diff --git a/Pf2eDataScrapper/Importers/AbstractImporter.cs b/Pf2eDataScrapper/Importers/AbstractImporter.cs index 785cb51..7eaac9b 100644 --- a/Pf2eDataScrapper/Importers/AbstractImporter.cs +++ b/Pf2eDataScrapper/Importers/AbstractImporter.cs @@ -35,5 +35,9 @@ /// public abstract void Import(); + /// + /// Построение схем данных + /// + public abstract void BuildSchemas(); } } diff --git a/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/Compendium/CompendiumFolderCollection.cs b/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/Compendium/CompendiumFolderCollection.cs index 8e3a538..4067ca1 100644 --- a/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/Compendium/CompendiumFolderCollection.cs +++ b/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/Compendium/CompendiumFolderCollection.cs @@ -21,7 +21,8 @@ /// public CompendiumFolderCollection Filter() { - return [.. this.FindAll(x => x.Type != CompendiumDocumentType.Actor)]; + // return [.. this.FindAll(x => x.Type != CompendiumDocumentType.Actor)]; + return [.. this.FindAll(x => x.Type != CompendiumDocumentType.Actor && x.Name == "Conditions")]; } } } diff --git a/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/CompendiumLoader.cs b/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/CompendiumLoader.cs index bb8468a..b95aa21 100644 --- a/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/CompendiumLoader.cs +++ b/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/CompendiumLoader.cs @@ -1,4 +1,5 @@ -using Newtonsoft.Json.Linq; +using Microsoft.Extensions.Logging; +using Newtonsoft.Json.Linq; using Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.Compendium; namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry @@ -119,12 +120,52 @@ namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry } } - foreach (var folder in Collection) - { - Console.WriteLine(folder); - } + // foreach (var folder in Collection) + // { + // Console.WriteLine(folder); + // } + + Collection = Collection.Filter(); Console.WriteLine($"Total items: {ItemCount}"); } + + public IEnumerable Enrties() + { + foreach (var folder in Collection.FindAll(f => f.Path is not null)) + { + var dirPath = Path.Combine(Importer.Dir, folder.Path!); + + foreach (var filePath in Directory.GetFiles(dirPath, "*.json", SearchOption.AllDirectories)) + { + var file = new FileInfo(filePath); + + if (file.Name == "_folders.json") + { + continue; + } + + using var reader = new StreamReader(file.FullName); + var itemJson = JObject.Parse(reader.ReadToEnd()); + + var itemFolder = folder; + if (itemJson.TryGetValue("folder", out var itemJsonFolder)) + { + itemFolder = Collection.GetById(itemJsonFolder.ToString()); + } + + yield return new LoaderEnrty(file, itemJson, itemFolder ?? folder); + } + } + } + + public class LoaderEnrty(FileInfo file, JObject itemJson, CompendiumFolder folder) + { + public FileInfo File { get; } = file; + public JObject Json { get; } = itemJson; + public CompendiumFolder Folder { get; } = folder; + public string Type { get; } = itemJson["type"]?.ToString() + ?? throw new Exception($"Expected to have `type` field in file {file.FullName}"); + } } } diff --git a/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/Pf2eFoundryImporter.cs b/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/Pf2eFoundryImporter.cs index 04b68b6..482563e 100644 --- a/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/Pf2eFoundryImporter.cs +++ b/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/Pf2eFoundryImporter.cs @@ -1,4 +1,7 @@ -namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry +using Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.DataBuilder; +using Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.SchemaBuilder; + +namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry { /// /// Импортёр для модуля Pf2e FoundryVTT. @@ -46,7 +49,21 @@ /// public override void Import() { - Loader.Prepare(); + var builder = new PF2eFoundryDataBuilder(Loader); + + builder.TruncateTables(); + builder.Build(); + } + + /// + /// Построение схем данных + /// + public override void BuildSchemas() + { + var builder = new CompendiumSchemaBuilder(Loader); + + builder.BuildSchemas(); + builder.Save(EnvSettings.OutputDir); } } } diff --git a/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/Compendium/CompendiumSchemaBuilder.cs b/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/SchemaBuilder/CompendiumSchemaBuilder.cs similarity index 84% rename from Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/Compendium/CompendiumSchemaBuilder.cs rename to Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/SchemaBuilder/CompendiumSchemaBuilder.cs index 4aa2d59..eb0651d 100644 --- a/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/Compendium/CompendiumSchemaBuilder.cs +++ b/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/SchemaBuilder/CompendiumSchemaBuilder.cs @@ -1,8 +1,9 @@ using Konsole; using Newtonsoft.Json.Linq; +using Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.Compendium; using Pf2eDataScrapper.SchemaBuilder; -namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.Compendium +namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.SchemaBuilder { /// /// Класс построения схемы компендиумов. @@ -97,36 +98,15 @@ namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.Compendium var bar = new ProgressBar(Loader.ItemCount); var i = 0; - foreach (var folder in Collection.FindAll(f => f.Path is not null)) + foreach (var entry in Loader.Enrties()) { - var dirPath = Path.Combine(Loader.Importer.Dir, folder.Path!); - - foreach (var filePath in Directory.GetFiles(dirPath, "*.json", SearchOption.AllDirectories)) + bar.Refresh(i++, entry.Folder.Name); + + var schemas = GetItemSchemas(entry.Folder, entry.Type); + + foreach (var schema in schemas) { - bar.Refresh(i++, folder.Name); - - var file = new FileInfo(filePath); - - if (file.Name == "_folders.json") - { - continue; - } - - using var reader = new StreamReader(file.FullName); - var itemJson = JObject.Parse(reader.ReadToEnd()); - - var itemFolder = folder; - if (itemJson.TryGetValue("folder", out var itemJsonFolder)) - { - itemFolder = Collection.GetById(itemJsonFolder.ToString()); - } - - var schemas = GetItemSchemas(itemFolder ?? folder, itemJson["type"]?.ToString()); - - foreach (var schema in schemas) - { - schema.AddValue(itemJson); - } + schema.AddValue(entry.Json); } } diff --git a/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/Compendium/CompendiumSchemaTypeResolver.cs b/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/SchemaBuilder/CompendiumSchemaTypeResolver.cs similarity index 99% rename from Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/Compendium/CompendiumSchemaTypeResolver.cs rename to Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/SchemaBuilder/CompendiumSchemaTypeResolver.cs index 14d08d3..d5f5f5c 100644 --- a/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/Compendium/CompendiumSchemaTypeResolver.cs +++ b/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/SchemaBuilder/CompendiumSchemaTypeResolver.cs @@ -7,7 +7,7 @@ using Newtonsoft.Json.Linq; using Pf2eDataScrapper.SchemaBuilder; using Pf2eDataScrapper.SchemaBuilder.Types; -namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.Compendium +namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.SchemaBuilder { internal class CompendiumSchemaTypeResolver : TypeResolver { diff --git a/Pf2eDataScrapper/Program.cs b/Pf2eDataScrapper/Program.cs index a7b09d2..3fa9c8a 100644 --- a/Pf2eDataScrapper/Program.cs +++ b/Pf2eDataScrapper/Program.cs @@ -1,46 +1,24 @@ -using Newtonsoft.Json.Linq; -using Pf2eDataScrapper; -using Pf2eDataScrapper.Importers; +using Pf2eDataScrapper; using Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry; -using Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.Compendium; -using Pf2eDataScrapper.SchemaBuilder; EnvSettings.Load(); -Console.WriteLine("Hello, World!"); - -//var loader = new GitLoader(EnvSettings.RepositoryDirPath); - var importer = new Pf2EFoundryImporter(); -//loader.UpdateImporter(importer); - importer.Prepare(); -var builder = new CompendiumSchemaBuilder(importer.Loader); - -builder.BuildSchemas(); -builder.Save(EnvSettings.OutputDir); - -// var q = importer.Loader.BuildSchemas(); -// SchemaDumper.Dump(q); - - - - - - -return; - -JObject data; - -using (var reader = new StreamReader("test.json")) +try { - data = JObject.Parse(reader.ReadToEnd()); + importer.Import(); } +catch (Exception ex) +{ + Console.WriteLine(ex); + //Console.WriteLine(ex.Message); + //Console.WriteLine(ex.InnerException); + //Console.WriteLine(ex.StackTrace); +} +// importer.BuildSchemas(); -SchemaNode schema = new(); -schema.AddValue(data); -Console.WriteLine(schema);