Изменение метода вывода данных

Pf2eDataScrapper/Importers/AbstractImporter.cs

@@ -28,6 +28,7 @@ namespace Pf2eDataScrapper.Importers
         /// </summary>
         public string Dir => Path.Combine(EnvSettings.DataDirPath, Slug);
 
+        public abstract void Prepare();
         public abstract void Import();
 
     }

Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/Compendium/CompendiumFolder.cs

@@ -3,17 +3,57 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
+using Newtonsoft.Json.Linq;
 
 namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.Compendium
 {
-    internal class CompendiumFolder(string name, string path, CompendiumDocumentType type)
+    internal class CompendiumFolder
     {
-        public string Name = name;
+        public readonly string Id;
 
-        public string Path = path;
+        public readonly string Name;
 
-        public CompendiumDocumentType Type = type;
+        public readonly string? Path;
 
-        public DirectoryInfo Directory = new(path);
+        public readonly bool IsContained;
+
+        public CompendiumFolder? Parent;
+
+        public readonly CompendiumDocumentType Type;
+
+        public CompendiumFolder(JObject json)
+        {
+            if (json.ContainsKey("_id"))
+            {
+                // Вложенная папка через _folders.json
+                Id = json["_id"]!.ToString();
+                Name = json["name"]!.ToString();
+                Path = null;
+                IsContained = true;
+            }
+            else
+            {
+                // Корневой пак в system.json
+                Id = json["name"]!.ToString();
+                Name = json["label"]!.ToString();
+                Path = json["path"]!.ToString();
+                IsContained = false;
+            }
+
+            Type = (CompendiumDocumentType)Enum.Parse(typeof(CompendiumDocumentType), json["type"]!.ToString());
+        }
+
+        public CompendiumFolder(string id, string name)
+        {
+            Id = id;
+            Name = name;
+            Path = null;
+            IsContained = false;
+        }
+
+        public override string ToString()
+        {
+            return $"CompendiumFolder([Id={Id}][Type={Type}][Name={Name}][Path={Path}][Parent={Parent?.Name ?? "null"}])";
+        }
     }
 }

Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/Compendium/CompendiumFolderCollection.cs

@@ -0,0 +1,19 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.Compendium
+{
+    internal class CompendiumFolderCollection : List<CompendiumFolder>
+    {
+
+        public CompendiumFolder? GetById(string id)
+        {
+            return this.FirstOrDefault(x => x.Id == id);
+        }
+
+        
+    }
+}

Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/Compendium/CompendiumPack.cs

@@ -1,31 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Diagnostics;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-
-namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.Compendium
-{
-    /// <summary>
-    /// Implementation of `CompendiumPack` class from pf2e voundry module
-    /// </summary>
-    internal class CompendiumPack
-    {
-        public string PackId;
-        public string PackDir;
-        public CompendiumDocumentType DocumentType;
-        public string SystemId;
-        
-        public CompendiumPack(string PackDir, object[] parsedData, object[] parsedFolders)
-        {
-
-        }
-
-
-        //public static CompendiumPack LoadJson(string path)
-        //{
-        //    ret
-        //}
-    }
-}

Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/Compendium/CompendiumSchemaBuilder.cs

@@ -0,0 +1,153 @@
+using Konsole;
+using Newtonsoft.Json.Linq;
+using Pf2eDataScrapper.SchemaBuilder;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.Compendium
+{
+    internal class CompendiumSchemaBuilder(CompendiumLoader loader)
+    {
+        public CompendiumLoader Loader = loader;
+
+        public CompendiumFolderCollection Collection = loader.Collection;
+
+        public CompendiumSchemaTypeResolver Resolver = new();
+
+        public Dictionary<CompendiumFolder, SchemaNode> PackSchemas = [];
+
+        public Dictionary<string, SchemaNode> TypeSchemas = [];
+
+        
+        protected List<SchemaNode> GetItemSchemas(CompendiumFolder folder, string? itemType)
+        {
+            if (!PackSchemas.ContainsKey(folder))
+            {
+                PackSchemas[folder] = new SchemaNode()
+                {
+                    Resolver = Resolver,
+                    Path = folder.Name
+                };
+            }
+
+            var key = folder.Type.ToString() + (itemType is not null ? $".{itemType}" : "");
+
+            if (!TypeSchemas.ContainsKey(key))
+            {
+                TypeSchemas[key] = new SchemaNode()
+                {
+                    Resolver = Resolver,
+                    Path = key
+                };
+            }
+
+            return [PackSchemas[folder], TypeSchemas[key]];
+        }
+
+        public void BuildSchemas()
+        {
+            Console.WriteLine("Building schemas...");
+
+            var bar = new ProgressBar(Loader.ItemCount);
+            var i = 0;
+
+            foreach (var folder in Collection.FindAll(f => f.Path is not null))
+            {
+                var dirPath = Path.Combine(Loader.Importer.Dir, folder.Path!);
+                
+                foreach (var filePath in Directory.GetFiles(dirPath, "*.json", SearchOption.AllDirectories))
+                {
+                    bar.Refresh(i++, folder.Name);
+
+                    var file = new FileInfo(filePath);
+                
+                    if (file.Name == "_folders.json")
+                    {
+                        continue;
+                    }
+                
+                    using var reader = new StreamReader(file.FullName);
+                    var itemJson = JObject.Parse(reader.ReadToEnd());
+
+                    var itemFolder = folder;
+                    if (itemJson.TryGetValue("folder", out var itemJsonFolder))
+                    {
+                        itemFolder = Collection.GetById(itemJsonFolder.ToString());
+                    }
+
+                    var schemas = GetItemSchemas(itemFolder ?? folder, itemJson["type"]?.ToString());
+
+                    foreach (var schema in schemas)
+                    {
+                        schema.AddValue(itemJson);
+                    }
+                }
+            }
+
+            bar.Refresh(Loader.ItemCount, "Done!");
+        }
+
+        public void Save(string targetDir)
+        {
+            Console.WriteLine("Dumping schemas...");
+
+            var bar = new ProgressBar(PackSchemas.Count + TypeSchemas.Count);
+            var i = 0;
+
+            targetDir = Path.Combine(targetDir, "pf2e");
+
+            if (!Directory.Exists(targetDir))
+            {
+                Directory.CreateDirectory(targetDir);
+            }
+            Array.ForEach(Directory.GetFiles(@targetDir), File.Delete);
+
+            // Packs
+            var packsDir = Path.Combine(targetDir, "packs");
+            Directory.CreateDirectory(packsDir);
+
+            foreach (var pair in PackSchemas)
+            {
+                bar.Refresh(i++, pair.Key.Name);
+                SaveSchema(packsDir, pair.Key.Name, pair.Value);
+            }
+
+            // Types
+            var typesDir = Path.Combine(targetDir, "types");
+            Directory.CreateDirectory(typesDir);
+            foreach (var pair in TypeSchemas)
+            {
+                bar.Refresh(i++, pair.Key);
+                SaveSchema(typesDir, pair.Key, pair.Value);
+            }
+
+            Console.WriteLine("Done!");
+        }
+
+        protected void SaveSchema(string dir, string name, SchemaNode schema)
+        {
+            var invalids = System.IO.Path.GetInvalidFileNameChars();
+            var fileName = string.Join("_", schema.Path.Split(invalids, StringSplitOptions.RemoveEmptyEntries)).TrimEnd('.');
+
+            var path = Path.Combine(dir, fileName + ".txt");
+
+
+            var dirPath = Path.GetDirectoryName(path);
+
+            if (!Directory.Exists(dirPath))
+            {
+                Directory.CreateDirectory(dirPath);
+            }
+
+            using var writer = new StreamWriter(path);
+
+            writer.WriteLine(name);
+            writer.WriteLine($"Different values: {schema.Values}");
+            writer.WriteLine("=====================================");
+            writer.Write(schema.ToString());
+        }
+    }
+}

Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/Compendium/CompendiumSchemaTypeResolver.cs

@@ -22,9 +22,16 @@ namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.Compendium
             }
         }
 
-        List<string> dictionaryPaths = [
+        protected readonly List<string> DictionaryPaths = [
-            "abomination-vaults-bestiary.items.[].system.overlays",
+            "Actor.character.items.[].system.items",
-            "abomination-vaults-bestiary.items.[].system.damageRolls",
+            "Item.ancestry.system.boosts",
+            "Item.background.system.boosts",
+            "Item.background.system.items",
+            "Item.class.system.items",
+            "Item.feat.system.subfeatures.proficiencies",
+            "Item.kit.system.items",
+            "Item.spell.system.damage",
+            "Item.spell.system.heightening.damage"
             ];
 
         protected bool ShouldBeDictionary(SchemaNode parent, JToken? value)
@@ -36,8 +43,8 @@ namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.Compendium
 
             //var p = (JProperty)value.Parent;
 
-            //return dictionaryPaths.Any(path => path == $"{parent.Path}.{p.Name}");
+            //return DictionaryPaths.Any(path => path == $"{parent.Path}.{p.Name}");
-            return dictionaryPaths.Any(path => path == parent.Path);
+            return DictionaryPaths.Any(path => path == parent.Path);
         }
 
         public override AbstractSchemaType Resolve(SchemaNode parent, JToken? value)

Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/CompendiumLoader.cs

@@ -7,121 +7,107 @@ using Pf2eDataScrapper.SchemaBuilder;
 
 namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry
 {
-    internal partial class CompendiumLoader(Pf2eFoundryImporter importer)
+    internal class CompendiumLoader(Pf2eFoundryImporter importer)
     {
         public readonly Pf2eFoundryImporter Importer = importer;
 
-        public List<CompendiumFolder> Folders = [];
+        public CompendiumFolderCollection Collection = [];
 
-        private static string SlugifyName(string s)
+        private int _folderCount = 0;
+
+        public int ItemCount { get; protected set; } = 0;
+
+        protected void ProcessPackFolders(JArray folders, CompendiumFolder? parent)
         {
-            s = SkippableCharactersRegex().Replace(s, "");
+            foreach (var token in folders)
-            s = NonAlphanumericRegex().Replace(s.ToLower(), "-");
+            {
+                JObject obj = (JObject)token;
 
-            return s;
+                var id = $"folder-{_folderCount++}";
+                var name = obj["name"]!.ToString();
+
+                var folder = new CompendiumFolder(id, name)
+                {
+                    Parent = parent
+                };
+                Collection.Add(folder);
+
+                if (obj.TryGetValue("packs", out var packs))
+                {
+                    foreach (var jToken in packs!)
+                    {
+                        var packId = jToken.ToString();
+                        var child = Collection.GetById(packId);
+
+                        if (child is not null)
+                        {
+                            child.Parent = folder;
+                        }
+                    }
                 }
 
-        private static CompendiumDocumentType GetType(string s)
+                if (obj.TryGetValue("folders", out var innerFolders))
                 {
-            return (CompendiumDocumentType)Enum.Parse(typeof(CompendiumDocumentType), s);
+                    ProcessPackFolders((JArray)innerFolders!, folder);
+                }
+            }
         }
 
-        void ResolveFolders()
+        public void Prepare()
         {
             using var systemFile = new StreamReader(Path.Combine(Importer.Dir, "static", "system.json"));
             var systemData = JObject.Parse(systemFile.ReadToEnd());
 
-            foreach (var pack in systemData["packs"])
+            foreach (var pack in systemData["packs"]!)
             {
-                var packPath = Path.Combine(importer.Dir, pack["path"].ToString());
+                var folder = new CompendiumFolder((JObject)pack);
-                var packType = GetType(pack["type"].ToString());
-                var packName = pack["path"].ToString().Replace("packs/", "");
 
-                if (!Directory.Exists(packPath))
+                Collection.Add(folder);
-                {
-                    throw new Exception("Pack not found: " + packPath);
             }
 
-                var folder = new CompendiumFolder(packName, packPath, packType);
+            if (systemData.TryGetValue("packFolders", out var packFolders))
-                Folders.Add(folder);
+            {
-
+                ProcessPackFolders((JArray)packFolders, null);
-                //var innerFolderPath = Path.Combine(folder.Directory.FullName, "_folders.json");
-                //if (File.Exists(innerFolderPath))
-                //{
-                //    using var innerFolderFile = new StreamReader(innerFolderPath);
-                //    var innerFolderData = JArray.Parse(innerFolderFile.ReadToEnd());
-
-                //    foreach (var item in innerFolderData)
-                //    {
-                //        var innerPackPath = Path.Combine(packPath, SlugifyName(item["name"].ToString()));
-                //        var innerPackType = GetType(item["type"].ToString());
-                //        var innerPackName = packName + "/" + SlugifyName(item["name"].ToString());
-
-                //        if (!Directory.Exists(packPath))
-                //        {
-                //            throw new Exception("Pack not found: " + innerPackPath);
-                //        }
-
-                //        var innerFolder = new CompendiumFolder(innerPackName, innerPackPath, innerPackType);
-                //        Folders.Add(innerFolder);
-                //    }
-                //}
             }
 
-            foreach (var item in Folders)
+            // Поиск _folders.json
+            foreach (var parent in Collection.ToList())
             {
-                Console.WriteLine($"{item.Directory.FullName} => {item.Type}");
+                if (parent.Path is null)
-            }
-            
-        }
-
-        public Dictionary<string, SchemaNode> BuildSchemas()
-        {
-            Dictionary<string, SchemaNode> Result = [];
-
-            var bar = new ProgressBar(Folders.Count);
-            var i = 0;
-            var resolver = new CompendiumSchemaTypeResolver();
-
-            foreach (var folder in Folders)
-            {
-                bar.Refresh(i++, folder.Name);
-
-                var schema = new SchemaNode() { 
-                    Resolver = resolver,
-                    Path = folder.Name
-                };
-                
-                foreach (var filePath in Directory.GetFiles(folder.Directory.FullName, "*.json", SearchOption.AllDirectories))
-                {
-                    var file = new FileInfo(filePath);
-
-                    if (file.Name == "_folders.json")
                 {
                     continue;
                 }
 
-                    using var reader = new StreamReader(file.FullName); // Открытие файла
+                var dirPath = Path.Combine(Importer.Dir, parent.Path);
-                    var data = JObject.Parse(reader.ReadToEnd()); // Загрузка данных
+
-                    schema.AddValue(data); // Добавление данных
+                ItemCount += Directory.GetFiles(dirPath, "*.json", SearchOption.AllDirectories)
+                    .Length;
+
+                foreach (var foldersFilePath in Directory.GetFiles(dirPath, "_folders.json", SearchOption.AllDirectories))
+                {
+                    ItemCount--;
+                    using var file = new StreamReader(foldersFilePath);
+                    var data = JArray.Parse(file.ReadToEnd());
+
+                    foreach (var pack in data)
+                    {
+                        var folder = new CompendiumFolder((JObject)pack) { Parent = parent };
+                        Collection.Add(folder);
+                    }
+                }
             }
 
-                Result.Add(folder.Name, schema);
+            foreach (var folder in Collection)
+            {
+                Console.WriteLine(folder);
             }
 
-            bar.Refresh(Folders.Count, "Done!");
+            Console.WriteLine($"Total items: {ItemCount}");
-            return Result;
         }
 
         public void Run()
         {
-            this.ResolveFolders();
+            this.Prepare();
-        }
+        }
-
-        [GeneratedRegex("[^a-zA-Z0-9]+")]
-        private static partial Regex NonAlphanumericRegex();
-
-        [GeneratedRegex("[\'?!]")]
-        private static partial Regex SkippableCharactersRegex();
     }
 }

Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/Pf2eFoundryImporter.cs

@@ -30,6 +30,11 @@ namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry
             Loader = new CompendiumLoader(this);
         }
 
+        public override void Prepare()
+        {
+            Loader.Prepare();
+        }
+
         public override void Import()
         {
             Loader.Run();

Pf2eDataScrapper/Program.cs

@@ -2,6 +2,7 @@
 using Pf2eDataScrapper;
 using Pf2eDataScrapper.Importers;
 using Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry;
+using Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.Compendium;
 using Pf2eDataScrapper.SchemaBuilder;
 
 EnvSettings.Load();
@@ -15,10 +16,15 @@ var importer = new Pf2eFoundryImporter();
 
 //loader.UpdateImporter(importer);
 
-importer.Import();
+importer.Prepare();
 
-var q = importer.Loader.BuildSchemas();
+var builder = new CompendiumSchemaBuilder(importer.Loader);
-SchemaDumper.Dump(q);
+
+builder.BuildSchemas();
+builder.Save(EnvSettings.OutputDir);
+
+// var q = importer.Loader.BuildSchemas();
+// SchemaDumper.Dump(q);
 
 
 

Pf2eDataScrapper/SchemaBuilder/ArraySchemaType.cs

@@ -53,10 +53,9 @@ namespace Pf2eDataScrapper.SchemaBuilder
         public override string ToString()
         {
             return string.Join("\n" + Tab, [
-                GetTypeName(),
+                GetTypeName() + $" {{{Parent?.Path ?? ""}}}",
                 $"Length: min={MinCount} max={MaxCount}",
-                "Value type:",
+                "Value type: " + Indent(ElementSchema.ToString())
-                Tab + Indent(ElementSchema.ToString())
             ]);
         }
     }

Pf2eDataScrapper/SchemaBuilder/DictionarySchemaType.cs

@@ -42,7 +42,7 @@ namespace Pf2eDataScrapper.SchemaBuilder
         public override string ToString()
         {
             return string.Join("\n" + Tab, [
-                GetTypeName(),
+                GetTypeName() + $" {{{Parent?.Path ?? ""}}}",
                 "Key:" + Indent(KeyNode.ToString()),
                 "Value:" + Indent(ItemNode.ToString())
             ]);

Pf2eDataScrapper/SchemaBuilder/ObjectSchemaType.cs

@@ -30,7 +30,7 @@ namespace Pf2eDataScrapper.SchemaBuilder
 
             foreach (var pair in obj)
             {
-                var field = Fields.Where(f => f.Name == pair.Key).FirstOrDefault();
+                var field = Fields.FirstOrDefault(f => f.Name == pair.Key);
 
                 if (field is null)
                 {
@@ -67,7 +67,7 @@ namespace Pf2eDataScrapper.SchemaBuilder
 
         public override string ToString()
         {
-            List<string> rows = ["object"];
+            List<string> rows = [$"object {{{Parent?.Path ?? ""}}}"];
 
             foreach (var field in Fields)
             {

Pf2eDataScrapper/SchemaBuilder/StaticSchemaType.cs

@@ -22,17 +22,31 @@ namespace Pf2eDataScrapper.SchemaBuilder
 
         public override void Beautify()
         {
-            // Do nothing
+            if (Type == JTokenType.String)
+            {
+                var newValues = new Dictionary<JToken, int>();
+
+                foreach (var pair in Values)
+                {
+                    var newValue = "'" + pair.Key.ToString().Trim().Replace("'", "\\'") + "'";
+                    newValues.TryAdd(newValue, 0);
+                    newValues[newValue] += pair.Value;
+                }
+
+                Values = newValues;
+            }
+
+            Values = Values.OrderByDescending(pair => pair.Value).ToDictionary(pair => pair.Key, pair => pair.Value);
         }
 
         public override string GetTypeName()
         {
-            return Type.ToString();
+            return Type.ToString().ToLower();
         }
 
         public override string ToString()
         {
-            var result = Type.ToString();
+            var result = GetTypeName() + $" {{{Parent?.Path ?? ""}}}";
             var count = Values.Count;
 
             if (count == Root.Values)
@@ -41,7 +55,13 @@ namespace Pf2eDataScrapper.SchemaBuilder
                 return result;
             }
 
-            result += $" (different values: {count})";
+            if (count == 1)
+            {
+                result += $" (same value: {Values.Keys.ToList()[0]})";
+                return result;
+            }
+
+            result += $" ({count} different values)";
 
             if (count is > ElementLimit or <= 0)
             {
@@ -50,7 +70,6 @@ namespace Pf2eDataScrapper.SchemaBuilder
 
 
             result += "\n";
-
             result = Values.Aggregate(result, (current, value) => current + $"{Tab}{value.Key} (x{value.Value})\n");
 
             return result.Trim();

Pf2eDataScrapper/SchemaDumper.cs

@@ -1,49 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-using Pf2eDataScrapper.SchemaBuilder;
-
-namespace Pf2eDataScrapper
-{
-    internal class SchemaDumper(Dictionary<string, SchemaNode> data)
-    {
-        public readonly Dictionary<string, SchemaNode> Data = data;
-
-        public readonly string OutputDir = EnvSettings.OutputDir;
-
-
-        public void Run()
-        {
-            Console.WriteLine("Dumping schema...");
-
-            foreach (var pair in Data)
-            {
-                var path = Path.Combine(OutputDir, pair.Key + ".txt");
-                var dir = Path.GetDirectoryName(path);
-
-                if (!Directory.Exists(dir))
-                {
-                    Directory.CreateDirectory(dir);
-                }
-
-
-                var schema = pair.Value;
-
-                using var writer = new StreamWriter(path);
-
-                writer.WriteLine(pair.Key);
-                writer.Write(schema.ToString());
-            }
-
-            Console.WriteLine("Done!");
-        }
-
-
-        public static void Dump(Dictionary<string, SchemaNode> data)
-        {
-            (new SchemaDumper(data)).Run();
-        }
-    }
-}