diff --git a/.gitignore b/.gitignore index 9491a2f..0ce0670 100644 --- a/.gitignore +++ b/.gitignore @@ -360,4 +360,6 @@ MigrationBackup/ .ionide/ # Fody - auto-generated XML schema -FodyWeavers.xsd \ No newline at end of file +FodyWeavers.xsd +/Pf2eDataScrapper/.env +/output diff --git a/Pf2eDataScrapper.sln b/Pf2eDataScrapper.sln index 43f91bc..13b25e5 100644 --- a/Pf2eDataScrapper.sln +++ b/Pf2eDataScrapper.sln @@ -5,6 +5,13 @@ VisualStudioVersion = 17.13.35919.96 d17.13 MinimumVisualStudioVersion = 10.0.40219.1 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Pf2eDataScrapper", "Pf2eDataScrapper\Pf2eDataScrapper.csproj", "{FF82896A-92CA-4C63-8FDC-959ECC513773}" EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Элементы решения", "Элементы решения", "{754FC069-D67B-A9D7-50A1-8D1CA196D8F1}" + ProjectSection(SolutionItems) = preProject + README.md = README.md + EndProjectSection +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "output", "output", "{36D591C7-65C7-A0D1-1CBC-10CDE441BDC8}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -19,6 +26,9 @@ Global GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE EndGlobalSection + GlobalSection(NestedProjects) = preSolution + {36D591C7-65C7-A0D1-1CBC-10CDE441BDC8} = {754FC069-D67B-A9D7-50A1-8D1CA196D8F1} + EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {CB89E1EC-DFB8-435C-9944-F4EACCF375DC} EndGlobalSection diff --git a/Pf2eDataScrapper/.env.example b/Pf2eDataScrapper/.env.example new file mode 100644 index 0000000..e4a1ea9 --- /dev/null +++ b/Pf2eDataScrapper/.env.example @@ -0,0 +1 @@ +DATA_DIR=D:\path\to\directory\with\repositories \ No newline at end of file diff --git a/Pf2eDataScrapper/EnvSettings.cs b/Pf2eDataScrapper/EnvSettings.cs new file mode 100644 index 0000000..39b438c --- /dev/null +++ b/Pf2eDataScrapper/EnvSettings.cs @@ -0,0 +1,27 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using dotenv.net; +using dotenv.net.Utilities; + +namespace Pf2eDataScrapper +{ + internal class EnvSettings + { + public static void Load() + { + DotEnv.Load(); + } + + public static string DataDirPath => EnvReader.GetStringValue("DATA_DIR"); + + public static DirectoryInfo DataDir => new(DataDirPath); + + public static string OutputDirPath => EnvReader.GetStringValue("OUTPUT_DIR"); + + public static string OutputDir => new(OutputDirPath); + + } +} diff --git a/Pf2eDataScrapper/Importers/AbstractImporter.cs b/Pf2eDataScrapper/Importers/AbstractImporter.cs new file mode 100644 index 0000000..60a5402 --- /dev/null +++ b/Pf2eDataScrapper/Importers/AbstractImporter.cs @@ -0,0 +1,34 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Pf2eDataScrapper.Importers +{ + internal abstract class AbstractImporter + { + /// + /// Отображаемое имя импортёра + /// + public abstract string Name { get; } + + /// + /// Короткое имя импортёра + /// + public abstract string Slug { get; } + + /// + /// Ссылка на git-репозиторий для клонирования + /// + public abstract string GitUrl { get; } + + /// + /// Путь к директории с данными для импортёра + /// + public string Dir => Path.Combine(EnvSettings.DataDirPath, Slug); + + public abstract void Import(); + + } +} diff --git a/Pf2eDataScrapper/Importers/GitLoader.cs b/Pf2eDataScrapper/Importers/GitLoader.cs new file mode 100644 index 0000000..2643721 --- /dev/null +++ b/Pf2eDataScrapper/Importers/GitLoader.cs @@ -0,0 +1,50 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Pf2eDataScrapper.Importers +{ + internal class GitLoader(string DataDirPath) + { + public readonly string DataDirPath = DataDirPath; + public DirectoryInfo DataDir = new(DataDirPath); + + public void UpdateImporter(AbstractImporter importer) + { + string gitPath = Path.Combine(DataDirPath, importer.Slug); + + if (Directory.Exists(gitPath)) + { + // git pull + var process = new Process + { + StartInfo = new ProcessStartInfo() + { + FileName = "git", + WorkingDirectory = gitPath, + Arguments = "pull", + } + }; + process.Start(); + } + else + { + // git clone + var process = new Process + { + StartInfo = new ProcessStartInfo() + { + FileName = "git", + WorkingDirectory = DataDirPath, + Arguments = $"clone {importer.GitUrl} {importer.Slug}", + } + }; + process.Start(); + + } + } + } +} diff --git a/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/Compendium/CompendiumDocumentType.cs b/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/Compendium/CompendiumDocumentType.cs new file mode 100644 index 0000000..b7f9b9b --- /dev/null +++ b/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/Compendium/CompendiumDocumentType.cs @@ -0,0 +1,21 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.Compendium +{ + internal enum CompendiumDocumentType + { + Actor, + Adventure, + Cards, + Item, + JournalEntry, + Macro, + Playlist, + RollTable, + Scene + } +} diff --git a/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/Compendium/CompendiumFolder.cs b/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/Compendium/CompendiumFolder.cs new file mode 100644 index 0000000..c4594b1 --- /dev/null +++ b/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/Compendium/CompendiumFolder.cs @@ -0,0 +1,19 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.Compendium +{ + internal class CompendiumFolder(string name, string path, CompendiumDocumentType type) + { + public string Name = name; + + public string Path = path; + + public CompendiumDocumentType Type = type; + + public DirectoryInfo Directory = new(path); + } +} diff --git a/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/Compendium/CompendiumPack.cs b/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/Compendium/CompendiumPack.cs new file mode 100644 index 0000000..734bdf5 --- /dev/null +++ b/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/Compendium/CompendiumPack.cs @@ -0,0 +1,31 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.Compendium +{ + /// + /// Implementation of `CompendiumPack` class from pf2e voundry module + /// + internal class CompendiumPack + { + public string PackId; + public string PackDir; + public CompendiumDocumentType DocumentType; + public string SystemId; + + public CompendiumPack(string PackDir, object[] parsedData, object[] parsedFolders) + { + + } + + + //public static CompendiumPack LoadJson(string path) + //{ + // ret + //} + } +} diff --git a/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/Compendium/CompendiumSchemaTypeResolver.cs b/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/Compendium/CompendiumSchemaTypeResolver.cs new file mode 100644 index 0000000..ec34727 --- /dev/null +++ b/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/Compendium/CompendiumSchemaTypeResolver.cs @@ -0,0 +1,58 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Newtonsoft.Json.Linq; +using Pf2eDataScrapper.SchemaBuilder; + +namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.Compendium +{ + internal class CompendiumSchemaTypeResolver : TypeResolver + { + public class FoundryRulesSchemaType(SchemaNode parent) : ArraySchemaType(parent) + { + public override string GetTypeName() + { + return "###FoundryRulesObject###"; + } + public override string ToString() + { + return GetTypeName(); + } + } + + List dictionaryPaths = [ + "abomination-vaults-bestiary.items.[].system.overlays", + "abomination-vaults-bestiary.items.[].system.damageRolls", + ]; + + protected bool ShouldBeDictionary(SchemaNode parent, JToken? value) + { + //if (value is not JValue || value.Parent is not JProperty) + //{ + // return false; + //} + + //var p = (JProperty)value.Parent; + + //return dictionaryPaths.Any(path => path == $"{parent.Path}.{p.Name}"); + return dictionaryPaths.Any(path => path == parent.Path); + } + + public override AbstractSchemaType Resolve(SchemaNode parent, JToken? value) + { + if (ShouldBeDictionary(parent, value)) + { + return new DictionarySchemaType(parent); + } + + if (parent.Path.EndsWith(".rules")) + { + return new FoundryRulesSchemaType(parent); + } + + return base.Resolve(parent, value); + } + } +} diff --git a/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/CompendiumLoader.cs b/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/CompendiumLoader.cs new file mode 100644 index 0000000..f2ba5d2 --- /dev/null +++ b/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/CompendiumLoader.cs @@ -0,0 +1,127 @@ +using System.Reflection.PortableExecutable; +using System.Text.RegularExpressions; +using Konsole; +using Newtonsoft.Json.Linq; +using Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.Compendium; +using Pf2eDataScrapper.SchemaBuilder; + +namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry +{ + internal partial class CompendiumLoader(Pf2eFoundryImporter importer) + { + public readonly Pf2eFoundryImporter Importer = importer; + + public List Folders = []; + + private static string SlugifyName(string s) + { + s = SkippableCharactersRegex().Replace(s, ""); + s = NonAlphanumericRegex().Replace(s.ToLower(), "-"); + + return s; + } + + private static CompendiumDocumentType GetType(string s) + { + return (CompendiumDocumentType)Enum.Parse(typeof(CompendiumDocumentType), s); + } + + void ResolveFolders() + { + using var systemFile = new StreamReader(Path.Combine(Importer.Dir, "static", "system.json")); + var systemData = JObject.Parse(systemFile.ReadToEnd()); + + foreach (var pack in systemData["packs"]) + { + var packPath = Path.Combine(importer.Dir, pack["path"].ToString()); + var packType = GetType(pack["type"].ToString()); + var packName = pack["path"].ToString().Replace("packs/", ""); + + if (!Directory.Exists(packPath)) + { + throw new Exception("Pack not found: " + packPath); + } + + var folder = new CompendiumFolder(packName, packPath, packType); + Folders.Add(folder); + + //var innerFolderPath = Path.Combine(folder.Directory.FullName, "_folders.json"); + //if (File.Exists(innerFolderPath)) + //{ + // using var innerFolderFile = new StreamReader(innerFolderPath); + // var innerFolderData = JArray.Parse(innerFolderFile.ReadToEnd()); + + // foreach (var item in innerFolderData) + // { + // var innerPackPath = Path.Combine(packPath, SlugifyName(item["name"].ToString())); + // var innerPackType = GetType(item["type"].ToString()); + // var innerPackName = packName + "/" + SlugifyName(item["name"].ToString()); + + // if (!Directory.Exists(packPath)) + // { + // throw new Exception("Pack not found: " + innerPackPath); + // } + + // var innerFolder = new CompendiumFolder(innerPackName, innerPackPath, innerPackType); + // Folders.Add(innerFolder); + // } + //} + } + + foreach (var item in Folders) + { + Console.WriteLine($"{item.Directory.FullName} => {item.Type}"); + } + + } + + public Dictionary BuildSchemas() + { + Dictionary Result = []; + + var bar = new ProgressBar(Folders.Count); + var i = 0; + var resolver = new CompendiumSchemaTypeResolver(); + + foreach (var folder in Folders) + { + bar.Refresh(i++, folder.Name); + + var schema = new SchemaNode() { + Resolver = resolver, + Path = folder.Name + }; + + foreach (var filePath in Directory.GetFiles(folder.Directory.FullName, "*.json", SearchOption.AllDirectories)) + { + var file = new FileInfo(filePath); + + if (file.Name == "_folders.json") + { + continue; + } + + using var reader = new StreamReader(file.FullName); // Открытие файла + var data = JObject.Parse(reader.ReadToEnd()); // Загрузка данных + schema.AddValue(data); // Добавление данных + } + + Result.Add(folder.Name, schema); + } + + bar.Refresh(Folders.Count, "Done!"); + return Result; + } + + public void Run() + { + this.ResolveFolders(); + } + + [GeneratedRegex("[^a-zA-Z0-9]+")] + private static partial Regex NonAlphanumericRegex(); + + [GeneratedRegex("[\'?!]")] + private static partial Regex SkippableCharactersRegex(); + } +} diff --git a/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/Pf2eFoundryImporter.cs b/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/Pf2eFoundryImporter.cs new file mode 100644 index 0000000..fe94e7b --- /dev/null +++ b/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/Pf2eFoundryImporter.cs @@ -0,0 +1,38 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry +{ + internal class Pf2eFoundryImporter : AbstractImporter + { + /// + /// Отображаемое имя импортёра + /// + public override string Name => "Pf2e FoundryVTT module"; + + /// + /// Короткое имя импортёра + /// + public override string Slug => "pf2e"; + + /// + /// Ссылка на git-репозиторий для клонирования + /// + public override string GitUrl => "https://github.com/foundryvtt/pf2e.git"; + + public CompendiumLoader Loader { get; } + + public Pf2eFoundryImporter() + { + Loader = new CompendiumLoader(this); + } + + public override void Import() + { + Loader.Run(); + } + } +} diff --git a/Pf2eDataScrapper/Pf2eDataScrapper.csproj b/Pf2eDataScrapper/Pf2eDataScrapper.csproj index 0f35fa8..72e34b2 100644 --- a/Pf2eDataScrapper/Pf2eDataScrapper.csproj +++ b/Pf2eDataScrapper/Pf2eDataScrapper.csproj @@ -8,12 +8,17 @@ + + + + Always + - PreserveNewest + Always diff --git a/Pf2eDataScrapper/Program.cs b/Pf2eDataScrapper/Program.cs index b8f448e..cb30ae9 100644 --- a/Pf2eDataScrapper/Program.cs +++ b/Pf2eDataScrapper/Program.cs @@ -1,8 +1,32 @@ using Newtonsoft.Json.Linq; +using Pf2eDataScrapper; +using Pf2eDataScrapper.Importers; +using Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry; using Pf2eDataScrapper.SchemaBuilder; +EnvSettings.Load(); + Console.WriteLine("Hello, World!"); +//var loader = new GitLoader(EnvSettings.DataDirPath); + + +var importer = new Pf2eFoundryImporter(); + +//loader.UpdateImporter(importer); + +importer.Import(); + +var q = importer.Loader.BuildSchemas(); +SchemaDumper.Dump(q); + + + + + + +return; + JObject data; using (var reader = new StreamReader("test.json")) @@ -10,8 +34,7 @@ using (var reader = new StreamReader("test.json")) data = JObject.Parse(reader.ReadToEnd()); } -AbstractSchemaType schema = new ObjectSchemaType(); -schema = schema.Adapt(data); +SchemaNode schema = new(); schema.AddValue(data); Console.WriteLine(schema); diff --git a/Pf2eDataScrapper/SchemaBuilder/AbstractSchemaType.cs b/Pf2eDataScrapper/SchemaBuilder/AbstractSchemaType.cs index f2ecf3f..5b1457b 100644 --- a/Pf2eDataScrapper/SchemaBuilder/AbstractSchemaType.cs +++ b/Pf2eDataScrapper/SchemaBuilder/AbstractSchemaType.cs @@ -1,48 +1,30 @@ -using Newtonsoft.Json.Linq; +using System.ComponentModel.Design.Serialization; +using Newtonsoft.Json.Linq; namespace Pf2eDataScrapper.SchemaBuilder { - internal abstract class AbstractSchemaType + internal abstract class AbstractSchemaType(SchemaNode parent) { - /// - /// If the value is nullable. - /// - public bool IsNullable = false; + public SchemaNode Parent = parent; - /// - /// Mark the value as nullable. - /// - /// This object - public AbstractSchemaType MarkNullable() - { - IsNullable = true; - return this; - } + public SchemaNode Root => Parent.RootNode; + + public TypeResolver Resolver => Root.Resolver; /// /// Check if type is compatible with current type. /// /// Value to be checked /// True if value is compatible with current type - public abstract bool IsCompatibleValue(JToken? value); - - /// - /// Adapt to new value. - /// If value is compatible with current type, `this` will be returned. - /// If value is not compatible, new type will be created. - /// - /// Value to be checked - /// Compatible type - public virtual AbstractSchemaType Adapt(JToken? value) - { - return IsCompatibleValue(value) ? this : new MixedSchemaType(this); - } + public abstract bool IsCompatibleValue(JToken value); /// /// Add new possible value. /// /// New value - public abstract void AddValue(JToken? value); + public abstract void AddValue(JToken value); + + public abstract void Beautify(); /// /// String representation of type. @@ -72,38 +54,5 @@ namespace Pf2eDataScrapper.SchemaBuilder { return text.Replace("\n", "\n" + Tab); } - - /// - /// Get string representation of value type. - /// - /// Value to be checked - /// String representation of value type - public static string ResolveType(JToken? value) - { - if (value is null || value.Type is JTokenType.Null or JTokenType.Undefined or JTokenType.None) - { - return "null"; - } - - return value.Type.ToString().ToLower(); - } - - /// - /// Create new type from value. - /// - /// Value to be checked - /// - public static AbstractSchemaType CreateFromValue(JToken? value) - { - var type = ResolveType(value); - - return type switch - { - "null" => new EmptySchemaType(), - "array" => new ArraySchemaType(), - "object" => new ObjectSchemaType(), - _ => new StaticSchemaType(type), - }; - } } } \ No newline at end of file diff --git a/Pf2eDataScrapper/SchemaBuilder/ArraySchemaType.cs b/Pf2eDataScrapper/SchemaBuilder/ArraySchemaType.cs index 0b3254c..70a9fac 100644 --- a/Pf2eDataScrapper/SchemaBuilder/ArraySchemaType.cs +++ b/Pf2eDataScrapper/SchemaBuilder/ArraySchemaType.cs @@ -7,93 +7,57 @@ using Newtonsoft.Json.Linq; namespace Pf2eDataScrapper.SchemaBuilder { - internal class ArraySchemaType : AbstractSchemaType + internal class ArraySchemaType(SchemaNode parent) : AbstractSchemaType(parent) { public int MinCount = int.MaxValue; public int MaxCount = int.MinValue; - protected AbstractSchemaType ElementType = new EmptySchemaType(); + protected SchemaNode ElementSchema = new(parent, "[]"); protected List Values = []; - - - public override void AddValue(JToken? value) + + public override bool IsCompatibleValue(JToken value) { - MinCount = Math.Min(MinCount, Values.Count); - MaxCount = Math.Max(MaxCount, Values.Count); - - foreach (var item in Values) - { - ElementType.AddValue(item); - } + return Resolver.TokenType(value) == JTokenType.Array; } - public override string GetTypeName() + public override void AddValue(JToken value) { - return $"array<{ElementType.GetTypeName()}>"; - } - - public override bool IsCompatibleValue(JToken? value) - { - var type = ResolveType(value); - - if (type == "null") + if (value.Type != JTokenType.Array) { - return true; - } - - if (type == "array") - { - return false; - } - - foreach (var item in Values) - { - if (!ElementType.IsCompatibleValue(item)) - { - return false; - } - } - - return true; - } - - public override AbstractSchemaType Adapt(JToken? value) - { - if (value is null || value.Type != JTokenType.Array) - { - throw new Exception("Value is not an array"); + throw new Exception("Value is not an array, did you forget to call IsCompatibleValue?"); } var array = (JArray)value; - if ((ResolveType(value) == "none") && (ElementType is EmptySchemaType)) + MinCount = Math.Min(MinCount, array.Count); + MaxCount = Math.Max(MaxCount, array.Count); + + foreach (var item in array) { - ElementType = CreateFromValue(array[0]); + ElementSchema.AddValue(item); } + } - if (IsCompatibleValue(value)) - { - foreach (var item in array) - { - ElementType.Adapt(item); - } + public override void Beautify() + { - return this; - } + } - return new MixedSchemaType(this); + public override string GetTypeName() + { + return $"array<{ElementSchema.GetTypeName()}>"; } public override string ToString() { - List rows = [ GetTypeName() ]; - - rows.Add($"Length: min={MinCount} max={MaxCount}"); - rows.Add("Value type:" + Indent(ElementType.GetTypeName())); - - return string.Join("\n" + Tab, rows); + return string.Join("\n" + Tab, [ + GetTypeName(), + $"Length: min={MinCount} max={MaxCount}", + "Value type:", + Tab + Indent(ElementSchema.ToString()) + ]); } } } diff --git a/Pf2eDataScrapper/SchemaBuilder/DictionarySchemaType.cs b/Pf2eDataScrapper/SchemaBuilder/DictionarySchemaType.cs new file mode 100644 index 0000000..183151d --- /dev/null +++ b/Pf2eDataScrapper/SchemaBuilder/DictionarySchemaType.cs @@ -0,0 +1,51 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Newtonsoft.Json.Linq; + +namespace Pf2eDataScrapper.SchemaBuilder +{ + internal class DictionarySchemaType(SchemaNode parent) : AbstractSchemaType(parent) + { + public SchemaNode KeyNode = new(parent, "[key]"); + + public SchemaNode ItemNode = new(parent, "[value]"); + + public override bool IsCompatibleValue(JToken value) + { + return Resolver.TokenType(value) == JTokenType.Object; + } + + public override void AddValue(JToken value) + { + var obj = (JObject)value; + + foreach (var pair in obj) + { + KeyNode.AddValue(pair.Key); + ItemNode.AddValue(pair.Value); + } + } + + public override void Beautify() + { + + } + + public override string GetTypeName() + { + return $"dict<{KeyNode.GetTypeName()}, {ItemNode.GetTypeName()}>"; + } + + public override string ToString() + { + return string.Join("\n" + Tab, [ + GetTypeName(), + "Key:" + Indent(KeyNode.ToString()), + "Value:" + Indent(ItemNode.ToString()) + ]); + } + } +} diff --git a/Pf2eDataScrapper/SchemaBuilder/EmptySchemaType.cs b/Pf2eDataScrapper/SchemaBuilder/EmptySchemaType.cs deleted file mode 100644 index 0de0449..0000000 --- a/Pf2eDataScrapper/SchemaBuilder/EmptySchemaType.cs +++ /dev/null @@ -1,41 +0,0 @@ -using Newtonsoft.Json.Linq; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace Pf2eDataScrapper.SchemaBuilder -{ - internal class EmptySchemaType : AbstractSchemaType - { - public override AbstractSchemaType Adapt(JToken? value) - { - if (IsCompatibleValue(value)) - { - return this; - } - - return CreateFromValue(value).MarkNullable(); - } - - public override void AddValue(JToken? value) - { - } - - public override string GetTypeName() - { - return "empty[] -> null"; - } - - public override bool IsCompatibleValue(JToken? value) - { - return ResolveType(value) == "null"; - } - - public override string ToString() - { - return "null"; - } - } -} diff --git a/Pf2eDataScrapper/SchemaBuilder/MixedSchemaType.cs b/Pf2eDataScrapper/SchemaBuilder/MixedSchemaType.cs deleted file mode 100644 index 44c7655..0000000 --- a/Pf2eDataScrapper/SchemaBuilder/MixedSchemaType.cs +++ /dev/null @@ -1,82 +0,0 @@ -using Newtonsoft.Json.Linq; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace Pf2eDataScrapper.SchemaBuilder -{ - internal class MixedSchemaType : AbstractSchemaType - { - public List Types = []; - - public MixedSchemaType(AbstractSchemaType baseType) - { - Types.Add(baseType); - } - - public AbstractSchemaType AddType(JToken value) - { - var newType = CreateFromValue(value); - - Types.Add(newType); - - return newType; - } - - public AbstractSchemaType? GetCompatibleType(JToken value) - { - foreach (var type in Types) - { - if (type.IsCompatibleValue(value)) - { - return type; - } - } - return null; - } - - public override AbstractSchemaType Adapt(JToken? value) - { - var type = GetCompatibleType(value); - - if (type == null) - { - AddType(value); - } - - return this; - } - - public override void AddValue(JToken? value) - { - GetCompatibleType(value)?.AddValue(value); - } - - public override string GetTypeName() - { - List typeNames = Types.Select(t => t.GetTypeName()).ToList(); - - return string.Join("|", typeNames); - } - - public override bool IsCompatibleValue(JToken? value) => true; - - /// - /// String representation of type. - /// - /// - public override string ToString() - { - List result = ["mixed"]; - - foreach (var type in Types) - { - result.Add(Indent(type?.ToString() ?? "")); - } - - return string.Join("\n" + Tab, result); - } - } -} diff --git a/Pf2eDataScrapper/SchemaBuilder/ObjectSchemaType.cs b/Pf2eDataScrapper/SchemaBuilder/ObjectSchemaType.cs index 6bb281a..b4db3f9 100644 --- a/Pf2eDataScrapper/SchemaBuilder/ObjectSchemaType.cs +++ b/Pf2eDataScrapper/SchemaBuilder/ObjectSchemaType.cs @@ -7,39 +7,56 @@ using System.Threading.Tasks; namespace Pf2eDataScrapper.SchemaBuilder { - internal class ObjectSchemaType : AbstractSchemaType + internal class ObjectSchemaType(SchemaNode parent) : AbstractSchemaType(parent) { - protected bool IsFresh = true; + public List Fields = []; - public Dictionary Fields = []; - - public override void AddValue(JToken? value) + public override bool IsCompatibleValue(JToken value) { - if (value is null || ResolveType(value) == "null") - { - MarkNullable(); - return; - } + return Resolver.TokenType(value) == JTokenType.Object; + } + public override void AddValue(JToken value) + { var obj = (JObject)value; - - if (obj.Count == 0) - { - MarkNullable(); - return; - } foreach (var field in Fields) { - if (!obj.ContainsKey(field.Key)) + if (!obj.ContainsKey(field.Name)) { - field.Value.MarkNullable(); + field.IsNullable = true; } } - foreach (var field in obj) + foreach (var pair in obj) { - Fields[field.Key].AddValue(field.Value); + var field = Fields.Where(f => f.Name == pair.Key).FirstOrDefault(); + + if (field is null) + { + field = new SchemaNode(this.Parent, pair.Key); + Fields.Add(field); + } + + field.AddValue(pair.Value); + } + } + + public override void Beautify() + { + Fields.Sort((a, b) => + { + if (a.IsNullable && !b.IsNullable) + { + return 1; + } + + return a.Name.CompareTo(b.Name); + }); + + foreach (var field in Fields) + { + field.Beautify(); } } @@ -48,71 +65,23 @@ namespace Pf2eDataScrapper.SchemaBuilder return "object"; } - public override bool IsCompatibleValue(JToken? value) - { - return (ResolveType(value) == "object") || (ResolveType(value) == "null"); - } - - public override AbstractSchemaType Adapt(JToken? value) - { - if (!IsCompatibleValue(value)) - { - return new MixedSchemaType(this); - } - - if (value is null) - { - return MarkNullable(); - } - - var obj = (JObject)value; - - foreach (var field in Fields) - { - if (!obj.ContainsKey(field.Key)) - { - field.Value.MarkNullable(); - } - } - - foreach (var field in obj) - { - if (!Fields.ContainsKey(field.Key)) - { - Fields[field.Key] = CreateFromValue(field.Value); - - if (!IsFresh) - { - Fields[field.Key].MarkNullable(); - } - } - - Fields[field.Key].Adapt(field.Value); - } - - IsFresh = false; - return this; - } - public override string ToString() { List rows = ["object"]; foreach (var field in Fields) { - string s = field.Key; + string s = field.Name; - if (field.Value.IsNullable) + if (field.IsNullable) { s = $"[{s}]"; } - s += " : " + Indent(field.Value.ToString()); - - rows.Add(Indent($"{field.Key}: {field.Value}")); + rows.Add(Indent(s + " : " + field.ToString())); } - return string.Join("\n", rows); + return string.Join("\n" + Tab, rows); } } diff --git a/Pf2eDataScrapper/SchemaBuilder/SchemaNode.cs b/Pf2eDataScrapper/SchemaBuilder/SchemaNode.cs new file mode 100644 index 0000000..e9a5786 --- /dev/null +++ b/Pf2eDataScrapper/SchemaBuilder/SchemaNode.cs @@ -0,0 +1,104 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Newtonsoft.Json.Linq; + +namespace Pf2eDataScrapper.SchemaBuilder +{ + internal class SchemaNode + { + public string Name = ""; + + public string Path = ""; + + public SchemaNode? Parent = null; + + public bool Root = false; + + public SchemaNode RootNode => Root ? this : Parent!.RootNode; + + public List Types = []; + + public bool IsNullable = false; + + public TypeResolver Resolver = new(); + + public int Values { get; protected set; } = 0; + + public bool IsSimple => Types.Count == 1; + + public SchemaNode() + { + this.Root = true; + } + + public SchemaNode(SchemaNode parent, string name) + { + this.Parent = parent; + this.Name = name; + this.Path = parent.Path + "." + name; + this.Resolver = parent.Resolver; + } + + public void AddValue(JToken? value) + { + if (value is null || Resolver.TokenType(value) == JTokenType.Null) + { + IsNullable = true; + return; + } + + Values++; + + foreach (var type in Types) + { + if (type.IsCompatibleValue(value)) + { + type.AddValue(value); + return; + } + } + + var newType = Resolver.Resolve(this, value); + newType.AddValue(value); + Types.Add(newType); + } + + public string GetTypeName() + { + if (IsSimple) + { + return Types[0].GetTypeName(); + } + + List typeNames = Types.Select(t => t.GetTypeName()).ToList(); + + return string.Join("|", typeNames); + } + + public override string ToString() + { + if (IsSimple) + { + return Types[0].ToString(); + } + + List result = ["mixed"]; + + foreach (var type in Types) + { + result.Add(AbstractSchemaType.Indent(type?.ToString() ?? "")); + } + + return string.Join("\n" + AbstractSchemaType.Tab, result); + } + + public void Beautify() + { + + } + } +} diff --git a/Pf2eDataScrapper/SchemaBuilder/StaticSchemaType.cs b/Pf2eDataScrapper/SchemaBuilder/StaticSchemaType.cs index 25e5fdd..b586f1d 100644 --- a/Pf2eDataScrapper/SchemaBuilder/StaticSchemaType.cs +++ b/Pf2eDataScrapper/SchemaBuilder/StaticSchemaType.cs @@ -2,34 +2,45 @@ namespace Pf2eDataScrapper.SchemaBuilder { - internal class StaticSchemaType(string type) : AbstractSchemaType + internal class StaticSchemaType(SchemaNode parent, JTokenType type) : AbstractSchemaType(parent) { - protected string Type = type; + protected JTokenType Type = type; - protected Dictionary Values = []; + protected Dictionary Values = []; - public override void AddValue(JToken? value) + public override bool IsCompatibleValue(JToken value) + { + return value.Type == Type; + } + + public override void AddValue(JToken value) { Values.TryAdd(value, 0); Values[value]++; } - public override string GetTypeName() + public override void Beautify() { - return Type; + // Do nothing } - public override bool IsCompatibleValue(JToken? value) + public override string GetTypeName() { - return ResolveType(value) == Type; + return Type.ToString(); } public override string ToString() { - var result = Type; + var result = Type.ToString(); var count = Values.Count; + if (count == Root.Values) + { + result += " (all different values)"; + return result; + } + result += $" (different values: {count})"; if (count is > ElementLimit or <= 0) @@ -37,6 +48,7 @@ namespace Pf2eDataScrapper.SchemaBuilder return result.Trim(); } + result += "\n"; result = Values.Aggregate(result, (current, value) => current + $"{Tab}{value.Key} (x{value.Value})\n"); diff --git a/Pf2eDataScrapper/SchemaBuilder/TypeResolver.cs b/Pf2eDataScrapper/SchemaBuilder/TypeResolver.cs new file mode 100644 index 0000000..167fdae --- /dev/null +++ b/Pf2eDataScrapper/SchemaBuilder/TypeResolver.cs @@ -0,0 +1,40 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Newtonsoft.Json.Linq; + +namespace Pf2eDataScrapper.SchemaBuilder +{ + internal class TypeResolver + { + public JTokenType TokenType(JToken? value) + { + if (value is null || value.Type is JTokenType.Null or JTokenType.Undefined or JTokenType.None) + { + return JTokenType.Null; + } + + return value.Type; + } + + public virtual AbstractSchemaType Resolve(SchemaNode parent, JToken? value) + { + var type = TokenType(value); + + if (type == JTokenType.Null) + { + throw new Exception("Value is null, cannot determine type"); + } + + return type switch + { + JTokenType.Array => new ArraySchemaType(parent), + JTokenType.Object => new ObjectSchemaType(parent), + _ => new StaticSchemaType(parent, type) + }; + } + + } +} diff --git a/Pf2eDataScrapper/SchemaDumper.cs b/Pf2eDataScrapper/SchemaDumper.cs new file mode 100644 index 0000000..7f23ad0 --- /dev/null +++ b/Pf2eDataScrapper/SchemaDumper.cs @@ -0,0 +1,49 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Pf2eDataScrapper.SchemaBuilder; + +namespace Pf2eDataScrapper +{ + internal class SchemaDumper(Dictionary data) + { + public readonly Dictionary Data = data; + + public readonly string OutputDir = EnvSettings.OutputDir; + + + public void Run() + { + Console.WriteLine("Dumping schema..."); + + foreach (var pair in Data) + { + var path = Path.Combine(OutputDir, pair.Key + ".txt"); + var dir = Path.GetDirectoryName(path); + + if (!Directory.Exists(dir)) + { + Directory.CreateDirectory(dir); + } + + + var schema = pair.Value; + + using var writer = new StreamWriter(path); + + writer.WriteLine(pair.Key); + writer.Write(schema.ToString()); + } + + Console.WriteLine("Done!"); + } + + + public static void Dump(Dictionary data) + { + (new SchemaDumper(data)).Run(); + } + } +} diff --git a/Pf2eDataScrapper/test.json b/Pf2eDataScrapper/test.json index e6c936f..62ba4be 100644 --- a/Pf2eDataScrapper/test.json +++ b/Pf2eDataScrapper/test.json @@ -1,239 +1,97 @@ -{ - "_id": "XwfcJuskrhI9GIjX", - "img": "systems/pf2e/icons/classes/alchemist.webp", - "name": "Alchemist", - "system": { - "ancestryFeatLevels": { - "value": [ - 1, - 5, - 9, - 13, - 17 - ] - }, - "attacks": { - "advanced": 0, - "martial": 0, - "other": { - "name": "Alchemical Bombs", - "rank": 1 - }, - "simple": 1, - "unarmed": 1 - }, - "classFeatLevels": { - "value": [ - 1, - 2, - 4, - 6, - 8, - 10, - 12, - 14, - 16, - 18, - 20 - ] - }, - "defenses": { - "heavy": 0, - "light": 1, - "medium": 1, - "unarmored": 1 - }, - "description": { - "value": "

There's no sight more beautiful to you than a strange brew bubbling in a beaker, and you consume your ingenious elixirs with abandon. You're fascinated by uncovering the secrets of science and the natural world, and you're constantly experimenting in your lab or on the go with inventive concoctions for every eventuality. You are fearless in the face of risk, hurling explosive or toxic creations at your foes. Your unique path toward greatness is lined with alchemical brews that push your mind and body to their limits.

\n

@UUID[Compendium.pf2e.journals.JournalEntry.kzxu2dI7tFxv6Ix6.JournalEntryPage.Om4e4tsgNWrComie]{Alchemist}

" - }, - "generalFeatLevels": { - "value": [ - 3, - 7, - 11, - 15, - 19 - ] - }, - "hp": 8, - "items": { - "4bdrn": { - "img": "icons/tools/laboratory/mortar-liquid-pink.webp", - "level": 1, - "name": "Alchemy", - "uuid": "Compendium.pf2e.classfeatures.Item.Alchemy" - }, - "CJ528": { - "img": "icons/magic/symbols/question-stone-yellow.webp", - "level": 11, - "name": "Chemical Hardiness", - "uuid": "Compendium.pf2e.classfeatures.Item.Chemical Hardiness" - }, - "JK2gV": { - "img": "icons/consumables/potions/potion-round-stopped-yellow-green.webp", - "level": 15, - "name": "Alchemical Weapon Mastery", - "uuid": "Compendium.pf2e.classfeatures.Item.Alchemical Weapon Mastery" - }, - "NY2Su": { - "img": "icons/creatures/eyes/human-single-blue.webp", - "level": 9, - "name": "Perception Expertise", - "uuid": "Compendium.pf2e.classfeatures.Item.Perception Expertise" - }, - "Q1cDZ": { - "img": "icons/magic/symbols/question-stone-yellow.webp", - "level": 15, - "name": "Explosion Dodger", - "uuid": "Compendium.pf2e.classfeatures.Item.Explosion Dodger" - }, - "SWNkG": { - "img": "icons/equipment/chest/breastplate-collared-steel.webp", - "level": 19, - "name": "Medium Armor Mastery", - "uuid": "Compendium.pf2e.classfeatures.Item.Medium Armor Mastery" - }, - "aay1e": { - "img": "systems/pf2e/icons/features/classes/walls-of-will.webp", - "level": 7, - "name": "Will Expertise", - "uuid": "Compendium.pf2e.classfeatures.Item.Will Expertise" - }, - "b3ohx": { - "img": "icons/magic/symbols/question-stone-yellow.webp", - "level": 1, - "name": "Research Field", - "uuid": "Compendium.pf2e.classfeatures.Item.Research Field" - }, - "cNnDR": { - "img": "icons/equipment/chest/breastplate-quilted-brown.webp", - "level": 13, - "name": "Medium Armor Expertise", - "uuid": "Compendium.pf2e.classfeatures.Item.Medium Armor Expertise" - }, - "enfdl": { - "img": "systems/pf2e/icons/features/classes/weapon-specialization.webp", - "level": 13, - "name": "Weapon Specialization", - "uuid": "Compendium.pf2e.classfeatures.Item.Weapon Specialization" - }, - "f4a0p": { - "img": "icons/magic/light/orbs-hand-gray.webp", - "level": 9, - "name": "Double Brew", - "uuid": "Compendium.pf2e.classfeatures.Item.Double Brew" - }, - "j5xnl": { - "img": "systems/pf2e/icons/features/classes/alchemical-weapon-expertise.webp", - "level": 7, - "name": "Alchemical Weapon Expertise", - "uuid": "Compendium.pf2e.classfeatures.Item.Alchemical Weapon Expertise" - }, - "l170j": { - "img": "icons/magic/symbols/question-stone-yellow.webp", - "level": 13, - "name": "(Choice) Greater Field Discovery", - "uuid": "Compendium.pf2e.classfeatures.Item.Greater Field Discovery" - }, - "plldb": { - "img": "icons/consumables/potions/potion-round-corked-glowing-green.webp", - "level": 9, - "name": "Alchemical Expertise", - "uuid": "Compendium.pf2e.classfeatures.Item.Alchemical Expertise" - }, - "rpCw8": { - "img": "icons/magic/symbols/question-stone-yellow.webp", - "level": 17, - "name": "Abundant Vials", - "uuid": "Compendium.pf2e.classfeatures.Item.Abundant Vials" - }, - "vOE57": { - "img": "icons/magic/symbols/question-stone-yellow.webp", - "level": 11, - "name": "Advanced Vials", - "uuid": "Compendium.pf2e.classfeatures.Item.Advanced Vials" - }, - "xzcdr": { - "img": "icons/magic/symbols/question-stone-yellow.webp", - "level": 5, - "name": "Field Discovery", - "uuid": "Compendium.pf2e.classfeatures.Item.Field Discovery" - }, - "yc9ak": { - "img": "icons/commodities/materials/bowl-powder-teal.webp", - "level": 5, - "name": "Powerful Alchemy", - "uuid": "Compendium.pf2e.classfeatures.Item.Powerful Alchemy" - }, - "yy1uz": { - "img": "icons/tools/cooking/mortar-stone-yellow.webp", - "level": 17, - "name": "Alchemical Mastery", - "uuid": "Compendium.pf2e.classfeatures.Item.Alchemical Mastery" - } - }, - "keyAbility": { - "value": [ - "int" - ] - }, - "perception": 1, - "publication": { - "license": "ORC", - "remaster": true, - "title": "Pathfinder Player Core 2" - }, - "rules": [ - { - "key": "ActiveEffectLike", - "mode": "upgrade", - "path": "system.proficiencies.attacks.weapon-base-alchemical-bomb.rank", - "value": 1 - } +{ + "_id": "fekYhQe1qrvwarBL", + "folder": "bGvA4p5QGO30eC0r", + "img": "systems/pf2e/icons/default-icons/hazard.svg", + "items": [ + { + "_id": "vsxjIPIKMzsQXXfM", + "img": "systems/pf2e/icons/actions/Reaction.webp", + "name": "Death Throes", + "sort": 0, + "system": { + "actionType": { + "value": "reaction" + }, + "actions": { + "value": null + }, + "category": null, + "description": { + "value": "

Trigger A creature touches the vampire's corpse

\n
\n

Effect A surge of void energy fills the room as the corpse lurches as though staked. The haunt deals @Damage[10d8[void]] damage to creatures within 30 feet of the corpse; this damage heals undead creatures for the same amount. Living creatures must attempt a @Check[will|dc:30|traits:damaging-effect] save.

\n
\n

Critical Success The creature is unaffected.

\n

Success The creature takes half damage and becomes @UUID[Compendium.pf2e.conditionitems.Item.Drained]{Drained 1}. This drained condition is reduced by 1 after 10 minutes.

\n

Failure The creature takes full damage and becomes drained 1.

\n

Critical Failure The creature takes double damage and is @UUID[Compendium.pf2e.conditionitems.Item.Drained]{Drained 2}.

" + }, + "publication": { + "license": "OGL", + "remaster": false, + "title": "" + }, + "rules": [], + "slug": null, + "traits": { + "rarity": "common", + "value": [] + } + }, + "type": "action" + } ], - "savingThrows": { - "fortitude": 2, - "reflex": 2, - "will": 1 + "name": "Benefactor's End", + "system": { + "attributes": { + "ac": { + "value": 0 + }, + "hardness": 0, + "hp": { + "details": "", + "max": 0, + "temp": 0, + "value": 0 + }, + "stealth": { + "details": "

(expert)

", + "value": 20 + } + }, + "creatureType": "", + "details": { + "description": "

A spectral assailant drives a stake into a dying vampire's heart; the vampire collapses and releases a surge of void energy.

", + "disable": "

@Check[religion|dc:28] (expert) to drain away the haunt's void energy

", + "isComplex": false, + "level": { + "value": 9 + }, + "publication": { + "license": "OGL", + "remaster": false, + "title": "Pathfinder #183: Field of Maidens" + }, + "reset": "

Unless disabled, the haunt materializes again on the next moonrise.

", + "routine": "" + }, + "saves": { + "fortitude": { + "saveDetail": "", + "value": 0 + }, + "reflex": { + "saveDetail": "", + "value": 0 + }, + "will": { + "saveDetail": "", + "value": 0 + } + }, + "statusEffects": [], + "traits": { + "rarity": "common", + "size": { + "value": "med" + }, + "value": [ + "haunt" + ] + } }, - "skillFeatLevels": { - "value": [ - 2, - 4, - 6, - 8, - 10, - 12, - 14, - 16, - 18, - 20 - ] - }, - "skillIncreaseLevels": { - "value": [ - 3, - 5, - 7, - 9, - 11, - 13, - 15, - 17, - 19 - ] - }, - "spellcasting": 0, - "trainedSkills": { - "additional": 3, - "value": [ - "crafting" - ] - }, - "traits": { - "rarity": "common", - "value": [] - } - }, - "type": "class" -} \ No newline at end of file + "type": "hazard" +} diff --git a/README.md b/README.md new file mode 100644 index 0000000..f5bdda2 --- /dev/null +++ b/README.md @@ -0,0 +1,23 @@ +# Pf2e Data Screapper + +Программа обработки данных для игры Pathfinder 2e для создания БД для дальнейшего использования на сайте локализации Pf2e. + +## Общая структура + +### `SchemaBuilder` + +Классы обработки вводимых JSON-данных и создание схемы данных для использовании при формировании обработчиков данных. + +Использование: +```csharp + +SchemaNode schema = new(); // Создание схемы данных + +data = JObject.Parse(reader.ReadToEnd()); // Загрузка данных + +schema.AddValue(data); // Добавление данных +``` + +### `Importers` + +Классы загрузки и обработки данных.