diff --git a/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/DataBuilder/AbstractEntityProcessor.cs b/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/DataBuilder/AbstractEntityProcessor.cs new file mode 100644 index 0000000..69912aa --- /dev/null +++ b/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/DataBuilder/AbstractEntityProcessor.cs @@ -0,0 +1,24 @@ +using Newtonsoft.Json.Linq; +using Slugify; + +namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.DataBuilder +{ + internal abstract class AbstractEntityProcessor(JObject input) : AbstractProcessor(input) + { + public abstract string DbType { get; } + + public override void Process() + { + Add("name", "name"); + Add("img", "img"); + Add("description", "system.description.value"); + Add("description_gm", "system.description.gm"); + Add("type", DbType); + Add("rarity", "system.traits.rarity"); + + SlugHelper sh = new(); + Data.Add("slug", sh.GenerateSlug(Data["name"]?.ToString() ?? throw new NullReferenceException("'name' is null"))); + } + + } +} diff --git a/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/DataBuilder/AbstractProcessor.cs b/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/DataBuilder/AbstractProcessor.cs new file mode 100644 index 0000000..15b4440 --- /dev/null +++ b/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/DataBuilder/AbstractProcessor.cs @@ -0,0 +1,30 @@ +using Newtonsoft.Json.Linq; +using Npgsql; + +namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.DataBuilder +{ + internal abstract class AbstractProcessor(JObject input) + { + public readonly JObject Input = input; + + public Pf2eDatabase Database = Pf2eDatabase.Instance; + + public DataBuilderHelper Helper = DataBuilderHelper.Instance; + + public abstract string Table { get; } + + public Dictionary Data = []; + + protected void Add(string key, string path) + { + Data.Add(key, DataBuilderHelper.JsonGet(Input, path)); + } + + public abstract void Process(); + + public int Save() + { + return Helper.Insert(Table, Data); + } + } +} diff --git a/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/DataBuilder/DataBuilderHelper.cs b/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/DataBuilder/DataBuilderHelper.cs new file mode 100644 index 0000000..cbab03d --- /dev/null +++ b/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/DataBuilder/DataBuilderHelper.cs @@ -0,0 +1,82 @@ +using Newtonsoft.Json.Linq; +using Npgsql; + +namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.DataBuilder +{ + internal class DataBuilderHelper + { + public Pf2eDatabase Database = Pf2eDatabase.Instance; + + private DataBuilderHelper() + { + } + + public NpgsqlCommand PrepareInsert(string table, Dictionary data, bool returnId) + { + List columns = []; + List patterns = []; + + foreach (var key in data.Keys) + { + columns.Add(key); + patterns.Add($":{key.ToUpper()}"); + } + + string dml = $"INSERT INTO {table} (\"{string.Join("\", \"", columns)}\") VALUES ({string.Join(", ", patterns)})" + (returnId ? " RETURNING id" : ""); + + NpgsqlCommand cmd = Database.GetCommand(dml); + foreach (var kvp in data) + { + cmd.Parameters.AddWithValue(":" + kvp.Key.ToUpper(), kvp.Value ?? DBNull.Value); + } + + Console.WriteLine(cmd.CommandText); + return cmd; + } + + public int Insert(string table, Dictionary data) + { + var cmd = PrepareInsert(table, data, true); + return (int)(cmd.ExecuteScalar() ?? throw new Exception("Not a number")); + } + + public void InsertBlind(string table, Dictionary data) + { + var cmd = PrepareInsert(table, data, false); + cmd.ExecuteNonQuery(); + } + + public static object? JsonGet(JObject root, string path) + { + var token = root.SelectToken(path); + + if (token == null) + { + return null; + } + + if (token.Type == JTokenType.String) + { + return token.ToString(); + } + else if (token.Type == JTokenType.Integer) + { + return token.ToObject(); + } + else if (token.Type == JTokenType.Float) + { + return token.ToObject(); + } + else if (token.Type == JTokenType.Boolean) + { + return token.ToObject(); + } + else + { + return token.ToString(); + } + } + + public static DataBuilderHelper Instance = new(); + } +} diff --git a/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/DataBuilder/PF2eFoundryDataBuilder.cs b/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/DataBuilder/PF2eFoundryDataBuilder.cs new file mode 100644 index 0000000..d9cc945 --- /dev/null +++ b/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/DataBuilder/PF2eFoundryDataBuilder.cs @@ -0,0 +1,85 @@ +using Konsole; +using Newtonsoft.Json.Linq; +using Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.Compendium; +using System.Reflection; + +namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.DataBuilder +{ + internal class PF2eFoundryDataBuilder + { + public Pf2eDatabase Database = Pf2eDatabase.Instance; + + /// + /// Загрузчик данных. + /// + public readonly CompendiumLoader Loader; + + /// + /// Коллекция папок компендиума. + /// + public CompendiumFolderCollection Collection => Loader.Collection; + + public Dictionary Processors = []; + + public PF2eFoundryDataBuilder(CompendiumLoader loader) + { + Loader = loader; + PrepareProcessors(); + } + + public void PrepareProcessors() + { + foreach (Type type in Assembly.GetExecutingAssembly().GetTypes()) + { + if (type.GetCustomAttributes(typeof(Pf2eTypeProcessor), true).Length > 0) + { + var attribute = type.GetCustomAttribute(); + Processors.Add(attribute!.Type, type.GetConstructor(new[] { typeof(JObject) })!); + } + } + } + + public AbstractProcessor? GetProcessor(string type, JObject input) + { + if (Processors.TryGetValue(type, out var constructor)) + { + return (AbstractProcessor)constructor.Invoke([input]); + } + + return null; + } + + public void TruncateTables() + { + Console.WriteLine("Truncating all old data..."); + + Database.ExecuteCommand($"SELECT f_truncate_tables();"); + + Console.WriteLine("Truncation completed!"); + } + + public void Build() + { + Console.WriteLine("Building data..."); + + var bar = new ProgressBar(Loader.ItemCount); + var i = 0; + + foreach (var entry in Loader.Enrties()) + { + bar.Refresh(i++, $"{entry.Folder.Name}/{entry.File.Name}"); + + var processor = GetProcessor($"{entry.Folder.Type}.{entry.Type}", entry.Json); + if (processor is null) + { + continue; + } + + processor.Process(); + processor.Save(); + } + + bar.Refresh(Loader.ItemCount, "Done!"); + } + } +} diff --git a/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/DataBuilder/Pf2eTypeProcessor.cs b/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/DataBuilder/Pf2eTypeProcessor.cs new file mode 100644 index 0000000..0720062 --- /dev/null +++ b/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/DataBuilder/Pf2eTypeProcessor.cs @@ -0,0 +1,11 @@ +using System.Reflection; + +namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.DataBuilder +{ + [AttributeUsage(AttributeTargets.Class)] + internal class Pf2eTypeProcessor(string type) : Attribute + { + public readonly string Type = type; + + } +} diff --git a/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/DataBuilder/Processors/ConditionProcessor.cs b/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/DataBuilder/Processors/ConditionProcessor.cs new file mode 100644 index 0000000..edb6f81 --- /dev/null +++ b/Pf2eDataScrapper/Importers/Implementations/Pf2eFoundry/DataBuilder/Processors/ConditionProcessor.cs @@ -0,0 +1,29 @@ +using Newtonsoft.Json.Linq; + +namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.DataBuilder.Processors +{ + [Pf2eTypeProcessor("Item.condition")] + internal class ConditionProcessor(JObject input) : AbstractEntityProcessor(input) + { + public override string Table => "condition"; + + public override string DbType => "condition"; + + public override void Process() + { + base.Process(); + + Add("group", "system.group"); + + JObject extra = new() + { + { "duration", Input.SelectToken("duration") }, + { "overrides", Input.SelectToken("overrides") }, + { "references", Input.SelectToken("references") }, + { "value", Input.SelectToken("value") } + }; + + Data.Add("extra", extra.ToString(Newtonsoft.Json.Formatting.Indented)); + } + } +}