Построение схем с Архивов Нефиса

2025-04-15 13:50:21 +04:00
parent 5475b6ab46
commit b67c964c80
104 changed files with 29299 additions and 42 deletions
@@ -1,4 +1,6 @@
-namespace Pf2eDataScrapper.Importers
+using Pf2eDataScrapper.Importers.Repositories;
+
+namespace Pf2eDataScrapper.Importers
 {
    /// <summary>
    /// Абстрактный класс импортёра
@@ -15,10 +17,7 @@
        /// </summary>
        public abstract string Slug { get; }

-        /// <summary>
-        /// Ссылка на git-репозиторий для клонирования
-        /// </summary>
-        public abstract string GitUrl { get; }
+        public abstract AbstractImporterRepository Repository { get; }

        /// <summary>
        /// Путь к директории с данными для импортёра
@@ -0,0 +1,40 @@
+using Pf2eDataScrapper.Importers.Repositories;
+using Pf2eDataScrapper.Importers.Implementations.Aonprd.SchemaBuilder;
+
+namespace Pf2eDataScrapper.Importers.Implementations.Aonprd
+{
+    internal class AonprdImporter : AbstractImporter
+    {
+        public override string Name => "Archives of Nethys Pathfinder 2e";
+
+        public override string Slug => "aonprd";
+
+        public override AbstractImporterRepository Repository => new AonprdRepository(this, "https://elasticsearch.aonprd.com");
+
+        public AonprdLoader Loader { get; }
+
+        public AonprdImporter()
+        {
+            Loader = new AonprdLoader(this);
+        }
+
+        public override void Prepare()
+        {
+            Repository.Update();
+            Loader.Prepare();
+        }
+
+        public override void BuildSchemas()
+        {
+            var builder = new AonprdSchemaBuilder(Loader);
+
+            builder.BuildSchemas();
+            builder.Save(EnvSettings.OutputDir);
+        }
+
+        public override void Import()
+        {
+            throw new NotImplementedException();
+        }
+    }
+}
@@ -0,0 +1,79 @@
+using Microsoft.EntityFrameworkCore.Metadata.Internal;
+using Newtonsoft.Json.Linq;
+using Pf2eDataScrapper.Importers.Implementations.Aonprd.Compendium;
+
+namespace Pf2eDataScrapper.Importers.Implementations.Aonprd
+{
+    internal class AonprdLoader (AonprdImporter importer)
+    {
+        public AonprdImporter Importer = importer;
+
+        public AonprdCompendium Compendium = new();
+
+        public int ItemCount => Compendium.Count;
+
+        public void Prepare()
+        {
+            using var indexFile = new StreamReader(Path.Combine(Importer.Dir, "index.json"));
+            var indexJson = JObject.Parse(indexFile.ReadToEnd());
+
+            foreach (var kvp in indexJson)
+            {
+                var file = kvp.Key.ToString();
+                var items = (JArray)kvp.Value!;
+
+                foreach (var item in items)
+                {
+                    Compendium.AddReference(item.ToString(), file);
+                }
+            }
+        }
+
+        public IEnumerable<LoaderEntry> Entries()
+        {
+            return Entries(Compendium.Types.Keys.ToArray());
+        }
+
+        public IEnumerable<LoaderEntry> Entries(string types)
+        {
+            return Entries([types]);
+        }
+
+        public IEnumerable<LoaderEntry> Entries(string[] types)
+        {
+            foreach (var file in Compendium.Files(types))
+            {
+                if (file is null)
+                {
+                    continue;
+                }
+
+                using var fileReader = new StreamReader(Path.Combine(Importer.Dir, $"{file}.json"));
+                var array = JArray.Parse(fileReader.ReadToEnd());
+
+                foreach (var item in array)
+                {
+                    var id = item.SelectToken("id")?.ToString();
+
+                    if (id is null)
+                    {
+                        continue;
+                    }
+
+                    var entry = new LoaderEntry(id, file, (JObject)item);
+
+                    if (types.Contains(entry.Type))
+                    {
+                        yield return entry;
+                    }
+                }
+            }
+        }
+
+
+        public class LoaderEntry(string id, string file, JObject itemJson) : AonprdReference(id, file)
+        {
+            public JObject Json { get; } = itemJson;
+        }
+    }
+}
@@ -0,0 +1,102 @@
+using Konsole;
+using Newtonsoft.Json.Linq;
+using Pf2eDataScrapper.Importers.Repositories;
+
+namespace Pf2eDataScrapper.Importers.Implementations.Aonprd
+{
+    internal class AonprdRepository(AbstractImporter importer, string searchUrl): AbstractImporterRepository(importer)
+    {
+        public readonly string SearchUrl = searchUrl;
+
+        public int Revision { get; private set; }
+
+        private string Load(string url)
+        {
+            if (!url.StartsWith("http://") && !url.StartsWith("https://"))
+            {
+                url = SearchUrl + url;
+            }
+
+            using var client = new HttpClient();
+            using var stream = client.GetStreamAsync(url);
+            using var reader = new StreamReader(stream.Result);
+
+            return reader.ReadToEnd();
+        }
+
+
+        public override void Update()
+        {
+            if (CheckRevision())
+            {
+                Console.WriteLine($"Данные Aonprd актуальны, текущая ревизия {Revision}.");
+                return;
+            }
+
+            Console.WriteLine($"Данные Aonprd устарели, обновляем {RepositoryDirPath} до ревизии {Revision}...");
+
+            if (Directory.Exists(RepositoryDirPath))
+            {
+                Directory.Delete(RepositoryDirPath, true);
+            }
+            Directory.CreateDirectory(RepositoryDirPath);
+
+            WriteFile("revision.txt", Revision.ToString());
+
+            Console.WriteLine("Загрузка индекса...");
+            var indexContent = Load($"/json-data/aon{Revision}-index.json");
+            WriteFile("index.json", indexContent);
+
+            var indexJson = JObject.Parse(indexContent);
+
+            Console.WriteLine("Загрузка индексных файлов...");
+
+            var bar = new ProgressBar(indexJson.Count);
+            var i = 0;
+
+            foreach (var kvp in indexJson)
+            {
+                var fileName = kvp.Key;
+                bar.Refresh(i++, fileName);
+
+                var fileContent = Load($"/json-data/{fileName}.json");
+                WriteFile($"{fileName}.json", fileContent);
+            }
+
+
+            bar.Refresh(indexJson.Count, "Готово!");
+        }
+
+        /// <summary>
+        /// Проверка актуальности данных
+        /// </summary>
+        /// <returns>true - актуальная ревизия, false - устаревшая/отсутсвует</returns>
+        private bool CheckRevision()
+        {
+            var page = JObject.Parse(Load("/aon/_doc/feat-1"));
+
+            Revision = Convert.ToInt32(page.SelectToken("_index")?.ToObject<string>()?.Replace("aon", "") ?? throw new Exception("Can't get revision"));
+
+            var revisionFile = Path.Combine(RepositoryDirPath, "revision.txt");
+            if (!File.Exists(revisionFile))
+            {
+                return false;
+            }
+
+            using var reader = new StreamReader(revisionFile);
+            var lastRevision = reader.ReadToEnd();
+            return lastRevision == Revision.ToString();
+        }
+
+        private void WriteFile (string path, string content)
+        {
+            var filePath = Path.Combine(RepositoryDirPath, path);
+            if (File.Exists(filePath))
+            {
+                File.Delete(filePath);
+            }
+            using var writer = new StreamWriter(filePath);
+            writer.Write(content);
+        }
+    }
+}
@@ -0,0 +1,28 @@
+namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Compendium
+{
+    internal class AonprdCompendium
+    {
+        public readonly Dictionary<string, AonprdType> Types = [];
+
+        public void AddReference(string id, string file)
+        {
+            AonprdReference reference = new(id, file);
+
+            if (Types.TryGetValue(reference.Type, out var type))
+            {
+                type.Add(reference);
+            }
+            else
+            {
+                Types.Add(reference.Type, new AonprdType(reference));
+            }
+        }
+
+        public int Count => Types.Values.Sum(x => x.Count);
+
+        public IEnumerable<string> Files(IEnumerable<string> types) => Types.Values
+            .Where(t => types.Contains(t.Type))
+            .SelectMany(t => t.Files)
+            .Distinct();
+    }
+}
@@ -0,0 +1,35 @@
+
+namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Compendium
+{
+    internal class AonprdReference(string id, string file)
+    {
+        public readonly string Id = id;
+
+        public readonly string Type = ResolveType(id);
+
+        public readonly string File = file;
+
+        public static string ResolveType(string id)
+        {
+            static string? subType(string type, string cat)
+            {
+                if (type.StartsWith($"{cat}-") && type != cat)
+                {
+                    if (type.EndsWith("-bonus"))
+                    {
+                        return $"{cat}-bonus";
+                    }
+                    else
+                    {
+                        return $"{cat}-sub";
+                    }
+                }
+                return null;
+            }
+
+            var type = string.Join('-', id.Split('-')[0..^1]);
+
+            return subType(type, "equipment") ?? subType(type, "weapon") ?? type;
+        }
+    }
+}
@@ -0,0 +1,23 @@
+namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Compendium
+{
+    internal class AonprdType(string type)
+    {
+        public readonly string Type = type;
+
+        public readonly List<AonprdReference> Items = [];
+
+        public AonprdType(AonprdReference reference) : this(reference.Type)
+        {
+            Items.Add(reference);
+        }
+
+        public void Add(AonprdReference reference)
+        {
+            Items.Add(reference);
+        }
+
+        public List<string> Files => [.. Items.Select(x => x.File).Distinct()];
+
+        public int Count => Items.Count;
+    }
+}
@@ -0,0 +1,87 @@
+using Konsole;
+using Pf2eDataScrapper.Importers.Implementations.Aonprd.Compendium;
+using Pf2eDataScrapper.SchemaBuilder;
+
+namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.SchemaBuilder
+{
+    internal class AonprdSchemaBuilder(AonprdLoader loader)
+    {
+        public readonly AonprdLoader Loader = loader;
+
+        public readonly AonprdCompendium Compendium = loader.Compendium;
+
+        // public readonly AonprdSchemaTypeResolver Resolver = new();
+
+        public readonly Dictionary<string, SchemaNode> Schemas = [];
+
+        private SchemaNode GetSchema(AonprdLoader.LoaderEntry entry)
+        {
+            string key = entry.Type;
+            if (!Schemas.TryGetValue(key, out SchemaNode? value))
+            {
+                value = new SchemaNode()
+                {
+                    // Resolver = Resolver,
+                    Path = key
+                };
+                Schemas[key] = value;
+            }
+            return value;
+        }
+
+        public void BuildSchemas()
+        {
+            Console.WriteLine("Building schemas...");
+
+            var bar = new ProgressBar(Loader.ItemCount);
+            var i = 0;
+
+            foreach (var entry in Loader.Entries())
+            {
+                bar.Refresh(i++, entry.Type);
+                GetSchema(entry).AddValue(entry.Json);
+            }
+
+            bar.Refresh(Loader.ItemCount, "Done!");
+        }
+
+        public void Save(string targetDir)
+        {
+            Console.WriteLine("Dumping schemas...");
+
+            targetDir = Path.Combine(targetDir, Loader.Importer.Slug);
+
+            var bar = new ProgressBar(Schemas.Count);
+            var i = 0;
+
+            foreach (var kvp in Schemas)
+            {
+                bar.Refresh(i++, kvp.Key);
+                SaveSchema(targetDir, kvp.Key, kvp.Value);
+            }
+        }
+
+        protected void SaveSchema(string dir, string name, SchemaNode schema)
+        {
+            var invalids = Path.GetInvalidFileNameChars();
+            var fileName = string.Join("_", schema.Path.Split(invalids, StringSplitOptions.RemoveEmptyEntries)).TrimEnd('.');
+            var path = Path.Combine(dir, fileName + ".yaml");
+
+            var dirPath = Path.GetDirectoryName(path);
+            if (dirPath is not null && !Directory.Exists(dirPath))
+            {
+                Directory.CreateDirectory(dirPath);
+            }
+
+            using var writer = new StreamWriter(path);
+
+            schema.Beautify();
+
+            writer.WriteLine(SchemaNode.FormatYaml(new Dictionary<string, string>()
+            {
+                ["name"] = name,
+                ["values"] = schema.Values.ToString()
+            }, schema.Types[0].GetYamlData()));
+        }
+    }
+}
@@ -1,5 +1,6 @@
 using Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.DataBuilder;
 using Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.SchemaBuilder;
+using Pf2eDataScrapper.Importers.Repositories;

 namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry
 {
@@ -18,10 +19,7 @@ namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry
        /// </summary>
        public override string Slug => "pf2e";

-        /// <summary>
-        /// Ссылка на git-репозиторий для клонирования.
-        /// </summary>
-        public override string GitUrl => "https://github.com/foundryvtt/pf2e.git";
+        public override AbstractImporterRepository Repository => new GitRepository(this, "https://github.com/foundryvtt/pf2e.git");

        /// <summary>
        /// Загрузчик данных компендиума.
@@ -123,7 +123,7 @@ namespace Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry.SchemaBuilder
            var bar = new ProgressBar(PackSchemas.Count + TypeSchemas.Count);
            var i = 0;

-            targetDir = Path.Combine(targetDir, "pf2e");
+            targetDir = Path.Combine(targetDir, Loader.Importer.Slug);

            if (!Directory.Exists(targetDir))
            {
@@ -0,0 +1,13 @@
+namespace Pf2eDataScrapper.Importers.Repositories
+{
+    internal abstract class AbstractImporterRepository (AbstractImporter importer)
+    {
+        public readonly AbstractImporter Importer = importer;
+
+        public string RepositoryDirPath => Importer.Dir;
+
+        public DirectoryInfo RepositoryDir => new(RepositoryDirPath);
+
+        public abstract void Update();
+    }
+}
@@ -1,28 +1,20 @@
-using System.Diagnostics;
+using System.Data.Common;
+using System.Diagnostics;

-namespace Pf2eDataScrapper.Importers
+namespace Pf2eDataScrapper.Importers.Repositories
 {
    /// <summary>
    /// Загрузчик данных из git-репозиториев
    /// </summary>
    /// <param name="repositoryDirPath"></param>
-    internal class GitLoader(string repositoryDirPath)
+    internal class GitRepository(AbstractImporter importer, string url) : AbstractImporterRepository(importer)
    {
-        /// <summary>
-        /// Путь к директории репозитория
-        /// </summary>
-        public readonly string RepositoryDirPath = repositoryDirPath;
-
-        /// <summary>
-        /// Директория репозитория
-        /// </summary>
-        public DirectoryInfo RepositoryDir = new(repositoryDirPath);
+        public readonly string Url = url;

        /// <summary>
        /// Обновление импортёра
        /// </summary>
-        /// <param name="importer"></param>
-        public void UpdateImporter(AbstractImporter importer)
+        public override void Update()
        {
            var gitPath = Path.Combine(RepositoryDirPath, importer.Slug);

@@ -49,7 +41,7 @@ namespace Pf2eDataScrapper.Importers
                    {
                        FileName = "git",
                        WorkingDirectory = RepositoryDirPath,
-                        Arguments = $"clone {importer.GitUrl} {importer.Slug}",
+                        Arguments = $"clone {Url} {Importer.Slug}",
                    }
                };
                process.Start();
@@ -1,26 +1,15 @@
 using Pf2eDataScrapper;
+using Pf2eDataScrapper.Importers.Implementations.Aonprd;
 using Pf2eDataScrapper.Importers.Implementations.Pf2eFoundry;

 EnvSettings.Load();

-
-var importer = new Pf2EFoundryImporter();
-
+var importer = new AonprdImporter();
 importer.Prepare();
+importer.BuildSchemas();

-importer.Import();
 /*
-try
-{
-}
-catch (Exception ex)
-{
-    Console.WriteLine(ex);
-    //Console.WriteLine(ex.Message);
-    //Console.WriteLine(ex.InnerException);
-    //Console.WriteLine(ex.StackTrace);
-}
-// importer.BuildSchemas();
-
-
+var importer = new Pf2EFoundryImporter();
+importer.Prepare();
+importer.Import();
 */