diff --git a/Pf2eDataScrapper/Importers/Implementations/Aonprd/DataBuilder/AonprdDataBuilder.cs b/Pf2eDataScrapper/Importers/Implementations/Aonprd/DataBuilder/AonprdDataBuilder.cs index aa26ef8..1fc0532 100644 --- a/Pf2eDataScrapper/Importers/Implementations/Aonprd/DataBuilder/AonprdDataBuilder.cs +++ b/Pf2eDataScrapper/Importers/Implementations/Aonprd/DataBuilder/AonprdDataBuilder.cs @@ -80,18 +80,20 @@ internal class AonprdDataBuilder(AonprdLoader loader) { ConfigureWindow(); + ProcessProgressBar.Refresh(-1, ""); + foreach (AonprdProcessorCollection.ProcessorItem item in Processors.Iterate()) { ProcessProgressBar.Next(item.Attribute.Name); + SubProcessProgressBar.Refresh(-1, "Starting..."); + if (item.Attribute.Type is null) { RunProcessor(item, []); continue; } - SubProcessProgressBar.Refresh(0, ""); - SubProcessProgressBar.Max = Compendium.Count([item.Attribute.Type]); foreach (AonprdLoader.LoaderEntry entry in Loader.Entries(item.Attribute.Type)) diff --git a/Pf2eDataScrapper/Importers/Implementations/Aonprd/DataBuilder/AonprdProcessorAttribute.cs b/Pf2eDataScrapper/Importers/Implementations/Aonprd/DataBuilder/AonprdProcessorAttribute.cs index 594ff4d..05dcbc2 100644 --- a/Pf2eDataScrapper/Importers/Implementations/Aonprd/DataBuilder/AonprdProcessorAttribute.cs +++ b/Pf2eDataScrapper/Importers/Implementations/Aonprd/DataBuilder/AonprdProcessorAttribute.cs @@ -1,20 +1,30 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder; [AttributeUsage(AttributeTargets.Class)] -public class AonprdProcessorAttribute(string name, string? type, string[] dependencies) : Attribute +public class AonprdProcessorAttribute( + string name, + string? type, + string[] dependencies, + bool includeDefaults = true +) : Attribute { + private static readonly string[] DefaultDependencies = ["trait", "source"]; public readonly string Name = name; public readonly string? Type = type; - public readonly IEnumerable Dependencies = dependencies; + public readonly IEnumerable Dependencies = + [ + .. includeDefaults ? DefaultDependencies : [], + .. dependencies, + ]; public AonprdProcessorAttribute(string type) - : this(type, type, ["trait", "source"]) { } + : this(type, type, DefaultDependencies) { } - public AonprdProcessorAttribute(string type, string[] dependencies) - : this(type, type, dependencies) { } + public AonprdProcessorAttribute(string type, string[] dependencies, bool includeDefaults = true) + : this(type, type, dependencies, includeDefaults) { } public AonprdProcessorAttribute(string name, string type) - : this(name, type, ["trait", "source"]) { } + : this(name, type, DefaultDependencies) { } } diff --git a/Pf2eDataScrapper/Importers/Implementations/Aonprd/DataBuilder/AonprdProcessorCollection.cs b/Pf2eDataScrapper/Importers/Implementations/Aonprd/DataBuilder/AonprdProcessorCollection.cs index b1104a1..7f3d24a 100644 --- a/Pf2eDataScrapper/Importers/Implementations/Aonprd/DataBuilder/AonprdProcessorCollection.cs +++ b/Pf2eDataScrapper/Importers/Implementations/Aonprd/DataBuilder/AonprdProcessorCollection.cs @@ -116,6 +116,12 @@ internal class AonprdProcessorCollection public void UseOnlyNames(string[] processorNames) { + if (processorNames.Length == 0) + { + Items = _allItems; + return; + } + Dictionary nameFlags = processorNames.ToDictionary(name => name, _ => false); while (nameFlags.Values.Any(v => !v)) diff --git a/Pf2eDataScrapper/Importers/Implementations/Aonprd/DataBuilder/Helpers/AonprdBuilderGlobals.cs b/Pf2eDataScrapper/Importers/Implementations/Aonprd/DataBuilder/Helpers/AonprdBuilderGlobals.cs index c6a4191..048c6be 100644 --- a/Pf2eDataScrapper/Importers/Implementations/Aonprd/DataBuilder/Helpers/AonprdBuilderGlobals.cs +++ b/Pf2eDataScrapper/Importers/Implementations/Aonprd/DataBuilder/Helpers/AonprdBuilderGlobals.cs @@ -8,5 +8,5 @@ internal class AonprdBuilderGlobals private AonprdBuilderGlobals() { } - public static AonprdBuilderGlobals Instance = new(); + public static readonly AonprdBuilderGlobals Instance = new(); } diff --git a/Pf2eDataScrapper/Importers/Implementations/Aonprd/DataBuilder/Processors/Common/ExternalProcessor.cs b/Pf2eDataScrapper/Importers/Implementations/Aonprd/DataBuilder/Processors/Common/ExternalProcessor.cs index d9a4ad8..a5cdc90 100644 --- a/Pf2eDataScrapper/Importers/Implementations/Aonprd/DataBuilder/Processors/Common/ExternalProcessor.cs +++ b/Pf2eDataScrapper/Importers/Implementations/Aonprd/DataBuilder/Processors/Common/ExternalProcessor.cs @@ -2,7 +2,7 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder.Processors.Common; -[AonprdProcessor("external", null, [])] +[AonprdProcessor("external", null, [], false)] internal class ExternalProcessor(AonprdProcessorInput input) : AbstractProcessor(input) { public ExternalTarget AonprdTarget = null!; diff --git a/Pf2eDataScrapper/Importers/Implementations/Aonprd/DataBuilder/Processors/Common/SkillProcessor.cs b/Pf2eDataScrapper/Importers/Implementations/Aonprd/DataBuilder/Processors/Common/SkillProcessor.cs index 1c8a112..fd67102 100644 --- a/Pf2eDataScrapper/Importers/Implementations/Aonprd/DataBuilder/Processors/Common/SkillProcessor.cs +++ b/Pf2eDataScrapper/Importers/Implementations/Aonprd/DataBuilder/Processors/Common/SkillProcessor.cs @@ -4,7 +4,7 @@ using Pf2eModel.Model.Enum; namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder.Processors.Common; -[AonprdProcessor("skill", [])] +[AonprdProcessor("skill", [], false)] internal class SkillProcessor(AonprdProcessorInput input) : AbstractProcessor(input) { public Training Training = null!; diff --git a/Pf2eDataScrapper/Importers/Implementations/Aonprd/DataBuilder/Processors/Common/SourceProcessor.cs b/Pf2eDataScrapper/Importers/Implementations/Aonprd/DataBuilder/Processors/Common/SourceProcessor.cs index 88a7224..9d25697 100644 --- a/Pf2eDataScrapper/Importers/Implementations/Aonprd/DataBuilder/Processors/Common/SourceProcessor.cs +++ b/Pf2eDataScrapper/Importers/Implementations/Aonprd/DataBuilder/Processors/Common/SourceProcessor.cs @@ -3,7 +3,7 @@ using Pf2eModel.Model.Enum; namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder.Processors.Common; -[AonprdProcessor("source", [])] +[AonprdProcessor("source", [], false)] internal class SourceProcessor(AonprdProcessorInput input) : AbstractProcessor(input) { protected Source Source = null!; diff --git a/Pf2eDataScrapper/Importers/Implementations/Aonprd/DataBuilder/Processors/Types/RuleProcessor.cs b/Pf2eDataScrapper/Importers/Implementations/Aonprd/DataBuilder/Processors/Types/RuleProcessor.cs deleted file mode 100644 index a18371c..0000000 --- a/Pf2eDataScrapper/Importers/Implementations/Aonprd/DataBuilder/Processors/Types/RuleProcessor.cs +++ /dev/null @@ -1,115 +0,0 @@ -using System.Data; -using Microsoft.EntityFrameworkCore.Migrations.Operations; -using Newtonsoft.Json.Linq; -using Pf2eModel.Model.Common.I18n; -using Pf2eModel.Model.Enum; -using Pf2eModel.Model.Structure; -using Rule = Pf2eModel.Model.Structure.Rule; - -namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder.Processors.Types; - -[AonprdProcessor("rules")] -internal class RuleProcessor(AonprdProcessorInput input) : AbstractEntityProcessor(input) -{ - public override EntityType Type => EntityType.Rule; - - public override Rule Object { get; } = new(); - - public override void Add() => Context.Rules.Add(Object); - - public override void Process() - { - base.Process(); - - string[] breadcrumbs = (Input.GetOptional("breadcrumbs") ?? []) - .Select(token => token.ToString()) - .ToArray(); - - RuleStructureItem item = GetStructureItem( - [.. breadcrumbs, Entity.Name[LocaleType.English]] - ); - - item.IsHandled = true; - item.RuleId = Object.Id; - item.Id = Input.GetRequired("id"); - - if (item.Parent?.IsHandled ?? false) - { - Object.ParentId = item.Parent.RuleId!; - // Object.Order = item.Parent.ChildrenOrder?.IndexOf(item.Id) ?? 0; - Object.Order = 0; - } - - item.ChildrenOrder = []; - - List childrenIds = item - .Children.Select(i => i.RuleId) - .Where(id => id != null) - .OfType() - .ToList(); - - List rules = Context.Rules.Where(i => childrenIds.Contains(i.Id)).ToList(); - - foreach (Rule rule in rules) - { - rule.Order = 0; - } - /* - * TODO - * Обработка порядка из полей - * - "next_link": { - "label": "Chapter 3: Classes", - "url": "/Rules.aspx?ID=134" - }, - "previous_link": { - "label": "Chapter 1: Introduction", - "url": "/Rules.aspx?ID=1" - }, - */ - } - - private static readonly List Roots = []; - - private RuleStructureItem GetStructureItem(string[] breadcrumbs, RuleStructureItem? from = null) - { - while (true) - { - List children = from?.Children ?? Roots; - - if (breadcrumbs.Length == 0) - { - return from!; - } - - string root = breadcrumbs[0]; - RuleStructureItem? item = children.FirstOrDefault(i => i.Name == root); - - if (item is null) - { - item = new(root, from); - children.Add(item); - } - - breadcrumbs = breadcrumbs[1..]; - from = item; - } - } -} - -internal class RuleStructureItem(string name, RuleStructureItem? parent) -{ - public string? Id { get; set; } = null; - - public string Name { get; set; } = name; - - public int? RuleId { get; set; } = null; - - public bool IsHandled { get; set; } = false; - - public RuleStructureItem? Parent { get; set; } = parent; - - public List? ChildrenOrder { get; set; } = null; - - public List Children { get; } = []; -} diff --git a/Pf2eDataScrapper/Importers/Implementations/Aonprd/DataBuilder/Processors/Types/Rules/RuleProcessor.cs b/Pf2eDataScrapper/Importers/Implementations/Aonprd/DataBuilder/Processors/Types/Rules/RuleProcessor.cs new file mode 100644 index 0000000..dd64ae3 --- /dev/null +++ b/Pf2eDataScrapper/Importers/Implementations/Aonprd/DataBuilder/Processors/Types/Rules/RuleProcessor.cs @@ -0,0 +1,115 @@ +using System.Data; +using Microsoft.EntityFrameworkCore.Migrations.Operations; +using Newtonsoft.Json.Linq; +using Pf2eModel.Model.Base.Source; +using Pf2eModel.Model.Base.Targets; +using Pf2eModel.Model.Common.I18n; +using Pf2eModel.Model.Enum; +using Pf2eModel.Model.Structure; +using Rule = Pf2eModel.Model.Structure.Rule; + +namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder.Processors.Types.Rules; + +[AonprdProcessor("rules", ["rules-toc"])] +internal class RuleProcessor(AonprdProcessorInput input) : AbstractEntityProcessor(input) +{ + public override EntityType Type => EntityType.Rule; + + public override Rule Object { get; } = new(); + + public override void Add() => Context.Rules.Add(Object); + + public readonly RuleTocCollection Toc = RuleTocCollection.Instance; + + public override void Process() + { + // base.Process(); + AddEntity(); + AddSource(); + AddExternal(); + AddTraits(); + + RuleTocBook book = Toc[GetSource().Id]; + + string[] breadcrumbs = (Input.GetOptional("breadcrumbs") ?? []) + .Select(token => token.ToString()) + .ToArray(); + + string id = GetExternal().Suffix.Replace("/Rules.aspx?ID=", "rules-"); + RuleTocItem? item; + + if (breadcrumbs.Length > 0) + { + RuleTocItem parent = book.Get(breadcrumbs); + item = parent.Children.FirstOrDefault(child => child.Id == id); + + if (item is null) + { + item = new(Entity.Name[LocaleType.English], parent); + parent.Children.Add(item); + } + } + else + { + item = book.Children.First(child => child.Id == id); + } + + item.Id = Input.GetRequired("id"); + item.RuleId = Object.Id; + item.IsHandled = true; + + if (item.Parent?.IsHandled ?? false) + { + Object.ParentId = item.Parent.RuleId!; + Object.Order = item.Parent.ChildrenOrder?.IndexOf(item.Id) ?? -1; + } + + // ЗАМЕТКА + // + // Индекс Архивов не включает поля "next_link" и "previous_link"! + + /* + if (item.ChildrenOrder is not null) + { + return; + } + + item.PreviousId = Input + .GetOptional("previous_link") + ?.Replace("/Rules.aspx?ID=", "rules-"); + item.NextId = Input.GetOptional("next_link")?.Replace("/Rules.aspx?ID=", "rules-"); + + if (item.Parent is not null) + { + if (item.PreviousId is null) + { + item.Parent.FirstChildId = item.Id; + } + if (item.NextId is null) + { + item.Parent.LastChildId = item.Id; + } + } + */ + } + + private Source GetSource() + { + foreach (EntitySource entitySource in Entity.Sources) + { + return entitySource.Source; + } + + throw new("Source not found"); + } + + private EntityExternal GetExternal() + { + foreach (EntityExternal entityExternal in Entity.Externals) + { + return entityExternal; + } + + throw new("External not found"); + } +} diff --git a/Pf2eDataScrapper/Importers/Implementations/Aonprd/DataBuilder/Processors/Types/Rules/RuleToc.cs b/Pf2eDataScrapper/Importers/Implementations/Aonprd/DataBuilder/Processors/Types/Rules/RuleToc.cs new file mode 100644 index 0000000..2db21cc --- /dev/null +++ b/Pf2eDataScrapper/Importers/Implementations/Aonprd/DataBuilder/Processors/Types/Rules/RuleToc.cs @@ -0,0 +1,219 @@ +using System.Linq; +using System.Runtime.CompilerServices; +using LinqToDB.DataProvider.ClickHouse; +using Pf2eModel.Model.Base.Source; + +namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder.Processors.Types.Rules; + +internal class RuleTocCollection +{ + public List Books { get; } = []; + + public RuleTocBook this[int id] => + Books.FirstOrDefault(i => i.SourceId == id) ?? throw new($"Book with id {id} not found"); + + public RuleTocBook this[Source source] + { + get + { + RuleTocBook? result = Books.FirstOrDefault(i => i.SourceId == source.Id); + + if (result is not null) + { + return result; + } + + result = new(source); + Books.Add(result); + return result; + } + } + + public RuleTocBook? FindName(string name) + { + foreach (RuleTocBook book in Books) + { + if (book.IsNameExists(name)) + { + return book; + } + } + + return null; + } + + public RuleTocItem? FindEntityId(int id) + { + foreach (RuleTocBook book in Books) + { + RuleTocItem? result = book.GetByEntityId(id); + + if (result is not null) + { + return result; + } + } + + return null; + } + + public static RuleTocCollection Instance { get; } = new(); + + public override string ToString() + { + return string.Join("\n", Books.Select(book => book.ToString()).ToList()); + } + + public void SaveFile() + { + string fileName = + $"rules-toc.{new DateTimeOffset(DateTime.UtcNow).ToUnixTimeSeconds()}.debug.yaml"; + + using FileStream fileStream = File.Create(fileName); + using StreamWriter writer = new(fileStream); + + writer.Write(ToString()); + } +} + +internal class RuleTocBook(Source source) +{ + public int SourceId { get; } = source.Id; + public string SourceName { get; } = source.Name; + + public List Children { get; } = []; + + public RuleTocItem Get(string[] breadcrumbs) + { + RuleTocItem? node = null; + + while (true) + { + List children = node?.Children ?? Children; + + if (breadcrumbs.Length == 0) + { + return node!; + } + + string root = breadcrumbs[0]; + RuleTocItem? item = children.FirstOrDefault(i => i.Name == root); + + if (item is null) + { + item = new(root, node); + children.Add(item); + } + + breadcrumbs = breadcrumbs[1..]; + node = item; + } + } + + public bool IsNameExists(string name, RuleTocItem? node = null) + { + foreach (RuleTocItem item in node?.Children ?? Children) + { + if (item.Name == name || IsNameExists(name, item)) + { + return true; + } + } + + return false; + } + + public RuleTocItem? GetByEntityId(int id, RuleTocItem? node = null) + { + if (node?.RuleId == id) + { + return node; + } + + foreach (RuleTocItem item in node?.Children ?? Children) + { + RuleTocItem? result = GetByEntityId(id, item); + + if (result is not null) + { + return result; + } + } + + return null; + } + + public override string ToString() + { + return $"\"{SourceName}\":\n" + + string.Join( + "\n", + Children.Select(child => RuleTocItem.Indent(child.ToString())).ToList() + ); + } +} + +internal class RuleTocItem(string name, RuleTocItem? parent) +{ + public string? Id { get; set; } = null; + + public string Name { get; } = name; + + public int? RuleId { get; set; } = null; + + public bool IsHandled { get; set; } = false; + + public RuleTocItem? Parent { get; } = parent; + + public List? ChildrenOrder { get; set; } = null; + + public List Children { get; } = []; + + public string? PreviousId { get; set; } + public string? NextId { get; set; } + + public string? FirstChildId { get; set; } + public string? LastChildId { get; set; } + + public override string ToString() + { + List rows = + [ + $"{Name}:", + $"{Tab}aonprd: {Id ?? "null"}", + $"{Tab}entity: {RuleId?.ToString() ?? "null"}", + ]; + + if (Children.Count == 0) + { + // rows.Add($"{TAB}children: None"); + return string.Join("\n", rows); + } + + rows.Add($"{Tab}children:"); + + foreach (RuleTocItem child in Children) + { + rows.Add(child.Indent(2)); + } + + return string.Join("\n", rows); + } + + public string Indent(int amount = 1) + { + string result = ToString(); + for (int i = 0; i < amount; i++) + { + result = Indent(result); + } + return result; + } + + public static string Indent(string str) + { + return Tab + str.Replace("\n", $"\n{Tab}"); + } + + public const string Tab = " "; +} diff --git a/Pf2eDataScrapper/Importers/Implementations/Aonprd/DataBuilder/Processors/Types/Rules/RulesOrderProcessor.cs b/Pf2eDataScrapper/Importers/Implementations/Aonprd/DataBuilder/Processors/Types/Rules/RulesOrderProcessor.cs new file mode 100644 index 0000000..11d7e53 --- /dev/null +++ b/Pf2eDataScrapper/Importers/Implementations/Aonprd/DataBuilder/Processors/Types/Rules/RulesOrderProcessor.cs @@ -0,0 +1,135 @@ +// ЗАМЕТКА +// +// В теории, этот метод должен был дополнительно устанавливать порядок разделов. +// Но индекс Архивов не включает поля "next_link" и "previous_link", +// так что ставить порядок ВСЕХ правил в архиве не получится, только тех, что есть в содержании. +// Ну а пока просто расставляем родительские и дочерние элементы. + +using System.Data; +using Pf2eModel.Model.Structure; +using Rule = Pf2eModel.Model.Structure.Rule; + +namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder.Processors.Types.Rules; + +[AonprdProcessor("rules-order", null, ["rules"], false)] +internal class RulesOrderProcessor(AonprdProcessorInput input) : AbstractProcessor(input) +{ + public readonly RuleTocCollection Toc = RuleTocCollection.Instance; + + private readonly List OrderItems = []; + + public override void Process() + { + Toc.SaveFile(); + + /* + foreach (RuleTocBook book in Toc.Books) + { + BuildIdOrderDictionary(book.Children); + } + */ + + int count = Context.Rules.Count(); + const int perPage = 50; + + for (int skip = 0; skip < count; skip += perPage) + { + Context.Rules.Skip(skip).Take(perPage).ToList().ForEach(Process); + } + } + + private void Process(Rule rule) + { + RuleTocItem? item = Toc.FindEntityId(rule.Id); + + if (item is null) + { + throw new($"Rule with id {rule.Id} not found in TOC"); + } + + rule.ParentId = item.Parent?.RuleId; + + // Дети будут раставлены автоматически по связи + } + + private void BuildIdOrderDictionary(RuleTocItem item) + { + List items = []; + + if (item.ChildrenOrder is null) + { + RuleTocItem? child = item.Children.FirstOrDefault(i => i.Id == item.FirstChildId); + + while (child is not null) + { + items.Add(child); + child = item.Children.FirstOrDefault(i => i.Id == child.NextId); + } + } + else + { + foreach (string id in item.ChildrenOrder) + { + RuleTocItem? child = item.Children.FirstOrDefault(i => i.Id == id); + if (child is null) + { + throw new($"Child with id {id} not found in {item.Name}"); + } + + items.Add(child); + } + } + + if (items.Count != item.Children.Count) + { + throw new($"Children count mismatch for {item.Name}"); + } + + BuildIdOrderDictionary(items); + } + + private void BuildIdOrderDictionary(List items) + { + for (int i = 0; i < items.Count; i++) + { + RuleTocItem item = items[i]; + + if (item.RuleId is null) + { + throw new($"RuleId is null for {item.Name} (id {item.Id}"); + } + + AddOrderItem((int)item.RuleId!, i, null); + BuildIdOrderDictionary(item); + } + } + + private void AddOrderItem(int id, int order, int? parentId) + { + if (OrderItems.Any(item => item.Id == id)) + { + throw new($"Order item with id {id} already exists"); + } + + OrderItems.Add( + new() + { + Id = id, + Order = order, + ParentId = parentId, + } + ); + } + + public override void Save() + { + Context.SaveChanges(); + } + + private record OrderItem + { + public int Id { get; init; } + public int Order { get; init; } + public int? ParentId { get; init; } + } +} diff --git a/Pf2eDataScrapper/Importers/Implementations/Aonprd/DataBuilder/Processors/Types/Rules/RulesTocProcessor.cs b/Pf2eDataScrapper/Importers/Implementations/Aonprd/DataBuilder/Processors/Types/Rules/RulesTocProcessor.cs new file mode 100644 index 0000000..bf202f3 --- /dev/null +++ b/Pf2eDataScrapper/Importers/Implementations/Aonprd/DataBuilder/Processors/Types/Rules/RulesTocProcessor.cs @@ -0,0 +1,104 @@ +using System.Text.RegularExpressions; +using System.Web; +using AngleSharp; +using AngleSharp.Dom; +using Pf2eModel.Model.Base.Source; + +namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder.Processors.Types.Rules; + +[AonprdProcessor("rules-toc", null, ["source"], false)] +internal partial class RulesTocProcessor(AonprdProcessorInput input) : AbstractProcessor(input) +{ + private const string Selector = "#ctl00_RadDrawer1_Content_MainContent_DetailedOutput"; + private const string Url = "https://2e.aonprd.com/Rules.aspx"; + + public readonly RuleTocCollection Toc = RuleTocCollection.Instance; + + public override void Process() + { + IDocument document = DownloadPage(); + + foreach (IElement block in document.QuerySelector(Selector)!.Children) + { + IElement summaryElement = + block.QuerySelector("summary") ?? throw new("Can't determine element"); + + string sourceName = summaryElement.TextContent; + Source source = + Context.Sources.FirstOrDefault(s => s.Name == sourceName) + ?? throw new($"Source {sourceName} not found"); + + RuleTocBook book = Toc[source]; + + for (int i = 1; i < block.Children.Length; i++) + { + IElement element = block.Children[i]; + + if (element.TagName != "B") + { + throw new($"Expected tag, but got <{element.TagName}>"); + } + + RuleTocItem item = ParseLink(element.InnerHtml); + item.ChildrenOrder = []; + + book.Children.Add(item); + + if (element.NextElementSibling?.TagName != "UL") + { + continue; + } + + IElement ul = block.Children[++i]; + + foreach (IElement li in ul.Children) + { + RuleTocItem child = ParseLink(li.InnerHtml); + + item.Children.Add(child); + item.ChildrenOrder.Add(child.Id!); + } + } + } + + Toc.SaveFile(); + } + + private RuleTocItem ParseLink(string value, RuleTocItem? parent = null) + { + Match linkMatch = LinkRegex().Match(value); + + if (!linkMatch.Success) + { + throw new($"Can't parse link {value}"); + } + + return new(HttpUtility.HtmlDecode(linkMatch.Groups[2].Value).Trim(), parent) + { + Id = $"rules-{linkMatch.Groups[1].Value}", + }; + } + + private IDocument DownloadPage() + { + IConfiguration config = Configuration.Default.WithDefaultLoader(); + IBrowsingContext context = BrowsingContext.New(config); + IDocument document = context.OpenAsync(Url).Result; + + IElement root = document.QuerySelector(Selector) ?? throw new("Broken page download"); + + string html = root.OuterHtml; + + foreach (string s in new string[] { "br", "\r", "\n", "", "", "&NoRedirect=1" }) + { + html = html.Replace(s, ""); + } + + return context.OpenAsync(req => req.Content(html)).Result; + } + + public override void Save() { } + + [GeneratedRegex(@"(.*?)")] + public static partial Regex LinkRegex(); +} diff --git a/Pf2eDataScrapper/Importers/Implementations/Aonprd/DataBuilder/Processors/Types/TraitProcessor.cs b/Pf2eDataScrapper/Importers/Implementations/Aonprd/DataBuilder/Processors/Types/TraitProcessor.cs index 6bbf758..49c6ac6 100644 --- a/Pf2eDataScrapper/Importers/Implementations/Aonprd/DataBuilder/Processors/Types/TraitProcessor.cs +++ b/Pf2eDataScrapper/Importers/Implementations/Aonprd/DataBuilder/Processors/Types/TraitProcessor.cs @@ -3,7 +3,7 @@ using Pf2eModel.Model.Structure; namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder.Processors.Types; -[AonprdProcessor("trait", ["external"])] +[AonprdProcessor("trait", ["external"], false)] internal class TraitProcessor(AonprdProcessorInput input) : AbstractEntityProcessor(input) { public override EntityType Type => EntityType.Trait; diff --git a/Pf2eDataScrapper/Pf2eDataScrapper.csproj b/Pf2eDataScrapper/Pf2eDataScrapper.csproj index 7793be0..a61becb 100644 --- a/Pf2eDataScrapper/Pf2eDataScrapper.csproj +++ b/Pf2eDataScrapper/Pf2eDataScrapper.csproj @@ -7,6 +7,7 @@ enable + diff --git a/Pf2eDataScrapper/Program.cs b/Pf2eDataScrapper/Program.cs index cfdf326..4c6da37 100644 --- a/Pf2eDataScrapper/Program.cs +++ b/Pf2eDataScrapper/Program.cs @@ -11,7 +11,7 @@ Pf2EFoundryImporter importer = new(); AonprdImporter importer = new(); -importer.ProcessorNames = ["rules"]; +// importer.ProcessorNames = ["rules", "rules-order"]; importer.Prepare(); importer.Import(); diff --git a/Pf2eModel/Migrations/20250507112626_1.Designer.cs b/Pf2eModel/Migrations/20250508101742_1.Designer.cs similarity index 99% rename from Pf2eModel/Migrations/20250507112626_1.Designer.cs rename to Pf2eModel/Migrations/20250508101742_1.Designer.cs index d96c48a..bb35ad4 100644 --- a/Pf2eModel/Migrations/20250507112626_1.Designer.cs +++ b/Pf2eModel/Migrations/20250508101742_1.Designer.cs @@ -13,7 +13,7 @@ using Pf2eModel.Model.Enum; namespace Pf2eModel.Migrations { [DbContext(typeof(Pf2eDbContext))] - [Migration("20250507112626_1")] + [Migration("20250508101742_1")] partial class _1 { /// @@ -236,22 +236,30 @@ namespace Pf2eModel.Migrations modelBuilder.Entity("Pf2eModel.Model.Base.Source.EntitySource", b => { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id") + .HasColumnOrder(0); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + b.Property("EntityId") .HasColumnType("integer") .HasColumnName("entity_id") .HasColumnOrder(1); - b.Property("SourceId") - .HasColumnType("integer") - .HasColumnName("source_id") - .HasColumnOrder(2); - b.Property("IsRemaster") .HasColumnType("boolean") .HasColumnName("is_remaster") .HasColumnOrder(4) .HasComment("Флаг, указывающий, что объект является частью ремастера"); + b.Property("SourceId") + .HasColumnType("integer") + .HasColumnName("source_id") + .HasColumnOrder(2); + b.Property("Suffix") .IsRequired() .HasMaxLength(255) @@ -260,9 +268,12 @@ namespace Pf2eModel.Migrations .HasColumnOrder(3) .HasComment("Номер страницы, эррата и т.д."); - b.HasKey("EntityId", "SourceId") + b.HasKey("Id") .HasName("pk_entity_sources"); + b.HasIndex("EntityId") + .HasDatabaseName("ix_entity_sources_entity_id"); + b.HasIndex("SourceId") .HasDatabaseName("ix_entity_sources_source_id"); diff --git a/Pf2eModel/Migrations/20250507112626_1.cs b/Pf2eModel/Migrations/20250508101742_1.cs similarity index 99% rename from Pf2eModel/Migrations/20250507112626_1.cs rename to Pf2eModel/Migrations/20250508101742_1.cs index 496b5da..2cf0e26 100644 --- a/Pf2eModel/Migrations/20250507112626_1.cs +++ b/Pf2eModel/Migrations/20250508101742_1.cs @@ -469,6 +469,8 @@ namespace Pf2eModel.Migrations name: "entity_sources", columns: table => new { + id = table.Column(type: "integer", nullable: false) + .Annotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn), entity_id = table.Column(type: "integer", nullable: false), source_id = table.Column(type: "integer", nullable: false), suffix = table.Column(type: "character varying(255)", maxLength: 255, nullable: false, comment: "Номер страницы, эррата и т.д."), @@ -476,7 +478,7 @@ namespace Pf2eModel.Migrations }, constraints: table => { - table.PrimaryKey("pk_entity_sources", x => new { x.entity_id, x.source_id }); + table.PrimaryKey("pk_entity_sources", x => x.id); table.ForeignKey( name: "fk_entity_sources_entities_entity_id", column: x => x.entity_id, @@ -1260,6 +1262,11 @@ namespace Pf2eModel.Migrations table: "entity_keyword", column: "keywords_id"); + migrationBuilder.CreateIndex( + name: "ix_entity_sources_entity_id", + table: "entity_sources", + column: "entity_id"); + migrationBuilder.CreateIndex( name: "ix_entity_sources_source_id", table: "entity_sources", diff --git a/Pf2eModel/Migrations/Pf2eDbContextModelSnapshot.cs b/Pf2eModel/Migrations/Pf2eDbContextModelSnapshot.cs index 07100d7..6e03cb4 100644 --- a/Pf2eModel/Migrations/Pf2eDbContextModelSnapshot.cs +++ b/Pf2eModel/Migrations/Pf2eDbContextModelSnapshot.cs @@ -233,22 +233,30 @@ namespace Pf2eModel.Migrations modelBuilder.Entity("Pf2eModel.Model.Base.Source.EntitySource", b => { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id") + .HasColumnOrder(0); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + b.Property("EntityId") .HasColumnType("integer") .HasColumnName("entity_id") .HasColumnOrder(1); - b.Property("SourceId") - .HasColumnType("integer") - .HasColumnName("source_id") - .HasColumnOrder(2); - b.Property("IsRemaster") .HasColumnType("boolean") .HasColumnName("is_remaster") .HasColumnOrder(4) .HasComment("Флаг, указывающий, что объект является частью ремастера"); + b.Property("SourceId") + .HasColumnType("integer") + .HasColumnName("source_id") + .HasColumnOrder(2); + b.Property("Suffix") .IsRequired() .HasMaxLength(255) @@ -257,9 +265,12 @@ namespace Pf2eModel.Migrations .HasColumnOrder(3) .HasComment("Номер страницы, эррата и т.д."); - b.HasKey("EntityId", "SourceId") + b.HasKey("Id") .HasName("pk_entity_sources"); + b.HasIndex("EntityId") + .HasDatabaseName("ix_entity_sources_entity_id"); + b.HasIndex("SourceId") .HasDatabaseName("ix_entity_sources_source_id"); diff --git a/Pf2eModel/Model/Base/Source/EntitySource.cs b/Pf2eModel/Model/Base/Source/EntitySource.cs index 6a6143f..df3daa5 100644 --- a/Pf2eModel/Model/Base/Source/EntitySource.cs +++ b/Pf2eModel/Model/Base/Source/EntitySource.cs @@ -4,9 +4,13 @@ using Microsoft.EntityFrameworkCore; namespace Pf2eModel.Model.Base.Source; -[PrimaryKey(nameof(EntityId), nameof(SourceId))] public class EntitySource { + [Key] + [DatabaseGenerated(DatabaseGeneratedOption.Identity)] + [Column(Order = 0)] + public int Id { get; set; } + [Column(Order = 1)] public int EntityId { get; set; } diff --git a/Pf2eModel/Pf2eDbContext.cs b/Pf2eModel/Pf2eDbContext.cs index 8e05de5..8b37bd2 100644 --- a/Pf2eModel/Pf2eDbContext.cs +++ b/Pf2eModel/Pf2eDbContext.cs @@ -125,6 +125,7 @@ public partial class Pf2eDbContext : DbContext // optionsBuilder.LogTo(Console.WriteLine); optionsBuilder.UseSnakeCaseNamingConvention(); + optionsBuilder.EnableSensitiveDataLogging(); optionsBuilder.UseNpgsql( databaseConnection, builder =>