kozhilya/Pf2eDataScrapper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Глобальное обновление #1

Merged
kozhilya merged 33 commits from dev into master 2025-06-01 22:28:15 +04:00
Conversation 0 Commits 33 Files Changed 1396
+921 -601
64 changed files with 921 additions and 601 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 34be3a5fc4 - Show all commits
Pf2eDataScrapper/App/ProcessorAttribute.cs
-79
View File
@@ -1,79 +0,0 @@
using System;
using System.Collections.Generic;
namespace Pf2eDataScrapper.App;
/// <summary>
/// Общий аттрибут обработчика.
/// </summary>
/// <param name="name">Название обработчика.</param>
[AttributeUsage(AttributeTargets.Class)]
public class ProcessorAttribute(string name) : Attribute
{
/// <summary>
/// Название обработчика.
/// </summary>
public string Name { get; init; } = name;
/// <summary>
/// Описание обработчика.
/// </summary>
public string Description { get; init; } = name;
/// <summary>
/// Группа обработчиков, к которому относится обработчик.
/// </summary>
public ProcessorGroupEnum Group { get; init; } = ProcessorGroupEnum.General;
/// <summary>
/// Процессор включён по умолчанию.
/// </summary>
public bool Active { get; init; } = true;
/// <summary>
/// Разрешить выполнять процессов в параллельном режиме
/// </summary>
public bool Parallel { get; init; } = false;
/// <summary>
/// Обработчик набора элементов
/// </summary>
public virtual bool Collection { get; } = false;
}
/// <summary>
/// Аттрибут обработчика набор элементов.
/// </summary>
/// <param name="name">Название обработчика.</param>
[AttributeUsage(AttributeTargets.Class)]
public class ProcessorCollectionAttribute(string name) : ProcessorAttribute(name)
{
/// <summary>
/// Название типов, который обрабатывает этот оброботчик.
/// <c>null</c> - обработать все типы.
/// </summary>
public required string? InnerType { get; init; }
/// <summary>
/// Тип используемого компендиума.
/// </summary>
public required ProcessorCompendiumEnum CompendiumType { get; init; }
/// <summary>
/// Обработчик набора элементов.
/// </summary>
public override bool Collection { get; } = true;
}
/// <summary>
/// Аттрибут зависимостей обработчика.
/// </summary>
/// <param name="dependencies">Коллекция названий обработчиков, которые должны быть выполнены до этого оброботчика.</param>
[AttributeUsage(AttributeTargets.Class)]
public class ProcessorDependenciesAttribute(string[] dependencies) : Attribute
{
/// <summary>
/// Коллекция названий обработчиков, которые должны быть выполнены до этого оброботчика.
/// </summary>
public ICollection<string> Dependencies { get; } = dependencies;
}
Pf2eDataScrapper/App/ProcessorCollection.cs
-261
View File
@@ -1,261 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Reflection;
using Pf2eDataScrapper.Processors;
namespace Pf2eDataScrapper.App;
/// <summary>
/// Класс с коллекцией обработчиков.
/// </summary>
internal class ProcessorCollection
{
/// <summary>
/// Коллекция всех доступных обработчиков.
/// </summary>
public ICollection<ProcessorCollectionItem> AllItems { get; } = [];
/// <summary>
/// Коллекция обработчиков, которые будут обрабатываться.
/// Для изменения необходимо использовать <see cref="FilterProcessors"/>.
/// </summary>
public List<ProcessorCollectionItem> Items { get; private set; } = [];
/// <summary>
/// Обеспечение доступа к элементам обработчиков.
/// </summary>
/// <param name="name"></param>
/// <returns></returns>
public ProcessorCollectionItem this[string name] => AllItems.First(item => item.Name == name);
/// <summary>
/// Конструктор класса <see cref="ProcessorCollection"/>.
/// </summary>
public ProcessorCollection()
{
Assembly assembly = Assembly.GetExecutingAssembly();
foreach (Type type in assembly.GetTypes())
{
if (type.GetCustomAttributes(typeof(ProcessorAttribute), true).Length <= 0)
{
continue;
}
AllItems.Add(CreateItem(type));
}
CheckDependencies();
FilterProcessors([]);
}
/// <summary>
/// Создание элемента коллекции обработчиков.
/// </summary>
/// <param name="type">Тип обработчика.</param>
/// <returns></returns>
private ProcessorCollectionItem CreateItem(Type type)
{
if (!type.IsAssignableTo(typeof(IProcessor)))
{
throw new(
$"'ProcessorAttribute' attribute should only be applied to children of 'IProcessor' class. '{type.FullName}' is not."
);
}
ProcessorAttribute processorAttribute = type.GetCustomAttribute<ProcessorAttribute>()!;
ProcessorDependenciesAttribute? dependenciesAttribute =
type.GetCustomAttribute<ProcessorDependenciesAttribute>();
bool isCollection = processorAttribute.Collection;
string? innerType = null;
ProcessorCompendiumEnum? compendium = null;
if (processorAttribute.GetType() == typeof(ProcessorCollectionAttribute))
{
ProcessorCollectionAttribute processorCollectionAttribute =
(ProcessorCollectionAttribute)processorAttribute;
innerType = processorCollectionAttribute.InnerType;
compendium = processorCollectionAttribute.CompendiumType;
}
return new()
{
Name = processorAttribute.Name,
Type = type,
Description = processorAttribute.Description,
Group = processorAttribute.Group,
Dependencies = dependenciesAttribute?.Dependencies ?? [],
IsCollection = isCollection,
InnerType = innerType,
Compendium = compendium,
Active = processorAttribute.Active,
Parallel = processorAttribute.Parallel,
};
}
/// <summary>
/// Проверка, что все зависимости всех обработчиков могут быть удовлетворены.
/// </summary>
private void CheckDependencies()
{
foreach (ProcessorCollectionItem item in Items)
{
foreach (string dependency in item.Dependencies)
{
if (Items.All(i => i.Name != dependency))
{
throw new($"Dependency '{dependency}' for processor '{item.Name}' not found.");
}
}
}
}
/// <summary>
/// Сортировать обработчики в соответсвии со всеми зависимостями.
/// </summary>
public void SortItems()
{
Items.ForEach(item => item.Priority = -1);
Dictionary<string, int> priorities = [];
while (priorities.Count < Items.Count)
{
foreach (ProcessorCollectionItem item in Items)
{
if (item.Priority >= 0)
{
continue;
}
IEnumerable<int> dependencyPriorities = item.Dependencies.Select(d =>
priorities.GetValueOrDefault(d, -1)
);
if (dependencyPriorities.Any(p => p == -1))
{
continue;
}
item.Priority = Math.Max(
dependencyPriorities.Any() ? dependencyPriorities.Max() + 1 : 1,
(int)item.Group * 100
);
priorities.Add(item.Name, item.Priority);
}
}
Items.Sort((a, b) => a.Priority - b.Priority);
}
/// <summary>
/// Ограничить список используемых обработчиков по названию.
/// Автоматически включает все обработчики, от которых зависят указанные в аргументе обработчики.
/// </summary>
/// <param name="processorNames">Коллекция названиий требуемых обработчиков.</param>
public void FilterProcessors(ICollection<string> processorNames)
{
if (processorNames.Count == 0)
{
Items = [.. AllItems];
SortItems();
return;
}
Dictionary<string, bool> nameFlags = processorNames.ToDictionary(name => name, _ => false);
while (nameFlags.Values.Any(v => !v))
{
foreach (
KeyValuePair<string, bool> pair in nameFlags.Where(pair => !pair.Value).ToList()
)
{
ProcessorCollectionItem item = AllItems.First(item => item.Name == pair.Key);
foreach (string dependency in item.Dependencies)
{
nameFlags.TryAdd(dependency, false);
}
nameFlags[pair.Key] = true;
}
}
Items = AllItems.Where(item => nameFlags.ContainsKey(item.Name)).ToList();
}
/// <summary>
/// Получить список типов, которые обрабатываются в компендиуме.
/// </summary>
/// <param name="compendium">Тип компендиума.</param>
/// <returns>Набор типов.</returns>
public ICollection<string> GetTypesOfCompendium(ProcessorCompendiumEnum compendium) =>
Items.Select(item => item.InnerType).OfType<string>().ToList();
}
/// <summary>
/// Элемент коллекции обработчиков.
/// </summary>
internal class ProcessorCollectionItem
{
/// <summary>
/// Название обработчика.
/// </summary>
public required string Name { get; init; }
/// <summary>
/// Тип класса обработчика.
/// </summary>
public required Type Type { get; init; }
/// <summary>
/// Описание обработчика.
/// </summary>
public string Description { get; init; } = "";
/// <summary>
/// Группа обработчика.
/// </summary>
public required ProcessorGroupEnum Group { get; init; }
/// <summary>
/// Коллекция названий обработчиков, после которых должен выполняться обработчик.
/// </summary>
public ICollection<string> Dependencies { get; init; } = [];
/// <summary>
/// Процессор включён по умолчанию.
/// </summary>
public bool Active { get; init; } = true;
/// <summary>
/// Приоритет выполнения обработчика.
/// </summary>
public int Priority { get; set; } = -1;
/// <summary>
/// Флаг, указывающий, обрабатывает ли обработчик коллекцию элементов.
/// Внутренний тип элементов указывается в <see cref="InnerType"/>.
/// </summary>
public required bool IsCollection { get; init; }
/// <summary>
/// Флаг, указывающий внутренний тип коллекции обрабатываемых элементов.
/// </summary>
public string? InnerType { get; init; }
/// <summary>
/// Название типов, который обрабатывает этот оброботчик.
/// <c>null</c> - обработать все типы.
/// </summary>
public ProcessorCompendiumEnum? Compendium { get; init; }
/// <summary>
/// Разрешить выполнять процессов в параллельном режиме
/// </summary>
public bool Parallel { get; init; } = true;
}
Pf2eDataScrapper/Helpers/EntityView.cs
+1
View File
@@ -91,6 +91,7 @@ public class EntityView
.Include(c => c.ClassTrainings) .Include(c => c.ClassTrainings)
.ThenInclude(t => t.Training) .ThenInclude(t => t.Training)
.Include(c => c.ClassProgressions) .Include(c => c.ClassProgressions)
.Include(c => c.SubclassGroups)
), ),
new Getter<Feat>( new Getter<Feat>(
EntityType.Feat, EntityType.Feat,
Pf2eDataScrapper/Helpers/ProcessProgress.cs
+1 -1
View File
@@ -1,4 +1,4 @@
using Pf2eDataScrapper.Processors; using Pf2eDataScrapper.Processors.Definitions;
namespace Pf2eDataScrapper.Helpers; namespace Pf2eDataScrapper.Helpers;
Pf2eDataScrapper/ImportApplication.cs
+2 -3
View File
@@ -2,10 +2,9 @@
using System.ComponentModel; using System.ComponentModel;
using System.Windows; using System.Windows;
using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection;
using Pf2eDataScrapper.App;
using Pf2eDataScrapper.Helpers; using Pf2eDataScrapper.Helpers;
using Pf2eDataScrapper.Processors; using Pf2eDataScrapper.Processors;
using Pf2eDataScrapper.WPF; using Pf2eDataScrapper.Processors.Collection;
using Pf2eDataScrapper.WPF.EntityViewer; using Pf2eDataScrapper.WPF.EntityViewer;
using Pf2eDataScrapper.WPF.ProgressForm; using Pf2eDataScrapper.WPF.ProgressForm;
using Pf2eDataScrapper.WPF.StartForm; using Pf2eDataScrapper.WPF.StartForm;
@@ -102,7 +101,7 @@ internal sealed class ImportApplication : Application
_entityForm = ActivatorUtilities.CreateInstance<EntityForm>(ServiceProvider); _entityForm = ActivatorUtilities.CreateInstance<EntityForm>(ServiceProvider);
_entityForm.WindowStartupLocation = WindowStartupLocation.CenterScreen; _entityForm.WindowStartupLocation = WindowStartupLocation.CenterScreen;
_entityForm.Closing += Wpf_HandleFormClosing; _entityForm.Closing += Wpf_HandleFormClosing;
//_entityForm.ApplicationStateChange += ApplicationStateChange; _entityForm.ApplicationStateChange += ApplicationStateChange;
_entityForm.Show(); _entityForm.Show();
CurrentWpfForm = _entityForm; CurrentWpfForm = _entityForm;
Pf2eDataScrapper/Importers/Implementations/Aonprd/Helpers/AonprdBuilderGlobals.cs
+43 -84
View File
@@ -1,4 +1,7 @@
using System.Collections.Generic; using System.Collections.Generic;
using System.IO;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using Pf2eModel.Model.Base.Targets; using Pf2eModel.Model.Base.Targets;
namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers; namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers;
@@ -16,108 +19,64 @@ internal class AonprdBuilderGlobals
/// <summary> /// <summary>
/// Словарь идентификаторов источников. /// Словарь идентификаторов источников.
/// </summary> /// </summary>
public Dictionary<string, int> SourceIds { get; set; } = []; public Dictionary<string, int> SourceIds { get; } = [];
/// <summary> /// <summary>
/// Словарь идентификаторов признаков. /// Словарь идентификаторов признаков.
/// </summary> /// </summary>
public Dictionary<string, int> TraitIds { get; set; } = []; public Dictionary<string, int> TraitIds { get; } = [];
/// <summary>
/// Конструктор класса <see cref="AonprdBuilderGlobals"/>.
/// </summary>
public AonprdBuilderGlobals() { }
/// <summary>
/// Получить суффикс для ссылки.
/// </summary>
/// <param name="id">Идентификатор Aounprd.</param>
/// <returns>Суффикс для ссылки или пустая строка.</returns>
public string GetSlugSuffix(string id)
{
return SlugSuffixes.TryGetValue(id, out var slug) ? $"-{slug}" : string.Empty;
}
/// <summary> /// <summary>
/// Словарей суффиксов для сущностей /// Словарей суффиксов для сущностей
/// </summary> /// </summary>
public Dictionary<string, string> SlugSuffixes { get; } = public Dictionary<string, string> SlugSuffixes { get; } = [];
new()
{
["feat-5997"] = "legacy",
["feat-5730"] = "dragonblood",
["feat-6469"] = "general",
["feat-4344"] = "archetype",
["feat-3549"] = "vampire",
["feat-7380"] = "mythic",
["feat-5737"] = "dragonblood",
["feat-6284"] = "bounty-hunter",
["feat-2664"] = "archetype",
["feat-6883"] = "kitsune",
["feat-6209"] = "investigator",
["feat-6477"] = "undead-slayer",
["feat-7637"] = "dandy",
["feat-7004"] = "yaoguai",
["feat-4380"] = "stonebound",
};
/// <summary> /// <summary>
/// Словарь допольнительных проверок связей. /// Словарь допольнительных проверок связей.
/// Ключ - устаревшая сущность, значение - обновлённая. /// Ключ - устаревшая сущность, значение - обновлённая.
/// </summary> /// </summary>
public Dictionary<string, string> LegacyRemasterExtra { get; } = public Dictionary<string, string> LegacyRemasterExtra { get; } = [];
new()
{
["archetype-83"] = "archetype-129",
["archetype-86"] = "archetype-198",
["feat-5997"] = "feat-7108",
["feat-4088"] = "feat-5491",
["feat-1177"] = "feat-4553",
["feat-2986"] = "feat-4340",
["feat-1753"] = "feat-6031",
["feat-2729"] = "feat-4077",
["feat-480"] = "feat-4850",
["feat-2989"] = "feat-4343",
["feat-2987"] = "feat-4341",
["feat-2990"] = "feat-4346",
["feat-936"] = "feat-2269",
["feat-2992"] = "feat-4350",
["feat-2991"] = "feat-4349",
["feat-2742"] = "feat-4078",
["feat-1169"] = "feat-5055",
["feat-2743"] = "feat-4079",
["feat-963"] = "feat-4519",
["feat-1171"] = "feat-4858",
["feat-1167"] = "feat-5880",
["feat-455"] = "feat-6007",
["feat-2993"] = "feat-4352",
["language-91"] = "language-180",
["ritual-31"] = "ritual-191",
["spell-571"] = "spell-1975",
["spell-1052"] = "spell-1309",
["spell-1053"] = "spell-1306",
["spell-843"] = "spell-2376",
["spell-578"] = "spell-1566",
["spell-551"] = "spell-2003",
["spell-553"] = "spell-2012",
["spell-646"] = "spell-1394",
["spell-1054"] = "spell-1305",
["spell-559"] = "spell-2417",
["spell-1055"] = "spell-1311",
["spell-1056"] = "spell-1310",
["spell-841"] = "spell-1107",
["spell-1057"] = "spell-1307",
["spell-560"] = "spell-2418",
["spell-1058"] = "spell-1304",
["spell-641"] = "spell-1413",
["spell-1059"] = "spell-1308",
};
/// <summary> /// <summary>
/// Словарь сущностей, которые нужно слить с другой/другими. /// Словарь сущностей, которые нужно слить с другой/другими.
/// Ключ - пропускаемая сущность. Значение - целевая сущность. /// Ключ - пропускаемая сущность. Значение - целевая сущность.
/// </summary> /// </summary>
public Dictionary<string, string> MergeSources { get; } = new() { ["spell-654"] = "spell-819" }; public Dictionary<string, string> MergeSources { get; } = [];
/// <summary>
/// Конструктор класса <see cref="AonprdBuilderGlobals"/>.
/// </summary>
public AonprdBuilderGlobals()
{
using StreamReader fileReader = new("Resources\\ImportAdaptation.json");
JObject? json = JObject
.Parse(fileReader.ReadToEnd())
.SelectToken("Aonprd")
?.ToObject<JObject>();
if (json is not null)
{
SlugSuffixes = JsonToDictionary(json, "SlugSuffixes");
LegacyRemasterExtra = JsonToDictionary(json, "LegacyRemasterExtra");
MergeSources = JsonToDictionary(json, "MergeSources");
}
}
/// <summary>
/// Получить суффикс для ссылки.
/// </summary>
/// <param name="id">Идентификатор Aonprd.</param>
/// <returns>Суффикс для ссылки или пустая строка.</returns>
public string GetSlugSuffix(string id)
{
return SlugSuffixes.TryGetValue(id, out string? slug) ? $"-{slug}" : string.Empty;
}
private Dictionary<string, string> JsonToDictionary(JObject json, string tokenPath)
{
return json.SelectToken(tokenPath)?.ToObject<Dictionary<string, string>>() ?? [];
}
} }
/* /*
Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/AbstractEntityProcessor.cs
+15 -1
View File
@@ -2,10 +2,11 @@
using System.Linq; using System.Linq;
using Newtonsoft.Json; using Newtonsoft.Json;
using Newtonsoft.Json.Linq; using Newtonsoft.Json.Linq;
using Pf2eDataScrapper.App;
using Pf2eDataScrapper.Helpers; using Pf2eDataScrapper.Helpers;
using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers; using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers;
using Pf2eDataScrapper.Processors; using Pf2eDataScrapper.Processors;
using Pf2eDataScrapper.Processors.Attributes;
using Pf2eDataScrapper.Processors.Definitions;
using Pf2eModel; using Pf2eModel;
using Pf2eModel.Model; using Pf2eModel.Model;
using Pf2eModel.Model.Base; using Pf2eModel.Model.Base;
@@ -60,6 +61,8 @@ internal abstract class AbstractEntityProcessor(
/// <inheritdoc/> /// <inheritdoc/>
public override void Process() public override void Process()
{ {
Check();
AddEntity(); AddEntity();
AddSource(); AddSource();
AddExternal(); AddExternal();
@@ -73,6 +76,17 @@ internal abstract class AbstractEntityProcessor(
Context.SaveChanges(); Context.SaveChanges();
} }
/// <summary>
/// Проверки необходимости импорта сущности.
/// </summary>
public virtual void Check()
{
if (Globals.MergeSources.ContainsKey(Input.GetRequired<string>("id")))
{
throw new SkipItemException(Input.GetRequired<string>("name"), "помечено на слияние.");
}
}
#region "Генерация описания" #region "Генерация описания"
/// <summary> /// <summary>
Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/AbstractSubclassProcessor.cs
+1 -1
View File
@@ -1,8 +1,8 @@
using System.Linq; using System.Linq;
using Pf2eDataScrapper.App;
using Pf2eDataScrapper.Helpers; using Pf2eDataScrapper.Helpers;
using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers; using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers;
using Pf2eDataScrapper.Processors; using Pf2eDataScrapper.Processors;
using Pf2eDataScrapper.Processors.Attributes;
using Pf2eModel; using Pf2eModel;
using Pf2eModel.Model.Base; using Pf2eModel.Model.Base;
using Pf2eModel.Model.Common.I18n; using Pf2eModel.Model.Common.I18n;
Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Common/ExternalProcessor.cs
+4 -3
View File
@@ -1,6 +1,7 @@
using Pf2eDataScrapper.App; using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers;
using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers;
using Pf2eDataScrapper.Processors; using Pf2eDataScrapper.Processors;
using Pf2eDataScrapper.Processors.Attributes;
using Pf2eDataScrapper.Processors.Definitions;
using Pf2eModel; using Pf2eModel;
using Pf2eModel.Model.Base.Targets; using Pf2eModel.Model.Base.Targets;
@@ -12,7 +13,7 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Common;
Group = ProcessorGroupEnum.AonprdPreparation Group = ProcessorGroupEnum.AonprdPreparation
)] )]
[ProcessorDependencies(["aonprd:repository"])] [ProcessorDependencies(["aonprd:repository"])]
internal class ExternalProcessor(AonprdBuilderGlobals globals) : AbstractProcessor internal class ExternalProcessor(AonprdBuilderGlobals globals) : AbstractSimpleProcessor
{ {
public Pf2eDbContext Context { get; } = new(); public Pf2eDbContext Context { get; } = new();
public AonprdBuilderGlobals Globals { get; } = globals; public AonprdBuilderGlobals Globals { get; } = globals;
Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Common/SkillProcessor.cs
+3 -2
View File
@@ -1,5 +1,6 @@
using Pf2eDataScrapper.App; using Pf2eDataScrapper.Processors;
using Pf2eDataScrapper.Processors; using Pf2eDataScrapper.Processors.Attributes;
using Pf2eDataScrapper.Processors.Definitions;
using Pf2eModel; using Pf2eModel;
using Pf2eModel.Model.Common; using Pf2eModel.Model.Common;
using Pf2eModel.Model.Common.I18n; using Pf2eModel.Model.Common.I18n;
Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Common/SourceProcessor.cs
+2 -1
View File
@@ -1,7 +1,8 @@
using System; using System;
using Pf2eDataScrapper.App;
using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers; using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers;
using Pf2eDataScrapper.Processors; using Pf2eDataScrapper.Processors;
using Pf2eDataScrapper.Processors.Attributes;
using Pf2eDataScrapper.Processors.Definitions;
using Pf2eModel; using Pf2eModel;
using Pf2eModel.Model.Base.Source; using Pf2eModel.Model.Base.Source;
using Pf2eModel.Model.Enum; using Pf2eModel.Model.Enum;
Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Finalization/LegacyRemasterProcessors.cs
+6 -13
View File
@@ -1,10 +1,10 @@
using System; using System.Collections.Generic;
using System.Collections.Generic;
using System.Linq; using System.Linq;
using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore;
using Newtonsoft.Json.Linq; using Newtonsoft.Json.Linq;
using Pf2eDataScrapper.App;
using Pf2eDataScrapper.Processors; using Pf2eDataScrapper.Processors;
using Pf2eDataScrapper.Processors.Attributes;
using Pf2eDataScrapper.Processors.Definitions;
using Pf2eModel; using Pf2eModel;
using Pf2eModel.Model.Base; using Pf2eModel.Model.Base;
using Pf2eModel.Model.Base.Targets; using Pf2eModel.Model.Base.Targets;
@@ -21,7 +21,7 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Finalizat
)] )]
internal class LegacyRemasterProcessors(ProcessorInput input) : AbstractInputProcessor(input) internal class LegacyRemasterProcessors(ProcessorInput input) : AbstractInputProcessor(input)
{ {
private Pf2eDbContext Context { get; set; } = new(); private Pf2eDbContext Context { get; } = new();
public override void Process() public override void Process()
{ {
@@ -32,27 +32,20 @@ internal class LegacyRemasterProcessors(ProcessorInput input) : AbstractInputPro
int entityId = entity.EntityId; int entityId = entity.EntityId;
Entity legacyEntity = Entity legacyEntity =
Context.Entities.FirstOrDefault(e => e.Id == entityId) Context.Entities.FirstOrDefault(e => e.Id == entityId)
?? throw new Exception($"Неизвестная сущность с ID {entityId}"); ?? throw new($"Неизвестная сущность с ID {entityId}");
foreach (int remasterId in GetEntityIds("remaster_id")) foreach (int remasterId in GetEntityIds("remaster_id"))
{ {
Context.LegacyRemasters.Add(new() { LegacyId = entityId, RemasterId = remasterId }); Context.LegacyRemasters.Add(new() { LegacyId = entityId, RemasterId = remasterId });
Entity remasterEntity = Entity remasterEntity =
Context.Entities.AsNoTracking().FirstOrDefault(e => e.Id == remasterId) Context.Entities.AsNoTracking().FirstOrDefault(e => e.Id == remasterId)
?? throw new Exception($"Неизвестная сущность с ID {remasterId}"); ?? throw new($"Неизвестная сущность с ID {remasterId}");
if (legacyEntity.Slug == remasterEntity.Slug) if (legacyEntity.Slug == remasterEntity.Slug)
{ {
legacyEntity.Slug += "-legacy"; legacyEntity.Slug += "-legacy";
} }
} }
/*
foreach (int legacyId in GetEntityIds("legacy_id"))
{
Context.LegacyRemasters.Add(new() { LegacyId = legacyId, RemasterId = entityId });
}
*/
} }
protected List<int> GetEntityIds(string path) protected List<int> GetEntityIds(string path)
Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Finalization/MissingLegacyRemasterProcessorFactory.cs
+88
View File
@@ -0,0 +1,88 @@
using System;
using System.Collections.Generic;
using System.Linq;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers;
using Pf2eDataScrapper.Processors;
using Pf2eDataScrapper.Processors.Attributes;
using Pf2eDataScrapper.Processors.Definitions;
using Pf2eDataScrapper.Processors.Factory;
using Pf2eModel;
using Pf2eModel.Model.Base;
using Pf2eModel.Model.Base.Targets;
namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Finalization;
[ProcessorFactory(
"aonprd:fix-legacy-remaster",
ProcessorType = typeof(MissingLegacyRemasterProcessor),
Description = "Дополнение связяй устаревших и обновлённых сущностей",
Group = ProcessorGroupEnum.AonprdFinalization,
Parallel = true
)]
[ProcessorDependencies(["aonprd:legacy-remaster"])]
internal class MissingLegacyRemasterProcessorFactory(
IServiceProvider serviceProvider,
AonprdBuilderGlobals globals
) : AbstractProcessorFactory
{
public IServiceProvider ServiceProvider { get; } = serviceProvider;
public AonprdBuilderGlobals Globals { get; } = globals;
public override int Count() => Globals.LegacyRemasterExtra.Count;
public override IEnumerable<Tuple<string, IProcessor>> GetEnumerator()
{
foreach ((string key, string value) in Globals.LegacyRemasterExtra)
{
yield return new(
$"{key} -> {value}",
ActivatorUtilities.CreateInstance<MissingLegacyRemasterProcessor>(
ServiceProvider,
key,
value
)
);
}
}
}
internal class MissingLegacyRemasterProcessor(string legacyId, string remasterId) : IProcessor
{
public Pf2eDbContext Context { get; } = new();
public string LegacyId { get; } = legacyId;
public string RemasterId { get; } = remasterId;
private Entity GetEntity(string aonprdId)
{
EntityExternal entity =
Context.EntityExternals.AsNoTracking().FirstOrDefault(x => x.ExternalId == aonprdId)
?? throw new SilentSkipItemException();
int entityId = entity.EntityId;
return Context.Entities.FirstOrDefault(e => e.Id == entityId)
?? throw new($"Неизвестная сущность с ID {entityId}");
}
public void Process()
{
Entity legacyEntity = GetEntity(LegacyId);
Entity remasterEntity = GetEntity(RemasterId);
Context.LegacyRemasters.Add(
new() { LegacyId = legacyEntity.Id, RemasterId = remasterEntity.Id }
);
if (legacyEntity.Slug == remasterEntity.Slug)
{
legacyEntity.Slug += "-legacy";
}
}
public void Save()
{
Context.SaveChanges();
}
}
Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Preparation/CompendiumLoadProcessor.cs
+2 -2
View File
@@ -1,9 +1,9 @@
using System.IO; using System.IO;
using Newtonsoft.Json.Linq; using Newtonsoft.Json.Linq;
using Pf2eDataScrapper.App;
using Pf2eDataScrapper.Helpers; using Pf2eDataScrapper.Helpers;
using Pf2eDataScrapper.Importers.Implementations.Aonprd.Compendium; using Pf2eDataScrapper.Importers.Implementations.Aonprd.Compendium;
using Pf2eDataScrapper.Processors; using Pf2eDataScrapper.Processors.Attributes;
using Pf2eDataScrapper.Processors.Definitions;
namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Preparation; namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Preparation;
Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Preparation/RepositoryUpdateProcessor.cs
+3 -3
View File
@@ -1,6 +1,6 @@
using Pf2eDataScrapper.App; using Pf2eDataScrapper.Helpers;
using Pf2eDataScrapper.Helpers; using Pf2eDataScrapper.Processors.Attributes;
using Pf2eDataScrapper.Processors; using Pf2eDataScrapper.Processors.Definitions;
namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Preparation; namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Preparation;
Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Preparation/SchemaBuildProcessor.cs
+1 -1
View File
@@ -2,9 +2,9 @@
using System.Collections.Generic; using System.Collections.Generic;
using System.IO; using System.IO;
using System.Linq; using System.Linq;
using Pf2eDataScrapper.App;
using Pf2eDataScrapper.Helpers; using Pf2eDataScrapper.Helpers;
using Pf2eDataScrapper.Importers.Implementations.Aonprd.Compendium; using Pf2eDataScrapper.Importers.Implementations.Aonprd.Compendium;
using Pf2eDataScrapper.Processors.Attributes;
using Pf2eDataScrapper.SchemaBuilder; using Pf2eDataScrapper.SchemaBuilder;
namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Preparation; namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Preparation;
Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Subclasses/ApparitionProcessor.cs
+3 -3
View File
@@ -1,9 +1,10 @@
using System.Collections.Generic; using System.Collections.Generic;
using System.Linq; using System.Linq;
using Pf2eDataScrapper.App;
using Pf2eDataScrapper.Helpers; using Pf2eDataScrapper.Helpers;
using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers; using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers;
using Pf2eDataScrapper.Processors; using Pf2eDataScrapper.Processors;
using Pf2eDataScrapper.Processors.Attributes;
using Pf2eDataScrapper.Processors.Collection;
using Pf2eModel; using Pf2eModel;
using Pf2eModel.Model.Base; using Pf2eModel.Model.Base;
using Pf2eModel.Model.Common; using Pf2eModel.Model.Common;
@@ -19,8 +20,7 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Subclasse
InnerType = "apparition", InnerType = "apparition",
CompendiumType = ProcessorCompendiumEnum.Aonprd, CompendiumType = ProcessorCompendiumEnum.Aonprd,
Description = "Apparitions", Description = "Apparitions",
Group = ProcessorGroupEnum.AonprdSubclasses, Group = ProcessorGroupEnum.AonprdSubclasses
Active = false
)] )]
[ProcessorDependencies( [ProcessorDependencies(
["aonprd:trait", "aonprd:source", "aonprd:class", "aonprd:spell", "aonprd:skill"] ["aonprd:trait", "aonprd:source", "aonprd:class", "aonprd:spell", "aonprd:skill"]
Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/ActionProcessor.cs
+6 -3
View File
@@ -1,7 +1,8 @@
using Pf2eDataScrapper.App; using Pf2eDataScrapper.Helpers;
using Pf2eDataScrapper.Helpers;
using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers; using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers;
using Pf2eDataScrapper.Processors; using Pf2eDataScrapper.Processors;
using Pf2eDataScrapper.Processors.Attributes;
using Pf2eDataScrapper.Processors.Collection;
using Pf2eModel.Model.Enum; using Pf2eModel.Model.Enum;
using StringHelper = Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers.StringHelper; using StringHelper = Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers.StringHelper;
@@ -42,8 +43,10 @@ internal class ActionProcessor(
Object.Extra = GenerateJsonString("school"); Object.Extra = GenerateJsonString("school");
} }
public void Check() public override void Check()
{ {
base.Check();
if (Input.GetRequired<string>("markdown").StartsWith("**Activate")) if (Input.GetRequired<string>("markdown").StartsWith("**Activate"))
{ {
throw new SkipItemException(Input.GetRequired<string>("name"), "activating action"); throw new SkipItemException(Input.GetRequired<string>("name"), "activating action");
Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/AncestryProcessor.cs
+2 -1
View File
@@ -2,10 +2,11 @@
using System.Collections.Generic; using System.Collections.Generic;
using System.Linq; using System.Linq;
using Newtonsoft.Json.Linq; using Newtonsoft.Json.Linq;
using Pf2eDataScrapper.App;
using Pf2eDataScrapper.Helpers; using Pf2eDataScrapper.Helpers;
using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers; using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers;
using Pf2eDataScrapper.Processors; using Pf2eDataScrapper.Processors;
using Pf2eDataScrapper.Processors.Attributes;
using Pf2eDataScrapper.Processors.Collection;
using Pf2eModel.Model.Enum; using Pf2eModel.Model.Enum;
using Pf2eModel.Model.Structure; using Pf2eModel.Model.Structure;
using StringHelper = Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers.StringHelper; using StringHelper = Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers.StringHelper;
Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/ArchetypeProcessor.cs
+3 -2
View File
@@ -1,7 +1,8 @@
using Pf2eDataScrapper.App; using Pf2eDataScrapper.Helpers;
using Pf2eDataScrapper.Helpers;
using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers; using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers;
using Pf2eDataScrapper.Processors; using Pf2eDataScrapper.Processors;
using Pf2eDataScrapper.Processors.Attributes;
using Pf2eDataScrapper.Processors.Collection;
using Pf2eModel.Model.Enum; using Pf2eModel.Model.Enum;
using Pf2eModel.Model.Structure; using Pf2eModel.Model.Structure;
using StringHelper = Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers.StringHelper; using StringHelper = Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers.StringHelper;
Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/BackgroundProcessor.cs
+2 -1
View File
@@ -1,8 +1,9 @@
using Newtonsoft.Json.Linq; using Newtonsoft.Json.Linq;
using Pf2eDataScrapper.App;
using Pf2eDataScrapper.Helpers; using Pf2eDataScrapper.Helpers;
using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers; using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers;
using Pf2eDataScrapper.Processors; using Pf2eDataScrapper.Processors;
using Pf2eDataScrapper.Processors.Attributes;
using Pf2eDataScrapper.Processors.Collection;
using Pf2eModel.Model.Enum; using Pf2eModel.Model.Enum;
using Pf2eModel.Model.Structure; using Pf2eModel.Model.Structure;
using Pf2eModel.Model.Structure.BackgroundRelation; using Pf2eModel.Model.Structure.BackgroundRelation;
Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/ClassProcessor.cs
+2 -1
View File
@@ -3,10 +3,11 @@ using System.Collections.Generic;
using System.Linq; using System.Linq;
using System.Text.RegularExpressions; using System.Text.RegularExpressions;
using Newtonsoft.Json.Linq; using Newtonsoft.Json.Linq;
using Pf2eDataScrapper.App;
using Pf2eDataScrapper.Helpers; using Pf2eDataScrapper.Helpers;
using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers; using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers;
using Pf2eDataScrapper.Processors; using Pf2eDataScrapper.Processors;
using Pf2eDataScrapper.Processors.Attributes;
using Pf2eDataScrapper.Processors.Collection;
using Pf2eModel; using Pf2eModel;
using Pf2eModel.Model.Common; using Pf2eModel.Model.Common;
using Pf2eModel.Model.Common.I18n; using Pf2eModel.Model.Common.I18n;
Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/FeatProcessor.cs
+1 -1
View File
@@ -1,9 +1,9 @@
using System.Linq; using System.Linq;
using Newtonsoft.Json.Linq; using Newtonsoft.Json.Linq;
using Pf2eDataScrapper.App;
using Pf2eDataScrapper.Helpers; using Pf2eDataScrapper.Helpers;
using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers; using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers;
using Pf2eDataScrapper.Processors; using Pf2eDataScrapper.Processors;
using Pf2eDataScrapper.Processors.Attributes;
using Pf2eModel; using Pf2eModel;
using Pf2eModel.Model.Enum; using Pf2eModel.Model.Enum;
using Pf2eModel.Model.Structure; using Pf2eModel.Model.Structure;
Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/LanguageProcessor.cs
+2 -2
View File
@@ -1,7 +1,7 @@
using Pf2eDataScrapper.App; using Pf2eDataScrapper.Helpers;
using Pf2eDataScrapper.Helpers;
using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers; using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers;
using Pf2eDataScrapper.Processors; using Pf2eDataScrapper.Processors;
using Pf2eDataScrapper.Processors.Attributes;
using Pf2eModel.Model.Enum; using Pf2eModel.Model.Enum;
using Pf2eModel.Model.Structure; using Pf2eModel.Model.Structure;
using StringHelper = Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers.StringHelper; using StringHelper = Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers.StringHelper;
Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/RitualProcessor.cs
+1 -1
View File
@@ -3,10 +3,10 @@ using System.Collections.Generic;
using System.Linq; using System.Linq;
using System.Text.RegularExpressions; using System.Text.RegularExpressions;
using Newtonsoft.Json.Linq; using Newtonsoft.Json.Linq;
using Pf2eDataScrapper.App;
using Pf2eDataScrapper.Helpers; using Pf2eDataScrapper.Helpers;
using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers; using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers;
using Pf2eDataScrapper.Processors; using Pf2eDataScrapper.Processors;
using Pf2eDataScrapper.Processors.Attributes;
using Pf2eModel; using Pf2eModel;
using Pf2eModel.Model.Common.I18n; using Pf2eModel.Model.Common.I18n;
using Pf2eModel.Model.Enum; using Pf2eModel.Model.Enum;
Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/Rules/RuleProcessor.cs
+1 -1
View File
@@ -1,9 +1,9 @@
using System.Linq; using System.Linq;
using Newtonsoft.Json.Linq; using Newtonsoft.Json.Linq;
using Pf2eDataScrapper.App;
using Pf2eDataScrapper.Helpers; using Pf2eDataScrapper.Helpers;
using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers; using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers;
using Pf2eDataScrapper.Processors; using Pf2eDataScrapper.Processors;
using Pf2eDataScrapper.Processors.Attributes;
using Pf2eModel.Model.Base.Source; using Pf2eModel.Model.Base.Source;
using Pf2eModel.Model.Base.Targets; using Pf2eModel.Model.Base.Targets;
using Pf2eModel.Model.Common.I18n; using Pf2eModel.Model.Common.I18n;
Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/Rules/RulesOrderProcessor.cs
+2 -2
View File
@@ -7,9 +7,9 @@
using System.Collections.Generic; using System.Collections.Generic;
using System.Linq; using System.Linq;
using Pf2eDataScrapper.App;
using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers; using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers;
using Pf2eDataScrapper.Processors; using Pf2eDataScrapper.Processors.Attributes;
using Pf2eDataScrapper.Processors.Definitions;
using Pf2eModel; using Pf2eModel;
using Rule = Pf2eModel.Model.Structure.Rule; using Rule = Pf2eModel.Model.Structure.Rule;
Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/Rules/RulesTocProcessor.cs
+4 -3
View File
@@ -4,9 +4,9 @@ using System.Text.RegularExpressions;
using System.Web; using System.Web;
using AngleSharp; using AngleSharp;
using AngleSharp.Dom; using AngleSharp.Dom;
using Pf2eDataScrapper.App;
using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers; using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers;
using Pf2eDataScrapper.Processors; using Pf2eDataScrapper.Processors.Attributes;
using Pf2eDataScrapper.Processors.Definitions;
using Pf2eModel; using Pf2eModel;
using Pf2eModel.Model.Base.Source; using Pf2eModel.Model.Base.Source;
@@ -120,7 +120,8 @@ internal partial class RulesTocProcessor(Pf2eDbContext context) : IProcessor
IDocument document = null!; IDocument document = null!;
bool completed = false; bool completed = false;
while (!completed){ while (!completed)
{
try try
{ {
document = context.OpenAsync(Url).Result; document = context.OpenAsync(Url).Result;
Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/SpellProcessor.cs
+1 -1
View File
@@ -1,10 +1,10 @@
using System.Collections.Generic; using System.Collections.Generic;
using System.Linq; using System.Linq;
using Newtonsoft.Json.Linq; using Newtonsoft.Json.Linq;
using Pf2eDataScrapper.App;
using Pf2eDataScrapper.Helpers; using Pf2eDataScrapper.Helpers;
using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers; using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers;
using Pf2eDataScrapper.Processors; using Pf2eDataScrapper.Processors;
using Pf2eDataScrapper.Processors.Attributes;
using Pf2eModel.Model.Common.I18n; using Pf2eModel.Model.Common.I18n;
using Pf2eModel.Model.Enum; using Pf2eModel.Model.Enum;
using Pf2eModel.Model.Structure; using Pf2eModel.Model.Structure;
Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/TraitProcessor.cs
+2 -2
View File
@@ -1,7 +1,7 @@
using Pf2eDataScrapper.App; using Pf2eDataScrapper.Helpers;
using Pf2eDataScrapper.Helpers;
using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers; using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers;
using Pf2eDataScrapper.Processors; using Pf2eDataScrapper.Processors;
using Pf2eDataScrapper.Processors.Attributes;
using Pf2eModel.Model.Common.I18n; using Pf2eModel.Model.Common.I18n;
using Pf2eModel.Model.Enum; using Pf2eModel.Model.Enum;
using Pf2eModel.Model.Structure; using Pf2eModel.Model.Structure;
Pf2eDataScrapper/Importers/Implementations/FoundryEn/Processors/Entities/ActionProcessor.cs
+2 -1
View File
@@ -1,7 +1,8 @@
using Newtonsoft.Json.Linq; using Newtonsoft.Json.Linq;
using Pf2eDataScrapper.App;
using Pf2eDataScrapper.Helpers; using Pf2eDataScrapper.Helpers;
using Pf2eDataScrapper.Processors; using Pf2eDataScrapper.Processors;
using Pf2eDataScrapper.Processors.Attributes;
using Pf2eDataScrapper.Processors.Collection;
using Pf2eModel; using Pf2eModel;
using Pf2eModel.Model.Enum; using Pf2eModel.Model.Enum;
Pf2eDataScrapper/Importers/Implementations/FoundryEn/Processors/Entities/ConditionProcessor.cs
+2 -1
View File
@@ -1,7 +1,8 @@
using Newtonsoft.Json.Linq; using Newtonsoft.Json.Linq;
using Pf2eDataScrapper.App;
using Pf2eDataScrapper.Helpers; using Pf2eDataScrapper.Helpers;
using Pf2eDataScrapper.Processors; using Pf2eDataScrapper.Processors;
using Pf2eDataScrapper.Processors.Attributes;
using Pf2eDataScrapper.Processors.Collection;
using Pf2eModel; using Pf2eModel;
using Pf2eModel.Model.Enum; using Pf2eModel.Model.Enum;
using Pf2eModel.Model.Structure; using Pf2eModel.Model.Structure;
Pf2eDataScrapper/Importers/Implementations/FoundryEn/Processors/FoundryEnAbstractProcessor.cs
+1
View File
@@ -7,6 +7,7 @@ using Microsoft.EntityFrameworkCore;
using Newtonsoft.Json.Linq; using Newtonsoft.Json.Linq;
using Pf2eDataScrapper.Helpers; using Pf2eDataScrapper.Helpers;
using Pf2eDataScrapper.Processors; using Pf2eDataScrapper.Processors;
using Pf2eDataScrapper.Processors.Definitions;
using Pf2eModel; using Pf2eModel;
using Pf2eModel.Model; using Pf2eModel.Model;
using Pf2eModel.Model.Base; using Pf2eModel.Model.Base;
Pf2eDataScrapper/Importers/Implementations/FoundryEn/Processors/Preparation/CompendiumFolderProcessor.cs
+2 -2
View File
@@ -1,8 +1,8 @@
using System.IO; using System.IO;
using Newtonsoft.Json.Linq; using Newtonsoft.Json.Linq;
using Pf2eDataScrapper.App;
using Pf2eDataScrapper.Importers.Implementations.FoundryEn.Compendium; using Pf2eDataScrapper.Importers.Implementations.FoundryEn.Compendium;
using Pf2eDataScrapper.Processors; using Pf2eDataScrapper.Processors.Attributes;
using Pf2eDataScrapper.Processors.Definitions;
namespace Pf2eDataScrapper.Importers.Implementations.FoundryEn.Processors.Preparation; namespace Pf2eDataScrapper.Importers.Implementations.FoundryEn.Processors.Preparation;
Pf2eDataScrapper/Importers/Implementations/FoundryEn/Processors/Preparation/CompendiumItemsProcessor.cs
+2 -2
View File
@@ -2,10 +2,10 @@
using System.IO; using System.IO;
using System.Linq; using System.Linq;
using Newtonsoft.Json.Linq; using Newtonsoft.Json.Linq;
using Pf2eDataScrapper.App;
using Pf2eDataScrapper.Helpers; using Pf2eDataScrapper.Helpers;
using Pf2eDataScrapper.Importers.Implementations.FoundryEn.Compendium; using Pf2eDataScrapper.Importers.Implementations.FoundryEn.Compendium;
using Pf2eDataScrapper.Processors; using Pf2eDataScrapper.Processors.Attributes;
using Pf2eDataScrapper.Processors.Definitions;
namespace Pf2eDataScrapper.Importers.Implementations.FoundryEn.Processors.Preparation; namespace Pf2eDataScrapper.Importers.Implementations.FoundryEn.Processors.Preparation;
Pf2eDataScrapper/Importers/Implementations/FoundryEn/Processors/Preparation/RepositoryUpdateProcessor.cs
+3 -3
View File
@@ -1,6 +1,6 @@
using Pf2eDataScrapper.App; using Pf2eDataScrapper.Helpers;
using Pf2eDataScrapper.Helpers; using Pf2eDataScrapper.Processors.Attributes;
using Pf2eDataScrapper.Processors; using Pf2eDataScrapper.Processors.Definitions;
namespace Pf2eDataScrapper.Importers.Implementations.FoundryEn.Processors.Preparation; namespace Pf2eDataScrapper.Importers.Implementations.FoundryEn.Processors.Preparation;
Pf2eDataScrapper/Importers/Implementations/FoundryEn/Processors/Preparation/SchemaBuildProcessor.cs
+1 -1
View File
@@ -2,9 +2,9 @@
using System.Collections.Generic; using System.Collections.Generic;
using System.IO; using System.IO;
using System.Linq; using System.Linq;
using Pf2eDataScrapper.App;
using Pf2eDataScrapper.Helpers; using Pf2eDataScrapper.Helpers;
using Pf2eDataScrapper.Importers.Implementations.FoundryEn.Compendium; using Pf2eDataScrapper.Importers.Implementations.FoundryEn.Compendium;
using Pf2eDataScrapper.Processors.Attributes;
using Pf2eDataScrapper.SchemaBuilder; using Pf2eDataScrapper.SchemaBuilder;
namespace Pf2eDataScrapper.Importers.Implementations.FoundryEn.Processors.Preparation; namespace Pf2eDataScrapper.Importers.Implementations.FoundryEn.Processors.Preparation;
Pf2eDataScrapper/Pf2eDataScrapper.csproj
+3
View File
@@ -35,6 +35,9 @@
<None Update=".env"> <None Update=".env">
<CopyToOutputDirectory>Always</CopyToOutputDirectory> <CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None> </None>
<None Update="Resources\ImportAdaptation.json">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
<None Update="schema.pgsql"> <None Update="schema.pgsql">
<CopyToOutputDirectory>Always</CopyToOutputDirectory> <CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None> </None>
Pf2eDataScrapper/Processors/Attributes/AbstractProcessorAttribute.cs
+35
View File
@@ -0,0 +1,35 @@
using System;
namespace Pf2eDataScrapper.Processors.Attributes;
/// <summary>
/// Абстрактный аттрибут обработчика.
/// </summary>
[AttributeUsage(AttributeTargets.Class)]
internal class AbstractProcessorAttribute(string name) : Attribute
{
/// <summary>
/// Название обработчика.
/// </summary>
public string Name { get; } = name;
/// <summary>
/// Описание обработчика.
/// </summary>
public string Description { get; init; } = name;
/// <summary>
/// Группа обработчиков, к которому относится обработчик.
/// </summary>
public ProcessorGroupEnum Group { get; init; } = ProcessorGroupEnum.General;
/// <summary>
/// Процессор включён по умолчанию.
/// </summary>
public bool Active { get; init; } = true;
/// <summary>
/// Разрешить выполнять процессов в параллельном режиме
/// </summary>
public bool Parallel { get; init; } = false;
}
Pf2eDataScrapper/Processors/Attributes/ProcessorAttribute.cs
+10
View File
@@ -0,0 +1,10 @@
using System;
namespace Pf2eDataScrapper.Processors.Attributes;
/// <summary>
/// Общий аттрибут обработчика.
/// </summary>
/// <param name="name">Название обработчика.</param>
[AttributeUsage(AttributeTargets.Class)]
internal class ProcessorAttribute(string name) : AbstractProcessorAttribute(name) { }
Pf2eDataScrapper/Processors/Attributes/ProcessorCollectionAttribute.cs
+22
View File
@@ -0,0 +1,22 @@
using System;
namespace Pf2eDataScrapper.Processors.Attributes;
/// <summary>
/// Аттрибут обработчика набор элементов.
/// </summary>
/// <param name="name">Название обработчика.</param>
[AttributeUsage(AttributeTargets.Class)]
internal class ProcessorCollectionAttribute(string name) : AbstractProcessorAttribute(name)
{
/// <summary>
/// Название типов, который обрабатывает этот оброботчик.
/// <c>null</c> - обработать все типы.
/// </summary>
public required string? InnerType { get; init; }
/// <summary>
/// Тип используемого компендиума.
/// </summary>
public required ProcessorCompendiumEnum CompendiumType { get; init; }
}
Pf2eDataScrapper/Processors/Attributes/ProcessorDependenciesAttribute.cs
+17
View File
@@ -0,0 +1,17 @@
using System;
using System.Collections.Generic;
namespace Pf2eDataScrapper.Processors.Attributes;
/// <summary>
/// Аттрибут зависимостей обработчика.
/// </summary>
/// <param name="dependencies">Коллекция названий обработчиков, которые должны быть выполнены до этого оброботчика.</param>
[AttributeUsage(AttributeTargets.Class)]
public class ProcessorDependenciesAttribute(string[] dependencies) : Attribute
{
/// <summary>
/// Коллекция названий обработчиков, которые должны быть выполнены до этого оброботчика.
/// </summary>
public ICollection<string> Dependencies { get; } = dependencies;
}
Pf2eDataScrapper/Processors/Attributes/ProcessorFactoryAttribute.cs
+11
View File
@@ -0,0 +1,11 @@
using System;
namespace Pf2eDataScrapper.Processors.Attributes;
internal class ProcessorFactoryAttribute(string name) : AbstractProcessorAttribute(name)
{
/// <summary>
/// Тип обрабатывающего процессора
/// </summary>
public required Type ProcessorType { get; init; }
}
Pf2eDataScrapper/App/ProcessorGroupEnum.cs → Pf2eDataScrapper/Processors/Attributes/ProcessorGroupEnum.cs
+8 -1
View File
@@ -1,7 +1,7 @@
using System.ComponentModel; using System.ComponentModel;
using System.Reflection; using System.Reflection;
namespace Pf2eDataScrapper.App; namespace Pf2eDataScrapper.Processors.Attributes;
public enum ProcessorGroupEnum public enum ProcessorGroupEnum
{ {
@@ -33,6 +33,13 @@ public enum ProcessorCompendiumEnum
FoundryEn, FoundryEn,
} }
public enum ProcessorFactoryTypeEnum
{
Simple,
Compendium,
Iterator,
}
public static class ProcessorGroupEnumExtension public static class ProcessorGroupEnumExtension
{ {
public static string GetDescription(this ProcessorGroupEnum group) public static string GetDescription(this ProcessorGroupEnum group)
Pf2eDataScrapper/Processors/Collection/ProcessorCollection.cs
+296
View File
@@ -0,0 +1,296 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Reflection;
using LinqToDB;
using Microsoft.Extensions.DependencyInjection;
using Pf2eDataScrapper.Processors.Attributes;
using Pf2eDataScrapper.Processors.Definitions;
using Pf2eDataScrapper.Processors.Factory;
namespace Pf2eDataScrapper.Processors.Collection;
/// <summary>
/// Класс с коллекцией обработчиков.
/// </summary>
internal class ProcessorCollection
{
public IServiceProvider ServiceProvider { get; }
/// <summary>
/// Коллекция всех доступных обработчиков.
/// </summary>
public ICollection<ProcessorRecord> AllItems { get; } = [];
/// <summary>
/// Коллекция обработчиков, которые будут обрабатываться.
/// Для изменения необходимо использовать <see cref="FilterProcessors"/>.
/// </summary>
public List<ProcessorRecord> Items { get; private set; } = [];
/// <summary>
/// Обеспечение доступа к элементам обработчиков.
/// </summary>
/// <param name="name"></param>
/// <returns></returns>
public ProcessorRecord this[string name] => AllItems.First(item => item.Name == name);
/// <summary>
/// Конструктор класса <see cref="ProcessorCollection"/>.
/// </summary>
public ProcessorCollection(IServiceProvider serviceProvider)
{
ServiceProvider = serviceProvider;
Assembly assembly = Assembly.GetExecutingAssembly();
foreach (Type type in assembly.GetTypes())
{
ProcessType(type, type.GetCustomAttribute<AbstractProcessorAttribute>());
}
CheckDependencies();
FilterProcessors([]);
}
private void ProcessType(Type type, AbstractProcessorAttribute? attribute)
{
switch (attribute)
{
case null:
break;
case ProcessorAttribute processorAttribute:
ProcessType(type, processorAttribute);
break;
case ProcessorCollectionAttribute processorCollectionAttribute:
ProcessType(type, processorCollectionAttribute);
break;
case ProcessorFactoryAttribute processorFactoryAttribute:
ProcessType(type, processorFactoryAttribute);
break;
default:
throw new(
$"Неизвестная реализация класса {typeof(AbstractProcessorAttribute)} - {attribute.GetType()}"
);
}
}
private void ProcessType(Type type, ProcessorAttribute? attribute)
{
if (attribute is null)
{
return;
}
if (!type.IsAssignableTo(typeof(IProcessor)))
{
throw new(
$"Процессор {type}, которому установлен аттрибут {attribute.GetType()}, не представляет интерфейс {typeof(IProcessor)}"
);
}
ProcessorDependenciesAttribute? dependenciesAttribute =
type.GetCustomAttribute<ProcessorDependenciesAttribute>();
ProcessorRecord item = new()
{
Name = attribute.Name,
Type = type,
Description = attribute.Description,
Group = attribute.Group,
Dependencies = dependenciesAttribute?.Dependencies ?? [],
Active = attribute.Active,
Parallel = attribute.Parallel,
};
item.Factory = ActivatorUtilities.CreateInstance<SimpleProcessorFactory>(
ServiceProvider,
item
);
AllItems.Add(item);
}
private void ProcessType(Type type, ProcessorCollectionAttribute? attribute)
{
if (attribute is null)
{
return;
}
if (!type.IsAssignableTo(typeof(AbstractInputProcessor)))
{
throw new(
$"Процессор {type}, которому установлен аттрибут {attribute.GetType()}, не реализует класс {typeof(AbstractInputProcessor)}."
);
}
ProcessorDependenciesAttribute? dependenciesAttribute =
type.GetCustomAttribute<ProcessorDependenciesAttribute>();
ProcessorRecord item = new()
{
Name = attribute.Name,
Type = type,
Description = attribute.Description,
Group = attribute.Group,
Dependencies = dependenciesAttribute?.Dependencies ?? [],
Active = attribute.Active,
Parallel = attribute.Parallel,
};
CompendiumProcessorFactory factory =
ActivatorUtilities.CreateInstance<CompendiumProcessorFactory>(ServiceProvider, item);
factory.CompendiumType = attribute.CompendiumType;
factory.InnerType = attribute.InnerType;
item.Factory = factory;
AllItems.Add(item);
}
private void ProcessType(Type type, ProcessorFactoryAttribute? attribute)
{
if (attribute is null)
{
return;
}
if (!type.IsAssignableTo(typeof(AbstractProcessorFactory)))
{
throw new(
$"Процессор {type}, которому установлен аттрибут {attribute.GetType()}, не реализует класс {typeof(AbstractProcessorFactory)}."
);
}
ProcessorDependenciesAttribute? dependenciesAttribute =
type.GetCustomAttribute<ProcessorDependenciesAttribute>();
ProcessorRecord item = new()
{
Name = attribute.Name,
Type = attribute.ProcessorType,
Description = attribute.Description,
Group = attribute.Group,
Dependencies = dependenciesAttribute?.Dependencies ?? [],
Active = attribute.Active,
Parallel = attribute.Parallel,
Factory = (AbstractProcessorFactory)
ActivatorUtilities.CreateInstance(ServiceProvider, type),
};
AllItems.Add(item);
}
/// <summary>
/// Проверка, что все зависимости всех обработчиков могут быть удовлетворены.
/// </summary>
private void CheckDependencies()
{
foreach (ProcessorRecord item in Items)
{
foreach (string dependency in item.Dependencies)
{
if (Items.All(i => i.Name != dependency))
{
throw new($"Dependency '{dependency}' for processor '{item.Name}' not found.");
}
}
}
}
/// <summary>
/// Сортировать обработчики в соответсвии со всеми зависимостями.
/// </summary>
public void SortItems()
{
Items.ForEach(item => item.Priority = -1);
Dictionary<string, int> priorities = [];
while (priorities.Count < Items.Count)
{
foreach (ProcessorRecord item in Items)
{
if (item.Priority >= 0)
{
continue;
}
IEnumerable<int> dependencyPriorities = item.Dependencies.Select(d =>
priorities.GetValueOrDefault(d, -1)
);
if (dependencyPriorities.Any(p => p == -1))
{
continue;
}
item.Priority = Math.Max(
dependencyPriorities.Any() ? dependencyPriorities.Max() + 1 : 1,
(int)item.Group * 100
);
priorities.Add(item.Name, item.Priority);
}
}
Items.Sort((a, b) => a.Priority - b.Priority);
}
/// <summary>
/// Ограничить список используемых обработчиков по названию.
/// Автоматически включает все обработчики, от которых зависят указанные в аргументе обработчики.
/// </summary>
/// <param name="processorNames">Коллекция названиий требуемых обработчиков.</param>
public void FilterProcessors(ICollection<string> processorNames)
{
if (processorNames.Count == 0)
{
Items = [.. AllItems];
SortItems();
return;
}
Dictionary<string, bool> nameFlags = processorNames.ToDictionary(name => name, _ => false);
while (nameFlags.Values.Any(v => !v))
{
foreach (
KeyValuePair<string, bool> pair in nameFlags.Where(pair => !pair.Value).ToList()
)
{
ProcessorRecord item = AllItems.First(item => item.Name == pair.Key);
foreach (string dependency in item.Dependencies)
{
nameFlags.TryAdd(dependency, false);
}
nameFlags[pair.Key] = true;
}
}
Items = AllItems.Where(item => nameFlags.ContainsKey(item.Name)).ToList();
}
/// <summary>
/// Получить список типов, которые обрабатываются в компендиуме.
/// </summary>
/// <param name="compendium">Тип компендиума.</param>
/// <returns>Набор типов.</returns>
public ICollection<string> GetTypesOfCompendium(ProcessorCompendiumEnum compendium) =>
Items
.Where(item =>
item.Factory.GetType().IsAssignableTo(typeof(CompendiumProcessorFactory))
)
.Where(item =>
(item.Factory as CompendiumProcessorFactory)!.CompendiumType == compendium
)
.Select(item => (item.Factory as CompendiumProcessorFactory)!.InnerType)
.OfType<string>()
.ToList();
}
Pf2eDataScrapper/Processors/Collection/ProcessorRecord.cs
+57
View File
@@ -0,0 +1,57 @@
using System;
using System.Collections.Generic;
using Pf2eDataScrapper.Processors.Attributes;
using Pf2eDataScrapper.Processors.Factory;
namespace Pf2eDataScrapper.Processors.Collection;
/// <summary>
/// Элемент коллекции обработчиков.
/// </summary>
internal class ProcessorRecord
{
/// <summary>
/// Название обработчика.
/// </summary>
public required string Name { get; init; }
/// <summary>
/// Тип класса обработчика.
/// </summary>
public required Type Type { get; init; }
/// <summary>
/// Описание обработчика.
/// </summary>
public string Description { get; init; } = "";
/// <summary>
/// Фабрика обработчиков.
/// </summary>
public AbstractProcessorFactory Factory { get; set; } = null!;
/// <summary>
/// Группа обработчика.
/// </summary>
public required ProcessorGroupEnum Group { get; init; }
/// <summary>
/// Коллекция названий обработчиков, после которых должен выполняться обработчик.
/// </summary>
public ICollection<string> Dependencies { get; init; } = [];
/// <summary>
/// Процессор включён по умолчанию.
/// </summary>
public bool Active { get; init; } = true;
/// <summary>
/// Приоритет выполнения обработчика.
/// </summary>
public int Priority { get; set; } = -1;
/// <summary>
/// Разрешить выполнять процессов в параллельном режиме
/// </summary>
public bool Parallel { get; init; } = true;
}
Pf2eDataScrapper/Processors/AbstractInputProcessor.cs → Pf2eDataScrapper/Processors/Definitions/AbstractInputProcessor.cs
+2 -2
View File
@@ -1,6 +1,6 @@
using Pf2eDataScrapper.App; using Pf2eDataScrapper.Processors.Attributes;
namespace Pf2eDataScrapper.Processors; namespace Pf2eDataScrapper.Processors.Definitions;
/// <summary> /// <summary>
/// Абстрактный класс-обработчик данных, полученных из Архивов Нефиса. /// Абстрактный класс-обработчик данных, полученных из Архивов Нефиса.
Pf2eDataScrapper/Processors/AbstractProcessor.cs → Pf2eDataScrapper/Processors/Definitions/AbstractSimpleProcessor.cs
+3 -3
View File
@@ -1,6 +1,6 @@
using Pf2eDataScrapper.App; using Pf2eDataScrapper.Processors.Attributes;
namespace Pf2eDataScrapper.Processors; namespace Pf2eDataScrapper.Processors.Definitions;
/// <summary> /// <summary>
/// Абстрактный класс-обработчик данных, полученных из Архивов Нефиса. /// Абстрактный класс-обработчик данных, полученных из Архивов Нефиса.
@@ -8,7 +8,7 @@ namespace Pf2eDataScrapper.Processors;
/// для которых указан атрибут <see cref="ProcessorAttribute"/>. /// для которых указан атрибут <see cref="ProcessorAttribute"/>.
/// </summary> /// </summary>
/// <seealso cref="ProcessorAttribute"/> /// <seealso cref="ProcessorAttribute"/>
internal abstract class AbstractProcessor : IProcessor internal abstract class AbstractSimpleProcessor : IProcessor
{ {
/// <summary> /// <summary>
/// Основной метод обработки данных. /// Основной метод обработки данных.
Pf2eDataScrapper/Processors/IProcessor.cs → Pf2eDataScrapper/Processors/Definitions/IProcessor.cs
+1 -1
View File
@@ -1,4 +1,4 @@
namespace Pf2eDataScrapper.Processors; namespace Pf2eDataScrapper.Processors.Definitions;
public interface IProcessor public interface IProcessor
{ {
Pf2eDataScrapper/Processors/Factory/AbstractProcessorFactory.cs
+29
View File
@@ -0,0 +1,29 @@
using System;
using System.Collections.Generic;
using Pf2eDataScrapper.Processors.Collection;
using Pf2eDataScrapper.Processors.Definitions;
namespace Pf2eDataScrapper.Processors.Factory;
/// <summary>
/// Абстрактная фабрика создания процессоров.
/// </summary>
internal abstract class AbstractProcessorFactory
{
/// <summary>
/// Подготовить фабрику к работе.
/// </summary>
public virtual void Prepare() { }
/// <summary>
/// Количество элеметов для итерирования.
/// </summary>
/// <returns>Количество элеметов</returns>
public abstract int Count();
/// <summary>
/// Итератор процессоров.
/// </summary>
/// <returns>Пара значений: [0] - описание элемента процессор, [1] - процессор.</returns>
public abstract IEnumerable<Tuple<string, IProcessor>> GetEnumerator();
}
Pf2eDataScrapper/Processors/Factory/CompendiumProcessorFactory.cs
+71
View File
@@ -0,0 +1,71 @@
using System;
using System.Collections.Generic;
using Microsoft.Extensions.DependencyInjection;
using Newtonsoft.Json.Linq;
using Pf2eDataScrapper.Importers;
using Pf2eDataScrapper.Importers.Implementations.Aonprd.Compendium;
using Pf2eDataScrapper.Importers.Implementations.FoundryEn.Compendium;
using Pf2eDataScrapper.Processors.Attributes;
using Pf2eDataScrapper.Processors.Collection;
using Pf2eDataScrapper.Processors.Definitions;
namespace Pf2eDataScrapper.Processors.Factory;
internal class CompendiumProcessorFactory(ProcessorRecord item, IServiceProvider serviceProvider)
: AbstractProcessorFactory
{
public ProcessorRecord Item { get; } = item;
public IServiceProvider ServiceProvider { get; } = serviceProvider;
public ProcessorCompendiumEnum? CompendiumType { get; set; } = null;
public string? InnerType { get; set; } = null;
public AbstractCompendium Compendium { get; private set; } = null!;
public ICollection<string> Types { get; private set; } = [];
public override void Prepare()
{
if (CompendiumType is null)
{
throw new ArgumentNullException(nameof(CompendiumType));
}
base.Prepare();
ProcessorCollection collection =
ServiceProvider.GetService<ProcessorCollection>() ?? throw new();
Compendium =
CompendiumType switch
{
ProcessorCompendiumEnum.Aonprd => ServiceProvider.GetService<AonprdCompendium>(),
ProcessorCompendiumEnum.FoundryEn =>
ServiceProvider.GetService<FoundryEnCompendium>(),
_ => (AbstractCompendium?)null,
} ?? throw new($"Не могу выбрать импортёр для группы {Item.Group}");
Types = InnerType is null
? collection.GetTypesOfCompendium(CompendiumType ?? throw new())
: [InnerType];
}
public override int Count() => Compendium.Count(Types);
public override IEnumerable<Tuple<string, IProcessor>> GetEnumerator()
{
foreach ((AbstractCompendiumItem compendiumItem, JObject json) in Compendium.Iterate(Types))
{
ProcessorInput processorInput = ActivatorUtilities.CreateInstance<ProcessorInput>(
ServiceProvider,
compendiumItem.Id,
json
);
yield return new(
compendiumItem.Id,
(IProcessor)
ActivatorUtilities.CreateInstance(ServiceProvider, Item.Type, processorInput)
);
}
}
}
Pf2eDataScrapper/Processors/Factory/SimpleProcessorFactory.cs
+24
View File
@@ -0,0 +1,24 @@
using System;
using System.Collections.Generic;
using Microsoft.Extensions.DependencyInjection;
using Pf2eDataScrapper.Processors.Collection;
using Pf2eDataScrapper.Processors.Definitions;
namespace Pf2eDataScrapper.Processors.Factory;
internal class SimpleProcessorFactory(ProcessorRecord item, IServiceProvider serviceProvider)
: AbstractProcessorFactory
{
public ProcessorRecord Item { get; } = item;
public IServiceProvider ServiceProvider { get; } = serviceProvider;
public override int Count() => 1;
public override IEnumerable<Tuple<string, IProcessor>> GetEnumerator()
{
yield return new(
Item.Description,
(IProcessor)ActivatorUtilities.CreateInstance(ServiceProvider, Item.Type)
);
}
}
Pf2eDataScrapper/Processors/General/DatabaseTruncateProcessor.cs
+2 -1
View File
@@ -1,4 +1,5 @@
using Pf2eDataScrapper.App; using Pf2eDataScrapper.Processors.Attributes;
using Pf2eDataScrapper.Processors.Definitions;
using Pf2eModel; using Pf2eModel;
namespace Pf2eDataScrapper.Processors.General; namespace Pf2eDataScrapper.Processors.General;
Pf2eDataScrapper/Processors/ProcessorExecutor.cs
+17 -59
View File
@@ -2,30 +2,21 @@
using System.Collections.Generic; using System.Collections.Generic;
using System.ComponentModel; using System.ComponentModel;
using System.Threading.Tasks; using System.Threading.Tasks;
using Microsoft.Extensions.DependencyInjection;
using Newtonsoft.Json.Linq;
using Pf2eDataScrapper.App;
using Pf2eDataScrapper.Helpers; using Pf2eDataScrapper.Helpers;
using Pf2eDataScrapper.Importers; using Pf2eDataScrapper.Processors.Collection;
using Pf2eDataScrapper.Importers.Implementations.Aonprd.Compendium; using Pf2eDataScrapper.Processors.Definitions;
using Pf2eDataScrapper.Importers.Implementations.FoundryEn.Compendium; using Pf2eDataScrapper.Processors.Factory;
namespace Pf2eDataScrapper.Processors; namespace Pf2eDataScrapper.Processors;
internal sealed class ProcessorExecutor( internal sealed class ProcessorExecutor(
IServiceProvider serviceProvider,
ProcessorCollection collection, ProcessorCollection collection,
ProcessProgress progress, ProcessProgress progress,
AonprdCompendium aonprdCompendium,
FoundryEnCompendium foundryEnCompendium,
ApplicationSettings settings ApplicationSettings settings
) )
{ {
public IServiceProvider ServiceProvider { get; } = serviceProvider;
public ProcessorCollection Collection { get; } = collection; public ProcessorCollection Collection { get; } = collection;
public ProcessProgress Progress { get; } = progress; public ProcessProgress Progress { get; } = progress;
public AonprdCompendium AonprdCompendium { get; } = aonprdCompendium;
public FoundryEnCompendium FoundryEnCompendium { get; } = foundryEnCompendium;
public ApplicationSettings Settings { get; } = settings; public ApplicationSettings Settings { get; } = settings;
public void Start() public void Start()
@@ -39,23 +30,25 @@ internal sealed class ProcessorExecutor(
private int _currentIndex; private int _currentIndex;
private AbstractProcessorFactory CurrentFactory { get; set; } = null!;
private void StartItem() private void StartItem()
{ {
ProcessorCollectionItem item = Collection.Items[_currentIndex]; ProcessorRecord item = Collection.Items[_currentIndex];
Progress.Processors.Start(item.Description); Progress.Processors.Start(item.Description);
Progress.Progress(Progress.Processors.ToString()); Progress.Progress(Progress.Processors.ToString());
BackgroundWorker worker = new(); BackgroundWorker worker = new();
CurrentFactory = item.Factory;
CurrentFactory.Prepare();
worker.DoWork += (_, _) => worker.DoWork += (_, _) =>
{ {
Parallel.ForEach( Parallel.ForEach(
IterateProcessors(item), IterateProcessors(),
new ParallelOptions() new() { MaxDegreeOfParallelism = item.Parallel ? Settings.MaxThreads : 1 },
{
MaxDegreeOfParallelism = item.Parallel ? Settings.MaxThreads : 1,
},
RunProcessor RunProcessor
); );
}; };
@@ -94,53 +87,18 @@ internal sealed class ProcessorExecutor(
else else
{ {
Progress.Processors.Complete(); Progress.Processors.Complete();
Progress.Complete();
} }
} }
private IEnumerable<IProcessor> IterateProcessors(ProcessorCollectionItem item) private IEnumerable<IProcessor> IterateProcessors()
{ {
if (!item.IsCollection) Progress.Local.Reset(CurrentFactory.Count());
foreach ((string title, IProcessor processor) in CurrentFactory.GetEnumerator())
{ {
Progress.Local.Reset(1); Progress.Local.Start(title);
Progress.Local.Start(item.Description);
yield return (IProcessor)ActivatorUtilities.CreateInstance(ServiceProvider, item.Type);
Progress.Local.Complete();
yield break;
}
if (item.Group == ProcessorGroupEnum.General) yield return processor;
{
throw new(
$"Обработчик {item.Description} помечен как общий (поле Group == General) и коллекционный (поле InnerType), что запрещено."
);
}
AbstractCompendium compendium = item.Compendium switch
{
ProcessorCompendiumEnum.Aonprd => AonprdCompendium,
ProcessorCompendiumEnum.FoundryEn => FoundryEnCompendium,
_ => throw new($"Не могу выбрать импортёр для группы {item.Group}"),
};
ICollection<string> types = item.InnerType is null
? Collection.GetTypesOfCompendium(item.Compendium ?? throw new("impossible"))
: [item.InnerType];
Progress.Local.Reset(compendium.Count(types));
foreach ((AbstractCompendiumItem compendiumItem, JObject json) in compendium.Iterate(types))
{
Progress.Local.Start(compendiumItem.Id);
ProcessorInput processorInput = ActivatorUtilities.CreateInstance<ProcessorInput>(
ServiceProvider,
compendiumItem.Id,
json
);
yield return (IProcessor)
ActivatorUtilities.CreateInstance(ServiceProvider, item.Type, processorInput);
Progress.Local.End(); Progress.Local.End();
} }
Pf2eDataScrapper/Resources/ImportAdaptation.json
+82
View File
@@ -0,0 +1,82 @@
{
"Aonprd": {
"SlugSuffixes": {
"feat-455": "initial",
"spell-843": "initial",
"feat-2664": "archetype",
"feat-3247": "game-hunter",
"feat-3477": "undead-slayer",
"feat-3549": "vampire",
"feat-4344": "archetype",
"feat-4380": "stonebound",
"feat-5730": "dragonblood",
"feat-5737": "dragonblood",
"feat-5997": "legacy",
"feat-6209": "investigator",
"feat-6469": "general",
"feat-6477": "undead-slayer",
"feat-6883": "kitsune",
"feat-7004": "yaoguai",
"feat-7380": "mythic",
"feat-7637": "dandy"
},
"LegacyRemasterExtra": {
"action-14": "action-2273",
"action-15": "action-2275",
"action-558": "action-2277",
"archetype-83": "archetype-129",
"archetype-86": "archetype-198",
"feat-455": "feat-6007",
"feat-480": "feat-4850",
"feat-936": "feat-2269",
"feat-1177": "feat-4553",
"feat-1753": "feat-6031",
"feat-2117": "feat-7577",
"feat-2729": "feat-4077",
"feat-2742": "feat-4078",
"feat-2743": "feat-4079",
"feat-2986": "feat-4340",
"feat-2987": "feat-4341",
"feat-2989": "feat-4343",
"feat-2990": "feat-4346",
"feat-2991": "feat-4349",
"feat-2992": "feat-4350",
"feat-2993": "feat-4352",
"feat-4088": "feat-5491",
"feat-5997": "feat-7108",
"language-91": "language-180",
"ritual-31": "ritual-191",
"spell-641": "spell-1413",
"spell-646": "spell-1394",
"spell-841": "spell-1107",
"spell-843": "spell-2376",
"spell-1052": "spell-1309",
"spell-1053": "spell-1306",
"spell-1054": "spell-1305",
"spell-1055": "spell-1311",
"spell-1056": "spell-1310",
"spell-1057": "spell-1307",
"spell-1058": "spell-1304",
"spell-1059": "spell-1308"
},
"MergeSources": {
"feat-963": "feat-4519",
"feat-1167": "feat-5880",
"feat-1169": "feat-5055",
"feat-1171": "feat-4858",
"spell-551": "spell-2003",
"spell-553": "spell-2012",
"spell-559": "spell-2417",
"spell-560": "spell-2418",
"spell-571": "spell-1975",
"spell-578": "spell-1566",
"spell-654": "spell-819"
}
}
}
Pf2eDataScrapper/SchemaBuilder/AbstractSchemaBuildProcessor.cs
+1 -1
View File
@@ -4,7 +4,7 @@ using System.IO;
using Newtonsoft.Json.Linq; using Newtonsoft.Json.Linq;
using Pf2eDataScrapper.Helpers; using Pf2eDataScrapper.Helpers;
using Pf2eDataScrapper.Importers; using Pf2eDataScrapper.Importers;
using Pf2eDataScrapper.Processors; using Pf2eDataScrapper.Processors.Definitions;
namespace Pf2eDataScrapper.SchemaBuilder; namespace Pf2eDataScrapper.SchemaBuilder;
Pf2eDataScrapper/WPF/EntityViewer/Component/EntityListControl.xaml.cs
+1 -28
View File
@@ -1,42 +1,15 @@
using System.Windows; namespace Pf2eDataScrapper.WPF.EntityViewer.Component;
using Pf2eDataScrapper.WPF.EntityViewer.ViewModel;
namespace Pf2eDataScrapper.WPF.EntityViewer.Component;
/// <summary> /// <summary>
/// Логика взаимодействия для DbEntityListControl.xaml /// Логика взаимодействия для DbEntityListControl.xaml
/// </summary> /// </summary>
internal partial class EntityListControl : System.Windows.Controls.UserControl internal partial class EntityListControl : System.Windows.Controls.UserControl
{ {
private EntityListViewModel? _viewModel;
public EntityListControl() public EntityListControl()
{ {
InitializeComponent(); InitializeComponent();
} }
//https://www.youtube.com/watch?v=54ZmhbpjBmg
private void FirstButton_OnClick(object sender, RoutedEventArgs e)
{
// DataContext?.FirstPage();
}
private void PreviousButton_OnClick(object sender, RoutedEventArgs e)
{
//ViewModel?.PreviousPage();
}
private void NextButton_OnClick(object sender, RoutedEventArgs e)
{
//ViewModel?.NextPage();
}
private void LastButton_OnClick(object sender, RoutedEventArgs e)
{
//ViewModel?.LastPage();
}
private void TextBox_PreviewTextInput( private void TextBox_PreviewTextInput(
object sender, object sender,
System.Windows.Input.TextCompositionEventArgs e System.Windows.Input.TextCompositionEventArgs e
Pf2eDataScrapper/WPF/EntityViewer/EntityForm.xaml.cs
+3 -1
View File
@@ -29,6 +29,8 @@ internal partial class EntityForm : Window
private void Button_Click(object sender, RoutedEventArgs e) private void Button_Click(object sender, RoutedEventArgs e)
{ {
(Application.Current as ImportApplication)!.CurrentStage = ApplicationStage.InitialSettings; ApplicationStateChange?.Invoke(this, ApplicationStage.InitialSettings);
} }
public event ImportApplication.ApplicationStateChangeDelegate? ApplicationStateChange;
} }
Pf2eDataScrapper/WPF/EntityViewer/EntityViewStructure/StructureRow.cs
-4
View File
@@ -124,10 +124,6 @@ internal class StructureRow : INotifyPropertyChanged
Field.PropertyChanged += Field_PropertyChanged; Field.PropertyChanged += Field_PropertyChanged;
} }
/// <summary>
///
/// </summary>
/// <param name="obj"></param>
private void Field_PropertyChanged(object? sender, PropertyChangedEventArgs e) private void Field_PropertyChanged(object? sender, PropertyChangedEventArgs e)
{ {
OnPropertyChanged(nameof(Field)); OnPropertyChanged(nameof(Field));
Pf2eDataScrapper/WPF/ProgressForm/ProgressForm.xaml.cs
+3 -2
View File
@@ -2,6 +2,7 @@
using System.Collections.ObjectModel; using System.Collections.ObjectModel;
using System.Drawing; using System.Drawing;
using System.Windows; using System.Windows;
using LinqToDB.Tools;
using Pf2eDataScrapper.Helpers; using Pf2eDataScrapper.Helpers;
namespace Pf2eDataScrapper.WPF.ProgressForm; namespace Pf2eDataScrapper.WPF.ProgressForm;
@@ -26,7 +27,6 @@ internal partial class ProgressForm : Window
Progress.ProgressComplete += Progress_ProgressComplete; Progress.ProgressComplete += Progress_ProgressComplete;
} }
private void Progress_ProgressUpdate(ProcessProgress sender) private void Progress_ProgressUpdate(ProcessProgress sender)
{ {
if (!Dispatcher.CheckAccess()) if (!Dispatcher.CheckAccess())
@@ -58,9 +58,10 @@ internal partial class ProgressForm : Window
MessageListView.ScrollIntoView(item); MessageListView.ScrollIntoView(item);
} }
private void Progress_ProgressComplete(ProcessProgress sender) private void Progress_ProgressComplete(ProcessProgress sender)
{ {
(Application.Current as ImportApplication)!.CurrentStage = ApplicationStage.Viewer; ApplicationStateChange?.Invoke(this, ApplicationStage.Viewer);
} }
public event ImportApplication.ApplicationStateChangeDelegate? ApplicationStateChange; public event ImportApplication.ApplicationStateChangeDelegate? ApplicationStateChange;
Pf2eDataScrapper/WPF/StartForm/Components/ProcessorCollectionControl.xaml.cs
+4 -7
View File
@@ -3,7 +3,8 @@ using System.Collections.Generic;
using System.ComponentModel; using System.ComponentModel;
using System.Linq; using System.Linq;
using System.Windows; using System.Windows;
using Pf2eDataScrapper.App; using Pf2eDataScrapper.Processors.Attributes;
using Pf2eDataScrapper.Processors.Collection;
using CheckBox = System.Windows.Controls.CheckBox; using CheckBox = System.Windows.Controls.CheckBox;
namespace Pf2eDataScrapper.WPF.StartForm.Components; namespace Pf2eDataScrapper.WPF.StartForm.Components;
@@ -61,11 +62,7 @@ internal partial class ProcessorCollectionControl : System.Windows.Controls.User
Style = (Style)Resources["ElementPanelStyle"]!, Style = (Style)Resources["ElementPanelStyle"]!,
}; };
foreach ( foreach (ProcessorRecord item in Collection.AllItems.Where(item => item.Group == group))
ProcessorCollectionItem item in Collection.AllItems.Where(item =>
item.Group == group
)
)
{ {
ItemCheckBoxes[item.Name] = new() ItemCheckBoxes[item.Name] = new()
{ {
@@ -96,7 +93,7 @@ internal partial class ProcessorCollectionControl : System.Windows.Controls.User
foreach ((string key, CheckBox? checkBox) in ItemCheckBoxes) foreach ((string key, CheckBox? checkBox) in ItemCheckBoxes)
{ {
ProcessorCollectionItem item = Collection[key]; ProcessorRecord item = Collection[key];
bool isChecked = checkBox.IsChecked ?? false; bool isChecked = checkBox.IsChecked ?? false;
if (GroupLastStats.TryGetValue(item.Group, out bool? state)) if (GroupLastStats.TryGetValue(item.Group, out bool? state))
Pf2eDataScrapper/WPF/StartForm/StartForm.xaml.cs
+1 -1
View File
@@ -1,5 +1,5 @@
using System.Windows; using System.Windows;
using Pf2eDataScrapper.App; using Pf2eDataScrapper.Processors.Collection;
namespace Pf2eDataScrapper.WPF.StartForm; namespace Pf2eDataScrapper.WPF.StartForm;
Pf2eModel/Migrations/20250521102908_1.Designer.cs → Pf2eModel/Migrations/20250521155437_1.Designer.cs
Generated
+1 -1
View File
@@ -13,7 +13,7 @@ using Pf2eModel.Model.Enum;
namespace Pf2eModel.Migrations namespace Pf2eModel.Migrations
{ {
[DbContext(typeof(Pf2eDbContext))] [DbContext(typeof(Pf2eDbContext))]
[Migration("20250521102908_1")] [Migration("20250521155437_1")]
partial class _1 partial class _1
{ {
/// <inheritdoc /> /// <inheritdoc />
Pf2eModel/Migrations/20250521102908_1.cs → Pf2eModel/Migrations/20250521155437_1.cs
View File