kozhilya/Pf2eDataScrapper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Изменения

Browse Source
This commit is contained in:
kozhilya
2025-05-15 01:37:27 +04:00
parent 995029ce8d
commit 2b61669e7a
36 changed files with 200 additions and 360 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Pf2eDataScrapper/.env.example
+3
View File
@@ -6,3 +6,6 @@ OUTPUT_DIR=D:\path\to\output\directory
# Строка подключения к БД
DATABASE_CONNECTION=Host=localhost;Port=5433;Database=usersdb2;Username=postgres;Password=password
# Максимальное количество потоков.
MAX_THREADS=60
Pf2eDataScrapper/App/ProcessorAttribute.cs
+1 -1
View File
@@ -30,7 +30,7 @@ public class ProcessorAttribute(string name) : Attribute
/// <summary>
/// Разрешить выполнять процессов в параллельном режиме
/// </summary>
public bool Parallel { get; init; } = true;
public bool Parallel { get; init; } = false;
/// <summary>
/// Обработчик набора элементов
Pf2eDataScrapper/App/ProcessorCollection.cs
+6
View File
@@ -90,6 +90,7 @@ internal class ProcessorCollection
InnerType = innerType,
Compendium = compendium,
Active = processorAttribute.Active,
Parallel = processorAttribute.Parallel,
};
}
@@ -241,4 +242,9 @@ internal class ProcessorCollectionItem
/// <c>null</c> - обработать все типы.
/// </summary>
public ProcessorCompendiumEnum? Compendium { get; init; }
/// <summary>
/// Разрешить выполнять процессов в параллельном режиме
/// </summary>
public bool Parallel { get; init; } = true;
}
Pf2eDataScrapper/ApplicationSettings.cs
+10
View File
@@ -19,6 +19,7 @@ internal class ApplicationSettings
DataDirPath = EnvReader.GetStringValue("DATA_DIR");
OutputDirPath = EnvReader.GetStringValue("OUTPUT_DIR");
DatabaseConnection = EnvReader.GetStringValue("DATABASE_CONNECTION");
MaxThreads = Convert.ToInt32(EnvReader.GetStringValue("MAX_THREADS"));
}
#region Настройки из .env
@@ -62,6 +63,15 @@ internal class ApplicationSettings
[Description("Редактирование через .env-файл.")]
public string DatabaseConnection { get; }
/// <summary>
/// Максимальное количество потоков.
/// </summary>
[Browsable(true)]
[ReadOnly(true)]
[DisplayName("Максимальное количество потоков.")]
[Description("Редактирование через .env-файл.")]
public int MaxThreads { get; }
#endregion
#region Основные настройки
Pf2eDataScrapper/Forms/Controls/EntityTreeViewer/EntityTreeViewerControl.cs
+1
View File
@@ -1,4 +1,5 @@
using System.ComponentModel;
using Pf2eDataScrapper.Forms.Controls.EntityTreeViewer;
namespace Pf2eViewer.EntityTreeViewer;
Pf2eDataScrapper/Forms/Controls/EntityTreeViewer/EntityTreeViewerControlRow.cs
+1
View File
@@ -1,4 +1,5 @@
using FontAwesome.Sharp;
using Pf2eDataScrapper.Forms.Controls.EntityTreeViewer;
using Font = System.Drawing.Font;
using FontStyle = System.Drawing.FontStyle;
using Image = System.Drawing.Image;
Pf2eDataScrapper/Forms/Controls/EntityTreeViewer/StructureBuilder.cs
+1 -1
View File
@@ -3,7 +3,7 @@ using System.Reflection;
using LinqToDB.Extensions;
using Pf2eModel.Model.Common.I18n;
namespace Pf2eViewer.EntityTreeViewer;
namespace Pf2eDataScrapper.Forms.Controls.EntityTreeViewer;
/// <summary>
/// Класс для построения структуры данных класса.
Pf2eDataScrapper/Forms/Controls/ProcessorCollectionControl.cs
+1 -1
View File
@@ -54,7 +54,7 @@ internal partial class ProcessorCollectionControl : UserControl
FlowDirection = FlowDirection.LeftToRight,
WrapContents = true,
BorderStyle = BorderStyle.FixedSingle,
Margin = new Padding(10, 0, 10, 15),
Margin = new(10, 0, 10, 15),
};
RootFlowLayoutPanel.Controls.Add(GroupCheckBoxes[group]);
Pf2eDataScrapper/Helpers/ProcessProgress.cs
-1
View File
@@ -125,7 +125,6 @@ internal class ProcessProgress : AbstractLogger
/// </summary>
public void End()
{
_text = "";
_current++;
Update?.Invoke();
}
Pf2eDataScrapper/ImportApplication.cs
+5
View File
@@ -177,7 +177,12 @@ internal sealed class ImportApplication : ApplicationContext
public static void RegisterServices(IServiceCollection services)
{
// Контекст базы данных
services.AddEntityFrameworkNpgsql();
services.AddDbContext<Pf2eDbContext>();
//services.AddDbContextPool<Pf2eDbContext>(options =>
//{
// options.Us
//});
// Основная программа
services.AddSingleton<ImportApplication>();
Pf2eDataScrapper/Importers/Implementations/Aonprd/AonprdImporter.cs
-1
View File
@@ -30,7 +30,6 @@ internal class AonprdImporter : AbstractImporter
services.AddSingleton<AonprdCompendium>();
services.AddSingleton<AonprdBuilderGlobals>();
services.AddSingleton<LegacyEntityHandler>();
services.AddSingleton<RuleTocCollection>();
services.AddTransient<StringHelper>();
Pf2eDataScrapper/Importers/Implementations/Aonprd/Helpers/AonprdBuilderGlobals.cs
+11 -2
View File
@@ -4,14 +4,23 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers;
/// <summary>
/// Класс-помощник, хранящий необходимые глобальные данные.
/// Реализует шаблон <c>Singleton</c>.
/// </summary>
internal class AonprdBuilderGlobals
{
/// <summary>
/// Цель внешних ссылок Архивов Нефисов.
/// </summary>
public ExternalTarget AonprdTarget = null!;
public ExternalTarget AonprdTarget { get; set; } = null!;
/// <summary>
/// Словарь идентификаторов источников.
/// </summary>
public Dictionary<string, int> SourceIds { get; set; } = [];
/// <summary>
/// Словарь идентификаторов признаков.
/// </summary>
public Dictionary<string, int> TraitIds { get; set; } = [];
/// <summary>
/// Конструктор класса <see cref="AonprdBuilderGlobals"/>.
Pf2eDataScrapper/Importers/Implementations/Aonprd/Helpers/LegacyEntityHandler.cs
-105
View File
@@ -1,105 +0,0 @@
using Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors;
using Pf2eModel.Model.Base;
using Pf2eModel.Model.Base.Source;
using Pf2eModel.Model.Common.I18n;
namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers;
/// <summary>
/// Обработчик сущностей, помеченных как устаревшие.
/// </summary>
internal class LegacyEntityHandler
{
/// <summary>
/// Идентификатор устаревшей сущности.
/// </summary>
public string LegacyId { get; private set; }
/// <summary>
/// Название устаревшей сущности.
/// </summary>
public string LegacyName { get; private set; } = "";
/// <summary>
/// Коллекция источников устревшей сущности.
/// </summary>
public List<EntitySource> LegacySources { get; private set; } = [];
/// <summary>
/// Флаг, указывающий, что сведения об устаревшей сущности готовы к добавлению в обновлённые.
/// </summary>
public bool LegacyPrepared { get; private set; } = false;
/// <summary>
/// Список обновлённых сущностей, что отметели устаревшую сущность как предшественника.
/// </summary>
public List<Entity> RemasterEntities { get; } = [];
/// <summary>
/// Конструктор класса <see cref="LegacyEntityHandler"/>, если сущность сама отмечена как устаревшая.
/// </summary>
/// <param name="processor">Обработчик устаревшей сущности.</param>
public LegacyEntityHandler(AbstractEntityProcessor processor)
{
LegacyId = processor.Input.GetRequired<string>("id");
Prepare(processor);
}
/// <summary>
/// Конструктор класса <see cref="LegacyEntityHandler"/>, если обновлённая сущность указывает на ещё недобавленную устаревшую сущность.
/// </summary>
/// <param name="legacyId">Идентификатор устаревшей сущности.</param>
/// <param name="remasterEntity">Обновлённая сущность.</param>
public LegacyEntityHandler(string legacyId, Entity remasterEntity)
{
LegacyId = legacyId;
LegacyPrepared = false;
RemasterEntities.Add(remasterEntity);
}
/// <summary>
/// Подготовить данные об устравшей сущности.
/// </summary>
/// <param name="processor">Обработчик устаревшей сущности.</param>
public void Prepare(AbstractEntityProcessor processor)
{
if (LegacyPrepared)
{
return;
}
LegacyName = processor.Input.GetRequired<string>("name");
LegacySources = processor.GetSources();
LegacyPrepared = true;
foreach (Entity remasterEntity in RemasterEntities)
{
Apply(remasterEntity);
}
}
/// <summary>
/// Применить данные устаревшей сущности к обновлённой.
/// </summary>
/// <param name="remasterEntity">Обновлённая сущность.</param>
public void Apply(Entity remasterEntity)
{
if (!remasterEntity.Name.Contains(LegacyName))
{
remasterEntity.Name[LocaleType.LegacyEnglish] = LegacyName;
}
foreach (EntitySource legacySource in LegacySources)
{
EntitySource entitySource = new()
{
Entity = remasterEntity,
Source = legacySource.Source,
Suffix = legacySource.Suffix,
IsRemaster = false,
};
remasterEntity.Sources.Add(entitySource);
}
}
}
Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/AbstractEntityProcessor.cs
+61 -149
View File
@@ -1,8 +1,4 @@
using System.Collections.Concurrent;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using Pf2eDataScrapper.App;
@@ -31,13 +27,12 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors;
/// <seealso cref="ProcessorAttribute"/>
internal abstract class AbstractEntityProcessor(
ProcessorInput input,
Pf2eDbContext context,
StringHelper stringHelper,
DatabaseHelper helper,
AonprdBuilderGlobals globals
) : AbstractInputProcessor(input)
{
public Pf2eDbContext Context { get; } = context;
public Pf2eDbContext Context { get; } = new();
public StringHelper StringHelper { get; } = stringHelper;
public DatabaseHelper Helper { get; } = helper;
public AonprdBuilderGlobals Globals { get; } = globals;
@@ -60,19 +55,22 @@ internal abstract class AbstractEntityProcessor(
/// <summary>
/// Добавление основной сущности в контекст базы данных.
/// </summary>
public abstract void Add();
protected abstract void Add();
/// <inheritdoc/>
public override void Process()
{
ProcessRemaster();
if (Input.GetOptional<JArray>("remaster_id") is not null)
{
throw new LegacyItemException(Input.GetOptional<string>("name", ""));
}
AddEntity();
AddSource();
AddExternal();
AddTraits();
ProcessLegacy();
AddLegacy();
}
/// <inheritdoc/>
@@ -116,6 +114,23 @@ internal abstract class AbstractEntityProcessor(
protected virtual string GetDescription() =>
Input.GetRequired<string>("markdown") + AddedDescription(AddedDescriptionPaths());
/// <summary>
/// Генерация JSON-строки из объектов, располагаюхися по путям, указанным в <paramref name="paths"/>.
/// </summary>
/// <param name="paths">Коллекция путей, из которых необходимо достатить объекты.</param>
/// <returns>JSON-строка.</returns>
protected string GenerateJsonString(params ICollection<string> paths)
{
Dictionary<string, object?> data = [];
foreach (string key in paths)
{
data.Add(key, Input.Input.SelectToken(key));
}
return JsonConvert.SerializeObject(data, new JsonSerializerSettings());
}
#endregion
#region "Генерация сущности"
@@ -146,8 +161,29 @@ internal abstract class AbstractEntityProcessor(
/// </summary>
protected virtual void AddSource()
{
foreach (EntitySource entitySource in GetSources())
string[] sourceNames = Input
.GetRequired<JArray>("source")
.Select(token => token.ToString())
.ToArray();
if (sourceNames.Length == 0)
{
throw new(
$"Источники для сущности \"{Entity.Name[LocaleType.English]}\" "
+ $"(Aonprd id \"{Input.GetRequired<string>("id")}\") не найдены."
);
}
foreach (string sourceName in sourceNames)
{
EntitySource entitySource = new()
{
Entity = Entity,
SourceId = Globals.SourceIds[sourceName],
Suffix = StringHelper.ResolveSource(Input.GetRequired<string>("source_markdown")),
IsRemaster = Input.GetOptional<JArray>("legacy_id") is not null,
};
Entity.Sources.Add(entitySource);
}
}
@@ -160,7 +196,7 @@ internal abstract class AbstractEntityProcessor(
EntityExternal external = new()
{
Entity = Entity,
Target = Globals.AonprdTarget,
TargetId = Globals.AonprdTarget.Id,
ExternalId = Input.GetRequired<string>("id"),
Suffix = Input.GetRequired<string>("url"),
Md5 = Input.Md5(),
@@ -182,26 +218,11 @@ internal abstract class AbstractEntityProcessor(
return;
}
IEnumerable<string> names = traitsJson.ToArray().Select(jToken => jToken.ToString());
List<Entity> traitEntities = Context
.Traits.Select(t => t.Entity)
// ReSharper disable once EntityFramework.UnsupportedServerSideFunctionCall
.Where(e => Pf2eDbContext.LocaleSearchArray(e.Name, names))
.ToList();
List<string> names = traitsJson.ToArray().Select(jToken => jToken.ToString()).ToList();
List<Trait> traits = Context
.Traits.Include(t => t.Entity)
.Where(t => traitEntities.Contains(t.Entity))
.ToList();
List<int> traitIds = names.Select(name => Globals.TraitIds[name]).ToList();
foreach (string name in names)
{
if (!traits.Any(t => t.Entity.Name.Contains(name)))
{
//TODO
//throw new Exception($"Trait '{name}' is not found!!!");
}
}
List<Trait> traits = Context.Traits.Where(t => traitIds.Contains(t.Id)).ToList();
foreach (Trait trait in traits)
{
@@ -209,138 +230,29 @@ internal abstract class AbstractEntityProcessor(
}
}
#endregion
/// <summary>
/// Обработка сущности, отмеченной как устаревшей.
///
/// Обработка происходит в классе <see cref="LegacyEntityHandler"/>
/// Добавление устаревших сущностей.
/// </summary>
protected virtual void ProcessLegacy()
protected virtual void AddLegacy()
{
JArray? legacyIdArray = Input.GetOptional<JArray>("legacy_id");
if (legacyIdArray is null)
if (legacyIdArray is null || legacyIdArray.Count == 0)
{
return;
}
string[] legacyIds = legacyIdArray.ToArray().Select(t => t.ToString()).ToArray();
List<LegacyEntity> legacyEntities = Context
.LegacyEntityExternals.Where(lee => legacyIds.Contains(lee.ExternalId))
.Select(lee => lee.LegacyEntity)
.ToList();
if (legacyIds.Length == 0)
foreach (LegacyEntity legacyEntity in legacyEntities)
{
return;
}
foreach (string legacyId in legacyIds)
{
LegacyEntityHandler? handler = LegacyHandlers.FirstOrDefault(h =>
h.LegacyId == legacyId
);
if (handler is not null)
{
handler.Apply(Entity);
}
else
{
handler = new(legacyId, Entity);
LegacyHandlers.Add(handler);
}
Entity.LegacyEntities.Add(legacyEntity);
}
}
protected virtual void ProcessRemaster()
{
JArray? remasterIdArray = Input.GetOptional<JArray>("remaster_id");
if (remasterIdArray is null)
{
return;
}
if (remasterIdArray.Count > 1)
{
throw new("Несколько отметок ремастер-сущности на устаревшие!");
}
string[] remasterIds = remasterIdArray.ToArray().Select(t => t.ToString()).ToArray();
foreach (string remasterId in remasterIds)
{
LegacyEntityHandler? handler = null;
//LegacyHandlers.FirstOrDefault(h =>
// h.RemasterEntities.Contains(remasterId)
//);
if (handler is null)
{
handler = new(this);
LegacyHandlers.Add(handler);
}
else { }
}
throw new LegacyItemException(Input.GetRequired<string>("name"));
}
/// <summary>
/// Получение списка источников сущности.
/// </summary>
/// <returns></returns>
public List<EntitySource> GetSources()
{
string[] sourceNames = Input
.GetRequired<JArray>("source")
.Select(token => token.ToString())
.ToArray();
if (sourceNames.Length == 0)
{
throw new(
$"Source for entity with id \"{Entity.Name[LocaleType.English]}\" "
+ $"(Aonprd id \"{Input.GetRequired<string>("id")}\") is not found"
);
}
List<EntitySource> entitySources = [];
foreach (string sourceName in sourceNames)
{
Source source =
Context.Sources.FirstOrDefault(s => s.Name == sourceName)
?? throw new($"Source \"{sourceName}\" did not found");
EntitySource entitySource = new()
{
Entity = Entity,
Source = source,
Suffix = StringHelper.ResolveSource(Input.GetRequired<string>("source_markdown")),
IsRemaster = Input.GetOptional<JArray>("legacy_id") is not null,
};
entitySources.Add(entitySource);
}
return entitySources;
}
/// <summary>
/// Генерация JSON-строки из объектов, располагаюхися по путям, указанным в <paramref name="paths"/>.
/// </summary>
/// <param name="paths">Коллекция путей, из которых необходимо достатить объекты.</param>
/// <returns>JSON-строка.</returns>
public string GenerateJsonString(params ICollection<string> paths)
{
Dictionary<string, object?> data = [];
foreach (string key in paths)
{
data.Add(key, Input.Input.SelectToken(key));
}
return JsonConvert.SerializeObject(data, new JsonSerializerSettings());
}
protected static readonly ConcurrentBag<LegacyEntityHandler> LegacyHandlers = [];
#endregion
}
Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/AbstractSubclassProcessor.cs
+2 -3
View File
@@ -18,11 +18,10 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors;
/// </summary>
internal abstract class AbstractSubclassProcessor(
ProcessorInput input,
Pf2eDbContext context,
StringHelper stringHelper,
DatabaseHelper helper,
AonprdBuilderGlobals globals
) : AbstractEntityProcessor(input, context, stringHelper, helper, globals)
) : AbstractEntityProcessor(input, stringHelper, helper, globals)
{
/// <inheritdoc/>
public override EntityType Type => EntityType.Subclass;
@@ -41,7 +40,7 @@ internal abstract class AbstractSubclassProcessor(
public abstract string GroupName { get; }
/// <inheritdoc/>
public override void Add() => Context.Subclasses.Add(Object);
protected override void Add() => Context.Subclasses.Add(Object);
/// <inheritdoc/>
public override void Process()
Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Common/ExternalProcessor.cs
+3 -3
View File
@@ -12,10 +12,9 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Common;
Group = ProcessorGroupEnum.AonprdPreparation
)]
[ProcessorDependencies(["aonprd:repository"])]
internal class ExternalProcessor(Pf2eDbContext context, AonprdBuilderGlobals globals)
: AbstractProcessor
internal class ExternalProcessor(AonprdBuilderGlobals globals) : AbstractProcessor
{
public Pf2eDbContext Context { get; } = context;
public Pf2eDbContext Context { get; } = new();
public AonprdBuilderGlobals Globals { get; } = globals;
public ExternalTarget AonprdTarget = null!;
@@ -33,6 +32,7 @@ internal class ExternalProcessor(Pf2eDbContext context, AonprdBuilderGlobals glo
public override void Save()
{
Context.ExternalTargets.Add(AonprdTarget);
Context.SaveChanges();
Globals.AonprdTarget = AonprdTarget;
}
Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Common/LegacyProcessor.cs
+18 -30
View File
@@ -5,7 +5,6 @@ using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers;
using Pf2eDataScrapper.Processors;
using Pf2eModel;
using Pf2eModel.Model.Base.Legacy;
using Pf2eModel.Model.Base.Source;
using Pf2eModel.Model.Common.I18n;
namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Common;
@@ -15,17 +14,17 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Common;
InnerType = null,
CompendiumType = ProcessorCompendiumEnum.Aonprd,
Description = "Создание ссылок на устаревшие сущности",
Group = ProcessorGroupEnum.AonprdPreparation
Group = ProcessorGroupEnum.AonprdPreparation,
Parallel = true
)]
[ProcessorDependencies(["aonprd:source", "aonprd:external"])]
internal class LegacyProcessor(
ProcessorInput input,
Pf2eDbContext context,
AonprdBuilderGlobals globals,
StringHelper stringHelper
) : AbstractInputProcessor(input)
{
public Pf2eDbContext Context { get; } = context;
public Pf2eDbContext Context { get; } = new();
public AonprdBuilderGlobals Globals { get; } = globals;
public StringHelper StringHelper { get; } = stringHelper;
@@ -33,8 +32,6 @@ internal class LegacyProcessor(
public override void Process()
{
JArray? remasterIdArray = Input.GetOptional<JArray>("remaster_id");
if (Input.GetOptional<JArray>("remaster_id") is null)
{
throw new SilentSkipItemException();
@@ -53,18 +50,6 @@ internal class LegacyProcessor(
/// Генерация источников сущности.
/// </summary>
protected virtual void AddSource()
{
foreach (LegacyEntitySource entitySource in GetSources())
{
LegacyEntity.Sources.Add(entitySource);
}
}
/// <summary>
/// Получение списка источников сущности.
/// </summary>
/// <returns></returns>
public List<LegacyEntitySource> GetSources()
{
string[] sourceNames = Input
.GetRequired<JArray>("source")
@@ -78,27 +63,25 @@ internal class LegacyProcessor(
+ $"(Aonprd id \"{Input.GetRequired<string>("id")}\") is not found"
);
}
List<LegacyEntitySource> entitySources = [];
/*
Dictionary<string, int> sourceIds = Context
.Sources.AsNoTracking()
.Where(s => sourceNames.Contains(s.Name))
.ToDictionary(s => s.Name, s => s.Id);
*/
foreach (string sourceName in sourceNames)
{
Source source =
Context.Sources.FirstOrDefault(s => s.Name == sourceName)
?? throw new($"Source \"{sourceName}\" did not found");
LegacyEntitySource entitySource = new()
{
LegacyEntity = LegacyEntity,
Source = source,
SourceId = Globals.SourceIds[sourceName],
Suffix = StringHelper.ResolveSource(Input.GetRequired<string>("source_markdown")),
IsRemaster = Input.GetOptional<JArray>("legacy_id") is not null,
};
entitySources.Add(entitySource);
Context.LegacyEntitySources.Add(entitySource);
LegacyEntity.Sources.Add(entitySource);
}
return entitySources;
}
/// <summary>
@@ -109,7 +92,7 @@ internal class LegacyProcessor(
LegacyEntityExternal external = new()
{
LegacyEntity = LegacyEntity,
Target = Globals.AonprdTarget,
TargetId = Globals.AonprdTarget.Id,
ExternalId = Input.GetRequired<string>("id"),
Suffix = Input.GetRequired<string>("url"),
Md5 = Input.Md5(),
@@ -121,5 +104,10 @@ internal class LegacyProcessor(
public override void Save()
{
Context.LegacyEntities.Add(LegacyEntity);
Context.SaveChanges(false);
Context.Dispose();
}
private static readonly Lock Lock = new();
}
Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Common/SourceProcessor.cs
+7 -2
View File
@@ -1,4 +1,5 @@
using Pf2eDataScrapper.App;
using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers;
using Pf2eDataScrapper.Processors;
using Pf2eModel;
using Pf2eModel.Model.Base.Source;
@@ -14,10 +15,11 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Common;
Group = ProcessorGroupEnum.AonprdPreparation
)]
[ProcessorDependencies(["aonprd:compendium"])]
internal class SourceProcessor(ProcessorInput input, Pf2eDbContext context)
internal class SourceProcessor(ProcessorInput input, AonprdBuilderGlobals globals)
: AbstractInputProcessor(input)
{
public Pf2eDbContext Context { get; } = context;
public AonprdBuilderGlobals Globals { get; } = globals;
public Pf2eDbContext Context { get; } = new();
/// <summary>
/// Источник сущности.
@@ -39,5 +41,8 @@ internal class SourceProcessor(ProcessorInput input, Pf2eDbContext context)
public override void Save()
{
Context.Sources.Add(Source);
Context.SaveChanges();
Globals.SourceIds.Add(Source.Name, Source.Id);
}
}
Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Subclasses/ApparitionProcessor.cs
+1 -2
View File
@@ -24,11 +24,10 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Subclasse
)]
internal class ApparitionProcessor(
ProcessorInput input,
Pf2eDbContext context,
StringHelper stringHelper,
DatabaseHelper helper,
AonprdBuilderGlobals globals
) : AbstractSubclassProcessor(input, context, stringHelper, helper, globals)
) : AbstractSubclassProcessor(input, stringHelper, helper, globals)
{
/// <inheritdoc/>
public override Subclass Object { get; } = new();
Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/ActionProcessor.cs
+2 -3
View File
@@ -18,11 +18,10 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Types;
[ProcessorDependencies(["aonprd:trait", "aonprd:source"])]
internal class ActionProcessor(
ProcessorInput input,
Pf2eDbContext context,
StringHelper stringHelper,
DatabaseHelper helper,
AonprdBuilderGlobals globals
) : AbstractEntityProcessor(input, context, stringHelper, helper, globals)
) : AbstractEntityProcessor(input, stringHelper, helper, globals)
{
/// <inheritdoc/>
public override EntityType Type => EntityType.Action;
@@ -31,7 +30,7 @@ internal class ActionProcessor(
public override Pf2eModel.Model.Structure.Action Object { get; } = new();
/// <inheritdoc/>
public override void Add() => Context.Actions.Add(Object);
protected override void Add() => Context.Actions.Add(Object);
/// <inheritdoc/>
public override void Process()
Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/AncestryProcessor.cs
+2 -3
View File
@@ -20,11 +20,10 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Types;
[ProcessorDependencies(["aonprd:trait", "aonprd:source", "aonprd:language"])]
internal class AncestryProcessor(
ProcessorInput input,
Pf2eDbContext context,
StringHelper stringHelper,
DatabaseHelper helper,
AonprdBuilderGlobals globals
) : AbstractEntityProcessor(input, context, stringHelper, helper, globals)
) : AbstractEntityProcessor(input, stringHelper, helper, globals)
{
/// <inheritdoc/>
public override EntityType Type => EntityType.Ancestry;
@@ -33,7 +32,7 @@ internal class AncestryProcessor(
public override Ancestry Object { get; } = new();
/// <inheritdoc/>
public override void Add() => Context.Ancestries.Add(Object);
protected override void Add() => Context.Ancestries.Add(Object);
/// <inheritdoc/>
public override void Process()
Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/ArchetypeProcessor.cs
+2 -3
View File
@@ -19,11 +19,10 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Types;
[ProcessorDependencies(["aonprd:trait", "aonprd:source"])]
internal class ArchetypeProcessor(
ProcessorInput input,
Pf2eDbContext context,
StringHelper stringHelper,
DatabaseHelper helper,
AonprdBuilderGlobals globals
) : AbstractEntityProcessor(input, context, stringHelper, helper, globals)
) : AbstractEntityProcessor(input, stringHelper, helper, globals)
{
/// <inheritdoc/>
public override EntityType Type => EntityType.Archetype;
@@ -32,7 +31,7 @@ internal class ArchetypeProcessor(
public override Archetype Object { get; } = new();
/// <inheritdoc/>
public override void Add() => Context.Archetypes.Add(Object);
protected override void Add() => Context.Archetypes.Add(Object);
/// <inheritdoc/>
public override void Process()
Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/BackgroundProcessor.cs
+2 -3
View File
@@ -21,11 +21,10 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Types;
[ProcessorDependencies(["aonprd:trait", "aonprd:source"])]
internal class BackgroundProcessor(
ProcessorInput input,
Pf2eDbContext context,
StringHelper stringHelper,
DatabaseHelper helper,
AonprdBuilderGlobals globals
) : AbstractEntityProcessor(input, context, stringHelper, helper, globals)
) : AbstractEntityProcessor(input, stringHelper, helper, globals)
{
/// <inheritdoc/>
public override EntityType Type => EntityType.Background;
@@ -34,7 +33,7 @@ internal class BackgroundProcessor(
public override Background Object { get; } = new();
/// <inheritdoc/>
public override void Add() => Context.Backgrounds.Add(Object);
protected override void Add() => Context.Backgrounds.Add(Object);
/// <inheritdoc/>
public override void Process()
Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/ClassProcessor.cs
+2 -3
View File
@@ -25,11 +25,10 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Types;
[ProcessorDependencies(["aonprd:trait", "aonprd:source", "aonprd:skill"])]
internal partial class ClassProcessor(
ProcessorInput input,
Pf2eDbContext context,
StringHelper stringHelper,
DatabaseHelper helper,
AonprdBuilderGlobals globals
) : AbstractEntityProcessor(input, context, stringHelper, helper, globals)
) : AbstractEntityProcessor(input, stringHelper, helper, globals)
{
/// <inheritdoc/>
public override EntityType Type => EntityType.Class;
@@ -38,7 +37,7 @@ internal partial class ClassProcessor(
public override Class Object { get; } = new();
/// <inheritdoc/>
public override void Add() => Context.Classes.Add(Object);
protected override void Add() => Context.Classes.Add(Object);
/// <inheritdoc/>
public override void Process()
Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/FeatProcessor.cs
+2 -3
View File
@@ -20,11 +20,10 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Types;
[ProcessorDependencies(["aonprd:trait", "aonprd:source", "aonprd:archetype"])]
internal class FeatProcessor(
ProcessorInput input,
Pf2eDbContext context,
StringHelper stringHelper,
DatabaseHelper helper,
AonprdBuilderGlobals globals
) : AbstractEntityProcessor(input, context, stringHelper, helper, globals)
) : AbstractEntityProcessor(input, stringHelper, helper, globals)
{
/// <inheritdoc/>
public override EntityType Type => EntityType.Feat;
@@ -33,7 +32,7 @@ internal class FeatProcessor(
public override Feat Object { get; } = new();
/// <inheritdoc/>
public override void Add() => Context.Feats.Add(Object);
protected override void Add() => Context.Feats.Add(Object);
/// <inheritdoc/>
public override void Process()
Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/LanguageProcessor.cs
+2 -3
View File
@@ -19,11 +19,10 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Types;
[ProcessorDependencies(["aonprd:trait", "aonprd:source"])]
internal class LanguageProcessor(
ProcessorInput input,
Pf2eDbContext context,
StringHelper stringHelper,
DatabaseHelper helper,
AonprdBuilderGlobals globals
) : AbstractEntityProcessor(input, context, stringHelper, helper, globals)
) : AbstractEntityProcessor(input, stringHelper, helper, globals)
{
/// <inheritdoc/>
public override EntityType Type => EntityType.Language;
@@ -32,5 +31,5 @@ internal class LanguageProcessor(
public override Language Object { get; } = new();
/// <inheritdoc/>
public override void Add() => Context.Languages.Add(Object);
protected override void Add() => Context.Languages.Add(Object);
}
Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/RitualProcessor.cs
+2 -3
View File
@@ -23,11 +23,10 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Types;
[ProcessorDependencies(["aonprd:trait", "aonprd:source", "aonprd:skill"])]
internal partial class RitualProcessor(
ProcessorInput input,
Pf2eDbContext context,
StringHelper stringHelper,
DatabaseHelper helper,
AonprdBuilderGlobals globals
) : AbstractEntityProcessor(input, context, stringHelper, helper, globals)
) : AbstractEntityProcessor(input, stringHelper, helper, globals)
{
/// <inheritdoc/>
public override EntityType Type => EntityType.Ritual;
@@ -36,7 +35,7 @@ internal partial class RitualProcessor(
public override Ritual Object { get; } = new();
/// <inheritdoc/>
public override void Add() => Context.Rituals.Add(Object);
protected override void Add() => Context.Rituals.Add(Object);
/// <inheritdoc/>
public override void Process()
Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/Rules/RuleProcessor.cs
+3 -5
View File
@@ -5,7 +5,6 @@ using Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder.Helpers;
using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers;
using Pf2eDataScrapper.Processors;
using Pf2eModel;
using Pf2eModel.Model.Base.Legacy;
using Pf2eModel.Model.Base.Source;
using Pf2eModel.Model.Base.Targets;
using Pf2eModel.Model.Common.I18n;
@@ -18,17 +17,16 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Types.Rul
"aonprd:rules",
InnerType = "rules",
CompendiumType = ProcessorCompendiumEnum.Aonprd,
Description = "Создание цели внешних ссылок",
Description = "Правила",
Group = ProcessorGroupEnum.AonprdEntities
)]
[ProcessorDependencies(["aonprd:trait", "aonprd:source", "aonprd:rules-toc"])]
internal class RuleProcessor(
ProcessorInput input,
Pf2eDbContext context,
StringHelper stringHelper,
DatabaseHelper helper,
AonprdBuilderGlobals globals
) : AbstractEntityProcessor(input, context, stringHelper, helper, globals)
) : AbstractEntityProcessor(input, stringHelper, helper, globals)
{
/// <inheritdoc/>
public override EntityType Type => EntityType.Rule;
@@ -37,7 +35,7 @@ internal class RuleProcessor(
public override Rule Object { get; } = new();
/// <inheritdoc/>
public override void Add() => Context.Rules.Add(Object);
protected override void Add() => Context.Rules.Add(Object);
/// <summary>
/// Структура генерации содержания правил.
Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/SpellProcessor.cs
+2 -3
View File
@@ -22,11 +22,10 @@ namespace Pf2eDataScrapper.Importers.Implementations.Aonprd.Processors.Types;
[ProcessorDependencies(["aonprd:trait", "aonprd:source"])]
internal class SpellProcessor(
ProcessorInput input,
Pf2eDbContext context,
StringHelper stringHelper,
DatabaseHelper helper,
AonprdBuilderGlobals globals
) : AbstractEntityProcessor(input, context, stringHelper, helper, globals)
) : AbstractEntityProcessor(input, stringHelper, helper, globals)
{
/// <inheritdoc/>
public override EntityType Type => EntityType.Spell;
@@ -35,7 +34,7 @@ internal class SpellProcessor(
public override Spell Object { get; } = new();
/// <inheritdoc/>
public override void Add() => Context.Spells.Add(Object);
protected override void Add() => Context.Spells.Add(Object);
/// <inheritdoc/>
public override void Process()
Pf2eDataScrapper/Importers/Implementations/Aonprd/Processors/Types/TraitProcessor.cs
+14 -2
View File
@@ -4,6 +4,8 @@ using Pf2eDataScrapper.Importers.Implementations.Aonprd.DataBuilder.Helpers;
using Pf2eDataScrapper.Importers.Implementations.Aonprd.Helpers;
using Pf2eDataScrapper.Processors;
using Pf2eModel;
using Pf2eModel.Model.Base.Legacy;
using Pf2eModel.Model.Common.I18n;
using Pf2eModel.Model.Enum;
using Pf2eModel.Model.Structure;
@@ -23,7 +25,7 @@ internal class TraitProcessor(
StringHelper stringHelper,
DatabaseHelper helper,
AonprdBuilderGlobals globals
) : AbstractEntityProcessor(input, context, stringHelper, helper, globals)
) : AbstractEntityProcessor(input, stringHelper, helper, globals)
{
/// <inheritdoc/>
public override EntityType Type => EntityType.Trait;
@@ -32,7 +34,17 @@ internal class TraitProcessor(
public override Trait Object { get; } = new();
/// <inheritdoc/>
public override void Add() => Context.Traits.Add(Object);
protected override void Add()
{
Context.Traits.Add(Object);
Globals.TraitIds[Entity.Name[LocaleType.English]] = Entity.Id;
foreach (LegacyEntity legacyEntity in Entity.LegacyEntities)
{
Globals.TraitIds[legacyEntity.Name[LocaleType.English]] = Entity.Id;
}
}
/// <inheritdoc/>
public override void Process()
Pf2eDataScrapper/Processors/ProcessorExecutor.cs
+18 -11
View File
@@ -1,4 +1,5 @@
using System.ComponentModel;
using System.Diagnostics;
using Microsoft.Extensions.DependencyInjection;
using Newtonsoft.Json.Linq;
using Pf2eDataScrapper.App;
@@ -16,15 +17,16 @@ internal sealed class ProcessorExecutor(
ProcessProgress progress,
Pf2eDbContext context,
AonprdCompendium aonprdCompendium,
FoundryEnCompendium foundryEnCompendium
FoundryEnCompendium foundryEnCompendium,
ApplicationSettings settings
)
{
public IServiceProvider ServiceProvider { get; } = serviceProvider;
public ProcessorCollection Collection { get; } = collection;
public ProcessProgress Progress { get; } = progress;
public Pf2eDbContext Context { get; } = context;
public AonprdCompendium AonprdCompendium { get; } = aonprdCompendium;
public FoundryEnCompendium FoundryEnCompendium { get; } = foundryEnCompendium;
public ApplicationSettings Settings { get; } = settings;
public void Start()
{
@@ -48,7 +50,19 @@ internal sealed class ProcessorExecutor(
worker.DoWork += (_, _) =>
{
foreach (IProcessor processor in IterateProcessors(item))
Parallel.ForEach(
IterateProcessors(item),
new() { MaxDegreeOfParallelism = item.Parallel ? Settings.MaxThreads : 1 },
RunProcessor
);
};
worker.RunWorkerCompleted += (_, _) => Continue();
worker.RunWorkerAsync();
}
private void RunProcessor(IProcessor processor, ParallelLoopState state, long arg3)
{
try
{
@@ -64,15 +78,8 @@ internal sealed class ProcessorExecutor(
//
}
}
Context.SaveChanges();
};
worker.RunWorkerCompleted += Continue;
worker.RunWorkerAsync();
}
private void Continue(object? sender, RunWorkerCompletedEventArgs e)
private void Continue()
{
Progress.Processors.End();
_currentIndex++;
Pf2eModel/EfCleanHelper.cs
+2 -2
View File
@@ -69,7 +69,7 @@ public static class EfCleanHelper
public static string Truncate<T>(this DbSet<T> dbSet)
where T : class
{
string cmd = $"TRUNCATE TABLE {TableName(dbSet)} CASCADE;";
string cmd = $"TRUNCATE TABLE {TableName(dbSet)} RESTART IDENTITY CASCADE;";
DbContext context = dbSet.GetService<ICurrentDbContext>().Context;
context.Database.ExecuteSqlRaw(cmd);
return cmd;
@@ -87,7 +87,7 @@ public static class EfCleanHelper
foreach (IEntityType entityType in tables)
{
string cmd = $"TRUNCATE TABLE {GetName(entityType)} CASCADE;";
string cmd = $"TRUNCATE TABLE {GetName(entityType)} RESTART IDENTITY CASCADE;";
context.Database.ExecuteSqlRaw(cmd);
cmdList.Add(cmd);
}
Pf2eModel/Migrations/20250514102154_1.Designer.cs → Pf2eModel/Migrations/20250514120355_1.Designer.cs
Generated
+1 -1
View File
@@ -13,7 +13,7 @@ using Pf2eModel.Model.Enum;
namespace Pf2eModel.Migrations
{
[DbContext(typeof(Pf2eDbContext))]
[Migration("20250514102154_1")]
[Migration("20250514120355_1")]
partial class _1
{
/// <inheritdoc />
Pf2eModel/Migrations/20250514102154_1.cs → Pf2eModel/Migrations/20250514120355_1.cs
View File
Pf2eModel/Model/Base/Entity.cs
+1 -1
View File
@@ -104,5 +104,5 @@ public class Entity
/// <summary>
/// Коллекция устаревших сущностей.
/// </summary>
public IEnumerable<LegacyEntity> LegacyEntities { get; set; } = [];
public ICollection<LegacyEntity> LegacyEntities { get; set; } = [];
}
Pf2eModel/Pf2eDbContext.cs
+1
View File
@@ -136,6 +136,7 @@ public partial class Pf2eDbContext : DbContext
// optionsBuilder.LogTo(Console.WriteLine);
optionsBuilder.UseSnakeCaseNamingConvention();
optionsBuilder.EnableSensitiveDataLogging();
optionsBuilder.UseNpgsql(
databaseConnection,