adding

2026-03-13 09:11:35 -05:00
parent f004c8205a
commit c2d6bda458
197 changed files with 14970 additions and 0 deletions
@@ -0,0 +1,121 @@
+using System.Text.RegularExpressions;
+using LCU.API.Internal.DocumentLib.Containers;
+using LCU.API.Internal.DocumentLib.Utilities;
+using MongoDB.Driver;
+using OwnerNumberMigration.ValueParsing;
+
+namespace OwnerNumberMigration;
+
+public class AttributeMigration<T> : IAttributeMigration
+{
+    private readonly IDocumentFilterFactory _filterFactory;
+    private readonly IMongoCollectionFactory _collectionFactory;
+    private readonly string _attributeName;
+    private readonly IValueParsingStrategy<T> _valueParsingStrategy;
+    private readonly string _destinationProperty;
+    private readonly IObserver<DocumentInformation> _observer;
+
+    public AttributeMigration
+    (
+        IMongoCollectionFactory collectionFactory,
+        string attributeName,
+        IValueParsingStrategy<T> valueParsingStrategy,
+        string destinationProperty,
+        IObserver<DocumentInformation> observer,
+        IDocumentFilterFactory filterFactory
+    )
+    {
+        _attributeName = attributeName;
+        _valueParsingStrategy = valueParsingStrategy;
+        _destinationProperty = destinationProperty;
+        _observer = observer;
+        _collectionFactory = collectionFactory;
+        _filterFactory = filterFactory;
+    }
+
+    public async Task Migrate(CancellationToken cancellationToken)
+    {
+        if (cancellationToken.IsCancellationRequested) return;
+
+        try
+        {
+            if (typeof(DocumentInformation).GetProperty(_destinationProperty) is null)
+            {
+                throw new InvalidOperationException();
+            }
+                           
+            var hasAttribute = _filterFactory.AppliesForMigrationFilterFactory(_attributeName, _destinationProperty);
+            var collection = _collectionFactory.GetCollection<DocumentInformation>();
+            var documents = await RetryOnTooManyRequests(async () => await collection.FindAsync(hasAttribute, cancellationToken: cancellationToken), cancellationToken);
+
+            var done = false;
+            while (!done)
+            {
+                try
+                {
+                    while (!cancellationToken.IsCancellationRequested && await documents.MoveNextAsync(cancellationToken))
+                    {
+                        foreach (var document in documents.Current)
+                        {
+                            if (cancellationToken.IsCancellationRequested) break;
+
+                            _observer.OnNext(document);
+
+                            var value = _valueParsingStrategy.Parse(document.Attributes[_attributeName]);
+                            await RetryOnTooManyRequests(async () => await collection.FindOneAndUpdateAsync
+                            (
+                                d => d.DocumentId == document.DocumentId,
+                                Builders<DocumentInformation>.Update.Set(_destinationProperty, value),
+                                cancellationToken: cancellationToken
+                            ), cancellationToken);
+
+                            _observer.OnCompleted();
+                        }
+                    }
+
+                    done = true;
+                }
+                catch (MongoCursorNotFoundException e)
+                {
+                    _observer.OnError(e);
+
+                    documents = await RetryOnTooManyRequests(async () => await collection.FindAsync(hasAttribute, cancellationToken: cancellationToken), cancellationToken);
+                }
+            }
+        }
+        catch (Exception e)
+        {
+            _observer.OnError(e);
+            throw;
+        }
+    }
+
+    private async Task<TU> RetryOnTooManyRequests<TU>(Func<Task<TU>> action, CancellationToken cancellationToken)
+        where TU : class
+    {
+        TU? result = null;
+        while (result is null)
+        {
+            try
+            {
+                result = await action();
+            }
+            catch (MongoCommandException e)
+            {
+                if (e.Code == 16500)
+                {
+                    _observer.OnError(e);
+
+                    var wait = int.Parse(Regex.Match(e.Result.GetValue("errmsg").AsString, "\\bRetryAfterMs=(\\d+)\\b").Groups[1].ValueSpan);
+                    await Task.Delay(wait, cancellationToken);
+                }
+                else
+                {
+                    throw;
+                }
+            }
+        }
+
+        return result;
+    }
+}
@@ -0,0 +1,13 @@
+using LCU.API.Internal.DocumentLib.Containers;
+using MongoDB.Driver;
+
+namespace OwnerNumberMigration;
+
+public class DocumentFilterFactory : IDocumentFilterFactory
+{
+    public FilterDefinition<DocumentInformation> AppliesForMigrationFilterFactory(string attributeName, string destinationAttributeName)
+    {
+        return Builders<DocumentInformation>.Filter.Exists($"Attributes.{attributeName}")
+               & !Builders<DocumentInformation>.Filter.Exists(destinationAttributeName);
+    }
+}
@@ -0,0 +1,15 @@
+using OwnerNumberMigration.ValueParsing;
+
+namespace OwnerNumberMigration;
+
+public delegate IAttributeMigration AttributeMigrationFactory<in T>
+(
+    string attributeName,
+    IValueParsingStrategy<T> valueParsingStrategy,
+    string destinationProperty
+);
+
+public interface IAttributeMigration
+{
+    Task Migrate(CancellationToken cancellationToken);
+}
@@ -0,0 +1,9 @@
+using LCU.API.Internal.DocumentLib.Containers;
+using MongoDB.Driver;
+
+namespace OwnerNumberMigration;
+
+public interface IDocumentFilterFactory
+{
+    FilterDefinition<DocumentInformation> AppliesForMigrationFilterFactory(string attributeName, string destinationAttributeName);
+}
@@ -0,0 +1,62 @@
+using LCU.API.Internal.DocumentLib.Containers;
+using LCU.API.Internal.DocumentLib.Utilities;
+using Microsoft.Extensions.Logging;
+using MongoDB.Driver;
+
+namespace OwnerNumberMigration.Logging;
+
+public delegate IAttributeMigration AttributeMigrationLoggingDecoratorFactory
+(
+    IAttributeMigration migration,
+    string attributeName,
+    string destinationAttributeName
+);
+
+public class AttributeMigrationLoggingDecorator : IAttributeMigration
+{
+    private readonly IAttributeMigration _migration;
+    private readonly ILogger _logger;
+    private readonly IMongoCollection<DocumentInformation> _documents;
+    private readonly string _attributeName;
+    private readonly string _destinationAttributeName;
+    private readonly IDocumentFilterFactory _filterFactory;
+
+    public AttributeMigrationLoggingDecorator
+    (
+        IAttributeMigration migration,
+        ILogger logger,
+        IMongoCollectionFactory collectionFactory,
+        string attributeName,
+        string destinationAttributeName,
+        IDocumentFilterFactory filterFactory
+    )
+    {
+        _migration = migration;
+        _logger = logger;
+        _documents = collectionFactory.GetCollection<DocumentInformation>();
+        _attributeName = attributeName;
+        _destinationAttributeName = destinationAttributeName;
+        _filterFactory = filterFactory;
+    }
+
+    public async Task Migrate(CancellationToken cancellationToken)
+    {
+        var countToMigration = await _documents.CountDocumentsAsync
+        (
+            _filterFactory.AppliesForMigrationFilterFactory(_attributeName, _destinationAttributeName),
+            cancellationToken: cancellationToken
+        );
+        _logger.LogInformation
+        (
+            "starting to migration {attribute} with {count} documents",
+            _attributeName,
+            countToMigration
+        );
+        var start = DateTime.Now;
+
+        await _migration.Migrate(cancellationToken);
+
+        var duration = DateTime.Now - start;
+        _logger.LogInformation("finished migration for {attribute} in {elapsed}", _attributeName, duration);
+    }
+}
@@ -0,0 +1,33 @@
+using LCU.API.Internal.DocumentLib.Containers;
+using Microsoft.Extensions.Logging;
+
+namespace OwnerNumberMigration.Logging;
+
+public delegate IObserver<DocumentInformation> AttributeMigrationLoggingObserverFactory(string attributeName);
+
+public class AttributeMigrationLoggingObserver : IObserver<DocumentInformation>
+{
+    private readonly ILogger _logger;
+    private readonly string _attributeName;
+
+    public AttributeMigrationLoggingObserver(ILogger logger, string attributeName)
+    {
+        _logger = logger;
+        _attributeName = attributeName;
+    }
+
+    public void OnCompleted()
+    {
+        _logger.LogInformation("completed one migration for {attribute}", _attributeName);
+    }
+
+    public void OnError(Exception error)
+    {
+        _logger.LogError("migration error {error} in migration for {attribute}", error, _attributeName);
+    }
+
+    public void OnNext(DocumentInformation value)
+    {
+        _logger.LogInformation("starting one migration for {attribute} on {documentId}", _attributeName, value.DocumentId);
+    }
+}
@@ -0,0 +1,24 @@
+using LCU.API.Internal.DocumentLib.Containers;
+using OwnerNumberMigration.ValueParsing;
+
+namespace OwnerNumberMigration.Migrations;
+
+public class AccountNumberMigration : AttributeMigrationTask<string>
+{
+    public AccountNumberMigration
+    (
+        AttributeMigrationFactory<string> factory,
+        IParsingStrategyFactory strategyFactory,
+        IMigrationManager manager
+    )
+        : base
+        (
+            "AccountNumber",
+            nameof(DocumentInformation.AccountNumber),
+            factory,
+            strategyFactory,
+            manager
+        )
+    {
+    }
+}
@@ -0,0 +1,62 @@
+using Microsoft.Extensions.Hosting;
+using OwnerNumberMigration.ValueParsing;
+
+namespace OwnerNumberMigration.Migrations;
+
+public abstract class AttributeMigrationTask<T> : IHostedService
+{
+    private readonly IAttributeMigration _migration;
+    private readonly CancellationTokenSource _cancellation = new();
+    private readonly IMigrationManager _manager;
+
+    private Task? _runningMigration;
+
+    protected AttributeMigrationTask
+    (
+        string sourceAttribute,
+        string destinationAttribute,
+        AttributeMigrationFactory<T> factory,
+        IParsingStrategyFactory strategyFactory,
+        IMigrationManager manager
+    )
+    {
+        _manager = manager;
+        _migration = factory
+        (
+            sourceAttribute,
+            strategyFactory.CreateParsingStrategy<T>(sourceAttribute),
+            destinationAttribute
+        );
+    }
+
+    public Task StartAsync(CancellationToken cancellationToken)
+    {
+        if (cancellationToken.IsCancellationRequested) return Task.CompletedTask;
+
+        _manager.Start(_runningMigration = _migration.Migrate(_cancellation.Token));
+
+        return Task.CompletedTask;
+    }
+
+    public async Task StopAsync(CancellationToken cancellationToken)
+    {
+        if (_runningMigration is not null)
+        {
+            if (!_runningMigration.IsCompleted)
+            {
+                _cancellation.Cancel();
+                try
+                {
+                    await _runningMigration;
+                }
+                catch (TaskCanceledException)
+                {
+                }
+            }
+
+            _runningMigration.Dispose();
+        }
+
+        _cancellation.Dispose();
+    }
+}
@@ -0,0 +1,6 @@
+namespace OwnerNumberMigration.Migrations;
+
+public interface IMigrationManager
+{
+    public void Start(Task migrationTask);
+}
@@ -0,0 +1,37 @@
+using Microsoft.Extensions.Hosting;
+
+namespace OwnerNumberMigration.Migrations;
+
+public class MigrationManager : IMigrationManager
+{
+    private readonly IHostApplicationLifetime _appLifetime;
+    private readonly IList<Task> _migrationTasks = new List<Task>();
+
+    private static readonly IReadOnlySet<TaskStatus> CompletedStatuses = new HashSet<TaskStatus>
+    {
+        TaskStatus.Faulted,
+        TaskStatus.Canceled,
+        TaskStatus.RanToCompletion
+    };
+
+    public MigrationManager(IHostApplicationLifetime appLifetime)
+    {
+        _appLifetime = appLifetime;
+    }
+
+    public void Start(Task migrationTask)
+    {
+        _migrationTasks.Add(migrationTask);
+
+        migrationTask.ContinueWith(_ =>
+        {
+            lock (_migrationTasks)
+            {
+                if (_migrationTasks.All(t => CompletedStatuses.Contains(t.Status)))
+                {
+                    _appLifetime.StopApplication();
+                }
+            }
+        });
+    }
+}
@@ -0,0 +1,24 @@
+using LCU.API.Internal.DocumentLib.Containers;
+using OwnerNumberMigration.ValueParsing;
+
+namespace OwnerNumberMigration.Migrations;
+
+public class OrganizationNumberMigration : AttributeMigrationTask<string[]>
+{
+    public OrganizationNumberMigration
+    (
+        AttributeMigrationFactory<string[]> factory,
+        IParsingStrategyFactory strategyFactory,
+        IMigrationManager manager
+    )
+        : base
+        (
+            "OrganizationNumber",
+            nameof(DocumentInformation.OrganizationNumbers),
+            factory,
+            strategyFactory,
+            manager
+        )
+    {
+    }
+}
@@ -0,0 +1,24 @@
+using LCU.API.Internal.DocumentLib.Containers;
+using OwnerNumberMigration.ValueParsing;
+
+namespace OwnerNumberMigration.Migrations;
+
+public class OrganizationNumbersMigration : AttributeMigrationTask<string[]>
+{
+    public OrganizationNumbersMigration
+    (
+        AttributeMigrationFactory<string[]> factory,
+        IParsingStrategyFactory strategyFactory,
+        IMigrationManager manager
+    )
+        : base
+        (
+            "OrganizationNumbers",
+            nameof(DocumentInformation.OrganizationNumbers),
+            factory,
+            strategyFactory,
+            manager
+        )
+    {
+    }
+}
@@ -0,0 +1,24 @@
+using LCU.API.Internal.DocumentLib.Containers;
+using OwnerNumberMigration.ValueParsing;
+
+namespace OwnerNumberMigration.Migrations;
+
+public class PersonNumberMigration : AttributeMigrationTask<string[]>
+{
+    public PersonNumberMigration
+    (
+        AttributeMigrationFactory<string[]> factory,
+        IParsingStrategyFactory strategyFactory,
+        IMigrationManager manager
+    )
+        : base
+        (
+            "PersonNumber",
+            nameof(DocumentInformation.PersonNumbers),
+            factory,
+            strategyFactory,
+            manager
+        )
+    {
+    }
+}
@@ -0,0 +1,24 @@
+using LCU.API.Internal.DocumentLib.Containers;
+using OwnerNumberMigration.ValueParsing;
+
+namespace OwnerNumberMigration.Migrations;
+
+public class PersonNumbersMigration : AttributeMigrationTask<string[]>
+{
+    public PersonNumbersMigration
+    (
+        AttributeMigrationFactory<string[]> factory,
+        IParsingStrategyFactory strategyFactory,
+        IMigrationManager manager
+    )
+        : base
+        (
+            "PersonNumbers",
+            nameof(DocumentInformation.PersonNumbers),
+            factory,
+            strategyFactory,
+            manager
+        )
+    {
+    }
+}
@@ -0,0 +1,33 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <Content Include="..\LCU.API.Internal.Document\appsettings.json" Link="appsettings.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+  </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Include="LCU.Library.Internal.Configuration" Version="1.0.0" />
+    <PackageReference Include="Serilog.Extensions.Hosting" Version="5.0.1" />
+    <PackageReference Include="Serilog.Extensions.Logging.File" Version="3.0.0" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\LCU.API.Internal.DocumentLib\LCU.API.Internal.DocumentLib.csproj" />
+    <ProjectReference Include="..\LCU.API.Internal.Document\LCU.API.Internal.Document.csproj" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <Folder Include="Logging\" />
+  </ItemGroup>
+
+  <ProjectExtensions><VisualStudio><UserProperties /></VisualStudio></ProjectExtensions>
+
+</Project>
@@ -0,0 +1,137 @@
+using LCU.API.Internal.DocumentLib.Configurations;
+using LCU.API.Internal.DocumentLib.Utilities;
+using LCU.Library.Internal.Configuration;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;
+using OwnerNumberMigration;
+using OwnerNumberMigration.Logging;
+using OwnerNumberMigration.Migrations;
+using ILogger = Microsoft.Extensions.Logging.ILogger;
+using Azure.Identity;
+using LCU.API.Internal.DocumentLib.Containers;
+using Microsoft.Extensions.Logging;
+using OwnerNumberMigration.ValueParsing;
+using Serilog;
+using LCU.Library.Internal.Logging.Models;
+using LCU.Library.Internal.Logging;
+
+await Host.CreateDefaultBuilder(args)
+    .ConfigureAppConfiguration((host, configuration) =>
+    {
+        var vaultConfiguration = configuration.Build();
+        var clientSecretCredentials = new ClientSecretCredential
+        (
+            vaultConfiguration["TenantId"],
+            vaultConfiguration["ClientId"],
+            vaultConfiguration["ClientSecret"]
+        );
+        var vaultLocation = new Uri($"https://{vaultConfiguration["AzureKeyVaultUrl"]}.vault.azure.net/");
+
+        configuration.AddAzureKeyVault(vaultLocation, clientSecretCredentials);
+    })
+/* remove for local */
+    .UseSerilog()
+/* to here */
+/* local:
+    .ConfigureLogging(logging =>
+    {
+        logging
+            .AddConsole()
+            .AddFile("Logs/OwnerMigration-{Date}.log", isJson: true)
+            .SetMinimumLevel(LogLevel.Information);
+    })
+*/
+    .ConfigureServices((host, services) =>
+    {
+        services.RegisterOwnerMigrationServices();
+
+        host.Configuration.Bind<DatabaseConfiguration>(services, "MongoDatabase");
+        host.Configuration.Bind<NotificationApiConfiguration>(services, "NotificationApiConfiguration");
+
+/* remove for local */
+        var loggingConfiguration = host.Configuration.GetSection("Primary");
+        var loggerConfiguration = new LoggingConfiguration
+        (
+            loggingConfiguration["AppName"],
+            loggingConfiguration["SeqUrl"],
+            loggingConfiguration["SeqApiKey"]
+        );
+        var lcuLogging = new LcuLogging(loggerConfiguration);
+        lcuLogging.UseLcuLogging();
+/* to here */
+    })
+    .RunConsoleAsync();
+
+public static class ServiceRegistrationExtension
+{
+    public static IServiceCollection RegisterOwnerMigrationServices(this IServiceCollection services)
+    {
+        ServiceRegistrations.AddDocumentRepositories(services);
+        services.AddTransient<ILogger>(p => p.GetRequiredService<ILogger<DocumentInformation>>());
+        services.AddTransient<SingleValueStrategy>();
+        services.AddTransient<SingleValueArrayStrategy>();
+        services.AddTransient<CommaStringArrayStrategy>();
+        services.AddTransient<IParsingStrategyFactory, ParsingStrategyFactory>();
+        services.AddTransient<IDocumentFilterFactory, DocumentFilterFactory>();
+        services.AddTransient<AttributeMigrationLoggingDecoratorFactory>(p => (migration, name, destination) =>
+            new AttributeMigrationLoggingDecorator
+            (
+                migration,
+                p.GetRequiredService<ILogger>(),
+                p.GetRequiredService<IMongoCollectionFactory>(),
+                name,
+                destination,
+                p.GetRequiredService<IDocumentFilterFactory>()
+            )
+        );
+        services.AddTransient<AttributeMigrationLoggingObserverFactory>(p => name =>
+            new AttributeMigrationLoggingObserver
+            (
+                p.GetRequiredService<ILogger>(),
+                name
+            )
+        );
+        services.AddTransient<AttributeMigrationFactory<string>>(p => (name, strategy, property) =>
+            p.GetRequiredService<AttributeMigrationLoggingDecoratorFactory>()
+            (
+                new AttributeMigration<string>
+                (
+                    p.GetRequiredService<IMongoCollectionFactory>(),
+                    name,
+                    strategy,
+                    property,
+                    p.GetRequiredService<AttributeMigrationLoggingObserverFactory>()(name),
+                    p.GetRequiredService<IDocumentFilterFactory>()
+                ),
+                name,
+                property
+            )
+        );
+        services.AddTransient<AttributeMigrationFactory<string[]>>(p => (name, strategy, property) =>
+            p.GetRequiredService<AttributeMigrationLoggingDecoratorFactory>()
+            (
+                new AttributeMigration<string[]>
+                (
+                    p.GetRequiredService<IMongoCollectionFactory>(),
+                    name,
+                    strategy,
+                    property,
+                    p.GetRequiredService<AttributeMigrationLoggingObserverFactory>()(name),
+                    p.GetRequiredService<IDocumentFilterFactory>()
+                ),
+                name,
+                property
+            )
+        );
+
+        services.AddSingleton<IMigrationManager, MigrationManager>();
+        services.AddHostedService<PersonNumberMigration>();
+        services.AddHostedService<PersonNumbersMigration>();
+        services.AddHostedService<OrganizationNumberMigration>();
+        services.AddHostedService<OrganizationNumbersMigration>();
+        services.AddHostedService<AccountNumberMigration>();
+
+        return services;
+    }
+}
@@ -0,0 +1,10 @@
+{
+  "profiles": {
+    "OwnerNumberMigration": {
+      "commandName": "Project",
+      "environmentVariables": {
+        "DOTNET_ENVIRONMENT": "Development"
+      }
+    }
+  }
+}
@@ -0,0 +1,6 @@
+namespace OwnerNumberMigration.ValueParsing;
+
+public class CommaStringArrayStrategy : IValueParsingStrategy<string[]>
+{
+    public string[] Parse(string value) => value.Split(",");
+}
@@ -0,0 +1,6 @@
+namespace OwnerNumberMigration.ValueParsing;
+
+public interface IParsingStrategyFactory
+{
+    IValueParsingStrategy<T> CreateParsingStrategy<T>(string attribute);
+}
@@ -0,0 +1,6 @@
+namespace OwnerNumberMigration.ValueParsing;
+
+public interface IValueParsingStrategy<out T>
+{
+    T Parse(string value);
+}
@@ -0,0 +1,23 @@
+using Microsoft.Extensions.DependencyInjection;
+
+namespace OwnerNumberMigration.ValueParsing;
+
+public class ParsingStrategyFactory : IParsingStrategyFactory
+{
+    private readonly IServiceProvider _services;
+
+    public ParsingStrategyFactory(IServiceProvider services)
+    {
+        _services = services;
+    }
+
+    public IValueParsingStrategy<T> CreateParsingStrategy<T>(string attribute) => attribute switch
+    {
+        "PersonNumber" => (IValueParsingStrategy<T>)_services.GetRequiredService<SingleValueArrayStrategy>(),
+        "PersonNumbers" => (IValueParsingStrategy<T>)_services.GetRequiredService<CommaStringArrayStrategy>(),
+        "OrganizationNumber" => (IValueParsingStrategy<T>)_services.GetRequiredService<SingleValueArrayStrategy>(),
+        "OrganizationNumbers" => (IValueParsingStrategy<T>)_services.GetRequiredService<CommaStringArrayStrategy>(),
+        "AccountNumber" => (IValueParsingStrategy<T>)_services.GetRequiredService<SingleValueStrategy>(),
+        _ => throw new NotImplementedException()
+    };
+}
@@ -0,0 +1,6 @@
+namespace OwnerNumberMigration.ValueParsing;
+
+public class SingleValueArrayStrategy : IValueParsingStrategy<string[]>
+{
+    public string[] Parse(string value) => new[] { value };
+}
@@ -0,0 +1,6 @@
+namespace OwnerNumberMigration.ValueParsing;
+
+public class SingleValueStrategy : IValueParsingStrategy<string>
+{
+    public string Parse(string value) => value;
+}