monorepo/LCU.API.Internal.DocumentIngestor/LCU.API.Internal.DocumentIngestor/Services/IngestionOperationServices/IndexService.cs

using LCU.API.Internal.DocumentIngestor.Services.IndexParserServices;
using LCU.API.Internal.DocumentIngestor.Services.MicroApiServices;
using LCU.API.Internal.DocumentIngestor.Services.QueueServices;
using LCU.Library.Internal.ApiResponse.Services;
using LCU.Library.Internal.DocumentIngestorModels.Constants;
using LCU.Library.Internal.DocumentIngestorModels.Entities;
using LCU.Library.Internal.DocumentIngestorModels.Enums;
using LCU.Library.Internal.DocumentIngestorModels.Models;
using LCU.Library.Internal.Exceptions;
using LCU.Library.Internal.LCUServiceFactory.Services;
using Serilog;
namespace LCU.API.Internal.DocumentIngestor.Services.IngestionOperationServices;

public class IndexService : IIndexService
{
    private readonly IQueueDocumentProcessingService _queueDocumentProcessing;
    private readonly IIngestorRunInfoService _ingestorRunInfoService;
    private readonly IDocumentProcessingInfoService _documentProcessingInfoService;
    private readonly IDocumentTypeRetrieverService _documentTypeRetrieverService;
    private readonly ILcuServiceFactory<IIndexParserService> _serviceFactory;

    public IndexService(IQueueDocumentProcessingService queueDocumentProcessing,
                        IIngestorRunInfoService ingestorRunInfoService,
                        IDocumentProcessingInfoService documentProcessingInfoService,
                        IDocumentTypeRetrieverService documentTypeRetrieverService,
                        ILcuServiceFactory<IIndexParserService> serviceFactory)
    {
        _queueDocumentProcessing = queueDocumentProcessing;
        _ingestorRunInfoService = ingestorRunInfoService;
        _documentProcessingInfoService = documentProcessingInfoService;
        _documentTypeRetrieverService = documentTypeRetrieverService;
        _serviceFactory = serviceFactory;
    }

    public async Task HandleIngestion(IngestorRequest request)
    {
        try
        {
            ArgumentNullException.ThrowIfNull(request);
            var result = await PerformIngestor(request);
            if (result.FatalFailure)
            {
                var errorMessage = string.Join("; ", result.MessageStack.ToArray());
                throw new Exception($"Fatal Failure occured. Error Message: {errorMessage}");
            }

            await SendProcessRequest(result);
        }
        catch (ArgumentNullException)
        {
            throw;
        }
        catch (Exception exception)
        {
            const string errorMessage = "Failure during document ingestion.";
            var errorMessageWithException = $"{errorMessage} Exception Message: {exception.Message}";
            Log.Error(exception, errorMessage);
            await QueueReturnResponse.QueueResponse(request, false, new List<string> { errorMessageWithException });
            throw new ServiceException(errorMessageWithException);
        }
    }

    public async Task<IngestorResult> PerformIngestor(IngestorRequest request)
    {
        try
        {
            var service = _serviceFactory.GetService(request.IngestorType);

            if (service is null)
            {
                throw new NotImplementedException();
            }

            return await service.HandleIndexRequest(request);
        }
        catch (Exception exception)
        {
            return new IngestorResult(true, exception.Message);
        }
    }

    /// <summary>
    /// This method is used to:
    ///     - Save IngestorRunInfo
    ///     - Create/Save DocumentProcessingInfo
    ///     - Queue for processing
    ///     
    /// NOTE: If there are any failures to save to database this process should be stopped by exception and will be 
    ///       caught, logged, maidbot will be used, and the IngestorRequest will be placed on the LCU.DocumentIngestor.Indexing_error.
    ///       
    /// NOTE: If failure to queue an exception will be logged, DocumentProcessInfo's status will be set to CriticalFailure, and a ProcessLog will be created.
    /// </summary>
    /// <param name="result"></param>
    /// <returns></returns>
    public async Task SendProcessRequest(IngestorResult result)
    {
        result.IngestorRunInfo = await _ingestorRunInfoService.SaveAsync(result.IngestorRunInfo);
        ArgumentNullException.ThrowIfNull(result.IngestorRunInfo);

        var documentCodeDetailInfos = await GetDistinctDocumentCodeDetailInfos(result.ProcessingRequests);

        List<DocumentProcessingInfo> documentProcessingInfos = new();

        foreach (var processRequest in result.ProcessingRequests)
        {
            var documentProcessingInfo = new DocumentProcessingInfo(result.IngestorRunInfo.IngestorRunInfoId,
                DateTime.Now,
                processRequest.DocumentValuePayload,
                ProcessStatusTypes.Processing,
                processRequest.DocumentFileName,
                documentCodeDetailInfos[processRequest.DocumentTypeDisplayName].DocumentType,
                documentCodeDetailInfos[processRequest.DocumentTypeDisplayName].DocumentBusinessName);

            documentProcessingInfos.Add(documentProcessingInfo);
        }

        await _documentProcessingInfoService.SaveRangeAsync(documentProcessingInfos.ToArray());

        var savedDocumentProcessingInfos = (await _documentProcessingInfoService.GetByIngestorRunInfoIdAsync(result.IngestorRunInfo.IngestorRunInfoId)).ToArray();

        var tasks = new List<Task>();

        tasks.AddRange(
            savedDocumentProcessingInfos.Select<DocumentProcessingInfo, Task>(
                (documentProcessingInfo) =>
                {
                    var processRequest = result.ProcessingRequests.First(processingRequest => processingRequest.DocumentFileName == documentProcessingInfo.DocumentFileName);
                    processRequest.DocumentProcessingInfoId = documentProcessingInfo.DocumentProcessingInfoId;

                    return SendDocumentProcessingRequest(processRequest, documentProcessingInfo);
                }
            )
        );

        await Task.WhenAll(tasks);
    }

    private async Task<Dictionary<string, DocumentCodeDetailInfo>> GetDistinctDocumentCodeDetailInfos(IEnumerable<ProcessingRequest> processingRequests)
    {
        var distinctProcessingRequests = processingRequests.Select(pr => pr.DocumentTypeDisplayName).Distinct();
        var documentCodeDetailInfos = new Dictionary<string, DocumentCodeDetailInfo>();

        foreach (var documentTypeDisplayName in distinctProcessingRequests)
        {
            // TODO: Get by range?
            var documentCodeDetail =
                await _documentTypeRetrieverService.GetDocumentCodeDetailByDisplayName(documentTypeDisplayName);
            documentCodeDetailInfos.Add(documentTypeDisplayName, documentCodeDetail);
        }

        return documentCodeDetailInfos;
    }

    public async Task SendDocumentProcessingRequest(ProcessingRequest processRequest, DocumentProcessingInfo documentProcessInfo)
    {
        try
        {
            await _queueDocumentProcessing.QueueDocumentProcessingRequest(processRequest);
        }
        catch (Exception exception)
        {
            var errorMessage = "Failure to Queue DocumentProcessingRequest. " +
                               $"DocumentProcessInfoId: {documentProcessInfo.DocumentProcessingInfoId},  " +
                               $"Exception Message: {exception.Message}";
            Log.Error(errorMessage);

            var documentProcessingInfo = await _documentProcessingInfoService.UpdateStatusAndAddLog(documentProcessInfo, 
                                                                                                    ProcessStatusTypes.CriticalFailure, 
                                                                                                    errorMessage,
                                                                                                    Users.System);
            if (documentProcessingInfo == null)
            {
                throw new Exception("Unable to update DocumentProcessingInfo, null returned.");
            }
        }
    }
}