monorepo/LCU.API.Internal.DocumentIngestor/LCU.API.Internal.DocumentIngestor/Services/IngestionOperationServices/PdfPartitionerService.cs

using System.Text.RegularExpressions;
using Serilog;
using Newtonsoft.Json;
using LCU.Library.Internal.ApiResponse.Services;
using LCU.Library.Internal.Exceptions;
using LCU.Library.Internal.DocumentIngestorModels.Models;
using LCU.API.Internal.DocumentIngestor.Services.MicroApiServices;
using LCU.API.Internal.DocumentIngestor.Services.PdfParserServices;
using LCU.Library.Internal.DocumentIngestorModels.Entities;
using LCU.Library.Internal.ApiResponse.Models;
using LCU.Library.Internal.LcuMassTransitModels;
using LCU.API.Internal.DocumentIngestor.Configurations;
using LCU.API.Internal.DocumentIngestor.Services.QueueServices;
using LCU.Library.Internal.FileOperationsModels.Models;
using LCU.Library.Internal.FileOperationsModels.Models.NameSettings;
using LCU.Library.Internal.FileOperationsModels.Shared;
using LCU.Library.Internal.LCUServiceFactory.Services;

namespace LCU.API.Internal.DocumentIngestor.Services.IngestionOperationServices;

public class PdfPartitionerService : IPdfPartitionerService
{
    private readonly ConnectionConfiguration _connectionConfiguration;
    private readonly ReceiveEndpointConfiguration _pdfPartitionerCompletionEndpoint;
    private readonly IPdfPartitionerRunInfoService _pdfPartitionerRunInfoService;
    private readonly IFileOperationsApiService _fileOperationsApiService;
    private readonly IQueuePdfPartitionerCompletionService _queuePdfPartitionerCompletionService;
    private readonly ILcuServiceFactory<IPdfParserService> _pdfParserServiceFactory;
    private const string AllPdfsRegexTag = "{Regex0}";
    private const string AllPdfsRegex = ".*";

    public PdfPartitionerService(ConnectionConfiguration connectionConfiguration,
                                 ReceiveEndpointConfigurations receiveEndpointConfigurations,
                                 IPdfPartitionerRunInfoService pdfPartitionerRunInfoService,
                                 IFileOperationsApiService fileOperationsApiService,
                                 IQueuePdfPartitionerCompletionService queuePdfPartitionerCompletionService,
                                 ILcuServiceFactory<IPdfParserService> pdfParserServiceFactory)
    {
        _connectionConfiguration = connectionConfiguration;
        _pdfPartitionerCompletionEndpoint = receiveEndpointConfigurations.PdfPartitionerCompletionEndpoint;
        _pdfPartitionerRunInfoService = pdfPartitionerRunInfoService;
        _fileOperationsApiService = fileOperationsApiService;
        _queuePdfPartitionerCompletionService = queuePdfPartitionerCompletionService;
        _pdfParserServiceFactory = pdfParserServiceFactory;
    }

    public async Task HandlePdfPartition(PdfPartitionerRequest request)
    {
        try
        {
            ArgumentNullException.ThrowIfNull(request);
            var dynamicDirectoryListRequest = new DynamicDirectoryListRequest
            {
                Path = request.FinalRootBasePath,
                DynamicDirectoryName = request.ParentDynamicDirectory,
                DynamicFileSettings = request.ParentDynamicDirectorySettings
            };
            var dynamicDirectories = await _fileOperationsApiService.BuildDynamicDirectoryList(dynamicDirectoryListRequest);

            var pdfPartitionerRunInfo = new PdfPartitionerRunInfo(DateTime.Now,
                                                                  request.PdfParserType,
                                                                  request.FinalRootBasePath,
                                                                  dynamicDirectories.ToList().Count);
            pdfPartitionerRunInfo = await _pdfPartitionerRunInfoService.SaveAsync(pdfPartitionerRunInfo);

            foreach (var directory in dynamicDirectories)
            {
                await PartitionPdf(request, directory, pdfPartitionerRunInfo);
            }
        }
        catch (ArgumentNullException)
        {
            throw;
        }
        catch (Exception exception)
        {
            const string errorMessage = "Failure during HandlePdfPartition.";
            var errorMessageWithException = $"{errorMessage} Exception Message: {exception.Message}";
            Log.Error(exception, errorMessage);
            await QueueReturnResponse.QueueResponse(request, false, new List<string> { errorMessageWithException });
            throw new ServiceException(errorMessageWithException);
        }
    }

    public async Task PartitionPdf(PdfPartitionerRequest request, string directory, PdfPartitionerRunInfo pdfPartitionerRunInfo)
    {
        var result = new PdfPartitionerResult(pdfPartitionerRunInfo.PdfPartitionerRunInfoId, request);
        try
        {
            var ingestionRootPath = DetermineIngestionRootPath(request, directory);
            var indexFile = await DetermineIndexFile(request, ingestionRootPath);
            var allPdfs = await DetermineAllPdfsInDirectory(ingestionRootPath);
            var mainPdf = DeterminMainPdfInDirectory(request, allPdfs);
            var batchPdfParserRequest = await PdfParserSwitch(request, ingestionRootPath, indexFile, mainPdf);

            var canSkipParsing = CanSkipPdfParse(request, allPdfs, batchPdfParserRequest);
            if (canSkipParsing)
            {
                result.MessageStack.Add($"Skipping Pdf Partition for Pdf {mainPdf} at the path {ingestionRootPath}");
                await _queuePdfPartitionerCompletionService.QueuePdfPartitionerCompletionRequest(result);
                return;
            }

            batchPdfParserRequest.RootPath = ingestionRootPath;
            batchPdfParserRequest.PdfName = mainPdf;
            batchPdfParserRequest = SetUpPdfPartitionerApiResponse(result, batchPdfParserRequest);
            await _fileOperationsApiService.QueueBatchPdfParser(batchPdfParserRequest);
        }
        catch (Exception exception)
        {
            Log.Error(exception.Message);
            result.AddFailureAndMessage(exception.Message);
            await _queuePdfPartitionerCompletionService.QueuePdfPartitionerCompletionRequest(result);
        }
    }

    public string DetermineIngestionRootPath(PdfPartitionerRequest request, string directory)
    {
        var subDirectoryPath = request.SubDirectoryPath.EndsWith("\\") || string.IsNullOrWhiteSpace(request.SubDirectoryPath) ?
                                   request.SubDirectoryPath :
                                   $"{request.SubDirectoryPath}\\";
        var formattedDirectory = directory.EndsWith("\\") ? directory : $"{directory}\\";
        var ingestionRootPath = request.FinalRootBasePath + formattedDirectory + subDirectoryPath;
        return ingestionRootPath;
    }

    public async Task<string> DetermineIndexFile(PdfPartitionerRequest request, string path)
    {
        var dynamicFileListRequest = new DynamicFileListRequest
        {
            Path = path,
            Filename = request.DynamicIndexFileName,
            DynamicFileSettings = request.DynamicFileSettings
        };

        return await _fileOperationsApiService.BuildDynamicFile(dynamicFileListRequest);
    }

    public async Task<List<string>> DetermineAllPdfsInDirectory(string directory)
    {
        var dynamicSettingsForAllPdfs = new DynamicNameSetting()
        {
            RegexSettings = new List<RegexSetting>
            {
                new RegexSetting
                {
                    Tag = AllPdfsRegexTag,
                    Regex = AllPdfsRegex
                }
            }
        };

        var dynamicFileListRequest = new DynamicFileListRequest
        {
            Path = directory,
            Filename = $"{AllPdfsRegexTag}.(?i:pdf)",
            DynamicFileSettings = dynamicSettingsForAllPdfs
        };

        var listOfPdfs = await _fileOperationsApiService.BuildDynamicFileList(dynamicFileListRequest);
        return listOfPdfs.ToList();
    }

    public string DeterminMainPdfInDirectory(PdfPartitionerRequest request, List<string> allPdfsInDirectory)
    {
        var regex = new Regex(request.DynamicPdfName);
        if (request?.DynamicPdfSettings?.RegexSettings != null)
        {
            regex = RegexNameHandler.BuildRegexName(request.DynamicPdfName, request.DynamicPdfSettings);
        }
       
        var mainPdf = allPdfsInDirectory.Where(pdf => regex.IsMatch(pdf)).ToList();
        if (mainPdf.Count > 1)
        {
            var message = $"There were more than one pdf that met the dynamic pdf settings. " +
                          $"DynamicPdfName:{request.DynamicPdfName}," +
                          $"DynamicPdfSettings:{JsonConvert.SerializeObject(request.DynamicPdfSettings)}";
            throw new Exception(message);
        }

        var pdf = mainPdf.FirstOrDefault();
        if (pdf == null)
        {
            var message = $"There were no pdfs that met the dynamic pdf settings. " +
                          $"DynamicPdfName:{request.DynamicPdfName}," +
                          $"DynamicPdfSettings:{JsonConvert.SerializeObject(request.DynamicPdfSettings)}";
            throw new Exception(message);
        }

        return pdf;
    }

    public async Task<BatchParsePdfRequest> PdfParserSwitch(PdfPartitionerRequest request, string rootPath, string indexFile, string pdfName)
    {
        
        var service = _pdfParserServiceFactory.GetService(request.PdfParserType);

        if (service is null)
        {
            throw new NotImplementedException();
        }

        return await service.DetermineBatchParsePdfRequest(rootPath, indexFile, pdfName);
    }

    public bool CanSkipPdfParse(PdfPartitionerRequest request, List<string> allPdfs, BatchParsePdfRequest batchParsePdfRequest)
    {
        if (!request.SkipParseIfCountsMatch)
        {
            return false;
        }

        var countsMatch = allPdfs.Count == batchParsePdfRequest.BatchPdfs.Count + 1;
        return countsMatch && request.SkipParseIfCountsMatch;
    }

    public BatchParsePdfRequest SetUpPdfPartitionerApiResponse(PdfPartitionerResult pdfPartitionerResult,
                                                                BatchParsePdfRequest batchParsePdfRequest)
    {
        var serializedResponse = JsonConvert.SerializeObject(pdfPartitionerResult);
        var newReturnItem = new ResponseItem
        {
            ResponsePayload = serializedResponse,
            ReturnQueueConnectionConfig = new QueueConnectionConfig
            {
                Url = _connectionConfiguration.HostUrl,
                Username = _connectionConfiguration.UserName,
                Password = _connectionConfiguration.Password
            },
            ReturnQueueConsumerConfig = new QueueConsumerConfig
            {
                ExchangeName = _pdfPartitionerCompletionEndpoint.ReceiveEndpoint
            }
        };

        batchParsePdfRequest.ResponseItems = new List<ResponseItem>
        {
            newReturnItem
        };
        return batchParsePdfRequest;
    }
}