monorepo/LCU.API.Internal.FileOperations/LCU.Api.Internal.FileOperations/Services/FileActionServices/ExtractIndexFileDataService.cs

using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Text.RegularExpressions;
using CsvHelper;
using CsvHelper.Configuration;
using CsvHelper.Configuration.Attributes;
using LCU.Api.Internal.FileOperations.Validations;
using LCU.Api.Internal.FileOperations.Models;
using LCU.Library.Internal.FileOperationsModels.Models;
using LCU.Library.Internal.FileOperationsModels.Shared;
using LCU.Library.Internal.LcuEvaluation.Services.EvaluationServices;
using LCU.Library.Internal.LCUServiceFactory.Configurations;
using LCU.Library.Internal.TypeExtentions.EnumExtensions;
using Newtonsoft.Json;
using Serilog;

namespace LCU.Api.Internal.FileOperations.Services.FileActionServices;

public class ExtractIndexFileDataService : FileActionService, IFileActionService, IActionService<ExtractIndexFileDataRequest>
{
    private readonly IRunRuleService _runRuleService;
    private const string RegexMatch = "(\")(.*?)(\")";

	public ExtractIndexFileDataService(IRunRuleService runRuleService, ServicesFactoryConfiguration servicesFactoryConfiguration)
        : base(servicesFactoryConfiguration)
	{
		_runRuleService = runRuleService;
	}

	public RequestResponse ExecuteAction(QueueRequest queueRequest)
    {
        var requestResponse = new RequestResponse();
        try
        {
            var request = JsonConvert.DeserializeObject<ExtractIndexFileDataRequest>(queueRequest.Request);
            var requests = DynamicNameBuilder.BuildListOfRequest(request);

            if (requests.Count > 1)
            {
                requestResponse.MessageStack.Add("More than one file was found.");
                requestResponse.IsSuccessful = false;
            }
            else
            {
                requestResponse = PerformRequest(requests.FirstOrDefault());
            }

            requestResponse.ReturnResponse = requests.FirstOrDefault();
            return requestResponse;
        }
        catch (Exception exception)
        {
            return RequestResponseHelper.ExceptionUpdateRequestResponse(requestResponse,
                                                                        queueRequest.FileOperationType,
                                                                        exception.Message);
        }
    }

    public RequestResponse PerformRequest(ExtractIndexFileDataRequest request)
    {
        return OperationService.PerformOperation(request, IsRequestValid, ExtractIndexFileData);
    }
    
    public RequestResponse PreQueueValidation(ExtractIndexFileDataRequest originalRequest)
    {
        var requestResponse = new RequestResponse();
        return requestResponse;
    }

    public RequestResponse IsRequestValid(ExtractIndexFileDataRequest request, RequestResponse requestResponse)
    {
        requestResponse = FileValidation.ValidateFileExists(request.FinalIndexFilePath, requestResponse);
        requestResponse = DirectoryValidation.ValidateDirectoryExists(request.FinalDestinationPath, requestResponse);

        return requestResponse;
    }
    
    private void ExtractIndexFileData(ExtractIndexFileDataRequest request)
    {
        try
        {
            var delimiter = request.DelimiterType.GetNameFromValue();
            
            var fileLines = GetFileContents(request.FinalIndexFilePath);
            
            if (fileLines is null || !fileLines.Any())
            {
                Log.Information($"No content was extracted from filepath: ({request.FinalIndexFilePath})");
                return;
            }
            
            var recordsAreWrappedInQuotes = HasQuotes(fileLines);
            var headers = request.IndexHasHeaders ? GetHeaders(fileLines, delimiter, recordsAreWrappedInQuotes) : new List<string>();
            var columnMax = GetColumnCount(fileLines, delimiter, recordsAreWrappedInQuotes);
            var fileContent = string.Join(Environment.NewLine, fileLines);
            var indexEvaluationRequests = GetCsvRecords(fileContent,
                                                        delimiter,
                                                        request.IndexHasHeaders);
            var evaluatedRequests = BuildAndRunRules(indexEvaluationRequests, request);
			
            var passedList = FilterColumns(evaluatedRequests.Passed,
                                           request.IndexesToExtract,
                                           columnMax,
                                           request.ExtractAllIndexes);
            var failedList = FilterColumns(evaluatedRequests.Failed,
                                           request.IndexesToExtract,
                                           columnMax,
                                           true);
            
            if (passedList is null || !passedList.Any())
            {
                Log.Information("No results passed evaluations to write to destination file.");
                return;
            }

            OutputResults(request, failedList, passedList, headers, recordsAreWrappedInQuotes);
        }
        catch (Exception exception)
        {
            throw new Exception(@$"Exception thrown while trying to ExecuteRequestAction.
                                   ExceptionMessage: {exception.Message}
                                   Request: {JsonConvert.SerializeObject(request)}");
        }
    }

	private (List<IndexEvaluationRequest> Passed, List<IndexEvaluationRequest> Failed) BuildAndRunRules(List<IndexEvaluationRequest> indexEvaluationRequests, ExtractIndexFileDataRequest request)
	{
        if (request.EvaluationRule is null)
        {
            return (indexEvaluationRequests, new List<IndexEvaluationRequest>());
        }
        
        var passedRules = new List<IndexEvaluationRequest>();
        var failedRules = new List<IndexEvaluationRequest>();
        
        foreach (var indexEvaluationRequest in indexEvaluationRequests)
        {
            if (_runRuleService.BuildAndRunRule(new List<IndexEvaluationRequest> { indexEvaluationRequest }, request.EvaluationRule))
            {
                passedRules.Add(indexEvaluationRequest);
            }
            else
            {
                failedRules.Add(indexEvaluationRequest);
            }
        }

        return (passedRules, failedRules);
	}

	private static List<string> GetFileContents(string filepath)
        => File.ReadAllLines(filepath).Where(line => !string.IsNullOrWhiteSpace(line)).ToList();

    private static bool HasQuotes(List<string> content)
        => Regex.Split(content.First(), RegexMatch).Length > 1;
    
    private static List<string> GetHeaders(List<string> content, string delimiter, bool recordsAreWrappedInQuotes)
        => recordsAreWrappedInQuotes ? Regex.Split(content.First(), RegexMatch).Where(x => !new List<string> { delimiter, "\"" }.Contains(x) 
                                                                                            && !string.IsNullOrWhiteSpace(x)).ToList()
                                        : content.First().Split(delimiter).ToList();

    private static int GetColumnCount(List<string> content, string delimiter, bool recordsAreWrappedInQuotes)
    {
        var line = content.First();
        
        if (!recordsAreWrappedInQuotes)
        {
            return line.Split(delimiter).Length;
        }
        
        var filterList = new List<string> { string.Empty, "\"", delimiter };
        return Regex.Split(line, RegexMatch).Where(x => !filterList.Contains(x)).ToList().Count;
    }

    private static List<IndexEvaluationRequest> GetCsvRecords(string fileContent, string delimiter, bool hasHeaders)
    {
        using var stringReader = new StringReader(fileContent);
        var config = new CsvConfiguration(CultureInfo.InvariantCulture)
        {
            Delimiter = delimiter,
            MissingFieldFound = null,
            HasHeaderRecord = hasHeaders,
            BadDataFound = null
        };

        using var csvReader = new CsvReader(stringReader, config);
        return csvReader.GetRecords<IndexEvaluationRequest>().ToList();
    }
    
    private static List<List<string>> FilterColumns(List<IndexEvaluationRequest> records, 
                                                    List<int> indexesToExtract, 
                                                    int columnMax, 
                                                    bool extractAllIndexes)
    {
        if (!extractAllIndexes && (indexesToExtract is null || !indexesToExtract.Any()))
        {
            throw new Exception("IndexesToExtract was empty");
        }
        
        var keepProperties = new List<string>();
        var removeProperties = new List<string>();
        
        var indexes = typeof(IndexEvaluationRequest).GetProperties().ToDictionary(
                prop => ((IndexAttribute)prop.GetCustomAttributes(true).FirstOrDefault(attr => attr.GetType() == typeof(IndexAttribute)))?.Index ?? int.MaxValue,
                prop => prop.Name);
            
        var recordsDictionary = JsonConvert.DeserializeObject<List<Dictionary<string,string>>>(JsonConvert.SerializeObject(records));

        keepProperties.AddRange(indexes.Where(index => extractAllIndexes ? index.Key < columnMax : indexesToExtract.Contains(index.Key))
                                                                                                                   .Select(x => x.Value).Distinct());
        
        return recordsDictionary.Select(recordList => 
                    recordList.Where(record => 
                        (!keepProperties.Any() || keepProperties.Contains(record.Key)) 
                        && !removeProperties.Contains(record.Key)
                    ).Select(record => record.Value).ToList()
                ).ToList();
    }

    private static string TransformToOutputString(IEnumerable<List<string>> content, string delimiter, string wrapText)
        => string.Join(Environment.NewLine, content.Select(x => string.Join(delimiter, x.Select(y => $"{wrapText}{y}{wrapText}"))));

    private static void OutputResults(ExtractIndexFileDataRequest request, 
                                      IEnumerable<List<string>> originalContent, 
                                      IEnumerable<List<string>> newContent, 
                                      List<string> headers, 
                                      bool recordIsWrappedInQuotes)
    {
        var delimiter = request.DelimiterType.GetNameFromValue();
        var wrapText = recordIsWrappedInQuotes ? "\"" : string.Empty;
        var destinationFileExists = File.Exists(request.FinalDestinationIndexFilePath);

        if (request.IndexHasHeaders)
        {
            originalContent = originalContent.Prepend(headers);
            
            if (!request.AppendToDestinationIndexFile || !destinationFileExists)
            {
                newContent = !request.ExtractAllIndexes && request.IndexesToExtract.Count > 0 
                                    ? newContent.Prepend(headers.Where((_, i) => request.IndexesToExtract.Contains(i)).ToList())
                                    : newContent.Prepend(headers);
            }
        }

        var originalFile = TransformToOutputString(originalContent, delimiter, wrapText);
        var newFile = TransformToOutputString(newContent, delimiter, wrapText);
        
        if (request.AppendToDestinationIndexFile && destinationFileExists)
        {
            File.AppendAllText(request.FinalDestinationIndexFilePath, Environment.NewLine + newFile);
        }
        else
        {   
            File.WriteAllText(request.FinalDestinationIndexFilePath, newFile);
        }

        File.WriteAllText(request.FinalIndexFilePath, originalFile);
    }
}