using System; using System.Collections.Generic; using System.Globalization; using System.IO; using System.Linq; using System.Text.RegularExpressions; using CsvHelper; using CsvHelper.Configuration; using CsvHelper.Configuration.Attributes; using LCU.Api.Internal.FileOperations.Validations; using LCU.Api.Internal.FileOperations.Models; using LCU.Library.Internal.FileOperationsModels.Models; using LCU.Library.Internal.FileOperationsModels.Shared; using LCU.Library.Internal.LcuEvaluation.Services.EvaluationServices; using LCU.Library.Internal.LCUServiceFactory.Configurations; using LCU.Library.Internal.TypeExtentions.EnumExtensions; using Newtonsoft.Json; using Serilog; namespace LCU.Api.Internal.FileOperations.Services.FileActionServices; public class ExtractIndexFileDataService : FileActionService, IFileActionService, IActionService { private readonly IRunRuleService _runRuleService; private const string RegexMatch = "(\")(.*?)(\")"; public ExtractIndexFileDataService(IRunRuleService runRuleService, ServicesFactoryConfiguration servicesFactoryConfiguration) : base(servicesFactoryConfiguration) { _runRuleService = runRuleService; } public RequestResponse ExecuteAction(QueueRequest queueRequest) { var requestResponse = new RequestResponse(); try { var request = JsonConvert.DeserializeObject(queueRequest.Request); var requests = DynamicNameBuilder.BuildListOfRequest(request); if (requests.Count > 1) { requestResponse.MessageStack.Add("More than one file was found."); requestResponse.IsSuccessful = false; } else { requestResponse = PerformRequest(requests.FirstOrDefault()); } requestResponse.ReturnResponse = requests.FirstOrDefault(); return requestResponse; } catch (Exception exception) { return RequestResponseHelper.ExceptionUpdateRequestResponse(requestResponse, queueRequest.FileOperationType, exception.Message); } } public RequestResponse PerformRequest(ExtractIndexFileDataRequest request) { return OperationService.PerformOperation(request, IsRequestValid, ExtractIndexFileData); } public RequestResponse PreQueueValidation(ExtractIndexFileDataRequest originalRequest) { var requestResponse = new RequestResponse(); return requestResponse; } public RequestResponse IsRequestValid(ExtractIndexFileDataRequest request, RequestResponse requestResponse) { requestResponse = FileValidation.ValidateFileExists(request.FinalIndexFilePath, requestResponse); requestResponse = DirectoryValidation.ValidateDirectoryExists(request.FinalDestinationPath, requestResponse); return requestResponse; } private void ExtractIndexFileData(ExtractIndexFileDataRequest request) { try { var delimiter = request.DelimiterType.GetNameFromValue(); var fileLines = GetFileContents(request.FinalIndexFilePath); if (fileLines is null || !fileLines.Any()) { Log.Information($"No content was extracted from filepath: ({request.FinalIndexFilePath})"); return; } var recordsAreWrappedInQuotes = HasQuotes(fileLines); var headers = request.IndexHasHeaders ? GetHeaders(fileLines, delimiter, recordsAreWrappedInQuotes) : new List(); var columnMax = GetColumnCount(fileLines, delimiter, recordsAreWrappedInQuotes); var fileContent = string.Join(Environment.NewLine, fileLines); var indexEvaluationRequests = GetCsvRecords(fileContent, delimiter, request.IndexHasHeaders); var evaluatedRequests = BuildAndRunRules(indexEvaluationRequests, request); var passedList = FilterColumns(evaluatedRequests.Passed, request.IndexesToExtract, columnMax, request.ExtractAllIndexes); var failedList = FilterColumns(evaluatedRequests.Failed, request.IndexesToExtract, columnMax, true); if (passedList is null || !passedList.Any()) { Log.Information("No results passed evaluations to write to destination file."); return; } OutputResults(request, failedList, passedList, headers, recordsAreWrappedInQuotes); } catch (Exception exception) { throw new Exception(@$"Exception thrown while trying to ExecuteRequestAction. ExceptionMessage: {exception.Message} Request: {JsonConvert.SerializeObject(request)}"); } } private (List Passed, List Failed) BuildAndRunRules(List indexEvaluationRequests, ExtractIndexFileDataRequest request) { if (request.EvaluationRule is null) { return (indexEvaluationRequests, new List()); } var passedRules = new List(); var failedRules = new List(); foreach (var indexEvaluationRequest in indexEvaluationRequests) { if (_runRuleService.BuildAndRunRule(new List { indexEvaluationRequest }, request.EvaluationRule)) { passedRules.Add(indexEvaluationRequest); } else { failedRules.Add(indexEvaluationRequest); } } return (passedRules, failedRules); } private static List GetFileContents(string filepath) => File.ReadAllLines(filepath).Where(line => !string.IsNullOrWhiteSpace(line)).ToList(); private static bool HasQuotes(List content) => Regex.Split(content.First(), RegexMatch).Length > 1; private static List GetHeaders(List content, string delimiter, bool recordsAreWrappedInQuotes) => recordsAreWrappedInQuotes ? Regex.Split(content.First(), RegexMatch).Where(x => !new List { delimiter, "\"" }.Contains(x) && !string.IsNullOrWhiteSpace(x)).ToList() : content.First().Split(delimiter).ToList(); private static int GetColumnCount(List content, string delimiter, bool recordsAreWrappedInQuotes) { var line = content.First(); if (!recordsAreWrappedInQuotes) { return line.Split(delimiter).Length; } var filterList = new List { string.Empty, "\"", delimiter }; return Regex.Split(line, RegexMatch).Where(x => !filterList.Contains(x)).ToList().Count; } private static List GetCsvRecords(string fileContent, string delimiter, bool hasHeaders) { using var stringReader = new StringReader(fileContent); var config = new CsvConfiguration(CultureInfo.InvariantCulture) { Delimiter = delimiter, MissingFieldFound = null, HasHeaderRecord = hasHeaders, BadDataFound = null }; using var csvReader = new CsvReader(stringReader, config); return csvReader.GetRecords().ToList(); } private static List> FilterColumns(List records, List indexesToExtract, int columnMax, bool extractAllIndexes) { if (!extractAllIndexes && (indexesToExtract is null || !indexesToExtract.Any())) { throw new Exception("IndexesToExtract was empty"); } var keepProperties = new List(); var removeProperties = new List(); var indexes = typeof(IndexEvaluationRequest).GetProperties().ToDictionary( prop => ((IndexAttribute)prop.GetCustomAttributes(true).FirstOrDefault(attr => attr.GetType() == typeof(IndexAttribute)))?.Index ?? int.MaxValue, prop => prop.Name); var recordsDictionary = JsonConvert.DeserializeObject>>(JsonConvert.SerializeObject(records)); keepProperties.AddRange(indexes.Where(index => extractAllIndexes ? index.Key < columnMax : indexesToExtract.Contains(index.Key)) .Select(x => x.Value).Distinct()); return recordsDictionary.Select(recordList => recordList.Where(record => (!keepProperties.Any() || keepProperties.Contains(record.Key)) && !removeProperties.Contains(record.Key) ).Select(record => record.Value).ToList() ).ToList(); } private static string TransformToOutputString(IEnumerable> content, string delimiter, string wrapText) => string.Join(Environment.NewLine, content.Select(x => string.Join(delimiter, x.Select(y => $"{wrapText}{y}{wrapText}")))); private static void OutputResults(ExtractIndexFileDataRequest request, IEnumerable> originalContent, IEnumerable> newContent, List headers, bool recordIsWrappedInQuotes) { var delimiter = request.DelimiterType.GetNameFromValue(); var wrapText = recordIsWrappedInQuotes ? "\"" : string.Empty; var destinationFileExists = File.Exists(request.FinalDestinationIndexFilePath); if (request.IndexHasHeaders) { originalContent = originalContent.Prepend(headers); if (!request.AppendToDestinationIndexFile || !destinationFileExists) { newContent = !request.ExtractAllIndexes && request.IndexesToExtract.Count > 0 ? newContent.Prepend(headers.Where((_, i) => request.IndexesToExtract.Contains(i)).ToList()) : newContent.Prepend(headers); } } var originalFile = TransformToOutputString(originalContent, delimiter, wrapText); var newFile = TransformToOutputString(newContent, delimiter, wrapText); if (request.AppendToDestinationIndexFile && destinationFileExists) { File.AppendAllText(request.FinalDestinationIndexFilePath, Environment.NewLine + newFile); } else { File.WriteAllText(request.FinalDestinationIndexFilePath, newFile); } File.WriteAllText(request.FinalIndexFilePath, originalFile); } }