431 lines
19 KiB
C#
431 lines
19 KiB
C#
using FileHelpers;
|
|
using LCU.Api.Internal.DataCleanUp.Configuration;
|
|
using LCU.Api.Internal.DataCleanUp.Models.ParserModels;
|
|
using LCU.Api.Internal.DataCleanUp.Services.Emailers;
|
|
using LCU.Api.Internal.DataCleanUp.Services.Validation;
|
|
using LCU.Library.Internal.FileOperationsModels.Models;
|
|
using LCU.Api.Internal.MaidBotLib.Models;
|
|
using LCU.Library.Internal.CoreApiWrapperModel.CoreModels.PostalAddresses;
|
|
using LCU.Library.Internal.DataCleanUpModel.Models;
|
|
using Newtonsoft.Json;
|
|
using Serilog;
|
|
using System.Globalization;
|
|
using System.Text.RegularExpressions;
|
|
using TimeZoneConverter;
|
|
|
|
namespace LCU.Api.Internal.DataCleanUp.Services;
|
|
|
|
public class UpdateNcoaProcessService : IUpdateNcoaProcessService
|
|
{
|
|
private readonly ICoreApiWrapperService _coreApiWrapperService;
|
|
private readonly IFileOperationsService _fileOperationsService;
|
|
private readonly NcoaConfiguration _ncoaConfiguration;
|
|
private readonly DataCleanUpConfiguration _dataCleanUpConfiguration;
|
|
private readonly INcoaValidationService _ncoaValidationService;
|
|
private readonly INotificationsApiService _notificationsApiService;
|
|
private readonly string _errorFileHeader = "ChangeDate|UserId|PersonNumber|Name|CurrentAddress1|CurrentAddress2|CurrentAddress3|CurrentAddress4|CurrentAddress5" +
|
|
"|New Address|City|State|Zip|ZipSuffix|ErrorMessage\n";
|
|
|
|
public UpdateNcoaProcessService(ICoreApiWrapperService coreApiWrapperService,
|
|
IFileOperationsService fileOperationsService,
|
|
NcoaConfiguration ncoaConfiguration,
|
|
DataCleanUpConfiguration dataCleanUpConfiguration,
|
|
INcoaValidationService ncoaValidationService,
|
|
INotificationsApiService notificationsApiService)
|
|
{
|
|
_coreApiWrapperService = coreApiWrapperService;
|
|
_fileOperationsService = fileOperationsService;
|
|
_ncoaConfiguration = ncoaConfiguration;
|
|
_dataCleanUpConfiguration = dataCleanUpConfiguration;
|
|
_ncoaValidationService = ncoaValidationService;
|
|
_notificationsApiService = notificationsApiService;
|
|
}
|
|
|
|
public async Task<DataCleanUpResult> CleanUp(DataCleanUpRequest dataCleanUpRequest)
|
|
{
|
|
Log.Information($"Starting Data CleanUp {dataCleanUpRequest.CleanUpType} at {DateTime.Now}");
|
|
var cleanUpResult = new DataCleanUpResult(dataCleanUpRequest, true, new List<string>());
|
|
var ncoaProcessRequest = JsonConvert.DeserializeObject<NcoaUpdateRequest>(dataCleanUpRequest.DataCleanUpPayload);
|
|
var ncoaUpdateResultList = new List<NcoaUpdateResult>();
|
|
var exclusionList = new List<string>();
|
|
|
|
try
|
|
{
|
|
var reader = new FileHelperEngine<NcoaRecord>();
|
|
|
|
var fileRequest = _fileOperationsService.BuildDynamicFileListRequest(ncoaProcessRequest);
|
|
var fileList = await _fileOperationsService.BuildDynamicFileList(fileRequest);
|
|
|
|
if (!fileList.Any())
|
|
{
|
|
var noFilesFound = "No files found for Update NCOA Process";
|
|
cleanUpResult.ResponseStack.Add(noFilesFound);
|
|
var emailSuccess = _notificationsApiService.SendEmail(ncoaProcessRequest.EmailSummaryRecipients, _ncoaConfiguration.SummaryEmailSubject, noFilesFound);
|
|
Log.Information($"Ending Data CleanUp {dataCleanUpRequest.CleanUpType} at {DateTime.Now}");
|
|
return cleanUpResult;
|
|
}
|
|
|
|
exclusionList = await GetExclusionList(ncoaProcessRequest.ExclusionFilePath);
|
|
|
|
foreach (var filename in fileList)
|
|
{
|
|
var updateResult = await HandleFileUpdate(filename, ncoaProcessRequest, reader, exclusionList);
|
|
|
|
ncoaUpdateResultList.Add(updateResult);
|
|
}
|
|
|
|
cleanUpResult = await HandleBatchCompletion(ncoaUpdateResultList, cleanUpResult, ncoaProcessRequest);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Log.Error(ex, "Error while updating NCOA Process.");
|
|
cleanUpResult.Successful = false;
|
|
cleanUpResult.ResponseStack.Add($"Error while updating NCOA Process. " +
|
|
$"Exception:{ex}");
|
|
}
|
|
|
|
Log.Information($"Ending Data CleanUp {dataCleanUpRequest.CleanUpType} at {DateTime.Now}");
|
|
return cleanUpResult;
|
|
}
|
|
|
|
public PersonAddrLine CheckForPoBoxAddressLineType(PersonAddrLine request)
|
|
{
|
|
if (request.AddressLine1.Contains(_ncoaConfiguration.PoBoxLabel))
|
|
{
|
|
request.AddressLineTypeCode1 = _ncoaConfiguration.PoBoxAddressTypeLabel;
|
|
}
|
|
|
|
return request;
|
|
}
|
|
|
|
public PersonAddrLine MapPersonAddrLine(int personNumber, PersonPostalAddress currentAddress, NcoaRecord ncoaRecord)
|
|
{
|
|
var currentDateTime = GetCurrentDateTimeString("yyyy-MM-ddTHH:mm:ss.fffK");
|
|
|
|
var coreNcoaRequest = new PersonAddrLine()
|
|
{
|
|
PersonNumber = personNumber,
|
|
Id = _ncoaConfiguration.PrimaryAddressLabel,
|
|
AddressUseCode = _ncoaConfiguration.PrimaryAddressLabel,
|
|
AddressNumber = currentAddress.AddressNumber,
|
|
AddressLine1 = ncoaRecord.NewAddressLine,
|
|
AddressLineTypeCode1 = _ncoaConfiguration.StreetTypeLabel,
|
|
AddressLine2 = null,
|
|
AddressLineTypeCode2 = null,
|
|
AddressLine3 = null,
|
|
AddressLineTypeCode3 = null,
|
|
CityName = ConvertToTitleCase(ncoaRecord.City.ToLower()),
|
|
CountrySubdivisionCode = ncoaRecord.StateCode,
|
|
PostalCode = ncoaRecord.ZipCode,
|
|
ZipPlus = ncoaRecord.ZipSuffix.Replace("-", ""),
|
|
CountryCode = _ncoaConfiguration.UsaLabel,
|
|
Url = _ncoaConfiguration.UrlBase.Replace(_ncoaConfiguration.PersonNumberLabel, personNumber.ToString()),
|
|
IsPerformIRSCorrection = false,
|
|
EffectiveDate = GetCurrentDateTimeString("yyyy-MM-dd")
|
|
};
|
|
|
|
coreNcoaRequest = CheckForPoBoxAddressLineType(coreNcoaRequest);
|
|
|
|
return coreNcoaRequest;
|
|
}
|
|
|
|
public async Task<bool> WriteNcoaErrorFile(string data, string destination)
|
|
{
|
|
try
|
|
{
|
|
var dateString = GetCurrentDateTimeString("yyyyMMdd");
|
|
|
|
var writeFileRequest = new WriteFileRequest()
|
|
{
|
|
DestinationPath = destination,
|
|
FileName = dateString + _ncoaConfiguration.ErrorFileDefaultName,
|
|
Extension = ".csv",
|
|
FileData = data,
|
|
Overwrite = false
|
|
};
|
|
|
|
await _fileOperationsService.WriteFile(writeFileRequest);
|
|
return true;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Log.Error(ex, "Exception while trying to WriteNCOA Error file");
|
|
return false;
|
|
}
|
|
}
|
|
|
|
public string GetCurrentDateTimeString(string format)
|
|
{
|
|
var assumedTimezone = TZConvert.GetTimeZoneInfo(_dataCleanUpConfiguration.TimeZone);
|
|
|
|
var currentDateTime = TimeZoneInfo.ConvertTime(DateTime.Now, assumedTimezone).Date;
|
|
|
|
return currentDateTime.ToString(format);
|
|
}
|
|
|
|
public string ConvertToTitleCase(string entry)
|
|
{
|
|
var textinfo = new CultureInfo("en-US", false).TextInfo;
|
|
|
|
return textinfo.ToTitleCase(entry);
|
|
}
|
|
|
|
public async Task<(DataCleanUpResult cleanUpResult, bool errorFileCreationSuccess)> CreateErrorFile(List<NcoaUpdateResult> updateResultList, string fileDestinationPath, DataCleanUpResult cleanUpResult)
|
|
{
|
|
var errorFileCreationSuccess = false;
|
|
var data = _errorFileHeader;
|
|
try
|
|
{
|
|
foreach (var updateResult in updateResultList)
|
|
{
|
|
if (updateResult.ErrorList.Count < 1)
|
|
{
|
|
cleanUpResult.ResponseStack.Add($"0 Error Records for file: {updateResult.Filename}");
|
|
continue;
|
|
}
|
|
|
|
foreach (var error in updateResult.ErrorList)
|
|
{
|
|
data += error;
|
|
}
|
|
}
|
|
|
|
errorFileCreationSuccess = await WriteNcoaErrorFile(data, fileDestinationPath);
|
|
|
|
if (!errorFileCreationSuccess)
|
|
{
|
|
throw new Exception("Affected Records: " + data);
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Log.ForContext(Enum.GetName(typeof(PropertyEnum), PropertyEnum.Group),
|
|
Enum.GetName(typeof(GroupEnum), GroupEnum.ITDevelopment))
|
|
.Error($"Error while writing NCOAError Record file. {ex.Message}");
|
|
cleanUpResult.ResponseStack.Add($"Error while writing {_ncoaConfiguration.ErrorFileDefaultName} file. Error file not produced. Data available in Logs.");
|
|
}
|
|
|
|
return (cleanUpResult, errorFileCreationSuccess);
|
|
}
|
|
|
|
public async Task<NcoaUpdateResult> ProcessNcoaRecord(NcoaRecord entry, NcoaUpdateResult updateResult,
|
|
NcoaUpdateRequest ncoaProcessRequest, List<string> exclusionList)
|
|
{
|
|
if (_ncoaValidationService.IsBusinessRecord(entry))
|
|
{
|
|
updateResult.AddStringToErrorList(entry, "Record appears to be a business account");
|
|
return updateResult;
|
|
}
|
|
|
|
var currentAgreement = await _coreApiWrapperService.GetAgreementByUserId(entry.UserId);
|
|
|
|
var personNumber = _ncoaValidationService.ParsePersonNumber(currentAgreement);
|
|
|
|
if (personNumber <= 0)
|
|
{
|
|
updateResult.AddStringToErrorList(entry, "Unable to parse or find PersonNumber from UserId");
|
|
return updateResult;
|
|
}
|
|
|
|
if (_ncoaValidationService.AgreementHasOrgNumber(currentAgreement))
|
|
{
|
|
updateResult.AddStringToErrorList(entry, "Record has an OrgNumber in Agreement from DNA");
|
|
return updateResult;
|
|
}
|
|
|
|
var personPostalAddressList = await _coreApiWrapperService.GetPersonPostalAddressByPersonNumber(personNumber);
|
|
|
|
var primaryAddress = _ncoaValidationService.IsAddrUseCodeFound(personPostalAddressList, _ncoaConfiguration.PrimaryAddressLabel);
|
|
|
|
if (primaryAddress == null)
|
|
{
|
|
updateResult.AddStringToErrorList(entry, "Unable to find a primary address by PersonNumber", personNumber.ToString());
|
|
return updateResult;
|
|
}
|
|
|
|
if (_ncoaValidationService.IsAddressUpdatedRecently(primaryAddress, ncoaProcessRequest.ExclusionDays, GetCurrentDateTimeString("yyyy-MM-dd")))
|
|
{
|
|
updateResult.AddStringToErrorList(entry, $"Primary Address has been updated within the last " +
|
|
$"{Math.Abs(ncoaProcessRequest.ExclusionDays).ToString()} days.", personNumber.ToString());
|
|
return updateResult;
|
|
}
|
|
|
|
var coreNcoaRequest = MapPersonAddrLine(personNumber, primaryAddress, entry);
|
|
|
|
if (_ncoaValidationService.IsAddrLineTooLong(coreNcoaRequest.AddressLine1))
|
|
{
|
|
updateResult.AddStringToErrorList(entry, "New Address Record is greater than 40 characters. Requires manual line split", personNumber.ToString());
|
|
return updateResult;
|
|
}
|
|
|
|
if(_ncoaValidationService.IsOnExclusionList(coreNcoaRequest.PersonNumber, exclusionList))
|
|
{
|
|
updateResult.AddStringToErrorList(entry, "Record matches an entry in exclusion list", personNumber.ToString());
|
|
return updateResult;
|
|
}
|
|
|
|
try
|
|
{
|
|
var CoreResult = await _coreApiWrapperService.UpdateAddrLineByPersonNumber(coreNcoaRequest);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
updateResult.AddStringToErrorList(entry, "Error while upating address in CoreApi", coreNcoaRequest.PersonNumber.ToString());
|
|
return updateResult;
|
|
}
|
|
|
|
updateResult.AddPersonAddrLineToUpdateList(coreNcoaRequest);
|
|
return updateResult;
|
|
}
|
|
|
|
public async Task<DataCleanUpResult> HandleBatchCompletion(List<NcoaUpdateResult> ncoaUpdateResultList,
|
|
DataCleanUpResult cleanUpResult, NcoaUpdateRequest ncoaProcessRequest)
|
|
{
|
|
var errorFileResult = await CreateErrorFile(ncoaUpdateResultList, ncoaProcessRequest.ErrorDestinationPath, cleanUpResult);
|
|
cleanUpResult = errorFileResult.cleanUpResult;
|
|
|
|
var totalBatchSummary = BuildTotalBatchSummary(ncoaUpdateResultList, errorFileResult.errorFileCreationSuccess);
|
|
|
|
var batchResultSummaryText = CreateTotalBatchSummaryString(totalBatchSummary, ncoaProcessRequest.ErrorDestinationPath);
|
|
|
|
foreach(var updateResult in totalBatchSummary.UpdateResults)
|
|
{
|
|
batchResultSummaryText += BuildFileUpdateSummary(updateResult);
|
|
}
|
|
|
|
var emailSuccess = _notificationsApiService.SendEmail(ncoaProcessRequest.EmailSummaryRecipients, _ncoaConfiguration.SummaryEmailSubject, batchResultSummaryText);
|
|
batchResultSummaryText += $"Summary Email Request Successful: {emailSuccess}";
|
|
|
|
Log.Information(batchResultSummaryText);
|
|
|
|
cleanUpResult.ResponseStack.Add(batchResultSummaryText);
|
|
return cleanUpResult;
|
|
}
|
|
|
|
public string CreateTotalBatchSummaryString(NcoaTotalBatchSummary summary, string errorFileLocation = "")
|
|
{
|
|
return summary.ErrorFileCreated ?
|
|
$"NCOA Update Total Batch Summary:<br><br>Total Files in Batch: {summary.UpdateResults.Count}<br><br>Total Records Received: " +
|
|
$"{summary.TotalRecordsFound}<br><br>Total Records Updated: {summary.TotalRecordsUpdated}<br><br>Total Error Records: " +
|
|
$"{summary.TotalErrorRecords}<br><br>Total Records Processed: {summary.TotalRecordsUpdated + summary.TotalErrorRecords}<br><br>" +
|
|
$"Error File Created: {summary.ErrorFileCreated}<br><br>Error File Location: {errorFileLocation}<br><br>"
|
|
:
|
|
$"NCOA Update Total Batch Summary:<br><br>Total Files in Batch: {summary.UpdateResults.Count}<br><br>Total Records Received: " +
|
|
$"{summary.TotalRecordsFound}<br><br>Total Records Updated: {summary.TotalRecordsUpdated}<br><br>Total Error Records: " +
|
|
$"{summary.TotalErrorRecords}<br><br>Total Records Processed: {summary.TotalRecordsUpdated + summary.TotalErrorRecords}<br><br>" +
|
|
$"Error File Created: {summary.ErrorFileCreated}<br><br>";
|
|
}
|
|
|
|
public NcoaTotalBatchSummary BuildTotalBatchSummary(List<NcoaUpdateResult> updateResults, bool errorFileCreationSuccess)
|
|
{
|
|
var totalBatchSummary = new NcoaTotalBatchSummary(updateResults, errorFileCreationSuccess);
|
|
|
|
foreach(var updateResult in totalBatchSummary.UpdateResults)
|
|
{
|
|
totalBatchSummary.TotalRecordsFound += updateResult.TotalRecordsFound;
|
|
totalBatchSummary.TotalRecordsUpdated += updateResult.UpdateList.Count;
|
|
totalBatchSummary.TotalErrorRecords += updateResult.ErrorList.Count;
|
|
}
|
|
|
|
return totalBatchSummary;
|
|
}
|
|
|
|
public string BuildFileUpdateSummary(NcoaUpdateResult updateResult)
|
|
{
|
|
return $"--------------<br><br>NCOA File In Batch Summary:<br><br>Filename: {updateResult.Filename}" +
|
|
$"<br><br>Total Records Received: {updateResult.TotalRecordsFound}<br><br>Total Records Updated: {updateResult.UpdateList.Count}" +
|
|
$"<br><br>Total Error Records: {updateResult.ErrorList.Count}<br><br>Total Records Processed: " +
|
|
$"{updateResult.UpdateList.Count + updateResult.ErrorList.Count}<br><br>";
|
|
}
|
|
|
|
public async Task<NcoaUpdateResult> HandleFileUpdate(string filename, NcoaUpdateRequest ncoaProcessRequest, FileHelperEngine<NcoaRecord> reader, List<string> exclusionList)
|
|
{
|
|
var updateResult = new NcoaUpdateResult(filename);
|
|
var fileData = await _fileOperationsService.GetContentData($"{ncoaProcessRequest.FinalSourcePath}{filename}");
|
|
var ncoaUpdates = reader.ReadString(fileData);
|
|
updateResult.TotalRecordsFound = ncoaUpdates.Length;
|
|
|
|
foreach (var entry in ncoaUpdates)
|
|
{
|
|
updateResult = await ProcessNcoaRecord(entry, updateResult, ncoaProcessRequest, exclusionList);
|
|
}
|
|
|
|
return updateResult;
|
|
}
|
|
|
|
public async Task<List<string>> GetExclusionList(string filepath)
|
|
{
|
|
var exclusionList = new List<string>();
|
|
try
|
|
{
|
|
var exclusionListFileData = await _fileOperationsService.GetContentData(filepath);
|
|
if (!string.IsNullOrWhiteSpace(exclusionListFileData))
|
|
{
|
|
exclusionList = exclusionListFileData.Split(_ncoaConfiguration.ExclusionListDelimiter).ToList();
|
|
}
|
|
|
|
return exclusionList;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
throw new Exception($"Error while trying to retrieve or split ExclusionList. Exception: {ex.Message}");
|
|
}
|
|
}
|
|
|
|
public struct NcoaUpdateResult
|
|
{
|
|
public List<PersonAddrLine> UpdateList { get; set; }
|
|
public List<string> ErrorList { get; set; }
|
|
public int TotalRecordsFound { get; set; }
|
|
public string Filename { get; set; }
|
|
|
|
public NcoaUpdateResult()
|
|
{
|
|
ErrorList = new List<string>();
|
|
UpdateList = new List<PersonAddrLine>();
|
|
TotalRecordsFound = 0;
|
|
Filename = null;
|
|
}
|
|
|
|
public NcoaUpdateResult(string filename)
|
|
{
|
|
ErrorList = new List<string>();
|
|
UpdateList = new List<PersonAddrLine>();
|
|
TotalRecordsFound = 0;
|
|
Filename = filename;
|
|
}
|
|
|
|
public void AddPersonAddrLineToUpdateList(PersonAddrLine entry)
|
|
{
|
|
UpdateList.Add(entry);
|
|
}
|
|
|
|
public void AddStringToErrorList(NcoaRecord record, string errorMessage, string personNumber = "")
|
|
{
|
|
var errorRecord = $"{record.ChangeDate}|{record.UserId}|{personNumber}|{record.PrimaryName}|{record.CurrentAddressOne}|" +
|
|
$"{record.CurrentAddressTwo}|{record.CurrentAddressThree}|{record.CurrentAddressFour}|{record.CurrentAddressFive}|" +
|
|
$"{record.NewAddressLine}|{record.City}|{record.StateCode}|{record.ZipCode}|{record.ZipSuffix.Replace("-", "")}|{errorMessage}\n";
|
|
|
|
ErrorList.Add(errorRecord);
|
|
}
|
|
}
|
|
|
|
public struct NcoaTotalBatchSummary
|
|
{
|
|
public List<NcoaUpdateResult> UpdateResults { get; set; }
|
|
public int TotalRecordsFound { get; set; }
|
|
public int TotalRecordsUpdated { get; set; }
|
|
public int TotalErrorRecords { get; set; }
|
|
public bool ErrorFileCreated { get; set; } = false;
|
|
|
|
public NcoaTotalBatchSummary(List<NcoaUpdateResult> updateResults, bool errorFileCreationSuccess)
|
|
{
|
|
UpdateResults = updateResults;
|
|
TotalErrorRecords = 0;
|
|
TotalRecordsUpdated = 0;
|
|
TotalRecordsFound = 0;
|
|
ErrorFileCreated = errorFileCreationSuccess;
|
|
}
|
|
}
|
|
}
|