This commit is contained in:
Zeeshaun
2026-03-25 14:50:13 -05:00
parent 883009770f
commit b690921fd7
592 changed files with 46618 additions and 0 deletions
@@ -0,0 +1,189 @@
using LCU.API.Internal.DocumentIngestor.Services.MicroApiServices;
using LCU.API.Internal.DocumentIngestor.Utilities;
using LCU.Library.DocumentType.Enums;
using LCU.Library.DocumentType.Extensions;
using LCU.Library.DocumentType.Models.Requests.Attributes;
using LCU.Library.Internal.CoreApiWrapperModel.CoreModels.Accounts;
using LCU.Library.Internal.DocumentIngestorModels.Entities;
using LCU.Library.Internal.DocumentIngestorModels.Enums;
using LCU.Library.Internal.DocumentIngestorModels.Models;
using LCU.Library.Internal.Exceptions;
using LCU.Library.Internal.CoreApiWrapperModel.CoreModels.Mixed;
namespace LCU.API.Internal.DocumentIngestor.Services.AttributeDeterminingServices;
public class AccountOwnerDeterminingService : BaseOwnerDeterminingService, IAccountOwnerDeterminingService
{
private readonly ICoreApiWrapperService _coreApiWrapperService;
public AccountOwnerDeterminingService(ICoreApiWrapperService coreApiWrapperService)
{
_coreApiWrapperService = coreApiWrapperService;
}
public async Task<ErrorCaptureResult<AccountAttribute>> DetermineAccountOwnerAttribute(string entityName,
string accountNumber,
DateTime creationDate,
OwnerOverrideInfo ownerOverrideInfo)
{
var accountOwners = await _coreApiWrapperService.GetOwnersByAccountNumber(accountNumber, creationDate.ToShortDateString());
var accountOwnerAttribute = BuildAccountAttribute(accountNumber, accountOwners, ownerOverrideInfo);
if (accountOwnerAttribute.PersonNumbers?.Count == 0 &&
accountOwnerAttribute.OrganizationNumbers?.Count == 0)
{
var message =
$"A PersonNumber or OrganizationNumber could not be determined for an Account Owner of AccountNumber: {accountNumber} and EntityName: {entityName}. " +
"The DNA call provided no results back.";
return ErrorCaptureResult<AccountAttribute>.CreateErrorPayload(message, accountOwnerAttribute);
}
if (ownerOverrideInfo != null)
{
return ErrorCaptureResult<AccountAttribute>.CreateSuccessPayload(accountOwnerAttribute);
}
return CheckPrimaryOwner(entityName, accountOwnerAttribute, accountOwners);
}
public async Task<ErrorCaptureResult<AccountAttribute>> DetermineAccountOwnerAttributeByMemberNumberOrUserId(string memberNumberOrUserId,
string accountNumber,
string docType,
DateTime creationDate,
OwnerOverrideInfo ownerOverrideInfo)
{
var accountOwners = await _coreApiWrapperService.GetOwnersByAccountNumber(accountNumber, creationDate.ToShortDateString());
var accountOwnerAttribute = BuildAccountAttribute(accountNumber, accountOwners, ownerOverrideInfo);
if (accountOwnerAttribute.PersonNumbers?.Count == 0 &&
accountOwnerAttribute.OrganizationNumbers?.Count == 0)
{
var message =
$"A PersonNumber or OrganizationNumber could not be determined for an Account Owner of AccountNumber: {accountNumber} and MemberNumberOrUserId: {memberNumberOrUserId}. " +
"The DNA call provided no results back.";
return ErrorCaptureResult<AccountAttribute>.CreateErrorPayload(message, accountOwnerAttribute);
}
if (ownerOverrideInfo != null)
{
return ErrorCaptureResult<AccountAttribute>.CreateSuccessPayload(accountOwnerAttribute);
}
long personNumber = 0;
long orgNumber = 0;
if (
EnumExtensions.GetValueFromName<AccountDocumentTypes>(docType) != AccountDocumentTypes.MtgNtOneHundredTen && EnumExtensions.GetValueFromName<AccountDocumentTypes>(docType) != AccountDocumentTypes.OpsNtOneHundredTwentyNine && EnumExtensions.GetValueFromName<AccountDocumentTypes>(docType) != AccountDocumentTypes.DmdNtOneHundredThirtyFour && EnumExtensions.GetValueFromName<AccountDocumentTypes>(docType) != AccountDocumentTypes.NegativeBalanceLetter ||
memberNumberOrUserId.ToUpper().StartsWith("M") &&
(EnumExtensions.GetValueFromName<AccountDocumentTypes>(docType) == AccountDocumentTypes.MtgNtOneHundredTen || EnumExtensions.GetValueFromName<AccountDocumentTypes>(docType) == AccountDocumentTypes.OpsNtOneHundredTwentyNine || EnumExtensions.GetValueFromName<AccountDocumentTypes>(docType) == AccountDocumentTypes.DmdNtOneHundredThirtyFour || EnumExtensions.GetValueFromName<AccountDocumentTypes>(docType) == AccountDocumentTypes.NegativeBalanceLetter)
)
{
try
{
PersonOrganization personOrganization = await _coreApiWrapperService.GetPersonOrOrganizationNumberByMemberId(memberNumberOrUserId.ToUpper().TrimStart('M'));
if(personOrganization.PersonNumber != null)
{
personNumber = long.Parse(personOrganization.PersonNumber);
}
if(personOrganization.OrganizationNumber != null)
{
orgNumber = long.Parse(personOrganization.OrganizationNumber);
}
}
catch (ServiceException ex)
{
return ErrorCaptureResult<AccountAttribute>.CreateErrorPayload(ex.Message, accountOwnerAttribute);
}
}
else
{
try
{
var agreementByMemberNumberOrUserId = await _coreApiWrapperService.GetAgreementByUserId(memberNumberOrUserId);
personNumber = agreementByMemberNumberOrUserId.PersonNumber;
orgNumber = agreementByMemberNumberOrUserId.ByOrgNbr;
}
catch (Exception ex)
{
return ErrorCaptureResult<AccountAttribute>.CreateErrorPayload(ex.Message, accountOwnerAttribute);
}
}
try
{
var agreementAccount = await _coreApiWrapperService.GetAgreementAccountByAccountId(accountNumber);
if (personNumber != agreementAccount.TaxReportForPersonNumber && orgNumber != agreementAccount.TaxReportForOrgNumber)
{
var message =
$"Person numbers ({personNumber}/{agreementAccount.TaxReportForPersonNumber}) and Organization numbers ({orgNumber}/{agreementAccount.TaxReportForOrgNumber}) did not match from GetAgreementAccountByAccountId for Account Number: {accountNumber}";
return ErrorCaptureResult<AccountAttribute>.CreateErrorPayload(message, accountOwnerAttribute);
}
return ErrorCaptureResult<AccountAttribute>.CreateSuccessPayload(accountOwnerAttribute);
}
catch (Exception ex)
{
return ErrorCaptureResult<AccountAttribute>.CreateErrorPayload(ex.Message, accountOwnerAttribute);
}
}
private AccountAttribute BuildAccountAttribute(string accountNumber, IEnumerable<AccountOwnership> accountOwnerships, OwnerOverrideInfo? ownerOverrideInfo)
{
var personNumbers = new List<string>();
var organizationNumbers = new List<string>();
if (ownerOverrideInfo != null)
{
switch (ownerOverrideInfo.EntityType)
{
case EntityTypes.Person:
personNumbers.Add(ownerOverrideInfo.EntityId);
break;
case EntityTypes.Organization:
organizationNumbers.Add(ownerOverrideInfo.EntityId);
break;
}
}
personNumbers.AddRange(accountOwnerships.Where(person => person.PersonNumber > 0)
.Select(p => p.PersonNumber.ToString()));
organizationNumbers.AddRange(accountOwnerships.Where(org => org.ByOrgNbr > 0)
.Select(o => o.ByOrgNbr.ToString()));
return new AccountAttribute
{
AccountNumber = accountNumber,
PersonNumbers = personNumbers.Distinct().ToList(),
OrganizationNumbers = organizationNumbers.Distinct().ToList()
};
}
private static ErrorCaptureResult<AccountAttribute> CheckPrimaryOwner(string entityName, AccountAttribute accountAttribute, IEnumerable<AccountOwnership> accountOwnerships)
{
var primaryOwner = accountOwnerships.FirstOrDefault(x => x.IsPrimaryOwner);
if (primaryOwner == null)
{
var message = $"A PersonNumber or OrganizationNumber could not be determined for an Account Owner of AccountNumber: {accountAttribute.AccountNumber} and EntityName: {entityName}. " +
$"The DNA call provided no primary owner back, but responded with {accountOwnerships.ToDetailString()}";
return ErrorCaptureResult<AccountAttribute>.CreateErrorPayload(message, accountAttribute);
}
var doesReturnNameMatch = primaryOwner.PersonNumber > 0
? DoesPersonNameMatchExpectedEntityName(primaryOwner, entityName)
: DoesNameMatchExpectedEntityName(primaryOwner.OrganizationName, entityName);
if (!doesReturnNameMatch)
{
var message = $"A PersonNumber or OrganizationNumber could not be determined for an Account Owner of AccountNumber: {accountAttribute.AccountNumber} and EntityName: {entityName}. " +
$"The DNA call provided a primary owner of: {primaryOwner.ToDetailString()}, which did not match the EntityName";
return ErrorCaptureResult<AccountAttribute>.CreateErrorPayload(message, accountAttribute);
}
return ErrorCaptureResult<AccountAttribute>.CreateSuccessPayload(accountAttribute);
}
}
@@ -0,0 +1,42 @@
using System.Text.RegularExpressions;
using LCU.Library.Internal.CoreApiWrapperModel.CoreModels.Accounts;
namespace LCU.API.Internal.DocumentIngestor.Services.AttributeDeterminingServices;
public abstract class BaseOwnerDeterminingService
{
protected static bool DoesPersonNameMatchExpectedEntityName(AccountOwnership primaryOwner, string entityName)
{
var trimmedFirstName = primaryOwner.FirstName?.Trim();
var trimmedMiddleName = primaryOwner.MiddleName?.Trim();
var trimmedMiddleInitial = primaryOwner.MiddleInitial?.Trim();
var trimmedLastName = primaryOwner.LastName?.Trim();
var trimmedSuffix = primaryOwner.Suffix?.Trim();
var fullNameMiddleInitial = $"{trimmedFirstName} {trimmedMiddleInitial} {trimmedLastName} {trimmedSuffix}";
var fullNameMiddleName = $"{trimmedFirstName} {trimmedMiddleName} {trimmedLastName} {trimmedSuffix}";
var fullNameMiddleInitialNoSuffix = $"{trimmedFirstName} {trimmedMiddleInitial} {trimmedLastName}";
var fullNameMiddleNameNoSuffix = $"{trimmedFirstName} {trimmedMiddleName} {trimmedLastName}";
var firstNameLastName = $"{trimmedFirstName} {trimmedLastName}";
return DoesNameMatchExpectedEntityName(entityName, fullNameMiddleInitial) ||
DoesNameMatchExpectedEntityName(entityName, fullNameMiddleName) ||
DoesNameMatchExpectedEntityName(entityName, fullNameMiddleInitialNoSuffix) ||
DoesNameMatchExpectedEntityName(entityName, fullNameMiddleNameNoSuffix) ||
DoesNameMatchExpectedEntityName(entityName, firstNameLastName);
}
protected static bool DoesNameMatchExpectedEntityName(string name, string expectedName)
{
var trimmedOrganizationName = RemoveSpecialCharactersAndSpacing(name);
return RemoveSpecialCharactersAndSpacing(expectedName).Equals(trimmedOrganizationName, StringComparison.InvariantCultureIgnoreCase);
}
protected static string RemoveSpecialCharactersAndSpacing(string name)
{
var nonSpecialString = string.IsNullOrWhiteSpace(name)
? ""
: Regex.Replace(name.Trim(), @"[^0-9a-zA-Z\-\&\s*]+", "");
var nonSpecialStringSpacing = nonSpecialString.Trim().Replace(" ", " ");
return nonSpecialStringSpacing.Length > 39 ? nonSpecialStringSpacing[..40].Trim() : nonSpecialStringSpacing;
}
}
@@ -0,0 +1,18 @@
using LCU.Library.DocumentType.Models.Requests.Attributes;
using LCU.Library.Internal.DocumentIngestorModels.Entities;
using LCU.Library.Internal.DocumentIngestorModels.Models;
namespace LCU.API.Internal.DocumentIngestor.Services.AttributeDeterminingServices;
public interface IAccountOwnerDeterminingService
{
Task<ErrorCaptureResult<AccountAttribute>> DetermineAccountOwnerAttribute(string entityName,
string accountNumber,
DateTime creationDate,
OwnerOverrideInfo overrideRequest);
Task<ErrorCaptureResult<AccountAttribute>> DetermineAccountOwnerAttributeByMemberNumberOrUserId(string id,
string accountNumber,
string docType,
DateTime creationDate,
OwnerOverrideInfo ownerOverrideInfo);
}
@@ -0,0 +1,28 @@
using LCU.Library.DocumentType.Models.Requests.Attributes;
using LCU.Library.Internal.DocumentIngestorModels.Entities;
using LCU.Library.Internal.DocumentIngestorModels.Models;
namespace LCU.API.Internal.DocumentIngestor.Services.AttributeDeterminingServices;
public interface IPrimaryOwnerDeterminingService
{
Task<ErrorCaptureResult<PrimaryOwnerAttribute>> DeterminePrimaryOwnerAttribute(string entityName,
string accountNumber,
string taxId,
OwnerOverrideInfo overrideRequest);
Task<ErrorCaptureResult<PrimaryOwnerAttribute>> DeterminePrimaryOwnerAttribute(string entityName,
string accountNumber,
DateTime creationDate,
OwnerOverrideInfo overrideRequest);
Task<ErrorCaptureResult<PrimaryOwnerAttribute>> DeterminePrimaryOwnerAttributeByMemberNumberOrUserId(string id,
string accountNumber,
string docType,
DateTime creationDate,
OwnerOverrideInfo overrideRequest);
Task<ErrorCaptureResult<PrimaryOwnerAttribute>> DeterminePrimaryOwnerAttributeByMemberNumberOrUserId(string memberNumberOrUserId,
string entityName,
OwnerOverrideInfo overrideRequest);
}
@@ -0,0 +1,418 @@
using LCU.API.Internal.DocumentIngestor.Services.MicroApiServices;
using LCU.API.Internal.DocumentIngestor.Utilities;
using LCU.Library.DocumentType.Enums;
using LCU.Library.DocumentType.Extensions;
using LCU.Library.DocumentType.Models.Requests.Attributes;
using LCU.Library.Internal.CoreApiWrapperModel.CoreModels.Agreements;
using LCU.Library.Internal.CoreApiWrapperModel.CoreModels.Mixed;
using LCU.Library.Internal.CoreApiWrapperModel.CoreModels.Tax;
using LCU.Library.Internal.DocumentIngestorModels.Entities;
using LCU.Library.Internal.DocumentIngestorModels.Enums;
using LCU.Library.Internal.DocumentIngestorModels.Models;
using LCU.Library.Internal.Exceptions;
namespace LCU.API.Internal.DocumentIngestor.Services.AttributeDeterminingServices;
public class PrimaryOwnerDeterminingService : BaseOwnerDeterminingService, IPrimaryOwnerDeterminingService
{
private readonly ICoreApiWrapperService _coreApiWrapperService;
public PrimaryOwnerDeterminingService(ICoreApiWrapperService coreApiWrapperService)
{
_coreApiWrapperService = coreApiWrapperService;
}
public async Task<ErrorCaptureResult<PrimaryOwnerAttribute>> DeterminePrimaryOwnerAttribute(string entityName,
string accountNumber,
DateTime creationDate,
OwnerOverrideInfo overrideRequest)
{
if (overrideRequest != null)
{
return await OverridePrimaryOwnerAttribute(accountNumber, overrideRequest);
}
var primaryOwnerAttribute = new PrimaryOwnerAttribute
{
AccountNumber = accountNumber
};
var accountOwnerships = await _coreApiWrapperService.GetOwnersByAccountNumber(accountNumber, creationDate.ToShortDateString());
if (!accountOwnerships.Any())
{
var message = $"A PersonNumber or OrganizationNumber could not be determined for a Primary Owner of AccountNumber: {accountNumber} and EntityName: {entityName}. " +
"The DNA call provided no results back.";
return ErrorCaptureResult<PrimaryOwnerAttribute>.CreateErrorPayload(message, primaryOwnerAttribute);
}
var primaryOwners = accountOwnerships.Where(account => account.IsPrimaryOwner).ToList();
if (primaryOwners.Count != 1)
{
var message = $"A PersonNumber or OrganizationNumber could not be determined for an Account Owner of AccountNumber: {accountNumber} and EntityName: {entityName}. " +
$"The DNA call provided no single primary owner back, but responded with {accountOwnerships.ToDetailString()}";
return ErrorCaptureResult<PrimaryOwnerAttribute>.CreateErrorPayload(message, primaryOwnerAttribute);
}
var primaryOwner = primaryOwners.First();
var doesReturnNameMatch = primaryOwner.PersonNumber > 0
? DoesPersonNameMatchExpectedEntityName(primaryOwner, entityName)
: DoesNameMatchExpectedEntityName(primaryOwner.OrganizationName, entityName);
if (!doesReturnNameMatch)
{
var message = $"A PersonNumber or OrganizationNumber could not be determined for an Account Owner of AccountNumber: {accountNumber} and EntityName: {entityName}. " +
$"The DNA call provided a primary owner of: {primaryOwner.ToDetailString()}, which did not match the EntityName";
return ErrorCaptureResult<PrimaryOwnerAttribute>.CreateErrorPayload(message, primaryOwnerAttribute);
}
if (primaryOwner.PersonNumber > 0)
{
primaryOwnerAttribute.PersonNumber = primaryOwner.PersonNumber.ToString();
}
else
{
primaryOwnerAttribute.OrganizationNumber = primaryOwner.ByOrgNbr.ToString();
}
return ErrorCaptureResult<PrimaryOwnerAttribute>.CreateSuccessPayload(primaryOwnerAttribute);
}
public async Task<ErrorCaptureResult<PrimaryOwnerAttribute>> DeterminePrimaryOwnerAttribute(string entityName,
string accountNumber,
string taxId,
OwnerOverrideInfo overrideRequest)
{
if (overrideRequest != null)
{
return await OverridePrimaryOwnerAttribute(accountNumber, overrideRequest);
}
var primaryOwnerAttribute = new PrimaryOwnerAttribute
{
AccountNumber = accountNumber
};
var personsTaxIdData = await _coreApiWrapperService.GetPersonsTaxIdData(taxId);
var personTaxIdDataResponses = personsTaxIdData.ToList();
if (DetermineIfPrimaryOwnerIsPerson(entityName, personTaxIdDataResponses, primaryOwnerAttribute, out var errorCaptureResult))
{
return errorCaptureResult;
}
var organizationTaxIdData = await _coreApiWrapperService.GetOrganizationsTaxIdData(taxId);
var organizationTaxIdDataResponses = organizationTaxIdData.ToList();
if (DetermineIfPrimaryOwnerIsOrganization(entityName, organizationTaxIdDataResponses, primaryOwnerAttribute, out var successPayload))
{
return successPayload;
}
if (!organizationTaxIdDataResponses.Any() && !personTaxIdDataResponses.Any())
{
var message = $"A PersonNumber or OrganizationNumber could not be determined for an Account Owner of AccountNumber: {accountNumber}, Tin: {taxId} and EntityName: {entityName}. " +
"The DNA call provided no results back.";
return ErrorCaptureResult<PrimaryOwnerAttribute>.CreateErrorPayload(message, primaryOwnerAttribute);
}
var errorMessage = $"A PersonNumber or OrganizationNumber could not be determined for an Account Owner of AccountNumber: {accountNumber}, Tin: {taxId} and EntityName: {entityName}. " +
"The DNA call provided no primary owner back, but responded with " +
string.Join
(
"; ",
personTaxIdDataResponses.ToDetailString(),
organizationTaxIdDataResponses.ToDetailString()
);
return ErrorCaptureResult<PrimaryOwnerAttribute>.CreateErrorPayload(errorMessage, primaryOwnerAttribute);
}
public async Task<ErrorCaptureResult<PrimaryOwnerAttribute>> DeterminePrimaryOwnerAttributeByMemberNumberOrUserId(string memberNumberOrUserId,
string accountNumber,
string docType,
DateTime creationDate,
OwnerOverrideInfo overrideRequest)
{
if (overrideRequest != null)
{
return await OverridePrimaryOwnerAttribute(accountNumber, overrideRequest);
}
var primaryOwnerAttribute = new PrimaryOwnerAttribute()
{
AccountNumber = accountNumber
};
Agreement? agreementByMemberNumberOrUserId;
long personNumber = 0;
long orgNumber = 0;
if (
memberNumberOrUserId.ToUpper().StartsWith("M") ||
EnumExtensions.GetValueFromName<PrimaryOwnerDocumentTypes>(docType) == PrimaryOwnerDocumentTypes.MmStmxt
)
{
try
{
PersonOrganization personOrganization = await _coreApiWrapperService.GetPersonOrOrganizationNumberByMemberId(memberNumberOrUserId.ToUpper().TrimStart('M'));
if (personOrganization.PersonNumber != null)
{
personNumber = long.Parse(personOrganization.PersonNumber);
}
else if (personOrganization.OrganizationNumber != null)
{
orgNumber = long.Parse(personOrganization.OrganizationNumber);
}
else
{
return ErrorCaptureResult<PrimaryOwnerAttribute>.CreateErrorPayload($"Error finding PersonOrganization by MemberId: {memberNumberOrUserId}", primaryOwnerAttribute);
}
}
catch (ServiceException ex)
{
return ErrorCaptureResult<PrimaryOwnerAttribute>.CreateErrorPayload(ex.Message, primaryOwnerAttribute);
}
}
else
{
try
{
agreementByMemberNumberOrUserId = await _coreApiWrapperService.GetAgreementByUserId(memberNumberOrUserId);
if (agreementByMemberNumberOrUserId == null)
{
var message =
$"An agreement could not be found from GetAgreementByUserId for User ID: {memberNumberOrUserId}";
return ErrorCaptureResult<PrimaryOwnerAttribute>.CreateErrorPayload(message, primaryOwnerAttribute);
}
personNumber = agreementByMemberNumberOrUserId.PersonNumber;
orgNumber = agreementByMemberNumberOrUserId.ByOrgNbr;
} catch (ServiceException ex)
{
return ErrorCaptureResult<PrimaryOwnerAttribute>.CreateErrorPayload(ex.Message, primaryOwnerAttribute);
}
}
if (EnumExtensions.GetValueFromName<PrimaryOwnerDocumentTypes>(docType) == PrimaryOwnerDocumentTypes.MmStmxt)
{
try
{
var agreementAccount = await _coreApiWrapperService.GetAgreementAccountByAccountId(accountNumber);
if (agreementAccount == null)
{
var message =
$"An agreement account could not be found from GetAgreementAccountByAccountId for Account Number: {accountNumber}";
return ErrorCaptureResult<PrimaryOwnerAttribute>.CreateErrorPayload(message, primaryOwnerAttribute);
}
if (personNumber != agreementAccount.TaxReportForPersonNumber && orgNumber != agreementAccount.TaxReportForOrgNumber)
{
var message =
$"Person numbers ({personNumber}/{agreementAccount.TaxReportForPersonNumber}) and Organization numbers ({orgNumber}/{agreementAccount.TaxReportForOrgNumber}) did not match from GetAgreementAccountByAccountId for Account Number: {accountNumber}";
return ErrorCaptureResult<PrimaryOwnerAttribute>.CreateErrorPayload(message, primaryOwnerAttribute);
}
}
catch (ServiceException ex)
{
return ErrorCaptureResult<PrimaryOwnerAttribute>.CreateErrorPayload(ex.Message, primaryOwnerAttribute);
}
}
if (orgNumber == 0)
{
primaryOwnerAttribute.PersonNumber = personNumber.ToString();
}
else
{
primaryOwnerAttribute.OrganizationNumber = orgNumber.ToString();
}
return ErrorCaptureResult<PrimaryOwnerAttribute>.CreateSuccessPayload(primaryOwnerAttribute);
}
public async Task<ErrorCaptureResult<PrimaryOwnerAttribute>> DeterminePrimaryOwnerAttributeByMemberNumberOrUserId(string memberNumberOrUserId,
string entityName,
OwnerOverrideInfo overrideRequest)
{
var primaryOwnerAttribute = new PrimaryOwnerAttribute();
if (overrideRequest != null)
{
return await OverridePrimaryOwnerAttribute(null, overrideRequest);
}
Agreement? agreementByMemberNumberOrUserId;
var returnedName = string.Empty;
long personNumber = 0;
long orgNumber = 0;
if (memberNumberOrUserId.ToUpper().StartsWith("M") || memberNumberOrUserId.ToUpper().StartsWith("C"))
{
try
{
PersonOrganization personOrganization = await _coreApiWrapperService.GetPersonOrOrganizationNumberByMemberId(memberNumberOrUserId.ToUpper().TrimStart('M').TrimStart('C'));
if (personOrganization.PersonNumber != null)
{
personNumber = long.Parse(personOrganization.PersonNumber);
returnedName = personOrganization.EntityName;
}
else if (personOrganization.OrganizationNumber != null)
{
orgNumber = long.Parse(personOrganization.OrganizationNumber);
returnedName = personOrganization.EntityName;
}
else
{
return ErrorCaptureResult<PrimaryOwnerAttribute>.CreateErrorPayload($"Error finding PersonOrganization by MemberId: {memberNumberOrUserId}", primaryOwnerAttribute);
}
}
catch (ServiceException ex)
{
return ErrorCaptureResult<PrimaryOwnerAttribute>.CreateErrorPayload(ex.Message, primaryOwnerAttribute);
}
}
else
{
try
{
agreementByMemberNumberOrUserId = await _coreApiWrapperService.GetAgreementByUserId(memberNumberOrUserId);
if (agreementByMemberNumberOrUserId == null)
{
// Checking to make sure UserId is not improperly mapped memberNumber
PersonOrganization personOrganization = await _coreApiWrapperService.GetPersonOrOrganizationNumberByMemberId(memberNumberOrUserId.ToUpper().TrimStart('M').TrimStart('C'));
if (personOrganization.PersonNumber != null)
{
personNumber = long.Parse(personOrganization.PersonNumber);
returnedName = personOrganization.EntityName;
}
else if (personOrganization.OrganizationNumber != null)
{
orgNumber = long.Parse(personOrganization.OrganizationNumber);
returnedName = personOrganization.EntityName;
}
else
{
var message =
$"An agreement could not be found from GetAgreementByUserId for User ID: {memberNumberOrUserId}";
return ErrorCaptureResult<PrimaryOwnerAttribute>.CreateErrorPayload(message, primaryOwnerAttribute);
}
}
else
{
personNumber = agreementByMemberNumberOrUserId.PersonNumber;
orgNumber = agreementByMemberNumberOrUserId.ByOrgNbr;
if (orgNumber == 0)
{
returnedName = $"{agreementByMemberNumberOrUserId.FirstName} {agreementByMemberNumberOrUserId.MiddleInitial} {agreementByMemberNumberOrUserId.LastName} {agreementByMemberNumberOrUserId.Suffix}";
}
else
{
returnedName = agreementByMemberNumberOrUserId.OrganizationName;
}
}
} catch (ServiceException ex)
{
return ErrorCaptureResult<PrimaryOwnerAttribute>.CreateErrorPayload(ex.Message, primaryOwnerAttribute);
}
}
if (!DoesNameMatchExpectedEntityName(returnedName, entityName))
{
var message =
$"The entityName ({entityName}) did not match from GetAgreementAccountByAccountId for returnedName: {returnedName}";
return ErrorCaptureResult<PrimaryOwnerAttribute>.CreateErrorPayload(message, primaryOwnerAttribute);
}
if (orgNumber == 0)
{
primaryOwnerAttribute.PersonNumber = personNumber.ToString();
}
else
{
primaryOwnerAttribute.OrganizationNumber = orgNumber.ToString();
}
return ErrorCaptureResult<PrimaryOwnerAttribute>.CreateSuccessPayload(primaryOwnerAttribute);
}
private async Task<ErrorCaptureResult<PrimaryOwnerAttribute>> OverridePrimaryOwnerAttribute(string accountNumber, OwnerOverrideInfo request)
{
var primaryOwnerAttribute = new PrimaryOwnerAttribute()
{
AccountNumber = accountNumber,
PersonNumber = request.EntityType == EntityTypes.Person ? request.EntityId : null,
OrganizationNumber = request.EntityType == EntityTypes.Organization ? request.EntityId : null,
};
return ErrorCaptureResult<PrimaryOwnerAttribute>.CreateSuccessPayload(primaryOwnerAttribute);
}
private static bool DetermineIfPrimaryOwnerIsPerson(string entityName,
IReadOnlyCollection<PersonTaxIdDataResponse> personTaxIdDataResponses,
PrimaryOwnerAttribute primaryOwnerAttribute,
out ErrorCaptureResult<PrimaryOwnerAttribute>? errorCaptureResult)
{
errorCaptureResult = null;
if (!personTaxIdDataResponses.Any())
{
return false;
}
var personNumber = personTaxIdDataResponses
.Where(person => DoesNameMatchExpectedEntityName(person.NameLastFirstMiddleSuffix, entityName))
.Select(p => p.PersonNumber).FirstOrDefault();
var personNumberString = personNumber <= 0 ? null : personNumber.ToString();
if (string.IsNullOrEmpty(personNumberString))
{
return false;
}
primaryOwnerAttribute.PersonNumber = personNumberString;
errorCaptureResult = ErrorCaptureResult<PrimaryOwnerAttribute>.CreateSuccessPayload(primaryOwnerAttribute);
return true;
}
private static bool DetermineIfPrimaryOwnerIsOrganization(string entityName,
IReadOnlyCollection<OrganizationTaxIdDataResponse> organizationTaxIdDataResponses,
PrimaryOwnerAttribute primaryOwnerAttribute,
out ErrorCaptureResult<PrimaryOwnerAttribute>? successPayload)
{
successPayload = null;
if (!organizationTaxIdDataResponses.Any())
{
return false;
}
var organizationNumber = organizationTaxIdDataResponses
.Where(org => DoesNameMatchExpectedEntityName(org.Name, entityName))
.Select(p => p.OrganizationNumber).FirstOrDefault();
var organizationNumberString = organizationNumber != 0 ? organizationNumber.ToString() : null;
if (string.IsNullOrEmpty(organizationNumberString))
{
return false;
}
primaryOwnerAttribute.OrganizationNumber = organizationNumberString;
successPayload = ErrorCaptureResult<PrimaryOwnerAttribute>.CreateSuccessPayload(primaryOwnerAttribute);
return true;
}
}
@@ -0,0 +1,638 @@
using System.Data.SqlClient;
using Serilog;
using LCU.Library.Internal.DocumentIngestorModels.Entities;
using LCU.Library.Internal.Exceptions;
using LCU.API.Internal.DocumentIngestorLib.Repositories;
using LCU.Library.Internal.DocumentIngestorModels.Enums;
using LCU.Library.Internal.DocumentIngestorModels.Models;
using LCU.Library.Internal.DocumentIngestorModels.Models.Pagination;
using LCU.Api.Internal.MaidBotLib.Models;
using LCU.Library.Internal.DocumentIngestorModels.Constants;
namespace LCU.API.Internal.DocumentIngestor.Services;
public class DocumentProcessingInfoService : IDocumentProcessingInfoService
{
private readonly IUnitOfWork _unitOfWork;
private readonly IDocumentProcessingInfoRepository _documentProcessingInfoRepository;
private readonly IProcessingLogService _processingLogService;
private readonly Dictionary<ProcessStatusTypes, List<ProcessStatusTypes>> _statusChangeRejectionList =
new ()
{
[ProcessStatusTypes.Processing] = new()
{
ProcessStatusTypes.Reprocessing,
ProcessStatusTypes.UploadCompletionReprocessing
},
[ProcessStatusTypes.Completed] = new()
{
ProcessStatusTypes.Processing,
ProcessStatusTypes.Completed,
ProcessStatusTypes.ArchiveFailure,
ProcessStatusTypes.CriticalFailure,
ProcessStatusTypes.DataMappingFailure,
ProcessStatusTypes.Reprocessing,
ProcessStatusTypes.NotificationDataFailure,
ProcessStatusTypes.UploadCompletionReprocessing
},
[ProcessStatusTypes.ArchiveFailure] = new()
{
ProcessStatusTypes.UploadCompletionReprocessing
},
[ProcessStatusTypes.CriticalFailure] = new()
{
ProcessStatusTypes.UploadCompletionReprocessing
},
[ProcessStatusTypes.DataMappingFailure] = new()
{
ProcessStatusTypes.Processing,
ProcessStatusTypes.Completed,
ProcessStatusTypes.ArchiveFailure,
ProcessStatusTypes.CriticalFailure,
ProcessStatusTypes.DataMappingFailure,
ProcessStatusTypes.Reprocessing,
ProcessStatusTypes.NotificationDataFailure,
ProcessStatusTypes.UploadCompletionReprocessing
},
[ProcessStatusTypes.Reprocessing] = new()
{
ProcessStatusTypes.Processing,
ProcessStatusTypes.UploadCompletionReprocessing
},
[ProcessStatusTypes.NotificationDataFailure] = new()
{
ProcessStatusTypes.Processing,
ProcessStatusTypes.ArchiveFailure,
ProcessStatusTypes.CriticalFailure,
ProcessStatusTypes.DataMappingFailure,
ProcessStatusTypes.Reprocessing,
ProcessStatusTypes.UploadCompletionReprocessing
},
[ProcessStatusTypes.UploadCompletionReprocessing] = new()
{
ProcessStatusTypes.Processing,
ProcessStatusTypes.ArchiveFailure,
ProcessStatusTypes.CriticalFailure,
ProcessStatusTypes.DataMappingFailure,
ProcessStatusTypes.Reprocessing
}
};
public DocumentProcessingInfoService(IUnitOfWork unitOfWork,
IDocumentProcessingInfoRepository documentProcessingInfoRepository,
IProcessingLogService processingLogService)
{
_unitOfWork = unitOfWork;
_documentProcessingInfoRepository = documentProcessingInfoRepository;
_processingLogService = processingLogService;
}
public async Task<IEnumerable<DocumentProcessingInfo>> GetAsync()
{
try
{
var documentProcessingInfos = await _documentProcessingInfoRepository.FindAsync();
return documentProcessingInfos;
}
catch (Exception exception)
{
Log.Error(exception, $"Failure retrieving DocumentProcessingInfos.");
throw new ServiceException($"Failure retrieving DocumentProcessingInfos.");
}
}
public async Task<DocumentProcessingInfo> GetByIdAsync(int id)
{
try
{
var documentProcessingInfo = await _documentProcessingInfoRepository.FindByIdAsync(id);
return documentProcessingInfo;
}
catch (Exception exception)
{
Log.Error(exception, $"Failure retrieving DocumentProcessingInfo with DocumentProcessingInfoId {id}.");
throw new ServiceException($"Failure retrieving DocumentProcessingInfo with DocumentProcessingInfoId {id}.");
}
}
public async Task<IEnumerable<DocumentProcessingInfo>> GetListByIdAsync(List<int> ids)
{
try
{
var documentProcessingInfo = await _documentProcessingInfoRepository.FindListByIdAsync(ids);
return documentProcessingInfo;
}
catch (Exception exception)
{
var message = $"Failure retrieving list of DocumentProcessingInfos with DocumentProcessingInfoIds: {String.Join(", ", ids.ToArray())}.";
Log.Error(exception, message);
throw new ServiceException(message);
}
}
public async Task<IEnumerable<DocumentProcessingInfo>> GetByIngestorRunInfoIdAsync(Guid ingestorRunInfoId)
{
try
{
var documentProcessingInfos = await _documentProcessingInfoRepository.FindByIngestorRunInfoIdAsync(ingestorRunInfoId);
return documentProcessingInfos;
}
catch (Exception exception)
{
Log.Error(exception, $"Failure retrieving DocumentProcessingInfos with IngestorRunInfoId {ingestorRunInfoId}.");
throw new ServiceException($"Failure retrieving DocumentProcessingInfos with IngestorRunInfoId {ingestorRunInfoId}.");
}
}
public async Task<IEnumerable<DocumentProcessingInfo>> GetByProcessStatusTypeAsync(ProcessStatusTypes statusType, bool? isUnknown = null)
{
try
{
return await _documentProcessingInfoRepository.FindByProcessStatusTypeAsync(statusType, isUnknown);
}
catch (Exception exception)
{
var message = $"Failure retrieving DocumentProcessingInfos with Process Status Type {statusType}. " +
"IsUnknown: " + (isUnknown is null ? "null" : isUnknown.ToString());
Log.Error(exception, message);
throw new ServiceException(message);
}
}
public async Task<IEnumerable<DocumentProcessingInfo>> GetByProcessStatusTypeListAsync(List<ProcessStatusTypes> statusTypes, bool? isUnknown = null)
{
try
{
return await _documentProcessingInfoRepository.FindByProcessStatusTypeListAsync(statusTypes, isUnknown);
}
catch (Exception exception)
{
var message = $"Failure retrieving DocumentProcessingInfos with Process Status Type {(statusTypes is null ? "null" : string.Join(",", statusTypes))}. " +
"IsUnknown: " + (isUnknown is null ? "null" : isUnknown.ToString());
Log.Error(exception, message);
throw new ServiceException(message);
}
}
public async Task<IEnumerable<DocumentProcessingInfo>> GetCompletedFirstPassByIngestorRunInfoIdAsync(Guid ingestorRunInfoId)
{
try
{
return await _documentProcessingInfoRepository.FindCompletedFirstPassByIngestorRunInfoIdAsync(ingestorRunInfoId);
}
catch (Exception exception)
{
var message =
"Failure retrieving CompletedFirstPassByIngestorRunInfoId DocumentProcessingInfos. " +
$"IngestorRunInfoId: {ingestorRunInfoId}";
Log.Error(exception, message);
throw new ServiceException(message);
}
}
public async Task<DocumentProcessingInfo> SaveAsync(DocumentProcessingInfo documentProcessingInfo)
{
try
{
var result = await _documentProcessingInfoRepository.AddAsync(documentProcessingInfo);
await _unitOfWork.CompleteAsync();
return result;
}
catch (Exception exception)
{
Log.Error(exception, $"Failure saving DocumentProcessingInfo.");
throw new ServiceException($"Failure saving DocumentProcessingInfo.");
}
}
public async Task SaveRangeAsync(DocumentProcessingInfo[] documentProcessingInfos)
{
try
{
await _documentProcessingInfoRepository.AddRangeAsync(documentProcessingInfos);
await _unitOfWork.CompleteAsync();
}
catch (Exception exception)
{
Log.Error(exception, "Failure saving range DocumentProcessingInfo.");
throw new ServiceException("Failure saving range DocumentProcessingInfo.");
}
}
public async Task<DocumentProcessingInfo?> UpdateStatusAndAddLog(int documentProcessingInfoId,
ProcessStatusTypes statusType,
string logMessage,
string userName,
bool? entityIsUnknown = null)
{
return await UpdateStatusAndAddLog(documentProcessingInfoId, statusType, new List<string>(new string[] { logMessage }), userName, entityIsUnknown);
}
public async Task<DocumentProcessingInfo?> UpdateStatusAndAddLog(int documentProcessingInfoId,
ProcessStatusTypes statusType,
List<string> logMessages,
string userName,
bool? entityIsUnknown = null)
{
try
{
var documentProcessingInfo = await GetByIdAsync(documentProcessingInfoId);
return await UpdateStatusAndAddLog(documentProcessingInfo, statusType, logMessages, userName, entityIsUnknown);
}
catch (Exception exception)
{
Log.Error(exception, $"Failure Updating Status and Adding Processing Log. DocumentProcessingInfoId: {documentProcessingInfoId}");
throw new ServiceException($"Failure Updating Status and Adding Processing Log. DocumentProcessingInfoId: {documentProcessingInfoId}");
}
}
public async Task<DocumentProcessingInfo?> UpdateStatusAndAddLog(DocumentProcessingInfo documentProcessingInfo,
ProcessStatusTypes statusType,
string logMessage,
string userName,
bool? entityIsUnknown = null)
{
return await UpdateStatusAndAddLog(documentProcessingInfo, statusType, new List<string>(new string[] { logMessage }), userName, entityIsUnknown);
}
public async Task<DocumentProcessingInfo?> UpdateStatusAndAddLog(DocumentProcessingInfo documentProcessingInfo,
ProcessStatusTypes statusType,
List<string> logMessages,
string userName,
bool? entityIsUnknown = null)
{
try
{
ArgumentNullException.ThrowIfNull(documentProcessingInfo);
documentProcessingInfo.StatusType = statusType;
if (entityIsUnknown is not null)
{
documentProcessingInfo.EntityUnknown = entityIsUnknown ?? documentProcessingInfo.EntityUnknown;
}
var updated = await Update(documentProcessingInfo);
if (updated == null)
{
return null;
}
foreach (var log in logMessages)
{
if (string.IsNullOrWhiteSpace(log)) continue;
var processingLog = new ProcessingLog(updated.DocumentProcessingInfoId, DateTime.Now, log, userName);
await _processingLogService.SaveAsync(processingLog);
}
return updated;
}
catch (Exception exception)
{
Log.Error(exception, $"Failure Updating Status and Adding Processing Log. DocumentProcessingInfoId: {documentProcessingInfo.DocumentProcessingInfoId}");
throw new ServiceException($"Failure Updating Status and Adding Processing Log. DocumentProcessingInfoId: {documentProcessingInfo.DocumentProcessingInfoId}");
}
}
public async Task<DocumentProcessingInfo?> UpdateProcessCountAndStatusAndAddLog(int documentProcessingInfoId,
ProcessStatusTypes statusType,
string logMessage,
string userName,
bool? entityIsUnknown = null)
{
return await UpdateProcessCountAndStatusAndAddLog(documentProcessingInfoId, statusType, new List<string>(new string[] { logMessage }), userName, entityIsUnknown);
}
public async Task<DocumentProcessingInfo?> UpdateProcessCountAndStatusAndAddLog(int documentProcessingInfoId,
ProcessStatusTypes statusType,
List<string> logMessages,
string userName,
bool? entityIsUnknown = null)
{
try
{
var documentProcessingInfo = await GetByIdAsync(documentProcessingInfoId);
return await UpdateProcessCountAndStatusAndAddLog(documentProcessingInfo, statusType, logMessages, userName, entityIsUnknown);
}
catch (Exception exception)
{
Log.Error(exception, $"Failure Updating Process Count, Status, and Adding Processing Log. DocumentProcessingInfoId: {documentProcessingInfoId}");
throw new ServiceException($"Failure Updating Process Count, Status, and Adding Processing Log. DocumentProcessingInfoId: {documentProcessingInfoId}");
}
}
public async Task<List<CompletionProcessingData>> UpdateProcessCountAndStatusAndAddLog(List<CompletionRequest> requests)
{
var processingLogs = new List<ProcessingLog>();
var documentProcessingInfos = await GetListByIdAsync(requests.Select(r => r.ProcessingRequest.DocumentProcessingInfoId).ToList());
var completionProcessingDatas = PairCompletionRequestAndDocumentProcessingInfo(requests, documentProcessingInfos);
foreach (var data in completionProcessingDatas)
{
data.DocumentProcessingInfo.ProcessCount++;
data.DocumentProcessingInfo.StatusType = data.CompletionRequest.ProcessResult.ProcessStatusType;
data.DocumentProcessingInfo.EntityUnknown = data.CompletionRequest.ProcessResult.EntityIsUnknown ?? data.DocumentProcessingInfo.EntityUnknown;
if(data.CompletionRequest.ProcessResult.MessageStack != null || data.CompletionRequest.ProcessResult.MessageStack.Count > 0)
{
foreach (var message in data.CompletionRequest.ProcessResult.MessageStack)
{
if (string.IsNullOrWhiteSpace(message)) continue;
processingLogs.Add(new ProcessingLog(data.CompletionRequest.ProcessingRequest.DocumentProcessingInfoId, DateTime.Now, message, Users.System));
}
}
}
await UpdateRangeAsync(completionProcessingDatas.Select(d => d.DocumentProcessingInfo).ToList());
if (processingLogs.Count() >= 1)
{
await _processingLogService.SaveRangeAsync(processingLogs);
}
return completionProcessingDatas;
}
public async Task<DocumentProcessingInfo?> UpdateProcessCountAndStatusAndAddLog(DocumentProcessingInfo documentProcessingInfo,
ProcessStatusTypes statusType,
string logMessage,
string userName,
bool? entityIsUnknown = null)
{
return await UpdateProcessCountAndStatusAndAddLog(documentProcessingInfo, statusType, new List<string>(new string[] { logMessage }), userName, entityIsUnknown);
}
public async Task<DocumentProcessingInfo?> UpdateProcessCountAndStatusAndAddLog(DocumentProcessingInfo documentProcessingInfo,
ProcessStatusTypes statusType,
List<string> logMessages,
string userName,
bool? entityIsUnknown = null)
{
try
{
ArgumentNullException.ThrowIfNull(documentProcessingInfo);
documentProcessingInfo.ProcessCount++;
return await UpdateStatusAndAddLog(documentProcessingInfo, statusType, logMessages, userName, entityIsUnknown);
}
catch (Exception exception)
{
Log.Error(exception, $"Failure Updating Process Count, Status, and Adding Processing Log. DocumentProcessingInfoId: {documentProcessingInfo.DocumentProcessingInfoId}");
throw new ServiceException($"Failure Updating Process Count, Status, and Adding Processing Log. DocumentProcessingInfoId: {documentProcessingInfo.DocumentProcessingInfoId}");
}
}
public List<CompletionProcessingData> PairCompletionRequestAndDocumentProcessingInfo(List<CompletionRequest> requests, IEnumerable<DocumentProcessingInfo> documentProcessingInfos)
{
var completionProcessingDatas = new List<CompletionProcessingData>();
foreach (var request in requests)
{
var documentProcessingInfo = documentProcessingInfos.Where(d => d.DocumentProcessingInfoId == request.ProcessingRequest.DocumentProcessingInfoId).FirstOrDefault();
if (documentProcessingInfo is null)
{
Log.ForContext(Enum.GetName(typeof(PropertyEnum), PropertyEnum.Group),
Enum.GetName(typeof(GroupEnum), GroupEnum.ITDevelopment))
.Error(new Exception(), $"Failure during PairCompletionRequestAndDocumentProcessingInfo. " +
$"The DocumentProcessingInfo was not found for DocumentProcessingInfoId: {request.ProcessingRequest.DocumentProcessingInfoId}");
continue;
}
completionProcessingDatas.Add(new CompletionProcessingData() { CompletionRequest = request, DocumentProcessingInfo = documentProcessingInfo });
}
return completionProcessingDatas;
}
public async Task<DocumentProcessingInfo?> Update(DocumentProcessingInfo documentProcessingInfo)
{
try
{
var existing = await GetByIdAsync(documentProcessingInfo.DocumentProcessingInfoId);
if (existing == null)
{
return existing;
}
existing.CreatedDate = documentProcessingInfo.CreatedDate;
existing.DocumentValuePayload = documentProcessingInfo.DocumentValuePayload;
existing.StatusType = documentProcessingInfo.StatusType;
existing.EntityUnknown = documentProcessingInfo.EntityUnknown;
_documentProcessingInfoRepository.Update(existing);
_unitOfWork.Complete();
return existing;
}
catch (SqlException sqlException)
{
Log.Error(sqlException, $"SQL Failure updating DocumentProcessingInfo. " +
$"DocumentProcessingInfoId: {documentProcessingInfo.DocumentProcessingInfoId}");
throw new ServiceException($"SQL Failure updating DocumentProcessingInfo. " +
$"DocumentProcessingInfoId: {documentProcessingInfo.DocumentProcessingInfoId}");
}
catch (Exception exception)
{
Log.Error(exception, $"Failure updating DocumentProcessingInfo. " +
$"DocumentProcessingInfoId: {documentProcessingInfo.DocumentProcessingInfoId}");
throw new ServiceException($"Failure updating DocumentProcessingInfo. " +
$"DocumentProcessingInfoId: {documentProcessingInfo.DocumentProcessingInfoId}");
}
}
public async Task<IEnumerable<DocumentProcessingInfo>> UpdateRangeAsync(IEnumerable<DocumentProcessingInfo> documentProcessingInfos)
{
try
{
var infos = documentProcessingInfos.ToList();
_documentProcessingInfoRepository.UpdateRange(infos);
await _unitOfWork.CompleteAsync();
return infos;
}
catch (SqlException sqlException)
{
var message = "SQL Failure updating list of DocumentProcessingInfos.";
Log.Error(sqlException, message);
throw new ServiceException(message);
}
catch (Exception exception)
{
var message = "Failure updating list of DocumentProcessingInfos.";
Log.Error(exception, message);
throw new ServiceException(message);
}
}
public async Task Delete(int id)
{
try
{
var documentProcessingInfo = await GetByIdAsync(id);
ArgumentNullException.ThrowIfNull(documentProcessingInfo);
_documentProcessingInfoRepository.Remove(documentProcessingInfo);
_unitOfWork.Complete();
}
catch (SqlException sqlException)
{
Log.Error(sqlException, $"SQL failure removing DocumentProcessingInfo. DocumentProcessingInfoId: {id}");
throw new ServiceException($"SQL failure removing DocumentProcessingInfo. DocumentProcessingInfoId: {id}");
}
catch (Exception exception)
{
Log.Error(exception, $"Failure removing DocumentProcessingInfo. DocumentProcessingInfoId: {id}");
throw new ServiceException($"Failure removing DocumentProcessingInfo. DocumentProcessingInfoId: {id}");
}
}
public async Task<PagedResult<DocumentProcessingInfo>> GetFilteredDocumentProcessingInfo(Filter filter)
{
try
{
return await _documentProcessingInfoRepository.FilteredListAsync(filter);
}
catch (Exception exception)
{
Log.Error(exception, "Failure retrieving filtered documentProcessingInfo.");
throw new ServiceException("Failure retrieving filtered documentProcessingInfo.");
}
}
public async Task<int> CountCompletedByIngestorRunInfoIdAsync(Guid ingestorRunInfoId)
{
try
{
return await _documentProcessingInfoRepository.CountCompletedByIngestorRunInfoIdAsync(ingestorRunInfoId);
}
catch (Exception exception)
{
Log.Error(exception, "Failure counting CompletedByIngestorRunInfoId DocumentProcessingInfos for run info {IngestorRunInfoId}", ingestorRunInfoId);
throw new ServiceException($"Failure counting CompletedByIngestorRunInfoId DocumentProcessingInfos for run info {ingestorRunInfoId}");
}
}
public async Task<int> CountCompletedFirstPassByIngestorRunInfoIdAsync(Guid ingestorRunInfoId)
{
try
{
return await _documentProcessingInfoRepository.CountCompletedFirstPassByIngestorRunInfoIdAsync(ingestorRunInfoId);
}
catch (Exception exception)
{
Log.Error(exception, "Failure counting CompletedFirstPassByIngestorRunInfoId DocumentProcessingInfos for run info {IngestorRunInfoId}", ingestorRunInfoId);
throw new ServiceException($"Failure counting CompletedFirstPassByIngestorRunInfoId DocumentProcessingInfos for run info {ingestorRunInfoId}");
}
}
public async Task<IEnumerable<DocumentProcessingInfo>?> SetDocumentsToEntityUnknown(EntityUnknownRequest request)
{
try
{
var documentProcessingInfos = await GetListByIdAsync(request.DocumentProcessingInfoIds);
if (!documentProcessingInfos.Any())
{
throw new ServiceException("No DocumentProcessingInfo(s) were found");
}
foreach (var documentProcessingInfo in documentProcessingInfos)
{
var logMessage = "Record's EntityUnknown set to True. DocumentProcessingInfoId:" +
$" {documentProcessingInfo.DocumentProcessingInfoId} by {request.UserName} on {DateTime.Now}";
var processingLog = new ProcessingLog(documentProcessingInfo.DocumentProcessingInfoId, DateTime.Now, logMessage, request.UserName);
processingLog = await _processingLogService.SaveAsync(processingLog);
if (processingLog == null)
{
throw new ServiceException("ProcessingLog could not be saved for DocumentProcessingInfoId: " +
documentProcessingInfo.DocumentProcessingInfoId);
}
documentProcessingInfo.EntityUnknown = true;
await Update(documentProcessingInfo);
}
return documentProcessingInfos;
}
catch (ArgumentNullException exception)
{
var message = "Failure during SetDocumentsToUnknown for " +
$"DocumentProcessingInfoIds: ({string.Join(", ", request.DocumentProcessingInfoIds.ToArray())}). " +
$"ExceptionMessage: {exception.Message}";
Log.Error(exception, message);
throw new ServiceException(message);
}
catch (Exception exception)
{
var message = "Failure during SetDocumentsToUnknown for " +
$"DocumentProcessingInfoIds: ({string.Join(", ", request.DocumentProcessingInfoIds.ToArray())}).";
Log.Error(exception, message);
throw new ServiceException(message);
}
}
public async Task<DocumentProcessingInfo?> UpdateDocumentProcessingInfoStatus(DocumentProcessingInfoStatusRequest request)
{
try
{
var documentProcessingInfo = await GetByIdAsync(request.DocumentProcessingInfoId);
if (documentProcessingInfo == null)
{
throw new ServiceException("DocumentProcessingInfo was not found.");
}
if (documentProcessingInfo.StatusType == request.ProcessStatus)
{
throw new ServiceException("DocumentProcessingInfo had the same status as the request and cannot be changed. " +
$"Status: {documentProcessingInfo.StatusType}");
}
var statusChangeNotAllowed = _statusChangeRejectionList[documentProcessingInfo.StatusType].Contains(request.ProcessStatus);
if (statusChangeNotAllowed)
{
throw new ServiceException($"DocumentProcessingInfo had a status of '{Enum.GetName(documentProcessingInfo.StatusType)}' " +
$"and cannot be changed to the requested status of '{Enum.GetName(request.ProcessStatus)}'.");
}
var logMessage = $"Record's ProcessStatus changed from {documentProcessingInfo.StatusType} to {request.ProcessStatus}." +
$" DocumentProcessingInfoId: {documentProcessingInfo.DocumentProcessingInfoId} by {request.Username} on {DateTime.Now}." +
$" Reason: {request.ReasonMessage}";
return await UpdateStatusAndAddLog(documentProcessingInfo, request.ProcessStatus, new List<string> { logMessage }, request.Username);
}
catch (ArgumentNullException exception)
{
var message = "Failure during UpdateDocumentProcessingInfoStatus for " +
$"DocumentProcessingInfoId: ({request.DocumentProcessingInfoId}). " +
$"ExceptionMessage: {exception.Message}";
Log.Error(exception, message);
throw new ServiceException(message);
}
catch (Exception exception)
{
var message = "Failure during UpdateDocumentProcessingInfoStatus for " +
$"DocumentProcessingInfoId: ({request.DocumentProcessingInfoId}).";
Log.Error(exception, message);
throw new ServiceException(message);
}
}
public async Task<IDictionary<Guid, DocumentProcessingInfo?>> GetByDocumentIdsAsync(Guid[] ids)
{
try
{
var infos = (await _documentProcessingInfoRepository.FindByDocumentIdsAsync(ids))
.Where(i => i.DocumentId != null)
.ToDictionary(i => i.DocumentId.Value);
foreach (var missingId in ids.Where(i => !infos.ContainsKey(i)))
{
infos[missingId] = null;
}
return infos;
}
catch (Exception exception)
{
var message = "Failure retrieving DocumentProcessingInfos by DocumentIds.";
Log.Error(exception, message);
throw new ServiceException(message);
}
}
}
@@ -0,0 +1,11 @@
using LCU.Library.Internal.LCUServiceFactory.Configurations;
using LCU.Library.Internal.LCUServiceFactory.Services;
namespace LCU.API.Internal.DocumentIngestor.Services.DocumentProcessorServices;
public abstract class DocumentProcessingService : LcuAbstractService
{
protected DocumentProcessingService(ServicesFactoryConfiguration servicesFactoryConfiguration)
: base(servicesFactoryConfiguration.GetServiceFactoryRegistrationByInterface<IDocumentProcessingService>())
{ }
}
@@ -0,0 +1,257 @@
using System.Globalization;
using LCU.API.Internal.DocumentIngestor.Services.AttributeDeterminingServices;
using LCU.API.Internal.DocumentIngestor.Services.MicroApiServices;
using LCU.Library.DocumentType.Enums;
using LCU.Library.DocumentType.Extensions;
using LCU.Library.DocumentType.Models.Requests.Account;
using LCU.Library.DocumentType.Models.Requests.Attributes;
using LCU.Library.DocumentType.Models.Requests.PrimaryOwner;
using LCU.Library.Internal.DocumentIngestorModels.Entities;
using LCU.Library.Internal.DocumentIngestorModels.Enums;
using LCU.Library.Internal.DocumentIngestorModels.Models;
using LCU.Library.Internal.DocumentIngestorModels.Models.IndexFileDocuments;
using LCU.Library.Internal.DocumentIngestorModels.Validations;
using LCU.Library.Internal.LCUServiceFactory.Configurations;
using Newtonsoft.Json;
using Serilog;
namespace LCU.API.Internal.DocumentIngestor.Services.DocumentProcessorServices
{
public class FosDnaDocumentProcessingService : DocumentProcessingService, IDocumentProcessingService
{
private const DataSourceTypes DataSource = DataSourceTypes.DNA;
private const SourceTypes Source = SourceTypes.FOS;
private readonly IPrimaryOwnerDeterminingService _primaryOwnerDeterminingService;
private readonly IAccountOwnerDeterminingService _accountOwnerDeterminingService;
private readonly IFileOperationsApiService _fileOperationsApiService;
private readonly IDocumentTypeUploaderApiService _documentTypeUploaderApiService;
public FosDnaDocumentProcessingService(IPrimaryOwnerDeterminingService primaryOwnerDeterminingService,
IAccountOwnerDeterminingService accountDeterminingService,
IFileOperationsApiService fileOperationsApiService,
IDocumentTypeUploaderApiService documentTypeUploaderApiService,
ServicesFactoryConfiguration servicesFactoryConfiguration)
: base(servicesFactoryConfiguration)
{
_documentTypeUploaderApiService = documentTypeUploaderApiService;
_fileOperationsApiService = fileOperationsApiService;
_primaryOwnerDeterminingService = primaryOwnerDeterminingService;
_accountOwnerDeterminingService = accountDeterminingService;
}
public async Task<ProcessResult> ProcessRequest(ProcessingRequest request)
{
var processResult = new ProcessResult();
try
{
Log.Debug("Starting FosDnaDocument ProcessRequest.");
ArgumentNullException.ThrowIfNull(request);
var documentValues = JsonConvert.DeserializeObject<FosDnaDocument>(request.DocumentValuePayload);
ArgumentNullException.ThrowIfNull(documentValues);
processResult = ValidateRecord(processResult, documentValues, request.OwnerOverrideInfo);
if (!processResult.Success)
{
return processResult;
}
if (!DateTime.TryParseExact(documentValues.CreationDate, "MMddyyyy", CultureInfo.InvariantCulture, DateTimeStyles.AssumeLocal, out var creationDate))
{
var message = $"A Creation Date of '{documentValues.CreationDate}' could not be properly formatted " +
$"for an Account Owner of AccountNumber: {documentValues?.AccountNumber} and EntityName: {documentValues?.EntityName}";
processResult = ProcessResult.ProcessDataMappingFailure(processResult, message);
return processResult;
}
if (EnumExtensions.IsDocumentTypeOf<PrimaryOwnerDocumentTypes>(documentValues.DocType))
{
processResult = await HandlePrimaryOwner(request, processResult, documentValues, creationDate);
}
else if (EnumExtensions.IsDocumentTypeOf<AccountDocumentTypes>(documentValues.DocType))
{
processResult = await HandleAccountOwner(request, processResult, documentValues, creationDate);
}
else
{
var message = $"Document Type of {documentValues.DocType} could not be found." +
$"for an Account Owner of AccountNumber: {documentValues?.AccountNumber} and EntityName: {documentValues?.EntityName}";
Log.Error(message);
processResult = ProcessResult.ProcessCriticalFailure(processResult, message);
return processResult;
}
Log.Debug("Ending FosDna ProcessRequest.");
}
catch (Exception exception)
{
var message = "An Exception occured while FosDnaDocument ProcessRequest. " +
$"Exception Message: {exception.Message} ";
Log.Error(exception, message);
processResult = ProcessResult.ProcessCriticalFailure(processResult, message);
}
return processResult;
}
public ProcessResult ValidateRecord(ProcessResult processResult,
FosDnaDocument documentValues,
OwnerOverrideInfo? ownerOverrideInfo)
{
// Keep at top to set EntityUnknown for PsInfoChangeNotice documents with invalid Account Numbers
// Business would like override to skip AccountNumber validation for PsInfoChangeNotice
if (!SkipAccountNumberValidationForPsInfoChangeNotice(documentValues.DocType, ownerOverrideInfo))
{
processResult = MetaDataValidation.ValidateAccountNumber(processResult, documentValues.AccountNumber);
processResult = SetToEntityUnknown(processResult, documentValues.DocType, ownerOverrideInfo);
}
processResult = MetaDataValidation.ValidateFileName(processResult, documentValues.DocName);
processResult = MetaDataValidation.ValidateFullName(processResult, documentValues.EntityName);
return processResult;
}
public async Task<ProcessResult> HandlePrimaryOwner(ProcessingRequest request,
ProcessResult processResult,
FosDnaDocument documentValues,
DateTime creationDate)
{
var primaryOwnerAttribute = await _primaryOwnerDeterminingService.DeterminePrimaryOwnerAttribute(documentValues.EntityName,
documentValues.AccountNumber,
creationDate,
request.OwnerOverrideInfo);
if (!primaryOwnerAttribute.Success)
{
processResult = ProcessResult.ProcessDataMappingFailure(processResult,
primaryOwnerAttribute.Message);
return processResult;
}
var primaryOwnerRequest = await BuildPrimaryOwnerUploadRequest(request, documentValues, creationDate, primaryOwnerAttribute.Payload);
await _documentTypeUploaderApiService.UploadPrimaryOwnerDocument(primaryOwnerRequest, request, processResult);
return processResult;
}
public async Task<ProcessResult> HandleAccountOwner(ProcessingRequest request,
ProcessResult processResult,
FosDnaDocument documentValues,
DateTime creationDate)
{
var accountOwnerAttribute = await _accountOwnerDeterminingService.DetermineAccountOwnerAttribute(documentValues.EntityName,
documentValues.AccountNumber,
creationDate,
request.OwnerOverrideInfo);
if (!accountOwnerAttribute.Success)
{
processResult = ProcessResult.ProcessDataMappingFailure(processResult,
accountOwnerAttribute.Message);
return processResult;
}
var accountOwnerRequest = await BuildAccountUploadRequest(request, documentValues, creationDate, accountOwnerAttribute.Payload);
await _documentTypeUploaderApiService.UploadAccountDocument(accountOwnerRequest, request, processResult);
return processResult;
}
public async Task<PrimaryOwnerDocumentRequest> BuildPrimaryOwnerUploadRequest(ProcessingRequest request,
FosDnaDocument documentValues,
DateTime creationDate,
PrimaryOwnerAttribute primaryOwnerAttribute)
{
var contentBytes = await _fileOperationsApiService.GetContentBytes(request.IngestorRequest.RootPath,
request.DocumentFileName);
var type = documentValues.DocType.GetValueFromName<PrimaryOwnerDocumentTypes>();
ArgumentNullException.ThrowIfNull(type);
var primaryOwnerDocumentRequest = new PrimaryOwnerDocumentRequest
{
Content = contentBytes,
PrimaryOwnerAttribute = primaryOwnerAttribute,
PrimaryOwnerDocumentMetadata = new PrimaryOwnerDocumentMetadata
{
Source = Source,
DataSource = DataSource,
Name = documentValues.DocName,
Type = type,
CreatedDate = creationDate,
TraceId = request.IngestorRunInfoId
}
};
return primaryOwnerDocumentRequest;
}
public async Task<AccountDocumentRequest> BuildAccountUploadRequest(ProcessingRequest request,
FosDnaDocument documentValues,
DateTime creationDate,
AccountAttribute accountAttribute)
{
var contentBytes = await _fileOperationsApiService.GetContentBytes(request.IngestorRequest.RootPath,
request.DocumentFileName);
var type = documentValues.DocType.GetValueFromName<AccountDocumentTypes>();
ArgumentNullException.ThrowIfNull(type);
var accountOwnerDocumentRequest = new AccountDocumentRequest
{
Content = contentBytes,
AccountAttribute = accountAttribute,
AccountDocumentMetadata = new AccountDocumentMetadata
{
Source = Source,
DataSource = DataSource,
Name = documentValues.DocName,
Type = type,
CreatedDate = creationDate,
TraceId = request.IngestorRunInfoId
}
};
return accountOwnerDocumentRequest;
}
private ProcessResult SetToEntityUnknown(ProcessResult processResult,
string documentType,
OwnerOverrideInfo? ownerOverrideInfo)
{
if (string.IsNullOrWhiteSpace(documentType)
|| processResult.Success
|| processResult.ProcessStatusType != ProcessStatusTypes.DataMappingFailure
|| ownerOverrideInfo is not null)
{
return processResult;
}
if (EnumExtensions.IsDocumentTypeOf<PrimaryOwnerDocumentTypes>(documentType))
{
switch (documentType.GetValueFromName<PrimaryOwnerDocumentTypes>())
{
case PrimaryOwnerDocumentTypes.PsInfoChangeNotice:
processResult.EntityIsUnknown = true;
break;
}
}
return processResult;
}
private bool SkipAccountNumberValidationForPsInfoChangeNotice(string docType, OwnerOverrideInfo? ownerOverrideInfo)
{
if (!EnumExtensions.IsDocumentTypeOf<PrimaryOwnerDocumentTypes>(docType))
{
return false;
}
if (ownerOverrideInfo is null
|| docType.GetValueFromName<PrimaryOwnerDocumentTypes>() != PrimaryOwnerDocumentTypes.PsInfoChangeNotice)
{
return false;
}
return true;
}
}
}
@@ -0,0 +1,128 @@
using System.Globalization;
using LCU.API.Internal.DocumentIngestor.Services.AttributeDeterminingServices;
using LCU.API.Internal.DocumentIngestor.Services.MicroApiServices;
using LCU.Library.DocumentType.Enums;
using LCU.Library.DocumentType.Extensions;
using LCU.Library.DocumentType.Models.Requests.Account;
using LCU.Library.DocumentType.Models.Requests.Attributes;
using LCU.Library.Internal.DocumentIngestorModels.Models;
using LCU.Library.Internal.DocumentIngestorModels.Models.IndexFileDocuments;
using LCU.Library.Internal.DocumentIngestorModels.Validations;
using LCU.Library.Internal.LCUServiceFactory.Configurations;
using Newtonsoft.Json;
using Serilog;
namespace LCU.API.Internal.DocumentIngestor.Services.DocumentProcessorServices
{
public class FosLoanStatementProcessingService : DocumentProcessingService, IDocumentProcessingService
{
private const DataSourceTypes DataSource = DataSourceTypes.DNA;
private const SourceTypes Source = SourceTypes.FOS;
private const string DateFormat = "yyyyMMdd";
private readonly IAccountOwnerDeterminingService _accountOwnerDeterminingService;
private readonly IFileOperationsApiService _fileOperationsApiService;
private readonly IDocumentTypeUploaderApiService _documentTypeUploaderApiService;
public FosLoanStatementProcessingService(IAccountOwnerDeterminingService accountDeterminingService,
IFileOperationsApiService fileOperationsApiService,
IDocumentTypeUploaderApiService documentTypeUploaderApiService,
ServicesFactoryConfiguration servicesFactoryConfiguration)
: base(servicesFactoryConfiguration)
{
_documentTypeUploaderApiService = documentTypeUploaderApiService;
_fileOperationsApiService = fileOperationsApiService;
_accountOwnerDeterminingService = accountDeterminingService;
}
public async Task<ProcessResult> ProcessRequest(ProcessingRequest request)
{
var processResult = new ProcessResult();
try
{
Log.Debug("Starting FosLoanStatement ProcessRequest.");
ArgumentNullException.ThrowIfNull(request);
var documentValues = JsonConvert.DeserializeObject<FosLoanStatement>(request.DocumentValuePayload);
ArgumentNullException.ThrowIfNull(documentValues);
processResult = ValidateRecord(processResult, documentValues);
if (!processResult.Success)
{
return processResult;
}
if (!DateTime.TryParseExact(documentValues.CreationDate, DateFormat, CultureInfo.InvariantCulture,
DateTimeStyles.AssumeLocal, out var creationDate))
{
var message =
$"A Creation Date of {documentValues.CreationDate} could not be properly formatted from format: {DateFormat}";
return ProcessResult.ProcessDataMappingFailure(processResult, message);
}
var accountOwnerAttribute = await _accountOwnerDeterminingService.DetermineAccountOwnerAttribute(
documentValues.EntityName,
documentValues.AccountNumber,
creationDate,
request.OwnerOverrideInfo);
if (!accountOwnerAttribute.Success)
{
return ProcessResult.ProcessDataMappingFailure(processResult,
accountOwnerAttribute.Message);
}
var accountOwnerRequest = await BuildAccountUploadRequest(request, documentValues, creationDate, accountOwnerAttribute.Payload);
await _documentTypeUploaderApiService.UploadAccountDocument(accountOwnerRequest, request, processResult);
Log.Debug("Ending FosLoanStatement ProcessRequest.");
}
catch (Exception exception)
{
var message = $"An Exception occured while FosLoanStatement ProcessRequest. " +
$"Exception Message: {exception.Message} ";
Log.Error(exception, message);
processResult = ProcessResult.ProcessCriticalFailure(processResult, message);
}
return processResult;
}
public ProcessResult ValidateRecord(ProcessResult processResult, FosLoanStatement documentValues)
{
processResult = MetaDataValidation.ValidateAccountNumber(processResult, documentValues.AccountNumber);
processResult = MetaDataValidation.ValidateFileName(processResult, documentValues.DocName);
processResult = MetaDataValidation.ValidateFullName(processResult, documentValues.EntityName);
processResult = MetaDataValidation.ValidateDocumentType<AccountDocumentTypes>(processResult, documentValues.DocType);
return processResult;
}
public async Task<AccountDocumentRequest> BuildAccountUploadRequest(ProcessingRequest request,
FosLoanStatement documentValues,
DateTime creationDate,
AccountAttribute accountAttribute)
{
var contentBytes = await _fileOperationsApiService.GetContentBytes(request.IngestorRequest.RootPath,
request.DocumentFileName);
var type = documentValues.DocType.GetValueFromName<AccountDocumentTypes>();
ArgumentNullException.ThrowIfNull(type);
var accountOwnerDocumentRequest = new AccountDocumentRequest
{
Content = contentBytes,
AccountAttribute = accountAttribute,
AccountDocumentMetadata = new AccountDocumentMetadata
{
Source = Source,
DataSource = DataSource,
Name = request.DocumentFileName,
Type = type,
CreatedDate = creationDate,
TraceId = request.IngestorRunInfoId
}
};
return accountOwnerDocumentRequest;
}
}
}
@@ -0,0 +1,161 @@
using Newtonsoft.Json;
using Serilog;
using LCU.Library.DocumentType.Enums;
using LCU.Library.DocumentType.Extensions;
using LCU.Library.DocumentType.Models.Requests.Attributes;
using LCU.Library.DocumentType.Models.Requests.PrimaryOwner;
using LCU.Library.DocumentType.Models.Requests.Generic;
using LCU.Library.Internal.DocumentIngestorModels.Models;
using LCU.Library.Internal.DocumentIngestorModels.Models.IndexFileDocuments;
using LCU.Library.Internal.DocumentIngestorModels.Validations;
using LCU.API.Internal.DocumentIngestor.Services.AttributeDeterminingServices;
using LCU.API.Internal.DocumentIngestor.Services.MicroApiServices;
using LCU.Library.Internal.LCUServiceFactory.Configurations;
namespace LCU.API.Internal.DocumentIngestor.Services.DocumentProcessorServices;
public class FosTaxStatementProcessingService : DocumentProcessingService, IDocumentProcessingService
{
private readonly IPrimaryOwnerDeterminingService _primaryOwnerDeterminingService;
private readonly IFileOperationsApiService _fileOperationsApiService;
private readonly IDocumentTypeUploaderApiService _documentTypeUploaderApiService;
private const DataSourceTypes DataSource = DataSourceTypes.DNA;
private const SourceTypes Source = SourceTypes.FOS;
public FosTaxStatementProcessingService(IPrimaryOwnerDeterminingService userIdDeterminingService,
IFileOperationsApiService fileOperationsApiService,
IDocumentTypeUploaderApiService documentTypeUploaderApiService,
ServicesFactoryConfiguration servicesFactoryConfiguration)
: base(servicesFactoryConfiguration)
{
_documentTypeUploaderApiService = documentTypeUploaderApiService;
_fileOperationsApiService = fileOperationsApiService;
_primaryOwnerDeterminingService = userIdDeterminingService;
}
public async Task<ProcessResult> ProcessRequest(ProcessingRequest request)
{
var processResult = new ProcessResult();
try
{
Log.Debug("Starting FosTaxStatement ProcessRequest.");
ArgumentNullException.ThrowIfNull(request);
var documentValues = JsonConvert.DeserializeObject<FosTaxStatementDocument>(request.DocumentValuePayload);
ArgumentNullException.ThrowIfNull(documentValues);
processResult = ValidateRecord(processResult, documentValues);
if (!processResult.Success)
{
return processResult;
}
if (EnumExtensions.IsDocumentTypeOf<PrimaryOwnerDocumentTypes>(documentValues.FileName))
{
var primaryOwnerAttribute = await _primaryOwnerDeterminingService.DeterminePrimaryOwnerAttribute(
documentValues.FullName,
documentValues.AccountNumber,
documentValues.PrimaryTIN,
request.OwnerOverrideInfo);
if (!primaryOwnerAttribute.Success)
{
processResult = ProcessResult.ProcessDataMappingFailure(processResult,
primaryOwnerAttribute.Message);
return processResult;
}
var uploadRequest = await BuildUploadRequest(request, documentValues, primaryOwnerAttribute.Payload);
await _documentTypeUploaderApiService.UploadPrimaryOwnerDocument(uploadRequest, request, processResult);
}
else if (EnumExtensions.IsDocumentTypeOf<GenericDocumentTypes>(documentValues.FileName))
{
var genericAttribute = new GenericDocumentAttribute
{
EntityName = documentValues.FullName
};
var uploadRequest = await BuildGenericUploadRequest(request, documentValues, genericAttribute);
await _documentTypeUploaderApiService.UploadGenericDocument(uploadRequest, request, processResult);
}
else
{
var message = $"Document Type of {documentValues.FileName} could not be found.";
Log.Error(message);
processResult = ProcessResult.ProcessCriticalFailure(processResult, message);
return processResult;
}
}
catch (Exception exception)
{
var message = "An Exception occured while FosTaxStatement ProcessRequest. " +
$"Exception Message: {exception.Message} ";
Log.Error(exception, message);
processResult = ProcessResult.ProcessCriticalFailure(processResult, message);
}
Log.Debug("Ending FosTaxStatement ProcessRequest.");
return processResult;
}
public ProcessResult ValidateRecord(ProcessResult processResult, FosTaxStatementDocument documentValues)
{
processResult = MetaDataValidation.ValidateAccountNumber(processResult, documentValues.AccountNumber);
processResult = MetaDataValidation.ValidateFileName(processResult, documentValues.FileName);
processResult = MetaDataValidation.ValidateFullName(processResult, documentValues.FullName);
processResult = MetaDataValidation.ValidatePdf(processResult, documentValues.PDF);
processResult = MetaDataValidation.ValidateTin(processResult, documentValues.PrimaryTIN);
return processResult;
}
public async Task<PrimaryOwnerDocumentRequest> BuildUploadRequest(ProcessingRequest request,
FosTaxStatementDocument documentValues,
PrimaryOwnerAttribute primaryOwnerAttribute)
{
var contentBytes = await _fileOperationsApiService.GetContentBytes(request.IngestorRequest.RootPath,
request.DocumentFileName);
var type = documentValues.FileName.GetValueFromName<PrimaryOwnerDocumentTypes>();
var primaryOwnerDocumentRequest = new PrimaryOwnerDocumentRequest
{
Content = contentBytes,
PrimaryOwnerAttribute = primaryOwnerAttribute,
PrimaryOwnerDocumentMetadata = new PrimaryOwnerDocumentMetadata
{
Source = Source,
DataSource = DataSource,
Name = documentValues.PDF,
Type = type,
CreatedDate = DateTime.Now.Date,
TraceId = request.IngestorRunInfoId
}
};
return primaryOwnerDocumentRequest;
}
public async Task<GenericDocumentRequest> BuildGenericUploadRequest(ProcessingRequest request,
FosTaxStatementDocument documentValues,
GenericDocumentAttribute genericDocumentAttribute)
{
var contentBytes = await _fileOperationsApiService.GetContentBytes(request.IngestorRequest.RootPath,
request.DocumentFileName);
var type = documentValues.FileName.GetValueFromName<GenericDocumentTypes>();
var genericDocumentRequest = new GenericDocumentRequest
{
Content = contentBytes,
GenericDocumentAttribute = genericDocumentAttribute,
GenericDocumentMetadata = new GenericDocumentMetadata
{
Source = Source,
DataSource = DataSource,
Name = documentValues.PDF,
Type = type,
CreatedDate = DateTime.Now.Date,
TraceId = request.IngestorRunInfoId
}
};
return genericDocumentRequest;
}
}
@@ -0,0 +1,9 @@
using LCU.Library.Internal.DocumentIngestorModels.Models;
using LCU.Library.Internal.LCUServiceFactory.Services;
namespace LCU.API.Internal.DocumentIngestor.Services.DocumentProcessorServices;
public interface IDocumentProcessingService : ILcuService
{
Task<ProcessResult> ProcessRequest(ProcessingRequest request);
}
@@ -0,0 +1,126 @@
using Newtonsoft.Json;
using Serilog;
using LCU.Library.DocumentType.Enums;
using LCU.Library.DocumentType.Extensions;
using LCU.Library.DocumentType.Models.Requests.Attributes;
using LCU.Library.Internal.DocumentIngestorModels.Models;
using LCU.Library.Internal.DocumentIngestorModels.Models.IndexFileDocuments;
using LCU.Library.Internal.DocumentIngestorModels.Validations;
using LCU.API.Internal.DocumentIngestor.Services.AttributeDeterminingServices;
using LCU.API.Internal.DocumentIngestor.Services.MicroApiServices;
using System.Globalization;
using LCU.Library.DocumentType.Models.Requests.Account;
using LCU.Library.Internal.LCUServiceFactory.Configurations;
namespace LCU.API.Internal.DocumentIngestor.Services.DocumentProcessorServices;
public class OsgDnaNoticeSplitProcessingService : DocumentProcessingService, IDocumentProcessingService
{
private readonly IAccountOwnerDeterminingService _accountOwnerDeterminingService;
private readonly IFileOperationsApiService _fileOperationsApiService;
private readonly IDocumentTypeUploaderApiService _documentTypeUploaderApiService;
private const DataSourceTypes DataSource = DataSourceTypes.DNA;
private const SourceTypes Source = SourceTypes.OSG;
private const string DateFormat = "MMddyyyy";
public OsgDnaNoticeSplitProcessingService(IFileOperationsApiService fileOperationsApiService,
IDocumentTypeUploaderApiService documentTypeUploaderApiService,
IAccountOwnerDeterminingService accountOwnerDeterminingService,
ServicesFactoryConfiguration servicesFactoryConfiguration)
: base(servicesFactoryConfiguration)
{
_documentTypeUploaderApiService = documentTypeUploaderApiService;
_fileOperationsApiService = fileOperationsApiService;
_accountOwnerDeterminingService = accountOwnerDeterminingService;
}
public async Task<ProcessResult> ProcessRequest(ProcessingRequest request)
{
var processResult = new ProcessResult();
try
{
Log.Debug("Starting OsgDnaNoticeSplit ProcessRequest.");
ArgumentNullException.ThrowIfNull(request);
var documentValues = JsonConvert.DeserializeObject<OsgDnaNoticeSplit>(request.DocumentValuePayload);
ArgumentNullException.ThrowIfNull(documentValues);
processResult = ValidateRecord(processResult, documentValues);
if (!processResult.Success)
{
return processResult;
}
if (!DateTime.TryParseExact(documentValues.CreationDate, DateFormat, CultureInfo.InvariantCulture, DateTimeStyles.AssumeLocal, out var creationDate))
{
var message = $"A Creation Date of '{documentValues.CreationDate}' could not be properly formatted " +
$"for an Account Owner of EntityName: {documentValues?.RecipientName} and EntityId: {documentValues?.AccountNumber}";
processResult = ProcessResult.ProcessDataMappingFailure(processResult, message);
return processResult;
}
var accountOwnerAttribute = await _accountOwnerDeterminingService.DetermineAccountOwnerAttributeByMemberNumberOrUserId(documentValues.UserId,
documentValues.AccountNumber,
documentValues.NoticeId,
creationDate,
request.OwnerOverrideInfo);
if (!accountOwnerAttribute.Success)
{
processResult = ProcessResult.ProcessDataMappingFailure(processResult,
accountOwnerAttribute.Message);
return processResult;
}
var accountOwnerRequest = await BuildUploadRequest(request, documentValues, creationDate, accountOwnerAttribute.Payload);
await _documentTypeUploaderApiService.UploadAccountDocument(accountOwnerRequest, request, processResult);
}
catch (Exception exception)
{
var message = "An Exception occured while OsgTaxStatement ProcessRequest. " +
$"Exception Message: {exception.Message} ";
Log.Error(exception, message);
processResult = ProcessResult.ProcessCriticalFailure(processResult, message);
}
Log.Debug("Ending OsgTaxStatement ProcessRequest.");
return processResult;
}
public ProcessResult ValidateRecord(ProcessResult processResult, OsgDnaNoticeSplit documentValues)
{
processResult = MetaDataValidation.ValidateAccountNumber(processResult, documentValues.AccountNumber);
processResult = MetaDataValidation.ValidateFileName(processResult, documentValues.NoticeName);
processResult = MetaDataValidation.ValidateFullName(processResult, documentValues.RecipientName);
processResult = MetaDataValidation.ValidatePdf(processResult, documentValues.PdfName);
return processResult;
}
public async Task<AccountDocumentRequest> BuildUploadRequest(ProcessingRequest request,
OsgDnaNoticeSplit documentValues,
DateTime creationDate,
AccountAttribute accountAttribute)
{
var contentBytes = await _fileOperationsApiService.GetContentBytes(request.IngestorRequest.RootPath,
request.DocumentFileName);
var type = documentValues.NoticeId.GetValueFromName<AccountDocumentTypes>();
ArgumentNullException.ThrowIfNull(type);
var accountOwnerDocumentRequest = new AccountDocumentRequest
{
Content = contentBytes,
AccountAttribute = accountAttribute,
AccountDocumentMetadata = new AccountDocumentMetadata
{
Source = Source,
DataSource = DataSource,
Name = documentValues.NoticeName,
Type = type,
CreatedDate = creationDate,
TraceId = request.IngestorRunInfoId
}
};
return accountOwnerDocumentRequest;
}
}
@@ -0,0 +1,187 @@
using LCU.API.Internal.DocumentIngestor.Services.AttributeDeterminingServices;
using LCU.API.Internal.DocumentIngestor.Services.MicroApiServices;
using LCU.Library.DocumentType.Enums;
using LCU.Library.DocumentType.Extensions;
using LCU.Library.DocumentType.Models.Requests.Account;
using LCU.Library.DocumentType.Models.Requests.Attributes;
using LCU.Library.DocumentType.Models.Requests.PrimaryOwner;
using LCU.Library.Internal.DocumentIngestorModels.Models;
using LCU.Library.Internal.DocumentIngestorModels.Models.IndexFileDocuments;
using LCU.Library.Internal.DocumentIngestorModels.Validations;
using Newtonsoft.Json;
using Serilog;
using System.Globalization;
using LCU.Library.Internal.LCUServiceFactory.Configurations;
namespace LCU.API.Internal.DocumentIngestor.Services.DocumentProcessorServices
{
public class OsgDnaNoticesProcessingService : DocumentProcessingService, IDocumentProcessingService
{
private const DataSourceTypes DataSource = DataSourceTypes.DNA;
private const SourceTypes Source = SourceTypes.OSG;
private readonly IPrimaryOwnerDeterminingService _primaryOwnerDeterminingService;
private readonly IFileOperationsApiService _fileOperationsApiService;
private readonly IDocumentTypeUploaderApiService _documentTypeUploaderApiService;
private readonly IAccountOwnerDeterminingService _accountOwnerDeterminingService;
private const string DateFormat = "yyyyMMdd";
public OsgDnaNoticesProcessingService(IPrimaryOwnerDeterminingService userIdDeterminingService,
IFileOperationsApiService fileOperationsApiService,
IDocumentTypeUploaderApiService documentTypeUploaderApiService,
IAccountOwnerDeterminingService accountDeterminingService,
ServicesFactoryConfiguration servicesFactoryConfiguration)
: base(servicesFactoryConfiguration)
{
_documentTypeUploaderApiService = documentTypeUploaderApiService;
_fileOperationsApiService = fileOperationsApiService;
_primaryOwnerDeterminingService = userIdDeterminingService;
_accountOwnerDeterminingService = accountDeterminingService;
}
public async Task<ProcessResult> ProcessRequest(ProcessingRequest request)
{
var processResult = new ProcessResult();
try
{
Log.Debug("Starting OsgDnaNoticesDocument ProcessRequest.");
ArgumentNullException.ThrowIfNull(request);
var documentValues = JsonConvert.DeserializeObject<OsgDnaNoticesDocument>(request.DocumentValuePayload);
ArgumentNullException.ThrowIfNull(documentValues);
processResult = ValidateRecord(processResult, documentValues);
if (!processResult.Success)
{
return processResult;
}
if (!DateTime.TryParseExact(documentValues.CreationDate, DateFormat, CultureInfo.InvariantCulture, DateTimeStyles.AssumeLocal, out var creationDate))
{
var message = $"A Creation Date of '{documentValues.CreationDate}' could not be properly formatted " +
$"for an Account Owner of AccountNumber: {documentValues?.AccountNumber} and EntityId: {documentValues?.MemberNumberOrUserId}";
processResult = ProcessResult.ProcessDataMappingFailure(processResult, message);
return processResult;
}
if (EnumExtensions.IsDocumentTypeOf<PrimaryOwnerDocumentTypes>(documentValues.DocType))
{
var primaryOwnerAttribute = await _primaryOwnerDeterminingService.DeterminePrimaryOwnerAttributeByMemberNumberOrUserId(documentValues.MemberNumberOrUserId,
documentValues.AccountNumber,
documentValues.DocType,
creationDate,
request.OwnerOverrideInfo);
if (!primaryOwnerAttribute.Success)
{
processResult = ProcessResult.ProcessDataMappingFailure(processResult,
primaryOwnerAttribute.Message);
return processResult;
}
var primaryOwnerRequest = await BuildPrimaryOwnerUploadRequest(request, documentValues, creationDate, primaryOwnerAttribute.Payload);
await _documentTypeUploaderApiService.UploadPrimaryOwnerDocument(primaryOwnerRequest, request, processResult);
}
else if (EnumExtensions.IsDocumentTypeOf<AccountDocumentTypes>(documentValues.DocType))
{
var accountOwnerAttribute = await _accountOwnerDeterminingService.DetermineAccountOwnerAttributeByMemberNumberOrUserId(documentValues.MemberNumberOrUserId,
documentValues.AccountNumber,
documentValues.DocType,
creationDate,
request.OwnerOverrideInfo);
if (!accountOwnerAttribute.Success)
{
processResult = ProcessResult.ProcessDataMappingFailure(processResult,
accountOwnerAttribute.Message);
return processResult;
}
var accountOwnerRequest = await BuildAccountUploadRequest(request, documentValues, creationDate, accountOwnerAttribute.Payload);
await _documentTypeUploaderApiService.UploadAccountDocument(accountOwnerRequest, request, processResult);
}
else
{
var message = $"Document Type of {documentValues.DocType} could not be found." +
$"for an Account Owner of AccountNumber: {documentValues?.AccountNumber} and EntityName: {documentValues?.MemberNumberOrUserId}";
Log.Error(message);
processResult = ProcessResult.ProcessCriticalFailure(processResult, message);
return processResult;
}
Log.Debug("Ending OsgDnaNotices ProcessRequest.");
}
catch (Exception exception)
{
var message = "An Exception occured while OsgDnaNoticesDocument ProcessRequest. " +
$"Exception Message: {exception.Message} ";
Log.Error(exception, message);
processResult = ProcessResult.ProcessCriticalFailure(processResult, message);
}
return processResult;
}
public async Task<AccountDocumentRequest> BuildAccountUploadRequest(ProcessingRequest request,
OsgDnaNoticesDocument documentValues,
DateTime creationDate,
AccountAttribute accountAttribute)
{
var contentBytes = await _fileOperationsApiService.GetContentBytes(request.IngestorRequest.RootPath,
request.DocumentFileName);
var type = documentValues.DocType.GetValueFromName<AccountDocumentTypes>();
ArgumentNullException.ThrowIfNull(type);
var accountOwnerDocumentRequest = new AccountDocumentRequest
{
Content = contentBytes,
AccountAttribute = accountAttribute,
AccountDocumentMetadata = new AccountDocumentMetadata
{
Source = Source,
DataSource = DataSource,
Name = request.DocumentFileName,
Type = type,
CreatedDate = creationDate,
TraceId = request.IngestorRunInfoId
}
};
return accountOwnerDocumentRequest;
}
public async Task<PrimaryOwnerDocumentRequest> BuildPrimaryOwnerUploadRequest(ProcessingRequest request,
OsgDnaNoticesDocument documentValues,
DateTime creationDate,
PrimaryOwnerAttribute primaryOwnerAttribute)
{
var contentBytes = await _fileOperationsApiService.GetContentBytes(request.IngestorRequest.RootPath,
request.DocumentFileName);
var type = documentValues.DocType.GetValueFromName<PrimaryOwnerDocumentTypes>();
ArgumentNullException.ThrowIfNull(type);
var primaryOwnerDocumentRequest = new PrimaryOwnerDocumentRequest
{
Content = contentBytes,
PrimaryOwnerAttribute = primaryOwnerAttribute,
PrimaryOwnerDocumentMetadata = new PrimaryOwnerDocumentMetadata
{
Source = Source,
DataSource = DataSource,
Name = request.DocumentFileName,
Type = type,
CreatedDate = creationDate,
TraceId = request.IngestorRunInfoId
}
};
return primaryOwnerDocumentRequest;
}
private static ProcessResult ValidateRecord(ProcessResult processResult, OsgDnaNoticesDocument documentValues)
{
processResult = MetaDataValidation.ValidateAccountNumber(processResult, documentValues.AccountNumber);
processResult = MetaDataValidation.ValidateDate(processResult, documentValues.CreationDate, DateFormat);
return processResult;
}
}
}
@@ -0,0 +1,134 @@
using LCU.API.Internal.DocumentIngestor.Services.AttributeDeterminingServices;
using LCU.API.Internal.DocumentIngestor.Services.MicroApiServices;
using LCU.Library.DocumentType.Enums;
using LCU.Library.DocumentType.Extensions;
using LCU.Library.DocumentType.Models.Requests.Attributes;
using LCU.Library.DocumentType.Models.Requests.PrimaryOwner;
using LCU.Library.Internal.DocumentIngestorModels.Models;
using LCU.Library.Internal.DocumentIngestorModels.Models.IndexFileDocuments;
using LCU.Library.Internal.DocumentIngestorModels.Validations;
using Newtonsoft.Json;
using Serilog;
using System.Globalization;
using LCU.Library.Internal.LCUServiceFactory.Configurations;
namespace LCU.API.Internal.DocumentIngestor.Services.DocumentProcessorServices
{
public class OsgHistoricalDocumentProcessingService : DocumentProcessingService, IDocumentProcessingService
{
private const DataSourceTypes DataSource = DataSourceTypes.DNA;
private const SourceTypes Source = SourceTypes.OSG;
private readonly IPrimaryOwnerDeterminingService _primaryOwnerDeterminingService;
private readonly IFileOperationsApiService _fileOperationsApiService;
private readonly IDocumentTypeUploaderApiService _documentTypeUploaderApiService;
private const string DateFormat = "yyyyMMdd";
public OsgHistoricalDocumentProcessingService(IPrimaryOwnerDeterminingService userIdDeterminingService,
IFileOperationsApiService fileOperationsApiService,
IDocumentTypeUploaderApiService documentTypeUploaderApiService,
ServicesFactoryConfiguration servicesFactoryConfiguration)
: base(servicesFactoryConfiguration)
{
_documentTypeUploaderApiService = documentTypeUploaderApiService;
_fileOperationsApiService = fileOperationsApiService;
_primaryOwnerDeterminingService = userIdDeterminingService;
}
public async Task<ProcessResult> ProcessRequest(ProcessingRequest request)
{
var processResult = new ProcessResult();
try
{
Log.Debug("Starting OsgHistoricalDocument ProcessRequest.");
ArgumentNullException.ThrowIfNull(request);
var documentValues = JsonConvert.DeserializeObject<OsgHistoricalDocument>(request.DocumentValuePayload);
ArgumentNullException.ThrowIfNull(documentValues);
processResult = ValidateRecord(processResult, documentValues);
if (!processResult.Success)
{
return processResult;
}
if (!DateTime.TryParseExact(documentValues.CreationDate, DateFormat, CultureInfo.InvariantCulture, DateTimeStyles.AssumeLocal, out var creationDate))
{
var message = $"A Creation Date of '{documentValues.CreationDate}' could not be properly formatted " +
$"for an Account Owner of EntityName: {documentValues?.EntityName} and EntityId: {documentValues?.MemberNumberOrUserId}";
processResult = ProcessResult.ProcessDataMappingFailure(processResult, message);
return processResult;
}
if (EnumExtensions.IsDocumentTypeOf<PrimaryOwnerDocumentTypes>(documentValues.DocType))
{
var primaryOwnerAttribute = await _primaryOwnerDeterminingService.DeterminePrimaryOwnerAttributeByMemberNumberOrUserId(documentValues.MemberNumberOrUserId,
documentValues.EntityName,
request.OwnerOverrideInfo);
if (!primaryOwnerAttribute.Success)
{
processResult = ProcessResult.ProcessDataMappingFailure(processResult, primaryOwnerAttribute.Message);
return processResult;
}
var primaryOwnerRequest = await BuildPrimaryOwnerUploadRequest(request, documentValues, creationDate, primaryOwnerAttribute.Payload);
await _documentTypeUploaderApiService.UploadPrimaryOwnerDocument(primaryOwnerRequest, request, processResult);
}
else
{
var message = $"Document Type of {documentValues.DocType} could not be found." +
$"for an Account Owner of EntityName: {documentValues?.EntityName} and EntityId: {documentValues?.MemberNumberOrUserId}";
Log.Error(message);
processResult = ProcessResult.ProcessCriticalFailure(processResult, message);
return processResult;
}
Log.Debug("Ending OsgHistoricalDocument ProcessRequest.");
}
catch (Exception exception)
{
var message = "An Exception occured while OsgHistoricalDocument ProcessRequest. " +
$"Exception Message: {exception.Message} ";
Log.Error(exception, message);
processResult = ProcessResult.ProcessCriticalFailure(processResult, message);
}
return processResult;
}
private async Task<PrimaryOwnerDocumentRequest> BuildPrimaryOwnerUploadRequest(ProcessingRequest request,
OsgHistoricalDocument documentValues,
DateTime creationDate,
PrimaryOwnerAttribute primaryOwnerAttribute)
{
var contentBytes = await _fileOperationsApiService.GetContentBytes(request.IngestorRequest.RootPath,
request.DocumentFileName);
var type = documentValues.DocType.GetValueFromName<PrimaryOwnerDocumentTypes>();
ArgumentNullException.ThrowIfNull(type);
var primaryOwnerDocumentRequest = new PrimaryOwnerDocumentRequest
{
Content = contentBytes,
PrimaryOwnerAttribute = primaryOwnerAttribute,
PrimaryOwnerDocumentMetadata = new PrimaryOwnerDocumentMetadata
{
Source = Source,
DataSource = DataSource,
Name = request.DocumentFileName,
Type = type,
CreatedDate = creationDate,
TraceId = request.IngestorRunInfoId
}
};
return primaryOwnerDocumentRequest;
}
private static ProcessResult ValidateRecord(ProcessResult processResult, OsgHistoricalDocument documentValues)
{
processResult = MetaDataValidation.ValidateFullName(processResult, documentValues.EntityName);
processResult = MetaDataValidation.ValidateDate(processResult, documentValues.CreationDate, DateFormat);
return processResult;
}
}
}
@@ -0,0 +1,173 @@
using Newtonsoft.Json;
using Serilog;
using LCU.Library.DocumentType.Enums;
using LCU.Library.DocumentType.Extensions;
using LCU.Library.DocumentType.Models.Requests.Attributes;
using LCU.Library.DocumentType.Models.Requests.PrimaryOwner;
using LCU.Library.DocumentType.Models.Requests.Generic;
using LCU.Library.Internal.DocumentIngestorModels.Models;
using LCU.Library.Internal.DocumentIngestorModels.Models.IndexFileDocuments;
using LCU.Library.Internal.DocumentIngestorModels.Validations;
using LCU.API.Internal.DocumentIngestor.Services.AttributeDeterminingServices;
using LCU.API.Internal.DocumentIngestor.Services.MicroApiServices;
using Microsoft.VisualBasic;
using System.Globalization;
using LCU.Library.Internal.LCUServiceFactory.Configurations;
namespace LCU.API.Internal.DocumentIngestor.Services.DocumentProcessorServices;
public class OsgTaxStatementProcessingService : DocumentProcessingService, IDocumentProcessingService
{
private readonly IPrimaryOwnerDeterminingService _primaryOwnerDeterminingService;
private readonly IFileOperationsApiService _fileOperationsApiService;
private readonly IDocumentTypeUploaderApiService _documentTypeUploaderApiService;
private const DataSourceTypes DataSource = DataSourceTypes.DNA;
private const SourceTypes Source = SourceTypes.OSG;
private const string DateFormat = "MMddyyyy";
public OsgTaxStatementProcessingService(IPrimaryOwnerDeterminingService userIdDeterminingService,
IFileOperationsApiService fileOperationsApiService,
IDocumentTypeUploaderApiService documentTypeUploaderApiService,
ServicesFactoryConfiguration servicesFactoryConfiguration)
: base(servicesFactoryConfiguration)
{
_documentTypeUploaderApiService = documentTypeUploaderApiService;
_fileOperationsApiService = fileOperationsApiService;
_primaryOwnerDeterminingService = userIdDeterminingService;
}
public async Task<ProcessResult> ProcessRequest(ProcessingRequest request)
{
var processResult = new ProcessResult();
try
{
Log.Debug("Starting OsgTaxStatement ProcessRequest.");
ArgumentNullException.ThrowIfNull(request);
var documentValues = JsonConvert.DeserializeObject<OsgTaxStatement>(request.DocumentValuePayload);
ArgumentNullException.ThrowIfNull(documentValues);
processResult = ValidateRecord(processResult, documentValues);
if (!processResult.Success)
{
return processResult;
}
if (!DateTime.TryParseExact(documentValues.CreationDate, DateFormat, CultureInfo.InvariantCulture, DateTimeStyles.AssumeLocal, out var creationDate))
{
var message = $"A Creation Date of '{documentValues.CreationDate}' could not be properly formatted " +
$"for an Account Owner of EntityName: {documentValues?.RecipientName} and EntityId: {documentValues?.AccountNumber}";
processResult = ProcessResult.ProcessDataMappingFailure(processResult, message);
return processResult;
}
if (EnumExtensions.IsDocumentTypeOf<PrimaryOwnerDocumentTypes>(documentValues.NoticeName))
{
var primaryOwnerAttribute = await _primaryOwnerDeterminingService.DeterminePrimaryOwnerAttribute(
documentValues.RecipientName,
documentValues.AccountNumber,
documentValues.SSN,
request.OwnerOverrideInfo);
if (!primaryOwnerAttribute.Success)
{
processResult = ProcessResult.ProcessDataMappingFailure(processResult,
primaryOwnerAttribute.Message);
return processResult;
}
var uploadRequest = await BuildUploadRequest(request, documentValues, primaryOwnerAttribute.Payload, creationDate);
await _documentTypeUploaderApiService.UploadPrimaryOwnerDocument(uploadRequest, request, processResult);
}
else if (EnumExtensions.IsDocumentTypeOf<GenericDocumentTypes>(documentValues.NoticeName))
{
var genericAttribute = new GenericDocumentAttribute
{
EntityName = documentValues.RecipientName
};
var uploadRequest = await BuildGenericUploadRequest(request, documentValues, genericAttribute);
await _documentTypeUploaderApiService.UploadGenericDocument(uploadRequest, request, processResult);
}
else
{
var message = $"Document Type of {documentValues.NoticeName} could not be found.";
Log.Error(message);
processResult = ProcessResult.ProcessCriticalFailure(processResult, message);
return processResult;
}
}
catch (Exception exception)
{
var message = "An Exception occured while OsgTaxStatement ProcessRequest. " +
$"Exception Message: {exception.Message} ";
Log.Error(exception, message);
processResult = ProcessResult.ProcessCriticalFailure(processResult, message);
}
Log.Debug("Ending OsgTaxStatement ProcessRequest.");
return processResult;
}
public ProcessResult ValidateRecord(ProcessResult processResult, OsgTaxStatement documentValues)
{
processResult = MetaDataValidation.ValidateAccountNumber(processResult, documentValues.AccountNumber);
processResult = MetaDataValidation.ValidateFileName(processResult, documentValues.NoticeName);
processResult = MetaDataValidation.ValidateFullName(processResult, documentValues.RecipientName);
processResult = MetaDataValidation.ValidatePdf(processResult, documentValues.PdfName);
processResult = MetaDataValidation.ValidateTin(processResult, documentValues.SSN);
return processResult;
}
public async Task<PrimaryOwnerDocumentRequest> BuildUploadRequest(ProcessingRequest request,
OsgTaxStatement documentValues,
PrimaryOwnerAttribute primaryOwnerAttribute,
DateTime creationDate)
{
var contentBytes = await _fileOperationsApiService.GetContentBytes(request.IngestorRequest.RootPath,
request.DocumentFileName);
var type = documentValues.NoticeName.GetValueFromName<PrimaryOwnerDocumentTypes>();
var primaryOwnerDocumentRequest = new PrimaryOwnerDocumentRequest
{
Content = contentBytes,
PrimaryOwnerAttribute = primaryOwnerAttribute,
PrimaryOwnerDocumentMetadata = new PrimaryOwnerDocumentMetadata
{
Source = Source,
DataSource = DataSource,
Name = documentValues.PdfName,
Type = type,
CreatedDate = creationDate,
TraceId = request.IngestorRunInfoId
}
};
return primaryOwnerDocumentRequest;
}
public async Task<GenericDocumentRequest> BuildGenericUploadRequest(ProcessingRequest request,
OsgTaxStatement documentValues,
GenericDocumentAttribute genericDocumentAttribute)
{
var contentBytes = await _fileOperationsApiService.GetContentBytes(request.IngestorRequest.RootPath,
request.DocumentFileName);
var type = documentValues.NoticeName.GetValueFromName<GenericDocumentTypes>();
var genericDocumentRequest = new GenericDocumentRequest
{
Content = contentBytes,
GenericDocumentAttribute = genericDocumentAttribute,
GenericDocumentMetadata = new GenericDocumentMetadata
{
Source = Source,
DataSource = DataSource,
Name = documentValues.PdfName,
Type = type,
CreatedDate = DateTime.Now.Date,
TraceId = request.IngestorRunInfoId
}
};
return genericDocumentRequest;
}
}
@@ -0,0 +1,131 @@
using System.Globalization;
using Newtonsoft.Json;
using Serilog;
using LCU.Library.DocumentType.Enums;
using LCU.Library.DocumentType.Extensions;
using LCU.Library.DocumentType.Models.Requests.Account;
using LCU.Library.DocumentType.Models.Requests.Attributes;
using LCU.Library.Internal.DocumentIngestorModels.Models;
using LCU.Library.Internal.DocumentIngestorModels.Models.IndexFileDocuments;
using LCU.Library.Internal.DocumentIngestorModels.Validations;
using LCU.API.Internal.DocumentIngestor.Services.AttributeDeterminingServices;
using LCU.API.Internal.DocumentIngestor.Services.MicroApiServices;
using LCU.Library.Internal.LCUServiceFactory.Configurations;
namespace LCU.API.Internal.DocumentIngestor.Services.DocumentProcessorServices;
public class OsgTemenosNoticeProcessingService : DocumentProcessingService, IDocumentProcessingService
{
private readonly IAccountOwnerDeterminingService _accountOwnerDeterminingService;
private readonly IFileOperationsApiService _fileOperationsApiService;
private readonly IDocumentTypeUploaderApiService _documentTypeUploaderApiService;
private const DataSourceTypes DataSource = DataSourceTypes.DNA;
private const SourceTypes Source = SourceTypes.XDI;
private const string DateFormat = "MMddyyyy";
public OsgTemenosNoticeProcessingService(IAccountOwnerDeterminingService accountOwnerDeterminingService,
IFileOperationsApiService fileOperationsApiService,
IDocumentTypeUploaderApiService documentTypeUploaderApiService,
ServicesFactoryConfiguration servicesFactoryConfiguration)
: base(servicesFactoryConfiguration)
{
_documentTypeUploaderApiService = documentTypeUploaderApiService;
_fileOperationsApiService = fileOperationsApiService;
_accountOwnerDeterminingService = accountOwnerDeterminingService;
}
public async Task<ProcessResult> ProcessRequest(ProcessingRequest request)
{
var processResult = new ProcessResult();
try
{
Log.Debug("Starting OsgTemenosNotice ProcessRequest.");
ArgumentNullException.ThrowIfNull(request);
var documentValues = JsonConvert.DeserializeObject<OsgTemenosNoticesDocument>(request.DocumentValuePayload);
ArgumentNullException.ThrowIfNull(documentValues);
processResult = ValidateRecord(processResult, documentValues);
if (!processResult.Success)
{
return processResult;
}
if (!DateTime.TryParseExact(documentValues.CreationDate,
DateFormat,
CultureInfo.InvariantCulture,
DateTimeStyles.AssumeLocal,
out var creationDate))
{
var message = $"A Creation Date of {documentValues.CreationDate} could not " +
$"be properly formatted from format: {DateFormat}";
return ProcessResult.ProcessDataMappingFailure(processResult,
message);
}
var accountOwnerAttribute = await _accountOwnerDeterminingService.DetermineAccountOwnerAttribute(documentValues.EntityName,
documentValues.AccountNumber,
creationDate,
request.OwnerOverrideInfo);
if (!accountOwnerAttribute.Success)
{
return ProcessResult.ProcessDataMappingFailure(processResult,
accountOwnerAttribute.Message);
}
var accountOwnerRequest = await BuildUploadRequest(request,
documentValues,
creationDate,
accountOwnerAttribute.Payload);
await _documentTypeUploaderApiService.UploadAccountDocument(accountOwnerRequest, request, processResult);
Log.Debug("Ending OsgTemenosNoticesDocument ProcessRequest.");
}
catch (Exception exception)
{
var message =
$"An Exception occured while OsgTemenosNoticesDocument ProcessRequest. Exception Message: {exception.Message}";
Log.Error(exception, message);
processResult = ProcessResult.ProcessCriticalFailure(processResult, message);
}
return processResult;
}
private static ProcessResult ValidateRecord(ProcessResult processResult, OsgTemenosNoticesDocument documentValues)
{
processResult = MetaDataValidation.ValidateAccountNumber(processResult, documentValues.AccountNumber);
processResult = MetaDataValidation.ValidatePdf(processResult, documentValues.PDF);
processResult = MetaDataValidation.ValidateFullName(processResult, documentValues.EntityName);
processResult = MetaDataValidation.ValidateDate(processResult, documentValues.CreationDate, DateFormat);
processResult = MetaDataValidation.ValidateDocumentType<AccountDocumentTypes>(processResult, documentValues.DocumentType);
return processResult;
}
private async Task<AccountDocumentRequest> BuildUploadRequest(ProcessingRequest request,
OsgTemenosNoticesDocument documentValues,
DateTime creationDate,
AccountAttribute accountAttribute)
{
var contentBytes = await _fileOperationsApiService.GetContentBytes(request.IngestorRequest.RootPath, request.DocumentFileName);
var type = documentValues.DocumentType.GetValueFromName<AccountDocumentTypes>();
ArgumentNullException.ThrowIfNull(type);
return new AccountDocumentRequest
{
Content = contentBytes,
AccountAttribute = accountAttribute,
AccountDocumentMetadata = new AccountDocumentMetadata
{
Source = Source,
DataSource = DataSource,
Name = documentValues.PDF,
Type = type,
CreatedDate = creationDate,
TraceId = request.IngestorRunInfoId
}
};
}
}
@@ -0,0 +1,126 @@
using Newtonsoft.Json;
using Serilog;
using LCU.Library.DocumentType.Enums;
using LCU.Library.DocumentType.Extensions;
using LCU.Library.DocumentType.Models.Requests.Attributes;
using LCU.Library.DocumentType.Models.Requests.PrimaryOwner;
using LCU.Library.DocumentType.Models.Requests.Generic;
using LCU.Library.Internal.DocumentIngestorModels.Models;
using LCU.Library.Internal.DocumentIngestorModels.Models.IndexFileDocuments;
using LCU.Library.Internal.DocumentIngestorModels.Validations;
using LCU.API.Internal.DocumentIngestor.Services.AttributeDeterminingServices;
using LCU.API.Internal.DocumentIngestor.Services.MicroApiServices;
using Microsoft.VisualBasic;
using System.Globalization;
using LCU.Library.Internal.LCUServiceFactory.Configurations;
namespace LCU.API.Internal.DocumentIngestor.Services.DocumentProcessorServices;
public class OsgVehicleTaxStatementProcessingService : DocumentProcessingService, IDocumentProcessingService
{
private readonly IPrimaryOwnerDeterminingService _primaryOwnerDeterminingService;
private readonly IFileOperationsApiService _fileOperationsApiService;
private readonly IDocumentTypeUploaderApiService _documentTypeUploaderApiService;
private const DataSourceTypes DataSource = DataSourceTypes.DNA;
private const SourceTypes Source = SourceTypes.OSG;
private const string DateFormat = "MMddyyyy";
public OsgVehicleTaxStatementProcessingService(IPrimaryOwnerDeterminingService userIdDeterminingService,
IFileOperationsApiService fileOperationsApiService,
IDocumentTypeUploaderApiService documentTypeUploaderApiService,
ServicesFactoryConfiguration servicesFactoryConfiguration)
: base(servicesFactoryConfiguration)
{
_documentTypeUploaderApiService = documentTypeUploaderApiService;
_fileOperationsApiService = fileOperationsApiService;
_primaryOwnerDeterminingService = userIdDeterminingService;
}
public async Task<ProcessResult> ProcessRequest(ProcessingRequest request)
{
var processResult = new ProcessResult();
try
{
Log.Debug("Starting OsgVehicleTaxStatement ProcessRequest.");
ArgumentNullException.ThrowIfNull(request);
var documentValues = JsonConvert.DeserializeObject<OsgTaxStatement>(request.DocumentValuePayload);
ArgumentNullException.ThrowIfNull(documentValues);
processResult = ValidateRecord(processResult, documentValues);
if (!processResult.Success)
{
return processResult;
}
if (!DateTime.TryParseExact(documentValues.CreationDate, DateFormat, CultureInfo.InvariantCulture, DateTimeStyles.AssumeLocal, out var creationDate))
{
var message = $"A Creation Date of '{documentValues.CreationDate}' could not be properly formatted " +
$"for an Account Owner of EntityName: {documentValues?.RecipientName} and EntityId: {documentValues?.AccountNumber}";
processResult = ProcessResult.ProcessDataMappingFailure(processResult, message);
return processResult;
}
var primaryOwnerAttribute = await _primaryOwnerDeterminingService.DeterminePrimaryOwnerAttribute(
documentValues.RecipientName,
documentValues.AccountNumber,
creationDate,
request.OwnerOverrideInfo);
if (!primaryOwnerAttribute.Success)
{
processResult = ProcessResult.ProcessDataMappingFailure(processResult,
primaryOwnerAttribute.Message);
return processResult;
}
var uploadRequest = await BuildUploadRequest(request, documentValues, primaryOwnerAttribute.Payload, creationDate);
await _documentTypeUploaderApiService.UploadPrimaryOwnerDocument(uploadRequest, request, processResult);
}
catch (Exception exception)
{
var message = "An Exception occured while OsgVehicleTaxStatement ProcessRequest. " +
$"Exception Message: {exception.Message} ";
Log.Error(exception, message);
processResult = ProcessResult.ProcessCriticalFailure(processResult, message);
}
Log.Debug("Ending OsgVehicleTaxStatement ProcessRequest.");
return processResult;
}
public ProcessResult ValidateRecord(ProcessResult processResult, OsgTaxStatement documentValues)
{
processResult = MetaDataValidation.ValidateAccountNumber(processResult, documentValues.AccountNumber);
processResult = MetaDataValidation.ValidateFileName(processResult, documentValues.NoticeName);
processResult = MetaDataValidation.ValidateFullName(processResult, documentValues.RecipientName);
processResult = MetaDataValidation.ValidatePdf(processResult, documentValues.PdfName);
return processResult;
}
public async Task<PrimaryOwnerDocumentRequest> BuildUploadRequest(ProcessingRequest request,
OsgTaxStatement documentValues,
PrimaryOwnerAttribute primaryOwnerAttribute,
DateTime creationDate)
{
var contentBytes = await _fileOperationsApiService.GetContentBytes(request.IngestorRequest.RootPath,
request.DocumentFileName);
var type = documentValues.NoticeId.GetValueFromName<PrimaryOwnerDocumentTypes>();
var primaryOwnerDocumentRequest = new PrimaryOwnerDocumentRequest
{
Content = contentBytes,
PrimaryOwnerAttribute = primaryOwnerAttribute,
PrimaryOwnerDocumentMetadata = new PrimaryOwnerDocumentMetadata
{
Source = Source,
DataSource = DataSource,
Name = documentValues.PdfName,
Type = type,
CreatedDate = creationDate,
TraceId = request.IngestorRunInfoId
}
};
return primaryOwnerDocumentRequest;
}
}
@@ -0,0 +1,137 @@
using LCU.API.Internal.DocumentIngestor.Services.AttributeDeterminingServices;
using LCU.API.Internal.DocumentIngestor.Services.MicroApiServices;
using LCU.Library.DocumentType.Enums;
using LCU.Library.DocumentType.Extensions;
using LCU.Library.DocumentType.Models.Requests.Attributes;
using LCU.Library.DocumentType.Models.Requests.Account;
using LCU.Library.Internal.DocumentIngestorModels.Models;
using LCU.Library.Internal.DocumentIngestorModels.Models.IndexFileDocuments;
using LCU.Library.Internal.DocumentIngestorModels.Validations;
using Newtonsoft.Json;
using Serilog;
using System.Globalization;
using LCU.Library.Internal.LCUServiceFactory.Configurations;
namespace LCU.API.Internal.DocumentIngestor.Services.DocumentProcessorServices
{
public class OsgWeilandStatementProcessingService : DocumentProcessingService, IDocumentProcessingService
{
private const DataSourceTypes DataSource = DataSourceTypes.WEILAND;
private const SourceTypes Source = SourceTypes.OSG;
private readonly IAccountOwnerDeterminingService _accountOwnerDeterminingService;
private readonly IFileOperationsApiService _fileOperationsApiService;
private readonly IDocumentTypeUploaderApiService _documentTypeUploaderApiService;
private const string DateFormat = "yyyyMMdd";
public OsgWeilandStatementProcessingService(IAccountOwnerDeterminingService accountOwnerDeterminingService,
IFileOperationsApiService fileOperationsApiService,
IDocumentTypeUploaderApiService documentTypeUploaderApiService,
ServicesFactoryConfiguration servicesFactoryConfiguration)
: base(servicesFactoryConfiguration)
{
_documentTypeUploaderApiService = documentTypeUploaderApiService;
_fileOperationsApiService = fileOperationsApiService;
_accountOwnerDeterminingService = accountOwnerDeterminingService;
}
public async Task<ProcessResult> ProcessRequest(ProcessingRequest request)
{
var processResult = new ProcessResult();
try
{
Log.Debug("Starting OsgWeilandDocument ProcessRequest.");
ArgumentNullException.ThrowIfNull(request);
var documentValues = JsonConvert.DeserializeObject<OsgWeilandStatement>(request.DocumentValuePayload);
ArgumentNullException.ThrowIfNull(documentValues);
processResult = ValidateRecord(processResult, documentValues);
if (!processResult.Success)
{
return processResult;
}
if (!DateTime.TryParseExact(documentValues.Date, DateFormat, CultureInfo.InvariantCulture, DateTimeStyles.AssumeLocal, out var creationDate))
{
var message = $"A Creation Date of '{documentValues.Date}' could not be properly formatted " +
$"for an Account Owner of EntityName: {documentValues?.AccountTitle} and EntityId: {documentValues?.AccountNumber}";
processResult = ProcessResult.ProcessDataMappingFailure(processResult, message);
return processResult;
}
if (EnumExtensions.IsDocumentTypeOf<AccountDocumentTypes>(documentValues.DocCode))
{
var accountOwnerAttribute = await _accountOwnerDeterminingService.DetermineAccountOwnerAttribute(documentValues.AccountTitle,
documentValues.AccountNumber,
creationDate,
request.OwnerOverrideInfo);
if (!accountOwnerAttribute.Success)
{
processResult = ProcessResult.ProcessDataMappingFailure(processResult, accountOwnerAttribute.Message);
return processResult;
}
var accountOwnerRequest = await BuildUploadRequest(request,
documentValues,
creationDate,
accountOwnerAttribute.Payload);
await _documentTypeUploaderApiService.UploadAccountDocument(accountOwnerRequest, request, processResult);
}
else
{
var message = $"Document Type of {documentValues.DocCode} could not be found." +
$"for an Account Owner of EntityName: {documentValues?.AccountTitle} and EntityId: {documentValues?.AccountNumber}";
Log.Error(message);
processResult = ProcessResult.ProcessCriticalFailure(processResult, message);
return processResult;
}
Log.Debug("Ending OsgWeilandStatement ProcessRequest.");
}
catch (Exception exception)
{
var message = "An Exception occured while OsgWeilandStatement ProcessRequest. " +
$"Exception Message: {exception.Message} ";
Log.Error(exception, message);
processResult = ProcessResult.ProcessCriticalFailure(processResult, message);
}
return processResult;
}
private async Task<AccountDocumentRequest> BuildUploadRequest(ProcessingRequest request,
OsgWeilandStatement documentValues,
DateTime creationDate,
AccountAttribute accountAttribute)
{
var contentBytes = await _fileOperationsApiService.GetContentBytes(request.IngestorRequest.RootPath, request.DocumentFileName);
var type = documentValues.DocCode.GetValueFromName<AccountDocumentTypes>();
ArgumentNullException.ThrowIfNull(type);
return new AccountDocumentRequest
{
Content = contentBytes,
AccountAttribute = accountAttribute,
AccountDocumentMetadata = new AccountDocumentMetadata
{
Source = Source,
DataSource = DataSource,
Name = documentValues.PdfName,
Type = type,
CreatedDate = creationDate,
TraceId = request.IngestorRunInfoId
}
};
}
private static ProcessResult ValidateRecord(ProcessResult processResult, OsgWeilandStatement documentValues)
{
processResult = MetaDataValidation.ValidateAccountNumber(processResult, documentValues.AccountNumber);
processResult = MetaDataValidation.ValidatePdf(processResult, documentValues.PdfName);
processResult = MetaDataValidation.ValidateDate(processResult, documentValues.Date, DateFormat);
processResult = MetaDataValidation.ValidateDocumentType<AccountDocumentTypes>(processResult, documentValues.DocCode);
return processResult;
}
}
}
@@ -0,0 +1,105 @@
using LCU.API.Internal.DocumentIngestor.Services.MicroApiServices;
using LCU.Library.DocumentType.Enums;
using LCU.Library.DocumentType.Extensions;
using LCU.Library.DocumentType.Models.Requests.Generic;
using LCU.Library.DocumentType.Models.Requests.Attributes;
using LCU.Library.Internal.DocumentIngestorModels.Models;
using LCU.Library.Internal.DocumentIngestorModels.Models.IndexFileDocuments;
using LCU.Library.Internal.DocumentIngestorModels.Validations;
using LCU.Library.Internal.LCUServiceFactory.Configurations;
using Newtonsoft.Json;
using Serilog;
namespace LCU.API.Internal.DocumentIngestor.Services.DocumentProcessorServices;
public class XdiActionNoticesProcessingService : DocumentProcessingService, IDocumentProcessingService
{
private const DataSourceTypes DataSource = DataSourceTypes.ACTION;
private const SourceTypes Source = SourceTypes.XDI;
private const string DateFormat = "mm/dd/yyyy";
private readonly IFileOperationsApiService _fileOperationsApiService;
private readonly IDocumentTypeUploaderApiService _documentTypeUploaderApiService;
public XdiActionNoticesProcessingService(IFileOperationsApiService fileOperationsApiService,
IDocumentTypeUploaderApiService documentTypeUploaderApiService,
ServicesFactoryConfiguration servicesFactoryConfiguration)
: base(servicesFactoryConfiguration)
{
_documentTypeUploaderApiService = documentTypeUploaderApiService;
_fileOperationsApiService = fileOperationsApiService;
}
public async Task<ProcessResult> ProcessRequest(ProcessingRequest request)
{
var processResult = new ProcessResult();
try
{
Log.Debug("Starting XdiActionNotice ProcessRequest.");
ArgumentNullException.ThrowIfNull(request);
var documentValues = JsonConvert.DeserializeObject<XdiActionNoticesDocument>(request.DocumentValuePayload);
ArgumentNullException.ThrowIfNull(documentValues);
processResult = ValidateRecord(processResult, documentValues);
if (!processResult.Success)
{
return processResult;
}
var genericAttribute = new GenericDocumentAttribute
{
EntityName = documentValues.EntityName
};
var uploadRequest = await BuildUploadRequest(request, documentValues, genericAttribute);
await _documentTypeUploaderApiService.UploadGenericDocument(uploadRequest, request, processResult);
Log.Debug("Ending XdiActionNoticeDocument ProcessRequest.");
}
catch (Exception exception)
{
var message =
$"An Exception occured while XdiActionNoticeDocument ProcessRequest. Exception Message: {exception.Message}";
Log.Error(exception, message);
processResult = ProcessResult.ProcessCriticalFailure(processResult, message);
}
return processResult;
}
private static ProcessResult ValidateRecord(ProcessResult processResult, XdiActionNoticesDocument documentValues)
{
processResult = MetaDataValidation.ValidateFullName(processResult, documentValues.EntityName);
processResult = MetaDataValidation.ValidateDate(processResult, documentValues.CreationDate, DateFormat);
processResult = MetaDataValidation.ValidateDocumentType<GenericDocumentTypes>(processResult, documentValues.DocumentType);
return processResult;
}
public async Task<GenericDocumentRequest> BuildUploadRequest(ProcessingRequest request,
XdiActionNoticesDocument documentValues,
GenericDocumentAttribute genericDocumentAttribute)
{
var contentBytes = await _fileOperationsApiService.GetContentBytes(request.IngestorRequest.RootPath,
request.DocumentFileName);
var type = documentValues.DocumentType.GetValueFromName<GenericDocumentTypes>();
var genericDocumentRequest = new GenericDocumentRequest
{
Content = contentBytes,
GenericDocumentAttribute = genericDocumentAttribute,
GenericDocumentMetadata = new GenericDocumentMetadata
{
Source = Source,
DataSource = DataSource,
Name = documentValues.DocumentName,
Type = type,
CreatedDate = DateTime.Parse(documentValues.CreationDate),
TraceId = request.IngestorRunInfoId
}
};
return genericDocumentRequest;
}
}
@@ -0,0 +1,131 @@
using System.Globalization;
using Newtonsoft.Json;
using Serilog;
using LCU.Library.DocumentType.Enums;
using LCU.Library.DocumentType.Extensions;
using LCU.Library.DocumentType.Models.Requests.Account;
using LCU.Library.DocumentType.Models.Requests.Attributes;
using LCU.Library.Internal.DocumentIngestorModels.Models;
using LCU.Library.Internal.DocumentIngestorModels.Models.IndexFileDocuments;
using LCU.Library.Internal.DocumentIngestorModels.Validations;
using LCU.API.Internal.DocumentIngestor.Services.AttributeDeterminingServices;
using LCU.API.Internal.DocumentIngestor.Services.MicroApiServices;
using LCU.Library.Internal.LCUServiceFactory.Configurations;
namespace LCU.API.Internal.DocumentIngestor.Services.DocumentProcessorServices;
public class XdiDnaNoticeDocumentProcessingService : DocumentProcessingService, IDocumentProcessingService
{
private readonly IAccountOwnerDeterminingService _accountOwnerDeterminingService;
private readonly IFileOperationsApiService _fileOperationsApiService;
private readonly IDocumentTypeUploaderApiService _documentTypeUploaderApiService;
private const DataSourceTypes DataSource = DataSourceTypes.DNA;
private const SourceTypes Source = SourceTypes.XDI;
private const string DateFormat = "MM/dd/yyyy";
public XdiDnaNoticeDocumentProcessingService(IAccountOwnerDeterminingService accountOwnerDeterminingService,
IFileOperationsApiService fileOperationsApiService,
IDocumentTypeUploaderApiService documentTypeUploaderApiService,
ServicesFactoryConfiguration servicesFactoryConfiguration)
: base(servicesFactoryConfiguration)
{
_documentTypeUploaderApiService = documentTypeUploaderApiService;
_fileOperationsApiService = fileOperationsApiService;
_accountOwnerDeterminingService = accountOwnerDeterminingService;
}
public async Task<ProcessResult> ProcessRequest(ProcessingRequest request)
{
var processResult = new ProcessResult();
try
{
Log.Debug("Starting XdiDnaNoticeDocument ProcessRequest.");
ArgumentNullException.ThrowIfNull(request);
var documentValues = JsonConvert.DeserializeObject<XdiDnaNoticesDocument>(request.DocumentValuePayload);
ArgumentNullException.ThrowIfNull(documentValues);
processResult = ValidateRecord(processResult, documentValues);
if (!processResult.Success)
{
return processResult;
}
if (!DateTime.TryParseExact(documentValues.CreationDate,
DateFormat,
CultureInfo.InvariantCulture,
DateTimeStyles.AssumeLocal,
out var creationDate))
{
var message = $"A Creation Date of {documentValues.CreationDate} could not " +
$"be properly formatted from format: {DateFormat}";
return ProcessResult.ProcessDataMappingFailure(processResult,
message);
}
var accountOwnerAttribute = await _accountOwnerDeterminingService.DetermineAccountOwnerAttribute(documentValues.EntityName,
documentValues.AccountNumber,
creationDate,
request.OwnerOverrideInfo);
if (!accountOwnerAttribute.Success)
{
return ProcessResult.ProcessDataMappingFailure(processResult,
accountOwnerAttribute.Message);
}
var accountOwnerRequest = await BuildUploadRequest(request,
documentValues,
creationDate,
accountOwnerAttribute.Payload);
await _documentTypeUploaderApiService.UploadAccountDocument(accountOwnerRequest, request, processResult);
Log.Debug("Ending XdiDnaNoticeDocument ProcessRequest.");
}
catch (Exception exception)
{
var message =
$"An Exception occured while XdiDnaNoticeDocument ProcessRequest. Exception Message: {exception.Message}";
Log.Error(exception, message);
processResult = ProcessResult.ProcessCriticalFailure(processResult, message);
}
return processResult;
}
private static ProcessResult ValidateRecord(ProcessResult processResult, XdiDnaNoticesDocument documentValues)
{
processResult = MetaDataValidation.ValidateAccountNumber(processResult, documentValues.AccountNumber);
processResult = MetaDataValidation.ValidatePdf(processResult, documentValues.PDF);
processResult = MetaDataValidation.ValidateFullName(processResult, documentValues.EntityName);
processResult = MetaDataValidation.ValidateDate(processResult, documentValues.CreationDate, DateFormat);
processResult = MetaDataValidation.ValidateDocumentType<AccountDocumentTypes>(processResult, documentValues.DocumentType);
return processResult;
}
private async Task<AccountDocumentRequest> BuildUploadRequest(ProcessingRequest request,
XdiDnaNoticesDocument documentValues,
DateTime creationDate,
AccountAttribute accountAttribute)
{
var contentBytes = await _fileOperationsApiService.GetContentBytes(request.IngestorRequest.RootPath, request.DocumentFileName);
var type = documentValues.DocumentType.GetValueFromName<AccountDocumentTypes>();
ArgumentNullException.ThrowIfNull(type);
return new AccountDocumentRequest
{
Content = contentBytes,
AccountAttribute = accountAttribute,
AccountDocumentMetadata = new AccountDocumentMetadata
{
Source = Source,
DataSource = DataSource,
Name = documentValues.PDF,
Type = type,
CreatedDate = creationDate,
TraceId = request.IngestorRunInfoId
}
};
}
}
@@ -0,0 +1,132 @@
using LCU.Library.Internal.DocumentIngestorModels.Entities;
using LCU.Library.Internal.DocumentIngestorModels.Enums;
using LCU.Library.Internal.DocumentIngestorModels.Models;
using LCU.Library.Internal.Exceptions;
using LCU.API.Internal.DocumentIngestor.Configurations;
using LCU.API.Internal.DocumentIngestor.Services.QueueServices;
using Newtonsoft.Json;
using Serilog;
namespace LCU.API.Internal.DocumentIngestor.Services;
public class DocumentReprocessingService : IDocumentReprocessingService
{
private readonly IIngestorRunInfoService _ingestorRunInfoService;
private readonly IDocumentProcessingInfoService _documentProcessingInfoService;
private readonly IOwnerOverrideInfoService _ownerOverrideInfoService;
private readonly IQueueDocumentProcessingService _queueDocumentProcessing;
private readonly ApiConfiguration _apiConfiguration;
public DocumentReprocessingService(IIngestorRunInfoService ingestorRunInfoService,
IDocumentProcessingInfoService documentProcessingInfoService,
IOwnerOverrideInfoService ownerOverrideInfoService,
IQueueDocumentProcessingService queueDocumentProcessing,
ApiConfiguration apiConfiguration)
{
_ingestorRunInfoService = ingestorRunInfoService;
_documentProcessingInfoService = documentProcessingInfoService;
_ownerOverrideInfoService = ownerOverrideInfoService;
_queueDocumentProcessing = queueDocumentProcessing;
_apiConfiguration = apiConfiguration;
}
public async Task QueueReprocessDocumentByDocumentProcessingInfoId(int documentProcessingInfoId, string userName)
{
try
{
var ownerOverrideInfos = await _ownerOverrideInfoService.GetByDocumentProcessingInfoIdAsync(documentProcessingInfoId);
var logMessage = $"Reprocess run of DocumentProcessingInfoId: {documentProcessingInfoId} by {userName} on {DateTime.Now}";
await QueueProcessingRequest(documentProcessingInfoId, ownerOverrideInfos?.FirstOrDefault(), logMessage, userName);
}
catch (ArgumentNullException exception)
{
Log.Error(exception, $"Failure to reprocess, DocumentProcessingInfo could not be found. " +
$"DocumentProcessingInfoId: {documentProcessingInfoId}");
throw;
}
catch (Exception exception)
{
Log.Error(exception, $"Failure reprocessing DocumentProcessingInfo. DocumentProcessingInfoId: {documentProcessingInfoId}");
throw new ServiceException($"Failure reprocessing DocumentProcessingInfo. DocumentProcessingInfoId: {documentProcessingInfoId}");
}
}
public async Task QueueOverrideDocumentByDocumentProcessingInfoId(OwnerOverrideInfo ownerOverrideInfo)
{
try
{
ownerOverrideInfo = await _ownerOverrideInfoService.SaveAsync(ownerOverrideInfo);
var logMessage = $"Override triggered by {ownerOverrideInfo.UserName} at {DateTime.Now} with values: {JsonConvert.SerializeObject(ownerOverrideInfo)}";
await QueueProcessingRequest(ownerOverrideInfo.DocumentProcessingInfoId, ownerOverrideInfo, logMessage, ownerOverrideInfo.UserName);
}
catch (ArgumentNullException exception)
{
Log.Error(exception, $"Failure to override, DocumentProcessingInfo could not be found. " +
$"DocumentProcessingInfoId: {ownerOverrideInfo.DocumentProcessingInfoId}");
throw;
}
catch (Exception exception)
{
Log.Error(exception, $"Failure overriding DocumentProcessingInfo. " +
$"DocumentProcessingInfoId: {ownerOverrideInfo.DocumentProcessingInfoId} | " +
$"EntityType: {Enum.GetName(typeof(EntityTypes), ownerOverrideInfo.EntityType)} | " +
$"EntityId: {ownerOverrideInfo.EntityId}");
throw new ServiceException($"Failure overriding DocumentProcessingInfo. DocumentProcessingInfoId: {ownerOverrideInfo.DocumentProcessingInfoId}");
}
}
public async Task QueueReprocessAllCriticalFailures(string userName)
{
try
{
var result = await _documentProcessingInfoService.GetByProcessStatusTypeAsync(ProcessStatusTypes.CriticalFailure);
var documentProcessingInfos = result.Take(_apiConfiguration.CriticalFailureReprocessLimit);
foreach (var documentProcessingInfo in documentProcessingInfos)
{
await QueueReprocessDocumentByDocumentProcessingInfoId(documentProcessingInfo.DocumentProcessingInfoId, userName);
}
}
catch (ArgumentNullException exception)
{
var message = "Failure during reprocess of DocumentProcessingInfos of ProcessStatusType: CriticalFailure."
+ $" ExceptionMessage: {exception.Message}";
Log.Error(exception, message);
throw new ServiceException(message);
}
catch (Exception exception)
{
Log.Error(exception, "Failure to reprocess all DocumentProcessingInfos with ProcessStatusType: CriticalFailure.");
throw new ServiceException("Failure to reprocess all DocumentProcessingInfos with ProcessStatusType: CriticalFailure.");
}
}
private async Task QueueProcessingRequest(int documentProcessingInfoId, OwnerOverrideInfo? ownerOverrideInfo, string logMessage, string userName)
{
var documentProcessingInfo = await _documentProcessingInfoService.GetByIdAsync(documentProcessingInfoId);
ArgumentNullException.ThrowIfNull(documentProcessingInfo);
var ingestorRunInfo = await _ingestorRunInfoService.GetByIdAsync(documentProcessingInfo.IngestorRunInfoId);
ArgumentNullException.ThrowIfNull(ingestorRunInfo);
var processingRequest = new ProcessingRequest()
{
DocumentProcessingInfoId = documentProcessingInfoId,
IngestorRunInfoId = documentProcessingInfo.IngestorRunInfoId,
DocumentFileName = documentProcessingInfo.DocumentFileName,
DocumentValuePayload = documentProcessingInfo.DocumentValuePayload,
IngestorRequest = new IngestorRequest()
{
IngestorType = ingestorRunInfo.IngestorType,
RootPath = ingestorRunInfo.IndexFilePath,
IndexFileName = ingestorRunInfo.IndexFileName
},
OwnerOverrideInfo = ownerOverrideInfo
};
await _documentProcessingInfoService.UpdateStatusAndAddLog(documentProcessingInfoId, ProcessStatusTypes.Reprocessing, logMessage, userName);
await _queueDocumentProcessing.QueueDocumentProcessingRequest(processingRequest);
}
}
@@ -0,0 +1,134 @@
using System.Data.SqlClient;
using Serilog;
using LCU.Library.Internal.DocumentIngestorModels.Entities;
using LCU.Library.Internal.Exceptions;
using LCU.API.Internal.DocumentIngestorLib.Repositories;
namespace LCU.API.Internal.DocumentIngestor.Services;
public class DocumentTypeArchiveDeleteInfoService : IDocumentTypeArchiveDeleteInfoService
{
private readonly IDocumentTypeArchiveDeleteInfoRepository _documentTypeArchiveDeleteInfoRepository;
private readonly IUnitOfWork _unitOfWork;
public DocumentTypeArchiveDeleteInfoService(IUnitOfWork unitOfWork,
IDocumentTypeArchiveDeleteInfoRepository documentTypeArchiveDeleteInfoRepository)
{
_documentTypeArchiveDeleteInfoRepository = documentTypeArchiveDeleteInfoRepository;
_unitOfWork = unitOfWork;
}
public async Task<IEnumerable<DocumentTypeArchiveDeleteInfo>> GetAsync()
{
try
{
var documentProcessingInfos = await _documentTypeArchiveDeleteInfoRepository.FindAsync();
return documentProcessingInfos;
}
catch (Exception exception)
{
Log.Error(exception, $"Failure retrieving DocumentTypeArchiveDeleteInfos.");
throw new ServiceException($"Failure retrieving DocumentTypeArchiveDeleteInfos.");
}
}
public async Task<DocumentTypeArchiveDeleteInfo> GetByDocumentTypeNameAsync<TEnum>(TEnum documentType) where TEnum : Enum
{
try
{
var typesName = Enum.GetName(typeof(TEnum), documentType);
ArgumentNullException.ThrowIfNull(typesName);
var processingLogs = await _documentTypeArchiveDeleteInfoRepository.FindByDocumentTypeNameAsync(typesName);
return processingLogs;
}
catch (Exception exception)
{
Log.Error(exception, $"Failure retrieving DocumentTypeArchiveDeleteInfo with DocumentType {documentType}.");
throw new ServiceException($"Failure retrieving DocumentTypeArchiveDeleteInfo with DocumentType {documentType}.");
}
}
public async Task<DocumentTypeArchiveDeleteInfo> GetByDocumentTypeNameAsync(string documentTypeName)
{
try
{
var processingLogs = await _documentTypeArchiveDeleteInfoRepository.FindByDocumentTypeNameAsync(documentTypeName);
return processingLogs;
}
catch (Exception exception)
{
Log.Error(exception, $"Failure retrieving DocumentTypeArchiveDeleteInfo with DocumentTypeName {documentTypeName}.");
throw new ServiceException($"Failure retrieving DocumentTypeArchiveDeleteInfo with DocumentType {documentTypeName}.");
}
}
public async Task<DocumentTypeArchiveDeleteInfo> SaveAsync(DocumentTypeArchiveDeleteInfo documentTypeArchiveDeleteInfo)
{
try
{
var result = await _documentTypeArchiveDeleteInfoRepository.AddAsync(documentTypeArchiveDeleteInfo);
await _unitOfWork.CompleteAsync();
return result;
}
catch (Exception exception)
{
Log.Error(exception, $"Failure saving DocumentTypeArchiveDeleteInfo.");
throw new ServiceException($"Failure saving DocumentTypeArchiveDeleteInfo. Exception Message: {exception.Message}");
}
}
public async Task<DocumentTypeArchiveDeleteInfo?> Update(DocumentTypeArchiveDeleteInfo documentTypeArchiveDeleteInfo)
{
try
{
var existing = await GetByDocumentTypeNameAsync(documentTypeArchiveDeleteInfo.DocumentTypeName);
if(existing == null)
{
return existing;
}
existing.ArchiveDays = documentTypeArchiveDeleteInfo.ArchiveDays;
existing.ArchiveMonths = documentTypeArchiveDeleteInfo.ArchiveMonths;
existing.DeleteDays = documentTypeArchiveDeleteInfo.DeleteDays;
_documentTypeArchiveDeleteInfoRepository.Update(existing);
_unitOfWork.Complete();
return existing;
}
catch (SqlException sqlException)
{
Log.Error(sqlException, $"SQL Failure updating DocumentTypeArchiveDeleteInfo. " +
$"DocumentTypeName: {documentTypeArchiveDeleteInfo.DocumentTypeName}");
throw new ServiceException($"SQL Failure updating DocumentTypeArchiveDeleteInfo. " +
$"DocumentTypeName: {documentTypeArchiveDeleteInfo.DocumentTypeName}");
}
catch (Exception exception)
{
Log.Error(exception, $"Failure updating DocumentTypeArchiveDeleteInfo. " +
$"DocumentTypeName: {documentTypeArchiveDeleteInfo.DocumentTypeName}");
throw new ServiceException($"Failure updating DocumentTypeArchiveDeleteInfo. " +
$"DocumentTypeName: {documentTypeArchiveDeleteInfo.DocumentTypeName}");
}
}
public async Task Delete(string documentTypeName)
{
try
{
var documentTypeArchiveDeleteInfo = await GetByDocumentTypeNameAsync(documentTypeName);
ArgumentNullException.ThrowIfNull(documentTypeArchiveDeleteInfo);
_documentTypeArchiveDeleteInfoRepository.Remove(documentTypeArchiveDeleteInfo);
_unitOfWork.Complete();
}
catch (SqlException sqlException)
{
Log.Error(sqlException, $"SQL failure removing DocumentTypeArchiveDeleteInfo. DocumentTypeName: {documentTypeName}");
throw new ServiceException($"SQL failure removing DocumentTypeArchiveDeleteInfo. DocumentTypeName: {documentTypeName}");
}
catch (Exception exception)
{
Log.Error(exception, $"Failure removing DocumentTypeArchiveDeleteInfo. DocumentTypeName: {documentTypeName}");
throw new ServiceException($"Failure removing DocumentTypeArchiveDeleteInfo. DocumentTypeName: {documentTypeName}");
}
}
}
@@ -0,0 +1,69 @@
using LCU.Library.Internal.DocumentIngestorModels.Entities;
using LCU.Library.Internal.DocumentIngestorModels.Enums;
using LCU.Library.Internal.DocumentIngestorModels.Models;
using LCU.Library.Internal.DocumentIngestorModels.Models.Pagination;
namespace LCU.API.Internal.DocumentIngestor.Services;
public interface IDocumentProcessingInfoService
{
Task<IEnumerable<DocumentProcessingInfo>> GetAsync();
Task<DocumentProcessingInfo> GetByIdAsync(int id);
Task<IEnumerable<DocumentProcessingInfo>> GetListByIdAsync(List<int> ids);
Task<IEnumerable<DocumentProcessingInfo>> GetByIngestorRunInfoIdAsync(Guid ingestorRunInfoId);
Task<IEnumerable<DocumentProcessingInfo>> GetByProcessStatusTypeAsync(ProcessStatusTypes statusType, bool? isUnknown = null);
Task<IEnumerable<DocumentProcessingInfo>> GetByProcessStatusTypeListAsync(List<ProcessStatusTypes> statusTypes, bool? isUnknown = null);
Task<IEnumerable<DocumentProcessingInfo>> GetCompletedFirstPassByIngestorRunInfoIdAsync(Guid ingestorRunInfoId);
Task<DocumentProcessingInfo> SaveAsync(DocumentProcessingInfo documentProcessingInfo);
Task SaveRangeAsync(DocumentProcessingInfo[] documentProcessingInfos);
Task<DocumentProcessingInfo?> UpdateStatusAndAddLog(int documentProcessingInfoId,
ProcessStatusTypes statusType,
string logMessage,
string userName,
bool? entityIsUnknown = null);
Task<DocumentProcessingInfo?> UpdateStatusAndAddLog(int documentProcessingInfoId,
ProcessStatusTypes statusType,
List<string> logMessages,
string userName,
bool? entityIsUnknown = null);
Task<DocumentProcessingInfo?> UpdateStatusAndAddLog(DocumentProcessingInfo documentProcessingInfo,
ProcessStatusTypes statusType,
string logMessage,
string userName,
bool? entityIsUnknown = null);
Task<DocumentProcessingInfo?> UpdateStatusAndAddLog(DocumentProcessingInfo documentProcessingInfo,
ProcessStatusTypes statusType,
List<string> logMessages,
string userName,
bool? entityIsUnknown = null);
Task<List<CompletionProcessingData>> UpdateProcessCountAndStatusAndAddLog(List<CompletionRequest> requests);
Task<DocumentProcessingInfo?> UpdateProcessCountAndStatusAndAddLog(int documentProcessingInfoId,
ProcessStatusTypes statusType,
string logMessage,
string userName,
bool? entityIsUnknown = null);
Task<DocumentProcessingInfo?> UpdateProcessCountAndStatusAndAddLog(int documentProcessingInfoId,
ProcessStatusTypes statusType,
List<string> logMessages,
string userName,
bool? entityIsUnknown = null);
Task<DocumentProcessingInfo?> UpdateProcessCountAndStatusAndAddLog(DocumentProcessingInfo documentProcessingInfo,
ProcessStatusTypes statusType,
string logMessage,
string userName,
bool? entityIsUnknown = null);
Task<DocumentProcessingInfo?> UpdateProcessCountAndStatusAndAddLog(DocumentProcessingInfo documentProcessingInfo,
ProcessStatusTypes statusType,
List<string> logMessages,
string userName,
bool? entityIsUnknown = null);
Task<DocumentProcessingInfo?> Update(DocumentProcessingInfo documentProcessingInfo);
Task<IEnumerable<DocumentProcessingInfo>> UpdateRangeAsync(IEnumerable<DocumentProcessingInfo> documentProcessingInfos);
Task Delete(int id);
Task<PagedResult<DocumentProcessingInfo>> GetFilteredDocumentProcessingInfo(Filter filter);
Task<int> CountCompletedFirstPassByIngestorRunInfoIdAsync(Guid ingestorRunInfoId);
Task<int> CountCompletedByIngestorRunInfoIdAsync(Guid ingestorRunInfoId);
Task<IEnumerable<DocumentProcessingInfo>?> SetDocumentsToEntityUnknown(EntityUnknownRequest request);
Task<DocumentProcessingInfo?> UpdateDocumentProcessingInfoStatus(DocumentProcessingInfoStatusRequest request);
Task<IDictionary<Guid, DocumentProcessingInfo?>> GetByDocumentIdsAsync(Guid[] ids);
}
@@ -0,0 +1,10 @@
using LCU.Library.Internal.DocumentIngestorModels.Entities;
namespace LCU.API.Internal.DocumentIngestor.Services;
public interface IDocumentReprocessingService
{
Task QueueReprocessDocumentByDocumentProcessingInfoId(int documentProcessingInfoId, string userName);
Task QueueOverrideDocumentByDocumentProcessingInfoId(OwnerOverrideInfo ownerOverrideRequest);
Task QueueReprocessAllCriticalFailures(string userName);
}
@@ -0,0 +1,12 @@
using LCU.Library.Internal.DocumentIngestorModels.Entities;
namespace LCU.API.Internal.DocumentIngestor.Services;
public interface IDocumentTypeArchiveDeleteInfoService
{
Task<IEnumerable<DocumentTypeArchiveDeleteInfo>> GetAsync();
Task<DocumentTypeArchiveDeleteInfo> GetByDocumentTypeNameAsync<TEnum>(TEnum documentType) where TEnum : Enum;
Task<DocumentTypeArchiveDeleteInfo> GetByDocumentTypeNameAsync(string documentType);
Task<DocumentTypeArchiveDeleteInfo> SaveAsync(DocumentTypeArchiveDeleteInfo documentTypeArchiveDeleteInfo);
Task<DocumentTypeArchiveDeleteInfo?> Update(DocumentTypeArchiveDeleteInfo documentTypeArchiveDeleteInfo);
Task Delete(string documentTypeName);
}
@@ -0,0 +1,12 @@
using LCU.Library.Internal.DocumentIngestorModels.Entities;
namespace LCU.API.Internal.DocumentIngestor.Services;
public interface IIngestorRunInfoService
{
Task<IngestorRunInfo?> GetByIdAsync(Guid id);
Task<IEnumerable<IngestorRunInfo>> GetListByIdAsync(List<Guid> ingestorRunInfoIds);
Task<IngestorRunInfo> SaveAsync(IngestorRunInfo ingestorRunInfo);
Task<IngestorRunInfo> Update(IngestorRunInfo ingestorRunInfo);
Task Delete(Guid ingestorRunInfoId);
IEnumerable<IngestorRunInfo> FindByIngestionEndDateNotArchived(DateTime ingestionEndDate);
}
@@ -0,0 +1,22 @@
using LCU.Library.Internal.DocumentIngestorModels.Entities;
using LCU.Library.Internal.DocumentIngestorModels.Enums;
using LCU.Library.Internal.DocumentIngestorModels.Models;
namespace LCU.API.Internal.DocumentIngestor.Services;
public interface IMemberNotificationInfoService
{
Task<IEnumerable<MemberNotificationInfo>> GetAsync();
Task<MemberNotificationInfo> GetByIdAsync(int id);
Task<IEnumerable<MemberNotificationInfo>> GetListByIdAsync(List<int> ids);
Task<IEnumerable<MemberNotificationInfo>> GetByDocumentProcessingInfoIdAsync(int documentProcessingInfoId);
Task<IEnumerable<MemberNotificationInfo>> GetByNotificationStatusAsync(NotificationStatusTypes status);
Task<IEnumerable<MemberNotificationInfo>> GetSendableMemberNotificationInfosAsync(DateTime minimumUploadDate, int maxAttempts);
Task<MemberNotificationInfo> SaveAsync(MemberNotificationInfo memberNotificationInfo);
Task SaveListAsync(List<MemberNotificationInfo> memberNotificationInfos);
Task<MemberNotificationInfo?> Update(MemberNotificationInfo memberNotificationInfo);
Task<IEnumerable<MemberNotificationInfo?>> UpdateRange(IEnumerable<MemberNotificationInfo> memberNotificationInfos);
Task Delete(int id);
Task<bool> SetDocumentNotificationStatus(int memberNotificationInfoId, NotificationStatusTypes notificationStatus, string userName, string reason);
Task<bool> BulkSetDocumentNotificationStatus(BulkNotificationStatusRequest request, string userName);
}
@@ -0,0 +1,17 @@
using LCU.Library.Internal.DocumentIngestorModels.Entities;
namespace LCU.API.Internal.DocumentIngestor.Services;
public interface IOsgPreprocessorInfoService
{
Task<IEnumerable<OsgPreprocessorInfo>> GetAsync();
Task<OsgPreprocessorInfo> GetByIdAsync(int id);
Task<IEnumerable<OsgPreprocessorInfo>> GetListByIdAsync(List<int> ids);
Task<IEnumerable<OsgPreprocessorInfo>> GetByDocumentTypeAsync(string documentType);
Task<IEnumerable<OsgPreprocessorInfo>> GetByMemberNumberAsync(string memberNumber);
Task<OsgPreprocessorInfo> SaveAsync(OsgPreprocessorInfo osgPreprocessorInfo);
Task SaveListAsync(List<OsgPreprocessorInfo> osgPreprocessorInfos);
Task<OsgPreprocessorInfo?> Update(OsgPreprocessorInfo osgPreprocessorInfo);
Task<IEnumerable<OsgPreprocessorInfo?>> UpdateRange(IEnumerable<OsgPreprocessorInfo> osgPreprocessorInfos);
Task Delete(int id);
}
@@ -0,0 +1,12 @@
using LCU.Library.Internal.DocumentIngestorModels.Entities;
namespace LCU.API.Internal.DocumentIngestor.Services;
public interface IOwnerOverrideInfoService
{
Task<IEnumerable<OwnerOverrideInfo>?> GetAsync();
Task<OwnerOverrideInfo?> GetByIdAsync(int id);
Task<IEnumerable<OwnerOverrideInfo>?> GetByDocumentProcessingInfoIdAsync(int documentProcessingInfoId);
Task<OwnerOverrideInfo> SaveAsync(OwnerOverrideInfo ownerOverrideInfo);
Task Delete(int id);
}
@@ -0,0 +1,12 @@
using LCU.Library.Internal.DocumentIngestorModels.Entities;
namespace LCU.API.Internal.DocumentIngestor.Services;
public interface IPdfPartitionerLogService
{
Task<IEnumerable<PdfPartitionerLog>> GetAsync();
Task<PdfPartitionerLog> GetByIdAsync(int id);
Task<IEnumerable<PdfPartitionerLog>> GetByPdfPartitionerRunInfoIdAsync(Guid pdfPartitionerRunInfoId);
Task SaveListAsync(Guid pdfPartitionerRunInfoId, List<string> messageLogs);
Task<PdfPartitionerLog> SaveAsync(PdfPartitionerLog pdfPartitionerLog);
Task Delete(int id);
}
@@ -0,0 +1,11 @@
using LCU.Library.Internal.DocumentIngestorModels.Entities;
namespace LCU.API.Internal.DocumentIngestor.Services;
public interface IPdfPartitionerRunInfoService
{
Task<PdfPartitionerRunInfo> GetByIdAsync(Guid id);
Task<PdfPartitionerRunInfo> SaveAsync(PdfPartitionerRunInfo pdfPartitionerRunInfo);
Task<PdfPartitionerRunInfo> IncreaseCountsAndUpdate(Guid pdfPartitionerRunInfoId, bool successful);
Task<PdfPartitionerRunInfo> Update(PdfPartitionerRunInfo pdfPartitionerRunInfo);
Task Delete(Guid pdfPartitionerRunInfoId);
}
@@ -0,0 +1,13 @@
using LCU.Library.Internal.DocumentIngestorModels.Entities;
namespace LCU.API.Internal.DocumentIngestor.Services;
public interface IProcessingLogService
{
Task<IEnumerable<ProcessingLog>> GetAsync();
Task<ProcessingLog> GetByIdAsync(int id);
Task<IEnumerable<ProcessingLog>> GetByDocumentProcessingInfoIdAsync(int documentProcessingInfoId);
Task<ProcessingLog> SaveAsync(ProcessingLog processingLog);
Task SaveRangeAsync(IEnumerable<ProcessingLog> processingLogs);
Task<ProcessingLog?> Update(ProcessingLog processingLog);
Task Delete(int id);
}
@@ -0,0 +1,7 @@
namespace LCU.API.Internal.DocumentIngestor.Services;
public interface IUploadCompletionReprocessingService
{
Task QueueReprocessUploadCompletionByDocumentProcessingInfoId(int documentProcessingInfoId, string userName);
Task QueueReprocessAllNotificationDataFailures(string userName);
}
@@ -0,0 +1,83 @@
using System.Globalization;
using Newtonsoft.Json;
using Serilog;
using CsvHelper;
using CsvHelper.Configuration;
using LCU.API.Internal.DocumentIngestor.Services.MicroApiServices;
using LCU.Library.Internal.DocumentIngestorModels.Entities;
using LCU.Library.Internal.DocumentIngestorModels.Models;
using LCU.Library.Internal.DocumentIngestorModels.Models.IndexFileDocuments;
using LCU.Library.Internal.LCUServiceFactory.Configurations;
namespace LCU.API.Internal.DocumentIngestor.Services.IndexParserServices;
public class FosDnaDocumentIndexService : IndexParserService, IIndexParserService
{
private readonly IFileOperationsApiService _fileOperationsApiService;
public FosDnaDocumentIndexService(IFileOperationsApiService fileOperationsApiService,
ServicesFactoryConfiguration servicesFactoryConfiguration)
: base(servicesFactoryConfiguration)
{
_fileOperationsApiService = fileOperationsApiService;
}
public async Task<IngestorResult> HandleIndexRequest(IngestorRequest request)
{
Log.Debug("Starting HandleIndexRequest for FosDnaDocument.");
var result = new IngestorResult();
try
{
ArgumentNullException.ThrowIfNull(request);
var records = await GetFileContentAndParseRecords<FosDnaDocument>(request.FinalIndexFilePath);
result = PrepareRunInfoAndProcessRequests(request, records, result);
}
catch (Exception exception)
{
Log.Error(exception, "An Exception occured while HandleDnaDocumentRequest.");
result = new IngestorResult(true, exception.Message);
}
Log.Debug("Ending HandleIndexRequest for FosDnaDocument.");
return result;
}
public async Task<List<T>> GetFileContentAndParseRecords<T>(string finalIndexFilePath)
{
var indexFileContent = await _fileOperationsApiService.GetFileContent(finalIndexFilePath);
using var stringReader = new StringReader(indexFileContent);
var config = new CsvConfiguration(CultureInfo.InvariantCulture) { Delimiter = ",", HasHeaderRecord = false };
using var csv = new CsvReader(stringReader, config);
var csvRecords = csv.GetRecords<T>();
return csvRecords.ToList();
}
public IngestorResult PrepareRunInfoAndProcessRequests(IngestorRequest request,
List<FosDnaDocument> documentMetaData,
IngestorResult result)
{
result.IngestorRunInfo = new IngestorRunInfo(Guid.NewGuid(),
DateTime.Now,
request.IngestorType,
request.RootPath,
request.IndexFileName,
documentMetaData.Count);
result.ProcessingRequests = new List<ProcessingRequest>();
foreach (var record in documentMetaData){
var processingRequest = new ProcessingRequest
{
IngestorRunInfoId = result.IngestorRunInfo.IngestorRunInfoId,
IngestorRequest = request,
DocumentFileName = record.DocName,
DocumentValuePayload = JsonConvert.SerializeObject(record),
DocumentTypeDisplayName = record.DocType
};
result.ProcessingRequests.Add(processingRequest);
}
return result;
}
}
@@ -0,0 +1,98 @@
using System.Text.RegularExpressions;
using System.Globalization;
using Newtonsoft.Json;
using Serilog;
using CsvHelper;
using CsvHelper.Configuration;
using LCU.API.Internal.DocumentIngestor.Services.MicroApiServices;
using LCU.Library.Internal.DocumentIngestorModels.Entities;
using LCU.Library.Internal.DocumentIngestorModels.Models;
using LCU.Library.Internal.DocumentIngestorModels.Models.IndexFileDocuments;
using LCU.Library.Internal.LCUServiceFactory.Configurations;
namespace LCU.API.Internal.DocumentIngestor.Services.IndexParserServices;
public class FosLoanStatementIndexService : IndexParserService, IPdfIndexParserService
{
private readonly IFileOperationsApiService _fileOperationsApiService;
public FosLoanStatementIndexService(IFileOperationsApiService fileOperationsApiService,
ServicesFactoryConfiguration servicesFactoryConfiguration)
: base(servicesFactoryConfiguration)
{
_fileOperationsApiService = fileOperationsApiService;
}
public async Task<IngestorResult> HandleIndexRequest(IngestorRequest request)
{
Log.Debug("Starting HandleIndexRequest for FosLoanStatement.");
var result = new IngestorResult();
try
{
ArgumentNullException.ThrowIfNull(request);
var records = await GetFileContentAndParseRecords<FosLoanStatement>(request.FinalIndexFilePath);
result = PrepareRunInfoAndProcessRequests(request, records, result);
}
catch (Exception exception)
{
Log.Error(exception, "An Exception occured while HandleLoanStatementRequest.");
result = new IngestorResult(true, exception.Message);
}
Log.Debug("Ending HandleIndexRequest for FosLoanStatement.");
return result;
}
public async Task<List<T>> GetFileContentAndParseRecords<T>(string finalIndexFilePath)
{
var indexFileContent = await _fileOperationsApiService.GetFileContent(finalIndexFilePath);
using var stringReader = new StringReader(indexFileContent);
var config = new CsvConfiguration(CultureInfo.InvariantCulture) { Delimiter = "|", HasHeaderRecord = false };
using var csv = new CsvReader(stringReader, config);
var csvRecords = csv.GetRecords<T>();
return csvRecords.ToList();
}
public IngestorResult PrepareRunInfoAndProcessRequests(IngestorRequest request,
List<FosLoanStatement> documentMetaData,
IngestorResult result)
{
result.IngestorRunInfo = new IngestorRunInfo(Guid.NewGuid(),
DateTime.Now,
request.IngestorType,
request.RootPath,
request.IndexFileName,
documentMetaData.Count());
result.ProcessingRequests = new List<ProcessingRequest>();
foreach (var record in documentMetaData)
{
var parsedPdfName = CreateParsedPdfName(record);
var processingRequest = new ProcessingRequest()
{
IngestorRunInfoId = result.IngestorRunInfo.IngestorRunInfoId,
IngestorRequest = request,
DocumentFileName = parsedPdfName,
DocumentValuePayload = JsonConvert.SerializeObject(record),
DocumentTypeDisplayName = record.DocType
};
result.ProcessingRequests.Add(processingRequest);
}
return result;
}
public string CreateParsedPdfName<T>(T record)
{
if (record is not FosLoanStatement fosLoanStatement)
{
throw new Exception("Record is not a FosLoanStatement");
}
var documentWithoutExtensions = Path.GetFileNameWithoutExtension(fosLoanStatement.DocName);
var entityName = Regex.Replace(fosLoanStatement.EntityName, "[^A-Za-z0-9 -]", "");
var parsedPdfName = $"{documentWithoutExtensions}_{entityName}_{fosLoanStatement.DocType}_{fosLoanStatement.StartingPage}.pdf";
return parsedPdfName;
}
}
@@ -0,0 +1,82 @@
using CsvHelper;
using CsvHelper.Configuration;
using LCU.API.Internal.DocumentIngestor.Services.MicroApiServices;
using LCU.Library.Internal.DocumentIngestorModels.Entities;
using LCU.Library.Internal.DocumentIngestorModels.Models;
using LCU.Library.Internal.DocumentIngestorModels.Models.IndexFileDocuments;
using Newtonsoft.Json;
using Serilog;
using System.Globalization;
using LCU.Library.Internal.LCUServiceFactory.Configurations;
namespace LCU.API.Internal.DocumentIngestor.Services.IndexParserServices;
public class FosTaxStatementIndexService : IndexParserService, IIndexParserService
{
private readonly IFileOperationsApiService _fileOperationsApiService;
public FosTaxStatementIndexService(IFileOperationsApiService fileOperationsApiService,
ServicesFactoryConfiguration servicesFactoryConfiguration)
: base(servicesFactoryConfiguration)
{
_fileOperationsApiService = fileOperationsApiService;
}
public async Task<IngestorResult> HandleIndexRequest(IngestorRequest request)
{
Log.Debug("Starting HandleIndexRequest for FosTaxStatement.");
var result = new IngestorResult();
try
{
ArgumentNullException.ThrowIfNull(request);
var records = await GetFileContentAndParseRecords<FosTaxStatementDocument>(request.FinalIndexFilePath);
result = PrepareRunInfoAndProcessRequests(request, records, result);
}
catch (Exception exception)
{
Log.Error(exception, "An Exception occured while HandleTaxStatementRequest.");
result = new IngestorResult(true, exception.Message);
}
Log.Debug("Ending HandleIndexRequest for FosTaxStatement.");
return result;
}
public async Task<List<T>> GetFileContentAndParseRecords<T>(string finalIndexFilePath)
{
var indexFileContent = await _fileOperationsApiService.GetFileContent(finalIndexFilePath);
using var stringReader = new StringReader(indexFileContent);
var config = new CsvConfiguration(CultureInfo.InvariantCulture) { Delimiter = ",", HasHeaderRecord = false };
using var csv = new CsvReader(stringReader, config);
var csvRecords = csv.GetRecords<T>();
return csvRecords.ToList();
}
public IngestorResult PrepareRunInfoAndProcessRequests(IngestorRequest request,
List<FosTaxStatementDocument> documentMetaData,
IngestorResult result)
{
result.IngestorRunInfo = new IngestorRunInfo(Guid.NewGuid(),
DateTime.Now,
request.IngestorType,
request.RootPath,
request.IndexFileName,
documentMetaData.Count());
result.ProcessingRequests = new List<ProcessingRequest>();
foreach (var record in documentMetaData)
{
var processingRequest = new ProcessingRequest()
{
IngestorRunInfoId = result.IngestorRunInfo.IngestorRunInfoId,
IngestorRequest = request,
DocumentFileName = record.PDF,
DocumentValuePayload = JsonConvert.SerializeObject(record),
DocumentTypeDisplayName = record.FileName
};
result.ProcessingRequests.Add(processingRequest);
}
return result;
}
}
@@ -0,0 +1,10 @@
using LCU.Library.Internal.DocumentIngestorModels.Models;
using LCU.Library.Internal.LCUServiceFactory.Services;
namespace LCU.API.Internal.DocumentIngestor.Services.IndexParserServices;
public interface IIndexParserService : ILcuService
{
Task<IngestorResult> HandleIndexRequest(IngestorRequest request);
Task<List<TIngestionType>> GetFileContentAndParseRecords<TIngestionType>(string finalIndexFilePath);
}
@@ -0,0 +1,9 @@
using LCU.Library.Internal.DocumentIngestorModels.Models;
using LCU.Library.Internal.LCUServiceFactory.Services;
namespace LCU.API.Internal.DocumentIngestor.Services.IndexParserServices;
public interface IPdfIndexParserService : IIndexParserService
{
public string CreateParsedPdfName<T>(T record);
}
@@ -0,0 +1,11 @@
using LCU.Library.Internal.LCUServiceFactory.Configurations;
using LCU.Library.Internal.LCUServiceFactory.Services;
namespace LCU.API.Internal.DocumentIngestor.Services.IndexParserServices;
public class IndexParserService : LcuAbstractService
{
protected IndexParserService(ServicesFactoryConfiguration servicesFactoryConfiguration)
: base(servicesFactoryConfiguration.GetServiceFactoryRegistrationByInterface<IIndexParserService>())
{ }
}
@@ -0,0 +1,82 @@
using CsvHelper;
using CsvHelper.Configuration;
using LCU.API.Internal.DocumentIngestor.Services.MicroApiServices;
using LCU.Library.Internal.DocumentIngestorModels.Entities;
using LCU.Library.Internal.DocumentIngestorModels.Models;
using LCU.Library.Internal.DocumentIngestorModels.Models.IndexFileDocuments;
using Newtonsoft.Json;
using Serilog;
using System.Globalization;
using LCU.Library.Internal.LCUServiceFactory.Configurations;
namespace LCU.API.Internal.DocumentIngestor.Services.IndexParserServices;
public class OsgDnaNoticeSplitIndexService : IndexParserService, IIndexParserService
{
private readonly IFileOperationsApiService _fileOperationsApiService;
public OsgDnaNoticeSplitIndexService(IFileOperationsApiService fileOperationsApiService,
ServicesFactoryConfiguration servicesFactoryConfiguration)
: base(servicesFactoryConfiguration)
{
_fileOperationsApiService = fileOperationsApiService;
}
public async Task<IngestorResult> HandleIndexRequest(IngestorRequest request)
{
Log.Debug("Starting HandleIndexRequest for OsgDnaNoticeSplitIndexService.");
var result = new IngestorResult();
try
{
ArgumentNullException.ThrowIfNull(request);
var records = await GetFileContentAndParseRecords<OsgDnaNoticeSplit>(request.FinalIndexFilePath);
result = PrepareRunInfoAndProcessRequests(request, records, result);
}
catch (Exception exception)
{
Log.Error(exception, "An Exception occured while HandleIndexRequest.");
result = new IngestorResult(true, exception.Message);
}
Log.Debug("Ending HandleIndexRequest for OsgDnaNoticeSplitIndexService.");
return result;
}
public async Task<List<T>> GetFileContentAndParseRecords<T>(string finalIndexFilePath)
{
var indexFileContent = await _fileOperationsApiService.GetFileContent(finalIndexFilePath);
using var stringReader = new StringReader(indexFileContent);
var config = new CsvConfiguration(CultureInfo.InvariantCulture) { Delimiter = "|", HasHeaderRecord = false };
using var csv = new CsvReader(stringReader, config);
var csvRecords = csv.GetRecords<T>();
return csvRecords.ToList();
}
public IngestorResult PrepareRunInfoAndProcessRequests(IngestorRequest request,
List<OsgDnaNoticeSplit> documentMetaData,
IngestorResult result)
{
result.IngestorRunInfo = new IngestorRunInfo(Guid.NewGuid(),
DateTime.Now,
request.IngestorType,
request.RootPath,
request.IndexFileName,
documentMetaData.Count());
result.ProcessingRequests = new List<ProcessingRequest>();
foreach (var record in documentMetaData)
{
var processingRequest = new ProcessingRequest()
{
IngestorRunInfoId = result.IngestorRunInfo.IngestorRunInfoId,
IngestorRequest = request,
DocumentFileName = record.PdfName,
DocumentValuePayload = JsonConvert.SerializeObject(record),
DocumentTypeDisplayName = record.NoticeId
};
result.ProcessingRequests.Add(processingRequest);
}
return result;
}
}
@@ -0,0 +1,94 @@
using CsvHelper;
using CsvHelper.Configuration;
using LCU.API.Internal.DocumentIngestor.Services.MicroApiServices;
using LCU.Library.Internal.DocumentIngestorModels.Entities;
using LCU.Library.Internal.DocumentIngestorModels.Models;
using LCU.Library.Internal.DocumentIngestorModels.Models.IndexFileDocuments;
using Newtonsoft.Json;
using Serilog;
using System.Globalization;
using System.Text.RegularExpressions;
using LCU.Library.Internal.LCUServiceFactory.Configurations;
namespace LCU.API.Internal.DocumentIngestor.Services.IndexParserServices
{
public class OsgDnaNoticesIndexService : IndexParserService, IPdfIndexParserService
{
private readonly IFileOperationsApiService _fileOperationsApiService;
public OsgDnaNoticesIndexService(IFileOperationsApiService fileOperationsApiService,
ServicesFactoryConfiguration servicesFactoryConfiguration)
: base(servicesFactoryConfiguration)
{
_fileOperationsApiService = fileOperationsApiService;
}
public async Task<List<T>> GetFileContentAndParseRecords<T>(string finalIndexFilePath)
{
var indexFileContent = await _fileOperationsApiService.GetFileContent(finalIndexFilePath);
using var stringReader = new StringReader(indexFileContent);
var config = new CsvConfiguration(CultureInfo.InvariantCulture) { Delimiter = "|", HasHeaderRecord = false };
using var csv = new CsvReader(stringReader, config);
return csv.GetRecords<T>().ToList();
}
public async Task<IngestorResult> HandleIndexRequest(IngestorRequest request)
{
Log.Debug("Starting HandleIndexRequest for OsgDnaNotices.");
var result = new IngestorResult();
try
{
ArgumentNullException.ThrowIfNull(request);
var records = await GetFileContentAndParseRecords<OsgDnaNoticesDocument>(request.FinalIndexFilePath);
result = PrepareRunInfoAndProcessRequests(request, records, result);
}
catch (Exception exception)
{
Log.Error(exception, "An Exception occured while HandleIndexRequest.");
result = new IngestorResult(true, exception.Message);
}
Log.Debug("Ending HandleIndexRequest for OsgDnaNotices.");
return result;
}
public IngestorResult PrepareRunInfoAndProcessRequests(IngestorRequest request, List<OsgDnaNoticesDocument> records, IngestorResult result)
{
result.IngestorRunInfo = new IngestorRunInfo(Guid.NewGuid(),
DateTime.Now,
request.IngestorType,
request.RootPath,
request.IndexFileName,
records.Count());
result.ProcessingRequests = new List<ProcessingRequest>();
foreach (var record in records)
{
var parsedPdfName = CreateParsedPdfName(record);
var processingRequest = new ProcessingRequest()
{
IngestorRunInfoId = result.IngestorRunInfo.IngestorRunInfoId,
IngestorRequest = request,
DocumentFileName = parsedPdfName,
DocumentValuePayload = JsonConvert.SerializeObject(record),
DocumentTypeDisplayName = record.DocType
};
result.ProcessingRequests.Add(processingRequest);
}
return result;
}
public string CreateParsedPdfName<T>(T record)
{
if (record is not OsgDnaNoticesDocument osgDnaNoticesDocument)
{
throw new Exception("Record is not a OsgDnaNoticesDocument");
}
var entityId = Regex.Replace(osgDnaNoticesDocument.MemberNumberOrUserId, "[^A-Za-z0-9 -]", "");
var accountNumber = string.IsNullOrWhiteSpace(osgDnaNoticesDocument.AccountNumber) ? "0" : osgDnaNoticesDocument.AccountNumber;
var parsedPdfName = $"{accountNumber}_{entityId}_{osgDnaNoticesDocument.DocType}_{osgDnaNoticesDocument.StartingPage}.pdf";
return parsedPdfName;
}
}
}
@@ -0,0 +1,94 @@
using CsvHelper;
using CsvHelper.Configuration;
using LCU.API.Internal.DocumentIngestor.Services.MicroApiServices;
using LCU.Library.Internal.DocumentIngestorModels.Entities;
using LCU.Library.Internal.DocumentIngestorModels.Models;
using LCU.Library.Internal.DocumentIngestorModels.Models.IndexFileDocuments;
using Newtonsoft.Json;
using Serilog;
using System.Globalization;
using System.Text.RegularExpressions;
using LCU.Library.Internal.LCUServiceFactory.Configurations;
namespace LCU.API.Internal.DocumentIngestor.Services.IndexParserServices
{
public class OsgHistoricalDocumentIndexService : IndexParserService, IPdfIndexParserService
{
private readonly IFileOperationsApiService _fileOperationsApiService;
public OsgHistoricalDocumentIndexService(IFileOperationsApiService fileOperationsApiService,
ServicesFactoryConfiguration servicesFactoryConfiguration)
: base(servicesFactoryConfiguration)
{
_fileOperationsApiService = fileOperationsApiService;
}
public async Task<List<T>> GetFileContentAndParseRecords<T>(string finalIndexFilePath)
{
var indexFileContent = await _fileOperationsApiService.GetFileContent(finalIndexFilePath);
using var stringReader = new StringReader(indexFileContent);
var config = new CsvConfiguration(CultureInfo.InvariantCulture) { Delimiter = "|", HasHeaderRecord = false };
using var csv = new CsvReader(stringReader, config);
return csv.GetRecords<T>().ToList();
}
public async Task<IngestorResult> HandleIndexRequest(IngestorRequest request)
{
Log.Debug("Starting HandleIndexRequest for OsgHistoricalDocument.");
var result = new IngestorResult();
try
{
ArgumentNullException.ThrowIfNull(request);
var records = await GetFileContentAndParseRecords<OsgHistoricalDocument>(request.FinalIndexFilePath);
result = PrepareRunInfoAndProcessRequests(request, records, result);
}
catch (Exception exception)
{
Log.Error(exception, "An Exception occured while HandleIndexRequest.");
result = new IngestorResult(true, exception.Message);
}
Log.Debug("Ending HandleIndexRequest for OsgHistoricalDocument.");
return result;
}
public IngestorResult PrepareRunInfoAndProcessRequests(IngestorRequest request, List<OsgHistoricalDocument> records, IngestorResult result)
{
result.IngestorRunInfo = new IngestorRunInfo(Guid.NewGuid(),
DateTime.Now,
request.IngestorType,
request.RootPath,
request.IndexFileName,
records.Count());
result.ProcessingRequests = new List<ProcessingRequest>();
foreach (var record in records)
{
var parsedPdfName = CreateParsedPdfName(record);
var processingRequest = new ProcessingRequest()
{
IngestorRunInfoId = result.IngestorRunInfo.IngestorRunInfoId,
IngestorRequest = request,
DocumentFileName = parsedPdfName,
DocumentValuePayload = JsonConvert.SerializeObject(record),
DocumentTypeDisplayName = record.DocType
};
result.ProcessingRequests.Add(processingRequest);
}
return result;
}
public string CreateParsedPdfName<T>(T record)
{
if (record is not OsgHistoricalDocument osgHistoricalDocument)
{
throw new Exception("Record is not a OsgHistoricalDocument");
}
var entityId = Regex.Replace(osgHistoricalDocument.MemberNumberOrUserId, "[^A-Za-z0-9 -]", "");
var entityName = Regex.Replace(osgHistoricalDocument.EntityName, "[^A-Za-z0-9 -]", "");
var parsedPdfName = $"{entityName}_{entityId}_{osgHistoricalDocument.DocType}_{osgHistoricalDocument.StartingPage}.pdf";
return parsedPdfName;
}
}
}
@@ -0,0 +1,82 @@
using CsvHelper;
using CsvHelper.Configuration;
using LCU.API.Internal.DocumentIngestor.Services.MicroApiServices;
using LCU.Library.Internal.DocumentIngestorModels.Entities;
using LCU.Library.Internal.DocumentIngestorModels.Models;
using LCU.Library.Internal.DocumentIngestorModels.Models.IndexFileDocuments;
using Newtonsoft.Json;
using Serilog;
using System.Globalization;
using LCU.Library.Internal.LCUServiceFactory.Configurations;
namespace LCU.API.Internal.DocumentIngestor.Services.IndexParserServices;
public class OsgTaxStatementIndexService : IndexParserService, IIndexParserService
{
private readonly IFileOperationsApiService _fileOperationsApiService;
public OsgTaxStatementIndexService(IFileOperationsApiService fileOperationsApiService,
ServicesFactoryConfiguration servicesFactoryConfiguration)
: base(servicesFactoryConfiguration)
{
_fileOperationsApiService = fileOperationsApiService;
}
public async Task<IngestorResult> HandleIndexRequest(IngestorRequest request)
{
Log.Debug("Starting HandleIndexRequest for OsgTaxStatement.");
var result = new IngestorResult();
try
{
ArgumentNullException.ThrowIfNull(request);
var records = await GetFileContentAndParseRecords<OsgTaxStatement>(request.FinalIndexFilePath);
result = PrepareRunInfoAndProcessRequests(request, records, result);
}
catch (Exception exception)
{
Log.Error(exception, "An Exception occured while HandleTaxStatementRequest.");
result = new IngestorResult(true, exception.Message);
}
Log.Debug("Ending HandleIndexRequest for OsgTaxStatement.");
return result;
}
public async Task<List<T>> GetFileContentAndParseRecords<T>(string finalIndexFilePath)
{
var indexFileContent = await _fileOperationsApiService.GetFileContent(finalIndexFilePath);
using var stringReader = new StringReader(indexFileContent);
var config = new CsvConfiguration(CultureInfo.InvariantCulture) { Delimiter = "|", HasHeaderRecord = false };
using var csv = new CsvReader(stringReader, config);
var csvRecords = csv.GetRecords<T>();
return csvRecords.ToList();
}
public IngestorResult PrepareRunInfoAndProcessRequests(IngestorRequest request,
List<OsgTaxStatement> documentMetaData,
IngestorResult result)
{
result.IngestorRunInfo = new IngestorRunInfo(Guid.NewGuid(),
DateTime.Now,
request.IngestorType,
request.RootPath,
request.IndexFileName,
documentMetaData.Count());
result.ProcessingRequests = new List<ProcessingRequest>();
foreach (var record in documentMetaData)
{
var processingRequest = new ProcessingRequest()
{
IngestorRunInfoId = result.IngestorRunInfo.IngestorRunInfoId,
IngestorRequest = request,
DocumentFileName = record.PdfName,
DocumentValuePayload = JsonConvert.SerializeObject(record),
DocumentTypeDisplayName = record.NoticeName
};
result.ProcessingRequests.Add(processingRequest);
}
return result;
}
}
@@ -0,0 +1,80 @@
using System.Globalization;
using Newtonsoft.Json;
using Serilog;
using CsvHelper.Configuration;
using CsvHelper;
using LCU.API.Internal.DocumentIngestor.Services.MicroApiServices;
using LCU.Library.Internal.DocumentIngestorModels.Entities;
using LCU.Library.Internal.DocumentIngestorModels.Models;
using LCU.Library.Internal.DocumentIngestorModels.Models.IndexFileDocuments;
using LCU.Library.Internal.LCUServiceFactory.Configurations;
namespace LCU.API.Internal.DocumentIngestor.Services.IndexParserServices;
public class OsgTemenosNoticeDocumentIndexService : IndexParserService, IIndexParserService
{
private readonly IFileOperationsApiService _fileOperationsApiService;
public OsgTemenosNoticeDocumentIndexService(IFileOperationsApiService fileOperationsApiService,
ServicesFactoryConfiguration servicesFactoryConfiguration)
: base(servicesFactoryConfiguration)
{
_fileOperationsApiService = fileOperationsApiService;
}
public async Task<IngestorResult> HandleIndexRequest(IngestorRequest request)
{
Log.Debug("Starting HandleIndexRequest for OsgTemenosNoticeRequest.");
var result = new IngestorResult();
try
{
ArgumentNullException.ThrowIfNull(request);
var records = await GetFileContentAndParseRecords<OsgTemenosNoticesDocument>(request.FinalIndexFilePath);
result = PrepareRunInfoAndProcessRequests(request, records, result);
}
catch (Exception exception)
{
Log.Error(exception, "An Exception occured while handling OsgTemenosNoticeRequest.");
result = new IngestorResult(true, exception.Message);
}
Log.Debug("Ending HandleIndexRequest for OsgTemenosNoticeRequest.");
return result;
}
public async Task<List<T>> GetFileContentAndParseRecords<T>(string finalIndexFilePath)
{
var indexFileContent = await _fileOperationsApiService.GetFileContent(finalIndexFilePath);
using var stringReader = new StringReader(indexFileContent);
var config = new CsvConfiguration(CultureInfo.InvariantCulture) { Delimiter = "|", HasHeaderRecord = false };
using var csv = new CsvReader(stringReader, config);
return csv.GetRecords<T>().ToList();
}
private static IngestorResult PrepareRunInfoAndProcessRequests(IngestorRequest request,
IReadOnlyCollection<OsgTemenosNoticesDocument> documentMetaData,
IngestorResult result)
{
result.IngestorRunInfo = new IngestorRunInfo(Guid.NewGuid(),
DateTime.Now,
request.IngestorType,
request.RootPath,
request.IndexFileName,
documentMetaData.Count);
result.ProcessingRequests = new List<ProcessingRequest>();
foreach (var processingRequest in documentMetaData.Select(record => new ProcessingRequest
{
IngestorRunInfoId = result.IngestorRunInfo.IngestorRunInfoId,
IngestorRequest = request,
DocumentFileName = record.PDF,
DocumentValuePayload = JsonConvert.SerializeObject(record),
DocumentTypeDisplayName = record.DocumentType
})){
result.ProcessingRequests.Add(processingRequest);
}
return result;
}
}
@@ -0,0 +1,82 @@
using CsvHelper;
using CsvHelper.Configuration;
using LCU.API.Internal.DocumentIngestor.Services.MicroApiServices;
using LCU.Library.Internal.DocumentIngestorModels.Entities;
using LCU.Library.Internal.DocumentIngestorModels.Models;
using LCU.Library.Internal.DocumentIngestorModels.Models.IndexFileDocuments;
using Newtonsoft.Json;
using Serilog;
using System.Globalization;
using LCU.Library.Internal.LCUServiceFactory.Configurations;
namespace LCU.API.Internal.DocumentIngestor.Services.IndexParserServices;
public class OsgVehicleTaxStatementIndexService : IndexParserService, IIndexParserService
{
private readonly IFileOperationsApiService _fileOperationsApiService;
public OsgVehicleTaxStatementIndexService(IFileOperationsApiService fileOperationsApiService,
ServicesFactoryConfiguration servicesFactoryConfiguration)
: base(servicesFactoryConfiguration)
{
_fileOperationsApiService = fileOperationsApiService;
}
public async Task<IngestorResult> HandleIndexRequest(IngestorRequest request)
{
Log.Debug("Starting HandleIndexRequest for OsgTaxStatement.");
var result = new IngestorResult();
try
{
ArgumentNullException.ThrowIfNull(request);
var records = await GetFileContentAndParseRecords<OsgTaxStatement>(request.FinalIndexFilePath);
result = PrepareRunInfoAndProcessRequests(request, records, result);
}
catch (Exception exception)
{
Log.Error(exception, "An Exception occured while HandleTaxStatementRequest.");
result = new IngestorResult(true, exception.Message);
}
Log.Debug("Ending HandleIndexRequest for OsgTaxStatement.");
return result;
}
public async Task<List<T>> GetFileContentAndParseRecords<T>(string finalIndexFilePath)
{
var indexFileContent = await _fileOperationsApiService.GetFileContent(finalIndexFilePath);
using var stringReader = new StringReader(indexFileContent);
var config = new CsvConfiguration(CultureInfo.InvariantCulture) { Delimiter = "|", HasHeaderRecord = false };
using var csv = new CsvReader(stringReader, config);
var csvRecords = csv.GetRecords<T>();
return csvRecords.ToList();
}
public IngestorResult PrepareRunInfoAndProcessRequests(IngestorRequest request,
List<OsgTaxStatement> documentMetaData,
IngestorResult result)
{
result.IngestorRunInfo = new IngestorRunInfo(Guid.NewGuid(),
DateTime.Now,
request.IngestorType,
request.RootPath,
request.IndexFileName,
documentMetaData.Count());
result.ProcessingRequests = new List<ProcessingRequest>();
foreach (var record in documentMetaData)
{
var processingRequest = new ProcessingRequest()
{
IngestorRunInfoId = result.IngestorRunInfo.IngestorRunInfoId,
IngestorRequest = request,
DocumentFileName = record.PdfName,
DocumentValuePayload = JsonConvert.SerializeObject(record),
DocumentTypeDisplayName = record.NoticeId
};
result.ProcessingRequests.Add(processingRequest);
}
return result;
}
}
@@ -0,0 +1,79 @@
using CsvHelper;
using CsvHelper.Configuration;
using LCU.API.Internal.DocumentIngestor.Services.MicroApiServices;
using LCU.Library.Internal.DocumentIngestorModels.Entities;
using LCU.Library.Internal.DocumentIngestorModels.Models;
using LCU.Library.Internal.DocumentIngestorModels.Models.IndexFileDocuments;
using Newtonsoft.Json;
using Serilog;
using System.Globalization;
using LCU.Library.Internal.LCUServiceFactory.Configurations;
namespace LCU.API.Internal.DocumentIngestor.Services.IndexParserServices
{
public class OsgWeilandStatementIndexService : IndexParserService, IIndexParserService
{
private readonly IFileOperationsApiService _fileOperationsApiService;
public OsgWeilandStatementIndexService(IFileOperationsApiService fileOperationsApiService,
ServicesFactoryConfiguration servicesFactoryConfiguration)
: base(servicesFactoryConfiguration)
{
_fileOperationsApiService = fileOperationsApiService;
}
public async Task<List<T>> GetFileContentAndParseRecords<T>(string finalIndexFilePath)
{
var indexFileContent = await _fileOperationsApiService.GetFileContent(finalIndexFilePath);
using var stringReader = new StringReader(indexFileContent);
var config = new CsvConfiguration(CultureInfo.InvariantCulture) { Delimiter = ",", HasHeaderRecord = true };
using var csv = new CsvReader(stringReader, config);
return csv.GetRecords<T>().ToList();
}
public async Task<IngestorResult> HandleIndexRequest(IngestorRequest request)
{
Log.Debug("Starting HandleIndexRequest for OsgWeilandDocument.");
var result = new IngestorResult();
try
{
ArgumentNullException.ThrowIfNull(request);
var records = await GetFileContentAndParseRecords<OsgWeilandStatement>(request.FinalIndexFilePath);
result = PrepareRunInfoAndProcessRequests(request, records, result);
}
catch (Exception exception)
{
Log.Error(exception, "An Exception occured while HandleIndexRequest.");
result = new IngestorResult(true, exception.Message);
}
Log.Debug("Ending HandleIndexRequest for OsgWeilandDocument.");
return result;
}
public IngestorResult PrepareRunInfoAndProcessRequests(IngestorRequest request, List<OsgWeilandStatement> records, IngestorResult result)
{
result.IngestorRunInfo = new IngestorRunInfo(Guid.NewGuid(),
DateTime.Now,
request.IngestorType,
request.RootPath,
request.IndexFileName,
records.Count());
result.ProcessingRequests = new List<ProcessingRequest>();
foreach (var record in records)
{
var processingRequest = new ProcessingRequest()
{
IngestorRunInfoId = result.IngestorRunInfo.IngestorRunInfoId,
IngestorRequest = request,
DocumentFileName = record.PdfName,
DocumentValuePayload = JsonConvert.SerializeObject(record),
DocumentTypeDisplayName = record.DocCode
};
result.ProcessingRequests.Add(processingRequest);
}
return result;
}
}
}
@@ -0,0 +1,81 @@
using System.Globalization;
using Newtonsoft.Json;
using Serilog;
using CsvHelper.Configuration;
using CsvHelper;
using LCU.API.Internal.DocumentIngestor.Services.MicroApiServices;
using LCU.Library.Internal.DocumentIngestorModels.Entities;
using LCU.Library.Internal.DocumentIngestorModels.Models;
using LCU.Library.Internal.DocumentIngestorModels.Models.IndexFileDocuments;
using LCU.Library.Internal.LCUServiceFactory.Configurations;
namespace LCU.API.Internal.DocumentIngestor.Services.IndexParserServices;
public class XdiActionNoticeDocumentIndexService : IndexParserService, IIndexParserService
{
private readonly IFileOperationsApiService _fileOperationsApiService;
public XdiActionNoticeDocumentIndexService(IFileOperationsApiService fileOperationsApiService,
ServicesFactoryConfiguration servicesFactoryConfiguration)
: base(servicesFactoryConfiguration)
{
_fileOperationsApiService = fileOperationsApiService;
}
public async Task<IngestorResult> HandleIndexRequest(IngestorRequest request)
{
Log.Debug("Starting HandleIndexRequest for XdiActionNoticeDocumentRequest.");
var result = new IngestorResult();
try
{
ArgumentNullException.ThrowIfNull(request);
var records = await GetFileContentAndParseRecords<XdiActionNoticesDocument>(request.FinalIndexFilePath);
result = PrepareRunInfoAndProcessRequests(request, records, result);
}
catch (Exception exception)
{
Log.Error(exception, "An Exception occured while handling XdiActionNoticeDocumentRequest.");
result = new IngestorResult(true, exception.Message);
}
Log.Debug("Ending HandleIndexRequest for XdiActionNoticeDocumentRequest.");
return result;
}
public async Task<List<T>> GetFileContentAndParseRecords<T>(string finalIndexFilePath)
{
var indexFileContent = await _fileOperationsApiService.GetFileContent(finalIndexFilePath);
using var stringReader = new StringReader(indexFileContent);
var config = new CsvConfiguration(CultureInfo.InvariantCulture) { Delimiter = "|", HasHeaderRecord = false };
using var csv = new CsvReader(stringReader, config);
return csv.GetRecords<T>().ToList();
}
private static IngestorResult PrepareRunInfoAndProcessRequests(IngestorRequest request,
IReadOnlyCollection<XdiActionNoticesDocument> documentMetaData,
IngestorResult result)
{
result.IngestorRunInfo = new IngestorRunInfo(Guid.NewGuid(),
DateTime.Now,
request.IngestorType,
request.RootPath,
request.IndexFileName,
documentMetaData.Count);
result.ProcessingRequests = new List<ProcessingRequest>();
foreach (var processingRequest in documentMetaData.Select(record => new ProcessingRequest
{
IngestorRunInfoId = result.IngestorRunInfo.IngestorRunInfoId,
IngestorRequest = request,
DocumentFileName = record.DocumentName,
DocumentValuePayload = JsonConvert.SerializeObject(record),
DocumentTypeDisplayName = record.DocumentType
}))
{
result.ProcessingRequests.Add(processingRequest);
}
return result;
}
}
@@ -0,0 +1,80 @@
using System.Globalization;
using Newtonsoft.Json;
using Serilog;
using CsvHelper.Configuration;
using CsvHelper;
using LCU.API.Internal.DocumentIngestor.Services.MicroApiServices;
using LCU.Library.Internal.DocumentIngestorModels.Entities;
using LCU.Library.Internal.DocumentIngestorModels.Models;
using LCU.Library.Internal.DocumentIngestorModels.Models.IndexFileDocuments;
using LCU.Library.Internal.LCUServiceFactory.Configurations;
namespace LCU.API.Internal.DocumentIngestor.Services.IndexParserServices;
public class XdiDnaNoticeDocumentIndexService : IndexParserService, IIndexParserService
{
private readonly IFileOperationsApiService _fileOperationsApiService;
public XdiDnaNoticeDocumentIndexService(IFileOperationsApiService fileOperationsApiService,
ServicesFactoryConfiguration servicesFactoryConfiguration)
: base(servicesFactoryConfiguration)
{
_fileOperationsApiService = fileOperationsApiService;
}
public async Task<IngestorResult> HandleIndexRequest(IngestorRequest request)
{
Log.Debug("Starting HandleIndexRequest for XdiDnaNoticeDocumentRequest.");
var result = new IngestorResult();
try
{
ArgumentNullException.ThrowIfNull(request);
var records = await GetFileContentAndParseRecords<XdiDnaNoticesDocument>(request.FinalIndexFilePath);
result = PrepareRunInfoAndProcessRequests(request, records, result);
}
catch (Exception exception)
{
Log.Error(exception, "An Exception occured while handling XdiDnaNoticeDocumentRequest.");
result = new IngestorResult(true, exception.Message);
}
Log.Debug("Ending HandleIndexRequest for XdiDnaNoticeDocumentRequest.");
return result;
}
public async Task<List<T>> GetFileContentAndParseRecords<T>(string finalIndexFilePath)
{
var indexFileContent = await _fileOperationsApiService.GetFileContent(finalIndexFilePath);
using var stringReader = new StringReader(indexFileContent);
var config = new CsvConfiguration(CultureInfo.InvariantCulture) { Delimiter = "|", HasHeaderRecord = false };
using var csv = new CsvReader(stringReader, config);
return csv.GetRecords<T>().ToList();
}
private static IngestorResult PrepareRunInfoAndProcessRequests(IngestorRequest request,
IReadOnlyCollection<XdiDnaNoticesDocument> documentMetaData,
IngestorResult result)
{
result.IngestorRunInfo = new IngestorRunInfo(Guid.NewGuid(),
DateTime.Now,
request.IngestorType,
request.RootPath,
request.IndexFileName,
documentMetaData.Count);
result.ProcessingRequests = new List<ProcessingRequest>();
foreach (var processingRequest in documentMetaData.Select(record => new ProcessingRequest
{
IngestorRunInfoId = result.IngestorRunInfo.IngestorRunInfoId,
IngestorRequest = request,
DocumentFileName = record.PDF,
DocumentValuePayload = JsonConvert.SerializeObject(record),
DocumentTypeDisplayName = record.DocumentType
})){
result.ProcessingRequests.Add(processingRequest);
}
return result;
}
}
@@ -0,0 +1,199 @@
using Serilog;
using TimeZoneConverter;
using LCU.Library.Internal.ApiResponse.Services;
using LCU.Library.Internal.Exceptions;
using LCU.API.Internal.DocumentIngestor.Services.MicroApiServices;
using LCU.Api.Internal.DispatcherSharedLib.BuilderServices;
using LCU.Library.Internal.DispatcherModels.Models;
using LCU.Library.Internal.DocumentIngestorModels.Models;
using LCU.API.Internal.DocumentIngestor.Configurations;
using LCU.Library.Internal.DispatcherModels.Enums;
using LCU.Library.Internal.FileOperationsModels.Models;
namespace LCU.API.Internal.DocumentIngestor.Services.IngestionOperationServices;
public class BatchIngestionService : IBatchIngestionService
{
public delegate void AddWorkItem(IngestorRequest request, WorkItemDetails workItemDetails);
public delegate AddWorkItem AddIngestorRequestWorkItemStrategy(IRequestBuilderService builder);
private readonly ApiConfiguration _apiConfiguration;
private readonly DispatcherJobCreationConfiguration _dispatcherJobCreationConfiguration;
private readonly IFileOperationsApiService _fileOperationsApiService;
private readonly IRequestBuilderService _dispatcherRequestBuilder;
private const string JobRequestingApplication = "Document Ingestor Api";
private const string JobName = "Document Ingestor - {IngestorType}";
private const string JobDescription = "This is an Single-Use job that has been created by Document Ingestor Api. After successfully processed it will no longer be needed.";
private const string ArchivedDirectory = "Archived";
private const string CreateArchivedWorkItemName = "Create Archived Directory";
private const string CreateArchivedWorkItemDescription = "Full Path: {Path}";
private const string CreateIngestorRequestWorkItemName = "Ingestor Request";
private const string CreateIngestorRequestWorkItemDescription = "IngestorType: {IngestorType}";
public BatchIngestionService
(
ApiConfiguration apiConfiguration,
DispatcherJobCreationConfiguration dispatcherJobCreationConfiguration,
IFileOperationsApiService fileOperationsApiService,
IRequestBuilderService dispatcherRequestBuilder
)
{
_apiConfiguration = apiConfiguration;
_dispatcherJobCreationConfiguration = dispatcherJobCreationConfiguration;
_fileOperationsApiService = fileOperationsApiService;
_dispatcherRequestBuilder = dispatcherRequestBuilder;
}
public async Task HandleBatchIngestion(BatchIngestionRequest request)
{
try
{
ArgumentNullException.ThrowIfNull(request);
var dynamicDirectoryListRequest = new DynamicDirectoryListRequest
{
Path = request.FinalRootBasePath,
DynamicDirectoryName = request.ParentDynamicDirectory,
DynamicFileSettings = request.ParentDynamicDirectorySettings
};
var dynamicDirectories = await _fileOperationsApiService.BuildDynamicDirectoryList(dynamicDirectoryListRequest);
var batchIngestionResult = new BatchIngestionResult();
foreach (var directory in dynamicDirectories)
{
batchIngestionResult = await CreateAndSendDispatcherRequest(request, directory, batchIngestionResult);
}
if (batchIngestionResult.IsFailure)
{
var message = $"There were {batchIngestionResult.FailureCount} Ingestor Requests that failed to be created.";
batchIngestionResult.Messages.Add(message);
}
await QueueReturnResponse.QueueResponse(request, batchIngestionResult.IsSuccessfull, batchIngestionResult.Messages);
}
catch (ArgumentNullException)
{
throw;
}
catch (Exception exception)
{
const string errorMessage = "Failure during document ingestion.";
var errorMessageWithException = $"{errorMessage} Exception Message: {exception.Message}";
Log.Error(exception, errorMessage);
await QueueReturnResponse.QueueResponse(request, false, new List<string> { errorMessageWithException });
throw new ServiceException(errorMessageWithException);
}
}
public async Task<BatchIngestionResult> CreateAndSendDispatcherRequest(BatchIngestionRequest request,
string directory,
BatchIngestionResult batchIngestionResult)
{
try
{
SetupJobDetails(request);
var ingestionRootPath = DetermineIngestionRootPath(request, directory);
SetupCreateArchivedDirectoryWorkItem(ingestionRootPath);
await SetupCreateIngestorRequestWorkItem(request, ingestionRootPath);
var successful = _dispatcherRequestBuilder.SendDispatcherRequest();
if (!successful)
{
throw new Exception("Failure to send dispatcher request.");
}
}
catch (Exception exception)
{
batchIngestionResult.AddToFailureCount(exception.Message);
}
return batchIngestionResult;
}
public void SetupJobDetails(BatchIngestionRequest request)
{
var timeZone = TZConvert.GetTimeZoneInfo(_apiConfiguration.TimeZone);
var dateNow = TimeZoneInfo.ConvertTime(DateTime.Now, timeZone);
var jobDetails = new JobDetails
{
RequestingApplication = JobRequestingApplication,
Name = JobName.Replace("{IngestorType}", request.IngestorType.ToString()),
Description = JobDescription,
CreatedBy = JobRequestingApplication,
CreatedAt = dateNow,
NextRunTime = dateNow,
ErrorRecipients = _dispatcherJobCreationConfiguration.DocumentIngestorDispatcherErrorRecipients,
JobTags = _dispatcherJobCreationConfiguration.JobTags,
JobStatus = JobStatus.ReadyToSchedule
};
_dispatcherRequestBuilder.CreateJob(jobDetails);
}
public string DetermineIngestionRootPath(BatchIngestionRequest request, string directory)
{
var subDirectoryPath = request.SubDirectoryPath.EndsWith("\\") || string.IsNullOrWhiteSpace(request.SubDirectoryPath) ?
request.SubDirectoryPath :
$"{request.SubDirectoryPath}\\";
var formattedDirectory = directory.EndsWith("\\") ? directory : $"{directory}\\";
var ingestionRootPath = request.FinalRootBasePath + formattedDirectory + subDirectoryPath;
return ingestionRootPath;
}
private void SetupCreateArchivedDirectoryWorkItem(string ingestionRootPath)
{
var createDirectoryRequest = new CreateDirectoryRequest
{
DirectoryName = ArchivedDirectory,
DirectoryPath = ingestionRootPath
};
var createDirectoryWorkItemDetails = new WorkItemDetails
{
Name = CreateArchivedWorkItemName,
Description = CreateArchivedWorkItemDescription.Replace("{Path}", ingestionRootPath + ArchivedDirectory),
Order = 1
};
_dispatcherRequestBuilder.AddWorkItem(createDirectoryRequest, createDirectoryWorkItemDetails, ActionType.CreateDirectory);
}
private async Task SetupCreateIngestorRequestWorkItem(BatchIngestionRequest request, string ingestionRootPath)
{
var indexFile = await DetermineIndexFile(request, ingestionRootPath);
var ingestorRequest = new IngestorRequest
{
IngestorType = request.IngestorType,
RootPath = ingestionRootPath,
IndexFileName = indexFile
};
var ingestorWorkItemDetails = new WorkItemDetails
{
Name = CreateIngestorRequestWorkItemName,
Description = CreateIngestorRequestWorkItemDescription.Replace("{IngestorType}", ingestorRequest.IngestorType.ToString()),
Order = 2
};
ActionType ingestionType;
if ((!_apiConfiguration.IsHistoric))
{
ingestionType = ActionType.DocumentIngestor;
}
else
{
ingestionType = ActionType.HistoricalDocumentIngestor;
}
_dispatcherRequestBuilder.AddWorkItem(ingestorRequest, ingestorWorkItemDetails, ingestionType);
}
public async Task<string> DetermineIndexFile(BatchIngestionRequest request, string directory)
{
var dynamicFileListRequest = new DynamicFileListRequest
{
Path = directory,
Filename = request.DynamicIndexFileName,
DynamicFileSettings = request.DynamicFileSettings
};
return await _fileOperationsApiService.BuildDynamicFile(dynamicFileListRequest);
}
}
@@ -0,0 +1,126 @@
using LCU.Api.Internal.MaidBotLib.Models;
using LCU.API.Internal.DocumentIngestor.Configurations;
using LCU.API.Internal.DocumentIngestor.Services.QueueServices;
using LCU.Library.Internal.DocumentIngestorModels.Enums;
using LCU.Library.Internal.DocumentIngestorModels.Models;
using LCU.Library.Internal.DocumentIngestorModels.Models.QueueResult;
using LCU.Library.Internal.Exceptions;
using MassTransit;
using Serilog;
using System.Data.SqlClient;
namespace LCU.API.Internal.DocumentIngestor.Services.IngestionOperationServices;
public class BatchUpdateMemberNotificationDataService : IBatchUpdateMemberNotificationDataService
{
public readonly IMemberNotificationInfoService _memberNotificationInfoService;
public readonly IQueueCompletionService _queueCompletionService;
public readonly ApiConfiguration _apiConfiguration;
public BatchUpdateMemberNotificationDataService(IMemberNotificationInfoService memberNotificationInfoService,
IQueueCompletionService queueCompletionService,
ApiConfiguration apiConfiguration)
{
_memberNotificationInfoService = memberNotificationInfoService;
_queueCompletionService = queueCompletionService;
_apiConfiguration = apiConfiguration;
}
public async Task HandleBatchUpdateMemberNotificationData(List<BatchUpdateMemberNotificationDataRequest> requests)
{
try
{
ArgumentNullException.ThrowIfNull(requests);
var completedBatchUpdateMemberNotificationDataRequests = await SaveBatchMemberNotificationInfos(requests);
foreach(var request in requests)
{
if (request.GatherMemberNotificationInfoRequest.CompletionRequest.ProcessResult.ProcessStatusType == ProcessStatusTypes.Processing)
{
ProcessResult.ProcessCompleted(request.GatherMemberNotificationInfoRequest.CompletionRequest.ProcessResult, null);
}
}
await QueueCompletionRequests(completedBatchUpdateMemberNotificationDataRequests);
}
catch (ServiceException serviceException)
{
Log.ForContext(Enum.GetName(typeof(PropertyEnum), PropertyEnum.Group),
Enum.GetName(typeof(GroupEnum), GroupEnum.ITDevelopment))
.Error(serviceException, $"Batch Update Member Notification Failure. " +
$"The List<BatchUpdateMemberNotificationDataRequest> is placed on LCU.DocumentIngestor.BatchUpdateMemberNotification_error.");
throw;
}
catch (Exception exception)
{
Log.ForContext(Enum.GetName(typeof(PropertyEnum), PropertyEnum.Group),
Enum.GetName(typeof(GroupEnum), GroupEnum.ITDevelopment))
.Fatal(exception, $"Unexpected Batch Update Member Notification Failure. " +
$"The List<BatchUpdateMemberNotificationDataRequest> is placed on LCU.DocumentIngestor.BatchUpdateMemberNotification_error.");
throw;
}
}
public async Task<List<BatchUpdateMemberNotificationDataRequest>> SaveBatchMemberNotificationInfos(List<BatchUpdateMemberNotificationDataRequest> requests)
{
try
{
await _memberNotificationInfoService.SaveListAsync(requests.SelectMany(r => r.MemberNotificationInfos).ToList());
}
catch (SqlException ex)
{
Log.Error("Error during SaveBatchMemberNotificationInfos. ProcessResults are being set to CriticalFailure");
foreach (var request in requests)
{
request.GatherMemberNotificationInfoRequest.CompletionRequest.ProcessResult =
ProcessResult.ProcessNotificationDataFailure(request.GatherMemberNotificationInfoRequest.CompletionRequest.ProcessResult,
"Failure during Batch save of MemberNotificationInfo");
}
}
return requests;
}
public async Task QueueCompletionRequests(List<BatchUpdateMemberNotificationDataRequest> requests)
{
var tasks = requests.Select(r => QueueCompletionRequest(r.GatherMemberNotificationInfoRequest.CompletionRequest));
var results = await Task.WhenAll(tasks);
var message = string.Join(Environment.NewLine,
results
.Where(r => !r.Success)
.Select(r => $"Failed to Queue Completion after BatchUpdateMemberNotification. " +
$"DocumentProcessingInfoId: {r.CompletionRequest.ProcessingRequest.DocumentProcessingInfoId}, " +
$"DocumentFileName: {r.CompletionRequest.ProcessingRequest.DocumentFileName}."));
if (!string.IsNullOrEmpty(message))
{
Log.ForContext(Enum.GetName(typeof(PropertyEnum), PropertyEnum.Group),
Enum.GetName(typeof(GroupEnum), GroupEnum.ITDevelopment))
.Error(new Exception(), message);
}
}
public async Task<QueueCompletionResult> QueueCompletionRequest(CompletionRequest request)
{
try
{
await _queueCompletionService.QueueCompletionRequest(request.ProcessingRequest, request.ProcessResult);
return new QueueCompletionResult
{
Success = true,
CompletionRequest = request
};
}
catch (Exception ex)
{
var errorMessage = $"Exception during QueueCompletionRequest. ErrorMessage: {ex.Message}";
Log.Warning(errorMessage);
return new QueueCompletionResult
{
Success = false,
CompletionRequest = request
};
}
}
}
@@ -0,0 +1,159 @@
using Dapper;
using LCU.Api.Internal.MaidBotLib.Models;
using LCU.API.Internal.DocumentIngestor.Services.QueueServices;
using LCU.Library.Internal.DocumentIngestorModels.Entities;
using LCU.Library.Internal.DocumentIngestorModels.Enums;
using LCU.Library.Internal.DocumentIngestorModels.Models;
using LCU.Library.Internal.Exceptions;
using MassTransit;
using Newtonsoft.Json;
using Serilog;
using Response = LCU.Library.Internal.ApiResponse.Models.Response;
namespace LCU.API.Internal.DocumentIngestor.Services.IngestionOperationServices;
public class CompletionService : ICompletionService
{
private readonly IIngestorRunInfoService _ingestorRunInfoService;
private readonly IDocumentProcessingInfoService _documentProcessingInfoService;
private readonly IResponseQueue _responseQueue;
public CompletionService
(
IIngestorRunInfoService ingestorRunInfoService,
IDocumentProcessingInfoService documentProcessingInfoService,
IResponseQueue responseQueue
)
{
_ingestorRunInfoService = ingestorRunInfoService;
_documentProcessingInfoService = documentProcessingInfoService;
_responseQueue = responseQueue;
}
public async Task HandleCompletion(List<Response> responses)
{
try
{
ArgumentNullException.ThrowIfNull(responses);
var requests = DeserializeResponses(responses);
ArgumentNullException.ThrowIfNull(requests);
await HandleMessagesAndUpdateDocumentProcessInfo(requests);
}
catch (ArgumentNullException)
{
throw;
}
catch (Exception exception)
{
throw new ServiceException($"Exception was thrown in HandleCompletion. " +
$"Exception Message: {exception.Message}");
}
}
public List<CompletionRequest> DeserializeResponses(List<Response> responses)
{
var requests = new List<CompletionRequest>();
foreach (var response in responses)
{
var request = JsonConvert.DeserializeObject<CompletionRequest>(response.ResponsePayload);
if (response.ResponseStack != null)
{
request.ProcessResult.MessageStack.AddRange(response.ResponseStack);
}
requests.Add(request);
}
return requests;
}
private async Task HandleMessagesAndUpdateDocumentProcessInfo(List<CompletionRequest> requests)
{
var completionProcessingDatas = await CreateAndFilterCompletionProcessingDatas(requests);
var ingestorRunInfoGroups = completionProcessingDatas.GroupBy(
d => d.DocumentProcessingInfo.IngestorRunInfoId,
d => d,
(key, d) => new { IngestorRunInfoId = key, CompletionProcessingData = d.ToList() });
var ingestorRunInfos = await GetAndFilterIngestorRunInfos(ingestorRunInfoGroups.Select(i => i.IngestorRunInfoId).ToList());
foreach (var ingestorRunInfoGroup in ingestorRunInfoGroups)
{
await ProcessRunInfos(ingestorRunInfoGroup.CompletionProcessingData,
ingestorRunInfos.Where(i => i.IngestorRunInfoId == ingestorRunInfoGroup.IngestorRunInfoId).FirstOrDefault());
}
}
public async Task<List<CompletionProcessingData>> CreateAndFilterCompletionProcessingDatas(List<CompletionRequest> requests)
{
var completionProcessingDatas = await _documentProcessingInfoService.UpdateProcessCountAndStatusAndAddLog(requests);
completionProcessingDatas.RemoveAll(d => d.DocumentProcessingInfo.ProcessCount > 1);
completionProcessingDatas.GroupBy(d => d.DocumentProcessingInfo.DocumentProcessingInfoId).Select(d => d.First()).ToList();
return completionProcessingDatas;
}
public async Task<IEnumerable<IngestorRunInfo>> GetAndFilterIngestorRunInfos(List<Guid> ingestorRunInfoIds)
{
var ingestorRunInfos = await _ingestorRunInfoService.GetListByIdAsync(ingestorRunInfoIds);
ingestorRunInfos.AsList().RemoveAll(i => i.EndTimeStamp != null);
return ingestorRunInfos;
}
public async Task ProcessRunInfos(List<CompletionProcessingData> completionProcessingDatas, IngestorRunInfo runInfo)
{
if (runInfo == null)
{
return;
}
runInfo.IngestionRunCount += completionProcessingDatas.Count();
if (runInfo.IngestionRunCount != runInfo.IndexTotal)
{
await _ingestorRunInfoService.Update(runInfo);
return;
}
runInfo.EndTimeStamp = DateTime.Now;
var documentProcessingInfosFromRunInfo = await _documentProcessingInfoService.GetCompletedFirstPassByIngestorRunInfoIdAsync(runInfo.IngestorRunInfoId);
await _ingestorRunInfoService.Update(runInfo);
await DetermineCountsAndQueueResponse(documentProcessingInfosFromRunInfo, completionProcessingDatas.FirstOrDefault().CompletionRequest, runInfo);
}
private async Task DetermineCountsAndQueueResponse(IEnumerable<DocumentProcessingInfo> documentProcessingInfos, CompletionRequest request, IngestorRunInfo ingestorRunInfo)
{
var successCount = documentProcessingInfos.Count(documentProcessingInfo =>
documentProcessingInfo.StatusType == ProcessStatusTypes.Completed
);
var dataMappingFailureCount = documentProcessingInfos.Count(documentProcessingInfo =>
documentProcessingInfo.StatusType == ProcessStatusTypes.DataMappingFailure
);
var criticalFailureCount = documentProcessingInfos.Count(documentProcessingInfo =>
documentProcessingInfo.StatusType == ProcessStatusTypes.CriticalFailure
);
var archiveFailureCount = documentProcessingInfos.Count(documentProcessingInfo =>
documentProcessingInfo.StatusType == ProcessStatusTypes.ArchiveFailure
);
var message = $"IngestorRunInfoId: {ingestorRunInfo.IngestorRunInfoId}, " +
$"Amount Ingested: {ingestorRunInfo.IndexTotal}, " +
$"Success Count: {successCount}, " +
$"Data Mapping Failure Count: {dataMappingFailureCount}, " +
$"Critical Failure Count: {criticalFailureCount}, " +
$"Archive Failure Count: {archiveFailureCount}";
if (criticalFailureCount + archiveFailureCount > 0)
{
Log.ForContext(Enum.GetName(typeof(PropertyEnum), PropertyEnum.Group),
Enum.GetName(typeof(GroupEnum), GroupEnum.ITDevelopment))
.Error(new Exception(), $"CriticalFailure or ArchiveFailure found during completion. Dispatcher Job is being marked as successful. " +
$"Please Investigate failures.{Environment.NewLine}" + message);
}
var messageStack = new List<string> { message };
Log.Information(message);
await _responseQueue.Enqueue(request.ProcessingRequest.IngestorRequest, true, messageStack);
}
}
@@ -0,0 +1,148 @@
using Serilog;
using LCU.API.Internal.DocumentIngestor.Configurations;
using LCU.Library.Internal.DocumentIngestorModels.Enums;
using LCU.Library.Internal.DocumentIngestorModels.Models;
using LCU.API.Internal.DocumentIngestor.Services.DocumentProcessorServices;
using LCU.API.Internal.DocumentIngestor.Services.MicroApiServices;
using LCU.API.Internal.DocumentIngestor.Services.QueueServices;
using LCU.API.Internal.DocumentIngestorLib.Exceptions;
using LCU.Library.Internal.LCUServiceFactory.Services;
using Newtonsoft.Json;
namespace LCU.API.Internal.DocumentIngestor.Services.IngestionOperationServices;
public class DocumentProcessorService : IDocumentProcessorService
{
private ApiConfiguration _apiConfiguration;
private readonly IQueueDocumentProcessingService _queueDocumentProcessingService;
private readonly IQueueCompletionService _queueCompletionService;
private readonly IFileOperationsApiService _fileOperationsApiService;
private readonly ILcuServiceFactory<IDocumentProcessingService> _serviceFactory;
public DocumentProcessorService(ApiConfiguration apiConfiguration,
IQueueDocumentProcessingService queueDocumentProcessingService,
IQueueCompletionService queueCompletionService,
IFileOperationsApiService fileOperationsApiService,
ILcuServiceFactory<IDocumentProcessingService> serviceFactory)
{
_apiConfiguration = apiConfiguration;
_queueDocumentProcessingService = queueDocumentProcessingService;
_queueCompletionService = queueCompletionService;
_fileOperationsApiService = fileOperationsApiService;
_serviceFactory = serviceFactory;
}
public async Task HandleProcessing(ProcessingRequest request)
{
try
{
ArgumentNullException.ThrowIfNull(request);
var documentNeedsProcessing = await ValidateDocumentNeedsProcessing(request);
if (!documentNeedsProcessing)
{
var skippedMessage = "Skipping: Document has already been processed and is archived.";
var processResult = ProcessResult.ProcessCompleted(new ProcessResult(), skippedMessage);
await _queueCompletionService.QueueCompletionRequest(request, processResult);
return;
}
var result = await PerformDocumentProcess(request);
if (!result.Success)
{
await HandlePostProcessingFailure(request, result);
return;
}
await ArchiveDocument(request);
}
catch (ArgumentNullException)
{
throw;
}
catch (ArchiveException)
{
throw;
}
catch (Exception exception)
{
var errorMessage = "Failure during document processing. " +
$"DocumentProcessingInfoId: {request.DocumentProcessingInfoId }, " +
$"IngestorRunInfoId:{request.IngestorRunInfoId}, " +
$"DocumentFileName: {request.DocumentFileName}, " +
$"Exception Message: {exception.Message}";
var result = ProcessResult.ProcessCriticalFailure(new ProcessResult(), errorMessage);
await HandlePostProcessingFailure(request, result);
}
}
private async Task<bool> ValidateDocumentNeedsProcessing(ProcessingRequest request)
{
var filePath = request.IngestorRequest.RootPath + request.DocumentFileName;
var archiveFilePath = $"{request.IngestorRequest.ArchivePath}" +
$"{request.DocumentFileName}";
var documentNeedsProcessing = await _fileOperationsApiService.ValidateFileExistsAndNotArchived(filePath, archiveFilePath);
return documentNeedsProcessing;
}
private async Task<ProcessResult> PerformDocumentProcess(ProcessingRequest request)
{
try
{
var service = _serviceFactory.GetService(request.IngestorRequest.IngestorType);
if (service is null)
{
throw new NotImplementedException();
}
return await service.ProcessRequest(request);
}
catch (Exception exception)
{
return new ProcessResult(true, exception.Message);
}
}
public async Task HandlePostProcessingFailure(ProcessingRequest request, ProcessResult result)
{
if (result.ProcessStatusType == ProcessStatusTypes.CriticalFailure &&
request.RetryCount < _apiConfiguration.DocumentProcessingReQueueCountLimit)
{
request.RetryCount++;
Log.Warning("Processing Request ran into a Critical Failure Type and is being Re-Queued." +
$"ProcessingRequest: {JsonConvert.SerializeObject(request)}");
await _queueDocumentProcessingService.QueueDocumentProcessingRequest(request);
return;
}
if(request.RetryCount > 0)
{
result.MessageStack.Add($"Document attempted to retry processing {request.RetryCount} time(s).");
}
await _queueCompletionService.QueueCompletionRequest(request, result);
}
private async Task ArchiveDocument(ProcessingRequest request)
{
try
{
var response = await _fileOperationsApiService.DirectMoveFile(request.IngestorRequest.RootPath,
request.DocumentFileName,
request.IngestorRequest.ArchivePath);
if (!response.IsSuccessful)
{
throw new Exception(string.Join("; ", response.MessageStack.ToArray()));
}
}
catch (Exception exception)
{
throw new ArchiveException("Failure to archive document. | " +
$"FileName: {request.DocumentFileName} | " +
$"RootPath: {request.IngestorRequest.RootPath} | " +
$"ArchivePath: {request.IngestorRequest.ArchivePath} | " +
$"Exception Message: {exception.Message}");
}
}
}
@@ -0,0 +1,57 @@
using LCU.Api.Internal.MaidBotLib.Models;
using LCU.API.Internal.DocumentIngestor.Services.MemberNotificationInfoDeterminingServices;
using LCU.API.Internal.DocumentIngestor.Services.QueueServices;
using LCU.Library.Internal.DocumentIngestorModels.Enums;
using LCU.Library.Internal.DocumentIngestorModels.Models;
using Serilog;
namespace LCU.API.Internal.DocumentIngestor.Services.IngestionOperationServices;
public class GatherMemberNotificationInfoService : IGatherMemberNotificationInfoService
{
private readonly IMemberNotificationInfoDeterminingService _memberNotificationInfoDeterminingService;
private readonly IQueueCompletionService _queueCompletionService;
private readonly IQueueBatchUpdateMemberNotificationDataService _queueBatchUpdateMemberNotificationDataService;
public GatherMemberNotificationInfoService(IMemberNotificationInfoDeterminingService memberNotificationInfoDeterminingService,
IQueueCompletionService queueCompletionService,
IQueueBatchUpdateMemberNotificationDataService queueBatchUpdateMemberNotificationDataService)
{
_memberNotificationInfoDeterminingService = memberNotificationInfoDeterminingService;
_queueCompletionService = queueCompletionService;
_queueBatchUpdateMemberNotificationDataService = queueBatchUpdateMemberNotificationDataService;
}
public async Task HandleMemberNotificationInfoGathering(GatherMemberNotificationInfoRequest request)
{
try
{
var memberNotificationInfos = await _memberNotificationInfoDeterminingService.HandleMemberNotificationInfos(request.CompletionRequest.MemberNotificationInfoRequest);
if(!memberNotificationInfos.Any())
{
if (request.CompletionRequest.ProcessResult.ProcessStatusType == ProcessStatusTypes.Processing)
{
ProcessResult.ProcessCompleted(request.CompletionRequest.ProcessResult, null);
}
await _queueCompletionService.QueueCompletionRequest(request.CompletionRequest.ProcessingRequest,
request.CompletionRequest.ProcessResult);
return;
}
await _queueBatchUpdateMemberNotificationDataService.QueueBatchUpdateMemberNotificationDataRequest(
new BatchUpdateMemberNotificationDataRequest(
memberNotificationInfos,
request));
}
catch (Exception exception)
{
var message = $"Failure to gather and create member notification info.";
request.CompletionRequest.ProcessResult = ProcessResult.ProcessNotificationDataFailure(request.CompletionRequest.ProcessResult, $"{message}, Exception Message: {exception.Message}");
Log.ForContext(Enum.GetName(typeof(PropertyEnum), PropertyEnum.Group),
Enum.GetName(typeof(GroupEnum), GroupEnum.ITDevelopment))
.Error(exception, message);
}
}
}
@@ -0,0 +1,9 @@
using LCU.Library.Internal.DocumentIngestorModels.Models;
namespace LCU.API.Internal.DocumentIngestor.Services.IngestionOperationServices;
public interface IBatchIngestionService
{
Task HandleBatchIngestion(BatchIngestionRequest request);
Task<string> DetermineIndexFile(BatchIngestionRequest request, string directory);
}
@@ -0,0 +1,8 @@
using LCU.Library.Internal.DocumentIngestorModels.Models;
namespace LCU.API.Internal.DocumentIngestor.Services.IngestionOperationServices;
public interface IBatchUpdateMemberNotificationDataService
{
Task HandleBatchUpdateMemberNotificationData(List<BatchUpdateMemberNotificationDataRequest> requests);
}
@@ -0,0 +1,7 @@
using LCU.Library.Internal.ApiResponse.Models;
namespace LCU.API.Internal.DocumentIngestor.Services.IngestionOperationServices;
public interface ICompletionService
{
Task HandleCompletion(List<Response> responses);
}
@@ -0,0 +1,8 @@
using LCU.Library.Internal.DocumentIngestorModels.Models;
namespace LCU.API.Internal.DocumentIngestor.Services.IngestionOperationServices;
public interface IDocumentProcessorService
{
Task HandleProcessing(ProcessingRequest request);
}
@@ -0,0 +1,8 @@
using LCU.Library.Internal.DocumentIngestorModels.Models;
namespace LCU.API.Internal.DocumentIngestor.Services.IngestionOperationServices;
public interface IGatherMemberNotificationInfoService
{
Task HandleMemberNotificationInfoGathering(GatherMemberNotificationInfoRequest request);
}
@@ -0,0 +1,8 @@
using LCU.Library.Internal.DocumentIngestorModels.Models;
namespace LCU.API.Internal.DocumentIngestor.Services.IngestionOperationServices;
public interface IIndexService
{
Task HandleIngestion(IngestorRequest request);
}
@@ -0,0 +1,8 @@
using LCU.Library.Internal.DocumentIngestorModels.Models;
namespace LCU.API.Internal.DocumentIngestor.Services.IngestionOperationServices;
public interface IIngestionFolderCleanUpService
{
Task HandleIngestionFolderCleanUp(IngestionFolderCleanUpRequest request);
}
@@ -0,0 +1,7 @@
using LCU.Library.Internal.DocumentIngestorModels.Models;
namespace LCU.API.Internal.DocumentIngestor.Services.IngestionOperationServices;
public interface IMemberDocumentNotifierService
{
Task HandleMemberDocumentNotification(MemberDocumentNotificationRequest request);
}
@@ -0,0 +1,7 @@
using LCU.Library.Internal.ApiResponse.Models;
namespace LCU.API.Internal.DocumentIngestor.Services.IngestionOperationServices;
public interface IMemberNotificationCompletionService
{
Task HandleMemberNotificationCompletion(Response returnResponse);
}
@@ -0,0 +1,7 @@
using LCU.Library.Internal.DocumentIngestorModels.Models;
namespace LCU.API.Internal.DocumentIngestor.Services.IngestionOperationServices;
public interface IOsgPreprocessorService
{
Task HandleOsgPreprocessing(OsgPreprocessorRequest request);
}
@@ -0,0 +1,8 @@
using LCU.Library.Internal.ApiResponse.Models;
namespace LCU.API.Internal.DocumentIngestor.Services.IngestionOperationServices;
public interface IPdfPartitionerCompletionService
{
Task HandlePdfPartitionerCompletion(Response response);
}
@@ -0,0 +1,7 @@
using LCU.Library.Internal.DocumentIngestorModels.Models;
namespace LCU.API.Internal.DocumentIngestor.Services.IngestionOperationServices;
public interface IPdfPartitionerService
{
Task HandlePdfPartition(PdfPartitionerRequest request);
}
@@ -0,0 +1,11 @@
using LCU.Library.Internal.DocumentIngestorModels.Entities;
using LCU.Library.Internal.DocumentIngestorModels.Requests;
namespace LCU.API.Internal.DocumentIngestor.Services.IngestionOperationServices
{
public interface IProcessStatusTypeEmailService
{
Task HandleProcessStatusTypeEmailRequest(ProcessStatusTypeEmailRequest request);
Task SendProcessStatusEmailAsync(IEnumerable<DocumentProcessingInfo> listDocumentProcessingInfo, ProcessStatusTypeEmailRequest request);
}
}
@@ -0,0 +1,7 @@
using LCU.Library.Internal.ApiResponse.Models;
namespace LCU.API.Internal.DocumentIngestor.Services.IngestionOperationServices;
public interface IUploadCompletionService
{
Task HandleUploadCompletion(List<Response> requests);
}
@@ -0,0 +1,177 @@
using LCU.API.Internal.DocumentIngestor.Services.IndexParserServices;
using LCU.API.Internal.DocumentIngestor.Services.MicroApiServices;
using LCU.API.Internal.DocumentIngestor.Services.QueueServices;
using LCU.Library.Internal.ApiResponse.Services;
using LCU.Library.Internal.DocumentIngestorModels.Constants;
using LCU.Library.Internal.DocumentIngestorModels.Entities;
using LCU.Library.Internal.DocumentIngestorModels.Enums;
using LCU.Library.Internal.DocumentIngestorModels.Models;
using LCU.Library.Internal.Exceptions;
using LCU.Library.Internal.LCUServiceFactory.Services;
using Serilog;
namespace LCU.API.Internal.DocumentIngestor.Services.IngestionOperationServices;
public class IndexService : IIndexService
{
private readonly IQueueDocumentProcessingService _queueDocumentProcessing;
private readonly IIngestorRunInfoService _ingestorRunInfoService;
private readonly IDocumentProcessingInfoService _documentProcessingInfoService;
private readonly IDocumentTypeRetrieverService _documentTypeRetrieverService;
private readonly ILcuServiceFactory<IIndexParserService> _serviceFactory;
public IndexService(IQueueDocumentProcessingService queueDocumentProcessing,
IIngestorRunInfoService ingestorRunInfoService,
IDocumentProcessingInfoService documentProcessingInfoService,
IDocumentTypeRetrieverService documentTypeRetrieverService,
ILcuServiceFactory<IIndexParserService> serviceFactory)
{
_queueDocumentProcessing = queueDocumentProcessing;
_ingestorRunInfoService = ingestorRunInfoService;
_documentProcessingInfoService = documentProcessingInfoService;
_documentTypeRetrieverService = documentTypeRetrieverService;
_serviceFactory = serviceFactory;
}
public async Task HandleIngestion(IngestorRequest request)
{
try
{
ArgumentNullException.ThrowIfNull(request);
var result = await PerformIngestor(request);
if (result.FatalFailure)
{
var errorMessage = string.Join("; ", result.MessageStack.ToArray());
throw new Exception($"Fatal Failure occured. Error Message: {errorMessage}");
}
await SendProcessRequest(result);
}
catch (ArgumentNullException)
{
throw;
}
catch (Exception exception)
{
const string errorMessage = "Failure during document ingestion.";
var errorMessageWithException = $"{errorMessage} Exception Message: {exception.Message}";
Log.Error(exception, errorMessage);
await QueueReturnResponse.QueueResponse(request, false, new List<string> { errorMessageWithException });
throw new ServiceException(errorMessageWithException);
}
}
public async Task<IngestorResult> PerformIngestor(IngestorRequest request)
{
try
{
var service = _serviceFactory.GetService(request.IngestorType);
if (service is null)
{
throw new NotImplementedException();
}
return await service.HandleIndexRequest(request);
}
catch (Exception exception)
{
return new IngestorResult(true, exception.Message);
}
}
/// <summary>
/// This method is used to:
/// - Save IngestorRunInfo
/// - Create/Save DocumentProcessingInfo
/// - Queue for processing
///
/// NOTE: If there are any failures to save to database this process should be stopped by exception and will be
/// caught, logged, maidbot will be used, and the IngestorRequest will be placed on the LCU.DocumentIngestor.Indexing_error.
///
/// NOTE: If failure to queue an exception will be logged, DocumentProcessInfo's status will be set to CriticalFailure, and a ProcessLog will be created.
/// </summary>
/// <param name="result"></param>
/// <returns></returns>
public async Task SendProcessRequest(IngestorResult result)
{
result.IngestorRunInfo = await _ingestorRunInfoService.SaveAsync(result.IngestorRunInfo);
ArgumentNullException.ThrowIfNull(result.IngestorRunInfo);
var documentCodeDetailInfos = await GetDistinctDocumentCodeDetailInfos(result.ProcessingRequests);
List<DocumentProcessingInfo> documentProcessingInfos = new();
foreach (var processRequest in result.ProcessingRequests)
{
var documentProcessingInfo = new DocumentProcessingInfo(result.IngestorRunInfo.IngestorRunInfoId,
DateTime.Now,
processRequest.DocumentValuePayload,
ProcessStatusTypes.Processing,
processRequest.DocumentFileName,
documentCodeDetailInfos[processRequest.DocumentTypeDisplayName].DocumentType,
documentCodeDetailInfos[processRequest.DocumentTypeDisplayName].DocumentBusinessName);
documentProcessingInfos.Add(documentProcessingInfo);
}
await _documentProcessingInfoService.SaveRangeAsync(documentProcessingInfos.ToArray());
var savedDocumentProcessingInfos = (await _documentProcessingInfoService.GetByIngestorRunInfoIdAsync(result.IngestorRunInfo.IngestorRunInfoId)).ToArray();
var tasks = new List<Task>();
tasks.AddRange(
savedDocumentProcessingInfos.Select<DocumentProcessingInfo, Task>(
(documentProcessingInfo) =>
{
var processRequest = result.ProcessingRequests.First(processingRequest => processingRequest.DocumentFileName == documentProcessingInfo.DocumentFileName);
processRequest.DocumentProcessingInfoId = documentProcessingInfo.DocumentProcessingInfoId;
return SendDocumentProcessingRequest(processRequest, documentProcessingInfo);
}
)
);
await Task.WhenAll(tasks);
}
private async Task<Dictionary<string, DocumentCodeDetailInfo>> GetDistinctDocumentCodeDetailInfos(IEnumerable<ProcessingRequest> processingRequests)
{
var distinctProcessingRequests = processingRequests.Select(pr => pr.DocumentTypeDisplayName).Distinct();
var documentCodeDetailInfos = new Dictionary<string, DocumentCodeDetailInfo>();
foreach (var documentTypeDisplayName in distinctProcessingRequests)
{
// TODO: Get by range?
var documentCodeDetail =
await _documentTypeRetrieverService.GetDocumentCodeDetailByDisplayName(documentTypeDisplayName);
documentCodeDetailInfos.Add(documentTypeDisplayName, documentCodeDetail);
}
return documentCodeDetailInfos;
}
public async Task SendDocumentProcessingRequest(ProcessingRequest processRequest, DocumentProcessingInfo documentProcessInfo)
{
try
{
await _queueDocumentProcessing.QueueDocumentProcessingRequest(processRequest);
}
catch (Exception exception)
{
var errorMessage = "Failure to Queue DocumentProcessingRequest. " +
$"DocumentProcessInfoId: {documentProcessInfo.DocumentProcessingInfoId}, " +
$"Exception Message: {exception.Message}";
Log.Error(errorMessage);
var documentProcessingInfo = await _documentProcessingInfoService.UpdateStatusAndAddLog(documentProcessInfo,
ProcessStatusTypes.CriticalFailure,
errorMessage,
Users.System);
if (documentProcessingInfo == null)
{
throw new Exception("Unable to update DocumentProcessingInfo, null returned.");
}
}
}
}
@@ -0,0 +1,329 @@
using System.Text.RegularExpressions;
using Serilog;
using LCU.Library.Internal.ApiResponse.Services;
using LCU.Library.Internal.Exceptions;
using LCU.API.Internal.DocumentIngestor.Services.MicroApiServices;
using LCU.Library.Internal.DocumentIngestorModels.Models;
using LCU.Library.Internal.DocumentIngestorModels.Entities;
using LCU.API.Internal.DocumentIngestor.Utilities;
using LCU.Library.Internal.FileOperationsModels.Models;
using LCU.Library.Internal.FileOperationsModels.Models.NameSettings;
namespace LCU.API.Internal.DocumentIngestor.Services.IngestionOperationServices;
public class IngestionFolderCleanUpService : IIngestionFolderCleanUpService
{
private readonly IFileOperationsApiService _fileOperationsApiService;
private readonly IIngestorRunInfoService _ingestorRunInfoService;
private readonly IDocumentProcessingInfoService _documentProcessingInfoService;
private readonly Regex _zipRootPathRegex = new(@"(?<ziprootpath>^.*\\\d{8}\\)", RegexOptions.Compiled | RegexOptions.IgnoreCase);
private readonly Regex _zipFileName = new(@"^.*\\\d{8}\\(?<zipfilename>[a-zA-Z0-9 _.\(\)-]+)\\", RegexOptions.Compiled | RegexOptions.IgnoreCase);
public IngestionFolderCleanUpService
(
IFileOperationsApiService fileOperationsApiService,
IIngestorRunInfoService ingestorRunInfoService,
IDocumentProcessingInfoService documentProcessingInfoService
)
{
_fileOperationsApiService = fileOperationsApiService;
_ingestorRunInfoService = ingestorRunInfoService;
_documentProcessingInfoService = documentProcessingInfoService;
}
public async Task HandleIngestionFolderCleanUp(IngestionFolderCleanUpRequest request)
{
try
{
var ingestionFolderCleanUpResult = new IngestionFolderCleanUpResult();
var archiveDirectoryWasCreated = false;
var ingestorRunInfos = _ingestorRunInfoService.FindByIngestionEndDateNotArchived(request.CleanUpIngestionFoldersUntilDate).ToList();
if (!ingestorRunInfos.Any())
{
var failureMessage =
$"No ingestions were found to clean up with the IngestionEndedDate: {request.IngestionEndedDate}";
ingestionFolderCleanUpResult.Messages.Add(failureMessage);
Log.Information($"Failure while handling IngestionFolderCleanUp. {failureMessage}");
}
foreach (var ingestorRunInfo in ingestorRunInfos)
{
var zipFileName = GetZipFileName(ingestorRunInfo.IndexFilePath);
if (string.IsNullOrWhiteSpace(zipFileName.Replace(".zip", string.Empty)))
{
var failureMessage = $"A ZipFileName with the required pattern could not be found for IngestorRunInfoId: {ingestorRunInfo.IngestorRunInfoId}. " +
$"IndexFilePath: {ingestorRunInfo.IndexFilePath} | " +
$"Example: '\\{DateTime.Now:yyyyMMdd}\\(ZipFileName).zip'.";
Log.Error("Failure while handling IngestionFolderCleanUp. " + failureMessage);
ingestionFolderCleanUpResult.AddToFailureCount(failureMessage);
continue;
}
var zipRootFolder = GetZipRootFolder(ingestorRunInfo.IndexFilePath);
if (string.IsNullOrWhiteSpace(zipRootFolder))
{
var failureMessage = $"A ZipRootFolder with the required pattern could not be found for IngestorRunInfoId: {ingestorRunInfo.IngestorRunInfoId}. " +
$"IndexFilePath: {ingestorRunInfo.IndexFilePath} | " +
$"Example: '\\(ZipRootFolder)\\{DateTime.Now:yyyyMMdd}\\'.";
Log.Error("Failure while handling IngestionFolderCleanUp. " + failureMessage);
ingestionFolderCleanUpResult.AddToFailureCount(failureMessage);
continue;
}
var unzippedFolderName = GetUnzippedFolderName(ingestorRunInfo.IndexFilePath);
if (string.IsNullOrWhiteSpace(unzippedFolderName))
{
var failureMessage = $"A UnzippedFolderName with the required pattern could not be found for IngestorRunInfoId: {ingestorRunInfo.IngestorRunInfoId}. " +
$"IndexFilePath: {ingestorRunInfo.IndexFilePath} | " +
$"Example: '\\{DateTime.Now:yyyyMMdd}\\(UnzippedFolderName)\\'.";;
Log.Error("Failure while handling IngestionFolderCleanUp. " + failureMessage);
ingestionFolderCleanUpResult.AddToFailureCount(failureMessage);
continue;
}
var countOfCompletedDocumentProcessingInfos = await _documentProcessingInfoService.CountCompletedByIngestorRunInfoIdAsync(ingestorRunInfo.IngestorRunInfoId);
var skipIncompleteIngestion = ingestorRunInfo.IndexTotal != countOfCompletedDocumentProcessingInfos;
if (skipIncompleteIngestion)
{
continue;
}
var archivedFileCountMatchResult = await ArchiveFileCountMatchesIndexTotal(ingestorRunInfo, ingestionFolderCleanUpResult);
ingestionFolderCleanUpResult = archivedFileCountMatchResult.result;
if (!archivedFileCountMatchResult.success)
{
continue;
}
var zipFileExistsResult = await ZipFileExists(ingestorRunInfo.IngestorRunInfoId, zipRootFolder, zipFileName, ingestionFolderCleanUpResult);
ingestionFolderCleanUpResult = zipFileExistsResult.result;
if (!zipFileExistsResult.success)
{
continue;
}
archiveDirectoryWasCreated = await CreateArchiveDirectory(request, archiveDirectoryWasCreated);
ingestionFolderCleanUpResult = await MoveZipFile(ingestorRunInfo.IngestorRunInfoId,
zipRootFolder,
zipFileName,
request.ArchiveRootPathAndFolder,
ingestionFolderCleanUpResult);
ingestionFolderCleanUpResult = await DeleteUnzippedFolder(ingestorRunInfo, zipRootFolder, unzippedFolderName, ingestionFolderCleanUpResult);
ingestionFolderCleanUpResult = await UpdateArchiveTimeStamp(ingestorRunInfo, ingestionFolderCleanUpResult);
}
await QueueReturnResponse.QueueResponse(request, ingestionFolderCleanUpResult.IsSuccessful, ingestionFolderCleanUpResult.Messages);
}
catch (ServiceException serviceException)
{
await QueueReturnResponse.QueueResponse(request, false, new List<string> { serviceException.Message });
Log.Error(serviceException, "Failure while handling IngestionFolderCleanUp.");
throw new ServiceException(serviceException.Message);
}
catch (Exception exception)
{
const string errorMessage = "Failure while handling IngestionFolderCleanUp.";
var errorMessageWithException = $"{errorMessage} Exception Message: {exception.Message}";
Log.Error(exception, errorMessage);
await QueueReturnResponse.QueueResponse(request, false, new List<string> { errorMessageWithException });
throw new ServiceException(errorMessageWithException);
}
}
private async Task<bool> CreateArchiveDirectory(IngestionFolderCleanUpRequest request, bool archiveDirectoryWasCreated)
{
if (archiveDirectoryWasCreated)
{
return archiveDirectoryWasCreated;
}
try
{
var createDirectory =
await _fileOperationsApiService.DirectCreateDirectory(request.ArchiveRootPath,
request.ArchiveFolderName);
if (!createDirectory.IsSuccessful)
{
throw new ServiceException(
$"Error when attempting to create archive directory: {request.ArchiveRootPathAndFolder}");
}
}
catch (ServiceException serviceException)
{
if (!serviceException.Message.Contains("Directory exists at path"))
{
throw;
}
}
return true;
}
private async Task<IngestionFolderCleanUpResult> MoveZipFile(Guid ingestorRunInfoId,
string zipRootFolder,
string zipFileName,
string archivePath,
IngestionFolderCleanUpResult ingestionFolderCleanUpResult)
{
try
{
await _fileOperationsApiService.DirectMoveFile(zipRootFolder, zipFileName, archivePath);
}
catch (ServiceException serviceException)
{
var failureMessage =
$"Failed to move zip file ({zipRootFolder + "\\" + zipFileName}) to ({archivePath}) for IngestorRunInfoId: {ingestorRunInfoId}. " +
$"Exception: {serviceException.Message}";
Log.Error(failureMessage);
ingestionFolderCleanUpResult.AddToFailureCount(failureMessage);
}
return ingestionFolderCleanUpResult;
}
private async Task<(bool success, IngestionFolderCleanUpResult result)> ArchiveFileCountMatchesIndexTotal(IngestorRunInfo ingestorRunInfo,
IngestionFolderCleanUpResult ingestionFolderCleanUpResult)
{
try
{
var archivedFilesRequest = new DynamicFileListRequest()
{
Path = ingestorRunInfo.IndexFilePath + "\\Archived",
Filename = "{ArchiveFiles}",
DynamicFileSettings = new DynamicNameSetting()
{
RegexSettings = new List<RegexSetting>()
{
new()
{
Tag = "{ArchiveFiles}",
Regex = ".*"
}
}
}
};
var archivedFilesCount = (await _fileOperationsApiService.BuildDynamicFileList(archivedFilesRequest)).Count();
if (archivedFilesCount != ingestorRunInfo.IndexTotal)
{
throw new Exception($"Count of files in Archived directory ({archivedFilesCount}) does not" +
$"match IndexTotal ({ingestorRunInfo.IndexTotal}) for IngestorRunInfoId: {ingestorRunInfo.IngestorRunInfoId}");
}
return (true, ingestionFolderCleanUpResult);
}
catch (Exception exception)
{
Log.Error("Failure while handling IngestionFolderCleanUp. " + exception.Message);
ingestionFolderCleanUpResult.AddToFailureCount(exception.Message);
return (false, ingestionFolderCleanUpResult);
}
}
private async Task<IngestionFolderCleanUpResult> DeleteUnzippedFolder(IngestorRunInfo ingestorRunInfo,
string zipRootFolder,
string unzippedFolderName,
IngestionFolderCleanUpResult ingestionFolderCleanUpResult)
{
try
{
var deleteDirectory =
await _fileOperationsApiService.DirectDeleteDirectory(unzippedFolderName,
zipRootFolder,
5);
if (!deleteDirectory.IsSuccessful)
{
var failureMessage =
$"Failed to delete directory {ingestorRunInfo.IndexFilePath} for IngestorRunInfoId {ingestorRunInfo.IngestorRunInfoId}";
Log.Error(failureMessage);
ingestionFolderCleanUpResult.AddToFailureCount(failureMessage);
}
}
catch (ServiceException serviceException)
{
var failureMessage =
$"Failed to delete directory {ingestorRunInfo.IndexFilePath} for IngestorRunInfoId {ingestorRunInfo.IngestorRunInfoId}. " +
$"Exception: {serviceException.Message}";
Log.Error(failureMessage);
ingestionFolderCleanUpResult.AddToFailureCount(failureMessage);
}
return ingestionFolderCleanUpResult;
}
private async Task<IngestionFolderCleanUpResult> UpdateArchiveTimeStamp(IngestorRunInfo ingestorRunInfo,
IngestionFolderCleanUpResult ingestionFolderCleanUpResult)
{
ingestorRunInfo.ArchiveTimeStamp = DateTime.Now;
try
{
await _ingestorRunInfoService.Update(ingestorRunInfo);
}
catch (Exception ex)
{
ingestionFolderCleanUpResult.AddToFailureCount(
$"Failure to update ArchiveTimeStamp for IngestorRunInfoId: {ingestorRunInfo.IngestorRunInfoId}");
Log.Error(ex.Message);
}
return ingestionFolderCleanUpResult;
}
private string GetZipRootFolder(string indexFilePath) =>
_zipRootPathRegex.Match(indexFilePath).Groups["ziprootpath"].Value;
private string GetZipFileName(string indexFilePath) =>
_zipFileName.Match(indexFilePath).Groups["zipfilename"].Value + ".zip";
private string GetUnzippedFolderName(string indexFilePath) =>
_zipFileName.Match(indexFilePath).Groups["zipfilename"].Value;
private async Task<(bool success, IngestionFolderCleanUpResult result)> ZipFileExists(Guid ingestorRunInfoId,
string zipRootFolder,
string zipFileName,
IngestionFolderCleanUpResult ingestionFolderCleanUpResult)
{
try
{
var escapedZipFileName = SpecialCharacters.EscapeSpecialCharacters(zipFileName);
var zipFileRequest = new DynamicFileListRequest()
{
Path = zipRootFolder,
Filename = "{ZipFileRegex}",
DynamicFileSettings = new DynamicNameSetting()
{
RegexSettings = new List<RegexSetting>()
{
new()
{
Tag = "{ZipFileRegex}",
// (?i) = ignore case
Regex = $"(?i){escapedZipFileName}"
}
}
}
};
var zipFiles = await _fileOperationsApiService.BuildDynamicFileList(zipFileRequest);
if (zipFiles.Count() != 1)
{
throw new Exception($"Failed to find zip file '{escapedZipFileName}' in '{zipRootFolder}' for IngestorRunInfoId: {ingestorRunInfoId}");
}
return (true, ingestionFolderCleanUpResult);
}
catch (Exception exception)
{
Log.Error("Failure while handling IngestionFolderCleanUp. " + exception.Message);
ingestionFolderCleanUpResult.AddToFailureCount(exception.Message);
return (false, ingestionFolderCleanUpResult);
}
}
}
@@ -0,0 +1,379 @@
using System.Text;
using System.Text.RegularExpressions;
using Newtonsoft.Json;
using Serilog;
using LCU.Api.Internal.DocumentIngestorLib.Extensions;
using LCU.API.Internal.DocumentIngestor.Configurations;
using LCU.API.Internal.DocumentIngestor.Services.MemberNotificationInfoDeterminingServices;
using LCU.API.Internal.DocumentIngestor.Services.MicroApiServices;
using LCU.API.Internal.DocumentIngestor.Services.QueueServices;
using LCU.Library.DocumentType.Models.Requests;
using LCU.Library.Internal.NotificationsModels.Models;
using LCU.Library.Internal.ApiResponse.Models;
using LCU.Library.Internal.ApiResponse.Services;
using LCU.Library.Internal.DocumentIngestorModels.Entities;
using LCU.Library.Internal.DocumentIngestorModels.Enums;
using LCU.Library.Internal.DocumentIngestorModels.Models;
using LCU.Library.Internal.Exceptions;
using LCU.Library.Internal.LcuMassTransitModels;
namespace LCU.API.Internal.DocumentIngestor.Services.IngestionOperationServices;
public class MemberDocumentNotifierService : IMemberDocumentNotifierService
{
private readonly IMemberNotificationInfoService _memberNotificationInfoService;
private readonly IDocumentTypeRetrieverService _documentTypeRetrieverService;
private readonly IDocumentProcessingInfoService _documentProcessingInfoService;
private readonly INotificationApiService _notificationApiService;
private readonly IMemberNotificationInfoDeterminingService _memberNotificationInfoDeterminingService;
private readonly IQueueMemberNotificationCompletionService _queueMemberNotificationCompletionService;
private readonly ApiConfiguration _apiConfiguration;
private readonly ConnectionConfiguration _connectionConfiguration;
private readonly ReceiveEndpointConfiguration _memberNotificationCompletionReceiveEndpointConfiguration;
private readonly DigitalBankingConfiguration _digitalBankingConfiguration;
private const string EntityNameTag = "{EntityName}";
private const string NumberOfDocumentsTag = "{NumberOfDocuments}";
private const string DocOrDocsTag = "{DocOrDocs}";
private const string TableRowsTag = "{TableRows}";
private const string DocumentNameTag = "{DocumentName}";
private const string DigitalBankingTag = "{DigitalBanking}";
private const string MobileBankingTag = "{MobileBanking}";
public MemberDocumentNotifierService(IMemberNotificationInfoService memberNotificationInfoService,
IDocumentTypeRetrieverService documentTypeRetrieverService,
IDocumentProcessingInfoService documentProcessingInfoService,
INotificationApiService notificationApiService,
IMemberNotificationInfoDeterminingService memberNotificationInfoDeterminingService,
IQueueMemberNotificationCompletionService queueMemberNotificationCompletionService,
ApiConfiguration apiConfiguration,
ConnectionConfiguration connectionConfiguration,
ReceiveEndpointConfigurations receiveEndpointConfigurations,
DigitalBankingConfiguration digitalBankingConfiguration)
{
_memberNotificationInfoService = memberNotificationInfoService;
_documentTypeRetrieverService = documentTypeRetrieverService;
_documentProcessingInfoService = documentProcessingInfoService;
_notificationApiService = notificationApiService;
_memberNotificationInfoDeterminingService = memberNotificationInfoDeterminingService;
_queueMemberNotificationCompletionService = queueMemberNotificationCompletionService;
_apiConfiguration = apiConfiguration;
_connectionConfiguration = connectionConfiguration;
_memberNotificationCompletionReceiveEndpointConfiguration = receiveEndpointConfigurations.MemberNotificationCompletionEndpoint;
_digitalBankingConfiguration = digitalBankingConfiguration;
}
public async Task HandleMemberDocumentNotification(MemberDocumentNotificationRequest request)
{
try
{
ArgumentNullException.ThrowIfNull(request);
var result = await PrepareAndSendNotifications(request);
await QueueReturnResponse.QueueResponse(request, result.IsSuccessful, result.Messages);
}
catch (ArgumentNullException)
{
throw;
}
catch (Exception exception)
{
throw new ServiceException(exception.Message);
}
}
public async Task<MemberDocumentNotificationResult> PrepareAndSendNotifications(MemberDocumentNotificationRequest request)
{
var result = new MemberDocumentNotificationResult();
try
{
var memberNotificationInfos = await GetMemberNotificationInfos(request);
if (!memberNotificationInfos.Any())
{
result.Messages.Add("No MemberNotificationInfos found.");
return result;
}
var memberNotificationInfoCount = memberNotificationInfos.Count();
var skipAmount = _apiConfiguration.BatchSkipAmount == null || _apiConfiguration.BatchSkipAmount == 0
? memberNotificationInfoCount
: _apiConfiguration.BatchSkipAmount;
var documentProcessingInfos = memberNotificationInfos.Chunk(skipAmount).Select(async memberNotificationInfoChunk => {
var documentProcessingInfoIds = memberNotificationInfoChunk
.Select(memberNotificationInfo => memberNotificationInfo.DocumentProcessingInfoId).ToList();
var documentProcessingInfoGroup = await _documentProcessingInfoService.GetListByIdAsync(documentProcessingInfoIds);
return documentProcessingInfoGroup;
}).SelectMany(memberNotificationInfoChunkResult => memberNotificationInfoChunkResult.Result).ToList();
var filteredResults = await RemoveMaliciousDocuments(memberNotificationInfos, documentProcessingInfos);
memberNotificationInfos = filteredResults.MemberNotificationInfos;
documentProcessingInfos = filteredResults.DocumentProcessingInfos.ToList();
memberNotificationInfos = await UpdateForSending(memberNotificationInfos);
var emailGroups = memberNotificationInfos.Where(x => x.DeliveryMethod != DeliveryMethod.Paper)
.GroupBy(x => ( x.AgreeUserId, x.EmailAddress ));
if (!emailGroups.Any())
{
result.Messages.Add("Result returned no emails to send.");
return result;
}
foreach (var emailGroup in emailGroups)
{
if (string.IsNullOrWhiteSpace(emailGroup.Key.EmailAddress))
{
var completionRequest = new MemberNotificationCompletionRequest()
{
MemberNotificationInfoIds = emailGroup.Select(x => x.MemberNotificationInfoId).ToList()
};
var message = $"Email address was null for AgreeUserId: {emailGroup.Key.AgreeUserId}";
await _queueMemberNotificationCompletionService.QueueMemberNotificationCompletionRequest(completionRequest, false, new() { message });
continue;
}
var memberNotificationDocIds = emailGroup.Select(x => x.DocumentProcessingInfoId).Distinct();
var userDocumentProcessingInfos = documentProcessingInfos.Where(x => memberNotificationDocIds.Contains(x.DocumentProcessingInfoId));
result = await PopulateAndSendNotification(emailGroup,
result,
userDocumentProcessingInfos);
}
}
catch (Exception exception)
{
var message = "Failed to PrepareAndSendNotifications.";
Log.Error(exception, message);
result.AddToFailureCount($"{message} Exception Message: {exception.Message}");
}
return result;
}
private async Task<MemberDocumentNotificationResult> PopulateAndSendNotification
(
IGrouping<(string AgreeUserId, string EmailAddress),
MemberNotificationInfo> memberNotificationInfos,
MemberDocumentNotificationResult result,
IEnumerable<DocumentProcessingInfo> documentProcessingInfos
)
{
const string errorMessage = "Failed to PopulateAndSendNotification.";
async Task<MemberDocumentNotificationResult> Unsendable(MemberDocumentNotificationResult result, string message)
{
result.AddToFailureCount($"{errorMessage} Error Message: {message}");
var updateList = memberNotificationInfos.ToList();
updateList.ForEach(x => x.NotificationStatusType = NotificationStatusTypes.CriticalFailure);
await UpdateMemberNotificationInfoByBatch(updateList);
return result;
}
try
{
var emailRequest = await CreateEmailRequest(memberNotificationInfos, documentProcessingInfos);
if (emailRequest == null)
{
throw new Exception("emailRequest returned null.");
}
if (_apiConfiguration.FeatureFlags.RestrictNotificationRecipients)
{
var pattern = new Regex(_apiConfiguration.Notifications.AllowedRecipientExpression, RegexOptions.IgnoreCase);
var unapprovedEmails = emailRequest.Recipient
.Where(s => !pattern.IsMatch(s))
.ToArray();
if (unapprovedEmails.Any())
{
var requestEmailAddresses = string.Join(", ", emailRequest.Recipient.ToArray());
var message =
$"{nameof(FeatureFlags.RestrictNotificationRecipients)} flag did not allow the email to send " +
$"because an address is outside the approved pattern of ({_apiConfiguration.Notifications.AllowedRecipientExpression}). " +
$"Emails: {requestEmailAddresses}";
return await Unsendable(result, message);
}
}
await _notificationApiService.SendEmailAsync(emailRequest);
}
catch (Exception exception)
{
Log.Error(exception, errorMessage);
return await Unsendable(result, exception.Message);
}
return result;
}
private async Task<IEnumerable<MemberNotificationInfo>> GetMemberNotificationInfos(MemberDocumentNotificationRequest request)
{
if (request.RunCriticalFailureNotifications)
{
return await _memberNotificationInfoService.GetByNotificationStatusAsync(NotificationStatusTypes.CriticalFailure);
}
var dateTimeNow = DateTime.Now.AddHours(_apiConfiguration.FileUploadCutoffHours)
.ConvertToTimeZone(_apiConfiguration.TimeZone);
return await _memberNotificationInfoService.GetSendableMemberNotificationInfosAsync(dateTimeNow,
_apiConfiguration.MaxMemberNotificationAttempts);
}
private async Task<IEnumerable<MemberNotificationInfo>> UpdateForSending(IEnumerable<MemberNotificationInfo> memberNotificationInfos)
{
foreach (var memberNotificationInfo in memberNotificationInfos)
{
if (memberNotificationInfo.NotificationStatusType == NotificationStatusTypes.Failure)
{
var contact = await _memberNotificationInfoDeterminingService.DetermineDeliveryMethod(memberNotificationInfo);
memberNotificationInfo.EmailAddress = contact.EmailAddress;
memberNotificationInfo.DeliveryMethod = contact.DeliveryMethod;
}
if (memberNotificationInfo.DeliveryMethod == DeliveryMethod.Paper)
{
memberNotificationInfo.NotificationStatusType = NotificationStatusTypes.Complete;
continue;
}
var dateTimeNow = DateTime.Now.ConvertToTimeZone(_apiConfiguration.TimeZone);
memberNotificationInfo.SentTimeStamp = dateTimeNow;
memberNotificationInfo.NotificationStatusType = NotificationStatusTypes.WaitingResponse;
}
await UpdateMemberNotificationInfoByBatch(memberNotificationInfos);
return memberNotificationInfos.Where(x => x.NotificationStatusType == NotificationStatusTypes.WaitingResponse);
}
private async Task<(IEnumerable<MemberNotificationInfo> MemberNotificationInfos,
IEnumerable<DocumentProcessingInfo> DocumentProcessingInfos)> RemoveMaliciousDocuments(IEnumerable<MemberNotificationInfo> memberNotificationInfos,
IEnumerable<DocumentProcessingInfo> documentProcessingInfos)
{
var dateTimes = memberNotificationInfos.Select(x => x.UploadDate.ConvertToTimeZone(_apiConfiguration.TimeZone)).OrderBy(x => x);
var minDateTime = dateTimes.First().AddHours(_apiConfiguration.MaliciousFileSearchPaddingHours * -1);
var maxDateTime = dateTimes.Last().AddHours(_apiConfiguration.MaliciousFileSearchPaddingHours);
var maliciousSearchFilter = new MaliciousDocumentSearchFilter()
{
MaliciousDateStart = minDateTime,
MaliciousDateEnd = maxDateTime
};
var maliciousDocuments = await _documentTypeRetrieverService.GetListOfMaliciousDocuments(maliciousSearchFilter);
if (maliciousDocuments == null)
{
return (memberNotificationInfos, documentProcessingInfos);
}
documentProcessingInfos = documentProcessingInfos.Where(documentProcessingInfo => {
return maliciousDocuments.Any(maliciousDocument =>
maliciousDocument.DocumentId != documentProcessingInfo.DocumentId);
});
memberNotificationInfos = memberNotificationInfos.Where(memberNotificationInfo => {
return documentProcessingInfos.Any(documentProcessingInfo =>
documentProcessingInfo.DocumentProcessingInfoId == memberNotificationInfo.DocumentProcessingInfoId);
});
return (memberNotificationInfos, documentProcessingInfos);
}
private async Task<EmailRequest?> CreateEmailRequest(IGrouping<(string AgreeUserId, string EmailAddress), MemberNotificationInfo> memberNotificationInfos,
IEnumerable<DocumentProcessingInfo> documentProcessingInfos)
{
var memberNotificationInfoIds = memberNotificationInfos.Select(x => x.MemberNotificationInfoId).ToList();
var entityName = memberNotificationInfos.FirstOrDefault()?.EntityName;
var documents = await CreateDocumentRows(documentProcessingInfos);
var documentCount = documentProcessingInfos.Count();
if (!documentProcessingInfos.Any() ||
string.IsNullOrWhiteSpace(entityName) ||
string.IsNullOrWhiteSpace(documents))
{
return null;
}
var notifierEmailTemplateReader = new StreamReader(_apiConfiguration.NotificationEmailTemplate);
string emailBody = notifierEmailTemplateReader.ReadToEnd();
emailBody = emailBody.Replace(EntityNameTag, entityName);
emailBody = emailBody.Replace(NumberOfDocumentsTag, documentCount.ToString());
emailBody = emailBody.Replace(DocOrDocsTag, documentCount == 1 ? "eDocument" : "eDocuments");
emailBody = emailBody.Replace(TableRowsTag, documents.ToString());
emailBody = emailBody.Replace(DigitalBankingTag, _digitalBankingConfiguration.DigitalBankingUrl);
emailBody = emailBody.Replace(MobileBankingTag, _digitalBankingConfiguration.MobileBankingUri.ToString());
var completionRequest = new MemberNotificationCompletionRequest()
{
MemberNotificationInfoIds = memberNotificationInfoIds.ToList()
};
// TODO: Update EmailRequest to Notification Library when available
return new EmailRequest()
{
Recipient = new List<string> { memberNotificationInfos.Key.EmailAddress },
Subject = $"{entityName}, view your new eDocuments from Landmark.",
Body = emailBody,
ResponseItems = new List<ResponseItem>
{
new ResponseItem()
{
ResponsePayload = JsonConvert.SerializeObject(completionRequest),
ReturnQueueConnectionConfig = new QueueConnectionConfig()
{
Url = _connectionConfiguration.HostUrl,
Username = _connectionConfiguration.UserName,
Password = _connectionConfiguration.Password
},
ReturnQueueConsumerConfig = new QueueConsumerConfig()
{
ExchangeName = _memberNotificationCompletionReceiveEndpointConfiguration.ReceiveEndpoint
}
}
}
};
}
private async Task<string> CreateDocumentRows(IEnumerable<DocumentProcessingInfo> documents)
{
var documentRowReader = new StreamReader(_apiConfiguration.NotificationEmailRowTemplate);
string documentRow = documentRowReader.ReadToEnd();
var documentRows = new StringBuilder();
foreach (var document in documents.OrderBy(x => x.DocumentBusinessName))
{
var docCodeDetail = await _documentTypeRetrieverService.GetDocumentCodeDetailByBusinessName(document.DocumentBusinessName);
documentRows.Append(documentRow.Replace(DocumentNameTag, docCodeDetail.DocumentMemberFacingName));
}
return documentRows.ToString();
}
private async Task<List<MemberNotificationInfo?>> UpdateMemberNotificationInfoByBatch(
IEnumerable<MemberNotificationInfo> memberNotificationInfos)
{
var memberNotificationInfoCount = memberNotificationInfos.Count();
var skipAmount = _apiConfiguration.BatchSkipAmount == null || _apiConfiguration.BatchSkipAmount == 0
? memberNotificationInfoCount
: _apiConfiguration.BatchSkipAmount;
var memberNotificationInfosChunks = memberNotificationInfos.Chunk(skipAmount);
var memberNotificationInfoResult = new List<MemberNotificationInfo?>();
foreach (IEnumerable<MemberNotificationInfo> memberNotificationInfoChunk in memberNotificationInfosChunks)
{
var result = await _memberNotificationInfoService.UpdateRange(memberNotificationInfoChunk);
memberNotificationInfoResult.AddRange(result);
}
return memberNotificationInfoResult;
}
}
@@ -0,0 +1,162 @@
using LCU.API.Internal.DocumentIngestor.Configurations;
using LCU.Api.Internal.DocumentIngestorLib.Extensions;
using LCU.API.Internal.DocumentIngestor.Services.MicroApiServices;
using LCU.Library.Internal.ApiResponse.Models;
using LCU.Library.Internal.DocumentIngestorModels.Constants;
using LCU.Library.Internal.DocumentIngestorModels.Entities;
using LCU.Library.Internal.DocumentIngestorModels.Enums;
using LCU.Library.Internal.DocumentIngestorModels.Models;
using LCU.Library.Internal.Exceptions;
using Newtonsoft.Json;
using Serilog;
namespace LCU.API.Internal.DocumentIngestor.Services.IngestionOperationServices;
public class MemberNotificationCompletionService : IMemberNotificationCompletionService
{
private readonly IMemberNotificationInfoService _memberNotificationInfoService;
private readonly IMemberPreferencesApiService _memberPreferencesApiService;
private readonly IProcessingLogService _processingLogService;
private readonly ApiConfiguration _apiConfiguration;
public MemberNotificationCompletionService(IMemberNotificationInfoService memberNotificationInfoService,
IMemberPreferencesApiService memberPreferencesApiService,
IProcessingLogService processingLogService,
ApiConfiguration apiConfiguration)
{
_memberNotificationInfoService = memberNotificationInfoService;
_memberPreferencesApiService = memberPreferencesApiService;
_processingLogService = processingLogService;
_apiConfiguration = apiConfiguration;
}
public async Task HandleMemberNotificationCompletion(Response returnResponse)
{
try
{
ArgumentNullException.ThrowIfNull(returnResponse);
var request = JsonConvert.DeserializeObject<MemberNotificationCompletionRequest>(returnResponse.ResponsePayload);
ArgumentNullException.ThrowIfNull(request);
await CompleteAndUpdateStatuses(request.MemberNotificationInfoIds,
returnResponse.Successful,
returnResponse.ResponseStack);
}
catch (ArgumentNullException)
{
throw;
}
catch (Exception exception)
{
throw new ServiceException("Exception was thrown in HandleMemberNotificationCompletion. " +
$"Response: {JsonConvert.SerializeObject(returnResponse)}" +
$"Exception Message: {exception.Message}");
}
}
public async Task CompleteAndUpdateStatuses(List<int> ids, bool success, List<string> logs)
{
try
{
var memberNotificationInfos = await _memberNotificationInfoService.GetListByIdAsync(ids);
var emailGroups = memberNotificationInfos.GroupBy(x => (x.AgreeUserId, x.EmailAddress));
foreach (var emailGroup in emailGroups)
{
var isUndeliverable = false;
foreach (var memberNotificationInfo in emailGroup)
{
var updatedMemberNotification = await UpdateStatusAndLog(memberNotificationInfo, success, logs);
if (updatedMemberNotification.NotificationStatusType == NotificationStatusTypes.Undeliverable)
{
isUndeliverable = true;
}
}
if (success || isUndeliverable)
{
await UpdatePreferencesEmailStatus(emailGroup.Key.AgreeUserId,
emailGroup.Key.EmailAddress,
success);
}
}
}
catch (Exception exception)
{
var message = "Failure on CompleteAndUpdateStatuses for MemberNotificationInfos.";
Log.Error(exception, message);
throw new ServiceException(message);
}
}
private async Task<MemberNotificationInfo> UpdateStatusAndLog(MemberNotificationInfo memberNotificationInfo, bool success, List<string> logs)
{
try
{
var dateTimeNow = DateTime.Now.ConvertToTimeZone(_apiConfiguration.TimeZone);
memberNotificationInfo.ResponseTimeStamp = dateTimeNow;
memberNotificationInfo = DetermineStatus(memberNotificationInfo, success);
await _memberNotificationInfoService.Update(memberNotificationInfo);
foreach (var log in logs)
{
var processingLog = new ProcessingLog(memberNotificationInfo.DocumentProcessingInfoId, DateTime.Now, log, Users.System);
await _processingLogService.SaveAsync(processingLog);
}
return memberNotificationInfo;
}
catch (Exception exception)
{
var message = "Failure on UpdateStatusAndLog for MemberNotificationInfo. " +
$"MemberNotificationId: {memberNotificationInfo.MemberNotificationInfoId}";
Log.Error(exception, message);
throw new ServiceException(message);
}
}
private MemberNotificationInfo DetermineStatus(MemberNotificationInfo memberNotificationInfo, bool success)
{
if (success)
{
memberNotificationInfo.NotificationStatusType = NotificationStatusTypes.Complete;
return memberNotificationInfo;
}
if (memberNotificationInfo.Attempts >= _apiConfiguration.MaxMemberNotificationAttempts)
{
memberNotificationInfo.NotificationStatusType = NotificationStatusTypes.Undeliverable;
return memberNotificationInfo;
}
memberNotificationInfo.NotificationStatusType = NotificationStatusTypes.Failure;
memberNotificationInfo.Attempts++;
return memberNotificationInfo;
}
private async Task UpdatePreferencesEmailStatus(string agreeUserId, string emailAddress, bool success)
{
try
{
await _memberPreferencesApiService.EmailStatusSend(agreeUserId,
emailAddress,
success);
}
catch (Exception exception)
{
var message = "Failure to UpdatePreferencesEmailStatus. Message: " + exception.Message;
if (exception.Message.Contains("NotFound"))
{
Log.Warning(exception, message);
}
else
{
throw new Exception(message);
}
}
}
}
@@ -0,0 +1,445 @@
using System.Text.RegularExpressions;
using System.Globalization;
using System.Xml;
using System.Xml.Serialization;
using Serilog;
using Newtonsoft.Json;
using LCU.Library.Internal.Exceptions;
using LCU.Library.Internal.DocumentIngestorModels.Entities;
using LCU.Library.Internal.DocumentIngestorModels.Models;
using LCU.Library.Internal.DocumentIngestorModels.Models.XmlParser;
using LCU.API.Internal.DocumentIngestor.Configurations;
using LCU.API.Internal.DocumentIngestor.Services.MicroApiServices;
using LCU.Api.Internal.DocumentIngestorLib.Extensions;
using LCU.Api.Internal.MaidBotLib.Models;
using LCU.Library.Internal.FileOperationsModels.Models.NameSettings;
using LCU.Library.Internal.FileOperationsModels.Models;
namespace LCU.API.Internal.DocumentIngestor.Services.IngestionOperationServices;
[Obsolete("06/02/2023 - Used to prepare historical documents for ingestion. No longer in use.")]
public class OsgPreprocessorService : IOsgPreprocessorService
{
private readonly IFileOperationsApiService _fileOperationsApiService;
private readonly IOsgPreprocessorInfoService _osgPreprocessorInfoService;
private readonly ApiConfiguration _apiConfiguration;
private const string AllPdfsRegexTag = "{Regex0}";
private const string _statement = "Statement";
private const string _mmSTMXT = "MM_STMXT";
private const string _document = "DOCUMENT";
private DateTime _minDocDate = new DateTime(2020,09,01);
private DateTime _maxDocDate = new DateTime(2022,08,31);
private const string AllRegex = ".*";
private const string SourceIndexExtension = ".XML";
private const string OutputIndexExtension = ".csv";
private const string PdfExtension = ".PDF";
private const string ZipExtension = ".ZIP";
private const string OsgPreprocessorRunInfoLogFileName = "OsgPreprocessorRunInfo";
private const string OsgPreprocessorRunInfoLogFileExt = "txt";
private const string _pdfPartitionDateFormat = "MMddyyyy";
private const string _osgIndexDateFormat = "yyyy/MM/dd";
private int _maxZipsPerDateFolder = 2;
public OsgPreprocessorService(IFileOperationsApiService fileOperationsApiService,
IOsgPreprocessorInfoService osgPreprocessorInfoService,
ApiConfiguration apiConfiguration)
{
_fileOperationsApiService = fileOperationsApiService;
_osgPreprocessorInfoService = osgPreprocessorInfoService;
_apiConfiguration = apiConfiguration;
}
public async Task HandleOsgPreprocessing(OsgPreprocessorRequest request)
{
try
{
ArgumentNullException.ThrowIfNull(request);
var allZipsInDirectory = await DetermineAllFilesInDirectory(request.SourceBasePath, ZipExtension);
var fileCounter = 1;
var directoryDate = request.ArchiveStartDate;
var batchRunInfoId = Guid.NewGuid();
foreach (var zipFile in allZipsInDirectory)
{
var zipFileStartTime = DateTime.Now.ConvertToTimeZone(_apiConfiguration.TimeZone);
await PrepareZipFile(zipFile, request);
var osgPreprocessorRunInfo = await HandleOsgFolder(zipFile, batchRunInfoId, zipFileStartTime, request);
if (osgPreprocessorRunInfo.HasStatements)
{
osgPreprocessorRunInfo.DirectoryDate = directoryDate.ToString(_pdfPartitionDateFormat);
await WriteOsgPreprocessorRunInfoLog(zipFile, osgPreprocessorRunInfo);
if (!osgPreprocessorRunInfo.Incomplete)
{
if (fileCounter == 1)
{
var createDateDirectoryRequest = await _fileOperationsApiService.DirectCreateDirectory(request.OutputBasePath, osgPreprocessorRunInfo.DirectoryDate);
if (!createDateDirectoryRequest.IsSuccessful)
{
var message = $"OsgPreprocessing failed. Failed to create directory ({request.OutputBasePath}{osgPreprocessorRunInfo.DirectoryDate}).";
Log.Error(message);
throw new Exception(message);
}
fileCounter++;
}
else if (fileCounter > 1 && fileCounter < _maxZipsPerDateFolder)
{
fileCounter++;
}
else
{
directoryDate = directoryDate.AddDays(1);
fileCounter = 1;
}
}
await ZipAndMoveResults(zipFile, osgPreprocessorRunInfo.FinalWorkingDirectory, osgPreprocessorRunInfo);
}
var deleteDirectoriesResult = await _fileOperationsApiService.DirectDeleteDirectory(osgPreprocessorRunInfo.UnZippedFolder, request.WorkingBasePath, 1);
if (!deleteDirectoriesResult.IsSuccessful)
{
throw new Exception($"Failure to DeleteDirectories for OSG Preprocessed. Message: {string.Join(",", deleteDirectoriesResult.MessageStack)}");
}
if (osgPreprocessorRunInfo.Incomplete)
{
Log.ForContext(Enum.GetName(typeof(PropertyEnum), PropertyEnum.Group),
Enum.GetName(typeof(GroupEnum), GroupEnum.ITDevelopment))
.Error($"There was an issue during preprocessing. The preprocessing run info has been " +
$"logged to the following zip file ({Path.GetFileNameWithoutExtension(zipFile)}_Final{ZipExtension}) " +
$"at location ({osgPreprocessorRunInfo.FinalWorkingDirectory})");
}
}
}
catch (ArgumentNullException)
{
throw;
}
catch (Exception exception)
{
const string errorMessage = "Failure during OsgPreprocessing. The OsgPreprocessor has been stopped.";
var errorMessageWithException = $"{errorMessage} Exception Message: {exception.Message}";
Log.ForContext(Enum.GetName(typeof(PropertyEnum), PropertyEnum.Group),
Enum.GetName(typeof(GroupEnum), GroupEnum.ITDevelopment))
.Error(exception, errorMessage);
throw new ServiceException(errorMessageWithException);
}
}
private async Task<OsgPreprocessorRunInfo> HandleOsgFolder(string zipFile, Guid batchRunInfoId, DateTime zipFileStartTime, OsgPreprocessorRequest request)
{
var osgPreprocessorRunInfo = new OsgPreprocessorRunInfo(request);
try
{
osgPreprocessorRunInfo.UnZippedFolder = Path.GetFileNameWithoutExtension(zipFile);
var indexFiles = await DetermineAllFilesInDirectory(osgPreprocessorRunInfo.FinalWorkingDirectory, SourceIndexExtension);
if (indexFiles?.Count() != 1)
{
throw new Exception($"Could not determine IndexFile for zip file: {request.WorkingBasePath}{zipFile}.");
}
var indexFile = indexFiles.FirstOrDefault();
if (indexFile == null)
{
throw new Exception($"IndexFile was null for zip file: {request.WorkingBasePath}{zipFile}.");
}
osgPreprocessorRunInfo.IndexFile = indexFile;
var indexFilePath = Path.GetFullPath(osgPreprocessorRunInfo.IndexFilePath).Replace(char.Parse("\\"), Path.DirectorySeparatorChar);
var records = ParseRecords(indexFilePath);
if (records == null || !records.Any())
{
osgPreprocessorRunInfo.HasStatements = false;
Log.Information($"OsgPreprocess Index file returned no statements. ZipFile: {request.WorkingBasePath}{zipFile}");
return osgPreprocessorRunInfo;
}
var indexFileGroups = records.GroupBy(x => x.ArchiveOutput.PdfIndex);
if (!indexFileGroups.Any())
{
throw new Exception("IndexFileGroups by PdfIndex returned no results.");
}
var pdfsInFolder = await DetermineAllFilesInDirectory(osgPreprocessorRunInfo.FinalWorkingDirectory, PdfExtension);
foreach (var indexFileGroup in indexFileGroups)
{
var parsingResult = await WriteParsedRecordGroup(indexFile, indexFileGroup.Key, indexFileGroup.ToList(), osgPreprocessorRunInfo.FinalWorkingDirectory, pdfsInFolder);
osgPreprocessorRunInfo.OsgPreprocessorResults.Add(parsingResult);
}
if (request.SaveXmlOutput)
{
var recordsToSave = TransformXmlDocsToInfos(batchRunInfoId, zipFile, zipFileStartTime, records).ToList();
await _osgPreprocessorInfoService.SaveListAsync(recordsToSave);
}
var anyIncomplete = osgPreprocessorRunInfo.OsgPreprocessorResults.Any(result => !result.Success);
osgPreprocessorRunInfo.Incomplete = anyIncomplete;
if (!anyIncomplete)
{
var deleteResponse = await _fileOperationsApiService.DirectDeleteFile(osgPreprocessorRunInfo.FinalWorkingDirectory, osgPreprocessorRunInfo.IndexFile);
if (!deleteResponse.IsSuccessful)
{
throw new Exception($"Failed to delete the source index file: {osgPreprocessorRunInfo.FinalWorkingDirectory}{osgPreprocessorRunInfo.IndexFile}.");
}
}
}
catch (Exception exception)
{
const string errorMessage = "Failure during OsgPreprocessing.";
Log.Error(exception, errorMessage);
var errorMessageWithException = $"{errorMessage} Exception Message: {exception.Message}";
osgPreprocessorRunInfo.Incomplete = true;
osgPreprocessorRunInfo.Messages.Add(errorMessageWithException);
}
return osgPreprocessorRunInfo;
}
private IEnumerable<XmlOsgHistoricalDocument> ParseRecords(string filePath)
{
using var xmlReader = XmlReader.Create(filePath, new XmlReaderSettings() { ConformanceLevel = ConformanceLevel.Fragment, IgnoreWhitespace = true });
var documents = SerializeHistoricalDocs(xmlReader);
var filtered = documents.Where(x => x.DocumentType == _statement &&
(DateTime.ParseExact(x.CreationDate, _osgIndexDateFormat, CultureInfo.InvariantCulture) >= _minDocDate &&
DateTime.ParseExact(x.CreationDate, _osgIndexDateFormat, CultureInfo.InvariantCulture) <= _maxDocDate));
return filtered.ToList();
}
private IEnumerable<XmlOsgHistoricalDocument> SerializeHistoricalDocs(XmlReader source)
{
var xmlSerializer = new XmlSerializer(typeof(XmlOsgHistoricalDocument));
while (!source.EOF)
{
if (source.NodeType == XmlNodeType.Element &&
source.Name == _document)
{
var sourceString = source.ReadOuterXml();
StringReader rdr = new StringReader(sourceString);
var xmlOsgHistoricalDocument = (XmlOsgHistoricalDocument)xmlSerializer.Deserialize(rdr);
xmlOsgHistoricalDocument.XmlOutput = sourceString;
yield return xmlOsgHistoricalDocument;
}
else
{
source.Read();
}
}
}
private async Task<OsgPreprocessorResult> WriteParsedRecordGroup(string indexFile,
int key,
List<XmlOsgHistoricalDocument> xmlOsgHistoricalDocuments,
string folderPath,
List<string> pdfsInFolder)
{
var result = new OsgPreprocessorResult();
try
{
var newFileName = $"{Path.GetFileNameWithoutExtension(indexFile)}.{key}";
var targetDirectory = Path.Combine(folderPath, $"{newFileName}\\");
var outputFile = result.OutputFile = Path.Combine(targetDirectory, $"{newFileName}{OutputIndexExtension}");
var doesParsedFileExist = await _fileOperationsApiService.ValidateFileExists(outputFile);
if (doesParsedFileExist)
{
var message = $"OsgPreprocessing failed. A parsed index file already exists. File: {outputFile}";
Log.Error(message);
result.AddFailureAndMessage(message);
return result;
}
var pdfRegex = new Regex(@".*\." + $"{key}.PDF", RegexOptions.IgnoreCase);
var pdfFile = pdfsInFolder.Where(x => pdfRegex.IsMatch(x)).FirstOrDefault();
if (string.IsNullOrWhiteSpace(pdfFile))
{
var message = $"OsgPreprocessing failed. A matching PDF could not be found.";
Log.Error(message);
result.AddFailureAndMessage(message);
return result;
}
var createResponse = await _fileOperationsApiService.DirectCreateDirectory(folderPath, newFileName);
if (!createResponse.IsSuccessful)
{
var message = $"OsgPreprocessing failed. Failed to create a directory. Messages: {string.Join(", ", createResponse.MessageStack)}";
Log.Error(message);
result.AddFailureAndMessage(message);
return result;
}
var moveResponse = await _fileOperationsApiService.DirectMoveFile(folderPath, pdfFile, targetDirectory);
if (!moveResponse.IsSuccessful)
{
var message = $"OsgPreprocessing failed. Failed to move pdf file ({folderPath}{pdfFile}). Messages: {string.Join(", ", moveResponse.MessageStack)}";
Log.Error(message);
result.AddFailureAndMessage(message);
return result;
}
var outputRows = xmlOsgHistoricalDocuments.Select(x =>
$"\"\"|\"{_mmSTMXT}\"|\"{x.OriginalAddress.EntityName}\"|\"{x.MemberNumberOrUserId}\"|\"{x.CreationDate.Replace("/", "")}\"|" +
$"\"{x.ArchiveOutput.StartingPage}\"|\"{x.ArchiveOutput.NumberOfPages}\"");
var writeResponse = await _fileOperationsApiService.DirectWriteFile(newFileName, OutputIndexExtension, string.Join("\n", outputRows), targetDirectory);
if (!writeResponse.IsSuccessful)
{
var message = $"OsgPreprocessing failed. Failed to write index file. Messages: {string.Join(", ", writeResponse.MessageStack)}";
Log.Error(message);
result.AddFailureAndMessage(message);
return result;
}
result.Success = true;
return result;
}
catch (Exception ex)
{
var message = $"Failure to WriteParsedRecordGroup. Exception: {ex.Message}";
Log.Error(message);
result.AddFailureAndMessage(message);
return result;
}
}
private async Task PrepareZipFile(string zipFile, OsgPreprocessorRequest request)
{
var moveZipFileRequest = await _fileOperationsApiService.DirectMoveFile(request.SourceBasePath,
zipFile,
request.WorkingBasePath);
if (!moveZipFileRequest.IsSuccessful)
{
var message = $"OsgPreprocessing failed. Failed to move zip file ({request.SourceBasePath}{zipFile}).";
Log.Error(message);
throw new Exception(message);
}
var unzipFileRequest = await _fileOperationsApiService.DirectUnzipFile(request.WorkingBasePath,
zipFile,
request.WorkingBasePath,
true);
if (!unzipFileRequest.IsSuccessful)
{
var message = $"OsgPreprocessing failed. Failed to unzip zip file ({request.WorkingBasePath}{zipFile}).";
Log.Error(message);
throw new Exception(message);
}
var deleteFileRequest = await _fileOperationsApiService.DirectDeleteFile(request.WorkingBasePath, zipFile);
if (!deleteFileRequest.IsSuccessful)
{
var message = $"OsgPreprocessing failed. Failed to delete zip file ({request.WorkingBasePath}{zipFile}).";
Log.Error(message);
throw new Exception(message);
}
}
private async Task ZipAndMoveResults(string zipFile, string finalWorkingDirectory, OsgPreprocessorRunInfo osgPreprocessorRunInfo)
{
var zipOutputName = $"{Path.GetFileNameWithoutExtension(zipFile)}_FINAL";
var zipFileName = $"{zipOutputName}{ZipExtension}";
var zipFileRequest = await _fileOperationsApiService.DirectZipDirectory(finalWorkingDirectory,
osgPreprocessorRunInfo.OsgPreprocessorRequest.WorkingBasePath,
zipFileName);
if (!zipFileRequest.IsSuccessful)
{
var message = $"OsgPreprocessing failed. Failed to zip files for source zip file: {zipFile}.";
Log.Error(message);
throw new Exception(message);
}
var moveZipRequest = await _fileOperationsApiService.DirectMoveFile(osgPreprocessorRunInfo.OsgPreprocessorRequest.WorkingBasePath,
zipFileName,
osgPreprocessorRunInfo.FinalOutputDirectory);
if (!moveZipRequest.IsSuccessful)
{
var message = $"OsgPreprocessing failed. Failed to move output zip file ({osgPreprocessorRunInfo.OsgPreprocessorRequest.WorkingBasePath}{zipFileName}).";
Log.Error(message);
throw new Exception(message);
}
if (!osgPreprocessorRunInfo.Incomplete)
{
var unzipFileRequest = await _fileOperationsApiService.DirectUnzipFile(osgPreprocessorRunInfo.FinalOutputDirectory, zipFileName, osgPreprocessorRunInfo.FinalOutputDirectory, false);
if(!unzipFileRequest.IsSuccessful)
{
var message = $"OsgPreprocessing failed. Failed to unzip file ({osgPreprocessorRunInfo.FinalOutputDirectory}).";
Log.Error(message);
throw new Exception(message);
}
}
}
private async Task<List<string>> DetermineAllFilesInDirectory(string directory, string extension)
{
var dynamicSettingsForAllPdfs = new DynamicNameSetting()
{
RegexSettings = new List<RegexSetting>
{
new RegexSetting
{
Tag = AllPdfsRegexTag,
Regex = AllRegex
}
}
};
var dynamicFileListRequest = new DynamicFileListRequest
{
Path = directory,
Filename = $"{AllPdfsRegexTag}{extension}",
DynamicFileSettings = dynamicSettingsForAllPdfs
};
var listOfPdfs = await _fileOperationsApiService.BuildDynamicFileList(dynamicFileListRequest);
return listOfPdfs.ToList();
}
private async Task WriteOsgPreprocessorRunInfoLog(string zipFile, OsgPreprocessorRunInfo osgPreprocessorRunInfo)
{
var serializedOsgPreprocessorRunInfo = JsonConvert.SerializeObject(osgPreprocessorRunInfo, Newtonsoft.Json.Formatting.Indented);
var writeResponse = await _fileOperationsApiService.DirectWriteFile($"{OsgPreprocessorRunInfoLogFileName}_{zipFile}",
OsgPreprocessorRunInfoLogFileExt,
serializedOsgPreprocessorRunInfo,
osgPreprocessorRunInfo.FinalWorkingDirectory);
if (!writeResponse.IsSuccessful)
{
throw new Exception("Failure to save Osg Preprocessor Run Info Log.");
}
}
private IEnumerable<OsgPreprocessorInfo> TransformXmlDocsToInfos(Guid batchRunInfoId,
string zipFileName,
DateTime zipFileStartTime,
IEnumerable<XmlOsgHistoricalDocument> records)
{
return records.Select(record =>
new OsgPreprocessorInfo()
{
BatchRunInfoId = batchRunInfoId,
ZipFileName = zipFileName,
ZipFileStartTime = zipFileStartTime,
ZipFileEndTime = DateTime.Now.ConvertToTimeZone(_apiConfiguration.TimeZone),
MemberNumber = record.MemberNumberOrUserId,
EntityName = record.OriginalAddress.EntityName,
DocumentType = record.DocumentType,
CreatedDate = DateTime.ParseExact(record.CreationDate, _osgIndexDateFormat, CultureInfo.InvariantCulture),
StartingPage = record.ArchiveOutput.StartingPage,
NumberOfPages = record.ArchiveOutput.NumberOfPages,
XmlOutput = record.XmlOutput
}
);
}
}
@@ -0,0 +1,61 @@
using Newtonsoft.Json;
using LCU.Library.Internal.ApiResponse.Services;
using LCU.Library.Internal.Exceptions;
using LCU.Library.Internal.ApiResponse.Models;
using LCU.Library.Internal.DocumentIngestorModels.Models;
namespace LCU.API.Internal.DocumentIngestor.Services.IngestionOperationServices;
public class PdfPartitionerCompletionService : IPdfPartitionerCompletionService
{
private readonly IPdfPartitionerRunInfoService _pdfPartitionerRunInfoService;
private readonly IPdfPartitionerLogService _pdfPartitionerLogService;
public PdfPartitionerCompletionService(IPdfPartitionerRunInfoService pdfPartitionerRunInfoService,
IPdfPartitionerLogService pdfPartitionerLogService)
{
_pdfPartitionerRunInfoService = pdfPartitionerRunInfoService;
_pdfPartitionerLogService = pdfPartitionerLogService;
}
public async Task HandlePdfPartitionerCompletion(Response response)
{
try
{
ArgumentNullException.ThrowIfNull(response);
var pdfPartitionerResult = JsonConvert.DeserializeObject<PdfPartitionerResult>(response.ResponsePayload);
ArgumentNullException.ThrowIfNull(pdfPartitionerResult);
var wasSuccessful = !pdfPartitionerResult.FailedToQueueToParse;
var pdfPartitionLogs = new List<string>();
if (wasSuccessful)
{
wasSuccessful = response.Successful;
if(response.ResponseStack != null)
{
pdfPartitionLogs.AddRange(response.ResponseStack);
}
}
pdfPartitionLogs.AddRange(pdfPartitionerResult.MessageStack);
var pdfPartitionerRunInfo = await _pdfPartitionerRunInfoService.IncreaseCountsAndUpdate(pdfPartitionerResult.PdfPartitionerRunInfoId, wasSuccessful);
await _pdfPartitionerLogService.SaveListAsync(pdfPartitionerRunInfo.PdfPartitionerRunInfoId, pdfPartitionLogs);
if (pdfPartitionerRunInfo.PdfPartitionerCompleted)
{
var pdfPartitionerLogs = await _pdfPartitionerLogService.GetByPdfPartitionerRunInfoIdAsync(pdfPartitionerRunInfo.PdfPartitionerRunInfoId);
var messageStack = pdfPartitionerLogs.Select(pdfPartitionerLog => pdfPartitionerLog.LogMessage).ToList();
await QueueReturnResponse.QueueResponse(pdfPartitionerResult.PdfPartitionerRequest, pdfPartitionerRunInfo.WasSuccessful, messageStack);
}
}
catch (ArgumentNullException)
{
throw;
}
catch (Exception exception)
{
throw new ServiceException($"Exception was thrown in HandlePdfPartitionerCompletion. " +
$"Response: {JsonConvert.SerializeObject(response)}" +
$"Exception Message: {exception.Message}");
}
}
}
@@ -0,0 +1,242 @@
using System.Text.RegularExpressions;
using Serilog;
using Newtonsoft.Json;
using LCU.Library.Internal.ApiResponse.Services;
using LCU.Library.Internal.Exceptions;
using LCU.Library.Internal.DocumentIngestorModels.Models;
using LCU.API.Internal.DocumentIngestor.Services.MicroApiServices;
using LCU.API.Internal.DocumentIngestor.Services.PdfParserServices;
using LCU.Library.Internal.DocumentIngestorModels.Entities;
using LCU.Library.Internal.ApiResponse.Models;
using LCU.Library.Internal.LcuMassTransitModels;
using LCU.API.Internal.DocumentIngestor.Configurations;
using LCU.API.Internal.DocumentIngestor.Services.QueueServices;
using LCU.Library.Internal.FileOperationsModels.Models;
using LCU.Library.Internal.FileOperationsModels.Models.NameSettings;
using LCU.Library.Internal.FileOperationsModels.Shared;
using LCU.Library.Internal.LCUServiceFactory.Services;
namespace LCU.API.Internal.DocumentIngestor.Services.IngestionOperationServices;
public class PdfPartitionerService : IPdfPartitionerService
{
private readonly ConnectionConfiguration _connectionConfiguration;
private readonly ReceiveEndpointConfiguration _pdfPartitionerCompletionEndpoint;
private readonly IPdfPartitionerRunInfoService _pdfPartitionerRunInfoService;
private readonly IFileOperationsApiService _fileOperationsApiService;
private readonly IQueuePdfPartitionerCompletionService _queuePdfPartitionerCompletionService;
private readonly ILcuServiceFactory<IPdfParserService> _pdfParserServiceFactory;
private const string AllPdfsRegexTag = "{Regex0}";
private const string AllPdfsRegex = ".*";
public PdfPartitionerService(ConnectionConfiguration connectionConfiguration,
ReceiveEndpointConfigurations receiveEndpointConfigurations,
IPdfPartitionerRunInfoService pdfPartitionerRunInfoService,
IFileOperationsApiService fileOperationsApiService,
IQueuePdfPartitionerCompletionService queuePdfPartitionerCompletionService,
ILcuServiceFactory<IPdfParserService> pdfParserServiceFactory)
{
_connectionConfiguration = connectionConfiguration;
_pdfPartitionerCompletionEndpoint = receiveEndpointConfigurations.PdfPartitionerCompletionEndpoint;
_pdfPartitionerRunInfoService = pdfPartitionerRunInfoService;
_fileOperationsApiService = fileOperationsApiService;
_queuePdfPartitionerCompletionService = queuePdfPartitionerCompletionService;
_pdfParserServiceFactory = pdfParserServiceFactory;
}
public async Task HandlePdfPartition(PdfPartitionerRequest request)
{
try
{
ArgumentNullException.ThrowIfNull(request);
var dynamicDirectoryListRequest = new DynamicDirectoryListRequest
{
Path = request.FinalRootBasePath,
DynamicDirectoryName = request.ParentDynamicDirectory,
DynamicFileSettings = request.ParentDynamicDirectorySettings
};
var dynamicDirectories = await _fileOperationsApiService.BuildDynamicDirectoryList(dynamicDirectoryListRequest);
var pdfPartitionerRunInfo = new PdfPartitionerRunInfo(DateTime.Now,
request.PdfParserType,
request.FinalRootBasePath,
dynamicDirectories.ToList().Count);
pdfPartitionerRunInfo = await _pdfPartitionerRunInfoService.SaveAsync(pdfPartitionerRunInfo);
foreach (var directory in dynamicDirectories)
{
await PartitionPdf(request, directory, pdfPartitionerRunInfo);
}
}
catch (ArgumentNullException)
{
throw;
}
catch (Exception exception)
{
const string errorMessage = "Failure during HandlePdfPartition.";
var errorMessageWithException = $"{errorMessage} Exception Message: {exception.Message}";
Log.Error(exception, errorMessage);
await QueueReturnResponse.QueueResponse(request, false, new List<string> { errorMessageWithException });
throw new ServiceException(errorMessageWithException);
}
}
public async Task PartitionPdf(PdfPartitionerRequest request, string directory, PdfPartitionerRunInfo pdfPartitionerRunInfo)
{
var result = new PdfPartitionerResult(pdfPartitionerRunInfo.PdfPartitionerRunInfoId, request);
try
{
var ingestionRootPath = DetermineIngestionRootPath(request, directory);
var indexFile = await DetermineIndexFile(request, ingestionRootPath);
var allPdfs = await DetermineAllPdfsInDirectory(ingestionRootPath);
var mainPdf = DeterminMainPdfInDirectory(request, allPdfs);
var batchPdfParserRequest = await PdfParserSwitch(request, ingestionRootPath, indexFile, mainPdf);
var canSkipParsing = CanSkipPdfParse(request, allPdfs, batchPdfParserRequest);
if (canSkipParsing)
{
result.MessageStack.Add($"Skipping Pdf Partition for Pdf {mainPdf} at the path {ingestionRootPath}");
await _queuePdfPartitionerCompletionService.QueuePdfPartitionerCompletionRequest(result);
return;
}
batchPdfParserRequest.RootPath = ingestionRootPath;
batchPdfParserRequest.PdfName = mainPdf;
batchPdfParserRequest = SetUpPdfPartitionerApiResponse(result, batchPdfParserRequest);
await _fileOperationsApiService.QueueBatchPdfParser(batchPdfParserRequest);
}
catch (Exception exception)
{
Log.Error(exception.Message);
result.AddFailureAndMessage(exception.Message);
await _queuePdfPartitionerCompletionService.QueuePdfPartitionerCompletionRequest(result);
}
}
public string DetermineIngestionRootPath(PdfPartitionerRequest request, string directory)
{
var subDirectoryPath = request.SubDirectoryPath.EndsWith("\\") || string.IsNullOrWhiteSpace(request.SubDirectoryPath) ?
request.SubDirectoryPath :
$"{request.SubDirectoryPath}\\";
var formattedDirectory = directory.EndsWith("\\") ? directory : $"{directory}\\";
var ingestionRootPath = request.FinalRootBasePath + formattedDirectory + subDirectoryPath;
return ingestionRootPath;
}
public async Task<string> DetermineIndexFile(PdfPartitionerRequest request, string path)
{
var dynamicFileListRequest = new DynamicFileListRequest
{
Path = path,
Filename = request.DynamicIndexFileName,
DynamicFileSettings = request.DynamicFileSettings
};
return await _fileOperationsApiService.BuildDynamicFile(dynamicFileListRequest);
}
public async Task<List<string>> DetermineAllPdfsInDirectory(string directory)
{
var dynamicSettingsForAllPdfs = new DynamicNameSetting()
{
RegexSettings = new List<RegexSetting>
{
new RegexSetting
{
Tag = AllPdfsRegexTag,
Regex = AllPdfsRegex
}
}
};
var dynamicFileListRequest = new DynamicFileListRequest
{
Path = directory,
Filename = $"{AllPdfsRegexTag}.(?i:pdf)",
DynamicFileSettings = dynamicSettingsForAllPdfs
};
var listOfPdfs = await _fileOperationsApiService.BuildDynamicFileList(dynamicFileListRequest);
return listOfPdfs.ToList();
}
public string DeterminMainPdfInDirectory(PdfPartitionerRequest request, List<string> allPdfsInDirectory)
{
var regex = new Regex(request.DynamicPdfName);
if (request?.DynamicPdfSettings?.RegexSettings != null)
{
regex = RegexNameHandler.BuildRegexName(request.DynamicPdfName, request.DynamicPdfSettings);
}
var mainPdf = allPdfsInDirectory.Where(pdf => regex.IsMatch(pdf)).ToList();
if (mainPdf.Count > 1)
{
var message = $"There were more than one pdf that met the dynamic pdf settings. " +
$"DynamicPdfName:{request.DynamicPdfName}," +
$"DynamicPdfSettings:{JsonConvert.SerializeObject(request.DynamicPdfSettings)}";
throw new Exception(message);
}
var pdf = mainPdf.FirstOrDefault();
if (pdf == null)
{
var message = $"There were no pdfs that met the dynamic pdf settings. " +
$"DynamicPdfName:{request.DynamicPdfName}," +
$"DynamicPdfSettings:{JsonConvert.SerializeObject(request.DynamicPdfSettings)}";
throw new Exception(message);
}
return pdf;
}
public async Task<BatchParsePdfRequest> PdfParserSwitch(PdfPartitionerRequest request, string rootPath, string indexFile, string pdfName)
{
var service = _pdfParserServiceFactory.GetService(request.PdfParserType);
if (service is null)
{
throw new NotImplementedException();
}
return await service.DetermineBatchParsePdfRequest(rootPath, indexFile, pdfName);
}
public bool CanSkipPdfParse(PdfPartitionerRequest request, List<string> allPdfs, BatchParsePdfRequest batchParsePdfRequest)
{
if (!request.SkipParseIfCountsMatch)
{
return false;
}
var countsMatch = allPdfs.Count == batchParsePdfRequest.BatchPdfs.Count + 1;
return countsMatch && request.SkipParseIfCountsMatch;
}
public BatchParsePdfRequest SetUpPdfPartitionerApiResponse(PdfPartitionerResult pdfPartitionerResult,
BatchParsePdfRequest batchParsePdfRequest)
{
var serializedResponse = JsonConvert.SerializeObject(pdfPartitionerResult);
var newReturnItem = new ResponseItem
{
ResponsePayload = serializedResponse,
ReturnQueueConnectionConfig = new QueueConnectionConfig
{
Url = _connectionConfiguration.HostUrl,
Username = _connectionConfiguration.UserName,
Password = _connectionConfiguration.Password
},
ReturnQueueConsumerConfig = new QueueConsumerConfig
{
ExchangeName = _pdfPartitionerCompletionEndpoint.ReceiveEndpoint
}
};
batchParsePdfRequest.ResponseItems = new List<ResponseItem>
{
newReturnItem
};
return batchParsePdfRequest;
}
}
@@ -0,0 +1,103 @@
using LCU.Api.Internal.MaidBotLib.Models;
using LCU.API.Internal.DocumentIngestor.Configurations;
using LCU.API.Internal.DocumentIngestor.Services.MicroApiServices;
using LCU.Library.Internal.ApiResponse.Services;
using LCU.Library.Internal.DocumentIngestorModels.Entities;
using LCU.Library.Internal.DocumentIngestorModels.Enums;
using LCU.Library.Internal.DocumentIngestorModels.Requests;
using LCU.Library.Internal.Exceptions;
using LCU.Library.Internal.NotificationsModels.Models;
using Microsoft.OpenApi.Extensions;
using Serilog;
using System.Text;
namespace LCU.API.Internal.DocumentIngestor.Services.IngestionOperationServices;
public class ProcessStatusTypeEmailService : IProcessStatusTypeEmailService
{
private readonly INotificationApiService _notificationApiService;
private readonly IDocumentProcessingInfoService _documentProcessingInfoService;
private readonly ApiConfiguration _apiConfiguration;
public ProcessStatusTypeEmailService(INotificationApiService notificationApiService, IDocumentProcessingInfoService documentProcessingInfoService,
ApiConfiguration apiConfiguration)
{
_notificationApiService = notificationApiService;
_documentProcessingInfoService = documentProcessingInfoService;
_apiConfiguration = apiConfiguration;
}
public async Task HandleProcessStatusTypeEmailRequest(ProcessStatusTypeEmailRequest request)
{
try
{
var processStatusTypeInfoResults = await _documentProcessingInfoService.GetByProcessStatusTypeListAsync(request.ProcessStatusTypes, false);
if (processStatusTypeInfoResults.Any())
{
await SendProcessStatusEmailAsync(processStatusTypeInfoResults, request);
}
var message = $"Handle Process Status Type Email Request of {string.Join(", ", request.ProcessStatusTypes)} statuses completed successfully.";
await QueueReturnResponse.QueueResponse(request, true, new List<string> { message });
}
catch (ServiceException exception)
{
Log.ForContext(Enum.GetName(typeof(PropertyEnum), PropertyEnum.Group),
Enum.GetName(typeof(GroupEnum), GroupEnum.ITDevelopment)).Error(exception, "Service Exception");
}
catch (Exception exception)
{
Log.ForContext(Enum.GetName(typeof(PropertyEnum), PropertyEnum.Group),
Enum.GetName(typeof(GroupEnum), GroupEnum.ITDevelopment)).Error(exception, "Unexpected Exception");
}
}
public async Task SendProcessStatusEmailAsync(IEnumerable<DocumentProcessingInfo> listDocumentProcessingInfo, ProcessStatusTypeEmailRequest request)
{
if (request != null)
{
var subject = CheckDataMappingFailures(request.ProcessStatusTypes) ? "Data mapping errors" :
string.Join(", ", request.ProcessStatusTypes.Select(x => x.GetDisplayName())) + " statuses";
var bodyMessage = GetEmailBodyMessage(request.ProcessStatusTypes, listDocumentProcessingInfo.ToList());
var emailRequest = new EmailRequest
{
Subject = $"Document Manager: {subject} found",
Recipient = request.EmailAddresses,
Body = GetLowerEnvironmentHtml() + "This is an auto generated message.<br>" + "<br>" + bodyMessage.ToString() +
request.ProcessStatusTypeEmailUrl
};
await _notificationApiService.SendEmailAsync(emailRequest);
}
}
private bool CheckDataMappingFailures(List<ProcessStatusTypes> statusTypes)
{
return statusTypes.Count == 1 && statusTypes.Contains(ProcessStatusTypes.DataMappingFailure);
}
private string GetEmailBodyMessage(List<ProcessStatusTypes> statusTypes, List<DocumentProcessingInfo> processingResults)
{
var bodyMessage = new StringBuilder();
foreach (var statusType in statusTypes)
{
var processCount = processingResults.Where(x => x.StatusType.Equals(statusType)).Count();
var typeDescription = CheckDataMappingFailures(statusTypes) ? " mapping errors" : $"{statusType} status.";
bodyMessage.Append($"There are {processCount} documents found that have {typeDescription} <br>" + "<br>");
}
return bodyMessage.ToString();
}
private string GetLowerEnvironmentHtml()
{
var message = string.Empty;
var env = Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT");
if (!string.IsNullOrEmpty(env) && !env.Equals("Production", StringComparison.Ordinal))
{
message = $"<span style='background-color: #FF0000; color: #FFFFFF'><strong>This is from the {env.ToUpper()} Environment</strong></span><br><br>";
}
return message;
}
}
@@ -0,0 +1,259 @@
using LCU.Api.Internal.MaidBotLib.Models;
using LCU.API.Internal.DocumentIngestor.Services.MemberNotificationInfoDeterminingServices;
using LCU.API.Internal.DocumentIngestor.Services.MicroApiServices;
using LCU.API.Internal.DocumentIngestor.Services.QueueServices;
using LCU.API.Internal.DocumentIngestorLib.Exceptions;
using LCU.Library.Internal.DocumentIngestorModels.Models;
using LCU.Library.Internal.DocumentIngestorModels.Models.QueueResult;
using LCU.Library.Internal.Exceptions;
using MassTransit;
using Newtonsoft.Json;
using Serilog;
using Response = LCU.Library.Internal.ApiResponse.Models.Response;
namespace LCU.API.Internal.DocumentIngestor.Services.IngestionOperationServices;
public class UploadCompletionService : IUploadCompletionService
{
private readonly IQueueCompletionService _queueCompletionService;
private readonly IFileOperationsApiService _fileOperationsApiService;
private readonly IDocumentProcessingInfoService _documentProcessingInfoService;
private readonly IMemberNotificationInfoDeterminingService _memberNotificationInfoDeterminingService;
private readonly IQueueGatherMemberNotificationInfoService _queueGatherMemberNotificationInfoService;
private const string _documentIdKey = "DocumentId";
public UploadCompletionService(IQueueCompletionService queueCompletionService,
IFileOperationsApiService fileOperationsApiService,
IDocumentProcessingInfoService documentProcessingInfoService,
IMemberNotificationInfoDeterminingService memberNotificationInfoDeterminingService,
IQueueGatherMemberNotificationInfoService queueGatherMemberNotificationInfoService)
{
_queueCompletionService = queueCompletionService;
_fileOperationsApiService = fileOperationsApiService;
_documentProcessingInfoService = documentProcessingInfoService;
_memberNotificationInfoDeterminingService = memberNotificationInfoDeterminingService;
_queueGatherMemberNotificationInfoService = queueGatherMemberNotificationInfoService;
}
public async Task HandleUploadCompletion(List<Response> responses)
{
try
{
ArgumentNullException.ThrowIfNull(responses);
var batchUploadCompletion = DeserializeAndSortResponses(responses);
if (batchUploadCompletion.SuccessfulUploadCompletions.Count() > 0)
{
batchUploadCompletion = await HandleSuccessfulCompletionRequests(batchUploadCompletion);
}
if (batchUploadCompletion.UnsuccessfulUploadCompletions.Count() > 0)
{
batchUploadCompletion = await HandleUnsuccessfulCompletionRequests(batchUploadCompletion);
}
}
catch (ArgumentNullException)
{
throw;
}
catch (ArchiveException)
{
throw;
}
catch (Exception exception)
{
throw new ServiceException("Exception was thrown in HandleCompletion. " +
$"Exception Message: {exception.Message}");
}
}
public BatchUploadCompletionRequest DeserializeAndSortResponses(List<Response> responses)
{
var batchUploadCompletion = new BatchUploadCompletionRequest()
{
SuccessfulUploadCompletions = new List<UploadCompletionRequest>(),
UnsuccessfulUploadCompletions = new List<UploadCompletionRequest>()
};
foreach (var response in responses)
{
var request = JsonConvert.DeserializeObject<CompletionRequest>(response.ResponsePayload);
if (response.Successful)
{
batchUploadCompletion.SuccessfulUploadCompletions.Add(new UploadCompletionRequest(request, response));
}
else
{
batchUploadCompletion.UnsuccessfulUploadCompletions.Add(new UploadCompletionRequest(request, response));
}
}
return batchUploadCompletion;
}
public async Task<BatchUploadCompletionRequest> HandleSuccessfulCompletionRequests(BatchUploadCompletionRequest batchUploadCompletionRequest)
{
batchUploadCompletionRequest.DocumentProcessingInfos = await _documentProcessingInfoService.GetListByIdAsync(batchUploadCompletionRequest.SuccessfulUploadCompletions
.Select(r => r.CompletionRequest.ProcessingRequest.DocumentProcessingInfoId)
.ToList());
batchUploadCompletionRequest = AddDocumentIdToDocumentProcessingInfo(batchUploadCompletionRequest);
batchUploadCompletionRequest.DocumentProcessingInfos = await _documentProcessingInfoService.UpdateRangeAsync(batchUploadCompletionRequest.DocumentProcessingInfos);
return await CreateAndQueueGatherMemberNotificationInfoRequest(batchUploadCompletionRequest);
}
public BatchUploadCompletionRequest AddDocumentIdToDocumentProcessingInfo(BatchUploadCompletionRequest batchUploadCompletionRequest)
{
foreach (var info in batchUploadCompletionRequest.DocumentProcessingInfos)
{
var request = batchUploadCompletionRequest.SuccessfulUploadCompletions.Where(r =>
r.CompletionRequest.ProcessingRequest.DocumentProcessingInfoId == info.DocumentProcessingInfoId).FirstOrDefault();
ArgumentNullException.ThrowIfNull(request);
var idResponse = "";
if (!request.Response.ReturnCollection.TryGetValue(_documentIdKey, out idResponse))
{
batchUploadCompletionRequest.UnsuccessfulUploadCompletions.Add(request);
batchUploadCompletionRequest.SuccessfulUploadCompletions.Remove(request);
continue;
}
Guid documentId;
if (!Guid.TryParse(idResponse, out documentId))
{
batchUploadCompletionRequest.UnsuccessfulUploadCompletions.Add(request);
batchUploadCompletionRequest.SuccessfulUploadCompletions.Remove(request);
continue;
}
info.DocumentId = documentId;
}
return batchUploadCompletionRequest;
}
public async Task<BatchUploadCompletionRequest> CreateAndQueueGatherMemberNotificationInfoRequest(BatchUploadCompletionRequest request)
{
var tasks = request.SuccessfulUploadCompletions.Select(r => QueueGatherMemberNotificationInfoRequest(
new GatherMemberNotificationInfoRequest()
{
CompletionRequest = r.CompletionRequest
}, r));
var results = await Task.WhenAll(tasks);
foreach (var result in results)
{
if (!result.Success)
{
request.UnsuccessfulUploadCompletions.Add(result.OriginalRequest);
}
}
return request;
}
public async Task<QueueGatherMemberNotificationInfoResult> QueueGatherMemberNotificationInfoRequest(GatherMemberNotificationInfoRequest GatherMemberNotificationInfoRequest,
UploadCompletionRequest uploadCompletionRequest)
{
try
{
await _queueGatherMemberNotificationInfoService.QueueGatherMemberNotificationInfoRequest(GatherMemberNotificationInfoRequest);
return new QueueGatherMemberNotificationInfoResult
{
Success = true,
OriginalRequest = uploadCompletionRequest
};
}
catch (Exception ex)
{
var errorMessage = $"Exception during CreateAndQueueGatherMemberNotificationInfoRequest. ErrorMessage: {ex.Message}";
Log.Warning(errorMessage);
return new QueueGatherMemberNotificationInfoResult
{
Success = false,
OriginalRequest = uploadCompletionRequest
};
}
}
public async Task<BatchUploadCompletionRequest> HandleUnsuccessfulCompletionRequests(BatchUploadCompletionRequest batchUploadCompletionRequest)
{
foreach (var request in batchUploadCompletionRequest.UnsuccessfulUploadCompletions)
{
await UnArchiveDocument(request.CompletionRequest.ProcessingRequest);
request.CompletionRequest.ProcessResult = ProcessResult.ProcessCriticalFailure(request.CompletionRequest.ProcessResult,
request.Response.ResponseStack);
}
await CreateAndQueueCompletionRequest(batchUploadCompletionRequest.UnsuccessfulUploadCompletions);
return batchUploadCompletionRequest;
}
private async Task UnArchiveDocument(ProcessingRequest request)
{
try
{
var response = await _fileOperationsApiService.DirectMoveFile(request.IngestorRequest.ArchivePath,
request.DocumentFileName,
request.IngestorRequest.RootPath);
if (!response.IsSuccessful)
{
throw new Exception(string.Join("; ", response.MessageStack.ToArray()));
}
}
catch (Exception exception)
{
throw new ArchiveException($"Failure to unarchive document. Exception Message: {exception.Message}");
}
}
private async Task CreateAndQueueCompletionRequest(List<UploadCompletionRequest> requests)
{
var tasks = requests.Select(r => QueueCompletionRequest(r.CompletionRequest));
var results = await Task.WhenAll(tasks);
var message = string.Join(Environment.NewLine,
results
.Where(r => !r.Success)
.Select(r => $"Failed to Queue Completion after UploadCompletion Failure. " +
$"DocumentProcessingInfoId: {r.CompletionRequest.ProcessingRequest.DocumentProcessingInfoId}, " +
$"DocumentFileName: {r.CompletionRequest.ProcessingRequest.DocumentFileName}."));
if (!string.IsNullOrEmpty(message))
{
Log.ForContext(Enum.GetName(typeof(PropertyEnum), PropertyEnum.Group),
Enum.GetName(typeof(GroupEnum), GroupEnum.ITDevelopment))
.Error(new Exception(), message);
}
}
private async Task<QueueCompletionResult> QueueCompletionRequest(CompletionRequest request)
{
try
{
await _queueCompletionService.QueueCompletionRequest(request.ProcessingRequest, request.ProcessResult);
return new QueueCompletionResult
{
Success = true,
CompletionRequest = request
};
}
catch (Exception ex)
{
var errorMessage = $"Exception during CreateAndQueueCompletionRequest. ErrorMessage: {ex.Message}";
Log.Warning(errorMessage);
return new QueueCompletionResult
{
Success = false,
CompletionRequest = request
};
}
}
}
@@ -0,0 +1,132 @@
using System.Data.SqlClient;
using Serilog;
using LCU.Library.Internal.DocumentIngestorModels.Entities;
using LCU.Library.Internal.Exceptions;
using LCU.API.Internal.DocumentIngestorLib.Repositories;
using LCU.Library.Internal.DocumentIngestorModels.Enums;
namespace LCU.API.Internal.DocumentIngestor.Services;
public class IngestorRunInfoService : IIngestorRunInfoService
{
private readonly IIngestorRunInfoRepository _ingestorRunInfoRepository;
private readonly IUnitOfWork _unitOfWork;
public IngestorRunInfoService(IUnitOfWork unitOfWork,
IIngestorRunInfoRepository ingestorRunInfoRepository)
{
_ingestorRunInfoRepository = ingestorRunInfoRepository;
_unitOfWork = unitOfWork;
}
public async Task<IngestorRunInfo> GetByIdAsync(Guid ingestorRunInfoId)
{
try
{
var ingestorRunInfo = await _ingestorRunInfoRepository.FindByIdAsync(ingestorRunInfoId);
return ingestorRunInfo;
}
catch (Exception exception)
{
Log.Error(exception, $"Failure retrieving IngestorRunInfo with IngestorRunInfoId {ingestorRunInfoId}.");
throw new ServiceException($"Failure retrieving IngestorRunInfo with IngestorRunInfoId {ingestorRunInfoId}.");
}
}
public async Task<IEnumerable<IngestorRunInfo>> GetListByIdAsync(List<Guid> ingestorRunInfoIds)
{
try
{
var ingestorRunInfos = await _ingestorRunInfoRepository.FindListByIdAsync(ingestorRunInfoIds);
return ingestorRunInfos;
}
catch (Exception exception)
{
Log.Error(exception, $"Failure retrieving List of IngestorRunInfo with IngestorRunInfoIds: {string.Join(", ", ingestorRunInfoIds)}.");
throw new ServiceException($"Failure retrieving List of IngestorRunInfo with IngestorRunInfoIds: {string.Join(", ", ingestorRunInfoIds)}.");
}
}
public IEnumerable<IngestorRunInfo> FindByIngestionEndDateNotArchived(DateTime ingestionEndDate)
{
try
{
var ingestorRunInfo = _ingestorRunInfoRepository.FindByIngestionEndDateNotArchived(ingestionEndDate);
return ingestorRunInfo;
}
catch (Exception exception)
{
Log.Error(exception, $"Failure retrieving IngestorRunInfo with IngestionEndDate {ingestionEndDate}.");
throw new ServiceException($"Failure retrieving IngestorRunInfo with IngestionEndDate {ingestionEndDate}.");
}
}
public async Task<IngestorRunInfo> SaveAsync(IngestorRunInfo ingestorRunInfo)
{
try
{
var result = await _ingestorRunInfoRepository.AddAsync(ingestorRunInfo);
await _unitOfWork.CompleteAsync();
return result;
}
catch (Exception exception)
{
Log.Error(exception, $"Failure saving IngestorRunInfo.");
throw new ServiceException($"Failure saving IngestorRunInfo.");
}
}
public async Task<IngestorRunInfo> Update(IngestorRunInfo ingestorRunInfo)
{
try
{
var existing = await GetByIdAsync(ingestorRunInfo.IngestorRunInfoId);
ArgumentNullException.ThrowIfNull(existing);
existing.StartTimeStamp = ingestorRunInfo.StartTimeStamp;
existing.EndTimeStamp = ingestorRunInfo.EndTimeStamp;
existing.IngestorType = ingestorRunInfo.IngestorType;
existing.IndexFilePath = ingestorRunInfo.IndexFilePath;
existing.IndexFileName = ingestorRunInfo.IndexFileName;
existing.IndexTotal = ingestorRunInfo.IndexTotal;
existing.ArchiveTimeStamp = ingestorRunInfo.ArchiveTimeStamp;
existing.IngestionRunCount = ingestorRunInfo.IngestionRunCount;
_ingestorRunInfoRepository.Update(existing);
_unitOfWork.Complete();
return existing;
}
catch (SqlException sqlException)
{
Log.Error(sqlException, $"SQL Failure updating IngestorRunInfo. IngestorRunInfo: {ingestorRunInfo.IngestorRunInfoId}");
throw new ServiceException($"SQL Failure updating IngestorRunInfo. IngestorRunInfo: {ingestorRunInfo.IngestorRunInfoId}");
}
catch (Exception exception)
{
Log.Error(exception, $"Failure updating IngestorRunInfo. IngestorRunInfo: {ingestorRunInfo.IngestorRunInfoId}");
throw new ServiceException($"Failure updating IngestorRunInfo. IngestorRunInfo: {ingestorRunInfo.IngestorRunInfoId}");
}
}
public async Task Delete(Guid ingestorRunInfoId)
{
try
{
var ingestorRunInfo = await _ingestorRunInfoRepository.FindByIdAsync(ingestorRunInfoId);
ArgumentNullException.ThrowIfNull(ingestorRunInfo);
_ingestorRunInfoRepository.Remove(ingestorRunInfo);
_unitOfWork.Complete();
}
catch (SqlException sqlException)
{
Log.Error(sqlException, $"SQL failure removing IngestorRunInfo. IngestorRunInfoId: {ingestorRunInfoId}");
throw new ServiceException($"SQL failure removing IngestorRunInfo. IngestorRunInfoId: {ingestorRunInfoId}");
}
catch (Exception exception)
{
Log.Error(exception, $"Failure removing IngestorRunInfo. IngestorRunInfoId: {ingestorRunInfoId}");
throw new ServiceException($"Failure removing IngestorRunInfo. IngestorRunInfoId: {ingestorRunInfoId}");
}
}
}
@@ -0,0 +1,78 @@
using LCU.Api.Internal.DocumentIngestorLib.Extensions;
using LCU.API.Internal.DocumentIngestor.Configurations;
using LCU.API.Internal.DocumentIngestor.Services.MicroApiServices;
using LCU.Library.Internal.DocumentIngestorModels.Entities;
using LCU.Library.Internal.DocumentIngestorModels.Enums;
using LCU.Library.Internal.Exceptions;
using Serilog;
namespace LCU.API.Internal.DocumentIngestor.Services.MemberNotificationInfoDeterminingServices;
public class AccountNotificationInfoDeterminingService : IAccountNotificationInfoDeterminingService
{
private readonly ICoreApiWrapperService _coreApiWrapperService;
private readonly ApiConfiguration _apiConfiguration;
public AccountNotificationInfoDeterminingService(ICoreApiWrapperService coreApiWrapperService,
ApiConfiguration apiConfiguration)
{
_coreApiWrapperService = coreApiWrapperService;
_apiConfiguration = apiConfiguration;
}
public async Task<List<MemberNotificationInfo>> DetermineAccountOwnerNotificationInfos(string accountNumber)
{
try
{
var dateTimeNow = DateTime.Now.ConvertToTimeZone(_apiConfiguration.TimeZone);
var agreeUserAccounts = await _coreApiWrapperService.GetAgreeUsersByAccount(accountNumber);
if (agreeUserAccounts?.Count() > 1)
{
throw new Exception("More then one AgreeUserAccounts returned. Expected only one back.");
}
var agreeUserAccount = agreeUserAccounts?.FirstOrDefault();
if (IsMemberNumberOrNull(agreeUserAccount?.UserId))
{
return new List<MemberNotificationInfo>()
{
new ()
{
UploadDate = dateTimeNow,
NotificationStatusType = NotificationStatusTypes.Complete,
Attempts = 1
}
};
}
var user = await _coreApiWrapperService.GetAgreementByUserId(agreeUserAccount?.UserId);
return new List<MemberNotificationInfo>()
{
new ()
{
AgreeUserId = agreeUserAccount.UserId,
EntityName = user.OrganizationName == null ? user.FirstName : user.OrganizationName,
UploadDate = dateTimeNow,
NotificationStatusType = NotificationStatusTypes.Pending,
Attempts = 1
}
};
}
catch (Exception ex)
{
var message = $"Failure to Determine Account Owner Notification Info by accountNumber: {accountNumber}. Exception Message: {ex.Message}";
Log.Error(message);
throw new ServiceException(message);
}
}
private bool IsMemberNumberOrNull(string userId)
{
if (userId == null)
{
return true;
}
return string.Equals(userId.Substring(0, 1), "M", StringComparison.OrdinalIgnoreCase);
}
}
@@ -0,0 +1,7 @@
using LCU.Library.Internal.DocumentIngestorModels.Entities;
namespace LCU.API.Internal.DocumentIngestor.Services.MemberNotificationInfoDeterminingServices;
public interface IAccountNotificationInfoDeterminingService
{
Task<List<MemberNotificationInfo>> DetermineAccountOwnerNotificationInfos(string accountNumber);
}
@@ -0,0 +1,10 @@
using LCU.Library.Internal.DocumentIngestorModels.Entities;
using LCU.Library.Internal.DocumentIngestorModels.Enums;
using LCU.Library.Internal.DocumentIngestorModels.Models;
namespace LCU.API.Internal.DocumentIngestor.Services.MemberNotificationInfoDeterminingServices;
public interface IMemberNotificationInfoDeterminingService
{
Task<List<MemberNotificationInfo>> HandleMemberNotificationInfos(MemberNotificationInfoRequest memberNotificationInfoRequest);
Task<(string EmailAddress, DeliveryMethod DeliveryMethod)> DetermineDeliveryMethod(MemberNotificationInfo memberNotificationInfo);
}
@@ -0,0 +1,9 @@
using LCU.Library.Internal.DocumentIngestorModels.Entities;
using LCU.Library.Internal.DocumentIngestorModels.Enums;
namespace LCU.API.Internal.DocumentIngestor.Services.MemberNotificationInfoDeterminingServices;
public interface IPrimaryNotificationInfoDeterminingService
{
Task<List<MemberNotificationInfo>> DeterminePrimaryOwnerNotificationInfos(string entityNumber,
EntityTypes entityType);
}
@@ -0,0 +1,109 @@
using LCU.API.Internal.DocumentIngestor.Services.MicroApiServices;
using LCU.Library.DocumentType.Extensions;
using LCU.Library.Internal.DocumentIngestorModels.Entities;
using LCU.Library.Internal.DocumentIngestorModels.Enums;
using LCU.Library.Internal.DocumentIngestorModels.Models;
using LCU.Library.Internal.Exceptions;
using Serilog;
namespace LCU.API.Internal.DocumentIngestor.Services.MemberNotificationInfoDeterminingServices;
public class MemberNotificationInfoDeterminingService : IMemberNotificationInfoDeterminingService
{
private readonly IAccountNotificationInfoDeterminingService _accountNotificationInfoDeterminingService;
private readonly IPrimaryNotificationInfoDeterminingService _primaryNotificationInfoDeterminingService;
private readonly IMemberNotificationInfoService _memberNotificationInfoService;
private readonly IMemberPreferencesApiService _memberPreferencesApiService;
private readonly IDocumentProcessingInfoService _documentProcessingInfoService;
public MemberNotificationInfoDeterminingService(IAccountNotificationInfoDeterminingService accountNotificationInfoDeterminingService,
IPrimaryNotificationInfoDeterminingService primaryNotificationInfoDeterminingService,
IMemberNotificationInfoService memberNotificationInfoService,
IMemberPreferencesApiService memberPreferencesApiService,
IDocumentProcessingInfoService documentProcessingInfoService)
{
_accountNotificationInfoDeterminingService = accountNotificationInfoDeterminingService;
_primaryNotificationInfoDeterminingService = primaryNotificationInfoDeterminingService;
_memberNotificationInfoService = memberNotificationInfoService;
_memberPreferencesApiService = memberPreferencesApiService;
_documentProcessingInfoService = documentProcessingInfoService;
}
public async Task<List<MemberNotificationInfo>> HandleMemberNotificationInfos(MemberNotificationInfoRequest memberNotificationInfoRequest)
{
if (memberNotificationInfoRequest.DocumentOwnerType == DocumentOwnerTypes.Generic)
{
return new List<MemberNotificationInfo>();
}
return await DetermineMemberNotificationInfos(memberNotificationInfoRequest);
}
public async Task<List<MemberNotificationInfo>> DetermineMemberNotificationInfos(MemberNotificationInfoRequest memberNotificationInfoRequest)
{
var memberNotificationInfos = memberNotificationInfoRequest.DocumentOwnerType switch
{
DocumentOwnerTypes.PrimaryOwner => await _primaryNotificationInfoDeterminingService.DeterminePrimaryOwnerNotificationInfos(memberNotificationInfoRequest.DocumentEntityNumber,
memberNotificationInfoRequest.EntityType),
DocumentOwnerTypes.AccountOwner => await _accountNotificationInfoDeterminingService.DetermineAccountOwnerNotificationInfos(memberNotificationInfoRequest.DocumentEntityNumber),
_ => throw new NotImplementedException()
};
foreach (var memberNotificationInfo in memberNotificationInfos)
{
memberNotificationInfo.DocumentProcessingInfoId = memberNotificationInfoRequest.DocumentProcessingInfoId;
var contact = await DetermineDeliveryMethod(memberNotificationInfo);
memberNotificationInfo.EmailAddress = contact.EmailAddress;
memberNotificationInfo.DeliveryMethod = contact.DeliveryMethod;
if (memberNotificationInfo.DeliveryMethod == DeliveryMethod.Paper)
{
memberNotificationInfo.NotificationStatusType = NotificationStatusTypes.Complete;
}
}
return memberNotificationInfos;
}
public async Task<(string EmailAddress, DeliveryMethod DeliveryMethod)> DetermineDeliveryMethod(MemberNotificationInfo memberNotificationInfo)
{
try
{
(string, DeliveryMethod) DeliveryMethodFactory(string? email, string? deliveryMethod)
{
var preference = deliveryMethod?.GetValueFromName<DeliveryMethod>() ?? DeliveryMethod.Paper;
return preference == DeliveryMethod.Electronic && !string.IsNullOrEmpty(email)
? (email, DeliveryMethod.Electronic)
: (string.Empty, DeliveryMethod.Paper);
}
if (string.IsNullOrWhiteSpace(memberNotificationInfo.AgreeUserId))
{
return DeliveryMethodFactory(null, null);
}
var memberOptInSettings = (await _memberPreferencesApiService.GetMemberOptInSettingsByUserId(memberNotificationInfo.AgreeUserId)).MemberOptinSettings;
if (memberOptInSettings is null)
{
return DeliveryMethodFactory(null, null);
}
if (memberOptInSettings.UniversalOptIn is not null)
{
return DeliveryMethodFactory(memberOptInSettings.UnifiedOptInEmail, memberOptInSettings.UniversalOptIn);
}
var documentProcessingInfo = await _documentProcessingInfoService.GetByIdAsync(memberNotificationInfo.DocumentProcessingInfoId);
return documentProcessingInfo.DocumentType is "Statement"
? DeliveryMethodFactory(memberOptInSettings.EStatementsEmail, memberOptInSettings.EStatements)
: DeliveryMethodFactory(memberOptInSettings.ENotificationsEmail, memberOptInSettings.ENotifications);
}
catch (Exception exception)
{
var message = $"Failure DeterminingDeliveryMethod for MemberNotificationInfo with Id:{memberNotificationInfo.MemberNotificationInfoId}.";
Log.Error(exception, message);
throw new ServiceException(message);
}
}
}
@@ -0,0 +1,109 @@
using LCU.Api.Internal.DocumentIngestorLib.Extensions;
using LCU.API.Internal.DocumentIngestor.Configurations;
using LCU.API.Internal.DocumentIngestor.Services.MicroApiServices;
using LCU.Library.Internal.DocumentIngestorModels.Entities;
using LCU.Library.Internal.DocumentIngestorModels.Enums;
using LCU.Library.Internal.Exceptions;
using Serilog;
namespace LCU.API.Internal.DocumentIngestor.Services.MemberNotificationInfoDeterminingServices;
public class PrimaryNotificationInfoDeterminingService : IPrimaryNotificationInfoDeterminingService
{
private readonly ICoreApiWrapperService _coreApiWrapperService;
private readonly ApiConfiguration _apiConfiguration;
public PrimaryNotificationInfoDeterminingService(ICoreApiWrapperService coreApiWrapperService,
ApiConfiguration apiConfiguration)
{
_coreApiWrapperService = coreApiWrapperService;
_apiConfiguration = apiConfiguration;
}
public async Task<List<MemberNotificationInfo>> DeterminePrimaryOwnerNotificationInfos(string entityNumber,
EntityTypes entityType)
{
try
{
var dateTimeNow = DateTime.Now.ConvertToTimeZone(_apiConfiguration.TimeZone);
var userIds = await GetUserIdsByEntityType(entityNumber, entityType);
var memberNotificationInfos = new List<MemberNotificationInfo>();
if (userIds is null || !userIds.Any())
{
return new List<MemberNotificationInfo>
{
new()
{
UploadDate = dateTimeNow,
NotificationStatusType = NotificationStatusTypes.Complete,
Attempts = 1
}
};
}
foreach (var userId in userIds)
{
var user = await _coreApiWrapperService.GetAgreementByUserId(userId);
if (user is null)
{
memberNotificationInfos.Add( new MemberNotificationInfo
{
UploadDate = dateTimeNow,
NotificationStatusType = NotificationStatusTypes.Complete,
Attempts = 1
});
continue;
}
memberNotificationInfos.Add( new MemberNotificationInfo()
{
AgreeUserId = userId,
EntityName = user.OrganizationName ?? user.FirstName,
UploadDate = dateTimeNow,
NotificationStatusType = NotificationStatusTypes.Pending,
Attempts = 1
});
}
return memberNotificationInfos.ToList();
}
catch (Exception ex)
{
var message = $"Failure to Determine Primary Owner Notification Info by entityNumber: {entityNumber}. Exception Message: {ex.Message}";
Log.Error(message);
throw new ServiceException(message);
}
}
private async Task<List<string>> GetUserIdsByEntityType(string entityNumber, EntityTypes entityType)
{
var userIds = new List<string>();
if (entityType == EntityTypes.Person)
{
var result = await _coreApiWrapperService.GetAgreeUsersByPersonNumber(entityNumber);
var ids = result?.Where(x => !string.IsNullOrWhiteSpace(x)).ToList();
if (ids is null || !ids.Any())
{
return new List<string>();
}
return ids;
}
else if (entityType == EntityTypes.Organization)
{
var result = await _coreApiWrapperService.GetAgreeUserByOrganizationNumber(entityNumber);
if (string.IsNullOrWhiteSpace(result))
{
return new List<string>();
}
userIds = new List<string>() { result };
}
return userIds;
}
}
@@ -0,0 +1,284 @@
using LCU.API.Internal.DocumentIngestor.Configurations;
using LCU.API.Internal.DocumentIngestorLib.Repositories;
using LCU.Library.Internal.DocumentIngestorModels.Entities;
using LCU.Library.Internal.DocumentIngestorModels.Enums;
using LCU.Library.Internal.DocumentIngestorModels.Models;
using LCU.Library.Internal.Exceptions;
using Serilog;
using System.Data.SqlClient;
namespace LCU.API.Internal.DocumentIngestor.Services;
public class MemberNotificationInfoService : IMemberNotificationInfoService
{
private readonly IUnitOfWork _unitOfWork;
private readonly ApiConfiguration _apiConfiguration;
private readonly IMemberNotificationInfoRepository _memberNotificationInfoRepository;
private readonly IProcessingLogService _processingLogService;
public MemberNotificationInfoService
(
IUnitOfWork unitOfWork,
ApiConfiguration apiConfiguration,
IMemberNotificationInfoRepository memberNotificationInfoRepository,
IProcessingLogService processingLogService
)
{
_unitOfWork = unitOfWork;
_apiConfiguration = apiConfiguration;
_memberNotificationInfoRepository = memberNotificationInfoRepository;
_processingLogService = processingLogService;
}
public async Task<IEnumerable<MemberNotificationInfo>> GetAsync()
{
try
{
var memberNotificationInfos = await _memberNotificationInfoRepository.FindAsync();
return memberNotificationInfos;
}
catch (Exception exception)
{
Log.Error(exception, "Failure retrieving MemberNotificationInfos.");
throw new ServiceException("Failure retrieving MemberNotificationInfos.");
}
}
public async Task<MemberNotificationInfo> GetByIdAsync(int id)
{
try
{
var memberNotificationInfo = await _memberNotificationInfoRepository.FindByIdAsync(id);
return memberNotificationInfo;
}
catch (Exception exception)
{
Log.Error(exception, $"Failure retrieving MemberNotificationInfo with MemberNotificationInfoId {id}.");
throw new ServiceException($"Failure retrieving MemberNotificationInfo with MemberNotificationInfoId {id}.");
}
}
public async Task<IEnumerable<MemberNotificationInfo>> GetListByIdAsync(List<int> ids)
{
try
{
var memberNotificationInfo = await _memberNotificationInfoRepository.FindListByIdAsync(ids);
return memberNotificationInfo;
}
catch (Exception exception)
{
var message = $"Failure retrieving list of MemberNotificationInfos with MemberNotificationInfoIds: {String.Join(", ", ids.ToArray())}.";
Log.Error(exception, message);
throw new ServiceException(message);
}
}
public async Task<IEnumerable<MemberNotificationInfo>> GetByDocumentProcessingInfoIdAsync(int documentProcessingInfoId)
{
try
{
var memberNotificationInfos = await _memberNotificationInfoRepository.FindByDocumentProcessingInfoIdAsync(documentProcessingInfoId);
return memberNotificationInfos;
}
catch (Exception exception)
{
Log.Error(exception, $"Failure retrieving MemberNotificationInfos with DocumentProcessingInfoId {documentProcessingInfoId}.");
throw new ServiceException($"Failure retrieving MemberNotificationInfos with DocumentProcessingInfoId {documentProcessingInfoId}.");
}
}
public async Task<IEnumerable<MemberNotificationInfo>> GetByNotificationStatusAsync(NotificationStatusTypes status)
{
try
{
var memberNotificationInfos = await _memberNotificationInfoRepository.FindByNotificationStatusAsync(status);
return memberNotificationInfos;
}
catch (Exception exception)
{
Log.Error(exception, $"Failure retrieving MemberNotificationInfos by Status. Status: {status}");
throw new ServiceException("Failure retrieving MemberNotificationInfos.");
}
}
public async Task<IEnumerable<MemberNotificationInfo>> GetSendableMemberNotificationInfosAsync(DateTime minimumUploadDate, int maxAttempts)
{
try
{
var memberNotificationInfos = await _memberNotificationInfoRepository.FindSendableMemberNotificationsAsync(minimumUploadDate, maxAttempts);
return memberNotificationInfos;
}
catch (Exception exception)
{
Log.Error(exception, $"Failure retrieving Sendable MemberNotificationInfos. MinimumUploadDate: {minimumUploadDate} | MaxAttempts: {maxAttempts}");
throw new ServiceException($"Failure retrieving MemberNotificationInfos.");
}
}
public async Task<MemberNotificationInfo> SaveAsync(MemberNotificationInfo memberNotificationInfo)
{
try
{
var result = await _memberNotificationInfoRepository.AddAsync(memberNotificationInfo);
await _unitOfWork.CompleteAsync();
return result;
}
catch (Exception exception)
{
Log.Error(exception, "Failure saving MemberNotificationInfo.");
throw new ServiceException("Failure saving MemberNotificationInfo.");
}
}
public async Task SaveListAsync(List<MemberNotificationInfo> memberNotificationInfos)
{
try
{
await _memberNotificationInfoRepository.AddListAsync(memberNotificationInfos);
await _unitOfWork.CompleteAsync();
}
catch (Exception exception)
{
Log.Error(exception, $"Failure saving list of MemberNotificationInfo.");
throw new ServiceException($"Failure saving list MemberNotificationInfo.");
}
}
public async Task<MemberNotificationInfo?> Update(MemberNotificationInfo memberNotificationInfo)
{
try
{
_memberNotificationInfoRepository.Update(memberNotificationInfo);
await _unitOfWork.CompleteAsync();
return memberNotificationInfo;
}
catch (SqlException sqlException)
{
Log.Error(sqlException, "SQL Failure updating MemberNotificationInfo. " +
$"MemberNotificationInfoId: {memberNotificationInfo.MemberNotificationInfoId}");
throw new ServiceException("SQL Failure updating MemberNotificationInfo. " +
$"MemberNotificationInfoId: {memberNotificationInfo.MemberNotificationInfoId}");
}
catch (Exception exception)
{
Log.Error(exception, $"Failure updating MemberNotificationInfo. " +
$"MemberNotificationInfoId: {memberNotificationInfo.MemberNotificationInfoId}");
throw new ServiceException($"Failure updating MemberNotificationInfo. " +
$"MemberNotificationInfoId: {memberNotificationInfo.MemberNotificationInfoId}");
}
}
public async Task<IEnumerable<MemberNotificationInfo?>> UpdateRange(IEnumerable<MemberNotificationInfo> memberNotificationInfos)
{
try
{
_memberNotificationInfoRepository.UpdateRange(memberNotificationInfos);
await _unitOfWork.CompleteAsync();
return memberNotificationInfos;
}
catch (SqlException sqlException)
{
Log.Error(sqlException, "SQL Failure updating list of MemberNotificationInfos.");
throw new ServiceException("SQL Failure updating list of MemberNotificationInfos.");
}
catch (Exception exception)
{
Log.Error(exception, "Failure updating list of MemberNotificationInfos.");
throw new ServiceException("Failure updating list of MemberNotificationInfos.");
}
}
public async Task Delete(int id)
{
try
{
var memberNotificationInfo = await GetByIdAsync(id);
ArgumentNullException.ThrowIfNull(memberNotificationInfo);
_memberNotificationInfoRepository.Remove(memberNotificationInfo);
_unitOfWork.Complete();
}
catch (SqlException sqlException)
{
Log.Error(sqlException, $"SQL failure removing MemberNotificationInfo. MemberNotificationInfoId: {id}");
throw new ServiceException($"SQL failure removing MemberNotificationInfo. MemberNotificationInfoId: {id}");
}
catch (Exception exception)
{
Log.Error(exception, $"Failure removing MemberNotificationInfo. MemberNotificationInfoId: {id}");
throw new ServiceException($"Failure removing MemberNotificationInfo. MemberNotificationInfoId: {id}");
}
}
public async Task<bool> SetDocumentNotificationStatus(int memberNotificationInfoId, NotificationStatusTypes notificationStatus, string userName, string reason)
{
try
{
var memberNotificationInfo = await GetByIdAsync(memberNotificationInfoId);
if (memberNotificationInfo is null) return false;
var originalStatus = memberNotificationInfo.NotificationStatusType;
memberNotificationInfo.NotificationStatusType = notificationStatus;
await Update(memberNotificationInfo);
var processingLog = new ProcessingLog(memberNotificationInfo.DocumentProcessingInfoId, DateTime.UtcNow,
$"Updated notification status for MemberNotificationInfoId: {memberNotificationInfoId} from {originalStatus} to {notificationStatus} because {reason}", userName);
await _processingLogService.SaveAsync(processingLog);
return true;
}
catch (Exception exception)
{
var message = $"Failure setting notification status for MemberNotificationInfoId: {memberNotificationInfoId} to {notificationStatus.ToString()}.";
Log.Error(exception, message);
throw new ServiceException(message);
}
}
public async Task<bool> BulkSetDocumentNotificationStatus(BulkNotificationStatusRequest request, string userName)
{
try
{
var memberNotificationInfos =
await _memberNotificationInfoRepository.GetMemberNotificationInfosByFilter(request.Filter);
var memberNotificationInfosChunks = memberNotificationInfos.Chunk(_apiConfiguration.BatchSkipAmount);
foreach (var memberNotificationInfoChunk in memberNotificationInfosChunks)
{
var listClone = memberNotificationInfoChunk.Select(x =>
new MemberNotificationInfo()
{
MemberNotificationInfoId = x.MemberNotificationInfoId,
DocumentProcessingInfoId = x.DocumentProcessingInfoId,
NotificationStatusType = x.NotificationStatusType
}).ToList();
var processingLogs = listClone.Select(x => new ProcessingLog(x.DocumentProcessingInfoId, DateTime.UtcNow,
$"Bulk notification status update for MemberNotificationId: {x.MemberNotificationInfoId} " +
$"from {x.NotificationStatusType} to {request.NotificationStatus} because {request.ReasonMessage}",
userName));
foreach (var memberNotificationInfo in memberNotificationInfoChunk)
{
if (memberNotificationInfo == null) continue;
memberNotificationInfo.NotificationStatusType = request.NotificationStatus;
}
_memberNotificationInfoRepository.UpdateRange(memberNotificationInfoChunk);
await _unitOfWork.CompleteAsync();
await _processingLogService.SaveRangeAsync(processingLogs);
}
return true;
}
catch (Exception exception)
{
var message = $"Failure on bulk update of notification status to {request.NotificationStatus.ToString()}.";
Log.Error(exception, message);
throw new ServiceException(message);
}
}
}
@@ -0,0 +1,182 @@
using Serilog;
using LCU.Library.Internal.Exceptions;
using LCU.Library.Internal.CoreApiWrapperModel.CoreModels.Tax;
using LCU.API.Internal.DocumentIngestor.DataAccess;
using LCU.Library.Internal.CoreApiWrapperModel.CoreModels.Accounts;
using LCU.Library.Internal.CoreApiWrapperModel.CoreModels.Agreements;
using LCU.Library.Internal.CoreApiWrapperModel.CoreModels.MemberAgreements;
using LCU.Library.Internal.CoreApiWrapperModel.CoreModels.Persons;
using LCU.Library.Internal.CoreApiWrapperModel.CoreModels.Organizations;
using LCU.Library.Internal.CoreApiWrapperModel.CoreModels.Mixed;
namespace LCU.API.Internal.DocumentIngestor.Services.MicroApiServices;
public class CoreApiWrapperService : ICoreApiWrapperService
{
private readonly ICoreApiWrapperDao _coreApiWrapperDao;
public CoreApiWrapperService(ICoreApiWrapperDao coreApiWrapperDao)
{
_coreApiWrapperDao = coreApiWrapperDao;
}
public async Task<IEnumerable<PersonTaxIdDataResponse>> GetPersonsTaxIdData(string taxId)
{
try
{
return await _coreApiWrapperDao.GetPersonsTaxIdData(taxId);
}
catch (Exception exception)
{
Log.Error(exception, $"Failure Get Persons TaxId Data From CoreApiWrapper. TaxId: {taxId}");
throw new ServiceException($"Failure Get Persons TaxId Data From CoreApiWrapper. TaxId: {taxId}, Exception Message: {exception.Message}");
}
}
public async Task<IEnumerable<OrganizationTaxIdDataResponse>> GetOrganizationsTaxIdData(string taxId)
{
try
{
return await _coreApiWrapperDao.GetOrganizationsTaxIdData(taxId);
}
catch (Exception exception)
{
Log.Error(exception, $"Failure Get Organizations TaxId Data From CoreApiWrapper. TaxId: {taxId}");
throw new ServiceException($"Failure Get Organizations TaxId Data From CoreApiWrapper. TaxId: {taxId}, Exception Message: {exception.Message}");
}
}
public async Task<IEnumerable<AccountOwnership>> GetOwnersByAccountNumber(string accountNumber, string creationDate)
{
try
{
return await _coreApiWrapperDao.GetOwnersByAccountNumber(accountNumber, creationDate);
}
catch (Exception exception)
{
Log.Error(exception, $"Failure Get Owners By Account Number From CoreApiWrapper. AccountNumber: {accountNumber} | CreationDate: {creationDate}");
throw new ServiceException($"Failure Get Owners By Account Number From CoreApiWrapper. AccountNumber: {accountNumber} | CreationDate: {creationDate}, Exception Message: {exception.Message}");
}
}
public async Task<IEnumerable<AccountOwnership>> GetOwnersByOrganizationName(string organizationName)
{
try
{
return await _coreApiWrapperDao.GetOwnersByOrganizationName(organizationName);
}
catch (Exception exception)
{
Log.Error(exception, $"Failure Get Owners By Organization Name From CoreApiWrapper. OrganizationName: {organizationName}");
throw new ServiceException($"Failure Get Owners By Organization Name From CoreApiWrapper. OrganizationName: {organizationName}, Exception Message: {exception.Message}");
}
}
public async Task<IEnumerable<AccountOwnership>> GetOwnersByPersonName(string personName)
{
try
{
return await _coreApiWrapperDao.GetOwnersByPersonName(personName);
}
catch (Exception exception)
{
Log.Error(exception, $"Failure Get Owners By Person Name From CoreApiWrapper. PersonName: {personName}");
throw new ServiceException($"Failure Get Owners By Person Name From CoreApiWrapper. PersonName: {personName}, Exception Message: {exception.Message}");
}
}
public async Task<IEnumerable<AgreeUserAccount>?> GetAgreeUsersByAccount(string accountNumber)
{
try
{
return await _coreApiWrapperDao.GetAgreeUsersByAccount(accountNumber);
}
catch (Exception exception)
{
var message = $"Failure Get Agree Users By AccountNumber From CoreApiWrapper. AccountNumber: {accountNumber}";
Log.Error(exception, message);
throw new ServiceException($"{message}, Exception Message: {exception.Message}");
}
}
public async Task<IEnumerable<string>?> GetAgreeUsersByPersonNumber(string personNumber)
{
try
{
return await _coreApiWrapperDao.GetAgreeUsersByPersonNumber(personNumber);
}
catch (Exception exception)
{
var message = $"Failure Get Agree Users By Person Number From CoreApiWrapper. PersonNumber: {personNumber}";
Log.Error(exception, message);
throw new ServiceException($"{message}, Exception Message: {exception.Message}");
}
}
public async Task<string?> GetAgreeUserByOrganizationNumber(string organizationNumber)
{
try
{
return await _coreApiWrapperDao.GetAgreeUserByOrganizationNumber(organizationNumber);
}
catch (Exception exception)
{
var message = $"Failure Get Agree User By Organization Number From CoreApiWrapper. OrganizationNumber: {organizationNumber}";
Log.Error(exception, message);
throw new ServiceException($"{message}, Exception Message: {exception.Message}");
}
}
public async Task<Agreement> GetAgreementByUserId(string userId)
{
try
{
return await _coreApiWrapperDao.GetAgreementByUserId(userId);
}
catch (Exception exception)
{
var message = $"Failure Get Agree User By UserId From CoreApiWrapper. UserId: {userId}";
Log.Error(exception, message);
throw new ServiceException($"{message}, Exception Message: {exception.Message}");
}
}
public async Task<AgreementAccount> GetAgreementAccountByAccountId(string accountId)
{
try
{
return await _coreApiWrapperDao.GetAgreementAccountByAccountId(accountId);
}
catch (Exception exception)
{
Log.Error(exception, $"Failure Get Agreement Account By Account Id From CoreApiWrapper. AccountId: {accountId}");
throw new ServiceException($"Failure Get Agreement Account By Account Id From CoreApiWrapper. AccountId: {accountId}, Exception Message: {exception.Message}");
}
}
public async Task<IEnumerable<AgreementAccount>?> GetAgreementAccountsByUserId(string userId)
{
try
{
return await _coreApiWrapperDao.GetAgreementAccountsByUserId(userId);
}
catch (Exception exception)
{
Log.Error(exception, $"Failure Get Agreement Account By User Id From CoreApiWrapper. UserId: {userId}");
throw new ServiceException($"Failure Get Agreement Account By User Id From CoreApiWrapper. UserId: {userId}, Exception Message: {exception.Message}");
}
}
public async Task<PersonOrganization> GetPersonOrOrganizationNumberByMemberId(string memberId)
{
try
{
return await _coreApiWrapperDao.GetPersonOrOrganizationNumberByMemberId(memberId);
}
catch (Exception exception)
{
Log.Error(exception, $"Failure Get PersonOrganization By Member Id From CoreApiWrapper. MemberId: {memberId}");
throw new ServiceException($"Failure Get PersonOrganization By Member Id From CoreApiWrapper. MemberId: {memberId}, Exception Message: {exception.Message}");
}
}
}
@@ -0,0 +1,156 @@
using LCU.Api.Internal.MaidBotLib.Models;
using LCU.API.Internal.DocumentIngestor.DataAccess;
using LCU.Library.DocumentType.Enums;
using LCU.Library.DocumentType.Extensions;
using LCU.Library.DocumentType.Models.Requests;
using LCU.Library.DocumentType.Models.Responses;
using LCU.Library.DocumentType.Models.Responses.External;
using LCU.Library.Internal.DocumentIngestorModels.Entities;
using LCU.Library.Internal.Exceptions;
using Serilog;
namespace LCU.API.Internal.DocumentIngestor.Services.MicroApiServices;
public class DocumentTypeRetrieverService : IDocumentTypeRetrieverService
{
private readonly IDocumentTypeRetrieverDao _documentTypeRetrieverDao;
private const string _unknown = "Unknown";
public DocumentTypeRetrieverService(IDocumentTypeRetrieverDao documentTypeRetrieverDao)
{
_documentTypeRetrieverDao = documentTypeRetrieverDao;
}
public async Task<DocumentCodeDetailsResponse> GetDocumentCodeDetail(string documentCode)
{
try
{
return await _documentTypeRetrieverDao.GetDocumentCodeDetail(documentCode);
}
catch (Exception exception)
{
Log.Error(exception, $"Failure To Get DocumentCodeDetail By DocumentCode From DocumentTypeRetriever. DocumentCode: {documentCode}");
throw new ServiceException($"Failure To Get DocumentCodeDetail By DocumentCode From DocumentTypeRetriever. DocumentCode: {documentCode} | " +
$"Exception Message: {exception.Message}");
}
}
public async Task<IEnumerable<DocumentCodeDetailsResponse>> GetDocumentCodeDetails()
{
try
{
return await _documentTypeRetrieverDao.GetDocumentCodeDetails();
}
catch (Exception exception)
{
Log.Error(exception, $"Failure To Get DocumentCodeDetails From DocumentTypeRetriever.");
throw new ServiceException($"Failure To Get DocumentCodeDetails From DocumentTypeRetriever. Exception Message: {exception.Message}");
}
}
public async Task<DocumentCodeDetailInfo> GetDocumentCodeDetailByDisplayName(string documentTypeDisplayName)
{
try
{
var documentCodeDetails = await _documentTypeRetrieverDao.GetDocumentCodeDetailsCache();
var documentCodeDetail = documentCodeDetails?
.Where(x => x.DocumentCode == documentTypeDisplayName.TranslateDocumentCode())
.FirstOrDefault();
if (documentCodeDetail == null)
{
Log.ForContext(Enum.GetName(typeof(PropertyEnum), PropertyEnum.Group),
Enum.GetName(typeof(GroupEnum), GroupEnum.ITDevelopment))
.Error($"Failure Get Document Code Detail. DocumentTypeDisplayName: {documentTypeDisplayName}");
return new DocumentCodeDetailInfo()
{
DocumentType = _unknown,
DocumentBusinessName = _unknown,
DocumentMemberFacingName = _unknown
};
}
return new DocumentCodeDetailInfo()
{
DocumentType = Enum.GetName(typeof(DocumentTypes), documentCodeDetail.DocumentType),
DocumentBusinessName = documentCodeDetail.DocumentBusinessName,
DocumentMemberFacingName = documentCodeDetail.DocumentMemberFacingName
};
}
catch (Exception exception)
{
Log.Error(exception, $"Failure To Get DocumentCodeDetails By DocumentCode From DocumentTypeRetriever. " +
$"DocumentTypeDisplayName: {documentTypeDisplayName}");
throw new ServiceException($"Failure To Get DocumentCodeDetails By DocumentCode From DocumentTypeRetriever. " +
$"DocumentTypeDisplayName: {documentTypeDisplayName} | Exception Message: {exception.Message}");
}
}
public async Task<DocumentCodeDetailInfo> GetDocumentCodeDetailByBusinessName(string documentBusinessName)
{
try
{
var documentCodeDetails = await _documentTypeRetrieverDao.GetDocumentCodeDetailsCache();
var documentCodeDetail = documentCodeDetails?
.Where(x => x.DocumentBusinessName == documentBusinessName)
.FirstOrDefault();
if (documentCodeDetail == null)
{
Log.ForContext(Enum.GetName(typeof(PropertyEnum), PropertyEnum.Group),
Enum.GetName(typeof(GroupEnum), GroupEnum.ITDevelopment))
.Error($"Failure Get Document Code Detail. DocumentBusinessName: {documentBusinessName}");
return new DocumentCodeDetailInfo()
{
DocumentType = _unknown,
DocumentBusinessName = _unknown,
DocumentMemberFacingName = _unknown
};
}
return new DocumentCodeDetailInfo()
{
DocumentType = Enum.GetName(typeof(DocumentTypes), documentCodeDetail.DocumentType),
DocumentBusinessName = documentCodeDetail.DocumentBusinessName,
DocumentMemberFacingName = documentCodeDetail.DocumentMemberFacingName
};
}
catch (Exception exception)
{
Log.Error(exception, $"Failure To Get DocumentCodeDetails By DocumentBusinessName From DocumentTypeRetriever. " +
$"DocumentBusinessName: {documentBusinessName}");
throw new ServiceException($"Failure To Get DocumentCodeDetails By DocumentBusinessName From DocumentTypeRetriever. " +
$"DocumentBusinessName: {documentBusinessName} | Exception Message: {exception.Message}");
}
}
public async Task<IEnumerable<DocumentMetadataResponse>> GetListOfMaliciousDocuments(MaliciousDocumentSearchFilter filter)
{
try
{
return await _documentTypeRetrieverDao.GetListOfMaliciousDocuments(filter);
}
catch (Exception exception)
{
Log.Error(exception, "Failure To Get List of MaliciousDocuments From DocumentTypeRetriever.");
throw new ServiceException($"Failure To Get List of MaliciousDocuments From DocumentTypeRetriever. Exception Message: {exception.Message}");
}
}
public async Task<EnrichedDocumentMetadataResponse> GetDocumentInformationById(Guid documentId)
{
try
{
return await _documentTypeRetrieverDao.GetDocumentInformationById(documentId);
}
catch (Exception exception)
{
var message =
$"Failure to get DocumentInformation with DocumentId ({documentId}) from DocumentTypeRetriever.";
Log.Error(exception, message);
throw new ServiceException($"{message} Exception Message: {exception.Message}");
}
}
}
@@ -0,0 +1,171 @@
using Serilog;
using Newtonsoft.Json;
using LCU.Library.Internal.Exceptions;
using LCU.Library.DocumentType.Models.Requests.PrimaryOwner;
using LCU.Library.DocumentType.Models.Requests.Account;
using LCU.Library.DocumentType.Models.Requests.Generic;
using LCU.Library.Internal.DocumentIngestorModels.Models;
using LCU.Library.Internal.DocumentIngestorModels.Enums;
using LCU.API.Internal.DocumentIngestor.Configurations;
using LCU.API.Internal.DocumentIngestor.DataAccess;
using LCU.Library.Internal.LcuMassTransitModels;
using LCU.Library.Internal.ApiResponse.Models;
namespace LCU.API.Internal.DocumentIngestor.Services.MicroApiServices;
public class DocumentTypeUploaderApiService : IDocumentTypeUploaderApiService
{
private readonly IDocumentTypeUploaderApiDao _documentTypeUploaderApiDao;
private readonly ConnectionConfiguration _connectionConfiguration;
private readonly ReceiveEndpointConfiguration _uploadCompletionReceiveEndpointConfiguration;
public DocumentTypeUploaderApiService(IDocumentTypeUploaderApiDao documentTypeUploaderApiDao,
ConnectionConfiguration connectionConfiguration,
ReceiveEndpointConfigurations receiveEndpointConfigurations)
{
_documentTypeUploaderApiDao = documentTypeUploaderApiDao;
_connectionConfiguration = connectionConfiguration;
_uploadCompletionReceiveEndpointConfiguration = receiveEndpointConfigurations.UploadCompletionEndpoint;
}
public async Task UploadPrimaryOwnerDocument(PrimaryOwnerDocumentRequest primaryOwnerDocumentRequest,
ProcessingRequest request,
ProcessResult result)
{
try
{
var isPersonEntityType = !string.IsNullOrEmpty(primaryOwnerDocumentRequest.PrimaryOwnerAttribute.PersonNumber);
var documentEntityNumber = isPersonEntityType ?
primaryOwnerDocumentRequest.PrimaryOwnerAttribute.PersonNumber :
primaryOwnerDocumentRequest.PrimaryOwnerAttribute.OrganizationNumber;
var entityType = isPersonEntityType ? EntityTypes.Person : EntityTypes.Organization;
var completionRequest = new CompletionRequest
{
ProcessingRequest = request,
ProcessResult = result,
MemberNotificationInfoRequest = new MemberNotificationInfoRequest
{
DocumentProcessingInfoId = request.DocumentProcessingInfoId,
DocumentOwnerType = DocumentOwnerTypes.PrimaryOwner,
DocumentEntityNumber = documentEntityNumber,
EntityType = entityType
}
};
primaryOwnerDocumentRequest.ResponseItems = new List<ResponseItem>
{
new ResponseItem()
{
ResponsePayload = JsonConvert.SerializeObject(completionRequest),
ReturnQueueConnectionConfig = new QueueConnectionConfig()
{
Url = _connectionConfiguration.HostUrl,
Username = _connectionConfiguration.UserName,
Password = _connectionConfiguration.Password
},
ReturnQueueConsumerConfig = new QueueConsumerConfig()
{
ExchangeName = _uploadCompletionReceiveEndpointConfiguration.ReceiveEndpoint
}
}
};
await _documentTypeUploaderApiDao.UploadPrimaryOwnerDocument(primaryOwnerDocumentRequest);
}
catch (Exception exception)
{
Log.Error(exception, "Failure Primary Owner Document Upload.");
throw new ServiceException($"Failure during Primary Owner Document Upload. Exception Message: {exception.Message}");
}
}
public async Task UploadAccountDocument(AccountDocumentRequest accountDocumentRequest,
ProcessingRequest request,
ProcessResult result)
{
try
{
var completionRequest = new CompletionRequest
{
ProcessingRequest = request,
ProcessResult = result,
MemberNotificationInfoRequest = new MemberNotificationInfoRequest
{
DocumentProcessingInfoId = request.DocumentProcessingInfoId,
DocumentOwnerType = DocumentOwnerTypes.AccountOwner,
DocumentEntityNumber = accountDocumentRequest.AccountAttribute.AccountNumber,
EntityType = EntityTypes.Unknown
}
};
accountDocumentRequest.ResponseItems = new List<ResponseItem>
{
new ResponseItem()
{
ResponsePayload = JsonConvert.SerializeObject(completionRequest),
ReturnQueueConnectionConfig = new QueueConnectionConfig()
{
Url = _connectionConfiguration.HostUrl,
Username = _connectionConfiguration.UserName,
Password = _connectionConfiguration.Password
},
ReturnQueueConsumerConfig = new QueueConsumerConfig()
{
ExchangeName = _uploadCompletionReceiveEndpointConfiguration.ReceiveEndpoint
}
}
};
await _documentTypeUploaderApiDao.UploadAccountDocument(accountDocumentRequest);
}
catch (Exception exception)
{
Log.Error(exception, "Failure Account Document Upload.");
throw new ServiceException($"Failure during Account Document Upload. Exception Message: {exception.Message}");
}
}
public async Task UploadGenericDocument(GenericDocumentRequest genericDocumentRequest,
ProcessingRequest request,
ProcessResult result)
{
try
{
var completionRequest = new CompletionRequest
{
ProcessingRequest = request,
ProcessResult = result,
MemberNotificationInfoRequest = new MemberNotificationInfoRequest
{
DocumentProcessingInfoId = request.DocumentProcessingInfoId,
DocumentOwnerType = DocumentOwnerTypes.Generic
}
};
genericDocumentRequest.ResponseItems = new List<ResponseItem>
{
new ResponseItem()
{
ResponsePayload = JsonConvert.SerializeObject(completionRequest),
ReturnQueueConnectionConfig = new QueueConnectionConfig()
{
Url = _connectionConfiguration.HostUrl,
Username = _connectionConfiguration.UserName,
Password = _connectionConfiguration.Password
},
ReturnQueueConsumerConfig = new QueueConsumerConfig()
{
ExchangeName = _uploadCompletionReceiveEndpointConfiguration.ReceiveEndpoint
}
}
};
await _documentTypeUploaderApiDao.UploadGenericDocument(genericDocumentRequest);
}
catch (Exception exception)
{
Log.Error(exception, "Failure Generic Document Upload.");
throw new ServiceException($"Failure during Generic Document Upload. Exception Message: {exception.Message}");
}
}
}
@@ -0,0 +1,321 @@
using Serilog;
using Newtonsoft.Json;
using LCU.Library.Internal.Exceptions;
using LCU.API.Internal.DocumentIngestor.DataAccess;
using LCU.Library.Internal.FileOperationsModels.Models;
namespace LCU.API.Internal.DocumentIngestor.Services.MicroApiServices;
public class FileOperationsApiService : IFileOperationsApiService
{
private readonly IFileOperationsApiDao _fileOperationsApiDao;
public FileOperationsApiService(IFileOperationsApiDao fileOperationsApiDao)
{
_fileOperationsApiDao = fileOperationsApiDao;
}
public async Task<RequestResponse> DirectMoveFile(string sourcePath, string sourceFile, string destinationPath)
{
try
{
var moveFileRequest = new MoveFileRequest
{
SourcePath = sourcePath,
SourceFile = sourceFile,
DestinationPath = destinationPath
};
return await _fileOperationsApiDao.DirectMoveFile(moveFileRequest);
}
catch (Exception exception)
{
var error = $"Failure during MoveFile. Exception Message: {exception.Message}";
Log.Error(error);
throw new ServiceException(error);
}
}
public async Task<RequestResponse> DirectWriteFile(string fileName, string extension, string fileData, string destinationPath)
{
try
{
var writeFileRequest = new WriteFileRequest
{
FileName = fileName,
Extension = extension,
FileData = fileData,
DestinationPath = destinationPath
};
return await _fileOperationsApiDao.DirectWriteFile(writeFileRequest);
}
catch (Exception exception)
{
var error = $"Failure during WriteFile. Exception Message: {exception.Message}";
Log.Error(error);
throw new ServiceException(error);
}
}
public async Task<RequestResponse> DirectCreateDirectory(string directoryPath, string directoryName)
{
try
{
var createDirectoryRequest = new CreateDirectoryRequest
{
DirectoryPath = directoryPath,
DirectoryName = directoryName
};
return await _fileOperationsApiDao.DirectCreateDirectory(createDirectoryRequest);
}
catch (Exception exception)
{
var error = $"Failure during CreateDirectory. Exception Message: {exception.Message}";
Log.Error(error);
throw new ServiceException(error);
}
}
public async Task<RequestResponse> DirectUnzipFile(string sourceDirectory,
string zipFileName,
string destinationPath,
bool useZipFileNameAsFinalDestination)
{
try
{
var unzipFileRequest = new UnzipFileRequest
{
SourceDirectory = sourceDirectory,
ZipFileName = zipFileName,
DestinationPath = destinationPath,
UseZipFileNameAsFinalDestination = useZipFileNameAsFinalDestination
};
return await _fileOperationsApiDao.DirectUnzipFile(unzipFileRequest);
}
catch (Exception exception)
{
var error = $"Failure during UnzipFile. Exception Message: {exception.Message}";
Log.Error(error);
throw new ServiceException(error);
}
}
public async Task<RequestResponse> DirectZipDirectory(string sourceDirectory, string zipPath, string zipFileName)
{
try
{
var zipDirectoryRequest = new ZipDirectoryRequest
{
SourceDirectory = sourceDirectory,
ZipPath = zipPath,
ZipFileName = zipFileName
};
return await _fileOperationsApiDao.DirectZipDirectory(zipDirectoryRequest);
}
catch (Exception exception)
{
var error = $"Failure during ZipDirectory. Exception Message: {exception.Message}";
Log.Error(error);
throw new ServiceException(error);
}
}
public async Task<RequestResponse> DirectDeleteDirectory(string directoryName, string directoryRootPath, int maxAmountOfDirectories)
{
try
{
var deleteDirectoryRequest = new DeleteDirectoryRequest
{
DirectoryName = directoryName,
DirectoryRootPath = directoryRootPath,
MaxAmountOfDirectories = maxAmountOfDirectories,
Recursive = true
};
return await _fileOperationsApiDao.DirectDeleteDirectory(deleteDirectoryRequest);
}
catch (Exception exception)
{
var error = $"Failure during DeleteDirectory. Exception Message: {exception.Message}";
Log.Error(error);
throw new ServiceException(error);
}
}
public async Task<RequestResponse> DirectDeleteFile(string sourcePath, string sourceFile)
{
try
{
var deleteFileRequest = new DeleteFileRequest
{
SourcePath = sourcePath,
SourceFile = sourceFile
};
return await _fileOperationsApiDao.DirectDeleteFile(deleteFileRequest);
}
catch (Exception exception)
{
var error = $"Failure during DeleteFile. Exception Message: {exception.Message}";
Log.Error(error);
throw new ServiceException(error);
}
}
public async Task<string> GetFileContent(string filePath)
{
try
{
return await _fileOperationsApiDao.GetFileContent(filePath);
}
catch (Exception exception)
{
Log.Error(exception, $"Failure to Get File Content. FilePath: {filePath}");
throw new ServiceException($"Failure to Get File Content. FilePath: {filePath}. " +
$"Exception Message: {exception.Message}");
}
}
public async Task<byte[]> GetPdfByteContent(string filePath)
{
try
{
return await _fileOperationsApiDao.GetPdfByteContent(filePath);
}
catch (Exception exception)
{
Log.Error(exception, $"Failure to Get PDF Content. FilePath: {filePath}");
throw new ServiceException($"Failure to Get PDF Content. FilePath: {filePath}. " +
$"Exception Message: {exception.Message}");
}
}
public async Task<byte[]> GetContentBytes(string documentPath, string documentFileName)
{
var filePath = documentPath + documentFileName;
return await GetPdfByteContent(filePath);
}
public async Task<bool> ValidateFileExists(string filePath)
{
try
{
return await _fileOperationsApiDao.ValidateFileExists(filePath);
}
catch (Exception exception)
{
Log.Error(exception, $"Failure to Validate File Exists. FilePath:{filePath}.");
throw new ServiceException($"Failure to Validate File Exists. FilePath:{filePath}. " +
$"Exception Message: {exception.Message}");
}
}
public async Task<bool> ValidateFileExistsAndNotArchived(string filePath, string archiveFilePath)
{
var fileExists = await ValidateFileExists(filePath);
if (fileExists)
{
return true;
}
var archiveFileExists = await ValidateFileExists(archiveFilePath);
if (archiveFileExists)
{
return false;
}
throw new Exception("File does not exist.");
}
public async Task<RequestResponse> ValidatePdfPageExists(string filePath, string pageNumber)
{
try
{
return await _fileOperationsApiDao.ValidatePdfPageExists(filePath, pageNumber);
}
catch (Exception exception)
{
Log.Error(exception, $"Failure to Validate PDF Page Exists. FilePath:{filePath} | PageNumber:{pageNumber}.");
throw new ServiceException($"Failure to Validate PDF Page Exists. FilePath:{filePath} | PageNumber: {pageNumber}. " +
$"Exception Message: {exception.Message}");
}
}
public async Task<IEnumerable<string>> BuildDynamicDirectoryList(DynamicDirectoryListRequest request)
{
try
{
return await _fileOperationsApiDao.BuildDynamicDirectoryList(request);
}
catch (Exception exception)
{
var message = $"Failure to Build Dynamic Directory List. " +
$"Path:{request.Path}" +
$"DynamicDirectoryName: {request.DynamicDirectoryName}.";
Log.Error(exception, message);
throw new ServiceException($"{message} Exception Message: {exception.Message}");
}
}
public async Task<string> BuildDynamicFile(DynamicFileListRequest request)
{
try
{
var dynamicFiles = await BuildDynamicFileList(request);
if (dynamicFiles.Count() > 1)
{
var message = $"There were more than one file that met the dynamic file settings. " +
$"DynamicFileListRequest:{JsonConvert.SerializeObject(request)}";
throw new Exception(message);
}
var file = dynamicFiles.FirstOrDefault();
if (file == null)
{
var message = $"There were no files that met the dynamic file settings. " +
$"DynamicFileListRequest:{JsonConvert.SerializeObject(request)}";
throw new Exception(message);
}
return file;
}
catch (Exception exception)
{
var message = $"Failure to Build Dynamic File. " +
$"Path:{request.Path}" +
$"Filename: {request.Filename}.";
Log.Error(exception, message);
throw new ServiceException($"{message} Exception Message: {exception.Message}");
}
}
public async Task<IEnumerable<string>> BuildDynamicFileList(DynamicFileListRequest request)
{
try
{
return await _fileOperationsApiDao.BuildDynamicFileList(request);
}
catch (Exception exception)
{
var message = $"Failure to Build Dynamic File List. " +
$"Path:{request.Path}" +
$"Filename: {request.Filename}.";
Log.Error(exception, message);
throw new ServiceException($"{message} Exception Message: {exception.Message}");
}
}
public async Task QueueBatchPdfParser(BatchParsePdfRequest batchParsePdfRequest)
{
try
{
await _fileOperationsApiDao.QueueBatchPdfParser(batchParsePdfRequest);
}
catch (Exception exception)
{
Log.Error(exception, $"Failure to Queue BatchParsePdfRequest. " +
$"FilePath:{batchParsePdfRequest.RootPath}/{batchParsePdfRequest.PdfName} ");
throw new ServiceException($"Failure to Queue BatchParsePdfRequest. " +
$"FilePath:{batchParsePdfRequest.RootPath}/{batchParsePdfRequest.PdfName} " +
$"Exception Message: {exception.Message}");
}
}
}
@@ -0,0 +1,25 @@
using LCU.Library.Internal.CoreApiWrapperModel.CoreModels.Accounts;
using LCU.Library.Internal.CoreApiWrapperModel.CoreModels.Agreements;
using LCU.Library.Internal.CoreApiWrapperModel.CoreModels.MemberAgreements;
using LCU.Library.Internal.CoreApiWrapperModel.CoreModels.Mixed;
using LCU.Library.Internal.CoreApiWrapperModel.CoreModels.Organizations;
using LCU.Library.Internal.CoreApiWrapperModel.CoreModels.Persons;
using LCU.Library.Internal.CoreApiWrapperModel.CoreModels.Tax;
namespace LCU.API.Internal.DocumentIngestor.Services.MicroApiServices;
public interface ICoreApiWrapperService
{
Task<IEnumerable<PersonTaxIdDataResponse>> GetPersonsTaxIdData(string taxId);
Task<IEnumerable<OrganizationTaxIdDataResponse>> GetOrganizationsTaxIdData(string taxId);
Task<IEnumerable<AccountOwnership>> GetOwnersByAccountNumber(string accountNumber, string creationDate);
Task<IEnumerable<AccountOwnership>> GetOwnersByOrganizationName(string organizationName);
Task<IEnumerable<AccountOwnership>> GetOwnersByPersonName(string personName);
Task<AgreementAccount> GetAgreementAccountByAccountId(string accountId);
Task<IEnumerable<AgreementAccount>?> GetAgreementAccountsByUserId(string userId);
Task<IEnumerable<AgreeUserAccount>?> GetAgreeUsersByAccount(string accountNumber);
Task<IEnumerable<string>?> GetAgreeUsersByPersonNumber(string personNumber);
Task<string?> GetAgreeUserByOrganizationNumber(string organizationNumber);
Task<Agreement> GetAgreementByUserId(string userId);
Task<PersonOrganization> GetPersonOrOrganizationNumberByMemberId(string memberId);
}
@@ -0,0 +1,16 @@
using LCU.Library.DocumentType.Models.Requests;
using LCU.Library.DocumentType.Models.Responses;
using LCU.Library.DocumentType.Models.Responses.External;
using LCU.Library.Internal.DocumentIngestorModels.Entities;
namespace LCU.API.Internal.DocumentIngestor.Services.MicroApiServices;
public interface IDocumentTypeRetrieverService
{
Task<DocumentCodeDetailsResponse> GetDocumentCodeDetail(string documentCode);
Task<IEnumerable<DocumentCodeDetailsResponse>> GetDocumentCodeDetails();
Task<DocumentCodeDetailInfo> GetDocumentCodeDetailByDisplayName(string documentTypeDisplayName);
Task<IEnumerable<DocumentMetadataResponse>> GetListOfMaliciousDocuments(MaliciousDocumentSearchFilter filter);
Task<EnrichedDocumentMetadataResponse> GetDocumentInformationById(Guid documentId);
Task<DocumentCodeDetailInfo> GetDocumentCodeDetailByBusinessName(string documentBusinessName);
}
@@ -0,0 +1,19 @@
using LCU.Library.DocumentType.Models.Requests.PrimaryOwner;
using LCU.Library.DocumentType.Models.Requests.Account;
using LCU.Library.DocumentType.Models.Requests.Generic;
using LCU.Library.Internal.DocumentIngestorModels.Models;
namespace LCU.API.Internal.DocumentIngestor.Services.MicroApiServices;
public interface IDocumentTypeUploaderApiService
{
Task UploadPrimaryOwnerDocument(PrimaryOwnerDocumentRequest primaryOwnerDocumentRequest,
ProcessingRequest request,
ProcessResult result);
Task UploadAccountDocument(AccountDocumentRequest accountDocumentRequest,
ProcessingRequest request,
ProcessResult result);
Task UploadGenericDocument(GenericDocumentRequest genericDocumentRequest,
ProcessingRequest request,
ProcessResult result);
}
@@ -0,0 +1,28 @@
using LCU.Library.Internal.FileOperationsModels.Models;
namespace LCU.API.Internal.DocumentIngestor.Services.MicroApiServices;
public interface IFileOperationsApiService
{
Task<RequestResponse> DirectMoveFile(string sourcePath, string sourceFile, string destinationPath);
Task<RequestResponse> DirectWriteFile(string fileName, string extension, string fileData, string destinationPath);
Task<RequestResponse> DirectCreateDirectory(string directoryPath, string directoryName);
Task<RequestResponse> DirectUnzipFile(string sourceDirectory,
string zipFileName,
string destinationPath,
bool useZipFileNameAsFinalDestination);
Task<RequestResponse> DirectZipDirectory(string sourceDirectory, string zipPath, string zipFileName);
Task<RequestResponse> DirectDeleteDirectory(string directoryName, string directoryRootPath, int maxAmountOfDirectories);
Task<RequestResponse> DirectDeleteFile(string sourcePath, string sourceFile);
Task<string> GetFileContent(string filePath);
Task<byte[]> GetPdfByteContent(string filePath);
Task<byte[]> GetContentBytes(string documentPath, string documentFileName);
Task<bool> ValidateFileExists(string filePath);
Task<bool> ValidateFileExistsAndNotArchived(string filePath, string archiveFilePath);
Task<RequestResponse> ValidatePdfPageExists(string filePath, string pageNumber);
Task<IEnumerable<string>> BuildDynamicDirectoryList(DynamicDirectoryListRequest request);
Task<string> BuildDynamicFile(DynamicFileListRequest request);
Task<IEnumerable<string>> BuildDynamicFileList(DynamicFileListRequest request);
Task QueueBatchPdfParser(BatchParsePdfRequest batchParsePdfRequest);
}
@@ -0,0 +1,9 @@
using LCU.API.Internal.DocumentIngestor.Models.MemberPreferencesDtos;
namespace LCU.API.Internal.DocumentIngestor.Services.MicroApiServices;
public interface IMemberPreferencesApiService
{
Task<MemberOptInAgreementInformation> GetMemberOptInAgreementInfoByUserId(string userId);
Task<MemberOptInInformation> GetMemberOptInSettingsByUserId(string userId);
Task EmailStatusSend(string userId, string emailAddress, bool success);
}
@@ -0,0 +1,10 @@
using LCU.API.Internal.DocumentIngestor.Models;
using LCU.Library.Internal.NotificationsModels.Models;
namespace LCU.API.Internal.DocumentIngestor.Services.MicroApiServices
{
public interface INotificationApiService
{
Task SendEmailAsync(EmailRequest emailRequest);
}
}
@@ -0,0 +1,65 @@
using Serilog;
using LCU.Library.Internal.Exceptions;
using LCU.API.Internal.DocumentIngestor.DataAccess;
using LCU.API.Internal.DocumentIngestor.Models.MemberPreferencesDtos;
namespace LCU.API.Internal.DocumentIngestor.Services.MicroApiServices;
public class MemberPreferencesApiService : IMemberPreferencesApiService
{
private readonly IMemberPreferencesApiDao _memberPreferencesApiDao;
public MemberPreferencesApiService(IMemberPreferencesApiDao memberPreferencesApiDao)
{
_memberPreferencesApiDao = memberPreferencesApiDao;
}
public async Task<MemberOptInAgreementInformation> GetMemberOptInAgreementInfoByUserId(string userId)
{
try
{
return await _memberPreferencesApiDao.GetMemberOptInAgreementInfoByUserId(userId);
}
catch (Exception exception)
{
var message = $"Failure Get Member Opt In Agreement Information From Member Preferences Api. UserId: {userId}";
Log.Error(exception, message);
throw new ServiceException($"{message}, Exception Message: {exception.Message}");
}
}
public async Task<MemberOptInInformation> GetMemberOptInSettingsByUserId(string userId)
{
try
{
return await _memberPreferencesApiDao.GetMemberOptInSettingsByUserId(userId);
}
catch (Exception exception)
{
var message = $"Failure Get Member Opt In Settings From Member Preferences Api. UserId: {userId}";
Log.Error(exception, message);
throw new ServiceException($"{message}, Exception Message: {exception.Message}");
}
}
public async Task EmailStatusSend(string userId, string emailAddress, bool success)
{
try
{
if (success)
{
await _memberPreferencesApiDao.EmailSuccessfulSend(userId, emailAddress);
}
else
{
await _memberPreferencesApiDao.EmailFailureSend(userId, emailAddress);
}
}
catch (Exception exception)
{
var message = $"Failure to notify Member Preferences Api with UserId({userId}) and EmailAddress({emailAddress}) with status: {success}).";
Log.Error(exception, message);
throw new ServiceException($"{message}, Exception Message: {exception.Message}");
}
}
}
@@ -0,0 +1,29 @@
using LCU.API.Internal.DocumentIngestor.DataAccess;
using LCU.Library.Internal.NotificationsModels.Models;
using LCU.Library.Internal.Exceptions;
using Serilog;
namespace LCU.API.Internal.DocumentIngestor.Services.MicroApiServices;
public class NotificationApiService : INotificationApiService
{
private readonly INotificationApiDao _notificationServiceApiDao;
public NotificationApiService(INotificationApiDao notificationApiServiceDao)
{
_notificationServiceApiDao = notificationApiServiceDao;
}
public async Task SendEmailAsync(EmailRequest emailRequest)
{
try
{
await _notificationServiceApiDao.SendEmailAsync(emailRequest);
}
catch (Exception exception)
{
Log.Error(exception, $"Failure to send email. Subject: {emailRequest.Subject}; Recipients: {string.Join(", ", emailRequest.Recipient)}");
throw new ServiceException($"Failure to send email. Subject: {emailRequest.Subject}; Recipients: {string.Join(", ", emailRequest.Recipient)}; Exception Message: {exception.Message}");
}
}
}
@@ -0,0 +1,230 @@
using LCU.API.Internal.DocumentIngestorLib.Repositories;
using LCU.Library.Internal.DocumentIngestorModels.Entities;
using LCU.Library.Internal.Exceptions;
using Serilog;
using System.Data.SqlClient;
namespace LCU.API.Internal.DocumentIngestor.Services;
[Obsolete("06/02/2023 - Used to prepare historical documents for ingestion. No longer in use.")]
public class OsgPreprocessorInfoService : IOsgPreprocessorInfoService
{
private readonly IUnitOfWork _unitOfWork;
private readonly IOsgPreprocessorInfoRepository _osgPreprocessorInfoRepository;
public OsgPreprocessorInfoService(IUnitOfWork unitOfWork,
IOsgPreprocessorInfoRepository osgPreprocessingInfoRepository)
{
_unitOfWork = unitOfWork;
_osgPreprocessorInfoRepository = osgPreprocessingInfoRepository;
}
public async Task<IEnumerable<OsgPreprocessorInfo>> GetAsync()
{
try
{
var osgPreprocessorInfos = await _osgPreprocessorInfoRepository.FindAsync();
return osgPreprocessorInfos;
}
catch (Exception exception)
{
Log.Error(exception, $"Failure retrieving OsgPreprocessorInfos.");
throw new ServiceException($"Failure retrieving OsgPreprocessorInfos.");
}
}
public async Task<OsgPreprocessorInfo> GetByIdAsync(int id)
{
try
{
var osgPreprocessorInfo = await _osgPreprocessorInfoRepository.FindByIdAsync(id);
return osgPreprocessorInfo;
}
catch (Exception exception)
{
Log.Error(exception, $"Failure retrieving OsgPreprocessorInfo with OsgPreprocessorInfoId {id}.");
throw new ServiceException($"Failure retrieving OsgPreprocessorInfo with OsgPreprocessorInfoId {id}.");
}
}
public async Task<IEnumerable<OsgPreprocessorInfo>> GetListByIdAsync(List<int> ids)
{
try
{
var osgPreprocessorInfo = await _osgPreprocessorInfoRepository.FindListByIdAsync(ids);
return osgPreprocessorInfo;
}
catch (Exception exception)
{
var message = $"Failure retrieving list of OsgPreprocessorInfos with OsgPreprocessorInfoIds: {String.Join(", ", ids.ToArray())}.";
Log.Error(exception, message);
throw new ServiceException(message);
}
}
public async Task<IEnumerable<OsgPreprocessorInfo>> GetByDocumentTypeAsync(string documentType)
{
try
{
var osgPreprocessorInfos = await _osgPreprocessorInfoRepository.FindByDocumentType(documentType);
return osgPreprocessorInfos;
}
catch (Exception exception)
{
Log.Error(exception, $"Failure retrieving OsgPreprocessorInfos with DocumentType: {documentType}.");
throw new ServiceException($"Failure retrieving OsgPreprocessorInfos with DocumentType: {documentType}.");
}
}
public async Task<IEnumerable<OsgPreprocessorInfo>> GetByMemberNumberAsync(string memberNumber)
{
try
{
var osgPreprocessorInfos = await _osgPreprocessorInfoRepository.FindByMemberNumber(memberNumber);
return osgPreprocessorInfos;
}
catch (Exception exception)
{
Log.Error(exception, $"Failure retrieving OsgPreprocessorInfos with MemberNumber: {memberNumber}.");
throw new ServiceException($"Failure retrieving OsgPreprocessorInfos with MemberNumber: {memberNumber}.");
}
}
public async Task<OsgPreprocessorInfo> SaveAsync(OsgPreprocessorInfo osgPreprocessorInfo)
{
try
{
var result = await _osgPreprocessorInfoRepository.AddAsync(osgPreprocessorInfo);
await _unitOfWork.CompleteAsync();
return result;
}
catch (Exception exception)
{
Log.Error(exception, $"Failure saving OsgPreprocessorInfo.");
throw new ServiceException($"Failure saving OsgPreprocessorInfo.");
}
}
public async Task SaveListAsync(List<OsgPreprocessorInfo> osgPreprocessorInfos)
{
try
{
await _osgPreprocessorInfoRepository.AddListAsync(osgPreprocessorInfos);
await _unitOfWork.CompleteAsync();
}
catch (Exception exception)
{
Log.Error(exception, $"Failure saving list of OsgPreprocessorInfo.");
throw new ServiceException($"Failure saving list OsgPreprocessorInfo.");
}
}
public async Task<OsgPreprocessorInfo?> Update(OsgPreprocessorInfo osgPreprocessorInfo)
{
try
{
var existing = await GetByIdAsync(osgPreprocessorInfo.OsgPreprocessorInfoId);
if (existing == null)
{
return existing;
}
existing.BatchRunInfoId = osgPreprocessorInfo.BatchRunInfoId;
existing.ZipFileName = osgPreprocessorInfo.ZipFileName;
existing.ZipFileStartTime = osgPreprocessorInfo.ZipFileStartTime;
existing.ZipFileEndTime = osgPreprocessorInfo.ZipFileEndTime;
existing.DocumentType = osgPreprocessorInfo.DocumentType;
existing.EntityName = osgPreprocessorInfo.EntityName;
existing.MemberNumber = osgPreprocessorInfo.MemberNumber;
existing.CreatedDate = osgPreprocessorInfo.CreatedDate;
existing.StartingPage = osgPreprocessorInfo.StartingPage;
existing.NumberOfPages = osgPreprocessorInfo.NumberOfPages;
existing.XmlOutput = osgPreprocessorInfo.XmlOutput;
_osgPreprocessorInfoRepository.Update(existing);
_unitOfWork.Complete();
return existing;
}
catch (SqlException sqlException)
{
Log.Error(sqlException, $"SQL Failure updating OsgPreprocessorInfo. " +
$"OsgPreprocessorInfoId: {osgPreprocessorInfo.OsgPreprocessorInfoId}");
throw new ServiceException($"SQL Failure updating DocumentProcessingInfo. " +
$"OsgPreprocessorInfoId: {osgPreprocessorInfo.OsgPreprocessorInfoId}");
}
catch (Exception exception)
{
Log.Error(exception, $"Failure updating OsgPreprocessorInfo. " +
$"OsgPreprocessorInfoId: {osgPreprocessorInfo.OsgPreprocessorInfoId}");
throw new ServiceException($"Failure updating OsgPreprocessorInfo. " +
$"OsgPreprocessorInfoId: {osgPreprocessorInfo.OsgPreprocessorInfoId}");
}
}
public async Task<IEnumerable<OsgPreprocessorInfo?>> UpdateRange(IEnumerable<OsgPreprocessorInfo> osgPreprocessorInfos)
{
try
{
var updateList = new List<OsgPreprocessorInfo>();
foreach (var osgPreprocessorInfo in osgPreprocessorInfos)
{
var existing = await GetByIdAsync(osgPreprocessorInfo.OsgPreprocessorInfoId);
if (existing == null)
{
continue;
}
existing.BatchRunInfoId = osgPreprocessorInfo.BatchRunInfoId;
existing.ZipFileName = osgPreprocessorInfo.ZipFileName;
existing.ZipFileStartTime = osgPreprocessorInfo.ZipFileStartTime;
existing.ZipFileEndTime = osgPreprocessorInfo.ZipFileEndTime;
existing.DocumentType = osgPreprocessorInfo.DocumentType;
existing.EntityName = osgPreprocessorInfo.EntityName;
existing.MemberNumber = osgPreprocessorInfo.MemberNumber;
existing.CreatedDate = osgPreprocessorInfo.CreatedDate;
existing.StartingPage = osgPreprocessorInfo.StartingPage;
existing.NumberOfPages = osgPreprocessorInfo.NumberOfPages;
existing.XmlOutput = osgPreprocessorInfo.XmlOutput;
updateList.Add(existing);
}
_osgPreprocessorInfoRepository.UpdateRange(updateList);
_unitOfWork.Complete();
return updateList;
}
catch (SqlException sqlException)
{
Log.Error(sqlException, $"SQL Failure updating list of OsgPreprocessorInfos.");
throw new ServiceException($"SQL Failure updating list of OsgPreprocessorInfos.");
}
catch (Exception exception)
{
Log.Error(exception, $"Failure updating list of OsgPreprocessorInfos.");
throw new ServiceException($"Failure updating list of OsgPreprocessorInfos.");
}
}
public async Task Delete(int id)
{
try
{
var osgPreprocessorInfo = await GetByIdAsync(id);
ArgumentNullException.ThrowIfNull(osgPreprocessorInfo);
_osgPreprocessorInfoRepository.Remove(osgPreprocessorInfo);
_unitOfWork.Complete();
}
catch (SqlException sqlException)
{
Log.Error(sqlException, $"SQL failure removing OsgPreprocessorInfo. OsgPreprocessorInfoId: {id}");
throw new ServiceException($"SQL failure removing OsgPreprocessorInfo. OsgPreprocessorInfoId: {id}");
}
catch (Exception exception)
{
Log.Error(exception, $"Failure removing OsgPreprocessorInfo. OsgPreprocessorInfoId: {id}");
throw new ServiceException($"Failure removing OsgPreprocessorInfo. OsgPreprocessorInfoId: {id}");
}
}
}
@@ -0,0 +1,97 @@
using LCU.API.Internal.DocumentIngestorLib.Repositories;
using LCU.Library.Internal.DocumentIngestorModels.Entities;
using LCU.Library.Internal.Exceptions;
using Serilog;
using System.Data.SqlClient;
namespace LCU.API.Internal.DocumentIngestor.Services;
public class OwnerOverrideInfoService : IOwnerOverrideInfoService
{
private readonly IUnitOfWork _unitOfWork;
private readonly IOwnerOverrideInfoRepository _ownerOverrideInfoRepository;
public OwnerOverrideInfoService(IUnitOfWork unitOfWork,
IOwnerOverrideInfoRepository ownerOverrideInfoRepository)
{
_unitOfWork = unitOfWork;
_ownerOverrideInfoRepository = ownerOverrideInfoRepository;
}
public async Task<IEnumerable<OwnerOverrideInfo>?> GetAsync()
{
try
{
var ownerOverrideInfos = await _ownerOverrideInfoRepository.FindAsync();
return ownerOverrideInfos;
}
catch (Exception exception)
{
Log.Error(exception, $"Failure retrieving OwnerOverrideInfos.");
throw new ServiceException($"Failure retrieving OwnerOverrideInfos. Exception Message: {exception.Message}");
}
}
public async Task<OwnerOverrideInfo?> GetByIdAsync(int id)
{
try
{
var ownerOverrideInfo = await _ownerOverrideInfoRepository.FindByIdAsync(id);
return ownerOverrideInfo;
}
catch (Exception exception)
{
Log.Error(exception, $"Failure retrieving OwnerOverrideInfo with OwnerOverrideInfoId {id}.");
throw new ServiceException($"Failure retrieving OwnerOverrideInfo with OwnerOverrideInfoId {id}. Exception Message: {exception.Message}");
}
}
public async Task<IEnumerable<OwnerOverrideInfo>?> GetByDocumentProcessingInfoIdAsync(int documentProcessingInfoId)
{
try
{
var ownerOverrideInfos = await _ownerOverrideInfoRepository.FindByDocumentProcessingInfoIdAsync(documentProcessingInfoId);
return ownerOverrideInfos;
}
catch (Exception exception)
{
Log.Error(exception, $"Failure retrieving OwnerOverrideInfos with DocumentProcessingInfoId {documentProcessingInfoId}.");
throw new ServiceException($"Failure retrieving OwnerOverrideInfos with DocumentProcessingInfoId {documentProcessingInfoId}. Exception Message: {exception.Message}");
}
}
public async Task<OwnerOverrideInfo> SaveAsync(OwnerOverrideInfo ownerOverrideInfo)
{
try
{
var result = await _ownerOverrideInfoRepository.AddAsync(ownerOverrideInfo);
await _unitOfWork.CompleteAsync();
return result;
}
catch (Exception exception)
{
Log.Error(exception, $"Failure saving OwnerOverrideInfo.");
throw new ServiceException($"Failure saving OwnerOverrideInfo.");
}
}
public async Task Delete(int id)
{
try
{
var ownerOverrideInfo = await GetByIdAsync(id);
ArgumentNullException.ThrowIfNull(ownerOverrideInfo);
_ownerOverrideInfoRepository.Remove(ownerOverrideInfo);
_unitOfWork.Complete();
}
catch (SqlException sqlException)
{
Log.Error(sqlException, $"SQL failure removing OwnerOverrideInfo. OwnerOverrideInfoId: {id}");
throw new ServiceException($"SQL failure removing OwnerOverrideInfo. OwnerOverrideInfoId: {id}");
}
catch (Exception exception)
{
Log.Error(exception, $"Failure removing OwnerOverrideInfo. OwnerOverrideInfoId: {id}");
throw new ServiceException($"Failure removing OwnerOverrideInfo. OwnerOverrideInfoId: {id}");
}
}
}
@@ -0,0 +1,135 @@
using Serilog;
using LCU.Library.Internal.Exceptions;
using LCU.Library.Internal.DocumentIngestorModels.Models.IndexFileDocuments;
using LCU.API.Internal.DocumentIngestor.Services.MicroApiServices;
using LCU.API.Internal.DocumentIngestor.Services.IndexParserServices;
using LCU.Library.Internal.DocumentIngestorModels.Enums;
using LCU.Library.Internal.FileOperationsModels.Models;
using LCU.Library.Internal.LCUServiceFactory.Configurations;
using LCU.Library.Internal.LCUServiceFactory.Services;
namespace LCU.API.Internal.DocumentIngestor.Services.PdfParserServices;
public class FosLoanStatementPdfService : PdfParserService, IPdfParserService
{
private readonly ILcuServiceFactory<IPdfIndexParserService> _indexParserServiceFactory;
private readonly IFileOperationsApiService _fileOperationsApiService;
public FosLoanStatementPdfService(IFileOperationsApiService fileOperationsApiService,
ILcuServiceFactory<IPdfIndexParserService> indexParserServiceFactory,
ServicesFactoryConfiguration servicesFactoryConfiguration)
: base(servicesFactoryConfiguration)
{
_indexParserServiceFactory = indexParserServiceFactory;
_fileOperationsApiService = fileOperationsApiService;
}
public async Task<BatchParsePdfRequest> DetermineBatchParsePdfRequest(string rootPath, string indexFile, string pdfName)
{
try
{
var indexFilePath = rootPath + indexFile;
var records = await GetFileContentAndParseRecords(indexFilePath);
var batchParsePdfRequest = await GetParsedPdfFiles(records, rootPath, pdfName);
return batchParsePdfRequest;
}
catch (Exception exception)
{
var message = $"An Exception occured while DetermineBatchParsePdfRequest. " +
$"IndexFile:{indexFile}, RootPath:{rootPath}, MainPdfName:{pdfName}";
Log.Error(exception, message);
throw new ServiceException($"{message}, ExceptionMessage:{exception.Message}");
}
}
private async Task<List<FosLoanStatement>> GetFileContentAndParseRecords(string indexFilePath)
{
var indexParserService = _indexParserServiceFactory.GetService(IngestorTypes.FosLoanStatement);
if (indexParserService is null)
{
throw new NotImplementedException();
}
return await indexParserService.GetFileContentAndParseRecords<FosLoanStatement>(indexFilePath);
}
public async Task<BatchParsePdfRequest> GetParsedPdfFiles(List<FosLoanStatement> records, string rootPath, string pdfName)
{
await ValidatePdfsCanBeParsed(records, pdfName, rootPath);
var batchParsePdfRequest = CreateBatchParsePdfRequest(records);
return batchParsePdfRequest;
}
private async Task ValidatePdfsCanBeParsed(List<FosLoanStatement> records,
string pdfName,
string rootPath)
{
var maxPage = records.Select(record =>
{
if (!int.TryParse(record.StartingPage, out int startingPage) || !int.TryParse(record.NumberOfPages, out int numberOfPages))
{
var message = $"'StartingPage' or 'NumberOfPages' for {pdfName} could not be parsed to int." +
$" StartingPage: {record.StartingPage} | NumberOfPages: {record.NumberOfPages}";
Log.Error(message);
throw new ServiceException(message);
}
return startingPage + (numberOfPages - 1);
}
).OrderByDescending(x => x).FirstOrDefault();
var pdfFullPath = string.Concat(rootPath, pdfName);
var pdfPageExists = await _fileOperationsApiService.ValidatePdfPageExists(pdfFullPath, maxPage.ToString());
if (!pdfPageExists.IsSuccessful)
{
var message = $"Page does not exist for {pdfName} at EndingPage: ({maxPage}). MessageStack: {String.Join("; ", pdfPageExists.MessageStack.ToArray())}";
Log.Error(message);
throw new ServiceException(message);
}
}
private BatchParsePdfRequest CreateBatchParsePdfRequest(List<FosLoanStatement> records)
{
var batchParsePdfRequest = new BatchParsePdfRequest
{
BatchPdfs = new List<BatchPdf>()
};
foreach (var record in records)
{
if (!int.TryParse(record.StartingPage, out int startingPage) || !int.TryParse(record.NumberOfPages, out int numberOfPages))
{
var message = $"'StartingPage' or 'NumberOfPages' for {record.DocName} could not be parsed to int." +
$" StartingPage: {record.StartingPage} | NumberOfPages: {record.NumberOfPages}";
Log.Error(message);
throw new ServiceException(message);
}
var endingPage = startingPage + (numberOfPages - 1);
var parsedPdfName = GetParsedPdfName(record);
var batchPdf = new BatchPdf()
{
PageStart = startingPage,
PageStop = endingPage,
ParsedPdfname = Path.GetFileNameWithoutExtension(parsedPdfName)
};
batchParsePdfRequest.BatchPdfs.Add(batchPdf);
}
return batchParsePdfRequest;
}
private string GetParsedPdfName(FosLoanStatement fosLoanStatement)
{
var indexParserService = _indexParserServiceFactory.GetService(IngestorTypes.FosLoanStatement);
if (indexParserService is null)
{
throw new NotImplementedException();
}
return indexParserService.CreateParsedPdfName(fosLoanStatement);
}
}
@@ -0,0 +1,11 @@
using LCU.Library.Internal.FileOperationsModels.Models;
using LCU.Library.Internal.LCUServiceFactory.Services;
namespace LCU.API.Internal.DocumentIngestor.Services.PdfParserServices;
public interface IPdfParserService : ILcuService
{
Task<BatchParsePdfRequest> DetermineBatchParsePdfRequest(string rootPath,
string indexFile,
string pdfName);
}

Some files were not shown because too many files have changed in this diff Show More