using System; using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; using Serilog; using Serilog.Events; using Moq; using NUnit.Framework; using LCU.API.Internal.DocumentIngestor.Configurations; using LCU.API.Internal.DocumentIngestor.Services; using LCU.API.Internal.DocumentIngestor.Services.QueueServices; using LCU.Library.Internal.DocumentIngestorModels.Entities; using LCU.Library.Internal.DocumentIngestorModels.Enums; using LCU.Library.Internal.DocumentIngestorModels.Models; using LCU.Library.Internal.Exceptions; using Newtonsoft.Json; namespace LCU.API.Internal.DocumentIngestorTests.Services; public class DocumentReprocessingServiceTest { private DocumentReprocessingService _documentReprocessingService; private Mock _mockIngestorRunInfoService; private Mock _mockDocumentProcessingInfoService; private Mock _mockQueueDocumentProcessing; private Mock _mockOwnerOverrideInfoService; private ApiConfiguration _apiConfiguration; private readonly Mock _logger = new(); private readonly ILogger _originalLogger = Log.Logger; [SetUp] public void Setup() { Log.Logger = _logger.Object; _mockIngestorRunInfoService = new Mock(); _mockDocumentProcessingInfoService = new Mock(); _mockQueueDocumentProcessing = new Mock(); _mockOwnerOverrideInfoService = new Mock(); _apiConfiguration = new ApiConfiguration(); _documentReprocessingService = new DocumentReprocessingService(_mockIngestorRunInfoService.Object, _mockDocumentProcessingInfoService.Object, _mockOwnerOverrideInfoService.Object, _mockQueueDocumentProcessing.Object, _apiConfiguration); } [TearDown] public void TearDown() { Log.Logger = _originalLogger; } [Test] public async Task QueueReprocessDocumentByDocumentProcessingInfoId_Success() { var documentProcessingInfoId = 1; var userName = "Test Tester"; var ingestorRunInfoId = new Guid(); var documentProcessingInfo = new DocumentProcessingInfo() { DocumentProcessingInfoId = documentProcessingInfoId, DocumentFileName = "TestDoc", DocumentValuePayload = "{jsonblah: hello}", IngestorRunInfoId = ingestorRunInfoId, CreatedDate = new DateTime(2022, 01, 01), StatusType = ProcessStatusTypes.ArchiveFailure }; var ingestorRunInfo = new IngestorRunInfo() { IndexFileName = "Test.txt", IndexFilePath = "\\\\Test\\Test\\", IngestorType = IngestorTypes.FosLoanStatement }; _mockDocumentProcessingInfoService.Setup(x => x.GetByIdAsync(documentProcessingInfoId)).ReturnsAsync(documentProcessingInfo); _mockIngestorRunInfoService.Setup(x => x.GetByIdAsync(ingestorRunInfoId)).ReturnsAsync(ingestorRunInfo); await _documentReprocessingService.QueueReprocessDocumentByDocumentProcessingInfoId(documentProcessingInfoId, userName); _mockDocumentProcessingInfoService.Verify(x => x.GetByIdAsync(documentProcessingInfoId), Times.Once); _mockIngestorRunInfoService.Verify(x => x.GetByIdAsync(It.IsAny()), Times.Once); _mockDocumentProcessingInfoService.Verify(x => x.UpdateStatusAndAddLog(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), null), Times.Once); _mockQueueDocumentProcessing.Verify(x => x.QueueDocumentProcessingRequest(It.IsAny()), Times.Once); } [Test] public void QueueReprocessDocumentByDocumentProcessingInfoId_Failure_Exception() { var id = 1; var userName = "Test Tester"; var ingestorRequest = new IngestorRequest() { RootPath = "\\Test\\Test" }; var request = new ProcessingRequest() { DocumentFileName = "000001Test.pdf", IngestorRequest = ingestorRequest }; _mockQueueDocumentProcessing.Setup(x => x.QueueDocumentProcessingRequest(request)).ThrowsAsync(new Exception("Test Error")); Assert.ThrowsAsync(async () => await _documentReprocessingService .QueueReprocessDocumentByDocumentProcessingInfoId(id, userName)); _mockDocumentProcessingInfoService.Verify(x => x.GetByIdAsync(id), Times.Once); _mockQueueDocumentProcessing.Verify(x => x.QueueDocumentProcessingRequest(It.IsAny()), Times.Never); } [Test] public void QueueReprocessDocumentByDocumentProcessingInfoId_Failure_ArgumentNullException() { var id = 1; var userName = "Test Tester"; _mockDocumentProcessingInfoService.Setup(x => x.GetByIdAsync(id)).ThrowsAsync(new ArgumentNullException("Test Error")); Assert.ThrowsAsync(async () => await _documentReprocessingService .QueueReprocessDocumentByDocumentProcessingInfoId(id, userName)); _mockDocumentProcessingInfoService.Verify(x => x.GetByIdAsync(id), Times.Once); _mockIngestorRunInfoService.Verify(x => x.GetByIdAsync(It.IsAny()), Times.Never); _mockDocumentProcessingInfoService.Verify(x => x.UpdateStatusAndAddLog(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), false), Times.Never); _mockQueueDocumentProcessing.Verify(x => x.QueueDocumentProcessingRequest(It.IsAny()), Times.Never); } private OwnerOverrideInfo OverrideFactory() { var info = new OwnerOverrideInfo { UserName = Guid.NewGuid().ToString(), EntityId = Guid.NewGuid().ToString(), EntityType = Random(), DocumentProcessingInfoId = DateTime.Now.Millisecond }; _mockOwnerOverrideInfoService .Setup(s => s.GetByDocumentProcessingInfoIdAsync(info.DocumentProcessingInfoId)) .ReturnsAsync(new[] { info }); return info; } [Test] public async Task QueueOverrideDocumentByDocumentProcessingInfoId_AllCases_SavesOwnerOverrides() { var ownerOverride = OverrideFactory(); try { await _documentReprocessingService.QueueOverrideDocumentByDocumentProcessingInfoId(ownerOverride); } catch { } _mockOwnerOverrideInfoService.Verify(s => s.SaveAsync(ownerOverride), Times.Once); } [Test] public void QueueOverrideDocumentByDocumentProcessingInfoId_OverrideSaveThrowsException_LogsAndThrowsServiceException() { var ownerOverride = OverrideFactory(); var saveException = new Exception(); var loggedErrors = new List(); _mockOwnerOverrideInfoService .Setup(s => s.SaveAsync(ownerOverride)) .ThrowsAsync(saveException); _logger .Setup(l => l.Write(LogEventLevel.Error, saveException, It.IsAny())) .Callback((_, _, log) => loggedErrors.Add(log)); var thrownException = Assert.ThrowsAsync(() => _documentReprocessingService.QueueOverrideDocumentByDocumentProcessingInfoId(ownerOverride)); StringAssert.Contains(ownerOverride.DocumentProcessingInfoId.ToString(), thrownException!.Message); Assert.AreEqual(1, loggedErrors.Count); var errorLog = loggedErrors.Single(); StringAssert.Contains(ownerOverride.DocumentProcessingInfoId.ToString(), errorLog); StringAssert.Contains(ownerOverride.EntityId, errorLog); StringAssert.Contains(ownerOverride.EntityType.ToString(), errorLog); } [Test] public async Task QueueOverrideDocumentByDocumentProcessingInfoId_OverrideSaveThrowsException_DoesNotRequeue() { var ownerOverride = OverrideFactory(); _mockOwnerOverrideInfoService .Setup(s => s.SaveAsync(ownerOverride)) .ThrowsAsync(new Exception()); try { await _documentReprocessingService.QueueOverrideDocumentByDocumentProcessingInfoId(ownerOverride); } catch { } _mockQueueDocumentProcessing.Verify(p => p.QueueDocumentProcessingRequest(It.IsAny()), Times.Never); } [Test] public void QueueOverrideDocumentByDocumentProcessingInfoId_FetchedDocumentInfoNull_LogsAndThrowsArgumentNull() { var ownerOverride = SetupSave(OverrideFactory()); var loggedErrors = new List<(Exception exception, string log)>(); _logger .Setup(l => l.Write(LogEventLevel.Error, It.IsAny(), It.IsAny())) .Callback((_, exception, log) => loggedErrors.Add((exception, log))); _mockDocumentProcessingInfoService .Setup(s => s.GetByIdAsync(ownerOverride.DocumentProcessingInfoId)) .ReturnsAsync((DocumentProcessingInfo)null); var thrownException = Assert.ThrowsAsync(() => _documentReprocessingService.QueueOverrideDocumentByDocumentProcessingInfoId(ownerOverride)); Assert.AreEqual(1, loggedErrors.Count); var (loggedException, log) = loggedErrors.Single(); Assert.AreSame(loggedException, thrownException); StringAssert.Contains(ownerOverride.DocumentProcessingInfoId.ToString(), log); } [Test] public async Task QueueOverrideDocumentByDocumentProcessingInfoId_FetchedDocumentInfoNull_DoesNotRequeue() { var ownerOverride = SetupSave(OverrideFactory()); _mockDocumentProcessingInfoService .Setup(s => s.GetByIdAsync(ownerOverride.DocumentProcessingInfoId)) .ReturnsAsync((DocumentProcessingInfo)null); try { await _documentReprocessingService.QueueOverrideDocumentByDocumentProcessingInfoId(ownerOverride); } catch { } _mockQueueDocumentProcessing.Verify(p => p.QueueDocumentProcessingRequest(It.IsAny()), Times.Never); } [Test] public void QueueOverrideDocumentByDocumentProcessingInfoId_FetchedIngestorRunNull_LogsAndThrowsArgumentNull() { var ownerOverride = SetupSave(OverrideFactory()); var info = SetupGetDocumentProcessingInfo(ownerOverride); var loggedErrors = new List<(Exception exception, string log)>(); _logger .Setup(l => l.Write(LogEventLevel.Error, It.IsAny(), It.IsAny())) .Callback((_, exception, log) => loggedErrors.Add((exception, log))); _mockIngestorRunInfoService .Setup(s => s.GetByIdAsync(info.IngestorRunInfoId)) .ReturnsAsync((IngestorRunInfo)null); var thrownException = Assert.ThrowsAsync(() => _documentReprocessingService.QueueOverrideDocumentByDocumentProcessingInfoId(ownerOverride)); Assert.AreEqual(1, loggedErrors.Count); var (loggedException, log) = loggedErrors.Single(); Assert.AreSame(loggedException, thrownException); StringAssert.Contains(ownerOverride.DocumentProcessingInfoId.ToString(), log); } [Test] public async Task QueueOverrideDocumentByDocumentProcessingInfoId_FetchedIngestorRunNull_DoesNotRequeue() { var ownerOverride = SetupSave(OverrideFactory()); var info = SetupGetDocumentProcessingInfo(ownerOverride); _mockIngestorRunInfoService .Setup(s => s.GetByIdAsync(info.IngestorRunInfoId)) .ReturnsAsync((IngestorRunInfo)null); try { await _documentReprocessingService.QueueOverrideDocumentByDocumentProcessingInfoId(ownerOverride); } catch { } _mockQueueDocumentProcessing.Verify(p => p.QueueDocumentProcessingRequest(It.IsAny()), Times.Never); } [Test] public async Task QueueOverrideDocumentByDocumentProcessingInfoId_SaveSucceeds_UpdatesStatusAndLogs() { var ownerOverride = SetupSave(OverrideFactory()); var documentInfo = SetupGetDocumentProcessingInfo(ownerOverride); SetupGetIngestorRunInfo(documentInfo); await _documentReprocessingService.QueueOverrideDocumentByDocumentProcessingInfoId(ownerOverride); _mockDocumentProcessingInfoService.Verify ( s => s.UpdateStatusAndAddLog ( ownerOverride.DocumentProcessingInfoId, ProcessStatusTypes.Reprocessing, It.Is(s => s.Contains(ownerOverride.UserName) && s.Contains(JsonConvert.SerializeObject(ownerOverride))), ownerOverride.UserName, It.IsAny() ), Times.Once ); } [Test] public void QueueOverrideDocumentByDocumentProcessingInfoId_UpdateStatusThrowsException_LogsAndThrowsServiceException() { var ownerOverride = SetupSave(OverrideFactory()); var documentInfo = SetupGetDocumentProcessingInfo(ownerOverride); SetupGetIngestorRunInfo(documentInfo); var exceptionToThrow = new Exception(); var loggedErrors = new List(); _logger .Setup(l => l.Write(LogEventLevel.Error, It.IsAny(), It.IsAny())) .Callback((_, _, log) => loggedErrors.Add(log)); _mockDocumentProcessingInfoService .Setup ( s => s.UpdateStatusAndAddLog ( ownerOverride.DocumentProcessingInfoId, ProcessStatusTypes.Reprocessing, It.Is(s => s.Contains(ownerOverride.UserName) && s.Contains(JsonConvert.SerializeObject(ownerOverride))), ownerOverride.UserName, null ) ) .ThrowsAsync(exceptionToThrow); var thrownException = Assert.ThrowsAsync(() => _documentReprocessingService.QueueOverrideDocumentByDocumentProcessingInfoId(ownerOverride)); StringAssert.Contains(ownerOverride.DocumentProcessingInfoId.ToString(), thrownException!.Message); Assert.AreEqual(1, loggedErrors.Count); var errorLog = loggedErrors.Single(); StringAssert.Contains(ownerOverride.DocumentProcessingInfoId.ToString(), errorLog); StringAssert.Contains(ownerOverride.EntityId, errorLog); StringAssert.Contains(ownerOverride.EntityType.ToString(), errorLog); } [Test] public async Task QueueOverrideDocumentByDocumentProcessingInfoId_UpdateStatusThrowsException_DoesNotRequeue() { var ownerOverride = SetupSave(OverrideFactory()); var documentInfo = SetupGetDocumentProcessingInfo(ownerOverride); SetupGetIngestorRunInfo(documentInfo); _mockDocumentProcessingInfoService .Setup ( s => s.UpdateStatusAndAddLog ( ownerOverride.DocumentProcessingInfoId, ProcessStatusTypes.Reprocessing, It.Is(s => s.Contains(ownerOverride.UserName) && s.Contains(JsonConvert.SerializeObject(ownerOverride))), ownerOverride.UserName, null ) ) .ThrowsAsync(new Exception()); try { await _documentReprocessingService.QueueOverrideDocumentByDocumentProcessingInfoId(ownerOverride); } catch { } _mockQueueDocumentProcessing.Verify(p => p.QueueDocumentProcessingRequest(It.IsAny()), Times.Never); } [Test] public async Task QueueOverrideDocumentByDocumentProcessingInfoId_SaveSucceeds_RequeuesProcessing() { var ownerOverride = SetupSave(OverrideFactory()); var documentInfo = SetupGetDocumentProcessingInfo(ownerOverride); var ingestorInfo = SetupGetIngestorRunInfo(documentInfo); var requeuedRequests = new List(); _mockQueueDocumentProcessing .Setup(p => p.QueueDocumentProcessingRequest(It.IsAny())) .Callback(requeuedRequests.Add); await _documentReprocessingService.QueueOverrideDocumentByDocumentProcessingInfoId(ownerOverride); Assert.AreEqual(1, requeuedRequests.Count); var requeued = requeuedRequests.Single(); Assert.AreEqual(requeued.DocumentProcessingInfoId, ownerOverride.DocumentProcessingInfoId); Assert.AreEqual(requeued.IngestorRunInfoId, documentInfo.IngestorRunInfoId); Assert.AreEqual(requeued.DocumentFileName, documentInfo.DocumentFileName); Assert.AreEqual(requeued.DocumentValuePayload, documentInfo.DocumentValuePayload); Assert.AreEqual(requeued.IngestorRequest.IngestorType, ingestorInfo.IngestorType); Assert.AreEqual(requeued.IngestorRequest.RootPath, ingestorInfo.IndexFilePath + "\\"); Assert.AreEqual(requeued.IngestorRequest.IndexFileName, ingestorInfo.IndexFileName); Assert.AreSame(requeued.OwnerOverrideInfo, ownerOverride); } [Test] public void QueueOverrideDocumentByDocumentProcessingInfoId_RequeueThrowsException_LogsAndThrowsServiceException() { var ownerOverride = SetupSave(OverrideFactory()); var documentInfo = SetupGetDocumentProcessingInfo(ownerOverride); SetupGetIngestorRunInfo(documentInfo); var exceptionToThrow = new Exception(); var loggedErrors = new List(); _logger .Setup(l => l.Write(LogEventLevel.Error, It.IsAny(), It.IsAny())) .Callback((_, _, log) => loggedErrors.Add(log)); _mockQueueDocumentProcessing .Setup(p => p.QueueDocumentProcessingRequest(It.IsAny())) .Throws(exceptionToThrow); var thrownException = Assert.ThrowsAsync(() => _documentReprocessingService.QueueOverrideDocumentByDocumentProcessingInfoId(ownerOverride)); StringAssert.Contains(ownerOverride.DocumentProcessingInfoId.ToString(), thrownException!.Message); Assert.AreEqual(1, loggedErrors.Count); var errorLog = loggedErrors.Single(); StringAssert.Contains(ownerOverride.DocumentProcessingInfoId.ToString(), errorLog); StringAssert.Contains(ownerOverride.EntityId, errorLog); StringAssert.Contains(ownerOverride.EntityType.ToString(), errorLog); } [Test] public void QueueReprocessAllCriticalFailures_InfoServiceThrowsArgumentNull_ThrowsServiceException() { var processStatus = ProcessStatusTypes.CriticalFailure; _mockDocumentProcessingInfoService .Setup(s => s.GetByProcessStatusTypeAsync(processStatus, null)) .ThrowsAsync(new ArgumentNullException()); Assert.ThrowsAsync(() => _documentReprocessingService.QueueReprocessAllCriticalFailures(Guid.NewGuid().ToString())); } [Test] public async Task QueueReprocessAllCriticalFailures_InfoServiceThrows_ThrowsServiceException() { var processStatus = ProcessStatusTypes.CriticalFailure; _mockDocumentProcessingInfoService .Setup(s => s.GetByProcessStatusTypeAsync(processStatus, null)) .ThrowsAsync(new Exception()); Assert.ThrowsAsync(() => _documentReprocessingService.QueueReprocessAllCriticalFailures(Guid.NewGuid().ToString())); } [Test] public async Task QueueReprocessAllCriticalFailures_InfoServiceReturnsMoreThanLimit_RequeuesUpToLimit() { var processStatus = ProcessStatusTypes.CriticalFailure; var username = Guid.NewGuid().ToString(); var criticalFailures = Enumerable .Repeat(() => SetupGetDocumentProcessingInfo(OverrideFactory()), 16) .Select(f => f()) .ToArray(); foreach (var info in criticalFailures) SetupGetIngestorRunInfo(info); _apiConfiguration.CriticalFailureReprocessLimit = criticalFailures.Length / 2; _mockDocumentProcessingInfoService .Setup(s => s.GetByProcessStatusTypeAsync(processStatus, null)) .ReturnsAsync(criticalFailures); await _documentReprocessingService.QueueReprocessAllCriticalFailures(username); _mockDocumentProcessingInfoService .Verify ( s => s.UpdateStatusAndAddLog ( It.Is(id => criticalFailures.Any(f => f.DocumentProcessingInfoId == id)), ProcessStatusTypes.Reprocessing, It.IsAny(), username, null ), Times.Exactly(_apiConfiguration.CriticalFailureReprocessLimit) ); _mockQueueDocumentProcessing .Verify ( dp => dp.QueueDocumentProcessingRequest ( It.Is(r => criticalFailures.Any(f => f.DocumentProcessingInfoId == r.DocumentProcessingInfoId)) ), Times.Exactly(_apiConfiguration.CriticalFailureReprocessLimit) ); } private OwnerOverrideInfo SetupSave(OwnerOverrideInfo ownerOverride) { _mockOwnerOverrideInfoService .Setup(s => s.SaveAsync(ownerOverride)) .ReturnsAsync(ownerOverride); return ownerOverride; } private DocumentProcessingInfo SetupGetDocumentProcessingInfo(OwnerOverrideInfo ownerOverride) { var info = new DocumentProcessingInfo { IngestorRunInfoId = Guid.NewGuid(), CreatedDate = DateTime.Now, DocumentValuePayload = Guid.NewGuid().ToString(), StatusType = Random(), DocumentFileName = Guid.NewGuid().ToString(), ProcessCount = DateTime.Now.Millisecond, DocumentType = Guid.NewGuid().ToString(), DocumentBusinessName = Guid.NewGuid().ToString(), DocumentProcessingInfoId = ownerOverride.DocumentProcessingInfoId }; _mockDocumentProcessingInfoService .Setup(s => s.GetByIdAsync(ownerOverride.DocumentProcessingInfoId)) .ReturnsAsync(info); return info; } private IngestorRunInfo SetupGetIngestorRunInfo(DocumentProcessingInfo ownerOverride) { var info = new IngestorRunInfo { IngestorRunInfoId = Guid.NewGuid(), StartTimeStamp = DateTime.Now, IngestorType = Random(), IndexFileName = Guid.NewGuid().ToString(), IndexFilePath = Guid.NewGuid().ToString(), IndexTotal = DateTime.Now.Millisecond }; _mockIngestorRunInfoService .Setup(s => s.GetByIdAsync(ownerOverride.IngestorRunInfoId)) .ReturnsAsync(info); return info; } private static T Random() where T : struct, Enum => Enum.GetValues()[DateTime.Now.Millisecond % (Enum.GetValues().Length - 1)]; }