Adding

2026-03-24 15:26:03 -05:00
parent 1e5beb292a
commit 9cc87337b8
333 changed files with 17931 additions and 0 deletions
@@ -0,0 +1,72 @@
+using LCU.Api.Internal.Dispatcher.Services.QueueServices;
+using LCU.Api.Internal.MaidBotLib.Models;
+using LCU.Library.Internal.DispatcherModels.Enums;
+using LCU.Library.Internal.DispatcherModels.Models.Requests;
+using System.Collections.Generic;
+using System.Threading;
+namespace LCU.Api.Internal.Dispatcher.Services.DispatcherOperationServices;
+
+public class BatchErroredJobsService : IBatchErroredJobsService
+{
+    private readonly IQueueSendErroredJobAlertService _queueSendErroredJobAlertService;
+    private readonly IJobService _jobService;
+
+    public BatchErroredJobsService(IJobService jobService,
+                                  IQueueSendErroredJobAlertService queueSendErroredJobAlertService)
+    {
+        _jobService = jobService;
+        _queueSendErroredJobAlertService = queueSendErroredJobAlertService;
+    }
+
+    public async Task PerformBatchErroredJobs(CancellationToken cancellationToken)
+    {
+        try
+        {
+            var jobs = await _jobService.GetErrorMonitoringJobs();
+            if (jobs == null || !jobs.Any())
+            {
+                return;
+            }
+
+            var batchedErroredJobAlertRequest = CreateBatchedErroredJobAlerts(jobs);
+
+            await _queueSendErroredJobAlertService.QueueSendErroredJobAlert(batchedErroredJobAlertRequest);
+        }
+        catch (Exception ex)
+        {
+            //Fatal as we need to know if we can't get out our errors
+            Log.ForContext(nameof(PropertyEnum.Group),
+                           nameof(GroupEnum.ITDevelopment))
+                .Fatal(ex, $"Failed to perform Error Monitoring.");
+        }
+    }
+
+    private static BatchedErroredJobAlertRequest CreateBatchedErroredJobAlerts(IEnumerable<Job> jobs)
+    {
+        var batchedErroredJobAlertRequests = new BatchedErroredJobAlertRequest()
+        {
+            ErroredJobAlertRequests = new List<ErroredJobAlertRequest>()
+        };
+
+        foreach (var job in jobs)
+        {
+            var failedOrProcessingWorkItem = job.WorkItems
+                                 .FirstOrDefault(wi => wi.WorkItemExecutions != null &&
+                                 wi.WorkItemExecutions.Any(wie => wie.Status == StatusType.Failed || wie.Status == StatusType.Processing));
+
+            if (failedOrProcessingWorkItem == null || failedOrProcessingWorkItem.CurrentWorkItemExecution == null)
+            {
+                Log.Warning($"Failed or Processing workItem not found during CreateBatchedErroredJobAlerts. JobId and Name: {job.JobId}: {job.Name}");
+                continue;
+            }
+
+            batchedErroredJobAlertRequests.ErroredJobAlertRequests.Add(new ErroredJobAlertRequest
+            {
+                CurrentJob = job,
+                CurrentWorkItem = failedOrProcessingWorkItem
+            });
+        }
+
+        return batchedErroredJobAlertRequests;
+    }
+}
@@ -0,0 +1,172 @@
+using LCU.Api.Internal.Dispatcher.Configurations;
+using LCU.Api.Internal.DispatcherLib.Repositories;
+using LCU.Api.Internal.MaidBotLib.Models;
+using LCU.Library.Internal.DispatcherModels.Models.Results;
+using Newtonsoft.Json;
+using System.Threading;
+
+namespace LCU.Api.Internal.Dispatcher.Services.DispatcherOperationServices;
+
+public class CleanUpJobsService : ICleanUpJobsService
+{
+    private readonly IJobService _jobService;
+    private readonly ICleanUpJobSettingService _cleanUpJobSettingService;
+    private readonly IUnitOfWork _unitOfWork;
+    private readonly ApiConfiguration _apiConfiguration;
+
+    public CleanUpJobsService(IJobService jobService,
+                                  ICleanUpJobSettingService cleanUpJobSettingService,
+                                  IUnitOfWork unitOfWork,
+                                  ApiConfiguration apiConfiguration)
+    {
+        _jobService = jobService;
+        _cleanUpJobSettingService = cleanUpJobSettingService;
+        _unitOfWork = unitOfWork;
+        _apiConfiguration = apiConfiguration;
+    }
+
+    public async Task PerformCleanUpJobs(CancellationToken cancellationToken)
+    {
+        try
+        {
+            var result = new CleanUpJobBatchResult();
+
+            result = await PerformSoftCleanUpJobTask(result);
+            result = await PerformHardDeleteCleanUpJobTask(result);
+
+            await _unitOfWork.CompleteAsync();
+
+            if (result.IsFailure)
+            {
+                Log.ForContext(nameof(PropertyEnum.Group),
+                               nameof(GroupEnum.ITDevelopment))
+                    .Error("CleanUpJob Batch Result: {@CleanUpJobBatchResult}", result);
+            }
+        }
+        catch (Exception ex)
+        {
+            Log.ForContext(nameof(PropertyEnum.Group),
+                           nameof(GroupEnum.ITDevelopment))
+                .Error(ex, $"Failed to perform Clean Up Jobs.");
+        }
+    }
+
+    public async Task<CleanUpJobBatchResult> PerformSoftCleanUpJobTask(CleanUpJobBatchResult result)
+    {
+        var jobs = await _jobService.GetInactiveExcludeSoftDeletedTrackedAsync();
+
+        var cleanUpJobSettings = await _cleanUpJobSettingService.GetAsync();
+
+        foreach (var setting in cleanUpJobSettings)
+        {
+            if (setting.CleanUpJobTags == null || !setting.CleanUpJobTags.Any())
+            {
+                continue;
+            }
+
+            foreach (var job in jobs)
+            {
+                result = PerformCleanUpTaskForJob(result, job, setting);
+            }
+        }
+
+        return result;
+    }
+
+    public bool ValidateShouldSoftDelete(CleanUpJobSetting setting, Job job, CleanUpJobBatchResult result)
+    {
+        var shouldSoftDelete = ShouldSoftDeleteByTag(setting, job);
+        
+        if (!shouldSoftDelete)
+        {
+            return false;
+        }
+
+        return ShouldSoftDeleteByDate(setting, job, result);
+    }
+
+    public bool ShouldSoftDeleteByDate(CleanUpJobSetting setting, Job job, CleanUpJobBatchResult result)
+    {
+        try
+        {
+			if (!job.WorkItems.Any() || !job.WorkItems.FirstOrDefault().WorkItemExecutions.Any())
+			{
+				result.AddToFailureCount($"Job would be cleaned but cannot determine if it is old enough to be cleaned due to not having any workItemExecutions. Job: {job.JobId} - {job.Name}");
+				return false;
+			}
+
+			var currentDate = DateTime.Now;
+            var cutoffDate = currentDate.AddDays(setting.CompletionDaysPast);
+            var jobLastCompletedTime = job.WorkItems.FirstOrDefault()
+                                            .WorkItemExecutions.OrderByDescending(wie => wie.CompletedTimestamp).FirstOrDefault().CompletedTimestamp.Value;
+
+            return DateTime.Compare(cutoffDate, jobLastCompletedTime) > 0;
+        }
+        catch (Exception ex)
+        {
+            var message = $"Error while attempting to compare CompletedTimestamp for SoftDeletion. Job: {job.JobId} - {job.Name}. ExeptionMessage: {ex.Message}";
+            Log.Error(message);
+            throw new Exception(message);
+        }
+    }
+
+    public bool ShouldSoftDeleteByTag(CleanUpJobSetting setting, Job job)
+    {
+        if (job.JobTags.Count != setting.CleanUpJobTags.Count)
+        {
+            return false;
+        }
+
+        var orderedJobTagIds = job.JobTags.Select(t => t.TagId).OrderBy(t => t).ToList();
+        var orderedCleanUpJobTagIds = setting.CleanUpJobTags.Select(t => t.TagId).OrderBy(t => t).ToList();
+
+        return orderedJobTagIds.SequenceEqual(orderedCleanUpJobTagIds);
+    }
+
+    public async Task<CleanUpJobBatchResult> PerformHardDeleteCleanUpJobTask(CleanUpJobBatchResult result)
+    {
+        var softDeletedJobs = await _jobService.GetSoftDeletedTrackedAsync(DateTime.Now.AddDays(_apiConfiguration.CleanUpJobsDaysToDelete));
+        foreach (var job in softDeletedJobs)
+        {
+            try
+            {
+                await _jobService.DeleteFromCleanupAsync(job);
+            }
+            catch (Exception ex)
+            {
+                Log.Error(ex.Message, "Failure to hard delete dispatcher job through Clean Up Job Task. Job: {@Job}", job);
+                result.AddToFailureCount(ex.Message);
+                continue;
+            }
+        }
+
+        return result;
+    }
+
+    private CleanUpJobBatchResult PerformCleanUpTaskForJob(CleanUpJobBatchResult result, Job job, CleanUpJobSetting setting)
+    {
+        try
+        {
+            var shouldSoftDelete = ValidateShouldSoftDelete(setting, job, result);
+
+            if (!shouldSoftDelete)
+            {
+                return result;
+            }
+
+            job.SoftDeleteDate = DateTime.Now;
+
+            var updatedJob = _jobService.UpdateExcludingWorkItemsAsyncWithoutSave(job);
+            if (updatedJob == null || updatedJob.Result.JobId != job.JobId)
+            {
+                throw new Exception($"Failure to setup soft delete update for job. Job:{JsonConvert.SerializeObject(job)}");
+            }
+        }
+        catch (Exception ex)
+        {
+            result.AddToFailureCount(ex.Message);
+        }
+
+        return result;
+    }
+}
@@ -0,0 +1,84 @@
+using Newtonsoft.Json;
+using LCU.Library.Internal.ApiResponse.Models;
+using LCU.Api.Internal.Dispatcher.Services.QueueServices;
+using LCU.Library.Internal.DispatcherModels.Models.Requests;
+using LCU.Library.Internal.DispatcherModels.Enums;
+using System.Collections.Generic;
+using RabbitMQ.Client;
+using LCU.Api.Internal.MaidBotLib.Models;
+namespace LCU.Api.Internal.Dispatcher.Services.DispatcherOperationServices;
+
+public class CompletionService : ICompletionService
+{
+    private readonly IJobService _jobService;
+    private readonly IWorkItemService _workItemService;
+    private readonly IExecutionMessageService _executionMessageService;
+    private readonly IQueueJobFailureService _queueJobFailureService;
+    private readonly IQueueDetermineOperationService _queueDetermineOperationService;
+
+    public CompletionService(IJobService jobService,
+                             IWorkItemService workItemService,
+                             IExecutionMessageService executionMessageService,
+                             IQueueJobFailureService queueJobFailureService,
+                             IQueueDetermineOperationService queueDetermineOperationService)
+    {
+        _jobService = jobService;
+        _workItemService = workItemService;
+        _executionMessageService = executionMessageService;
+        _queueJobFailureService = queueJobFailureService;
+        _queueDetermineOperationService = queueDetermineOperationService;
+    }
+
+    public async Task HandleCompletionRequest(Response response)
+    {
+        var dispatcherReturnResponse = JsonConvert.DeserializeObject<DispatcherReturnResponse>(response.ResponsePayload);
+        try
+        {          
+            ArgumentNullException.ThrowIfNull(dispatcherReturnResponse);
+
+            var job = await _jobService.GetByWorkItemIdExcludingWorkItemsAsync(dispatcherReturnResponse.WorkItemId);
+
+            if (!response.Successful)
+            {
+                await _queueJobFailureService.QueueJobFailure(job.JobId,
+                                                                dispatcherReturnResponse.WorkItemId, 
+                                                                response.ResponseStack == null || response.ResponseStack.Any() ? response.ResponseStack : new List<string>());
+                return;
+            }
+
+            var workItem = await _workItemService.GetByIdAsync(dispatcherReturnResponse.WorkItemId);
+            if (workItem.CurrentWorkItemExecution == null)
+            {
+                Log.Warning($"CurrentWorkitemExecution was null for WorkItemId:{dispatcherReturnResponse.WorkItemId}");
+                return;
+            }
+            await _executionMessageService.SaveExecutionMessages(workItem.CurrentWorkItemExecution.WorkItemExecutionId, response.ResponseStack);
+
+            if (workItem.CanceledJobCounter > 0)
+            {
+                workItem = await _workItemService.ResetCanceledJobCounter(workItem);
+            }
+
+            job.JobStatus = JobStatus.DeterminingOperation;
+            var updatedJob = await _jobService.UpdateAsync(job);
+
+            await _queueDetermineOperationService.QueueDetermineOperation(updatedJob, workItem, SetDetermineOperationType(dispatcherReturnResponse.Skipped));
+        }
+        catch (Exception exception)
+        {
+            var exceptionMessage = $"Failed to perform completion. Job may be stuck in an inoperable status. WorkItemId: {dispatcherReturnResponse.WorkItemId}";
+            Log.Error(exception, exceptionMessage);
+            throw new ServiceException(exceptionMessage);
+        }
+    }
+
+    public DetermineOperationType SetDetermineOperationType(bool isSkipped)
+    {
+        if(isSkipped)
+        {
+            return DetermineOperationType.Skipped;
+        }
+
+        return DetermineOperationType.Completed;
+    }
+}
@@ -0,0 +1,82 @@
+using LCU.Api.Internal.Dispatcher.Services.QueueServices;
+using LCU.Api.Internal.MaidBotLib.Models;
+using LCU.Library.Internal.DispatcherModels.Entities;
+using LCU.Library.Internal.DispatcherModels.Enums;
+using LCU.Library.Internal.DispatcherModels.Models.Requests;
+
+namespace LCU.Api.Internal.Dispatcher.Services.DispatcherOperationServices
+{
+    public class DetermineOperationService : IDetermineOperationService
+	{
+        private readonly IJobService _jobService;
+        private readonly IWorkItemService _workItemService;
+        private readonly IWorkItemExecutionService _workItemExecutionService;
+        private readonly IQueueExecutionService _queueExecutionService;
+
+        public DetermineOperationService(IJobService jobService, 
+                                            IWorkItemService workItemService, 
+                                            IWorkItemExecutionService workItemExecutionService, 
+                                            IQueueExecutionService queueExecutionService)
+        {
+            _jobService = jobService;
+            _workItemService = workItemService;
+            _workItemExecutionService = workItemExecutionService;
+            _queueExecutionService = queueExecutionService;
+        }
+
+        public async Task HandleDetermineOperationRequest(DetermineOperationRequest determineOperationRequest)
+		{
+			try
+			{
+                switch (determineOperationRequest.DetermineOperationType)
+                {
+                    case DetermineOperationType.Failed:
+                        await HandleFailedWorkItem(determineOperationRequest);
+                        break;
+                    case DetermineOperationType.Skipped:
+                        await HandleWorkItemCompletion(determineOperationRequest, StatusType.Skipped);
+                        break;
+                    case DetermineOperationType.Completed:
+                        await HandleWorkItemCompletion(determineOperationRequest, StatusType.Completed);
+                        break;
+                    default:
+                        throw new NotImplementedException();
+                }
+                return;
+            }
+            catch (ServiceException serviceException)
+            {
+                Log.Error(serviceException, $"Dispatcher Determine Operation Failure. " +
+                                            $"The Dispatcher Determine Operation is placed on LCU.Dispatcher.DetermineOperation_error.");
+                throw new ServiceException($"Failed to handle DetermineOperationRequest.");
+            }
+            catch (Exception exception)
+            {
+                Log.Error(exception, $"Failed to handle DetermineOperationRequest.");
+                throw new ServiceException($"Failed to handle DetermineOperationRequest.");
+            }
+		}
+
+        public async Task HandleFailedWorkItem(DetermineOperationRequest determineOperationRequest)
+        {
+            await _jobService.CompleteJobAndWorkItem(determineOperationRequest.CurrentJob, determineOperationRequest.CurrentWorkItem.CurrentWorkItemExecution, StatusType.Failed);
+        }
+
+        public async Task HandleWorkItemCompletion(DetermineOperationRequest determineOperationRequest, StatusType status)
+        {
+            var nextWorkItem = await _workItemService.GetNextWorkItemByJobId(determineOperationRequest.CurrentJob.JobId, determineOperationRequest.CurrentWorkItem.Order);
+            if (nextWorkItem == null)
+            {
+                await _jobService.CompleteJobAndWorkItem(determineOperationRequest.CurrentJob, determineOperationRequest.CurrentWorkItem.CurrentWorkItemExecution, status);
+                return;
+            }
+
+            await _jobService.CompleteWorkItemAndContinueProcessingJob(determineOperationRequest.CurrentJob,
+                                                                        determineOperationRequest.CurrentWorkItem.CurrentWorkItemExecution,
+                                                                        status);
+            var newWorkItemExecution = await _workItemExecutionService.CreateWorkItemExecution(nextWorkItem.WorkItemId);
+
+            await _queueExecutionService.QueueExecution(newWorkItemExecution);
+        }
+    }
+}
@@ -0,0 +1,96 @@
+using LCU.Api.Internal.Dispatcher.Services.QueueServices;
+using LCU.Library.Internal.DispatcherModels.Enums;
+using LCU.Library.Internal.LcuHttpClient.HttpRequest;
+using System.Collections.Generic;
+namespace LCU.Api.Internal.Dispatcher.Services.DispatcherOperationServices;
+public class ExecutionService : IExecutionService
+{
+    private readonly IPostRequest _postRequest;
+    private readonly IWorkItemService _workItemService;
+    private readonly IJobService _jobService;
+    private readonly IQueueCompletionService _queueCompletionService;
+
+	private const string HttRequestIsNullMessage = "Http Request is null";
+
+	public ExecutionService(IPostRequest postRequest,
+							IWorkItemService workItemService,
+
+							IJobService jobService,
+                            IQueueCompletionService queueCompletionService)
+	{
+        _postRequest = postRequest;
+		_workItemService = workItemService;
+		_jobService = jobService;
+        _queueCompletionService = queueCompletionService;
+	}
+
+	public async Task HandleExecutionRequest(WorkItemExecution request)
+    {
+        try
+        {
+            var executionResponse = await ExecuteRequest(request);
+            if (!executionResponse.IsSuccess)
+            {
+                await _queueCompletionService.QueueCompletion(new DispatcherReturnResponse() { WorkItemId = executionResponse.WorkItemExecution.WorkItemId },
+                                                                    executionResponse.Messages, executionResponse.IsSuccess);
+            }
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, $"Failed to perform execution. WorkItemId: {request.WorkItemId}");
+            throw new ServiceException($"Failed to perform execution. WorkItemId: {request.WorkItemId}");
+        }
+    }
+
+    public async Task<ExecutionResponse> ExecuteRequest(WorkItemExecution request)
+    {
+        try
+        {
+            ArgumentNullException.ThrowIfNull(request);
+
+            var requestedWorkItem = await _workItemService.GetByIdAsync(request.WorkItemId);
+
+            await UpdateJobStatusToProcessing(requestedWorkItem);
+
+            var result = await _postRequest.AsyncWithNewtonSerializedNoThrow(new Uri(requestedWorkItem.Url), requestedWorkItem.SerializedPayload);
+            if (!result.IsSuccessStatusCode)
+            {
+                var content = await result.Content.ReadAsStringAsync();
+                return new ExecutionResponse(false, request, content);
+            }
+            return new ExecutionResponse(true, request);
+        }
+        catch (Exception ex)
+        {
+            var errorMessage = $"Exception during ExecuteRequest. Exception Message: {ex.Message}. Inner Exception: {ex.InnerException}";
+            Log.Error(errorMessage);
+            return new ExecutionResponse(false, request, errorMessage);
+        }
+    }
+
+    private async Task UpdateJobStatusToProcessing(WorkItem requestedWorkItem)
+	{
+		var job = await _jobService.GetByWorkItemIdExcludingWorkItemsAsync(requestedWorkItem.WorkItemId);
+        if (job.JobStatus == JobStatus.Processing)
+        {
+            return;
+        }
+		job.JobStatus = JobStatus.Processing;
+		await _jobService.UpdateAsync(job);
+		Log.Information("Job Id {JobId} was saved with status {Status}", job.JobId, job.JobStatus);
+	}
+
+	public struct ExecutionResponse
+    {
+        public ExecutionResponse(bool isSuccess, WorkItemExecution workItemExecution, string messages = null)
+        {
+            IsSuccess = isSuccess;
+            WorkItemExecution = workItemExecution;
+            Messages = messages != null ? new List<string>() { messages } : null;
+        }
+
+        public bool IsSuccess { get; set; }
+        public List<string> Messages { get; set; }
+        public WorkItemExecution WorkItemExecution { get; set; }
+    }
+}
@@ -0,0 +1,7 @@
+using System.Threading;
+namespace LCU.Api.Internal.Dispatcher.Services.DispatcherOperationServices;
+
+public interface IBatchErroredJobsService
+{
+    Task PerformBatchErroredJobs(CancellationToken cancellationToken);
+}
@@ -0,0 +1,8 @@
+using System.Threading;
+
+namespace LCU.Api.Internal.Dispatcher.Services.DispatcherOperationServices;
+
+public interface ICleanUpJobsService
+{
+    Task PerformCleanUpJobs(CancellationToken cancellationToken);
+}
@@ -0,0 +1,7 @@
+using LCU.Library.Internal.ApiResponse.Models;
+namespace LCU.Api.Internal.Dispatcher.Services.DispatcherOperationServices;
+
+public interface ICompletionService
+{
+    Task HandleCompletionRequest(Response response);
+}
@@ -0,0 +1,9 @@
+using LCU.Library.Internal.DispatcherModels.Models.Requests;
+
+namespace LCU.Api.Internal.Dispatcher.Services.DispatcherOperationServices
+{
+    public interface IDetermineOperationService
+	{
+		Task HandleDetermineOperationRequest(DetermineOperationRequest determineOperationRequest);
+	}
+}
@@ -0,0 +1,6 @@
+namespace LCU.Api.Internal.Dispatcher.Services.DispatcherOperationServices;
+
+public interface IExecutionService
+{
+    Task HandleExecutionRequest(WorkItemExecution request);
+}
@@ -0,0 +1,7 @@
+namespace LCU.Api.Internal.Dispatcher.Services.DispatcherOperationServices;
+
+public interface IIngestionService
+{
+    Task HandleIngestionRequest(Job request);
+    Task<bool> PerformIngestion(Job queueRequest);
+}
@@ -0,0 +1,9 @@
+using LCU.Library.Internal.DispatcherModels.Models.Requests;
+
+namespace LCU.Api.Internal.Dispatcher.Services.DispatcherOperationServices
+{
+    public interface IJobFailureService
+	{
+		Task HandleJobFailureRequest(JobFailureRequest jobFailureRequest);
+	}
+}
@@ -0,0 +1,7 @@
+using System.Threading;
+namespace LCU.Api.Internal.Dispatcher.Services.DispatcherOperationServices;
+
+public interface IJobScheduleService
+{
+    Task PerformSchedule(CancellationToken cancellationToken);
+}
@@ -0,0 +1,10 @@
+using LCU.Library.Internal.DispatcherModels.Models.Requests;
+using System.Collections.Generic;
+
+namespace LCU.Api.Internal.Dispatcher.Services.DispatcherOperationServices
+{
+    public interface ISendErroredJobAlertService
+	{
+		Task HandleSendingErroredJobAlerts(List<BatchedErroredJobAlertRequest> batchedErroredJobAlertRequestList);
+	}
+}
@@ -0,0 +1,159 @@
+using LCU.Library.Internal.DispatcherModels.Enums;
+using System.Collections.Generic;
+namespace LCU.Api.Internal.Dispatcher.Services.DispatcherOperationServices;
+
+public class IngestionService : IIngestionService
+{
+    private readonly IJobService _jobService;
+    private readonly IWorkItemService _workItemService;
+    private readonly IJobErrorRecipientService _jobErrorRecipientService;
+    private readonly IJobTagService _jobTagService;
+
+    public IngestionService(IJobService jobService,
+                            IWorkItemService workItemService,
+                            IJobErrorRecipientService jobErrorRecipientService,
+                            IJobTagService jobTagService)
+    {
+        _jobService = jobService;
+        _workItemService = workItemService;
+        _jobErrorRecipientService = jobErrorRecipientService;
+        _jobTagService = jobTagService;
+    }
+
+    public async Task HandleIngestionRequest(Job request)
+    {
+        try
+        {
+            ArgumentNullException.ThrowIfNull(request);
+            var result = await PerformIngestion(request);
+
+            if (!result)
+            {
+                throw new ServiceException("Failure to PerformIngestion.");
+            }
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, $"Failed to perform execution.");
+            throw new ServiceException($"Failed to perform ingestion.");
+        }
+    }
+
+    public async Task<bool> PerformIngestion(Job request)
+    {
+        try
+        {
+            if (request.IsNewJob)
+            {
+                return await SaveNewRequest(request);
+            }
+
+            return await UpdateRequest(request);
+        }
+        catch (Exception ex)
+        {
+            Log.Information(ex, $"Failed to perform ingestion.");
+            return false;
+        }
+    }
+
+    public async Task<bool> SaveNewRequest(Job request)
+    {
+        request = await _jobService.SaveAsync(request);
+        if (request == null)
+        {
+            return false;
+        }
+
+        var updatedWorkItems = await _workItemService.PopulateActionFieldsAndUpdateList(request.WorkItems);
+        if (updatedWorkItems == null)
+        {
+            return false;
+        }
+
+        return true;
+    }
+
+    public async Task<bool> UpdateRequest(Job request)
+    {
+        var existingJob = await _jobService.GetByIdAsync(request.JobId);
+
+        await UpdateRequestWorkItems(request, existingJob);
+        UpdateRequestErrorRecipients(request, existingJob);
+        await UpdateRequestJobTags(request, existingJob);
+        request.JobStatus = (request.NextRunTime == null && request.CronSchedule == null) ? JobStatus.Inactive : existingJob.JobStatus;
+
+        var updatedJob = await _jobService.UpdateAsync(request);
+        if (updatedJob == null)
+        {
+            return false;
+        }
+
+        return true;
+    }
+
+    public async Task UpdateRequestWorkItems(Job request, Job existingJob)
+    {
+        var removedWorkItems = existingJob.WorkItems.Where(wi =>
+                                                     !request.WorkItems.Any(newWI =>
+                                                              newWI.WorkItemId == wi.WorkItemId)).ToList();
+        await _workItemService.DeleteList(removedWorkItems);
+
+        var newWorkItems = request.WorkItems.Where(wi => wi.IsNewWorkItem).ToList();
+        _workItemService.SaveList(newWorkItems);
+
+        var updatedWorkItems = await _workItemService.PopulateActionFieldsAndUpdateList(request.WorkItems);
+        if (updatedWorkItems == null)
+        {
+            throw new Exception($"WorkItem Error while Populating Action Fields and updating.");
+        }
+    }
+
+    public void UpdateRequestErrorRecipients(Job request, Job existingJob)
+    {
+        var removedJobErrorRecipients = existingJob.JobErrorRecipients.Where(recipient =>
+                                                     !request.JobErrorRecipients.Any(newRecipient =>
+                                                              newRecipient.Recipient == recipient.Recipient)).ToList();
+        _jobErrorRecipientService.DeleteList(removedJobErrorRecipients);
+
+        var newJobErrorRecipients = new List<JobErrorRecipient>();
+        foreach (var recipient in request.JobErrorRecipients)
+        {
+            var jobErrorRecipient = existingJob.JobErrorRecipients.FirstOrDefault(existing =>
+                                            existing.Recipient == recipient.Recipient);
+
+            if (jobErrorRecipient == null)
+            {
+                newJobErrorRecipients.Add(recipient);
+            }
+        }
+
+        _jobErrorRecipientService.SaveList(newJobErrorRecipients);
+    }
+
+    public async Task UpdateRequestJobTags(Job request, Job existingJob)
+    {
+        var removedJobTags = existingJob.JobTags.Where(tag =>
+                                              !request.JobTags.Any(jobTags =>
+                                                       jobTags.TagId == tag.TagId)).ToList();
+
+        foreach (var tag in removedJobTags)
+        {
+            await _jobTagService.Delete(tag.JobTagId);
+        }
+
+        var newJobTags = new List<JobTag>();
+        foreach (var tag in request.JobTags)
+        {
+            var existingTag = existingJob.JobTags.FirstOrDefault(existing =>
+                                            existing.TagId == tag.TagId);
+
+            if (existingTag == null)
+            {
+                newJobTags.Add(tag);
+                var result = await _jobTagService.SaveAsync(tag);
+            }
+
+        }
+    }
+}
@@ -0,0 +1,91 @@
+using LCU.Api.Internal.Dispatcher.Services.QueueServices;
+using LCU.Api.Internal.MaidBotLib.Models;
+using LCU.Library.Internal.DispatcherModels.Entities;
+using LCU.Library.Internal.DispatcherModels.Enums;
+using LCU.Library.Internal.DispatcherModels.Models;
+using LCU.Library.Internal.DispatcherModels.Models.Requests;
+using LCU.Library.Internal.DocumentIngestorModels.Models.IndexFileDocuments;
+using LCU.Library.Internal.Exceptions;
+using System;
+using System.Collections.Generic;
+
+namespace LCU.Api.Internal.Dispatcher.Services.DispatcherOperationServices
+{
+    public class JobFailureService : IJobFailureService
+	{
+        private readonly IWorkItemService _workItemService;
+        private readonly IWorkItemExecutionService _workItemExecutionService;
+        private readonly IJobService _jobService;
+        private readonly IExecutionMessageService _executionMessageService;
+        private readonly IQueueDetermineOperationService _queueDetermineOperationService;
+        private readonly IQueueSendErroredJobAlertService _queueSendErroredJobAlertService;
+
+        public JobFailureService(IWorkItemService workItemService, 
+                                    IWorkItemExecutionService workItemExecutionService, 
+                                    IJobService jobService, 
+                                    IExecutionMessageService executionMessageService,
+                                    IQueueDetermineOperationService queueDetermineOperationService,
+                                    IQueueSendErroredJobAlertService queueSendErroredJobAlertService)
+        {
+            _workItemService = workItemService;
+            _workItemExecutionService = workItemExecutionService;
+            _jobService = jobService;
+            _executionMessageService = executionMessageService;
+            _queueDetermineOperationService = queueDetermineOperationService;
+            _queueSendErroredJobAlertService = queueSendErroredJobAlertService;
+        }
+
+        public async Task HandleJobFailureRequest(JobFailureRequest jobFailureRequest)
+		{
+            try
+            {
+                var workItem = await _workItemService.GetByIdAsync(jobFailureRequest.WorkItemId);
+
+                var updatedWorkItemExecution = await _workItemExecutionService.UpdateFailedWorkItemExecution(workItem.CurrentWorkItemExecution);
+
+                if (workItem.CancelJobOnFailure)
+                {
+                    workItem = await _workItemService.IncreaseCanceledJobCounter(workItem);
+                }
+
+                var job = await _jobService.GetByWorkItemIdExcludingWorkItemsIncludeErrorRecipientsAsync(jobFailureRequest.WorkItemId);
+
+                if (workItem.RequiresJobCancellation)
+                {
+                    var message = $"This Job will be canceled and if scheduleable, rescheduled. The " +
+                              $"WorkItem's canceled Job counter is currently at {workItem.CanceledJobCounter} and has " +
+                              $"not yet went over the cancel Job limit of {workItem.CancelJobLimit}.";
+                    jobFailureRequest.ExecutionMessages.Add(message);
+                    job.JobStatus = JobStatus.DeterminingOperation;
+                }
+                else
+                {
+                    job.JobStatus = JobStatus.Failed;
+                }
+
+                await _executionMessageService.SaveExecutionMessages(workItem.CurrentWorkItemExecution.WorkItemExecutionId, jobFailureRequest.ExecutionMessages);
+                var updatedJob = await _jobService.UpdateAsync(job);
+
+                if (updatedJob.JobStatus == JobStatus.DeterminingOperation)
+                {
+                    await _queueDetermineOperationService.QueueDetermineOperation(updatedJob, workItem, DetermineOperationType.Failed);
+                }
+                else
+                {
+                    await _queueSendErroredJobAlertService.QueueSendErroredJobAlert(updatedJob, workItem);
+                }
+            }
+            catch (ServiceException serviceException)
+            {
+                Log.Error(serviceException, $"Dispatcher Job Failure Request Failure. " +
+                                            $"The Dispatcher Job Failure Request is placed on LCU.Dispatcher.JobFailure_error.");
+                throw new ServiceException($"Failed to handle JobFailureRequest.");
+            }
+            catch(Exception exception)
+            {
+                Log.Error(exception, $"Failed to handle JobFailureRequest.");
+                throw new ServiceException($"Failed to handle JobFailureRequest.");
+            }
+		}
+	}
+}
@@ -0,0 +1,67 @@
+using System.Threading;
+using LCU.Api.Internal.MaidBotLib.Models;
+using LCU.Api.Internal.Dispatcher.Services.QueueServices;
+using LCU.Library.Internal.DispatcherModels.Enums;
+using System;
+namespace LCU.Api.Internal.Dispatcher.Services.DispatcherOperationServices;
+
+public class JobScheduleService : IJobScheduleService
+{
+    private readonly IQueueExecutionService _queueExecutionService;
+    private readonly IJobService _jobService;
+    private readonly IWorkItemService _workItemService;
+    private readonly IWorkItemExecutionService _workItemExecutionService;
+
+    public JobScheduleService(IQueueExecutionService queueExecutionService,
+                              IJobService jobService,
+                              IWorkItemService workItemService,
+                              IWorkItemExecutionService workItemExecutionService)
+    {
+        _queueExecutionService = queueExecutionService;
+        _jobService = jobService;
+        _workItemService = workItemService;
+        _workItemExecutionService = workItemExecutionService;
+    }
+
+    public async Task PerformSchedule(CancellationToken cancellationToken)
+    {
+        try
+        {
+            var readyToScheduleJobs = await _jobService.GetJobsReadyToBeScheduled();
+
+            foreach (var job in readyToScheduleJobs)
+            {
+                var workItem = await _workItemService.GetFirstOrDefaultWorkItemByJobId(job.JobId);
+
+                if (workItem == null)
+                {
+                    // THIS SHOULD NEVER HAPPEN
+                    Log.ForContext(Enum.GetName(typeof(PropertyEnum), PropertyEnum.Group),
+                           Enum.GetName(typeof(GroupEnum), GroupEnum.ITDevelopment))
+                    .Error($"Failure in Dispatcher Job Scheduler. Attempting to schedule Job: {job.JobId} - {job.Name} and NO workItems were found to process.");
+                    continue;
+                }
+
+                var workItemExecution = await _workItemExecutionService.CreateWorkItemExecution(workItem.WorkItemId);
+
+                job.JobStatus = JobStatus.PendingToRun;
+
+                if (job.SoftDeleteDate != null)
+                {
+                    job.SoftDeleteDate = null;
+                }
+
+                await _jobService.UpdateAsync(job);
+                Log.Information("Job Id {JobId} was saved with status {Status}", job.JobId, job.JobStatus);
+
+				await _queueExecutionService.QueueExecution(workItemExecution);
+            }
+        }
+        catch (Exception exception)
+        {
+            Log.ForContext(Enum.GetName(typeof(PropertyEnum), PropertyEnum.Group),
+                           Enum.GetName(typeof(GroupEnum), GroupEnum.ITDevelopment))
+                .Error(exception, $"Failed to perform schedule.");
+        }
+    }
+}
@@ -0,0 +1,146 @@
+using LCU.Api.Internal.Dispatcher.Configurations;
+using LCU.Library.Internal.DispatcherModels.Models.Requests;
+using System.Collections.Generic;
+using System.IO;
+
+namespace LCU.Api.Internal.Dispatcher.Services.DispatcherOperationServices
+{
+    public class SendErroredJobAlertService : ISendErroredJobAlertService
+	{
+        private readonly ApiConfiguration _apiConfiguration;
+        private readonly INotificationService _notificationService;
+
+        public SendErroredJobAlertService(ApiConfiguration apiConfiguration, INotificationService notificationService)
+        {
+            _apiConfiguration = apiConfiguration;
+            _notificationService = notificationService;
+        }
+
+        public Task HandleSendingErroredJobAlerts(List<BatchedErroredJobAlertRequest> batchedErroredJobAlertRequestList)
+        {
+            var erroredJobAlertRequests = CombineFromBatch(batchedErroredJobAlertRequestList);
+
+            return SendErroredJobsReport(erroredJobAlertRequests);
+        }
+
+        public async Task SendErroredJobsReport(List<ErroredJobAlertRequest> erroredJobAlertRequests)
+        {
+            var groupedJobsByRecipients = GroupJobsByRecipients(erroredJobAlertRequests);
+            var result = true;
+            foreach (var recipientGroup in groupedJobsByRecipients)
+            {
+                var successful = await HandleBatchErroredJobs(recipientGroup.Value, recipientGroup.Key);
+                if (!successful)
+                {
+                    result = false;
+                }
+            }
+
+            if (!result)
+            {
+                throw new ServiceException($"Failure to Send Errored Jobs Report. Result: {result}");
+            }
+        }
+
+        public List<ErroredJobAlertRequest> CombineFromBatch(List<BatchedErroredJobAlertRequest> batchedErroredJobAlertRequestList)
+        {
+            var erroredJobAlertRequests = new List<ErroredJobAlertRequest>();
+            foreach (var batch in batchedErroredJobAlertRequestList)
+            {
+                erroredJobAlertRequests.AddRange(batch.ErroredJobAlertRequests);
+            }
+            return erroredJobAlertRequests;
+        }
+
+        public Dictionary<string, List<ErroredJobAlertRequest>> GroupJobsByRecipients(List<ErroredJobAlertRequest> erroredJobAlertRequests)
+        {
+            var groupedRecipients = new Dictionary<string, List<ErroredJobAlertRequest>>();
+            foreach (var erroredJobAlertRequest in erroredJobAlertRequests)
+            {
+                AddRecipients(erroredJobAlertRequest, groupedRecipients);
+            }
+
+            return groupedRecipients;
+        }
+
+        public void AddRecipients(ErroredJobAlertRequest erroredJbAlertRequest, Dictionary<string, List<ErroredJobAlertRequest>> groupedRecipients)
+        {
+            if (erroredJbAlertRequest.CurrentJob.JobErrorRecipients == null ||
+                erroredJbAlertRequest.CurrentJob.JobErrorRecipients.Count == 0)
+            {
+                AddRecipient(_apiConfiguration.DefaultErrorRecipient, erroredJbAlertRequest, groupedRecipients);
+                return;
+            }
+
+            foreach (var jobErrorRecipient in erroredJbAlertRequest.CurrentJob.JobErrorRecipients)
+            {
+                AddRecipient(jobErrorRecipient.Recipient, erroredJbAlertRequest, groupedRecipients);
+            }
+        }
+
+        public void AddRecipient(string defaultErrorRecipient, ErroredJobAlertRequest erroredJobAlertRequest, Dictionary<string, List<ErroredJobAlertRequest>> groupedRecipients)
+        {
+            var recipientLower = defaultErrorRecipient.ToLower();
+            if (!groupedRecipients.ContainsKey(recipientLower))
+            {
+                groupedRecipients.Add(recipientLower, new List<ErroredJobAlertRequest>());
+            }
+
+            groupedRecipients[recipientLower].Add(erroredJobAlertRequest);
+        }
+
+        public async Task<bool> HandleBatchErroredJobs(List<ErroredJobAlertRequest> erroredJobAlertRequests, string recipient)
+        {
+            string rowsOfErrors = "";
+            foreach (ErroredJobAlertRequest erroredJobAlertRequest in erroredJobAlertRequests)
+            {
+                var jobErrorDetail = SetUpJobErrorDetails(erroredJobAlertRequest);
+                if (jobErrorDetail != null)
+                {
+                    rowsOfErrors += CreateErrorRow(jobErrorDetail);
+                }
+            }
+
+            var emailTemplate = CreateBatchErroredJobsEmail(rowsOfErrors);
+            var successfullySent = await _notificationService.SendEmailAsync(recipient,
+                                                                       _apiConfiguration.BatchErroredJobsSubject,
+                                                                       emailTemplate);
+            return successfullySent;
+        }
+
+        public JobErrorDetail SetUpJobErrorDetails(ErroredJobAlertRequest erroredJobAlertRequest)
+        {
+            var jobErrorDetail = new JobErrorDetail()
+            {
+                JobName = erroredJobAlertRequest?.CurrentJob?.Name,
+                WorkItemName = erroredJobAlertRequest?.CurrentWorkItem?.Name,
+                JobStatus = erroredJobAlertRequest?.CurrentJob?.JobStatus.ToString(),
+                StartTime = erroredJobAlertRequest?.CurrentWorkItem?.CurrentWorkItemExecution?.StartedTimestamp.ToString()
+            };
+
+            return jobErrorDetail;
+        }
+
+        public string CreateErrorRow(JobErrorDetail jobErrorDetails)
+        {
+            using var errorRowReader = new StreamReader(_apiConfiguration.ErrorRowTemplate);
+            string errorRowOutput = errorRowReader.ReadToEnd();
+
+            foreach (var property in jobErrorDetails.GetType().GetProperties())
+            {
+                string value = property.GetValue(jobErrorDetails) == null ? string.Empty :
+                                        property.GetValue(jobErrorDetails).ToString();
+                errorRowOutput = errorRowOutput.Replace($"{{{property.Name}}}", value);
+            }
+
+            return errorRowOutput;
+        }
+
+        public string CreateBatchErroredJobsEmail(string errorRows)
+        {
+            using var batchErroredJobsReader = new StreamReader(_apiConfiguration.BatchErroredJobsTemplate);
+            string batchErroredJobsOutput = batchErroredJobsReader.ReadToEnd();
+            return batchErroredJobsOutput.Replace("{TableRows}", errorRows);
+        }
+    }
+}