Adding

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Configurations/ApiConfiguration.cs

@@ -0,0 +1,26 @@
+using LCU.Library.Internal.DispatcherModels.Enums;
+using System.Collections.Generic;
+
+namespace LCU.Api.Internal.Dispatcher.Configurations;
+
+public class ApiConfiguration
+{
+    public string AppName { get; set; }
+    public string SeqUrl { get; set; }
+    public string SeqApiKey { get; set; }
+    public string[] CorsAllowedOrigins { get; set; }
+    public string SchedulersCronSchedule { get; set; }
+    public string BatchErroredJobsCronSchedule { get; set; }
+    public int ProcessingLimitInMinutes { get; set; }
+    public int NumberOfWorkItemExecutions { get; set; }
+    public int NumberOfExecutionMessages { get; set; }
+    public string BatchErroredJobsTemplate { get; set; }
+    public string ErrorRowTemplate { get; set; }
+    public string BatchErroredJobsSubject { get; set; }
+    public string DefaultErrorRecipient { get; set; }
+    public string CleanUpJobsCronSchedule { get; set; }
+    public int CleanUpJobsDaysToDelete { get; set; }
+    public int DefaultDelayMilliseconds { get; set; }
+    public FeatureFlags FeatureFlags { get; set; }   
+    public List<JobStatus> JobStatusesToMonitorByTimeLimit { get; set; }
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Configurations/ConnectionsConfiguration.cs

@@ -0,0 +1,22 @@
+using System.Collections.Generic;
+namespace LCU.Api.Internal.Dispatcher.Configurations;
+
+public class ConnectionsConfiguration
+{
+    public Sql Sql { get; set; }
+}
+
+public class Sql
+{
+    public string DispatcherDb { get; set; }
+    public SQLOptions SQLOptions { get; set; }
+}
+
+public class SQLOptions
+{
+    public int MaxRetryCount { get; set; }
+    public TimeSpan MaxRetryDelay { get; set; }
+    public ICollection<int> ErrorNumbersToAdd { get; set; }
+    public bool EnableDetailedErrors { get; set; }
+    public int CommandTimeout { get; set; }
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Configurations/FeatureFlags.cs

@@ -0,0 +1,13 @@
+namespace LCU.Api.Internal.Dispatcher.Configurations;
+
+public class FeatureFlags
+{
+    public bool RunCompletionConsumer { get; set; }
+    public bool RunExecutionConsumer { get; set; }
+    public bool RunJobSchedulerHostedService { get; set; }
+    public bool RunBatchErroredJobsHostedService { get; set; }
+    public bool RunCleanUpJobsHostedService { get; set; }
+    public bool RunDetermineOperationConsumer { get; set; }
+    public bool RunJobFailureConsumer { get; set; }
+    public bool RunSendErroredJobAlertConsumer { get; set; }
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Configurations/ReceiveEndpointConfigurations.cs

@@ -0,0 +1,11 @@
+using LCU.Library.Internal.LcuMassTransitModels;
+namespace LCU.Api.Internal.Dispatcher.Configurations;
+
+public class ReceiveEndpointConfigurations
+{
+    public ReceiveEndpointConfiguration ExecutionReceiveEndpoint { get; set; }
+    public ReceiveEndpointConfiguration CompletionReceiveEndpoint { get; set; }
+    public ReceiveEndpointConfiguration DetermineOperationReceiveEndpoint { get; set; }
+    public ReceiveEndpointConfiguration JobFailureReceiveEndpoint { get; set; }
+    public ReceiveEndpointConfiguration SendErroredJobAlertReceiveEndpoint { get; set; }
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Consumers/CompletionConsumer.cs

@@ -0,0 +1,47 @@
+using LCU.Api.Internal.Dispatcher.Services.DispatcherOperationServices;
+using LCU.Api.Internal.MaidBotLib.Models;
+using LCU.Library.Internal.Logging.Performance;
+using MassTransit;
+using Microsoft.Extensions.DependencyInjection;
+using Newtonsoft.Json;
+using System.Collections.Generic;
+using ApiResponse = LCU.Library.Internal.ApiResponse.Models.Response;
+namespace LCU.Api.Internal.Dispatcher.Consumers;
+
+public class CompletionConsumer : IConsumer<ApiResponse>
+{
+    private readonly IServiceScopeFactory _scopeFactory;
+
+    public CompletionConsumer(IServiceScopeFactory scopeFactory)
+    {
+        _scopeFactory = scopeFactory;
+    }
+
+    public async Task Consume(ConsumeContext<ApiResponse> context)
+    {
+        try
+        {
+            using var scope = _scopeFactory.CreateAsyncScope();
+            var dispatcherReturnResponse = JsonConvert.DeserializeObject<DispatcherReturnResponse>(context.Message.ResponsePayload);
+            var performanceMonitor = new PerformanceMonitor("DispatcherCompletionConsumerMonitor",
+                                                                Guid.NewGuid());
+
+            await scope.ServiceProvider.GetRequiredService<ICompletionService>().HandleCompletionRequest(context.Message);
+            performanceMonitor.LogPerformance((new Dictionary<string, string>(){
+                                                {"WorkItemId", dispatcherReturnResponse.WorkItemId.ToString()}
+            }));
+        }
+        catch (ServiceException serviceException)
+        {
+            Log.ForContext(Enum.GetName(typeof(PropertyEnum), PropertyEnum.Group),
+                           Enum.GetName(typeof(GroupEnum), GroupEnum.ITDevelopment))
+               .Error(serviceException, $"Dispatcher Completion Failure. " +
+                                        $"The Dispatcher Completion Request is placed on LCU.Dispatcher.Completion_error.");
+            return;
+        }
+        catch (Exception exception)
+        {
+            Log.Fatal(exception, $"Unexpected Failure");
+        }
+    }
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Consumers/DetermineOperationConsumer.cs

@@ -0,0 +1,52 @@
+using LCU.Api.Internal.Dispatcher.Services.DispatcherOperationServices;
+using LCU.Api.Internal.MaidBotLib.Models;
+using LCU.Library.Internal.DispatcherModels.Models;
+using LCU.Library.Internal.DispatcherModels.Models.Requests;
+using LCU.Library.Internal.Logging.Performance;
+using MassTransit;
+using Microsoft.Extensions.DependencyInjection;
+using System.Collections.Generic;
+
+namespace LCU.Api.Internal.Dispatcher.Consumers
+{
+    public class DetermineOperationConsumer : IConsumer<DetermineOperationRequest>
+	{
+        private readonly IServiceScopeFactory _scopeFactory;
+
+		public DetermineOperationConsumer(IServiceScopeFactory scopeFactory)
+		{
+			_scopeFactory = scopeFactory;
+		}
+		
+		public async Task Consume(ConsumeContext<DetermineOperationRequest> context)
+		{
+			try
+            {
+                using var scope = _scopeFactory.CreateAsyncScope();
+                var performanceMonitor = new PerformanceMonitor("DispatcherDetermineOperationConsumerMonitor", Guid.NewGuid());
+                await scope.ServiceProvider.GetRequiredService<IDetermineOperationService>().HandleDetermineOperationRequest(context.Message);
+                performanceMonitor.LogPerformance((new Dictionary<string, string>(){
+                                                {"JobId", context.Message.CurrentJob.JobId.ToString()},
+                                                {"WorkItemId", context.Message.CurrentWorkItem.WorkItemId.ToString()},
+                                                {"WorkItemExecutionId", context.Message.CurrentWorkItem?.CurrentWorkItemExecution?.WorkItemExecutionId.ToString()}
+                }));
+            }
+            catch (ServiceException serviceException)
+            {
+                Log.ForContext(Enum.GetName(typeof(PropertyEnum), PropertyEnum.Group),
+                               Enum.GetName(typeof(GroupEnum), GroupEnum.ITDevelopment))
+                   .Error(serviceException, $"Dispatcher Determine Operation Failure. " +
+                                            $"The Dispatcher Determine Operation Request is placed on LCU.Dispatcher.DetermineOperation_error.");
+                throw;
+            }
+            catch (Exception exception)
+            {
+                Log.ForContext(Enum.GetName(typeof(PropertyEnum), PropertyEnum.Group),
+                               Enum.GetName(typeof(GroupEnum), GroupEnum.ITDevelopment))
+                   .Fatal(exception, $"Unexpected Dispatcher Determine Operation Failure. " +
+                                     $"The Dispatcher Determine Operation Request is placed on LCU.Dispatcher.DetermineOperation_error.");
+                throw;
+            }
+		}
+	}
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Consumers/ExecutionConsumer.cs

@@ -0,0 +1,47 @@
+using MassTransit;
+using Microsoft.Extensions.DependencyInjection;
+using LCU.Api.Internal.MaidBotLib.Models;
+using LCU.Api.Internal.Dispatcher.Services.DispatcherOperationServices;
+using System.Collections.Generic;
+using LCU.Library.Internal.Logging.Performance;
+namespace LCU.Api.Internal.Dispatcher.Consumers;
+
+public class ExecutionConsumer : IConsumer<WorkItemExecution>
+{
+    private readonly IServiceScopeFactory _scopeFactory;
+
+    public ExecutionConsumer(IServiceScopeFactory scopeFactory)
+    {
+        _scopeFactory = scopeFactory;
+    }
+
+    public async Task Consume(ConsumeContext<WorkItemExecution> context)
+    {
+        try
+        {
+            using var scope = _scopeFactory.CreateAsyncScope();
+            var performanceMonitor = new PerformanceMonitor("DispatcherExecutionConsumerMonitor", Guid.NewGuid());
+            await scope.ServiceProvider.GetRequiredService<IExecutionService>().HandleExecutionRequest(context.Message);
+            performanceMonitor.LogPerformance((new Dictionary<string, string>(){
+                                                {"WorkItemId", context.Message.WorkItemId.ToString()},
+                                                {"WorkItemExecutionId", context.Message.WorkItemExecutionId.ToString()}
+                }));
+        }
+        catch (ServiceException serviceException)
+        {
+            Log.ForContext(Enum.GetName(typeof(PropertyEnum), PropertyEnum.Group),
+                           Enum.GetName(typeof(GroupEnum), GroupEnum.ITDevelopment))
+               .Error(serviceException, $"Dispatcher Execution Failure. " +
+                                        $"The Dispatcher Execution Request is placed on LCU.Dispatcher.Execution_error.");
+            throw;
+        }
+        catch (Exception exception)
+        {
+            Log.ForContext(Enum.GetName(typeof(PropertyEnum), PropertyEnum.Group),
+                           Enum.GetName(typeof(GroupEnum), GroupEnum.ITDevelopment))
+               .Fatal(exception, $"Unexpected Dispatcher Execution Failure. " +
+                                 $"The Dispatcher Execution Request is placed on LCU.Dispatcher.Execution_error.");
+            throw;
+        }
+    }
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Consumers/JobFailureConsumer.cs

@@ -0,0 +1,50 @@
+using LCU.Api.Internal.Dispatcher.Services.DispatcherOperationServices;
+using LCU.Api.Internal.MaidBotLib.Models;
+using LCU.Library.Internal.DispatcherModels.Models.Requests;
+using LCU.Library.Internal.Logging.Performance;
+using MassTransit;
+using Microsoft.Extensions.DependencyInjection;
+using System.Collections.Generic;
+
+namespace LCU.Api.Internal.Dispatcher.Consumers
+{
+    public class JobFailureConsumer : IConsumer<JobFailureRequest>
+	{
+		private readonly IServiceScopeFactory _scopeFactory;
+
+		public JobFailureConsumer(IServiceScopeFactory scopeFactory)
+		{
+			_scopeFactory = scopeFactory;
+		}
+
+		public async Task Consume(ConsumeContext<JobFailureRequest> context)
+		{
+			try
+            {
+                using var scope = _scopeFactory.CreateAsyncScope();
+                var performanceMonitor = new PerformanceMonitor("DispatcherJobFailureConsumerMonitor", Guid.NewGuid());
+                await scope.ServiceProvider.GetRequiredService<IJobFailureService>().HandleJobFailureRequest(context.Message);
+                performanceMonitor.LogPerformance((new Dictionary<string, string>(){
+                                                {"JobId", context.Message.JobId.ToString()},
+                                                {"WorkItemId", context.Message.WorkItemId.ToString()}
+                }));
+            }
+            catch (ServiceException serviceException)
+            {
+                Log.ForContext(Enum.GetName(typeof(PropertyEnum), PropertyEnum.Group),
+                               Enum.GetName(typeof(GroupEnum), GroupEnum.ITDevelopment))
+                   .Error(serviceException, $"Dispatcher Job Failure. " +
+                                            $"The Dispatcher Job Failure Request is placed on LCU.Dispatcher.JobFailure_error.");
+                throw;
+            }
+            catch (Exception exception)
+            {
+                Log.ForContext(Enum.GetName(typeof(PropertyEnum), PropertyEnum.Group),
+                               Enum.GetName(typeof(GroupEnum), GroupEnum.ITDevelopment))
+                   .Fatal(exception, $"Unexpected Dispatcher Job Failure. " +
+                                     $"The Dispatcher Job Failure Request is placed on LCU.Dispatcher.JobFailure_error.");
+                throw;
+            }
+		}
+	}
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Consumers/SendErroredJobAlertConsumer.cs

@@ -0,0 +1,57 @@
+using LCU.Api.Internal.Dispatcher.Services.DispatcherOperationServices;
+using LCU.Api.Internal.MaidBotLib.Models;
+using LCU.Library.Internal.DispatcherModels.Models.Requests;
+using LCU.Library.Internal.Logging.Performance;
+using MassTransit;
+using Microsoft.Extensions.DependencyInjection;
+using System.Collections.Generic;
+
+namespace LCU.Api.Internal.Dispatcher.Consumers
+{
+    public class SendErroredJobAlertConsumer : IConsumer<Batch<BatchedErroredJobAlertRequest>>
+	{
+		private readonly IServiceScopeFactory _scopeFactory;
+
+		public SendErroredJobAlertConsumer(IServiceScopeFactory scopeFactory)
+		{
+			_scopeFactory = scopeFactory;
+		}
+
+		public async Task Consume(ConsumeContext<Batch<BatchedErroredJobAlertRequest>> context)
+		{
+			try
+            {
+                using var scope = _scopeFactory.CreateAsyncScope();
+                var performanceMonitor = new PerformanceMonitor("DispatcherBatchErroredJobsConsumerMonitor", Guid.NewGuid());
+
+                var batchedErroredJobAlertRequestlist = new List<BatchedErroredJobAlertRequest>();
+                for (int index = 0; index < context.Message.Length; index++)
+                {
+                    ConsumeContext<BatchedErroredJobAlertRequest> messageContext = context.Message[index];
+                    batchedErroredJobAlertRequestlist.Add(messageContext.Message);
+                }
+
+                await scope.ServiceProvider.GetRequiredService<ISendErroredJobAlertService>().HandleSendingErroredJobAlerts(batchedErroredJobAlertRequestlist);
+                performanceMonitor.LogPerformance((new Dictionary<string, string>(){
+                                                {"NumberOfBatchedErrors", batchedErroredJobAlertRequestlist.Count().ToString()}
+                }));
+            }
+            catch (ServiceException serviceException)
+            {
+                Log.ForContext(Enum.GetName(typeof(PropertyEnum), PropertyEnum.Group),
+                               Enum.GetName(typeof(GroupEnum), GroupEnum.ITDevelopment))
+                   .Error(serviceException, $"Dispatcher Send Errored Job Alert. " +
+                                            $"The Dispatcher Send Errored Job Alert Request is placed on LCU.Dispatcher.SendErroredJobAlert_error.");
+                throw;
+            }
+            catch (Exception exception)
+            {
+                Log.ForContext(Enum.GetName(typeof(PropertyEnum), PropertyEnum.Group),
+                               Enum.GetName(typeof(GroupEnum), GroupEnum.ITDevelopment))
+                   .Fatal(exception, $"Unexpected Dispatcher Send Errored Job Alert. " +
+                                     $"The Dispatcher Send Errored Job Alert Request is placed on LCU.Dispatcher.SendErroredJobAlert_error.");
+                throw;
+            }
+		}
+	}
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Controllers/CleanUpJobProcessController.cs

@@ -0,0 +1,42 @@
+using LCU.Api.Internal.Dispatcher.Services.DispatcherOperationServices;
+using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.Mvc;
+using System.Threading;
+
+namespace LCU.Api.Internal.Dispatcher.Controllers;
+
+[ApiController]
+[Route("api/[controller]")]
+public class CleanUpJobProcessController : ControllerBase
+{
+    private readonly ICleanUpJobsService _cleanUpJobsService;
+
+    public CleanUpJobProcessController(ICleanUpJobsService cleanUpJobsService)
+    {
+        _cleanUpJobsService = cleanUpJobsService;
+    }
+
+    [HttpPost("RunCleanUpJobProcessNow", Name = "RunCleanUpJobProcessNow")]
+    [ProducesResponseType(200)]
+    [ProducesResponseType(500)]
+    public async Task<IActionResult> RunCleanUpJobNow()
+    {
+        try
+        {
+            Log.Information("Manually Kicked Off Dispatcher's Clean Up Job Hosted Service.");
+            var token = new CancellationToken();
+            await _cleanUpJobsService.PerformCleanUpJobs(token);
+
+            return Ok();
+        }
+        catch (ServiceException)
+        {
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, "Unexpected Exception");
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+    }
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Controllers/CleanUpJobSettingController.cs

@@ -0,0 +1,161 @@
+using LCU.Api.Internal.Dispatcher.Services;
+using LCU.Api.Internal.Dispatcher.Services.DispatcherOperationServices;
+using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.Mvc;
+using System.Collections.Generic;
+
+namespace LCU.Api.Internal.Dispatcher.Controllers;
+
+[ApiController]
+[Route("api/[controller]")]
+public class CleanUpJobSettingController : ControllerBase
+{
+    private readonly ICleanUpJobSettingService _cleanUpJobSettingService;
+
+    public CleanUpJobSettingController(ICleanUpJobSettingService cleanUpJobSettingService)
+    {
+        _cleanUpJobSettingService = cleanUpJobSettingService;
+    }
+
+    [HttpGet("GetAllCleanUpJobSetting", Name = "GetAllCleanUpJobSetting")]
+    [ProducesResponseType(typeof(IEnumerable<CleanUpJobSetting>), 200)]
+    [ProducesResponseType(204)]
+    [ProducesResponseType(500)]
+    public async Task<IActionResult> Get()
+    {
+        try
+        {
+            var result = await _cleanUpJobSettingService.GetAsync();
+            if (result == null || !result.Any())
+            {
+                return NoContent();
+            }
+
+            return Ok(result);
+        }
+        catch (ServiceException)
+        {
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, "Unexpected Exception");
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+    }
+
+    [HttpGet("GetByCleanUpJobSettingIdAsync", Name = "GetByCleanUpJobSettingIdAsync")]
+    [ProducesResponseType(typeof(CleanUpJobSetting), 200)]
+    [ProducesResponseType(204)]
+    [ProducesResponseType(400)]
+    [ProducesResponseType(500)]
+    public async Task<IActionResult> GetByCleanUpJobSettingIdAsync(int cleanUpJobSettingId)
+    {
+        try
+        {
+            if (cleanUpJobSettingId <= 0)
+            {
+                return BadRequest("Invalid CleanUpJobSettingId.");
+            }
+
+            var result = await _cleanUpJobSettingService.GetByCleanUpJobSettingIdAsync(cleanUpJobSettingId);
+            if (result == null)
+            {
+                return NoContent();
+            }
+
+            return Ok(result);
+        }
+        catch (ServiceException)
+        {
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, "Unexpected Exception");
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+    }
+
+    [HttpPost]
+    [ProducesResponseType(typeof(CleanUpJobSetting), 200)]
+    [ProducesResponseType(400)]
+    [ProducesResponseType(406)]
+    [ProducesResponseType(500)]
+    public async Task<IActionResult> Post([FromBody] CleanUpJobSetting request)
+    {
+        try
+        {
+            var result = await _cleanUpJobSettingService.SaveAsync(request);
+            if (result == null)
+            {
+                return StatusCode(StatusCodes.Status406NotAcceptable);
+            }
+
+            return Ok(result);
+        }
+        catch (ServiceException)
+        {
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, "Unexpected Exception");
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+    }
+
+    [HttpPut()]
+    [ProducesResponseType(typeof(CleanUpJobSetting), 200)]
+    [ProducesResponseType(406)]
+    [ProducesResponseType(500)]
+    public async Task<IActionResult> Put([FromBody] CleanUpJobSetting request)
+    {
+        try
+        {
+            var result = await _cleanUpJobSettingService.UpdateAsync(request);
+            if (result == null)
+            {
+                return StatusCode(StatusCodes.Status406NotAcceptable);
+            }
+
+            return Ok(result);
+        }
+        catch (ServiceException)
+        {
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, "Unexpected Exception");
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+    }
+
+    [HttpDelete()]
+    [ProducesResponseType(200)]
+    [ProducesResponseType(400)]
+    [ProducesResponseType(500)]
+    public async Task<IActionResult> Delete(int cleanUpJobSettingId)
+    {
+        try
+        {
+            if (cleanUpJobSettingId <= 0)
+            {
+                return BadRequest("Invalid CleanUpJobSettingId");
+            }
+
+            await _cleanUpJobSettingService.DeleteAsync(cleanUpJobSettingId);
+            return Ok();
+        }
+        catch (ServiceException)
+        {
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, "Unexpected Exception");
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+    }
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Controllers/CleanUpJobTagController.cs

@@ -0,0 +1,192 @@
+using LCU.Api.Internal.Dispatcher.Services;
+using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.Mvc;
+using System.Collections.Generic;
+
+namespace LCU.Api.Internal.Dispatcher.Controllers;
+
+[Route("api/[controller]")]
+[ApiController]
+public class CleanUpJobTagController : ControllerBase
+{
+    private readonly ICleanUpJobTagService _cleanUpJobTagService;
+    public CleanUpJobTagController(ICleanUpJobTagService cleanUpJobTagService)
+    {
+        _cleanUpJobTagService = cleanUpJobTagService;
+    }
+
+    [HttpGet(Name = "GetAllCleanUpJobTag")]
+    [ProducesResponseType(typeof(IEnumerable<CleanUpJobTag>), 200)]
+    [ProducesResponseType(204)]
+    [ProducesResponseType(500)]
+    public async Task<IActionResult> Get()
+    {
+        try
+        {
+            var result = await _cleanUpJobTagService.GetAsync();
+            if (result == null || !result.Any())
+            {
+                return NoContent();
+            }
+            return Ok(result);
+        }
+        catch (ServiceException)
+        {
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, "Unexpected Exception");
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+    }
+
+    [HttpGet("GetByCleanUpJobTagIdAsync", Name = "GetByCleanUpJobTagIdAsync")]
+    [ProducesResponseType(typeof(CleanUpJobTag), 200)]
+    [ProducesResponseType(204)]
+    [ProducesResponseType(400)]
+    [ProducesResponseType(500)]
+    public async Task<IActionResult> GetByCleanUpJobTagIdAsync(int cleanUpJobTagId)
+    {
+        try
+        {
+            if (cleanUpJobTagId <= 0)
+            {
+                return BadRequest("Invalid CleanUpJobTagIdId.");
+            }
+
+            var result = await _cleanUpJobTagService.GetByCleanUpJobTagIdAsync(cleanUpJobTagId);
+            if (result == null)
+            {
+                return NoContent();
+            }
+            return Ok(result);
+        }
+        catch (ServiceException)
+        {
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, "Unexpected Exception");
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+    }
+
+    [HttpGet("GetTagsByCleanUpJobSettingIdAsync", Name = "GetTagsByCleanUpJobSettingIdAsync")]
+    [ProducesResponseType(typeof(IEnumerable<CleanUpJobTag>), 200)]
+    [ProducesResponseType(204)]
+    [ProducesResponseType(400)]
+    [ProducesResponseType(500)]
+    public async Task<IActionResult> GetByCleanUpJobSettingIdAsync(int cleanUpJobSettingId)
+    {
+        try
+        {
+            if (cleanUpJobSettingId <= 0)
+            {
+                return BadRequest("Invalid CleanUpJobSettingId.");
+            }
+
+            var result = await _cleanUpJobTagService.GetByCleanUpJobSettingIdAsync(cleanUpJobSettingId);
+            if (result == null)
+            {
+                return NoContent();
+            }
+
+            return Ok(result);
+        }
+        catch (ArgumentNullException)
+        {
+            return StatusCode(StatusCodes.Status404NotFound);
+        }
+        catch (ServiceException)
+        {
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, "Unexpected Exception");
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+    }
+
+    [HttpPost()]
+    [ProducesResponseType(typeof(CleanUpJobTag), 200)]
+    [ProducesResponseType(406)]
+    [ProducesResponseType(500)]
+    public async Task<IActionResult> Post([FromBody] CleanUpJobTag request)
+    {
+        try
+        {
+            var result = await _cleanUpJobTagService.SaveAsync(request);
+            if (result == null)
+            {
+                return StatusCode(StatusCodes.Status406NotAcceptable);
+            }
+            return Ok(result);
+        }
+        catch (ServiceException)
+        {
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, "Unexpected Exception");
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+    }
+
+    [HttpDelete("DeleteByCleanUpJobTagId", Name = "DeleteByCleanUpJobTagId")]
+    [ProducesResponseType(200)]
+    [ProducesResponseType(400)]
+    [ProducesResponseType(500)]
+    public async Task<IActionResult> Delete(int cleanUpJobTagId)
+    {
+        try
+        {
+            if (cleanUpJobTagId <= 0)
+            {
+                return BadRequest("Invalid CleanUpJobTagId");
+            }
+
+            await _cleanUpJobTagService.DeleteAsync(cleanUpJobTagId);
+            return Ok();
+        }
+        catch (ServiceException)
+        {
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, "Unexpected Exception");
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+    }
+
+    [HttpDelete("DeleteByCleanUpJobSettingId", Name = "DeleteByCleanUpJobSettingId")]
+    [ProducesResponseType(200)]
+    [ProducesResponseType(400)]
+    [ProducesResponseType(500)]
+    public async Task<IActionResult> DeleteByCleanUpJobSettingId(int cleanUpJobSettingId)
+    {
+        try
+        {
+            if (cleanUpJobSettingId <= 0)
+            {
+                return BadRequest("Invalid CleanUpJobSettingId");
+            }
+
+            await _cleanUpJobTagService.DeleteByCleanUpJobSettingIdAsync(cleanUpJobSettingId);
+            return Ok();
+        }
+        catch (ServiceException)
+        {
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, "Unexpected Exception");
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+    }
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Controllers/ExecutionMessageController.cs

@@ -0,0 +1,43 @@
+using System.Collections.Generic;
+using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.Mvc;
+using LCU.Api.Internal.Dispatcher.Services;
+namespace LCU.Api.Internal.Dispatcher.Controllers;
+
+[Route("[controller]")]
+[ApiController]
+public class ExecutionMessageController : ControllerBase
+{
+    private readonly IExecutionMessageService _executionMessageService;
+    public ExecutionMessageController(IExecutionMessageService executionMessageService)
+    {
+        _executionMessageService = executionMessageService;
+    }
+
+    [HttpPost()]
+    [ProducesResponseType(typeof(List<ExecutionMessage>), 200)]
+    [ProducesResponseType(400)]
+    [ProducesResponseType(500)]
+    public async Task<IActionResult> Post([FromBody] List<ExecutionMessage> request)
+    {
+        try
+        {
+            if (request == null || !request.Any())
+            {
+                return BadRequest("Execution messages are required.");
+            }
+
+            await _executionMessageService.SaveExecutionMessages(request);
+            return Ok();
+        }
+        catch (ServiceException)
+        {
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, "Unexpected Exception");
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+    }
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Controllers/HostedServicesController.cs

@@ -0,0 +1,163 @@
+using LCU.Api.Internal.Dispatcher.HostedServices;
+using LCU.Library.Internal.ApiSecurity.Attributes;
+using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.Mvc;
+
+namespace LCU.Api.Internal.Dispatcher.Controllers
+{
+	[ApiController]
+	[Route("api/[controller]")]
+	public class HostedServicesController : ControllerBase
+	{
+		private readonly ICleanUpJobsHostedService _cleanUpJobsHostedServiceStarter;
+		private readonly IBatchErroredJobsHostedService _batchErroredJobsServiceStarter;
+		private readonly IJobSchedulerHostedService _jobSchedulerHostedServiceStarter;
+
+		public HostedServicesController(ICleanUpJobsHostedService cleanUpJobsHostedService,
+										IBatchErroredJobsHostedService batchErroredJobsServiceStarter,
+										IJobSchedulerHostedService jobSchedulerHostedServiceStarter)
+		{
+			_cleanUpJobsHostedServiceStarter = cleanUpJobsHostedService;
+			_batchErroredJobsServiceStarter = batchErroredJobsServiceStarter;
+			_jobSchedulerHostedServiceStarter = jobSchedulerHostedServiceStarter;
+		}
+
+		[ApiKey]
+		[HttpPost("StartCleanUpJobs", Name = "StartCleanUpJobs")]
+		[ProducesResponseType(200)]
+		[ProducesResponseType(500)]
+		public async Task<IActionResult> StartCleanUpJobs()
+		{
+			try
+			{
+				await _cleanUpJobsHostedServiceStarter.StartAsync();
+
+				return Ok();
+			}
+			catch (ServiceException)
+			{
+				return StatusCode(StatusCodes.Status500InternalServerError);
+			}
+			catch (Exception exception)
+			{
+				Log.Error(exception, "Unexpected Exception");
+				return StatusCode(StatusCodes.Status500InternalServerError);
+			}
+		}
+
+		[ApiKey]
+		[HttpPost("StopCleanUpJobs", Name = "StopCleanUpJobs")]
+		[ProducesResponseType(200)]
+		[ProducesResponseType(500)]
+		public async Task<IActionResult> StopCleanUpJobs()
+		{
+			try
+			{
+				await _cleanUpJobsHostedServiceStarter.StopAsync();
+
+				return Ok();
+			}
+			catch (ServiceException)
+			{
+				return StatusCode(StatusCodes.Status500InternalServerError);
+			}
+			catch (Exception exception)
+			{
+				Log.Error(exception, "Unexpected Exception");
+				return StatusCode(StatusCodes.Status500InternalServerError);
+			}
+		}
+
+		[ApiKey]
+		[HttpPost("StartBatchErroredJobs", Name = "StartBatchErroredJobs")]
+		[ProducesResponseType(200)]
+		[ProducesResponseType(500)]
+		public async Task<IActionResult> StartBatchErroredJobs()
+		{
+			try
+			{
+				await _batchErroredJobsServiceStarter.StartAsync();
+
+				return Ok();
+			}
+			catch (ServiceException)
+			{
+				return StatusCode(StatusCodes.Status500InternalServerError);
+			}
+			catch (Exception exception)
+			{
+				Log.Error(exception, "Unexpected Exception");
+				return StatusCode(StatusCodes.Status500InternalServerError);
+			}
+		}
+
+		[ApiKey]
+		[HttpPost("StopBatchErroredJobs", Name = "StopBatchErroredJobs")]
+		[ProducesResponseType(200)]
+		[ProducesResponseType(500)]
+		public async Task<IActionResult> StopBatchErroredJobs()
+		{
+			try
+			{
+				await _batchErroredJobsServiceStarter.StopAsync();
+
+				return Ok();
+			}
+			catch (ServiceException)
+			{
+				return StatusCode(StatusCodes.Status500InternalServerError);
+			}
+			catch (Exception exception)
+			{
+				Log.Error(exception, "Unexpected Exception");
+				return StatusCode(StatusCodes.Status500InternalServerError);
+			}
+		}
+
+		[ApiKey]
+		[HttpPost("StartJobScheduler", Name = "StartJobScheduler")]
+		[ProducesResponseType(200)]
+		[ProducesResponseType(500)]
+		public async Task<IActionResult> StartJobScheduler()
+		{
+			try
+			{
+				await _jobSchedulerHostedServiceStarter.StartAsync();
+
+				return Ok();
+			}
+			catch (ServiceException)
+			{
+				return StatusCode(StatusCodes.Status500InternalServerError);
+			}
+			catch (Exception exception)
+			{
+				Log.Error(exception, "Unexpected Exception");
+				return StatusCode(StatusCodes.Status500InternalServerError);
+			}
+		}
+
+		[ApiKey]
+		[HttpPost("StopJobScheduler", Name = "StopJobScheduler")]
+		[ProducesResponseType(200)]
+		[ProducesResponseType(500)]
+		public async Task<IActionResult> StopJobScheduler()
+		{
+			try
+			{
+				await _jobSchedulerHostedServiceStarter.StopAsync();
+
+				return Ok();
+			}
+			catch (ServiceException)
+			{
+				return StatusCode(StatusCodes.Status500InternalServerError);
+			}
+			catch (Exception exception)
+			{
+				Log.Error(exception, "Unexpected Exception");
+				return StatusCode(StatusCodes.Status500InternalServerError);
+			}
+		}
+	}
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Controllers/IngestionController.cs

@@ -0,0 +1,51 @@
+using System.Collections.Generic;
+using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.Mvc;
+using LCU.Api.Internal.Dispatcher.Extensions;
+using LCU.Api.Internal.Dispatcher.Services.DispatcherOperationServices;
+namespace LCU.Api.Internal.Dispatcher.Controllers;
+
+[Route("api/[controller]")]
+[ApiController]
+public class IngestionController : ControllerBase
+{
+    private readonly IIngestionService _ingestionService;
+
+    public IngestionController(IIngestionService ingestionService)
+    {
+        _ingestionService = ingestionService;
+    }
+
+    // POST api/Post
+    [HttpPost]
+    [ProducesResponseType(201)]
+    [ProducesResponseType(typeof(List<string>), 400)]
+    [ProducesResponseType(500)]
+    public async Task<IActionResult> Post([FromBody] Job request)
+    {
+        try
+        {
+            if (!ModelState.IsValid)
+            {
+                return BadRequest(ModelState.GetErrorMessages());
+            }
+
+            if (!request.WorkItems.Any())
+            {
+                return BadRequest("Workitems are required.");
+            }
+
+            await _ingestionService.HandleIngestionRequest(request);
+            return Ok();
+        }
+        catch (ServiceException)
+        {
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, "Unexpected Exception");
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+    }
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Controllers/JobController.cs

@@ -0,0 +1,304 @@
+using System.Collections.Generic;
+using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.Mvc;
+using LCU.Api.Internal.Dispatcher.Services;
+using LCU.Library.Internal.DispatcherModels.Models.Requests;
+
+namespace LCU.Api.Internal.Dispatcher.Controllers;
+
+[Route("api/[controller]")]
+[ApiController]
+public class JobController : ControllerBase
+{
+    private readonly IJobService _jobService;
+    public JobController(IJobService jobService)
+    {
+        _jobService = jobService;
+    }
+
+    [HttpPost("GetFilteredJobs")]
+    [ProducesResponseType(typeof(PagedResponse<Job>), 200)]
+    [ProducesResponseType(500)]
+    public async Task<IActionResult> GetFilteredJobs([FromBody] Filter filter)
+    {
+        try
+        {
+            var filterResult = await _jobService.GetFilteredJobs(filter);
+            var result = new PagedResponse<Job>(filterResult, filter);
+            return Ok(result);
+        }
+        catch (ServiceException)
+        {
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, "Unexpected Exception");
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+    }
+
+    [HttpGet(Name = "GetAllJobs")]
+    [ProducesResponseType(typeof(IEnumerable<Job>), 200)]
+    [ProducesResponseType(500)]
+    public async Task<IActionResult> Get()
+    {
+        try
+        {
+            var result = await _jobService.GetAsync();
+            return Ok(result);
+        }
+        catch (ServiceException)
+        {
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, "Unexpected Exception");
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+    }
+
+    [HttpGet("GetInactiveJobs", Name = "GetInactiveJobs")]
+    [ProducesResponseType(typeof(IEnumerable<Job>), 200)]
+    [ProducesResponseType(500)]
+    public async Task<IActionResult> GetInactive()
+    {
+        try
+        {
+            var result = await _jobService.GetInactiveAsync();
+            return Ok(result);
+        }
+        catch (ServiceException)
+        {
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, "Unexpected Exception");
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+    }
+
+    [HttpGet("GetInactiveJobsExcludeSoftDeleted", Name = "GetInactiveJobsExcludeSoftDeleted")]
+    [ProducesResponseType(typeof(IEnumerable<Job>), 200)]
+    [ProducesResponseType(500)]
+    public async Task<IActionResult> GetInactiveExcludeSoftDeleted()
+    {
+        try
+        {
+            var result = await _jobService.GetInactiveExcludeSoftDeletedTrackedAsync();
+            return Ok(result);
+        }
+        catch (ServiceException)
+        {
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, "Unexpected Exception");
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+    }
+
+    [HttpGet("GetSoftDeletedJobs", Name = "GetSoftDeletedJobs")]
+    [ProducesResponseType(typeof(IEnumerable<Job>), 200)]
+    [ProducesResponseType(500)]
+    public async Task<IActionResult> GetSoftDeleted(DateTime cutOffDate)
+    {
+        try
+        {
+            var result = await _jobService.GetSoftDeletedTrackedAsync(cutOffDate);
+            return Ok(result);
+        }
+        catch (ServiceException)
+        {
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, "Unexpected Exception");
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+    }
+
+    [HttpGet("GetJobByIdExcludingWorkItems", Name = "GetJobByIdExcludingWorkItems")]
+    [ProducesResponseType(typeof(Job), 200)]
+    [ProducesResponseType(500)]
+    public async Task<IActionResult> GetJobByIdExcludingWorkItems(int jobId)
+    {
+        try
+        {
+            var result = await _jobService.GetByIdExcludingWorkItemsAsync(jobId);
+            return Ok(result);
+        }
+        catch (ServiceException)
+        {
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, "Unexpected Exception");
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+    }
+
+    [HttpPut("RunJobNow", Name = "RunJobNow")]
+    [ProducesResponseType(typeof(Job), 200)]
+    [ProducesResponseType(500)]
+    public async Task<IActionResult> RunJobNow([FromBody] Job job)
+    {
+        try
+        {
+            var result = await _jobService.RunJobNowAsync(job);
+            return Ok(result);
+        }
+        catch (ServiceException)
+        {
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, "Unexpected Exception");
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+    }
+
+    [HttpPut("RunJobNowById", Name = "RunJobNowById")]
+    [ProducesResponseType(typeof(Job), 200)]
+    [ProducesResponseType(500)]
+    public async Task<IActionResult> RunJobNowById(int id)
+    {
+        try
+        {
+            var result = await _jobService.RunJobNowByIdAsync(id);
+            return Ok(result);
+        }
+        catch (ServiceException)
+        {
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, "Unexpected Exception");
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+    }
+
+    [HttpPut("DeactivateJob", Name = "DeactivateJob")]
+    [ProducesResponseType(typeof(Job), 200)]
+    [ProducesResponseType(500)]
+    public async Task<IActionResult> DeactivateJob([FromBody] Job job)
+    {
+        try
+        {
+            var result = await _jobService.DeactivateJobAsync(job);
+            return Ok(result);
+        }
+        catch (ServiceException)
+        {
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, "Unexpected Exception");
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+    }
+
+    [HttpPut("RescheduleJob", Name = "RescheduleJob")]
+    [ProducesResponseType(typeof(Job), 200)]
+    [ProducesResponseType(500)]
+    public async Task<IActionResult> RescheduleJob([FromBody] Job job)
+    {
+        try
+        {
+            var result = await _jobService.RescheduleJobAsync(job);
+            return Ok(result);
+        }
+        catch (ServiceException)
+        {
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, "Unexpected Exception");
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+    }
+
+    [HttpPut("RemoveSoftDeleteById", Name = "RemoveSoftDeleteById")]
+    [ProducesResponseType(typeof(Job), 200)]
+    [ProducesResponseType(500)]
+    public async Task<IActionResult> RemoveSoftDeleteById(int id)
+    {
+        try
+        {
+            var result = await _jobService.RemoveSoftDeleteById(id);
+            return Ok(result);
+        }
+        catch (ServiceException)
+        {
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, "Unexpected Exception");
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+    }
+
+    [HttpDelete()]
+    [ProducesResponseType(200)]
+    [ProducesResponseType(400)]
+    [ProducesResponseType(500)]
+    public async Task<IActionResult> Delete(int jobId)
+    {
+        try
+        {
+            if (jobId <= 0)
+            {
+                return BadRequest("Invalid Id");
+            }
+
+            await _jobService.DeleteAsync(jobId);
+            return Ok();
+        }
+        catch (ServiceException)
+        {
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, "Unexpected Exception");
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+    }
+    
+    [HttpPost("Copy")]
+    [ProducesResponseType(typeof(Job), 200)]
+    [ProducesResponseType(400)]
+    [ProducesResponseType(500)]
+    public async Task<IActionResult> Copy(CopyJobRequest request)
+    {
+        try
+        {
+            if (request == null)
+            {
+                return BadRequest("Request cannot be null");
+            }
+
+            var job = await _jobService.CopyJobAsync(request);
+            return Ok(job);
+        }
+        catch (ServiceException)
+        {
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, "Unexpected Exception");
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+    }
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Controllers/JobTagController.cs

@@ -0,0 +1,112 @@
+using System.Collections.Generic;
+using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.Mvc;
+using LCU.Api.Internal.Dispatcher.Services;
+namespace LCU.Api.Internal.Dispatcher.Controllers;
+
+[Route("[controller]")]
+[ApiController]
+public class JobTagController : ControllerBase
+{
+    private readonly IJobTagService _jobTagService;
+    public JobTagController(IJobTagService jobTagService)
+    {
+        _jobTagService = jobTagService;
+    }
+
+    [HttpGet]
+    [ProducesResponseType(typeof(IEnumerable<JobTag>), 200)]
+    [ProducesResponseType(500)]
+    public async Task<IActionResult> Get()
+    {
+        try
+        {
+            var result = await _jobTagService.GetAsync();
+            return Ok(result);
+        }
+        catch (ServiceException)
+        {
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, "Unexpected Exception");
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+    }
+
+    [HttpGet("GetByTagIdAsync")]
+    [ProducesResponseType(typeof(IEnumerable<JobTag>), 200)]
+    [ProducesResponseType(400)]
+    [ProducesResponseType(500)]
+    public async Task<IActionResult> GetByTagIdAsync(int tagId)
+    {
+        try
+        {
+            if (tagId <= 0)
+            {
+                return BadRequest("Invalid TagId.");
+            }
+
+            var result = await _jobTagService.GetByTagIdAsync(tagId);
+            return Ok(result);
+        }
+        catch (ServiceException)
+        {
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, "Unexpected Exception");
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+    }
+
+    [HttpPost()]
+    [ProducesResponseType(typeof(JobTag), 200)]
+    [ProducesResponseType(500)]
+    public async Task<IActionResult> Post([FromBody] JobTag request)
+    {
+        try
+        {
+            var result = await _jobTagService.SaveAsync(request);
+            return Ok(result);
+        }
+        catch (ServiceException)
+        {
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, "Unexpected Exception");
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+    }
+
+    [HttpDelete()]
+    [ProducesResponseType(200)]
+    [ProducesResponseType(400)]
+    [ProducesResponseType(500)]
+    public async Task<IActionResult> Delete(int jobTagId)
+    {
+        try
+        {
+            if (jobTagId <= 0)
+            {
+                return BadRequest("Invalid JobTagId");
+            }
+
+            await _jobTagService.Delete(jobTagId);
+            return Ok();
+        }
+        catch (ServiceException)
+        {
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, "Unexpected Exception");
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+    }
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Controllers/QueueCompletionController.cs

@@ -0,0 +1,45 @@
+using System.Collections.Generic;
+using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.Mvc;
+using LCU.Api.Internal.Dispatcher.Extensions;
+using LCU.Api.Internal.Dispatcher.Services.QueueServices;
+namespace LCU.Api.Internal.Dispatcher.Controllers;
+
+[Route("api/[controller]")]
+[ApiController]
+public class QueueCompletionController : ControllerBase
+{
+    private readonly IQueueCompletionService _queueCompletionService;
+
+    public QueueCompletionController(IQueueCompletionService completionService)
+    {
+        _queueCompletionService = completionService;
+    }
+
+    [HttpPost]
+    [ProducesResponseType(202)]
+    [ProducesResponseType(typeof(List<string>), 400)]
+    [ProducesResponseType(500)]
+    public async Task<IActionResult> Post([FromBody] DispatcherReturnResponse request)
+    {
+        try
+        {
+            if (!ModelState.IsValid)
+            {
+                return BadRequest(ModelState.GetErrorMessages());
+            }
+
+            await _queueCompletionService.QueueCompletion(request);
+            return Accepted();
+        }
+        catch (ServiceException)
+        {
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, "Unexpected Exception");
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+    }
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Controllers/QueueExecutionController.cs

@@ -0,0 +1,45 @@
+using System.Collections.Generic;
+using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.Mvc;
+using LCU.Api.Internal.Dispatcher.Extensions;
+using LCU.Api.Internal.Dispatcher.Services.QueueServices;
+namespace LCU.Api.Internal.Dispatcher.Controllers;
+
+[Route("api/[controller]")]
+[ApiController]
+public class QueueExecutionController : ControllerBase
+{
+    private readonly IQueueExecutionService _queueExecutionService;
+
+    public QueueExecutionController(IQueueExecutionService executionService)
+    {
+        _queueExecutionService = executionService;
+    }
+
+    [HttpPost("QueueRetryExecution")]
+    [ProducesResponseType(202)]
+    [ProducesResponseType(typeof(List<string>), 400)]
+    [ProducesResponseType(500)]
+    public async Task<IActionResult> QueueRetryExecutionController([FromBody] WorkItemExecution request)
+    {
+        try
+        {
+            if (!ModelState.IsValid)
+            {
+                return BadRequest(ModelState.GetErrorMessages());
+            }
+
+            await _queueExecutionService.QueueRetryExecution(request);
+            return Accepted();
+        }
+        catch (ServiceException)
+        {
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, "Unexpected Exception");
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+    }
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Controllers/StatusController.cs

@@ -0,0 +1,31 @@
+using Microsoft.AspNetCore.Mvc;
+using LCU.Api.Internal.Dispatcher.Models;
+namespace LCU.Api.Internal.Dispatcher.Controllers;
+
+[Route("api/[controller]")]
+[ApiController]
+public class StatusController : ControllerBase
+{
+    private readonly IEnvironmentView _environmentView;
+
+    public StatusController(IEnvironmentView environmentView)
+    {
+        _environmentView = environmentView;
+    }
+
+    [HttpGet()]
+    [ProducesResponseType(200)]
+    [ProducesResponseType(500)]
+    public IActionResult Get()
+    {
+        try
+        {
+            return Ok(_environmentView);
+        }
+        catch (Exception ex)
+        {
+            Log.Error(ex, "Failure to return status");
+            return StatusCode((int)System.Net.HttpStatusCode.InternalServerError);
+        }
+    }
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Controllers/TagController.cs

@@ -0,0 +1,137 @@
+using System.Collections.Generic;
+using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.Mvc;
+using LCU.Api.Internal.Dispatcher.Services;
+namespace LCU.Api.Internal.Dispatcher.Controllers;
+
+[Route("[controller]")]
+[ApiController]
+public class TagController : ControllerBase
+{
+    private readonly ITagService _tagService;
+    public TagController(ITagService tagService)
+    {
+        _tagService = tagService;
+    }
+
+    [HttpGet]
+    [ProducesResponseType(typeof(IEnumerable<Tag>), 200)]
+    [ProducesResponseType(500)]
+    public async Task<IActionResult> Get()
+    {
+        try
+        {
+            var result = await _tagService.GetAsync();
+            return Ok(result);
+        }
+        catch (ServiceException)
+        {
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, "Unexpected Exception");
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+    }
+
+    [HttpGet("GetByIdAsync")]
+    [ProducesResponseType(typeof(Tag), 200)]
+    [ProducesResponseType(400)]
+    [ProducesResponseType(500)]
+    public async Task<IActionResult> GetByIdAsync(int tagId)
+    {
+        try
+        {
+            if (tagId <= 0)
+            {
+                return BadRequest("Invalid TagId.");
+            }
+
+            var result = await _tagService.GetByIdAsync(tagId);
+            return Ok(result);
+        }
+        catch (ArgumentNullException)
+        {
+            return StatusCode(StatusCodes.Status404NotFound);
+        }
+        catch (ServiceException)
+        {
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, "Unexpected Exception");
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+    }
+
+    [HttpPost()]
+    [ProducesResponseType(typeof(Tag), 200)]
+    [ProducesResponseType(500)]
+    public async Task<IActionResult> Post([FromBody] Tag request)
+    {
+        try
+        {
+            var result = await _tagService.SaveAsync(request);
+            return Ok(result);
+        }
+        catch (ServiceException)
+        {
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, "Unexpected Exception");
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+    }
+
+    [HttpPut()]
+    [ProducesResponseType(typeof(Tag), 200)]
+    [ProducesResponseType(500)]
+    public async Task<IActionResult> Put([FromBody] Tag request)
+    {
+        try
+        {
+            var result = await _tagService.UpdateAsync(request);
+            return Ok(result);
+        }
+        catch (ServiceException)
+        {
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, "Unexpected Exception");
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+    }
+
+    [HttpDelete()]
+    [ProducesResponseType(200)]
+    [ProducesResponseType(400)]
+    [ProducesResponseType(500)]
+    public async Task<IActionResult> Delete(int tagId)
+    {
+        try
+        {
+            if (tagId <= 0)
+            {
+                return BadRequest("Invalid TagId");
+            }
+
+            await _tagService.DeleteAsync(tagId);
+            return Ok();
+        }
+        catch (ServiceException)
+        {
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, "Unexpected Exception");
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+    }
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Controllers/WorkItemController.cs

@@ -0,0 +1,39 @@
+using LCU.Api.Internal.Dispatcher.Services;
+using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.Mvc;
+using System.Collections.Generic;
+
+namespace LCU.Api.Internal.Dispatcher.Controllers;
+
+[Route("api/[controller]")]
+[ApiController]
+public class WorkItemController : ControllerBase
+{
+    private readonly IWorkItemService _workItemService;
+
+    public WorkItemController(IWorkItemService workItemService)
+    {
+        _workItemService = workItemService;
+    }
+
+    [HttpPost("GetWorkItemsByJobId")]
+    [ProducesResponseType(typeof(IEnumerable<WorkItem>), 200)]
+    [ProducesResponseType(500)]
+    public async Task<IActionResult> GetWorkItemsByJobId(int jobId)
+    {
+        try
+        {
+            var result = await _workItemService.GetByJobIdWithExecutionsAsync(jobId);
+            return Ok(result);
+        }
+        catch (ServiceException)
+        {
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, "Unexpected Exception");
+            return StatusCode(StatusCodes.Status500InternalServerError);
+        }
+    }
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/DataAccess/INotificationsApiDao.cs

@@ -0,0 +1,8 @@
+using LCU.Library.Internal.NotificationsModels.Models;
+
+namespace LCU.Api.Internal.Dispatcher.DataAccess;
+
+public interface INotificationsApiDao
+{
+    Task SendEmailAsync(EmailRequest request);
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/DataAccess/NotificationsApiDao.cs

@@ -0,0 +1,21 @@
+using LCU.Api.Internal.DispatcherSharedLib.Configurations;
+using LCU.Library.Internal.LcuHttpClient.HttpRequest;
+using LCU.Library.Internal.NotificationsModels.Models;
+namespace LCU.Api.Internal.Dispatcher.DataAccess;
+
+public class NotificationsApiDao : INotificationsApiDao
+{
+    private readonly NotificationApiConfiguration _configuration;
+    private readonly IPostRequest _postRequest;
+
+    public NotificationsApiDao(NotificationApiConfiguration configuration, IPostRequest postRequest)
+    {
+        _configuration = configuration;
+        _postRequest = postRequest;
+    }
+
+    public async Task SendEmailAsync(EmailRequest request)
+    {
+        await _postRequest.Async(new Uri(_configuration.EmailEndpointUrl), request);
+    }
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Extensions/ModelStateMessages.cs

@@ -0,0 +1,14 @@
+using System.Collections.Generic;
+using System.Linq;
+using Microsoft.AspNetCore.Mvc.ModelBinding;
+namespace LCU.Api.Internal.Dispatcher.Extensions;
+
+public static class ModelStateMessages
+{
+    public static List<string> GetErrorMessages(this ModelStateDictionary dictionary)
+    {
+        return dictionary.SelectMany(m => m.Value.Errors)
+            .Select(m => m.ErrorMessage)
+            .ToList();
+    }
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/HostedServices/BatchErroredJobsHostedService.cs

@@ -0,0 +1,68 @@
+using System.Threading;
+using Microsoft.Extensions.DependencyInjection;
+using NCrontab;
+using LCU.Api.Internal.DispatcherSharedLib.Utilities;
+using LCU.Api.Internal.Dispatcher.Configurations;
+using LCU.Api.Internal.Dispatcher.Services.DispatcherOperationServices;
+using LCU.Library.Internal.LCUHostedService.LCUBackgroundService;
+namespace LCU.Api.Internal.Dispatcher.HostedServices;
+
+public class BatchErroredJobsHostedService : LCUBackgroundService, IBatchErroredJobsHostedService
+{
+    private readonly ApiConfiguration _apiConfiguration;
+    private readonly IServiceScopeFactory _scopeFactory;
+    private readonly CrontabSchedule _batchErroredJobsSchedule;
+    private DateTime _nextScheduleRun;
+    public const string DefaultBatchErroredJobsCronPattern = "*/15 * * * *";
+    private CancellationToken _cancellationToken;
+
+    public BatchErroredJobsHostedService(IServiceScopeFactory scopeFactory,
+                                  ApiConfiguration apiConfiguration)
+    {
+        _scopeFactory = scopeFactory;
+        _apiConfiguration = apiConfiguration;
+        _batchErroredJobsSchedule = CronHelpers.SetCronSchedule(_apiConfiguration.BatchErroredJobsCronSchedule,
+                                                                DefaultBatchErroredJobsCronPattern);
+        _nextScheduleRun = _batchErroredJobsSchedule.GetNextOccurrence(DateTime.Now);
+        _cancellationToken = new CancellationToken();
+    }
+
+    protected override async Task ExecuteAsync(CancellationToken cancellationToken)
+    {
+        if (!_apiConfiguration.FeatureFlags.RunBatchErroredJobsHostedService)
+        {
+			Log.Error("Batch Errored Jobs Hosted Service is disabled by feature flag. Service not started.");
+			return;
+		}
+
+        Log.Debug("Dispatcher's Batch Errored Jobs Hosted Service is starting.");
+        try
+        {
+            while (!cancellationToken.IsCancellationRequested)
+            {
+                await Task.Delay(UntilNextExecution(), cancellationToken); // wait until next time
+
+                using (var scope = _scopeFactory.CreateAsyncScope())
+                {
+                    Log.Debug("Running Batch Errored Jobs Service");
+                    await scope.ServiceProvider.GetRequiredService<IBatchErroredJobsService>().PerformBatchErroredJobs(_cancellationToken);
+                }
+
+                _nextScheduleRun = _batchErroredJobsSchedule.GetNextOccurrence(DateTime.Now);
+            }
+        }
+        catch(OperationCanceledException taskCanceledEx)
+        {
+            Log.Error(taskCanceledEx, $"Manual start/stop of Dispatcher's Batch Errored Jobs Hosted Service was initiated.");
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception,
+                "Error trying to startup main thread for Dispatcher's Batch Errored Jobs Hosted Service");
+        }
+
+        Log.Debug("Dispatcher's Batch Errored Jobs Hosted Service is stopping.");
+    }
+
+    private int UntilNextExecution() => Math.Max(0, (int)_nextScheduleRun.Subtract(DateTime.Now).TotalMilliseconds);
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/HostedServices/CleanUpJobsHostedService.cs

@@ -0,0 +1,69 @@
+using LCU.Api.Internal.Dispatcher.Configurations;
+using LCU.Api.Internal.DispatcherSharedLib.Utilities;
+using LCU.Api.Internal.Dispatcher.Services.DispatcherOperationServices;
+using Microsoft.Extensions.DependencyInjection;
+using NCrontab;
+using System.Threading;
+using LCU.Library.Internal.LCUHostedService.LCUBackgroundService;
+
+namespace LCU.Api.Internal.Dispatcher.HostedServices;
+
+public class CleanUpJobsHostedService : LCUBackgroundService, ICleanUpJobsHostedService
+{
+    private readonly ApiConfiguration _apiConfiguration;
+    private readonly IServiceScopeFactory _scopeFactory;
+    private readonly CrontabSchedule _cleanUpJobSchedule;
+    private DateTime _nextScheduleRun;
+    public const string DefaultCleanUpJobCronPattern = "* 22 * * *";
+    private CancellationToken _cancellationToken;
+
+    public CleanUpJobsHostedService(IServiceScopeFactory scopeFactory,
+                                  ApiConfiguration apiConfiguration)
+    {
+        _scopeFactory = scopeFactory;
+        _apiConfiguration = apiConfiguration;
+        _cleanUpJobSchedule = CronHelpers.SetCronSchedule(_apiConfiguration.CleanUpJobsCronSchedule,
+                                                                DefaultCleanUpJobCronPattern);
+        _nextScheduleRun = _cleanUpJobSchedule.GetNextOccurrence(DateTime.Now);
+        _cancellationToken = new CancellationToken();
+    }
+
+    protected override async Task ExecuteAsync(CancellationToken cancellationToken)
+    {
+        if (!_apiConfiguration.FeatureFlags.RunCleanUpJobsHostedService)
+        {
+			Log.Error("Clean Up Job Hosted Service is disabled by feature flag. Service not started.");
+			return;
+		}
+
+        Log.Debug("Dispatcher's Clean Up Job Hosted Service is starting.");
+        try
+        {
+            while (!cancellationToken.IsCancellationRequested)
+            {
+                await Task.Delay(UntilNextExecution(), cancellationToken); // wait until next time
+
+                using (var scope = _scopeFactory.CreateAsyncScope())
+                {
+                    Log.Debug("Running Clean Up Job Service");
+                    await scope.ServiceProvider.GetRequiredService<ICleanUpJobsService>().PerformCleanUpJobs(_cancellationToken);
+                }
+
+                _nextScheduleRun = _cleanUpJobSchedule.GetNextOccurrence(DateTime.Now);
+            }
+        }
+        catch(OperationCanceledException taskCanceledEx)
+        {
+            Log.Error(taskCanceledEx, $"Manual start/stop of Dispatcher's Clean Up Job Hosted Service was initiated.");
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception,
+                "Error trying to startup main thread for Dispatcher's Clean Up Job Hosted Service");
+        }
+
+        Log.Debug("Dispatcher's Clean Up Job Hosted Service is stopping.");
+    }
+
+    private int UntilNextExecution() => Math.Max(0, (int)_nextScheduleRun.Subtract(DateTime.Now).TotalMilliseconds);
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/HostedServices/IBatchErroredJobsHostedService.cs

@@ -0,0 +1,8 @@
+using LCU.Library.Internal.LCUHostedService.LCUBackgroundService;
+
+namespace LCU.Api.Internal.Dispatcher.HostedServices
+{
+	public interface IBatchErroredJobsHostedService : ILCUBackgroundService
+	{
+	}
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/HostedServices/ICleanUpJobsHostedService.cs

@@ -0,0 +1,8 @@
+using LCU.Library.Internal.LCUHostedService.LCUBackgroundService;
+
+namespace LCU.Api.Internal.Dispatcher.HostedServices
+{
+	public interface ICleanUpJobsHostedService : ILCUBackgroundService
+	{
+	}
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/HostedServices/IJobSchedulerHostedService.cs

@@ -0,0 +1,8 @@
+using LCU.Library.Internal.LCUHostedService.LCUBackgroundService;
+
+namespace LCU.Api.Internal.Dispatcher.HostedServices
+{
+	public interface IJobSchedulerHostedService : ILCUBackgroundService
+	{
+	}
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/HostedServices/JobSchedulerHostedService.cs

@@ -0,0 +1,58 @@
+using System.Threading;
+using Microsoft.Extensions.DependencyInjection;
+using NCrontab;
+using LCU.Api.Internal.DispatcherSharedLib.Utilities;
+using LCU.Api.Internal.Dispatcher.Configurations;
+using LCU.Api.Internal.Dispatcher.Services.DispatcherOperationServices;
+using LCU.Library.Internal.LCUHostedService.LCUBackgroundService;
+namespace LCU.Api.Internal.Dispatcher.HostedServices;
+
+public class JobSchedulerHostedService : LCUBackgroundService, IJobSchedulerHostedService
+{
+    private readonly ApiConfiguration _apiConfiguration;
+    private readonly IServiceScopeFactory _scopeFactory;
+    private CancellationToken _cancellationToken;
+
+    public JobSchedulerHostedService(IServiceScopeFactory scopeFactory,
+                                  ApiConfiguration apiConfiguration)
+    {
+        _scopeFactory = scopeFactory;
+        _apiConfiguration = apiConfiguration;
+        _cancellationToken = new CancellationToken();
+    }
+
+    protected override async Task ExecuteAsync(CancellationToken cancellationToken)
+    {
+		if (!_apiConfiguration.FeatureFlags.RunJobSchedulerHostedService)
+		{
+			Log.Error("Job Scheduler Hosted Service is disabled by feature flag. Service not started.");
+			return;
+		}
+
+		Log.Debug("Dispatcher's Job Schedule Hosted Service is starting.");
+        try
+        {
+            while (!cancellationToken.IsCancellationRequested)
+            {
+                await Task.Delay(_apiConfiguration.DefaultDelayMilliseconds, cancellationToken); // wait until next time
+
+                using (var scope = _scopeFactory.CreateAsyncScope())
+                {
+                    Log.Debug("Running Job Schedule Service");
+                    await scope.ServiceProvider.GetRequiredService<IJobScheduleService>().PerformSchedule(_cancellationToken);
+                }
+            }
+        }
+        catch(OperationCanceledException taskCanceledEx)
+        {
+            Log.Error(taskCanceledEx, $"Manual start/stop of Dispatcher's Job Scheduler Hosted Service was initiated.");
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception,
+                "Error trying to startup main thread for Dispatcher's Job Schedule Hosted Service");
+        }
+
+        Log.Debug("Dispatcher's Job Schedule Hosted Service is stopping.");
+    }
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/LCU.Api.Internal.Dispatcher.csproj

@@ -0,0 +1,96 @@
+<Project Sdk="Microsoft.NET.Sdk.Web">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <AspNetCoreHostingModel>InProcess</AspNetCoreHostingModel>
+    <Version>2.8.1</Version>
+    <Description>
+      1.1.0 - Added NotificationsApi EmailRequest
+      1.2.0 - Added FileOperations CreateDirectory and RenameDirectory
+      1.3.0 - Added required end points and methods to handle Job Run now, Deactivation, Reschedule and Delete.
+      1.4.0 - Fix to always copy HTML Templates for Error Monitoring on build.
+      1.4.1 - Fix scheduler for Error Monitoring
+      1.5.0 - Addition of the JobErrorRecipient and functionality.
+      1.6.0 - Add Dynamic Zip and UnzipFileRequest
+      1.7.0 - Add ArchiveRequest
+      1.8.0 - Add Encrypt/Decrypt and SucceedOnError to Move/Copy/Rename/Delete File
+      1.9.0 - Upgrade to .net 6, upgrade to use new apiresponse, and some refactoring
+      2.0.0 - No longer Queueing Ingestion, removal of QueueLib, and clean up to be fully using MassTransit
+      2.1.0 - Added DocumentIngestor IngestorRequest
+      2.2.0 - Added DocumentIngestor IngestorPdfPartitionerRequest
+      2.3.0 - Added MemberDocumentNotificationRequest Workitem
+      2.4.0 - Added RemoteDepositCaptureXmlRequest Workitem and updated DocumentIngestor models
+      2.5.0 - Added DeleteDirectoryRequest Workitem and updated DocumentIngestor models
+      2.6.0 - Added UpdatePersonForCheckHoldRequest and DataCleanup Workitems
+      2.7.0 - Update FileOperations and FTP Api nuget for additions to SFTP upload and modify file
+      2.7.1 - Added HistoricalDocumentIngestor, HistoricalDocumentBatchIngestion, and HistoricalIngestorPdfPartitioner WorkItems
+      2.8.0 - Added ExtractRequest and ProcessingEmailRequest WorkItems from NoticeProcessor API
+      2.8.1 - Use new ActionBuilder and ActionService classes and other Refactors.
+    </Description>
+    <PackageReleaseNotes>
+      1.1.0 - Added NotificationsApi EmailRequest
+      1.2.0 - Added FileOperations CreateDirectory and RenameDirectory
+      1.3.0 - Added required end points and methods to handle Job Run now, Deactivation, Reschedule and Delete.
+      1.4.0 - Fix to always copy HTML Templates for Error Monitoring on build.
+      1.4.1 - Fix scheduler for Error Monitoring.
+      1.5.0 - Addition of the JobErrorRecipient and functionality.
+      1.6.0 - Add Dynamic Zip and UnzipFileRequest
+      1.7.0 - Add ArchiveRequest
+      1.8.0 - Add Encrypt/Decrypt and SucceedOnError to Move/Copy/Rename/Delete File
+      1.9.0 - Upgrade to .net 6, upgrade to use new apiresponse, and some refactoring
+      2.0.0 - No longer Queueing Ingestion, removal of QueueLib, and clean up to be fully using MassTransit
+      2.1.0 - Added DocumentIngestor IngestorRequest
+      2.2.0 - Added DocumentIngestor IngestorPdfPartitionerRequest
+      2.3.0 - Added MemberDocumentNotificationRequest Workitem
+      2.4.0 - Added RemoteDepositCaptureXmlRequest Workitem and updated DocumentIngestor models
+      2.5.0 - Added DeleteDirectoryRequest Workitem and updated DocumentIngestor models
+      2.6.0 - Added UpdatePersonForCheckHoldRequest and DataCleanup Workitems
+      2.7.0 - Update FileOperations and FTP Api nuget for additions to SFTP upload and modify file
+      2.7.1 - Added HistoricalDocumentIngestor, HistoricalDocumentBatchIngestion, and HistoricalIngestorPdfPartitioner WorkItems
+      2.8.0 - Added ExtractRequest and ProcessingEmailRequest WorkItems from NoticeProcessor API
+      2.8.1 - Use new ActionBuilder and ActionService classes and other Refactors.
+    </PackageReleaseNotes>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="LCU.Api.Internal.MaidBotLib" Version="1.0.2" />
+    <PackageReference Include="LCU.Library.Internal.ApiResponse" Version="3.4.1" />
+    <PackageReference Include="LCU.Library.Internal.ApiSecurity" Version="2.0.0" />
+    <PackageReference Include="LCU.Library.Internal.Configuration" Version="1.0.0" />
+    <PackageReference Include="LCU.Library.Internal.Exceptions" Version="1.0.0" />
+    <PackageReference Include="LCU.Library.Internal.LCUHostedService" Version="1.0.0" />
+    <PackageReference Include="LCU.Library.Internal.LcuHttpClient" Version="2.0.1" />
+    <PackageReference Include="LCU.Library.Internal.LcuMassTransit" Version="1.0.0" />
+    <PackageReference Include="LCU.Library.Internal.LcuMassTransitModels" Version="1.1.0" />
+    <PackageReference Include="LCU.Library.Internal.Logging" Version="2.2.0" />
+    <PackageReference Include="LCU.Library.Internal.RepoApiModels" Version="1.0.0" />
+    <PackageReference Include="LCU.Library.Internal.TypeExtensions" Version="1.2.2" />
+    <PackageReference Include="LCU.Shared.Collections.RepoApiLib" Version="2.4.5" />
+    <PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="3.0.0" />
+    <PackageReference Include="Microsoft.AspNetCore.Hosting" Version="2.2.7" />
+    <PackageReference Include="Microsoft.EntityFrameworkCore.SqlServer" Version="7.0.5" />
+    <PackageReference Include="Microsoft.VisualStudio.Web.CodeGeneration.Design" Version="2.2.3" />
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
+    <PackageReference Include="Swashbuckle.AspNetCore" Version="5.6.3" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\LCU.Api.Internal.DispatcherLib\LCU.Api.Internal.DispatcherLib.csproj" />
+    <ProjectReference Include="..\LCU.Api.Internal.DispatcherSharedLib\LCU.Api.Internal.DispatcherSharedLib.csproj" />
+    <ProjectReference Include="..\LCU.Library.Internal.DispatcherModels\LCU.Library.Internal.DispatcherModels.csproj" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <Folder Include="Properties\" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <None Update="Templates\BatchErroredJobsTemplate.html">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="Templates\ErrorRowTemplate.html">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+  </ItemGroup>
+
+</Project>

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Models/EnvironmentView.cs

@@ -0,0 +1,30 @@
+using System;
+using System.Reflection;
+using Microsoft.AspNetCore.Hosting;
+namespace LCU.Api.Internal.Dispatcher.Models;
+
+public class EnvironmentView : IEnvironmentView
+{
+    public string Application => _env.ApplicationName;
+    public DateTime CurrentDate => DateTime.Now;
+    public string BuildVersion => Assembly.GetExecutingAssembly().GetName().Version.ToString();
+    public DateTime BuildDate => AssemblyBuildDate();
+    public string Environment => _env.EnvironmentName;
+    private readonly IWebHostEnvironment _env;
+
+    public EnvironmentView(IWebHostEnvironment env)
+    {
+        _env = env;
+    }
+
+    public static DateTime AssemblyBuildDate()
+    {
+        //https://stackoverflow.com/a/804310/266594
+        System.Reflection.Assembly assembly = System.Reflection.Assembly.GetExecutingAssembly();
+        System.IO.FileInfo fileInfo = new System.IO.FileInfo(assembly.Location);
+        DateTime lastModified = fileInfo.LastWriteTime;
+
+        return lastModified;
+    }
+
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Models/IEnvironmentView.cs

@@ -0,0 +1,11 @@
+using System;
+namespace LCU.Api.Internal.Dispatcher.Models;
+
+public interface IEnvironmentView
+{
+    string Application { get; }
+    DateTime CurrentDate { get; }
+    string BuildVersion { get; }
+    DateTime BuildDate { get; }
+    string Environment { get; }
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Program.cs

@@ -0,0 +1,396 @@
+using System.IO;
+using System.Reflection;
+using Microsoft.AspNetCore.Builder;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;
+using Microsoft.OpenApi.Models;
+using Newtonsoft.Json;
+using MassTransit;
+using LCU.Library.Internal.Configuration;
+using LCU.Library.Internal.LcuMassTransit;
+using LCU.Library.Internal.LcuMassTransitModels;
+using LCU.Library.Internal.Logging;
+using LCU.Library.Internal.Logging.Middleware;
+using LCU.Library.Internal.Logging.Models;
+using LCU.Api.Internal.Dispatcher.Configurations;
+using LCU.Api.Internal.Dispatcher.Consumers;
+using LCU.Api.Internal.Dispatcher.HostedServices;
+using LCU.Api.Internal.Dispatcher.Models;
+using LCU.Api.Internal.Dispatcher.Services;
+using LCU.Api.Internal.Dispatcher.Services.DispatcherOperationServices;
+using LCU.Api.Internal.Dispatcher.Services.QueueServices;
+using LCU.Api.Internal.DispatcherLib.Persistence;
+using LCU.Api.Internal.DispatcherLib.Repositories;
+using LCU.Api.Internal.DispatcherSharedLib;
+using System.Collections.Generic;
+using LCU.Library.Internal.ApiSecurity.Constants;
+using LCU.Library.Internal.ApiSecurity;
+using LCU.Library.Internal.LcuHttpClient;
+using LCU.Api.Internal.Dispatcher.DataAccess;
+
+string MyAllowSpecificOrigins = "_myAllowSpecificOrigins";
+ApiConfiguration _apiConfiguration;
+
+var builder = BuildBuilder(args);
+BuildServicesAndConfigurations(builder);
+BuildAppAndRun(builder);
+
+WebApplicationBuilder BuildBuilder(string[] args)
+{
+    var builder = WebApplication.CreateBuilder(args);
+    builder.Configuration.AddJsonFile("/app/config/environment-appsettings/appsettings.json", optional: true, reloadOnChange: true);
+    builder.Configuration.AddJsonFile("/app/config/workload-appsettings/appsettings.json", optional: true, reloadOnChange: true);
+    builder.Configuration.AddJsonFile("/vault/secrets/dispatcher-api-database-connection.json", optional: true, reloadOnChange: true);
+    builder.Configuration.AddJsonFile("/vault/secrets/dispatcher-api-seq-api-key.json", optional: true, reloadOnChange: true);
+    builder.Configuration.AddJsonFile("/vault/secrets/dispatcher-api-rabbitmq-username.json", optional: true, reloadOnChange: true);
+    builder.Configuration.AddJsonFile("/vault/secrets/dispatcher-api-rabbitmq-password.json", optional: true, reloadOnChange: true);
+    builder.Configuration.AddJsonFile("/vault/secrets/dispatcher-api-external-api-key-set.json", optional: true, reloadOnChange: true);
+	builder.Host.UseSerilog();
+    return builder;
+}
+
+void BuildServicesAndConfigurations(WebApplicationBuilder builder)
+{
+    var services = builder.Services;
+	var configuration = builder.Configuration;
+    services.AddControllers();
+    services.AddEndpointsApiExplorer();
+    services.AddSwaggerGen();
+    services.AddHttpClient();    
+
+	_apiConfiguration = configuration.Bind<ApiConfiguration>(services, "Primary");
+
+    var connectionsConfiguration = configuration.Bind<ConnectionsConfiguration>(services);    
+
+    var loggerConfiguration = new LoggingConfiguration(_apiConfiguration.AppName,
+                                                       _apiConfiguration.SeqApiKey,
+                                                       _apiConfiguration.SeqUrl);
+    var lcuLogging = new LcuLogging(loggerConfiguration);
+    lcuLogging.UseLcuLogging();    
+
+    var connectionConfiguration = MassTransitConfigurationSetUp.AddConnectionConfiguration(ref services, configuration);
+    //debugging
+    Console.WriteLine(JsonConvert.SerializeObject(connectionConfiguration));
+
+    var receiveEndpointConfigurations = new ReceiveEndpointConfigurations()
+    {
+        ExecutionReceiveEndpoint = MassTransitConfigurationSetUp.GetReceiveEndpointConfiguration(configuration,
+                                                                                                 "ExecutionReceiveEndpoint"),
+        CompletionReceiveEndpoint = MassTransitConfigurationSetUp.GetReceiveEndpointConfiguration(configuration,
+                                                                                                  "CompletionReceiveEndpoint"),
+        DetermineOperationReceiveEndpoint = MassTransitConfigurationSetUp.GetReceiveEndpointConfiguration(configuration,
+                                                                                                  "DetermineOperationReceiveEndpoint"),
+        JobFailureReceiveEndpoint = MassTransitConfigurationSetUp.GetReceiveEndpointConfiguration(configuration,
+                                                                                                  "JobFailureReceiveEndpoint"),
+        SendErroredJobAlertReceiveEndpoint = MassTransitConfigurationSetUp.GetReceiveEndpointConfiguration(configuration,
+                                                                                                  "SendErroredJobAlertReceiveEndpoint")
+    };
+    services.AddSingleton(receiveEndpointConfigurations);
+    //debugging
+    Console.WriteLine(JsonConvert.SerializeObject(receiveEndpointConfigurations));
+
+    services.AddCors(options =>
+    {
+        options.AddPolicy(MyAllowSpecificOrigins,
+            builder =>
+            {
+                builder.WithOrigins(
+                    _apiConfiguration.CorsAllowedOrigins
+                ).AllowAnyHeader().AllowAnyMethod();
+            });
+    });
+
+    services.AddDbContextPool<AppDbContext>(options =>
+    {
+        options.UseSqlServer(
+            connectionsConfiguration.Sql.DispatcherDb,
+            sqlServerOptionsAction: sqlOptions =>
+            {
+                sqlOptions.EnableRetryOnFailure(
+                    maxRetryCount: connectionsConfiguration.Sql.SQLOptions.MaxRetryCount,
+                    maxRetryDelay: connectionsConfiguration.Sql.SQLOptions.MaxRetryDelay,
+                    errorNumbersToAdd: connectionsConfiguration.Sql.SQLOptions.ErrorNumbersToAdd
+                );
+                sqlOptions.CommandTimeout(connectionsConfiguration.Sql.SQLOptions.CommandTimeout);
+            }
+        );
+        options.EnableDetailedErrors(connectionsConfiguration.Sql.SQLOptions.EnableDetailedErrors);
+    });
+
+    AddMassTransitServices(ref services, connectionConfiguration, receiveEndpointConfigurations, _apiConfiguration);
+    services.AddDispatcherServices(configuration);
+    services.AddLCUHttpServices();
+    AddServices(services);
+    AddRepositories(services);
+    AddHostedServices(services);
+
+    services.AddSwaggerGen(c =>
+    {
+        c.SwaggerDoc("v1", new OpenApiInfo { Title = $"{_apiConfiguration.AppName}", Version = "v1" });
+
+        //Add Security Entry Fields In Swagger
+        c.AddSecurityRequirement(new OpenApiSecurityRequirement
+        {
+            {
+                new OpenApiSecurityScheme
+                {
+                    Reference = new OpenApiReference
+                    {
+                        Id = HeaderKeys.ApiKey, // Must match the "AddSecurityDefinition"
+                        Type = ReferenceType.SecurityScheme
+                    }
+                },
+                new List<string>()
+            }
+        });
+
+        c.AddSecurityRequirement(new OpenApiSecurityRequirement
+        {
+            {
+                new OpenApiSecurityScheme
+                {
+                    Reference = new OpenApiReference
+                    {
+                        Id = HeaderKeys.Client, // Must match the "AddSecurityDefinition"
+                        Type = ReferenceType.SecurityScheme
+                    }
+                },
+                new List<string>()
+            }
+        });
+
+        c.AddSecurityDefinition(HeaderKeys.ApiKey,
+            new OpenApiSecurityScheme
+            {
+                Description = HeaderKeys.ApiKey,
+                In = ParameterLocation.Header,
+                Name = HeaderKeys.ApiKey,
+                Type = SecuritySchemeType.ApiKey,
+                Scheme = "apikey"
+            });
+
+        c.AddSecurityDefinition(HeaderKeys.Client,
+            new OpenApiSecurityScheme
+            {
+                Description = HeaderKeys.Client,
+                In = ParameterLocation.Header,
+                Name = HeaderKeys.Client,
+                Type = SecuritySchemeType.ApiKey,
+                Scheme = "apikey"
+            });
+    });
+
+    builder.Services.AddApiKeySecurity(builder.Configuration);
+}
+
+void AddMassTransitServices(ref IServiceCollection services,
+                            ConnectionConfiguration connectionConfiguration,
+                            ReceiveEndpointConfigurations receiveEndpointConfigurations,
+                            ApiConfiguration apiConfiguration)
+{
+    services.AddMassTransit(x =>
+    {
+        x.AddConsumer<ExecutionConsumer>();
+        x.AddConsumer<CompletionConsumer>();
+        x.AddConsumer<DetermineOperationConsumer>();
+        x.AddConsumer<JobFailureConsumer>();
+        x.AddConsumer<SendErroredJobAlertConsumer>();
+
+		x.UsingRabbitMq((context, cfg) =>
+        {
+            cfg.Host(connectionConfiguration.HostUrl, "/", host =>
+            {
+                host.Username(connectionConfiguration.UserName);
+                host.Password(connectionConfiguration.Password);
+            });
+
+            if (apiConfiguration.FeatureFlags.RunExecutionConsumer)
+            {
+                cfg.ReceiveEndpoint(receiveEndpointConfigurations.ExecutionReceiveEndpoint.ReceiveEndpoint, e =>
+                {
+                    e.ConfigureConsumer<ExecutionConsumer>(context);
+                    e.SetQuorumQueue();
+                    e.PrefetchCount = receiveEndpointConfigurations.CompletionReceiveEndpoint.PrefetchLimit;
+                });
+            }
+
+            if (apiConfiguration.FeatureFlags.RunCompletionConsumer)
+            {
+                cfg.ReceiveEndpoint(receiveEndpointConfigurations.CompletionReceiveEndpoint.ReceiveEndpoint, e =>
+                {
+                    e.ConfigureConsumer<CompletionConsumer>(context);
+                    e.SetQuorumQueue();
+                    e.PrefetchCount = receiveEndpointConfigurations.CompletionReceiveEndpoint.PrefetchLimit;
+                });
+            }
+            
+            if (apiConfiguration.FeatureFlags.RunDetermineOperationConsumer)
+            {
+                cfg.ReceiveEndpoint(receiveEndpointConfigurations.DetermineOperationReceiveEndpoint.ReceiveEndpoint, e =>
+                {
+                    e.ConfigureConsumer<DetermineOperationConsumer>(context);
+                    e.SetQuorumQueue();
+                    e.PrefetchCount = receiveEndpointConfigurations.DetermineOperationReceiveEndpoint.PrefetchLimit;
+                });
+            }
+
+            if (apiConfiguration.FeatureFlags.RunJobFailureConsumer)
+            {
+                cfg.ReceiveEndpoint(receiveEndpointConfigurations.JobFailureReceiveEndpoint.ReceiveEndpoint, e =>
+                {
+                    e.ConfigureConsumer<JobFailureConsumer>(context);
+                    e.SetQuorumQueue();
+                    e.PrefetchCount = receiveEndpointConfigurations.JobFailureReceiveEndpoint.PrefetchLimit;
+                });
+            }
+
+            if (apiConfiguration.FeatureFlags.RunSendErroredJobAlertConsumer)
+			{
+				cfg.ReceiveEndpoint(receiveEndpointConfigurations.SendErroredJobAlertReceiveEndpoint.ReceiveEndpoint, e =>
+				{
+					e.ConfigureConsumer<SendErroredJobAlertConsumer>(context, cons =>
+                    {
+                        cons.Options<BatchOptions>(options => options
+                            .SetMessageLimit(200)
+                            .SetTimeLimit(5000)
+                            .SetConcurrencyLimit(1));
+                    });
+                    e.SetQuorumQueue();
+					e.PrefetchCount = receiveEndpointConfigurations.SendErroredJobAlertReceiveEndpoint.PrefetchLimit;
+				});
+			}
+        });
+    });
+    services.AddMassTransitHostedService();
+}
+
+void AddServices(IServiceCollection services)
+{
+    services.AddScoped<IEnvironmentView, EnvironmentView>();
+
+    AddDispatcherOperationServices(services);
+    AddQueueServices(services);
+    services.AddScoped<IJobService, JobService>();
+    services.AddScoped<IJobTagService, JobTagService>();
+    services.AddScoped<ITagService, TagService>();
+    services.AddScoped<IWorkItemService, WorkItemService>();
+    services.AddScoped<IWorkItemExecutionService, WorkItemExecutionService>();
+    services.AddScoped<IExecutionMessageService, ExecutionMessageService>();
+    services.AddScoped<IJobErrorRecipientService, JobErrorRecipientService>();
+    services.AddScoped<INotificationService, NotificationService>();
+    services.AddScoped<ICleanUpJobSettingService, CleanUpJobSettingService>();
+    services.AddScoped<ICleanUpJobTagService, CleanUpJobTagService>();
+}
+
+void AddDispatcherOperationServices(IServiceCollection services)
+{
+    services.AddScoped<IJobScheduleService, JobScheduleService>();
+    services.AddScoped<IBatchErroredJobsService, BatchErroredJobsService>();
+    services.AddScoped<IIngestionService, IngestionService>();
+    services.AddScoped<IExecutionService, ExecutionService>();
+    services.AddScoped<ICompletionService, CompletionService>();
+
+    services.AddScoped<ICleanUpJobsService, CleanUpJobsService>();
+    services.AddScoped<IDetermineOperationService, DetermineOperationService >();
+    services.AddScoped<IJobFailureService, JobFailureService >();
+    services.AddScoped<ISendErroredJobAlertService, SendErroredJobAlertService >();
+}
+
+void AddQueueServices(IServiceCollection services)
+{
+    services.AddScoped<IQueueExecutionService, QueueExecutionService>();
+    services.AddScoped<IQueueCompletionService, QueueCompletionService>();
+    services.AddScoped<IQueueSendErroredJobAlertService, QueueSendErroredJobAlertService>();
+    services.AddScoped<IQueueDetermineOperationService, QueueDetermineOperationService>();
+    services.AddScoped<IQueueJobFailureService, QueueJobFailureService>();
+}
+
+void AddRepositories(IServiceCollection services)
+{
+    services.AddScoped<IUnitOfWork, UnitOfWork>();
+    services.AddScoped<IJobRepository, JobRepository>();
+    services.AddScoped<IJobErrorRecipientRepository, JobErrorRecipientRepository>();
+    services.AddScoped<IJobTagRepository, JobTagRepository>();
+    services.AddScoped<ITagRepository, TagRepository>();
+    services.AddScoped<IWorkItemRepository, WorkItemRepository>();
+    services.AddScoped<IWorkItemExecutionRepository, WorkItemExecutionRepository>();
+    services.AddScoped<IExecutionMessageRepository, ExecutionMessageRepository>();
+    services.AddScoped<ICleanUpJobSettingRepository, CleanUpJobSettingRepository>();
+    services.AddScoped<ICleanUpJobTagRepository, CleanUpJobTagRepository>();
+    services.AddScoped<INotificationsApiDao, NotificationsApiDao>();
+}
+
+void AddHostedServices(IServiceCollection services)
+{
+	services.AddSingleton<ICleanUpJobsHostedService, CleanUpJobsHostedService>();
+    services.AddSingleton<IBatchErroredJobsHostedService, BatchErroredJobsHostedService>();
+    services.AddSingleton<IJobSchedulerHostedService, JobSchedulerHostedService>();
+}
+
+void BuildAppAndRun(WebApplicationBuilder builder)
+{
+    var app = builder.Build();
+
+    if (_apiConfiguration.FeatureFlags.RunCleanUpJobsHostedService)
+    {
+        var service = app.Services.GetRequiredService<ICleanUpJobsHostedService>();
+        service.StartAsync().Wait();
+	}
+
+    if (_apiConfiguration.FeatureFlags.RunJobSchedulerHostedService)
+    {
+        var service = app.Services.GetRequiredService<IJobSchedulerHostedService>();
+        service.StartAsync().Wait();
+    }
+
+    if (_apiConfiguration.FeatureFlags.RunBatchErroredJobsHostedService)
+    {
+        var service = app.Services.GetRequiredService<IBatchErroredJobsHostedService>();
+        service.StartAsync().Wait();
+    }
+
+    // Configure the HTTP request pipeline.
+    if (app.Environment.IsDevelopment())
+    {
+        app.UseDeveloperExceptionPage();
+    }
+
+    app.UseSwagger(c =>
+    {
+        c.PreSerializeFilters.Add((swaggerDoc, request) =>
+        {
+            const string prefixHeader = "X-Forwarded-Prefix";
+            if (request.Headers.ContainsKey(prefixHeader)) return;
+
+            var serverUrl = request.Headers[prefixHeader];
+            if (!String.IsNullOrWhiteSpace(serverUrl))
+            {
+                swaggerDoc.Servers = new List<OpenApiServer>()
+                {
+                    new() { Description = serverUrl, Url = serverUrl }
+                };
+            }
+        });
+    });
+    app.UseSwaggerUI(c =>
+    {
+        c.RoutePrefix = "swagger";
+        c.SwaggerEndpoint("v1/swagger.json", "LCU.Api.Internal.Dispatcher v1");
+    });
+    app.UseMiddleware<RequestResponseLoggingMiddleware>();
+    app.UseSerilogRequestLogging(options =>
+    {
+        options.EnrichDiagnosticContext = RequestResponseLoggingMiddleware.RequestDataParser.EnrichFromRequest;
+    });
+
+    app.UseRouting();
+    app.UseCors(MyAllowSpecificOrigins);
+    app.UseHttpsRedirection();
+    app.UseAuthorization();
+    app.MapControllers();
+    app.UseEndpoints(endpoints => { endpoints.MapControllers(); });
+
+    app.Run();
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Services/CleanUpJobSettingService.cs

@@ -0,0 +1,114 @@
+using LCU.Api.Internal.DispatcherLib.Repositories;
+using Microsoft.Data.SqlClient;
+using System.Collections.Generic;
+
+namespace LCU.Api.Internal.Dispatcher.Services
+{
+    public class CleanUpJobSettingService : ICleanUpJobSettingService
+    {
+        private readonly ICleanUpJobSettingRepository _cleanUpJobSettingRepository;
+        private readonly IUnitOfWork _unitOfWork;
+
+        public CleanUpJobSettingService(IUnitOfWork unitOfWork,
+                          ICleanUpJobSettingRepository cleanUpJobSettingRepository)
+        {
+            _unitOfWork = unitOfWork;
+            _cleanUpJobSettingRepository = cleanUpJobSettingRepository;
+        }
+
+        public async Task<IEnumerable<CleanUpJobSetting>> GetAsync()
+        {
+            try
+            {
+                return await _cleanUpJobSettingRepository.ListAsync();
+            }
+            catch (Exception exception)
+            {
+                Log.Error(exception, "Failure retrieving CleanUpJobSetting.");
+                throw new ServiceException("Failure retrieving CleanUpJobSetting.");
+            }
+        }
+
+        public async Task<CleanUpJobSetting> GetByCleanUpJobSettingIdAsync(int cleanUpJobSettingId)
+        {
+            try
+            {
+                return await _cleanUpJobSettingRepository.FindByCleanUpJobSettingIdAsync(cleanUpJobSettingId);
+            }
+            catch (ArgumentNullException exception)
+            {
+                Log.Error(exception, $"Failure getting dispatcher Clean Up Job Settings by Id. CleanUpJobSettingId was null");
+                return null;
+            }
+            catch (Exception exception)
+            {
+                Log.Error(exception, "Failure retrieving dispatcher Clean Up Job Settings. CleanUpJobSettingId: {@cleanUpJobSettingId}", cleanUpJobSettingId);
+                throw new ServiceException($"Failure retrieving dispatcher Clean Up Job Settings. CleanUpJobSettingId: {cleanUpJobSettingId}");
+            }
+        }
+
+        public async Task<CleanUpJobSetting> SaveAsync(CleanUpJobSetting request)
+        {
+            try
+            {
+                var result = await _cleanUpJobSettingRepository.AddAsync(request);
+                await _unitOfWork.CompleteAsync();
+                return result;
+            }
+            catch (Exception exception)
+            {
+                Log.Error(exception, "Failure saving dispatcher Clean Up Job Settings. CleanUpJobSettingId: {@request.CleanUpJobSettingId}", request.CleanUpJobSettingId);
+                throw new ServiceException($"Failure saving dispatcher Clean Up Job Settings. CleanUpJobSettingId: {request.CleanUpJobSettingId}");
+            }
+        }
+
+        public async Task<CleanUpJobSetting> UpdateAsync(CleanUpJobSetting request)
+        {
+            try
+            {
+                var existing = await GetByCleanUpJobSettingIdAsync(request.CleanUpJobSettingId);
+                if(existing == null)
+                {
+                    return null;
+                }
+
+                existing.CompletionDaysPast = request.CompletionDaysPast;
+                _cleanUpJobSettingRepository.Update(existing);
+                await _unitOfWork.CompleteAsync();
+
+                return existing;
+            }
+            catch (SqlException sqlException)
+            {
+                Log.Error(sqlException, "SQL Failure updating dispatcher Clean Up Job Settings. CleanUpJobSettingId: {@request.CleanUpJobSettingId}", request.CleanUpJobSettingId);
+                throw new ServiceException($"SQL Failure updating dispatcher Clean Up Job Settings. CleanUpJobSettingId: {request.CleanUpJobSettingId}");
+            }
+            catch (Exception exception)
+            {
+                Log.Error(exception, "SQL Failure updating dispatcher Clean Up Job Settings. CleanUpJobSettingId: {@request.CleanUpJobSettingId}", request.CleanUpJobSettingId);
+                throw new ServiceException($"SQL Failure updating dispatcher Clean Up Job Settings. CleanUpJobSettingId: {request.CleanUpJobSettingId}");
+            }
+        }
+
+        public async Task DeleteAsync(int cleanUpJobSettingId)
+        {
+            try
+            {
+                var cleanUpJobSetting = await _cleanUpJobSettingRepository.FindByCleanUpJobSettingIdAsync(cleanUpJobSettingId);
+                ArgumentNullException.ThrowIfNull(cleanUpJobSetting);
+                _cleanUpJobSettingRepository.Remove(cleanUpJobSetting);
+                await _unitOfWork.CompleteAsync();
+            }
+            catch (SqlException sqlException)
+            {
+                Log.Error(sqlException, "SQL Failure removing dispatcher Clean Up Job Settings. CleanUpJobSettingId: {@cleanUpJobSettingId}", cleanUpJobSettingId);
+                throw new ServiceException($"SQL Failure removing dispatcher Clean Up Job Settings. CleanUpJobSettingId: {cleanUpJobSettingId}");
+            }
+            catch (Exception exception)
+            {
+                Log.Error(exception, "Failure removing dispatcher Clean Up Job Settings. CleanUpJobSettingId: {@cleanUpJobSettingId}", cleanUpJobSettingId);
+                throw new ServiceException($"Failure removing dispatcher Clean Up Job Settings. CleanUpJobSettingId: {cleanUpJobSettingId}");
+            }
+        }
+    }
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Services/CleanUpJobTagService.cs

@@ -0,0 +1,135 @@
+using LCU.Api.Internal.DispatcherLib.Repositories;
+using Microsoft.Data.SqlClient;
+using System.Collections.Generic;
+
+namespace LCU.Api.Internal.Dispatcher.Services;
+
+public class CleanUpJobTagService : ICleanUpJobTagService
+{
+    private readonly ICleanUpJobTagRepository _cleanUpJobTagRepository;
+    private readonly IUnitOfWork _unitOfWork;
+
+    public CleanUpJobTagService(ICleanUpJobTagRepository cleanUpJobRepository,
+                                    IUnitOfWork unitOfWork)
+    {
+        _cleanUpJobTagRepository = cleanUpJobRepository;
+        _unitOfWork = unitOfWork;
+    }
+
+    public async Task<IEnumerable<CleanUpJobTag>> GetAsync()
+    {
+        try
+        {
+            return await _cleanUpJobTagRepository.ListAsync();
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, "Failure retrieving CleanUpJobag.");
+            throw new ServiceException("Failure retrieving CleanUpJobTag.");
+        }
+    }
+
+    public async Task<CleanUpJobTag> GetByCleanUpJobTagIdAsync(int cleanUpJobTagId)
+    {
+        try
+        {
+            ArgumentNullException.ThrowIfNull(cleanUpJobTagId);
+            return await _cleanUpJobTagRepository.FindByCleanUpJobTagIdAsync(cleanUpJobTagId);
+        }
+        catch (ArgumentNullException exception)
+        {
+            Log.Error(exception, $"Failure getting dispatcher Clean Up Job Tag by Id. CleanUpJobTagId was null");
+            return null;
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, "Failure retrieving dispatcher Clean Up Job Tag. CleanUpJobTagId: {@cleanUpJobTagId}", cleanUpJobTagId);
+            throw new ServiceException($"Failure retrieving dispatcher Clean Up Job Tag. CleanUpJobTagId: {cleanUpJobTagId}");
+        }
+    }
+
+    public async Task<IEnumerable<CleanUpJobTag>> GetByCleanUpJobSettingIdAsync(int cleanUpJobSettingId)
+    {
+        try
+        {
+            ArgumentNullException.ThrowIfNull(cleanUpJobSettingId);
+            return await _cleanUpJobTagRepository.FindByCleanUpJobSettingIdAsync(cleanUpJobSettingId);
+        }
+        catch (ArgumentNullException exception)
+        {
+            Log.Error(exception, $"Failure getting dispatcher Clean Up Job Tags bySettings Id. CleanUpJobSettingId was null");
+            return null;
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, "Failure retrieving dispatcher Clean Up Job Tag. CleanUpJobSettingId: {@cleanUpJobSettingId}", cleanUpJobSettingId);
+            throw new ServiceException($"Failure retrieving dispatcher Clean Up Job Tag. CleanUpJobSettingId: {cleanUpJobSettingId}");
+        }
+    }
+
+    public async Task<CleanUpJobTag> SaveAsync(CleanUpJobTag request)
+    {
+        try
+        {
+            var result = await _cleanUpJobTagRepository.AddAsync(request);
+            await _unitOfWork.CompleteAsync();
+            return result;
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, "Failure saving dispatcher Clean Up Job Tag. CleanUpJobTagId: {@request.CleanUpJobTagId}", request.CleanUpJobTagId);
+            throw new ServiceException($"Failure saving dispatcher Clean Up Job Tag. CleanUpJobTagId: {request.CleanUpJobTagId}");
+        }
+    }
+
+    public async Task DeleteAsync(int cleanUpJobTagId)
+    {
+        try
+        {
+            var cleanUpJobTag = await _cleanUpJobTagRepository.FindByCleanUpJobTagIdAsync(cleanUpJobTagId);
+            ArgumentNullException.ThrowIfNull(cleanUpJobTag);
+            _cleanUpJobTagRepository.Remove(cleanUpJobTag);
+            await _unitOfWork.CompleteAsync();
+        }
+        catch (SqlException sqlException)
+        {
+            Log.Error(sqlException, "SQL Failure removing dispatcher Clean Up Job Tag. CleanUpJobTagId: {@cleanUpJobTagId}", cleanUpJobTagId);
+            throw new ServiceException($"SQL Failure removing dispatcher Clean Up Job Tag. CleanUpJobTagId: {cleanUpJobTagId}");
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, "Failure removing dispatcher Clean Up Job Tag. CleanUpJobTagId: {@cleanUpJobTagId}", cleanUpJobTagId);
+            throw new ServiceException($"Failure removing dispatcher Clean Up Job Tag. CleanUpJobTagId: {cleanUpJobTagId}");
+        }
+    }
+
+    public async Task DeleteByCleanUpJobSettingIdAsync(int cleanUpJobSettingId)
+    {
+        try
+        {
+            var cleanUpJobTags = await _cleanUpJobTagRepository.FindByCleanUpJobSettingIdAsync(cleanUpJobSettingId);
+
+            if(!cleanUpJobTags.Any())
+            {
+                throw new Exception($"No CleanUpJobTags found.");
+            }
+
+            foreach (var tag in cleanUpJobTags)
+            {
+                _cleanUpJobTagRepository.Remove(tag);
+            }
+
+            await _unitOfWork.CompleteAsync();
+        }
+        catch (SqlException sqlException)
+        {
+            Log.Error(sqlException, "SQL Failure removing dispatcher Clean Up Job Tag. CleanUpJobSettingId: {@cleanUpJobSettingId}", cleanUpJobSettingId);
+            throw new ServiceException($"SQL Failure removing dispatcher Clean Up Job Tag. CleanUpJobSettingId: {cleanUpJobSettingId}");
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, "Failure removing dispatcher Clean Up Job Tag. CleanUpJobSettingId: {@cleanUpJobSettingId}", cleanUpJobSettingId);
+            throw new ServiceException($"Failure removing dispatcher Clean Up Job Tag. CleanUpJobSettingId: {cleanUpJobSettingId}");
+        }
+    }
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Services/DispatcherOperationServices/BatchErroredJobsService.cs

@@ -0,0 +1,72 @@
+using LCU.Api.Internal.Dispatcher.Services.QueueServices;
+using LCU.Api.Internal.MaidBotLib.Models;
+using LCU.Library.Internal.DispatcherModels.Enums;
+using LCU.Library.Internal.DispatcherModels.Models.Requests;
+using System.Collections.Generic;
+using System.Threading;
+namespace LCU.Api.Internal.Dispatcher.Services.DispatcherOperationServices;
+
+public class BatchErroredJobsService : IBatchErroredJobsService
+{
+    private readonly IQueueSendErroredJobAlertService _queueSendErroredJobAlertService;
+    private readonly IJobService _jobService;
+
+    public BatchErroredJobsService(IJobService jobService,
+                                  IQueueSendErroredJobAlertService queueSendErroredJobAlertService)
+    {
+        _jobService = jobService;
+        _queueSendErroredJobAlertService = queueSendErroredJobAlertService;
+    }
+
+    public async Task PerformBatchErroredJobs(CancellationToken cancellationToken)
+    {
+        try
+        {
+            var jobs = await _jobService.GetErrorMonitoringJobs();
+            if (jobs == null || !jobs.Any())
+            {
+                return;
+            }
+
+            var batchedErroredJobAlertRequest = CreateBatchedErroredJobAlerts(jobs);
+
+            await _queueSendErroredJobAlertService.QueueSendErroredJobAlert(batchedErroredJobAlertRequest);
+        }
+        catch (Exception ex)
+        {
+            //Fatal as we need to know if we can't get out our errors
+            Log.ForContext(nameof(PropertyEnum.Group),
+                           nameof(GroupEnum.ITDevelopment))
+                .Fatal(ex, $"Failed to perform Error Monitoring.");
+        }
+    }
+
+    private static BatchedErroredJobAlertRequest CreateBatchedErroredJobAlerts(IEnumerable<Job> jobs)
+    {
+        var batchedErroredJobAlertRequests = new BatchedErroredJobAlertRequest()
+        {
+            ErroredJobAlertRequests = new List<ErroredJobAlertRequest>()
+        };
+
+        foreach (var job in jobs)
+        {
+            var failedOrProcessingWorkItem = job.WorkItems
+                                 .FirstOrDefault(wi => wi.WorkItemExecutions != null &&
+                                 wi.WorkItemExecutions.Any(wie => wie.Status == StatusType.Failed || wie.Status == StatusType.Processing));
+
+            if (failedOrProcessingWorkItem == null || failedOrProcessingWorkItem.CurrentWorkItemExecution == null)
+            {
+                Log.Warning($"Failed or Processing workItem not found during CreateBatchedErroredJobAlerts. JobId and Name: {job.JobId}: {job.Name}");
+                continue;
+            }
+
+            batchedErroredJobAlertRequests.ErroredJobAlertRequests.Add(new ErroredJobAlertRequest
+            {
+                CurrentJob = job,
+                CurrentWorkItem = failedOrProcessingWorkItem
+            });
+        }
+
+        return batchedErroredJobAlertRequests;
+    }
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Services/DispatcherOperationServices/CleanUpJobsService.cs

@@ -0,0 +1,172 @@
+using LCU.Api.Internal.Dispatcher.Configurations;
+using LCU.Api.Internal.DispatcherLib.Repositories;
+using LCU.Api.Internal.MaidBotLib.Models;
+using LCU.Library.Internal.DispatcherModels.Models.Results;
+using Newtonsoft.Json;
+using System.Threading;
+
+namespace LCU.Api.Internal.Dispatcher.Services.DispatcherOperationServices;
+
+public class CleanUpJobsService : ICleanUpJobsService
+{
+    private readonly IJobService _jobService;
+    private readonly ICleanUpJobSettingService _cleanUpJobSettingService;
+    private readonly IUnitOfWork _unitOfWork;
+    private readonly ApiConfiguration _apiConfiguration;
+
+    public CleanUpJobsService(IJobService jobService,
+                                  ICleanUpJobSettingService cleanUpJobSettingService,
+                                  IUnitOfWork unitOfWork,
+                                  ApiConfiguration apiConfiguration)
+    {
+        _jobService = jobService;
+        _cleanUpJobSettingService = cleanUpJobSettingService;
+        _unitOfWork = unitOfWork;
+        _apiConfiguration = apiConfiguration;
+    }
+
+    public async Task PerformCleanUpJobs(CancellationToken cancellationToken)
+    {
+        try
+        {
+            var result = new CleanUpJobBatchResult();
+
+            result = await PerformSoftCleanUpJobTask(result);
+            result = await PerformHardDeleteCleanUpJobTask(result);
+
+            await _unitOfWork.CompleteAsync();
+
+            if (result.IsFailure)
+            {
+                Log.ForContext(nameof(PropertyEnum.Group),
+                               nameof(GroupEnum.ITDevelopment))
+                    .Error("CleanUpJob Batch Result: {@CleanUpJobBatchResult}", result);
+            }
+        }
+        catch (Exception ex)
+        {
+            Log.ForContext(nameof(PropertyEnum.Group),
+                           nameof(GroupEnum.ITDevelopment))
+                .Error(ex, $"Failed to perform Clean Up Jobs.");
+        }
+    }
+
+    public async Task<CleanUpJobBatchResult> PerformSoftCleanUpJobTask(CleanUpJobBatchResult result)
+    {
+        var jobs = await _jobService.GetInactiveExcludeSoftDeletedTrackedAsync();
+
+        var cleanUpJobSettings = await _cleanUpJobSettingService.GetAsync();
+
+        foreach (var setting in cleanUpJobSettings)
+        {
+            if (setting.CleanUpJobTags == null || !setting.CleanUpJobTags.Any())
+            {
+                continue;
+            }
+
+            foreach (var job in jobs)
+            {
+                result = PerformCleanUpTaskForJob(result, job, setting);
+            }
+        }
+
+        return result;
+    }
+
+    public bool ValidateShouldSoftDelete(CleanUpJobSetting setting, Job job, CleanUpJobBatchResult result)
+    {
+        var shouldSoftDelete = ShouldSoftDeleteByTag(setting, job);
+        
+        if (!shouldSoftDelete)
+        {
+            return false;
+        }
+
+        return ShouldSoftDeleteByDate(setting, job, result);
+    }
+
+    public bool ShouldSoftDeleteByDate(CleanUpJobSetting setting, Job job, CleanUpJobBatchResult result)
+    {
+        try
+        {
+			if (!job.WorkItems.Any() || !job.WorkItems.FirstOrDefault().WorkItemExecutions.Any())
+			{
+				result.AddToFailureCount($"Job would be cleaned but cannot determine if it is old enough to be cleaned due to not having any workItemExecutions. Job: {job.JobId} - {job.Name}");
+				return false;
+			}
+
+			var currentDate = DateTime.Now;
+            var cutoffDate = currentDate.AddDays(setting.CompletionDaysPast);
+            var jobLastCompletedTime = job.WorkItems.FirstOrDefault()
+                                            .WorkItemExecutions.OrderByDescending(wie => wie.CompletedTimestamp).FirstOrDefault().CompletedTimestamp.Value;
+
+            return DateTime.Compare(cutoffDate, jobLastCompletedTime) > 0;
+        }
+        catch (Exception ex)
+        {
+            var message = $"Error while attempting to compare CompletedTimestamp for SoftDeletion. Job: {job.JobId} - {job.Name}. ExeptionMessage: {ex.Message}";
+            Log.Error(message);
+            throw new Exception(message);
+        }
+    }
+
+    public bool ShouldSoftDeleteByTag(CleanUpJobSetting setting, Job job)
+    {
+        if (job.JobTags.Count != setting.CleanUpJobTags.Count)
+        {
+            return false;
+        }
+
+        var orderedJobTagIds = job.JobTags.Select(t => t.TagId).OrderBy(t => t).ToList();
+        var orderedCleanUpJobTagIds = setting.CleanUpJobTags.Select(t => t.TagId).OrderBy(t => t).ToList();
+
+        return orderedJobTagIds.SequenceEqual(orderedCleanUpJobTagIds);
+    }
+
+    public async Task<CleanUpJobBatchResult> PerformHardDeleteCleanUpJobTask(CleanUpJobBatchResult result)
+    {
+        var softDeletedJobs = await _jobService.GetSoftDeletedTrackedAsync(DateTime.Now.AddDays(_apiConfiguration.CleanUpJobsDaysToDelete));
+        foreach (var job in softDeletedJobs)
+        {
+            try
+            {
+                await _jobService.DeleteFromCleanupAsync(job);
+            }
+            catch (Exception ex)
+            {
+                Log.Error(ex.Message, "Failure to hard delete dispatcher job through Clean Up Job Task. Job: {@Job}", job);
+                result.AddToFailureCount(ex.Message);
+                continue;
+            }
+        }
+
+        return result;
+    }
+
+    private CleanUpJobBatchResult PerformCleanUpTaskForJob(CleanUpJobBatchResult result, Job job, CleanUpJobSetting setting)
+    {
+        try
+        {
+            var shouldSoftDelete = ValidateShouldSoftDelete(setting, job, result);
+
+            if (!shouldSoftDelete)
+            {
+                return result;
+            }
+
+            job.SoftDeleteDate = DateTime.Now;
+
+            var updatedJob = _jobService.UpdateExcludingWorkItemsAsyncWithoutSave(job);
+            if (updatedJob == null || updatedJob.Result.JobId != job.JobId)
+            {
+                throw new Exception($"Failure to setup soft delete update for job. Job:{JsonConvert.SerializeObject(job)}");
+            }
+        }
+        catch (Exception ex)
+        {
+            result.AddToFailureCount(ex.Message);
+        }
+
+        return result;
+    }
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Services/DispatcherOperationServices/CompletionService.cs

@@ -0,0 +1,84 @@
+using Newtonsoft.Json;
+using LCU.Library.Internal.ApiResponse.Models;
+using LCU.Api.Internal.Dispatcher.Services.QueueServices;
+using LCU.Library.Internal.DispatcherModels.Models.Requests;
+using LCU.Library.Internal.DispatcherModels.Enums;
+using System.Collections.Generic;
+using RabbitMQ.Client;
+using LCU.Api.Internal.MaidBotLib.Models;
+namespace LCU.Api.Internal.Dispatcher.Services.DispatcherOperationServices;
+
+public class CompletionService : ICompletionService
+{
+    private readonly IJobService _jobService;
+    private readonly IWorkItemService _workItemService;
+    private readonly IExecutionMessageService _executionMessageService;
+    private readonly IQueueJobFailureService _queueJobFailureService;
+    private readonly IQueueDetermineOperationService _queueDetermineOperationService;
+
+    public CompletionService(IJobService jobService,
+                             IWorkItemService workItemService,
+                             IExecutionMessageService executionMessageService,
+                             IQueueJobFailureService queueJobFailureService,
+                             IQueueDetermineOperationService queueDetermineOperationService)
+    {
+        _jobService = jobService;
+        _workItemService = workItemService;
+        _executionMessageService = executionMessageService;
+        _queueJobFailureService = queueJobFailureService;
+        _queueDetermineOperationService = queueDetermineOperationService;
+    }
+
+    public async Task HandleCompletionRequest(Response response)
+    {
+        var dispatcherReturnResponse = JsonConvert.DeserializeObject<DispatcherReturnResponse>(response.ResponsePayload);
+        try
+        {          
+            ArgumentNullException.ThrowIfNull(dispatcherReturnResponse);
+
+            var job = await _jobService.GetByWorkItemIdExcludingWorkItemsAsync(dispatcherReturnResponse.WorkItemId);
+
+            if (!response.Successful)
+            {
+                await _queueJobFailureService.QueueJobFailure(job.JobId,
+                                                                dispatcherReturnResponse.WorkItemId, 
+                                                                response.ResponseStack == null || response.ResponseStack.Any() ? response.ResponseStack : new List<string>());
+                return;
+            }
+
+            var workItem = await _workItemService.GetByIdAsync(dispatcherReturnResponse.WorkItemId);
+            if (workItem.CurrentWorkItemExecution == null)
+            {
+                Log.Warning($"CurrentWorkitemExecution was null for WorkItemId:{dispatcherReturnResponse.WorkItemId}");
+                return;
+            }
+            await _executionMessageService.SaveExecutionMessages(workItem.CurrentWorkItemExecution.WorkItemExecutionId, response.ResponseStack);
+
+            if (workItem.CanceledJobCounter > 0)
+            {
+                workItem = await _workItemService.ResetCanceledJobCounter(workItem);
+            }
+
+            job.JobStatus = JobStatus.DeterminingOperation;
+            var updatedJob = await _jobService.UpdateAsync(job);
+
+            await _queueDetermineOperationService.QueueDetermineOperation(updatedJob, workItem, SetDetermineOperationType(dispatcherReturnResponse.Skipped));
+        }
+        catch (Exception exception)
+        {
+            var exceptionMessage = $"Failed to perform completion. Job may be stuck in an inoperable status. WorkItemId: {dispatcherReturnResponse.WorkItemId}";
+            Log.Error(exception, exceptionMessage);
+            throw new ServiceException(exceptionMessage);
+        }
+    }
+
+    public DetermineOperationType SetDetermineOperationType(bool isSkipped)
+    {
+        if(isSkipped)
+        {
+            return DetermineOperationType.Skipped;
+        }
+
+        return DetermineOperationType.Completed;
+    }
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Services/DispatcherOperationServices/DetermineOperationService.cs

@@ -0,0 +1,82 @@
+using LCU.Api.Internal.Dispatcher.Services.QueueServices;
+using LCU.Api.Internal.MaidBotLib.Models;
+using LCU.Library.Internal.DispatcherModels.Entities;
+using LCU.Library.Internal.DispatcherModels.Enums;
+using LCU.Library.Internal.DispatcherModels.Models.Requests;
+
+namespace LCU.Api.Internal.Dispatcher.Services.DispatcherOperationServices
+{
+    public class DetermineOperationService : IDetermineOperationService
+	{
+        private readonly IJobService _jobService;
+        private readonly IWorkItemService _workItemService;
+        private readonly IWorkItemExecutionService _workItemExecutionService;
+        private readonly IQueueExecutionService _queueExecutionService;
+
+        public DetermineOperationService(IJobService jobService, 
+                                            IWorkItemService workItemService, 
+                                            IWorkItemExecutionService workItemExecutionService, 
+                                            IQueueExecutionService queueExecutionService)
+        {
+            _jobService = jobService;
+            _workItemService = workItemService;
+            _workItemExecutionService = workItemExecutionService;
+            _queueExecutionService = queueExecutionService;
+        }
+
+        public async Task HandleDetermineOperationRequest(DetermineOperationRequest determineOperationRequest)
+		{
+			try
+			{
+                switch (determineOperationRequest.DetermineOperationType)
+                {
+                    case DetermineOperationType.Failed:
+                        await HandleFailedWorkItem(determineOperationRequest);
+                        break;
+                    case DetermineOperationType.Skipped:
+                        await HandleWorkItemCompletion(determineOperationRequest, StatusType.Skipped);
+                        break;
+                    case DetermineOperationType.Completed:
+                        await HandleWorkItemCompletion(determineOperationRequest, StatusType.Completed);
+                        break;
+                    default:
+                        throw new NotImplementedException();
+                }
+                return;
+            }
+            catch (ServiceException serviceException)
+            {
+                Log.Error(serviceException, $"Dispatcher Determine Operation Failure. " +
+                                            $"The Dispatcher Determine Operation is placed on LCU.Dispatcher.DetermineOperation_error.");
+                throw new ServiceException($"Failed to handle DetermineOperationRequest.");
+            }
+            catch (Exception exception)
+            {
+                Log.Error(exception, $"Failed to handle DetermineOperationRequest.");
+                throw new ServiceException($"Failed to handle DetermineOperationRequest.");
+            }
+		}
+
+        public async Task HandleFailedWorkItem(DetermineOperationRequest determineOperationRequest)
+        {
+            await _jobService.CompleteJobAndWorkItem(determineOperationRequest.CurrentJob, determineOperationRequest.CurrentWorkItem.CurrentWorkItemExecution, StatusType.Failed);
+        }
+
+        public async Task HandleWorkItemCompletion(DetermineOperationRequest determineOperationRequest, StatusType status)
+        {
+            var nextWorkItem = await _workItemService.GetNextWorkItemByJobId(determineOperationRequest.CurrentJob.JobId, determineOperationRequest.CurrentWorkItem.Order);
+            if (nextWorkItem == null)
+            {
+                await _jobService.CompleteJobAndWorkItem(determineOperationRequest.CurrentJob, determineOperationRequest.CurrentWorkItem.CurrentWorkItemExecution, status);
+                return;
+            }
+
+            await _jobService.CompleteWorkItemAndContinueProcessingJob(determineOperationRequest.CurrentJob,
+                                                                        determineOperationRequest.CurrentWorkItem.CurrentWorkItemExecution,
+                                                                        status);
+            var newWorkItemExecution = await _workItemExecutionService.CreateWorkItemExecution(nextWorkItem.WorkItemId);
+
+            await _queueExecutionService.QueueExecution(newWorkItemExecution);
+        }
+    }
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Services/DispatcherOperationServices/ExecutionService.cs

@@ -0,0 +1,96 @@
+using LCU.Api.Internal.Dispatcher.Services.QueueServices;
+using LCU.Library.Internal.DispatcherModels.Enums;
+using LCU.Library.Internal.LcuHttpClient.HttpRequest;
+using System.Collections.Generic;
+namespace LCU.Api.Internal.Dispatcher.Services.DispatcherOperationServices;
+public class ExecutionService : IExecutionService
+{
+    private readonly IPostRequest _postRequest;
+    private readonly IWorkItemService _workItemService;
+    private readonly IJobService _jobService;
+    private readonly IQueueCompletionService _queueCompletionService;
+
+	private const string HttRequestIsNullMessage = "Http Request is null";
+
+	public ExecutionService(IPostRequest postRequest,
+							IWorkItemService workItemService,
+
+							IJobService jobService,
+                            IQueueCompletionService queueCompletionService)
+	{
+        _postRequest = postRequest;
+		_workItemService = workItemService;
+		_jobService = jobService;
+        _queueCompletionService = queueCompletionService;
+	}
+
+	public async Task HandleExecutionRequest(WorkItemExecution request)
+    {
+        try
+        {
+            var executionResponse = await ExecuteRequest(request);
+            if (!executionResponse.IsSuccess)
+            {
+                await _queueCompletionService.QueueCompletion(new DispatcherReturnResponse() { WorkItemId = executionResponse.WorkItemExecution.WorkItemId },
+                                                                    executionResponse.Messages, executionResponse.IsSuccess);
+            }
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, $"Failed to perform execution. WorkItemId: {request.WorkItemId}");
+            throw new ServiceException($"Failed to perform execution. WorkItemId: {request.WorkItemId}");
+        }
+    }
+
+    public async Task<ExecutionResponse> ExecuteRequest(WorkItemExecution request)
+    {
+        try
+        {
+            ArgumentNullException.ThrowIfNull(request);
+
+            var requestedWorkItem = await _workItemService.GetByIdAsync(request.WorkItemId);
+
+            await UpdateJobStatusToProcessing(requestedWorkItem);
+
+            var result = await _postRequest.AsyncWithNewtonSerializedNoThrow(new Uri(requestedWorkItem.Url), requestedWorkItem.SerializedPayload);
+            if (!result.IsSuccessStatusCode)
+            {
+                var content = await result.Content.ReadAsStringAsync();
+                return new ExecutionResponse(false, request, content);
+            }
+            return new ExecutionResponse(true, request);
+        }
+        catch (Exception ex)
+        {
+            var errorMessage = $"Exception during ExecuteRequest. Exception Message: {ex.Message}. Inner Exception: {ex.InnerException}";
+            Log.Error(errorMessage);
+            return new ExecutionResponse(false, request, errorMessage);
+        }
+    }
+
+    private async Task UpdateJobStatusToProcessing(WorkItem requestedWorkItem)
+	{
+		var job = await _jobService.GetByWorkItemIdExcludingWorkItemsAsync(requestedWorkItem.WorkItemId);
+        if (job.JobStatus == JobStatus.Processing)
+        {
+            return;
+        }
+		job.JobStatus = JobStatus.Processing;
+		await _jobService.UpdateAsync(job);
+		Log.Information("Job Id {JobId} was saved with status {Status}", job.JobId, job.JobStatus);
+	}
+
+	public struct ExecutionResponse
+    {
+        public ExecutionResponse(bool isSuccess, WorkItemExecution workItemExecution, string messages = null)
+        {
+            IsSuccess = isSuccess;
+            WorkItemExecution = workItemExecution;
+            Messages = messages != null ? new List<string>() { messages } : null;
+        }
+
+        public bool IsSuccess { get; set; }
+        public List<string> Messages { get; set; }
+        public WorkItemExecution WorkItemExecution { get; set; }
+    }
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Services/DispatcherOperationServices/IBatchErroredJobsService.cs

@@ -0,0 +1,7 @@
+using System.Threading;
+namespace LCU.Api.Internal.Dispatcher.Services.DispatcherOperationServices;
+
+public interface IBatchErroredJobsService
+{
+    Task PerformBatchErroredJobs(CancellationToken cancellationToken);
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Services/DispatcherOperationServices/ICleanUpJobsService.cs

@@ -0,0 +1,8 @@
+using System.Threading;
+
+namespace LCU.Api.Internal.Dispatcher.Services.DispatcherOperationServices;
+
+public interface ICleanUpJobsService
+{
+    Task PerformCleanUpJobs(CancellationToken cancellationToken);
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Services/DispatcherOperationServices/ICompletionService.cs

@@ -0,0 +1,7 @@
+using LCU.Library.Internal.ApiResponse.Models;
+namespace LCU.Api.Internal.Dispatcher.Services.DispatcherOperationServices;
+
+public interface ICompletionService
+{
+    Task HandleCompletionRequest(Response response);
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Services/DispatcherOperationServices/IDetermineOperationService.cs

@@ -0,0 +1,9 @@
+using LCU.Library.Internal.DispatcherModels.Models.Requests;
+
+namespace LCU.Api.Internal.Dispatcher.Services.DispatcherOperationServices
+{
+    public interface IDetermineOperationService
+	{
+		Task HandleDetermineOperationRequest(DetermineOperationRequest determineOperationRequest);
+	}
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Services/DispatcherOperationServices/IExecutionService.cs

@@ -0,0 +1,6 @@
+namespace LCU.Api.Internal.Dispatcher.Services.DispatcherOperationServices;
+
+public interface IExecutionService
+{
+    Task HandleExecutionRequest(WorkItemExecution request);
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Services/DispatcherOperationServices/IIngestionService.cs

@@ -0,0 +1,7 @@
+namespace LCU.Api.Internal.Dispatcher.Services.DispatcherOperationServices;
+
+public interface IIngestionService
+{
+    Task HandleIngestionRequest(Job request);
+    Task<bool> PerformIngestion(Job queueRequest);
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Services/DispatcherOperationServices/IJobFailureService.cs

@@ -0,0 +1,9 @@
+using LCU.Library.Internal.DispatcherModels.Models.Requests;
+
+namespace LCU.Api.Internal.Dispatcher.Services.DispatcherOperationServices
+{
+    public interface IJobFailureService
+	{
+		Task HandleJobFailureRequest(JobFailureRequest jobFailureRequest);
+	}
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Services/DispatcherOperationServices/IJobScheduleService.cs

@@ -0,0 +1,7 @@
+using System.Threading;
+namespace LCU.Api.Internal.Dispatcher.Services.DispatcherOperationServices;
+
+public interface IJobScheduleService
+{
+    Task PerformSchedule(CancellationToken cancellationToken);
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Services/DispatcherOperationServices/ISendErroredJobAlertService.cs

@@ -0,0 +1,10 @@
+using LCU.Library.Internal.DispatcherModels.Models.Requests;
+using System.Collections.Generic;
+
+namespace LCU.Api.Internal.Dispatcher.Services.DispatcherOperationServices
+{
+    public interface ISendErroredJobAlertService
+	{
+		Task HandleSendingErroredJobAlerts(List<BatchedErroredJobAlertRequest> batchedErroredJobAlertRequestList);
+	}
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Services/DispatcherOperationServices/IngestionService.cs

@@ -0,0 +1,159 @@
+using LCU.Library.Internal.DispatcherModels.Enums;
+using System.Collections.Generic;
+namespace LCU.Api.Internal.Dispatcher.Services.DispatcherOperationServices;
+
+public class IngestionService : IIngestionService
+{
+    private readonly IJobService _jobService;
+    private readonly IWorkItemService _workItemService;
+    private readonly IJobErrorRecipientService _jobErrorRecipientService;
+    private readonly IJobTagService _jobTagService;
+
+    public IngestionService(IJobService jobService,
+                            IWorkItemService workItemService,
+                            IJobErrorRecipientService jobErrorRecipientService,
+                            IJobTagService jobTagService)
+    {
+        _jobService = jobService;
+        _workItemService = workItemService;
+        _jobErrorRecipientService = jobErrorRecipientService;
+        _jobTagService = jobTagService;
+    }
+
+    public async Task HandleIngestionRequest(Job request)
+    {
+        try
+        {
+            ArgumentNullException.ThrowIfNull(request);
+            var result = await PerformIngestion(request);
+
+            if (!result)
+            {
+                throw new ServiceException("Failure to PerformIngestion.");
+            }
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, $"Failed to perform execution.");
+            throw new ServiceException($"Failed to perform ingestion.");
+        }
+    }
+
+    public async Task<bool> PerformIngestion(Job request)
+    {
+        try
+        {
+            if (request.IsNewJob)
+            {
+                return await SaveNewRequest(request);
+            }
+
+            return await UpdateRequest(request);
+        }
+        catch (Exception ex)
+        {
+            Log.Information(ex, $"Failed to perform ingestion.");
+            return false;
+        }
+    }
+
+    public async Task<bool> SaveNewRequest(Job request)
+    {
+        request = await _jobService.SaveAsync(request);
+        if (request == null)
+        {
+            return false;
+        }
+
+        var updatedWorkItems = await _workItemService.PopulateActionFieldsAndUpdateList(request.WorkItems);
+        if (updatedWorkItems == null)
+        {
+            return false;
+        }
+
+        return true;
+    }
+
+    public async Task<bool> UpdateRequest(Job request)
+    {
+        var existingJob = await _jobService.GetByIdAsync(request.JobId);
+
+        await UpdateRequestWorkItems(request, existingJob);
+        UpdateRequestErrorRecipients(request, existingJob);
+        await UpdateRequestJobTags(request, existingJob);
+        request.JobStatus = (request.NextRunTime == null && request.CronSchedule == null) ? JobStatus.Inactive : existingJob.JobStatus;
+
+        var updatedJob = await _jobService.UpdateAsync(request);
+        if (updatedJob == null)
+        {
+            return false;
+        }
+
+        return true;
+    }
+
+    public async Task UpdateRequestWorkItems(Job request, Job existingJob)
+    {
+        var removedWorkItems = existingJob.WorkItems.Where(wi =>
+                                                     !request.WorkItems.Any(newWI =>
+                                                              newWI.WorkItemId == wi.WorkItemId)).ToList();
+        await _workItemService.DeleteList(removedWorkItems);
+
+        var newWorkItems = request.WorkItems.Where(wi => wi.IsNewWorkItem).ToList();
+        _workItemService.SaveList(newWorkItems);
+
+        var updatedWorkItems = await _workItemService.PopulateActionFieldsAndUpdateList(request.WorkItems);
+        if (updatedWorkItems == null)
+        {
+            throw new Exception($"WorkItem Error while Populating Action Fields and updating.");
+        }
+    }
+
+    public void UpdateRequestErrorRecipients(Job request, Job existingJob)
+    {
+        var removedJobErrorRecipients = existingJob.JobErrorRecipients.Where(recipient =>
+                                                     !request.JobErrorRecipients.Any(newRecipient =>
+                                                              newRecipient.Recipient == recipient.Recipient)).ToList();
+        _jobErrorRecipientService.DeleteList(removedJobErrorRecipients);
+
+        var newJobErrorRecipients = new List<JobErrorRecipient>();
+        foreach (var recipient in request.JobErrorRecipients)
+        {
+            var jobErrorRecipient = existingJob.JobErrorRecipients.FirstOrDefault(existing =>
+                                            existing.Recipient == recipient.Recipient);
+
+            if (jobErrorRecipient == null)
+            {
+                newJobErrorRecipients.Add(recipient);
+            }
+        }
+
+        _jobErrorRecipientService.SaveList(newJobErrorRecipients);
+    }
+
+    public async Task UpdateRequestJobTags(Job request, Job existingJob)
+    {
+        var removedJobTags = existingJob.JobTags.Where(tag =>
+                                              !request.JobTags.Any(jobTags =>
+                                                       jobTags.TagId == tag.TagId)).ToList();
+
+        foreach (var tag in removedJobTags)
+        {
+            await _jobTagService.Delete(tag.JobTagId);
+        }
+
+        var newJobTags = new List<JobTag>();
+        foreach (var tag in request.JobTags)
+        {
+            var existingTag = existingJob.JobTags.FirstOrDefault(existing =>
+                                            existing.TagId == tag.TagId);
+
+            if (existingTag == null)
+            {
+                newJobTags.Add(tag);
+                var result = await _jobTagService.SaveAsync(tag);
+            }
+
+        }
+    }
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Services/DispatcherOperationServices/JobFailureService.cs

@@ -0,0 +1,91 @@
+using LCU.Api.Internal.Dispatcher.Services.QueueServices;
+using LCU.Api.Internal.MaidBotLib.Models;
+using LCU.Library.Internal.DispatcherModels.Entities;
+using LCU.Library.Internal.DispatcherModels.Enums;
+using LCU.Library.Internal.DispatcherModels.Models;
+using LCU.Library.Internal.DispatcherModels.Models.Requests;
+using LCU.Library.Internal.DocumentIngestorModels.Models.IndexFileDocuments;
+using LCU.Library.Internal.Exceptions;
+using System;
+using System.Collections.Generic;
+
+namespace LCU.Api.Internal.Dispatcher.Services.DispatcherOperationServices
+{
+    public class JobFailureService : IJobFailureService
+	{
+        private readonly IWorkItemService _workItemService;
+        private readonly IWorkItemExecutionService _workItemExecutionService;
+        private readonly IJobService _jobService;
+        private readonly IExecutionMessageService _executionMessageService;
+        private readonly IQueueDetermineOperationService _queueDetermineOperationService;
+        private readonly IQueueSendErroredJobAlertService _queueSendErroredJobAlertService;
+
+        public JobFailureService(IWorkItemService workItemService, 
+                                    IWorkItemExecutionService workItemExecutionService, 
+                                    IJobService jobService, 
+                                    IExecutionMessageService executionMessageService,
+                                    IQueueDetermineOperationService queueDetermineOperationService,
+                                    IQueueSendErroredJobAlertService queueSendErroredJobAlertService)
+        {
+            _workItemService = workItemService;
+            _workItemExecutionService = workItemExecutionService;
+            _jobService = jobService;
+            _executionMessageService = executionMessageService;
+            _queueDetermineOperationService = queueDetermineOperationService;
+            _queueSendErroredJobAlertService = queueSendErroredJobAlertService;
+        }
+
+        public async Task HandleJobFailureRequest(JobFailureRequest jobFailureRequest)
+		{
+            try
+            {
+                var workItem = await _workItemService.GetByIdAsync(jobFailureRequest.WorkItemId);
+
+                var updatedWorkItemExecution = await _workItemExecutionService.UpdateFailedWorkItemExecution(workItem.CurrentWorkItemExecution);
+
+                if (workItem.CancelJobOnFailure)
+                {
+                    workItem = await _workItemService.IncreaseCanceledJobCounter(workItem);
+                }
+
+                var job = await _jobService.GetByWorkItemIdExcludingWorkItemsIncludeErrorRecipientsAsync(jobFailureRequest.WorkItemId);
+
+                if (workItem.RequiresJobCancellation)
+                {
+                    var message = $"This Job will be canceled and if scheduleable, rescheduled. The " +
+                              $"WorkItem's canceled Job counter is currently at {workItem.CanceledJobCounter} and has " +
+                              $"not yet went over the cancel Job limit of {workItem.CancelJobLimit}.";
+                    jobFailureRequest.ExecutionMessages.Add(message);
+                    job.JobStatus = JobStatus.DeterminingOperation;
+                }
+                else
+                {
+                    job.JobStatus = JobStatus.Failed;
+                }
+
+                await _executionMessageService.SaveExecutionMessages(workItem.CurrentWorkItemExecution.WorkItemExecutionId, jobFailureRequest.ExecutionMessages);
+                var updatedJob = await _jobService.UpdateAsync(job);
+
+                if (updatedJob.JobStatus == JobStatus.DeterminingOperation)
+                {
+                    await _queueDetermineOperationService.QueueDetermineOperation(updatedJob, workItem, DetermineOperationType.Failed);
+                }
+                else
+                {
+                    await _queueSendErroredJobAlertService.QueueSendErroredJobAlert(updatedJob, workItem);
+                }
+            }
+            catch (ServiceException serviceException)
+            {
+                Log.Error(serviceException, $"Dispatcher Job Failure Request Failure. " +
+                                            $"The Dispatcher Job Failure Request is placed on LCU.Dispatcher.JobFailure_error.");
+                throw new ServiceException($"Failed to handle JobFailureRequest.");
+            }
+            catch(Exception exception)
+            {
+                Log.Error(exception, $"Failed to handle JobFailureRequest.");
+                throw new ServiceException($"Failed to handle JobFailureRequest.");
+            }
+		}
+	}
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Services/DispatcherOperationServices/JobScheduleService.cs

@@ -0,0 +1,67 @@
+using System.Threading;
+using LCU.Api.Internal.MaidBotLib.Models;
+using LCU.Api.Internal.Dispatcher.Services.QueueServices;
+using LCU.Library.Internal.DispatcherModels.Enums;
+using System;
+namespace LCU.Api.Internal.Dispatcher.Services.DispatcherOperationServices;
+
+public class JobScheduleService : IJobScheduleService
+{
+    private readonly IQueueExecutionService _queueExecutionService;
+    private readonly IJobService _jobService;
+    private readonly IWorkItemService _workItemService;
+    private readonly IWorkItemExecutionService _workItemExecutionService;
+
+    public JobScheduleService(IQueueExecutionService queueExecutionService,
+                              IJobService jobService,
+                              IWorkItemService workItemService,
+                              IWorkItemExecutionService workItemExecutionService)
+    {
+        _queueExecutionService = queueExecutionService;
+        _jobService = jobService;
+        _workItemService = workItemService;
+        _workItemExecutionService = workItemExecutionService;
+    }
+
+    public async Task PerformSchedule(CancellationToken cancellationToken)
+    {
+        try
+        {
+            var readyToScheduleJobs = await _jobService.GetJobsReadyToBeScheduled();
+
+            foreach (var job in readyToScheduleJobs)
+            {
+                var workItem = await _workItemService.GetFirstOrDefaultWorkItemByJobId(job.JobId);
+
+                if (workItem == null)
+                {
+                    // THIS SHOULD NEVER HAPPEN
+                    Log.ForContext(Enum.GetName(typeof(PropertyEnum), PropertyEnum.Group),
+                           Enum.GetName(typeof(GroupEnum), GroupEnum.ITDevelopment))
+                    .Error($"Failure in Dispatcher Job Scheduler. Attempting to schedule Job: {job.JobId} - {job.Name} and NO workItems were found to process.");
+                    continue;
+                }
+
+                var workItemExecution = await _workItemExecutionService.CreateWorkItemExecution(workItem.WorkItemId);
+
+                job.JobStatus = JobStatus.PendingToRun;
+
+                if (job.SoftDeleteDate != null)
+                {
+                    job.SoftDeleteDate = null;
+                }
+
+                await _jobService.UpdateAsync(job);
+                Log.Information("Job Id {JobId} was saved with status {Status}", job.JobId, job.JobStatus);
+
+				await _queueExecutionService.QueueExecution(workItemExecution);
+            }
+        }
+        catch (Exception exception)
+        {
+            Log.ForContext(Enum.GetName(typeof(PropertyEnum), PropertyEnum.Group),
+                           Enum.GetName(typeof(GroupEnum), GroupEnum.ITDevelopment))
+                .Error(exception, $"Failed to perform schedule.");
+        }
+    }
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Services/DispatcherOperationServices/SendErroredJobAlertService.cs

@@ -0,0 +1,146 @@
+using LCU.Api.Internal.Dispatcher.Configurations;
+using LCU.Library.Internal.DispatcherModels.Models.Requests;
+using System.Collections.Generic;
+using System.IO;
+
+namespace LCU.Api.Internal.Dispatcher.Services.DispatcherOperationServices
+{
+    public class SendErroredJobAlertService : ISendErroredJobAlertService
+	{
+        private readonly ApiConfiguration _apiConfiguration;
+        private readonly INotificationService _notificationService;
+
+        public SendErroredJobAlertService(ApiConfiguration apiConfiguration, INotificationService notificationService)
+        {
+            _apiConfiguration = apiConfiguration;
+            _notificationService = notificationService;
+        }
+
+        public Task HandleSendingErroredJobAlerts(List<BatchedErroredJobAlertRequest> batchedErroredJobAlertRequestList)
+        {
+            var erroredJobAlertRequests = CombineFromBatch(batchedErroredJobAlertRequestList);
+
+            return SendErroredJobsReport(erroredJobAlertRequests);
+        }
+
+        public async Task SendErroredJobsReport(List<ErroredJobAlertRequest> erroredJobAlertRequests)
+        {
+            var groupedJobsByRecipients = GroupJobsByRecipients(erroredJobAlertRequests);
+            var result = true;
+            foreach (var recipientGroup in groupedJobsByRecipients)
+            {
+                var successful = await HandleBatchErroredJobs(recipientGroup.Value, recipientGroup.Key);
+                if (!successful)
+                {
+                    result = false;
+                }
+            }
+
+            if (!result)
+            {
+                throw new ServiceException($"Failure to Send Errored Jobs Report. Result: {result}");
+            }
+        }
+
+        public List<ErroredJobAlertRequest> CombineFromBatch(List<BatchedErroredJobAlertRequest> batchedErroredJobAlertRequestList)
+        {
+            var erroredJobAlertRequests = new List<ErroredJobAlertRequest>();
+            foreach (var batch in batchedErroredJobAlertRequestList)
+            {
+                erroredJobAlertRequests.AddRange(batch.ErroredJobAlertRequests);
+            }
+            return erroredJobAlertRequests;
+        }
+
+        public Dictionary<string, List<ErroredJobAlertRequest>> GroupJobsByRecipients(List<ErroredJobAlertRequest> erroredJobAlertRequests)
+        {
+            var groupedRecipients = new Dictionary<string, List<ErroredJobAlertRequest>>();
+            foreach (var erroredJobAlertRequest in erroredJobAlertRequests)
+            {
+                AddRecipients(erroredJobAlertRequest, groupedRecipients);
+            }
+
+            return groupedRecipients;
+        }
+
+        public void AddRecipients(ErroredJobAlertRequest erroredJbAlertRequest, Dictionary<string, List<ErroredJobAlertRequest>> groupedRecipients)
+        {
+            if (erroredJbAlertRequest.CurrentJob.JobErrorRecipients == null ||
+                erroredJbAlertRequest.CurrentJob.JobErrorRecipients.Count == 0)
+            {
+                AddRecipient(_apiConfiguration.DefaultErrorRecipient, erroredJbAlertRequest, groupedRecipients);
+                return;
+            }
+
+            foreach (var jobErrorRecipient in erroredJbAlertRequest.CurrentJob.JobErrorRecipients)
+            {
+                AddRecipient(jobErrorRecipient.Recipient, erroredJbAlertRequest, groupedRecipients);
+            }
+        }
+
+        public void AddRecipient(string defaultErrorRecipient, ErroredJobAlertRequest erroredJobAlertRequest, Dictionary<string, List<ErroredJobAlertRequest>> groupedRecipients)
+        {
+            var recipientLower = defaultErrorRecipient.ToLower();
+            if (!groupedRecipients.ContainsKey(recipientLower))
+            {
+                groupedRecipients.Add(recipientLower, new List<ErroredJobAlertRequest>());
+            }
+
+            groupedRecipients[recipientLower].Add(erroredJobAlertRequest);
+        }
+
+        public async Task<bool> HandleBatchErroredJobs(List<ErroredJobAlertRequest> erroredJobAlertRequests, string recipient)
+        {
+            string rowsOfErrors = "";
+            foreach (ErroredJobAlertRequest erroredJobAlertRequest in erroredJobAlertRequests)
+            {
+                var jobErrorDetail = SetUpJobErrorDetails(erroredJobAlertRequest);
+                if (jobErrorDetail != null)
+                {
+                    rowsOfErrors += CreateErrorRow(jobErrorDetail);
+                }
+            }
+
+            var emailTemplate = CreateBatchErroredJobsEmail(rowsOfErrors);
+            var successfullySent = await _notificationService.SendEmailAsync(recipient,
+                                                                       _apiConfiguration.BatchErroredJobsSubject,
+                                                                       emailTemplate);
+            return successfullySent;
+        }
+
+        public JobErrorDetail SetUpJobErrorDetails(ErroredJobAlertRequest erroredJobAlertRequest)
+        {
+            var jobErrorDetail = new JobErrorDetail()
+            {
+                JobName = erroredJobAlertRequest?.CurrentJob?.Name,
+                WorkItemName = erroredJobAlertRequest?.CurrentWorkItem?.Name,
+                JobStatus = erroredJobAlertRequest?.CurrentJob?.JobStatus.ToString(),
+                StartTime = erroredJobAlertRequest?.CurrentWorkItem?.CurrentWorkItemExecution?.StartedTimestamp.ToString()
+            };
+
+            return jobErrorDetail;
+        }
+
+        public string CreateErrorRow(JobErrorDetail jobErrorDetails)
+        {
+            using var errorRowReader = new StreamReader(_apiConfiguration.ErrorRowTemplate);
+            string errorRowOutput = errorRowReader.ReadToEnd();
+
+            foreach (var property in jobErrorDetails.GetType().GetProperties())
+            {
+                string value = property.GetValue(jobErrorDetails) == null ? string.Empty :
+                                        property.GetValue(jobErrorDetails).ToString();
+                errorRowOutput = errorRowOutput.Replace($"{{{property.Name}}}", value);
+            }
+
+            return errorRowOutput;
+        }
+
+        public string CreateBatchErroredJobsEmail(string errorRows)
+        {
+            using var batchErroredJobsReader = new StreamReader(_apiConfiguration.BatchErroredJobsTemplate);
+            string batchErroredJobsOutput = batchErroredJobsReader.ReadToEnd();
+            return batchErroredJobsOutput.Replace("{TableRows}", errorRows);
+        }
+    }
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Services/ExecutionMessageService.cs

@@ -0,0 +1,90 @@
+using System.Collections.Generic;
+using LCU.Api.Internal.DispatcherLib.Repositories;
+namespace LCU.Api.Internal.Dispatcher.Services;
+
+public class ExecutionMessageService : IExecutionMessageService
+{
+    private readonly IUnitOfWork _unitOfWork;
+    private readonly IExecutionMessageRepository _executionMessageRepository;    
+
+    public ExecutionMessageService(IUnitOfWork unitOfWork,
+                                   IExecutionMessageRepository executionMessageRepository)
+    {
+        _unitOfWork = unitOfWork;
+        _executionMessageRepository = executionMessageRepository;        
+    }
+
+    public async Task SaveExecutionMessages(List<ExecutionMessage> executionMessageList)
+    {
+        try
+        {
+            if (executionMessageList == null || !executionMessageList.Any())
+            {
+                return;
+            }
+
+            foreach (var message in executionMessageList)
+            {
+                if (string.IsNullOrWhiteSpace(message.Message))
+                {
+                    continue;
+                }
+
+                var savedExecutionMessage = await SaveAsync(message);
+                ArgumentNullException.ThrowIfNull(savedExecutionMessage);
+            }
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, "During SaveExecutionMessages there was an error. ");
+            throw new ServiceException($"During SaveExecutionMessages there was an error. " +
+                                       "ExceptionMessage: {exception.Message}");
+        }
+    }
+
+    public async Task SaveExecutionMessages(int workItemExecutionId, List<string> messageList)
+    {
+        try
+        {
+            if (messageList == null || !messageList.Any())
+            {
+                return;
+            }
+
+            foreach (var message in messageList)
+            {
+                if (string.IsNullOrWhiteSpace(message))
+                {
+                    continue;
+                }
+
+                var executionMessage = new ExecutionMessage(workItemExecutionId, message);
+                var savedExecutionMessage = await SaveAsync(executionMessage);
+                ArgumentNullException.ThrowIfNull(savedExecutionMessage);
+            }
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, $"During SaveExecutionMessages there was an error. " +
+                                 $"WorkItemExecutionId: {workItemExecutionId}");
+            throw new ServiceException($"During SaveExecutionMessages there was an error. " +
+                                       $"WorkItemExecutionId: {workItemExecutionId}, " +
+                                       $"ExceptionMessage: {exception.Message}");
+        }
+    }
+
+    public async Task<ExecutionMessage> SaveAsync(ExecutionMessage workItemExecution)
+    {
+        try
+        {
+            var result = await _executionMessageRepository.AddAsync(workItemExecution);
+            await _unitOfWork.CompleteAsync();
+            return result;
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, "Failure saving dispatcher Execution Messages", workItemExecution);
+            throw new ServiceException($"Failure saving dispatcher Execution Messages. ExceptionMessage: { exception.Message }");
+        }
+    }
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Services/ICleanUpJobSettingService.cs

@@ -0,0 +1,13 @@
+using System.Collections.Generic;
+
+namespace LCU.Api.Internal.Dispatcher.Services
+{
+    public interface ICleanUpJobSettingService
+    {
+        Task<IEnumerable<CleanUpJobSetting>> GetAsync();
+        Task<CleanUpJobSetting> GetByCleanUpJobSettingIdAsync(int cleanUpJobSettingId);
+        Task<CleanUpJobSetting> SaveAsync(CleanUpJobSetting request);
+        Task<CleanUpJobSetting> UpdateAsync(CleanUpJobSetting request);
+        Task DeleteAsync(int cleanUpJobSettingId);
+    }
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Services/ICleanUpJobTagService.cs

@@ -0,0 +1,13 @@
+using System.Collections.Generic;
+
+namespace LCU.Api.Internal.Dispatcher.Services;
+
+public interface ICleanUpJobTagService
+{
+    Task<IEnumerable<CleanUpJobTag>> GetAsync();
+    Task<CleanUpJobTag> GetByCleanUpJobTagIdAsync(int cleanUpJobTagId);
+    Task<IEnumerable<CleanUpJobTag>> GetByCleanUpJobSettingIdAsync(int cleanUpJobSettingId);
+    Task<CleanUpJobTag> SaveAsync(CleanUpJobTag request);
+    Task DeleteAsync(int cleanUpJobTagId);
+    Task DeleteByCleanUpJobSettingIdAsync(int cleanUpJobSettingId);
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Services/IExecutionMessageService.cs

@@ -0,0 +1,9 @@
+using System.Collections.Generic;
+namespace LCU.Api.Internal.Dispatcher.Services;
+
+public interface IExecutionMessageService
+{
+    Task SaveExecutionMessages(List<ExecutionMessage> executionMessageList);
+    Task SaveExecutionMessages(int workItemExecutionId, List<string> messageList);
+    Task<ExecutionMessage> SaveAsync(ExecutionMessage workItemExecution);
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Services/IJobErrorRecipientService.cs

@@ -0,0 +1,9 @@
+using System.Collections.Generic;
+using LCU.Library.Internal.DispatcherModels.Entities;
+namespace LCU.Api.Internal.Dispatcher.Services;
+
+public interface IJobErrorRecipientService
+{
+    void SaveList(ICollection<JobErrorRecipient> jobErrorRecipient);
+    void DeleteList(ICollection<JobErrorRecipient> jobErrorRecipient);
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Services/IJobService.cs

@@ -0,0 +1,35 @@
+using System.Collections.Generic;
+using LCU.Library.Internal.DispatcherModels.Enums;
+using LCU.Library.Internal.DispatcherModels.Models.Requests;
+using LCU.Library.Internal.DispatcherModels.Models.Results;
+
+namespace LCU.Api.Internal.Dispatcher.Services;
+
+public interface IJobService
+{
+    Task<PagedResult<Job>> GetFilteredJobs(Filter filter);
+    Task<IEnumerable<Job>> GetAsync();
+    Task<IEnumerable<Job>> GetInactiveAsync();
+    Task<IEnumerable<Job>> GetInactiveExcludeSoftDeletedTrackedAsync();
+    Task<IEnumerable<Job>> GetSoftDeletedTrackedAsync(DateTime cutOffDate);
+    Task<Job> GetByIdAsync(int id);
+    Task<Job> GetByIdExcludingWorkItemsAsync(int id);
+    Task<Job> GetByWorkItemIdAsync(int workItemId);
+    Task<Job> GetByWorkItemIdExcludingWorkItemsAsync(int workItemId);
+    Task<Job> GetByWorkItemIdExcludingWorkItemsIncludeErrorRecipientsAsync(int workItemId);
+    Task<IEnumerable<Job>> GetJobsReadyToBeScheduled();
+    Task<IEnumerable<Job>> GetErrorMonitoringJobs();
+    Task<Job> SaveAsync(Job job);
+    Task CompleteJobAndWorkItem(Job job, WorkItemExecution currentWorkItemExecution, StatusType finalStatus);
+    Task CompleteWorkItemAndContinueProcessingJob(Job job, WorkItemExecution currentWorkItemExecution, StatusType finalStatus);
+    Task<Job> UpdateAsync(Job job);
+    Task<Job> UpdateExcludingWorkItemsAsyncWithoutSave(Job job);
+    Task<Job> RemoveSoftDeleteById(int id);
+    Task<Job> RunJobNowAsync(Job job);
+    Task<Job> RunJobNowByIdAsync(int id);
+    Task<Job> DeactivateJobAsync(Job job);
+    Task<Job> RescheduleJobAsync(Job job);
+    Task DeleteAsync(int jobId);
+    Task DeleteFromCleanupAsync(Job job);
+    Task<Job> CopyJobAsync(CopyJobRequest request);
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Services/IJobTagService.cs

@@ -0,0 +1,10 @@
+using System.Collections.Generic;
+namespace LCU.Api.Internal.Dispatcher.Services;
+
+public interface IJobTagService
+{
+    Task<IEnumerable<JobTag>> GetAsync();
+    Task<IEnumerable<JobTag>> GetByTagIdAsync(int tagId);
+    Task<JobTag> SaveAsync(JobTag jobTag);
+    Task Delete(int jobTagId);
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Services/INotificationService.cs

@@ -0,0 +1,7 @@
+using System.Threading.Tasks;
+namespace LCU.Api.Internal.Dispatcher.Services;
+
+public interface INotificationService
+{
+    Task<bool> SendEmailAsync(string recipient, string subject, string emailTemplate);
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Services/ITagService.cs

@@ -0,0 +1,11 @@
+using System.Collections.Generic;
+namespace LCU.Api.Internal.Dispatcher.Services;
+
+public interface ITagService
+{
+    Task<IEnumerable<Tag>> GetAsync();
+    Task<Tag> GetByIdAsync(int tagId);
+    Task<Tag> SaveAsync(Tag tag);
+    Task<Tag> UpdateAsync(Tag tag);
+    Task DeleteAsync(int tagId);
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Services/IWorkItemExecutionService.cs

@@ -0,0 +1,17 @@
+using LCU.Library.Internal.DispatcherModels.Enums;
+using System.Collections.Generic;
+namespace LCU.Api.Internal.Dispatcher.Services;
+
+public interface IWorkItemExecutionService
+{
+    Task<WorkItemExecution> GetByIdAsync(int workItemExecutionId);
+    Task<WorkItemExecution> UpdateCompleteWorkItemExecution(WorkItemExecution workItemExecution, StatusType workItemFinalStatus);
+    Task<WorkItemExecution> UpdateCompleteWorkItemExecutionWithoutSave(WorkItemExecution workItemExecution, StatusType workItemFinalStatus);
+    Task<WorkItemExecution> UpdateAsync(WorkItemExecution workItemExecution);
+    Task<WorkItemExecution> CreateWorkItemExecution(int workItemId);
+    Task<WorkItemExecution> SaveAsync(WorkItemExecution workItemExecution);
+    Task<WorkItemExecution> UpdateProcessingWorkItemExecution(WorkItemExecution workItemExecution);
+    Task<WorkItemExecution> UpdateFailedWorkItemExecution(WorkItemExecution workItemExecution);
+    Task<IEnumerable<WorkItemExecution>> GetAllUnFinishedWorkItemExecutions(int workItemExecutionId);
+    Task<IEnumerable<WorkItemExecution>> GetAllByWorkItemId(int workItemId);
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Services/IWorkItemService.cs

@@ -0,0 +1,18 @@
+using System.Collections.Generic;
+namespace LCU.Api.Internal.Dispatcher.Services;
+
+public interface IWorkItemService
+{
+    Task<WorkItem> GetByIdAsync(int workItemId);
+    Task<IEnumerable<WorkItem>> GetByJobIdAsync(int jobId);
+    Task<IEnumerable<WorkItem>> GetByJobIdWithExecutionsAsync(int jobId);
+    Task<WorkItem> GetFirstOrDefaultWorkItemByJobId(int jobId);
+    Task<WorkItem> GetNextWorkItemByJobId(int jobId, int currentWorkItemOrder);
+    void SaveList(ICollection<WorkItem> workItems);
+    Task<ICollection<WorkItem>> PopulateActionFieldsAndUpdateList(ICollection<WorkItem> workItems);
+    Task<WorkItem> ResetCanceledJobCounter(WorkItem workItem);
+    Task<WorkItem> IncreaseCanceledJobCounter(WorkItem workItem);
+    Task<WorkItem> UpdateAsync(WorkItem workItem);
+    Task DeleteList(ICollection<WorkItem> workItems);
+    WorkItem PopulateActionFields(WorkItem workItem);
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Services/JobErrorRecipientService.cs

@@ -0,0 +1,55 @@
+using System.Collections.Generic;
+using Microsoft.Data.SqlClient;
+using LCU.Api.Internal.DispatcherLib.Repositories;
+namespace LCU.Api.Internal.Dispatcher.Services;
+
+public class JobErrorRecipientService : IJobErrorRecipientService
+{
+    private readonly IUnitOfWork _unitOfWork;
+    private readonly IJobErrorRecipientRepository _jobErrorRecipientRepository;
+
+    public JobErrorRecipientService(IUnitOfWork unitOfWork,
+                                    IJobErrorRecipientRepository jobErrorRecipientRepository)
+    {
+        _unitOfWork = unitOfWork;
+        _jobErrorRecipientRepository = jobErrorRecipientRepository;
+    }
+
+    public void SaveList(ICollection<JobErrorRecipient> jobErrorRecipient)
+    {
+        try
+        {
+            _jobErrorRecipientRepository.AddList(jobErrorRecipient);
+            _unitOfWork.Complete();
+        }
+        catch (SqlException ex)
+        {
+            Log.Error(ex, $"SQL Failure saving list of job error recipients.");
+            throw new ServiceException($"SQL Failure saving list of job error recipients.");
+        }
+        catch (Exception ex)
+        {
+            Log.Error(ex, $"Failure saving list of  job error recipients.");
+            throw new ServiceException($"Failure saving list of  job error recipients.");
+        }
+    }
+
+    public void DeleteList(ICollection<JobErrorRecipient> jobErrorRecipient)
+    {
+        try
+        {
+            _jobErrorRecipientRepository.RemoveList(jobErrorRecipient);
+            _unitOfWork.Complete();
+        }
+        catch (SqlException ex)
+        {
+            Log.Error(ex, $"SQL Failure removing list of job error recipients.");
+            throw new ServiceException($"SQL Failure removing list of job error recipients.");
+        }
+        catch (Exception ex)
+        {
+            Log.Error(ex, $"Failure removing list of job error recipients.");
+            throw new ServiceException($"Failure removing list of job error recipients.");
+        }
+    }
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Services/JobService.cs

@@ -0,0 +1,583 @@
+using System.Collections.Generic;
+using Microsoft.Data.SqlClient;
+using Newtonsoft.Json;
+using LCU.Library.Internal.DispatcherModels.Enums;
+using LCU.Api.Internal.DispatcherLib.Repositories;
+using LCU.Api.Internal.DispatcherSharedLib.Utilities;
+using LCU.Api.Internal.Dispatcher.Configurations;
+using LCU.Library.Internal.TypeExtensions.ObjectExtensions;
+using LCU.Library.Internal.DispatcherModels.Models.Requests;
+using LCU.Library.Internal.DispatcherModels.Models.Results;
+using LCU.Library.Internal.Logging.Performance;
+
+namespace LCU.Api.Internal.Dispatcher.Services;
+
+public class JobService : IJobService
+{
+    private readonly IJobRepository _jobRepository;
+    private readonly IWorkItemService _workItemService;
+    private readonly IWorkItemExecutionService _workItemExecutionService;
+    private readonly IUnitOfWork _unitOfWork;
+    private readonly ApiConfiguration _apiConfiguration;
+
+    public JobService(IUnitOfWork unitOfWork,
+                      IJobRepository jobRepository,
+                      IWorkItemService workItemService,
+                      IWorkItemExecutionService workItemExecutionService,
+                      ApiConfiguration apiConfiguration)
+    {
+        _unitOfWork = unitOfWork;
+        _jobRepository = jobRepository;
+        _workItemService = workItemService;
+        _workItemExecutionService = workItemExecutionService;
+        _apiConfiguration = apiConfiguration;
+    }
+
+    public async Task<PagedResult<Job>> GetFilteredJobs(Filter filter)
+    {
+        try
+        {
+            var performanceMonitorId = Guid.NewGuid();
+            var getFilteredJobsQueryMonitor = new PerformanceMonitor("GetFilteredJobs", performanceMonitorId);
+
+            var filteredJobs = await _jobRepository.FilteredListAsync(filter,
+                                                            _apiConfiguration.NumberOfWorkItemExecutions,
+                                                            _apiConfiguration.NumberOfExecutionMessages);
+
+            getFilteredJobsQueryMonitor.LogPerformance();
+
+            return filteredJobs;
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, "Failure retrieving filtered dispatcher jobs.");
+            throw new ServiceException("Failure retrieving filtered dispatcher jobs.");
+        }
+    }
+
+    public async Task<IEnumerable<Job>> GetAsync()
+    {
+        try
+        {
+            return await _jobRepository.ListAsync(_apiConfiguration.NumberOfWorkItemExecutions, _apiConfiguration.NumberOfExecutionMessages);
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, "Failure retrieving dispatcher jobs.");
+            throw new ServiceException("Failure retrieving dispatcher jobs.");
+        }
+    }
+
+    public async Task<IEnumerable<Job>> GetInactiveAsync()
+    {
+        try
+        {
+            return await _jobRepository.ListInactiveAsync(_apiConfiguration.NumberOfWorkItemExecutions, _apiConfiguration.NumberOfExecutionMessages);
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, "Failure retrieving inactive dispatcher jobs.");
+            throw new ServiceException("Failure retrieving inactive dispatcher jobs.");
+        }
+    }
+
+    public async Task<IEnumerable<Job>> GetInactiveExcludeSoftDeletedTrackedAsync()
+    {
+        try
+        {
+            return await _jobRepository.ListInactiveExcludeSoftDeletedTrackedAsync(_apiConfiguration.NumberOfWorkItemExecutions, _apiConfiguration.NumberOfExecutionMessages);
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, "Failure retrieving non-SoftDeleted inactive dispatcher jobs.");
+            throw new ServiceException("Failure retrieving non-SoftDeleted inactive dispatcher jobs.");
+        }
+    }
+
+    public async Task<IEnumerable<Job>> GetSoftDeletedTrackedAsync(DateTime cutOffDate)
+    {
+        try
+        {
+            return await _jobRepository.ListSoftDeletedTrackedAsync(_apiConfiguration.NumberOfWorkItemExecutions, _apiConfiguration.NumberOfExecutionMessages, cutOffDate);
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, "Failure retrieving soft deleted dispatcher jobs.");
+            throw new ServiceException("Failure retrieving soft deleted dispatcher jobs.");
+        }
+    }
+
+    public async Task<Job> GetByIdAsync(int id)
+    {
+        try
+        {
+            var job = await _jobRepository.FindByIdAsync(id, 
+                                                         _apiConfiguration.NumberOfWorkItemExecutions, 
+                                                         _apiConfiguration.NumberOfExecutionMessages);
+            ArgumentNullException.ThrowIfNull(job);
+            return job;
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, $"Failure retrieving dispatcher jobs with id {id}.");
+            throw new ServiceException($"Failure retrieving dispatcher jobs with id {id}.");
+        }
+    }
+
+    public async Task<Job> GetByIdExcludingWorkItemsAsync(int id)
+    {
+        try
+        {
+            var job = await _jobRepository.FindByIdExcludingWorkItemsAsync(id);
+            ArgumentNullException.ThrowIfNull(job);
+            return job;
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, $"Failure retrieving dispatcher jobs with id {id}.");
+            throw new ServiceException($"Failure retrieving dispatcher jobs with id {id}.");
+        }
+    }
+
+    public async Task<Job> GetByWorkItemIdAsync(int workItemId)
+    {
+        try
+        {
+            var job = await _jobRepository.FindByWorkItemIdAsync(workItemId, 
+                                                                 _apiConfiguration.NumberOfWorkItemExecutions, 
+                                                                 _apiConfiguration.NumberOfExecutionMessages);
+            ArgumentNullException.ThrowIfNull(job);
+            return job;
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, $"Failure retrieving dispatcher jobs by WorkItemId {workItemId}.");
+            throw new ServiceException($"Failure retrieving dispatcher jobs by WorkItemId {workItemId}.");
+        }
+    }
+
+    public async Task<Job> GetByWorkItemIdExcludingWorkItemsAsync(int workItemId)
+    {
+        try
+        {
+            var job = await _jobRepository.FindByWorkItemIdExcludingWorkItemsAsync(workItemId);
+            ArgumentNullException.ThrowIfNull(job);
+            return job;
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, $"Failure retrieving dispatcher jobs excluding WorkItems by WorkItemId {workItemId}.");
+            throw new ServiceException($"Failure retrieving dispatcher jobs excluding WorkItems by WorkItemId {workItemId}.");
+        }
+    }
+
+    public async Task<Job> GetByWorkItemIdExcludingWorkItemsIncludeErrorRecipientsAsync(int workItemId)
+    {
+        try
+        {
+            var job = await _jobRepository.FindByWorkItemIdExcludingWorkItemsIncludeErrorRecipientsAsync(workItemId);
+            ArgumentNullException.ThrowIfNull(job);
+            return job;
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, $"Failure retrieving dispatcher jobs excluding WorkItems by WorkItemId {workItemId}.");
+            throw new ServiceException($"Failure retrieving dispatcher jobs excluding WorkItems by WorkItemId {workItemId}.");
+        }
+    }
+
+    public async Task<IEnumerable<Job>> GetJobsReadyToBeScheduled()
+    {
+        try
+        {
+            var result = await _jobRepository.FindJobsReadyToBeScheduled(DateTime.Now);
+            return result;
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, "Failure getting pending Jobs");
+            throw new ServiceException("Failure getting pending Jobs");
+        }
+    }
+
+    public async Task<IEnumerable<Job>> GetErrorMonitoringJobs()
+    {
+        try
+        {
+            var result = await _jobRepository.FindJobsPastDateByStatuses(DateTime.Now, 
+                                                               _apiConfiguration.NumberOfWorkItemExecutions, 
+                                                               _apiConfiguration.NumberOfExecutionMessages,
+                                                               _apiConfiguration.JobStatusesToMonitorByTimeLimit);
+            ArgumentNullException.ThrowIfNull(result);
+            var erroredAndStuckJobs = result.Where(job => job.JobStatus == JobStatus.Failed).ToList();
+            if (_apiConfiguration.JobStatusesToMonitorByTimeLimit is not null && _apiConfiguration.JobStatusesToMonitorByTimeLimit.Count() > 0)
+            {
+                erroredAndStuckJobs.AddRange(GetJobsInStatusLongerThanTimeLimit(result.Except(erroredAndStuckJobs).ToList(),
+                                                                                _apiConfiguration.ProcessingLimitInMinutes));
+            }
+
+            return erroredAndStuckJobs;
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, "Failure getting next errored and stuck in processing Jobs");
+            throw new ServiceException("Failure getting next errored and stuck in processing Jobs");
+        }
+    }
+
+    public List<Job> GetJobsInStatusLongerThanTimeLimit(List<Job> matchingStatusJobs, int processingLimitInMinutes)
+    {
+        var stuckInStatusJobs = new List<Job>();
+
+        foreach (var job in matchingStatusJobs)
+        {
+            var currentWorkItem = job.WorkItems.FirstOrDefault(wi => wi.CurrentWorkItemExecution != null);
+            if (currentWorkItem == null)
+            {
+                Log.Warning($"Current Work Item came back as null when attempting to get stuck in processing Jobs. JobId: {job.JobId}");
+                continue;
+            }
+
+            var startTime = Convert.ToDateTime(currentWorkItem.CurrentWorkItemExecution.StartedTimestamp);
+            var minutesInStatus = DateTime.Now.Subtract(startTime).TotalMinutes;
+            if (minutesInStatus > processingLimitInMinutes)
+            {
+                stuckInStatusJobs.Add(job);
+            }
+        }
+
+        return stuckInStatusJobs;
+    }
+
+    public async Task<Job> SaveAsync(Job job)
+    {
+        try
+        {
+            var result = await _jobRepository.AddAsync(job);
+            await _unitOfWork.CompleteAsync();
+            ArgumentNullException.ThrowIfNull(result);
+            return result;
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, "Failure saving dispatcher job", job);
+            throw new ServiceException("Failure saving dispatcher job");
+        }
+    }
+
+    public async Task CompleteJobAndWorkItem(Job job, WorkItemExecution currentWorkItemExecution, StatusType workItemFinalStatus)
+    {
+        try
+        {
+            job.ResetRunTimeAndStatus();
+			CronHelpers.SetNextScheduledRunTime(ref job);
+            var updatedJob = await UpdateExcludingWorkItemsAsyncWithoutSave(job);
+            if (updatedJob == null)
+            {
+                throw new Exception($"Failure to setup update for job. Job:{JsonConvert.SerializeObject(job)}");
+            }
+
+            var updatedWorkItemExecution = await _workItemExecutionService.UpdateCompleteWorkItemExecutionWithoutSave(currentWorkItemExecution, workItemFinalStatus);
+            if (updatedWorkItemExecution == null)
+            {
+                throw new Exception($"Failure to setup update for WorkItemExecution. WorkItemExecution:{JsonConvert.SerializeObject(currentWorkItemExecution)}");
+            }            
+            await _unitOfWork.CompleteAsync();
+		    Log.Information("Job Id {JobId} was saved with status {Status}", job.JobId, job.JobStatus);
+        }
+        catch (SqlException sqlException)
+        {
+            var message = $"SQL Failure to Complete Job and WorkItemExecution. JobId: {job.JobId}, WorkItemExecutionId: {currentWorkItemExecution.WorkItemExecutionId}.";
+            Log.Error(sqlException, message);
+            throw new ServiceException($"{message} ExceptionMessage: {sqlException.Message}");
+        }
+        catch (Exception exception)
+        {
+            var message = $"Failure to Complete Job and WorkItemExecution. JobId: {job.JobId}, WorkItemExecutionId: {currentWorkItemExecution.WorkItemExecutionId}.";
+            Log.Error(exception, message);
+            throw new ServiceException($"{message} ExceptionMessage: {exception.Message}");
+        }       
+    }
+
+    public async Task CompleteWorkItemAndContinueProcessingJob(Job job, WorkItemExecution currentWorkItemExecution, StatusType workItemFinalStatus)
+    {
+        try
+        {
+            job.JobStatus = JobStatus.Processing;
+            var updatedJob = await UpdateExcludingWorkItemsAsyncWithoutSave(job);
+            if (updatedJob == null)
+            {
+                throw new Exception($"Failure to setup update for job. Job:{JsonConvert.SerializeObject(job)}");
+            }
+
+            var updatedWorkItemExecution = await _workItemExecutionService.UpdateCompleteWorkItemExecutionWithoutSave(currentWorkItemExecution, workItemFinalStatus);
+            if (updatedWorkItemExecution == null)
+            {
+                throw new Exception($"Failure to setup update for WorkItemExecution. WorkItemExecution:{JsonConvert.SerializeObject(currentWorkItemExecution)}");
+            }
+            await _unitOfWork.CompleteAsync();
+            Log.Information("Job Id {JobId} was saved with status {Status}", job.JobId, job.JobStatus);
+        }
+        catch (SqlException sqlException)
+        {
+            var message = $"SQL Failure to Complete Job and WorkItemExecution. JobId: {job.JobId}, WorkItemExecutionId: {currentWorkItemExecution.WorkItemExecutionId}.";
+            Log.Error(sqlException, message);
+            throw new ServiceException($"{message} ExceptionMessage: {sqlException.Message}");
+        }
+        catch (Exception exception)
+        {
+            var message = $"Failure to Complete Job and WorkItemExecution. JobId: {job.JobId}, WorkItemExecutionId: {currentWorkItemExecution.WorkItemExecutionId}.";
+            Log.Error(exception, message);
+            throw new ServiceException($"{message} ExceptionMessage: {exception.Message}");
+        }
+    }
+
+    public async Task<Job> UpdateAsync(Job job)
+    {
+        try
+        {
+            var updatedJob = await _jobRepository.Update(job);
+            await _unitOfWork.CompleteAsync();
+
+            return updatedJob;
+        }
+        catch (SqlException sqlException)
+        {
+            Log.Error(sqlException, "SQL Failure updating dispatcher job", job);
+            throw new ServiceException("SQL Failure updating dispatcher job");
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, "Failure updating dispatcher job", job);
+            throw new ServiceException("Failure updating dispatcher job");
+        }
+    }
+
+    public async Task<Job> UpdateExcludingWorkItemsAsyncWithoutSave(Job job)
+    {
+        return await _jobRepository.Update(job);
+    }
+
+    public async Task<Job> RemoveSoftDeleteById(int id)
+    {
+        try
+        {
+            var existing = await GetByIdAsync(id);
+            ArgumentNullException.ThrowIfNull(existing);
+
+            existing.SoftDeleteDate = null;
+            _jobRepository.Update(existing);
+            await _unitOfWork.CompleteAsync();
+
+            return existing;
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, $"Failure to remove soft delete date for dispatcher job. JobId: {id}");
+            throw new ServiceException($"Failure to remove soft delete date for dispatcher job. JobId: {id}");
+        }
+    }
+
+    public async Task DeleteAsync(int jobId)
+    {
+        try
+        {
+            var job = await _jobRepository.FindByIdAsync(jobId,
+                                                         _apiConfiguration.NumberOfWorkItemExecutions,
+                                                         _apiConfiguration.NumberOfExecutionMessages);
+            ArgumentNullException.ThrowIfNull(job);
+            
+            _jobRepository.Remove(job);
+            await _unitOfWork.CompleteAsync();
+        }
+        catch (SqlException sqlException)
+        {
+            Log.Error(sqlException, $"SQL Failure removing dispatcher job. JobId: {jobId}");
+            throw new ServiceException($"SQL Failure removing dispatcher job. JobId: {jobId}");
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, $"Failure removing dispatcher job. JobId: {jobId}");
+            throw new ServiceException($"Failure removing dispatcher job. JobId: {jobId}");
+        }
+    }
+
+    public async Task DeleteFromCleanupAsync(Job job)
+    {
+        try
+        {
+            ArgumentNullException.ThrowIfNull(job);
+
+            _jobRepository.Remove(job);
+        }
+        catch (SqlException sqlException)
+        {
+            Log.Error(sqlException, $"SQL Failure removing dispatcher job. JobId: {job.JobId}");
+            throw new ServiceException($"SQL Failure removing dispatcher job. JobId: {job.JobId}");
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, $"Failure removing dispatcher job. JobId: {job.JobId}");
+            throw new ServiceException($"Failure removing dispatcher job. JobId: {job.JobId}");
+        }
+    }
+
+    public async Task<Job> RunJobNowAsync(Job job)
+    {
+        try
+        {
+            job = await ResetJob(job);
+            job.NextRunTime = DateTime.Now;
+            job.JobStatus = JobStatus.ReadyToSchedule;
+            var updatedJob = await UpdateAsync(job);
+            return updatedJob;
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, "Failure to set job to run now.");
+            throw new ServiceException("Failure to set job to run now.");
+        }
+    }
+
+    public async Task<Job> RunJobNowByIdAsync(int id)
+    {
+        try
+        {
+            var job = await GetByIdAsync(id);
+            var updatedJob = await RunJobNowAsync(job);
+            return updatedJob;
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, $"Failure to set job to run now by id. Id: {id}");
+            throw new ServiceException($"Failure to set job to run now by id. Id: {id}");
+        }
+    }
+
+    public async Task<Job> DeactivateJobAsync(Job job)
+    {
+        try
+        {
+            job = await ResetJob(job);
+            var updatedJob = await UpdateAsync(job);
+            return updatedJob;
+        }
+        catch (Exception exception)
+        {
+
+            Log.Error(exception, "Failure to deactivate job.");
+            throw new ServiceException("Failure to deactivate job.");
+        }
+    }
+
+    public async Task<Job> RescheduleJobAsync(Job job)
+    {
+        try
+        {
+            job = await ResetJob(job);
+            CronHelpers.SetNextScheduledRunTime(ref job);
+			var updatedJob = await UpdateAsync(job);
+		    Log.Information("Job Id {JobId} was saved with status {Status}", job.JobId, job.JobStatus);
+            return updatedJob;
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, "Failure to reschedule job.");
+            throw new ServiceException("Failure to reschedule job.");
+        }
+    }
+
+    public async Task<Job> ResetJob(Job job)
+    {
+        foreach (var workItem in job.WorkItems)
+        {
+            var executionsToReset = await _workItemExecutionService.GetAllUnFinishedWorkItemExecutions(workItem.WorkItemId);
+            foreach (var workItemExecution in executionsToReset)
+            {
+                workItemExecution.StartedTimestamp = workItemExecution.StartedTimestamp == null ?
+                                                        DateTime.Now :
+                                                        workItemExecution.StartedTimestamp;
+                workItemExecution.CompletedTimestamp = workItemExecution.CompletedTimestamp == null ?
+                                                        DateTime.Now :
+                                                        workItemExecution.CompletedTimestamp;
+
+                var updatedExecution = await _workItemExecutionService.UpdateAsync(workItemExecution);
+                ArgumentNullException.ThrowIfNull(updatedExecution);
+            }
+        }
+
+        job.ResetRunTimeAndStatus();
+        return job;
+    }
+
+    public async Task<Job> CopyJobAsync(CopyJobRequest request)
+    {
+        try
+        {
+            var job = (await GetByIdAsync(request.JobId)).DeepCopy();
+
+            if (job == null)
+            {
+                throw new Exception($"Job with id {request.JobId} not found.");
+            }
+
+            job.Name = $"(COPY) {job.Name}";
+            job.JobId = 0;
+            job.CronSchedule = null;
+            job.CreatedAt = DateTime.Now;
+            job.ModifiedDate = null;
+            job.ResetRunTimeAndStatus();
+
+            foreach (var errorRecipient in job.JobErrorRecipients)
+            {
+                errorRecipient.JobErrorRecipientId = 0;
+                errorRecipient.JobId = 0;
+            }
+
+            if (request.ChangeEnvironment)
+            {
+                job.JobTags = null;
+                
+                foreach (var workItem in job.WorkItems)
+                {
+                    workItem.WorkItemId = 0;
+                    workItem.JobId = 0;
+                    workItem.Url = null;
+                    workItem.WorkItemExecutions = null;
+                }
+
+                return job;
+            }
+            
+            foreach (var workItem in job.WorkItems)
+            {
+                workItem.WorkItemId = 0;
+                workItem.JobId = 0; 
+                workItem.Url = null;
+                workItem.WorkItemExecutions = null;
+            }
+
+            foreach (var jobTag in job.JobTags)
+            {
+                jobTag.JobTagId = 0;
+                jobTag.JobId = 0;
+                jobTag.Tag = null;
+            }
+
+            var savedJob = await SaveAsync(job);
+
+            if (savedJob == null)
+            {
+                throw new Exception("Failure to save new job.");
+            }
+
+            return savedJob;
+        }
+        catch (Exception exception)
+        {
+            var message = $"Failure to copy dispatcher job. Request: {JsonConvert.SerializeObject(request)}";
+            Log.Error(exception, message);
+            throw new ServiceException($"{message}. Exception: {exception.Message}");
+        }
+    }
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Services/JobTagService.cs

@@ -0,0 +1,81 @@
+using System.Collections.Generic;
+using Microsoft.Data.SqlClient;
+using LCU.Api.Internal.DispatcherLib.Repositories;
+namespace LCU.Api.Internal.Dispatcher.Services;
+
+public class JobTagService : IJobTagService
+{
+    private readonly IJobTagRepository _jobTagRepository;
+    private readonly IUnitOfWork _unitOfWork;
+
+    public JobTagService(IUnitOfWork unitOfWork,
+                      IJobTagRepository jobTagRepository)
+    {
+        _unitOfWork = unitOfWork;
+        _jobTagRepository = jobTagRepository;
+    }
+
+    public async Task<IEnumerable<JobTag>> GetAsync()
+    {
+        try
+        {
+            return await _jobTagRepository.ListAsync();
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, "Failure retrieving JobTags.");
+            throw new ServiceException("Failure retrieving JobTags.");
+        }
+    }
+
+    public async Task<IEnumerable<JobTag>> GetByTagIdAsync(int tagId)
+    {
+        try
+        {
+            ArgumentNullException.ThrowIfNull(tagId);
+            return await _jobTagRepository.FindByTagIdAsync(tagId);
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, $"Failure retrieving dispatcher Job Tags. TagId: {tagId}");
+            throw new ServiceException($"Failure retrieving dispatcher Job Tags. TagId: {tagId}");
+        }
+    }
+
+    public async Task<JobTag> SaveAsync(JobTag jobTag)
+    {
+        try
+        {
+            var result = await _jobTagRepository.AddAsync(jobTag);
+            await _unitOfWork.CompleteAsync();
+            ArgumentNullException.ThrowIfNull(result);
+            return result;
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, $"Failure saving dispatcher JobTag. JobTagId: {jobTag.JobTagId}");
+            throw new ServiceException($"Failure saving dispatcher JobTag. JobTagId: {jobTag.JobTagId}");
+        }
+    }
+
+    public async Task Delete(int jobTagId)
+    {
+        try
+        {
+            var jobTag = await _jobTagRepository.FindByIdAsync(jobTagId);
+            ArgumentNullException.ThrowIfNull(jobTag);
+            _jobTagRepository.Remove(jobTag);
+            await _unitOfWork.CompleteAsync();
+        }
+        catch (SqlException sqlException)
+        {
+            Log.Error(sqlException, $"SQL Failure removing dispatcher JobTag. JobTagId: {jobTagId}");
+            throw new ServiceException($"SQL Failure removing dispatcher JobTag. JobTagId: {jobTagId}");
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, $"Failure removing dispatcher JobTag. JobTagId: {jobTagId}");
+            throw new ServiceException($"Failure removing dispatcher JobTag. JobTagId: {jobTagId}");
+        }
+    }
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Services/NotificationService.cs

@@ -0,0 +1,40 @@
+using System.Collections.Generic;
+using Newtonsoft.Json;
+using LCU.Library.Internal.NotificationsModels.Models;
+using LCU.Api.Internal.DispatcherSharedLib.Configurations;
+using LCU.Api.Internal.DispatcherSharedLib.Utilities;
+using LCU.Api.Internal.Dispatcher.DataAccess;
+
+namespace LCU.Api.Internal.Dispatcher.Services;
+
+public class NotificationService : INotificationService
+{
+    private readonly INotificationsApiDao _notificationsApiDao;
+
+	public NotificationService(INotificationsApiDao notificationsApiDao)
+	{
+        _notificationsApiDao = notificationsApiDao;
+	}
+
+	public async Task<bool> SendEmailAsync(string recipient, string subject, string emailTemplate)
+    {
+        try
+        {
+            var emailRequest = new EmailRequest
+            {
+                Recipient = new List<string>() { recipient },
+                Subject = subject,
+                Body = emailTemplate
+            };
+
+            await _notificationsApiDao.SendEmailAsync(emailRequest);
+
+            return true;
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, $"Failure to send email from Dispatcher");
+            return false;
+        }
+    }
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Services/QueueServices/IQueueCompletionService.cs

@@ -0,0 +1,8 @@
+using System.Collections.Generic;
+
+namespace LCU.Api.Internal.Dispatcher.Services.QueueServices;
+
+public interface IQueueCompletionService
+{
+    Task QueueCompletion(DispatcherReturnResponse request, List<string> executionMessages = null, bool successful = true);
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Services/QueueServices/IQueueDetermineOperationService.cs

@@ -0,0 +1,10 @@
+using LCU.Library.Internal.DispatcherModels.Enums;
+using LCU.Library.Internal.DispatcherModels.Models.Requests;
+
+namespace LCU.Api.Internal.Dispatcher.Services.QueueServices
+{
+    public interface IQueueDetermineOperationService
+	{
+		Task QueueDetermineOperation(Job job, WorkItem workItem, DetermineOperationType type);
+	}
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Services/QueueServices/IQueueExecutionService.cs

@@ -0,0 +1,7 @@
+namespace LCU.Api.Internal.Dispatcher.Services.QueueServices;
+
+public interface IQueueExecutionService
+{
+    Task QueueExecution(WorkItemExecution request);
+    Task QueueRetryExecution(WorkItemExecution request);
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Services/QueueServices/IQueueJobFailureService.cs

@@ -0,0 +1,10 @@
+using LCU.Library.Internal.DispatcherModels.Models.Requests;
+using System.Collections.Generic;
+
+namespace LCU.Api.Internal.Dispatcher.Services.QueueServices
+{
+    public interface IQueueJobFailureService
+	{
+		public Task QueueJobFailure(int jobId, int workItemId, List<string> messages);
+	}
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Services/QueueServices/IQueueSendErroredJobAlertService.cs

@@ -0,0 +1,10 @@
+using LCU.Library.Internal.DispatcherModels.Models.Requests;
+
+namespace LCU.Api.Internal.Dispatcher.Services.QueueServices
+{
+    public interface IQueueSendErroredJobAlertService
+	{
+        public Task QueueSendErroredJobAlert(BatchedErroredJobAlertRequest request);
+        public Task QueueSendErroredJobAlert(Job job, WorkItem workItem);
+    }
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Services/QueueServices/QueueCompletionService.cs

@@ -0,0 +1,45 @@
+using System.Collections.Generic;
+using System.Threading;
+using MassTransit;
+using Newtonsoft.Json;
+using ApiResponse = LCU.Library.Internal.ApiResponse.Models;
+using LCU.Api.Internal.MaidBotLib.Models;
+using LCU.Api.Internal.Dispatcher.Configurations;
+namespace LCU.Api.Internal.Dispatcher.Services.QueueServices;
+
+public class QueueCompletionService : IQueueCompletionService
+{
+    private readonly ISendEndpoint _sendEndpoint;
+    private readonly ReceiveEndpointConfigurations _queueReceiveEndpointsConfiguration;
+
+    public QueueCompletionService(IBusControl busControl,
+                                  ReceiveEndpointConfigurations queueReceiveEndpointsConfiguration)
+    {
+        _queueReceiveEndpointsConfiguration = queueReceiveEndpointsConfiguration;
+        _sendEndpoint = busControl.GetSendEndpoint(_queueReceiveEndpointsConfiguration.CompletionReceiveEndpoint.ReceiveEndpointUri).Result;
+    }
+
+    public async Task QueueCompletion(DispatcherReturnResponse request, List<string> messages = null, bool successful = true)
+    {
+        try
+        {
+            ArgumentNullException.ThrowIfNull(request);
+
+            var response = new ApiResponse.Response()
+            {
+                Successful = successful,
+                ResponseStack = messages == null ? new List<string>() : messages,
+                ResponsePayload = JsonConvert.SerializeObject(request)
+            };
+            var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30));
+            await _sendEndpoint.Send(response, cancellationTokenSource.Token);
+        }
+        catch (Exception exception)
+        {
+            Log.ForContext(Enum.GetName(typeof(PropertyEnum), PropertyEnum.Group),
+                           Enum.GetName(typeof(GroupEnum), GroupEnum.ITDevelopment))
+               .Fatal(exception, $"Failed to queue completion workitem request. WorkItemId: {request?.WorkItemId}");
+            throw new ServiceException($"Failed to queue completion workitem request. WorkItemId: {request?.WorkItemId}");
+        }
+    }
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Services/QueueServices/QueueDetermineOperationService.cs

@@ -0,0 +1,48 @@
+using System.Collections.Generic;
+using System.Threading;
+using LCU.Api.Internal.Dispatcher.Configurations;
+using LCU.Api.Internal.MaidBotLib.Models;
+using LCU.Library.Internal.DispatcherModels.Enums;
+using LCU.Library.Internal.DispatcherModels.Models.Requests;
+using MassTransit;
+using Newtonsoft.Json;
+using ApiResponse = LCU.Library.Internal.ApiResponse.Models;
+
+namespace LCU.Api.Internal.Dispatcher.Services.QueueServices
+{
+    public class QueueDetermineOperationService : IQueueDetermineOperationService
+	{
+		private readonly ISendEndpoint _sendEndpoint;
+		private readonly ReceiveEndpointConfigurations _queueReceiveEndpointsConfiguration;
+
+		public QueueDetermineOperationService(IBusControl busControl,
+                                  ReceiveEndpointConfigurations queueReceiveEndpointsConfiguration)
+		{
+			_queueReceiveEndpointsConfiguration = queueReceiveEndpointsConfiguration;
+			_sendEndpoint = busControl.GetSendEndpoint(_queueReceiveEndpointsConfiguration.DetermineOperationReceiveEndpoint.ReceiveEndpointUri).Result;
+		}
+		
+        public async Task QueueDetermineOperation(Job job, WorkItem workItem, DetermineOperationType type)
+		{
+			try
+            {
+                var request = new DetermineOperationRequest
+                {
+                    CurrentJob = job,
+                    CurrentWorkItem = workItem,
+                    DetermineOperationType = type
+                };
+
+                var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30));
+                await _sendEndpoint.Send(request, cancellationTokenSource.Token);
+            }
+            catch (Exception exception)
+            {
+                Log.ForContext(Enum.GetName(typeof(PropertyEnum), PropertyEnum.Group),
+                               Enum.GetName(typeof(GroupEnum), GroupEnum.ITDevelopment))
+                   .Fatal(exception, $"Failed to queue determine operation request.  JobName: {job.Name} WorkItemId: {workItem.WorkItemId}");
+                throw new ServiceException($"Failed to queue determine operation request.  JobName: {job.Name} WorkItemId: {workItem.WorkItemId}");
+            }
+		}
+	}
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Services/QueueServices/QueueExecutionService.cs

@@ -0,0 +1,55 @@
+using System.Threading;
+using MassTransit;
+using LCU.Api.Internal.MaidBotLib.Models;
+using LCU.Api.Internal.Dispatcher.Configurations;
+namespace LCU.Api.Internal.Dispatcher.Services.QueueServices;
+
+public class QueueExecutionService : IQueueExecutionService
+{
+    private readonly ISendEndpoint _sendEndpoint;
+    private readonly IWorkItemExecutionService _workItemExecutionService;
+    private readonly ReceiveEndpointConfigurations _queueReceiveEndpointsConfiguration;
+
+    public QueueExecutionService(IBusControl busControl,
+                                 IWorkItemExecutionService workItemExecutionService,
+                                 ReceiveEndpointConfigurations queueReceiveEndpointsConfiguration)
+    {
+        _queueReceiveEndpointsConfiguration = queueReceiveEndpointsConfiguration;
+        _workItemExecutionService = workItemExecutionService;
+        _sendEndpoint = busControl.GetSendEndpoint(_queueReceiveEndpointsConfiguration.ExecutionReceiveEndpoint.ReceiveEndpointUri).Result;
+    }
+
+    public async Task QueueExecution(WorkItemExecution request)
+    {
+        try
+        {
+            var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30));
+            await _sendEndpoint.Send(request, cancellationTokenSource.Token);
+        }
+        catch (Exception exception)
+        {
+            Log.ForContext(Enum.GetName(typeof(PropertyEnum), PropertyEnum.Group),
+                           Enum.GetName(typeof(GroupEnum), GroupEnum.ITDevelopment))
+               .Fatal(exception, $"Failed to queue workitemexecution request. WorkItemExecutionId: {request?.WorkItemExecutionId}");
+            throw new ServiceException($"Failed to queue workitemexecution request. WorkItemExecutionId: {request?.WorkItemExecutionId}");
+        }
+    }
+
+    public async Task QueueRetryExecution(WorkItemExecution request)
+    {
+        try
+        {
+            var updatedWorkItemExection = _workItemExecutionService.UpdateProcessingWorkItemExecution(request);
+
+            var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30));
+            await _sendEndpoint.Send(request, cancellationTokenSource.Token);
+        }
+        catch (Exception exception)
+        {
+            Log.ForContext(Enum.GetName(typeof(PropertyEnum), PropertyEnum.Group),
+                           Enum.GetName(typeof(GroupEnum), GroupEnum.ITDevelopment))
+               .Fatal(exception, $"Failed to queue workitemexecution request. WorkItemExecutionId: {request?.WorkItemExecutionId}");
+            throw new ServiceException($"Failed to queue workitemexecution request. WorkItemExecutionId: {request?.WorkItemExecutionId}");
+        }
+    }
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Services/QueueServices/QueueJobFailureService.cs

@@ -0,0 +1,49 @@
+using System.Collections.Generic;
+using System.Threading;
+using LCU.Api.Internal.Dispatcher.Configurations;
+using LCU.Api.Internal.MaidBotLib.Models;
+using LCU.Library.Internal.ApiResponse.Models;
+using LCU.Library.Internal.DispatcherModels.Entities;
+using LCU.Library.Internal.DispatcherModels.Models;
+using LCU.Library.Internal.DispatcherModels.Models.Requests;
+using MassTransit;
+using Newtonsoft.Json;
+using ApiResponse = LCU.Library.Internal.ApiResponse.Models;
+
+namespace LCU.Api.Internal.Dispatcher.Services.QueueServices
+{
+    public class QueueJobFailureService : IQueueJobFailureService
+	{
+		private readonly ISendEndpoint _sendEndpoint;
+		private readonly ReceiveEndpointConfigurations _queueReceiveEndpointsConfiguration;
+
+		public QueueJobFailureService(IBusControl busControl,
+			ReceiveEndpointConfigurations queueReceiveEndpointsConfiguration)
+		{
+			_queueReceiveEndpointsConfiguration = queueReceiveEndpointsConfiguration;
+			_sendEndpoint = busControl.GetSendEndpoint(_queueReceiveEndpointsConfiguration.JobFailureReceiveEndpoint.ReceiveEndpointUri).Result;
+		}
+		public async Task QueueJobFailure(int jobId, int workItemId, List<string> messages)
+        {
+            try
+			{
+                var request = new JobFailureRequest()
+                {
+                    JobId = jobId,
+                    WorkItemId = workItemId,
+                    ExecutionMessages = messages
+                };
+
+				var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30));
+				await _sendEndpoint.Send(request, cancellationTokenSource.Token);
+			}
+			catch (Exception exception)
+			{
+				Log.ForContext(Enum.GetName(typeof(PropertyEnum), PropertyEnum.Group),
+							   Enum.GetName(typeof(GroupEnum), GroupEnum.ITDevelopment))
+				   .Fatal(exception, $"Failed to queue job failure request. JobId: {jobId}, WorkItemId: {workItemId}");
+				throw new ServiceException($"Failed to queue job failure request. JobId: {jobId}, WorkItemId: {workItemId}");
+			}
+		}
+	}	
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Services/QueueServices/QueueSendErroredJobAlertService.cs

@@ -0,0 +1,63 @@
+using System.Collections.Generic;
+using System.Threading;
+using LCU.Api.Internal.Dispatcher.Configurations;
+using LCU.Api.Internal.MaidBotLib.Models;
+using LCU.Library.Internal.DispatcherModels.Models.Requests;
+using MassTransit;
+using Newtonsoft.Json;
+using ApiResponse = LCU.Library.Internal.ApiResponse.Models;
+
+namespace LCU.Api.Internal.Dispatcher.Services.QueueServices
+{
+    public class QueueSendErroredJobAlertService : IQueueSendErroredJobAlertService
+	{
+		private readonly ISendEndpoint _sendEndpoint;
+		private readonly ReceiveEndpointConfigurations _queueReceiveEndpointsConfiguration;
+
+		public QueueSendErroredJobAlertService(IBusControl busControl,
+			ReceiveEndpointConfigurations queueReceiveEndpointConfigurations)
+		{
+			_queueReceiveEndpointsConfiguration = queueReceiveEndpointConfigurations;
+			_sendEndpoint = busControl.GetSendEndpoint(_queueReceiveEndpointsConfiguration.SendErroredJobAlertReceiveEndpoint.ReceiveEndpointUri).Result;
+		}
+        public async Task QueueSendErroredJobAlert(BatchedErroredJobAlertRequest request)
+        {
+			try
+			{
+                ArgumentNullException.ThrowIfNull(request);
+				var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30));
+				await _sendEndpoint.Send(request, cancellationTokenSource.Token);
+			}
+			catch (Exception exception)
+			{
+                var workItemIds = string.Join(",",
+                                  request.ErroredJobAlertRequests
+								  .Where(x => x.CurrentWorkItem != null)
+                                  .Select(x => x.CurrentWorkItem.WorkItemId));
+
+                Log.ForContext(Enum.GetName(typeof(PropertyEnum), PropertyEnum.Group),
+							   Enum.GetName(typeof(GroupEnum), GroupEnum.ITDevelopment))
+                    .Fatal(exception, $"Failed to queue send errored job alert workitem requests. WorkItemIds: {workItemIds}");
+                throw new ServiceException($"Failed to queue send errored job alert workitem requests. WorkItemIds: {workItemIds}");
+            }
+		}
+
+		public async Task QueueSendErroredJobAlert(Job job, WorkItem workItem)
+		{
+			var request = new ErroredJobAlertRequest()
+			{
+				CurrentJob = job,
+				CurrentWorkItem = workItem
+			};
+			var requestList = new List<ErroredJobAlertRequest>()
+			{
+				request
+			};
+			var batchedRequest = new BatchedErroredJobAlertRequest()
+			{
+				ErroredJobAlertRequests = requestList
+			};
+			await QueueSendErroredJobAlert(batchedRequest);
+        }
+    }
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Services/TagService.cs

@@ -0,0 +1,135 @@
+using System.Collections.Generic;
+using Microsoft.Data.SqlClient;
+using LCU.Api.Internal.DispatcherLib.Repositories;
+namespace LCU.Api.Internal.Dispatcher.Services;
+
+public class TagService : ITagService
+{
+    private readonly ITagRepository _tagRepository;
+    private readonly IUnitOfWork _unitOfWork;
+    private readonly IJobTagService _jobTagService;
+
+    public TagService(IUnitOfWork unitOfWork,
+                      ITagRepository tagRepository, IJobTagService jobTagService)
+    {
+        _unitOfWork = unitOfWork;
+        _tagRepository = tagRepository;
+        _jobTagService = jobTagService;
+    }
+
+    public async Task<IEnumerable<Tag>> GetAsync()
+    {
+        try
+        {
+            return await _tagRepository.ListAsync();
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, "Failure retrieving dispatcher Tags.");
+            throw new ServiceException("Failure retrieving dispatcher Tags.");
+        }
+    }
+
+    public async Task<Tag> GetByIdAsync(int id)
+    {
+        try
+        {
+            var tag = await _tagRepository.FindByIdAsync(id);
+            ArgumentNullException.ThrowIfNull(tag);
+            return tag;
+        }
+        catch (ArgumentNullException nullEx)
+        {
+            Log.Error(nullEx, $"Failure retrieving dispatcher Tags. TagId: {id}");
+            throw new ArgumentNullException($"Failure retrieving dispatcher Tags. TagId: {id}");
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, $"Failure retrieving dispatcher Tags. TagId: {id}");
+            throw new ServiceException($"Failure retrieving dispatcher Tags. TagId: {id}");
+        }
+    }
+
+    public async Task<Tag> SaveAsync(Tag tag)
+    {
+        try
+        {
+            var result = await _tagRepository.AddAsync(tag);
+            await _unitOfWork.CompleteAsync();
+            ArgumentNullException.ThrowIfNull(result);
+            return result;
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, $"Failure saving dispatcher Tag. TagId: {tag.TagId}");
+            throw new ServiceException($"Failure saving dispatcher Tag. TagId: {tag.TagId}");
+        }
+    }
+
+    public async Task<Tag> UpdateAsync(Tag tag)
+    {
+        try
+        {
+            _tagRepository.Update(tag);
+            await _unitOfWork.CompleteAsync();
+
+            return tag;
+        }
+        catch (SqlException sqlException)
+        {
+            Log.Error(sqlException, $"SQL Failure updating dispatcher Tag. TagId: {tag.TagId}");
+            throw new ServiceException($"SQL Failure updating dispatcher Tag. TagId: {tag.TagId}");
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, $"SQL Failure updating dispatcher Tag. TagId: {tag.TagId}");
+            throw new ServiceException($"SQL Failure updating dispatcher Tag. TagId: {tag.TagId}");
+        }
+    }
+
+    public async Task DeleteAsync(int tagId)
+    {
+        try
+        {
+            var jobTags = await _jobTagService.GetByTagIdAsync(tagId);
+            if (jobTags == null || !jobTags.Any()) 
+            { 
+                await RemoveJobTags(jobTags.ToList());
+            }
+
+            var tag = await _tagRepository.FindByIdAsync(tagId);
+            ArgumentNullException.ThrowIfNull(tag);
+            _tagRepository.Remove(tag);
+            await _unitOfWork.CompleteAsync();
+        }
+        catch (SqlException sqlException)
+        {
+            Log.Error(sqlException, $"SQL Failure removing dispatcher Tag. TagId: {tagId}");
+            throw new ServiceException($"SQL Failure removing dispatcher Tag. TagId: {tagId}");
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, $"Failure removing dispatcher Tag. TagId: {tagId}");
+            throw new ServiceException($"Failure removing dispatcher Tag. TagId: {tagId}");
+        }
+    }
+
+    private async Task<bool> RemoveJobTags(List<JobTag> jobTags)
+    {
+        bool isSuccessful = false;
+        try
+        {
+            foreach (var jobTag in jobTags)
+            {
+                await _jobTagService.Delete(jobTag.JobTagId);
+                isSuccessful = true;
+            }
+            return isSuccessful;
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, "Failure to remove job tags.");
+            throw new ServiceException("Failure to remove job tags.");
+        }
+    }
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Services/WorkItemExecutionService.cs

@@ -0,0 +1,150 @@
+using System.Collections.Generic;
+using Microsoft.Data.SqlClient;
+using LCU.Library.Internal.DispatcherModels.Enums;
+using LCU.Api.Internal.DispatcherLib.Repositories;
+namespace LCU.Api.Internal.Dispatcher.Services;
+
+public class WorkItemExecutionService : IWorkItemExecutionService
+{
+    private readonly IUnitOfWork _unitOfWork;
+    private readonly IWorkItemExecutionRepository _workItemExecutionRepository;
+
+    public WorkItemExecutionService(IUnitOfWork unitOfWork,
+                                    IWorkItemExecutionRepository workItemExecutionRepository)
+    {
+        _unitOfWork = unitOfWork;
+        _workItemExecutionRepository = workItemExecutionRepository;
+    }
+
+    public async Task<WorkItemExecution> GetByIdAsync(int id)
+    {
+        WorkItemExecution workItemExecution = null;
+        try
+        {
+            workItemExecution = await _workItemExecutionRepository.FindByIdAsync(id);
+            ArgumentNullException.ThrowIfNull(workItemExecution);
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, $"Failure retrieving dispatcher WorkItemExecution with id {id}.");
+            throw new ServiceException($"Failure retrieving dispatcher WorkItemExecution with id {id}.");
+        }
+
+        return workItemExecution;
+    }
+
+    public async Task<WorkItemExecution> UpdateCompleteWorkItemExecution(WorkItemExecution workItemExecution, StatusType finalStatus)
+    {
+        workItemExecution = WorkItemExecution.SetAsFinal(workItemExecution, finalStatus);
+        var updatedFailedWorkItem = await UpdateAsync(workItemExecution);
+        return updatedFailedWorkItem;
+    }
+
+    public async Task<WorkItemExecution> UpdateCompleteWorkItemExecutionWithoutSave(WorkItemExecution workItemExecution, StatusType finalStatus)
+    {
+        workItemExecution = WorkItemExecution.SetAsFinal(workItemExecution, finalStatus);
+        var updatedFailedWorkItem = await UpdateAsyncWithoutSave(workItemExecution);
+        return updatedFailedWorkItem;
+    }
+
+    public async Task<WorkItemExecution> UpdateProcessingWorkItemExecution(WorkItemExecution workItemExecution)
+    {
+        workItemExecution.Status = StatusType.Processing;
+        return await UpdateAsync(workItemExecution);
+    }
+    public async Task<WorkItemExecution> UpdateFailedWorkItemExecution(WorkItemExecution workItemExecution)
+    {
+        workItemExecution.Status = StatusType.Failed;
+        return await UpdateAsync(workItemExecution);
+    }
+
+    public async Task<WorkItemExecution> UpdateAsync(WorkItemExecution workItemExecution)
+    {
+        try
+        {
+            var updatedWorkItemExecution = await UpdateAsyncWithoutSave(workItemExecution);
+            await _unitOfWork.CompleteAsync();
+
+            return updatedWorkItemExecution;
+        }
+        catch (SqlException sqlException)
+        {
+            Log.Error(sqlException, $"SQL Failure updating dispatcher WorkItemExecution with id {workItemExecution.WorkItemId}");
+            throw new ServiceException($"SQL Failure updating dispatcher WorkItemExecution with id {workItemExecution.WorkItemId}");
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, $"Failure updating dispatcher WorkItemExecution with id {workItemExecution.WorkItemId}");
+            throw new ServiceException($"Failure updating dispatcher WorkItemExecution with id {workItemExecution.WorkItemId}");
+        }
+    }
+
+    public async Task<WorkItemExecution> UpdateAsyncWithoutSave(WorkItemExecution workItemExecution)
+    {
+        try
+        {
+            _workItemExecutionRepository.Update(workItemExecution);
+
+            return workItemExecution;
+        }
+        catch (SqlException sqlException)
+        {
+            Log.Error(sqlException, $"SQL Failure updating dispatcher WorkItemExecution with id {workItemExecution.WorkItemId}");
+            throw new ServiceException($"SQL Failure updating dispatcher WorkItemExecution with id {workItemExecution.WorkItemId}");
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, $"Failure updating dispatcher WorkItemExecution with id {workItemExecution.WorkItemId}");
+            throw new ServiceException($"Failure updating dispatcher WorkItemExecution with id {workItemExecution.WorkItemId}");
+        }
+    }
+
+    public async Task<WorkItemExecution> CreateWorkItemExecution(int workItemId)
+    {
+        var newWorkItemExecution = new WorkItemExecution(workItemId);
+        return await SaveAsync(newWorkItemExecution);
+    }
+
+    public async Task<WorkItemExecution> SaveAsync(WorkItemExecution workItemExecution)
+    {
+        try
+        {
+            var result = await _workItemExecutionRepository.AddAsync(workItemExecution);
+            await _unitOfWork.CompleteAsync();
+            return result;
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, "Failure saving dispatcher WorkItemExecution", workItemExecution);
+            throw new ServiceException("Failure saving dispatcher WorkItemExecution");
+        }
+    }
+
+    public async Task<IEnumerable<WorkItemExecution>> GetAllUnFinishedWorkItemExecutions(int workItemId)
+    {
+        try
+        {
+            var result = await _workItemExecutionRepository.FindUnFinishedWorkItemExecutions(workItemId);
+            return result;
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, $"Failure retrieving dispatcher WorkItemExecutions with id {workItemId}.");
+            throw new ServiceException($"Failure retrieving dispatcher WorkItemExecutions with id {workItemId}.");
+        }
+    }
+
+    public async Task<IEnumerable<WorkItemExecution>> GetAllByWorkItemId(int workItemId)
+    {
+        try
+        {
+            var result = await _workItemExecutionRepository.FindAllByWorkItemId(workItemId);
+            return result;
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, $"Failure retrieving dispatcher WorkItemExecutions with WorkItemId {workItemId}.");
+            throw new ServiceException($"Failure retrieving dispatcher WorkItemExecutions with WorkItemId {workItemId}.");
+        }
+    }
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Services/WorkItemService.cs

@@ -0,0 +1,258 @@
+using LCU.Api.Internal.Dispatcher.Configurations;
+using LCU.Api.Internal.DispatcherLib.Repositories;
+using LCU.Api.Internal.DispatcherSharedLib.BuilderServices;
+using LCU.Library.Internal.ApiResponse.Models;
+using LCU.Library.Internal.LcuMassTransitModels;
+using Microsoft.Data.SqlClient;
+using Newtonsoft.Json;
+using System.Collections.Generic;
+namespace LCU.Api.Internal.Dispatcher.Services;
+
+public class WorkItemService : IWorkItemService
+{
+    private readonly IUnitOfWork _unitOfWork;
+    private readonly IWorkItemRepository _workItemRepository;
+    private readonly IWorkItemExecutionService _workItemExecutionService;
+    private readonly IActionBuilderService _actionBuilderService;
+    private readonly ConnectionConfiguration _connectionConfiguration;
+    private readonly ReceiveEndpointConfiguration _completionReceiveEndpoint;
+    private readonly ApiConfiguration _apiConfiguration;
+
+    public WorkItemService(IUnitOfWork unitOfWork,
+                           IWorkItemRepository workItemRepository,
+                           IWorkItemExecutionService workItemExecutionService,
+                           IActionBuilderService actionBuilderService,
+                           ConnectionConfiguration connectionConfiguration,
+                           ReceiveEndpointConfigurations queueReceiveEndpointsConfigurations,
+                           ApiConfiguration apiConfiguration)
+    {
+        _unitOfWork = unitOfWork;
+        _workItemRepository = workItemRepository;
+        _workItemExecutionService = workItemExecutionService;
+        _actionBuilderService = actionBuilderService;
+        _connectionConfiguration = connectionConfiguration;
+        _completionReceiveEndpoint = queueReceiveEndpointsConfigurations.CompletionReceiveEndpoint;
+        _apiConfiguration = apiConfiguration;
+    }
+
+    public async Task<WorkItem> GetByIdAsync(int id)
+    {
+        WorkItem workItem = null;
+        try
+        {
+            workItem = await _workItemRepository.FindByIdAsync(id);
+            ArgumentNullException.ThrowIfNull(workItem);
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, $"Failure retrieving dispatcher workitem with id {id}.");
+            throw new ServiceException($"Failure retrieving dispatcher workitem with id {id}.");
+        }
+
+        return workItem;
+    }
+
+    public async Task<IEnumerable<WorkItem>> GetByJobIdAsync(int jobId)
+    {
+        IEnumerable<WorkItem> workItems = null;
+        try
+        {
+            workItems = await _workItemRepository.FindByJobIdAsync(jobId);
+            ArgumentNullException.ThrowIfNull(workItems);
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, $"Failure retrieving dispatcher workitems with JobId {jobId}.");
+            throw new ServiceException($"Failure retrieving dispatcher workitems with JobId {jobId}.");
+        }
+
+        return workItems;
+    }
+
+    
+    public async Task<IEnumerable<WorkItem>> GetByJobIdWithExecutionsAsync(int jobId)
+    {
+        IEnumerable<WorkItem> workItems = null;
+        try
+        {
+            workItems = await _workItemRepository.FindByJobIdWithExecutionsAsync(jobId,
+                                                                                    _apiConfiguration.NumberOfWorkItemExecutions,
+                                                                                    _apiConfiguration.NumberOfExecutionMessages);
+            ArgumentNullException.ThrowIfNull(workItems);
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, $"Failure retrieving dispatcher workitems and executions with JobId {jobId}.");
+            throw new ServiceException($"Failure retrieving dispatcher workitems and executions with JobId {jobId}.");
+        }
+
+        return workItems;
+    }
+
+    public async Task<WorkItem> GetFirstOrDefaultWorkItemByJobId(int jobId)
+    {
+        WorkItem workItem = null;
+        try
+        {
+            workItem = await _workItemRepository.GetFirstOrDefaultWorkItemByJobId(jobId);
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, $"Failure retrieving first or default workItem with JobId: {jobId}.");
+            throw new ServiceException($"Failure retrieving first or default workItem with JobId: {jobId}.");
+        }
+
+        return workItem;
+    }
+
+    public async Task<WorkItem> GetNextWorkItemByJobId(int jobId, int currentWorkItemOrder)
+    {
+        WorkItem workItem = null;
+        try
+        {
+            workItem = await _workItemRepository.GetNextWorkItemByJobId(jobId, currentWorkItemOrder);
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, $"Failure retrieving next workItem with JobId: {jobId}.");
+            throw new ServiceException($"Failure retrieving next workItem with JobId: {jobId}.");
+        }
+
+        return workItem;
+    }
+
+    public async Task<ICollection<WorkItem>> PopulateActionFieldsAndUpdateList(ICollection<WorkItem> workItems)
+    {
+        try
+        {
+            var updatedWorkItems = new List<WorkItem>();
+            foreach (var workItem in workItems)
+            {
+                var newWorkItem = workItem;
+                newWorkItem = PopulateActionFields(newWorkItem);
+                newWorkItem = await UpdateAsync(newWorkItem);
+                updatedWorkItems.Add(newWorkItem);
+            }
+
+            return updatedWorkItems;
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, $"Failure populating action fields and updating dispatcher workitems.");
+        }
+
+        return null;
+    }
+
+    public async Task<WorkItem> ResetCanceledJobCounter(WorkItem workItem)
+    {
+        var trackedWorkItem = await _workItemRepository.GetTrackedWorkItem(workItem.WorkItemId);
+        trackedWorkItem.CanceledJobCounter = 0;
+        await _unitOfWork.CompleteAsync();
+        return await _workItemRepository.FindByIdAsync(trackedWorkItem.WorkItemId);
+    }
+
+    public async Task<WorkItem> IncreaseCanceledJobCounter(WorkItem workItem)
+    {
+        var trackedWorkItem = await _workItemRepository.GetTrackedWorkItem(workItem.WorkItemId);
+        trackedWorkItem.CanceledJobCounter++;
+        await _unitOfWork.CompleteAsync();
+        return await _workItemRepository.FindByIdAsync(trackedWorkItem.WorkItemId);
+    }
+
+    public async Task<WorkItem> UpdateAsync(WorkItem workItem)
+    {
+        try
+        {
+            _workItemRepository.Update(workItem);
+            await _unitOfWork.CompleteAsync();
+
+            return workItem;
+        }
+        catch (SqlException sqlException)
+        {
+            Log.Error(sqlException, $"SQL Failure updating dispatcher workitem with id {workItem.WorkItemId}");
+            throw new ServiceException($"SQL Failure updating dispatcher workitem with id {workItem.WorkItemId}");
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, $"Failure updating dispatcher workitem with id {workItem.WorkItemId}");
+            throw new ServiceException($"Failure updating dispatcher workitem with id {workItem.WorkItemId}");
+        }
+    }
+
+    public void SaveList(ICollection<WorkItem> workItems)
+    {
+        try
+        {
+            _workItemRepository.AddList(workItems);
+            _unitOfWork.Complete();
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, $"Failure saving list of workitems.");
+            throw new ServiceException($"Failure saving list of workitems.");
+        }
+    }
+
+    public async Task DeleteList(ICollection<WorkItem> workItems)
+    {
+        try
+        {
+            foreach (var workItem in workItems)
+            {
+                var workItemExecutions = await _workItemExecutionService.GetAllByWorkItemId(workItem.WorkItemId);
+                workItem.WorkItemExecutions = workItemExecutions.ToList();
+            }
+
+            _workItemRepository.RemoveList(workItems);
+            await _unitOfWork.CompleteAsync();
+        }
+        catch (SqlException sqlException)
+        {
+            Log.Error(sqlException, $"SQL Failure removing dispatcher workitems");
+            throw new ServiceException($"SQL Failure removing dispatcher workitems");
+        }
+        catch (Exception exception)
+        {
+            Log.Error(exception, $"Failure removing dispatcher workitems");
+            throw new ServiceException($"Failure removing dispatcher workitems");
+        }
+
+    }
+
+    public WorkItem PopulateActionFields(WorkItem workItem)
+    {
+        var actionBuilder = _actionBuilderService.BuildAction(workItem.ActionType);
+        var responseItem = CreateResponseItem(workItem.WorkItemId);
+        workItem = actionBuilder.AddUrlAndHttpMethod(workItem);
+        workItem = actionBuilder.AddWorkItemReturnResponse(workItem, responseItem);
+        return workItem;
+    }
+
+    private ResponseItem CreateResponseItem(int workItemId)
+    {
+        var dispatcherReturnResponse = new DispatcherReturnResponse
+        {
+            WorkItemId = workItemId
+        };
+        var serializeDispatcherReturnResponse = JsonConvert.SerializeObject(dispatcherReturnResponse);
+
+        var newReturnItem = new ResponseItem
+        {
+            ResponsePayload = serializeDispatcherReturnResponse,
+            ReturnQueueConnectionConfig = new QueueConnectionConfig()
+            {
+                Url = _connectionConfiguration.HostUrl,
+                Username = _connectionConfiguration.UserName,
+                Password = _connectionConfiguration.Password
+            },
+            ReturnQueueConsumerConfig = new QueueConsumerConfig()
+            {
+                ExchangeName = _completionReceiveEndpoint.ReceiveEndpoint
+            }
+        };
+
+        return newReturnItem;
+    }
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Templates/BatchErroredJobsTemplate.html

@@ -0,0 +1,58 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <style>
+        #errors {
+            font-family: Arial, Helvetica, sans-serif;
+            border-collapse: collapse;
+            width: 100%;
+        }
+
+            #errors td, #errors th {
+                border: 1px solid #ddd;
+                padding: 8px;
+            }
+
+            #errors tr:nth-child(even) {
+                background-color: #f2f2f2;
+            }
+
+            #errors th {
+                padding-top: 12px;
+                padding-bottom: 12px;
+                text-align: left;
+                background-color: #4CAF50;
+                color: white;
+            }
+
+        #emailHeader {
+            color: #0054a2;
+        }
+
+        #errorTableHeader {
+            background-color: #0054a2;
+            color: white;
+        }
+    </style>
+
+    <meta charset="utf-8">
+    <title>Dispatcher: Error Monitoring</title>
+</head>
+<body>
+    <h1><span id="emailHeader">Dispatcher: Error Monitoring</span></h1>
+    <table id="errors">
+        <thead id="errorTableHeader">
+            <tr style="height: 17px;">
+                <td style="max-width: 250px;font-weight: bolder;">Job Name</td>
+                <td style="max-width: 200;font-weight: bolder;">WorkItem Name</td>
+                <td style="max-width: 79px;font-weight: bolder;">JobStatus</td>
+                <td style="max-width: 150px;font-weight: bolder;">StartTime</td>
+            </tr>
+        </thead>
+        <tbody id="errorRows">
+            {TableRows}
+        </tbody>
+    </table>
+
+</body>
+</html>

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Templates/ErrorRowTemplate.html

@@ -0,0 +1,6 @@
+<tr id="errorRow" style="height: 22px;">
+    <td style="max-width: 250px;">{JobName}</td>
+    <td style="max-width: 200px;">{WorkItemName}</td>
+    <td style="max-width: 79px;">{JobStatus}</td>
+    <td style="max-width: 150px;">{StartTime}</td>
+</tr>

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/Using.cs

@@ -0,0 +1,7 @@
+global using System;
+global using System.Threading.Tasks;
+global using System.Linq;
+global using Serilog;
+global using LCU.Library.Internal.Exceptions;
+global using LCU.Library.Internal.DispatcherModels.Entities;
+global using LCU.Library.Internal.DispatcherModels.Models;

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/appsettings.json

@@ -0,0 +1,216 @@
+{
+  "Primary": {
+    "AppName": "Dispatcher",
+    "SeqUrl": "http://localhost:5080",
+    "SeqApiKey": "9MQu0vLDwQ1LCFzGixZM",
+    "CorsAllowedOrigins": [
+      "http://localhost:3000",
+      "http://lcu-internal.com",
+      "https://lcu-internal.com",
+      "https://qa-api.lcu-internal.com",
+      "https://uat-api.lcu-internal.com",
+      "https://api.lcu-internal.com"
+    ],
+    "SchedulersCronSchedule": "* * * * *",
+    "BatchErroredJobsCronSchedule": "*/15 * * * *",
+    "CleanUpJobsCronSchedule": "00 22 * * *",
+    "ProcessingLimitInMinutes": 30,
+    "NumberOfWorkItemExecutions": 10,
+    "NumberOfExecutionMessages": 20,
+    "CleanUpJobsDaysToDelete": -30,
+    "BatchErroredJobsTemplate": "./Templates/BatchErroredJobsTemplate.html",
+    "ErrorRowTemplate": "./Templates/ErrorRowTemplate.html",
+    "BatchErroredJobsSubject": "Dispatcher: Error Monitoring",
+    "DefaultErrorRecipient": "[email protected]",
+    "JobStatusesToMonitorByTimeLimit": [
+      "Processing",
+      "DeterminingOperation",
+      "PendingToRun"
+    ],
+    "DefaultDelayMilliseconds": 5000,
+    "FeatureFlags": {
+      "RunCompletionConsumer": false,
+      "RunExecutionConsumer": false,
+      "RunJobSchedulerHostedService": false,
+      "RunBatchErroredJobsHostedService": false,
+      "RunCleanUpJobsHostedService": false,
+      "RunDetermineOperationConsumer": false,
+      "RunJobFailureConsumer": false,
+      "RunSendErroredJobAlertConsumer": false
+    }
+  },
+  "FileOperationsApiConfiguration": {
+    "Url": "https://localhost:56595",
+    "MoveFileEndpoint": "/api/FileOperations/MoveFile",
+    "CopyFileEndpoint": "/api/FileOperations/CopyFile",
+    "ModifyFileEndpoint": "/api/FileOperations/ModifyFile",
+    "RenameFileEndpoint": "/api/FileOperations/RenameFile",
+    "DeleteFileEndpoint": "/api/FileOperations/DeleteFile",
+    "ZipDirectoryEndpoint": "/api/FileOperations/ZipDirectory",
+    "CreateDirectoryEndpoint": "/api/FileOperations/CreateDirectory",
+    "RenameDirectoryEndpoint": "/api/FileOperations/RenameDirectory",
+    "UnzipFileEndpoint": "/api/FileOperations/UnzipFile",
+    "EncryptFileDataEndpoint": "/api/FileOperations/EncryptFileData",
+    "DecryptFileDataEndpoint": "/api/FileOperations/DecryptFileData",
+    "UnzipPasswordProtectedFileEndpoint": "/api/FileOperations/UnzipPasswordProtectedFile",
+    "DeleteDirectoryEndpoint": "/api/FileOperations/DeleteDirectory",
+    "FileValidationEndpoint": "/FileValidation/QueueValidateFile",
+    "ExtractIndexFileDataEndpoint": "/api/IndexFile/QueueExtractIndexFileData",
+    "ValidateFileSizeInRange": "/FileValidation/QueueValidateFileSizeInRange",
+    "DeleteEmptyDirectoryEndpoint": "/api/FileOperations/DeleteEmptyDirectory"
+  },
+  "FtpApiConfiguration": {
+    "Url": "https://localhost:44351",
+    "SftpFileEndpoint": "/api/Sftp",
+    "QueueSftpDynamicFileEndpoint": "/api/Sftp/QueueSftpDynamicFileRequest",
+    "QueueSftpUploadFileEndpoint": "/api/Sftp/QueueSftpUploadRequest",
+    "QueueSftpDownloadFileEndpoint": "/api/Sftp/QueueSftpDownloadRequest"
+  },
+  "NotificationApiConfiguration": {
+    "Url": "https://qa-notifications-api.apps.azure.lcu-internal.com",
+    "EmailEndpoint": "/api/Email"
+  },
+  "ArchiverApiConfiguration": {
+    "Url": "https://localhost:44353",
+    "QueueArchiverEndpoint": "/QueueArchiver/QueueArchiver"
+  },
+  "BlendApiConfiguration": {
+    "Url": "https://localhost:54004",
+    "QueueDataImportEndpoint": "/api/Processing/QueueSyncDataFromBlend"
+  },
+  "DialerApiConfiguration": {
+    "Url": "https://localhost:1234",
+    "QueueDialerUploadEndpoint": "/api/DialerUpload/QueueDialerUploadRequest",
+    "QueueUploadDoNotCallEndpoint": "/api/UploadDoNotCall/QueueUploadDoNotCallRequest",
+    "QueueCreateVoAppsCampaignEndpoint": "/api/CreateVoAppsCampaign/QueueCreateVoAppsCampaignRequest",
+    "QueueUploadVoAppsCampaignResultsEndpoint": "/api/UploadVoAppsCampaignResults/QueueUploadVoAppsCampaignRequest",
+    "QueueUploadVoAppsCampaignErrorRequestEndpoint": "/api/UploadVoAppsCampaignResultsError/QueueUploadVoAppsCampaignErrorRequest"
+  },
+  "DocumentIngestorApiConfiguration": {
+    "Url": "https://localhost:7139",
+    "QueueDocumentIngestorEndpoint": "/DocumentIngestor/QueueDocumentIngestor",
+    "QueueDocumentBatchIngestionEndpoint": "/BatchIngestion/QueueBatchIngestion",
+    "QueueIngestorPdfPartitionerEndpoint": "/PdfPartitioner/QueueIngestorPdfPartitioner",
+    "QueueMemberDocumentNotifierEndpoint": "/MemberDocumentNotifier/QueueMemberDocumentNotifier",
+    "QueueIngestionFolderCleanUpEndpoint": "/IngestionFolderCleanUp/QueueIngestionFolderCleanUp",
+    "QueueProcessStatusTypeEmailRequestEndpoint": "/QueueProcessStatusTypeEmail/QueueProcessStatusTypeEmailRequest"
+  },
+  "DocumentTypeCleanupApiConfiguration": {
+    "Url": "https://localhost:7066",
+    "QueueDocumentRetentionCleanupEndpoint": "/DocumentTypeCleanup/QueueDocumentRetentionCleanup"
+  },
+  "HistoricalDocumentIngestorApiConfiguration": {
+    "Url": "https://localhost:44346",
+    "QueueDocumentIngestorEndpoint": "/DocumentIngestor/QueueDocumentIngestor",
+    "QueueDocumentBatchIngestionEndpoint": "/BatchIngestion/QueueBatchIngestion",
+    "QueueIngestorPdfPartitionerEndpoint": "/PdfPartitioner/QueueIngestorPdfPartitioner"
+  },
+  "RemoteDepositCaptureApiConfiguration": {
+    "Url": "https://localhost:44305",
+    "QueueRemoteDepositCaptureXmlEndpoint": "/api/RemoteDepositCaptureXml/QueueRemoteDepositCaptureXmlRequest"
+  },
+  "DataCleanUpApiConfiguration": {
+    "Url": "https://localhost:44357",
+    "QueueDataCleanUpEndpoint": "/QueueDataCleanUp/QueueDataCleanUp",
+    "QueueUpdatePersonForCheckHoldsEndpoint": "/QueueDataCleanUp/QueueUpdatePersonForCheckHolds",
+    "QueueNcoaUpdateEndpoint": "/QueueDataCleanUp/QueueUpdateNcoaProcess",
+    "QueueUpdateFannieMaeMortgageEndpoint": "/QueueDataCleanUp/QueueUpdateFannieMaeMortgage",
+    "QueueIndirectFundingEndpoint": "/QueueDataCleanUp/QueueIndirectFunding"
+  },
+  "NoticeProcessorApiConfiguration": {
+    "Url": "https://localhost:7275",
+    "QueueExtractEndpoint": "/QueueExtractRequest/RunNoticeProcessorByExtractRequestNow",
+    "QueueProcessingEmailSummaryEndpoint": "/QueueProcessingEmailRequest/RunProcessingEmailRequestNow"
+  },
+  "MomBotApiConfiguration": {
+    "Url": "https://localhost:5001",
+    "QueueMomBotEndpoint": "/api/QueueMomBot"
+  },
+  "RealTimeAlertsApiConfiguration": {
+    "Url": "https://localhost:59626",
+    "QueueInitiateRealTimeAlertsEndpoint": "/api/TransactionAlerts/QueueInitiateTransactionAlertsRequest",
+    "QueueErroredTransactionCleanUpEndpoint": "/api/TransactionAlerts/QueueInitiateErroredTransactionCleanUpRequest"
+  },
+  "RepoApiConfiguration": {
+    "Url": "https://localhost:7066",
+    "AccountBatchEndpoint": "/api/RepoBatchProcessing/QueueRepoBatchRequest"
+  },
+  "ConnectionsConfiguration": {
+    "SQL": {
+      "DispatcherDb": "data source=localhost;initial catalog=Dispatcher;Integrated Security=SSPI;MultipleActiveResultSets=True;App=EntityFramework;Encrypt=false",
+      "SQLOptions": {
+        "MaxRetryCount": 5,
+        "MaxRetryDelay": "0.00:05:00",
+        "ErrorNumbersToAdd": [],
+        "EnableDetailedErrors": true,
+        "CommandTimeout": 60
+      }
+    }
+  },
+  "DispatcherRequestBuilderConfiguration": {
+    "Url": "https://localhost:44380/api/Ingestion"
+  },
+  "DispatcherCompletionConsumer": {
+    "QueueName": "LCU.Queue.Dispatcher.Completion",
+    "ExchangeName": "LCU.Dispatcher.Direct",
+    "Subject": "LCU.Queue.Dispatcher.Completion"
+  },
+  "ConnectionConfiguration": {
+    "HostUrl": "localhost",
+    "username": "guest",
+    "password": "guest"
+  },
+  "ExecutionReceiveEndpoint": {
+    "ReceiveEndpoint": "LCU.Dispatcher.Execution",
+    "ReceiveEndpointType": "exchange",
+    "PrefetchLimit": 1
+  },
+  "CompletionReceiveEndpoint": {
+    "ReceiveEndpoint": "LCU.Dispatcher.Completion",
+    "ReceiveEndpointType": "exchange",
+    "PrefetchLimit": 1
+  },
+  "DetermineOperationReceiveEndpoint": {
+    "ReceiveEndpoint": "LCU.Dispatcher.DetermineOperation",
+    "ReceiveEndpointType": "exchange",
+    "PrefetchLimit": 1
+  },
+  "JobFailureReceiveEndpoint": {
+    "ReceiveEndpoint": "LCU.Dispatcher.JobFailure",
+    "ReceiveEndpointType": "exchange",
+    "PrefetchLimit": 1
+  },
+  "SendErroredJobAlertReceiveEndpoint": {
+    "ReceiveEndpoint": "LCU.Dispatcher.SendErroredJobAlert",
+    "ReceiveEndpointType": "exchange",
+    "PrefetchLimit": 100
+  },
+  "ApiSecurityConfiguration": {
+    "ApiKeyAuthenticationFailureBehavior": "Error",
+    "ExternalApiKeySet": {
+      "DispatcherHostedServiceClient": "VALUE-IN-VAULT"
+    }
+  },
+  "IncentiveManagerConfiguration": {
+    "Url": "https://localhost:20001/api",
+    "QueueCreditCardImportEndpoint": "/ActivityImport/QueueCreditCardImportRequest",
+    "QueueVehicleImportEndpoint": "/ActivityImport/QueueIndirectVehicleImportRequest",
+    "QueueActionCreditCardEndpoint": "/ActivityImport/QueueActionCreditCardRequest",
+    "QueueActionVehicleEndpoint": "/ActivityImport/QueueActionVehicleRequest",
+    "QueueAlliedGapMbpEndpoint": "/ActivityImport/QueueAlliedGapMbpRequest",
+    "QueueIncentiveProcessEndpoint": "/Processing/QueueIncentiveProcess",
+    "QueueNightlyIncentiveProcessEndpoint": "/Processing/QueueNightlyIncentiveProcess",
+    "QueuePayoutProcessEndpoint": "/Processing/QueuePayoutProcess",
+    "QueueReportingProcessEndpoint": "/Processing/QueueReportingRequest",
+    "QueueGoalDispersalProcessEndpoint": "/Processing/QueueGoalDispersalSyncProcess",
+    "QueueDayforceProcessEndpoint": "/Processing/QueueDayforceSyncProcess"
+  },
+  "FileUpdaterApiConfiguration": {
+    "Url": "https://localhost:56076",
+    "QueueIndexFieldMapper": "/api/FileUpdater/QueueIndexFieldMapper"
+  },
+  "TemenosNotificationApiConfiguration": {
+    "Url": "https://localhost:7076",
+    "QueueDealerAchNotification": "/api/QueueDealerAchNotification"
+  }
+}

LCU.API.Internal.Dispatcher/LCU.Api.Internal.Dispatcher/libman.json

@@ -0,0 +1,5 @@
+{
+  "version": "1.0",
+  "defaultProvider": "cdnjs",
+  "libraries": []
+}