using MongoDB.Bson; using MongoDB.Driver; using QuestionsApi.Models; namespace QuestionsApi.Services; public class ExamSimulationStore { public const int RequiredQuestions = 232; public static readonly int[] BlockSizes = [20, 19, 19, 20, 19, 19, 20, 19, 19, 20, 19, 19]; private readonly IMongoCollection _simulations; private readonly IMongoCollection _outcomes; private readonly QuestionStore _questions; private readonly QuestionAttemptStore _attempts; private readonly AnswerStatisticsStore _statistics; private readonly RatingStore _ratings; private readonly ServiceMetrics _metrics; public ExamSimulationStore( MongoDatabaseProvider mongo, QuestionStore questions, QuestionAttemptStore attempts, AnswerStatisticsStore statistics, RatingStore ratings, ServiceMetrics metrics) { _simulations = mongo.Database.GetCollection("ExamSimulations"); _outcomes = mongo.Database.GetCollection("Step3Outcomes"); _questions = questions; _attempts = attempts; _statistics = statistics; _ratings = ratings; _metrics = metrics; _simulations.Indexes.CreateOne(new CreateIndexModel( Builders.IndexKeys.Ascending(item => item.UserId).Ascending(item => item.State))); _simulations.Indexes.CreateOne(new CreateIndexModel( Builders.IndexKeys.Ascending(item => item.UserId).Ascending(item => item.State).Descending(item => item.CompletedUtc))); _simulations.Indexes.CreateOne(new CreateIndexModel( Builders.IndexKeys.Ascending(item => item.UserId).Descending(item => item.CreatedUtc))); _simulations.Indexes.CreateOne(new CreateIndexModel( Builders.IndexKeys.Ascending(item => item.UpdatedUtc))); _outcomes.Indexes.CreateOne(new CreateIndexModel( Builders.IndexKeys.Ascending(item => item.UserId), new CreateIndexOptions { Unique = true })); } public async Task GetReadinessAsync(string userId) { var count = (await _questions.GetPublishedAsync()).Count(question => question.Choices.Count >= 5); return new ExamSimulationReadinessResponse { Ready = count >= RequiredQuestions, AvailableQuestions = count, HasActiveSimulation = await FindActiveAsync(userId) is not null }; } public async Task CreateAsync(string userId) { if (await FindActiveAsync(userId) is not null) throw new InvalidOperationException("An active Step 3 simulation already exists."); var bank = (await _questions.GetPublishedAsync()).Where(question => question.Choices.Count >= 5).ToList(); if (bank.Count < RequiredQuestions) throw new InvalidOperationException($"At least {RequiredQuestions} published questions are required."); var seen = (await _attempts.GetUserQuestionIdsAsync(userId)).ToHashSet(StringComparer.Ordinal); var elos = await _ratings.GetQuestionElosAsync(bank.Select(question => question.ExternalId)); var targets = BuildTargets(); var selected = SelectForm(bank, seen, targets); var items = new List(); var position = 0; for (var block = 0; block < BlockSizes.Length; block++) { for (var offset = 0; offset < BlockSizes[block]; offset++) { var question = selected[position]; items.Add(new ExamSimulationItem { QuestionExternalId = question.ExternalId, BlockIndex = block, Position = offset, PresentedChoiceIds = SelectChoices(question), QuestionElo = elos.GetValueOrDefault(question.ExternalId, 1000), WasSeen = seen.Contains(question.ExternalId), BlueprintCategory = BlueprintCategory(question) }); position++; } } var simulation = new ExamSimulation { UserId = userId, Items = items, BlueprintTargets = targets, BlueprintActual = items.GroupBy(item => item.BlueprintCategory).ToDictionary(group => group.Key, group => group.Count()) }; await _simulations.InsertOneAsync(simulation); return simulation; } public async Task BuildResultsPreviewAsync(string userId) => await ToResponseAsync(await BuildSampleAsync(userId, false)); public Task CreateSampleCompletedAsync(string userId) => BuildSampleAsync(userId, true); private async Task BuildSampleAsync(string userId, bool persist) { var bank = (await _questions.GetPublishedAsync()).Where(question => question.Choices.Count >= 5).ToList(); if (bank.Count < RequiredQuestions) throw new InvalidOperationException($"At least {RequiredQuestions} published questions are required."); var targets = BuildTargets(); var selected = SelectForm(bank, [], targets); var items = selected.Select((question, index) => new ExamSimulationItem { QuestionExternalId = question.ExternalId, BlockIndex = BlockIndexForPosition(index), Position = PositionWithinBlock(index), PresentedChoiceIds = SelectChoices(question), QuestionElo = 1000, WasSeen = false, BlueprintCategory = BlueprintCategory(question) }).ToList(); var questionMap = selected.ToDictionary(question => question.ExternalId, StringComparer.Ordinal); var responses = items.Select((item, index) => { var question = questionMap[item.QuestionExternalId]; var choiceId = index % 3 == 0 ? item.PresentedChoiceIds.First(id => id != question.CorrectChoiceId) : question.CorrectChoiceId; return new ExamSimulationResponse { QuestionExternalId = item.QuestionExternalId, ChoiceId = choiceId, ElapsedSeconds = 75 + index % 45, Flagged = index % 11 == 0 }; }).ToList(); var responseMap = responses.ToDictionary(item => item.QuestionExternalId, StringComparer.Ordinal); var now = DateTime.UtcNow; var simulation = new ExamSimulation { UserId = userId, State = ExamSimulationStates.Completed, IsSample = true, CurrentBlock = BlockSizes.Length - 1, BreakSecondsRemaining = 3300, Items = items, Responses = responses, Score = ExamSimulationScoring.Calculate(items, responseMap, questionMap), CreatedUtc = now, UpdatedUtc = now, CompletedUtc = now, BlueprintTargets = targets, BlueprintActual = items.GroupBy(item => item.BlueprintCategory).ToDictionary(group => group.Key, group => group.Count()) }; if (persist) await _simulations.InsertOneAsync(simulation); return simulation; } public async Task FindActiveAsync(string userId) => await _simulations.Find(item => item.UserId == userId && (item.State == ExamSimulationStates.BetweenBlocks || item.State == ExamSimulationStates.BlockActive)) .SortByDescending(item => item.UpdatedUtc).FirstOrDefaultAsync(); public async Task GetOwnedAsync(string id, string userId) { var simulation = await _simulations.Find(item => item.Id == id && item.UserId == userId).FirstOrDefaultAsync(); if (simulation is not null && simulation.State == ExamSimulationStates.BlockActive && simulation.BlockDeadlineUtc <= DateTime.UtcNow) simulation = await SubmitBlockAsync(simulation, true); return simulation; } public async Task GetCompletedAsync(string userId, int limit, DateTime? before) { limit = Math.Clamp(limit, 1, 50); var filter = Builders.Filter.Eq(item => item.UserId, userId) & Builders.Filter.Eq(item => item.State, ExamSimulationStates.Completed); if (before is not null) filter &= Builders.Filter.Lt(item => item.CompletedUtc, before.Value); var simulations = await _simulations.Find(filter).SortByDescending(item => item.CompletedUtc).Limit(limit + 1).ToListAsync(); var hasMore = simulations.Count > limit; if (hasMore) simulations.RemoveAt(simulations.Count - 1); return new CompletedExamSimulationPage { Items = simulations.Select(item => new CompletedExamSimulationSummary { Id = item.Id ?? "", CompletedUtc = item.CompletedUtc ?? item.UpdatedUtc, PredictedScore = item.Score?.PredictedScore ?? 0, RangeLow = item.Score?.RangeLow ?? 0, RangeHigh = item.Score?.RangeHigh ?? 0, Correct = item.Score?.Correct ?? 0, Total = item.Score?.Total ?? item.Items.Count, SeenCount = item.Score?.SeenCount ?? 0 }).ToList(), NextBefore = hasMore && simulations.Count > 0 ? simulations[^1].CompletedUtc : null }; } public async Task GetHistoryAsync(string userId, int limit, DateTime? before) { limit = Math.Clamp(limit, 1, 50); var filter = Builders.Filter.Eq(item => item.UserId, userId); if (before is not null) filter &= Builders.Filter.Lt(item => item.CreatedUtc, before.Value); var simulations = await _simulations.Find(filter).SortByDescending(item => item.CreatedUtc).Limit(limit + 1).ToListAsync(); var hasMore = simulations.Count > limit; if (hasMore) simulations.RemoveAt(simulations.Count - 1); return new ExamSimulationHistoryPage { Items = simulations.Select(ToHistorySummary).ToList(), NextBefore = hasMore && simulations.Count > 0 ? simulations[^1].CreatedUtc : null }; } public static ExamSimulationHistorySummary ToHistorySummary(ExamSimulation item) => new() { Id = item.Id ?? "", State = item.State, IsSample = item.IsSample, CreatedUtc = item.CreatedUtc, UpdatedUtc = item.UpdatedUtc, CompletedUtc = item.CompletedUtc, CurrentBlock = item.CurrentBlock, PredictedScore = item.Score?.PredictedScore, RangeLow = item.Score?.RangeLow, RangeHigh = item.Score?.RangeHigh, Correct = item.Score?.Correct, Total = item.Score?.Total ?? item.Items.Count }; public async Task StartNextBlockAsync(ExamSimulation simulation) { if (simulation.State != ExamSimulationStates.BetweenBlocks) throw new InvalidOperationException("The simulation is not between blocks."); if (simulation.CurrentBlock + 1 >= BlockSizes.Length) throw new InvalidOperationException("All blocks are complete."); simulation.CurrentBlock++; simulation.State = ExamSimulationStates.BlockActive; simulation.BlockDeadlineUtc = DateTime.UtcNow.AddMinutes(30); simulation.UpdatedUtc = DateTime.UtcNow; await ReplaceAsync(simulation); return simulation; } public async Task SaveResponseAsync(ExamSimulation simulation, SaveSimulationResponseRequest request) { if (simulation.State != ExamSimulationStates.BlockActive || simulation.BlockDeadlineUtc <= DateTime.UtcNow) throw new InvalidOperationException("The active block has ended."); var item = simulation.Items.FirstOrDefault(candidate => candidate.BlockIndex == simulation.CurrentBlock && candidate.QuestionExternalId == request.QuestionExternalId) ?? throw new ArgumentException("Question is not in the active block."); if (!item.PresentedChoiceIds.Contains(request.ChoiceId, StringComparer.Ordinal) && request.ChoiceId != "") throw new ArgumentException("Choice is not presented for this question."); var response = simulation.Responses.FirstOrDefault(candidate => candidate.QuestionExternalId == request.QuestionExternalId); if (response is null) { response = new ExamSimulationResponse { QuestionExternalId = request.QuestionExternalId }; simulation.Responses.Add(response); } response.ChoiceId = request.ChoiceId; response.EliminatedChoiceIds = request.EliminatedChoiceIds.Where(item.PresentedChoiceIds.Contains).Distinct().ToList(); response.Flagged = request.Flagged; response.ElapsedSeconds = Math.Clamp(request.ElapsedSeconds, 0, 1800); response.UpdatedUtc = DateTime.UtcNow; simulation.UpdatedUtc = DateTime.UtcNow; await ReplaceAsync(simulation); return simulation; } public async Task SubmitBlockAsync(ExamSimulation simulation, bool expired = false) { if (simulation.State != ExamSimulationStates.BlockActive) throw new InvalidOperationException("There is no active block to submit."); if (!expired && simulation.BlockDeadlineUtc is not null) simulation.BreakSecondsRemaining += Math.Max(0, (int)(simulation.BlockDeadlineUtc.Value - DateTime.UtcNow).TotalSeconds); simulation.BlockDeadlineUtc = null; simulation.UpdatedUtc = DateTime.UtcNow; if (simulation.CurrentBlock + 1 < BlockSizes.Length) { simulation.State = ExamSimulationStates.BetweenBlocks; await ReplaceAsync(simulation); return simulation; } await CompleteAsync(simulation); return simulation; } public async Task AbandonAsync(ExamSimulation simulation) { simulation.State = ExamSimulationStates.Abandoned; simulation.BlockDeadlineUtc = null; simulation.UpdatedUtc = DateTime.UtcNow; await ReplaceAsync(simulation); } public async Task ToResponseAsync(ExamSimulation simulation) { var questionMap = await _questions.GetByExternalIdsAsync(simulation.Items.Select(item => item.QuestionExternalId)); var response = new ExamSimulationStateResponse { Id = simulation.Id ?? "", State = simulation.State, CurrentBlock = simulation.CurrentBlock, BreakSecondsRemaining = simulation.BreakSecondsRemaining, BlockDeadlineUtc = simulation.BlockDeadlineUtc, Responses = simulation.Responses, Score = simulation.Score, BlueprintTargets = simulation.BlueprintTargets, BlueprintActual = simulation.BlueprintActual }; if (simulation.State == ExamSimulationStates.BlockActive) { response.Questions = simulation.Items.Where(item => item.BlockIndex == simulation.CurrentBlock) .OrderBy(item => item.Position) .Where(item => questionMap.ContainsKey(item.QuestionExternalId)) .Select(item => SafeQuestion(questionMap[item.QuestionExternalId], item.PresentedChoiceIds)).ToList(); } if (simulation.State == ExamSimulationStates.Completed) { response.ReviewQuestions = simulation.Items.OrderBy(item => item.BlockIndex).ThenBy(item => item.Position) .Where(item => questionMap.ContainsKey(item.QuestionExternalId)) .Select(item => QuestionResponse.FromModelWithChoices(questionMap[item.QuestionExternalId], item.PresentedChoiceIds, item.QuestionElo)).ToList(); } return response; } public async Task GetOutcomeAsync(string userId) => await _outcomes.Find(item => item.UserId == userId).FirstOrDefaultAsync(); public async Task UpsertOutcomeAsync(string userId, UpsertStep3OutcomeRequest request) { if (!request.Consent) throw new ArgumentException("Consent is required."); if (request.OfficialScore is < 1 or > 300) throw new ArgumentException("Official score must be between 1 and 300."); if (request.ExamDate.Date > DateTime.UtcNow.Date) throw new ArgumentException("Exam date cannot be in the future."); var now = DateTime.UtcNow; return await _outcomes.FindOneAndUpdateAsync( item => item.UserId == userId, Builders.Update .SetOnInsert(item => item.UserId, userId) .SetOnInsert(item => item.ConsentUtc, now) .Set(item => item.OfficialScore, request.OfficialScore) .Set(item => item.ExamDate, request.ExamDate) .Set(item => item.UpdatedUtc, now), new FindOneAndUpdateOptions { IsUpsert = true, ReturnDocument = ReturnDocument.After }); } public Task DeleteOutcomeAsync(string userId) => _outcomes.DeleteOneAsync(item => item.UserId == userId); private async Task CompleteAsync(ExamSimulation simulation) { var questionMap = await _questions.GetByExternalIdsAsync(simulation.Items.Select(item => item.QuestionExternalId)); var responses = simulation.Responses.ToDictionary(item => item.QuestionExternalId, StringComparer.Ordinal); simulation.Score = ExamSimulationScoring.Calculate(simulation.Items, responses, questionMap); simulation.State = ExamSimulationStates.Completed; simulation.CompletedUtc = DateTime.UtcNow; simulation.UpdatedUtc = DateTime.UtcNow; await ReplaceAsync(simulation); foreach (var item in simulation.Items.Where(item => !item.WasSeen)) { if (!questionMap.TryGetValue(item.QuestionExternalId, out var question)) continue; responses.TryGetValue(item.QuestionExternalId, out var response); var choiceId = response?.ChoiceId ?? ""; var record = await _attempts.RecordFirstAttemptAsync(item.QuestionExternalId, simulation.UserId, choiceId, choiceId == question.CorrectChoiceId, choiceId == "", item.PresentedChoiceIds); if (!record.Inserted) continue; _metrics.AttemptCreated(); try { await _statistics.IncrementAsync(record.Attempt); } catch { _metrics.CounterFailure(); } var owner = Guid.NewGuid().ToString("N"); var attemptId = record.Attempt.Id ?? ""; var shouldRate = false; try { shouldRate = await _attempts.TryAcquireRatingLeaseAsync(attemptId, owner, TimeSpan.FromSeconds(30)); await _ratings.ApplyFirstResultAsync(simulation.UserId, question, record.Attempt.IsCorrect, shouldRate); if (shouldRate) await _attempts.MarkRatingAppliedAsync(attemptId, owner); } catch { _metrics.CounterFailure(); if (shouldRate) await _attempts.ReleaseRatingLeaseAsync(attemptId, owner); } } } private Task ReplaceAsync(ExamSimulation simulation) => _simulations.ReplaceOneAsync(item => item.Id == simulation.Id && item.UserId == simulation.UserId, simulation); private static int BlockIndexForPosition(int position) { var consumed = 0; for (var block = 0; block < BlockSizes.Length; block++) { if (position < consumed + BlockSizes[block]) return block; consumed += BlockSizes[block]; } return BlockSizes.Length - 1; } private static int PositionWithinBlock(int position) { var consumed = 0; foreach (var size in BlockSizes) { if (position < consumed + size) return position - consumed; consumed += size; } return 0; } private static ExamSimulationQuestionResponse SafeQuestion(Question question, List presentedIds) => new() { ExternalId = question.ExternalId, Stem = question.Stem, LeadIn = question.LeadIn, Choices = presentedIds.Select(id => question.Choices.First(choice => choice.Id == id)).ToList(), Discipline = question.Discipline, System = question.System, PerformanceCategory = question.PerformanceCategory, Labs = question.Labs, Media = question.Media }; private static List SelectChoices(Question question) { var distractors = question.Choices.Where(choice => choice.Id != question.CorrectChoiceId).OrderBy(_ => Random.Shared.Next()).Take(4).Select(choice => choice.Id).ToList(); distractors.Add(question.CorrectChoiceId); return distractors.OrderBy(_ => Random.Shared.Next()).ToList(); } private static List SelectForm(List bank, HashSet seen, Dictionary targets) { var remaining = bank.OrderBy(question => seen.Contains(question.ExternalId) ? 1 : 0).ThenBy(_ => Random.Shared.Next()).ToList(); var selected = new List(); foreach (var target in targets) { var matches = remaining.Where(question => BlueprintCategory(question) == target.Key).Take(target.Value).ToList(); selected.AddRange(matches); foreach (var match in matches) remaining.Remove(match); } selected.AddRange(remaining.Take(RequiredQuestions - selected.Count)); return selected.OrderBy(_ => Random.Shared.Next()).ToList(); } public static Dictionary BuildTargets() { var weights = new Dictionary { ["Human Development"] = 2, ["Immune/Blood/Multisystem"] = 7, ["Behavioral Health"] = 5, ["Nervous System"] = 9, ["Skin"] = 5, ["Musculoskeletal"] = 6, ["Cardiovascular"] = 10, ["Respiratory"] = 9, ["Gastrointestinal"] = 7, ["Renal/Urinary/Male"] = 5, ["Pregnancy/Female/Breast"] = 8, ["Endocrine"] = 6, ["Biostatistics/Population Health"] = 12, ["Social Sciences"] = 8 }; var total = weights.Values.Sum(); var exact = weights.ToDictionary(pair => pair.Key, pair => pair.Value / total * RequiredQuestions); var result = exact.ToDictionary(pair => pair.Key, pair => (int)Math.Floor(pair.Value)); foreach (var key in exact.OrderByDescending(pair => pair.Value - Math.Floor(pair.Value)).Take(RequiredQuestions - result.Values.Sum()).Select(pair => pair.Key)) result[key]++; return result; } public static string BlueprintCategory(Question question) { var value = $"{question.System} {question.PerformanceCategory} {question.Discipline}".ToLowerInvariant(); if (value.Contains("biostat") || value.Contains("epidemi") || value.Contains("population")) return "Biostatistics/Population Health"; if (value.Contains("ethic") || value.Contains("communication") || value.Contains("professional") || value.Contains("patient safety")) return "Social Sciences"; if (value.Contains("pregnan") || value.Contains("obstetric") || value.Contains("female") || value.Contains("breast") || value.Contains("gyne")) return "Pregnancy/Female/Breast"; if (value.Contains("renal") || value.Contains("urinary") || value.Contains("male") || value.Contains("genitourinary")) return "Renal/Urinary/Male"; if (value.Contains("cardio")) return "Cardiovascular"; if (value.Contains("resp") || value.Contains("pulmonary")) return "Respiratory"; if (value.Contains("gastro") || value.Contains("hepato")) return "Gastrointestinal"; if (value.Contains("neuro") || value.Contains("special sense")) return "Nervous System"; if (value.Contains("musculo") || value.Contains("orthop")) return "Musculoskeletal"; if (value.Contains("skin") || value.Contains("derm")) return "Skin"; if (value.Contains("behavior") || value.Contains("psychiatr")) return "Behavioral Health"; if (value.Contains("endocrine") || value.Contains("diabetes") || value.Contains("metabolism")) return "Endocrine"; if (value.Contains("immune") || value.Contains("hemat") || value.Contains("blood") || value.Contains("multisystem")) return "Immune/Blood/Multisystem"; return "Human Development"; } }