Files
step3/api/QuestionsApi/Services/ExamSimulationStore.cs
T

466 lines
24 KiB
C#

using MongoDB.Bson;
using MongoDB.Driver;
using QuestionsApi.Models;
namespace QuestionsApi.Services;
public class ExamSimulationStore
{
public const int RequiredQuestions = 232;
public static readonly int[] BlockSizes = [20, 19, 19, 20, 19, 19, 20, 19, 19, 20, 19, 19];
private readonly IMongoCollection<ExamSimulation> _simulations;
private readonly IMongoCollection<Step3Outcome> _outcomes;
private readonly QuestionStore _questions;
private readonly QuestionAttemptStore _attempts;
private readonly AnswerStatisticsStore _statistics;
private readonly RatingStore _ratings;
private readonly ServiceMetrics _metrics;
public ExamSimulationStore(
MongoDatabaseProvider mongo,
QuestionStore questions,
QuestionAttemptStore attempts,
AnswerStatisticsStore statistics,
RatingStore ratings,
ServiceMetrics metrics)
{
_simulations = mongo.Database.GetCollection<ExamSimulation>("ExamSimulations");
_outcomes = mongo.Database.GetCollection<Step3Outcome>("Step3Outcomes");
_questions = questions;
_attempts = attempts;
_statistics = statistics;
_ratings = ratings;
_metrics = metrics;
_simulations.Indexes.CreateOne(new CreateIndexModel<ExamSimulation>(
Builders<ExamSimulation>.IndexKeys.Ascending(item => item.UserId).Ascending(item => item.State)));
_simulations.Indexes.CreateOne(new CreateIndexModel<ExamSimulation>(
Builders<ExamSimulation>.IndexKeys.Ascending(item => item.UserId).Ascending(item => item.State).Descending(item => item.CompletedUtc)));
_simulations.Indexes.CreateOne(new CreateIndexModel<ExamSimulation>(
Builders<ExamSimulation>.IndexKeys.Ascending(item => item.UserId).Descending(item => item.CreatedUtc)));
_simulations.Indexes.CreateOne(new CreateIndexModel<ExamSimulation>(
Builders<ExamSimulation>.IndexKeys.Ascending(item => item.UpdatedUtc)));
_outcomes.Indexes.CreateOne(new CreateIndexModel<Step3Outcome>(
Builders<Step3Outcome>.IndexKeys.Ascending(item => item.UserId),
new CreateIndexOptions { Unique = true }));
}
public async Task<ExamSimulationReadinessResponse> GetReadinessAsync(string userId)
{
var count = (await _questions.GetPublishedAsync()).Count(question => question.Choices.Count >= 5);
return new ExamSimulationReadinessResponse
{
Ready = count >= RequiredQuestions,
AvailableQuestions = count,
HasActiveSimulation = await FindActiveAsync(userId) is not null
};
}
public async Task<ExamSimulation> CreateAsync(string userId)
{
if (await FindActiveAsync(userId) is not null)
throw new InvalidOperationException("An active Step 3 simulation already exists.");
var bank = (await _questions.GetPublishedAsync()).Where(question => question.Choices.Count >= 5).ToList();
if (bank.Count < RequiredQuestions)
throw new InvalidOperationException($"At least {RequiredQuestions} published questions are required.");
var seen = (await _attempts.GetUserQuestionIdsAsync(userId)).ToHashSet(StringComparer.Ordinal);
var elos = await _ratings.GetQuestionElosAsync(bank.Select(question => question.ExternalId));
var targets = BuildTargets();
var selected = SelectForm(bank, seen, targets);
var items = new List<ExamSimulationItem>();
var position = 0;
for (var block = 0; block < BlockSizes.Length; block++)
{
for (var offset = 0; offset < BlockSizes[block]; offset++)
{
var question = selected[position];
items.Add(new ExamSimulationItem
{
QuestionExternalId = question.ExternalId,
BlockIndex = block,
Position = offset,
PresentedChoiceIds = SelectChoices(question),
QuestionElo = elos.GetValueOrDefault(question.ExternalId, 1000),
WasSeen = seen.Contains(question.ExternalId),
BlueprintCategory = BlueprintCategory(question)
});
position++;
}
}
var simulation = new ExamSimulation
{
UserId = userId,
Items = items,
BlueprintTargets = targets,
BlueprintActual = items.GroupBy(item => item.BlueprintCategory).ToDictionary(group => group.Key, group => group.Count())
};
await _simulations.InsertOneAsync(simulation);
return simulation;
}
public async Task<ExamSimulationStateResponse> BuildResultsPreviewAsync(string userId)
{
var bank = (await _questions.GetPublishedAsync()).Where(question => question.Choices.Count >= 5).ToList();
if (bank.Count < RequiredQuestions)
throw new InvalidOperationException($"At least {RequiredQuestions} published questions are required.");
var targets = BuildTargets();
var selected = SelectForm(bank, [], targets);
var items = selected.Select((question, index) => new ExamSimulationItem
{
QuestionExternalId = question.ExternalId,
BlockIndex = BlockIndexForPosition(index),
Position = PositionWithinBlock(index),
PresentedChoiceIds = SelectChoices(question),
QuestionElo = 1000,
WasSeen = false,
BlueprintCategory = BlueprintCategory(question)
}).ToList();
var questionMap = selected.ToDictionary(question => question.ExternalId, StringComparer.Ordinal);
var responses = items.Select((item, index) =>
{
var question = questionMap[item.QuestionExternalId];
var choiceId = index % 3 == 0
? item.PresentedChoiceIds.First(id => id != question.CorrectChoiceId)
: question.CorrectChoiceId;
return new ExamSimulationResponse
{
QuestionExternalId = item.QuestionExternalId,
ChoiceId = choiceId,
ElapsedSeconds = 75 + index % 45,
Flagged = index % 11 == 0
};
}).ToList();
var responseMap = responses.ToDictionary(item => item.QuestionExternalId, StringComparer.Ordinal);
return new ExamSimulationStateResponse
{
Id = "preview",
State = ExamSimulationStates.Completed,
CurrentBlock = BlockSizes.Length - 1,
BreakSecondsRemaining = 3300,
Responses = responses,
Score = ExamSimulationScoring.Calculate(items, responseMap, questionMap),
BlueprintTargets = targets,
BlueprintActual = items.GroupBy(item => item.BlueprintCategory).ToDictionary(group => group.Key, group => group.Count()),
ReviewQuestions = items.Select(item => QuestionResponse.FromModelWithChoices(
questionMap[item.QuestionExternalId], item.PresentedChoiceIds, item.QuestionElo)).ToList()
};
}
public async Task<ExamSimulation?> FindActiveAsync(string userId) =>
await _simulations.Find(item => item.UserId == userId &&
(item.State == ExamSimulationStates.BetweenBlocks || item.State == ExamSimulationStates.BlockActive))
.SortByDescending(item => item.UpdatedUtc).FirstOrDefaultAsync();
public async Task<ExamSimulation?> GetOwnedAsync(string id, string userId)
{
var simulation = await _simulations.Find(item => item.Id == id && item.UserId == userId).FirstOrDefaultAsync();
if (simulation is not null && simulation.State == ExamSimulationStates.BlockActive && simulation.BlockDeadlineUtc <= DateTime.UtcNow)
simulation = await SubmitBlockAsync(simulation, true);
return simulation;
}
public async Task<CompletedExamSimulationPage> GetCompletedAsync(string userId, int limit, DateTime? before)
{
limit = Math.Clamp(limit, 1, 50);
var filter = Builders<ExamSimulation>.Filter.Eq(item => item.UserId, userId) &
Builders<ExamSimulation>.Filter.Eq(item => item.State, ExamSimulationStates.Completed);
if (before is not null)
filter &= Builders<ExamSimulation>.Filter.Lt(item => item.CompletedUtc, before.Value);
var simulations = await _simulations.Find(filter).SortByDescending(item => item.CompletedUtc).Limit(limit + 1).ToListAsync();
var hasMore = simulations.Count > limit;
if (hasMore) simulations.RemoveAt(simulations.Count - 1);
return new CompletedExamSimulationPage
{
Items = simulations.Select(item => new CompletedExamSimulationSummary
{
Id = item.Id ?? "",
CompletedUtc = item.CompletedUtc ?? item.UpdatedUtc,
PredictedScore = item.Score?.PredictedScore ?? 0,
RangeLow = item.Score?.RangeLow ?? 0,
RangeHigh = item.Score?.RangeHigh ?? 0,
Correct = item.Score?.Correct ?? 0,
Total = item.Score?.Total ?? item.Items.Count,
SeenCount = item.Score?.SeenCount ?? 0
}).ToList(),
NextBefore = hasMore && simulations.Count > 0 ? simulations[^1].CompletedUtc : null
};
}
public async Task<ExamSimulationHistoryPage> GetHistoryAsync(string userId, int limit, DateTime? before)
{
limit = Math.Clamp(limit, 1, 50);
var filter = Builders<ExamSimulation>.Filter.Eq(item => item.UserId, userId);
if (before is not null)
filter &= Builders<ExamSimulation>.Filter.Lt(item => item.CreatedUtc, before.Value);
var simulations = await _simulations.Find(filter).SortByDescending(item => item.CreatedUtc).Limit(limit + 1).ToListAsync();
var hasMore = simulations.Count > limit;
if (hasMore) simulations.RemoveAt(simulations.Count - 1);
return new ExamSimulationHistoryPage
{
Items = simulations.Select(ToHistorySummary).ToList(),
NextBefore = hasMore && simulations.Count > 0 ? simulations[^1].CreatedUtc : null
};
}
public static ExamSimulationHistorySummary ToHistorySummary(ExamSimulation item) => new()
{
Id = item.Id ?? "",
State = item.State,
CreatedUtc = item.CreatedUtc,
UpdatedUtc = item.UpdatedUtc,
CompletedUtc = item.CompletedUtc,
CurrentBlock = item.CurrentBlock,
PredictedScore = item.Score?.PredictedScore,
RangeLow = item.Score?.RangeLow,
RangeHigh = item.Score?.RangeHigh,
Correct = item.Score?.Correct,
Total = item.Score?.Total ?? item.Items.Count
};
public async Task<ExamSimulation> StartNextBlockAsync(ExamSimulation simulation)
{
if (simulation.State != ExamSimulationStates.BetweenBlocks)
throw new InvalidOperationException("The simulation is not between blocks.");
if (simulation.CurrentBlock + 1 >= BlockSizes.Length)
throw new InvalidOperationException("All blocks are complete.");
simulation.CurrentBlock++;
simulation.State = ExamSimulationStates.BlockActive;
simulation.BlockDeadlineUtc = DateTime.UtcNow.AddMinutes(30);
simulation.UpdatedUtc = DateTime.UtcNow;
await ReplaceAsync(simulation);
return simulation;
}
public async Task<ExamSimulation> SaveResponseAsync(ExamSimulation simulation, SaveSimulationResponseRequest request)
{
if (simulation.State != ExamSimulationStates.BlockActive || simulation.BlockDeadlineUtc <= DateTime.UtcNow)
throw new InvalidOperationException("The active block has ended.");
var item = simulation.Items.FirstOrDefault(candidate => candidate.BlockIndex == simulation.CurrentBlock && candidate.QuestionExternalId == request.QuestionExternalId)
?? throw new ArgumentException("Question is not in the active block.");
if (!item.PresentedChoiceIds.Contains(request.ChoiceId, StringComparer.Ordinal) && request.ChoiceId != "")
throw new ArgumentException("Choice is not presented for this question.");
var response = simulation.Responses.FirstOrDefault(candidate => candidate.QuestionExternalId == request.QuestionExternalId);
if (response is null)
{
response = new ExamSimulationResponse { QuestionExternalId = request.QuestionExternalId };
simulation.Responses.Add(response);
}
response.ChoiceId = request.ChoiceId;
response.EliminatedChoiceIds = request.EliminatedChoiceIds.Where(item.PresentedChoiceIds.Contains).Distinct().ToList();
response.Flagged = request.Flagged;
response.ElapsedSeconds = Math.Clamp(request.ElapsedSeconds, 0, 1800);
response.UpdatedUtc = DateTime.UtcNow;
simulation.UpdatedUtc = DateTime.UtcNow;
await ReplaceAsync(simulation);
return simulation;
}
public async Task<ExamSimulation> SubmitBlockAsync(ExamSimulation simulation, bool expired = false)
{
if (simulation.State != ExamSimulationStates.BlockActive)
throw new InvalidOperationException("There is no active block to submit.");
if (!expired && simulation.BlockDeadlineUtc is not null)
simulation.BreakSecondsRemaining += Math.Max(0, (int)(simulation.BlockDeadlineUtc.Value - DateTime.UtcNow).TotalSeconds);
simulation.BlockDeadlineUtc = null;
simulation.UpdatedUtc = DateTime.UtcNow;
if (simulation.CurrentBlock + 1 < BlockSizes.Length)
{
simulation.State = ExamSimulationStates.BetweenBlocks;
await ReplaceAsync(simulation);
return simulation;
}
await CompleteAsync(simulation);
return simulation;
}
public async Task AbandonAsync(ExamSimulation simulation)
{
simulation.State = ExamSimulationStates.Abandoned;
simulation.BlockDeadlineUtc = null;
simulation.UpdatedUtc = DateTime.UtcNow;
await ReplaceAsync(simulation);
}
public async Task<ExamSimulationStateResponse> ToResponseAsync(ExamSimulation simulation)
{
var questionMap = await _questions.GetByExternalIdsAsync(simulation.Items.Select(item => item.QuestionExternalId));
var response = new ExamSimulationStateResponse
{
Id = simulation.Id ?? "",
State = simulation.State,
CurrentBlock = simulation.CurrentBlock,
BreakSecondsRemaining = simulation.BreakSecondsRemaining,
BlockDeadlineUtc = simulation.BlockDeadlineUtc,
Responses = simulation.Responses,
Score = simulation.Score,
BlueprintTargets = simulation.BlueprintTargets,
BlueprintActual = simulation.BlueprintActual
};
if (simulation.State == ExamSimulationStates.BlockActive)
{
response.Questions = simulation.Items.Where(item => item.BlockIndex == simulation.CurrentBlock)
.OrderBy(item => item.Position)
.Where(item => questionMap.ContainsKey(item.QuestionExternalId))
.Select(item => SafeQuestion(questionMap[item.QuestionExternalId], item.PresentedChoiceIds)).ToList();
}
if (simulation.State == ExamSimulationStates.Completed)
{
response.ReviewQuestions = simulation.Items.OrderBy(item => item.BlockIndex).ThenBy(item => item.Position)
.Where(item => questionMap.ContainsKey(item.QuestionExternalId))
.Select(item => QuestionResponse.FromModelWithChoices(questionMap[item.QuestionExternalId], item.PresentedChoiceIds, item.QuestionElo)).ToList();
}
return response;
}
public async Task<Step3Outcome?> GetOutcomeAsync(string userId) =>
await _outcomes.Find(item => item.UserId == userId).FirstOrDefaultAsync();
public async Task<Step3Outcome> UpsertOutcomeAsync(string userId, UpsertStep3OutcomeRequest request)
{
if (!request.Consent) throw new ArgumentException("Consent is required.");
if (request.OfficialScore is < 1 or > 300) throw new ArgumentException("Official score must be between 1 and 300.");
if (request.ExamDate.Date > DateTime.UtcNow.Date) throw new ArgumentException("Exam date cannot be in the future.");
var now = DateTime.UtcNow;
return await _outcomes.FindOneAndUpdateAsync(
item => item.UserId == userId,
Builders<Step3Outcome>.Update
.SetOnInsert(item => item.UserId, userId)
.SetOnInsert(item => item.ConsentUtc, now)
.Set(item => item.OfficialScore, request.OfficialScore)
.Set(item => item.ExamDate, request.ExamDate)
.Set(item => item.UpdatedUtc, now),
new FindOneAndUpdateOptions<Step3Outcome> { IsUpsert = true, ReturnDocument = ReturnDocument.After });
}
public Task DeleteOutcomeAsync(string userId) => _outcomes.DeleteOneAsync(item => item.UserId == userId);
private async Task CompleteAsync(ExamSimulation simulation)
{
var questionMap = await _questions.GetByExternalIdsAsync(simulation.Items.Select(item => item.QuestionExternalId));
var responses = simulation.Responses.ToDictionary(item => item.QuestionExternalId, StringComparer.Ordinal);
simulation.Score = ExamSimulationScoring.Calculate(simulation.Items, responses, questionMap);
simulation.State = ExamSimulationStates.Completed;
simulation.CompletedUtc = DateTime.UtcNow;
simulation.UpdatedUtc = DateTime.UtcNow;
await ReplaceAsync(simulation);
foreach (var item in simulation.Items.Where(item => !item.WasSeen))
{
if (!questionMap.TryGetValue(item.QuestionExternalId, out var question)) continue;
responses.TryGetValue(item.QuestionExternalId, out var response);
var choiceId = response?.ChoiceId ?? "";
var record = await _attempts.RecordFirstAttemptAsync(item.QuestionExternalId, simulation.UserId, choiceId, choiceId == question.CorrectChoiceId, choiceId == "", item.PresentedChoiceIds);
if (!record.Inserted) continue;
_metrics.AttemptCreated();
try { await _statistics.IncrementAsync(record.Attempt); } catch { _metrics.CounterFailure(); }
var owner = Guid.NewGuid().ToString("N");
var attemptId = record.Attempt.Id ?? "";
var shouldRate = false;
try
{
shouldRate = await _attempts.TryAcquireRatingLeaseAsync(attemptId, owner, TimeSpan.FromSeconds(30));
await _ratings.ApplyFirstResultAsync(simulation.UserId, question, record.Attempt.IsCorrect, shouldRate);
if (shouldRate) await _attempts.MarkRatingAppliedAsync(attemptId, owner);
}
catch
{
_metrics.CounterFailure();
if (shouldRate) await _attempts.ReleaseRatingLeaseAsync(attemptId, owner);
}
}
}
private Task ReplaceAsync(ExamSimulation simulation) =>
_simulations.ReplaceOneAsync(item => item.Id == simulation.Id && item.UserId == simulation.UserId, simulation);
private static int BlockIndexForPosition(int position)
{
var consumed = 0;
for (var block = 0; block < BlockSizes.Length; block++)
{
if (position < consumed + BlockSizes[block]) return block;
consumed += BlockSizes[block];
}
return BlockSizes.Length - 1;
}
private static int PositionWithinBlock(int position)
{
var consumed = 0;
foreach (var size in BlockSizes)
{
if (position < consumed + size) return position - consumed;
consumed += size;
}
return 0;
}
private static ExamSimulationQuestionResponse SafeQuestion(Question question, List<string> presentedIds) => new()
{
ExternalId = question.ExternalId,
Stem = question.Stem,
LeadIn = question.LeadIn,
Choices = presentedIds.Select(id => question.Choices.First(choice => choice.Id == id)).ToList(),
Discipline = question.Discipline,
System = question.System,
PerformanceCategory = question.PerformanceCategory,
Labs = question.Labs,
Media = question.Media
};
private static List<string> SelectChoices(Question question)
{
var distractors = question.Choices.Where(choice => choice.Id != question.CorrectChoiceId).OrderBy(_ => Random.Shared.Next()).Take(4).Select(choice => choice.Id).ToList();
distractors.Add(question.CorrectChoiceId);
return distractors.OrderBy(_ => Random.Shared.Next()).ToList();
}
private static List<Question> SelectForm(List<Question> bank, HashSet<string> seen, Dictionary<string, int> targets)
{
var remaining = bank.OrderBy(question => seen.Contains(question.ExternalId) ? 1 : 0).ThenBy(_ => Random.Shared.Next()).ToList();
var selected = new List<Question>();
foreach (var target in targets)
{
var matches = remaining.Where(question => BlueprintCategory(question) == target.Key).Take(target.Value).ToList();
selected.AddRange(matches);
foreach (var match in matches) remaining.Remove(match);
}
selected.AddRange(remaining.Take(RequiredQuestions - selected.Count));
return selected.OrderBy(_ => Random.Shared.Next()).ToList();
}
public static Dictionary<string, int> BuildTargets()
{
var weights = new Dictionary<string, double>
{
["Human Development"] = 2, ["Immune/Blood/Multisystem"] = 7, ["Behavioral Health"] = 5,
["Nervous System"] = 9, ["Skin"] = 5, ["Musculoskeletal"] = 6, ["Cardiovascular"] = 10,
["Respiratory"] = 9, ["Gastrointestinal"] = 7, ["Renal/Urinary/Male"] = 5,
["Pregnancy/Female/Breast"] = 8, ["Endocrine"] = 6, ["Biostatistics/Population Health"] = 12,
["Social Sciences"] = 8
};
var total = weights.Values.Sum();
var exact = weights.ToDictionary(pair => pair.Key, pair => pair.Value / total * RequiredQuestions);
var result = exact.ToDictionary(pair => pair.Key, pair => (int)Math.Floor(pair.Value));
foreach (var key in exact.OrderByDescending(pair => pair.Value - Math.Floor(pair.Value)).Take(RequiredQuestions - result.Values.Sum()).Select(pair => pair.Key)) result[key]++;
return result;
}
public static string BlueprintCategory(Question question)
{
var value = $"{question.System} {question.PerformanceCategory} {question.Discipline}".ToLowerInvariant();
if (value.Contains("biostat") || value.Contains("epidemi") || value.Contains("population")) return "Biostatistics/Population Health";
if (value.Contains("ethic") || value.Contains("communication") || value.Contains("professional") || value.Contains("patient safety")) return "Social Sciences";
if (value.Contains("pregnan") || value.Contains("obstetric") || value.Contains("female") || value.Contains("breast") || value.Contains("gyne")) return "Pregnancy/Female/Breast";
if (value.Contains("renal") || value.Contains("urinary") || value.Contains("male") || value.Contains("genitourinary")) return "Renal/Urinary/Male";
if (value.Contains("cardio")) return "Cardiovascular";
if (value.Contains("resp") || value.Contains("pulmonary")) return "Respiratory";
if (value.Contains("gastro") || value.Contains("hepato")) return "Gastrointestinal";
if (value.Contains("neuro") || value.Contains("special sense")) return "Nervous System";
if (value.Contains("musculo") || value.Contains("orthop")) return "Musculoskeletal";
if (value.Contains("skin") || value.Contains("derm")) return "Skin";
if (value.Contains("behavior") || value.Contains("psychiatr")) return "Behavioral Health";
if (value.Contains("endocrine") || value.Contains("diabetes") || value.Contains("metabolism")) return "Endocrine";
if (value.Contains("immune") || value.Contains("hemat") || value.Contains("blood") || value.Contains("multisystem")) return "Immune/Blood/Multisystem";
return "Human Development";
}
}