Fix exam simulation score prediction

2026-07-30 16:00:29 -05:00
parent ae6b37eb7d
commit 778da32e73
9 changed files with 282 additions and 57 deletions
@@ -91,6 +91,7 @@ public class ExamSimulationStore
                    Position = offset,
                    PresentedChoiceIds = SelectChoices(question, choicePercentages.GetValueOrDefault(question.ExternalId)),
                    QuestionElo = elos.GetValueOrDefault(question.ExternalId, 1000),
+                    QuestionDifficulty = question.Difficulty,
                    WasSeen = seen.Contains(question.ExternalId),
                    BlueprintCategory = BlueprintCategory(question)
                });
@@ -128,6 +129,7 @@ public class ExamSimulationStore
            Position = PositionWithinBlock(index),
            PresentedChoiceIds = SelectChoices(question, choicePercentages.GetValueOrDefault(question.ExternalId)),
            QuestionElo = 1000,
+            QuestionDifficulty = question.Difficulty,
            WasSeen = false,
            BlueprintCategory = BlueprintCategory(question)
        }).ToList();
@@ -173,15 +175,22 @@ public class ExamSimulationStore
            (item.State == ExamSimulationStates.BetweenBlocks || item.State == ExamSimulationStates.BlockActive))
            .SortByDescending(item => item.UpdatedUtc).FirstOrDefaultAsync();

-    public async Task<ExamSimulationScore?> GetLatestCompletedScoreAsync(string userId) =>
-        (await _simulations.Find(item => item.UserId == userId && item.State == ExamSimulationStates.Completed && item.Score != null)
-            .SortByDescending(item => item.CompletedUtc).FirstOrDefaultAsync())?.Score;
+    public async Task<ExamSimulationScore?> GetLatestCompletedScoreAsync(string userId)
+    {
+        var simulation = await _simulations
+            .Find(item => item.UserId == userId && item.State == ExamSimulationStates.Completed && item.Score != null)
+            .SortByDescending(item => item.CompletedUtc)
+            .FirstOrDefaultAsync();
+        return simulation is null ? null : (await UpgradeScoringModelAsync(simulation)).Score;
+    }

    public async Task<ExamSimulation?> GetOwnedAsync(string id, string userId)
    {
        var simulation = await _simulations.Find(item => item.Id == id && item.UserId == userId).FirstOrDefaultAsync();
        if (simulation is not null && simulation.State == ExamSimulationStates.BlockActive && simulation.BlockDeadlineUtc <= DateTime.UtcNow)
            simulation = await SubmitBlockAsync(simulation, true);
+        if (simulation is not null)
+            simulation = await UpgradeScoringModelAsync(simulation);
        return simulation;
    }

@@ -195,6 +204,7 @@ public class ExamSimulationStore
        var simulations = await _simulations.Find(filter).SortByDescending(item => item.CompletedUtc).Limit(limit + 1).ToListAsync();
        var hasMore = simulations.Count > limit;
        if (hasMore) simulations.RemoveAt(simulations.Count - 1);
+        await UpgradeScoringModelsAsync(simulations);
        return new CompletedExamSimulationPage
        {
            Items = simulations.Select(item => new CompletedExamSimulationSummary
@@ -261,6 +271,7 @@ public class ExamSimulationStore
        var simulations = await _simulations.Find(filter).Sort(sortDefinition).Limit(limit + 1).ToListAsync();
        var hasMore = simulations.Count > limit;
        if (hasMore) simulations.RemoveAt(simulations.Count - 1);
+        await UpgradeScoringModelsAsync(simulations.Where(item => item.State == ExamSimulationStates.Completed));
        var last = simulations.LastOrDefault();
        return new ExamSimulationHistoryPage
        {
@@ -442,7 +453,9 @@ public class ExamSimulationStore
    {
        var questionMap = await _questions.GetByExternalIdsAsync(simulation.Items.Select(item => item.QuestionExternalId));
        var responses = simulation.Responses.ToDictionary(item => item.QuestionExternalId, StringComparer.Ordinal);
+        SnapshotDifficulties(simulation, questionMap);
        simulation.Score = ExamSimulationScoring.Calculate(simulation.Items, responses, questionMap);
+        simulation.ModelVersion = ExamSimulationScoring.ModelVersion;
        simulation.State = ExamSimulationStates.Completed;
        simulation.CompletedUtc = DateTime.UtcNow;
        simulation.UpdatedUtc = DateTime.UtcNow;
@@ -480,6 +493,71 @@ public class ExamSimulationStore
    private Task ReplaceAsync(ExamSimulation simulation) =>
        _simulations.ReplaceOneAsync(item => item.Id == simulation.Id && item.UserId == simulation.UserId, simulation);

+    public async Task<ExamSimulationBackfillResult> BackfillScoringModelAsync(CancellationToken cancellationToken = default)
+    {
+        var filter = Builders<ExamSimulation>.Filter.Eq(item => item.State, ExamSimulationStates.Completed) &
+            Builders<ExamSimulation>.Filter.Ne(item => item.ModelVersion, ExamSimulationScoring.ModelVersion);
+        var simulations = await _simulations.Find(filter).ToListAsync(cancellationToken);
+        var missingQuestionIds = new List<string>();
+        var updated = 0;
+        foreach (var simulation in simulations)
+        {
+            cancellationToken.ThrowIfCancellationRequested();
+            var questionMap = await _questions.GetByExternalIdsAsync(simulation.Items.Select(item => item.QuestionExternalId));
+            missingQuestionIds.AddRange(simulation.Items
+                .Where(item => !questionMap.ContainsKey(item.QuestionExternalId))
+                .Select(item => $"{simulation.Id}:{item.QuestionExternalId}"));
+            SnapshotDifficulties(simulation, questionMap);
+            var responses = simulation.Responses.ToDictionary(item => item.QuestionExternalId, StringComparer.Ordinal);
+            simulation.Score = ExamSimulationScoring.Calculate(simulation.Items, responses, questionMap);
+            simulation.ModelVersion = ExamSimulationScoring.ModelVersion;
+            simulation.UpdatedUtc = DateTime.UtcNow;
+            await ReplaceAsync(simulation);
+            updated++;
+        }
+        return new ExamSimulationBackfillResult(updated, missingQuestionIds);
+    }
+
+    private async Task UpgradeScoringModelsAsync(IEnumerable<ExamSimulation> simulations)
+    {
+        foreach (var simulation in simulations)
+            await UpgradeScoringModelAsync(simulation);
+    }
+
+    private async Task<ExamSimulation> UpgradeScoringModelAsync(ExamSimulation simulation)
+    {
+        var needsDifficultySnapshot = simulation.Items.Any(item => item.QuestionDifficulty is < 1 or > 5);
+        var needsScoreUpgrade = simulation.State == ExamSimulationStates.Completed &&
+            simulation.ModelVersion != ExamSimulationScoring.ModelVersion;
+        if (!needsDifficultySnapshot && !needsScoreUpgrade)
+            return simulation;
+
+        var questionMap = await _questions.GetByExternalIdsAsync(simulation.Items.Select(item => item.QuestionExternalId));
+        SnapshotDifficulties(simulation, questionMap);
+        if (simulation.State == ExamSimulationStates.Completed)
+        {
+            var responses = simulation.Responses.ToDictionary(item => item.QuestionExternalId, StringComparer.Ordinal);
+            simulation.Score = ExamSimulationScoring.Calculate(simulation.Items, responses, questionMap);
+        }
+        simulation.ModelVersion = ExamSimulationScoring.ModelVersion;
+        simulation.UpdatedUtc = DateTime.UtcNow;
+        await ReplaceAsync(simulation);
+        return simulation;
+    }
+
+    private static void SnapshotDifficulties(
+        ExamSimulation simulation,
+        IReadOnlyDictionary<string, Question> questions)
+    {
+        foreach (var item in simulation.Items)
+        {
+            questions.TryGetValue(item.QuestionExternalId, out var question);
+            item.QuestionDifficulty = ExamSimulationScoring.ResolveDifficulty(item, question);
+        }
+    }
+
+    public sealed record ExamSimulationBackfillResult(int UpdatedCount, IReadOnlyList<string> MissingQuestionIds);
+
    private static int BlockIndexForPosition(int position)
    {
        var consumed = 0;