From 74398133f775d625bc5d36a835f29c11edb9f33b Mon Sep 17 00:00:00 2001 From: hz Date: Sun, 26 Jul 2026 14:46:45 -0500 Subject: [PATCH] Exclude sample exam simulations from the dashboard's readiness score The SUPER-role "preview results" QA tooling seeds a completed ExamSimulation with scripted (not real) answers, flagged IsSample. The dashboard's readiness lookup picked whichever completed simulation was most recent regardless of that flag, so running the QA preview on an account could override its real Exam Readiness card with a synthetic pass probability. Recent-activity/weekly-activity queries got the same exclusion since they read from the same collection for the same reason. Co-Authored-By: Claude Sonnet 5 --- api/QuestionsApi.Tests/DashboardTests.cs | 31 +++++++++++++++++++++ api/QuestionsApi/Services/DashboardStore.cs | 17 +++++++---- 2 files changed, 43 insertions(+), 5 deletions(-) diff --git a/api/QuestionsApi.Tests/DashboardTests.cs b/api/QuestionsApi.Tests/DashboardTests.cs index 7765d42..a3ae6f1 100644 --- a/api/QuestionsApi.Tests/DashboardTests.cs +++ b/api/QuestionsApi.Tests/DashboardTests.cs @@ -199,4 +199,35 @@ public sealed class DashboardTests { Assert.Equal(expected, DashboardStore.ReadinessLabel(passProbability)); } + + [Fact] + public void SelectLatestReadinessScoreSkipsSampleSimulationsEvenWhenMostRecent() + { + var today = new DateTime(2026, 7, 25, 0, 0, 0, DateTimeKind.Utc); + var simulations = new[] + { + CompletedSimulation(isSample: true, completedUtc: today, passProbability: 0.98), + CompletedSimulation(isSample: false, completedUtc: today.AddDays(-3), passProbability: 0.12) + }; + + var score = DashboardStore.SelectLatestReadinessScore(simulations); + + Assert.NotNull(score); + Assert.Equal(0.12, score.PassProbability); + } + + [Fact] + public void SelectLatestReadinessScoreReturnsNullWhenOnlySamplesExist() + { + var simulations = new[] { CompletedSimulation(isSample: true, completedUtc: DateTime.UtcNow, passProbability: 0.98) }; + + Assert.Null(DashboardStore.SelectLatestReadinessScore(simulations)); + } + + private static ExamSimulation CompletedSimulation(bool isSample, DateTime completedUtc, double passProbability) => new() + { + IsSample = isSample, + CompletedUtc = completedUtc, + Score = new ExamSimulationScore { PassProbability = passProbability } + }; } diff --git a/api/QuestionsApi/Services/DashboardStore.cs b/api/QuestionsApi/Services/DashboardStore.cs index f823a7f..3cb77de 100644 --- a/api/QuestionsApi/Services/DashboardStore.cs +++ b/api/QuestionsApi/Services/DashboardStore.cs @@ -79,10 +79,10 @@ public sealed class DashboardStore var ccsAllAttemptsTask = _ccsAttempts.Find(value => value.UserId == userId).ToListAsync(); var completedCcsCountTask = _ccsAttempts.CountDocumentsAsync(value => value.UserId == userId && value.State == CcsAttemptStates.Completed && value.CompletedUtc.HasValue); - var simulationsTask = _simulations.Find(value => value.UserId == userId && value.CreatedUtc >= trendStart).ToListAsync(); - var latestScoreTask = _simulations.Find(value => value.UserId == userId + var simulationsTask = _simulations.Find(value => value.UserId == userId && !value.IsSample && value.CreatedUtc >= trendStart).ToListAsync(); + var completedSimulationsTask = _simulations.Find(value => value.UserId == userId && value.State == ExamSimulationStates.Completed && value.Score != null) - .SortByDescending(value => value.CompletedUtc).FirstOrDefaultAsync(); + .SortByDescending(value => value.CompletedUtc).ToListAsync(); var bookmarkCountTask = _bookmarks.CountDocumentsAsync(value => value.UserId == userId); var notepadTask = _notes.Find(value => value.UserId == userId).FirstOrDefaultAsync(); var reviewAttemptsTask = _reviewQueue.GetReviewsSinceAsync(userId, weekStart); @@ -90,7 +90,7 @@ public sealed class DashboardStore var flashcardReviewsTask = _flashcardSchedules.GetReviewsSinceAsync(userId, weekStart); var dynamicReviewsTask = _dynamicQuestions.DueReviewsAsync(userId, nowUtc); await Task.WhenAll(attemptsTask, ratingTask, ccsRatingTask, ccsAttemptsTask, ccsAllAttemptsTask, completedCcsCountTask, - simulationsTask, latestScoreTask, bookmarkCountTask, notepadTask, reviewAttemptsTask, flashcardCountsTask, + simulationsTask, completedSimulationsTask, bookmarkCountTask, notepadTask, reviewAttemptsTask, flashcardCountsTask, flashcardReviewsTask, dynamicReviewsTask); var attempts = attemptsTask.Result; @@ -112,7 +112,7 @@ public sealed class DashboardStore var ccsTotalTask = _ccsRatings.CountDocumentsAsync(FilterDefinition.Empty); await Task.WhenAll(qbankHigherTask, qbankTotalTask, ccsHigherTask, ccsTotalTask); - var latestScore = latestScoreTask.Result?.Score; + var latestScore = SelectLatestReadinessScore(completedSimulationsTask.Result); var accuracySampleSize = recentAnswered.Count; var firstAttemptAccuracy = Percentage(recentAnswered.Count(value => value.IsCorrect), recentAnswered.Count); var performanceTrend = BuildPerformanceTrend(attempts, ratingTask.Result, ccsAllAttemptsTask.Result, ccsRatingTask.Result, today); @@ -190,6 +190,13 @@ public sealed class DashboardStore public static string ReadinessLabel(double passProbability) => passProbability >= 0.7 ? "High" : passProbability >= 0.4 ? "Medium" : "Low"; + // completedSimulationsByRecency must already be sorted newest-first (by CompletedUtc). + // Sample simulations (QA-only, seeded via the SUPER-role "preview results" tooling with + // scripted answers) never reflect real performance, so they're skipped here rather than + // in the Mongo filter - keeping the "which one counts" decision unit-testable. + public static ExamSimulationScore? SelectLatestReadinessScore(IEnumerable completedSimulationsByRecency) => + completedSimulationsByRecency.FirstOrDefault(value => !value.IsSample)?.Score; + public static int CalculateCurrentStreak(IEnumerable activeDates, DateTime todayUtc) { var dates = activeDates.Select(value => value.Date).ToHashSet();