Exclude sample exam simulations from the dashboard's readiness score

The SUPER-role "preview results" QA tooling seeds a completed
ExamSimulation with scripted (not real) answers, flagged IsSample.
The dashboard's readiness lookup picked whichever completed
simulation was most recent regardless of that flag, so running the
QA preview on an account could override its real Exam Readiness card
with a synthetic pass probability. Recent-activity/weekly-activity
queries got the same exclusion since they read from the same
collection for the same reason.

Co-Authored-By: Claude Sonnet 5 <[email protected]>
This commit is contained in:
hz
2026-07-26 14:47:07 -05:00
co-authored by Claude Sonnet 5
parent 447f080552
commit 74398133f7
2 changed files with 43 additions and 5 deletions
+31
View File
@@ -199,4 +199,35 @@ public sealed class DashboardTests
{ {
Assert.Equal(expected, DashboardStore.ReadinessLabel(passProbability)); Assert.Equal(expected, DashboardStore.ReadinessLabel(passProbability));
} }
[Fact]
public void SelectLatestReadinessScoreSkipsSampleSimulationsEvenWhenMostRecent()
{
var today = new DateTime(2026, 7, 25, 0, 0, 0, DateTimeKind.Utc);
var simulations = new[]
{
CompletedSimulation(isSample: true, completedUtc: today, passProbability: 0.98),
CompletedSimulation(isSample: false, completedUtc: today.AddDays(-3), passProbability: 0.12)
};
var score = DashboardStore.SelectLatestReadinessScore(simulations);
Assert.NotNull(score);
Assert.Equal(0.12, score.PassProbability);
}
[Fact]
public void SelectLatestReadinessScoreReturnsNullWhenOnlySamplesExist()
{
var simulations = new[] { CompletedSimulation(isSample: true, completedUtc: DateTime.UtcNow, passProbability: 0.98) };
Assert.Null(DashboardStore.SelectLatestReadinessScore(simulations));
}
private static ExamSimulation CompletedSimulation(bool isSample, DateTime completedUtc, double passProbability) => new()
{
IsSample = isSample,
CompletedUtc = completedUtc,
Score = new ExamSimulationScore { PassProbability = passProbability }
};
} }
+12 -5
View File
@@ -79,10 +79,10 @@ public sealed class DashboardStore
var ccsAllAttemptsTask = _ccsAttempts.Find(value => value.UserId == userId).ToListAsync(); var ccsAllAttemptsTask = _ccsAttempts.Find(value => value.UserId == userId).ToListAsync();
var completedCcsCountTask = _ccsAttempts.CountDocumentsAsync(value => value.UserId == userId var completedCcsCountTask = _ccsAttempts.CountDocumentsAsync(value => value.UserId == userId
&& value.State == CcsAttemptStates.Completed && value.CompletedUtc.HasValue); && value.State == CcsAttemptStates.Completed && value.CompletedUtc.HasValue);
var simulationsTask = _simulations.Find(value => value.UserId == userId && value.CreatedUtc >= trendStart).ToListAsync(); var simulationsTask = _simulations.Find(value => value.UserId == userId && !value.IsSample && value.CreatedUtc >= trendStart).ToListAsync();
var latestScoreTask = _simulations.Find(value => value.UserId == userId var completedSimulationsTask = _simulations.Find(value => value.UserId == userId
&& value.State == ExamSimulationStates.Completed && value.Score != null) && value.State == ExamSimulationStates.Completed && value.Score != null)
.SortByDescending(value => value.CompletedUtc).FirstOrDefaultAsync(); .SortByDescending(value => value.CompletedUtc).ToListAsync();
var bookmarkCountTask = _bookmarks.CountDocumentsAsync(value => value.UserId == userId); var bookmarkCountTask = _bookmarks.CountDocumentsAsync(value => value.UserId == userId);
var notepadTask = _notes.Find(value => value.UserId == userId).FirstOrDefaultAsync(); var notepadTask = _notes.Find(value => value.UserId == userId).FirstOrDefaultAsync();
var reviewAttemptsTask = _reviewQueue.GetReviewsSinceAsync(userId, weekStart); var reviewAttemptsTask = _reviewQueue.GetReviewsSinceAsync(userId, weekStart);
@@ -90,7 +90,7 @@ public sealed class DashboardStore
var flashcardReviewsTask = _flashcardSchedules.GetReviewsSinceAsync(userId, weekStart); var flashcardReviewsTask = _flashcardSchedules.GetReviewsSinceAsync(userId, weekStart);
var dynamicReviewsTask = _dynamicQuestions.DueReviewsAsync(userId, nowUtc); var dynamicReviewsTask = _dynamicQuestions.DueReviewsAsync(userId, nowUtc);
await Task.WhenAll(attemptsTask, ratingTask, ccsRatingTask, ccsAttemptsTask, ccsAllAttemptsTask, completedCcsCountTask, await Task.WhenAll(attemptsTask, ratingTask, ccsRatingTask, ccsAttemptsTask, ccsAllAttemptsTask, completedCcsCountTask,
simulationsTask, latestScoreTask, bookmarkCountTask, notepadTask, reviewAttemptsTask, flashcardCountsTask, simulationsTask, completedSimulationsTask, bookmarkCountTask, notepadTask, reviewAttemptsTask, flashcardCountsTask,
flashcardReviewsTask, dynamicReviewsTask); flashcardReviewsTask, dynamicReviewsTask);
var attempts = attemptsTask.Result; var attempts = attemptsTask.Result;
@@ -112,7 +112,7 @@ public sealed class DashboardStore
var ccsTotalTask = _ccsRatings.CountDocumentsAsync(FilterDefinition<CcsUserRating>.Empty); var ccsTotalTask = _ccsRatings.CountDocumentsAsync(FilterDefinition<CcsUserRating>.Empty);
await Task.WhenAll(qbankHigherTask, qbankTotalTask, ccsHigherTask, ccsTotalTask); await Task.WhenAll(qbankHigherTask, qbankTotalTask, ccsHigherTask, ccsTotalTask);
var latestScore = latestScoreTask.Result?.Score; var latestScore = SelectLatestReadinessScore(completedSimulationsTask.Result);
var accuracySampleSize = recentAnswered.Count; var accuracySampleSize = recentAnswered.Count;
var firstAttemptAccuracy = Percentage(recentAnswered.Count(value => value.IsCorrect), recentAnswered.Count); var firstAttemptAccuracy = Percentage(recentAnswered.Count(value => value.IsCorrect), recentAnswered.Count);
var performanceTrend = BuildPerformanceTrend(attempts, ratingTask.Result, ccsAllAttemptsTask.Result, ccsRatingTask.Result, today); var performanceTrend = BuildPerformanceTrend(attempts, ratingTask.Result, ccsAllAttemptsTask.Result, ccsRatingTask.Result, today);
@@ -190,6 +190,13 @@ public sealed class DashboardStore
public static string ReadinessLabel(double passProbability) => public static string ReadinessLabel(double passProbability) =>
passProbability >= 0.7 ? "High" : passProbability >= 0.4 ? "Medium" : "Low"; passProbability >= 0.7 ? "High" : passProbability >= 0.4 ? "Medium" : "Low";
// completedSimulationsByRecency must already be sorted newest-first (by CompletedUtc).
// Sample simulations (QA-only, seeded via the SUPER-role "preview results" tooling with
// scripted answers) never reflect real performance, so they're skipped here rather than
// in the Mongo filter - keeping the "which one counts" decision unit-testable.
public static ExamSimulationScore? SelectLatestReadinessScore(IEnumerable<ExamSimulation> completedSimulationsByRecency) =>
completedSimulationsByRecency.FirstOrDefault(value => !value.IsSample)?.Score;
public static int CalculateCurrentStreak(IEnumerable<DateTime> activeDates, DateTime todayUtc) public static int CalculateCurrentStreak(IEnumerable<DateTime> activeDates, DateTime todayUtc)
{ {
var dates = activeDates.Select(value => value.Date).ToHashSet(); var dates = activeDates.Select(value => value.Date).ToHashSet();