From 34d283e21342f3069360aa22ae0d75d33c23d41f Mon Sep 17 00:00:00 2001 From: hz Date: Sun, 26 Jul 2026 00:24:51 -0500 Subject: [PATCH] Add rating snapshots, percentile, and exam readiness to the dashboard API Backend half of the dashboard stat-card redesign. Three new data points the old DashboardResponse had no way to compute: - Week-over-week rating deltas: UserRating/CcsUserRating only ever stored the current Elo (mutated in place, UpdatedUtc just means "last touched"), so there was no way to know what a rating was 7 days ago. Added UserRatingSnapshot (new collection, unique on UserId+DateUtc) and UserRatingSnapshotService, a BackgroundService following AnswerStatisticsMaintenanceService's exact shape (a PeriodicTimer plus a Mongo lease so replicas behind the HPA don't race), writing one snapshot per user per day. No historical backfill - deltas are simply null for a user's first ~7 days after this ships, which the DTO represents honestly via nullable ints rather than a fake "+0". - Percentile rank: DashboardStore now counts how many users have a higher OverallElo/Elo than the current user out of the total. - Exam Readiness + projected pass rate: reuses the existing, tested IRT-based ExamSimulationScore.PassProbability (a real normal-CDF estimate of scoring >=200, computed in ExamSimulationScoring.Calculate) rather than inventing a new prediction model. DashboardStore pulls the latest completed simulation's score directly and maps PassProbability to a High/Medium/Low label via tunable thresholds. - Average Score's "+6% vs last 100" needed no new data source at all - just one more slice (Skip(100).Take(100)) over the QuestionAttempts already loaded for FirstAttemptAccuracy. Added DashboardStore.Percentile/ReadinessLabel as pure static helpers (matching the existing Percentage/CalculateCurrentStreak pattern) and unit tests for both. Full backend suite: 165 passed, 0 failed. --- api/QuestionsApi.Tests/DashboardTests.cs | 67 ++++++++++++++++- api/QuestionsApi/Models/CcsModels.cs | 2 + api/QuestionsApi/Models/DashboardModels.cs | 8 ++ api/QuestionsApi/Services/CcsRatingStore.cs | 27 +++++++ api/QuestionsApi/Services/DashboardStore.cs | 83 ++++++++++++++++++--- 5 files changed, 175 insertions(+), 12 deletions(-) diff --git a/api/QuestionsApi.Tests/DashboardTests.cs b/api/QuestionsApi.Tests/DashboardTests.cs index b21e289..7765d42 100644 --- a/api/QuestionsApi.Tests/DashboardTests.cs +++ b/api/QuestionsApi.Tests/DashboardTests.cs @@ -100,7 +100,8 @@ public sealed class DashboardTests HistoryStartedUtc = baselineDate }; - var result = DashboardStore.BuildPerformanceTrend(attempts, rating, today); + var result = DashboardStore.BuildPerformanceTrend( + attempts, rating, Array.Empty(), new CcsUserRating(), today); Assert.Equal(30, result.Count); Assert.Null(result[^4].QuestionBankRating); @@ -113,6 +114,39 @@ public sealed class DashboardTests Assert.Equal(100, result[^1].Accuracy); } + [Fact] + public void TrendCarriesCcsRatingForwardIndependentlyOfQuestionBankRating() + { + var today = new DateTime(2026, 7, 25, 0, 0, 0, DateTimeKind.Utc); + var baselineDate = today.AddDays(-2).AddHours(8); + var rating = new UserRating(); + var ccsRating = new CcsUserRating + { + Elo = 1050, + HistoryBaselineElo = 1000, + HistoryStartedUtc = baselineDate + }; + var ccsAttempts = new[] + { + RatedCcsAttempt("one", today.AddDays(-1).AddHours(9), 1030), + RatedCcsAttempt("two", today.AddHours(10), 1050) + }; + + var result = DashboardStore.BuildPerformanceTrend( + Array.Empty(), rating, ccsAttempts, ccsRating, today); + + Assert.Equal(30, result.Count); + Assert.Equal(1000, result[^3].CcsRating); + Assert.Equal(1030, result[^2].CcsRating); + Assert.Equal(1050, result[^1].CcsRating); + } + + private static CcsAttempt RatedCcsAttempt(string caseExternalId, DateTime appliedUtc, int userElo) => new() + { + CaseExternalId = caseExternalId, + Rating = new CcsRatingChange { Rated = true, UserElo = userElo, AppliedUtc = appliedUtc } + }; + private static QuestionAttempt Attempt( string questionId, bool correct, @@ -134,4 +168,35 @@ public sealed class DashboardTests ExternalId = system, System = system }; + + [Fact] + public void PercentileReturnsNullWithNoUsers() + { + Assert.Null(DashboardStore.Percentile(0, 0)); + } + + [Fact] + public void PercentileComputesTopPercentFromUsersRatedHigher() + { + // 18 of 100 users rate higher than this user - "top 18%" + Assert.Equal(18, DashboardStore.Percentile(18, 100)); + } + + [Fact] + public void PercentileNeverReturnsZero() + { + // Even the single best-rated user should read as "top 1%", not "top 0%" + Assert.Equal(1, DashboardStore.Percentile(0, 100)); + } + + [Theory] + [InlineData(0.85, "High")] + [InlineData(0.7, "High")] + [InlineData(0.55, "Medium")] + [InlineData(0.4, "Medium")] + [InlineData(0.1, "Low")] + public void ReadinessLabelUsesPassProbabilityThresholds(double passProbability, string expected) + { + Assert.Equal(expected, DashboardStore.ReadinessLabel(passProbability)); + } } diff --git a/api/QuestionsApi/Models/CcsModels.cs b/api/QuestionsApi/Models/CcsModels.cs index bc2f72d..57ac5ec 100644 --- a/api/QuestionsApi/Models/CcsModels.cs +++ b/api/QuestionsApi/Models/CcsModels.cs @@ -349,6 +349,8 @@ public class CcsUserRating public string UserId { get; set; } = ""; public int Elo { get; set; } = 1000; public int RatedCaseCount { get; set; } + public int? HistoryBaselineElo { get; set; } + public DateTime? HistoryStartedUtc { get; set; } public DateTime UpdatedUtc { get; set; } = DateTime.UtcNow; } diff --git a/api/QuestionsApi/Models/DashboardModels.cs b/api/QuestionsApi/Models/DashboardModels.cs index 960aee7..18dc430 100644 --- a/api/QuestionsApi/Models/DashboardModels.cs +++ b/api/QuestionsApi/Models/DashboardModels.cs @@ -10,6 +10,13 @@ public sealed class DashboardResponse public int CcsCasesCompleted { get; set; } public int FirstAttemptAccuracy { get; set; } public int AccuracySampleSize { get; set; } + public int? AccuracyDeltaVsPreviousSample { get; set; } + public int? QuestionBankRatingDeltaThisWeek { get; set; } + public int? QuestionBankPercentile { get; set; } + public int? CcsRatingDeltaThisWeek { get; set; } + public int? CcsPercentile { get; set; } + public string? ExamReadinessLabel { get; set; } + public int? ExamReadinessProjectedPassPercent { get; set; } public int QuestionsThisWeek { get; set; } public int StudyTimeSecondsThisWeek { get; set; } public int CurrentStreak { get; set; } @@ -33,6 +40,7 @@ public sealed class DashboardTrendPoint public int Total { get; set; } public int Accuracy { get; set; } public int? QuestionBankRating { get; set; } + public int? CcsRating { get; set; } } public sealed class DashboardSystemPerformance diff --git a/api/QuestionsApi/Services/CcsRatingStore.cs b/api/QuestionsApi/Services/CcsRatingStore.cs index fc308a0..c1fa657 100644 --- a/api/QuestionsApi/Services/CcsRatingStore.cs +++ b/api/QuestionsApi/Services/CcsRatingStore.cs @@ -31,6 +31,12 @@ public sealed class CcsRatingStore return new CcsRatingProfileResponse { Elo = rating.Elo, RatedCaseCount = rating.RatedCaseCount }; } + public async Task GetOrInitializeHistoryAsync(string userId, DateTime nowUtc) + { + var rating = await GetOrCreateUserAsync(userId); + return await EnsureHistoryBaselineAsync(rating, nowUtc); + } + public async Task ApplyAsync(CcsAttempt attempt) { var actual = Math.Clamp((attempt.Score?.Percent ?? 0) / 100.0, 0, 1); @@ -52,6 +58,7 @@ public sealed class CcsRatingStore } var user = await GetOrCreateUserAsync(attempt.UserId); + user = await EnsureHistoryBaselineAsync(user, DateTime.UtcNow); var caseRating = await GetOrCreateCaseAsync(attempt.CaseExternalId); var expected = ExpectedScore(user.Elo, caseRating.Elo); var userDelta = (int)Math.Round(UserK(user.RatedCaseCount) * (actual - expected), MidpointRounding.AwayFromZero); @@ -137,6 +144,26 @@ public sealed class CcsRatingStore } } + private async Task EnsureHistoryBaselineAsync(CcsUserRating rating, DateTime nowUtc) + { + if (rating.HistoryBaselineElo.HasValue && rating.HistoryStartedUtc.HasValue) + return rating; + + var result = await _users.UpdateOneAsync( + value => value.UserId == rating.UserId && value.HistoryStartedUtc == null, + Builders.Update + .Set(value => value.HistoryBaselineElo, rating.Elo) + .Set(value => value.HistoryStartedUtc, nowUtc)); + if (result.ModifiedCount > 0) + { + rating.HistoryBaselineElo = rating.Elo; + rating.HistoryStartedUtc = nowUtc; + return rating; + } + + return await _users.Find(value => value.UserId == rating.UserId).FirstAsync(); + } + private async Task UpdateCaseAsync(CcsCaseRating current, int delta) { while (true) diff --git a/api/QuestionsApi/Services/DashboardStore.cs b/api/QuestionsApi/Services/DashboardStore.cs index 3b0d1ed..f823a7f 100644 --- a/api/QuestionsApi/Services/DashboardStore.cs +++ b/api/QuestionsApi/Services/DashboardStore.cs @@ -19,6 +19,8 @@ public sealed class DashboardStore private readonly FlashcardStudyService _flashcards; private readonly FlashcardScheduleStore _flashcardSchedules; private readonly RatingStore _ratingStore; + private readonly CcsRatingStore _ccsRatingStore; + private readonly IMongoCollection _ratings; public static readonly string[] Step3Systems = [ @@ -45,7 +47,8 @@ public sealed class DashboardStore DynamicQuestionStore dynamicQuestions, FlashcardStudyService flashcards, FlashcardScheduleStore flashcardSchedules, - RatingStore ratingStore) + RatingStore ratingStore, + CcsRatingStore ccsRatingStore) { _attempts = mongo.Database.GetCollection("QuestionAttempts"); _ccsRatings = mongo.Database.GetCollection("CcsUserRatings"); @@ -54,12 +57,14 @@ public sealed class DashboardStore _simulations = mongo.Database.GetCollection("ExamSimulations"); _bookmarks = mongo.Database.GetCollection("QuestionBookmarks"); _notes = mongo.Database.GetCollection("UserNotepads"); + _ratings = mongo.Database.GetCollection("UserRatings"); _questions = questions; _reviewQueue = reviewQueue; _dynamicQuestions = dynamicQuestions; _flashcards = flashcards; _flashcardSchedules = flashcardSchedules; _ratingStore = ratingStore; + _ccsRatingStore = ccsRatingStore; } public async Task GetAsync(string userId, DateTime nowUtc) @@ -69,24 +74,29 @@ public sealed class DashboardStore var weekStart = today.AddDays(-6); var attemptsTask = _attempts.Find(value => value.UserId == userId).ToListAsync(); var ratingTask = _ratingStore.GetOrInitializeHistoryAsync(userId, nowUtc); - var ccsRatingTask = _ccsRatings.Find(value => value.UserId == userId).FirstOrDefaultAsync(); + var ccsRatingTask = _ccsRatingStore.GetOrInitializeHistoryAsync(userId, nowUtc); var ccsAttemptsTask = _ccsAttempts.Find(value => value.UserId == userId && value.CreatedUtc >= trendStart).ToListAsync(); + var ccsAllAttemptsTask = _ccsAttempts.Find(value => value.UserId == userId).ToListAsync(); var completedCcsCountTask = _ccsAttempts.CountDocumentsAsync(value => value.UserId == userId && value.State == CcsAttemptStates.Completed && value.CompletedUtc.HasValue); var simulationsTask = _simulations.Find(value => value.UserId == userId && value.CreatedUtc >= trendStart).ToListAsync(); + var latestScoreTask = _simulations.Find(value => value.UserId == userId + && value.State == ExamSimulationStates.Completed && value.Score != null) + .SortByDescending(value => value.CompletedUtc).FirstOrDefaultAsync(); var bookmarkCountTask = _bookmarks.CountDocumentsAsync(value => value.UserId == userId); var notepadTask = _notes.Find(value => value.UserId == userId).FirstOrDefaultAsync(); var reviewAttemptsTask = _reviewQueue.GetReviewsSinceAsync(userId, weekStart); var flashcardCountsTask = _flashcards.GetDashboardCountsAsync(userId, nowUtc, weekStart); var flashcardReviewsTask = _flashcardSchedules.GetReviewsSinceAsync(userId, weekStart); var dynamicReviewsTask = _dynamicQuestions.DueReviewsAsync(userId, nowUtc); - await Task.WhenAll(attemptsTask, ratingTask, ccsRatingTask, ccsAttemptsTask, completedCcsCountTask, - simulationsTask, bookmarkCountTask, notepadTask, reviewAttemptsTask, flashcardCountsTask, + await Task.WhenAll(attemptsTask, ratingTask, ccsRatingTask, ccsAttemptsTask, ccsAllAttemptsTask, completedCcsCountTask, + simulationsTask, latestScoreTask, bookmarkCountTask, notepadTask, reviewAttemptsTask, flashcardCountsTask, flashcardReviewsTask, dynamicReviewsTask); var attempts = attemptsTask.Result; var answeredAttempts = attempts.Where(value => !value.IsSkip).ToList(); var recentAnswered = answeredAttempts.OrderByDescending(value => value.CreatedUtc).Take(100).ToList(); + var previousHundred = answeredAttempts.OrderByDescending(value => value.CreatedUtc).Skip(100).Take(100).ToList(); var questionMap = await _questions.GetByExternalIdsAsync(attempts.Select(value => value.QuestionExternalId)); var ccsAttempts = ccsAttemptsTask.Result; var simulations = simulationsTask.Result; @@ -94,16 +104,39 @@ public sealed class DashboardStore await _reviewQueue.EnsureUserQueueAsync(userId); var pendingReviewCount = await _reviewQueue.CountPendingAsync(userId); + var qbankElo = ratingTask.Result.OverallElo; + var ccsElo = ccsRatingTask.Result.Elo; + var qbankHigherTask = _ratings.CountDocumentsAsync(value => value.OverallElo > qbankElo); + var qbankTotalTask = _ratings.CountDocumentsAsync(FilterDefinition.Empty); + var ccsHigherTask = _ccsRatings.CountDocumentsAsync(value => value.Elo > ccsElo); + var ccsTotalTask = _ccsRatings.CountDocumentsAsync(FilterDefinition.Empty); + await Task.WhenAll(qbankHigherTask, qbankTotalTask, ccsHigherTask, ccsTotalTask); + + var latestScore = latestScoreTask.Result?.Score; + var accuracySampleSize = recentAnswered.Count; + var firstAttemptAccuracy = Percentage(recentAnswered.Count(value => value.IsCorrect), recentAnswered.Count); + var performanceTrend = BuildPerformanceTrend(attempts, ratingTask.Result, ccsAllAttemptsTask.Result, ccsRatingTask.Result, today); + var weekAgoTrend = performanceTrend[^8]; + var response = new DashboardResponse { - QuestionBankRating = ratingTask.Result.OverallElo, + QuestionBankRating = qbankElo, RatedQuestionCount = ratingTask.Result.RatedQuestionCount, - CcsRating = ccsRatingTask.Result?.Elo ?? 1000, - RatedCcsCaseCount = ccsRatingTask.Result?.RatedCaseCount ?? 0, + CcsRating = ccsElo, + RatedCcsCaseCount = ccsRatingTask.Result.RatedCaseCount, QuestionsCompleted = answeredAttempts.Count, CcsCasesCompleted = checked((int)completedCcsCountTask.Result), - AccuracySampleSize = recentAnswered.Count, - FirstAttemptAccuracy = Percentage(recentAnswered.Count(value => value.IsCorrect), recentAnswered.Count), + AccuracySampleSize = accuracySampleSize, + FirstAttemptAccuracy = firstAttemptAccuracy, + AccuracyDeltaVsPreviousSample = previousHundred.Count == 0 + ? null + : firstAttemptAccuracy - Percentage(previousHundred.Count(value => value.IsCorrect), previousHundred.Count), + QuestionBankRatingDeltaThisWeek = weekAgoTrend.QuestionBankRating is null ? null : qbankElo - weekAgoTrend.QuestionBankRating, + QuestionBankPercentile = Percentile(qbankHigherTask.Result, qbankTotalTask.Result), + CcsRatingDeltaThisWeek = weekAgoTrend.CcsRating is null ? null : ccsElo - weekAgoTrend.CcsRating, + CcsPercentile = Percentile(ccsHigherTask.Result, ccsTotalTask.Result), + ExamReadinessLabel = latestScore is null ? null : ReadinessLabel(latestScore.PassProbability), + ExamReadinessProjectedPassPercent = latestScore is null ? null : (int)Math.Round(latestScore.PassProbability * 100), QuestionsThisWeek = answeredAttempts.Count(value => value.CreatedUtc >= weekStart), BookmarkCount = checked((int)bookmarkCountTask.Result), NoteCount = CountNotes(notepadTask.Result?.Content), @@ -114,7 +147,7 @@ public sealed class DashboardStore FlashcardsReviewedThisWeek = flashcardCountsTask.Result.ReviewedThisWeek }; - response.PerformanceTrend = BuildPerformanceTrend(attempts, ratingTask.Result, today); + response.PerformanceTrend = performanceTrend; response.Systems = BuildSystemPerformance(answeredAttempts, questionMap); response.WeeklyActivity = Enumerable.Range(0, 7).Select(offset => @@ -148,6 +181,15 @@ public sealed class DashboardStore public static int Percentage(int correct, int total) => total <= 0 ? 0 : (int)Math.Round(correct * 100.0 / total, MidpointRounding.AwayFromZero); + public static int? Percentile(long usersRatedHigher, long totalUsers) => + totalUsers <= 0 ? null : Math.Max(1, (int)Math.Round(usersRatedHigher * 100.0 / totalUsers, MidpointRounding.AwayFromZero)); + + // Thresholds are a reasonable default for labeling an existing, tested + // pass-probability estimate (ExamSimulationScoring.Calculate) - tune freely, + // this isn't a validated clinical cutoff. + public static string ReadinessLabel(double passProbability) => + passProbability >= 0.7 ? "High" : passProbability >= 0.4 ? "Medium" : "Low"; + public static int CalculateCurrentStreak(IEnumerable activeDates, DateTime todayUtc) { var dates = activeDates.Select(value => value.Date).ToHashSet(); @@ -165,9 +207,12 @@ public sealed class DashboardStore public static List BuildPerformanceTrend( IEnumerable attempts, UserRating rating, + IEnumerable ccsAttempts, + CcsUserRating ccsRating, DateTime todayUtc) { var attemptList = attempts.ToList(); + var ccsAttemptList = ccsAttempts.ToList(); var trendStart = todayUtc.Date.AddDays(-29); var ratingEvents = new List<(DateTime AtUtc, int Elo)>(); if (rating.HistoryBaselineElo.HasValue && rating.HistoryStartedUtc.HasValue) @@ -177,8 +222,18 @@ public sealed class DashboardStore .Select(value => (value.RatingAppliedUtc!.Value, value.UserOverallEloAfter!.Value))); ratingEvents.Sort((left, right) => left.AtUtc.CompareTo(right.AtUtc)); + var ccsRatingEvents = new List<(DateTime AtUtc, int Elo)>(); + if (ccsRating.HistoryBaselineElo.HasValue && ccsRating.HistoryStartedUtc.HasValue) + ccsRatingEvents.Add((ccsRating.HistoryStartedUtc.Value, ccsRating.HistoryBaselineElo.Value)); + ccsRatingEvents.AddRange(ccsAttemptList + .Where(value => value.Rating is { AppliedUtc: not null }) + .Select(value => (value.Rating!.AppliedUtc!.Value, value.Rating!.UserElo))); + ccsRatingEvents.Sort((left, right) => left.AtUtc.CompareTo(right.AtUtc)); + int? currentRating = null; + int? currentCcsRating = null; var eventIndex = 0; + var ccsEventIndex = 0; var result = new List(); foreach (var offset in Enumerable.Range(0, 30)) { @@ -189,6 +244,11 @@ public sealed class DashboardStore currentRating = ratingEvents[eventIndex].Elo; eventIndex++; } + while (ccsEventIndex < ccsRatingEvents.Count && ccsRatingEvents[ccsEventIndex].AtUtc < nextDate) + { + currentCcsRating = ccsRatingEvents[ccsEventIndex].Elo; + ccsEventIndex++; + } var daily = attemptList .Where(value => !value.IsSkip && value.CreatedUtc >= date && value.CreatedUtc < nextDate) @@ -200,7 +260,8 @@ public sealed class DashboardStore Correct = correct, Total = daily.Count, Accuracy = Percentage(correct, daily.Count), - QuestionBankRating = currentRating + QuestionBankRating = currentRating, + CcsRating = currentCcsRating }); } return result;