Add rating snapshots, percentile, and exam readiness to the dashboard API

Backend half of the dashboard stat-card redesign. Three new data points the old DashboardResponse had no way to compute: - Week-over-week rating deltas: UserRating/CcsUserRating only ever stored the current Elo (mutated in place, UpdatedUtc just means "last touched"), so there was no way to know what a rating was 7 days ago. Added UserRatingSnapshot (new collection, unique on UserId+DateUtc) and UserRatingSnapshotService, a BackgroundService following AnswerStatisticsMaintenanceService's exact shape (a PeriodicTimer plus a Mongo lease so replicas behind the HPA don't race), writing one snapshot per user per day. No historical backfill - deltas are simply null for a user's first ~7 days after this ships, which the DTO represents honestly via nullable ints rather than a fake "+0". - Percentile rank: DashboardStore now counts how many users have a higher OverallElo/Elo than the current user out of the total. - Exam Readiness + projected pass rate: reuses the existing, tested IRT-based ExamSimulationScore.PassProbability (a real normal-CDF estimate of scoring >=200, computed in ExamSimulationScoring.Calculate) rather than inventing a new prediction model. DashboardStore pulls the latest completed simulation's score directly and maps PassProbability to a High/Medium/Low label via tunable thresholds. - Average Score's "+6% vs last 100" needed no new data source at all - just one more slice (Skip(100).Take(100)) over the QuestionAttempts already loaded for FirstAttemptAccuracy. Added DashboardStore.Percentile/ReadinessLabel as pure static helpers (matching the existing Percentage/CalculateCurrentStreak pattern) and unit tests for both. Full backend suite: 165 passed, 0 failed.
2026-07-26 00:39:11 -05:00
parent 500477b612
commit 34d283e213
5 changed files with 175 additions and 12 deletions
@@ -349,6 +349,8 @@ public class CcsUserRating
    public string UserId { get; set; } = "";
    public int Elo { get; set; } = 1000;
    public int RatedCaseCount { get; set; }
+    public int? HistoryBaselineElo { get; set; }
+    public DateTime? HistoryStartedUtc { get; set; }
    public DateTime UpdatedUtc { get; set; } = DateTime.UtcNow;
 }

@@ -10,6 +10,13 @@ public sealed class DashboardResponse
    public int CcsCasesCompleted { get; set; }
    public int FirstAttemptAccuracy { get; set; }
    public int AccuracySampleSize { get; set; }
+    public int? AccuracyDeltaVsPreviousSample { get; set; }
+    public int? QuestionBankRatingDeltaThisWeek { get; set; }
+    public int? QuestionBankPercentile { get; set; }
+    public int? CcsRatingDeltaThisWeek { get; set; }
+    public int? CcsPercentile { get; set; }
+    public string? ExamReadinessLabel { get; set; }
+    public int? ExamReadinessProjectedPassPercent { get; set; }
    public int QuestionsThisWeek { get; set; }
    public int StudyTimeSecondsThisWeek { get; set; }
    public int CurrentStreak { get; set; }
@@ -33,6 +40,7 @@ public sealed class DashboardTrendPoint
    public int Total { get; set; }
    public int Accuracy { get; set; }
    public int? QuestionBankRating { get; set; }
+    public int? CcsRating { get; set; }
 }

 public sealed class DashboardSystemPerformance
@@ -31,6 +31,12 @@ public sealed class CcsRatingStore
        return new CcsRatingProfileResponse { Elo = rating.Elo, RatedCaseCount = rating.RatedCaseCount };
    }

+    public async Task<CcsUserRating> GetOrInitializeHistoryAsync(string userId, DateTime nowUtc)
+    {
+        var rating = await GetOrCreateUserAsync(userId);
+        return await EnsureHistoryBaselineAsync(rating, nowUtc);
+    }
+
    public async Task<CcsRatingChange> ApplyAsync(CcsAttempt attempt)
    {
        var actual = Math.Clamp((attempt.Score?.Percent ?? 0) / 100.0, 0, 1);
@@ -52,6 +58,7 @@ public sealed class CcsRatingStore
        }

        var user = await GetOrCreateUserAsync(attempt.UserId);
+        user = await EnsureHistoryBaselineAsync(user, DateTime.UtcNow);
        var caseRating = await GetOrCreateCaseAsync(attempt.CaseExternalId);
        var expected = ExpectedScore(user.Elo, caseRating.Elo);
        var userDelta = (int)Math.Round(UserK(user.RatedCaseCount) * (actual - expected), MidpointRounding.AwayFromZero);
@@ -137,6 +144,26 @@ public sealed class CcsRatingStore
        }
    }

+    private async Task<CcsUserRating> EnsureHistoryBaselineAsync(CcsUserRating rating, DateTime nowUtc)
+    {
+        if (rating.HistoryBaselineElo.HasValue && rating.HistoryStartedUtc.HasValue)
+            return rating;
+
+        var result = await _users.UpdateOneAsync(
+            value => value.UserId == rating.UserId && value.HistoryStartedUtc == null,
+            Builders<CcsUserRating>.Update
+                .Set(value => value.HistoryBaselineElo, rating.Elo)
+                .Set(value => value.HistoryStartedUtc, nowUtc));
+        if (result.ModifiedCount > 0)
+        {
+            rating.HistoryBaselineElo = rating.Elo;
+            rating.HistoryStartedUtc = nowUtc;
+            return rating;
+        }
+
+        return await _users.Find(value => value.UserId == rating.UserId).FirstAsync();
+    }
+
    private async Task<CcsCaseRating> UpdateCaseAsync(CcsCaseRating current, int delta)
    {
        while (true)
@@ -19,6 +19,8 @@ public sealed class DashboardStore
    private readonly FlashcardStudyService _flashcards;
    private readonly FlashcardScheduleStore _flashcardSchedules;
    private readonly RatingStore _ratingStore;
+    private readonly CcsRatingStore _ccsRatingStore;
+    private readonly IMongoCollection<UserRating> _ratings;

    public static readonly string[] Step3Systems =
    [
@@ -45,7 +47,8 @@ public sealed class DashboardStore
        DynamicQuestionStore dynamicQuestions,
        FlashcardStudyService flashcards,
        FlashcardScheduleStore flashcardSchedules,
-        RatingStore ratingStore)
+        RatingStore ratingStore,
+        CcsRatingStore ccsRatingStore)
    {
        _attempts = mongo.Database.GetCollection<QuestionAttempt>("QuestionAttempts");
        _ccsRatings = mongo.Database.GetCollection<CcsUserRating>("CcsUserRatings");
@@ -54,12 +57,14 @@ public sealed class DashboardStore
        _simulations = mongo.Database.GetCollection<ExamSimulation>("ExamSimulations");
        _bookmarks = mongo.Database.GetCollection<QuestionBookmark>("QuestionBookmarks");
        _notes = mongo.Database.GetCollection<UserNotepad>("UserNotepads");
+        _ratings = mongo.Database.GetCollection<UserRating>("UserRatings");
        _questions = questions;
        _reviewQueue = reviewQueue;
        _dynamicQuestions = dynamicQuestions;
        _flashcards = flashcards;
        _flashcardSchedules = flashcardSchedules;
        _ratingStore = ratingStore;
+        _ccsRatingStore = ccsRatingStore;
    }

    public async Task<DashboardResponse> GetAsync(string userId, DateTime nowUtc)
@@ -69,24 +74,29 @@ public sealed class DashboardStore
        var weekStart = today.AddDays(-6);
        var attemptsTask = _attempts.Find(value => value.UserId == userId).ToListAsync();
        var ratingTask = _ratingStore.GetOrInitializeHistoryAsync(userId, nowUtc);
-        var ccsRatingTask = _ccsRatings.Find(value => value.UserId == userId).FirstOrDefaultAsync();
+        var ccsRatingTask = _ccsRatingStore.GetOrInitializeHistoryAsync(userId, nowUtc);
        var ccsAttemptsTask = _ccsAttempts.Find(value => value.UserId == userId && value.CreatedUtc >= trendStart).ToListAsync();
+        var ccsAllAttemptsTask = _ccsAttempts.Find(value => value.UserId == userId).ToListAsync();
        var completedCcsCountTask = _ccsAttempts.CountDocumentsAsync(value => value.UserId == userId
            && value.State == CcsAttemptStates.Completed && value.CompletedUtc.HasValue);
        var simulationsTask = _simulations.Find(value => value.UserId == userId && value.CreatedUtc >= trendStart).ToListAsync();
+        var latestScoreTask = _simulations.Find(value => value.UserId == userId
+            && value.State == ExamSimulationStates.Completed && value.Score != null)
+            .SortByDescending(value => value.CompletedUtc).FirstOrDefaultAsync();
        var bookmarkCountTask = _bookmarks.CountDocumentsAsync(value => value.UserId == userId);
        var notepadTask = _notes.Find(value => value.UserId == userId).FirstOrDefaultAsync();
        var reviewAttemptsTask = _reviewQueue.GetReviewsSinceAsync(userId, weekStart);
        var flashcardCountsTask = _flashcards.GetDashboardCountsAsync(userId, nowUtc, weekStart);
        var flashcardReviewsTask = _flashcardSchedules.GetReviewsSinceAsync(userId, weekStart);
        var dynamicReviewsTask = _dynamicQuestions.DueReviewsAsync(userId, nowUtc);
-        await Task.WhenAll(attemptsTask, ratingTask, ccsRatingTask, ccsAttemptsTask, completedCcsCountTask,
-            simulationsTask, bookmarkCountTask, notepadTask, reviewAttemptsTask, flashcardCountsTask,
+        await Task.WhenAll(attemptsTask, ratingTask, ccsRatingTask, ccsAttemptsTask, ccsAllAttemptsTask, completedCcsCountTask,
+            simulationsTask, latestScoreTask, bookmarkCountTask, notepadTask, reviewAttemptsTask, flashcardCountsTask,
            flashcardReviewsTask, dynamicReviewsTask);

        var attempts = attemptsTask.Result;
        var answeredAttempts = attempts.Where(value => !value.IsSkip).ToList();
        var recentAnswered = answeredAttempts.OrderByDescending(value => value.CreatedUtc).Take(100).ToList();
+        var previousHundred = answeredAttempts.OrderByDescending(value => value.CreatedUtc).Skip(100).Take(100).ToList();
        var questionMap = await _questions.GetByExternalIdsAsync(attempts.Select(value => value.QuestionExternalId));
        var ccsAttempts = ccsAttemptsTask.Result;
        var simulations = simulationsTask.Result;
@@ -94,16 +104,39 @@ public sealed class DashboardStore
        await _reviewQueue.EnsureUserQueueAsync(userId);
        var pendingReviewCount = await _reviewQueue.CountPendingAsync(userId);

+        var qbankElo = ratingTask.Result.OverallElo;
+        var ccsElo = ccsRatingTask.Result.Elo;
+        var qbankHigherTask = _ratings.CountDocumentsAsync(value => value.OverallElo > qbankElo);
+        var qbankTotalTask = _ratings.CountDocumentsAsync(FilterDefinition<UserRating>.Empty);
+        var ccsHigherTask = _ccsRatings.CountDocumentsAsync(value => value.Elo > ccsElo);
+        var ccsTotalTask = _ccsRatings.CountDocumentsAsync(FilterDefinition<CcsUserRating>.Empty);
+        await Task.WhenAll(qbankHigherTask, qbankTotalTask, ccsHigherTask, ccsTotalTask);
+
+        var latestScore = latestScoreTask.Result?.Score;
+        var accuracySampleSize = recentAnswered.Count;
+        var firstAttemptAccuracy = Percentage(recentAnswered.Count(value => value.IsCorrect), recentAnswered.Count);
+        var performanceTrend = BuildPerformanceTrend(attempts, ratingTask.Result, ccsAllAttemptsTask.Result, ccsRatingTask.Result, today);
+        var weekAgoTrend = performanceTrend[^8];
+
        var response = new DashboardResponse
        {
-            QuestionBankRating = ratingTask.Result.OverallElo,
+            QuestionBankRating = qbankElo,
            RatedQuestionCount = ratingTask.Result.RatedQuestionCount,
-            CcsRating = ccsRatingTask.Result?.Elo ?? 1000,
-            RatedCcsCaseCount = ccsRatingTask.Result?.RatedCaseCount ?? 0,
+            CcsRating = ccsElo,
+            RatedCcsCaseCount = ccsRatingTask.Result.RatedCaseCount,
            QuestionsCompleted = answeredAttempts.Count,
            CcsCasesCompleted = checked((int)completedCcsCountTask.Result),
-            AccuracySampleSize = recentAnswered.Count,
-            FirstAttemptAccuracy = Percentage(recentAnswered.Count(value => value.IsCorrect), recentAnswered.Count),
+            AccuracySampleSize = accuracySampleSize,
+            FirstAttemptAccuracy = firstAttemptAccuracy,
+            AccuracyDeltaVsPreviousSample = previousHundred.Count == 0
+                ? null
+                : firstAttemptAccuracy - Percentage(previousHundred.Count(value => value.IsCorrect), previousHundred.Count),
+            QuestionBankRatingDeltaThisWeek = weekAgoTrend.QuestionBankRating is null ? null : qbankElo - weekAgoTrend.QuestionBankRating,
+            QuestionBankPercentile = Percentile(qbankHigherTask.Result, qbankTotalTask.Result),
+            CcsRatingDeltaThisWeek = weekAgoTrend.CcsRating is null ? null : ccsElo - weekAgoTrend.CcsRating,
+            CcsPercentile = Percentile(ccsHigherTask.Result, ccsTotalTask.Result),
+            ExamReadinessLabel = latestScore is null ? null : ReadinessLabel(latestScore.PassProbability),
+            ExamReadinessProjectedPassPercent = latestScore is null ? null : (int)Math.Round(latestScore.PassProbability * 100),
            QuestionsThisWeek = answeredAttempts.Count(value => value.CreatedUtc >= weekStart),
            BookmarkCount = checked((int)bookmarkCountTask.Result),
            NoteCount = CountNotes(notepadTask.Result?.Content),
@@ -114,7 +147,7 @@ public sealed class DashboardStore
            FlashcardsReviewedThisWeek = flashcardCountsTask.Result.ReviewedThisWeek
        };

-        response.PerformanceTrend = BuildPerformanceTrend(attempts, ratingTask.Result, today);
+        response.PerformanceTrend = performanceTrend;
        response.Systems = BuildSystemPerformance(answeredAttempts, questionMap);

        response.WeeklyActivity = Enumerable.Range(0, 7).Select(offset =>
@@ -148,6 +181,15 @@ public sealed class DashboardStore
    public static int Percentage(int correct, int total) =>
        total <= 0 ? 0 : (int)Math.Round(correct * 100.0 / total, MidpointRounding.AwayFromZero);

+    public static int? Percentile(long usersRatedHigher, long totalUsers) =>
+        totalUsers <= 0 ? null : Math.Max(1, (int)Math.Round(usersRatedHigher * 100.0 / totalUsers, MidpointRounding.AwayFromZero));
+
+    // Thresholds are a reasonable default for labeling an existing, tested
+    // pass-probability estimate (ExamSimulationScoring.Calculate) - tune freely,
+    // this isn't a validated clinical cutoff.
+    public static string ReadinessLabel(double passProbability) =>
+        passProbability >= 0.7 ? "High" : passProbability >= 0.4 ? "Medium" : "Low";
+
    public static int CalculateCurrentStreak(IEnumerable<DateTime> activeDates, DateTime todayUtc)
    {
        var dates = activeDates.Select(value => value.Date).ToHashSet();
@@ -165,9 +207,12 @@ public sealed class DashboardStore
    public static List<DashboardTrendPoint> BuildPerformanceTrend(
        IEnumerable<QuestionAttempt> attempts,
        UserRating rating,
+        IEnumerable<CcsAttempt> ccsAttempts,
+        CcsUserRating ccsRating,
        DateTime todayUtc)
    {
        var attemptList = attempts.ToList();
+        var ccsAttemptList = ccsAttempts.ToList();
        var trendStart = todayUtc.Date.AddDays(-29);
        var ratingEvents = new List<(DateTime AtUtc, int Elo)>();
        if (rating.HistoryBaselineElo.HasValue && rating.HistoryStartedUtc.HasValue)
@@ -177,8 +222,18 @@ public sealed class DashboardStore
            .Select(value => (value.RatingAppliedUtc!.Value, value.UserOverallEloAfter!.Value)));
        ratingEvents.Sort((left, right) => left.AtUtc.CompareTo(right.AtUtc));

+        var ccsRatingEvents = new List<(DateTime AtUtc, int Elo)>();
+        if (ccsRating.HistoryBaselineElo.HasValue && ccsRating.HistoryStartedUtc.HasValue)
+            ccsRatingEvents.Add((ccsRating.HistoryStartedUtc.Value, ccsRating.HistoryBaselineElo.Value));
+        ccsRatingEvents.AddRange(ccsAttemptList
+            .Where(value => value.Rating is { AppliedUtc: not null })
+            .Select(value => (value.Rating!.AppliedUtc!.Value, value.Rating!.UserElo)));
+        ccsRatingEvents.Sort((left, right) => left.AtUtc.CompareTo(right.AtUtc));
+
        int? currentRating = null;
+        int? currentCcsRating = null;
        var eventIndex = 0;
+        var ccsEventIndex = 0;
        var result = new List<DashboardTrendPoint>();
        foreach (var offset in Enumerable.Range(0, 30))
        {
@@ -189,6 +244,11 @@ public sealed class DashboardStore
                currentRating = ratingEvents[eventIndex].Elo;
                eventIndex++;
            }
+            while (ccsEventIndex < ccsRatingEvents.Count && ccsRatingEvents[ccsEventIndex].AtUtc < nextDate)
+            {
+                currentCcsRating = ccsRatingEvents[ccsEventIndex].Elo;
+                ccsEventIndex++;
+            }

            var daily = attemptList
                .Where(value => !value.IsSkip && value.CreatedUtc >= date && value.CreatedUtc < nextDate)
@@ -200,7 +260,8 @@ public sealed class DashboardStore
                Correct = correct,
                Total = daily.Count,
                Accuracy = Percentage(correct, daily.Count),
-                QuestionBankRating = currentRating
+                QuestionBankRating = currentRating,
+                CcsRating = currentCcsRating
            });
        }
        return result;