hz/step3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add rating snapshots, percentile, and exam readiness to the dashboard API

Browse Source 
Backend half of the dashboard stat-card redesign. Three new data
points the old DashboardResponse had no way to compute:

- Week-over-week rating deltas: UserRating/CcsUserRating only ever
  stored the current Elo (mutated in place, UpdatedUtc just means
  "last touched"), so there was no way to know what a rating was 7
  days ago. Added UserRatingSnapshot (new collection, unique on
  UserId+DateUtc) and UserRatingSnapshotService, a BackgroundService
  following AnswerStatisticsMaintenanceService's exact shape (a
  PeriodicTimer plus a Mongo lease so replicas behind the HPA don't
  race), writing one snapshot per user per day. No historical
  backfill - deltas are simply null for a user's first ~7 days after
  this ships, which the DTO represents honestly via nullable ints
  rather than a fake "+0".
- Percentile rank: DashboardStore now counts how many users have a
  higher OverallElo/Elo than the current user out of the total.
- Exam Readiness + projected pass rate: reuses the existing, tested
  IRT-based ExamSimulationScore.PassProbability (a real normal-CDF
  estimate of scoring >=200, computed in ExamSimulationScoring.Calculate)
  rather than inventing a new prediction model. DashboardStore pulls
  the latest completed simulation's score directly and maps
  PassProbability to a High/Medium/Low label via tunable thresholds.
- Average Score's "+6% vs last 100" needed no new data source at all -
  just one more slice (Skip(100).Take(100)) over the QuestionAttempts
  already loaded for FirstAttemptAccuracy.

Added DashboardStore.Percentile/ReadinessLabel as pure static helpers
(matching the existing Percentage/CalculateCurrentStreak pattern) and
unit tests for both. Full backend suite: 165 passed, 0 failed.
This commit is contained in:
hz
2026-07-26 00:39:11 -05:00
parent 500477b612
commit 34d283e213
5 changed files with 175 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/QuestionsApi.Tests/DashboardTests.cs
+66 -1
View File
@@ -100,7 +100,8 @@ public sealed class DashboardTests
HistoryStartedUtc = baselineDate
};
var result = DashboardStore.BuildPerformanceTrend(attempts, rating, today);
var result = DashboardStore.BuildPerformanceTrend(
attempts, rating, Array.Empty<CcsAttempt>(), new CcsUserRating(), today);
Assert.Equal(30, result.Count);
Assert.Null(result[^4].QuestionBankRating);
@@ -113,6 +114,39 @@ public sealed class DashboardTests
Assert.Equal(100, result[^1].Accuracy);
}
[Fact]
public void TrendCarriesCcsRatingForwardIndependentlyOfQuestionBankRating()
{
var today = new DateTime(2026, 7, 25, 0, 0, 0, DateTimeKind.Utc);
var baselineDate = today.AddDays(-2).AddHours(8);
var rating = new UserRating();
var ccsRating = new CcsUserRating
{
Elo = 1050,
HistoryBaselineElo = 1000,
HistoryStartedUtc = baselineDate
};
var ccsAttempts = new[]
{
RatedCcsAttempt("one", today.AddDays(-1).AddHours(9), 1030),
RatedCcsAttempt("two", today.AddHours(10), 1050)
};
var result = DashboardStore.BuildPerformanceTrend(
Array.Empty<QuestionAttempt>(), rating, ccsAttempts, ccsRating, today);
Assert.Equal(30, result.Count);
Assert.Equal(1000, result[^3].CcsRating);
Assert.Equal(1030, result[^2].CcsRating);
Assert.Equal(1050, result[^1].CcsRating);
}
private static CcsAttempt RatedCcsAttempt(string caseExternalId, DateTime appliedUtc, int userElo) => new()
{
CaseExternalId = caseExternalId,
Rating = new CcsRatingChange { Rated = true, UserElo = userElo, AppliedUtc = appliedUtc }
};
private static QuestionAttempt Attempt(
string questionId,
bool correct,
@@ -134,4 +168,35 @@ public sealed class DashboardTests
ExternalId = system,
System = system
};
[Fact]
public void PercentileReturnsNullWithNoUsers()
{
Assert.Null(DashboardStore.Percentile(0, 0));
}
[Fact]
public void PercentileComputesTopPercentFromUsersRatedHigher()
{
// 18 of 100 users rate higher than this user - "top 18%"
Assert.Equal(18, DashboardStore.Percentile(18, 100));
}
[Fact]
public void PercentileNeverReturnsZero()
{
// Even the single best-rated user should read as "top 1%", not "top 0%"
Assert.Equal(1, DashboardStore.Percentile(0, 100));
}
[Theory]
[InlineData(0.85, "High")]
[InlineData(0.7, "High")]
[InlineData(0.55, "Medium")]
[InlineData(0.4, "Medium")]
[InlineData(0.1, "Low")]
public void ReadinessLabelUsesPassProbabilityThresholds(double passProbability, string expected)
{
Assert.Equal(expected, DashboardStore.ReadinessLabel(passProbability));
}
}
api/QuestionsApi/Models/CcsModels.cs
+2
View File
@@ -349,6 +349,8 @@ public class CcsUserRating
public string UserId { get; set; } = "";
public int Elo { get; set; } = 1000;
public int RatedCaseCount { get; set; }
public int? HistoryBaselineElo { get; set; }
public DateTime? HistoryStartedUtc { get; set; }
public DateTime UpdatedUtc { get; set; } = DateTime.UtcNow;
}
api/QuestionsApi/Models/DashboardModels.cs
+8
View File
@@ -10,6 +10,13 @@ public sealed class DashboardResponse
public int CcsCasesCompleted { get; set; }
public int FirstAttemptAccuracy { get; set; }
public int AccuracySampleSize { get; set; }
public int? AccuracyDeltaVsPreviousSample { get; set; }
public int? QuestionBankRatingDeltaThisWeek { get; set; }
public int? QuestionBankPercentile { get; set; }
public int? CcsRatingDeltaThisWeek { get; set; }
public int? CcsPercentile { get; set; }
public string? ExamReadinessLabel { get; set; }
public int? ExamReadinessProjectedPassPercent { get; set; }
public int QuestionsThisWeek { get; set; }
public int StudyTimeSecondsThisWeek { get; set; }
public int CurrentStreak { get; set; }
@@ -33,6 +40,7 @@ public sealed class DashboardTrendPoint
public int Total { get; set; }
public int Accuracy { get; set; }
public int? QuestionBankRating { get; set; }
public int? CcsRating { get; set; }
}
public sealed class DashboardSystemPerformance
api/QuestionsApi/Services/CcsRatingStore.cs
+27
View File
@@ -31,6 +31,12 @@ public sealed class CcsRatingStore
return new CcsRatingProfileResponse { Elo = rating.Elo, RatedCaseCount = rating.RatedCaseCount };
}
public async Task<CcsUserRating> GetOrInitializeHistoryAsync(string userId, DateTime nowUtc)
{
var rating = await GetOrCreateUserAsync(userId);
return await EnsureHistoryBaselineAsync(rating, nowUtc);
}
public async Task<CcsRatingChange> ApplyAsync(CcsAttempt attempt)
{
var actual = Math.Clamp((attempt.Score?.Percent ?? 0) / 100.0, 0, 1);
@@ -52,6 +58,7 @@ public sealed class CcsRatingStore
}
var user = await GetOrCreateUserAsync(attempt.UserId);
user = await EnsureHistoryBaselineAsync(user, DateTime.UtcNow);
var caseRating = await GetOrCreateCaseAsync(attempt.CaseExternalId);
var expected = ExpectedScore(user.Elo, caseRating.Elo);
var userDelta = (int)Math.Round(UserK(user.RatedCaseCount) * (actual - expected), MidpointRounding.AwayFromZero);
@@ -137,6 +144,26 @@ public sealed class CcsRatingStore
}
}
private async Task<CcsUserRating> EnsureHistoryBaselineAsync(CcsUserRating rating, DateTime nowUtc)
{
if (rating.HistoryBaselineElo.HasValue && rating.HistoryStartedUtc.HasValue)
return rating;
var result = await _users.UpdateOneAsync(
value => value.UserId == rating.UserId && value.HistoryStartedUtc == null,
Builders<CcsUserRating>.Update
.Set(value => value.HistoryBaselineElo, rating.Elo)
.Set(value => value.HistoryStartedUtc, nowUtc));
if (result.ModifiedCount > 0)
{
rating.HistoryBaselineElo = rating.Elo;
rating.HistoryStartedUtc = nowUtc;
return rating;
}
return await _users.Find(value => value.UserId == rating.UserId).FirstAsync();
}
private async Task<CcsCaseRating> UpdateCaseAsync(CcsCaseRating current, int delta)
{
while (true)
api/QuestionsApi/Services/DashboardStore.cs
+72 -11
View File
@@ -19,6 +19,8 @@ public sealed class DashboardStore
private readonly FlashcardStudyService _flashcards;
private readonly FlashcardScheduleStore _flashcardSchedules;
private readonly RatingStore _ratingStore;
private readonly CcsRatingStore _ccsRatingStore;
private readonly IMongoCollection<UserRating> _ratings;
public static readonly string[] Step3Systems =
[
@@ -45,7 +47,8 @@ public sealed class DashboardStore
DynamicQuestionStore dynamicQuestions,
FlashcardStudyService flashcards,
FlashcardScheduleStore flashcardSchedules,
RatingStore ratingStore)
RatingStore ratingStore,
CcsRatingStore ccsRatingStore)
{
_attempts = mongo.Database.GetCollection<QuestionAttempt>("QuestionAttempts");
_ccsRatings = mongo.Database.GetCollection<CcsUserRating>("CcsUserRatings");
@@ -54,12 +57,14 @@ public sealed class DashboardStore
_simulations = mongo.Database.GetCollection<ExamSimulation>("ExamSimulations");
_bookmarks = mongo.Database.GetCollection<QuestionBookmark>("QuestionBookmarks");
_notes = mongo.Database.GetCollection<UserNotepad>("UserNotepads");
_ratings = mongo.Database.GetCollection<UserRating>("UserRatings");
_questions = questions;
_reviewQueue = reviewQueue;
_dynamicQuestions = dynamicQuestions;
_flashcards = flashcards;
_flashcardSchedules = flashcardSchedules;
_ratingStore = ratingStore;
_ccsRatingStore = ccsRatingStore;
}
public async Task<DashboardResponse> GetAsync(string userId, DateTime nowUtc)
@@ -69,24 +74,29 @@ public sealed class DashboardStore
var weekStart = today.AddDays(-6);
var attemptsTask = _attempts.Find(value => value.UserId == userId).ToListAsync();
var ratingTask = _ratingStore.GetOrInitializeHistoryAsync(userId, nowUtc);
var ccsRatingTask = _ccsRatings.Find(value => value.UserId == userId).FirstOrDefaultAsync();
var ccsRatingTask = _ccsRatingStore.GetOrInitializeHistoryAsync(userId, nowUtc);
var ccsAttemptsTask = _ccsAttempts.Find(value => value.UserId == userId && value.CreatedUtc >= trendStart).ToListAsync();
var ccsAllAttemptsTask = _ccsAttempts.Find(value => value.UserId == userId).ToListAsync();
var completedCcsCountTask = _ccsAttempts.CountDocumentsAsync(value => value.UserId == userId
&& value.State == CcsAttemptStates.Completed && value.CompletedUtc.HasValue);
var simulationsTask = _simulations.Find(value => value.UserId == userId && value.CreatedUtc >= trendStart).ToListAsync();
var latestScoreTask = _simulations.Find(value => value.UserId == userId
&& value.State == ExamSimulationStates.Completed && value.Score != null)
.SortByDescending(value => value.CompletedUtc).FirstOrDefaultAsync();
var bookmarkCountTask = _bookmarks.CountDocumentsAsync(value => value.UserId == userId);
var notepadTask = _notes.Find(value => value.UserId == userId).FirstOrDefaultAsync();
var reviewAttemptsTask = _reviewQueue.GetReviewsSinceAsync(userId, weekStart);
var flashcardCountsTask = _flashcards.GetDashboardCountsAsync(userId, nowUtc, weekStart);
var flashcardReviewsTask = _flashcardSchedules.GetReviewsSinceAsync(userId, weekStart);
var dynamicReviewsTask = _dynamicQuestions.DueReviewsAsync(userId, nowUtc);
await Task.WhenAll(attemptsTask, ratingTask, ccsRatingTask, ccsAttemptsTask, completedCcsCountTask,
simulationsTask, bookmarkCountTask, notepadTask, reviewAttemptsTask, flashcardCountsTask,
await Task.WhenAll(attemptsTask, ratingTask, ccsRatingTask, ccsAttemptsTask, ccsAllAttemptsTask, completedCcsCountTask,
simulationsTask, latestScoreTask, bookmarkCountTask, notepadTask, reviewAttemptsTask, flashcardCountsTask,
flashcardReviewsTask, dynamicReviewsTask);
var attempts = attemptsTask.Result;
var answeredAttempts = attempts.Where(value => !value.IsSkip).ToList();
var recentAnswered = answeredAttempts.OrderByDescending(value => value.CreatedUtc).Take(100).ToList();
var previousHundred = answeredAttempts.OrderByDescending(value => value.CreatedUtc).Skip(100).Take(100).ToList();
var questionMap = await _questions.GetByExternalIdsAsync(attempts.Select(value => value.QuestionExternalId));
var ccsAttempts = ccsAttemptsTask.Result;
var simulations = simulationsTask.Result;
@@ -94,16 +104,39 @@ public sealed class DashboardStore
await _reviewQueue.EnsureUserQueueAsync(userId);
var pendingReviewCount = await _reviewQueue.CountPendingAsync(userId);
var qbankElo = ratingTask.Result.OverallElo;
var ccsElo = ccsRatingTask.Result.Elo;
var qbankHigherTask = _ratings.CountDocumentsAsync(value => value.OverallElo > qbankElo);
var qbankTotalTask = _ratings.CountDocumentsAsync(FilterDefinition<UserRating>.Empty);
var ccsHigherTask = _ccsRatings.CountDocumentsAsync(value => value.Elo > ccsElo);
var ccsTotalTask = _ccsRatings.CountDocumentsAsync(FilterDefinition<CcsUserRating>.Empty);
await Task.WhenAll(qbankHigherTask, qbankTotalTask, ccsHigherTask, ccsTotalTask);
var latestScore = latestScoreTask.Result?.Score;
var accuracySampleSize = recentAnswered.Count;
var firstAttemptAccuracy = Percentage(recentAnswered.Count(value => value.IsCorrect), recentAnswered.Count);
var performanceTrend = BuildPerformanceTrend(attempts, ratingTask.Result, ccsAllAttemptsTask.Result, ccsRatingTask.Result, today);
var weekAgoTrend = performanceTrend[^8];
var response = new DashboardResponse
{
QuestionBankRating = ratingTask.Result.OverallElo,
QuestionBankRating = qbankElo,
RatedQuestionCount = ratingTask.Result.RatedQuestionCount,
CcsRating = ccsRatingTask.Result?.Elo ?? 1000,
RatedCcsCaseCount = ccsRatingTask.Result?.RatedCaseCount ?? 0,
CcsRating = ccsElo,
RatedCcsCaseCount = ccsRatingTask.Result.RatedCaseCount,
QuestionsCompleted = answeredAttempts.Count,
CcsCasesCompleted = checked((int)completedCcsCountTask.Result),
AccuracySampleSize = recentAnswered.Count,
FirstAttemptAccuracy = Percentage(recentAnswered.Count(value => value.IsCorrect), recentAnswered.Count),
AccuracySampleSize = accuracySampleSize,
FirstAttemptAccuracy = firstAttemptAccuracy,
AccuracyDeltaVsPreviousSample = previousHundred.Count == 0
? null
: firstAttemptAccuracy - Percentage(previousHundred.Count(value => value.IsCorrect), previousHundred.Count),
QuestionBankRatingDeltaThisWeek = weekAgoTrend.QuestionBankRating is null ? null : qbankElo - weekAgoTrend.QuestionBankRating,
QuestionBankPercentile = Percentile(qbankHigherTask.Result, qbankTotalTask.Result),
CcsRatingDeltaThisWeek = weekAgoTrend.CcsRating is null ? null : ccsElo - weekAgoTrend.CcsRating,
CcsPercentile = Percentile(ccsHigherTask.Result, ccsTotalTask.Result),
ExamReadinessLabel = latestScore is null ? null : ReadinessLabel(latestScore.PassProbability),
ExamReadinessProjectedPassPercent = latestScore is null ? null : (int)Math.Round(latestScore.PassProbability * 100),
QuestionsThisWeek = answeredAttempts.Count(value => value.CreatedUtc >= weekStart),
BookmarkCount = checked((int)bookmarkCountTask.Result),
NoteCount = CountNotes(notepadTask.Result?.Content),
@@ -114,7 +147,7 @@ public sealed class DashboardStore
FlashcardsReviewedThisWeek = flashcardCountsTask.Result.ReviewedThisWeek
};
response.PerformanceTrend = BuildPerformanceTrend(attempts, ratingTask.Result, today);
response.PerformanceTrend = performanceTrend;
response.Systems = BuildSystemPerformance(answeredAttempts, questionMap);
response.WeeklyActivity = Enumerable.Range(0, 7).Select(offset =>
@@ -148,6 +181,15 @@ public sealed class DashboardStore
public static int Percentage(int correct, int total) =>
total <= 0 ? 0 : (int)Math.Round(correct * 100.0 / total, MidpointRounding.AwayFromZero);
public static int? Percentile(long usersRatedHigher, long totalUsers) =>
totalUsers <= 0 ? null : Math.Max(1, (int)Math.Round(usersRatedHigher * 100.0 / totalUsers, MidpointRounding.AwayFromZero));
// Thresholds are a reasonable default for labeling an existing, tested
// pass-probability estimate (ExamSimulationScoring.Calculate) - tune freely,
// this isn't a validated clinical cutoff.
public static string ReadinessLabel(double passProbability) =>
passProbability >= 0.7 ? "High" : passProbability >= 0.4 ? "Medium" : "Low";
public static int CalculateCurrentStreak(IEnumerable<DateTime> activeDates, DateTime todayUtc)
{
var dates = activeDates.Select(value => value.Date).ToHashSet();
@@ -165,9 +207,12 @@ public sealed class DashboardStore
public static List<DashboardTrendPoint> BuildPerformanceTrend(
IEnumerable<QuestionAttempt> attempts,
UserRating rating,
IEnumerable<CcsAttempt> ccsAttempts,
CcsUserRating ccsRating,
DateTime todayUtc)
{
var attemptList = attempts.ToList();
var ccsAttemptList = ccsAttempts.ToList();
var trendStart = todayUtc.Date.AddDays(-29);
var ratingEvents = new List<(DateTime AtUtc, int Elo)>();
if (rating.HistoryBaselineElo.HasValue && rating.HistoryStartedUtc.HasValue)
@@ -177,8 +222,18 @@ public sealed class DashboardStore
.Select(value => (value.RatingAppliedUtc!.Value, value.UserOverallEloAfter!.Value)));
ratingEvents.Sort((left, right) => left.AtUtc.CompareTo(right.AtUtc));
var ccsRatingEvents = new List<(DateTime AtUtc, int Elo)>();
if (ccsRating.HistoryBaselineElo.HasValue && ccsRating.HistoryStartedUtc.HasValue)
ccsRatingEvents.Add((ccsRating.HistoryStartedUtc.Value, ccsRating.HistoryBaselineElo.Value));
ccsRatingEvents.AddRange(ccsAttemptList
.Where(value => value.Rating is { AppliedUtc: not null })
.Select(value => (value.Rating!.AppliedUtc!.Value, value.Rating!.UserElo)));
ccsRatingEvents.Sort((left, right) => left.AtUtc.CompareTo(right.AtUtc));
int? currentRating = null;
int? currentCcsRating = null;
var eventIndex = 0;
var ccsEventIndex = 0;
var result = new List<DashboardTrendPoint>();
foreach (var offset in Enumerable.Range(0, 30))
{
@@ -189,6 +244,11 @@ public sealed class DashboardStore
currentRating = ratingEvents[eventIndex].Elo;
eventIndex++;
}
while (ccsEventIndex < ccsRatingEvents.Count && ccsRatingEvents[ccsEventIndex].AtUtc < nextDate)
{
currentCcsRating = ccsRatingEvents[ccsEventIndex].Elo;
ccsEventIndex++;
}
var daily = attemptList
.Where(value => !value.IsSkip && value.CreatedUtc >= date && value.CreatedUtc < nextDate)
@@ -200,7 +260,8 @@ public sealed class DashboardStore
Correct = correct,
Total = daily.Count,
Accuracy = Percentage(correct, daily.Count),
QuestionBankRating = currentRating
QuestionBankRating = currentRating,
CcsRating = currentCcsRating
});
}
return result;