Files
step3/api/QuestionsApi/Services/DashboardStore.cs
T
hzandClaude Sonnet 5 74398133f7 Exclude sample exam simulations from the dashboard's readiness score
The SUPER-role "preview results" QA tooling seeds a completed
ExamSimulation with scripted (not real) answers, flagged IsSample.
The dashboard's readiness lookup picked whichever completed
simulation was most recent regardless of that flag, so running the
QA preview on an account could override its real Exam Readiness card
with a synthetic pass probability. Recent-activity/weekly-activity
queries got the same exclusion since they read from the same
collection for the same reason.

Co-Authored-By: Claude Sonnet 5 <[email protected]>
2026-07-26 14:47:07 -05:00

415 lines
22 KiB
C#

using MongoDB.Driver;
using QuestionsApi.Models;
using QuestionsApi.Services.Flashcards;
namespace QuestionsApi.Services;
public sealed class DashboardStore
{
private readonly IMongoCollection<QuestionAttempt> _attempts;
private readonly IMongoCollection<CcsUserRating> _ccsRatings;
private readonly IMongoCollection<CcsAttempt> _ccsAttempts;
private readonly IMongoCollection<CcsCaseRevision> _ccsCases;
private readonly IMongoCollection<ExamSimulation> _simulations;
private readonly IMongoCollection<QuestionBookmark> _bookmarks;
private readonly IMongoCollection<UserNotepad> _notes;
private readonly QuestionStore _questions;
private readonly ReviewQueueStore _reviewQueue;
private readonly DynamicQuestionStore _dynamicQuestions;
private readonly FlashcardStudyService _flashcards;
private readonly FlashcardScheduleStore _flashcardSchedules;
private readonly RatingStore _ratingStore;
private readonly CcsRatingStore _ccsRatingStore;
private readonly IMongoCollection<UserRating> _ratings;
public static readonly string[] Step3Systems =
[
"Human Development",
"Immune System, Blood & Lymphoreticular System, and Multisystem Processes/Disorders",
"Behavioral Health",
"Nervous System & Special Senses",
"Skin & Subcutaneous Tissue",
"Musculoskeletal System",
"Cardiovascular System",
"Respiratory System",
"Gastrointestinal System",
"Renal/Urinary & Male Reproductive Systems",
"Pregnancy/Childbirth & Female Reproductive System & Breast",
"Endocrine System",
"Biostatistics & Epidemiology/Population Health & Interpretation of the Medical Literature",
"Social Sciences: Communication Skills/Ethics/Patient Safety"
];
public DashboardStore(
MongoDatabaseProvider mongo,
QuestionStore questions,
ReviewQueueStore reviewQueue,
DynamicQuestionStore dynamicQuestions,
FlashcardStudyService flashcards,
FlashcardScheduleStore flashcardSchedules,
RatingStore ratingStore,
CcsRatingStore ccsRatingStore)
{
_attempts = mongo.Database.GetCollection<QuestionAttempt>("QuestionAttempts");
_ccsRatings = mongo.Database.GetCollection<CcsUserRating>("CcsUserRatings");
_ccsAttempts = mongo.Database.GetCollection<CcsAttempt>("CcsAttempts");
_ccsCases = mongo.Database.GetCollection<CcsCaseRevision>("CcsCaseRevisions");
_simulations = mongo.Database.GetCollection<ExamSimulation>("ExamSimulations");
_bookmarks = mongo.Database.GetCollection<QuestionBookmark>("QuestionBookmarks");
_notes = mongo.Database.GetCollection<UserNotepad>("UserNotepads");
_ratings = mongo.Database.GetCollection<UserRating>("UserRatings");
_questions = questions;
_reviewQueue = reviewQueue;
_dynamicQuestions = dynamicQuestions;
_flashcards = flashcards;
_flashcardSchedules = flashcardSchedules;
_ratingStore = ratingStore;
_ccsRatingStore = ccsRatingStore;
}
public async Task<DashboardResponse> GetAsync(string userId, DateTime nowUtc)
{
var today = nowUtc.Date;
var trendStart = today.AddDays(-29);
var weekStart = today.AddDays(-6);
var attemptsTask = _attempts.Find(value => value.UserId == userId).ToListAsync();
var ratingTask = _ratingStore.GetOrInitializeHistoryAsync(userId, nowUtc);
var ccsRatingTask = _ccsRatingStore.GetOrInitializeHistoryAsync(userId, nowUtc);
var ccsAttemptsTask = _ccsAttempts.Find(value => value.UserId == userId && value.CreatedUtc >= trendStart).ToListAsync();
var ccsAllAttemptsTask = _ccsAttempts.Find(value => value.UserId == userId).ToListAsync();
var completedCcsCountTask = _ccsAttempts.CountDocumentsAsync(value => value.UserId == userId
&& value.State == CcsAttemptStates.Completed && value.CompletedUtc.HasValue);
var simulationsTask = _simulations.Find(value => value.UserId == userId && !value.IsSample && value.CreatedUtc >= trendStart).ToListAsync();
var completedSimulationsTask = _simulations.Find(value => value.UserId == userId
&& value.State == ExamSimulationStates.Completed && value.Score != null)
.SortByDescending(value => value.CompletedUtc).ToListAsync();
var bookmarkCountTask = _bookmarks.CountDocumentsAsync(value => value.UserId == userId);
var notepadTask = _notes.Find(value => value.UserId == userId).FirstOrDefaultAsync();
var reviewAttemptsTask = _reviewQueue.GetReviewsSinceAsync(userId, weekStart);
var flashcardCountsTask = _flashcards.GetDashboardCountsAsync(userId, nowUtc, weekStart);
var flashcardReviewsTask = _flashcardSchedules.GetReviewsSinceAsync(userId, weekStart);
var dynamicReviewsTask = _dynamicQuestions.DueReviewsAsync(userId, nowUtc);
await Task.WhenAll(attemptsTask, ratingTask, ccsRatingTask, ccsAttemptsTask, ccsAllAttemptsTask, completedCcsCountTask,
simulationsTask, completedSimulationsTask, bookmarkCountTask, notepadTask, reviewAttemptsTask, flashcardCountsTask,
flashcardReviewsTask, dynamicReviewsTask);
var attempts = attemptsTask.Result;
var answeredAttempts = attempts.Where(value => !value.IsSkip).ToList();
var recentAnswered = answeredAttempts.OrderByDescending(value => value.CreatedUtc).Take(100).ToList();
var previousHundred = answeredAttempts.OrderByDescending(value => value.CreatedUtc).Skip(100).Take(100).ToList();
var questionMap = await _questions.GetByExternalIdsAsync(attempts.Select(value => value.QuestionExternalId));
var ccsAttempts = ccsAttemptsTask.Result;
var simulations = simulationsTask.Result;
var completedCcs = ccsAttempts.Where(value => value.State == CcsAttemptStates.Completed && value.CompletedUtc.HasValue).ToList();
await _reviewQueue.EnsureUserQueueAsync(userId);
var pendingReviewCount = await _reviewQueue.CountPendingAsync(userId);
var qbankElo = ratingTask.Result.OverallElo;
var ccsElo = ccsRatingTask.Result.Elo;
var qbankHigherTask = _ratings.CountDocumentsAsync(value => value.OverallElo > qbankElo);
var qbankTotalTask = _ratings.CountDocumentsAsync(FilterDefinition<UserRating>.Empty);
var ccsHigherTask = _ccsRatings.CountDocumentsAsync(value => value.Elo > ccsElo);
var ccsTotalTask = _ccsRatings.CountDocumentsAsync(FilterDefinition<CcsUserRating>.Empty);
await Task.WhenAll(qbankHigherTask, qbankTotalTask, ccsHigherTask, ccsTotalTask);
var latestScore = SelectLatestReadinessScore(completedSimulationsTask.Result);
var accuracySampleSize = recentAnswered.Count;
var firstAttemptAccuracy = Percentage(recentAnswered.Count(value => value.IsCorrect), recentAnswered.Count);
var performanceTrend = BuildPerformanceTrend(attempts, ratingTask.Result, ccsAllAttemptsTask.Result, ccsRatingTask.Result, today);
var weekAgoTrend = performanceTrend[^8];
var response = new DashboardResponse
{
QuestionBankRating = qbankElo,
RatedQuestionCount = ratingTask.Result.RatedQuestionCount,
CcsRating = ccsElo,
RatedCcsCaseCount = ccsRatingTask.Result.RatedCaseCount,
QuestionsCompleted = answeredAttempts.Count,
CcsCasesCompleted = checked((int)completedCcsCountTask.Result),
AccuracySampleSize = accuracySampleSize,
FirstAttemptAccuracy = firstAttemptAccuracy,
AccuracyDeltaVsPreviousSample = previousHundred.Count == 0
? null
: firstAttemptAccuracy - Percentage(previousHundred.Count(value => value.IsCorrect), previousHundred.Count),
QuestionBankRatingDeltaThisWeek = weekAgoTrend.QuestionBankRating is null ? null : qbankElo - weekAgoTrend.QuestionBankRating,
QuestionBankPercentile = Percentile(qbankHigherTask.Result, qbankTotalTask.Result),
CcsRatingDeltaThisWeek = weekAgoTrend.CcsRating is null ? null : ccsElo - weekAgoTrend.CcsRating,
CcsPercentile = Percentile(ccsHigherTask.Result, ccsTotalTask.Result),
ExamReadinessLabel = latestScore is null ? null : ReadinessLabel(latestScore.PassProbability),
ExamReadinessProjectedPassPercent = latestScore is null ? null : (int)Math.Round(latestScore.PassProbability * 100),
QuestionsThisWeek = answeredAttempts.Count(value => value.CreatedUtc >= weekStart),
BookmarkCount = checked((int)bookmarkCountTask.Result),
NoteCount = CountNotes(notepadTask.Result?.Content),
MissedQuestionCount = checked((int)pendingReviewCount),
DynamicReviewsDue = dynamicReviewsTask.Result.Count,
FlashcardsDue = flashcardCountsTask.Result.Due,
FlashcardsNewAvailable = flashcardCountsTask.Result.NewAvailable,
FlashcardsReviewedThisWeek = flashcardCountsTask.Result.ReviewedThisWeek
};
response.PerformanceTrend = performanceTrend;
response.Systems = BuildSystemPerformance(answeredAttempts, questionMap);
response.WeeklyActivity = Enumerable.Range(0, 7).Select(offset =>
{
var date = weekStart.AddDays(offset);
var dailyQuestions = answeredAttempts.Where(value => value.CreatedUtc.Date == date).ToList();
var dailyCcs = completedCcs.Where(value => value.CompletedUtc!.Value.Date == date).ToList();
var dailySimulations = simulations.Where(value => value.CompletedUtc?.Date == date).ToList();
return new DashboardActivityDay
{
DateUtc = date,
Questions = dailyQuestions.Count,
CcsCases = dailyCcs.Count,
StudyTimeSeconds = dailyQuestions.Sum(value => value.DurationSeconds)
+ dailyCcs.Sum(CcsDurationSeconds)
+ dailySimulations.Sum(value => value.Responses.Sum(item => Math.Max(0, item.ElapsedSeconds)))
+ reviewAttemptsTask.Result.Where(value => value.CreatedUtc.Date == date).Sum(value => value.DurationSeconds)
+ flashcardReviewsTask.Result.Where(value => value.ReviewedUtc.Date == date).Sum(value => value.DurationSeconds)
};
}).ToList();
response.StudyTimeSecondsThisWeek = response.WeeklyActivity.Sum(value => value.StudyTimeSeconds);
var activeDates = attempts.Select(value => value.CreatedUtc.Date)
.Concat(completedCcs.Select(value => value.CompletedUtc!.Value.Date))
.Concat(simulations.Where(value => value.CompletedUtc.HasValue).Select(value => value.CompletedUtc!.Value.Date))
.Concat(flashcardReviewsTask.Result.Select(value => value.ReviewedUtc.Date));
response.CurrentStreak = CalculateCurrentStreak(activeDates, today);
response.RecentActivity = await BuildRecentActivityAsync(answeredAttempts, questionMap, completedCcs, simulations);
return response;
}
public static int Percentage(int correct, int total) =>
total <= 0 ? 0 : (int)Math.Round(correct * 100.0 / total, MidpointRounding.AwayFromZero);
public static int? Percentile(long usersRatedHigher, long totalUsers) =>
totalUsers <= 0 ? null : Math.Max(1, (int)Math.Round(usersRatedHigher * 100.0 / totalUsers, MidpointRounding.AwayFromZero));
// Thresholds are a reasonable default for labeling an existing, tested
// pass-probability estimate (ExamSimulationScoring.Calculate) - tune freely,
// this isn't a validated clinical cutoff.
public static string ReadinessLabel(double passProbability) =>
passProbability >= 0.7 ? "High" : passProbability >= 0.4 ? "Medium" : "Low";
// completedSimulationsByRecency must already be sorted newest-first (by CompletedUtc).
// Sample simulations (QA-only, seeded via the SUPER-role "preview results" tooling with
// scripted answers) never reflect real performance, so they're skipped here rather than
// in the Mongo filter - keeping the "which one counts" decision unit-testable.
public static ExamSimulationScore? SelectLatestReadinessScore(IEnumerable<ExamSimulation> completedSimulationsByRecency) =>
completedSimulationsByRecency.FirstOrDefault(value => !value.IsSample)?.Score;
public static int CalculateCurrentStreak(IEnumerable<DateTime> activeDates, DateTime todayUtc)
{
var dates = activeDates.Select(value => value.Date).ToHashSet();
var cursor = todayUtc.Date;
if (!dates.Contains(cursor)) cursor = cursor.AddDays(-1);
var streak = 0;
while (dates.Contains(cursor))
{
streak++;
cursor = cursor.AddDays(-1);
}
return streak;
}
public static List<DashboardTrendPoint> BuildPerformanceTrend(
IEnumerable<QuestionAttempt> attempts,
UserRating rating,
IEnumerable<CcsAttempt> ccsAttempts,
CcsUserRating ccsRating,
DateTime todayUtc)
{
var attemptList = attempts.ToList();
var ccsAttemptList = ccsAttempts.ToList();
var trendStart = todayUtc.Date.AddDays(-29);
var ratingEvents = new List<(DateTime AtUtc, int Elo)>();
if (rating.HistoryBaselineElo.HasValue && rating.HistoryStartedUtc.HasValue)
ratingEvents.Add((rating.HistoryStartedUtc.Value, rating.HistoryBaselineElo.Value));
ratingEvents.AddRange(attemptList
.Where(value => value.RatingAppliedUtc.HasValue && value.UserOverallEloAfter.HasValue)
.Select(value => (value.RatingAppliedUtc!.Value, value.UserOverallEloAfter!.Value)));
ratingEvents.Sort((left, right) => left.AtUtc.CompareTo(right.AtUtc));
var ccsRatingEvents = new List<(DateTime AtUtc, int Elo)>();
if (ccsRating.HistoryBaselineElo.HasValue && ccsRating.HistoryStartedUtc.HasValue)
ccsRatingEvents.Add((ccsRating.HistoryStartedUtc.Value, ccsRating.HistoryBaselineElo.Value));
ccsRatingEvents.AddRange(ccsAttemptList
.Where(value => value.Rating is { AppliedUtc: not null })
.Select(value => (value.Rating!.AppliedUtc!.Value, value.Rating!.UserElo)));
ccsRatingEvents.Sort((left, right) => left.AtUtc.CompareTo(right.AtUtc));
int? currentRating = null;
int? currentCcsRating = null;
var eventIndex = 0;
var ccsEventIndex = 0;
var result = new List<DashboardTrendPoint>();
foreach (var offset in Enumerable.Range(0, 30))
{
var date = trendStart.AddDays(offset);
var nextDate = date.AddDays(1);
while (eventIndex < ratingEvents.Count && ratingEvents[eventIndex].AtUtc < nextDate)
{
currentRating = ratingEvents[eventIndex].Elo;
eventIndex++;
}
while (ccsEventIndex < ccsRatingEvents.Count && ccsRatingEvents[ccsEventIndex].AtUtc < nextDate)
{
currentCcsRating = ccsRatingEvents[ccsEventIndex].Elo;
ccsEventIndex++;
}
var daily = attemptList
.Where(value => !value.IsSkip && value.CreatedUtc >= date && value.CreatedUtc < nextDate)
.ToList();
var correct = daily.Count(value => value.IsCorrect);
result.Add(new DashboardTrendPoint
{
DateUtc = date,
Correct = correct,
Total = daily.Count,
Accuracy = Percentage(correct, daily.Count),
QuestionBankRating = currentRating,
CcsRating = currentCcsRating
});
}
return result;
}
public static List<DashboardSystemPerformance> BuildSystemPerformance(
IEnumerable<QuestionAttempt> attempts,
IReadOnlyDictionary<string, Question> questionMap)
{
var totals = Step3Systems.ToDictionary(
system => system,
_ => (Correct: 0, Total: 0),
StringComparer.Ordinal);
foreach (var attempt in attempts.Where(value => !value.IsSkip))
{
if (!questionMap.TryGetValue(attempt.QuestionExternalId, out var question))
continue;
var system = NormalizeStep3System(question);
if (system is null)
continue;
var current = totals[system];
totals[system] = (current.Correct + (attempt.IsCorrect ? 1 : 0), current.Total + 1);
}
return Step3Systems.Select(system => new DashboardSystemPerformance
{
System = system,
Correct = totals[system].Correct,
Total = totals[system].Total,
Accuracy = Percentage(totals[system].Correct, totals[system].Total)
}).ToList();
}
public static string? NormalizeStep3System(Question question)
{
foreach (var candidate in new[] { question.System, question.PerformanceCategory, question.Discipline })
{
var normalized = NormalizeStep3System(candidate);
if (normalized is not null)
return normalized;
}
return null;
}
public static string? NormalizeStep3System(string? value)
{
if (string.IsNullOrWhiteSpace(value))
return null;
var exact = Step3Systems.FirstOrDefault(
system => string.Equals(system, value.Trim(), StringComparison.OrdinalIgnoreCase));
if (exact is not null)
return exact;
var lower = value.Trim().ToLowerInvariant();
if (ContainsAny(lower, "human development", "well patient", "preventive", "health maintenance", "geriatr", "age-related"))
return Step3Systems[0];
if (ContainsAny(lower, "social science", "communication", "ethic", "patient safety", "professionalism", "systems-based"))
return Step3Systems[13];
if (ContainsAny(lower, "pregnan", "obstetric", "gynec", "female reproductive", "breast", "puerper"))
return Step3Systems[10];
if (ContainsAny(lower, "renal", "urinary", "male reproductive", "urolog", "nephro"))
return Step3Systems[9];
if (ContainsAny(lower, "biostat", "epidemi", "population health", "medical literature"))
return Step3Systems[12];
if (ContainsAny(lower, "immune", "immun", "blood", "hemat", "lymph", "multisystem", "infectious"))
return Step3Systems[1];
if (ContainsAny(lower, "behavior", "psychi", "mental health"))
return Step3Systems[2];
if (ContainsAny(lower, "nervous", "neuro", "special sense", "ophthalm", "otolaryng", " ent"))
return Step3Systems[3];
if (ContainsAny(lower, "skin", "derm", "subcutaneous"))
return Step3Systems[4];
if (ContainsAny(lower, "musculo", "orthop", "rheumat"))
return Step3Systems[5];
if (ContainsAny(lower, "cardio", "circulatory"))
return Step3Systems[6];
if (ContainsAny(lower, "resp", "pulmon"))
return Step3Systems[7];
if (ContainsAny(lower, "gastro", "digest", "hepato"))
return Step3Systems[8];
if (ContainsAny(lower, "endocr", "metabolic"))
return Step3Systems[11];
return null;
}
private static bool ContainsAny(string value, params string[] fragments) =>
fragments.Any(value.Contains);
private async Task<List<DashboardRecentActivity>> BuildRecentActivityAsync(
List<QuestionAttempt> attempts,
Dictionary<string, Question> questionMap,
List<CcsAttempt> ccsAttempts,
List<ExamSimulation> simulations)
{
var caseIds = ccsAttempts.Select(value => value.CaseExternalId).Distinct().ToList();
var caseTitles = caseIds.Count == 0
? []
: await _ccsCases.Find(value => caseIds.Contains(value.ExternalId) && value.IsCurrent).ToListAsync();
var titleMap = caseTitles.GroupBy(value => value.ExternalId).ToDictionary(group => group.Key, group => group.First().Title);
var activity = attempts.OrderByDescending(value => value.CreatedUtc).Take(6).Select(value =>
{
questionMap.TryGetValue(value.QuestionExternalId, out var question);
return new DashboardRecentActivity
{
Type = "question",
Title = $"Answered {NormalizeSystem(question?.System)} question",
Detail = value.IsCorrect ? "Correct" : "Incorrect",
OccurredUtc = value.CreatedUtc,
Route = "question_bank"
};
}).ToList();
activity.AddRange(ccsAttempts.OrderByDescending(value => value.CompletedUtc).Take(4).Select(value =>
new DashboardRecentActivity
{
Type = "ccs",
Title = $"Completed {titleMap.GetValueOrDefault(value.CaseExternalId, "CCS case")}",
Detail = $"{Percentage(value.Score?.EarnedPoints ?? 0, value.Score?.PossiblePoints ?? 0)}%",
OccurredUtc = value.CompletedUtc ?? value.UpdatedUtc,
Route = "ccs"
}));
activity.AddRange(simulations.Where(value => value.State == ExamSimulationStates.Completed)
.OrderByDescending(value => value.CompletedUtc).Take(3).Select(value =>
new DashboardRecentActivity
{
Type = "simulation",
Title = "Completed Step 3 simulation",
Detail = value.Score is null ? "Completed" : $"Predicted score {value.Score.PredictedScore}",
OccurredUtc = value.CompletedUtc ?? value.UpdatedUtc,
Route = "exams"
}));
return activity.OrderByDescending(value => value.OccurredUtc).Take(6).ToList();
}
private static int CcsDurationSeconds(CcsAttempt attempt) => attempt.CompletedUtc.HasValue
? Math.Max(0, (int)(attempt.CompletedUtc.Value - attempt.CreatedUtc).TotalSeconds)
: 0;
private static int CountNotes(string? content) => string.IsNullOrWhiteSpace(content)
? 0
: content.Split('\n').Count(line => !string.IsNullOrWhiteSpace(line));
private static string NormalizeSystem(string? value) => string.IsNullOrWhiteSpace(value) ? "General" : value.Trim();
}