hz/step3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add dashboard performance trends

Browse Source
This commit is contained in:
hz
2026-07-25 22:43:34 -05:00
parent 46953ee71a
commit 500477b612
15 changed files with 640 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/QuestionsApi/Controllers/QuestionsController.cs
+2 -2
View File
@@ -187,7 +187,7 @@ public class QuestionsController : ControllerBase
record.Attempt.IsCorrect,
shouldRate);
if (shouldRate)
await _attempts.MarkRatingAppliedAsync(record.Attempt.Id ?? "", ratingOwner);
await _attempts.MarkRatingAppliedAsync(record.Attempt.Id ?? "", ratingOwner, rating.UserOverallElo);
}
catch
{
@@ -286,7 +286,7 @@ public class QuestionsController : ControllerBase
{
rating = await _ratings.ApplyFirstResultAsync(userId, question, false, shouldRate);
if (shouldRate)
await _attempts.MarkRatingAppliedAsync(record.Attempt.Id ?? "", ratingOwner);
await _attempts.MarkRatingAppliedAsync(record.Attempt.Id ?? "", ratingOwner, rating.UserOverallElo);
}
catch
{
api/QuestionsApi/Models/DashboardModels.cs
+1
View File
@@ -32,6 +32,7 @@ public sealed class DashboardTrendPoint
public int Correct { get; set; }
public int Total { get; set; }
public int Accuracy { get; set; }
public int? QuestionBankRating { get; set; }
}
public sealed class DashboardSystemPerformance
api/QuestionsApi/Models/QuestionAttempt.cs
+2
View File
@@ -19,6 +19,8 @@ public class QuestionAttempt
public int DurationSeconds { get; set; }
public int CounterBucket { get; set; }
public bool? RatingApplied { get; set; }
public int? UserOverallEloAfter { get; set; }
public DateTime? RatingAppliedUtc { get; set; }
public string RatingLeaseOwner { get; set; } = "";
public DateTime RatingLeaseUntilUtc { get; set; }
public DateTime CreatedUtc { get; set; } = DateTime.UtcNow;
api/QuestionsApi/Models/RatingModels.cs
+2
View File
@@ -13,6 +13,8 @@ public class UserRating
public int OverallElo { get; set; } = 1000;
public Dictionary<string, int> CategoryElos { get; set; } = [];
public int RatedQuestionCount { get; set; }
public int? HistoryBaselineElo { get; set; }
public DateTime? HistoryStartedUtc { get; set; }
public DateTime UpdatedUtc { get; set; } = DateTime.UtcNow;
}
api/QuestionsApi/Services/DashboardStore.cs
+154 -31
View File
@@ -7,7 +7,6 @@ namespace QuestionsApi.Services;
public sealed class DashboardStore
{
private readonly IMongoCollection<QuestionAttempt> _attempts;
private readonly IMongoCollection<UserRating> _ratings;
private readonly IMongoCollection<CcsUserRating> _ccsRatings;
private readonly IMongoCollection<CcsAttempt> _ccsAttempts;
private readonly IMongoCollection<CcsCaseRevision> _ccsCases;
@@ -19,6 +18,25 @@ public sealed class DashboardStore
private readonly DynamicQuestionStore _dynamicQuestions;
private readonly FlashcardStudyService _flashcards;
private readonly FlashcardScheduleStore _flashcardSchedules;
private readonly RatingStore _ratingStore;
public static readonly string[] Step3Systems =
[
"Human Development",
"Immune System, Blood & Lymphoreticular System, and Multisystem Processes/Disorders",
"Behavioral Health",
"Nervous System & Special Senses",
"Skin & Subcutaneous Tissue",
"Musculoskeletal System",
"Cardiovascular System",
"Respiratory System",
"Gastrointestinal System",
"Renal/Urinary & Male Reproductive Systems",
"Pregnancy/Childbirth & Female Reproductive System & Breast",
"Endocrine System",
"Biostatistics & Epidemiology/Population Health & Interpretation of the Medical Literature",
"Social Sciences: Communication Skills/Ethics/Patient Safety"
];
public DashboardStore(
MongoDatabaseProvider mongo,
@@ -26,10 +44,10 @@ public sealed class DashboardStore
ReviewQueueStore reviewQueue,
DynamicQuestionStore dynamicQuestions,
FlashcardStudyService flashcards,
FlashcardScheduleStore flashcardSchedules)
FlashcardScheduleStore flashcardSchedules,
RatingStore ratingStore)
{
_attempts = mongo.Database.GetCollection<QuestionAttempt>("QuestionAttempts");
_ratings = mongo.Database.GetCollection<UserRating>("UserRatings");
_ccsRatings = mongo.Database.GetCollection<CcsUserRating>("CcsUserRatings");
_ccsAttempts = mongo.Database.GetCollection<CcsAttempt>("CcsAttempts");
_ccsCases = mongo.Database.GetCollection<CcsCaseRevision>("CcsCaseRevisions");
@@ -41,6 +59,7 @@ public sealed class DashboardStore
_dynamicQuestions = dynamicQuestions;
_flashcards = flashcards;
_flashcardSchedules = flashcardSchedules;
_ratingStore = ratingStore;
}
public async Task<DashboardResponse> GetAsync(string userId, DateTime nowUtc)
@@ -49,7 +68,7 @@ public sealed class DashboardStore
var trendStart = today.AddDays(-29);
var weekStart = today.AddDays(-6);
var attemptsTask = _attempts.Find(value => value.UserId == userId).ToListAsync();
var ratingTask = _ratings.Find(value => value.UserId == userId).FirstOrDefaultAsync();
var ratingTask = _ratingStore.GetOrInitializeHistoryAsync(userId, nowUtc);
var ccsRatingTask = _ccsRatings.Find(value => value.UserId == userId).FirstOrDefaultAsync();
var ccsAttemptsTask = _ccsAttempts.Find(value => value.UserId == userId && value.CreatedUtc >= trendStart).ToListAsync();
var completedCcsCountTask = _ccsAttempts.CountDocumentsAsync(value => value.UserId == userId
@@ -77,8 +96,8 @@ public sealed class DashboardStore
var response = new DashboardResponse
{
QuestionBankRating = ratingTask.Result?.OverallElo ?? 1000,
RatedQuestionCount = ratingTask.Result?.RatedQuestionCount ?? 0,
QuestionBankRating = ratingTask.Result.OverallElo,
RatedQuestionCount = ratingTask.Result.RatedQuestionCount,
CcsRating = ccsRatingTask.Result?.Elo ?? 1000,
RatedCcsCaseCount = ccsRatingTask.Result?.RatedCaseCount ?? 0,
QuestionsCompleted = answeredAttempts.Count,
@@ -95,31 +114,8 @@ public sealed class DashboardStore
FlashcardsReviewedThisWeek = flashcardCountsTask.Result.ReviewedThisWeek
};
response.PerformanceTrend = Enumerable.Range(0, 30).Select(offset =>
{
var date = trendStart.AddDays(offset);
var daily = answeredAttempts.Where(value => value.CreatedUtc.Date == date).ToList();
return new DashboardTrendPoint
{
DateUtc = date,
Correct = daily.Count(value => value.IsCorrect),
Total = daily.Count,
Accuracy = Percentage(daily.Count(value => value.IsCorrect), daily.Count)
};
}).ToList();
response.Systems = answeredAttempts
.Where(value => questionMap.ContainsKey(value.QuestionExternalId))
.GroupBy(value => NormalizeSystem(questionMap[value.QuestionExternalId].System))
.Select(group => new DashboardSystemPerformance
{
System = group.Key,
Correct = group.Count(value => value.IsCorrect),
Total = group.Count(),
Accuracy = Percentage(group.Count(value => value.IsCorrect), group.Count())
})
.OrderByDescending(value => value.Total).ThenByDescending(value => value.Accuracy)
.ToList();
response.PerformanceTrend = BuildPerformanceTrend(attempts, ratingTask.Result, today);
response.Systems = BuildSystemPerformance(answeredAttempts, questionMap);
response.WeeklyActivity = Enumerable.Range(0, 7).Select(offset =>
{
@@ -166,6 +162,133 @@ public sealed class DashboardStore
return streak;
}
public static List<DashboardTrendPoint> BuildPerformanceTrend(
IEnumerable<QuestionAttempt> attempts,
UserRating rating,
DateTime todayUtc)
{
var attemptList = attempts.ToList();
var trendStart = todayUtc.Date.AddDays(-29);
var ratingEvents = new List<(DateTime AtUtc, int Elo)>();
if (rating.HistoryBaselineElo.HasValue && rating.HistoryStartedUtc.HasValue)
ratingEvents.Add((rating.HistoryStartedUtc.Value, rating.HistoryBaselineElo.Value));
ratingEvents.AddRange(attemptList
.Where(value => value.RatingAppliedUtc.HasValue && value.UserOverallEloAfter.HasValue)
.Select(value => (value.RatingAppliedUtc!.Value, value.UserOverallEloAfter!.Value)));
ratingEvents.Sort((left, right) => left.AtUtc.CompareTo(right.AtUtc));
int? currentRating = null;
var eventIndex = 0;
var result = new List<DashboardTrendPoint>();
foreach (var offset in Enumerable.Range(0, 30))
{
var date = trendStart.AddDays(offset);
var nextDate = date.AddDays(1);
while (eventIndex < ratingEvents.Count && ratingEvents[eventIndex].AtUtc < nextDate)
{
currentRating = ratingEvents[eventIndex].Elo;
eventIndex++;
}
var daily = attemptList
.Where(value => !value.IsSkip && value.CreatedUtc >= date && value.CreatedUtc < nextDate)
.ToList();
var correct = daily.Count(value => value.IsCorrect);
result.Add(new DashboardTrendPoint
{
DateUtc = date,
Correct = correct,
Total = daily.Count,
Accuracy = Percentage(correct, daily.Count),
QuestionBankRating = currentRating
});
}
return result;
}
public static List<DashboardSystemPerformance> BuildSystemPerformance(
IEnumerable<QuestionAttempt> attempts,
IReadOnlyDictionary<string, Question> questionMap)
{
var totals = Step3Systems.ToDictionary(
system => system,
_ => (Correct: 0, Total: 0),
StringComparer.Ordinal);
foreach (var attempt in attempts.Where(value => !value.IsSkip))
{
if (!questionMap.TryGetValue(attempt.QuestionExternalId, out var question))
continue;
var system = NormalizeStep3System(question);
if (system is null)
continue;
var current = totals[system];
totals[system] = (current.Correct + (attempt.IsCorrect ? 1 : 0), current.Total + 1);
}
return Step3Systems.Select(system => new DashboardSystemPerformance
{
System = system,
Correct = totals[system].Correct,
Total = totals[system].Total,
Accuracy = Percentage(totals[system].Correct, totals[system].Total)
}).ToList();
}
public static string? NormalizeStep3System(Question question)
{
foreach (var candidate in new[] { question.System, question.PerformanceCategory, question.Discipline })
{
var normalized = NormalizeStep3System(candidate);
if (normalized is not null)
return normalized;
}
return null;
}
public static string? NormalizeStep3System(string? value)
{
if (string.IsNullOrWhiteSpace(value))
return null;
var exact = Step3Systems.FirstOrDefault(
system => string.Equals(system, value.Trim(), StringComparison.OrdinalIgnoreCase));
if (exact is not null)
return exact;
var lower = value.Trim().ToLowerInvariant();
if (ContainsAny(lower, "human development", "well patient", "preventive", "health maintenance", "geriatr", "age-related"))
return Step3Systems[0];
if (ContainsAny(lower, "social science", "communication", "ethic", "patient safety", "professionalism", "systems-based"))
return Step3Systems[13];
if (ContainsAny(lower, "pregnan", "obstetric", "gynec", "female reproductive", "breast", "puerper"))
return Step3Systems[10];
if (ContainsAny(lower, "renal", "urinary", "male reproductive", "urolog", "nephro"))
return Step3Systems[9];
if (ContainsAny(lower, "biostat", "epidemi", "population health", "medical literature"))
return Step3Systems[12];
if (ContainsAny(lower, "immune", "immun", "blood", "hemat", "lymph", "multisystem", "infectious"))
return Step3Systems[1];
if (ContainsAny(lower, "behavior", "psychi", "mental health"))
return Step3Systems[2];
if (ContainsAny(lower, "nervous", "neuro", "special sense", "ophthalm", "otolaryng", " ent"))
return Step3Systems[3];
if (ContainsAny(lower, "skin", "derm", "subcutaneous"))
return Step3Systems[4];
if (ContainsAny(lower, "musculo", "orthop", "rheumat"))
return Step3Systems[5];
if (ContainsAny(lower, "cardio", "circulatory"))
return Step3Systems[6];
if (ContainsAny(lower, "resp", "pulmon"))
return Step3Systems[7];
if (ContainsAny(lower, "gastro", "digest", "hepato"))
return Step3Systems[8];
if (ContainsAny(lower, "endocr", "metabolic"))
return Step3Systems[11];
return null;
}
private static bool ContainsAny(string value, params string[] fragments) =>
fragments.Any(value.Contains);
private async Task<List<DashboardRecentActivity>> BuildRecentActivityAsync(
List<QuestionAttempt> attempts,
Dictionary<string, Question> questionMap,
api/QuestionsApi/Services/ExamSimulationStore.cs
+2 -2
View File
@@ -454,8 +454,8 @@ public class ExamSimulationStore
try
{
shouldRate = await _attempts.TryAcquireRatingLeaseAsync(attemptId, owner, TimeSpan.FromSeconds(30));
await _ratings.ApplyFirstResultAsync(simulation.UserId, question, record.Attempt.IsCorrect, shouldRate);
if (shouldRate) await _attempts.MarkRatingAppliedAsync(attemptId, owner);
var rating = await _ratings.ApplyFirstResultAsync(simulation.UserId, question, record.Attempt.IsCorrect, shouldRate);
if (shouldRate) await _attempts.MarkRatingAppliedAsync(attemptId, owner, rating.UserOverallElo);
}
catch
{
api/QuestionsApi/Services/QuestionAttemptStore.cs
+3 -1
View File
@@ -103,11 +103,13 @@ public class QuestionAttemptStore
return leased?.RatingLeaseOwner == owner;
}
public Task MarkRatingAppliedAsync(string attemptId, string owner) =>
public Task MarkRatingAppliedAsync(string attemptId, string owner, int userOverallElo) =>
_col.UpdateOneAsync(
attempt => attempt.Id == attemptId && attempt.RatingLeaseOwner == owner,
Builders<QuestionAttempt>.Update
.Set(attempt => attempt.RatingApplied, true)
.Set(attempt => attempt.UserOverallEloAfter, userOverallElo)
.Set(attempt => attempt.RatingAppliedUtc, DateTime.UtcNow)
.Set(attempt => attempt.RatingLeaseUntilUtc, DateTime.MinValue));
public Task ReleaseRatingLeaseAsync(string attemptId, string owner) =>
api/QuestionsApi/Services/RatingStore.cs
+27
View File
@@ -35,6 +35,12 @@ public class RatingStore
};
}
public async Task<UserRating> GetOrInitializeHistoryAsync(string userId, DateTime nowUtc)
{
var rating = await GetOrCreateUserRatingAsync(userId);
return await EnsureHistoryBaselineAsync(rating, nowUtc);
}
public async Task<Dictionary<string, int>> GetQuestionElosAsync(IEnumerable<string> externalIds)
{
var ids = externalIds.Where(id => !string.IsNullOrWhiteSpace(id)).Distinct().ToList();
@@ -71,6 +77,7 @@ public class RatingStore
};
}
userRating = await EnsureHistoryBaselineAsync(userRating, DateTime.UtcNow);
var actual = isCorrect ? 1.0 : 0.0;
var userK = userRating.RatedQuestionCount < 20 ? 40 : 24;
var questionK = questionRating.AttemptCount < 20 ? 32 : 24;
@@ -164,6 +171,26 @@ public class RatingStore
return rating;
}
private async Task<UserRating> EnsureHistoryBaselineAsync(UserRating rating, DateTime nowUtc)
{
if (rating.HistoryBaselineElo.HasValue && rating.HistoryStartedUtc.HasValue)
return rating;
var result = await _users.UpdateOneAsync(
value => value.UserId == rating.UserId && value.HistoryStartedUtc == null,
Builders<UserRating>.Update
.Set(value => value.HistoryBaselineElo, rating.OverallElo)
.Set(value => value.HistoryStartedUtc, nowUtc));
if (result.ModifiedCount > 0)
{
rating.HistoryBaselineElo = rating.OverallElo;
rating.HistoryStartedUtc = nowUtc;
return rating;
}
return await _users.Find(value => value.UserId == rating.UserId).FirstAsync();
}
private async Task<QuestionRating> GetOrCreateQuestionRatingAsync(string questionExternalId)
{
var create = Builders<QuestionRating>.Update