Add dashboard performance trends

2026-07-25 22:43:34 -05:00
parent 46953ee71a
commit 500477b612
15 changed files with 640 additions and 78 deletions
@@ -1,3 +1,4 @@
+using QuestionsApi.Models;
 using QuestionsApi.Services;
 using Xunit;

@@ -27,4 +28,110 @@ public sealed class DashboardTests
        var dates = new[] { today, today.AddDays(-1), today.AddDays(-3) };
        Assert.Equal(2, DashboardStore.CalculateCurrentStreak(dates, today));
    }
+
+    [Fact]
+    public void SystemPerformanceReturnsEveryOfficialSystemInOrder()
+    {
+        var attempts = new[]
+        {
+            Attempt("cardio-1", true),
+            Attempt("cardio-2", false),
+            Attempt("ethics-1", true),
+            Attempt("ignored", true),
+            Attempt("skipped-cardio", false, true)
+        };
+        var questions = new Dictionary<string, Question>
+        {
+            ["cardio-1"] = Question("Cardiology"),
+            ["cardio-2"] = Question("Cardiovascular System"),
+            ["ethics-1"] = Question("Medical Ethics"),
+            ["ignored"] = Question("Unmapped Specialty"),
+            ["skipped-cardio"] = Question("Cardiology")
+        };
+
+        var result = DashboardStore.BuildSystemPerformance(attempts, questions);
+
+        Assert.Equal(14, result.Count);
+        Assert.Equal(DashboardStore.Step3Systems, result.Select(value => value.System));
+        var cardiovascular = result.Single(value => value.System == "Cardiovascular System");
+        Assert.Equal(1, cardiovascular.Correct);
+        Assert.Equal(2, cardiovascular.Total);
+        Assert.Equal(50, cardiovascular.Accuracy);
+        var social = result.Single(value => value.System.StartsWith("Social Sciences"));
+        Assert.Equal(1, social.Correct);
+        Assert.Equal(1, social.Total);
+        Assert.All(result.Where(value => value.System != cardiovascular.System && value.System != social.System),
+            value => Assert.Equal(0, value.Total));
+    }
+
+    [Theory]
+    [InlineData("Preventive Medicine", "Human Development")]
+    [InlineData("Infectious Disease", "Immune System, Blood & Lymphoreticular System, and Multisystem Processes/Disorders")]
+    [InlineData("Dermatology", "Skin & Subcutaneous Tissue")]
+    [InlineData("Pulmonary", "Respiratory System")]
+    [InlineData("Pregnancy", "Pregnancy/Childbirth & Female Reproductive System & Breast")]
+    [InlineData("Biostatistics", "Biostatistics & Epidemiology/Population Health & Interpretation of the Medical Literature")]
+    [InlineData("Medical Ethics", "Social Sciences: Communication Skills/Ethics/Patient Safety")]
+    public void SystemAliasesMapToOfficialCategories(string source, string expected)
+    {
+        Assert.Equal(expected, DashboardStore.NormalizeStep3System(source));
+    }
+
+    [Fact]
+    public void TrendCarriesRatingForwardAndLeavesInactiveAccuracyAsGapData()
+    {
+        var today = new DateTime(2026, 7, 25, 0, 0, 0, DateTimeKind.Utc);
+        var baselineDate = today.AddDays(-2).AddHours(8);
+        var attempts = new[]
+        {
+            Attempt("one", true, createdUtc: today.AddDays(-2).AddHours(9),
+                rating: 1018, ratingUtc: today.AddDays(-2).AddHours(9)),
+            Attempt("two", false, createdUtc: today.AddDays(-2).AddHours(10),
+                rating: 1002, ratingUtc: today.AddDays(-2).AddHours(10)),
+            Attempt("three", true, createdUtc: today.AddHours(11),
+                rating: 1020, ratingUtc: today.AddHours(11)),
+            Attempt("skip", false, true, today.AddDays(-1).AddHours(4),
+                990, today.AddDays(-1).AddHours(4))
+        };
+        var rating = new UserRating
+        {
+            OverallElo = 1020,
+            HistoryBaselineElo = 1000,
+            HistoryStartedUtc = baselineDate
+        };
+
+        var result = DashboardStore.BuildPerformanceTrend(attempts, rating, today);
+
+        Assert.Equal(30, result.Count);
+        Assert.Null(result[^4].QuestionBankRating);
+        Assert.Equal(1002, result[^3].QuestionBankRating);
+        Assert.Equal(50, result[^3].Accuracy);
+        Assert.Equal(2, result[^3].Total);
+        Assert.Equal(990, result[^2].QuestionBankRating);
+        Assert.Equal(0, result[^2].Total);
+        Assert.Equal(1020, result[^1].QuestionBankRating);
+        Assert.Equal(100, result[^1].Accuracy);
+    }
+
+    private static QuestionAttempt Attempt(
+        string questionId,
+        bool correct,
+        bool skip = false,
+        DateTime? createdUtc = null,
+        int? rating = null,
+        DateTime? ratingUtc = null) => new()
+        {
+            QuestionExternalId = questionId,
+            IsCorrect = correct,
+            IsSkip = skip,
+            CreatedUtc = createdUtc ?? DateTime.UtcNow,
+            UserOverallEloAfter = rating,
+            RatingAppliedUtc = ratingUtc
+        };
+
+    private static Question Question(string system) => new()
+    {
+        ExternalId = system,
+        System = system
+    };
 }
@@ -187,7 +187,7 @@ public class QuestionsController : ControllerBase
                record.Attempt.IsCorrect,
                shouldRate);
            if (shouldRate)
-                await _attempts.MarkRatingAppliedAsync(record.Attempt.Id ?? "", ratingOwner);
+                await _attempts.MarkRatingAppliedAsync(record.Attempt.Id ?? "", ratingOwner, rating.UserOverallElo);
        }
        catch
        {
@@ -286,7 +286,7 @@ public class QuestionsController : ControllerBase
        {
            rating = await _ratings.ApplyFirstResultAsync(userId, question, false, shouldRate);
            if (shouldRate)
-                await _attempts.MarkRatingAppliedAsync(record.Attempt.Id ?? "", ratingOwner);
+                await _attempts.MarkRatingAppliedAsync(record.Attempt.Id ?? "", ratingOwner, rating.UserOverallElo);
        }
        catch
        {
@@ -32,6 +32,7 @@ public sealed class DashboardTrendPoint
    public int Correct { get; set; }
    public int Total { get; set; }
    public int Accuracy { get; set; }
+    public int? QuestionBankRating { get; set; }
 }

 public sealed class DashboardSystemPerformance
@@ -19,6 +19,8 @@ public class QuestionAttempt
    public int DurationSeconds { get; set; }
    public int CounterBucket { get; set; }
    public bool? RatingApplied { get; set; }
+    public int? UserOverallEloAfter { get; set; }
+    public DateTime? RatingAppliedUtc { get; set; }
    public string RatingLeaseOwner { get; set; } = "";
    public DateTime RatingLeaseUntilUtc { get; set; }
    public DateTime CreatedUtc { get; set; } = DateTime.UtcNow;
@@ -13,6 +13,8 @@ public class UserRating
    public int OverallElo { get; set; } = 1000;
    public Dictionary<string, int> CategoryElos { get; set; } = [];
    public int RatedQuestionCount { get; set; }
+    public int? HistoryBaselineElo { get; set; }
+    public DateTime? HistoryStartedUtc { get; set; }
    public DateTime UpdatedUtc { get; set; } = DateTime.UtcNow;
 }

@@ -7,7 +7,6 @@ namespace QuestionsApi.Services;
 public sealed class DashboardStore
 {
    private readonly IMongoCollection<QuestionAttempt> _attempts;
-    private readonly IMongoCollection<UserRating> _ratings;
    private readonly IMongoCollection<CcsUserRating> _ccsRatings;
    private readonly IMongoCollection<CcsAttempt> _ccsAttempts;
    private readonly IMongoCollection<CcsCaseRevision> _ccsCases;
@@ -19,6 +18,25 @@ public sealed class DashboardStore
    private readonly DynamicQuestionStore _dynamicQuestions;
    private readonly FlashcardStudyService _flashcards;
    private readonly FlashcardScheduleStore _flashcardSchedules;
+    private readonly RatingStore _ratingStore;
+
+    public static readonly string[] Step3Systems =
+    [
+        "Human Development",
+        "Immune System, Blood & Lymphoreticular System, and Multisystem Processes/Disorders",
+        "Behavioral Health",
+        "Nervous System & Special Senses",
+        "Skin & Subcutaneous Tissue",
+        "Musculoskeletal System",
+        "Cardiovascular System",
+        "Respiratory System",
+        "Gastrointestinal System",
+        "Renal/Urinary & Male Reproductive Systems",
+        "Pregnancy/Childbirth & Female Reproductive System & Breast",
+        "Endocrine System",
+        "Biostatistics & Epidemiology/Population Health & Interpretation of the Medical Literature",
+        "Social Sciences: Communication Skills/Ethics/Patient Safety"
+    ];

    public DashboardStore(
        MongoDatabaseProvider mongo,
@@ -26,10 +44,10 @@ public sealed class DashboardStore
        ReviewQueueStore reviewQueue,
        DynamicQuestionStore dynamicQuestions,
        FlashcardStudyService flashcards,
-        FlashcardScheduleStore flashcardSchedules)
+        FlashcardScheduleStore flashcardSchedules,
+        RatingStore ratingStore)
    {
        _attempts = mongo.Database.GetCollection<QuestionAttempt>("QuestionAttempts");
-        _ratings = mongo.Database.GetCollection<UserRating>("UserRatings");
        _ccsRatings = mongo.Database.GetCollection<CcsUserRating>("CcsUserRatings");
        _ccsAttempts = mongo.Database.GetCollection<CcsAttempt>("CcsAttempts");
        _ccsCases = mongo.Database.GetCollection<CcsCaseRevision>("CcsCaseRevisions");
@@ -41,6 +59,7 @@ public sealed class DashboardStore
        _dynamicQuestions = dynamicQuestions;
        _flashcards = flashcards;
        _flashcardSchedules = flashcardSchedules;
+        _ratingStore = ratingStore;
    }

    public async Task<DashboardResponse> GetAsync(string userId, DateTime nowUtc)
@@ -49,7 +68,7 @@ public sealed class DashboardStore
        var trendStart = today.AddDays(-29);
        var weekStart = today.AddDays(-6);
        var attemptsTask = _attempts.Find(value => value.UserId == userId).ToListAsync();
-        var ratingTask = _ratings.Find(value => value.UserId == userId).FirstOrDefaultAsync();
+        var ratingTask = _ratingStore.GetOrInitializeHistoryAsync(userId, nowUtc);
        var ccsRatingTask = _ccsRatings.Find(value => value.UserId == userId).FirstOrDefaultAsync();
        var ccsAttemptsTask = _ccsAttempts.Find(value => value.UserId == userId && value.CreatedUtc >= trendStart).ToListAsync();
        var completedCcsCountTask = _ccsAttempts.CountDocumentsAsync(value => value.UserId == userId
@@ -77,8 +96,8 @@ public sealed class DashboardStore

        var response = new DashboardResponse
        {
-            QuestionBankRating = ratingTask.Result?.OverallElo ?? 1000,
-            RatedQuestionCount = ratingTask.Result?.RatedQuestionCount ?? 0,
+            QuestionBankRating = ratingTask.Result.OverallElo,
+            RatedQuestionCount = ratingTask.Result.RatedQuestionCount,
            CcsRating = ccsRatingTask.Result?.Elo ?? 1000,
            RatedCcsCaseCount = ccsRatingTask.Result?.RatedCaseCount ?? 0,
            QuestionsCompleted = answeredAttempts.Count,
@@ -95,31 +114,8 @@ public sealed class DashboardStore
            FlashcardsReviewedThisWeek = flashcardCountsTask.Result.ReviewedThisWeek
        };

-        response.PerformanceTrend = Enumerable.Range(0, 30).Select(offset =>
-        {
-            var date = trendStart.AddDays(offset);
-            var daily = answeredAttempts.Where(value => value.CreatedUtc.Date == date).ToList();
-            return new DashboardTrendPoint
-            {
-                DateUtc = date,
-                Correct = daily.Count(value => value.IsCorrect),
-                Total = daily.Count,
-                Accuracy = Percentage(daily.Count(value => value.IsCorrect), daily.Count)
-            };
-        }).ToList();
-
-        response.Systems = answeredAttempts
-            .Where(value => questionMap.ContainsKey(value.QuestionExternalId))
-            .GroupBy(value => NormalizeSystem(questionMap[value.QuestionExternalId].System))
-            .Select(group => new DashboardSystemPerformance
-            {
-                System = group.Key,
-                Correct = group.Count(value => value.IsCorrect),
-                Total = group.Count(),
-                Accuracy = Percentage(group.Count(value => value.IsCorrect), group.Count())
-            })
-            .OrderByDescending(value => value.Total).ThenByDescending(value => value.Accuracy)
-            .ToList();
+        response.PerformanceTrend = BuildPerformanceTrend(attempts, ratingTask.Result, today);
+        response.Systems = BuildSystemPerformance(answeredAttempts, questionMap);

        response.WeeklyActivity = Enumerable.Range(0, 7).Select(offset =>
        {
@@ -166,6 +162,133 @@ public sealed class DashboardStore
        return streak;
    }

+    public static List<DashboardTrendPoint> BuildPerformanceTrend(
+        IEnumerable<QuestionAttempt> attempts,
+        UserRating rating,
+        DateTime todayUtc)
+    {
+        var attemptList = attempts.ToList();
+        var trendStart = todayUtc.Date.AddDays(-29);
+        var ratingEvents = new List<(DateTime AtUtc, int Elo)>();
+        if (rating.HistoryBaselineElo.HasValue && rating.HistoryStartedUtc.HasValue)
+            ratingEvents.Add((rating.HistoryStartedUtc.Value, rating.HistoryBaselineElo.Value));
+        ratingEvents.AddRange(attemptList
+            .Where(value => value.RatingAppliedUtc.HasValue && value.UserOverallEloAfter.HasValue)
+            .Select(value => (value.RatingAppliedUtc!.Value, value.UserOverallEloAfter!.Value)));
+        ratingEvents.Sort((left, right) => left.AtUtc.CompareTo(right.AtUtc));
+
+        int? currentRating = null;
+        var eventIndex = 0;
+        var result = new List<DashboardTrendPoint>();
+        foreach (var offset in Enumerable.Range(0, 30))
+        {
+            var date = trendStart.AddDays(offset);
+            var nextDate = date.AddDays(1);
+            while (eventIndex < ratingEvents.Count && ratingEvents[eventIndex].AtUtc < nextDate)
+            {
+                currentRating = ratingEvents[eventIndex].Elo;
+                eventIndex++;
+            }
+
+            var daily = attemptList
+                .Where(value => !value.IsSkip && value.CreatedUtc >= date && value.CreatedUtc < nextDate)
+                .ToList();
+            var correct = daily.Count(value => value.IsCorrect);
+            result.Add(new DashboardTrendPoint
+            {
+                DateUtc = date,
+                Correct = correct,
+                Total = daily.Count,
+                Accuracy = Percentage(correct, daily.Count),
+                QuestionBankRating = currentRating
+            });
+        }
+        return result;
+    }
+
+    public static List<DashboardSystemPerformance> BuildSystemPerformance(
+        IEnumerable<QuestionAttempt> attempts,
+        IReadOnlyDictionary<string, Question> questionMap)
+    {
+        var totals = Step3Systems.ToDictionary(
+            system => system,
+            _ => (Correct: 0, Total: 0),
+            StringComparer.Ordinal);
+        foreach (var attempt in attempts.Where(value => !value.IsSkip))
+        {
+            if (!questionMap.TryGetValue(attempt.QuestionExternalId, out var question))
+                continue;
+            var system = NormalizeStep3System(question);
+            if (system is null)
+                continue;
+            var current = totals[system];
+            totals[system] = (current.Correct + (attempt.IsCorrect ? 1 : 0), current.Total + 1);
+        }
+
+        return Step3Systems.Select(system => new DashboardSystemPerformance
+        {
+            System = system,
+            Correct = totals[system].Correct,
+            Total = totals[system].Total,
+            Accuracy = Percentage(totals[system].Correct, totals[system].Total)
+        }).ToList();
+    }
+
+    public static string? NormalizeStep3System(Question question)
+    {
+        foreach (var candidate in new[] { question.System, question.PerformanceCategory, question.Discipline })
+        {
+            var normalized = NormalizeStep3System(candidate);
+            if (normalized is not null)
+                return normalized;
+        }
+        return null;
+    }
+
+    public static string? NormalizeStep3System(string? value)
+    {
+        if (string.IsNullOrWhiteSpace(value))
+            return null;
+        var exact = Step3Systems.FirstOrDefault(
+            system => string.Equals(system, value.Trim(), StringComparison.OrdinalIgnoreCase));
+        if (exact is not null)
+            return exact;
+
+        var lower = value.Trim().ToLowerInvariant();
+        if (ContainsAny(lower, "human development", "well patient", "preventive", "health maintenance", "geriatr", "age-related"))
+            return Step3Systems[0];
+        if (ContainsAny(lower, "social science", "communication", "ethic", "patient safety", "professionalism", "systems-based"))
+            return Step3Systems[13];
+        if (ContainsAny(lower, "pregnan", "obstetric", "gynec", "female reproductive", "breast", "puerper"))
+            return Step3Systems[10];
+        if (ContainsAny(lower, "renal", "urinary", "male reproductive", "urolog", "nephro"))
+            return Step3Systems[9];
+        if (ContainsAny(lower, "biostat", "epidemi", "population health", "medical literature"))
+            return Step3Systems[12];
+        if (ContainsAny(lower, "immune", "immun", "blood", "hemat", "lymph", "multisystem", "infectious"))
+            return Step3Systems[1];
+        if (ContainsAny(lower, "behavior", "psychi", "mental health"))
+            return Step3Systems[2];
+        if (ContainsAny(lower, "nervous", "neuro", "special sense", "ophthalm", "otolaryng", " ent"))
+            return Step3Systems[3];
+        if (ContainsAny(lower, "skin", "derm", "subcutaneous"))
+            return Step3Systems[4];
+        if (ContainsAny(lower, "musculo", "orthop", "rheumat"))
+            return Step3Systems[5];
+        if (ContainsAny(lower, "cardio", "circulatory"))
+            return Step3Systems[6];
+        if (ContainsAny(lower, "resp", "pulmon"))
+            return Step3Systems[7];
+        if (ContainsAny(lower, "gastro", "digest", "hepato"))
+            return Step3Systems[8];
+        if (ContainsAny(lower, "endocr", "metabolic"))
+            return Step3Systems[11];
+        return null;
+    }
+
+    private static bool ContainsAny(string value, params string[] fragments) =>
+        fragments.Any(value.Contains);
+
    private async Task<List<DashboardRecentActivity>> BuildRecentActivityAsync(
        List<QuestionAttempt> attempts,
        Dictionary<string, Question> questionMap,
@@ -454,8 +454,8 @@ public class ExamSimulationStore
            try
            {
                shouldRate = await _attempts.TryAcquireRatingLeaseAsync(attemptId, owner, TimeSpan.FromSeconds(30));
-                await _ratings.ApplyFirstResultAsync(simulation.UserId, question, record.Attempt.IsCorrect, shouldRate);
-                if (shouldRate) await _attempts.MarkRatingAppliedAsync(attemptId, owner);
+                var rating = await _ratings.ApplyFirstResultAsync(simulation.UserId, question, record.Attempt.IsCorrect, shouldRate);
+                if (shouldRate) await _attempts.MarkRatingAppliedAsync(attemptId, owner, rating.UserOverallElo);
            }
            catch
            {
@@ -103,11 +103,13 @@ public class QuestionAttemptStore
        return leased?.RatingLeaseOwner == owner;
    }

-    public Task MarkRatingAppliedAsync(string attemptId, string owner) =>
+    public Task MarkRatingAppliedAsync(string attemptId, string owner, int userOverallElo) =>
        _col.UpdateOneAsync(
            attempt => attempt.Id == attemptId && attempt.RatingLeaseOwner == owner,
            Builders<QuestionAttempt>.Update
                .Set(attempt => attempt.RatingApplied, true)
+                .Set(attempt => attempt.UserOverallEloAfter, userOverallElo)
+                .Set(attempt => attempt.RatingAppliedUtc, DateTime.UtcNow)
                .Set(attempt => attempt.RatingLeaseUntilUtc, DateTime.MinValue));

    public Task ReleaseRatingLeaseAsync(string attemptId, string owner) =>
@@ -35,6 +35,12 @@ public class RatingStore
        };
    }

+    public async Task<UserRating> GetOrInitializeHistoryAsync(string userId, DateTime nowUtc)
+    {
+        var rating = await GetOrCreateUserRatingAsync(userId);
+        return await EnsureHistoryBaselineAsync(rating, nowUtc);
+    }
+
    public async Task<Dictionary<string, int>> GetQuestionElosAsync(IEnumerable<string> externalIds)
    {
        var ids = externalIds.Where(id => !string.IsNullOrWhiteSpace(id)).Distinct().ToList();
@@ -71,6 +77,7 @@ public class RatingStore
            };
        }

+        userRating = await EnsureHistoryBaselineAsync(userRating, DateTime.UtcNow);
        var actual = isCorrect ? 1.0 : 0.0;
        var userK = userRating.RatedQuestionCount < 20 ? 40 : 24;
        var questionK = questionRating.AttemptCount < 20 ? 32 : 24;
@@ -164,6 +171,26 @@ public class RatingStore
        return rating;
    }

+    private async Task<UserRating> EnsureHistoryBaselineAsync(UserRating rating, DateTime nowUtc)
+    {
+        if (rating.HistoryBaselineElo.HasValue && rating.HistoryStartedUtc.HasValue)
+            return rating;
+
+        var result = await _users.UpdateOneAsync(
+            value => value.UserId == rating.UserId && value.HistoryStartedUtc == null,
+            Builders<UserRating>.Update
+                .Set(value => value.HistoryBaselineElo, rating.OverallElo)
+                .Set(value => value.HistoryStartedUtc, nowUtc));
+        if (result.ModifiedCount > 0)
+        {
+            rating.HistoryBaselineElo = rating.OverallElo;
+            rating.HistoryStartedUtc = nowUtc;
+            return rating;
+        }
+
+        return await _users.Find(value => value.UserId == rating.UserId).FirstAsync();
+    }
+
    private async Task<QuestionRating> GetOrCreateQuestionRatingAsync(string questionExternalId)
    {
        var create = Builders<QuestionRating>.Update