Improve dynamic question learning loop

2026-07-23 16:50:53 -05:00
parent c91e70deb5
commit 32c455bba0
14 changed files with 422 additions and 16 deletions
@@ -10,6 +10,7 @@ public sealed class DynamicQuestionStore
    private readonly IMongoCollection<DynamicQuestionAttempt> _attempts;
    private readonly IMongoCollection<DynamicObjectiveMastery> _mastery;
    private readonly IMongoCollection<DynamicFamilyRating> _familyRatings;
+    private readonly IMongoCollection<DynamicReviewItem> _reviews;
    private readonly ServiceMetrics? _metrics;

    public DynamicQuestionStore(MongoDatabaseProvider mongo, ServiceMetrics? metrics = null)
@@ -19,6 +20,7 @@ public sealed class DynamicQuestionStore
        _attempts = mongo.Database.GetCollection<DynamicQuestionAttempt>("DynamicQuestionAttempts");
        _mastery = mongo.Database.GetCollection<DynamicObjectiveMastery>("DynamicObjectiveMastery");
        _familyRatings = mongo.Database.GetCollection<DynamicFamilyRating>("DynamicFamilyRatings");
+        _reviews = mongo.Database.GetCollection<DynamicReviewItem>("DynamicReviewItems");
        _blueprints.Indexes.CreateOne(new CreateIndexModel<DynamicQuestionBlueprint>(
            Builders<DynamicQuestionBlueprint>.IndexKeys.Ascending(value => value.ExternalId).Ascending(value => value.Version),
            new CreateIndexOptions { Unique = true }));
@@ -34,6 +36,19 @@ public sealed class DynamicQuestionStore
                .Ascending(value => value.BlueprintVersion)
                .Ascending(value => value.CaseFamilyId),
            new CreateIndexOptions { Unique = true }));
+        _reviews.Indexes.CreateMany([
+            new CreateIndexModel<DynamicReviewItem>(
+                Builders<DynamicReviewItem>.IndexKeys
+                    .Ascending(value => value.UserId)
+                    .Ascending(value => value.ObjectiveId),
+                new CreateIndexOptions { Unique = true, Name = "dynamic_review_user_objective" }),
+            new CreateIndexModel<DynamicReviewItem>(
+                Builders<DynamicReviewItem>.IndexKeys
+                    .Ascending(value => value.UserId)
+                    .Ascending(value => value.State)
+                    .Ascending(value => value.DueUtc),
+                new CreateIndexOptions { Name = "dynamic_review_user_due" })
+        ]);
    }

    public async Task<List<DynamicQuestionBlueprint>> PublishedAsync() =>
@@ -46,6 +61,11 @@ public sealed class DynamicQuestionStore
        (await _mastery.Find(value => value.UserId == userId).ToListAsync())
        .ToDictionary(value => value.ObjectiveId, StringComparer.Ordinal);

+    public async Task<List<DynamicReviewItem>> DueReviewsAsync(string userId, DateTime nowUtc) =>
+        await _reviews.Find(value => value.UserId == userId
+            && value.State == DynamicReviewStates.Pending
+            && value.DueUtc <= nowUtc).ToListAsync();
+
    public async Task<Dictionary<string, DynamicFamilyRating>> FamilyRatingsAsync(DynamicQuestionBlueprint blueprint)
    {
        var ratings = await _familyRatings.Find(value =>
@@ -60,6 +80,14 @@ public sealed class DynamicQuestionStore
        DynamicAttemptRequest request)
    {
        var isCorrect = !request.IsSkip && request.ChoiceId == generated.Question.CorrectChoiceId;
+        var selectedAnswer = generated.Blueprint.Distractors
+            .Concat(generated.Blueprint.CaseFamilies
+                .Where(value => value.Id == claims.CaseFamilyId)
+                .SelectMany(value => value.Distractors))
+            .FirstOrDefault(value => value.Id == request.ChoiceId);
+        var misconceptionId = isCorrect || request.IsSkip
+            ? ""
+            : selectedAnswer?.MisconceptionId ?? "";
        var attempt = new DynamicQuestionAttempt
        {
            UserId = claims.UserId,
@@ -74,6 +102,8 @@ public sealed class DynamicQuestionStore
            ChoiceId = request.ChoiceId,
            IsCorrect = isCorrect,
            IsSkip = request.IsSkip,
+            SelectedMisconceptionId = misconceptionId,
+            IsScheduledReview = claims.IsScheduledReview,
            DurationSeconds = Math.Clamp(request.DurationSeconds, 0, 86_400),
            CreatedUtc = DateTime.UtcNow
        };
@@ -117,6 +147,9 @@ public sealed class DynamicQuestionStore
                .Min(value => value.Elo, DynamicDifficultyPolicy.MaximumElo));
        if (!string.IsNullOrWhiteSpace(claims.CaseFamilyId))
            await UpdateFamilyRatingAsync(claims, generated, isCorrect);
+        DateTime? nextReviewUtc = null;
+        if (!request.IsSkip)
+            nextReviewUtc = await UpdateReviewAsync(claims, misconceptionId, isCorrect);
        return new DynamicAttemptResponse
        {
            Inserted = true,
@@ -124,10 +157,61 @@ public sealed class DynamicQuestionStore
            CorrectChoiceId = generated.Question.CorrectChoiceId,
            ObjectiveId = claims.ObjectiveId,
            ObjectiveElo = updated.Elo,
-            RatingChange = delta
+            RatingChange = delta,
+            SelectedMisconceptionId = misconceptionId,
+            IsScheduledReview = claims.IsScheduledReview,
+            NextReviewUtc = nextReviewUtc
        };
    }

+    private async Task<DateTime?> UpdateReviewAsync(
+        DynamicGenerationClaims claims,
+        string misconceptionId,
+        bool isCorrect)
+    {
+        var now = DateTime.UtcNow;
+        var existing = await _reviews.Find(value =>
+            value.UserId == claims.UserId && value.ObjectiveId == claims.ObjectiveId).FirstOrDefaultAsync();
+
+        if (!claims.IsScheduledReview)
+        {
+            if (isCorrect) return existing?.DueUtc;
+            var due = DynamicReviewPolicy.FirstDue(now);
+            var update = Builders<DynamicReviewItem>.Update
+                .SetOnInsert(value => value.UserId, claims.UserId)
+                .SetOnInsert(value => value.ObjectiveId, claims.ObjectiveId)
+                .SetOnInsert(value => value.CreatedUtc, now)
+                .Set(value => value.BlueprintExternalId, claims.BlueprintExternalId)
+                .Set(value => value.BlueprintVersion, claims.BlueprintVersion)
+                .Set(value => value.MisconceptionId, misconceptionId)
+                .Set(value => value.State, DynamicReviewStates.Pending)
+                .Set(value => value.Stage, 0)
+                .Set(value => value.DueUtc, due)
+                .Set(value => value.UpdatedUtc, now);
+            await _reviews.UpdateOneAsync(
+                value => value.UserId == claims.UserId && value.ObjectiveId == claims.ObjectiveId,
+                update,
+                new UpdateOptions { IsUpsert = true });
+            return due;
+        }
+
+        if (existing is null || existing.Id != claims.ReviewItemId || existing.Stage != claims.ReviewStage)
+            return null;
+        var next = DynamicReviewPolicy.AfterReview(existing.Stage, isCorrect, now);
+        var reviewUpdate = Builders<DynamicReviewItem>.Update
+            .Set(value => value.Stage, next.Stage)
+            .Set(value => value.State, next.Mastered ? DynamicReviewStates.Mastered : DynamicReviewStates.Pending)
+            .Set(value => value.MisconceptionId, isCorrect ? existing.MisconceptionId : misconceptionId)
+            .Set(value => value.UpdatedUtc, now)
+            .Inc(value => value.ReviewCount, 1);
+        if (next.DueUtc.HasValue)
+            reviewUpdate = reviewUpdate.Set(value => value.DueUtc, next.DueUtc.Value);
+        await _reviews.UpdateOneAsync(
+            value => value.Id == existing.Id && value.Stage == claims.ReviewStage,
+            reviewUpdate);
+        return next.DueUtc;
+    }
+
    private async Task EnsureMasteryAsync(string userId, string objectiveId)
    {
        try
@@ -194,18 +278,55 @@ public sealed class DynamicQuestionStore

    public async Task<DynamicQuestionStatusResponse> StatusAsync(string userId)
    {
-        var objectiveCount = (await PublishedAsync()).Select(value => value.ObjectiveId).Distinct().Count();
+        var blueprints = (await PublishedAsync())
+            .GroupBy(value => value.ObjectiveId)
+            .Select(value => value.OrderByDescending(item => item.Version).First())
+            .ToList();
+        var objectiveCount = blueprints.Count;
        var mastery = await _mastery.Find(value => value.UserId == userId).ToListAsync();
+        var masteryByObjective = mastery.ToDictionary(value => value.ObjectiveId, StringComparer.Ordinal);
+        var due = await _reviews.CountDocumentsAsync(value => value.UserId == userId
+            && value.State == DynamicReviewStates.Pending
+            && value.DueUtc <= DateTime.UtcNow);
        return new DynamicQuestionStatusResponse
        {
            Available = objectiveCount > 0,
            PublishedObjectives = objectiveCount,
            AttemptedObjectives = mastery.Count,
            TotalAttempts = mastery.Sum(value => value.Attempts),
-            AverageObjectiveElo = mastery.Count == 0 ? 1000 : (int)Math.Round(mastery.Average(value => value.Elo))
+            AverageObjectiveElo = mastery.Count == 0 ? 1000 : (int)Math.Round(mastery.Average(value => value.Elo)),
+            ReviewsDue = checked((int)due),
+            SystemCoverage = Coverage(blueprints, masteryByObjective, value => value.System),
+            PhysicianTaskCoverage = Coverage(blueprints, masteryByObjective, value => value.PhysicianTask)
        };
    }

+    private static List<DynamicCoverageSummary> Coverage(
+        IReadOnlyCollection<DynamicQuestionBlueprint> blueprints,
+        IReadOnlyDictionary<string, DynamicObjectiveMastery> mastery,
+        Func<DynamicQuestionBlueprint, string> keySelector) =>
+        blueprints
+            .GroupBy(value => string.IsNullOrWhiteSpace(keySelector(value)) ? "Unclassified" : keySelector(value))
+            .Select(group =>
+            {
+                var attempts = group
+                    .Where(value => mastery.ContainsKey(value.ObjectiveId))
+                    .Select(value => mastery[value.ObjectiveId])
+                    .ToList();
+                return new DynamicCoverageSummary
+                {
+                    Id = group.Key,
+                    PublishedObjectives = group.Count(),
+                    AttemptedObjectives = attempts.Count,
+                    Attempts = attempts.Sum(value => value.Attempts),
+                    CoveragePercent = group.Count() == 0 ? 0 : Math.Round(attempts.Count * 100.0 / group.Count(), 1),
+                    AverageElo = attempts.Count == 0 ? 1000 : (int)Math.Round(attempts.Average(value => value.Elo))
+                };
+            })
+            .OrderBy(value => value.CoveragePercent)
+            .ThenBy(value => value.Id)
+            .ToList();
+
    public async Task<(int Inserted, int Unchanged)> UpsertBlueprintsAsync(IEnumerable<DynamicQuestionBlueprint> blueprints)
    {
        var inserted = 0;