Add question bank mastery paths and gap questions

2026-07-31 23:46:46 -05:00
parent 012d0cf196
commit 8c439c6294
28 changed files with 1409 additions and 121 deletions
@@ -22,23 +22,7 @@ public sealed class DashboardStore
    private readonly CcsRatingStore _ccsRatingStore;
    private readonly IMongoCollection<UserRating> _ratings;

-    public static readonly string[] Step3Systems =
-    [
-        "Human Development",
-        "Immune System, Blood & Lymphoreticular System, and Multisystem Processes/Disorders",
-        "Behavioral Health",
-        "Nervous System & Special Senses",
-        "Skin & Subcutaneous Tissue",
-        "Musculoskeletal System",
-        "Cardiovascular System",
-        "Respiratory System",
-        "Gastrointestinal System",
-        "Renal/Urinary & Male Reproductive Systems",
-        "Pregnancy/Childbirth & Female Reproductive System & Breast",
-        "Endocrine System",
-        "Biostatistics & Epidemiology/Population Health & Interpretation of the Medical Literature",
-        "Social Sciences: Communication Skills/Ethics/Patient Safety"
-    ];
+    public static readonly string[] Step3Systems = Step3SystemCatalog.All;

    public DashboardStore(
        MongoDatabaseProvider mongo,
@@ -69,6 +53,7 @@ public sealed class DashboardStore

    public async Task<DashboardResponse> GetAsync(string userId, DateTime nowUtc)
    {
+        await _reviewQueue.EnsureUserQueueAsync(userId);
        var today = nowUtc.Date;
        var trendStart = today.AddDays(-29);
        var weekStart = today.AddDays(-6);
@@ -86,22 +71,25 @@ public sealed class DashboardStore
        var bookmarkCountTask = _bookmarks.CountDocumentsAsync(value => value.UserId == userId);
        var notepadTask = _notes.Find(value => value.UserId == userId).FirstOrDefaultAsync();
        var reviewAttemptsTask = _reviewQueue.GetReviewsSinceAsync(userId, weekStart);
+        var allReviewAttemptsTask = _reviewQueue.GetUserReviewsAsync(userId);
+        var reviewItemsTask = _reviewQueue.GetUserItemsAsync(userId);
        var flashcardCountsTask = _flashcards.GetDashboardCountsAsync(userId, nowUtc, weekStart);
        var flashcardReviewsTask = _flashcardSchedules.GetReviewsSinceAsync(userId, weekStart);
        var dynamicReviewsTask = _dynamicQuestions.DueReviewsAsync(userId, nowUtc);
        await Task.WhenAll(attemptsTask, ratingTask, ccsRatingTask, ccsAttemptsTask, ccsAllAttemptsTask, completedCcsCountTask,
            simulationsTask, completedSimulationsTask, bookmarkCountTask, notepadTask, reviewAttemptsTask, flashcardCountsTask,
-            flashcardReviewsTask, dynamicReviewsTask);
+            flashcardReviewsTask, dynamicReviewsTask, allReviewAttemptsTask, reviewItemsTask);

        var attempts = attemptsTask.Result;
        var answeredAttempts = attempts.Where(value => !value.IsSkip).ToList();
        var recentAnswered = answeredAttempts.OrderByDescending(value => value.CreatedUtc).Take(100).ToList();
        var previousHundred = answeredAttempts.OrderByDescending(value => value.CreatedUtc).Skip(100).Take(100).ToList();
-        var questionMap = await _questions.GetByExternalIdsAsync(attempts.Select(value => value.QuestionExternalId));
+        var questionMap = await _questions.GetByExternalIdsAsync(attempts.Select(value => value.QuestionExternalId)
+            .Concat(reviewItemsTask.Result.Select(value => value.QuestionExternalId))
+            .Concat(allReviewAttemptsTask.Result.Select(value => value.QuestionExternalId)));
        var ccsAttempts = ccsAttemptsTask.Result;
        var simulations = simulationsTask.Result;
        var completedCcs = ccsAttempts.Where(value => value.State == CcsAttemptStates.Completed && value.CompletedUtc.HasValue).ToList();
-        await _reviewQueue.EnsureUserQueueAsync(userId);
        var pendingReviewCount = await _reviewQueue.CountPendingAsync(userId);

        var qbankElo = ratingTask.Result.OverallElo;
@@ -148,7 +136,8 @@ public sealed class DashboardStore
        };

        response.PerformanceTrend = performanceTrend;
-        response.Systems = BuildSystemPerformance(answeredAttempts, questionMap);
+        response.Systems = MasteryCalculator.Calculate(
+            attempts, allReviewAttemptsTask.Result, reviewItemsTask.Result, questionMap, nowUtc);

        response.WeeklyActivity = Enumerable.Range(0, 7).Select(offset =>
        {
@@ -303,59 +292,10 @@ public sealed class DashboardStore
    }

    public static string? NormalizeStep3System(Question question)
-    {
-        foreach (var candidate in new[] { question.System, question.PerformanceCategory, question.Discipline })
-        {
-            var normalized = NormalizeStep3System(candidate);
-            if (normalized is not null)
-                return normalized;
-        }
-        return null;
-    }
+        => Step3SystemCatalog.Normalize(question);

    public static string? NormalizeStep3System(string? value)
-    {
-        if (string.IsNullOrWhiteSpace(value))
-            return null;
-        var exact = Step3Systems.FirstOrDefault(
-            system => string.Equals(system, value.Trim(), StringComparison.OrdinalIgnoreCase));
-        if (exact is not null)
-            return exact;
-
-        var lower = value.Trim().ToLowerInvariant();
-        if (ContainsAny(lower, "human development", "well patient", "preventive", "health maintenance", "geriatr", "age-related"))
-            return Step3Systems[0];
-        if (ContainsAny(lower, "social science", "communication", "ethic", "patient safety", "professionalism", "systems-based"))
-            return Step3Systems[13];
-        if (ContainsAny(lower, "pregnan", "obstetric", "gynec", "female reproductive", "breast", "puerper"))
-            return Step3Systems[10];
-        if (ContainsAny(lower, "renal", "urinary", "male reproductive", "urolog", "nephro"))
-            return Step3Systems[9];
-        if (ContainsAny(lower, "biostat", "epidemi", "population health", "medical literature"))
-            return Step3Systems[12];
-        if (ContainsAny(lower, "immune", "immun", "blood", "hemat", "lymph", "multisystem", "infectious"))
-            return Step3Systems[1];
-        if (ContainsAny(lower, "behavior", "psychi", "mental health"))
-            return Step3Systems[2];
-        if (ContainsAny(lower, "nervous", "neuro", "special sense", "ophthalm", "otolaryng", " ent"))
-            return Step3Systems[3];
-        if (ContainsAny(lower, "skin", "derm", "subcutaneous"))
-            return Step3Systems[4];
-        if (ContainsAny(lower, "musculo", "orthop", "rheumat"))
-            return Step3Systems[5];
-        if (ContainsAny(lower, "cardio", "circulatory"))
-            return Step3Systems[6];
-        if (ContainsAny(lower, "resp", "pulmon"))
-            return Step3Systems[7];
-        if (ContainsAny(lower, "gastro", "digest", "hepato"))
-            return Step3Systems[8];
-        if (ContainsAny(lower, "endocr", "metabolic"))
-            return Step3Systems[11];
-        return null;
-    }
-
-    private static bool ContainsAny(string value, params string[] fragments) =>
-        fragments.Any(value.Contains);
+        => Step3SystemCatalog.Normalize(value);

    private async Task<List<DashboardRecentActivity>> BuildRecentActivityAsync(
        List<QuestionAttempt> attempts,
@@ -0,0 +1,109 @@
+using QuestionsApi.Models;
+
+namespace QuestionsApi.Services;
+
+public static class MasteryCalculator
+{
+    private static readonly IReadOnlyDictionary<int, double> DifficultyWeights = new Dictionary<int, double>
+    {
+        [1] = 0.6, [2] = 0.8, [3] = 1.0, [4] = 1.2, [5] = 1.4
+    };
+
+    public static List<DashboardSystemPerformance> Calculate(
+        IEnumerable<QuestionAttempt> attempts,
+        IEnumerable<QuestionReviewAttempt> reviews,
+        IEnumerable<ReviewQueueItem> queueItems,
+        IReadOnlyDictionary<string, Question> questions,
+        DateTime nowUtc)
+    {
+        var mappedAttempts = attempts.Where(attempt => questions.ContainsKey(attempt.QuestionExternalId)).ToList();
+        var answered = mappedAttempts.Where(attempt => !attempt.IsSkip).ToList();
+        var maxCycle = mappedAttempts.Count == 0 ? 0 : mappedAttempts.Max(attempt => attempt.BankCycle);
+        var reviewList = reviews.OrderBy(review => review.CreatedUtc).ToList();
+        var queueList = queueItems.ToList();
+        var results = new List<DashboardSystemPerformance>();
+        foreach (var system in Step3SystemCatalog.All)
+        {
+            var systemAttempts = answered.Where(attempt =>
+                Step3SystemCatalog.Normalize(questions[attempt.QuestionExternalId]) == system).ToList();
+            var ids = systemAttempts.Select(attempt => attempt.QuestionExternalId).Distinct(StringComparer.Ordinal).ToHashSet(StringComparer.Ordinal);
+            var systemReviews = reviewList.Where(review => ids.Contains(review.QuestionExternalId)).ToList();
+            var pending = queueList.Count(item => item.State == ReviewQueueStates.Pending
+                && questions.TryGetValue(item.QuestionExternalId, out var question)
+                && Step3SystemCatalog.Normalize(question) == system);
+            double earned = 0;
+            double possible = 0;
+            var advancedSuccesses = new HashSet<string>(StringComparer.Ordinal);
+            foreach (var attempt in systemAttempts)
+            {
+                var question = questions[attempt.QuestionExternalId];
+                var weight = DifficultyWeights.GetValueOrDefault(Math.Clamp(question.Difficulty, 1, 5), 1.0)
+                    * Math.Pow(0.5, Math.Max(0, maxCycle - attempt.BankCycle));
+                var outcome = attempt.IsCorrect ? 1.0 : RecoveryOutcome(attempt, systemAttempts, systemReviews);
+                earned += weight * outcome;
+                possible += weight;
+                if (question.Difficulty >= 4 && outcome >= 0.75) advancedSuccesses.Add(question.ExternalId);
+            }
+            var score = possible == 0 ? 0 : (int)Math.Round(earned / possible * 100, MidpointRounding.AwayFromZero);
+            var state = State(ids.Count, score, advancedSuccesses.Count, pending);
+            var lastEvidence = systemAttempts.Select(value => (DateTime?)value.CreatedUtc)
+                .Concat(systemReviews.Select(value => (DateTime?)value.CreatedUtc)).Max();
+            var refreshDue = state is MasteryStates.Strong or MasteryStates.Mastered
+                && lastEvidence.HasValue && lastEvidence.Value < nowUtc.AddDays(-30);
+            results.Add(new DashboardSystemPerformance
+            {
+                System = system,
+                Correct = systemAttempts.Count(value => value.IsCorrect),
+                Total = systemAttempts.Count,
+                Accuracy = DashboardStore.Percentage(systemAttempts.Count(value => value.IsCorrect), systemAttempts.Count),
+                MasteryScore = score,
+                MasteryState = state,
+                DistinctQuestions = ids.Count,
+                Difficulty4PlusSuccessful = advancedSuccesses.Count,
+                PendingReviews = pending,
+                LastEvidenceUtc = lastEvidence,
+                RefreshDue = refreshDue,
+                RecommendedAction = Action(state, pending, refreshDue)
+            });
+        }
+        return results;
+    }
+
+    private static double RecoveryOutcome(
+        QuestionAttempt attempt,
+        IReadOnlyCollection<QuestionAttempt> attempts,
+        IReadOnlyCollection<QuestionReviewAttempt> reviews)
+    {
+        var nextAttempt = attempts.Where(value => value.QuestionExternalId == attempt.QuestionExternalId
+                && value.CreatedUtc > attempt.CreatedUtc)
+            .OrderBy(value => value.CreatedUtc).FirstOrDefault();
+        var recovery = reviews.Where(value => value.QuestionExternalId == attempt.QuestionExternalId
+                && value.IsCorrect && value.CreatedUtc > attempt.CreatedUtc
+                && (nextAttempt is null || value.CreatedUtc < nextAttempt.CreatedUtc))
+            .OrderBy(value => value.CreatedUtc).FirstOrDefault();
+        if (recovery is null) return 0;
+        return recovery.CreatedUtc - attempt.CreatedUtc >= TimeSpan.FromHours(24) ? 0.75 : 0.5;
+    }
+
+    private static string State(int distinct, int score, int advanced, int pending)
+    {
+        if (distinct == 0) return MasteryStates.Unseen;
+        if (distinct < 5) return MasteryStates.Learning;
+        if (distinct >= 25 && score >= 85 && advanced >= 5 && pending == 0) return MasteryStates.Mastered;
+        if (distinct >= 15 && score >= 75) return MasteryStates.Strong;
+        return MasteryStates.Practicing;
+    }
+
+    private static DashboardMasteryAction Action(string state, int pending, bool refreshDue)
+    {
+        if (pending > 0) return new() { Type = "review", Label = "Review missed concepts", QuestionCount = pending };
+        return state switch
+        {
+            MasteryStates.Unseen or MasteryStates.Learning => new() { Type = "practice", Label = "Start learning", QuestionCount = 10 },
+            MasteryStates.Practicing => new() { Type = "practice", Label = "Build mastery", QuestionCount = 10, DifficultyMin = 3, DifficultyMax = 4 },
+            MasteryStates.Strong => new() { Type = "practice", Label = "Challenge this system", QuestionCount = 10, DifficultyMin = 4, DifficultyMax = 5 },
+            MasteryStates.Mastered when refreshDue => new() { Type = "practice", Label = "Refresh mastery", QuestionCount = 5, DifficultyMin = 4, DifficultyMax = 5 },
+            _ => new() { Type = "practice", Label = "Maintain mastery", QuestionCount = 5, DifficultyMin = 4, DifficultyMax = 5 }
+        };
+    }
+}
@@ -21,6 +21,10 @@ public class QuestionStore
        _col.Indexes.CreateOne(new CreateIndexModel<Question>(systemIndex));
        var tagsIndex = Builders<Question>.IndexKeys.Ascending(q => q.Tags);
        _col.Indexes.CreateOne(new CreateIndexModel<Question>(tagsIndex));
+        var step3SystemIndex = Builders<Question>.IndexKeys.Ascending(q => q.Step3System)
+            .Ascending(q => q.IsPublished).Ascending(q => q.FlaggedForReviewUtc).Ascending(q => q.Difficulty);
+        _col.Indexes.CreateOne(new CreateIndexModel<Question>(step3SystemIndex,
+            new CreateIndexOptions { Name = "step3_system_visibility_difficulty" }));
    }

    public async Task<List<Question>> SearchAsync(QuestionSearchRequest req, bool includeUnpublished)
@@ -55,10 +59,18 @@ public class QuestionStore

        var filter = filters.Count == 0 ? builder.Empty : builder.And(filters);
        var limit = Math.Clamp(req.Limit, 1, 80);
-        return await _col.Aggregate()
-            .Match(filter)
-            .Sample(limit)
-            .ToListAsync();
+        if (string.IsNullOrWhiteSpace(req.Step3System))
+            return await _col.Aggregate().Match(filter).Sample(limit).ToListAsync();
+
+        var requested = Step3SystemCatalog.Normalize(req.Step3System)
+            ?? throw new ArgumentException("step3System must be an official Step 3 system");
+        // Include legacy records until the idempotent Step3System backfill has run.
+        var systemFilter = builder.Eq(question => question.Step3System, requested)
+            | builder.Exists(nameof(Question.Step3System), false)
+            | builder.Eq(question => question.Step3System, "");
+        var candidates = await _col.Find(filter & systemFilter).ToListAsync();
+        return candidates.Where(question => Step3SystemCatalog.Normalize(question) == requested)
+            .OrderBy(_ => Random.Shared.Next()).Take(limit).ToList();
    }

    public async Task<Question?> GetByIdAsync(string id) =>
@@ -280,6 +292,11 @@ public class QuestionStore
        ChoiceSelectionPercentages = req.ChoiceSelectionPercentages,
        Discipline = req.Discipline.Trim(),
        System = req.System.Trim(),
+        Step3System = Step3SystemCatalog.Derive(new Question
+        {
+            ExternalId = req.ExternalId.Trim(), System = req.System.Trim(),
+            PerformanceCategory = req.PerformanceCategory.Trim(), Discipline = req.Discipline.Trim()
+        }) ?? "",
        PerformanceCategory = RatingCategory.Normalize(req.PerformanceCategory, req.System, req.Discipline),
        Difficulty = req.Difficulty,
        Labs = req.Labs,
@@ -46,15 +46,20 @@ public sealed class ReviewQueueStore
        var update = Builders<ReviewQueueItem>.Update
            .SetOnInsert(value => value.UserId, attempt.UserId)
            .SetOnInsert(value => value.QuestionExternalId, attempt.QuestionExternalId)
-            .SetOnInsert(value => value.SourceAttemptId, attempt.Id ?? "")
-            .SetOnInsert(value => value.State, ReviewQueueStates.Pending)
            .SetOnInsert(value => value.Cycle, 1)
            .SetOnInsert(value => value.CreatedUtc, attempt.CreatedUtc)
-            .SetOnInsert(value => value.UpdatedUtc, now);
+            .Set(value => value.SourceAttemptId, attempt.Id ?? "")
+            .Set(value => value.State, ReviewQueueStates.Pending)
+            .Set(value => value.MasteredUtc, null)
+            .Set(value => value.UpdatedUtc, now);
+        var filter = Builders<ReviewQueueItem>.Filter.Eq(value => value.UserId, attempt.UserId)
+            & Builders<ReviewQueueItem>.Filter.Eq(value => value.QuestionExternalId, attempt.QuestionExternalId)
+            & (Builders<ReviewQueueItem>.Filter.Ne(value => value.State, ReviewQueueStates.Mastered)
+                | Builders<ReviewQueueItem>.Filter.Ne(value => value.SourceAttemptId, attempt.Id ?? ""));
        try
        {
            await _items.UpdateOneAsync(
-                value => value.UserId == attempt.UserId && value.QuestionExternalId == attempt.QuestionExternalId,
+                filter,
                update,
                new UpdateOptions { IsUpsert = true });
        }
@@ -66,26 +71,36 @@ public sealed class ReviewQueueStore

    public async Task EnsureUserQueueAsync(string userId)
    {
-        var checkpoint = await _backfills.Find(value => value.UserId == userId).FirstOrDefaultAsync();
        var scanThroughUtc = DateTime.UtcNow;
        var filter = Builders<QuestionAttempt>.Filter.Where(value => value.UserId == userId && !value.IsCorrect && !value.IsSkip);
-        if (checkpoint is not null)
-            filter &= Builders<QuestionAttempt>.Filter.Gte(value => value.CreatedUtc, checkpoint.CompletedUtc);
        filter &= Builders<QuestionAttempt>.Filter.Lte(value => value.CreatedUtc, scanThroughUtc);
-        var misses = await _firstAttempts.Find(filter).ToListAsync();
+        var allMisses = await _firstAttempts.Find(filter).ToListAsync();
+        var existingItems = (await _items.Find(value => value.UserId == userId).ToListAsync())
+            .ToDictionary(value => value.QuestionExternalId, StringComparer.Ordinal);
+        var misses = allMisses.GroupBy(value => value.QuestionExternalId, StringComparer.Ordinal)
+            .Select(group => group.OrderByDescending(value => value.CreatedUtc).First())
+            .Where(attempt => !existingItems.TryGetValue(attempt.QuestionExternalId, out var item)
+                || (item.State == ReviewQueueStates.Mastered
+                    && item.SourceAttemptId != (attempt.Id ?? "")
+                    && (!item.MasteredUtc.HasValue || attempt.CreatedUtc > item.MasteredUtc.Value)))
+            .ToList();
        if (misses.Count > 0)
        {
            var now = DateTime.UtcNow;
            var writes = misses.Select(attempt => new UpdateOneModel<ReviewQueueItem>(
-                Builders<ReviewQueueItem>.Filter.Where(value => value.UserId == userId && value.QuestionExternalId == attempt.QuestionExternalId),
+                Builders<ReviewQueueItem>.Filter.Eq(value => value.UserId, userId)
+                    & Builders<ReviewQueueItem>.Filter.Eq(value => value.QuestionExternalId, attempt.QuestionExternalId)
+                    & (Builders<ReviewQueueItem>.Filter.Ne(value => value.State, ReviewQueueStates.Mastered)
+                        | Builders<ReviewQueueItem>.Filter.Ne(value => value.SourceAttemptId, attempt.Id ?? "")),
                Builders<ReviewQueueItem>.Update
                    .SetOnInsert(value => value.UserId, userId)
                    .SetOnInsert(value => value.QuestionExternalId, attempt.QuestionExternalId)
-                    .SetOnInsert(value => value.SourceAttemptId, attempt.Id ?? "")
-                    .SetOnInsert(value => value.State, ReviewQueueStates.Pending)
                    .SetOnInsert(value => value.Cycle, 1)
                    .SetOnInsert(value => value.CreatedUtc, attempt.CreatedUtc)
-                    .SetOnInsert(value => value.UpdatedUtc, now))
+                    .Set(value => value.SourceAttemptId, attempt.Id ?? "")
+                    .Set(value => value.State, ReviewQueueStates.Pending)
+                    .Set(value => value.MasteredUtc, null)
+                    .Set(value => value.UpdatedUtc, now))
            { IsUpsert = true }).ToList();
            try
            {
@@ -115,27 +130,50 @@ public sealed class ReviewQueueStore
    public Task<long> CountPendingAsync(string userId) =>
        _items.CountDocumentsAsync(value => value.UserId == userId && value.State == ReviewQueueStates.Pending);

-    public async Task<ReviewQueueItem?> GetRandomPendingAsync(string userId, IEnumerable<string> excludedIds)
+    public async Task<long> CountPendingAsync(string userId, string? step3System)
+    {
+        if (string.IsNullOrWhiteSpace(step3System)) return await CountPendingAsync(userId);
+        var requested = Step3SystemCatalog.Normalize(step3System)
+            ?? throw new ArgumentException("step3System must be an official Step 3 system");
+        var pending = await _items.Find(value => value.UserId == userId && value.State == ReviewQueueStates.Pending).ToListAsync();
+        var count = 0L;
+        foreach (var item in pending)
+        {
+            var question = await _questions.GetByExternalIdAsync(item.QuestionExternalId);
+            if (question is not null && Step3SystemCatalog.Normalize(question) == requested) count++;
+        }
+        return count;
+    }
+
+    public Task<List<ReviewQueueItem>> GetUserItemsAsync(string userId) =>
+        _items.Find(value => value.UserId == userId).ToListAsync();
+
+    public Task<List<QuestionReviewAttempt>> GetUserReviewsAsync(string userId) =>
+        _reviews.Find(value => value.UserId == userId).ToListAsync();
+
+    public async Task<ReviewQueueItem?> GetRandomPendingAsync(
+        string userId, IEnumerable<string> excludedIds, string? step3System = null)
    {
        var excluded = excludedIds.Where(value => !string.IsNullOrWhiteSpace(value)).Distinct(StringComparer.Ordinal).ToList();
-        for (var pass = 0; pass < 5; pass++)
+        var requestedSystem = string.IsNullOrWhiteSpace(step3System) ? null : Step3SystemCatalog.Normalize(step3System);
+        if (!string.IsNullOrWhiteSpace(step3System) && requestedSystem is null)
+            throw new ArgumentException("step3System must be an official Step 3 system");
+        var filter = Builders<ReviewQueueItem>.Filter.Eq(value => value.UserId, userId)
+            & Builders<ReviewQueueItem>.Filter.Eq(value => value.State, ReviewQueueStates.Pending);
+        if (excluded.Count > 0) filter &= Builders<ReviewQueueItem>.Filter.Nin(value => value.QuestionExternalId, excluded);
+        var candidates = await _items.Find(filter).ToListAsync();
+        foreach (var candidate in candidates.OrderBy(_ => Random.Shared.Next()))
        {
-            var filter = Builders<ReviewQueueItem>.Filter.Eq(value => value.UserId, userId)
-                & Builders<ReviewQueueItem>.Filter.Eq(value => value.State, ReviewQueueStates.Pending);
-            if (excluded.Count > 0)
-                filter &= Builders<ReviewQueueItem>.Filter.Nin(value => value.QuestionExternalId, excluded);
-            var candidates = await _items.Aggregate().Match(filter).Sample(20).ToListAsync();
-            if (candidates.Count == 0) return null;
-            foreach (var candidate in candidates)
+            var question = await _questions.GetByExternalIdAsync(candidate.QuestionExternalId);
+            if (question is { IsPublished: true, FlaggedForReviewUtc: null })
            {
-                var question = await _questions.GetByExternalIdAsync(candidate.QuestionExternalId);
-                if (question is { IsPublished: true, FlaggedForReviewUtc: null }) return candidate;
-                await _items.UpdateOneAsync(
-                    value => value.Id == candidate.Id && value.State == ReviewQueueStates.Pending,
-                    Builders<ReviewQueueItem>.Update
-                        .Set(value => value.State, ReviewQueueStates.Unavailable)
-                        .Set(value => value.UpdatedUtc, DateTime.UtcNow));
+                if (requestedSystem is null || Step3SystemCatalog.Normalize(question) == requestedSystem) return candidate;
+                continue;
            }
+            await _items.UpdateOneAsync(
+                value => value.Id == candidate.Id && value.State == ReviewQueueStates.Pending,
+                Builders<ReviewQueueItem>.Update.Set(value => value.State, ReviewQueueStates.Unavailable)
+                    .Set(value => value.UpdatedUtc, DateTime.UtcNow));
        }
        return null;
    }
@@ -0,0 +1,53 @@
+using MongoDB.Bson;
+using MongoDB.Driver;
+using QuestionsApi.Models;
+
+namespace QuestionsApi.Services;
+
+public static class Step3SystemBackfillCommand
+{
+    public const string Confirmation = "BACKFILL_STEP3_SYSTEMS";
+
+    public static async Task<int> RunAsync(IConfiguration configuration, string[] args, TextWriter output)
+    {
+        var apply = args.Contains("--apply", StringComparer.Ordinal);
+        var confirmIndex = Array.IndexOf(args, "--confirm");
+        var confirmation = confirmIndex >= 0 && confirmIndex + 1 < args.Length ? args[confirmIndex + 1] : "";
+        var database = new MongoDatabaseProvider(configuration).Database;
+        var questions = database.GetCollection<Question>("Questions");
+        var all = await questions.Find(FilterDefinition<Question>.Empty).ToListAsync();
+        var classified = all.Select(question => (Question: question, System: Step3SystemCatalog.Derive(question) ?? "")).ToList();
+        var changes = classified.Where(item => item.System.Length > 0 && item.Question.Step3System != item.System).ToList();
+        var unmapped = classified.Count(item => item.System.Length == 0);
+        await output.WriteLineAsync($"Questions requiring Step3System backfill: {changes.Count}; unmapped: {unmapped}.");
+        foreach (var group in classified.Where(item => item.System.Length == 0)
+                     .GroupBy(item => $"{item.Question.System} | {item.Question.PerformanceCategory} | {item.Question.Discipline}")
+                     .OrderByDescending(group => group.Count()).ThenBy(group => group.Key, StringComparer.Ordinal))
+            await output.WriteLineAsync($"Unmapped ({group.Count()}): {group.Key}; ids: {string.Join(", ", group.Select(item => item.Question.ExternalId).Take(10))}");
+        if (!apply)
+        {
+            await output.WriteLineAsync($"Dry run only. Re-run with --apply --confirm {Confirmation}.");
+            return 0;
+        }
+        if (confirmation != Confirmation)
+        {
+            await output.WriteLineAsync($"Refusing migration: --confirm {Confirmation} is required.");
+            return 2;
+        }
+        if (changes.Count > 0)
+        {
+            var writes = changes.Select(item => new UpdateOneModel<Question>(
+                Builders<Question>.Filter.Eq(question => question.Id, item.Question.Id),
+                Builders<Question>.Update.Set(question => question.Step3System, item.System))).ToList();
+            await questions.BulkWriteAsync(writes, new BulkWriteOptions { IsOrdered = false });
+        }
+        var remaining = await questions.CountDocumentsAsync(new BsonDocumentFilterDefinition<Question>(
+            new BsonDocument("$or", new BsonArray
+            {
+                new BsonDocument(nameof(Question.Step3System), new BsonDocument("$exists", false)),
+                new BsonDocument(nameof(Question.Step3System), "")
+            })));
+        await output.WriteLineAsync($"Backfilled {changes.Count}; unmapped or empty Step3System: {remaining}.");
+        return 0;
+    }
+}
@@ -0,0 +1,74 @@
+using QuestionsApi.Models;
+
+namespace QuestionsApi.Services;
+
+public static class Step3SystemCatalog
+{
+    private static readonly IReadOnlyDictionary<string, string> Overrides = new Dictionary<string, string>(StringComparer.Ordinal)
+    {
+        ["d5-expansion-pediatrics-urea-cycle-hyperammonemia-001"] = "Endocrine System",
+        ["d5-expansion-pediatrics-neonatal-hsv-disseminated-001"] = "Immune System, Blood & Lymphoreticular System, and Multisystem Processes/Disorders",
+        ["d5-expansion-pediatrics-cdh-delivery-stabilization-001"] = "Respiratory System",
+        ["d5-expansion-pediatrics-hie-therapeutic-hypothermia-001"] = "Nervous System & Special Senses",
+        ["d5-expansion-pediatrics-fnait-intracranial-bleeding-001"] = "Immune System, Blood & Lymphoreticular System, and Multisystem Processes/Disorders",
+        ["nbme6-s3-i029-infant-hyperchloremic-acidosis-growth-risk"] = "Renal/Urinary & Male Reproductive Systems",
+        ["form7-s1-i042-genitourinary-syndrome-menopause"] = "Pregnancy/Childbirth & Female Reproductive System & Breast",
+        ["free137-i019-inguinal-node-vulvar-source"] = "Pregnancy/Childbirth & Female Reproductive System & Breast",
+        ["free137-i061-klinefelter-confirmation"] = "Renal/Urinary & Male Reproductive Systems",
+        ["form7-s1-i009-cremasteric-reflex"] = "Renal/Urinary & Male Reproductive Systems",
+        ["form7-s2-i015-acrosome-head"] = "Renal/Urinary & Male Reproductive Systems",
+        ["nbme6-s1-i033-prostate-primary-osteoblastic-metastases"] = "Renal/Urinary & Male Reproductive Systems",
+        ["nbme6-s3-i043-lytic-lesion-hematuria-staging"] = "Renal/Urinary & Male Reproductive Systems"
+    };
+    public static readonly string[] All =
+    [
+        "Human Development",
+        "Immune System, Blood & Lymphoreticular System, and Multisystem Processes/Disorders",
+        "Behavioral Health",
+        "Nervous System & Special Senses",
+        "Skin & Subcutaneous Tissue",
+        "Musculoskeletal System",
+        "Cardiovascular System",
+        "Respiratory System",
+        "Gastrointestinal System",
+        "Renal/Urinary & Male Reproductive Systems",
+        "Pregnancy/Childbirth & Female Reproductive System & Breast",
+        "Endocrine System",
+        "Biostatistics & Epidemiology/Population Health & Interpretation of the Medical Literature",
+        "Social Sciences: Communication Skills/Ethics/Patient Safety"
+    ];
+
+    public static string? Normalize(Question question) =>
+        Normalize(question.Step3System)
+        ?? Derive(question);
+
+    public static string? Derive(Question question) =>
+        Overrides.GetValueOrDefault(question.ExternalId)
+        ?? Normalize(question.System) ?? Normalize(question.PerformanceCategory) ?? Normalize(question.Discipline);
+
+    public static string? Normalize(string? value)
+    {
+        if (string.IsNullOrWhiteSpace(value)) return null;
+        var trimmed = value.Trim();
+        var exact = All.FirstOrDefault(system => string.Equals(system, trimmed, StringComparison.OrdinalIgnoreCase));
+        if (exact is not null) return exact;
+        var lower = trimmed.ToLowerInvariant();
+        if (Has(lower, "human development", "well patient", "preventive", "health maintenance", "geriatr", "age-related", "neonat")) return All[0];
+        if (Has(lower, "social science", "communication", "ethic", "patient safety", "professionalism", "systems-based")) return All[13];
+        if (Has(lower, "pregnan", "obstetric", "gynec", "female reproductive", "breast", "puerper")) return All[10];
+        if (Has(lower, "renal", "urinary", "male reproductive", "urolog", "nephro")) return All[9];
+        if (Has(lower, "biostat", "epidemi", "population health", "medical literature")) return All[12];
+        if (Has(lower, "immune", "immun", "blood", "hemat", "lymph", "multisystem", "infectious", "toxicolog", "poisoning", "oncolog", "trauma")) return All[1];
+        if (Has(lower, "behavior", "psychi", "mental health")) return All[2];
+        if (Has(lower, "nervous", "neuro", "special sense", "ophthalm", "eye & adnexa", "otolaryng", "ear, nose", " ent")) return All[3];
+        if (Has(lower, "skin", "derm", "subcutaneous", "integument")) return All[4];
+        if (Has(lower, "musculo", "orthop", "rheumat")) return All[5];
+        if (Has(lower, "cardio", "circulatory", "peripheral vascular")) return All[6];
+        if (Has(lower, "resp", "pulmon")) return All[7];
+        if (Has(lower, "gastro", "digest", "hepato", "oral/dental")) return All[8];
+        if (Has(lower, "endocr", "metabolic")) return All[11];
+        return null;
+    }
+
+    private static bool Has(string value, params string[] fragments) => fragments.Any(value.Contains);
+}