diff --git a/api/QuestionsApi.Tests/DynamicQuestionTests.cs b/api/QuestionsApi.Tests/DynamicQuestionTests.cs index 107c66d..e844a4e 100644 --- a/api/QuestionsApi.Tests/DynamicQuestionTests.cs +++ b/api/QuestionsApi.Tests/DynamicQuestionTests.cs @@ -119,6 +119,63 @@ public sealed class DynamicQuestionTests Assert.Equal(1800, DynamicDifficultyPolicy.Clamp(5000)); } + [Fact] + public void ReviewPolicy_UsesFixedIntervals_AndResetsAfterAMiss() + { + var now = new DateTime(2026, 7, 23, 12, 0, 0, DateTimeKind.Utc); + + Assert.Equal(now.AddDays(1), DynamicReviewPolicy.FirstDue(now)); + Assert.Equal((1, now.AddDays(3), false), DynamicReviewPolicy.AfterReview(0, true, now)); + Assert.Equal((2, now.AddDays(7), false), DynamicReviewPolicy.AfterReview(1, true, now)); + Assert.Equal((3, now.AddDays(21), false), DynamicReviewPolicy.AfterReview(2, true, now)); + Assert.Equal((4, now.AddDays(45), false), DynamicReviewPolicy.AfterReview(3, true, now)); + Assert.Equal((5, null, true), DynamicReviewPolicy.AfterReview(4, true, now)); + Assert.Equal((0, now.AddDays(1), false), DynamicReviewPolicy.AfterReview(3, false, now)); + } + + [Fact] + public void QualityStandardFour_RequiresCoverageReviewAndMisconceptionMetadata() + { + var options = new System.Text.Json.JsonSerializerOptions(System.Text.Json.JsonSerializerDefaults.Web) + { PropertyNameCaseInsensitive = true }; + var root = Path.GetFullPath(Path.Combine( + AppContext.BaseDirectory, "..", "..", "..", "..", "..", "content", "dynamic_questions")); + var source = Directory.EnumerateFiles(root, "*.json") + .SelectMany(file => System.Text.Json.JsonSerializer.Deserialize>( + File.ReadAllText(file), options) ?? []) + .ToList(); + var blueprint = DynamicBlueprintResolver.ResolveAll(source) + .First(value => value.QualityStandardVersion == 3); + blueprint.QualityStandardVersion = 4; + blueprint.BlueprintWeight = 6; + + Assert.Contains("encounterFrame", DynamicQuestionGenerator.Validate(blueprint)); + + blueprint.System = "Endocrine"; + blueprint.PhysicianTask = "Pharmacotherapy"; + blueprint.EncounterFrame = "Urgent Intervention"; + blueprint.SiteOfCare = "Emergency Department"; + blueprint.ConceptIds = ["test.concept"]; + blueprint.MedicallyReviewedUtc = DateTime.UtcNow.AddDays(-1); + blueprint.MedicalReviewExpiresUtc = DateTime.UtcNow.AddYears(1); + blueprint.MedicalReviewer = "Medical editor"; + foreach (var distractor in blueprint.Distractors) + distractor.MisconceptionId = $"misconception.{distractor.Id}"; + + Assert.Null(DynamicQuestionGenerator.Validate(blueprint)); + } + + [Fact] + public void CoveragePolicy_UsesOfficialDefaults_AndAllowsVersionedOverrides() + { + Assert.Equal(12, DynamicBlueprintCoveragePolicy.Weight( + new DynamicQuestionBlueprint { System = "Biostatistics" })); + Assert.Equal(10, DynamicBlueprintCoveragePolicy.Weight( + new DynamicQuestionBlueprint { System = "Cardiovascular" })); + Assert.Equal(2.5, DynamicBlueprintCoveragePolicy.Weight( + new DynamicQuestionBlueprint { System = "Cardiovascular", BlueprintWeight = 2.5 })); + } + [Fact] public void AdaptiveContent_ResolvesToFourCompleteTiers() { diff --git a/api/QuestionsApi/Models/DashboardModels.cs b/api/QuestionsApi/Models/DashboardModels.cs index 37968fa..43f8508 100644 --- a/api/QuestionsApi/Models/DashboardModels.cs +++ b/api/QuestionsApi/Models/DashboardModels.cs @@ -14,6 +14,7 @@ public sealed class DashboardResponse public int StudyTimeSecondsThisWeek { get; set; } public int CurrentStreak { get; set; } public int MissedQuestionCount { get; set; } + public int DynamicReviewsDue { get; set; } public int BookmarkCount { get; set; } public int NoteCount { get; set; } public int FlashcardsDue { get; set; } diff --git a/api/QuestionsApi/Models/DynamicQuestionModels.cs b/api/QuestionsApi/Models/DynamicQuestionModels.cs index 899afdb..0b79ef5 100644 --- a/api/QuestionsApi/Models/DynamicQuestionModels.cs +++ b/api/QuestionsApi/Models/DynamicQuestionModels.cs @@ -16,6 +16,14 @@ public sealed class DynamicQuestionBlueprint public string Discipline { get; set; } = ""; public string PerformanceCategory { get; set; } = ""; public string PhysicianTask { get; set; } = ""; + public string EncounterFrame { get; set; } = ""; + public string SiteOfCare { get; set; } = ""; + public double BlueprintWeight { get; set; } + public List ConceptIds { get; set; } = []; + public List PrerequisiteConceptIds { get; set; } = []; + public DateTime? MedicallyReviewedUtc { get; set; } + public DateTime? MedicalReviewExpiresUtc { get; set; } + public string MedicalReviewer { get; set; } = ""; public int Difficulty { get; set; } = 3; public int QualityStandardVersion { get; set; } = 1; public string CorrectConceptId { get; set; } = ""; @@ -115,6 +123,8 @@ public sealed class DynamicAnswerTemplate public string Id { get; set; } = ""; public string TextTemplate { get; set; } = ""; public string ExplanationTemplate { get; set; } = ""; + public string MisconceptionId { get; set; } = ""; + public List RemediationConceptIds { get; set; } = []; } public sealed class DynamicGenerationClaims @@ -130,6 +140,9 @@ public sealed class DynamicGenerationClaims public string DifficultyTier { get; set; } = ""; public int EffectiveAbilityElo { get; set; } public int FamilyElo { get; set; } + public bool IsScheduledReview { get; set; } + public string ReviewItemId { get; set; } = ""; + public int ReviewStage { get; set; } public DateTime ExpiresUtc { get; set; } } @@ -183,6 +196,9 @@ public sealed class DynamicAttemptResponse public string ObjectiveId { get; set; } = ""; public int ObjectiveElo { get; set; } public int RatingChange { get; set; } + public string SelectedMisconceptionId { get; set; } = ""; + public bool IsScheduledReview { get; set; } + public DateTime? NextReviewUtc { get; set; } } public sealed class DynamicReportRequest @@ -199,6 +215,19 @@ public sealed class DynamicQuestionStatusResponse public int AttemptedObjectives { get; set; } public int TotalAttempts { get; set; } public int AverageObjectiveElo { get; set; } = 1000; + public int ReviewsDue { get; set; } + public List SystemCoverage { get; set; } = []; + public List PhysicianTaskCoverage { get; set; } = []; +} + +public sealed class DynamicCoverageSummary +{ + public string Id { get; set; } = ""; + public int PublishedObjectives { get; set; } + public int AttemptedObjectives { get; set; } + public int Attempts { get; set; } + public double CoveragePercent { get; set; } + public int AverageElo { get; set; } = 1000; } public sealed class DynamicQuestionAttempt @@ -218,10 +247,36 @@ public sealed class DynamicQuestionAttempt public string ChoiceId { get; set; } = ""; public bool IsCorrect { get; set; } public bool IsSkip { get; set; } + public string SelectedMisconceptionId { get; set; } = ""; + public bool IsScheduledReview { get; set; } public int DurationSeconds { get; set; } public DateTime CreatedUtc { get; set; } = DateTime.UtcNow; } +public static class DynamicReviewStates +{ + public const string Pending = "pending"; + public const string Mastered = "mastered"; +} + +public sealed class DynamicReviewItem +{ + [BsonId] + [BsonRepresentation(BsonType.ObjectId)] + public string? Id { get; set; } + public string UserId { get; set; } = ""; + public string ObjectiveId { get; set; } = ""; + public string BlueprintExternalId { get; set; } = ""; + public int BlueprintVersion { get; set; } + public string MisconceptionId { get; set; } = ""; + public string State { get; set; } = DynamicReviewStates.Pending; + public int Stage { get; set; } + public int ReviewCount { get; set; } + public DateTime DueUtc { get; set; } + public DateTime CreatedUtc { get; set; } = DateTime.UtcNow; + public DateTime UpdatedUtc { get; set; } = DateTime.UtcNow; +} + public sealed class DynamicObjectiveMastery { [BsonId] diff --git a/api/QuestionsApi/Services/DashboardStore.cs b/api/QuestionsApi/Services/DashboardStore.cs index 7557636..44fe6da 100644 --- a/api/QuestionsApi/Services/DashboardStore.cs +++ b/api/QuestionsApi/Services/DashboardStore.cs @@ -16,6 +16,7 @@ public sealed class DashboardStore private readonly IMongoCollection _notes; private readonly QuestionStore _questions; private readonly ReviewQueueStore _reviewQueue; + private readonly DynamicQuestionStore _dynamicQuestions; private readonly FlashcardStudyService _flashcards; private readonly FlashcardScheduleStore _flashcardSchedules; @@ -23,6 +24,7 @@ public sealed class DashboardStore MongoDatabaseProvider mongo, QuestionStore questions, ReviewQueueStore reviewQueue, + DynamicQuestionStore dynamicQuestions, FlashcardStudyService flashcards, FlashcardScheduleStore flashcardSchedules) { @@ -36,6 +38,7 @@ public sealed class DashboardStore _notes = mongo.Database.GetCollection("UserNotepads"); _questions = questions; _reviewQueue = reviewQueue; + _dynamicQuestions = dynamicQuestions; _flashcards = flashcards; _flashcardSchedules = flashcardSchedules; } @@ -57,8 +60,10 @@ public sealed class DashboardStore var reviewAttemptsTask = _reviewQueue.GetReviewsSinceAsync(userId, weekStart); var flashcardCountsTask = _flashcards.GetDashboardCountsAsync(userId, nowUtc, weekStart); var flashcardReviewsTask = _flashcardSchedules.GetReviewsSinceAsync(userId, weekStart); + var dynamicReviewsTask = _dynamicQuestions.DueReviewsAsync(userId, nowUtc); await Task.WhenAll(attemptsTask, ratingTask, ccsRatingTask, ccsAttemptsTask, completedCcsCountTask, - simulationsTask, bookmarkCountTask, notepadTask, reviewAttemptsTask, flashcardCountsTask, flashcardReviewsTask); + simulationsTask, bookmarkCountTask, notepadTask, reviewAttemptsTask, flashcardCountsTask, + flashcardReviewsTask, dynamicReviewsTask); var attempts = attemptsTask.Result; var answeredAttempts = attempts.Where(value => !value.IsSkip).ToList(); @@ -84,6 +89,7 @@ public sealed class DashboardStore BookmarkCount = checked((int)bookmarkCountTask.Result), NoteCount = CountNotes(notepadTask.Result?.Content), MissedQuestionCount = checked((int)pendingReviewCount), + DynamicReviewsDue = dynamicReviewsTask.Result.Count, FlashcardsDue = flashcardCountsTask.Result.Due, FlashcardsNewAvailable = flashcardCountsTask.Result.NewAvailable, FlashcardsReviewedThisWeek = flashcardCountsTask.Result.ReviewedThisWeek diff --git a/api/QuestionsApi/Services/DynamicBlueprintCoveragePolicy.cs b/api/QuestionsApi/Services/DynamicBlueprintCoveragePolicy.cs new file mode 100644 index 0000000..b1bc76b --- /dev/null +++ b/api/QuestionsApi/Services/DynamicBlueprintCoveragePolicy.cs @@ -0,0 +1,37 @@ +using QuestionsApi.Models; + +namespace QuestionsApi.Services; + +public static class DynamicBlueprintCoveragePolicy +{ + private static readonly Dictionary SystemWeights = + new(StringComparer.OrdinalIgnoreCase) + { + ["Nutrition"] = 17.5, + ["Biostatistics"] = 12, + ["Cardiovascular"] = 10, + ["Neurologic"] = 9, + ["Nervous System"] = 9, + ["Pulmonary"] = 9, + ["Respiratory"] = 9, + ["Pregnancy"] = 8, + ["Social Sciences"] = 8, + ["Gastrointestinal"] = 7, + ["Immune System"] = 7, + ["Hematology"] = 7, + ["Multisystem"] = 7, + ["Infectious Disease"] = 7, + ["Endocrine"] = 6, + ["Musculoskeletal"] = 6, + ["Behavioral Health"] = 5, + ["Renal"] = 5, + ["Skin"] = 5, + ["Human Development"] = 2 + }; + + public static double Weight(DynamicQuestionBlueprint blueprint) + { + if (blueprint.BlueprintWeight > 0) return blueprint.BlueprintWeight; + return SystemWeights.GetValueOrDefault(blueprint.System, 5); + } +} diff --git a/api/QuestionsApi/Services/DynamicBlueprintResolver.cs b/api/QuestionsApi/Services/DynamicBlueprintResolver.cs index e9a1410..bb1e3db 100644 --- a/api/QuestionsApi/Services/DynamicBlueprintResolver.cs +++ b/api/QuestionsApi/Services/DynamicBlueprintResolver.cs @@ -29,7 +29,18 @@ public static class DynamicBlueprintResolver materialized.BaseBlueprintVersion = blueprint.BaseBlueprintVersion; materialized.QualityStandardVersion = blueprint.QualityStandardVersion; materialized.IsPublished = blueprint.IsPublished; - materialized.CaseFamilies = blueprint.CaseFamilies.Select(overlay => + materialized.EncounterFrame = blueprint.EncounterFrame; + materialized.SiteOfCare = blueprint.SiteOfCare; + materialized.BlueprintWeight = blueprint.BlueprintWeight; + materialized.ConceptIds = blueprint.ConceptIds; + materialized.PrerequisiteConceptIds = blueprint.PrerequisiteConceptIds; + materialized.MedicallyReviewedUtc = blueprint.MedicallyReviewedUtc; + materialized.MedicalReviewExpiresUtc = blueprint.MedicalReviewExpiresUtc; + materialized.MedicalReviewer = blueprint.MedicalReviewer; + if (blueprint.Distractors.Count > 0) + materialized.Distractors = blueprint.Distractors; + if (blueprint.CaseFamilies.Count > 0) + materialized.CaseFamilies = blueprint.CaseFamilies.Select(overlay => { var baseFamily = baseBlueprint.CaseFamilies.FirstOrDefault(value => value.Id == overlay.BaseFamilyId) ?? throw new InvalidDataException( diff --git a/api/QuestionsApi/Services/DynamicQuestionGenerator.cs b/api/QuestionsApi/Services/DynamicQuestionGenerator.cs index 26199c0..d19ecce 100644 --- a/api/QuestionsApi/Services/DynamicQuestionGenerator.cs +++ b/api/QuestionsApi/Services/DynamicQuestionGenerator.cs @@ -317,6 +317,11 @@ public sealed class DynamicQuestionGenerator if (string.IsNullOrWhiteSpace(blueprint.ExternalId)) return "externalId required"; if (string.IsNullOrWhiteSpace(blueprint.ObjectiveId)) return "objectiveId required"; if (blueprint.Version < 1) return "version must be positive"; + if (blueprint.BlueprintWeight < 0) return "blueprintWeight cannot be negative"; + if (blueprint.IsPublished + && blueprint.MedicalReviewExpiresUtc is { } expiresUtc + && expiresUtc <= DateTime.UtcNow) + return "medical review has expired"; if (blueprint.QualityStandardVersion >= 2) return ValidateV2(blueprint); if (string.IsNullOrWhiteSpace(blueprint.StemTemplate) || string.IsNullOrWhiteSpace(blueprint.LeadIn)) return "stemTemplate and leadIn required"; @@ -355,6 +360,22 @@ public sealed class DynamicQuestionGenerator if (blueprint.CaseFamilies.Any(value => !DynamicDifficultyPolicy.TierElos.ContainsKey(value.DifficultyTier))) return "difficultyTier must be foundation, standard, advanced, or expert"; } + if (blueprint.QualityStandardVersion >= 4) + { + if (blueprint.BlueprintWeight <= 0) return "quality standard 4 requires blueprintWeight"; + if (string.IsNullOrWhiteSpace(blueprint.System) + || string.IsNullOrWhiteSpace(blueprint.PhysicianTask) + || string.IsNullOrWhiteSpace(blueprint.EncounterFrame) + || string.IsNullOrWhiteSpace(blueprint.SiteOfCare)) + return "quality standard 4 requires system, physicianTask, encounterFrame, and siteOfCare"; + if (blueprint.ConceptIds.Count == 0) return "quality standard 4 requires conceptIds"; + if (blueprint.MedicallyReviewedUtc is null + || blueprint.MedicalReviewExpiresUtc is null + || string.IsNullOrWhiteSpace(blueprint.MedicalReviewer)) + return "quality standard 4 requires medical review metadata"; + if (blueprint.Distractors.Any(value => string.IsNullOrWhiteSpace(value.MisconceptionId))) + return "quality standard 4 requires a misconceptionId for every distractor"; + } foreach (var family in blueprint.CaseFamilies) { if (string.IsNullOrWhiteSpace(family.Id)) return "case family id required"; @@ -367,6 +388,9 @@ public sealed class DynamicQuestionGenerator var reviews = family.ReviewFollowups.Count == 0 ? blueprint.ReviewFollowups : family.ReviewFollowups; var wrong = family.WrongAnswerFollowups.Count == 0 ? blueprint.WrongAnswerFollowups : family.WrongAnswerFollowups; if (distractors.Count < 6) return $"{family.Id} requires at least six distractors"; + if (blueprint.QualityStandardVersion >= 4 + && distractors.Any(value => string.IsNullOrWhiteSpace(value.MisconceptionId))) + return $"{family.Id} every distractor requires a misconceptionId"; var answers = distractors.Append(correctAnswer).ToList(); if (answers.Any(value => string.IsNullOrWhiteSpace(value.Id) || string.IsNullOrWhiteSpace(value.TextTemplate) diff --git a/api/QuestionsApi/Services/DynamicQuestionService.cs b/api/QuestionsApi/Services/DynamicQuestionService.cs index 9b69f88..074481a 100644 --- a/api/QuestionsApi/Services/DynamicQuestionService.cs +++ b/api/QuestionsApi/Services/DynamicQuestionService.cs @@ -49,13 +49,52 @@ public sealed class DynamicQuestionService } var mastery = await _store.MasteryAsync(userId); + var dueReviews = await _store.DueReviewsAsync(userId, DateTime.UtcNow); + var dueByObjective = dueReviews.ToDictionary(value => value.ObjectiveId, StringComparer.Ordinal); var candidates = unseen .Where(group => group.Key != run.PreviousObjectiveId || unseen.Count == 1) - .OrderBy(group => mastery.TryGetValue(group.Key, out var value) ? value.Elo : 1000) - .ThenBy(_ => RandomNumberGenerator.GetInt32(int.MaxValue)) .ToList(); - var weakestBand = candidates.Take(Math.Min(4, candidates.Count)).ToList(); - var selectedObjective = weakestBand[RandomNumberGenerator.GetInt32(weakestBand.Count)]; + var roll = RandomNumberGenerator.GetInt32(100); + IGrouping selectedObjective; + var dueCandidates = candidates.Where(value => dueByObjective.ContainsKey(value.Key)).ToList(); + if (roll < 50 && dueCandidates.Count == 0 && dueByObjective.Count > 0) + dueCandidates = byObjective + .Where(value => dueByObjective.ContainsKey(value.Key) + && (value.Key != run.PreviousObjectiveId || dueByObjective.Count == 1)) + .ToList(); + if (roll < 50 && dueCandidates.Count > 0) + { + selectedObjective = dueCandidates + .OrderBy(value => dueByObjective[value.Key].DueUtc) + .ThenBy(_ => RandomNumberGenerator.GetInt32(int.MaxValue)) + .First(); + } + else if (roll < 80) + { + var systemAttempts = byObjective + .GroupBy(group => group.OrderByDescending(value => value.Version).First().System) + .ToDictionary( + group => group.Key, + group => group.Sum(objective => mastery.TryGetValue(objective.Key, out var value) ? value.Attempts : 0), + StringComparer.Ordinal); + selectedObjective = candidates + .OrderBy(group => + { + var latest = group.OrderByDescending(value => value.Version).First(); + var weight = DynamicBlueprintCoveragePolicy.Weight(latest); + return (systemAttempts.GetValueOrDefault(latest.System) + 1) / weight; + }) + .ThenBy(_ => RandomNumberGenerator.GetInt32(int.MaxValue)) + .First(); + } + else + { + var weakestBand = candidates + .OrderBy(group => mastery.TryGetValue(group.Key, out var value) ? value.Elo : 1000) + .Take(Math.Min(4, candidates.Count)) + .ToList(); + selectedObjective = weakestBand[RandomNumberGenerator.GetInt32(weakestBand.Count)]; + } var blueprint = selectedObjective .OrderByDescending(value => value.Version) .First(); @@ -97,11 +136,15 @@ public sealed class DynamicQuestionService DifficultyTier = selectedFamily.DifficultyTier, EffectiveAbilityElo = effectiveAbility, FamilyElo = selectedFamilyElo, + IsScheduledReview = dueByObjective.TryGetValue(blueprint.ObjectiveId, out var reviewItem), + ReviewItemId = reviewItem?.Id ?? "", + ReviewStage = reviewItem?.Stage ?? 0, ExpiresUtc = DateTime.UtcNow.AddDays(7) }; var generated = _generator.Generate( blueprint, claims.Seed, claims.Nonce, claims.GeneratorVersion, claims.CaseFamilyId); - run.SeenObjectiveIds.Add(blueprint.ObjectiveId); + if (!run.SeenObjectiveIds.Contains(blueprint.ObjectiveId, StringComparer.Ordinal)) + run.SeenObjectiveIds.Add(blueprint.ObjectiveId); run.PreviousObjectiveId = blueprint.ObjectiveId; run.RecentCaseFamilyIds.Add(generated.CaseFamilyId); run.RecentCaseFamilyIds = run.RecentCaseFamilyIds.TakeLast(30).ToList(); diff --git a/api/QuestionsApi/Services/DynamicQuestionStore.cs b/api/QuestionsApi/Services/DynamicQuestionStore.cs index 50e7779..932ca7a 100644 --- a/api/QuestionsApi/Services/DynamicQuestionStore.cs +++ b/api/QuestionsApi/Services/DynamicQuestionStore.cs @@ -10,6 +10,7 @@ public sealed class DynamicQuestionStore private readonly IMongoCollection _attempts; private readonly IMongoCollection _mastery; private readonly IMongoCollection _familyRatings; + private readonly IMongoCollection _reviews; private readonly ServiceMetrics? _metrics; public DynamicQuestionStore(MongoDatabaseProvider mongo, ServiceMetrics? metrics = null) @@ -19,6 +20,7 @@ public sealed class DynamicQuestionStore _attempts = mongo.Database.GetCollection("DynamicQuestionAttempts"); _mastery = mongo.Database.GetCollection("DynamicObjectiveMastery"); _familyRatings = mongo.Database.GetCollection("DynamicFamilyRatings"); + _reviews = mongo.Database.GetCollection("DynamicReviewItems"); _blueprints.Indexes.CreateOne(new CreateIndexModel( Builders.IndexKeys.Ascending(value => value.ExternalId).Ascending(value => value.Version), new CreateIndexOptions { Unique = true })); @@ -34,6 +36,19 @@ public sealed class DynamicQuestionStore .Ascending(value => value.BlueprintVersion) .Ascending(value => value.CaseFamilyId), new CreateIndexOptions { Unique = true })); + _reviews.Indexes.CreateMany([ + new CreateIndexModel( + Builders.IndexKeys + .Ascending(value => value.UserId) + .Ascending(value => value.ObjectiveId), + new CreateIndexOptions { Unique = true, Name = "dynamic_review_user_objective" }), + new CreateIndexModel( + Builders.IndexKeys + .Ascending(value => value.UserId) + .Ascending(value => value.State) + .Ascending(value => value.DueUtc), + new CreateIndexOptions { Name = "dynamic_review_user_due" }) + ]); } public async Task> PublishedAsync() => @@ -46,6 +61,11 @@ public sealed class DynamicQuestionStore (await _mastery.Find(value => value.UserId == userId).ToListAsync()) .ToDictionary(value => value.ObjectiveId, StringComparer.Ordinal); + public async Task> DueReviewsAsync(string userId, DateTime nowUtc) => + await _reviews.Find(value => value.UserId == userId + && value.State == DynamicReviewStates.Pending + && value.DueUtc <= nowUtc).ToListAsync(); + public async Task> FamilyRatingsAsync(DynamicQuestionBlueprint blueprint) { var ratings = await _familyRatings.Find(value => @@ -60,6 +80,14 @@ public sealed class DynamicQuestionStore DynamicAttemptRequest request) { var isCorrect = !request.IsSkip && request.ChoiceId == generated.Question.CorrectChoiceId; + var selectedAnswer = generated.Blueprint.Distractors + .Concat(generated.Blueprint.CaseFamilies + .Where(value => value.Id == claims.CaseFamilyId) + .SelectMany(value => value.Distractors)) + .FirstOrDefault(value => value.Id == request.ChoiceId); + var misconceptionId = isCorrect || request.IsSkip + ? "" + : selectedAnswer?.MisconceptionId ?? ""; var attempt = new DynamicQuestionAttempt { UserId = claims.UserId, @@ -74,6 +102,8 @@ public sealed class DynamicQuestionStore ChoiceId = request.ChoiceId, IsCorrect = isCorrect, IsSkip = request.IsSkip, + SelectedMisconceptionId = misconceptionId, + IsScheduledReview = claims.IsScheduledReview, DurationSeconds = Math.Clamp(request.DurationSeconds, 0, 86_400), CreatedUtc = DateTime.UtcNow }; @@ -117,6 +147,9 @@ public sealed class DynamicQuestionStore .Min(value => value.Elo, DynamicDifficultyPolicy.MaximumElo)); if (!string.IsNullOrWhiteSpace(claims.CaseFamilyId)) await UpdateFamilyRatingAsync(claims, generated, isCorrect); + DateTime? nextReviewUtc = null; + if (!request.IsSkip) + nextReviewUtc = await UpdateReviewAsync(claims, misconceptionId, isCorrect); return new DynamicAttemptResponse { Inserted = true, @@ -124,10 +157,61 @@ public sealed class DynamicQuestionStore CorrectChoiceId = generated.Question.CorrectChoiceId, ObjectiveId = claims.ObjectiveId, ObjectiveElo = updated.Elo, - RatingChange = delta + RatingChange = delta, + SelectedMisconceptionId = misconceptionId, + IsScheduledReview = claims.IsScheduledReview, + NextReviewUtc = nextReviewUtc }; } + private async Task UpdateReviewAsync( + DynamicGenerationClaims claims, + string misconceptionId, + bool isCorrect) + { + var now = DateTime.UtcNow; + var existing = await _reviews.Find(value => + value.UserId == claims.UserId && value.ObjectiveId == claims.ObjectiveId).FirstOrDefaultAsync(); + + if (!claims.IsScheduledReview) + { + if (isCorrect) return existing?.DueUtc; + var due = DynamicReviewPolicy.FirstDue(now); + var update = Builders.Update + .SetOnInsert(value => value.UserId, claims.UserId) + .SetOnInsert(value => value.ObjectiveId, claims.ObjectiveId) + .SetOnInsert(value => value.CreatedUtc, now) + .Set(value => value.BlueprintExternalId, claims.BlueprintExternalId) + .Set(value => value.BlueprintVersion, claims.BlueprintVersion) + .Set(value => value.MisconceptionId, misconceptionId) + .Set(value => value.State, DynamicReviewStates.Pending) + .Set(value => value.Stage, 0) + .Set(value => value.DueUtc, due) + .Set(value => value.UpdatedUtc, now); + await _reviews.UpdateOneAsync( + value => value.UserId == claims.UserId && value.ObjectiveId == claims.ObjectiveId, + update, + new UpdateOptions { IsUpsert = true }); + return due; + } + + if (existing is null || existing.Id != claims.ReviewItemId || existing.Stage != claims.ReviewStage) + return null; + var next = DynamicReviewPolicy.AfterReview(existing.Stage, isCorrect, now); + var reviewUpdate = Builders.Update + .Set(value => value.Stage, next.Stage) + .Set(value => value.State, next.Mastered ? DynamicReviewStates.Mastered : DynamicReviewStates.Pending) + .Set(value => value.MisconceptionId, isCorrect ? existing.MisconceptionId : misconceptionId) + .Set(value => value.UpdatedUtc, now) + .Inc(value => value.ReviewCount, 1); + if (next.DueUtc.HasValue) + reviewUpdate = reviewUpdate.Set(value => value.DueUtc, next.DueUtc.Value); + await _reviews.UpdateOneAsync( + value => value.Id == existing.Id && value.Stage == claims.ReviewStage, + reviewUpdate); + return next.DueUtc; + } + private async Task EnsureMasteryAsync(string userId, string objectiveId) { try @@ -194,18 +278,55 @@ public sealed class DynamicQuestionStore public async Task StatusAsync(string userId) { - var objectiveCount = (await PublishedAsync()).Select(value => value.ObjectiveId).Distinct().Count(); + var blueprints = (await PublishedAsync()) + .GroupBy(value => value.ObjectiveId) + .Select(value => value.OrderByDescending(item => item.Version).First()) + .ToList(); + var objectiveCount = blueprints.Count; var mastery = await _mastery.Find(value => value.UserId == userId).ToListAsync(); + var masteryByObjective = mastery.ToDictionary(value => value.ObjectiveId, StringComparer.Ordinal); + var due = await _reviews.CountDocumentsAsync(value => value.UserId == userId + && value.State == DynamicReviewStates.Pending + && value.DueUtc <= DateTime.UtcNow); return new DynamicQuestionStatusResponse { Available = objectiveCount > 0, PublishedObjectives = objectiveCount, AttemptedObjectives = mastery.Count, TotalAttempts = mastery.Sum(value => value.Attempts), - AverageObjectiveElo = mastery.Count == 0 ? 1000 : (int)Math.Round(mastery.Average(value => value.Elo)) + AverageObjectiveElo = mastery.Count == 0 ? 1000 : (int)Math.Round(mastery.Average(value => value.Elo)), + ReviewsDue = checked((int)due), + SystemCoverage = Coverage(blueprints, masteryByObjective, value => value.System), + PhysicianTaskCoverage = Coverage(blueprints, masteryByObjective, value => value.PhysicianTask) }; } + private static List Coverage( + IReadOnlyCollection blueprints, + IReadOnlyDictionary mastery, + Func keySelector) => + blueprints + .GroupBy(value => string.IsNullOrWhiteSpace(keySelector(value)) ? "Unclassified" : keySelector(value)) + .Select(group => + { + var attempts = group + .Where(value => mastery.ContainsKey(value.ObjectiveId)) + .Select(value => mastery[value.ObjectiveId]) + .ToList(); + return new DynamicCoverageSummary + { + Id = group.Key, + PublishedObjectives = group.Count(), + AttemptedObjectives = attempts.Count, + Attempts = attempts.Sum(value => value.Attempts), + CoveragePercent = group.Count() == 0 ? 0 : Math.Round(attempts.Count * 100.0 / group.Count(), 1), + AverageElo = attempts.Count == 0 ? 1000 : (int)Math.Round(attempts.Average(value => value.Elo)) + }; + }) + .OrderBy(value => value.CoveragePercent) + .ThenBy(value => value.Id) + .ToList(); + public async Task<(int Inserted, int Unchanged)> UpsertBlueprintsAsync(IEnumerable blueprints) { var inserted = 0; diff --git a/api/QuestionsApi/Services/DynamicReviewPolicy.cs b/api/QuestionsApi/Services/DynamicReviewPolicy.cs new file mode 100644 index 0000000..e19e282 --- /dev/null +++ b/api/QuestionsApi/Services/DynamicReviewPolicy.cs @@ -0,0 +1,22 @@ +namespace QuestionsApi.Services; + +public static class DynamicReviewPolicy +{ + private static readonly int[] IntervalsDays = [1, 3, 7, 21, 45]; + + public static DateTime FirstDue(DateTime nowUtc) => nowUtc.AddDays(IntervalsDays[0]); + + public static (int Stage, DateTime? DueUtc, bool Mastered) AfterReview( + int currentStage, + bool isCorrect, + DateTime nowUtc) + { + if (!isCorrect) + return (0, FirstDue(nowUtc), false); + + var nextStage = Math.Clamp(currentStage + 1, 1, IntervalsDays.Length); + if (nextStage >= IntervalsDays.Length) + return (nextStage, null, true); + return (nextStage, nowUtc.AddDays(IntervalsDays[nextStage]), false); + } +} diff --git a/api/QuestionsApi/Services/UserProfileStore.cs b/api/QuestionsApi/Services/UserProfileStore.cs index c943c55..2e3c086 100644 --- a/api/QuestionsApi/Services/UserProfileStore.cs +++ b/api/QuestionsApi/Services/UserProfileStore.cs @@ -14,6 +14,7 @@ public sealed class UserProfileStore [ "QuestionAttempts", "UserRatings", "QuestionBookmarks", "UserNotepads", "ReviewQueueItems", "QuestionReviewAttempts", "ReviewQueueBackfillStates", + "DynamicQuestionAttempts", "DynamicObjectiveMastery", "DynamicReviewItems", "CcsAttempts", "CcsUserRatings", "CcsRatedCaseResults", "ExamSimulations", "Step3Outcomes", "QuestionReports", "CcsCaseReports", "AppFeedback", "UserStudyProfiles", "ExternalIdentities", "ExternalAuthTransactions", diff --git a/docs/dynamic-question-quality.md b/docs/dynamic-question-quality.md index 0d7e283..160ccf3 100644 --- a/docs/dynamic-question-quality.md +++ b/docs/dynamic-question-quality.md @@ -11,6 +11,24 @@ Effective ability begins as an equal blend of overall and category question-bank Elo, then shifts toward dynamic objective Elo up to an 80% objective weight after 10 attempts. Aggregate outcomes calibrate each case family's Elo over time. +Quality-standard-4 blueprints add Step 3 coverage and lifecycle metadata. They +must declare the organ system, physician task, encounter frame, site of care, +tested concepts, medical reviewer, review date, and review expiration. Every +distractor must identify the misconception it represents. An expired medical +review prevents publication. + +## Adaptive learning loop + +New questions are chosen using a 50/30/20 policy: + +- 50% scheduled concept reviews, when any are due. +- 30% underexposed blueprint areas, adjusted by `blueprintWeight`. +- 20% interleaved maintenance from the learner's weakest objectives. + +Incorrect answers enter concept review; skips and correct first attempts do not. +Reviews use a freshly generated variant at 1, 3, 7, 21, and 45 days. An +incorrect review resets the interval to one day. + ## Content requirements - At least three curated case families per objective. @@ -20,6 +38,8 @@ Elo, then shifts toward dynamic objective Elo up to an 80% objective weight afte - A complete explanation, key takeaway, USMLE pearl, review follow-up, and wrong-answer remediation. - At least one authoritative reference. +- Quality-standard-4 content includes misconception IDs and current medical + review metadata. Promotion validates 10,000 seeds for every quality-standard-2 blueprint. Existing blueprint versions are immutable; content changes require a version increment. @@ -37,7 +57,7 @@ dotnet QuestionsApi.dll export-dynamic-review \ Keep `answer-key.json` separate from `review-packet.json`. Two clinical reviewers score realism, distractor quality, explanation quality, and Step 3 relevance from -1–5 and predict whether each item is authored or dynamic. +1-5 and predict whether each item is authored or dynamic. The pilot passes when there are no critical medical or ambiguity findings, every dimension averages at least 4/5, and source-identification accuracy is no greater diff --git a/scripts/ui/dashboard_screen.gd b/scripts/ui/dashboard_screen.gd index f3960b5..6051c07 100644 --- a/scripts/ui/dashboard_screen.gd +++ b/scripts/ui/dashboard_screen.gd @@ -244,6 +244,10 @@ func _render_review(data: Dictionary) -> void: missed_action.disabled = missed_count <= 0 or pending_review_launch missed_action.pressed.connect(_start_review_queue) review_list.add_child(missed_action) + var dynamic_due := int(data.get("dynamicReviewsDue", 0)) + review_list.add_child(_review_action( + "%d dynamic concepts due" % dynamic_due if dynamic_due > 0 else "Dynamic concepts up to date", + "activity", Brand.PURPLE if dynamic_due > 0 else Brand.ROYAL_BLUE, "dynamic_questions")) var flashcards_due := int(data.get("flashcardsDue", 0)) review_list.add_child(_review_action( "%d flashcards due" % flashcards_due if flashcards_due > 0 else "Flashcards up to date", diff --git a/scripts/ui/dynamic_questions_screen.gd b/scripts/ui/dynamic_questions_screen.gd index 81e25c8..8f890fe 100644 --- a/scripts/ui/dynamic_questions_screen.gd +++ b/scripts/ui/dynamic_questions_screen.gd @@ -28,7 +28,7 @@ func _ready() -> void: description.horizontal_alignment = HORIZONTAL_ALIGNMENT_CENTER card.add_child(description) status_label = Label.new() - status_label.text = "Checking available objectives…" + status_label.text = "Checking available objectives..." status_label.horizontal_alignment = HORIZONTAL_ALIGNMENT_CENTER card.add_child(status_label) start_button = Button.new() @@ -44,7 +44,7 @@ func _ready() -> void: func on_show(_payload: Variant = null) -> void: start_button.disabled = true - status_label.text = "Checking available objectives…" + status_label.text = "Checking available objectives..." DynamicQuestionService.load_status() @@ -52,8 +52,12 @@ func _on_status(status: Dictionary) -> void: var available := bool(status.get("available", false)) var count := int(status.get("publishedObjectives", 0)) var attempts := int(status.get("totalAttempts", 0)) - status_label.text = "%d objectives • %d attempts • mastery %d" % [ - count, attempts, int(status.get("averageObjectiveElo", 1000))] + var due := int(status.get("reviewsDue", 0)) + var coverage := 0 + if count > 0: + coverage = int(round(float(status.get("attemptedObjectives", 0)) * 100.0 / float(count))) + status_label.text = "%d objectives | %d%% explored | %d reviews due | %d attempts" % [ + count, coverage, due, attempts] start_button.disabled = not available if not available: status_label.text = "Dynamic question content has not been published yet." @@ -61,7 +65,7 @@ func _on_status(status: Dictionary) -> void: func _start() -> void: start_button.disabled = true - start_button.text = "Generating…" + start_button.text = "Generating..." DynamicQuestionService.reset_run() DynamicQuestionService.load_next()