hz/step3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve dynamic question learning loop

Browse Source
This commit is contained in:
hz
2026-07-23 16:50:53 -05:00
parent c91e70deb5
commit 32c455bba0
14 changed files with 422 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/QuestionsApi.Tests/DynamicQuestionTests.cs
+57
View File
@@ -119,6 +119,63 @@ public sealed class DynamicQuestionTests
Assert.Equal(1800, DynamicDifficultyPolicy.Clamp(5000));
}
[Fact]
public void ReviewPolicy_UsesFixedIntervals_AndResetsAfterAMiss()
{
var now = new DateTime(2026, 7, 23, 12, 0, 0, DateTimeKind.Utc);
Assert.Equal(now.AddDays(1), DynamicReviewPolicy.FirstDue(now));
Assert.Equal((1, now.AddDays(3), false), DynamicReviewPolicy.AfterReview(0, true, now));
Assert.Equal((2, now.AddDays(7), false), DynamicReviewPolicy.AfterReview(1, true, now));
Assert.Equal((3, now.AddDays(21), false), DynamicReviewPolicy.AfterReview(2, true, now));
Assert.Equal((4, now.AddDays(45), false), DynamicReviewPolicy.AfterReview(3, true, now));
Assert.Equal((5, null, true), DynamicReviewPolicy.AfterReview(4, true, now));
Assert.Equal((0, now.AddDays(1), false), DynamicReviewPolicy.AfterReview(3, false, now));
}
[Fact]
public void QualityStandardFour_RequiresCoverageReviewAndMisconceptionMetadata()
{
var options = new System.Text.Json.JsonSerializerOptions(System.Text.Json.JsonSerializerDefaults.Web)
{ PropertyNameCaseInsensitive = true };
var root = Path.GetFullPath(Path.Combine(
AppContext.BaseDirectory, "..", "..", "..", "..", "..", "content", "dynamic_questions"));
var source = Directory.EnumerateFiles(root, "*.json")
.SelectMany(file => System.Text.Json.JsonSerializer.Deserialize<List<DynamicQuestionBlueprint>>(
File.ReadAllText(file), options) ?? [])
.ToList();
var blueprint = DynamicBlueprintResolver.ResolveAll(source)
.First(value => value.QualityStandardVersion == 3);
blueprint.QualityStandardVersion = 4;
blueprint.BlueprintWeight = 6;
Assert.Contains("encounterFrame", DynamicQuestionGenerator.Validate(blueprint));
blueprint.System = "Endocrine";
blueprint.PhysicianTask = "Pharmacotherapy";
blueprint.EncounterFrame = "Urgent Intervention";
blueprint.SiteOfCare = "Emergency Department";
blueprint.ConceptIds = ["test.concept"];
blueprint.MedicallyReviewedUtc = DateTime.UtcNow.AddDays(-1);
blueprint.MedicalReviewExpiresUtc = DateTime.UtcNow.AddYears(1);
blueprint.MedicalReviewer = "Medical editor";
foreach (var distractor in blueprint.Distractors)
distractor.MisconceptionId = $"misconception.{distractor.Id}";
Assert.Null(DynamicQuestionGenerator.Validate(blueprint));
}
[Fact]
public void CoveragePolicy_UsesOfficialDefaults_AndAllowsVersionedOverrides()
{
Assert.Equal(12, DynamicBlueprintCoveragePolicy.Weight(
new DynamicQuestionBlueprint { System = "Biostatistics" }));
Assert.Equal(10, DynamicBlueprintCoveragePolicy.Weight(
new DynamicQuestionBlueprint { System = "Cardiovascular" }));
Assert.Equal(2.5, DynamicBlueprintCoveragePolicy.Weight(
new DynamicQuestionBlueprint { System = "Cardiovascular", BlueprintWeight = 2.5 }));
}
[Fact]
public void AdaptiveContent_ResolvesToFourCompleteTiers()
{
api/QuestionsApi/Models/DashboardModels.cs
+1
View File
@@ -14,6 +14,7 @@ public sealed class DashboardResponse
public int StudyTimeSecondsThisWeek { get; set; }
public int CurrentStreak { get; set; }
public int MissedQuestionCount { get; set; }
public int DynamicReviewsDue { get; set; }
public int BookmarkCount { get; set; }
public int NoteCount { get; set; }
public int FlashcardsDue { get; set; }
api/QuestionsApi/Models/DynamicQuestionModels.cs
+55
View File
@@ -16,6 +16,14 @@ public sealed class DynamicQuestionBlueprint
public string Discipline { get; set; } = "";
public string PerformanceCategory { get; set; } = "";
public string PhysicianTask { get; set; } = "";
public string EncounterFrame { get; set; } = "";
public string SiteOfCare { get; set; } = "";
public double BlueprintWeight { get; set; }
public List<string> ConceptIds { get; set; } = [];
public List<string> PrerequisiteConceptIds { get; set; } = [];
public DateTime? MedicallyReviewedUtc { get; set; }
public DateTime? MedicalReviewExpiresUtc { get; set; }
public string MedicalReviewer { get; set; } = "";
public int Difficulty { get; set; } = 3;
public int QualityStandardVersion { get; set; } = 1;
public string CorrectConceptId { get; set; } = "";
@@ -115,6 +123,8 @@ public sealed class DynamicAnswerTemplate
public string Id { get; set; } = "";
public string TextTemplate { get; set; } = "";
public string ExplanationTemplate { get; set; } = "";
public string MisconceptionId { get; set; } = "";
public List<string> RemediationConceptIds { get; set; } = [];
}
public sealed class DynamicGenerationClaims
@@ -130,6 +140,9 @@ public sealed class DynamicGenerationClaims
public string DifficultyTier { get; set; } = "";
public int EffectiveAbilityElo { get; set; }
public int FamilyElo { get; set; }
public bool IsScheduledReview { get; set; }
public string ReviewItemId { get; set; } = "";
public int ReviewStage { get; set; }
public DateTime ExpiresUtc { get; set; }
}
@@ -183,6 +196,9 @@ public sealed class DynamicAttemptResponse
public string ObjectiveId { get; set; } = "";
public int ObjectiveElo { get; set; }
public int RatingChange { get; set; }
public string SelectedMisconceptionId { get; set; } = "";
public bool IsScheduledReview { get; set; }
public DateTime? NextReviewUtc { get; set; }
}
public sealed class DynamicReportRequest
@@ -199,6 +215,19 @@ public sealed class DynamicQuestionStatusResponse
public int AttemptedObjectives { get; set; }
public int TotalAttempts { get; set; }
public int AverageObjectiveElo { get; set; } = 1000;
public int ReviewsDue { get; set; }
public List<DynamicCoverageSummary> SystemCoverage { get; set; } = [];
public List<DynamicCoverageSummary> PhysicianTaskCoverage { get; set; } = [];
}
public sealed class DynamicCoverageSummary
{
public string Id { get; set; } = "";
public int PublishedObjectives { get; set; }
public int AttemptedObjectives { get; set; }
public int Attempts { get; set; }
public double CoveragePercent { get; set; }
public int AverageElo { get; set; } = 1000;
}
public sealed class DynamicQuestionAttempt
@@ -218,10 +247,36 @@ public sealed class DynamicQuestionAttempt
public string ChoiceId { get; set; } = "";
public bool IsCorrect { get; set; }
public bool IsSkip { get; set; }
public string SelectedMisconceptionId { get; set; } = "";
public bool IsScheduledReview { get; set; }
public int DurationSeconds { get; set; }
public DateTime CreatedUtc { get; set; } = DateTime.UtcNow;
}
public static class DynamicReviewStates
{
public const string Pending = "pending";
public const string Mastered = "mastered";
}
public sealed class DynamicReviewItem
{
[BsonId]
[BsonRepresentation(BsonType.ObjectId)]
public string? Id { get; set; }
public string UserId { get; set; } = "";
public string ObjectiveId { get; set; } = "";
public string BlueprintExternalId { get; set; } = "";
public int BlueprintVersion { get; set; }
public string MisconceptionId { get; set; } = "";
public string State { get; set; } = DynamicReviewStates.Pending;
public int Stage { get; set; }
public int ReviewCount { get; set; }
public DateTime DueUtc { get; set; }
public DateTime CreatedUtc { get; set; } = DateTime.UtcNow;
public DateTime UpdatedUtc { get; set; } = DateTime.UtcNow;
}
public sealed class DynamicObjectiveMastery
{
[BsonId]
api/QuestionsApi/Services/DashboardStore.cs
+7 -1
View File
@@ -16,6 +16,7 @@ public sealed class DashboardStore
private readonly IMongoCollection<UserNotepad> _notes;
private readonly QuestionStore _questions;
private readonly ReviewQueueStore _reviewQueue;
private readonly DynamicQuestionStore _dynamicQuestions;
private readonly FlashcardStudyService _flashcards;
private readonly FlashcardScheduleStore _flashcardSchedules;
@@ -23,6 +24,7 @@ public sealed class DashboardStore
MongoDatabaseProvider mongo,
QuestionStore questions,
ReviewQueueStore reviewQueue,
DynamicQuestionStore dynamicQuestions,
FlashcardStudyService flashcards,
FlashcardScheduleStore flashcardSchedules)
{
@@ -36,6 +38,7 @@ public sealed class DashboardStore
_notes = mongo.Database.GetCollection<UserNotepad>("UserNotepads");
_questions = questions;
_reviewQueue = reviewQueue;
_dynamicQuestions = dynamicQuestions;
_flashcards = flashcards;
_flashcardSchedules = flashcardSchedules;
}
@@ -57,8 +60,10 @@ public sealed class DashboardStore
var reviewAttemptsTask = _reviewQueue.GetReviewsSinceAsync(userId, weekStart);
var flashcardCountsTask = _flashcards.GetDashboardCountsAsync(userId, nowUtc, weekStart);
var flashcardReviewsTask = _flashcardSchedules.GetReviewsSinceAsync(userId, weekStart);
var dynamicReviewsTask = _dynamicQuestions.DueReviewsAsync(userId, nowUtc);
await Task.WhenAll(attemptsTask, ratingTask, ccsRatingTask, ccsAttemptsTask, completedCcsCountTask,
simulationsTask, bookmarkCountTask, notepadTask, reviewAttemptsTask, flashcardCountsTask, flashcardReviewsTask);
simulationsTask, bookmarkCountTask, notepadTask, reviewAttemptsTask, flashcardCountsTask,
flashcardReviewsTask, dynamicReviewsTask);
var attempts = attemptsTask.Result;
var answeredAttempts = attempts.Where(value => !value.IsSkip).ToList();
@@ -84,6 +89,7 @@ public sealed class DashboardStore
BookmarkCount = checked((int)bookmarkCountTask.Result),
NoteCount = CountNotes(notepadTask.Result?.Content),
MissedQuestionCount = checked((int)pendingReviewCount),
DynamicReviewsDue = dynamicReviewsTask.Result.Count,
FlashcardsDue = flashcardCountsTask.Result.Due,
FlashcardsNewAvailable = flashcardCountsTask.Result.NewAvailable,
FlashcardsReviewedThisWeek = flashcardCountsTask.Result.ReviewedThisWeek
api/QuestionsApi/Services/DynamicBlueprintCoveragePolicy.cs
+37
View File
@@ -0,0 +1,37 @@
using QuestionsApi.Models;
namespace QuestionsApi.Services;
public static class DynamicBlueprintCoveragePolicy
{
private static readonly Dictionary<string, double> SystemWeights =
new(StringComparer.OrdinalIgnoreCase)
{
["Nutrition"] = 17.5,
["Biostatistics"] = 12,
["Cardiovascular"] = 10,
["Neurologic"] = 9,
["Nervous System"] = 9,
["Pulmonary"] = 9,
["Respiratory"] = 9,
["Pregnancy"] = 8,
["Social Sciences"] = 8,
["Gastrointestinal"] = 7,
["Immune System"] = 7,
["Hematology"] = 7,
["Multisystem"] = 7,
["Infectious Disease"] = 7,
["Endocrine"] = 6,
["Musculoskeletal"] = 6,
["Behavioral Health"] = 5,
["Renal"] = 5,
["Skin"] = 5,
["Human Development"] = 2
};
public static double Weight(DynamicQuestionBlueprint blueprint)
{
if (blueprint.BlueprintWeight > 0) return blueprint.BlueprintWeight;
return SystemWeights.GetValueOrDefault(blueprint.System, 5);
}
}
api/QuestionsApi/Services/DynamicBlueprintResolver.cs
+12 -1
View File
@@ -29,7 +29,18 @@ public static class DynamicBlueprintResolver
materialized.BaseBlueprintVersion = blueprint.BaseBlueprintVersion;
materialized.QualityStandardVersion = blueprint.QualityStandardVersion;
materialized.IsPublished = blueprint.IsPublished;
materialized.CaseFamilies = blueprint.CaseFamilies.Select(overlay =>
materialized.EncounterFrame = blueprint.EncounterFrame;
materialized.SiteOfCare = blueprint.SiteOfCare;
materialized.BlueprintWeight = blueprint.BlueprintWeight;
materialized.ConceptIds = blueprint.ConceptIds;
materialized.PrerequisiteConceptIds = blueprint.PrerequisiteConceptIds;
materialized.MedicallyReviewedUtc = blueprint.MedicallyReviewedUtc;
materialized.MedicalReviewExpiresUtc = blueprint.MedicalReviewExpiresUtc;
materialized.MedicalReviewer = blueprint.MedicalReviewer;
if (blueprint.Distractors.Count > 0)
materialized.Distractors = blueprint.Distractors;
if (blueprint.CaseFamilies.Count > 0)
materialized.CaseFamilies = blueprint.CaseFamilies.Select(overlay =>
{
var baseFamily = baseBlueprint.CaseFamilies.FirstOrDefault(value => value.Id == overlay.BaseFamilyId)
?? throw new InvalidDataException(
api/QuestionsApi/Services/DynamicQuestionGenerator.cs
+24
View File
@@ -317,6 +317,11 @@ public sealed class DynamicQuestionGenerator
if (string.IsNullOrWhiteSpace(blueprint.ExternalId)) return "externalId required";
if (string.IsNullOrWhiteSpace(blueprint.ObjectiveId)) return "objectiveId required";
if (blueprint.Version < 1) return "version must be positive";
if (blueprint.BlueprintWeight < 0) return "blueprintWeight cannot be negative";
if (blueprint.IsPublished
&& blueprint.MedicalReviewExpiresUtc is { } expiresUtc
&& expiresUtc <= DateTime.UtcNow)
return "medical review has expired";
if (blueprint.QualityStandardVersion >= 2)
return ValidateV2(blueprint);
if (string.IsNullOrWhiteSpace(blueprint.StemTemplate) || string.IsNullOrWhiteSpace(blueprint.LeadIn)) return "stemTemplate and leadIn required";
@@ -355,6 +360,22 @@ public sealed class DynamicQuestionGenerator
if (blueprint.CaseFamilies.Any(value => !DynamicDifficultyPolicy.TierElos.ContainsKey(value.DifficultyTier)))
return "difficultyTier must be foundation, standard, advanced, or expert";
}
if (blueprint.QualityStandardVersion >= 4)
{
if (blueprint.BlueprintWeight <= 0) return "quality standard 4 requires blueprintWeight";
if (string.IsNullOrWhiteSpace(blueprint.System)
|| string.IsNullOrWhiteSpace(blueprint.PhysicianTask)
|| string.IsNullOrWhiteSpace(blueprint.EncounterFrame)
|| string.IsNullOrWhiteSpace(blueprint.SiteOfCare))
return "quality standard 4 requires system, physicianTask, encounterFrame, and siteOfCare";
if (blueprint.ConceptIds.Count == 0) return "quality standard 4 requires conceptIds";
if (blueprint.MedicallyReviewedUtc is null
|| blueprint.MedicalReviewExpiresUtc is null
|| string.IsNullOrWhiteSpace(blueprint.MedicalReviewer))
return "quality standard 4 requires medical review metadata";
if (blueprint.Distractors.Any(value => string.IsNullOrWhiteSpace(value.MisconceptionId)))
return "quality standard 4 requires a misconceptionId for every distractor";
}
foreach (var family in blueprint.CaseFamilies)
{
if (string.IsNullOrWhiteSpace(family.Id)) return "case family id required";
@@ -367,6 +388,9 @@ public sealed class DynamicQuestionGenerator
var reviews = family.ReviewFollowups.Count == 0 ? blueprint.ReviewFollowups : family.ReviewFollowups;
var wrong = family.WrongAnswerFollowups.Count == 0 ? blueprint.WrongAnswerFollowups : family.WrongAnswerFollowups;
if (distractors.Count < 6) return $"{family.Id} requires at least six distractors";
if (blueprint.QualityStandardVersion >= 4
&& distractors.Any(value => string.IsNullOrWhiteSpace(value.MisconceptionId)))
return $"{family.Id} every distractor requires a misconceptionId";
var answers = distractors.Append(correctAnswer).ToList();
if (answers.Any(value => string.IsNullOrWhiteSpace(value.Id)
|| string.IsNullOrWhiteSpace(value.TextTemplate)
api/QuestionsApi/Services/DynamicQuestionService.cs
+48 -5
View File
@@ -49,13 +49,52 @@ public sealed class DynamicQuestionService
}
var mastery = await _store.MasteryAsync(userId);
var dueReviews = await _store.DueReviewsAsync(userId, DateTime.UtcNow);
var dueByObjective = dueReviews.ToDictionary(value => value.ObjectiveId, StringComparer.Ordinal);
var candidates = unseen
.Where(group => group.Key != run.PreviousObjectiveId || unseen.Count == 1)
.OrderBy(group => mastery.TryGetValue(group.Key, out var value) ? value.Elo : 1000)
.ThenBy(_ => RandomNumberGenerator.GetInt32(int.MaxValue))
.ToList();
var weakestBand = candidates.Take(Math.Min(4, candidates.Count)).ToList();
var selectedObjective = weakestBand[RandomNumberGenerator.GetInt32(weakestBand.Count)];
var roll = RandomNumberGenerator.GetInt32(100);
IGrouping<string, DynamicQuestionBlueprint> selectedObjective;
var dueCandidates = candidates.Where(value => dueByObjective.ContainsKey(value.Key)).ToList();
if (roll < 50 && dueCandidates.Count == 0 && dueByObjective.Count > 0)
dueCandidates = byObjective
.Where(value => dueByObjective.ContainsKey(value.Key)
&& (value.Key != run.PreviousObjectiveId || dueByObjective.Count == 1))
.ToList();
if (roll < 50 && dueCandidates.Count > 0)
{
selectedObjective = dueCandidates
.OrderBy(value => dueByObjective[value.Key].DueUtc)
.ThenBy(_ => RandomNumberGenerator.GetInt32(int.MaxValue))
.First();
}
else if (roll < 80)
{
var systemAttempts = byObjective
.GroupBy(group => group.OrderByDescending(value => value.Version).First().System)
.ToDictionary(
group => group.Key,
group => group.Sum(objective => mastery.TryGetValue(objective.Key, out var value) ? value.Attempts : 0),
StringComparer.Ordinal);
selectedObjective = candidates
.OrderBy(group =>
{
var latest = group.OrderByDescending(value => value.Version).First();
var weight = DynamicBlueprintCoveragePolicy.Weight(latest);
return (systemAttempts.GetValueOrDefault(latest.System) + 1) / weight;
})
.ThenBy(_ => RandomNumberGenerator.GetInt32(int.MaxValue))
.First();
}
else
{
var weakestBand = candidates
.OrderBy(group => mastery.TryGetValue(group.Key, out var value) ? value.Elo : 1000)
.Take(Math.Min(4, candidates.Count))
.ToList();
selectedObjective = weakestBand[RandomNumberGenerator.GetInt32(weakestBand.Count)];
}
var blueprint = selectedObjective
.OrderByDescending(value => value.Version)
.First();
@@ -97,11 +136,15 @@ public sealed class DynamicQuestionService
DifficultyTier = selectedFamily.DifficultyTier,
EffectiveAbilityElo = effectiveAbility,
FamilyElo = selectedFamilyElo,
IsScheduledReview = dueByObjective.TryGetValue(blueprint.ObjectiveId, out var reviewItem),
ReviewItemId = reviewItem?.Id ?? "",
ReviewStage = reviewItem?.Stage ?? 0,
ExpiresUtc = DateTime.UtcNow.AddDays(7)
};
var generated = _generator.Generate(
blueprint, claims.Seed, claims.Nonce, claims.GeneratorVersion, claims.CaseFamilyId);
run.SeenObjectiveIds.Add(blueprint.ObjectiveId);
if (!run.SeenObjectiveIds.Contains(blueprint.ObjectiveId, StringComparer.Ordinal))
run.SeenObjectiveIds.Add(blueprint.ObjectiveId);
run.PreviousObjectiveId = blueprint.ObjectiveId;
run.RecentCaseFamilyIds.Add(generated.CaseFamilyId);
run.RecentCaseFamilyIds = run.RecentCaseFamilyIds.TakeLast(30).ToList();
api/QuestionsApi/Services/DynamicQuestionStore.cs
+124 -3
View File
@@ -10,6 +10,7 @@ public sealed class DynamicQuestionStore
private readonly IMongoCollection<DynamicQuestionAttempt> _attempts;
private readonly IMongoCollection<DynamicObjectiveMastery> _mastery;
private readonly IMongoCollection<DynamicFamilyRating> _familyRatings;
private readonly IMongoCollection<DynamicReviewItem> _reviews;
private readonly ServiceMetrics? _metrics;
public DynamicQuestionStore(MongoDatabaseProvider mongo, ServiceMetrics? metrics = null)
@@ -19,6 +20,7 @@ public sealed class DynamicQuestionStore
_attempts = mongo.Database.GetCollection<DynamicQuestionAttempt>("DynamicQuestionAttempts");
_mastery = mongo.Database.GetCollection<DynamicObjectiveMastery>("DynamicObjectiveMastery");
_familyRatings = mongo.Database.GetCollection<DynamicFamilyRating>("DynamicFamilyRatings");
_reviews = mongo.Database.GetCollection<DynamicReviewItem>("DynamicReviewItems");
_blueprints.Indexes.CreateOne(new CreateIndexModel<DynamicQuestionBlueprint>(
Builders<DynamicQuestionBlueprint>.IndexKeys.Ascending(value => value.ExternalId).Ascending(value => value.Version),
new CreateIndexOptions { Unique = true }));
@@ -34,6 +36,19 @@ public sealed class DynamicQuestionStore
.Ascending(value => value.BlueprintVersion)
.Ascending(value => value.CaseFamilyId),
new CreateIndexOptions { Unique = true }));
_reviews.Indexes.CreateMany([
new CreateIndexModel<DynamicReviewItem>(
Builders<DynamicReviewItem>.IndexKeys
.Ascending(value => value.UserId)
.Ascending(value => value.ObjectiveId),
new CreateIndexOptions { Unique = true, Name = "dynamic_review_user_objective" }),
new CreateIndexModel<DynamicReviewItem>(
Builders<DynamicReviewItem>.IndexKeys
.Ascending(value => value.UserId)
.Ascending(value => value.State)
.Ascending(value => value.DueUtc),
new CreateIndexOptions { Name = "dynamic_review_user_due" })
]);
}
public async Task<List<DynamicQuestionBlueprint>> PublishedAsync() =>
@@ -46,6 +61,11 @@ public sealed class DynamicQuestionStore
(await _mastery.Find(value => value.UserId == userId).ToListAsync())
.ToDictionary(value => value.ObjectiveId, StringComparer.Ordinal);
public async Task<List<DynamicReviewItem>> DueReviewsAsync(string userId, DateTime nowUtc) =>
await _reviews.Find(value => value.UserId == userId
&& value.State == DynamicReviewStates.Pending
&& value.DueUtc <= nowUtc).ToListAsync();
public async Task<Dictionary<string, DynamicFamilyRating>> FamilyRatingsAsync(DynamicQuestionBlueprint blueprint)
{
var ratings = await _familyRatings.Find(value =>
@@ -60,6 +80,14 @@ public sealed class DynamicQuestionStore
DynamicAttemptRequest request)
{
var isCorrect = !request.IsSkip && request.ChoiceId == generated.Question.CorrectChoiceId;
var selectedAnswer = generated.Blueprint.Distractors
.Concat(generated.Blueprint.CaseFamilies
.Where(value => value.Id == claims.CaseFamilyId)
.SelectMany(value => value.Distractors))
.FirstOrDefault(value => value.Id == request.ChoiceId);
var misconceptionId = isCorrect || request.IsSkip
? ""
: selectedAnswer?.MisconceptionId ?? "";
var attempt = new DynamicQuestionAttempt
{
UserId = claims.UserId,
@@ -74,6 +102,8 @@ public sealed class DynamicQuestionStore
ChoiceId = request.ChoiceId,
IsCorrect = isCorrect,
IsSkip = request.IsSkip,
SelectedMisconceptionId = misconceptionId,
IsScheduledReview = claims.IsScheduledReview,
DurationSeconds = Math.Clamp(request.DurationSeconds, 0, 86_400),
CreatedUtc = DateTime.UtcNow
};
@@ -117,6 +147,9 @@ public sealed class DynamicQuestionStore
.Min(value => value.Elo, DynamicDifficultyPolicy.MaximumElo));
if (!string.IsNullOrWhiteSpace(claims.CaseFamilyId))
await UpdateFamilyRatingAsync(claims, generated, isCorrect);
DateTime? nextReviewUtc = null;
if (!request.IsSkip)
nextReviewUtc = await UpdateReviewAsync(claims, misconceptionId, isCorrect);
return new DynamicAttemptResponse
{
Inserted = true,
@@ -124,10 +157,61 @@ public sealed class DynamicQuestionStore
CorrectChoiceId = generated.Question.CorrectChoiceId,
ObjectiveId = claims.ObjectiveId,
ObjectiveElo = updated.Elo,
RatingChange = delta
RatingChange = delta,
SelectedMisconceptionId = misconceptionId,
IsScheduledReview = claims.IsScheduledReview,
NextReviewUtc = nextReviewUtc
};
}
private async Task<DateTime?> UpdateReviewAsync(
DynamicGenerationClaims claims,
string misconceptionId,
bool isCorrect)
{
var now = DateTime.UtcNow;
var existing = await _reviews.Find(value =>
value.UserId == claims.UserId && value.ObjectiveId == claims.ObjectiveId).FirstOrDefaultAsync();
if (!claims.IsScheduledReview)
{
if (isCorrect) return existing?.DueUtc;
var due = DynamicReviewPolicy.FirstDue(now);
var update = Builders<DynamicReviewItem>.Update
.SetOnInsert(value => value.UserId, claims.UserId)
.SetOnInsert(value => value.ObjectiveId, claims.ObjectiveId)
.SetOnInsert(value => value.CreatedUtc, now)
.Set(value => value.BlueprintExternalId, claims.BlueprintExternalId)
.Set(value => value.BlueprintVersion, claims.BlueprintVersion)
.Set(value => value.MisconceptionId, misconceptionId)
.Set(value => value.State, DynamicReviewStates.Pending)
.Set(value => value.Stage, 0)
.Set(value => value.DueUtc, due)
.Set(value => value.UpdatedUtc, now);
await _reviews.UpdateOneAsync(
value => value.UserId == claims.UserId && value.ObjectiveId == claims.ObjectiveId,
update,
new UpdateOptions { IsUpsert = true });
return due;
}
if (existing is null || existing.Id != claims.ReviewItemId || existing.Stage != claims.ReviewStage)
return null;
var next = DynamicReviewPolicy.AfterReview(existing.Stage, isCorrect, now);
var reviewUpdate = Builders<DynamicReviewItem>.Update
.Set(value => value.Stage, next.Stage)
.Set(value => value.State, next.Mastered ? DynamicReviewStates.Mastered : DynamicReviewStates.Pending)
.Set(value => value.MisconceptionId, isCorrect ? existing.MisconceptionId : misconceptionId)
.Set(value => value.UpdatedUtc, now)
.Inc(value => value.ReviewCount, 1);
if (next.DueUtc.HasValue)
reviewUpdate = reviewUpdate.Set(value => value.DueUtc, next.DueUtc.Value);
await _reviews.UpdateOneAsync(
value => value.Id == existing.Id && value.Stage == claims.ReviewStage,
reviewUpdate);
return next.DueUtc;
}
private async Task EnsureMasteryAsync(string userId, string objectiveId)
{
try
@@ -194,18 +278,55 @@ public sealed class DynamicQuestionStore
public async Task<DynamicQuestionStatusResponse> StatusAsync(string userId)
{
var objectiveCount = (await PublishedAsync()).Select(value => value.ObjectiveId).Distinct().Count();
var blueprints = (await PublishedAsync())
.GroupBy(value => value.ObjectiveId)
.Select(value => value.OrderByDescending(item => item.Version).First())
.ToList();
var objectiveCount = blueprints.Count;
var mastery = await _mastery.Find(value => value.UserId == userId).ToListAsync();
var masteryByObjective = mastery.ToDictionary(value => value.ObjectiveId, StringComparer.Ordinal);
var due = await _reviews.CountDocumentsAsync(value => value.UserId == userId
&& value.State == DynamicReviewStates.Pending
&& value.DueUtc <= DateTime.UtcNow);
return new DynamicQuestionStatusResponse
{
Available = objectiveCount > 0,
PublishedObjectives = objectiveCount,
AttemptedObjectives = mastery.Count,
TotalAttempts = mastery.Sum(value => value.Attempts),
AverageObjectiveElo = mastery.Count == 0 ? 1000 : (int)Math.Round(mastery.Average(value => value.Elo))
AverageObjectiveElo = mastery.Count == 0 ? 1000 : (int)Math.Round(mastery.Average(value => value.Elo)),
ReviewsDue = checked((int)due),
SystemCoverage = Coverage(blueprints, masteryByObjective, value => value.System),
PhysicianTaskCoverage = Coverage(blueprints, masteryByObjective, value => value.PhysicianTask)
};
}
private static List<DynamicCoverageSummary> Coverage(
IReadOnlyCollection<DynamicQuestionBlueprint> blueprints,
IReadOnlyDictionary<string, DynamicObjectiveMastery> mastery,
Func<DynamicQuestionBlueprint, string> keySelector) =>
blueprints
.GroupBy(value => string.IsNullOrWhiteSpace(keySelector(value)) ? "Unclassified" : keySelector(value))
.Select(group =>
{
var attempts = group
.Where(value => mastery.ContainsKey(value.ObjectiveId))
.Select(value => mastery[value.ObjectiveId])
.ToList();
return new DynamicCoverageSummary
{
Id = group.Key,
PublishedObjectives = group.Count(),
AttemptedObjectives = attempts.Count,
Attempts = attempts.Sum(value => value.Attempts),
CoveragePercent = group.Count() == 0 ? 0 : Math.Round(attempts.Count * 100.0 / group.Count(), 1),
AverageElo = attempts.Count == 0 ? 1000 : (int)Math.Round(attempts.Average(value => value.Elo))
};
})
.OrderBy(value => value.CoveragePercent)
.ThenBy(value => value.Id)
.ToList();
public async Task<(int Inserted, int Unchanged)> UpsertBlueprintsAsync(IEnumerable<DynamicQuestionBlueprint> blueprints)
{
var inserted = 0;
api/QuestionsApi/Services/DynamicReviewPolicy.cs
+22
View File
@@ -0,0 +1,22 @@
namespace QuestionsApi.Services;
public static class DynamicReviewPolicy
{
private static readonly int[] IntervalsDays = [1, 3, 7, 21, 45];
public static DateTime FirstDue(DateTime nowUtc) => nowUtc.AddDays(IntervalsDays[0]);
public static (int Stage, DateTime? DueUtc, bool Mastered) AfterReview(
int currentStage,
bool isCorrect,
DateTime nowUtc)
{
if (!isCorrect)
return (0, FirstDue(nowUtc), false);
var nextStage = Math.Clamp(currentStage + 1, 1, IntervalsDays.Length);
if (nextStage >= IntervalsDays.Length)
return (nextStage, null, true);
return (nextStage, nowUtc.AddDays(IntervalsDays[nextStage]), false);
}
}
api/QuestionsApi/Services/UserProfileStore.cs
+1
View File
@@ -14,6 +14,7 @@ public sealed class UserProfileStore
[
"QuestionAttempts", "UserRatings", "QuestionBookmarks", "UserNotepads",
"ReviewQueueItems", "QuestionReviewAttempts", "ReviewQueueBackfillStates",
"DynamicQuestionAttempts", "DynamicObjectiveMastery", "DynamicReviewItems",
"CcsAttempts", "CcsUserRatings", "CcsRatedCaseResults", "ExamSimulations",
"Step3Outcomes", "QuestionReports", "CcsCaseReports", "AppFeedback", "UserStudyProfiles",
"ExternalIdentities", "ExternalAuthTransactions",
docs/dynamic-question-quality.md
+21 -1
View File
@@ -11,6 +11,24 @@ Effective ability begins as an equal blend of overall and category question-bank
Elo, then shifts toward dynamic objective Elo up to an 80% objective weight after
10 attempts. Aggregate outcomes calibrate each case family's Elo over time.
Quality-standard-4 blueprints add Step 3 coverage and lifecycle metadata. They
must declare the organ system, physician task, encounter frame, site of care,
tested concepts, medical reviewer, review date, and review expiration. Every
distractor must identify the misconception it represents. An expired medical
review prevents publication.
## Adaptive learning loop
New questions are chosen using a 50/30/20 policy:
- 50% scheduled concept reviews, when any are due.
- 30% underexposed blueprint areas, adjusted by `blueprintWeight`.
- 20% interleaved maintenance from the learner's weakest objectives.
Incorrect answers enter concept review; skips and correct first attempts do not.
Reviews use a freshly generated variant at 1, 3, 7, 21, and 45 days. An
incorrect review resets the interval to one day.
## Content requirements
- At least three curated case families per objective.
@@ -20,6 +38,8 @@ Elo, then shifts toward dynamic objective Elo up to an 80% objective weight afte
- A complete explanation, key takeaway, USMLE pearl, review follow-up, and
wrong-answer remediation.
- At least one authoritative reference.
- Quality-standard-4 content includes misconception IDs and current medical
review metadata.
Promotion validates 10,000 seeds for every quality-standard-2 blueprint. Existing
blueprint versions are immutable; content changes require a version increment.
@@ -37,7 +57,7 @@ dotnet QuestionsApi.dll export-dynamic-review \
Keep `answer-key.json` separate from `review-packet.json`. Two clinical reviewers
score realism, distractor quality, explanation quality, and Step 3 relevance from
1–5 and predict whether each item is authored or dynamic.
1-5 and predict whether each item is authored or dynamic.
The pilot passes when there are no critical medical or ambiguity findings, every
dimension averages at least 4/5, and source-identification accuracy is no greater
scripts/ui/dashboard_screen.gd
+4
View File
@@ -244,6 +244,10 @@ func _render_review(data: Dictionary) -> void:
missed_action.disabled = missed_count <= 0 or pending_review_launch
missed_action.pressed.connect(_start_review_queue)
review_list.add_child(missed_action)
var dynamic_due := int(data.get("dynamicReviewsDue", 0))
review_list.add_child(_review_action(
"%d dynamic concepts due" % dynamic_due if dynamic_due > 0 else "Dynamic concepts up to date",
"activity", Brand.PURPLE if dynamic_due > 0 else Brand.ROYAL_BLUE, "dynamic_questions"))
var flashcards_due := int(data.get("flashcardsDue", 0))
review_list.add_child(_review_action(
"%d flashcards due" % flashcards_due if flashcards_due > 0 else "Flashcards up to date",
scripts/ui/dynamic_questions_screen.gd
+9 -5
View File
@@ -28,7 +28,7 @@ func _ready() -> void:
description.horizontal_alignment = HORIZONTAL_ALIGNMENT_CENTER
card.add_child(description)
status_label = Label.new()
status_label.text = "Checking available objectives…"
status_label.text = "Checking available objectives..."
status_label.horizontal_alignment = HORIZONTAL_ALIGNMENT_CENTER
card.add_child(status_label)
start_button = Button.new()
@@ -44,7 +44,7 @@ func _ready() -> void:
func on_show(_payload: Variant = null) -> void:
start_button.disabled = true
status_label.text = "Checking available objectives…"
status_label.text = "Checking available objectives..."
DynamicQuestionService.load_status()
@@ -52,8 +52,12 @@ func _on_status(status: Dictionary) -> void:
var available := bool(status.get("available", false))
var count := int(status.get("publishedObjectives", 0))
var attempts := int(status.get("totalAttempts", 0))
status_label.text = "%d objectives • %d attempts • mastery %d" % [
count, attempts, int(status.get("averageObjectiveElo", 1000))]
var due := int(status.get("reviewsDue", 0))
var coverage := 0
if count > 0:
coverage = int(round(float(status.get("attemptedObjectives", 0)) * 100.0 / float(count)))
status_label.text = "%d objectives | %d%% explored | %d reviews due | %d attempts" % [
count, coverage, due, attempts]
start_button.disabled = not available
if not available:
status_label.text = "Dynamic question content has not been published yet."
@@ -61,7 +65,7 @@ func _on_status(status: Dictionary) -> void:
func _start() -> void:
start_button.disabled = true
start_button.text = "Generating…"
start_button.text = "Generating..."
DynamicQuestionService.reset_run()
DynamicQuestionService.load_next()