From 92c5688b556dafcd2fac4e09bac716bc44058d06 Mon Sep 17 00:00:00 2001 From: hz Date: Thu, 23 Jul 2026 14:11:18 -0500 Subject: [PATCH] Upgrade dynamic questions to authored parity --- .../DynamicQuestionTests.cs | 78 ++++++ .../Models/DynamicQuestionModels.cs | 44 ++++ api/QuestionsApi/Program.cs | 6 + .../Services/DynamicBlueprintCommand.cs | 11 +- .../Services/DynamicQuestionGenerator.cs | 238 +++++++++++++++++- .../Services/DynamicQuestionService.cs | 23 +- .../Services/DynamicQuestionStore.cs | 28 ++- .../Services/DynamicReviewExportCommand.cs | 74 ++++++ .../Services/QuestionReportStore.cs | 3 +- content/dynamic_questions/gold-pilot-v2.json | 144 +++++++++++ docs/dynamic-question-quality.md | 36 +++ tests/smoke/uat-smoke.spec.mjs | 23 ++ 12 files changed, 690 insertions(+), 18 deletions(-) create mode 100644 api/QuestionsApi/Services/DynamicReviewExportCommand.cs create mode 100644 content/dynamic_questions/gold-pilot-v2.json create mode 100644 docs/dynamic-question-quality.md diff --git a/api/QuestionsApi.Tests/DynamicQuestionTests.cs b/api/QuestionsApi.Tests/DynamicQuestionTests.cs index 0565aa9..738d48d 100644 --- a/api/QuestionsApi.Tests/DynamicQuestionTests.cs +++ b/api/QuestionsApi.Tests/DynamicQuestionTests.cs @@ -48,6 +48,38 @@ public sealed class DynamicQuestionTests Assert.ThrowsAny(() => service.ReadGeneration(token + "x", "user-one")); } + [Fact] + public void GoldGenerator_UsesCaseFamilies_AndProducesAuthoredTeachingShape() + { + var blueprint = GoldBlueprint(); + var generator = new DynamicQuestionGenerator(); + var generated = Enumerable.Range(0, 100) + .Select(seed => generator.Generate(blueprint, seed.ToString(), seed.ToString())) + .ToList(); + + Assert.Equal(3, generated.Select(value => value.CaseFamilyId).Distinct().Count()); + Assert.All(generated, value => + { + Assert.True(value.Question.Stem.Length >= 220); + Assert.Equal(5, value.Question.Choices.Count); + Assert.Equal(5, value.Question.ChoiceExplanations.Count); + Assert.Single(value.Question.ReviewFollowups); + Assert.Equal(4, value.Question.WrongAnswerFollowups.Count); + Assert.Contains("quality-standard:2", value.Question.Tags); + }); + Assert.Null(DynamicQuestionGenerator.Validate(blueprint)); + } + + [Fact] + public void LegacyGeneratorVersion_RemainsDeterministic() + { + var generator = new DynamicQuestionGenerator(); + var first = generator.Generate(Blueprint(), "legacy-seed", "one", 1).Question; + var regenerated = generator.Generate(Blueprint(), "legacy-seed", "two", 1).Question; + Assert.Equal(first.Stem, regenerated.Stem); + Assert.Equal(first.Choices.Select(value => value.Id), regenerated.Choices.Select(value => value.Id)); + } + private static DynamicQuestionBlueprint Blueprint() => new() { ExternalId = "test", @@ -68,4 +100,50 @@ public sealed class DynamicQuestionTests ], References = [new QuestionReference { Title = "Reference", Url = "https://example.test" }] }; + + private static DynamicQuestionBlueprint GoldBlueprint() + { + var stem = string.Join(" ", Enumerable.Repeat( + "A patient presents with a carefully correlated history, examination, vital signs, and diagnostic findings that establish the clinical context without naming the diagnosis.", 3)); + return new DynamicQuestionBlueprint + { + ExternalId = "gold-test", + Version = 2, + ObjectiveId = "gold.objective", + QualityStandardVersion = 2, + CorrectConceptId = "gold.correct", + CorrectAnswer = new DynamicAnswerTemplate + { Id = "correct", TextTemplate = "Correct action", ExplanationTemplate = "This addresses the decisive finding." }, + Distractors = Enumerable.Range(1, 6).Select(index => new DynamicAnswerTemplate + { + Id = $"d{index}", + TextTemplate = $"Distractor {index}", + ExplanationTemplate = $"Distractor {index} does not address the decisive finding." + }).ToList(), + ExplanationTemplate = "A complete explanation connects the presentation to the action.", + KeyTakeaway = "Use the decisive finding.", + UsmlePearl = "Sequence urgent care correctly.", + ReviewFollowups = + [ + new DynamicReviewFollowupTemplate + { + PromptTemplate = "What is decisive?", + Choices = + [ + new() { Id = "yes", TextTemplate = "The unstable finding" }, + new() { Id = "no", TextTemplate = "An incidental finding" } + ], + CorrectChoiceId = "yes", + TeachingPointTemplate = "The unstable finding determines management." + } + ], + CaseFamilies = + [ + new() { Id = "family-a", StemTemplate = stem, LeadIn = "What is next?" }, + new() { Id = "family-b", StemTemplate = stem + " Family B.", LeadIn = "What is next?" }, + new() { Id = "family-c", StemTemplate = stem + " Family C.", LeadIn = "What is next?" } + ], + References = [new QuestionReference { Title = "Reference", Url = "https://example.test" }] + }; + } } diff --git a/api/QuestionsApi/Models/DynamicQuestionModels.cs b/api/QuestionsApi/Models/DynamicQuestionModels.cs index c018a20..62fdb93 100644 --- a/api/QuestionsApi/Models/DynamicQuestionModels.cs +++ b/api/QuestionsApi/Models/DynamicQuestionModels.cs @@ -17,6 +17,8 @@ public sealed class DynamicQuestionBlueprint public string PerformanceCategory { get; set; } = ""; public string PhysicianTask { get; set; } = ""; public int Difficulty { get; set; } = 3; + public int QualityStandardVersion { get; set; } = 1; + public string CorrectConceptId { get; set; } = ""; public bool IsPublished { get; set; } = true; public string StemTemplate { get; set; } = ""; public string LeadIn { get; set; } = ""; @@ -31,6 +33,44 @@ public sealed class DynamicQuestionBlueprint public string UsmlePearl { get; set; } = ""; public List References { get; set; } = []; public List Tags { get; set; } = []; + public List ReviewFollowups { get; set; } = []; + public Dictionary WrongAnswerFollowups { get; set; } = []; + public List CaseFamilies { get; set; } = []; +} + +public sealed class DynamicCaseFamily +{ + public string Id { get; set; } = ""; + public string StemTemplate { get; set; } = ""; + public string LeadIn { get; set; } = ""; + public Dictionary Variables { get; set; } = []; + public List> ValueBundles { get; set; } = []; + public Dictionary Derived { get; set; } = []; + public List Constraints { get; set; } = []; + public List Labs { get; set; } = []; + public DynamicAnswerTemplate CorrectAnswer { get; set; } = new(); + public List Distractors { get; set; } = []; + public string ExplanationTemplate { get; set; } = ""; + public string KeyTakeaway { get; set; } = ""; + public string UsmlePearl { get; set; } = ""; + public List ReviewFollowups { get; set; } = []; + public Dictionary WrongAnswerFollowups { get; set; } = []; +} + +public sealed class DynamicReviewFollowupTemplate +{ + public string PromptTemplate { get; set; } = ""; + public List Choices { get; set; } = []; + public string CorrectChoiceId { get; set; } = ""; + public string TeachingPointTemplate { get; set; } = ""; +} + +public sealed class DynamicWrongAnswerFollowupTemplate +{ + public string PromptTemplate { get; set; } = ""; + public List Choices { get; set; } = []; + public string CorrectChoiceId { get; set; } = ""; + public string TeachingPointTemplate { get; set; } = ""; } public sealed class DynamicVariableDefinition @@ -82,6 +122,7 @@ public sealed class DynamicGenerationClaims public string Seed { get; set; } = ""; public string Nonce { get; set; } = ""; public int GeneratorVersion { get; set; } = 1; + public string CaseFamilyId { get; set; } = ""; public DateTime ExpiresUtc { get; set; } } @@ -91,6 +132,7 @@ public sealed class DynamicRunClaims public List SeenObjectiveIds { get; set; } = []; public string PreviousObjectiveId { get; set; } = ""; public int Cycle { get; set; } + public List RecentCaseFamilyIds { get; set; } = []; public DateTime ExpiresUtc { get; set; } } @@ -98,6 +140,7 @@ public sealed class DynamicGeneratedQuestion { public Question Question { get; set; } = new(); public DynamicQuestionBlueprint Blueprint { get; set; } = new(); + public string CaseFamilyId { get; set; } = ""; } public sealed class DynamicQuestionEnvelope @@ -187,4 +230,5 @@ public sealed class DynamicGenerationDescriptor public int BlueprintVersion { get; set; } public int GeneratorVersion { get; set; } public string Seed { get; set; } = ""; + public string CaseFamilyId { get; set; } = ""; } diff --git a/api/QuestionsApi/Program.cs b/api/QuestionsApi/Program.cs index 754fdc7..9d7c1c5 100644 --- a/api/QuestionsApi/Program.cs +++ b/api/QuestionsApi/Program.cs @@ -23,6 +23,12 @@ if (args.FirstOrDefault() == "promote-dynamic-questions") return; } +if (args.FirstOrDefault() == "export-dynamic-review") +{ + Environment.ExitCode = await DynamicReviewExportCommand.RunAsync(args.Skip(1).ToArray(), Console.Out); + return; +} + if (args.FirstOrDefault() == "seed-flashcards") { var seedConfiguration = new ConfigurationBuilder() diff --git a/api/QuestionsApi/Services/DynamicBlueprintCommand.cs b/api/QuestionsApi/Services/DynamicBlueprintCommand.cs index e405967..1729ec1 100644 --- a/api/QuestionsApi/Services/DynamicBlueprintCommand.cs +++ b/api/QuestionsApi/Services/DynamicBlueprintCommand.cs @@ -24,15 +24,18 @@ public static class DynamicBlueprintCommand blueprints.AddRange(JsonSerializer.Deserialize>(await File.ReadAllTextAsync(file), json) ?? []); var generator = new DynamicQuestionGenerator(); + var generatedVariants = 0; foreach (var blueprint in blueprints) { var error = DynamicQuestionGenerator.Validate(blueprint); if (error is not null) throw new InvalidDataException($"{blueprint.ExternalId} v{blueprint.Version}: {error}"); - for (var seed = 0; seed < 1000; seed++) + var validationSeeds = blueprint.QualityStandardVersion >= 2 ? 10_000 : 1_000; + for (var seed = 0; seed < validationSeeds; seed++) generator.Generate(blueprint, seed.ToString(), seed.ToString()); + generatedVariants += validationSeeds; } - await output.WriteLineAsync($"Validated {blueprints.Count} blueprints across 1,000 seeds each."); + await output.WriteLineAsync($"Validated {blueprints.Count} blueprints across {generatedVariants:N0} generated variants."); if (!apply) { await output.WriteLineAsync("Dry run only. Pass --apply to publish."); @@ -43,8 +46,8 @@ public static class DynamicBlueprintCommand .AddEnvironmentVariables() .Build(); var store = new DynamicQuestionStore(new MongoDatabaseProvider(configuration)); - await store.UpsertBlueprintsAsync(blueprints); - await output.WriteLineAsync($"Published {blueprints.Count} immutable blueprint versions."); + var result = await store.UpsertBlueprintsAsync(blueprints); + await output.WriteLineAsync($"Promotion complete: {result.Inserted} inserted, {result.Unchanged} unchanged."); return 0; } } diff --git a/api/QuestionsApi/Services/DynamicQuestionGenerator.cs b/api/QuestionsApi/Services/DynamicQuestionGenerator.cs index ed3b6aa..f90636a 100644 --- a/api/QuestionsApi/Services/DynamicQuestionGenerator.cs +++ b/api/QuestionsApi/Services/DynamicQuestionGenerator.cs @@ -8,12 +8,17 @@ namespace QuestionsApi.Services; public sealed class DynamicQuestionGenerator { - public const int CurrentVersion = 1; + public const int CurrentVersion = 2; private static readonly Regex Placeholder = new(@"\{\{([a-zA-Z0-9_]+)\}\}", RegexOptions.Compiled); - public DynamicGeneratedQuestion Generate(DynamicQuestionBlueprint blueprint, string seed, string nonce) + public DynamicGeneratedQuestion Generate(DynamicQuestionBlueprint blueprint, string seed, string nonce, int generatorVersion = CurrentVersion) + => blueprint.QualityStandardVersion >= 2 + ? GenerateV2(blueprint, seed, nonce) + : GenerateLegacy(blueprint, seed, nonce, generatorVersion); + + private DynamicGeneratedQuestion GenerateLegacy(DynamicQuestionBlueprint blueprint, string seed, string nonce, int generatorVersion) { - var draw = new DeterministicDraw($"{blueprint.ExternalId}|{blueprint.Version}|{CurrentVersion}|{seed}"); + var draw = new DeterministicDraw($"{blueprint.ExternalId}|{blueprint.Version}|{generatorVersion}|{seed}"); var values = new Dictionary(StringComparer.Ordinal); var numbers = new Dictionary(StringComparer.Ordinal); foreach (var item in blueprint.Variables.OrderBy(value => value.Key, StringComparer.Ordinal)) @@ -112,11 +117,193 @@ public sealed class DynamicQuestionGenerator return new DynamicGeneratedQuestion { Question = question, Blueprint = blueprint }; } + private DynamicGeneratedQuestion GenerateV2(DynamicQuestionBlueprint blueprint, string seed, string nonce) + { + if (blueprint.CaseFamilies.Count == 0) + throw new InvalidOperationException("quality-standard-2 blueprints require case families"); + var draw = new DeterministicDraw($"{blueprint.ExternalId}|{blueprint.Version}|{CurrentVersion}|{seed}"); + var family = blueprint.CaseFamilies.OrderBy(value => value.Id, StringComparer.Ordinal) + .ElementAt(draw.Next(blueprint.CaseFamilies.Count)); + var values = new Dictionary(StringComparer.Ordinal); + var numbers = new Dictionary(StringComparer.Ordinal); + if (family.ValueBundles.Count > 0) + foreach (var item in family.ValueBundles[draw.Next(family.ValueBundles.Count)]) + values[item.Key] = item.Value; + PopulateVariables(family.Variables, values, numbers, draw); + PopulateDerived(family.Derived, values, numbers); + ValidateConstraints(family.Constraints, numbers); + + var correctAnswer = string.IsNullOrWhiteSpace(family.CorrectAnswer.Id) ? blueprint.CorrectAnswer : family.CorrectAnswer; + var distractorPool = family.Distractors.Count == 0 ? blueprint.Distractors : family.Distractors; + var reviewTemplates = family.ReviewFollowups.Count == 0 ? blueprint.ReviewFollowups : family.ReviewFollowups; + var wrongTemplates = family.WrongAnswerFollowups.Count == 0 ? blueprint.WrongAnswerFollowups : family.WrongAnswerFollowups; + var explanationTemplate = string.IsNullOrWhiteSpace(family.ExplanationTemplate) ? blueprint.ExplanationTemplate : family.ExplanationTemplate; + var keyTakeaway = string.IsNullOrWhiteSpace(family.KeyTakeaway) ? blueprint.KeyTakeaway : family.KeyTakeaway; + var usmlePearl = string.IsNullOrWhiteSpace(family.UsmlePearl) ? blueprint.UsmlePearl : family.UsmlePearl; + var distractors = distractorPool.OrderBy(_ => draw.Next(int.MaxValue)).Take(4).ToList(); + if (distractors.Count != 4) throw new InvalidOperationException($"{family.Id} requires at least four distractors"); + var answerTemplates = distractors.Append(correctAnswer).OrderBy(_ => draw.Next(int.MaxValue)).ToList(); + var choices = answerTemplates.Select(answer => new QuestionChoice + { + Id = answer.Id, + Text = Render(answer.TextTemplate, values) + }).ToList(); + if (choices.Select(value => value.Id).Distinct(StringComparer.Ordinal).Count() != 5 + || choices.Select(value => value.Text).Distinct(StringComparer.OrdinalIgnoreCase).Count() != 5) + throw new InvalidOperationException($"{family.Id} generated choices must be unique"); + + var question = new Question + { + Id = $"dynamic-{nonce}", + ExternalId = $"dynamic-{nonce}", + Stem = Render(family.StemTemplate, values), + LeadIn = Render(family.LeadIn, values), + Choices = choices, + CorrectChoiceId = correctAnswer.Id, + Explanation = Render(explanationTemplate, values), + KeyTakeaway = Render(keyTakeaway, values), + UsmlePearl = Render(usmlePearl, values), + ChoiceExplanations = answerTemplates.ToDictionary( + value => value.Id, value => Render(value.ExplanationTemplate, values)), + ReviewFollowups = reviewTemplates.Select(followup => new QuestionReviewFollowup + { + Prompt = Render(followup.PromptTemplate, values), + Choices = followup.Choices.Select(choice => new QuestionChoice + { Id = choice.Id, Text = Render(choice.TextTemplate, values) }).ToList(), + CorrectChoiceId = followup.CorrectChoiceId, + TeachingPoint = Render(followup.TeachingPointTemplate, values) + }).ToList(), + WrongAnswerFollowups = BuildWrongAnswerFollowups( + distractors, correctAnswer, wrongTemplates, values), + Discipline = blueprint.Discipline, + System = blueprint.System, + PerformanceCategory = blueprint.PerformanceCategory, + Difficulty = blueprint.Difficulty, + Labs = family.Labs.Select(lab => new QuestionLab + { + Name = lab.Name, + Value = Render(lab.ValueTemplate, values), + Unit = lab.Unit, + Reference = lab.Reference + }).ToList(), + Tags = [.. blueprint.Tags, "dynamic", "quality-standard:2", $"objective:{blueprint.ObjectiveId}", $"family:{family.Id}"], + References = blueprint.References, + IsPublished = true + }; + return new DynamicGeneratedQuestion { Question = question, Blueprint = blueprint, CaseFamilyId = family.Id }; + } + + private static void PopulateVariables( + IReadOnlyDictionary definitions, + IDictionary values, + IDictionary numbers, + DeterministicDraw draw) + { + foreach (var item in definitions.OrderBy(value => value.Key, StringComparer.Ordinal)) + { + var definition = item.Value; + if (definition.Type == "enum") + { + if (definition.Values.Count == 0) throw new InvalidOperationException($"{item.Key} requires values"); + values[item.Key] = definition.Values[draw.Next(definition.Values.Count)]; + continue; + } + var step = definition.Step <= 0 ? 1 : definition.Step; + var count = (int)Math.Floor((definition.Maximum - definition.Minimum) / step) + 1; + if (count < 1) throw new InvalidOperationException($"{item.Key} range is invalid"); + var number = Math.Round(definition.Minimum + draw.Next(count) * step, Math.Clamp(definition.Precision, 0, 8)); + numbers[item.Key] = number; + values[item.Key] = Format(number, definition.Precision); + } + } + + private static void PopulateDerived( + IReadOnlyDictionary definitions, + IDictionary values, + Dictionary numbers) + { + foreach (var item in definitions) + { + var definition = item.Value; + var left = ResolveNumber(definition.Left, numbers); + var right = ResolveNumber(definition.Right, numbers); + var number = definition.Operation switch + { + "add" => left + right, + "subtract" => left - right, + "multiply" => left * right, + "divide" when Math.Abs(right) > double.Epsilon => left / right, + _ => throw new InvalidOperationException($"{item.Key} operation is invalid") + }; + number = Math.Round(number, Math.Clamp(definition.Precision, 0, 8)); + numbers[item.Key] = number; + values[item.Key] = Format(number, definition.Precision); + } + } + + private static void ValidateConstraints(IEnumerable constraints, IReadOnlyDictionary numbers) + { + foreach (var constraint in constraints) + { + if (!numbers.TryGetValue(constraint.Field, out var actual)) + throw new InvalidOperationException($"constraint field {constraint.Field} is not numeric"); + var valid = constraint.Operator switch + { + "lt" => actual < constraint.Value, + "lte" => actual <= constraint.Value, + "gt" => actual > constraint.Value, + "gte" => actual >= constraint.Value, + "eq" => Math.Abs(actual - constraint.Value) < 0.0000001, + "neq" => Math.Abs(actual - constraint.Value) >= 0.0000001, + _ => false + }; + if (!valid) throw new InvalidOperationException($"generated value violates {constraint.Field} {constraint.Operator}"); + } + } + + private static Dictionary BuildWrongAnswerFollowups( + IEnumerable distractors, + DynamicAnswerTemplate correct, + IReadOnlyDictionary templates, + IReadOnlyDictionary values) + { + var result = new Dictionary(); + foreach (var distractor in distractors) + { + if (templates.TryGetValue(distractor.Id, out var template)) + { + result[distractor.Id] = new WrongAnswerFollowup + { + Prompt = Render(template.PromptTemplate, values), + Choices = template.Choices.Select(choice => new QuestionChoice + { Id = choice.Id, Text = Render(choice.TextTemplate, values) }).ToList(), + CorrectChoiceId = template.CorrectChoiceId, + TeachingPoint = Render(template.TeachingPointTemplate, values) + }; + continue; + } + result[distractor.Id] = new WrongAnswerFollowup + { + Prompt = "Which option best addresses the decisive finding in this case?", + Choices = + [ + new QuestionChoice { Id = correct.Id, Text = Render(correct.TextTemplate, values) }, + new QuestionChoice { Id = distractor.Id, Text = Render(distractor.TextTemplate, values) } + ], + CorrectChoiceId = correct.Id, + TeachingPoint = $"{Render(distractor.ExplanationTemplate, values)} {Render(correct.ExplanationTemplate, values)}" + }; + } + return result; + } + public static string? Validate(DynamicQuestionBlueprint blueprint) { if (string.IsNullOrWhiteSpace(blueprint.ExternalId)) return "externalId required"; if (string.IsNullOrWhiteSpace(blueprint.ObjectiveId)) return "objectiveId required"; if (blueprint.Version < 1) return "version must be positive"; + if (blueprint.QualityStandardVersion >= 2) + return ValidateV2(blueprint); if (string.IsNullOrWhiteSpace(blueprint.StemTemplate) || string.IsNullOrWhiteSpace(blueprint.LeadIn)) return "stemTemplate and leadIn required"; if (string.IsNullOrWhiteSpace(blueprint.CorrectAnswer.Id) || string.IsNullOrWhiteSpace(blueprint.CorrectAnswer.TextTemplate)) return "correctAnswer required"; if (blueprint.Distractors.Count < 4) return "at least four distractors required"; @@ -133,6 +320,51 @@ public sealed class DynamicQuestionGenerator return null; } + private static string? ValidateV2(DynamicQuestionBlueprint blueprint) + { + if (string.IsNullOrWhiteSpace(blueprint.CorrectConceptId)) return "correctConceptId required"; + if (blueprint.CaseFamilies.Count < 3) return "at least three case families required"; + if (blueprint.References.Count == 0) return "at least one reference required"; + if (blueprint.CaseFamilies.Select(value => value.Id).Distinct(StringComparer.Ordinal).Count() != blueprint.CaseFamilies.Count) + return "case family ids must be unique"; + foreach (var family in blueprint.CaseFamilies) + { + if (string.IsNullOrWhiteSpace(family.Id)) return "case family id required"; + if (family.StemTemplate.Length < 220) return $"{family.Id} stem is too short for quality standard 2"; + if (string.IsNullOrWhiteSpace(family.LeadIn)) return $"{family.Id} leadIn required"; + if (!string.IsNullOrWhiteSpace(family.CorrectAnswer.Id)) + return $"{family.Id} must use the blueprint-level invariant correct answer"; + var correctAnswer = string.IsNullOrWhiteSpace(family.CorrectAnswer.Id) ? blueprint.CorrectAnswer : family.CorrectAnswer; + var distractors = family.Distractors.Count == 0 ? blueprint.Distractors : family.Distractors; + var reviews = family.ReviewFollowups.Count == 0 ? blueprint.ReviewFollowups : family.ReviewFollowups; + var wrong = family.WrongAnswerFollowups.Count == 0 ? blueprint.WrongAnswerFollowups : family.WrongAnswerFollowups; + if (distractors.Count < 6) return $"{family.Id} requires at least six distractors"; + var answers = distractors.Append(correctAnswer).ToList(); + if (answers.Any(value => string.IsNullOrWhiteSpace(value.Id) + || string.IsNullOrWhiteSpace(value.TextTemplate) + || string.IsNullOrWhiteSpace(value.ExplanationTemplate))) + return $"{family.Id} every answer requires id, text, and explanation"; + if (answers.Select(value => value.Id).Distinct(StringComparer.Ordinal).Count() != answers.Count) + return $"{family.Id} answer ids must be unique"; + if (reviews.Count == 0) return $"{family.Id} review followup required"; + if (string.IsNullOrWhiteSpace(family.ExplanationTemplate) && string.IsNullOrWhiteSpace(blueprint.ExplanationTemplate) + || string.IsNullOrWhiteSpace(family.KeyTakeaway) && string.IsNullOrWhiteSpace(blueprint.KeyTakeaway) + || string.IsNullOrWhiteSpace(family.UsmlePearl) && string.IsNullOrWhiteSpace(blueprint.UsmlePearl)) + return $"{family.Id} complete teaching content required"; + } + try + { + var generator = new DynamicQuestionGenerator(); + for (var index = 0; index < 300; index++) + { + var generated = generator.Generate(blueprint, $"validation-{index}", $"validation-{index}"); + if (generated.Question.ChoiceExplanations.Count != 5) return "every displayed choice requires an explanation"; + } + } + catch (Exception exception) { return exception.Message; } + return null; + } + private static string Render(string template, IReadOnlyDictionary values) { var rendered = Placeholder.Replace(template ?? "", match => diff --git a/api/QuestionsApi/Services/DynamicQuestionService.cs b/api/QuestionsApi/Services/DynamicQuestionService.cs index ce6b39c..782186d 100644 --- a/api/QuestionsApi/Services/DynamicQuestionService.cs +++ b/api/QuestionsApi/Services/DynamicQuestionService.cs @@ -65,9 +65,22 @@ public sealed class DynamicQuestionService GeneratorVersion = DynamicQuestionGenerator.CurrentVersion, ExpiresUtc = DateTime.UtcNow.AddDays(7) }; - var generated = _generator.Generate(blueprint, claims.Seed, claims.Nonce); + var generated = _generator.Generate(blueprint, claims.Seed, claims.Nonce, claims.GeneratorVersion); + var familyIds = blueprint.CaseFamilies.Select(value => value.Id).ToHashSet(StringComparer.Ordinal); + var recentForObjective = run.RecentCaseFamilyIds + .Where(familyIds.Contains) + .TakeLast(Math.Max(1, blueprint.CaseFamilies.Count - 1)) + .ToHashSet(StringComparer.Ordinal); + for (var retry = 0; retry < 20 && recentForObjective.Contains(generated.CaseFamilyId); retry++) + { + claims.Seed = Convert.ToHexString(RandomNumberGenerator.GetBytes(16)).ToLowerInvariant(); + generated = _generator.Generate(blueprint, claims.Seed, claims.Nonce); + } + claims.CaseFamilyId = generated.CaseFamilyId; run.SeenObjectiveIds.Add(blueprint.ObjectiveId); run.PreviousObjectiveId = blueprint.ObjectiveId; + run.RecentCaseFamilyIds.Add(generated.CaseFamilyId); + run.RecentCaseFamilyIds = run.RecentCaseFamilyIds.TakeLast(30).ToList(); run.ExpiresUtc = DateTime.UtcNow.AddDays(7); return new DynamicQuestionEnvelope { @@ -92,11 +105,15 @@ public sealed class DynamicQuestionService string generationToken) { var claims = _tokens.ReadGeneration(generationToken, userId); - if (claims.GeneratorVersion != DynamicQuestionGenerator.CurrentVersion) + if (claims.GeneratorVersion is < 1 or > DynamicQuestionGenerator.CurrentVersion) throw new InvalidOperationException("This generated question version is no longer supported."); var blueprint = await _store.GetAsync(claims.BlueprintExternalId, claims.BlueprintVersion) ?? throw new KeyNotFoundException("The question blueprint is no longer available."); - return (claims, _generator.Generate(blueprint, claims.Seed, claims.Nonce)); + var generated = _generator.Generate(blueprint, claims.Seed, claims.Nonce, claims.GeneratorVersion); + if (!string.IsNullOrWhiteSpace(claims.CaseFamilyId) + && claims.CaseFamilyId != generated.CaseFamilyId) + throw new InvalidOperationException("Generated case family does not match its signed descriptor."); + return (claims, generated); } private static DynamicRunClaims NewRun(string userId) => new() diff --git a/api/QuestionsApi/Services/DynamicQuestionStore.cs b/api/QuestionsApi/Services/DynamicQuestionStore.cs index ff5ecec..cc4dd3b 100644 --- a/api/QuestionsApi/Services/DynamicQuestionStore.cs +++ b/api/QuestionsApi/Services/DynamicQuestionStore.cs @@ -27,7 +27,7 @@ public sealed class DynamicQuestionStore } public async Task> PublishedAsync() => - await _blueprints.Find(value => value.IsPublished).ToListAsync(); + await _blueprints.Find(value => value.IsPublished && value.QualityStandardVersion >= 2).ToListAsync(); public async Task GetAsync(string externalId, int version) => await _blueprints.Find(value => value.ExternalId == externalId && value.Version == version).FirstOrDefaultAsync(); @@ -114,18 +114,32 @@ public sealed class DynamicQuestionStore }; } - public async Task UpsertBlueprintsAsync(IEnumerable blueprints) + public async Task<(int Inserted, int Unchanged)> UpsertBlueprintsAsync(IEnumerable blueprints) { + var inserted = 0; + var unchanged = 0; foreach (var blueprint in blueprints) { var existing = await _blueprints.Find( value => value.ExternalId == blueprint.ExternalId && value.Version == blueprint.Version) .FirstOrDefaultAsync(); - blueprint.Id = existing is null ? ObjectId.GenerateNewId().ToString() : existing.Id; - await _blueprints.ReplaceOneAsync( - value => value.ExternalId == blueprint.ExternalId && value.Version == blueprint.Version, - blueprint, - new ReplaceOptions { IsUpsert = true }); + if (existing is not null) + { + blueprint.Id = existing.Id; + var incomingDocument = blueprint.ToBsonDocument(); + var existingDocument = existing.ToBsonDocument(); + incomingDocument.Remove("_id"); + existingDocument.Remove("_id"); + if (incomingDocument != existingDocument) + throw new InvalidDataException( + $"{blueprint.ExternalId} v{blueprint.Version} is immutable; publish changes with a new version"); + unchanged++; + continue; + } + blueprint.Id = ObjectId.GenerateNewId().ToString(); + await _blueprints.InsertOneAsync(blueprint); + inserted++; } + return (inserted, unchanged); } } diff --git a/api/QuestionsApi/Services/DynamicReviewExportCommand.cs b/api/QuestionsApi/Services/DynamicReviewExportCommand.cs new file mode 100644 index 0000000..48ebe60 --- /dev/null +++ b/api/QuestionsApi/Services/DynamicReviewExportCommand.cs @@ -0,0 +1,74 @@ +using Microsoft.Extensions.Configuration; +using QuestionsApi.Models; +using System.Text.Json; + +namespace QuestionsApi.Services; + +public static class DynamicReviewExportCommand +{ + public static async Task RunAsync(string[] args, TextWriter output) + { + static string Argument(string[] values, string name, string fallback = "") + { + var index = Array.FindIndex(values, value => value.Equals(name, StringComparison.OrdinalIgnoreCase)); + return index >= 0 && index + 1 < values.Length ? values[index + 1] : fallback; + } + var source = Argument(args, "--source", Path.Combine(AppContext.BaseDirectory, "content", "dynamic_questions")); + var destination = Argument(args, "--output"); + if (!Directory.Exists(source) || string.IsNullOrWhiteSpace(destination)) + { + await output.WriteLineAsync("Usage: export-dynamic-review --source --output "); + return 2; + } + var jsonOptions = new JsonSerializerOptions(JsonSerializerDefaults.Web) + { PropertyNameCaseInsensitive = true, WriteIndented = true }; + var blueprints = new List(); + foreach (var file in Directory.EnumerateFiles(source, "*.json", SearchOption.AllDirectories)) + blueprints.AddRange(JsonSerializer.Deserialize>( + await File.ReadAllTextAsync(file), jsonOptions) ?? []); + blueprints = blueprints.Where(value => value.IsPublished && value.QualityStandardVersion >= 2).ToList(); + + var configuration = new ConfigurationBuilder() + .AddJsonFile("appsettings.json", optional: true) + .AddEnvironmentVariables() + .Build(); + var authored = (await new QuestionStore(new MongoDatabaseProvider(configuration)).GetPublishedAsync()) + .OrderBy(value => value.ExternalId, StringComparer.Ordinal) + .Take(blueprints.Count * 3) + .ToList(); + if (authored.Count < blueprints.Count * 3) + throw new InvalidOperationException("Not enough published authored questions for a balanced review packet."); + + var generator = new DynamicQuestionGenerator(); + var entries = new List<(Question Question, string Source, string Detail)>(); + foreach (var blueprint in blueprints) + for (var seed = 0; seed < 3; seed++) + { + var generated = generator.Generate(blueprint, $"review-{seed}", $"review-{blueprint.ExternalId}-{seed}"); + entries.Add((generated.Question, "dynamic", $"{blueprint.ExternalId}:{generated.CaseFamilyId}")); + } + entries.AddRange(authored.Select(value => (value, "authored", value.ExternalId))); + var ordered = entries.OrderBy(value => Convert.ToHexString( + System.Security.Cryptography.SHA256.HashData(System.Text.Encoding.UTF8.GetBytes(value.Detail)))).ToList(); + + Directory.CreateDirectory(destination); + var packet = ordered.Select((entry, index) => new + { + reviewId = $"Q{index + 1:D2}", + entry.Question.Stem, + entry.Question.LeadIn, + entry.Question.Choices, + entry.Question.Labs, + entry.Question.Explanation, + entry.Question.ChoiceExplanations, + entry.Question.ReviewFollowups, + entry.Question.WrongAnswerFollowups, + scoring = new { realism = 0, distractors = 0, explanation = 0, step3Relevance = 0, criticalIssue = "", predictedSource = "" } + }); + var key = ordered.Select((entry, index) => new { reviewId = $"Q{index + 1:D2}", entry.Source, entry.Detail }); + await File.WriteAllTextAsync(Path.Combine(destination, "review-packet.json"), JsonSerializer.Serialize(packet, jsonOptions)); + await File.WriteAllTextAsync(Path.Combine(destination, "answer-key.json"), JsonSerializer.Serialize(key, jsonOptions)); + await output.WriteLineAsync($"Exported {ordered.Count} blinded questions to {destination}."); + return 0; + } +} diff --git a/api/QuestionsApi/Services/QuestionReportStore.cs b/api/QuestionsApi/Services/QuestionReportStore.cs index 5027cd5..94e4742 100644 --- a/api/QuestionsApi/Services/QuestionReportStore.cs +++ b/api/QuestionsApi/Services/QuestionReportStore.cs @@ -99,7 +99,8 @@ public class QuestionReportStore BlueprintExternalId = claims.BlueprintExternalId, BlueprintVersion = claims.BlueprintVersion, GeneratorVersion = claims.GeneratorVersion, - Seed = claims.Seed + Seed = claims.Seed, + CaseFamilyId = claims.CaseFamilyId }, CreatedUtc = now, UpdatedUtc = now diff --git a/content/dynamic_questions/gold-pilot-v2.json b/content/dynamic_questions/gold-pilot-v2.json new file mode 100644 index 0000000..be85011 --- /dev/null +++ b/content/dynamic_questions/gold-pilot-v2.json @@ -0,0 +1,144 @@ +[ + { + "externalId": "dyn-gold-dka", "version": 2, "objectiveId": "endocrine.dka-initial-management", + "title": "Initial management of diabetic ketoacidosis", "system": "Endocrine", "discipline": "Medicine", + "performanceCategory": "Patient Care: Management", "physicianTask": "Emergency Management", "difficulty": 3, + "qualityStandardVersion": 2, "correctConceptId": "dka.restore-intravascular-volume", "isPublished": true, + "correctAnswer": {"id":"correct","textTemplate":"Begin intravenous 0.9% saline","explanationTemplate":"DKA causes major osmotic diuresis and extracellular volume depletion; isotonic crystalloid is the immediate first intervention while potassium and insulin plans are prepared."}, + "distractors": [ + {"id":"insulin","textTemplate":"Begin an intravenous insulin infusion before giving fluid","explanationTemplate":"Insulin is essential, but initial volume restoration comes first and serum potassium must be known because insulin can precipitate dangerous hypokalemia."}, + {"id":"bicarbonate","textTemplate":"Give intravenous sodium bicarbonate immediately","explanationTemplate":"Bicarbonate is not routine DKA therapy and is generally reserved for extreme acidemia, typically pH below 6.9."}, + {"id":"potassium","textTemplate":"Give intravenous potassium without further assessment","explanationTemplate":"Total-body potassium is depleted, but replacement depends on the measured potassium and urine output; empiric immediate potassium can be hazardous."}, + {"id":"dextrose","textTemplate":"Begin intravenous dextrose-containing fluid","explanationTemplate":"Dextrose is added later as glucose approaches the treatment threshold so insulin can continue clearing ketones."}, + {"id":"dialysis","textTemplate":"Arrange immediate hemodialysis","explanationTemplate":"Routine DKA responds to fluids, potassium-guided insulin, and treatment of the trigger; dialysis is not the initial intervention."}, + {"id":"subcutaneous","textTemplate":"Give only a dose of subcutaneous long-acting insulin","explanationTemplate":"A critically ill patient with DKA requires controlled acute resuscitation, not long-acting insulin as sole therapy."} + ], + "explanationTemplate":"This patient has DKA with severe extracellular volume depletion from osmotic diuresis. The first action is rapid isotonic crystalloid resuscitation. Measure potassium promptly, begin intravenous insulin when potassium is safe, add dextrose as glucose falls, and treat the precipitating illness.", + "keyTakeaway":"In DKA, restore intravascular volume first; potassium status determines when insulin can safely begin.", + "usmlePearl":"A normal or elevated initial serum potassium does not mean potassium stores are adequate; total-body potassium is depleted.", + "reviewFollowups":[{"promptTemplate":"After initial fluid resuscitation, the serum potassium is 2.9 mEq/L. What is the next step?","choices":[{"id":"k","textTemplate":"Replace potassium before insulin"},{"id":"i","textTemplate":"Start insulin immediately"},{"id":"b","textTemplate":"Give bicarbonate"},{"id":"d","textTemplate":"Start dextrose"}],"correctChoiceId":"k","teachingPointTemplate":"When potassium is below 3.3 mEq/L, replace potassium and hold insulin until potassium rises because insulin shifts potassium intracellularly."}], + "caseFamilies": [ + { + "id":"dka-infection","stemTemplate":"A {{age}}-year-old woman with type 1 diabetes mellitus is brought to the emergency department for 2 days of dysuria, fever, progressive nausea, and diffuse abdominal pain. She has missed several insulin doses because she was not eating. Temperature is {{temp}}°C, blood pressure is {{sbp}}/{{dbp}} mm Hg, pulse is {{pulse}}/min, and respirations are deep at {{resp}}/min. She is drowsy, has dry mucous membranes, and has suprapubic tenderness without guarding. Urinalysis shows pyuria and positive nitrites. Additional laboratory results are shown. Which of the following is the most appropriate immediate next step?","leadIn":"Which of the following is the most appropriate immediate next step?", + "variables":{"age":{"type":"integer","minimum":19,"maximum":42},"temp":{"type":"decimal","minimum":38.1,"maximum":39.2,"step":0.1,"precision":1},"sbp":{"type":"integer","minimum":84,"maximum":96},"dbp":{"type":"integer","minimum":48,"maximum":60},"pulse":{"type":"integer","minimum":112,"maximum":132},"resp":{"type":"integer","minimum":26,"maximum":34},"glucose":{"type":"integer","minimum":480,"maximum":680,"step":10},"sodium":{"type":"integer","minimum":128,"maximum":134},"potassium":{"type":"decimal","minimum":4.6,"maximum":5.6,"step":0.1,"precision":1},"bicarb":{"type":"integer","minimum":8,"maximum":14},"ph":{"type":"decimal","minimum":7.08,"maximum":7.22,"step":0.01,"precision":2}}, + "labs":[{"name":"Glucose","valueTemplate":"{{glucose}}","unit":"mg/dL","reference":"70–100"},{"name":"Sodium","valueTemplate":"{{sodium}}","unit":"mEq/L","reference":"136–145"},{"name":"Potassium","valueTemplate":"{{potassium}}","unit":"mEq/L","reference":"3.5–5.0"},{"name":"Bicarbonate","valueTemplate":"{{bicarb}}","unit":"mEq/L","reference":"22–28"},{"name":"Arterial pH","valueTemplate":"{{ph}}","unit":"","reference":"7.35–7.45"}] + }, + { + "id":"dka-new-onset","stemTemplate":"A {{age}}-year-old man with no known medical conditions presents with 3 weeks of polyuria, polydipsia, and weight loss followed by repeated vomiting and increasing confusion. Blood pressure is {{sbp}}/{{dbp}} mm Hg, pulse is {{pulse}}/min, and respirations are rapid and deep. Examination shows sunken eyes, markedly dry oral mucosa, delayed capillary refill, and a fruity odor to the breath. The abdomen is mildly tender without rebound. Serum beta-hydroxybutyrate is elevated, and laboratory results are shown. Which intervention should be performed first?","leadIn":"Which intervention should be performed first?", + "variables":{"age":{"type":"integer","minimum":18,"maximum":34},"sbp":{"type":"integer","minimum":86,"maximum":100},"dbp":{"type":"integer","minimum":50,"maximum":64},"pulse":{"type":"integer","minimum":108,"maximum":128},"glucose":{"type":"integer","minimum":410,"maximum":620,"step":10},"potassium":{"type":"decimal","minimum":4.1,"maximum":5.3,"step":0.1,"precision":1},"bicarb":{"type":"integer","minimum":7,"maximum":13},"gap":{"type":"integer","minimum":24,"maximum":34}}, + "labs":[{"name":"Glucose","valueTemplate":"{{glucose}}","unit":"mg/dL","reference":"70–100"},{"name":"Potassium","valueTemplate":"{{potassium}}","unit":"mEq/L","reference":"3.5–5.0"},{"name":"Bicarbonate","valueTemplate":"{{bicarb}}","unit":"mEq/L","reference":"22–28"},{"name":"Anion gap","valueTemplate":"{{gap}}","unit":"mEq/L","reference":"8–12"}] + }, + { + "id":"dka-pump-failure","stemTemplate":"A {{age}}-year-old woman with type 1 diabetes mellitus managed with an insulin pump presents after awakening with intense thirst, vomiting, and shortness of breath. The pump tubing is found to be kinked. She is lethargic but follows commands. Blood pressure is {{sbp}}/{{dbp}} mm Hg, pulse is {{pulse}}/min, and respirations are {{resp}}/min with a deep pattern. Her extremities are cool, mucous membranes are dry, and jugular venous pressure is low. Bedside glucose reads HIGH; confirmatory laboratory results are shown. What is the best initial treatment?","leadIn":"What is the best initial treatment?", + "variables":{"age":{"type":"integer","minimum":20,"maximum":48},"sbp":{"type":"integer","minimum":88,"maximum":102},"dbp":{"type":"integer","minimum":52,"maximum":66},"pulse":{"type":"integer","minimum":110,"maximum":130},"resp":{"type":"integer","minimum":25,"maximum":33},"glucose":{"type":"integer","minimum":520,"maximum":720,"step":10},"potassium":{"type":"decimal","minimum":4.4,"maximum":5.7,"step":0.1,"precision":1},"bicarb":{"type":"integer","minimum":9,"maximum":15},"ph":{"type":"decimal","minimum":7.10,"maximum":7.24,"step":0.01,"precision":2}}, + "labs":[{"name":"Glucose","valueTemplate":"{{glucose}}","unit":"mg/dL","reference":"70–100"},{"name":"Potassium","valueTemplate":"{{potassium}}","unit":"mEq/L","reference":"3.5–5.0"},{"name":"Bicarbonate","valueTemplate":"{{bicarb}}","unit":"mEq/L","reference":"22–28"},{"name":"Venous pH","valueTemplate":"{{ph}}","unit":"","reference":"7.32–7.42"}] + } + ], + "references":[{"title":"Hyperglycemic Crises in Adults With Diabetes","publisher":"American Diabetes Association","url":"https://diabetesjournals.org/care/article/47/8/1257/156806/Hyperglycemic-Crises-in-Adults-With-Diabetes-A","supports":"Fluid, potassium, and insulin sequence"}], + "tags":["emergency","diabetes","gold"] + }, + { + "externalId":"dyn-gold-af","version":2,"objectiveId":"cardio.unstable-atrial-fibrillation", + "title":"Unstable atrial fibrillation", "system":"Cardiovascular", "discipline":"Medicine", + "performanceCategory":"Patient Care: Management", "physicianTask":"Emergency Management", "difficulty": 3, + "qualityStandardVersion":2,"correctConceptId":"tachyarrhythmia.synchronized-cardioversion","isPublished":true, + "correctAnswer":{"id":"correct","textTemplate":"Perform immediate synchronized cardioversion","explanationTemplate":"A tachyarrhythmia causing hypotension, ischemia, shock, pulmonary edema, or altered mental status requires synchronized cardioversion."}, + "distractors":[ + {"id":"metoprolol","textTemplate":"Give intravenous metoprolol","explanationTemplate":"Rate-control medication can worsen hypotension and delays definitive therapy in an unstable patient."}, + {"id":"diltiazem","textTemplate":"Begin an intravenous diltiazem infusion","explanationTemplate":"Diltiazem is appropriate for selected stable patients but can worsen shock and is not first-line when instability is attributable to the rhythm."}, + {"id":"adenosine","textTemplate":"Give a rapid intravenous bolus of adenosine","explanationTemplate":"Adenosine may clarify or terminate regular narrow-complex tachycardia; it does not treat irregular atrial fibrillation."}, + {"id":"amiodarone","textTemplate":"Begin an intravenous amiodarone infusion and observe","explanationTemplate":"Medication should not delay synchronized cardioversion when atrial fibrillation is producing hemodynamic instability."}, + {"id":"anticoagulation","textTemplate":"Start anticoagulation and arrange cardioversion in 3 weeks","explanationTemplate":"The usual elective anticoagulation strategy does not apply when immediate cardioversion is needed to stabilize the patient."}, + {"id":"defibrillation","textTemplate":"Perform unsynchronized defibrillation","explanationTemplate":"Unsynchronized shock is used for pulseless rhythms or polymorphic ventricular tachycardia; a patient with unstable atrial fibrillation and a pulse needs synchronization."} + ], + "explanationTemplate":"The irregular narrow-complex tachycardia is causing end-organ hypoperfusion. Immediate synchronized cardioversion is indicated. Sedate if feasible without delaying shock, then address precipitants and thromboembolic risk after stabilization.", + "keyTakeaway":"Cardiovert an unstable tachyarrhythmia immediately when the rhythm is causing hypotension or other end-organ compromise.", + "usmlePearl":"The duration of atrial fibrillation changes anticoagulation planning, but it does not justify delaying cardioversion in an unstable patient.", + "reviewFollowups":[{"promptTemplate":"Which feature most clearly makes this an unstable tachyarrhythmia?","choices":[{"id":"h","textTemplate":"Hypotension with altered mental status"},{"id":"r","textTemplate":"An irregular rhythm alone"},{"id":"a","textTemplate":"Age older than 65 years"},{"id":"d","textTemplate":"A duration longer than 30 minutes"}],"correctChoiceId":"h","teachingPointTemplate":"Instability is defined by consequences such as hypotension, shock, ischemic chest discomfort, acute heart failure, or altered mental status attributable to the rhythm."}], + "caseFamilies":[ + {"id":"af-sepsis","stemTemplate":"A {{age}}-year-old woman hospitalized for pneumonia suddenly becomes confused and diaphoretic. Blood pressure falls to {{sbp}}/{{dbp}} mm Hg, pulse is irregular at {{rate}}/min, respirations are 28/min, and oxygen saturation is 93% with supplemental oxygen. Cardiac examination shows an irregularly irregular rhythm without a murmur. The lungs have right basilar crackles. An ECG shows absent discrete P waves and irregularly spaced narrow QRS complexes. A fluid bolus has not improved the blood pressure. Which intervention is most appropriate now?","leadIn":"Which intervention is most appropriate now?","variables":{"age":{"type":"integer","minimum":54,"maximum":79},"sbp":{"type":"integer","minimum":72,"maximum":84},"dbp":{"type":"integer","minimum":42,"maximum":54},"rate":{"type":"integer","minimum":158,"maximum":188}}}, + {"id":"af-pulmonary-edema","stemTemplate":"A {{age}}-year-old man with hypertension presents with abrupt palpitations and severe dyspnea. He is anxious and can speak only in short phrases. Blood pressure is {{sbp}}/{{dbp}} mm Hg, pulse is irregular at {{rate}}/min, and oxygen saturation is {{sat}}% on room air. Jugular venous pressure is elevated, diffuse inspiratory crackles are present, and the extremities are cool. ECG shows atrial fibrillation with rapid ventricular response and no ST-segment elevation. Noninvasive oxygen support is initiated. What is the next step in management?","leadIn":"What is the next step in management?","variables":{"age":{"type":"integer","minimum":58,"maximum":84},"sbp":{"type":"integer","minimum":76,"maximum":88},"dbp":{"type":"integer","minimum":44,"maximum":58},"rate":{"type":"integer","minimum":150,"maximum":182},"sat":{"type":"integer","minimum":78,"maximum":87}}}, + {"id":"af-ischemia","stemTemplate":"A {{age}}-year-old woman develops sudden palpitations, substernal pressure, and lightheadedness while recovering after abdominal surgery. Blood pressure is {{sbp}}/{{dbp}} mm Hg and pulse is irregular at {{rate}}/min. She is pale, intermittently confused, and has weak peripheral pulses. ECG demonstrates an irregular narrow-complex tachycardia without visible P waves and diffuse rate-related ST depressions. Electrolyte samples have been drawn, and intravenous access is established. Which of the following is the best immediate management?","leadIn":"Which of the following is the best immediate management?","variables":{"age":{"type":"integer","minimum":47,"maximum":73},"sbp":{"type":"integer","minimum":70,"maximum":86},"dbp":{"type":"integer","minimum":40,"maximum":54},"rate":{"type":"integer","minimum":162,"maximum":194}}} + ], + "references":[{"title":"Adult Tachycardia With a Pulse Algorithm","publisher":"American Heart Association","url":"https://cpr.heart.org/en/resuscitation-science/cpr-and-ecc-guidelines/algorithms","supports":"Immediate synchronized cardioversion for unstable tachyarrhythmia"}], + "tags":["cardiology","emergency","gold"] + }, + { + "externalId":"dyn-gold-pyelo-pregnancy","version":2,"objectiveId":"ob.pregnancy-pyelonephritis", + "title":"Pyelonephritis in pregnancy", "system":"Pregnancy", "discipline":"Obstetrics", + "performanceCategory":"Patient Care: Management", "physicianTask":"Pharmacotherapy", "difficulty":3, + "qualityStandardVersion":2,"correctConceptId":"pregnancy.pyelonephritis.inpatient-parenteral-antibiotics","isPublished":true, + "correctAnswer":{"id":"correct","textTemplate":"Admit for intravenous ceftriaxone","explanationTemplate":"Acute pyelonephritis in pregnancy warrants initial inpatient management with a parenteral antibiotic that penetrates renal tissue, followed by culture-directed therapy."}, + "distractors":[ + {"id":"nitrofurantoin","textTemplate":"Discharge with oral nitrofurantoin","explanationTemplate":"Nitrofurantoin treats lower urinary infection but does not reach adequate renal parenchymal concentrations for pyelonephritis."}, + {"id":"fosfomycin","textTemplate":"Give one oral dose of fosfomycin","explanationTemplate":"Fosfomycin is a lower-tract option and is inadequate for a febrile upper urinary tract infection."}, + {"id":"await","textTemplate":"Withhold antibiotics until urine culture results return","explanationTemplate":"Delaying therapy risks maternal sepsis, respiratory complications, and preterm labor; obtain cultures and treat empirically."}, + {"id":"ciprofloxacin","textTemplate":"Discharge with oral ciprofloxacin","explanationTemplate":"Pregnancy and systemic upper-tract illness make outpatient fluoroquinolone treatment inappropriate."}, + {"id":"delivery","textTemplate":"Proceed with immediate delivery","explanationTemplate":"Pyelonephritis alone is not an indication for delivery; stabilize and treat the infection while monitoring mother and fetus."}, + {"id":"phenazopyridine","textTemplate":"Treat with phenazopyridine alone","explanationTemplate":"Urinary analgesia does not eradicate bacterial infection or prevent maternal and fetal complications."} + ], + "explanationTemplate":"Fever, flank pain, costovertebral-angle tenderness, and pyuria indicate acute pyelonephritis. Pregnancy increases the risk of sepsis, ARDS, and preterm labor, so begin inpatient parenteral therapy such as ceftriaxone after cultures and tailor treatment to susceptibility results.", + "keyTakeaway":"Manage pyelonephritis in pregnancy initially in the hospital with renal-penetrating parenteral antibiotics.", + "usmlePearl":"Nitrofurantoin and fosfomycin may treat cystitis but should not be used for pyelonephritis because renal tissue levels are inadequate.", + "reviewFollowups":[{"promptTemplate":"Why is nitrofurantoin not appropriate in this patient?","choices":[{"id":"t","textTemplate":"It does not achieve therapeutic renal tissue levels"},{"id":"f","textTemplate":"It always causes fetal malformations"},{"id":"r","textTemplate":"All urinary pathogens are resistant"},{"id":"p","textTemplate":"It induces premature labor"}],"correctChoiceId":"t","teachingPointTemplate":"The key distinction is infection site: agents concentrated in urine can treat cystitis yet fail to treat renal parenchymal infection."}], + "caseFamilies":[ + {"id":"pyelo-classic","stemTemplate":"A {{age}}-year-old woman at {{weeks}} weeks' gestation presents with 24 hours of fever, chills, nausea, and right flank pain. Temperature is {{temp}}°C, blood pressure is {{sbp}}/{{dbp}} mm Hg, and pulse is {{pulse}}/min. The uterus is nontender, fetal heart rate is reassuring, and there is marked right costovertebral-angle tenderness. Urinalysis shows positive leukocyte esterase, nitrites, numerous leukocytes, and bacteriuria. Blood and urine cultures are obtained. Which management is most appropriate?","leadIn":"Which management is most appropriate?","variables":{"age":{"type":"integer","minimum":20,"maximum":36},"weeks":{"type":"integer","minimum":16,"maximum":30},"temp":{"type":"decimal","minimum":38.2,"maximum":39.4,"step":0.1,"precision":1},"sbp":{"type":"integer","minimum":96,"maximum":112},"dbp":{"type":"integer","minimum":56,"maximum":70},"pulse":{"type":"integer","minimum":104,"maximum":124}}}, + {"id":"pyelo-recurrent","stemTemplate":"A {{age}}-year-old pregnant woman at {{weeks}} weeks' gestation has fever, vomiting, dysuria, and left-sided back pain. She was treated for asymptomatic bacteriuria earlier in pregnancy. Temperature is {{temp}}°C, blood pressure is {{sbp}}/{{dbp}} mm Hg, and pulse is {{pulse}}/min. Examination shows dry mucous membranes and left costovertebral-angle tenderness; there are no contractions and the cervix is closed. Urinalysis reveals pyuria and white blood cell casts. What is the best next step?","leadIn":"What is the best next step?","variables":{"age":{"type":"integer","minimum":19,"maximum":38},"weeks":{"type":"integer","minimum":18,"maximum":33},"temp":{"type":"decimal","minimum":38.0,"maximum":39.1,"step":0.1,"precision":1},"sbp":{"type":"integer","minimum":92,"maximum":108},"dbp":{"type":"integer","minimum":54,"maximum":68},"pulse":{"type":"integer","minimum":102,"maximum":122}}}, + {"id":"pyelo-systemic","stemTemplate":"A {{age}}-year-old woman at {{weeks}} weeks' gestation presents with fever, rigors, urinary urgency, and severe flank pain. Temperature is {{temp}}°C, blood pressure is {{sbp}}/{{dbp}} mm Hg, pulse is {{pulse}}/min, and respirations are 24/min. She appears ill and has suprapubic and costovertebral-angle tenderness. Fetal monitoring is reactive. Urinalysis shows bacteriuria, pyuria, and positive nitrites; serum lactate is mildly elevated. After obtaining cultures, which treatment is most appropriate?","leadIn":"After obtaining cultures, which treatment is most appropriate?","variables":{"age":{"type":"integer","minimum":22,"maximum":39},"weeks":{"type":"integer","minimum":14,"maximum":28},"temp":{"type":"decimal","minimum":38.4,"maximum":39.6,"step":0.1,"precision":1},"sbp":{"type":"integer","minimum":88,"maximum":104},"dbp":{"type":"integer","minimum":50,"maximum":64},"pulse":{"type":"integer","minimum":112,"maximum":132}}} + ], + "references":[{"title":"Urinary Tract Infections in Pregnant Individuals","publisher":"ACOG","url":"https://www.acog.org/clinical/clinical-guidance/clinical-consensus/articles/2023/08/urinary-tract-infections-in-pregnant-individuals","supports":"Inpatient parenteral treatment of pyelonephritis"}], + "tags":["obstetrics","infectious-disease","gold"] + }, + { + "externalId":"dyn-gold-sah","version":2,"objectiveId":"neuro.subarachnoid-hemorrhage-initial-test", + "title":"Initial testing for subarachnoid hemorrhage", "system":"Neurologic", "discipline":"Medicine", + "performanceCategory":"Patient Care: Diagnosis", "physicianTask":"Diagnostic Studies", "difficulty":3, + "qualityStandardVersion":2,"correctConceptId":"sah.initial-noncontrast-head-ct","isPublished":true, + "correctAnswer":{"id":"correct","textTemplate":"Obtain an emergent noncontrast CT scan of the head","explanationTemplate":"Noncontrast head CT is the rapid initial study for suspected subarachnoid hemorrhage and can identify acute blood and complications."}, + "distractors":[ + {"id":"mri","textTemplate":"Obtain a routine brain MRI with contrast","explanationTemplate":"MRI is slower and is not the initial emergency study for a classic thunderclap headache suspicious for subarachnoid hemorrhage."}, + {"id":"lpfirst","textTemplate":"Perform lumbar puncture before neuroimaging","explanationTemplate":"Noncontrast CT is obtained first; further testing after a negative CT depends on timing, image quality, and residual suspicion."}, + {"id":"ctaonly","textTemplate":"Obtain CT angiography without a noncontrast study","explanationTemplate":"Angiography helps define an aneurysm, but the initial diagnostic study for acute subarachnoid blood is noncontrast CT."}, + {"id":"eeg","textTemplate":"Order electroencephalography","explanationTemplate":"EEG evaluates seizure activity and does not diagnose subarachnoid hemorrhage."}, + {"id":"migraine","textTemplate":"Treat as migraine and discharge without imaging","explanationTemplate":"A first sudden maximal-at-onset headache with meningismus or neurologic symptoms requires emergency evaluation for hemorrhage."}, + {"id":"carotid","textTemplate":"Order carotid duplex ultrasonography","explanationTemplate":"Carotid ultrasound does not evaluate the subarachnoid space or intracranial aneurysmal bleeding."} + ], + "explanationTemplate":"A headache that reaches maximal intensity immediately, especially with vomiting, meningismus, syncope, or a focal deficit, is subarachnoid hemorrhage until proven otherwise. Obtain emergent noncontrast head CT; if it is nondiagnostic and suspicion remains, proceed according to timing and local diagnostic pathways.", + "keyTakeaway":"The initial test for suspected acute subarachnoid hemorrhage is emergent noncontrast head CT.", + "usmlePearl":"Do not label a first thunderclap headache as migraine simply because the neurologic examination improves.", + "reviewFollowups":[{"promptTemplate":"If the initial CT is nondiagnostic but clinical suspicion remains high, what principle should guide the next step?","choices":[{"id":"f","textTemplate":"Pursue additional evaluation for subarachnoid blood or aneurysm"},{"id":"d","textTemplate":"Discharge because CT excludes all cases"},{"id":"e","textTemplate":"Order EEG only"},{"id":"s","textTemplate":"Schedule routine follow-up in 6 months"}],"correctChoiceId":"f","teachingPointTemplate":"CT sensitivity depends on factors including time from onset; persistent high suspicion requires additional evaluation rather than reassurance alone."}], + "caseFamilies":[ + {"id":"sah-exertion","stemTemplate":"A {{age}}-year-old man develops the worst headache of his life while lifting a heavy object. The pain reaches maximal intensity within seconds and is followed by vomiting and a brief loss of consciousness. On arrival, blood pressure is {{sbp}}/{{dbp}} mm Hg and pulse is 88/min. He is photophobic and has neck stiffness but no focal weakness. He has no prior history of similar headaches and takes no anticoagulants. Which diagnostic study should be obtained first?","leadIn":"Which diagnostic study should be obtained first?","variables":{"age":{"type":"integer","minimum":41,"maximum":68},"sbp":{"type":"integer","minimum":154,"maximum":194},"dbp":{"type":"integer","minimum":88,"maximum":112}}}, + {"id":"sah-sentinel","stemTemplate":"A {{age}}-year-old woman presents 90 minutes after sudden severe occipital pain that began during sexual activity. She says the headache was instantly maximal and unlike prior mild tension headaches. She vomited twice and now has painful neck flexion. Blood pressure is {{sbp}}/{{dbp}} mm Hg. She is alert, and cranial nerves, strength, and sensation are normal. Her mother died after a ruptured cerebral aneurysm. What is the most appropriate initial diagnostic test?","leadIn":"What is the most appropriate initial diagnostic test?","variables":{"age":{"type":"integer","minimum":36,"maximum":61},"sbp":{"type":"integer","minimum":146,"maximum":186},"dbp":{"type":"integer","minimum":84,"maximum":108}}}, + {"id":"sah-focal","stemTemplate":"A {{age}}-year-old man abruptly develops a severe diffuse headache while at work, falls to the floor, and is confused for several minutes. In the emergency department, blood pressure is {{sbp}}/{{dbp}} mm Hg. He has meningismus and mild left eyelid ptosis but otherwise normal strength and speech. Finger-stick glucose is normal, and there is no evidence of trauma. His headache remains intense despite an analgesic. Which study is the best first step in establishing the diagnosis?","leadIn":"Which study is the best first step in establishing the diagnosis?","variables":{"age":{"type":"integer","minimum":44,"maximum":72},"sbp":{"type":"integer","minimum":158,"maximum":198},"dbp":{"type":"integer","minimum":90,"maximum":116}}} + ], + "references":[{"title":"Guideline for the Management of Aneurysmal Subarachnoid Hemorrhage","publisher":"American Heart Association","url":"https://www.ahajournals.org/doi/10.1161/STR.0000000000000436","supports":"Emergency diagnostic evaluation"}], + "tags":["neurology","emergency","gold"] + }, + { + "externalId":"dyn-gold-arr","version":2,"objectiveId":"biostats.absolute-risk-reduction", + "title":"Absolute risk reduction", "system":"Biostatistics", "discipline":"Biostatistics", + "performanceCategory":"Applying Foundational Science Concepts", "physicianTask":"Interpretation of Medical Literature", "difficulty":3, + "qualityStandardVersion":2,"correctConceptId":"biostats.control-event-rate-minus-treatment-event-rate","isPublished":true, + "correctAnswer":{"id":"correct","textTemplate":"{{arr}} percentage points","explanationTemplate":"Absolute risk reduction equals the control event rate minus the treatment event rate: {{control}}% − {{treatment}}% = {{arr}} percentage points."}, + "distractors":[ + {"id":"control","textTemplate":"{{control}}% (the control event rate)","explanationTemplate":"This is the control event rate, not the difference attributable to treatment."}, + {"id":"treatment","textTemplate":"{{treatment}}% (the treatment event rate)","explanationTemplate":"This is the treatment event rate, not the absolute reduction compared with control."}, + {"id":"rrr","textTemplate":"{{rrr}}% (the relative risk reduction)","explanationTemplate":"This is the relative risk reduction: the absolute difference divided by the control event rate."}, + {"id":"nnt","textTemplate":"{{nnt}}","explanationTemplate":"This is the number needed to treat, the reciprocal of the absolute risk reduction expressed as a proportion."}, + {"id":"sum","textTemplate":"{{sum}} percentage points","explanationTemplate":"Adding the two event rates does not measure the treatment-associated reduction in risk."}, + {"id":"ratio","textTemplate":"{{ratio}}% (the relative risk)","explanationTemplate":"This represents the treatment-to-control risk ratio expressed as a percentage, not absolute risk reduction."} + ], + "explanationTemplate":"Use absolute—not relative—risk. In this trial, ARR = control event rate − treatment event rate = {{control}}% − {{treatment}}% = {{arr}} percentage points. Relative risk reduction and number needed to treat answer different questions and should not be substituted.", + "keyTakeaway":"Absolute risk reduction is the arithmetic difference between event rates in the control and treatment groups.", + "usmlePearl":"Report ARR in percentage points; convert it to a proportion only when calculating NNT.", + "reviewFollowups":[{"promptTemplate":"Which calculation gives the number needed to treat for this trial?","choices":[{"id":"n","textTemplate":"1 ÷ ({{arr}}/100)"},{"id":"r","textTemplate":"{{control}} ÷ {{treatment}}"},{"id":"a","textTemplate":"{{control}} + {{treatment}}"},{"id":"s","textTemplate":"1 ÷ {{control}}"}],"correctChoiceId":"n","teachingPointTemplate":"NNT is the reciprocal of ARR expressed as a decimal proportion."}], + "caseFamilies":[ + {"id":"arr-cardiovascular","stemTemplate":"Investigators randomly assign {{size}} adults with established cardiovascular disease to receive a new antiplatelet drug or placebo in addition to standard therapy. Baseline characteristics and follow-up duration are similar between groups, and analysis is by intention to treat. A major cardiovascular event occurs in {{treatment}}% of the treatment group and {{control}}% of the placebo group. Adverse-event discontinuation is similar. Based on these results, what is the absolute risk reduction associated with the new drug?","leadIn":"What is the absolute risk reduction associated with the new drug?","variables":{"size":{"type":"integer","minimum":1200,"maximum":3600,"step":100},"control":{"type":"integer","minimum":18,"maximum":24},"treatment":{"type":"integer","minimum":8,"maximum":12}},"derived":{"arr":{"operation":"subtract","left":"control","right":"treatment","precision":0},"sum":{"operation":"add","left":"control","right":"treatment","precision":0},"rrrFraction":{"operation":"divide","left":"arr","right":"control","precision":4},"rrr":{"operation":"multiply","left":"rrrFraction","right":"100","precision":0},"ratioFraction":{"operation":"divide","left":"treatment","right":"control","precision":4},"ratio":{"operation":"multiply","left":"ratioFraction","right":"100","precision":0},"nnt":{"operation":"divide","left":"100","right":"arr","precision":0}},"constraints":[{"field":"arr","operator":"gt","value":5}]}, + {"id":"arr-infection","stemTemplate":"A multicenter randomized trial evaluates a prophylactic medication in {{size}} immunocompromised adults. Participants, clinicians, and outcome assessors are blinded. During the prespecified follow-up period, invasive infection develops in {{control}}% of patients receiving placebo and {{treatment}}% receiving prophylaxis. Loss to follow-up is minimal and balanced between groups. The investigators report both relative and absolute effect measures. What is the absolute risk reduction produced by prophylaxis?","leadIn":"What is the absolute risk reduction produced by prophylaxis?","variables":{"size":{"type":"integer","minimum":800,"maximum":2400,"step":100},"control":{"type":"integer","minimum":20,"maximum":28},"treatment":{"type":"integer","minimum":9,"maximum":14}},"derived":{"arr":{"operation":"subtract","left":"control","right":"treatment","precision":0},"sum":{"operation":"add","left":"control","right":"treatment","precision":0},"rrrFraction":{"operation":"divide","left":"arr","right":"control","precision":4},"rrr":{"operation":"multiply","left":"rrrFraction","right":"100","precision":0},"ratioFraction":{"operation":"divide","left":"treatment","right":"control","precision":4},"ratio":{"operation":"multiply","left":"ratioFraction","right":"100","precision":0},"nnt":{"operation":"divide","left":"100","right":"arr","precision":0}},"constraints":[{"field":"arr","operator":"gt","value":5}]}, + {"id":"arr-readmission","stemTemplate":"A randomized controlled trial enrolls {{size}} patients discharged after an exacerbation of chronic heart failure. Patients receive either a structured transitional-care program or usual discharge planning. Groups have comparable disease severity, and all participants are analyzed in their assigned group. Hospital readmission within 30 days occurs in {{control}}% of the usual-care group and {{treatment}}% of the intervention group. Which value is the absolute risk reduction attributable to the program?","leadIn":"Which value is the absolute risk reduction attributable to the program?","variables":{"size":{"type":"integer","minimum":600,"maximum":1800,"step":100},"control":{"type":"integer","minimum":22,"maximum":30},"treatment":{"type":"integer","minimum":11,"maximum":16}},"derived":{"arr":{"operation":"subtract","left":"control","right":"treatment","precision":0},"sum":{"operation":"add","left":"control","right":"treatment","precision":0},"rrrFraction":{"operation":"divide","left":"arr","right":"control","precision":4},"rrr":{"operation":"multiply","left":"rrrFraction","right":"100","precision":0},"ratioFraction":{"operation":"divide","left":"treatment","right":"control","precision":4},"ratio":{"operation":"multiply","left":"ratioFraction","right":"100","precision":0},"nnt":{"operation":"divide","left":"100","right":"arr","precision":0}},"constraints":[{"field":"arr","operator":"gt","value":5}]} + ], + "references":[{"title":"Principles of Epidemiology: Analyzing and Interpreting Data","publisher":"CDC","url":"https://www.cdc.gov/field-epi-manual/php/chapters/analyzing-interpreting-data.html","supports":"Absolute measures of association"}], + "tags":["biostatistics","evidence-based-medicine","gold"] + } +] diff --git a/docs/dynamic-question-quality.md b/docs/dynamic-question-quality.md new file mode 100644 index 0000000..ab29879 --- /dev/null +++ b/docs/dynamic-question-quality.md @@ -0,0 +1,36 @@ +# Dynamic question quality standard + +Quality-standard-2 blueprints are eligible for learner runs. Older blueprints +remain stored only so signed questions and reports can be regenerated. + +## Content requirements + +- At least three curated case families per objective. +- One invariant `correctConceptId` across every family. +- A complete clinical vignette, focused lead-in, and coherent correlated data. +- At least six plausible distractors with an explanation for every answer. +- A complete explanation, key takeaway, USMLE pearl, review follow-up, and + wrong-answer remediation. +- At least one authoritative reference. + +Promotion validates 10,000 seeds for every quality-standard-2 blueprint. Existing +blueprint versions are immutable; content changes require a version increment. + +## Blinded acceptance + +Create a balanced packet of three generated variants per gold objective and the +same number of authored questions: + +```text +dotnet QuestionsApi.dll export-dynamic-review \ + --source /app/content/dynamic_questions \ + --output /tmp/dynamic-review +``` + +Keep `answer-key.json` separate from `review-packet.json`. Two clinical reviewers +score realism, distractor quality, explanation quality, and Step 3 relevance from +1–5 and predict whether each item is authored or dynamic. + +The pilot passes when there are no critical medical or ambiguity findings, every +dimension averages at least 4/5, and source-identification accuracy is no greater +than 65%. Corrections are published as a new blueprint version and reviewed again. diff --git a/tests/smoke/uat-smoke.spec.mjs b/tests/smoke/uat-smoke.spec.mjs index a96ba2e..5c27669 100644 --- a/tests/smoke/uat-smoke.spec.mjs +++ b/tests/smoke/uat-smoke.spec.mjs @@ -140,6 +140,29 @@ test("public API and disposable learner journey", async () => { await call("/api/dashboard", { token }); await call("/api/ratings/me", { token }); + const { data: dynamicStatus } = await call("/api/dynamic-questions/status", { token }); + expect(dynamicStatus.available).toBe(true); + expect(dynamicStatus.publishedObjectives).toBe(5); + const { data: dynamicQuestion } = await call("/api/dynamic-questions/next", { + method: "POST", token, body: { runToken: "" } + }); + expect(dynamicQuestion.generationToken).toBeTruthy(); + expect(dynamicQuestion.runToken).toBeTruthy(); + expect(dynamicQuestion.question.stem.length).toBeGreaterThan(220); + expect(dynamicQuestion.question.choices).toHaveLength(5); + expect(Object.keys(dynamicQuestion.question.choiceExplanations)).toHaveLength(5); + expect(dynamicQuestion.question.reviewFollowups.length).toBeGreaterThan(0); + const dynamicChoice = dynamicQuestion.question.choices[0].id; + const { data: dynamicAttempt } = await call("/api/dynamic-questions/attempt", { + method: "POST", token, body: { + generationToken: dynamicQuestion.generationToken, + choiceId: dynamicChoice, + isSkip: false, + durationSeconds: 5 + } + }); + expect(dynamicAttempt.objectiveId).toBeTruthy(); + const { data: queue } = await call("/api/review-queue/next", { method: "POST", token, body: { excludeExternalIds: [], supportsChoicePools: true } });