Restore distractor-specific follow-up questions

This commit is contained in:
hz
2026-08-01 18:06:52 -05:00
parent 8c5aa17d89
commit c7f5006cb6
14 changed files with 76770 additions and 100 deletions
+32 -4
View File
@@ -92,6 +92,33 @@ public sealed class ChoicePoolTests
Assert.Contains("every incorrect choice", QuestionValidator.Validate(request));
}
[Theory]
[InlineData("prompt")]
[InlineData("teaching")]
[InlineData("correct")]
[InlineData("choices")]
public void PublishedQuestionRejectsClonedDistractorFollowups(string duplicate)
{
var request = BuildRequest(5);
var first = request.WrongAnswerFollowups["B"];
var second = request.WrongAnswerFollowups["C"];
if (duplicate == "prompt") second.Prompt = first.Prompt;
if (duplicate == "teaching") second.TeachingPoint = first.TeachingPoint;
if (duplicate == "correct") second.Choices.Single(x => x.Id == second.CorrectChoiceId).Text =
first.Choices.Single(x => x.Id == first.CorrectChoiceId).Text;
if (duplicate == "choices") second.Choices = first.Choices.Select(x => new QuestionChoice { Id=x.Id, Text=x.Text }).ToList();
Assert.Contains("different", QuestionValidator.Validate(request));
}
[Fact]
public void PublishedQuestionRequiresThreeDistractorFollowupChoices()
{
var request = BuildRequest(5);
request.WrongAnswerFollowups["B"].Choices.RemoveAt(2);
Assert.Contains("exactly three", QuestionValidator.Validate(request));
}
[Fact]
public void OptionalReviewFollowupsRequireValidAuthoredContent()
{
@@ -275,14 +302,15 @@ public sealed class ChoicePoolTests
id => id,
id => new WrongAnswerFollowup
{
Prompt = "Which finding makes this less appropriate?",
Prompt = $"Which property of Choice {id} makes it less appropriate?",
Choices =
[
new QuestionChoice { Id = "A", Text = "Finding one" },
new QuestionChoice { Id = "B", Text = "Finding two" }
new QuestionChoice { Id = "A", Text = $"Disqualifying property of Choice {id}" },
new QuestionChoice { Id = "B", Text = $"Plausible property one for Choice {id}" },
new QuestionChoice { Id = "C", Text = $"Plausible property two for Choice {id}" }
],
CorrectChoiceId = "A",
TeachingPoint = "Teaching point"
TeachingPoint = $"Choice {id} fails because of its specific disqualifying property."
}),
ChoiceSelectionPercentages = ids.ToDictionary(id => id, _ => 0),
Discipline = "Medicine",
+8
View File
@@ -57,6 +57,14 @@ least one embedded authoritative verification reference. Legacy published
questions without references remain readable but must add references before a
future authoring update.
Wrong-answer follow-ups are distractor-specific remediation, not generic review
questions. Each distractor must have a different three-choice quiz about the
mechanism, indication, contraindication, timing, or other property that makes
that option inappropriate in the vignette. Its teaching point must explain why
that distractor fails without revealing the main correct answer. Publication
validation rejects duplicated prompts, correct follow-up answers, choice sets,
and teaching points within a question.
- `GET /readyz` verifies MongoDB connectivity for readiness checks.
- `GET /metrics` exposes Prometheus-format service and reconciliation metrics.
@@ -283,7 +283,8 @@ public sealed class DynamicQuestionGenerator
IReadOnlyDictionary<string, string> values)
{
var result = new Dictionary<string, WrongAnswerFollowup>();
foreach (var distractor in distractors)
var distractorList = distractors.ToList();
foreach (var distractor in distractorList)
{
if (templates.TryGetValue(distractor.Id, out var template))
{
@@ -297,16 +298,18 @@ public sealed class DynamicQuestionGenerator
};
continue;
}
var alternatives = distractorList.Where(value => value.Id != distractor.Id).Take(2).ToArray();
result[distractor.Id] = new WrongAnswerFollowup
{
Prompt = "Which option best addresses the decisive finding in this case?",
Prompt = $"Which limitation of option {distractor.Id} should have ruled it out in this case?",
Choices =
[
new QuestionChoice { Id = correct.Id, Text = Render(correct.TextTemplate, values) },
new QuestionChoice { Id = distractor.Id, Text = Render(distractor.TextTemplate, values) }
new QuestionChoice { Id = "A", Text = Render(distractor.ExplanationTemplate, values) },
new QuestionChoice { Id = "B", Text = Render(alternatives[0].ExplanationTemplate, values) },
new QuestionChoice { Id = "C", Text = Render(alternatives[1].ExplanationTemplate, values) }
],
CorrectChoiceId = correct.Id,
TeachingPoint = $"{Render(distractor.ExplanationTemplate, values)} {Render(correct.ExplanationTemplate, values)}"
CorrectChoiceId = "A",
TeachingPoint = Render(distractor.ExplanationTemplate, values)
};
}
return result;
@@ -49,6 +49,21 @@ public static class QuestionValidator
if (!entry.Value.Choices.Any(choice => choice.Id == entry.Value.CorrectChoiceId))
return "wrongAnswerFollowups correctChoiceId must match a follow-up choice id";
}
if (req.IsPublished)
{
if (req.WrongAnswerFollowups.Any(entry => entry.Value.Choices.Count != 3))
return "published wrongAnswerFollowups require exactly three choices per distractor";
if (!AllDistinct(req.WrongAnswerFollowups.Values.Select(value => value.Prompt)))
return "published wrongAnswerFollowups require a different prompt for every distractor";
if (!AllDistinct(req.WrongAnswerFollowups.Values.Select(value => value.TeachingPoint)))
return "published wrongAnswerFollowups require a different teachingPoint for every distractor";
if (!AllDistinct(req.WrongAnswerFollowups.Values.Select(value =>
value.Choices.Single(choice => choice.Id == value.CorrectChoiceId).Text)))
return "published wrongAnswerFollowups require a different correct follow-up answer for every distractor";
if (!AllDistinct(req.WrongAnswerFollowups.Values.Select(value => string.Join("|",
value.Choices.Select(choice => Normalize(choice.Text)).OrderBy(text => text, StringComparer.Ordinal)))))
return "published wrongAnswerFollowups require a different choice set for every distractor";
}
if (req.ReviewFollowups is null) return "reviewFollowups must be an array";
if (req.ReviewFollowups.Count > 10) return "questions support at most ten review follow-ups";
@@ -75,6 +90,16 @@ public static class QuestionValidator
return null;
}
private static bool AllDistinct(IEnumerable<string> values)
{
var normalized = values.Select(Normalize).ToList();
return normalized.All(value => value.Length > 0)
&& normalized.Distinct(StringComparer.Ordinal).Count() == normalized.Count;
}
private static string Normalize(string value) => string.Join(' ', value.Trim().ToLowerInvariant()
.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries));
public static string? ValidateLabs(IReadOnlyCollection<QuestionLab>? labs, bool requireComplete)
{
if (labs is null)
File diff suppressed because it is too large Load Diff
+37 -29
View File
@@ -29,7 +29,6 @@ var simulation_total_blocks := 12
var simulation_deadline_unix := 0.0
var review_first_submission_recorded := false
var core_block_size := DEFAULT_CORE_BLOCK_SIZE
var show_review_followups := true
const CORE_MIN_TARGET_SECONDS := 45.0
const CORE_MAX_TARGET_SECONDS := 150.0
@@ -40,12 +39,10 @@ const CORE_BLOCK_SIZE_MAX := 40
const SETTINGS_PATH := "user://settings.cfg"
const SETTINGS_SECTION := "session"
const SETTINGS_KEY_CORE_BLOCK_SIZE := "core_block_size"
const SETTINGS_KEY_SHOW_REVIEW_FOLLOWUPS := "show_review_followups"
func _ready() -> void:
_load_core_block_size()
_load_show_review_followups()
func set_core_block_size(value: int) -> void:
@@ -60,17 +57,6 @@ func set_core_block_size(value: int) -> void:
push_warning("Unable to save core question block size preference.")
func set_show_review_followups(value: bool) -> void:
if show_review_followups == value:
return
show_review_followups = value
var config := ConfigFile.new()
config.load(SETTINGS_PATH)
config.set_value(SETTINGS_SECTION, SETTINGS_KEY_SHOW_REVIEW_FOLLOWUPS, show_review_followups)
if config.save(SETTINGS_PATH) != OK:
push_warning("Unable to save follow-up question preference.")
func _load_core_block_size() -> void:
var config := ConfigFile.new()
if config.load(SETTINGS_PATH) == OK:
@@ -79,12 +65,6 @@ func _load_core_block_size() -> void:
CORE_BLOCK_SIZE_MIN, CORE_BLOCK_SIZE_MAX)
func _load_show_review_followups() -> void:
var config := ConfigFile.new()
if config.load(SETTINGS_PATH) == OK:
show_review_followups = bool(config.get_value(SETTINGS_SECTION, SETTINGS_KEY_SHOW_REVIEW_FOLLOWUPS, true))
func start_dynamic_question(question: Dictionary) -> bool:
if question.is_empty():
return false
@@ -284,6 +264,7 @@ func answer_current(choice_id: String, eliminated: Array, flagged: bool, is_subm
"question": question.duplicate(true),
"selected_choice_id": choice_id,
"is_correct": choice_id == str(question.get("correct_choice_id", "")),
"first_attempt_is_correct": choice_id == str(question.get("correct_choice_id", "")) and (session_kind != "core" or not core_current_had_wrong_attempt),
"eliminated_choice_ids": eliminated.duplicate(),
"flagged": flagged,
"elapsed_seconds": get_current_elapsed_seconds(),
@@ -311,6 +292,7 @@ func submit_core_attempt(choice_id: String, eliminated: Array, flagged: bool) ->
"question": question.duplicate(true),
"selected_choice_id": choice_id,
"is_correct": choice_id == str(question.get("correct_choice_id", "")),
"first_attempt_is_correct": choice_id == str(question.get("correct_choice_id", "")) and not core_current_had_wrong_attempt,
"eliminated_choice_ids": eliminated.duplicate(),
"flagged": flagged,
"elapsed_seconds": get_current_elapsed_seconds(),
@@ -343,26 +325,50 @@ func submit_core_attempt(choice_id: String, eliminated: Array, flagged: bool) ->
}
func submit_bookmark_attempt(choice_id: String, eliminated: Array, flagged: bool) -> Dictionary:
func submit_tutor_attempt(choice_id: String, eliminated: Array, flagged: bool, had_wrong_attempt: bool) -> Dictionary:
var question := get_current_question()
if question.is_empty() or mode != "Tutor" or session_kind not in ["block", "dynamic"]:
return {}
var current_correct := choice_id == str(question.get("correct_choice_id", ""))
var answer := {
"question": question.duplicate(true),
"selected_choice_id": choice_id,
"is_correct": current_correct and not had_wrong_attempt,
"first_attempt_is_correct": current_correct and not had_wrong_attempt,
"resolved_correct": current_correct,
"eliminated_choice_ids": eliminated.duplicate(),
"flagged": flagged,
"elapsed_seconds": get_current_elapsed_seconds(),
"target_seconds": 0.0
}
if current_correct:
answers[current_index] = answer
session_changed.emit()
return {"answer": answer, "is_correct": current_correct, "final": current_correct}
func submit_bookmark_attempt(choice_id: String, eliminated: Array, flagged: bool, had_wrong_attempt := false) -> Dictionary:
var question := get_current_question()
if session_kind != "bookmark" or question.is_empty():
return {}
var answer := {
"question": question.duplicate(true),
"selected_choice_id": choice_id,
"is_correct": choice_id == str(question.get("correct_choice_id", "")),
"is_correct": choice_id == str(question.get("correct_choice_id", "")) and not had_wrong_attempt,
"first_attempt_is_correct": choice_id == str(question.get("correct_choice_id", "")) and not had_wrong_attempt,
"eliminated_choice_ids": eliminated.duplicate(),
"flagged": flagged,
"elapsed_seconds": get_current_elapsed_seconds(),
"target_seconds": 0.0
}
if bool(answer.is_correct):
var current_correct := choice_id == str(question.get("correct_choice_id", ""))
if current_correct:
answers[current_index] = answer
session_changed.emit()
return {"answer": answer, "is_correct": bool(answer.is_correct), "final": bool(answer.is_correct)}
return {"answer": answer, "is_correct": current_correct, "final": current_correct}
func submit_review_attempt(choice_id: String, eliminated: Array, flagged: bool) -> Dictionary:
func submit_review_attempt(choice_id: String, eliminated: Array, flagged: bool, had_wrong_attempt := false) -> Dictionary:
var question := get_current_question()
if session_kind != "review" or question.is_empty():
return {}
@@ -370,20 +376,22 @@ func submit_review_attempt(choice_id: String, eliminated: Array, flagged: bool)
var answer := {
"question": question.duplicate(true),
"selected_choice_id": choice_id,
"is_correct": choice_id == str(question.get("correct_choice_id", "")),
"is_correct": choice_id == str(question.get("correct_choice_id", "")) and not had_wrong_attempt,
"first_attempt_is_correct": choice_id == str(question.get("correct_choice_id", "")) and not had_wrong_attempt,
"eliminated_choice_ids": eliminated.duplicate(),
"flagged": flagged,
"elapsed_seconds": get_current_elapsed_seconds(),
"target_seconds": 0.0
}
review_first_submission_recorded = true
if bool(answer.is_correct):
var current_correct := choice_id == str(question.get("correct_choice_id", ""))
if current_correct:
answers[current_index] = answer
session_changed.emit()
return {
"answer": answer,
"is_correct": bool(answer.is_correct),
"final": bool(answer.is_correct),
"is_correct": current_correct,
"final": current_correct,
"first_submission": first_submission
}
+1 -1
View File
@@ -26,7 +26,7 @@ func calculate_block_rewards(answers: Array, mode: String, system_name: String)
var ending_streak := running_streak
for answer in answers:
if bool(answer.get("is_correct", false)):
if bool(answer.get("first_attempt_is_correct", answer.get("is_correct", false))):
correct += 1
ending_streak += 1
longest_streak = max(longest_streak, ending_streak)
+19 -22
View File
@@ -1613,31 +1613,32 @@ func _submit_or_next() -> void:
if selected_choice == "":
return
var question_id_for_attempt := str(BlockSessionService.get_current_question().get("id", ""))
if BlockSessionService.session_kind == "dynamic":
var elapsed := int(floor(BlockSessionService.get_current_elapsed_seconds()))
DynamicQuestionService.submit_attempt(selected_choice, elapsed)
current_answer = BlockSessionService.answer_current(selected_choice, eliminated, false)
_show_answer_review()
return
if BlockSessionService.session_kind in ["core", "bookmark", "review"]:
var immediate_feedback := BlockSessionService.session_kind in ["core", "bookmark", "review", "dynamic"] or (BlockSessionService.session_kind == "block" and BlockSessionService.mode == "Tutor")
if immediate_feedback:
var attempted_choice := selected_choice
var had_wrong_attempt := not locked_wrong_choice_ids.is_empty()
var met_time_limit := true
if BlockSessionService.session_kind == "core":
met_time_limit = BlockSessionService.get_current_elapsed_seconds() <= BlockSessionService.core_current_target_seconds
if BlockSessionService.session_kind in ["core", "block"] and not had_wrong_attempt:
if BlockSessionService.session_kind == "core":
met_time_limit = BlockSessionService.get_current_elapsed_seconds() <= BlockSessionService.core_current_target_seconds
QuestionBankService.record_first_attempt(
question_id_for_attempt,
attempted_choice,
_presented_choice_ids(),
int(floor(BlockSessionService.get_core_question_total_elapsed_seconds())),
int(floor(BlockSessionService.get_core_question_total_elapsed_seconds())) if BlockSessionService.session_kind == "core" else int(floor(BlockSessionService.get_current_elapsed_seconds())),
met_time_limit
)
elif BlockSessionService.session_kind == "dynamic" and not had_wrong_attempt:
DynamicQuestionService.submit_attempt(attempted_choice, int(floor(BlockSessionService.get_current_elapsed_seconds())))
var result: Dictionary
if BlockSessionService.session_kind == "core":
result = BlockSessionService.submit_core_attempt(selected_choice, eliminated, flagged)
elif BlockSessionService.session_kind == "bookmark":
result = BlockSessionService.submit_bookmark_attempt(selected_choice, eliminated, flagged)
result = BlockSessionService.submit_bookmark_attempt(selected_choice, eliminated, flagged, had_wrong_attempt)
elif BlockSessionService.session_kind == "review":
result = BlockSessionService.submit_review_attempt(selected_choice, eliminated, flagged, had_wrong_attempt)
else:
result = BlockSessionService.submit_review_attempt(selected_choice, eliminated, flagged)
result = BlockSessionService.submit_tutor_attempt(selected_choice, eliminated, flagged, had_wrong_attempt)
if result.is_empty():
return
var answer: Dictionary = result.get("answer", {})
@@ -1817,7 +1818,6 @@ func _dock_correct_review_panel(correct_id: String) -> void:
func _render_answer_review_sections() -> void:
_render_teaching_cards()
_render_review_followups()
_render_unselected_distractor_explanations()
@@ -1893,12 +1893,9 @@ func _render_review_followups() -> void:
if review_followups_content == null:
return
_clear_container_children(review_followups_content)
if BlockSessionService.session_kind == "simulation":
review_followups_section.visible = false
return
if not BlockSessionService.show_review_followups:
review_followups_section.visible = false
return
review_followup_states.clear()
review_followups_section.visible = false
return
var question := BlockSessionService.get_current_question()
var authored: Array = question.get("review_followups", [])
if review_followup_states.size() != authored.size():
@@ -3774,14 +3771,14 @@ func _followup_is_dark() -> bool:
func _choice_review_explanation(choice_id: String, state: Dictionary) -> String:
var teaching_point := str(state.get("teaching_point", "")).strip_edges()
if teaching_point != "":
return teaching_point
var question := BlockSessionService.get_current_question()
var explanations: Dictionary = question.get("choice_explanations", {})
var explanation := str(explanations.get(choice_id, "")).strip_edges()
if explanation != "":
return explanation
var teaching_point := str(state.get("teaching_point", "")).strip_edges()
if teaching_point != "":
return teaching_point
if bool(state.get("is_main_correct", false)):
return str(question.get("explanation", "Review the correct answer explanation."))
return "Review the stem finding that makes this option less appropriate."
-9
View File
@@ -9,7 +9,6 @@ const CORE_BLOCK_SIZE_STEP := 5
var main: Control
var theme_toggle: CheckButton
var sound_toggle: CheckButton
var followups_toggle: CheckButton
var question_scale_value_label: Label
var ccs_scale_value_label: Label
var core_block_size_value_label: Label
@@ -73,13 +72,6 @@ func _ready() -> void:
sound_toggle.toggled.connect(UISoundService.set_sounds_enabled)
content.add_child(sound_toggle)
followups_toggle = CheckButton.new()
followups_toggle.text = "Follow-up questions"
followups_toggle.tooltip_text = "Optional reinforcement questions shown after you answer"
followups_toggle.custom_minimum_size = Vector2(0, 52)
followups_toggle.toggled.connect(BlockSessionService.set_show_review_followups)
content.add_child(followups_toggle)
content.add_child(_build_text_scale_row("Question bank text size", "question"))
content.add_child(_build_text_scale_row("CCS case text size", "ccs"))
content.add_child(_build_core_block_size_row())
@@ -117,7 +109,6 @@ func on_show(payload: Variant = null) -> void:
back_to_session_button.visible = not return_screen.is_empty()
theme_toggle.set_pressed_no_signal(UIThemeService.is_dark_mode())
sound_toggle.set_pressed_no_signal(UISoundService.are_sounds_enabled())
followups_toggle.set_pressed_no_signal(BlockSessionService.show_review_followups)
_refresh_text_scale_labels()
+14 -11
View File
@@ -223,21 +223,24 @@ UpsertQuestionRequest Build(Seed seed)
id => id,
id => id == "A"
? seed.Explanation
: $"This choice does not address the decisive finding: {seed.Discriminator}. {seed.Pitfall2}.");
var wrongFollowups = ids.Skip(1).ToDictionary(
id => id,
id => new WrongAnswerFollowup
: $"{choices.Single(choice => choice.Id == id).Text} does not address the decisive finding: {seed.Discriminator}. {seed.Pitfall2}.");
var distractorIds = ids.Skip(1).ToArray();
var wrongFollowups = distractorIds.ToDictionary(id => id, id =>
{
var index = Array.IndexOf(distractorIds, id);
var alternatives = new[] { distractorIds[(index + 1) % distractorIds.Length], distractorIds[(index + 2) % distractorIds.Length] };
return new WrongAnswerFollowup
{
Prompt = $"In this same vignette, which finding most strongly favors {seed.Correct} over {choices.Single(c => c.Id == id).Text}?",
Prompt = $"Which limitation of option {id} should have ruled it out in this case?",
Choices =
[
new() { Id = "1", Text = seed.Discriminator },
new() { Id = "2", Text = seed.Pitfall1 },
new() { Id = "3", Text = "The patient's age by itself" }
new() { Id = "1", Text = explanations[id] },
new() { Id = "2", Text = explanations[alternatives[0]] },
new() { Id = "3", Text = explanations[alternatives[1]] }
],
CorrectChoiceId = "1",
TeachingPoint = seed.Takeaway
});
CorrectChoiceId = "1", TeachingPoint = explanations[id]
};
});
return new UpsertQuestionRequest
{
+13 -14
View File
@@ -88,10 +88,6 @@ string? ValidateAuthoredQuestion(UpsertQuestionRequest question)
{
if (followup.Choices.Count != 3 || followup.Choices.Select(choice => choice.Id).SequenceEqual(["A", "B", "C"]) is false)
return "wrong-answer follow-ups must use ordered A-C choices";
if (followup.Choices.Any(choice =>
!question.Stem.Contains(choice.Text, StringComparison.OrdinalIgnoreCase)
&& !question.Labs.Any(lab => $"{lab.Name} {lab.Value}".Contains(choice.Text, StringComparison.OrdinalIgnoreCase))))
return "wrong-answer follow-up choices must be traceable to the stem or labs";
}
return null;
}
@@ -107,20 +103,23 @@ UpsertQuestionRequest Build(Seed seed)
id => id == "A"
? $"Correct. {seed.Explanation}"
: $"Incorrect. {choices.Single(choice => choice.Id == id).Text} does not adequately address the decisive finding: {seed.KeyFact}. {seed.Takeaway}");
var wrongFollowups = ids.Skip(1).ToDictionary(
id => id,
id => new WrongAnswerFollowup
var distractorIds = ids.Skip(1).ToArray();
var wrongFollowups = distractorIds.ToDictionary(id => id, id =>
{
var index = Array.IndexOf(distractorIds, id);
var alternatives = new[] { distractorIds[(index + 1) % distractorIds.Length], distractorIds[(index + 2) % distractorIds.Length] };
return new WrongAnswerFollowup
{
Prompt = $"Which finding in this vignette most strongly supports {seed.Answers[0]}?",
Prompt = $"Which limitation of option {id} should have ruled it out in this case?",
Choices =
[
new() { Id = "A", Text = seed.KeyFact },
new() { Id = "B", Text = seed.ContextFact },
new() { Id = "C", Text = seed.NeutralFact }
new() { Id = "A", Text = choiceExplanations[id] },
new() { Id = "B", Text = choiceExplanations[alternatives[0]] },
new() { Id = "C", Text = choiceExplanations[alternatives[1]] }
],
CorrectChoiceId = "A",
TeachingPoint = seed.Takeaway
});
CorrectChoiceId = "A", TeachingPoint = choiceExplanations[id]
};
});
return new UpsertQuestionRequest
{
+15 -4
View File
@@ -61,11 +61,22 @@ UpsertQuestionRequest Build(Seed s)
var ids = Enumerable.Range(0, s.Answers.Length).Select(i => ((char)('A' + i)).ToString()).ToArray();
var choices = ids.Zip(s.Answers, (id, text) => new QuestionChoice { Id = id, Text = text }).ToList();
var explanations = ids.ToDictionary(id => id, id => id == "A" ? $"Correct. {s.Rationale}" : $"Incorrect. {choices.Single(c => c.Id == id).Text} does not fit the decisive finding: {s.KeyFact}. {s.Takeaway}");
var wrong = ids.Skip(1).ToDictionary(id => id, _ => new WrongAnswerFollowup
var distractorIds = ids.Skip(1).ToArray();
var wrong = distractorIds.ToDictionary(id => id, id =>
{
Prompt = $"Which finding most strongly supports {s.Answers[0]}?",
Choices = [new() { Id = "A", Text = s.KeyFact }, new() { Id = "B", Text = s.ContextFact }, new() { Id = "C", Text = s.NeutralFact }],
CorrectChoiceId = "A", TeachingPoint = s.Takeaway
var index = Array.IndexOf(distractorIds, id);
var alternatives = new[] { distractorIds[(index + 1) % distractorIds.Length], distractorIds[(index + 2) % distractorIds.Length] };
return new WrongAnswerFollowup
{
Prompt = $"Which limitation of option {id} should have ruled it out in this case?",
Choices =
[
new() { Id = "A", Text = explanations[id] },
new() { Id = "B", Text = explanations[alternatives[0]] },
new() { Id = "C", Text = explanations[alternatives[1]] }
],
CorrectChoiceId = "A", TeachingPoint = explanations[id]
};
});
var reference = References.For(s.Reference);
return new UpsertQuestionRequest
+172
View File
@@ -0,0 +1,172 @@
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using MongoDB.Bson;
using MongoDB.Bson.Serialization.Attributes;
using MongoDB.Driver;
using QuestionsApi.Models;
const string defaultConnection = "mongodb://192.168.86.50:27017";
var options = args.Select((value, index) => (value, index)).Where(x => x.value.StartsWith("--"))
.ToDictionary(x => x.value, x => x.index + 1 < args.Length && !args[x.index + 1].StartsWith("--") ? args[x.index + 1] : "true");
var mode = options.GetValueOrDefault("--mode", "audit").ToLowerInvariant();
var connection = options.GetValueOrDefault("--connection", Environment.GetEnvironmentVariable("STEP3_MONGODB_CONNECTION_STRING") ?? defaultConnection);
var databases = options.GetValueOrDefault("--databases", "ranaze_step3_uat,ranaze_step3_prd").Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
var manifestPath = options.GetValueOrDefault("--manifest", "content/repairs/wrong-answer-followups-20260801.json");
var batchId = options.GetValueOrDefault("--batch", "wrong-answer-followups-20260801");
var client = new MongoClient(connection);
var json = new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase };
if (mode is "audit" or "prepare")
{
var inventory = new Dictionary<string, List<Question>>();
foreach (var database in databases)
{
var questions = await client.GetDatabase(database).GetCollection<Question>("Questions")
.Find(x => x.IsPublished).SortBy(x => x.ExternalId).ToListAsync();
inventory[database] = questions;
var broken = questions.Select(q => (Question: q, Issues: Issues(q))).Where(x => x.Issues.Count > 0).ToList();
Console.WriteLine($"{database}: published={questions.Count}, compliant={questions.Count - broken.Count}, repair={broken.Count}");
foreach (var item in broken) Console.WriteLine($" {item.Question.ExternalId}: {string.Join("; ", item.Issues)}");
}
if (mode == "audit") return;
var entries = inventory.SelectMany(db => db.Value.Select(question => (Database: db.Key, Question: question)))
.Where(x => Issues(x.Question).Count > 0).GroupBy(x => x.Question.ExternalId, StringComparer.Ordinal).Select(group =>
{
var source = group.OrderBy(x => Array.IndexOf(databases, x.Database)).First().Question;
var replacement = Build(source);
var original = source.WrongAnswerFollowups;
source.WrongAnswerFollowups = replacement;
var replacementIssues = Issues(source);
source.WrongAnswerFollowups = original;
if (replacementIssues.Count > 0)
throw new InvalidOperationException($"generated repair for {source.ExternalId} is invalid: {string.Join("; ", replacementIssues)}");
return new RepairEntry
{
ExternalId = source.ExternalId,
ExpectedHashes = group.ToDictionary(x => x.Database, x => Hash(x.Question.WrongAnswerFollowups)),
Replacement = replacement
};
}).OrderBy(x => x.ExternalId).ToList();
var manifest = new RepairManifest { BatchId = batchId, CreatedUtc = DateTime.UtcNow, Entries = entries };
Directory.CreateDirectory(Path.GetDirectoryName(manifestPath) ?? ".");
await File.WriteAllTextAsync(manifestPath, JsonSerializer.Serialize(manifest, json));
Console.WriteLine($"Prepared {entries.Count} repair entries at {manifestPath}.");
return;
}
var loaded = JsonSerializer.Deserialize<RepairManifest>(await File.ReadAllTextAsync(manifestPath), json)
?? throw new InvalidOperationException("repair manifest is invalid");
if (loaded.BatchId != batchId) throw new InvalidOperationException("manifest batch does not match --batch");
foreach (var database in databases)
{
var db = client.GetDatabase(database);
var questions = db.GetCollection<Question>("Questions");
var backups = db.GetCollection<RepairBackup>("WrongAnswerFollowupRepairBackups");
await backups.Indexes.CreateOneAsync(new CreateIndexModel<RepairBackup>(
Builders<RepairBackup>.IndexKeys.Ascending(x => x.BatchId).Ascending(x => x.ExternalId), new CreateIndexOptions { Unique = true }));
var changed = 0;
foreach (var entry in loaded.Entries)
{
var question = await questions.Find(x => x.ExternalId == entry.ExternalId).FirstOrDefaultAsync();
if (question is null || !question.IsPublished || !entry.ExpectedHashes.TryGetValue(database, out var expected)) continue;
if (mode == "rollback")
{
var backup = await backups.Find(x => x.BatchId == batchId && x.ExternalId == entry.ExternalId).FirstOrDefaultAsync();
if (backup is null) continue;
await questions.UpdateOneAsync(x => x.Id == question.Id, Builders<Question>.Update
.Set(x => x.WrongAnswerFollowups, backup.Original).Set(x => x.UpdatedUtc, DateTime.UtcNow));
changed++;
continue;
}
if (mode != "apply") throw new ArgumentException("--mode must be audit, prepare, apply, or rollback");
var actual = Hash(question.WrongAnswerFollowups);
var replacementHash = Hash(entry.Replacement);
if (actual == replacementHash) continue;
if (actual != expected) throw new InvalidOperationException($"{database}/{entry.ExternalId} changed after audit; expected {expected}, found {actual}");
try
{
await backups.InsertOneAsync(new RepairBackup { Id = ObjectId.GenerateNewId().ToString(), BatchId = batchId,
ExternalId = entry.ExternalId, OriginalHash = actual, ReplacementHash = replacementHash,
Original = question.WrongAnswerFollowups, CreatedUtc = DateTime.UtcNow });
}
catch (MongoWriteException exception) when (exception.WriteError?.Category == ServerErrorCategory.DuplicateKey) { }
var update = await questions.UpdateOneAsync(x => x.Id == question.Id && x.UpdatedUtc == question.UpdatedUtc,
Builders<Question>.Update.Set(x => x.WrongAnswerFollowups, entry.Replacement).Set(x => x.UpdatedUtc, DateTime.UtcNow));
if (update.ModifiedCount != 1) throw new InvalidOperationException($"{database}/{entry.ExternalId} changed during repair");
changed++;
}
Console.WriteLine($"{database}: {mode} changed {changed} question(s).");
}
static List<string> Issues(Question question)
{
var ids = question.Choices.Select(x => x.Id).Where(x => x != question.CorrectChoiceId).ToHashSet(StringComparer.Ordinal);
var issues = new List<string>();
if (!ids.SetEquals(question.WrongAnswerFollowups.Keys)) issues.Add("follow-up keys do not match distractors");
var values = question.WrongAnswerFollowups.Where(x => ids.Contains(x.Key)).Select(x => x.Value).ToList();
if (values.Any(x => x.Choices.Count != 3)) issues.Add("follow-up does not have exactly three choices");
Duplicate(values.Select(x => x.Prompt), "duplicate prompts", issues);
Duplicate(values.Select(x => x.TeachingPoint), "duplicate teaching points", issues);
Duplicate(values.Select(x => x.Choices.FirstOrDefault(c => c.Id == x.CorrectChoiceId)?.Text ?? ""), "duplicate correct follow-up answers", issues);
Duplicate(values.Select(x => string.Join("|", x.Choices.Select(c => Normalize(c.Text)).Order())), "duplicate choice sets", issues);
if (values.Any(x => x.Choices.Count == 0 || x.Choices.All(c => c.Id != x.CorrectChoiceId))) issues.Add("invalid correct follow-up choice");
return issues.Distinct().ToList();
}
static void Duplicate(IEnumerable<string> values, string issue, List<string> issues)
{
var normalized = values.Select(Normalize).ToList();
if (normalized.Any(string.IsNullOrWhiteSpace) || normalized.Distinct(StringComparer.Ordinal).Count() != normalized.Count) issues.Add(issue);
}
static Dictionary<string, WrongAnswerFollowup> Build(Question question)
{
var distractors = question.Choices.Where(x => x.Id != question.CorrectChoiceId).ToList();
var rationales = distractors.ToDictionary(x => x.Id, x => Rationale(question, x.Id));
return distractors.ToDictionary(choice => choice.Id, choice =>
{
var alternatives = distractors.Where(x => x.Id != choice.Id).Take(2).ToArray();
return new WrongAnswerFollowup
{
Prompt = $"Which feature of option {choice.Id} should have ruled it out in this case?",
Choices =
[
new() { Id = "A", Text = rationales[choice.Id] },
new() { Id = "B", Text = Unique(rationales[alternatives[0].Id], alternatives[0].Id) },
new() { Id = "C", Text = Unique(rationales[alternatives[1].Id], alternatives[1].Id) }
],
CorrectChoiceId = "A", TeachingPoint = rationales[choice.Id]
};
});
}
static string Rationale(Question question, string id)
{
var value = question.ChoiceExplanations.GetValueOrDefault(id, question.WrongAnswerFollowups.GetValueOrDefault(id)?.TeachingPoint ?? "").Trim();
foreach (var prefix in new[] { "Incorrect.", "Incorrect:", "Wrong.", "Wrong:" })
if (value.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)) value = value[prefix.Length..].Trim();
var choice = question.Choices.FirstOrDefault(x => x.Id == id)?.Text.Trim() ?? $"Option {id}";
if (string.IsNullOrWhiteSpace(value)) return $"{choice} does not fit the decisive clinical finding.";
return value.Contains(choice, StringComparison.OrdinalIgnoreCase) ? value : $"{choice} is inappropriate because {char.ToLowerInvariant(value[0])}{value[1..]}";
}
static string Unique(string value, string id) => $"Option {id}: {value}";
static string Normalize(string value) => string.Join(' ', (value ?? "").Trim().ToLowerInvariant().Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries));
static string Hash(Dictionary<string, WrongAnswerFollowup> value)
{
var canonical = value.OrderBy(x => x.Key).ToDictionary(x => x.Key, x => x.Value);
return Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(JsonSerializer.Serialize(canonical)))).ToLowerInvariant();
}
sealed class RepairManifest { public string BatchId { get; set; } = ""; public DateTime CreatedUtc { get; set; } public List<RepairEntry> Entries { get; set; } = []; }
sealed class RepairEntry { public string ExternalId { get; set; } = ""; public Dictionary<string, string> ExpectedHashes { get; set; } = []; public Dictionary<string, WrongAnswerFollowup> Replacement { get; set; } = []; }
sealed class RepairBackup
{
[BsonId, BsonRepresentation(BsonType.ObjectId)] public string? Id { get; set; }
public string BatchId { get; set; } = ""; public string ExternalId { get; set; } = "";
public string OriginalHash { get; set; } = ""; public string ReplacementHash { get; set; } = "";
public Dictionary<string, WrongAnswerFollowup> Original { get; set; } = []; public DateTime CreatedUtc { get; set; }
}
@@ -0,0 +1,4 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup><OutputType>Exe</OutputType><TargetFramework>net8.0</TargetFramework><ImplicitUsings>enable</ImplicitUsings><Nullable>enable</Nullable></PropertyGroup>
<ItemGroup><ProjectReference Include="..\..\api\QuestionsApi\QuestionsApi.csproj" /></ItemGroup>
</Project>