Restore distractor-specific follow-up questions
This commit is contained in:
@@ -92,6 +92,33 @@ public sealed class ChoicePoolTests
|
||||
Assert.Contains("every incorrect choice", QuestionValidator.Validate(request));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("prompt")]
|
||||
[InlineData("teaching")]
|
||||
[InlineData("correct")]
|
||||
[InlineData("choices")]
|
||||
public void PublishedQuestionRejectsClonedDistractorFollowups(string duplicate)
|
||||
{
|
||||
var request = BuildRequest(5);
|
||||
var first = request.WrongAnswerFollowups["B"];
|
||||
var second = request.WrongAnswerFollowups["C"];
|
||||
if (duplicate == "prompt") second.Prompt = first.Prompt;
|
||||
if (duplicate == "teaching") second.TeachingPoint = first.TeachingPoint;
|
||||
if (duplicate == "correct") second.Choices.Single(x => x.Id == second.CorrectChoiceId).Text =
|
||||
first.Choices.Single(x => x.Id == first.CorrectChoiceId).Text;
|
||||
if (duplicate == "choices") second.Choices = first.Choices.Select(x => new QuestionChoice { Id=x.Id, Text=x.Text }).ToList();
|
||||
|
||||
Assert.Contains("different", QuestionValidator.Validate(request));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void PublishedQuestionRequiresThreeDistractorFollowupChoices()
|
||||
{
|
||||
var request = BuildRequest(5);
|
||||
request.WrongAnswerFollowups["B"].Choices.RemoveAt(2);
|
||||
Assert.Contains("exactly three", QuestionValidator.Validate(request));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void OptionalReviewFollowupsRequireValidAuthoredContent()
|
||||
{
|
||||
@@ -275,14 +302,15 @@ public sealed class ChoicePoolTests
|
||||
id => id,
|
||||
id => new WrongAnswerFollowup
|
||||
{
|
||||
Prompt = "Which finding makes this less appropriate?",
|
||||
Prompt = $"Which property of Choice {id} makes it less appropriate?",
|
||||
Choices =
|
||||
[
|
||||
new QuestionChoice { Id = "A", Text = "Finding one" },
|
||||
new QuestionChoice { Id = "B", Text = "Finding two" }
|
||||
new QuestionChoice { Id = "A", Text = $"Disqualifying property of Choice {id}" },
|
||||
new QuestionChoice { Id = "B", Text = $"Plausible property one for Choice {id}" },
|
||||
new QuestionChoice { Id = "C", Text = $"Plausible property two for Choice {id}" }
|
||||
],
|
||||
CorrectChoiceId = "A",
|
||||
TeachingPoint = "Teaching point"
|
||||
TeachingPoint = $"Choice {id} fails because of its specific disqualifying property."
|
||||
}),
|
||||
ChoiceSelectionPercentages = ids.ToDictionary(id => id, _ => 0),
|
||||
Discipline = "Medicine",
|
||||
|
||||
@@ -57,6 +57,14 @@ least one embedded authoritative verification reference. Legacy published
|
||||
questions without references remain readable but must add references before a
|
||||
future authoring update.
|
||||
|
||||
Wrong-answer follow-ups are distractor-specific remediation, not generic review
|
||||
questions. Each distractor must have a different three-choice quiz about the
|
||||
mechanism, indication, contraindication, timing, or other property that makes
|
||||
that option inappropriate in the vignette. Its teaching point must explain why
|
||||
that distractor fails without revealing the main correct answer. Publication
|
||||
validation rejects duplicated prompts, correct follow-up answers, choice sets,
|
||||
and teaching points within a question.
|
||||
|
||||
- `GET /readyz` verifies MongoDB connectivity for readiness checks.
|
||||
- `GET /metrics` exposes Prometheus-format service and reconciliation metrics.
|
||||
|
||||
|
||||
@@ -283,7 +283,8 @@ public sealed class DynamicQuestionGenerator
|
||||
IReadOnlyDictionary<string, string> values)
|
||||
{
|
||||
var result = new Dictionary<string, WrongAnswerFollowup>();
|
||||
foreach (var distractor in distractors)
|
||||
var distractorList = distractors.ToList();
|
||||
foreach (var distractor in distractorList)
|
||||
{
|
||||
if (templates.TryGetValue(distractor.Id, out var template))
|
||||
{
|
||||
@@ -297,16 +298,18 @@ public sealed class DynamicQuestionGenerator
|
||||
};
|
||||
continue;
|
||||
}
|
||||
var alternatives = distractorList.Where(value => value.Id != distractor.Id).Take(2).ToArray();
|
||||
result[distractor.Id] = new WrongAnswerFollowup
|
||||
{
|
||||
Prompt = "Which option best addresses the decisive finding in this case?",
|
||||
Prompt = $"Which limitation of option {distractor.Id} should have ruled it out in this case?",
|
||||
Choices =
|
||||
[
|
||||
new QuestionChoice { Id = correct.Id, Text = Render(correct.TextTemplate, values) },
|
||||
new QuestionChoice { Id = distractor.Id, Text = Render(distractor.TextTemplate, values) }
|
||||
new QuestionChoice { Id = "A", Text = Render(distractor.ExplanationTemplate, values) },
|
||||
new QuestionChoice { Id = "B", Text = Render(alternatives[0].ExplanationTemplate, values) },
|
||||
new QuestionChoice { Id = "C", Text = Render(alternatives[1].ExplanationTemplate, values) }
|
||||
],
|
||||
CorrectChoiceId = correct.Id,
|
||||
TeachingPoint = $"{Render(distractor.ExplanationTemplate, values)} {Render(correct.ExplanationTemplate, values)}"
|
||||
CorrectChoiceId = "A",
|
||||
TeachingPoint = Render(distractor.ExplanationTemplate, values)
|
||||
};
|
||||
}
|
||||
return result;
|
||||
|
||||
@@ -49,6 +49,21 @@ public static class QuestionValidator
|
||||
if (!entry.Value.Choices.Any(choice => choice.Id == entry.Value.CorrectChoiceId))
|
||||
return "wrongAnswerFollowups correctChoiceId must match a follow-up choice id";
|
||||
}
|
||||
if (req.IsPublished)
|
||||
{
|
||||
if (req.WrongAnswerFollowups.Any(entry => entry.Value.Choices.Count != 3))
|
||||
return "published wrongAnswerFollowups require exactly three choices per distractor";
|
||||
if (!AllDistinct(req.WrongAnswerFollowups.Values.Select(value => value.Prompt)))
|
||||
return "published wrongAnswerFollowups require a different prompt for every distractor";
|
||||
if (!AllDistinct(req.WrongAnswerFollowups.Values.Select(value => value.TeachingPoint)))
|
||||
return "published wrongAnswerFollowups require a different teachingPoint for every distractor";
|
||||
if (!AllDistinct(req.WrongAnswerFollowups.Values.Select(value =>
|
||||
value.Choices.Single(choice => choice.Id == value.CorrectChoiceId).Text)))
|
||||
return "published wrongAnswerFollowups require a different correct follow-up answer for every distractor";
|
||||
if (!AllDistinct(req.WrongAnswerFollowups.Values.Select(value => string.Join("|",
|
||||
value.Choices.Select(choice => Normalize(choice.Text)).OrderBy(text => text, StringComparer.Ordinal)))))
|
||||
return "published wrongAnswerFollowups require a different choice set for every distractor";
|
||||
}
|
||||
|
||||
if (req.ReviewFollowups is null) return "reviewFollowups must be an array";
|
||||
if (req.ReviewFollowups.Count > 10) return "questions support at most ten review follow-ups";
|
||||
@@ -75,6 +90,16 @@ public static class QuestionValidator
|
||||
return null;
|
||||
}
|
||||
|
||||
private static bool AllDistinct(IEnumerable<string> values)
|
||||
{
|
||||
var normalized = values.Select(Normalize).ToList();
|
||||
return normalized.All(value => value.Length > 0)
|
||||
&& normalized.Distinct(StringComparer.Ordinal).Count() == normalized.Count;
|
||||
}
|
||||
|
||||
private static string Normalize(string value) => string.Join(' ', value.Trim().ToLowerInvariant()
|
||||
.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries));
|
||||
|
||||
public static string? ValidateLabs(IReadOnlyCollection<QuestionLab>? labs, bool requireComplete)
|
||||
{
|
||||
if (labs is null)
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -29,7 +29,6 @@ var simulation_total_blocks := 12
|
||||
var simulation_deadline_unix := 0.0
|
||||
var review_first_submission_recorded := false
|
||||
var core_block_size := DEFAULT_CORE_BLOCK_SIZE
|
||||
var show_review_followups := true
|
||||
|
||||
const CORE_MIN_TARGET_SECONDS := 45.0
|
||||
const CORE_MAX_TARGET_SECONDS := 150.0
|
||||
@@ -40,12 +39,10 @@ const CORE_BLOCK_SIZE_MAX := 40
|
||||
const SETTINGS_PATH := "user://settings.cfg"
|
||||
const SETTINGS_SECTION := "session"
|
||||
const SETTINGS_KEY_CORE_BLOCK_SIZE := "core_block_size"
|
||||
const SETTINGS_KEY_SHOW_REVIEW_FOLLOWUPS := "show_review_followups"
|
||||
|
||||
|
||||
func _ready() -> void:
|
||||
_load_core_block_size()
|
||||
_load_show_review_followups()
|
||||
|
||||
|
||||
func set_core_block_size(value: int) -> void:
|
||||
@@ -60,17 +57,6 @@ func set_core_block_size(value: int) -> void:
|
||||
push_warning("Unable to save core question block size preference.")
|
||||
|
||||
|
||||
func set_show_review_followups(value: bool) -> void:
|
||||
if show_review_followups == value:
|
||||
return
|
||||
show_review_followups = value
|
||||
var config := ConfigFile.new()
|
||||
config.load(SETTINGS_PATH)
|
||||
config.set_value(SETTINGS_SECTION, SETTINGS_KEY_SHOW_REVIEW_FOLLOWUPS, show_review_followups)
|
||||
if config.save(SETTINGS_PATH) != OK:
|
||||
push_warning("Unable to save follow-up question preference.")
|
||||
|
||||
|
||||
func _load_core_block_size() -> void:
|
||||
var config := ConfigFile.new()
|
||||
if config.load(SETTINGS_PATH) == OK:
|
||||
@@ -79,12 +65,6 @@ func _load_core_block_size() -> void:
|
||||
CORE_BLOCK_SIZE_MIN, CORE_BLOCK_SIZE_MAX)
|
||||
|
||||
|
||||
func _load_show_review_followups() -> void:
|
||||
var config := ConfigFile.new()
|
||||
if config.load(SETTINGS_PATH) == OK:
|
||||
show_review_followups = bool(config.get_value(SETTINGS_SECTION, SETTINGS_KEY_SHOW_REVIEW_FOLLOWUPS, true))
|
||||
|
||||
|
||||
func start_dynamic_question(question: Dictionary) -> bool:
|
||||
if question.is_empty():
|
||||
return false
|
||||
@@ -284,6 +264,7 @@ func answer_current(choice_id: String, eliminated: Array, flagged: bool, is_subm
|
||||
"question": question.duplicate(true),
|
||||
"selected_choice_id": choice_id,
|
||||
"is_correct": choice_id == str(question.get("correct_choice_id", "")),
|
||||
"first_attempt_is_correct": choice_id == str(question.get("correct_choice_id", "")) and (session_kind != "core" or not core_current_had_wrong_attempt),
|
||||
"eliminated_choice_ids": eliminated.duplicate(),
|
||||
"flagged": flagged,
|
||||
"elapsed_seconds": get_current_elapsed_seconds(),
|
||||
@@ -311,6 +292,7 @@ func submit_core_attempt(choice_id: String, eliminated: Array, flagged: bool) ->
|
||||
"question": question.duplicate(true),
|
||||
"selected_choice_id": choice_id,
|
||||
"is_correct": choice_id == str(question.get("correct_choice_id", "")),
|
||||
"first_attempt_is_correct": choice_id == str(question.get("correct_choice_id", "")) and not core_current_had_wrong_attempt,
|
||||
"eliminated_choice_ids": eliminated.duplicate(),
|
||||
"flagged": flagged,
|
||||
"elapsed_seconds": get_current_elapsed_seconds(),
|
||||
@@ -343,26 +325,50 @@ func submit_core_attempt(choice_id: String, eliminated: Array, flagged: bool) ->
|
||||
}
|
||||
|
||||
|
||||
func submit_bookmark_attempt(choice_id: String, eliminated: Array, flagged: bool) -> Dictionary:
|
||||
func submit_tutor_attempt(choice_id: String, eliminated: Array, flagged: bool, had_wrong_attempt: bool) -> Dictionary:
|
||||
var question := get_current_question()
|
||||
if question.is_empty() or mode != "Tutor" or session_kind not in ["block", "dynamic"]:
|
||||
return {}
|
||||
var current_correct := choice_id == str(question.get("correct_choice_id", ""))
|
||||
var answer := {
|
||||
"question": question.duplicate(true),
|
||||
"selected_choice_id": choice_id,
|
||||
"is_correct": current_correct and not had_wrong_attempt,
|
||||
"first_attempt_is_correct": current_correct and not had_wrong_attempt,
|
||||
"resolved_correct": current_correct,
|
||||
"eliminated_choice_ids": eliminated.duplicate(),
|
||||
"flagged": flagged,
|
||||
"elapsed_seconds": get_current_elapsed_seconds(),
|
||||
"target_seconds": 0.0
|
||||
}
|
||||
if current_correct:
|
||||
answers[current_index] = answer
|
||||
session_changed.emit()
|
||||
return {"answer": answer, "is_correct": current_correct, "final": current_correct}
|
||||
|
||||
|
||||
func submit_bookmark_attempt(choice_id: String, eliminated: Array, flagged: bool, had_wrong_attempt := false) -> Dictionary:
|
||||
var question := get_current_question()
|
||||
if session_kind != "bookmark" or question.is_empty():
|
||||
return {}
|
||||
var answer := {
|
||||
"question": question.duplicate(true),
|
||||
"selected_choice_id": choice_id,
|
||||
"is_correct": choice_id == str(question.get("correct_choice_id", "")),
|
||||
"is_correct": choice_id == str(question.get("correct_choice_id", "")) and not had_wrong_attempt,
|
||||
"first_attempt_is_correct": choice_id == str(question.get("correct_choice_id", "")) and not had_wrong_attempt,
|
||||
"eliminated_choice_ids": eliminated.duplicate(),
|
||||
"flagged": flagged,
|
||||
"elapsed_seconds": get_current_elapsed_seconds(),
|
||||
"target_seconds": 0.0
|
||||
}
|
||||
if bool(answer.is_correct):
|
||||
var current_correct := choice_id == str(question.get("correct_choice_id", ""))
|
||||
if current_correct:
|
||||
answers[current_index] = answer
|
||||
session_changed.emit()
|
||||
return {"answer": answer, "is_correct": bool(answer.is_correct), "final": bool(answer.is_correct)}
|
||||
return {"answer": answer, "is_correct": current_correct, "final": current_correct}
|
||||
|
||||
|
||||
func submit_review_attempt(choice_id: String, eliminated: Array, flagged: bool) -> Dictionary:
|
||||
func submit_review_attempt(choice_id: String, eliminated: Array, flagged: bool, had_wrong_attempt := false) -> Dictionary:
|
||||
var question := get_current_question()
|
||||
if session_kind != "review" or question.is_empty():
|
||||
return {}
|
||||
@@ -370,20 +376,22 @@ func submit_review_attempt(choice_id: String, eliminated: Array, flagged: bool)
|
||||
var answer := {
|
||||
"question": question.duplicate(true),
|
||||
"selected_choice_id": choice_id,
|
||||
"is_correct": choice_id == str(question.get("correct_choice_id", "")),
|
||||
"is_correct": choice_id == str(question.get("correct_choice_id", "")) and not had_wrong_attempt,
|
||||
"first_attempt_is_correct": choice_id == str(question.get("correct_choice_id", "")) and not had_wrong_attempt,
|
||||
"eliminated_choice_ids": eliminated.duplicate(),
|
||||
"flagged": flagged,
|
||||
"elapsed_seconds": get_current_elapsed_seconds(),
|
||||
"target_seconds": 0.0
|
||||
}
|
||||
review_first_submission_recorded = true
|
||||
if bool(answer.is_correct):
|
||||
var current_correct := choice_id == str(question.get("correct_choice_id", ""))
|
||||
if current_correct:
|
||||
answers[current_index] = answer
|
||||
session_changed.emit()
|
||||
return {
|
||||
"answer": answer,
|
||||
"is_correct": bool(answer.is_correct),
|
||||
"final": bool(answer.is_correct),
|
||||
"is_correct": current_correct,
|
||||
"final": current_correct,
|
||||
"first_submission": first_submission
|
||||
}
|
||||
|
||||
|
||||
@@ -26,7 +26,7 @@ func calculate_block_rewards(answers: Array, mode: String, system_name: String)
|
||||
var ending_streak := running_streak
|
||||
|
||||
for answer in answers:
|
||||
if bool(answer.get("is_correct", false)):
|
||||
if bool(answer.get("first_attempt_is_correct", answer.get("is_correct", false))):
|
||||
correct += 1
|
||||
ending_streak += 1
|
||||
longest_streak = max(longest_streak, ending_streak)
|
||||
|
||||
@@ -1613,31 +1613,32 @@ func _submit_or_next() -> void:
|
||||
if selected_choice == "":
|
||||
return
|
||||
var question_id_for_attempt := str(BlockSessionService.get_current_question().get("id", ""))
|
||||
if BlockSessionService.session_kind == "dynamic":
|
||||
var elapsed := int(floor(BlockSessionService.get_current_elapsed_seconds()))
|
||||
DynamicQuestionService.submit_attempt(selected_choice, elapsed)
|
||||
current_answer = BlockSessionService.answer_current(selected_choice, eliminated, false)
|
||||
_show_answer_review()
|
||||
return
|
||||
if BlockSessionService.session_kind in ["core", "bookmark", "review"]:
|
||||
var immediate_feedback := BlockSessionService.session_kind in ["core", "bookmark", "review", "dynamic"] or (BlockSessionService.session_kind == "block" and BlockSessionService.mode == "Tutor")
|
||||
if immediate_feedback:
|
||||
var attempted_choice := selected_choice
|
||||
var had_wrong_attempt := not locked_wrong_choice_ids.is_empty()
|
||||
var met_time_limit := true
|
||||
if BlockSessionService.session_kind == "core":
|
||||
met_time_limit = BlockSessionService.get_current_elapsed_seconds() <= BlockSessionService.core_current_target_seconds
|
||||
if BlockSessionService.session_kind in ["core", "block"] and not had_wrong_attempt:
|
||||
if BlockSessionService.session_kind == "core":
|
||||
met_time_limit = BlockSessionService.get_current_elapsed_seconds() <= BlockSessionService.core_current_target_seconds
|
||||
QuestionBankService.record_first_attempt(
|
||||
question_id_for_attempt,
|
||||
attempted_choice,
|
||||
_presented_choice_ids(),
|
||||
int(floor(BlockSessionService.get_core_question_total_elapsed_seconds())),
|
||||
int(floor(BlockSessionService.get_core_question_total_elapsed_seconds())) if BlockSessionService.session_kind == "core" else int(floor(BlockSessionService.get_current_elapsed_seconds())),
|
||||
met_time_limit
|
||||
)
|
||||
elif BlockSessionService.session_kind == "dynamic" and not had_wrong_attempt:
|
||||
DynamicQuestionService.submit_attempt(attempted_choice, int(floor(BlockSessionService.get_current_elapsed_seconds())))
|
||||
var result: Dictionary
|
||||
if BlockSessionService.session_kind == "core":
|
||||
result = BlockSessionService.submit_core_attempt(selected_choice, eliminated, flagged)
|
||||
elif BlockSessionService.session_kind == "bookmark":
|
||||
result = BlockSessionService.submit_bookmark_attempt(selected_choice, eliminated, flagged)
|
||||
result = BlockSessionService.submit_bookmark_attempt(selected_choice, eliminated, flagged, had_wrong_attempt)
|
||||
elif BlockSessionService.session_kind == "review":
|
||||
result = BlockSessionService.submit_review_attempt(selected_choice, eliminated, flagged, had_wrong_attempt)
|
||||
else:
|
||||
result = BlockSessionService.submit_review_attempt(selected_choice, eliminated, flagged)
|
||||
result = BlockSessionService.submit_tutor_attempt(selected_choice, eliminated, flagged, had_wrong_attempt)
|
||||
if result.is_empty():
|
||||
return
|
||||
var answer: Dictionary = result.get("answer", {})
|
||||
@@ -1817,7 +1818,6 @@ func _dock_correct_review_panel(correct_id: String) -> void:
|
||||
|
||||
func _render_answer_review_sections() -> void:
|
||||
_render_teaching_cards()
|
||||
_render_review_followups()
|
||||
_render_unselected_distractor_explanations()
|
||||
|
||||
|
||||
@@ -1893,12 +1893,9 @@ func _render_review_followups() -> void:
|
||||
if review_followups_content == null:
|
||||
return
|
||||
_clear_container_children(review_followups_content)
|
||||
if BlockSessionService.session_kind == "simulation":
|
||||
review_followups_section.visible = false
|
||||
return
|
||||
if not BlockSessionService.show_review_followups:
|
||||
review_followups_section.visible = false
|
||||
return
|
||||
review_followup_states.clear()
|
||||
review_followups_section.visible = false
|
||||
return
|
||||
var question := BlockSessionService.get_current_question()
|
||||
var authored: Array = question.get("review_followups", [])
|
||||
if review_followup_states.size() != authored.size():
|
||||
@@ -3774,14 +3771,14 @@ func _followup_is_dark() -> bool:
|
||||
|
||||
|
||||
func _choice_review_explanation(choice_id: String, state: Dictionary) -> String:
|
||||
var teaching_point := str(state.get("teaching_point", "")).strip_edges()
|
||||
if teaching_point != "":
|
||||
return teaching_point
|
||||
var question := BlockSessionService.get_current_question()
|
||||
var explanations: Dictionary = question.get("choice_explanations", {})
|
||||
var explanation := str(explanations.get(choice_id, "")).strip_edges()
|
||||
if explanation != "":
|
||||
return explanation
|
||||
var teaching_point := str(state.get("teaching_point", "")).strip_edges()
|
||||
if teaching_point != "":
|
||||
return teaching_point
|
||||
if bool(state.get("is_main_correct", false)):
|
||||
return str(question.get("explanation", "Review the correct answer explanation."))
|
||||
return "Review the stem finding that makes this option less appropriate."
|
||||
|
||||
@@ -9,7 +9,6 @@ const CORE_BLOCK_SIZE_STEP := 5
|
||||
var main: Control
|
||||
var theme_toggle: CheckButton
|
||||
var sound_toggle: CheckButton
|
||||
var followups_toggle: CheckButton
|
||||
var question_scale_value_label: Label
|
||||
var ccs_scale_value_label: Label
|
||||
var core_block_size_value_label: Label
|
||||
@@ -73,13 +72,6 @@ func _ready() -> void:
|
||||
sound_toggle.toggled.connect(UISoundService.set_sounds_enabled)
|
||||
content.add_child(sound_toggle)
|
||||
|
||||
followups_toggle = CheckButton.new()
|
||||
followups_toggle.text = "Follow-up questions"
|
||||
followups_toggle.tooltip_text = "Optional reinforcement questions shown after you answer"
|
||||
followups_toggle.custom_minimum_size = Vector2(0, 52)
|
||||
followups_toggle.toggled.connect(BlockSessionService.set_show_review_followups)
|
||||
content.add_child(followups_toggle)
|
||||
|
||||
content.add_child(_build_text_scale_row("Question bank text size", "question"))
|
||||
content.add_child(_build_text_scale_row("CCS case text size", "ccs"))
|
||||
content.add_child(_build_core_block_size_row())
|
||||
@@ -117,7 +109,6 @@ func on_show(payload: Variant = null) -> void:
|
||||
back_to_session_button.visible = not return_screen.is_empty()
|
||||
theme_toggle.set_pressed_no_signal(UIThemeService.is_dark_mode())
|
||||
sound_toggle.set_pressed_no_signal(UISoundService.are_sounds_enabled())
|
||||
followups_toggle.set_pressed_no_signal(BlockSessionService.show_review_followups)
|
||||
_refresh_text_scale_labels()
|
||||
|
||||
|
||||
|
||||
@@ -223,21 +223,24 @@ UpsertQuestionRequest Build(Seed seed)
|
||||
id => id,
|
||||
id => id == "A"
|
||||
? seed.Explanation
|
||||
: $"This choice does not address the decisive finding: {seed.Discriminator}. {seed.Pitfall2}.");
|
||||
var wrongFollowups = ids.Skip(1).ToDictionary(
|
||||
id => id,
|
||||
id => new WrongAnswerFollowup
|
||||
: $"{choices.Single(choice => choice.Id == id).Text} does not address the decisive finding: {seed.Discriminator}. {seed.Pitfall2}.");
|
||||
var distractorIds = ids.Skip(1).ToArray();
|
||||
var wrongFollowups = distractorIds.ToDictionary(id => id, id =>
|
||||
{
|
||||
var index = Array.IndexOf(distractorIds, id);
|
||||
var alternatives = new[] { distractorIds[(index + 1) % distractorIds.Length], distractorIds[(index + 2) % distractorIds.Length] };
|
||||
return new WrongAnswerFollowup
|
||||
{
|
||||
Prompt = $"In this same vignette, which finding most strongly favors {seed.Correct} over {choices.Single(c => c.Id == id).Text}?",
|
||||
Prompt = $"Which limitation of option {id} should have ruled it out in this case?",
|
||||
Choices =
|
||||
[
|
||||
new() { Id = "1", Text = seed.Discriminator },
|
||||
new() { Id = "2", Text = seed.Pitfall1 },
|
||||
new() { Id = "3", Text = "The patient's age by itself" }
|
||||
new() { Id = "1", Text = explanations[id] },
|
||||
new() { Id = "2", Text = explanations[alternatives[0]] },
|
||||
new() { Id = "3", Text = explanations[alternatives[1]] }
|
||||
],
|
||||
CorrectChoiceId = "1",
|
||||
TeachingPoint = seed.Takeaway
|
||||
});
|
||||
CorrectChoiceId = "1", TeachingPoint = explanations[id]
|
||||
};
|
||||
});
|
||||
|
||||
return new UpsertQuestionRequest
|
||||
{
|
||||
|
||||
@@ -88,10 +88,6 @@ string? ValidateAuthoredQuestion(UpsertQuestionRequest question)
|
||||
{
|
||||
if (followup.Choices.Count != 3 || followup.Choices.Select(choice => choice.Id).SequenceEqual(["A", "B", "C"]) is false)
|
||||
return "wrong-answer follow-ups must use ordered A-C choices";
|
||||
if (followup.Choices.Any(choice =>
|
||||
!question.Stem.Contains(choice.Text, StringComparison.OrdinalIgnoreCase)
|
||||
&& !question.Labs.Any(lab => $"{lab.Name} {lab.Value}".Contains(choice.Text, StringComparison.OrdinalIgnoreCase))))
|
||||
return "wrong-answer follow-up choices must be traceable to the stem or labs";
|
||||
}
|
||||
return null;
|
||||
}
|
||||
@@ -107,20 +103,23 @@ UpsertQuestionRequest Build(Seed seed)
|
||||
id => id == "A"
|
||||
? $"Correct. {seed.Explanation}"
|
||||
: $"Incorrect. {choices.Single(choice => choice.Id == id).Text} does not adequately address the decisive finding: {seed.KeyFact}. {seed.Takeaway}");
|
||||
var wrongFollowups = ids.Skip(1).ToDictionary(
|
||||
id => id,
|
||||
id => new WrongAnswerFollowup
|
||||
var distractorIds = ids.Skip(1).ToArray();
|
||||
var wrongFollowups = distractorIds.ToDictionary(id => id, id =>
|
||||
{
|
||||
var index = Array.IndexOf(distractorIds, id);
|
||||
var alternatives = new[] { distractorIds[(index + 1) % distractorIds.Length], distractorIds[(index + 2) % distractorIds.Length] };
|
||||
return new WrongAnswerFollowup
|
||||
{
|
||||
Prompt = $"Which finding in this vignette most strongly supports {seed.Answers[0]}?",
|
||||
Prompt = $"Which limitation of option {id} should have ruled it out in this case?",
|
||||
Choices =
|
||||
[
|
||||
new() { Id = "A", Text = seed.KeyFact },
|
||||
new() { Id = "B", Text = seed.ContextFact },
|
||||
new() { Id = "C", Text = seed.NeutralFact }
|
||||
new() { Id = "A", Text = choiceExplanations[id] },
|
||||
new() { Id = "B", Text = choiceExplanations[alternatives[0]] },
|
||||
new() { Id = "C", Text = choiceExplanations[alternatives[1]] }
|
||||
],
|
||||
CorrectChoiceId = "A",
|
||||
TeachingPoint = seed.Takeaway
|
||||
});
|
||||
CorrectChoiceId = "A", TeachingPoint = choiceExplanations[id]
|
||||
};
|
||||
});
|
||||
|
||||
return new UpsertQuestionRequest
|
||||
{
|
||||
|
||||
@@ -61,11 +61,22 @@ UpsertQuestionRequest Build(Seed s)
|
||||
var ids = Enumerable.Range(0, s.Answers.Length).Select(i => ((char)('A' + i)).ToString()).ToArray();
|
||||
var choices = ids.Zip(s.Answers, (id, text) => new QuestionChoice { Id = id, Text = text }).ToList();
|
||||
var explanations = ids.ToDictionary(id => id, id => id == "A" ? $"Correct. {s.Rationale}" : $"Incorrect. {choices.Single(c => c.Id == id).Text} does not fit the decisive finding: {s.KeyFact}. {s.Takeaway}");
|
||||
var wrong = ids.Skip(1).ToDictionary(id => id, _ => new WrongAnswerFollowup
|
||||
var distractorIds = ids.Skip(1).ToArray();
|
||||
var wrong = distractorIds.ToDictionary(id => id, id =>
|
||||
{
|
||||
Prompt = $"Which finding most strongly supports {s.Answers[0]}?",
|
||||
Choices = [new() { Id = "A", Text = s.KeyFact }, new() { Id = "B", Text = s.ContextFact }, new() { Id = "C", Text = s.NeutralFact }],
|
||||
CorrectChoiceId = "A", TeachingPoint = s.Takeaway
|
||||
var index = Array.IndexOf(distractorIds, id);
|
||||
var alternatives = new[] { distractorIds[(index + 1) % distractorIds.Length], distractorIds[(index + 2) % distractorIds.Length] };
|
||||
return new WrongAnswerFollowup
|
||||
{
|
||||
Prompt = $"Which limitation of option {id} should have ruled it out in this case?",
|
||||
Choices =
|
||||
[
|
||||
new() { Id = "A", Text = explanations[id] },
|
||||
new() { Id = "B", Text = explanations[alternatives[0]] },
|
||||
new() { Id = "C", Text = explanations[alternatives[1]] }
|
||||
],
|
||||
CorrectChoiceId = "A", TeachingPoint = explanations[id]
|
||||
};
|
||||
});
|
||||
var reference = References.For(s.Reference);
|
||||
return new UpsertQuestionRequest
|
||||
|
||||
@@ -0,0 +1,172 @@
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Bson.Serialization.Attributes;
|
||||
using MongoDB.Driver;
|
||||
using QuestionsApi.Models;
|
||||
|
||||
const string defaultConnection = "mongodb://192.168.86.50:27017";
|
||||
var options = args.Select((value, index) => (value, index)).Where(x => x.value.StartsWith("--"))
|
||||
.ToDictionary(x => x.value, x => x.index + 1 < args.Length && !args[x.index + 1].StartsWith("--") ? args[x.index + 1] : "true");
|
||||
var mode = options.GetValueOrDefault("--mode", "audit").ToLowerInvariant();
|
||||
var connection = options.GetValueOrDefault("--connection", Environment.GetEnvironmentVariable("STEP3_MONGODB_CONNECTION_STRING") ?? defaultConnection);
|
||||
var databases = options.GetValueOrDefault("--databases", "ranaze_step3_uat,ranaze_step3_prd").Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
|
||||
var manifestPath = options.GetValueOrDefault("--manifest", "content/repairs/wrong-answer-followups-20260801.json");
|
||||
var batchId = options.GetValueOrDefault("--batch", "wrong-answer-followups-20260801");
|
||||
var client = new MongoClient(connection);
|
||||
var json = new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase };
|
||||
|
||||
if (mode is "audit" or "prepare")
|
||||
{
|
||||
var inventory = new Dictionary<string, List<Question>>();
|
||||
foreach (var database in databases)
|
||||
{
|
||||
var questions = await client.GetDatabase(database).GetCollection<Question>("Questions")
|
||||
.Find(x => x.IsPublished).SortBy(x => x.ExternalId).ToListAsync();
|
||||
inventory[database] = questions;
|
||||
var broken = questions.Select(q => (Question: q, Issues: Issues(q))).Where(x => x.Issues.Count > 0).ToList();
|
||||
Console.WriteLine($"{database}: published={questions.Count}, compliant={questions.Count - broken.Count}, repair={broken.Count}");
|
||||
foreach (var item in broken) Console.WriteLine($" {item.Question.ExternalId}: {string.Join("; ", item.Issues)}");
|
||||
}
|
||||
if (mode == "audit") return;
|
||||
|
||||
var entries = inventory.SelectMany(db => db.Value.Select(question => (Database: db.Key, Question: question)))
|
||||
.Where(x => Issues(x.Question).Count > 0).GroupBy(x => x.Question.ExternalId, StringComparer.Ordinal).Select(group =>
|
||||
{
|
||||
var source = group.OrderBy(x => Array.IndexOf(databases, x.Database)).First().Question;
|
||||
var replacement = Build(source);
|
||||
var original = source.WrongAnswerFollowups;
|
||||
source.WrongAnswerFollowups = replacement;
|
||||
var replacementIssues = Issues(source);
|
||||
source.WrongAnswerFollowups = original;
|
||||
if (replacementIssues.Count > 0)
|
||||
throw new InvalidOperationException($"generated repair for {source.ExternalId} is invalid: {string.Join("; ", replacementIssues)}");
|
||||
return new RepairEntry
|
||||
{
|
||||
ExternalId = source.ExternalId,
|
||||
ExpectedHashes = group.ToDictionary(x => x.Database, x => Hash(x.Question.WrongAnswerFollowups)),
|
||||
Replacement = replacement
|
||||
};
|
||||
}).OrderBy(x => x.ExternalId).ToList();
|
||||
var manifest = new RepairManifest { BatchId = batchId, CreatedUtc = DateTime.UtcNow, Entries = entries };
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(manifestPath) ?? ".");
|
||||
await File.WriteAllTextAsync(manifestPath, JsonSerializer.Serialize(manifest, json));
|
||||
Console.WriteLine($"Prepared {entries.Count} repair entries at {manifestPath}.");
|
||||
return;
|
||||
}
|
||||
|
||||
var loaded = JsonSerializer.Deserialize<RepairManifest>(await File.ReadAllTextAsync(manifestPath), json)
|
||||
?? throw new InvalidOperationException("repair manifest is invalid");
|
||||
if (loaded.BatchId != batchId) throw new InvalidOperationException("manifest batch does not match --batch");
|
||||
|
||||
foreach (var database in databases)
|
||||
{
|
||||
var db = client.GetDatabase(database);
|
||||
var questions = db.GetCollection<Question>("Questions");
|
||||
var backups = db.GetCollection<RepairBackup>("WrongAnswerFollowupRepairBackups");
|
||||
await backups.Indexes.CreateOneAsync(new CreateIndexModel<RepairBackup>(
|
||||
Builders<RepairBackup>.IndexKeys.Ascending(x => x.BatchId).Ascending(x => x.ExternalId), new CreateIndexOptions { Unique = true }));
|
||||
var changed = 0;
|
||||
foreach (var entry in loaded.Entries)
|
||||
{
|
||||
var question = await questions.Find(x => x.ExternalId == entry.ExternalId).FirstOrDefaultAsync();
|
||||
if (question is null || !question.IsPublished || !entry.ExpectedHashes.TryGetValue(database, out var expected)) continue;
|
||||
if (mode == "rollback")
|
||||
{
|
||||
var backup = await backups.Find(x => x.BatchId == batchId && x.ExternalId == entry.ExternalId).FirstOrDefaultAsync();
|
||||
if (backup is null) continue;
|
||||
await questions.UpdateOneAsync(x => x.Id == question.Id, Builders<Question>.Update
|
||||
.Set(x => x.WrongAnswerFollowups, backup.Original).Set(x => x.UpdatedUtc, DateTime.UtcNow));
|
||||
changed++;
|
||||
continue;
|
||||
}
|
||||
if (mode != "apply") throw new ArgumentException("--mode must be audit, prepare, apply, or rollback");
|
||||
var actual = Hash(question.WrongAnswerFollowups);
|
||||
var replacementHash = Hash(entry.Replacement);
|
||||
if (actual == replacementHash) continue;
|
||||
if (actual != expected) throw new InvalidOperationException($"{database}/{entry.ExternalId} changed after audit; expected {expected}, found {actual}");
|
||||
try
|
||||
{
|
||||
await backups.InsertOneAsync(new RepairBackup { Id = ObjectId.GenerateNewId().ToString(), BatchId = batchId,
|
||||
ExternalId = entry.ExternalId, OriginalHash = actual, ReplacementHash = replacementHash,
|
||||
Original = question.WrongAnswerFollowups, CreatedUtc = DateTime.UtcNow });
|
||||
}
|
||||
catch (MongoWriteException exception) when (exception.WriteError?.Category == ServerErrorCategory.DuplicateKey) { }
|
||||
var update = await questions.UpdateOneAsync(x => x.Id == question.Id && x.UpdatedUtc == question.UpdatedUtc,
|
||||
Builders<Question>.Update.Set(x => x.WrongAnswerFollowups, entry.Replacement).Set(x => x.UpdatedUtc, DateTime.UtcNow));
|
||||
if (update.ModifiedCount != 1) throw new InvalidOperationException($"{database}/{entry.ExternalId} changed during repair");
|
||||
changed++;
|
||||
}
|
||||
Console.WriteLine($"{database}: {mode} changed {changed} question(s).");
|
||||
}
|
||||
|
||||
static List<string> Issues(Question question)
|
||||
{
|
||||
var ids = question.Choices.Select(x => x.Id).Where(x => x != question.CorrectChoiceId).ToHashSet(StringComparer.Ordinal);
|
||||
var issues = new List<string>();
|
||||
if (!ids.SetEquals(question.WrongAnswerFollowups.Keys)) issues.Add("follow-up keys do not match distractors");
|
||||
var values = question.WrongAnswerFollowups.Where(x => ids.Contains(x.Key)).Select(x => x.Value).ToList();
|
||||
if (values.Any(x => x.Choices.Count != 3)) issues.Add("follow-up does not have exactly three choices");
|
||||
Duplicate(values.Select(x => x.Prompt), "duplicate prompts", issues);
|
||||
Duplicate(values.Select(x => x.TeachingPoint), "duplicate teaching points", issues);
|
||||
Duplicate(values.Select(x => x.Choices.FirstOrDefault(c => c.Id == x.CorrectChoiceId)?.Text ?? ""), "duplicate correct follow-up answers", issues);
|
||||
Duplicate(values.Select(x => string.Join("|", x.Choices.Select(c => Normalize(c.Text)).Order())), "duplicate choice sets", issues);
|
||||
if (values.Any(x => x.Choices.Count == 0 || x.Choices.All(c => c.Id != x.CorrectChoiceId))) issues.Add("invalid correct follow-up choice");
|
||||
return issues.Distinct().ToList();
|
||||
}
|
||||
|
||||
static void Duplicate(IEnumerable<string> values, string issue, List<string> issues)
|
||||
{
|
||||
var normalized = values.Select(Normalize).ToList();
|
||||
if (normalized.Any(string.IsNullOrWhiteSpace) || normalized.Distinct(StringComparer.Ordinal).Count() != normalized.Count) issues.Add(issue);
|
||||
}
|
||||
|
||||
static Dictionary<string, WrongAnswerFollowup> Build(Question question)
|
||||
{
|
||||
var distractors = question.Choices.Where(x => x.Id != question.CorrectChoiceId).ToList();
|
||||
var rationales = distractors.ToDictionary(x => x.Id, x => Rationale(question, x.Id));
|
||||
return distractors.ToDictionary(choice => choice.Id, choice =>
|
||||
{
|
||||
var alternatives = distractors.Where(x => x.Id != choice.Id).Take(2).ToArray();
|
||||
return new WrongAnswerFollowup
|
||||
{
|
||||
Prompt = $"Which feature of option {choice.Id} should have ruled it out in this case?",
|
||||
Choices =
|
||||
[
|
||||
new() { Id = "A", Text = rationales[choice.Id] },
|
||||
new() { Id = "B", Text = Unique(rationales[alternatives[0].Id], alternatives[0].Id) },
|
||||
new() { Id = "C", Text = Unique(rationales[alternatives[1].Id], alternatives[1].Id) }
|
||||
],
|
||||
CorrectChoiceId = "A", TeachingPoint = rationales[choice.Id]
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
static string Rationale(Question question, string id)
|
||||
{
|
||||
var value = question.ChoiceExplanations.GetValueOrDefault(id, question.WrongAnswerFollowups.GetValueOrDefault(id)?.TeachingPoint ?? "").Trim();
|
||||
foreach (var prefix in new[] { "Incorrect.", "Incorrect:", "Wrong.", "Wrong:" })
|
||||
if (value.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)) value = value[prefix.Length..].Trim();
|
||||
var choice = question.Choices.FirstOrDefault(x => x.Id == id)?.Text.Trim() ?? $"Option {id}";
|
||||
if (string.IsNullOrWhiteSpace(value)) return $"{choice} does not fit the decisive clinical finding.";
|
||||
return value.Contains(choice, StringComparison.OrdinalIgnoreCase) ? value : $"{choice} is inappropriate because {char.ToLowerInvariant(value[0])}{value[1..]}";
|
||||
}
|
||||
|
||||
static string Unique(string value, string id) => $"Option {id}: {value}";
|
||||
static string Normalize(string value) => string.Join(' ', (value ?? "").Trim().ToLowerInvariant().Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries));
|
||||
static string Hash(Dictionary<string, WrongAnswerFollowup> value)
|
||||
{
|
||||
var canonical = value.OrderBy(x => x.Key).ToDictionary(x => x.Key, x => x.Value);
|
||||
return Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(JsonSerializer.Serialize(canonical)))).ToLowerInvariant();
|
||||
}
|
||||
|
||||
sealed class RepairManifest { public string BatchId { get; set; } = ""; public DateTime CreatedUtc { get; set; } public List<RepairEntry> Entries { get; set; } = []; }
|
||||
sealed class RepairEntry { public string ExternalId { get; set; } = ""; public Dictionary<string, string> ExpectedHashes { get; set; } = []; public Dictionary<string, WrongAnswerFollowup> Replacement { get; set; } = []; }
|
||||
sealed class RepairBackup
|
||||
{
|
||||
[BsonId, BsonRepresentation(BsonType.ObjectId)] public string? Id { get; set; }
|
||||
public string BatchId { get; set; } = ""; public string ExternalId { get; set; } = "";
|
||||
public string OriginalHash { get; set; } = ""; public string ReplacementHash { get; set; } = "";
|
||||
public Dictionary<string, WrongAnswerFollowup> Original { get; set; } = []; public DateTime CreatedUtc { get; set; }
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<PropertyGroup><OutputType>Exe</OutputType><TargetFramework>net8.0</TargetFramework><ImplicitUsings>enable</ImplicitUsings><Nullable>enable</Nullable></PropertyGroup>
|
||||
<ItemGroup><ProjectReference Include="..\..\api\QuestionsApi\QuestionsApi.csproj" /></ItemGroup>
|
||||
</Project>
|
||||
Reference in New Issue
Block a user