From 121e80b2ea77b2fb6a869427e74c7e1d0c879e62 Mon Sep 17 00:00:00 2001 From: hz Date: Mon, 20 Jul 2026 02:44:58 -0500 Subject: [PATCH] Prefer high-performing distractors --- api/QuestionsApi.Tests/ChoicePoolTests.cs | 35 ++++++++++ api/QuestionsApi/Models/ChoicePoolSelector.cs | 65 +++++++++++++++++++ api/QuestionsApi/Models/QuestionResponse.cs | 17 ++--- .../Services/ExamSimulationStore.cs | 16 ++--- scripts/autoload/block_session_service.gd | 29 ++++++++- tests/choice_pool_test.gd | 10 +++ 6 files changed, 152 insertions(+), 20 deletions(-) create mode 100644 api/QuestionsApi/Models/ChoicePoolSelector.cs diff --git a/api/QuestionsApi.Tests/ChoicePoolTests.cs b/api/QuestionsApi.Tests/ChoicePoolTests.cs index f5a8eef..8f80e14 100644 --- a/api/QuestionsApi.Tests/ChoicePoolTests.cs +++ b/api/QuestionsApi.Tests/ChoicePoolTests.cs @@ -35,6 +35,41 @@ public sealed class ChoicePoolTests Assert.Single(response.ReviewFollowups); } + [Fact] + public void HigherSelectionRatesIncreaseDistractorPresentationFrequency() + { + var question = BuildQuestion(9); + var rates = question.Choices.ToDictionary(choice => choice.Id, _ => 0); + rates["B"] = 100; + var random = new Random(12345); + var counts = question.Choices.ToDictionary(choice => choice.Id, _ => 0); + + for (var iteration = 0; iteration < 2000; iteration++) + { + foreach (var id in ChoicePoolSelector.SelectChoiceIds(question, rates, random)) + counts[id]++; + } + + Assert.All(question.Choices.Where(choice => choice.Id is not "A" and not "B"), + choice => Assert.True(counts["B"] > counts[choice.Id])); + Assert.Equal(2000, counts["A"]); + } + + [Fact] + public void MissingSelectionRatesStillAllowDifferentDistractorSets() + { + var question = BuildQuestion(9); + var random = new Random(9876); + var sets = Enumerable.Range(0, 40) + .Select(_ => string.Join(",", ChoicePoolSelector.SelectChoiceIds( + question, + new Dictionary(), + random).Order())) + .ToHashSet(); + + Assert.True(sets.Count > 1); + } + [Fact] public void ExamReviewContainsOnlyChoicesPresentedDuringTheExam() { diff --git a/api/QuestionsApi/Models/ChoicePoolSelector.cs b/api/QuestionsApi/Models/ChoicePoolSelector.cs new file mode 100644 index 0000000..e478d84 --- /dev/null +++ b/api/QuestionsApi/Models/ChoicePoolSelector.cs @@ -0,0 +1,65 @@ +namespace QuestionsApi.Models; + +public static class ChoicePoolSelector +{ + private const int PresentedChoiceCount = 5; + private const double MaximumRateMultiplier = 5.0; + + public static List SelectChoiceIds( + Question question, + IReadOnlyDictionary? liveSelectionPercentages = null, + Random? random = null) + { + random ??= Random.Shared; + var rates = liveSelectionPercentages is { Count: > 0 } + ? liveSelectionPercentages + : question.ChoiceSelectionPercentages; + var correct = question.Choices.FirstOrDefault(choice => choice.Id == question.CorrectChoiceId); + var remaining = question.Choices + .Where(choice => choice.Id != question.CorrectChoiceId) + .ToList(); + var selected = new List(); + + while (selected.Count < PresentedChoiceCount - 1 && remaining.Count > 0) + { + var totalWeight = remaining.Sum(choice => Weight(choice.Id, rates)); + var threshold = random.NextDouble() * totalWeight; + var cumulative = 0.0; + var selectedIndex = remaining.Count - 1; + for (var index = 0; index < remaining.Count; index++) + { + cumulative += Weight(remaining[index].Id, rates); + if (threshold < cumulative) + { + selectedIndex = index; + break; + } + } + selected.Add(remaining[selectedIndex]); + remaining.RemoveAt(selectedIndex); + } + + if (correct is not null) + selected.Add(correct); + else + selected = question.Choices.Take(PresentedChoiceCount).ToList(); + + Shuffle(selected, random); + return selected.Select(choice => choice.Id).ToList(); + } + + private static double Weight(string choiceId, IReadOnlyDictionary? rates) + { + var rate = Math.Clamp(rates?.GetValueOrDefault(choiceId, 0) ?? 0, 0, 100); + return 1.0 + MaximumRateMultiplier * rate / 100.0; + } + + private static void Shuffle(IList items, Random random) + { + for (var index = items.Count - 1; index > 0; index--) + { + var replacement = random.Next(index + 1); + (items[index], items[replacement]) = (items[replacement], items[index]); + } + } +} diff --git a/api/QuestionsApi/Models/QuestionResponse.cs b/api/QuestionsApi/Models/QuestionResponse.cs index 63590cd..f9240ec 100644 --- a/api/QuestionsApi/Models/QuestionResponse.cs +++ b/api/QuestionsApi/Models/QuestionResponse.cs @@ -33,7 +33,7 @@ public class QuestionResponse int questionElo = 1000, bool includeChoicePool = true) { - var choices = includeChoicePool ? question.Choices : SelectLegacyChoices(question); + var choices = includeChoicePool ? question.Choices : SelectLegacyChoices(question, liveChoiceSelectionPercentages); var choiceIds = choices.Select(choice => choice.Id).ToHashSet(StringComparer.Ordinal); var percentages = BuildChoiceSelectionPercentages(question, liveChoiceSelectionPercentages); return new QuestionResponse @@ -87,20 +87,17 @@ public class QuestionResponse return response; } - private static List SelectLegacyChoices(Question question) + private static List SelectLegacyChoices( + Question question, + IReadOnlyDictionary? liveChoiceSelectionPercentages) { if (question.Choices.Count <= 5) return question.Choices; - var correct = question.Choices.First(choice => choice.Id == question.CorrectChoiceId); - var distractors = question.Choices - .Where(choice => choice.Id != question.CorrectChoiceId) - .OrderBy(_ => Random.Shared.Next()) - .Take(4) - .Append(correct) - .OrderBy(_ => Random.Shared.Next()) + var selectedIds = ChoicePoolSelector.SelectChoiceIds(question, liveChoiceSelectionPercentages); + return selectedIds + .Select(id => question.Choices.First(choice => choice.Id == id)) .ToList(); - return distractors; } public static Dictionary BuildChoiceSelectionPercentages(Question question, Dictionary? liveChoiceSelectionPercentages) diff --git a/api/QuestionsApi/Services/ExamSimulationStore.cs b/api/QuestionsApi/Services/ExamSimulationStore.cs index 014da74..ec862fa 100644 --- a/api/QuestionsApi/Services/ExamSimulationStore.cs +++ b/api/QuestionsApi/Services/ExamSimulationStore.cs @@ -72,6 +72,7 @@ public class ExamSimulationStore var elos = await _ratings.GetQuestionElosAsync(bank.Select(question => question.ExternalId)); var targets = BuildTargets(); var selected = SelectForm(bank, seen, targets); + var choicePercentages = await _statistics.GetPercentagesByQuestionAsync(selected.Select(question => question.ExternalId)); var items = new List(); var position = 0; for (var block = 0; block < BlockSizes.Length; block++) @@ -84,7 +85,7 @@ public class ExamSimulationStore QuestionExternalId = question.ExternalId, BlockIndex = block, Position = offset, - PresentedChoiceIds = SelectChoices(question), + PresentedChoiceIds = SelectChoices(question, choicePercentages.GetValueOrDefault(question.ExternalId)), QuestionElo = elos.GetValueOrDefault(question.ExternalId, 1000), WasSeen = seen.Contains(question.ExternalId), BlueprintCategory = BlueprintCategory(question) @@ -115,12 +116,13 @@ public class ExamSimulationStore throw new InvalidOperationException($"At least {RequiredQuestions} published questions are required."); var targets = BuildTargets(); var selected = SelectForm(bank, [], targets); + var choicePercentages = await _statistics.GetPercentagesByQuestionAsync(selected.Select(question => question.ExternalId)); var items = selected.Select((question, index) => new ExamSimulationItem { QuestionExternalId = question.ExternalId, BlockIndex = BlockIndexForPosition(index), Position = PositionWithinBlock(index), - PresentedChoiceIds = SelectChoices(question), + PresentedChoiceIds = SelectChoices(question, choicePercentages.GetValueOrDefault(question.ExternalId)), QuestionElo = 1000, WasSeen = false, BlueprintCategory = BlueprintCategory(question) @@ -501,12 +503,10 @@ public class ExamSimulationStore Media = question.Media }; - private static List SelectChoices(Question question) - { - var distractors = question.Choices.Where(choice => choice.Id != question.CorrectChoiceId).OrderBy(_ => Random.Shared.Next()).Take(4).Select(choice => choice.Id).ToList(); - distractors.Add(question.CorrectChoiceId); - return distractors.OrderBy(_ => Random.Shared.Next()).ToList(); - } + private static List SelectChoices( + Question question, + IReadOnlyDictionary? liveChoiceSelectionPercentages) => + ChoicePoolSelector.SelectChoiceIds(question, liveChoiceSelectionPercentages); private static List SelectForm(List bank, HashSet seen, Dictionary targets) { diff --git a/scripts/autoload/block_session_service.gd b/scripts/autoload/block_session_service.gd index 5fcf109..70df3b9 100644 --- a/scripts/autoload/block_session_service.gd +++ b/scripts/autoload/block_session_service.gd @@ -546,6 +546,7 @@ func _prepare_question_choices(question: Dictionary) -> Dictionary: var all_choices: Array = question.get("choices", []).duplicate(true) var correct_id := str(question.get("correct_choice_id", "")) + var percentages: Dictionary = question.get("choice_selection_percentages", {}) var correct_choice: Dictionary = {} var distractors: Array = [] for choice in all_choices: @@ -556,8 +557,7 @@ func _prepare_question_choices(question: Dictionary) -> Dictionary: else: distractors.append(choice) - distractors.shuffle() - var presented: Array = distractors.slice(0, min(4, distractors.size())) + var presented: Array = _select_weighted_distractors(distractors, percentages, 4) if not correct_choice.is_empty(): presented.append(correct_choice) elif not all_choices.is_empty(): @@ -567,6 +567,31 @@ func _prepare_question_choices(question: Dictionary) -> Dictionary: return question +func _select_weighted_distractors(distractors: Array, percentages: Dictionary, count: int) -> Array: + var remaining: Array = distractors.duplicate(true) + var selected: Array = [] + while selected.size() < count and not remaining.is_empty(): + var total_weight := 0.0 + for choice in remaining: + total_weight += _distractor_weight(str(choice.get("id", "")), percentages) + var threshold := randf() * total_weight + var cumulative := 0.0 + var selected_index := remaining.size() - 1 + for index in range(remaining.size()): + cumulative += _distractor_weight(str(remaining[index].get("id", "")), percentages) + if threshold < cumulative: + selected_index = index + break + selected.append(remaining[selected_index]) + remaining.remove_at(selected_index) + return selected + + +func _distractor_weight(choice_id: String, percentages: Dictionary) -> float: + var selection_rate := clampf(float(percentages.get(choice_id, 0)), 0.0, 100.0) + return 1.0 + 5.0 * selection_rate / 100.0 + + func _build_core_question_pool() -> Array: var pool: Array = [] var profile := ProfileService.get_profile() diff --git a/tests/choice_pool_test.gd b/tests/choice_pool_test.gd index c421969..5356c3e 100644 --- a/tests/choice_pool_test.gd +++ b/tests/choice_pool_test.gd @@ -21,6 +21,16 @@ func _initialize() -> void: observed_sets[",".join(ids)] = true _assert(observed_sets.size() > 1, "future sessions should be able to present different distractors") + seed(12345) + var weighted_distractors: Array = question.get("choices", []).slice(1) + var weighted_counts := {"B": 0, "C": 0, "D": 0, "E": 0, "F": 0, "G": 0, "H": 0, "I": 0} + for _iteration in range(2000): + var selected: Array = service._select_weighted_distractors(weighted_distractors, {"B": 100}, 1) + var selected_id := str(selected[0].get("id", "")) + weighted_counts[selected_id] = int(weighted_counts.get(selected_id, 0)) + 1 + for low_rate_id in ["C", "D", "E", "F", "G", "H", "I"]: + _assert(weighted_counts["B"] > weighted_counts[low_rate_id], "higher-rate distractors should be preferred") + var legacy: Dictionary = service._prepare_question_choices(_question(5)) _check_presented(legacy) _assert(_ids(legacy.get("presented_choices", [])).size() == 5, "legacy five-choice questions must show all choices")