Prefer high-performing distractors

api/QuestionsApi.Tests/ChoicePoolTests.cs

@@ -35,6 +35,41 @@ public sealed class ChoicePoolTests
         Assert.Single(response.ReviewFollowups);
     }
 
+    [Fact]
+    public void HigherSelectionRatesIncreaseDistractorPresentationFrequency()
+    {
+        var question = BuildQuestion(9);
+        var rates = question.Choices.ToDictionary(choice => choice.Id, _ => 0);
+        rates["B"] = 100;
+        var random = new Random(12345);
+        var counts = question.Choices.ToDictionary(choice => choice.Id, _ => 0);
+
+        for (var iteration = 0; iteration < 2000; iteration++)
+        {
+            foreach (var id in ChoicePoolSelector.SelectChoiceIds(question, rates, random))
+                counts[id]++;
+        }
+
+        Assert.All(question.Choices.Where(choice => choice.Id is not "A" and not "B"),
+            choice => Assert.True(counts["B"] > counts[choice.Id]));
+        Assert.Equal(2000, counts["A"]);
+    }
+
+    [Fact]
+    public void MissingSelectionRatesStillAllowDifferentDistractorSets()
+    {
+        var question = BuildQuestion(9);
+        var random = new Random(9876);
+        var sets = Enumerable.Range(0, 40)
+            .Select(_ => string.Join(",", ChoicePoolSelector.SelectChoiceIds(
+                question,
+                new Dictionary<string, int>(),
+                random).Order()))
+            .ToHashSet();
+
+        Assert.True(sets.Count > 1);
+    }
+
     [Fact]
     public void ExamReviewContainsOnlyChoicesPresentedDuringTheExam()
     {

api/QuestionsApi/Models/ChoicePoolSelector.cs

@@ -0,0 +1,65 @@
+namespace QuestionsApi.Models;
+
+public static class ChoicePoolSelector
+{
+    private const int PresentedChoiceCount = 5;
+    private const double MaximumRateMultiplier = 5.0;
+
+    public static List<string> SelectChoiceIds(
+        Question question,
+        IReadOnlyDictionary<string, int>? liveSelectionPercentages = null,
+        Random? random = null)
+    {
+        random ??= Random.Shared;
+        var rates = liveSelectionPercentages is { Count: > 0 }
+            ? liveSelectionPercentages
+            : question.ChoiceSelectionPercentages;
+        var correct = question.Choices.FirstOrDefault(choice => choice.Id == question.CorrectChoiceId);
+        var remaining = question.Choices
+            .Where(choice => choice.Id != question.CorrectChoiceId)
+            .ToList();
+        var selected = new List<QuestionChoice>();
+
+        while (selected.Count < PresentedChoiceCount - 1 && remaining.Count > 0)
+        {
+            var totalWeight = remaining.Sum(choice => Weight(choice.Id, rates));
+            var threshold = random.NextDouble() * totalWeight;
+            var cumulative = 0.0;
+            var selectedIndex = remaining.Count - 1;
+            for (var index = 0; index < remaining.Count; index++)
+            {
+                cumulative += Weight(remaining[index].Id, rates);
+                if (threshold < cumulative)
+                {
+                    selectedIndex = index;
+                    break;
+                }
+            }
+            selected.Add(remaining[selectedIndex]);
+            remaining.RemoveAt(selectedIndex);
+        }
+
+        if (correct is not null)
+            selected.Add(correct);
+        else
+            selected = question.Choices.Take(PresentedChoiceCount).ToList();
+
+        Shuffle(selected, random);
+        return selected.Select(choice => choice.Id).ToList();
+    }
+
+    private static double Weight(string choiceId, IReadOnlyDictionary<string, int>? rates)
+    {
+        var rate = Math.Clamp(rates?.GetValueOrDefault(choiceId, 0) ?? 0, 0, 100);
+        return 1.0 + MaximumRateMultiplier * rate / 100.0;
+    }
+
+    private static void Shuffle<T>(IList<T> items, Random random)
+    {
+        for (var index = items.Count - 1; index > 0; index--)
+        {
+            var replacement = random.Next(index + 1);
+            (items[index], items[replacement]) = (items[replacement], items[index]);
+        }
+    }
+}

api/QuestionsApi/Models/QuestionResponse.cs

@@ -33,7 +33,7 @@ public class QuestionResponse
         int questionElo = 1000,
         bool includeChoicePool = true)
     {
-        var choices = includeChoicePool ? question.Choices : SelectLegacyChoices(question);
+        var choices = includeChoicePool ? question.Choices : SelectLegacyChoices(question, liveChoiceSelectionPercentages);
         var choiceIds = choices.Select(choice => choice.Id).ToHashSet(StringComparer.Ordinal);
         var percentages = BuildChoiceSelectionPercentages(question, liveChoiceSelectionPercentages);
         return new QuestionResponse
@@ -87,20 +87,17 @@ public class QuestionResponse
         return response;
     }
 
-    private static List<QuestionChoice> SelectLegacyChoices(Question question)
+    private static List<QuestionChoice> SelectLegacyChoices(
+        Question question,
+        IReadOnlyDictionary<string, int>? liveChoiceSelectionPercentages)
     {
         if (question.Choices.Count <= 5)
             return question.Choices;
 
-        var correct = question.Choices.First(choice => choice.Id == question.CorrectChoiceId);
+        var selectedIds = ChoicePoolSelector.SelectChoiceIds(question, liveChoiceSelectionPercentages);
-        var distractors = question.Choices
+        return selectedIds
-            .Where(choice => choice.Id != question.CorrectChoiceId)
+            .Select(id => question.Choices.First(choice => choice.Id == id))
-            .OrderBy(_ => Random.Shared.Next())
-            .Take(4)
-            .Append(correct)
-            .OrderBy(_ => Random.Shared.Next())
             .ToList();
-        return distractors;
     }
 
     public static Dictionary<string, int> BuildChoiceSelectionPercentages(Question question, Dictionary<string, int>? liveChoiceSelectionPercentages)

api/QuestionsApi/Services/ExamSimulationStore.cs

@@ -72,6 +72,7 @@ public class ExamSimulationStore
         var elos = await _ratings.GetQuestionElosAsync(bank.Select(question => question.ExternalId));
         var targets = BuildTargets();
         var selected = SelectForm(bank, seen, targets);
+        var choicePercentages = await _statistics.GetPercentagesByQuestionAsync(selected.Select(question => question.ExternalId));
         var items = new List<ExamSimulationItem>();
         var position = 0;
         for (var block = 0; block < BlockSizes.Length; block++)
@@ -84,7 +85,7 @@ public class ExamSimulationStore
                     QuestionExternalId = question.ExternalId,
                     BlockIndex = block,
                     Position = offset,
-                    PresentedChoiceIds = SelectChoices(question),
+                    PresentedChoiceIds = SelectChoices(question, choicePercentages.GetValueOrDefault(question.ExternalId)),
                     QuestionElo = elos.GetValueOrDefault(question.ExternalId, 1000),
                     WasSeen = seen.Contains(question.ExternalId),
                     BlueprintCategory = BlueprintCategory(question)
@@ -115,12 +116,13 @@ public class ExamSimulationStore
             throw new InvalidOperationException($"At least {RequiredQuestions} published questions are required.");
         var targets = BuildTargets();
         var selected = SelectForm(bank, [], targets);
+        var choicePercentages = await _statistics.GetPercentagesByQuestionAsync(selected.Select(question => question.ExternalId));
         var items = selected.Select((question, index) => new ExamSimulationItem
         {
             QuestionExternalId = question.ExternalId,
             BlockIndex = BlockIndexForPosition(index),
             Position = PositionWithinBlock(index),
-            PresentedChoiceIds = SelectChoices(question),
+            PresentedChoiceIds = SelectChoices(question, choicePercentages.GetValueOrDefault(question.ExternalId)),
             QuestionElo = 1000,
             WasSeen = false,
             BlueprintCategory = BlueprintCategory(question)
@@ -501,12 +503,10 @@ public class ExamSimulationStore
         Media = question.Media
     };
 
-    private static List<string> SelectChoices(Question question)
+    private static List<string> SelectChoices(
-    {
+        Question question,
-        var distractors = question.Choices.Where(choice => choice.Id != question.CorrectChoiceId).OrderBy(_ => Random.Shared.Next()).Take(4).Select(choice => choice.Id).ToList();
+        IReadOnlyDictionary<string, int>? liveChoiceSelectionPercentages) =>
-        distractors.Add(question.CorrectChoiceId);
+        ChoicePoolSelector.SelectChoiceIds(question, liveChoiceSelectionPercentages);
-        return distractors.OrderBy(_ => Random.Shared.Next()).ToList();
-    }
 
     private static List<Question> SelectForm(List<Question> bank, HashSet<string> seen, Dictionary<string, int> targets)
     {

scripts/autoload/block_session_service.gd

@@ -546,6 +546,7 @@ func _prepare_question_choices(question: Dictionary) -> Dictionary:
 
 	var all_choices: Array = question.get("choices", []).duplicate(true)
 	var correct_id := str(question.get("correct_choice_id", ""))
+	var percentages: Dictionary = question.get("choice_selection_percentages", {})
 	var correct_choice: Dictionary = {}
 	var distractors: Array = []
 	for choice in all_choices:
@@ -556,8 +557,7 @@ func _prepare_question_choices(question: Dictionary) -> Dictionary:
 		else:
 			distractors.append(choice)
 
-	distractors.shuffle()
+	var presented: Array = _select_weighted_distractors(distractors, percentages, 4)
-	var presented: Array = distractors.slice(0, min(4, distractors.size()))
 	if not correct_choice.is_empty():
 		presented.append(correct_choice)
 	elif not all_choices.is_empty():
@@ -567,6 +567,31 @@ func _prepare_question_choices(question: Dictionary) -> Dictionary:
 	return question
 
 
+func _select_weighted_distractors(distractors: Array, percentages: Dictionary, count: int) -> Array:
+	var remaining: Array = distractors.duplicate(true)
+	var selected: Array = []
+	while selected.size() < count and not remaining.is_empty():
+		var total_weight := 0.0
+		for choice in remaining:
+			total_weight += _distractor_weight(str(choice.get("id", "")), percentages)
+		var threshold := randf() * total_weight
+		var cumulative := 0.0
+		var selected_index := remaining.size() - 1
+		for index in range(remaining.size()):
+			cumulative += _distractor_weight(str(remaining[index].get("id", "")), percentages)
+			if threshold < cumulative:
+				selected_index = index
+				break
+		selected.append(remaining[selected_index])
+		remaining.remove_at(selected_index)
+	return selected
+
+
+func _distractor_weight(choice_id: String, percentages: Dictionary) -> float:
+	var selection_rate := clampf(float(percentages.get(choice_id, 0)), 0.0, 100.0)
+	return 1.0 + 5.0 * selection_rate / 100.0
+
+
 func _build_core_question_pool() -> Array:
 	var pool: Array = []
 	var profile := ProfileService.get_profile()

tests/choice_pool_test.gd

@@ -21,6 +21,16 @@ func _initialize() -> void:
 		observed_sets[",".join(ids)] = true
 	_assert(observed_sets.size() > 1, "future sessions should be able to present different distractors")
 
+	seed(12345)
+	var weighted_distractors: Array = question.get("choices", []).slice(1)
+	var weighted_counts := {"B": 0, "C": 0, "D": 0, "E": 0, "F": 0, "G": 0, "H": 0, "I": 0}
+	for _iteration in range(2000):
+		var selected: Array = service._select_weighted_distractors(weighted_distractors, {"B": 100}, 1)
+		var selected_id := str(selected[0].get("id", ""))
+		weighted_counts[selected_id] = int(weighted_counts.get(selected_id, 0)) + 1
+	for low_rate_id in ["C", "D", "E", "F", "G", "H", "I"]:
+		_assert(weighted_counts["B"] > weighted_counts[low_rate_id], "higher-rate distractors should be preferred")
+
 	var legacy: Dictionary = service._prepare_question_choices(_question(5))
 	_check_presented(legacy)
 	_assert(_ids(legacy.get("presented_choices", [])).size() == 5, "legacy five-choice questions must show all choices")