Prefer high-performing distractors

api/QuestionsApi.Tests/ChoicePoolTests.cs

@@ -35,6 +35,41 @@ public sealed class ChoicePoolTests
         Assert.Single(response.ReviewFollowups);
     }
 
+    [Fact]
+    public void HigherSelectionRatesIncreaseDistractorPresentationFrequency()
+    {
+        var question = BuildQuestion(9);
+        var rates = question.Choices.ToDictionary(choice => choice.Id, _ => 0);
+        rates["B"] = 100;
+        var random = new Random(12345);
+        var counts = question.Choices.ToDictionary(choice => choice.Id, _ => 0);
+
+        for (var iteration = 0; iteration < 2000; iteration++)
+        {
+            foreach (var id in ChoicePoolSelector.SelectChoiceIds(question, rates, random))
+                counts[id]++;
+        }
+
+        Assert.All(question.Choices.Where(choice => choice.Id is not "A" and not "B"),
+            choice => Assert.True(counts["B"] > counts[choice.Id]));
+        Assert.Equal(2000, counts["A"]);
+    }
+
+    [Fact]
+    public void MissingSelectionRatesStillAllowDifferentDistractorSets()
+    {
+        var question = BuildQuestion(9);
+        var random = new Random(9876);
+        var sets = Enumerable.Range(0, 40)
+            .Select(_ => string.Join(",", ChoicePoolSelector.SelectChoiceIds(
+                question,
+                new Dictionary<string, int>(),
+                random).Order()))
+            .ToHashSet();
+
+        Assert.True(sets.Count > 1);
+    }
+
     [Fact]
     public void ExamReviewContainsOnlyChoicesPresentedDuringTheExam()
     {

api/QuestionsApi/Models/ChoicePoolSelector.cs

@@ -0,0 +1,65 @@
+namespace QuestionsApi.Models;
+
+public static class ChoicePoolSelector
+{
+    private const int PresentedChoiceCount = 5;
+    private const double MaximumRateMultiplier = 5.0;
+
+    public static List<string> SelectChoiceIds(
+        Question question,
+        IReadOnlyDictionary<string, int>? liveSelectionPercentages = null,
+        Random? random = null)
+    {
+        random ??= Random.Shared;
+        var rates = liveSelectionPercentages is { Count: > 0 }
+            ? liveSelectionPercentages
+            : question.ChoiceSelectionPercentages;
+        var correct = question.Choices.FirstOrDefault(choice => choice.Id == question.CorrectChoiceId);
+        var remaining = question.Choices
+            .Where(choice => choice.Id != question.CorrectChoiceId)
+            .ToList();
+        var selected = new List<QuestionChoice>();
+
+        while (selected.Count < PresentedChoiceCount - 1 && remaining.Count > 0)
+        {
+            var totalWeight = remaining.Sum(choice => Weight(choice.Id, rates));
+            var threshold = random.NextDouble() * totalWeight;
+            var cumulative = 0.0;
+            var selectedIndex = remaining.Count - 1;
+            for (var index = 0; index < remaining.Count; index++)
+            {
+                cumulative += Weight(remaining[index].Id, rates);
+                if (threshold < cumulative)
+                {
+                    selectedIndex = index;
+                    break;
+                }
+            }
+            selected.Add(remaining[selectedIndex]);
+            remaining.RemoveAt(selectedIndex);
+        }
+
+        if (correct is not null)
+            selected.Add(correct);
+        else
+            selected = question.Choices.Take(PresentedChoiceCount).ToList();
+
+        Shuffle(selected, random);
+        return selected.Select(choice => choice.Id).ToList();
+    }
+
+    private static double Weight(string choiceId, IReadOnlyDictionary<string, int>? rates)
+    {
+        var rate = Math.Clamp(rates?.GetValueOrDefault(choiceId, 0) ?? 0, 0, 100);
+        return 1.0 + MaximumRateMultiplier * rate / 100.0;
+    }
+
+    private static void Shuffle<T>(IList<T> items, Random random)
+    {
+        for (var index = items.Count - 1; index > 0; index--)
+        {
+            var replacement = random.Next(index + 1);
+            (items[index], items[replacement]) = (items[replacement], items[index]);
+        }
+    }
+}

api/QuestionsApi/Models/QuestionResponse.cs

@@ -33,7 +33,7 @@ public class QuestionResponse
         int questionElo = 1000,
         bool includeChoicePool = true)
     {
-        var choices = includeChoicePool ? question.Choices : SelectLegacyChoices(question);
+        var choices = includeChoicePool ? question.Choices : SelectLegacyChoices(question, liveChoiceSelectionPercentages);
         var choiceIds = choices.Select(choice => choice.Id).ToHashSet(StringComparer.Ordinal);
         var percentages = BuildChoiceSelectionPercentages(question, liveChoiceSelectionPercentages);
         return new QuestionResponse
@@ -87,20 +87,17 @@ public class QuestionResponse
         return response;
     }
 
-    private static List<QuestionChoice> SelectLegacyChoices(Question question)
+    private static List<QuestionChoice> SelectLegacyChoices(
+        Question question,
+        IReadOnlyDictionary<string, int>? liveChoiceSelectionPercentages)
     {
         if (question.Choices.Count <= 5)
             return question.Choices;
 
-        var correct = question.Choices.First(choice => choice.Id == question.CorrectChoiceId);
-        var distractors = question.Choices
-            .Where(choice => choice.Id != question.CorrectChoiceId)
-            .OrderBy(_ => Random.Shared.Next())
-            .Take(4)
-            .Append(correct)
-            .OrderBy(_ => Random.Shared.Next())
+        var selectedIds = ChoicePoolSelector.SelectChoiceIds(question, liveChoiceSelectionPercentages);
+        return selectedIds
+            .Select(id => question.Choices.First(choice => choice.Id == id))
             .ToList();
-        return distractors;
     }
 
     public static Dictionary<string, int> BuildChoiceSelectionPercentages(Question question, Dictionary<string, int>? liveChoiceSelectionPercentages)

api/QuestionsApi/Services/ExamSimulationStore.cs

@@ -72,6 +72,7 @@ public class ExamSimulationStore
         var elos = await _ratings.GetQuestionElosAsync(bank.Select(question => question.ExternalId));
         var targets = BuildTargets();
         var selected = SelectForm(bank, seen, targets);
+        var choicePercentages = await _statistics.GetPercentagesByQuestionAsync(selected.Select(question => question.ExternalId));
         var items = new List<ExamSimulationItem>();
         var position = 0;
         for (var block = 0; block < BlockSizes.Length; block++)
@@ -84,7 +85,7 @@ public class ExamSimulationStore
                     QuestionExternalId = question.ExternalId,
                     BlockIndex = block,
                     Position = offset,
-                    PresentedChoiceIds = SelectChoices(question),
+                    PresentedChoiceIds = SelectChoices(question, choicePercentages.GetValueOrDefault(question.ExternalId)),
                     QuestionElo = elos.GetValueOrDefault(question.ExternalId, 1000),
                     WasSeen = seen.Contains(question.ExternalId),
                     BlueprintCategory = BlueprintCategory(question)
@@ -115,12 +116,13 @@ public class ExamSimulationStore
             throw new InvalidOperationException($"At least {RequiredQuestions} published questions are required.");
         var targets = BuildTargets();
         var selected = SelectForm(bank, [], targets);
+        var choicePercentages = await _statistics.GetPercentagesByQuestionAsync(selected.Select(question => question.ExternalId));
         var items = selected.Select((question, index) => new ExamSimulationItem
         {
             QuestionExternalId = question.ExternalId,
             BlockIndex = BlockIndexForPosition(index),
             Position = PositionWithinBlock(index),
-            PresentedChoiceIds = SelectChoices(question),
+            PresentedChoiceIds = SelectChoices(question, choicePercentages.GetValueOrDefault(question.ExternalId)),
             QuestionElo = 1000,
             WasSeen = false,
             BlueprintCategory = BlueprintCategory(question)
@@ -501,12 +503,10 @@ public class ExamSimulationStore
         Media = question.Media
     };
 
-    private static List<string> SelectChoices(Question question)
-    {
-        var distractors = question.Choices.Where(choice => choice.Id != question.CorrectChoiceId).OrderBy(_ => Random.Shared.Next()).Take(4).Select(choice => choice.Id).ToList();
-        distractors.Add(question.CorrectChoiceId);
-        return distractors.OrderBy(_ => Random.Shared.Next()).ToList();
-    }
+    private static List<string> SelectChoices(
+        Question question,
+        IReadOnlyDictionary<string, int>? liveChoiceSelectionPercentages) =>
+        ChoicePoolSelector.SelectChoiceIds(question, liveChoiceSelectionPercentages);
 
     private static List<Question> SelectForm(List<Question> bank, HashSet<string> seen, Dictionary<string, int> targets)
     {