Fix exam simulation score prediction

api/QuestionsApi.Tests/ExamSimulationTests.cs

@@ -44,8 +44,104 @@ public sealed class ExamSimulationTests
         Assert.Equal(0, failing.Correct);
     }
 
+    [Theory]
+    [InlineData(0, 50)]
+    [InlineData(20, 100)]
+    [InlineData(60, 200)]
+    [InlineData(100, 300)]
+    public void NeutralDifficultyUsesTransparentScoreAnchors(int correctCount, int expectedScore)
+    {
+        var items = BuildItems(100);
+        var questions = BuildQuestions(100);
+        var responses = items.Take(correctCount).ToDictionary(
+            item => item.QuestionExternalId,
+            item => new ExamSimulationResponse { QuestionExternalId = item.QuestionExternalId, ChoiceId = "A" });
+
+        var score = ExamSimulationScoring.Calculate(items, responses, questions);
+
+        Assert.Equal(expectedScore, score.PredictedScore);
+        Assert.Equal(correctCount / 100.0, score.RawAccuracy, 4);
+        Assert.Equal(correctCount / 100.0, score.DifficultyAdjustedAccuracy, 4);
+    }
+
     [Fact]
-    public void SeenItemsAreReportedAndCarryLessScoringWeight()
+    public void CorrectHarderQuestionsProduceAHigherAdjustedPrediction()
+    {
+        var questions = BuildQuestions(10);
+        var hardCorrect = BuildItems(10);
+        var easyCorrect = BuildItems(10);
+        for (var index = 0; index < 10; index++)
+        {
+            hardCorrect[index].QuestionDifficulty = index < 5 ? 5 : 1;
+            easyCorrect[index].QuestionDifficulty = index < 5 ? 1 : 5;
+        }
+        var responses = hardCorrect.Take(5).ToDictionary(
+            item => item.QuestionExternalId,
+            item => new ExamSimulationResponse { QuestionExternalId = item.QuestionExternalId, ChoiceId = "A" });
+
+        var hardScore = ExamSimulationScoring.Calculate(hardCorrect, responses, questions);
+        var easyScore = ExamSimulationScoring.Calculate(easyCorrect, responses, questions);
+
+        Assert.Equal(hardScore.RawAccuracy, easyScore.RawAccuracy);
+        Assert.True(hardScore.DifficultyAdjustedAccuracy > easyScore.DifficultyAdjustedAccuracy);
+        Assert.True(hardScore.PredictedScore > easyScore.PredictedScore);
+    }
+
+    [Fact]
+    public void EmpiricalQuestionEloDoesNotAffectPrediction()
+    {
+        var questions = BuildQuestions(20);
+        var lowElos = BuildItems(20);
+        var highElos = BuildItems(20);
+        foreach (var item in lowElos) item.QuestionElo = 100;
+        foreach (var item in highElos) item.QuestionElo = 1900;
+        var responses = lowElos.Take(10).ToDictionary(
+            item => item.QuestionExternalId,
+            item => new ExamSimulationResponse { QuestionExternalId = item.QuestionExternalId, ChoiceId = "A" });
+
+        var lowEloScore = ExamSimulationScoring.Calculate(lowElos, responses, questions);
+        var highEloScore = ExamSimulationScoring.Calculate(highElos, responses, questions);
+
+        Assert.Equal(lowEloScore.PredictedScore, highEloScore.PredictedScore);
+        Assert.Equal(lowEloScore.PassProbability, highEloScore.PassProbability);
+    }
+
+    [Fact]
+    public void SkippedAndUnansweredQuestionsCountAsIncorrect()
+    {
+        var items = BuildItems(10);
+        var questions = BuildQuestions(10);
+        var responses = new Dictionary<string, ExamSimulationResponse>
+        {
+            [items[0].QuestionExternalId] = new() { QuestionExternalId = items[0].QuestionExternalId, ChoiceId = "A" },
+            [items[1].QuestionExternalId] = new() { QuestionExternalId = items[1].QuestionExternalId, ChoiceId = "" }
+        };
+
+        var score = ExamSimulationScoring.Calculate(items, responses, questions);
+
+        Assert.Equal(1, score.Correct);
+        Assert.Equal(10, score.Total);
+        Assert.Equal(0.1, score.RawAccuracy, 4);
+    }
+
+    [Fact]
+    public void LowAccuracyFullLengthSimulationIsNotInflated()
+    {
+        var items = BuildItems(232);
+        var questions = BuildQuestions(232);
+        var responses = items.Take(49).ToDictionary(
+            item => item.QuestionExternalId,
+            item => new ExamSimulationResponse { QuestionExternalId = item.QuestionExternalId, ChoiceId = "A" });
+
+        var score = ExamSimulationScoring.Calculate(items, responses, questions);
+
+        Assert.Equal(103, score.PredictedScore);
+        Assert.True(score.PassProbability < 0.001);
+        Assert.True(score.RangeHigh < 130);
+    }
+
+    [Fact]
+    public void SeenItemsAreReportedButDoNotChangeScoring()
     {
         var questions = BuildQuestions(20);
         var unseen = BuildItems(20);
@@ -59,7 +155,9 @@ public sealed class ExamSimulationTests
 
         Assert.Equal(0, unseenScore.SeenCount);
         Assert.Equal(20, seenScore.SeenCount);
-        Assert.True(unseenScore.AbilityElo > seenScore.AbilityElo);
+        Assert.Equal(unseenScore.PredictedScore, seenScore.PredictedScore);
+        Assert.Null(unseenScore.AbilityElo);
+        Assert.Null(seenScore.AbilityElo);
     }
 
     [Theory]
@@ -126,6 +224,7 @@ public sealed class ExamSimulationTests
         {
             QuestionExternalId = $"question-{index}",
             QuestionElo = 1000,
+            QuestionDifficulty = 3,
             WasSeen = wasSeen,
             BlueprintCategory = "Cardiovascular"
         }).ToList();
@@ -133,5 +232,5 @@ public sealed class ExamSimulationTests
     private static Dictionary<string, Question> BuildQuestions(int count) =>
         Enumerable.Range(0, count).ToDictionary(
             index => $"question-{index}",
-            index => new Question { ExternalId = $"question-{index}", CorrectChoiceId = "A" });
+            index => new Question { ExternalId = $"question-{index}", CorrectChoiceId = "A", Difficulty = 3 });
 }