Add full-length Step 3 Day 1 simulation

2026-07-14 16:54:11 -05:00
parent 8aaf7d61e9
commit 206a5c0124
19 changed files with 1520 additions and 32 deletions
@@ -0,0 +1,100 @@
+using QuestionsApi.Models;
+
+namespace QuestionsApi.Services;
+
+public static class ExamSimulationScoring
+{
+    public const string ModelVersion = "provisional-1";
+    private const double PriorMean = 1000.0;
+    private const double PriorSd = 400.0;
+
+    public static ExamSimulationScore Calculate(
+        IReadOnlyList<ExamSimulationItem> items,
+        IReadOnlyDictionary<string, ExamSimulationResponse> responses,
+        IReadOnlyDictionary<string, Question> questions)
+    {
+        var ability = EstimateAbility(items, responses, questions);
+        var predicted = Math.Clamp((int)Math.Round(227.0 + (ability - 1000.0) * 15.0 / 200.0), 1, 300);
+        var information = 1.0 / (PriorSd * PriorSd);
+        var correct = 0;
+        var seen = 0;
+        var categories = new Dictionary<string, ExamSimulationCategoryScore>();
+
+        foreach (var item in items)
+        {
+            if (!questions.TryGetValue(item.QuestionExternalId, out var question))
+                continue;
+            responses.TryGetValue(item.QuestionExternalId, out var response);
+            var isCorrect = response?.ChoiceId == question.CorrectChoiceId;
+            if (isCorrect) correct++;
+            if (item.WasSeen) seen++;
+            var weight = item.WasSeen ? 0.5 : 1.0;
+            var probability = Expected(ability, item.QuestionElo);
+            information += weight * probability * (1.0 - probability) * Math.Pow(Math.Log(10.0) / 400.0, 2);
+            if (!categories.TryGetValue(item.BlueprintCategory, out var category))
+            {
+                category = new ExamSimulationCategoryScore();
+                categories[item.BlueprintCategory] = category;
+            }
+            category.Total++;
+            if (isCorrect) category.Correct++;
+        }
+
+        var abilitySe = Math.Sqrt(1.0 / information);
+        var scoreSe = abilitySe * 15.0 / 200.0;
+        var combinedSe = Math.Sqrt(7.0 * 7.0 + 15.0 * 15.0 + scoreSe * scoreSe);
+        var range = (int)Math.Ceiling(combinedSe);
+        var passProbability = NormalCdf((predicted - 200.0) / combinedSe);
+        return new ExamSimulationScore
+        {
+            PredictedScore = predicted,
+            RangeLow = Math.Clamp(predicted - range, 1, 300),
+            RangeHigh = Math.Clamp(predicted + range, 1, 300),
+            PassProbability = Math.Round(passProbability, 4),
+            Correct = correct,
+            Total = items.Count,
+            SeenCount = seen,
+            AbilityElo = Math.Round(ability, 1),
+            Categories = categories
+        };
+    }
+
+    private static double EstimateAbility(
+        IReadOnlyList<ExamSimulationItem> items,
+        IReadOnlyDictionary<string, ExamSimulationResponse> responses,
+        IReadOnlyDictionary<string, Question> questions)
+    {
+        var low = 100.0;
+        var high = 1900.0;
+        for (var iteration = 0; iteration < 80; iteration++)
+        {
+            var ability = (low + high) / 2.0;
+            var derivative = -(ability - PriorMean) / (PriorSd * PriorSd);
+            foreach (var item in items)
+            {
+                if (!questions.TryGetValue(item.QuestionExternalId, out var question)) continue;
+                responses.TryGetValue(item.QuestionExternalId, out var response);
+                var actual = response?.ChoiceId == question.CorrectChoiceId ? 1.0 : 0.0;
+                var weight = item.WasSeen ? 0.5 : 1.0;
+                derivative += weight * (actual - Expected(ability, item.QuestionElo)) * Math.Log(10.0) / 400.0;
+            }
+            if (derivative > 0) low = ability; else high = ability;
+        }
+        return (low + high) / 2.0;
+    }
+
+    private static double Expected(double ability, int difficulty) =>
+        1.0 / (1.0 + Math.Pow(10.0, (difficulty - ability) / 400.0));
+
+    private static double NormalCdf(double value) => 0.5 * (1.0 + Erf(value / Math.Sqrt(2.0)));
+
+    private static double Erf(double value)
+    {
+        var sign = value < 0 ? -1.0 : 1.0;
+        value = Math.Abs(value);
+        const double p = 0.3275911;
+        var t = 1.0 / (1.0 + p * value);
+        var y = 1.0 - (((((1.061405429 * t - 1.453152027) * t) + 1.421413741) * t - 0.284496736) * t + 0.254829592) * t * Math.Exp(-value * value);
+        return sign * y;
+    }
+}