hz/step3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix exam simulation score prediction

Browse Source
This commit is contained in:
hz
2026-07-30 16:00:29 -05:00
parent ae6b37eb7d
commit 778da32e73
9 changed files with 282 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/QuestionsApi/Services/ExamSimulationScoring.cs
+47 -43
View File
@@ -4,33 +4,35 @@ namespace QuestionsApi.Services;
public static class ExamSimulationScoring
{
public const string ModelVersion = "provisional-1";
private const double PriorMean = 1000.0;
private const double PriorSd = 400.0;
public const string ModelVersion = "provisional-2";
private const double PassingAccuracy = 0.60;
private const double ScoreIntercept = 50.0;
private const double ScoreScale = 250.0;
public static ExamSimulationScore Calculate(
IReadOnlyList<ExamSimulationItem> items,
IReadOnlyDictionary<string, ExamSimulationResponse> responses,
IReadOnlyDictionary<string, Question> questions)
{
var ability = EstimateAbility(items, responses, questions);
var predicted = Math.Clamp((int)Math.Round(227.0 + (ability - 1000.0) * 15.0 / 200.0), 1, 300);
var information = 1.0 / (PriorSd * PriorSd);
var correct = 0;
var seen = 0;
var weightedCorrect = 0.0;
var weightSum = 0.0;
var squaredWeightSum = 0.0;
var categories = new Dictionary<string, ExamSimulationCategoryScore>();
foreach (var item in items)
{
if (!questions.TryGetValue(item.QuestionExternalId, out var question))
continue;
questions.TryGetValue(item.QuestionExternalId, out var question);
responses.TryGetValue(item.QuestionExternalId, out var response);
var isCorrect = response?.ChoiceId == question.CorrectChoiceId;
var isCorrect = question is not null && response?.ChoiceId == question.CorrectChoiceId;
if (isCorrect) correct++;
if (item.WasSeen) seen++;
var weight = item.WasSeen ? 0.5 : 1.0;
var probability = Expected(ability, item.QuestionElo);
information += weight * probability * (1.0 - probability) * Math.Pow(Math.Log(10.0) / 400.0, 2);
var difficulty = ResolveDifficulty(item, question);
var weight = DifficultyWeight(difficulty);
weightSum += weight;
squaredWeightSum += weight * weight;
if (isCorrect) weightedCorrect += weight;
if (!categories.TryGetValue(item.BlueprintCategory, out var category))
{
category = new ExamSimulationCategoryScore();
@@ -40,51 +42,53 @@ public static class ExamSimulationScoring
if (isCorrect) category.Correct++;
}
var abilitySe = Math.Sqrt(1.0 / information);
var scoreSe = abilitySe * 15.0 / 200.0;
var combinedSe = Math.Sqrt(7.0 * 7.0 + 15.0 * 15.0 + scoreSe * scoreSe);
var range = (int)Math.Ceiling(combinedSe);
var passProbability = NormalCdf((predicted - 200.0) / combinedSe);
var rawAccuracy = items.Count == 0 ? 0.0 : (double)correct / items.Count;
var adjustedAccuracy = weightSum == 0.0 ? 0.0 : weightedCorrect / weightSum;
var predicted = ToScore(adjustedAccuracy);
var effectiveSampleSize = squaredWeightSum == 0.0 ? 0.0 : weightSum * weightSum / squaredWeightSum;
var accuracySe = effectiveSampleSize == 0.0
? 0.0
: Math.Sqrt(adjustedAccuracy * (1.0 - adjustedAccuracy) / effectiveSampleSize);
var accuracyMargin = 1.96 * accuracySe;
var rangeLow = ToScore(Math.Clamp(adjustedAccuracy - accuracyMargin, 0.0, 1.0));
var rangeHigh = ToScore(Math.Clamp(adjustedAccuracy + accuracyMargin, 0.0, 1.0));
var passProbability = accuracySe == 0.0
? adjustedAccuracy >= PassingAccuracy ? 1.0 : 0.0
: NormalCdf((adjustedAccuracy - PassingAccuracy) / accuracySe);
return new ExamSimulationScore
{
PredictedScore = predicted,
RangeLow = Math.Clamp(predicted - range, 1, 300),
RangeHigh = Math.Clamp(predicted + range, 1, 300),
RangeLow = rangeLow,
RangeHigh = rangeHigh,
PassProbability = Math.Round(passProbability, 4),
Correct = correct,
Total = items.Count,
SeenCount = seen,
AbilityElo = Math.Round(ability, 1),
RawAccuracy = Math.Round(rawAccuracy, 4),
DifficultyAdjustedAccuracy = Math.Round(adjustedAccuracy, 4),
AbilityElo = null,
Categories = categories
};
}
private static double EstimateAbility(
IReadOnlyList<ExamSimulationItem> items,
IReadOnlyDictionary<string, ExamSimulationResponse> responses,
IReadOnlyDictionary<string, Question> questions)
public static int ResolveDifficulty(ExamSimulationItem item, Question? question = null)
{
var low = 100.0;
var high = 1900.0;
for (var iteration = 0; iteration < 80; iteration++)
{
var ability = (low + high) / 2.0;
var derivative = -(ability - PriorMean) / (PriorSd * PriorSd);
foreach (var item in items)
{
if (!questions.TryGetValue(item.QuestionExternalId, out var question)) continue;
responses.TryGetValue(item.QuestionExternalId, out var response);
var actual = response?.ChoiceId == question.CorrectChoiceId ? 1.0 : 0.0;
var weight = item.WasSeen ? 0.5 : 1.0;
derivative += weight * (actual - Expected(ability, item.QuestionElo)) * Math.Log(10.0) / 400.0;
}
if (derivative > 0) low = ability; else high = ability;
}
return (low + high) / 2.0;
if (item.QuestionDifficulty is >= 1 and <= 5) return item.QuestionDifficulty;
return question?.Difficulty is >= 1 and <= 5 ? question.Difficulty : 3;
}
private static double Expected(double ability, int difficulty) =>
1.0 / (1.0 + Math.Pow(10.0, (difficulty - ability) / 400.0));
private static double DifficultyWeight(int difficulty) => difficulty switch
{
1 => 0.8,
2 => 0.9,
3 => 1.0,
4 => 1.1,
5 => 1.2,
_ => 1.0
};
private static int ToScore(double accuracy) =>
Math.Clamp((int)Math.Round(ScoreIntercept + ScoreScale * accuracy), 1, 300);
private static double NormalCdf(double value) => 0.5 * (1.0 + Erf(value / Math.Sqrt(2.0)));