using QuestionsApi.Models; namespace QuestionsApi.Services; public static class ExamSimulationScoring { public const string ModelVersion = "provisional-2"; private const double PassingAccuracy = 0.60; private const double ScoreIntercept = 50.0; private const double ScoreScale = 250.0; public static ExamSimulationScore Calculate( IReadOnlyList items, IReadOnlyDictionary responses, IReadOnlyDictionary questions) { var correct = 0; var seen = 0; var weightedCorrect = 0.0; var weightSum = 0.0; var squaredWeightSum = 0.0; var categories = new Dictionary(); foreach (var item in items) { questions.TryGetValue(item.QuestionExternalId, out var question); responses.TryGetValue(item.QuestionExternalId, out var response); var isCorrect = question is not null && response?.ChoiceId == question.CorrectChoiceId; if (isCorrect) correct++; if (item.WasSeen) seen++; var difficulty = ResolveDifficulty(item, question); var weight = DifficultyWeight(difficulty); weightSum += weight; squaredWeightSum += weight * weight; if (isCorrect) weightedCorrect += weight; if (!categories.TryGetValue(item.BlueprintCategory, out var category)) { category = new ExamSimulationCategoryScore(); categories[item.BlueprintCategory] = category; } category.Total++; if (isCorrect) category.Correct++; } var rawAccuracy = items.Count == 0 ? 0.0 : (double)correct / items.Count; var adjustedAccuracy = weightSum == 0.0 ? 0.0 : weightedCorrect / weightSum; var predicted = ToScore(adjustedAccuracy); var effectiveSampleSize = squaredWeightSum == 0.0 ? 0.0 : weightSum * weightSum / squaredWeightSum; var accuracySe = effectiveSampleSize == 0.0 ? 0.0 : Math.Sqrt(adjustedAccuracy * (1.0 - adjustedAccuracy) / effectiveSampleSize); var accuracyMargin = 1.96 * accuracySe; var rangeLow = ToScore(Math.Clamp(adjustedAccuracy - accuracyMargin, 0.0, 1.0)); var rangeHigh = ToScore(Math.Clamp(adjustedAccuracy + accuracyMargin, 0.0, 1.0)); var passProbability = accuracySe == 0.0 ? adjustedAccuracy >= PassingAccuracy ? 1.0 : 0.0 : NormalCdf((adjustedAccuracy - PassingAccuracy) / accuracySe); return new ExamSimulationScore { PredictedScore = predicted, RangeLow = rangeLow, RangeHigh = rangeHigh, PassProbability = Math.Round(passProbability, 4), Correct = correct, Total = items.Count, SeenCount = seen, RawAccuracy = Math.Round(rawAccuracy, 4), DifficultyAdjustedAccuracy = Math.Round(adjustedAccuracy, 4), AbilityElo = null, Categories = categories }; } public static int ResolveDifficulty(ExamSimulationItem item, Question? question = null) { if (item.QuestionDifficulty is >= 1 and <= 5) return item.QuestionDifficulty; return question?.Difficulty is >= 1 and <= 5 ? question.Difficulty : 3; } private static double DifficultyWeight(int difficulty) => difficulty switch { 1 => 0.8, 2 => 0.9, 3 => 1.0, 4 => 1.1, 5 => 1.2, _ => 1.0 }; private static int ToScore(double accuracy) => Math.Clamp((int)Math.Round(ScoreIntercept + ScoreScale * accuracy), 1, 300); private static double NormalCdf(double value) => 0.5 * (1.0 + Erf(value / Math.Sqrt(2.0))); private static double Erf(double value) { var sign = value < 0 ? -1.0 : 1.0; value = Math.Abs(value); const double p = 0.3275911; var t = 1.0 / (1.0 + p * value); var y = 1.0 - (((((1.061405429 * t - 1.453152027) * t) + 1.421413741) * t - 0.284496736) * t + 0.254829592) * t * Math.Exp(-value * value); return sign * y; } }