105 lines
4.1 KiB
C#
105 lines
4.1 KiB
C#
using QuestionsApi.Models;
|
|
|
|
namespace QuestionsApi.Services;
|
|
|
|
public static class ExamSimulationScoring
|
|
{
|
|
public const string ModelVersion = "provisional-2";
|
|
private const double PassingAccuracy = 0.60;
|
|
private const double ScoreIntercept = 50.0;
|
|
private const double ScoreScale = 250.0;
|
|
|
|
public static ExamSimulationScore Calculate(
|
|
IReadOnlyList<ExamSimulationItem> items,
|
|
IReadOnlyDictionary<string, ExamSimulationResponse> responses,
|
|
IReadOnlyDictionary<string, Question> questions)
|
|
{
|
|
var correct = 0;
|
|
var seen = 0;
|
|
var weightedCorrect = 0.0;
|
|
var weightSum = 0.0;
|
|
var squaredWeightSum = 0.0;
|
|
var categories = new Dictionary<string, ExamSimulationCategoryScore>();
|
|
|
|
foreach (var item in items)
|
|
{
|
|
questions.TryGetValue(item.QuestionExternalId, out var question);
|
|
responses.TryGetValue(item.QuestionExternalId, out var response);
|
|
var isCorrect = question is not null && response?.ChoiceId == question.CorrectChoiceId;
|
|
if (isCorrect) correct++;
|
|
if (item.WasSeen) seen++;
|
|
var difficulty = ResolveDifficulty(item, question);
|
|
var weight = DifficultyWeight(difficulty);
|
|
weightSum += weight;
|
|
squaredWeightSum += weight * weight;
|
|
if (isCorrect) weightedCorrect += weight;
|
|
if (!categories.TryGetValue(item.BlueprintCategory, out var category))
|
|
{
|
|
category = new ExamSimulationCategoryScore();
|
|
categories[item.BlueprintCategory] = category;
|
|
}
|
|
category.Total++;
|
|
if (isCorrect) category.Correct++;
|
|
}
|
|
|
|
var rawAccuracy = items.Count == 0 ? 0.0 : (double)correct / items.Count;
|
|
var adjustedAccuracy = weightSum == 0.0 ? 0.0 : weightedCorrect / weightSum;
|
|
var predicted = ToScore(adjustedAccuracy);
|
|
var effectiveSampleSize = squaredWeightSum == 0.0 ? 0.0 : weightSum * weightSum / squaredWeightSum;
|
|
var accuracySe = effectiveSampleSize == 0.0
|
|
? 0.0
|
|
: Math.Sqrt(adjustedAccuracy * (1.0 - adjustedAccuracy) / effectiveSampleSize);
|
|
var accuracyMargin = 1.96 * accuracySe;
|
|
var rangeLow = ToScore(Math.Clamp(adjustedAccuracy - accuracyMargin, 0.0, 1.0));
|
|
var rangeHigh = ToScore(Math.Clamp(adjustedAccuracy + accuracyMargin, 0.0, 1.0));
|
|
var passProbability = accuracySe == 0.0
|
|
? adjustedAccuracy >= PassingAccuracy ? 1.0 : 0.0
|
|
: NormalCdf((adjustedAccuracy - PassingAccuracy) / accuracySe);
|
|
return new ExamSimulationScore
|
|
{
|
|
PredictedScore = predicted,
|
|
RangeLow = rangeLow,
|
|
RangeHigh = rangeHigh,
|
|
PassProbability = Math.Round(passProbability, 4),
|
|
Correct = correct,
|
|
Total = items.Count,
|
|
SeenCount = seen,
|
|
RawAccuracy = Math.Round(rawAccuracy, 4),
|
|
DifficultyAdjustedAccuracy = Math.Round(adjustedAccuracy, 4),
|
|
AbilityElo = null,
|
|
Categories = categories
|
|
};
|
|
}
|
|
|
|
public static int ResolveDifficulty(ExamSimulationItem item, Question? question = null)
|
|
{
|
|
if (item.QuestionDifficulty is >= 1 and <= 5) return item.QuestionDifficulty;
|
|
return question?.Difficulty is >= 1 and <= 5 ? question.Difficulty : 3;
|
|
}
|
|
|
|
private static double DifficultyWeight(int difficulty) => difficulty switch
|
|
{
|
|
1 => 0.8,
|
|
2 => 0.9,
|
|
3 => 1.0,
|
|
4 => 1.1,
|
|
5 => 1.2,
|
|
_ => 1.0
|
|
};
|
|
|
|
private static int ToScore(double accuracy) =>
|
|
Math.Clamp((int)Math.Round(ScoreIntercept + ScoreScale * accuracy), 1, 300);
|
|
|
|
private static double NormalCdf(double value) => 0.5 * (1.0 + Erf(value / Math.Sqrt(2.0)));
|
|
|
|
private static double Erf(double value)
|
|
{
|
|
var sign = value < 0 ? -1.0 : 1.0;
|
|
value = Math.Abs(value);
|
|
const double p = 0.3275911;
|
|
var t = 1.0 / (1.0 + p * value);
|
|
var y = 1.0 - (((((1.061405429 * t - 1.453152027) * t) + 1.421413741) * t - 0.284496736) * t + 0.254829592) * t * Math.Exp(-value * value);
|
|
return sign * y;
|
|
}
|
|
}
|