101 lines
4.2 KiB
C#
101 lines
4.2 KiB
C#
using QuestionsApi.Models;
|
|
|
|
namespace QuestionsApi.Services;
|
|
|
|
public static class ExamSimulationScoring
|
|
{
|
|
public const string ModelVersion = "provisional-1";
|
|
private const double PriorMean = 1000.0;
|
|
private const double PriorSd = 400.0;
|
|
|
|
public static ExamSimulationScore Calculate(
|
|
IReadOnlyList<ExamSimulationItem> items,
|
|
IReadOnlyDictionary<string, ExamSimulationResponse> responses,
|
|
IReadOnlyDictionary<string, Question> questions)
|
|
{
|
|
var ability = EstimateAbility(items, responses, questions);
|
|
var predicted = Math.Clamp((int)Math.Round(227.0 + (ability - 1000.0) * 15.0 / 200.0), 1, 300);
|
|
var information = 1.0 / (PriorSd * PriorSd);
|
|
var correct = 0;
|
|
var seen = 0;
|
|
var categories = new Dictionary<string, ExamSimulationCategoryScore>();
|
|
|
|
foreach (var item in items)
|
|
{
|
|
if (!questions.TryGetValue(item.QuestionExternalId, out var question))
|
|
continue;
|
|
responses.TryGetValue(item.QuestionExternalId, out var response);
|
|
var isCorrect = response?.ChoiceId == question.CorrectChoiceId;
|
|
if (isCorrect) correct++;
|
|
if (item.WasSeen) seen++;
|
|
var weight = item.WasSeen ? 0.5 : 1.0;
|
|
var probability = Expected(ability, item.QuestionElo);
|
|
information += weight * probability * (1.0 - probability) * Math.Pow(Math.Log(10.0) / 400.0, 2);
|
|
if (!categories.TryGetValue(item.BlueprintCategory, out var category))
|
|
{
|
|
category = new ExamSimulationCategoryScore();
|
|
categories[item.BlueprintCategory] = category;
|
|
}
|
|
category.Total++;
|
|
if (isCorrect) category.Correct++;
|
|
}
|
|
|
|
var abilitySe = Math.Sqrt(1.0 / information);
|
|
var scoreSe = abilitySe * 15.0 / 200.0;
|
|
var combinedSe = Math.Sqrt(7.0 * 7.0 + 15.0 * 15.0 + scoreSe * scoreSe);
|
|
var range = (int)Math.Ceiling(combinedSe);
|
|
var passProbability = NormalCdf((predicted - 200.0) / combinedSe);
|
|
return new ExamSimulationScore
|
|
{
|
|
PredictedScore = predicted,
|
|
RangeLow = Math.Clamp(predicted - range, 1, 300),
|
|
RangeHigh = Math.Clamp(predicted + range, 1, 300),
|
|
PassProbability = Math.Round(passProbability, 4),
|
|
Correct = correct,
|
|
Total = items.Count,
|
|
SeenCount = seen,
|
|
AbilityElo = Math.Round(ability, 1),
|
|
Categories = categories
|
|
};
|
|
}
|
|
|
|
private static double EstimateAbility(
|
|
IReadOnlyList<ExamSimulationItem> items,
|
|
IReadOnlyDictionary<string, ExamSimulationResponse> responses,
|
|
IReadOnlyDictionary<string, Question> questions)
|
|
{
|
|
var low = 100.0;
|
|
var high = 1900.0;
|
|
for (var iteration = 0; iteration < 80; iteration++)
|
|
{
|
|
var ability = (low + high) / 2.0;
|
|
var derivative = -(ability - PriorMean) / (PriorSd * PriorSd);
|
|
foreach (var item in items)
|
|
{
|
|
if (!questions.TryGetValue(item.QuestionExternalId, out var question)) continue;
|
|
responses.TryGetValue(item.QuestionExternalId, out var response);
|
|
var actual = response?.ChoiceId == question.CorrectChoiceId ? 1.0 : 0.0;
|
|
var weight = item.WasSeen ? 0.5 : 1.0;
|
|
derivative += weight * (actual - Expected(ability, item.QuestionElo)) * Math.Log(10.0) / 400.0;
|
|
}
|
|
if (derivative > 0) low = ability; else high = ability;
|
|
}
|
|
return (low + high) / 2.0;
|
|
}
|
|
|
|
private static double Expected(double ability, int difficulty) =>
|
|
1.0 / (1.0 + Math.Pow(10.0, (difficulty - ability) / 400.0));
|
|
|
|
private static double NormalCdf(double value) => 0.5 * (1.0 + Erf(value / Math.Sqrt(2.0)));
|
|
|
|
private static double Erf(double value)
|
|
{
|
|
var sign = value < 0 ? -1.0 : 1.0;
|
|
value = Math.Abs(value);
|
|
const double p = 0.3275911;
|
|
var t = 1.0 / (1.0 + p * value);
|
|
var y = 1.0 - (((((1.061405429 * t - 1.453152027) * t) + 1.421413741) * t - 0.284496736) * t + 0.254829592) * t * Math.Exp(-value * value);
|
|
return sign * y;
|
|
}
|
|
}
|