hz/step3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix exam simulation score prediction

Browse Source
This commit is contained in:
hz
2026-07-30 16:00:29 -05:00
parent ae6b37eb7d
commit 778da32e73
9 changed files with 282 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/QuestionsApi.Tests/ExamSimulationTests.cs
+102 -3
View File
@@ -44,8 +44,104 @@ public sealed class ExamSimulationTests
Assert.Equal(0, failing.Correct);
}
[Theory]
[InlineData(0, 50)]
[InlineData(20, 100)]
[InlineData(60, 200)]
[InlineData(100, 300)]
public void NeutralDifficultyUsesTransparentScoreAnchors(int correctCount, int expectedScore)
{
var items = BuildItems(100);
var questions = BuildQuestions(100);
var responses = items.Take(correctCount).ToDictionary(
item => item.QuestionExternalId,
item => new ExamSimulationResponse { QuestionExternalId = item.QuestionExternalId, ChoiceId = "A" });
var score = ExamSimulationScoring.Calculate(items, responses, questions);
Assert.Equal(expectedScore, score.PredictedScore);
Assert.Equal(correctCount / 100.0, score.RawAccuracy, 4);
Assert.Equal(correctCount / 100.0, score.DifficultyAdjustedAccuracy, 4);
}
[Fact]
public void SeenItemsAreReportedAndCarryLessScoringWeight()
public void CorrectHarderQuestionsProduceAHigherAdjustedPrediction()
{
var questions = BuildQuestions(10);
var hardCorrect = BuildItems(10);
var easyCorrect = BuildItems(10);
for (var index = 0; index < 10; index++)
{
hardCorrect[index].QuestionDifficulty = index < 5 ? 5 : 1;
easyCorrect[index].QuestionDifficulty = index < 5 ? 1 : 5;
}
var responses = hardCorrect.Take(5).ToDictionary(
item => item.QuestionExternalId,
item => new ExamSimulationResponse { QuestionExternalId = item.QuestionExternalId, ChoiceId = "A" });
var hardScore = ExamSimulationScoring.Calculate(hardCorrect, responses, questions);
var easyScore = ExamSimulationScoring.Calculate(easyCorrect, responses, questions);
Assert.Equal(hardScore.RawAccuracy, easyScore.RawAccuracy);
Assert.True(hardScore.DifficultyAdjustedAccuracy > easyScore.DifficultyAdjustedAccuracy);
Assert.True(hardScore.PredictedScore > easyScore.PredictedScore);
}
[Fact]
public void EmpiricalQuestionEloDoesNotAffectPrediction()
{
var questions = BuildQuestions(20);
var lowElos = BuildItems(20);
var highElos = BuildItems(20);
foreach (var item in lowElos) item.QuestionElo = 100;
foreach (var item in highElos) item.QuestionElo = 1900;
var responses = lowElos.Take(10).ToDictionary(
item => item.QuestionExternalId,
item => new ExamSimulationResponse { QuestionExternalId = item.QuestionExternalId, ChoiceId = "A" });
var lowEloScore = ExamSimulationScoring.Calculate(lowElos, responses, questions);
var highEloScore = ExamSimulationScoring.Calculate(highElos, responses, questions);
Assert.Equal(lowEloScore.PredictedScore, highEloScore.PredictedScore);
Assert.Equal(lowEloScore.PassProbability, highEloScore.PassProbability);
}
[Fact]
public void SkippedAndUnansweredQuestionsCountAsIncorrect()
{
var items = BuildItems(10);
var questions = BuildQuestions(10);
var responses = new Dictionary<string, ExamSimulationResponse>
{
[items[0].QuestionExternalId] = new() { QuestionExternalId = items[0].QuestionExternalId, ChoiceId = "A" },
[items[1].QuestionExternalId] = new() { QuestionExternalId = items[1].QuestionExternalId, ChoiceId = "" }
};
var score = ExamSimulationScoring.Calculate(items, responses, questions);
Assert.Equal(1, score.Correct);
Assert.Equal(10, score.Total);
Assert.Equal(0.1, score.RawAccuracy, 4);
}
[Fact]
public void LowAccuracyFullLengthSimulationIsNotInflated()
{
var items = BuildItems(232);
var questions = BuildQuestions(232);
var responses = items.Take(49).ToDictionary(
item => item.QuestionExternalId,
item => new ExamSimulationResponse { QuestionExternalId = item.QuestionExternalId, ChoiceId = "A" });
var score = ExamSimulationScoring.Calculate(items, responses, questions);
Assert.Equal(103, score.PredictedScore);
Assert.True(score.PassProbability < 0.001);
Assert.True(score.RangeHigh < 130);
}
[Fact]
public void SeenItemsAreReportedButDoNotChangeScoring()
{
var questions = BuildQuestions(20);
var unseen = BuildItems(20);
@@ -59,7 +155,9 @@ public sealed class ExamSimulationTests
Assert.Equal(0, unseenScore.SeenCount);
Assert.Equal(20, seenScore.SeenCount);
Assert.True(unseenScore.AbilityElo > seenScore.AbilityElo);
Assert.Equal(unseenScore.PredictedScore, seenScore.PredictedScore);
Assert.Null(unseenScore.AbilityElo);
Assert.Null(seenScore.AbilityElo);
}
[Theory]
@@ -126,6 +224,7 @@ public sealed class ExamSimulationTests
{
QuestionExternalId = $"question-{index}",
QuestionElo = 1000,
QuestionDifficulty = 3,
WasSeen = wasSeen,
BlueprintCategory = "Cardiovascular"
}).ToList();
@@ -133,5 +232,5 @@ public sealed class ExamSimulationTests
private static Dictionary<string, Question> BuildQuestions(int count) =>
Enumerable.Range(0, count).ToDictionary(
index => $"question-{index}",
index => new Question { ExternalId = $"question-{index}", CorrectChoiceId = "A" });
index => new Question { ExternalId = $"question-{index}", CorrectChoiceId = "A", Difficulty = 3 });
}
api/QuestionsApi/Models/ExamSimulation.cs
+4 -1
View File
@@ -41,6 +41,7 @@ public class ExamSimulationItem
public int Position { get; set; }
public List<string> PresentedChoiceIds { get; set; } = [];
public int QuestionElo { get; set; } = 1000;
public int QuestionDifficulty { get; set; }
public bool WasSeen { get; set; }
public string BlueprintCategory { get; set; } = "Other";
}
@@ -64,7 +65,9 @@ public class ExamSimulationScore
public int Correct { get; set; }
public int Total { get; set; }
public int SeenCount { get; set; }
public double AbilityElo { get; set; }
public double RawAccuracy { get; set; }
public double DifficultyAdjustedAccuracy { get; set; }
public double? AbilityElo { get; set; }
public Dictionary<string, ExamSimulationCategoryScore> Categories { get; set; } = [];
}
api/QuestionsApi/Program.cs
+1
View File
@@ -125,6 +125,7 @@ builder.Services.AddSingleton<DonationStore>();
builder.Services.AddSingleton<IStripeDonationGateway, StripeDonationGateway>();
builder.Services.AddSingleton<ServiceMetrics>();
builder.Services.AddHostedService<AnswerStatisticsMaintenanceService>();
builder.Services.AddHostedService<ExamSimulationScoringMigrationService>();
builder.Services.AddHttpClient();
builder.Services.AddRateLimiter(options =>
{
api/QuestionsApi/Services/ExamSimulationScoring.cs
+47 -43
View File
@@ -4,33 +4,35 @@ namespace QuestionsApi.Services;
public static class ExamSimulationScoring
{
public const string ModelVersion = "provisional-1";
private const double PriorMean = 1000.0;
private const double PriorSd = 400.0;
public const string ModelVersion = "provisional-2";
private const double PassingAccuracy = 0.60;
private const double ScoreIntercept = 50.0;
private const double ScoreScale = 250.0;
public static ExamSimulationScore Calculate(
IReadOnlyList<ExamSimulationItem> items,
IReadOnlyDictionary<string, ExamSimulationResponse> responses,
IReadOnlyDictionary<string, Question> questions)
{
var ability = EstimateAbility(items, responses, questions);
var predicted = Math.Clamp((int)Math.Round(227.0 + (ability - 1000.0) * 15.0 / 200.0), 1, 300);
var information = 1.0 / (PriorSd * PriorSd);
var correct = 0;
var seen = 0;
var weightedCorrect = 0.0;
var weightSum = 0.0;
var squaredWeightSum = 0.0;
var categories = new Dictionary<string, ExamSimulationCategoryScore>();
foreach (var item in items)
{
if (!questions.TryGetValue(item.QuestionExternalId, out var question))
continue;
questions.TryGetValue(item.QuestionExternalId, out var question);
responses.TryGetValue(item.QuestionExternalId, out var response);
var isCorrect = response?.ChoiceId == question.CorrectChoiceId;
var isCorrect = question is not null && response?.ChoiceId == question.CorrectChoiceId;
if (isCorrect) correct++;
if (item.WasSeen) seen++;
var weight = item.WasSeen ? 0.5 : 1.0;
var probability = Expected(ability, item.QuestionElo);
information += weight * probability * (1.0 - probability) * Math.Pow(Math.Log(10.0) / 400.0, 2);
var difficulty = ResolveDifficulty(item, question);
var weight = DifficultyWeight(difficulty);
weightSum += weight;
squaredWeightSum += weight * weight;
if (isCorrect) weightedCorrect += weight;
if (!categories.TryGetValue(item.BlueprintCategory, out var category))
{
category = new ExamSimulationCategoryScore();
@@ -40,51 +42,53 @@ public static class ExamSimulationScoring
if (isCorrect) category.Correct++;
}
var abilitySe = Math.Sqrt(1.0 / information);
var scoreSe = abilitySe * 15.0 / 200.0;
var combinedSe = Math.Sqrt(7.0 * 7.0 + 15.0 * 15.0 + scoreSe * scoreSe);
var range = (int)Math.Ceiling(combinedSe);
var passProbability = NormalCdf((predicted - 200.0) / combinedSe);
var rawAccuracy = items.Count == 0 ? 0.0 : (double)correct / items.Count;
var adjustedAccuracy = weightSum == 0.0 ? 0.0 : weightedCorrect / weightSum;
var predicted = ToScore(adjustedAccuracy);
var effectiveSampleSize = squaredWeightSum == 0.0 ? 0.0 : weightSum * weightSum / squaredWeightSum;
var accuracySe = effectiveSampleSize == 0.0
? 0.0
: Math.Sqrt(adjustedAccuracy * (1.0 - adjustedAccuracy) / effectiveSampleSize);
var accuracyMargin = 1.96 * accuracySe;
var rangeLow = ToScore(Math.Clamp(adjustedAccuracy - accuracyMargin, 0.0, 1.0));
var rangeHigh = ToScore(Math.Clamp(adjustedAccuracy + accuracyMargin, 0.0, 1.0));
var passProbability = accuracySe == 0.0
? adjustedAccuracy >= PassingAccuracy ? 1.0 : 0.0
: NormalCdf((adjustedAccuracy - PassingAccuracy) / accuracySe);
return new ExamSimulationScore
{
PredictedScore = predicted,
RangeLow = Math.Clamp(predicted - range, 1, 300),
RangeHigh = Math.Clamp(predicted + range, 1, 300),
RangeLow = rangeLow,
RangeHigh = rangeHigh,
PassProbability = Math.Round(passProbability, 4),
Correct = correct,
Total = items.Count,
SeenCount = seen,
AbilityElo = Math.Round(ability, 1),
RawAccuracy = Math.Round(rawAccuracy, 4),
DifficultyAdjustedAccuracy = Math.Round(adjustedAccuracy, 4),
AbilityElo = null,
Categories = categories
};
}
private static double EstimateAbility(
IReadOnlyList<ExamSimulationItem> items,
IReadOnlyDictionary<string, ExamSimulationResponse> responses,
IReadOnlyDictionary<string, Question> questions)
public static int ResolveDifficulty(ExamSimulationItem item, Question? question = null)
{
var low = 100.0;
var high = 1900.0;
for (var iteration = 0; iteration < 80; iteration++)
{
var ability = (low + high) / 2.0;
var derivative = -(ability - PriorMean) / (PriorSd * PriorSd);
foreach (var item in items)
{
if (!questions.TryGetValue(item.QuestionExternalId, out var question)) continue;
responses.TryGetValue(item.QuestionExternalId, out var response);
var actual = response?.ChoiceId == question.CorrectChoiceId ? 1.0 : 0.0;
var weight = item.WasSeen ? 0.5 : 1.0;
derivative += weight * (actual - Expected(ability, item.QuestionElo)) * Math.Log(10.0) / 400.0;
}
if (derivative > 0) low = ability; else high = ability;
}
return (low + high) / 2.0;
if (item.QuestionDifficulty is >= 1 and <= 5) return item.QuestionDifficulty;
return question?.Difficulty is >= 1 and <= 5 ? question.Difficulty : 3;
}
private static double Expected(double ability, int difficulty) =>
1.0 / (1.0 + Math.Pow(10.0, (difficulty - ability) / 400.0));
private static double DifficultyWeight(int difficulty) => difficulty switch
{
1 => 0.8,
2 => 0.9,
3 => 1.0,
4 => 1.1,
5 => 1.2,
_ => 1.0
};
private static int ToScore(double accuracy) =>
Math.Clamp((int)Math.Round(ScoreIntercept + ScoreScale * accuracy), 1, 300);
private static double NormalCdf(double value) => 0.5 * (1.0 + Erf(value / Math.Sqrt(2.0)));
api/QuestionsApi/Services/ExamSimulationScoringMigrationService.cs
+37
View File
@@ -0,0 +1,37 @@
namespace QuestionsApi.Services;
public sealed class ExamSimulationScoringMigrationService : BackgroundService
{
private readonly ExamSimulationStore _simulations;
private readonly ILogger<ExamSimulationScoringMigrationService> _logger;
public ExamSimulationScoringMigrationService(
ExamSimulationStore simulations,
ILogger<ExamSimulationScoringMigrationService> logger)
{
_simulations = simulations;
_logger = logger;
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
try
{
var result = await _simulations.BackfillScoringModelAsync(stoppingToken);
_logger.LogInformation(
"Exam simulation scoring migration completed; recalculated {UpdatedCount} simulation(s)",
result.UpdatedCount);
foreach (var reference in result.MissingQuestionIds)
_logger.LogWarning(
"Exam simulation scoring migration used neutral difficulty for missing question {Reference}",
reference);
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
{
}
catch (Exception exception)
{
_logger.LogError(exception, "Exam simulation scoring migration failed");
}
}
}
api/QuestionsApi/Services/ExamSimulationStore.cs
+81 -3
View File
@@ -91,6 +91,7 @@ public class ExamSimulationStore
Position = offset,
PresentedChoiceIds = SelectChoices(question, choicePercentages.GetValueOrDefault(question.ExternalId)),
QuestionElo = elos.GetValueOrDefault(question.ExternalId, 1000),
QuestionDifficulty = question.Difficulty,
WasSeen = seen.Contains(question.ExternalId),
BlueprintCategory = BlueprintCategory(question)
});
@@ -128,6 +129,7 @@ public class ExamSimulationStore
Position = PositionWithinBlock(index),
PresentedChoiceIds = SelectChoices(question, choicePercentages.GetValueOrDefault(question.ExternalId)),
QuestionElo = 1000,
QuestionDifficulty = question.Difficulty,
WasSeen = false,
BlueprintCategory = BlueprintCategory(question)
}).ToList();
@@ -173,15 +175,22 @@ public class ExamSimulationStore
(item.State == ExamSimulationStates.BetweenBlocks || item.State == ExamSimulationStates.BlockActive))
.SortByDescending(item => item.UpdatedUtc).FirstOrDefaultAsync();
public async Task<ExamSimulationScore?> GetLatestCompletedScoreAsync(string userId) =>
(await _simulations.Find(item => item.UserId == userId && item.State == ExamSimulationStates.Completed && item.Score != null)
.SortByDescending(item => item.CompletedUtc).FirstOrDefaultAsync())?.Score;
public async Task<ExamSimulationScore?> GetLatestCompletedScoreAsync(string userId)
{
var simulation = await _simulations
.Find(item => item.UserId == userId && item.State == ExamSimulationStates.Completed && item.Score != null)
.SortByDescending(item => item.CompletedUtc)
.FirstOrDefaultAsync();
return simulation is null ? null : (await UpgradeScoringModelAsync(simulation)).Score;
}
public async Task<ExamSimulation?> GetOwnedAsync(string id, string userId)
{
var simulation = await _simulations.Find(item => item.Id == id && item.UserId == userId).FirstOrDefaultAsync();
if (simulation is not null && simulation.State == ExamSimulationStates.BlockActive && simulation.BlockDeadlineUtc <= DateTime.UtcNow)
simulation = await SubmitBlockAsync(simulation, true);
if (simulation is not null)
simulation = await UpgradeScoringModelAsync(simulation);
return simulation;
}
@@ -195,6 +204,7 @@ public class ExamSimulationStore
var simulations = await _simulations.Find(filter).SortByDescending(item => item.CompletedUtc).Limit(limit + 1).ToListAsync();
var hasMore = simulations.Count > limit;
if (hasMore) simulations.RemoveAt(simulations.Count - 1);
await UpgradeScoringModelsAsync(simulations);
return new CompletedExamSimulationPage
{
Items = simulations.Select(item => new CompletedExamSimulationSummary
@@ -261,6 +271,7 @@ public class ExamSimulationStore
var simulations = await _simulations.Find(filter).Sort(sortDefinition).Limit(limit + 1).ToListAsync();
var hasMore = simulations.Count > limit;
if (hasMore) simulations.RemoveAt(simulations.Count - 1);
await UpgradeScoringModelsAsync(simulations.Where(item => item.State == ExamSimulationStates.Completed));
var last = simulations.LastOrDefault();
return new ExamSimulationHistoryPage
{
@@ -442,7 +453,9 @@ public class ExamSimulationStore
{
var questionMap = await _questions.GetByExternalIdsAsync(simulation.Items.Select(item => item.QuestionExternalId));
var responses = simulation.Responses.ToDictionary(item => item.QuestionExternalId, StringComparer.Ordinal);
SnapshotDifficulties(simulation, questionMap);
simulation.Score = ExamSimulationScoring.Calculate(simulation.Items, responses, questionMap);
simulation.ModelVersion = ExamSimulationScoring.ModelVersion;
simulation.State = ExamSimulationStates.Completed;
simulation.CompletedUtc = DateTime.UtcNow;
simulation.UpdatedUtc = DateTime.UtcNow;
@@ -480,6 +493,71 @@ public class ExamSimulationStore
private Task ReplaceAsync(ExamSimulation simulation) =>
_simulations.ReplaceOneAsync(item => item.Id == simulation.Id && item.UserId == simulation.UserId, simulation);
public async Task<ExamSimulationBackfillResult> BackfillScoringModelAsync(CancellationToken cancellationToken = default)
{
var filter = Builders<ExamSimulation>.Filter.Eq(item => item.State, ExamSimulationStates.Completed) &
Builders<ExamSimulation>.Filter.Ne(item => item.ModelVersion, ExamSimulationScoring.ModelVersion);
var simulations = await _simulations.Find(filter).ToListAsync(cancellationToken);
var missingQuestionIds = new List<string>();
var updated = 0;
foreach (var simulation in simulations)
{
cancellationToken.ThrowIfCancellationRequested();
var questionMap = await _questions.GetByExternalIdsAsync(simulation.Items.Select(item => item.QuestionExternalId));
missingQuestionIds.AddRange(simulation.Items
.Where(item => !questionMap.ContainsKey(item.QuestionExternalId))
.Select(item => $"{simulation.Id}:{item.QuestionExternalId}"));
SnapshotDifficulties(simulation, questionMap);
var responses = simulation.Responses.ToDictionary(item => item.QuestionExternalId, StringComparer.Ordinal);
simulation.Score = ExamSimulationScoring.Calculate(simulation.Items, responses, questionMap);
simulation.ModelVersion = ExamSimulationScoring.ModelVersion;
simulation.UpdatedUtc = DateTime.UtcNow;
await ReplaceAsync(simulation);
updated++;
}
return new ExamSimulationBackfillResult(updated, missingQuestionIds);
}
private async Task UpgradeScoringModelsAsync(IEnumerable<ExamSimulation> simulations)
{
foreach (var simulation in simulations)
await UpgradeScoringModelAsync(simulation);
}
private async Task<ExamSimulation> UpgradeScoringModelAsync(ExamSimulation simulation)
{
var needsDifficultySnapshot = simulation.Items.Any(item => item.QuestionDifficulty is < 1 or > 5);
var needsScoreUpgrade = simulation.State == ExamSimulationStates.Completed &&
simulation.ModelVersion != ExamSimulationScoring.ModelVersion;
if (!needsDifficultySnapshot && !needsScoreUpgrade)
return simulation;
var questionMap = await _questions.GetByExternalIdsAsync(simulation.Items.Select(item => item.QuestionExternalId));
SnapshotDifficulties(simulation, questionMap);
if (simulation.State == ExamSimulationStates.Completed)
{
var responses = simulation.Responses.ToDictionary(item => item.QuestionExternalId, StringComparer.Ordinal);
simulation.Score = ExamSimulationScoring.Calculate(simulation.Items, responses, questionMap);
}
simulation.ModelVersion = ExamSimulationScoring.ModelVersion;
simulation.UpdatedUtc = DateTime.UtcNow;
await ReplaceAsync(simulation);
return simulation;
}
private static void SnapshotDifficulties(
ExamSimulation simulation,
IReadOnlyDictionary<string, Question> questions)
{
foreach (var item in simulation.Items)
{
questions.TryGetValue(item.QuestionExternalId, out var question);
item.QuestionDifficulty = ExamSimulationScoring.ResolveDifficulty(item, question);
}
}
public sealed record ExamSimulationBackfillResult(int UpdatedCount, IReadOnlyList<string> MissingQuestionIds);
private static int BlockIndexForPosition(int position)
{
var consumed = 0;
scripts/ui/ccs_hub.gd
+1 -1
View File
@@ -297,7 +297,7 @@ func _build_performance_summary(dark: bool) -> void:
if combined != null:
prediction_value = "%d" % int(combined)
prediction_detail = "Range %d-%d" % [int(CCSService.prediction.get("rangeLow", 0)), int(CCSService.prediction.get("rangeHigh", 0))]
_add_metric(row, "Combined Prediction", prediction_value, prediction_detail, dark)
_add_metric(row, "Provisional Combined Estimate", prediction_value, prediction_detail, dark)
func _add_metric(parent: BoxContainer, caption: String, value: String, detail: String, dark: bool) -> void:
scripts/ui/dashboard_screen.gd
+3 -3
View File
@@ -94,7 +94,7 @@ func _build() -> void:
readiness_segment_bar = DashboardSegmentBar.new()
readiness_segment_bar.custom_minimum_size = Vector2(0, 10)
_add_stat_card("readiness", "Exam Readiness", "chart-no-axes-column-increasing", Brand.PURPLE, readiness_segment_bar,
"Predicted pass probability from your most recent completed exam simulation.")
"Heuristic pass probability from your most recent completed exam simulation.")
accuracy_progress_bar = ProgressBar.new()
accuracy_progress_bar.show_percentage = false
@@ -254,8 +254,8 @@ func _render_readiness_card(data: Dictionary) -> void:
var label := str(label_value)
var pass_percent := int(data.get("examReadinessProjectedPassPercent", 0))
var status_color := Brand.SUCCESS if label == "High" else (Brand.AMBER if label == "Medium" else Brand.MAGENTA)
var caption := "You're on track!" if label != "Low" else "Keep practicing"
_set_stat_card("readiness", label, "Projected pass: %d%%" % pass_percent, status_color, caption)
var caption := "Provisional estimate" if label != "Low" else "Keep practicing"
_set_stat_card("readiness", label, "Heuristic pass: %d%%" % pass_percent, status_color, caption)
(metric_values.get("readiness") as Label).add_theme_color_override("font_color", status_color)
readiness_segment_bar.set_progress(int(round(pass_percent / 100.0 * 4.0)), 4, status_color, dark)
scripts/ui/simulation_results.gd
+6 -3
View File
@@ -67,9 +67,12 @@ func on_show(payload: Variant = null) -> void:
var low := int(score.get("rangeLow", 0))
var high := int(score.get("rangeHigh", 0))
var probability := int(round(float(score.get("passProbability", 0.0)) * 100.0))
score_label.text = "Predicted Step 3 score: %d (%d-%d)" % [predicted, low, high]
summary_label.text = "%d/%d correct | Estimated probability of reaching 200: %d%% | Previously seen questions: %d\nThis provisional educational estimate is not an official USMLE or NBME score and excludes CCS performance." % [
int(score.get("correct", 0)), int(score.get("total", 0)), probability, int(score.get("seenCount", 0))]
var raw_accuracy := float(score.get("rawAccuracy", 0.0)) * 100.0
var adjusted_accuracy := float(score.get("difficultyAdjustedAccuracy", 0.0)) * 100.0
score_label.text = "Provisional Step 3 estimate: %d (%d-%d)" % [predicted, low, high]
summary_label.text = "%d/%d correct (%.1f%%) | Difficulty-adjusted accuracy: %.1f%%\nHeuristic probability of provisional score >=200: %d%% | Previously seen questions: %d\nThis transparent, unvalidated educational heuristic is not an official USMLE or NBME score and excludes CCS performance." % [
int(score.get("correct", 0)), int(score.get("total", 0)), raw_accuracy, adjusted_accuracy,
probability, int(score.get("seenCount", 0))]
_clear(category_list)
for category in score.get("categories", {}):
var value: Dictionary = score["categories"][category]