From 778da32e735fe27d8368d9387ba63283e29c9e53 Mon Sep 17 00:00:00 2001 From: hz Date: Thu, 30 Jul 2026 16:00:29 -0500 Subject: [PATCH] Fix exam simulation score prediction --- api/QuestionsApi.Tests/ExamSimulationTests.cs | 105 +++++++++++++++++- api/QuestionsApi/Models/ExamSimulation.cs | 5 +- api/QuestionsApi/Program.cs | 1 + .../Services/ExamSimulationScoring.cs | 90 ++++++++------- .../ExamSimulationScoringMigrationService.cs | 37 ++++++ .../Services/ExamSimulationStore.cs | 84 +++++++++++++- scripts/ui/ccs_hub.gd | 2 +- scripts/ui/dashboard_screen.gd | 6 +- scripts/ui/simulation_results.gd | 9 +- 9 files changed, 282 insertions(+), 57 deletions(-) create mode 100644 api/QuestionsApi/Services/ExamSimulationScoringMigrationService.cs diff --git a/api/QuestionsApi.Tests/ExamSimulationTests.cs b/api/QuestionsApi.Tests/ExamSimulationTests.cs index 5713108..f51b7bb 100644 --- a/api/QuestionsApi.Tests/ExamSimulationTests.cs +++ b/api/QuestionsApi.Tests/ExamSimulationTests.cs @@ -44,8 +44,104 @@ public sealed class ExamSimulationTests Assert.Equal(0, failing.Correct); } + [Theory] + [InlineData(0, 50)] + [InlineData(20, 100)] + [InlineData(60, 200)] + [InlineData(100, 300)] + public void NeutralDifficultyUsesTransparentScoreAnchors(int correctCount, int expectedScore) + { + var items = BuildItems(100); + var questions = BuildQuestions(100); + var responses = items.Take(correctCount).ToDictionary( + item => item.QuestionExternalId, + item => new ExamSimulationResponse { QuestionExternalId = item.QuestionExternalId, ChoiceId = "A" }); + + var score = ExamSimulationScoring.Calculate(items, responses, questions); + + Assert.Equal(expectedScore, score.PredictedScore); + Assert.Equal(correctCount / 100.0, score.RawAccuracy, 4); + Assert.Equal(correctCount / 100.0, score.DifficultyAdjustedAccuracy, 4); + } + [Fact] - public void SeenItemsAreReportedAndCarryLessScoringWeight() + public void CorrectHarderQuestionsProduceAHigherAdjustedPrediction() + { + var questions = BuildQuestions(10); + var hardCorrect = BuildItems(10); + var easyCorrect = BuildItems(10); + for (var index = 0; index < 10; index++) + { + hardCorrect[index].QuestionDifficulty = index < 5 ? 5 : 1; + easyCorrect[index].QuestionDifficulty = index < 5 ? 1 : 5; + } + var responses = hardCorrect.Take(5).ToDictionary( + item => item.QuestionExternalId, + item => new ExamSimulationResponse { QuestionExternalId = item.QuestionExternalId, ChoiceId = "A" }); + + var hardScore = ExamSimulationScoring.Calculate(hardCorrect, responses, questions); + var easyScore = ExamSimulationScoring.Calculate(easyCorrect, responses, questions); + + Assert.Equal(hardScore.RawAccuracy, easyScore.RawAccuracy); + Assert.True(hardScore.DifficultyAdjustedAccuracy > easyScore.DifficultyAdjustedAccuracy); + Assert.True(hardScore.PredictedScore > easyScore.PredictedScore); + } + + [Fact] + public void EmpiricalQuestionEloDoesNotAffectPrediction() + { + var questions = BuildQuestions(20); + var lowElos = BuildItems(20); + var highElos = BuildItems(20); + foreach (var item in lowElos) item.QuestionElo = 100; + foreach (var item in highElos) item.QuestionElo = 1900; + var responses = lowElos.Take(10).ToDictionary( + item => item.QuestionExternalId, + item => new ExamSimulationResponse { QuestionExternalId = item.QuestionExternalId, ChoiceId = "A" }); + + var lowEloScore = ExamSimulationScoring.Calculate(lowElos, responses, questions); + var highEloScore = ExamSimulationScoring.Calculate(highElos, responses, questions); + + Assert.Equal(lowEloScore.PredictedScore, highEloScore.PredictedScore); + Assert.Equal(lowEloScore.PassProbability, highEloScore.PassProbability); + } + + [Fact] + public void SkippedAndUnansweredQuestionsCountAsIncorrect() + { + var items = BuildItems(10); + var questions = BuildQuestions(10); + var responses = new Dictionary + { + [items[0].QuestionExternalId] = new() { QuestionExternalId = items[0].QuestionExternalId, ChoiceId = "A" }, + [items[1].QuestionExternalId] = new() { QuestionExternalId = items[1].QuestionExternalId, ChoiceId = "" } + }; + + var score = ExamSimulationScoring.Calculate(items, responses, questions); + + Assert.Equal(1, score.Correct); + Assert.Equal(10, score.Total); + Assert.Equal(0.1, score.RawAccuracy, 4); + } + + [Fact] + public void LowAccuracyFullLengthSimulationIsNotInflated() + { + var items = BuildItems(232); + var questions = BuildQuestions(232); + var responses = items.Take(49).ToDictionary( + item => item.QuestionExternalId, + item => new ExamSimulationResponse { QuestionExternalId = item.QuestionExternalId, ChoiceId = "A" }); + + var score = ExamSimulationScoring.Calculate(items, responses, questions); + + Assert.Equal(103, score.PredictedScore); + Assert.True(score.PassProbability < 0.001); + Assert.True(score.RangeHigh < 130); + } + + [Fact] + public void SeenItemsAreReportedButDoNotChangeScoring() { var questions = BuildQuestions(20); var unseen = BuildItems(20); @@ -59,7 +155,9 @@ public sealed class ExamSimulationTests Assert.Equal(0, unseenScore.SeenCount); Assert.Equal(20, seenScore.SeenCount); - Assert.True(unseenScore.AbilityElo > seenScore.AbilityElo); + Assert.Equal(unseenScore.PredictedScore, seenScore.PredictedScore); + Assert.Null(unseenScore.AbilityElo); + Assert.Null(seenScore.AbilityElo); } [Theory] @@ -126,6 +224,7 @@ public sealed class ExamSimulationTests { QuestionExternalId = $"question-{index}", QuestionElo = 1000, + QuestionDifficulty = 3, WasSeen = wasSeen, BlueprintCategory = "Cardiovascular" }).ToList(); @@ -133,5 +232,5 @@ public sealed class ExamSimulationTests private static Dictionary BuildQuestions(int count) => Enumerable.Range(0, count).ToDictionary( index => $"question-{index}", - index => new Question { ExternalId = $"question-{index}", CorrectChoiceId = "A" }); + index => new Question { ExternalId = $"question-{index}", CorrectChoiceId = "A", Difficulty = 3 }); } diff --git a/api/QuestionsApi/Models/ExamSimulation.cs b/api/QuestionsApi/Models/ExamSimulation.cs index fab1927..5a2d8b1 100644 --- a/api/QuestionsApi/Models/ExamSimulation.cs +++ b/api/QuestionsApi/Models/ExamSimulation.cs @@ -41,6 +41,7 @@ public class ExamSimulationItem public int Position { get; set; } public List PresentedChoiceIds { get; set; } = []; public int QuestionElo { get; set; } = 1000; + public int QuestionDifficulty { get; set; } public bool WasSeen { get; set; } public string BlueprintCategory { get; set; } = "Other"; } @@ -64,7 +65,9 @@ public class ExamSimulationScore public int Correct { get; set; } public int Total { get; set; } public int SeenCount { get; set; } - public double AbilityElo { get; set; } + public double RawAccuracy { get; set; } + public double DifficultyAdjustedAccuracy { get; set; } + public double? AbilityElo { get; set; } public Dictionary Categories { get; set; } = []; } diff --git a/api/QuestionsApi/Program.cs b/api/QuestionsApi/Program.cs index c511ed3..3f8f002 100644 --- a/api/QuestionsApi/Program.cs +++ b/api/QuestionsApi/Program.cs @@ -125,6 +125,7 @@ builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddHostedService(); +builder.Services.AddHostedService(); builder.Services.AddHttpClient(); builder.Services.AddRateLimiter(options => { diff --git a/api/QuestionsApi/Services/ExamSimulationScoring.cs b/api/QuestionsApi/Services/ExamSimulationScoring.cs index fae77a4..b398ae6 100644 --- a/api/QuestionsApi/Services/ExamSimulationScoring.cs +++ b/api/QuestionsApi/Services/ExamSimulationScoring.cs @@ -4,33 +4,35 @@ namespace QuestionsApi.Services; public static class ExamSimulationScoring { - public const string ModelVersion = "provisional-1"; - private const double PriorMean = 1000.0; - private const double PriorSd = 400.0; + public const string ModelVersion = "provisional-2"; + private const double PassingAccuracy = 0.60; + private const double ScoreIntercept = 50.0; + private const double ScoreScale = 250.0; public static ExamSimulationScore Calculate( IReadOnlyList items, IReadOnlyDictionary responses, IReadOnlyDictionary questions) { - var ability = EstimateAbility(items, responses, questions); - var predicted = Math.Clamp((int)Math.Round(227.0 + (ability - 1000.0) * 15.0 / 200.0), 1, 300); - var information = 1.0 / (PriorSd * PriorSd); var correct = 0; var seen = 0; + var weightedCorrect = 0.0; + var weightSum = 0.0; + var squaredWeightSum = 0.0; var categories = new Dictionary(); foreach (var item in items) { - if (!questions.TryGetValue(item.QuestionExternalId, out var question)) - continue; + questions.TryGetValue(item.QuestionExternalId, out var question); responses.TryGetValue(item.QuestionExternalId, out var response); - var isCorrect = response?.ChoiceId == question.CorrectChoiceId; + var isCorrect = question is not null && response?.ChoiceId == question.CorrectChoiceId; if (isCorrect) correct++; if (item.WasSeen) seen++; - var weight = item.WasSeen ? 0.5 : 1.0; - var probability = Expected(ability, item.QuestionElo); - information += weight * probability * (1.0 - probability) * Math.Pow(Math.Log(10.0) / 400.0, 2); + var difficulty = ResolveDifficulty(item, question); + var weight = DifficultyWeight(difficulty); + weightSum += weight; + squaredWeightSum += weight * weight; + if (isCorrect) weightedCorrect += weight; if (!categories.TryGetValue(item.BlueprintCategory, out var category)) { category = new ExamSimulationCategoryScore(); @@ -40,51 +42,53 @@ public static class ExamSimulationScoring if (isCorrect) category.Correct++; } - var abilitySe = Math.Sqrt(1.0 / information); - var scoreSe = abilitySe * 15.0 / 200.0; - var combinedSe = Math.Sqrt(7.0 * 7.0 + 15.0 * 15.0 + scoreSe * scoreSe); - var range = (int)Math.Ceiling(combinedSe); - var passProbability = NormalCdf((predicted - 200.0) / combinedSe); + var rawAccuracy = items.Count == 0 ? 0.0 : (double)correct / items.Count; + var adjustedAccuracy = weightSum == 0.0 ? 0.0 : weightedCorrect / weightSum; + var predicted = ToScore(adjustedAccuracy); + var effectiveSampleSize = squaredWeightSum == 0.0 ? 0.0 : weightSum * weightSum / squaredWeightSum; + var accuracySe = effectiveSampleSize == 0.0 + ? 0.0 + : Math.Sqrt(adjustedAccuracy * (1.0 - adjustedAccuracy) / effectiveSampleSize); + var accuracyMargin = 1.96 * accuracySe; + var rangeLow = ToScore(Math.Clamp(adjustedAccuracy - accuracyMargin, 0.0, 1.0)); + var rangeHigh = ToScore(Math.Clamp(adjustedAccuracy + accuracyMargin, 0.0, 1.0)); + var passProbability = accuracySe == 0.0 + ? adjustedAccuracy >= PassingAccuracy ? 1.0 : 0.0 + : NormalCdf((adjustedAccuracy - PassingAccuracy) / accuracySe); return new ExamSimulationScore { PredictedScore = predicted, - RangeLow = Math.Clamp(predicted - range, 1, 300), - RangeHigh = Math.Clamp(predicted + range, 1, 300), + RangeLow = rangeLow, + RangeHigh = rangeHigh, PassProbability = Math.Round(passProbability, 4), Correct = correct, Total = items.Count, SeenCount = seen, - AbilityElo = Math.Round(ability, 1), + RawAccuracy = Math.Round(rawAccuracy, 4), + DifficultyAdjustedAccuracy = Math.Round(adjustedAccuracy, 4), + AbilityElo = null, Categories = categories }; } - private static double EstimateAbility( - IReadOnlyList items, - IReadOnlyDictionary responses, - IReadOnlyDictionary questions) + public static int ResolveDifficulty(ExamSimulationItem item, Question? question = null) { - var low = 100.0; - var high = 1900.0; - for (var iteration = 0; iteration < 80; iteration++) - { - var ability = (low + high) / 2.0; - var derivative = -(ability - PriorMean) / (PriorSd * PriorSd); - foreach (var item in items) - { - if (!questions.TryGetValue(item.QuestionExternalId, out var question)) continue; - responses.TryGetValue(item.QuestionExternalId, out var response); - var actual = response?.ChoiceId == question.CorrectChoiceId ? 1.0 : 0.0; - var weight = item.WasSeen ? 0.5 : 1.0; - derivative += weight * (actual - Expected(ability, item.QuestionElo)) * Math.Log(10.0) / 400.0; - } - if (derivative > 0) low = ability; else high = ability; - } - return (low + high) / 2.0; + if (item.QuestionDifficulty is >= 1 and <= 5) return item.QuestionDifficulty; + return question?.Difficulty is >= 1 and <= 5 ? question.Difficulty : 3; } - private static double Expected(double ability, int difficulty) => - 1.0 / (1.0 + Math.Pow(10.0, (difficulty - ability) / 400.0)); + private static double DifficultyWeight(int difficulty) => difficulty switch + { + 1 => 0.8, + 2 => 0.9, + 3 => 1.0, + 4 => 1.1, + 5 => 1.2, + _ => 1.0 + }; + + private static int ToScore(double accuracy) => + Math.Clamp((int)Math.Round(ScoreIntercept + ScoreScale * accuracy), 1, 300); private static double NormalCdf(double value) => 0.5 * (1.0 + Erf(value / Math.Sqrt(2.0))); diff --git a/api/QuestionsApi/Services/ExamSimulationScoringMigrationService.cs b/api/QuestionsApi/Services/ExamSimulationScoringMigrationService.cs new file mode 100644 index 0000000..ef1af6a --- /dev/null +++ b/api/QuestionsApi/Services/ExamSimulationScoringMigrationService.cs @@ -0,0 +1,37 @@ +namespace QuestionsApi.Services; + +public sealed class ExamSimulationScoringMigrationService : BackgroundService +{ + private readonly ExamSimulationStore _simulations; + private readonly ILogger _logger; + + public ExamSimulationScoringMigrationService( + ExamSimulationStore simulations, + ILogger logger) + { + _simulations = simulations; + _logger = logger; + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + try + { + var result = await _simulations.BackfillScoringModelAsync(stoppingToken); + _logger.LogInformation( + "Exam simulation scoring migration completed; recalculated {UpdatedCount} simulation(s)", + result.UpdatedCount); + foreach (var reference in result.MissingQuestionIds) + _logger.LogWarning( + "Exam simulation scoring migration used neutral difficulty for missing question {Reference}", + reference); + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + } + catch (Exception exception) + { + _logger.LogError(exception, "Exam simulation scoring migration failed"); + } + } +} diff --git a/api/QuestionsApi/Services/ExamSimulationStore.cs b/api/QuestionsApi/Services/ExamSimulationStore.cs index 356a5ae..d1bd2bf 100644 --- a/api/QuestionsApi/Services/ExamSimulationStore.cs +++ b/api/QuestionsApi/Services/ExamSimulationStore.cs @@ -91,6 +91,7 @@ public class ExamSimulationStore Position = offset, PresentedChoiceIds = SelectChoices(question, choicePercentages.GetValueOrDefault(question.ExternalId)), QuestionElo = elos.GetValueOrDefault(question.ExternalId, 1000), + QuestionDifficulty = question.Difficulty, WasSeen = seen.Contains(question.ExternalId), BlueprintCategory = BlueprintCategory(question) }); @@ -128,6 +129,7 @@ public class ExamSimulationStore Position = PositionWithinBlock(index), PresentedChoiceIds = SelectChoices(question, choicePercentages.GetValueOrDefault(question.ExternalId)), QuestionElo = 1000, + QuestionDifficulty = question.Difficulty, WasSeen = false, BlueprintCategory = BlueprintCategory(question) }).ToList(); @@ -173,15 +175,22 @@ public class ExamSimulationStore (item.State == ExamSimulationStates.BetweenBlocks || item.State == ExamSimulationStates.BlockActive)) .SortByDescending(item => item.UpdatedUtc).FirstOrDefaultAsync(); - public async Task GetLatestCompletedScoreAsync(string userId) => - (await _simulations.Find(item => item.UserId == userId && item.State == ExamSimulationStates.Completed && item.Score != null) - .SortByDescending(item => item.CompletedUtc).FirstOrDefaultAsync())?.Score; + public async Task GetLatestCompletedScoreAsync(string userId) + { + var simulation = await _simulations + .Find(item => item.UserId == userId && item.State == ExamSimulationStates.Completed && item.Score != null) + .SortByDescending(item => item.CompletedUtc) + .FirstOrDefaultAsync(); + return simulation is null ? null : (await UpgradeScoringModelAsync(simulation)).Score; + } public async Task GetOwnedAsync(string id, string userId) { var simulation = await _simulations.Find(item => item.Id == id && item.UserId == userId).FirstOrDefaultAsync(); if (simulation is not null && simulation.State == ExamSimulationStates.BlockActive && simulation.BlockDeadlineUtc <= DateTime.UtcNow) simulation = await SubmitBlockAsync(simulation, true); + if (simulation is not null) + simulation = await UpgradeScoringModelAsync(simulation); return simulation; } @@ -195,6 +204,7 @@ public class ExamSimulationStore var simulations = await _simulations.Find(filter).SortByDescending(item => item.CompletedUtc).Limit(limit + 1).ToListAsync(); var hasMore = simulations.Count > limit; if (hasMore) simulations.RemoveAt(simulations.Count - 1); + await UpgradeScoringModelsAsync(simulations); return new CompletedExamSimulationPage { Items = simulations.Select(item => new CompletedExamSimulationSummary @@ -261,6 +271,7 @@ public class ExamSimulationStore var simulations = await _simulations.Find(filter).Sort(sortDefinition).Limit(limit + 1).ToListAsync(); var hasMore = simulations.Count > limit; if (hasMore) simulations.RemoveAt(simulations.Count - 1); + await UpgradeScoringModelsAsync(simulations.Where(item => item.State == ExamSimulationStates.Completed)); var last = simulations.LastOrDefault(); return new ExamSimulationHistoryPage { @@ -442,7 +453,9 @@ public class ExamSimulationStore { var questionMap = await _questions.GetByExternalIdsAsync(simulation.Items.Select(item => item.QuestionExternalId)); var responses = simulation.Responses.ToDictionary(item => item.QuestionExternalId, StringComparer.Ordinal); + SnapshotDifficulties(simulation, questionMap); simulation.Score = ExamSimulationScoring.Calculate(simulation.Items, responses, questionMap); + simulation.ModelVersion = ExamSimulationScoring.ModelVersion; simulation.State = ExamSimulationStates.Completed; simulation.CompletedUtc = DateTime.UtcNow; simulation.UpdatedUtc = DateTime.UtcNow; @@ -480,6 +493,71 @@ public class ExamSimulationStore private Task ReplaceAsync(ExamSimulation simulation) => _simulations.ReplaceOneAsync(item => item.Id == simulation.Id && item.UserId == simulation.UserId, simulation); + public async Task BackfillScoringModelAsync(CancellationToken cancellationToken = default) + { + var filter = Builders.Filter.Eq(item => item.State, ExamSimulationStates.Completed) & + Builders.Filter.Ne(item => item.ModelVersion, ExamSimulationScoring.ModelVersion); + var simulations = await _simulations.Find(filter).ToListAsync(cancellationToken); + var missingQuestionIds = new List(); + var updated = 0; + foreach (var simulation in simulations) + { + cancellationToken.ThrowIfCancellationRequested(); + var questionMap = await _questions.GetByExternalIdsAsync(simulation.Items.Select(item => item.QuestionExternalId)); + missingQuestionIds.AddRange(simulation.Items + .Where(item => !questionMap.ContainsKey(item.QuestionExternalId)) + .Select(item => $"{simulation.Id}:{item.QuestionExternalId}")); + SnapshotDifficulties(simulation, questionMap); + var responses = simulation.Responses.ToDictionary(item => item.QuestionExternalId, StringComparer.Ordinal); + simulation.Score = ExamSimulationScoring.Calculate(simulation.Items, responses, questionMap); + simulation.ModelVersion = ExamSimulationScoring.ModelVersion; + simulation.UpdatedUtc = DateTime.UtcNow; + await ReplaceAsync(simulation); + updated++; + } + return new ExamSimulationBackfillResult(updated, missingQuestionIds); + } + + private async Task UpgradeScoringModelsAsync(IEnumerable simulations) + { + foreach (var simulation in simulations) + await UpgradeScoringModelAsync(simulation); + } + + private async Task UpgradeScoringModelAsync(ExamSimulation simulation) + { + var needsDifficultySnapshot = simulation.Items.Any(item => item.QuestionDifficulty is < 1 or > 5); + var needsScoreUpgrade = simulation.State == ExamSimulationStates.Completed && + simulation.ModelVersion != ExamSimulationScoring.ModelVersion; + if (!needsDifficultySnapshot && !needsScoreUpgrade) + return simulation; + + var questionMap = await _questions.GetByExternalIdsAsync(simulation.Items.Select(item => item.QuestionExternalId)); + SnapshotDifficulties(simulation, questionMap); + if (simulation.State == ExamSimulationStates.Completed) + { + var responses = simulation.Responses.ToDictionary(item => item.QuestionExternalId, StringComparer.Ordinal); + simulation.Score = ExamSimulationScoring.Calculate(simulation.Items, responses, questionMap); + } + simulation.ModelVersion = ExamSimulationScoring.ModelVersion; + simulation.UpdatedUtc = DateTime.UtcNow; + await ReplaceAsync(simulation); + return simulation; + } + + private static void SnapshotDifficulties( + ExamSimulation simulation, + IReadOnlyDictionary questions) + { + foreach (var item in simulation.Items) + { + questions.TryGetValue(item.QuestionExternalId, out var question); + item.QuestionDifficulty = ExamSimulationScoring.ResolveDifficulty(item, question); + } + } + + public sealed record ExamSimulationBackfillResult(int UpdatedCount, IReadOnlyList MissingQuestionIds); + private static int BlockIndexForPosition(int position) { var consumed = 0; diff --git a/scripts/ui/ccs_hub.gd b/scripts/ui/ccs_hub.gd index 7a540a2..ba79569 100644 --- a/scripts/ui/ccs_hub.gd +++ b/scripts/ui/ccs_hub.gd @@ -297,7 +297,7 @@ func _build_performance_summary(dark: bool) -> void: if combined != null: prediction_value = "%d" % int(combined) prediction_detail = "Range %d-%d" % [int(CCSService.prediction.get("rangeLow", 0)), int(CCSService.prediction.get("rangeHigh", 0))] - _add_metric(row, "Combined Prediction", prediction_value, prediction_detail, dark) + _add_metric(row, "Provisional Combined Estimate", prediction_value, prediction_detail, dark) func _add_metric(parent: BoxContainer, caption: String, value: String, detail: String, dark: bool) -> void: diff --git a/scripts/ui/dashboard_screen.gd b/scripts/ui/dashboard_screen.gd index a97ba79..59db4f7 100644 --- a/scripts/ui/dashboard_screen.gd +++ b/scripts/ui/dashboard_screen.gd @@ -94,7 +94,7 @@ func _build() -> void: readiness_segment_bar = DashboardSegmentBar.new() readiness_segment_bar.custom_minimum_size = Vector2(0, 10) _add_stat_card("readiness", "Exam Readiness", "chart-no-axes-column-increasing", Brand.PURPLE, readiness_segment_bar, - "Predicted pass probability from your most recent completed exam simulation.") + "Heuristic pass probability from your most recent completed exam simulation.") accuracy_progress_bar = ProgressBar.new() accuracy_progress_bar.show_percentage = false @@ -254,8 +254,8 @@ func _render_readiness_card(data: Dictionary) -> void: var label := str(label_value) var pass_percent := int(data.get("examReadinessProjectedPassPercent", 0)) var status_color := Brand.SUCCESS if label == "High" else (Brand.AMBER if label == "Medium" else Brand.MAGENTA) - var caption := "You're on track!" if label != "Low" else "Keep practicing" - _set_stat_card("readiness", label, "Projected pass: %d%%" % pass_percent, status_color, caption) + var caption := "Provisional estimate" if label != "Low" else "Keep practicing" + _set_stat_card("readiness", label, "Heuristic pass: %d%%" % pass_percent, status_color, caption) (metric_values.get("readiness") as Label).add_theme_color_override("font_color", status_color) readiness_segment_bar.set_progress(int(round(pass_percent / 100.0 * 4.0)), 4, status_color, dark) diff --git a/scripts/ui/simulation_results.gd b/scripts/ui/simulation_results.gd index 446b804..d2a9250 100644 --- a/scripts/ui/simulation_results.gd +++ b/scripts/ui/simulation_results.gd @@ -67,9 +67,12 @@ func on_show(payload: Variant = null) -> void: var low := int(score.get("rangeLow", 0)) var high := int(score.get("rangeHigh", 0)) var probability := int(round(float(score.get("passProbability", 0.0)) * 100.0)) - score_label.text = "Predicted Step 3 score: %d (%d-%d)" % [predicted, low, high] - summary_label.text = "%d/%d correct | Estimated probability of reaching 200: %d%% | Previously seen questions: %d\nThis provisional educational estimate is not an official USMLE or NBME score and excludes CCS performance." % [ - int(score.get("correct", 0)), int(score.get("total", 0)), probability, int(score.get("seenCount", 0))] + var raw_accuracy := float(score.get("rawAccuracy", 0.0)) * 100.0 + var adjusted_accuracy := float(score.get("difficultyAdjustedAccuracy", 0.0)) * 100.0 + score_label.text = "Provisional Step 3 estimate: %d (%d-%d)" % [predicted, low, high] + summary_label.text = "%d/%d correct (%.1f%%) | Difficulty-adjusted accuracy: %.1f%%\nHeuristic probability of provisional score >=200: %d%% | Previously seen questions: %d\nThis transparent, unvalidated educational heuristic is not an official USMLE or NBME score and excludes CCS performance." % [ + int(score.get("correct", 0)), int(score.get("total", 0)), raw_accuracy, adjusted_accuracy, + probability, int(score.get("seenCount", 0))] _clear(category_list) for category in score.get("categories", {}): var value: Dictionary = score["categories"][category]