Add full-length Step 3 Day 1 simulation
This commit is contained in:
@@ -0,0 +1,87 @@
|
||||
using QuestionsApi.Models;
|
||||
using QuestionsApi.Services;
|
||||
using Xunit;
|
||||
|
||||
namespace QuestionsApi.Tests;
|
||||
|
||||
public sealed class ExamSimulationTests
|
||||
{
|
||||
[Fact]
|
||||
public void DayOneFormHasOfficialItemAndBlockCounts()
|
||||
{
|
||||
Assert.Equal(232, ExamSimulationStore.BlockSizes.Sum());
|
||||
Assert.Equal(12, ExamSimulationStore.BlockSizes.Length);
|
||||
Assert.All(ExamSimulationStore.BlockSizes, size => Assert.InRange(size, 18, 20));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BlueprintTargetsFillTheCompleteForm()
|
||||
{
|
||||
var targets = ExamSimulationStore.BuildTargets();
|
||||
Assert.Equal(ExamSimulationStore.RequiredQuestions, targets.Values.Sum());
|
||||
Assert.Equal(14, targets.Count);
|
||||
Assert.All(targets.Values, count => Assert.True(count > 0));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void MoreCorrectAnswersProduceAHigherPrediction()
|
||||
{
|
||||
var items = BuildItems(100);
|
||||
var questions = BuildQuestions(100);
|
||||
var allCorrect = items.ToDictionary(
|
||||
item => item.QuestionExternalId,
|
||||
item => new ExamSimulationResponse { QuestionExternalId = item.QuestionExternalId, ChoiceId = "A" });
|
||||
var allWrong = items.ToDictionary(
|
||||
item => item.QuestionExternalId,
|
||||
item => new ExamSimulationResponse { QuestionExternalId = item.QuestionExternalId, ChoiceId = "B" });
|
||||
|
||||
var passing = ExamSimulationScoring.Calculate(items, allCorrect, questions);
|
||||
var failing = ExamSimulationScoring.Calculate(items, allWrong, questions);
|
||||
|
||||
Assert.True(passing.PredictedScore > failing.PredictedScore);
|
||||
Assert.True(passing.PassProbability > failing.PassProbability);
|
||||
Assert.Equal(100, passing.Correct);
|
||||
Assert.Equal(0, failing.Correct);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SeenItemsAreReportedAndCarryLessScoringWeight()
|
||||
{
|
||||
var questions = BuildQuestions(20);
|
||||
var unseen = BuildItems(20);
|
||||
var seen = BuildItems(20, wasSeen: true);
|
||||
var responses = unseen.ToDictionary(
|
||||
item => item.QuestionExternalId,
|
||||
item => new ExamSimulationResponse { QuestionExternalId = item.QuestionExternalId, ChoiceId = "A" });
|
||||
|
||||
var unseenScore = ExamSimulationScoring.Calculate(unseen, responses, questions);
|
||||
var seenScore = ExamSimulationScoring.Calculate(seen, responses, questions);
|
||||
|
||||
Assert.Equal(0, unseenScore.SeenCount);
|
||||
Assert.Equal(20, seenScore.SeenCount);
|
||||
Assert.True(unseenScore.AbilityElo > seenScore.AbilityElo);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("Cardiovascular System", "Cardiovascular")]
|
||||
[InlineData("Professionalism and Ethics", "Social Sciences")]
|
||||
[InlineData("Endocrine, Diabetes & Metabolism", "Endocrine")]
|
||||
public void QuestionsMapToExpectedBlueprintCategory(string system, string expected)
|
||||
{
|
||||
Assert.Equal(expected, ExamSimulationStore.BlueprintCategory(new Question { System = system }));
|
||||
}
|
||||
|
||||
private static List<ExamSimulationItem> BuildItems(int count, bool wasSeen = false) =>
|
||||
Enumerable.Range(0, count).Select(index => new ExamSimulationItem
|
||||
{
|
||||
QuestionExternalId = $"question-{index}",
|
||||
QuestionElo = 1000,
|
||||
WasSeen = wasSeen,
|
||||
BlueprintCategory = "Cardiovascular"
|
||||
}).ToList();
|
||||
|
||||
private static Dictionary<string, Question> BuildQuestions(int count) =>
|
||||
Enumerable.Range(0, count).ToDictionary(
|
||||
index => $"question-{index}",
|
||||
index => new Question { ExternalId = $"question-{index}", CorrectChoiceId = "A" });
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
using Microsoft.AspNetCore.Authorization;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using QuestionsApi.Models;
|
||||
using QuestionsApi.Services;
|
||||
using System.Security.Claims;
|
||||
|
||||
namespace QuestionsApi.Controllers;
|
||||
|
||||
[ApiController]
|
||||
[Route("api/exam-simulations")]
|
||||
[Authorize(Roles = "USER,SUPER")]
|
||||
public class ExamSimulationsController(ExamSimulationStore simulations) : ControllerBase
|
||||
{
|
||||
[HttpGet("readiness")]
|
||||
public async Task<IActionResult> Readiness() => Ok(await simulations.GetReadinessAsync(UserId()));
|
||||
|
||||
[HttpPost]
|
||||
public async Task<IActionResult> Create()
|
||||
{
|
||||
try
|
||||
{
|
||||
var simulation = await simulations.CreateAsync(UserId());
|
||||
return Ok(await simulations.ToResponseAsync(simulation));
|
||||
}
|
||||
catch (InvalidOperationException exception) { return Conflict(exception.Message); }
|
||||
}
|
||||
|
||||
[HttpGet("active")]
|
||||
public async Task<IActionResult> Active()
|
||||
{
|
||||
var simulation = await simulations.FindActiveAsync(UserId());
|
||||
return simulation is null ? NoContent() : Ok(await simulations.ToResponseAsync(simulation));
|
||||
}
|
||||
|
||||
[HttpGet("{id}")]
|
||||
public async Task<IActionResult> Get(string id)
|
||||
{
|
||||
var simulation = await simulations.GetOwnedAsync(id, UserId());
|
||||
return simulation is null ? NotFound() : Ok(await simulations.ToResponseAsync(simulation));
|
||||
}
|
||||
|
||||
[HttpPost("{id}/blocks/start")]
|
||||
public async Task<IActionResult> StartBlock(string id)
|
||||
{
|
||||
var simulation = await simulations.GetOwnedAsync(id, UserId());
|
||||
if (simulation is null) return NotFound();
|
||||
try { return Ok(await simulations.ToResponseAsync(await simulations.StartNextBlockAsync(simulation))); }
|
||||
catch (InvalidOperationException exception) { return Conflict(exception.Message); }
|
||||
}
|
||||
|
||||
[HttpPut("{id}/responses")]
|
||||
public async Task<IActionResult> SaveResponse(string id, SaveSimulationResponseRequest request)
|
||||
{
|
||||
var simulation = await simulations.GetOwnedAsync(id, UserId());
|
||||
if (simulation is null) return NotFound();
|
||||
try { return Ok(await simulations.ToResponseAsync(await simulations.SaveResponseAsync(simulation, request))); }
|
||||
catch (ArgumentException exception) { return BadRequest(exception.Message); }
|
||||
catch (InvalidOperationException exception) { return Conflict(exception.Message); }
|
||||
}
|
||||
|
||||
[HttpPost("{id}/blocks/submit")]
|
||||
public async Task<IActionResult> SubmitBlock(string id)
|
||||
{
|
||||
var simulation = await simulations.GetOwnedAsync(id, UserId());
|
||||
if (simulation is null) return NotFound();
|
||||
try { return Ok(await simulations.ToResponseAsync(await simulations.SubmitBlockAsync(simulation))); }
|
||||
catch (InvalidOperationException exception) { return Conflict(exception.Message); }
|
||||
}
|
||||
|
||||
[HttpPost("{id}/abandon")]
|
||||
public async Task<IActionResult> Abandon(string id)
|
||||
{
|
||||
var simulation = await simulations.GetOwnedAsync(id, UserId());
|
||||
if (simulation is null) return NotFound();
|
||||
await simulations.AbandonAsync(simulation);
|
||||
return NoContent();
|
||||
}
|
||||
|
||||
private string UserId() => User.FindFirstValue(ClaimTypes.NameIdentifier)
|
||||
?? User.FindFirstValue(ClaimTypes.Name)
|
||||
?? User.Identity?.Name
|
||||
?? throw new UnauthorizedAccessException();
|
||||
}
|
||||
|
||||
[ApiController]
|
||||
[Route("api/step3-outcomes")]
|
||||
[Authorize(Roles = "USER,SUPER")]
|
||||
public class Step3OutcomesController(ExamSimulationStore simulations) : ControllerBase
|
||||
{
|
||||
[HttpGet("me")]
|
||||
public async Task<IActionResult> Get()
|
||||
{
|
||||
var outcome = await simulations.GetOutcomeAsync(UserId());
|
||||
return outcome is null ? NoContent() : Ok(outcome);
|
||||
}
|
||||
|
||||
[HttpPut("me")]
|
||||
public async Task<IActionResult> Put(UpsertStep3OutcomeRequest request)
|
||||
{
|
||||
try { return Ok(await simulations.UpsertOutcomeAsync(UserId(), request)); }
|
||||
catch (ArgumentException exception) { return BadRequest(exception.Message); }
|
||||
}
|
||||
|
||||
[HttpDelete("me")]
|
||||
public async Task<IActionResult> Delete()
|
||||
{
|
||||
await simulations.DeleteOutcomeAsync(UserId());
|
||||
return NoContent();
|
||||
}
|
||||
|
||||
private string UserId() => User.FindFirstValue(ClaimTypes.NameIdentifier)
|
||||
?? User.FindFirstValue(ClaimTypes.Name)
|
||||
?? User.Identity?.Name
|
||||
?? throw new UnauthorizedAccessException();
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Bson.Serialization.Attributes;
|
||||
using QuestionsApi.Services;
|
||||
|
||||
namespace QuestionsApi.Models;
|
||||
|
||||
public static class ExamSimulationStates
|
||||
{
|
||||
public const string BetweenBlocks = "between_blocks";
|
||||
public const string BlockActive = "block_active";
|
||||
public const string Completed = "completed";
|
||||
public const string Abandoned = "abandoned";
|
||||
}
|
||||
|
||||
public class ExamSimulation
|
||||
{
|
||||
[BsonId]
|
||||
[BsonRepresentation(BsonType.ObjectId)]
|
||||
public string? Id { get; set; }
|
||||
public string UserId { get; set; } = "";
|
||||
public string State { get; set; } = ExamSimulationStates.BetweenBlocks;
|
||||
public string ModelVersion { get; set; } = ExamSimulationScoring.ModelVersion;
|
||||
public List<ExamSimulationItem> Items { get; set; } = [];
|
||||
public List<ExamSimulationResponse> Responses { get; set; } = [];
|
||||
public int CurrentBlock { get; set; } = -1;
|
||||
public int BreakSecondsRemaining { get; set; } = 3300;
|
||||
public DateTime? BlockDeadlineUtc { get; set; }
|
||||
public DateTime CreatedUtc { get; set; } = DateTime.UtcNow;
|
||||
public DateTime UpdatedUtc { get; set; } = DateTime.UtcNow;
|
||||
public DateTime? CompletedUtc { get; set; }
|
||||
public ExamSimulationScore? Score { get; set; }
|
||||
public Dictionary<string, int> BlueprintTargets { get; set; } = [];
|
||||
public Dictionary<string, int> BlueprintActual { get; set; } = [];
|
||||
}
|
||||
|
||||
public class ExamSimulationItem
|
||||
{
|
||||
public string QuestionExternalId { get; set; } = "";
|
||||
public int BlockIndex { get; set; }
|
||||
public int Position { get; set; }
|
||||
public List<string> PresentedChoiceIds { get; set; } = [];
|
||||
public int QuestionElo { get; set; } = 1000;
|
||||
public bool WasSeen { get; set; }
|
||||
public string BlueprintCategory { get; set; } = "Other";
|
||||
}
|
||||
|
||||
public class ExamSimulationResponse
|
||||
{
|
||||
public string QuestionExternalId { get; set; } = "";
|
||||
public string ChoiceId { get; set; } = "";
|
||||
public List<string> EliminatedChoiceIds { get; set; } = [];
|
||||
public bool Flagged { get; set; }
|
||||
public int ElapsedSeconds { get; set; }
|
||||
public DateTime UpdatedUtc { get; set; } = DateTime.UtcNow;
|
||||
}
|
||||
|
||||
public class ExamSimulationScore
|
||||
{
|
||||
public int PredictedScore { get; set; }
|
||||
public int RangeLow { get; set; }
|
||||
public int RangeHigh { get; set; }
|
||||
public double PassProbability { get; set; }
|
||||
public int Correct { get; set; }
|
||||
public int Total { get; set; }
|
||||
public int SeenCount { get; set; }
|
||||
public double AbilityElo { get; set; }
|
||||
public Dictionary<string, ExamSimulationCategoryScore> Categories { get; set; } = [];
|
||||
}
|
||||
|
||||
public class ExamSimulationCategoryScore
|
||||
{
|
||||
public int Correct { get; set; }
|
||||
public int Total { get; set; }
|
||||
}
|
||||
|
||||
public class SaveSimulationResponseRequest
|
||||
{
|
||||
public string QuestionExternalId { get; set; } = "";
|
||||
public string ChoiceId { get; set; } = "";
|
||||
public List<string> EliminatedChoiceIds { get; set; } = [];
|
||||
public bool Flagged { get; set; }
|
||||
public int ElapsedSeconds { get; set; }
|
||||
}
|
||||
|
||||
public class ExamSimulationQuestionResponse
|
||||
{
|
||||
public string ExternalId { get; set; } = "";
|
||||
public string Stem { get; set; } = "";
|
||||
public string LeadIn { get; set; } = "";
|
||||
public List<QuestionChoice> Choices { get; set; } = [];
|
||||
public string Discipline { get; set; } = "";
|
||||
public string System { get; set; } = "";
|
||||
public string PerformanceCategory { get; set; } = "";
|
||||
public List<QuestionLab> Labs { get; set; } = [];
|
||||
public List<string> Media { get; set; } = [];
|
||||
}
|
||||
|
||||
public class ExamSimulationStateResponse
|
||||
{
|
||||
public string Id { get; set; } = "";
|
||||
public string State { get; set; } = "";
|
||||
public int CurrentBlock { get; set; }
|
||||
public int TotalBlocks { get; set; } = 12;
|
||||
public int BreakSecondsRemaining { get; set; }
|
||||
public DateTime? BlockDeadlineUtc { get; set; }
|
||||
public List<ExamSimulationQuestionResponse> Questions { get; set; } = [];
|
||||
public List<ExamSimulationResponse> Responses { get; set; } = [];
|
||||
public ExamSimulationScore? Score { get; set; }
|
||||
public Dictionary<string, int> BlueprintTargets { get; set; } = [];
|
||||
public Dictionary<string, int> BlueprintActual { get; set; } = [];
|
||||
public List<QuestionResponse> ReviewQuestions { get; set; } = [];
|
||||
}
|
||||
|
||||
public class ExamSimulationReadinessResponse
|
||||
{
|
||||
public bool Ready { get; set; }
|
||||
public int AvailableQuestions { get; set; }
|
||||
public int RequiredQuestions { get; set; } = 232;
|
||||
public bool HasActiveSimulation { get; set; }
|
||||
}
|
||||
|
||||
public class Step3Outcome
|
||||
{
|
||||
[BsonId]
|
||||
[BsonRepresentation(BsonType.ObjectId)]
|
||||
public string? Id { get; set; }
|
||||
public string UserId { get; set; } = "";
|
||||
public int OfficialScore { get; set; }
|
||||
public DateTime ExamDate { get; set; }
|
||||
public DateTime ConsentUtc { get; set; }
|
||||
public DateTime UpdatedUtc { get; set; }
|
||||
}
|
||||
|
||||
public class UpsertStep3OutcomeRequest
|
||||
{
|
||||
public int OfficialScore { get; set; }
|
||||
public DateTime ExamDate { get; set; }
|
||||
public bool Consent { get; set; }
|
||||
}
|
||||
@@ -16,6 +16,7 @@ builder.Services.AddSingleton<AnswerStatisticsStore>();
|
||||
builder.Services.AddSingleton<RatingStore>();
|
||||
builder.Services.AddSingleton<QuestionMediaStore>();
|
||||
builder.Services.AddSingleton<QuestionReportStore>();
|
||||
builder.Services.AddSingleton<ExamSimulationStore>();
|
||||
builder.Services.AddSingleton<ServiceMetrics>();
|
||||
builder.Services.AddHostedService<AnswerStatisticsMaintenanceService>();
|
||||
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
using QuestionsApi.Models;
|
||||
|
||||
namespace QuestionsApi.Services;
|
||||
|
||||
public static class ExamSimulationScoring
|
||||
{
|
||||
public const string ModelVersion = "provisional-1";
|
||||
private const double PriorMean = 1000.0;
|
||||
private const double PriorSd = 400.0;
|
||||
|
||||
public static ExamSimulationScore Calculate(
|
||||
IReadOnlyList<ExamSimulationItem> items,
|
||||
IReadOnlyDictionary<string, ExamSimulationResponse> responses,
|
||||
IReadOnlyDictionary<string, Question> questions)
|
||||
{
|
||||
var ability = EstimateAbility(items, responses, questions);
|
||||
var predicted = Math.Clamp((int)Math.Round(227.0 + (ability - 1000.0) * 15.0 / 200.0), 1, 300);
|
||||
var information = 1.0 / (PriorSd * PriorSd);
|
||||
var correct = 0;
|
||||
var seen = 0;
|
||||
var categories = new Dictionary<string, ExamSimulationCategoryScore>();
|
||||
|
||||
foreach (var item in items)
|
||||
{
|
||||
if (!questions.TryGetValue(item.QuestionExternalId, out var question))
|
||||
continue;
|
||||
responses.TryGetValue(item.QuestionExternalId, out var response);
|
||||
var isCorrect = response?.ChoiceId == question.CorrectChoiceId;
|
||||
if (isCorrect) correct++;
|
||||
if (item.WasSeen) seen++;
|
||||
var weight = item.WasSeen ? 0.5 : 1.0;
|
||||
var probability = Expected(ability, item.QuestionElo);
|
||||
information += weight * probability * (1.0 - probability) * Math.Pow(Math.Log(10.0) / 400.0, 2);
|
||||
if (!categories.TryGetValue(item.BlueprintCategory, out var category))
|
||||
{
|
||||
category = new ExamSimulationCategoryScore();
|
||||
categories[item.BlueprintCategory] = category;
|
||||
}
|
||||
category.Total++;
|
||||
if (isCorrect) category.Correct++;
|
||||
}
|
||||
|
||||
var abilitySe = Math.Sqrt(1.0 / information);
|
||||
var scoreSe = abilitySe * 15.0 / 200.0;
|
||||
var combinedSe = Math.Sqrt(7.0 * 7.0 + 15.0 * 15.0 + scoreSe * scoreSe);
|
||||
var range = (int)Math.Ceiling(combinedSe);
|
||||
var passProbability = NormalCdf((predicted - 200.0) / combinedSe);
|
||||
return new ExamSimulationScore
|
||||
{
|
||||
PredictedScore = predicted,
|
||||
RangeLow = Math.Clamp(predicted - range, 1, 300),
|
||||
RangeHigh = Math.Clamp(predicted + range, 1, 300),
|
||||
PassProbability = Math.Round(passProbability, 4),
|
||||
Correct = correct,
|
||||
Total = items.Count,
|
||||
SeenCount = seen,
|
||||
AbilityElo = Math.Round(ability, 1),
|
||||
Categories = categories
|
||||
};
|
||||
}
|
||||
|
||||
private static double EstimateAbility(
|
||||
IReadOnlyList<ExamSimulationItem> items,
|
||||
IReadOnlyDictionary<string, ExamSimulationResponse> responses,
|
||||
IReadOnlyDictionary<string, Question> questions)
|
||||
{
|
||||
var low = 100.0;
|
||||
var high = 1900.0;
|
||||
for (var iteration = 0; iteration < 80; iteration++)
|
||||
{
|
||||
var ability = (low + high) / 2.0;
|
||||
var derivative = -(ability - PriorMean) / (PriorSd * PriorSd);
|
||||
foreach (var item in items)
|
||||
{
|
||||
if (!questions.TryGetValue(item.QuestionExternalId, out var question)) continue;
|
||||
responses.TryGetValue(item.QuestionExternalId, out var response);
|
||||
var actual = response?.ChoiceId == question.CorrectChoiceId ? 1.0 : 0.0;
|
||||
var weight = item.WasSeen ? 0.5 : 1.0;
|
||||
derivative += weight * (actual - Expected(ability, item.QuestionElo)) * Math.Log(10.0) / 400.0;
|
||||
}
|
||||
if (derivative > 0) low = ability; else high = ability;
|
||||
}
|
||||
return (low + high) / 2.0;
|
||||
}
|
||||
|
||||
private static double Expected(double ability, int difficulty) =>
|
||||
1.0 / (1.0 + Math.Pow(10.0, (difficulty - ability) / 400.0));
|
||||
|
||||
private static double NormalCdf(double value) => 0.5 * (1.0 + Erf(value / Math.Sqrt(2.0)));
|
||||
|
||||
private static double Erf(double value)
|
||||
{
|
||||
var sign = value < 0 ? -1.0 : 1.0;
|
||||
value = Math.Abs(value);
|
||||
const double p = 0.3275911;
|
||||
var t = 1.0 / (1.0 + p * value);
|
||||
var y = 1.0 - (((((1.061405429 * t - 1.453152027) * t) + 1.421413741) * t - 0.284496736) * t + 0.254829592) * t * Math.Exp(-value * value);
|
||||
return sign * y;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,333 @@
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Driver;
|
||||
using QuestionsApi.Models;
|
||||
|
||||
namespace QuestionsApi.Services;
|
||||
|
||||
public class ExamSimulationStore
|
||||
{
|
||||
public const int RequiredQuestions = 232;
|
||||
public static readonly int[] BlockSizes = [20, 19, 19, 20, 19, 19, 20, 19, 19, 20, 19, 19];
|
||||
private readonly IMongoCollection<ExamSimulation> _simulations;
|
||||
private readonly IMongoCollection<Step3Outcome> _outcomes;
|
||||
private readonly QuestionStore _questions;
|
||||
private readonly QuestionAttemptStore _attempts;
|
||||
private readonly AnswerStatisticsStore _statistics;
|
||||
private readonly RatingStore _ratings;
|
||||
private readonly ServiceMetrics _metrics;
|
||||
|
||||
public ExamSimulationStore(
|
||||
MongoDatabaseProvider mongo,
|
||||
QuestionStore questions,
|
||||
QuestionAttemptStore attempts,
|
||||
AnswerStatisticsStore statistics,
|
||||
RatingStore ratings,
|
||||
ServiceMetrics metrics)
|
||||
{
|
||||
_simulations = mongo.Database.GetCollection<ExamSimulation>("ExamSimulations");
|
||||
_outcomes = mongo.Database.GetCollection<Step3Outcome>("Step3Outcomes");
|
||||
_questions = questions;
|
||||
_attempts = attempts;
|
||||
_statistics = statistics;
|
||||
_ratings = ratings;
|
||||
_metrics = metrics;
|
||||
_simulations.Indexes.CreateOne(new CreateIndexModel<ExamSimulation>(
|
||||
Builders<ExamSimulation>.IndexKeys.Ascending(item => item.UserId).Ascending(item => item.State)));
|
||||
_simulations.Indexes.CreateOne(new CreateIndexModel<ExamSimulation>(
|
||||
Builders<ExamSimulation>.IndexKeys.Ascending(item => item.UpdatedUtc)));
|
||||
_outcomes.Indexes.CreateOne(new CreateIndexModel<Step3Outcome>(
|
||||
Builders<Step3Outcome>.IndexKeys.Ascending(item => item.UserId),
|
||||
new CreateIndexOptions { Unique = true }));
|
||||
}
|
||||
|
||||
public async Task<ExamSimulationReadinessResponse> GetReadinessAsync(string userId)
|
||||
{
|
||||
var count = (await _questions.GetPublishedAsync()).Count(question => question.Choices.Count >= 5);
|
||||
return new ExamSimulationReadinessResponse
|
||||
{
|
||||
Ready = count >= RequiredQuestions,
|
||||
AvailableQuestions = count,
|
||||
HasActiveSimulation = await FindActiveAsync(userId) is not null
|
||||
};
|
||||
}
|
||||
|
||||
public async Task<ExamSimulation> CreateAsync(string userId)
|
||||
{
|
||||
if (await FindActiveAsync(userId) is not null)
|
||||
throw new InvalidOperationException("An active Step 3 simulation already exists.");
|
||||
var bank = (await _questions.GetPublishedAsync()).Where(question => question.Choices.Count >= 5).ToList();
|
||||
if (bank.Count < RequiredQuestions)
|
||||
throw new InvalidOperationException($"At least {RequiredQuestions} published questions are required.");
|
||||
var seen = (await _attempts.GetUserQuestionIdsAsync(userId)).ToHashSet(StringComparer.Ordinal);
|
||||
var elos = await _ratings.GetQuestionElosAsync(bank.Select(question => question.ExternalId));
|
||||
var targets = BuildTargets();
|
||||
var selected = SelectForm(bank, seen, targets);
|
||||
var items = new List<ExamSimulationItem>();
|
||||
var position = 0;
|
||||
for (var block = 0; block < BlockSizes.Length; block++)
|
||||
{
|
||||
for (var offset = 0; offset < BlockSizes[block]; offset++)
|
||||
{
|
||||
var question = selected[position];
|
||||
items.Add(new ExamSimulationItem
|
||||
{
|
||||
QuestionExternalId = question.ExternalId,
|
||||
BlockIndex = block,
|
||||
Position = offset,
|
||||
PresentedChoiceIds = SelectChoices(question),
|
||||
QuestionElo = elos.GetValueOrDefault(question.ExternalId, 1000),
|
||||
WasSeen = seen.Contains(question.ExternalId),
|
||||
BlueprintCategory = BlueprintCategory(question)
|
||||
});
|
||||
position++;
|
||||
}
|
||||
}
|
||||
var simulation = new ExamSimulation
|
||||
{
|
||||
UserId = userId,
|
||||
Items = items,
|
||||
BlueprintTargets = targets,
|
||||
BlueprintActual = items.GroupBy(item => item.BlueprintCategory).ToDictionary(group => group.Key, group => group.Count())
|
||||
};
|
||||
await _simulations.InsertOneAsync(simulation);
|
||||
return simulation;
|
||||
}
|
||||
|
||||
public async Task<ExamSimulation?> FindActiveAsync(string userId) =>
|
||||
await _simulations.Find(item => item.UserId == userId &&
|
||||
(item.State == ExamSimulationStates.BetweenBlocks || item.State == ExamSimulationStates.BlockActive))
|
||||
.SortByDescending(item => item.UpdatedUtc).FirstOrDefaultAsync();
|
||||
|
||||
public async Task<ExamSimulation?> GetOwnedAsync(string id, string userId)
|
||||
{
|
||||
var simulation = await _simulations.Find(item => item.Id == id && item.UserId == userId).FirstOrDefaultAsync();
|
||||
if (simulation is not null && simulation.State == ExamSimulationStates.BlockActive && simulation.BlockDeadlineUtc <= DateTime.UtcNow)
|
||||
simulation = await SubmitBlockAsync(simulation, true);
|
||||
return simulation;
|
||||
}
|
||||
|
||||
public async Task<ExamSimulation> StartNextBlockAsync(ExamSimulation simulation)
|
||||
{
|
||||
if (simulation.State != ExamSimulationStates.BetweenBlocks)
|
||||
throw new InvalidOperationException("The simulation is not between blocks.");
|
||||
if (simulation.CurrentBlock + 1 >= BlockSizes.Length)
|
||||
throw new InvalidOperationException("All blocks are complete.");
|
||||
simulation.CurrentBlock++;
|
||||
simulation.State = ExamSimulationStates.BlockActive;
|
||||
simulation.BlockDeadlineUtc = DateTime.UtcNow.AddMinutes(30);
|
||||
simulation.UpdatedUtc = DateTime.UtcNow;
|
||||
await ReplaceAsync(simulation);
|
||||
return simulation;
|
||||
}
|
||||
|
||||
public async Task<ExamSimulation> SaveResponseAsync(ExamSimulation simulation, SaveSimulationResponseRequest request)
|
||||
{
|
||||
if (simulation.State != ExamSimulationStates.BlockActive || simulation.BlockDeadlineUtc <= DateTime.UtcNow)
|
||||
throw new InvalidOperationException("The active block has ended.");
|
||||
var item = simulation.Items.FirstOrDefault(candidate => candidate.BlockIndex == simulation.CurrentBlock && candidate.QuestionExternalId == request.QuestionExternalId)
|
||||
?? throw new ArgumentException("Question is not in the active block.");
|
||||
if (!item.PresentedChoiceIds.Contains(request.ChoiceId, StringComparer.Ordinal) && request.ChoiceId != "")
|
||||
throw new ArgumentException("Choice is not presented for this question.");
|
||||
var response = simulation.Responses.FirstOrDefault(candidate => candidate.QuestionExternalId == request.QuestionExternalId);
|
||||
if (response is null)
|
||||
{
|
||||
response = new ExamSimulationResponse { QuestionExternalId = request.QuestionExternalId };
|
||||
simulation.Responses.Add(response);
|
||||
}
|
||||
response.ChoiceId = request.ChoiceId;
|
||||
response.EliminatedChoiceIds = request.EliminatedChoiceIds.Where(item.PresentedChoiceIds.Contains).Distinct().ToList();
|
||||
response.Flagged = request.Flagged;
|
||||
response.ElapsedSeconds = Math.Clamp(request.ElapsedSeconds, 0, 1800);
|
||||
response.UpdatedUtc = DateTime.UtcNow;
|
||||
simulation.UpdatedUtc = DateTime.UtcNow;
|
||||
await ReplaceAsync(simulation);
|
||||
return simulation;
|
||||
}
|
||||
|
||||
public async Task<ExamSimulation> SubmitBlockAsync(ExamSimulation simulation, bool expired = false)
|
||||
{
|
||||
if (simulation.State != ExamSimulationStates.BlockActive)
|
||||
throw new InvalidOperationException("There is no active block to submit.");
|
||||
if (!expired && simulation.BlockDeadlineUtc is not null)
|
||||
simulation.BreakSecondsRemaining += Math.Max(0, (int)(simulation.BlockDeadlineUtc.Value - DateTime.UtcNow).TotalSeconds);
|
||||
simulation.BlockDeadlineUtc = null;
|
||||
simulation.UpdatedUtc = DateTime.UtcNow;
|
||||
if (simulation.CurrentBlock + 1 < BlockSizes.Length)
|
||||
{
|
||||
simulation.State = ExamSimulationStates.BetweenBlocks;
|
||||
await ReplaceAsync(simulation);
|
||||
return simulation;
|
||||
}
|
||||
await CompleteAsync(simulation);
|
||||
return simulation;
|
||||
}
|
||||
|
||||
public async Task AbandonAsync(ExamSimulation simulation)
|
||||
{
|
||||
simulation.State = ExamSimulationStates.Abandoned;
|
||||
simulation.BlockDeadlineUtc = null;
|
||||
simulation.UpdatedUtc = DateTime.UtcNow;
|
||||
await ReplaceAsync(simulation);
|
||||
}
|
||||
|
||||
public async Task<ExamSimulationStateResponse> ToResponseAsync(ExamSimulation simulation)
|
||||
{
|
||||
var questionMap = await _questions.GetByExternalIdsAsync(simulation.Items.Select(item => item.QuestionExternalId));
|
||||
var response = new ExamSimulationStateResponse
|
||||
{
|
||||
Id = simulation.Id ?? "",
|
||||
State = simulation.State,
|
||||
CurrentBlock = simulation.CurrentBlock,
|
||||
BreakSecondsRemaining = simulation.BreakSecondsRemaining,
|
||||
BlockDeadlineUtc = simulation.BlockDeadlineUtc,
|
||||
Responses = simulation.Responses,
|
||||
Score = simulation.Score,
|
||||
BlueprintTargets = simulation.BlueprintTargets,
|
||||
BlueprintActual = simulation.BlueprintActual
|
||||
};
|
||||
if (simulation.State == ExamSimulationStates.BlockActive)
|
||||
{
|
||||
response.Questions = simulation.Items.Where(item => item.BlockIndex == simulation.CurrentBlock)
|
||||
.OrderBy(item => item.Position)
|
||||
.Where(item => questionMap.ContainsKey(item.QuestionExternalId))
|
||||
.Select(item => SafeQuestion(questionMap[item.QuestionExternalId], item.PresentedChoiceIds)).ToList();
|
||||
}
|
||||
if (simulation.State == ExamSimulationStates.Completed)
|
||||
{
|
||||
response.ReviewQuestions = simulation.Items.OrderBy(item => item.BlockIndex).ThenBy(item => item.Position)
|
||||
.Where(item => questionMap.ContainsKey(item.QuestionExternalId))
|
||||
.Select(item => QuestionResponse.FromModel(questionMap[item.QuestionExternalId], questionElo: item.QuestionElo, includeChoicePool: true)).ToList();
|
||||
}
|
||||
return response;
|
||||
}
|
||||
|
||||
public async Task<Step3Outcome?> GetOutcomeAsync(string userId) =>
|
||||
await _outcomes.Find(item => item.UserId == userId).FirstOrDefaultAsync();
|
||||
|
||||
public async Task<Step3Outcome> UpsertOutcomeAsync(string userId, UpsertStep3OutcomeRequest request)
|
||||
{
|
||||
if (!request.Consent) throw new ArgumentException("Consent is required.");
|
||||
if (request.OfficialScore is < 1 or > 300) throw new ArgumentException("Official score must be between 1 and 300.");
|
||||
if (request.ExamDate.Date > DateTime.UtcNow.Date) throw new ArgumentException("Exam date cannot be in the future.");
|
||||
var now = DateTime.UtcNow;
|
||||
return await _outcomes.FindOneAndUpdateAsync(
|
||||
item => item.UserId == userId,
|
||||
Builders<Step3Outcome>.Update
|
||||
.SetOnInsert(item => item.UserId, userId)
|
||||
.SetOnInsert(item => item.ConsentUtc, now)
|
||||
.Set(item => item.OfficialScore, request.OfficialScore)
|
||||
.Set(item => item.ExamDate, request.ExamDate)
|
||||
.Set(item => item.UpdatedUtc, now),
|
||||
new FindOneAndUpdateOptions<Step3Outcome> { IsUpsert = true, ReturnDocument = ReturnDocument.After });
|
||||
}
|
||||
|
||||
public Task DeleteOutcomeAsync(string userId) => _outcomes.DeleteOneAsync(item => item.UserId == userId);
|
||||
|
||||
private async Task CompleteAsync(ExamSimulation simulation)
|
||||
{
|
||||
var questionMap = await _questions.GetByExternalIdsAsync(simulation.Items.Select(item => item.QuestionExternalId));
|
||||
var responses = simulation.Responses.ToDictionary(item => item.QuestionExternalId, StringComparer.Ordinal);
|
||||
simulation.Score = ExamSimulationScoring.Calculate(simulation.Items, responses, questionMap);
|
||||
simulation.State = ExamSimulationStates.Completed;
|
||||
simulation.CompletedUtc = DateTime.UtcNow;
|
||||
simulation.UpdatedUtc = DateTime.UtcNow;
|
||||
await ReplaceAsync(simulation);
|
||||
foreach (var item in simulation.Items.Where(item => !item.WasSeen))
|
||||
{
|
||||
if (!questionMap.TryGetValue(item.QuestionExternalId, out var question)) continue;
|
||||
responses.TryGetValue(item.QuestionExternalId, out var response);
|
||||
var choiceId = response?.ChoiceId ?? "";
|
||||
var record = await _attempts.RecordFirstAttemptAsync(item.QuestionExternalId, simulation.UserId, choiceId, choiceId == question.CorrectChoiceId, choiceId == "", item.PresentedChoiceIds);
|
||||
if (!record.Inserted) continue;
|
||||
_metrics.AttemptCreated();
|
||||
try { await _statistics.IncrementAsync(record.Attempt); } catch { _metrics.CounterFailure(); }
|
||||
var owner = Guid.NewGuid().ToString("N");
|
||||
var attemptId = record.Attempt.Id ?? "";
|
||||
var shouldRate = false;
|
||||
try
|
||||
{
|
||||
shouldRate = await _attempts.TryAcquireRatingLeaseAsync(attemptId, owner, TimeSpan.FromSeconds(30));
|
||||
await _ratings.ApplyFirstResultAsync(simulation.UserId, question, record.Attempt.IsCorrect, shouldRate);
|
||||
if (shouldRate) await _attempts.MarkRatingAppliedAsync(attemptId, owner);
|
||||
}
|
||||
catch
|
||||
{
|
||||
_metrics.CounterFailure();
|
||||
if (shouldRate) await _attempts.ReleaseRatingLeaseAsync(attemptId, owner);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private Task ReplaceAsync(ExamSimulation simulation) =>
|
||||
_simulations.ReplaceOneAsync(item => item.Id == simulation.Id && item.UserId == simulation.UserId, simulation);
|
||||
|
||||
private static ExamSimulationQuestionResponse SafeQuestion(Question question, List<string> presentedIds) => new()
|
||||
{
|
||||
ExternalId = question.ExternalId,
|
||||
Stem = question.Stem,
|
||||
LeadIn = question.LeadIn,
|
||||
Choices = presentedIds.Select(id => question.Choices.First(choice => choice.Id == id)).ToList(),
|
||||
Discipline = question.Discipline,
|
||||
System = question.System,
|
||||
PerformanceCategory = question.PerformanceCategory,
|
||||
Labs = question.Labs,
|
||||
Media = question.Media
|
||||
};
|
||||
|
||||
private static List<string> SelectChoices(Question question)
|
||||
{
|
||||
var distractors = question.Choices.Where(choice => choice.Id != question.CorrectChoiceId).OrderBy(_ => Random.Shared.Next()).Take(4).Select(choice => choice.Id).ToList();
|
||||
distractors.Add(question.CorrectChoiceId);
|
||||
return distractors.OrderBy(_ => Random.Shared.Next()).ToList();
|
||||
}
|
||||
|
||||
private static List<Question> SelectForm(List<Question> bank, HashSet<string> seen, Dictionary<string, int> targets)
|
||||
{
|
||||
var remaining = bank.OrderBy(question => seen.Contains(question.ExternalId) ? 1 : 0).ThenBy(_ => Random.Shared.Next()).ToList();
|
||||
var selected = new List<Question>();
|
||||
foreach (var target in targets)
|
||||
{
|
||||
var matches = remaining.Where(question => BlueprintCategory(question) == target.Key).Take(target.Value).ToList();
|
||||
selected.AddRange(matches);
|
||||
foreach (var match in matches) remaining.Remove(match);
|
||||
}
|
||||
selected.AddRange(remaining.Take(RequiredQuestions - selected.Count));
|
||||
return selected.OrderBy(_ => Random.Shared.Next()).ToList();
|
||||
}
|
||||
|
||||
public static Dictionary<string, int> BuildTargets()
|
||||
{
|
||||
var weights = new Dictionary<string, double>
|
||||
{
|
||||
["Human Development"] = 2, ["Immune/Blood/Multisystem"] = 7, ["Behavioral Health"] = 5,
|
||||
["Nervous System"] = 9, ["Skin"] = 5, ["Musculoskeletal"] = 6, ["Cardiovascular"] = 10,
|
||||
["Respiratory"] = 9, ["Gastrointestinal"] = 7, ["Renal/Urinary/Male"] = 5,
|
||||
["Pregnancy/Female/Breast"] = 8, ["Endocrine"] = 6, ["Biostatistics/Population Health"] = 12,
|
||||
["Social Sciences"] = 8
|
||||
};
|
||||
var total = weights.Values.Sum();
|
||||
var exact = weights.ToDictionary(pair => pair.Key, pair => pair.Value / total * RequiredQuestions);
|
||||
var result = exact.ToDictionary(pair => pair.Key, pair => (int)Math.Floor(pair.Value));
|
||||
foreach (var key in exact.OrderByDescending(pair => pair.Value - Math.Floor(pair.Value)).Take(RequiredQuestions - result.Values.Sum()).Select(pair => pair.Key)) result[key]++;
|
||||
return result;
|
||||
}
|
||||
|
||||
public static string BlueprintCategory(Question question)
|
||||
{
|
||||
var value = $"{question.System} {question.PerformanceCategory} {question.Discipline}".ToLowerInvariant();
|
||||
if (value.Contains("biostat") || value.Contains("epidemi") || value.Contains("population")) return "Biostatistics/Population Health";
|
||||
if (value.Contains("ethic") || value.Contains("communication") || value.Contains("professional") || value.Contains("patient safety")) return "Social Sciences";
|
||||
if (value.Contains("pregnan") || value.Contains("obstetric") || value.Contains("female") || value.Contains("breast") || value.Contains("gyne")) return "Pregnancy/Female/Breast";
|
||||
if (value.Contains("renal") || value.Contains("urinary") || value.Contains("male") || value.Contains("genitourinary")) return "Renal/Urinary/Male";
|
||||
if (value.Contains("cardio")) return "Cardiovascular";
|
||||
if (value.Contains("resp") || value.Contains("pulmonary")) return "Respiratory";
|
||||
if (value.Contains("gastro") || value.Contains("hepato")) return "Gastrointestinal";
|
||||
if (value.Contains("neuro") || value.Contains("special sense")) return "Nervous System";
|
||||
if (value.Contains("musculo") || value.Contains("orthop")) return "Musculoskeletal";
|
||||
if (value.Contains("skin") || value.Contains("derm")) return "Skin";
|
||||
if (value.Contains("behavior") || value.Contains("psychiatr")) return "Behavioral Health";
|
||||
if (value.Contains("endocrine") || value.Contains("diabetes") || value.Contains("metabolism")) return "Endocrine";
|
||||
if (value.Contains("immune") || value.Contains("hemat") || value.Contains("blood") || value.Contains("multisystem")) return "Immune/Blood/Multisystem";
|
||||
return "Human Development";
|
||||
}
|
||||
}
|
||||
@@ -71,6 +71,12 @@ public class QuestionAttemptStore
|
||||
Builders<QuestionAttempt>.Filter.Gte(attempt => attempt.CreatedUtc, sinceUtc))
|
||||
.ToListAsync();
|
||||
|
||||
public Task<List<string>> GetUserQuestionIdsAsync(string userId) =>
|
||||
_col.Distinct(
|
||||
attempt => attempt.QuestionExternalId,
|
||||
Builders<QuestionAttempt>.Filter.Eq(attempt => attempt.UserId, userId))
|
||||
.ToListAsync();
|
||||
|
||||
public IMongoCollection<QuestionAttempt> Collection => _col;
|
||||
|
||||
public async Task<bool> TryAcquireRatingLeaseAsync(string attemptId, string owner, TimeSpan duration)
|
||||
|
||||
@@ -64,6 +64,16 @@ public class QuestionStore
|
||||
public async Task<List<Question>> GetAllAsync() =>
|
||||
await _col.Find(Builders<Question>.Filter.Empty).ToListAsync();
|
||||
|
||||
public async Task<List<Question>> GetPublishedAsync() =>
|
||||
await _col.Find(question => question.IsPublished).ToListAsync();
|
||||
|
||||
public async Task<Dictionary<string, Question>> GetByExternalIdsAsync(IEnumerable<string> externalIds)
|
||||
{
|
||||
var ids = externalIds.Distinct(StringComparer.Ordinal).ToList();
|
||||
var questions = await _col.Find(question => ids.Contains(question.ExternalId)).ToListAsync();
|
||||
return questions.ToDictionary(question => question.ExternalId, StringComparer.Ordinal);
|
||||
}
|
||||
|
||||
public async Task<(List<Question> Items, string? NextCursor, bool HasMore)> ExportPageAsync(
|
||||
string? after,
|
||||
int requestedLimit)
|
||||
|
||||
Reference in New Issue
Block a user