679 lines
34 KiB
C#
679 lines
34 KiB
C#
using MongoDB.Bson;
|
|
using MongoDB.Driver;
|
|
using System.Text;
|
|
using QuestionsApi.Models;
|
|
|
|
namespace QuestionsApi.Services;
|
|
|
|
public class ExamSimulationStore
|
|
{
|
|
public const int RequiredQuestions = 232;
|
|
public static readonly int[] BlockSizes = [20, 19, 19, 20, 19, 19, 20, 19, 19, 20, 19, 19];
|
|
private readonly IMongoCollection<ExamSimulation> _simulations;
|
|
private readonly IMongoCollection<Step3Outcome> _outcomes;
|
|
private readonly QuestionStore _questions;
|
|
private readonly QuestionAttemptStore _attempts;
|
|
private readonly CreditStore _credits;
|
|
private readonly AnswerStatisticsStore _statistics;
|
|
private readonly RatingStore _ratings;
|
|
private readonly ServiceMetrics _metrics;
|
|
private readonly ReviewQueueStore _reviewQueue;
|
|
|
|
public ExamSimulationStore(
|
|
MongoDatabaseProvider mongo,
|
|
QuestionStore questions,
|
|
QuestionAttemptStore attempts,
|
|
CreditStore credits,
|
|
AnswerStatisticsStore statistics,
|
|
RatingStore ratings,
|
|
ServiceMetrics metrics,
|
|
ReviewQueueStore reviewQueue)
|
|
{
|
|
_simulations = mongo.Database.GetCollection<ExamSimulation>("ExamSimulations");
|
|
_outcomes = mongo.Database.GetCollection<Step3Outcome>("Step3Outcomes");
|
|
_questions = questions;
|
|
_attempts = attempts;
|
|
_credits = credits;
|
|
_statistics = statistics;
|
|
_ratings = ratings;
|
|
_metrics = metrics;
|
|
_reviewQueue = reviewQueue;
|
|
_simulations.Indexes.CreateOne(new CreateIndexModel<ExamSimulation>(
|
|
Builders<ExamSimulation>.IndexKeys.Ascending(item => item.UserId).Ascending(item => item.State)));
|
|
_simulations.Indexes.CreateOne(new CreateIndexModel<ExamSimulation>(
|
|
Builders<ExamSimulation>.IndexKeys.Ascending(item => item.UserId).Ascending(item => item.State).Descending(item => item.CompletedUtc)));
|
|
_simulations.Indexes.CreateOne(new CreateIndexModel<ExamSimulation>(
|
|
Builders<ExamSimulation>.IndexKeys.Ascending(item => item.UserId).Descending(item => item.CreatedUtc)));
|
|
_simulations.Indexes.CreateOne(new CreateIndexModel<ExamSimulation>(
|
|
Builders<ExamSimulation>.IndexKeys.Ascending(item => item.UserId).Ascending(item => item.State).Descending(item => item.CreatedUtc).Descending(item => item.Id)));
|
|
_simulations.Indexes.CreateOne(new CreateIndexModel<ExamSimulation>(
|
|
Builders<ExamSimulation>.IndexKeys.Ascending(item => item.UpdatedUtc)));
|
|
_outcomes.Indexes.CreateOne(new CreateIndexModel<Step3Outcome>(
|
|
Builders<Step3Outcome>.IndexKeys.Ascending(item => item.UserId),
|
|
new CreateIndexOptions { Unique = true }));
|
|
}
|
|
|
|
public async Task<ExamSimulationReadinessResponse> GetReadinessAsync(string userId)
|
|
{
|
|
var count = (await _questions.GetPublishedAsync()).Count(question => question.Choices.Count >= 5);
|
|
return new ExamSimulationReadinessResponse
|
|
{
|
|
Ready = count >= RequiredQuestions,
|
|
AvailableQuestions = count,
|
|
HasActiveSimulation = await FindActiveAsync(userId) is not null
|
|
};
|
|
}
|
|
|
|
public async Task<ExamSimulation> CreateAsync(string userId)
|
|
{
|
|
if (await FindActiveAsync(userId) is not null)
|
|
throw new InvalidOperationException("An active Step 3 simulation already exists.");
|
|
var bank = (await _questions.GetPublishedAsync()).Where(question => question.Choices.Count >= 5).ToList();
|
|
if (bank.Count < RequiredQuestions)
|
|
throw new InvalidOperationException($"At least {RequiredQuestions} published questions are required.");
|
|
var bankCycle = await _credits.GetBankCycleAsync(userId);
|
|
var seen = (await _attempts.GetUserQuestionIdsAsync(userId, bankCycle)).ToHashSet(StringComparer.Ordinal);
|
|
var elos = await _ratings.GetQuestionElosAsync(bank.Select(question => question.ExternalId));
|
|
var targets = BuildTargets();
|
|
var selected = SelectForm(bank, seen, targets);
|
|
var choicePercentages = await _statistics.GetPercentagesByQuestionAsync(selected.Select(question => question.ExternalId));
|
|
var items = new List<ExamSimulationItem>();
|
|
var position = 0;
|
|
for (var block = 0; block < BlockSizes.Length; block++)
|
|
{
|
|
for (var offset = 0; offset < BlockSizes[block]; offset++)
|
|
{
|
|
var question = selected[position];
|
|
items.Add(new ExamSimulationItem
|
|
{
|
|
QuestionExternalId = question.ExternalId,
|
|
BlockIndex = block,
|
|
Position = offset,
|
|
PresentedChoiceIds = SelectChoices(question, choicePercentages.GetValueOrDefault(question.ExternalId)),
|
|
QuestionElo = elos.GetValueOrDefault(question.ExternalId, 1000),
|
|
QuestionDifficulty = question.Difficulty,
|
|
WasSeen = seen.Contains(question.ExternalId),
|
|
BlueprintCategory = BlueprintCategory(question)
|
|
});
|
|
position++;
|
|
}
|
|
}
|
|
var simulation = new ExamSimulation
|
|
{
|
|
UserId = userId,
|
|
Items = items,
|
|
BlueprintTargets = targets,
|
|
BlueprintActual = items.GroupBy(item => item.BlueprintCategory).ToDictionary(group => group.Key, group => group.Count())
|
|
};
|
|
await _simulations.InsertOneAsync(simulation);
|
|
return simulation;
|
|
}
|
|
|
|
public async Task<ExamSimulationStateResponse> BuildResultsPreviewAsync(string userId) =>
|
|
await ToResponseAsync(await BuildSampleAsync(userId, false));
|
|
|
|
public Task<ExamSimulation> CreateSampleCompletedAsync(string userId) => BuildSampleAsync(userId, true);
|
|
|
|
private async Task<ExamSimulation> BuildSampleAsync(string userId, bool persist)
|
|
{
|
|
var bank = (await _questions.GetPublishedAsync()).Where(question => question.Choices.Count >= 5).ToList();
|
|
if (bank.Count < RequiredQuestions)
|
|
throw new InvalidOperationException($"At least {RequiredQuestions} published questions are required.");
|
|
var targets = BuildTargets();
|
|
var selected = SelectForm(bank, [], targets);
|
|
var choicePercentages = await _statistics.GetPercentagesByQuestionAsync(selected.Select(question => question.ExternalId));
|
|
var items = selected.Select((question, index) => new ExamSimulationItem
|
|
{
|
|
QuestionExternalId = question.ExternalId,
|
|
BlockIndex = BlockIndexForPosition(index),
|
|
Position = PositionWithinBlock(index),
|
|
PresentedChoiceIds = SelectChoices(question, choicePercentages.GetValueOrDefault(question.ExternalId)),
|
|
QuestionElo = 1000,
|
|
QuestionDifficulty = question.Difficulty,
|
|
WasSeen = false,
|
|
BlueprintCategory = BlueprintCategory(question)
|
|
}).ToList();
|
|
var questionMap = selected.ToDictionary(question => question.ExternalId, StringComparer.Ordinal);
|
|
var responses = items.Select((item, index) =>
|
|
{
|
|
var question = questionMap[item.QuestionExternalId];
|
|
var choiceId = index % 3 == 0
|
|
? item.PresentedChoiceIds.First(id => id != question.CorrectChoiceId)
|
|
: question.CorrectChoiceId;
|
|
return new ExamSimulationResponse
|
|
{
|
|
QuestionExternalId = item.QuestionExternalId,
|
|
ChoiceId = choiceId,
|
|
ElapsedSeconds = 75 + index % 45,
|
|
Flagged = index % 11 == 0
|
|
};
|
|
}).ToList();
|
|
var responseMap = responses.ToDictionary(item => item.QuestionExternalId, StringComparer.Ordinal);
|
|
var now = DateTime.UtcNow;
|
|
var simulation = new ExamSimulation
|
|
{
|
|
UserId = userId,
|
|
State = ExamSimulationStates.Completed,
|
|
IsSample = true,
|
|
CurrentBlock = BlockSizes.Length - 1,
|
|
BreakSecondsRemaining = 3300,
|
|
Items = items,
|
|
Responses = responses,
|
|
Score = ExamSimulationScoring.Calculate(items, responseMap, questionMap),
|
|
CreatedUtc = now,
|
|
UpdatedUtc = now,
|
|
CompletedUtc = now,
|
|
BlueprintTargets = targets,
|
|
BlueprintActual = items.GroupBy(item => item.BlueprintCategory).ToDictionary(group => group.Key, group => group.Count())
|
|
};
|
|
if (persist) await _simulations.InsertOneAsync(simulation);
|
|
return simulation;
|
|
}
|
|
|
|
public async Task<ExamSimulation?> FindActiveAsync(string userId) =>
|
|
await _simulations.Find(item => item.UserId == userId &&
|
|
(item.State == ExamSimulationStates.BetweenBlocks || item.State == ExamSimulationStates.BlockActive))
|
|
.SortByDescending(item => item.UpdatedUtc).FirstOrDefaultAsync();
|
|
|
|
public async Task<ExamSimulationScore?> GetLatestCompletedScoreAsync(string userId)
|
|
{
|
|
var simulation = await _simulations
|
|
.Find(item => item.UserId == userId && item.State == ExamSimulationStates.Completed && item.Score != null)
|
|
.SortByDescending(item => item.CompletedUtc)
|
|
.FirstOrDefaultAsync();
|
|
return simulation is null ? null : (await UpgradeScoringModelAsync(simulation)).Score;
|
|
}
|
|
|
|
public async Task<ExamSimulation?> GetOwnedAsync(string id, string userId)
|
|
{
|
|
var simulation = await _simulations.Find(item => item.Id == id && item.UserId == userId).FirstOrDefaultAsync();
|
|
if (simulation is not null && simulation.State == ExamSimulationStates.BlockActive && simulation.BlockDeadlineUtc <= DateTime.UtcNow)
|
|
simulation = await SubmitBlockAsync(simulation, true);
|
|
if (simulation is not null)
|
|
simulation = await UpgradeScoringModelAsync(simulation);
|
|
return simulation;
|
|
}
|
|
|
|
public async Task<CompletedExamSimulationPage> GetCompletedAsync(string userId, int limit, DateTime? before)
|
|
{
|
|
limit = Math.Clamp(limit, 1, 50);
|
|
var filter = Builders<ExamSimulation>.Filter.Eq(item => item.UserId, userId) &
|
|
Builders<ExamSimulation>.Filter.Eq(item => item.State, ExamSimulationStates.Completed);
|
|
if (before is not null)
|
|
filter &= Builders<ExamSimulation>.Filter.Lt(item => item.CompletedUtc, before.Value);
|
|
var simulations = await _simulations.Find(filter).SortByDescending(item => item.CompletedUtc).Limit(limit + 1).ToListAsync();
|
|
var hasMore = simulations.Count > limit;
|
|
if (hasMore) simulations.RemoveAt(simulations.Count - 1);
|
|
await UpgradeScoringModelsAsync(simulations);
|
|
return new CompletedExamSimulationPage
|
|
{
|
|
Items = simulations.Select(item => new CompletedExamSimulationSummary
|
|
{
|
|
Id = item.Id ?? "",
|
|
CompletedUtc = item.CompletedUtc ?? item.UpdatedUtc,
|
|
PredictedScore = item.Score?.PredictedScore ?? 0,
|
|
RangeLow = item.Score?.RangeLow ?? 0,
|
|
RangeHigh = item.Score?.RangeHigh ?? 0,
|
|
Correct = item.Score?.Correct ?? 0,
|
|
Total = item.Score?.Total ?? item.Items.Count,
|
|
SeenCount = item.Score?.SeenCount ?? 0
|
|
}).ToList(),
|
|
NextBefore = hasMore && simulations.Count > 0 ? simulations[^1].CompletedUtc : null
|
|
};
|
|
}
|
|
|
|
public async Task<ExamSimulationHistoryPage> GetHistoryAsync(
|
|
string userId,
|
|
int limit,
|
|
string status = "all",
|
|
string sort = "newest",
|
|
string? cursor = null,
|
|
DateTime? legacyBefore = null)
|
|
{
|
|
limit = Math.Clamp(limit, 1, 50);
|
|
status = status.Trim().ToLowerInvariant();
|
|
sort = sort.Trim().ToLowerInvariant();
|
|
if (status is not ("all" or "completed" or "abandoned"))
|
|
throw new ArgumentException("The simulation-history status is invalid.");
|
|
if (sort is not ("newest" or "oldest"))
|
|
throw new ArgumentException("The simulation-history sort order is invalid.");
|
|
|
|
var filters = Builders<ExamSimulation>.Filter;
|
|
var filter = filters.Eq(item => item.UserId, userId);
|
|
filter &= status switch
|
|
{
|
|
"completed" => filters.Eq(item => item.State, ExamSimulationStates.Completed),
|
|
"abandoned" => filters.Eq(item => item.State, ExamSimulationStates.Abandoned),
|
|
_ => filters.In(item => item.State, [ExamSimulationStates.Completed, ExamSimulationStates.Abandoned])
|
|
};
|
|
|
|
HistoryCursor? decodedCursor = null;
|
|
if (!string.IsNullOrWhiteSpace(cursor))
|
|
{
|
|
if (!TryDecodeHistoryCursor(cursor, out decodedCursor))
|
|
throw new ArgumentException("The simulation-history cursor is invalid.");
|
|
var dateComparison = sort == "newest"
|
|
? filters.Lt(item => item.CreatedUtc, decodedCursor!.CreatedUtc)
|
|
: filters.Gt(item => item.CreatedUtc, decodedCursor!.CreatedUtc);
|
|
var idComparison = sort == "newest"
|
|
? filters.Lt(item => item.Id, decodedCursor!.Id)
|
|
: filters.Gt(item => item.Id, decodedCursor!.Id);
|
|
filter &= dateComparison | (filters.Eq(item => item.CreatedUtc, decodedCursor!.CreatedUtc) & idComparison);
|
|
}
|
|
else if (legacyBefore is not null)
|
|
{
|
|
filter &= filters.Lt(item => item.CreatedUtc, legacyBefore.Value);
|
|
}
|
|
|
|
var sortDefinition = sort == "newest"
|
|
? Builders<ExamSimulation>.Sort.Descending(item => item.CreatedUtc).Descending(item => item.Id)
|
|
: Builders<ExamSimulation>.Sort.Ascending(item => item.CreatedUtc).Ascending(item => item.Id);
|
|
var simulations = await _simulations.Find(filter).Sort(sortDefinition).Limit(limit + 1).ToListAsync();
|
|
var hasMore = simulations.Count > limit;
|
|
if (hasMore) simulations.RemoveAt(simulations.Count - 1);
|
|
await UpgradeScoringModelsAsync(simulations.Where(item => item.State == ExamSimulationStates.Completed));
|
|
var last = simulations.LastOrDefault();
|
|
return new ExamSimulationHistoryPage
|
|
{
|
|
Items = simulations.Select(ToHistorySummary).ToList(),
|
|
NextBefore = sort == "newest" && hasMore && last is not null ? last.CreatedUtc : null,
|
|
NextCursor = hasMore && last is not null ? EncodeHistoryCursor(last.CreatedUtc, last.Id ?? "") : null
|
|
};
|
|
}
|
|
|
|
public static string EncodeHistoryCursor(DateTime createdUtc, string id) =>
|
|
Convert.ToBase64String(Encoding.UTF8.GetBytes($"{createdUtc.ToUniversalTime().Ticks}|{id}"));
|
|
|
|
public static bool TryDecodeHistoryCursor(string value, out HistoryCursor? cursor)
|
|
{
|
|
cursor = null;
|
|
try
|
|
{
|
|
var decoded = Encoding.UTF8.GetString(Convert.FromBase64String(value));
|
|
var parts = decoded.Split('|', 2);
|
|
if (parts.Length != 2 || !long.TryParse(parts[0], out var ticks) || !ObjectId.TryParse(parts[1], out _))
|
|
return false;
|
|
cursor = new HistoryCursor(new DateTime(ticks, DateTimeKind.Utc), parts[1]);
|
|
return true;
|
|
}
|
|
catch (FormatException)
|
|
{
|
|
return false;
|
|
}
|
|
catch (ArgumentOutOfRangeException)
|
|
{
|
|
return false;
|
|
}
|
|
}
|
|
|
|
public sealed record HistoryCursor(DateTime CreatedUtc, string Id);
|
|
|
|
public static ExamSimulationHistorySummary ToHistorySummary(ExamSimulation item) => new()
|
|
{
|
|
Id = item.Id ?? "",
|
|
State = item.State,
|
|
IsSample = item.IsSample,
|
|
CreatedUtc = item.CreatedUtc,
|
|
UpdatedUtc = item.UpdatedUtc,
|
|
CompletedUtc = item.CompletedUtc,
|
|
CurrentBlock = item.CurrentBlock,
|
|
PredictedScore = item.Score?.PredictedScore,
|
|
RangeLow = item.Score?.RangeLow,
|
|
RangeHigh = item.Score?.RangeHigh,
|
|
PassProbability = item.Score?.PassProbability,
|
|
Correct = item.Score?.Correct,
|
|
Total = item.Score?.Total ?? item.Items.Count
|
|
};
|
|
|
|
public async Task<ExamSimulation> StartNextBlockAsync(ExamSimulation simulation)
|
|
{
|
|
if (simulation.State != ExamSimulationStates.BetweenBlocks)
|
|
throw new InvalidOperationException("The simulation is not between blocks.");
|
|
if (simulation.CurrentBlock + 1 >= BlockSizes.Length)
|
|
throw new InvalidOperationException("All blocks are complete.");
|
|
simulation.CurrentBlock++;
|
|
simulation.State = ExamSimulationStates.BlockActive;
|
|
simulation.BlockDeadlineUtc = DateTime.UtcNow.AddMinutes(30);
|
|
simulation.UpdatedUtc = DateTime.UtcNow;
|
|
await ReplaceAsync(simulation);
|
|
return simulation;
|
|
}
|
|
|
|
public async Task<ExamSimulation> SaveResponseAsync(ExamSimulation simulation, SaveSimulationResponseRequest request)
|
|
{
|
|
if (simulation.State != ExamSimulationStates.BlockActive || simulation.BlockDeadlineUtc <= DateTime.UtcNow)
|
|
throw new InvalidOperationException("The active block has ended.");
|
|
var item = simulation.Items.FirstOrDefault(candidate => candidate.BlockIndex == simulation.CurrentBlock && candidate.QuestionExternalId == request.QuestionExternalId)
|
|
?? throw new ArgumentException("Question is not in the active block.");
|
|
if (!item.PresentedChoiceIds.Contains(request.ChoiceId, StringComparer.Ordinal) && request.ChoiceId != "")
|
|
throw new ArgumentException("Choice is not presented for this question.");
|
|
var response = simulation.Responses.FirstOrDefault(candidate => candidate.QuestionExternalId == request.QuestionExternalId);
|
|
if (response is not null && IsResponseLocked(response))
|
|
{
|
|
if (ResponseContentMatches(response, request, item.PresentedChoiceIds))
|
|
return simulation;
|
|
throw new InvalidOperationException("A submitted simulation answer cannot be changed.");
|
|
}
|
|
if (response is null)
|
|
{
|
|
response = new ExamSimulationResponse
|
|
{
|
|
QuestionExternalId = request.QuestionExternalId,
|
|
IsSubmitted = false
|
|
};
|
|
simulation.Responses.Add(response);
|
|
}
|
|
response.ChoiceId = request.ChoiceId;
|
|
response.EliminatedChoiceIds = request.EliminatedChoiceIds.Where(item.PresentedChoiceIds.Contains).Distinct().ToList();
|
|
response.Flagged = request.Flagged;
|
|
response.ElapsedSeconds = Math.Clamp(request.ElapsedSeconds, 0, 1800);
|
|
response.IsSubmitted = request.IsSubmitted;
|
|
response.UpdatedUtc = DateTime.UtcNow;
|
|
simulation.UpdatedUtc = DateTime.UtcNow;
|
|
await ReplaceAsync(simulation);
|
|
return simulation;
|
|
}
|
|
|
|
public static bool IsResponseLocked(ExamSimulationResponse response) =>
|
|
response.IsSubmitted == true ||
|
|
(response.IsSubmitted is null && !string.IsNullOrEmpty(response.ChoiceId));
|
|
|
|
public static bool ResponseContentMatches(
|
|
ExamSimulationResponse response,
|
|
SaveSimulationResponseRequest request,
|
|
IReadOnlyCollection<string> presentedChoiceIds)
|
|
{
|
|
var eliminated = request.EliminatedChoiceIds
|
|
.Where(presentedChoiceIds.Contains)
|
|
.ToHashSet(StringComparer.Ordinal);
|
|
return response.ChoiceId == request.ChoiceId &&
|
|
response.Flagged == request.Flagged &&
|
|
response.EliminatedChoiceIds.ToHashSet(StringComparer.Ordinal).SetEquals(eliminated);
|
|
}
|
|
|
|
public async Task<ExamSimulation> SubmitBlockAsync(ExamSimulation simulation, bool expired = false)
|
|
{
|
|
if (IsAlreadySubmittedState(simulation.State))
|
|
return simulation;
|
|
if (simulation.State != ExamSimulationStates.BlockActive)
|
|
throw new InvalidOperationException("There is no active block to submit.");
|
|
if (!expired && simulation.BlockDeadlineUtc is not null)
|
|
simulation.BreakSecondsRemaining += Math.Max(0, (int)(simulation.BlockDeadlineUtc.Value - DateTime.UtcNow).TotalSeconds);
|
|
simulation.BlockDeadlineUtc = null;
|
|
simulation.UpdatedUtc = DateTime.UtcNow;
|
|
if (simulation.CurrentBlock + 1 < BlockSizes.Length)
|
|
{
|
|
simulation.State = ExamSimulationStates.BetweenBlocks;
|
|
await ReplaceAsync(simulation);
|
|
return simulation;
|
|
}
|
|
await CompleteAsync(simulation);
|
|
return simulation;
|
|
}
|
|
|
|
public static bool IsAlreadySubmittedState(string state) =>
|
|
state is ExamSimulationStates.BetweenBlocks or ExamSimulationStates.Completed;
|
|
|
|
public async Task AbandonAsync(ExamSimulation simulation)
|
|
{
|
|
simulation.State = ExamSimulationStates.Abandoned;
|
|
simulation.BlockDeadlineUtc = null;
|
|
simulation.UpdatedUtc = DateTime.UtcNow;
|
|
await ReplaceAsync(simulation);
|
|
}
|
|
|
|
public async Task<ExamSimulationStateResponse> ToResponseAsync(ExamSimulation simulation)
|
|
{
|
|
var questionMap = await _questions.GetByExternalIdsAsync(simulation.Items.Select(item => item.QuestionExternalId));
|
|
var response = new ExamSimulationStateResponse
|
|
{
|
|
Id = simulation.Id ?? "",
|
|
State = simulation.State,
|
|
CurrentBlock = simulation.CurrentBlock,
|
|
BreakSecondsRemaining = simulation.BreakSecondsRemaining,
|
|
BlockDeadlineUtc = simulation.BlockDeadlineUtc,
|
|
CreatedUtc = simulation.CreatedUtc,
|
|
UpdatedUtc = simulation.UpdatedUtc,
|
|
Responses = simulation.Responses,
|
|
Score = simulation.Score,
|
|
BlueprintTargets = simulation.BlueprintTargets,
|
|
BlueprintActual = simulation.BlueprintActual
|
|
};
|
|
if (simulation.State == ExamSimulationStates.BlockActive)
|
|
{
|
|
response.Questions = simulation.Items.Where(item => item.BlockIndex == simulation.CurrentBlock)
|
|
.OrderBy(item => item.Position)
|
|
.Where(item => questionMap.ContainsKey(item.QuestionExternalId))
|
|
.Select(item => SafeQuestion(questionMap[item.QuestionExternalId], item.PresentedChoiceIds)).ToList();
|
|
}
|
|
if (simulation.State == ExamSimulationStates.Completed)
|
|
{
|
|
response.ReviewQuestions = simulation.Items.OrderBy(item => item.BlockIndex).ThenBy(item => item.Position)
|
|
.Where(item => questionMap.ContainsKey(item.QuestionExternalId))
|
|
.Select(item => QuestionResponse.FromModelWithChoices(questionMap[item.QuestionExternalId], item.PresentedChoiceIds, item.QuestionElo)).ToList();
|
|
}
|
|
return response;
|
|
}
|
|
|
|
public async Task<Step3Outcome?> GetOutcomeAsync(string userId) =>
|
|
await _outcomes.Find(item => item.UserId == userId).FirstOrDefaultAsync();
|
|
|
|
public async Task<Step3Outcome> UpsertOutcomeAsync(string userId, UpsertStep3OutcomeRequest request)
|
|
{
|
|
if (!request.Consent) throw new ArgumentException("Consent is required.");
|
|
if (request.OfficialScore is < 1 or > 300) throw new ArgumentException("Official score must be between 1 and 300.");
|
|
if (request.ExamDate.Date > DateTime.UtcNow.Date) throw new ArgumentException("Exam date cannot be in the future.");
|
|
var now = DateTime.UtcNow;
|
|
return await _outcomes.FindOneAndUpdateAsync(
|
|
item => item.UserId == userId,
|
|
Builders<Step3Outcome>.Update
|
|
.SetOnInsert(item => item.UserId, userId)
|
|
.SetOnInsert(item => item.ConsentUtc, now)
|
|
.Set(item => item.OfficialScore, request.OfficialScore)
|
|
.Set(item => item.ExamDate, request.ExamDate)
|
|
.Set(item => item.UpdatedUtc, now),
|
|
new FindOneAndUpdateOptions<Step3Outcome> { IsUpsert = true, ReturnDocument = ReturnDocument.After });
|
|
}
|
|
|
|
public Task DeleteOutcomeAsync(string userId) => _outcomes.DeleteOneAsync(item => item.UserId == userId);
|
|
|
|
private async Task CompleteAsync(ExamSimulation simulation)
|
|
{
|
|
var questionMap = await _questions.GetByExternalIdsAsync(simulation.Items.Select(item => item.QuestionExternalId));
|
|
var responses = simulation.Responses.ToDictionary(item => item.QuestionExternalId, StringComparer.Ordinal);
|
|
SnapshotDifficulties(simulation, questionMap);
|
|
simulation.Score = ExamSimulationScoring.Calculate(simulation.Items, responses, questionMap);
|
|
simulation.ModelVersion = ExamSimulationScoring.ModelVersion;
|
|
simulation.State = ExamSimulationStates.Completed;
|
|
simulation.CompletedUtc = DateTime.UtcNow;
|
|
simulation.UpdatedUtc = DateTime.UtcNow;
|
|
await ReplaceAsync(simulation);
|
|
foreach (var item in simulation.Items.Where(item => !item.WasSeen))
|
|
{
|
|
if (!questionMap.TryGetValue(item.QuestionExternalId, out var question)) continue;
|
|
responses.TryGetValue(item.QuestionExternalId, out var response);
|
|
var choiceId = response?.ChoiceId ?? "";
|
|
var record = await _attempts.RecordFirstAttemptAsync(
|
|
item.QuestionExternalId, simulation.UserId, choiceId,
|
|
choiceId == question.CorrectChoiceId, choiceId == "",
|
|
item.PresentedChoiceIds, bankCycle: await _credits.GetBankCycleAsync(simulation.UserId));
|
|
await _reviewQueue.AddFromFirstAttemptAsync(record.Attempt);
|
|
if (!record.Inserted) continue;
|
|
_metrics.AttemptCreated();
|
|
try { await _statistics.IncrementAsync(record.Attempt); } catch { _metrics.CounterFailure(); }
|
|
var owner = Guid.NewGuid().ToString("N");
|
|
var attemptId = record.Attempt.Id ?? "";
|
|
var shouldRate = false;
|
|
try
|
|
{
|
|
shouldRate = await _attempts.TryAcquireRatingLeaseAsync(attemptId, owner, TimeSpan.FromSeconds(30));
|
|
var rating = await _ratings.ApplyFirstResultAsync(simulation.UserId, question, record.Attempt.IsCorrect, shouldRate);
|
|
if (shouldRate) await _attempts.MarkRatingAppliedAsync(attemptId, owner, rating.UserOverallElo);
|
|
}
|
|
catch
|
|
{
|
|
_metrics.CounterFailure();
|
|
if (shouldRate) await _attempts.ReleaseRatingLeaseAsync(attemptId, owner);
|
|
}
|
|
}
|
|
}
|
|
|
|
private Task ReplaceAsync(ExamSimulation simulation) =>
|
|
_simulations.ReplaceOneAsync(item => item.Id == simulation.Id && item.UserId == simulation.UserId, simulation);
|
|
|
|
public async Task<ExamSimulationBackfillResult> BackfillScoringModelAsync(CancellationToken cancellationToken = default)
|
|
{
|
|
var filter = Builders<ExamSimulation>.Filter.Eq(item => item.State, ExamSimulationStates.Completed) &
|
|
Builders<ExamSimulation>.Filter.Ne(item => item.ModelVersion, ExamSimulationScoring.ModelVersion);
|
|
var simulations = await _simulations.Find(filter).ToListAsync(cancellationToken);
|
|
var missingQuestionIds = new List<string>();
|
|
var updated = 0;
|
|
foreach (var simulation in simulations)
|
|
{
|
|
cancellationToken.ThrowIfCancellationRequested();
|
|
var questionMap = await _questions.GetByExternalIdsAsync(simulation.Items.Select(item => item.QuestionExternalId));
|
|
missingQuestionIds.AddRange(simulation.Items
|
|
.Where(item => !questionMap.ContainsKey(item.QuestionExternalId))
|
|
.Select(item => $"{simulation.Id}:{item.QuestionExternalId}"));
|
|
SnapshotDifficulties(simulation, questionMap);
|
|
var responses = simulation.Responses.ToDictionary(item => item.QuestionExternalId, StringComparer.Ordinal);
|
|
simulation.Score = ExamSimulationScoring.Calculate(simulation.Items, responses, questionMap);
|
|
simulation.ModelVersion = ExamSimulationScoring.ModelVersion;
|
|
simulation.UpdatedUtc = DateTime.UtcNow;
|
|
await ReplaceAsync(simulation);
|
|
updated++;
|
|
}
|
|
return new ExamSimulationBackfillResult(updated, missingQuestionIds);
|
|
}
|
|
|
|
private async Task UpgradeScoringModelsAsync(IEnumerable<ExamSimulation> simulations)
|
|
{
|
|
foreach (var simulation in simulations)
|
|
await UpgradeScoringModelAsync(simulation);
|
|
}
|
|
|
|
private async Task<ExamSimulation> UpgradeScoringModelAsync(ExamSimulation simulation)
|
|
{
|
|
var needsDifficultySnapshot = simulation.Items.Any(item => item.QuestionDifficulty is < 1 or > 5);
|
|
var needsScoreUpgrade = simulation.State == ExamSimulationStates.Completed &&
|
|
simulation.ModelVersion != ExamSimulationScoring.ModelVersion;
|
|
if (!needsDifficultySnapshot && !needsScoreUpgrade)
|
|
return simulation;
|
|
|
|
var questionMap = await _questions.GetByExternalIdsAsync(simulation.Items.Select(item => item.QuestionExternalId));
|
|
SnapshotDifficulties(simulation, questionMap);
|
|
if (simulation.State == ExamSimulationStates.Completed)
|
|
{
|
|
var responses = simulation.Responses.ToDictionary(item => item.QuestionExternalId, StringComparer.Ordinal);
|
|
simulation.Score = ExamSimulationScoring.Calculate(simulation.Items, responses, questionMap);
|
|
}
|
|
simulation.ModelVersion = ExamSimulationScoring.ModelVersion;
|
|
simulation.UpdatedUtc = DateTime.UtcNow;
|
|
await ReplaceAsync(simulation);
|
|
return simulation;
|
|
}
|
|
|
|
private static void SnapshotDifficulties(
|
|
ExamSimulation simulation,
|
|
IReadOnlyDictionary<string, Question> questions)
|
|
{
|
|
foreach (var item in simulation.Items)
|
|
{
|
|
questions.TryGetValue(item.QuestionExternalId, out var question);
|
|
item.QuestionDifficulty = ExamSimulationScoring.ResolveDifficulty(item, question);
|
|
}
|
|
}
|
|
|
|
public sealed record ExamSimulationBackfillResult(int UpdatedCount, IReadOnlyList<string> MissingQuestionIds);
|
|
|
|
private static int BlockIndexForPosition(int position)
|
|
{
|
|
var consumed = 0;
|
|
for (var block = 0; block < BlockSizes.Length; block++)
|
|
{
|
|
if (position < consumed + BlockSizes[block]) return block;
|
|
consumed += BlockSizes[block];
|
|
}
|
|
return BlockSizes.Length - 1;
|
|
}
|
|
|
|
private static int PositionWithinBlock(int position)
|
|
{
|
|
var consumed = 0;
|
|
foreach (var size in BlockSizes)
|
|
{
|
|
if (position < consumed + size) return position - consumed;
|
|
consumed += size;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
private static ExamSimulationQuestionResponse SafeQuestion(Question question, List<string> presentedIds) => new()
|
|
{
|
|
ExternalId = question.ExternalId,
|
|
Stem = question.Stem,
|
|
LeadIn = question.LeadIn,
|
|
Choices = presentedIds.Select(id => question.Choices.First(choice => choice.Id == id)).ToList(),
|
|
Discipline = question.Discipline,
|
|
System = question.System,
|
|
PerformanceCategory = question.PerformanceCategory,
|
|
Labs = question.Labs,
|
|
Media = question.Media
|
|
};
|
|
|
|
private static List<string> SelectChoices(
|
|
Question question,
|
|
IReadOnlyDictionary<string, int>? liveChoiceSelectionPercentages) =>
|
|
ChoicePoolSelector.SelectChoiceIds(question, liveChoiceSelectionPercentages);
|
|
|
|
private static List<Question> SelectForm(List<Question> bank, HashSet<string> seen, Dictionary<string, int> targets)
|
|
{
|
|
var remaining = bank.OrderBy(question => seen.Contains(question.ExternalId) ? 1 : 0).ThenBy(_ => Random.Shared.Next()).ToList();
|
|
var selected = new List<Question>();
|
|
foreach (var target in targets)
|
|
{
|
|
var matches = remaining.Where(question => BlueprintCategory(question) == target.Key).Take(target.Value).ToList();
|
|
selected.AddRange(matches);
|
|
foreach (var match in matches) remaining.Remove(match);
|
|
}
|
|
selected.AddRange(remaining.Take(RequiredQuestions - selected.Count));
|
|
return selected.OrderBy(_ => Random.Shared.Next()).ToList();
|
|
}
|
|
|
|
public static Dictionary<string, int> BuildTargets()
|
|
{
|
|
var weights = new Dictionary<string, double>
|
|
{
|
|
["Human Development"] = 2, ["Immune/Blood/Multisystem"] = 7, ["Behavioral Health"] = 5,
|
|
["Nervous System"] = 9, ["Skin"] = 5, ["Musculoskeletal"] = 6, ["Cardiovascular"] = 10,
|
|
["Respiratory"] = 9, ["Gastrointestinal"] = 7, ["Renal/Urinary/Male"] = 5,
|
|
["Pregnancy/Female/Breast"] = 8, ["Endocrine"] = 6, ["Biostatistics/Population Health"] = 12,
|
|
["Social Sciences"] = 8
|
|
};
|
|
var total = weights.Values.Sum();
|
|
var exact = weights.ToDictionary(pair => pair.Key, pair => pair.Value / total * RequiredQuestions);
|
|
var result = exact.ToDictionary(pair => pair.Key, pair => (int)Math.Floor(pair.Value));
|
|
foreach (var key in exact.OrderByDescending(pair => pair.Value - Math.Floor(pair.Value)).Take(RequiredQuestions - result.Values.Sum()).Select(pair => pair.Key)) result[key]++;
|
|
return result;
|
|
}
|
|
|
|
public static string BlueprintCategory(Question question)
|
|
{
|
|
var value = $"{question.System} {question.PerformanceCategory} {question.Discipline}".ToLowerInvariant();
|
|
if (value.Contains("biostat") || value.Contains("epidemi") || value.Contains("population")) return "Biostatistics/Population Health";
|
|
if (value.Contains("ethic") || value.Contains("communication") || value.Contains("professional") || value.Contains("patient safety")) return "Social Sciences";
|
|
if (value.Contains("pregnan") || value.Contains("obstetric") || value.Contains("female") || value.Contains("breast") || value.Contains("gyne")) return "Pregnancy/Female/Breast";
|
|
if (value.Contains("renal") || value.Contains("urinary") || value.Contains("male") || value.Contains("genitourinary")) return "Renal/Urinary/Male";
|
|
if (value.Contains("cardio")) return "Cardiovascular";
|
|
if (value.Contains("resp") || value.Contains("pulmonary")) return "Respiratory";
|
|
if (value.Contains("gastro") || value.Contains("hepato")) return "Gastrointestinal";
|
|
if (value.Contains("neuro") || value.Contains("special sense")) return "Nervous System";
|
|
if (value.Contains("musculo") || value.Contains("orthop")) return "Musculoskeletal";
|
|
if (value.Contains("skin") || value.Contains("derm")) return "Skin";
|
|
if (value.Contains("behavior") || value.Contains("psychiatr")) return "Behavioral Health";
|
|
if (value.Contains("endocrine") || value.Contains("diabetes") || value.Contains("metabolism")) return "Endocrine";
|
|
if (value.Contains("immune") || value.Contains("hemat") || value.Contains("blood") || value.Contains("multisystem")) return "Immune/Blood/Multisystem";
|
|
return "Human Development";
|
|
}
|
|
}
|