using MongoDB.Driver; using MongoDB.Bson; using QuestionsApi.Models; namespace QuestionsApi.Services; public sealed class DynamicQuestionStore { private readonly IMongoCollection _blueprints; private readonly IMongoCollection _attempts; private readonly IMongoCollection _mastery; private readonly IMongoCollection _familyRatings; private readonly IMongoCollection _reviews; private readonly ServiceMetrics? _metrics; public DynamicQuestionStore(MongoDatabaseProvider mongo, ServiceMetrics? metrics = null) { _metrics = metrics; _blueprints = mongo.Database.GetCollection("DynamicQuestionBlueprints"); _attempts = mongo.Database.GetCollection("DynamicQuestionAttempts"); _mastery = mongo.Database.GetCollection("DynamicObjectiveMastery"); _familyRatings = mongo.Database.GetCollection("DynamicFamilyRatings"); _reviews = mongo.Database.GetCollection("DynamicReviewItems"); _blueprints.Indexes.CreateOne(new CreateIndexModel( Builders.IndexKeys.Ascending(value => value.ExternalId).Ascending(value => value.Version), new CreateIndexOptions { Unique = true })); _attempts.Indexes.CreateOne(new CreateIndexModel( Builders.IndexKeys.Ascending(value => value.UserId).Ascending(value => value.Nonce), new CreateIndexOptions { Unique = true })); _mastery.Indexes.CreateOne(new CreateIndexModel( Builders.IndexKeys.Ascending(value => value.UserId).Ascending(value => value.ObjectiveId), new CreateIndexOptions { Unique = true })); _familyRatings.Indexes.CreateOne(new CreateIndexModel( Builders.IndexKeys .Ascending(value => value.BlueprintExternalId) .Ascending(value => value.BlueprintVersion) .Ascending(value => value.CaseFamilyId), new CreateIndexOptions { Unique = true })); _reviews.Indexes.CreateMany([ new CreateIndexModel( Builders.IndexKeys .Ascending(value => value.UserId) .Ascending(value => value.ObjectiveId), new CreateIndexOptions { Unique = true, Name = "dynamic_review_user_objective" }), new CreateIndexModel( Builders.IndexKeys .Ascending(value => value.UserId) .Ascending(value => value.State) .Ascending(value => value.DueUtc), new CreateIndexOptions { Name = "dynamic_review_user_due" }) ]); } public async Task> PublishedAsync() => await _blueprints.Find(value => value.IsPublished && value.QualityStandardVersion >= 2).ToListAsync(); public async Task GetAsync(string externalId, int version) => await _blueprints.Find(value => value.ExternalId == externalId && value.Version == version).FirstOrDefaultAsync(); public async Task> MasteryAsync(string userId) => (await _mastery.Find(value => value.UserId == userId).ToListAsync()) .ToDictionary(value => value.ObjectiveId, StringComparer.Ordinal); public async Task> DueReviewsAsync(string userId, DateTime nowUtc) => await _reviews.Find(value => value.UserId == userId && value.State == DynamicReviewStates.Pending && value.DueUtc <= nowUtc).ToListAsync(); public async Task> FamilyRatingsAsync(DynamicQuestionBlueprint blueprint) { var ratings = await _familyRatings.Find(value => value.BlueprintExternalId == blueprint.ExternalId && value.BlueprintVersion == blueprint.Version).ToListAsync(); return ratings.ToDictionary(value => value.CaseFamilyId, StringComparer.Ordinal); } public async Task RecordAsync( DynamicGenerationClaims claims, DynamicGeneratedQuestion generated, DynamicAttemptRequest request) { var isCorrect = !request.IsSkip && request.ChoiceId == generated.Question.CorrectChoiceId; var selectedAnswer = generated.Blueprint.Distractors .Concat(generated.Blueprint.CaseFamilies .Where(value => value.Id == claims.CaseFamilyId) .SelectMany(value => value.Distractors)) .FirstOrDefault(value => value.Id == request.ChoiceId); var misconceptionId = isCorrect || request.IsSkip ? "" : selectedAnswer?.MisconceptionId ?? ""; var attempt = new DynamicQuestionAttempt { UserId = claims.UserId, Nonce = claims.Nonce, BlueprintExternalId = claims.BlueprintExternalId, BlueprintVersion = claims.BlueprintVersion, ObjectiveId = claims.ObjectiveId, CaseFamilyId = claims.CaseFamilyId, DifficultyTier = claims.DifficultyTier, EffectiveAbilityElo = claims.EffectiveAbilityElo, FamilyElo = claims.FamilyElo, ChoiceId = request.ChoiceId, IsCorrect = isCorrect, IsSkip = request.IsSkip, SelectedMisconceptionId = misconceptionId, IsScheduledReview = claims.IsScheduledReview, DurationSeconds = Math.Clamp(request.DurationSeconds, 0, 86_400), CreatedUtc = DateTime.UtcNow }; try { await _attempts.InsertOneAsync(attempt); } catch (MongoWriteException exception) when (exception.WriteError.Category == ServerErrorCategory.DuplicateKey) { var existingMastery = await _mastery.Find(value => value.UserId == claims.UserId && value.ObjectiveId == claims.ObjectiveId).FirstOrDefaultAsync(); return new DynamicAttemptResponse { Inserted = false, IsCorrect = isCorrect, CorrectChoiceId = generated.Question.CorrectChoiceId, ObjectiveId = claims.ObjectiveId, ObjectiveElo = existingMastery?.Elo ?? 1000 }; } _metrics?.DynamicAttempt(claims.DifficultyTier, isCorrect); await EnsureMasteryAsync(claims.UserId, claims.ObjectiveId); var current = await _mastery.Find(value => value.UserId == claims.UserId && value.ObjectiveId == claims.ObjectiveId).FirstAsync() ?? throw new InvalidOperationException("Dynamic objective mastery could not be initialized."); var oldElo = current.Elo; var familyElo = claims.FamilyElo > 0 ? claims.FamilyElo : 800 + Math.Clamp(generated.Blueprint.Difficulty, 1, 5) * 100; var delta = DynamicDifficultyPolicy.LearnerDelta(oldElo, familyElo, isCorrect, current.Attempts); var now = DateTime.UtcNow; var update = Builders.Update .Inc(value => value.Elo, delta) .Inc(value => value.Attempts, 1) .Inc(value => value.Correct, isCorrect ? 1 : 0) .Set(value => value.LastSeenUtc, now); var updated = await _mastery.FindOneAndUpdateAsync( value => value.UserId == claims.UserId && value.ObjectiveId == claims.ObjectiveId, update, new FindOneAndUpdateOptions { ReturnDocument = ReturnDocument.After }); await _mastery.UpdateOneAsync( value => value.UserId == claims.UserId && value.ObjectiveId == claims.ObjectiveId, Builders.Update .Max(value => value.Elo, DynamicDifficultyPolicy.MinimumElo) .Min(value => value.Elo, DynamicDifficultyPolicy.MaximumElo)); if (!string.IsNullOrWhiteSpace(claims.CaseFamilyId)) await UpdateFamilyRatingAsync(claims, generated, isCorrect); DateTime? nextReviewUtc = null; if (!request.IsSkip) nextReviewUtc = await UpdateReviewAsync(claims, misconceptionId, isCorrect); return new DynamicAttemptResponse { Inserted = true, IsCorrect = isCorrect, CorrectChoiceId = generated.Question.CorrectChoiceId, ObjectiveId = claims.ObjectiveId, ObjectiveElo = updated.Elo, RatingChange = delta, SelectedMisconceptionId = misconceptionId, IsScheduledReview = claims.IsScheduledReview, NextReviewUtc = nextReviewUtc }; } private async Task UpdateReviewAsync( DynamicGenerationClaims claims, string misconceptionId, bool isCorrect) { var now = DateTime.UtcNow; var existing = await _reviews.Find(value => value.UserId == claims.UserId && value.ObjectiveId == claims.ObjectiveId).FirstOrDefaultAsync(); if (!claims.IsScheduledReview) { if (isCorrect) return existing?.DueUtc; var due = DynamicReviewPolicy.FirstDue(now); var update = Builders.Update .SetOnInsert(value => value.UserId, claims.UserId) .SetOnInsert(value => value.ObjectiveId, claims.ObjectiveId) .SetOnInsert(value => value.CreatedUtc, now) .Set(value => value.BlueprintExternalId, claims.BlueprintExternalId) .Set(value => value.BlueprintVersion, claims.BlueprintVersion) .Set(value => value.MisconceptionId, misconceptionId) .Set(value => value.State, DynamicReviewStates.Pending) .Set(value => value.Stage, 0) .Set(value => value.DueUtc, due) .Set(value => value.UpdatedUtc, now); await _reviews.UpdateOneAsync( value => value.UserId == claims.UserId && value.ObjectiveId == claims.ObjectiveId, update, new UpdateOptions { IsUpsert = true }); return due; } if (existing is null || existing.Id != claims.ReviewItemId || existing.Stage != claims.ReviewStage) return null; var next = DynamicReviewPolicy.AfterReview(existing.Stage, isCorrect, now); var reviewUpdate = Builders.Update .Set(value => value.Stage, next.Stage) .Set(value => value.State, next.Mastered ? DynamicReviewStates.Mastered : DynamicReviewStates.Pending) .Set(value => value.MisconceptionId, isCorrect ? existing.MisconceptionId : misconceptionId) .Set(value => value.UpdatedUtc, now) .Inc(value => value.ReviewCount, 1); if (next.DueUtc.HasValue) reviewUpdate = reviewUpdate.Set(value => value.DueUtc, next.DueUtc.Value); await _reviews.UpdateOneAsync( value => value.Id == existing.Id && value.Stage == claims.ReviewStage, reviewUpdate); return next.DueUtc; } private async Task EnsureMasteryAsync(string userId, string objectiveId) { try { await _mastery.UpdateOneAsync( value => value.UserId == userId && value.ObjectiveId == objectiveId, Builders.Update .SetOnInsert(value => value.UserId, userId) .SetOnInsert(value => value.ObjectiveId, objectiveId) .SetOnInsert(value => value.Elo, 1000) .SetOnInsert(value => value.Attempts, 0) .SetOnInsert(value => value.Correct, 0), new UpdateOptions { IsUpsert = true }); } catch (MongoWriteException exception) when (exception.WriteError.Category == ServerErrorCategory.DuplicateKey) { } } private async Task UpdateFamilyRatingAsync( DynamicGenerationClaims claims, DynamicGeneratedQuestion generated, bool isCorrect) { var family = generated.Blueprint.CaseFamilies.First(value => value.Id == claims.CaseFamilyId); try { await _familyRatings.UpdateOneAsync( value => value.BlueprintExternalId == claims.BlueprintExternalId && value.BlueprintVersion == claims.BlueprintVersion && value.CaseFamilyId == claims.CaseFamilyId, Builders.Update .SetOnInsert(value => value.BlueprintExternalId, claims.BlueprintExternalId) .SetOnInsert(value => value.BlueprintVersion, claims.BlueprintVersion) .SetOnInsert(value => value.CaseFamilyId, claims.CaseFamilyId) .SetOnInsert(value => value.DifficultyTier, family.DifficultyTier) .SetOnInsert(value => value.NominalElo, family.NominalElo) .SetOnInsert(value => value.Elo, family.NominalElo) .SetOnInsert(value => value.Attempts, 0) .SetOnInsert(value => value.Correct, 0), new UpdateOptions { IsUpsert = true }); } catch (MongoWriteException exception) when (exception.WriteError.Category == ServerErrorCategory.DuplicateKey) { } var current = await _familyRatings.Find(value => value.BlueprintExternalId == claims.BlueprintExternalId && value.BlueprintVersion == claims.BlueprintVersion && value.CaseFamilyId == claims.CaseFamilyId).FirstAsync(); var delta = DynamicDifficultyPolicy.FamilyDelta( claims.EffectiveAbilityElo > 0 ? claims.EffectiveAbilityElo : 1000, claims.FamilyElo > 0 ? claims.FamilyElo : current.Elo, isCorrect, current.Attempts); await _familyRatings.UpdateOneAsync( value => value.Id == current.Id, Builders.Update .Inc(value => value.Elo, delta) .Inc(value => value.Attempts, 1) .Inc(value => value.Correct, isCorrect ? 1 : 0) .Set(value => value.UpdatedUtc, DateTime.UtcNow)); await _familyRatings.UpdateOneAsync( value => value.Id == current.Id, Builders.Update .Max(value => value.Elo, DynamicDifficultyPolicy.MinimumElo) .Min(value => value.Elo, DynamicDifficultyPolicy.MaximumElo)); } public async Task StatusAsync(string userId) { var blueprints = (await PublishedAsync()) .GroupBy(value => value.ObjectiveId) .Select(value => value.OrderByDescending(item => item.Version).First()) .ToList(); var objectiveCount = blueprints.Count; var mastery = await _mastery.Find(value => value.UserId == userId).ToListAsync(); var masteryByObjective = mastery.ToDictionary(value => value.ObjectiveId, StringComparer.Ordinal); var due = await _reviews.CountDocumentsAsync(value => value.UserId == userId && value.State == DynamicReviewStates.Pending && value.DueUtc <= DateTime.UtcNow); return new DynamicQuestionStatusResponse { Available = objectiveCount > 0, PublishedObjectives = objectiveCount, AttemptedObjectives = mastery.Count, TotalAttempts = mastery.Sum(value => value.Attempts), AverageObjectiveElo = mastery.Count == 0 ? 1000 : (int)Math.Round(mastery.Average(value => value.Elo)), ReviewsDue = checked((int)due), SystemCoverage = Coverage(blueprints, masteryByObjective, value => value.System), PhysicianTaskCoverage = Coverage(blueprints, masteryByObjective, value => value.PhysicianTask) }; } private static List Coverage( IReadOnlyCollection blueprints, IReadOnlyDictionary mastery, Func keySelector) => blueprints .GroupBy(value => string.IsNullOrWhiteSpace(keySelector(value)) ? "Unclassified" : keySelector(value)) .Select(group => { var attempts = group .Where(value => mastery.ContainsKey(value.ObjectiveId)) .Select(value => mastery[value.ObjectiveId]) .ToList(); return new DynamicCoverageSummary { Id = group.Key, PublishedObjectives = group.Count(), AttemptedObjectives = attempts.Count, Attempts = attempts.Sum(value => value.Attempts), CoveragePercent = group.Count() == 0 ? 0 : Math.Round(attempts.Count * 100.0 / group.Count(), 1), AverageElo = attempts.Count == 0 ? 1000 : (int)Math.Round(attempts.Average(value => value.Elo)) }; }) .OrderBy(value => value.CoveragePercent) .ThenBy(value => value.Id) .ToList(); public async Task<(int Inserted, int Unchanged)> UpsertBlueprintsAsync(IEnumerable blueprints) { var inserted = 0; var unchanged = 0; foreach (var blueprint in blueprints) { var existing = await _blueprints.Find( value => value.ExternalId == blueprint.ExternalId && value.Version == blueprint.Version) .FirstOrDefaultAsync(); if (existing is not null) { blueprint.Id = existing.Id; var incomingDocument = blueprint.ToBsonDocument(); var existingDocument = existing.ToBsonDocument(); incomingDocument.Remove("_id"); existingDocument.Remove("_id"); if (incomingDocument != existingDocument) throw new InvalidDataException( $"{blueprint.ExternalId} v{blueprint.Version} is immutable; publish changes with a new version"); unchanged++; continue; } blueprint.Id = ObjectId.GenerateNewId().ToString(); await _blueprints.InsertOneAsync(blueprint); inserted++; } return (inserted, unchanged); } }