365 lines
19 KiB
C#
365 lines
19 KiB
C#
using MongoDB.Driver;
|
|
using MongoDB.Bson;
|
|
using QuestionsApi.Models;
|
|
|
|
namespace QuestionsApi.Services;
|
|
|
|
public sealed class DynamicQuestionStore
|
|
{
|
|
private readonly IMongoCollection<DynamicQuestionBlueprint> _blueprints;
|
|
private readonly IMongoCollection<DynamicQuestionAttempt> _attempts;
|
|
private readonly IMongoCollection<DynamicObjectiveMastery> _mastery;
|
|
private readonly IMongoCollection<DynamicFamilyRating> _familyRatings;
|
|
private readonly IMongoCollection<DynamicReviewItem> _reviews;
|
|
private readonly ServiceMetrics? _metrics;
|
|
|
|
public DynamicQuestionStore(MongoDatabaseProvider mongo, ServiceMetrics? metrics = null)
|
|
{
|
|
_metrics = metrics;
|
|
_blueprints = mongo.Database.GetCollection<DynamicQuestionBlueprint>("DynamicQuestionBlueprints");
|
|
_attempts = mongo.Database.GetCollection<DynamicQuestionAttempt>("DynamicQuestionAttempts");
|
|
_mastery = mongo.Database.GetCollection<DynamicObjectiveMastery>("DynamicObjectiveMastery");
|
|
_familyRatings = mongo.Database.GetCollection<DynamicFamilyRating>("DynamicFamilyRatings");
|
|
_reviews = mongo.Database.GetCollection<DynamicReviewItem>("DynamicReviewItems");
|
|
_blueprints.Indexes.CreateOne(new CreateIndexModel<DynamicQuestionBlueprint>(
|
|
Builders<DynamicQuestionBlueprint>.IndexKeys.Ascending(value => value.ExternalId).Ascending(value => value.Version),
|
|
new CreateIndexOptions { Unique = true }));
|
|
_attempts.Indexes.CreateOne(new CreateIndexModel<DynamicQuestionAttempt>(
|
|
Builders<DynamicQuestionAttempt>.IndexKeys.Ascending(value => value.UserId).Ascending(value => value.Nonce),
|
|
new CreateIndexOptions { Unique = true }));
|
|
_mastery.Indexes.CreateOne(new CreateIndexModel<DynamicObjectiveMastery>(
|
|
Builders<DynamicObjectiveMastery>.IndexKeys.Ascending(value => value.UserId).Ascending(value => value.ObjectiveId),
|
|
new CreateIndexOptions { Unique = true }));
|
|
_familyRatings.Indexes.CreateOne(new CreateIndexModel<DynamicFamilyRating>(
|
|
Builders<DynamicFamilyRating>.IndexKeys
|
|
.Ascending(value => value.BlueprintExternalId)
|
|
.Ascending(value => value.BlueprintVersion)
|
|
.Ascending(value => value.CaseFamilyId),
|
|
new CreateIndexOptions { Unique = true }));
|
|
_reviews.Indexes.CreateMany([
|
|
new CreateIndexModel<DynamicReviewItem>(
|
|
Builders<DynamicReviewItem>.IndexKeys
|
|
.Ascending(value => value.UserId)
|
|
.Ascending(value => value.ObjectiveId),
|
|
new CreateIndexOptions { Unique = true, Name = "dynamic_review_user_objective" }),
|
|
new CreateIndexModel<DynamicReviewItem>(
|
|
Builders<DynamicReviewItem>.IndexKeys
|
|
.Ascending(value => value.UserId)
|
|
.Ascending(value => value.State)
|
|
.Ascending(value => value.DueUtc),
|
|
new CreateIndexOptions { Name = "dynamic_review_user_due" })
|
|
]);
|
|
}
|
|
|
|
public async Task<List<DynamicQuestionBlueprint>> PublishedAsync() =>
|
|
await _blueprints.Find(value => value.IsPublished && value.QualityStandardVersion >= 2).ToListAsync();
|
|
|
|
public async Task<DynamicQuestionBlueprint?> GetAsync(string externalId, int version) =>
|
|
await _blueprints.Find(value => value.ExternalId == externalId && value.Version == version).FirstOrDefaultAsync();
|
|
|
|
public async Task<Dictionary<string, DynamicObjectiveMastery>> MasteryAsync(string userId) =>
|
|
(await _mastery.Find(value => value.UserId == userId).ToListAsync())
|
|
.ToDictionary(value => value.ObjectiveId, StringComparer.Ordinal);
|
|
|
|
public async Task<List<DynamicReviewItem>> DueReviewsAsync(string userId, DateTime nowUtc) =>
|
|
await _reviews.Find(value => value.UserId == userId
|
|
&& value.State == DynamicReviewStates.Pending
|
|
&& value.DueUtc <= nowUtc).ToListAsync();
|
|
|
|
public async Task<Dictionary<string, DynamicFamilyRating>> FamilyRatingsAsync(DynamicQuestionBlueprint blueprint)
|
|
{
|
|
var ratings = await _familyRatings.Find(value =>
|
|
value.BlueprintExternalId == blueprint.ExternalId
|
|
&& value.BlueprintVersion == blueprint.Version).ToListAsync();
|
|
return ratings.ToDictionary(value => value.CaseFamilyId, StringComparer.Ordinal);
|
|
}
|
|
|
|
public async Task<DynamicAttemptResponse> RecordAsync(
|
|
DynamicGenerationClaims claims,
|
|
DynamicGeneratedQuestion generated,
|
|
DynamicAttemptRequest request)
|
|
{
|
|
var isCorrect = !request.IsSkip && request.ChoiceId == generated.Question.CorrectChoiceId;
|
|
var selectedAnswer = generated.Blueprint.Distractors
|
|
.Concat(generated.Blueprint.CaseFamilies
|
|
.Where(value => value.Id == claims.CaseFamilyId)
|
|
.SelectMany(value => value.Distractors))
|
|
.FirstOrDefault(value => value.Id == request.ChoiceId);
|
|
var misconceptionId = isCorrect || request.IsSkip
|
|
? ""
|
|
: selectedAnswer?.MisconceptionId ?? "";
|
|
var attempt = new DynamicQuestionAttempt
|
|
{
|
|
UserId = claims.UserId,
|
|
Nonce = claims.Nonce,
|
|
BlueprintExternalId = claims.BlueprintExternalId,
|
|
BlueprintVersion = claims.BlueprintVersion,
|
|
ObjectiveId = claims.ObjectiveId,
|
|
CaseFamilyId = claims.CaseFamilyId,
|
|
DifficultyTier = claims.DifficultyTier,
|
|
EffectiveAbilityElo = claims.EffectiveAbilityElo,
|
|
FamilyElo = claims.FamilyElo,
|
|
ChoiceId = request.ChoiceId,
|
|
IsCorrect = isCorrect,
|
|
IsSkip = request.IsSkip,
|
|
SelectedMisconceptionId = misconceptionId,
|
|
IsScheduledReview = claims.IsScheduledReview,
|
|
DurationSeconds = Math.Clamp(request.DurationSeconds, 0, 86_400),
|
|
CreatedUtc = DateTime.UtcNow
|
|
};
|
|
try { await _attempts.InsertOneAsync(attempt); }
|
|
catch (MongoWriteException exception) when (exception.WriteError.Category == ServerErrorCategory.DuplicateKey)
|
|
{
|
|
var existingMastery = await _mastery.Find(value => value.UserId == claims.UserId && value.ObjectiveId == claims.ObjectiveId).FirstOrDefaultAsync();
|
|
return new DynamicAttemptResponse
|
|
{
|
|
Inserted = false,
|
|
IsCorrect = isCorrect,
|
|
CorrectChoiceId = generated.Question.CorrectChoiceId,
|
|
ObjectiveId = claims.ObjectiveId,
|
|
ObjectiveElo = existingMastery?.Elo ?? 1000
|
|
};
|
|
}
|
|
_metrics?.DynamicAttempt(claims.DifficultyTier, isCorrect);
|
|
|
|
await EnsureMasteryAsync(claims.UserId, claims.ObjectiveId);
|
|
var current = await _mastery.Find(value => value.UserId == claims.UserId && value.ObjectiveId == claims.ObjectiveId).FirstAsync()
|
|
?? throw new InvalidOperationException("Dynamic objective mastery could not be initialized.");
|
|
var oldElo = current.Elo;
|
|
var familyElo = claims.FamilyElo > 0
|
|
? claims.FamilyElo
|
|
: 800 + Math.Clamp(generated.Blueprint.Difficulty, 1, 5) * 100;
|
|
var delta = DynamicDifficultyPolicy.LearnerDelta(oldElo, familyElo, isCorrect, current.Attempts);
|
|
var now = DateTime.UtcNow;
|
|
var update = Builders<DynamicObjectiveMastery>.Update
|
|
.Inc(value => value.Elo, delta)
|
|
.Inc(value => value.Attempts, 1)
|
|
.Inc(value => value.Correct, isCorrect ? 1 : 0)
|
|
.Set(value => value.LastSeenUtc, now);
|
|
var updated = await _mastery.FindOneAndUpdateAsync(
|
|
value => value.UserId == claims.UserId && value.ObjectiveId == claims.ObjectiveId,
|
|
update,
|
|
new FindOneAndUpdateOptions<DynamicObjectiveMastery> { ReturnDocument = ReturnDocument.After });
|
|
await _mastery.UpdateOneAsync(
|
|
value => value.UserId == claims.UserId && value.ObjectiveId == claims.ObjectiveId,
|
|
Builders<DynamicObjectiveMastery>.Update
|
|
.Max(value => value.Elo, DynamicDifficultyPolicy.MinimumElo));
|
|
await _mastery.UpdateOneAsync(
|
|
value => value.UserId == claims.UserId && value.ObjectiveId == claims.ObjectiveId,
|
|
Builders<DynamicObjectiveMastery>.Update
|
|
.Min(value => value.Elo, DynamicDifficultyPolicy.MaximumElo));
|
|
if (!string.IsNullOrWhiteSpace(claims.CaseFamilyId))
|
|
await UpdateFamilyRatingAsync(claims, generated, isCorrect);
|
|
DateTime? nextReviewUtc = null;
|
|
if (!request.IsSkip)
|
|
nextReviewUtc = await UpdateReviewAsync(claims, misconceptionId, isCorrect);
|
|
return new DynamicAttemptResponse
|
|
{
|
|
Inserted = true,
|
|
IsCorrect = isCorrect,
|
|
CorrectChoiceId = generated.Question.CorrectChoiceId,
|
|
ObjectiveId = claims.ObjectiveId,
|
|
ObjectiveElo = DynamicDifficultyPolicy.Clamp(updated.Elo),
|
|
RatingChange = delta,
|
|
SelectedMisconceptionId = misconceptionId,
|
|
IsScheduledReview = claims.IsScheduledReview,
|
|
NextReviewUtc = nextReviewUtc
|
|
};
|
|
}
|
|
|
|
private async Task<DateTime?> UpdateReviewAsync(
|
|
DynamicGenerationClaims claims,
|
|
string misconceptionId,
|
|
bool isCorrect)
|
|
{
|
|
var now = DateTime.UtcNow;
|
|
var existing = await _reviews.Find(value =>
|
|
value.UserId == claims.UserId && value.ObjectiveId == claims.ObjectiveId).FirstOrDefaultAsync();
|
|
|
|
if (!claims.IsScheduledReview)
|
|
{
|
|
if (isCorrect) return existing?.DueUtc;
|
|
var due = DynamicReviewPolicy.FirstDue(now);
|
|
var update = Builders<DynamicReviewItem>.Update
|
|
.SetOnInsert(value => value.UserId, claims.UserId)
|
|
.SetOnInsert(value => value.ObjectiveId, claims.ObjectiveId)
|
|
.SetOnInsert(value => value.CreatedUtc, now)
|
|
.Set(value => value.BlueprintExternalId, claims.BlueprintExternalId)
|
|
.Set(value => value.BlueprintVersion, claims.BlueprintVersion)
|
|
.Set(value => value.MisconceptionId, misconceptionId)
|
|
.Set(value => value.State, DynamicReviewStates.Pending)
|
|
.Set(value => value.Stage, 0)
|
|
.Set(value => value.DueUtc, due)
|
|
.Set(value => value.UpdatedUtc, now);
|
|
await _reviews.UpdateOneAsync(
|
|
value => value.UserId == claims.UserId && value.ObjectiveId == claims.ObjectiveId,
|
|
update,
|
|
new UpdateOptions { IsUpsert = true });
|
|
return due;
|
|
}
|
|
|
|
if (existing is null || existing.Id != claims.ReviewItemId || existing.Stage != claims.ReviewStage)
|
|
return null;
|
|
var next = DynamicReviewPolicy.AfterReview(existing.Stage, isCorrect, now);
|
|
var reviewUpdate = Builders<DynamicReviewItem>.Update
|
|
.Set(value => value.Stage, next.Stage)
|
|
.Set(value => value.State, next.Mastered ? DynamicReviewStates.Mastered : DynamicReviewStates.Pending)
|
|
.Set(value => value.MisconceptionId, isCorrect ? existing.MisconceptionId : misconceptionId)
|
|
.Set(value => value.UpdatedUtc, now)
|
|
.Inc(value => value.ReviewCount, 1);
|
|
if (next.DueUtc.HasValue)
|
|
reviewUpdate = reviewUpdate.Set(value => value.DueUtc, next.DueUtc.Value);
|
|
await _reviews.UpdateOneAsync(
|
|
value => value.Id == existing.Id && value.Stage == claims.ReviewStage,
|
|
reviewUpdate);
|
|
return next.DueUtc;
|
|
}
|
|
|
|
private async Task EnsureMasteryAsync(string userId, string objectiveId)
|
|
{
|
|
try
|
|
{
|
|
await _mastery.UpdateOneAsync(
|
|
value => value.UserId == userId && value.ObjectiveId == objectiveId,
|
|
Builders<DynamicObjectiveMastery>.Update
|
|
.SetOnInsert(value => value.UserId, userId)
|
|
.SetOnInsert(value => value.ObjectiveId, objectiveId)
|
|
.SetOnInsert(value => value.Elo, 1000)
|
|
.SetOnInsert(value => value.Attempts, 0)
|
|
.SetOnInsert(value => value.Correct, 0),
|
|
new UpdateOptions { IsUpsert = true });
|
|
}
|
|
catch (MongoWriteException exception) when (exception.WriteError.Category == ServerErrorCategory.DuplicateKey) { }
|
|
}
|
|
|
|
private async Task UpdateFamilyRatingAsync(
|
|
DynamicGenerationClaims claims,
|
|
DynamicGeneratedQuestion generated,
|
|
bool isCorrect)
|
|
{
|
|
var family = generated.Blueprint.CaseFamilies.First(value => value.Id == claims.CaseFamilyId);
|
|
try
|
|
{
|
|
await _familyRatings.UpdateOneAsync(
|
|
value => value.BlueprintExternalId == claims.BlueprintExternalId
|
|
&& value.BlueprintVersion == claims.BlueprintVersion
|
|
&& value.CaseFamilyId == claims.CaseFamilyId,
|
|
Builders<DynamicFamilyRating>.Update
|
|
.SetOnInsert(value => value.BlueprintExternalId, claims.BlueprintExternalId)
|
|
.SetOnInsert(value => value.BlueprintVersion, claims.BlueprintVersion)
|
|
.SetOnInsert(value => value.CaseFamilyId, claims.CaseFamilyId)
|
|
.SetOnInsert(value => value.DifficultyTier, family.DifficultyTier)
|
|
.SetOnInsert(value => value.NominalElo, family.NominalElo)
|
|
.SetOnInsert(value => value.Elo, family.NominalElo)
|
|
.SetOnInsert(value => value.Attempts, 0)
|
|
.SetOnInsert(value => value.Correct, 0),
|
|
new UpdateOptions { IsUpsert = true });
|
|
}
|
|
catch (MongoWriteException exception) when (exception.WriteError.Category == ServerErrorCategory.DuplicateKey) { }
|
|
var current = await _familyRatings.Find(value =>
|
|
value.BlueprintExternalId == claims.BlueprintExternalId
|
|
&& value.BlueprintVersion == claims.BlueprintVersion
|
|
&& value.CaseFamilyId == claims.CaseFamilyId).FirstAsync();
|
|
var delta = DynamicDifficultyPolicy.FamilyDelta(
|
|
claims.EffectiveAbilityElo > 0 ? claims.EffectiveAbilityElo : 1000,
|
|
claims.FamilyElo > 0 ? claims.FamilyElo : current.Elo,
|
|
isCorrect,
|
|
current.Attempts);
|
|
await _familyRatings.UpdateOneAsync(
|
|
value => value.Id == current.Id,
|
|
Builders<DynamicFamilyRating>.Update
|
|
.Inc(value => value.Elo, delta)
|
|
.Inc(value => value.Attempts, 1)
|
|
.Inc(value => value.Correct, isCorrect ? 1 : 0)
|
|
.Set(value => value.UpdatedUtc, DateTime.UtcNow));
|
|
await _familyRatings.UpdateOneAsync(
|
|
value => value.Id == current.Id,
|
|
Builders<DynamicFamilyRating>.Update
|
|
.Max(value => value.Elo, DynamicDifficultyPolicy.MinimumElo));
|
|
await _familyRatings.UpdateOneAsync(
|
|
value => value.Id == current.Id,
|
|
Builders<DynamicFamilyRating>.Update
|
|
.Min(value => value.Elo, DynamicDifficultyPolicy.MaximumElo));
|
|
}
|
|
|
|
public async Task<DynamicQuestionStatusResponse> StatusAsync(string userId)
|
|
{
|
|
var blueprints = (await PublishedAsync())
|
|
.GroupBy(value => value.ObjectiveId)
|
|
.Select(value => value.OrderByDescending(item => item.Version).First())
|
|
.ToList();
|
|
var objectiveCount = blueprints.Count;
|
|
var mastery = await _mastery.Find(value => value.UserId == userId).ToListAsync();
|
|
var masteryByObjective = mastery.ToDictionary(value => value.ObjectiveId, StringComparer.Ordinal);
|
|
var due = await _reviews.CountDocumentsAsync(value => value.UserId == userId
|
|
&& value.State == DynamicReviewStates.Pending
|
|
&& value.DueUtc <= DateTime.UtcNow);
|
|
return new DynamicQuestionStatusResponse
|
|
{
|
|
Available = objectiveCount > 0,
|
|
PublishedObjectives = objectiveCount,
|
|
AttemptedObjectives = mastery.Count,
|
|
TotalAttempts = mastery.Sum(value => value.Attempts),
|
|
AverageObjectiveElo = mastery.Count == 0 ? 1000 : (int)Math.Round(mastery.Average(value => value.Elo)),
|
|
ReviewsDue = checked((int)due),
|
|
SystemCoverage = Coverage(blueprints, masteryByObjective, value => value.System),
|
|
PhysicianTaskCoverage = Coverage(blueprints, masteryByObjective, value => value.PhysicianTask)
|
|
};
|
|
}
|
|
|
|
private static List<DynamicCoverageSummary> Coverage(
|
|
IReadOnlyCollection<DynamicQuestionBlueprint> blueprints,
|
|
IReadOnlyDictionary<string, DynamicObjectiveMastery> mastery,
|
|
Func<DynamicQuestionBlueprint, string> keySelector) =>
|
|
blueprints
|
|
.GroupBy(value => string.IsNullOrWhiteSpace(keySelector(value)) ? "Unclassified" : keySelector(value))
|
|
.Select(group =>
|
|
{
|
|
var attempts = group
|
|
.Where(value => mastery.ContainsKey(value.ObjectiveId))
|
|
.Select(value => mastery[value.ObjectiveId])
|
|
.ToList();
|
|
return new DynamicCoverageSummary
|
|
{
|
|
Id = group.Key,
|
|
PublishedObjectives = group.Count(),
|
|
AttemptedObjectives = attempts.Count,
|
|
Attempts = attempts.Sum(value => value.Attempts),
|
|
CoveragePercent = group.Count() == 0 ? 0 : Math.Round(attempts.Count * 100.0 / group.Count(), 1),
|
|
AverageElo = attempts.Count == 0 ? 1000 : (int)Math.Round(attempts.Average(value => value.Elo))
|
|
};
|
|
})
|
|
.OrderBy(value => value.CoveragePercent)
|
|
.ThenBy(value => value.Id)
|
|
.ToList();
|
|
|
|
public async Task<(int Inserted, int Unchanged)> UpsertBlueprintsAsync(IEnumerable<DynamicQuestionBlueprint> blueprints)
|
|
{
|
|
var inserted = 0;
|
|
var unchanged = 0;
|
|
foreach (var blueprint in blueprints)
|
|
{
|
|
var existing = await _blueprints.Find(
|
|
value => value.ExternalId == blueprint.ExternalId && value.Version == blueprint.Version)
|
|
.FirstOrDefaultAsync();
|
|
if (existing is not null)
|
|
{
|
|
blueprint.Id = existing.Id;
|
|
var incomingDocument = blueprint.ToBsonDocument();
|
|
var existingDocument = existing.ToBsonDocument();
|
|
incomingDocument.Remove("_id");
|
|
existingDocument.Remove("_id");
|
|
if (incomingDocument != existingDocument)
|
|
throw new InvalidDataException(
|
|
$"{blueprint.ExternalId} v{blueprint.Version} is immutable; publish changes with a new version");
|
|
unchanged++;
|
|
continue;
|
|
}
|
|
blueprint.Id = ObjectId.GenerateNewId().ToString();
|
|
await _blueprints.InsertOneAsync(blueprint);
|
|
inserted++;
|
|
}
|
|
return (inserted, unchanged);
|
|
}
|
|
}
|