hz/step3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Adapt dynamic question difficulty to learner Elo

Browse Source
This commit is contained in:
hz
2026-07-23 16:18:44 -05:00
parent 5b5741de37
commit fae38d2ded
13 changed files with 519 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/QuestionsApi/Services/DynamicQuestionStore.cs
+105 -11
View File
@@ -9,12 +9,16 @@ public sealed class DynamicQuestionStore
private readonly IMongoCollection<DynamicQuestionBlueprint> _blueprints;
private readonly IMongoCollection<DynamicQuestionAttempt> _attempts;
private readonly IMongoCollection<DynamicObjectiveMastery> _mastery;
private readonly IMongoCollection<DynamicFamilyRating> _familyRatings;
private readonly ServiceMetrics? _metrics;
public DynamicQuestionStore(MongoDatabaseProvider mongo)
public DynamicQuestionStore(MongoDatabaseProvider mongo, ServiceMetrics? metrics = null)
{
_metrics = metrics;
_blueprints = mongo.Database.GetCollection<DynamicQuestionBlueprint>("DynamicQuestionBlueprints");
_attempts = mongo.Database.GetCollection<DynamicQuestionAttempt>("DynamicQuestionAttempts");
_mastery = mongo.Database.GetCollection<DynamicObjectiveMastery>("DynamicObjectiveMastery");
_familyRatings = mongo.Database.GetCollection<DynamicFamilyRating>("DynamicFamilyRatings");
_blueprints.Indexes.CreateOne(new CreateIndexModel<DynamicQuestionBlueprint>(
Builders<DynamicQuestionBlueprint>.IndexKeys.Ascending(value => value.ExternalId).Ascending(value => value.Version),
new CreateIndexOptions { Unique = true }));
@@ -24,6 +28,12 @@ public sealed class DynamicQuestionStore
_mastery.Indexes.CreateOne(new CreateIndexModel<DynamicObjectiveMastery>(
Builders<DynamicObjectiveMastery>.IndexKeys.Ascending(value => value.UserId).Ascending(value => value.ObjectiveId),
new CreateIndexOptions { Unique = true }));
_familyRatings.Indexes.CreateOne(new CreateIndexModel<DynamicFamilyRating>(
Builders<DynamicFamilyRating>.IndexKeys
.Ascending(value => value.BlueprintExternalId)
.Ascending(value => value.BlueprintVersion)
.Ascending(value => value.CaseFamilyId),
new CreateIndexOptions { Unique = true }));
}
public async Task<List<DynamicQuestionBlueprint>> PublishedAsync() =>
@@ -36,6 +46,14 @@ public sealed class DynamicQuestionStore
(await _mastery.Find(value => value.UserId == userId).ToListAsync())
.ToDictionary(value => value.ObjectiveId, StringComparer.Ordinal);
public async Task<Dictionary<string, DynamicFamilyRating>> FamilyRatingsAsync(DynamicQuestionBlueprint blueprint)
{
var ratings = await _familyRatings.Find(value =>
value.BlueprintExternalId == blueprint.ExternalId
&& value.BlueprintVersion == blueprint.Version).ToListAsync();
return ratings.ToDictionary(value => value.CaseFamilyId, StringComparer.Ordinal);
}
public async Task<DynamicAttemptResponse> RecordAsync(
DynamicGenerationClaims claims,
DynamicGeneratedQuestion generated,
@@ -49,6 +67,10 @@ public sealed class DynamicQuestionStore
BlueprintExternalId = claims.BlueprintExternalId,
BlueprintVersion = claims.BlueprintVersion,
ObjectiveId = claims.ObjectiveId,
CaseFamilyId = claims.CaseFamilyId,
DifficultyTier = claims.DifficultyTier,
EffectiveAbilityElo = claims.EffectiveAbilityElo,
FamilyElo = claims.FamilyElo,
ChoiceId = request.ChoiceId,
IsCorrect = isCorrect,
IsSkip = request.IsSkip,
@@ -68,25 +90,33 @@ public sealed class DynamicQuestionStore
ObjectiveElo = existingMastery?.Elo ?? 1000
};
}
_metrics?.DynamicAttempt(claims.DifficultyTier, isCorrect);
var current = await _mastery.Find(value => value.UserId == claims.UserId && value.ObjectiveId == claims.ObjectiveId).FirstOrDefaultAsync();
var oldElo = current?.Elo ?? 1000;
var objectiveDifficulty = 800 + Math.Clamp(generated.Blueprint.Difficulty, 1, 5) * 100;
var expected = 1.0 / (1.0 + Math.Pow(10, (objectiveDifficulty - oldElo) / 400.0));
var k = (current?.Attempts ?? 0) < 5 ? 40 : 24;
var delta = (int)Math.Round(k * ((isCorrect ? 1.0 : 0.0) - expected));
await EnsureMasteryAsync(claims.UserId, claims.ObjectiveId);
var current = await _mastery.Find(value => value.UserId == claims.UserId && value.ObjectiveId == claims.ObjectiveId).FirstAsync()
?? throw new InvalidOperationException("Dynamic objective mastery could not be initialized.");
var oldElo = current.Elo;
var familyElo = claims.FamilyElo > 0
? claims.FamilyElo
: 800 + Math.Clamp(generated.Blueprint.Difficulty, 1, 5) * 100;
var delta = DynamicDifficultyPolicy.LearnerDelta(oldElo, familyElo, isCorrect, current.Attempts);
var now = DateTime.UtcNow;
var update = Builders<DynamicObjectiveMastery>.Update
.SetOnInsert(value => value.UserId, claims.UserId)
.SetOnInsert(value => value.ObjectiveId, claims.ObjectiveId)
.Set(value => value.Elo, oldElo + delta)
.Inc(value => value.Elo, delta)
.Inc(value => value.Attempts, 1)
.Inc(value => value.Correct, isCorrect ? 1 : 0)
.Set(value => value.LastSeenUtc, now);
var updated = await _mastery.FindOneAndUpdateAsync(
value => value.UserId == claims.UserId && value.ObjectiveId == claims.ObjectiveId,
update,
new FindOneAndUpdateOptions<DynamicObjectiveMastery> { IsUpsert = true, ReturnDocument = ReturnDocument.After });
new FindOneAndUpdateOptions<DynamicObjectiveMastery> { ReturnDocument = ReturnDocument.After });
await _mastery.UpdateOneAsync(
value => value.UserId == claims.UserId && value.ObjectiveId == claims.ObjectiveId,
Builders<DynamicObjectiveMastery>.Update
.Max(value => value.Elo, DynamicDifficultyPolicy.MinimumElo)
.Min(value => value.Elo, DynamicDifficultyPolicy.MaximumElo));
if (!string.IsNullOrWhiteSpace(claims.CaseFamilyId))
await UpdateFamilyRatingAsync(claims, generated, isCorrect);
return new DynamicAttemptResponse
{
Inserted = true,
@@ -98,6 +128,70 @@ public sealed class DynamicQuestionStore
};
}
private async Task EnsureMasteryAsync(string userId, string objectiveId)
{
try
{
await _mastery.UpdateOneAsync(
value => value.UserId == userId && value.ObjectiveId == objectiveId,
Builders<DynamicObjectiveMastery>.Update
.SetOnInsert(value => value.UserId, userId)
.SetOnInsert(value => value.ObjectiveId, objectiveId)
.SetOnInsert(value => value.Elo, 1000)
.SetOnInsert(value => value.Attempts, 0)
.SetOnInsert(value => value.Correct, 0),
new UpdateOptions { IsUpsert = true });
}
catch (MongoWriteException exception) when (exception.WriteError.Category == ServerErrorCategory.DuplicateKey) { }
}
private async Task UpdateFamilyRatingAsync(
DynamicGenerationClaims claims,
DynamicGeneratedQuestion generated,
bool isCorrect)
{
var family = generated.Blueprint.CaseFamilies.First(value => value.Id == claims.CaseFamilyId);
try
{
await _familyRatings.UpdateOneAsync(
value => value.BlueprintExternalId == claims.BlueprintExternalId
&& value.BlueprintVersion == claims.BlueprintVersion
&& value.CaseFamilyId == claims.CaseFamilyId,
Builders<DynamicFamilyRating>.Update
.SetOnInsert(value => value.BlueprintExternalId, claims.BlueprintExternalId)
.SetOnInsert(value => value.BlueprintVersion, claims.BlueprintVersion)
.SetOnInsert(value => value.CaseFamilyId, claims.CaseFamilyId)
.SetOnInsert(value => value.DifficultyTier, family.DifficultyTier)
.SetOnInsert(value => value.NominalElo, family.NominalElo)
.SetOnInsert(value => value.Elo, family.NominalElo)
.SetOnInsert(value => value.Attempts, 0)
.SetOnInsert(value => value.Correct, 0),
new UpdateOptions { IsUpsert = true });
}
catch (MongoWriteException exception) when (exception.WriteError.Category == ServerErrorCategory.DuplicateKey) { }
var current = await _familyRatings.Find(value =>
value.BlueprintExternalId == claims.BlueprintExternalId
&& value.BlueprintVersion == claims.BlueprintVersion
&& value.CaseFamilyId == claims.CaseFamilyId).FirstAsync();
var delta = DynamicDifficultyPolicy.FamilyDelta(
claims.EffectiveAbilityElo > 0 ? claims.EffectiveAbilityElo : 1000,
claims.FamilyElo > 0 ? claims.FamilyElo : current.Elo,
isCorrect,
current.Attempts);
await _familyRatings.UpdateOneAsync(
value => value.Id == current.Id,
Builders<DynamicFamilyRating>.Update
.Inc(value => value.Elo, delta)
.Inc(value => value.Attempts, 1)
.Inc(value => value.Correct, isCorrect ? 1 : 0)
.Set(value => value.UpdatedUtc, DateTime.UtcNow));
await _familyRatings.UpdateOneAsync(
value => value.Id == current.Id,
Builders<DynamicFamilyRating>.Update
.Max(value => value.Elo, DynamicDifficultyPolicy.MinimumElo)
.Min(value => value.Elo, DynamicDifficultyPolicy.MaximumElo));
}
public async Task<DynamicQuestionStatusResponse> StatusAsync(string userId)
{
var objectiveCount = (await PublishedAsync()).Select(value => value.ObjectiveId).Distinct().Count();