hz/step3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Adapt dynamic question difficulty to learner Elo

Browse Source
This commit is contained in:
hz
2026-07-23 16:18:44 -05:00
parent 5b5741de37
commit fae38d2ded
13 changed files with 519 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/QuestionsApi/Services/DynamicBlueprintCommand.cs
+1
View File
@@ -22,6 +22,7 @@ public static class DynamicBlueprintCommand
var blueprints = new List<DynamicQuestionBlueprint>();
foreach (var file in Directory.EnumerateFiles(source, "*.json", SearchOption.AllDirectories))
blueprints.AddRange(JsonSerializer.Deserialize<List<DynamicQuestionBlueprint>>(await File.ReadAllTextAsync(file), json) ?? []);
blueprints = DynamicBlueprintResolver.ResolveAll(blueprints);
var generator = new DynamicQuestionGenerator();
var generatedVariants = 0;
api/QuestionsApi/Services/DynamicBlueprintResolver.cs
+61
View File
@@ -0,0 +1,61 @@
using QuestionsApi.Models;
using System.Text.Json;
namespace QuestionsApi.Services;
public static class DynamicBlueprintResolver
{
private static readonly JsonSerializerOptions Json = new(JsonSerializerDefaults.Web);
public static List<DynamicQuestionBlueprint> ResolveAll(IEnumerable<DynamicQuestionBlueprint> source)
{
var all = source.ToList();
var resolved = new List<DynamicQuestionBlueprint>();
foreach (var blueprint in all.OrderBy(value => value.Version))
{
if (blueprint.BaseBlueprintVersion <= 0)
{
resolved.Add(blueprint);
continue;
}
var baseBlueprint = resolved.FirstOrDefault(value =>
value.ExternalId == blueprint.ExternalId
&& value.Version == blueprint.BaseBlueprintVersion)
?? throw new InvalidDataException(
$"{blueprint.ExternalId} v{blueprint.Version} base version {blueprint.BaseBlueprintVersion} not found");
var materialized = Clone(baseBlueprint);
materialized.Id = null;
materialized.Version = blueprint.Version;
materialized.BaseBlueprintVersion = blueprint.BaseBlueprintVersion;
materialized.QualityStandardVersion = blueprint.QualityStandardVersion;
materialized.IsPublished = blueprint.IsPublished;
materialized.CaseFamilies = blueprint.CaseFamilies.Select(overlay =>
{
var baseFamily = baseBlueprint.CaseFamilies.FirstOrDefault(value => value.Id == overlay.BaseFamilyId)
?? throw new InvalidDataException(
$"{blueprint.ExternalId} v{blueprint.Version} base family {overlay.BaseFamilyId} not found");
var family = Clone(baseFamily);
family.Id = overlay.Id;
family.BaseFamilyId = overlay.BaseFamilyId;
family.DifficultyTier = overlay.DifficultyTier;
family.NominalElo = overlay.NominalElo;
if (!string.IsNullOrWhiteSpace(family.LeadIn)
&& family.StemTemplate.EndsWith(family.LeadIn, StringComparison.Ordinal))
family.StemTemplate = family.StemTemplate[..^family.LeadIn.Length].TrimEnd();
if (!string.IsNullOrWhiteSpace(overlay.StemTemplate))
family.StemTemplate = $"{family.StemTemplate} {overlay.StemTemplate}".Trim();
if (!string.IsNullOrWhiteSpace(overlay.LeadIn))
family.LeadIn = overlay.LeadIn;
if (overlay.Distractors.Count > 0)
family.Distractors = overlay.Distractors;
return family;
}).ToList();
resolved.Add(materialized);
}
return resolved;
}
private static T Clone<T>(T value) =>
JsonSerializer.Deserialize<T>(JsonSerializer.Serialize(value, Json), Json)
?? throw new InvalidOperationException("Blueprint clone failed.");
}
api/QuestionsApi/Services/DynamicDifficultyPolicy.cs
+69
View File
@@ -0,0 +1,69 @@
using QuestionsApi.Models;
namespace QuestionsApi.Services;
public static class DynamicDifficultyPolicy
{
public const int MinimumElo = 600;
public const int MaximumElo = 1800;
public const int TargetOffset = 70;
public static readonly IReadOnlyDictionary<string, int> TierElos =
new Dictionary<string, int>(StringComparer.Ordinal)
{
["foundation"] = 850,
["standard"] = 1050,
["advanced"] = 1250,
["expert"] = 1450
};
public static int EffectiveAbility(
int overallElo,
int categoryElo,
DynamicObjectiveMastery? objective)
{
var bankAbility = (int)Math.Round((overallElo + categoryElo) / 2.0);
if (objective is null || objective.Attempts <= 0)
return Clamp(bankAbility);
var objectiveWeight = Math.Min(0.8, objective.Attempts * 0.08);
return Clamp((int)Math.Round(bankAbility * (1.0 - objectiveWeight) + objective.Elo * objectiveWeight));
}
public static int TargetFamilyElo(int effectiveAbility) => Clamp(effectiveAbility - TargetOffset);
public static double ExpectedScore(int playerElo, int familyElo) =>
1.0 / (1.0 + Math.Pow(10, (familyElo - playerElo) / 400.0));
public static int LearnerDelta(int playerElo, int familyElo, bool correct, int attempts) =>
(int)Math.Round((attempts < 5 ? 40 : 24)
* ((correct ? 1.0 : 0.0) - ExpectedScore(playerElo, familyElo)));
public static int FamilyDelta(int playerElo, int familyElo, bool correct, int attempts) =>
(int)Math.Round((attempts < 20 ? 32 : 24)
* (ExpectedScore(playerElo, familyElo) - (correct ? 1.0 : 0.0)));
public static DynamicCaseFamily SelectFamily(
IReadOnlyList<DynamicCaseFamily> families,
IReadOnlyDictionary<string, DynamicFamilyRating> ratings,
int targetElo,
ISet<string> recentFamilyIds,
int tieBreaker)
{
if (families.Count == 0) throw new InvalidOperationException("No case families are available.");
var eligible = families.Where(value => !recentFamilyIds.Contains(value.Id)).ToList();
if (eligible.Count == 0) eligible = families.ToList();
var ranked = eligible
.Select(value => new
{
Family = value,
Elo = ratings.TryGetValue(value.Id, out var rating) ? rating.Elo : value.NominalElo
})
.OrderBy(value => Math.Abs(value.Elo - targetElo))
.ThenBy(value => value.Family.Id, StringComparer.Ordinal)
.ToList();
var bestDistance = Math.Abs(ranked[0].Elo - targetElo);
var tied = ranked.Where(value => Math.Abs(value.Elo - targetElo) == bestDistance).ToList();
return tied[Math.Abs(tieBreaker) % tied.Count].Family;
}
public static int Clamp(int elo) => Math.Clamp(elo, MinimumElo, MaximumElo);
}
api/QuestionsApi/Services/DynamicQuestionGenerator.cs
+34 -6
View File
@@ -11,9 +11,14 @@ public sealed class DynamicQuestionGenerator
public const int CurrentVersion = 2;
private static readonly Regex Placeholder = new(@"\{\{([a-zA-Z0-9_]+)\}\}", RegexOptions.Compiled);
public DynamicGeneratedQuestion Generate(DynamicQuestionBlueprint blueprint, string seed, string nonce, int generatorVersion = CurrentVersion)
public DynamicGeneratedQuestion Generate(
DynamicQuestionBlueprint blueprint,
string seed,
string nonce,
int generatorVersion = CurrentVersion,
string caseFamilyId = "")
=> blueprint.QualityStandardVersion >= 2
? GenerateV2(blueprint, seed, nonce)
? GenerateV2(blueprint, seed, nonce, caseFamilyId)
: GenerateLegacy(blueprint, seed, nonce, generatorVersion);
private DynamicGeneratedQuestion GenerateLegacy(DynamicQuestionBlueprint blueprint, string seed, string nonce, int generatorVersion)
@@ -117,13 +122,16 @@ public sealed class DynamicQuestionGenerator
return new DynamicGeneratedQuestion { Question = question, Blueprint = blueprint };
}
private DynamicGeneratedQuestion GenerateV2(DynamicQuestionBlueprint blueprint, string seed, string nonce)
private DynamicGeneratedQuestion GenerateV2(DynamicQuestionBlueprint blueprint, string seed, string nonce, string caseFamilyId)
{
if (blueprint.CaseFamilies.Count == 0)
throw new InvalidOperationException("quality-standard-2 blueprints require case families");
var draw = new DeterministicDraw($"{blueprint.ExternalId}|{blueprint.Version}|{CurrentVersion}|{seed}");
var family = blueprint.CaseFamilies.OrderBy(value => value.Id, StringComparer.Ordinal)
.ElementAt(draw.Next(blueprint.CaseFamilies.Count));
var orderedFamilies = blueprint.CaseFamilies.OrderBy(value => value.Id, StringComparer.Ordinal).ToList();
var family = string.IsNullOrWhiteSpace(caseFamilyId)
? orderedFamilies[draw.Next(orderedFamilies.Count)]
: orderedFamilies.FirstOrDefault(value => value.Id == caseFamilyId)
?? throw new InvalidOperationException($"case family {caseFamilyId} is unavailable");
var values = new Dictionary<string, string>(StringComparer.Ordinal);
var numbers = new Dictionary<string, double>(StringComparer.Ordinal);
if (family.ValueBundles.Count > 0)
@@ -178,7 +186,14 @@ public sealed class DynamicQuestionGenerator
Discipline = blueprint.Discipline,
System = blueprint.System,
PerformanceCategory = blueprint.PerformanceCategory,
Difficulty = blueprint.Difficulty,
Difficulty = family.DifficultyTier switch
{
"foundation" => 1,
"standard" => 2,
"advanced" => 4,
"expert" => 5,
_ => blueprint.Difficulty
},
Labs = family.Labs.Select(lab => new QuestionLab
{
Name = lab.Name,
@@ -327,6 +342,19 @@ public sealed class DynamicQuestionGenerator
if (blueprint.References.Count == 0) return "at least one reference required";
if (blueprint.CaseFamilies.Select(value => value.Id).Distinct(StringComparer.Ordinal).Count() != blueprint.CaseFamilies.Count)
return "case family ids must be unique";
if (blueprint.QualityStandardVersion >= 3)
{
foreach (var tier in DynamicDifficultyPolicy.TierElos)
{
var tierFamilies = blueprint.CaseFamilies
.Where(value => value.DifficultyTier == tier.Key).ToList();
if (tierFamilies.Count < 2) return $"at least two {tier.Key} case families required";
if (tierFamilies.Any(value => Math.Abs(value.NominalElo - tier.Value) > 100))
return $"{tier.Key} nominal Elo must be within 100 points of {tier.Value}";
}
if (blueprint.CaseFamilies.Any(value => !DynamicDifficultyPolicy.TierElos.ContainsKey(value.DifficultyTier)))
return "difficultyTier must be foundation, standard, advanced, or expert";
}
foreach (var family in blueprint.CaseFamilies)
{
if (string.IsNullOrWhiteSpace(family.Id)) return "case family id required";
api/QuestionsApi/Services/DynamicQuestionService.cs
+40 -15
View File
@@ -8,15 +8,21 @@ public sealed class DynamicQuestionService
private readonly DynamicQuestionStore _store;
private readonly DynamicQuestionGenerator _generator;
private readonly DynamicQuestionTokenService _tokens;
private readonly RatingStore _ratings;
private readonly ServiceMetrics _metrics;
public DynamicQuestionService(
DynamicQuestionStore store,
DynamicQuestionGenerator generator,
DynamicQuestionTokenService tokens)
DynamicQuestionTokenService tokens,
RatingStore ratings,
ServiceMetrics metrics)
{
_store = store;
_generator = generator;
_tokens = tokens;
_ratings = ratings;
_metrics = metrics;
}
public Task<DynamicQuestionStatusResponse> StatusAsync(string userId) => _store.StatusAsync(userId);
@@ -53,6 +59,30 @@ public sealed class DynamicQuestionService
var blueprint = selectedObjective
.OrderByDescending(value => value.Version)
.First();
var ratingProfile = await _ratings.GetProfileAsync(userId);
mastery.TryGetValue(blueprint.ObjectiveId, out var objectiveMastery);
var category = RatingCategory.Normalize(
blueprint.PerformanceCategory, blueprint.System, blueprint.Discipline);
var categoryElo = ratingProfile.CategoryElos.GetValueOrDefault(category, ratingProfile.OverallElo);
var effectiveAbility = DynamicDifficultyPolicy.EffectiveAbility(
ratingProfile.OverallElo, categoryElo, objectiveMastery);
var targetElo = DynamicDifficultyPolicy.TargetFamilyElo(effectiveAbility);
var familyRatings = await _store.FamilyRatingsAsync(blueprint);
var blueprintFamilyIds = blueprint.CaseFamilies.Select(value => value.Id).ToHashSet(StringComparer.Ordinal);
var recentForObjective = run.RecentCaseFamilyIds
.Where(blueprintFamilyIds.Contains)
.TakeLast(Math.Max(1, blueprint.CaseFamilies.Count / 4))
.ToHashSet(StringComparer.Ordinal);
var selectedFamily = DynamicDifficultyPolicy.SelectFamily(
blueprint.CaseFamilies,
familyRatings,
targetElo,
recentForObjective,
RandomNumberGenerator.GetInt32(int.MaxValue));
var selectedFamilyElo = familyRatings.TryGetValue(selectedFamily.Id, out var selectedRating)
? selectedRating.Elo
: selectedFamily.NominalElo;
_metrics.DynamicSelected(selectedFamily.DifficultyTier, selectedFamilyElo - targetElo);
var claims = new DynamicGenerationClaims
{
@@ -63,20 +93,14 @@ public sealed class DynamicQuestionService
Seed = Convert.ToHexString(RandomNumberGenerator.GetBytes(16)).ToLowerInvariant(),
Nonce = Guid.NewGuid().ToString("N"),
GeneratorVersion = DynamicQuestionGenerator.CurrentVersion,
CaseFamilyId = selectedFamily.Id,
DifficultyTier = selectedFamily.DifficultyTier,
EffectiveAbilityElo = effectiveAbility,
FamilyElo = selectedFamilyElo,
ExpiresUtc = DateTime.UtcNow.AddDays(7)
};
var generated = _generator.Generate(blueprint, claims.Seed, claims.Nonce, claims.GeneratorVersion);
var familyIds = blueprint.CaseFamilies.Select(value => value.Id).ToHashSet(StringComparer.Ordinal);
var recentForObjective = run.RecentCaseFamilyIds
.Where(familyIds.Contains)
.TakeLast(Math.Max(1, blueprint.CaseFamilies.Count - 1))
.ToHashSet(StringComparer.Ordinal);
for (var retry = 0; retry < 20 && recentForObjective.Contains(generated.CaseFamilyId); retry++)
{
claims.Seed = Convert.ToHexString(RandomNumberGenerator.GetBytes(16)).ToLowerInvariant();
generated = _generator.Generate(blueprint, claims.Seed, claims.Nonce);
}
claims.CaseFamilyId = generated.CaseFamilyId;
var generated = _generator.Generate(
blueprint, claims.Seed, claims.Nonce, claims.GeneratorVersion, claims.CaseFamilyId);
run.SeenObjectiveIds.Add(blueprint.ObjectiveId);
run.PreviousObjectiveId = blueprint.ObjectiveId;
run.RecentCaseFamilyIds.Add(generated.CaseFamilyId);
@@ -84,7 +108,7 @@ public sealed class DynamicQuestionService
run.ExpiresUtc = DateTime.UtcNow.AddDays(7);
return new DynamicQuestionEnvelope
{
Question = QuestionResponse.FromModel(generated.Question),
Question = QuestionResponse.FromModel(generated.Question, questionElo: selectedFamilyElo),
ObjectiveId = blueprint.ObjectiveId,
ObjectiveTitle = blueprint.Title,
GenerationToken = _tokens.SignGeneration(claims),
@@ -109,7 +133,8 @@ public sealed class DynamicQuestionService
throw new InvalidOperationException("This generated question version is no longer supported.");
var blueprint = await _store.GetAsync(claims.BlueprintExternalId, claims.BlueprintVersion)
?? throw new KeyNotFoundException("The question blueprint is no longer available.");
var generated = _generator.Generate(blueprint, claims.Seed, claims.Nonce, claims.GeneratorVersion);
var generated = _generator.Generate(
blueprint, claims.Seed, claims.Nonce, claims.GeneratorVersion, claims.CaseFamilyId);
if (!string.IsNullOrWhiteSpace(claims.CaseFamilyId)
&& claims.CaseFamilyId != generated.CaseFamilyId)
throw new InvalidOperationException("Generated case family does not match its signed descriptor.");
api/QuestionsApi/Services/DynamicQuestionStore.cs
+105 -11
View File
@@ -9,12 +9,16 @@ public sealed class DynamicQuestionStore
private readonly IMongoCollection<DynamicQuestionBlueprint> _blueprints;
private readonly IMongoCollection<DynamicQuestionAttempt> _attempts;
private readonly IMongoCollection<DynamicObjectiveMastery> _mastery;
private readonly IMongoCollection<DynamicFamilyRating> _familyRatings;
private readonly ServiceMetrics? _metrics;
public DynamicQuestionStore(MongoDatabaseProvider mongo)
public DynamicQuestionStore(MongoDatabaseProvider mongo, ServiceMetrics? metrics = null)
{
_metrics = metrics;
_blueprints = mongo.Database.GetCollection<DynamicQuestionBlueprint>("DynamicQuestionBlueprints");
_attempts = mongo.Database.GetCollection<DynamicQuestionAttempt>("DynamicQuestionAttempts");
_mastery = mongo.Database.GetCollection<DynamicObjectiveMastery>("DynamicObjectiveMastery");
_familyRatings = mongo.Database.GetCollection<DynamicFamilyRating>("DynamicFamilyRatings");
_blueprints.Indexes.CreateOne(new CreateIndexModel<DynamicQuestionBlueprint>(
Builders<DynamicQuestionBlueprint>.IndexKeys.Ascending(value => value.ExternalId).Ascending(value => value.Version),
new CreateIndexOptions { Unique = true }));
@@ -24,6 +28,12 @@ public sealed class DynamicQuestionStore
_mastery.Indexes.CreateOne(new CreateIndexModel<DynamicObjectiveMastery>(
Builders<DynamicObjectiveMastery>.IndexKeys.Ascending(value => value.UserId).Ascending(value => value.ObjectiveId),
new CreateIndexOptions { Unique = true }));
_familyRatings.Indexes.CreateOne(new CreateIndexModel<DynamicFamilyRating>(
Builders<DynamicFamilyRating>.IndexKeys
.Ascending(value => value.BlueprintExternalId)
.Ascending(value => value.BlueprintVersion)
.Ascending(value => value.CaseFamilyId),
new CreateIndexOptions { Unique = true }));
}
public async Task<List<DynamicQuestionBlueprint>> PublishedAsync() =>
@@ -36,6 +46,14 @@ public sealed class DynamicQuestionStore
(await _mastery.Find(value => value.UserId == userId).ToListAsync())
.ToDictionary(value => value.ObjectiveId, StringComparer.Ordinal);
public async Task<Dictionary<string, DynamicFamilyRating>> FamilyRatingsAsync(DynamicQuestionBlueprint blueprint)
{
var ratings = await _familyRatings.Find(value =>
value.BlueprintExternalId == blueprint.ExternalId
&& value.BlueprintVersion == blueprint.Version).ToListAsync();
return ratings.ToDictionary(value => value.CaseFamilyId, StringComparer.Ordinal);
}
public async Task<DynamicAttemptResponse> RecordAsync(
DynamicGenerationClaims claims,
DynamicGeneratedQuestion generated,
@@ -49,6 +67,10 @@ public sealed class DynamicQuestionStore
BlueprintExternalId = claims.BlueprintExternalId,
BlueprintVersion = claims.BlueprintVersion,
ObjectiveId = claims.ObjectiveId,
CaseFamilyId = claims.CaseFamilyId,
DifficultyTier = claims.DifficultyTier,
EffectiveAbilityElo = claims.EffectiveAbilityElo,
FamilyElo = claims.FamilyElo,
ChoiceId = request.ChoiceId,
IsCorrect = isCorrect,
IsSkip = request.IsSkip,
@@ -68,25 +90,33 @@ public sealed class DynamicQuestionStore
ObjectiveElo = existingMastery?.Elo ?? 1000
};
}
_metrics?.DynamicAttempt(claims.DifficultyTier, isCorrect);
var current = await _mastery.Find(value => value.UserId == claims.UserId && value.ObjectiveId == claims.ObjectiveId).FirstOrDefaultAsync();
var oldElo = current?.Elo ?? 1000;
var objectiveDifficulty = 800 + Math.Clamp(generated.Blueprint.Difficulty, 1, 5) * 100;
var expected = 1.0 / (1.0 + Math.Pow(10, (objectiveDifficulty - oldElo) / 400.0));
var k = (current?.Attempts ?? 0) < 5 ? 40 : 24;
var delta = (int)Math.Round(k * ((isCorrect ? 1.0 : 0.0) - expected));
await EnsureMasteryAsync(claims.UserId, claims.ObjectiveId);
var current = await _mastery.Find(value => value.UserId == claims.UserId && value.ObjectiveId == claims.ObjectiveId).FirstAsync()
?? throw new InvalidOperationException("Dynamic objective mastery could not be initialized.");
var oldElo = current.Elo;
var familyElo = claims.FamilyElo > 0
? claims.FamilyElo
: 800 + Math.Clamp(generated.Blueprint.Difficulty, 1, 5) * 100;
var delta = DynamicDifficultyPolicy.LearnerDelta(oldElo, familyElo, isCorrect, current.Attempts);
var now = DateTime.UtcNow;
var update = Builders<DynamicObjectiveMastery>.Update
.SetOnInsert(value => value.UserId, claims.UserId)
.SetOnInsert(value => value.ObjectiveId, claims.ObjectiveId)
.Set(value => value.Elo, oldElo + delta)
.Inc(value => value.Elo, delta)
.Inc(value => value.Attempts, 1)
.Inc(value => value.Correct, isCorrect ? 1 : 0)
.Set(value => value.LastSeenUtc, now);
var updated = await _mastery.FindOneAndUpdateAsync(
value => value.UserId == claims.UserId && value.ObjectiveId == claims.ObjectiveId,
update,
new FindOneAndUpdateOptions<DynamicObjectiveMastery> { IsUpsert = true, ReturnDocument = ReturnDocument.After });
new FindOneAndUpdateOptions<DynamicObjectiveMastery> { ReturnDocument = ReturnDocument.After });
await _mastery.UpdateOneAsync(
value => value.UserId == claims.UserId && value.ObjectiveId == claims.ObjectiveId,
Builders<DynamicObjectiveMastery>.Update
.Max(value => value.Elo, DynamicDifficultyPolicy.MinimumElo)
.Min(value => value.Elo, DynamicDifficultyPolicy.MaximumElo));
if (!string.IsNullOrWhiteSpace(claims.CaseFamilyId))
await UpdateFamilyRatingAsync(claims, generated, isCorrect);
return new DynamicAttemptResponse
{
Inserted = true,
@@ -98,6 +128,70 @@ public sealed class DynamicQuestionStore
};
}
private async Task EnsureMasteryAsync(string userId, string objectiveId)
{
try
{
await _mastery.UpdateOneAsync(
value => value.UserId == userId && value.ObjectiveId == objectiveId,
Builders<DynamicObjectiveMastery>.Update
.SetOnInsert(value => value.UserId, userId)
.SetOnInsert(value => value.ObjectiveId, objectiveId)
.SetOnInsert(value => value.Elo, 1000)
.SetOnInsert(value => value.Attempts, 0)
.SetOnInsert(value => value.Correct, 0),
new UpdateOptions { IsUpsert = true });
}
catch (MongoWriteException exception) when (exception.WriteError.Category == ServerErrorCategory.DuplicateKey) { }
}
private async Task UpdateFamilyRatingAsync(
DynamicGenerationClaims claims,
DynamicGeneratedQuestion generated,
bool isCorrect)
{
var family = generated.Blueprint.CaseFamilies.First(value => value.Id == claims.CaseFamilyId);
try
{
await _familyRatings.UpdateOneAsync(
value => value.BlueprintExternalId == claims.BlueprintExternalId
&& value.BlueprintVersion == claims.BlueprintVersion
&& value.CaseFamilyId == claims.CaseFamilyId,
Builders<DynamicFamilyRating>.Update
.SetOnInsert(value => value.BlueprintExternalId, claims.BlueprintExternalId)
.SetOnInsert(value => value.BlueprintVersion, claims.BlueprintVersion)
.SetOnInsert(value => value.CaseFamilyId, claims.CaseFamilyId)
.SetOnInsert(value => value.DifficultyTier, family.DifficultyTier)
.SetOnInsert(value => value.NominalElo, family.NominalElo)
.SetOnInsert(value => value.Elo, family.NominalElo)
.SetOnInsert(value => value.Attempts, 0)
.SetOnInsert(value => value.Correct, 0),
new UpdateOptions { IsUpsert = true });
}
catch (MongoWriteException exception) when (exception.WriteError.Category == ServerErrorCategory.DuplicateKey) { }
var current = await _familyRatings.Find(value =>
value.BlueprintExternalId == claims.BlueprintExternalId
&& value.BlueprintVersion == claims.BlueprintVersion
&& value.CaseFamilyId == claims.CaseFamilyId).FirstAsync();
var delta = DynamicDifficultyPolicy.FamilyDelta(
claims.EffectiveAbilityElo > 0 ? claims.EffectiveAbilityElo : 1000,
claims.FamilyElo > 0 ? claims.FamilyElo : current.Elo,
isCorrect,
current.Attempts);
await _familyRatings.UpdateOneAsync(
value => value.Id == current.Id,
Builders<DynamicFamilyRating>.Update
.Inc(value => value.Elo, delta)
.Inc(value => value.Attempts, 1)
.Inc(value => value.Correct, isCorrect ? 1 : 0)
.Set(value => value.UpdatedUtc, DateTime.UtcNow));
await _familyRatings.UpdateOneAsync(
value => value.Id == current.Id,
Builders<DynamicFamilyRating>.Update
.Max(value => value.Elo, DynamicDifficultyPolicy.MinimumElo)
.Min(value => value.Elo, DynamicDifficultyPolicy.MaximumElo));
}
public async Task<DynamicQuestionStatusResponse> StatusAsync(string userId)
{
var objectiveCount = (await PublishedAsync()).Select(value => value.ObjectiveId).Distinct().Count();
api/QuestionsApi/Services/DynamicReviewExportCommand.cs
+1
View File
@@ -26,6 +26,7 @@ public static class DynamicReviewExportCommand
foreach (var file in Directory.EnumerateFiles(source, "*.json", SearchOption.AllDirectories))
blueprints.AddRange(JsonSerializer.Deserialize<List<DynamicQuestionBlueprint>>(
await File.ReadAllTextAsync(file), jsonOptions) ?? []);
blueprints = DynamicBlueprintResolver.ResolveAll(blueprints);
blueprints = blueprints.Where(value => value.IsPublished && value.QualityStandardVersion >= 2).ToList();
var configuration = new ConfigurationBuilder()
api/QuestionsApi/Services/QuestionReportStore.cs
+4 -1
View File
@@ -100,7 +100,10 @@ public class QuestionReportStore
BlueprintVersion = claims.BlueprintVersion,
GeneratorVersion = claims.GeneratorVersion,
Seed = claims.Seed,
CaseFamilyId = claims.CaseFamilyId
CaseFamilyId = claims.CaseFamilyId,
DifficultyTier = claims.DifficultyTier,
FamilyElo = claims.FamilyElo,
EffectiveAbilityElo = claims.EffectiveAbilityElo
},
CreatedUtc = now,
UpdatedUtc = now
api/QuestionsApi/Services/ServiceMetrics.cs
+29
View File
@@ -13,6 +13,10 @@ public sealed class ServiceMetrics
private long _httpRequests;
private long _httpDurationMicroseconds;
private long _answerStatisticsBackfillComplete;
private readonly System.Collections.Concurrent.ConcurrentDictionary<string, long> _dynamicSelections = new();
private readonly System.Collections.Concurrent.ConcurrentDictionary<string, long> _dynamicAttempts = new();
private readonly System.Collections.Concurrent.ConcurrentDictionary<string, long> _dynamicCorrect = new();
private long _dynamicTargetGapTotal;
public void AttemptCreated() => Interlocked.Increment(ref _attemptsCreated);
public void AttemptDuplicate() => Interlocked.Increment(ref _attemptDuplicates);
@@ -29,6 +33,16 @@ public sealed class ServiceMetrics
Interlocked.Increment(ref _httpRequests);
Interlocked.Add(ref _httpDurationMicroseconds, (long)(duration.TotalMilliseconds * 1000));
}
public void DynamicSelected(string tier, int targetGap)
{
_dynamicSelections.AddOrUpdate(tier, 1, (_, value) => value + 1);
Interlocked.Add(ref _dynamicTargetGapTotal, Math.Abs(targetGap));
}
public void DynamicAttempt(string tier, bool correct)
{
_dynamicAttempts.AddOrUpdate(tier, 1, (_, value) => value + 1);
if (correct) _dynamicCorrect.AddOrUpdate(tier, 1, (_, value) => value + 1);
}
public string RenderPrometheus()
{
@@ -39,6 +53,16 @@ public sealed class ServiceMetrics
AppendCounter(output, "step3_rating_failures_total", Interlocked.Read(ref _ratingFailures));
AppendCounter(output, "step3_answer_reconciliations_total", Interlocked.Read(ref _reconciliations));
AppendCounter(output, "step3_http_requests_total", Interlocked.Read(ref _httpRequests));
output.AppendLine("# TYPE step3_dynamic_selections_total counter");
output.AppendLine("# TYPE step3_dynamic_attempts_total counter");
output.AppendLine("# TYPE step3_dynamic_correct_total counter");
foreach (var tier in _dynamicSelections.Keys.Order())
{
AppendLabeledSample(output, "step3_dynamic_selections_total", tier, _dynamicSelections.GetValueOrDefault(tier));
AppendLabeledSample(output, "step3_dynamic_attempts_total", tier, _dynamicAttempts.GetValueOrDefault(tier));
AppendLabeledSample(output, "step3_dynamic_correct_total", tier, _dynamicCorrect.GetValueOrDefault(tier));
}
AppendCounter(output, "step3_dynamic_target_gap_points_total", Interlocked.Read(ref _dynamicTargetGapTotal));
output.AppendLine("# TYPE step3_http_request_duration_seconds summary");
output.Append("step3_http_request_duration_seconds_count ")
.Append(Interlocked.Read(ref _httpRequests))
@@ -62,4 +86,9 @@ public sealed class ServiceMetrics
output.Append("# TYPE ").Append(name).AppendLine(" counter");
output.Append(name).Append(' ').Append(value).AppendLine();
}
private static void AppendLabeledSample(StringBuilder output, string name, string tier, long value)
{
output.Append(name).Append("{tier=\"").Append(tier.Replace("\"", "")).Append("\"} ").Append(value).AppendLine();
}
}