diff --git a/api/QuestionsApi.Tests/DynamicQuestionTests.cs b/api/QuestionsApi.Tests/DynamicQuestionTests.cs index 738d48d..107c66d 100644 --- a/api/QuestionsApi.Tests/DynamicQuestionTests.cs +++ b/api/QuestionsApi.Tests/DynamicQuestionTests.cs @@ -80,6 +80,71 @@ public sealed class DynamicQuestionTests Assert.Equal(first.Choices.Select(value => value.Id), regenerated.Choices.Select(value => value.Id)); } + [Fact] + public void DifficultyPolicy_BlendsThenSpecializes_AndTargetsSixtyPercent() + { + Assert.Equal(1200, DynamicDifficultyPolicy.EffectiveAbility(1100, 1300, null)); + var early = new DynamicObjectiveMastery { Elo = 1500, Attempts = 1 }; + var established = new DynamicObjectiveMastery { Elo = 1500, Attempts = 10 }; + Assert.Equal(1224, DynamicDifficultyPolicy.EffectiveAbility(1100, 1300, early)); + Assert.Equal(1440, DynamicDifficultyPolicy.EffectiveAbility(1100, 1300, established)); + Assert.Equal(1370, DynamicDifficultyPolicy.TargetFamilyElo(1440)); + } + + [Fact] + public void DifficultyPolicy_SelectsClosestNonrecentFamily() + { + var families = new List + { + new() { Id = "foundation", DifficultyTier = "foundation", NominalElo = 850 }, + new() { Id = "standard", DifficultyTier = "standard", NominalElo = 1050 }, + new() { Id = "advanced-a", DifficultyTier = "advanced", NominalElo = 1250 }, + new() { Id = "advanced-b", DifficultyTier = "advanced", NominalElo = 1250 }, + new() { Id = "expert", DifficultyTier = "expert", NominalElo = 1450 } + }; + var selected = DynamicDifficultyPolicy.SelectFamily( + families, new Dictionary(), 1260, + new HashSet { "advanced-a" }, 0); + Assert.Equal("advanced-b", selected.Id); + } + + [Fact] + public void DifficultyPolicy_RatingsMoveInOppositeDirections() + { + Assert.True(DynamicDifficultyPolicy.LearnerDelta(1000, 1050, true, 0) > 0); + Assert.True(DynamicDifficultyPolicy.FamilyDelta(1000, 1050, true, 0) < 0); + Assert.True(DynamicDifficultyPolicy.LearnerDelta(1000, 1050, false, 0) < 0); + Assert.True(DynamicDifficultyPolicy.FamilyDelta(1000, 1050, false, 0) > 0); + Assert.Equal(600, DynamicDifficultyPolicy.Clamp(-100)); + Assert.Equal(1800, DynamicDifficultyPolicy.Clamp(5000)); + } + + [Fact] + public void AdaptiveContent_ResolvesToFourCompleteTiers() + { + var options = new System.Text.Json.JsonSerializerOptions(System.Text.Json.JsonSerializerDefaults.Web) + { PropertyNameCaseInsensitive = true }; + var root = Path.GetFullPath(Path.Combine( + AppContext.BaseDirectory, "..", "..", "..", "..", "..", "content", "dynamic_questions")); + var source = Directory.EnumerateFiles(root, "*.json") + .SelectMany(file => System.Text.Json.JsonSerializer.Deserialize>( + File.ReadAllText(file), options) ?? []) + .ToList(); + var adaptive = DynamicBlueprintResolver.ResolveAll(source) + .Where(value => value.QualityStandardVersion >= 3).ToList(); + + Assert.Equal(5, adaptive.Count); + Assert.All(adaptive, blueprint => + { + Assert.Equal(8, blueprint.CaseFamilies.Count); + Assert.All(DynamicDifficultyPolicy.TierElos.Keys, tier => + Assert.Equal(2, blueprint.CaseFamilies.Count(value => value.DifficultyTier == tier))); + Assert.All(blueprint.CaseFamilies, family => + Assert.False(family.StemTemplate.EndsWith(family.LeadIn, StringComparison.Ordinal))); + Assert.Null(DynamicQuestionGenerator.Validate(blueprint)); + }); + } + private static DynamicQuestionBlueprint Blueprint() => new() { ExternalId = "test", diff --git a/api/QuestionsApi/Models/DynamicQuestionModels.cs b/api/QuestionsApi/Models/DynamicQuestionModels.cs index 62fdb93..899afdb 100644 --- a/api/QuestionsApi/Models/DynamicQuestionModels.cs +++ b/api/QuestionsApi/Models/DynamicQuestionModels.cs @@ -19,6 +19,7 @@ public sealed class DynamicQuestionBlueprint public int Difficulty { get; set; } = 3; public int QualityStandardVersion { get; set; } = 1; public string CorrectConceptId { get; set; } = ""; + public int BaseBlueprintVersion { get; set; } public bool IsPublished { get; set; } = true; public string StemTemplate { get; set; } = ""; public string LeadIn { get; set; } = ""; @@ -41,6 +42,9 @@ public sealed class DynamicQuestionBlueprint public sealed class DynamicCaseFamily { public string Id { get; set; } = ""; + public string BaseFamilyId { get; set; } = ""; + public string DifficultyTier { get; set; } = "standard"; + public int NominalElo { get; set; } = 1050; public string StemTemplate { get; set; } = ""; public string LeadIn { get; set; } = ""; public Dictionary Variables { get; set; } = []; @@ -123,6 +127,9 @@ public sealed class DynamicGenerationClaims public string Nonce { get; set; } = ""; public int GeneratorVersion { get; set; } = 1; public string CaseFamilyId { get; set; } = ""; + public string DifficultyTier { get; set; } = ""; + public int EffectiveAbilityElo { get; set; } + public int FamilyElo { get; set; } public DateTime ExpiresUtc { get; set; } } @@ -204,6 +211,10 @@ public sealed class DynamicQuestionAttempt public string BlueprintExternalId { get; set; } = ""; public int BlueprintVersion { get; set; } public string ObjectiveId { get; set; } = ""; + public string CaseFamilyId { get; set; } = ""; + public string DifficultyTier { get; set; } = ""; + public int EffectiveAbilityElo { get; set; } + public int FamilyElo { get; set; } public string ChoiceId { get; set; } = ""; public bool IsCorrect { get; set; } public bool IsSkip { get; set; } @@ -224,6 +235,22 @@ public sealed class DynamicObjectiveMastery public DateTime LastSeenUtc { get; set; } } +public sealed class DynamicFamilyRating +{ + [BsonId] + [BsonRepresentation(BsonType.ObjectId)] + public string? Id { get; set; } + public string BlueprintExternalId { get; set; } = ""; + public int BlueprintVersion { get; set; } + public string CaseFamilyId { get; set; } = ""; + public string DifficultyTier { get; set; } = ""; + public int NominalElo { get; set; } + public int Elo { get; set; } + public int Attempts { get; set; } + public int Correct { get; set; } + public DateTime UpdatedUtc { get; set; } = DateTime.UtcNow; +} + public sealed class DynamicGenerationDescriptor { public string BlueprintExternalId { get; set; } = ""; @@ -231,4 +258,7 @@ public sealed class DynamicGenerationDescriptor public int GeneratorVersion { get; set; } public string Seed { get; set; } = ""; public string CaseFamilyId { get; set; } = ""; + public string DifficultyTier { get; set; } = ""; + public int FamilyElo { get; set; } + public int EffectiveAbilityElo { get; set; } } diff --git a/api/QuestionsApi/Services/DynamicBlueprintCommand.cs b/api/QuestionsApi/Services/DynamicBlueprintCommand.cs index 1729ec1..d68fdac 100644 --- a/api/QuestionsApi/Services/DynamicBlueprintCommand.cs +++ b/api/QuestionsApi/Services/DynamicBlueprintCommand.cs @@ -22,6 +22,7 @@ public static class DynamicBlueprintCommand var blueprints = new List(); foreach (var file in Directory.EnumerateFiles(source, "*.json", SearchOption.AllDirectories)) blueprints.AddRange(JsonSerializer.Deserialize>(await File.ReadAllTextAsync(file), json) ?? []); + blueprints = DynamicBlueprintResolver.ResolveAll(blueprints); var generator = new DynamicQuestionGenerator(); var generatedVariants = 0; diff --git a/api/QuestionsApi/Services/DynamicBlueprintResolver.cs b/api/QuestionsApi/Services/DynamicBlueprintResolver.cs new file mode 100644 index 0000000..e9a1410 --- /dev/null +++ b/api/QuestionsApi/Services/DynamicBlueprintResolver.cs @@ -0,0 +1,61 @@ +using QuestionsApi.Models; +using System.Text.Json; + +namespace QuestionsApi.Services; + +public static class DynamicBlueprintResolver +{ + private static readonly JsonSerializerOptions Json = new(JsonSerializerDefaults.Web); + + public static List ResolveAll(IEnumerable source) + { + var all = source.ToList(); + var resolved = new List(); + foreach (var blueprint in all.OrderBy(value => value.Version)) + { + if (blueprint.BaseBlueprintVersion <= 0) + { + resolved.Add(blueprint); + continue; + } + var baseBlueprint = resolved.FirstOrDefault(value => + value.ExternalId == blueprint.ExternalId + && value.Version == blueprint.BaseBlueprintVersion) + ?? throw new InvalidDataException( + $"{blueprint.ExternalId} v{blueprint.Version} base version {blueprint.BaseBlueprintVersion} not found"); + var materialized = Clone(baseBlueprint); + materialized.Id = null; + materialized.Version = blueprint.Version; + materialized.BaseBlueprintVersion = blueprint.BaseBlueprintVersion; + materialized.QualityStandardVersion = blueprint.QualityStandardVersion; + materialized.IsPublished = blueprint.IsPublished; + materialized.CaseFamilies = blueprint.CaseFamilies.Select(overlay => + { + var baseFamily = baseBlueprint.CaseFamilies.FirstOrDefault(value => value.Id == overlay.BaseFamilyId) + ?? throw new InvalidDataException( + $"{blueprint.ExternalId} v{blueprint.Version} base family {overlay.BaseFamilyId} not found"); + var family = Clone(baseFamily); + family.Id = overlay.Id; + family.BaseFamilyId = overlay.BaseFamilyId; + family.DifficultyTier = overlay.DifficultyTier; + family.NominalElo = overlay.NominalElo; + if (!string.IsNullOrWhiteSpace(family.LeadIn) + && family.StemTemplate.EndsWith(family.LeadIn, StringComparison.Ordinal)) + family.StemTemplate = family.StemTemplate[..^family.LeadIn.Length].TrimEnd(); + if (!string.IsNullOrWhiteSpace(overlay.StemTemplate)) + family.StemTemplate = $"{family.StemTemplate} {overlay.StemTemplate}".Trim(); + if (!string.IsNullOrWhiteSpace(overlay.LeadIn)) + family.LeadIn = overlay.LeadIn; + if (overlay.Distractors.Count > 0) + family.Distractors = overlay.Distractors; + return family; + }).ToList(); + resolved.Add(materialized); + } + return resolved; + } + + private static T Clone(T value) => + JsonSerializer.Deserialize(JsonSerializer.Serialize(value, Json), Json) + ?? throw new InvalidOperationException("Blueprint clone failed."); +} diff --git a/api/QuestionsApi/Services/DynamicDifficultyPolicy.cs b/api/QuestionsApi/Services/DynamicDifficultyPolicy.cs new file mode 100644 index 0000000..0cd2b91 --- /dev/null +++ b/api/QuestionsApi/Services/DynamicDifficultyPolicy.cs @@ -0,0 +1,69 @@ +using QuestionsApi.Models; + +namespace QuestionsApi.Services; + +public static class DynamicDifficultyPolicy +{ + public const int MinimumElo = 600; + public const int MaximumElo = 1800; + public const int TargetOffset = 70; + public static readonly IReadOnlyDictionary TierElos = + new Dictionary(StringComparer.Ordinal) + { + ["foundation"] = 850, + ["standard"] = 1050, + ["advanced"] = 1250, + ["expert"] = 1450 + }; + + public static int EffectiveAbility( + int overallElo, + int categoryElo, + DynamicObjectiveMastery? objective) + { + var bankAbility = (int)Math.Round((overallElo + categoryElo) / 2.0); + if (objective is null || objective.Attempts <= 0) + return Clamp(bankAbility); + var objectiveWeight = Math.Min(0.8, objective.Attempts * 0.08); + return Clamp((int)Math.Round(bankAbility * (1.0 - objectiveWeight) + objective.Elo * objectiveWeight)); + } + + public static int TargetFamilyElo(int effectiveAbility) => Clamp(effectiveAbility - TargetOffset); + + public static double ExpectedScore(int playerElo, int familyElo) => + 1.0 / (1.0 + Math.Pow(10, (familyElo - playerElo) / 400.0)); + + public static int LearnerDelta(int playerElo, int familyElo, bool correct, int attempts) => + (int)Math.Round((attempts < 5 ? 40 : 24) + * ((correct ? 1.0 : 0.0) - ExpectedScore(playerElo, familyElo))); + + public static int FamilyDelta(int playerElo, int familyElo, bool correct, int attempts) => + (int)Math.Round((attempts < 20 ? 32 : 24) + * (ExpectedScore(playerElo, familyElo) - (correct ? 1.0 : 0.0))); + + public static DynamicCaseFamily SelectFamily( + IReadOnlyList families, + IReadOnlyDictionary ratings, + int targetElo, + ISet recentFamilyIds, + int tieBreaker) + { + if (families.Count == 0) throw new InvalidOperationException("No case families are available."); + var eligible = families.Where(value => !recentFamilyIds.Contains(value.Id)).ToList(); + if (eligible.Count == 0) eligible = families.ToList(); + var ranked = eligible + .Select(value => new + { + Family = value, + Elo = ratings.TryGetValue(value.Id, out var rating) ? rating.Elo : value.NominalElo + }) + .OrderBy(value => Math.Abs(value.Elo - targetElo)) + .ThenBy(value => value.Family.Id, StringComparer.Ordinal) + .ToList(); + var bestDistance = Math.Abs(ranked[0].Elo - targetElo); + var tied = ranked.Where(value => Math.Abs(value.Elo - targetElo) == bestDistance).ToList(); + return tied[Math.Abs(tieBreaker) % tied.Count].Family; + } + + public static int Clamp(int elo) => Math.Clamp(elo, MinimumElo, MaximumElo); +} diff --git a/api/QuestionsApi/Services/DynamicQuestionGenerator.cs b/api/QuestionsApi/Services/DynamicQuestionGenerator.cs index f90636a..26199c0 100644 --- a/api/QuestionsApi/Services/DynamicQuestionGenerator.cs +++ b/api/QuestionsApi/Services/DynamicQuestionGenerator.cs @@ -11,9 +11,14 @@ public sealed class DynamicQuestionGenerator public const int CurrentVersion = 2; private static readonly Regex Placeholder = new(@"\{\{([a-zA-Z0-9_]+)\}\}", RegexOptions.Compiled); - public DynamicGeneratedQuestion Generate(DynamicQuestionBlueprint blueprint, string seed, string nonce, int generatorVersion = CurrentVersion) + public DynamicGeneratedQuestion Generate( + DynamicQuestionBlueprint blueprint, + string seed, + string nonce, + int generatorVersion = CurrentVersion, + string caseFamilyId = "") => blueprint.QualityStandardVersion >= 2 - ? GenerateV2(blueprint, seed, nonce) + ? GenerateV2(blueprint, seed, nonce, caseFamilyId) : GenerateLegacy(blueprint, seed, nonce, generatorVersion); private DynamicGeneratedQuestion GenerateLegacy(DynamicQuestionBlueprint blueprint, string seed, string nonce, int generatorVersion) @@ -117,13 +122,16 @@ public sealed class DynamicQuestionGenerator return new DynamicGeneratedQuestion { Question = question, Blueprint = blueprint }; } - private DynamicGeneratedQuestion GenerateV2(DynamicQuestionBlueprint blueprint, string seed, string nonce) + private DynamicGeneratedQuestion GenerateV2(DynamicQuestionBlueprint blueprint, string seed, string nonce, string caseFamilyId) { if (blueprint.CaseFamilies.Count == 0) throw new InvalidOperationException("quality-standard-2 blueprints require case families"); var draw = new DeterministicDraw($"{blueprint.ExternalId}|{blueprint.Version}|{CurrentVersion}|{seed}"); - var family = blueprint.CaseFamilies.OrderBy(value => value.Id, StringComparer.Ordinal) - .ElementAt(draw.Next(blueprint.CaseFamilies.Count)); + var orderedFamilies = blueprint.CaseFamilies.OrderBy(value => value.Id, StringComparer.Ordinal).ToList(); + var family = string.IsNullOrWhiteSpace(caseFamilyId) + ? orderedFamilies[draw.Next(orderedFamilies.Count)] + : orderedFamilies.FirstOrDefault(value => value.Id == caseFamilyId) + ?? throw new InvalidOperationException($"case family {caseFamilyId} is unavailable"); var values = new Dictionary(StringComparer.Ordinal); var numbers = new Dictionary(StringComparer.Ordinal); if (family.ValueBundles.Count > 0) @@ -178,7 +186,14 @@ public sealed class DynamicQuestionGenerator Discipline = blueprint.Discipline, System = blueprint.System, PerformanceCategory = blueprint.PerformanceCategory, - Difficulty = blueprint.Difficulty, + Difficulty = family.DifficultyTier switch + { + "foundation" => 1, + "standard" => 2, + "advanced" => 4, + "expert" => 5, + _ => blueprint.Difficulty + }, Labs = family.Labs.Select(lab => new QuestionLab { Name = lab.Name, @@ -327,6 +342,19 @@ public sealed class DynamicQuestionGenerator if (blueprint.References.Count == 0) return "at least one reference required"; if (blueprint.CaseFamilies.Select(value => value.Id).Distinct(StringComparer.Ordinal).Count() != blueprint.CaseFamilies.Count) return "case family ids must be unique"; + if (blueprint.QualityStandardVersion >= 3) + { + foreach (var tier in DynamicDifficultyPolicy.TierElos) + { + var tierFamilies = blueprint.CaseFamilies + .Where(value => value.DifficultyTier == tier.Key).ToList(); + if (tierFamilies.Count < 2) return $"at least two {tier.Key} case families required"; + if (tierFamilies.Any(value => Math.Abs(value.NominalElo - tier.Value) > 100)) + return $"{tier.Key} nominal Elo must be within 100 points of {tier.Value}"; + } + if (blueprint.CaseFamilies.Any(value => !DynamicDifficultyPolicy.TierElos.ContainsKey(value.DifficultyTier))) + return "difficultyTier must be foundation, standard, advanced, or expert"; + } foreach (var family in blueprint.CaseFamilies) { if (string.IsNullOrWhiteSpace(family.Id)) return "case family id required"; diff --git a/api/QuestionsApi/Services/DynamicQuestionService.cs b/api/QuestionsApi/Services/DynamicQuestionService.cs index 782186d..9b69f88 100644 --- a/api/QuestionsApi/Services/DynamicQuestionService.cs +++ b/api/QuestionsApi/Services/DynamicQuestionService.cs @@ -8,15 +8,21 @@ public sealed class DynamicQuestionService private readonly DynamicQuestionStore _store; private readonly DynamicQuestionGenerator _generator; private readonly DynamicQuestionTokenService _tokens; + private readonly RatingStore _ratings; + private readonly ServiceMetrics _metrics; public DynamicQuestionService( DynamicQuestionStore store, DynamicQuestionGenerator generator, - DynamicQuestionTokenService tokens) + DynamicQuestionTokenService tokens, + RatingStore ratings, + ServiceMetrics metrics) { _store = store; _generator = generator; _tokens = tokens; + _ratings = ratings; + _metrics = metrics; } public Task StatusAsync(string userId) => _store.StatusAsync(userId); @@ -53,6 +59,30 @@ public sealed class DynamicQuestionService var blueprint = selectedObjective .OrderByDescending(value => value.Version) .First(); + var ratingProfile = await _ratings.GetProfileAsync(userId); + mastery.TryGetValue(blueprint.ObjectiveId, out var objectiveMastery); + var category = RatingCategory.Normalize( + blueprint.PerformanceCategory, blueprint.System, blueprint.Discipline); + var categoryElo = ratingProfile.CategoryElos.GetValueOrDefault(category, ratingProfile.OverallElo); + var effectiveAbility = DynamicDifficultyPolicy.EffectiveAbility( + ratingProfile.OverallElo, categoryElo, objectiveMastery); + var targetElo = DynamicDifficultyPolicy.TargetFamilyElo(effectiveAbility); + var familyRatings = await _store.FamilyRatingsAsync(blueprint); + var blueprintFamilyIds = blueprint.CaseFamilies.Select(value => value.Id).ToHashSet(StringComparer.Ordinal); + var recentForObjective = run.RecentCaseFamilyIds + .Where(blueprintFamilyIds.Contains) + .TakeLast(Math.Max(1, blueprint.CaseFamilies.Count / 4)) + .ToHashSet(StringComparer.Ordinal); + var selectedFamily = DynamicDifficultyPolicy.SelectFamily( + blueprint.CaseFamilies, + familyRatings, + targetElo, + recentForObjective, + RandomNumberGenerator.GetInt32(int.MaxValue)); + var selectedFamilyElo = familyRatings.TryGetValue(selectedFamily.Id, out var selectedRating) + ? selectedRating.Elo + : selectedFamily.NominalElo; + _metrics.DynamicSelected(selectedFamily.DifficultyTier, selectedFamilyElo - targetElo); var claims = new DynamicGenerationClaims { @@ -63,20 +93,14 @@ public sealed class DynamicQuestionService Seed = Convert.ToHexString(RandomNumberGenerator.GetBytes(16)).ToLowerInvariant(), Nonce = Guid.NewGuid().ToString("N"), GeneratorVersion = DynamicQuestionGenerator.CurrentVersion, + CaseFamilyId = selectedFamily.Id, + DifficultyTier = selectedFamily.DifficultyTier, + EffectiveAbilityElo = effectiveAbility, + FamilyElo = selectedFamilyElo, ExpiresUtc = DateTime.UtcNow.AddDays(7) }; - var generated = _generator.Generate(blueprint, claims.Seed, claims.Nonce, claims.GeneratorVersion); - var familyIds = blueprint.CaseFamilies.Select(value => value.Id).ToHashSet(StringComparer.Ordinal); - var recentForObjective = run.RecentCaseFamilyIds - .Where(familyIds.Contains) - .TakeLast(Math.Max(1, blueprint.CaseFamilies.Count - 1)) - .ToHashSet(StringComparer.Ordinal); - for (var retry = 0; retry < 20 && recentForObjective.Contains(generated.CaseFamilyId); retry++) - { - claims.Seed = Convert.ToHexString(RandomNumberGenerator.GetBytes(16)).ToLowerInvariant(); - generated = _generator.Generate(blueprint, claims.Seed, claims.Nonce); - } - claims.CaseFamilyId = generated.CaseFamilyId; + var generated = _generator.Generate( + blueprint, claims.Seed, claims.Nonce, claims.GeneratorVersion, claims.CaseFamilyId); run.SeenObjectiveIds.Add(blueprint.ObjectiveId); run.PreviousObjectiveId = blueprint.ObjectiveId; run.RecentCaseFamilyIds.Add(generated.CaseFamilyId); @@ -84,7 +108,7 @@ public sealed class DynamicQuestionService run.ExpiresUtc = DateTime.UtcNow.AddDays(7); return new DynamicQuestionEnvelope { - Question = QuestionResponse.FromModel(generated.Question), + Question = QuestionResponse.FromModel(generated.Question, questionElo: selectedFamilyElo), ObjectiveId = blueprint.ObjectiveId, ObjectiveTitle = blueprint.Title, GenerationToken = _tokens.SignGeneration(claims), @@ -109,7 +133,8 @@ public sealed class DynamicQuestionService throw new InvalidOperationException("This generated question version is no longer supported."); var blueprint = await _store.GetAsync(claims.BlueprintExternalId, claims.BlueprintVersion) ?? throw new KeyNotFoundException("The question blueprint is no longer available."); - var generated = _generator.Generate(blueprint, claims.Seed, claims.Nonce, claims.GeneratorVersion); + var generated = _generator.Generate( + blueprint, claims.Seed, claims.Nonce, claims.GeneratorVersion, claims.CaseFamilyId); if (!string.IsNullOrWhiteSpace(claims.CaseFamilyId) && claims.CaseFamilyId != generated.CaseFamilyId) throw new InvalidOperationException("Generated case family does not match its signed descriptor."); diff --git a/api/QuestionsApi/Services/DynamicQuestionStore.cs b/api/QuestionsApi/Services/DynamicQuestionStore.cs index 42499f6..50e7779 100644 --- a/api/QuestionsApi/Services/DynamicQuestionStore.cs +++ b/api/QuestionsApi/Services/DynamicQuestionStore.cs @@ -9,12 +9,16 @@ public sealed class DynamicQuestionStore private readonly IMongoCollection _blueprints; private readonly IMongoCollection _attempts; private readonly IMongoCollection _mastery; + private readonly IMongoCollection _familyRatings; + private readonly ServiceMetrics? _metrics; - public DynamicQuestionStore(MongoDatabaseProvider mongo) + public DynamicQuestionStore(MongoDatabaseProvider mongo, ServiceMetrics? metrics = null) { + _metrics = metrics; _blueprints = mongo.Database.GetCollection("DynamicQuestionBlueprints"); _attempts = mongo.Database.GetCollection("DynamicQuestionAttempts"); _mastery = mongo.Database.GetCollection("DynamicObjectiveMastery"); + _familyRatings = mongo.Database.GetCollection("DynamicFamilyRatings"); _blueprints.Indexes.CreateOne(new CreateIndexModel( Builders.IndexKeys.Ascending(value => value.ExternalId).Ascending(value => value.Version), new CreateIndexOptions { Unique = true })); @@ -24,6 +28,12 @@ public sealed class DynamicQuestionStore _mastery.Indexes.CreateOne(new CreateIndexModel( Builders.IndexKeys.Ascending(value => value.UserId).Ascending(value => value.ObjectiveId), new CreateIndexOptions { Unique = true })); + _familyRatings.Indexes.CreateOne(new CreateIndexModel( + Builders.IndexKeys + .Ascending(value => value.BlueprintExternalId) + .Ascending(value => value.BlueprintVersion) + .Ascending(value => value.CaseFamilyId), + new CreateIndexOptions { Unique = true })); } public async Task> PublishedAsync() => @@ -36,6 +46,14 @@ public sealed class DynamicQuestionStore (await _mastery.Find(value => value.UserId == userId).ToListAsync()) .ToDictionary(value => value.ObjectiveId, StringComparer.Ordinal); + public async Task> FamilyRatingsAsync(DynamicQuestionBlueprint blueprint) + { + var ratings = await _familyRatings.Find(value => + value.BlueprintExternalId == blueprint.ExternalId + && value.BlueprintVersion == blueprint.Version).ToListAsync(); + return ratings.ToDictionary(value => value.CaseFamilyId, StringComparer.Ordinal); + } + public async Task RecordAsync( DynamicGenerationClaims claims, DynamicGeneratedQuestion generated, @@ -49,6 +67,10 @@ public sealed class DynamicQuestionStore BlueprintExternalId = claims.BlueprintExternalId, BlueprintVersion = claims.BlueprintVersion, ObjectiveId = claims.ObjectiveId, + CaseFamilyId = claims.CaseFamilyId, + DifficultyTier = claims.DifficultyTier, + EffectiveAbilityElo = claims.EffectiveAbilityElo, + FamilyElo = claims.FamilyElo, ChoiceId = request.ChoiceId, IsCorrect = isCorrect, IsSkip = request.IsSkip, @@ -68,25 +90,33 @@ public sealed class DynamicQuestionStore ObjectiveElo = existingMastery?.Elo ?? 1000 }; } + _metrics?.DynamicAttempt(claims.DifficultyTier, isCorrect); - var current = await _mastery.Find(value => value.UserId == claims.UserId && value.ObjectiveId == claims.ObjectiveId).FirstOrDefaultAsync(); - var oldElo = current?.Elo ?? 1000; - var objectiveDifficulty = 800 + Math.Clamp(generated.Blueprint.Difficulty, 1, 5) * 100; - var expected = 1.0 / (1.0 + Math.Pow(10, (objectiveDifficulty - oldElo) / 400.0)); - var k = (current?.Attempts ?? 0) < 5 ? 40 : 24; - var delta = (int)Math.Round(k * ((isCorrect ? 1.0 : 0.0) - expected)); + await EnsureMasteryAsync(claims.UserId, claims.ObjectiveId); + var current = await _mastery.Find(value => value.UserId == claims.UserId && value.ObjectiveId == claims.ObjectiveId).FirstAsync() + ?? throw new InvalidOperationException("Dynamic objective mastery could not be initialized."); + var oldElo = current.Elo; + var familyElo = claims.FamilyElo > 0 + ? claims.FamilyElo + : 800 + Math.Clamp(generated.Blueprint.Difficulty, 1, 5) * 100; + var delta = DynamicDifficultyPolicy.LearnerDelta(oldElo, familyElo, isCorrect, current.Attempts); var now = DateTime.UtcNow; var update = Builders.Update - .SetOnInsert(value => value.UserId, claims.UserId) - .SetOnInsert(value => value.ObjectiveId, claims.ObjectiveId) - .Set(value => value.Elo, oldElo + delta) + .Inc(value => value.Elo, delta) .Inc(value => value.Attempts, 1) .Inc(value => value.Correct, isCorrect ? 1 : 0) .Set(value => value.LastSeenUtc, now); var updated = await _mastery.FindOneAndUpdateAsync( value => value.UserId == claims.UserId && value.ObjectiveId == claims.ObjectiveId, update, - new FindOneAndUpdateOptions { IsUpsert = true, ReturnDocument = ReturnDocument.After }); + new FindOneAndUpdateOptions { ReturnDocument = ReturnDocument.After }); + await _mastery.UpdateOneAsync( + value => value.UserId == claims.UserId && value.ObjectiveId == claims.ObjectiveId, + Builders.Update + .Max(value => value.Elo, DynamicDifficultyPolicy.MinimumElo) + .Min(value => value.Elo, DynamicDifficultyPolicy.MaximumElo)); + if (!string.IsNullOrWhiteSpace(claims.CaseFamilyId)) + await UpdateFamilyRatingAsync(claims, generated, isCorrect); return new DynamicAttemptResponse { Inserted = true, @@ -98,6 +128,70 @@ public sealed class DynamicQuestionStore }; } + private async Task EnsureMasteryAsync(string userId, string objectiveId) + { + try + { + await _mastery.UpdateOneAsync( + value => value.UserId == userId && value.ObjectiveId == objectiveId, + Builders.Update + .SetOnInsert(value => value.UserId, userId) + .SetOnInsert(value => value.ObjectiveId, objectiveId) + .SetOnInsert(value => value.Elo, 1000) + .SetOnInsert(value => value.Attempts, 0) + .SetOnInsert(value => value.Correct, 0), + new UpdateOptions { IsUpsert = true }); + } + catch (MongoWriteException exception) when (exception.WriteError.Category == ServerErrorCategory.DuplicateKey) { } + } + + private async Task UpdateFamilyRatingAsync( + DynamicGenerationClaims claims, + DynamicGeneratedQuestion generated, + bool isCorrect) + { + var family = generated.Blueprint.CaseFamilies.First(value => value.Id == claims.CaseFamilyId); + try + { + await _familyRatings.UpdateOneAsync( + value => value.BlueprintExternalId == claims.BlueprintExternalId + && value.BlueprintVersion == claims.BlueprintVersion + && value.CaseFamilyId == claims.CaseFamilyId, + Builders.Update + .SetOnInsert(value => value.BlueprintExternalId, claims.BlueprintExternalId) + .SetOnInsert(value => value.BlueprintVersion, claims.BlueprintVersion) + .SetOnInsert(value => value.CaseFamilyId, claims.CaseFamilyId) + .SetOnInsert(value => value.DifficultyTier, family.DifficultyTier) + .SetOnInsert(value => value.NominalElo, family.NominalElo) + .SetOnInsert(value => value.Elo, family.NominalElo) + .SetOnInsert(value => value.Attempts, 0) + .SetOnInsert(value => value.Correct, 0), + new UpdateOptions { IsUpsert = true }); + } + catch (MongoWriteException exception) when (exception.WriteError.Category == ServerErrorCategory.DuplicateKey) { } + var current = await _familyRatings.Find(value => + value.BlueprintExternalId == claims.BlueprintExternalId + && value.BlueprintVersion == claims.BlueprintVersion + && value.CaseFamilyId == claims.CaseFamilyId).FirstAsync(); + var delta = DynamicDifficultyPolicy.FamilyDelta( + claims.EffectiveAbilityElo > 0 ? claims.EffectiveAbilityElo : 1000, + claims.FamilyElo > 0 ? claims.FamilyElo : current.Elo, + isCorrect, + current.Attempts); + await _familyRatings.UpdateOneAsync( + value => value.Id == current.Id, + Builders.Update + .Inc(value => value.Elo, delta) + .Inc(value => value.Attempts, 1) + .Inc(value => value.Correct, isCorrect ? 1 : 0) + .Set(value => value.UpdatedUtc, DateTime.UtcNow)); + await _familyRatings.UpdateOneAsync( + value => value.Id == current.Id, + Builders.Update + .Max(value => value.Elo, DynamicDifficultyPolicy.MinimumElo) + .Min(value => value.Elo, DynamicDifficultyPolicy.MaximumElo)); + } + public async Task StatusAsync(string userId) { var objectiveCount = (await PublishedAsync()).Select(value => value.ObjectiveId).Distinct().Count(); diff --git a/api/QuestionsApi/Services/DynamicReviewExportCommand.cs b/api/QuestionsApi/Services/DynamicReviewExportCommand.cs index 48ebe60..fb85faf 100644 --- a/api/QuestionsApi/Services/DynamicReviewExportCommand.cs +++ b/api/QuestionsApi/Services/DynamicReviewExportCommand.cs @@ -26,6 +26,7 @@ public static class DynamicReviewExportCommand foreach (var file in Directory.EnumerateFiles(source, "*.json", SearchOption.AllDirectories)) blueprints.AddRange(JsonSerializer.Deserialize>( await File.ReadAllTextAsync(file), jsonOptions) ?? []); + blueprints = DynamicBlueprintResolver.ResolveAll(blueprints); blueprints = blueprints.Where(value => value.IsPublished && value.QualityStandardVersion >= 2).ToList(); var configuration = new ConfigurationBuilder() diff --git a/api/QuestionsApi/Services/QuestionReportStore.cs b/api/QuestionsApi/Services/QuestionReportStore.cs index 94e4742..1b8b5f4 100644 --- a/api/QuestionsApi/Services/QuestionReportStore.cs +++ b/api/QuestionsApi/Services/QuestionReportStore.cs @@ -100,7 +100,10 @@ public class QuestionReportStore BlueprintVersion = claims.BlueprintVersion, GeneratorVersion = claims.GeneratorVersion, Seed = claims.Seed, - CaseFamilyId = claims.CaseFamilyId + CaseFamilyId = claims.CaseFamilyId, + DifficultyTier = claims.DifficultyTier, + FamilyElo = claims.FamilyElo, + EffectiveAbilityElo = claims.EffectiveAbilityElo }, CreatedUtc = now, UpdatedUtc = now diff --git a/api/QuestionsApi/Services/ServiceMetrics.cs b/api/QuestionsApi/Services/ServiceMetrics.cs index 8270bac..f1365d3 100644 --- a/api/QuestionsApi/Services/ServiceMetrics.cs +++ b/api/QuestionsApi/Services/ServiceMetrics.cs @@ -13,6 +13,10 @@ public sealed class ServiceMetrics private long _httpRequests; private long _httpDurationMicroseconds; private long _answerStatisticsBackfillComplete; + private readonly System.Collections.Concurrent.ConcurrentDictionary _dynamicSelections = new(); + private readonly System.Collections.Concurrent.ConcurrentDictionary _dynamicAttempts = new(); + private readonly System.Collections.Concurrent.ConcurrentDictionary _dynamicCorrect = new(); + private long _dynamicTargetGapTotal; public void AttemptCreated() => Interlocked.Increment(ref _attemptsCreated); public void AttemptDuplicate() => Interlocked.Increment(ref _attemptDuplicates); @@ -29,6 +33,16 @@ public sealed class ServiceMetrics Interlocked.Increment(ref _httpRequests); Interlocked.Add(ref _httpDurationMicroseconds, (long)(duration.TotalMilliseconds * 1000)); } + public void DynamicSelected(string tier, int targetGap) + { + _dynamicSelections.AddOrUpdate(tier, 1, (_, value) => value + 1); + Interlocked.Add(ref _dynamicTargetGapTotal, Math.Abs(targetGap)); + } + public void DynamicAttempt(string tier, bool correct) + { + _dynamicAttempts.AddOrUpdate(tier, 1, (_, value) => value + 1); + if (correct) _dynamicCorrect.AddOrUpdate(tier, 1, (_, value) => value + 1); + } public string RenderPrometheus() { @@ -39,6 +53,16 @@ public sealed class ServiceMetrics AppendCounter(output, "step3_rating_failures_total", Interlocked.Read(ref _ratingFailures)); AppendCounter(output, "step3_answer_reconciliations_total", Interlocked.Read(ref _reconciliations)); AppendCounter(output, "step3_http_requests_total", Interlocked.Read(ref _httpRequests)); + output.AppendLine("# TYPE step3_dynamic_selections_total counter"); + output.AppendLine("# TYPE step3_dynamic_attempts_total counter"); + output.AppendLine("# TYPE step3_dynamic_correct_total counter"); + foreach (var tier in _dynamicSelections.Keys.Order()) + { + AppendLabeledSample(output, "step3_dynamic_selections_total", tier, _dynamicSelections.GetValueOrDefault(tier)); + AppendLabeledSample(output, "step3_dynamic_attempts_total", tier, _dynamicAttempts.GetValueOrDefault(tier)); + AppendLabeledSample(output, "step3_dynamic_correct_total", tier, _dynamicCorrect.GetValueOrDefault(tier)); + } + AppendCounter(output, "step3_dynamic_target_gap_points_total", Interlocked.Read(ref _dynamicTargetGapTotal)); output.AppendLine("# TYPE step3_http_request_duration_seconds summary"); output.Append("step3_http_request_duration_seconds_count ") .Append(Interlocked.Read(ref _httpRequests)) @@ -62,4 +86,9 @@ public sealed class ServiceMetrics output.Append("# TYPE ").Append(name).AppendLine(" counter"); output.Append(name).Append(' ').Append(value).AppendLine(); } + + private static void AppendLabeledSample(StringBuilder output, string name, string tier, long value) + { + output.Append(name).Append("{tier=\"").Append(tier.Replace("\"", "")).Append("\"} ").Append(value).AppendLine(); + } } diff --git a/content/dynamic_questions/adaptive-v3.json b/content/dynamic_questions/adaptive-v3.json new file mode 100644 index 0000000..f5ee1f1 --- /dev/null +++ b/content/dynamic_questions/adaptive-v3.json @@ -0,0 +1,72 @@ +[ + { + "externalId":"dyn-gold-dka","version":3,"baseBlueprintVersion":2, + "qualityStandardVersion":3,"isPublished":true, + "caseFamilies":[ + {"id":"dka-fnd-infection","baseFamilyId":"dka-infection","difficultyTier":"foundation","nominalElo":850,"stemTemplate":"The combination of hyperglycemia, ketonemia, and an anion-gap metabolic acidosis confirms diabetic ketoacidosis."}, + {"id":"dka-fnd-new","baseFamilyId":"dka-new-onset","difficultyTier":"foundation","nominalElo":850,"stemTemplate":"The patient is clinically volume depleted, and there is no evidence of heart or kidney failure."}, + {"id":"dka-std-pump","baseFamilyId":"dka-pump-failure","difficultyTier":"standard","nominalElo":1050,"stemTemplate":"The emergency team prepares simultaneous cardiac monitoring and repeat electrolyte measurements."}, + {"id":"dka-std-infection","baseFamilyId":"dka-infection","difficultyTier":"standard","nominalElo":1050,"stemTemplate":"The urinary infection will require treatment, but the team must first sequence the metabolic resuscitation correctly."}, + {"id":"dka-adv-new","baseFamilyId":"dka-new-onset","difficultyTier":"advanced","nominalElo":1250,"stemTemplate":"Although the measured serum potassium is not low, the patient has had substantial urinary potassium losses and has not yet received insulin."}, + {"id":"dka-adv-pump","baseFamilyId":"dka-pump-failure","difficultyTier":"advanced","nominalElo":1250,"stemTemplate":"A family member asks whether reconnecting the pump alone will reverse the acute physiology."}, + {"id":"dka-exp-infection","baseFamilyId":"dka-infection","difficultyTier":"expert","nominalElo":1450,"stemTemplate":"The patient received one antibiotic dose at urgent care; this does not explain the persistent shock physiology or alter the immediate metabolic priority."}, + {"id":"dka-exp-pump","baseFamilyId":"dka-pump-failure","difficultyTier":"expert","nominalElo":1450,"stemTemplate":"Point-of-care ultrasound shows a small collapsible inferior vena cava and preserved left ventricular systolic function."} + ] + }, + { + "externalId":"dyn-gold-af","version":3,"baseBlueprintVersion":2, + "qualityStandardVersion":3,"isPublished":true, + "caseFamilies":[ + {"id":"af-fnd-sepsis","baseFamilyId":"af-sepsis","difficultyTier":"foundation","nominalElo":850,"stemTemplate":"The hypotension and confusion began with the rapid rhythm and are attributed to the tachyarrhythmia."}, + {"id":"af-fnd-edema","baseFamilyId":"af-pulmonary-edema","difficultyTier":"foundation","nominalElo":850,"stemTemplate":"The patient has a pulse but has acute pulmonary edema and shock caused by the rapid rhythm."}, + {"id":"af-std-ischemia","baseFamilyId":"af-ischemia","difficultyTier":"standard","nominalElo":1050,"stemTemplate":"The exact duration of atrial fibrillation is unknown."}, + {"id":"af-std-sepsis","baseFamilyId":"af-sepsis","difficultyTier":"standard","nominalElo":1050,"stemTemplate":"Antibiotics and source control are underway, but perfusion continues to deteriorate as the ventricular rate rises."}, + {"id":"af-adv-edema","baseFamilyId":"af-pulmonary-edema","difficultyTier":"advanced","nominalElo":1250,"stemTemplate":"The patient took the usual beta blocker this morning, and additional atrioventricular nodal blockade could worsen the low-output state."}, + {"id":"af-adv-ischemia","baseFamilyId":"af-ischemia","difficultyTier":"advanced","nominalElo":1250,"stemTemplate":"A postoperative hemoglobin measurement is mildly decreased but stable and does not account for the abrupt hemodynamic change."}, + {"id":"af-exp-sepsis","baseFamilyId":"af-sepsis","difficultyTier":"expert","nominalElo":1450,"stemTemplate":"A prior ECG showed sinus rhythm with a normal PR interval; the current tracing has no pre-excitation and the instability persists after a cautious fluid challenge."}, + {"id":"af-exp-ischemia","baseFamilyId":"af-ischemia","difficultyTier":"expert","nominalElo":1450,"stemTemplate":"Bedside echocardiography shows no pericardial effusion or right ventricular dilation, narrowing the cause of shock to the rapid atrial arrhythmia."} + ] + }, + { + "externalId":"dyn-gold-pyelo-pregnancy","version":3,"baseBlueprintVersion":2, + "qualityStandardVersion":3,"isPublished":true, + "caseFamilies":[ + {"id":"pyelo-fnd-classic","baseFamilyId":"pyelo-classic","difficultyTier":"foundation","nominalElo":850,"stemTemplate":"The fever and costovertebral-angle tenderness distinguish this upper urinary tract infection from uncomplicated cystitis."}, + {"id":"pyelo-fnd-recurrent","baseFamilyId":"pyelo-recurrent","difficultyTier":"foundation","nominalElo":850,"stemTemplate":"There is no drug allergy, and renal function is normal."}, + {"id":"pyelo-std-systemic","baseFamilyId":"pyelo-systemic","difficultyTier":"standard","nominalElo":1050,"stemTemplate":"There is no evidence of obstructing nephrolithiasis on bedside assessment."}, + {"id":"pyelo-std-classic","baseFamilyId":"pyelo-classic","difficultyTier":"standard","nominalElo":1050,"stemTemplate":"The patient can take oral medication but remains febrile and systemically ill."}, + {"id":"pyelo-adv-recurrent","baseFamilyId":"pyelo-recurrent","difficultyTier":"advanced","nominalElo":1250,"stemTemplate":"The earlier urine isolate was susceptible to nitrofurantoin, but susceptibility does not overcome inadequate renal parenchymal penetration."}, + {"id":"pyelo-adv-systemic","baseFamilyId":"pyelo-systemic","difficultyTier":"advanced","nominalElo":1250,"stemTemplate":"Mild uterine irritability resolves with hydration, and there is no indication for delivery."}, + {"id":"pyelo-exp-classic","baseFamilyId":"pyelo-classic","difficultyTier":"expert","nominalElo":1450,"stemTemplate":"A preliminary culture reports a gram-negative rod; definitive susceptibilities will not be available until tomorrow."}, + {"id":"pyelo-exp-systemic","baseFamilyId":"pyelo-systemic","difficultyTier":"expert","nominalElo":1450,"stemTemplate":"The lactate and respiratory rate raise concern for evolving maternal sepsis, making disposition as important as antibiotic spectrum."} + ] + }, + { + "externalId":"dyn-gold-sah","version":3,"baseBlueprintVersion":2, + "qualityStandardVersion":3,"isPublished":true, + "caseFamilies":[ + {"id":"sah-fnd-exertion","baseFamilyId":"sah-exertion","difficultyTier":"foundation","nominalElo":850,"stemTemplate":"The sudden maximal-at-onset headache and meningismus are classic warning features for subarachnoid hemorrhage."}, + {"id":"sah-fnd-sentinel","baseFamilyId":"sah-sentinel","difficultyTier":"foundation","nominalElo":850,"stemTemplate":"This is a first thunderclap headache, not a recurrence of a previously evaluated migraine syndrome."}, + {"id":"sah-std-focal","baseFamilyId":"sah-focal","difficultyTier":"standard","nominalElo":1050,"stemTemplate":"The patient is stable enough for immediate diagnostic imaging."}, + {"id":"sah-std-exertion","baseFamilyId":"sah-exertion","difficultyTier":"standard","nominalElo":1050,"stemTemplate":"The absence of persistent focal weakness does not reduce the urgency of excluding hemorrhage."}, + {"id":"sah-adv-sentinel","baseFamilyId":"sah-sentinel","difficultyTier":"advanced","nominalElo":1250,"stemTemplate":"The presentation occurred recently enough that the initial imaging strategy has high diagnostic yield, but a negative result must still be interpreted in clinical context."}, + {"id":"sah-adv-focal","baseFamilyId":"sah-focal","difficultyTier":"advanced","nominalElo":1250,"stemTemplate":"The partial third-nerve finding raises concern for an aneurysmal source but does not replace the initial study for acute blood."}, + {"id":"sah-exp-exertion","baseFamilyId":"sah-exertion","difficultyTier":"expert","nominalElo":1450,"stemTemplate":"A nonfocal examination and transient improvement after analgesia are potentially misleading and should not alter the first diagnostic step."}, + {"id":"sah-exp-sentinel","baseFamilyId":"sah-sentinel","difficultyTier":"expert","nominalElo":1450,"stemTemplate":"Family history increases pretest probability, while the choice between subsequent lumbar puncture and angiographic evaluation depends on the result and timing of the initial study."} + ] + }, + { + "externalId":"dyn-gold-arr","version":3,"baseBlueprintVersion":2, + "qualityStandardVersion":3,"isPublished":true, + "caseFamilies":[ + {"id":"arr-fnd-cardio","baseFamilyId":"arr-cardiovascular","difficultyTier":"foundation","nominalElo":850,"stemTemplate":"Calculate the arithmetic difference between the two event rates rather than a relative measure."}, + {"id":"arr-fnd-infection","baseFamilyId":"arr-infection","difficultyTier":"foundation","nominalElo":850,"stemTemplate":"The requested measure should be reported in percentage points."}, + {"id":"arr-std-readmit","baseFamilyId":"arr-readmission","difficultyTier":"standard","nominalElo":1050,"stemTemplate":"Do not convert the result to number needed to treat unless specifically asked."}, + {"id":"arr-std-cardio","baseFamilyId":"arr-cardiovascular","difficultyTier":"standard","nominalElo":1050,"stemTemplate":"The trial reports sufficient data to calculate absolute and relative effects directly."}, + {"id":"arr-adv-infection","baseFamilyId":"arr-infection","difficultyTier":"advanced","nominalElo":1250,"stemTemplate":"A secondary table emphasizes a large relative reduction, but the question asks for the absolute treatment effect."}, + {"id":"arr-adv-readmit","baseFamilyId":"arr-readmission","difficultyTier":"advanced","nominalElo":1250,"stemTemplate":"The intervention has similar follow-up completeness in both groups, so no adjustment for attrition is needed."}, + {"id":"arr-exp-cardio","baseFamilyId":"arr-cardiovascular","difficultyTier":"expert","nominalElo":1450,"stemTemplate":"The authors additionally report an adjusted hazard ratio; that time-to-event estimate is not needed for this requested absolute measure."}, + {"id":"arr-exp-readmit","baseFamilyId":"arr-readmission","difficultyTier":"expert","nominalElo":1450,"stemTemplate":"Cost-effectiveness and relative-risk results are presented as distractors, but neither changes the calculation from the observed event proportions."} + ] + } +] diff --git a/docs/dynamic-question-quality.md b/docs/dynamic-question-quality.md index ab29879..0d7e283 100644 --- a/docs/dynamic-question-quality.md +++ b/docs/dynamic-question-quality.md @@ -3,6 +3,14 @@ Quality-standard-2 blueprints are eligible for learner runs. Older blueprints remain stored only so signed questions and reports can be regenerated. +Quality-standard-3 blueprints add Elo-adaptive difficulty. Each active objective +must provide at least two families at every tier: foundation (850), standard +(1050), advanced (1250), and expert (1450). The service targets an item about 70 +Elo below effective learner ability, approximately 60% expected correctness. +Effective ability begins as an equal blend of overall and category question-bank +Elo, then shifts toward dynamic objective Elo up to an 80% objective weight after +10 attempts. Aggregate outcomes calibrate each case family's Elo over time. + ## Content requirements - At least three curated case families per objective.