272 lines
12 KiB
C#
272 lines
12 KiB
C#
using Microsoft.Extensions.Configuration;
|
|
using QuestionsApi.Models;
|
|
using QuestionsApi.Services;
|
|
using Xunit;
|
|
|
|
namespace QuestionsApi.Tests;
|
|
|
|
public sealed class DynamicQuestionTests
|
|
{
|
|
[Fact]
|
|
public void Generator_IsDeterministic_AndPreservesCorrectAnswer()
|
|
{
|
|
var blueprint = Blueprint();
|
|
var generator = new DynamicQuestionGenerator();
|
|
|
|
var first = generator.Generate(blueprint, "same-seed", "one").Question;
|
|
var second = generator.Generate(blueprint, "same-seed", "two").Question;
|
|
|
|
Assert.Equal(first.Stem, second.Stem);
|
|
Assert.Equal(first.Choices.Select(value => value.Text), second.Choices.Select(value => value.Text));
|
|
Assert.Equal("correct", first.CorrectChoiceId);
|
|
Assert.Null(DynamicQuestionGenerator.Validate(blueprint));
|
|
}
|
|
|
|
[Fact]
|
|
public void SignedTokens_AreUserBound_AndRejectTampering()
|
|
{
|
|
var configuration = new ConfigurationBuilder()
|
|
.AddInMemoryCollection(new Dictionary<string, string?> { ["DynamicQuestions:SigningKey"] = "unit-test-key" })
|
|
.Build();
|
|
var service = new DynamicQuestionTokenService(configuration);
|
|
var claims = new DynamicGenerationClaims
|
|
{
|
|
UserId = "user-one",
|
|
BlueprintExternalId = "test",
|
|
BlueprintVersion = 1,
|
|
ObjectiveId = "objective",
|
|
Seed = "seed",
|
|
Nonce = "nonce",
|
|
GeneratorVersion = 1,
|
|
ExpiresUtc = DateTime.UtcNow.AddMinutes(5)
|
|
};
|
|
|
|
var token = service.SignGeneration(claims);
|
|
|
|
Assert.Equal("seed", service.ReadGeneration(token, "user-one").Seed);
|
|
Assert.Throws<ArgumentException>(() => service.ReadGeneration(token, "user-two"));
|
|
Assert.ThrowsAny<Exception>(() => service.ReadGeneration(token + "x", "user-one"));
|
|
}
|
|
|
|
[Fact]
|
|
public void GoldGenerator_UsesCaseFamilies_AndProducesAuthoredTeachingShape()
|
|
{
|
|
var blueprint = GoldBlueprint();
|
|
var generator = new DynamicQuestionGenerator();
|
|
var generated = Enumerable.Range(0, 100)
|
|
.Select(seed => generator.Generate(blueprint, seed.ToString(), seed.ToString()))
|
|
.ToList();
|
|
|
|
Assert.Equal(3, generated.Select(value => value.CaseFamilyId).Distinct().Count());
|
|
Assert.All(generated, value =>
|
|
{
|
|
Assert.True(value.Question.Stem.Length >= 220);
|
|
Assert.Equal(5, value.Question.Choices.Count);
|
|
Assert.Equal(5, value.Question.ChoiceExplanations.Count);
|
|
Assert.Single(value.Question.ReviewFollowups);
|
|
Assert.Equal(4, value.Question.WrongAnswerFollowups.Count);
|
|
Assert.Contains("quality-standard:2", value.Question.Tags);
|
|
});
|
|
Assert.Null(DynamicQuestionGenerator.Validate(blueprint));
|
|
}
|
|
|
|
[Fact]
|
|
public void LegacyGeneratorVersion_RemainsDeterministic()
|
|
{
|
|
var generator = new DynamicQuestionGenerator();
|
|
var first = generator.Generate(Blueprint(), "legacy-seed", "one", 1).Question;
|
|
var regenerated = generator.Generate(Blueprint(), "legacy-seed", "two", 1).Question;
|
|
Assert.Equal(first.Stem, regenerated.Stem);
|
|
Assert.Equal(first.Choices.Select(value => value.Id), regenerated.Choices.Select(value => value.Id));
|
|
}
|
|
|
|
[Fact]
|
|
public void DifficultyPolicy_BlendsThenSpecializes_AndTargetsSixtyPercent()
|
|
{
|
|
Assert.Equal(1200, DynamicDifficultyPolicy.EffectiveAbility(1100, 1300, null));
|
|
var early = new DynamicObjectiveMastery { Elo = 1500, Attempts = 1 };
|
|
var established = new DynamicObjectiveMastery { Elo = 1500, Attempts = 10 };
|
|
Assert.Equal(1224, DynamicDifficultyPolicy.EffectiveAbility(1100, 1300, early));
|
|
Assert.Equal(1440, DynamicDifficultyPolicy.EffectiveAbility(1100, 1300, established));
|
|
Assert.Equal(1370, DynamicDifficultyPolicy.TargetFamilyElo(1440));
|
|
}
|
|
|
|
[Fact]
|
|
public void DifficultyPolicy_SelectsClosestNonrecentFamily()
|
|
{
|
|
var families = new List<DynamicCaseFamily>
|
|
{
|
|
new() { Id = "foundation", DifficultyTier = "foundation", NominalElo = 850 },
|
|
new() { Id = "standard", DifficultyTier = "standard", NominalElo = 1050 },
|
|
new() { Id = "advanced-a", DifficultyTier = "advanced", NominalElo = 1250 },
|
|
new() { Id = "advanced-b", DifficultyTier = "advanced", NominalElo = 1250 },
|
|
new() { Id = "expert", DifficultyTier = "expert", NominalElo = 1450 }
|
|
};
|
|
var selected = DynamicDifficultyPolicy.SelectFamily(
|
|
families, new Dictionary<string, DynamicFamilyRating>(), 1260,
|
|
new HashSet<string> { "advanced-a" }, 0);
|
|
Assert.Equal("advanced-b", selected.Id);
|
|
}
|
|
|
|
[Fact]
|
|
public void DifficultyPolicy_RatingsMoveInOppositeDirections()
|
|
{
|
|
Assert.True(DynamicDifficultyPolicy.LearnerDelta(1000, 1050, true, 0) > 0);
|
|
Assert.True(DynamicDifficultyPolicy.FamilyDelta(1000, 1050, true, 0) < 0);
|
|
Assert.True(DynamicDifficultyPolicy.LearnerDelta(1000, 1050, false, 0) < 0);
|
|
Assert.True(DynamicDifficultyPolicy.FamilyDelta(1000, 1050, false, 0) > 0);
|
|
Assert.Equal(600, DynamicDifficultyPolicy.Clamp(-100));
|
|
Assert.Equal(1800, DynamicDifficultyPolicy.Clamp(5000));
|
|
}
|
|
|
|
[Fact]
|
|
public void ReviewPolicy_UsesFixedIntervals_AndResetsAfterAMiss()
|
|
{
|
|
var now = new DateTime(2026, 7, 23, 12, 0, 0, DateTimeKind.Utc);
|
|
|
|
Assert.Equal(now.AddDays(1), DynamicReviewPolicy.FirstDue(now));
|
|
Assert.Equal((1, now.AddDays(3), false), DynamicReviewPolicy.AfterReview(0, true, now));
|
|
Assert.Equal((2, now.AddDays(7), false), DynamicReviewPolicy.AfterReview(1, true, now));
|
|
Assert.Equal((3, now.AddDays(21), false), DynamicReviewPolicy.AfterReview(2, true, now));
|
|
Assert.Equal((4, now.AddDays(45), false), DynamicReviewPolicy.AfterReview(3, true, now));
|
|
Assert.Equal((5, null, true), DynamicReviewPolicy.AfterReview(4, true, now));
|
|
Assert.Equal((0, now.AddDays(1), false), DynamicReviewPolicy.AfterReview(3, false, now));
|
|
}
|
|
|
|
[Fact]
|
|
public void QualityStandardFour_RequiresCoverageReviewAndMisconceptionMetadata()
|
|
{
|
|
var options = new System.Text.Json.JsonSerializerOptions(System.Text.Json.JsonSerializerDefaults.Web)
|
|
{ PropertyNameCaseInsensitive = true };
|
|
var root = Path.GetFullPath(Path.Combine(
|
|
AppContext.BaseDirectory, "..", "..", "..", "..", "..", "content", "dynamic_questions"));
|
|
var source = Directory.EnumerateFiles(root, "*.json")
|
|
.SelectMany(file => System.Text.Json.JsonSerializer.Deserialize<List<DynamicQuestionBlueprint>>(
|
|
File.ReadAllText(file), options) ?? [])
|
|
.ToList();
|
|
var blueprint = DynamicBlueprintResolver.ResolveAll(source)
|
|
.First(value => value.QualityStandardVersion == 3);
|
|
blueprint.QualityStandardVersion = 4;
|
|
blueprint.BlueprintWeight = 6;
|
|
|
|
Assert.Contains("encounterFrame", DynamicQuestionGenerator.Validate(blueprint));
|
|
|
|
blueprint.System = "Endocrine";
|
|
blueprint.PhysicianTask = "Pharmacotherapy";
|
|
blueprint.EncounterFrame = "Urgent Intervention";
|
|
blueprint.SiteOfCare = "Emergency Department";
|
|
blueprint.ConceptIds = ["test.concept"];
|
|
blueprint.MedicallyReviewedUtc = DateTime.UtcNow.AddDays(-1);
|
|
blueprint.MedicalReviewExpiresUtc = DateTime.UtcNow.AddYears(1);
|
|
blueprint.MedicalReviewer = "Medical editor";
|
|
foreach (var distractor in blueprint.Distractors)
|
|
distractor.MisconceptionId = $"misconception.{distractor.Id}";
|
|
|
|
Assert.Null(DynamicQuestionGenerator.Validate(blueprint));
|
|
}
|
|
|
|
[Fact]
|
|
public void CoveragePolicy_UsesOfficialDefaults_AndAllowsVersionedOverrides()
|
|
{
|
|
Assert.Equal(12, DynamicBlueprintCoveragePolicy.Weight(
|
|
new DynamicQuestionBlueprint { System = "Biostatistics" }));
|
|
Assert.Equal(10, DynamicBlueprintCoveragePolicy.Weight(
|
|
new DynamicQuestionBlueprint { System = "Cardiovascular" }));
|
|
Assert.Equal(2.5, DynamicBlueprintCoveragePolicy.Weight(
|
|
new DynamicQuestionBlueprint { System = "Cardiovascular", BlueprintWeight = 2.5 }));
|
|
}
|
|
|
|
[Fact]
|
|
public void AdaptiveContent_ResolvesToFourCompleteTiers()
|
|
{
|
|
var options = new System.Text.Json.JsonSerializerOptions(System.Text.Json.JsonSerializerDefaults.Web)
|
|
{ PropertyNameCaseInsensitive = true };
|
|
var root = Path.GetFullPath(Path.Combine(
|
|
AppContext.BaseDirectory, "..", "..", "..", "..", "..", "content", "dynamic_questions"));
|
|
var source = Directory.EnumerateFiles(root, "*.json")
|
|
.SelectMany(file => System.Text.Json.JsonSerializer.Deserialize<List<DynamicQuestionBlueprint>>(
|
|
File.ReadAllText(file), options) ?? [])
|
|
.ToList();
|
|
var adaptive = DynamicBlueprintResolver.ResolveAll(source)
|
|
.Where(value => value.QualityStandardVersion >= 3).ToList();
|
|
|
|
Assert.Equal(5, adaptive.Count);
|
|
Assert.All(adaptive, blueprint =>
|
|
{
|
|
Assert.Equal(8, blueprint.CaseFamilies.Count);
|
|
Assert.All(DynamicDifficultyPolicy.TierElos.Keys, tier =>
|
|
Assert.Equal(2, blueprint.CaseFamilies.Count(value => value.DifficultyTier == tier)));
|
|
Assert.All(blueprint.CaseFamilies, family =>
|
|
Assert.False(family.StemTemplate.EndsWith(family.LeadIn, StringComparison.Ordinal)));
|
|
Assert.Null(DynamicQuestionGenerator.Validate(blueprint));
|
|
});
|
|
}
|
|
|
|
private static DynamicQuestionBlueprint Blueprint() => new()
|
|
{
|
|
ExternalId = "test",
|
|
ObjectiveId = "objective",
|
|
StemTemplate = "Patient age {{age}}.",
|
|
LeadIn = "What is correct?",
|
|
Variables = new()
|
|
{
|
|
["age"] = new DynamicVariableDefinition { Minimum = 20, Maximum = 80, Step = 1 }
|
|
},
|
|
CorrectAnswer = new DynamicAnswerTemplate { Id = "correct", TextTemplate = "Correct" },
|
|
Distractors =
|
|
[
|
|
new() { Id = "a", TextTemplate = "A" },
|
|
new() { Id = "b", TextTemplate = "B" },
|
|
new() { Id = "c", TextTemplate = "C" },
|
|
new() { Id = "d", TextTemplate = "D" }
|
|
],
|
|
References = [new QuestionReference { Title = "Reference", Url = "https://example.test" }]
|
|
};
|
|
|
|
private static DynamicQuestionBlueprint GoldBlueprint()
|
|
{
|
|
var stem = string.Join(" ", Enumerable.Repeat(
|
|
"A patient presents with a carefully correlated history, examination, vital signs, and diagnostic findings that establish the clinical context without naming the diagnosis.", 3));
|
|
return new DynamicQuestionBlueprint
|
|
{
|
|
ExternalId = "gold-test",
|
|
Version = 2,
|
|
ObjectiveId = "gold.objective",
|
|
QualityStandardVersion = 2,
|
|
CorrectConceptId = "gold.correct",
|
|
CorrectAnswer = new DynamicAnswerTemplate
|
|
{ Id = "correct", TextTemplate = "Correct action", ExplanationTemplate = "This addresses the decisive finding." },
|
|
Distractors = Enumerable.Range(1, 6).Select(index => new DynamicAnswerTemplate
|
|
{
|
|
Id = $"d{index}",
|
|
TextTemplate = $"Distractor {index}",
|
|
ExplanationTemplate = $"Distractor {index} does not address the decisive finding."
|
|
}).ToList(),
|
|
ExplanationTemplate = "A complete explanation connects the presentation to the action.",
|
|
KeyTakeaway = "Use the decisive finding.",
|
|
UsmlePearl = "Sequence urgent care correctly.",
|
|
ReviewFollowups =
|
|
[
|
|
new DynamicReviewFollowupTemplate
|
|
{
|
|
PromptTemplate = "What is decisive?",
|
|
Choices =
|
|
[
|
|
new() { Id = "yes", TextTemplate = "The unstable finding" },
|
|
new() { Id = "no", TextTemplate = "An incidental finding" }
|
|
],
|
|
CorrectChoiceId = "yes",
|
|
TeachingPointTemplate = "The unstable finding determines management."
|
|
}
|
|
],
|
|
CaseFamilies =
|
|
[
|
|
new() { Id = "family-a", StemTemplate = stem, LeadIn = "What is next?" },
|
|
new() { Id = "family-b", StemTemplate = stem + " Family B.", LeadIn = "What is next?" },
|
|
new() { Id = "family-c", StemTemplate = stem + " Family C.", LeadIn = "What is next?" }
|
|
],
|
|
References = [new QuestionReference { Title = "Reference", Url = "https://example.test" }]
|
|
};
|
|
}
|
|
}
|