Upgrade dynamic questions to authored parity
This commit is contained in:
@@ -8,12 +8,17 @@ namespace QuestionsApi.Services;
|
||||
|
||||
public sealed class DynamicQuestionGenerator
|
||||
{
|
||||
public const int CurrentVersion = 1;
|
||||
public const int CurrentVersion = 2;
|
||||
private static readonly Regex Placeholder = new(@"\{\{([a-zA-Z0-9_]+)\}\}", RegexOptions.Compiled);
|
||||
|
||||
public DynamicGeneratedQuestion Generate(DynamicQuestionBlueprint blueprint, string seed, string nonce)
|
||||
public DynamicGeneratedQuestion Generate(DynamicQuestionBlueprint blueprint, string seed, string nonce, int generatorVersion = CurrentVersion)
|
||||
=> blueprint.QualityStandardVersion >= 2
|
||||
? GenerateV2(blueprint, seed, nonce)
|
||||
: GenerateLegacy(blueprint, seed, nonce, generatorVersion);
|
||||
|
||||
private DynamicGeneratedQuestion GenerateLegacy(DynamicQuestionBlueprint blueprint, string seed, string nonce, int generatorVersion)
|
||||
{
|
||||
var draw = new DeterministicDraw($"{blueprint.ExternalId}|{blueprint.Version}|{CurrentVersion}|{seed}");
|
||||
var draw = new DeterministicDraw($"{blueprint.ExternalId}|{blueprint.Version}|{generatorVersion}|{seed}");
|
||||
var values = new Dictionary<string, string>(StringComparer.Ordinal);
|
||||
var numbers = new Dictionary<string, double>(StringComparer.Ordinal);
|
||||
foreach (var item in blueprint.Variables.OrderBy(value => value.Key, StringComparer.Ordinal))
|
||||
@@ -112,11 +117,193 @@ public sealed class DynamicQuestionGenerator
|
||||
return new DynamicGeneratedQuestion { Question = question, Blueprint = blueprint };
|
||||
}
|
||||
|
||||
private DynamicGeneratedQuestion GenerateV2(DynamicQuestionBlueprint blueprint, string seed, string nonce)
|
||||
{
|
||||
if (blueprint.CaseFamilies.Count == 0)
|
||||
throw new InvalidOperationException("quality-standard-2 blueprints require case families");
|
||||
var draw = new DeterministicDraw($"{blueprint.ExternalId}|{blueprint.Version}|{CurrentVersion}|{seed}");
|
||||
var family = blueprint.CaseFamilies.OrderBy(value => value.Id, StringComparer.Ordinal)
|
||||
.ElementAt(draw.Next(blueprint.CaseFamilies.Count));
|
||||
var values = new Dictionary<string, string>(StringComparer.Ordinal);
|
||||
var numbers = new Dictionary<string, double>(StringComparer.Ordinal);
|
||||
if (family.ValueBundles.Count > 0)
|
||||
foreach (var item in family.ValueBundles[draw.Next(family.ValueBundles.Count)])
|
||||
values[item.Key] = item.Value;
|
||||
PopulateVariables(family.Variables, values, numbers, draw);
|
||||
PopulateDerived(family.Derived, values, numbers);
|
||||
ValidateConstraints(family.Constraints, numbers);
|
||||
|
||||
var correctAnswer = string.IsNullOrWhiteSpace(family.CorrectAnswer.Id) ? blueprint.CorrectAnswer : family.CorrectAnswer;
|
||||
var distractorPool = family.Distractors.Count == 0 ? blueprint.Distractors : family.Distractors;
|
||||
var reviewTemplates = family.ReviewFollowups.Count == 0 ? blueprint.ReviewFollowups : family.ReviewFollowups;
|
||||
var wrongTemplates = family.WrongAnswerFollowups.Count == 0 ? blueprint.WrongAnswerFollowups : family.WrongAnswerFollowups;
|
||||
var explanationTemplate = string.IsNullOrWhiteSpace(family.ExplanationTemplate) ? blueprint.ExplanationTemplate : family.ExplanationTemplate;
|
||||
var keyTakeaway = string.IsNullOrWhiteSpace(family.KeyTakeaway) ? blueprint.KeyTakeaway : family.KeyTakeaway;
|
||||
var usmlePearl = string.IsNullOrWhiteSpace(family.UsmlePearl) ? blueprint.UsmlePearl : family.UsmlePearl;
|
||||
var distractors = distractorPool.OrderBy(_ => draw.Next(int.MaxValue)).Take(4).ToList();
|
||||
if (distractors.Count != 4) throw new InvalidOperationException($"{family.Id} requires at least four distractors");
|
||||
var answerTemplates = distractors.Append(correctAnswer).OrderBy(_ => draw.Next(int.MaxValue)).ToList();
|
||||
var choices = answerTemplates.Select(answer => new QuestionChoice
|
||||
{
|
||||
Id = answer.Id,
|
||||
Text = Render(answer.TextTemplate, values)
|
||||
}).ToList();
|
||||
if (choices.Select(value => value.Id).Distinct(StringComparer.Ordinal).Count() != 5
|
||||
|| choices.Select(value => value.Text).Distinct(StringComparer.OrdinalIgnoreCase).Count() != 5)
|
||||
throw new InvalidOperationException($"{family.Id} generated choices must be unique");
|
||||
|
||||
var question = new Question
|
||||
{
|
||||
Id = $"dynamic-{nonce}",
|
||||
ExternalId = $"dynamic-{nonce}",
|
||||
Stem = Render(family.StemTemplate, values),
|
||||
LeadIn = Render(family.LeadIn, values),
|
||||
Choices = choices,
|
||||
CorrectChoiceId = correctAnswer.Id,
|
||||
Explanation = Render(explanationTemplate, values),
|
||||
KeyTakeaway = Render(keyTakeaway, values),
|
||||
UsmlePearl = Render(usmlePearl, values),
|
||||
ChoiceExplanations = answerTemplates.ToDictionary(
|
||||
value => value.Id, value => Render(value.ExplanationTemplate, values)),
|
||||
ReviewFollowups = reviewTemplates.Select(followup => new QuestionReviewFollowup
|
||||
{
|
||||
Prompt = Render(followup.PromptTemplate, values),
|
||||
Choices = followup.Choices.Select(choice => new QuestionChoice
|
||||
{ Id = choice.Id, Text = Render(choice.TextTemplate, values) }).ToList(),
|
||||
CorrectChoiceId = followup.CorrectChoiceId,
|
||||
TeachingPoint = Render(followup.TeachingPointTemplate, values)
|
||||
}).ToList(),
|
||||
WrongAnswerFollowups = BuildWrongAnswerFollowups(
|
||||
distractors, correctAnswer, wrongTemplates, values),
|
||||
Discipline = blueprint.Discipline,
|
||||
System = blueprint.System,
|
||||
PerformanceCategory = blueprint.PerformanceCategory,
|
||||
Difficulty = blueprint.Difficulty,
|
||||
Labs = family.Labs.Select(lab => new QuestionLab
|
||||
{
|
||||
Name = lab.Name,
|
||||
Value = Render(lab.ValueTemplate, values),
|
||||
Unit = lab.Unit,
|
||||
Reference = lab.Reference
|
||||
}).ToList(),
|
||||
Tags = [.. blueprint.Tags, "dynamic", "quality-standard:2", $"objective:{blueprint.ObjectiveId}", $"family:{family.Id}"],
|
||||
References = blueprint.References,
|
||||
IsPublished = true
|
||||
};
|
||||
return new DynamicGeneratedQuestion { Question = question, Blueprint = blueprint, CaseFamilyId = family.Id };
|
||||
}
|
||||
|
||||
private static void PopulateVariables(
|
||||
IReadOnlyDictionary<string, DynamicVariableDefinition> definitions,
|
||||
IDictionary<string, string> values,
|
||||
IDictionary<string, double> numbers,
|
||||
DeterministicDraw draw)
|
||||
{
|
||||
foreach (var item in definitions.OrderBy(value => value.Key, StringComparer.Ordinal))
|
||||
{
|
||||
var definition = item.Value;
|
||||
if (definition.Type == "enum")
|
||||
{
|
||||
if (definition.Values.Count == 0) throw new InvalidOperationException($"{item.Key} requires values");
|
||||
values[item.Key] = definition.Values[draw.Next(definition.Values.Count)];
|
||||
continue;
|
||||
}
|
||||
var step = definition.Step <= 0 ? 1 : definition.Step;
|
||||
var count = (int)Math.Floor((definition.Maximum - definition.Minimum) / step) + 1;
|
||||
if (count < 1) throw new InvalidOperationException($"{item.Key} range is invalid");
|
||||
var number = Math.Round(definition.Minimum + draw.Next(count) * step, Math.Clamp(definition.Precision, 0, 8));
|
||||
numbers[item.Key] = number;
|
||||
values[item.Key] = Format(number, definition.Precision);
|
||||
}
|
||||
}
|
||||
|
||||
private static void PopulateDerived(
|
||||
IReadOnlyDictionary<string, DynamicDerivedDefinition> definitions,
|
||||
IDictionary<string, string> values,
|
||||
Dictionary<string, double> numbers)
|
||||
{
|
||||
foreach (var item in definitions)
|
||||
{
|
||||
var definition = item.Value;
|
||||
var left = ResolveNumber(definition.Left, numbers);
|
||||
var right = ResolveNumber(definition.Right, numbers);
|
||||
var number = definition.Operation switch
|
||||
{
|
||||
"add" => left + right,
|
||||
"subtract" => left - right,
|
||||
"multiply" => left * right,
|
||||
"divide" when Math.Abs(right) > double.Epsilon => left / right,
|
||||
_ => throw new InvalidOperationException($"{item.Key} operation is invalid")
|
||||
};
|
||||
number = Math.Round(number, Math.Clamp(definition.Precision, 0, 8));
|
||||
numbers[item.Key] = number;
|
||||
values[item.Key] = Format(number, definition.Precision);
|
||||
}
|
||||
}
|
||||
|
||||
private static void ValidateConstraints(IEnumerable<DynamicConstraint> constraints, IReadOnlyDictionary<string, double> numbers)
|
||||
{
|
||||
foreach (var constraint in constraints)
|
||||
{
|
||||
if (!numbers.TryGetValue(constraint.Field, out var actual))
|
||||
throw new InvalidOperationException($"constraint field {constraint.Field} is not numeric");
|
||||
var valid = constraint.Operator switch
|
||||
{
|
||||
"lt" => actual < constraint.Value,
|
||||
"lte" => actual <= constraint.Value,
|
||||
"gt" => actual > constraint.Value,
|
||||
"gte" => actual >= constraint.Value,
|
||||
"eq" => Math.Abs(actual - constraint.Value) < 0.0000001,
|
||||
"neq" => Math.Abs(actual - constraint.Value) >= 0.0000001,
|
||||
_ => false
|
||||
};
|
||||
if (!valid) throw new InvalidOperationException($"generated value violates {constraint.Field} {constraint.Operator}");
|
||||
}
|
||||
}
|
||||
|
||||
private static Dictionary<string, WrongAnswerFollowup> BuildWrongAnswerFollowups(
|
||||
IEnumerable<DynamicAnswerTemplate> distractors,
|
||||
DynamicAnswerTemplate correct,
|
||||
IReadOnlyDictionary<string, DynamicWrongAnswerFollowupTemplate> templates,
|
||||
IReadOnlyDictionary<string, string> values)
|
||||
{
|
||||
var result = new Dictionary<string, WrongAnswerFollowup>();
|
||||
foreach (var distractor in distractors)
|
||||
{
|
||||
if (templates.TryGetValue(distractor.Id, out var template))
|
||||
{
|
||||
result[distractor.Id] = new WrongAnswerFollowup
|
||||
{
|
||||
Prompt = Render(template.PromptTemplate, values),
|
||||
Choices = template.Choices.Select(choice => new QuestionChoice
|
||||
{ Id = choice.Id, Text = Render(choice.TextTemplate, values) }).ToList(),
|
||||
CorrectChoiceId = template.CorrectChoiceId,
|
||||
TeachingPoint = Render(template.TeachingPointTemplate, values)
|
||||
};
|
||||
continue;
|
||||
}
|
||||
result[distractor.Id] = new WrongAnswerFollowup
|
||||
{
|
||||
Prompt = "Which option best addresses the decisive finding in this case?",
|
||||
Choices =
|
||||
[
|
||||
new QuestionChoice { Id = correct.Id, Text = Render(correct.TextTemplate, values) },
|
||||
new QuestionChoice { Id = distractor.Id, Text = Render(distractor.TextTemplate, values) }
|
||||
],
|
||||
CorrectChoiceId = correct.Id,
|
||||
TeachingPoint = $"{Render(distractor.ExplanationTemplate, values)} {Render(correct.ExplanationTemplate, values)}"
|
||||
};
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
public static string? Validate(DynamicQuestionBlueprint blueprint)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(blueprint.ExternalId)) return "externalId required";
|
||||
if (string.IsNullOrWhiteSpace(blueprint.ObjectiveId)) return "objectiveId required";
|
||||
if (blueprint.Version < 1) return "version must be positive";
|
||||
if (blueprint.QualityStandardVersion >= 2)
|
||||
return ValidateV2(blueprint);
|
||||
if (string.IsNullOrWhiteSpace(blueprint.StemTemplate) || string.IsNullOrWhiteSpace(blueprint.LeadIn)) return "stemTemplate and leadIn required";
|
||||
if (string.IsNullOrWhiteSpace(blueprint.CorrectAnswer.Id) || string.IsNullOrWhiteSpace(blueprint.CorrectAnswer.TextTemplate)) return "correctAnswer required";
|
||||
if (blueprint.Distractors.Count < 4) return "at least four distractors required";
|
||||
@@ -133,6 +320,51 @@ public sealed class DynamicQuestionGenerator
|
||||
return null;
|
||||
}
|
||||
|
||||
private static string? ValidateV2(DynamicQuestionBlueprint blueprint)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(blueprint.CorrectConceptId)) return "correctConceptId required";
|
||||
if (blueprint.CaseFamilies.Count < 3) return "at least three case families required";
|
||||
if (blueprint.References.Count == 0) return "at least one reference required";
|
||||
if (blueprint.CaseFamilies.Select(value => value.Id).Distinct(StringComparer.Ordinal).Count() != blueprint.CaseFamilies.Count)
|
||||
return "case family ids must be unique";
|
||||
foreach (var family in blueprint.CaseFamilies)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(family.Id)) return "case family id required";
|
||||
if (family.StemTemplate.Length < 220) return $"{family.Id} stem is too short for quality standard 2";
|
||||
if (string.IsNullOrWhiteSpace(family.LeadIn)) return $"{family.Id} leadIn required";
|
||||
if (!string.IsNullOrWhiteSpace(family.CorrectAnswer.Id))
|
||||
return $"{family.Id} must use the blueprint-level invariant correct answer";
|
||||
var correctAnswer = string.IsNullOrWhiteSpace(family.CorrectAnswer.Id) ? blueprint.CorrectAnswer : family.CorrectAnswer;
|
||||
var distractors = family.Distractors.Count == 0 ? blueprint.Distractors : family.Distractors;
|
||||
var reviews = family.ReviewFollowups.Count == 0 ? blueprint.ReviewFollowups : family.ReviewFollowups;
|
||||
var wrong = family.WrongAnswerFollowups.Count == 0 ? blueprint.WrongAnswerFollowups : family.WrongAnswerFollowups;
|
||||
if (distractors.Count < 6) return $"{family.Id} requires at least six distractors";
|
||||
var answers = distractors.Append(correctAnswer).ToList();
|
||||
if (answers.Any(value => string.IsNullOrWhiteSpace(value.Id)
|
||||
|| string.IsNullOrWhiteSpace(value.TextTemplate)
|
||||
|| string.IsNullOrWhiteSpace(value.ExplanationTemplate)))
|
||||
return $"{family.Id} every answer requires id, text, and explanation";
|
||||
if (answers.Select(value => value.Id).Distinct(StringComparer.Ordinal).Count() != answers.Count)
|
||||
return $"{family.Id} answer ids must be unique";
|
||||
if (reviews.Count == 0) return $"{family.Id} review followup required";
|
||||
if (string.IsNullOrWhiteSpace(family.ExplanationTemplate) && string.IsNullOrWhiteSpace(blueprint.ExplanationTemplate)
|
||||
|| string.IsNullOrWhiteSpace(family.KeyTakeaway) && string.IsNullOrWhiteSpace(blueprint.KeyTakeaway)
|
||||
|| string.IsNullOrWhiteSpace(family.UsmlePearl) && string.IsNullOrWhiteSpace(blueprint.UsmlePearl))
|
||||
return $"{family.Id} complete teaching content required";
|
||||
}
|
||||
try
|
||||
{
|
||||
var generator = new DynamicQuestionGenerator();
|
||||
for (var index = 0; index < 300; index++)
|
||||
{
|
||||
var generated = generator.Generate(blueprint, $"validation-{index}", $"validation-{index}");
|
||||
if (generated.Question.ChoiceExplanations.Count != 5) return "every displayed choice requires an explanation";
|
||||
}
|
||||
}
|
||||
catch (Exception exception) { return exception.Message; }
|
||||
return null;
|
||||
}
|
||||
|
||||
private static string Render(string template, IReadOnlyDictionary<string, string> values)
|
||||
{
|
||||
var rendered = Placeholder.Replace(template ?? "", match =>
|
||||
|
||||
Reference in New Issue
Block a user