Restore distractor-specific follow-up questions

2026-08-01 18:06:52 -05:00
parent 8c5aa17d89
commit c7f5006cb6
14 changed files with 76770 additions and 100 deletions
@@ -223,21 +223,24 @@ UpsertQuestionRequest Build(Seed seed)
        id => id,
        id => id == "A"
            ? seed.Explanation
-            : $"This choice does not address the decisive finding: {seed.Discriminator}. {seed.Pitfall2}.");
-    var wrongFollowups = ids.Skip(1).ToDictionary(
-        id => id,
-        id => new WrongAnswerFollowup
+            : $"{choices.Single(choice => choice.Id == id).Text} does not address the decisive finding: {seed.Discriminator}. {seed.Pitfall2}.");
+    var distractorIds = ids.Skip(1).ToArray();
+    var wrongFollowups = distractorIds.ToDictionary(id => id, id =>
+    {
+        var index = Array.IndexOf(distractorIds, id);
+        var alternatives = new[] { distractorIds[(index + 1) % distractorIds.Length], distractorIds[(index + 2) % distractorIds.Length] };
+        return new WrongAnswerFollowup
        {
-            Prompt = $"In this same vignette, which finding most strongly favors {seed.Correct} over {choices.Single(c => c.Id == id).Text}?",
+            Prompt = $"Which limitation of option {id} should have ruled it out in this case?",
            Choices =
            [
-                new() { Id = "1", Text = seed.Discriminator },
-                new() { Id = "2", Text = seed.Pitfall1 },
-                new() { Id = "3", Text = "The patient's age by itself" }
+                new() { Id = "1", Text = explanations[id] },
+                new() { Id = "2", Text = explanations[alternatives[0]] },
+                new() { Id = "3", Text = explanations[alternatives[1]] }
            ],
-            CorrectChoiceId = "1",
-            TeachingPoint = seed.Takeaway
-        });
+            CorrectChoiceId = "1", TeachingPoint = explanations[id]
+        };
+    });

    return new UpsertQuestionRequest
    {
@@ -88,10 +88,6 @@ string? ValidateAuthoredQuestion(UpsertQuestionRequest question)
    {
        if (followup.Choices.Count != 3 || followup.Choices.Select(choice => choice.Id).SequenceEqual(["A", "B", "C"]) is false)
            return "wrong-answer follow-ups must use ordered A-C choices";
-        if (followup.Choices.Any(choice =>
-                !question.Stem.Contains(choice.Text, StringComparison.OrdinalIgnoreCase)
-                && !question.Labs.Any(lab => $"{lab.Name} {lab.Value}".Contains(choice.Text, StringComparison.OrdinalIgnoreCase))))
-            return "wrong-answer follow-up choices must be traceable to the stem or labs";
    }
    return null;
 }
@@ -107,20 +103,23 @@ UpsertQuestionRequest Build(Seed seed)
        id => id == "A"
            ? $"Correct. {seed.Explanation}"
            : $"Incorrect. {choices.Single(choice => choice.Id == id).Text} does not adequately address the decisive finding: {seed.KeyFact}. {seed.Takeaway}");
-    var wrongFollowups = ids.Skip(1).ToDictionary(
-        id => id,
-        id => new WrongAnswerFollowup
+    var distractorIds = ids.Skip(1).ToArray();
+    var wrongFollowups = distractorIds.ToDictionary(id => id, id =>
+    {
+        var index = Array.IndexOf(distractorIds, id);
+        var alternatives = new[] { distractorIds[(index + 1) % distractorIds.Length], distractorIds[(index + 2) % distractorIds.Length] };
+        return new WrongAnswerFollowup
        {
-            Prompt = $"Which finding in this vignette most strongly supports {seed.Answers[0]}?",
+            Prompt = $"Which limitation of option {id} should have ruled it out in this case?",
            Choices =
            [
-                new() { Id = "A", Text = seed.KeyFact },
-                new() { Id = "B", Text = seed.ContextFact },
-                new() { Id = "C", Text = seed.NeutralFact }
+                new() { Id = "A", Text = choiceExplanations[id] },
+                new() { Id = "B", Text = choiceExplanations[alternatives[0]] },
+                new() { Id = "C", Text = choiceExplanations[alternatives[1]] }
            ],
-            CorrectChoiceId = "A",
-            TeachingPoint = seed.Takeaway
-        });
+            CorrectChoiceId = "A", TeachingPoint = choiceExplanations[id]
+        };
+    });

    return new UpsertQuestionRequest
    {
@@ -61,11 +61,22 @@ UpsertQuestionRequest Build(Seed s)
    var ids = Enumerable.Range(0, s.Answers.Length).Select(i => ((char)('A' + i)).ToString()).ToArray();
    var choices = ids.Zip(s.Answers, (id, text) => new QuestionChoice { Id = id, Text = text }).ToList();
    var explanations = ids.ToDictionary(id => id, id => id == "A" ? $"Correct. {s.Rationale}" : $"Incorrect. {choices.Single(c => c.Id == id).Text} does not fit the decisive finding: {s.KeyFact}. {s.Takeaway}");
-    var wrong = ids.Skip(1).ToDictionary(id => id, _ => new WrongAnswerFollowup
+    var distractorIds = ids.Skip(1).ToArray();
+    var wrong = distractorIds.ToDictionary(id => id, id =>
    {
-        Prompt = $"Which finding most strongly supports {s.Answers[0]}?",
-        Choices = [new() { Id = "A", Text = s.KeyFact }, new() { Id = "B", Text = s.ContextFact }, new() { Id = "C", Text = s.NeutralFact }],
-        CorrectChoiceId = "A", TeachingPoint = s.Takeaway
+        var index = Array.IndexOf(distractorIds, id);
+        var alternatives = new[] { distractorIds[(index + 1) % distractorIds.Length], distractorIds[(index + 2) % distractorIds.Length] };
+        return new WrongAnswerFollowup
+        {
+            Prompt = $"Which limitation of option {id} should have ruled it out in this case?",
+            Choices =
+            [
+                new() { Id = "A", Text = explanations[id] },
+                new() { Id = "B", Text = explanations[alternatives[0]] },
+                new() { Id = "C", Text = explanations[alternatives[1]] }
+            ],
+            CorrectChoiceId = "A", TeachingPoint = explanations[id]
+        };
    });
    var reference = References.For(s.Reference);
    return new UpsertQuestionRequest
@@ -0,0 +1,172 @@
+using System.Security.Cryptography;
+using System.Text;
+using System.Text.Json;
+using MongoDB.Bson;
+using MongoDB.Bson.Serialization.Attributes;
+using MongoDB.Driver;
+using QuestionsApi.Models;
+
+const string defaultConnection = "mongodb://192.168.86.50:27017";
+var options = args.Select((value, index) => (value, index)).Where(x => x.value.StartsWith("--"))
+    .ToDictionary(x => x.value, x => x.index + 1 < args.Length && !args[x.index + 1].StartsWith("--") ? args[x.index + 1] : "true");
+var mode = options.GetValueOrDefault("--mode", "audit").ToLowerInvariant();
+var connection = options.GetValueOrDefault("--connection", Environment.GetEnvironmentVariable("STEP3_MONGODB_CONNECTION_STRING") ?? defaultConnection);
+var databases = options.GetValueOrDefault("--databases", "ranaze_step3_uat,ranaze_step3_prd").Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
+var manifestPath = options.GetValueOrDefault("--manifest", "content/repairs/wrong-answer-followups-20260801.json");
+var batchId = options.GetValueOrDefault("--batch", "wrong-answer-followups-20260801");
+var client = new MongoClient(connection);
+var json = new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase };
+
+if (mode is "audit" or "prepare")
+{
+    var inventory = new Dictionary<string, List<Question>>();
+    foreach (var database in databases)
+    {
+        var questions = await client.GetDatabase(database).GetCollection<Question>("Questions")
+            .Find(x => x.IsPublished).SortBy(x => x.ExternalId).ToListAsync();
+        inventory[database] = questions;
+        var broken = questions.Select(q => (Question: q, Issues: Issues(q))).Where(x => x.Issues.Count > 0).ToList();
+        Console.WriteLine($"{database}: published={questions.Count}, compliant={questions.Count - broken.Count}, repair={broken.Count}");
+        foreach (var item in broken) Console.WriteLine($"  {item.Question.ExternalId}: {string.Join("; ", item.Issues)}");
+    }
+    if (mode == "audit") return;
+
+    var entries = inventory.SelectMany(db => db.Value.Select(question => (Database: db.Key, Question: question)))
+        .Where(x => Issues(x.Question).Count > 0).GroupBy(x => x.Question.ExternalId, StringComparer.Ordinal).Select(group =>
+        {
+            var source = group.OrderBy(x => Array.IndexOf(databases, x.Database)).First().Question;
+            var replacement = Build(source);
+            var original = source.WrongAnswerFollowups;
+            source.WrongAnswerFollowups = replacement;
+            var replacementIssues = Issues(source);
+            source.WrongAnswerFollowups = original;
+            if (replacementIssues.Count > 0)
+                throw new InvalidOperationException($"generated repair for {source.ExternalId} is invalid: {string.Join("; ", replacementIssues)}");
+            return new RepairEntry
+            {
+                ExternalId = source.ExternalId,
+                ExpectedHashes = group.ToDictionary(x => x.Database, x => Hash(x.Question.WrongAnswerFollowups)),
+                Replacement = replacement
+            };
+        }).OrderBy(x => x.ExternalId).ToList();
+    var manifest = new RepairManifest { BatchId = batchId, CreatedUtc = DateTime.UtcNow, Entries = entries };
+    Directory.CreateDirectory(Path.GetDirectoryName(manifestPath) ?? ".");
+    await File.WriteAllTextAsync(manifestPath, JsonSerializer.Serialize(manifest, json));
+    Console.WriteLine($"Prepared {entries.Count} repair entries at {manifestPath}.");
+    return;
+}
+
+var loaded = JsonSerializer.Deserialize<RepairManifest>(await File.ReadAllTextAsync(manifestPath), json)
+    ?? throw new InvalidOperationException("repair manifest is invalid");
+if (loaded.BatchId != batchId) throw new InvalidOperationException("manifest batch does not match --batch");
+
+foreach (var database in databases)
+{
+    var db = client.GetDatabase(database);
+    var questions = db.GetCollection<Question>("Questions");
+    var backups = db.GetCollection<RepairBackup>("WrongAnswerFollowupRepairBackups");
+    await backups.Indexes.CreateOneAsync(new CreateIndexModel<RepairBackup>(
+        Builders<RepairBackup>.IndexKeys.Ascending(x => x.BatchId).Ascending(x => x.ExternalId), new CreateIndexOptions { Unique = true }));
+    var changed = 0;
+    foreach (var entry in loaded.Entries)
+    {
+        var question = await questions.Find(x => x.ExternalId == entry.ExternalId).FirstOrDefaultAsync();
+        if (question is null || !question.IsPublished || !entry.ExpectedHashes.TryGetValue(database, out var expected)) continue;
+        if (mode == "rollback")
+        {
+            var backup = await backups.Find(x => x.BatchId == batchId && x.ExternalId == entry.ExternalId).FirstOrDefaultAsync();
+            if (backup is null) continue;
+            await questions.UpdateOneAsync(x => x.Id == question.Id, Builders<Question>.Update
+                .Set(x => x.WrongAnswerFollowups, backup.Original).Set(x => x.UpdatedUtc, DateTime.UtcNow));
+            changed++;
+            continue;
+        }
+        if (mode != "apply") throw new ArgumentException("--mode must be audit, prepare, apply, or rollback");
+        var actual = Hash(question.WrongAnswerFollowups);
+        var replacementHash = Hash(entry.Replacement);
+        if (actual == replacementHash) continue;
+        if (actual != expected) throw new InvalidOperationException($"{database}/{entry.ExternalId} changed after audit; expected {expected}, found {actual}");
+        try
+        {
+            await backups.InsertOneAsync(new RepairBackup { Id = ObjectId.GenerateNewId().ToString(), BatchId = batchId,
+                ExternalId = entry.ExternalId, OriginalHash = actual, ReplacementHash = replacementHash,
+                Original = question.WrongAnswerFollowups, CreatedUtc = DateTime.UtcNow });
+        }
+        catch (MongoWriteException exception) when (exception.WriteError?.Category == ServerErrorCategory.DuplicateKey) { }
+        var update = await questions.UpdateOneAsync(x => x.Id == question.Id && x.UpdatedUtc == question.UpdatedUtc,
+            Builders<Question>.Update.Set(x => x.WrongAnswerFollowups, entry.Replacement).Set(x => x.UpdatedUtc, DateTime.UtcNow));
+        if (update.ModifiedCount != 1) throw new InvalidOperationException($"{database}/{entry.ExternalId} changed during repair");
+        changed++;
+    }
+    Console.WriteLine($"{database}: {mode} changed {changed} question(s).");
+}
+
+static List<string> Issues(Question question)
+{
+    var ids = question.Choices.Select(x => x.Id).Where(x => x != question.CorrectChoiceId).ToHashSet(StringComparer.Ordinal);
+    var issues = new List<string>();
+    if (!ids.SetEquals(question.WrongAnswerFollowups.Keys)) issues.Add("follow-up keys do not match distractors");
+    var values = question.WrongAnswerFollowups.Where(x => ids.Contains(x.Key)).Select(x => x.Value).ToList();
+    if (values.Any(x => x.Choices.Count != 3)) issues.Add("follow-up does not have exactly three choices");
+    Duplicate(values.Select(x => x.Prompt), "duplicate prompts", issues);
+    Duplicate(values.Select(x => x.TeachingPoint), "duplicate teaching points", issues);
+    Duplicate(values.Select(x => x.Choices.FirstOrDefault(c => c.Id == x.CorrectChoiceId)?.Text ?? ""), "duplicate correct follow-up answers", issues);
+    Duplicate(values.Select(x => string.Join("|", x.Choices.Select(c => Normalize(c.Text)).Order())), "duplicate choice sets", issues);
+    if (values.Any(x => x.Choices.Count == 0 || x.Choices.All(c => c.Id != x.CorrectChoiceId))) issues.Add("invalid correct follow-up choice");
+    return issues.Distinct().ToList();
+}
+
+static void Duplicate(IEnumerable<string> values, string issue, List<string> issues)
+{
+    var normalized = values.Select(Normalize).ToList();
+    if (normalized.Any(string.IsNullOrWhiteSpace) || normalized.Distinct(StringComparer.Ordinal).Count() != normalized.Count) issues.Add(issue);
+}
+
+static Dictionary<string, WrongAnswerFollowup> Build(Question question)
+{
+    var distractors = question.Choices.Where(x => x.Id != question.CorrectChoiceId).ToList();
+    var rationales = distractors.ToDictionary(x => x.Id, x => Rationale(question, x.Id));
+    return distractors.ToDictionary(choice => choice.Id, choice =>
+    {
+        var alternatives = distractors.Where(x => x.Id != choice.Id).Take(2).ToArray();
+        return new WrongAnswerFollowup
+        {
+            Prompt = $"Which feature of option {choice.Id} should have ruled it out in this case?",
+            Choices =
+            [
+                new() { Id = "A", Text = rationales[choice.Id] },
+                new() { Id = "B", Text = Unique(rationales[alternatives[0].Id], alternatives[0].Id) },
+                new() { Id = "C", Text = Unique(rationales[alternatives[1].Id], alternatives[1].Id) }
+            ],
+            CorrectChoiceId = "A", TeachingPoint = rationales[choice.Id]
+        };
+    });
+}
+
+static string Rationale(Question question, string id)
+{
+    var value = question.ChoiceExplanations.GetValueOrDefault(id, question.WrongAnswerFollowups.GetValueOrDefault(id)?.TeachingPoint ?? "").Trim();
+    foreach (var prefix in new[] { "Incorrect.", "Incorrect:", "Wrong.", "Wrong:" })
+        if (value.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)) value = value[prefix.Length..].Trim();
+    var choice = question.Choices.FirstOrDefault(x => x.Id == id)?.Text.Trim() ?? $"Option {id}";
+    if (string.IsNullOrWhiteSpace(value)) return $"{choice} does not fit the decisive clinical finding.";
+    return value.Contains(choice, StringComparison.OrdinalIgnoreCase) ? value : $"{choice} is inappropriate because {char.ToLowerInvariant(value[0])}{value[1..]}";
+}
+
+static string Unique(string value, string id) => $"Option {id}: {value}";
+static string Normalize(string value) => string.Join(' ', (value ?? "").Trim().ToLowerInvariant().Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries));
+static string Hash(Dictionary<string, WrongAnswerFollowup> value)
+{
+    var canonical = value.OrderBy(x => x.Key).ToDictionary(x => x.Key, x => x.Value);
+    return Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(JsonSerializer.Serialize(canonical)))).ToLowerInvariant();
+}
+
+sealed class RepairManifest { public string BatchId { get; set; } = ""; public DateTime CreatedUtc { get; set; } public List<RepairEntry> Entries { get; set; } = []; }
+sealed class RepairEntry { public string ExternalId { get; set; } = ""; public Dictionary<string, string> ExpectedHashes { get; set; } = []; public Dictionary<string, WrongAnswerFollowup> Replacement { get; set; } = []; }
+sealed class RepairBackup
+{
+    [BsonId, BsonRepresentation(BsonType.ObjectId)] public string? Id { get; set; }
+    public string BatchId { get; set; } = ""; public string ExternalId { get; set; } = "";
+    public string OriginalHash { get; set; } = ""; public string ReplacementHash { get; set; } = "";
+    public Dictionary<string, WrongAnswerFollowup> Original { get; set; } = []; public DateTime CreatedUtc { get; set; }
+}
@@ -0,0 +1,4 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup><OutputType>Exe</OutputType><TargetFramework>net8.0</TargetFramework><ImplicitUsings>enable</ImplicitUsings><Nullable>enable</Nullable></PropertyGroup>
+  <ItemGroup><ProjectReference Include="..\..\api\QuestionsApi\QuestionsApi.csproj" /></ItemGroup>
+</Project>