From 93154d592508f7778d6ce12ecbcb215a19cf9481 Mon Sep 17 00:00:00 2001 From: hz Date: Mon, 13 Jul 2026 10:15:05 -0500 Subject: [PATCH] Add variable question distractor pools --- QBANK_PDF_IMPORT_RUNBOOK.md | 342 ++++++++++++++++++ README.md | 22 ++ api/QuestionsApi.Tests/ChoicePoolTests.cs | 104 ++++++ .../QuestionsApi.Tests.csproj | 14 + .../Controllers/QuestionsController.cs | 105 +++--- api/QuestionsApi/Models/AnswerStatistics.cs | 20 +- .../Models/QuestionAdminExport.cs | 8 + api/QuestionsApi/Models/QuestionAttempt.cs | 2 + api/QuestionsApi/Models/QuestionRequests.cs | 2 + api/QuestionsApi/Models/QuestionResponse.cs | 70 +++- api/QuestionsApi/Program.cs | 6 +- api/QuestionsApi/README.md | 22 +- .../AnswerStatisticsMaintenanceService.cs | 7 + .../Services/AnswerStatisticsStore.cs | 282 +++++++++------ .../Services/QuestionAttemptStore.cs | 48 +-- api/QuestionsApi/Services/QuestionStore.cs | 26 ++ .../Services/QuestionValidator.cs | 57 +++ api/QuestionsApi/Services/ServiceMetrics.cs | 6 + scripts/autoload/block_session_service.gd | 32 +- scripts/autoload/question_bank_service.gd | 14 +- scripts/ui/question_player.gd | 23 +- step3.sln | 15 + tests/choice_pool_test.gd | 59 +++ tests/choice_pool_test.gd.uid | 1 + 24 files changed, 1040 insertions(+), 247 deletions(-) create mode 100644 QBANK_PDF_IMPORT_RUNBOOK.md create mode 100644 api/QuestionsApi.Tests/ChoicePoolTests.cs create mode 100644 api/QuestionsApi.Tests/QuestionsApi.Tests.csproj create mode 100644 api/QuestionsApi/Models/QuestionAdminExport.cs create mode 100644 api/QuestionsApi/Services/QuestionValidator.cs create mode 100644 tests/choice_pool_test.gd create mode 100644 tests/choice_pool_test.gd.uid diff --git a/QBANK_PDF_IMPORT_RUNBOOK.md b/QBANK_PDF_IMPORT_RUNBOOK.md new file mode 100644 index 0000000..9aa1432 --- /dev/null +++ b/QBANK_PDF_IMPORT_RUNBOOK.md @@ -0,0 +1,342 @@ +# Question Bank PDF Import Runbook + +This runbook describes how to turn an image-based board-review PDF into +original Step 3-style questions, verify them, insert them through the Questions +API, and confirm the production result. It records the workflow used for the +July 2026 NBME 5 import and strengthens it with the repository's current +reference, originality, image, and duplicate-review requirements. + +The source material is used only to identify tested concepts, reasoning paths, +and plausible distractors. Do not copy source wording, vignette structure, +distinctive fact combinations, answer wording, or images. + +## July 2026 Result + +- 221 image-only PDF pages were rendered and OCRed. +- Page markers were mapped to 176 source questions across five sections. +- Questions were generated in batches of four. +- The first 72 questions passed the initial hosted-model workflow. +- Of the remaining 104, an independent audit approved 95 and rejected 9. +- The 9 rejected questions were regenerated from scratch with the audit issues + supplied, then all 9 passed a second independent audit. +- The bulk API inserted 104 remaining records, bringing the imported set to + 176 records. +- Direct database verification found 0 missing external IDs and 0 answer-key + mismatches. + +Known gap in that historical run: model-based medical review was completed, but +authoritative references were not added to `QUESTION_REFERENCE_REVIEWS.md` for +the `nbme5-*` records. Future runs must complete the reference-review gate below +before setting `isPublished` to `true`. The historical records should be treated +as awaiting that separate reference review. + +## Prerequisites + +- Python 3 with `PyMuPDF`, `Pillow`, and `pytesseract`. +- A local Tesseract OCR installation available on `PATH`. +- PowerShell, `curl`, the .NET SDK, and access to this repository. +- Access to a capable generation model and a separate audit model. +- A `SUPER` JWT for the target Questions API. +- Optional direct MongoDB read access for final production verification. + +Keep secrets out of scripts, prompts, logs, and the repository. Configure them +in the current process: + +```powershell +$env:QUESTIONS_API_BASE = "https://example.invalid" +$env:QUESTIONS_SUPER_JWT = "" +$env:QUESTIONS_MONGODB = "" +``` + +Create all intermediate files outside the repository: + +```powershell +$RunId = Get-Date -Format "yyyyMMdd-HHmmss" +$WorkDir = Join-Path $env:TEMP "step3-qbank-import-$RunId" +New-Item -ItemType Directory -Path $WorkDir | Out-Null +"ocr", "mapping", "prompts", "generated", "audits", "repairs" | + ForEach-Object { New-Item -ItemType Directory -Path (Join-Path $WorkDir $_) | Out-Null } +``` + +Suggested layout: + +```text +step3-qbank-import-/ + ocr/pages.jsonl + mapping/items.jsonl + prompts/ + generated/ + audits/ + repairs/ + validation.json + references.json + insert-payload.json +``` + +## 1. Establish The Import Set + +1. Hash and record the source PDF before processing. +2. Record the expected source, section, and question counts if known. +3. Query existing external IDs before generation. Never assume the database is + empty, and never overwrite an existing ID unless that update is intentional. +4. Choose a deterministic ID prefix, for example + `nbme5-s3-i007-`. +5. Keep the original PDF unchanged and outside generated-output directories. + +The bulk endpoint upserts by `externalId`. Deterministic IDs make retries +idempotent, but they also make accidental overwrites possible. Compare the +planned IDs with production before insertion. + +## 2. Render And OCR The PDF + +For an image-only PDF, render each page at approximately 250-300 DPI and run +Tesseract on the rendered bitmap. A minimal Python outline is: + +```powershell +py -m pip install pymupdf pillow pytesseract +``` + +```python +import io, json, sys, fitz, pytesseract +from PIL import Image + +source_pdf = sys.argv[1] +pages_jsonl = sys.argv[2] +document = fitz.open(source_pdf) +with open(pages_jsonl, "w", encoding="utf-8") as output: + for page_number, page in enumerate(document, start=1): + pixmap = page.get_pixmap(matrix=fitz.Matrix(3, 3), alpha=False) + image = Image.open(io.BytesIO(pixmap.tobytes("png"))) + text = pytesseract.image_to_string(image, config="--psm 6") + output.write(json.dumps({ + "page": page_number, + "text": text, + }, ensure_ascii=False) + "\n") +``` + +Save the snippet as `ocr_pdf.py` in the working directory and run: + +```powershell +py "$WorkDir\ocr_pdf.py" "C:\path\source.pdf" "$WorkDir\ocr\pages.jsonl" +``` + +Do not silently discard low-confidence or empty pages. Save page numbers and OCR +text so every generated question remains traceable to its source pages during +review. + +## 3. Map Pages To Source Questions + +Parse repeated headers such as `Exam Section N: Item M of K`. Group all pages +for the same section/item pair and append continuation pages to the preceding +item when necessary. + +Each `items.jsonl` record should contain: + +```json +{"section":3,"item":7,"pages":[95],"sourceText":"..."} +``` + +Before generation, verify: + +- every detected section/item pair is unique; +- the item sequence has no unexplained gaps; +- multipage stems and answer choices are complete; +- the total mapped-question count matches a manual review; +- OCR did not convert option letters, decimal points, inequality signs, units, + or answer-choice text into unusable content; +- image-dependent questions are identified for special handling. + +The July 2026 source had 221 pages but only 176 questions. Page count is not a +question-count proxy. + +## 4. Generate Original Questions + +Generate small batches, typically four source items per request. Small batches +limit loss when a model call fails and make outputs easier to review. + +The generation prompt must require: + +- a substantially original clinical scenario testing only the same concept; +- changes to age, sex when clinically possible, setting, chronology, numeric + values, sentence sequence, and answer wording; +- one defensible best answer and a variable pool of plausible choices in an + array of `{id, text}` objects, not a map keyed by letters; +- a stem containing only the vignette and a separate `leadIn`; +- `choiceExplanations` with exactly one entry for every main choice; +- `wrongAnswerFollowups` for every incorrect main answer; +- three A-C follow-up choices, all of which restate true facts explicitly found + in the new stem or displayed labs; +- Celsius and Fahrenheit together for every temperature; +- labs only when useful and only when their values also appear in the stem; +- `media: []` unless separately reviewed, licensed media will be added later; +- selection percentages for every authored choice initialized to zero; +- established Step categories, 3-6 tags, and a unique deterministic ID. + +For image-dependent source items, rewrite the concept as a text-based question +when that can be done without changing the learning objective. Otherwise stop +and perform the image copyright workflow in `README.md`; do not import the +source image. + +Run a four-question gate before scaling out. Inspect it manually for +originality, ambiguity, schema shape, medical accuracy, and stem-anchored +follow-ups. Reject the model or revise the prompt if this gate fails. + +## 5. Run Deterministic Validation + +Combine generated batches only after each parses as JSON. The validator should +fail the run for any of the following: + +- missing, duplicate, or incorrectly prefixed external IDs; +- a count different from the mapped source count; +- fewer than five main choices, more than twenty, duplicate choice IDs, or a + main choice not represented as an `{id, text}` object; +- an invalid `correctChoiceId`; +- missing choice explanations or a correct explanation that does not begin + with `Correct.`; +- follow-up keys that do not exactly equal the incorrect main-answer IDs; +- follow-up choices not represented by an ordered A-C array; +- follow-up text that cannot be traced to the new stem or labs; +- nonzero initial selection percentages; +- invalid difficulty, category, tag count, publication state, or media shape; +- a Celsius temperature without its Fahrenheit equivalent; +- labs referenced by explanations or follow-ups but absent from the stem; +- suspiciously long contiguous overlap with source wording. + +Treat automated overlap and anchoring warnings as review queues, not automatic +approval or rejection. Record the disposition of every warning. + +## 6. Perform Independent Content Review + +Use a different model or reviewer from the generator. Provide the OCR source +and generated question, and request a machine-readable report: + +```json +{"externalId":"...","approved":false,"issues":["concrete issue"]} +``` + +The reviewer must check: + +1. The new item tests the same core reasoning path. +2. It is structurally and textually original. +3. The keyed answer is correct under current practice. +4. No distractor is reasonably co-correct. +5. All explanations are accurate. +6. Every follow-up choice is an explicit fact from the generated stem or labs. +7. The vignette is internally consistent and sufficient to answer. + +Regenerate rejected items from scratch with the audit issues included. Do not +patch only the reported sentence when the underlying scenario is ambiguous. +Audit every repaired item again. Merge a repair only after it receives an +independent approval with an empty issue list. + +## 7. Complete Required Reference And Duplicate Reviews + +Model agreement is not authoritative medical verification. Before publication: + +1. Verify each keyed answer and material explanatory claim against current, + direct, authoritative sources such as professional guidelines, government + agencies, official labeling, or primary research. +2. Add an entry for every question to `QUESTION_REFERENCE_REVIEWS.md`, including + title, publisher, direct URL, supported claim, access date, and reviewer. +3. Search the entire question bank for exact and near-duplicate learning tasks, + not just duplicate text or IDs. +4. Record duplicate-review dispositions and substantially differentiate or + remove near-duplicates. +5. Complete the image licensing review for every non-empty `media` field. + +Keep new questions unpublished until these gates are complete. If generation +produces `isPublished: true`, change it to `false` before insertion and publish +only after the reviews are recorded. + +## 8. Build And Submit The Payload + +The API expects this wrapper: + +```json +{"questions":[{...},{...}]} +``` + +Submit the validated file exactly as UTF-8 bytes: + +```powershell +curl.exe --fail-with-body ` + -X POST "$env:QUESTIONS_API_BASE/api/questions/bulk" ` + -H "Authorization: Bearer $env:QUESTIONS_SUPER_JWT" ` + -H "Content-Type: application/json; charset=utf-8" ` + --data-binary "@${WorkDir}\insert-payload.json" +``` + +Expect `{"upserted":N}` with HTTP 200. PowerShell 5.1 can transmit a string body +with an unexpected encoding; prefer `curl --data-binary` or pass explicit UTF-8 +bytes to `Invoke-RestMethod`. + +Do not insert partial batches merely because generation is expensive. Complete +the validation, review, repair, reference, and duplicate gates first. + +## 9. Verify Production + +Verification is complete only when all of these pass: + +- the API reports the expected upsert count; +- the database contains every expected external ID exactly once; +- every stored `correctChoiceId` matches the final payload; +- a targeted deployed-API search returns at least one newly inserted question; +- the returned question has the expected choices, answer key, and publication + state; +- repository state is unchanged unless documentation or tooling was + intentionally committed. + +Do not use an unfiltered search to prove completeness. The API limits search +results and randomizes the returned order. Compare IDs directly through a +read-only database query or a dedicated administrative verification tool, then +use a highly specific tag combination for the deployed-API read check. + +The final verification report should include at least: + +```text +EXPECTED= +STORED= +MISSING= +DUPLICATES= +ANSWER_MISMATCHES= +API_READ_RESULTS= +``` + +## 10. Retain Reviews, Then Clean Up + +Retain the reference ledger, duplicate-review record, validation summary, audit +decisions, and final ID/answer-key manifest according to project policy. These +are review records, not disposable generation scratch files. + +After production verification: + +1. Delete rendered page images, raw model logs, rejected generations, temporary + prompts, local test projects, and disposable payload copies. +2. Preserve or securely dispose of the original PDF according to its license + and organizational policy. +3. Remove large local models installed only for the import. +4. Confirm the repository is clean with `git status --short --branch`. +5. Commit and push only maintained documentation, tooling, and review records. + +Use explicit, workspace-contained paths for cleanup. Resolve each path before a +recursive deletion and never delete the repository or source PDF as part of a +generated-artifact cleanup. + +## Acceptance Checklist + +- [ ] Source PDF hash and import scope recorded. +- [ ] OCR page count and mapped-question count reconciled. +- [ ] Every source item mapped exactly once. +- [ ] Four-question generation gate passed. +- [ ] Full schema and ID validation passed with zero errors. +- [ ] All anchoring and source-overlap warnings reviewed. +- [ ] Independent content audit approved every question. +- [ ] Every rejected question regenerated and re-audited. +- [ ] Authoritative references recorded for every question. +- [ ] Full-bank duplicate review completed. +- [ ] Image rights cleared for every media-bearing question. +- [ ] Final payload contains only intended new or intentionally updated IDs. +- [ ] Bulk API returned the expected upsert count. +- [ ] Database has zero missing IDs and answer-key mismatches. +- [ ] Deployed API read check passed. +- [ ] Required review records retained and scratch files removed. diff --git a/README.md b/README.md index f208459..6d8b06c 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,12 @@ The Godot client reads this from `project.godot`: step3/questions_api_url ``` +## Question Bank PDF Imports + +Use [QBANK_PDF_IMPORT_RUNBOOK.md](QBANK_PDF_IMPORT_RUNBOOK.md) for the complete +OCR, original-authoring, validation, independent-review, reference-review, bulk +insertion, production-verification, and cleanup workflow. + ## Question Image Copyright Review All questions containing images must receive a copyright and licensing review @@ -102,6 +108,22 @@ copyright requirements still apply. If the question schema does not expose a reference field, retain this information in the associated authoring or review record until it can be stored with the question. +## Variable Distractor Pools + +Published questions may contain more than the five choices shown to a learner. +The player always presents the correct answer plus four randomly selected +distractors and keeps that set stable for the duration of the active session. +Choice IDs are stable content identifiers; the A-E labels shown in the player +are based on presentation order and must not be stored as answer analytics. + +Add only distractors that are plausible for the exact vignette while remaining +unambiguously incorrect. Every added distractor requires its own explanation, +stem-anchored wrong-answer follow-up, independent accuracy review, and +authoritative reference coverage. There is no required number of additional +distractors: record an exception instead of forcing a weak or co-correct +answer. Do not expand production pools until the v2 exposure-statistics +backfill metric reports completion. + ## Duplicate Question Review Every question must be checked for duplicates before publication. This review diff --git a/api/QuestionsApi.Tests/ChoicePoolTests.cs b/api/QuestionsApi.Tests/ChoicePoolTests.cs new file mode 100644 index 0000000..8c0b60c --- /dev/null +++ b/api/QuestionsApi.Tests/ChoicePoolTests.cs @@ -0,0 +1,104 @@ +using QuestionsApi.Models; +using QuestionsApi.Services; +using Xunit; + +namespace QuestionsApi.Tests; + +public sealed class ChoicePoolTests +{ + [Fact] + public void LegacyResponseContainsFiveChoicesAndAlwaysIncludesCorrectAnswer() + { + var question = BuildQuestion(9); + + for (var iteration = 0; iteration < 50; iteration++) + { + var response = QuestionResponse.FromModel(question, includeChoicePool: false); + Assert.Equal(5, response.Choices.Count); + Assert.Equal(5, response.Choices.Select(choice => choice.Id).Distinct().Count()); + Assert.Contains(response.Choices, choice => choice.Id == question.CorrectChoiceId); + Assert.All(response.ChoiceExplanations.Keys, id => Assert.Contains(response.Choices, choice => choice.Id == id)); + Assert.All(response.WrongAnswerFollowups.Keys, id => Assert.Contains(response.Choices, choice => choice.Id == id)); + } + } + + [Fact] + public void PoolCapableResponseContainsCompletePool() + { + var question = BuildQuestion(9); + var response = QuestionResponse.FromModel(question, includeChoicePool: true); + Assert.Equal(9, response.Choices.Count); + } + + [Fact] + public void PublishedQuestionRequiresCompletePerChoiceTeachingContent() + { + var request = BuildRequest(6); + Assert.Null(QuestionValidator.Validate(request)); + + request.WrongAnswerFollowups.Remove("F"); + Assert.Contains("every incorrect choice", QuestionValidator.Validate(request)); + } + + [Fact] + public void PublishedQuestionRequiresAtLeastFiveUniqueChoices() + { + var request = BuildRequest(4); + Assert.Contains("at least five", QuestionValidator.Validate(request)); + + request = BuildRequest(5); + request.Choices[4].Id = "A"; + Assert.Contains("unique", QuestionValidator.Validate(request)); + } + + private static Question BuildQuestion(int count) + { + var request = BuildRequest(count); + return new Question + { + ExternalId = request.ExternalId, + Stem = request.Stem, + LeadIn = request.LeadIn, + Choices = request.Choices, + CorrectChoiceId = request.CorrectChoiceId, + Explanation = request.Explanation, + ChoiceExplanations = request.ChoiceExplanations, + WrongAnswerFollowups = request.WrongAnswerFollowups, + ChoiceSelectionPercentages = request.ChoiceSelectionPercentages + }; + } + + private static UpsertQuestionRequest BuildRequest(int count) + { + var ids = Enumerable.Range(0, count).Select(index => ((char)('A' + index)).ToString()).ToList(); + var choices = ids.Select(id => new QuestionChoice { Id = id, Text = $"Choice {id}" }).ToList(); + return new UpsertQuestionRequest + { + ExternalId = "choice-pool-test", + Stem = "A complete clinical vignette.", + LeadIn = "Which option is most appropriate?", + Choices = choices, + CorrectChoiceId = "A", + Explanation = "Explanation", + ChoiceExplanations = ids.ToDictionary(id => id, id => id == "A" ? "Correct." : "Incorrect."), + WrongAnswerFollowups = ids.Where(id => id != "A").ToDictionary( + id => id, + id => new WrongAnswerFollowup + { + Prompt = "Which finding makes this less appropriate?", + Choices = + [ + new QuestionChoice { Id = "A", Text = "Finding one" }, + new QuestionChoice { Id = "B", Text = "Finding two" } + ], + CorrectChoiceId = "A", + TeachingPoint = "Teaching point" + }), + ChoiceSelectionPercentages = ids.ToDictionary(id => id, _ => 0), + Discipline = "Medicine", + System = "Cardiovascular System", + Difficulty = 2, + IsPublished = true + }; + } +} diff --git a/api/QuestionsApi.Tests/QuestionsApi.Tests.csproj b/api/QuestionsApi.Tests/QuestionsApi.Tests.csproj new file mode 100644 index 0000000..998bf9a --- /dev/null +++ b/api/QuestionsApi.Tests/QuestionsApi.Tests.csproj @@ -0,0 +1,14 @@ + + + net8.0 + enable + enable + false + + + + + + + + diff --git a/api/QuestionsApi/Controllers/QuestionsController.cs b/api/QuestionsApi/Controllers/QuestionsController.cs index 4e7bfa9..8310e62 100644 --- a/api/QuestionsApi/Controllers/QuestionsController.cs +++ b/api/QuestionsApi/Controllers/QuestionsController.cs @@ -52,10 +52,34 @@ public class QuestionsController : ControllerBase return Ok(questions.Select(q => { percentages.TryGetValue(q.ExternalId, out var questionPercentages); - return QuestionResponse.FromModel(q, questionPercentages, questionElos.GetValueOrDefault(q.ExternalId, 1000)); + return QuestionResponse.FromModel( + q, + questionPercentages, + questionElos.GetValueOrDefault(q.ExternalId, 1000), + req.SupportsChoicePools); }).ToList()); } + [HttpGet("admin/export")] + [Authorize(Roles = "SUPER")] + public async Task AdminExport([FromQuery] string? after = null, [FromQuery] int limit = 100) + { + try + { + var page = await _questions.ExportPageAsync(after, limit); + return Ok(new QuestionAdminExportResponse + { + Items = page.Items.Select(question => QuestionResponse.FromModel(question)).ToList(), + NextCursor = page.NextCursor, + HasMore = page.HasMore + }); + } + catch (ArgumentException exception) + { + return BadRequest(exception.Message); + } + } + [HttpGet("{id}")] [Authorize(Roles = "USER,SUPER")] public async Task Get(string id) @@ -84,6 +108,22 @@ public class QuestionsController : ControllerBase if (!question.Choices.Any(c => c.Id == req.ChoiceId.Trim())) return BadRequest("choiceId must match a choice id"); + var presentedChoiceIds = req.PresentedChoiceIds + .Where(id => !string.IsNullOrWhiteSpace(id)) + .Select(id => id.Trim()) + .ToList(); + if (presentedChoiceIds.Count > 0) + { + var validChoiceIds = question.Choices.Select(choice => choice.Id).ToHashSet(StringComparer.Ordinal); + if (presentedChoiceIds.Count != 5 || presentedChoiceIds.Distinct(StringComparer.Ordinal).Count() != 5) + return BadRequest("presentedChoiceIds must contain exactly five unique choice ids"); + if (presentedChoiceIds.Any(id => !validChoiceIds.Contains(id))) + return BadRequest("presentedChoiceIds must match question choice ids"); + if (!presentedChoiceIds.Contains(req.ChoiceId.Trim(), StringComparer.Ordinal) + || !presentedChoiceIds.Contains(question.CorrectChoiceId, StringComparer.Ordinal)) + return BadRequest("presentedChoiceIds must include the selected and correct choices"); + } + var userId = User.FindFirstValue(ClaimTypes.NameIdentifier) ?? User.FindFirstValue(ClaimTypes.Name) ?? User.Identity?.Name @@ -95,7 +135,9 @@ public class QuestionsController : ControllerBase question.ExternalId, userId, req.ChoiceId.Trim(), - req.ChoiceId.Trim() == question.CorrectChoiceId); + req.ChoiceId.Trim() == question.CorrectChoiceId, + false, + presentedChoiceIds); if (record.Inserted) { _metrics.AttemptCreated(); @@ -252,7 +294,7 @@ public class QuestionsController : ControllerBase [Authorize(Roles = "SUPER")] public async Task Create([FromBody] UpsertQuestionRequest req) { - var validation = ValidateQuestion(req); + var validation = QuestionValidator.Validate(req); if (validation is not null) return BadRequest(validation); @@ -272,7 +314,7 @@ public class QuestionsController : ControllerBase foreach (var question in req.Questions) { - var validation = ValidateQuestion(question); + var validation = QuestionValidator.Validate(question); if (validation is not null) return BadRequest($"{question.ExternalId}: {validation}"); } @@ -285,7 +327,7 @@ public class QuestionsController : ControllerBase [Authorize(Roles = "SUPER")] public async Task Update(string id, [FromBody] UpsertQuestionRequest req) { - var validation = ValidateQuestion(req); + var validation = QuestionValidator.Validate(req); if (validation is not null) return BadRequest(validation); @@ -314,59 +356,6 @@ public class QuestionsController : ControllerBase return Ok("Deleted"); } - private static string? ValidateQuestion(UpsertQuestionRequest req) - { - if (string.IsNullOrWhiteSpace(req.ExternalId)) - return "externalId required"; - if (string.IsNullOrWhiteSpace(req.Stem)) - return "stem required"; - if (string.IsNullOrWhiteSpace(req.LeadIn)) - return "leadIn required"; - if (req.Choices.Count < 2) - return "at least two choices required"; - if (string.IsNullOrWhiteSpace(req.CorrectChoiceId)) - return "correctChoiceId required"; - if (!req.Choices.Any(c => c.Id == req.CorrectChoiceId)) - return "correctChoiceId must match a choice id"; - if (string.IsNullOrWhiteSpace(req.Explanation)) - return "explanation required"; - if (req.ChoiceSelectionPercentages.Any(entry => entry.Value is < 0 or > 100)) - return "choiceSelectionPercentages values must be between 0 and 100"; - var followupValidation = ValidateWrongAnswerFollowups(req); - if (followupValidation is not null) - return followupValidation; - if (string.IsNullOrWhiteSpace(req.System)) - return "system required"; - if (string.IsNullOrWhiteSpace(req.Discipline)) - return "discipline required"; - if (req.Difficulty is < 1 or > 5) - return "difficulty must be between 1 and 5"; - - return null; - } - - private static string? ValidateWrongAnswerFollowups(UpsertQuestionRequest req) - { - var choiceIds = req.Choices.Select(c => c.Id).Where(id => !string.IsNullOrWhiteSpace(id)).ToHashSet(); - foreach (var entry in req.WrongAnswerFollowups) - { - if (!choiceIds.Contains(entry.Key)) - return "wrongAnswerFollowups keys must match a choice id"; - if (entry.Key == req.CorrectChoiceId) - return "wrongAnswerFollowups should only target incorrect choices"; - if (string.IsNullOrWhiteSpace(entry.Value.Prompt)) - return "wrongAnswerFollowups prompt required"; - if (string.IsNullOrWhiteSpace(entry.Value.TeachingPoint)) - return "wrongAnswerFollowups teachingPoint required"; - if (entry.Value.Choices.Count == 0) - return "wrongAnswerFollowups choices required"; - if (!entry.Value.Choices.Any(c => c.Id == entry.Value.CorrectChoiceId)) - return "wrongAnswerFollowups correctChoiceId must match a follow-up choice id"; - } - - return null; - } - private string CurrentUserId() => User.FindFirstValue(ClaimTypes.NameIdentifier) ?? User.FindFirstValue(ClaimTypes.Name) diff --git a/api/QuestionsApi/Models/AnswerStatistics.cs b/api/QuestionsApi/Models/AnswerStatistics.cs index 0986416..1b54bf5 100644 --- a/api/QuestionsApi/Models/AnswerStatistics.cs +++ b/api/QuestionsApi/Models/AnswerStatistics.cs @@ -7,7 +7,8 @@ public sealed class QuestionAttemptCounter public string QuestionExternalId { get; set; } = ""; public int Bucket { get; set; } public string ChoiceId { get; set; } = ""; - public long Count { get; set; } + public long SelectionCount { get; set; } + public long PresentationCount { get; set; } public DateTime UpdatedUtc { get; set; } = DateTime.UtcNow; } @@ -15,14 +16,16 @@ public sealed class QuestionAttemptCounterTotal { [BsonId] public QuestionChoiceCounterKey Key { get; set; } = new(); - public long Count { get; set; } + public long SelectionCount { get; set; } + public long PresentationCount { get; set; } } public sealed class QuestionAttemptCounterAggregate { [BsonId] public QuestionChoiceBucketKey Key { get; set; } = new(); - public long Count { get; set; } + public long SelectionCount { get; set; } + public long PresentationCount { get; set; } } public sealed class QuestionChoiceBucketKey @@ -41,9 +44,18 @@ public sealed class QuestionChoiceCounterKey public sealed class AnswerStatisticsState { [BsonId] - public string Id { get; set; } = "answer-statistics-v1"; + public string Id { get; set; } = "answer-statistics-v2"; public bool Completed { get; set; } public string LeaseOwner { get; set; } = ""; public DateTime LeaseUntilUtc { get; set; } public DateTime UpdatedUtc { get; set; } = DateTime.UtcNow; + public DateTime CutoverUtc { get; set; } +} + +public sealed class QuestionPresentationBaseline +{ + [BsonId] + public string QuestionExternalId { get; set; } = ""; + public List ChoiceIds { get; set; } = []; + public DateTime CreatedUtc { get; set; } = DateTime.UtcNow; } diff --git a/api/QuestionsApi/Models/QuestionAdminExport.cs b/api/QuestionsApi/Models/QuestionAdminExport.cs new file mode 100644 index 0000000..163cd2b --- /dev/null +++ b/api/QuestionsApi/Models/QuestionAdminExport.cs @@ -0,0 +1,8 @@ +namespace QuestionsApi.Models; + +public sealed class QuestionAdminExportResponse +{ + public List Items { get; set; } = []; + public string? NextCursor { get; set; } + public bool HasMore { get; set; } +} diff --git a/api/QuestionsApi/Models/QuestionAttempt.cs b/api/QuestionsApi/Models/QuestionAttempt.cs index 29721a1..4657406 100644 --- a/api/QuestionsApi/Models/QuestionAttempt.cs +++ b/api/QuestionsApi/Models/QuestionAttempt.cs @@ -12,6 +12,8 @@ public class QuestionAttempt public string QuestionExternalId { get; set; } = ""; public string UserId { get; set; } = ""; public string ChoiceId { get; set; } = ""; + public List PresentedChoiceIds { get; set; } = []; + public bool TracksPresentation { get; set; } public bool IsCorrect { get; set; } public bool IsSkip { get; set; } public int CounterBucket { get; set; } diff --git a/api/QuestionsApi/Models/QuestionRequests.cs b/api/QuestionsApi/Models/QuestionRequests.cs index 6b1e6fa..2c9fb0b 100644 --- a/api/QuestionsApi/Models/QuestionRequests.cs +++ b/api/QuestionsApi/Models/QuestionRequests.cs @@ -8,6 +8,7 @@ public class QuestionSearchRequest public int? DifficultyMin { get; set; } public int? DifficultyMax { get; set; } public int Limit { get; set; } = 40; + public bool SupportsChoicePools { get; set; } } public class UpsertQuestionRequest @@ -35,4 +36,5 @@ public class FirstQuestionAttemptRequest { public string QuestionExternalId { get; set; } = ""; public string ChoiceId { get; set; } = ""; + public List PresentedChoiceIds { get; set; } = []; } diff --git a/api/QuestionsApi/Models/QuestionResponse.cs b/api/QuestionsApi/Models/QuestionResponse.cs index cf5d52a..ccf009e 100644 --- a/api/QuestionsApi/Models/QuestionResponse.cs +++ b/api/QuestionsApi/Models/QuestionResponse.cs @@ -12,6 +12,7 @@ public class QuestionResponse public Dictionary ChoiceExplanations { get; set; } = []; public Dictionary WrongAnswerFollowups { get; set; } = []; public Dictionary ChoiceSelectionPercentages { get; set; } = []; + public string ChoiceSelectionRateBasis { get; set; } = "presented"; public string Discipline { get; set; } = ""; public string System { get; set; } = ""; public string PerformanceCategory { get; set; } = ""; @@ -24,27 +25,56 @@ public class QuestionResponse public static QuestionResponse FromModel( Question question, Dictionary? liveChoiceSelectionPercentages = null, - int questionElo = 1000) => new() + int questionElo = 1000, + bool includeChoicePool = true) { - Id = question.Id ?? "", - ExternalId = question.ExternalId, - Stem = question.Stem, - LeadIn = question.LeadIn, - Choices = question.Choices, - CorrectChoiceId = question.CorrectChoiceId, - Explanation = question.Explanation, - ChoiceExplanations = question.ChoiceExplanations, - WrongAnswerFollowups = question.WrongAnswerFollowups, - ChoiceSelectionPercentages = BuildChoiceSelectionPercentages(question, liveChoiceSelectionPercentages), - Discipline = question.Discipline, - System = question.System, - PerformanceCategory = RatingCategory.Normalize(question.PerformanceCategory, question.System, question.Discipline), - Difficulty = question.Difficulty, - QuestionElo = questionElo, - Labs = question.Labs, - Media = question.Media, - Tags = question.Tags - }; + var choices = includeChoicePool ? question.Choices : SelectLegacyChoices(question); + var choiceIds = choices.Select(choice => choice.Id).ToHashSet(StringComparer.Ordinal); + var percentages = BuildChoiceSelectionPercentages(question, liveChoiceSelectionPercentages); + return new QuestionResponse + { + Id = question.Id ?? "", + ExternalId = question.ExternalId, + Stem = question.Stem, + LeadIn = question.LeadIn, + Choices = choices, + CorrectChoiceId = question.CorrectChoiceId, + Explanation = question.Explanation, + ChoiceExplanations = question.ChoiceExplanations + .Where(entry => choiceIds.Contains(entry.Key)) + .ToDictionary(), + WrongAnswerFollowups = question.WrongAnswerFollowups + .Where(entry => choiceIds.Contains(entry.Key)) + .ToDictionary(), + ChoiceSelectionPercentages = percentages + .Where(entry => choiceIds.Contains(entry.Key)) + .ToDictionary(), + Discipline = question.Discipline, + System = question.System, + PerformanceCategory = RatingCategory.Normalize(question.PerformanceCategory, question.System, question.Discipline), + Difficulty = question.Difficulty, + QuestionElo = questionElo, + Labs = question.Labs, + Media = question.Media, + Tags = question.Tags + }; + } + + private static List SelectLegacyChoices(Question question) + { + if (question.Choices.Count <= 5) + return question.Choices; + + var correct = question.Choices.First(choice => choice.Id == question.CorrectChoiceId); + var distractors = question.Choices + .Where(choice => choice.Id != question.CorrectChoiceId) + .OrderBy(_ => Random.Shared.Next()) + .Take(4) + .Append(correct) + .OrderBy(_ => Random.Shared.Next()) + .ToList(); + return distractors; + } public static Dictionary BuildChoiceSelectionPercentages(Question question, Dictionary? liveChoiceSelectionPercentages) { diff --git a/api/QuestionsApi/Program.cs b/api/QuestionsApi/Program.cs index d79f5b8..fa94c4b 100644 --- a/api/QuestionsApi/Program.cs +++ b/api/QuestionsApi/Program.cs @@ -53,8 +53,10 @@ builder.Services.AddAuthentication(JwtBearerDefaults.AuthenticationScheme) { o.TokenValidationParameters = new TokenValidationParameters { - ValidateIssuer = true, ValidIssuer = issuer, - ValidateAudience = true, ValidAudience = aud, + ValidateIssuer = true, + ValidIssuer = issuer, + ValidateAudience = true, + ValidAudience = aud, ValidateIssuerSigningKey = true, IssuerSigningKey = new SymmetricSecurityKey(Encoding.UTF8.GetBytes(jwtKey)), ValidateLifetime = true, diff --git a/api/QuestionsApi/README.md b/api/QuestionsApi/README.md index e197eae..e72a4a8 100644 --- a/api/QuestionsApi/README.md +++ b/api/QuestionsApi/README.md @@ -7,6 +7,7 @@ STEP-style question bank service for study games and clients. - `GET /healthz` Health check. - `GET /api/questions/systems` List available systems. - `POST /api/questions/search` Search or build a block of questions. +- `GET /api/questions/admin/export` Cursor-paginated full-bank export (SUPER only). - `GET /api/questions/{id}` Get one question. - `POST /api/questions` Create question (SUPER only). - `POST /api/questions/bulk` Upsert questions by `externalId` (SUPER only). @@ -22,10 +23,23 @@ Player reads require `USER` or `SUPER` JWT auth. Authoring endpoints require `SU ## Answer tracking The `QuestionAttempts` collection is the immutable source of truth and enforces -one first attempt per question and user with a unique compound index. Choice -percentages are served from 32-way materialized counter buckets and cached for -up to five minutes. A background service backfills counters on the first deploy -and reconciles recently active questions every five minutes. +one first attempt per question and user with a unique compound index. Updated +clients submit the five choice IDs that were presented. Choice rates are the +percentage of presentations that resulted in selection, served from 32-way +materialized counter buckets and cached for up to five minutes. These rates are +independent and are not expected to sum to 100% when users see different +distractor subsets. + +A background service snapshots the original choice IDs, backfills historical +attempts into v2 exposure counters, and reconciles recently active questions +every five minutes. The metric +`step3_answer_statistics_v2_backfill_complete` must equal `1` before adding +extra distractors to existing questions. + +Search clients that set `supportsChoicePools: true` receive each complete +choice pool. Older clients receive five server-selected choices that always +include the correct answer. Published questions require 5-20 unique choices, +an explanation for every choice, and a follow-up for every incorrect choice. - `GET /readyz` verifies MongoDB connectivity for readiness checks. - `GET /metrics` exposes Prometheus-format service and reconciliation metrics. diff --git a/api/QuestionsApi/Services/AnswerStatisticsMaintenanceService.cs b/api/QuestionsApi/Services/AnswerStatisticsMaintenanceService.cs index 460bcc3..dde221a 100644 --- a/api/QuestionsApi/Services/AnswerStatisticsMaintenanceService.cs +++ b/api/QuestionsApi/Services/AnswerStatisticsMaintenanceService.cs @@ -32,8 +32,13 @@ public sealed class AnswerStatisticsMaintenanceService : BackgroundService _logger.LogInformation("Backfilling materialized answer statistics"); await _statistics.RebuildAllAsync(); await _statistics.MarkBackfillCompleteAsync(_owner); + _metrics.AnswerStatisticsBackfillCompleted(); _metrics.Reconciled(); } + else if (await _statistics.IsBackfillCompleteAsync()) + { + _metrics.AnswerStatisticsBackfillCompleted(); + } } catch (Exception exception) { @@ -45,6 +50,8 @@ public sealed class AnswerStatisticsMaintenanceService : BackgroundService { try { + if (await _statistics.IsBackfillCompleteAsync()) + _metrics.AnswerStatisticsBackfillCompleted(); var activeIds = await _attempts.GetActiveQuestionIdsAsync(DateTime.UtcNow.Subtract(_interval).Subtract(_interval)); await _statistics.RebuildQuestionsAsync(activeIds); _metrics.Reconciled(); diff --git a/api/QuestionsApi/Services/AnswerStatisticsStore.cs b/api/QuestionsApi/Services/AnswerStatisticsStore.cs index ae03c96..9167d02 100644 --- a/api/QuestionsApi/Services/AnswerStatisticsStore.cs +++ b/api/QuestionsApi/Services/AnswerStatisticsStore.cs @@ -1,3 +1,4 @@ +using MongoDB.Bson; using MongoDB.Driver; using QuestionsApi.Models; using System.Collections.Concurrent; @@ -6,20 +7,27 @@ namespace QuestionsApi.Services; public sealed class AnswerStatisticsStore { - private const string StateId = "answer-statistics-v1"; + private const string StateId = "answer-statistics-v2"; private static readonly TimeSpan CacheDuration = TimeSpan.FromMinutes(5); private readonly IMongoCollection _counters; private readonly IMongoCollection _state; + private readonly IMongoCollection _baselines; private readonly QuestionAttemptStore _attempts; + private readonly QuestionStore _questions; private readonly ConcurrentDictionary _cache = new(); private volatile bool _backfillComplete; - public AnswerStatisticsStore(MongoDatabaseProvider mongo, QuestionAttemptStore attempts) + public AnswerStatisticsStore( + MongoDatabaseProvider mongo, + QuestionAttemptStore attempts, + QuestionStore questions) { _attempts = attempts; - _counters = mongo.Database.GetCollection("QuestionAttemptCounters"); + _questions = questions; + _counters = mongo.Database.GetCollection("QuestionChoiceExposureCounters"); _state = mongo.Database.GetCollection("SystemState"); + _baselines = mongo.Database.GetCollection("QuestionPresentationBaselines"); var counterIndex = Builders.IndexKeys .Ascending(counter => counter.QuestionExternalId) @@ -32,27 +40,31 @@ public sealed class AnswerStatisticsStore public async Task IncrementAsync(QuestionAttempt attempt) { - if (attempt.IsSkip) + if (attempt.IsSkip || !attempt.TracksPresentation || attempt.PresentedChoiceIds.Count == 0) return; - var filter = Builders.Filter.And( - Builders.Filter.Eq(counter => counter.QuestionExternalId, attempt.QuestionExternalId), - Builders.Filter.Eq(counter => counter.Bucket, attempt.CounterBucket), - Builders.Filter.Eq(counter => counter.ChoiceId, attempt.ChoiceId)); - var update = Builders.Update - .Inc(counter => counter.Count, 1) - .Set(counter => counter.UpdatedUtc, DateTime.UtcNow) - .SetOnInsert(counter => counter.QuestionExternalId, attempt.QuestionExternalId) - .SetOnInsert(counter => counter.Bucket, attempt.CounterBucket) - .SetOnInsert(counter => counter.ChoiceId, attempt.ChoiceId); + var now = DateTime.UtcNow; + var writes = attempt.PresentedChoiceIds.Distinct(StringComparer.Ordinal).Select(choiceId => + { + var filter = CounterFilter(attempt.QuestionExternalId, attempt.CounterBucket, choiceId); + var update = Builders.Update + .Inc(counter => counter.PresentationCount, 1) + .Inc(counter => counter.SelectionCount, choiceId == attempt.ChoiceId ? 1 : 0) + .Set(counter => counter.UpdatedUtc, now) + .SetOnInsert(counter => counter.QuestionExternalId, attempt.QuestionExternalId) + .SetOnInsert(counter => counter.Bucket, attempt.CounterBucket) + .SetOnInsert(counter => counter.ChoiceId, choiceId); + return new UpdateOneModel(filter, update) { IsUpsert = true }; + }).ToList(); try { - await _counters.UpdateOneAsync(filter, update, new UpdateOptions { IsUpsert = true }); + await _counters.BulkWriteAsync(writes, new BulkWriteOptions { IsOrdered = false }); } - catch (MongoWriteException exception) when (exception.WriteError.Category == ServerErrorCategory.DuplicateKey) + catch (MongoBulkWriteException exception) + when (exception.WriteErrors.All(error => error.Category == ServerErrorCategory.DuplicateKey)) { - await _counters.UpdateOneAsync(filter, update); + await _counters.BulkWriteAsync(writes, new BulkWriteOptions { IsOrdered = false }); } _cache.TryRemove(attempt.QuestionExternalId, out _); } @@ -78,23 +90,9 @@ public sealed class AnswerStatisticsStore else missing.Add(id); } - - if (missing.Count == 0) + if (missing.Count == 0 || !await IsBackfillCompleteAsync()) return output; - var backfillComplete = await IsBackfillCompleteAsync(); - if (!backfillComplete) - { - var rawPercentages = await _attempts.GetPercentagesByQuestionAsync(missing); - foreach (var id in missing) - { - var percentages = rawPercentages.GetValueOrDefault(id, []); - output[id] = percentages; - _cache[id] = new CachedPercentages(percentages, now.Add(CacheDuration)); - } - return output; - } - var totals = await _counters.Aggregate() .Match(counter => missing.Contains(counter.QuestionExternalId)) .Group( @@ -106,34 +104,22 @@ public sealed class AnswerStatisticsStore QuestionExternalId = group.Key.QuestionExternalId, ChoiceId = group.Key.ChoiceId }, - Count = group.Sum(counter => counter.Count) + SelectionCount = group.Sum(counter => counter.SelectionCount), + PresentationCount = group.Sum(counter => counter.PresentationCount) }) .ToListAsync(); - var grouped = totals.GroupBy(total => total.Key.QuestionExternalId) - .ToDictionary(group => group.Key, group => group.ToList()); - + var grouped = totals.GroupBy(total => total.Key.QuestionExternalId).ToDictionary(group => group.Key); foreach (var id in missing) { - Dictionary percentages; - if (grouped.TryGetValue(id, out var questionTotals)) - { - var total = questionTotals.Sum(item => item.Count); - percentages = total == 0 - ? [] - : questionTotals.ToDictionary( - item => item.Key.ChoiceId, - item => (int)Math.Round(item.Count * 100.0 / total)); - } - else - { - percentages = []; - } - + var percentages = grouped.TryGetValue(id, out var choices) + ? choices.Where(item => item.PresentationCount > 0).ToDictionary( + item => item.Key.ChoiceId, + item => (int)Math.Round(item.SelectionCount * 100.0 / item.PresentationCount)) + : []; output[id] = percentages; _cache[id] = new CachedPercentages(percentages, now.Add(CacheDuration)); } - return output; } @@ -149,15 +135,14 @@ public sealed class AnswerStatisticsStore var update = Builders.Update .SetOnInsert(state => state.Id, StateId) .SetOnInsert(state => state.Completed, false) + .SetOnInsert(state => state.CutoverUtc, now) .Set(state => state.LeaseOwner, owner) .Set(state => state.LeaseUntilUtc, now.Add(duration)) .Set(state => state.UpdatedUtc, now); try { - var leased = await _state.FindOneAndUpdateAsync( - filter, - update, + var leased = await _state.FindOneAndUpdateAsync(filter, update, new FindOneAndUpdateOptions { IsUpsert = true, @@ -173,64 +158,131 @@ public sealed class AnswerStatisticsStore public async Task MarkBackfillCompleteAsync(string owner) { - var update = Builders.Update - .Set(state => state.Completed, true) - .Set(state => state.LeaseOwner, owner) - .Set(state => state.UpdatedUtc, DateTime.UtcNow); - await _state.UpdateOneAsync(state => state.Id == StateId && state.LeaseOwner == owner, update); + await _state.UpdateOneAsync( + state => state.Id == StateId && state.LeaseOwner == owner, + Builders.Update + .Set(state => state.Completed, true) + .Set(state => state.UpdatedUtc, DateTime.UtcNow)); _backfillComplete = true; } - public async Task RebuildAllAsync() => await RebuildAsync(null); + public Task RebuildAllAsync() => RebuildAsync(null); - public async Task RebuildQuestionsAsync(IReadOnlyCollection questionExternalIds) + public Task RebuildQuestionsAsync(IReadOnlyCollection questionExternalIds) => + questionExternalIds.Count == 0 ? Task.CompletedTask : RebuildAsync(questionExternalIds); + + public async Task IsBackfillCompleteAsync() { - if (questionExternalIds.Count > 0) - await RebuildAsync(questionExternalIds); + if (_backfillComplete) + return true; + _backfillComplete = (await _state.Find(state => state.Id == StateId).FirstOrDefaultAsync())?.Completed == true; + return _backfillComplete; } private async Task RebuildAsync(IReadOnlyCollection? questionExternalIds) { - var aggregate = _attempts.Collection.Aggregate().Match(attempt => !attempt.IsSkip); - if (questionExternalIds is not null) - aggregate = aggregate.Match(attempt => questionExternalIds.Contains(attempt.QuestionExternalId)); - - var totals = await aggregate - .Group( - attempt => new { attempt.QuestionExternalId, attempt.CounterBucket, attempt.ChoiceId }, - group => new QuestionAttemptCounterAggregate - { - Key = new QuestionChoiceBucketKey - { - QuestionExternalId = group.Key.QuestionExternalId, - Bucket = group.Key.CounterBucket, - ChoiceId = group.Key.ChoiceId - }, - Count = group.Sum(_ => 1L) - }) - .ToListAsync(); - - if (totals.Count > 0) + await EnsureBaselinesAsync(); + var state = await _state.Find(item => item.Id == StateId).FirstAsync(); + var baselines = await _baselines.Find(Builders.Filter.Empty).ToListAsync(); + var baselineByQuestion = baselines.ToDictionary(item => item.QuestionExternalId, item => item.ChoiceIds); + var match = new BsonDocument { - var now = DateTime.UtcNow; - var replacements = totals.Select(total => - { - var counter = new QuestionAttemptCounter + { "IsSkip", false }, + { "$or", new BsonArray { - QuestionExternalId = total.Key.QuestionExternalId, - Bucket = total.Key.Bucket, - ChoiceId = total.Key.ChoiceId, - Count = total.Count, - UpdatedUtc = now - }; - var filter = Builders.Filter.And( - Builders.Filter.Eq(item => item.QuestionExternalId, counter.QuestionExternalId), - Builders.Filter.Eq(item => item.Bucket, counter.Bucket), - Builders.Filter.Eq(item => item.ChoiceId, counter.ChoiceId)); - return new ReplaceOneModel(filter, counter) { IsUpsert = true }; - }).ToList(); - await _counters.BulkWriteAsync(replacements, new BulkWriteOptions { IsOrdered = false }); + new BsonDocument("TracksPresentation", true), + new BsonDocument("CreatedUtc", new BsonDocument("$lt", state.CutoverUtc)) + } + } + }; + if (questionExternalIds is not null) + match["QuestionExternalId"] = new BsonDocument("$in", new BsonArray(questionExternalIds)); + + var selectionPipeline = new[] + { + new BsonDocument("$match", match), + new BsonDocument("$group", new BsonDocument + { + { "_id", new BsonDocument { { "q", "$QuestionExternalId" }, { "b", "$CounterBucket" }, { "c", "$ChoiceId" } } }, + { "count", new BsonDocument("$sum", 1) } + }) + }; + var presentationMatch = new BsonDocument(match) { ["TracksPresentation"] = true }; + var presentationPipeline = new[] + { + new BsonDocument("$match", presentationMatch), + new BsonDocument("$unwind", "$PresentedChoiceIds"), + new BsonDocument("$group", new BsonDocument + { + { "_id", new BsonDocument { { "q", "$QuestionExternalId" }, { "b", "$CounterBucket" }, { "c", "$PresentedChoiceIds" } } }, + { "count", new BsonDocument("$sum", 1) } + }) + }; + var legacyMatch = new BsonDocument + { + { "IsSkip", false }, + { "TracksPresentation", new BsonDocument("$ne", true) }, + { "CreatedUtc", new BsonDocument("$lt", state.CutoverUtc) } + }; + if (questionExternalIds is not null) + legacyMatch["QuestionExternalId"] = new BsonDocument("$in", new BsonArray(questionExternalIds)); + var legacyPipeline = new[] + { + new BsonDocument("$match", legacyMatch), + new BsonDocument("$group", new BsonDocument + { + { "_id", new BsonDocument { { "q", "$QuestionExternalId" }, { "b", "$CounterBucket" } } }, + { "count", new BsonDocument("$sum", 1) } + }) + }; + + var selections = await _attempts.Collection.Aggregate(selectionPipeline).ToListAsync(); + var presentations = await _attempts.Collection.Aggregate(presentationPipeline).ToListAsync(); + var legacy = await _attempts.Collection.Aggregate(legacyPipeline).ToListAsync(); + var totals = new Dictionary<(string Question, int Bucket, string Choice), (long Selected, long Presented)>(); + + foreach (var row in selections) + { + var key = CounterKey(row["_id"].AsBsonDocument); + totals[key] = (row["count"].ToInt64(), totals.GetValueOrDefault(key).Presented); } + foreach (var row in presentations) + { + var key = CounterKey(row["_id"].AsBsonDocument); + totals[key] = (totals.GetValueOrDefault(key).Selected, row["count"].ToInt64()); + } + foreach (var row in legacy) + { + var id = row["_id"].AsBsonDocument; + var questionId = id["q"].AsString; + if (!baselineByQuestion.TryGetValue(questionId, out var choiceIds)) + continue; + foreach (var choiceId in choiceIds) + { + var key = (questionId, id["b"].ToInt32(), choiceId); + totals[key] = (totals.GetValueOrDefault(key).Selected, + totals.GetValueOrDefault(key).Presented + row["count"].ToInt64()); + } + } + + var now = DateTime.UtcNow; + var writes = totals.Select(item => + new ReplaceOneModel( + CounterFilter(item.Key.Question, item.Key.Bucket, item.Key.Choice), + new QuestionAttemptCounter + { + QuestionExternalId = item.Key.Question, + Bucket = item.Key.Bucket, + ChoiceId = item.Key.Choice, + SelectionCount = item.Value.Selected, + PresentationCount = item.Value.Presented, + UpdatedUtc = now + }) + { + IsUpsert = true + }).ToList(); + if (writes.Count > 0) + await _counters.BulkWriteAsync(writes, new BulkWriteOptions { IsOrdered = false }); if (questionExternalIds is null) _cache.Clear(); @@ -239,13 +291,31 @@ public sealed class AnswerStatisticsStore _cache.TryRemove(id, out _); } - private async Task IsBackfillCompleteAsync() + private async Task EnsureBaselinesAsync() { - if (_backfillComplete) - return true; - _backfillComplete = (await _state.Find(state => state.Id == StateId).FirstOrDefaultAsync())?.Completed == true; - return _backfillComplete; + var questions = await _questions.GetAllAsync(); + var writes = questions.Select(question => + new UpdateOneModel( + Builders.Filter.Eq(item => item.QuestionExternalId, question.ExternalId), + Builders.Update + .SetOnInsert(item => item.QuestionExternalId, question.ExternalId) + .SetOnInsert(item => item.ChoiceIds, question.Choices.Select(choice => choice.Id).ToList()) + .SetOnInsert(item => item.CreatedUtc, DateTime.UtcNow)) + { + IsUpsert = true + }).ToList(); + if (writes.Count > 0) + await _baselines.BulkWriteAsync(writes, new BulkWriteOptions { IsOrdered = false }); } + private static FilterDefinition CounterFilter(string questionId, int bucket, string choiceId) => + Builders.Filter.And( + Builders.Filter.Eq(counter => counter.QuestionExternalId, questionId), + Builders.Filter.Eq(counter => counter.Bucket, bucket), + Builders.Filter.Eq(counter => counter.ChoiceId, choiceId)); + + private static (string Question, int Bucket, string Choice) CounterKey(BsonDocument id) => + (id["q"].AsString, id["b"].ToInt32(), id["c"].AsString); + private sealed record CachedPercentages(Dictionary Percentages, DateTime ExpiresUtc); } diff --git a/api/QuestionsApi/Services/QuestionAttemptStore.cs b/api/QuestionsApi/Services/QuestionAttemptStore.cs index 1036ab5..4cd99bd 100644 --- a/api/QuestionsApi/Services/QuestionAttemptStore.cs +++ b/api/QuestionsApi/Services/QuestionAttemptStore.cs @@ -34,13 +34,16 @@ public class QuestionAttemptStore string userId, string choiceId, bool isCorrect, - bool isSkip = false) + bool isSkip = false, + IReadOnlyCollection? presentedChoiceIds = null) { var attempt = new QuestionAttempt { QuestionExternalId = questionExternalId, UserId = userId, ChoiceId = choiceId, + PresentedChoiceIds = presentedChoiceIds?.ToList() ?? [], + TracksPresentation = presentedChoiceIds is { Count: > 0 }, IsCorrect = isCorrect, IsSkip = isSkip, CounterBucket = GetCounterBucket(userId), @@ -62,49 +65,6 @@ public class QuestionAttemptStore } } - public async Task> GetPercentagesForQuestionAsync(string questionExternalId) - { - var results = await GetPercentagesByQuestionAsync([questionExternalId]); - return results.GetValueOrDefault(questionExternalId, []); - } - - public async Task>> GetPercentagesByQuestionAsync(IEnumerable questionExternalIds) - { - var ids = questionExternalIds.Where(id => !string.IsNullOrWhiteSpace(id)).Distinct().ToList(); - if (ids.Count == 0) - return []; - - var totals = await _col.Aggregate() - .Match(a => ids.Contains(a.QuestionExternalId) && !a.IsSkip) - .Group( - attempt => new { attempt.QuestionExternalId, attempt.ChoiceId }, - group => new QuestionAttemptCounterTotal - { - Key = new QuestionChoiceCounterKey - { - QuestionExternalId = group.Key.QuestionExternalId, - ChoiceId = group.Key.ChoiceId - }, - Count = group.Sum(_ => 1L) - }) - .ToListAsync(); - var output = new Dictionary>(); - - foreach (var group in totals.GroupBy(total => total.Key.QuestionExternalId)) - { - var total = group.Sum(item => item.Count); - if (total == 0) - continue; - - output[group.Key] = group - .ToDictionary( - item => item.Key.ChoiceId, - item => (int)Math.Round(item.Count * 100.0 / total)); - } - - return output; - } - public Task> GetActiveQuestionIdsAsync(DateTime sinceUtc) => _col.Distinct( attempt => attempt.QuestionExternalId, diff --git a/api/QuestionsApi/Services/QuestionStore.cs b/api/QuestionsApi/Services/QuestionStore.cs index b39c00a..aeedec6 100644 --- a/api/QuestionsApi/Services/QuestionStore.cs +++ b/api/QuestionsApi/Services/QuestionStore.cs @@ -50,6 +50,32 @@ public class QuestionStore public async Task GetByExternalIdAsync(string externalId) => await _col.Find(q => q.ExternalId == externalId).FirstOrDefaultAsync(); + public async Task> GetAllAsync() => + await _col.Find(Builders.Filter.Empty).ToListAsync(); + + public async Task<(List Items, string? NextCursor, bool HasMore)> ExportPageAsync( + string? after, + int requestedLimit) + { + var limit = Math.Clamp(requestedLimit, 1, 200); + var filter = Builders.Filter.Empty; + if (!string.IsNullOrWhiteSpace(after)) + { + if (!ObjectId.TryParse(after, out var cursor)) + throw new ArgumentException("after must be a valid question cursor", nameof(after)); + filter = new BsonDocument("_id", new BsonDocument("$gt", cursor)); + } + + var page = await _col.Find(filter) + .Sort(Builders.Sort.Ascending("_id")) + .Limit(limit + 1) + .ToListAsync(); + var hasMore = page.Count > limit; + if (hasMore) + page.RemoveAt(page.Count - 1); + return (page, hasMore ? page[^1].Id : null, hasMore); + } + public async Task CreateAsync(UpsertQuestionRequest req) { var question = FromRequest(req); diff --git a/api/QuestionsApi/Services/QuestionValidator.cs b/api/QuestionsApi/Services/QuestionValidator.cs new file mode 100644 index 0000000..4275320 --- /dev/null +++ b/api/QuestionsApi/Services/QuestionValidator.cs @@ -0,0 +1,57 @@ +using QuestionsApi.Models; + +namespace QuestionsApi.Services; + +public static class QuestionValidator +{ + public static string? Validate(UpsertQuestionRequest req) + { + if (string.IsNullOrWhiteSpace(req.ExternalId)) return "externalId required"; + if (string.IsNullOrWhiteSpace(req.Stem)) return "stem required"; + if (string.IsNullOrWhiteSpace(req.LeadIn)) return "leadIn required"; + if (req.Choices.Count < (req.IsPublished ? 5 : 2)) + return req.IsPublished ? "published questions require at least five choices" : "at least two choices required"; + if (req.Choices.Count > 20) return "questions support at most twenty choices"; + if (req.Choices.Any(choice => string.IsNullOrWhiteSpace(choice.Id) || string.IsNullOrWhiteSpace(choice.Text))) + return "choice ids and text are required"; + + var choiceIds = req.Choices.Select(choice => choice.Id.Trim()).ToList(); + if (choiceIds.Distinct(StringComparer.Ordinal).Count() != choiceIds.Count) + return "choice ids must be unique"; + if (string.IsNullOrWhiteSpace(req.CorrectChoiceId)) return "correctChoiceId required"; + if (!choiceIds.Contains(req.CorrectChoiceId.Trim(), StringComparer.Ordinal)) + return "correctChoiceId must match a choice id"; + if (string.IsNullOrWhiteSpace(req.Explanation)) return "explanation required"; + + var choiceIdSet = choiceIds.ToHashSet(StringComparer.Ordinal); + if (!choiceIdSet.SetEquals(req.ChoiceExplanations.Keys)) + return "choiceExplanations must contain exactly one entry for every choice"; + var incorrectChoiceIds = choiceIdSet + .Where(id => id != req.CorrectChoiceId.Trim()) + .ToHashSet(StringComparer.Ordinal); + if (!incorrectChoiceIds.SetEquals(req.WrongAnswerFollowups.Keys)) + return "wrongAnswerFollowups must contain exactly one entry for every incorrect choice"; + if (req.ChoiceSelectionPercentages.Keys.Any(id => !choiceIdSet.Contains(id))) + return "choiceSelectionPercentages keys must match choice ids"; + if (req.ChoiceSelectionPercentages.Any(entry => entry.Value is < 0 or > 100)) + return "choiceSelectionPercentages values must be between 0 and 100"; + + foreach (var entry in req.WrongAnswerFollowups) + { + if (string.IsNullOrWhiteSpace(entry.Value.Prompt)) return "wrongAnswerFollowups prompt required"; + if (string.IsNullOrWhiteSpace(entry.Value.TeachingPoint)) return "wrongAnswerFollowups teachingPoint required"; + if (entry.Value.Choices.Count == 0) return "wrongAnswerFollowups choices required"; + if (entry.Value.Choices.Any(choice => string.IsNullOrWhiteSpace(choice.Id) || string.IsNullOrWhiteSpace(choice.Text))) + return "wrongAnswerFollowups choice ids and text are required"; + if (entry.Value.Choices.Select(choice => choice.Id).Distinct(StringComparer.Ordinal).Count() != entry.Value.Choices.Count) + return "wrongAnswerFollowups choice ids must be unique"; + if (!entry.Value.Choices.Any(choice => choice.Id == entry.Value.CorrectChoiceId)) + return "wrongAnswerFollowups correctChoiceId must match a follow-up choice id"; + } + + if (string.IsNullOrWhiteSpace(req.System)) return "system required"; + if (string.IsNullOrWhiteSpace(req.Discipline)) return "discipline required"; + if (req.Difficulty is < 1 or > 5) return "difficulty must be between 1 and 5"; + return null; + } +} diff --git a/api/QuestionsApi/Services/ServiceMetrics.cs b/api/QuestionsApi/Services/ServiceMetrics.cs index 9ff06d2..8270bac 100644 --- a/api/QuestionsApi/Services/ServiceMetrics.cs +++ b/api/QuestionsApi/Services/ServiceMetrics.cs @@ -12,11 +12,13 @@ public sealed class ServiceMetrics private long _lastReconciliationUnixSeconds; private long _httpRequests; private long _httpDurationMicroseconds; + private long _answerStatisticsBackfillComplete; public void AttemptCreated() => Interlocked.Increment(ref _attemptsCreated); public void AttemptDuplicate() => Interlocked.Increment(ref _attemptDuplicates); public void CounterFailure() => Interlocked.Increment(ref _counterFailures); public void RatingFailure() => Interlocked.Increment(ref _ratingFailures); + public void AnswerStatisticsBackfillCompleted() => Interlocked.Exchange(ref _answerStatisticsBackfillComplete, 1); public void Reconciled() { Interlocked.Increment(ref _reconciliations); @@ -48,6 +50,10 @@ public sealed class ServiceMetrics output.Append("step3_answer_last_reconciliation_unixtime ") .Append(Interlocked.Read(ref _lastReconciliationUnixSeconds)) .AppendLine(); + output.AppendLine("# TYPE step3_answer_statistics_v2_backfill_complete gauge"); + output.Append("step3_answer_statistics_v2_backfill_complete ") + .Append(Interlocked.Read(ref _answerStatisticsBackfillComplete)) + .AppendLine(); return output.ToString(); } diff --git a/scripts/autoload/block_session_service.gd b/scripts/autoload/block_session_service.gd index 65c7dfb..e20c47e 100644 --- a/scripts/autoload/block_session_service.gd +++ b/scripts/autoload/block_session_service.gd @@ -30,6 +30,8 @@ func start_block(selected_system: String, block_size: int, selected_mode: String mode = selected_mode session_kind = "block" active_questions = QuestionBankService.build_block(selected_system, block_size) + for index in range(active_questions.size()): + active_questions[index] = _prepare_question_choices(active_questions[index]) answers.clear() answers.resize(active_questions.size()) current_index = 0 @@ -293,10 +295,38 @@ func _draw_core_question() -> Dictionary: var candidate: Dictionary = core_question_pool[index] if str(candidate.get("id", "")) != last_id or core_question_pool.size() == 1: core_question_pool.remove_at(index) - return candidate.duplicate(true) + return _prepare_question_choices(candidate.duplicate(true)) return {} +func _prepare_question_choices(question: Dictionary) -> Dictionary: + var existing: Array = question.get("presented_choices", []) + if not existing.is_empty(): + return question + + var all_choices: Array = question.get("choices", []).duplicate(true) + var correct_id := str(question.get("correct_choice_id", "")) + var correct_choice: Dictionary = {} + var distractors: Array = [] + for choice in all_choices: + if typeof(choice) != TYPE_DICTIONARY: + continue + if str(choice.get("id", "")) == correct_id: + correct_choice = choice + else: + distractors.append(choice) + + distractors.shuffle() + var presented: Array = distractors.slice(0, min(4, distractors.size())) + if not correct_choice.is_empty(): + presented.append(correct_choice) + elif not all_choices.is_empty(): + presented = all_choices.slice(0, min(5, all_choices.size())) + presented.shuffle() + question["presented_choices"] = presented + return question + + func _build_core_question_pool() -> Array: var pool: Array = [] var profile := ProfileService.get_profile() diff --git a/scripts/autoload/question_bank_service.gd b/scripts/autoload/question_bank_service.gd index 9df273e..75e563e 100644 --- a/scripts/autoload/question_bank_service.gd +++ b/scripts/autoload/question_bank_service.gd @@ -103,7 +103,8 @@ func load_questions_from_api(reset_retry: bool = true) -> void: "Authorization: Bearer %s" % AuthState.access_token ]) var payload := { - "limit": 80 + "limit": 80, + "supportsChoicePools": true } var err := request.request(questions_api_url, headers, HTTPClient.METHOD_POST, JSON.stringify(payload)) if err != OK: @@ -177,10 +178,10 @@ func clear_questions() -> void: ratings_changed.emit() -func record_first_attempt(question_id: String, choice_id: String) -> void: +func record_first_attempt(question_id: String, choice_id: String, presented_choice_ids: Array = []) -> void: if AuthState.access_token.is_empty() or question_id == "" or choice_id == "": return - _enqueue_attempt("answer", question_id, choice_id) + _enqueue_attempt("answer", question_id, choice_id, presented_choice_ids) func skip_question(question_id: String) -> void: @@ -294,7 +295,7 @@ func _on_retry_timer_timeout() -> void: load_questions_from_api(false) -func _enqueue_attempt(kind: String, question_id: String, choice_id: String) -> void: +func _enqueue_attempt(kind: String, question_id: String, choice_id: String, presented_choice_ids: Array = []) -> void: var account := AuthState.username.strip_edges() if account == "": return @@ -306,6 +307,7 @@ func _enqueue_attempt(kind: String, question_id: String, choice_id: String) -> v "kind": kind, "question_id": question_id, "choice_id": choice_id, + "presented_choice_ids": presented_choice_ids.duplicate(), "retry_count": 0 }) _save_attempt_queue() @@ -335,7 +337,8 @@ func _process_attempt_queue() -> void: var url := question_attempt_api_url var body := JSON.stringify({ "questionExternalId": question_id, - "choiceId": str(queued.get("choice_id", "")) + "choiceId": str(queued.get("choice_id", "")), + "presentedChoiceIds": queued.get("presented_choice_ids", []) }) if str(queued.get("kind", "answer")) == "skip": url = "%s/%s/skip" % [question_api_base_url.rstrip("/"), question_id.uri_encode()] @@ -457,6 +460,7 @@ func _load_attempt_queue() -> void: "kind": kind, "question_id": question_id, "choice_id": str(item.get("choice_id", "")), + "presented_choice_ids": item.get("presented_choice_ids", []).duplicate(), "retry_count": max(0, int(item.get("retry_count", 0))) }) diff --git a/scripts/ui/question_player.gd b/scripts/ui/question_player.gd index fc17a96..469af18 100644 --- a/scripts/ui/question_player.gd +++ b/scripts/ui/question_player.gd @@ -442,8 +442,7 @@ func _load_question() -> void: lead_label.text = question.get("lead_in", "") _refresh_labs_button() - presented_choices = question.get("choices", []).duplicate(true) - presented_choices.shuffle() + presented_choices = question.get("presented_choices", question.get("choices", [])).duplicate(true) var choices := presented_choices for index in range(choice_buttons.size()): var button := choice_buttons[index] @@ -613,7 +612,11 @@ func _submit_or_next() -> void: _collapse_resolved_followups() var attempted_choice := selected_choice var attempted_question_id := str(BlockSessionService.get_current_question().get("id", "")) - QuestionBankService.record_first_attempt(attempted_question_id, attempted_choice) + QuestionBankService.record_first_attempt( + attempted_question_id, + attempted_choice, + _presented_choice_ids() + ) var result := BlockSessionService.submit_core_attempt(selected_choice, eliminated, flagged) if result.is_empty(): return @@ -632,6 +635,12 @@ func _submit_or_next() -> void: _trigger_wrong_feedback() _apply_saved_choice_state() return + var attempted_question_id := str(BlockSessionService.get_current_question().get("id", "")) + QuestionBankService.record_first_attempt( + attempted_question_id, + selected_choice, + _presented_choice_ids() + ) current_answer = BlockSessionService.answer_current(selected_choice, eliminated, flagged) if BlockSessionService.mode == "Tutor": _show_explanation() @@ -1871,6 +1880,14 @@ func _get_presented_choices() -> Array: return BlockSessionService.get_current_question().get("choices", []) +func _presented_choice_ids() -> Array: + var ids: Array = [] + for choice in _get_presented_choices(): + if typeof(choice) == TYPE_DICTIONARY: + ids.append(str(choice.get("id", ""))) + return ids + + func _display_label_for_index(index: int) -> String: return String.chr(65 + index) diff --git a/step3.sln b/step3.sln index 4550b08..ef7fecf 100644 --- a/step3.sln +++ b/step3.sln @@ -7,6 +7,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "api", "api", "{D82C56D6-B2C EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "QuestionsApi", "api\QuestionsApi\QuestionsApi.csproj", "{0B2F6FEF-3C08-4A72-BBA9-90C225BC24EC}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "QuestionsApi.Tests", "api\QuestionsApi.Tests\QuestionsApi.Tests.csproj", "{1897C6D6-452D-4142-BCCC-FA1A46E7C869}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -29,11 +31,24 @@ Global {0B2F6FEF-3C08-4A72-BBA9-90C225BC24EC}.Release|x64.Build.0 = Release|Any CPU {0B2F6FEF-3C08-4A72-BBA9-90C225BC24EC}.Release|x86.ActiveCfg = Release|Any CPU {0B2F6FEF-3C08-4A72-BBA9-90C225BC24EC}.Release|x86.Build.0 = Release|Any CPU + {1897C6D6-452D-4142-BCCC-FA1A46E7C869}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {1897C6D6-452D-4142-BCCC-FA1A46E7C869}.Debug|Any CPU.Build.0 = Debug|Any CPU + {1897C6D6-452D-4142-BCCC-FA1A46E7C869}.Debug|x64.ActiveCfg = Debug|Any CPU + {1897C6D6-452D-4142-BCCC-FA1A46E7C869}.Debug|x64.Build.0 = Debug|Any CPU + {1897C6D6-452D-4142-BCCC-FA1A46E7C869}.Debug|x86.ActiveCfg = Debug|Any CPU + {1897C6D6-452D-4142-BCCC-FA1A46E7C869}.Debug|x86.Build.0 = Debug|Any CPU + {1897C6D6-452D-4142-BCCC-FA1A46E7C869}.Release|Any CPU.ActiveCfg = Release|Any CPU + {1897C6D6-452D-4142-BCCC-FA1A46E7C869}.Release|Any CPU.Build.0 = Release|Any CPU + {1897C6D6-452D-4142-BCCC-FA1A46E7C869}.Release|x64.ActiveCfg = Release|Any CPU + {1897C6D6-452D-4142-BCCC-FA1A46E7C869}.Release|x64.Build.0 = Release|Any CPU + {1897C6D6-452D-4142-BCCC-FA1A46E7C869}.Release|x86.ActiveCfg = Release|Any CPU + {1897C6D6-452D-4142-BCCC-FA1A46E7C869}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE EndGlobalSection GlobalSection(NestedProjects) = preSolution {0B2F6FEF-3C08-4A72-BBA9-90C225BC24EC} = {D82C56D6-B2CD-F611-9E31-04638F7EC30E} + {1897C6D6-452D-4142-BCCC-FA1A46E7C869} = {D82C56D6-B2CD-F611-9E31-04638F7EC30E} EndGlobalSection EndGlobal diff --git a/tests/choice_pool_test.gd b/tests/choice_pool_test.gd new file mode 100644 index 0000000..6c9c08b --- /dev/null +++ b/tests/choice_pool_test.gd @@ -0,0 +1,59 @@ +extends SceneTree + +var failures := 0 + + +func _initialize() -> void: + var service = load("res://scripts/autoload/block_session_service.gd").new() + var question := _question(9) + var prepared: Dictionary = service._prepare_question_choices(question.duplicate(true)) + _check_presented(prepared) + + var stable: Dictionary = service._prepare_question_choices(prepared.duplicate(true)) + _assert(stable.get("presented_choices", []) == prepared.get("presented_choices", []), "session choices must remain stable") + + var observed_sets := {} + for iteration in range(40): + var rerolled: Dictionary = service._prepare_question_choices(question.duplicate(true)) + _check_presented(rerolled) + var ids := _ids(rerolled.get("presented_choices", [])) + ids.sort() + observed_sets[",".join(ids)] = true + _assert(observed_sets.size() > 1, "future sessions should be able to present different distractors") + + var legacy: Dictionary = service._prepare_question_choices(_question(5)) + _check_presented(legacy) + _assert(_ids(legacy.get("presented_choices", [])).size() == 5, "legacy five-choice questions must show all choices") + service.free() + quit(1 if failures > 0 else 0) + + +func _question(count: int) -> Dictionary: + var choices: Array = [] + for index in range(count): + choices.append({"id": String.chr(65 + index), "text": "Choice %d" % index}) + return {"choices": choices, "correct_choice_id": "A"} + + +func _check_presented(question: Dictionary) -> void: + var ids := _ids(question.get("presented_choices", [])) + _assert(ids.size() == 5, "exactly five choices must be presented") + _assert(ids.has("A"), "the correct answer must be presented") + var unique := {} + for id in ids: + unique[id] = true + _assert(unique.size() == ids.size(), "presented choices must be unique") + + +func _ids(choices: Array) -> Array: + var ids: Array = [] + for choice in choices: + ids.append(str(choice.get("id", ""))) + return ids + + +func _assert(condition: bool, message: String) -> void: + if condition: + return + failures += 1 + push_error(message) diff --git a/tests/choice_pool_test.gd.uid b/tests/choice_pool_test.gd.uid new file mode 100644 index 0000000..303fee7 --- /dev/null +++ b/tests/choice_pool_test.gd.uid @@ -0,0 +1 @@ +uid://dphnt1gskhhna