Add variable question distractor pools
This commit is contained in:
@@ -0,0 +1,342 @@
|
||||
# Question Bank PDF Import Runbook
|
||||
|
||||
This runbook describes how to turn an image-based board-review PDF into
|
||||
original Step 3-style questions, verify them, insert them through the Questions
|
||||
API, and confirm the production result. It records the workflow used for the
|
||||
July 2026 NBME 5 import and strengthens it with the repository's current
|
||||
reference, originality, image, and duplicate-review requirements.
|
||||
|
||||
The source material is used only to identify tested concepts, reasoning paths,
|
||||
and plausible distractors. Do not copy source wording, vignette structure,
|
||||
distinctive fact combinations, answer wording, or images.
|
||||
|
||||
## July 2026 Result
|
||||
|
||||
- 221 image-only PDF pages were rendered and OCRed.
|
||||
- Page markers were mapped to 176 source questions across five sections.
|
||||
- Questions were generated in batches of four.
|
||||
- The first 72 questions passed the initial hosted-model workflow.
|
||||
- Of the remaining 104, an independent audit approved 95 and rejected 9.
|
||||
- The 9 rejected questions were regenerated from scratch with the audit issues
|
||||
supplied, then all 9 passed a second independent audit.
|
||||
- The bulk API inserted 104 remaining records, bringing the imported set to
|
||||
176 records.
|
||||
- Direct database verification found 0 missing external IDs and 0 answer-key
|
||||
mismatches.
|
||||
|
||||
Known gap in that historical run: model-based medical review was completed, but
|
||||
authoritative references were not added to `QUESTION_REFERENCE_REVIEWS.md` for
|
||||
the `nbme5-*` records. Future runs must complete the reference-review gate below
|
||||
before setting `isPublished` to `true`. The historical records should be treated
|
||||
as awaiting that separate reference review.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Python 3 with `PyMuPDF`, `Pillow`, and `pytesseract`.
|
||||
- A local Tesseract OCR installation available on `PATH`.
|
||||
- PowerShell, `curl`, the .NET SDK, and access to this repository.
|
||||
- Access to a capable generation model and a separate audit model.
|
||||
- A `SUPER` JWT for the target Questions API.
|
||||
- Optional direct MongoDB read access for final production verification.
|
||||
|
||||
Keep secrets out of scripts, prompts, logs, and the repository. Configure them
|
||||
in the current process:
|
||||
|
||||
```powershell
|
||||
$env:QUESTIONS_API_BASE = "https://example.invalid"
|
||||
$env:QUESTIONS_SUPER_JWT = "<short-lived SUPER token>"
|
||||
$env:QUESTIONS_MONGODB = "<optional read-only connection string>"
|
||||
```
|
||||
|
||||
Create all intermediate files outside the repository:
|
||||
|
||||
```powershell
|
||||
$RunId = Get-Date -Format "yyyyMMdd-HHmmss"
|
||||
$WorkDir = Join-Path $env:TEMP "step3-qbank-import-$RunId"
|
||||
New-Item -ItemType Directory -Path $WorkDir | Out-Null
|
||||
"ocr", "mapping", "prompts", "generated", "audits", "repairs" |
|
||||
ForEach-Object { New-Item -ItemType Directory -Path (Join-Path $WorkDir $_) | Out-Null }
|
||||
```
|
||||
|
||||
Suggested layout:
|
||||
|
||||
```text
|
||||
step3-qbank-import-<run-id>/
|
||||
ocr/pages.jsonl
|
||||
mapping/items.jsonl
|
||||
prompts/
|
||||
generated/
|
||||
audits/
|
||||
repairs/
|
||||
validation.json
|
||||
references.json
|
||||
insert-payload.json
|
||||
```
|
||||
|
||||
## 1. Establish The Import Set
|
||||
|
||||
1. Hash and record the source PDF before processing.
|
||||
2. Record the expected source, section, and question counts if known.
|
||||
3. Query existing external IDs before generation. Never assume the database is
|
||||
empty, and never overwrite an existing ID unless that update is intentional.
|
||||
4. Choose a deterministic ID prefix, for example
|
||||
`nbme5-s3-i007-<concept-slug>`.
|
||||
5. Keep the original PDF unchanged and outside generated-output directories.
|
||||
|
||||
The bulk endpoint upserts by `externalId`. Deterministic IDs make retries
|
||||
idempotent, but they also make accidental overwrites possible. Compare the
|
||||
planned IDs with production before insertion.
|
||||
|
||||
## 2. Render And OCR The PDF
|
||||
|
||||
For an image-only PDF, render each page at approximately 250-300 DPI and run
|
||||
Tesseract on the rendered bitmap. A minimal Python outline is:
|
||||
|
||||
```powershell
|
||||
py -m pip install pymupdf pillow pytesseract
|
||||
```
|
||||
|
||||
```python
|
||||
import io, json, sys, fitz, pytesseract
|
||||
from PIL import Image
|
||||
|
||||
source_pdf = sys.argv[1]
|
||||
pages_jsonl = sys.argv[2]
|
||||
document = fitz.open(source_pdf)
|
||||
with open(pages_jsonl, "w", encoding="utf-8") as output:
|
||||
for page_number, page in enumerate(document, start=1):
|
||||
pixmap = page.get_pixmap(matrix=fitz.Matrix(3, 3), alpha=False)
|
||||
image = Image.open(io.BytesIO(pixmap.tobytes("png")))
|
||||
text = pytesseract.image_to_string(image, config="--psm 6")
|
||||
output.write(json.dumps({
|
||||
"page": page_number,
|
||||
"text": text,
|
||||
}, ensure_ascii=False) + "\n")
|
||||
```
|
||||
|
||||
Save the snippet as `ocr_pdf.py` in the working directory and run:
|
||||
|
||||
```powershell
|
||||
py "$WorkDir\ocr_pdf.py" "C:\path\source.pdf" "$WorkDir\ocr\pages.jsonl"
|
||||
```
|
||||
|
||||
Do not silently discard low-confidence or empty pages. Save page numbers and OCR
|
||||
text so every generated question remains traceable to its source pages during
|
||||
review.
|
||||
|
||||
## 3. Map Pages To Source Questions
|
||||
|
||||
Parse repeated headers such as `Exam Section N: Item M of K`. Group all pages
|
||||
for the same section/item pair and append continuation pages to the preceding
|
||||
item when necessary.
|
||||
|
||||
Each `items.jsonl` record should contain:
|
||||
|
||||
```json
|
||||
{"section":3,"item":7,"pages":[95],"sourceText":"..."}
|
||||
```
|
||||
|
||||
Before generation, verify:
|
||||
|
||||
- every detected section/item pair is unique;
|
||||
- the item sequence has no unexplained gaps;
|
||||
- multipage stems and answer choices are complete;
|
||||
- the total mapped-question count matches a manual review;
|
||||
- OCR did not convert option letters, decimal points, inequality signs, units,
|
||||
or answer-choice text into unusable content;
|
||||
- image-dependent questions are identified for special handling.
|
||||
|
||||
The July 2026 source had 221 pages but only 176 questions. Page count is not a
|
||||
question-count proxy.
|
||||
|
||||
## 4. Generate Original Questions
|
||||
|
||||
Generate small batches, typically four source items per request. Small batches
|
||||
limit loss when a model call fails and make outputs easier to review.
|
||||
|
||||
The generation prompt must require:
|
||||
|
||||
- a substantially original clinical scenario testing only the same concept;
|
||||
- changes to age, sex when clinically possible, setting, chronology, numeric
|
||||
values, sentence sequence, and answer wording;
|
||||
- one defensible best answer and a variable pool of plausible choices in an
|
||||
array of `{id, text}` objects, not a map keyed by letters;
|
||||
- a stem containing only the vignette and a separate `leadIn`;
|
||||
- `choiceExplanations` with exactly one entry for every main choice;
|
||||
- `wrongAnswerFollowups` for every incorrect main answer;
|
||||
- three A-C follow-up choices, all of which restate true facts explicitly found
|
||||
in the new stem or displayed labs;
|
||||
- Celsius and Fahrenheit together for every temperature;
|
||||
- labs only when useful and only when their values also appear in the stem;
|
||||
- `media: []` unless separately reviewed, licensed media will be added later;
|
||||
- selection percentages for every authored choice initialized to zero;
|
||||
- established Step categories, 3-6 tags, and a unique deterministic ID.
|
||||
|
||||
For image-dependent source items, rewrite the concept as a text-based question
|
||||
when that can be done without changing the learning objective. Otherwise stop
|
||||
and perform the image copyright workflow in `README.md`; do not import the
|
||||
source image.
|
||||
|
||||
Run a four-question gate before scaling out. Inspect it manually for
|
||||
originality, ambiguity, schema shape, medical accuracy, and stem-anchored
|
||||
follow-ups. Reject the model or revise the prompt if this gate fails.
|
||||
|
||||
## 5. Run Deterministic Validation
|
||||
|
||||
Combine generated batches only after each parses as JSON. The validator should
|
||||
fail the run for any of the following:
|
||||
|
||||
- missing, duplicate, or incorrectly prefixed external IDs;
|
||||
- a count different from the mapped source count;
|
||||
- fewer than five main choices, more than twenty, duplicate choice IDs, or a
|
||||
main choice not represented as an `{id, text}` object;
|
||||
- an invalid `correctChoiceId`;
|
||||
- missing choice explanations or a correct explanation that does not begin
|
||||
with `Correct.`;
|
||||
- follow-up keys that do not exactly equal the incorrect main-answer IDs;
|
||||
- follow-up choices not represented by an ordered A-C array;
|
||||
- follow-up text that cannot be traced to the new stem or labs;
|
||||
- nonzero initial selection percentages;
|
||||
- invalid difficulty, category, tag count, publication state, or media shape;
|
||||
- a Celsius temperature without its Fahrenheit equivalent;
|
||||
- labs referenced by explanations or follow-ups but absent from the stem;
|
||||
- suspiciously long contiguous overlap with source wording.
|
||||
|
||||
Treat automated overlap and anchoring warnings as review queues, not automatic
|
||||
approval or rejection. Record the disposition of every warning.
|
||||
|
||||
## 6. Perform Independent Content Review
|
||||
|
||||
Use a different model or reviewer from the generator. Provide the OCR source
|
||||
and generated question, and request a machine-readable report:
|
||||
|
||||
```json
|
||||
{"externalId":"...","approved":false,"issues":["concrete issue"]}
|
||||
```
|
||||
|
||||
The reviewer must check:
|
||||
|
||||
1. The new item tests the same core reasoning path.
|
||||
2. It is structurally and textually original.
|
||||
3. The keyed answer is correct under current practice.
|
||||
4. No distractor is reasonably co-correct.
|
||||
5. All explanations are accurate.
|
||||
6. Every follow-up choice is an explicit fact from the generated stem or labs.
|
||||
7. The vignette is internally consistent and sufficient to answer.
|
||||
|
||||
Regenerate rejected items from scratch with the audit issues included. Do not
|
||||
patch only the reported sentence when the underlying scenario is ambiguous.
|
||||
Audit every repaired item again. Merge a repair only after it receives an
|
||||
independent approval with an empty issue list.
|
||||
|
||||
## 7. Complete Required Reference And Duplicate Reviews
|
||||
|
||||
Model agreement is not authoritative medical verification. Before publication:
|
||||
|
||||
1. Verify each keyed answer and material explanatory claim against current,
|
||||
direct, authoritative sources such as professional guidelines, government
|
||||
agencies, official labeling, or primary research.
|
||||
2. Add an entry for every question to `QUESTION_REFERENCE_REVIEWS.md`, including
|
||||
title, publisher, direct URL, supported claim, access date, and reviewer.
|
||||
3. Search the entire question bank for exact and near-duplicate learning tasks,
|
||||
not just duplicate text or IDs.
|
||||
4. Record duplicate-review dispositions and substantially differentiate or
|
||||
remove near-duplicates.
|
||||
5. Complete the image licensing review for every non-empty `media` field.
|
||||
|
||||
Keep new questions unpublished until these gates are complete. If generation
|
||||
produces `isPublished: true`, change it to `false` before insertion and publish
|
||||
only after the reviews are recorded.
|
||||
|
||||
## 8. Build And Submit The Payload
|
||||
|
||||
The API expects this wrapper:
|
||||
|
||||
```json
|
||||
{"questions":[{...},{...}]}
|
||||
```
|
||||
|
||||
Submit the validated file exactly as UTF-8 bytes:
|
||||
|
||||
```powershell
|
||||
curl.exe --fail-with-body `
|
||||
-X POST "$env:QUESTIONS_API_BASE/api/questions/bulk" `
|
||||
-H "Authorization: Bearer $env:QUESTIONS_SUPER_JWT" `
|
||||
-H "Content-Type: application/json; charset=utf-8" `
|
||||
--data-binary "@${WorkDir}\insert-payload.json"
|
||||
```
|
||||
|
||||
Expect `{"upserted":N}` with HTTP 200. PowerShell 5.1 can transmit a string body
|
||||
with an unexpected encoding; prefer `curl --data-binary` or pass explicit UTF-8
|
||||
bytes to `Invoke-RestMethod`.
|
||||
|
||||
Do not insert partial batches merely because generation is expensive. Complete
|
||||
the validation, review, repair, reference, and duplicate gates first.
|
||||
|
||||
## 9. Verify Production
|
||||
|
||||
Verification is complete only when all of these pass:
|
||||
|
||||
- the API reports the expected upsert count;
|
||||
- the database contains every expected external ID exactly once;
|
||||
- every stored `correctChoiceId` matches the final payload;
|
||||
- a targeted deployed-API search returns at least one newly inserted question;
|
||||
- the returned question has the expected choices, answer key, and publication
|
||||
state;
|
||||
- repository state is unchanged unless documentation or tooling was
|
||||
intentionally committed.
|
||||
|
||||
Do not use an unfiltered search to prove completeness. The API limits search
|
||||
results and randomizes the returned order. Compare IDs directly through a
|
||||
read-only database query or a dedicated administrative verification tool, then
|
||||
use a highly specific tag combination for the deployed-API read check.
|
||||
|
||||
The final verification report should include at least:
|
||||
|
||||
```text
|
||||
EXPECTED=<count>
|
||||
STORED=<count>
|
||||
MISSING=<count>
|
||||
DUPLICATES=<count>
|
||||
ANSWER_MISMATCHES=<count>
|
||||
API_READ_RESULTS=<count>
|
||||
```
|
||||
|
||||
## 10. Retain Reviews, Then Clean Up
|
||||
|
||||
Retain the reference ledger, duplicate-review record, validation summary, audit
|
||||
decisions, and final ID/answer-key manifest according to project policy. These
|
||||
are review records, not disposable generation scratch files.
|
||||
|
||||
After production verification:
|
||||
|
||||
1. Delete rendered page images, raw model logs, rejected generations, temporary
|
||||
prompts, local test projects, and disposable payload copies.
|
||||
2. Preserve or securely dispose of the original PDF according to its license
|
||||
and organizational policy.
|
||||
3. Remove large local models installed only for the import.
|
||||
4. Confirm the repository is clean with `git status --short --branch`.
|
||||
5. Commit and push only maintained documentation, tooling, and review records.
|
||||
|
||||
Use explicit, workspace-contained paths for cleanup. Resolve each path before a
|
||||
recursive deletion and never delete the repository or source PDF as part of a
|
||||
generated-artifact cleanup.
|
||||
|
||||
## Acceptance Checklist
|
||||
|
||||
- [ ] Source PDF hash and import scope recorded.
|
||||
- [ ] OCR page count and mapped-question count reconciled.
|
||||
- [ ] Every source item mapped exactly once.
|
||||
- [ ] Four-question generation gate passed.
|
||||
- [ ] Full schema and ID validation passed with zero errors.
|
||||
- [ ] All anchoring and source-overlap warnings reviewed.
|
||||
- [ ] Independent content audit approved every question.
|
||||
- [ ] Every rejected question regenerated and re-audited.
|
||||
- [ ] Authoritative references recorded for every question.
|
||||
- [ ] Full-bank duplicate review completed.
|
||||
- [ ] Image rights cleared for every media-bearing question.
|
||||
- [ ] Final payload contains only intended new or intentionally updated IDs.
|
||||
- [ ] Bulk API returned the expected upsert count.
|
||||
- [ ] Database has zero missing IDs and answer-key mismatches.
|
||||
- [ ] Deployed API read check passed.
|
||||
- [ ] Required review records retained and scratch files removed.
|
||||
@@ -32,6 +32,12 @@ The Godot client reads this from `project.godot`:
|
||||
step3/questions_api_url
|
||||
```
|
||||
|
||||
## Question Bank PDF Imports
|
||||
|
||||
Use [QBANK_PDF_IMPORT_RUNBOOK.md](QBANK_PDF_IMPORT_RUNBOOK.md) for the complete
|
||||
OCR, original-authoring, validation, independent-review, reference-review, bulk
|
||||
insertion, production-verification, and cleanup workflow.
|
||||
|
||||
## Question Image Copyright Review
|
||||
|
||||
All questions containing images must receive a copyright and licensing review
|
||||
@@ -102,6 +108,22 @@ copyright requirements still apply. If the question schema does not expose a
|
||||
reference field, retain this information in the associated authoring or review
|
||||
record until it can be stored with the question.
|
||||
|
||||
## Variable Distractor Pools
|
||||
|
||||
Published questions may contain more than the five choices shown to a learner.
|
||||
The player always presents the correct answer plus four randomly selected
|
||||
distractors and keeps that set stable for the duration of the active session.
|
||||
Choice IDs are stable content identifiers; the A-E labels shown in the player
|
||||
are based on presentation order and must not be stored as answer analytics.
|
||||
|
||||
Add only distractors that are plausible for the exact vignette while remaining
|
||||
unambiguously incorrect. Every added distractor requires its own explanation,
|
||||
stem-anchored wrong-answer follow-up, independent accuracy review, and
|
||||
authoritative reference coverage. There is no required number of additional
|
||||
distractors: record an exception instead of forcing a weak or co-correct
|
||||
answer. Do not expand production pools until the v2 exposure-statistics
|
||||
backfill metric reports completion.
|
||||
|
||||
## Duplicate Question Review
|
||||
|
||||
Every question must be checked for duplicates before publication. This review
|
||||
|
||||
@@ -0,0 +1,104 @@
|
||||
using QuestionsApi.Models;
|
||||
using QuestionsApi.Services;
|
||||
using Xunit;
|
||||
|
||||
namespace QuestionsApi.Tests;
|
||||
|
||||
public sealed class ChoicePoolTests
|
||||
{
|
||||
[Fact]
|
||||
public void LegacyResponseContainsFiveChoicesAndAlwaysIncludesCorrectAnswer()
|
||||
{
|
||||
var question = BuildQuestion(9);
|
||||
|
||||
for (var iteration = 0; iteration < 50; iteration++)
|
||||
{
|
||||
var response = QuestionResponse.FromModel(question, includeChoicePool: false);
|
||||
Assert.Equal(5, response.Choices.Count);
|
||||
Assert.Equal(5, response.Choices.Select(choice => choice.Id).Distinct().Count());
|
||||
Assert.Contains(response.Choices, choice => choice.Id == question.CorrectChoiceId);
|
||||
Assert.All(response.ChoiceExplanations.Keys, id => Assert.Contains(response.Choices, choice => choice.Id == id));
|
||||
Assert.All(response.WrongAnswerFollowups.Keys, id => Assert.Contains(response.Choices, choice => choice.Id == id));
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void PoolCapableResponseContainsCompletePool()
|
||||
{
|
||||
var question = BuildQuestion(9);
|
||||
var response = QuestionResponse.FromModel(question, includeChoicePool: true);
|
||||
Assert.Equal(9, response.Choices.Count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void PublishedQuestionRequiresCompletePerChoiceTeachingContent()
|
||||
{
|
||||
var request = BuildRequest(6);
|
||||
Assert.Null(QuestionValidator.Validate(request));
|
||||
|
||||
request.WrongAnswerFollowups.Remove("F");
|
||||
Assert.Contains("every incorrect choice", QuestionValidator.Validate(request));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void PublishedQuestionRequiresAtLeastFiveUniqueChoices()
|
||||
{
|
||||
var request = BuildRequest(4);
|
||||
Assert.Contains("at least five", QuestionValidator.Validate(request));
|
||||
|
||||
request = BuildRequest(5);
|
||||
request.Choices[4].Id = "A";
|
||||
Assert.Contains("unique", QuestionValidator.Validate(request));
|
||||
}
|
||||
|
||||
private static Question BuildQuestion(int count)
|
||||
{
|
||||
var request = BuildRequest(count);
|
||||
return new Question
|
||||
{
|
||||
ExternalId = request.ExternalId,
|
||||
Stem = request.Stem,
|
||||
LeadIn = request.LeadIn,
|
||||
Choices = request.Choices,
|
||||
CorrectChoiceId = request.CorrectChoiceId,
|
||||
Explanation = request.Explanation,
|
||||
ChoiceExplanations = request.ChoiceExplanations,
|
||||
WrongAnswerFollowups = request.WrongAnswerFollowups,
|
||||
ChoiceSelectionPercentages = request.ChoiceSelectionPercentages
|
||||
};
|
||||
}
|
||||
|
||||
private static UpsertQuestionRequest BuildRequest(int count)
|
||||
{
|
||||
var ids = Enumerable.Range(0, count).Select(index => ((char)('A' + index)).ToString()).ToList();
|
||||
var choices = ids.Select(id => new QuestionChoice { Id = id, Text = $"Choice {id}" }).ToList();
|
||||
return new UpsertQuestionRequest
|
||||
{
|
||||
ExternalId = "choice-pool-test",
|
||||
Stem = "A complete clinical vignette.",
|
||||
LeadIn = "Which option is most appropriate?",
|
||||
Choices = choices,
|
||||
CorrectChoiceId = "A",
|
||||
Explanation = "Explanation",
|
||||
ChoiceExplanations = ids.ToDictionary(id => id, id => id == "A" ? "Correct." : "Incorrect."),
|
||||
WrongAnswerFollowups = ids.Where(id => id != "A").ToDictionary(
|
||||
id => id,
|
||||
id => new WrongAnswerFollowup
|
||||
{
|
||||
Prompt = "Which finding makes this less appropriate?",
|
||||
Choices =
|
||||
[
|
||||
new QuestionChoice { Id = "A", Text = "Finding one" },
|
||||
new QuestionChoice { Id = "B", Text = "Finding two" }
|
||||
],
|
||||
CorrectChoiceId = "A",
|
||||
TeachingPoint = "Teaching point"
|
||||
}),
|
||||
ChoiceSelectionPercentages = ids.ToDictionary(id => id, _ => 0),
|
||||
Discipline = "Medicine",
|
||||
System = "Cardiovascular System",
|
||||
Difficulty = 2,
|
||||
IsPublished = true
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net8.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
<IsPackable>false</IsPackable>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.12.0" />
|
||||
<PackageReference Include="xunit" Version="2.9.2" />
|
||||
<PackageReference Include="xunit.runner.visualstudio" Version="2.8.2" />
|
||||
<ProjectReference Include="..\QuestionsApi\QuestionsApi.csproj" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
@@ -52,10 +52,34 @@ public class QuestionsController : ControllerBase
|
||||
return Ok(questions.Select(q =>
|
||||
{
|
||||
percentages.TryGetValue(q.ExternalId, out var questionPercentages);
|
||||
return QuestionResponse.FromModel(q, questionPercentages, questionElos.GetValueOrDefault(q.ExternalId, 1000));
|
||||
return QuestionResponse.FromModel(
|
||||
q,
|
||||
questionPercentages,
|
||||
questionElos.GetValueOrDefault(q.ExternalId, 1000),
|
||||
req.SupportsChoicePools);
|
||||
}).ToList());
|
||||
}
|
||||
|
||||
[HttpGet("admin/export")]
|
||||
[Authorize(Roles = "SUPER")]
|
||||
public async Task<IActionResult> AdminExport([FromQuery] string? after = null, [FromQuery] int limit = 100)
|
||||
{
|
||||
try
|
||||
{
|
||||
var page = await _questions.ExportPageAsync(after, limit);
|
||||
return Ok(new QuestionAdminExportResponse
|
||||
{
|
||||
Items = page.Items.Select(question => QuestionResponse.FromModel(question)).ToList(),
|
||||
NextCursor = page.NextCursor,
|
||||
HasMore = page.HasMore
|
||||
});
|
||||
}
|
||||
catch (ArgumentException exception)
|
||||
{
|
||||
return BadRequest(exception.Message);
|
||||
}
|
||||
}
|
||||
|
||||
[HttpGet("{id}")]
|
||||
[Authorize(Roles = "USER,SUPER")]
|
||||
public async Task<IActionResult> Get(string id)
|
||||
@@ -84,6 +108,22 @@ public class QuestionsController : ControllerBase
|
||||
if (!question.Choices.Any(c => c.Id == req.ChoiceId.Trim()))
|
||||
return BadRequest("choiceId must match a choice id");
|
||||
|
||||
var presentedChoiceIds = req.PresentedChoiceIds
|
||||
.Where(id => !string.IsNullOrWhiteSpace(id))
|
||||
.Select(id => id.Trim())
|
||||
.ToList();
|
||||
if (presentedChoiceIds.Count > 0)
|
||||
{
|
||||
var validChoiceIds = question.Choices.Select(choice => choice.Id).ToHashSet(StringComparer.Ordinal);
|
||||
if (presentedChoiceIds.Count != 5 || presentedChoiceIds.Distinct(StringComparer.Ordinal).Count() != 5)
|
||||
return BadRequest("presentedChoiceIds must contain exactly five unique choice ids");
|
||||
if (presentedChoiceIds.Any(id => !validChoiceIds.Contains(id)))
|
||||
return BadRequest("presentedChoiceIds must match question choice ids");
|
||||
if (!presentedChoiceIds.Contains(req.ChoiceId.Trim(), StringComparer.Ordinal)
|
||||
|| !presentedChoiceIds.Contains(question.CorrectChoiceId, StringComparer.Ordinal))
|
||||
return BadRequest("presentedChoiceIds must include the selected and correct choices");
|
||||
}
|
||||
|
||||
var userId = User.FindFirstValue(ClaimTypes.NameIdentifier)
|
||||
?? User.FindFirstValue(ClaimTypes.Name)
|
||||
?? User.Identity?.Name
|
||||
@@ -95,7 +135,9 @@ public class QuestionsController : ControllerBase
|
||||
question.ExternalId,
|
||||
userId,
|
||||
req.ChoiceId.Trim(),
|
||||
req.ChoiceId.Trim() == question.CorrectChoiceId);
|
||||
req.ChoiceId.Trim() == question.CorrectChoiceId,
|
||||
false,
|
||||
presentedChoiceIds);
|
||||
if (record.Inserted)
|
||||
{
|
||||
_metrics.AttemptCreated();
|
||||
@@ -252,7 +294,7 @@ public class QuestionsController : ControllerBase
|
||||
[Authorize(Roles = "SUPER")]
|
||||
public async Task<IActionResult> Create([FromBody] UpsertQuestionRequest req)
|
||||
{
|
||||
var validation = ValidateQuestion(req);
|
||||
var validation = QuestionValidator.Validate(req);
|
||||
if (validation is not null)
|
||||
return BadRequest(validation);
|
||||
|
||||
@@ -272,7 +314,7 @@ public class QuestionsController : ControllerBase
|
||||
|
||||
foreach (var question in req.Questions)
|
||||
{
|
||||
var validation = ValidateQuestion(question);
|
||||
var validation = QuestionValidator.Validate(question);
|
||||
if (validation is not null)
|
||||
return BadRequest($"{question.ExternalId}: {validation}");
|
||||
}
|
||||
@@ -285,7 +327,7 @@ public class QuestionsController : ControllerBase
|
||||
[Authorize(Roles = "SUPER")]
|
||||
public async Task<IActionResult> Update(string id, [FromBody] UpsertQuestionRequest req)
|
||||
{
|
||||
var validation = ValidateQuestion(req);
|
||||
var validation = QuestionValidator.Validate(req);
|
||||
if (validation is not null)
|
||||
return BadRequest(validation);
|
||||
|
||||
@@ -314,59 +356,6 @@ public class QuestionsController : ControllerBase
|
||||
return Ok("Deleted");
|
||||
}
|
||||
|
||||
private static string? ValidateQuestion(UpsertQuestionRequest req)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(req.ExternalId))
|
||||
return "externalId required";
|
||||
if (string.IsNullOrWhiteSpace(req.Stem))
|
||||
return "stem required";
|
||||
if (string.IsNullOrWhiteSpace(req.LeadIn))
|
||||
return "leadIn required";
|
||||
if (req.Choices.Count < 2)
|
||||
return "at least two choices required";
|
||||
if (string.IsNullOrWhiteSpace(req.CorrectChoiceId))
|
||||
return "correctChoiceId required";
|
||||
if (!req.Choices.Any(c => c.Id == req.CorrectChoiceId))
|
||||
return "correctChoiceId must match a choice id";
|
||||
if (string.IsNullOrWhiteSpace(req.Explanation))
|
||||
return "explanation required";
|
||||
if (req.ChoiceSelectionPercentages.Any(entry => entry.Value is < 0 or > 100))
|
||||
return "choiceSelectionPercentages values must be between 0 and 100";
|
||||
var followupValidation = ValidateWrongAnswerFollowups(req);
|
||||
if (followupValidation is not null)
|
||||
return followupValidation;
|
||||
if (string.IsNullOrWhiteSpace(req.System))
|
||||
return "system required";
|
||||
if (string.IsNullOrWhiteSpace(req.Discipline))
|
||||
return "discipline required";
|
||||
if (req.Difficulty is < 1 or > 5)
|
||||
return "difficulty must be between 1 and 5";
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static string? ValidateWrongAnswerFollowups(UpsertQuestionRequest req)
|
||||
{
|
||||
var choiceIds = req.Choices.Select(c => c.Id).Where(id => !string.IsNullOrWhiteSpace(id)).ToHashSet();
|
||||
foreach (var entry in req.WrongAnswerFollowups)
|
||||
{
|
||||
if (!choiceIds.Contains(entry.Key))
|
||||
return "wrongAnswerFollowups keys must match a choice id";
|
||||
if (entry.Key == req.CorrectChoiceId)
|
||||
return "wrongAnswerFollowups should only target incorrect choices";
|
||||
if (string.IsNullOrWhiteSpace(entry.Value.Prompt))
|
||||
return "wrongAnswerFollowups prompt required";
|
||||
if (string.IsNullOrWhiteSpace(entry.Value.TeachingPoint))
|
||||
return "wrongAnswerFollowups teachingPoint required";
|
||||
if (entry.Value.Choices.Count == 0)
|
||||
return "wrongAnswerFollowups choices required";
|
||||
if (!entry.Value.Choices.Any(c => c.Id == entry.Value.CorrectChoiceId))
|
||||
return "wrongAnswerFollowups correctChoiceId must match a follow-up choice id";
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private string CurrentUserId() =>
|
||||
User.FindFirstValue(ClaimTypes.NameIdentifier)
|
||||
?? User.FindFirstValue(ClaimTypes.Name)
|
||||
|
||||
@@ -7,7 +7,8 @@ public sealed class QuestionAttemptCounter
|
||||
public string QuestionExternalId { get; set; } = "";
|
||||
public int Bucket { get; set; }
|
||||
public string ChoiceId { get; set; } = "";
|
||||
public long Count { get; set; }
|
||||
public long SelectionCount { get; set; }
|
||||
public long PresentationCount { get; set; }
|
||||
public DateTime UpdatedUtc { get; set; } = DateTime.UtcNow;
|
||||
}
|
||||
|
||||
@@ -15,14 +16,16 @@ public sealed class QuestionAttemptCounterTotal
|
||||
{
|
||||
[BsonId]
|
||||
public QuestionChoiceCounterKey Key { get; set; } = new();
|
||||
public long Count { get; set; }
|
||||
public long SelectionCount { get; set; }
|
||||
public long PresentationCount { get; set; }
|
||||
}
|
||||
|
||||
public sealed class QuestionAttemptCounterAggregate
|
||||
{
|
||||
[BsonId]
|
||||
public QuestionChoiceBucketKey Key { get; set; } = new();
|
||||
public long Count { get; set; }
|
||||
public long SelectionCount { get; set; }
|
||||
public long PresentationCount { get; set; }
|
||||
}
|
||||
|
||||
public sealed class QuestionChoiceBucketKey
|
||||
@@ -41,9 +44,18 @@ public sealed class QuestionChoiceCounterKey
|
||||
public sealed class AnswerStatisticsState
|
||||
{
|
||||
[BsonId]
|
||||
public string Id { get; set; } = "answer-statistics-v1";
|
||||
public string Id { get; set; } = "answer-statistics-v2";
|
||||
public bool Completed { get; set; }
|
||||
public string LeaseOwner { get; set; } = "";
|
||||
public DateTime LeaseUntilUtc { get; set; }
|
||||
public DateTime UpdatedUtc { get; set; } = DateTime.UtcNow;
|
||||
public DateTime CutoverUtc { get; set; }
|
||||
}
|
||||
|
||||
public sealed class QuestionPresentationBaseline
|
||||
{
|
||||
[BsonId]
|
||||
public string QuestionExternalId { get; set; } = "";
|
||||
public List<string> ChoiceIds { get; set; } = [];
|
||||
public DateTime CreatedUtc { get; set; } = DateTime.UtcNow;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
namespace QuestionsApi.Models;
|
||||
|
||||
public sealed class QuestionAdminExportResponse
|
||||
{
|
||||
public List<QuestionResponse> Items { get; set; } = [];
|
||||
public string? NextCursor { get; set; }
|
||||
public bool HasMore { get; set; }
|
||||
}
|
||||
@@ -12,6 +12,8 @@ public class QuestionAttempt
|
||||
public string QuestionExternalId { get; set; } = "";
|
||||
public string UserId { get; set; } = "";
|
||||
public string ChoiceId { get; set; } = "";
|
||||
public List<string> PresentedChoiceIds { get; set; } = [];
|
||||
public bool TracksPresentation { get; set; }
|
||||
public bool IsCorrect { get; set; }
|
||||
public bool IsSkip { get; set; }
|
||||
public int CounterBucket { get; set; }
|
||||
|
||||
@@ -8,6 +8,7 @@ public class QuestionSearchRequest
|
||||
public int? DifficultyMin { get; set; }
|
||||
public int? DifficultyMax { get; set; }
|
||||
public int Limit { get; set; } = 40;
|
||||
public bool SupportsChoicePools { get; set; }
|
||||
}
|
||||
|
||||
public class UpsertQuestionRequest
|
||||
@@ -35,4 +36,5 @@ public class FirstQuestionAttemptRequest
|
||||
{
|
||||
public string QuestionExternalId { get; set; } = "";
|
||||
public string ChoiceId { get; set; } = "";
|
||||
public List<string> PresentedChoiceIds { get; set; } = [];
|
||||
}
|
||||
|
||||
@@ -12,6 +12,7 @@ public class QuestionResponse
|
||||
public Dictionary<string, string> ChoiceExplanations { get; set; } = [];
|
||||
public Dictionary<string, WrongAnswerFollowup> WrongAnswerFollowups { get; set; } = [];
|
||||
public Dictionary<string, int> ChoiceSelectionPercentages { get; set; } = [];
|
||||
public string ChoiceSelectionRateBasis { get; set; } = "presented";
|
||||
public string Discipline { get; set; } = "";
|
||||
public string System { get; set; } = "";
|
||||
public string PerformanceCategory { get; set; } = "";
|
||||
@@ -24,18 +25,30 @@ public class QuestionResponse
|
||||
public static QuestionResponse FromModel(
|
||||
Question question,
|
||||
Dictionary<string, int>? liveChoiceSelectionPercentages = null,
|
||||
int questionElo = 1000) => new()
|
||||
int questionElo = 1000,
|
||||
bool includeChoicePool = true)
|
||||
{
|
||||
var choices = includeChoicePool ? question.Choices : SelectLegacyChoices(question);
|
||||
var choiceIds = choices.Select(choice => choice.Id).ToHashSet(StringComparer.Ordinal);
|
||||
var percentages = BuildChoiceSelectionPercentages(question, liveChoiceSelectionPercentages);
|
||||
return new QuestionResponse
|
||||
{
|
||||
Id = question.Id ?? "",
|
||||
ExternalId = question.ExternalId,
|
||||
Stem = question.Stem,
|
||||
LeadIn = question.LeadIn,
|
||||
Choices = question.Choices,
|
||||
Choices = choices,
|
||||
CorrectChoiceId = question.CorrectChoiceId,
|
||||
Explanation = question.Explanation,
|
||||
ChoiceExplanations = question.ChoiceExplanations,
|
||||
WrongAnswerFollowups = question.WrongAnswerFollowups,
|
||||
ChoiceSelectionPercentages = BuildChoiceSelectionPercentages(question, liveChoiceSelectionPercentages),
|
||||
ChoiceExplanations = question.ChoiceExplanations
|
||||
.Where(entry => choiceIds.Contains(entry.Key))
|
||||
.ToDictionary(),
|
||||
WrongAnswerFollowups = question.WrongAnswerFollowups
|
||||
.Where(entry => choiceIds.Contains(entry.Key))
|
||||
.ToDictionary(),
|
||||
ChoiceSelectionPercentages = percentages
|
||||
.Where(entry => choiceIds.Contains(entry.Key))
|
||||
.ToDictionary(),
|
||||
Discipline = question.Discipline,
|
||||
System = question.System,
|
||||
PerformanceCategory = RatingCategory.Normalize(question.PerformanceCategory, question.System, question.Discipline),
|
||||
@@ -45,6 +58,23 @@ public class QuestionResponse
|
||||
Media = question.Media,
|
||||
Tags = question.Tags
|
||||
};
|
||||
}
|
||||
|
||||
private static List<QuestionChoice> SelectLegacyChoices(Question question)
|
||||
{
|
||||
if (question.Choices.Count <= 5)
|
||||
return question.Choices;
|
||||
|
||||
var correct = question.Choices.First(choice => choice.Id == question.CorrectChoiceId);
|
||||
var distractors = question.Choices
|
||||
.Where(choice => choice.Id != question.CorrectChoiceId)
|
||||
.OrderBy(_ => Random.Shared.Next())
|
||||
.Take(4)
|
||||
.Append(correct)
|
||||
.OrderBy(_ => Random.Shared.Next())
|
||||
.ToList();
|
||||
return distractors;
|
||||
}
|
||||
|
||||
public static Dictionary<string, int> BuildChoiceSelectionPercentages(Question question, Dictionary<string, int>? liveChoiceSelectionPercentages)
|
||||
{
|
||||
|
||||
@@ -53,8 +53,10 @@ builder.Services.AddAuthentication(JwtBearerDefaults.AuthenticationScheme)
|
||||
{
|
||||
o.TokenValidationParameters = new TokenValidationParameters
|
||||
{
|
||||
ValidateIssuer = true, ValidIssuer = issuer,
|
||||
ValidateAudience = true, ValidAudience = aud,
|
||||
ValidateIssuer = true,
|
||||
ValidIssuer = issuer,
|
||||
ValidateAudience = true,
|
||||
ValidAudience = aud,
|
||||
ValidateIssuerSigningKey = true,
|
||||
IssuerSigningKey = new SymmetricSecurityKey(Encoding.UTF8.GetBytes(jwtKey)),
|
||||
ValidateLifetime = true,
|
||||
|
||||
@@ -7,6 +7,7 @@ STEP-style question bank service for study games and clients.
|
||||
- `GET /healthz` Health check.
|
||||
- `GET /api/questions/systems` List available systems.
|
||||
- `POST /api/questions/search` Search or build a block of questions.
|
||||
- `GET /api/questions/admin/export` Cursor-paginated full-bank export (SUPER only).
|
||||
- `GET /api/questions/{id}` Get one question.
|
||||
- `POST /api/questions` Create question (SUPER only).
|
||||
- `POST /api/questions/bulk` Upsert questions by `externalId` (SUPER only).
|
||||
@@ -22,10 +23,23 @@ Player reads require `USER` or `SUPER` JWT auth. Authoring endpoints require `SU
|
||||
## Answer tracking
|
||||
|
||||
The `QuestionAttempts` collection is the immutable source of truth and enforces
|
||||
one first attempt per question and user with a unique compound index. Choice
|
||||
percentages are served from 32-way materialized counter buckets and cached for
|
||||
up to five minutes. A background service backfills counters on the first deploy
|
||||
and reconciles recently active questions every five minutes.
|
||||
one first attempt per question and user with a unique compound index. Updated
|
||||
clients submit the five choice IDs that were presented. Choice rates are the
|
||||
percentage of presentations that resulted in selection, served from 32-way
|
||||
materialized counter buckets and cached for up to five minutes. These rates are
|
||||
independent and are not expected to sum to 100% when users see different
|
||||
distractor subsets.
|
||||
|
||||
A background service snapshots the original choice IDs, backfills historical
|
||||
attempts into v2 exposure counters, and reconciles recently active questions
|
||||
every five minutes. The metric
|
||||
`step3_answer_statistics_v2_backfill_complete` must equal `1` before adding
|
||||
extra distractors to existing questions.
|
||||
|
||||
Search clients that set `supportsChoicePools: true` receive each complete
|
||||
choice pool. Older clients receive five server-selected choices that always
|
||||
include the correct answer. Published questions require 5-20 unique choices,
|
||||
an explanation for every choice, and a follow-up for every incorrect choice.
|
||||
|
||||
- `GET /readyz` verifies MongoDB connectivity for readiness checks.
|
||||
- `GET /metrics` exposes Prometheus-format service and reconciliation metrics.
|
||||
|
||||
@@ -32,8 +32,13 @@ public sealed class AnswerStatisticsMaintenanceService : BackgroundService
|
||||
_logger.LogInformation("Backfilling materialized answer statistics");
|
||||
await _statistics.RebuildAllAsync();
|
||||
await _statistics.MarkBackfillCompleteAsync(_owner);
|
||||
_metrics.AnswerStatisticsBackfillCompleted();
|
||||
_metrics.Reconciled();
|
||||
}
|
||||
else if (await _statistics.IsBackfillCompleteAsync())
|
||||
{
|
||||
_metrics.AnswerStatisticsBackfillCompleted();
|
||||
}
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
@@ -45,6 +50,8 @@ public sealed class AnswerStatisticsMaintenanceService : BackgroundService
|
||||
{
|
||||
try
|
||||
{
|
||||
if (await _statistics.IsBackfillCompleteAsync())
|
||||
_metrics.AnswerStatisticsBackfillCompleted();
|
||||
var activeIds = await _attempts.GetActiveQuestionIdsAsync(DateTime.UtcNow.Subtract(_interval).Subtract(_interval));
|
||||
await _statistics.RebuildQuestionsAsync(activeIds);
|
||||
_metrics.Reconciled();
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Driver;
|
||||
using QuestionsApi.Models;
|
||||
using System.Collections.Concurrent;
|
||||
@@ -6,20 +7,27 @@ namespace QuestionsApi.Services;
|
||||
|
||||
public sealed class AnswerStatisticsStore
|
||||
{
|
||||
private const string StateId = "answer-statistics-v1";
|
||||
private const string StateId = "answer-statistics-v2";
|
||||
private static readonly TimeSpan CacheDuration = TimeSpan.FromMinutes(5);
|
||||
|
||||
private readonly IMongoCollection<QuestionAttemptCounter> _counters;
|
||||
private readonly IMongoCollection<AnswerStatisticsState> _state;
|
||||
private readonly IMongoCollection<QuestionPresentationBaseline> _baselines;
|
||||
private readonly QuestionAttemptStore _attempts;
|
||||
private readonly QuestionStore _questions;
|
||||
private readonly ConcurrentDictionary<string, CachedPercentages> _cache = new();
|
||||
private volatile bool _backfillComplete;
|
||||
|
||||
public AnswerStatisticsStore(MongoDatabaseProvider mongo, QuestionAttemptStore attempts)
|
||||
public AnswerStatisticsStore(
|
||||
MongoDatabaseProvider mongo,
|
||||
QuestionAttemptStore attempts,
|
||||
QuestionStore questions)
|
||||
{
|
||||
_attempts = attempts;
|
||||
_counters = mongo.Database.GetCollection<QuestionAttemptCounter>("QuestionAttemptCounters");
|
||||
_questions = questions;
|
||||
_counters = mongo.Database.GetCollection<QuestionAttemptCounter>("QuestionChoiceExposureCounters");
|
||||
_state = mongo.Database.GetCollection<AnswerStatisticsState>("SystemState");
|
||||
_baselines = mongo.Database.GetCollection<QuestionPresentationBaseline>("QuestionPresentationBaselines");
|
||||
|
||||
var counterIndex = Builders<QuestionAttemptCounter>.IndexKeys
|
||||
.Ascending(counter => counter.QuestionExternalId)
|
||||
@@ -32,27 +40,31 @@ public sealed class AnswerStatisticsStore
|
||||
|
||||
public async Task IncrementAsync(QuestionAttempt attempt)
|
||||
{
|
||||
if (attempt.IsSkip)
|
||||
if (attempt.IsSkip || !attempt.TracksPresentation || attempt.PresentedChoiceIds.Count == 0)
|
||||
return;
|
||||
|
||||
var filter = Builders<QuestionAttemptCounter>.Filter.And(
|
||||
Builders<QuestionAttemptCounter>.Filter.Eq(counter => counter.QuestionExternalId, attempt.QuestionExternalId),
|
||||
Builders<QuestionAttemptCounter>.Filter.Eq(counter => counter.Bucket, attempt.CounterBucket),
|
||||
Builders<QuestionAttemptCounter>.Filter.Eq(counter => counter.ChoiceId, attempt.ChoiceId));
|
||||
var now = DateTime.UtcNow;
|
||||
var writes = attempt.PresentedChoiceIds.Distinct(StringComparer.Ordinal).Select(choiceId =>
|
||||
{
|
||||
var filter = CounterFilter(attempt.QuestionExternalId, attempt.CounterBucket, choiceId);
|
||||
var update = Builders<QuestionAttemptCounter>.Update
|
||||
.Inc(counter => counter.Count, 1)
|
||||
.Set(counter => counter.UpdatedUtc, DateTime.UtcNow)
|
||||
.Inc(counter => counter.PresentationCount, 1)
|
||||
.Inc(counter => counter.SelectionCount, choiceId == attempt.ChoiceId ? 1 : 0)
|
||||
.Set(counter => counter.UpdatedUtc, now)
|
||||
.SetOnInsert(counter => counter.QuestionExternalId, attempt.QuestionExternalId)
|
||||
.SetOnInsert(counter => counter.Bucket, attempt.CounterBucket)
|
||||
.SetOnInsert(counter => counter.ChoiceId, attempt.ChoiceId);
|
||||
.SetOnInsert(counter => counter.ChoiceId, choiceId);
|
||||
return new UpdateOneModel<QuestionAttemptCounter>(filter, update) { IsUpsert = true };
|
||||
}).ToList();
|
||||
|
||||
try
|
||||
{
|
||||
await _counters.UpdateOneAsync(filter, update, new UpdateOptions { IsUpsert = true });
|
||||
await _counters.BulkWriteAsync(writes, new BulkWriteOptions { IsOrdered = false });
|
||||
}
|
||||
catch (MongoWriteException exception) when (exception.WriteError.Category == ServerErrorCategory.DuplicateKey)
|
||||
catch (MongoBulkWriteException<QuestionAttemptCounter> exception)
|
||||
when (exception.WriteErrors.All(error => error.Category == ServerErrorCategory.DuplicateKey))
|
||||
{
|
||||
await _counters.UpdateOneAsync(filter, update);
|
||||
await _counters.BulkWriteAsync(writes, new BulkWriteOptions { IsOrdered = false });
|
||||
}
|
||||
_cache.TryRemove(attempt.QuestionExternalId, out _);
|
||||
}
|
||||
@@ -78,23 +90,9 @@ public sealed class AnswerStatisticsStore
|
||||
else
|
||||
missing.Add(id);
|
||||
}
|
||||
|
||||
if (missing.Count == 0)
|
||||
if (missing.Count == 0 || !await IsBackfillCompleteAsync())
|
||||
return output;
|
||||
|
||||
var backfillComplete = await IsBackfillCompleteAsync();
|
||||
if (!backfillComplete)
|
||||
{
|
||||
var rawPercentages = await _attempts.GetPercentagesByQuestionAsync(missing);
|
||||
foreach (var id in missing)
|
||||
{
|
||||
var percentages = rawPercentages.GetValueOrDefault(id, []);
|
||||
output[id] = percentages;
|
||||
_cache[id] = new CachedPercentages(percentages, now.Add(CacheDuration));
|
||||
}
|
||||
return output;
|
||||
}
|
||||
|
||||
var totals = await _counters.Aggregate()
|
||||
.Match(counter => missing.Contains(counter.QuestionExternalId))
|
||||
.Group(
|
||||
@@ -106,34 +104,22 @@ public sealed class AnswerStatisticsStore
|
||||
QuestionExternalId = group.Key.QuestionExternalId,
|
||||
ChoiceId = group.Key.ChoiceId
|
||||
},
|
||||
Count = group.Sum(counter => counter.Count)
|
||||
SelectionCount = group.Sum(counter => counter.SelectionCount),
|
||||
PresentationCount = group.Sum(counter => counter.PresentationCount)
|
||||
})
|
||||
.ToListAsync();
|
||||
|
||||
var grouped = totals.GroupBy(total => total.Key.QuestionExternalId)
|
||||
.ToDictionary(group => group.Key, group => group.ToList());
|
||||
|
||||
var grouped = totals.GroupBy(total => total.Key.QuestionExternalId).ToDictionary(group => group.Key);
|
||||
foreach (var id in missing)
|
||||
{
|
||||
Dictionary<string, int> percentages;
|
||||
if (grouped.TryGetValue(id, out var questionTotals))
|
||||
{
|
||||
var total = questionTotals.Sum(item => item.Count);
|
||||
percentages = total == 0
|
||||
? []
|
||||
: questionTotals.ToDictionary(
|
||||
var percentages = grouped.TryGetValue(id, out var choices)
|
||||
? choices.Where(item => item.PresentationCount > 0).ToDictionary(
|
||||
item => item.Key.ChoiceId,
|
||||
item => (int)Math.Round(item.Count * 100.0 / total));
|
||||
}
|
||||
else
|
||||
{
|
||||
percentages = [];
|
||||
}
|
||||
|
||||
item => (int)Math.Round(item.SelectionCount * 100.0 / item.PresentationCount))
|
||||
: [];
|
||||
output[id] = percentages;
|
||||
_cache[id] = new CachedPercentages(percentages, now.Add(CacheDuration));
|
||||
}
|
||||
|
||||
return output;
|
||||
}
|
||||
|
||||
@@ -149,15 +135,14 @@ public sealed class AnswerStatisticsStore
|
||||
var update = Builders<AnswerStatisticsState>.Update
|
||||
.SetOnInsert(state => state.Id, StateId)
|
||||
.SetOnInsert(state => state.Completed, false)
|
||||
.SetOnInsert(state => state.CutoverUtc, now)
|
||||
.Set(state => state.LeaseOwner, owner)
|
||||
.Set(state => state.LeaseUntilUtc, now.Add(duration))
|
||||
.Set(state => state.UpdatedUtc, now);
|
||||
|
||||
try
|
||||
{
|
||||
var leased = await _state.FindOneAndUpdateAsync(
|
||||
filter,
|
||||
update,
|
||||
var leased = await _state.FindOneAndUpdateAsync(filter, update,
|
||||
new FindOneAndUpdateOptions<AnswerStatisticsState>
|
||||
{
|
||||
IsUpsert = true,
|
||||
@@ -173,64 +158,131 @@ public sealed class AnswerStatisticsStore
|
||||
|
||||
public async Task MarkBackfillCompleteAsync(string owner)
|
||||
{
|
||||
var update = Builders<AnswerStatisticsState>.Update
|
||||
await _state.UpdateOneAsync(
|
||||
state => state.Id == StateId && state.LeaseOwner == owner,
|
||||
Builders<AnswerStatisticsState>.Update
|
||||
.Set(state => state.Completed, true)
|
||||
.Set(state => state.LeaseOwner, owner)
|
||||
.Set(state => state.UpdatedUtc, DateTime.UtcNow);
|
||||
await _state.UpdateOneAsync(state => state.Id == StateId && state.LeaseOwner == owner, update);
|
||||
.Set(state => state.UpdatedUtc, DateTime.UtcNow));
|
||||
_backfillComplete = true;
|
||||
}
|
||||
|
||||
public async Task RebuildAllAsync() => await RebuildAsync(null);
|
||||
public Task RebuildAllAsync() => RebuildAsync(null);
|
||||
|
||||
public async Task RebuildQuestionsAsync(IReadOnlyCollection<string> questionExternalIds)
|
||||
public Task RebuildQuestionsAsync(IReadOnlyCollection<string> questionExternalIds) =>
|
||||
questionExternalIds.Count == 0 ? Task.CompletedTask : RebuildAsync(questionExternalIds);
|
||||
|
||||
public async Task<bool> IsBackfillCompleteAsync()
|
||||
{
|
||||
if (questionExternalIds.Count > 0)
|
||||
await RebuildAsync(questionExternalIds);
|
||||
if (_backfillComplete)
|
||||
return true;
|
||||
_backfillComplete = (await _state.Find(state => state.Id == StateId).FirstOrDefaultAsync())?.Completed == true;
|
||||
return _backfillComplete;
|
||||
}
|
||||
|
||||
private async Task RebuildAsync(IReadOnlyCollection<string>? questionExternalIds)
|
||||
{
|
||||
var aggregate = _attempts.Collection.Aggregate().Match(attempt => !attempt.IsSkip);
|
||||
if (questionExternalIds is not null)
|
||||
aggregate = aggregate.Match(attempt => questionExternalIds.Contains(attempt.QuestionExternalId));
|
||||
|
||||
var totals = await aggregate
|
||||
.Group(
|
||||
attempt => new { attempt.QuestionExternalId, attempt.CounterBucket, attempt.ChoiceId },
|
||||
group => new QuestionAttemptCounterAggregate
|
||||
await EnsureBaselinesAsync();
|
||||
var state = await _state.Find(item => item.Id == StateId).FirstAsync();
|
||||
var baselines = await _baselines.Find(Builders<QuestionPresentationBaseline>.Filter.Empty).ToListAsync();
|
||||
var baselineByQuestion = baselines.ToDictionary(item => item.QuestionExternalId, item => item.ChoiceIds);
|
||||
var match = new BsonDocument
|
||||
{
|
||||
Key = new QuestionChoiceBucketKey
|
||||
{ "IsSkip", false },
|
||||
{ "$or", new BsonArray
|
||||
{
|
||||
QuestionExternalId = group.Key.QuestionExternalId,
|
||||
Bucket = group.Key.CounterBucket,
|
||||
ChoiceId = group.Key.ChoiceId
|
||||
},
|
||||
Count = group.Sum(_ => 1L)
|
||||
})
|
||||
.ToListAsync();
|
||||
|
||||
if (totals.Count > 0)
|
||||
{
|
||||
var now = DateTime.UtcNow;
|
||||
var replacements = totals.Select(total =>
|
||||
{
|
||||
var counter = new QuestionAttemptCounter
|
||||
{
|
||||
QuestionExternalId = total.Key.QuestionExternalId,
|
||||
Bucket = total.Key.Bucket,
|
||||
ChoiceId = total.Key.ChoiceId,
|
||||
Count = total.Count,
|
||||
UpdatedUtc = now
|
||||
};
|
||||
var filter = Builders<QuestionAttemptCounter>.Filter.And(
|
||||
Builders<QuestionAttemptCounter>.Filter.Eq(item => item.QuestionExternalId, counter.QuestionExternalId),
|
||||
Builders<QuestionAttemptCounter>.Filter.Eq(item => item.Bucket, counter.Bucket),
|
||||
Builders<QuestionAttemptCounter>.Filter.Eq(item => item.ChoiceId, counter.ChoiceId));
|
||||
return new ReplaceOneModel<QuestionAttemptCounter>(filter, counter) { IsUpsert = true };
|
||||
}).ToList();
|
||||
await _counters.BulkWriteAsync(replacements, new BulkWriteOptions { IsOrdered = false });
|
||||
new BsonDocument("TracksPresentation", true),
|
||||
new BsonDocument("CreatedUtc", new BsonDocument("$lt", state.CutoverUtc))
|
||||
}
|
||||
}
|
||||
};
|
||||
if (questionExternalIds is not null)
|
||||
match["QuestionExternalId"] = new BsonDocument("$in", new BsonArray(questionExternalIds));
|
||||
|
||||
var selectionPipeline = new[]
|
||||
{
|
||||
new BsonDocument("$match", match),
|
||||
new BsonDocument("$group", new BsonDocument
|
||||
{
|
||||
{ "_id", new BsonDocument { { "q", "$QuestionExternalId" }, { "b", "$CounterBucket" }, { "c", "$ChoiceId" } } },
|
||||
{ "count", new BsonDocument("$sum", 1) }
|
||||
})
|
||||
};
|
||||
var presentationMatch = new BsonDocument(match) { ["TracksPresentation"] = true };
|
||||
var presentationPipeline = new[]
|
||||
{
|
||||
new BsonDocument("$match", presentationMatch),
|
||||
new BsonDocument("$unwind", "$PresentedChoiceIds"),
|
||||
new BsonDocument("$group", new BsonDocument
|
||||
{
|
||||
{ "_id", new BsonDocument { { "q", "$QuestionExternalId" }, { "b", "$CounterBucket" }, { "c", "$PresentedChoiceIds" } } },
|
||||
{ "count", new BsonDocument("$sum", 1) }
|
||||
})
|
||||
};
|
||||
var legacyMatch = new BsonDocument
|
||||
{
|
||||
{ "IsSkip", false },
|
||||
{ "TracksPresentation", new BsonDocument("$ne", true) },
|
||||
{ "CreatedUtc", new BsonDocument("$lt", state.CutoverUtc) }
|
||||
};
|
||||
if (questionExternalIds is not null)
|
||||
legacyMatch["QuestionExternalId"] = new BsonDocument("$in", new BsonArray(questionExternalIds));
|
||||
var legacyPipeline = new[]
|
||||
{
|
||||
new BsonDocument("$match", legacyMatch),
|
||||
new BsonDocument("$group", new BsonDocument
|
||||
{
|
||||
{ "_id", new BsonDocument { { "q", "$QuestionExternalId" }, { "b", "$CounterBucket" } } },
|
||||
{ "count", new BsonDocument("$sum", 1) }
|
||||
})
|
||||
};
|
||||
|
||||
var selections = await _attempts.Collection.Aggregate<BsonDocument>(selectionPipeline).ToListAsync();
|
||||
var presentations = await _attempts.Collection.Aggregate<BsonDocument>(presentationPipeline).ToListAsync();
|
||||
var legacy = await _attempts.Collection.Aggregate<BsonDocument>(legacyPipeline).ToListAsync();
|
||||
var totals = new Dictionary<(string Question, int Bucket, string Choice), (long Selected, long Presented)>();
|
||||
|
||||
foreach (var row in selections)
|
||||
{
|
||||
var key = CounterKey(row["_id"].AsBsonDocument);
|
||||
totals[key] = (row["count"].ToInt64(), totals.GetValueOrDefault(key).Presented);
|
||||
}
|
||||
foreach (var row in presentations)
|
||||
{
|
||||
var key = CounterKey(row["_id"].AsBsonDocument);
|
||||
totals[key] = (totals.GetValueOrDefault(key).Selected, row["count"].ToInt64());
|
||||
}
|
||||
foreach (var row in legacy)
|
||||
{
|
||||
var id = row["_id"].AsBsonDocument;
|
||||
var questionId = id["q"].AsString;
|
||||
if (!baselineByQuestion.TryGetValue(questionId, out var choiceIds))
|
||||
continue;
|
||||
foreach (var choiceId in choiceIds)
|
||||
{
|
||||
var key = (questionId, id["b"].ToInt32(), choiceId);
|
||||
totals[key] = (totals.GetValueOrDefault(key).Selected,
|
||||
totals.GetValueOrDefault(key).Presented + row["count"].ToInt64());
|
||||
}
|
||||
}
|
||||
|
||||
var now = DateTime.UtcNow;
|
||||
var writes = totals.Select(item =>
|
||||
new ReplaceOneModel<QuestionAttemptCounter>(
|
||||
CounterFilter(item.Key.Question, item.Key.Bucket, item.Key.Choice),
|
||||
new QuestionAttemptCounter
|
||||
{
|
||||
QuestionExternalId = item.Key.Question,
|
||||
Bucket = item.Key.Bucket,
|
||||
ChoiceId = item.Key.Choice,
|
||||
SelectionCount = item.Value.Selected,
|
||||
PresentationCount = item.Value.Presented,
|
||||
UpdatedUtc = now
|
||||
})
|
||||
{
|
||||
IsUpsert = true
|
||||
}).ToList();
|
||||
if (writes.Count > 0)
|
||||
await _counters.BulkWriteAsync(writes, new BulkWriteOptions { IsOrdered = false });
|
||||
|
||||
if (questionExternalIds is null)
|
||||
_cache.Clear();
|
||||
@@ -239,13 +291,31 @@ public sealed class AnswerStatisticsStore
|
||||
_cache.TryRemove(id, out _);
|
||||
}
|
||||
|
||||
private async Task<bool> IsBackfillCompleteAsync()
|
||||
private async Task EnsureBaselinesAsync()
|
||||
{
|
||||
if (_backfillComplete)
|
||||
return true;
|
||||
_backfillComplete = (await _state.Find(state => state.Id == StateId).FirstOrDefaultAsync())?.Completed == true;
|
||||
return _backfillComplete;
|
||||
var questions = await _questions.GetAllAsync();
|
||||
var writes = questions.Select(question =>
|
||||
new UpdateOneModel<QuestionPresentationBaseline>(
|
||||
Builders<QuestionPresentationBaseline>.Filter.Eq(item => item.QuestionExternalId, question.ExternalId),
|
||||
Builders<QuestionPresentationBaseline>.Update
|
||||
.SetOnInsert(item => item.QuestionExternalId, question.ExternalId)
|
||||
.SetOnInsert(item => item.ChoiceIds, question.Choices.Select(choice => choice.Id).ToList())
|
||||
.SetOnInsert(item => item.CreatedUtc, DateTime.UtcNow))
|
||||
{
|
||||
IsUpsert = true
|
||||
}).ToList();
|
||||
if (writes.Count > 0)
|
||||
await _baselines.BulkWriteAsync(writes, new BulkWriteOptions { IsOrdered = false });
|
||||
}
|
||||
|
||||
private static FilterDefinition<QuestionAttemptCounter> CounterFilter(string questionId, int bucket, string choiceId) =>
|
||||
Builders<QuestionAttemptCounter>.Filter.And(
|
||||
Builders<QuestionAttemptCounter>.Filter.Eq(counter => counter.QuestionExternalId, questionId),
|
||||
Builders<QuestionAttemptCounter>.Filter.Eq(counter => counter.Bucket, bucket),
|
||||
Builders<QuestionAttemptCounter>.Filter.Eq(counter => counter.ChoiceId, choiceId));
|
||||
|
||||
private static (string Question, int Bucket, string Choice) CounterKey(BsonDocument id) =>
|
||||
(id["q"].AsString, id["b"].ToInt32(), id["c"].AsString);
|
||||
|
||||
private sealed record CachedPercentages(Dictionary<string, int> Percentages, DateTime ExpiresUtc);
|
||||
}
|
||||
|
||||
@@ -34,13 +34,16 @@ public class QuestionAttemptStore
|
||||
string userId,
|
||||
string choiceId,
|
||||
bool isCorrect,
|
||||
bool isSkip = false)
|
||||
bool isSkip = false,
|
||||
IReadOnlyCollection<string>? presentedChoiceIds = null)
|
||||
{
|
||||
var attempt = new QuestionAttempt
|
||||
{
|
||||
QuestionExternalId = questionExternalId,
|
||||
UserId = userId,
|
||||
ChoiceId = choiceId,
|
||||
PresentedChoiceIds = presentedChoiceIds?.ToList() ?? [],
|
||||
TracksPresentation = presentedChoiceIds is { Count: > 0 },
|
||||
IsCorrect = isCorrect,
|
||||
IsSkip = isSkip,
|
||||
CounterBucket = GetCounterBucket(userId),
|
||||
@@ -62,49 +65,6 @@ public class QuestionAttemptStore
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<Dictionary<string, int>> GetPercentagesForQuestionAsync(string questionExternalId)
|
||||
{
|
||||
var results = await GetPercentagesByQuestionAsync([questionExternalId]);
|
||||
return results.GetValueOrDefault(questionExternalId, []);
|
||||
}
|
||||
|
||||
public async Task<Dictionary<string, Dictionary<string, int>>> GetPercentagesByQuestionAsync(IEnumerable<string> questionExternalIds)
|
||||
{
|
||||
var ids = questionExternalIds.Where(id => !string.IsNullOrWhiteSpace(id)).Distinct().ToList();
|
||||
if (ids.Count == 0)
|
||||
return [];
|
||||
|
||||
var totals = await _col.Aggregate()
|
||||
.Match(a => ids.Contains(a.QuestionExternalId) && !a.IsSkip)
|
||||
.Group(
|
||||
attempt => new { attempt.QuestionExternalId, attempt.ChoiceId },
|
||||
group => new QuestionAttemptCounterTotal
|
||||
{
|
||||
Key = new QuestionChoiceCounterKey
|
||||
{
|
||||
QuestionExternalId = group.Key.QuestionExternalId,
|
||||
ChoiceId = group.Key.ChoiceId
|
||||
},
|
||||
Count = group.Sum(_ => 1L)
|
||||
})
|
||||
.ToListAsync();
|
||||
var output = new Dictionary<string, Dictionary<string, int>>();
|
||||
|
||||
foreach (var group in totals.GroupBy(total => total.Key.QuestionExternalId))
|
||||
{
|
||||
var total = group.Sum(item => item.Count);
|
||||
if (total == 0)
|
||||
continue;
|
||||
|
||||
output[group.Key] = group
|
||||
.ToDictionary(
|
||||
item => item.Key.ChoiceId,
|
||||
item => (int)Math.Round(item.Count * 100.0 / total));
|
||||
}
|
||||
|
||||
return output;
|
||||
}
|
||||
|
||||
public Task<List<string>> GetActiveQuestionIdsAsync(DateTime sinceUtc) =>
|
||||
_col.Distinct(
|
||||
attempt => attempt.QuestionExternalId,
|
||||
|
||||
@@ -50,6 +50,32 @@ public class QuestionStore
|
||||
public async Task<Question?> GetByExternalIdAsync(string externalId) =>
|
||||
await _col.Find(q => q.ExternalId == externalId).FirstOrDefaultAsync();
|
||||
|
||||
public async Task<List<Question>> GetAllAsync() =>
|
||||
await _col.Find(Builders<Question>.Filter.Empty).ToListAsync();
|
||||
|
||||
public async Task<(List<Question> Items, string? NextCursor, bool HasMore)> ExportPageAsync(
|
||||
string? after,
|
||||
int requestedLimit)
|
||||
{
|
||||
var limit = Math.Clamp(requestedLimit, 1, 200);
|
||||
var filter = Builders<Question>.Filter.Empty;
|
||||
if (!string.IsNullOrWhiteSpace(after))
|
||||
{
|
||||
if (!ObjectId.TryParse(after, out var cursor))
|
||||
throw new ArgumentException("after must be a valid question cursor", nameof(after));
|
||||
filter = new BsonDocument("_id", new BsonDocument("$gt", cursor));
|
||||
}
|
||||
|
||||
var page = await _col.Find(filter)
|
||||
.Sort(Builders<Question>.Sort.Ascending("_id"))
|
||||
.Limit(limit + 1)
|
||||
.ToListAsync();
|
||||
var hasMore = page.Count > limit;
|
||||
if (hasMore)
|
||||
page.RemoveAt(page.Count - 1);
|
||||
return (page, hasMore ? page[^1].Id : null, hasMore);
|
||||
}
|
||||
|
||||
public async Task<Question?> CreateAsync(UpsertQuestionRequest req)
|
||||
{
|
||||
var question = FromRequest(req);
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
using QuestionsApi.Models;
|
||||
|
||||
namespace QuestionsApi.Services;
|
||||
|
||||
public static class QuestionValidator
|
||||
{
|
||||
public static string? Validate(UpsertQuestionRequest req)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(req.ExternalId)) return "externalId required";
|
||||
if (string.IsNullOrWhiteSpace(req.Stem)) return "stem required";
|
||||
if (string.IsNullOrWhiteSpace(req.LeadIn)) return "leadIn required";
|
||||
if (req.Choices.Count < (req.IsPublished ? 5 : 2))
|
||||
return req.IsPublished ? "published questions require at least five choices" : "at least two choices required";
|
||||
if (req.Choices.Count > 20) return "questions support at most twenty choices";
|
||||
if (req.Choices.Any(choice => string.IsNullOrWhiteSpace(choice.Id) || string.IsNullOrWhiteSpace(choice.Text)))
|
||||
return "choice ids and text are required";
|
||||
|
||||
var choiceIds = req.Choices.Select(choice => choice.Id.Trim()).ToList();
|
||||
if (choiceIds.Distinct(StringComparer.Ordinal).Count() != choiceIds.Count)
|
||||
return "choice ids must be unique";
|
||||
if (string.IsNullOrWhiteSpace(req.CorrectChoiceId)) return "correctChoiceId required";
|
||||
if (!choiceIds.Contains(req.CorrectChoiceId.Trim(), StringComparer.Ordinal))
|
||||
return "correctChoiceId must match a choice id";
|
||||
if (string.IsNullOrWhiteSpace(req.Explanation)) return "explanation required";
|
||||
|
||||
var choiceIdSet = choiceIds.ToHashSet(StringComparer.Ordinal);
|
||||
if (!choiceIdSet.SetEquals(req.ChoiceExplanations.Keys))
|
||||
return "choiceExplanations must contain exactly one entry for every choice";
|
||||
var incorrectChoiceIds = choiceIdSet
|
||||
.Where(id => id != req.CorrectChoiceId.Trim())
|
||||
.ToHashSet(StringComparer.Ordinal);
|
||||
if (!incorrectChoiceIds.SetEquals(req.WrongAnswerFollowups.Keys))
|
||||
return "wrongAnswerFollowups must contain exactly one entry for every incorrect choice";
|
||||
if (req.ChoiceSelectionPercentages.Keys.Any(id => !choiceIdSet.Contains(id)))
|
||||
return "choiceSelectionPercentages keys must match choice ids";
|
||||
if (req.ChoiceSelectionPercentages.Any(entry => entry.Value is < 0 or > 100))
|
||||
return "choiceSelectionPercentages values must be between 0 and 100";
|
||||
|
||||
foreach (var entry in req.WrongAnswerFollowups)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(entry.Value.Prompt)) return "wrongAnswerFollowups prompt required";
|
||||
if (string.IsNullOrWhiteSpace(entry.Value.TeachingPoint)) return "wrongAnswerFollowups teachingPoint required";
|
||||
if (entry.Value.Choices.Count == 0) return "wrongAnswerFollowups choices required";
|
||||
if (entry.Value.Choices.Any(choice => string.IsNullOrWhiteSpace(choice.Id) || string.IsNullOrWhiteSpace(choice.Text)))
|
||||
return "wrongAnswerFollowups choice ids and text are required";
|
||||
if (entry.Value.Choices.Select(choice => choice.Id).Distinct(StringComparer.Ordinal).Count() != entry.Value.Choices.Count)
|
||||
return "wrongAnswerFollowups choice ids must be unique";
|
||||
if (!entry.Value.Choices.Any(choice => choice.Id == entry.Value.CorrectChoiceId))
|
||||
return "wrongAnswerFollowups correctChoiceId must match a follow-up choice id";
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(req.System)) return "system required";
|
||||
if (string.IsNullOrWhiteSpace(req.Discipline)) return "discipline required";
|
||||
if (req.Difficulty is < 1 or > 5) return "difficulty must be between 1 and 5";
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -12,11 +12,13 @@ public sealed class ServiceMetrics
|
||||
private long _lastReconciliationUnixSeconds;
|
||||
private long _httpRequests;
|
||||
private long _httpDurationMicroseconds;
|
||||
private long _answerStatisticsBackfillComplete;
|
||||
|
||||
public void AttemptCreated() => Interlocked.Increment(ref _attemptsCreated);
|
||||
public void AttemptDuplicate() => Interlocked.Increment(ref _attemptDuplicates);
|
||||
public void CounterFailure() => Interlocked.Increment(ref _counterFailures);
|
||||
public void RatingFailure() => Interlocked.Increment(ref _ratingFailures);
|
||||
public void AnswerStatisticsBackfillCompleted() => Interlocked.Exchange(ref _answerStatisticsBackfillComplete, 1);
|
||||
public void Reconciled()
|
||||
{
|
||||
Interlocked.Increment(ref _reconciliations);
|
||||
@@ -48,6 +50,10 @@ public sealed class ServiceMetrics
|
||||
output.Append("step3_answer_last_reconciliation_unixtime ")
|
||||
.Append(Interlocked.Read(ref _lastReconciliationUnixSeconds))
|
||||
.AppendLine();
|
||||
output.AppendLine("# TYPE step3_answer_statistics_v2_backfill_complete gauge");
|
||||
output.Append("step3_answer_statistics_v2_backfill_complete ")
|
||||
.Append(Interlocked.Read(ref _answerStatisticsBackfillComplete))
|
||||
.AppendLine();
|
||||
return output.ToString();
|
||||
}
|
||||
|
||||
|
||||
@@ -30,6 +30,8 @@ func start_block(selected_system: String, block_size: int, selected_mode: String
|
||||
mode = selected_mode
|
||||
session_kind = "block"
|
||||
active_questions = QuestionBankService.build_block(selected_system, block_size)
|
||||
for index in range(active_questions.size()):
|
||||
active_questions[index] = _prepare_question_choices(active_questions[index])
|
||||
answers.clear()
|
||||
answers.resize(active_questions.size())
|
||||
current_index = 0
|
||||
@@ -293,10 +295,38 @@ func _draw_core_question() -> Dictionary:
|
||||
var candidate: Dictionary = core_question_pool[index]
|
||||
if str(candidate.get("id", "")) != last_id or core_question_pool.size() == 1:
|
||||
core_question_pool.remove_at(index)
|
||||
return candidate.duplicate(true)
|
||||
return _prepare_question_choices(candidate.duplicate(true))
|
||||
return {}
|
||||
|
||||
|
||||
func _prepare_question_choices(question: Dictionary) -> Dictionary:
|
||||
var existing: Array = question.get("presented_choices", [])
|
||||
if not existing.is_empty():
|
||||
return question
|
||||
|
||||
var all_choices: Array = question.get("choices", []).duplicate(true)
|
||||
var correct_id := str(question.get("correct_choice_id", ""))
|
||||
var correct_choice: Dictionary = {}
|
||||
var distractors: Array = []
|
||||
for choice in all_choices:
|
||||
if typeof(choice) != TYPE_DICTIONARY:
|
||||
continue
|
||||
if str(choice.get("id", "")) == correct_id:
|
||||
correct_choice = choice
|
||||
else:
|
||||
distractors.append(choice)
|
||||
|
||||
distractors.shuffle()
|
||||
var presented: Array = distractors.slice(0, min(4, distractors.size()))
|
||||
if not correct_choice.is_empty():
|
||||
presented.append(correct_choice)
|
||||
elif not all_choices.is_empty():
|
||||
presented = all_choices.slice(0, min(5, all_choices.size()))
|
||||
presented.shuffle()
|
||||
question["presented_choices"] = presented
|
||||
return question
|
||||
|
||||
|
||||
func _build_core_question_pool() -> Array:
|
||||
var pool: Array = []
|
||||
var profile := ProfileService.get_profile()
|
||||
|
||||
@@ -103,7 +103,8 @@ func load_questions_from_api(reset_retry: bool = true) -> void:
|
||||
"Authorization: Bearer %s" % AuthState.access_token
|
||||
])
|
||||
var payload := {
|
||||
"limit": 80
|
||||
"limit": 80,
|
||||
"supportsChoicePools": true
|
||||
}
|
||||
var err := request.request(questions_api_url, headers, HTTPClient.METHOD_POST, JSON.stringify(payload))
|
||||
if err != OK:
|
||||
@@ -177,10 +178,10 @@ func clear_questions() -> void:
|
||||
ratings_changed.emit()
|
||||
|
||||
|
||||
func record_first_attempt(question_id: String, choice_id: String) -> void:
|
||||
func record_first_attempt(question_id: String, choice_id: String, presented_choice_ids: Array = []) -> void:
|
||||
if AuthState.access_token.is_empty() or question_id == "" or choice_id == "":
|
||||
return
|
||||
_enqueue_attempt("answer", question_id, choice_id)
|
||||
_enqueue_attempt("answer", question_id, choice_id, presented_choice_ids)
|
||||
|
||||
|
||||
func skip_question(question_id: String) -> void:
|
||||
@@ -294,7 +295,7 @@ func _on_retry_timer_timeout() -> void:
|
||||
load_questions_from_api(false)
|
||||
|
||||
|
||||
func _enqueue_attempt(kind: String, question_id: String, choice_id: String) -> void:
|
||||
func _enqueue_attempt(kind: String, question_id: String, choice_id: String, presented_choice_ids: Array = []) -> void:
|
||||
var account := AuthState.username.strip_edges()
|
||||
if account == "":
|
||||
return
|
||||
@@ -306,6 +307,7 @@ func _enqueue_attempt(kind: String, question_id: String, choice_id: String) -> v
|
||||
"kind": kind,
|
||||
"question_id": question_id,
|
||||
"choice_id": choice_id,
|
||||
"presented_choice_ids": presented_choice_ids.duplicate(),
|
||||
"retry_count": 0
|
||||
})
|
||||
_save_attempt_queue()
|
||||
@@ -335,7 +337,8 @@ func _process_attempt_queue() -> void:
|
||||
var url := question_attempt_api_url
|
||||
var body := JSON.stringify({
|
||||
"questionExternalId": question_id,
|
||||
"choiceId": str(queued.get("choice_id", ""))
|
||||
"choiceId": str(queued.get("choice_id", "")),
|
||||
"presentedChoiceIds": queued.get("presented_choice_ids", [])
|
||||
})
|
||||
if str(queued.get("kind", "answer")) == "skip":
|
||||
url = "%s/%s/skip" % [question_api_base_url.rstrip("/"), question_id.uri_encode()]
|
||||
@@ -457,6 +460,7 @@ func _load_attempt_queue() -> void:
|
||||
"kind": kind,
|
||||
"question_id": question_id,
|
||||
"choice_id": str(item.get("choice_id", "")),
|
||||
"presented_choice_ids": item.get("presented_choice_ids", []).duplicate(),
|
||||
"retry_count": max(0, int(item.get("retry_count", 0)))
|
||||
})
|
||||
|
||||
|
||||
@@ -442,8 +442,7 @@ func _load_question() -> void:
|
||||
lead_label.text = question.get("lead_in", "")
|
||||
_refresh_labs_button()
|
||||
|
||||
presented_choices = question.get("choices", []).duplicate(true)
|
||||
presented_choices.shuffle()
|
||||
presented_choices = question.get("presented_choices", question.get("choices", [])).duplicate(true)
|
||||
var choices := presented_choices
|
||||
for index in range(choice_buttons.size()):
|
||||
var button := choice_buttons[index]
|
||||
@@ -613,7 +612,11 @@ func _submit_or_next() -> void:
|
||||
_collapse_resolved_followups()
|
||||
var attempted_choice := selected_choice
|
||||
var attempted_question_id := str(BlockSessionService.get_current_question().get("id", ""))
|
||||
QuestionBankService.record_first_attempt(attempted_question_id, attempted_choice)
|
||||
QuestionBankService.record_first_attempt(
|
||||
attempted_question_id,
|
||||
attempted_choice,
|
||||
_presented_choice_ids()
|
||||
)
|
||||
var result := BlockSessionService.submit_core_attempt(selected_choice, eliminated, flagged)
|
||||
if result.is_empty():
|
||||
return
|
||||
@@ -632,6 +635,12 @@ func _submit_or_next() -> void:
|
||||
_trigger_wrong_feedback()
|
||||
_apply_saved_choice_state()
|
||||
return
|
||||
var attempted_question_id := str(BlockSessionService.get_current_question().get("id", ""))
|
||||
QuestionBankService.record_first_attempt(
|
||||
attempted_question_id,
|
||||
selected_choice,
|
||||
_presented_choice_ids()
|
||||
)
|
||||
current_answer = BlockSessionService.answer_current(selected_choice, eliminated, flagged)
|
||||
if BlockSessionService.mode == "Tutor":
|
||||
_show_explanation()
|
||||
@@ -1871,6 +1880,14 @@ func _get_presented_choices() -> Array:
|
||||
return BlockSessionService.get_current_question().get("choices", [])
|
||||
|
||||
|
||||
func _presented_choice_ids() -> Array:
|
||||
var ids: Array = []
|
||||
for choice in _get_presented_choices():
|
||||
if typeof(choice) == TYPE_DICTIONARY:
|
||||
ids.append(str(choice.get("id", "")))
|
||||
return ids
|
||||
|
||||
|
||||
func _display_label_for_index(index: int) -> String:
|
||||
return String.chr(65 + index)
|
||||
|
||||
|
||||
@@ -7,6 +7,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "api", "api", "{D82C56D6-B2C
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "QuestionsApi", "api\QuestionsApi\QuestionsApi.csproj", "{0B2F6FEF-3C08-4A72-BBA9-90C225BC24EC}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "QuestionsApi.Tests", "api\QuestionsApi.Tests\QuestionsApi.Tests.csproj", "{1897C6D6-452D-4142-BCCC-FA1A46E7C869}"
|
||||
EndProject
|
||||
Global
|
||||
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||
Debug|Any CPU = Debug|Any CPU
|
||||
@@ -29,11 +31,24 @@ Global
|
||||
{0B2F6FEF-3C08-4A72-BBA9-90C225BC24EC}.Release|x64.Build.0 = Release|Any CPU
|
||||
{0B2F6FEF-3C08-4A72-BBA9-90C225BC24EC}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{0B2F6FEF-3C08-4A72-BBA9-90C225BC24EC}.Release|x86.Build.0 = Release|Any CPU
|
||||
{1897C6D6-452D-4142-BCCC-FA1A46E7C869}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{1897C6D6-452D-4142-BCCC-FA1A46E7C869}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{1897C6D6-452D-4142-BCCC-FA1A46E7C869}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||
{1897C6D6-452D-4142-BCCC-FA1A46E7C869}.Debug|x64.Build.0 = Debug|Any CPU
|
||||
{1897C6D6-452D-4142-BCCC-FA1A46E7C869}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{1897C6D6-452D-4142-BCCC-FA1A46E7C869}.Debug|x86.Build.0 = Debug|Any CPU
|
||||
{1897C6D6-452D-4142-BCCC-FA1A46E7C869}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{1897C6D6-452D-4142-BCCC-FA1A46E7C869}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{1897C6D6-452D-4142-BCCC-FA1A46E7C869}.Release|x64.ActiveCfg = Release|Any CPU
|
||||
{1897C6D6-452D-4142-BCCC-FA1A46E7C869}.Release|x64.Build.0 = Release|Any CPU
|
||||
{1897C6D6-452D-4142-BCCC-FA1A46E7C869}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{1897C6D6-452D-4142-BCCC-FA1A46E7C869}.Release|x86.Build.0 = Release|Any CPU
|
||||
EndGlobalSection
|
||||
GlobalSection(SolutionProperties) = preSolution
|
||||
HideSolutionNode = FALSE
|
||||
EndGlobalSection
|
||||
GlobalSection(NestedProjects) = preSolution
|
||||
{0B2F6FEF-3C08-4A72-BBA9-90C225BC24EC} = {D82C56D6-B2CD-F611-9E31-04638F7EC30E}
|
||||
{1897C6D6-452D-4142-BCCC-FA1A46E7C869} = {D82C56D6-B2CD-F611-9E31-04638F7EC30E}
|
||||
EndGlobalSection
|
||||
EndGlobal
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
extends SceneTree
|
||||
|
||||
var failures := 0
|
||||
|
||||
|
||||
func _initialize() -> void:
|
||||
var service = load("res://scripts/autoload/block_session_service.gd").new()
|
||||
var question := _question(9)
|
||||
var prepared: Dictionary = service._prepare_question_choices(question.duplicate(true))
|
||||
_check_presented(prepared)
|
||||
|
||||
var stable: Dictionary = service._prepare_question_choices(prepared.duplicate(true))
|
||||
_assert(stable.get("presented_choices", []) == prepared.get("presented_choices", []), "session choices must remain stable")
|
||||
|
||||
var observed_sets := {}
|
||||
for iteration in range(40):
|
||||
var rerolled: Dictionary = service._prepare_question_choices(question.duplicate(true))
|
||||
_check_presented(rerolled)
|
||||
var ids := _ids(rerolled.get("presented_choices", []))
|
||||
ids.sort()
|
||||
observed_sets[",".join(ids)] = true
|
||||
_assert(observed_sets.size() > 1, "future sessions should be able to present different distractors")
|
||||
|
||||
var legacy: Dictionary = service._prepare_question_choices(_question(5))
|
||||
_check_presented(legacy)
|
||||
_assert(_ids(legacy.get("presented_choices", [])).size() == 5, "legacy five-choice questions must show all choices")
|
||||
service.free()
|
||||
quit(1 if failures > 0 else 0)
|
||||
|
||||
|
||||
func _question(count: int) -> Dictionary:
|
||||
var choices: Array = []
|
||||
for index in range(count):
|
||||
choices.append({"id": String.chr(65 + index), "text": "Choice %d" % index})
|
||||
return {"choices": choices, "correct_choice_id": "A"}
|
||||
|
||||
|
||||
func _check_presented(question: Dictionary) -> void:
|
||||
var ids := _ids(question.get("presented_choices", []))
|
||||
_assert(ids.size() == 5, "exactly five choices must be presented")
|
||||
_assert(ids.has("A"), "the correct answer must be presented")
|
||||
var unique := {}
|
||||
for id in ids:
|
||||
unique[id] = true
|
||||
_assert(unique.size() == ids.size(), "presented choices must be unique")
|
||||
|
||||
|
||||
func _ids(choices: Array) -> Array:
|
||||
var ids: Array = []
|
||||
for choice in choices:
|
||||
ids.append(str(choice.get("id", "")))
|
||||
return ids
|
||||
|
||||
|
||||
func _assert(condition: bool, message: String) -> void:
|
||||
if condition:
|
||||
return
|
||||
failures += 1
|
||||
push_error(message)
|
||||
@@ -0,0 +1 @@
|
||||
uid://dphnt1gskhhna
|
||||
Reference in New Issue
Block a user