Add variable question distractor pools

api/QuestionsApi/Controllers/QuestionsController.cs

@@ -52,10 +52,34 @@ public class QuestionsController : ControllerBase
         return Ok(questions.Select(q =>
         {
             percentages.TryGetValue(q.ExternalId, out var questionPercentages);
-            return QuestionResponse.FromModel(q, questionPercentages, questionElos.GetValueOrDefault(q.ExternalId, 1000));
+            return QuestionResponse.FromModel(
+                q,
+                questionPercentages,
+                questionElos.GetValueOrDefault(q.ExternalId, 1000),
+                req.SupportsChoicePools);
         }).ToList());
     }
 
+    [HttpGet("admin/export")]
+    [Authorize(Roles = "SUPER")]
+    public async Task<IActionResult> AdminExport([FromQuery] string? after = null, [FromQuery] int limit = 100)
+    {
+        try
+        {
+            var page = await _questions.ExportPageAsync(after, limit);
+            return Ok(new QuestionAdminExportResponse
+            {
+                Items = page.Items.Select(question => QuestionResponse.FromModel(question)).ToList(),
+                NextCursor = page.NextCursor,
+                HasMore = page.HasMore
+            });
+        }
+        catch (ArgumentException exception)
+        {
+            return BadRequest(exception.Message);
+        }
+    }
+
     [HttpGet("{id}")]
     [Authorize(Roles = "USER,SUPER")]
     public async Task<IActionResult> Get(string id)
@@ -84,6 +108,22 @@ public class QuestionsController : ControllerBase
         if (!question.Choices.Any(c => c.Id == req.ChoiceId.Trim()))
             return BadRequest("choiceId must match a choice id");
 
+        var presentedChoiceIds = req.PresentedChoiceIds
+            .Where(id => !string.IsNullOrWhiteSpace(id))
+            .Select(id => id.Trim())
+            .ToList();
+        if (presentedChoiceIds.Count > 0)
+        {
+            var validChoiceIds = question.Choices.Select(choice => choice.Id).ToHashSet(StringComparer.Ordinal);
+            if (presentedChoiceIds.Count != 5 || presentedChoiceIds.Distinct(StringComparer.Ordinal).Count() != 5)
+                return BadRequest("presentedChoiceIds must contain exactly five unique choice ids");
+            if (presentedChoiceIds.Any(id => !validChoiceIds.Contains(id)))
+                return BadRequest("presentedChoiceIds must match question choice ids");
+            if (!presentedChoiceIds.Contains(req.ChoiceId.Trim(), StringComparer.Ordinal)
+                || !presentedChoiceIds.Contains(question.CorrectChoiceId, StringComparer.Ordinal))
+                return BadRequest("presentedChoiceIds must include the selected and correct choices");
+        }
+
         var userId = User.FindFirstValue(ClaimTypes.NameIdentifier)
             ?? User.FindFirstValue(ClaimTypes.Name)
             ?? User.Identity?.Name
@@ -95,7 +135,9 @@ public class QuestionsController : ControllerBase
             question.ExternalId,
             userId,
             req.ChoiceId.Trim(),
-            req.ChoiceId.Trim() == question.CorrectChoiceId);
+            req.ChoiceId.Trim() == question.CorrectChoiceId,
+            false,
+            presentedChoiceIds);
         if (record.Inserted)
         {
             _metrics.AttemptCreated();
@@ -252,7 +294,7 @@ public class QuestionsController : ControllerBase
     [Authorize(Roles = "SUPER")]
     public async Task<IActionResult> Create([FromBody] UpsertQuestionRequest req)
     {
-        var validation = ValidateQuestion(req);
+        var validation = QuestionValidator.Validate(req);
         if (validation is not null)
             return BadRequest(validation);
 
@@ -272,7 +314,7 @@ public class QuestionsController : ControllerBase
 
         foreach (var question in req.Questions)
         {
-            var validation = ValidateQuestion(question);
+            var validation = QuestionValidator.Validate(question);
             if (validation is not null)
                 return BadRequest($"{question.ExternalId}: {validation}");
         }
@@ -285,7 +327,7 @@ public class QuestionsController : ControllerBase
     [Authorize(Roles = "SUPER")]
     public async Task<IActionResult> Update(string id, [FromBody] UpsertQuestionRequest req)
     {
-        var validation = ValidateQuestion(req);
+        var validation = QuestionValidator.Validate(req);
         if (validation is not null)
             return BadRequest(validation);
 
@@ -314,59 +356,6 @@ public class QuestionsController : ControllerBase
         return Ok("Deleted");
     }
 
-    private static string? ValidateQuestion(UpsertQuestionRequest req)
-    {
-        if (string.IsNullOrWhiteSpace(req.ExternalId))
-            return "externalId required";
-        if (string.IsNullOrWhiteSpace(req.Stem))
-            return "stem required";
-        if (string.IsNullOrWhiteSpace(req.LeadIn))
-            return "leadIn required";
-        if (req.Choices.Count < 2)
-            return "at least two choices required";
-        if (string.IsNullOrWhiteSpace(req.CorrectChoiceId))
-            return "correctChoiceId required";
-        if (!req.Choices.Any(c => c.Id == req.CorrectChoiceId))
-            return "correctChoiceId must match a choice id";
-        if (string.IsNullOrWhiteSpace(req.Explanation))
-            return "explanation required";
-        if (req.ChoiceSelectionPercentages.Any(entry => entry.Value is < 0 or > 100))
-            return "choiceSelectionPercentages values must be between 0 and 100";
-        var followupValidation = ValidateWrongAnswerFollowups(req);
-        if (followupValidation is not null)
-            return followupValidation;
-        if (string.IsNullOrWhiteSpace(req.System))
-            return "system required";
-        if (string.IsNullOrWhiteSpace(req.Discipline))
-            return "discipline required";
-        if (req.Difficulty is < 1 or > 5)
-            return "difficulty must be between 1 and 5";
-
-        return null;
-    }
-
-    private static string? ValidateWrongAnswerFollowups(UpsertQuestionRequest req)
-    {
-        var choiceIds = req.Choices.Select(c => c.Id).Where(id => !string.IsNullOrWhiteSpace(id)).ToHashSet();
-        foreach (var entry in req.WrongAnswerFollowups)
-        {
-            if (!choiceIds.Contains(entry.Key))
-                return "wrongAnswerFollowups keys must match a choice id";
-            if (entry.Key == req.CorrectChoiceId)
-                return "wrongAnswerFollowups should only target incorrect choices";
-            if (string.IsNullOrWhiteSpace(entry.Value.Prompt))
-                return "wrongAnswerFollowups prompt required";
-            if (string.IsNullOrWhiteSpace(entry.Value.TeachingPoint))
-                return "wrongAnswerFollowups teachingPoint required";
-            if (entry.Value.Choices.Count == 0)
-                return "wrongAnswerFollowups choices required";
-            if (!entry.Value.Choices.Any(c => c.Id == entry.Value.CorrectChoiceId))
-                return "wrongAnswerFollowups correctChoiceId must match a follow-up choice id";
-        }
-
-        return null;
-    }
-
     private string CurrentUserId() =>
         User.FindFirstValue(ClaimTypes.NameIdentifier)
         ?? User.FindFirstValue(ClaimTypes.Name)

api/QuestionsApi/Models/AnswerStatistics.cs

@@ -7,7 +7,8 @@ public sealed class QuestionAttemptCounter
     public string QuestionExternalId { get; set; } = "";
     public int Bucket { get; set; }
     public string ChoiceId { get; set; } = "";
-    public long Count { get; set; }
+    public long SelectionCount { get; set; }
+    public long PresentationCount { get; set; }
     public DateTime UpdatedUtc { get; set; } = DateTime.UtcNow;
 }
 
@@ -15,14 +16,16 @@ public sealed class QuestionAttemptCounterTotal
 {
     [BsonId]
     public QuestionChoiceCounterKey Key { get; set; } = new();
-    public long Count { get; set; }
+    public long SelectionCount { get; set; }
+    public long PresentationCount { get; set; }
 }
 
 public sealed class QuestionAttemptCounterAggregate
 {
     [BsonId]
     public QuestionChoiceBucketKey Key { get; set; } = new();
-    public long Count { get; set; }
+    public long SelectionCount { get; set; }
+    public long PresentationCount { get; set; }
 }
 
 public sealed class QuestionChoiceBucketKey
@@ -41,9 +44,18 @@ public sealed class QuestionChoiceCounterKey
 public sealed class AnswerStatisticsState
 {
     [BsonId]
-    public string Id { get; set; } = "answer-statistics-v1";
+    public string Id { get; set; } = "answer-statistics-v2";
     public bool Completed { get; set; }
     public string LeaseOwner { get; set; } = "";
     public DateTime LeaseUntilUtc { get; set; }
     public DateTime UpdatedUtc { get; set; } = DateTime.UtcNow;
+    public DateTime CutoverUtc { get; set; }
+}
+
+public sealed class QuestionPresentationBaseline
+{
+    [BsonId]
+    public string QuestionExternalId { get; set; } = "";
+    public List<string> ChoiceIds { get; set; } = [];
+    public DateTime CreatedUtc { get; set; } = DateTime.UtcNow;
 }

api/QuestionsApi/Models/QuestionAdminExport.cs

@@ -0,0 +1,8 @@
+namespace QuestionsApi.Models;
+
+public sealed class QuestionAdminExportResponse
+{
+    public List<QuestionResponse> Items { get; set; } = [];
+    public string? NextCursor { get; set; }
+    public bool HasMore { get; set; }
+}

api/QuestionsApi/Models/QuestionAttempt.cs

@@ -12,6 +12,8 @@ public class QuestionAttempt
     public string QuestionExternalId { get; set; } = "";
     public string UserId { get; set; } = "";
     public string ChoiceId { get; set; } = "";
+    public List<string> PresentedChoiceIds { get; set; } = [];
+    public bool TracksPresentation { get; set; }
     public bool IsCorrect { get; set; }
     public bool IsSkip { get; set; }
     public int CounterBucket { get; set; }

api/QuestionsApi/Models/QuestionRequests.cs

@@ -8,6 +8,7 @@ public class QuestionSearchRequest
     public int? DifficultyMin { get; set; }
     public int? DifficultyMax { get; set; }
     public int Limit { get; set; } = 40;
+    public bool SupportsChoicePools { get; set; }
 }
 
 public class UpsertQuestionRequest
@@ -35,4 +36,5 @@ public class FirstQuestionAttemptRequest
 {
     public string QuestionExternalId { get; set; } = "";
     public string ChoiceId { get; set; } = "";
+    public List<string> PresentedChoiceIds { get; set; } = [];
 }

api/QuestionsApi/Models/QuestionResponse.cs

@@ -12,6 +12,7 @@ public class QuestionResponse
     public Dictionary<string, string> ChoiceExplanations { get; set; } = [];
     public Dictionary<string, WrongAnswerFollowup> WrongAnswerFollowups { get; set; } = [];
     public Dictionary<string, int> ChoiceSelectionPercentages { get; set; } = [];
+    public string ChoiceSelectionRateBasis { get; set; } = "presented";
     public string Discipline { get; set; } = "";
     public string System { get; set; } = "";
     public string PerformanceCategory { get; set; } = "";
@@ -24,27 +25,56 @@ public class QuestionResponse
     public static QuestionResponse FromModel(
         Question question,
         Dictionary<string, int>? liveChoiceSelectionPercentages = null,
-        int questionElo = 1000) => new()
+        int questionElo = 1000,
+        bool includeChoicePool = true)
     {
-        Id = question.Id ?? "",
-        ExternalId = question.ExternalId,
-        Stem = question.Stem,
-        LeadIn = question.LeadIn,
-        Choices = question.Choices,
-        CorrectChoiceId = question.CorrectChoiceId,
-        Explanation = question.Explanation,
-        ChoiceExplanations = question.ChoiceExplanations,
-        WrongAnswerFollowups = question.WrongAnswerFollowups,
-        ChoiceSelectionPercentages = BuildChoiceSelectionPercentages(question, liveChoiceSelectionPercentages),
-        Discipline = question.Discipline,
-        System = question.System,
-        PerformanceCategory = RatingCategory.Normalize(question.PerformanceCategory, question.System, question.Discipline),
-        Difficulty = question.Difficulty,
-        QuestionElo = questionElo,
-        Labs = question.Labs,
-        Media = question.Media,
-        Tags = question.Tags
-    };
+        var choices = includeChoicePool ? question.Choices : SelectLegacyChoices(question);
+        var choiceIds = choices.Select(choice => choice.Id).ToHashSet(StringComparer.Ordinal);
+        var percentages = BuildChoiceSelectionPercentages(question, liveChoiceSelectionPercentages);
+        return new QuestionResponse
+        {
+            Id = question.Id ?? "",
+            ExternalId = question.ExternalId,
+            Stem = question.Stem,
+            LeadIn = question.LeadIn,
+            Choices = choices,
+            CorrectChoiceId = question.CorrectChoiceId,
+            Explanation = question.Explanation,
+            ChoiceExplanations = question.ChoiceExplanations
+                .Where(entry => choiceIds.Contains(entry.Key))
+                .ToDictionary(),
+            WrongAnswerFollowups = question.WrongAnswerFollowups
+                .Where(entry => choiceIds.Contains(entry.Key))
+                .ToDictionary(),
+            ChoiceSelectionPercentages = percentages
+                .Where(entry => choiceIds.Contains(entry.Key))
+                .ToDictionary(),
+            Discipline = question.Discipline,
+            System = question.System,
+            PerformanceCategory = RatingCategory.Normalize(question.PerformanceCategory, question.System, question.Discipline),
+            Difficulty = question.Difficulty,
+            QuestionElo = questionElo,
+            Labs = question.Labs,
+            Media = question.Media,
+            Tags = question.Tags
+        };
+    }
+
+    private static List<QuestionChoice> SelectLegacyChoices(Question question)
+    {
+        if (question.Choices.Count <= 5)
+            return question.Choices;
+
+        var correct = question.Choices.First(choice => choice.Id == question.CorrectChoiceId);
+        var distractors = question.Choices
+            .Where(choice => choice.Id != question.CorrectChoiceId)
+            .OrderBy(_ => Random.Shared.Next())
+            .Take(4)
+            .Append(correct)
+            .OrderBy(_ => Random.Shared.Next())
+            .ToList();
+        return distractors;
+    }
 
     public static Dictionary<string, int> BuildChoiceSelectionPercentages(Question question, Dictionary<string, int>? liveChoiceSelectionPercentages)
     {

api/QuestionsApi/Program.cs

@@ -53,8 +53,10 @@ builder.Services.AddAuthentication(JwtBearerDefaults.AuthenticationScheme)
     {
         o.TokenValidationParameters = new TokenValidationParameters
         {
-            ValidateIssuer = true, ValidIssuer = issuer,
-            ValidateAudience = true, ValidAudience = aud,
+            ValidateIssuer = true,
+            ValidIssuer = issuer,
+            ValidateAudience = true,
+            ValidAudience = aud,
             ValidateIssuerSigningKey = true,
             IssuerSigningKey = new SymmetricSecurityKey(Encoding.UTF8.GetBytes(jwtKey)),
             ValidateLifetime = true,

api/QuestionsApi/README.md

@@ -7,6 +7,7 @@ STEP-style question bank service for study games and clients.
 - `GET /healthz` Health check.
 - `GET /api/questions/systems` List available systems.
 - `POST /api/questions/search` Search or build a block of questions.
+- `GET /api/questions/admin/export` Cursor-paginated full-bank export (SUPER only).
 - `GET /api/questions/{id}` Get one question.
 - `POST /api/questions` Create question (SUPER only).
 - `POST /api/questions/bulk` Upsert questions by `externalId` (SUPER only).
@@ -22,10 +23,23 @@ Player reads require `USER` or `SUPER` JWT auth. Authoring endpoints require `SU
 ## Answer tracking
 
 The `QuestionAttempts` collection is the immutable source of truth and enforces
-one first attempt per question and user with a unique compound index. Choice
-percentages are served from 32-way materialized counter buckets and cached for
-up to five minutes. A background service backfills counters on the first deploy
-and reconciles recently active questions every five minutes.
+one first attempt per question and user with a unique compound index. Updated
+clients submit the five choice IDs that were presented. Choice rates are the
+percentage of presentations that resulted in selection, served from 32-way
+materialized counter buckets and cached for up to five minutes. These rates are
+independent and are not expected to sum to 100% when users see different
+distractor subsets.
+
+A background service snapshots the original choice IDs, backfills historical
+attempts into v2 exposure counters, and reconciles recently active questions
+every five minutes. The metric
+`step3_answer_statistics_v2_backfill_complete` must equal `1` before adding
+extra distractors to existing questions.
+
+Search clients that set `supportsChoicePools: true` receive each complete
+choice pool. Older clients receive five server-selected choices that always
+include the correct answer. Published questions require 5-20 unique choices,
+an explanation for every choice, and a follow-up for every incorrect choice.
 
 - `GET /readyz` verifies MongoDB connectivity for readiness checks.
 - `GET /metrics` exposes Prometheus-format service and reconciliation metrics.

api/QuestionsApi/Services/AnswerStatisticsMaintenanceService.cs

@@ -32,8 +32,13 @@ public sealed class AnswerStatisticsMaintenanceService : BackgroundService
                 _logger.LogInformation("Backfilling materialized answer statistics");
                 await _statistics.RebuildAllAsync();
                 await _statistics.MarkBackfillCompleteAsync(_owner);
+                _metrics.AnswerStatisticsBackfillCompleted();
                 _metrics.Reconciled();
             }
+            else if (await _statistics.IsBackfillCompleteAsync())
+            {
+                _metrics.AnswerStatisticsBackfillCompleted();
+            }
         }
         catch (Exception exception)
         {
@@ -45,6 +50,8 @@ public sealed class AnswerStatisticsMaintenanceService : BackgroundService
         {
             try
             {
+                if (await _statistics.IsBackfillCompleteAsync())
+                    _metrics.AnswerStatisticsBackfillCompleted();
                 var activeIds = await _attempts.GetActiveQuestionIdsAsync(DateTime.UtcNow.Subtract(_interval).Subtract(_interval));
                 await _statistics.RebuildQuestionsAsync(activeIds);
                 _metrics.Reconciled();

api/QuestionsApi/Services/AnswerStatisticsStore.cs

@@ -1,3 +1,4 @@
+using MongoDB.Bson;
 using MongoDB.Driver;
 using QuestionsApi.Models;
 using System.Collections.Concurrent;
@@ -6,20 +7,27 @@ namespace QuestionsApi.Services;
 
 public sealed class AnswerStatisticsStore
 {
-    private const string StateId = "answer-statistics-v1";
+    private const string StateId = "answer-statistics-v2";
     private static readonly TimeSpan CacheDuration = TimeSpan.FromMinutes(5);
 
     private readonly IMongoCollection<QuestionAttemptCounter> _counters;
     private readonly IMongoCollection<AnswerStatisticsState> _state;
+    private readonly IMongoCollection<QuestionPresentationBaseline> _baselines;
     private readonly QuestionAttemptStore _attempts;
+    private readonly QuestionStore _questions;
     private readonly ConcurrentDictionary<string, CachedPercentages> _cache = new();
     private volatile bool _backfillComplete;
 
-    public AnswerStatisticsStore(MongoDatabaseProvider mongo, QuestionAttemptStore attempts)
+    public AnswerStatisticsStore(
+        MongoDatabaseProvider mongo,
+        QuestionAttemptStore attempts,
+        QuestionStore questions)
     {
         _attempts = attempts;
-        _counters = mongo.Database.GetCollection<QuestionAttemptCounter>("QuestionAttemptCounters");
+        _questions = questions;
+        _counters = mongo.Database.GetCollection<QuestionAttemptCounter>("QuestionChoiceExposureCounters");
         _state = mongo.Database.GetCollection<AnswerStatisticsState>("SystemState");
+        _baselines = mongo.Database.GetCollection<QuestionPresentationBaseline>("QuestionPresentationBaselines");
 
         var counterIndex = Builders<QuestionAttemptCounter>.IndexKeys
             .Ascending(counter => counter.QuestionExternalId)
@@ -32,27 +40,31 @@ public sealed class AnswerStatisticsStore
 
     public async Task IncrementAsync(QuestionAttempt attempt)
     {
-        if (attempt.IsSkip)
+        if (attempt.IsSkip || !attempt.TracksPresentation || attempt.PresentedChoiceIds.Count == 0)
             return;
 
-        var filter = Builders<QuestionAttemptCounter>.Filter.And(
-            Builders<QuestionAttemptCounter>.Filter.Eq(counter => counter.QuestionExternalId, attempt.QuestionExternalId),
-            Builders<QuestionAttemptCounter>.Filter.Eq(counter => counter.Bucket, attempt.CounterBucket),
-            Builders<QuestionAttemptCounter>.Filter.Eq(counter => counter.ChoiceId, attempt.ChoiceId));
-        var update = Builders<QuestionAttemptCounter>.Update
-            .Inc(counter => counter.Count, 1)
-            .Set(counter => counter.UpdatedUtc, DateTime.UtcNow)
-            .SetOnInsert(counter => counter.QuestionExternalId, attempt.QuestionExternalId)
-            .SetOnInsert(counter => counter.Bucket, attempt.CounterBucket)
-            .SetOnInsert(counter => counter.ChoiceId, attempt.ChoiceId);
+        var now = DateTime.UtcNow;
+        var writes = attempt.PresentedChoiceIds.Distinct(StringComparer.Ordinal).Select(choiceId =>
+        {
+            var filter = CounterFilter(attempt.QuestionExternalId, attempt.CounterBucket, choiceId);
+            var update = Builders<QuestionAttemptCounter>.Update
+                .Inc(counter => counter.PresentationCount, 1)
+                .Inc(counter => counter.SelectionCount, choiceId == attempt.ChoiceId ? 1 : 0)
+                .Set(counter => counter.UpdatedUtc, now)
+                .SetOnInsert(counter => counter.QuestionExternalId, attempt.QuestionExternalId)
+                .SetOnInsert(counter => counter.Bucket, attempt.CounterBucket)
+                .SetOnInsert(counter => counter.ChoiceId, choiceId);
+            return new UpdateOneModel<QuestionAttemptCounter>(filter, update) { IsUpsert = true };
+        }).ToList();
 
         try
         {
-            await _counters.UpdateOneAsync(filter, update, new UpdateOptions { IsUpsert = true });
+            await _counters.BulkWriteAsync(writes, new BulkWriteOptions { IsOrdered = false });
         }
-        catch (MongoWriteException exception) when (exception.WriteError.Category == ServerErrorCategory.DuplicateKey)
+        catch (MongoBulkWriteException<QuestionAttemptCounter> exception)
+            when (exception.WriteErrors.All(error => error.Category == ServerErrorCategory.DuplicateKey))
         {
-            await _counters.UpdateOneAsync(filter, update);
+            await _counters.BulkWriteAsync(writes, new BulkWriteOptions { IsOrdered = false });
         }
         _cache.TryRemove(attempt.QuestionExternalId, out _);
     }
@@ -78,23 +90,9 @@ public sealed class AnswerStatisticsStore
             else
                 missing.Add(id);
         }
-
-        if (missing.Count == 0)
+        if (missing.Count == 0 || !await IsBackfillCompleteAsync())
             return output;
 
-        var backfillComplete = await IsBackfillCompleteAsync();
-        if (!backfillComplete)
-        {
-            var rawPercentages = await _attempts.GetPercentagesByQuestionAsync(missing);
-            foreach (var id in missing)
-            {
-                var percentages = rawPercentages.GetValueOrDefault(id, []);
-                output[id] = percentages;
-                _cache[id] = new CachedPercentages(percentages, now.Add(CacheDuration));
-            }
-            return output;
-        }
-
         var totals = await _counters.Aggregate()
             .Match(counter => missing.Contains(counter.QuestionExternalId))
             .Group(
@@ -106,34 +104,22 @@ public sealed class AnswerStatisticsStore
                         QuestionExternalId = group.Key.QuestionExternalId,
                         ChoiceId = group.Key.ChoiceId
                     },
-                    Count = group.Sum(counter => counter.Count)
+                    SelectionCount = group.Sum(counter => counter.SelectionCount),
+                    PresentationCount = group.Sum(counter => counter.PresentationCount)
                 })
             .ToListAsync();
 
-        var grouped = totals.GroupBy(total => total.Key.QuestionExternalId)
-            .ToDictionary(group => group.Key, group => group.ToList());
-
+        var grouped = totals.GroupBy(total => total.Key.QuestionExternalId).ToDictionary(group => group.Key);
         foreach (var id in missing)
         {
-            Dictionary<string, int> percentages;
-            if (grouped.TryGetValue(id, out var questionTotals))
-            {
-                var total = questionTotals.Sum(item => item.Count);
-                percentages = total == 0
-                    ? []
-                    : questionTotals.ToDictionary(
-                        item => item.Key.ChoiceId,
-                        item => (int)Math.Round(item.Count * 100.0 / total));
-            }
-            else
-            {
-                percentages = [];
-            }
-
+            var percentages = grouped.TryGetValue(id, out var choices)
+                ? choices.Where(item => item.PresentationCount > 0).ToDictionary(
+                    item => item.Key.ChoiceId,
+                    item => (int)Math.Round(item.SelectionCount * 100.0 / item.PresentationCount))
+                : [];
             output[id] = percentages;
             _cache[id] = new CachedPercentages(percentages, now.Add(CacheDuration));
         }
-
         return output;
     }
 
@@ -149,15 +135,14 @@ public sealed class AnswerStatisticsStore
         var update = Builders<AnswerStatisticsState>.Update
             .SetOnInsert(state => state.Id, StateId)
             .SetOnInsert(state => state.Completed, false)
+            .SetOnInsert(state => state.CutoverUtc, now)
             .Set(state => state.LeaseOwner, owner)
             .Set(state => state.LeaseUntilUtc, now.Add(duration))
             .Set(state => state.UpdatedUtc, now);
 
         try
         {
-            var leased = await _state.FindOneAndUpdateAsync(
-                filter,
-                update,
+            var leased = await _state.FindOneAndUpdateAsync(filter, update,
                 new FindOneAndUpdateOptions<AnswerStatisticsState>
                 {
                     IsUpsert = true,
@@ -173,64 +158,131 @@ public sealed class AnswerStatisticsStore
 
     public async Task MarkBackfillCompleteAsync(string owner)
     {
-        var update = Builders<AnswerStatisticsState>.Update
-            .Set(state => state.Completed, true)
-            .Set(state => state.LeaseOwner, owner)
-            .Set(state => state.UpdatedUtc, DateTime.UtcNow);
-        await _state.UpdateOneAsync(state => state.Id == StateId && state.LeaseOwner == owner, update);
+        await _state.UpdateOneAsync(
+            state => state.Id == StateId && state.LeaseOwner == owner,
+            Builders<AnswerStatisticsState>.Update
+                .Set(state => state.Completed, true)
+                .Set(state => state.UpdatedUtc, DateTime.UtcNow));
         _backfillComplete = true;
     }
 
-    public async Task RebuildAllAsync() => await RebuildAsync(null);
+    public Task RebuildAllAsync() => RebuildAsync(null);
 
-    public async Task RebuildQuestionsAsync(IReadOnlyCollection<string> questionExternalIds)
+    public Task RebuildQuestionsAsync(IReadOnlyCollection<string> questionExternalIds) =>
+        questionExternalIds.Count == 0 ? Task.CompletedTask : RebuildAsync(questionExternalIds);
+
+    public async Task<bool> IsBackfillCompleteAsync()
     {
-        if (questionExternalIds.Count > 0)
-            await RebuildAsync(questionExternalIds);
+        if (_backfillComplete)
+            return true;
+        _backfillComplete = (await _state.Find(state => state.Id == StateId).FirstOrDefaultAsync())?.Completed == true;
+        return _backfillComplete;
     }
 
     private async Task RebuildAsync(IReadOnlyCollection<string>? questionExternalIds)
     {
-        var aggregate = _attempts.Collection.Aggregate().Match(attempt => !attempt.IsSkip);
-        if (questionExternalIds is not null)
-            aggregate = aggregate.Match(attempt => questionExternalIds.Contains(attempt.QuestionExternalId));
-
-        var totals = await aggregate
-            .Group(
-                attempt => new { attempt.QuestionExternalId, attempt.CounterBucket, attempt.ChoiceId },
-                group => new QuestionAttemptCounterAggregate
-                {
-                    Key = new QuestionChoiceBucketKey
-                    {
-                        QuestionExternalId = group.Key.QuestionExternalId,
-                        Bucket = group.Key.CounterBucket,
-                        ChoiceId = group.Key.ChoiceId
-                    },
-                    Count = group.Sum(_ => 1L)
-                })
-            .ToListAsync();
-
-        if (totals.Count > 0)
+        await EnsureBaselinesAsync();
+        var state = await _state.Find(item => item.Id == StateId).FirstAsync();
+        var baselines = await _baselines.Find(Builders<QuestionPresentationBaseline>.Filter.Empty).ToListAsync();
+        var baselineByQuestion = baselines.ToDictionary(item => item.QuestionExternalId, item => item.ChoiceIds);
+        var match = new BsonDocument
         {
-            var now = DateTime.UtcNow;
-            var replacements = totals.Select(total =>
-            {
-                var counter = new QuestionAttemptCounter
+            { "IsSkip", false },
+            { "$or", new BsonArray
                 {
-                    QuestionExternalId = total.Key.QuestionExternalId,
-                    Bucket = total.Key.Bucket,
-                    ChoiceId = total.Key.ChoiceId,
-                    Count = total.Count,
-                    UpdatedUtc = now
-                };
-                var filter = Builders<QuestionAttemptCounter>.Filter.And(
-                    Builders<QuestionAttemptCounter>.Filter.Eq(item => item.QuestionExternalId, counter.QuestionExternalId),
-                    Builders<QuestionAttemptCounter>.Filter.Eq(item => item.Bucket, counter.Bucket),
-                    Builders<QuestionAttemptCounter>.Filter.Eq(item => item.ChoiceId, counter.ChoiceId));
-                return new ReplaceOneModel<QuestionAttemptCounter>(filter, counter) { IsUpsert = true };
-            }).ToList();
-            await _counters.BulkWriteAsync(replacements, new BulkWriteOptions { IsOrdered = false });
+                    new BsonDocument("TracksPresentation", true),
+                    new BsonDocument("CreatedUtc", new BsonDocument("$lt", state.CutoverUtc))
+                }
+            }
+        };
+        if (questionExternalIds is not null)
+            match["QuestionExternalId"] = new BsonDocument("$in", new BsonArray(questionExternalIds));
+
+        var selectionPipeline = new[]
+        {
+            new BsonDocument("$match", match),
+            new BsonDocument("$group", new BsonDocument
+            {
+                { "_id", new BsonDocument { { "q", "$QuestionExternalId" }, { "b", "$CounterBucket" }, { "c", "$ChoiceId" } } },
+                { "count", new BsonDocument("$sum", 1) }
+            })
+        };
+        var presentationMatch = new BsonDocument(match) { ["TracksPresentation"] = true };
+        var presentationPipeline = new[]
+        {
+            new BsonDocument("$match", presentationMatch),
+            new BsonDocument("$unwind", "$PresentedChoiceIds"),
+            new BsonDocument("$group", new BsonDocument
+            {
+                { "_id", new BsonDocument { { "q", "$QuestionExternalId" }, { "b", "$CounterBucket" }, { "c", "$PresentedChoiceIds" } } },
+                { "count", new BsonDocument("$sum", 1) }
+            })
+        };
+        var legacyMatch = new BsonDocument
+        {
+            { "IsSkip", false },
+            { "TracksPresentation", new BsonDocument("$ne", true) },
+            { "CreatedUtc", new BsonDocument("$lt", state.CutoverUtc) }
+        };
+        if (questionExternalIds is not null)
+            legacyMatch["QuestionExternalId"] = new BsonDocument("$in", new BsonArray(questionExternalIds));
+        var legacyPipeline = new[]
+        {
+            new BsonDocument("$match", legacyMatch),
+            new BsonDocument("$group", new BsonDocument
+            {
+                { "_id", new BsonDocument { { "q", "$QuestionExternalId" }, { "b", "$CounterBucket" } } },
+                { "count", new BsonDocument("$sum", 1) }
+            })
+        };
+
+        var selections = await _attempts.Collection.Aggregate<BsonDocument>(selectionPipeline).ToListAsync();
+        var presentations = await _attempts.Collection.Aggregate<BsonDocument>(presentationPipeline).ToListAsync();
+        var legacy = await _attempts.Collection.Aggregate<BsonDocument>(legacyPipeline).ToListAsync();
+        var totals = new Dictionary<(string Question, int Bucket, string Choice), (long Selected, long Presented)>();
+
+        foreach (var row in selections)
+        {
+            var key = CounterKey(row["_id"].AsBsonDocument);
+            totals[key] = (row["count"].ToInt64(), totals.GetValueOrDefault(key).Presented);
         }
+        foreach (var row in presentations)
+        {
+            var key = CounterKey(row["_id"].AsBsonDocument);
+            totals[key] = (totals.GetValueOrDefault(key).Selected, row["count"].ToInt64());
+        }
+        foreach (var row in legacy)
+        {
+            var id = row["_id"].AsBsonDocument;
+            var questionId = id["q"].AsString;
+            if (!baselineByQuestion.TryGetValue(questionId, out var choiceIds))
+                continue;
+            foreach (var choiceId in choiceIds)
+            {
+                var key = (questionId, id["b"].ToInt32(), choiceId);
+                totals[key] = (totals.GetValueOrDefault(key).Selected,
+                    totals.GetValueOrDefault(key).Presented + row["count"].ToInt64());
+            }
+        }
+
+        var now = DateTime.UtcNow;
+        var writes = totals.Select(item =>
+            new ReplaceOneModel<QuestionAttemptCounter>(
+                CounterFilter(item.Key.Question, item.Key.Bucket, item.Key.Choice),
+                new QuestionAttemptCounter
+                {
+                    QuestionExternalId = item.Key.Question,
+                    Bucket = item.Key.Bucket,
+                    ChoiceId = item.Key.Choice,
+                    SelectionCount = item.Value.Selected,
+                    PresentationCount = item.Value.Presented,
+                    UpdatedUtc = now
+                })
+            {
+                IsUpsert = true
+            }).ToList();
+        if (writes.Count > 0)
+            await _counters.BulkWriteAsync(writes, new BulkWriteOptions { IsOrdered = false });
 
         if (questionExternalIds is null)
             _cache.Clear();
@@ -239,13 +291,31 @@ public sealed class AnswerStatisticsStore
                 _cache.TryRemove(id, out _);
     }
 
-    private async Task<bool> IsBackfillCompleteAsync()
+    private async Task EnsureBaselinesAsync()
     {
-        if (_backfillComplete)
-            return true;
-        _backfillComplete = (await _state.Find(state => state.Id == StateId).FirstOrDefaultAsync())?.Completed == true;
-        return _backfillComplete;
+        var questions = await _questions.GetAllAsync();
+        var writes = questions.Select(question =>
+            new UpdateOneModel<QuestionPresentationBaseline>(
+                Builders<QuestionPresentationBaseline>.Filter.Eq(item => item.QuestionExternalId, question.ExternalId),
+                Builders<QuestionPresentationBaseline>.Update
+                    .SetOnInsert(item => item.QuestionExternalId, question.ExternalId)
+                    .SetOnInsert(item => item.ChoiceIds, question.Choices.Select(choice => choice.Id).ToList())
+                    .SetOnInsert(item => item.CreatedUtc, DateTime.UtcNow))
+            {
+                IsUpsert = true
+            }).ToList();
+        if (writes.Count > 0)
+            await _baselines.BulkWriteAsync(writes, new BulkWriteOptions { IsOrdered = false });
     }
 
+    private static FilterDefinition<QuestionAttemptCounter> CounterFilter(string questionId, int bucket, string choiceId) =>
+        Builders<QuestionAttemptCounter>.Filter.And(
+            Builders<QuestionAttemptCounter>.Filter.Eq(counter => counter.QuestionExternalId, questionId),
+            Builders<QuestionAttemptCounter>.Filter.Eq(counter => counter.Bucket, bucket),
+            Builders<QuestionAttemptCounter>.Filter.Eq(counter => counter.ChoiceId, choiceId));
+
+    private static (string Question, int Bucket, string Choice) CounterKey(BsonDocument id) =>
+        (id["q"].AsString, id["b"].ToInt32(), id["c"].AsString);
+
     private sealed record CachedPercentages(Dictionary<string, int> Percentages, DateTime ExpiresUtc);
 }

api/QuestionsApi/Services/QuestionAttemptStore.cs

@@ -34,13 +34,16 @@ public class QuestionAttemptStore
         string userId,
         string choiceId,
         bool isCorrect,
-        bool isSkip = false)
+        bool isSkip = false,
+        IReadOnlyCollection<string>? presentedChoiceIds = null)
     {
         var attempt = new QuestionAttempt
         {
             QuestionExternalId = questionExternalId,
             UserId = userId,
             ChoiceId = choiceId,
+            PresentedChoiceIds = presentedChoiceIds?.ToList() ?? [],
+            TracksPresentation = presentedChoiceIds is { Count: > 0 },
             IsCorrect = isCorrect,
             IsSkip = isSkip,
             CounterBucket = GetCounterBucket(userId),
@@ -62,49 +65,6 @@ public class QuestionAttemptStore
         }
     }
 
-    public async Task<Dictionary<string, int>> GetPercentagesForQuestionAsync(string questionExternalId)
-    {
-        var results = await GetPercentagesByQuestionAsync([questionExternalId]);
-        return results.GetValueOrDefault(questionExternalId, []);
-    }
-
-    public async Task<Dictionary<string, Dictionary<string, int>>> GetPercentagesByQuestionAsync(IEnumerable<string> questionExternalIds)
-    {
-        var ids = questionExternalIds.Where(id => !string.IsNullOrWhiteSpace(id)).Distinct().ToList();
-        if (ids.Count == 0)
-            return [];
-
-        var totals = await _col.Aggregate()
-            .Match(a => ids.Contains(a.QuestionExternalId) && !a.IsSkip)
-            .Group(
-                attempt => new { attempt.QuestionExternalId, attempt.ChoiceId },
-                group => new QuestionAttemptCounterTotal
-                {
-                    Key = new QuestionChoiceCounterKey
-                    {
-                        QuestionExternalId = group.Key.QuestionExternalId,
-                        ChoiceId = group.Key.ChoiceId
-                    },
-                    Count = group.Sum(_ => 1L)
-                })
-            .ToListAsync();
-        var output = new Dictionary<string, Dictionary<string, int>>();
-
-        foreach (var group in totals.GroupBy(total => total.Key.QuestionExternalId))
-        {
-            var total = group.Sum(item => item.Count);
-            if (total == 0)
-                continue;
-
-            output[group.Key] = group
-                .ToDictionary(
-                    item => item.Key.ChoiceId,
-                    item => (int)Math.Round(item.Count * 100.0 / total));
-        }
-
-        return output;
-    }
-
     public Task<List<string>> GetActiveQuestionIdsAsync(DateTime sinceUtc) =>
         _col.Distinct(
             attempt => attempt.QuestionExternalId,

api/QuestionsApi/Services/QuestionStore.cs

@@ -50,6 +50,32 @@ public class QuestionStore
     public async Task<Question?> GetByExternalIdAsync(string externalId) =>
         await _col.Find(q => q.ExternalId == externalId).FirstOrDefaultAsync();
 
+    public async Task<List<Question>> GetAllAsync() =>
+        await _col.Find(Builders<Question>.Filter.Empty).ToListAsync();
+
+    public async Task<(List<Question> Items, string? NextCursor, bool HasMore)> ExportPageAsync(
+        string? after,
+        int requestedLimit)
+    {
+        var limit = Math.Clamp(requestedLimit, 1, 200);
+        var filter = Builders<Question>.Filter.Empty;
+        if (!string.IsNullOrWhiteSpace(after))
+        {
+            if (!ObjectId.TryParse(after, out var cursor))
+                throw new ArgumentException("after must be a valid question cursor", nameof(after));
+            filter = new BsonDocument("_id", new BsonDocument("$gt", cursor));
+        }
+
+        var page = await _col.Find(filter)
+            .Sort(Builders<Question>.Sort.Ascending("_id"))
+            .Limit(limit + 1)
+            .ToListAsync();
+        var hasMore = page.Count > limit;
+        if (hasMore)
+            page.RemoveAt(page.Count - 1);
+        return (page, hasMore ? page[^1].Id : null, hasMore);
+    }
+
     public async Task<Question?> CreateAsync(UpsertQuestionRequest req)
     {
         var question = FromRequest(req);

api/QuestionsApi/Services/QuestionValidator.cs

@@ -0,0 +1,57 @@
+using QuestionsApi.Models;
+
+namespace QuestionsApi.Services;
+
+public static class QuestionValidator
+{
+    public static string? Validate(UpsertQuestionRequest req)
+    {
+        if (string.IsNullOrWhiteSpace(req.ExternalId)) return "externalId required";
+        if (string.IsNullOrWhiteSpace(req.Stem)) return "stem required";
+        if (string.IsNullOrWhiteSpace(req.LeadIn)) return "leadIn required";
+        if (req.Choices.Count < (req.IsPublished ? 5 : 2))
+            return req.IsPublished ? "published questions require at least five choices" : "at least two choices required";
+        if (req.Choices.Count > 20) return "questions support at most twenty choices";
+        if (req.Choices.Any(choice => string.IsNullOrWhiteSpace(choice.Id) || string.IsNullOrWhiteSpace(choice.Text)))
+            return "choice ids and text are required";
+
+        var choiceIds = req.Choices.Select(choice => choice.Id.Trim()).ToList();
+        if (choiceIds.Distinct(StringComparer.Ordinal).Count() != choiceIds.Count)
+            return "choice ids must be unique";
+        if (string.IsNullOrWhiteSpace(req.CorrectChoiceId)) return "correctChoiceId required";
+        if (!choiceIds.Contains(req.CorrectChoiceId.Trim(), StringComparer.Ordinal))
+            return "correctChoiceId must match a choice id";
+        if (string.IsNullOrWhiteSpace(req.Explanation)) return "explanation required";
+
+        var choiceIdSet = choiceIds.ToHashSet(StringComparer.Ordinal);
+        if (!choiceIdSet.SetEquals(req.ChoiceExplanations.Keys))
+            return "choiceExplanations must contain exactly one entry for every choice";
+        var incorrectChoiceIds = choiceIdSet
+            .Where(id => id != req.CorrectChoiceId.Trim())
+            .ToHashSet(StringComparer.Ordinal);
+        if (!incorrectChoiceIds.SetEquals(req.WrongAnswerFollowups.Keys))
+            return "wrongAnswerFollowups must contain exactly one entry for every incorrect choice";
+        if (req.ChoiceSelectionPercentages.Keys.Any(id => !choiceIdSet.Contains(id)))
+            return "choiceSelectionPercentages keys must match choice ids";
+        if (req.ChoiceSelectionPercentages.Any(entry => entry.Value is < 0 or > 100))
+            return "choiceSelectionPercentages values must be between 0 and 100";
+
+        foreach (var entry in req.WrongAnswerFollowups)
+        {
+            if (string.IsNullOrWhiteSpace(entry.Value.Prompt)) return "wrongAnswerFollowups prompt required";
+            if (string.IsNullOrWhiteSpace(entry.Value.TeachingPoint)) return "wrongAnswerFollowups teachingPoint required";
+            if (entry.Value.Choices.Count == 0) return "wrongAnswerFollowups choices required";
+            if (entry.Value.Choices.Any(choice => string.IsNullOrWhiteSpace(choice.Id) || string.IsNullOrWhiteSpace(choice.Text)))
+                return "wrongAnswerFollowups choice ids and text are required";
+            if (entry.Value.Choices.Select(choice => choice.Id).Distinct(StringComparer.Ordinal).Count() != entry.Value.Choices.Count)
+                return "wrongAnswerFollowups choice ids must be unique";
+            if (!entry.Value.Choices.Any(choice => choice.Id == entry.Value.CorrectChoiceId))
+                return "wrongAnswerFollowups correctChoiceId must match a follow-up choice id";
+        }
+
+        if (string.IsNullOrWhiteSpace(req.System)) return "system required";
+        if (string.IsNullOrWhiteSpace(req.Discipline)) return "discipline required";
+        if (req.Difficulty is < 1 or > 5) return "difficulty must be between 1 and 5";
+        return null;
+    }
+}

api/QuestionsApi/Services/ServiceMetrics.cs

@@ -12,11 +12,13 @@ public sealed class ServiceMetrics
     private long _lastReconciliationUnixSeconds;
     private long _httpRequests;
     private long _httpDurationMicroseconds;
+    private long _answerStatisticsBackfillComplete;
 
     public void AttemptCreated() => Interlocked.Increment(ref _attemptsCreated);
     public void AttemptDuplicate() => Interlocked.Increment(ref _attemptDuplicates);
     public void CounterFailure() => Interlocked.Increment(ref _counterFailures);
     public void RatingFailure() => Interlocked.Increment(ref _ratingFailures);
+    public void AnswerStatisticsBackfillCompleted() => Interlocked.Exchange(ref _answerStatisticsBackfillComplete, 1);
     public void Reconciled()
     {
         Interlocked.Increment(ref _reconciliations);
@@ -48,6 +50,10 @@ public sealed class ServiceMetrics
         output.Append("step3_answer_last_reconciliation_unixtime ")
             .Append(Interlocked.Read(ref _lastReconciliationUnixSeconds))
             .AppendLine();
+        output.AppendLine("# TYPE step3_answer_statistics_v2_backfill_complete gauge");
+        output.Append("step3_answer_statistics_v2_backfill_complete ")
+            .Append(Interlocked.Read(ref _answerStatisticsBackfillComplete))
+            .AppendLine();
         return output.ToString();
     }