Add variable question distractor pools

2026-07-13 10:15:05 -05:00
parent f2c59388eb
commit 93154d5925
24 changed files with 1040 additions and 247 deletions
@@ -32,8 +32,13 @@ public sealed class AnswerStatisticsMaintenanceService : BackgroundService
                _logger.LogInformation("Backfilling materialized answer statistics");
                await _statistics.RebuildAllAsync();
                await _statistics.MarkBackfillCompleteAsync(_owner);
+                _metrics.AnswerStatisticsBackfillCompleted();
                _metrics.Reconciled();
            }
+            else if (await _statistics.IsBackfillCompleteAsync())
+            {
+                _metrics.AnswerStatisticsBackfillCompleted();
+            }
        }
        catch (Exception exception)
        {
@@ -45,6 +50,8 @@ public sealed class AnswerStatisticsMaintenanceService : BackgroundService
        {
            try
            {
+                if (await _statistics.IsBackfillCompleteAsync())
+                    _metrics.AnswerStatisticsBackfillCompleted();
                var activeIds = await _attempts.GetActiveQuestionIdsAsync(DateTime.UtcNow.Subtract(_interval).Subtract(_interval));
                await _statistics.RebuildQuestionsAsync(activeIds);
                _metrics.Reconciled();
@@ -1,3 +1,4 @@
+using MongoDB.Bson;
 using MongoDB.Driver;
 using QuestionsApi.Models;
 using System.Collections.Concurrent;
@@ -6,20 +7,27 @@ namespace QuestionsApi.Services;

 public sealed class AnswerStatisticsStore
 {
-    private const string StateId = "answer-statistics-v1";
+    private const string StateId = "answer-statistics-v2";
    private static readonly TimeSpan CacheDuration = TimeSpan.FromMinutes(5);

    private readonly IMongoCollection<QuestionAttemptCounter> _counters;
    private readonly IMongoCollection<AnswerStatisticsState> _state;
+    private readonly IMongoCollection<QuestionPresentationBaseline> _baselines;
    private readonly QuestionAttemptStore _attempts;
+    private readonly QuestionStore _questions;
    private readonly ConcurrentDictionary<string, CachedPercentages> _cache = new();
    private volatile bool _backfillComplete;

-    public AnswerStatisticsStore(MongoDatabaseProvider mongo, QuestionAttemptStore attempts)
+    public AnswerStatisticsStore(
+        MongoDatabaseProvider mongo,
+        QuestionAttemptStore attempts,
+        QuestionStore questions)
    {
        _attempts = attempts;
-        _counters = mongo.Database.GetCollection<QuestionAttemptCounter>("QuestionAttemptCounters");
+        _questions = questions;
+        _counters = mongo.Database.GetCollection<QuestionAttemptCounter>("QuestionChoiceExposureCounters");
        _state = mongo.Database.GetCollection<AnswerStatisticsState>("SystemState");
+        _baselines = mongo.Database.GetCollection<QuestionPresentationBaseline>("QuestionPresentationBaselines");

        var counterIndex = Builders<QuestionAttemptCounter>.IndexKeys
            .Ascending(counter => counter.QuestionExternalId)
@@ -32,27 +40,31 @@ public sealed class AnswerStatisticsStore

    public async Task IncrementAsync(QuestionAttempt attempt)
    {
-        if (attempt.IsSkip)
+        if (attempt.IsSkip || !attempt.TracksPresentation || attempt.PresentedChoiceIds.Count == 0)
            return;

-        var filter = Builders<QuestionAttemptCounter>.Filter.And(
-            Builders<QuestionAttemptCounter>.Filter.Eq(counter => counter.QuestionExternalId, attempt.QuestionExternalId),
-            Builders<QuestionAttemptCounter>.Filter.Eq(counter => counter.Bucket, attempt.CounterBucket),
-            Builders<QuestionAttemptCounter>.Filter.Eq(counter => counter.ChoiceId, attempt.ChoiceId));
-        var update = Builders<QuestionAttemptCounter>.Update
-            .Inc(counter => counter.Count, 1)
-            .Set(counter => counter.UpdatedUtc, DateTime.UtcNow)
-            .SetOnInsert(counter => counter.QuestionExternalId, attempt.QuestionExternalId)
-            .SetOnInsert(counter => counter.Bucket, attempt.CounterBucket)
-            .SetOnInsert(counter => counter.ChoiceId, attempt.ChoiceId);
+        var now = DateTime.UtcNow;
+        var writes = attempt.PresentedChoiceIds.Distinct(StringComparer.Ordinal).Select(choiceId =>
+        {
+            var filter = CounterFilter(attempt.QuestionExternalId, attempt.CounterBucket, choiceId);
+            var update = Builders<QuestionAttemptCounter>.Update
+                .Inc(counter => counter.PresentationCount, 1)
+                .Inc(counter => counter.SelectionCount, choiceId == attempt.ChoiceId ? 1 : 0)
+                .Set(counter => counter.UpdatedUtc, now)
+                .SetOnInsert(counter => counter.QuestionExternalId, attempt.QuestionExternalId)
+                .SetOnInsert(counter => counter.Bucket, attempt.CounterBucket)
+                .SetOnInsert(counter => counter.ChoiceId, choiceId);
+            return new UpdateOneModel<QuestionAttemptCounter>(filter, update) { IsUpsert = true };
+        }).ToList();

        try
        {
-            await _counters.UpdateOneAsync(filter, update, new UpdateOptions { IsUpsert = true });
+            await _counters.BulkWriteAsync(writes, new BulkWriteOptions { IsOrdered = false });
        }
-        catch (MongoWriteException exception) when (exception.WriteError.Category == ServerErrorCategory.DuplicateKey)
+        catch (MongoBulkWriteException<QuestionAttemptCounter> exception)
+            when (exception.WriteErrors.All(error => error.Category == ServerErrorCategory.DuplicateKey))
        {
-            await _counters.UpdateOneAsync(filter, update);
+            await _counters.BulkWriteAsync(writes, new BulkWriteOptions { IsOrdered = false });
        }
        _cache.TryRemove(attempt.QuestionExternalId, out _);
    }
@@ -78,23 +90,9 @@ public sealed class AnswerStatisticsStore
            else
                missing.Add(id);
        }
-
-        if (missing.Count == 0)
+        if (missing.Count == 0 || !await IsBackfillCompleteAsync())
            return output;

-        var backfillComplete = await IsBackfillCompleteAsync();
-        if (!backfillComplete)
-        {
-            var rawPercentages = await _attempts.GetPercentagesByQuestionAsync(missing);
-            foreach (var id in missing)
-            {
-                var percentages = rawPercentages.GetValueOrDefault(id, []);
-                output[id] = percentages;
-                _cache[id] = new CachedPercentages(percentages, now.Add(CacheDuration));
-            }
-            return output;
-        }
-
        var totals = await _counters.Aggregate()
            .Match(counter => missing.Contains(counter.QuestionExternalId))
            .Group(
@@ -106,34 +104,22 @@ public sealed class AnswerStatisticsStore
                        QuestionExternalId = group.Key.QuestionExternalId,
                        ChoiceId = group.Key.ChoiceId
                    },
-                    Count = group.Sum(counter => counter.Count)
+                    SelectionCount = group.Sum(counter => counter.SelectionCount),
+                    PresentationCount = group.Sum(counter => counter.PresentationCount)
                })
            .ToListAsync();

-        var grouped = totals.GroupBy(total => total.Key.QuestionExternalId)
-            .ToDictionary(group => group.Key, group => group.ToList());
-
+        var grouped = totals.GroupBy(total => total.Key.QuestionExternalId).ToDictionary(group => group.Key);
        foreach (var id in missing)
        {
-            Dictionary<string, int> percentages;
-            if (grouped.TryGetValue(id, out var questionTotals))
-            {
-                var total = questionTotals.Sum(item => item.Count);
-                percentages = total == 0
-                    ? []
-                    : questionTotals.ToDictionary(
-                        item => item.Key.ChoiceId,
-                        item => (int)Math.Round(item.Count * 100.0 / total));
-            }
-            else
-            {
-                percentages = [];
-            }
-
+            var percentages = grouped.TryGetValue(id, out var choices)
+                ? choices.Where(item => item.PresentationCount > 0).ToDictionary(
+                    item => item.Key.ChoiceId,
+                    item => (int)Math.Round(item.SelectionCount * 100.0 / item.PresentationCount))
+                : [];
            output[id] = percentages;
            _cache[id] = new CachedPercentages(percentages, now.Add(CacheDuration));
        }
-
        return output;
    }

@@ -149,15 +135,14 @@ public sealed class AnswerStatisticsStore
        var update = Builders<AnswerStatisticsState>.Update
            .SetOnInsert(state => state.Id, StateId)
            .SetOnInsert(state => state.Completed, false)
+            .SetOnInsert(state => state.CutoverUtc, now)
            .Set(state => state.LeaseOwner, owner)
            .Set(state => state.LeaseUntilUtc, now.Add(duration))
            .Set(state => state.UpdatedUtc, now);

        try
        {
-            var leased = await _state.FindOneAndUpdateAsync(
-                filter,
-                update,
+            var leased = await _state.FindOneAndUpdateAsync(filter, update,
                new FindOneAndUpdateOptions<AnswerStatisticsState>
                {
                    IsUpsert = true,
@@ -173,64 +158,131 @@ public sealed class AnswerStatisticsStore

    public async Task MarkBackfillCompleteAsync(string owner)
    {
-        var update = Builders<AnswerStatisticsState>.Update
-            .Set(state => state.Completed, true)
-            .Set(state => state.LeaseOwner, owner)
-            .Set(state => state.UpdatedUtc, DateTime.UtcNow);
-        await _state.UpdateOneAsync(state => state.Id == StateId && state.LeaseOwner == owner, update);
+        await _state.UpdateOneAsync(
+            state => state.Id == StateId && state.LeaseOwner == owner,
+            Builders<AnswerStatisticsState>.Update
+                .Set(state => state.Completed, true)
+                .Set(state => state.UpdatedUtc, DateTime.UtcNow));
        _backfillComplete = true;
    }

-    public async Task RebuildAllAsync() => await RebuildAsync(null);
+    public Task RebuildAllAsync() => RebuildAsync(null);

-    public async Task RebuildQuestionsAsync(IReadOnlyCollection<string> questionExternalIds)
+    public Task RebuildQuestionsAsync(IReadOnlyCollection<string> questionExternalIds) =>
+        questionExternalIds.Count == 0 ? Task.CompletedTask : RebuildAsync(questionExternalIds);
+
+    public async Task<bool> IsBackfillCompleteAsync()
    {
-        if (questionExternalIds.Count > 0)
-            await RebuildAsync(questionExternalIds);
+        if (_backfillComplete)
+            return true;
+        _backfillComplete = (await _state.Find(state => state.Id == StateId).FirstOrDefaultAsync())?.Completed == true;
+        return _backfillComplete;
    }

    private async Task RebuildAsync(IReadOnlyCollection<string>? questionExternalIds)
    {
-        var aggregate = _attempts.Collection.Aggregate().Match(attempt => !attempt.IsSkip);
-        if (questionExternalIds is not null)
-            aggregate = aggregate.Match(attempt => questionExternalIds.Contains(attempt.QuestionExternalId));
-
-        var totals = await aggregate
-            .Group(
-                attempt => new { attempt.QuestionExternalId, attempt.CounterBucket, attempt.ChoiceId },
-                group => new QuestionAttemptCounterAggregate
-                {
-                    Key = new QuestionChoiceBucketKey
-                    {
-                        QuestionExternalId = group.Key.QuestionExternalId,
-                        Bucket = group.Key.CounterBucket,
-                        ChoiceId = group.Key.ChoiceId
-                    },
-                    Count = group.Sum(_ => 1L)
-                })
-            .ToListAsync();
-
-        if (totals.Count > 0)
+        await EnsureBaselinesAsync();
+        var state = await _state.Find(item => item.Id == StateId).FirstAsync();
+        var baselines = await _baselines.Find(Builders<QuestionPresentationBaseline>.Filter.Empty).ToListAsync();
+        var baselineByQuestion = baselines.ToDictionary(item => item.QuestionExternalId, item => item.ChoiceIds);
+        var match = new BsonDocument
        {
-            var now = DateTime.UtcNow;
-            var replacements = totals.Select(total =>
-            {
-                var counter = new QuestionAttemptCounter
+            { "IsSkip", false },
+            { "$or", new BsonArray
                {
-                    QuestionExternalId = total.Key.QuestionExternalId,
-                    Bucket = total.Key.Bucket,
-                    ChoiceId = total.Key.ChoiceId,
-                    Count = total.Count,
-                    UpdatedUtc = now
-                };
-                var filter = Builders<QuestionAttemptCounter>.Filter.And(
-                    Builders<QuestionAttemptCounter>.Filter.Eq(item => item.QuestionExternalId, counter.QuestionExternalId),
-                    Builders<QuestionAttemptCounter>.Filter.Eq(item => item.Bucket, counter.Bucket),
-                    Builders<QuestionAttemptCounter>.Filter.Eq(item => item.ChoiceId, counter.ChoiceId));
-                return new ReplaceOneModel<QuestionAttemptCounter>(filter, counter) { IsUpsert = true };
-            }).ToList();
-            await _counters.BulkWriteAsync(replacements, new BulkWriteOptions { IsOrdered = false });
+                    new BsonDocument("TracksPresentation", true),
+                    new BsonDocument("CreatedUtc", new BsonDocument("$lt", state.CutoverUtc))
+                }
+            }
+        };
+        if (questionExternalIds is not null)
+            match["QuestionExternalId"] = new BsonDocument("$in", new BsonArray(questionExternalIds));
+
+        var selectionPipeline = new[]
+        {
+            new BsonDocument("$match", match),
+            new BsonDocument("$group", new BsonDocument
+            {
+                { "_id", new BsonDocument { { "q", "$QuestionExternalId" }, { "b", "$CounterBucket" }, { "c", "$ChoiceId" } } },
+                { "count", new BsonDocument("$sum", 1) }
+            })
+        };
+        var presentationMatch = new BsonDocument(match) { ["TracksPresentation"] = true };
+        var presentationPipeline = new[]
+        {
+            new BsonDocument("$match", presentationMatch),
+            new BsonDocument("$unwind", "$PresentedChoiceIds"),
+            new BsonDocument("$group", new BsonDocument
+            {
+                { "_id", new BsonDocument { { "q", "$QuestionExternalId" }, { "b", "$CounterBucket" }, { "c", "$PresentedChoiceIds" } } },
+                { "count", new BsonDocument("$sum", 1) }
+            })
+        };
+        var legacyMatch = new BsonDocument
+        {
+            { "IsSkip", false },
+            { "TracksPresentation", new BsonDocument("$ne", true) },
+            { "CreatedUtc", new BsonDocument("$lt", state.CutoverUtc) }
+        };
+        if (questionExternalIds is not null)
+            legacyMatch["QuestionExternalId"] = new BsonDocument("$in", new BsonArray(questionExternalIds));
+        var legacyPipeline = new[]
+        {
+            new BsonDocument("$match", legacyMatch),
+            new BsonDocument("$group", new BsonDocument
+            {
+                { "_id", new BsonDocument { { "q", "$QuestionExternalId" }, { "b", "$CounterBucket" } } },
+                { "count", new BsonDocument("$sum", 1) }
+            })
+        };
+
+        var selections = await _attempts.Collection.Aggregate<BsonDocument>(selectionPipeline).ToListAsync();
+        var presentations = await _attempts.Collection.Aggregate<BsonDocument>(presentationPipeline).ToListAsync();
+        var legacy = await _attempts.Collection.Aggregate<BsonDocument>(legacyPipeline).ToListAsync();
+        var totals = new Dictionary<(string Question, int Bucket, string Choice), (long Selected, long Presented)>();
+
+        foreach (var row in selections)
+        {
+            var key = CounterKey(row["_id"].AsBsonDocument);
+            totals[key] = (row["count"].ToInt64(), totals.GetValueOrDefault(key).Presented);
        }
+        foreach (var row in presentations)
+        {
+            var key = CounterKey(row["_id"].AsBsonDocument);
+            totals[key] = (totals.GetValueOrDefault(key).Selected, row["count"].ToInt64());
+        }
+        foreach (var row in legacy)
+        {
+            var id = row["_id"].AsBsonDocument;
+            var questionId = id["q"].AsString;
+            if (!baselineByQuestion.TryGetValue(questionId, out var choiceIds))
+                continue;
+            foreach (var choiceId in choiceIds)
+            {
+                var key = (questionId, id["b"].ToInt32(), choiceId);
+                totals[key] = (totals.GetValueOrDefault(key).Selected,
+                    totals.GetValueOrDefault(key).Presented + row["count"].ToInt64());
+            }
+        }
+
+        var now = DateTime.UtcNow;
+        var writes = totals.Select(item =>
+            new ReplaceOneModel<QuestionAttemptCounter>(
+                CounterFilter(item.Key.Question, item.Key.Bucket, item.Key.Choice),
+                new QuestionAttemptCounter
+                {
+                    QuestionExternalId = item.Key.Question,
+                    Bucket = item.Key.Bucket,
+                    ChoiceId = item.Key.Choice,
+                    SelectionCount = item.Value.Selected,
+                    PresentationCount = item.Value.Presented,
+                    UpdatedUtc = now
+                })
+            {
+                IsUpsert = true
+            }).ToList();
+        if (writes.Count > 0)
+            await _counters.BulkWriteAsync(writes, new BulkWriteOptions { IsOrdered = false });

        if (questionExternalIds is null)
            _cache.Clear();
@@ -239,13 +291,31 @@ public sealed class AnswerStatisticsStore
                _cache.TryRemove(id, out _);
    }

-    private async Task<bool> IsBackfillCompleteAsync()
+    private async Task EnsureBaselinesAsync()
    {
-        if (_backfillComplete)
-            return true;
-        _backfillComplete = (await _state.Find(state => state.Id == StateId).FirstOrDefaultAsync())?.Completed == true;
-        return _backfillComplete;
+        var questions = await _questions.GetAllAsync();
+        var writes = questions.Select(question =>
+            new UpdateOneModel<QuestionPresentationBaseline>(
+                Builders<QuestionPresentationBaseline>.Filter.Eq(item => item.QuestionExternalId, question.ExternalId),
+                Builders<QuestionPresentationBaseline>.Update
+                    .SetOnInsert(item => item.QuestionExternalId, question.ExternalId)
+                    .SetOnInsert(item => item.ChoiceIds, question.Choices.Select(choice => choice.Id).ToList())
+                    .SetOnInsert(item => item.CreatedUtc, DateTime.UtcNow))
+            {
+                IsUpsert = true
+            }).ToList();
+        if (writes.Count > 0)
+            await _baselines.BulkWriteAsync(writes, new BulkWriteOptions { IsOrdered = false });
    }

+    private static FilterDefinition<QuestionAttemptCounter> CounterFilter(string questionId, int bucket, string choiceId) =>
+        Builders<QuestionAttemptCounter>.Filter.And(
+            Builders<QuestionAttemptCounter>.Filter.Eq(counter => counter.QuestionExternalId, questionId),
+            Builders<QuestionAttemptCounter>.Filter.Eq(counter => counter.Bucket, bucket),
+            Builders<QuestionAttemptCounter>.Filter.Eq(counter => counter.ChoiceId, choiceId));
+
+    private static (string Question, int Bucket, string Choice) CounterKey(BsonDocument id) =>
+        (id["q"].AsString, id["b"].ToInt32(), id["c"].AsString);
+
    private sealed record CachedPercentages(Dictionary<string, int> Percentages, DateTime ExpiresUtc);
 }
@@ -34,13 +34,16 @@ public class QuestionAttemptStore
        string userId,
        string choiceId,
        bool isCorrect,
-        bool isSkip = false)
+        bool isSkip = false,
+        IReadOnlyCollection<string>? presentedChoiceIds = null)
    {
        var attempt = new QuestionAttempt
        {
            QuestionExternalId = questionExternalId,
            UserId = userId,
            ChoiceId = choiceId,
+            PresentedChoiceIds = presentedChoiceIds?.ToList() ?? [],
+            TracksPresentation = presentedChoiceIds is { Count: > 0 },
            IsCorrect = isCorrect,
            IsSkip = isSkip,
            CounterBucket = GetCounterBucket(userId),
@@ -62,49 +65,6 @@ public class QuestionAttemptStore
        }
    }

-    public async Task<Dictionary<string, int>> GetPercentagesForQuestionAsync(string questionExternalId)
-    {
-        var results = await GetPercentagesByQuestionAsync([questionExternalId]);
-        return results.GetValueOrDefault(questionExternalId, []);
-    }
-
-    public async Task<Dictionary<string, Dictionary<string, int>>> GetPercentagesByQuestionAsync(IEnumerable<string> questionExternalIds)
-    {
-        var ids = questionExternalIds.Where(id => !string.IsNullOrWhiteSpace(id)).Distinct().ToList();
-        if (ids.Count == 0)
-            return [];
-
-        var totals = await _col.Aggregate()
-            .Match(a => ids.Contains(a.QuestionExternalId) && !a.IsSkip)
-            .Group(
-                attempt => new { attempt.QuestionExternalId, attempt.ChoiceId },
-                group => new QuestionAttemptCounterTotal
-                {
-                    Key = new QuestionChoiceCounterKey
-                    {
-                        QuestionExternalId = group.Key.QuestionExternalId,
-                        ChoiceId = group.Key.ChoiceId
-                    },
-                    Count = group.Sum(_ => 1L)
-                })
-            .ToListAsync();
-        var output = new Dictionary<string, Dictionary<string, int>>();
-
-        foreach (var group in totals.GroupBy(total => total.Key.QuestionExternalId))
-        {
-            var total = group.Sum(item => item.Count);
-            if (total == 0)
-                continue;
-
-            output[group.Key] = group
-                .ToDictionary(
-                    item => item.Key.ChoiceId,
-                    item => (int)Math.Round(item.Count * 100.0 / total));
-        }
-
-        return output;
-    }
-
    public Task<List<string>> GetActiveQuestionIdsAsync(DateTime sinceUtc) =>
        _col.Distinct(
            attempt => attempt.QuestionExternalId,
@@ -50,6 +50,32 @@ public class QuestionStore
    public async Task<Question?> GetByExternalIdAsync(string externalId) =>
        await _col.Find(q => q.ExternalId == externalId).FirstOrDefaultAsync();

+    public async Task<List<Question>> GetAllAsync() =>
+        await _col.Find(Builders<Question>.Filter.Empty).ToListAsync();
+
+    public async Task<(List<Question> Items, string? NextCursor, bool HasMore)> ExportPageAsync(
+        string? after,
+        int requestedLimit)
+    {
+        var limit = Math.Clamp(requestedLimit, 1, 200);
+        var filter = Builders<Question>.Filter.Empty;
+        if (!string.IsNullOrWhiteSpace(after))
+        {
+            if (!ObjectId.TryParse(after, out var cursor))
+                throw new ArgumentException("after must be a valid question cursor", nameof(after));
+            filter = new BsonDocument("_id", new BsonDocument("$gt", cursor));
+        }
+
+        var page = await _col.Find(filter)
+            .Sort(Builders<Question>.Sort.Ascending("_id"))
+            .Limit(limit + 1)
+            .ToListAsync();
+        var hasMore = page.Count > limit;
+        if (hasMore)
+            page.RemoveAt(page.Count - 1);
+        return (page, hasMore ? page[^1].Id : null, hasMore);
+    }
+
    public async Task<Question?> CreateAsync(UpsertQuestionRequest req)
    {
        var question = FromRequest(req);
@@ -0,0 +1,57 @@
+using QuestionsApi.Models;
+
+namespace QuestionsApi.Services;
+
+public static class QuestionValidator
+{
+    public static string? Validate(UpsertQuestionRequest req)
+    {
+        if (string.IsNullOrWhiteSpace(req.ExternalId)) return "externalId required";
+        if (string.IsNullOrWhiteSpace(req.Stem)) return "stem required";
+        if (string.IsNullOrWhiteSpace(req.LeadIn)) return "leadIn required";
+        if (req.Choices.Count < (req.IsPublished ? 5 : 2))
+            return req.IsPublished ? "published questions require at least five choices" : "at least two choices required";
+        if (req.Choices.Count > 20) return "questions support at most twenty choices";
+        if (req.Choices.Any(choice => string.IsNullOrWhiteSpace(choice.Id) || string.IsNullOrWhiteSpace(choice.Text)))
+            return "choice ids and text are required";
+
+        var choiceIds = req.Choices.Select(choice => choice.Id.Trim()).ToList();
+        if (choiceIds.Distinct(StringComparer.Ordinal).Count() != choiceIds.Count)
+            return "choice ids must be unique";
+        if (string.IsNullOrWhiteSpace(req.CorrectChoiceId)) return "correctChoiceId required";
+        if (!choiceIds.Contains(req.CorrectChoiceId.Trim(), StringComparer.Ordinal))
+            return "correctChoiceId must match a choice id";
+        if (string.IsNullOrWhiteSpace(req.Explanation)) return "explanation required";
+
+        var choiceIdSet = choiceIds.ToHashSet(StringComparer.Ordinal);
+        if (!choiceIdSet.SetEquals(req.ChoiceExplanations.Keys))
+            return "choiceExplanations must contain exactly one entry for every choice";
+        var incorrectChoiceIds = choiceIdSet
+            .Where(id => id != req.CorrectChoiceId.Trim())
+            .ToHashSet(StringComparer.Ordinal);
+        if (!incorrectChoiceIds.SetEquals(req.WrongAnswerFollowups.Keys))
+            return "wrongAnswerFollowups must contain exactly one entry for every incorrect choice";
+        if (req.ChoiceSelectionPercentages.Keys.Any(id => !choiceIdSet.Contains(id)))
+            return "choiceSelectionPercentages keys must match choice ids";
+        if (req.ChoiceSelectionPercentages.Any(entry => entry.Value is < 0 or > 100))
+            return "choiceSelectionPercentages values must be between 0 and 100";
+
+        foreach (var entry in req.WrongAnswerFollowups)
+        {
+            if (string.IsNullOrWhiteSpace(entry.Value.Prompt)) return "wrongAnswerFollowups prompt required";
+            if (string.IsNullOrWhiteSpace(entry.Value.TeachingPoint)) return "wrongAnswerFollowups teachingPoint required";
+            if (entry.Value.Choices.Count == 0) return "wrongAnswerFollowups choices required";
+            if (entry.Value.Choices.Any(choice => string.IsNullOrWhiteSpace(choice.Id) || string.IsNullOrWhiteSpace(choice.Text)))
+                return "wrongAnswerFollowups choice ids and text are required";
+            if (entry.Value.Choices.Select(choice => choice.Id).Distinct(StringComparer.Ordinal).Count() != entry.Value.Choices.Count)
+                return "wrongAnswerFollowups choice ids must be unique";
+            if (!entry.Value.Choices.Any(choice => choice.Id == entry.Value.CorrectChoiceId))
+                return "wrongAnswerFollowups correctChoiceId must match a follow-up choice id";
+        }
+
+        if (string.IsNullOrWhiteSpace(req.System)) return "system required";
+        if (string.IsNullOrWhiteSpace(req.Discipline)) return "discipline required";
+        if (req.Difficulty is < 1 or > 5) return "difficulty must be between 1 and 5";
+        return null;
+    }
+}
@@ -12,11 +12,13 @@ public sealed class ServiceMetrics
    private long _lastReconciliationUnixSeconds;
    private long _httpRequests;
    private long _httpDurationMicroseconds;
+    private long _answerStatisticsBackfillComplete;

    public void AttemptCreated() => Interlocked.Increment(ref _attemptsCreated);
    public void AttemptDuplicate() => Interlocked.Increment(ref _attemptDuplicates);
    public void CounterFailure() => Interlocked.Increment(ref _counterFailures);
    public void RatingFailure() => Interlocked.Increment(ref _ratingFailures);
+    public void AnswerStatisticsBackfillCompleted() => Interlocked.Exchange(ref _answerStatisticsBackfillComplete, 1);
    public void Reconciled()
    {
        Interlocked.Increment(ref _reconciliations);
@@ -48,6 +50,10 @@ public sealed class ServiceMetrics
        output.Append("step3_answer_last_reconciliation_unixtime ")
            .Append(Interlocked.Read(ref _lastReconciliationUnixSeconds))
            .AppendLine();
+        output.AppendLine("# TYPE step3_answer_statistics_v2_backfill_complete gauge");
+        output.Append("step3_answer_statistics_v2_backfill_complete ")
+            .Append(Interlocked.Read(ref _answerStatisticsBackfillComplete))
+            .AppendLine();
        return output.ToString();
    }