Add variable question distractor pools

2026-07-13 10:15:05 -05:00
parent f2c59388eb
commit 93154d5925
24 changed files with 1040 additions and 247 deletions
@@ -1,3 +1,4 @@
+using MongoDB.Bson;
 using MongoDB.Driver;
 using QuestionsApi.Models;
 using System.Collections.Concurrent;
@@ -6,20 +7,27 @@ namespace QuestionsApi.Services;

 public sealed class AnswerStatisticsStore
 {
-    private const string StateId = "answer-statistics-v1";
+    private const string StateId = "answer-statistics-v2";
    private static readonly TimeSpan CacheDuration = TimeSpan.FromMinutes(5);

    private readonly IMongoCollection<QuestionAttemptCounter> _counters;
    private readonly IMongoCollection<AnswerStatisticsState> _state;
+    private readonly IMongoCollection<QuestionPresentationBaseline> _baselines;
    private readonly QuestionAttemptStore _attempts;
+    private readonly QuestionStore _questions;
    private readonly ConcurrentDictionary<string, CachedPercentages> _cache = new();
    private volatile bool _backfillComplete;

-    public AnswerStatisticsStore(MongoDatabaseProvider mongo, QuestionAttemptStore attempts)
+    public AnswerStatisticsStore(
+        MongoDatabaseProvider mongo,
+        QuestionAttemptStore attempts,
+        QuestionStore questions)
    {
        _attempts = attempts;
-        _counters = mongo.Database.GetCollection<QuestionAttemptCounter>("QuestionAttemptCounters");
+        _questions = questions;
+        _counters = mongo.Database.GetCollection<QuestionAttemptCounter>("QuestionChoiceExposureCounters");
        _state = mongo.Database.GetCollection<AnswerStatisticsState>("SystemState");
+        _baselines = mongo.Database.GetCollection<QuestionPresentationBaseline>("QuestionPresentationBaselines");

        var counterIndex = Builders<QuestionAttemptCounter>.IndexKeys
            .Ascending(counter => counter.QuestionExternalId)
@@ -32,27 +40,31 @@ public sealed class AnswerStatisticsStore

    public async Task IncrementAsync(QuestionAttempt attempt)
    {
-        if (attempt.IsSkip)
+        if (attempt.IsSkip || !attempt.TracksPresentation || attempt.PresentedChoiceIds.Count == 0)
            return;

-        var filter = Builders<QuestionAttemptCounter>.Filter.And(
-            Builders<QuestionAttemptCounter>.Filter.Eq(counter => counter.QuestionExternalId, attempt.QuestionExternalId),
-            Builders<QuestionAttemptCounter>.Filter.Eq(counter => counter.Bucket, attempt.CounterBucket),
-            Builders<QuestionAttemptCounter>.Filter.Eq(counter => counter.ChoiceId, attempt.ChoiceId));
-        var update = Builders<QuestionAttemptCounter>.Update
-            .Inc(counter => counter.Count, 1)
-            .Set(counter => counter.UpdatedUtc, DateTime.UtcNow)
-            .SetOnInsert(counter => counter.QuestionExternalId, attempt.QuestionExternalId)
-            .SetOnInsert(counter => counter.Bucket, attempt.CounterBucket)
-            .SetOnInsert(counter => counter.ChoiceId, attempt.ChoiceId);
+        var now = DateTime.UtcNow;
+        var writes = attempt.PresentedChoiceIds.Distinct(StringComparer.Ordinal).Select(choiceId =>
+        {
+            var filter = CounterFilter(attempt.QuestionExternalId, attempt.CounterBucket, choiceId);
+            var update = Builders<QuestionAttemptCounter>.Update
+                .Inc(counter => counter.PresentationCount, 1)
+                .Inc(counter => counter.SelectionCount, choiceId == attempt.ChoiceId ? 1 : 0)
+                .Set(counter => counter.UpdatedUtc, now)
+                .SetOnInsert(counter => counter.QuestionExternalId, attempt.QuestionExternalId)
+                .SetOnInsert(counter => counter.Bucket, attempt.CounterBucket)
+                .SetOnInsert(counter => counter.ChoiceId, choiceId);
+            return new UpdateOneModel<QuestionAttemptCounter>(filter, update) { IsUpsert = true };
+        }).ToList();

        try
        {
-            await _counters.UpdateOneAsync(filter, update, new UpdateOptions { IsUpsert = true });
+            await _counters.BulkWriteAsync(writes, new BulkWriteOptions { IsOrdered = false });
        }
-        catch (MongoWriteException exception) when (exception.WriteError.Category == ServerErrorCategory.DuplicateKey)
+        catch (MongoBulkWriteException<QuestionAttemptCounter> exception)
+            when (exception.WriteErrors.All(error => error.Category == ServerErrorCategory.DuplicateKey))
        {
-            await _counters.UpdateOneAsync(filter, update);
+            await _counters.BulkWriteAsync(writes, new BulkWriteOptions { IsOrdered = false });
        }
        _cache.TryRemove(attempt.QuestionExternalId, out _);
    }
@@ -78,23 +90,9 @@ public sealed class AnswerStatisticsStore
            else
                missing.Add(id);
        }
-
-        if (missing.Count == 0)
+        if (missing.Count == 0 || !await IsBackfillCompleteAsync())
            return output;

-        var backfillComplete = await IsBackfillCompleteAsync();
-        if (!backfillComplete)
-        {
-            var rawPercentages = await _attempts.GetPercentagesByQuestionAsync(missing);
-            foreach (var id in missing)
-            {
-                var percentages = rawPercentages.GetValueOrDefault(id, []);
-                output[id] = percentages;
-                _cache[id] = new CachedPercentages(percentages, now.Add(CacheDuration));
-            }
-            return output;
-        }
-
        var totals = await _counters.Aggregate()
            .Match(counter => missing.Contains(counter.QuestionExternalId))
            .Group(
@@ -106,34 +104,22 @@ public sealed class AnswerStatisticsStore
                        QuestionExternalId = group.Key.QuestionExternalId,
                        ChoiceId = group.Key.ChoiceId
                    },
-                    Count = group.Sum(counter => counter.Count)
+                    SelectionCount = group.Sum(counter => counter.SelectionCount),
+                    PresentationCount = group.Sum(counter => counter.PresentationCount)
                })
            .ToListAsync();

-        var grouped = totals.GroupBy(total => total.Key.QuestionExternalId)
-            .ToDictionary(group => group.Key, group => group.ToList());
-
+        var grouped = totals.GroupBy(total => total.Key.QuestionExternalId).ToDictionary(group => group.Key);
        foreach (var id in missing)
        {
-            Dictionary<string, int> percentages;
-            if (grouped.TryGetValue(id, out var questionTotals))
-            {
-                var total = questionTotals.Sum(item => item.Count);
-                percentages = total == 0
-                    ? []
-                    : questionTotals.ToDictionary(
-                        item => item.Key.ChoiceId,
-                        item => (int)Math.Round(item.Count * 100.0 / total));
-            }
-            else
-            {
-                percentages = [];
-            }
-
+            var percentages = grouped.TryGetValue(id, out var choices)
+                ? choices.Where(item => item.PresentationCount > 0).ToDictionary(
+                    item => item.Key.ChoiceId,
+                    item => (int)Math.Round(item.SelectionCount * 100.0 / item.PresentationCount))
+                : [];
            output[id] = percentages;
            _cache[id] = new CachedPercentages(percentages, now.Add(CacheDuration));
        }
-
        return output;
    }

@@ -149,15 +135,14 @@ public sealed class AnswerStatisticsStore
        var update = Builders<AnswerStatisticsState>.Update
            .SetOnInsert(state => state.Id, StateId)
            .SetOnInsert(state => state.Completed, false)
+            .SetOnInsert(state => state.CutoverUtc, now)
            .Set(state => state.LeaseOwner, owner)
            .Set(state => state.LeaseUntilUtc, now.Add(duration))
            .Set(state => state.UpdatedUtc, now);

        try
        {
-            var leased = await _state.FindOneAndUpdateAsync(
-                filter,
-                update,
+            var leased = await _state.FindOneAndUpdateAsync(filter, update,
                new FindOneAndUpdateOptions<AnswerStatisticsState>
                {
                    IsUpsert = true,
@@ -173,64 +158,131 @@ public sealed class AnswerStatisticsStore

    public async Task MarkBackfillCompleteAsync(string owner)
    {
-        var update = Builders<AnswerStatisticsState>.Update
-            .Set(state => state.Completed, true)
-            .Set(state => state.LeaseOwner, owner)
-            .Set(state => state.UpdatedUtc, DateTime.UtcNow);
-        await _state.UpdateOneAsync(state => state.Id == StateId && state.LeaseOwner == owner, update);
+        await _state.UpdateOneAsync(
+            state => state.Id == StateId && state.LeaseOwner == owner,
+            Builders<AnswerStatisticsState>.Update
+                .Set(state => state.Completed, true)
+                .Set(state => state.UpdatedUtc, DateTime.UtcNow));
        _backfillComplete = true;
    }

-    public async Task RebuildAllAsync() => await RebuildAsync(null);
+    public Task RebuildAllAsync() => RebuildAsync(null);

-    public async Task RebuildQuestionsAsync(IReadOnlyCollection<string> questionExternalIds)
+    public Task RebuildQuestionsAsync(IReadOnlyCollection<string> questionExternalIds) =>
+        questionExternalIds.Count == 0 ? Task.CompletedTask : RebuildAsync(questionExternalIds);
+
+    public async Task<bool> IsBackfillCompleteAsync()
    {
-        if (questionExternalIds.Count > 0)
-            await RebuildAsync(questionExternalIds);
+        if (_backfillComplete)
+            return true;
+        _backfillComplete = (await _state.Find(state => state.Id == StateId).FirstOrDefaultAsync())?.Completed == true;
+        return _backfillComplete;
    }

    private async Task RebuildAsync(IReadOnlyCollection<string>? questionExternalIds)
    {
-        var aggregate = _attempts.Collection.Aggregate().Match(attempt => !attempt.IsSkip);
-        if (questionExternalIds is not null)
-            aggregate = aggregate.Match(attempt => questionExternalIds.Contains(attempt.QuestionExternalId));
-
-        var totals = await aggregate
-            .Group(
-                attempt => new { attempt.QuestionExternalId, attempt.CounterBucket, attempt.ChoiceId },
-                group => new QuestionAttemptCounterAggregate
-                {
-                    Key = new QuestionChoiceBucketKey
-                    {
-                        QuestionExternalId = group.Key.QuestionExternalId,
-                        Bucket = group.Key.CounterBucket,
-                        ChoiceId = group.Key.ChoiceId
-                    },
-                    Count = group.Sum(_ => 1L)
-                })
-            .ToListAsync();
-
-        if (totals.Count > 0)
+        await EnsureBaselinesAsync();
+        var state = await _state.Find(item => item.Id == StateId).FirstAsync();
+        var baselines = await _baselines.Find(Builders<QuestionPresentationBaseline>.Filter.Empty).ToListAsync();
+        var baselineByQuestion = baselines.ToDictionary(item => item.QuestionExternalId, item => item.ChoiceIds);
+        var match = new BsonDocument
        {
-            var now = DateTime.UtcNow;
-            var replacements = totals.Select(total =>
-            {
-                var counter = new QuestionAttemptCounter
+            { "IsSkip", false },
+            { "$or", new BsonArray
                {
-                    QuestionExternalId = total.Key.QuestionExternalId,
-                    Bucket = total.Key.Bucket,
-                    ChoiceId = total.Key.ChoiceId,
-                    Count = total.Count,
-                    UpdatedUtc = now
-                };
-                var filter = Builders<QuestionAttemptCounter>.Filter.And(
-                    Builders<QuestionAttemptCounter>.Filter.Eq(item => item.QuestionExternalId, counter.QuestionExternalId),
-                    Builders<QuestionAttemptCounter>.Filter.Eq(item => item.Bucket, counter.Bucket),
-                    Builders<QuestionAttemptCounter>.Filter.Eq(item => item.ChoiceId, counter.ChoiceId));
-                return new ReplaceOneModel<QuestionAttemptCounter>(filter, counter) { IsUpsert = true };
-            }).ToList();
-            await _counters.BulkWriteAsync(replacements, new BulkWriteOptions { IsOrdered = false });
+                    new BsonDocument("TracksPresentation", true),
+                    new BsonDocument("CreatedUtc", new BsonDocument("$lt", state.CutoverUtc))
+                }
+            }
+        };
+        if (questionExternalIds is not null)
+            match["QuestionExternalId"] = new BsonDocument("$in", new BsonArray(questionExternalIds));
+
+        var selectionPipeline = new[]
+        {
+            new BsonDocument("$match", match),
+            new BsonDocument("$group", new BsonDocument
+            {
+                { "_id", new BsonDocument { { "q", "$QuestionExternalId" }, { "b", "$CounterBucket" }, { "c", "$ChoiceId" } } },
+                { "count", new BsonDocument("$sum", 1) }
+            })
+        };
+        var presentationMatch = new BsonDocument(match) { ["TracksPresentation"] = true };
+        var presentationPipeline = new[]
+        {
+            new BsonDocument("$match", presentationMatch),
+            new BsonDocument("$unwind", "$PresentedChoiceIds"),
+            new BsonDocument("$group", new BsonDocument
+            {
+                { "_id", new BsonDocument { { "q", "$QuestionExternalId" }, { "b", "$CounterBucket" }, { "c", "$PresentedChoiceIds" } } },
+                { "count", new BsonDocument("$sum", 1) }
+            })
+        };
+        var legacyMatch = new BsonDocument
+        {
+            { "IsSkip", false },
+            { "TracksPresentation", new BsonDocument("$ne", true) },
+            { "CreatedUtc", new BsonDocument("$lt", state.CutoverUtc) }
+        };
+        if (questionExternalIds is not null)
+            legacyMatch["QuestionExternalId"] = new BsonDocument("$in", new BsonArray(questionExternalIds));
+        var legacyPipeline = new[]
+        {
+            new BsonDocument("$match", legacyMatch),
+            new BsonDocument("$group", new BsonDocument
+            {
+                { "_id", new BsonDocument { { "q", "$QuestionExternalId" }, { "b", "$CounterBucket" } } },
+                { "count", new BsonDocument("$sum", 1) }
+            })
+        };
+
+        var selections = await _attempts.Collection.Aggregate<BsonDocument>(selectionPipeline).ToListAsync();
+        var presentations = await _attempts.Collection.Aggregate<BsonDocument>(presentationPipeline).ToListAsync();
+        var legacy = await _attempts.Collection.Aggregate<BsonDocument>(legacyPipeline).ToListAsync();
+        var totals = new Dictionary<(string Question, int Bucket, string Choice), (long Selected, long Presented)>();
+
+        foreach (var row in selections)
+        {
+            var key = CounterKey(row["_id"].AsBsonDocument);
+            totals[key] = (row["count"].ToInt64(), totals.GetValueOrDefault(key).Presented);
        }
+        foreach (var row in presentations)
+        {
+            var key = CounterKey(row["_id"].AsBsonDocument);
+            totals[key] = (totals.GetValueOrDefault(key).Selected, row["count"].ToInt64());
+        }
+        foreach (var row in legacy)
+        {
+            var id = row["_id"].AsBsonDocument;
+            var questionId = id["q"].AsString;
+            if (!baselineByQuestion.TryGetValue(questionId, out var choiceIds))
+                continue;
+            foreach (var choiceId in choiceIds)
+            {
+                var key = (questionId, id["b"].ToInt32(), choiceId);
+                totals[key] = (totals.GetValueOrDefault(key).Selected,
+                    totals.GetValueOrDefault(key).Presented + row["count"].ToInt64());
+            }
+        }
+
+        var now = DateTime.UtcNow;
+        var writes = totals.Select(item =>
+            new ReplaceOneModel<QuestionAttemptCounter>(
+                CounterFilter(item.Key.Question, item.Key.Bucket, item.Key.Choice),
+                new QuestionAttemptCounter
+                {
+                    QuestionExternalId = item.Key.Question,
+                    Bucket = item.Key.Bucket,
+                    ChoiceId = item.Key.Choice,
+                    SelectionCount = item.Value.Selected,
+                    PresentationCount = item.Value.Presented,
+                    UpdatedUtc = now
+                })
+            {
+                IsUpsert = true
+            }).ToList();
+        if (writes.Count > 0)
+            await _counters.BulkWriteAsync(writes, new BulkWriteOptions { IsOrdered = false });

        if (questionExternalIds is null)
            _cache.Clear();
@@ -239,13 +291,31 @@ public sealed class AnswerStatisticsStore
                _cache.TryRemove(id, out _);
    }

-    private async Task<bool> IsBackfillCompleteAsync()
+    private async Task EnsureBaselinesAsync()
    {
-        if (_backfillComplete)
-            return true;
-        _backfillComplete = (await _state.Find(state => state.Id == StateId).FirstOrDefaultAsync())?.Completed == true;
-        return _backfillComplete;
+        var questions = await _questions.GetAllAsync();
+        var writes = questions.Select(question =>
+            new UpdateOneModel<QuestionPresentationBaseline>(
+                Builders<QuestionPresentationBaseline>.Filter.Eq(item => item.QuestionExternalId, question.ExternalId),
+                Builders<QuestionPresentationBaseline>.Update
+                    .SetOnInsert(item => item.QuestionExternalId, question.ExternalId)
+                    .SetOnInsert(item => item.ChoiceIds, question.Choices.Select(choice => choice.Id).ToList())
+                    .SetOnInsert(item => item.CreatedUtc, DateTime.UtcNow))
+            {
+                IsUpsert = true
+            }).ToList();
+        if (writes.Count > 0)
+            await _baselines.BulkWriteAsync(writes, new BulkWriteOptions { IsOrdered = false });
    }

+    private static FilterDefinition<QuestionAttemptCounter> CounterFilter(string questionId, int bucket, string choiceId) =>
+        Builders<QuestionAttemptCounter>.Filter.And(
+            Builders<QuestionAttemptCounter>.Filter.Eq(counter => counter.QuestionExternalId, questionId),
+            Builders<QuestionAttemptCounter>.Filter.Eq(counter => counter.Bucket, bucket),
+            Builders<QuestionAttemptCounter>.Filter.Eq(counter => counter.ChoiceId, choiceId));
+
+    private static (string Question, int Bucket, string Choice) CounterKey(BsonDocument id) =>
+        (id["q"].AsString, id["b"].ToInt32(), id["c"].AsString);
+
    private sealed record CachedPercentages(Dictionary<string, int> Percentages, DateTime ExpiresUtc);
 }