Scale unique answer tracking

api/QuestionsApi/Controllers/QuestionsController.cs

@@ -15,17 +15,23 @@ public class QuestionsController : ControllerBase
     private readonly QuestionAttemptStore _attempts;
     private readonly RatingStore _ratings;
     private readonly QuestionMediaStore _media;
+    private readonly AnswerStatisticsStore _statistics;
+    private readonly ServiceMetrics _metrics;
 
     public QuestionsController(
         QuestionStore questions,
         QuestionAttemptStore attempts,
         RatingStore ratings,
-        QuestionMediaStore media)
+        QuestionMediaStore media,
+        AnswerStatisticsStore statistics,
+        ServiceMetrics metrics)
     {
         _questions = questions;
         _attempts = attempts;
         _ratings = ratings;
         _media = media;
+        _statistics = statistics;
+        _metrics = metrics;
     }
 
     [HttpGet("systems")]
@@ -41,7 +47,7 @@ public class QuestionsController : ControllerBase
     public async Task<IActionResult> Search([FromBody] QuestionSearchRequest req)
     {
         var questions = await _questions.SearchAsync(req, User.IsInRole("SUPER"));
-        var percentages = await _attempts.GetPercentagesByQuestionAsync(questions.Select(q => q.ExternalId));
+        var percentages = await _statistics.GetPercentagesByQuestionAsync(questions.Select(q => q.ExternalId));
         var questionElos = await _ratings.GetQuestionElosAsync(questions.Select(q => q.ExternalId));
         return Ok(questions.Select(q =>
         {
@@ -58,7 +64,7 @@ public class QuestionsController : ControllerBase
         if (question is null || (!question.IsPublished && !User.IsInRole("SUPER")))
             return NotFound();
 
-        var percentages = await _attempts.GetPercentagesForQuestionAsync(question.ExternalId);
+        var percentages = await _statistics.GetPercentagesForQuestionAsync(question.ExternalId);
         var questionElo = await _ratings.GetQuestionEloAsync(question.ExternalId);
         return Ok(QuestionResponse.FromModel(question, percentages, questionElo));
     }
@@ -85,23 +91,58 @@ public class QuestionsController : ControllerBase
         if (string.IsNullOrWhiteSpace(userId))
             return Unauthorized();
 
-        var wasFirstAttempt = await _attempts.RecordFirstAttemptAsync(
+        var record = await _attempts.RecordFirstAttemptAsync(
             question.ExternalId,
             userId,
             req.ChoiceId.Trim(),
             req.ChoiceId.Trim() == question.CorrectChoiceId);
-        var rating = await _ratings.ApplyFirstResultAsync(
+        if (record.Inserted)
-            userId,
+        {
-            question,
+            _metrics.AttemptCreated();
-            req.ChoiceId.Trim() == question.CorrectChoiceId,
+            try
-            wasFirstAttempt);
+            {
+                await _statistics.IncrementAsync(record.Attempt);
+            }
+            catch
+            {
+                _metrics.CounterFailure();
+            }
+        }
+        else
+        {
+            _metrics.AttemptDuplicate();
+        }
+
+        var ratingOwner = Guid.NewGuid().ToString("N");
+        var shouldRate = await _attempts.TryAcquireRatingLeaseAsync(
+            record.Attempt.Id ?? "",
+            ratingOwner,
+            TimeSpan.FromSeconds(30));
+        RatingChangeResponse rating;
+        try
+        {
+            rating = await _ratings.ApplyFirstResultAsync(
+                userId,
+                question,
+                record.Attempt.IsCorrect,
+                shouldRate);
+            if (shouldRate)
+                await _attempts.MarkRatingAppliedAsync(record.Attempt.Id ?? "", ratingOwner);
+        }
+        catch
+        {
+            _metrics.RatingFailure();
+            if (shouldRate)
+                await _attempts.ReleaseRatingLeaseAsync(record.Attempt.Id ?? "", ratingOwner);
+            throw;
+        }
 
         return Ok(new FirstQuestionAttemptResponse
         {
             QuestionExternalId = question.ExternalId,
             ChoiceSelectionPercentages = QuestionResponse.BuildChoiceSelectionPercentages(
                 question,
-                await _attempts.GetPercentagesForQuestionAsync(question.ExternalId)),
+                await _statistics.GetPercentagesForQuestionAsync(question.ExternalId)),
             QuestionElo = rating.QuestionElo,
             Rating = rating
         });
@@ -164,13 +205,36 @@ public class QuestionsController : ControllerBase
         if (string.IsNullOrWhiteSpace(userId))
             return Unauthorized();
 
-        var wasFirstAttempt = await _attempts.RecordFirstAttemptAsync(
+        var record = await _attempts.RecordFirstAttemptAsync(
             question.ExternalId,
             userId,
             "",
             false,
             true);
-        var rating = await _ratings.ApplyFirstResultAsync(userId, question, false, wasFirstAttempt);
+        if (record.Inserted)
+            _metrics.AttemptCreated();
+        else
+            _metrics.AttemptDuplicate();
+
+        var ratingOwner = Guid.NewGuid().ToString("N");
+        var shouldRate = await _attempts.TryAcquireRatingLeaseAsync(
+            record.Attempt.Id ?? "",
+            ratingOwner,
+            TimeSpan.FromSeconds(30));
+        RatingChangeResponse rating;
+        try
+        {
+            rating = await _ratings.ApplyFirstResultAsync(userId, question, false, shouldRate);
+            if (shouldRate)
+                await _attempts.MarkRatingAppliedAsync(record.Attempt.Id ?? "", ratingOwner);
+        }
+        catch
+        {
+            _metrics.RatingFailure();
+            if (shouldRate)
+                await _attempts.ReleaseRatingLeaseAsync(record.Attempt.Id ?? "", ratingOwner);
+            throw;
+        }
 
         return Ok(new SkipQuestionResponse
         {
@@ -178,7 +242,7 @@ public class QuestionsController : ControllerBase
             CorrectChoiceId = question.CorrectChoiceId,
             ChoiceSelectionPercentages = QuestionResponse.BuildChoiceSelectionPercentages(
                 question,
-                await _attempts.GetPercentagesForQuestionAsync(question.ExternalId)),
+                await _statistics.GetPercentagesForQuestionAsync(question.ExternalId)),
             QuestionElo = rating.QuestionElo,
             Rating = rating
         });

api/QuestionsApi/Models/AnswerStatistics.cs

@@ -0,0 +1,49 @@
+using MongoDB.Bson.Serialization.Attributes;
+
+namespace QuestionsApi.Models;
+
+public sealed class QuestionAttemptCounter
+{
+    public string QuestionExternalId { get; set; } = "";
+    public int Bucket { get; set; }
+    public string ChoiceId { get; set; } = "";
+    public long Count { get; set; }
+    public DateTime UpdatedUtc { get; set; } = DateTime.UtcNow;
+}
+
+public sealed class QuestionAttemptCounterTotal
+{
+    [BsonId]
+    public QuestionChoiceCounterKey Key { get; set; } = new();
+    public long Count { get; set; }
+}
+
+public sealed class QuestionAttemptCounterAggregate
+{
+    [BsonId]
+    public QuestionChoiceBucketKey Key { get; set; } = new();
+    public long Count { get; set; }
+}
+
+public sealed class QuestionChoiceBucketKey
+{
+    public string QuestionExternalId { get; set; } = "";
+    public int Bucket { get; set; }
+    public string ChoiceId { get; set; } = "";
+}
+
+public sealed class QuestionChoiceCounterKey
+{
+    public string QuestionExternalId { get; set; } = "";
+    public string ChoiceId { get; set; } = "";
+}
+
+public sealed class AnswerStatisticsState
+{
+    [BsonId]
+    public string Id { get; set; } = "answer-statistics-v1";
+    public bool Completed { get; set; }
+    public string LeaseOwner { get; set; } = "";
+    public DateTime LeaseUntilUtc { get; set; }
+    public DateTime UpdatedUtc { get; set; } = DateTime.UtcNow;
+}

api/QuestionsApi/Models/QuestionAttempt.cs

@@ -14,5 +14,15 @@ public class QuestionAttempt
     public string ChoiceId { get; set; } = "";
     public bool IsCorrect { get; set; }
     public bool IsSkip { get; set; }
+    public int CounterBucket { get; set; }
+    public bool? RatingApplied { get; set; }
+    public string RatingLeaseOwner { get; set; } = "";
+    public DateTime RatingLeaseUntilUtc { get; set; }
     public DateTime CreatedUtc { get; set; } = DateTime.UtcNow;
 }
+
+public sealed class QuestionAttemptRecordResult
+{
+    public bool Inserted { get; set; }
+    public QuestionAttempt Attempt { get; set; } = new();
+}

api/QuestionsApi/Program.cs

@@ -3,16 +3,21 @@ using Microsoft.AspNetCore.Diagnostics;
 using Microsoft.IdentityModel.Tokens;
 using Microsoft.OpenApi.Models;
 using QuestionsApi.Services;
+using MongoDB.Bson;
 using System.Text;
 
 var builder = WebApplication.CreateBuilder(args);
 builder.Services.AddControllers();
 
+builder.Services.AddSingleton<MongoDatabaseProvider>();
 builder.Services.AddSingleton<QuestionStore>();
 builder.Services.AddSingleton<QuestionAttemptStore>();
+builder.Services.AddSingleton<AnswerStatisticsStore>();
 builder.Services.AddSingleton<RatingStore>();
 builder.Services.AddSingleton<QuestionMediaStore>();
 builder.Services.AddSingleton<QuestionReportStore>();
+builder.Services.AddSingleton<ServiceMetrics>();
+builder.Services.AddHostedService<AnswerStatisticsMaintenanceService>();
 
 builder.Services.AddEndpointsApiExplorer();
 builder.Services.AddSwaggerGen(c =>
@@ -61,6 +66,19 @@ builder.Services.AddAuthorization();
 
 var app = builder.Build();
 
+app.Use(async (context, next) =>
+{
+    var started = System.Diagnostics.Stopwatch.StartNew();
+    try
+    {
+        await next();
+    }
+    finally
+    {
+        context.RequestServices.GetRequiredService<ServiceMetrics>().HttpRequest(started.Elapsed);
+    }
+});
+
 app.UseExceptionHandler(errorApp =>
 {
     errorApp.Run(async context =>
@@ -96,6 +114,12 @@ app.UseExceptionHandler(errorApp =>
 });
 
 app.MapGet("/healthz", () => Results.Ok("ok"));
+app.MapGet("/readyz", async (MongoDatabaseProvider mongo) =>
+{
+    await mongo.Database.RunCommandAsync<BsonDocument>(new BsonDocument("ping", 1));
+    return Results.Ok("ready");
+});
+app.MapGet("/metrics", (ServiceMetrics metrics) => Results.Text(metrics.RenderPrometheus(), "text/plain; version=0.0.4"));
 app.UseSwagger();
 app.UseSwaggerUI(o =>
 {

api/QuestionsApi/README.md

@@ -19,6 +19,21 @@ STEP-style question bank service for study games and clients.
 
 Player reads require `USER` or `SUPER` JWT auth. Authoring endpoints require `SUPER`.
 
+## Answer tracking
+
+The `QuestionAttempts` collection is the immutable source of truth and enforces
+one first attempt per question and user with a unique compound index. Choice
+percentages are served from 32-way materialized counter buckets and cached for
+up to five minutes. A background service backfills counters on the first deploy
+and reconciles recently active questions every five minutes.
+
+- `GET /readyz` verifies MongoDB connectivity for readiness checks.
+- `GET /metrics` exposes Prometheus-format service and reconciliation metrics.
+
+The manifests in `k8s/` configure 2-10 API replicas. The cluster must have
+Kubernetes Metrics Server installed before applying them with
+`kubectl apply -k k8s/`.
+
 Questions submitted through authoring endpoints must have authoritative direct
 reference links recorded as described in the repository's **Question
 Verification References** policy. The references must verify the keyed answer

api/QuestionsApi/Services/AnswerStatisticsMaintenanceService.cs

@@ -0,0 +1,58 @@
+namespace QuestionsApi.Services;
+
+public sealed class AnswerStatisticsMaintenanceService : BackgroundService
+{
+    private readonly AnswerStatisticsStore _statistics;
+    private readonly QuestionAttemptStore _attempts;
+    private readonly ServiceMetrics _metrics;
+    private readonly ILogger<AnswerStatisticsMaintenanceService> _logger;
+    private readonly TimeSpan _interval;
+    private readonly string _owner = $"{Environment.MachineName}-{Guid.NewGuid():N}";
+
+    public AnswerStatisticsMaintenanceService(
+        AnswerStatisticsStore statistics,
+        QuestionAttemptStore attempts,
+        ServiceMetrics metrics,
+        IConfiguration configuration,
+        ILogger<AnswerStatisticsMaintenanceService> logger)
+    {
+        _statistics = statistics;
+        _attempts = attempts;
+        _metrics = metrics;
+        _logger = logger;
+        _interval = TimeSpan.FromSeconds(Math.Max(30, configuration.GetValue("AnswerStatistics:ReconcileSeconds", 300)));
+    }
+
+    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+    {
+        try
+        {
+            if (await _statistics.TryAcquireBackfillLeaseAsync(_owner, TimeSpan.FromMinutes(30)))
+            {
+                _logger.LogInformation("Backfilling materialized answer statistics");
+                await _statistics.RebuildAllAsync();
+                await _statistics.MarkBackfillCompleteAsync(_owner);
+                _metrics.Reconciled();
+            }
+        }
+        catch (Exception exception)
+        {
+            _logger.LogError(exception, "Initial answer-statistics backfill failed");
+        }
+
+        using var timer = new PeriodicTimer(_interval);
+        while (await timer.WaitForNextTickAsync(stoppingToken))
+        {
+            try
+            {
+                var activeIds = await _attempts.GetActiveQuestionIdsAsync(DateTime.UtcNow.Subtract(_interval).Subtract(_interval));
+                await _statistics.RebuildQuestionsAsync(activeIds);
+                _metrics.Reconciled();
+            }
+            catch (Exception exception)
+            {
+                _logger.LogError(exception, "Answer-statistics reconciliation failed");
+            }
+        }
+    }
+}

api/QuestionsApi/Services/AnswerStatisticsStore.cs

@@ -0,0 +1,251 @@
+using MongoDB.Driver;
+using QuestionsApi.Models;
+using System.Collections.Concurrent;
+
+namespace QuestionsApi.Services;
+
+public sealed class AnswerStatisticsStore
+{
+    private const string StateId = "answer-statistics-v1";
+    private static readonly TimeSpan CacheDuration = TimeSpan.FromMinutes(5);
+
+    private readonly IMongoCollection<QuestionAttemptCounter> _counters;
+    private readonly IMongoCollection<AnswerStatisticsState> _state;
+    private readonly QuestionAttemptStore _attempts;
+    private readonly ConcurrentDictionary<string, CachedPercentages> _cache = new();
+    private volatile bool _backfillComplete;
+
+    public AnswerStatisticsStore(MongoDatabaseProvider mongo, QuestionAttemptStore attempts)
+    {
+        _attempts = attempts;
+        _counters = mongo.Database.GetCollection<QuestionAttemptCounter>("QuestionAttemptCounters");
+        _state = mongo.Database.GetCollection<AnswerStatisticsState>("SystemState");
+
+        var counterIndex = Builders<QuestionAttemptCounter>.IndexKeys
+            .Ascending(counter => counter.QuestionExternalId)
+            .Ascending(counter => counter.Bucket)
+            .Ascending(counter => counter.ChoiceId);
+        _counters.Indexes.CreateOne(new CreateIndexModel<QuestionAttemptCounter>(
+            counterIndex,
+            new CreateIndexOptions { Unique = true }));
+    }
+
+    public async Task IncrementAsync(QuestionAttempt attempt)
+    {
+        if (attempt.IsSkip)
+            return;
+
+        var filter = Builders<QuestionAttemptCounter>.Filter.And(
+            Builders<QuestionAttemptCounter>.Filter.Eq(counter => counter.QuestionExternalId, attempt.QuestionExternalId),
+            Builders<QuestionAttemptCounter>.Filter.Eq(counter => counter.Bucket, attempt.CounterBucket),
+            Builders<QuestionAttemptCounter>.Filter.Eq(counter => counter.ChoiceId, attempt.ChoiceId));
+        var update = Builders<QuestionAttemptCounter>.Update
+            .Inc(counter => counter.Count, 1)
+            .Set(counter => counter.UpdatedUtc, DateTime.UtcNow)
+            .SetOnInsert(counter => counter.QuestionExternalId, attempt.QuestionExternalId)
+            .SetOnInsert(counter => counter.Bucket, attempt.CounterBucket)
+            .SetOnInsert(counter => counter.ChoiceId, attempt.ChoiceId);
+
+        try
+        {
+            await _counters.UpdateOneAsync(filter, update, new UpdateOptions { IsUpsert = true });
+        }
+        catch (MongoWriteException exception) when (exception.WriteError.Category == ServerErrorCategory.DuplicateKey)
+        {
+            await _counters.UpdateOneAsync(filter, update);
+        }
+        _cache.TryRemove(attempt.QuestionExternalId, out _);
+    }
+
+    public async Task<Dictionary<string, int>> GetPercentagesForQuestionAsync(string questionExternalId)
+    {
+        var result = await GetPercentagesByQuestionAsync([questionExternalId]);
+        return result.GetValueOrDefault(questionExternalId, []);
+    }
+
+    public async Task<Dictionary<string, Dictionary<string, int>>> GetPercentagesByQuestionAsync(
+        IEnumerable<string> questionExternalIds)
+    {
+        var ids = questionExternalIds.Where(id => !string.IsNullOrWhiteSpace(id)).Distinct().ToList();
+        var output = new Dictionary<string, Dictionary<string, int>>();
+        var missing = new List<string>();
+        var now = DateTime.UtcNow;
+
+        foreach (var id in ids)
+        {
+            if (_cache.TryGetValue(id, out var cached) && cached.ExpiresUtc > now)
+                output[id] = new Dictionary<string, int>(cached.Percentages);
+            else
+                missing.Add(id);
+        }
+
+        if (missing.Count == 0)
+            return output;
+
+        var backfillComplete = await IsBackfillCompleteAsync();
+        if (!backfillComplete)
+        {
+            var rawPercentages = await _attempts.GetPercentagesByQuestionAsync(missing);
+            foreach (var id in missing)
+            {
+                var percentages = rawPercentages.GetValueOrDefault(id, []);
+                output[id] = percentages;
+                _cache[id] = new CachedPercentages(percentages, now.Add(CacheDuration));
+            }
+            return output;
+        }
+
+        var totals = await _counters.Aggregate()
+            .Match(counter => missing.Contains(counter.QuestionExternalId))
+            .Group(
+                counter => new { counter.QuestionExternalId, counter.ChoiceId },
+                group => new QuestionAttemptCounterTotal
+                {
+                    Key = new QuestionChoiceCounterKey
+                    {
+                        QuestionExternalId = group.Key.QuestionExternalId,
+                        ChoiceId = group.Key.ChoiceId
+                    },
+                    Count = group.Sum(counter => counter.Count)
+                })
+            .ToListAsync();
+
+        var grouped = totals.GroupBy(total => total.Key.QuestionExternalId)
+            .ToDictionary(group => group.Key, group => group.ToList());
+
+        foreach (var id in missing)
+        {
+            Dictionary<string, int> percentages;
+            if (grouped.TryGetValue(id, out var questionTotals))
+            {
+                var total = questionTotals.Sum(item => item.Count);
+                percentages = total == 0
+                    ? []
+                    : questionTotals.ToDictionary(
+                        item => item.Key.ChoiceId,
+                        item => (int)Math.Round(item.Count * 100.0 / total));
+            }
+            else
+            {
+                percentages = [];
+            }
+
+            output[id] = percentages;
+            _cache[id] = new CachedPercentages(percentages, now.Add(CacheDuration));
+        }
+
+        return output;
+    }
+
+    public async Task<bool> TryAcquireBackfillLeaseAsync(string owner, TimeSpan duration)
+    {
+        var now = DateTime.UtcNow;
+        var filter = Builders<AnswerStatisticsState>.Filter.And(
+            Builders<AnswerStatisticsState>.Filter.Eq(state => state.Id, StateId),
+            Builders<AnswerStatisticsState>.Filter.Eq(state => state.Completed, false),
+            Builders<AnswerStatisticsState>.Filter.Or(
+                Builders<AnswerStatisticsState>.Filter.Lt(state => state.LeaseUntilUtc, now),
+                Builders<AnswerStatisticsState>.Filter.Eq(state => state.LeaseOwner, owner)));
+        var update = Builders<AnswerStatisticsState>.Update
+            .SetOnInsert(state => state.Id, StateId)
+            .SetOnInsert(state => state.Completed, false)
+            .Set(state => state.LeaseOwner, owner)
+            .Set(state => state.LeaseUntilUtc, now.Add(duration))
+            .Set(state => state.UpdatedUtc, now);
+
+        try
+        {
+            var leased = await _state.FindOneAndUpdateAsync(
+                filter,
+                update,
+                new FindOneAndUpdateOptions<AnswerStatisticsState>
+                {
+                    IsUpsert = true,
+                    ReturnDocument = ReturnDocument.After
+                });
+            return leased?.LeaseOwner == owner;
+        }
+        catch (MongoWriteException exception) when (exception.WriteError.Category == ServerErrorCategory.DuplicateKey)
+        {
+            return false;
+        }
+    }
+
+    public async Task MarkBackfillCompleteAsync(string owner)
+    {
+        var update = Builders<AnswerStatisticsState>.Update
+            .Set(state => state.Completed, true)
+            .Set(state => state.LeaseOwner, owner)
+            .Set(state => state.UpdatedUtc, DateTime.UtcNow);
+        await _state.UpdateOneAsync(state => state.Id == StateId && state.LeaseOwner == owner, update);
+        _backfillComplete = true;
+    }
+
+    public async Task RebuildAllAsync() => await RebuildAsync(null);
+
+    public async Task RebuildQuestionsAsync(IReadOnlyCollection<string> questionExternalIds)
+    {
+        if (questionExternalIds.Count > 0)
+            await RebuildAsync(questionExternalIds);
+    }
+
+    private async Task RebuildAsync(IReadOnlyCollection<string>? questionExternalIds)
+    {
+        var aggregate = _attempts.Collection.Aggregate().Match(attempt => !attempt.IsSkip);
+        if (questionExternalIds is not null)
+            aggregate = aggregate.Match(attempt => questionExternalIds.Contains(attempt.QuestionExternalId));
+
+        var totals = await aggregate
+            .Group(
+                attempt => new { attempt.QuestionExternalId, attempt.CounterBucket, attempt.ChoiceId },
+                group => new QuestionAttemptCounterAggregate
+                {
+                    Key = new QuestionChoiceBucketKey
+                    {
+                        QuestionExternalId = group.Key.QuestionExternalId,
+                        Bucket = group.Key.CounterBucket,
+                        ChoiceId = group.Key.ChoiceId
+                    },
+                    Count = group.Sum(_ => 1L)
+                })
+            .ToListAsync();
+
+        if (totals.Count > 0)
+        {
+            var now = DateTime.UtcNow;
+            var replacements = totals.Select(total =>
+            {
+                var counter = new QuestionAttemptCounter
+                {
+                    QuestionExternalId = total.Key.QuestionExternalId,
+                    Bucket = total.Key.Bucket,
+                    ChoiceId = total.Key.ChoiceId,
+                    Count = total.Count,
+                    UpdatedUtc = now
+                };
+                var filter = Builders<QuestionAttemptCounter>.Filter.And(
+                    Builders<QuestionAttemptCounter>.Filter.Eq(item => item.QuestionExternalId, counter.QuestionExternalId),
+                    Builders<QuestionAttemptCounter>.Filter.Eq(item => item.Bucket, counter.Bucket),
+                    Builders<QuestionAttemptCounter>.Filter.Eq(item => item.ChoiceId, counter.ChoiceId));
+                return new ReplaceOneModel<QuestionAttemptCounter>(filter, counter) { IsUpsert = true };
+            }).ToList();
+            await _counters.BulkWriteAsync(replacements, new BulkWriteOptions { IsOrdered = false });
+        }
+
+        if (questionExternalIds is null)
+            _cache.Clear();
+        else
+            foreach (var id in questionExternalIds)
+                _cache.TryRemove(id, out _);
+    }
+
+    private async Task<bool> IsBackfillCompleteAsync()
+    {
+        if (_backfillComplete)
+            return true;
+        _backfillComplete = (await _state.Find(state => state.Id == StateId).FirstOrDefaultAsync())?.Completed == true;
+        return _backfillComplete;
+    }
+
+    private sealed record CachedPercentages(Dictionary<string, int> Percentages, DateTime ExpiresUtc);
+}

api/QuestionsApi/Services/MongoDatabaseProvider.cs

@@ -0,0 +1,17 @@
+using MongoDB.Driver;
+
+namespace QuestionsApi.Services;
+
+public sealed class MongoDatabaseProvider
+{
+    public IMongoClient Client { get; }
+    public IMongoDatabase Database { get; }
+
+    public MongoDatabaseProvider(IConfiguration configuration)
+    {
+        var connectionString = configuration["MongoDB:ConnectionString"] ?? "mongodb://127.0.0.1:27017";
+        var databaseName = configuration["MongoDB:DatabaseName"] ?? "GameDb";
+        Client = new MongoClient(connectionString);
+        Database = Client.GetDatabase(databaseName);
+    }
+}

api/QuestionsApi/Services/QuestionAttemptStore.cs

@@ -1,5 +1,7 @@
 using MongoDB.Driver;
 using QuestionsApi.Models;
+using System.Security.Cryptography;
+using System.Text;
 
 namespace QuestionsApi.Services;
 
@@ -7,13 +9,11 @@ public class QuestionAttemptStore
 {
     private readonly IMongoCollection<QuestionAttempt> _col;
 
-    public QuestionAttemptStore(IConfiguration cfg)
+    public const int CounterBucketCount = 32;
+
+    public QuestionAttemptStore(MongoDatabaseProvider mongo)
     {
-        var cs = cfg["MongoDB:ConnectionString"] ?? "mongodb://127.0.0.1:27017";
+        _col = mongo.Database.GetCollection<QuestionAttempt>("QuestionAttempts");
-        var dbName = cfg["MongoDB:DatabaseName"] ?? "GameDb";
-        var client = new MongoClient(cs);
-        var db = client.GetDatabase(dbName);
-        _col = db.GetCollection<QuestionAttempt>("QuestionAttempts");
 
         var firstAttemptIndex = Builders<QuestionAttempt>.IndexKeys
             .Ascending(a => a.QuestionExternalId)
@@ -24,9 +24,17 @@ public class QuestionAttemptStore
 
         var questionIndex = Builders<QuestionAttempt>.IndexKeys.Ascending(a => a.QuestionExternalId);
         _col.Indexes.CreateOne(new CreateIndexModel<QuestionAttempt>(questionIndex));
+
+        var createdIndex = Builders<QuestionAttempt>.IndexKeys.Ascending(a => a.CreatedUtc);
+        _col.Indexes.CreateOne(new CreateIndexModel<QuestionAttempt>(createdIndex));
     }
 
-    public async Task<bool> RecordFirstAttemptAsync(string questionExternalId, string userId, string choiceId, bool isCorrect, bool isSkip = false)
+    public async Task<QuestionAttemptRecordResult> RecordFirstAttemptAsync(
+        string questionExternalId,
+        string userId,
+        string choiceId,
+        bool isCorrect,
+        bool isSkip = false)
     {
         var attempt = new QuestionAttempt
         {
@@ -35,17 +43,22 @@ public class QuestionAttemptStore
             ChoiceId = choiceId,
             IsCorrect = isCorrect,
             IsSkip = isSkip,
+            CounterBucket = GetCounterBucket(userId),
+            RatingApplied = false,
             CreatedUtc = DateTime.UtcNow
         };
 
         try
         {
             await _col.InsertOneAsync(attempt);
-            return true;
+            return new QuestionAttemptRecordResult { Inserted = true, Attempt = attempt };
         }
         catch (MongoWriteException ex) when (ex.WriteError.Category == ServerErrorCategory.DuplicateKey)
         {
-            return false;
+            var existing = await _col
+                .Find(a => a.QuestionExternalId == questionExternalId && a.UserId == userId)
+                .FirstAsync();
+            return new QuestionAttemptRecordResult { Inserted = false, Attempt = existing };
         }
     }
 
@@ -61,22 +74,80 @@ public class QuestionAttemptStore
         if (ids.Count == 0)
             return [];
 
-        var attempts = await _col.Find(a => ids.Contains(a.QuestionExternalId) && !a.IsSkip).ToListAsync();
+        var totals = await _col.Aggregate()
+            .Match(a => ids.Contains(a.QuestionExternalId) && !a.IsSkip)
+            .Group(
+                attempt => new { attempt.QuestionExternalId, attempt.ChoiceId },
+                group => new QuestionAttemptCounterTotal
+                {
+                    Key = new QuestionChoiceCounterKey
+                    {
+                        QuestionExternalId = group.Key.QuestionExternalId,
+                        ChoiceId = group.Key.ChoiceId
+                    },
+                    Count = group.Sum(_ => 1L)
+                })
+            .ToListAsync();
         var output = new Dictionary<string, Dictionary<string, int>>();
 
-        foreach (var group in attempts.GroupBy(a => a.QuestionExternalId))
+        foreach (var group in totals.GroupBy(total => total.Key.QuestionExternalId))
         {
-            var total = group.Count();
+            var total = group.Sum(item => item.Count);
             if (total == 0)
                 continue;
 
             output[group.Key] = group
-                .GroupBy(a => a.ChoiceId)
                 .ToDictionary(
-                    choiceGroup => choiceGroup.Key,
+                    item => item.Key.ChoiceId,
-                    choiceGroup => (int)Math.Round(choiceGroup.Count() * 100.0 / total));
+                    item => (int)Math.Round(item.Count * 100.0 / total));
         }
 
         return output;
     }
+
+    public Task<List<string>> GetActiveQuestionIdsAsync(DateTime sinceUtc) =>
+        _col.Distinct(
+            attempt => attempt.QuestionExternalId,
+            Builders<QuestionAttempt>.Filter.Gte(attempt => attempt.CreatedUtc, sinceUtc))
+            .ToListAsync();
+
+    public IMongoCollection<QuestionAttempt> Collection => _col;
+
+    public async Task<bool> TryAcquireRatingLeaseAsync(string attemptId, string owner, TimeSpan duration)
+    {
+        var now = DateTime.UtcNow;
+        var filter = Builders<QuestionAttempt>.Filter.And(
+            Builders<QuestionAttempt>.Filter.Eq(attempt => attempt.Id, attemptId),
+            Builders<QuestionAttempt>.Filter.Exists(attempt => attempt.RatingApplied, true),
+            Builders<QuestionAttempt>.Filter.Eq(attempt => attempt.RatingApplied, false),
+            Builders<QuestionAttempt>.Filter.Or(
+                Builders<QuestionAttempt>.Filter.Lt(attempt => attempt.RatingLeaseUntilUtc, now),
+                Builders<QuestionAttempt>.Filter.Eq(attempt => attempt.RatingLeaseOwner, owner)));
+        var update = Builders<QuestionAttempt>.Update
+            .Set(attempt => attempt.RatingLeaseOwner, owner)
+            .Set(attempt => attempt.RatingLeaseUntilUtc, now.Add(duration));
+        var leased = await _col.FindOneAndUpdateAsync(
+            filter,
+            update,
+            new FindOneAndUpdateOptions<QuestionAttempt> { ReturnDocument = ReturnDocument.After });
+        return leased?.RatingLeaseOwner == owner;
+    }
+
+    public Task MarkRatingAppliedAsync(string attemptId, string owner) =>
+        _col.UpdateOneAsync(
+            attempt => attempt.Id == attemptId && attempt.RatingLeaseOwner == owner,
+            Builders<QuestionAttempt>.Update
+                .Set(attempt => attempt.RatingApplied, true)
+                .Set(attempt => attempt.RatingLeaseUntilUtc, DateTime.MinValue));
+
+    public Task ReleaseRatingLeaseAsync(string attemptId, string owner) =>
+        _col.UpdateOneAsync(
+            attempt => attempt.Id == attemptId && attempt.RatingLeaseOwner == owner && attempt.RatingApplied == false,
+            Builders<QuestionAttempt>.Update.Set(attempt => attempt.RatingLeaseUntilUtc, DateTime.MinValue));
+
+    public static int GetCounterBucket(string userId)
+    {
+        var hash = SHA256.HashData(Encoding.UTF8.GetBytes(userId));
+        return (int)(BitConverter.ToUInt32(hash, 0) % CounterBucketCount);
+    }
 }

api/QuestionsApi/Services/QuestionMediaStore.cs

@@ -8,13 +8,9 @@ public class QuestionMediaStore
 {
     private readonly IMongoCollection<QuestionMediaAsset> _col;
 
-    public QuestionMediaStore(IConfiguration cfg)
+    public QuestionMediaStore(MongoDatabaseProvider mongo)
     {
-        var cs = cfg["MongoDB:ConnectionString"] ?? "mongodb://127.0.0.1:27017";
+        _col = mongo.Database.GetCollection<QuestionMediaAsset>("QuestionMediaAssets");
-        var dbName = cfg["MongoDB:DatabaseName"] ?? "GameDb";
-        var client = new MongoClient(cs);
-        var db = client.GetDatabase(dbName);
-        _col = db.GetCollection<QuestionMediaAsset>("QuestionMediaAssets");
     }
 
     public async Task<QuestionMediaAsset> CreateAsync(string fileName, string contentType, byte[] bytes)

api/QuestionsApi/Services/QuestionReportStore.cs

@@ -8,13 +8,9 @@ public class QuestionReportStore
 {
     private readonly IMongoCollection<QuestionReport> _reports;
 
-    public QuestionReportStore(IConfiguration cfg)
+    public QuestionReportStore(MongoDatabaseProvider mongo)
     {
-        var connectionString = cfg["MongoDB:ConnectionString"] ?? "mongodb://127.0.0.1:27017";
+        _reports = mongo.Database.GetCollection<QuestionReport>("QuestionReports");
-        var databaseName = cfg["MongoDB:DatabaseName"] ?? "GameDb";
-        var client = new MongoClient(connectionString);
-        var database = client.GetDatabase(databaseName);
-        _reports = database.GetCollection<QuestionReport>("QuestionReports");
 
         _reports.Indexes.CreateMany(
         [

api/QuestionsApi/Services/QuestionStore.cs

@@ -8,13 +8,9 @@ public class QuestionStore
 {
     private readonly IMongoCollection<Question> _col;
 
-    public QuestionStore(IConfiguration cfg)
+    public QuestionStore(MongoDatabaseProvider mongo)
     {
-        var cs = cfg["MongoDB:ConnectionString"] ?? "mongodb://127.0.0.1:27017";
+        _col = mongo.Database.GetCollection<Question>("Questions");
-        var dbName = cfg["MongoDB:DatabaseName"] ?? "GameDb";
-        var client = new MongoClient(cs);
-        var db = client.GetDatabase(dbName);
-        _col = db.GetCollection<Question>("Questions");
 
         var externalIdIndex = Builders<Question>.IndexKeys.Ascending(q => q.ExternalId);
         _col.Indexes.CreateOne(new CreateIndexModel<Question>(externalIdIndex, new CreateIndexOptions { Unique = true }));

api/QuestionsApi/Services/RatingStore.cs

@@ -11,14 +11,10 @@ public class RatingStore
     private readonly IMongoCollection<UserRating> _users;
     private readonly IMongoCollection<QuestionRating> _questions;
 
-    public RatingStore(IConfiguration cfg)
+    public RatingStore(MongoDatabaseProvider mongo)
     {
-        var cs = cfg["MongoDB:ConnectionString"] ?? "mongodb://127.0.0.1:27017";
+        _users = mongo.Database.GetCollection<UserRating>("UserRatings");
-        var dbName = cfg["MongoDB:DatabaseName"] ?? "GameDb";
+        _questions = mongo.Database.GetCollection<QuestionRating>("QuestionRatings");
-        var client = new MongoClient(cs);
-        var db = client.GetDatabase(dbName);
-        _users = db.GetCollection<UserRating>("UserRatings");
-        _questions = db.GetCollection<QuestionRating>("QuestionRatings");
 
         _users.Indexes.CreateOne(new CreateIndexModel<UserRating>(
             Builders<UserRating>.IndexKeys.Ascending(r => r.UserId),
@@ -89,64 +85,107 @@ public class RatingStore
         var categoryDelta = (int)Math.Round(userK * (actual - categoryExpected));
         var questionDelta = (int)Math.Round(questionK * (overallExpected - actual));
 
-        userRating.OverallElo = Math.Max(MinElo, oldOverall + overallDelta);
+        var now = DateTime.UtcNow;
-        userRating.CategoryElos[category] = Math.Max(MinElo, oldCategory + categoryDelta);
+        var categoryPath = $"CategoryElos.{category}";
-        userRating.RatedQuestionCount++;
+        var userUpdate = Builders<UserRating>.Update
-        userRating.UpdatedUtc = DateTime.UtcNow;
+            .Inc(rating => rating.OverallElo, overallDelta)
+            .Inc(categoryPath, categoryDelta)
+            .Inc(rating => rating.RatedQuestionCount, 1)
+            .Set(rating => rating.UpdatedUtc, now);
+        await _users.UpdateOneAsync(rating => rating.UserId == userId, userUpdate);
+        await _users.UpdateOneAsync(
+            rating => rating.UserId == userId,
+            Builders<UserRating>.Update
+                .Max(rating => rating.OverallElo, MinElo)
+                .Max(categoryPath, MinElo));
 
-        questionRating.Elo = Math.Max(MinElo, oldQuestion + questionDelta);
+        var questionUpdate = Builders<QuestionRating>.Update
-        questionRating.AttemptCount++;
+            .Inc(rating => rating.Elo, questionDelta)
-        questionRating.UpdatedUtc = DateTime.UtcNow;
+            .Inc(rating => rating.AttemptCount, 1)
+            .Set(rating => rating.UpdatedUtc, now);
+        await _questions.UpdateOneAsync(rating => rating.QuestionExternalId == question.ExternalId, questionUpdate);
+        await _questions.UpdateOneAsync(
+            rating => rating.QuestionExternalId == question.ExternalId,
+            Builders<QuestionRating>.Update.Max(rating => rating.Elo, MinElo));
 
-        await _users.ReplaceOneAsync(r => r.UserId == userRating.UserId, userRating, new ReplaceOptions { IsUpsert = true });
+        var updatedUser = await _users.Find(rating => rating.UserId == userId).FirstAsync();
-        await _questions.ReplaceOneAsync(r => r.QuestionExternalId == questionRating.QuestionExternalId, questionRating, new ReplaceOptions { IsUpsert = true });
+        var updatedQuestion = await _questions.Find(rating => rating.QuestionExternalId == question.ExternalId).FirstAsync();
 
         return new RatingChangeResponse
         {
             Rated = true,
             Category = category,
-            UserOverallElo = userRating.OverallElo,
+            UserOverallElo = updatedUser.OverallElo,
-            UserOverallDelta = userRating.OverallElo - oldOverall,
+            UserOverallDelta = overallDelta,
-            UserCategoryElo = userRating.CategoryElos[category],
+            UserCategoryElo = updatedUser.CategoryElos.GetValueOrDefault(category, BaseElo),
-            UserCategoryDelta = userRating.CategoryElos[category] - oldCategory,
+            UserCategoryDelta = categoryDelta,
-            QuestionElo = questionRating.Elo,
+            QuestionElo = updatedQuestion.Elo,
-            QuestionEloDelta = questionRating.Elo - oldQuestion
+            QuestionEloDelta = questionDelta
         };
     }
 
     private async Task<UserRating> GetOrCreateUserRatingAsync(string userId)
     {
-        var rating = await _users.Find(r => r.UserId == userId).FirstOrDefaultAsync();
+        var create = Builders<UserRating>.Update
-        if (rating is null)
+            .SetOnInsert(rating => rating.UserId, userId)
+            .SetOnInsert(rating => rating.OverallElo, BaseElo)
+            .SetOnInsert(rating => rating.RatedQuestionCount, 0)
+            .SetOnInsert(rating => rating.CategoryElos, DefaultCategoryElos())
+            .SetOnInsert(rating => rating.UpdatedUtc, DateTime.UtcNow);
+        UserRating rating;
+        try
         {
-            rating = new UserRating { UserId = userId, CategoryElos = DefaultCategoryElos() };
+            rating = await _users.FindOneAndUpdateAsync(
-            await _users.InsertOneAsync(rating);
+                item => item.UserId == userId,
-            return rating;
+                create,
+                new FindOneAndUpdateOptions<UserRating>
+                {
+                    IsUpsert = true,
+                    ReturnDocument = ReturnDocument.After
+                });
+        }
+        catch (MongoWriteException exception) when (exception.WriteError.Category == ServerErrorCategory.DuplicateKey)
+        {
+            rating = await _users.Find(item => item.UserId == userId).FirstAsync();
         }
 
-        var changed = false;
+        var missingCategories = RatingCategory.Categories
-        foreach (var category in RatingCategory.Categories)
+            .Where(category => !rating.CategoryElos.ContainsKey(category))
+            .ToList();
+        if (missingCategories.Count > 0)
         {
-            if (!rating.CategoryElos.ContainsKey(category))
+            var updates = missingCategories
-            {
+                .Select(category => Builders<UserRating>.Update.Set($"CategoryElos.{category}", BaseElo))
-                rating.CategoryElos[category] = BaseElo;
+                .ToList();
-                changed = true;
+            if (updates.Count > 0)
-            }
+                await _users.UpdateOneAsync(item => item.UserId == userId, Builders<UserRating>.Update.Combine(updates));
+            rating = await _users.Find(item => item.UserId == userId).FirstAsync();
         }
-        if (changed)
-            await _users.ReplaceOneAsync(r => r.UserId == userId, rating);
         return rating;
     }
 
     private async Task<QuestionRating> GetOrCreateQuestionRatingAsync(string questionExternalId)
     {
-        var rating = await _questions.Find(r => r.QuestionExternalId == questionExternalId).FirstOrDefaultAsync();
+        var create = Builders<QuestionRating>.Update
-        if (rating is not null)
+            .SetOnInsert(rating => rating.QuestionExternalId, questionExternalId)
-            return rating;
+            .SetOnInsert(rating => rating.Elo, BaseElo)
-
+            .SetOnInsert(rating => rating.AttemptCount, 0)
-        rating = new QuestionRating { QuestionExternalId = questionExternalId };
+            .SetOnInsert(rating => rating.UpdatedUtc, DateTime.UtcNow);
-        await _questions.InsertOneAsync(rating);
+        try
-        return rating;
+        {
+            return await _questions.FindOneAndUpdateAsync(
+                rating => rating.QuestionExternalId == questionExternalId,
+                create,
+                new FindOneAndUpdateOptions<QuestionRating>
+                {
+                    IsUpsert = true,
+                    ReturnDocument = ReturnDocument.After
+                });
+        }
+        catch (MongoWriteException exception) when (exception.WriteError.Category == ServerErrorCategory.DuplicateKey)
+        {
+            return await _questions.Find(rating => rating.QuestionExternalId == questionExternalId).FirstAsync();
+        }
     }
 
     private static Dictionary<string, int> DefaultCategoryElos() =>

api/QuestionsApi/Services/ServiceMetrics.cs

@@ -0,0 +1,59 @@
+using System.Text;
+
+namespace QuestionsApi.Services;
+
+public sealed class ServiceMetrics
+{
+    private long _attemptsCreated;
+    private long _attemptDuplicates;
+    private long _counterFailures;
+    private long _ratingFailures;
+    private long _reconciliations;
+    private long _lastReconciliationUnixSeconds;
+    private long _httpRequests;
+    private long _httpDurationMicroseconds;
+
+    public void AttemptCreated() => Interlocked.Increment(ref _attemptsCreated);
+    public void AttemptDuplicate() => Interlocked.Increment(ref _attemptDuplicates);
+    public void CounterFailure() => Interlocked.Increment(ref _counterFailures);
+    public void RatingFailure() => Interlocked.Increment(ref _ratingFailures);
+    public void Reconciled()
+    {
+        Interlocked.Increment(ref _reconciliations);
+        Interlocked.Exchange(ref _lastReconciliationUnixSeconds, DateTimeOffset.UtcNow.ToUnixTimeSeconds());
+    }
+    public void HttpRequest(TimeSpan duration)
+    {
+        Interlocked.Increment(ref _httpRequests);
+        Interlocked.Add(ref _httpDurationMicroseconds, (long)(duration.TotalMilliseconds * 1000));
+    }
+
+    public string RenderPrometheus()
+    {
+        var output = new StringBuilder();
+        AppendCounter(output, "step3_question_attempts_created_total", Interlocked.Read(ref _attemptsCreated));
+        AppendCounter(output, "step3_question_attempt_duplicates_total", Interlocked.Read(ref _attemptDuplicates));
+        AppendCounter(output, "step3_answer_counter_failures_total", Interlocked.Read(ref _counterFailures));
+        AppendCounter(output, "step3_rating_failures_total", Interlocked.Read(ref _ratingFailures));
+        AppendCounter(output, "step3_answer_reconciliations_total", Interlocked.Read(ref _reconciliations));
+        AppendCounter(output, "step3_http_requests_total", Interlocked.Read(ref _httpRequests));
+        output.AppendLine("# TYPE step3_http_request_duration_seconds summary");
+        output.Append("step3_http_request_duration_seconds_count ")
+            .Append(Interlocked.Read(ref _httpRequests))
+            .AppendLine();
+        output.Append("step3_http_request_duration_seconds_sum ")
+            .Append((Interlocked.Read(ref _httpDurationMicroseconds) / 1_000_000.0).ToString("0.######", System.Globalization.CultureInfo.InvariantCulture))
+            .AppendLine();
+        output.AppendLine("# TYPE step3_answer_last_reconciliation_unixtime gauge");
+        output.Append("step3_answer_last_reconciliation_unixtime ")
+            .Append(Interlocked.Read(ref _lastReconciliationUnixSeconds))
+            .AppendLine();
+        return output.ToString();
+    }
+
+    private static void AppendCounter(StringBuilder output, string name, long value)
+    {
+        output.Append("# TYPE ").Append(name).AppendLine(" counter");
+        output.Append(name).Append(' ').Append(value).AppendLine();
+    }
+}

api/QuestionsApi/appsettings.Development.json

@@ -1,6 +1,7 @@
 {
   "Kestrel": { "Endpoints": { "Http": { "Url": "http://0.0.0.0:5008" } } },
   "MongoDB": { "ConnectionString": "mongodb://192.168.86.50:27017", "DatabaseName": "promiscuity" },
+  "AnswerStatistics": { "ReconcileSeconds": 300 },
   "Jwt": { "Key": "SuperUltraSecureJwtKeyWithAtLeast32Chars!!", "Issuer": "promiscuity", "Audience": "promiscuity-auth-api" },
   "Logging": { "LogLevel": { "Default": "Information" } }
 }

api/QuestionsApi/appsettings.json

@@ -1,6 +1,7 @@
 {
   "Kestrel": { "Endpoints": { "Http": { "Url": "http://0.0.0.0:5008" } } },
   "MongoDB": { "ConnectionString": "mongodb://192.168.86.50:27017", "DatabaseName": "promiscuity" },
+  "AnswerStatistics": { "ReconcileSeconds": 300 },
   "Jwt": { "Key": "SuperUltraSecureJwtKeyWithAtLeast32Chars!!", "Issuer": "promiscuity", "Audience": "promiscuity-auth-api" },
   "Logging": { "LogLevel": { "Default": "Information" } },
   "AllowedHosts": "*"

api/QuestionsApi/k8s/deployment.yaml

@@ -6,6 +6,11 @@ metadata:
     app: step3-questions
 spec:
   replicas: 2
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: 0
+      maxSurge: 1
   selector:
     matchLabels:
       app: step3-questions
@@ -13,6 +18,10 @@ spec:
     metadata:
       labels:
         app: step3-questions
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/path: "/metrics"
+        prometheus.io/port: "5008"
     spec:
       containers:
         - name: step3-questions
@@ -20,9 +29,26 @@ spec:
           imagePullPolicy: IfNotPresent
           ports:
             - containerPort: 5008
+          resources:
+            requests:
+              cpu: 250m
+              memory: 256Mi
+            limits:
+              cpu: "1"
+              memory: 512Mi
           readinessProbe:
             httpGet:
-              path: /healthz
+              path: /readyz
               port: 5008
             initialDelaySeconds: 5
             periodSeconds: 10
+            timeoutSeconds: 3
+            failureThreshold: 3
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: 5008
+            initialDelaySeconds: 15
+            periodSeconds: 20
+            timeoutSeconds: 3
+            failureThreshold: 3

api/QuestionsApi/k8s/hpa.yaml

@@ -0,0 +1,35 @@
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+  name: step3-questions
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: step3-questions
+  minReplicas: 2
+  maxReplicas: 10
+  behavior:
+    scaleUp:
+      stabilizationWindowSeconds: 0
+      policies:
+        - type: Percent
+          value: 100
+          periodSeconds: 60
+        - type: Pods
+          value: 2
+          periodSeconds: 60
+      selectPolicy: Max
+    scaleDown:
+      stabilizationWindowSeconds: 300
+      policies:
+        - type: Percent
+          value: 25
+          periodSeconds: 60
+  metrics:
+    - type: Resource
+      resource:
+        name: cpu
+        target:
+          type: Utilization
+          averageUtilization: 60

api/QuestionsApi/k8s/kustomization.yaml

@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - deployment.yaml
+  - service.yaml
+  - hpa.yaml
+  - pdb.yaml

api/QuestionsApi/k8s/pdb.yaml

@@ -0,0 +1,9 @@
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: step3-questions
+spec:
+  minAvailable: 1
+  selector:
+    matchLabels:
+      app: step3-questions

api/QuestionsApi/load/README.md

@@ -0,0 +1,14 @@
+# Answer Tracking Load Test
+
+Generate 5,000 or more short-lived USER JWTs for dedicated load-test accounts and
+store them as a JSON string array outside the repository. Never reuse production
+user credentials.
+
+Run the test from this directory:
+
+```powershell
+k6 run -e BASE_URL=https://s3q.ranaze.com -e AUTH_TOKENS_PATH=./tokens.json answer-tracking.js
+```
+
+The test ramps to 5,000 concurrent users and validates the production latency and
+error-rate targets. Run it against a staging environment before production.

api/QuestionsApi/load/answer-tracking.js

@@ -0,0 +1,76 @@
+import http from "k6/http";
+import { check, sleep } from "k6";
+import { SharedArray } from "k6/data";
+
+const baseUrl = __ENV.BASE_URL || "https://s3q.ranaze.com";
+const tokenPath = __ENV.AUTH_TOKENS_PATH || "./tokens.json";
+const tokens = new SharedArray("test-user tokens", () => JSON.parse(open(tokenPath)));
+
+if (tokens.length < 5000) {
+  throw new Error("The load test requires at least 5,000 distinct USER tokens.");
+}
+
+export const options = {
+  scenarios: {
+    answer_tracking: {
+      executor: "ramping-vus",
+      startVUs: 0,
+      stages: [
+        { duration: "2m", target: 5000 },
+        { duration: "5m", target: 5000 },
+        { duration: "1m", target: 0 },
+      ],
+      gracefulRampDown: "30s",
+    },
+  },
+  thresholds: {
+    http_req_failed: ["rate<0.01"],
+    "http_req_duration{operation:search}": ["p(95)<1000"],
+    "http_req_duration{operation:answer}": ["p(95)<500"],
+  },
+};
+
+export default function () {
+  const token = tokens[(__VU - 1) % tokens.length];
+  const headers = {
+    Authorization: `Bearer ${token}`,
+    "Content-Type": "application/json",
+  };
+
+  const search = http.post(
+    `${baseUrl}/api/questions/search`,
+    JSON.stringify({ limit: 80 }),
+    { headers, tags: { operation: "search" } },
+  );
+  if (!check(search, { "search succeeded": (response) => response.status === 200 })) {
+    sleep(1);
+    return;
+  }
+
+  const questions = search.json();
+  if (!questions || questions.length === 0) {
+    sleep(1);
+    return;
+  }
+
+  const question = questions[Math.floor(Math.random() * questions.length)];
+  if (Math.random() < 0.1) {
+    const skipped = http.post(
+      `${baseUrl}/api/questions/${encodeURIComponent(question.externalId)}/skip`,
+      null,
+      { headers, tags: { operation: "answer" } },
+    );
+    check(skipped, { "skip recorded": (response) => response.status === 200 });
+    sleep(Math.random() * 2 + 1);
+    return;
+  }
+
+  const choice = question.choices[Math.floor(Math.random() * question.choices.length)];
+  const answer = http.post(
+    `${baseUrl}/api/questions/attempts/first`,
+    JSON.stringify({ questionExternalId: question.externalId, choiceId: choice.id }),
+    { headers, tags: { operation: "answer" } },
+  );
+  check(answer, { "answer recorded": (response) => response.status === 200 });
+  sleep(Math.random() * 2 + 1);
+}

scripts/autoload/question_bank_service.gd

@@ -6,6 +6,7 @@ const DEFAULT_QUESTION_API_BASE_URL := "https://s3q.ranaze.com/api/questions"
 const DEFAULT_RATINGS_API_URL := "https://s3q.ranaze.com/api/ratings/me"
 const DEFAULT_QUESTION_REPORTS_API_URL := "https://s3q.ranaze.com/api/question-reports"
 const RETRY_DELAYS := [2, 5, 10, 20, 30]
+const ATTEMPT_QUEUE_PATH := "user://pending_question_attempts.json"
 
 signal questions_changed
 signal questions_load_failed(message: String)
@@ -20,9 +21,9 @@ var questions: Array = []
 var request: HTTPRequest
 var attempt_request: HTTPRequest
 var rating_request: HTTPRequest
-var skip_request: HTTPRequest
 var report_request: HTTPRequest
 var retry_timer: Timer
+var attempt_retry_timer: Timer
 var questions_api_url := DEFAULT_QUESTIONS_API_URL
 var question_attempt_api_url := DEFAULT_QUESTION_ATTEMPT_API_URL
 var question_api_base_url := DEFAULT_QUESTION_API_BASE_URL
@@ -38,6 +39,9 @@ var rated_question_count := 0
 var category_elos := {}
 var last_rating_change := {}
 var pending_report_question_id := ""
+var attempt_queue: Array = []
+var active_attempt_index := -1
+var attempt_request_active := false
 
 func _ready() -> void:
 	questions_api_url = str(ProjectSettings.get_setting("step3/questions_api_url", DEFAULT_QUESTIONS_API_URL))
@@ -57,10 +61,6 @@ func _ready() -> void:
 	rating_request.request_completed.connect(_on_ratings_loaded)
 	add_child(rating_request)
 
-	skip_request = HTTPRequest.new()
-	skip_request.request_completed.connect(_on_question_skipped)
-	add_child(skip_request)
-
 	report_request = HTTPRequest.new()
 	report_request.request_completed.connect(_on_question_report_submitted)
 	add_child(report_request)
@@ -70,6 +70,16 @@ func _ready() -> void:
 	retry_timer.timeout.connect(_on_retry_timer_timeout)
 	add_child(retry_timer)
 
+	attempt_retry_timer = Timer.new()
+	attempt_retry_timer.one_shot = true
+	attempt_retry_timer.timeout.connect(_process_attempt_queue)
+	add_child(attempt_retry_timer)
+
+	_load_attempt_queue()
+	if not AuthState.auth_changed.is_connected(_on_auth_changed_for_attempt_queue):
+		AuthState.auth_changed.connect(_on_auth_changed_for_attempt_queue)
+	_process_attempt_queue()
+
 
 func load_questions_from_api(reset_retry: bool = true) -> void:
 	if AuthState.access_token.is_empty():
@@ -144,12 +154,14 @@ func clear_questions() -> void:
 		attempt_request.cancel_request()
 	if rating_request != null:
 		rating_request.cancel_request()
-	if skip_request != null:
-		skip_request.cancel_request()
 	if report_request != null:
 		report_request.cancel_request()
 	if retry_timer != null:
 		retry_timer.stop()
+	if attempt_retry_timer != null:
+		attempt_retry_timer.stop()
+	attempt_request_active = false
+	active_attempt_index = -1
 	questions.clear()
 	is_loading = false
 	last_error = ""
@@ -168,32 +180,13 @@ func clear_questions() -> void:
 func record_first_attempt(question_id: String, choice_id: String) -> void:
 	if AuthState.access_token.is_empty() or question_id == "" or choice_id == "":
 		return
-
+	_enqueue_attempt("answer", question_id, choice_id)
-	var headers := PackedStringArray([
-		"Content-Type: application/json",
-		"Authorization: Bearer %s" % AuthState.access_token
-	])
-	var payload := {
-		"questionExternalId": question_id,
-		"choiceId": choice_id
-	}
-	var err := attempt_request.request(question_attempt_api_url, headers, HTTPClient.METHOD_POST, JSON.stringify(payload))
-	if err != OK:
-		push_warning("Failed to record first question attempt.")
 
 
 func skip_question(question_id: String) -> void:
 	if AuthState.access_token.is_empty() or question_id == "":
 		return
-
+	_enqueue_attempt("skip", question_id, "")
-	var headers := PackedStringArray([
-		"Content-Type: application/json",
-		"Authorization: Bearer %s" % AuthState.access_token
-	])
-	var url := "%s/%s/skip" % [question_api_base_url.rstrip("/"), question_id.uri_encode()]
-	var err := skip_request.request(url, headers, HTTPClient.METHOD_POST, "")
-	if err != OK:
-		push_warning("Failed to record skipped question.")
 
 
 func submit_question_report(question_id: String, category: String, details: String) -> bool:
@@ -301,16 +294,111 @@ func _on_retry_timer_timeout() -> void:
 	load_questions_from_api(false)
 
 
+func _enqueue_attempt(kind: String, question_id: String, choice_id: String) -> void:
+	var account := AuthState.username.strip_edges()
+	if account == "":
+		return
+	for queued in attempt_queue:
+		if str(queued.get("account", "")) == account and str(queued.get("question_id", "")) == question_id:
+			return
+	attempt_queue.append({
+		"account": account,
+		"kind": kind,
+		"question_id": question_id,
+		"choice_id": choice_id,
+		"retry_count": 0
+	})
+	_save_attempt_queue()
+	_process_attempt_queue()
+
+
+func _process_attempt_queue() -> void:
+	if attempt_request_active or attempt_queue.is_empty() or AuthState.access_token.is_empty():
+		return
+	if attempt_retry_timer != null and not attempt_retry_timer.is_stopped():
+		return
+
+	active_attempt_index = -1
+	for index in range(attempt_queue.size()):
+		if str(attempt_queue[index].get("account", "")) == AuthState.username:
+			active_attempt_index = index
+			break
+	if active_attempt_index < 0:
+		return
+
+	var queued: Dictionary = attempt_queue[active_attempt_index]
+	var headers := PackedStringArray([
+		"Content-Type: application/json",
+		"Authorization: Bearer %s" % AuthState.access_token
+	])
+	var question_id := str(queued.get("question_id", ""))
+	var url := question_attempt_api_url
+	var body := JSON.stringify({
+		"questionExternalId": question_id,
+		"choiceId": str(queued.get("choice_id", ""))
+	})
+	if str(queued.get("kind", "answer")) == "skip":
+		url = "%s/%s/skip" % [question_api_base_url.rstrip("/"), question_id.uri_encode()]
+		body = ""
+
+	attempt_request_active = true
+	var err := attempt_request.request(url, headers, HTTPClient.METHOD_POST, body)
+	if err != OK:
+		attempt_request_active = false
+		_schedule_attempt_retry()
+
+
+func _schedule_attempt_retry() -> void:
+	if active_attempt_index < 0 or active_attempt_index >= attempt_queue.size():
+		return
+	var queued: Dictionary = attempt_queue[active_attempt_index]
+	var retry_count := int(queued.get("retry_count", 0)) + 1
+	queued["retry_count"] = retry_count
+	attempt_queue[active_attempt_index] = queued
+	_save_attempt_queue()
+	var delay_index: int = min(retry_count - 1, RETRY_DELAYS.size() - 1)
+	attempt_retry_timer.start(float(RETRY_DELAYS[delay_index]))
+
+
+func _remove_active_attempt() -> void:
+	if active_attempt_index >= 0 and active_attempt_index < attempt_queue.size():
+		attempt_queue.remove_at(active_attempt_index)
+	_save_attempt_queue()
+	active_attempt_index = -1
+
+
 func _on_first_attempt_recorded(result: int, response_code: int, _headers: PackedStringArray, body: PackedByteArray) -> void:
-	if result != HTTPRequest.RESULT_SUCCESS or response_code < 200 or response_code >= 300:
+	attempt_request_active = false
-		push_warning("Question attempt API request failed with status %d." % response_code)
+	if active_attempt_index < 0 or active_attempt_index >= attempt_queue.size():
+		return
+	var queued: Dictionary = attempt_queue[active_attempt_index]
+	var succeeded := result == HTTPRequest.RESULT_SUCCESS and response_code >= 200 and response_code < 300
+	if not succeeded:
+		if response_code == 401:
+			active_attempt_index = -1
+		elif response_code in [400, 403, 404]:
+			push_warning("Discarding invalid question attempt with status %d." % response_code)
+			_remove_active_attempt()
+			_process_attempt_queue()
+		else:
+			_schedule_attempt_retry()
 		return
 
 	var parsed: Variant = JSON.parse_string(body.get_string_from_utf8())
 	if typeof(parsed) != TYPE_DICTIONARY:
-		push_warning("Question attempt API response was not an object.")
+		_schedule_attempt_retry()
 		return
 
+	_remove_active_attempt()
+	if str(queued.get("kind", "answer")) == "skip":
+		_apply_skip_response(parsed)
+	else:
+		_apply_first_attempt_response(parsed)
+	_process_attempt_queue()
+
+
+func _apply_first_attempt_response(parsed: Dictionary) -> void:
+
 	var question_id := str(parsed.get("questionExternalId", parsed.get("question_external_id", "")))
 	var percentages: Dictionary = parsed.get("choiceSelectionPercentages", parsed.get("choice_selection_percentages", {}))
 	if question_id == "":
@@ -325,16 +413,7 @@ func _on_first_attempt_recorded(result: int, response_code: int, _headers: Packe
 	question_rating_updated.emit(question_id, question_elo, rating)
 
 
-func _on_question_skipped(result: int, response_code: int, _headers: PackedStringArray, body: PackedByteArray) -> void:
+func _apply_skip_response(parsed: Dictionary) -> void:
-	if result != HTTPRequest.RESULT_SUCCESS or response_code < 200 or response_code >= 300:
-		push_warning("Question skip API request failed with status %d." % response_code)
-		return
-
-	var parsed: Variant = JSON.parse_string(body.get_string_from_utf8())
-	if typeof(parsed) != TYPE_DICTIONARY:
-		push_warning("Question skip API response was not an object.")
-		return
-
 	var question_id := str(parsed.get("questionExternalId", parsed.get("question_external_id", "")))
 	var correct_choice_id := str(parsed.get("correctChoiceId", parsed.get("correct_choice_id", "")))
 	var percentages: Dictionary = parsed.get("choiceSelectionPercentages", parsed.get("choice_selection_percentages", {}))
@@ -350,6 +429,46 @@ func _on_question_skipped(result: int, response_code: int, _headers: PackedStrin
 	question_skipped.emit(question_id, correct_choice_id, question_elo, rating)
 
 
+func _on_auth_changed_for_attempt_queue() -> void:
+	if AuthState.access_token.is_empty():
+		if attempt_retry_timer != null:
+			attempt_retry_timer.stop()
+		return
+	_process_attempt_queue()
+
+
+func _load_attempt_queue() -> void:
+	attempt_queue = []
+	if not FileAccess.file_exists(ATTEMPT_QUEUE_PATH):
+		return
+	var parsed: Variant = JSON.parse_string(FileAccess.get_file_as_string(ATTEMPT_QUEUE_PATH))
+	if typeof(parsed) != TYPE_ARRAY:
+		return
+	for item in parsed:
+		if typeof(item) != TYPE_DICTIONARY:
+			continue
+		var account := str(item.get("account", "")).strip_edges()
+		var question_id := str(item.get("question_id", "")).strip_edges()
+		var kind := str(item.get("kind", "answer"))
+		if account == "" or question_id == "" or kind not in ["answer", "skip"]:
+			continue
+		attempt_queue.append({
+			"account": account,
+			"kind": kind,
+			"question_id": question_id,
+			"choice_id": str(item.get("choice_id", "")),
+			"retry_count": max(0, int(item.get("retry_count", 0)))
+		})
+
+
+func _save_attempt_queue() -> void:
+	var file := FileAccess.open(ATTEMPT_QUEUE_PATH, FileAccess.WRITE)
+	if file == null:
+		push_warning("Could not persist pending question attempts.")
+		return
+	file.store_string(JSON.stringify(attempt_queue))
+
+
 func _on_question_report_submitted(result: int, response_code: int, _headers: PackedStringArray, body: PackedByteArray) -> void:
 	var parsed: Variant = JSON.parse_string(body.get_string_from_utf8())
 	var question_id := ""