diff --git a/api/QuestionsApi/Controllers/QuestionsController.cs b/api/QuestionsApi/Controllers/QuestionsController.cs index d615e0f..4e7bfa9 100644 --- a/api/QuestionsApi/Controllers/QuestionsController.cs +++ b/api/QuestionsApi/Controllers/QuestionsController.cs @@ -15,17 +15,23 @@ public class QuestionsController : ControllerBase private readonly QuestionAttemptStore _attempts; private readonly RatingStore _ratings; private readonly QuestionMediaStore _media; + private readonly AnswerStatisticsStore _statistics; + private readonly ServiceMetrics _metrics; public QuestionsController( QuestionStore questions, QuestionAttemptStore attempts, RatingStore ratings, - QuestionMediaStore media) + QuestionMediaStore media, + AnswerStatisticsStore statistics, + ServiceMetrics metrics) { _questions = questions; _attempts = attempts; _ratings = ratings; _media = media; + _statistics = statistics; + _metrics = metrics; } [HttpGet("systems")] @@ -41,7 +47,7 @@ public class QuestionsController : ControllerBase public async Task Search([FromBody] QuestionSearchRequest req) { var questions = await _questions.SearchAsync(req, User.IsInRole("SUPER")); - var percentages = await _attempts.GetPercentagesByQuestionAsync(questions.Select(q => q.ExternalId)); + var percentages = await _statistics.GetPercentagesByQuestionAsync(questions.Select(q => q.ExternalId)); var questionElos = await _ratings.GetQuestionElosAsync(questions.Select(q => q.ExternalId)); return Ok(questions.Select(q => { @@ -58,7 +64,7 @@ public class QuestionsController : ControllerBase if (question is null || (!question.IsPublished && !User.IsInRole("SUPER"))) return NotFound(); - var percentages = await _attempts.GetPercentagesForQuestionAsync(question.ExternalId); + var percentages = await _statistics.GetPercentagesForQuestionAsync(question.ExternalId); var questionElo = await _ratings.GetQuestionEloAsync(question.ExternalId); return Ok(QuestionResponse.FromModel(question, percentages, questionElo)); } @@ -85,23 +91,58 @@ public class QuestionsController : ControllerBase if (string.IsNullOrWhiteSpace(userId)) return Unauthorized(); - var wasFirstAttempt = await _attempts.RecordFirstAttemptAsync( + var record = await _attempts.RecordFirstAttemptAsync( question.ExternalId, userId, req.ChoiceId.Trim(), req.ChoiceId.Trim() == question.CorrectChoiceId); - var rating = await _ratings.ApplyFirstResultAsync( - userId, - question, - req.ChoiceId.Trim() == question.CorrectChoiceId, - wasFirstAttempt); + if (record.Inserted) + { + _metrics.AttemptCreated(); + try + { + await _statistics.IncrementAsync(record.Attempt); + } + catch + { + _metrics.CounterFailure(); + } + } + else + { + _metrics.AttemptDuplicate(); + } + + var ratingOwner = Guid.NewGuid().ToString("N"); + var shouldRate = await _attempts.TryAcquireRatingLeaseAsync( + record.Attempt.Id ?? "", + ratingOwner, + TimeSpan.FromSeconds(30)); + RatingChangeResponse rating; + try + { + rating = await _ratings.ApplyFirstResultAsync( + userId, + question, + record.Attempt.IsCorrect, + shouldRate); + if (shouldRate) + await _attempts.MarkRatingAppliedAsync(record.Attempt.Id ?? "", ratingOwner); + } + catch + { + _metrics.RatingFailure(); + if (shouldRate) + await _attempts.ReleaseRatingLeaseAsync(record.Attempt.Id ?? "", ratingOwner); + throw; + } return Ok(new FirstQuestionAttemptResponse { QuestionExternalId = question.ExternalId, ChoiceSelectionPercentages = QuestionResponse.BuildChoiceSelectionPercentages( question, - await _attempts.GetPercentagesForQuestionAsync(question.ExternalId)), + await _statistics.GetPercentagesForQuestionAsync(question.ExternalId)), QuestionElo = rating.QuestionElo, Rating = rating }); @@ -164,13 +205,36 @@ public class QuestionsController : ControllerBase if (string.IsNullOrWhiteSpace(userId)) return Unauthorized(); - var wasFirstAttempt = await _attempts.RecordFirstAttemptAsync( + var record = await _attempts.RecordFirstAttemptAsync( question.ExternalId, userId, "", false, true); - var rating = await _ratings.ApplyFirstResultAsync(userId, question, false, wasFirstAttempt); + if (record.Inserted) + _metrics.AttemptCreated(); + else + _metrics.AttemptDuplicate(); + + var ratingOwner = Guid.NewGuid().ToString("N"); + var shouldRate = await _attempts.TryAcquireRatingLeaseAsync( + record.Attempt.Id ?? "", + ratingOwner, + TimeSpan.FromSeconds(30)); + RatingChangeResponse rating; + try + { + rating = await _ratings.ApplyFirstResultAsync(userId, question, false, shouldRate); + if (shouldRate) + await _attempts.MarkRatingAppliedAsync(record.Attempt.Id ?? "", ratingOwner); + } + catch + { + _metrics.RatingFailure(); + if (shouldRate) + await _attempts.ReleaseRatingLeaseAsync(record.Attempt.Id ?? "", ratingOwner); + throw; + } return Ok(new SkipQuestionResponse { @@ -178,7 +242,7 @@ public class QuestionsController : ControllerBase CorrectChoiceId = question.CorrectChoiceId, ChoiceSelectionPercentages = QuestionResponse.BuildChoiceSelectionPercentages( question, - await _attempts.GetPercentagesForQuestionAsync(question.ExternalId)), + await _statistics.GetPercentagesForQuestionAsync(question.ExternalId)), QuestionElo = rating.QuestionElo, Rating = rating }); diff --git a/api/QuestionsApi/Models/AnswerStatistics.cs b/api/QuestionsApi/Models/AnswerStatistics.cs new file mode 100644 index 0000000..0986416 --- /dev/null +++ b/api/QuestionsApi/Models/AnswerStatistics.cs @@ -0,0 +1,49 @@ +using MongoDB.Bson.Serialization.Attributes; + +namespace QuestionsApi.Models; + +public sealed class QuestionAttemptCounter +{ + public string QuestionExternalId { get; set; } = ""; + public int Bucket { get; set; } + public string ChoiceId { get; set; } = ""; + public long Count { get; set; } + public DateTime UpdatedUtc { get; set; } = DateTime.UtcNow; +} + +public sealed class QuestionAttemptCounterTotal +{ + [BsonId] + public QuestionChoiceCounterKey Key { get; set; } = new(); + public long Count { get; set; } +} + +public sealed class QuestionAttemptCounterAggregate +{ + [BsonId] + public QuestionChoiceBucketKey Key { get; set; } = new(); + public long Count { get; set; } +} + +public sealed class QuestionChoiceBucketKey +{ + public string QuestionExternalId { get; set; } = ""; + public int Bucket { get; set; } + public string ChoiceId { get; set; } = ""; +} + +public sealed class QuestionChoiceCounterKey +{ + public string QuestionExternalId { get; set; } = ""; + public string ChoiceId { get; set; } = ""; +} + +public sealed class AnswerStatisticsState +{ + [BsonId] + public string Id { get; set; } = "answer-statistics-v1"; + public bool Completed { get; set; } + public string LeaseOwner { get; set; } = ""; + public DateTime LeaseUntilUtc { get; set; } + public DateTime UpdatedUtc { get; set; } = DateTime.UtcNow; +} diff --git a/api/QuestionsApi/Models/QuestionAttempt.cs b/api/QuestionsApi/Models/QuestionAttempt.cs index 36dda80..29721a1 100644 --- a/api/QuestionsApi/Models/QuestionAttempt.cs +++ b/api/QuestionsApi/Models/QuestionAttempt.cs @@ -14,5 +14,15 @@ public class QuestionAttempt public string ChoiceId { get; set; } = ""; public bool IsCorrect { get; set; } public bool IsSkip { get; set; } + public int CounterBucket { get; set; } + public bool? RatingApplied { get; set; } + public string RatingLeaseOwner { get; set; } = ""; + public DateTime RatingLeaseUntilUtc { get; set; } public DateTime CreatedUtc { get; set; } = DateTime.UtcNow; } + +public sealed class QuestionAttemptRecordResult +{ + public bool Inserted { get; set; } + public QuestionAttempt Attempt { get; set; } = new(); +} diff --git a/api/QuestionsApi/Program.cs b/api/QuestionsApi/Program.cs index 64e1dfa..d79f5b8 100644 --- a/api/QuestionsApi/Program.cs +++ b/api/QuestionsApi/Program.cs @@ -3,16 +3,21 @@ using Microsoft.AspNetCore.Diagnostics; using Microsoft.IdentityModel.Tokens; using Microsoft.OpenApi.Models; using QuestionsApi.Services; +using MongoDB.Bson; using System.Text; var builder = WebApplication.CreateBuilder(args); builder.Services.AddControllers(); +builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); +builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddHostedService(); builder.Services.AddEndpointsApiExplorer(); builder.Services.AddSwaggerGen(c => @@ -61,6 +66,19 @@ builder.Services.AddAuthorization(); var app = builder.Build(); +app.Use(async (context, next) => +{ + var started = System.Diagnostics.Stopwatch.StartNew(); + try + { + await next(); + } + finally + { + context.RequestServices.GetRequiredService().HttpRequest(started.Elapsed); + } +}); + app.UseExceptionHandler(errorApp => { errorApp.Run(async context => @@ -96,6 +114,12 @@ app.UseExceptionHandler(errorApp => }); app.MapGet("/healthz", () => Results.Ok("ok")); +app.MapGet("/readyz", async (MongoDatabaseProvider mongo) => +{ + await mongo.Database.RunCommandAsync(new BsonDocument("ping", 1)); + return Results.Ok("ready"); +}); +app.MapGet("/metrics", (ServiceMetrics metrics) => Results.Text(metrics.RenderPrometheus(), "text/plain; version=0.0.4")); app.UseSwagger(); app.UseSwaggerUI(o => { diff --git a/api/QuestionsApi/README.md b/api/QuestionsApi/README.md index 66fec1a..e197eae 100644 --- a/api/QuestionsApi/README.md +++ b/api/QuestionsApi/README.md @@ -19,6 +19,21 @@ STEP-style question bank service for study games and clients. Player reads require `USER` or `SUPER` JWT auth. Authoring endpoints require `SUPER`. +## Answer tracking + +The `QuestionAttempts` collection is the immutable source of truth and enforces +one first attempt per question and user with a unique compound index. Choice +percentages are served from 32-way materialized counter buckets and cached for +up to five minutes. A background service backfills counters on the first deploy +and reconciles recently active questions every five minutes. + +- `GET /readyz` verifies MongoDB connectivity for readiness checks. +- `GET /metrics` exposes Prometheus-format service and reconciliation metrics. + +The manifests in `k8s/` configure 2-10 API replicas. The cluster must have +Kubernetes Metrics Server installed before applying them with +`kubectl apply -k k8s/`. + Questions submitted through authoring endpoints must have authoritative direct reference links recorded as described in the repository's **Question Verification References** policy. The references must verify the keyed answer diff --git a/api/QuestionsApi/Services/AnswerStatisticsMaintenanceService.cs b/api/QuestionsApi/Services/AnswerStatisticsMaintenanceService.cs new file mode 100644 index 0000000..460bcc3 --- /dev/null +++ b/api/QuestionsApi/Services/AnswerStatisticsMaintenanceService.cs @@ -0,0 +1,58 @@ +namespace QuestionsApi.Services; + +public sealed class AnswerStatisticsMaintenanceService : BackgroundService +{ + private readonly AnswerStatisticsStore _statistics; + private readonly QuestionAttemptStore _attempts; + private readonly ServiceMetrics _metrics; + private readonly ILogger _logger; + private readonly TimeSpan _interval; + private readonly string _owner = $"{Environment.MachineName}-{Guid.NewGuid():N}"; + + public AnswerStatisticsMaintenanceService( + AnswerStatisticsStore statistics, + QuestionAttemptStore attempts, + ServiceMetrics metrics, + IConfiguration configuration, + ILogger logger) + { + _statistics = statistics; + _attempts = attempts; + _metrics = metrics; + _logger = logger; + _interval = TimeSpan.FromSeconds(Math.Max(30, configuration.GetValue("AnswerStatistics:ReconcileSeconds", 300))); + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + try + { + if (await _statistics.TryAcquireBackfillLeaseAsync(_owner, TimeSpan.FromMinutes(30))) + { + _logger.LogInformation("Backfilling materialized answer statistics"); + await _statistics.RebuildAllAsync(); + await _statistics.MarkBackfillCompleteAsync(_owner); + _metrics.Reconciled(); + } + } + catch (Exception exception) + { + _logger.LogError(exception, "Initial answer-statistics backfill failed"); + } + + using var timer = new PeriodicTimer(_interval); + while (await timer.WaitForNextTickAsync(stoppingToken)) + { + try + { + var activeIds = await _attempts.GetActiveQuestionIdsAsync(DateTime.UtcNow.Subtract(_interval).Subtract(_interval)); + await _statistics.RebuildQuestionsAsync(activeIds); + _metrics.Reconciled(); + } + catch (Exception exception) + { + _logger.LogError(exception, "Answer-statistics reconciliation failed"); + } + } + } +} diff --git a/api/QuestionsApi/Services/AnswerStatisticsStore.cs b/api/QuestionsApi/Services/AnswerStatisticsStore.cs new file mode 100644 index 0000000..ae03c96 --- /dev/null +++ b/api/QuestionsApi/Services/AnswerStatisticsStore.cs @@ -0,0 +1,251 @@ +using MongoDB.Driver; +using QuestionsApi.Models; +using System.Collections.Concurrent; + +namespace QuestionsApi.Services; + +public sealed class AnswerStatisticsStore +{ + private const string StateId = "answer-statistics-v1"; + private static readonly TimeSpan CacheDuration = TimeSpan.FromMinutes(5); + + private readonly IMongoCollection _counters; + private readonly IMongoCollection _state; + private readonly QuestionAttemptStore _attempts; + private readonly ConcurrentDictionary _cache = new(); + private volatile bool _backfillComplete; + + public AnswerStatisticsStore(MongoDatabaseProvider mongo, QuestionAttemptStore attempts) + { + _attempts = attempts; + _counters = mongo.Database.GetCollection("QuestionAttemptCounters"); + _state = mongo.Database.GetCollection("SystemState"); + + var counterIndex = Builders.IndexKeys + .Ascending(counter => counter.QuestionExternalId) + .Ascending(counter => counter.Bucket) + .Ascending(counter => counter.ChoiceId); + _counters.Indexes.CreateOne(new CreateIndexModel( + counterIndex, + new CreateIndexOptions { Unique = true })); + } + + public async Task IncrementAsync(QuestionAttempt attempt) + { + if (attempt.IsSkip) + return; + + var filter = Builders.Filter.And( + Builders.Filter.Eq(counter => counter.QuestionExternalId, attempt.QuestionExternalId), + Builders.Filter.Eq(counter => counter.Bucket, attempt.CounterBucket), + Builders.Filter.Eq(counter => counter.ChoiceId, attempt.ChoiceId)); + var update = Builders.Update + .Inc(counter => counter.Count, 1) + .Set(counter => counter.UpdatedUtc, DateTime.UtcNow) + .SetOnInsert(counter => counter.QuestionExternalId, attempt.QuestionExternalId) + .SetOnInsert(counter => counter.Bucket, attempt.CounterBucket) + .SetOnInsert(counter => counter.ChoiceId, attempt.ChoiceId); + + try + { + await _counters.UpdateOneAsync(filter, update, new UpdateOptions { IsUpsert = true }); + } + catch (MongoWriteException exception) when (exception.WriteError.Category == ServerErrorCategory.DuplicateKey) + { + await _counters.UpdateOneAsync(filter, update); + } + _cache.TryRemove(attempt.QuestionExternalId, out _); + } + + public async Task> GetPercentagesForQuestionAsync(string questionExternalId) + { + var result = await GetPercentagesByQuestionAsync([questionExternalId]); + return result.GetValueOrDefault(questionExternalId, []); + } + + public async Task>> GetPercentagesByQuestionAsync( + IEnumerable questionExternalIds) + { + var ids = questionExternalIds.Where(id => !string.IsNullOrWhiteSpace(id)).Distinct().ToList(); + var output = new Dictionary>(); + var missing = new List(); + var now = DateTime.UtcNow; + + foreach (var id in ids) + { + if (_cache.TryGetValue(id, out var cached) && cached.ExpiresUtc > now) + output[id] = new Dictionary(cached.Percentages); + else + missing.Add(id); + } + + if (missing.Count == 0) + return output; + + var backfillComplete = await IsBackfillCompleteAsync(); + if (!backfillComplete) + { + var rawPercentages = await _attempts.GetPercentagesByQuestionAsync(missing); + foreach (var id in missing) + { + var percentages = rawPercentages.GetValueOrDefault(id, []); + output[id] = percentages; + _cache[id] = new CachedPercentages(percentages, now.Add(CacheDuration)); + } + return output; + } + + var totals = await _counters.Aggregate() + .Match(counter => missing.Contains(counter.QuestionExternalId)) + .Group( + counter => new { counter.QuestionExternalId, counter.ChoiceId }, + group => new QuestionAttemptCounterTotal + { + Key = new QuestionChoiceCounterKey + { + QuestionExternalId = group.Key.QuestionExternalId, + ChoiceId = group.Key.ChoiceId + }, + Count = group.Sum(counter => counter.Count) + }) + .ToListAsync(); + + var grouped = totals.GroupBy(total => total.Key.QuestionExternalId) + .ToDictionary(group => group.Key, group => group.ToList()); + + foreach (var id in missing) + { + Dictionary percentages; + if (grouped.TryGetValue(id, out var questionTotals)) + { + var total = questionTotals.Sum(item => item.Count); + percentages = total == 0 + ? [] + : questionTotals.ToDictionary( + item => item.Key.ChoiceId, + item => (int)Math.Round(item.Count * 100.0 / total)); + } + else + { + percentages = []; + } + + output[id] = percentages; + _cache[id] = new CachedPercentages(percentages, now.Add(CacheDuration)); + } + + return output; + } + + public async Task TryAcquireBackfillLeaseAsync(string owner, TimeSpan duration) + { + var now = DateTime.UtcNow; + var filter = Builders.Filter.And( + Builders.Filter.Eq(state => state.Id, StateId), + Builders.Filter.Eq(state => state.Completed, false), + Builders.Filter.Or( + Builders.Filter.Lt(state => state.LeaseUntilUtc, now), + Builders.Filter.Eq(state => state.LeaseOwner, owner))); + var update = Builders.Update + .SetOnInsert(state => state.Id, StateId) + .SetOnInsert(state => state.Completed, false) + .Set(state => state.LeaseOwner, owner) + .Set(state => state.LeaseUntilUtc, now.Add(duration)) + .Set(state => state.UpdatedUtc, now); + + try + { + var leased = await _state.FindOneAndUpdateAsync( + filter, + update, + new FindOneAndUpdateOptions + { + IsUpsert = true, + ReturnDocument = ReturnDocument.After + }); + return leased?.LeaseOwner == owner; + } + catch (MongoWriteException exception) when (exception.WriteError.Category == ServerErrorCategory.DuplicateKey) + { + return false; + } + } + + public async Task MarkBackfillCompleteAsync(string owner) + { + var update = Builders.Update + .Set(state => state.Completed, true) + .Set(state => state.LeaseOwner, owner) + .Set(state => state.UpdatedUtc, DateTime.UtcNow); + await _state.UpdateOneAsync(state => state.Id == StateId && state.LeaseOwner == owner, update); + _backfillComplete = true; + } + + public async Task RebuildAllAsync() => await RebuildAsync(null); + + public async Task RebuildQuestionsAsync(IReadOnlyCollection questionExternalIds) + { + if (questionExternalIds.Count > 0) + await RebuildAsync(questionExternalIds); + } + + private async Task RebuildAsync(IReadOnlyCollection? questionExternalIds) + { + var aggregate = _attempts.Collection.Aggregate().Match(attempt => !attempt.IsSkip); + if (questionExternalIds is not null) + aggregate = aggregate.Match(attempt => questionExternalIds.Contains(attempt.QuestionExternalId)); + + var totals = await aggregate + .Group( + attempt => new { attempt.QuestionExternalId, attempt.CounterBucket, attempt.ChoiceId }, + group => new QuestionAttemptCounterAggregate + { + Key = new QuestionChoiceBucketKey + { + QuestionExternalId = group.Key.QuestionExternalId, + Bucket = group.Key.CounterBucket, + ChoiceId = group.Key.ChoiceId + }, + Count = group.Sum(_ => 1L) + }) + .ToListAsync(); + + if (totals.Count > 0) + { + var now = DateTime.UtcNow; + var replacements = totals.Select(total => + { + var counter = new QuestionAttemptCounter + { + QuestionExternalId = total.Key.QuestionExternalId, + Bucket = total.Key.Bucket, + ChoiceId = total.Key.ChoiceId, + Count = total.Count, + UpdatedUtc = now + }; + var filter = Builders.Filter.And( + Builders.Filter.Eq(item => item.QuestionExternalId, counter.QuestionExternalId), + Builders.Filter.Eq(item => item.Bucket, counter.Bucket), + Builders.Filter.Eq(item => item.ChoiceId, counter.ChoiceId)); + return new ReplaceOneModel(filter, counter) { IsUpsert = true }; + }).ToList(); + await _counters.BulkWriteAsync(replacements, new BulkWriteOptions { IsOrdered = false }); + } + + if (questionExternalIds is null) + _cache.Clear(); + else + foreach (var id in questionExternalIds) + _cache.TryRemove(id, out _); + } + + private async Task IsBackfillCompleteAsync() + { + if (_backfillComplete) + return true; + _backfillComplete = (await _state.Find(state => state.Id == StateId).FirstOrDefaultAsync())?.Completed == true; + return _backfillComplete; + } + + private sealed record CachedPercentages(Dictionary Percentages, DateTime ExpiresUtc); +} diff --git a/api/QuestionsApi/Services/MongoDatabaseProvider.cs b/api/QuestionsApi/Services/MongoDatabaseProvider.cs new file mode 100644 index 0000000..b273157 --- /dev/null +++ b/api/QuestionsApi/Services/MongoDatabaseProvider.cs @@ -0,0 +1,17 @@ +using MongoDB.Driver; + +namespace QuestionsApi.Services; + +public sealed class MongoDatabaseProvider +{ + public IMongoClient Client { get; } + public IMongoDatabase Database { get; } + + public MongoDatabaseProvider(IConfiguration configuration) + { + var connectionString = configuration["MongoDB:ConnectionString"] ?? "mongodb://127.0.0.1:27017"; + var databaseName = configuration["MongoDB:DatabaseName"] ?? "GameDb"; + Client = new MongoClient(connectionString); + Database = Client.GetDatabase(databaseName); + } +} diff --git a/api/QuestionsApi/Services/QuestionAttemptStore.cs b/api/QuestionsApi/Services/QuestionAttemptStore.cs index 7592c37..1036ab5 100644 --- a/api/QuestionsApi/Services/QuestionAttemptStore.cs +++ b/api/QuestionsApi/Services/QuestionAttemptStore.cs @@ -1,5 +1,7 @@ using MongoDB.Driver; using QuestionsApi.Models; +using System.Security.Cryptography; +using System.Text; namespace QuestionsApi.Services; @@ -7,13 +9,11 @@ public class QuestionAttemptStore { private readonly IMongoCollection _col; - public QuestionAttemptStore(IConfiguration cfg) + public const int CounterBucketCount = 32; + + public QuestionAttemptStore(MongoDatabaseProvider mongo) { - var cs = cfg["MongoDB:ConnectionString"] ?? "mongodb://127.0.0.1:27017"; - var dbName = cfg["MongoDB:DatabaseName"] ?? "GameDb"; - var client = new MongoClient(cs); - var db = client.GetDatabase(dbName); - _col = db.GetCollection("QuestionAttempts"); + _col = mongo.Database.GetCollection("QuestionAttempts"); var firstAttemptIndex = Builders.IndexKeys .Ascending(a => a.QuestionExternalId) @@ -24,9 +24,17 @@ public class QuestionAttemptStore var questionIndex = Builders.IndexKeys.Ascending(a => a.QuestionExternalId); _col.Indexes.CreateOne(new CreateIndexModel(questionIndex)); + + var createdIndex = Builders.IndexKeys.Ascending(a => a.CreatedUtc); + _col.Indexes.CreateOne(new CreateIndexModel(createdIndex)); } - public async Task RecordFirstAttemptAsync(string questionExternalId, string userId, string choiceId, bool isCorrect, bool isSkip = false) + public async Task RecordFirstAttemptAsync( + string questionExternalId, + string userId, + string choiceId, + bool isCorrect, + bool isSkip = false) { var attempt = new QuestionAttempt { @@ -35,17 +43,22 @@ public class QuestionAttemptStore ChoiceId = choiceId, IsCorrect = isCorrect, IsSkip = isSkip, + CounterBucket = GetCounterBucket(userId), + RatingApplied = false, CreatedUtc = DateTime.UtcNow }; try { await _col.InsertOneAsync(attempt); - return true; + return new QuestionAttemptRecordResult { Inserted = true, Attempt = attempt }; } catch (MongoWriteException ex) when (ex.WriteError.Category == ServerErrorCategory.DuplicateKey) { - return false; + var existing = await _col + .Find(a => a.QuestionExternalId == questionExternalId && a.UserId == userId) + .FirstAsync(); + return new QuestionAttemptRecordResult { Inserted = false, Attempt = existing }; } } @@ -61,22 +74,80 @@ public class QuestionAttemptStore if (ids.Count == 0) return []; - var attempts = await _col.Find(a => ids.Contains(a.QuestionExternalId) && !a.IsSkip).ToListAsync(); + var totals = await _col.Aggregate() + .Match(a => ids.Contains(a.QuestionExternalId) && !a.IsSkip) + .Group( + attempt => new { attempt.QuestionExternalId, attempt.ChoiceId }, + group => new QuestionAttemptCounterTotal + { + Key = new QuestionChoiceCounterKey + { + QuestionExternalId = group.Key.QuestionExternalId, + ChoiceId = group.Key.ChoiceId + }, + Count = group.Sum(_ => 1L) + }) + .ToListAsync(); var output = new Dictionary>(); - foreach (var group in attempts.GroupBy(a => a.QuestionExternalId)) + foreach (var group in totals.GroupBy(total => total.Key.QuestionExternalId)) { - var total = group.Count(); + var total = group.Sum(item => item.Count); if (total == 0) continue; output[group.Key] = group - .GroupBy(a => a.ChoiceId) .ToDictionary( - choiceGroup => choiceGroup.Key, - choiceGroup => (int)Math.Round(choiceGroup.Count() * 100.0 / total)); + item => item.Key.ChoiceId, + item => (int)Math.Round(item.Count * 100.0 / total)); } return output; } + + public Task> GetActiveQuestionIdsAsync(DateTime sinceUtc) => + _col.Distinct( + attempt => attempt.QuestionExternalId, + Builders.Filter.Gte(attempt => attempt.CreatedUtc, sinceUtc)) + .ToListAsync(); + + public IMongoCollection Collection => _col; + + public async Task TryAcquireRatingLeaseAsync(string attemptId, string owner, TimeSpan duration) + { + var now = DateTime.UtcNow; + var filter = Builders.Filter.And( + Builders.Filter.Eq(attempt => attempt.Id, attemptId), + Builders.Filter.Exists(attempt => attempt.RatingApplied, true), + Builders.Filter.Eq(attempt => attempt.RatingApplied, false), + Builders.Filter.Or( + Builders.Filter.Lt(attempt => attempt.RatingLeaseUntilUtc, now), + Builders.Filter.Eq(attempt => attempt.RatingLeaseOwner, owner))); + var update = Builders.Update + .Set(attempt => attempt.RatingLeaseOwner, owner) + .Set(attempt => attempt.RatingLeaseUntilUtc, now.Add(duration)); + var leased = await _col.FindOneAndUpdateAsync( + filter, + update, + new FindOneAndUpdateOptions { ReturnDocument = ReturnDocument.After }); + return leased?.RatingLeaseOwner == owner; + } + + public Task MarkRatingAppliedAsync(string attemptId, string owner) => + _col.UpdateOneAsync( + attempt => attempt.Id == attemptId && attempt.RatingLeaseOwner == owner, + Builders.Update + .Set(attempt => attempt.RatingApplied, true) + .Set(attempt => attempt.RatingLeaseUntilUtc, DateTime.MinValue)); + + public Task ReleaseRatingLeaseAsync(string attemptId, string owner) => + _col.UpdateOneAsync( + attempt => attempt.Id == attemptId && attempt.RatingLeaseOwner == owner && attempt.RatingApplied == false, + Builders.Update.Set(attempt => attempt.RatingLeaseUntilUtc, DateTime.MinValue)); + + public static int GetCounterBucket(string userId) + { + var hash = SHA256.HashData(Encoding.UTF8.GetBytes(userId)); + return (int)(BitConverter.ToUInt32(hash, 0) % CounterBucketCount); + } } diff --git a/api/QuestionsApi/Services/QuestionMediaStore.cs b/api/QuestionsApi/Services/QuestionMediaStore.cs index cd3598f..ee05cef 100644 --- a/api/QuestionsApi/Services/QuestionMediaStore.cs +++ b/api/QuestionsApi/Services/QuestionMediaStore.cs @@ -8,13 +8,9 @@ public class QuestionMediaStore { private readonly IMongoCollection _col; - public QuestionMediaStore(IConfiguration cfg) + public QuestionMediaStore(MongoDatabaseProvider mongo) { - var cs = cfg["MongoDB:ConnectionString"] ?? "mongodb://127.0.0.1:27017"; - var dbName = cfg["MongoDB:DatabaseName"] ?? "GameDb"; - var client = new MongoClient(cs); - var db = client.GetDatabase(dbName); - _col = db.GetCollection("QuestionMediaAssets"); + _col = mongo.Database.GetCollection("QuestionMediaAssets"); } public async Task CreateAsync(string fileName, string contentType, byte[] bytes) diff --git a/api/QuestionsApi/Services/QuestionReportStore.cs b/api/QuestionsApi/Services/QuestionReportStore.cs index 52ec8f6..0935a49 100644 --- a/api/QuestionsApi/Services/QuestionReportStore.cs +++ b/api/QuestionsApi/Services/QuestionReportStore.cs @@ -8,13 +8,9 @@ public class QuestionReportStore { private readonly IMongoCollection _reports; - public QuestionReportStore(IConfiguration cfg) + public QuestionReportStore(MongoDatabaseProvider mongo) { - var connectionString = cfg["MongoDB:ConnectionString"] ?? "mongodb://127.0.0.1:27017"; - var databaseName = cfg["MongoDB:DatabaseName"] ?? "GameDb"; - var client = new MongoClient(connectionString); - var database = client.GetDatabase(databaseName); - _reports = database.GetCollection("QuestionReports"); + _reports = mongo.Database.GetCollection("QuestionReports"); _reports.Indexes.CreateMany( [ diff --git a/api/QuestionsApi/Services/QuestionStore.cs b/api/QuestionsApi/Services/QuestionStore.cs index f10aafe..b39c00a 100644 --- a/api/QuestionsApi/Services/QuestionStore.cs +++ b/api/QuestionsApi/Services/QuestionStore.cs @@ -8,13 +8,9 @@ public class QuestionStore { private readonly IMongoCollection _col; - public QuestionStore(IConfiguration cfg) + public QuestionStore(MongoDatabaseProvider mongo) { - var cs = cfg["MongoDB:ConnectionString"] ?? "mongodb://127.0.0.1:27017"; - var dbName = cfg["MongoDB:DatabaseName"] ?? "GameDb"; - var client = new MongoClient(cs); - var db = client.GetDatabase(dbName); - _col = db.GetCollection("Questions"); + _col = mongo.Database.GetCollection("Questions"); var externalIdIndex = Builders.IndexKeys.Ascending(q => q.ExternalId); _col.Indexes.CreateOne(new CreateIndexModel(externalIdIndex, new CreateIndexOptions { Unique = true })); diff --git a/api/QuestionsApi/Services/RatingStore.cs b/api/QuestionsApi/Services/RatingStore.cs index 9616ba0..df83118 100644 --- a/api/QuestionsApi/Services/RatingStore.cs +++ b/api/QuestionsApi/Services/RatingStore.cs @@ -11,14 +11,10 @@ public class RatingStore private readonly IMongoCollection _users; private readonly IMongoCollection _questions; - public RatingStore(IConfiguration cfg) + public RatingStore(MongoDatabaseProvider mongo) { - var cs = cfg["MongoDB:ConnectionString"] ?? "mongodb://127.0.0.1:27017"; - var dbName = cfg["MongoDB:DatabaseName"] ?? "GameDb"; - var client = new MongoClient(cs); - var db = client.GetDatabase(dbName); - _users = db.GetCollection("UserRatings"); - _questions = db.GetCollection("QuestionRatings"); + _users = mongo.Database.GetCollection("UserRatings"); + _questions = mongo.Database.GetCollection("QuestionRatings"); _users.Indexes.CreateOne(new CreateIndexModel( Builders.IndexKeys.Ascending(r => r.UserId), @@ -89,64 +85,107 @@ public class RatingStore var categoryDelta = (int)Math.Round(userK * (actual - categoryExpected)); var questionDelta = (int)Math.Round(questionK * (overallExpected - actual)); - userRating.OverallElo = Math.Max(MinElo, oldOverall + overallDelta); - userRating.CategoryElos[category] = Math.Max(MinElo, oldCategory + categoryDelta); - userRating.RatedQuestionCount++; - userRating.UpdatedUtc = DateTime.UtcNow; + var now = DateTime.UtcNow; + var categoryPath = $"CategoryElos.{category}"; + var userUpdate = Builders.Update + .Inc(rating => rating.OverallElo, overallDelta) + .Inc(categoryPath, categoryDelta) + .Inc(rating => rating.RatedQuestionCount, 1) + .Set(rating => rating.UpdatedUtc, now); + await _users.UpdateOneAsync(rating => rating.UserId == userId, userUpdate); + await _users.UpdateOneAsync( + rating => rating.UserId == userId, + Builders.Update + .Max(rating => rating.OverallElo, MinElo) + .Max(categoryPath, MinElo)); - questionRating.Elo = Math.Max(MinElo, oldQuestion + questionDelta); - questionRating.AttemptCount++; - questionRating.UpdatedUtc = DateTime.UtcNow; + var questionUpdate = Builders.Update + .Inc(rating => rating.Elo, questionDelta) + .Inc(rating => rating.AttemptCount, 1) + .Set(rating => rating.UpdatedUtc, now); + await _questions.UpdateOneAsync(rating => rating.QuestionExternalId == question.ExternalId, questionUpdate); + await _questions.UpdateOneAsync( + rating => rating.QuestionExternalId == question.ExternalId, + Builders.Update.Max(rating => rating.Elo, MinElo)); - await _users.ReplaceOneAsync(r => r.UserId == userRating.UserId, userRating, new ReplaceOptions { IsUpsert = true }); - await _questions.ReplaceOneAsync(r => r.QuestionExternalId == questionRating.QuestionExternalId, questionRating, new ReplaceOptions { IsUpsert = true }); + var updatedUser = await _users.Find(rating => rating.UserId == userId).FirstAsync(); + var updatedQuestion = await _questions.Find(rating => rating.QuestionExternalId == question.ExternalId).FirstAsync(); return new RatingChangeResponse { Rated = true, Category = category, - UserOverallElo = userRating.OverallElo, - UserOverallDelta = userRating.OverallElo - oldOverall, - UserCategoryElo = userRating.CategoryElos[category], - UserCategoryDelta = userRating.CategoryElos[category] - oldCategory, - QuestionElo = questionRating.Elo, - QuestionEloDelta = questionRating.Elo - oldQuestion + UserOverallElo = updatedUser.OverallElo, + UserOverallDelta = overallDelta, + UserCategoryElo = updatedUser.CategoryElos.GetValueOrDefault(category, BaseElo), + UserCategoryDelta = categoryDelta, + QuestionElo = updatedQuestion.Elo, + QuestionEloDelta = questionDelta }; } private async Task GetOrCreateUserRatingAsync(string userId) { - var rating = await _users.Find(r => r.UserId == userId).FirstOrDefaultAsync(); - if (rating is null) + var create = Builders.Update + .SetOnInsert(rating => rating.UserId, userId) + .SetOnInsert(rating => rating.OverallElo, BaseElo) + .SetOnInsert(rating => rating.RatedQuestionCount, 0) + .SetOnInsert(rating => rating.CategoryElos, DefaultCategoryElos()) + .SetOnInsert(rating => rating.UpdatedUtc, DateTime.UtcNow); + UserRating rating; + try { - rating = new UserRating { UserId = userId, CategoryElos = DefaultCategoryElos() }; - await _users.InsertOneAsync(rating); - return rating; + rating = await _users.FindOneAndUpdateAsync( + item => item.UserId == userId, + create, + new FindOneAndUpdateOptions + { + IsUpsert = true, + ReturnDocument = ReturnDocument.After + }); + } + catch (MongoWriteException exception) when (exception.WriteError.Category == ServerErrorCategory.DuplicateKey) + { + rating = await _users.Find(item => item.UserId == userId).FirstAsync(); } - var changed = false; - foreach (var category in RatingCategory.Categories) + var missingCategories = RatingCategory.Categories + .Where(category => !rating.CategoryElos.ContainsKey(category)) + .ToList(); + if (missingCategories.Count > 0) { - if (!rating.CategoryElos.ContainsKey(category)) - { - rating.CategoryElos[category] = BaseElo; - changed = true; - } + var updates = missingCategories + .Select(category => Builders.Update.Set($"CategoryElos.{category}", BaseElo)) + .ToList(); + if (updates.Count > 0) + await _users.UpdateOneAsync(item => item.UserId == userId, Builders.Update.Combine(updates)); + rating = await _users.Find(item => item.UserId == userId).FirstAsync(); } - if (changed) - await _users.ReplaceOneAsync(r => r.UserId == userId, rating); return rating; } private async Task GetOrCreateQuestionRatingAsync(string questionExternalId) { - var rating = await _questions.Find(r => r.QuestionExternalId == questionExternalId).FirstOrDefaultAsync(); - if (rating is not null) - return rating; - - rating = new QuestionRating { QuestionExternalId = questionExternalId }; - await _questions.InsertOneAsync(rating); - return rating; + var create = Builders.Update + .SetOnInsert(rating => rating.QuestionExternalId, questionExternalId) + .SetOnInsert(rating => rating.Elo, BaseElo) + .SetOnInsert(rating => rating.AttemptCount, 0) + .SetOnInsert(rating => rating.UpdatedUtc, DateTime.UtcNow); + try + { + return await _questions.FindOneAndUpdateAsync( + rating => rating.QuestionExternalId == questionExternalId, + create, + new FindOneAndUpdateOptions + { + IsUpsert = true, + ReturnDocument = ReturnDocument.After + }); + } + catch (MongoWriteException exception) when (exception.WriteError.Category == ServerErrorCategory.DuplicateKey) + { + return await _questions.Find(rating => rating.QuestionExternalId == questionExternalId).FirstAsync(); + } } private static Dictionary DefaultCategoryElos() => diff --git a/api/QuestionsApi/Services/ServiceMetrics.cs b/api/QuestionsApi/Services/ServiceMetrics.cs new file mode 100644 index 0000000..9ff06d2 --- /dev/null +++ b/api/QuestionsApi/Services/ServiceMetrics.cs @@ -0,0 +1,59 @@ +using System.Text; + +namespace QuestionsApi.Services; + +public sealed class ServiceMetrics +{ + private long _attemptsCreated; + private long _attemptDuplicates; + private long _counterFailures; + private long _ratingFailures; + private long _reconciliations; + private long _lastReconciliationUnixSeconds; + private long _httpRequests; + private long _httpDurationMicroseconds; + + public void AttemptCreated() => Interlocked.Increment(ref _attemptsCreated); + public void AttemptDuplicate() => Interlocked.Increment(ref _attemptDuplicates); + public void CounterFailure() => Interlocked.Increment(ref _counterFailures); + public void RatingFailure() => Interlocked.Increment(ref _ratingFailures); + public void Reconciled() + { + Interlocked.Increment(ref _reconciliations); + Interlocked.Exchange(ref _lastReconciliationUnixSeconds, DateTimeOffset.UtcNow.ToUnixTimeSeconds()); + } + public void HttpRequest(TimeSpan duration) + { + Interlocked.Increment(ref _httpRequests); + Interlocked.Add(ref _httpDurationMicroseconds, (long)(duration.TotalMilliseconds * 1000)); + } + + public string RenderPrometheus() + { + var output = new StringBuilder(); + AppendCounter(output, "step3_question_attempts_created_total", Interlocked.Read(ref _attemptsCreated)); + AppendCounter(output, "step3_question_attempt_duplicates_total", Interlocked.Read(ref _attemptDuplicates)); + AppendCounter(output, "step3_answer_counter_failures_total", Interlocked.Read(ref _counterFailures)); + AppendCounter(output, "step3_rating_failures_total", Interlocked.Read(ref _ratingFailures)); + AppendCounter(output, "step3_answer_reconciliations_total", Interlocked.Read(ref _reconciliations)); + AppendCounter(output, "step3_http_requests_total", Interlocked.Read(ref _httpRequests)); + output.AppendLine("# TYPE step3_http_request_duration_seconds summary"); + output.Append("step3_http_request_duration_seconds_count ") + .Append(Interlocked.Read(ref _httpRequests)) + .AppendLine(); + output.Append("step3_http_request_duration_seconds_sum ") + .Append((Interlocked.Read(ref _httpDurationMicroseconds) / 1_000_000.0).ToString("0.######", System.Globalization.CultureInfo.InvariantCulture)) + .AppendLine(); + output.AppendLine("# TYPE step3_answer_last_reconciliation_unixtime gauge"); + output.Append("step3_answer_last_reconciliation_unixtime ") + .Append(Interlocked.Read(ref _lastReconciliationUnixSeconds)) + .AppendLine(); + return output.ToString(); + } + + private static void AppendCounter(StringBuilder output, string name, long value) + { + output.Append("# TYPE ").Append(name).AppendLine(" counter"); + output.Append(name).Append(' ').Append(value).AppendLine(); + } +} diff --git a/api/QuestionsApi/appsettings.Development.json b/api/QuestionsApi/appsettings.Development.json index 55e3ab0..b6ce260 100644 --- a/api/QuestionsApi/appsettings.Development.json +++ b/api/QuestionsApi/appsettings.Development.json @@ -1,6 +1,7 @@ { "Kestrel": { "Endpoints": { "Http": { "Url": "http://0.0.0.0:5008" } } }, "MongoDB": { "ConnectionString": "mongodb://192.168.86.50:27017", "DatabaseName": "promiscuity" }, + "AnswerStatistics": { "ReconcileSeconds": 300 }, "Jwt": { "Key": "SuperUltraSecureJwtKeyWithAtLeast32Chars!!", "Issuer": "promiscuity", "Audience": "promiscuity-auth-api" }, "Logging": { "LogLevel": { "Default": "Information" } } } diff --git a/api/QuestionsApi/appsettings.json b/api/QuestionsApi/appsettings.json index 01adb6c..226f593 100644 --- a/api/QuestionsApi/appsettings.json +++ b/api/QuestionsApi/appsettings.json @@ -1,6 +1,7 @@ { "Kestrel": { "Endpoints": { "Http": { "Url": "http://0.0.0.0:5008" } } }, "MongoDB": { "ConnectionString": "mongodb://192.168.86.50:27017", "DatabaseName": "promiscuity" }, + "AnswerStatistics": { "ReconcileSeconds": 300 }, "Jwt": { "Key": "SuperUltraSecureJwtKeyWithAtLeast32Chars!!", "Issuer": "promiscuity", "Audience": "promiscuity-auth-api" }, "Logging": { "LogLevel": { "Default": "Information" } }, "AllowedHosts": "*" diff --git a/api/QuestionsApi/k8s/deployment.yaml b/api/QuestionsApi/k8s/deployment.yaml index 849b6d1..d5543d4 100644 --- a/api/QuestionsApi/k8s/deployment.yaml +++ b/api/QuestionsApi/k8s/deployment.yaml @@ -6,6 +6,11 @@ metadata: app: step3-questions spec: replicas: 2 + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 0 + maxSurge: 1 selector: matchLabels: app: step3-questions @@ -13,6 +18,10 @@ spec: metadata: labels: app: step3-questions + annotations: + prometheus.io/scrape: "true" + prometheus.io/path: "/metrics" + prometheus.io/port: "5008" spec: containers: - name: step3-questions @@ -20,9 +29,26 @@ spec: imagePullPolicy: IfNotPresent ports: - containerPort: 5008 + resources: + requests: + cpu: 250m + memory: 256Mi + limits: + cpu: "1" + memory: 512Mi readinessProbe: httpGet: - path: /healthz + path: /readyz port: 5008 initialDelaySeconds: 5 periodSeconds: 10 + timeoutSeconds: 3 + failureThreshold: 3 + livenessProbe: + httpGet: + path: /healthz + port: 5008 + initialDelaySeconds: 15 + periodSeconds: 20 + timeoutSeconds: 3 + failureThreshold: 3 diff --git a/api/QuestionsApi/k8s/hpa.yaml b/api/QuestionsApi/k8s/hpa.yaml new file mode 100644 index 0000000..01fb6c8 --- /dev/null +++ b/api/QuestionsApi/k8s/hpa.yaml @@ -0,0 +1,35 @@ +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: step3-questions +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: step3-questions + minReplicas: 2 + maxReplicas: 10 + behavior: + scaleUp: + stabilizationWindowSeconds: 0 + policies: + - type: Percent + value: 100 + periodSeconds: 60 + - type: Pods + value: 2 + periodSeconds: 60 + selectPolicy: Max + scaleDown: + stabilizationWindowSeconds: 300 + policies: + - type: Percent + value: 25 + periodSeconds: 60 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 60 diff --git a/api/QuestionsApi/k8s/kustomization.yaml b/api/QuestionsApi/k8s/kustomization.yaml new file mode 100644 index 0000000..b528f32 --- /dev/null +++ b/api/QuestionsApi/k8s/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - deployment.yaml + - service.yaml + - hpa.yaml + - pdb.yaml diff --git a/api/QuestionsApi/k8s/pdb.yaml b/api/QuestionsApi/k8s/pdb.yaml new file mode 100644 index 0000000..98aa095 --- /dev/null +++ b/api/QuestionsApi/k8s/pdb.yaml @@ -0,0 +1,9 @@ +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: step3-questions +spec: + minAvailable: 1 + selector: + matchLabels: + app: step3-questions diff --git a/api/QuestionsApi/load/README.md b/api/QuestionsApi/load/README.md new file mode 100644 index 0000000..fce88a9 --- /dev/null +++ b/api/QuestionsApi/load/README.md @@ -0,0 +1,14 @@ +# Answer Tracking Load Test + +Generate 5,000 or more short-lived USER JWTs for dedicated load-test accounts and +store them as a JSON string array outside the repository. Never reuse production +user credentials. + +Run the test from this directory: + +```powershell +k6 run -e BASE_URL=https://s3q.ranaze.com -e AUTH_TOKENS_PATH=./tokens.json answer-tracking.js +``` + +The test ramps to 5,000 concurrent users and validates the production latency and +error-rate targets. Run it against a staging environment before production. diff --git a/api/QuestionsApi/load/answer-tracking.js b/api/QuestionsApi/load/answer-tracking.js new file mode 100644 index 0000000..9362609 --- /dev/null +++ b/api/QuestionsApi/load/answer-tracking.js @@ -0,0 +1,76 @@ +import http from "k6/http"; +import { check, sleep } from "k6"; +import { SharedArray } from "k6/data"; + +const baseUrl = __ENV.BASE_URL || "https://s3q.ranaze.com"; +const tokenPath = __ENV.AUTH_TOKENS_PATH || "./tokens.json"; +const tokens = new SharedArray("test-user tokens", () => JSON.parse(open(tokenPath))); + +if (tokens.length < 5000) { + throw new Error("The load test requires at least 5,000 distinct USER tokens."); +} + +export const options = { + scenarios: { + answer_tracking: { + executor: "ramping-vus", + startVUs: 0, + stages: [ + { duration: "2m", target: 5000 }, + { duration: "5m", target: 5000 }, + { duration: "1m", target: 0 }, + ], + gracefulRampDown: "30s", + }, + }, + thresholds: { + http_req_failed: ["rate<0.01"], + "http_req_duration{operation:search}": ["p(95)<1000"], + "http_req_duration{operation:answer}": ["p(95)<500"], + }, +}; + +export default function () { + const token = tokens[(__VU - 1) % tokens.length]; + const headers = { + Authorization: `Bearer ${token}`, + "Content-Type": "application/json", + }; + + const search = http.post( + `${baseUrl}/api/questions/search`, + JSON.stringify({ limit: 80 }), + { headers, tags: { operation: "search" } }, + ); + if (!check(search, { "search succeeded": (response) => response.status === 200 })) { + sleep(1); + return; + } + + const questions = search.json(); + if (!questions || questions.length === 0) { + sleep(1); + return; + } + + const question = questions[Math.floor(Math.random() * questions.length)]; + if (Math.random() < 0.1) { + const skipped = http.post( + `${baseUrl}/api/questions/${encodeURIComponent(question.externalId)}/skip`, + null, + { headers, tags: { operation: "answer" } }, + ); + check(skipped, { "skip recorded": (response) => response.status === 200 }); + sleep(Math.random() * 2 + 1); + return; + } + + const choice = question.choices[Math.floor(Math.random() * question.choices.length)]; + const answer = http.post( + `${baseUrl}/api/questions/attempts/first`, + JSON.stringify({ questionExternalId: question.externalId, choiceId: choice.id }), + { headers, tags: { operation: "answer" } }, + ); + check(answer, { "answer recorded": (response) => response.status === 200 }); + sleep(Math.random() * 2 + 1); +} diff --git a/scripts/autoload/question_bank_service.gd b/scripts/autoload/question_bank_service.gd index bb69c4f..9df273e 100644 --- a/scripts/autoload/question_bank_service.gd +++ b/scripts/autoload/question_bank_service.gd @@ -6,6 +6,7 @@ const DEFAULT_QUESTION_API_BASE_URL := "https://s3q.ranaze.com/api/questions" const DEFAULT_RATINGS_API_URL := "https://s3q.ranaze.com/api/ratings/me" const DEFAULT_QUESTION_REPORTS_API_URL := "https://s3q.ranaze.com/api/question-reports" const RETRY_DELAYS := [2, 5, 10, 20, 30] +const ATTEMPT_QUEUE_PATH := "user://pending_question_attempts.json" signal questions_changed signal questions_load_failed(message: String) @@ -20,9 +21,9 @@ var questions: Array = [] var request: HTTPRequest var attempt_request: HTTPRequest var rating_request: HTTPRequest -var skip_request: HTTPRequest var report_request: HTTPRequest var retry_timer: Timer +var attempt_retry_timer: Timer var questions_api_url := DEFAULT_QUESTIONS_API_URL var question_attempt_api_url := DEFAULT_QUESTION_ATTEMPT_API_URL var question_api_base_url := DEFAULT_QUESTION_API_BASE_URL @@ -38,6 +39,9 @@ var rated_question_count := 0 var category_elos := {} var last_rating_change := {} var pending_report_question_id := "" +var attempt_queue: Array = [] +var active_attempt_index := -1 +var attempt_request_active := false func _ready() -> void: questions_api_url = str(ProjectSettings.get_setting("step3/questions_api_url", DEFAULT_QUESTIONS_API_URL)) @@ -57,10 +61,6 @@ func _ready() -> void: rating_request.request_completed.connect(_on_ratings_loaded) add_child(rating_request) - skip_request = HTTPRequest.new() - skip_request.request_completed.connect(_on_question_skipped) - add_child(skip_request) - report_request = HTTPRequest.new() report_request.request_completed.connect(_on_question_report_submitted) add_child(report_request) @@ -70,6 +70,16 @@ func _ready() -> void: retry_timer.timeout.connect(_on_retry_timer_timeout) add_child(retry_timer) + attempt_retry_timer = Timer.new() + attempt_retry_timer.one_shot = true + attempt_retry_timer.timeout.connect(_process_attempt_queue) + add_child(attempt_retry_timer) + + _load_attempt_queue() + if not AuthState.auth_changed.is_connected(_on_auth_changed_for_attempt_queue): + AuthState.auth_changed.connect(_on_auth_changed_for_attempt_queue) + _process_attempt_queue() + func load_questions_from_api(reset_retry: bool = true) -> void: if AuthState.access_token.is_empty(): @@ -144,12 +154,14 @@ func clear_questions() -> void: attempt_request.cancel_request() if rating_request != null: rating_request.cancel_request() - if skip_request != null: - skip_request.cancel_request() if report_request != null: report_request.cancel_request() if retry_timer != null: retry_timer.stop() + if attempt_retry_timer != null: + attempt_retry_timer.stop() + attempt_request_active = false + active_attempt_index = -1 questions.clear() is_loading = false last_error = "" @@ -168,32 +180,13 @@ func clear_questions() -> void: func record_first_attempt(question_id: String, choice_id: String) -> void: if AuthState.access_token.is_empty() or question_id == "" or choice_id == "": return - - var headers := PackedStringArray([ - "Content-Type: application/json", - "Authorization: Bearer %s" % AuthState.access_token - ]) - var payload := { - "questionExternalId": question_id, - "choiceId": choice_id - } - var err := attempt_request.request(question_attempt_api_url, headers, HTTPClient.METHOD_POST, JSON.stringify(payload)) - if err != OK: - push_warning("Failed to record first question attempt.") + _enqueue_attempt("answer", question_id, choice_id) func skip_question(question_id: String) -> void: if AuthState.access_token.is_empty() or question_id == "": return - - var headers := PackedStringArray([ - "Content-Type: application/json", - "Authorization: Bearer %s" % AuthState.access_token - ]) - var url := "%s/%s/skip" % [question_api_base_url.rstrip("/"), question_id.uri_encode()] - var err := skip_request.request(url, headers, HTTPClient.METHOD_POST, "") - if err != OK: - push_warning("Failed to record skipped question.") + _enqueue_attempt("skip", question_id, "") func submit_question_report(question_id: String, category: String, details: String) -> bool: @@ -301,16 +294,111 @@ func _on_retry_timer_timeout() -> void: load_questions_from_api(false) +func _enqueue_attempt(kind: String, question_id: String, choice_id: String) -> void: + var account := AuthState.username.strip_edges() + if account == "": + return + for queued in attempt_queue: + if str(queued.get("account", "")) == account and str(queued.get("question_id", "")) == question_id: + return + attempt_queue.append({ + "account": account, + "kind": kind, + "question_id": question_id, + "choice_id": choice_id, + "retry_count": 0 + }) + _save_attempt_queue() + _process_attempt_queue() + + +func _process_attempt_queue() -> void: + if attempt_request_active or attempt_queue.is_empty() or AuthState.access_token.is_empty(): + return + if attempt_retry_timer != null and not attempt_retry_timer.is_stopped(): + return + + active_attempt_index = -1 + for index in range(attempt_queue.size()): + if str(attempt_queue[index].get("account", "")) == AuthState.username: + active_attempt_index = index + break + if active_attempt_index < 0: + return + + var queued: Dictionary = attempt_queue[active_attempt_index] + var headers := PackedStringArray([ + "Content-Type: application/json", + "Authorization: Bearer %s" % AuthState.access_token + ]) + var question_id := str(queued.get("question_id", "")) + var url := question_attempt_api_url + var body := JSON.stringify({ + "questionExternalId": question_id, + "choiceId": str(queued.get("choice_id", "")) + }) + if str(queued.get("kind", "answer")) == "skip": + url = "%s/%s/skip" % [question_api_base_url.rstrip("/"), question_id.uri_encode()] + body = "" + + attempt_request_active = true + var err := attempt_request.request(url, headers, HTTPClient.METHOD_POST, body) + if err != OK: + attempt_request_active = false + _schedule_attempt_retry() + + +func _schedule_attempt_retry() -> void: + if active_attempt_index < 0 or active_attempt_index >= attempt_queue.size(): + return + var queued: Dictionary = attempt_queue[active_attempt_index] + var retry_count := int(queued.get("retry_count", 0)) + 1 + queued["retry_count"] = retry_count + attempt_queue[active_attempt_index] = queued + _save_attempt_queue() + var delay_index: int = min(retry_count - 1, RETRY_DELAYS.size() - 1) + attempt_retry_timer.start(float(RETRY_DELAYS[delay_index])) + + +func _remove_active_attempt() -> void: + if active_attempt_index >= 0 and active_attempt_index < attempt_queue.size(): + attempt_queue.remove_at(active_attempt_index) + _save_attempt_queue() + active_attempt_index = -1 + + func _on_first_attempt_recorded(result: int, response_code: int, _headers: PackedStringArray, body: PackedByteArray) -> void: - if result != HTTPRequest.RESULT_SUCCESS or response_code < 200 or response_code >= 300: - push_warning("Question attempt API request failed with status %d." % response_code) + attempt_request_active = false + if active_attempt_index < 0 or active_attempt_index >= attempt_queue.size(): + return + var queued: Dictionary = attempt_queue[active_attempt_index] + var succeeded := result == HTTPRequest.RESULT_SUCCESS and response_code >= 200 and response_code < 300 + if not succeeded: + if response_code == 401: + active_attempt_index = -1 + elif response_code in [400, 403, 404]: + push_warning("Discarding invalid question attempt with status %d." % response_code) + _remove_active_attempt() + _process_attempt_queue() + else: + _schedule_attempt_retry() return var parsed: Variant = JSON.parse_string(body.get_string_from_utf8()) if typeof(parsed) != TYPE_DICTIONARY: - push_warning("Question attempt API response was not an object.") + _schedule_attempt_retry() return + _remove_active_attempt() + if str(queued.get("kind", "answer")) == "skip": + _apply_skip_response(parsed) + else: + _apply_first_attempt_response(parsed) + _process_attempt_queue() + + +func _apply_first_attempt_response(parsed: Dictionary) -> void: + var question_id := str(parsed.get("questionExternalId", parsed.get("question_external_id", ""))) var percentages: Dictionary = parsed.get("choiceSelectionPercentages", parsed.get("choice_selection_percentages", {})) if question_id == "": @@ -325,16 +413,7 @@ func _on_first_attempt_recorded(result: int, response_code: int, _headers: Packe question_rating_updated.emit(question_id, question_elo, rating) -func _on_question_skipped(result: int, response_code: int, _headers: PackedStringArray, body: PackedByteArray) -> void: - if result != HTTPRequest.RESULT_SUCCESS or response_code < 200 or response_code >= 300: - push_warning("Question skip API request failed with status %d." % response_code) - return - - var parsed: Variant = JSON.parse_string(body.get_string_from_utf8()) - if typeof(parsed) != TYPE_DICTIONARY: - push_warning("Question skip API response was not an object.") - return - +func _apply_skip_response(parsed: Dictionary) -> void: var question_id := str(parsed.get("questionExternalId", parsed.get("question_external_id", ""))) var correct_choice_id := str(parsed.get("correctChoiceId", parsed.get("correct_choice_id", ""))) var percentages: Dictionary = parsed.get("choiceSelectionPercentages", parsed.get("choice_selection_percentages", {})) @@ -350,6 +429,46 @@ func _on_question_skipped(result: int, response_code: int, _headers: PackedStrin question_skipped.emit(question_id, correct_choice_id, question_elo, rating) +func _on_auth_changed_for_attempt_queue() -> void: + if AuthState.access_token.is_empty(): + if attempt_retry_timer != null: + attempt_retry_timer.stop() + return + _process_attempt_queue() + + +func _load_attempt_queue() -> void: + attempt_queue = [] + if not FileAccess.file_exists(ATTEMPT_QUEUE_PATH): + return + var parsed: Variant = JSON.parse_string(FileAccess.get_file_as_string(ATTEMPT_QUEUE_PATH)) + if typeof(parsed) != TYPE_ARRAY: + return + for item in parsed: + if typeof(item) != TYPE_DICTIONARY: + continue + var account := str(item.get("account", "")).strip_edges() + var question_id := str(item.get("question_id", "")).strip_edges() + var kind := str(item.get("kind", "answer")) + if account == "" or question_id == "" or kind not in ["answer", "skip"]: + continue + attempt_queue.append({ + "account": account, + "kind": kind, + "question_id": question_id, + "choice_id": str(item.get("choice_id", "")), + "retry_count": max(0, int(item.get("retry_count", 0))) + }) + + +func _save_attempt_queue() -> void: + var file := FileAccess.open(ATTEMPT_QUEUE_PATH, FileAccess.WRITE) + if file == null: + push_warning("Could not persist pending question attempts.") + return + file.store_string(JSON.stringify(attempt_queue)) + + func _on_question_report_submitted(result: int, response_code: int, _headers: PackedStringArray, body: PackedByteArray) -> void: var parsed: Variant = JSON.parse_string(body.get_string_from_utf8()) var question_id := ""