hz/step3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Complete CCS results and rating flow

Browse Source
This commit is contained in:
hz
2026-07-15 11:08:13 -05:00
parent 9f379cda5c
commit 52d435fa2e
14 changed files with 402 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/QuestionsApi.Tests/CcsCaseTests.cs
+29 -4
View File
@@ -45,9 +45,9 @@ public sealed class CcsCaseTests
var definition = ValidImport().Cases[0];
definition.ScoringCriteria =
[
new() { Id = "lab", ActionType = "order", TargetCode = "bmp", Points = 5, LatestMinute = 10, Rationale = "Check potassium." },
new() { Id = "insulin", ActionType = "order", TargetCode = "insulin", AfterTargetCode = "bmp", Points = 10, Rationale = "Start after potassium assessment." },
new() { Id = "bicarb", ActionType = "order", TargetCode = "bicarb", Points = -4, Rationale = "Avoid routine bicarbonate." }
new() { Id = "lab", Title = "Check electrolytes", ActionType = "order", TargetCode = "bmp", Points = 5, LatestMinute = 10, Rationale = "Check potassium." },
new() { Id = "insulin", Title = "Start insulin safely", ActionType = "order", TargetCode = "insulin", AfterTargetCode = "bmp", Points = 10, Rationale = "Start after potassium assessment." },
new() { Id = "bicarb", Title = "Avoid bicarbonate", ActionType = "order", TargetCode = "bicarb", Points = -4, Rationale = "Avoid routine bicarbonate." }
];
var attempt = new CcsAttempt { Actions =
[
@@ -70,6 +70,31 @@ public sealed class CcsCaseTests
Assert.Empty(CcsCaseValidator.ValidateImport(ValidImport(), new HashSet<string>()));
}
[Fact]
public void CcsExpectedScoreUsesSeparateUserAndCaseElo()
{
Assert.Equal(0.5, CcsRatingStore.ExpectedScore(1000, 1000), 6);
Assert.True(CcsRatingStore.ExpectedScore(1200, 1000) > 0.75);
Assert.True(CcsRatingStore.ExpectedScore(800, 1000) < 0.25);
}
[Fact]
public void ScoringReviewIncludesExpectedAndObservedEvidence()
{
var definition = ValidImport().Cases[0];
var attempt = new CcsAttempt
{
Actions = [new() { Type = "order", Code = "bmp", SimulatedMinute = 4 }]
};
var criterion = CcsStore.CalculateScore(attempt, definition).Criteria.Single();
Assert.Equal("Check electrolytes", criterion.Title);
Assert.Contains("Bmp", criterion.ExpectedAction, StringComparison.OrdinalIgnoreCase);
Assert.Equal("bmp", criterion.ObservedAction);
Assert.Equal(4, criterion.ObservedMinute);
}
private static CcsImportRequest ValidImport() => new()
{
Orders =
@@ -87,7 +112,7 @@ public sealed class CcsCaseTests
IsPublished = true, RealTimeMinutes = 20, Difficulty = 2,
InitialChart = new() { PatientSummary = "Adult with diabetes", ReasonForEncounter = "Vomiting" },
OrderResponses = [new() { OrderCode = "bmp", Result = "Potassium 4.9 mEq/L" }],
ScoringCriteria = [new() { Id = "lab", ActionType = "order", TargetCode = "bmp", Points = 5 }],
ScoringCriteria = [new() { Id = "lab", Title = "Check electrolytes", ActionType = "order", TargetCode = "bmp", Points = 5 }],
References = [new() { Title = "Consensus report", Publisher = "Example Society", Url = "https://example.org/consensus", Supports = "Case management and scoring", AccessedOn = "2026-07-15", Reviewer = "Question bank editorial review" }]
}
]
api/QuestionsApi/Controllers/CcsController.cs
+4 -1
View File
@@ -9,7 +9,7 @@ namespace QuestionsApi.Controllers;
[ApiController]
[Route("api/ccs")]
[Authorize(Roles = "USER,SUPER")]
public sealed class CcsController(CcsStore store) : ControllerBase
public sealed class CcsController(CcsStore store, CcsRatingStore ratings) : ControllerBase
{
[HttpGet("cases")]
public async Task<IActionResult> Cases() => Ok(await store.ListCasesAsync());
@@ -59,6 +59,9 @@ public sealed class CcsController(CcsStore store) : ControllerBase
[HttpGet("prediction")]
public async Task<IActionResult> Prediction() => Ok(await store.PredictionAsync(UserId()));
[HttpGet("ratings/me")]
public async Task<IActionResult> Rating() => Ok(await ratings.GetProfileAsync(UserId()));
[HttpPost("admin/import")]
[Authorize(Roles = "SUPER")]
public async Task<IActionResult> Import(CcsImportRequest request)
api/QuestionsApi/Models/CcsModels.cs
+57
View File
@@ -92,6 +92,7 @@ public class CcsCaseEvent
public class CcsScoringCriterion
{
public string Id { get; set; } = "";
public string Title { get; set; } = "";
public string ActionType { get; set; } = "order";
public string TargetCode { get; set; } = "";
public List<string> TargetCodes { get; set; } = [];
@@ -127,6 +128,8 @@ public class CcsAttempt
public DateTime UpdatedUtc { get; set; } = DateTime.UtcNow;
public DateTime? CompletedUtc { get; set; }
public CcsCaseScore? Score { get; set; }
public bool RatingEligible { get; set; }
public CcsRatingChange? Rating { get; set; }
}
public class CcsAttemptAction
@@ -163,6 +166,11 @@ public class CcsCaseScore
public class CcsCriterionResult
{
public string CriterionId { get; set; } = "";
public string Title { get; set; } = "";
public string ActionType { get; set; } = "";
public string ExpectedAction { get; set; } = "";
public string ObservedAction { get; set; } = "";
public int? ObservedMinute { get; set; }
public string Status { get; set; } = "missed";
public int Points { get; set; }
public string Rationale { get; set; } = "";
@@ -231,6 +239,7 @@ public class CcsAttemptResponse
public List<CcsExamDefinition> AvailableExams { get; set; } = [];
public List<string> AllowedLocations { get; set; } = [];
public CcsCaseScore? Score { get; set; }
public CcsRatingChange? Rating { get; set; }
public List<QuestionReference> References { get; set; } = [];
}
@@ -242,9 +251,57 @@ public class CcsHistoryItem
public string Mode { get; set; } = "";
public string State { get; set; } = "";
public double? Score { get; set; }
public CcsRatingChange? Rating { get; set; }
public DateTime CreatedUtc { get; set; }
}
public class CcsUserRating
{
[BsonId, BsonRepresentation(BsonType.ObjectId)] public string? Id { get; set; }
public string UserId { get; set; } = "";
public int Elo { get; set; } = 1000;
public int RatedCaseCount { get; set; }
public DateTime UpdatedUtc { get; set; } = DateTime.UtcNow;
}
public class CcsCaseRating
{
[BsonId, BsonRepresentation(BsonType.ObjectId)] public string? Id { get; set; }
public string CaseExternalId { get; set; } = "";
public int Elo { get; set; } = 1000;
public int RatedAttemptCount { get; set; }
public DateTime UpdatedUtc { get; set; } = DateTime.UtcNow;
}
public class CcsRatedCaseResult
{
[BsonId, BsonRepresentation(BsonType.ObjectId)] public string? Id { get; set; }
public string UserId { get; set; } = "";
public string CaseExternalId { get; set; } = "";
[BsonRepresentation(BsonType.ObjectId)] public string AttemptId { get; set; } = "";
public DateTime CreatedUtc { get; set; } = DateTime.UtcNow;
}
public class CcsRatingChange
{
public bool Rated { get; set; }
public string Reason { get; set; } = "";
public int UserEloBefore { get; set; }
public int UserElo { get; set; }
public int UserEloDelta { get; set; }
public int CaseEloBefore { get; set; }
public int CaseElo { get; set; }
public int CaseEloDelta { get; set; }
public double ActualScore { get; set; }
public DateTime? AppliedUtc { get; set; }
}
public class CcsRatingProfileResponse
{
public int Elo { get; set; } = 1000;
public int RatedCaseCount { get; set; }
}
public class CcsPredictionResponse
{
public double? CcsScore { get; set; }
api/QuestionsApi/Program.cs
+1
View File
@@ -18,6 +18,7 @@ builder.Services.AddSingleton<QuestionMediaStore>();
builder.Services.AddSingleton<QuestionReportStore>();
builder.Services.AddSingleton<ExamSimulationStore>();
builder.Services.AddSingleton<CcsStore>();
builder.Services.AddSingleton<CcsRatingStore>();
builder.Services.AddSingleton<CcsCaseReportStore>();
builder.Services.AddSingleton<ServiceMetrics>();
builder.Services.AddHostedService<AnswerStatisticsMaintenanceService>();
api/QuestionsApi/Services/CcsCaseValidator.cs
+1
View File
@@ -58,6 +58,7 @@ public static class CcsCaseValidator
errors.Add(prefix + "scoring criterion ids must be present and unique");
foreach (var criterion in item.ScoringCriteria)
{
if (string.IsNullOrWhiteSpace(criterion.Title)) errors.Add(prefix + $"criterion {criterion.Id} requires a title");
var targets = criterion.TargetCodes.Count > 0 ? criterion.TargetCodes : [criterion.TargetCode];
if (criterion.Points == 0) errors.Add(prefix + $"criterion {criterion.Id} points cannot be zero");
if (targets.Any(string.IsNullOrWhiteSpace)) errors.Add(prefix + $"criterion {criterion.Id} requires a target code");
api/QuestionsApi/Services/CcsRatingStore.cs
+154
View File
@@ -0,0 +1,154 @@
using MongoDB.Driver;
using QuestionsApi.Models;
namespace QuestionsApi.Services;
public sealed class CcsRatingStore
{
private const int BaseElo = 1000;
private const int MinimumElo = 100;
private readonly IMongoCollection<CcsUserRating> _users;
private readonly IMongoCollection<CcsCaseRating> _cases;
private readonly IMongoCollection<CcsRatedCaseResult> _results;
public CcsRatingStore(MongoDatabaseProvider mongo)
{
_users = mongo.Database.GetCollection<CcsUserRating>("CcsUserRatings");
_cases = mongo.Database.GetCollection<CcsCaseRating>("CcsCaseRatings");
_results = mongo.Database.GetCollection<CcsRatedCaseResult>("CcsRatedCaseResults");
_users.Indexes.CreateOne(new CreateIndexModel<CcsUserRating>(
Builders<CcsUserRating>.IndexKeys.Ascending(value => value.UserId), new CreateIndexOptions { Unique = true }));
_cases.Indexes.CreateOne(new CreateIndexModel<CcsCaseRating>(
Builders<CcsCaseRating>.IndexKeys.Ascending(value => value.CaseExternalId), new CreateIndexOptions { Unique = true }));
_results.Indexes.CreateOne(new CreateIndexModel<CcsRatedCaseResult>(
Builders<CcsRatedCaseResult>.IndexKeys.Ascending(value => value.UserId).Ascending(value => value.CaseExternalId),
new CreateIndexOptions { Unique = true }));
}
public async Task<CcsRatingProfileResponse> GetProfileAsync(string userId)
{
var rating = await GetOrCreateUserAsync(userId);
return new CcsRatingProfileResponse { Elo = rating.Elo, RatedCaseCount = rating.RatedCaseCount };
}
public async Task<CcsRatingChange> ApplyAsync(CcsAttempt attempt)
{
var actual = Math.Clamp((attempt.Score?.Percent ?? 0) / 100.0, 0, 1);
if (!attempt.RatingEligible || attempt.Mode != CcsAttemptModes.Timed)
return Unrated("not_eligible", actual);
try
{
await _results.InsertOneAsync(new CcsRatedCaseResult
{
UserId = attempt.UserId,
CaseExternalId = attempt.CaseExternalId,
AttemptId = attempt.Id ?? ""
});
}
catch (MongoWriteException exception) when (exception.WriteError?.Category == ServerErrorCategory.DuplicateKey)
{
return Unrated("repeat_case", actual);
}
var user = await GetOrCreateUserAsync(attempt.UserId);
var caseRating = await GetOrCreateCaseAsync(attempt.CaseExternalId);
var expected = ExpectedScore(user.Elo, caseRating.Elo);
var userDelta = (int)Math.Round(UserK(user.RatedCaseCount) * (actual - expected), MidpointRounding.AwayFromZero);
var caseDelta = (int)Math.Round(CaseK(caseRating.RatedAttemptCount) * (expected - actual), MidpointRounding.AwayFromZero);
var updatedUser = await UpdateUserAsync(user, userDelta);
var updatedCase = await UpdateCaseAsync(caseRating, caseDelta);
return new CcsRatingChange
{
Rated = true,
Reason = "first_timed_attempt",
UserEloBefore = user.Elo,
UserElo = updatedUser.Elo,
UserEloDelta = updatedUser.Elo - user.Elo,
CaseEloBefore = caseRating.Elo,
CaseElo = updatedCase.Elo,
CaseEloDelta = updatedCase.Elo - caseRating.Elo,
ActualScore = actual,
AppliedUtc = DateTime.UtcNow
};
}
public static double ExpectedScore(int userElo, int caseElo) =>
1.0 / (1.0 + Math.Pow(10.0, (caseElo - userElo) / 400.0));
private static int UserK(int count) => count < 20 ? 40 : 24;
private static int CaseK(int count) => count < 20 ? 32 : 24;
private static CcsRatingChange Unrated(string reason, double actual) => new()
{
Rated = false, Reason = reason, ActualScore = actual
};
private async Task<CcsUserRating> GetOrCreateUserAsync(string userId)
{
try
{
await _users.UpdateOneAsync(value => value.UserId == userId,
Builders<CcsUserRating>.Update
.SetOnInsert(value => value.UserId, userId)
.SetOnInsert(value => value.Elo, BaseElo)
.SetOnInsert(value => value.RatedCaseCount, 0)
.SetOnInsert(value => value.UpdatedUtc, DateTime.UtcNow),
new UpdateOptions { IsUpsert = true });
}
catch (MongoWriteException exception) when (exception.WriteError?.Category == ServerErrorCategory.DuplicateKey)
{
// Another API replica created the initial rating first.
}
return await _users.Find(value => value.UserId == userId).FirstAsync();
}
private async Task<CcsCaseRating> GetOrCreateCaseAsync(string externalId)
{
try
{
await _cases.UpdateOneAsync(value => value.CaseExternalId == externalId,
Builders<CcsCaseRating>.Update
.SetOnInsert(value => value.CaseExternalId, externalId)
.SetOnInsert(value => value.Elo, BaseElo)
.SetOnInsert(value => value.RatedAttemptCount, 0)
.SetOnInsert(value => value.UpdatedUtc, DateTime.UtcNow),
new UpdateOptions { IsUpsert = true });
}
catch (MongoWriteException exception) when (exception.WriteError?.Category == ServerErrorCategory.DuplicateKey)
{
// Another API replica created the initial rating first.
}
return await _cases.Find(value => value.CaseExternalId == externalId).FirstAsync();
}
private async Task<CcsUserRating> UpdateUserAsync(CcsUserRating current, int delta)
{
while (true)
{
var nextElo = Math.Max(MinimumElo, current.Elo + delta);
var updated = await _users.FindOneAndUpdateAsync(
value => value.UserId == current.UserId && value.Elo == current.Elo && value.RatedCaseCount == current.RatedCaseCount,
Builders<CcsUserRating>.Update.Set(value => value.Elo, nextElo)
.Inc(value => value.RatedCaseCount, 1).Set(value => value.UpdatedUtc, DateTime.UtcNow),
new FindOneAndUpdateOptions<CcsUserRating> { ReturnDocument = ReturnDocument.After });
if (updated is not null) return updated;
current = await _users.Find(value => value.UserId == current.UserId).FirstAsync();
}
}
private async Task<CcsCaseRating> UpdateCaseAsync(CcsCaseRating current, int delta)
{
while (true)
{
var nextElo = Math.Max(MinimumElo, current.Elo + delta);
var updated = await _cases.FindOneAndUpdateAsync(
value => value.CaseExternalId == current.CaseExternalId && value.Elo == current.Elo && value.RatedAttemptCount == current.RatedAttemptCount,
Builders<CcsCaseRating>.Update.Set(value => value.Elo, nextElo)
.Inc(value => value.RatedAttemptCount, 1).Set(value => value.UpdatedUtc, DateTime.UtcNow),
new FindOneAndUpdateOptions<CcsCaseRating> { ReturnDocument = ReturnDocument.After });
if (updated is not null) return updated;
current = await _cases.Find(value => value.CaseExternalId == current.CaseExternalId).FirstAsync();
}
}
}
api/QuestionsApi/Services/CcsStore.cs
+23 -3
View File
@@ -9,13 +9,15 @@ public sealed class CcsStore
private readonly IMongoCollection<CcsOrderDefinition> _orders;
private readonly IMongoCollection<CcsAttempt> _attempts;
private readonly ExamSimulationStore _examSimulations;
private readonly CcsRatingStore _ratings;
public CcsStore(MongoDatabaseProvider mongo, ExamSimulationStore examSimulations)
public CcsStore(MongoDatabaseProvider mongo, ExamSimulationStore examSimulations, CcsRatingStore ratings)
{
_cases = mongo.Database.GetCollection<CcsCaseRevision>("CcsCaseRevisions");
_orders = mongo.Database.GetCollection<CcsOrderDefinition>("CcsOrderCatalog");
_attempts = mongo.Database.GetCollection<CcsAttempt>("CcsAttempts");
_examSimulations = examSimulations;
_ratings = ratings;
_cases.Indexes.CreateOne(new CreateIndexModel<CcsCaseRevision>(
Builders<CcsCaseRevision>.IndexKeys.Ascending(value => value.ExternalId).Ascending(value => value.Version),
new CreateIndexOptions { Unique = true }));
@@ -108,6 +110,7 @@ public sealed class CcsStore
CurrentVitals = new Dictionary<string, string>(definition.InitialChart.Vitals),
ActiveOrderCodes = definition.InitialChart.InitialOrderCodes.Distinct(StringComparer.OrdinalIgnoreCase).ToList(),
DeadlineUtc = request.Mode == CcsAttemptModes.Timed ? now.AddMinutes(definition.RealTimeMinutes) : null,
RatingEligible = request.Mode == CcsAttemptModes.Timed,
Chart =
[
new CcsChartEntry { Category = "progress", Text = definition.InitialChart.ReasonForEncounter },
@@ -181,7 +184,7 @@ public sealed class CcsStore
{
Id = value.Id ?? "", CaseExternalId = value.CaseExternalId,
CaseTitle = definitions.GetValueOrDefault(value.CaseRevisionId)?.Title ?? value.CaseExternalId,
Mode = value.Mode, State = value.State, Score = value.Score?.Percent, CreatedUtc = value.CreatedUtc
Mode = value.Mode, State = value.State, Score = value.Score?.Percent, Rating = value.Rating, CreatedUtc = value.CreatedUtc
}).ToList();
}
@@ -227,7 +230,8 @@ public sealed class CcsStore
PendingResults = attempt.PendingResults.Select(value => new CcsPendingResult { OrderCode = value.OrderCode, DueMinute = value.DueMinute }).ToList(),
AvailableExams = attempt.State == CcsAttemptStates.Active ? definition.Exams : [],
AllowedLocations = attempt.State == CcsAttemptStates.Active ? definition.AllowedLocations : [],
Score = completed ? attempt.Score : null, References = completed ? definition.References : []
Score = completed ? attempt.Score : null, Rating = completed ? attempt.Rating : null,
References = completed ? definition.References : []
};
}
@@ -341,6 +345,7 @@ public sealed class CcsStore
attempt.State = CcsAttemptStates.Completed;
attempt.CompletedUtc = DateTime.UtcNow;
attempt.Score = CalculateScore(attempt, definition);
attempt.Rating = await _ratings.ApplyAsync(attempt);
attempt.UpdatedUtc = DateTime.UtcNow;
await SaveAsync(attempt);
}
@@ -367,6 +372,11 @@ public sealed class CcsStore
results.Add(new CcsCriterionResult
{
CriterionId = criterion.Id,
Title = string.IsNullOrWhiteSpace(criterion.Title) ? Humanize(criterion.Id) : criterion.Title,
ActionType = criterion.ActionType,
ExpectedAction = DescribeExpected(criterion, targets),
ObservedAction = action?.Code ?? "Not performed",
ObservedMinute = action?.SimulatedMinute,
Status = met ? criterion.Points < 0 ? "harmful" : "credited" : criterion.Points > 0 ? "missed" : "avoided",
Points = awarded, Rationale = criterion.Rationale
});
@@ -384,6 +394,16 @@ public sealed class CcsStore
private Task SaveAsync(CcsAttempt attempt) => _attempts.ReplaceOneAsync(value => value.Id == attempt.Id, attempt);
private static CcsAttemptAction Action(string type, string code, int minute, DateTime now) => new() { Type = type, Code = code, SimulatedMinute = minute, RealUtc = now };
private static string Humanize(string value) => string.Join(' ', value.Split('_', StringSplitOptions.RemoveEmptyEntries).Select(word => char.ToUpperInvariant(word[0]) + word[1..]));
private static string DescribeExpected(CcsScoringCriterion criterion, List<string> targets)
{
var text = $"{Humanize(criterion.ActionType)}: {string.Join(" or ", targets.Select(Humanize))}";
if (criterion.EarliestMinute is not null && criterion.LatestMinute is not null) text += $" between minute {criterion.EarliestMinute} and {criterion.LatestMinute}";
else if (criterion.EarliestMinute is not null) text += $" at or after minute {criterion.EarliestMinute}";
else if (criterion.LatestMinute is not null) text += $" by minute {criterion.LatestMinute}";
if (!string.IsNullOrWhiteSpace(criterion.AfterTargetCode)) text += $" after {Humanize(criterion.AfterTargetCode)}";
return text;
}
private static CcsCaseSummary ToSummary(CcsCaseRevision value) => new()
{
ExternalId = value.ExternalId, Title = value.Title, System = value.System,