hz/step3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve dynamic question learning loop

Browse Source
This commit is contained in:
hz
2026-07-23 16:50:53 -05:00
parent c91e70deb5
commit 32c455bba0
14 changed files with 422 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/QuestionsApi/Services/DashboardStore.cs
+7 -1
View File
@@ -16,6 +16,7 @@ public sealed class DashboardStore
private readonly IMongoCollection<UserNotepad> _notes;
private readonly QuestionStore _questions;
private readonly ReviewQueueStore _reviewQueue;
private readonly DynamicQuestionStore _dynamicQuestions;
private readonly FlashcardStudyService _flashcards;
private readonly FlashcardScheduleStore _flashcardSchedules;
@@ -23,6 +24,7 @@ public sealed class DashboardStore
MongoDatabaseProvider mongo,
QuestionStore questions,
ReviewQueueStore reviewQueue,
DynamicQuestionStore dynamicQuestions,
FlashcardStudyService flashcards,
FlashcardScheduleStore flashcardSchedules)
{
@@ -36,6 +38,7 @@ public sealed class DashboardStore
_notes = mongo.Database.GetCollection<UserNotepad>("UserNotepads");
_questions = questions;
_reviewQueue = reviewQueue;
_dynamicQuestions = dynamicQuestions;
_flashcards = flashcards;
_flashcardSchedules = flashcardSchedules;
}
@@ -57,8 +60,10 @@ public sealed class DashboardStore
var reviewAttemptsTask = _reviewQueue.GetReviewsSinceAsync(userId, weekStart);
var flashcardCountsTask = _flashcards.GetDashboardCountsAsync(userId, nowUtc, weekStart);
var flashcardReviewsTask = _flashcardSchedules.GetReviewsSinceAsync(userId, weekStart);
var dynamicReviewsTask = _dynamicQuestions.DueReviewsAsync(userId, nowUtc);
await Task.WhenAll(attemptsTask, ratingTask, ccsRatingTask, ccsAttemptsTask, completedCcsCountTask,
simulationsTask, bookmarkCountTask, notepadTask, reviewAttemptsTask, flashcardCountsTask, flashcardReviewsTask);
simulationsTask, bookmarkCountTask, notepadTask, reviewAttemptsTask, flashcardCountsTask,
flashcardReviewsTask, dynamicReviewsTask);
var attempts = attemptsTask.Result;
var answeredAttempts = attempts.Where(value => !value.IsSkip).ToList();
@@ -84,6 +89,7 @@ public sealed class DashboardStore
BookmarkCount = checked((int)bookmarkCountTask.Result),
NoteCount = CountNotes(notepadTask.Result?.Content),
MissedQuestionCount = checked((int)pendingReviewCount),
DynamicReviewsDue = dynamicReviewsTask.Result.Count,
FlashcardsDue = flashcardCountsTask.Result.Due,
FlashcardsNewAvailable = flashcardCountsTask.Result.NewAvailable,
FlashcardsReviewedThisWeek = flashcardCountsTask.Result.ReviewedThisWeek
api/QuestionsApi/Services/DynamicBlueprintCoveragePolicy.cs
+37
View File
@@ -0,0 +1,37 @@
using QuestionsApi.Models;
namespace QuestionsApi.Services;
public static class DynamicBlueprintCoveragePolicy
{
private static readonly Dictionary<string, double> SystemWeights =
new(StringComparer.OrdinalIgnoreCase)
{
["Nutrition"] = 17.5,
["Biostatistics"] = 12,
["Cardiovascular"] = 10,
["Neurologic"] = 9,
["Nervous System"] = 9,
["Pulmonary"] = 9,
["Respiratory"] = 9,
["Pregnancy"] = 8,
["Social Sciences"] = 8,
["Gastrointestinal"] = 7,
["Immune System"] = 7,
["Hematology"] = 7,
["Multisystem"] = 7,
["Infectious Disease"] = 7,
["Endocrine"] = 6,
["Musculoskeletal"] = 6,
["Behavioral Health"] = 5,
["Renal"] = 5,
["Skin"] = 5,
["Human Development"] = 2
};
public static double Weight(DynamicQuestionBlueprint blueprint)
{
if (blueprint.BlueprintWeight > 0) return blueprint.BlueprintWeight;
return SystemWeights.GetValueOrDefault(blueprint.System, 5);
}
}
api/QuestionsApi/Services/DynamicBlueprintResolver.cs
+12 -1
View File
@@ -29,7 +29,18 @@ public static class DynamicBlueprintResolver
materialized.BaseBlueprintVersion = blueprint.BaseBlueprintVersion;
materialized.QualityStandardVersion = blueprint.QualityStandardVersion;
materialized.IsPublished = blueprint.IsPublished;
materialized.CaseFamilies = blueprint.CaseFamilies.Select(overlay =>
materialized.EncounterFrame = blueprint.EncounterFrame;
materialized.SiteOfCare = blueprint.SiteOfCare;
materialized.BlueprintWeight = blueprint.BlueprintWeight;
materialized.ConceptIds = blueprint.ConceptIds;
materialized.PrerequisiteConceptIds = blueprint.PrerequisiteConceptIds;
materialized.MedicallyReviewedUtc = blueprint.MedicallyReviewedUtc;
materialized.MedicalReviewExpiresUtc = blueprint.MedicalReviewExpiresUtc;
materialized.MedicalReviewer = blueprint.MedicalReviewer;
if (blueprint.Distractors.Count > 0)
materialized.Distractors = blueprint.Distractors;
if (blueprint.CaseFamilies.Count > 0)
materialized.CaseFamilies = blueprint.CaseFamilies.Select(overlay =>
{
var baseFamily = baseBlueprint.CaseFamilies.FirstOrDefault(value => value.Id == overlay.BaseFamilyId)
?? throw new InvalidDataException(
api/QuestionsApi/Services/DynamicQuestionGenerator.cs
+24
View File
@@ -317,6 +317,11 @@ public sealed class DynamicQuestionGenerator
if (string.IsNullOrWhiteSpace(blueprint.ExternalId)) return "externalId required";
if (string.IsNullOrWhiteSpace(blueprint.ObjectiveId)) return "objectiveId required";
if (blueprint.Version < 1) return "version must be positive";
if (blueprint.BlueprintWeight < 0) return "blueprintWeight cannot be negative";
if (blueprint.IsPublished
&& blueprint.MedicalReviewExpiresUtc is { } expiresUtc
&& expiresUtc <= DateTime.UtcNow)
return "medical review has expired";
if (blueprint.QualityStandardVersion >= 2)
return ValidateV2(blueprint);
if (string.IsNullOrWhiteSpace(blueprint.StemTemplate) || string.IsNullOrWhiteSpace(blueprint.LeadIn)) return "stemTemplate and leadIn required";
@@ -355,6 +360,22 @@ public sealed class DynamicQuestionGenerator
if (blueprint.CaseFamilies.Any(value => !DynamicDifficultyPolicy.TierElos.ContainsKey(value.DifficultyTier)))
return "difficultyTier must be foundation, standard, advanced, or expert";
}
if (blueprint.QualityStandardVersion >= 4)
{
if (blueprint.BlueprintWeight <= 0) return "quality standard 4 requires blueprintWeight";
if (string.IsNullOrWhiteSpace(blueprint.System)
|| string.IsNullOrWhiteSpace(blueprint.PhysicianTask)
|| string.IsNullOrWhiteSpace(blueprint.EncounterFrame)
|| string.IsNullOrWhiteSpace(blueprint.SiteOfCare))
return "quality standard 4 requires system, physicianTask, encounterFrame, and siteOfCare";
if (blueprint.ConceptIds.Count == 0) return "quality standard 4 requires conceptIds";
if (blueprint.MedicallyReviewedUtc is null
|| blueprint.MedicalReviewExpiresUtc is null
|| string.IsNullOrWhiteSpace(blueprint.MedicalReviewer))
return "quality standard 4 requires medical review metadata";
if (blueprint.Distractors.Any(value => string.IsNullOrWhiteSpace(value.MisconceptionId)))
return "quality standard 4 requires a misconceptionId for every distractor";
}
foreach (var family in blueprint.CaseFamilies)
{
if (string.IsNullOrWhiteSpace(family.Id)) return "case family id required";
@@ -367,6 +388,9 @@ public sealed class DynamicQuestionGenerator
var reviews = family.ReviewFollowups.Count == 0 ? blueprint.ReviewFollowups : family.ReviewFollowups;
var wrong = family.WrongAnswerFollowups.Count == 0 ? blueprint.WrongAnswerFollowups : family.WrongAnswerFollowups;
if (distractors.Count < 6) return $"{family.Id} requires at least six distractors";
if (blueprint.QualityStandardVersion >= 4
&& distractors.Any(value => string.IsNullOrWhiteSpace(value.MisconceptionId)))
return $"{family.Id} every distractor requires a misconceptionId";
var answers = distractors.Append(correctAnswer).ToList();
if (answers.Any(value => string.IsNullOrWhiteSpace(value.Id)
|| string.IsNullOrWhiteSpace(value.TextTemplate)
api/QuestionsApi/Services/DynamicQuestionService.cs
+48 -5
View File
@@ -49,13 +49,52 @@ public sealed class DynamicQuestionService
}
var mastery = await _store.MasteryAsync(userId);
var dueReviews = await _store.DueReviewsAsync(userId, DateTime.UtcNow);
var dueByObjective = dueReviews.ToDictionary(value => value.ObjectiveId, StringComparer.Ordinal);
var candidates = unseen
.Where(group => group.Key != run.PreviousObjectiveId || unseen.Count == 1)
.OrderBy(group => mastery.TryGetValue(group.Key, out var value) ? value.Elo : 1000)
.ThenBy(_ => RandomNumberGenerator.GetInt32(int.MaxValue))
.ToList();
var weakestBand = candidates.Take(Math.Min(4, candidates.Count)).ToList();
var selectedObjective = weakestBand[RandomNumberGenerator.GetInt32(weakestBand.Count)];
var roll = RandomNumberGenerator.GetInt32(100);
IGrouping<string, DynamicQuestionBlueprint> selectedObjective;
var dueCandidates = candidates.Where(value => dueByObjective.ContainsKey(value.Key)).ToList();
if (roll < 50 && dueCandidates.Count == 0 && dueByObjective.Count > 0)
dueCandidates = byObjective
.Where(value => dueByObjective.ContainsKey(value.Key)
&& (value.Key != run.PreviousObjectiveId || dueByObjective.Count == 1))
.ToList();
if (roll < 50 && dueCandidates.Count > 0)
{
selectedObjective = dueCandidates
.OrderBy(value => dueByObjective[value.Key].DueUtc)
.ThenBy(_ => RandomNumberGenerator.GetInt32(int.MaxValue))
.First();
}
else if (roll < 80)
{
var systemAttempts = byObjective
.GroupBy(group => group.OrderByDescending(value => value.Version).First().System)
.ToDictionary(
group => group.Key,
group => group.Sum(objective => mastery.TryGetValue(objective.Key, out var value) ? value.Attempts : 0),
StringComparer.Ordinal);
selectedObjective = candidates
.OrderBy(group =>
{
var latest = group.OrderByDescending(value => value.Version).First();
var weight = DynamicBlueprintCoveragePolicy.Weight(latest);
return (systemAttempts.GetValueOrDefault(latest.System) + 1) / weight;
})
.ThenBy(_ => RandomNumberGenerator.GetInt32(int.MaxValue))
.First();
}
else
{
var weakestBand = candidates
.OrderBy(group => mastery.TryGetValue(group.Key, out var value) ? value.Elo : 1000)
.Take(Math.Min(4, candidates.Count))
.ToList();
selectedObjective = weakestBand[RandomNumberGenerator.GetInt32(weakestBand.Count)];
}
var blueprint = selectedObjective
.OrderByDescending(value => value.Version)
.First();
@@ -97,11 +136,15 @@ public sealed class DynamicQuestionService
DifficultyTier = selectedFamily.DifficultyTier,
EffectiveAbilityElo = effectiveAbility,
FamilyElo = selectedFamilyElo,
IsScheduledReview = dueByObjective.TryGetValue(blueprint.ObjectiveId, out var reviewItem),
ReviewItemId = reviewItem?.Id ?? "",
ReviewStage = reviewItem?.Stage ?? 0,
ExpiresUtc = DateTime.UtcNow.AddDays(7)
};
var generated = _generator.Generate(
blueprint, claims.Seed, claims.Nonce, claims.GeneratorVersion, claims.CaseFamilyId);
run.SeenObjectiveIds.Add(blueprint.ObjectiveId);
if (!run.SeenObjectiveIds.Contains(blueprint.ObjectiveId, StringComparer.Ordinal))
run.SeenObjectiveIds.Add(blueprint.ObjectiveId);
run.PreviousObjectiveId = blueprint.ObjectiveId;
run.RecentCaseFamilyIds.Add(generated.CaseFamilyId);
run.RecentCaseFamilyIds = run.RecentCaseFamilyIds.TakeLast(30).ToList();
api/QuestionsApi/Services/DynamicQuestionStore.cs
+124 -3
View File
@@ -10,6 +10,7 @@ public sealed class DynamicQuestionStore
private readonly IMongoCollection<DynamicQuestionAttempt> _attempts;
private readonly IMongoCollection<DynamicObjectiveMastery> _mastery;
private readonly IMongoCollection<DynamicFamilyRating> _familyRatings;
private readonly IMongoCollection<DynamicReviewItem> _reviews;
private readonly ServiceMetrics? _metrics;
public DynamicQuestionStore(MongoDatabaseProvider mongo, ServiceMetrics? metrics = null)
@@ -19,6 +20,7 @@ public sealed class DynamicQuestionStore
_attempts = mongo.Database.GetCollection<DynamicQuestionAttempt>("DynamicQuestionAttempts");
_mastery = mongo.Database.GetCollection<DynamicObjectiveMastery>("DynamicObjectiveMastery");
_familyRatings = mongo.Database.GetCollection<DynamicFamilyRating>("DynamicFamilyRatings");
_reviews = mongo.Database.GetCollection<DynamicReviewItem>("DynamicReviewItems");
_blueprints.Indexes.CreateOne(new CreateIndexModel<DynamicQuestionBlueprint>(
Builders<DynamicQuestionBlueprint>.IndexKeys.Ascending(value => value.ExternalId).Ascending(value => value.Version),
new CreateIndexOptions { Unique = true }));
@@ -34,6 +36,19 @@ public sealed class DynamicQuestionStore
.Ascending(value => value.BlueprintVersion)
.Ascending(value => value.CaseFamilyId),
new CreateIndexOptions { Unique = true }));
_reviews.Indexes.CreateMany([
new CreateIndexModel<DynamicReviewItem>(
Builders<DynamicReviewItem>.IndexKeys
.Ascending(value => value.UserId)
.Ascending(value => value.ObjectiveId),
new CreateIndexOptions { Unique = true, Name = "dynamic_review_user_objective" }),
new CreateIndexModel<DynamicReviewItem>(
Builders<DynamicReviewItem>.IndexKeys
.Ascending(value => value.UserId)
.Ascending(value => value.State)
.Ascending(value => value.DueUtc),
new CreateIndexOptions { Name = "dynamic_review_user_due" })
]);
}
public async Task<List<DynamicQuestionBlueprint>> PublishedAsync() =>
@@ -46,6 +61,11 @@ public sealed class DynamicQuestionStore
(await _mastery.Find(value => value.UserId == userId).ToListAsync())
.ToDictionary(value => value.ObjectiveId, StringComparer.Ordinal);
public async Task<List<DynamicReviewItem>> DueReviewsAsync(string userId, DateTime nowUtc) =>
await _reviews.Find(value => value.UserId == userId
&& value.State == DynamicReviewStates.Pending
&& value.DueUtc <= nowUtc).ToListAsync();
public async Task<Dictionary<string, DynamicFamilyRating>> FamilyRatingsAsync(DynamicQuestionBlueprint blueprint)
{
var ratings = await _familyRatings.Find(value =>
@@ -60,6 +80,14 @@ public sealed class DynamicQuestionStore
DynamicAttemptRequest request)
{
var isCorrect = !request.IsSkip && request.ChoiceId == generated.Question.CorrectChoiceId;
var selectedAnswer = generated.Blueprint.Distractors
.Concat(generated.Blueprint.CaseFamilies
.Where(value => value.Id == claims.CaseFamilyId)
.SelectMany(value => value.Distractors))
.FirstOrDefault(value => value.Id == request.ChoiceId);
var misconceptionId = isCorrect || request.IsSkip
? ""
: selectedAnswer?.MisconceptionId ?? "";
var attempt = new DynamicQuestionAttempt
{
UserId = claims.UserId,
@@ -74,6 +102,8 @@ public sealed class DynamicQuestionStore
ChoiceId = request.ChoiceId,
IsCorrect = isCorrect,
IsSkip = request.IsSkip,
SelectedMisconceptionId = misconceptionId,
IsScheduledReview = claims.IsScheduledReview,
DurationSeconds = Math.Clamp(request.DurationSeconds, 0, 86_400),
CreatedUtc = DateTime.UtcNow
};
@@ -117,6 +147,9 @@ public sealed class DynamicQuestionStore
.Min(value => value.Elo, DynamicDifficultyPolicy.MaximumElo));
if (!string.IsNullOrWhiteSpace(claims.CaseFamilyId))
await UpdateFamilyRatingAsync(claims, generated, isCorrect);
DateTime? nextReviewUtc = null;
if (!request.IsSkip)
nextReviewUtc = await UpdateReviewAsync(claims, misconceptionId, isCorrect);
return new DynamicAttemptResponse
{
Inserted = true,
@@ -124,10 +157,61 @@ public sealed class DynamicQuestionStore
CorrectChoiceId = generated.Question.CorrectChoiceId,
ObjectiveId = claims.ObjectiveId,
ObjectiveElo = updated.Elo,
RatingChange = delta
RatingChange = delta,
SelectedMisconceptionId = misconceptionId,
IsScheduledReview = claims.IsScheduledReview,
NextReviewUtc = nextReviewUtc
};
}
private async Task<DateTime?> UpdateReviewAsync(
DynamicGenerationClaims claims,
string misconceptionId,
bool isCorrect)
{
var now = DateTime.UtcNow;
var existing = await _reviews.Find(value =>
value.UserId == claims.UserId && value.ObjectiveId == claims.ObjectiveId).FirstOrDefaultAsync();
if (!claims.IsScheduledReview)
{
if (isCorrect) return existing?.DueUtc;
var due = DynamicReviewPolicy.FirstDue(now);
var update = Builders<DynamicReviewItem>.Update
.SetOnInsert(value => value.UserId, claims.UserId)
.SetOnInsert(value => value.ObjectiveId, claims.ObjectiveId)
.SetOnInsert(value => value.CreatedUtc, now)
.Set(value => value.BlueprintExternalId, claims.BlueprintExternalId)
.Set(value => value.BlueprintVersion, claims.BlueprintVersion)
.Set(value => value.MisconceptionId, misconceptionId)
.Set(value => value.State, DynamicReviewStates.Pending)
.Set(value => value.Stage, 0)
.Set(value => value.DueUtc, due)
.Set(value => value.UpdatedUtc, now);
await _reviews.UpdateOneAsync(
value => value.UserId == claims.UserId && value.ObjectiveId == claims.ObjectiveId,
update,
new UpdateOptions { IsUpsert = true });
return due;
}
if (existing is null || existing.Id != claims.ReviewItemId || existing.Stage != claims.ReviewStage)
return null;
var next = DynamicReviewPolicy.AfterReview(existing.Stage, isCorrect, now);
var reviewUpdate = Builders<DynamicReviewItem>.Update
.Set(value => value.Stage, next.Stage)
.Set(value => value.State, next.Mastered ? DynamicReviewStates.Mastered : DynamicReviewStates.Pending)
.Set(value => value.MisconceptionId, isCorrect ? existing.MisconceptionId : misconceptionId)
.Set(value => value.UpdatedUtc, now)
.Inc(value => value.ReviewCount, 1);
if (next.DueUtc.HasValue)
reviewUpdate = reviewUpdate.Set(value => value.DueUtc, next.DueUtc.Value);
await _reviews.UpdateOneAsync(
value => value.Id == existing.Id && value.Stage == claims.ReviewStage,
reviewUpdate);
return next.DueUtc;
}
private async Task EnsureMasteryAsync(string userId, string objectiveId)
{
try
@@ -194,18 +278,55 @@ public sealed class DynamicQuestionStore
public async Task<DynamicQuestionStatusResponse> StatusAsync(string userId)
{
var objectiveCount = (await PublishedAsync()).Select(value => value.ObjectiveId).Distinct().Count();
var blueprints = (await PublishedAsync())
.GroupBy(value => value.ObjectiveId)
.Select(value => value.OrderByDescending(item => item.Version).First())
.ToList();
var objectiveCount = blueprints.Count;
var mastery = await _mastery.Find(value => value.UserId == userId).ToListAsync();
var masteryByObjective = mastery.ToDictionary(value => value.ObjectiveId, StringComparer.Ordinal);
var due = await _reviews.CountDocumentsAsync(value => value.UserId == userId
&& value.State == DynamicReviewStates.Pending
&& value.DueUtc <= DateTime.UtcNow);
return new DynamicQuestionStatusResponse
{
Available = objectiveCount > 0,
PublishedObjectives = objectiveCount,
AttemptedObjectives = mastery.Count,
TotalAttempts = mastery.Sum(value => value.Attempts),
AverageObjectiveElo = mastery.Count == 0 ? 1000 : (int)Math.Round(mastery.Average(value => value.Elo))
AverageObjectiveElo = mastery.Count == 0 ? 1000 : (int)Math.Round(mastery.Average(value => value.Elo)),
ReviewsDue = checked((int)due),
SystemCoverage = Coverage(blueprints, masteryByObjective, value => value.System),
PhysicianTaskCoverage = Coverage(blueprints, masteryByObjective, value => value.PhysicianTask)
};
}
private static List<DynamicCoverageSummary> Coverage(
IReadOnlyCollection<DynamicQuestionBlueprint> blueprints,
IReadOnlyDictionary<string, DynamicObjectiveMastery> mastery,
Func<DynamicQuestionBlueprint, string> keySelector) =>
blueprints
.GroupBy(value => string.IsNullOrWhiteSpace(keySelector(value)) ? "Unclassified" : keySelector(value))
.Select(group =>
{
var attempts = group
.Where(value => mastery.ContainsKey(value.ObjectiveId))
.Select(value => mastery[value.ObjectiveId])
.ToList();
return new DynamicCoverageSummary
{
Id = group.Key,
PublishedObjectives = group.Count(),
AttemptedObjectives = attempts.Count,
Attempts = attempts.Sum(value => value.Attempts),
CoveragePercent = group.Count() == 0 ? 0 : Math.Round(attempts.Count * 100.0 / group.Count(), 1),
AverageElo = attempts.Count == 0 ? 1000 : (int)Math.Round(attempts.Average(value => value.Elo))
};
})
.OrderBy(value => value.CoveragePercent)
.ThenBy(value => value.Id)
.ToList();
public async Task<(int Inserted, int Unchanged)> UpsertBlueprintsAsync(IEnumerable<DynamicQuestionBlueprint> blueprints)
{
var inserted = 0;
api/QuestionsApi/Services/DynamicReviewPolicy.cs
+22
View File
@@ -0,0 +1,22 @@
namespace QuestionsApi.Services;
public static class DynamicReviewPolicy
{
private static readonly int[] IntervalsDays = [1, 3, 7, 21, 45];
public static DateTime FirstDue(DateTime nowUtc) => nowUtc.AddDays(IntervalsDays[0]);
public static (int Stage, DateTime? DueUtc, bool Mastered) AfterReview(
int currentStage,
bool isCorrect,
DateTime nowUtc)
{
if (!isCorrect)
return (0, FirstDue(nowUtc), false);
var nextStage = Math.Clamp(currentStage + 1, 1, IntervalsDays.Length);
if (nextStage >= IntervalsDays.Length)
return (nextStage, null, true);
return (nextStage, nowUtc.AddDays(IntervalsDays[nextStage]), false);
}
}
api/QuestionsApi/Services/UserProfileStore.cs
+1
View File
@@ -14,6 +14,7 @@ public sealed class UserProfileStore
[
"QuestionAttempts", "UserRatings", "QuestionBookmarks", "UserNotepads",
"ReviewQueueItems", "QuestionReviewAttempts", "ReviewQueueBackfillStates",
"DynamicQuestionAttempts", "DynamicObjectiveMastery", "DynamicReviewItems",
"CcsAttempts", "CcsUserRatings", "CcsRatedCaseResults", "ExamSimulations",
"Step3Outcomes", "QuestionReports", "CcsCaseReports", "AppFeedback", "UserStudyProfiles",
"ExternalIdentities", "ExternalAuthTransactions",