287 lines
13 KiB
C#
287 lines
13 KiB
C#
using MongoDB.Bson;
|
|
using MongoDB.Driver;
|
|
using System.Security.Cryptography;
|
|
using System.Text;
|
|
using System.Text.Json;
|
|
|
|
namespace QuestionsApi.Services;
|
|
|
|
public static class QuestionPromotionCommand
|
|
{
|
|
public const string Confirmation = "PROMOTE_TO_PRD";
|
|
|
|
public static async Task<int> RunAsync(string[] args, TextWriter output)
|
|
{
|
|
try
|
|
{
|
|
var options = Parse(args);
|
|
if (options.Apply && options.Confirmation != Confirmation)
|
|
throw new ArgumentException($"--confirm {Confirmation} is required with --apply");
|
|
if (options.SourceDatabase.Equals(options.TargetDatabase, StringComparison.OrdinalIgnoreCase)
|
|
&& options.SourceConnection == options.TargetConnection)
|
|
throw new ArgumentException("source and target databases must be different");
|
|
|
|
var source = new MongoClient(options.SourceConnection).GetDatabase(options.SourceDatabase);
|
|
var target = new MongoClient(options.TargetConnection).GetDatabase(options.TargetDatabase);
|
|
await source.RunCommandAsync<BsonDocument>(new BsonDocument("ping", 1));
|
|
await target.RunCommandAsync<BsonDocument>(new BsonDocument("ping", 1));
|
|
|
|
var questionFilter = Builders<BsonDocument>.Filter.Eq("IsPublished", true)
|
|
& Builders<BsonDocument>.Filter.Eq("FlaggedForReviewUtc", BsonNull.Value);
|
|
var questions = await source.GetCollection<BsonDocument>("Questions")
|
|
.Find(questionFilter).ToListAsync();
|
|
var duplicateIds = questions
|
|
.GroupBy(ExternalId, StringComparer.Ordinal)
|
|
.Where(group => group.Count() > 1)
|
|
.Select(group => group.Key)
|
|
.ToList();
|
|
if (duplicateIds.Count > 0)
|
|
throw new InvalidOperationException($"duplicate eligible ExternalIds: {string.Join(", ", duplicateIds)}");
|
|
|
|
var mediaIds = CollectMediaIds(questions);
|
|
var media = mediaIds.Count == 0
|
|
? []
|
|
: await source.GetCollection<BsonDocument>("QuestionMediaAssets")
|
|
.Find(Builders<BsonDocument>.Filter.In("_id", mediaIds)).ToListAsync();
|
|
if (media.Count != mediaIds.Count)
|
|
throw new InvalidOperationException($"{mediaIds.Count - media.Count} referenced question media assets are missing");
|
|
|
|
var sourceIds = questions.Select(ExternalId).ToHashSet(StringComparer.Ordinal);
|
|
var targetCollection = target.GetCollection<BsonDocument>("Questions");
|
|
var targetQuestions = sourceIds.Count == 0
|
|
? []
|
|
: await targetCollection.Find(
|
|
Builders<BsonDocument>.Filter.In("ExternalId", sourceIds)).ToListAsync();
|
|
var targetTotal = await targetCollection.CountDocumentsAsync(FilterDefinition<BsonDocument>.Empty);
|
|
var comparison = ClassifyQuestions(questions, targetQuestions);
|
|
var summary = new PromotionSummary(
|
|
questions.Count,
|
|
media.Count,
|
|
comparison.Inserted,
|
|
comparison.Updated,
|
|
comparison.Unchanged,
|
|
checked((int)Math.Max(0, targetTotal - targetQuestions.Count)),
|
|
HashQuestions(questions));
|
|
|
|
await WriteSummaryAsync(output, options.Apply ? "apply" : "dry-run", options, summary);
|
|
if (!options.Apply)
|
|
return 0;
|
|
|
|
await UpsertMediaAsync(target, media);
|
|
await UpsertQuestionsAsync(target, questions);
|
|
|
|
var verifiedQuestions = sourceIds.Count == 0
|
|
? []
|
|
: await targetCollection.Find(
|
|
Builders<BsonDocument>.Filter.In("ExternalId", sourceIds)).ToListAsync();
|
|
var verification = ClassifyQuestions(questions, verifiedQuestions);
|
|
if (verification.Inserted != 0 || verification.Updated != 0)
|
|
throw new InvalidOperationException(
|
|
$"target verification failed: {verification.Inserted} missing and {verification.Updated} mismatched questions");
|
|
var verifiedMedia = mediaIds.Count == 0
|
|
? 0
|
|
: checked((int)await target.GetCollection<BsonDocument>("QuestionMediaAssets")
|
|
.CountDocumentsAsync(Builders<BsonDocument>.Filter.In("_id", mediaIds)));
|
|
if (verifiedMedia != mediaIds.Count)
|
|
throw new InvalidOperationException(
|
|
$"target verification failed: {mediaIds.Count - verifiedMedia} media assets are missing");
|
|
|
|
await target.GetCollection<BsonDocument>("ContentPromotions").InsertOneAsync(new BsonDocument
|
|
{
|
|
["PromotionId"] = Guid.NewGuid().ToString("N"),
|
|
["AppliedUtc"] = DateTime.UtcNow,
|
|
["Operator"] = options.Operator,
|
|
["BuildRevision"] = options.BuildRevision,
|
|
["SourceDatabase"] = options.SourceDatabase,
|
|
["TargetDatabase"] = options.TargetDatabase,
|
|
["Scope"] = "questions",
|
|
["Eligibility"] = "published_unflagged",
|
|
["Questions"] = summary.EligibleQuestions,
|
|
["Inserted"] = summary.Inserted,
|
|
["Updated"] = summary.Updated,
|
|
["Unchanged"] = summary.Unchanged,
|
|
["MediaAssets"] = summary.MediaAssets,
|
|
["TargetExtras"] = summary.TargetExtras,
|
|
["ContentHash"] = summary.ContentHash
|
|
});
|
|
await output.WriteLineAsync("Question promotion completed and verified.");
|
|
return 0;
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
await output.WriteLineAsync($"Question promotion failed: {exception.Message}");
|
|
return 1;
|
|
}
|
|
}
|
|
|
|
internal static bool IsEligible(BsonDocument question) =>
|
|
question.GetValue("IsPublished", false).ToBoolean()
|
|
&& (!question.TryGetValue("FlaggedForReviewUtc", out var flagged) || flagged.IsBsonNull);
|
|
|
|
internal static HashSet<ObjectId> CollectMediaIds(IEnumerable<BsonDocument> questions)
|
|
{
|
|
var ids = new HashSet<ObjectId>();
|
|
foreach (var question in questions)
|
|
{
|
|
if (!question.TryGetValue("Media", out var raw) || !raw.IsBsonArray)
|
|
continue;
|
|
foreach (var value in raw.AsBsonArray)
|
|
{
|
|
if (!value.IsString)
|
|
continue;
|
|
var reference = value.AsString.Trim();
|
|
var candidate = reference.TrimEnd('/').Split('/').LastOrDefault() ?? "";
|
|
if (ObjectId.TryParse(reference, out var direct))
|
|
ids.Add(direct);
|
|
else if (ObjectId.TryParse(candidate, out var fromPath))
|
|
ids.Add(fromPath);
|
|
else if (reference.Contains("/api/questions/media/", StringComparison.OrdinalIgnoreCase))
|
|
throw new InvalidOperationException(
|
|
$"question {ExternalId(question)} has an invalid local media reference: {reference}");
|
|
}
|
|
}
|
|
return ids;
|
|
}
|
|
|
|
internal static QuestionComparison ClassifyQuestions(
|
|
IEnumerable<BsonDocument> sourceQuestions,
|
|
IEnumerable<BsonDocument> targetQuestions)
|
|
{
|
|
var target = targetQuestions.ToDictionary(ExternalId, StringComparer.Ordinal);
|
|
var inserted = 0;
|
|
var updated = 0;
|
|
var unchanged = 0;
|
|
foreach (var source in sourceQuestions)
|
|
{
|
|
var externalId = ExternalId(source);
|
|
if (!target.TryGetValue(externalId, out var existing))
|
|
inserted++;
|
|
else if (EquivalentQuestion(source, existing))
|
|
unchanged++;
|
|
else
|
|
updated++;
|
|
}
|
|
return new QuestionComparison(inserted, updated, unchanged);
|
|
}
|
|
|
|
internal static string HashQuestions(IEnumerable<BsonDocument> questions)
|
|
{
|
|
using var hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256);
|
|
foreach (var document in questions.OrderBy(ExternalId, StringComparer.Ordinal))
|
|
hash.AppendData(Encoding.UTF8.GetBytes(CanonicalQuestion(document).ToJson()));
|
|
return Convert.ToHexString(hash.GetHashAndReset()).ToLowerInvariant();
|
|
}
|
|
|
|
private static bool EquivalentQuestion(BsonDocument source, BsonDocument target) =>
|
|
CanonicalQuestion(source).Equals(CanonicalQuestion(target));
|
|
|
|
private static BsonDocument CanonicalQuestion(BsonDocument source) =>
|
|
Canonicalize(new BsonDocument(source.Where(element => element.Name != "_id"))).AsBsonDocument;
|
|
|
|
private static BsonValue Canonicalize(BsonValue value)
|
|
{
|
|
if (value.IsBsonDocument)
|
|
return new BsonDocument(value.AsBsonDocument
|
|
.OrderBy(element => element.Name, StringComparer.Ordinal)
|
|
.Select(element => new BsonElement(element.Name, Canonicalize(element.Value))));
|
|
if (value.IsBsonArray)
|
|
return new BsonArray(value.AsBsonArray.Select(Canonicalize));
|
|
return value;
|
|
}
|
|
|
|
private static string ExternalId(BsonDocument document) =>
|
|
document.GetValue("ExternalId", "").AsString;
|
|
|
|
private static async Task UpsertQuestionsAsync(IMongoDatabase target, IEnumerable<BsonDocument> questions)
|
|
{
|
|
var collection = target.GetCollection<BsonDocument>("Questions");
|
|
foreach (var source in questions)
|
|
{
|
|
var replacement = new BsonDocument(source.Where(element => element.Name != "_id"));
|
|
await collection.ReplaceOneAsync(
|
|
Builders<BsonDocument>.Filter.Eq("ExternalId", source["ExternalId"]),
|
|
replacement,
|
|
new ReplaceOptions { IsUpsert = true });
|
|
}
|
|
}
|
|
|
|
private static async Task UpsertMediaAsync(IMongoDatabase target, IEnumerable<BsonDocument> media)
|
|
{
|
|
var collection = target.GetCollection<BsonDocument>("QuestionMediaAssets");
|
|
foreach (var source in media)
|
|
await collection.ReplaceOneAsync(
|
|
Builders<BsonDocument>.Filter.Eq("_id", source["_id"]),
|
|
source,
|
|
new ReplaceOptions { IsUpsert = true });
|
|
}
|
|
|
|
private static Task WriteSummaryAsync(
|
|
TextWriter output,
|
|
string mode,
|
|
PromotionOptions options,
|
|
PromotionSummary summary) =>
|
|
output.WriteLineAsync(JsonSerializer.Serialize(new
|
|
{
|
|
mode,
|
|
scope = "questions",
|
|
eligibility = "published_unflagged",
|
|
source = options.SourceDatabase,
|
|
target = options.TargetDatabase,
|
|
eligibleQuestions = summary.EligibleQuestions,
|
|
mediaAssets = summary.MediaAssets,
|
|
inserted = summary.Inserted,
|
|
updated = summary.Updated,
|
|
unchanged = summary.Unchanged,
|
|
targetExtras = summary.TargetExtras,
|
|
contentHash = summary.ContentHash
|
|
}));
|
|
|
|
private static PromotionOptions Parse(IReadOnlyList<string> args)
|
|
{
|
|
string Required(string name, string environmentName)
|
|
{
|
|
var index = args.IndexOf($"--{name}");
|
|
var value = index >= 0 && index + 1 < args.Count
|
|
? args[index + 1]
|
|
: Environment.GetEnvironmentVariable(environmentName);
|
|
return string.IsNullOrWhiteSpace(value)
|
|
? throw new ArgumentException($"--{name} is required")
|
|
: value;
|
|
}
|
|
string Optional(string name, string fallback)
|
|
{
|
|
var index = args.IndexOf($"--{name}");
|
|
return index >= 0 && index + 1 < args.Count ? args[index + 1] : fallback;
|
|
}
|
|
return new PromotionOptions(
|
|
Required("source-connection", "SOURCE_MONGODB_CONNECTION_STRING"),
|
|
Required("source-database", "SOURCE_MONGODB_DATABASE"),
|
|
Required("target-connection", "TARGET_MONGODB_CONNECTION_STRING"),
|
|
Required("target-database", "TARGET_MONGODB_DATABASE"),
|
|
args.Contains("--apply"),
|
|
Optional("confirm", ""),
|
|
Optional("operator", "unknown"),
|
|
Optional("build-revision", "unknown"));
|
|
}
|
|
|
|
private sealed record PromotionOptions(
|
|
string SourceConnection,
|
|
string SourceDatabase,
|
|
string TargetConnection,
|
|
string TargetDatabase,
|
|
bool Apply,
|
|
string Confirmation,
|
|
string Operator,
|
|
string BuildRevision);
|
|
|
|
internal sealed record QuestionComparison(int Inserted, int Updated, int Unchanged);
|
|
internal sealed record PromotionSummary(
|
|
int EligibleQuestions,
|
|
int MediaAssets,
|
|
int Inserted,
|
|
int Updated,
|
|
int Unchanged,
|
|
int TargetExtras,
|
|
string ContentHash);
|
|
}
|