step3/api/QuestionsApi/Services/Flashcards/FlashcardImportExportService.cs

using System.Text;
using System.Text.Json;
using MongoDB.Driver;
using QuestionsApi.Models;

namespace QuestionsApi.Services.Flashcards;

public sealed class FlashcardImportExportService
{
    private const int MaxNotesPerImport = 10_000;
    private const long MaxImportBytes = 25 * 1024 * 1024;

    private static readonly JsonSerializerOptions JsonOptions = new() { PropertyNameCaseInsensitive = true };

    private readonly FlashcardDeckStore _decks;
    private readonly FlashcardNoteStore _notes;
    private readonly FlashcardScheduleStore _schedules;
    private readonly FlashcardMediaStore _media;
    private readonly FlashcardSettingsStore _settings;
    private readonly IMongoCollection<FlashcardImportPreview> _previews;

    public FlashcardImportExportService(
        MongoDatabaseProvider mongo,
        FlashcardDeckStore decks,
        FlashcardNoteStore notes,
        FlashcardScheduleStore schedules,
        FlashcardMediaStore media,
        FlashcardSettingsStore settings)
    {
        _decks = decks;
        _notes = notes;
        _schedules = schedules;
        _media = media;
        _settings = settings;
        _previews = mongo.Database.GetCollection<FlashcardImportPreview>("FlashcardImportPreviews");
        _previews.Indexes.CreateOne(new CreateIndexModel<FlashcardImportPreview>(
            Builders<FlashcardImportPreview>.IndexKeys.Ascending(v => v.ExpiresUtc),
            new CreateIndexOptions { ExpireAfter = TimeSpan.Zero, Name = "flashcard_import_preview_ttl" }));
    }

    public static void ValidateSize(long byteLength)
    {
        if (byteLength > MaxImportBytes)
            throw new FlashcardValidationException("import file must be 25 MB or smaller");
    }

    public async Task<(string Csv, List<FlashcardExportOmission> Omissions)> ExportCsvAsync(string userId)
    {
        var deckList = await _decks.ListAsync(userId);
        var deckNames = deckList.ToDictionary(v => v.Id!, v => v.Name, StringComparer.Ordinal);
        var allNotes = await _notes.ListAllActiveAsync(userId);

        var sb = new StringBuilder();
        sb.Append(FlashcardCsv.WriteRow(FlashcardCsv.Headers)).Append('\n');
        foreach (var note in allNotes.OrderBy(v => v.CreatedUtc))
        {
            var deckName = deckNames.GetValueOrDefault(note.DeckId, "Default");
            sb.Append(FlashcardCsv.WriteRow([
                note.StableId, deckName, note.Type, note.Prompt, note.Answer, note.Extra,
                string.Join(';', note.Tags), note.SourceQuestionExternalId
            ])).Append('\n');
        }

        var omissions = new List<FlashcardExportOmission>
        {
            new() { Kind = "schedule", Reason = "CSV export omits FSRS scheduling state" },
            new() { Kind = "history", Reason = "CSV export omits review history" }
        };
        if (allNotes.Any(v => v.Images.Count > 0))
            omissions.Insert(0, new FlashcardExportOmission { Kind = "images", Reason = "CSV export omits attached images" });
        return (sb.ToString(), omissions);
    }

    public async Task<FlashcardJsonExport> ExportJsonAsync(string userId)
    {
        var export = new FlashcardJsonExport
        {
            Settings = await _settings.GetAsync(userId),
            Decks = await _decks.ListAsync(userId)
        };

        foreach (var note in await _notes.ListAllActiveAsync(userId))
        {
            var cards = await _schedules.GetByNoteIdAsync(userId, note.Id!);
            var logs = new List<FlashcardReviewLog>();
            foreach (var card in cards)
                logs.AddRange(await _schedules.GetReviewLogsForCardAsync(userId, card.Id!));

            var images = new List<FlashcardJsonExportImage>();
            foreach (var image in note.Images)
            {
                var asset = await _media.GetAsync(userId, image.AssetId);
                if (asset is null) continue;
                images.Add(new FlashcardJsonExportImage
                {
                    AssetId = asset.Id!,
                    FileName = asset.FileName,
                    ContentType = asset.ContentType,
                    Base64Bytes = Convert.ToBase64String(asset.Bytes)
                });
            }

            export.Notes.Add(new FlashcardJsonExportNote { Note = note, Cards = cards, ReviewLogs = logs, Images = images });
        }
        return export;
    }

    public async Task<FlashcardImportPreviewResponse> PreviewCsvAsync(string userId, string csvText)
    {
        var rows = FlashcardCsv.ParseRows(csvText);
        if (rows.Count == 0) throw new FlashcardValidationException("CSV file is empty");
        if (!rows[0].Select(v => v.Trim()).SequenceEqual(FlashcardCsv.Headers, StringComparer.OrdinalIgnoreCase))
            throw new FlashcardValidationException($"CSV header must be: {string.Join(",", FlashcardCsv.Headers)}");
        if (rows.Count - 1 > MaxNotesPerImport)
            throw new FlashcardValidationException($"import limited to {MaxNotesPerImport:N0} notes");

        var existingNotes = await _notes.ListAllActiveAsync(userId);
        var byStableId = existingNotes.Where(v => v.StableId.Length > 0)
            .ToDictionary(v => v.StableId, StringComparer.Ordinal);
        var fingerprints = existingNotes.Select(v => v.ContentFingerprint).ToHashSet(StringComparer.Ordinal);

        var records = new List<FlashcardImportPendingRecord>();
        for (var r = 1; r < rows.Count; r++)
        {
            var cols = rows[r];
            var record = new FlashcardImportPendingRecord { RowNumber = r + 1 };
            if (cols.Length != FlashcardCsv.Headers.Length)
            {
                record.Action = FlashcardImportRowActions.Error;
                record.Error = $"expected {FlashcardCsv.Headers.Length} columns, found {cols.Length}";
                records.Add(record);
                continue;
            }

            var stableId = cols[0].Trim();
            var deckName = cols[1].Trim();
            var type = cols[2].Trim().ToLowerInvariant();
            var prompt = cols[3];
            var answer = cols[4];
            var extra = cols[5];
            var tags = cols[6].Split(';', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries).ToList();
            var sourceQuestionId = cols[7].Trim();

            record.StableId = stableId;
            record.DeckName = deckName.Length == 0 ? "Default" : deckName;
            record.Type = type;
            record.Prompt = prompt;
            record.Answer = answer;
            record.Extra = extra;
            record.Tags = tags;
            record.SourceQuestionExternalId = sourceQuestionId;

            if (type != FlashcardTypes.Basic && type != FlashcardTypes.Cloze)
            {
                record.Action = FlashcardImportRowActions.Error;
                record.Error = "type must be 'basic' or 'cloze'";
            }
            else if (prompt.Trim().Length == 0)
            {
                record.Action = FlashcardImportRowActions.Error;
                record.Error = "prompt is required";
            }
            else if (type == FlashcardTypes.Cloze && !FlashcardClozeParser.HasValidCloze(prompt))
            {
                record.Action = FlashcardImportRowActions.Error;
                record.Error = "cloze prompt must contain at least one {{cN::answer}} marker";
            }
            else
            {
                var fingerprint = FlashcardNoteStore.Fingerprint(type, prompt, answer);
                if (stableId.Length > 0 && byStableId.TryGetValue(stableId, out var existing))
                {
                    record.Action = existing.ContentFingerprint == fingerprint
                        ? FlashcardImportRowActions.Skip
                        : FlashcardImportRowActions.Update;
                }
                else if (fingerprints.Contains(fingerprint))
                {
                    record.Action = FlashcardImportRowActions.Skip;
                }
                else
                {
                    record.Action = FlashcardImportRowActions.Create;
                    if (stableId.Length == 0) record.StableId = Guid.NewGuid().ToString();
                }
            }
            records.Add(record);
        }

        return await SavePreviewAsync(userId, FlashcardImportFormats.Csv, records);
    }

    public async Task<FlashcardImportPreviewResponse> PreviewJsonAsync(string userId, string jsonText)
    {
        FlashcardJsonExport parsed;
        try
        {
            parsed = JsonSerializer.Deserialize<FlashcardJsonExport>(jsonText, JsonOptions)
                ?? throw new FlashcardValidationException("empty JSON document");
        }
        catch (JsonException exception)
        {
            throw new FlashcardValidationException($"invalid JSON: {exception.Message}");
        }

        if (parsed.Notes.Count > MaxNotesPerImport)
            throw new FlashcardValidationException($"import limited to {MaxNotesPerImport:N0} notes");

        var existingNotes = await _notes.ListAllActiveAsync(userId);
        var byStableId = existingNotes.Where(v => v.StableId.Length > 0)
            .ToDictionary(v => v.StableId, StringComparer.Ordinal);

        var records = new List<FlashcardImportPendingRecord>();
        var rowNumber = 0;
        foreach (var item in parsed.Notes)
        {
            rowNumber++;
            var note = item.Note;
            var stableId = string.IsNullOrWhiteSpace(note.StableId) ? Guid.NewGuid().ToString() : note.StableId;
            var type = FlashcardTypes.All.Contains(note.Type) ? note.Type : FlashcardTypes.Basic;
            var record = new FlashcardImportPendingRecord
            {
                RowNumber = rowNumber,
                StableId = stableId,
                DeckName = parsed.Decks.FirstOrDefault(d => d.Id == note.DeckId)?.Name ?? "Default",
                Type = type,
                Prompt = note.Prompt,
                Answer = note.Answer,
                Extra = note.Extra,
                Tags = note.Tags,
                SourceQuestionExternalId = note.SourceQuestionExternalId,
                Cards = item.Cards,
                ReviewLogs = item.ReviewLogs
            };

            record.Action = type == FlashcardTypes.Cloze && !FlashcardClozeParser.HasValidCloze(note.Prompt)
                ? FlashcardImportRowActions.Error
                : byStableId.ContainsKey(stableId) ? FlashcardImportRowActions.Update : FlashcardImportRowActions.Create;
            if (record.Action == FlashcardImportRowActions.Error)
                record.Error = "cloze prompt must contain at least one {{cN::answer}} marker";

            records.Add(record);
        }

        return await SavePreviewAsync(userId, FlashcardImportFormats.Json, records);
    }

    public async Task<FlashcardImportCommitResponse> CommitAsync(string userId, string importToken)
    {
        var preview = await _previews.Find(v => v.Id == importToken && v.UserId == userId).FirstOrDefaultAsync();
        if (preview is null) throw new FlashcardValidationException("import preview not found or expired; re-run preview");

        var deckCache = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
        var response = new FlashcardImportCommitResponse();

        foreach (var record in preview.Records)
        {
            if (record.Action == FlashcardImportRowActions.Error)
            {
                response.Errored++;
                response.Errors.Add(ToRow(record));
                continue;
            }
            if (record.Action == FlashcardImportRowActions.Skip)
            {
                response.Skipped++;
                continue;
            }

            var deckId = await ResolveDeckIdAsync(userId, record.DeckName, deckCache);
            var existing = await _notes.GetByStableIdAsync(userId, record.StableId);
            var request = new UpsertFlashcardNoteRequest
            {
                DeckId = deckId,
                StableId = record.StableId,
                Type = record.Type,
                Prompt = record.Prompt,
                Answer = record.Answer,
                Extra = record.Extra,
                Tags = record.Tags,
                SourceQuestionExternalId = record.SourceQuestionExternalId,
                ExpectedRevision = existing?.Revision
            };

            FlashcardNoteUpdateResult result;
            try
            {
                result = await _notes.UpsertAsync(userId, request, existing?.Id);
            }
            catch (FlashcardValidationException exception)
            {
                response.Errored++;
                response.Errors.Add(new FlashcardImportPreviewRow
                {
                    RowNumber = record.RowNumber, Action = FlashcardImportRowActions.Error,
                    Error = exception.Message, StableId = record.StableId
                });
                continue;
            }

            if (!result.Succeeded)
            {
                response.Errored++;
                response.Errors.Add(new FlashcardImportPreviewRow
                {
                    RowNumber = record.RowNumber, Action = FlashcardImportRowActions.Error,
                    Error = "note changed since preview; re-run import", StableId = record.StableId
                });
                continue;
            }

            if (record.Action == FlashcardImportRowActions.Create) response.Created++; else response.Updated++;

            foreach (var card in record.Cards)
                await _schedules.ImportScheduleStateAsync(userId, result.Note!.Id!, card);
            foreach (var log in record.ReviewLogs)
            {
                var targetCard = (await _schedules.GetByNoteIdAsync(userId, result.Note!.Id!))
                    .FirstOrDefault(v => v.Variant == record.Cards.FirstOrDefault(c => c.Id == log.CardId)?.Variant);
                if (targetCard is not null)
                    await _schedules.ImportReviewLogAsync(userId, targetCard.Id!, log);
            }
        }

        await _previews.DeleteOneAsync(v => v.Id == importToken);
        return response;
    }

    private async Task<FlashcardImportPreviewResponse> SavePreviewAsync(
        string userId, string format, List<FlashcardImportPendingRecord> records)
    {
        var preview = new FlashcardImportPreview
        {
            Id = Guid.NewGuid().ToString(),
            UserId = userId,
            Format = format,
            Records = records
        };
        await _previews.InsertOneAsync(preview);

        return new FlashcardImportPreviewResponse
        {
            ImportToken = preview.Id,
            CreateCount = records.Count(v => v.Action == FlashcardImportRowActions.Create),
            UpdateCount = records.Count(v => v.Action == FlashcardImportRowActions.Update),
            SkipCount = records.Count(v => v.Action == FlashcardImportRowActions.Skip),
            ErrorCount = records.Count(v => v.Action == FlashcardImportRowActions.Error),
            Rows = records.Select(ToRow).ToList()
        };
    }

    private static FlashcardImportPreviewRow ToRow(FlashcardImportPendingRecord record) => new()
    {
        RowNumber = record.RowNumber,
        Action = record.Action,
        Error = record.Error,
        StableId = record.StableId,
        Deck = record.DeckName,
        Type = record.Type,
        Prompt = Truncate(record.Prompt),
        Answer = Truncate(record.Answer)
    };

    private static string Truncate(string value) => value.Length > 200 ? value[..200] + "…" : value;

    private async Task<string> ResolveDeckIdAsync(string userId, string deckName, Dictionary<string, string> cache)
    {
        var key = deckName.Trim();
        if (key.Length == 0) key = "Default";
        if (cache.TryGetValue(key, out var cached)) return cached;

        if (string.Equals(key, "Default", StringComparison.OrdinalIgnoreCase))
        {
            var defaultDeck = await _decks.GetOrCreateDefaultDeckAsync(userId);
            cache[key] = defaultDeck.Id!;
            return defaultDeck.Id!;
        }

        var normalized = FlashcardDeckStore.Normalize(key);
        var existing = await _decks.GetByNameAsync(userId, normalized);
        if (existing is not null)
        {
            cache[key] = existing.Id!;
            return existing.Id!;
        }

        var created = await _decks.UpsertAsync(userId, new UpsertFlashcardDeckRequest { Name = key }, null);
        var deckId = created.Deck?.Id ?? (await _decks.GetByNameAsync(userId, normalized))?.Id
            ?? throw new InvalidOperationException("deck resolution failed during import");
        cache[key] = deckId;
        return deckId;
    }
}