using System.Security.Claims; using System.Text.Json; using MiaoJiZhang.Api.Contracts; using MiaoJiZhang.Api.Services; using MiaoJiZhang.Domain.Entities; using MiaoJiZhang.Domain.Enums; using MiaoJiZhang.Infrastructure.Persistence; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; using Microsoft.AspNetCore.RateLimiting; using Microsoft.EntityFrameworkCore; namespace MiaoJiZhang.Api.Controllers; /// /// OCR/语音/图片辅助解析。客户端确认后再调用交易接口正式入账。 /// 图片模式由后端多模态模型提取一笔或多笔账单草稿。 /// [ApiController] [Authorize] [Route("api/parse")] [RequireAiPermission] public class ParseController(AppDbContext db, ILlmClient llm, AgentService agent) : ControllerBase { private long Uid => long.Parse( User.FindFirstValue(ClaimTypes.NameIdentifier) ?? User.FindFirstValue("sub")!); [HttpPost] [EnableRateLimiting("ai")] public async Task> Parse(OcrParseRequest req) { return await ParseWithAgent(req, HttpContext.RequestAborted); } private async Task> ParseWithAgent( OcrParseRequest req, CancellationToken ct) { if (string.IsNullOrWhiteSpace(req.Text)) return BadRequest(new ApiError("TEXT_EMPTY", "内容不能为空")); var featureKey = req.Source == "voice" ? "feature.voice_enabled" : "feature.ocr_enabled"; if (!await FeatureEnabled(featureKey)) return StatusCode(403, new ApiError("FEATURE_DISABLED", "该解析功能已由后台关闭")); IReadOnlyList drafts; try { drafts = await agent.ParseDraftsAsync(Uid, req.Text.Trim(), ct); } catch (InvalidOperationException ex) { return StatusCode( 503, new ApiError("AGENT_PARSE_UNAVAILABLE", ex.Message)); } catch (Exception) { return StatusCode( 502, new ApiError( "AGENT_PARSE_FAILED", "AI 暂时无法解析,没有生成账单草稿")); } if (drafts.Count == 0) { var fallback = await db.Categories.FirstAsync( c => c.Name == "其他" && c.Type == TransactionType.Expense && c.UserId == null, ct); return Ok(new OcrParseResponse( false, fallback.Id, fallback.Name, fallback.IconKey, 0, null, req.Text, "expense")); } var bill = drafts[0]; var category = await db.Categories.FirstAsync( c => c.Id == bill.CategoryId && c.Type == bill.Type.CategoryType(bill.TransferDirection), ct); return Ok(new OcrParseResponse( true, category.Id, category.Name, category.IconKey, bill.Amount, bill.PaymentMethod, bill.Note, bill.Type.ToWire(), bill.TransferDirection.ToWire(), bill.Counterparty)); } /// 上传截屏或小票,提取其中全部独立交易。 [HttpPost("image")] [EnableRateLimiting("upload")] [RequestSizeLimit(10 * 1024 * 1024)] [RequestFormLimits(MultipartBodyLengthLimit = 10 * 1024 * 1024)] public async Task> ParseImage( IFormFile file, [FromQuery] string source = "image", CancellationToken ct = default) { var featureKey = source == "screenshot" ? "feature.screenshot_bookkeeping_enabled" : "feature.ocr_enabled"; if (!await FeatureEnabled(featureKey)) return StatusCode(403, new ApiError("FEATURE_DISABLED", "该图片识别功能已由后台关闭")); if (file is null || file.Length == 0) return BadRequest(new ApiError("FILE_EMPTY", "请上传图片")); if (!file.ContentType.StartsWith("image/")) return BadRequest(new ApiError("NOT_IMAGE", "仅支持图片文件")); if (file.Length > 10 * 1024 * 1024) return BadRequest(new ApiError("FILE_TOO_LARGE", "图片不能超过 10MB")); if (!llm.IsEnabled) { return StatusCode( 503, new ApiError( "LLM_NOT_CONFIGURED", "AI 图片解析未配置,请在管理后台设置 LLM API Key")); } using var stream = new MemoryStream(); await file.CopyToAsync(stream, ct); IReadOnlyList? results; try { results = await llm.AnalyzeImageAsync( stream.ToArray(), file.ContentType, ct); } catch (InvalidOperationException ex) { return StatusCode(502, new ApiError("LLM_IMAGE_ERROR", ex.Message)); } if (results is null) return StatusCode(500, new ApiError("LLM_ERROR", "AI 解析失败,请稍后重试")); var categories = await db.Categories .Where(c => !c.IsDeleted && (c.UserId == null || c.UserId == Uid)) .ToListAsync(ct); if (results.Count == 0) { return Ok(new ImageParseResponse( false, 0, "待确认", "tag", 0, null, "", "unknown", [], null)); } var items = new List(); foreach (var result in results.Take(20)) { var normalizedType = result.Type.Trim().ToLowerInvariant(); var transferDirection = result.TransferDirection is "in" or "out" ? result.TransferDirection : null; if (normalizedType is not ("income" or "expense" or "transfer") || normalizedType == "transfer" && transferDirection is null) { items.Add(new ImageParseItemResponse( false, 0, "待确认", "tag", "unknown", result.Amount, result.PaymentMethod, result.Note, result.OccurredAt, transferDirection, result.Counterparty)); continue; } var categoryType = normalizedType == "income" || transferDirection == "in" ? TransactionType.Income : TransactionType.Expense; var category = FindCategory(categories, categoryType, result.CategoryName); items.Add(new ImageParseItemResponse( true, category.Id, category.Name, category.IconKey, normalizedType, result.Amount, result.PaymentMethod, result.Note, result.OccurredAt, transferDirection, result.Counterparty)); } var first = items[0]; return Ok(new ImageParseResponse( true, first.CategoryId, first.CategoryName, first.CategoryIcon, first.Amount, first.PaymentMethod, first.Note, first.Type, items, first.OccurredAt, first.TransferDirection, first.Counterparty)); } [HttpPost("recognition-batch")] [EnableRateLimiting("upload")] [RequestSizeLimit(10 * 1024 * 1024)] [RequestFormLimits(MultipartBodyLengthLimit = 10 * 1024 * 1024)] public async Task> ReconcileRecognitionBatch( CancellationToken ct = default) { if (!await FeatureEnabled("feature.screenshot_bookkeeping_enabled")) return StatusCode(403, new ApiError("FEATURE_DISABLED", "AI 截图补全已由后台关闭")); if (!llm.IsEnabled) return StatusCode(503, new ApiError("LLM_NOT_CONFIGURED", "AI 图片解析未配置")); var form = await Request.ReadFormAsync(ct); var manifestText = form["manifest"].FirstOrDefault(); if (string.IsNullOrWhiteSpace(manifestText)) return BadRequest(new ApiError("BATCH_MANIFEST_EMPTY", "批次清单不能为空")); RecognitionBatchManifestRequest? manifest; try { manifest = JsonSerializer.Deserialize( manifestText, new JsonSerializerOptions(JsonSerializerDefaults.Web)); } catch (JsonException) { return BadRequest(new ApiError("BATCH_MANIFEST_INVALID", "批次清单格式无效")); } if (manifest is null || !Guid.TryParse(manifest.BatchId, out _) || manifest.Candidates.Count > 10 || manifest.Evidence.Count > 10 || manifest.Candidates.Count + manifest.Evidence.Count == 0) { return BadRequest(new ApiError("BATCH_MANIFEST_INVALID", "批次数量或标识无效")); } if (manifest.Candidates.Select(item => item.CandidateId).Distinct().Count() != manifest.Candidates.Count || manifest.Evidence.Select(item => item.EvidenceId).Distinct().Count() != manifest.Evidence.Count) { return BadRequest(new ApiError("BATCH_ID_DUPLICATED", "批次中存在重复标识")); } var evidenceById = manifest.Evidence.ToDictionary(item => item.EvidenceId); var candidateIds = manifest.Candidates.Select(item => item.CandidateId).ToHashSet(); if (manifest.Evidence.Any(item => item.CandidateId is not null && !candidateIds.Contains(item.CandidateId)) || manifest.Candidates.Any(candidate => (candidate.EvidenceIds ?? []).Any(evidenceId => !evidenceById.TryGetValue(evidenceId, out var evidence) || evidence.CandidateId != candidate.CandidateId))) { return BadRequest(new ApiError("BATCH_EVIDENCE_INVALID", "截图与候选关联无效")); } var imageInputs = new List(); var uploadedEvidenceIds = new HashSet(); long totalBytes = 0; foreach (var file in form.Files) { var evidenceId = Path.GetFileNameWithoutExtension(file.FileName); if (!evidenceById.TryGetValue(evidenceId, out var evidence) || !uploadedEvidenceIds.Add(evidenceId)) return BadRequest(new ApiError("BATCH_EVIDENCE_UNKNOWN", "截图与批次清单不匹配")); if (!file.ContentType.StartsWith("image/") || file.Length <= 0 || file.Length > 1024 * 1024) return BadRequest(new ApiError("BATCH_EVIDENCE_INVALID", "单张截图必须是 1MB 以内的图片")); totalBytes += file.Length; if (totalBytes > 9 * 1024 * 1024) return BadRequest(new ApiError("BATCH_TOO_LARGE", "批次截图总大小不能超过 9MB")); using var stream = new MemoryStream(); await file.CopyToAsync(stream, ct); imageInputs.Add(new RecognitionBatchModelEvidence( evidence.EvidenceId, evidence.CandidateId, evidence.FlowSessionId, evidence.PackageName, evidence.CapturedAt, stream.ToArray(), file.ContentType)); } if (uploadedEvidenceIds.Count != evidenceById.Count) return BadRequest(new ApiError("BATCH_EVIDENCE_MISSING", "批次截图上传不完整")); var categories = await db.Categories .Where(category => !category.IsDeleted && (category.UserId == null || category.UserId == Uid)) .ToListAsync(ct); var candidates = manifest.Candidates.Select(candidate => new RecognitionBatchModelCandidate( candidate.CandidateId, candidate.FlowSessionId, candidate.PackageName, candidate.Type, candidate.Amount, candidate.Merchant, candidate.OrderId, candidate.OccurredAt, candidate.RecognitionKind, candidate.CategoryHint, candidate.Confidence, candidate.EvidenceIds ?? [], candidate.TransferDirection, candidate.Counterparty, candidate.ProviderTransactionId, candidate.RecognitionOccurrenceId, candidate.IdentityConfidence)).ToList(); IReadOnlyList? modelActions; try { modelActions = await llm.ReconcileRecognitionBatchAsync( new RecognitionBatchModelInput( manifest.BatchId, candidates, imageInputs, categories.Where(category => category.Type == TransactionType.Expense) .Select(category => category.Name).Distinct().ToList(), categories.Where(category => category.Type == TransactionType.Income) .Select(category => category.Name).Distinct().ToList()), ct); } catch (InvalidOperationException ex) { return StatusCode(502, new ApiError("BATCH_AI_FAILED", ex.Message)); } finally { foreach (var evidence in imageInputs) Array.Clear(evidence.ImageBytes); } if (modelActions is null) return StatusCode(502, new ApiError("BATCH_AI_FAILED", "AI 未返回批次对账结果")); var actions = ValidateBatchActions(manifest, modelActions, categories); return Ok(new RecognitionBatchResponse(manifest.BatchId, actions)); } private List ValidateBatchActions( RecognitionBatchManifestRequest manifest, IReadOnlyList modelActions, List categories) { var candidateIds = manifest.Candidates.Select(item => item.CandidateId).ToHashSet(); var evidenceById = manifest.Evidence.ToDictionary(item => item.EvidenceId); var selected = modelActions .Where(action => action.CandidateId != null && candidateIds.Contains(action.CandidateId)) .GroupBy(action => action.CandidateId!) .ToDictionary(group => group.Key, group => group.First()); var result = new List(); foreach (var candidate in manifest.Candidates) { selected.TryGetValue(candidate.CandidateId, out var model); var action = model?.Action is "update" or "drop" ? model.Action : "keep"; var reason = model?.Reason ?? "保留本地识别结果"; var confidence = model?.Confidence ?? 1; if (action == "drop" && (confidence < 0.9 || string.IsNullOrWhiteSpace(model?.EvidenceId) || !evidenceById.TryGetValue(model.EvidenceId, out var dropEvidence) || dropEvidence.CandidateId != candidate.CandidateId)) { action = "keep"; reason = "撤销证据不足,已保留本地结果"; } var type = model?.Type is "income" or "expense" or "transfer" ? model.Type : candidate.Type; var transferDirection = type == "transfer" ? model?.TransferDirection is "in" or "out" ? model.TransferDirection : candidate.TransferDirection is "in" or "out" ? candidate.TransferDirection : null : null; if (type == "transfer" && transferDirection is null) { action = "keep"; type = candidate.Type; transferDirection = candidate.TransferDirection; reason = "转账方向不明确,已保留本地结果"; } var amount = model?.Amount is > 0 ? model.Amount.Value : candidate.Amount; var category = FindCategory( categories, type == "income" || transferDirection == "in" ? TransactionType.Income : TransactionType.Expense, model?.CategoryName ?? candidate.CategoryHint ?? "其他"); result.Add(new RecognitionBatchActionResponse( action, model?.ActionId ?? $"keep-{candidate.CandidateId}", candidate.CandidateId, model?.EvidenceId, category.Id, category.Name, type, amount, model?.PaymentMethod, string.IsNullOrWhiteSpace(model?.Note) ? candidate.Merchant : model.Note, model?.OccurredAt ?? candidate.OccurredAt, confidence, reason, transferDirection, string.IsNullOrWhiteSpace(model?.Counterparty) ? candidate.Counterparty : model.Counterparty)); } var usedEvidence = new HashSet(); foreach (var model in modelActions.Where(action => action.Action == "create")) { if (string.IsNullOrWhiteSpace(model.EvidenceId) || !evidenceById.TryGetValue(model.EvidenceId, out var evidence) || !string.IsNullOrWhiteSpace(evidence.CandidateId) || !usedEvidence.Add(model.EvidenceId) || model.Type is not ("income" or "expense" or "transfer") || model.Type == "transfer" && model.TransferDirection is not ("in" or "out") || model.Amount is not > 0) { continue; } var categoryType = model.Type == "income" || model.TransferDirection == "in" ? TransactionType.Income : TransactionType.Expense; var category = FindCategory(categories, categoryType, model.CategoryName ?? "其他"); result.Add(new RecognitionBatchActionResponse( "create", model.ActionId, null, model.EvidenceId, category.Id, category.Name, model.Type, model.Amount, model.PaymentMethod, model.Note, model.OccurredAt ?? evidence.CapturedAt, model.Confidence, model.Reason, model.TransferDirection, model.Counterparty)); } return result.Take(20).ToList(); } private async Task FeatureEnabled(string key) { var value = await db.AppConfigs .Where(config => config.Key == key) .Select(config => config.Value) .FirstOrDefaultAsync(); return value is null || !value.Equals("false", StringComparison.OrdinalIgnoreCase); } private Category FindCategory( List categories, TransactionType type, string requestedName) { if (type == TransactionType.Income && requestedName is "退款" or "返现" or "退回") { requestedName = "报销"; } return categories .Where(c => c.Type == type && c.Name == requestedName) .OrderByDescending(c => c.UserId == Uid) .FirstOrDefault() ?? categories.First( c => c.Type == type && c.Name == "其他" && c.UserId == null); } }