524 lines
21 KiB
C#
524 lines
21 KiB
C#
using System.Security.Claims;
|
|
using System.Text.Json;
|
|
using MiaoJiZhang.Api.Contracts;
|
|
using MiaoJiZhang.Api.Services;
|
|
using MiaoJiZhang.Domain.Entities;
|
|
using MiaoJiZhang.Domain.Enums;
|
|
using MiaoJiZhang.Infrastructure.Persistence;
|
|
using Microsoft.AspNetCore.Authorization;
|
|
using Microsoft.AspNetCore.Mvc;
|
|
using Microsoft.AspNetCore.RateLimiting;
|
|
using Microsoft.EntityFrameworkCore;
|
|
|
|
namespace MiaoJiZhang.Api.Controllers;
|
|
|
|
/// <summary>
|
|
/// OCR/语音/图片辅助解析。客户端确认后再调用交易接口正式入账。
|
|
/// 图片模式由后端多模态模型提取一笔或多笔账单草稿。
|
|
/// </summary>
|
|
[ApiController]
|
|
[Authorize]
|
|
[Route("api/parse")]
|
|
[RequireAiPermission]
|
|
public class ParseController(AppDbContext db, ILlmClient llm, AgentService agent) : ControllerBase
|
|
{
|
|
private long Uid => long.Parse(
|
|
User.FindFirstValue(ClaimTypes.NameIdentifier) ??
|
|
User.FindFirstValue("sub")!);
|
|
|
|
[HttpPost]
|
|
[EnableRateLimiting("ai")]
|
|
public async Task<ActionResult<OcrParseResponse>> Parse(OcrParseRequest req)
|
|
{
|
|
return await ParseWithAgent(req, HttpContext.RequestAborted);
|
|
}
|
|
|
|
private async Task<ActionResult<OcrParseResponse>> ParseWithAgent(
|
|
OcrParseRequest req,
|
|
CancellationToken ct)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(req.Text))
|
|
return BadRequest(new ApiError("TEXT_EMPTY", "内容不能为空"));
|
|
var featureKey = req.Source == "voice"
|
|
? "feature.voice_enabled"
|
|
: "feature.ocr_enabled";
|
|
if (!await FeatureEnabled(featureKey))
|
|
return StatusCode(403, new ApiError("FEATURE_DISABLED", "该解析功能已由后台关闭"));
|
|
|
|
IReadOnlyList<ParsedBill> drafts;
|
|
try
|
|
{
|
|
drafts = await agent.ParseDraftsAsync(Uid, req.Text.Trim(), ct);
|
|
}
|
|
catch (InvalidOperationException ex)
|
|
{
|
|
return StatusCode(
|
|
503,
|
|
new ApiError("AGENT_PARSE_UNAVAILABLE", ex.Message));
|
|
}
|
|
catch (Exception)
|
|
{
|
|
return StatusCode(
|
|
502,
|
|
new ApiError(
|
|
"AGENT_PARSE_FAILED",
|
|
"AI 暂时无法解析，没有生成账单草稿"));
|
|
}
|
|
|
|
if (drafts.Count == 0)
|
|
{
|
|
var fallback = await db.Categories.FirstAsync(
|
|
c => c.Name == "其他" &&
|
|
c.Type == TransactionType.Expense &&
|
|
c.UserId == null,
|
|
ct);
|
|
return Ok(new OcrParseResponse(
|
|
false,
|
|
fallback.Id,
|
|
fallback.Name,
|
|
fallback.IconKey,
|
|
0,
|
|
null,
|
|
req.Text,
|
|
"expense"));
|
|
}
|
|
|
|
var bill = drafts[0];
|
|
var category = await db.Categories.FirstAsync(
|
|
c => c.Id == bill.CategoryId &&
|
|
c.Type == bill.Type.CategoryType(bill.TransferDirection),
|
|
ct);
|
|
return Ok(new OcrParseResponse(
|
|
true,
|
|
category.Id,
|
|
category.Name,
|
|
category.IconKey,
|
|
bill.Amount,
|
|
bill.PaymentMethod,
|
|
bill.Note,
|
|
bill.Type.ToWire(),
|
|
bill.TransferDirection.ToWire(),
|
|
bill.Counterparty));
|
|
}
|
|
|
|
/// <summary>上传截屏或小票，提取其中全部独立交易。</summary>
|
|
[HttpPost("image")]
|
|
[EnableRateLimiting("upload")]
|
|
[RequestSizeLimit(10 * 1024 * 1024)]
|
|
[RequestFormLimits(MultipartBodyLengthLimit = 10 * 1024 * 1024)]
|
|
public async Task<ActionResult<ImageParseResponse>> ParseImage(
|
|
IFormFile file,
|
|
[FromQuery] string source = "image",
|
|
CancellationToken ct = default)
|
|
{
|
|
var featureKey = source == "screenshot"
|
|
? "feature.screenshot_bookkeeping_enabled"
|
|
: "feature.ocr_enabled";
|
|
if (!await FeatureEnabled(featureKey))
|
|
return StatusCode(403, new ApiError("FEATURE_DISABLED", "该图片识别功能已由后台关闭"));
|
|
if (file is null || file.Length == 0)
|
|
return BadRequest(new ApiError("FILE_EMPTY", "请上传图片"));
|
|
if (!file.ContentType.StartsWith("image/"))
|
|
return BadRequest(new ApiError("NOT_IMAGE", "仅支持图片文件"));
|
|
if (file.Length > 10 * 1024 * 1024)
|
|
return BadRequest(new ApiError("FILE_TOO_LARGE", "图片不能超过 10MB"));
|
|
|
|
if (!llm.IsEnabled)
|
|
{
|
|
return StatusCode(
|
|
503,
|
|
new ApiError(
|
|
"LLM_NOT_CONFIGURED",
|
|
"AI 图片解析未配置，请在管理后台设置 LLM API Key"));
|
|
}
|
|
|
|
using var stream = new MemoryStream();
|
|
await file.CopyToAsync(stream, ct);
|
|
|
|
IReadOnlyList<ImageParseResult>? results;
|
|
try
|
|
{
|
|
results = await llm.AnalyzeImageAsync(
|
|
stream.ToArray(),
|
|
file.ContentType,
|
|
ct);
|
|
}
|
|
catch (InvalidOperationException ex)
|
|
{
|
|
return StatusCode(502, new ApiError("LLM_IMAGE_ERROR", ex.Message));
|
|
}
|
|
|
|
if (results is null)
|
|
return StatusCode(500, new ApiError("LLM_ERROR", "AI 解析失败，请稍后重试"));
|
|
|
|
var categories = await db.Categories
|
|
.Where(c => !c.IsDeleted && (c.UserId == null || c.UserId == Uid))
|
|
.ToListAsync(ct);
|
|
|
|
if (results.Count == 0)
|
|
{
|
|
return Ok(new ImageParseResponse(
|
|
false,
|
|
0,
|
|
"待确认",
|
|
"tag",
|
|
0,
|
|
null,
|
|
"",
|
|
"unknown",
|
|
[],
|
|
null));
|
|
}
|
|
|
|
var items = new List<ImageParseItemResponse>();
|
|
foreach (var result in results.Take(20))
|
|
{
|
|
var normalizedType = result.Type.Trim().ToLowerInvariant();
|
|
var transferDirection = result.TransferDirection is "in" or "out"
|
|
? result.TransferDirection
|
|
: null;
|
|
if (normalizedType is not ("income" or "expense" or "transfer") ||
|
|
normalizedType == "transfer" && transferDirection is null)
|
|
{
|
|
items.Add(new ImageParseItemResponse(
|
|
false, 0, "待确认", "tag", "unknown",
|
|
result.Amount, result.PaymentMethod, result.Note,
|
|
result.OccurredAt,
|
|
transferDirection,
|
|
result.Counterparty));
|
|
continue;
|
|
}
|
|
var categoryType = normalizedType == "income" || transferDirection == "in"
|
|
? TransactionType.Income
|
|
: TransactionType.Expense;
|
|
var category = FindCategory(categories, categoryType, result.CategoryName);
|
|
items.Add(new ImageParseItemResponse(
|
|
true,
|
|
category.Id,
|
|
category.Name,
|
|
category.IconKey,
|
|
normalizedType,
|
|
result.Amount,
|
|
result.PaymentMethod,
|
|
result.Note,
|
|
result.OccurredAt,
|
|
transferDirection,
|
|
result.Counterparty));
|
|
}
|
|
|
|
var first = items[0];
|
|
return Ok(new ImageParseResponse(
|
|
true,
|
|
first.CategoryId,
|
|
first.CategoryName,
|
|
first.CategoryIcon,
|
|
first.Amount,
|
|
first.PaymentMethod,
|
|
first.Note,
|
|
first.Type,
|
|
items,
|
|
first.OccurredAt,
|
|
first.TransferDirection,
|
|
first.Counterparty));
|
|
}
|
|
|
|
[HttpPost("recognition-batch")]
|
|
[EnableRateLimiting("upload")]
|
|
[RequestSizeLimit(10 * 1024 * 1024)]
|
|
[RequestFormLimits(MultipartBodyLengthLimit = 10 * 1024 * 1024)]
|
|
public async Task<ActionResult<RecognitionBatchResponse>> ReconcileRecognitionBatch(
|
|
CancellationToken ct = default)
|
|
{
|
|
if (!await FeatureEnabled("feature.screenshot_bookkeeping_enabled"))
|
|
return StatusCode(403, new ApiError("FEATURE_DISABLED", "AI 截图补全已由后台关闭"));
|
|
if (!llm.IsEnabled)
|
|
return StatusCode(503, new ApiError("LLM_NOT_CONFIGURED", "AI 图片解析未配置"));
|
|
|
|
var form = await Request.ReadFormAsync(ct);
|
|
var manifestText = form["manifest"].FirstOrDefault();
|
|
if (string.IsNullOrWhiteSpace(manifestText))
|
|
return BadRequest(new ApiError("BATCH_MANIFEST_EMPTY", "批次清单不能为空"));
|
|
|
|
RecognitionBatchManifestRequest? manifest;
|
|
try
|
|
{
|
|
manifest = JsonSerializer.Deserialize<RecognitionBatchManifestRequest>(
|
|
manifestText,
|
|
new JsonSerializerOptions(JsonSerializerDefaults.Web));
|
|
}
|
|
catch (JsonException)
|
|
{
|
|
return BadRequest(new ApiError("BATCH_MANIFEST_INVALID", "批次清单格式无效"));
|
|
}
|
|
if (manifest is null || !Guid.TryParse(manifest.BatchId, out _) ||
|
|
manifest.Candidates.Count > 10 || manifest.Evidence.Count > 10 ||
|
|
manifest.Candidates.Count + manifest.Evidence.Count == 0)
|
|
{
|
|
return BadRequest(new ApiError("BATCH_MANIFEST_INVALID", "批次数量或标识无效"));
|
|
}
|
|
if (manifest.Candidates.Select(item => item.CandidateId).Distinct().Count() !=
|
|
manifest.Candidates.Count ||
|
|
manifest.Evidence.Select(item => item.EvidenceId).Distinct().Count() !=
|
|
manifest.Evidence.Count)
|
|
{
|
|
return BadRequest(new ApiError("BATCH_ID_DUPLICATED", "批次中存在重复标识"));
|
|
}
|
|
|
|
var evidenceById = manifest.Evidence.ToDictionary(item => item.EvidenceId);
|
|
var candidateIds = manifest.Candidates.Select(item => item.CandidateId).ToHashSet();
|
|
if (manifest.Evidence.Any(item => item.CandidateId is not null &&
|
|
!candidateIds.Contains(item.CandidateId)) ||
|
|
manifest.Candidates.Any(candidate =>
|
|
(candidate.EvidenceIds ?? []).Any(evidenceId =>
|
|
!evidenceById.TryGetValue(evidenceId, out var evidence) ||
|
|
evidence.CandidateId != candidate.CandidateId)))
|
|
{
|
|
return BadRequest(new ApiError("BATCH_EVIDENCE_INVALID", "截图与候选关联无效"));
|
|
}
|
|
var imageInputs = new List<RecognitionBatchModelEvidence>();
|
|
var uploadedEvidenceIds = new HashSet<string>();
|
|
long totalBytes = 0;
|
|
foreach (var file in form.Files)
|
|
{
|
|
var evidenceId = Path.GetFileNameWithoutExtension(file.FileName);
|
|
if (!evidenceById.TryGetValue(evidenceId, out var evidence) ||
|
|
!uploadedEvidenceIds.Add(evidenceId))
|
|
return BadRequest(new ApiError("BATCH_EVIDENCE_UNKNOWN", "截图与批次清单不匹配"));
|
|
if (!file.ContentType.StartsWith("image/") || file.Length <= 0 || file.Length > 1024 * 1024)
|
|
return BadRequest(new ApiError("BATCH_EVIDENCE_INVALID", "单张截图必须是 1MB 以内的图片"));
|
|
totalBytes += file.Length;
|
|
if (totalBytes > 9 * 1024 * 1024)
|
|
return BadRequest(new ApiError("BATCH_TOO_LARGE", "批次截图总大小不能超过 9MB"));
|
|
using var stream = new MemoryStream();
|
|
await file.CopyToAsync(stream, ct);
|
|
imageInputs.Add(new RecognitionBatchModelEvidence(
|
|
evidence.EvidenceId,
|
|
evidence.CandidateId,
|
|
evidence.FlowSessionId,
|
|
evidence.PackageName,
|
|
evidence.CapturedAt,
|
|
stream.ToArray(),
|
|
file.ContentType));
|
|
}
|
|
if (uploadedEvidenceIds.Count != evidenceById.Count)
|
|
return BadRequest(new ApiError("BATCH_EVIDENCE_MISSING", "批次截图上传不完整"));
|
|
|
|
var categories = await db.Categories
|
|
.Where(category => !category.IsDeleted &&
|
|
(category.UserId == null || category.UserId == Uid))
|
|
.ToListAsync(ct);
|
|
var candidates = manifest.Candidates.Select(candidate =>
|
|
new RecognitionBatchModelCandidate(
|
|
candidate.CandidateId,
|
|
candidate.FlowSessionId,
|
|
candidate.PackageName,
|
|
candidate.Type,
|
|
candidate.Amount,
|
|
candidate.Merchant,
|
|
candidate.OrderId,
|
|
candidate.OccurredAt,
|
|
candidate.RecognitionKind,
|
|
candidate.CategoryHint,
|
|
candidate.Confidence,
|
|
candidate.EvidenceIds ?? [],
|
|
candidate.TransferDirection,
|
|
candidate.Counterparty,
|
|
candidate.ProviderTransactionId,
|
|
candidate.RecognitionOccurrenceId,
|
|
candidate.IdentityConfidence)).ToList();
|
|
|
|
IReadOnlyList<RecognitionBatchModelAction>? modelActions;
|
|
try
|
|
{
|
|
modelActions = await llm.ReconcileRecognitionBatchAsync(
|
|
new RecognitionBatchModelInput(
|
|
manifest.BatchId,
|
|
candidates,
|
|
imageInputs,
|
|
categories.Where(category => category.Type == TransactionType.Expense)
|
|
.Select(category => category.Name).Distinct().ToList(),
|
|
categories.Where(category => category.Type == TransactionType.Income)
|
|
.Select(category => category.Name).Distinct().ToList()),
|
|
ct);
|
|
}
|
|
catch (InvalidOperationException ex)
|
|
{
|
|
return StatusCode(502, new ApiError("BATCH_AI_FAILED", ex.Message));
|
|
}
|
|
finally
|
|
{
|
|
foreach (var evidence in imageInputs)
|
|
Array.Clear(evidence.ImageBytes);
|
|
}
|
|
if (modelActions is null)
|
|
return StatusCode(502, new ApiError("BATCH_AI_FAILED", "AI 未返回批次对账结果"));
|
|
|
|
var actions = ValidateBatchActions(manifest, modelActions, categories);
|
|
return Ok(new RecognitionBatchResponse(manifest.BatchId, actions));
|
|
}
|
|
|
|
private List<RecognitionBatchActionResponse> ValidateBatchActions(
|
|
RecognitionBatchManifestRequest manifest,
|
|
IReadOnlyList<RecognitionBatchModelAction> modelActions,
|
|
List<Category> categories)
|
|
{
|
|
var candidateIds = manifest.Candidates.Select(item => item.CandidateId).ToHashSet();
|
|
var evidenceById = manifest.Evidence.ToDictionary(item => item.EvidenceId);
|
|
var evidenceOwners = evidenceById.ToDictionary(
|
|
entry => entry.Key,
|
|
entry => entry.Value.CandidateId);
|
|
var selected = modelActions
|
|
.Where(action => action.CandidateId != null && candidateIds.Contains(action.CandidateId))
|
|
.GroupBy(action => action.CandidateId!)
|
|
.ToDictionary(group => group.Key, group => group.First());
|
|
var result = new List<RecognitionBatchActionResponse>();
|
|
|
|
foreach (var candidate in manifest.Candidates)
|
|
{
|
|
selected.TryGetValue(candidate.CandidateId, out var model);
|
|
var action = model?.Action is "update" or "drop" ? model.Action : "keep";
|
|
var reason = model?.Reason ?? "保留本地识别结果";
|
|
var confidence = model?.Confidence ?? 1;
|
|
if (action == "drop" && (confidence < 0.9 ||
|
|
string.IsNullOrWhiteSpace(model?.EvidenceId) ||
|
|
!evidenceById.TryGetValue(model.EvidenceId, out var dropEvidence) ||
|
|
dropEvidence.CandidateId != candidate.CandidateId))
|
|
{
|
|
action = "keep";
|
|
reason = "撤销证据不足，已保留本地结果";
|
|
}
|
|
var type = model?.Type is "income" or "expense" or "transfer"
|
|
? model.Type
|
|
: candidate.Type;
|
|
var transferDirection = type == "transfer"
|
|
? model?.TransferDirection is "in" or "out"
|
|
? model.TransferDirection
|
|
: candidate.TransferDirection is "in" or "out"
|
|
? candidate.TransferDirection
|
|
: null
|
|
: null;
|
|
if (type == "transfer" && transferDirection is null)
|
|
{
|
|
action = "keep";
|
|
type = candidate.Type;
|
|
transferDirection = candidate.TransferDirection;
|
|
reason = "转账方向不明确，已保留本地结果";
|
|
}
|
|
var proposedAmount = model?.Amount is > 0 ? model.Amount.Value : candidate.Amount;
|
|
var amountChanged = proposedAmount != candidate.Amount;
|
|
var amountUpdateAllowed = !amountChanged ||
|
|
model?.Action == "update" && CanApplyAmountUpdate(
|
|
candidate.CandidateId,
|
|
model.EvidenceId,
|
|
model.Confidence,
|
|
evidenceOwners);
|
|
var amount = amountUpdateAllowed ? proposedAmount : candidate.Amount;
|
|
if (amountChanged && !amountUpdateAllowed)
|
|
{
|
|
reason = "金额修改证据不足，已保留本地金额";
|
|
}
|
|
var category = FindCategory(
|
|
categories,
|
|
type == "income" || transferDirection == "in"
|
|
? TransactionType.Income
|
|
: TransactionType.Expense,
|
|
model?.CategoryName ?? candidate.CategoryHint ?? "其他");
|
|
result.Add(new RecognitionBatchActionResponse(
|
|
action,
|
|
model?.ActionId ?? $"keep-{candidate.CandidateId}",
|
|
candidate.CandidateId,
|
|
model?.EvidenceId,
|
|
category.Id,
|
|
category.Name,
|
|
type,
|
|
amount,
|
|
model?.PaymentMethod,
|
|
string.IsNullOrWhiteSpace(model?.Note) ? candidate.Merchant : model.Note,
|
|
model?.OccurredAt ?? candidate.OccurredAt,
|
|
confidence,
|
|
reason,
|
|
transferDirection,
|
|
string.IsNullOrWhiteSpace(model?.Counterparty)
|
|
? candidate.Counterparty
|
|
: model.Counterparty));
|
|
}
|
|
|
|
var usedEvidence = new HashSet<string>();
|
|
foreach (var model in modelActions.Where(action => action.Action == "create"))
|
|
{
|
|
if (string.IsNullOrWhiteSpace(model.EvidenceId) ||
|
|
!evidenceById.TryGetValue(model.EvidenceId, out var evidence) ||
|
|
!string.IsNullOrWhiteSpace(evidence.CandidateId) ||
|
|
!usedEvidence.Add(model.EvidenceId) ||
|
|
model.Type is not ("income" or "expense" or "transfer") ||
|
|
model.Type == "transfer" && model.TransferDirection is not ("in" or "out") ||
|
|
model.Amount is not > 0)
|
|
{
|
|
continue;
|
|
}
|
|
var categoryType = model.Type == "income" || model.TransferDirection == "in"
|
|
? TransactionType.Income
|
|
: TransactionType.Expense;
|
|
var category = FindCategory(categories, categoryType, model.CategoryName ?? "其他");
|
|
result.Add(new RecognitionBatchActionResponse(
|
|
"create",
|
|
model.ActionId,
|
|
null,
|
|
model.EvidenceId,
|
|
category.Id,
|
|
category.Name,
|
|
model.Type,
|
|
model.Amount,
|
|
model.PaymentMethod,
|
|
model.Note,
|
|
model.OccurredAt ?? evidence.CapturedAt,
|
|
model.Confidence,
|
|
model.Reason,
|
|
model.TransferDirection,
|
|
model.Counterparty));
|
|
}
|
|
return result.Take(20).ToList();
|
|
}
|
|
|
|
internal static bool CanApplyAmountUpdate(
|
|
string candidateId,
|
|
string? evidenceId,
|
|
double confidence,
|
|
IReadOnlyDictionary<string, string?> evidenceOwners)
|
|
{
|
|
return confidence >= 0.9 &&
|
|
!string.IsNullOrWhiteSpace(evidenceId) &&
|
|
evidenceOwners.TryGetValue(evidenceId, out var owner) &&
|
|
owner == candidateId;
|
|
}
|
|
|
|
private async Task<bool> FeatureEnabled(string key)
|
|
{
|
|
var value = await db.AppConfigs
|
|
.Where(config => config.Key == key)
|
|
.Select(config => config.Value)
|
|
.FirstOrDefaultAsync();
|
|
return value is null || !value.Equals("false", StringComparison.OrdinalIgnoreCase);
|
|
}
|
|
|
|
private Category FindCategory(
|
|
List<Category> categories,
|
|
TransactionType type,
|
|
string requestedName)
|
|
{
|
|
if (type == TransactionType.Income &&
|
|
requestedName is "退款" or "返现" or "退回")
|
|
{
|
|
requestedName = "报销";
|
|
}
|
|
return categories
|
|
.Where(c => c.Type == type && c.Name == requestedName)
|
|
.OrderByDescending(c => c.UserId == Uid)
|
|
.FirstOrDefault()
|
|
?? categories.First(
|
|
c => c.Type == type &&
|
|
c.Name == "其他" &&
|
|
c.UserId == null);
|
|
}
|
|
}
|