Add AI batch reconciliation for accessibility bills

This commit is contained in:
2026-07-25 18:41:09 +08:00
parent eb8909a192
commit 7cca34b331
25 changed files with 2771 additions and 230 deletions
@@ -353,7 +353,7 @@ public sealed class ApiIntegrationTests(ApiFixture fixture)
}
[Fact]
public async Task RecognitionClientRequestId_IsIdempotentAcrossConcurrentChannels()
public async Task RecognitionClientRequestId_IsIdempotentAcrossConcurrentChannels()
{
using var client = await fixture.RegisterAsync("recognition_idempotency");
var ledgers = await client.GetFromJsonAsync<JsonElement>("/api/ledgers");
@@ -389,9 +389,89 @@ public sealed class ApiIntegrationTests(ApiFixture fixture)
var month = await client.GetFromJsonAsync<JsonElement>(
$"/api/transactions/month?year=2026&month=7&ledgerId={ledgerId}");
Assert.Equal(1, month.GetProperty("count").GetInt32());
}
}
Assert.Equal(1, month.GetProperty("count").GetInt32());
}
[Fact]
public async Task RecognitionBatch_PreservesThreeConsecutiveTransfers_AndIsIdempotent()
{
using var client = await fixture.RegisterAsync("recognition_batch_three_transfers");
var ledgers = await client.GetFromJsonAsync<JsonElement>("/api/ledgers");
var ledgerId = ledgers[0].GetProperty("id").GetInt64();
var categories = await client.GetFromJsonAsync<JsonElement>(
"/api/categories?type=expense");
var categoryId = categories[0].GetProperty("id").GetInt64();
var batchId = Guid.NewGuid().ToString();
var payload = new
{
batchId,
ledgerId,
items = new[]
{
new
{
candidateId = "transfer-1",
clientRequestId = "recognition-wechat-flow-1",
categoryId,
type = "expense",
amount = 20m,
note = "转账给张三",
paymentMethod = "微信",
occurredAt = "2026-07-25T10:00:01+08:00",
source = "recognition_ai",
sourceText = "微信转账成功",
},
new
{
candidateId = "transfer-2",
clientRequestId = "recognition-wechat-flow-2",
categoryId,
type = "expense",
amount = 20m,
note = "转账给张三",
paymentMethod = "微信",
occurredAt = "2026-07-25T10:00:10+08:00",
source = "recognition_ai",
sourceText = "微信转账成功",
},
new
{
candidateId = "transfer-3",
clientRequestId = "recognition-wechat-flow-3",
categoryId,
type = "expense",
amount = 30m,
note = "转账给张三",
paymentMethod = "微信",
occurredAt = "2026-07-25T10:00:20+08:00",
source = "recognition_ai",
sourceText = "微信转账成功",
},
},
};
var responses = await Task.WhenAll(
client.PostAsJsonAsync("/api/transactions/recognition-batch", payload),
client.PostAsJsonAsync("/api/transactions/recognition-batch", payload));
Assert.All(responses, response => response.EnsureSuccessStatusCode());
var results = await Task.WhenAll(
responses.Select(response => response.Content.ReadFromJsonAsync<JsonElement>()));
var first = results[0];
var retry = results[1];
Assert.Equal(3, first.GetArrayLength());
Assert.Equal(3, retry.GetArrayLength());
Assert.Equal(
first.EnumerateArray()
.Select(item => item.GetProperty("transaction").GetProperty("id").GetInt64()),
retry.EnumerateArray()
.Select(item => item.GetProperty("transaction").GetProperty("id").GetInt64()));
var month = await client.GetFromJsonAsync<JsonElement>(
$"/api/transactions/month?year=2026&month=7&ledgerId={ledgerId}");
Assert.Equal(3, month.GetProperty("count").GetInt32());
Assert.Equal(70m, month.GetProperty("expense").GetDecimal());
}
}
public sealed class ChinaClockTests
{
@@ -427,7 +507,7 @@ public sealed class ChinaClockTests
yearEnd);
}
}
public sealed class ImageParseResultTests
public sealed class ImageParseResultTests
{
private static readonly MethodInfo ParseMethod =
typeof(OpenAiVisionClient).GetMethod(
@@ -479,7 +559,52 @@ public sealed class ImageParseResultTests
results[1].OccurredAt);
Assert.Null(results[2].OccurredAt);
}
}
}
public sealed class RecognitionBatchActionParserTests
{
private static readonly MethodInfo ParseMethod =
typeof(OpenAiVisionClient).GetMethod(
"ParseRecognitionBatchActions",
BindingFlags.NonPublic | BindingFlags.Static)
?? throw new InvalidOperationException("Recognition batch parser not found");
[Fact]
public void Actions_AreParsedFromFencedJson_AndInvalidActionsAreIgnored()
{
const string payload =
"""
result:
```json
{
"actions": [
{
"action": "update",
"actionId": "a1",
"candidateId": "candidate-1",
"amount": 20,
"confidence": 1.4,
"reason": "修正金额"
},
{
"action": "merge",
"candidateId": "candidate-2"
}
]
}
```
""";
var actions = Assert.IsAssignableFrom<IReadOnlyList<RecognitionBatchModelAction>>(
ParseMethod.Invoke(null, [payload]));
var action = Assert.Single(actions);
Assert.Equal("update", action.Action);
Assert.Equal("candidate-1", action.CandidateId);
Assert.Equal(20m, action.Amount);
Assert.Equal(1, action.Confidence);
}
}
public sealed class BudgetRecommendationValidationTests
{
@@ -138,10 +138,78 @@ public record ImageParseResponse(
decimal Amount, string? PaymentMethod, string Note, string Type,
List<ImageParseItemResponse> Items, DateTime? OccurredAt);
public record RecognitionBatchCandidateRequest(
string CandidateId,
string ClientRequestId,
string? FlowSessionId,
string PackageName,
string Type,
decimal Amount,
string? Merchant,
string? OrderId,
DateTime OccurredAt,
string RecognitionKind,
string? CategoryHint,
string Confidence,
string? SourceText,
List<string>? EvidenceIds = null);
public record RecognitionBatchEvidenceRequest(
string EvidenceId,
string? CandidateId,
string? FlowSessionId,
string PackageName,
DateTime CapturedAt);
public record RecognitionBatchManifestRequest(
string BatchId,
DateTime OpenedAt,
List<RecognitionBatchCandidateRequest> Candidates,
List<RecognitionBatchEvidenceRequest> Evidence);
public record RecognitionBatchActionResponse(
string Action,
string ActionId,
string? CandidateId,
string? EvidenceId,
long? CategoryId,
string? CategoryName,
string? Type,
decimal? Amount,
string? PaymentMethod,
string? Note,
DateTime? OccurredAt,
double Confidence,
string Reason);
public record RecognitionBatchResponse(
string BatchId,
List<RecognitionBatchActionResponse> Actions);
public record RecognitionBatchTransactionItemRequest(
string CandidateId,
string ClientRequestId,
long CategoryId,
string Type,
decimal Amount,
string? Note,
string? PaymentMethod,
DateTime OccurredAt,
string? Source,
string? SourceText);
public record CreateRecognitionBatchRequest(
string BatchId,
long? LedgerId,
List<RecognitionBatchTransactionItemRequest> Items);
public record RecognitionBatchTransactionDto(
string CandidateId,
TransactionDto Transaction);
// ---- Chat ----
public record SendChatRequest(string Content, string Type = "text", long? LedgerId = null); // text | sticker
public record ChatMessageDto(
long Id, string Role, string Type, string Content,
TransactionDto? Transaction, DateTime CreatedAt);
public record SendChatResponse(List<ChatMessageDto> Messages);
@@ -1,4 +1,5 @@
using System.Security.Claims;
using System.Security.Claims;
using System.Text.Json;
using MiaoJiZhang.Api.Contracts;
using MiaoJiZhang.Api.Services;
using MiaoJiZhang.Domain.Entities;
@@ -102,7 +103,7 @@ public class ParseController(AppDbContext db, ILlmClient llm, AgentService agent
[EnableRateLimiting("upload")]
[RequestSizeLimit(10 * 1024 * 1024)]
[RequestFormLimits(MultipartBodyLengthLimit = 10 * 1024 * 1024)]
public async Task<ActionResult<ImageParseResponse>> ParseImage(
public async Task<ActionResult<ImageParseResponse>> ParseImage(
IFormFile file,
[FromQuery] string source = "image",
CancellationToken ct = default)
@@ -195,7 +196,7 @@ public class ParseController(AppDbContext db, ILlmClient llm, AgentService agent
}
var first = items[0];
return Ok(new ImageParseResponse(
return Ok(new ImageParseResponse(
true,
first.CategoryId,
first.CategoryName,
@@ -205,8 +206,219 @@ public class ParseController(AppDbContext db, ILlmClient llm, AgentService agent
first.Note,
first.Type,
items,
first.OccurredAt));
}
first.OccurredAt));
}
[HttpPost("recognition-batch")]
[EnableRateLimiting("upload")]
[RequestSizeLimit(10 * 1024 * 1024)]
[RequestFormLimits(MultipartBodyLengthLimit = 10 * 1024 * 1024)]
public async Task<ActionResult<RecognitionBatchResponse>> ReconcileRecognitionBatch(
CancellationToken ct = default)
{
if (!await FeatureEnabled("feature.screenshot_bookkeeping_enabled"))
return StatusCode(403, new ApiError("FEATURE_DISABLED", "AI 截图补全已由后台关闭"));
if (!llm.IsEnabled)
return StatusCode(503, new ApiError("LLM_NOT_CONFIGURED", "AI 图片解析未配置"));
var form = await Request.ReadFormAsync(ct);
var manifestText = form["manifest"].FirstOrDefault();
if (string.IsNullOrWhiteSpace(manifestText))
return BadRequest(new ApiError("BATCH_MANIFEST_EMPTY", "批次清单不能为空"));
RecognitionBatchManifestRequest? manifest;
try
{
manifest = JsonSerializer.Deserialize<RecognitionBatchManifestRequest>(
manifestText,
new JsonSerializerOptions(JsonSerializerDefaults.Web));
}
catch (JsonException)
{
return BadRequest(new ApiError("BATCH_MANIFEST_INVALID", "批次清单格式无效"));
}
if (manifest is null || !Guid.TryParse(manifest.BatchId, out _) ||
manifest.Candidates.Count > 10 || manifest.Evidence.Count > 10 ||
manifest.Candidates.Count + manifest.Evidence.Count == 0)
{
return BadRequest(new ApiError("BATCH_MANIFEST_INVALID", "批次数量或标识无效"));
}
if (manifest.Candidates.Select(item => item.CandidateId).Distinct().Count() !=
manifest.Candidates.Count ||
manifest.Evidence.Select(item => item.EvidenceId).Distinct().Count() !=
manifest.Evidence.Count)
{
return BadRequest(new ApiError("BATCH_ID_DUPLICATED", "批次中存在重复标识"));
}
var evidenceById = manifest.Evidence.ToDictionary(item => item.EvidenceId);
var candidateIds = manifest.Candidates.Select(item => item.CandidateId).ToHashSet();
if (manifest.Evidence.Any(item => item.CandidateId is not null &&
!candidateIds.Contains(item.CandidateId)) ||
manifest.Candidates.Any(candidate =>
(candidate.EvidenceIds ?? []).Any(evidenceId =>
!evidenceById.TryGetValue(evidenceId, out var evidence) ||
evidence.CandidateId != candidate.CandidateId)))
{
return BadRequest(new ApiError("BATCH_EVIDENCE_INVALID", "截图与候选关联无效"));
}
var imageInputs = new List<RecognitionBatchModelEvidence>();
var uploadedEvidenceIds = new HashSet<string>();
long totalBytes = 0;
foreach (var file in form.Files)
{
var evidenceId = Path.GetFileNameWithoutExtension(file.FileName);
if (!evidenceById.TryGetValue(evidenceId, out var evidence) ||
!uploadedEvidenceIds.Add(evidenceId))
return BadRequest(new ApiError("BATCH_EVIDENCE_UNKNOWN", "截图与批次清单不匹配"));
if (!file.ContentType.StartsWith("image/") || file.Length <= 0 || file.Length > 1024 * 1024)
return BadRequest(new ApiError("BATCH_EVIDENCE_INVALID", "单张截图必须是 1MB 以内的图片"));
totalBytes += file.Length;
if (totalBytes > 9 * 1024 * 1024)
return BadRequest(new ApiError("BATCH_TOO_LARGE", "批次截图总大小不能超过 9MB"));
using var stream = new MemoryStream();
await file.CopyToAsync(stream, ct);
imageInputs.Add(new RecognitionBatchModelEvidence(
evidence.EvidenceId,
evidence.CandidateId,
evidence.FlowSessionId,
evidence.PackageName,
evidence.CapturedAt,
stream.ToArray(),
file.ContentType));
}
if (uploadedEvidenceIds.Count != evidenceById.Count)
return BadRequest(new ApiError("BATCH_EVIDENCE_MISSING", "批次截图上传不完整"));
var categories = await db.Categories
.Where(category => !category.IsDeleted &&
(category.UserId == null || category.UserId == Uid))
.ToListAsync(ct);
var candidates = manifest.Candidates.Select(candidate =>
new RecognitionBatchModelCandidate(
candidate.CandidateId,
candidate.FlowSessionId,
candidate.PackageName,
candidate.Type,
candidate.Amount,
candidate.Merchant,
candidate.OrderId,
candidate.OccurredAt,
candidate.RecognitionKind,
candidate.CategoryHint,
candidate.Confidence,
candidate.EvidenceIds ?? [])).ToList();
IReadOnlyList<RecognitionBatchModelAction>? modelActions;
try
{
modelActions = await llm.ReconcileRecognitionBatchAsync(
new RecognitionBatchModelInput(
manifest.BatchId,
candidates,
imageInputs,
categories.Where(category => category.Type == TransactionType.Expense)
.Select(category => category.Name).Distinct().ToList(),
categories.Where(category => category.Type == TransactionType.Income)
.Select(category => category.Name).Distinct().ToList()),
ct);
}
catch (InvalidOperationException ex)
{
return StatusCode(502, new ApiError("BATCH_AI_FAILED", ex.Message));
}
finally
{
foreach (var evidence in imageInputs)
Array.Clear(evidence.ImageBytes);
}
if (modelActions is null)
return StatusCode(502, new ApiError("BATCH_AI_FAILED", "AI 未返回批次对账结果"));
var actions = ValidateBatchActions(manifest, modelActions, categories);
return Ok(new RecognitionBatchResponse(manifest.BatchId, actions));
}
private List<RecognitionBatchActionResponse> ValidateBatchActions(
RecognitionBatchManifestRequest manifest,
IReadOnlyList<RecognitionBatchModelAction> modelActions,
List<Category> categories)
{
var candidateIds = manifest.Candidates.Select(item => item.CandidateId).ToHashSet();
var evidenceById = manifest.Evidence.ToDictionary(item => item.EvidenceId);
var selected = modelActions
.Where(action => action.CandidateId != null && candidateIds.Contains(action.CandidateId))
.GroupBy(action => action.CandidateId!)
.ToDictionary(group => group.Key, group => group.First());
var result = new List<RecognitionBatchActionResponse>();
foreach (var candidate in manifest.Candidates)
{
selected.TryGetValue(candidate.CandidateId, out var model);
var action = model?.Action is "update" or "drop" ? model.Action : "keep";
var reason = model?.Reason ?? "保留本地识别结果";
var confidence = model?.Confidence ?? 1;
if (action == "drop" && (confidence < 0.9 ||
string.IsNullOrWhiteSpace(model?.EvidenceId) ||
!evidenceById.TryGetValue(model.EvidenceId, out var dropEvidence) ||
dropEvidence.CandidateId != candidate.CandidateId))
{
action = "keep";
reason = "撤销证据不足,已保留本地结果";
}
var type = model?.Type is "income" or "expense" ? model.Type : candidate.Type;
var amount = model?.Amount is > 0 ? model.Amount.Value : candidate.Amount;
var category = FindCategory(
categories,
type == "income" ? TransactionType.Income : TransactionType.Expense,
model?.CategoryName ?? candidate.CategoryHint ?? "其他");
result.Add(new RecognitionBatchActionResponse(
action,
model?.ActionId ?? $"keep-{candidate.CandidateId}",
candidate.CandidateId,
model?.EvidenceId,
category.Id,
category.Name,
type,
amount,
model?.PaymentMethod,
string.IsNullOrWhiteSpace(model?.Note) ? candidate.Merchant : model.Note,
model?.OccurredAt ?? candidate.OccurredAt,
confidence,
reason));
}
var usedEvidence = new HashSet<string>();
foreach (var model in modelActions.Where(action => action.Action == "create"))
{
if (string.IsNullOrWhiteSpace(model.EvidenceId) ||
!evidenceById.TryGetValue(model.EvidenceId, out var evidence) ||
!string.IsNullOrWhiteSpace(evidence.CandidateId) ||
!usedEvidence.Add(model.EvidenceId) ||
model.Type is not ("income" or "expense") ||
model.Amount is not > 0)
{
continue;
}
var type = model.Type == "income" ? TransactionType.Income : TransactionType.Expense;
var category = FindCategory(categories, type, model.CategoryName ?? "其他");
result.Add(new RecognitionBatchActionResponse(
"create",
model.ActionId,
null,
model.EvidenceId,
category.Id,
category.Name,
model.Type,
model.Amount,
model.PaymentMethod,
model.Note,
model.OccurredAt ?? evidence.CapturedAt,
model.Confidence,
model.Reason));
}
return result.Take(20).ToList();
}
private async Task<bool> FeatureEnabled(string key)
{
@@ -61,17 +61,7 @@ public class TransactionsController(AppDbContext db, LedgerResolver ledgers) : C
Amount = req.Amount,
Note = req.Note,
PaymentMethod = req.PaymentMethod,
Source = req.Source switch
{
"voice" => TransactionSource.Voice,
"ocr" => TransactionSource.ReceiptOcr,
"screenshot" => TransactionSource.Screenshot,
"accessibility" => TransactionSource.Accessibility,
"notification" => TransactionSource.Notification,
"recognition_ai" => TransactionSource.RecognitionAi,
"local_ocr" => TransactionSource.LocalOcr,
_ => TransactionSource.Manual,
},
Source = SourceFromWire(req.Source),
SourceText = req.SourceText,
ClientRequestId = clientRequestId,
CreatedAt = DateTime.UtcNow,
@@ -98,6 +88,107 @@ public class TransactionsController(AppDbContext db, LedgerResolver ledgers) : C
}
}
[HttpPost("recognition-batch")]
public async Task<ActionResult<List<RecognitionBatchTransactionDto>>> CreateRecognitionBatch(
CreateRecognitionBatchRequest req,
CancellationToken ct)
{
if (!Guid.TryParse(req.BatchId, out _) || req.Items.Count is < 1 or > 20)
return BadRequest(new ApiError("BATCH_INVALID", "批次标识或账单数量无效"));
if (req.Items.Select(item => item.CandidateId).Distinct().Count() != req.Items.Count ||
req.Items.Select(item => item.ClientRequestId).Distinct().Count() != req.Items.Count)
{
return BadRequest(new ApiError("BATCH_DUPLICATED", "批次中存在重复账单标识"));
}
if (req.Items.Any(item => item.Amount <= 0 ||
item.ClientRequestId.Length is < 1 or > 64 ||
ParseType(item.Type) is null))
{
return BadRequest(new ApiError("BATCH_ITEM_INVALID", "批次中存在无效账单"));
}
var ledgerId = await ledgers.ResolveAsync(Uid, req.LedgerId);
if (!ledgerId.HasValue)
return BadRequest(new ApiError("LEDGER_NOT_FOUND", "账本不存在或无权访问"));
var categoryIds = req.Items.Select(item => item.CategoryId).Distinct().ToList();
var categories = await db.Categories
.Where(category => categoryIds.Contains(category.Id) && !category.IsDeleted &&
(category.UserId == null || category.UserId == Uid))
.ToDictionaryAsync(category => category.Id, ct);
foreach (var item in req.Items)
{
var type = ParseType(item.Type)!.Value;
if (!categories.TryGetValue(item.CategoryId, out var category) || category.Type != type)
return BadRequest(new ApiError("CATEGORY_TYPE_MISMATCH", "分类与收支类型不一致"));
}
var requestIds = req.Items.Select(item => item.ClientRequestId).ToList();
var existing = await db.Transactions
.Include(transaction => transaction.Category)
.Where(transaction => transaction.UserId == Uid &&
transaction.ClientRequestId != null &&
requestIds.Contains(transaction.ClientRequestId))
.ToDictionaryAsync(transaction => transaction.ClientRequestId!, ct);
await using var transactionScope = await db.Database.BeginTransactionAsync(ct);
var mapped = new List<(string CandidateId, Transaction Transaction)>();
foreach (var item in req.Items)
{
if (existing.TryGetValue(item.ClientRequestId, out var found))
{
mapped.Add((item.CandidateId, found));
continue;
}
var category = categories[item.CategoryId];
var transaction = new Transaction
{
LedgerId = ledgerId.Value,
UserId = Uid,
CategoryId = category.Id,
Category = category,
Type = ParseType(item.Type)!.Value,
Amount = item.Amount,
Note = item.Note?.Trim(),
PaymentMethod = item.PaymentMethod?.Trim(),
OccurredAt = NormalizeOccurredAt(item.OccurredAt),
Source = SourceFromWire(item.Source),
SourceText = item.SourceText,
ClientRequestId = item.ClientRequestId,
CreatedAt = DateTime.UtcNow,
UpdatedAt = DateTime.UtcNow,
};
db.Transactions.Add(transaction);
mapped.Add((item.CandidateId, transaction));
}
try
{
await db.SaveChangesAsync(ct);
await transactionScope.CommitAsync(ct);
return Ok(mapped.Select(item => new RecognitionBatchTransactionDto(
item.CandidateId,
ToDto(item.Transaction, item.Transaction.Category))).ToList());
}
catch (DbUpdateException)
{
await transactionScope.RollbackAsync(ct);
foreach (var entry in db.ChangeTracker.Entries<Transaction>()
.Where(entry => entry.State == EntityState.Added))
{
entry.State = EntityState.Detached;
}
var raced = await db.Transactions
.AsNoTracking()
.Include(transaction => transaction.Category)
.Where(transaction => transaction.UserId == Uid &&
transaction.ClientRequestId != null &&
requestIds.Contains(transaction.ClientRequestId))
.ToDictionaryAsync(transaction => transaction.ClientRequestId!, ct);
if (raced.Count != requestIds.Count) throw;
return Ok(req.Items.Select(item => new RecognitionBatchTransactionDto(
item.CandidateId,
ToDto(raced[item.ClientRequestId], raced[item.ClientRequestId].Category))).ToList());
}
}
/// <summary>账单详情(P11:AI 来源追溯、分类、备注、时间)</summary>
[HttpGet("{id:long}")]
public async Task<ActionResult<TransactionDto>> Detail(long id)
@@ -512,6 +603,18 @@ public class TransactionsController(AppDbContext db, LedgerResolver ledgers) : C
: baseUpdatedAt.Value.ToUniversalTime();
return transaction.UpdatedAt > baseline.AddMilliseconds(1);
}
private static TransactionSource SourceFromWire(string? source) => source switch
{
"voice" => TransactionSource.Voice,
"ocr" => TransactionSource.ReceiptOcr,
"screenshot" => TransactionSource.Screenshot,
"accessibility" => TransactionSource.Accessibility,
"notification" => TransactionSource.Notification,
"recognition_ai" => TransactionSource.RecognitionAi,
"local_ocr" => TransactionSource.LocalOcr,
_ => TransactionSource.Manual,
};
private static DateTime NormalizeOccurredAt(DateTime value) =>
value.Kind == DateTimeKind.Utc ? value : ChinaClock.ToUtc(value);
+60 -8
View File
@@ -5,10 +5,13 @@ public interface ILlmClient
bool IsEnabled { get; }
Task<IntentResult?> TryParseIntentAsync(string userText, CancellationToken ct = default);
Task<string?> TryGenerateReplyAsync(string systemPrompt, string userText, CancellationToken ct = default);
Task<IReadOnlyList<ImageParseResult>?> AnalyzeImageAsync(
byte[] imageBytes,
string mimeType,
CancellationToken ct = default);
Task<IReadOnlyList<ImageParseResult>?> AnalyzeImageAsync(
byte[] imageBytes,
string mimeType,
CancellationToken ct = default);
Task<IReadOnlyList<RecognitionBatchModelAction>?> ReconcileRecognitionBatchAsync(
RecognitionBatchModelInput input,
CancellationToken ct = default);
Task<(bool Ok, string? Error)> TestConnectionAsync(CancellationToken ct = default);
Task<AgentRunResponse> RunAgentAsync(
@@ -42,13 +45,57 @@ public record AgentRunResponse(
string Text,
int ToolCallCount);
public record ImageParseResult(
public record ImageParseResult(
string Type,
decimal Amount,
string CategoryName,
string? PaymentMethod,
string Note,
DateTime? OccurredAt);
DateTime? OccurredAt);
public record RecognitionBatchModelCandidate(
string CandidateId,
string? FlowSessionId,
string PackageName,
string Type,
decimal Amount,
string? Merchant,
string? OrderId,
DateTime OccurredAt,
string RecognitionKind,
string? CategoryHint,
string Confidence,
IReadOnlyList<string> EvidenceIds);
public record RecognitionBatchModelEvidence(
string EvidenceId,
string? CandidateId,
string? FlowSessionId,
string PackageName,
DateTime CapturedAt,
byte[] ImageBytes,
string MimeType);
public record RecognitionBatchModelInput(
string BatchId,
IReadOnlyList<RecognitionBatchModelCandidate> Candidates,
IReadOnlyList<RecognitionBatchModelEvidence> Evidence,
IReadOnlyList<string> ExpenseCategories,
IReadOnlyList<string> IncomeCategories);
public record RecognitionBatchModelAction(
string Action,
string ActionId,
string? CandidateId,
string? EvidenceId,
string? Type,
decimal? Amount,
string? CategoryName,
string? PaymentMethod,
string? Note,
DateTime? OccurredAt,
double Confidence,
string Reason);
public class NullLlmClient : ILlmClient
{
@@ -65,11 +112,16 @@ public class NullLlmClient : ILlmClient
CancellationToken ct = default) =>
Task.FromResult<string?>(null);
public Task<IReadOnlyList<ImageParseResult>?> AnalyzeImageAsync(
public Task<IReadOnlyList<ImageParseResult>?> AnalyzeImageAsync(
byte[] imageBytes,
string mimeType,
CancellationToken ct = default) =>
Task.FromResult<IReadOnlyList<ImageParseResult>?>(null);
Task.FromResult<IReadOnlyList<ImageParseResult>?>(null);
public Task<IReadOnlyList<RecognitionBatchModelAction>?> ReconcileRecognitionBatchAsync(
RecognitionBatchModelInput input,
CancellationToken ct = default) =>
Task.FromResult<IReadOnlyList<RecognitionBatchModelAction>?>(null);
public Task<(bool Ok, string? Error)> TestConnectionAsync(
CancellationToken ct = default) =>
@@ -51,7 +51,7 @@ public partial class OpenAiVisionClient : ILlmClient
public async Task<string?> TryGenerateReplyAsync(string s, string u, CancellationToken ct = default)
{ if (!IsEnabled) return null; return await L(s, u, ct); }
public async Task<IReadOnlyList<ImageParseResult>?> AnalyzeImageAsync(
public async Task<IReadOnlyList<ImageParseResult>?> AnalyzeImageAsync(
byte[] img,
string mime,
CancellationToken ct = default)
@@ -157,10 +157,193 @@ public partial class OpenAiVisionClient : ILlmClient
providerRequestId,
SanitizeOutputSnippet(output));
throw new InvalidOperationException("图片模型返回格式无法解析,请重试", ex);
}
}
private static IReadOnlyList<ImageParseResult> ParseImageResults(string raw)
}
}
public async Task<IReadOnlyList<RecognitionBatchModelAction>?> ReconcileRecognitionBatchAsync(
RecognitionBatchModelInput input,
CancellationToken ct = default)
{
if (!IsEnabled) return null;
var candidateJson = JsonSerializer.Serialize(
input.Candidates.Select(candidate => new
{
candidateId = candidate.CandidateId,
flowSessionId = candidate.FlowSessionId,
packageName = candidate.PackageName,
type = candidate.Type,
amount = candidate.Amount,
merchant = candidate.Merchant,
orderId = candidate.OrderId,
occurredAt = candidate.OccurredAt,
recognitionKind = candidate.RecognitionKind,
categoryHint = candidate.CategoryHint,
confidence = candidate.Confidence,
evidenceIds = candidate.EvidenceIds,
}),
new JsonSerializerOptions(JsonSerializerDefaults.Web));
var systemPrompt = $$"""
你是支付结果批次对账器。只返回 JSON:
{"actions":[{"action":"keep|update|create|drop","actionId":"a1","candidateId":null,"evidenceId":null,"type":null,"amount":null,"categoryName":null,"paymentMethod":null,"note":null,"occurredAt":null,"confidence":0.0,"reason":""}]}
{{candidateJson}}
{{string.Join('/', input.ExpenseCategories)}}
{{string.Join('/', input.IncomeCategories)}}
keepupdate drop
flowSessionId 使
flowSessionId drop
evidenceId create evidenceId
update/create type expense incomeamount 0
沿reason 40
""";
var messages = new List<object>();
var content = new List<object>();
if (_protocol == "messages")
{
content.Add(new { type = "text", text = systemPrompt });
foreach (var evidence in input.Evidence)
{
content.Add(new { type = "text", text = $"证据 {evidence.EvidenceId}" });
content.Add(new
{
type = "image",
source = new
{
type = "base64",
media_type = evidence.MimeType,
data = Convert.ToBase64String(evidence.ImageBytes),
},
});
}
messages.Add(new { role = "user", content });
}
else if (_protocol == "responses")
{
messages.Add(new { role = "system", content = systemPrompt });
foreach (var evidence in input.Evidence)
{
content.Add(new { type = "input_text", text = $"证据 {evidence.EvidenceId}" });
content.Add(new
{
type = "input_image",
image_url = "data:" + evidence.MimeType + ";base64," +
Convert.ToBase64String(evidence.ImageBytes),
});
}
if (content.Count == 0)
content.Add(new { type = "input_text", text = "仅根据候选结构化信息完成对账" });
messages.Add(new { role = "user", content });
}
else
{
messages.Add(new { role = "system", content = systemPrompt });
foreach (var evidence in input.Evidence)
{
content.Add(new { type = "text", text = $"证据 {evidence.EvidenceId}" });
content.Add(new
{
type = "image_url",
image_url = new
{
url = "data:" + evidence.MimeType + ";base64," +
Convert.ToBase64String(evidence.ImageBytes),
},
});
}
if (content.Count == 0)
content.Add(new { type = "text", text = "仅根据候选结构化信息完成对账" });
messages.Add(new { role = "user", content });
}
var (json, error) = await CA(BuildBody(messages, 2048, 0.1), ct);
if (json is null)
{
_logger.LogWarning(
"Recognition batch model request failed. batchId={BatchId} candidates={Candidates} evidence={Evidence} error={Error}",
input.BatchId,
input.Candidates.Count,
input.Evidence.Count,
error);
throw new InvalidOperationException(error ?? "批次对账模型未返回内容");
}
try
{
return ParseRecognitionBatchActions(EX(json));
}
catch (Exception ex) when (ex is JsonException or InvalidOperationException)
{
_logger.LogWarning(
ex,
"Recognition batch output was invalid. batchId={BatchId} candidates={Candidates} evidence={Evidence}",
input.BatchId,
input.Candidates.Count,
input.Evidence.Count);
throw new InvalidOperationException("批次对账模型返回格式无法解析", ex);
}
}
private static IReadOnlyList<RecognitionBatchModelAction> ParseRecognitionBatchActions(
string raw)
{
foreach (var candidate in EnumerateJsonCandidates(raw))
{
try
{
using var document = JsonDocument.Parse(candidate);
var root = document.RootElement;
JsonElement actions;
if (root.ValueKind == JsonValueKind.Array)
{
actions = root;
}
else if (!TryGetProperty(root, "actions", out actions) ||
actions.ValueKind != JsonValueKind.Array)
{
continue;
}
var results = new List<RecognitionBatchModelAction>();
foreach (var item in actions.EnumerateArray().Take(30))
{
if (item.ValueKind != JsonValueKind.Object) continue;
var action = ReadText(item, "action")?.Trim().ToLowerInvariant();
if (action is not ("keep" or "update" or "create" or "drop"))
continue;
decimal? amount = TryGetProperty(item, "amount", out var amountValue) &&
amountValue.ValueKind != JsonValueKind.Null
? ReadAmount(item)
: null;
var confidence = TryGetProperty(item, "confidence", out var confidenceValue) &&
confidenceValue.TryGetDouble(out var parsedConfidence)
? Math.Clamp(parsedConfidence, 0, 1)
: 0;
var reason = ReadText(item, "reason")?.Trim() ?? "AI 对账";
results.Add(new RecognitionBatchModelAction(
action,
ReadText(item, "actionId", "action_id")?.Trim() ?? $"a{results.Count + 1}",
ReadText(item, "candidateId", "candidate_id")?.Trim(),
ReadText(item, "evidenceId", "evidence_id")?.Trim(),
ReadText(item, "type")?.Trim().ToLowerInvariant(),
amount,
ReadText(item, "categoryName", "category_name")?.Trim(),
ReadText(item, "paymentMethod", "payment_method")?.Trim(),
ReadText(item, "note", "merchant")?.Trim(),
ReadOccurredAt(item),
confidence,
reason.Length > 80 ? reason[..80] : reason));
}
return results;
}
catch (JsonException)
{
// Models may wrap the JSON in prose; continue scanning candidates.
}
}
throw new InvalidOperationException("JSON 中没有 actions 数组");
}
private static IReadOnlyList<ImageParseResult> ParseImageResults(string raw)
{
if (string.IsNullOrWhiteSpace(raw))
throw new InvalidOperationException("模型没有返回可解析内容");
@@ -809,4 +992,4 @@ public partial class OpenAiVisionClient : ILlmClient
return null;
}
}
}
}