Add AI batch reconciliation for accessibility bills

2026-07-25 18:41:09 +08:00
parent eb8909a192
commit 7cca34b331
25 changed files with 2771 additions and 230 deletions
@@ -5,10 +5,13 @@ public interface ILlmClient
    bool IsEnabled { get; }
    Task<IntentResult?> TryParseIntentAsync(string userText, CancellationToken ct = default);
    Task<string?> TryGenerateReplyAsync(string systemPrompt, string userText, CancellationToken ct = default);
-    Task<IReadOnlyList<ImageParseResult>?> AnalyzeImageAsync(
-        byte[] imageBytes,
-        string mimeType,
-        CancellationToken ct = default);
+    Task<IReadOnlyList<ImageParseResult>?> AnalyzeImageAsync(
+        byte[] imageBytes,
+        string mimeType,
+        CancellationToken ct = default);
+    Task<IReadOnlyList<RecognitionBatchModelAction>?> ReconcileRecognitionBatchAsync(
+        RecognitionBatchModelInput input,
+        CancellationToken ct = default);
    Task<(bool Ok, string? Error)> TestConnectionAsync(CancellationToken ct = default);

    Task<AgentRunResponse> RunAgentAsync(
@@ -42,13 +45,57 @@ public record AgentRunResponse(
    string Text,
    int ToolCallCount);

-public record ImageParseResult(
+public record ImageParseResult(
    string Type,
    decimal Amount,
    string CategoryName,
    string? PaymentMethod,
    string Note,
-    DateTime? OccurredAt);
+    DateTime? OccurredAt);
+
+public record RecognitionBatchModelCandidate(
+    string CandidateId,
+    string? FlowSessionId,
+    string PackageName,
+    string Type,
+    decimal Amount,
+    string? Merchant,
+    string? OrderId,
+    DateTime OccurredAt,
+    string RecognitionKind,
+    string? CategoryHint,
+    string Confidence,
+    IReadOnlyList<string> EvidenceIds);
+
+public record RecognitionBatchModelEvidence(
+    string EvidenceId,
+    string? CandidateId,
+    string? FlowSessionId,
+    string PackageName,
+    DateTime CapturedAt,
+    byte[] ImageBytes,
+    string MimeType);
+
+public record RecognitionBatchModelInput(
+    string BatchId,
+    IReadOnlyList<RecognitionBatchModelCandidate> Candidates,
+    IReadOnlyList<RecognitionBatchModelEvidence> Evidence,
+    IReadOnlyList<string> ExpenseCategories,
+    IReadOnlyList<string> IncomeCategories);
+
+public record RecognitionBatchModelAction(
+    string Action,
+    string ActionId,
+    string? CandidateId,
+    string? EvidenceId,
+    string? Type,
+    decimal? Amount,
+    string? CategoryName,
+    string? PaymentMethod,
+    string? Note,
+    DateTime? OccurredAt,
+    double Confidence,
+    string Reason);

 public class NullLlmClient : ILlmClient
 {
@@ -65,11 +112,16 @@ public class NullLlmClient : ILlmClient
        CancellationToken ct = default) =>
        Task.FromResult<string?>(null);

-    public Task<IReadOnlyList<ImageParseResult>?> AnalyzeImageAsync(
+    public Task<IReadOnlyList<ImageParseResult>?> AnalyzeImageAsync(
        byte[] imageBytes,
        string mimeType,
        CancellationToken ct = default) =>
-        Task.FromResult<IReadOnlyList<ImageParseResult>?>(null);
+        Task.FromResult<IReadOnlyList<ImageParseResult>?>(null);
+
+    public Task<IReadOnlyList<RecognitionBatchModelAction>?> ReconcileRecognitionBatchAsync(
+        RecognitionBatchModelInput input,
+        CancellationToken ct = default) =>
+        Task.FromResult<IReadOnlyList<RecognitionBatchModelAction>?>(null);

    public Task<(bool Ok, string? Error)> TestConnectionAsync(
        CancellationToken ct = default) =>
@@ -51,7 +51,7 @@ public partial class OpenAiVisionClient : ILlmClient
    public async Task<string?> TryGenerateReplyAsync(string s, string u, CancellationToken ct = default)
    { if (!IsEnabled) return null; return await L(s, u, ct); }

-    public async Task<IReadOnlyList<ImageParseResult>?> AnalyzeImageAsync(
+    public async Task<IReadOnlyList<ImageParseResult>?> AnalyzeImageAsync(
        byte[] img,
        string mime,
        CancellationToken ct = default)
@@ -157,10 +157,193 @@ public partial class OpenAiVisionClient : ILlmClient
                providerRequestId,
                SanitizeOutputSnippet(output));
            throw new InvalidOperationException("图片模型返回格式无法解析，请重试", ex);
-        }
-    }
-
-    private static IReadOnlyList<ImageParseResult> ParseImageResults(string raw)
+        }
+    }
+
+    public async Task<IReadOnlyList<RecognitionBatchModelAction>?> ReconcileRecognitionBatchAsync(
+        RecognitionBatchModelInput input,
+        CancellationToken ct = default)
+    {
+        if (!IsEnabled) return null;
+
+        var candidateJson = JsonSerializer.Serialize(
+            input.Candidates.Select(candidate => new
+            {
+                candidateId = candidate.CandidateId,
+                flowSessionId = candidate.FlowSessionId,
+                packageName = candidate.PackageName,
+                type = candidate.Type,
+                amount = candidate.Amount,
+                merchant = candidate.Merchant,
+                orderId = candidate.OrderId,
+                occurredAt = candidate.OccurredAt,
+                recognitionKind = candidate.RecognitionKind,
+                categoryHint = candidate.CategoryHint,
+                confidence = candidate.Confidence,
+                evidenceIds = candidate.EvidenceIds,
+            }),
+            new JsonSerializerOptions(JsonSerializerDefaults.Web));
+        var systemPrompt = $$"""
+            你是支付结果批次对账器。只返回 JSON：
+            {"actions":[{"action":"keep|update|create|drop","actionId":"a1","candidateId":null,"evidenceId":null,"type":null,"amount":null,"categoryName":null,"paymentMethod":null,"note":null,"occurredAt":null,"confidence":0.0,"reason":""}]}
+            当前批次候选：{{candidateJson}}
+            支出分类只能是：{{string.Join('/', input.ExpenseCategories)}}。
+            收入分类只能是：{{string.Join('/', input.IncomeCategories)}}。
+            每个候选必须且只能返回一个 keep、update 或 drop；缺失字段用候选原值。
+            不同 flowSessionId 代表不同支付流程。即使收款人、金额和时间相同，也绝不能据此合并或删除。
+            只有截图明确显示失败、取消、待支付，或明确是同一 flowSessionId 的重复结果页时才可 drop。
+            只有存在没有对应候选的 evidenceId 且截图明确显示交易成功时才可 create，并必须引用该 evidenceId。
+            update/create 的 type 只能是 expense 或 income，amount 必须大于 0。
+            截图未明确显示时间时沿用候选时间，禁止猜测时间。reason 不超过 40 个汉字。
+            """;
+        var messages = new List<object>();
+        var content = new List<object>();
+        if (_protocol == "messages")
+        {
+            content.Add(new { type = "text", text = systemPrompt });
+            foreach (var evidence in input.Evidence)
+            {
+                content.Add(new { type = "text", text = $"证据 {evidence.EvidenceId}" });
+                content.Add(new
+                {
+                    type = "image",
+                    source = new
+                    {
+                        type = "base64",
+                        media_type = evidence.MimeType,
+                        data = Convert.ToBase64String(evidence.ImageBytes),
+                    },
+                });
+            }
+            messages.Add(new { role = "user", content });
+        }
+        else if (_protocol == "responses")
+        {
+            messages.Add(new { role = "system", content = systemPrompt });
+            foreach (var evidence in input.Evidence)
+            {
+                content.Add(new { type = "input_text", text = $"证据 {evidence.EvidenceId}" });
+                content.Add(new
+                {
+                    type = "input_image",
+                    image_url = "data:" + evidence.MimeType + ";base64," +
+                        Convert.ToBase64String(evidence.ImageBytes),
+                });
+            }
+            if (content.Count == 0)
+                content.Add(new { type = "input_text", text = "仅根据候选结构化信息完成对账" });
+            messages.Add(new { role = "user", content });
+        }
+        else
+        {
+            messages.Add(new { role = "system", content = systemPrompt });
+            foreach (var evidence in input.Evidence)
+            {
+                content.Add(new { type = "text", text = $"证据 {evidence.EvidenceId}" });
+                content.Add(new
+                {
+                    type = "image_url",
+                    image_url = new
+                    {
+                        url = "data:" + evidence.MimeType + ";base64," +
+                            Convert.ToBase64String(evidence.ImageBytes),
+                    },
+                });
+            }
+            if (content.Count == 0)
+                content.Add(new { type = "text", text = "仅根据候选结构化信息完成对账" });
+            messages.Add(new { role = "user", content });
+        }
+
+        var (json, error) = await CA(BuildBody(messages, 2048, 0.1), ct);
+        if (json is null)
+        {
+            _logger.LogWarning(
+                "Recognition batch model request failed. batchId={BatchId} candidates={Candidates} evidence={Evidence} error={Error}",
+                input.BatchId,
+                input.Candidates.Count,
+                input.Evidence.Count,
+                error);
+            throw new InvalidOperationException(error ?? "批次对账模型未返回内容");
+        }
+
+        try
+        {
+            return ParseRecognitionBatchActions(EX(json));
+        }
+        catch (Exception ex) when (ex is JsonException or InvalidOperationException)
+        {
+            _logger.LogWarning(
+                ex,
+                "Recognition batch output was invalid. batchId={BatchId} candidates={Candidates} evidence={Evidence}",
+                input.BatchId,
+                input.Candidates.Count,
+                input.Evidence.Count);
+            throw new InvalidOperationException("批次对账模型返回格式无法解析", ex);
+        }
+    }
+
+    private static IReadOnlyList<RecognitionBatchModelAction> ParseRecognitionBatchActions(
+        string raw)
+    {
+        foreach (var candidate in EnumerateJsonCandidates(raw))
+        {
+            try
+            {
+                using var document = JsonDocument.Parse(candidate);
+                var root = document.RootElement;
+                JsonElement actions;
+                if (root.ValueKind == JsonValueKind.Array)
+                {
+                    actions = root;
+                }
+                else if (!TryGetProperty(root, "actions", out actions) ||
+                         actions.ValueKind != JsonValueKind.Array)
+                {
+                    continue;
+                }
+
+                var results = new List<RecognitionBatchModelAction>();
+                foreach (var item in actions.EnumerateArray().Take(30))
+                {
+                    if (item.ValueKind != JsonValueKind.Object) continue;
+                    var action = ReadText(item, "action")?.Trim().ToLowerInvariant();
+                    if (action is not ("keep" or "update" or "create" or "drop"))
+                        continue;
+                    decimal? amount = TryGetProperty(item, "amount", out var amountValue) &&
+                                      amountValue.ValueKind != JsonValueKind.Null
+                        ? ReadAmount(item)
+                        : null;
+                    var confidence = TryGetProperty(item, "confidence", out var confidenceValue) &&
+                                     confidenceValue.TryGetDouble(out var parsedConfidence)
+                        ? Math.Clamp(parsedConfidence, 0, 1)
+                        : 0;
+                    var reason = ReadText(item, "reason")?.Trim() ?? "AI 对账";
+                    results.Add(new RecognitionBatchModelAction(
+                        action,
+                        ReadText(item, "actionId", "action_id")?.Trim() ?? $"a{results.Count + 1}",
+                        ReadText(item, "candidateId", "candidate_id")?.Trim(),
+                        ReadText(item, "evidenceId", "evidence_id")?.Trim(),
+                        ReadText(item, "type")?.Trim().ToLowerInvariant(),
+                        amount,
+                        ReadText(item, "categoryName", "category_name")?.Trim(),
+                        ReadText(item, "paymentMethod", "payment_method")?.Trim(),
+                        ReadText(item, "note", "merchant")?.Trim(),
+                        ReadOccurredAt(item),
+                        confidence,
+                        reason.Length > 80 ? reason[..80] : reason));
+                }
+                return results;
+            }
+            catch (JsonException)
+            {
+                // Models may wrap the JSON in prose; continue scanning candidates.
+            }
+        }
+        throw new InvalidOperationException("JSON 中没有 actions 数组");
+    }
+
+    private static IReadOnlyList<ImageParseResult> ParseImageResults(string raw)
    {
        if (string.IsNullOrWhiteSpace(raw))
            throw new InvalidOperationException("模型没有返回可解析内容");
@@ -809,4 +992,4 @@ public partial class OpenAiVisionClient : ILlmClient
            return null;
        }
    }
-}
+}