diff --git a/backend/MiaoJiZhang.Api.Tests/ApiIntegrationTests.cs b/backend/MiaoJiZhang.Api.Tests/ApiIntegrationTests.cs index 53dee2f..8117f6e 100644 --- a/backend/MiaoJiZhang.Api.Tests/ApiIntegrationTests.cs +++ b/backend/MiaoJiZhang.Api.Tests/ApiIntegrationTests.cs @@ -353,7 +353,7 @@ public sealed class ApiIntegrationTests(ApiFixture fixture) } [Fact] - public async Task RecognitionClientRequestId_IsIdempotentAcrossConcurrentChannels() + public async Task RecognitionClientRequestId_IsIdempotentAcrossConcurrentChannels() { using var client = await fixture.RegisterAsync("recognition_idempotency"); var ledgers = await client.GetFromJsonAsync("/api/ledgers"); @@ -389,9 +389,89 @@ public sealed class ApiIntegrationTests(ApiFixture fixture) var month = await client.GetFromJsonAsync( $"/api/transactions/month?year=2026&month=7&ledgerId={ledgerId}"); - Assert.Equal(1, month.GetProperty("count").GetInt32()); - } -} + Assert.Equal(1, month.GetProperty("count").GetInt32()); + } + + [Fact] + public async Task RecognitionBatch_PreservesThreeConsecutiveTransfers_AndIsIdempotent() + { + using var client = await fixture.RegisterAsync("recognition_batch_three_transfers"); + var ledgers = await client.GetFromJsonAsync("/api/ledgers"); + var ledgerId = ledgers[0].GetProperty("id").GetInt64(); + var categories = await client.GetFromJsonAsync( + "/api/categories?type=expense"); + var categoryId = categories[0].GetProperty("id").GetInt64(); + var batchId = Guid.NewGuid().ToString(); + var payload = new + { + batchId, + ledgerId, + items = new[] + { + new + { + candidateId = "transfer-1", + clientRequestId = "recognition-wechat-flow-1", + categoryId, + type = "expense", + amount = 20m, + note = "转账给张三", + paymentMethod = "微信", + occurredAt = "2026-07-25T10:00:01+08:00", + source = "recognition_ai", + sourceText = "微信转账成功", + }, + new + { + candidateId = "transfer-2", + clientRequestId = "recognition-wechat-flow-2", + categoryId, + type = "expense", + amount = 20m, + note = "转账给张三", + paymentMethod = "微信", + occurredAt = "2026-07-25T10:00:10+08:00", + source = "recognition_ai", + sourceText = "微信转账成功", + }, + new + { + candidateId = "transfer-3", + clientRequestId = "recognition-wechat-flow-3", + categoryId, + type = "expense", + amount = 30m, + note = "转账给张三", + paymentMethod = "微信", + occurredAt = "2026-07-25T10:00:20+08:00", + source = "recognition_ai", + sourceText = "微信转账成功", + }, + }, + }; + + var responses = await Task.WhenAll( + client.PostAsJsonAsync("/api/transactions/recognition-batch", payload), + client.PostAsJsonAsync("/api/transactions/recognition-batch", payload)); + Assert.All(responses, response => response.EnsureSuccessStatusCode()); + var results = await Task.WhenAll( + responses.Select(response => response.Content.ReadFromJsonAsync())); + var first = results[0]; + var retry = results[1]; + Assert.Equal(3, first.GetArrayLength()); + Assert.Equal(3, retry.GetArrayLength()); + Assert.Equal( + first.EnumerateArray() + .Select(item => item.GetProperty("transaction").GetProperty("id").GetInt64()), + retry.EnumerateArray() + .Select(item => item.GetProperty("transaction").GetProperty("id").GetInt64())); + + var month = await client.GetFromJsonAsync( + $"/api/transactions/month?year=2026&month=7&ledgerId={ledgerId}"); + Assert.Equal(3, month.GetProperty("count").GetInt32()); + Assert.Equal(70m, month.GetProperty("expense").GetDecimal()); + } +} public sealed class ChinaClockTests { @@ -427,7 +507,7 @@ public sealed class ChinaClockTests yearEnd); } } -public sealed class ImageParseResultTests +public sealed class ImageParseResultTests { private static readonly MethodInfo ParseMethod = typeof(OpenAiVisionClient).GetMethod( @@ -479,7 +559,52 @@ public sealed class ImageParseResultTests results[1].OccurredAt); Assert.Null(results[2].OccurredAt); } -} +} + +public sealed class RecognitionBatchActionParserTests +{ + private static readonly MethodInfo ParseMethod = + typeof(OpenAiVisionClient).GetMethod( + "ParseRecognitionBatchActions", + BindingFlags.NonPublic | BindingFlags.Static) + ?? throw new InvalidOperationException("Recognition batch parser not found"); + + [Fact] + public void Actions_AreParsedFromFencedJson_AndInvalidActionsAreIgnored() + { + const string payload = + """ + result: + ```json + { + "actions": [ + { + "action": "update", + "actionId": "a1", + "candidateId": "candidate-1", + "amount": 20, + "confidence": 1.4, + "reason": "修正金额" + }, + { + "action": "merge", + "candidateId": "candidate-2" + } + ] + } + ``` + """; + + var actions = Assert.IsAssignableFrom>( + ParseMethod.Invoke(null, [payload])); + + var action = Assert.Single(actions); + Assert.Equal("update", action.Action); + Assert.Equal("candidate-1", action.CandidateId); + Assert.Equal(20m, action.Amount); + Assert.Equal(1, action.Confidence); + } +} public sealed class BudgetRecommendationValidationTests { diff --git a/backend/MiaoJiZhang.Api/Contracts/BusinessContracts.cs b/backend/MiaoJiZhang.Api/Contracts/BusinessContracts.cs index 79824ff..bb8b354 100644 --- a/backend/MiaoJiZhang.Api/Contracts/BusinessContracts.cs +++ b/backend/MiaoJiZhang.Api/Contracts/BusinessContracts.cs @@ -138,10 +138,78 @@ public record ImageParseResponse( decimal Amount, string? PaymentMethod, string Note, string Type, List Items, DateTime? OccurredAt); +public record RecognitionBatchCandidateRequest( + string CandidateId, + string ClientRequestId, + string? FlowSessionId, + string PackageName, + string Type, + decimal Amount, + string? Merchant, + string? OrderId, + DateTime OccurredAt, + string RecognitionKind, + string? CategoryHint, + string Confidence, + string? SourceText, + List? EvidenceIds = null); + +public record RecognitionBatchEvidenceRequest( + string EvidenceId, + string? CandidateId, + string? FlowSessionId, + string PackageName, + DateTime CapturedAt); + +public record RecognitionBatchManifestRequest( + string BatchId, + DateTime OpenedAt, + List Candidates, + List Evidence); + +public record RecognitionBatchActionResponse( + string Action, + string ActionId, + string? CandidateId, + string? EvidenceId, + long? CategoryId, + string? CategoryName, + string? Type, + decimal? Amount, + string? PaymentMethod, + string? Note, + DateTime? OccurredAt, + double Confidence, + string Reason); + +public record RecognitionBatchResponse( + string BatchId, + List Actions); + +public record RecognitionBatchTransactionItemRequest( + string CandidateId, + string ClientRequestId, + long CategoryId, + string Type, + decimal Amount, + string? Note, + string? PaymentMethod, + DateTime OccurredAt, + string? Source, + string? SourceText); + +public record CreateRecognitionBatchRequest( + string BatchId, + long? LedgerId, + List Items); + +public record RecognitionBatchTransactionDto( + string CandidateId, + TransactionDto Transaction); + // ---- Chat ---- public record SendChatRequest(string Content, string Type = "text", long? LedgerId = null); // text | sticker public record ChatMessageDto( long Id, string Role, string Type, string Content, TransactionDto? Transaction, DateTime CreatedAt); public record SendChatResponse(List Messages); - diff --git a/backend/MiaoJiZhang.Api/Controllers/ParseController.cs b/backend/MiaoJiZhang.Api/Controllers/ParseController.cs index 237c3e8..e4dece2 100644 --- a/backend/MiaoJiZhang.Api/Controllers/ParseController.cs +++ b/backend/MiaoJiZhang.Api/Controllers/ParseController.cs @@ -1,4 +1,5 @@ -using System.Security.Claims; +using System.Security.Claims; +using System.Text.Json; using MiaoJiZhang.Api.Contracts; using MiaoJiZhang.Api.Services; using MiaoJiZhang.Domain.Entities; @@ -102,7 +103,7 @@ public class ParseController(AppDbContext db, ILlmClient llm, AgentService agent [EnableRateLimiting("upload")] [RequestSizeLimit(10 * 1024 * 1024)] [RequestFormLimits(MultipartBodyLengthLimit = 10 * 1024 * 1024)] - public async Task> ParseImage( + public async Task> ParseImage( IFormFile file, [FromQuery] string source = "image", CancellationToken ct = default) @@ -195,7 +196,7 @@ public class ParseController(AppDbContext db, ILlmClient llm, AgentService agent } var first = items[0]; - return Ok(new ImageParseResponse( + return Ok(new ImageParseResponse( true, first.CategoryId, first.CategoryName, @@ -205,8 +206,219 @@ public class ParseController(AppDbContext db, ILlmClient llm, AgentService agent first.Note, first.Type, items, - first.OccurredAt)); - } + first.OccurredAt)); + } + + [HttpPost("recognition-batch")] + [EnableRateLimiting("upload")] + [RequestSizeLimit(10 * 1024 * 1024)] + [RequestFormLimits(MultipartBodyLengthLimit = 10 * 1024 * 1024)] + public async Task> ReconcileRecognitionBatch( + CancellationToken ct = default) + { + if (!await FeatureEnabled("feature.screenshot_bookkeeping_enabled")) + return StatusCode(403, new ApiError("FEATURE_DISABLED", "AI 截图补全已由后台关闭")); + if (!llm.IsEnabled) + return StatusCode(503, new ApiError("LLM_NOT_CONFIGURED", "AI 图片解析未配置")); + + var form = await Request.ReadFormAsync(ct); + var manifestText = form["manifest"].FirstOrDefault(); + if (string.IsNullOrWhiteSpace(manifestText)) + return BadRequest(new ApiError("BATCH_MANIFEST_EMPTY", "批次清单不能为空")); + + RecognitionBatchManifestRequest? manifest; + try + { + manifest = JsonSerializer.Deserialize( + manifestText, + new JsonSerializerOptions(JsonSerializerDefaults.Web)); + } + catch (JsonException) + { + return BadRequest(new ApiError("BATCH_MANIFEST_INVALID", "批次清单格式无效")); + } + if (manifest is null || !Guid.TryParse(manifest.BatchId, out _) || + manifest.Candidates.Count > 10 || manifest.Evidence.Count > 10 || + manifest.Candidates.Count + manifest.Evidence.Count == 0) + { + return BadRequest(new ApiError("BATCH_MANIFEST_INVALID", "批次数量或标识无效")); + } + if (manifest.Candidates.Select(item => item.CandidateId).Distinct().Count() != + manifest.Candidates.Count || + manifest.Evidence.Select(item => item.EvidenceId).Distinct().Count() != + manifest.Evidence.Count) + { + return BadRequest(new ApiError("BATCH_ID_DUPLICATED", "批次中存在重复标识")); + } + + var evidenceById = manifest.Evidence.ToDictionary(item => item.EvidenceId); + var candidateIds = manifest.Candidates.Select(item => item.CandidateId).ToHashSet(); + if (manifest.Evidence.Any(item => item.CandidateId is not null && + !candidateIds.Contains(item.CandidateId)) || + manifest.Candidates.Any(candidate => + (candidate.EvidenceIds ?? []).Any(evidenceId => + !evidenceById.TryGetValue(evidenceId, out var evidence) || + evidence.CandidateId != candidate.CandidateId))) + { + return BadRequest(new ApiError("BATCH_EVIDENCE_INVALID", "截图与候选关联无效")); + } + var imageInputs = new List(); + var uploadedEvidenceIds = new HashSet(); + long totalBytes = 0; + foreach (var file in form.Files) + { + var evidenceId = Path.GetFileNameWithoutExtension(file.FileName); + if (!evidenceById.TryGetValue(evidenceId, out var evidence) || + !uploadedEvidenceIds.Add(evidenceId)) + return BadRequest(new ApiError("BATCH_EVIDENCE_UNKNOWN", "截图与批次清单不匹配")); + if (!file.ContentType.StartsWith("image/") || file.Length <= 0 || file.Length > 1024 * 1024) + return BadRequest(new ApiError("BATCH_EVIDENCE_INVALID", "单张截图必须是 1MB 以内的图片")); + totalBytes += file.Length; + if (totalBytes > 9 * 1024 * 1024) + return BadRequest(new ApiError("BATCH_TOO_LARGE", "批次截图总大小不能超过 9MB")); + using var stream = new MemoryStream(); + await file.CopyToAsync(stream, ct); + imageInputs.Add(new RecognitionBatchModelEvidence( + evidence.EvidenceId, + evidence.CandidateId, + evidence.FlowSessionId, + evidence.PackageName, + evidence.CapturedAt, + stream.ToArray(), + file.ContentType)); + } + if (uploadedEvidenceIds.Count != evidenceById.Count) + return BadRequest(new ApiError("BATCH_EVIDENCE_MISSING", "批次截图上传不完整")); + + var categories = await db.Categories + .Where(category => !category.IsDeleted && + (category.UserId == null || category.UserId == Uid)) + .ToListAsync(ct); + var candidates = manifest.Candidates.Select(candidate => + new RecognitionBatchModelCandidate( + candidate.CandidateId, + candidate.FlowSessionId, + candidate.PackageName, + candidate.Type, + candidate.Amount, + candidate.Merchant, + candidate.OrderId, + candidate.OccurredAt, + candidate.RecognitionKind, + candidate.CategoryHint, + candidate.Confidence, + candidate.EvidenceIds ?? [])).ToList(); + + IReadOnlyList? modelActions; + try + { + modelActions = await llm.ReconcileRecognitionBatchAsync( + new RecognitionBatchModelInput( + manifest.BatchId, + candidates, + imageInputs, + categories.Where(category => category.Type == TransactionType.Expense) + .Select(category => category.Name).Distinct().ToList(), + categories.Where(category => category.Type == TransactionType.Income) + .Select(category => category.Name).Distinct().ToList()), + ct); + } + catch (InvalidOperationException ex) + { + return StatusCode(502, new ApiError("BATCH_AI_FAILED", ex.Message)); + } + finally + { + foreach (var evidence in imageInputs) + Array.Clear(evidence.ImageBytes); + } + if (modelActions is null) + return StatusCode(502, new ApiError("BATCH_AI_FAILED", "AI 未返回批次对账结果")); + + var actions = ValidateBatchActions(manifest, modelActions, categories); + return Ok(new RecognitionBatchResponse(manifest.BatchId, actions)); + } + + private List ValidateBatchActions( + RecognitionBatchManifestRequest manifest, + IReadOnlyList modelActions, + List categories) + { + var candidateIds = manifest.Candidates.Select(item => item.CandidateId).ToHashSet(); + var evidenceById = manifest.Evidence.ToDictionary(item => item.EvidenceId); + var selected = modelActions + .Where(action => action.CandidateId != null && candidateIds.Contains(action.CandidateId)) + .GroupBy(action => action.CandidateId!) + .ToDictionary(group => group.Key, group => group.First()); + var result = new List(); + + foreach (var candidate in manifest.Candidates) + { + selected.TryGetValue(candidate.CandidateId, out var model); + var action = model?.Action is "update" or "drop" ? model.Action : "keep"; + var reason = model?.Reason ?? "保留本地识别结果"; + var confidence = model?.Confidence ?? 1; + if (action == "drop" && (confidence < 0.9 || + string.IsNullOrWhiteSpace(model?.EvidenceId) || + !evidenceById.TryGetValue(model.EvidenceId, out var dropEvidence) || + dropEvidence.CandidateId != candidate.CandidateId)) + { + action = "keep"; + reason = "撤销证据不足，已保留本地结果"; + } + var type = model?.Type is "income" or "expense" ? model.Type : candidate.Type; + var amount = model?.Amount is > 0 ? model.Amount.Value : candidate.Amount; + var category = FindCategory( + categories, + type == "income" ? TransactionType.Income : TransactionType.Expense, + model?.CategoryName ?? candidate.CategoryHint ?? "其他"); + result.Add(new RecognitionBatchActionResponse( + action, + model?.ActionId ?? $"keep-{candidate.CandidateId}", + candidate.CandidateId, + model?.EvidenceId, + category.Id, + category.Name, + type, + amount, + model?.PaymentMethod, + string.IsNullOrWhiteSpace(model?.Note) ? candidate.Merchant : model.Note, + model?.OccurredAt ?? candidate.OccurredAt, + confidence, + reason)); + } + + var usedEvidence = new HashSet(); + foreach (var model in modelActions.Where(action => action.Action == "create")) + { + if (string.IsNullOrWhiteSpace(model.EvidenceId) || + !evidenceById.TryGetValue(model.EvidenceId, out var evidence) || + !string.IsNullOrWhiteSpace(evidence.CandidateId) || + !usedEvidence.Add(model.EvidenceId) || + model.Type is not ("income" or "expense") || + model.Amount is not > 0) + { + continue; + } + var type = model.Type == "income" ? TransactionType.Income : TransactionType.Expense; + var category = FindCategory(categories, type, model.CategoryName ?? "其他"); + result.Add(new RecognitionBatchActionResponse( + "create", + model.ActionId, + null, + model.EvidenceId, + category.Id, + category.Name, + model.Type, + model.Amount, + model.PaymentMethod, + model.Note, + model.OccurredAt ?? evidence.CapturedAt, + model.Confidence, + model.Reason)); + } + return result.Take(20).ToList(); + } private async Task FeatureEnabled(string key) { diff --git a/backend/MiaoJiZhang.Api/Controllers/TransactionsController.cs b/backend/MiaoJiZhang.Api/Controllers/TransactionsController.cs index 5b399ff..2b9aa00 100644 --- a/backend/MiaoJiZhang.Api/Controllers/TransactionsController.cs +++ b/backend/MiaoJiZhang.Api/Controllers/TransactionsController.cs @@ -61,17 +61,7 @@ public class TransactionsController(AppDbContext db, LedgerResolver ledgers) : C Amount = req.Amount, Note = req.Note, PaymentMethod = req.PaymentMethod, - Source = req.Source switch - { - "voice" => TransactionSource.Voice, - "ocr" => TransactionSource.ReceiptOcr, - "screenshot" => TransactionSource.Screenshot, - "accessibility" => TransactionSource.Accessibility, - "notification" => TransactionSource.Notification, - "recognition_ai" => TransactionSource.RecognitionAi, - "local_ocr" => TransactionSource.LocalOcr, - _ => TransactionSource.Manual, - }, + Source = SourceFromWire(req.Source), SourceText = req.SourceText, ClientRequestId = clientRequestId, CreatedAt = DateTime.UtcNow, @@ -98,6 +88,107 @@ public class TransactionsController(AppDbContext db, LedgerResolver ledgers) : C } } + [HttpPost("recognition-batch")] + public async Task>> CreateRecognitionBatch( + CreateRecognitionBatchRequest req, + CancellationToken ct) + { + if (!Guid.TryParse(req.BatchId, out _) || req.Items.Count is < 1 or > 20) + return BadRequest(new ApiError("BATCH_INVALID", "批次标识或账单数量无效")); + if (req.Items.Select(item => item.CandidateId).Distinct().Count() != req.Items.Count || + req.Items.Select(item => item.ClientRequestId).Distinct().Count() != req.Items.Count) + { + return BadRequest(new ApiError("BATCH_DUPLICATED", "批次中存在重复账单标识")); + } + if (req.Items.Any(item => item.Amount <= 0 || + item.ClientRequestId.Length is < 1 or > 64 || + ParseType(item.Type) is null)) + { + return BadRequest(new ApiError("BATCH_ITEM_INVALID", "批次中存在无效账单")); + } + + var ledgerId = await ledgers.ResolveAsync(Uid, req.LedgerId); + if (!ledgerId.HasValue) + return BadRequest(new ApiError("LEDGER_NOT_FOUND", "账本不存在或无权访问")); + var categoryIds = req.Items.Select(item => item.CategoryId).Distinct().ToList(); + var categories = await db.Categories + .Where(category => categoryIds.Contains(category.Id) && !category.IsDeleted && + (category.UserId == null || category.UserId == Uid)) + .ToDictionaryAsync(category => category.Id, ct); + foreach (var item in req.Items) + { + var type = ParseType(item.Type)!.Value; + if (!categories.TryGetValue(item.CategoryId, out var category) || category.Type != type) + return BadRequest(new ApiError("CATEGORY_TYPE_MISMATCH", "分类与收支类型不一致")); + } + + var requestIds = req.Items.Select(item => item.ClientRequestId).ToList(); + var existing = await db.Transactions + .Include(transaction => transaction.Category) + .Where(transaction => transaction.UserId == Uid && + transaction.ClientRequestId != null && + requestIds.Contains(transaction.ClientRequestId)) + .ToDictionaryAsync(transaction => transaction.ClientRequestId!, ct); + await using var transactionScope = await db.Database.BeginTransactionAsync(ct); + var mapped = new List<(string CandidateId, Transaction Transaction)>(); + foreach (var item in req.Items) + { + if (existing.TryGetValue(item.ClientRequestId, out var found)) + { + mapped.Add((item.CandidateId, found)); + continue; + } + var category = categories[item.CategoryId]; + var transaction = new Transaction + { + LedgerId = ledgerId.Value, + UserId = Uid, + CategoryId = category.Id, + Category = category, + Type = ParseType(item.Type)!.Value, + Amount = item.Amount, + Note = item.Note?.Trim(), + PaymentMethod = item.PaymentMethod?.Trim(), + OccurredAt = NormalizeOccurredAt(item.OccurredAt), + Source = SourceFromWire(item.Source), + SourceText = item.SourceText, + ClientRequestId = item.ClientRequestId, + CreatedAt = DateTime.UtcNow, + UpdatedAt = DateTime.UtcNow, + }; + db.Transactions.Add(transaction); + mapped.Add((item.CandidateId, transaction)); + } + try + { + await db.SaveChangesAsync(ct); + await transactionScope.CommitAsync(ct); + return Ok(mapped.Select(item => new RecognitionBatchTransactionDto( + item.CandidateId, + ToDto(item.Transaction, item.Transaction.Category))).ToList()); + } + catch (DbUpdateException) + { + await transactionScope.RollbackAsync(ct); + foreach (var entry in db.ChangeTracker.Entries() + .Where(entry => entry.State == EntityState.Added)) + { + entry.State = EntityState.Detached; + } + var raced = await db.Transactions + .AsNoTracking() + .Include(transaction => transaction.Category) + .Where(transaction => transaction.UserId == Uid && + transaction.ClientRequestId != null && + requestIds.Contains(transaction.ClientRequestId)) + .ToDictionaryAsync(transaction => transaction.ClientRequestId!, ct); + if (raced.Count != requestIds.Count) throw; + return Ok(req.Items.Select(item => new RecognitionBatchTransactionDto( + item.CandidateId, + ToDto(raced[item.ClientRequestId], raced[item.ClientRequestId].Category))).ToList()); + } + } + /// 账单详情（P11：AI 来源追溯、分类、备注、时间） [HttpGet("{id:long}")] public async Task> Detail(long id) @@ -512,6 +603,18 @@ public class TransactionsController(AppDbContext db, LedgerResolver ledgers) : C : baseUpdatedAt.Value.ToUniversalTime(); return transaction.UpdatedAt > baseline.AddMilliseconds(1); } + + private static TransactionSource SourceFromWire(string? source) => source switch + { + "voice" => TransactionSource.Voice, + "ocr" => TransactionSource.ReceiptOcr, + "screenshot" => TransactionSource.Screenshot, + "accessibility" => TransactionSource.Accessibility, + "notification" => TransactionSource.Notification, + "recognition_ai" => TransactionSource.RecognitionAi, + "local_ocr" => TransactionSource.LocalOcr, + _ => TransactionSource.Manual, + }; private static DateTime NormalizeOccurredAt(DateTime value) => value.Kind == DateTimeKind.Utc ? value : ChinaClock.ToUtc(value); diff --git a/backend/MiaoJiZhang.Api/Services/LlmClient.cs b/backend/MiaoJiZhang.Api/Services/LlmClient.cs index 64a87f7..4638356 100644 --- a/backend/MiaoJiZhang.Api/Services/LlmClient.cs +++ b/backend/MiaoJiZhang.Api/Services/LlmClient.cs @@ -5,10 +5,13 @@ public interface ILlmClient bool IsEnabled { get; } Task TryParseIntentAsync(string userText, CancellationToken ct = default); Task TryGenerateReplyAsync(string systemPrompt, string userText, CancellationToken ct = default); - Task?> AnalyzeImageAsync( - byte[] imageBytes, - string mimeType, - CancellationToken ct = default); + Task?> AnalyzeImageAsync( + byte[] imageBytes, + string mimeType, + CancellationToken ct = default); + Task?> ReconcileRecognitionBatchAsync( + RecognitionBatchModelInput input, + CancellationToken ct = default); Task<(bool Ok, string? Error)> TestConnectionAsync(CancellationToken ct = default); Task RunAgentAsync( @@ -42,13 +45,57 @@ public record AgentRunResponse( string Text, int ToolCallCount); -public record ImageParseResult( +public record ImageParseResult( string Type, decimal Amount, string CategoryName, string? PaymentMethod, string Note, - DateTime? OccurredAt); + DateTime? OccurredAt); + +public record RecognitionBatchModelCandidate( + string CandidateId, + string? FlowSessionId, + string PackageName, + string Type, + decimal Amount, + string? Merchant, + string? OrderId, + DateTime OccurredAt, + string RecognitionKind, + string? CategoryHint, + string Confidence, + IReadOnlyList EvidenceIds); + +public record RecognitionBatchModelEvidence( + string EvidenceId, + string? CandidateId, + string? FlowSessionId, + string PackageName, + DateTime CapturedAt, + byte[] ImageBytes, + string MimeType); + +public record RecognitionBatchModelInput( + string BatchId, + IReadOnlyList Candidates, + IReadOnlyList Evidence, + IReadOnlyList ExpenseCategories, + IReadOnlyList IncomeCategories); + +public record RecognitionBatchModelAction( + string Action, + string ActionId, + string? CandidateId, + string? EvidenceId, + string? Type, + decimal? Amount, + string? CategoryName, + string? PaymentMethod, + string? Note, + DateTime? OccurredAt, + double Confidence, + string Reason); public class NullLlmClient : ILlmClient { @@ -65,11 +112,16 @@ public class NullLlmClient : ILlmClient CancellationToken ct = default) => Task.FromResult(null); - public Task?> AnalyzeImageAsync( + public Task?> AnalyzeImageAsync( byte[] imageBytes, string mimeType, CancellationToken ct = default) => - Task.FromResult?>(null); + Task.FromResult?>(null); + + public Task?> ReconcileRecognitionBatchAsync( + RecognitionBatchModelInput input, + CancellationToken ct = default) => + Task.FromResult?>(null); public Task<(bool Ok, string? Error)> TestConnectionAsync( CancellationToken ct = default) => diff --git a/backend/MiaoJiZhang.Api/Services/OpenAiVisionClient.cs b/backend/MiaoJiZhang.Api/Services/OpenAiVisionClient.cs index f98b2cc..d28e64b 100644 --- a/backend/MiaoJiZhang.Api/Services/OpenAiVisionClient.cs +++ b/backend/MiaoJiZhang.Api/Services/OpenAiVisionClient.cs @@ -51,7 +51,7 @@ public partial class OpenAiVisionClient : ILlmClient public async Task TryGenerateReplyAsync(string s, string u, CancellationToken ct = default) { if (!IsEnabled) return null; return await L(s, u, ct); } - public async Task?> AnalyzeImageAsync( + public async Task?> AnalyzeImageAsync( byte[] img, string mime, CancellationToken ct = default) @@ -157,10 +157,193 @@ public partial class OpenAiVisionClient : ILlmClient providerRequestId, SanitizeOutputSnippet(output)); throw new InvalidOperationException("图片模型返回格式无法解析，请重试", ex); - } - } - - private static IReadOnlyList ParseImageResults(string raw) + } + } + + public async Task?> ReconcileRecognitionBatchAsync( + RecognitionBatchModelInput input, + CancellationToken ct = default) + { + if (!IsEnabled) return null; + + var candidateJson = JsonSerializer.Serialize( + input.Candidates.Select(candidate => new + { + candidateId = candidate.CandidateId, + flowSessionId = candidate.FlowSessionId, + packageName = candidate.PackageName, + type = candidate.Type, + amount = candidate.Amount, + merchant = candidate.Merchant, + orderId = candidate.OrderId, + occurredAt = candidate.OccurredAt, + recognitionKind = candidate.RecognitionKind, + categoryHint = candidate.CategoryHint, + confidence = candidate.Confidence, + evidenceIds = candidate.EvidenceIds, + }), + new JsonSerializerOptions(JsonSerializerDefaults.Web)); + var systemPrompt = $$""" + 你是支付结果批次对账器。只返回 JSON： + {"actions":[{"action":"keep|update|create|drop","actionId":"a1","candidateId":null,"evidenceId":null,"type":null,"amount":null,"categoryName":null,"paymentMethod":null,"note":null,"occurredAt":null,"confidence":0.0,"reason":""}]} + 当前批次候选：{{candidateJson}} + 支出分类只能是：{{string.Join('/', input.ExpenseCategories)}}。 + 收入分类只能是：{{string.Join('/', input.IncomeCategories)}}。 + 每个候选必须且只能返回一个 keep、update 或 drop；缺失字段用候选原值。 + 不同 flowSessionId 代表不同支付流程。即使收款人、金额和时间相同，也绝不能据此合并或删除。 + 只有截图明确显示失败、取消、待支付，或明确是同一 flowSessionId 的重复结果页时才可 drop。 + 只有存在没有对应候选的 evidenceId 且截图明确显示交易成功时才可 create，并必须引用该 evidenceId。 + update/create 的 type 只能是 expense 或 income，amount 必须大于 0。 + 截图未明确显示时间时沿用候选时间，禁止猜测时间。reason 不超过 40 个汉字。 + """; + var messages = new List(); + var content = new List(); + if (_protocol == "messages") + { + content.Add(new { type = "text", text = systemPrompt }); + foreach (var evidence in input.Evidence) + { + content.Add(new { type = "text", text = $"证据 {evidence.EvidenceId}" }); + content.Add(new + { + type = "image", + source = new + { + type = "base64", + media_type = evidence.MimeType, + data = Convert.ToBase64String(evidence.ImageBytes), + }, + }); + } + messages.Add(new { role = "user", content }); + } + else if (_protocol == "responses") + { + messages.Add(new { role = "system", content = systemPrompt }); + foreach (var evidence in input.Evidence) + { + content.Add(new { type = "input_text", text = $"证据 {evidence.EvidenceId}" }); + content.Add(new + { + type = "input_image", + image_url = "data:" + evidence.MimeType + ";base64," + + Convert.ToBase64String(evidence.ImageBytes), + }); + } + if (content.Count == 0) + content.Add(new { type = "input_text", text = "仅根据候选结构化信息完成对账" }); + messages.Add(new { role = "user", content }); + } + else + { + messages.Add(new { role = "system", content = systemPrompt }); + foreach (var evidence in input.Evidence) + { + content.Add(new { type = "text", text = $"证据 {evidence.EvidenceId}" }); + content.Add(new + { + type = "image_url", + image_url = new + { + url = "data:" + evidence.MimeType + ";base64," + + Convert.ToBase64String(evidence.ImageBytes), + }, + }); + } + if (content.Count == 0) + content.Add(new { type = "text", text = "仅根据候选结构化信息完成对账" }); + messages.Add(new { role = "user", content }); + } + + var (json, error) = await CA(BuildBody(messages, 2048, 0.1), ct); + if (json is null) + { + _logger.LogWarning( + "Recognition batch model request failed. batchId={BatchId} candidates={Candidates} evidence={Evidence} error={Error}", + input.BatchId, + input.Candidates.Count, + input.Evidence.Count, + error); + throw new InvalidOperationException(error ?? "批次对账模型未返回内容"); + } + + try + { + return ParseRecognitionBatchActions(EX(json)); + } + catch (Exception ex) when (ex is JsonException or InvalidOperationException) + { + _logger.LogWarning( + ex, + "Recognition batch output was invalid. batchId={BatchId} candidates={Candidates} evidence={Evidence}", + input.BatchId, + input.Candidates.Count, + input.Evidence.Count); + throw new InvalidOperationException("批次对账模型返回格式无法解析", ex); + } + } + + private static IReadOnlyList ParseRecognitionBatchActions( + string raw) + { + foreach (var candidate in EnumerateJsonCandidates(raw)) + { + try + { + using var document = JsonDocument.Parse(candidate); + var root = document.RootElement; + JsonElement actions; + if (root.ValueKind == JsonValueKind.Array) + { + actions = root; + } + else if (!TryGetProperty(root, "actions", out actions) || + actions.ValueKind != JsonValueKind.Array) + { + continue; + } + + var results = new List(); + foreach (var item in actions.EnumerateArray().Take(30)) + { + if (item.ValueKind != JsonValueKind.Object) continue; + var action = ReadText(item, "action")?.Trim().ToLowerInvariant(); + if (action is not ("keep" or "update" or "create" or "drop")) + continue; + decimal? amount = TryGetProperty(item, "amount", out var amountValue) && + amountValue.ValueKind != JsonValueKind.Null + ? ReadAmount(item) + : null; + var confidence = TryGetProperty(item, "confidence", out var confidenceValue) && + confidenceValue.TryGetDouble(out var parsedConfidence) + ? Math.Clamp(parsedConfidence, 0, 1) + : 0; + var reason = ReadText(item, "reason")?.Trim() ?? "AI 对账"; + results.Add(new RecognitionBatchModelAction( + action, + ReadText(item, "actionId", "action_id")?.Trim() ?? $"a{results.Count + 1}", + ReadText(item, "candidateId", "candidate_id")?.Trim(), + ReadText(item, "evidenceId", "evidence_id")?.Trim(), + ReadText(item, "type")?.Trim().ToLowerInvariant(), + amount, + ReadText(item, "categoryName", "category_name")?.Trim(), + ReadText(item, "paymentMethod", "payment_method")?.Trim(), + ReadText(item, "note", "merchant")?.Trim(), + ReadOccurredAt(item), + confidence, + reason.Length > 80 ? reason[..80] : reason)); + } + return results; + } + catch (JsonException) + { + // Models may wrap the JSON in prose; continue scanning candidates. + } + } + throw new InvalidOperationException("JSON 中没有 actions 数组"); + } + + private static IReadOnlyList ParseImageResults(string raw) { if (string.IsNullOrWhiteSpace(raw)) throw new InvalidOperationException("模型没有返回可解析内容"); @@ -809,4 +992,4 @@ public partial class OpenAiVisionClient : ILlmClient return null; } } -} \ No newline at end of file +} diff --git a/frontend/android/app/src/main/kotlin/com/nx/miaoji/BackgroundAiRecognizer.kt b/frontend/android/app/src/main/kotlin/com/nx/miaoji/BackgroundAiRecognizer.kt deleted file mode 100644 index 2ce441d..0000000 --- a/frontend/android/app/src/main/kotlin/com/nx/miaoji/BackgroundAiRecognizer.kt +++ /dev/null @@ -1,112 +0,0 @@ -package com.nx.miaoji - -import android.content.Context -import android.util.Log -import org.json.JSONObject -import java.io.ByteArrayOutputStream -import java.net.HttpURLConnection -import java.net.URL -import java.time.Instant -import java.util.UUID -import java.util.concurrent.Executors - -object BackgroundAiRecognizer { - private val executor = Executors.newSingleThreadExecutor() - - fun analyze( - context: Context, - packageName: String, - image: ByteArray, - flowSessionId: String? = null, - ) { - val appContext = context.applicationContext - executor.execute { - try { - val settings = RecognitionSettings.snapshot(appContext) - val token = RecognitionSettings.runtimeToken(appContext) - val baseUrl = settings.baseUrl?.trimEnd('/') - if (!settings.aiScreenshot || !settings.aiAllowed || !settings.hasAccount || - token.isNullOrBlank() || baseUrl.isNullOrBlank() - ) { - return@execute - } - val response = upload("$baseUrl/api/parse/image?source=screenshot", token, image) - if (response.first == HttpURLConnection.HTTP_FORBIDDEN && - response.second.contains("AI_PERMISSION_DENIED") - ) { - RecognitionSettings.disableRuntimeAi(appContext) - return@execute - } - if (response.first !in 200..299) { - Log.w(TAG, "Background AI parse failed status=${response.first}") - return@execute - } - val root = JSONObject(response.second) - val items = root.optJSONArray("items") ?: return@execute - for (index in 0 until minOf(items.length(), 10)) { - val item = items.optJSONObject(index) ?: continue - val type = item.optString("type").lowercase() - val amount = item.optDouble("amount", 0.0) - if (type !in setOf("income", "expense") || amount <= 0) continue - val occurredAt = runCatching { - Instant.parse(item.optString("occurredAt")).toEpochMilli() - }.getOrElse { System.currentTimeMillis() } - val note = item.optString("note").takeIf { it.isNotBlank() } - RecognitionCoordinator.get(appContext).submit( - PaymentSignal( - packageName = packageName, - channel = "recognition_ai", - amountCents = kotlin.math.round(amount * 100).toLong(), - type = type, - merchant = note, - orderId = null, - occurredAtEpochMs = occurredAt, - knownTemplate = false, - sourceEventId = "ai:" + (flowSessionId ?: UUID.randomUUID().toString()), - sourceText = "AI 截图补全 · " + PaymentParser.appName(packageName), - flowSessionId = flowSessionId, - evidenceConfidence = "confirm", - ), - ) - break - } - } catch (error: Exception) { - Log.w(TAG, "Background AI parse unavailable", error) - } finally { - image.fill(0) - } - } - } - - private fun upload( - endpoint: String, - token: String, - image: ByteArray, - ): Pair { - val boundary = "----Jizhi${UUID.randomUUID()}" - val connection = URL(endpoint).openConnection() as HttpURLConnection - connection.connectTimeout = 10_000 - connection.readTimeout = 120_000 - connection.requestMethod = "POST" - connection.doOutput = true - connection.setRequestProperty("Authorization", "Bearer $token") - connection.setRequestProperty("Content-Type", "multipart/form-data; boundary=$boundary") - connection.outputStream.use { output -> - output.write("--$boundary\r\n".toByteArray()) - output.write( - "Content-Disposition: form-data; name=\"file\"; filename=\"recognition.png\"\r\n" - .toByteArray(), - ) - output.write("Content-Type: image/png\r\n\r\n".toByteArray()) - output.write(image) - output.write("\r\n--$boundary--\r\n".toByteArray()) - } - val code = connection.responseCode - val stream = if (code in 200..299) connection.inputStream else connection.errorStream - val body = stream?.bufferedReader(Charsets.UTF_8)?.use { it.readText() }.orEmpty() - connection.disconnect() - return code to body - } - - private const val TAG = "JizhiRecognition" -} diff --git a/frontend/android/app/src/main/kotlin/com/nx/miaoji/MainActivity.kt b/frontend/android/app/src/main/kotlin/com/nx/miaoji/MainActivity.kt index c72f959..679092a 100644 --- a/frontend/android/app/src/main/kotlin/com/nx/miaoji/MainActivity.kt +++ b/frontend/android/app/src/main/kotlin/com/nx/miaoji/MainActivity.kt @@ -42,7 +42,8 @@ class MainActivity : FlutterActivity() { const val ACTION_SCREENSHOT_ERROR = "screenshot_error" const val ACTION_RECOGNITION_CONFIRM = "recognition_confirm" const val ACTION_RECOGNITION_UNDO = "recognition_undo" - const val ACTION_RECOGNITION_EDIT = "recognition_edit" + const val ACTION_RECOGNITION_EDIT = "recognition_edit" + const val ACTION_RECOGNITION_BATCH_REVIEW = "recognition_batch_review" const val EXTRA_SCREENSHOT_PATH = "screenshotPath" const val EXTRA_SCREENSHOT_ERROR = "screenshotError" const val EXTRA_SCREENSHOT_SESSION_ID = "screenshotSessionId" @@ -152,7 +153,24 @@ class MainActivity : FlutterActivity() { ) result.success(response?.getStringArrayList("candidates") ?: arrayListOf()) } - "ackRecognitionCandidate" -> acknowledgeRecognition(call, result) + "ackRecognitionCandidate" -> acknowledgeRecognition(call, result) + "listRecognitionBatches" -> { + val response = RecognitionBridge.call( + this, + RecognitionBridgeProvider.METHOD_BATCHES, + ) + result.success(response?.getStringArrayList("batches") ?: arrayListOf()) + } + "restoreDroppedRecognition" -> { + val response = RecognitionBridge.call( + this, + RecognitionBridgeProvider.METHOD_RESTORE_DROPPED, + extras = Bundle().apply { + putString("candidateId", call.argument("candidateId")) + }, + ) + result.success(response?.getBoolean("success") == true) + } "openAccessibilitySettings" -> { startActivity(Intent(Settings.ACTION_ACCESSIBILITY_SETTINGS)) result.success(true) @@ -240,18 +258,22 @@ class MainActivity : FlutterActivity() { dispatchPendingScreenshot() } } - ACTION_RECOGNITION_CONFIRM, - ACTION_RECOGNITION_UNDO, - ACTION_RECOGNITION_EDIT -> { + ACTION_RECOGNITION_CONFIRM, + ACTION_RECOGNITION_UNDO, + ACTION_RECOGNITION_EDIT, + ACTION_RECOGNITION_BATCH_REVIEW -> { pendingRecognitionAction = mapOf( "action" to incoming.getStringExtra(EXTRA_ACTION), "candidateId" to incoming.getStringExtra( RecognitionCoordinator.EXTRA_CANDIDATE_ID, ), - "transactionId" to incoming.getLongExtra( + "transactionId" to incoming.getLongExtra( EXTRA_TRANSACTION_ID, Long.MIN_VALUE, - ).takeIf { it != Long.MIN_VALUE }, + ).takeIf { it != Long.MIN_VALUE }, + "batchId" to incoming.getStringExtra( + RecognitionCoordinator.EXTRA_BATCH_ID, + ), ) dispatchPendingRecognitionAction() } @@ -810,9 +832,12 @@ class MainActivity : FlutterActivity() { override fun onReceive(context: Context?, intent: Intent?) { pendingRecognitionAction = mapOf( "action" to "ready", - "candidateId" to intent?.getStringExtra( + "candidateId" to intent?.getStringExtra( RecognitionCoordinator.EXTRA_CANDIDATE_ID, - ), + ), + "batchId" to intent?.getStringExtra( + RecognitionCoordinator.EXTRA_BATCH_ID, + ), ) dispatchPendingRecognitionAction() } diff --git a/frontend/android/app/src/main/kotlin/com/nx/miaoji/RecognitionBatchProcessor.kt b/frontend/android/app/src/main/kotlin/com/nx/miaoji/RecognitionBatchProcessor.kt new file mode 100644 index 0000000..a688c33 --- /dev/null +++ b/frontend/android/app/src/main/kotlin/com/nx/miaoji/RecognitionBatchProcessor.kt @@ -0,0 +1,92 @@ +package com.nx.miaoji + +import android.content.Context +import android.util.Log +import java.net.HttpURLConnection +import java.net.URL +import java.util.UUID + +class RecognitionBatchProcessor(private val context: Context) { + fun process(batch: PendingRecognitionBatch): BatchProcessResult { + return try { + val settings = RecognitionSettings.snapshot(context) + val token = RecognitionSettings.runtimeToken(context) + val baseUrl = settings.baseUrl?.trimEnd('/') + if (!settings.aiScreenshot || !settings.aiAllowed || !settings.hasAccount || + token.isNullOrBlank() || baseUrl.isNullOrBlank() + ) { + return BatchProcessResult.Fallback("AI 截图补全不可用") + } + val response = upload( + "$baseUrl/api/parse/recognition-batch", + token, + batch, + ) + if (response.code == HttpURLConnection.HTTP_FORBIDDEN && + response.body.contains("AI_PERMISSION_DENIED") + ) { + RecognitionSettings.disableRuntimeAi(context) + } + if (response.code !in 200..299) { + Log.w(TAG, "Recognition batch failed status=${response.code} batchId=${batch.id}") + BatchProcessResult.Fallback("AI 批次对账失败（${response.code}）") + } else { + BatchProcessResult.Success(response.body) + } + } catch (error: Exception) { + Log.w(TAG, "Recognition batch unavailable batchId=${batch.id}", error) + BatchProcessResult.Fallback("AI 批次对账超时或网络不可用") + } finally { + batch.images.forEach { it.bytes.fill(0) } + } + } + + private fun upload( + endpoint: String, + token: String, + batch: PendingRecognitionBatch, + ): HttpResponse { + val boundary = "----JizhiBatch${UUID.randomUUID()}" + val connection = URL(endpoint).openConnection() as HttpURLConnection + connection.connectTimeout = 10_000 + connection.readTimeout = 120_000 + connection.requestMethod = "POST" + connection.doOutput = true + connection.setRequestProperty("Authorization", "Bearer $token") + connection.setRequestProperty("Content-Type", "multipart/form-data; boundary=$boundary") + connection.outputStream.use { output -> + output.write("--$boundary\r\n".toByteArray()) + output.write("Content-Disposition: form-data; name=\"manifest\"\r\n".toByteArray()) + output.write("Content-Type: application/json; charset=utf-8\r\n\r\n".toByteArray()) + output.write(batch.manifest.toByteArray(Charsets.UTF_8)) + output.write("\r\n".toByteArray()) + batch.images.forEach { image -> + output.write("--$boundary\r\n".toByteArray()) + output.write( + "Content-Disposition: form-data; name=\"files\"; filename=\"${image.evidenceId}.jpg\"\r\n" + .toByteArray(), + ) + output.write("Content-Type: image/jpeg\r\n\r\n".toByteArray()) + output.write(image.bytes) + output.write("\r\n".toByteArray()) + } + output.write("--$boundary--\r\n".toByteArray()) + } + val code = connection.responseCode + val stream = if (code in 200..299) connection.inputStream else connection.errorStream + val body = stream?.bufferedReader(Charsets.UTF_8)?.use { it.readText() }.orEmpty() + connection.disconnect() + return HttpResponse(code, body) + } + + private data class HttpResponse(val code: Int, val body: String) + + companion object { + private const val TAG = "JizhiRecognition" + } +} + +sealed interface BatchProcessResult { + data class Success(val responseBody: String) : BatchProcessResult + data class Fallback(val reason: String) : BatchProcessResult +} diff --git a/frontend/android/app/src/main/kotlin/com/nx/miaoji/RecognitionBridgeProvider.kt b/frontend/android/app/src/main/kotlin/com/nx/miaoji/RecognitionBridgeProvider.kt index a0afe22..b1a2fd9 100644 --- a/frontend/android/app/src/main/kotlin/com/nx/miaoji/RecognitionBridgeProvider.kt +++ b/frontend/android/app/src/main/kotlin/com/nx/miaoji/RecognitionBridgeProvider.kt @@ -73,6 +73,19 @@ class RecognitionBridgeProvider : ContentProvider() { ) putBoolean("success", candidate != null) } + METHOD_BATCHES -> Bundle().apply { + putStringArrayList( + "batches", + ArrayList(RecognitionCoordinator.get(appContext).recentBatches()), + ) + } + METHOD_RESTORE_DROPPED -> Bundle().apply { + putBoolean( + "success", + RecognitionCoordinator.get(appContext) + .restoreDropped(extras?.getString("candidateId").orEmpty()) != null, + ) + } else -> super.call(method, arg, extras) ?: Bundle() } } @@ -132,6 +145,8 @@ class RecognitionBridgeProvider : ContentProvider() { const val METHOD_SCREENSHOT_RESULT = "screenshotResult" const val METHOD_DRAIN = "drain" const val METHOD_ACK = "ack" + const val METHOD_BATCHES = "batches" + const val METHOD_RESTORE_DROPPED = "restoreDropped" } } diff --git a/frontend/android/app/src/main/kotlin/com/nx/miaoji/RecognitionCoordinator.kt b/frontend/android/app/src/main/kotlin/com/nx/miaoji/RecognitionCoordinator.kt index 99e9d77..54815d8 100644 --- a/frontend/android/app/src/main/kotlin/com/nx/miaoji/RecognitionCoordinator.kt +++ b/frontend/android/app/src/main/kotlin/com/nx/miaoji/RecognitionCoordinator.kt @@ -4,19 +4,101 @@ import android.content.Context import android.content.Intent import android.os.Handler import android.os.HandlerThread +import android.os.Looper import android.util.Log class RecognitionCoordinator private constructor(private val context: Context) { private val store = RecognitionStore(context) + private val batchProcessor = RecognitionBatchProcessor(context) private val thread = HandlerThread("jizhi-recognition").apply { start() } private val handler = Handler(thread.looper) + private val mainHandler = Handler(Looper.getMainLooper()) + private val batchRunnable = Runnable { processDueBatches() } - fun submit(signal: PaymentSignal) { + init { handler.post { - runCatching { - val id = store.upsert(signal) - handler.postDelayed({ finalize(id) }, 1_650L) - }.onFailure { Log.e(TAG, "Unable to store recognition signal", it) } + store.recoverInterruptedBatches() + scheduleBatchProcessing() + } + } + + fun submit( + signal: PaymentSignal, + evidenceImage: ByteArray? = null, + onStored: ((StoredSubmission) -> Unit)? = null, + ) { + handler.post { + try { + val batchMode = RecognitionSettings.snapshot(context).let { + it.aiScreenshot && it.aiAllowed && it.hasAccount + } + val submission = store.upsert(signal, batchMode) + if (submission.batchId != null && evidenceImage != null) { + store.addBatchImage( + submission.batchId, + submission.candidateId, + signal.flowSessionId, + signal.packageName, + System.currentTimeMillis(), + evidenceImage, + ) + } + if (submission.batchId == null) { + handler.postDelayed({ finalize(submission.candidateId) }, 1_650L) + } else { + scheduleBatchProcessing() + } + onStored?.let { callback -> + mainHandler.post { callback(submission) } + } + } catch (error: Exception) { + Log.e(TAG, "Unable to store recognition signal", error) + } finally { + evidenceImage?.fill(0) + } + } + } + + fun attachBatchImage( + submission: StoredSubmission, + signal: PaymentSignal, + image: ByteArray, + ) { + val batchId = submission.batchId + if (batchId == null) { + image.fill(0) + return + } + handler.post { + try { + store.addBatchImage( + batchId, + submission.candidateId, + signal.flowSessionId, + signal.packageName, + System.currentTimeMillis(), + image, + ) + scheduleBatchProcessing() + } finally { + image.fill(0) + } + } + } + + fun submitEvidenceOnly( + packageName: String, + flowSessionId: String?, + capturedAt: Long, + image: ByteArray, + ) { + handler.post { + try { + store.addEvidenceOnly(packageName, flowSessionId, capturedAt, image) + scheduleBatchProcessing() + } finally { + image.fill(0) + } } } @@ -24,12 +106,18 @@ class RecognitionCoordinator private constructor(private val context: Context) { fun acknowledge(id: String, state: String, transactionId: Long?): StoredCandidate? { val candidate = store.acknowledge(id, state, transactionId) - if (candidate != null && state == "imported") { + if (candidate != null && state == "imported" && candidate.batchId == null) { RecognitionNotifier.showImported(context, candidate) } return candidate } + fun recentBatches(): List = store.recentBatches() + + fun restoreDropped(candidateId: String): StoredCandidate? { + return store.restoreDropped(candidateId) + } + fun latestStatus(): String? = store.latestStatus() private fun finalize(id: String) { @@ -42,9 +130,48 @@ class RecognitionCoordinator private constructor(private val context: Context) { RecognitionNotifier.showReady(context, candidate) } + private fun scheduleBatchProcessing() { + handler.removeCallbacks(batchRunnable) + val dueAt = store.nextBatchDueAt() ?: return + handler.postDelayed(batchRunnable, (dueAt - System.currentTimeMillis()).coerceAtLeast(0L)) + } + + private fun processDueBatches() { + while (true) { + val batch = store.claimDueBatch(System.currentTimeMillis()) ?: break + val fallback = when (val result = batchProcessor.process(batch)) { + is BatchProcessResult.Success -> { + val applied = runCatching { + store.applyBatchResponse(batch.id, result.responseBody) + }.onFailure { + Log.e(TAG, "Unable to apply recognition batch ${batch.id}", it) + }.getOrDefault(false) + if (applied) { + false + } else { + store.fallbackBatch(batch.id, "AI 返回结果无法应用") + true + } + } + is BatchProcessResult.Fallback -> { + store.fallbackBatch(batch.id, result.reason) + true + } + } + context.sendBroadcast( + Intent(ACTION_READY) + .setPackage(context.packageName) + .putExtra(EXTRA_BATCH_ID, batch.id), + ) + RecognitionNotifier.showBatchReady(context, batch.id, fallback) + } + scheduleBatchProcessing() + } + companion object { const val ACTION_READY = "com.nx.miaoji.RECOGNITION_READY" const val EXTRA_CANDIDATE_ID = "candidateId" + const val EXTRA_BATCH_ID = "batchId" private const val TAG = "JizhiRecognition" @Volatile diff --git a/frontend/android/app/src/main/kotlin/com/nx/miaoji/RecognitionNotifier.kt b/frontend/android/app/src/main/kotlin/com/nx/miaoji/RecognitionNotifier.kt index e2e4295..6dbb725 100644 --- a/frontend/android/app/src/main/kotlin/com/nx/miaoji/RecognitionNotifier.kt +++ b/frontend/android/app/src/main/kotlin/com/nx/miaoji/RecognitionNotifier.kt @@ -68,6 +68,26 @@ object RecognitionNotifier { notify(context, candidate.id.hashCode(), notification) } + fun showBatchReady(context: Context, batchId: String, fallback: Boolean) { + val intent = Intent(context, MainActivity::class.java).apply { + addFlags(Intent.FLAG_ACTIVITY_NEW_TASK or Intent.FLAG_ACTIVITY_SINGLE_TOP) + putExtra(MainActivity.EXTRA_ACTION, MainActivity.ACTION_RECOGNITION_BATCH_REVIEW) + putExtra(RecognitionCoordinator.EXTRA_BATCH_ID, batchId) + } + val notification = Notification.Builder(context, ensureChannel(context)) + .setSmallIcon(R.mipmap.ic_launcher) + .setContentTitle(if (fallback) "本地识别结果已就绪" else "AI 批次对账已完成") + .setContentText( + if (fallback) "AI 暂时不可用，已按本地结果处理" + else "点按查看本批次的保留、修正、补全和剔除结果", + ) + .setAutoCancel(true) + .setContentIntent(pendingActivity(context, batchId.hashCode(), intent)) + .setCategory(Notification.CATEGORY_STATUS) + .build() + notify(context, batchId.hashCode(), notification) + } + private fun actionIntent(context: Context, candidate: StoredCandidate, action: String) = Intent(context, MainActivity::class.java).apply { addFlags(Intent.FLAG_ACTIVITY_NEW_TASK or Intent.FLAG_ACTIVITY_SINGLE_TOP) diff --git a/frontend/android/app/src/main/kotlin/com/nx/miaoji/RecognitionSettings.kt b/frontend/android/app/src/main/kotlin/com/nx/miaoji/RecognitionSettings.kt index 11bb86c..93d0272 100644 --- a/frontend/android/app/src/main/kotlin/com/nx/miaoji/RecognitionSettings.kt +++ b/frontend/android/app/src/main/kotlin/com/nx/miaoji/RecognitionSettings.kt @@ -81,7 +81,12 @@ object RecognitionSettings { if (token.isNullOrBlank()) { editor.remove(KEY_TOKEN) } else { - val encrypted = NativeCrypto.encrypt(token.toByteArray(Charsets.UTF_8)) + val tokenBytes = token.toByteArray(Charsets.UTF_8) + val encrypted = try { + NativeCrypto.encrypt(tokenBytes) + } finally { + tokenBytes.fill(0) + } if (encrypted != null) editor.putString(KEY_TOKEN, encrypted) } editor.apply() @@ -97,7 +102,12 @@ object RecognitionSettings { fun runtimeToken(context: Context): String? { val encoded = context.getSharedPreferences(PREFS, Context.MODE_PRIVATE) .getString(KEY_TOKEN, null) ?: return null - return NativeCrypto.decrypt(encoded)?.toString(Charsets.UTF_8) + val decrypted = NativeCrypto.decrypt(encoded) ?: return null + return try { + decrypted.toString(Charsets.UTF_8) + } finally { + decrypted.fill(0) + } } fun statusJson(context: Context): String { diff --git a/frontend/android/app/src/main/kotlin/com/nx/miaoji/RecognitionStore.kt b/frontend/android/app/src/main/kotlin/com/nx/miaoji/RecognitionStore.kt index e9b3917..fb64167 100644 --- a/frontend/android/app/src/main/kotlin/com/nx/miaoji/RecognitionStore.kt +++ b/frontend/android/app/src/main/kotlin/com/nx/miaoji/RecognitionStore.kt @@ -6,6 +6,8 @@ import android.database.Cursor import android.database.sqlite.SQLiteDatabase import android.database.sqlite.SQLiteOpenHelper import org.json.JSONObject +import org.json.JSONArray +import java.time.Instant import java.util.UUID import kotlin.math.abs @@ -15,10 +17,29 @@ data class StoredCandidate( val state: String, val transactionId: Long?, val json: String, + val batchId: String? = null, +) + +data class StoredSubmission(val candidateId: String, val batchId: String?) + +data class PendingBatchImage( + val evidenceId: String, + val candidateId: String?, + val flowSessionId: String?, + val packageName: String, + val capturedAt: Long, + val bytes: ByteArray, +) + +data class PendingRecognitionBatch( + val id: String, + val openedAt: Long, + val manifest: String, + val images: List, ) class RecognitionStore(context: Context) : - SQLiteOpenHelper(context, "recognition_queue.db", null, 2) { + SQLiteOpenHelper(context, "recognition_queue.db", null, 3) { override fun onCreate(db: SQLiteDatabase) { db.execSQL( """ @@ -39,7 +60,11 @@ class RecognitionStore(context: Context) : available_at INTEGER NOT NULL, first_seen INTEGER NOT NULL, updated_at INTEGER NOT NULL, - transaction_id INTEGER + transaction_id INTEGER, + batch_id TEXT, + ai_action TEXT, + ai_reason TEXT, + original_payload_encrypted TEXT ) """.trimIndent(), ) @@ -64,6 +89,7 @@ class RecognitionStore(context: Context) : db.execSQL( "CREATE INDEX ix_recognition_state ON candidates(state, available_at)", ) + createBatchTables(db) } override fun onUpgrade(db: SQLiteDatabase, oldVersion: Int, newVersion: Int) { @@ -73,10 +99,50 @@ class RecognitionStore(context: Context) : "ON candidates(state, available_at)", ) } + if (oldVersion < 3) { + db.execSQL("ALTER TABLE candidates ADD COLUMN batch_id TEXT") + db.execSQL("ALTER TABLE candidates ADD COLUMN ai_action TEXT") + db.execSQL("ALTER TABLE candidates ADD COLUMN ai_reason TEXT") + db.execSQL("ALTER TABLE candidates ADD COLUMN original_payload_encrypted TEXT") + createBatchTables(db) + } + } + + private fun createBatchTables(db: SQLiteDatabase) { + db.execSQL( + """ + CREATE TABLE IF NOT EXISTS recognition_batches ( + id TEXT PRIMARY KEY, + state TEXT NOT NULL, + opened_at INTEGER NOT NULL, + last_seen INTEGER NOT NULL, + flush_at INTEGER NOT NULL, + hard_deadline INTEGER NOT NULL, + completed_at INTEGER, + summary_json TEXT, + failure_reason TEXT + ) + """.trimIndent(), + ) + db.execSQL( + """ + CREATE TABLE IF NOT EXISTS batch_images ( + evidence_id TEXT PRIMARY KEY, + batch_id TEXT NOT NULL, + candidate_id TEXT, + flow_session_id TEXT, + package_name TEXT NOT NULL, + captured_at INTEGER NOT NULL, + image_encrypted TEXT NOT NULL + ) + """.trimIndent(), + ) + db.execSQL("CREATE INDEX IF NOT EXISTS ix_candidates_batch ON candidates(batch_id, state)") + db.execSQL("CREATE INDEX IF NOT EXISTS ix_batches_due ON recognition_batches(state, flush_at)") } @Synchronized - fun upsert(signal: PaymentSignal): String { + fun upsert(signal: PaymentSignal, batchMode: Boolean = false): StoredSubmission { val now = System.currentTimeMillis() val channelBit = channelBit(signal.channel) val sourceHash = PaymentParser.sha256(signal.sourceEventId) @@ -88,7 +154,8 @@ class RecognitionStore(context: Context) : ).use { cursor -> if (cursor.moveToFirst()) { writableDatabase.setTransactionSuccessful() - return cursor.getString(0) + val candidateId = cursor.getString(0) + return StoredSubmission(candidateId, batchIdForCandidate(candidateId)) } } @@ -98,15 +165,14 @@ class RecognitionStore(context: Context) : val orderStrongKey = signal.orderId?.takeIf { it.isNotBlank() }?.let { PaymentParser.sha256("${signal.packageName}|$it") } - val flowStrongKey = signal.flowSessionId?.takeIf { it.isNotBlank() }?.let { - PaymentParser.sha256("${signal.packageName}|flow|$it") - } + val flowStrongKey = flowStrongKeyFor(signal) val strongKey = orderStrongKey ?: flowStrongKey val clientRequestId = clientRequestIdFor(signal) val signalHigh = signal.channel in setOf("accessibility", "local_ocr") && signal.evidenceConfidence == "high" val existing = findMergeCandidate(signal, channelBit, merchantHash, strongKey, now) ?: findByClientRequestId(clientRequestId) + val batchId = existing?.batchId ?: if (batchMode) activeBatch(now) else null val id: String if (existing != null) { id = existing.id @@ -128,6 +194,10 @@ class RecognitionStore(context: Context) : if (high && existing.state == "pending_confirm") put("state", "auto_ready") put("updated_at", now) put("available_at", now + MERGE_DELAY_MS) + if (batchId != null) { + put("batch_id", batchId) + put("state", "batch_collecting") + } }, "id = ?", arrayOf(id), @@ -151,11 +221,12 @@ class RecognitionStore(context: Context) : put("known_template", if (signal.knownTemplate) 1 else 0) put("occurred_at", signal.occurredAtEpochMs) put("payload_encrypted", encryptPayload(payload)) - put("state", "pending_merge") + put("state", if (batchId == null) "pending_merge" else "batch_collecting") put("high_confidence", if (high) 1 else 0) put("available_at", now + MERGE_DELAY_MS) put("first_seen", now) put("updated_at", now) + if (batchId != null) put("batch_id", batchId) }, ) } @@ -169,8 +240,9 @@ class RecognitionStore(context: Context) : put("created_at", now) }, ) + if (batchId != null) touchBatch(batchId, now) writableDatabase.setTransactionSuccessful() - return id + return StoredSubmission(id, batchId) } finally { writableDatabase.endTransaction() } @@ -184,6 +256,563 @@ class RecognitionStore(context: Context) : if (cursor.moveToFirst()) row(cursor) else null } + private fun batchIdForCandidate(candidateId: String): String? = readableDatabase.rawQuery( + "SELECT batch_id FROM candidates WHERE id = ? LIMIT 1", + arrayOf(candidateId), + ).use { cursor -> + if (!cursor.moveToFirst() || cursor.isNull(0)) null else cursor.getString(0) + } + + private fun activeBatch(now: Long): String { + readableDatabase.rawQuery( + """ + SELECT b.id, + (SELECT COUNT(*) FROM candidates c WHERE c.batch_id = b.id) + + (SELECT COUNT(*) FROM batch_images i WHERE i.batch_id = b.id AND i.candidate_id IS NULL) + FROM recognition_batches b + WHERE b.state = 'collecting' AND b.hard_deadline > ? + ORDER BY b.opened_at DESC LIMIT 1 + """.trimIndent(), + arrayOf(now.toString()), + ).use { cursor -> + if (cursor.moveToFirst() && cursor.getInt(1) < MAX_BATCH_ITEMS) { + return cursor.getString(0) + } + } + val id = UUID.randomUUID().toString() + writableDatabase.insertOrThrow( + "recognition_batches", + null, + ContentValues().apply { + put("id", id) + put("state", "collecting") + put("opened_at", now) + put("last_seen", now) + put("flush_at", now + BATCH_IDLE_MS) + put("hard_deadline", now + BATCH_HARD_LIMIT_MS) + }, + ) + return id + } + + private fun touchBatch(batchId: String, now: Long) { + writableDatabase.execSQL( + """ + UPDATE recognition_batches + SET last_seen = ?, + flush_at = MIN(hard_deadline, ?) + WHERE id = ? AND state = 'collecting' + """.trimIndent(), + arrayOf(now, now + BATCH_IDLE_MS, batchId), + ) + val itemCount = readableDatabase.rawQuery( + """ + SELECT + (SELECT COUNT(*) FROM candidates WHERE batch_id = ?) + + (SELECT COUNT(*) FROM batch_images WHERE batch_id = ? AND candidate_id IS NULL) + """.trimIndent(), + arrayOf(batchId, batchId), + ).use { cursor -> if (cursor.moveToFirst()) cursor.getInt(0) else 0 } + if (itemCount >= MAX_BATCH_ITEMS) { + writableDatabase.execSQL( + "UPDATE recognition_batches SET flush_at = ? WHERE id = ?", + arrayOf(now, batchId), + ) + } + } + + @Synchronized + fun addBatchImage( + batchId: String, + candidateId: String?, + flowSessionId: String?, + packageName: String, + capturedAt: Long, + image: ByteArray, + ): String? { + if (image.isEmpty() || image.size > MAX_BATCH_IMAGE_BYTES) return null + val state = readableDatabase.rawQuery( + "SELECT state FROM recognition_batches WHERE id = ? LIMIT 1", + arrayOf(batchId), + ).use { cursor -> if (cursor.moveToFirst()) cursor.getString(0) else null } + if (state != "collecting") return null + val existing = readableDatabase.rawQuery( + """ + SELECT evidence_id FROM batch_images + WHERE batch_id = ? AND ( + (? IS NOT NULL AND candidate_id = ?) OR + (? IS NULL AND candidate_id IS NULL AND flow_session_id = ?) + ) LIMIT 1 + """.trimIndent(), + arrayOf(batchId, candidateId, candidateId, candidateId, flowSessionId), + ).use { cursor -> if (cursor.moveToFirst()) cursor.getString(0) else null } + if (existing != null) return existing + val encrypted = NativeCrypto.encrypt(image) ?: return null + val evidenceId = UUID.randomUUID().toString() + writableDatabase.insertOrThrow( + "batch_images", + null, + ContentValues().apply { + put("evidence_id", evidenceId) + put("batch_id", batchId) + if (candidateId == null) putNull("candidate_id") else put("candidate_id", candidateId) + if (flowSessionId == null) putNull("flow_session_id") else put("flow_session_id", flowSessionId) + put("package_name", packageName) + put("captured_at", capturedAt) + put("image_encrypted", encrypted) + }, + ) + touchBatch(batchId, capturedAt) + return evidenceId + } + + @Synchronized + fun addEvidenceOnly( + packageName: String, + flowSessionId: String?, + capturedAt: Long, + image: ByteArray, + ): String? { + writableDatabase.beginTransaction() + return try { + val batchId = activeBatch(capturedAt) + addBatchImage(batchId, null, flowSessionId, packageName, capturedAt, image) + writableDatabase.setTransactionSuccessful() + batchId + } finally { + writableDatabase.endTransaction() + } + } + + @Synchronized + fun nextBatchDueAt(): Long? = readableDatabase.rawQuery( + "SELECT MIN(flush_at) FROM recognition_batches WHERE state = 'collecting'", + null, + ).use { cursor -> + if (!cursor.moveToFirst() || cursor.isNull(0)) null else cursor.getLong(0) + } + + @Synchronized + fun claimDueBatch(now: Long): PendingRecognitionBatch? { + val batch = readableDatabase.rawQuery( + """ + SELECT id, opened_at FROM recognition_batches + WHERE state = 'collecting' AND (flush_at <= ? OR hard_deadline <= ?) + ORDER BY opened_at LIMIT 1 + """.trimIndent(), + arrayOf(now.toString(), now.toString()), + ).use { cursor -> + if (!cursor.moveToFirst()) null else cursor.getString(0) to cursor.getLong(1) + } ?: return null + writableDatabase.update( + "recognition_batches", + ContentValues().apply { put("state", "processing") }, + "id = ? AND state = 'collecting'", + arrayOf(batch.first), + ) + + val images = readableDatabase.rawQuery( + "SELECT * FROM batch_images WHERE batch_id = ? ORDER BY captured_at", + arrayOf(batch.first), + ).use { cursor -> + buildList { + while (cursor.moveToNext()) { + val bytes = NativeCrypto.decrypt(cursor.getString(cursor.getColumnIndexOrThrow("image_encrypted"))) + ?: continue + val candidateIndex = cursor.getColumnIndexOrThrow("candidate_id") + val flowIndex = cursor.getColumnIndexOrThrow("flow_session_id") + add( + PendingBatchImage( + evidenceId = cursor.getString(cursor.getColumnIndexOrThrow("evidence_id")), + candidateId = if (cursor.isNull(candidateIndex)) null else cursor.getString(candidateIndex), + flowSessionId = if (cursor.isNull(flowIndex)) null else cursor.getString(flowIndex), + packageName = cursor.getString(cursor.getColumnIndexOrThrow("package_name")), + capturedAt = cursor.getLong(cursor.getColumnIndexOrThrow("captured_at")), + bytes = bytes, + ), + ) + } + } + } + val evidenceByCandidate = images.filter { it.candidateId != null }.groupBy { it.candidateId } + val candidates = JSONArray() + readableDatabase.rawQuery( + "SELECT * FROM candidates WHERE batch_id = ? AND state = 'batch_collecting' ORDER BY occurred_at, first_seen", + arrayOf(batch.first), + ).use { cursor -> + while (cursor.moveToNext()) { + val payload = decryptPayload(cursor.getString(cursor.getColumnIndexOrThrow("payload_encrypted"))) + ?: continue + candidates.put( + JSONObject() + .put("candidateId", cursor.getString(cursor.getColumnIndexOrThrow("id"))) + .put("clientRequestId", cursor.getString(cursor.getColumnIndexOrThrow("client_request_id"))) + .put("flowSessionId", payload.optString("flowSessionId").takeIf(String::isNotBlank)) + .put("packageName", payload.optString("packageName")) + .put("type", payload.optString("type")) + .put("amount", payload.optDouble("amount")) + .put("merchant", payload.optString("merchant").takeIf(String::isNotBlank)) + .put("orderId", payload.optString("orderId").takeIf(String::isNotBlank)) + .put("occurredAt", Instant.ofEpochMilli(payload.optLong("occurredAtEpochMs")).toString()) + .put("recognitionKind", payload.optString("recognitionKind", "payment")) + .put("categoryHint", payload.optString("categoryHint").takeIf(String::isNotBlank)) + .put("confidence", if (cursor.getInt(cursor.getColumnIndexOrThrow("high_confidence")) == 1) "auto" else "confirm") + .put("sourceText", payload.optString("sourceText").takeIf(String::isNotBlank)) + .put( + "evidenceIds", + JSONArray(evidenceByCandidate[cursor.getString(cursor.getColumnIndexOrThrow("id"))] + .orEmpty().map(PendingBatchImage::evidenceId)), + ), + ) + } + } + val evidence = JSONArray(images.map { image -> + JSONObject() + .put("evidenceId", image.evidenceId) + .put("candidateId", image.candidateId) + .put("flowSessionId", image.flowSessionId) + .put("packageName", image.packageName) + .put("capturedAt", Instant.ofEpochMilli(image.capturedAt).toString()) + }) + val manifest = JSONObject() + .put("batchId", batch.first) + .put("openedAt", Instant.ofEpochMilli(batch.second).toString()) + .put("candidates", candidates) + .put("evidence", evidence) + .toString() + return PendingRecognitionBatch(batch.first, batch.second, manifest, images) + } + + @Synchronized + fun recoverInterruptedBatches() { + val now = System.currentTimeMillis() + writableDatabase.execSQL( + "UPDATE recognition_batches SET state = 'collecting', flush_at = ? WHERE state = 'processing'", + arrayOf(now), + ) + cleanupBatchHistory(now) + } + + @Synchronized + fun applyBatchResponse(batchId: String, responseBody: String): Boolean { + val root = runCatching { JSONObject(responseBody) }.getOrNull() ?: return false + if (root.optString("batchId") != batchId) return false + val actions = root.optJSONArray("actions") ?: return false + val now = System.currentTimeMillis() + writableDatabase.beginTransaction() + return try { + val seenCandidates = HashSet() + var kept = 0 + var updated = 0 + var created = 0 + var dropped = 0 + for (index in 0 until actions.length()) { + val action = actions.optJSONObject(index) ?: continue + when (action.optString("action")) { + "keep", "update", "drop" -> { + val candidateId = action.optString("candidateId") + if (candidateId.isBlank() || !seenCandidates.add(candidateId)) continue + val row = readableDatabase.rawQuery( + "SELECT * FROM candidates WHERE id = ? AND batch_id = ? LIMIT 1", + arrayOf(candidateId, batchId), + ).use { cursor -> if (cursor.moveToFirst()) row(cursor) else null } ?: continue + val original = JSONObject(row.payload.toString()) + val kind = action.optString("action") + val payload = JSONObject(row.payload.toString()) + if (kind == "update") applyActionFields(payload, action) + payload.put("sourceOverride", "recognition_ai") + val reason = action.optString("reason", "AI 对账") + writableDatabase.update( + "candidates", + ContentValues().apply { + put("original_payload_encrypted", encryptPayload(original)) + put("payload_encrypted", encryptPayload(payload)) + put("amount_cents", kotlin.math.round(payload.optDouble("amount") * 100).toLong()) + put("direction", payload.optString("type")) + put("merchant_hash", PaymentParser.sha256(payload.optString("merchant").lowercase())) + put("occurred_at", payload.optLong("occurredAtEpochMs")) + put("channel_mask", row.channelMask or channelBit("recognition_ai")) + put("ai_action", kind) + put("ai_reason", reason.take(80)) + put("state", if (kind == "drop") "ai_dropped" else "auto_ready") + put("high_confidence", 1) + put("updated_at", now) + }, + "id = ? AND batch_id = ?", + arrayOf(candidateId, batchId), + ) + when (kind) { + "keep" -> kept += 1 + "update" -> updated += 1 + "drop" -> dropped += 1 + } + } + "create" -> { + val evidenceId = action.optString("evidenceId") + if (evidenceId.isBlank()) continue + val image = readableDatabase.rawQuery( + "SELECT * FROM batch_images WHERE evidence_id = ? AND batch_id = ? AND candidate_id IS NULL LIMIT 1", + arrayOf(evidenceId, batchId), + ).use { cursor -> + if (!cursor.moveToFirst()) null else Triple( + cursor.getString(cursor.getColumnIndexOrThrow("package_name")), + cursor.getLong(cursor.getColumnIndexOrThrow("captured_at")), + cursor.getColumnIndexOrThrow("flow_session_id").let { flowIndex -> + if (cursor.isNull(flowIndex)) null else cursor.getString(flowIndex) + }, + ) + } ?: continue + val amount = action.optDouble("amount", 0.0) + val type = action.optString("type") + if (amount <= 0 || type !in setOf("income", "expense")) continue + val candidateId = UUID.randomUUID().toString() + val requestId = "recognition-" + PaymentParser.sha256("$batchId|create|$evidenceId").take(52) + val payload = JSONObject() + .put("packageName", image.first) + .put("appName", PaymentParser.appName(image.first)) + .put("type", type) + .put("amount", amount) + .put("merchant", action.optionalString("note")) + .put("orderId", JSONObject.NULL) + .put("occurredAtEpochMs", action.optionalInstantEpoch("occurredAt") ?: image.second) + .put("sourceText", "AI 批次补全 · ${PaymentParser.appName(image.first)}") + .put("flowSessionId", image.third) + .put("evidenceConfidence", "high") + .put("recognitionKind", "payment") + .put("categoryHint", action.optionalString("categoryName")) + .put("categoryId", action.optLong("categoryId").takeIf { it > 0 }) + .put("amountSource", "ai_batch") + .put("resultFingerprint", PaymentParser.sha256("$batchId|$evidenceId")) + .put("note", action.optionalString("note") ?: PaymentParser.appName(image.first)) + .put("paymentMethod", action.optionalString("paymentMethod")) + .put("sourceOverride", "recognition_ai") + writableDatabase.insertOrThrow( + "candidates", + null, + ContentValues().apply { + put("id", candidateId) + put("client_request_id", requestId) + put("package_name", image.first) + put("amount_cents", kotlin.math.round(amount * 100).toLong()) + put("direction", type) + put("merchant_hash", PaymentParser.sha256(payload.optString("merchant").lowercase())) + put("strong_key", PaymentParser.sha256("$batchId|create|$evidenceId")) + put("channel_mask", channelBit("recognition_ai")) + put("known_template", 0) + put("occurred_at", payload.optLong("occurredAtEpochMs")) + put("payload_encrypted", encryptPayload(payload)) + put("state", "auto_ready") + put("high_confidence", 1) + put("available_at", now) + put("first_seen", now) + put("updated_at", now) + put("batch_id", batchId) + put("ai_action", "create") + put("ai_reason", action.optString("reason", "AI 补全").take(80)) + }, + ) + created += 1 + } + } + } + // The server guarantees one action per candidate. Preserve anything omitted + // by a malformed response instead of silently losing a payment. + writableDatabase.rawQuery( + "SELECT id FROM candidates WHERE batch_id = ? AND state = 'batch_collecting'", + arrayOf(batchId), + ).use { cursor -> + while (cursor.moveToNext()) { + writableDatabase.update( + "candidates", + ContentValues().apply { + put("state", "auto_ready") + put("ai_action", "keep") + put("ai_reason", "AI 未返回该候选，已保留本地结果") + put("updated_at", now) + }, + "id = ?", + arrayOf(cursor.getString(0)), + ) + kept += 1 + } + } + val summary = JSONObject() + .put("kept", kept) + .put("updated", updated) + .put("created", created) + .put("dropped", dropped) + .put("fallback", false) + finishBatch(batchId, "ready", summary, null, now) + writableDatabase.setTransactionSuccessful() + true + } finally { + writableDatabase.endTransaction() + } + } + + @Synchronized + fun fallbackBatch(batchId: String, reason: String): Boolean { + val now = System.currentTimeMillis() + writableDatabase.beginTransaction() + return try { + writableDatabase.execSQL( + """ + UPDATE candidates + SET state = CASE WHEN high_confidence = 1 THEN 'auto_ready' ELSE 'pending_confirm' END, + ai_action = 'fallback', ai_reason = ?, updated_at = ? + WHERE batch_id = ? AND state = 'batch_collecting' + """.trimIndent(), + arrayOf(reason.take(80), now, batchId), + ) + val count = readableDatabase.rawQuery( + "SELECT COUNT(*) FROM candidates WHERE batch_id = ? AND state IN ('auto_ready','pending_confirm')", + arrayOf(batchId), + ).use { cursor -> if (cursor.moveToFirst()) cursor.getInt(0) else 0 } + val summary = JSONObject() + .put("kept", count) + .put("updated", 0) + .put("created", 0) + .put("dropped", 0) + .put("fallback", true) + finishBatch(batchId, "fallback", summary, reason, now) + writableDatabase.setTransactionSuccessful() + true + } finally { + writableDatabase.endTransaction() + } + } + + private fun finishBatch( + batchId: String, + state: String, + summary: JSONObject, + failureReason: String?, + now: Long, + ) { + writableDatabase.update( + "recognition_batches", + ContentValues().apply { + put("state", state) + put("completed_at", now) + put("summary_json", summary.toString()) + if (failureReason == null) putNull("failure_reason") else put("failure_reason", failureReason.take(80)) + }, + "id = ?", + arrayOf(batchId), + ) + writableDatabase.delete("batch_images", "batch_id = ?", arrayOf(batchId)) + } + + private fun applyActionFields(payload: JSONObject, action: JSONObject) { + action.optionalString("type")?.takeIf { it in setOf("income", "expense") }?.let { + payload.put("type", it) + } + action.optDouble("amount", 0.0).takeIf { it > 0 }?.let { payload.put("amount", it) } + action.optionalString("note")?.let { + payload.put("merchant", it.take(40)) + payload.put("note", it.take(40)) + } + action.optionalString("paymentMethod")?.let { payload.put("paymentMethod", it.take(40)) } + action.optionalString("categoryName")?.let { payload.put("categoryHint", it.take(40)) } + action.optLong("categoryId").takeIf { it > 0 }?.let { payload.put("categoryId", it) } + action.optionalInstantEpoch("occurredAt")?.let { payload.put("occurredAtEpochMs", it) } + } + + @Synchronized + fun recentBatches(): List { + cleanupBatchHistory(System.currentTimeMillis()) + return readableDatabase.rawQuery( + """ + SELECT id, state, opened_at, completed_at, summary_json, failure_reason + FROM recognition_batches + WHERE completed_at IS NOT NULL + ORDER BY completed_at DESC LIMIT 20 + """.trimIndent(), + null, + ).use { cursor -> + buildList { + while (cursor.moveToNext()) { + val batchId = cursor.getString(0) + val items = JSONArray() + readableDatabase.rawQuery( + "SELECT * FROM candidates WHERE batch_id = ? ORDER BY first_seen", + arrayOf(batchId), + ).use { candidates -> + while (candidates.moveToNext()) { + val payload = decryptPayload(candidates.getString(candidates.getColumnIndexOrThrow("payload_encrypted"))) + ?: continue + val actionIndex = candidates.getColumnIndexOrThrow("ai_action") + val reasonIndex = candidates.getColumnIndexOrThrow("ai_reason") + val action = if (candidates.isNull(actionIndex)) "keep" else candidates.getString(actionIndex) + items.put( + JSONObject() + .put("candidateId", candidates.getString(candidates.getColumnIndexOrThrow("id"))) + .put("action", action) + .put("reason", if (candidates.isNull(reasonIndex)) "" else candidates.getString(reasonIndex)) + .put("type", payload.optString("type")) + .put("amount", payload.optDouble("amount")) + .put("merchant", payload.optString("merchant").takeIf(String::isNotBlank)) + .put("state", candidates.getString(candidates.getColumnIndexOrThrow("state"))) + .put("canRestore", action == "drop" && candidates.getString(candidates.getColumnIndexOrThrow("state")) == "ai_dropped"), + ) + } + } + add( + JSONObject() + .put("id", batchId) + .put("state", cursor.getString(1)) + .put("openedAt", cursor.getLong(2)) + .put("completedAt", if (cursor.isNull(3)) JSONObject.NULL else cursor.getLong(3)) + .put("summary", cursor.getString(4)?.let(::JSONObject) ?: JSONObject()) + .put("failureReason", if (cursor.isNull(5)) JSONObject.NULL else cursor.getString(5)) + .put("items", items) + .toString(), + ) + } + } + } + } + + @Synchronized + fun restoreDropped(candidateId: String): StoredCandidate? { + val changed = writableDatabase.update( + "candidates", + ContentValues().apply { + put("state", "auto_ready") + put("ai_action", "restored") + put("ai_reason", "用户恢复 AI 剔除项") + put("updated_at", System.currentTimeMillis()) + }, + "id = ? AND state = 'ai_dropped'", + arrayOf(candidateId), + ) + return if (changed == 1) loadById(candidateId) else null + } + + private fun cleanupBatchHistory(now: Long) { + val cutoff = now - BATCH_HISTORY_MS + writableDatabase.delete("batch_images", "batch_id IN (SELECT id FROM recognition_batches WHERE completed_at < ?)", arrayOf(cutoff.toString())) + writableDatabase.execSQL( + """ + UPDATE candidates + SET state = CASE WHEN state = 'ai_dropped' THEN 'expired' ELSE state END, + batch_id = NULL, ai_action = NULL, ai_reason = NULL, + original_payload_encrypted = NULL + WHERE batch_id IN ( + SELECT id FROM recognition_batches WHERE completed_at < ? + ) + """.trimIndent(), + arrayOf(cutoff), + ) + writableDatabase.delete("recognition_batches", "completed_at < ?", arrayOf(cutoff.toString())) + } + + private fun JSONObject.optionalString(name: String): String? = + if (!has(name) || isNull(name)) null else optString(name).trim().takeIf(String::isNotBlank) + + private fun JSONObject.optionalInstantEpoch(name: String): Long? = + optionalString(name)?.let { value -> runCatching { Instant.parse(value).toEpochMilli() }.getOrNull() } + @Synchronized fun finalizeCandidate(id: String): StoredCandidate? { val now = System.currentTimeMillis() @@ -277,6 +906,9 @@ class RecognitionStore(context: Context) : ).use { cursor -> if (cursor.moveToFirst()) return row(cursor) } + // A new accessibility/OCR flow is a new payment, even when amount and + // counterparty are identical to another transaction in the same window. + return null } val since = signal.occurredAtEpochMs - NO_ORDER_WINDOW_MS val until = signal.occurredAtEpochMs + NO_ORDER_WINDOW_MS @@ -327,7 +959,17 @@ class RecognitionStore(context: Context) : .put("clientRequestId", cursor.getString(cursor.getColumnIndexOrThrow("client_request_id"))) .put("state", cursor.getString(cursor.getColumnIndexOrThrow("state"))) .put("confidence", if (cursor.getInt(cursor.getColumnIndexOrThrow("high_confidence")) == 1) "auto" else "confirm") - .put("source", sourceFromMask(cursor.getInt(cursor.getColumnIndexOrThrow("channel_mask")))) + .put( + "source", + payload.optString("sourceOverride").takeIf(String::isNotBlank) + ?: sourceFromMask(cursor.getInt(cursor.getColumnIndexOrThrow("channel_mask"))), + ) + val batchIndex = cursor.getColumnIndex("batch_id") + val actionIndex = cursor.getColumnIndex("ai_action") + val reasonIndex = cursor.getColumnIndex("ai_reason") + if (batchIndex >= 0 && !cursor.isNull(batchIndex)) payload.put("batchId", cursor.getString(batchIndex)) + if (actionIndex >= 0 && !cursor.isNull(actionIndex)) payload.put("aiAction", cursor.getString(actionIndex)) + if (reasonIndex >= 0 && !cursor.isNull(reasonIndex)) payload.put("aiReason", cursor.getString(reasonIndex)) val txIndex = cursor.getColumnIndexOrThrow("transaction_id") val txId = if (cursor.isNull(txIndex)) null else cursor.getLong(txIndex) if (txId != null) payload.put("transactionId", txId) @@ -337,6 +979,9 @@ class RecognitionStore(context: Context) : state = payload.getString("state"), transactionId = txId, json = payload.toString(), + batchId = cursor.getColumnIndex("batch_id").takeIf { it >= 0 }?.let { index -> + if (cursor.isNull(index)) null else cursor.getString(index) + }, ) } @@ -352,6 +997,9 @@ class RecognitionStore(context: Context) : updatedAt = cursor.getLong(cursor.getColumnIndexOrThrow("updated_at")), resultFingerprint = payload.optString("resultFingerprint").takeIf(String::isNotBlank), payload = payload, + batchId = cursor.getColumnIndex("batch_id").takeIf { it >= 0 }?.let { index -> + if (cursor.isNull(index)) null else cursor.getString(index) + }, ) } @@ -405,7 +1053,7 @@ class RecognitionStore(context: Context) : private fun expireOld(now: Long) { writableDatabase.execSQL( "UPDATE candidates SET state = 'expired', updated_at = ? " + - "WHERE state NOT IN ('imported','undone','expired') AND first_seen < ?", + "WHERE state NOT IN ('imported','undone','expired','ai_dropped') AND first_seen < ?", arrayOf(now, now - EXPIRE_MS), ) writableDatabase.delete("evidence", "created_at < ?", arrayOf((now - EXPIRE_MS).toString())) @@ -435,6 +1083,7 @@ class RecognitionStore(context: Context) : val updatedAt: Long, val resultFingerprint: String?, val payload: JSONObject, + val batchId: String?, ) companion object { @@ -443,6 +1092,11 @@ class RecognitionStore(context: Context) : private const val NO_ORDER_WINDOW_MS = 90_000L private const val SAME_CHANNEL_DEBOUNCE_MS = 10_000L private const val EXPIRE_MS = 7L * 24L * 60L * 60L * 1000L + private const val BATCH_IDLE_MS = 30_000L + private const val BATCH_HARD_LIMIT_MS = 120_000L + private const val BATCH_HISTORY_MS = 7L * 24L * 60L * 60L * 1000L + private const val MAX_BATCH_ITEMS = 10 + private const val MAX_BATCH_IMAGE_BYTES = 1024 * 1024 internal fun clientRequestIdFor(signal: PaymentSignal): String { val basis = when { @@ -455,5 +1109,10 @@ class RecognitionStore(context: Context) : } return "recognition-${PaymentParser.sha256(basis).take(52)}" } + + internal fun flowStrongKeyFor(signal: PaymentSignal): String? = + signal.flowSessionId?.takeIf { it.isNotBlank() }?.let { + PaymentParser.sha256("${signal.packageName}|flow|$it") + } } } diff --git a/frontend/android/app/src/main/kotlin/com/nx/miaoji/ScreenshotAccessibilityService.kt b/frontend/android/app/src/main/kotlin/com/nx/miaoji/ScreenshotAccessibilityService.kt index 1632782..c94897e 100644 --- a/frontend/android/app/src/main/kotlin/com/nx/miaoji/ScreenshotAccessibilityService.kt +++ b/frontend/android/app/src/main/kotlin/com/nx/miaoji/ScreenshotAccessibilityService.kt @@ -33,6 +33,7 @@ class ScreenshotAccessibilityService : AccessibilityService() { private var ocrInProgress = false private var visualOperationId: String? = null private var visualTimeout: Runnable? = null + private var batchCaptureTimeout: Runnable? = null private data class CollectedPage(val text: String, val nodeCount: Int) @@ -53,6 +54,7 @@ class ScreenshotAccessibilityService : AccessibilityService() { var resultPageHash: String? = null, var resultFingerprint: String? = null, var completed: Boolean = false, + var resultSurfaceExited: Boolean = false, var retryCount: Int = 0, var probeCount: Int = 0, ) @@ -146,6 +148,9 @@ class ScreenshotAccessibilityService : AccessibilityService() { val resultSurface = status.strength != PaymentStatusStrength.NONE || (existingFlow?.kind == "red_packet_send" && PaymentParser.hasRedPacketSentSurface(combined)) + if (existingFlow?.completed == true && !resultSurface && combined.isNotBlank()) { + existingFlow.resultSurfaceExited = true + } if (existingFlow?.completed == true && resultSurface) { val currentKind = if (existingFlow.kind == "red_packet_send" && PaymentParser.hasRedPacketSentSurface(combined) @@ -160,7 +165,9 @@ class ScreenshotAccessibilityService : AccessibilityService() { val sameOutgoingResult = currentKind in setOf("payment", "transfer") && existingFlow.kind in setOf("payment", "transfer") - if (currentKind == existingFlow.kind || sameOutgoingResult) { + if ((currentKind == existingFlow.kind || sameOutgoingResult) && + shouldSuppressCompletedResult(existingFlow.resultSurfaceExited) + ) { RecognitionDiagnostics.record( this, recognizedPackage, @@ -282,6 +289,8 @@ class ScreenshotAccessibilityService : AccessibilityService() { pendingVisualCapture = null visualTimeout?.let(handler::removeCallbacks) visualTimeout = null + batchCaptureTimeout?.let(handler::removeCallbacks) + batchCaptureTimeout = null visualOperationId = null captureInProgress = false ocrInProgress = false @@ -392,20 +401,43 @@ class ScreenshotAccessibilityService : AccessibilityService() { nodeCount: Int, stage: String, recordDiagnostic: Boolean = true, + evidenceImage: ByteArray? = null, ) { - if (flow.completed) return + if (flow.completed) { + evidenceImage?.fill(0) + return + } flow.completed = true flow.resultFingerprint = signal.resultFingerprint - RecognitionCoordinator.get(this).submit(signal) + val coordinator = RecognitionCoordinator.get(this) + val settings = RecognitionSettings.snapshot(this) + val batchEnabled = settings.aiScreenshot && settings.aiAllowed && settings.hasAccount + if (evidenceImage != null) { + coordinator.submit(signal, evidenceImage) + } else if (batchEnabled) { + coordinator.submit(signal) { submission -> + if (submission.batchId != null) { + captureBatchEvidence(signal, submission) + } + } + } else { + coordinator.submit(signal) + } if (recordDiagnostic) { RecognitionDiagnostics.record( this, signal.packageName, stage = stage, - result = if (signal.evidenceConfidence == "high") "auto_ready" else "confirm", + result = if (batchEnabled) { + "batched" + } else if (signal.evidenceConfidence == "high") { + "auto_ready" + } else { + "confirm" + }, nodeCount = nodeCount, amountCandidates = 1, - reason = "success", + reason = if (batchEnabled) "queued_for_ai" else "success", expectedAmountMatched = flow.expectedAmountCents?.let { it == signal.amountCents }, @@ -695,12 +727,21 @@ class ScreenshotAccessibilityService : AccessibilityService() { ) val signal = outcome.signal if (signal != null) { + val evidence = if (RecognitionSettings.snapshot(this).let { + it.aiScreenshot && it.aiAllowed && it.hasAccount + } + ) { + runCatching { bitmapToBatchBytes(bitmap) }.getOrNull() + } else { + null + } submitOnce( signal, flow, nodeCount, "local_ocr", recordDiagnostic = false, + evidenceImage = evidence, ) return } @@ -725,8 +766,13 @@ class ScreenshotAccessibilityService : AccessibilityService() { val settings = RecognitionSettings.snapshot(this) if (!settings.aiScreenshot || !settings.aiAllowed || !settings.hasAccount) return runCatching { - val bytes = bitmapToBytes(bitmap) - BackgroundAiRecognizer.analyze(this, flow.packageName, bytes, flow.id) + val bytes = bitmapToBatchBytes(bitmap) + RecognitionCoordinator.get(this).submitEvidenceOnly( + flow.packageName, + flow.id, + System.currentTimeMillis(), + bytes, + ) }.onFailure { RecognitionDiagnostics.record( this, @@ -738,6 +784,71 @@ class ScreenshotAccessibilityService : AccessibilityService() { } } + private fun captureBatchEvidence(signal: PaymentSignal, submission: StoredSubmission) { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.R || captureInProgress) return + captureInProgress = true + var finished = false + fun finish(): Boolean { + if (finished) return false + finished = true + batchCaptureTimeout?.let(handler::removeCallbacks) + batchCaptureTimeout = null + captureInProgress = false + return true + } + batchCaptureTimeout = Runnable { + if (finish()) { + RecognitionDiagnostics.record( + this, + signal.packageName, + stage = "ai_batch_capture", + result = "failed", + reason = "capture_timeout", + ) + } + }.also { handler.postDelayed(it, CAPTURE_CALLBACK_TIMEOUT_MS) } + val callback = object : TakeScreenshotCallback { + override fun onSuccess(screenshot: ScreenshotResult) { + if (!finish()) { + screenshot.hardwareBuffer.close() + return + } + var bitmap: Bitmap? = null + runCatching { + bitmap = copyBitmap(screenshot) + val bytes = bitmapToBatchBytes(requireNotNull(bitmap)) + RecognitionCoordinator.get(this@ScreenshotAccessibilityService) + .attachBatchImage(submission, signal, bytes) + }.onFailure { + RecognitionDiagnostics.record( + this@ScreenshotAccessibilityService, + signal.packageName, + stage = "ai_batch_capture", + result = "failed", + reason = "image_encode_failed", + ) + } + bitmap?.recycle() + } + + override fun onFailure(errorCode: Int) { + if (!finish()) return + RecognitionDiagnostics.record( + this@ScreenshotAccessibilityService, + signal.packageName, + stage = "ai_batch_capture", + result = "failed", + reason = screenshotDiagnosticReason(errorCode), + ) + } + } + runCatching { + takeScreenshot(Display.DEFAULT_DISPLAY, mainExecutor, callback) + }.onFailure { + finish() + } + } + private fun handleVisualFailure(flow: PaymentFlow, nodeCount: Int, reason: String) { RecognitionDiagnostics.record( this, @@ -918,13 +1029,43 @@ class ScreenshotAccessibilityService : AccessibilityService() { } } - private fun bitmapToBytes(bitmap: Bitmap): ByteArray = - ByteArrayOutputStream().use { output -> - check(bitmap.compress(Bitmap.CompressFormat.PNG, 92, output)) { - "无法编码截屏" - } - output.toByteArray() + private fun bitmapToBatchBytes(bitmap: Bitmap): ByteArray { + val longest = maxOf(bitmap.width, bitmap.height) + val scaled = if (longest > BATCH_IMAGE_MAX_EDGE) { + val ratio = BATCH_IMAGE_MAX_EDGE.toDouble() / longest + Bitmap.createScaledBitmap( + bitmap, + (bitmap.width * ratio).toInt().coerceAtLeast(1), + (bitmap.height * ratio).toInt().coerceAtLeast(1), + true, + ) + } else { + bitmap } + return try { + var quality = 82 + var bytes = ByteArray(0) + try { + do { + bytes.fill(0) + bytes = ByteArrayOutputStream().use { output -> + check(scaled.compress(Bitmap.CompressFormat.JPEG, quality, output)) { + "无法编码批次截图" + } + output.toByteArray() + } + quality -= 10 + } while (bytes.size > BATCH_IMAGE_MAX_BYTES && quality >= 52) + check(bytes.size <= BATCH_IMAGE_MAX_BYTES) { "批次截图压缩后仍然过大" } + bytes + } catch (error: Exception) { + bytes.fill(0) + throw error + } + } finally { + if (scaled !== bitmap) scaled.recycle() + } + } private fun copyBitmap(screenshot: ScreenshotResult): Bitmap { val buffer = screenshot.hardwareBuffer @@ -976,6 +1117,8 @@ class ScreenshotAccessibilityService : AccessibilityService() { private const val MAX_TREE_NODES = 160 private const val MAX_CHILDREN_PER_NODE = 40 private const val MAX_TEXT_CHARS = 8_000 + private const val BATCH_IMAGE_MAX_EDGE = 1280 + private const val BATCH_IMAGE_MAX_BYTES = 900 * 1024 private val MAJOR_WINDOW_EVENTS = setOf( AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED, AccessibilityEvent.TYPE_WINDOWS_CHANGED, @@ -991,6 +1134,9 @@ class ScreenshotAccessibilityService : AccessibilityService() { @Volatile private var activeInstance: ScreenshotAccessibilityService? = null + internal fun shouldSuppressCompletedResult(resultSurfaceExited: Boolean): Boolean = + !resultSurfaceExited + @Volatile var isConnected = false private set @@ -1010,4 +1156,3 @@ class ScreenshotAccessibilityService : AccessibilityService() { } } } - diff --git a/frontend/android/app/src/test/kotlin/com/nx/miaoji/PaymentParserTest.kt b/frontend/android/app/src/test/kotlin/com/nx/miaoji/PaymentParserTest.kt index 4708709..023f53c 100644 --- a/frontend/android/app/src/test/kotlin/com/nx/miaoji/PaymentParserTest.kt +++ b/frontend/android/app/src/test/kotlin/com/nx/miaoji/PaymentParserTest.kt @@ -301,6 +301,47 @@ class PaymentParserTest { ) } + @Test + fun completedResultStaysDeduplicatedUntilTheSurfaceIsExited() { + assertTrue(ScreenshotAccessibilityService.shouldSuppressCompletedResult(false)) + assertFalse(ScreenshotAccessibilityService.shouldSuppressCompletedResult(true)) + } + + @Test + fun consecutiveIdenticalTransfersKeepDistinctFlowIdentities() { + val first = paymentSignal( + channel = "accessibility", + sourceEventId = "a:wechat:transfer-1", + flowSessionId = "transfer-1", + ) + val second = paymentSignal( + channel = "accessibility", + sourceEventId = "a:wechat:transfer-2", + flowSessionId = "transfer-2", + ) + val third = paymentSignal( + channel = "accessibility", + sourceEventId = "a:wechat:transfer-3", + flowSessionId = "transfer-3", + ).copy(amountCents = 3_000L) + + assertNotEquals( + RecognitionStore.flowStrongKeyFor(first), + RecognitionStore.flowStrongKeyFor(second), + ) + assertNotEquals( + RecognitionStore.flowStrongKeyFor(second), + RecognitionStore.flowStrongKeyFor(third), + ) + assertEquals( + 3, + listOf(first, second, third) + .map(RecognitionStore::flowStrongKeyFor) + .toSet() + .size, + ) + } + private fun paymentSignal( channel: String, sourceEventId: String, diff --git a/frontend/lib/app/app.dart b/frontend/lib/app/app.dart index d086496..545086f 100644 --- a/frontend/lib/app/app.dart +++ b/frontend/lib/app/app.dart @@ -19,6 +19,7 @@ import 'package:miaoji_zhang/features/settings/category_manage_page.dart'; import 'package:miaoji_zhang/features/settings/companion_page.dart'; import 'package:miaoji_zhang/features/settings/me_page.dart'; import 'package:miaoji_zhang/features/settings/recycle_bin_page.dart'; +import 'package:miaoji_zhang/features/settings/recognition_batch_page.dart'; import 'package:miaoji_zhang/features/settings/legal_document_page.dart'; import 'package:miaoji_zhang/features/settings/screenshot_settings_page.dart'; import 'package:miaoji_zhang/features/settings/sync_conflicts_page.dart'; @@ -91,6 +92,12 @@ final router = GoRouter( path: '/screenshot-settings', builder: (_, __) => const ScreenshotSettingsPage(), ), + GoRoute( + path: '/recognition-batches', + builder: (_, state) => RecognitionBatchPage( + initialBatchId: state.uri.queryParameters['batchId'], + ), + ), StatefulShellRoute.indexedStack( builder: (_, __, shell) => MainShell(shell: shell), branches: [ diff --git a/frontend/lib/features/settings/legal_document_page.dart b/frontend/lib/features/settings/legal_document_page.dart index a8982af..70e2dc4 100644 --- a/frontend/lib/features/settings/legal_document_page.dart +++ b/frontend/lib/features/settings/legal_document_page.dart @@ -19,7 +19,7 @@ class LegalDocumentPage extends StatelessWidget { final LegalDocumentKind kind; const LegalDocumentPage({super.key, required this.kind}); - static const _effectiveDate = '2026 年 7 月 21 日'; + static const _effectiveDate = '2026 年 7 月 25 日'; @override Widget build(BuildContext context) { @@ -83,7 +83,7 @@ class LegalDocumentPage extends StatelessWidget { ), _LegalSection( '二、语音、图片与 AI 数据', - '使用语音记账时，麦克风音频由设备系统语音识别能力处理，记之接收识别后的文字；使用拍照或相册识别时，只有你主动选择的图片会用于本次识别。开启无障碍事件识别后，记之会在微信、支付宝疑似支付流程结束时按需截取当前页面，并由设备内置 OCR 在内存中识别，图片不落盘且处理后立即释放。只有你另行开启 AI 截图补全时，当前支付页图片才会发送至我们配置的火山方舟大模型服务。我们不会将完整账单历史无差别发送给模型，也不会把这些数据用于广告画像。', + '使用语音记账时，麦克风音频由设备系统语音识别能力处理，记之接收识别后的文字；使用拍照或相册识别时，只有你主动选择的图片会用于本次识别。开启无障碍事件识别后，记之会在微信、支付宝疑似支付流程结束时按需截取当前页面，并由设备内置 OCR 在内存中识别。只有你另行开启 AI 批次对账时，支付结果页截图与本地候选才会按 30 秒空闲窗口成批发送至我们配置的火山方舟大模型服务，单批最长等待 2 分钟或累计 10 条。AI 仅可对当前批次执行保留、修正、补全或剔除，不能改动批次外账单。待提交图片只在设备上临时加密保存，批次完成或失败回退后立即删除。我们不会将完整账单历史无差别发送给模型，也不会把这些数据用于广告画像。', ), _LegalSection( '三、设备、网络与日志', @@ -91,7 +91,7 @@ class LegalDocumentPage extends StatelessWidget { ), _LegalSection( '四、存储期限与安全', - '账号数据在你使用服务期间保存。你删除的账单进入 30 天回收站；截屏识别文件在完成、取消或失败后清理，遗留文件会在超过 24 小时后清理。账号注销进入 15 天后悔期，到期后永久删除账号关联数据。导出文件由你主动分享，应用会在分享完成或失败后清理临时副本。', + '账号数据在你使用服务期间保存。你删除的账单进入 30 天回收站；AI 批次图片在批次完成或失败回退后立即删除，最近 7 天仅保留不含图片的批次操作摘要用于核对与恢复。手动截屏识别文件在完成、取消或失败后清理，遗留文件会在超过 24 小时后清理。账号注销进入 15 天后悔期，到期后永久删除账号关联数据。导出文件由你主动分享，应用会在分享完成或失败后清理临时副本。', ), _LegalSection( '五、你的权利', @@ -138,7 +138,7 @@ class LegalDocumentPage extends StatelessWidget { ), _LegalSection( '无障碍服务', - '可选权限。用于快捷磁贴静默截屏，以及在你主动开启“无障碍事件识别”后，仅处理微信、支付宝的支付流程事件、当前页面可见文字和按需本地截图 OCR。不会监听或拦截音量键，不会保存完整控件树或本地 OCR 截图；关闭后基础手工记账仍可使用。', + '可选权限。用于快捷磁贴静默截屏，以及在你主动开启“无障碍事件识别”后，仅处理微信、支付宝的支付流程事件、当前页面可见文字和按需本地截图 OCR。不会监听或拦截音量键，不会保存完整控件树；未开启 AI 批次对账时，本地 OCR 截图只在内存中处理。开启 AI 批次对账后，支付结果页会临时加密保存并按批上传，批次结束立即删除。关闭后基础手工记账仍可使用。', ), _LegalSection( '屏幕录制 / 截屏授权', @@ -176,7 +176,7 @@ class LegalDocumentPage extends StatelessWidget { ), _LegalSection( '火山方舟大模型服务（字节跳动）', - '用于 AI 对话、账单文本解析、图片识别及预算草稿调整。会处理完成对应请求所需的文本、图片和最小化财务上下文，不接收密码、JWT 或 API Key。', + '用于 AI 对话、账单文本解析、图片识别、无障碍支付结果批次对账及预算草稿调整。会处理完成对应请求所需的文本、图片和最小化财务上下文，不接收密码、JWT 或 API Key。批次对账仅处理当前批次，图片在处理结束后由应用删除。', ), _LegalSection( 'Android / iOS 系统能力', diff --git a/frontend/lib/features/settings/recognition_batch_page.dart b/frontend/lib/features/settings/recognition_batch_page.dart new file mode 100644 index 0000000..b1982d5 --- /dev/null +++ b/frontend/lib/features/settings/recognition_batch_page.dart @@ -0,0 +1,316 @@ +import 'package:flutter/material.dart'; +import 'package:miaoji_zhang/shared/api/api_client.dart'; +import 'package:miaoji_zhang/shared/services/recognition_import_service.dart'; +import 'package:miaoji_zhang/shared/services/screenshot_channel.dart'; +import 'package:miaoji_zhang/shared/services/shanghai_time.dart'; +import 'package:miaoji_zhang/shared/theme/app_theme.dart'; +import 'package:miaoji_zhang/shared/widgets/app_controls.dart'; + +class RecognitionBatchPage extends StatefulWidget { + final String? initialBatchId; + + const RecognitionBatchPage({super.key, this.initialBatchId}); + + @override + State createState() => _RecognitionBatchPageState(); +} + +class _RecognitionBatchPageState extends State { + List _batches = const []; + bool _loading = true; + String? _error; + String? _restoringId; + String? _confirmingId; + + @override + void initState() { + super.initState(); + _load(); + } + + Future _load() async { + try { + final batches = await ScreenshotChannel.listRecognitionBatches(); + if (!mounted) return; + setState(() { + _batches = batches; + _loading = false; + _error = null; + }); + } catch (error) { + if (!mounted) return; + setState(() { + _loading = false; + _error = apiErrorMessage(error); + }); + } + } + + Future _restore(RecognitionBatchItem item) async { + setState(() => _restoringId = item.candidateId); + try { + final restored = await ScreenshotChannel.restoreDroppedRecognition( + item.candidateId, + ); + if (!restored) throw StateError('这条候选已恢复或已过期'); + await RecognitionImportService.importAutomatic(); + await _load(); + if (mounted) { + ScaffoldMessenger.of( + context, + ).showSnackBar(const SnackBar(content: Text('候选已恢复并入账'))); + } + } catch (error) { + if (mounted) { + ScaffoldMessenger.of( + context, + ).showSnackBar(SnackBar(content: Text(apiErrorMessage(error)))); + } + } finally { + if (mounted) setState(() => _restoringId = null); + } + } + + Future _confirm(RecognitionBatchItem item) async { + setState(() => _confirmingId = item.candidateId); + try { + await RecognitionImportService.handleAction(context, { + 'action': 'recognition_confirm', + 'candidateId': item.candidateId, + }); + await _load(); + } finally { + if (mounted) setState(() => _confirmingId = null); + } + } + + @override + Widget build(BuildContext context) { + return Scaffold( + appBar: AppBar(title: const Text('最近 AI 对账')), + body: RefreshIndicator(onRefresh: _load, child: _body()), + ); + } + + Widget _body() { + if (_loading) { + return const Center(child: CircularProgressIndicator()); + } + if (_error != null) { + return ListView( + children: [ + SizedBox(height: MediaQuery.sizeOf(context).height * 0.28), + Center( + child: Text(_error!, style: TextStyle(color: context.jz.text2)), + ), + const SizedBox(height: 12), + Center( + child: JzActionButton( + label: '重试', + secondary: true, + onPressed: _load, + ), + ), + ], + ); + } + if (_batches.isEmpty) { + return ListView( + children: [ + SizedBox(height: MediaQuery.sizeOf(context).height * 0.3), + Icon(Icons.fact_check_outlined, size: 42, color: context.jz.text3), + const SizedBox(height: 12), + Center( + child: Text( + '暂无 AI 对账记录', + style: TextStyle(color: context.jz.text3), + ), + ), + ], + ); + } + return ListView.separated( + padding: const EdgeInsets.fromLTRB(16, 8, 16, 28), + itemCount: _batches.length, + separatorBuilder: (_, _) => const SizedBox(height: 10), + itemBuilder: (_, index) => _batchCard(_batches[index]), + ); + } + + Widget _batchCard(RecognitionBatch batch) { + final count = batch.kept + batch.updated + batch.created + batch.dropped; + return Card( + clipBehavior: Clip.antiAlias, + child: ExpansionTile( + initiallyExpanded: batch.id == widget.initialBatchId, + leading: Icon( + batch.fallback + ? Icons.offline_bolt_outlined + : Icons.auto_fix_high_rounded, + color: batch.fallback ? AppTheme.orange : AppTheme.primary, + ), + title: Text( + '${ShanghaiTime.formatDateTime(batch.completedAt)} · $count 条候选', + style: const TextStyle(fontSize: 14, fontWeight: FontWeight.w700), + ), + subtitle: Padding( + padding: const EdgeInsets.only(top: 4), + child: Text( + batch.fallback + ? 'AI 不可用，已按本地识别结果处理' + : '保留 ${batch.kept} · 修正 ${batch.updated} · 补全 ${batch.created} · 剔除 ${batch.dropped}', + style: TextStyle(fontSize: 11.5, color: context.jz.text2), + ), + ), + children: [ + Divider(height: 1, color: context.jz.line), + for (var index = 0; index < batch.items.length; index++) ...[ + _batchItem(batch.items[index]), + if (index != batch.items.length - 1) + Divider(height: 1, indent: 54, color: context.jz.line), + ], + if (batch.items.isEmpty) + Padding( + padding: const EdgeInsets.all(16), + child: Text( + '本批次没有候选明细', + style: TextStyle(color: context.jz.text3), + ), + ), + ], + ), + ); + } + + Widget _batchItem(RecognitionBatchItem item) { + final action = _actionDisplay(item.action); + final restoring = _restoringId == item.candidateId; + final confirming = _confirmingId == item.candidateId; + return Padding( + padding: const EdgeInsets.fromLTRB(16, 13, 12, 13), + child: Row( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + SizedBox( + width: 30, + height: 30, + child: Icon(action.icon, size: 19, color: action.color), + ), + const SizedBox(width: 8), + Expanded( + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Row( + children: [ + Expanded( + child: Text( + item.merchant?.trim().isNotEmpty == true + ? item.merchant!.trim() + : '智能识别账单', + maxLines: 1, + overflow: TextOverflow.ellipsis, + style: const TextStyle( + fontSize: 13, + fontWeight: FontWeight.w700, + ), + ), + ), + const SizedBox(width: 8), + Text( + '${item.type == 'income' ? '+' : '-'}¥${item.amount.toStringAsFixed(2)}', + style: TextStyle( + fontSize: 13, + fontWeight: FontWeight.w700, + color: item.type == 'income' + ? AppTheme.primary + : AppTheme.red, + ), + ), + ], + ), + const SizedBox(height: 4), + Text( + '${action.label}${item.reason.isEmpty ? '' : ' · ${item.reason}'}', + style: TextStyle( + fontSize: 11.5, + height: 1.45, + color: context.jz.text2, + ), + ), + ], + ), + ), + if (item.canRestore) ...[ + const SizedBox(width: 8), + IconButton( + tooltip: '恢复并入账', + onPressed: restoring ? null : () => _restore(item), + icon: restoring + ? const SizedBox( + width: 18, + height: 18, + child: CircularProgressIndicator(strokeWidth: 2), + ) + : const Icon(Icons.restore_rounded), + ), + ] else if (item.state == 'pending_confirm') ...[ + const SizedBox(width: 8), + IconButton( + tooltip: '确认入账', + onPressed: confirming ? null : () => _confirm(item), + icon: confirming + ? const SizedBox( + width: 18, + height: 18, + child: CircularProgressIndicator(strokeWidth: 2), + ) + : const Icon(Icons.check_rounded), + ), + ], + ], + ), + ); + } + + _ActionDisplay _actionDisplay(String action) => switch (action) { + 'update' => const _ActionDisplay( + 'AI 已修正', + Icons.edit_note_rounded, + AppTheme.orange, + ), + 'create' => const _ActionDisplay( + 'AI 已补全', + Icons.add_circle_outline_rounded, + AppTheme.primary, + ), + 'drop' => const _ActionDisplay( + 'AI 已剔除', + Icons.remove_circle_outline_rounded, + AppTheme.red, + ), + 'restored' => const _ActionDisplay( + '已手动恢复', + Icons.restore_rounded, + AppTheme.primary, + ), + 'fallback' => const _ActionDisplay( + '本地结果', + Icons.offline_bolt_outlined, + AppTheme.orange, + ), + _ => const _ActionDisplay( + 'AI 已保留', + Icons.check_circle_outline_rounded, + AppTheme.primary, + ), + }; +} + +class _ActionDisplay { + final String label; + final IconData icon; + final Color color; + + const _ActionDisplay(this.label, this.icon, this.color); +} diff --git a/frontend/lib/features/settings/screenshot_settings_page.dart b/frontend/lib/features/settings/screenshot_settings_page.dart index 76dadbc..ec64ff3 100644 --- a/frontend/lib/features/settings/screenshot_settings_page.dart +++ b/frontend/lib/features/settings/screenshot_settings_page.dart @@ -1,6 +1,7 @@ import 'dart:async'; import 'package:flutter/material.dart'; +import 'package:go_router/go_router.dart'; import 'package:shared_preferences/shared_preferences.dart'; import 'package:miaoji_zhang/shared/services/screenshot_channel.dart'; import 'package:miaoji_zhang/shared/services/session_store.dart'; @@ -147,6 +148,9 @@ class _ScreenshotSettingsPageState extends State !await _confirmLocalOcrConsent()) { return; } + if (enabled && key == 'ai_screenshot' && !await _confirmAiBatchConsent()) { + return; + } if (!enabled) { _pendingAuthorizationKey = null; final changed = await ScreenshotChannel.setRecognitionToggle(key, false); @@ -275,6 +279,71 @@ class _ScreenshotSettingsPageState extends State return false; } + Future _confirmAiBatchConsent() async { + final preferences = await SharedPreferences.getInstance(); + if (preferences.getBool('ai_batch_consent_v2') == true) return true; + if (!mounted) return false; + final accepted = await showModalBottomSheet( + context: context, + useSafeArea: true, + isScrollControlled: true, + backgroundColor: Colors.transparent, + builder: (sheetContext) => Container( + padding: const EdgeInsets.fromLTRB(20, 0, 20, 20), + decoration: BoxDecoration( + color: sheetContext.jz.card, + borderRadius: const BorderRadius.vertical(top: Radius.circular(24)), + ), + child: Column( + mainAxisSize: MainAxisSize.min, + children: [ + const JzSheetHeader( + title: '启用 AI 批次对账', + subtitle: '请确认支付页截图的批量处理方式', + ), + const SizedBox(height: 12), + const _ConsentPoint( + icon: Icons.schedule_rounded, + text: '支付结果会按 30 秒空闲窗口归为一批，最长等待 2 分钟或累计 10 条。', + ), + const _ConsentPoint( + icon: Icons.auto_fix_high_rounded, + text: '本批截图和本地候选会发送给 AI，仅允许在当前批次内保留、修正、补全或剔除。', + ), + const _ConsentPoint( + icon: Icons.enhanced_encryption_outlined, + text: '待提交图片仅在本机临时加密保存，批次完成或回退后立即删除，不进入最近对账记录。', + ), + const SizedBox(height: 16), + Row( + children: [ + Expanded( + child: JzActionButton( + label: '暂不开启', + secondary: true, + onPressed: () => Navigator.pop(sheetContext, false), + ), + ), + const SizedBox(width: 10), + Expanded( + child: JzActionButton( + label: '同意并开启', + onPressed: () => Navigator.pop(sheetContext, true), + ), + ), + ], + ), + ], + ), + ), + ); + if (accepted == true) { + await preferences.setBool('ai_batch_consent_v2', true); + return true; + } + return false; + } + void _showMessage(String message) { ScaffoldMessenger.of( context, @@ -413,24 +482,46 @@ class _ScreenshotSettingsPageState extends State _RecognitionCard( icon: Icons.document_scanner_outlined, title: 'AI 截图补全', - description: '本地 OCR 已确认支付成功但字段仍不足时才在线分析。截图完成、失败或超时后立即释放。', + description: '支付结果按 30 秒空闲窗口批量提交，AI 只在本批次内纠错、补全或剔除。处理结束立即删除图片。', authorized: aiAvailable, connected: aiAvailable && status.accessibilityConnected, statusLabel: !aiAvailable ? 'AI 不可用' : null, onOpenSettings: status.accessibilityAuthorized ? null : ScreenshotChannel.openAccessibilitySettings, - child: JzSwitchTile( - value: status.aiScreenshot, - title: '补全开关', - subtitle: !aiAvailable - ? '需要登录且账号具备 AI 权限' - : !status.accessibilityAuthorized - ? '需要先授权无障碍截屏能力' - : '默认关闭，仅在支付应用前台运行', - onChanged: !aiAvailable - ? null - : (value) => _toggle('ai_screenshot', value), + child: Column( + children: [ + JzSwitchTile( + value: status.aiScreenshot, + title: '批次对账开关', + subtitle: !aiAvailable + ? '需要登录且账号具备 AI 权限' + : !status.accessibilityAuthorized + ? '需要先授权无障碍截屏能力' + : '默认关闭，仅在支付应用前台运行', + onChanged: !aiAvailable + ? null + : (value) => _toggle('ai_screenshot', value), + ), + Divider(height: 1, color: context.jz.line), + InkWell( + onTap: () => context.push('/recognition-batches'), + child: const Padding( + padding: EdgeInsets.symmetric( + horizontal: 4, + vertical: 13, + ), + child: Row( + children: [ + Icon(Icons.fact_check_outlined, size: 20), + SizedBox(width: 10), + Expanded(child: Text('最近 AI 对账')), + Icon(Icons.chevron_right_rounded), + ], + ), + ), + ), + ], ), ), if (status.latestDiagnostic != null) ...[ @@ -487,7 +578,7 @@ class _ScreenshotSettingsPageState extends State const SizedBox(width: 9), Expanded( child: Text( - '银行密码页等安全窗口由系统禁止截屏，记之不会绕过限制。本地 OCR 图片仅在内存中处理；只有你开启 AI 截图补全时才会上传当前支付页。', + '银行密码页等安全窗口由系统禁止截屏，记之不会绕过限制。本地 OCR 图片仅在内存中处理；开启 AI 批次对账后，支付结果页会临时加密并按批上传，处理结束立即删除。', style: TextStyle( color: palette.text2, fontSize: 11.5, diff --git a/frontend/lib/shared/api/business_api.dart b/frontend/lib/shared/api/business_api.dart index d031d5b..655a638 100644 --- a/frontend/lib/shared/api/business_api.dart +++ b/frontend/lib/shared/api/business_api.dart @@ -185,6 +185,27 @@ class PeriodStats { analysis = json['analysis'] as String?; } +class RecognitionBatchDraft { + final String candidateId, clientRequestId, type, source; + final int categoryId; + final double amount; + final DateTime occurredAt; + final String? note, paymentMethod, sourceText; + + const RecognitionBatchDraft({ + required this.candidateId, + required this.clientRequestId, + required this.categoryId, + required this.type, + required this.amount, + required this.occurredAt, + required this.source, + this.note, + this.paymentMethod, + this.sourceText, + }); +} + class TxApi { static final _dio = ApiClient.instance.dio; @@ -361,6 +382,86 @@ class TxApi { } } + static Future> createRecognitionBatch( + String batchId, + List drafts, + ) async { + if (drafts.isEmpty) return const {}; + final localPayloads = drafts + .map( + (draft) => { + 'ledgerId': _ledgerId, + 'categoryId': draft.categoryId, + 'type': draft.type, + 'amount': draft.amount, + 'note': draft.note, + 'paymentMethod': draft.paymentMethod, + 'source': draft.source, + 'sourceText': draft.sourceText, + 'occurredAt': ShanghaiTime.civilToUtc( + draft.occurredAt, + ).toIso8601String(), + 'clientRequestId': draft.clientRequestId, + }, + ) + .toList(growable: false); + + Map createLocal() { + final values = LocalDatabase.instance.createTransactionsBatch( + localPayloads, + enqueueSyncChanges: _queueOfflineChanges, + ); + return { + for (var index = 0; index < drafts.length; index++) + drafts[index].candidateId: TxItem.fromJson(values[index]), + }; + } + + final session = SessionStore.instance; + if (session.shouldUseLocalOnly || + _ledgerId < 0 || + drafts.any((draft) => draft.categoryId < 0)) { + return createLocal(); + } + try { + final response = await _dio.post( + '/api/transactions/recognition-batch', + data: { + 'batchId': batchId, + 'ledgerId': _ledgerId, + 'items': [ + for (var index = 0; index < drafts.length; index++) + { + 'candidateId': drafts[index].candidateId, + 'clientRequestId': drafts[index].clientRequestId, + 'categoryId': drafts[index].categoryId, + 'type': drafts[index].type, + 'amount': drafts[index].amount, + 'note': drafts[index].note, + 'paymentMethod': drafts[index].paymentMethod, + 'occurredAt': localPayloads[index]['occurredAt'], + 'source': drafts[index].source, + 'sourceText': drafts[index].sourceText, + }, + ], + }, + ); + final result = {}; + for (final raw in response.data as List) { + final item = Map.from(raw as Map); + final transaction = Map.from( + item['transaction'] as Map, + ); + LocalDatabase.instance.cacheTransaction(transaction); + result[item['candidateId'].toString()] = TxItem.fromJson(transaction); + } + return result; + } catch (error) { + if (!isConnectivityError(error) || !session.isAccount) rethrow; + return createLocal(); + } + } + static Future delete(int id) async { final baseUpdatedAt = LocalDatabase.instance.transaction( id, diff --git a/frontend/lib/shared/services/local_database.dart b/frontend/lib/shared/services/local_database.dart index 82762f1..5f7f6b4 100644 --- a/frontend/lib/shared/services/local_database.dart +++ b/frontend/lib/shared/services/local_database.dart @@ -951,6 +951,37 @@ class LocalDatabase { return transaction(id, includeDeleted: true)!; } + List> createTransactionsBatch( + List> values, { + bool enqueueSyncChanges = false, + }) { + _db.execute('BEGIN'); + try { + final created = >[]; + for (final value in values) { + final clientRequestId = value['clientRequestId'] as String?; + final existed = + clientRequestId != null && + clientRequestId.isNotEmpty && + _db.select( + 'SELECT 1 FROM transactions WHERE client_request_id = ? LIMIT 1', + [clientRequestId], + ).isNotEmpty; + final transaction = createTransaction(value); + created.add(transaction); + final id = (transaction['id'] as num).toInt(); + if (enqueueSyncChanges && id < 0 && !existed) { + enqueueSync('transaction', id, 'create', value); + } + } + _db.execute('COMMIT'); + return created; + } catch (_) { + _db.execute('ROLLBACK'); + rethrow; + } + } + Map? transaction(int id, {bool includeDeleted = false}) { final rows = _db.select( ''' diff --git a/frontend/lib/shared/services/recognition_import_service.dart b/frontend/lib/shared/services/recognition_import_service.dart index ff3a995..2465936 100644 --- a/frontend/lib/shared/services/recognition_import_service.dart +++ b/frontend/lib/shared/services/recognition_import_service.dart @@ -1,4 +1,5 @@ import 'package:flutter/material.dart'; +import 'package:go_router/go_router.dart'; import 'package:miaoji_zhang/features/home/pages/transaction_edit_page.dart'; import 'package:miaoji_zhang/shared/api/api_client.dart'; import 'package:miaoji_zhang/shared/api/business_api.dart'; @@ -10,14 +11,20 @@ import 'package:miaoji_zhang/shared/services/shanghai_time.dart'; import 'package:miaoji_zhang/shared/services/transaction_events.dart'; import 'package:miaoji_zhang/shared/theme/app_theme.dart'; import 'package:miaoji_zhang/shared/widgets/app_controls.dart'; +import 'package:shared_preferences/shared_preferences.dart'; class RecognitionImportService { RecognitionImportService._(); - static bool _processing = false; + static Future? _activeImport; + static bool _rerunRequested = false; static Future configureNativeContext() async { final session = SessionStore.instance; + final preferences = await SharedPreferences.getInstance(); + if (preferences.getBool('ai_batch_consent_v2') != true) { + await ScreenshotChannel.setRecognitionToggle('ai_screenshot', false); + } await ScreenshotChannel.configureRecognitionContext( hasAccount: session.isAccount, aiAllowed: session.aiEnabled, @@ -26,17 +33,46 @@ class RecognitionImportService { ); } - static Future importAutomatic() async { - if (_processing || !SessionStore.instance.hasSession) return; - _processing = true; + static Future importAutomatic() { + if (!SessionStore.instance.hasSession) return Future.value(); + final active = _activeImport; + if (active != null) { + _rerunRequested = true; + return active; + } + final operation = _runAutomaticImports(); + _activeImport = operation; + return operation.whenComplete(() { + if (identical(_activeImport, operation)) _activeImport = null; + }); + } + + static Future _runAutomaticImports() async { try { - await CurrentLedgerStore.instance.ensureLoaded(); - final candidates = await ScreenshotChannel.drainRecognitionCandidates(); - for (final candidate in candidates.where((item) => item.canAutoImport)) { - await _import(candidate); - } + do { + _rerunRequested = false; + await CurrentLedgerStore.instance.ensureLoaded(); + final candidates = await ScreenshotChannel.drainRecognitionCandidates(); + final automatic = candidates + .where((item) => item.canAutoImport) + .toList(); + final batches = >{}; + for (final candidate in automatic) { + final batchId = candidate.batchId; + if (batchId == null || batchId.isEmpty) continue; + batches.putIfAbsent(batchId, () => []).add(candidate); + } + for (final entry in batches.entries) { + await _importBatch(entry.key, entry.value); + } + for (final candidate in automatic.where( + (item) => item.batchId == null || item.batchId!.isEmpty, + )) { + await _import(candidate); + } + } while (_rerunRequested); } finally { - _processing = false; + _rerunRequested = false; } } @@ -49,6 +85,20 @@ class RecognitionImportService { await importAutomatic(); return; } + if (kind == 'recognition_batch_review') { + await importAutomatic(); + if (!context.mounted) return; + final batchId = action['batchId']?.toString(); + context.push( + Uri( + path: '/recognition-batches', + queryParameters: batchId == null || batchId.isEmpty + ? null + : {'batchId': batchId}, + ).toString(), + ); + return; + } if (kind == 'recognition_undo') { final transactionId = (action['transactionId'] as num?)?.toInt(); final candidateId = action['candidateId']?.toString(); @@ -130,17 +180,7 @@ class RecognitionImportService { } static Future _import(RecognitionCandidate candidate) async { - if (candidate.type != 'income' && candidate.type != 'expense') { - throw StateError('识别结果缺少明确的收支类型'); - } - final categories = await TxApi.categories(candidate.type); - if (categories.isEmpty) throw StateError('当前账本没有可用分类'); - final category = - categories - .where((item) => item.name == candidate.categoryHint) - .firstOrNull ?? - categories.where((item) => item.name == '其他').firstOrNull ?? - categories.first; + final category = await _resolveCategory(candidate, {}); final occurredUtc = validOccurredAtUtc(candidate.occurredAtEpochMs); final transaction = await TxApi.create( categoryId: category.id, @@ -164,6 +204,73 @@ class RecognitionImportService { return transaction; } + static Future _importBatch( + String batchId, + List candidates, + ) async { + final categoryCache = >{}; + final drafts = []; + for (final candidate in candidates) { + final category = await _resolveCategory(candidate, categoryCache); + final occurredUtc = validOccurredAtUtc(candidate.occurredAtEpochMs); + drafts.add( + RecognitionBatchDraft( + candidateId: candidate.id, + clientRequestId: candidate.clientRequestId, + categoryId: category.id, + type: candidate.type, + amount: candidate.amount, + note: candidate.merchant?.trim().isNotEmpty == true + ? candidate.merchant!.trim() + : (candidate.note ?? '智能识别'), + paymentMethod: candidate.appName, + source: candidate.source, + sourceText: candidate.sourceText, + occurredAt: ShanghaiTime.toCivil(occurredUtc), + ), + ); + } + final transactions = await TxApi.createRecognitionBatch(batchId, drafts); + if (transactions.length != drafts.length) { + throw StateError('批次入账结果不完整，请稍后重试'); + } + for (final candidate in candidates) { + final transaction = transactions[candidate.id]; + if (transaction == null) { + throw StateError('批次入账缺少候选 ${candidate.id}'); + } + final acknowledged = + await ScreenshotChannel.acknowledgeRecognitionCandidate( + candidate.id, + 'imported', + transactionId: transaction.id, + ); + if (!acknowledged) throw StateError('批次状态确认失败，请稍后重试'); + } + TransactionEvents.notifyChanged(); + } + + static Future _resolveCategory( + RecognitionCandidate candidate, + Map> cache, + ) async { + if (candidate.type != 'income' && candidate.type != 'expense') { + throw StateError('识别结果缺少明确的收支类型'); + } + final categories = cache[candidate.type] ??= await TxApi.categories( + candidate.type, + ); + if (categories.isEmpty) throw StateError('当前账本没有可用分类'); + return categories + .where((item) => item.id == candidate.categoryId) + .firstOrNull ?? + categories + .where((item) => item.name == candidate.categoryHint) + .firstOrNull ?? + categories.where((item) => item.name == '其他').firstOrNull ?? + categories.first; + } + static DateTime validOccurredAtUtc(int epochMs, {DateTime? now}) { final current = (now ?? DateTime.now()).toUtc(); final parsed = DateTime.fromMillisecondsSinceEpoch(epochMs, isUtc: true); diff --git a/frontend/lib/shared/services/screenshot_channel.dart b/frontend/lib/shared/services/screenshot_channel.dart index 2c3ceeb..80df5c9 100644 --- a/frontend/lib/shared/services/screenshot_channel.dart +++ b/frontend/lib/shared/services/screenshot_channel.dart @@ -138,6 +138,8 @@ class RecognitionCandidate { final String? merchant, orderId, sourceText, note; final String recognitionKind, amountSource; final String? categoryHint, resultFingerprint; + final String? batchId, aiAction, aiReason; + final int? categoryId; final int occurredAtEpochMs; RecognitionCandidate.fromJson(Map value) @@ -155,13 +157,68 @@ class RecognitionCandidate { note = value['note'] as String?, recognitionKind = value['recognitionKind']?.toString() ?? 'payment', categoryHint = value['categoryHint']?.toString(), + categoryId = (value['categoryId'] as num?)?.toInt(), amountSource = value['amountSource']?.toString() ?? 'result', resultFingerprint = value['resultFingerprint']?.toString(), + batchId = value['batchId']?.toString(), + aiAction = value['aiAction']?.toString(), + aiReason = value['aiReason']?.toString(), occurredAtEpochMs = (value['occurredAtEpochMs'] as num).toInt(); bool get canAutoImport => state == 'auto_ready' && confidence == 'auto'; } +class RecognitionBatchItem { + final String candidateId, action, reason, state, type; + final double amount; + final String? merchant; + final bool canRestore; + + RecognitionBatchItem.fromJson(Map value) + : candidateId = value['candidateId']?.toString() ?? '', + action = value['action']?.toString() ?? 'keep', + reason = value['reason']?.toString() ?? '', + state = value['state']?.toString() ?? '', + type = value['type']?.toString() ?? 'expense', + amount = (value['amount'] as num?)?.toDouble() ?? 0, + merchant = value['merchant']?.toString(), + canRestore = value['canRestore'] as bool? ?? false; +} + +class RecognitionBatch { + final String id, state; + final DateTime openedAt, completedAt; + final int kept, updated, created, dropped; + final bool fallback; + final String? failureReason; + final List items; + + RecognitionBatch.fromJson(Map value) + : id = value['id']?.toString() ?? '', + state = value['state']?.toString() ?? '', + openedAt = DateTime.fromMillisecondsSinceEpoch( + (value['openedAt'] as num?)?.toInt() ?? 0, + isUtc: true, + ), + completedAt = DateTime.fromMillisecondsSinceEpoch( + (value['completedAt'] as num?)?.toInt() ?? 0, + isUtc: true, + ), + kept = ((value['summary'] as Map?)?['kept'] as num?)?.toInt() ?? 0, + updated = ((value['summary'] as Map?)?['updated'] as num?)?.toInt() ?? 0, + created = ((value['summary'] as Map?)?['created'] as num?)?.toInt() ?? 0, + dropped = ((value['summary'] as Map?)?['dropped'] as num?)?.toInt() ?? 0, + fallback = ((value['summary'] as Map?)?['fallback'] as bool?) ?? false, + failureReason = value['failureReason']?.toString(), + items = (value['items'] as List? ?? const []) + .map( + (item) => RecognitionBatchItem.fromJson( + Map.from(item as Map), + ), + ) + .toList(growable: false); +} + class SpeechEvent { final String type; final String? text; @@ -398,6 +455,37 @@ class ScreenshotChannel { } } + static Future> listRecognitionBatches() async { + try { + final values = + await _channel.invokeMethod>( + 'listRecognitionBatches', + ) ?? + const []; + return values + .whereType() + .map( + (value) => RecognitionBatch.fromJson( + jsonDecode(value) as Map, + ), + ) + .toList(growable: false); + } on MissingPluginException { + return const []; + } + } + + static Future restoreDroppedRecognition(String candidateId) async { + try { + return await _channel.invokeMethod('restoreDroppedRecognition', { + 'candidateId': candidateId, + }) ?? + false; + } on MissingPluginException { + return false; + } + } + static Future requestNotificationPermission() async { try { return await _channel.invokeMethod( diff --git a/frontend/test/theme_and_time_test.dart b/frontend/test/theme_and_time_test.dart index 3ae902a..3d4d80c 100644 --- a/frontend/test/theme_and_time_test.dart +++ b/frontend/test/theme_and_time_test.dart @@ -91,6 +91,41 @@ void main() { expect(candidate.resultFingerprint, 'abc123'); }); + test('AI 批次模型保留动作、原因与可恢复状态', () { + final batch = RecognitionBatch.fromJson({ + 'id': 'batch-1', + 'state': 'ready', + 'openedAt': DateTime.utc(2026, 7, 25, 8).millisecondsSinceEpoch, + 'completedAt': DateTime.utc(2026, 7, 25, 8, 1).millisecondsSinceEpoch, + 'summary': { + 'kept': 1, + 'updated': 1, + 'created': 0, + 'dropped': 1, + 'fallback': false, + }, + 'items': [ + { + 'candidateId': 'candidate-1', + 'action': 'drop', + 'reason': '同一流程重复结果页', + 'state': 'ai_dropped', + 'type': 'expense', + 'amount': 20, + 'merchant': '张三', + 'canRestore': true, + }, + ], + }); + + expect(batch.completedAt.isUtc, isTrue); + expect(batch.updated, 1); + expect(batch.dropped, 1); + expect(batch.items.single.action, 'drop'); + expect(batch.items.single.reason, '同一流程重复结果页'); + expect(batch.items.single.canRestore, isTrue); + }); + test('智能识别诊断摘要能区分关键失败类型', () { RecognitionDiagnostic diagnostic(String result, String reason) { return RecognitionDiagnostic(