From 294a7de61db99b8da981d8b0d65b0efb9828781a Mon Sep 17 00:00:00 2001 From: nanxun Date: Fri, 21 Aug 2026 22:07:45 +0800 Subject: [PATCH] fix: isolate payment amount evidence --- .../ApiIntegrationTests.cs | 28 ++- .../Controllers/ParseController.cs | 29 ++- .../Services/OpenAiVisionClient.cs | 1 + .../kotlin/com/nx/miaoji/LocalPaymentOcr.kt | 40 +++- .../kotlin/com/nx/miaoji/PaymentParser.kt | 201 ++++++++++++++++-- .../kotlin/com/nx/miaoji/RecognitionStore.kt | 162 +++++++++++++- .../miaoji/ScreenshotAccessibilityService.kt | 92 +++++--- .../kotlin/com/nx/miaoji/PaymentParserTest.kt | 170 ++++++++++++++- .../recognition_diagnostic_formatter.dart | 10 +- frontend/test/theme_and_time_test.dart | 12 ++ 10 files changed, 673 insertions(+), 72 deletions(-) diff --git a/backend/MiaoJiZhang.Api.Tests/ApiIntegrationTests.cs b/backend/MiaoJiZhang.Api.Tests/ApiIntegrationTests.cs index 01a76f0..a4e978c 100644 --- a/backend/MiaoJiZhang.Api.Tests/ApiIntegrationTests.cs +++ b/backend/MiaoJiZhang.Api.Tests/ApiIntegrationTests.cs @@ -3,8 +3,9 @@ using System.Net; using System.Net.Http.Headers; using System.Net.Http.Json; using System.Reflection; -using System.Text.Json; -using MiaoJiZhang.Api.Services; +using System.Text.Json; +using MiaoJiZhang.Api.Controllers; +using MiaoJiZhang.Api.Services; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Mvc.Testing; using Microsoft.AspNetCore.TestHost; @@ -859,6 +860,29 @@ public sealed class RecognitionBatchActionParserTests Assert.Equal(1, action.Confidence); } } + +public sealed class RecognitionAmountUpdateEvidenceTests +{ + private static readonly IReadOnlyDictionary EvidenceOwners = + new Dictionary + { + ["evidence-a"] = "candidate-a", + ["evidence-b"] = "candidate-b" + }; + + [Fact] + public void AmountUpdateRequiresLinkedEvidenceAndHighConfidence() + { + Assert.False(ParseController.CanApplyAmountUpdate( + "candidate-a", null, 0.99, EvidenceOwners)); + Assert.False(ParseController.CanApplyAmountUpdate( + "candidate-a", "evidence-a", 0.89, EvidenceOwners)); + Assert.False(ParseController.CanApplyAmountUpdate( + "candidate-a", "evidence-b", 0.99, EvidenceOwners)); + Assert.True(ParseController.CanApplyAmountUpdate( + "candidate-a", "evidence-a", 0.9, EvidenceOwners)); + } +} public sealed class BudgetRecommendationValidationTests { diff --git a/backend/MiaoJiZhang.Api/Controllers/ParseController.cs b/backend/MiaoJiZhang.Api/Controllers/ParseController.cs index 65bab68..49b1b71 100644 --- a/backend/MiaoJiZhang.Api/Controllers/ParseController.cs +++ b/backend/MiaoJiZhang.Api/Controllers/ParseController.cs @@ -364,6 +364,9 @@ public class ParseController(AppDbContext db, ILlmClient llm, AgentService agent { var candidateIds = manifest.Candidates.Select(item => item.CandidateId).ToHashSet(); var evidenceById = manifest.Evidence.ToDictionary(item => item.EvidenceId); + var evidenceOwners = evidenceById.ToDictionary( + entry => entry.Key, + entry => entry.Value.CandidateId); var selected = modelActions .Where(action => action.CandidateId != null && candidateIds.Contains(action.CandidateId)) .GroupBy(action => action.CandidateId!) @@ -401,7 +404,19 @@ public class ParseController(AppDbContext db, ILlmClient llm, AgentService agent transferDirection = candidate.TransferDirection; reason = "转账方向不明确，已保留本地结果"; } - var amount = model?.Amount is > 0 ? model.Amount.Value : candidate.Amount; + var proposedAmount = model?.Amount is > 0 ? model.Amount.Value : candidate.Amount; + var amountChanged = proposedAmount != candidate.Amount; + var amountUpdateAllowed = !amountChanged || + model?.Action == "update" && CanApplyAmountUpdate( + candidate.CandidateId, + model.EvidenceId, + model.Confidence, + evidenceOwners); + var amount = amountUpdateAllowed ? proposedAmount : candidate.Amount; + if (amountChanged && !amountUpdateAllowed) + { + reason = "金额修改证据不足，已保留本地金额"; + } var category = FindCategory( categories, type == "income" || transferDirection == "in" @@ -464,6 +479,18 @@ public class ParseController(AppDbContext db, ILlmClient llm, AgentService agent } return result.Take(20).ToList(); } + + internal static bool CanApplyAmountUpdate( + string candidateId, + string? evidenceId, + double confidence, + IReadOnlyDictionary evidenceOwners) + { + return confidence >= 0.9 && + !string.IsNullOrWhiteSpace(evidenceId) && + evidenceOwners.TryGetValue(evidenceId, out var owner) && + owner == candidateId; + } private async Task FeatureEnabled(string key) { diff --git a/backend/MiaoJiZhang.Api/Services/OpenAiVisionClient.cs b/backend/MiaoJiZhang.Api/Services/OpenAiVisionClient.cs index 2240e99..d4b7190 100644 --- a/backend/MiaoJiZhang.Api/Services/OpenAiVisionClient.cs +++ b/backend/MiaoJiZhang.Api/Services/OpenAiVisionClient.cs @@ -200,6 +200,7 @@ public partial class OpenAiVisionClient : ILlmClient 只有截图明确显示失败、取消、待支付，或明确是同一 flowSessionId 的重复结果页时才可 drop。 只有存在没有对应候选的 evidenceId 且截图明确显示交易成功时才可 create，并必须引用该 evidenceId。 update/create 的 type 只能是 expense、income 或 transfer，amount 必须大于 0；transfer 必须返回 transferDirection=in|out。 + update 修改 amount 时必须引用属于该 candidateId 的 evidenceId，且 confidence 不低于 0.9；证据不足时保持候选金额。 截图未明确显示时间时沿用候选时间，禁止猜测时间。reason 不超过 40 个汉字。 """; var messages = new List(); diff --git a/frontend/android/app/src/main/kotlin/com/nx/miaoji/LocalPaymentOcr.kt b/frontend/android/app/src/main/kotlin/com/nx/miaoji/LocalPaymentOcr.kt index 278a331..7ee188d 100644 --- a/frontend/android/app/src/main/kotlin/com/nx/miaoji/LocalPaymentOcr.kt +++ b/frontend/android/app/src/main/kotlin/com/nx/miaoji/LocalPaymentOcr.kt @@ -101,7 +101,7 @@ object LocalPaymentOcr { flowSessionId: String, trustedFlow: Boolean, capturedAt: Long, - expectedAmountCents: Long?, + expectedAmountEvidence: ExpectedAmountEvidence?, expectedType: String?, resultTransitionObserved: Boolean, submittedFlow: Boolean, @@ -120,7 +120,7 @@ object LocalPaymentOcr { flowSessionId = flowSessionId, trustedFlow = trustedFlow, capturedAt = capturedAt, - expectedAmountCents = expectedAmountCents, + expectedAmountEvidence = expectedAmountEvidence, expectedType = expectedType, resultTransitionObserved = resultTransitionObserved, submittedFlow = submittedFlow, @@ -199,7 +199,7 @@ object LocalPaymentOcr { flowSessionId: String, trustedFlow: Boolean, capturedAt: Long, - expectedAmountCents: Long?, + expectedAmountEvidence: ExpectedAmountEvidence?, expectedType: String?, resultTransitionObserved: Boolean, submittedFlow: Boolean, @@ -322,6 +322,7 @@ object LocalPaymentOcr { ), ) val distinctCents = ranked.map { (it.amount * 100).roundToLong() }.distinct() + val expectedAmountCents = expectedAmountEvidence?.amountCents val expectedCandidate = expectedAmountCents?.let { expected -> ranked.firstOrNull { (it.amount * 100).roundToLong() == expected } } @@ -348,15 +349,21 @@ object LocalPaymentOcr { val selectedCents = resultSelectedCents ?: expectedAmountCents!! val amountSource = if (resultSelectedCents == null) "expected" else "result" - val expectedMatched = expectedAmountCents?.let { it == selectedCents } + val expectedMatched = if (expectedAmountCents != null && resultSelectedCents != null) { + expectedAmountCents == resultSelectedCents + } else { + null + } + val amountConflict = expectedMatched == false val fresh = System.currentTimeMillis() - capturedAt <= MAX_SCREENSHOT_AGE_MS val resultAmountSafe = distinctCents.size == 1 && (expectedAmountCents == null || expectedMatched == true) val highConfidence = when { - amountSource == "expected" -> trustedFlow && fresh && submittedFlow && - resultTransitionObserved && expectedType == direction + amountSource == "expected" -> expectedAmountEvidence?.isStrong == true && + trustedFlow && fresh && submittedFlow && resultTransitionObserved && + expectedType == direction status.strength == PaymentStatusStrength.STRONG -> trustedFlow && fresh && - submittedFlow && resultTransitionObserved && resultAmountSafe + submittedFlow && resultTransitionObserved && resultAmountSafe && !amountConflict status.strength == PaymentStatusStrength.WEAK -> OcrEvidenceEvaluator.qualifiesWeakAuto( trustedFlow = trustedFlow && submittedFlow, freshScreenshot = fresh, @@ -399,8 +406,27 @@ object LocalPaymentOcr { if (it == "income") "in" else "out" }, counterparty = merchant.takeIf { kind == "transfer" }, + amountEvidenceStrength = if (amountSource == "result" || + expectedAmountEvidence?.isStrong == true + ) { + "strong" + } else { + "weak" + }, + amountCandidateCount = distinctCents.size, + expectedAmountMatched = expectedMatched, + amountIssueReason = when { + amountConflict -> "result_amount_conflict" + amountSource == "expected" && expectedAmountEvidence?.isStrong != true -> + "weak_expected_fallback" + amountSource == "expected" -> "expected_amount_fallback" + else -> null + }, ) val reason = when { + amountConflict -> "result_amount_conflict" + amountSource == "expected" && expectedAmountEvidence?.isStrong != true -> + "weak_expected_fallback" highConfidence && amountSource == "expected" -> "expected_amount_fallback" highConfidence && status.strength == PaymentStatusStrength.WEAK -> "combined_high_confidence" diff --git a/frontend/android/app/src/main/kotlin/com/nx/miaoji/PaymentParser.kt b/frontend/android/app/src/main/kotlin/com/nx/miaoji/PaymentParser.kt index 102d86c..ddd8e30 100644 --- a/frontend/android/app/src/main/kotlin/com/nx/miaoji/PaymentParser.kt +++ b/frontend/android/app/src/main/kotlin/com/nx/miaoji/PaymentParser.kt @@ -25,6 +25,36 @@ data class PaymentSignal( val identityConfidence: String = "strong", val transferDirection: String? = null, val counterparty: String? = null, + val amountEvidenceStrength: String = "strong", + val amountCandidateCount: Int = 1, + val expectedAmountMatched: Boolean? = null, + val amountIssueReason: String? = null, +) + +data class ExpectedAmountEvidence( + val amountCents: Long, + val flowSessionId: String, + val windowId: Int, + val pageFingerprint: String, + val capturedAtEpochMs: Long, + val candidateCount: Int, + val source: String, + val strength: String, +) { + val isStrong: Boolean get() = strength == "strong" + + fun belongsTo(flowId: String, flowStartedAt: Long, committedAt: Long?): Boolean = + flowSessionId == flowId && + capturedAtEpochMs >= flowStartedAt && + (committedAt == null || capturedAtEpochMs <= committedAt) +} + +data class PaymentParseOutcome( + val signal: PaymentSignal?, + val amountCandidateCount: Int, + val resultAmountCents: Long?, + val expectedAmountMatched: Boolean?, + val reason: String?, ) enum class PaymentStatusStrength(val wireValue: String) { @@ -44,7 +74,7 @@ object PaymentParser { val supportedPackages = setOf(WECHAT, ALIPAY) private val amountPatterns = listOf( - Regex("""(?:实付|付款金额|支付金额|收款金额|到账金额|交易金额|金额)[：:\s]*[¥￥]?\s*([0-9]+(?:\.[0-9]{1,2})?)"""), + Regex("""(?:实付|付款金额|支付金额|转账金额|红包金额|收款金额|到账金额|交易金额|金额)[：:\s]*[¥￥]?\s*([0-9]+(?:\.[0-9]{1,2})?)"""), Regex("""[¥￥]\s*([0-9]+(?:\.[0-9]{1,2})?)"""), Regex("""([0-9]+(?:\.[0-9]{1,2})?)\s*元"""), ) @@ -128,17 +158,47 @@ object PaymentParser { windowId: Int, flowSessionId: String? = null, trustedFlow: Boolean = false, - expectedAmountCents: Long? = null, + expectedAmountEvidence: ExpectedAmountEvidence? = null, expectedType: String? = null, resultTransitionObserved: Boolean = false, submittedFlow: Boolean = false, flowKind: String? = null, - ): PaymentSignal? { + ): PaymentSignal? = fromAccessibilityOutcome( + packageName = packageName, + text = text, + eventTime = eventTime, + windowId = windowId, + flowSessionId = flowSessionId, + trustedFlow = trustedFlow, + expectedAmountEvidence = expectedAmountEvidence, + expectedType = expectedType, + resultTransitionObserved = resultTransitionObserved, + submittedFlow = submittedFlow, + flowKind = flowKind, + ).signal + + fun fromAccessibilityOutcome( + packageName: String, + text: String, + eventTime: Long, + windowId: Int, + flowSessionId: String? = null, + trustedFlow: Boolean = false, + expectedAmountEvidence: ExpectedAmountEvidence? = null, + expectedType: String? = null, + resultTransitionObserved: Boolean = false, + submittedFlow: Boolean = false, + flowKind: String? = null, + ): PaymentParseOutcome { + val expectedEvidence = expectedAmountEvidence + val expectedCents = expectedEvidence?.amountCents + val resultCandidates = amountCandidateCents(text) + val uniqueResultAmount = resultCandidates.singleOrNull() if (packageName !in supportedPackages || containsBlockedStatus(text) || isHistoryPageText(text) || isPaymentInputPage(text) - ) return null + ) return PaymentParseOutcome(null, resultCandidates.size, null, null, null) val eventId = flowSessionId?.let { "a:" + packageName + ":" + it } ?: "a:" + windowId + ":" + sha256(normalize(text)) + ":" + (eventTime / 10_000L) val status = detectStatus(text, expectedType) @@ -156,46 +216,85 @@ object PaymentParser { null } if (parsed != null) { - val uniqueResultAmount = uniqueAmountCents(text) val standaloneRedPacketIncome = parsed.recognitionKind in setOf( "red_packet_receive", "red_packet_refund", ) - val expectedMatches = expectedAmountCents == null || - uniqueResultAmount == expectedAmountCents + val expectedMatches = expectedCents == null || + uniqueResultAmount == expectedCents + val conflict = expectedCents != null && uniqueResultAmount != null && + uniqueResultAmount != expectedCents val evidenceHigh = when { standaloneRedPacketIncome -> uniqueResultAmount == parsed.amountCents submittedFlow -> trustedFlow && resultTransitionObserved && - uniqueResultAmount == parsed.amountCents && expectedMatches + uniqueResultAmount == parsed.amountCents && expectedMatches && !conflict else -> false } - return parsed.copy( + val signal = parsed.copy( evidenceConfidence = if (evidenceHigh) "high" else "confirm", + amountEvidenceStrength = if (uniqueResultAmount != null) "strong" else "weak", + amountCandidateCount = resultCandidates.size, + expectedAmountMatched = if (expectedCents != null && uniqueResultAmount != null) { + expectedMatches + } else { + null + }, + amountIssueReason = if (conflict) "result_amount_conflict" else null, + ) + return PaymentParseOutcome( + signal, + resultCandidates.size, + uniqueResultAmount, + signal.expectedAmountMatched, + signal.amountIssueReason, ) } val redPacketSentSurface = flowKind == "red_packet_send" && hasRedPacketSentSurface(text) - if (status.strength == PaymentStatusStrength.NONE && !redPacketSentSurface) return null - val direction = status.direction ?: if (redPacketSentSurface) "expense" else return null - val resultAmount = uniqueAmountCents(text) + if (status.strength == PaymentStatusStrength.NONE && !redPacketSentSurface) { + return PaymentParseOutcome(null, resultCandidates.size, null, null, null) + } + val direction = status.direction ?: if (redPacketSentSurface) "expense" else { + return PaymentParseOutcome(null, resultCandidates.size, null, null, "direction_unknown") + } + val resultAmount = uniqueResultAmount val canUseExpectedAmount = resultAmount == null && - expectedAmountCents != null && + expectedCents != null && submittedFlow && resultTransitionObserved && expectedType == direction - val amountCents = resultAmount ?: expectedAmountCents?.takeIf { canUseExpectedAmount } - ?: return null - val expectedMatched = expectedAmountCents != null && amountCents == expectedAmountCents + val amountCents = resultAmount ?: expectedCents?.takeIf { canUseExpectedAmount } + ?: return PaymentParseOutcome( + null, + resultCandidates.size, + resultAmount, + null, + if (expectedCents == null) "expected_amount_missing" else "missing_amount", + ) + val expectedMatched = if (expectedCents != null && resultAmount != null) { + resultAmount == expectedCents + } else { + null + } + val conflict = expectedMatched == false + val expectedStrong = expectedEvidence?.isStrong == true val evidenceHigh = trustedFlow && submittedFlow && resultTransitionObserved && - expectedMatched && expectedType == direction + expectedType == direction && !conflict && + (resultAmount != null || expectedStrong) val kind = flowKind ?: recognitionKind(text, direction) val merchant = extractMerchant(text) val orderId = extractOrderId(text) val transferDirection = direction.takeIf { kind == "transfer" }?.let { if (it == "income") "in" else "out" } - return PaymentSignal( + val issueReason = when { + conflict -> "result_amount_conflict" + resultAmount == null && !expectedStrong -> "weak_expected_fallback" + resultAmount == null -> "expected_amount_fallback" + else -> null + } + val signal = PaymentSignal( packageName = packageName, channel = "accessibility", amountCents = amountCents, @@ -222,6 +321,17 @@ object PaymentParser { ), transferDirection = transferDirection, counterparty = merchant.takeIf { kind == "transfer" }, + amountEvidenceStrength = if (resultAmount != null || expectedStrong) "strong" else "weak", + amountCandidateCount = resultCandidates.size, + expectedAmountMatched = expectedMatched, + amountIssueReason = issueReason, + ) + return PaymentParseOutcome( + signal, + resultCandidates.size, + resultAmount, + expectedMatched, + issueReason, ) } fun fromNotification(notification: StatusBarNotification): PaymentSignal? { @@ -313,8 +423,12 @@ object PaymentParser { } fun uniqueAmountCents(value: String): Long? { + return amountCandidateCents(value).singleOrNull() + } + + fun amountCandidateCents(value: String): List { val normalized = normalize(value) - val cents = buildList { + return buildList { amountPatterns.forEach { pattern -> pattern.findAll(normalized).forEach { match -> match.groupValues.getOrNull(1)?.toDoubleOrNull()?.let { amount -> @@ -332,7 +446,50 @@ object PaymentParser { } } }.distinct() - return cents.singleOrNull() + } + + fun expectedAmountEvidence( + pageText: String, + eventText: String, + flowSessionId: String, + windowId: Int, + capturedAtEpochMs: Long, + ): ExpectedAmountEvidence? { + val pageFingerprint = sha256(normalize(pageText)) + val labeled = LABELED_AMOUNT.findAll(normalize(pageText)) + .mapNotNull { match -> + match.groupValues.getOrNull(1)?.toDoubleOrNull()?.let { + (it * 100).roundToLong() + } + } + .filter { it > 0 } + .distinct() + .toList() + if (labeled.size == 1) { + return ExpectedAmountEvidence( + labeled.single(), + flowSessionId, + windowId, + pageFingerprint, + capturedAtEpochMs, + labeled.size, + "labeled_page", + "strong", + ) + } + val eventCandidates = amountCandidateCents(eventText) + if (eventCandidates.size != 1) return null + val explicitEvent = EXPLICIT_AMOUNT.containsMatchIn(normalize(eventText)) + return ExpectedAmountEvidence( + eventCandidates.single(), + flowSessionId, + windowId, + pageFingerprint, + capturedAtEpochMs, + eventCandidates.size, + "payment_event", + if (explicitEvent) "strong" else "weak", + ) } fun isPaymentInputPage(value: String): Boolean { @@ -548,6 +705,10 @@ object PaymentParser { private val STANDALONE_AMOUNT = Regex( """^\s*([¥￥]?)\s*([0-9]{1,8}(?:\.[0-9]{1,2})?)\s*(元?)\s*$""", ) + private val LABELED_AMOUNT = Regex( + """(?:实付|付款金额|支付金额|转账金额|红包金额|收款金额|到账金额|交易金额)[：: \t]*(?:\r?\n[：: \t]*)?[¥￥]?[ \t]*([0-9]+(?:\.[0-9]{1,2})?)""", + ) + private val EXPLICIT_AMOUNT = Regex("""[¥￥]\s*[0-9]+(?:\.[0-9]{1,2})?|[0-9]+\.[0-9]{1,2}""") private const val MAX_SUCCESS_HEADING_CHARS = 48 private const val MAX_PAYMENT_SCAN_LINES = 48 private const val MAX_HISTORY_TITLE_LINES = 3 diff --git a/frontend/android/app/src/main/kotlin/com/nx/miaoji/RecognitionStore.kt b/frontend/android/app/src/main/kotlin/com/nx/miaoji/RecognitionStore.kt index 076226f..6dab745 100644 --- a/frontend/android/app/src/main/kotlin/com/nx/miaoji/RecognitionStore.kt +++ b/frontend/android/app/src/main/kotlin/com/nx/miaoji/RecognitionStore.kt @@ -37,6 +37,13 @@ data class PendingRecognitionBatch( val images: List, ) +internal data class AmountMergeResult( + val amountCents: Long, + val source: String, + val strength: String, + val conflict: Boolean, +) + class RecognitionStore(context: Context) : SQLiteOpenHelper(context, "recognition_queue.db", null, 3) { override fun onCreate(db: SQLiteDatabase) { @@ -178,20 +185,27 @@ class RecognitionStore(context: Context) : id = existing.id val mergedMask = existing.channelMask or channelBit val hasNonAiEvidence = mergedMask and channelBit("recognition_ai").inv() != 0 - val high = hasNonAiEvidence && + val mergedPayload = mergePayload(existing.payload, signal) + val amountConflict = mergedPayload.optBoolean("amountConflict", false) + val high = !amountConflict && hasNonAiEvidence && (signalHigh || (mergedMask and ACCESSIBILITY_NOTIFICATION_MASK) == ACCESSIBILITY_NOTIFICATION_MASK || existing.highConfidence) - val mergedPayload = mergePayload(existing.payload, signal) writableDatabase.update( "candidates", ContentValues().apply { put("channel_mask", mergedMask) put("known_template", if (existing.knownTemplate || signal.knownTemplate) 1 else 0) + put("amount_cents", kotlin.math.round(mergedPayload.optDouble("amount") * 100).toLong()) + put("direction", mergedPayload.optString("type", signal.type)) put("high_confidence", if (high) 1 else 0) put("payload_encrypted", encryptPayload(mergedPayload)) - if (high && existing.state == "pending_confirm") put("state", "auto_ready") + if (amountConflict) { + put("state", "pending_confirm") + } else if (high && existing.state == "pending_confirm") { + put("state", "auto_ready") + } put("updated_at", now) put("available_at", now + MERGE_DELAY_MS) if (batchId != null) { @@ -525,9 +539,21 @@ class RecognitionStore(context: Context) : val original = JSONObject(row.payload.toString()) val kind = action.optString("action") val payload = JSONObject(row.payload.toString()) - if (kind == "update") applyActionFields(payload, action) + if (kind == "update") { + applyActionFields(payload, action) + if (payload.optDouble("amount") != original.optDouble("amount")) { + payload + .put("amountSource", "ai_batch") + .put("amountEvidenceStrength", "strong") + .put("amountConflict", false) + .remove("conflictingAmount") + payload.remove("conflictingAmountSource") + payload.remove("amountIssueReason") + } + } payload.put("sourceOverride", "recognition_ai") val reason = action.optString("reason", "AI 对账") + val amountConflict = payload.optBoolean("amountConflict", false) writableDatabase.update( "candidates", ContentValues().apply { @@ -540,8 +566,15 @@ class RecognitionStore(context: Context) : put("channel_mask", row.channelMask or channelBit("recognition_ai")) put("ai_action", kind) put("ai_reason", reason.take(80)) - put("state", if (kind == "drop") "ai_dropped" else "auto_ready") - put("high_confidence", 1) + put( + "state", + when { + kind == "drop" -> "ai_dropped" + amountConflict -> "pending_confirm" + else -> "auto_ready" + }, + ) + put("high_confidence", if (amountConflict) 0 else 1) put("updated_at", now) }, "id = ? AND batch_id = ?", @@ -631,14 +664,17 @@ class RecognitionStore(context: Context) : // The server guarantees one action per candidate. Preserve anything omitted // by a malformed response instead of silently losing a payment. writableDatabase.rawQuery( - "SELECT id FROM candidates WHERE batch_id = ? AND state = 'batch_collecting'", + "SELECT id, payload_encrypted FROM candidates WHERE batch_id = ? AND state = 'batch_collecting'", arrayOf(batchId), ).use { cursor -> while (cursor.moveToNext()) { + val amountConflict = decryptPayload(cursor.getString(1)) + ?.optBoolean("amountConflict", false) == true writableDatabase.update( "candidates", ContentValues().apply { - put("state", "auto_ready") + put("state", if (amountConflict) "pending_confirm" else "auto_ready") + put("high_confidence", if (amountConflict) 0 else 1) put("ai_action", "keep") put("ai_reason", "AI 未返回该候选，已保留本地结果") put("updated_at", now) @@ -1011,6 +1047,10 @@ class RecognitionStore(context: Context) : .put("recognitionKind", signal.recognitionKind) .put("categoryHint", signal.categoryHint) .put("amountSource", signal.amountSource) + .put("amountEvidenceStrength", signal.amountEvidenceStrength) + .put("amountCandidateCount", signal.amountCandidateCount) + .put("expectedAmountMatched", signal.expectedAmountMatched) + .put("amountIssueReason", signal.amountIssueReason) .put("resultFingerprint", signal.resultFingerprint) .put("identityConfidence", signal.identityConfidence) .put("transferDirection", signal.transferDirection) @@ -1043,7 +1083,36 @@ class RecognitionStore(context: Context) : if (existing.isNull("sourceText") && signal.sourceText != null) existing.put("sourceText", signal.sourceText) if (existing.optString("recognitionKind").isBlank()) existing.put("recognitionKind", signal.recognitionKind) if (existing.isNull("categoryHint") && signal.categoryHint != null) existing.put("categoryHint", signal.categoryHint) - if (existing.optString("amountSource").isBlank()) existing.put("amountSource", signal.amountSource) + val existingAmountCents = kotlin.math.round(existing.optDouble("amount") * 100).toLong() + val amountMerge = mergeAmountEvidence( + existingAmountCents = existingAmountCents, + existingSource = existing.optString("amountSource", "result"), + existingStrength = existing.optString("amountEvidenceStrength", "strong"), + incoming = signal, + ) + val amountConflict = existing.optBoolean("amountConflict", false) || + amountMerge.conflict + existing + .put("amount", amountMerge.amountCents / 100.0) + .put("amountSource", amountMerge.source) + .put("amountEvidenceStrength", amountMerge.strength) + .put("amountConflict", amountConflict) + .put("amountCandidateCount", maxOf( + existing.optInt("amountCandidateCount", 0), + signal.amountCandidateCount, + )) + if (amountMerge.conflict) { + existing + .put("conflictingAmount", signal.amountCents / 100.0) + .put("conflictingAmountSource", signal.amountSource) + } + if (amountConflict) { + existing.put("amountIssueReason", "result_amount_conflict") + } else if (signal.amountIssueReason != null) { + existing.put("amountIssueReason", signal.amountIssueReason) + } else if (amountMerge.source == "result") { + existing.remove("amountIssueReason") + } if (existing.isNull("resultFingerprint") && signal.resultFingerprint != null) { existing.put("resultFingerprint", signal.resultFingerprint) } @@ -1111,6 +1180,81 @@ class RecognitionStore(context: Context) : private const val MAX_BATCH_ITEMS = 10 private const val MAX_BATCH_IMAGE_BYTES = 1024 * 1024 + internal fun mergeAmountEvidence( + existingAmountCents: Long, + existingSource: String, + existingStrength: String, + incoming: PaymentSignal, + ): AmountMergeResult { + if (existingAmountCents == incoming.amountCents) { + val incomingRank = amountEvidenceRank( + incoming.amountSource, + incoming.amountEvidenceStrength, + ) + val existingRank = amountEvidenceRank(existingSource, existingStrength) + return if (incomingRank > existingRank) { + AmountMergeResult( + incoming.amountCents, + incoming.amountSource, + incoming.amountEvidenceStrength, + false, + ) + } else { + AmountMergeResult( + existingAmountCents, + existingSource, + existingStrength, + false, + ) + } + } + + val incomingRank = amountEvidenceRank( + incoming.amountSource, + incoming.amountEvidenceStrength, + ) + val existingRank = amountEvidenceRank(existingSource, existingStrength) + val bothStrong = existingStrength == "strong" && + incoming.amountEvidenceStrength == "strong" + val independentConflict = bothStrong && + (existingSource == "result" || incoming.amountSource == "result") + if (independentConflict) { + return AmountMergeResult( + existingAmountCents, + existingSource, + existingStrength, + true, + ) + } + val useIncoming = when { + incoming.amountSource == "result" && existingSource != "result" -> true + existingSource == "result" && incoming.amountSource != "result" -> false + else -> incomingRank > existingRank + } + return if (useIncoming) { + AmountMergeResult( + incoming.amountCents, + incoming.amountSource, + incoming.amountEvidenceStrength, + false, + ) + } else { + AmountMergeResult( + existingAmountCents, + existingSource, + existingStrength, + false, + ) + } + } + + private fun amountEvidenceRank(source: String, strength: String): Int = when { + source == "result" && strength == "strong" -> 4 + source == "result" -> 3 + strength == "strong" -> 2 + else -> 1 + } + internal fun clientRequestIdFor(signal: PaymentSignal): String { val basis = when { !signal.orderId.isNullOrBlank() -> diff --git a/frontend/android/app/src/main/kotlin/com/nx/miaoji/ScreenshotAccessibilityService.kt b/frontend/android/app/src/main/kotlin/com/nx/miaoji/ScreenshotAccessibilityService.kt index d9af38d..9805f4f 100644 --- a/frontend/android/app/src/main/kotlin/com/nx/miaoji/ScreenshotAccessibilityService.kt +++ b/frontend/android/app/src/main/kotlin/com/nx/miaoji/ScreenshotAccessibilityService.kt @@ -47,7 +47,7 @@ class ScreenshotAccessibilityService : AccessibilityService() { var kind: String, var originWindowId: Int = windowId, var originPageHash: String? = null, - var expectedAmountCents: Long? = null, + var expectedAmountEvidence: ExpectedAmountEvidence? = null, var expectedType: String? = null, var committedAt: Long? = null, var completedAt: Long? = null, @@ -188,7 +188,7 @@ class ScreenshotAccessibilityService : AccessibilityService() { ) return } - val observed = PaymentParser.fromAccessibility( + val observedOutcome = PaymentParser.fromAccessibilityOutcome( packageName = recognizedPackage, text = combined, eventTime = currentEvent.eventTime, @@ -196,6 +196,7 @@ class ScreenshotAccessibilityService : AccessibilityService() { expectedType = status.direction ?: existingFlow.expectedType, flowKind = currentKind, ) + val observed = observedOutcome.signal val resultChanged = observed?.resultFingerprint != null && existingFlow.resultFingerprint != null && observed.resultFingerprint != existingFlow.resultFingerprint @@ -229,6 +230,7 @@ class ScreenshotAccessibilityService : AccessibilityService() { nextFlow, page.nodeCount, "tree", + parseOutcome = observedOutcome, ) return } @@ -267,13 +269,14 @@ class ScreenshotAccessibilityService : AccessibilityService() { currentEvent.windowId, now, forceNew = existingFlow?.completed == true || + existingFlow?.committedAt != null || existingFlow?.packageName != recognizedPackage, kind = inferredKind, ) else -> existingFlow?.takeIf { !it.completed } } if (armedFlow != null) { - updateFlowEvidence(armedFlow, combined) + updateFlowEvidence(armedFlow, combined, eventText, now) if (clickedPaymentAction) { armedFlow.committedAt = armedFlow.committedAt ?: now armedFlow.expectedType = armedFlow.expectedType ?: "expense" @@ -284,19 +287,28 @@ class ScreenshotAccessibilityService : AccessibilityService() { } val directFlow = paymentFlow?.takeIf { !it.completed } - val directSignal = if (combined.isBlank()) null else PaymentParser.fromAccessibility( - packageName = recognizedPackage, - text = combined, - eventTime = currentEvent.eventTime, - windowId = currentEvent.windowId, - flowSessionId = directFlow?.id, - trustedFlow = directFlow?.trusted == true, - expectedAmountCents = directFlow?.expectedAmountCents, - expectedType = directFlow?.expectedType, - resultTransitionObserved = directFlow?.resultTransitionObserved == true, - submittedFlow = directFlow?.committedAt != null, - flowKind = directFlow?.kind, - ) + val directOutcome = if (combined.isBlank()) null else { + PaymentParser.fromAccessibilityOutcome( + packageName = recognizedPackage, + text = combined, + eventTime = currentEvent.eventTime, + windowId = currentEvent.windowId, + flowSessionId = directFlow?.id, + trustedFlow = directFlow?.trusted == true, + expectedAmountEvidence = directFlow?.expectedAmountEvidence?.takeIf { + it.belongsTo( + directFlow.id, + directFlow.startedAt, + directFlow.committedAt, + ) + }, + expectedType = directFlow?.expectedType, + resultTransitionObserved = directFlow?.resultTransitionObserved == true, + submittedFlow = directFlow?.committedAt != null, + flowKind = directFlow?.kind, + ) + } + val directSignal = directOutcome?.signal if (directSignal != null) { val flow = directFlow ?: armPaymentFlow( recognizedPackage, @@ -313,6 +325,7 @@ class ScreenshotAccessibilityService : AccessibilityService() { flow, page.nodeCount, "tree", + parseOutcome = directOutcome, ) return } @@ -411,13 +424,35 @@ class ScreenshotAccessibilityService : AccessibilityService() { ).also { paymentFlow = it } } - private fun updateFlowEvidence(flow: PaymentFlow, text: String) { - if (text.isBlank() || flow.completed || flow.resultTransitionObserved) return + private fun updateFlowEvidence( + flow: PaymentFlow, + text: String, + eventText: String, + now: Long, + ) { + if (text.isBlank() || flow.completed || flow.committedAt != null || + flow.resultTransitionObserved + ) { + return + } if (PaymentParser.detectStatus(text).strength != PaymentStatusStrength.NONE) return val pageHash = PaymentParser.sha256(text) flow.originPageHash = flow.originPageHash ?: pageHash - flow.expectedAmountCents = flow.expectedAmountCents - ?: PaymentParser.uniqueAmountCents(text) + val observed = PaymentParser.expectedAmountEvidence( + pageText = text, + eventText = eventText, + flowSessionId = flow.id, + windowId = flow.windowId, + capturedAtEpochMs = now, + ) + if (observed != null) { + val existing = flow.expectedAmountEvidence + if (existing == null || observed.isStrong && !existing.isStrong || + observed.source == existing.source + ) { + flow.expectedAmountEvidence = observed + } + } flow.expectedType = flow.expectedType ?: PaymentParser.inferContextDirection(text) PaymentParser.detectFlowKind(text)?.let { detected -> @@ -468,6 +503,7 @@ class ScreenshotAccessibilityService : AccessibilityService() { stage: String, recordDiagnostic: Boolean = true, evidenceImage: ByteArray? = null, + parseOutcome: PaymentParseOutcome? = null, ) { if (flow.completed) { evidenceImage?.fill(0) @@ -503,11 +539,13 @@ class ScreenshotAccessibilityService : AccessibilityService() { "confirm" }, nodeCount = nodeCount, - amountCandidates = 1, - reason = if (batchEnabled) "queued_for_ai" else "success", - expectedAmountMatched = flow.expectedAmountCents?.let { - it == signal.amountCents - }, + amountCandidates = parseOutcome?.amountCandidateCount + ?: signal.amountCandidateCount, + reason = signal.amountIssueReason + ?: parseOutcome?.reason + ?: if (batchEnabled) "queued_for_ai" else "success", + expectedAmountMatched = parseOutcome?.expectedAmountMatched + ?: signal.expectedAmountMatched, resultTransitionObserved = flow.resultTransitionObserved, recognitionKind = signal.recognitionKind, amountSource = signal.amountSource, @@ -671,7 +709,9 @@ class ScreenshotAccessibilityService : AccessibilityService() { flowSessionId = flow.id, trustedFlow = flow.trusted, capturedAt = capturedAt, - expectedAmountCents = flow.expectedAmountCents, + expectedAmountEvidence = flow.expectedAmountEvidence?.takeIf { + it.belongsTo(flow.id, flow.startedAt, flow.committedAt) + }, expectedType = flow.expectedType, resultTransitionObserved = visualTransitionObserved, submittedFlow = flow.committedAt != null, diff --git a/frontend/android/app/src/test/kotlin/com/nx/miaoji/PaymentParserTest.kt b/frontend/android/app/src/test/kotlin/com/nx/miaoji/PaymentParserTest.kt index ab1db68..1f0f127 100644 --- a/frontend/android/app/src/test/kotlin/com/nx/miaoji/PaymentParserTest.kt +++ b/frontend/android/app/src/test/kotlin/com/nx/miaoji/PaymentParserTest.kt @@ -93,7 +93,7 @@ class PaymentParserTest { windowId = 7, flowSessionId = "flow-a", trustedFlow = true, - expectedAmountCents = 2_000L, + expectedAmountEvidence = expectedEvidence(2_000L, "strong", "flow-a"), expectedType = "expense", resultTransitionObserved = true, submittedFlow = true, @@ -107,7 +107,7 @@ class PaymentParserTest { windowId = 7, flowSessionId = "flow-b", trustedFlow = true, - expectedAmountCents = 3_000L, + expectedAmountEvidence = expectedEvidence(3_000L, "strong", "flow-b"), expectedType = "expense", resultTransitionObserved = true, ) @@ -149,7 +149,7 @@ class PaymentParserTest { windowId = 8, flowSessionId = "payment-flow", trustedFlow = true, - expectedAmountCents = 1_880L, + expectedAmountEvidence = expectedEvidence(1_880L, "strong", "payment-flow"), expectedType = "expense", resultTransitionObserved = true, submittedFlow = true, @@ -171,7 +171,7 @@ class PaymentParserTest { windowId = 8, flowSessionId = "payment-flow", trustedFlow = true, - expectedAmountCents = 1_880L, + expectedAmountEvidence = expectedEvidence(1_880L, "strong", "payment-flow"), expectedType = "expense", resultTransitionObserved = true, submittedFlow = false, @@ -190,7 +190,7 @@ class PaymentParserTest { windowId = 9, flowSessionId = "red-packet-flow", trustedFlow = true, - expectedAmountCents = 2_000L, + expectedAmountEvidence = expectedEvidence(2_000L, "strong", "red-packet-flow"), expectedType = "expense", resultTransitionObserved = true, submittedFlow = true, @@ -381,6 +381,166 @@ class PaymentParserTest { ) } + @Test + fun expectedAmountEvidenceIgnoresUnlabelledChatHistory() { + val evidence = PaymentParser.expectedAmountEvidence( + pageText = "聊天记录\n¥0.01\n你发起了一笔转账\n¥0.02\n确认转账", + eventText = "确认转账", + flowSessionId = "transfer-003", + windowId = 7, + capturedAtEpochMs = 1_000L, + ) + + assertNull(evidence) + } + + @Test + fun labeledCurrentTransferAmountWinsOverChatHistory() { + val evidence = PaymentParser.expectedAmountEvidence( + pageText = "聊天记录\n¥0.01\n¥0.02\n转账金额\n¥0.03\n确认转账", + eventText = "确认转账", + flowSessionId = "transfer-003", + windowId = 7, + capturedAtEpochMs = 1_000L, + ) + + assertEquals(3L, evidence?.amountCents) + assertEquals("strong", evidence?.strength) + assertEquals("labeled_page", evidence?.source) + } + + @Test + fun labeledAmountMustBeOnTheSameOrAdjacentNode() { + val evidence = PaymentParser.expectedAmountEvidence( + pageText = "转账金额\n付款说明\n¥0.01", + eventText = "确认转账", + flowSessionId = "transfer-003", + windowId = 7, + capturedAtEpochMs = 1_000L, + ) + + assertNull(evidence) + } + + @Test + fun resultWithoutAmountUsesOnlyStrongCurrentFlowEvidence() { + val outcome = PaymentParser.fromAccessibilityOutcome( + packageName = PaymentParser.WECHAT, + text = "转账成功", + eventTime = 2_000L, + windowId = 7, + flowSessionId = "transfer-003", + trustedFlow = true, + expectedAmountEvidence = expectedEvidence(3L, "strong"), + expectedType = "expense", + resultTransitionObserved = true, + submittedFlow = true, + flowKind = "transfer", + ) + + assertEquals(3L, outcome.signal?.amountCents) + assertEquals("high", outcome.signal?.evidenceConfidence) + assertEquals("expected_amount_fallback", outcome.reason) + assertNull(outcome.expectedAmountMatched) + } + + @Test + fun weakFallbackAndConflictingResultRequireConfirmation() { + val weak = PaymentParser.fromAccessibilityOutcome( + packageName = PaymentParser.WECHAT, + text = "转账成功", + eventTime = 2_000L, + windowId = 7, + flowSessionId = "transfer-003", + trustedFlow = true, + expectedAmountEvidence = expectedEvidence(3L, "weak"), + expectedType = "expense", + resultTransitionObserved = true, + submittedFlow = true, + flowKind = "transfer", + ) + val conflict = PaymentParser.fromAccessibilityOutcome( + packageName = PaymentParser.WECHAT, + text = "转账成功\n¥0.01", + eventTime = 2_000L, + windowId = 7, + flowSessionId = "transfer-003", + trustedFlow = true, + expectedAmountEvidence = expectedEvidence(3L, "strong"), + expectedType = "expense", + resultTransitionObserved = true, + submittedFlow = true, + flowKind = "transfer", + ) + + assertEquals("confirm", weak.signal?.evidenceConfidence) + assertEquals("weak_expected_fallback", weak.reason) + assertEquals(1L, conflict.signal?.amountCents) + assertEquals("confirm", conflict.signal?.evidenceConfidence) + assertEquals(false, conflict.expectedAmountMatched) + assertEquals("result_amount_conflict", conflict.reason) + } + + @Test + fun strongerResultReplacesWeakFallbackButStrongConflictStaysFlagged() { + val resultSignal = paymentSignal( + channel = "local_ocr", + sourceEventId = "ocr:wechat:transfer-003", + flowSessionId = "transfer-003", + ).copy( + amountCents = 3L, + amountSource = "result", + amountEvidenceStrength = "strong", + ) + val replacement = RecognitionStore.mergeAmountEvidence( + existingAmountCents = 1L, + existingSource = "expected", + existingStrength = "weak", + incoming = resultSignal, + ) + val conflict = RecognitionStore.mergeAmountEvidence( + existingAmountCents = 1L, + existingSource = "expected", + existingStrength = "strong", + incoming = resultSignal, + ) + + assertEquals(3L, replacement.amountCents) + assertFalse(replacement.conflict) + assertEquals(1L, conflict.amountCents) + assertTrue(conflict.conflict) + } + + @Test + fun consecutiveSmallTransfersKeepFourFlowsAndSumToSevenCents() { + val amounts = listOf(1L, 1L, 2L, 3L) + val signals = amounts.mapIndexed { index, cents -> + paymentSignal( + channel = "accessibility", + sourceEventId = "a:wechat:transfer-$index", + flowSessionId = "transfer-$index", + ).copy(amountCents = cents) + } + + assertEquals(7L, signals.sumOf { it.amountCents }) + assertEquals(4, signals.map(RecognitionStore::flowStrongKeyFor).toSet().size) + } + + private fun expectedEvidence( + amountCents: Long, + strength: String, + flowSessionId: String = "transfer-003", + ) = ExpectedAmountEvidence( + amountCents = amountCents, + flowSessionId = flowSessionId, + windowId = 7, + pageFingerprint = "payment-page", + capturedAtEpochMs = 1_500L, + candidateCount = 1, + source = "labeled_page", + strength = strength, + ) + private fun paymentSignal( channel: String, sourceEventId: String, diff --git a/frontend/lib/shared/services/recognition_diagnostic_formatter.dart b/frontend/lib/shared/services/recognition_diagnostic_formatter.dart index d6ee682..f00a6bc 100644 --- a/frontend/lib/shared/services/recognition_diagnostic_formatter.dart +++ b/frontend/lib/shared/services/recognition_diagnostic_formatter.dart @@ -80,7 +80,10 @@ class RecognitionDiagnosticDisplay { 'direction_unknown' => '识别到完成状态，但无法确认收支方向', 'missing_amount' => '成功状态已识别，但没有找到金额', 'expected_amount_missing' => '结果页没有金额，且付款前金额不唯一或未捕获', - 'expected_amount_fallback' => '结果页金额缺失，已使用付款前确认的唯一金额', + 'expected_amount_fallback' => '结果页未读到金额，已使用本次付款流程确认的金额', + 'weak_expected_fallback' => '结果页未读到金额，付款前金额来源不可靠，请确认后入账', + 'result_amount_conflict' => '结果页金额与本次付款金额不一致，请确认后入账', + 'amount_source_untrusted' => '金额无法与当前付款流程可靠关联，请确认后入账', 'red_packet_not_settled' => '红包尚未明确到账或退回，不会自动入账', 'duplicate_result_surface' => '同一结果页已处理，本次刷新已忽略', 'weak_status_confirm' => '只识别到弱完成状态，组合证据不足，需确认后入账', @@ -123,7 +126,7 @@ class RecognitionDiagnosticDisplay { static String? _amountSourceLabel(String? value) { return switch (value) { - 'expected' => '使用付款前金额', + 'expected' => '使用本次付款金额', 'result' => '使用结果页金额', _ => null, }; @@ -172,5 +175,8 @@ class RecognitionDiagnosticDisplay { 'red_packet_not_settled', 'weak_status_confirm', 'ambiguous_or_unarmed', + 'weak_expected_fallback', + 'result_amount_conflict', + 'amount_source_untrusted', }; } diff --git a/frontend/test/theme_and_time_test.dart b/frontend/test/theme_and_time_test.dart index 3d4d80c..04192b1 100644 --- a/frontend/test/theme_and_time_test.dart +++ b/frontend/test/theme_and_time_test.dart @@ -163,5 +163,17 @@ void main() { ).summaryLabel, '已合并', ); + expect( + RecognitionDiagnosticDisplay.from( + diagnostic('confirm', 'expected_amount_fallback'), + ).reasonLabel, + '结果页未读到金额，已使用本次付款流程确认的金额', + ); + expect( + RecognitionDiagnosticDisplay.from( + diagnostic('confirm', 'result_amount_conflict'), + ).reasonLabel, + '结果页金额与本次付款金额不一致，请确认后入账', + ); }); }