nanxun/jizhi
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: isolate payment amount evidence

Browse Source
This commit is contained in:
nanxun
2026-08-21 22:07:45 +08:00
parent 979684a06e
commit 294a7de61d
10 changed files with 673 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/MiaoJiZhang.Api.Tests/ApiIntegrationTests.cs
+26 -2
View File
@@ -3,8 +3,9 @@ using System.Net;
using System.Net.Http.Headers;
using System.Net.Http.Json;
using System.Reflection;
using System.Text.Json;
using MiaoJiZhang.Api.Services;
using System.Text.Json;
using MiaoJiZhang.Api.Controllers;
using MiaoJiZhang.Api.Services;
using Microsoft.AspNetCore.Hosting;
using Microsoft.AspNetCore.Mvc.Testing;
using Microsoft.AspNetCore.TestHost;
@@ -859,6 +860,29 @@ public sealed class RecognitionBatchActionParserTests
Assert.Equal(1, action.Confidence);
}
}
public sealed class RecognitionAmountUpdateEvidenceTests
{
private static readonly IReadOnlyDictionary<string, string?> EvidenceOwners =
new Dictionary<string, string?>
{
["evidence-a"] = "candidate-a",
["evidence-b"] = "candidate-b"
};
[Fact]
public void AmountUpdateRequiresLinkedEvidenceAndHighConfidence()
{
Assert.False(ParseController.CanApplyAmountUpdate(
"candidate-a", null, 0.99, EvidenceOwners));
Assert.False(ParseController.CanApplyAmountUpdate(
"candidate-a", "evidence-a", 0.89, EvidenceOwners));
Assert.False(ParseController.CanApplyAmountUpdate(
"candidate-a", "evidence-b", 0.99, EvidenceOwners));
Assert.True(ParseController.CanApplyAmountUpdate(
"candidate-a", "evidence-a", 0.9, EvidenceOwners));
}
}
public sealed class BudgetRecommendationValidationTests
{
backend/MiaoJiZhang.Api/Controllers/ParseController.cs
+28 -1
View File
@@ -364,6 +364,9 @@ public class ParseController(AppDbContext db, ILlmClient llm, AgentService agent
{
var candidateIds = manifest.Candidates.Select(item => item.CandidateId).ToHashSet();
var evidenceById = manifest.Evidence.ToDictionary(item => item.EvidenceId);
var evidenceOwners = evidenceById.ToDictionary(
entry => entry.Key,
entry => entry.Value.CandidateId);
var selected = modelActions
.Where(action => action.CandidateId != null && candidateIds.Contains(action.CandidateId))
.GroupBy(action => action.CandidateId!)
@@ -401,7 +404,19 @@ public class ParseController(AppDbContext db, ILlmClient llm, AgentService agent
transferDirection = candidate.TransferDirection;
reason = "转账方向不明确，已保留本地结果";
}
var amount = model?.Amount is > 0 ? model.Amount.Value : candidate.Amount;
var proposedAmount = model?.Amount is > 0 ? model.Amount.Value : candidate.Amount;
var amountChanged = proposedAmount != candidate.Amount;
var amountUpdateAllowed = !amountChanged ||
model?.Action == "update" && CanApplyAmountUpdate(
candidate.CandidateId,
model.EvidenceId,
model.Confidence,
evidenceOwners);
var amount = amountUpdateAllowed ? proposedAmount : candidate.Amount;
if (amountChanged && !amountUpdateAllowed)
{
reason = "金额修改证据不足，已保留本地金额";
}
var category = FindCategory(
categories,
type == "income" || transferDirection == "in"
@@ -464,6 +479,18 @@ public class ParseController(AppDbContext db, ILlmClient llm, AgentService agent
}
return result.Take(20).ToList();
}
internal static bool CanApplyAmountUpdate(
string candidateId,
string? evidenceId,
double confidence,
IReadOnlyDictionary<string, string?> evidenceOwners)
{
return confidence >= 0.9 &&
!string.IsNullOrWhiteSpace(evidenceId) &&
evidenceOwners.TryGetValue(evidenceId, out var owner) &&
owner == candidateId;
}
private async Task<bool> FeatureEnabled(string key)
{
backend/MiaoJiZhang.Api/Services/OpenAiVisionClient.cs
+1
View File
@@ -200,6 +200,7 @@ public partial class OpenAiVisionClient : ILlmClient
只有截图明确显示失败、取消、待支付，或明确是同一 flowSessionId 的重复结果页时才可 drop。
只有存在没有对应候选的 evidenceId 且截图明确显示交易成功时才可 create，并必须引用该 evidenceId。
update/create 的 type 只能是 expense、income 或 transfer，amount 必须大于 0；transfer 必须返回 transferDirection=in|out。
update 修改 amount 时必须引用属于该 candidateId 的 evidenceId，且 confidence 不低于 0.9；证据不足时保持候选金额。
截图未明确显示时间时沿用候选时间，禁止猜测时间。reason 不超过 40 个汉字。
""";
var messages = new List<object>();
frontend/android/app/src/main/kotlin/com/nx/miaoji/LocalPaymentOcr.kt
+33 -7
View File
@@ -101,7 +101,7 @@ object LocalPaymentOcr {
flowSessionId: String,
trustedFlow: Boolean,
capturedAt: Long,
expectedAmountCents: Long?,
expectedAmountEvidence: ExpectedAmountEvidence?,
expectedType: String?,
resultTransitionObserved: Boolean,
submittedFlow: Boolean,
@@ -120,7 +120,7 @@ object LocalPaymentOcr {
flowSessionId = flowSessionId,
trustedFlow = trustedFlow,
capturedAt = capturedAt,
expectedAmountCents = expectedAmountCents,
expectedAmountEvidence = expectedAmountEvidence,
expectedType = expectedType,
resultTransitionObserved = resultTransitionObserved,
submittedFlow = submittedFlow,
@@ -199,7 +199,7 @@ object LocalPaymentOcr {
flowSessionId: String,
trustedFlow: Boolean,
capturedAt: Long,
expectedAmountCents: Long?,
expectedAmountEvidence: ExpectedAmountEvidence?,
expectedType: String?,
resultTransitionObserved: Boolean,
submittedFlow: Boolean,
@@ -322,6 +322,7 @@ object LocalPaymentOcr {
),
)
val distinctCents = ranked.map { (it.amount * 100).roundToLong() }.distinct()
val expectedAmountCents = expectedAmountEvidence?.amountCents
val expectedCandidate = expectedAmountCents?.let { expected ->
ranked.firstOrNull { (it.amount * 100).roundToLong() == expected }
}
@@ -348,15 +349,21 @@ object LocalPaymentOcr {
val selectedCents = resultSelectedCents ?: expectedAmountCents!!
val amountSource = if (resultSelectedCents == null) "expected" else "result"
val expectedMatched = expectedAmountCents?.let { it == selectedCents }
val expectedMatched = if (expectedAmountCents != null && resultSelectedCents != null) {
expectedAmountCents == resultSelectedCents
} else {
null
}
val amountConflict = expectedMatched == false
val fresh = System.currentTimeMillis() - capturedAt <= MAX_SCREENSHOT_AGE_MS
val resultAmountSafe = distinctCents.size == 1 &&
(expectedAmountCents == null || expectedMatched == true)
val highConfidence = when {
amountSource == "expected" -> trustedFlow && fresh && submittedFlow &&
resultTransitionObserved && expectedType == direction
amountSource == "expected" -> expectedAmountEvidence?.isStrong == true &&
trustedFlow && fresh && submittedFlow && resultTransitionObserved &&
expectedType == direction
status.strength == PaymentStatusStrength.STRONG -> trustedFlow && fresh &&
submittedFlow && resultTransitionObserved && resultAmountSafe
submittedFlow && resultTransitionObserved && resultAmountSafe && !amountConflict
status.strength == PaymentStatusStrength.WEAK -> OcrEvidenceEvaluator.qualifiesWeakAuto(
trustedFlow = trustedFlow && submittedFlow,
freshScreenshot = fresh,
@@ -399,8 +406,27 @@ object LocalPaymentOcr {
if (it == "income") "in" else "out"
},
counterparty = merchant.takeIf { kind == "transfer" },
amountEvidenceStrength = if (amountSource == "result" ||
expectedAmountEvidence?.isStrong == true
) {
"strong"
} else {
"weak"
},
amountCandidateCount = distinctCents.size,
expectedAmountMatched = expectedMatched,
amountIssueReason = when {
amountConflict -> "result_amount_conflict"
amountSource == "expected" && expectedAmountEvidence?.isStrong != true ->
"weak_expected_fallback"
amountSource == "expected" -> "expected_amount_fallback"
else -> null
},
)
val reason = when {
amountConflict -> "result_amount_conflict"
amountSource == "expected" && expectedAmountEvidence?.isStrong != true ->
"weak_expected_fallback"
highConfidence && amountSource == "expected" -> "expected_amount_fallback"
highConfidence && status.strength == PaymentStatusStrength.WEAK ->
"combined_high_confidence"
frontend/android/app/src/main/kotlin/com/nx/miaoji/PaymentParser.kt
+181 -20
View File
@@ -25,6 +25,36 @@ data class PaymentSignal(
val identityConfidence: String = "strong",
val transferDirection: String? = null,
val counterparty: String? = null,
val amountEvidenceStrength: String = "strong",
val amountCandidateCount: Int = 1,
val expectedAmountMatched: Boolean? = null,
val amountIssueReason: String? = null,
)
data class ExpectedAmountEvidence(
val amountCents: Long,
val flowSessionId: String,
val windowId: Int,
val pageFingerprint: String,
val capturedAtEpochMs: Long,
val candidateCount: Int,
val source: String,
val strength: String,
) {
val isStrong: Boolean get() = strength == "strong"
fun belongsTo(flowId: String, flowStartedAt: Long, committedAt: Long?): Boolean =
flowSessionId == flowId &&
capturedAtEpochMs >= flowStartedAt &&
(committedAt == null || capturedAtEpochMs <= committedAt)
}
data class PaymentParseOutcome(
val signal: PaymentSignal?,
val amountCandidateCount: Int,
val resultAmountCents: Long?,
val expectedAmountMatched: Boolean?,
val reason: String?,
)
enum class PaymentStatusStrength(val wireValue: String) {
@@ -44,7 +74,7 @@ object PaymentParser {
val supportedPackages = setOf(WECHAT, ALIPAY)
private val amountPatterns = listOf(
Regex("""(?:实付|付款金额|支付金额|收款金额|到账金额|交易金额|金额)[：:\s]*[¥￥]?\s*([0-9]+(?:\.[0-9]{1,2})?)"""),
Regex("""(?:实付|付款金额|支付金额|转账金额|红包金额|收款金额|到账金额|交易金额|金额)[：:\s]*[¥￥]?\s*([0-9]+(?:\.[0-9]{1,2})?)"""),
Regex("""[¥￥]\s*([0-9]+(?:\.[0-9]{1,2})?)"""),
Regex("""([0-9]+(?:\.[0-9]{1,2})?)\s*元"""),
)
@@ -128,17 +158,47 @@ object PaymentParser {
windowId: Int,
flowSessionId: String? = null,
trustedFlow: Boolean = false,
expectedAmountCents: Long? = null,
expectedAmountEvidence: ExpectedAmountEvidence? = null,
expectedType: String? = null,
resultTransitionObserved: Boolean = false,
submittedFlow: Boolean = false,
flowKind: String? = null,
): PaymentSignal? {
): PaymentSignal? = fromAccessibilityOutcome(
packageName = packageName,
text = text,
eventTime = eventTime,
windowId = windowId,
flowSessionId = flowSessionId,
trustedFlow = trustedFlow,
expectedAmountEvidence = expectedAmountEvidence,
expectedType = expectedType,
resultTransitionObserved = resultTransitionObserved,
submittedFlow = submittedFlow,
flowKind = flowKind,
).signal
fun fromAccessibilityOutcome(
packageName: String,
text: String,
eventTime: Long,
windowId: Int,
flowSessionId: String? = null,
trustedFlow: Boolean = false,
expectedAmountEvidence: ExpectedAmountEvidence? = null,
expectedType: String? = null,
resultTransitionObserved: Boolean = false,
submittedFlow: Boolean = false,
flowKind: String? = null,
): PaymentParseOutcome {
val expectedEvidence = expectedAmountEvidence
val expectedCents = expectedEvidence?.amountCents
val resultCandidates = amountCandidateCents(text)
val uniqueResultAmount = resultCandidates.singleOrNull()
if (packageName !in supportedPackages ||
containsBlockedStatus(text) ||
isHistoryPageText(text) ||
isPaymentInputPage(text)
) return null
) return PaymentParseOutcome(null, resultCandidates.size, null, null, null)
val eventId = flowSessionId?.let { "a:" + packageName + ":" + it }
?: "a:" + windowId + ":" + sha256(normalize(text)) + ":" + (eventTime / 10_000L)
val status = detectStatus(text, expectedType)
@@ -156,46 +216,85 @@ object PaymentParser {
null
}
if (parsed != null) {
val uniqueResultAmount = uniqueAmountCents(text)
val standaloneRedPacketIncome = parsed.recognitionKind in setOf(
"red_packet_receive",
"red_packet_refund",
)
val expectedMatches = expectedAmountCents == null ||
uniqueResultAmount == expectedAmountCents
val expectedMatches = expectedCents == null ||
uniqueResultAmount == expectedCents
val conflict = expectedCents != null && uniqueResultAmount != null &&
uniqueResultAmount != expectedCents
val evidenceHigh = when {
standaloneRedPacketIncome -> uniqueResultAmount == parsed.amountCents
submittedFlow -> trustedFlow && resultTransitionObserved &&
uniqueResultAmount == parsed.amountCents && expectedMatches
uniqueResultAmount == parsed.amountCents && expectedMatches && !conflict
else -> false
}
return parsed.copy(
val signal = parsed.copy(
evidenceConfidence = if (evidenceHigh) "high" else "confirm",
amountEvidenceStrength = if (uniqueResultAmount != null) "strong" else "weak",
amountCandidateCount = resultCandidates.size,
expectedAmountMatched = if (expectedCents != null && uniqueResultAmount != null) {
expectedMatches
} else {
null
},
amountIssueReason = if (conflict) "result_amount_conflict" else null,
)
return PaymentParseOutcome(
signal,
resultCandidates.size,
uniqueResultAmount,
signal.expectedAmountMatched,
signal.amountIssueReason,
)
}
val redPacketSentSurface = flowKind == "red_packet_send" &&
hasRedPacketSentSurface(text)
if (status.strength == PaymentStatusStrength.NONE && !redPacketSentSurface) return null
val direction = status.direction ?: if (redPacketSentSurface) "expense" else return null
val resultAmount = uniqueAmountCents(text)
if (status.strength == PaymentStatusStrength.NONE && !redPacketSentSurface) {
return PaymentParseOutcome(null, resultCandidates.size, null, null, null)
}
val direction = status.direction ?: if (redPacketSentSurface) "expense" else {
return PaymentParseOutcome(null, resultCandidates.size, null, null, "direction_unknown")
}
val resultAmount = uniqueResultAmount
val canUseExpectedAmount = resultAmount == null &&
expectedAmountCents != null &&
expectedCents != null &&
submittedFlow &&
resultTransitionObserved &&
expectedType == direction
val amountCents = resultAmount ?: expectedAmountCents?.takeIf { canUseExpectedAmount }
?: return null
val expectedMatched = expectedAmountCents != null && amountCents == expectedAmountCents
val amountCents = resultAmount ?: expectedCents?.takeIf { canUseExpectedAmount }
?: return PaymentParseOutcome(
null,
resultCandidates.size,
resultAmount,
null,
if (expectedCents == null) "expected_amount_missing" else "missing_amount",
)
val expectedMatched = if (expectedCents != null && resultAmount != null) {
resultAmount == expectedCents
} else {
null
}
val conflict = expectedMatched == false
val expectedStrong = expectedEvidence?.isStrong == true
val evidenceHigh = trustedFlow && submittedFlow && resultTransitionObserved &&
expectedMatched && expectedType == direction
expectedType == direction && !conflict &&
(resultAmount != null || expectedStrong)
val kind = flowKind ?: recognitionKind(text, direction)
val merchant = extractMerchant(text)
val orderId = extractOrderId(text)
val transferDirection = direction.takeIf { kind == "transfer" }?.let {
if (it == "income") "in" else "out"
}
return PaymentSignal(
val issueReason = when {
conflict -> "result_amount_conflict"
resultAmount == null && !expectedStrong -> "weak_expected_fallback"
resultAmount == null -> "expected_amount_fallback"
else -> null
}
val signal = PaymentSignal(
packageName = packageName,
channel = "accessibility",
amountCents = amountCents,
@@ -222,6 +321,17 @@ object PaymentParser {
),
transferDirection = transferDirection,
counterparty = merchant.takeIf { kind == "transfer" },
amountEvidenceStrength = if (resultAmount != null || expectedStrong) "strong" else "weak",
amountCandidateCount = resultCandidates.size,
expectedAmountMatched = expectedMatched,
amountIssueReason = issueReason,
)
return PaymentParseOutcome(
signal,
resultCandidates.size,
resultAmount,
expectedMatched,
issueReason,
)
}
fun fromNotification(notification: StatusBarNotification): PaymentSignal? {
@@ -313,8 +423,12 @@ object PaymentParser {
}
fun uniqueAmountCents(value: String): Long? {
return amountCandidateCents(value).singleOrNull()
}
fun amountCandidateCents(value: String): List<Long> {
val normalized = normalize(value)
val cents = buildList {
return buildList {
amountPatterns.forEach { pattern ->
pattern.findAll(normalized).forEach { match ->
match.groupValues.getOrNull(1)?.toDoubleOrNull()?.let { amount ->
@@ -332,7 +446,50 @@ object PaymentParser {
}
}
}.distinct()
return cents.singleOrNull()
}
fun expectedAmountEvidence(
pageText: String,
eventText: String,
flowSessionId: String,
windowId: Int,
capturedAtEpochMs: Long,
): ExpectedAmountEvidence? {
val pageFingerprint = sha256(normalize(pageText))
val labeled = LABELED_AMOUNT.findAll(normalize(pageText))
.mapNotNull { match ->
match.groupValues.getOrNull(1)?.toDoubleOrNull()?.let {
(it * 100).roundToLong()
}
}
.filter { it > 0 }
.distinct()
.toList()
if (labeled.size == 1) {
return ExpectedAmountEvidence(
labeled.single(),
flowSessionId,
windowId,
pageFingerprint,
capturedAtEpochMs,
labeled.size,
"labeled_page",
"strong",
)
}
val eventCandidates = amountCandidateCents(eventText)
if (eventCandidates.size != 1) return null
val explicitEvent = EXPLICIT_AMOUNT.containsMatchIn(normalize(eventText))
return ExpectedAmountEvidence(
eventCandidates.single(),
flowSessionId,
windowId,
pageFingerprint,
capturedAtEpochMs,
eventCandidates.size,
"payment_event",
if (explicitEvent) "strong" else "weak",
)
}
fun isPaymentInputPage(value: String): Boolean {
@@ -548,6 +705,10 @@ object PaymentParser {
private val STANDALONE_AMOUNT = Regex(
"""^\s*([¥￥]?)\s*([0-9]{1,8}(?:\.[0-9]{1,2})?)\s*(元?)\s*$""",
)
private val LABELED_AMOUNT = Regex(
"""(?:实付|付款金额|支付金额|转账金额|红包金额|收款金额|到账金额|交易金额)[：: \t]*(?:\r?\n[：: \t]*)?[¥￥]?[ \t]*([0-9]+(?:\.[0-9]{1,2})?)""",
)
private val EXPLICIT_AMOUNT = Regex("""[¥￥]\s*[0-9]+(?:\.[0-9]{1,2})?|[0-9]+\.[0-9]{1,2}""")
private const val MAX_SUCCESS_HEADING_CHARS = 48
private const val MAX_PAYMENT_SCAN_LINES = 48
private const val MAX_HISTORY_TITLE_LINES = 3
frontend/android/app/src/main/kotlin/com/nx/miaoji/RecognitionStore.kt
+153 -9
View File
@@ -37,6 +37,13 @@ data class PendingRecognitionBatch(
val images: List<PendingBatchImage>,
)
internal data class AmountMergeResult(
val amountCents: Long,
val source: String,
val strength: String,
val conflict: Boolean,
)
class RecognitionStore(context: Context) :
SQLiteOpenHelper(context, "recognition_queue.db", null, 3) {
override fun onCreate(db: SQLiteDatabase) {
@@ -178,20 +185,27 @@ class RecognitionStore(context: Context) :
id = existing.id
val mergedMask = existing.channelMask or channelBit
val hasNonAiEvidence = mergedMask and channelBit("recognition_ai").inv() != 0
val high = hasNonAiEvidence &&
val mergedPayload = mergePayload(existing.payload, signal)
val amountConflict = mergedPayload.optBoolean("amountConflict", false)
val high = !amountConflict && hasNonAiEvidence &&
(signalHigh ||
(mergedMask and ACCESSIBILITY_NOTIFICATION_MASK) ==
ACCESSIBILITY_NOTIFICATION_MASK ||
existing.highConfidence)
val mergedPayload = mergePayload(existing.payload, signal)
writableDatabase.update(
"candidates",
ContentValues().apply {
put("channel_mask", mergedMask)
put("known_template", if (existing.knownTemplate || signal.knownTemplate) 1 else 0)
put("amount_cents", kotlin.math.round(mergedPayload.optDouble("amount") * 100).toLong())
put("direction", mergedPayload.optString("type", signal.type))
put("high_confidence", if (high) 1 else 0)
put("payload_encrypted", encryptPayload(mergedPayload))
if (high && existing.state == "pending_confirm") put("state", "auto_ready")
if (amountConflict) {
put("state", "pending_confirm")
} else if (high && existing.state == "pending_confirm") {
put("state", "auto_ready")
}
put("updated_at", now)
put("available_at", now + MERGE_DELAY_MS)
if (batchId != null) {
@@ -525,9 +539,21 @@ class RecognitionStore(context: Context) :
val original = JSONObject(row.payload.toString())
val kind = action.optString("action")
val payload = JSONObject(row.payload.toString())
if (kind == "update") applyActionFields(payload, action)
if (kind == "update") {
applyActionFields(payload, action)
if (payload.optDouble("amount") != original.optDouble("amount")) {
payload
.put("amountSource", "ai_batch")
.put("amountEvidenceStrength", "strong")
.put("amountConflict", false)
.remove("conflictingAmount")
payload.remove("conflictingAmountSource")
payload.remove("amountIssueReason")
}
}
payload.put("sourceOverride", "recognition_ai")
val reason = action.optString("reason", "AI 对账")
val amountConflict = payload.optBoolean("amountConflict", false)
writableDatabase.update(
"candidates",
ContentValues().apply {
@@ -540,8 +566,15 @@ class RecognitionStore(context: Context) :
put("channel_mask", row.channelMask or channelBit("recognition_ai"))
put("ai_action", kind)
put("ai_reason", reason.take(80))
put("state", if (kind == "drop") "ai_dropped" else "auto_ready")
put("high_confidence", 1)
put(
"state",
when {
kind == "drop" -> "ai_dropped"
amountConflict -> "pending_confirm"
else -> "auto_ready"
},
)
put("high_confidence", if (amountConflict) 0 else 1)
put("updated_at", now)
},
"id = ? AND batch_id = ?",
@@ -631,14 +664,17 @@ class RecognitionStore(context: Context) :
// The server guarantees one action per candidate. Preserve anything omitted
// by a malformed response instead of silently losing a payment.
writableDatabase.rawQuery(
"SELECT id FROM candidates WHERE batch_id = ? AND state = 'batch_collecting'",
"SELECT id, payload_encrypted FROM candidates WHERE batch_id = ? AND state = 'batch_collecting'",
arrayOf(batchId),
).use { cursor ->
while (cursor.moveToNext()) {
val amountConflict = decryptPayload(cursor.getString(1))
?.optBoolean("amountConflict", false) == true
writableDatabase.update(
"candidates",
ContentValues().apply {
put("state", "auto_ready")
put("state", if (amountConflict) "pending_confirm" else "auto_ready")
put("high_confidence", if (amountConflict) 0 else 1)
put("ai_action", "keep")
put("ai_reason", "AI 未返回该候选，已保留本地结果")
put("updated_at", now)
@@ -1011,6 +1047,10 @@ class RecognitionStore(context: Context) :
.put("recognitionKind", signal.recognitionKind)
.put("categoryHint", signal.categoryHint)
.put("amountSource", signal.amountSource)
.put("amountEvidenceStrength", signal.amountEvidenceStrength)
.put("amountCandidateCount", signal.amountCandidateCount)
.put("expectedAmountMatched", signal.expectedAmountMatched)
.put("amountIssueReason", signal.amountIssueReason)
.put("resultFingerprint", signal.resultFingerprint)
.put("identityConfidence", signal.identityConfidence)
.put("transferDirection", signal.transferDirection)
@@ -1043,7 +1083,36 @@ class RecognitionStore(context: Context) :
if (existing.isNull("sourceText") && signal.sourceText != null) existing.put("sourceText", signal.sourceText)
if (existing.optString("recognitionKind").isBlank()) existing.put("recognitionKind", signal.recognitionKind)
if (existing.isNull("categoryHint") && signal.categoryHint != null) existing.put("categoryHint", signal.categoryHint)
if (existing.optString("amountSource").isBlank()) existing.put("amountSource", signal.amountSource)
val existingAmountCents = kotlin.math.round(existing.optDouble("amount") * 100).toLong()
val amountMerge = mergeAmountEvidence(
existingAmountCents = existingAmountCents,
existingSource = existing.optString("amountSource", "result"),
existingStrength = existing.optString("amountEvidenceStrength", "strong"),
incoming = signal,
)
val amountConflict = existing.optBoolean("amountConflict", false) ||
amountMerge.conflict
existing
.put("amount", amountMerge.amountCents / 100.0)
.put("amountSource", amountMerge.source)
.put("amountEvidenceStrength", amountMerge.strength)
.put("amountConflict", amountConflict)
.put("amountCandidateCount", maxOf(
existing.optInt("amountCandidateCount", 0),
signal.amountCandidateCount,
))
if (amountMerge.conflict) {
existing
.put("conflictingAmount", signal.amountCents / 100.0)
.put("conflictingAmountSource", signal.amountSource)
}
if (amountConflict) {
existing.put("amountIssueReason", "result_amount_conflict")
} else if (signal.amountIssueReason != null) {
existing.put("amountIssueReason", signal.amountIssueReason)
} else if (amountMerge.source == "result") {
existing.remove("amountIssueReason")
}
if (existing.isNull("resultFingerprint") && signal.resultFingerprint != null) {
existing.put("resultFingerprint", signal.resultFingerprint)
}
@@ -1111,6 +1180,81 @@ class RecognitionStore(context: Context) :
private const val MAX_BATCH_ITEMS = 10
private const val MAX_BATCH_IMAGE_BYTES = 1024 * 1024
internal fun mergeAmountEvidence(
existingAmountCents: Long,
existingSource: String,
existingStrength: String,
incoming: PaymentSignal,
): AmountMergeResult {
if (existingAmountCents == incoming.amountCents) {
val incomingRank = amountEvidenceRank(
incoming.amountSource,
incoming.amountEvidenceStrength,
)
val existingRank = amountEvidenceRank(existingSource, existingStrength)
return if (incomingRank > existingRank) {
AmountMergeResult(
incoming.amountCents,
incoming.amountSource,
incoming.amountEvidenceStrength,
false,
)
} else {
AmountMergeResult(
existingAmountCents,
existingSource,
existingStrength,
false,
)
}
}
val incomingRank = amountEvidenceRank(
incoming.amountSource,
incoming.amountEvidenceStrength,
)
val existingRank = amountEvidenceRank(existingSource, existingStrength)
val bothStrong = existingStrength == "strong" &&
incoming.amountEvidenceStrength == "strong"
val independentConflict = bothStrong &&
(existingSource == "result" || incoming.amountSource == "result")
if (independentConflict) {
return AmountMergeResult(
existingAmountCents,
existingSource,
existingStrength,
true,
)
}
val useIncoming = when {
incoming.amountSource == "result" && existingSource != "result" -> true
existingSource == "result" && incoming.amountSource != "result" -> false
else -> incomingRank > existingRank
}
return if (useIncoming) {
AmountMergeResult(
incoming.amountCents,
incoming.amountSource,
incoming.amountEvidenceStrength,
false,
)
} else {
AmountMergeResult(
existingAmountCents,
existingSource,
existingStrength,
false,
)
}
}
private fun amountEvidenceRank(source: String, strength: String): Int = when {
source == "result" && strength == "strong" -> 4
source == "result" -> 3
strength == "strong" -> 2
else -> 1
}
internal fun clientRequestIdFor(signal: PaymentSignal): String {
val basis = when {
!signal.orderId.isNullOrBlank() ->
frontend/android/app/src/main/kotlin/com/nx/miaoji/ScreenshotAccessibilityService.kt
+66 -26
View File
@@ -47,7 +47,7 @@ class ScreenshotAccessibilityService : AccessibilityService() {
var kind: String,
var originWindowId: Int = windowId,
var originPageHash: String? = null,
var expectedAmountCents: Long? = null,
var expectedAmountEvidence: ExpectedAmountEvidence? = null,
var expectedType: String? = null,
var committedAt: Long? = null,
var completedAt: Long? = null,
@@ -188,7 +188,7 @@ class ScreenshotAccessibilityService : AccessibilityService() {
)
return
}
val observed = PaymentParser.fromAccessibility(
val observedOutcome = PaymentParser.fromAccessibilityOutcome(
packageName = recognizedPackage,
text = combined,
eventTime = currentEvent.eventTime,
@@ -196,6 +196,7 @@ class ScreenshotAccessibilityService : AccessibilityService() {
expectedType = status.direction ?: existingFlow.expectedType,
flowKind = currentKind,
)
val observed = observedOutcome.signal
val resultChanged = observed?.resultFingerprint != null &&
existingFlow.resultFingerprint != null &&
observed.resultFingerprint != existingFlow.resultFingerprint
@@ -229,6 +230,7 @@ class ScreenshotAccessibilityService : AccessibilityService() {
nextFlow,
page.nodeCount,
"tree",
parseOutcome = observedOutcome,
)
return
}
@@ -267,13 +269,14 @@ class ScreenshotAccessibilityService : AccessibilityService() {
currentEvent.windowId,
now,
forceNew = existingFlow?.completed == true ||
existingFlow?.committedAt != null ||
existingFlow?.packageName != recognizedPackage,
kind = inferredKind,
)
else -> existingFlow?.takeIf { !it.completed }
}
if (armedFlow != null) {
updateFlowEvidence(armedFlow, combined)
updateFlowEvidence(armedFlow, combined, eventText, now)
if (clickedPaymentAction) {
armedFlow.committedAt = armedFlow.committedAt ?: now
armedFlow.expectedType = armedFlow.expectedType ?: "expense"
@@ -284,19 +287,28 @@ class ScreenshotAccessibilityService : AccessibilityService() {
}
val directFlow = paymentFlow?.takeIf { !it.completed }
val directSignal = if (combined.isBlank()) null else PaymentParser.fromAccessibility(
packageName = recognizedPackage,
text = combined,
eventTime = currentEvent.eventTime,
windowId = currentEvent.windowId,
flowSessionId = directFlow?.id,
trustedFlow = directFlow?.trusted == true,
expectedAmountCents = directFlow?.expectedAmountCents,
expectedType = directFlow?.expectedType,
resultTransitionObserved = directFlow?.resultTransitionObserved == true,
submittedFlow = directFlow?.committedAt != null,
flowKind = directFlow?.kind,
)
val directOutcome = if (combined.isBlank()) null else {
PaymentParser.fromAccessibilityOutcome(
packageName = recognizedPackage,
text = combined,
eventTime = currentEvent.eventTime,
windowId = currentEvent.windowId,
flowSessionId = directFlow?.id,
trustedFlow = directFlow?.trusted == true,
expectedAmountEvidence = directFlow?.expectedAmountEvidence?.takeIf {
it.belongsTo(
directFlow.id,
directFlow.startedAt,
directFlow.committedAt,
)
},
expectedType = directFlow?.expectedType,
resultTransitionObserved = directFlow?.resultTransitionObserved == true,
submittedFlow = directFlow?.committedAt != null,
flowKind = directFlow?.kind,
)
}
val directSignal = directOutcome?.signal
if (directSignal != null) {
val flow = directFlow ?: armPaymentFlow(
recognizedPackage,
@@ -313,6 +325,7 @@ class ScreenshotAccessibilityService : AccessibilityService() {
flow,
page.nodeCount,
"tree",
parseOutcome = directOutcome,
)
return
}
@@ -411,13 +424,35 @@ class ScreenshotAccessibilityService : AccessibilityService() {
).also { paymentFlow = it }
}
private fun updateFlowEvidence(flow: PaymentFlow, text: String) {
if (text.isBlank() || flow.completed || flow.resultTransitionObserved) return
private fun updateFlowEvidence(
flow: PaymentFlow,
text: String,
eventText: String,
now: Long,
) {
if (text.isBlank() || flow.completed || flow.committedAt != null ||
flow.resultTransitionObserved
) {
return
}
if (PaymentParser.detectStatus(text).strength != PaymentStatusStrength.NONE) return
val pageHash = PaymentParser.sha256(text)
flow.originPageHash = flow.originPageHash ?: pageHash
flow.expectedAmountCents = flow.expectedAmountCents
?: PaymentParser.uniqueAmountCents(text)
val observed = PaymentParser.expectedAmountEvidence(
pageText = text,
eventText = eventText,
flowSessionId = flow.id,
windowId = flow.windowId,
capturedAtEpochMs = now,
)
if (observed != null) {
val existing = flow.expectedAmountEvidence
if (existing == null || observed.isStrong && !existing.isStrong ||
observed.source == existing.source
) {
flow.expectedAmountEvidence = observed
}
}
flow.expectedType = flow.expectedType
?: PaymentParser.inferContextDirection(text)
PaymentParser.detectFlowKind(text)?.let { detected ->
@@ -468,6 +503,7 @@ class ScreenshotAccessibilityService : AccessibilityService() {
stage: String,
recordDiagnostic: Boolean = true,
evidenceImage: ByteArray? = null,
parseOutcome: PaymentParseOutcome? = null,
) {
if (flow.completed) {
evidenceImage?.fill(0)
@@ -503,11 +539,13 @@ class ScreenshotAccessibilityService : AccessibilityService() {
"confirm"
},
nodeCount = nodeCount,
amountCandidates = 1,
reason = if (batchEnabled) "queued_for_ai" else "success",
expectedAmountMatched = flow.expectedAmountCents?.let {
it == signal.amountCents
},
amountCandidates = parseOutcome?.amountCandidateCount
?: signal.amountCandidateCount,
reason = signal.amountIssueReason
?: parseOutcome?.reason
?: if (batchEnabled) "queued_for_ai" else "success",
expectedAmountMatched = parseOutcome?.expectedAmountMatched
?: signal.expectedAmountMatched,
resultTransitionObserved = flow.resultTransitionObserved,
recognitionKind = signal.recognitionKind,
amountSource = signal.amountSource,
@@ -671,7 +709,9 @@ class ScreenshotAccessibilityService : AccessibilityService() {
flowSessionId = flow.id,
trustedFlow = flow.trusted,
capturedAt = capturedAt,
expectedAmountCents = flow.expectedAmountCents,
expectedAmountEvidence = flow.expectedAmountEvidence?.takeIf {
it.belongsTo(flow.id, flow.startedAt, flow.committedAt)
},
expectedType = flow.expectedType,
resultTransitionObserved = visualTransitionObserved,
submittedFlow = flow.committedAt != null,
frontend/android/app/src/test/kotlin/com/nx/miaoji/PaymentParserTest.kt
+165 -5
View File
@@ -93,7 +93,7 @@ class PaymentParserTest {
windowId = 7,
flowSessionId = "flow-a",
trustedFlow = true,
expectedAmountCents = 2_000L,
expectedAmountEvidence = expectedEvidence(2_000L, "strong", "flow-a"),
expectedType = "expense",
resultTransitionObserved = true,
submittedFlow = true,
@@ -107,7 +107,7 @@ class PaymentParserTest {
windowId = 7,
flowSessionId = "flow-b",
trustedFlow = true,
expectedAmountCents = 3_000L,
expectedAmountEvidence = expectedEvidence(3_000L, "strong", "flow-b"),
expectedType = "expense",
resultTransitionObserved = true,
)
@@ -149,7 +149,7 @@ class PaymentParserTest {
windowId = 8,
flowSessionId = "payment-flow",
trustedFlow = true,
expectedAmountCents = 1_880L,
expectedAmountEvidence = expectedEvidence(1_880L, "strong", "payment-flow"),
expectedType = "expense",
resultTransitionObserved = true,
submittedFlow = true,
@@ -171,7 +171,7 @@ class PaymentParserTest {
windowId = 8,
flowSessionId = "payment-flow",
trustedFlow = true,
expectedAmountCents = 1_880L,
expectedAmountEvidence = expectedEvidence(1_880L, "strong", "payment-flow"),
expectedType = "expense",
resultTransitionObserved = true,
submittedFlow = false,
@@ -190,7 +190,7 @@ class PaymentParserTest {
windowId = 9,
flowSessionId = "red-packet-flow",
trustedFlow = true,
expectedAmountCents = 2_000L,
expectedAmountEvidence = expectedEvidence(2_000L, "strong", "red-packet-flow"),
expectedType = "expense",
resultTransitionObserved = true,
submittedFlow = true,
@@ -381,6 +381,166 @@ class PaymentParserTest {
)
}
@Test
fun expectedAmountEvidenceIgnoresUnlabelledChatHistory() {
val evidence = PaymentParser.expectedAmountEvidence(
pageText = "聊天记录\n¥0.01\n你发起了一笔转账\n¥0.02\n确认转账",
eventText = "确认转账",
flowSessionId = "transfer-003",
windowId = 7,
capturedAtEpochMs = 1_000L,
)
assertNull(evidence)
}
@Test
fun labeledCurrentTransferAmountWinsOverChatHistory() {
val evidence = PaymentParser.expectedAmountEvidence(
pageText = "聊天记录\n¥0.01\n¥0.02\n转账金额\n¥0.03\n确认转账",
eventText = "确认转账",
flowSessionId = "transfer-003",
windowId = 7,
capturedAtEpochMs = 1_000L,
)
assertEquals(3L, evidence?.amountCents)
assertEquals("strong", evidence?.strength)
assertEquals("labeled_page", evidence?.source)
}
@Test
fun labeledAmountMustBeOnTheSameOrAdjacentNode() {
val evidence = PaymentParser.expectedAmountEvidence(
pageText = "转账金额\n付款说明\n¥0.01",
eventText = "确认转账",
flowSessionId = "transfer-003",
windowId = 7,
capturedAtEpochMs = 1_000L,
)
assertNull(evidence)
}
@Test
fun resultWithoutAmountUsesOnlyStrongCurrentFlowEvidence() {
val outcome = PaymentParser.fromAccessibilityOutcome(
packageName = PaymentParser.WECHAT,
text = "转账成功",
eventTime = 2_000L,
windowId = 7,
flowSessionId = "transfer-003",
trustedFlow = true,
expectedAmountEvidence = expectedEvidence(3L, "strong"),
expectedType = "expense",
resultTransitionObserved = true,
submittedFlow = true,
flowKind = "transfer",
)
assertEquals(3L, outcome.signal?.amountCents)
assertEquals("high", outcome.signal?.evidenceConfidence)
assertEquals("expected_amount_fallback", outcome.reason)
assertNull(outcome.expectedAmountMatched)
}
@Test
fun weakFallbackAndConflictingResultRequireConfirmation() {
val weak = PaymentParser.fromAccessibilityOutcome(
packageName = PaymentParser.WECHAT,
text = "转账成功",
eventTime = 2_000L,
windowId = 7,
flowSessionId = "transfer-003",
trustedFlow = true,
expectedAmountEvidence = expectedEvidence(3L, "weak"),
expectedType = "expense",
resultTransitionObserved = true,
submittedFlow = true,
flowKind = "transfer",
)
val conflict = PaymentParser.fromAccessibilityOutcome(
packageName = PaymentParser.WECHAT,
text = "转账成功\n¥0.01",
eventTime = 2_000L,
windowId = 7,
flowSessionId = "transfer-003",
trustedFlow = true,
expectedAmountEvidence = expectedEvidence(3L, "strong"),
expectedType = "expense",
resultTransitionObserved = true,
submittedFlow = true,
flowKind = "transfer",
)
assertEquals("confirm", weak.signal?.evidenceConfidence)
assertEquals("weak_expected_fallback", weak.reason)
assertEquals(1L, conflict.signal?.amountCents)
assertEquals("confirm", conflict.signal?.evidenceConfidence)
assertEquals(false, conflict.expectedAmountMatched)
assertEquals("result_amount_conflict", conflict.reason)
}
@Test
fun strongerResultReplacesWeakFallbackButStrongConflictStaysFlagged() {
val resultSignal = paymentSignal(
channel = "local_ocr",
sourceEventId = "ocr:wechat:transfer-003",
flowSessionId = "transfer-003",
).copy(
amountCents = 3L,
amountSource = "result",
amountEvidenceStrength = "strong",
)
val replacement = RecognitionStore.mergeAmountEvidence(
existingAmountCents = 1L,
existingSource = "expected",
existingStrength = "weak",
incoming = resultSignal,
)
val conflict = RecognitionStore.mergeAmountEvidence(
existingAmountCents = 1L,
existingSource = "expected",
existingStrength = "strong",
incoming = resultSignal,
)
assertEquals(3L, replacement.amountCents)
assertFalse(replacement.conflict)
assertEquals(1L, conflict.amountCents)
assertTrue(conflict.conflict)
}
@Test
fun consecutiveSmallTransfersKeepFourFlowsAndSumToSevenCents() {
val amounts = listOf(1L, 1L, 2L, 3L)
val signals = amounts.mapIndexed { index, cents ->
paymentSignal(
channel = "accessibility",
sourceEventId = "a:wechat:transfer-$index",
flowSessionId = "transfer-$index",
).copy(amountCents = cents)
}
assertEquals(7L, signals.sumOf { it.amountCents })
assertEquals(4, signals.map(RecognitionStore::flowStrongKeyFor).toSet().size)
}
private fun expectedEvidence(
amountCents: Long,
strength: String,
flowSessionId: String = "transfer-003",
) = ExpectedAmountEvidence(
amountCents = amountCents,
flowSessionId = flowSessionId,
windowId = 7,
pageFingerprint = "payment-page",
capturedAtEpochMs = 1_500L,
candidateCount = 1,
source = "labeled_page",
strength = strength,
)
private fun paymentSignal(
channel: String,
sourceEventId: String,
frontend/lib/shared/services/recognition_diagnostic_formatter.dart
+8 -2
View File
@@ -80,7 +80,10 @@ class RecognitionDiagnosticDisplay {
'direction_unknown' => '识别到完成状态，但无法确认收支方向',
'missing_amount' => '成功状态已识别，但没有找到金额',
'expected_amount_missing' => '结果页没有金额，且付款前金额不唯一或未捕获',
'expected_amount_fallback' => '结果页金额缺失，已使用付款前确认的唯一金额',
'expected_amount_fallback' => '结果页未读到金额，已使用本次付款流程确认的金额',
'weak_expected_fallback' => '结果页未读到金额，付款前金额来源不可靠，请确认后入账',
'result_amount_conflict' => '结果页金额与本次付款金额不一致，请确认后入账',
'amount_source_untrusted' => '金额无法与当前付款流程可靠关联，请确认后入账',
'red_packet_not_settled' => '红包尚未明确到账或退回，不会自动入账',
'duplicate_result_surface' => '同一结果页已处理，本次刷新已忽略',
'weak_status_confirm' => '只识别到弱完成状态，组合证据不足，需确认后入账',
@@ -123,7 +126,7 @@ class RecognitionDiagnosticDisplay {
static String? _amountSourceLabel(String? value) {
return switch (value) {
'expected' => '使用付款前金额',
'expected' => '使用本次付款金额',
'result' => '使用结果页金额',
_ => null,
};
@@ -172,5 +175,8 @@ class RecognitionDiagnosticDisplay {
'red_packet_not_settled',
'weak_status_confirm',
'ambiguous_or_unarmed',
'weak_expected_fallback',
'result_amount_conflict',
'amount_source_untrusted',
};
}
frontend/test/theme_and_time_test.dart
+12
View File
@@ -163,5 +163,17 @@ void main() {
).summaryLabel,
'已合并',
);
expect(
RecognitionDiagnosticDisplay.from(
diagnostic('confirm', 'expected_amount_fallback'),
).reasonLabel,
'结果页未读到金额，已使用本次付款流程确认的金额',
);
expect(
RecognitionDiagnosticDisplay.from(
diagnostic('confirm', 'result_amount_conflict'),
).reasonLabel,
'结果页金额与本次付款金额不一致，请确认后入账',
);
});
}