Files
jizhi/frontend/lib/shared/services/screenshot_channel.dart
T

764 lines
26 KiB
Dart

import 'dart:convert';
import 'package:flutter/services.dart';
class RecognitionDiagnostic {
final DateTime at;
final String appName, stage, result, reason, statusStrength;
final String? recognitionKind, amountSource, resultFingerprint;
final int nodeCount, amountCandidates;
final int? ocrMs;
final bool? expectedAmountMatched, resultTransitionObserved;
final List<String> ocrPreview;
final DateTime? previewExpiresAt;
const RecognitionDiagnostic({
required this.at,
required this.appName,
required this.stage,
required this.result,
required this.reason,
required this.nodeCount,
required this.amountCandidates,
this.ocrMs,
this.statusStrength = 'none',
this.recognitionKind,
this.amountSource,
this.resultFingerprint,
this.expectedAmountMatched,
this.resultTransitionObserved,
this.ocrPreview = const [],
this.previewExpiresAt,
});
factory RecognitionDiagnostic.fromJson(Map<String, dynamic> value) {
final epoch = (value['at'] as num?)?.toInt() ?? 0;
return RecognitionDiagnostic(
at: DateTime.fromMillisecondsSinceEpoch(epoch),
appName: value['appName']?.toString() ?? '支付应用',
stage: value['stage']?.toString() ?? 'event',
result: value['result']?.toString() ?? 'unknown',
reason: value['reason']?.toString() ?? '',
nodeCount: (value['nodeCount'] as num?)?.toInt() ?? 0,
amountCandidates: (value['amountCandidates'] as num?)?.toInt() ?? 0,
ocrMs: (value['ocrMs'] as num?)?.toInt(),
statusStrength: value['statusStrength']?.toString() ?? 'none',
recognitionKind: value['recognitionKind']?.toString(),
amountSource: value['amountSource']?.toString(),
resultFingerprint: value['resultFingerprint']?.toString(),
expectedAmountMatched: value['expectedAmountMatched'] as bool?,
resultTransitionObserved: value['resultTransitionObserved'] as bool?,
ocrPreview:
(value['ocrPreview'] as List<dynamic>?)
?.map((item) => item.toString())
.toList(growable: false) ??
const [],
previewExpiresAt: (value['previewExpiresAt'] as num?) == null
? null
: DateTime.fromMillisecondsSinceEpoch(
(value['previewExpiresAt'] as num).toInt(),
),
);
}
}
enum AccessibilityConnectionState {
unauthorized,
reconnecting,
connected,
disconnected;
static AccessibilityConnectionState parse(
Object? value, {
required bool authorized,
required bool connected,
}) {
return switch (value?.toString()) {
'reconnecting' => AccessibilityConnectionState.reconnecting,
'connected' => AccessibilityConnectionState.connected,
'disconnected' => AccessibilityConnectionState.disconnected,
'unauthorized' => AccessibilityConnectionState.unauthorized,
_ =>
!authorized
? unauthorized
: connected
? AccessibilityConnectionState.connected
: disconnected,
};
}
}
class RecognitionStatus {
final bool accessibilityAuthorized;
final bool accessibilityConnected;
final AccessibilityConnectionState accessibilityConnectionState;
final DateTime? accessibilityLastConnectedAt;
final DateTime? accessibilityLastDisconnectedAt;
final DateTime? recognitionProcessStartedAt;
final bool keepAliveExpected;
final bool keepAliveRunning;
final String? keepAliveError;
final bool recentsProtectionExpected;
final bool recentsProtectionActive;
final DateTime? lastRecognitionExitAt;
final String? lastRecognitionExitReason;
final bool taskCleanerRecoveryNeeded;
final bool notificationAuthorized;
final bool notificationConnected;
final bool postNotificationsGranted;
final bool accessibilityEvents;
final bool notificationEvents;
final bool aiScreenshot;
final bool ocrDiagnosticPreview;
final DateTime? ocrDiagnosticPreviewExpiresAt;
final bool batteryOptimizationIgnored;
final String manufacturer;
final String? latestStatus;
final RecognitionDiagnostic? latestDiagnostic;
const RecognitionStatus({
required this.accessibilityAuthorized,
required this.accessibilityConnected,
this.accessibilityConnectionState =
AccessibilityConnectionState.unauthorized,
this.accessibilityLastConnectedAt,
this.accessibilityLastDisconnectedAt,
this.recognitionProcessStartedAt,
this.keepAliveExpected = false,
this.keepAliveRunning = false,
this.keepAliveError,
this.recentsProtectionExpected = false,
this.recentsProtectionActive = false,
this.lastRecognitionExitAt,
this.lastRecognitionExitReason,
this.taskCleanerRecoveryNeeded = false,
required this.notificationAuthorized,
required this.notificationConnected,
required this.postNotificationsGranted,
required this.accessibilityEvents,
required this.notificationEvents,
required this.aiScreenshot,
this.ocrDiagnosticPreview = false,
this.ocrDiagnosticPreviewExpiresAt,
this.batteryOptimizationIgnored = false,
this.manufacturer = '',
this.latestStatus,
this.latestDiagnostic,
});
factory RecognitionStatus.fromMap(Map<Object?, Object?> value) {
final rawSettings = value['settings']?.toString();
final rawDiagnostic = value['latestDiagnostic']?.toString();
final settings = rawSettings == null || rawSettings.isEmpty
? const <String, dynamic>{}
: jsonDecode(rawSettings) as Map<String, dynamic>;
final accessibilityAuthorized =
value['accessibilityAuthorized'] as bool? ?? false;
final accessibilityConnected =
value['accessibilityConnected'] as bool? ?? false;
DateTime? epochDate(String key) {
final epoch = (value[key] as num?)?.toInt() ?? 0;
return epoch <= 0 ? null : DateTime.fromMillisecondsSinceEpoch(epoch);
}
return RecognitionStatus(
accessibilityAuthorized: accessibilityAuthorized,
accessibilityConnected: accessibilityConnected,
accessibilityConnectionState: AccessibilityConnectionState.parse(
value['accessibilityConnectionState'],
authorized: accessibilityAuthorized,
connected: accessibilityConnected,
),
accessibilityLastConnectedAt: epochDate('accessibilityLastConnectedAt'),
accessibilityLastDisconnectedAt: epochDate(
'accessibilityLastDisconnectedAt',
),
recognitionProcessStartedAt: epochDate('recognitionProcessStartedAt'),
keepAliveExpected: value['keepAliveExpected'] as bool? ?? false,
keepAliveRunning: value['keepAliveRunning'] as bool? ?? false,
keepAliveError: value['keepAliveError']?.toString(),
recentsProtectionExpected:
value['recentsProtectionExpected'] as bool? ?? false,
recentsProtectionActive:
value['recentsProtectionActive'] as bool? ?? false,
lastRecognitionExitAt: epochDate('lastRecognitionExitAt'),
lastRecognitionExitReason: value['lastRecognitionExitReason']?.toString(),
taskCleanerRecoveryNeeded:
value['taskCleanerRecoveryNeeded'] as bool? ?? false,
notificationAuthorized: value['notificationAuthorized'] as bool? ?? false,
notificationConnected: value['notificationConnected'] as bool? ?? false,
postNotificationsGranted:
value['postNotificationsGranted'] as bool? ?? true,
accessibilityEvents: settings['accessibilityEvents'] as bool? ?? false,
notificationEvents: settings['notificationEvents'] as bool? ?? false,
aiScreenshot: settings['aiScreenshot'] as bool? ?? false,
ocrDiagnosticPreview: settings['ocrDiagnosticPreview'] as bool? ?? false,
ocrDiagnosticPreviewExpiresAt:
(settings['ocrDiagnosticPreviewExpiresAt'] as num?) == null
? null
: DateTime.fromMillisecondsSinceEpoch(
(settings['ocrDiagnosticPreviewExpiresAt'] as num).toInt(),
),
batteryOptimizationIgnored:
value['batteryOptimizationIgnored'] as bool? ?? false,
manufacturer: value['manufacturer']?.toString() ?? '',
latestStatus: value['latestStatus']?.toString(),
latestDiagnostic: rawDiagnostic == null || rawDiagnostic.isEmpty
? null
: RecognitionDiagnostic.fromJson(
jsonDecode(rawDiagnostic) as Map<String, dynamic>,
),
);
}
bool get accessibilityNeedsRecovery =>
accessibilityAuthorized &&
!accessibilityConnected &&
accessibilityConnectionState == AccessibilityConnectionState.disconnected;
bool get keepAliveNeedsRecovery => keepAliveExpected && !keepAliveRunning;
String get accessibilityConnectionLabel =>
switch (accessibilityConnectionState) {
AccessibilityConnectionState.unauthorized => '未授权',
AccessibilityConnectionState.reconnecting => '正在重连',
AccessibilityConnectionState.connected => '已连接',
AccessibilityConnectionState.disconnected => '服务未连接',
};
}
class RecognitionCandidate {
final String id, clientRequestId, state, confidence, type, source, appName;
final double amount;
final String? merchant, orderId, sourceText, note;
final String? transferDirection, counterparty;
final String? provider, providerTransactionId, recognitionOccurrenceId;
final String? evidenceFingerprint;
final String recognitionKind, amountSource;
final String identityConfidence;
final String? categoryHint, resultFingerprint;
final String? batchId, aiAction, aiReason;
final int? categoryId;
final int occurredAtEpochMs;
RecognitionCandidate.fromJson(Map<String, dynamic> value)
: id = value['id'] as String,
clientRequestId = value['clientRequestId'] as String,
state = value['state'] as String,
confidence = value['confidence'] as String,
type = value['type'] as String,
source = value['source'] as String,
appName = value['appName'] as String? ?? '支付应用',
amount = (value['amount'] as num).toDouble(),
merchant = value['merchant'] as String?,
orderId = value['orderId'] as String?,
sourceText = value['sourceText'] as String?,
note = value['note'] as String?,
transferDirection = value['transferDirection']?.toString(),
counterparty = value['counterparty']?.toString(),
provider = value['provider']?.toString(),
providerTransactionId = value['providerTransactionId']?.toString(),
recognitionOccurrenceId = value['recognitionOccurrenceId']?.toString(),
evidenceFingerprint = value['evidenceFingerprint']?.toString(),
recognitionKind = value['recognitionKind']?.toString() ?? 'payment',
categoryHint = value['categoryHint']?.toString(),
categoryId = (value['categoryId'] as num?)?.toInt(),
amountSource = value['amountSource']?.toString() ?? 'result',
resultFingerprint = value['resultFingerprint']?.toString(),
identityConfidence = value['identityConfidence']?.toString() ?? 'strong',
batchId = value['batchId']?.toString(),
aiAction = value['aiAction']?.toString(),
aiReason = value['aiReason']?.toString(),
occurredAtEpochMs = (value['occurredAtEpochMs'] as num).toInt();
bool get canAutoImport => state == 'auto_ready' && confidence == 'auto';
bool get isIncome =>
type == 'income' || type == 'transfer' && transferDirection == 'in';
String get typeLabel => type == 'transfer'
? transferDirection == 'in'
? '转入'
: '转出'
: isIncome
? '收入'
: '支出';
}
class RecognitionBatchItem {
final String candidateId, action, reason, state, type;
final double amount;
final String? merchant, transferDirection;
final bool canRestore;
RecognitionBatchItem.fromJson(Map<String, dynamic> value)
: candidateId = value['candidateId']?.toString() ?? '',
action = value['action']?.toString() ?? 'keep',
reason = value['reason']?.toString() ?? '',
state = value['state']?.toString() ?? '',
type = value['type']?.toString() ?? 'expense',
amount = (value['amount'] as num?)?.toDouble() ?? 0,
merchant = value['merchant']?.toString(),
transferDirection = value['transferDirection']?.toString(),
canRestore = value['canRestore'] as bool? ?? false;
bool get isIncome =>
type == 'income' || type == 'transfer' && transferDirection == 'in';
}
class RecognitionBatch {
final String id, state;
final DateTime openedAt, completedAt;
final int kept, updated, created, dropped;
final bool fallback;
final String? failureReason;
final List<RecognitionBatchItem> items;
RecognitionBatch.fromJson(Map<String, dynamic> value)
: id = value['id']?.toString() ?? '',
state = value['state']?.toString() ?? '',
openedAt = DateTime.fromMillisecondsSinceEpoch(
(value['openedAt'] as num?)?.toInt() ?? 0,
isUtc: true,
),
completedAt = DateTime.fromMillisecondsSinceEpoch(
(value['completedAt'] as num?)?.toInt() ?? 0,
isUtc: true,
),
kept = ((value['summary'] as Map?)?['kept'] as num?)?.toInt() ?? 0,
updated = ((value['summary'] as Map?)?['updated'] as num?)?.toInt() ?? 0,
created = ((value['summary'] as Map?)?['created'] as num?)?.toInt() ?? 0,
dropped = ((value['summary'] as Map?)?['dropped'] as num?)?.toInt() ?? 0,
fallback = ((value['summary'] as Map?)?['fallback'] as bool?) ?? false,
failureReason = value['failureReason']?.toString(),
items = (value['items'] as List<dynamic>? ?? const [])
.map(
(item) => RecognitionBatchItem.fromJson(
Map<String, dynamic>.from(item as Map),
),
)
.toList(growable: false);
}
class SpeechEvent {
final String type;
final String? text;
final double rms;
const SpeechEvent({required this.type, this.text, this.rms = 0});
}
/// Android native capabilities for screenshots, AI progress and speech.
class ScreenshotChannel {
static const _channel = MethodChannel('com.miaoji/screenshot');
static Future<RecognitionStatus>? _activeAccessibilityConnectionWait;
static void Function(String path)? _screenshotReady;
static void Function(String error)? _screenshotError;
static void Function(SpeechEvent event)? _speechEvent;
static void Function(Map<String, dynamic> action)? _recognitionAction;
static bool _handlerInstalled = false;
static Future<void> cleanupStaleScreenshots() async {
try {
await _channel.invokeMethod<int>('cleanupStaleScreenshots');
} on PlatformException {
// Cleanup is best-effort and will run again on the next launch.
} on MissingPluginException {
// Screenshot storage is Android-only.
}
}
static Future<String?> capture() async {
try {
return await _channel.invokeMethod<String>('captureScreenshot');
} on PlatformException catch (e) {
throw Exception(e.message ?? '截屏失败,请重试');
} on MissingPluginException {
return null;
}
}
static Future<void> startAiProgress() async {
try {
await _channel.invokeMethod<bool>('startAiProgress');
} on MissingPluginException {
// Non-Android platforms do not expose native progress notifications.
}
}
static Future<void> updateAiProgress(int count) async {
try {
await _channel.invokeMethod<bool>('updateAiProgress', {'count': count});
} on MissingPluginException {
// Non-Android platforms do not expose native progress notifications.
}
}
static Future<void> finishAiProgress(int count, double total) async {
try {
await _channel.invokeMethod<bool>('finishAiProgress', {
'count': count,
'total': total,
});
} on MissingPluginException {
// Non-Android platforms do not expose native progress notifications.
}
}
static Future<void> failAiProgress(String message) async {
try {
await _channel.invokeMethod<bool>('failAiProgress', {'message': message});
} on MissingPluginException {
// Non-Android platforms do not expose native progress notifications.
}
}
static Future<String?> recognizeSpeech() async {
try {
return await _channel.invokeMethod<String>('recognizeSpeech');
} on PlatformException catch (e) {
throw Exception(e.message ?? '语音识别不可用');
} on MissingPluginException {
return null;
}
}
static Future<void> startSpeechRecognition(
void Function(SpeechEvent event) onEvent,
) async {
_speechEvent = onEvent;
_installHandler();
try {
await _channel.invokeMethod<bool>('startSpeechRecognition');
} on PlatformException catch (e) {
_speechEvent = null;
throw Exception(e.message ?? '语音识别不可用');
} on MissingPluginException {
_speechEvent = null;
throw Exception('当前设备不支持语音识别');
}
}
static Future<void> stopSpeechRecognition() async {
try {
await _channel.invokeMethod<bool>('stopSpeechRecognition');
} on MissingPluginException {
// No native recognizer to stop.
}
}
static Future<void> cancelSpeechRecognition() async {
_speechEvent = null;
try {
await _channel.invokeMethod<bool>('cancelSpeechRecognition');
} on MissingPluginException {
// No native recognizer to cancel.
}
}
static Future<bool> isAccessibilityEnabled() async {
try {
return await _channel.invokeMethod<bool>('isAccessibilityEnabled') ??
false;
} on MissingPluginException {
return false;
}
}
static Future<void> openAccessibilitySettings() async {
try {
await _channel.invokeMethod('openAccessibilitySettings');
} on MissingPluginException {
// Accessibility shortcut is Android-only.
}
}
static Future<void> openQuickSettings() async {
try {
await _channel.invokeMethod('openQuickSettings');
} on MissingPluginException {
// Quick settings tiles are Android-only.
}
}
static Future<RecognitionStatus> recognitionStatus() async {
try {
final raw = await _channel.invokeMethod<Map<Object?, Object?>>(
'getRecognitionStatus',
);
return RecognitionStatus.fromMap(raw ?? const {});
} on MissingPluginException {
return const RecognitionStatus(
accessibilityAuthorized: false,
accessibilityConnected: false,
notificationAuthorized: false,
notificationConnected: false,
postNotificationsGranted: true,
accessibilityEvents: false,
notificationEvents: false,
aiScreenshot: false,
);
}
}
static Future<RecognitionStatus> waitForAccessibilityConnection({
Duration timeout = const Duration(seconds: 5),
Duration interval = const Duration(milliseconds: 500),
}) {
final active = _activeAccessibilityConnectionWait;
if (active != null) return active;
final operation = _waitForAccessibilityConnection(
timeout: timeout,
interval: interval,
);
_activeAccessibilityConnectionWait = operation;
return operation.whenComplete(() {
if (identical(_activeAccessibilityConnectionWait, operation)) {
_activeAccessibilityConnectionWait = null;
}
});
}
static Future<RecognitionStatus> _waitForAccessibilityConnection({
required Duration timeout,
required Duration interval,
}) async {
var status = await recognitionStatus();
final recognitionEnabled =
status.accessibilityEvents || status.aiScreenshot;
if (!recognitionEnabled ||
!status.accessibilityAuthorized ||
status.accessibilityConnected ||
status.taskCleanerRecoveryNeeded) {
return status;
}
final deadline = DateTime.now().add(timeout);
while (DateTime.now().isBefore(deadline)) {
await Future<void>.delayed(interval);
status = await recognitionStatus();
if (!status.accessibilityAuthorized ||
status.accessibilityConnected ||
status.taskCleanerRecoveryNeeded ||
!(status.accessibilityEvents || status.aiScreenshot)) {
return status;
}
}
return status;
}
static Future<bool> ensureRecognitionKeepAlive() async {
try {
return await _channel.invokeMethod<bool>('ensureRecognitionKeepAlive') ??
false;
} on MissingPluginException {
return false;
}
}
static Future<bool> clearRecognitionDiagnostic() async {
try {
return await _channel.invokeMethod<bool>('clearRecognitionDiagnostic') ??
false;
} on MissingPluginException {
return false;
}
}
static Future<bool> setRecognitionToggle(String key, bool enabled) async {
try {
return await _channel.invokeMethod<bool>('setRecognitionToggle', {
'key': key,
'enabled': enabled,
}) ??
false;
} on MissingPluginException {
return false;
}
}
static Future<void> configureRecognitionContext({
required bool hasAccount,
required bool aiAllowed,
required String baseUrl,
String? token,
}) async {
try {
await _channel.invokeMethod<bool>('configureRecognitionContext', {
'hasAccount': hasAccount,
'aiAllowed': aiAllowed,
'baseUrl': baseUrl,
'token': token,
});
} on MissingPluginException {
// Native background recognition is Android-only.
}
}
static Future<List<RecognitionCandidate>> drainRecognitionCandidates() async {
try {
final values =
await _channel.invokeMethod<List<Object?>>(
'drainRecognitionCandidates',
) ??
const [];
return values
.whereType<String>()
.map(
(value) => RecognitionCandidate.fromJson(
jsonDecode(value) as Map<String, dynamic>,
),
)
.toList();
} on MissingPluginException {
return const [];
}
}
static Future<List<RecognitionCandidate>> listRecognitionCandidates() async {
try {
final values =
await _channel.invokeMethod<List<Object?>>(
'listRecognitionCandidates',
) ??
const [];
return values
.whereType<String>()
.map(
(value) => RecognitionCandidate.fromJson(
jsonDecode(value) as Map<String, dynamic>,
),
)
.toList(growable: false);
} on MissingPluginException {
return const [];
}
}
static Future<bool> acknowledgeRecognitionCandidate(
String id,
String state, {
int? transactionId,
}) async {
try {
return await _channel.invokeMethod<bool>('ackRecognitionCandidate', {
'id': id,
'state': state,
if (transactionId != null) 'transactionId': transactionId,
}) ??
false;
} on MissingPluginException {
return false;
}
}
static Future<List<RecognitionBatch>> listRecognitionBatches() async {
try {
final values =
await _channel.invokeMethod<List<Object?>>(
'listRecognitionBatches',
) ??
const [];
return values
.whereType<String>()
.map(
(value) => RecognitionBatch.fromJson(
jsonDecode(value) as Map<String, dynamic>,
),
)
.toList(growable: false);
} on MissingPluginException {
return const [];
}
}
static Future<bool> restoreDroppedRecognition(String candidateId) async {
try {
return await _channel.invokeMethod<bool>('restoreDroppedRecognition', {
'candidateId': candidateId,
}) ??
false;
} on MissingPluginException {
return false;
}
}
static Future<bool> requestNotificationPermission() async {
try {
return await _channel.invokeMethod<bool>(
'requestNotificationPermission',
) ??
false;
} on MissingPluginException {
return true;
}
}
static Future<void> openNotificationAccessSettings() async {
try {
await _channel.invokeMethod('openNotificationAccessSettings');
} on MissingPluginException {
// Notification access is Android-only.
}
}
static Future<void> openBatteryOptimizationSettings() async {
try {
await _channel.invokeMethod('openBatteryOptimizationSettings');
} on MissingPluginException {
// Background battery settings are Android-only.
}
}
static Future<void> openBackgroundStartupSettings() async {
try {
await _channel.invokeMethod('openBackgroundStartupSettings');
} on MissingPluginException {
// Vendor background startup settings are Android-only.
}
}
static void onRecognitionAction(
void Function(Map<String, dynamic> action) callback,
) {
_recognitionAction = callback;
_installHandler();
}
static Future<void> onScreenshotReady(
void Function(String path) callback, {
void Function(String error)? onError,
}) async {
_screenshotReady = callback;
_screenshotError = onError;
_installHandler();
}
static void _installHandler() {
if (_handlerInstalled) return;
_handlerInstalled = true;
_channel.setMethodCallHandler((call) async {
if (call.method == 'onScreenshotReady') {
final path = call.arguments as String?;
if (path != null) _screenshotReady?.call(path);
} else if (call.method == 'onScreenshotError') {
_screenshotError?.call(call.arguments?.toString() ?? '截屏失败,请重试');
} else if (call.method == 'onSpeechEvent') {
final raw = Map<Object?, Object?>.from(call.arguments as Map);
final type = raw['type']?.toString() ?? 'error';
final event = SpeechEvent(
type: type,
text: raw['text']?.toString(),
rms: (raw['rms'] as num?)?.toDouble() ?? 0,
);
_speechEvent?.call(event);
if (type == 'final' || type == 'error' || type == 'cancelled') {
_speechEvent = null;
}
} else if (call.method == 'onRecognitionAction') {
final raw = Map<Object?, Object?>.from(call.arguments as Map);
_recognitionAction?.call(
raw.map((key, value) => MapEntry(key.toString(), value)),
);
}
});
}
}