From 48f4efaf3dc440433fccd9f698239b6a0a5c1fc0 Mon Sep 17 00:00:00 2001 From: nanxun Date: Wed, 12 Aug 2026 13:32:10 +0800 Subject: [PATCH] fix(audio): detect transcript-wide hallucinations --- backend/imagefind/api.py | 12 +++ backend/imagefind/audio_worker.py | 58 +++++++++-- backend/imagefind/speech.py | 11 +- backend/imagefind/speech_quality.py | 57 ++++++++++ frontend/src/App.tsx | 6 +- scripts/audio_small_ab.py | 109 ++++++++++++++++++++ scripts/audio_small_ab_windows.example.json | 12 +++ tests/test_api.py | 3 + tests/test_audio_index.py | 36 +++++++ tests/test_speech_quality.py | 27 +++++ 10 files changed, 318 insertions(+), 13 deletions(-) create mode 100644 scripts/audio_small_ab.py create mode 100644 scripts/audio_small_ab_windows.example.json diff --git a/backend/imagefind/api.py b/backend/imagefind/api.py index 5f8fe3f..726081a 100644 --- a/backend/imagefind/api.py +++ b/backend/imagefind/api.py @@ -2138,6 +2138,15 @@ def video_transcript( int(video["audio_rejected_segments"] or 0), quality_flags, ) + repeated_phrases = [] + for flag in quality_flags: + if not isinstance(flag, str) or not flag.startswith("repeated_phrase:"): + continue + try: + phrase, count = flag.removeprefix("repeated_phrase:").rsplit(":", 1) + repeated_phrases.append({"text": phrase, "count": int(count)}) + except (TypeError, ValueError): + continue if job and job["status"] in {"queued", "running"}: status = job["status"] elif current: @@ -2159,6 +2168,9 @@ def video_transcript( else "legacy", "rejected_segments": int(video["audio_rejected_segments"] or 0), "quality_flags": quality_flags, + "aggregate_risk": quality_state == "low_quality", + "repeated_phrases": repeated_phrases, + "quality_score_semantics": "rule_check", "items": [dict(row) for row in rows], "page": page, "page_size": page_size, diff --git a/backend/imagefind/audio_worker.py b/backend/imagefind/audio_worker.py index 40513f3..1944249 100644 --- a/backend/imagefind/audio_worker.py +++ b/backend/imagefind/audio_worker.py @@ -15,7 +15,14 @@ from pathlib import Path from .accelerator import AcceleratorService from .config import Settings from .speech import SpeechService, SpeechStageError -from .speech_quality import normalize_language, pcm16_voiced_regions, select_language, transcript_quality +from .speech_quality import ( + aggregate_transcript_quality, + normalize_language, + pcm16_speech_ratio, + pcm16_voiced_regions, + select_language, + transcript_quality, +) def _emit(payload: dict) -> None: @@ -39,6 +46,7 @@ def _arguments() -> argparse.Namespace: parser.add_argument("--language-policy", choices=("zh_priority", "auto", "zh"), default="zh_priority") parser.add_argument("--quality-profile", choices=("speed", "balanced", "accuracy"), default="accuracy") parser.add_argument("--model-variant", choices=("small", "medium"), default="small") + parser.add_argument("--segmentation", choices=("vad", "continuous"), default="vad") return parser.parse_args() @@ -240,6 +248,7 @@ def run(args: argparse.Namespace) -> int: strongest_voiced_frame = 0 rejected_candidate_units = 0 verification_duration_ms = 0 + accepted_all: list[dict] = [] with wave.open(str(wav_path), "rb") as handle: sample_rate = handle.getframerate() total_frames = handle.getnframes() @@ -273,12 +282,15 @@ def run(args: argparse.Namespace) -> int: upper_ms = end_ms - (args.overlap_seconds * 500 if end_frame < total_frames else 0) energy = _sample_energy(samples) segments = [] - regions = pcm16_voiced_regions(raw, sample_rate, mode=2) - if not regions and energy > 1e-6: - regions = pcm16_voiced_regions(raw, sample_rate, mode=1) + if args.segmentation == "continuous": + regions = [(0, len(raw) // 2, pcm16_speech_ratio(raw, sample_rate, mode=2))] + else: + regions = pcm16_voiced_regions(raw, sample_rate, mode=2) + if not regions and energy > 1e-6: + regions = pcm16_voiced_regions(raw, sample_rate, mode=1) windows: list[tuple[list[float], float, int, int]] = [] for region_start, region_end, speech_ratio in regions: - if speech_ratio < 0.08: + if speech_ratio < 0.08 and args.segmentation != "continuous": continue if speech_ratio > strongest_voiced_ratio: strongest_voiced_ratio = speech_ratio @@ -341,6 +353,29 @@ def run(args: argparse.Namespace) -> int: rejected_total += rejected quality_flags.extend(flags) quality_scores.append(candidate_score) + elapsed = time.monotonic() - inference_started + _emit( + { + "event": "window_result", + "chunk_index": index + 1, + "window_index": offset + 1, + "start_ms": window_base_ms, + "duration_ms": window_duration_ms, + "speech_ratio": round(speech_ratio, 4), + "language_candidates": detected, + "forced_language": resolved_language, + "raw_output": raw_segments, + "accepted_output": scored, + "quality_score": candidate_score, + "quality_flags": flags, + "rejected_segments": rejected, + "device": args.device, + "beam": 1 if args.device == "GPU" else speech.generation_beams(quality_profile), + "elapsed_seconds": round(elapsed, 3), + "rtf": round(elapsed / max(0.001, window_duration_ms / 1000), 3), + "segmentation": args.segmentation, + } + ) for segment in scored: segment["start_ms"] += window_base_ms segment["end_ms"] += window_base_ms @@ -349,6 +384,7 @@ def run(args: argparse.Namespace) -> int: midpoint = (segment["start_ms"] + segment["end_ms"]) // 2 if lower_ms <= midpoint <= upper_ms: segments.append(segment) + accepted_all.append(dict(segment)) recognized_segments += len(segments) if not segments: if energy > strongest_empty_energy: @@ -397,20 +433,26 @@ def run(args: argparse.Namespace) -> int: "segmentation": "webrtcvad", }, ) + aggregate = aggregate_transcript_quality( + accepted_all, + base_score=round(sum(quality_scores) / len(quality_scores), 3) if quality_scores else None, + base_flags=quality_flags, + ) _emit( { "event": "complete", "detected_language": resolved_language, - "quality_score": round(sum(quality_scores) / len(quality_scores), 3) if quality_scores else None, + "quality_score": aggregate.score, "rejected_segments": rejected_total, - "quality_flags": list(dict.fromkeys(quality_flags)), + "quality_flags": list(aggregate.flags), "inference_diagnostics": { "requested_device": args.device, "actual_device": component.get("actual_device") or component.get("device") or args.device, "primary_backend": "direct_generate", "model_variant": model_variant, "num_beams": 1 if args.device == "GPU" else speech.generation_beams(quality_profile), - "segmentation": "webrtcvad", + "segmentation": args.segmentation, + "quality_score_semantics": "rule_check", "language_candidates": detected, "selected_language": resolved_language, "fallback_scope": None, diff --git a/backend/imagefind/speech.py b/backend/imagefind/speech.py index a8501cc..91ecbfa 100644 --- a/backend/imagefind/speech.py +++ b/backend/imagefind/speech.py @@ -25,7 +25,7 @@ from .database import Database, utcnow from .jobs import JobCancelled, JobQueue, JobRetry from .media import MediaInput, MediaService from .runtime import RuntimeToolManager -from .speech_quality import normalize_language, transcript_quality_state +from .speech_quality import aggregate_transcript_quality, normalize_language, transcript_quality_state from .text import search_tokens logger = logging.getLogger(__name__) @@ -1550,6 +1550,11 @@ class AudioIndexer: quality_score = getattr(segments, "quality_score", None) rejected_segments = int(getattr(segments, "rejected_segments", 0) or 0) quality_flags = list(getattr(segments, "quality_flags", []) or []) + aggregate_quality = aggregate_transcript_quality( + segments, base_score=quality_score, base_flags=quality_flags + ) + quality_score = aggregate_quality.score + quality_flags = list(aggregate_quality.flags) quality_state = transcript_quality_state( quality_score, len(segments), @@ -1581,7 +1586,9 @@ class AudioIndexer: utcnow(), ), ) - if tokens: + # Risky transcripts remain visible for diagnosis and playback, + # but repeated hallucinations must not pollute global search. + if tokens and quality_state != "low_quality": conn.execute("INSERT INTO text_fts(entry_id,tokens) VALUES(?,?)", (entry_id, tokens)) conn.execute( "UPDATE videos SET audio_model_version=?,audio_index_revision=?,audio_detected_language=?," diff --git a/backend/imagefind/speech_quality.py b/backend/imagefind/speech_quality.py index 9848430..f0706a5 100644 --- a/backend/imagefind/speech_quality.py +++ b/backend/imagefind/speech_quality.py @@ -43,6 +43,60 @@ class TranscriptQuality: units: int +@dataclass(frozen=True, slots=True) +class AggregateTranscriptQuality: + score: float + flags: tuple[str, ...] + repeated_phrases: tuple[tuple[str, int], ...] + short_segment_ratio: float + + +_ENDING_HALLUCINATIONS = { + "拜拜", "再见", "谢谢观看", "谢谢大家观看", "谢谢大家收看", + "感谢观看", "下期再见", "謝謝觀看", "謝謝大家觀看", "謝謝大家收看", + "感謝觀看", "下期再見", "我去看看", "thanksforwatching", +} + + +def _phrase_key(text: object) -> str: + value = unicodedata.normalize("NFKC", str(text or "")).casefold() + return "".join(character for character in value if character.isalnum()) + + +def aggregate_transcript_quality( + segments: list[dict] | tuple[dict, ...], + *, + base_score: float | None = None, + base_flags: list[str] | tuple[str, ...] = (), +) -> AggregateTranscriptQuality: + """Detect risks that only become visible across a whole transcript.""" + texts = [_phrase_key(item.get("text")) for item in segments] + texts = [text for text in texts if text] + flags = list(dict.fromkeys(str(flag) for flag in base_flags if flag)) + if not texts: + return AggregateTranscriptQuality(round(float(base_score or 0.0), 3), tuple(flags), (), 0.0) + counts = Counter(texts) + repeated = tuple(sorted( + ((text, count) for text, count in counts.items() if count >= 3), + key=lambda item: (-item[1], item[0]), + )[:12]) + short_ratio = sum(len(text) < 4 for text in texts) / len(texts) + repeated_units = sum(count for _text, count in repeated) + penalty = 0.0 + if repeated and (repeated_units >= 6 or repeated_units / len(texts) >= 0.20): + flags.append("whole_repeated_phrase") + penalty += 0.35 + if short_ratio >= 0.65 and len(texts) >= 10: + flags.append("whole_short_segment_dominance") + penalty += 0.35 + if any(text in _ENDING_HALLUCINATIONS for text, _count in repeated): + flags.append("whole_ending_hallucination") + penalty += 0.45 + flags.extend(f"repeated_phrase:{text}:{count}" for text, count in repeated) + score = max(0.0, min(1.0, float(1.0 if base_score is None else base_score) - penalty)) + return AggregateTranscriptQuality(round(score, 3), tuple(dict.fromkeys(flags)), repeated, round(short_ratio, 3)) + + def normalize_language(value: object) -> str | None: language = str(value or "").strip().lower() if language.startswith("<|") and language.endswith("|>"): @@ -98,6 +152,9 @@ def transcript_quality_state( "no_speech_hallucination", "language_script_conflict", "untrusted_language", + "whole_repeated_phrase", + "whole_short_segment_dominance", + "whole_ending_hallucination", } if high_risk.intersection(flags or ()): return "low_quality" diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 1a9937e..07494ee 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -40,7 +40,7 @@ type Result = { }; type SearchCoverage = {available:boolean;model_version?:string;total:number;indexed:number;searchable:number;empty:number;queued:number;running:number;failed:number;low_quality?:number;pending:number;percent:number;current_model_percent:number}; type TranscriptItem = {id:string;start_ms:number;end_ms:number;raw_text:string;language?:string;quality_score?:number}; -type TranscriptPage = {status:"ready"|"filtered"|"low_quality"|"empty"|"queued"|"running"|"failed"|"unavailable"|"pending";model_version?:string;indexed_version?:string;detected_language?:string;quality_score?:number;quality_state:"ready"|"filtered"|"low_quality"|"empty"|"legacy";rejected_segments:number;quality_flags:string[];items:TranscriptItem[];page:number;page_size:number;total:number;pages:number;job?:Pick}; +type TranscriptPage = {status:"ready"|"filtered"|"low_quality"|"empty"|"queued"|"running"|"failed"|"unavailable"|"pending";model_version?:string;indexed_version?:string;detected_language?:string;quality_score?:number;quality_state:"ready"|"filtered"|"low_quality"|"empty"|"legacy";rejected_segments:number;quality_flags:string[];aggregate_risk?:boolean;repeated_phrases?:{text:string;count:number}[];quality_score_semantics?:"rule_check";items:TranscriptItem[];page:number;page_size:number;total:number;pages:number;job?:Pick}; type SpeechConfig = {language_policy:"zh_priority"|"auto"|"zh";quality_profile:"speed"|"balanced"|"accuracy";model_variant:"small"|"medium";index_revision:number}; type RecognitionType = "visual"|"ocr"|"person"|"audio"|"subtitle"|"metadata"; type UploadTask = { @@ -628,7 +628,7 @@ function PlayerPage({ video, startMs, videos, collections, onBack, onPlay, onEdi
- {transcriptOpen&&
{transcriptLoading?:<>{transcript&&
{transcript.detected_language?`识别语言：${({zh:"中文",en:"英语",ja:"日语",ko:"韩语"} as Record)[transcript.detected_language]||transcript.detected_language}`:"识别语言：自动"}{transcript.quality_state==="low_quality"?`整体质量偏低，已保留可信片段并仅自动增强一次`:transcript.quality_state==="filtered"?`已过滤 ${transcript.rejected_segments} 个不可信片段`:transcript.quality_state==="empty"?"有界语音检测未发现可识别内容":transcript.quality_state==="legacy"?"旧版转写，建议重新识别":"质量检查已通过"}{typeof transcript.quality_score==="number"?` · 可信度 ${Math.round(transcript.quality_score*100)}%`:""}{["filtered","low_quality","legacy"].includes(transcript.quality_state)?:}
}{transcript?.items.length?<>
{transcript.items.map(item=>)}
{transcript.pages>1&&
{transcript.page} / {transcript.pages}
}:

{transcript?.job?.error||transcript?.job?.message||"当前视频还没有可显示的音频转写。"}

}}{transcriptError&&
{transcriptError}
}
} + {transcriptOpen&&
{transcriptLoading?:<>{transcript&&
{transcript.detected_language?`识别语言：${({zh:"中文",en:"英语",ja:"日语",ko:"韩语"} as Record)[transcript.detected_language]||transcript.detected_language}`:"识别语言：自动"}{transcript.quality_state==="low_quality"?`检测到整片重复或低质量风险，结果仅供诊断且不进入搜索`:transcript.quality_state==="filtered"?`已过滤 ${transcript.rejected_segments} 个不可信片段`:transcript.quality_state==="empty"?"有界语音检测未发现可识别内容":transcript.quality_state==="legacy"?"旧版转写，建议重新识别":"规则检查未发现明显异常"}{typeof transcript.quality_score==="number"?` · 规则检查分 ${Math.round(transcript.quality_score*100)}%`:""}{transcript.repeated_phrases?.length?<>{" · 重复："+transcript.repeated_phrases.slice(0,3).map(item=>item.text+"×"+item.count).join("、")}:null}{["filtered","low_quality","legacy"].includes(transcript.quality_state)?:}
}{transcript?.items.length?<>
{transcript.items.map(item=>)}
{transcript.pages>1&&
{transcript.page} / {transcript.pages}
}:

{transcript?.job?.error||transcript?.job?.message||"当前视频还没有可显示的音频转写。"}

}}{transcriptError&&
{transcriptError}
}
}