fix(audio): detect transcript-wide hallucinations

backend/imagefind/api.py

@@ -2138,6 +2138,15 @@ def video_transcript(
         int(video["audio_rejected_segments"] or 0),
         quality_flags,
     )
+    repeated_phrases = []
+    for flag in quality_flags:
+        if not isinstance(flag, str) or not flag.startswith("repeated_phrase:"):
+            continue
+        try:
+            phrase, count = flag.removeprefix("repeated_phrase:").rsplit(":", 1)
+            repeated_phrases.append({"text": phrase, "count": int(count)})
+        except (TypeError, ValueError):
+            continue
     if job and job["status"] in {"queued", "running"}:
         status = job["status"]
     elif current:
@@ -2159,6 +2168,9 @@ def video_transcript(
         else "legacy",
         "rejected_segments": int(video["audio_rejected_segments"] or 0),
         "quality_flags": quality_flags,
+        "aggregate_risk": quality_state == "low_quality",
+        "repeated_phrases": repeated_phrases,
+        "quality_score_semantics": "rule_check",
         "items": [dict(row) for row in rows],
         "page": page,
         "page_size": page_size,

backend/imagefind/audio_worker.py

@@ -15,7 +15,14 @@ from pathlib import Path
 from .accelerator import AcceleratorService
 from .config import Settings
 from .speech import SpeechService, SpeechStageError
-from .speech_quality import normalize_language, pcm16_voiced_regions, select_language, transcript_quality
+from .speech_quality import (
+    aggregate_transcript_quality,
+    normalize_language,
+    pcm16_speech_ratio,
+    pcm16_voiced_regions,
+    select_language,
+    transcript_quality,
+)
 
 
 def _emit(payload: dict) -> None:
@@ -39,6 +46,7 @@ def _arguments() -> argparse.Namespace:
     parser.add_argument("--language-policy", choices=("zh_priority", "auto", "zh"), default="zh_priority")
     parser.add_argument("--quality-profile", choices=("speed", "balanced", "accuracy"), default="accuracy")
     parser.add_argument("--model-variant", choices=("small", "medium"), default="small")
+    parser.add_argument("--segmentation", choices=("vad", "continuous"), default="vad")
     return parser.parse_args()
 
 
@@ -240,6 +248,7 @@ def run(args: argparse.Namespace) -> int:
     strongest_voiced_frame = 0
     rejected_candidate_units = 0
     verification_duration_ms = 0
+    accepted_all: list[dict] = []
     with wave.open(str(wav_path), "rb") as handle:
         sample_rate = handle.getframerate()
         total_frames = handle.getnframes()
@@ -273,12 +282,15 @@ def run(args: argparse.Namespace) -> int:
             upper_ms = end_ms - (args.overlap_seconds * 500 if end_frame < total_frames else 0)
             energy = _sample_energy(samples)
             segments = []
-            regions = pcm16_voiced_regions(raw, sample_rate, mode=2)
+            if args.segmentation == "continuous":
-            if not regions and energy > 1e-6:
+                regions = [(0, len(raw) // 2, pcm16_speech_ratio(raw, sample_rate, mode=2))]
-                regions = pcm16_voiced_regions(raw, sample_rate, mode=1)
+            else:
+                regions = pcm16_voiced_regions(raw, sample_rate, mode=2)
+                if not regions and energy > 1e-6:
+                    regions = pcm16_voiced_regions(raw, sample_rate, mode=1)
             windows: list[tuple[list[float], float, int, int]] = []
             for region_start, region_end, speech_ratio in regions:
-                if speech_ratio < 0.08:
+                if speech_ratio < 0.08 and args.segmentation != "continuous":
                     continue
                 if speech_ratio > strongest_voiced_ratio:
                     strongest_voiced_ratio = speech_ratio
@@ -341,6 +353,29 @@ def run(args: argparse.Namespace) -> int:
                     rejected_total += rejected
                     quality_flags.extend(flags)
                     quality_scores.append(candidate_score)
+                    elapsed = time.monotonic() - inference_started
+                    _emit(
+                        {
+                            "event": "window_result",
+                            "chunk_index": index + 1,
+                            "window_index": offset + 1,
+                            "start_ms": window_base_ms,
+                            "duration_ms": window_duration_ms,
+                            "speech_ratio": round(speech_ratio, 4),
+                            "language_candidates": detected,
+                            "forced_language": resolved_language,
+                            "raw_output": raw_segments,
+                            "accepted_output": scored,
+                            "quality_score": candidate_score,
+                            "quality_flags": flags,
+                            "rejected_segments": rejected,
+                            "device": args.device,
+                            "beam": 1 if args.device == "GPU" else speech.generation_beams(quality_profile),
+                            "elapsed_seconds": round(elapsed, 3),
+                            "rtf": round(elapsed / max(0.001, window_duration_ms / 1000), 3),
+                            "segmentation": args.segmentation,
+                        }
+                    )
                     for segment in scored:
                         segment["start_ms"] += window_base_ms
                         segment["end_ms"] += window_base_ms
@@ -349,6 +384,7 @@ def run(args: argparse.Namespace) -> int:
                         midpoint = (segment["start_ms"] + segment["end_ms"]) // 2
                         if lower_ms <= midpoint <= upper_ms:
                             segments.append(segment)
+                            accepted_all.append(dict(segment))
             recognized_segments += len(segments)
             if not segments:
                 if energy > strongest_empty_energy:
@@ -397,20 +433,26 @@ def run(args: argparse.Namespace) -> int:
                 "segmentation": "webrtcvad",
             },
         )
+    aggregate = aggregate_transcript_quality(
+        accepted_all,
+        base_score=round(sum(quality_scores) / len(quality_scores), 3) if quality_scores else None,
+        base_flags=quality_flags,
+    )
     _emit(
         {
             "event": "complete",
             "detected_language": resolved_language,
-            "quality_score": round(sum(quality_scores) / len(quality_scores), 3) if quality_scores else None,
+            "quality_score": aggregate.score,
             "rejected_segments": rejected_total,
-            "quality_flags": list(dict.fromkeys(quality_flags)),
+            "quality_flags": list(aggregate.flags),
             "inference_diagnostics": {
                 "requested_device": args.device,
                 "actual_device": component.get("actual_device") or component.get("device") or args.device,
                 "primary_backend": "direct_generate",
                 "model_variant": model_variant,
                 "num_beams": 1 if args.device == "GPU" else speech.generation_beams(quality_profile),
-                "segmentation": "webrtcvad",
+                "segmentation": args.segmentation,
+                "quality_score_semantics": "rule_check",
                 "language_candidates": detected,
                 "selected_language": resolved_language,
                 "fallback_scope": None,

backend/imagefind/speech.py

@@ -25,7 +25,7 @@ from .database import Database, utcnow
 from .jobs import JobCancelled, JobQueue, JobRetry
 from .media import MediaInput, MediaService
 from .runtime import RuntimeToolManager
-from .speech_quality import normalize_language, transcript_quality_state
+from .speech_quality import aggregate_transcript_quality, normalize_language, transcript_quality_state
 from .text import search_tokens
 
 logger = logging.getLogger(__name__)
@@ -1550,6 +1550,11 @@ class AudioIndexer:
         quality_score = getattr(segments, "quality_score", None)
         rejected_segments = int(getattr(segments, "rejected_segments", 0) or 0)
         quality_flags = list(getattr(segments, "quality_flags", []) or [])
+        aggregate_quality = aggregate_transcript_quality(
+            segments, base_score=quality_score, base_flags=quality_flags
+        )
+        quality_score = aggregate_quality.score
+        quality_flags = list(aggregate_quality.flags)
         quality_state = transcript_quality_state(
             quality_score,
             len(segments),
@@ -1581,7 +1586,9 @@ class AudioIndexer:
                         utcnow(),
                     ),
                 )
-                if tokens:
+                # Risky transcripts remain visible for diagnosis and playback,
+                # but repeated hallucinations must not pollute global search.
+                if tokens and quality_state != "low_quality":
                     conn.execute("INSERT INTO text_fts(entry_id,tokens) VALUES(?,?)", (entry_id, tokens))
             conn.execute(
                 "UPDATE videos SET audio_model_version=?,audio_index_revision=?,audio_detected_language=?,"

backend/imagefind/speech_quality.py

@@ -43,6 +43,60 @@ class TranscriptQuality:
     units: int
 
 
+@dataclass(frozen=True, slots=True)
+class AggregateTranscriptQuality:
+    score: float
+    flags: tuple[str, ...]
+    repeated_phrases: tuple[tuple[str, int], ...]
+    short_segment_ratio: float
+
+
+_ENDING_HALLUCINATIONS = {
+    "拜拜", "再见", "谢谢观看", "谢谢大家观看", "谢谢大家收看",
+    "感谢观看", "下期再见", "謝謝觀看", "謝謝大家觀看", "謝謝大家收看",
+    "感謝觀看", "下期再見", "我去看看", "thanksforwatching",
+}
+
+
+def _phrase_key(text: object) -> str:
+    value = unicodedata.normalize("NFKC", str(text or "")).casefold()
+    return "".join(character for character in value if character.isalnum())
+
+
+def aggregate_transcript_quality(
+    segments: list[dict] | tuple[dict, ...],
+    *,
+    base_score: float | None = None,
+    base_flags: list[str] | tuple[str, ...] = (),
+) -> AggregateTranscriptQuality:
+    """Detect risks that only become visible across a whole transcript."""
+    texts = [_phrase_key(item.get("text")) for item in segments]
+    texts = [text for text in texts if text]
+    flags = list(dict.fromkeys(str(flag) for flag in base_flags if flag))
+    if not texts:
+        return AggregateTranscriptQuality(round(float(base_score or 0.0), 3), tuple(flags), (), 0.0)
+    counts = Counter(texts)
+    repeated = tuple(sorted(
+        ((text, count) for text, count in counts.items() if count >= 3),
+        key=lambda item: (-item[1], item[0]),
+    )[:12])
+    short_ratio = sum(len(text) < 4 for text in texts) / len(texts)
+    repeated_units = sum(count for _text, count in repeated)
+    penalty = 0.0
+    if repeated and (repeated_units >= 6 or repeated_units / len(texts) >= 0.20):
+        flags.append("whole_repeated_phrase")
+        penalty += 0.35
+    if short_ratio >= 0.65 and len(texts) >= 10:
+        flags.append("whole_short_segment_dominance")
+        penalty += 0.35
+    if any(text in _ENDING_HALLUCINATIONS for text, _count in repeated):
+        flags.append("whole_ending_hallucination")
+        penalty += 0.45
+    flags.extend(f"repeated_phrase:{text}:{count}" for text, count in repeated)
+    score = max(0.0, min(1.0, float(1.0 if base_score is None else base_score) - penalty))
+    return AggregateTranscriptQuality(round(score, 3), tuple(dict.fromkeys(flags)), repeated, round(short_ratio, 3))
+
+
 def normalize_language(value: object) -> str | None:
     language = str(value or "").strip().lower()
     if language.startswith("<|") and language.endswith("|>"):
@@ -98,6 +152,9 @@ def transcript_quality_state(
         "no_speech_hallucination",
         "language_script_conflict",
         "untrusted_language",
+        "whole_repeated_phrase",
+        "whole_short_segment_dominance",
+        "whole_ending_hallucination",
     }
     if high_risk.intersection(flags or ()):
         return "low_quality"

frontend/src/App.tsx

@@ -40,7 +40,7 @@ type Result = {
 };
 type SearchCoverage = {available:boolean;model_version?:string;total:number;indexed:number;searchable:number;empty:number;queued:number;running:number;failed:number;low_quality?:number;pending:number;percent:number;current_model_percent:number};
 type TranscriptItem = {id:string;start_ms:number;end_ms:number;raw_text:string;language?:string;quality_score?:number};
-type TranscriptPage = {status:"ready"|"filtered"|"low_quality"|"empty"|"queued"|"running"|"failed"|"unavailable"|"pending";model_version?:string;indexed_version?:string;detected_language?:string;quality_score?:number;quality_state:"ready"|"filtered"|"low_quality"|"empty"|"legacy";rejected_segments:number;quality_flags:string[];items:TranscriptItem[];page:number;page_size:number;total:number;pages:number;job?:Pick<Job,"id"|"status"|"progress"|"message"|"error">};
+type TranscriptPage = {status:"ready"|"filtered"|"low_quality"|"empty"|"queued"|"running"|"failed"|"unavailable"|"pending";model_version?:string;indexed_version?:string;detected_language?:string;quality_score?:number;quality_state:"ready"|"filtered"|"low_quality"|"empty"|"legacy";rejected_segments:number;quality_flags:string[];aggregate_risk?:boolean;repeated_phrases?:{text:string;count:number}[];quality_score_semantics?:"rule_check";items:TranscriptItem[];page:number;page_size:number;total:number;pages:number;job?:Pick<Job,"id"|"status"|"progress"|"message"|"error">};
 type SpeechConfig = {language_policy:"zh_priority"|"auto"|"zh";quality_profile:"speed"|"balanced"|"accuracy";model_variant:"small"|"medium";index_revision:number};
 type RecognitionType = "visual"|"ocr"|"person"|"audio"|"subtitle"|"metadata";
 type UploadTask = {
@@ -628,7 +628,7 @@ function PlayerPage({ video, startMs, videos, collections, onBack, onPlay, onEdi
       </section>
       <section className={`transcript-panel ${transcriptOpen?"open":""}`}>
         <button className="transcript-toggle" aria-expanded={transcriptOpen} onClick={()=>setTranscriptOpen(value=>!value)}><span><Volume2/><span><strong>音频转写</strong><small>{transcriptLoading?"正在读取识别状态…":transcript?.status==="ready"?`${transcript.total} 个高可信时间片段`:transcript?.status==="filtered"?`${transcript.total} 个可搜索片段，部分内容已过滤`:transcript?.status==="low_quality"?"识别质量偏低，系统将进行一次增强修复":transcript?.status==="empty"?"已识别，未检测到语音":transcript?.status==="running"?`正在识别 ${Math.round((transcript.job?.progress||0)*100)}%`:transcript?.status==="queued"?"等待后台识别":transcript?.status==="failed"?"识别失败":transcript?.status==="unavailable"?"音频模型未安装":"尚未识别"}</small></span></span>{transcriptLoading?<RefreshCw className="spin"/>:transcriptOpen?<ArrowUp/>:<ArrowDown/>}</button>
-        {transcriptOpen&&<div className="transcript-body">{transcriptLoading?<AsyncNotice compact label="正在读取音频转写…"/>:<>{transcript&&<div className={`transcript-quality ${transcript.quality_state}`} role="status"><span><strong>{transcript.detected_language?`识别语言：${({zh:"中文",en:"英语",ja:"日语",ko:"韩语"} as Record<string,string>)[transcript.detected_language]||transcript.detected_language}`:"识别语言：自动"}</strong><small>{transcript.quality_state==="low_quality"?`整体质量偏低，已保留可信片段并仅自动增强一次`:transcript.quality_state==="filtered"?`已过滤 ${transcript.rejected_segments} 个不可信片段`:transcript.quality_state==="empty"?"有界语音检测未发现可识别内容":transcript.quality_state==="legacy"?"旧版转写，建议重新识别":"质量检查已通过"}{typeof transcript.quality_score==="number"?` · 可信度 ${Math.round(transcript.quality_score*100)}%`:""}</small></span>{["filtered","low_quality","legacy"].includes(transcript.quality_state)?<AlertCircle/>:<ShieldCheck/>}</div>}{transcript?.items.length?<><div className="transcript-list">{transcript.items.map(item=><button key={item.id} onClick={()=>{seekTo(item.start_ms/1000);void element.current?.play().catch(()=>{})}}><time>{formatTime(item.start_ms)}</time><span>{item.raw_text}</span><Play/></button>)}</div>{transcript.pages>1&&<div className="transcript-pages"><button disabled={transcript.page<=1} onClick={()=>void loadTranscript(transcript.page-1)}><ChevronLeft/>上一页</button><span>{transcript.page} / {transcript.pages}</span><button disabled={transcript.page>=transcript.pages} onClick={()=>void loadTranscript(transcript.page+1)}>下一页<ChevronRight/></button></div>}</>:<div className="transcript-empty"><VolumeX/><p>{transcript?.job?.error||transcript?.job?.message||"当前视频还没有可显示的音频转写。"}</p></div>}</>}{transcriptError&&<div className="error"><AlertCircle/>{transcriptError}</div>}<button className="secondary transcript-reindex" disabled={transcriptBusy||transcript?.status==="running"||transcript?.status==="queued"} onClick={()=>setTranscriptLanguageOpen(true)}>{transcriptBusy?<RefreshCw className="spin"/>:<RefreshCw/>}{transcript?.status==="failed"?"重试识别":"重新识别"}</button></div>}
+        {transcriptOpen&&<div className="transcript-body">{transcriptLoading?<AsyncNotice compact label="正在读取音频转写…"/>:<>{transcript&&<div className={`transcript-quality ${transcript.quality_state}`} role="status"><span><strong>{transcript.detected_language?`识别语言：${({zh:"中文",en:"英语",ja:"日语",ko:"韩语"} as Record<string,string>)[transcript.detected_language]||transcript.detected_language}`:"识别语言：自动"}</strong><small>{transcript.quality_state==="low_quality"?`检测到整片重复或低质量风险，结果仅供诊断且不进入搜索`:transcript.quality_state==="filtered"?`已过滤 ${transcript.rejected_segments} 个不可信片段`:transcript.quality_state==="empty"?"有界语音检测未发现可识别内容":transcript.quality_state==="legacy"?"旧版转写，建议重新识别":"规则检查未发现明显异常"}{typeof transcript.quality_score==="number"?` · 规则检查分 ${Math.round(transcript.quality_score*100)}%`:""}{transcript.repeated_phrases?.length?<>{" · 重复："+transcript.repeated_phrases.slice(0,3).map(item=>item.text+"×"+item.count).join("、")}</>:null}</small></span>{["filtered","low_quality","legacy"].includes(transcript.quality_state)?<AlertCircle/>:<ShieldCheck/>}</div>}{transcript?.items.length?<><div className="transcript-list">{transcript.items.map(item=><button key={item.id} onClick={()=>{seekTo(item.start_ms/1000);void element.current?.play().catch(()=>{})}}><time>{formatTime(item.start_ms)}</time><span>{item.raw_text}</span><Play/></button>)}</div>{transcript.pages>1&&<div className="transcript-pages"><button disabled={transcript.page<=1} onClick={()=>void loadTranscript(transcript.page-1)}><ChevronLeft/>上一页</button><span>{transcript.page} / {transcript.pages}</span><button disabled={transcript.page>=transcript.pages} onClick={()=>void loadTranscript(transcript.page+1)}>下一页<ChevronRight/></button></div>}</>:<div className="transcript-empty"><VolumeX/><p>{transcript?.job?.error||transcript?.job?.message||"当前视频还没有可显示的音频转写。"}</p></div>}</>}{transcriptError&&<div className="error"><AlertCircle/>{transcriptError}</div>}<button className="secondary transcript-reindex" disabled={transcriptBusy||transcript?.status==="running"||transcript?.status==="queued"} onClick={()=>setTranscriptLanguageOpen(true)}>{transcriptBusy?<RefreshCw className="spin"/>:<RefreshCw/>}{transcript?.status==="failed"?"重试识别":"重新识别"}</button></div>}
       </section>
     </div>
     <aside className={`related player-side-panel ${collectionDetail?"player-collection-directory":""}`}>
@@ -1359,7 +1359,7 @@ function SettingsPage({ preferences, sources, onPreferences }: {preferences:Pref
         {speechLoad.phase==="loading"||speechLoad.phase==="idle"?<AsyncNotice compact label="正在读取语音识别策略…"/>:speechLoad.phase==="error"&&!speechConfig?<div className="error"><AlertCircle/>{speechLoad.error}<button onClick={()=>void loadSpeech()}>重试</button></div>:speechConfig&&<>
           <fieldset><legend>识别模型</legend><div className="audio-model-variants">{(["small","medium"] as const).map(variant=>{const info=models.audio_variants?.[variant]||{};const state=models.installations?.[`audio:${variant}`] as ModelInstallation|undefined;const active=state?.status==="queued"||state?.status==="running";const progress=Math.round(Math.max(0,Math.min(1,state?.progress||0))*100);const selected=speechConfig.model_variant===variant;return <article className={`${selected?"selected":""} ${info.installed?"installed":"missing"}`} key={variant}><button type="button" className="audio-model-choice" disabled={!info.installed||active||(variant==="medium"&&!gpuId)} aria-pressed={selected} onClick={()=>setSpeechConfig({...speechConfig,model_variant:variant})}><span>{selected?<CheckCircle2/>:<Boxes/>}</span><div><strong>Whisper {variant==="medium"?"Medium INT8":"Small"}</strong><small>{variant==="medium"?"高质量 · 仅 Intel GPU · 约 0.78 GB":"标准模型 · 约 0.49 GB · 纯 CPU 设备可用"}</small></div></button>{active?<div className="audio-model-progress" role="status"><span>{state?.message||"正在下载安装"}</span><b>{state?.status==="queued"?"等待":`${progress}%`}</b><i><span style={{width:`${state?.status==="queued"?4:progress}%`}}/></i></div>:info.needs_update?<button className="secondary audio-model-install" disabled={modelBusy||Boolean(modelJob)||(variant==="medium"&&!gpuId)} onClick={()=>void installOnline("audio",variant)}><RefreshCw/>更新为 INT8 · {formatSize(info.size_bytes||0)}</button>:info.installed?<div className="audio-model-actions"><span>{selected?"当前选择":formatSize(info.size_bytes||0)}</span>{!selected&&<button className="danger-text" disabled={modelBusy} onClick={()=>void removeAudioVariant(variant)}><Trash2/>删除</button>}</div>:<button className="secondary audio-model-install" disabled={modelBusy||Boolean(modelJob)||(variant==="medium"&&!gpuId)} onClick={()=>void installOnline("audio",variant)}><Download/>{variant==="medium"&&!gpuId?"需要 Intel GPU":"下载安装"}</button>}</article>})}</div></fieldset>
           <fieldset><legend>语言检测</legend><div className="speech-policy-options">{([['zh_priority','中文优先','音轨标签只作提示；弱检测或冲突时回到中文'],['auto','智能检测','由多个有声片段多数决，音轨标签仅用于平票'],['zh','固定中文','所有任务均明确按中文解码']] as const).map(([key,label,description])=><button type="button" className={speechConfig.language_policy===key?"active":""} aria-pressed={speechConfig.language_policy===key} key={key} onClick={()=>setSpeechConfig({...speechConfig,language_policy:key})}><strong>{label}</strong><small>{description}</small>{speechConfig.language_policy===key&&<CheckCircle2/>}</button>)}</div></fieldset>
-          <fieldset><legend>质量档位</legend><div className="speech-policy-options quality">{([['speed','速度优先','单 beam 解码，适合快速建立索引'],['balanced','均衡','提高解码候选数，兼顾速度与准确率'],['accuracy','准确率优先','使用更多 beam，处理时间更长']] as const).map(([key,label,description])=><button type="button" className={speechConfig.quality_profile===key?"active":""} aria-pressed={speechConfig.quality_profile===key} key={key} onClick={()=>setSpeechConfig({...speechConfig,quality_profile:key})}><strong>{label}</strong><small>{description}</small>{speechConfig.quality_profile===key&&<CheckCircle2/>}</button>)}</div></fieldset>
+          <fieldset><legend>质量档位</legend><div className="speech-policy-options quality">{([['speed','速度优先','单 beam 解码，适合快速建立索引'],['balanced','均衡','CPU 会增加候选；Intel GPU 仍为单 beam'],['accuracy','准确率优先','CPU 使用 5 beam；Intel GPU 受限为单 beam']] as const).map(([key,label,description])=><button type="button" className={speechConfig.quality_profile===key?"active":""} aria-pressed={speechConfig.quality_profile===key} key={key} onClick={()=>setSpeechConfig({...speechConfig,quality_profile:key})}><strong>{label}</strong><small>{description}</small>{speechConfig.quality_profile===key&&<CheckCircle2/>}</button>)}</div></fieldset>
           <p className="speech-resource-note"><ShieldCheck/>Medium 在 GPU、内存或推理验证失败时会停止任务，不会回退 CPU；旧转写会保留到新结果成功写入。</p>
           {speechLoad.error&&<div className="error"><AlertCircle/>{speechLoad.error}<button onClick={()=>void loadSpeech()}>重新读取</button></div>}
           <div className="speech-policy-actions"><button className="primary" disabled={speechBusy} onClick={()=>void saveSpeechConfig()}>{speechBusy?<RefreshCw className="spin"/>:<CheckCircle2/>}保存识别策略</button><button className="secondary" disabled={speechBusy} onClick={()=>void reconcileSpeech()}><ShieldCheck/>重建旧版音频索引</button></div>{speechMessage&&<p className="status-line"><Activity/>{speechMessage}</p>}

scripts/audio_small_ab.py

@@ -0,0 +1,109 @@
+from __future__ import annotations
+
+import argparse
+import json
+import subprocess
+import sys
+import tempfile
+import time
+import wave
+from pathlib import Path
+
+
+COMBINATIONS = (
+    ("small_vad_gpu_1beam", "small", "GPU", "vad"),
+    ("small_context_gpu_1beam", "small", "GPU", "continuous"),
+    ("small_context_cpu_5beam", "small", "CPU", "continuous"),
+    ("medium_current_baseline", "medium", "GPU", "vad"),
+)
+
+
+def arguments() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Read-only Small/Medium audio A/B diagnostics")
+    parser.add_argument("--data-dir", required=True, type=Path)
+    parser.add_argument("--media", required=True, type=Path, action="append")
+    parser.add_argument("--windows", required=True, type=Path)
+    parser.add_argument("--output", required=True, type=Path)
+    parser.add_argument("--ffmpeg", default="ffmpeg")
+    parser.add_argument("--force", action="store_true")
+    return parser.parse_args()
+
+
+def worker_run(data_dir: Path, wav: Path, window: dict, combination: tuple[str, ...]) -> dict:
+    name, model, device, segmentation = combination
+    with wave.open(str(wav), "rb") as handle:
+        sample_rate = handle.getframerate()
+    start_ms = int(window["start_ms"])
+    duration_ms = int(window["end_ms"]) - start_ms
+    command = [
+        sys.executable, "-m", "imagefind.audio_worker",
+        "--data-dir", str(data_dir), "--wav", str(wav),
+        "--device", device, "--model-variant", model,
+        "--segmentation", segmentation, "--quality-profile", "accuracy",
+        "--language-policy", str(window.get("language_policy", "zh_priority")),
+        "--chunk-seconds", str(max(15, min(30, (duration_ms + 999) // 1000))),
+        "--overlap-seconds", "0",
+        "--start-frame", str(start_ms * sample_rate // 1000),
+        "--max-chunks", "1", "--cpu-threads", "2",
+    ]
+    started = time.monotonic()
+    process = subprocess.run(command, text=True, capture_output=True, encoding="utf-8", errors="replace")
+    elapsed = time.monotonic() - started
+    events = []
+    for line in process.stdout.splitlines():
+        try:
+            events.append(json.loads(line))
+        except json.JSONDecodeError:
+            pass
+    return {
+        "combination": name,
+        "model": model,
+        "device": device,
+        "beam": 1 if device == "GPU" else 5,
+        "segmentation": segmentation,
+        "elapsed_seconds": round(elapsed, 3),
+        "window_seconds": round(duration_ms / 1000, 3),
+        "rtf": round(elapsed / max(0.001, duration_ms / 1000), 3),
+        "exit_code": process.returncode,
+        "events": events,
+        "stderr_tail": process.stderr[-2000:],
+    }
+
+
+def main() -> int:
+    args = arguments()
+    if args.output.exists() and not args.force:
+        raise SystemExit("output exists; pass --force to replace this diagnostic artifact")
+    windows = json.loads(args.windows.read_text(encoding="utf-8"))
+    artifact = {
+        "schema_version": 1,
+        "created_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+        "database_writes": False,
+        "runs": [],
+    }
+    with tempfile.TemporaryDirectory(prefix="imagefind-audio-ab-") as work:
+        for media in args.media:
+            wav = Path(work) / f"{media.stem}.wav"
+            subprocess.run(
+                [
+                    args.ffmpeg, "-nostdin", "-y", "-i", str(media), "-vn",
+                    "-ac", "1", "-ar", "16000", "-c:a", "pcm_s16le", str(wav),
+                ],
+                check=True,
+                capture_output=True,
+            )
+            selected = windows.get(media.name)
+            if not isinstance(selected, list) or len(selected) != 8:
+                raise SystemExit(f"{media.name}: windows JSON must contain exactly 8 labelled windows")
+            for window in selected:
+                for combination in COMBINATIONS:
+                    result = worker_run(args.data_dir, wav, window, combination)
+                    result.update({"media": media.name, "window": window})
+                    artifact["runs"].append(result)
+    args.output.parent.mkdir(parents=True, exist_ok=True)
+    args.output.write_text(json.dumps(artifact, ensure_ascii=False, indent=2), encoding="utf-8")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/audio_small_ab_windows.example.json

@@ -0,0 +1,12 @@
+{
+  "M3316 直男与0-勾引直男体育生2.mp4": [
+    {"label": "clear_dialogue_1", "start_ms": 0, "end_ms": 20000},
+    {"label": "clear_dialogue_2", "start_ms": 20000, "end_ms": 40000},
+    {"label": "short_dense_1", "start_ms": 40000, "end_ms": 60000},
+    {"label": "short_dense_2", "start_ms": 60000, "end_ms": 80000},
+    {"label": "suspected_hallucination_1", "start_ms": 80000, "end_ms": 100000},
+    {"label": "suspected_hallucination_2", "start_ms": 100000, "end_ms": 120000},
+    {"label": "vad_miss_1", "start_ms": 120000, "end_ms": 140000},
+    {"label": "vad_miss_2", "start_ms": 140000, "end_ms": 160000}
+  ]
+}

tests/test_api.py

@@ -638,6 +638,9 @@ def test_transcript_coverage_pagination_and_manual_reindex(tmp_path: Path, monke
             assert payload["items"][0]["start_ms"] == 10_000
             assert payload["detected_language"] == "zh"
             assert payload["quality_state"] == "ready"
+            assert payload["aggregate_risk"] is False
+            assert payload["repeated_phrases"] == []
+            assert payload["quality_score_semantics"] == "rule_check"
             assert payload["quality_flags"] == []
 
             speech = await client.get("/api/v1/speech/config", headers=headers)

tests/test_audio_index.py

@@ -120,6 +120,42 @@ def test_audio_indexer_writes_timed_fts_entries_and_reconciles(tmp_path: Path, m
     assert json.loads(job["payload_json"]) == {"video_id": "video"}
 
 
+def test_repeated_hallucinations_are_saved_but_not_searchable(tmp_path: Path, monkeypatch):
+    app = _app(tmp_path)
+    service = app.state.services
+    monkeypatch.setattr(service.media, "input_for", lambda _video: MediaInput("movie.mp4"))
+    monkeypatch.setattr(
+        service.media,
+        "probe",
+        lambda _media: {"raw": {"streams": [{"codec_type": "audio"}]}, "duration_ms": 30_000},
+    )
+    monkeypatch.setattr(
+        service.speech,
+        "transcribe",
+        lambda *_args, **_kwargs: [
+            {"text": "拜拜", "start_ms": index * 1000, "end_ms": index * 1000 + 800}
+            for index in range(8)
+        ],
+    )
+    job_id = service.jobs.enqueue("transcribe_audio", {"video_id": "video"}, dedupe_key="audio:video")
+    service.audio_indexer.index(job_id, "video", quality_repair=True)
+    with service.db.read() as conn:
+        entries = conn.execute(
+            "SELECT count(*) FROM text_entries WHERE video_id='video' AND kind='audio'"
+        ).fetchone()[0]
+        searchable = conn.execute(
+            "SELECT count(*) FROM text_fts WHERE entry_id IN "
+            "(SELECT id FROM text_entries WHERE video_id='video' AND kind='audio')"
+        ).fetchone()[0]
+        video = conn.execute(
+            "SELECT audio_quality_score,audio_quality_flags_json FROM videos WHERE id='video'"
+        ).fetchone()
+    assert entries == 8
+    assert searchable == 0
+    assert video["audio_quality_score"] < 0.7
+    assert "whole_ending_hallucination" in json.loads(video["audio_quality_flags_json"])
+
+
 def test_audio_indexer_retries_medium_memory_pressure(tmp_path: Path, monkeypatch):
     app = _app(tmp_path)
     service = app.state.services

tests/test_speech_quality.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 from types import SimpleNamespace
 
 from imagefind.speech_quality import (
+    aggregate_transcript_quality,
     normalize_language,
     pcm16_voiced_regions,
     select_language,
@@ -10,6 +11,32 @@ from imagefind.speech_quality import (
 )
 
 
+def test_aggregate_quality_rejects_repeated_short_hallucinations() -> None:
+    segments = ([{"text": "好"}] * 11) + ([{"text": "拜拜"}] * 7) + ([{"text": "正常对话内容"}] * 4)
+    result = aggregate_transcript_quality(segments, base_score=1.0)
+    assert result.score < 0.7
+    assert "whole_repeated_phrase" in result.flags
+    assert "whole_short_segment_dominance" in result.flags
+    assert "whole_ending_hallucination" in result.flags
+    assert ("拜拜", 7) in result.repeated_phrases
+
+
+def test_sports_samples_patterns_cannot_remain_ready() -> None:
+    sports_2 = ([{"text": "好"}] * 11) + ([{"text": "拜拜"}] * 7) + ([{"text": "嗯"}] * 5)
+    sports_2 += [{"text": f"正常对话{i}"} for i in range(19)]
+    sports_3 = ([{"text": "拜拜"}] * 8) + ([{"text": "啊"}] * 7) + ([{"text": "嗯"}] * 7)
+    sports_3 += ([{"text": "谢谢大家收看"}] * 3) + [{"text": f"正常内容{i}"} for i in range(32)]
+    for segments in (sports_2, sports_3):
+        result = aggregate_transcript_quality(segments, base_score=1.0)
+        assert result.score < 0.7
+        assert "whole_repeated_phrase" in result.flags
+
+
+def test_traditional_ending_phrase_is_flagged() -> None:
+    result = aggregate_transcript_quality([{"text": "謝謝大家收看"}] * 3, base_score=1.0)
+    assert "whole_ending_hallucination" in result.flags
+
+
 def test_language_policy_treats_stream_metadata_as_weak_hint() -> None:
     assert select_language(["en", "zh"], policy="zh_priority") == "zh"
     assert select_language(["en", "zh"], policy="auto") == "en"