nanxun/imagefind
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(audio): detect transcript-wide hallucinations

Browse Source
This commit is contained in:
nanxun
2026-08-12 13:32:10 +08:00
parent c321d020dd
commit 48f4efaf3d
10 changed files with 318 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/imagefind/api.py
+12
View File
@@ -2138,6 +2138,15 @@ def video_transcript(
int(video["audio_rejected_segments"] or 0),
quality_flags,
)
repeated_phrases = []
for flag in quality_flags:
if not isinstance(flag, str) or not flag.startswith("repeated_phrase:"):
continue
try:
phrase, count = flag.removeprefix("repeated_phrase:").rsplit(":", 1)
repeated_phrases.append({"text": phrase, "count": int(count)})
except (TypeError, ValueError):
continue
if job and job["status"] in {"queued", "running"}:
status = job["status"]
elif current:
@@ -2159,6 +2168,9 @@ def video_transcript(
else "legacy",
"rejected_segments": int(video["audio_rejected_segments"] or 0),
"quality_flags": quality_flags,
"aggregate_risk": quality_state == "low_quality",
"repeated_phrases": repeated_phrases,
"quality_score_semantics": "rule_check",
"items": [dict(row) for row in rows],
"page": page,
"page_size": page_size,
backend/imagefind/audio_worker.py
+50 -8
View File
@@ -15,7 +15,14 @@ from pathlib import Path
from .accelerator import AcceleratorService
from .config import Settings
from .speech import SpeechService, SpeechStageError
from .speech_quality import normalize_language, pcm16_voiced_regions, select_language, transcript_quality
from .speech_quality import (
aggregate_transcript_quality,
normalize_language,
pcm16_speech_ratio,
pcm16_voiced_regions,
select_language,
transcript_quality,
)
def _emit(payload: dict) -> None:
@@ -39,6 +46,7 @@ def _arguments() -> argparse.Namespace:
parser.add_argument("--language-policy", choices=("zh_priority", "auto", "zh"), default="zh_priority")
parser.add_argument("--quality-profile", choices=("speed", "balanced", "accuracy"), default="accuracy")
parser.add_argument("--model-variant", choices=("small", "medium"), default="small")
parser.add_argument("--segmentation", choices=("vad", "continuous"), default="vad")
return parser.parse_args()
@@ -240,6 +248,7 @@ def run(args: argparse.Namespace) -> int:
strongest_voiced_frame = 0
rejected_candidate_units = 0
verification_duration_ms = 0
accepted_all: list[dict] = []
with wave.open(str(wav_path), "rb") as handle:
sample_rate = handle.getframerate()
total_frames = handle.getnframes()
@@ -273,12 +282,15 @@ def run(args: argparse.Namespace) -> int:
upper_ms = end_ms - (args.overlap_seconds * 500 if end_frame < total_frames else 0)
energy = _sample_energy(samples)
segments = []
regions = pcm16_voiced_regions(raw, sample_rate, mode=2)
if not regions and energy > 1e-6:
regions = pcm16_voiced_regions(raw, sample_rate, mode=1)
if args.segmentation == "continuous":
regions = [(0, len(raw) // 2, pcm16_speech_ratio(raw, sample_rate, mode=2))]
else:
regions = pcm16_voiced_regions(raw, sample_rate, mode=2)
if not regions and energy > 1e-6:
regions = pcm16_voiced_regions(raw, sample_rate, mode=1)
windows: list[tuple[list[float], float, int, int]] = []
for region_start, region_end, speech_ratio in regions:
if speech_ratio < 0.08:
if speech_ratio < 0.08 and args.segmentation != "continuous":
continue
if speech_ratio > strongest_voiced_ratio:
strongest_voiced_ratio = speech_ratio
@@ -341,6 +353,29 @@ def run(args: argparse.Namespace) -> int:
rejected_total += rejected
quality_flags.extend(flags)
quality_scores.append(candidate_score)
elapsed = time.monotonic() - inference_started
_emit(
{
"event": "window_result",
"chunk_index": index + 1,
"window_index": offset + 1,
"start_ms": window_base_ms,
"duration_ms": window_duration_ms,
"speech_ratio": round(speech_ratio, 4),
"language_candidates": detected,
"forced_language": resolved_language,
"raw_output": raw_segments,
"accepted_output": scored,
"quality_score": candidate_score,
"quality_flags": flags,
"rejected_segments": rejected,
"device": args.device,
"beam": 1 if args.device == "GPU" else speech.generation_beams(quality_profile),
"elapsed_seconds": round(elapsed, 3),
"rtf": round(elapsed / max(0.001, window_duration_ms / 1000), 3),
"segmentation": args.segmentation,
}
)
for segment in scored:
segment["start_ms"] += window_base_ms
segment["end_ms"] += window_base_ms
@@ -349,6 +384,7 @@ def run(args: argparse.Namespace) -> int:
midpoint = (segment["start_ms"] + segment["end_ms"]) // 2
if lower_ms <= midpoint <= upper_ms:
segments.append(segment)
accepted_all.append(dict(segment))
recognized_segments += len(segments)
if not segments:
if energy > strongest_empty_energy:
@@ -397,20 +433,26 @@ def run(args: argparse.Namespace) -> int:
"segmentation": "webrtcvad",
},
)
aggregate = aggregate_transcript_quality(
accepted_all,
base_score=round(sum(quality_scores) / len(quality_scores), 3) if quality_scores else None,
base_flags=quality_flags,
)
_emit(
{
"event": "complete",
"detected_language": resolved_language,
"quality_score": round(sum(quality_scores) / len(quality_scores), 3) if quality_scores else None,
"quality_score": aggregate.score,
"rejected_segments": rejected_total,
"quality_flags": list(dict.fromkeys(quality_flags)),
"quality_flags": list(aggregate.flags),
"inference_diagnostics": {
"requested_device": args.device,
"actual_device": component.get("actual_device") or component.get("device") or args.device,
"primary_backend": "direct_generate",
"model_variant": model_variant,
"num_beams": 1 if args.device == "GPU" else speech.generation_beams(quality_profile),
"segmentation": "webrtcvad",
"segmentation": args.segmentation,
"quality_score_semantics": "rule_check",
"language_candidates": detected,
"selected_language": resolved_language,
"fallback_scope": None,
backend/imagefind/speech.py
+9 -2
View File
@@ -25,7 +25,7 @@ from .database import Database, utcnow
from .jobs import JobCancelled, JobQueue, JobRetry
from .media import MediaInput, MediaService
from .runtime import RuntimeToolManager
from .speech_quality import normalize_language, transcript_quality_state
from .speech_quality import aggregate_transcript_quality, normalize_language, transcript_quality_state
from .text import search_tokens
logger = logging.getLogger(__name__)
@@ -1550,6 +1550,11 @@ class AudioIndexer:
quality_score = getattr(segments, "quality_score", None)
rejected_segments = int(getattr(segments, "rejected_segments", 0) or 0)
quality_flags = list(getattr(segments, "quality_flags", []) or [])
aggregate_quality = aggregate_transcript_quality(
segments, base_score=quality_score, base_flags=quality_flags
)
quality_score = aggregate_quality.score
quality_flags = list(aggregate_quality.flags)
quality_state = transcript_quality_state(
quality_score,
len(segments),
@@ -1581,7 +1586,9 @@ class AudioIndexer:
utcnow(),
),
)
if tokens:
# Risky transcripts remain visible for diagnosis and playback,
# but repeated hallucinations must not pollute global search.
if tokens and quality_state != "low_quality":
conn.execute("INSERT INTO text_fts(entry_id,tokens) VALUES(?,?)", (entry_id, tokens))
conn.execute(
"UPDATE videos SET audio_model_version=?,audio_index_revision=?,audio_detected_language=?,"
backend/imagefind/speech_quality.py
+57
View File
@@ -43,6 +43,60 @@ class TranscriptQuality:
units: int
@dataclass(frozen=True, slots=True)
class AggregateTranscriptQuality:
score: float
flags: tuple[str, ...]
repeated_phrases: tuple[tuple[str, int], ...]
short_segment_ratio: float
_ENDING_HALLUCINATIONS = {
"拜拜", "再见", "谢谢观看", "谢谢大家观看", "谢谢大家收看",
"感谢观看", "下期再见", "謝謝觀看", "謝謝大家觀看", "謝謝大家收看",
"感謝觀看", "下期再見", "我去看看", "thanksforwatching",
}
def _phrase_key(text: object) -> str:
value = unicodedata.normalize("NFKC", str(text or "")).casefold()
return "".join(character for character in value if character.isalnum())
def aggregate_transcript_quality(
segments: list[dict] | tuple[dict, ...],
*,
base_score: float | None = None,
base_flags: list[str] | tuple[str, ...] = (),
) -> AggregateTranscriptQuality:
"""Detect risks that only become visible across a whole transcript."""
texts = [_phrase_key(item.get("text")) for item in segments]
texts = [text for text in texts if text]
flags = list(dict.fromkeys(str(flag) for flag in base_flags if flag))
if not texts:
return AggregateTranscriptQuality(round(float(base_score or 0.0), 3), tuple(flags), (), 0.0)
counts = Counter(texts)
repeated = tuple(sorted(
((text, count) for text, count in counts.items() if count >= 3),
key=lambda item: (-item[1], item[0]),
)[:12])
short_ratio = sum(len(text) < 4 for text in texts) / len(texts)
repeated_units = sum(count for _text, count in repeated)
penalty = 0.0
if repeated and (repeated_units >= 6 or repeated_units / len(texts) >= 0.20):
flags.append("whole_repeated_phrase")
penalty += 0.35
if short_ratio >= 0.65 and len(texts) >= 10:
flags.append("whole_short_segment_dominance")
penalty += 0.35
if any(text in _ENDING_HALLUCINATIONS for text, _count in repeated):
flags.append("whole_ending_hallucination")
penalty += 0.45
flags.extend(f"repeated_phrase:{text}:{count}" for text, count in repeated)
score = max(0.0, min(1.0, float(1.0 if base_score is None else base_score) - penalty))
return AggregateTranscriptQuality(round(score, 3), tuple(dict.fromkeys(flags)), repeated, round(short_ratio, 3))
def normalize_language(value: object) -> str | None:
language = str(value or "").strip().lower()
if language.startswith("<|") and language.endswith("|>"):
@@ -98,6 +152,9 @@ def transcript_quality_state(
"no_speech_hallucination",
"language_script_conflict",
"untrusted_language",
"whole_repeated_phrase",
"whole_short_segment_dominance",
"whole_ending_hallucination",
}
if high_risk.intersection(flags or ()):
return "low_quality"