from __future__ import annotations import json import os import sys import wave from pathlib import Path from types import SimpleNamespace import pytest from imagefind import audio_worker from imagefind.ai import ModelUnavailable from imagefind.config import Settings from imagefind.database import utcnow from imagefind.jobs import JobRetry from imagefind.main import create_app from imagefind.media import MediaInput from imagefind.speech import SPEECH_INDEX_REVISION, SpeechService, SpeechStageError from imagefind.speech_quality import aggregate_transcript_quality from imagefind.text import fts_query def test_ending_hallucination_is_risky_twice_or_once_in_weak_speech(): repeated = aggregate_transcript_quality([ {"text": "拜拜"}, {"text": "正常对话"}, {"text": "拜拜"}, ]) assert "whole_ending_hallucination" in repeated.flags assert ("拜拜", 2) in repeated.repeated_phrases weak = aggregate_transcript_quality([ {"text": "谢谢大家收看", "speech_ratio": 0.1}, {"text": "正常对话", "speech_ratio": 0.8}, ]) assert "whole_ending_hallucination" in weak.flags single = aggregate_transcript_quality([ {"text": "拜拜", "speech_ratio": 0.8}, {"text": "正常对话", "speech_ratio": 0.8}, ]) assert "whole_ending_hallucination" not in single.flags def _app(tmp_path: Path): settings = Settings(data_dir=tmp_path / "data", embedding_backend="hash", upload_reserve_gb=0) settings.prepare() app = create_app(settings) audio = settings.models_dir / "audio" audio.mkdir(parents=True) for name in ( "config.json", "preprocessor_config.json", "tokenizer_config.json", "vocab.json", "openvino_encoder_model.xml", "openvino_encoder_model.bin", "openvino_decoder_model.xml", "openvino_decoder_model.bin", "openvino_decoder_with_past_model.xml", "openvino_decoder_with_past_model.bin", ): (audio / name).write_text("{}" if name.endswith(".json") else "model") (settings.models_dir / "manifest.json").write_text( json.dumps({"version": "v6", "components": {"audio": {"version": "whisper-small-int8"}}}) ) service = app.state.services now = utcnow() media = tmp_path / "movie.mp4" media.write_bytes(b"video") with service.db.transaction() as conn: conn.execute( "INSERT INTO sources(id,kind,name,config_json,created_at,updated_at) VALUES(?,?,?,?,?,?)", ("source", "local", "本地", json.dumps({"path": str(tmp_path)}), now, now), ) conn.execute( "INSERT INTO videos(id,source_id,source_key,display_name,location,fingerprint,basic_fingerprint," "visual_model_version,status,available,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?,?,?,?,?)", ( "video", "source", "movie.mp4", "movie.mp4", str(media), "fingerprint", "fingerprint", "hash-v1", "indexed", 1, now, now, ), ) return app def test_audio_indexer_writes_timed_fts_entries_and_reconciles(tmp_path: Path, monkeypatch): app = _app(tmp_path) service = app.state.services monkeypatch.setattr(service.media, "input_for", lambda _video: MediaInput("movie.mp4")) monkeypatch.setattr( service.media, "probe", lambda _media: {"raw": {"streams": [{"codec_type": "audio"}]}, "duration_ms": 90_000}, ) def transcribe(_media, _probe, progress, **_kwargs): progress(0.5, "识别默认音轨 50%") return [{"text": "今天讨论本地人工智能", "start_ms": 12_000, "end_ms": 15_500}] monkeypatch.setattr(service.speech, "transcribe", transcribe) job_id = service.jobs.enqueue("transcribe_audio", {"video_id": "video"}, dedupe_key="audio:video") service.audio_indexer.index(job_id, "video") with service.db.read() as conn: entry = conn.execute("SELECT kind,start_ms,end_ms,raw_text FROM text_entries WHERE video_id='video'").fetchone() version = conn.execute("SELECT audio_model_version FROM videos WHERE id='video'").fetchone()[0] fts = conn.execute("SELECT entry_id FROM text_fts WHERE text_fts MATCH ?", (fts_query("人工智能"),)).fetchone() assert dict(entry) == { "kind": "audio", "start_ms": 12_000, "end_ms": 15_500, "raw_text": "今天讨论本地人工智能", } assert version == "whisper-small-int8" assert fts is not None service.audio_indexer.index(job_id, "video", quality_repair=True) with service.db.read() as conn: repair_revision = conn.execute( "SELECT audio_quality_repair_revision FROM videos WHERE id='video'" ).fetchone()[0] assert repair_revision == SPEECH_INDEX_REVISION with service.db.transaction() as conn: conn.execute("DELETE FROM jobs") conn.execute("UPDATE videos SET audio_model_version=NULL WHERE id='video'") queued = service.reconcile_ai() assert len(queued) == 1 with service.db.read() as conn: job = conn.execute("SELECT kind,payload_json FROM jobs WHERE id=?", (queued[0],)).fetchone() assert job["kind"] == "transcribe_audio" assert json.loads(job["payload_json"]) == {"video_id": "video"} def test_repeated_hallucinations_are_saved_but_not_searchable(tmp_path: Path, monkeypatch): app = _app(tmp_path) service = app.state.services monkeypatch.setattr(service.media, "input_for", lambda _video: MediaInput("movie.mp4")) monkeypatch.setattr( service.media, "probe", lambda _media: {"raw": {"streams": [{"codec_type": "audio"}]}, "duration_ms": 30_000}, ) monkeypatch.setattr( service.speech, "transcribe", lambda *_args, **_kwargs: [ {"text": "拜拜", "start_ms": index * 1000, "end_ms": index * 1000 + 800} for index in range(8) ], ) job_id = service.jobs.enqueue("transcribe_audio", {"video_id": "video"}, dedupe_key="audio:video") service.audio_indexer.index(job_id, "video", quality_repair=True) with service.db.read() as conn: entries = conn.execute( "SELECT count(*) FROM text_entries WHERE video_id='video' AND kind='audio'" ).fetchone()[0] searchable = conn.execute( "SELECT count(*) FROM text_fts WHERE entry_id IN " "(SELECT id FROM text_entries WHERE video_id='video' AND kind='audio')" ).fetchone()[0] video = conn.execute( "SELECT audio_quality_score,audio_quality_flags_json FROM videos WHERE id='video'" ).fetchone() assert entries == 8 assert searchable == 0 assert video["audio_quality_score"] < 0.7 assert "whole_ending_hallucination" in json.loads(video["audio_quality_flags_json"]) def test_audio_indexer_retries_medium_memory_pressure(tmp_path: Path, monkeypatch): app = _app(tmp_path) service = app.state.services monkeypatch.setattr(service.media, "input_for", lambda _video: MediaInput("movie.mp4")) monkeypatch.setattr( service.media, "probe", lambda _media: {"raw": {"streams": [{"codec_type": "audio"}]}, "duration_ms": 90_000}, ) def transcribe(*_args, **_kwargs): raise ModelUnavailable("Whisper Medium 可用内存不足:至少需要额外 4.0 GiB,且需保留系统内存") monkeypatch.setattr(service.speech, "transcribe", transcribe) job_id = service.jobs.enqueue("transcribe_audio", {"video_id": "video"}, dedupe_key="audio:video") with pytest.raises(JobRetry, match="资源释放后自动重试") as retry: service.audio_indexer.index(job_id, "video") assert retry.value.delay_seconds == 120 def test_reconcile_ai_safely_replaces_historical_locked_index_job(tmp_path: Path): app = _app(tmp_path) service = app.state.services failed_job = service.jobs.enqueue( "index_video", {"video_id": "video"}, dedupe_key="index:video", ) with service.db.transaction() as conn: conn.execute( "UPDATE jobs SET status='failed',error='deadlock detected',finished_at=? WHERE id=?", (utcnow(), failed_job), ) conn.execute( "UPDATE videos SET indexed_fingerprint=NULL,basic_fingerprint=NULL WHERE id='video'" ) first = service.reconcile_ai() second = service.reconcile_ai() assert len(first) == 1 assert second == first with service.db.read() as conn: active = conn.execute( "SELECT count(*) FROM jobs WHERE kind='index_video' AND status IN ('queued','running')" ).fetchone()[0] replacement = conn.execute( "SELECT payload_json FROM jobs WHERE id=?", (first[0],), ).fetchone() assert active == 1 assert json.loads(replacement["payload_json"])["video_id"] == "video" def test_speech_service_skips_video_without_audio_stream(tmp_path: Path): app = _app(tmp_path) speech = app.state.services.speech result = speech.transcribe(MediaInput("movie.mp4"), {"raw": {"streams": []}, "duration_ms": 1}) assert result == [] def test_speech_service_loads_local_openvino_model_without_library_inference(tmp_path: Path): app = _app(tmp_path) speech = app.state.services.speech calls: dict[str, object] = {} class ConfigFactory: @staticmethod def from_pretrained(path, **kwargs): calls["config"] = (path, kwargs) return SimpleNamespace(architectures=["WhisperForConditionalGeneration"]) class ProcessorFactory: @staticmethod def from_pretrained(path, **kwargs): calls["processor"] = (path, kwargs) return SimpleNamespace(tokenizer="tokenizer", feature_extractor="feature-extractor") class ModelFactory: class LoadedModel: def compile(self): calls["compiled"] = True @classmethod def from_pretrained(cls, *_args, **_kwargs): raise AssertionError("public loader must not run library inference") @classmethod def _from_pretrained(cls, path, config, **kwargs): calls["model"] = (path, config, kwargs) return cls.LoadedModel() def pipeline_factory(task, **kwargs): calls["pipeline"] = (task, kwargs) return "speech-pipeline" result = speech._build_pipeline( "CPU", ConfigFactory, ProcessorFactory, ModelFactory, pipeline_factory, ) assert result == "speech-pipeline" model_path, _, model_kwargs = calls["model"] assert model_path == speech.model_path.resolve() assert model_kwargs["local_files_only"] is True assert model_kwargs["compile"] is False # Whisper generation needs the regular model wrapper. ``compile_only`` # can compile the IR successfully while returning only the first word. assert model_kwargs["compile_only"] is False assert model_kwargs["use_cache"] is True assert calls["compiled"] is True assert calls["pipeline"][1]["framework"] == "pt" def test_speech_service_accepts_export_without_optional_decoder_cache(tmp_path: Path): app = _app(tmp_path) speech = app.state.services.speech (speech.model_path / "openvino_decoder_with_past_model.xml").unlink() (speech.model_path / "openvino_decoder_with_past_model.bin").unlink() calls: dict[str, object] = {} class ConfigFactory: @staticmethod def from_pretrained(_path, **_kwargs): return SimpleNamespace(architectures=["WhisperForConditionalGeneration"]) class ProcessorFactory: @staticmethod def from_pretrained(_path, **_kwargs): return SimpleNamespace(tokenizer="tokenizer", feature_extractor="feature-extractor") class ModelFactory: @classmethod def _from_pretrained(cls, _path, config=None, **kwargs): assert config is not None calls["model"] = kwargs return "openvino-model" result = speech._build_pipeline( "CPU", ConfigFactory, ProcessorFactory, ModelFactory, lambda _task, **_kwargs: "speech-pipeline", ) assert speech.ready() is True assert result == "speech-pipeline" assert calls["model"]["use_cache"] is False def test_int8_whisper_is_marked_for_fp16_repair_when_intel_gpu_is_available(tmp_path: Path): app = _app(tmp_path) services = app.state.services services.accelerator.gpu_device = "GPU.0" manifest_path = services.settings.models_dir / "manifest.json" manifest_path.write_text( json.dumps( { "version": "legacy", "components": { "audio": {"version": "openai/whisper-small-openvino-int8"}, }, } ) ) health = services.models.component_health()["audio"] assert health["state"] == "outdated" assert "FP16" in health["error"] assert services.models.runnable_component_versions()["audio"] is None repair_job_id = services.queue_missing_ai_runtime() with services.db.read() as connection: repair_job = connection.execute( "SELECT kind,payload_json,dedupe_key FROM jobs WHERE id=?", (repair_job_id,) ).fetchone() assert repair_job["kind"] == "install_models" assert json.loads(repair_job["payload_json"]) == {"component": "audio"} assert repair_job["dedupe_key"] == "install-model:audio" def test_legacy_locally_exported_fp16_whisper_is_replaced_by_official_model(tmp_path: Path): app = _app(tmp_path) services = app.state.services manifest_path = services.settings.models_dir / "manifest.json" manifest_path.write_text( json.dumps( { "version": "legacy", "components": { "audio": {"version": "openai/whisper-small-openvino-fp16-v1"}, }, } ) ) health = services.models.component_health()["audio"] assert Settings(data_dir=tmp_path / "defaults").audio_model == "OpenVINO/whisper-small-fp16-ov" assert health["state"] == "outdated" assert "OpenVINO 官方" in health["error"] assert services.models.runnable_component_versions()["audio"] is None def test_speech_service_uses_quality_profile_beams_for_whisper_generation(tmp_path: Path, monkeypatch): app = _app(tmp_path) speech = app.state.services.speech calls: dict[str, object] = {} def engine(inputs, **kwargs): calls["inputs"] = inputs calls["kwargs"] = kwargs return {"chunks": []} monkeypatch.setattr(speech, "_load", lambda: engine) assert speech._transcribe_samples([0.0], 16_000) == [] generation = calls["kwargs"]["generate_kwargs"] assert generation == { "task": "transcribe", "num_beams": 5, "max_new_tokens": 192, "do_sample": False, "use_cache": True, "early_stopping": True, } assert calls["kwargs"]["return_timestamps"] is True def test_speech_service_can_retry_without_timestamp_generation(tmp_path: Path, monkeypatch): settings = Settings(data_dir=tmp_path / "data", embedding_backend="hash") settings.prepare() speech = SpeechService(settings) calls: list[dict] = [] def engine(_inputs, **kwargs): calls.append(kwargs) return {"text": "兼容解码返回文字", "chunks": []} monkeypatch.setattr(speech, "_load", lambda: engine) assert speech._transcribe_samples( [0.1] * 16_000, 16_000, return_timestamps=False, ) == [{"text": "兼容解码返回文字", "start_ms": 0, "end_ms": 1000}] assert calls[0]["return_timestamps"] is False def test_speech_service_native_generate_fallback_returns_seekable_chunk(tmp_path: Path, monkeypatch): app = _app(tmp_path) speech = app.state.services.speech calls: dict[str, object] = {} class Processor: def __call__(self, samples, **kwargs): calls["processor"] = (samples, kwargs) return SimpleNamespace(input_features="features", attention_mask="mask") @staticmethod def batch_decode(tokens, **kwargs): calls["decode"] = (tokens, kwargs) return ["The verification keyword is nebula seven two nine."] class Model: @staticmethod def generate(**kwargs): calls["generate"] = kwargs return [[1, 2, 3]] speech._model = Model() speech._processor = Processor() speech._pipeline_device = "GPU.0" monkeypatch.setattr(speech, "_load", lambda: object()) result = speech._direct_transcribe_samples([0.1] * 32_000, 16_000) assert result == [ { "text": "The verification keyword is nebula seven two nine.", "start_ms": 0, "end_ms": 2000, } ] assert calls["generate"] == { "input_features": "features", "attention_mask": "mask", "task": "transcribe", # Intel GPU uses the stateful decoder's deterministic greedy path; # beam expansion can stall before the first segment is emitted. "num_beams": 1, "max_new_tokens": 96, "do_sample": False, "use_cache": True, "early_stopping": True, "return_timestamps": False, } assert calls["decode"][1] == {"skip_special_tokens": True} assert calls["processor"][1]["return_attention_mask"] is True def test_speech_full_text_fallback_uses_real_chunk_duration(tmp_path: Path, monkeypatch): app = _app(tmp_path) speech = app.state.services.speech monkeypatch.setattr(speech, "_load", lambda: lambda *_args, **_kwargs: {"text": "完整文本"}) assert speech._transcribe_samples([0.1] * 32_000, 16_000) == [{"text": "完整文本", "start_ms": 0, "end_ms": 2000}] def test_speech_accepts_segment_timestamp_variants_and_rejects_non_silent_empty( tmp_path: Path, monkeypatch, ): app = _app(tmp_path) speech = app.state.services.speech monkeypatch.setattr( speech, "_load", lambda: ( lambda *_args, **_kwargs: { "segments": [ {"text": "第一段", "timestamps": {"start": 0.25, "end": 0.75}}, {"text": "第二段", "start_ms": 900, "end_ms": 1300}, ] } ), ) assert speech._transcribe_samples([0.1] * 32_000, 16_000) == [ {"text": "第一段", "start_ms": 250, "end_ms": 750}, {"text": "第二段", "start_ms": 900, "end_ms": 1300}, ] monkeypatch.setattr(speech, "_load", lambda: lambda *_args, **_kwargs: {"chunks": []}) with pytest.raises(RuntimeError, match="未返回文字或时间片段"): speech._transcribe_samples([0.1] * 16_000, 16_000) assert ( speech._transcribe_samples( [0.1] * 16_000, 16_000, allow_non_silent_empty=True, ) == [] ) def test_speech_service_recognizes_modern_stateful_decoder_cache(tmp_path: Path, monkeypatch): app = _app(tmp_path) speech = app.state.services.speech (speech.model_path / "openvino_decoder_with_past_model.xml").unlink() (speech.model_path / "openvino_decoder_with_past_model.bin").unlink() (speech.model_path / "openvino_decoder_model.xml").write_text( '' ) speech.accelerator.gpu_device = "GPU.0" calls = [] def run_once(_wav_path, requested, _stall_seconds, _progress, *_args, **_kwargs): calls.append(requested) return [{"text": "gpu", "start_ms": 0, "end_ms": 1000}] monkeypatch.setattr(speech, "_isolated_transcribe_once", run_once) assert speech.has_decoder_cache() is True result = speech._isolated_transcribe(tmp_path / "audio.wav", lambda *_args: None) assert result[0]["text"] == "gpu" assert calls == ["GPU.0"] health = app.state.services.models.component_health()["audio"] assert health == {"state": "ready", "error": None} def test_isolated_audio_rejects_gpu_when_decoder_cache_is_missing(tmp_path: Path, monkeypatch): app = _app(tmp_path) speech = app.state.services.speech (speech.model_path / "openvino_decoder_with_past_model.xml").unlink() (speech.model_path / "openvino_decoder_with_past_model.bin").unlink() speech.accelerator.gpu_device = "GPU.0" calls = [] def run_once(_wav_path, requested, _stall_seconds, _progress, *_args, **_kwargs): calls.append(requested) return [{"text": "cpu fallback", "start_ms": 0, "end_ms": 1000}] monkeypatch.setattr(speech, "_isolated_transcribe_once", run_once) with pytest.raises(Exception, match="stateful decoder"): speech._isolated_transcribe(tmp_path / "audio.wav", lambda *_args: None) assert calls == [] health = app.state.services.models.component_health()["audio"] assert health["state"] == "ready" assert health["gpu_eligible"] is False assert "decoder_with_past" in health["warning"] def test_isolated_audio_never_retries_a_gpu_stall_as_full_cpu_job(tmp_path: Path, monkeypatch): app = _app(tmp_path) speech = app.state.services.speech speech.accelerator.gpu_device = "GPU.0" calls = [] def run_once(_wav_path, requested, stall_seconds, _progress, *_args, **_kwargs): calls.append((requested, stall_seconds)) if requested.startswith("GPU"): raise TimeoutError("gpu stalled") return [{"text": "cpu retry", "start_ms": 0, "end_ms": 1000}] monkeypatch.setattr(speech, "_isolated_transcribe_once", run_once) with pytest.raises(TimeoutError, match="gpu stalled"): speech._isolated_transcribe(tmp_path / "audio.wav", lambda *_args: None) assert calls == [("GPU.0", speech.settings.audio_gpu_stall_seconds)] audio = speech.accelerator.status()["components"]["audio"] assert audio["state"] == "job_fallback" assert audio["failure_stage"] == "inference_stall" assert audio["fallback_scope"] == "job" assert speech.accelerator.device_for("audio") == "GPU.0" def test_isolated_audio_verifies_empty_gpu_generation_on_cpu(tmp_path: Path, monkeypatch): app = _app(tmp_path) speech = app.state.services.speech speech.accelerator.gpu_device = "GPU.0" calls = [] def run_once(_wav_path, requested, _stall_seconds, _progress, *_args, **_kwargs): calls.append(requested) if requested.startswith("GPU"): from imagefind.speech import SpeechStageError raise SpeechStageError("empty_result", "GPU returned no text") return [] messages = [] monkeypatch.setattr(speech, "_isolated_transcribe_once", run_once) result = speech._isolated_transcribe(tmp_path / "audio.wav", lambda _, message: messages.append(message)) assert result == [] assert calls == ["GPU.0", "CPU"] assert any("复核" in message for message in messages) assert any("均未检测到" in message for message in messages) # Content-specific no-speech must not permanently disable a healthy GPU. audio = speech.accelerator.status()["components"]["audio"] assert audio["state"] == "ready" assert audio["actual_device"] == "GPU.0" def test_isolated_audio_marks_gpu_result_untrusted_when_bounded_verifier_finds_speech( tmp_path: Path, monkeypatch, ): app = _app(tmp_path) speech = app.state.services.speech speech.accelerator.gpu_device = "GPU.0" calls = [] def run_once(_wav_path, requested, _stall_seconds, _progress, *_args, **kwargs): calls.append((requested, kwargs.get("max_chunks", 0), kwargs.get("verification", False))) if requested.startswith("GPU"): from imagefind.speech import SpeechStageError raise SpeechStageError( "empty_result", "GPU returned no text", {"verification_start_frame": 32_000}, ) if kwargs.get("verification"): return [{"text": "cpu sample", "start_ms": 2000, "end_ms": 3000}] return [{"text": "cpu full", "start_ms": 0, "end_ms": 1000}] monkeypatch.setattr(speech, "_isolated_transcribe_once", run_once) with pytest.raises(SpeechStageError, match="未通过短片段复核") as error: speech._isolated_transcribe(tmp_path / "audio.wav", lambda *_args: None) assert error.value.stage == "gpu_result_untrusted" assert calls == [("GPU.0", 0, False), ("CPU", 1, True)] audio = speech.accelerator.status()["components"]["audio"] assert audio["state"] == "ready" assert audio["failure_stage"] is None def test_isolated_audio_marks_gpu_result_untrusted_when_cpu_sample_is_materially_richer( tmp_path: Path, monkeypatch, ): app = _app(tmp_path) speech = app.state.services.speech speech.accelerator.gpu_device = "GPU.0" candidate = [{"text": "And", "start_ms": 0, "end_ms": 11_000}] calls = [] def run_once(_wav_path, requested, _stall_seconds, _progress, *_args, **kwargs): calls.append((requested, kwargs.get("max_chunks", 0), kwargs.get("verification", False))) if requested.startswith("GPU"): raise SpeechStageError( "low_quality_result", "GPU result was too short", { "verification_start_frame": 0, "gpu_text_units": 3, "candidate_segments": candidate, }, ) if kwargs.get("verification"): return [ { "text": "Ask not what your country can do for you", "start_ms": 0, "end_ms": 5000, } ] return [{"text": "complete cpu transcript", "start_ms": 0, "end_ms": 11_000}] monkeypatch.setattr(speech, "_isolated_transcribe_once", run_once) with pytest.raises(SpeechStageError, match="未通过短片段复核") as error: speech._isolated_transcribe(tmp_path / "audio.wav", lambda *_args: None) assert error.value.stage == "gpu_result_untrusted" assert calls == [("GPU.0", 0, False), ("CPU", 1, True)] audio = speech.accelerator.status()["components"]["audio"] assert audio["state"] == "ready" assert audio["failure_stage"] is None def test_isolated_audio_keeps_short_gpu_text_when_cpu_is_not_materially_richer( tmp_path: Path, monkeypatch, ): app = _app(tmp_path) speech = app.state.services.speech speech.accelerator.gpu_device = "GPU.0" candidate = [{"text": "And", "start_ms": 0, "end_ms": 11_000}] calls = [] def run_once(_wav_path, requested, _stall_seconds, _progress, *_args, **kwargs): calls.append((requested, kwargs.get("verification", False))) if requested.startswith("GPU"): raise SpeechStageError( "low_quality_result", "GPU result was short", { "verification_start_frame": 0, "gpu_text_units": 3, "candidate_segments": candidate, }, ) return [{"text": "And", "start_ms": 0, "end_ms": 11_000}] monkeypatch.setattr(speech, "_isolated_transcribe_once", run_once) result = speech._isolated_transcribe(tmp_path / "audio.wav", lambda *_args: None) assert result == candidate assert calls == [("GPU.0", False), ("CPU", True)] audio = speech.accelerator.status()["components"]["audio"] assert audio["state"] == "ready" assert audio["actual_device"] == "GPU.0" def test_audio_acceleration_verification_releases_pipeline_but_preserves_status( tmp_path: Path, monkeypatch, ): app = _app(tmp_path) speech = app.state.services.speech speech.accelerator.gpu_device = "GPU.0" monkeypatch.setitem( sys.modules, "numpy", SimpleNamespace(float32="float32", zeros=lambda *_args, **_kwargs: [0.0]), ) def verify_sample(_samples, _sample_rate): speech._pipeline = object() speech._pipeline_device = "GPU.0" speech.accelerator.mark_ready("audio", "GPU.0", ["GPU.0"]) return [] monkeypatch.setattr(speech, "_transcribe_samples", verify_sample) status = speech.verify_acceleration() assert status["state"] == "ready" assert status["actual_device"] == "GPU.0" assert speech._pipeline is None assert speech._pipeline_device is None assert speech.accelerator.status()["components"]["audio"]["state"] == "ready" def test_orphaned_audio_worker_discovery_is_scoped_to_data_dir_and_parent(tmp_path: Path): app = _app(tmp_path) speech = app.state.services.speech proc_root = tmp_path / "proc" def process(pid: int, parent: int, data_dir: str) -> None: root = proc_root / str(pid) root.mkdir(parents=True) (root / "status").write_text(f"PPid:\t{parent}\nUid:\t{os.geteuid()} 0 0 0\n") (root / "cmdline").write_bytes( b"python\0-m\0imagefind.audio_worker\0--data-dir\0" + data_dir.encode() + b"\0--wav\0audio.wav\0" ) process(41001, 1, str(speech.settings.data_dir.resolve())) process(41002, 1, str((tmp_path / "another-app").resolve())) process(41003, os.getpid(), str(speech.settings.data_dir.resolve())) assert speech.orphaned_worker_pids(proc_root) == [41001] def test_audio_worker_emits_chunk_progress_for_split_wav(tmp_path: Path, monkeypatch, capsys): wav_path = tmp_path / "audio.wav" with wave.open(str(wav_path), "wb") as handle: handle.setnchannels(1) handle.setsampwidth(2) handle.setframerate(16_000) handle.writeframes(b"\0\0" * 16_000 * 3) monkeypatch.setattr("imagefind.speech.SpeechService.ready", lambda _self: True) monkeypatch.setattr("imagefind.speech.SpeechService._load", lambda _self: object()) def transcribe_samples(_self, samples, _sample_rate, **_kwargs): return [{"text": f"{len(samples)} samples", "start_ms": 0, "end_ms": 500}] monkeypatch.setattr("imagefind.speech.SpeechService._direct_transcribe_samples", transcribe_samples) result = audio_worker.run( SimpleNamespace( data_dir=tmp_path / "data", wav=wav_path, device="CPU", chunk_seconds=1, overlap_seconds=0, cpu_threads=1, ) ) events = [json.loads(line) for line in capsys.readouterr().out.splitlines()] assert result == 0 assert events[0]["event"] == "ready" chunks = [event for event in events if event["event"] == "chunk"] assert len(chunks) == 3 assert [chunk["index"] for chunk in chunks] == [1, 2, 3] assert chunks[-1]["fraction"] == 1 assert events[-1]["event"] == "complete" def test_audio_worker_uses_native_generate_once_per_worker_chunk( tmp_path: Path, monkeypatch, capsys, ): wav_path = tmp_path / "speech.wav" with wave.open(str(wav_path), "wb") as handle: handle.setnchannels(1) handle.setsampwidth(2) handle.setframerate(16_000) handle.writeframes((b"\x00\x20" + b"\x00\xe0") * 8_000) monkeypatch.setattr("imagefind.speech.SpeechService.ready", lambda _self: True) monkeypatch.setattr("imagefind.speech.SpeechService._load", lambda _self: object()) calls: list[dict] = [] def transcribe_samples(_self, _samples, _sample_rate, **kwargs): calls.append(kwargs) return [{"text": "有效中文识别结果", "start_ms": 0, "end_ms": 1000}] monkeypatch.setattr("imagefind.speech.SpeechService._direct_transcribe_samples", transcribe_samples) monkeypatch.setattr( audio_worker, "pcm16_voiced_regions", lambda raw, _rate, **_kwargs: [(0, len(raw) // 2, 1.0)], ) result = audio_worker.run( SimpleNamespace( data_dir=tmp_path / "data", wav=wav_path, device="GPU", chunk_seconds=30, overlap_seconds=0, cpu_threads=1, ) ) events = [json.loads(line) for line in capsys.readouterr().out.splitlines()] chunks = [event for event in events if event["event"] == "chunk"] assert result == 0 assert len(calls) == 1 assert calls[0]["language"] == "zh" assert chunks[0]["segments"] == [ { "text": "有效中文识别结果", "start_ms": 0, "end_ms": 1000, "quality_score": 1.0, "language": "zh", } ] def test_audio_worker_flags_nonempty_but_implausibly_short_gpu_transcript( tmp_path: Path, monkeypatch, ): wav_path = tmp_path / "speech.wav" with wave.open(str(wav_path), "wb") as handle: handle.setnchannels(1) handle.setsampwidth(2) handle.setframerate(16_000) handle.writeframes((b"\x00\x20" + b"\x00\xe0") * 8_000 * 11) monkeypatch.setattr("imagefind.speech.SpeechService.ready", lambda _self: True) monkeypatch.setattr("imagefind.speech.SpeechService._load", lambda _self: object()) monkeypatch.setattr( "imagefind.speech.SpeechService._direct_transcribe_samples", lambda *_args, **_kwargs: [{"text": "And", "start_ms": 0, "end_ms": 11_000}], ) monkeypatch.setattr( audio_worker, "pcm16_voiced_regions", lambda raw, _rate, **_kwargs: [(0, len(raw) // 2, 1.0)], ) with pytest.raises(SpeechStageError) as error: audio_worker.run( SimpleNamespace( data_dir=tmp_path / "data", wav=wav_path, device="GPU", chunk_seconds=30, overlap_seconds=0, cpu_threads=1, start_frame=0, max_chunks=0, ) ) assert error.value.stage == "low_quality_result" assert error.value.details["gpu_text_units"] == 3 assert error.value.details["verification_duration_ms"] == 11_000 def test_audio_worker_uses_native_generate_when_pipeline_text_is_implausibly_short( tmp_path: Path, monkeypatch, capsys, ): wav_path = tmp_path / "speech.wav" with wave.open(str(wav_path), "wb") as handle: handle.setnchannels(1) handle.setsampwidth(2) handle.setframerate(16_000) handle.writeframes((b"\x00\x20" + b"\x00\xe0") * 8_000 * 11) monkeypatch.setattr("imagefind.speech.SpeechService.ready", lambda _self: True) monkeypatch.setattr("imagefind.speech.SpeechService._load", lambda _self: object()) monkeypatch.setattr( "imagefind.speech.SpeechService._transcribe_samples", lambda *_args, **_kwargs: [{"text": "And", "start_ms": 0, "end_ms": 11_000}], ) monkeypatch.setattr( "imagefind.speech.SpeechService._direct_transcribe_samples", lambda *_args, **_kwargs: [ { "text": "The verification keyword is nebula seven two nine", "start_ms": 0, "end_ms": 11_000, } ], ) monkeypatch.setattr( audio_worker, "pcm16_voiced_regions", lambda raw, _rate, **_kwargs: [(0, len(raw) // 2, 1.0)], ) assert audio_worker.run( SimpleNamespace( data_dir=tmp_path / "data", wav=wav_path, device="GPU", chunk_seconds=30, overlap_seconds=0, cpu_threads=1, start_frame=0, max_chunks=0, max_new_tokens=0, language="en", ) ) == 0 events = [json.loads(line) for line in capsys.readouterr().out.splitlines()] chunk = next(event for event in events if event["event"] == "chunk") assert "nebula" in chunk["segments"][0]["text"] def test_audio_worker_and_service_default_to_one_pipeline_sized_chunk( tmp_path: Path, monkeypatch, ): settings = Settings(data_dir=tmp_path / "data") monkeypatch.setattr( sys, "argv", [ "imagefind.audio_worker", "--data-dir", str(settings.data_dir), "--wav", str(tmp_path / "audio.wav"), "--device", "CPU", ], ) arguments = audio_worker._arguments() assert settings.audio_chunk_seconds == 30 assert arguments.chunk_seconds == 30 def test_audio_worker_rejects_medium_on_cpu_before_loading_model(tmp_path: Path): with pytest.raises(SpeechStageError, match="仅支持 Intel GPU") as error: audio_worker.run( SimpleNamespace( data_dir=tmp_path / "data", wav=tmp_path / "unused.wav", device="CPU", chunk_seconds=30, overlap_seconds=0, cpu_threads=1, model_variant="medium", ) ) assert error.value.stage == "device_policy" def test_medium_verification_preflights_memory_before_starting_worker(tmp_path: Path, monkeypatch): settings = Settings(data_dir=tmp_path / "data", audio_model_variant="medium") settings.prepare() audio = settings.models_dir / "audio" / "medium" audio.mkdir(parents=True) for name in ( "config.json", "preprocessor_config.json", "tokenizer_config.json", "tokenizer.json", "openvino_encoder_model.xml", "openvino_encoder_model.bin", "openvino_decoder_model.xml", "openvino_decoder_model.bin", ): (audio / name).write_text("{}", encoding="utf-8") speech = SpeechService(settings) speech.accelerator.gpu_device = "GPU.0" original_read_text = Path.read_text def read_text(path, *args, **kwargs): if Path(path) == Path("/proc/meminfo"): return "MemTotal: 4096000 kB\nMemAvailable: 512000 kB\n" return original_read_text(path, *args, **kwargs) def forbidden_worker(*_args, **_kwargs): raise AssertionError("resource preflight must run before the isolated worker starts") monkeypatch.setattr(Path, "read_text", read_text) monkeypatch.setattr(speech, "_isolated_transcribe_once", forbidden_worker) with pytest.raises(ModelUnavailable, match="可用内存不足"): speech.verify_acceleration_isolated() def test_candidate_variant_verification_does_not_mutate_active_settings(tmp_path: Path, monkeypatch): settings = Settings(data_dir=tmp_path / "data", audio_model_variant="small") settings.prepare() speech = SpeechService(settings) observed: list[str] = [] def verify(candidate: SpeechService, *, stall_seconds=None): observed.append(candidate.settings.audio_model_variant) return {"state": "ready", "stall_seconds": stall_seconds} monkeypatch.setattr(SpeechService, "verify_acceleration_isolated", verify) result = speech.verify_variant_acceleration_isolated("medium", stall_seconds=90) assert result == {"state": "ready", "stall_seconds": 90} assert observed == ["medium"] assert settings.audio_model_variant == "small"