1077 lines
38 KiB
Python
1077 lines
38 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import sys
|
|
import wave
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
from imagefind import audio_worker
|
|
from imagefind.ai import ModelUnavailable
|
|
from imagefind.config import Settings
|
|
from imagefind.database import utcnow
|
|
from imagefind.jobs import JobRetry
|
|
from imagefind.main import create_app
|
|
from imagefind.media import MediaInput
|
|
from imagefind.speech import SPEECH_INDEX_REVISION, SpeechService, SpeechStageError
|
|
from imagefind.speech_quality import aggregate_transcript_quality
|
|
from imagefind.text import fts_query
|
|
|
|
|
|
def test_ending_hallucination_is_risky_twice_or_once_in_weak_speech():
|
|
repeated = aggregate_transcript_quality([
|
|
{"text": "拜拜"},
|
|
{"text": "正常对话"},
|
|
{"text": "拜拜"},
|
|
])
|
|
assert "whole_ending_hallucination" in repeated.flags
|
|
assert ("拜拜", 2) in repeated.repeated_phrases
|
|
|
|
weak = aggregate_transcript_quality([
|
|
{"text": "谢谢大家收看", "speech_ratio": 0.1},
|
|
{"text": "正常对话", "speech_ratio": 0.8},
|
|
])
|
|
assert "whole_ending_hallucination" in weak.flags
|
|
|
|
single = aggregate_transcript_quality([
|
|
{"text": "拜拜", "speech_ratio": 0.8},
|
|
{"text": "正常对话", "speech_ratio": 0.8},
|
|
])
|
|
assert "whole_ending_hallucination" not in single.flags
|
|
|
|
|
|
def _app(tmp_path: Path):
|
|
settings = Settings(data_dir=tmp_path / "data", embedding_backend="hash", upload_reserve_gb=0)
|
|
settings.prepare()
|
|
app = create_app(settings)
|
|
audio = settings.models_dir / "audio"
|
|
audio.mkdir(parents=True)
|
|
for name in (
|
|
"config.json",
|
|
"preprocessor_config.json",
|
|
"tokenizer_config.json",
|
|
"vocab.json",
|
|
"openvino_encoder_model.xml",
|
|
"openvino_encoder_model.bin",
|
|
"openvino_decoder_model.xml",
|
|
"openvino_decoder_model.bin",
|
|
"openvino_decoder_with_past_model.xml",
|
|
"openvino_decoder_with_past_model.bin",
|
|
):
|
|
(audio / name).write_text("{}" if name.endswith(".json") else "model")
|
|
(settings.models_dir / "manifest.json").write_text(
|
|
json.dumps({"version": "v6", "components": {"audio": {"version": "whisper-small-int8"}}})
|
|
)
|
|
service = app.state.services
|
|
now = utcnow()
|
|
media = tmp_path / "movie.mp4"
|
|
media.write_bytes(b"video")
|
|
with service.db.transaction() as conn:
|
|
conn.execute(
|
|
"INSERT INTO sources(id,kind,name,config_json,created_at,updated_at) VALUES(?,?,?,?,?,?)",
|
|
("source", "local", "本地", json.dumps({"path": str(tmp_path)}), now, now),
|
|
)
|
|
conn.execute(
|
|
"INSERT INTO videos(id,source_id,source_key,display_name,location,fingerprint,basic_fingerprint,"
|
|
"visual_model_version,status,available,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?,?,?,?,?)",
|
|
(
|
|
"video",
|
|
"source",
|
|
"movie.mp4",
|
|
"movie.mp4",
|
|
str(media),
|
|
"fingerprint",
|
|
"fingerprint",
|
|
"hash-v1",
|
|
"indexed",
|
|
1,
|
|
now,
|
|
now,
|
|
),
|
|
)
|
|
return app
|
|
|
|
|
|
def test_audio_indexer_writes_timed_fts_entries_and_reconciles(tmp_path: Path, monkeypatch):
|
|
app = _app(tmp_path)
|
|
service = app.state.services
|
|
monkeypatch.setattr(service.media, "input_for", lambda _video: MediaInput("movie.mp4"))
|
|
monkeypatch.setattr(
|
|
service.media,
|
|
"probe",
|
|
lambda _media: {"raw": {"streams": [{"codec_type": "audio"}]}, "duration_ms": 90_000},
|
|
)
|
|
|
|
def transcribe(_media, _probe, progress, **_kwargs):
|
|
progress(0.5, "识别默认音轨 50%")
|
|
return [{"text": "今天讨论本地人工智能", "start_ms": 12_000, "end_ms": 15_500}]
|
|
|
|
monkeypatch.setattr(service.speech, "transcribe", transcribe)
|
|
job_id = service.jobs.enqueue("transcribe_audio", {"video_id": "video"}, dedupe_key="audio:video")
|
|
service.audio_indexer.index(job_id, "video")
|
|
|
|
with service.db.read() as conn:
|
|
entry = conn.execute("SELECT kind,start_ms,end_ms,raw_text FROM text_entries WHERE video_id='video'").fetchone()
|
|
version = conn.execute("SELECT audio_model_version FROM videos WHERE id='video'").fetchone()[0]
|
|
fts = conn.execute("SELECT entry_id FROM text_fts WHERE text_fts MATCH ?", (fts_query("人工智能"),)).fetchone()
|
|
assert dict(entry) == {
|
|
"kind": "audio",
|
|
"start_ms": 12_000,
|
|
"end_ms": 15_500,
|
|
"raw_text": "今天讨论本地人工智能",
|
|
}
|
|
assert version == "whisper-small-int8"
|
|
assert fts is not None
|
|
|
|
service.audio_indexer.index(job_id, "video", quality_repair=True)
|
|
with service.db.read() as conn:
|
|
repair_revision = conn.execute(
|
|
"SELECT audio_quality_repair_revision FROM videos WHERE id='video'"
|
|
).fetchone()[0]
|
|
assert repair_revision == SPEECH_INDEX_REVISION
|
|
|
|
with service.db.transaction() as conn:
|
|
conn.execute("DELETE FROM jobs")
|
|
conn.execute("UPDATE videos SET audio_model_version=NULL WHERE id='video'")
|
|
queued = service.reconcile_ai()
|
|
assert len(queued) == 1
|
|
with service.db.read() as conn:
|
|
job = conn.execute("SELECT kind,payload_json FROM jobs WHERE id=?", (queued[0],)).fetchone()
|
|
assert job["kind"] == "transcribe_audio"
|
|
assert json.loads(job["payload_json"]) == {"video_id": "video"}
|
|
|
|
|
|
def test_repeated_hallucinations_are_saved_but_not_searchable(tmp_path: Path, monkeypatch):
|
|
app = _app(tmp_path)
|
|
service = app.state.services
|
|
monkeypatch.setattr(service.media, "input_for", lambda _video: MediaInput("movie.mp4"))
|
|
monkeypatch.setattr(
|
|
service.media,
|
|
"probe",
|
|
lambda _media: {"raw": {"streams": [{"codec_type": "audio"}]}, "duration_ms": 30_000},
|
|
)
|
|
monkeypatch.setattr(
|
|
service.speech,
|
|
"transcribe",
|
|
lambda *_args, **_kwargs: [
|
|
{"text": "拜拜", "start_ms": index * 1000, "end_ms": index * 1000 + 800}
|
|
for index in range(8)
|
|
],
|
|
)
|
|
job_id = service.jobs.enqueue("transcribe_audio", {"video_id": "video"}, dedupe_key="audio:video")
|
|
service.audio_indexer.index(job_id, "video", quality_repair=True)
|
|
with service.db.read() as conn:
|
|
entries = conn.execute(
|
|
"SELECT count(*) FROM text_entries WHERE video_id='video' AND kind='audio'"
|
|
).fetchone()[0]
|
|
searchable = conn.execute(
|
|
"SELECT count(*) FROM text_fts WHERE entry_id IN "
|
|
"(SELECT id FROM text_entries WHERE video_id='video' AND kind='audio')"
|
|
).fetchone()[0]
|
|
video = conn.execute(
|
|
"SELECT audio_quality_score,audio_quality_flags_json FROM videos WHERE id='video'"
|
|
).fetchone()
|
|
assert entries == 8
|
|
assert searchable == 0
|
|
assert video["audio_quality_score"] < 0.7
|
|
assert "whole_ending_hallucination" in json.loads(video["audio_quality_flags_json"])
|
|
|
|
|
|
def test_audio_indexer_retries_medium_memory_pressure(tmp_path: Path, monkeypatch):
|
|
app = _app(tmp_path)
|
|
service = app.state.services
|
|
monkeypatch.setattr(service.media, "input_for", lambda _video: MediaInput("movie.mp4"))
|
|
monkeypatch.setattr(
|
|
service.media,
|
|
"probe",
|
|
lambda _media: {"raw": {"streams": [{"codec_type": "audio"}]}, "duration_ms": 90_000},
|
|
)
|
|
|
|
def transcribe(*_args, **_kwargs):
|
|
raise ModelUnavailable("Whisper Medium 可用内存不足：至少需要额外 4.0 GiB，且需保留系统内存")
|
|
|
|
monkeypatch.setattr(service.speech, "transcribe", transcribe)
|
|
job_id = service.jobs.enqueue("transcribe_audio", {"video_id": "video"}, dedupe_key="audio:video")
|
|
|
|
with pytest.raises(JobRetry, match="资源释放后自动重试") as retry:
|
|
service.audio_indexer.index(job_id, "video")
|
|
assert retry.value.delay_seconds == 120
|
|
|
|
|
|
def test_reconcile_ai_safely_replaces_historical_locked_index_job(tmp_path: Path):
|
|
app = _app(tmp_path)
|
|
service = app.state.services
|
|
failed_job = service.jobs.enqueue(
|
|
"index_video",
|
|
{"video_id": "video"},
|
|
dedupe_key="index:video",
|
|
)
|
|
with service.db.transaction() as conn:
|
|
conn.execute(
|
|
"UPDATE jobs SET status='failed',error='deadlock detected',finished_at=? WHERE id=?",
|
|
(utcnow(), failed_job),
|
|
)
|
|
conn.execute(
|
|
"UPDATE videos SET indexed_fingerprint=NULL,basic_fingerprint=NULL WHERE id='video'"
|
|
)
|
|
|
|
first = service.reconcile_ai()
|
|
second = service.reconcile_ai()
|
|
|
|
assert len(first) == 1
|
|
assert second == first
|
|
with service.db.read() as conn:
|
|
active = conn.execute(
|
|
"SELECT count(*) FROM jobs WHERE kind='index_video' AND status IN ('queued','running')"
|
|
).fetchone()[0]
|
|
replacement = conn.execute(
|
|
"SELECT payload_json FROM jobs WHERE id=?",
|
|
(first[0],),
|
|
).fetchone()
|
|
assert active == 1
|
|
assert json.loads(replacement["payload_json"])["video_id"] == "video"
|
|
|
|
|
|
def test_speech_service_skips_video_without_audio_stream(tmp_path: Path):
|
|
app = _app(tmp_path)
|
|
speech = app.state.services.speech
|
|
result = speech.transcribe(MediaInput("movie.mp4"), {"raw": {"streams": []}, "duration_ms": 1})
|
|
assert result == []
|
|
|
|
|
|
def test_speech_service_loads_local_openvino_model_without_library_inference(tmp_path: Path):
|
|
app = _app(tmp_path)
|
|
speech = app.state.services.speech
|
|
calls: dict[str, object] = {}
|
|
|
|
class ConfigFactory:
|
|
@staticmethod
|
|
def from_pretrained(path, **kwargs):
|
|
calls["config"] = (path, kwargs)
|
|
return SimpleNamespace(architectures=["WhisperForConditionalGeneration"])
|
|
|
|
class ProcessorFactory:
|
|
@staticmethod
|
|
def from_pretrained(path, **kwargs):
|
|
calls["processor"] = (path, kwargs)
|
|
return SimpleNamespace(tokenizer="tokenizer", feature_extractor="feature-extractor")
|
|
|
|
class ModelFactory:
|
|
class LoadedModel:
|
|
def compile(self):
|
|
calls["compiled"] = True
|
|
|
|
@classmethod
|
|
def from_pretrained(cls, *_args, **_kwargs):
|
|
raise AssertionError("public loader must not run library inference")
|
|
|
|
@classmethod
|
|
def _from_pretrained(cls, path, config, **kwargs):
|
|
calls["model"] = (path, config, kwargs)
|
|
return cls.LoadedModel()
|
|
|
|
def pipeline_factory(task, **kwargs):
|
|
calls["pipeline"] = (task, kwargs)
|
|
return "speech-pipeline"
|
|
|
|
result = speech._build_pipeline(
|
|
"CPU",
|
|
ConfigFactory,
|
|
ProcessorFactory,
|
|
ModelFactory,
|
|
pipeline_factory,
|
|
)
|
|
|
|
assert result == "speech-pipeline"
|
|
model_path, _, model_kwargs = calls["model"]
|
|
assert model_path == speech.model_path.resolve()
|
|
assert model_kwargs["local_files_only"] is True
|
|
assert model_kwargs["compile"] is False
|
|
# Whisper generation needs the regular model wrapper. ``compile_only``
|
|
# can compile the IR successfully while returning only the first word.
|
|
assert model_kwargs["compile_only"] is False
|
|
assert model_kwargs["use_cache"] is True
|
|
assert calls["compiled"] is True
|
|
assert calls["pipeline"][1]["framework"] == "pt"
|
|
|
|
|
|
def test_speech_service_accepts_export_without_optional_decoder_cache(tmp_path: Path):
|
|
app = _app(tmp_path)
|
|
speech = app.state.services.speech
|
|
(speech.model_path / "openvino_decoder_with_past_model.xml").unlink()
|
|
(speech.model_path / "openvino_decoder_with_past_model.bin").unlink()
|
|
calls: dict[str, object] = {}
|
|
|
|
class ConfigFactory:
|
|
@staticmethod
|
|
def from_pretrained(_path, **_kwargs):
|
|
return SimpleNamespace(architectures=["WhisperForConditionalGeneration"])
|
|
|
|
class ProcessorFactory:
|
|
@staticmethod
|
|
def from_pretrained(_path, **_kwargs):
|
|
return SimpleNamespace(tokenizer="tokenizer", feature_extractor="feature-extractor")
|
|
|
|
class ModelFactory:
|
|
@classmethod
|
|
def _from_pretrained(cls, _path, config=None, **kwargs):
|
|
assert config is not None
|
|
calls["model"] = kwargs
|
|
return "openvino-model"
|
|
|
|
result = speech._build_pipeline(
|
|
"CPU",
|
|
ConfigFactory,
|
|
ProcessorFactory,
|
|
ModelFactory,
|
|
lambda _task, **_kwargs: "speech-pipeline",
|
|
)
|
|
|
|
assert speech.ready() is True
|
|
assert result == "speech-pipeline"
|
|
assert calls["model"]["use_cache"] is False
|
|
|
|
|
|
def test_int8_whisper_is_marked_for_fp16_repair_when_intel_gpu_is_available(tmp_path: Path):
|
|
app = _app(tmp_path)
|
|
services = app.state.services
|
|
services.accelerator.gpu_device = "GPU.0"
|
|
manifest_path = services.settings.models_dir / "manifest.json"
|
|
manifest_path.write_text(
|
|
json.dumps(
|
|
{
|
|
"version": "legacy",
|
|
"components": {
|
|
"audio": {"version": "openai/whisper-small-openvino-int8"},
|
|
},
|
|
}
|
|
)
|
|
)
|
|
|
|
health = services.models.component_health()["audio"]
|
|
|
|
assert health["state"] == "outdated"
|
|
assert "FP16" in health["error"]
|
|
assert services.models.runnable_component_versions()["audio"] is None
|
|
repair_job_id = services.queue_missing_ai_runtime()
|
|
with services.db.read() as connection:
|
|
repair_job = connection.execute(
|
|
"SELECT kind,payload_json,dedupe_key FROM jobs WHERE id=?", (repair_job_id,)
|
|
).fetchone()
|
|
assert repair_job["kind"] == "install_models"
|
|
assert json.loads(repair_job["payload_json"]) == {"component": "audio"}
|
|
assert repair_job["dedupe_key"] == "install-model:audio"
|
|
|
|
|
|
def test_legacy_locally_exported_fp16_whisper_is_replaced_by_official_model(tmp_path: Path):
|
|
app = _app(tmp_path)
|
|
services = app.state.services
|
|
manifest_path = services.settings.models_dir / "manifest.json"
|
|
manifest_path.write_text(
|
|
json.dumps(
|
|
{
|
|
"version": "legacy",
|
|
"components": {
|
|
"audio": {"version": "openai/whisper-small-openvino-fp16-v1"},
|
|
},
|
|
}
|
|
)
|
|
)
|
|
|
|
health = services.models.component_health()["audio"]
|
|
|
|
assert Settings(data_dir=tmp_path / "defaults").audio_model == "OpenVINO/whisper-small-fp16-ov"
|
|
assert health["state"] == "outdated"
|
|
assert "OpenVINO 官方" in health["error"]
|
|
assert services.models.runnable_component_versions()["audio"] is None
|
|
|
|
|
|
def test_speech_service_uses_quality_profile_beams_for_whisper_generation(tmp_path: Path, monkeypatch):
|
|
app = _app(tmp_path)
|
|
speech = app.state.services.speech
|
|
calls: dict[str, object] = {}
|
|
|
|
def engine(inputs, **kwargs):
|
|
calls["inputs"] = inputs
|
|
calls["kwargs"] = kwargs
|
|
return {"chunks": []}
|
|
|
|
monkeypatch.setattr(speech, "_load", lambda: engine)
|
|
|
|
assert speech._transcribe_samples([0.0], 16_000) == []
|
|
generation = calls["kwargs"]["generate_kwargs"]
|
|
assert generation == {
|
|
"task": "transcribe",
|
|
"num_beams": 5,
|
|
"max_new_tokens": 192,
|
|
"do_sample": False,
|
|
"use_cache": True,
|
|
"early_stopping": True,
|
|
}
|
|
assert calls["kwargs"]["return_timestamps"] is True
|
|
|
|
|
|
def test_speech_service_can_retry_without_timestamp_generation(tmp_path: Path, monkeypatch):
|
|
settings = Settings(data_dir=tmp_path / "data", embedding_backend="hash")
|
|
settings.prepare()
|
|
speech = SpeechService(settings)
|
|
calls: list[dict] = []
|
|
|
|
def engine(_inputs, **kwargs):
|
|
calls.append(kwargs)
|
|
return {"text": "兼容解码返回文字", "chunks": []}
|
|
|
|
monkeypatch.setattr(speech, "_load", lambda: engine)
|
|
|
|
assert speech._transcribe_samples(
|
|
[0.1] * 16_000,
|
|
16_000,
|
|
return_timestamps=False,
|
|
) == [{"text": "兼容解码返回文字", "start_ms": 0, "end_ms": 1000}]
|
|
assert calls[0]["return_timestamps"] is False
|
|
|
|
|
|
def test_speech_service_native_generate_fallback_returns_seekable_chunk(tmp_path: Path, monkeypatch):
|
|
app = _app(tmp_path)
|
|
speech = app.state.services.speech
|
|
calls: dict[str, object] = {}
|
|
|
|
class Processor:
|
|
def __call__(self, samples, **kwargs):
|
|
calls["processor"] = (samples, kwargs)
|
|
return SimpleNamespace(input_features="features", attention_mask="mask")
|
|
|
|
@staticmethod
|
|
def batch_decode(tokens, **kwargs):
|
|
calls["decode"] = (tokens, kwargs)
|
|
return ["The verification keyword is nebula seven two nine."]
|
|
|
|
class Model:
|
|
@staticmethod
|
|
def generate(**kwargs):
|
|
calls["generate"] = kwargs
|
|
return [[1, 2, 3]]
|
|
|
|
speech._model = Model()
|
|
speech._processor = Processor()
|
|
speech._pipeline_device = "GPU.0"
|
|
monkeypatch.setattr(speech, "_load", lambda: object())
|
|
|
|
result = speech._direct_transcribe_samples([0.1] * 32_000, 16_000)
|
|
|
|
assert result == [
|
|
{
|
|
"text": "The verification keyword is nebula seven two nine.",
|
|
"start_ms": 0,
|
|
"end_ms": 2000,
|
|
}
|
|
]
|
|
assert calls["generate"] == {
|
|
"input_features": "features",
|
|
"attention_mask": "mask",
|
|
"task": "transcribe",
|
|
# Intel GPU uses the stateful decoder's deterministic greedy path;
|
|
# beam expansion can stall before the first segment is emitted.
|
|
"num_beams": 1,
|
|
"max_new_tokens": 96,
|
|
"do_sample": False,
|
|
"use_cache": True,
|
|
"early_stopping": True,
|
|
"return_timestamps": False,
|
|
}
|
|
assert calls["decode"][1] == {"skip_special_tokens": True}
|
|
assert calls["processor"][1]["return_attention_mask"] is True
|
|
|
|
|
|
def test_speech_full_text_fallback_uses_real_chunk_duration(tmp_path: Path, monkeypatch):
|
|
app = _app(tmp_path)
|
|
speech = app.state.services.speech
|
|
monkeypatch.setattr(speech, "_load", lambda: lambda *_args, **_kwargs: {"text": "完整文本"})
|
|
|
|
assert speech._transcribe_samples([0.1] * 32_000, 16_000) == [{"text": "完整文本", "start_ms": 0, "end_ms": 2000}]
|
|
|
|
|
|
def test_speech_accepts_segment_timestamp_variants_and_rejects_non_silent_empty(
|
|
tmp_path: Path,
|
|
monkeypatch,
|
|
):
|
|
app = _app(tmp_path)
|
|
speech = app.state.services.speech
|
|
monkeypatch.setattr(
|
|
speech,
|
|
"_load",
|
|
lambda: (
|
|
lambda *_args, **_kwargs: {
|
|
"segments": [
|
|
{"text": "第一段", "timestamps": {"start": 0.25, "end": 0.75}},
|
|
{"text": "第二段", "start_ms": 900, "end_ms": 1300},
|
|
]
|
|
}
|
|
),
|
|
)
|
|
assert speech._transcribe_samples([0.1] * 32_000, 16_000) == [
|
|
{"text": "第一段", "start_ms": 250, "end_ms": 750},
|
|
{"text": "第二段", "start_ms": 900, "end_ms": 1300},
|
|
]
|
|
|
|
monkeypatch.setattr(speech, "_load", lambda: lambda *_args, **_kwargs: {"chunks": []})
|
|
with pytest.raises(RuntimeError, match="未返回文字或时间片段"):
|
|
speech._transcribe_samples([0.1] * 16_000, 16_000)
|
|
assert (
|
|
speech._transcribe_samples(
|
|
[0.1] * 16_000,
|
|
16_000,
|
|
allow_non_silent_empty=True,
|
|
)
|
|
== []
|
|
)
|
|
|
|
|
|
def test_speech_service_recognizes_modern_stateful_decoder_cache(tmp_path: Path, monkeypatch):
|
|
app = _app(tmp_path)
|
|
speech = app.state.services.speech
|
|
(speech.model_path / "openvino_decoder_with_past_model.xml").unlink()
|
|
(speech.model_path / "openvino_decoder_with_past_model.bin").unlink()
|
|
(speech.model_path / "openvino_decoder_model.xml").write_text(
|
|
'<net><layers><layer id="0" type="ReadValue" version="opset6"/></layers></net>'
|
|
)
|
|
speech.accelerator.gpu_device = "GPU.0"
|
|
calls = []
|
|
|
|
def run_once(_wav_path, requested, _stall_seconds, _progress, *_args, **_kwargs):
|
|
calls.append(requested)
|
|
return [{"text": "gpu", "start_ms": 0, "end_ms": 1000}]
|
|
|
|
monkeypatch.setattr(speech, "_isolated_transcribe_once", run_once)
|
|
|
|
assert speech.has_decoder_cache() is True
|
|
result = speech._isolated_transcribe(tmp_path / "audio.wav", lambda *_args: None)
|
|
|
|
assert result[0]["text"] == "gpu"
|
|
assert calls == ["GPU.0"]
|
|
health = app.state.services.models.component_health()["audio"]
|
|
assert health == {"state": "ready", "error": None}
|
|
|
|
|
|
def test_isolated_audio_rejects_gpu_when_decoder_cache_is_missing(tmp_path: Path, monkeypatch):
|
|
app = _app(tmp_path)
|
|
speech = app.state.services.speech
|
|
(speech.model_path / "openvino_decoder_with_past_model.xml").unlink()
|
|
(speech.model_path / "openvino_decoder_with_past_model.bin").unlink()
|
|
speech.accelerator.gpu_device = "GPU.0"
|
|
calls = []
|
|
|
|
def run_once(_wav_path, requested, _stall_seconds, _progress, *_args, **_kwargs):
|
|
calls.append(requested)
|
|
return [{"text": "cpu fallback", "start_ms": 0, "end_ms": 1000}]
|
|
|
|
monkeypatch.setattr(speech, "_isolated_transcribe_once", run_once)
|
|
with pytest.raises(Exception, match="stateful decoder"):
|
|
speech._isolated_transcribe(tmp_path / "audio.wav", lambda *_args: None)
|
|
|
|
assert calls == []
|
|
health = app.state.services.models.component_health()["audio"]
|
|
assert health["state"] == "ready"
|
|
assert health["gpu_eligible"] is False
|
|
assert "decoder_with_past" in health["warning"]
|
|
|
|
|
|
def test_isolated_audio_never_retries_a_gpu_stall_as_full_cpu_job(tmp_path: Path, monkeypatch):
|
|
app = _app(tmp_path)
|
|
speech = app.state.services.speech
|
|
speech.accelerator.gpu_device = "GPU.0"
|
|
calls = []
|
|
|
|
def run_once(_wav_path, requested, stall_seconds, _progress, *_args, **_kwargs):
|
|
calls.append((requested, stall_seconds))
|
|
if requested.startswith("GPU"):
|
|
raise TimeoutError("gpu stalled")
|
|
return [{"text": "cpu retry", "start_ms": 0, "end_ms": 1000}]
|
|
|
|
monkeypatch.setattr(speech, "_isolated_transcribe_once", run_once)
|
|
with pytest.raises(TimeoutError, match="gpu stalled"):
|
|
speech._isolated_transcribe(tmp_path / "audio.wav", lambda *_args: None)
|
|
|
|
assert calls == [("GPU.0", speech.settings.audio_gpu_stall_seconds)]
|
|
audio = speech.accelerator.status()["components"]["audio"]
|
|
assert audio["state"] == "job_fallback"
|
|
assert audio["failure_stage"] == "inference_stall"
|
|
assert audio["fallback_scope"] == "job"
|
|
assert speech.accelerator.device_for("audio") == "GPU.0"
|
|
|
|
|
|
def test_isolated_audio_verifies_empty_gpu_generation_on_cpu(tmp_path: Path, monkeypatch):
|
|
app = _app(tmp_path)
|
|
speech = app.state.services.speech
|
|
speech.accelerator.gpu_device = "GPU.0"
|
|
calls = []
|
|
|
|
def run_once(_wav_path, requested, _stall_seconds, _progress, *_args, **_kwargs):
|
|
calls.append(requested)
|
|
if requested.startswith("GPU"):
|
|
from imagefind.speech import SpeechStageError
|
|
|
|
raise SpeechStageError("empty_result", "GPU returned no text")
|
|
return []
|
|
|
|
messages = []
|
|
monkeypatch.setattr(speech, "_isolated_transcribe_once", run_once)
|
|
result = speech._isolated_transcribe(tmp_path / "audio.wav", lambda _, message: messages.append(message))
|
|
|
|
assert result == []
|
|
assert calls == ["GPU.0", "CPU"]
|
|
assert any("复核" in message for message in messages)
|
|
assert any("均未检测到" in message for message in messages)
|
|
# Content-specific no-speech must not permanently disable a healthy GPU.
|
|
audio = speech.accelerator.status()["components"]["audio"]
|
|
assert audio["state"] == "ready"
|
|
assert audio["actual_device"] == "GPU.0"
|
|
|
|
|
|
def test_isolated_audio_marks_gpu_result_untrusted_when_bounded_verifier_finds_speech(
|
|
tmp_path: Path,
|
|
monkeypatch,
|
|
):
|
|
app = _app(tmp_path)
|
|
speech = app.state.services.speech
|
|
speech.accelerator.gpu_device = "GPU.0"
|
|
calls = []
|
|
|
|
def run_once(_wav_path, requested, _stall_seconds, _progress, *_args, **kwargs):
|
|
calls.append((requested, kwargs.get("max_chunks", 0), kwargs.get("verification", False)))
|
|
if requested.startswith("GPU"):
|
|
from imagefind.speech import SpeechStageError
|
|
|
|
raise SpeechStageError(
|
|
"empty_result",
|
|
"GPU returned no text",
|
|
{"verification_start_frame": 32_000},
|
|
)
|
|
if kwargs.get("verification"):
|
|
return [{"text": "cpu sample", "start_ms": 2000, "end_ms": 3000}]
|
|
return [{"text": "cpu full", "start_ms": 0, "end_ms": 1000}]
|
|
|
|
monkeypatch.setattr(speech, "_isolated_transcribe_once", run_once)
|
|
|
|
with pytest.raises(SpeechStageError, match="未通过短片段复核") as error:
|
|
speech._isolated_transcribe(tmp_path / "audio.wav", lambda *_args: None)
|
|
|
|
assert error.value.stage == "gpu_result_untrusted"
|
|
assert calls == [("GPU.0", 0, False), ("CPU", 1, True)]
|
|
audio = speech.accelerator.status()["components"]["audio"]
|
|
assert audio["state"] == "ready"
|
|
assert audio["failure_stage"] is None
|
|
|
|
|
|
def test_isolated_audio_marks_gpu_result_untrusted_when_cpu_sample_is_materially_richer(
|
|
tmp_path: Path,
|
|
monkeypatch,
|
|
):
|
|
app = _app(tmp_path)
|
|
speech = app.state.services.speech
|
|
speech.accelerator.gpu_device = "GPU.0"
|
|
candidate = [{"text": "And", "start_ms": 0, "end_ms": 11_000}]
|
|
calls = []
|
|
|
|
def run_once(_wav_path, requested, _stall_seconds, _progress, *_args, **kwargs):
|
|
calls.append((requested, kwargs.get("max_chunks", 0), kwargs.get("verification", False)))
|
|
if requested.startswith("GPU"):
|
|
raise SpeechStageError(
|
|
"low_quality_result",
|
|
"GPU result was too short",
|
|
{
|
|
"verification_start_frame": 0,
|
|
"gpu_text_units": 3,
|
|
"candidate_segments": candidate,
|
|
},
|
|
)
|
|
if kwargs.get("verification"):
|
|
return [
|
|
{
|
|
"text": "Ask not what your country can do for you",
|
|
"start_ms": 0,
|
|
"end_ms": 5000,
|
|
}
|
|
]
|
|
return [{"text": "complete cpu transcript", "start_ms": 0, "end_ms": 11_000}]
|
|
|
|
monkeypatch.setattr(speech, "_isolated_transcribe_once", run_once)
|
|
|
|
with pytest.raises(SpeechStageError, match="未通过短片段复核") as error:
|
|
speech._isolated_transcribe(tmp_path / "audio.wav", lambda *_args: None)
|
|
|
|
assert error.value.stage == "gpu_result_untrusted"
|
|
assert calls == [("GPU.0", 0, False), ("CPU", 1, True)]
|
|
audio = speech.accelerator.status()["components"]["audio"]
|
|
assert audio["state"] == "ready"
|
|
assert audio["failure_stage"] is None
|
|
|
|
|
|
def test_isolated_audio_keeps_short_gpu_text_when_cpu_is_not_materially_richer(
|
|
tmp_path: Path,
|
|
monkeypatch,
|
|
):
|
|
app = _app(tmp_path)
|
|
speech = app.state.services.speech
|
|
speech.accelerator.gpu_device = "GPU.0"
|
|
candidate = [{"text": "And", "start_ms": 0, "end_ms": 11_000}]
|
|
calls = []
|
|
|
|
def run_once(_wav_path, requested, _stall_seconds, _progress, *_args, **kwargs):
|
|
calls.append((requested, kwargs.get("verification", False)))
|
|
if requested.startswith("GPU"):
|
|
raise SpeechStageError(
|
|
"low_quality_result",
|
|
"GPU result was short",
|
|
{
|
|
"verification_start_frame": 0,
|
|
"gpu_text_units": 3,
|
|
"candidate_segments": candidate,
|
|
},
|
|
)
|
|
return [{"text": "And", "start_ms": 0, "end_ms": 11_000}]
|
|
|
|
monkeypatch.setattr(speech, "_isolated_transcribe_once", run_once)
|
|
|
|
result = speech._isolated_transcribe(tmp_path / "audio.wav", lambda *_args: None)
|
|
|
|
assert result == candidate
|
|
assert calls == [("GPU.0", False), ("CPU", True)]
|
|
audio = speech.accelerator.status()["components"]["audio"]
|
|
assert audio["state"] == "ready"
|
|
assert audio["actual_device"] == "GPU.0"
|
|
|
|
|
|
def test_audio_acceleration_verification_releases_pipeline_but_preserves_status(
|
|
tmp_path: Path,
|
|
monkeypatch,
|
|
):
|
|
app = _app(tmp_path)
|
|
speech = app.state.services.speech
|
|
speech.accelerator.gpu_device = "GPU.0"
|
|
monkeypatch.setitem(
|
|
sys.modules,
|
|
"numpy",
|
|
SimpleNamespace(float32="float32", zeros=lambda *_args, **_kwargs: [0.0]),
|
|
)
|
|
|
|
def verify_sample(_samples, _sample_rate):
|
|
speech._pipeline = object()
|
|
speech._pipeline_device = "GPU.0"
|
|
speech.accelerator.mark_ready("audio", "GPU.0", ["GPU.0"])
|
|
return []
|
|
|
|
monkeypatch.setattr(speech, "_transcribe_samples", verify_sample)
|
|
|
|
status = speech.verify_acceleration()
|
|
|
|
assert status["state"] == "ready"
|
|
assert status["actual_device"] == "GPU.0"
|
|
assert speech._pipeline is None
|
|
assert speech._pipeline_device is None
|
|
assert speech.accelerator.status()["components"]["audio"]["state"] == "ready"
|
|
|
|
|
|
def test_orphaned_audio_worker_discovery_is_scoped_to_data_dir_and_parent(tmp_path: Path):
|
|
app = _app(tmp_path)
|
|
speech = app.state.services.speech
|
|
proc_root = tmp_path / "proc"
|
|
|
|
def process(pid: int, parent: int, data_dir: str) -> None:
|
|
root = proc_root / str(pid)
|
|
root.mkdir(parents=True)
|
|
(root / "status").write_text(f"PPid:\t{parent}\nUid:\t{os.geteuid()} 0 0 0\n")
|
|
(root / "cmdline").write_bytes(
|
|
b"python\0-m\0imagefind.audio_worker\0--data-dir\0" + data_dir.encode() + b"\0--wav\0audio.wav\0"
|
|
)
|
|
|
|
process(41001, 1, str(speech.settings.data_dir.resolve()))
|
|
process(41002, 1, str((tmp_path / "another-app").resolve()))
|
|
process(41003, os.getpid(), str(speech.settings.data_dir.resolve()))
|
|
|
|
assert speech.orphaned_worker_pids(proc_root) == [41001]
|
|
|
|
|
|
def test_audio_worker_emits_chunk_progress_for_split_wav(tmp_path: Path, monkeypatch, capsys):
|
|
wav_path = tmp_path / "audio.wav"
|
|
with wave.open(str(wav_path), "wb") as handle:
|
|
handle.setnchannels(1)
|
|
handle.setsampwidth(2)
|
|
handle.setframerate(16_000)
|
|
handle.writeframes(b"\0\0" * 16_000 * 3)
|
|
|
|
monkeypatch.setattr("imagefind.speech.SpeechService.ready", lambda _self: True)
|
|
monkeypatch.setattr("imagefind.speech.SpeechService._load", lambda _self: object())
|
|
|
|
def transcribe_samples(_self, samples, _sample_rate, **_kwargs):
|
|
return [{"text": f"{len(samples)} samples", "start_ms": 0, "end_ms": 500}]
|
|
|
|
monkeypatch.setattr("imagefind.speech.SpeechService._direct_transcribe_samples", transcribe_samples)
|
|
|
|
result = audio_worker.run(
|
|
SimpleNamespace(
|
|
data_dir=tmp_path / "data",
|
|
wav=wav_path,
|
|
device="CPU",
|
|
chunk_seconds=1,
|
|
overlap_seconds=0,
|
|
cpu_threads=1,
|
|
)
|
|
)
|
|
|
|
events = [json.loads(line) for line in capsys.readouterr().out.splitlines()]
|
|
assert result == 0
|
|
assert events[0]["event"] == "ready"
|
|
chunks = [event for event in events if event["event"] == "chunk"]
|
|
assert len(chunks) == 3
|
|
assert [chunk["index"] for chunk in chunks] == [1, 2, 3]
|
|
assert chunks[-1]["fraction"] == 1
|
|
assert events[-1]["event"] == "complete"
|
|
|
|
|
|
def test_audio_worker_uses_native_generate_once_per_worker_chunk(
|
|
tmp_path: Path,
|
|
monkeypatch,
|
|
capsys,
|
|
):
|
|
wav_path = tmp_path / "speech.wav"
|
|
with wave.open(str(wav_path), "wb") as handle:
|
|
handle.setnchannels(1)
|
|
handle.setsampwidth(2)
|
|
handle.setframerate(16_000)
|
|
handle.writeframes((b"\x00\x20" + b"\x00\xe0") * 8_000)
|
|
|
|
monkeypatch.setattr("imagefind.speech.SpeechService.ready", lambda _self: True)
|
|
monkeypatch.setattr("imagefind.speech.SpeechService._load", lambda _self: object())
|
|
calls: list[dict] = []
|
|
|
|
def transcribe_samples(_self, _samples, _sample_rate, **kwargs):
|
|
calls.append(kwargs)
|
|
return [{"text": "有效中文识别结果", "start_ms": 0, "end_ms": 1000}]
|
|
|
|
monkeypatch.setattr("imagefind.speech.SpeechService._direct_transcribe_samples", transcribe_samples)
|
|
monkeypatch.setattr(
|
|
audio_worker,
|
|
"pcm16_voiced_regions",
|
|
lambda raw, _rate, **_kwargs: [(0, len(raw) // 2, 1.0)],
|
|
)
|
|
|
|
result = audio_worker.run(
|
|
SimpleNamespace(
|
|
data_dir=tmp_path / "data",
|
|
wav=wav_path,
|
|
device="GPU",
|
|
chunk_seconds=30,
|
|
overlap_seconds=0,
|
|
cpu_threads=1,
|
|
)
|
|
)
|
|
|
|
events = [json.loads(line) for line in capsys.readouterr().out.splitlines()]
|
|
chunks = [event for event in events if event["event"] == "chunk"]
|
|
assert result == 0
|
|
assert len(calls) == 1
|
|
assert calls[0]["language"] == "zh"
|
|
assert chunks[0]["segments"] == [
|
|
{
|
|
"text": "有效中文识别结果",
|
|
"start_ms": 0,
|
|
"end_ms": 1000,
|
|
"quality_score": 1.0,
|
|
"language": "zh",
|
|
}
|
|
]
|
|
|
|
|
|
def test_audio_worker_flags_nonempty_but_implausibly_short_gpu_transcript(
|
|
tmp_path: Path,
|
|
monkeypatch,
|
|
):
|
|
wav_path = tmp_path / "speech.wav"
|
|
with wave.open(str(wav_path), "wb") as handle:
|
|
handle.setnchannels(1)
|
|
handle.setsampwidth(2)
|
|
handle.setframerate(16_000)
|
|
handle.writeframes((b"\x00\x20" + b"\x00\xe0") * 8_000 * 11)
|
|
|
|
monkeypatch.setattr("imagefind.speech.SpeechService.ready", lambda _self: True)
|
|
monkeypatch.setattr("imagefind.speech.SpeechService._load", lambda _self: object())
|
|
monkeypatch.setattr(
|
|
"imagefind.speech.SpeechService._direct_transcribe_samples",
|
|
lambda *_args, **_kwargs: [{"text": "And", "start_ms": 0, "end_ms": 11_000}],
|
|
)
|
|
monkeypatch.setattr(
|
|
audio_worker,
|
|
"pcm16_voiced_regions",
|
|
lambda raw, _rate, **_kwargs: [(0, len(raw) // 2, 1.0)],
|
|
)
|
|
|
|
with pytest.raises(SpeechStageError) as error:
|
|
audio_worker.run(
|
|
SimpleNamespace(
|
|
data_dir=tmp_path / "data",
|
|
wav=wav_path,
|
|
device="GPU",
|
|
chunk_seconds=30,
|
|
overlap_seconds=0,
|
|
cpu_threads=1,
|
|
start_frame=0,
|
|
max_chunks=0,
|
|
)
|
|
)
|
|
|
|
assert error.value.stage == "low_quality_result"
|
|
assert error.value.details["gpu_text_units"] == 3
|
|
assert error.value.details["verification_duration_ms"] == 11_000
|
|
|
|
|
|
def test_audio_worker_uses_native_generate_when_pipeline_text_is_implausibly_short(
|
|
tmp_path: Path,
|
|
monkeypatch,
|
|
capsys,
|
|
):
|
|
wav_path = tmp_path / "speech.wav"
|
|
with wave.open(str(wav_path), "wb") as handle:
|
|
handle.setnchannels(1)
|
|
handle.setsampwidth(2)
|
|
handle.setframerate(16_000)
|
|
handle.writeframes((b"\x00\x20" + b"\x00\xe0") * 8_000 * 11)
|
|
|
|
monkeypatch.setattr("imagefind.speech.SpeechService.ready", lambda _self: True)
|
|
monkeypatch.setattr("imagefind.speech.SpeechService._load", lambda _self: object())
|
|
monkeypatch.setattr(
|
|
"imagefind.speech.SpeechService._transcribe_samples",
|
|
lambda *_args, **_kwargs: [{"text": "And", "start_ms": 0, "end_ms": 11_000}],
|
|
)
|
|
monkeypatch.setattr(
|
|
"imagefind.speech.SpeechService._direct_transcribe_samples",
|
|
lambda *_args, **_kwargs: [
|
|
{
|
|
"text": "The verification keyword is nebula seven two nine",
|
|
"start_ms": 0,
|
|
"end_ms": 11_000,
|
|
}
|
|
],
|
|
)
|
|
monkeypatch.setattr(
|
|
audio_worker,
|
|
"pcm16_voiced_regions",
|
|
lambda raw, _rate, **_kwargs: [(0, len(raw) // 2, 1.0)],
|
|
)
|
|
|
|
assert audio_worker.run(
|
|
SimpleNamespace(
|
|
data_dir=tmp_path / "data",
|
|
wav=wav_path,
|
|
device="GPU",
|
|
chunk_seconds=30,
|
|
overlap_seconds=0,
|
|
cpu_threads=1,
|
|
start_frame=0,
|
|
max_chunks=0,
|
|
max_new_tokens=0,
|
|
language="en",
|
|
)
|
|
) == 0
|
|
|
|
events = [json.loads(line) for line in capsys.readouterr().out.splitlines()]
|
|
chunk = next(event for event in events if event["event"] == "chunk")
|
|
assert "nebula" in chunk["segments"][0]["text"]
|
|
|
|
|
|
def test_audio_worker_and_service_default_to_one_pipeline_sized_chunk(
|
|
tmp_path: Path,
|
|
monkeypatch,
|
|
):
|
|
settings = Settings(data_dir=tmp_path / "data")
|
|
monkeypatch.setattr(
|
|
sys,
|
|
"argv",
|
|
[
|
|
"imagefind.audio_worker",
|
|
"--data-dir",
|
|
str(settings.data_dir),
|
|
"--wav",
|
|
str(tmp_path / "audio.wav"),
|
|
"--device",
|
|
"CPU",
|
|
],
|
|
)
|
|
arguments = audio_worker._arguments()
|
|
|
|
assert settings.audio_chunk_seconds == 30
|
|
assert arguments.chunk_seconds == 30
|
|
|
|
|
|
def test_audio_worker_rejects_medium_on_cpu_before_loading_model(tmp_path: Path):
|
|
with pytest.raises(SpeechStageError, match="仅支持 Intel GPU") as error:
|
|
audio_worker.run(
|
|
SimpleNamespace(
|
|
data_dir=tmp_path / "data",
|
|
wav=tmp_path / "unused.wav",
|
|
device="CPU",
|
|
chunk_seconds=30,
|
|
overlap_seconds=0,
|
|
cpu_threads=1,
|
|
model_variant="medium",
|
|
)
|
|
)
|
|
assert error.value.stage == "device_policy"
|
|
|
|
|
|
def test_medium_verification_preflights_memory_before_starting_worker(tmp_path: Path, monkeypatch):
|
|
settings = Settings(data_dir=tmp_path / "data", audio_model_variant="medium")
|
|
settings.prepare()
|
|
audio = settings.models_dir / "audio" / "medium"
|
|
audio.mkdir(parents=True)
|
|
for name in (
|
|
"config.json",
|
|
"preprocessor_config.json",
|
|
"tokenizer_config.json",
|
|
"tokenizer.json",
|
|
"openvino_encoder_model.xml",
|
|
"openvino_encoder_model.bin",
|
|
"openvino_decoder_model.xml",
|
|
"openvino_decoder_model.bin",
|
|
):
|
|
(audio / name).write_text("{}", encoding="utf-8")
|
|
speech = SpeechService(settings)
|
|
speech.accelerator.gpu_device = "GPU.0"
|
|
|
|
original_read_text = Path.read_text
|
|
|
|
def read_text(path, *args, **kwargs):
|
|
if Path(path) == Path("/proc/meminfo"):
|
|
return "MemTotal: 4096000 kB\nMemAvailable: 512000 kB\n"
|
|
return original_read_text(path, *args, **kwargs)
|
|
|
|
def forbidden_worker(*_args, **_kwargs):
|
|
raise AssertionError("resource preflight must run before the isolated worker starts")
|
|
|
|
monkeypatch.setattr(Path, "read_text", read_text)
|
|
monkeypatch.setattr(speech, "_isolated_transcribe_once", forbidden_worker)
|
|
|
|
with pytest.raises(ModelUnavailable, match="可用内存不足"):
|
|
speech.verify_acceleration_isolated()
|
|
|
|
|
|
def test_candidate_variant_verification_does_not_mutate_active_settings(tmp_path: Path, monkeypatch):
|
|
settings = Settings(data_dir=tmp_path / "data", audio_model_variant="small")
|
|
settings.prepare()
|
|
speech = SpeechService(settings)
|
|
observed: list[str] = []
|
|
|
|
def verify(candidate: SpeechService, *, stall_seconds=None):
|
|
observed.append(candidate.settings.audio_model_variant)
|
|
return {"state": "ready", "stall_seconds": stall_seconds}
|
|
|
|
monkeypatch.setattr(SpeechService, "verify_acceleration_isolated", verify)
|
|
|
|
result = speech.verify_variant_acceleration_isolated("medium", stall_seconds=90)
|
|
|
|
assert result == {"state": "ready", "stall_seconds": 90}
|
|
assert observed == ["medium"]
|
|
assert settings.audio_model_variant == "small"
|