from __future__ import annotations import logging import os import time from datetime import UTC, datetime, timedelta from pathlib import Path from threading import Lock from .config import Settings logger = logging.getLogger(__name__) COMPONENTS = ("visual", "ocr", "faces", "audio") GPU_MODES = { "low": {"hint": "LATENCY", "streams": 1, "batch_size": 1}, "balanced": {"hint": "THROUGHPUT", "streams": 2, "batch_size": 2}, "throughput": {"hint": "THROUGHPUT", "streams": 3, "batch_size": 4}, } AUDIO_CIRCUIT_FAILURES = 3 AUDIO_CIRCUIT_COOLDOWN_SECONDS = 600 class AcceleratorService: """Detect and track the OpenVINO device used by each AI component. ImageFind uses explicit GPU compilation instead of OpenVINO AUTO so the settings page can truthfully report where inference ran. A failed GPU component is pinned to a bounded CPU fallback until it is reset. """ def __init__(self, settings: Settings): self.settings = settings self._lock = Lock() self._runtime_gpu_mode: str | None = None self._resource_pressure_samples = 0 self._resource_recovery_samples = 0 self._gpu_busy_previous: dict[str, tuple[int, float]] = {} self.openvino = False self.devices: list[str] = [] self.device_names: dict[str, str] = {} self.gpu_device: str | None = None self.render_nodes: list[str] = [] self.render_accessible = False self.unavailable_reason = "" self._components = { name: { "state": "not_loaded", "device": None, "requested_device": None, "actual_device": None, "execution_devices": [], "failure_stage": None, "fallback_reason": None, "fallback_scope": None, "circuit_state": "closed", "failure_count": 0, "retry_at": None, "last_verified_at": None, } for name in COMPONENTS } self.refresh() def refresh(self) -> None: nodes = sorted(Path("/dev/dri").glob("renderD*")) self.render_nodes = [str(path) for path in nodes] self.render_accessible = any(os.access(path, os.R_OK | os.W_OK) for path in nodes) try: import openvino as ov core = ov.Core() devices = list(core.available_devices) names: dict[str, str] = {} for device in devices: try: names[device] = str(core.get_property(device, "FULL_DEVICE_NAME")) except Exception: names[device] = device self.openvino = True self.devices = devices self.device_names = names self.gpu_device = next((device for device in devices if device.upper().startswith("GPU")), None) except ModuleNotFoundError: logger.info("OpenVINO runtime is not installed; AI acceleration will use CPU fallbacks") self.openvino = False self.devices = [] self.device_names = {} self.gpu_device = None except Exception: logger.warning("OpenVINO device discovery failed", exc_info=True) self.openvino = False self.devices = [] self.device_names = {} self.gpu_device = None if self.gpu_device: self.unavailable_reason = "" elif not self.openvino: self.unavailable_reason = "OpenVINO 运行时不可用" elif not nodes: self.unavailable_reason = "未检测到 Intel render 设备" elif not self.render_accessible: self.unavailable_reason = "Intel render 设备权限不足" else: self.unavailable_reason = "OpenVINO 未发现 Intel GPU，请检查核显驱动" logger.info( "AI accelerator devices=%s gpu=%s render_accessible=%s", self.devices, self.gpu_device or "none", self.render_accessible, ) def device_for(self, component: str) -> str: with self._lock: state = self._components[component] if state["state"] == "fallback": retry_at = state.get("retry_at") if component == "audio" and retry_at and self.gpu_device: try: retry_due = datetime.fromisoformat(str(retry_at)) <= datetime.now(UTC) except ValueError: retry_due = True if retry_due: state.update( state="probing_gpu", device=self.gpu_device, requested_device=self.gpu_device, actual_device=None, execution_devices=[], fallback_scope="component", circuit_state="half_open", retry_at=None, ) return self.gpu_device return "CPU" return self.gpu_device or "CPU" def ov_config(self, device: str) -> dict[str, object]: if device.upper().startswith("CPU"): return { "INFERENCE_NUM_THREADS": self.settings.ai_cpu_threads, "PERFORMANCE_HINT": "LATENCY", } cache_dir = self.settings.runtime_dir / "openvino-cache" cache_dir.mkdir(parents=True, exist_ok=True) mode = self.effective_gpu_mode() profile = GPU_MODES[mode] return { "PERFORMANCE_HINT": profile["hint"], "NUM_STREAMS": str(profile["streams"]), "CACHE_DIR": str(cache_dir), "INFERENCE_PRECISION_HINT": "f32", } def gpu_profile(self) -> dict[str, object]: selected = self.settings.ai_gpu_mode if self.settings.ai_gpu_mode in GPU_MODES else "balanced" mode = self.effective_gpu_mode() return {"mode": mode, "selected_mode": selected, "degraded": mode != selected, **GPU_MODES[mode]} def effective_gpu_mode(self) -> str: selected = self.settings.ai_gpu_mode if self.settings.ai_gpu_mode in GPU_MODES else "balanced" with self._lock: return self._runtime_gpu_mode or selected def reset_gpu_profile(self) -> None: with self._lock: self._runtime_gpu_mode = None self._resource_pressure_samples = 0 self._resource_recovery_samples = 0 def apply_resource_pressure(self, pressured: bool) -> None: """Temporarily reduce GPU concurrency under sustained host pressure.""" with self._lock: if pressured: self._resource_pressure_samples += 1 self._resource_recovery_samples = 0 if self._resource_pressure_samples >= 3: selected = self.settings.ai_gpu_mode if self.settings.ai_gpu_mode in GPU_MODES else "balanced" self._runtime_gpu_mode = "low" if selected in {"balanced", "throughput"} else selected else: self._resource_pressure_samples = 0 self._resource_recovery_samples += 1 if self._resource_recovery_samples >= 5: self._runtime_gpu_mode = None self._resource_recovery_samples = 0 def hardware_metrics(self) -> dict: """Collect Intel DRM engine utilization without inventing VRAM data.""" now = time.monotonic() engines: dict[str, float | None] = {} busy_files = sorted(Path("/sys/class/drm").glob("card*/engine/*/busy")) for path in busy_files: try: value = int(path.read_text(encoding="utf-8").strip()) except (OSError, ValueError): continue key = f"{path.parents[2].name}/{path.parent.name}" previous = self._gpu_busy_previous.get(str(path)) utilization = None if previous is not None and now > previous[1] and value >= previous[0]: utilization = max(0.0, min(100.0, (value - previous[0]) / ((now - previous[1]) * 1_000_000_000) * 100)) self._gpu_busy_previous[str(path)] = (value, now) engines[key] = round(utilization, 1) if utilization is not None else None sampled = [value for value in engines.values() if value is not None] utilization = max(sampled) if sampled else None memory_total = memory_used = None for card in sorted(Path("/sys/class/drm").glob("card*/device")): total_path = card / "mem_info_vram_total" used_path = card / "mem_info_vram_used" try: total_value = int(total_path.read_text(encoding="utf-8").strip()) used_value = int(used_path.read_text(encoding="utf-8").strip()) except (OSError, ValueError): continue memory_total = max(0, total_value) memory_used = max(0, used_value) break supported = bool(engines) reason = None if not supported: reason = ( "Intel DRM 未提供 engine busy 指标" if self.gpu_device else self.unavailable_reason or "未检测到可用 GPU" ) return { "supported": supported, "utilization_percent": utilization, "engines": engines, "memory_supported": memory_total is not None, "memory_total_bytes": memory_total, "memory_used_bytes": memory_used, "collector": "drm_sysfs" if supported else None, "reason": reason, "collected_at": datetime.now(UTC).isoformat(), } def _degrade_gpu_profile(self) -> None: order = ("low", "balanced", "throughput") current = self.effective_gpu_mode() index = order.index(current) if index <= 0: return with self._lock: self._runtime_gpu_mode = order[index - 1] logger.warning("GPU profile degraded from %s to %s after inference failure", current, order[index - 1]) @staticmethod def _now() -> str: return datetime.now(UTC).isoformat() @staticmethod def _normalise_execution_devices(devices) -> list[str]: if devices is None: return [] if isinstance(devices, str): values = [devices] else: try: values = list(devices) except TypeError: values = [devices] result = [] for value in values: name = str(value).strip() if name and name not in result: result.append(name) return result @classmethod def execution_devices_from(cls, *roots) -> list[str]: """Read EXECUTION_DEVICES from OpenVINO/Optimum model wrappers.""" devices: list[str] = [] seen: set[int] = set() pending = [root for root in roots if root is not None] attribute_names = ( "model", "auto_model", "compiled_model", "request", "encoder", "decoder", "decoder_with_past", ) while pending: value = pending.pop() identity = id(value) if identity in seen: continue seen.add(identity) try: reported = value.get_property("EXECUTION_DEVICES") # OpenVINO plugins do not expose exactly the same property set. # In particular, some Intel GPU driver/runtime combinations raise # ValueError rather than RuntimeError for EXECUTION_DEVICES even # though the compiled request itself is valid. Treat that branch # as non-reporting and continue inspecting the other wrappers. except (AttributeError, RuntimeError, TypeError, ValueError): reported = None for device in cls._normalise_execution_devices(reported): if device not in devices: devices.append(device) try: compiled = value.get_compiled_model() except (AttributeError, RuntimeError, TypeError, ValueError): compiled = None if compiled is not None: pending.append(compiled) for name in attribute_names: try: child = getattr(value, name) except (AttributeError, RuntimeError, ValueError): continue if child is not None and child is not value: pending.append(child) try: modules = list(value.children()) except (AttributeError, RuntimeError, TypeError, ValueError): modules = [] pending.extend(modules) return devices def mark_ready(self, component: str, device: str, execution_devices=None) -> None: actual_devices = self._normalise_execution_devices(execution_devices) or [device] actual = " / ".join(actual_devices) with self._lock: prior_reason = self._components[component].get("fallback_reason") self._components[component] = { "state": "fallback" if prior_reason and device == "CPU" else "ready", "device": actual, "requested_device": device, "actual_device": actual, "execution_devices": actual_devices, "failure_stage": self._components[component].get("failure_stage") if device == "CPU" else None, "fallback_reason": prior_reason if device == "CPU" else None, "fallback_scope": self._components[component].get("fallback_scope") if device == "CPU" else None, "circuit_state": self._components[component].get("circuit_state", "closed"), "failure_count": int(self._components[component].get("failure_count") or 0), "retry_at": self._components[component].get("retry_at"), "last_verified_at": self._now(), } def mark_inference_success(self, component: str, device: str, execution_devices=None) -> None: """Close a transient circuit only after inference, not merely model compilation.""" actual_devices = self._normalise_execution_devices(execution_devices) or [device] actual = " / ".join(actual_devices) with self._lock: self._components[component] = { "state": "ready", "device": actual, "requested_device": device, "actual_device": actual, "execution_devices": actual_devices, "failure_stage": None, "fallback_reason": None, "fallback_scope": None, "circuit_state": "closed", "failure_count": 0, "retry_at": None, "last_verified_at": self._now(), } def mark_verifying_cpu( self, component: str, reason: str = "GPU 未返回文字，正在使用 CPU 复核", *, stage: str = "empty_result", ) -> None: """Expose a bounded CPU cross-check without pinning the component to CPU. Empty Whisper output can be either valid no-speech content or a GPU generation defect. The verifier is deliberately transient: callers still get the preferred GPU from :meth:`device_for` until a CPU sample proves that the GPU result was wrong. """ with self._lock: self._components[component] = { "state": "verifying_cpu", "device": "CPU", "requested_device": self.gpu_device, "actual_device": "CPU", "execution_devices": ["CPU"], "failure_stage": stage, "fallback_reason": reason[:160], "fallback_scope": "job", "circuit_state": self._components[component].get("circuit_state", "closed"), "failure_count": int(self._components[component].get("failure_count") or 0), "retry_at": self._components[component].get("retry_at"), "last_verified_at": self._now(), } def record_transient_failure( self, component: str, reason: BaseException | str, *, stage: str = "inference", ) -> bool: """Record one task-local device failure and open the audio circuit after a threshold.""" if component != "audio": return self.fall_back(component, reason, stage=stage) detail = type(reason).__name__ if isinstance(reason, BaseException) else str(reason).strip()[:120] detail = detail or "未知错误" with self._lock: current = self._components[component] failures = int(current.get("failure_count") or 0) + 1 if failures < AUDIO_CIRCUIT_FAILURES: current.update( state="job_fallback", device="CPU", requested_device=self.gpu_device, actual_device="CPU", execution_devices=["CPU"], failure_stage=stage, fallback_reason=f"当前任务 GPU 推理失败，使用 CPU（{detail}）", fallback_scope="job", circuit_state="closed", failure_count=failures, retry_at=None, last_verified_at=self._now(), ) logger.warning( "AI component %s task-local GPU failure %s/%s: %s", component, failures, AUDIO_CIRCUIT_FAILURES, detail, ) return False retry_at = datetime.now(UTC) + timedelta(seconds=AUDIO_CIRCUIT_COOLDOWN_SECONDS) current.update( state="fallback", device="CPU", requested_device=self.gpu_device, actual_device="CPU", execution_devices=["CPU"], failure_stage=stage, fallback_reason=f"GPU 连续推理失败，暂时回退 CPU（{detail}）", fallback_scope="component", circuit_state="open", failure_count=failures, retry_at=retry_at.isoformat(), last_verified_at=self._now(), ) logger.warning("AI component %s circuit opened after %s failures: %s", component, failures, detail) return True def fall_back(self, component: str, reason: BaseException | str, *, stage: str = "inference") -> bool: """Pin one GPU component to CPU and report whether a retry is useful.""" with self._lock: current = self._components[component] # A temporary CPU verification reports device=CPU as well, but it # must still be promotable to a real, persistent fallback when the # comparison sample contains speech that GPU failed to return. if current.get("state") == "fallback": return False if isinstance(reason, BaseException): detail = f"{type(reason).__name__}" else: detail = str(reason).strip()[:120] or "未知错误" current.update( state="fallback", device="CPU", requested_device=self.gpu_device, actual_device="CPU", execution_devices=["CPU"], failure_stage=stage, fallback_reason=f"GPU 推理失败，已回退 CPU（{detail}）", fallback_scope="component", circuit_state="open" if component == "audio" else "closed", failure_count=max(1, int(current.get("failure_count") or 0)), retry_at=( (datetime.now(UTC) + timedelta(seconds=AUDIO_CIRCUIT_COOLDOWN_SECONDS)).isoformat() if component == "audio" else None ), last_verified_at=self._now(), ) if stage in {"encoder_compile", "decoder_compile", "inference", "inference_stall"}: self._degrade_gpu_profile() logger.warning("AI component %s fell back from GPU to CPU: %s", component, detail) return True @staticmethod def is_device_error(reason: BaseException) -> bool: """Only retry errors that plausibly originate from OpenVINO device compilation/inference.""" module = type(reason).__module__.lower() name = type(reason).__name__.lower() text = str(reason).lower() if "huggingface" in module or "hfvalidation" in name or name == "modelunavailable": return False non_device = ("repo id", "repository id", "modules.json", "config.json", "local path", "not a valid model") if any(value in text for value in non_device): return False # Python bindings for Intel GPU compilation and property discovery can # surface plugin failures as ValueError. The same model is still worth # retrying on CPU; Hugging Face/configuration ValueErrors were excluded # above so genuine model layout problems remain visible. if isinstance(reason, ValueError): return True device_markers = ( "openvino", "gpu", "device", "compile_model", "cldnn", "level zero", "ze_result", "intel", ) return "openvino" in module or any(value in text for value in device_markers) def mark_unavailable( self, component: str, reason: BaseException | str, *, stage: str = "inference", ) -> None: detail = type(reason).__name__ if isinstance(reason, BaseException) else str(reason).strip()[:120] with self._lock: self._components[component] = { "state": "unavailable", "device": None, "requested_device": self.gpu_device or "CPU", "actual_device": None, "execution_devices": [], "failure_stage": stage, "fallback_reason": detail or "模型不可用", "fallback_scope": "component", "circuit_state": "open" if component == "audio" else "closed", "failure_count": int(self._components[component].get("failure_count") or 0), "retry_at": None, "last_verified_at": self._now(), } def reset(self, component: str) -> None: with self._lock: previous = self._components[component] self._components[component] = { "state": "not_loaded", "device": None, # Releasing a model only means it is no longer resident in # memory. Keep the last verified device as observability # history so a completed stage does not immediately revert # to the misleading "never loaded" state in the UI. "requested_device": previous.get("requested_device"), "actual_device": previous.get("actual_device"), "execution_devices": list(previous.get("execution_devices") or []), "failure_stage": None, "fallback_reason": None, "fallback_scope": None, "circuit_state": "closed", "failure_count": 0, "retry_at": None, "last_verified_at": previous.get("last_verified_at"), } def merge_worker_status(self, status: dict) -> None: """Mirror observable state reported by the isolated inference worker.""" components = status.get("components") if not isinstance(components, dict): return with self._lock: for name in COMPONENTS: value = components.get(name) if isinstance(value, dict): self._components[name] = dict(value) def status(self) -> dict: with self._lock: components = {name: dict(value) for name, value in self._components.items()} active = sorted({value["device"] for value in components.values() if value.get("device")}) if active: selected = " / ".join(active) elif self.gpu_device: selected = "Intel GPU 优先（首次推理时加载）" else: selected = "CPU（未检测到可用 Intel GPU）" return { "policy": "gpu_preferred", "openvino": self.openvino, "devices": list(self.devices), "device_names": dict(self.device_names), "selected": selected, "render_device": { "available": bool(self.render_nodes), "accessible": self.render_accessible, "count": len(self.render_nodes), }, "unavailable_reason": self.unavailable_reason or None, "cpu_threads": self.settings.ai_cpu_threads, "gpu_profile": self.gpu_profile(), "components": components, }