607 lines
25 KiB
Python
607 lines
25 KiB
Python
from __future__ import annotations
|
||
|
||
import logging
|
||
import os
|
||
import time
|
||
from datetime import UTC, datetime, timedelta
|
||
from pathlib import Path
|
||
from threading import Lock
|
||
|
||
from .config import Settings
|
||
|
||
logger = logging.getLogger(__name__)
|
||
COMPONENTS = ("visual", "ocr", "faces", "audio")
|
||
GPU_MODES = {
|
||
"low": {"hint": "LATENCY", "streams": 1, "batch_size": 1},
|
||
"balanced": {"hint": "THROUGHPUT", "streams": 2, "batch_size": 2},
|
||
"throughput": {"hint": "THROUGHPUT", "streams": 3, "batch_size": 4},
|
||
}
|
||
AUDIO_CIRCUIT_FAILURES = 3
|
||
AUDIO_CIRCUIT_COOLDOWN_SECONDS = 600
|
||
|
||
|
||
class AcceleratorService:
|
||
"""Detect and track the OpenVINO device used by each AI component.
|
||
|
||
ImageFind uses explicit GPU compilation instead of OpenVINO AUTO so the
|
||
settings page can truthfully report where inference ran. A failed GPU
|
||
component is pinned to a bounded CPU fallback until it is reset.
|
||
"""
|
||
|
||
def __init__(self, settings: Settings):
|
||
self.settings = settings
|
||
self._lock = Lock()
|
||
self._runtime_gpu_mode: str | None = None
|
||
self._resource_pressure_samples = 0
|
||
self._resource_recovery_samples = 0
|
||
self._gpu_busy_previous: dict[str, tuple[int, float]] = {}
|
||
self.openvino = False
|
||
self.devices: list[str] = []
|
||
self.device_names: dict[str, str] = {}
|
||
self.gpu_device: str | None = None
|
||
self.render_nodes: list[str] = []
|
||
self.render_accessible = False
|
||
self.unavailable_reason = ""
|
||
self._components = {
|
||
name: {
|
||
"state": "not_loaded",
|
||
"device": None,
|
||
"requested_device": None,
|
||
"actual_device": None,
|
||
"execution_devices": [],
|
||
"failure_stage": None,
|
||
"fallback_reason": None,
|
||
"fallback_scope": None,
|
||
"circuit_state": "closed",
|
||
"failure_count": 0,
|
||
"retry_at": None,
|
||
"last_verified_at": None,
|
||
}
|
||
for name in COMPONENTS
|
||
}
|
||
self.refresh()
|
||
|
||
def refresh(self) -> None:
|
||
nodes = sorted(Path("/dev/dri").glob("renderD*"))
|
||
self.render_nodes = [str(path) for path in nodes]
|
||
self.render_accessible = any(os.access(path, os.R_OK | os.W_OK) for path in nodes)
|
||
try:
|
||
import openvino as ov
|
||
|
||
core = ov.Core()
|
||
devices = list(core.available_devices)
|
||
names: dict[str, str] = {}
|
||
for device in devices:
|
||
try:
|
||
names[device] = str(core.get_property(device, "FULL_DEVICE_NAME"))
|
||
except Exception:
|
||
names[device] = device
|
||
self.openvino = True
|
||
self.devices = devices
|
||
self.device_names = names
|
||
self.gpu_device = next((device for device in devices if device.upper().startswith("GPU")), None)
|
||
except ModuleNotFoundError:
|
||
logger.info("OpenVINO runtime is not installed; AI acceleration will use CPU fallbacks")
|
||
self.openvino = False
|
||
self.devices = []
|
||
self.device_names = {}
|
||
self.gpu_device = None
|
||
except Exception:
|
||
logger.warning("OpenVINO device discovery failed", exc_info=True)
|
||
self.openvino = False
|
||
self.devices = []
|
||
self.device_names = {}
|
||
self.gpu_device = None
|
||
|
||
if self.gpu_device:
|
||
self.unavailable_reason = ""
|
||
elif not self.openvino:
|
||
self.unavailable_reason = "OpenVINO 运行时不可用"
|
||
elif not nodes:
|
||
self.unavailable_reason = "未检测到 Intel render 设备"
|
||
elif not self.render_accessible:
|
||
self.unavailable_reason = "Intel render 设备权限不足"
|
||
else:
|
||
self.unavailable_reason = "OpenVINO 未发现 Intel GPU，请检查核显驱动"
|
||
|
||
logger.info(
|
||
"AI accelerator devices=%s gpu=%s render_accessible=%s",
|
||
self.devices,
|
||
self.gpu_device or "none",
|
||
self.render_accessible,
|
||
)
|
||
|
||
def device_for(self, component: str) -> str:
|
||
with self._lock:
|
||
state = self._components[component]
|
||
if state["state"] == "fallback":
|
||
retry_at = state.get("retry_at")
|
||
if component == "audio" and retry_at and self.gpu_device:
|
||
try:
|
||
retry_due = datetime.fromisoformat(str(retry_at)) <= datetime.now(UTC)
|
||
except ValueError:
|
||
retry_due = True
|
||
if retry_due:
|
||
state.update(
|
||
state="probing_gpu",
|
||
device=self.gpu_device,
|
||
requested_device=self.gpu_device,
|
||
actual_device=None,
|
||
execution_devices=[],
|
||
fallback_scope="component",
|
||
circuit_state="half_open",
|
||
retry_at=None,
|
||
)
|
||
return self.gpu_device
|
||
return "CPU"
|
||
return self.gpu_device or "CPU"
|
||
|
||
def ov_config(self, device: str) -> dict[str, object]:
|
||
if device.upper().startswith("CPU"):
|
||
return {
|
||
"INFERENCE_NUM_THREADS": self.settings.ai_cpu_threads,
|
||
"PERFORMANCE_HINT": "LATENCY",
|
||
}
|
||
cache_dir = self.settings.runtime_dir / "openvino-cache"
|
||
cache_dir.mkdir(parents=True, exist_ok=True)
|
||
mode = self.effective_gpu_mode()
|
||
profile = GPU_MODES[mode]
|
||
return {
|
||
"PERFORMANCE_HINT": profile["hint"],
|
||
"NUM_STREAMS": str(profile["streams"]),
|
||
"CACHE_DIR": str(cache_dir),
|
||
"INFERENCE_PRECISION_HINT": "f32",
|
||
}
|
||
|
||
def gpu_profile(self) -> dict[str, object]:
|
||
selected = self.settings.ai_gpu_mode if self.settings.ai_gpu_mode in GPU_MODES else "balanced"
|
||
mode = self.effective_gpu_mode()
|
||
return {"mode": mode, "selected_mode": selected, "degraded": mode != selected, **GPU_MODES[mode]}
|
||
|
||
def effective_gpu_mode(self) -> str:
|
||
selected = self.settings.ai_gpu_mode if self.settings.ai_gpu_mode in GPU_MODES else "balanced"
|
||
with self._lock:
|
||
return self._runtime_gpu_mode or selected
|
||
|
||
def reset_gpu_profile(self) -> None:
|
||
with self._lock:
|
||
self._runtime_gpu_mode = None
|
||
self._resource_pressure_samples = 0
|
||
self._resource_recovery_samples = 0
|
||
|
||
def apply_resource_pressure(self, pressured: bool) -> None:
|
||
"""Temporarily reduce GPU concurrency under sustained host pressure."""
|
||
|
||
with self._lock:
|
||
if pressured:
|
||
self._resource_pressure_samples += 1
|
||
self._resource_recovery_samples = 0
|
||
if self._resource_pressure_samples >= 3:
|
||
selected = self.settings.ai_gpu_mode if self.settings.ai_gpu_mode in GPU_MODES else "balanced"
|
||
self._runtime_gpu_mode = "low" if selected in {"balanced", "throughput"} else selected
|
||
else:
|
||
self._resource_pressure_samples = 0
|
||
self._resource_recovery_samples += 1
|
||
if self._resource_recovery_samples >= 5:
|
||
self._runtime_gpu_mode = None
|
||
self._resource_recovery_samples = 0
|
||
|
||
def hardware_metrics(self) -> dict:
|
||
"""Collect Intel DRM engine utilization without inventing VRAM data."""
|
||
|
||
now = time.monotonic()
|
||
engines: dict[str, float | None] = {}
|
||
busy_files = sorted(Path("/sys/class/drm").glob("card*/engine/*/busy"))
|
||
for path in busy_files:
|
||
try:
|
||
value = int(path.read_text(encoding="utf-8").strip())
|
||
except (OSError, ValueError):
|
||
continue
|
||
key = f"{path.parents[2].name}/{path.parent.name}"
|
||
previous = self._gpu_busy_previous.get(str(path))
|
||
utilization = None
|
||
if previous is not None and now > previous[1] and value >= previous[0]:
|
||
utilization = max(0.0, min(100.0, (value - previous[0]) / ((now - previous[1]) * 1_000_000_000) * 100))
|
||
self._gpu_busy_previous[str(path)] = (value, now)
|
||
engines[key] = round(utilization, 1) if utilization is not None else None
|
||
sampled = [value for value in engines.values() if value is not None]
|
||
utilization = max(sampled) if sampled else None
|
||
memory_total = memory_used = None
|
||
for card in sorted(Path("/sys/class/drm").glob("card*/device")):
|
||
total_path = card / "mem_info_vram_total"
|
||
used_path = card / "mem_info_vram_used"
|
||
try:
|
||
total_value = int(total_path.read_text(encoding="utf-8").strip())
|
||
used_value = int(used_path.read_text(encoding="utf-8").strip())
|
||
except (OSError, ValueError):
|
||
continue
|
||
memory_total = max(0, total_value)
|
||
memory_used = max(0, used_value)
|
||
break
|
||
supported = bool(engines)
|
||
reason = None
|
||
if not supported:
|
||
reason = (
|
||
"Intel DRM 未提供 engine busy 指标"
|
||
if self.gpu_device
|
||
else self.unavailable_reason or "未检测到可用 GPU"
|
||
)
|
||
return {
|
||
"supported": supported,
|
||
"utilization_percent": utilization,
|
||
"engines": engines,
|
||
"memory_supported": memory_total is not None,
|
||
"memory_total_bytes": memory_total,
|
||
"memory_used_bytes": memory_used,
|
||
"collector": "drm_sysfs" if supported else None,
|
||
"reason": reason,
|
||
"collected_at": datetime.now(UTC).isoformat(),
|
||
}
|
||
|
||
def _degrade_gpu_profile(self) -> None:
|
||
order = ("low", "balanced", "throughput")
|
||
current = self.effective_gpu_mode()
|
||
index = order.index(current)
|
||
if index <= 0:
|
||
return
|
||
with self._lock:
|
||
self._runtime_gpu_mode = order[index - 1]
|
||
logger.warning("GPU profile degraded from %s to %s after inference failure", current, order[index - 1])
|
||
|
||
@staticmethod
|
||
def _now() -> str:
|
||
return datetime.now(UTC).isoformat()
|
||
|
||
@staticmethod
|
||
def _normalise_execution_devices(devices) -> list[str]:
|
||
if devices is None:
|
||
return []
|
||
if isinstance(devices, str):
|
||
values = [devices]
|
||
else:
|
||
try:
|
||
values = list(devices)
|
||
except TypeError:
|
||
values = [devices]
|
||
result = []
|
||
for value in values:
|
||
name = str(value).strip()
|
||
if name and name not in result:
|
||
result.append(name)
|
||
return result
|
||
|
||
@classmethod
|
||
def execution_devices_from(cls, *roots) -> list[str]:
|
||
"""Read EXECUTION_DEVICES from OpenVINO/Optimum model wrappers."""
|
||
|
||
devices: list[str] = []
|
||
seen: set[int] = set()
|
||
pending = [root for root in roots if root is not None]
|
||
attribute_names = (
|
||
"model",
|
||
"auto_model",
|
||
"compiled_model",
|
||
"request",
|
||
"encoder",
|
||
"decoder",
|
||
"decoder_with_past",
|
||
)
|
||
while pending:
|
||
value = pending.pop()
|
||
identity = id(value)
|
||
if identity in seen:
|
||
continue
|
||
seen.add(identity)
|
||
try:
|
||
reported = value.get_property("EXECUTION_DEVICES")
|
||
# OpenVINO plugins do not expose exactly the same property set.
|
||
# In particular, some Intel GPU driver/runtime combinations raise
|
||
# ValueError rather than RuntimeError for EXECUTION_DEVICES even
|
||
# though the compiled request itself is valid. Treat that branch
|
||
# as non-reporting and continue inspecting the other wrappers.
|
||
except (AttributeError, RuntimeError, TypeError, ValueError):
|
||
reported = None
|
||
for device in cls._normalise_execution_devices(reported):
|
||
if device not in devices:
|
||
devices.append(device)
|
||
try:
|
||
compiled = value.get_compiled_model()
|
||
except (AttributeError, RuntimeError, TypeError, ValueError):
|
||
compiled = None
|
||
if compiled is not None:
|
||
pending.append(compiled)
|
||
for name in attribute_names:
|
||
try:
|
||
child = getattr(value, name)
|
||
except (AttributeError, RuntimeError, ValueError):
|
||
continue
|
||
if child is not None and child is not value:
|
||
pending.append(child)
|
||
try:
|
||
modules = list(value.children())
|
||
except (AttributeError, RuntimeError, TypeError, ValueError):
|
||
modules = []
|
||
pending.extend(modules)
|
||
return devices
|
||
|
||
def mark_ready(self, component: str, device: str, execution_devices=None) -> None:
|
||
actual_devices = self._normalise_execution_devices(execution_devices) or [device]
|
||
actual = " / ".join(actual_devices)
|
||
with self._lock:
|
||
prior_reason = self._components[component].get("fallback_reason")
|
||
self._components[component] = {
|
||
"state": "fallback" if prior_reason and device == "CPU" else "ready",
|
||
"device": actual,
|
||
"requested_device": device,
|
||
"actual_device": actual,
|
||
"execution_devices": actual_devices,
|
||
"failure_stage": self._components[component].get("failure_stage") if device == "CPU" else None,
|
||
"fallback_reason": prior_reason if device == "CPU" else None,
|
||
"fallback_scope": self._components[component].get("fallback_scope") if device == "CPU" else None,
|
||
"circuit_state": self._components[component].get("circuit_state", "closed"),
|
||
"failure_count": int(self._components[component].get("failure_count") or 0),
|
||
"retry_at": self._components[component].get("retry_at"),
|
||
"last_verified_at": self._now(),
|
||
}
|
||
|
||
def mark_inference_success(self, component: str, device: str, execution_devices=None) -> None:
|
||
"""Close a transient circuit only after inference, not merely model compilation."""
|
||
|
||
actual_devices = self._normalise_execution_devices(execution_devices) or [device]
|
||
actual = " / ".join(actual_devices)
|
||
with self._lock:
|
||
self._components[component] = {
|
||
"state": "ready",
|
||
"device": actual,
|
||
"requested_device": device,
|
||
"actual_device": actual,
|
||
"execution_devices": actual_devices,
|
||
"failure_stage": None,
|
||
"fallback_reason": None,
|
||
"fallback_scope": None,
|
||
"circuit_state": "closed",
|
||
"failure_count": 0,
|
||
"retry_at": None,
|
||
"last_verified_at": self._now(),
|
||
}
|
||
|
||
def mark_verifying_cpu(
|
||
self,
|
||
component: str,
|
||
reason: str = "GPU 未返回文字，正在使用 CPU 复核",
|
||
*,
|
||
stage: str = "empty_result",
|
||
) -> None:
|
||
"""Expose a bounded CPU cross-check without pinning the component to CPU.
|
||
|
||
Empty Whisper output can be either valid no-speech content or a GPU
|
||
generation defect. The verifier is deliberately transient: callers
|
||
still get the preferred GPU from :meth:`device_for` until a CPU sample
|
||
proves that the GPU result was wrong.
|
||
"""
|
||
|
||
with self._lock:
|
||
self._components[component] = {
|
||
"state": "verifying_cpu",
|
||
"device": "CPU",
|
||
"requested_device": self.gpu_device,
|
||
"actual_device": "CPU",
|
||
"execution_devices": ["CPU"],
|
||
"failure_stage": stage,
|
||
"fallback_reason": reason[:160],
|
||
"fallback_scope": "job",
|
||
"circuit_state": self._components[component].get("circuit_state", "closed"),
|
||
"failure_count": int(self._components[component].get("failure_count") or 0),
|
||
"retry_at": self._components[component].get("retry_at"),
|
||
"last_verified_at": self._now(),
|
||
}
|
||
|
||
def record_transient_failure(
|
||
self,
|
||
component: str,
|
||
reason: BaseException | str,
|
||
*,
|
||
stage: str = "inference",
|
||
) -> bool:
|
||
"""Record one task-local device failure and open the audio circuit after a threshold."""
|
||
|
||
if component != "audio":
|
||
return self.fall_back(component, reason, stage=stage)
|
||
detail = type(reason).__name__ if isinstance(reason, BaseException) else str(reason).strip()[:120]
|
||
detail = detail or "未知错误"
|
||
with self._lock:
|
||
current = self._components[component]
|
||
failures = int(current.get("failure_count") or 0) + 1
|
||
if failures < AUDIO_CIRCUIT_FAILURES:
|
||
current.update(
|
||
state="job_fallback",
|
||
device="CPU",
|
||
requested_device=self.gpu_device,
|
||
actual_device="CPU",
|
||
execution_devices=["CPU"],
|
||
failure_stage=stage,
|
||
fallback_reason=f"当前任务 GPU 推理失败，使用 CPU（{detail}）",
|
||
fallback_scope="job",
|
||
circuit_state="closed",
|
||
failure_count=failures,
|
||
retry_at=None,
|
||
last_verified_at=self._now(),
|
||
)
|
||
logger.warning(
|
||
"AI component %s task-local GPU failure %s/%s: %s",
|
||
component,
|
||
failures,
|
||
AUDIO_CIRCUIT_FAILURES,
|
||
detail,
|
||
)
|
||
return False
|
||
retry_at = datetime.now(UTC) + timedelta(seconds=AUDIO_CIRCUIT_COOLDOWN_SECONDS)
|
||
current.update(
|
||
state="fallback",
|
||
device="CPU",
|
||
requested_device=self.gpu_device,
|
||
actual_device="CPU",
|
||
execution_devices=["CPU"],
|
||
failure_stage=stage,
|
||
fallback_reason=f"GPU 连续推理失败，暂时回退 CPU（{detail}）",
|
||
fallback_scope="component",
|
||
circuit_state="open",
|
||
failure_count=failures,
|
||
retry_at=retry_at.isoformat(),
|
||
last_verified_at=self._now(),
|
||
)
|
||
logger.warning("AI component %s circuit opened after %s failures: %s", component, failures, detail)
|
||
return True
|
||
|
||
def fall_back(self, component: str, reason: BaseException | str, *, stage: str = "inference") -> bool:
|
||
"""Pin one GPU component to CPU and report whether a retry is useful."""
|
||
|
||
with self._lock:
|
||
current = self._components[component]
|
||
# A temporary CPU verification reports device=CPU as well, but it
|
||
# must still be promotable to a real, persistent fallback when the
|
||
# comparison sample contains speech that GPU failed to return.
|
||
if current.get("state") == "fallback":
|
||
return False
|
||
if isinstance(reason, BaseException):
|
||
detail = f"{type(reason).__name__}"
|
||
else:
|
||
detail = str(reason).strip()[:120] or "未知错误"
|
||
current.update(
|
||
state="fallback",
|
||
device="CPU",
|
||
requested_device=self.gpu_device,
|
||
actual_device="CPU",
|
||
execution_devices=["CPU"],
|
||
failure_stage=stage,
|
||
fallback_reason=f"GPU 推理失败，已回退 CPU（{detail}）",
|
||
fallback_scope="component",
|
||
circuit_state="open" if component == "audio" else "closed",
|
||
failure_count=max(1, int(current.get("failure_count") or 0)),
|
||
retry_at=(
|
||
(datetime.now(UTC) + timedelta(seconds=AUDIO_CIRCUIT_COOLDOWN_SECONDS)).isoformat()
|
||
if component == "audio"
|
||
else None
|
||
),
|
||
last_verified_at=self._now(),
|
||
)
|
||
if stage in {"encoder_compile", "decoder_compile", "inference", "inference_stall"}:
|
||
self._degrade_gpu_profile()
|
||
logger.warning("AI component %s fell back from GPU to CPU: %s", component, detail)
|
||
return True
|
||
|
||
@staticmethod
|
||
def is_device_error(reason: BaseException) -> bool:
|
||
"""Only retry errors that plausibly originate from OpenVINO device compilation/inference."""
|
||
|
||
module = type(reason).__module__.lower()
|
||
name = type(reason).__name__.lower()
|
||
text = str(reason).lower()
|
||
if "huggingface" in module or "hfvalidation" in name or name == "modelunavailable":
|
||
return False
|
||
non_device = ("repo id", "repository id", "modules.json", "config.json", "local path", "not a valid model")
|
||
if any(value in text for value in non_device):
|
||
return False
|
||
# Python bindings for Intel GPU compilation and property discovery can
|
||
# surface plugin failures as ValueError. The same model is still worth
|
||
# retrying on CPU; Hugging Face/configuration ValueErrors were excluded
|
||
# above so genuine model layout problems remain visible.
|
||
if isinstance(reason, ValueError):
|
||
return True
|
||
device_markers = (
|
||
"openvino",
|
||
"gpu",
|
||
"device",
|
||
"compile_model",
|
||
"cldnn",
|
||
"level zero",
|
||
"ze_result",
|
||
"intel",
|
||
)
|
||
return "openvino" in module or any(value in text for value in device_markers)
|
||
|
||
def mark_unavailable(
|
||
self,
|
||
component: str,
|
||
reason: BaseException | str,
|
||
*,
|
||
stage: str = "inference",
|
||
) -> None:
|
||
detail = type(reason).__name__ if isinstance(reason, BaseException) else str(reason).strip()[:120]
|
||
with self._lock:
|
||
self._components[component] = {
|
||
"state": "unavailable",
|
||
"device": None,
|
||
"requested_device": self.gpu_device or "CPU",
|
||
"actual_device": None,
|
||
"execution_devices": [],
|
||
"failure_stage": stage,
|
||
"fallback_reason": detail or "模型不可用",
|
||
"fallback_scope": "component",
|
||
"circuit_state": "open" if component == "audio" else "closed",
|
||
"failure_count": int(self._components[component].get("failure_count") or 0),
|
||
"retry_at": None,
|
||
"last_verified_at": self._now(),
|
||
}
|
||
|
||
def reset(self, component: str) -> None:
|
||
with self._lock:
|
||
previous = self._components[component]
|
||
self._components[component] = {
|
||
"state": "not_loaded",
|
||
"device": None,
|
||
# Releasing a model only means it is no longer resident in
|
||
# memory. Keep the last verified device as observability
|
||
# history so a completed stage does not immediately revert
|
||
# to the misleading "never loaded" state in the UI.
|
||
"requested_device": previous.get("requested_device"),
|
||
"actual_device": previous.get("actual_device"),
|
||
"execution_devices": list(previous.get("execution_devices") or []),
|
||
"failure_stage": None,
|
||
"fallback_reason": None,
|
||
"fallback_scope": None,
|
||
"circuit_state": "closed",
|
||
"failure_count": 0,
|
||
"retry_at": None,
|
||
"last_verified_at": previous.get("last_verified_at"),
|
||
}
|
||
|
||
def merge_worker_status(self, status: dict) -> None:
|
||
"""Mirror observable state reported by the isolated inference worker."""
|
||
|
||
components = status.get("components")
|
||
if not isinstance(components, dict):
|
||
return
|
||
with self._lock:
|
||
for name in COMPONENTS:
|
||
value = components.get(name)
|
||
if isinstance(value, dict):
|
||
self._components[name] = dict(value)
|
||
|
||
def status(self) -> dict:
|
||
with self._lock:
|
||
components = {name: dict(value) for name, value in self._components.items()}
|
||
active = sorted({value["device"] for value in components.values() if value.get("device")})
|
||
if active:
|
||
selected = " / ".join(active)
|
||
elif self.gpu_device:
|
||
selected = "Intel GPU 优先（首次推理时加载）"
|
||
else:
|
||
selected = "CPU（未检测到可用 Intel GPU）"
|
||
return {
|
||
"policy": "gpu_preferred",
|
||
"openvino": self.openvino,
|
||
"devices": list(self.devices),
|
||
"device_names": dict(self.device_names),
|
||
"selected": selected,
|
||
"render_device": {
|
||
"available": bool(self.render_nodes),
|
||
"accessible": self.render_accessible,
|
||
"count": len(self.render_nodes),
|
||
},
|
||
"unavailable_reason": self.unavailable_reason or None,
|
||
"cpu_threads": self.settings.ai_cpu_threads,
|
||
"gpu_profile": self.gpu_profile(),
|
||
"components": components,
|
||
}
|