121 lines
4.3 KiB
Python
121 lines
4.3 KiB
Python
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import math
|
|
import threading
|
|
import time
|
|
from collections import deque
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from starlette.types import ASGIApp, Message, Receive, Scope, Send
|
|
|
|
|
|
class Diagnostics:
|
|
def __init__(self) -> None:
|
|
self._guard = threading.Lock()
|
|
self._requests: deque[dict[str, Any]] = deque(maxlen=200)
|
|
self._loop_lag_ms = 0.0
|
|
self._loop_max_lag_ms = 0.0
|
|
|
|
def record_request(self, method: str, path: str, status: int, duration_ms: float) -> None:
|
|
with self._guard:
|
|
self._requests.append(
|
|
{
|
|
"method": method,
|
|
"path": path,
|
|
"status": status,
|
|
"duration_ms": round(duration_ms, 2),
|
|
"at": time.time(),
|
|
}
|
|
)
|
|
|
|
async def monitor(self, stopping: asyncio.Event) -> None:
|
|
interval = 1.0
|
|
expected = time.monotonic() + interval
|
|
while not stopping.is_set():
|
|
try:
|
|
await asyncio.wait_for(stopping.wait(), timeout=interval)
|
|
break
|
|
except TimeoutError:
|
|
pass
|
|
now = time.monotonic()
|
|
lag = max(0.0, (now - expected) * 1000)
|
|
expected = now + interval
|
|
with self._guard:
|
|
self._loop_lag_ms = lag
|
|
self._loop_max_lag_ms = max(self._loop_max_lag_ms, lag)
|
|
|
|
@staticmethod
|
|
def _rss_bytes() -> int:
|
|
try:
|
|
for line in Path("/proc/self/status").read_text(encoding="utf-8").splitlines():
|
|
if line.startswith("VmRSS:"):
|
|
return int(line.split()[1]) * 1024
|
|
except (OSError, ValueError, IndexError):
|
|
pass
|
|
return 0
|
|
|
|
def status(self) -> dict[str, Any]:
|
|
with self._guard:
|
|
requests = list(self._requests)
|
|
lag = self._loop_lag_ms
|
|
max_lag = self._loop_max_lag_ms
|
|
durations = sorted(float(item["duration_ms"]) for item in requests)
|
|
p95_index = max(0, math.ceil(len(durations) * 0.95) - 1)
|
|
return {
|
|
"process_rss_bytes": self._rss_bytes(),
|
|
"event_loop_lag_ms": round(lag, 2),
|
|
"event_loop_max_lag_ms": round(max_lag, 2),
|
|
"request_count_window": len(requests),
|
|
"request_p95_ms": durations[p95_index] if durations else 0,
|
|
"slow_requests": [item for item in requests if item["duration_ms"] >= 1000][-30:],
|
|
"recent_errors": [item for item in requests if item["status"] >= 500][-20:],
|
|
}
|
|
|
|
|
|
class ObservabilityMiddleware:
|
|
SECURITY_HEADERS = (
|
|
(b"x-content-type-options", b"nosniff"),
|
|
(b"referrer-policy", b"same-origin"),
|
|
(b"permissions-policy", b"camera=(), microphone=(), geolocation=()"),
|
|
(
|
|
b"content-security-policy",
|
|
b"default-src 'self'; script-src 'self' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; "
|
|
b"img-src 'self' data: blob:; media-src 'self' blob:; connect-src 'self'; "
|
|
b"font-src 'self'; frame-ancestors 'self'",
|
|
),
|
|
)
|
|
|
|
def __init__(self, app: ASGIApp, diagnostics: Diagnostics):
|
|
self.app = app
|
|
self.diagnostics = diagnostics
|
|
|
|
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
|
|
if scope["type"] != "http":
|
|
await self.app(scope, receive, send)
|
|
return
|
|
started = time.monotonic()
|
|
status = 500
|
|
|
|
async def observed(message: Message) -> None:
|
|
nonlocal status
|
|
if message["type"] == "http.response.start":
|
|
status = int(message["status"])
|
|
headers = list(message.get("headers", []))
|
|
existing = {name.lower() for name, _ in headers}
|
|
headers.extend(header for header in self.SECURITY_HEADERS if header[0] not in existing)
|
|
message = {**message, "headers": headers}
|
|
await send(message)
|
|
|
|
try:
|
|
await self.app(scope, receive, observed)
|
|
finally:
|
|
duration_ms = (time.monotonic() - started) * 1000
|
|
self.diagnostics.record_request(
|
|
str(scope.get("method") or ""),
|
|
str(scope.get("path") or "")[:500],
|
|
status,
|
|
duration_ms,
|
|
)
|