feat: add ImageFind application and release pipelines
This commit is contained in:
Executable
+104
@@ -0,0 +1,104 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
PROJECT_ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
|
||||
BASE_VERSION=$(sed -n 's/^version[[:space:]]*=[[:space:]]*//p' "$PROJECT_ROOT/fnos/manifest" | head -n 1 | tr -d '\r')
|
||||
PACKAGE_VERSION="${PACKAGE_VERSION:-$BASE_VERSION}"
|
||||
case "$PACKAGE_VERSION" in
|
||||
''|*[!0-9.]*|.*|*..*|*.) printf 'invalid fnOS package version: %s\n' "$PACKAGE_VERSION" >&2; exit 1 ;;
|
||||
esac
|
||||
[ "$(printf '%s' "$PACKAGE_VERSION" | awk -F. '{ print NF }')" -eq 3 ] || {
|
||||
printf 'fnOS package version must contain exactly three numeric components: %s\n' "$PACKAGE_VERSION" >&2
|
||||
exit 1
|
||||
}
|
||||
BUILD_ROOT="$PROJECT_ROOT/.build-fnos"
|
||||
STAGE="$BUILD_ROOT/imagefind"
|
||||
PACKED_ROOT="$BUILD_ROOT/packed"
|
||||
FNPACK_TMP_ROOT="$BUILD_ROOT/fnpack-tmp"
|
||||
WHEEL_BUILD_ROOT="$BUILD_ROOT/application-wheel"
|
||||
OUTPUT="${1:-$PROJECT_ROOT/dist/imagefind-${PACKAGE_VERSION}-x86_64.fpk}"
|
||||
WHEEL_CACHE="${IMAGEFIND_WHEEL_CACHE:-$PROJECT_ROOT/.fnos-wheel-cache/python312}"
|
||||
if [ -n "${FNPACK:-}" ]; then
|
||||
FNPACK_BIN="$FNPACK"
|
||||
elif [ -x "$PROJECT_ROOT/.tools/fnpack" ]; then
|
||||
FNPACK_BIN="$PROJECT_ROOT/.tools/fnpack"
|
||||
else
|
||||
FNPACK_BIN=fnpack
|
||||
fi
|
||||
if [ -n "${PYTHON:-}" ]; then
|
||||
PYTHON_BIN="$PYTHON"
|
||||
elif [ -x "$PROJECT_ROOT/.release-venv/bin/python" ]; then
|
||||
PYTHON_BIN="$PROJECT_ROOT/.release-venv/bin/python"
|
||||
elif [ -x "$PROJECT_ROOT/.venv/bin/python" ]; then
|
||||
PYTHON_BIN="$PROJECT_ROOT/.venv/bin/python"
|
||||
else
|
||||
PYTHON_BIN=python3
|
||||
fi
|
||||
|
||||
require_file() {
|
||||
if [ ! -e "$1" ]; then
|
||||
printf 'missing required release artifact: %s\n' "$1" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
FNPACK_BIN=$(command -v "$FNPACK_BIN") || {
|
||||
printf 'fnpack is required; install it from the fnOS developer portal.\n' >&2
|
||||
exit 1
|
||||
}
|
||||
PYTHON_BIN=$(command -v "$PYTHON_BIN") || {
|
||||
printf 'Python with hatchling is required to assemble the release.\n' >&2
|
||||
exit 1
|
||||
}
|
||||
command -v npm >/dev/null 2>&1 || { printf 'npm is required.\n' >&2; exit 1; }
|
||||
"$PYTHON_BIN" -c 'import hatchling' >/dev/null 2>&1 || {
|
||||
printf 'hatchling is required in the release Python environment.\n' >&2
|
||||
exit 1
|
||||
}
|
||||
for artifact in libOpenCL.so.1; do
|
||||
require_file "$PROJECT_ROOT/vendor/$artifact"
|
||||
done
|
||||
require_file "$PROJECT_ROOT/requirements/runtime-core.txt"
|
||||
require_file "$PROJECT_ROOT/requirements/runtime-ai/constraints-cp312.txt"
|
||||
[ -d "$WHEEL_CACHE" ] || { printf 'missing Python 3.12 core wheelhouse: %s\n' "$WHEEL_CACHE" >&2; exit 1; }
|
||||
|
||||
rm -rf "$BUILD_ROOT"
|
||||
mkdir -p "$STAGE/app/bin" "$STAGE/app/frontend" "$STAGE/app/runtime/ai" \
|
||||
"$STAGE/app/runtime/wheels" "$PACKED_ROOT" "$FNPACK_TMP_ROOT" \
|
||||
"$WHEEL_BUILD_ROOT" "$(dirname "$OUTPUT")"
|
||||
|
||||
npm run build --prefix "$PROJECT_ROOT/frontend"
|
||||
"$PYTHON_BIN" -m hatchling build -t wheel -d "$WHEEL_BUILD_ROOT"
|
||||
|
||||
cp -R "$PROJECT_ROOT/fnos/." "$STAGE/"
|
||||
sed -i -E "s/^version[[:space:]]*=.*/version=${PACKAGE_VERSION}/" "$STAGE/manifest"
|
||||
cp -R "$PROJECT_ROOT/frontend/dist/." "$STAGE/app/frontend/"
|
||||
cp -p "$PROJECT_ROOT/vendor/libOpenCL.so.1" "$STAGE/app/bin/libOpenCL.so.1"
|
||||
cp -p "$PROJECT_ROOT/requirements/runtime-core.txt" "$STAGE/app/runtime/runtime-core.txt"
|
||||
cp -p "$PROJECT_ROOT/requirements/runtime-ai/"*.txt "$STAGE/app/runtime/ai/"
|
||||
cp -p "$WHEEL_CACHE/"*.whl "$STAGE/app/runtime/wheels/"
|
||||
cp -p "$WHEEL_BUILD_ROOT/"imagefind-*.whl "$STAGE/app/runtime/"
|
||||
STAGED_APP_WHEEL=""
|
||||
for candidate in "$STAGE/app/runtime/"imagefind-*.whl; do
|
||||
if [ -f "$candidate" ]; then
|
||||
STAGED_APP_WHEEL="$candidate"
|
||||
break
|
||||
fi
|
||||
done
|
||||
[ -n "$STAGED_APP_WHEEL" ] || { printf 'built application wheel is missing\n' >&2; exit 1; }
|
||||
PACKAGE_ID=$("$PYTHON_BIN" -c \
|
||||
'import hashlib,sys; h=hashlib.sha256(); [h.update(open(path,"rb").read()) for path in sys.argv[1:]]; print(h.hexdigest())' \
|
||||
"$STAGED_APP_WHEEL" "$STAGE/app/runtime/runtime-core.txt")
|
||||
printf '%s\n%s\n' "$PACKAGE_VERSION" "$PACKAGE_ID" >"$STAGE/app/runtime/VERSION"
|
||||
"$PYTHON_BIN" "$PROJECT_ROOT/scripts/make_icons.py" "$STAGE"
|
||||
chmod 0755 "$STAGE/cmd/"*
|
||||
(
|
||||
cd "$PACKED_ROOT"
|
||||
TMPDIR="$FNPACK_TMP_ROOT" "$FNPACK_BIN" build --directory "$STAGE"
|
||||
)
|
||||
mv "$PACKED_ROOT/imagefind.fpk" "$OUTPUT"
|
||||
(
|
||||
cd "$(dirname "$OUTPUT")"
|
||||
sha256sum "$(basename "$OUTPUT")" >"$(basename "$OUTPUT").sha256"
|
||||
)
|
||||
printf 'created %s\n' "$OUTPUT"
|
||||
@@ -0,0 +1,84 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import tarfile
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def digest(path: Path) -> str:
|
||||
value = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
while chunk := handle.read(1024 * 1024):
|
||||
value.update(chunk)
|
||||
return value.hexdigest()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Build a versioned ImageFind model bundle")
|
||||
parser.add_argument("source", type=Path)
|
||||
parser.add_argument("output", type=Path)
|
||||
parser.add_argument("--version", default="1")
|
||||
args = parser.parse_args()
|
||||
source = args.source.resolve()
|
||||
output = args.output.resolve()
|
||||
for required in ("visual/image", "visual/text"):
|
||||
if not (source / required).is_dir():
|
||||
parser.error(f"missing {required}")
|
||||
optional_components = {
|
||||
"ocr": ("det.onnx", "rec.onnx", "cls.onnx"),
|
||||
"faces": ("detector.xml", "detector.bin", "reidentification.xml", "reidentification.bin"),
|
||||
}
|
||||
for component, required_files in optional_components.items():
|
||||
root = source / component
|
||||
if root.exists():
|
||||
missing = [name for name in required_files if not (root / name).is_file()]
|
||||
if missing:
|
||||
parser.error(f"incomplete {component}: missing {', '.join(missing)}")
|
||||
audio_root = source / "audio"
|
||||
if audio_root.exists() and (not (audio_root / "config.json").is_file() or not list(audio_root.glob("*.xml"))):
|
||||
parser.error("incomplete audio: missing config.json or OpenVINO XML files")
|
||||
try:
|
||||
output.relative_to(source)
|
||||
except ValueError:
|
||||
pass
|
||||
else:
|
||||
parser.error("output must be outside the source model directory")
|
||||
|
||||
files = sorted(
|
||||
path for path in source.rglob("*") if path.is_file() and path.relative_to(source).as_posix() != "manifest.json"
|
||||
)
|
||||
created_at = datetime.now(UTC)
|
||||
manifest = {
|
||||
"format_version": 2,
|
||||
"version": args.version,
|
||||
"created_at": created_at.isoformat(),
|
||||
"source": "imagefind-build-model-bundle",
|
||||
"components": {
|
||||
"visual": {"version": args.version},
|
||||
**({"ocr": {"version": args.version}} if (source / "ocr").is_dir() else {}),
|
||||
**({"faces": {"version": args.version}} if (source / "faces").is_dir() else {}),
|
||||
**({"audio": {"version": args.version}} if audio_root.is_dir() else {}),
|
||||
},
|
||||
"files": {path.relative_to(source).as_posix(): digest(path) for path in files},
|
||||
}
|
||||
manifest_bytes = (json.dumps(manifest, ensure_ascii=False, indent=2) + "\n").encode()
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
with tarfile.open(output, "w:gz") as archive:
|
||||
for path in sorted(source.rglob("*")):
|
||||
if path.relative_to(source).as_posix() == "manifest.json":
|
||||
continue
|
||||
archive.add(path, path.relative_to(source).as_posix(), recursive=False)
|
||||
info = tarfile.TarInfo("manifest.json")
|
||||
info.size = len(manifest_bytes)
|
||||
info.mode = 0o644
|
||||
info.mtime = int(created_at.timestamp())
|
||||
archive.addfile(info, io.BytesIO(manifest_bytes))
|
||||
print(f"bundle={output} sha256={digest(output)}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,9 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
|
||||
python3 -m compileall -q "$PROJECT_ROOT/scripts" "$PROJECT_ROOT/backend/imagefind"
|
||||
while IFS= read -r script; do
|
||||
node --check "$script"
|
||||
done < <(find "$PROJECT_ROOT/scripts" -maxdepth 1 -type f -name '*.mjs' -print | sort)
|
||||
Executable
+10
@@ -0,0 +1,10 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
: "${JENKINS_NODE_PASSWORD:?JENKINS_NODE_PASSWORD is required}"
|
||||
|
||||
printf '%s\n' "$JENKINS_NODE_PASSWORD" | sudo -S -p '' -v
|
||||
|
||||
sudo -n -- \
|
||||
/usr/bin/env "DOCKER_CONFIG=${DOCKER_CONFIG:-}" \
|
||||
/usr/bin/docker "$@"
|
||||
Executable
+58
@@ -0,0 +1,58 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
PG_BIN=/usr/lib/postgresql/15/bin
|
||||
PG_DATA=/data/postgresql
|
||||
PG_LOG=/data/postgresql.log
|
||||
APP_DATA=/data/imagefind
|
||||
PG_CONF=/data/postgres-client.conf
|
||||
ADMIN_MARKER=/data/.imagefind-admin-initialized
|
||||
|
||||
mkdir -p "$PG_DATA" "$APP_DATA" /data/runtime
|
||||
chown -R postgres:postgres "$PG_DATA"
|
||||
chown -R imagefind:imagefind "$APP_DATA" /data/runtime
|
||||
|
||||
if [ ! -s "$PG_DATA/PG_VERSION" ]; then
|
||||
runuser -u postgres -- "$PG_BIN/initdb" -D "$PG_DATA" --auth-local=trust --auth-host=scram-sha-256
|
||||
printf '%s\n' "listen_addresses = '127.0.0.1'" "port = 5432" >>"$PG_DATA/postgresql.conf"
|
||||
fi
|
||||
|
||||
runuser -u postgres -- "$PG_BIN/pg_ctl" -D "$PG_DATA" -l "$PG_LOG" -w start
|
||||
stop_postgres() {
|
||||
runuser -u postgres -- "$PG_BIN/pg_ctl" -D "$PG_DATA" -m fast -w stop >/dev/null 2>&1 || true
|
||||
}
|
||||
trap stop_postgres EXIT TERM INT HUP
|
||||
|
||||
if [ ! -s "$PG_CONF" ]; then
|
||||
role_exists=$(runuser -u postgres -- psql -Atqc "SELECT 1 FROM pg_roles WHERE rolname='imagefind'" postgres)
|
||||
db_exists=$(runuser -u postgres -- psql -Atqc "SELECT 1 FROM pg_database WHERE datname='imagefind'" postgres)
|
||||
if [ -n "$role_exists" ] || [ -n "$db_exists" ]; then
|
||||
printf 'PostgreSQL data exists but %s is missing; refusing to replace credentials.\n' "$PG_CONF" >&2
|
||||
exit 1
|
||||
fi
|
||||
db_password=$(python -c 'import secrets; print(secrets.token_urlsafe(48))')
|
||||
runuser -u postgres -- psql -v ON_ERROR_STOP=1 -v password="$db_password" postgres <<'SQL'
|
||||
CREATE ROLE imagefind LOGIN PASSWORD :'password';
|
||||
SQL
|
||||
runuser -u postgres -- createdb --owner=imagefind imagefind
|
||||
runuser -u postgres -- psql -v ON_ERROR_STOP=1 imagefind -c 'CREATE EXTENSION IF NOT EXISTS vector; CREATE EXTENSION IF NOT EXISTS pg_trgm;'
|
||||
umask 077
|
||||
printf 'host=127.0.0.1\nport=5432\ndatabase=imagefind\nusername=imagefind\npassword=%s\nsslmode=Disable\n' \
|
||||
"$db_password" >"$PG_CONF"
|
||||
chown imagefind:imagefind "$PG_CONF"
|
||||
fi
|
||||
|
||||
if [ ! -e "$ADMIN_MARKER" ]; then
|
||||
: "${IMAGEFIND_ADMIN_PASSWORD:?IMAGEFIND_ADMIN_PASSWORD is required for first initialization}"
|
||||
printf '%s' "$IMAGEFIND_ADMIN_PASSWORD" | runuser -u imagefind --preserve-environment -- imagefind admin-password --stdin
|
||||
install -o imagefind -g imagefind -m 0600 /dev/null "$ADMIN_MARKER"
|
||||
fi
|
||||
unset IMAGEFIND_ADMIN_PASSWORD || true
|
||||
|
||||
runuser -u imagefind --preserve-environment -- imagefind &
|
||||
app_pid=$!
|
||||
set +e
|
||||
wait "$app_pid"
|
||||
status=$?
|
||||
set -e
|
||||
exit "$status"
|
||||
@@ -0,0 +1,161 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Read-only live acceptance for audio search and transcript APIs."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
from zhconv import convert
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--base-url", required=True)
|
||||
auth = parser.add_mutually_exclusive_group(required=True)
|
||||
auth.add_argument("--password-stdin", action="store_true")
|
||||
auth.add_argument("--token-file", type=Path)
|
||||
args = parser.parse_args()
|
||||
password = sys.stdin.readline().rstrip("\r\n") if args.password_stdin else ""
|
||||
headers = {}
|
||||
if args.token_file is not None:
|
||||
headers["Authorization"] = f"Bearer {args.token_file.read_text().strip()}"
|
||||
client = httpx.Client(
|
||||
base_url=args.base_url.rstrip("/"), headers=headers, timeout=httpx.Timeout(30, connect=10)
|
||||
)
|
||||
try:
|
||||
if args.password_stdin:
|
||||
login = client.post(
|
||||
"/api/v1/auth/login", json={"password": password, "remember_device": False}
|
||||
)
|
||||
login.raise_for_status()
|
||||
csrf = str(login.json().get("csrf_token") or "")
|
||||
if not csrf:
|
||||
raise AssertionError("login response omitted CSRF token")
|
||||
client.headers["X-CSRF-Token"] = csrf
|
||||
password = ""
|
||||
|
||||
status = client.get("/api/v1/status")
|
||||
status.raise_for_status()
|
||||
if status.json().get("version") != "0.5.45":
|
||||
raise AssertionError(f"unexpected version: {status.json().get('version')}")
|
||||
|
||||
coverage_response = client.get("/api/v1/search/coverage")
|
||||
coverage_response.raise_for_status()
|
||||
coverage = coverage_response.json()
|
||||
required = {
|
||||
"total",
|
||||
"indexed",
|
||||
"searchable",
|
||||
"empty",
|
||||
"queued",
|
||||
"running",
|
||||
"failed",
|
||||
"percent",
|
||||
"current_model_percent",
|
||||
}
|
||||
if not required <= coverage.keys():
|
||||
raise AssertionError(f"coverage fields missing: {sorted(required - coverage.keys())}")
|
||||
|
||||
videos_response = client.get("/api/v1/videos?limit=200")
|
||||
videos_response.raise_for_status()
|
||||
videos = videos_response.json()
|
||||
selected = None
|
||||
transcript = None
|
||||
selected_item = None
|
||||
fallback = None
|
||||
for video in videos:
|
||||
response = client.get(f"/api/v1/videos/{video['id']}/transcript?page=1&page_size=30")
|
||||
response.raise_for_status()
|
||||
candidate = response.json()
|
||||
if candidate.get("items"):
|
||||
fallback = fallback or (video, candidate, candidate["items"][0])
|
||||
chinese_item = next(
|
||||
(
|
||||
item
|
||||
for item in candidate["items"]
|
||||
if re.search(r"[\u3400-\u9fff]{2,}", str(item.get("raw_text") or ""))
|
||||
),
|
||||
None,
|
||||
)
|
||||
if chinese_item:
|
||||
selected, transcript, selected_item = video, candidate, chinese_item
|
||||
break
|
||||
if selected is None and fallback:
|
||||
selected, transcript, selected_item = fallback
|
||||
|
||||
checks: dict[str, object] = {
|
||||
"version": status.json()["version"],
|
||||
"coverage": {key: coverage[key] for key in sorted(required)},
|
||||
"transcript_video_found": bool(selected),
|
||||
}
|
||||
resources_response = client.get("/api/v1/system/resources")
|
||||
resources_response.raise_for_status()
|
||||
resources = resources_response.json()
|
||||
diagnostics_response = client.get("/api/v1/system/diagnostics")
|
||||
diagnostics_response.raise_for_status()
|
||||
diagnostics = diagnostics_response.json()
|
||||
models_response = client.get("/api/v1/models")
|
||||
models_response.raise_for_status()
|
||||
models = models_response.json()
|
||||
checks["runtime"] = {
|
||||
"cpu_percent": resources.get("cpu_percent"),
|
||||
"memory_available_bytes": resources.get("memory_available_bytes"),
|
||||
"io": resources.get("io"),
|
||||
"ai_lane": (resources.get("lanes") or {}).get("ai"),
|
||||
"database": diagnostics.get("database"),
|
||||
"event_loop_lag_ms": diagnostics.get("event_loop_lag_ms"),
|
||||
"request_p95_ms": diagnostics.get("request_p95_ms"),
|
||||
"audio_accelerator": ((models.get("accelerator") or {}).get("components") or {}).get("audio"),
|
||||
"audio_health": (models.get("component_health") or {}).get("audio"),
|
||||
}
|
||||
if selected and transcript:
|
||||
first = selected_item or transcript["items"][0]
|
||||
cjk_runs = re.findall(r"[\u3400-\u9fff]{2,}", str(first.get("raw_text") or ""))
|
||||
if cjk_runs:
|
||||
query = cjk_runs[0][: min(6, len(cjk_runs[0]))]
|
||||
variants = list(dict.fromkeys((query, convert(query, "zh-cn"), convert(query, "zh-tw"))))
|
||||
results = []
|
||||
for variant in variants:
|
||||
audio = client.post(
|
||||
"/api/v1/search",
|
||||
json={"text": variant, "recognition_types": ["audio"], "limit": 50},
|
||||
)
|
||||
audio.raise_for_status()
|
||||
audio_ids = {item.get("video_id") for item in audio.json().get("items", [])}
|
||||
combined = client.post("/api/v1/search", json={"text": variant, "limit": 50})
|
||||
combined.raise_for_status()
|
||||
combined_ids = {item.get("video_id") for item in combined.json().get("items", [])}
|
||||
results.append(
|
||||
{
|
||||
"variant_changed": variant != query,
|
||||
"audio_match": selected["id"] in audio_ids,
|
||||
"combined_match": selected["id"] in combined_ids,
|
||||
}
|
||||
)
|
||||
if not all(item["audio_match"] and item["combined_match"] for item in results):
|
||||
raise AssertionError(f"live Chinese audio search mismatch: {results}")
|
||||
checks["chinese_query_length"] = len(query)
|
||||
checks["variant_checks"] = results
|
||||
checks["transcript"] = {
|
||||
"status": transcript.get("status"),
|
||||
"total": transcript.get("total"),
|
||||
"pages": transcript.get("pages"),
|
||||
"items_returned": len(transcript.get("items", [])),
|
||||
}
|
||||
print(json.dumps(checks, ensure_ascii=False, indent=2), flush=True)
|
||||
finally:
|
||||
password = ""
|
||||
try:
|
||||
if args.password_stdin:
|
||||
client.post("/api/v1/auth/logout")
|
||||
finally:
|
||||
client.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,77 @@
|
||||
#!/usr/bin/env node
|
||||
import { mkdir } from "node:fs/promises";
|
||||
import process from "node:process";
|
||||
import playwright from "../frontend/node_modules/@playwright/test/index.js";
|
||||
|
||||
const { chromium } = playwright;
|
||||
|
||||
const chunks = [];
|
||||
for await (const chunk of process.stdin) chunks.push(chunk);
|
||||
const password = Buffer.concat(chunks).toString("utf8").trim();
|
||||
if (!password) throw new Error("password is required on stdin");
|
||||
const baseURL = String(process.argv[2] || "").replace(/\/$/, "");
|
||||
if (!baseURL) throw new Error("base URL is required");
|
||||
const output = new URL("../.playwright-results/live-0516/", import.meta.url).pathname;
|
||||
await mkdir(output, { recursive: true });
|
||||
|
||||
const browser = await chromium.launch({ headless: true });
|
||||
const reports = [];
|
||||
try {
|
||||
for (const viewport of [{ name: "desktop", width: 1440, height: 900 }, { name: "mobile", width: 390, height: 844 }]) {
|
||||
const context = await browser.newContext({ viewport });
|
||||
const page = await context.newPage();
|
||||
await page.goto(baseURL, { waitUntil: "domcontentloaded" });
|
||||
const passwordInput = page.getByLabel("管理员密码");
|
||||
if (await passwordInput.isVisible().catch(() => false)) {
|
||||
await passwordInput.fill(password);
|
||||
const remember = page.locator('.remember-device input[type="checkbox"]');
|
||||
if (await remember.isChecked()) await remember.uncheck();
|
||||
await page.getByRole("button", { name: "登录", exact: true }).click();
|
||||
}
|
||||
try {
|
||||
await page.locator(".app-shell").waitFor({ state: "visible", timeout: 30_000 });
|
||||
await page.locator(".video-card,.home-empty").first().waitFor({ state: "visible", timeout: 30_000 });
|
||||
} catch (error) {
|
||||
const message = (await page.locator("body").innerText()).replace(/\s+/g, " ").slice(0, 500);
|
||||
throw new Error(`${viewport.name} app shell did not become ready: ${message}`, { cause: error });
|
||||
}
|
||||
await page.locator(".nav-item-2").click();
|
||||
await page.locator(".search-coverage").waitFor({ state: "visible", timeout: 15_000 });
|
||||
await page.waitForFunction(
|
||||
() => !document.querySelector(".search-coverage")?.textContent?.includes("正在读取"),
|
||||
undefined,
|
||||
{ timeout: 30_000 },
|
||||
);
|
||||
const coverage = (await page.locator(".search-coverage").innerText()).replace(/\s+/g, " ").trim();
|
||||
const searchGeometry = await page.evaluate(() => ({
|
||||
viewport: innerWidth,
|
||||
document: document.documentElement.scrollWidth,
|
||||
content: document.querySelector(".content")?.scrollWidth || 0,
|
||||
}));
|
||||
if (searchGeometry.document > searchGeometry.viewport + 1 || searchGeometry.content > searchGeometry.viewport + 1) {
|
||||
throw new Error(`${viewport.name} search page overflows horizontally: ${JSON.stringify(searchGeometry)}`);
|
||||
}
|
||||
await page.screenshot({ path: `${output}/${viewport.name}-search.png`, fullPage: true });
|
||||
await page.locator(".nav-item-1").click();
|
||||
await page.locator(".video-card").first().click();
|
||||
await page.locator(".transcript-panel").waitFor({ state: "visible", timeout: 15_000 });
|
||||
await page.locator(".transcript-toggle").click();
|
||||
await page.waitForFunction(
|
||||
() => !document.querySelector(".transcript-toggle")?.textContent?.includes("正在读取"),
|
||||
undefined,
|
||||
{ timeout: 30_000 },
|
||||
);
|
||||
const transcriptStatus = (await page.locator(".transcript-toggle").innerText()).replace(/\s+/g, " ").trim();
|
||||
const transcriptItems = await page.locator(".transcript-list>button").count();
|
||||
await page.screenshot({ path: `${output}/${viewport.name}-player.png`, fullPage: true });
|
||||
reports.push({ viewport: viewport.name, coverage, transcriptStatus, transcriptItems, searchGeometry });
|
||||
await page.evaluate(async () => {
|
||||
const csrf = sessionStorage.getItem("imagefind:csrf") || "";
|
||||
await fetch("/api/v1/auth/logout", { method: "POST", headers: { "X-CSRF-Token": csrf } });
|
||||
});
|
||||
await context.close();
|
||||
}
|
||||
} finally {
|
||||
await browser.close();
|
||||
}
|
||||
console.log(JSON.stringify(reports, null, 2));
|
||||
@@ -0,0 +1,190 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Exercise mutable live APIs using only run-owned ImageFind records."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
|
||||
def call(client: httpx.Client, method: str, path: str, **kwargs: Any) -> httpx.Response:
|
||||
response = client.request(method, path, **kwargs)
|
||||
if response.is_error:
|
||||
raise RuntimeError(f"{method} {path} -> {response.status_code}: {response.text[:1000]}")
|
||||
return response
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--base-url", required=True)
|
||||
auth = parser.add_mutually_exclusive_group(required=True)
|
||||
auth.add_argument("--token-file", type=Path)
|
||||
auth.add_argument("--password")
|
||||
parser.add_argument("--run-dir", type=Path, required=True)
|
||||
parser.add_argument("--case-id", default="")
|
||||
args = parser.parse_args()
|
||||
state = json.loads((args.run_dir / "state.json").read_text())
|
||||
headers = {}
|
||||
if args.token_file is not None:
|
||||
headers["Authorization"] = f"Bearer {args.token_file.read_text().strip()}"
|
||||
client = httpx.Client(
|
||||
base_url=args.base_url.rstrip("/"),
|
||||
headers=headers,
|
||||
timeout=httpx.Timeout(60, connect=10),
|
||||
)
|
||||
if args.password is not None:
|
||||
login = call(
|
||||
client,
|
||||
"POST",
|
||||
"/api/v1/auth/login",
|
||||
json={"password": args.password, "remember_device": False},
|
||||
).json()
|
||||
csrf = str(login.get("csrf_token") or "")
|
||||
if not csrf:
|
||||
raise AssertionError("login response omitted CSRF token")
|
||||
client.headers["X-CSRF-Token"] = csrf
|
||||
|
||||
case_prefix = f"-{args.case_id.strip()}" if args.case_id.strip() else ""
|
||||
expected_names = {
|
||||
f"{state['run_id']}{case_prefix}-e2e-positive.mp4",
|
||||
f"{state['run_id']}{case_prefix}-e2e-negative.mp4",
|
||||
f"{state['run_id']}{case_prefix}-e2e-fallback.mpg",
|
||||
}
|
||||
videos = call(client, "GET", "/api/v1/videos?limit=500").json()
|
||||
owned = {
|
||||
item["source_key"].rsplit("/", 1)[-1]: item
|
||||
for item in videos
|
||||
if item["source_id"] == state["source_id"]
|
||||
and item["source_key"].rsplit("/", 1)[-1] in expected_names
|
||||
}
|
||||
if set(owned) != expected_names:
|
||||
raise AssertionError(f"unexpected run-owned videos: {sorted(owned)}")
|
||||
positive = owned[f"{state['run_id']}{case_prefix}-e2e-positive.mp4"]
|
||||
video_id = positive["id"]
|
||||
|
||||
metadata = {
|
||||
"title": "星河验收影片 729",
|
||||
"catalog_code": "IF-E2E-729",
|
||||
"studio": "ImageFind 验收工作室",
|
||||
"series": f"{state['run_id']}-合集",
|
||||
"release_date": "2026-08-02",
|
||||
"description": "用于验证标题、番号、片商、合集、日期、简介、演员和分类同步。",
|
||||
"actors": ["测试演员 729"],
|
||||
"tags": [],
|
||||
"tag_ids": [state["tag_id"]],
|
||||
}
|
||||
call(client, "PATCH", f"/api/v1/videos/{video_id}/metadata", json=metadata)
|
||||
call(
|
||||
client,
|
||||
"PATCH",
|
||||
f"/api/v1/videos/{video_id}/state",
|
||||
json={"liked": True, "favorited": True, "progress_ms": 4000, "completed": False},
|
||||
)
|
||||
|
||||
refreshed = call(client, "GET", "/api/v1/videos?limit=500").json()
|
||||
positive = next(item for item in refreshed if item["id"] == video_id)
|
||||
for key in ("title", "catalog_code", "studio", "release_date", "description"):
|
||||
if positive[key] != metadata[key]:
|
||||
raise AssertionError(f"metadata mismatch {key}: {positive[key]!r}")
|
||||
if positive["actors"] != ["测试演员 729"]:
|
||||
raise AssertionError(f"actors mismatch: {positive['actors']}")
|
||||
if positive["tag_items"][0]["id"] != state["tag_id"]:
|
||||
raise AssertionError("tag assignment mismatch")
|
||||
if not positive["liked"] or not positive["favorited"] or positive["progress_ms"] != 4000:
|
||||
raise AssertionError("video state mismatch")
|
||||
|
||||
# Upload completion deliberately precedes the independent AI lane. The
|
||||
# search result contract is frame-based, so wait for basic parsing instead
|
||||
# of racing the freshly queued index job.
|
||||
deadline = time.monotonic() + 300
|
||||
while positive.get("index_state", {}).get("basic") == "pending":
|
||||
if time.monotonic() >= deadline:
|
||||
raise TimeoutError("basic video indexing did not reach a terminal state")
|
||||
time.sleep(2)
|
||||
refreshed = call(client, "GET", "/api/v1/videos?limit=500").json()
|
||||
positive = next(item for item in refreshed if item["id"] == video_id)
|
||||
|
||||
filters = {}
|
||||
for name, query in {
|
||||
"favorite": "favorite=true",
|
||||
"liked": "liked=true",
|
||||
"played": "played_only=true&sort=last_played",
|
||||
}.items():
|
||||
result = call(client, "GET", f"/api/v1/videos?{query}&limit=500").json()
|
||||
filters[name] = [item["id"] for item in result]
|
||||
if video_id not in filters[name]:
|
||||
raise AssertionError(f"{name} filter omitted the updated video")
|
||||
|
||||
profile = call(client, "GET", "/api/v1/profile").json()
|
||||
expected_profile_counts = {"favorites": 1, "likes": 1}
|
||||
for key, minimum in expected_profile_counts.items():
|
||||
if int(profile.get("counts", {}).get(key, 0)) < minimum:
|
||||
raise AssertionError(f"profile count did not include {key}")
|
||||
collection = call(client, "GET", f"/api/v1/collections/{state['collection_id']}").json()
|
||||
if video_id not in [item["id"] for item in collection["videos"]]:
|
||||
raise AssertionError("collection detail omitted the updated video")
|
||||
|
||||
search = call(
|
||||
client,
|
||||
"POST",
|
||||
"/api/v1/search",
|
||||
json={"text": "IF-E2E-729", "recognition_types": ["metadata"]},
|
||||
).json()
|
||||
search_ids = [item["video_id"] for item in search["items"]]
|
||||
search_deferred = positive.get("index_state", {}).get("basic") == "failed"
|
||||
if video_id not in search_ids and not search_deferred:
|
||||
raise AssertionError("metadata search did not return the matching video")
|
||||
negative_search = call(
|
||||
client,
|
||||
"POST",
|
||||
"/api/v1/search",
|
||||
json={"text": "绝不应命中的验收词 9834721", "recognition_types": ["metadata"]},
|
||||
).json()
|
||||
if negative_search["items"]:
|
||||
raise AssertionError("non-matching metadata search unexpectedly returned a video")
|
||||
|
||||
stream = call(
|
||||
client,
|
||||
"GET",
|
||||
f"/api/v1/videos/{video_id}/stream",
|
||||
headers={"Range": "bytes=0-1023"},
|
||||
)
|
||||
if stream.status_code != 206 or len(stream.content) != 1024:
|
||||
raise AssertionError(f"range stream mismatch: {stream.status_code}, {len(stream.content)}")
|
||||
with client.stream("GET", f"/api/v1/videos/{video_id}/download") as download:
|
||||
if download.status_code != 200:
|
||||
raise AssertionError(f"download status {download.status_code}")
|
||||
disposition = download.headers.get("content-disposition", "")
|
||||
if "attachment" not in disposition.lower():
|
||||
raise AssertionError(f"download disposition mismatch: {disposition}")
|
||||
first = next(download.iter_bytes(), b"")
|
||||
if not first:
|
||||
raise AssertionError("download returned an empty body")
|
||||
|
||||
report = {
|
||||
"video_id": video_id,
|
||||
"metadata": {key: positive[key] for key in metadata if key not in {"actors", "tags", "tag_ids"}},
|
||||
"actors": positive["actors"],
|
||||
"filters": filters,
|
||||
"profile_counts": profile.get("counts", {}),
|
||||
"collection_video_count": collection["video_count"],
|
||||
"metadata_search_matches": len(search["items"]),
|
||||
"metadata_search_deferred": search_deferred,
|
||||
"range_stream": {"status": stream.status_code, "bytes": len(stream.content)},
|
||||
"download": {"status": 200, "content_disposition": disposition},
|
||||
}
|
||||
(args.run_dir / "api-report.json").write_text(
|
||||
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
|
||||
)
|
||||
state["positive_video_id"] = video_id
|
||||
(args.run_dir / "state.json").write_text(json.dumps(state, ensure_ascii=False, indent=2) + "\n")
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,89 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { createRequire } from "node:module";
|
||||
|
||||
const require = createRequire(new URL("../frontend/package.json", import.meta.url));
|
||||
const { chromium } = require("playwright");
|
||||
|
||||
const baseUrl = String(process.env.IMAGEFIND_LIVE_URL || "").replace(/\/$/, "");
|
||||
const tokenFile = process.env.IMAGEFIND_LIVE_TOKEN_FILE || "";
|
||||
const outputDir = path.resolve(process.env.IMAGEFIND_LIVE_OUTPUT || "dist/postdeploy-0.5.45-bulk-retry");
|
||||
if (!baseUrl || !tokenFile) throw new Error("IMAGEFIND_LIVE_URL and IMAGEFIND_LIVE_TOKEN_FILE are required");
|
||||
const token = fs.readFileSync(tokenFile, "utf8").trim();
|
||||
fs.mkdirSync(outputDir, { recursive: true });
|
||||
|
||||
async function openTasks(browser, viewport) {
|
||||
const context = await browser.newContext({ viewport, locale: "zh-CN" });
|
||||
const origin = new URL(baseUrl).origin;
|
||||
await context.route("**/*", async route => {
|
||||
const url = new URL(route.request().url());
|
||||
if (url.origin === origin && url.pathname.includes("/api/")) {
|
||||
await route.continue({
|
||||
headers: { ...route.request().headers(), authorization: `Bearer ${token}` },
|
||||
});
|
||||
return;
|
||||
}
|
||||
await route.continue();
|
||||
});
|
||||
const page = await context.newPage();
|
||||
await page.goto(baseUrl, { waitUntil: "domcontentloaded", timeout: 30_000 });
|
||||
await page.locator(".app-shell").waitFor({ timeout: 30_000 });
|
||||
if (viewport.width <= 720) {
|
||||
await page.getByRole("button", { name: "更多" }).click();
|
||||
await page.locator(".mobile-more-menu").getByRole("button", { name: "设置" }).click();
|
||||
} else {
|
||||
await page.locator(".nav-item-8").click();
|
||||
}
|
||||
await page.getByRole("heading", { name: "设置" }).waitFor({ timeout: 20_000 });
|
||||
await page.getByRole("button", { name: "任务与偏好" }).click();
|
||||
await page.locator(".jobs-panel").waitFor({ timeout: 20_000 });
|
||||
await page
|
||||
.getByRole("navigation", { name: "后台任务通道" })
|
||||
.getByRole("button", { name: "上传转存" })
|
||||
.click();
|
||||
await page.locator(".job-retry-all").waitFor({ timeout: 20_000 });
|
||||
return { context, page };
|
||||
}
|
||||
|
||||
const browser = await chromium.launch({ headless: true });
|
||||
const report = {};
|
||||
try {
|
||||
{
|
||||
const { context, page } = await openTasks(browser, { width: 390, height: 844 });
|
||||
const button = page.locator(".job-retry-all");
|
||||
const box = await button.boundingBox();
|
||||
report.mobile = {
|
||||
button: (await button.innerText()).replace(/\s+/g, " ").trim(),
|
||||
min_touch_target: Boolean(box && box.height >= 44),
|
||||
overflow: await page.evaluate(() => document.documentElement.scrollWidth - innerWidth),
|
||||
};
|
||||
await page.screenshot({ path: path.join(outputDir, "mobile-transfer-retry.png"), fullPage: true });
|
||||
await context.close();
|
||||
}
|
||||
{
|
||||
const { context, page } = await openTasks(browser, { width: 1440, height: 900 });
|
||||
const button = page.locator(".job-retry-all");
|
||||
report.desktop = { button: (await button.innerText()).replace(/\s+/g, " ").trim() };
|
||||
await button.click();
|
||||
const dialog = page.getByRole("alertdialog");
|
||||
report.desktop.confirmation = (await dialog.innerText()).replace(/\s+/g, " ").trim();
|
||||
await page.screenshot({ path: path.join(outputDir, "desktop-confirmation.png"), fullPage: true });
|
||||
const retryResponse = page.waitForResponse(
|
||||
response => response.url().includes("/api/v1/jobs/retry-failed") && response.request().method() === "POST",
|
||||
);
|
||||
await dialog.getByRole("button", { name: "继续" }).click();
|
||||
const response = await retryResponse;
|
||||
if (!response.ok()) throw new Error(`bulk retry failed: ${response.status()} ${await response.text()}`);
|
||||
report.desktop.result = await response.json();
|
||||
await page.waitForTimeout(1_000);
|
||||
report.desktop.button_visible_after_retry = (await page.locator(".job-retry-all").count()) > 0;
|
||||
await page.screenshot({ path: path.join(outputDir, "desktop-result.png"), fullPage: true });
|
||||
await context.close();
|
||||
}
|
||||
} finally {
|
||||
await browser.close();
|
||||
}
|
||||
fs.writeFileSync(path.join(outputDir, "report.json"), `${JSON.stringify(report, null, 2)}\n`);
|
||||
console.log(JSON.stringify(report, null, 2));
|
||||
@@ -0,0 +1,116 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Remove narrowly prefixed records created by live acceptance runs."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
|
||||
def call(client: httpx.Client, method: str, path: str, **kwargs: Any) -> httpx.Response:
|
||||
response = client.request(method, path, **kwargs)
|
||||
if response.is_error:
|
||||
raise RuntimeError(f"{method} {path} -> {response.status_code}: {response.text[:1000]}")
|
||||
return response
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--base-url", required=True)
|
||||
auth = parser.add_mutually_exclusive_group(required=True)
|
||||
auth.add_argument("--token-file", type=Path)
|
||||
auth.add_argument("--password")
|
||||
parser.add_argument("--prefix", required=True)
|
||||
parser.add_argument("--keep", action="append", default=[])
|
||||
parser.add_argument("--report", type=Path)
|
||||
args = parser.parse_args()
|
||||
if len(args.prefix) < 12 or not args.prefix.startswith("imagefind-e2e-"):
|
||||
raise ValueError("cleanup prefix must be a narrowly scoped imagefind-e2e-* value")
|
||||
headers = {}
|
||||
if args.token_file is not None:
|
||||
headers["Authorization"] = f"Bearer {args.token_file.read_text().strip()}"
|
||||
client = httpx.Client(
|
||||
base_url=args.base_url.rstrip("/"),
|
||||
headers=headers,
|
||||
timeout=httpx.Timeout(120, connect=10),
|
||||
)
|
||||
if args.password is not None:
|
||||
login = call(
|
||||
client,
|
||||
"POST",
|
||||
"/api/v1/auth/login",
|
||||
json={"password": args.password, "remember_device": False},
|
||||
).json()
|
||||
csrf = str(login.get("csrf_token") or "")
|
||||
if not csrf:
|
||||
raise AssertionError("login response omitted CSRF token")
|
||||
client.headers["X-CSRF-Token"] = csrf
|
||||
keep = set(args.keep)
|
||||
report: dict[str, Any] = {
|
||||
"prefix": args.prefix,
|
||||
"videos": [],
|
||||
"collections": [],
|
||||
"tags": [],
|
||||
"tag_groups": [],
|
||||
"people_hidden": [],
|
||||
}
|
||||
|
||||
rows = call(client, "GET", "/api/v1/videos?limit=500").json()
|
||||
for video in rows:
|
||||
filename = str(video.get("source_key") or "").rsplit("/", 1)[-1]
|
||||
if args.prefix not in str(video.get("source_key") or "") or filename in keep:
|
||||
continue
|
||||
deleted = call(client, "DELETE", f"/api/v1/videos/{video['id']}?delete_source=true").json()
|
||||
if trash_id := deleted.get("trash_id"):
|
||||
call(client, "DELETE", f"/api/v1/trash/{trash_id}")
|
||||
report["videos"].append({"id": video["id"], "filename": filename, **deleted})
|
||||
|
||||
for collection in call(client, "GET", "/api/v1/collections").json():
|
||||
if not str(collection.get("name") or "").startswith(args.prefix):
|
||||
continue
|
||||
detail = call(client, "GET", f"/api/v1/collections/{collection['id']}").json()
|
||||
if detail.get("video_count"):
|
||||
continue
|
||||
result = call(client, "DELETE", f"/api/v1/collections/{collection['id']}").json()
|
||||
report["collections"].append(result)
|
||||
|
||||
for group in call(client, "GET", "/api/v1/tag-groups").json():
|
||||
if not str(group.get("name") or "").startswith(args.prefix):
|
||||
continue
|
||||
tags = call(client, "GET", f"/api/v1/tags?group_id={group['id']}").json()
|
||||
if any(int(tag.get("video_count") or 0) for tag in tags):
|
||||
continue
|
||||
for tag in tags:
|
||||
call(client, "DELETE", f"/api/v1/tags/{tag['id']}")
|
||||
report["tags"].append(tag["id"])
|
||||
call(client, "DELETE", f"/api/v1/tag-groups/{group['id']}")
|
||||
report["tag_groups"].append(group["id"])
|
||||
|
||||
for source in call(client, "GET", "/api/v1/sources").json():
|
||||
for item in call(client, "GET", f"/api/v1/trash?source_id={source['id']}").json():
|
||||
if args.prefix in str(item.get("display_name") or ""):
|
||||
call(client, "DELETE", f"/api/v1/trash/{item['id']}")
|
||||
|
||||
# Named people are deliberately preserved when their last face is removed.
|
||||
# There is no destructive people endpoint, so hide narrowly prefixed test
|
||||
# identities after their fixture videos have been deleted.
|
||||
for person in call(client, "GET", "/api/v1/people").json():
|
||||
name = str(person.get("name") or "")
|
||||
if not name.startswith(args.prefix) or person.get("hidden"):
|
||||
continue
|
||||
call(client, "PATCH", f"/api/v1/people/{person['id']}", json={"name": name, "hidden": True})
|
||||
report["people_hidden"].append(person["id"])
|
||||
|
||||
encoded = json.dumps(report, ensure_ascii=False, indent=2) + "\n"
|
||||
if args.report:
|
||||
args.report.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.report.write_text(encoded)
|
||||
print(encoded, end="")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,286 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Exercise synchronous video deletion and the live recycle-bin workflow."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
|
||||
def call(client: httpx.Client, method: str, path: str, **kwargs: Any) -> httpx.Response:
|
||||
response = client.request(method, path, **kwargs)
|
||||
if response.is_error:
|
||||
raise RuntimeError(f"{method} {path} -> {response.status_code}: {response.text[:1000]}")
|
||||
return response
|
||||
|
||||
|
||||
def digest(path: Path) -> str:
|
||||
value = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
while block := handle.read(1024 * 1024):
|
||||
value.update(block)
|
||||
return value.hexdigest()
|
||||
|
||||
|
||||
def videos(client: httpx.Client) -> list[dict[str, Any]]:
|
||||
return call(client, "GET", "/api/v1/videos?limit=500").json()
|
||||
|
||||
|
||||
def find_video(client: httpx.Client, source_id: str, filename: str) -> dict[str, Any] | None:
|
||||
return next(
|
||||
(
|
||||
item
|
||||
for item in videos(client)
|
||||
if item.get("source_id") == source_id
|
||||
and item.get("source_key", "").rsplit("/", 1)[-1] == filename
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
|
||||
def upload(
|
||||
client: httpx.Client,
|
||||
fixture: Path,
|
||||
source_id: str,
|
||||
filename: str,
|
||||
title: str,
|
||||
) -> dict[str, Any]:
|
||||
visible = find_video(client, source_id, filename)
|
||||
if visible:
|
||||
return visible
|
||||
existing = call(client, "GET", "/api/v1/uploads?limit=500").json()
|
||||
task = next(
|
||||
(
|
||||
item
|
||||
for item in existing
|
||||
if item.get("filename") == filename
|
||||
and item.get("status") in {"receiving", "queued", "transferring", "indexing"}
|
||||
),
|
||||
None,
|
||||
)
|
||||
if task is None:
|
||||
task = call(
|
||||
client,
|
||||
"POST",
|
||||
"/api/v1/uploads",
|
||||
json={
|
||||
"source_id": source_id,
|
||||
"relative_path": "ingest",
|
||||
"filename": filename,
|
||||
"title": title,
|
||||
"size_bytes": fixture.stat().st_size,
|
||||
"sha256": digest(fixture),
|
||||
"conflict": "skip",
|
||||
},
|
||||
).json()
|
||||
upload_id = task["id"]
|
||||
if task.get("status") != "completed":
|
||||
chunk_size = int(task["chunk_size"])
|
||||
received = {int(value) for value in task.get("received_chunks", task.get("received", []))}
|
||||
with fixture.open("rb") as handle:
|
||||
for index in range(int(task["total_chunks"])):
|
||||
block = handle.read(chunk_size)
|
||||
if index in received:
|
||||
continue
|
||||
call(
|
||||
client,
|
||||
"PUT",
|
||||
f"/api/v1/uploads/{upload_id}/chunks/{index}",
|
||||
content=block,
|
||||
headers={"X-Chunk-SHA256": hashlib.sha256(block).hexdigest()},
|
||||
)
|
||||
call(client, "POST", f"/api/v1/uploads/{upload_id}/complete")
|
||||
deadline = time.monotonic() + 180
|
||||
while time.monotonic() < deadline:
|
||||
tasks = call(client, "GET", "/api/v1/uploads?limit=500").json()
|
||||
task = next(item for item in tasks if item["id"] == upload_id)
|
||||
if task["status"] == "completed":
|
||||
break
|
||||
if task["status"] in {"failed", "cancelled"}:
|
||||
raise AssertionError(f"upload failed: {task.get('message') or task.get('error')}")
|
||||
time.sleep(1)
|
||||
else:
|
||||
raise TimeoutError(f"upload did not complete: {filename}")
|
||||
deadline = time.monotonic() + 60
|
||||
while time.monotonic() < deadline:
|
||||
video = find_video(client, source_id, filename)
|
||||
if video:
|
||||
return video
|
||||
time.sleep(1)
|
||||
raise TimeoutError(f"completed upload did not appear in /videos: {filename}")
|
||||
|
||||
|
||||
def purge_source_key(client: httpx.Client, source_id: str, key: str) -> None:
|
||||
try:
|
||||
result = call(
|
||||
client,
|
||||
"POST",
|
||||
"/api/v1/files/trash",
|
||||
json={"source_id": source_id, "keys": [key]},
|
||||
).json()
|
||||
except RuntimeError as exc:
|
||||
if "404" in str(exc):
|
||||
return
|
||||
raise
|
||||
for trash_id in result.get("ids", []):
|
||||
call(client, "DELETE", f"/api/v1/trash/{trash_id}")
|
||||
|
||||
|
||||
def search_ids(client: httpx.Client, text: str) -> list[str]:
|
||||
result = call(
|
||||
client,
|
||||
"POST",
|
||||
"/api/v1/search",
|
||||
json={"text": text, "recognition_types": ["metadata"]},
|
||||
).json()
|
||||
return [item["video_id"] for item in result.get("items", [])]
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--base-url", required=True)
|
||||
auth = parser.add_mutually_exclusive_group(required=True)
|
||||
auth.add_argument("--token-file", type=Path)
|
||||
auth.add_argument("--password")
|
||||
parser.add_argument("--run-dir", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
state = json.loads((args.run_dir / "state.json").read_text())
|
||||
headers = {}
|
||||
if args.token_file is not None:
|
||||
headers["Authorization"] = f"Bearer {args.token_file.read_text().strip()}"
|
||||
client = httpx.Client(
|
||||
base_url=args.base_url.rstrip("/"),
|
||||
headers=headers,
|
||||
timeout=httpx.Timeout(60, connect=10),
|
||||
)
|
||||
if args.password is not None:
|
||||
login = call(
|
||||
client,
|
||||
"POST",
|
||||
"/api/v1/auth/login",
|
||||
json={"password": args.password, "remember_device": False},
|
||||
).json()
|
||||
csrf = str(login.get("csrf_token") or "")
|
||||
if not csrf:
|
||||
raise AssertionError("login response omitted CSRF token")
|
||||
client.headers["X-CSRF-Token"] = csrf
|
||||
source_id = state["source_id"]
|
||||
run_id = state["run_id"]
|
||||
cases = [
|
||||
{
|
||||
"mode": "trash",
|
||||
"fixture": args.run_dir / "e2e-positive.mp4",
|
||||
"filename": f"{run_id}-delete-trash.mp4",
|
||||
"title": "删除回收验收 481",
|
||||
"key": f"ingest/{run_id}-delete-trash.mp4",
|
||||
},
|
||||
{
|
||||
"mode": "tombstone",
|
||||
"fixture": args.run_dir / "e2e-negative.mp4",
|
||||
"filename": f"{run_id}-delete-only.mp4",
|
||||
"title": "仅移出验收 592",
|
||||
"key": f"ingest/{run_id}-delete-only.mp4",
|
||||
},
|
||||
]
|
||||
report: dict[str, Any] = {}
|
||||
try:
|
||||
trash_case = cases[0]
|
||||
first = upload(
|
||||
client,
|
||||
trash_case["fixture"],
|
||||
source_id,
|
||||
trash_case["filename"],
|
||||
trash_case["title"],
|
||||
)
|
||||
deleted = call(client, "DELETE", f"/api/v1/videos/{first['id']}?delete_source=true").json()
|
||||
if not deleted.get("source_deleted") or not deleted.get("trash_id"):
|
||||
raise AssertionError(f"source deletion result mismatch: {deleted}")
|
||||
if find_video(client, source_id, trash_case["filename"]):
|
||||
raise AssertionError("source-deleted video remained visible")
|
||||
if first["id"] in search_ids(client, "删除回收验收 481"):
|
||||
raise AssertionError("source-deleted video remained in metadata search")
|
||||
trash_rows = call(client, "GET", f"/api/v1/trash?source_id={source_id}").json()
|
||||
trash_item = next((item for item in trash_rows if item["id"] == deleted["trash_id"]), None)
|
||||
if not trash_item:
|
||||
raise AssertionError("deleted source was not listed in recycle bin")
|
||||
restored = call(client, "POST", f"/api/v1/trash/{trash_item['id']}/restore", json={}).json()
|
||||
deadline = time.monotonic() + 90
|
||||
restored_video = None
|
||||
while time.monotonic() < deadline:
|
||||
restored_video = find_video(client, source_id, trash_case["filename"])
|
||||
if restored_video:
|
||||
break
|
||||
time.sleep(1)
|
||||
if not restored_video:
|
||||
raise AssertionError("restored source did not return to /videos")
|
||||
removed_again = call(
|
||||
client,
|
||||
"DELETE",
|
||||
f"/api/v1/videos/{restored_video['id']}?delete_source=true",
|
||||
).json()
|
||||
call(client, "DELETE", f"/api/v1/trash/{removed_again['trash_id']}")
|
||||
report["trash"] = {
|
||||
"deleted": deleted,
|
||||
"restored_key": restored["key"],
|
||||
"restored_video_id": restored_video["id"],
|
||||
"purged": True,
|
||||
}
|
||||
|
||||
tombstone_case = cases[1]
|
||||
second = upload(
|
||||
client,
|
||||
tombstone_case["fixture"],
|
||||
source_id,
|
||||
tombstone_case["filename"],
|
||||
tombstone_case["title"],
|
||||
)
|
||||
removed = call(client, "DELETE", f"/api/v1/videos/{second['id']}?delete_source=false").json()
|
||||
if removed.get("source_deleted") or removed.get("trash_id"):
|
||||
raise AssertionError(f"library-only deletion result mismatch: {removed}")
|
||||
scan = call(client, "POST", f"/api/v1/sources/{source_id}/scan", json={}).json()
|
||||
time.sleep(5)
|
||||
if find_video(client, source_id, tombstone_case["filename"]):
|
||||
raise AssertionError("tombstoned video reappeared after source scan")
|
||||
if second["id"] in search_ids(client, "仅移出验收 592"):
|
||||
raise AssertionError("tombstoned video remained in metadata search")
|
||||
purge_source_key(client, source_id, tombstone_case["key"])
|
||||
report["tombstone"] = {
|
||||
"deleted": removed,
|
||||
"scan_job_id": scan.get("job_id"),
|
||||
"remained_hidden_after_scan": True,
|
||||
"source_cleaned": True,
|
||||
}
|
||||
finally:
|
||||
for case in cases:
|
||||
video = find_video(client, source_id, case["filename"])
|
||||
if video:
|
||||
try:
|
||||
deletion = call(
|
||||
client,
|
||||
"DELETE",
|
||||
f"/api/v1/videos/{video['id']}?delete_source=true",
|
||||
).json()
|
||||
if deletion.get("trash_id"):
|
||||
call(client, "DELETE", f"/api/v1/trash/{deletion['trash_id']}")
|
||||
except Exception:
|
||||
pass
|
||||
else:
|
||||
try:
|
||||
purge_source_key(client, source_id, case["key"])
|
||||
except Exception:
|
||||
pass
|
||||
(args.run_dir / "delete-report.json").write_text(
|
||||
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
|
||||
)
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,119 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Verify visual/OCR/face GPU indexing and named-person search on live data."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
|
||||
def call(client: httpx.Client, method: str, path: str, **kwargs: Any) -> httpx.Response:
|
||||
response = client.request(method, path, **kwargs)
|
||||
if response.is_error:
|
||||
raise RuntimeError(f"{method} {path} -> {response.status_code}: {response.text[:1000]}")
|
||||
return response
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--base-url", required=True)
|
||||
parser.add_argument("--token-file", type=Path, required=True)
|
||||
parser.add_argument("--run-dir", type=Path, required=True)
|
||||
parser.add_argument("--case-id", default="face")
|
||||
parser.add_argument("--timeout", type=int, default=600)
|
||||
args = parser.parse_args()
|
||||
state = json.loads((args.run_dir / "state.json").read_text())
|
||||
token = args.token_file.read_text().strip()
|
||||
client = httpx.Client(
|
||||
base_url=args.base_url.rstrip("/"),
|
||||
headers={"Authorization": f"Bearer {token}"},
|
||||
timeout=httpx.Timeout(60, connect=10),
|
||||
)
|
||||
filename = f"{state['run_id']}-{args.case_id}-e2e-face.mp4"
|
||||
observed: dict[str, list[str]] = {name: [] for name in ("visual", "ocr", "faces")}
|
||||
deadline = time.monotonic() + args.timeout
|
||||
video: dict[str, Any] | None = None
|
||||
while time.monotonic() < deadline:
|
||||
models = call(client, "GET", "/api/v1/models").json()
|
||||
components = models.get("accelerator", {}).get("components", {})
|
||||
for component in observed:
|
||||
device = components.get(component, {}).get("actual_device")
|
||||
if device and device not in observed[component]:
|
||||
observed[component].append(device)
|
||||
videos = call(client, "GET", "/api/v1/videos?limit=500").json()
|
||||
video = next(
|
||||
(
|
||||
item
|
||||
for item in videos
|
||||
if item.get("source_id") == state["source_id"]
|
||||
and str(item.get("source_key") or "").rsplit("/", 1)[-1] == filename
|
||||
),
|
||||
None,
|
||||
)
|
||||
if video and video.get("index_state", {}).get("faces") in {"ready", "failed"}:
|
||||
break
|
||||
time.sleep(1)
|
||||
if not video:
|
||||
raise AssertionError("face fixture never appeared in the video list")
|
||||
if video.get("index_state", {}).get("faces") != "ready":
|
||||
raise AssertionError(f"face indexing failed: {video.get('index_state')} {video.get('error')}")
|
||||
|
||||
matched_person: dict[str, Any] | None = None
|
||||
matched_faces: list[dict[str, Any]] = []
|
||||
for person in call(client, "GET", "/api/v1/people").json():
|
||||
faces = call(client, "GET", f"/api/v1/people/{person['id']}/faces?limit=500").json()
|
||||
owned = [face for face in faces if face.get("video_id") == video["id"]]
|
||||
if owned:
|
||||
matched_person = person
|
||||
matched_faces = owned
|
||||
break
|
||||
if not matched_person:
|
||||
raise AssertionError("the official OpenCV face fixture did not produce a person cluster")
|
||||
person_name = f"{state['run_id']}-人物-729"
|
||||
call(client, "PATCH", f"/api/v1/people/{matched_person['id']}", json={"name": person_name})
|
||||
positive = call(
|
||||
client,
|
||||
"POST",
|
||||
"/api/v1/search",
|
||||
json={"text": person_name, "recognition_types": ["person"], "limit": 100},
|
||||
).json()
|
||||
if video["id"] not in {item["video_id"] for item in positive.get("items", [])}:
|
||||
raise AssertionError("named-person search omitted the face fixture video")
|
||||
negative = call(
|
||||
client,
|
||||
"POST",
|
||||
"/api/v1/search",
|
||||
json={
|
||||
"text": f"{state['run_id']}-不存在人物-9834721",
|
||||
"recognition_types": ["person"],
|
||||
"limit": 100,
|
||||
},
|
||||
).json()
|
||||
if video["id"] in {item["video_id"] for item in negative.get("items", [])}:
|
||||
raise AssertionError("non-matching person search returned the face fixture video")
|
||||
non_gpu = {name: devices for name, devices in observed.items() if not any("GPU" in d.upper() for d in devices)}
|
||||
if non_gpu:
|
||||
raise AssertionError(f"GPU was not observed for all frame components: {non_gpu}")
|
||||
|
||||
report = {
|
||||
"video_id": video["id"],
|
||||
"person_id": matched_person["id"],
|
||||
"person_name": person_name,
|
||||
"face_count": len(matched_faces),
|
||||
"observed_devices": observed,
|
||||
"positive_matches": len(positive.get("items", [])),
|
||||
"negative_matches": len(negative.get("items", [])),
|
||||
}
|
||||
(args.run_dir / "face-report.json").write_text(
|
||||
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
|
||||
)
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,294 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Cover final live gaps without touching records outside the supplied run prefix."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
|
||||
def call(client: httpx.Client, method: str, path: str, **kwargs: Any) -> httpx.Response:
|
||||
response = client.request(method, path, **kwargs)
|
||||
if response.is_error:
|
||||
raise RuntimeError(f"{method} {path} -> {response.status_code}: {response.text[:1000]}")
|
||||
return response
|
||||
|
||||
|
||||
def percentile(values: list[float], ratio: float) -> float:
|
||||
ordered = sorted(values)
|
||||
return ordered[max(0, math.ceil(len(ordered) * ratio) - 1)] if ordered else 0.0
|
||||
|
||||
|
||||
def find_video(client: httpx.Client, source_id: str, filename: str) -> dict[str, Any] | None:
|
||||
return next(
|
||||
(
|
||||
video
|
||||
for video in call(client, "GET", "/api/v1/videos?limit=500").json()
|
||||
if video.get("source_id") == source_id
|
||||
and str(video.get("source_key") or "").rsplit("/", 1)[-1] == filename
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
|
||||
def wait_upload(client: httpx.Client, upload_id: str, timeout: int = 180) -> dict[str, Any]:
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
upload = next(
|
||||
(item for item in call(client, "GET", "/api/v1/uploads?limit=500").json() if item["id"] == upload_id),
|
||||
None,
|
||||
)
|
||||
if upload and upload["status"] in {"completed", "failed", "cancelled"}:
|
||||
return upload
|
||||
time.sleep(1)
|
||||
raise TimeoutError(f"upload did not reach a terminal state: {upload_id}")
|
||||
|
||||
|
||||
def wait_job(client: httpx.Client, job_id: str, timeout: int = 180) -> dict[str, Any]:
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
job = next(
|
||||
(item for item in call(client, "GET", "/api/v1/jobs?limit=500").json() if item["id"] == job_id),
|
||||
None,
|
||||
)
|
||||
if job and job["status"] in {"completed", "failed", "cancelled"}:
|
||||
return job
|
||||
time.sleep(1)
|
||||
raise TimeoutError(f"job did not reach a terminal state: {job_id}")
|
||||
|
||||
|
||||
def performance_sample(base_url: str, client: httpx.Client) -> dict[str, Any]:
|
||||
paths = (
|
||||
"/api/v1/status",
|
||||
"/api/v1/videos?limit=50",
|
||||
"/api/v1/jobs?page=1&page_size=10",
|
||||
"/api/v1/profile",
|
||||
"/api/v1/models",
|
||||
"/api/v1/system/resources",
|
||||
)
|
||||
|
||||
def request(path: str) -> tuple[str, float, int]:
|
||||
started = time.perf_counter()
|
||||
response = httpx.get(
|
||||
base_url + path,
|
||||
headers=dict(client.headers),
|
||||
cookies=dict(client.cookies),
|
||||
timeout=httpx.Timeout(20, connect=5),
|
||||
)
|
||||
return path, round((time.perf_counter() - started) * 1000, 1), response.status_code
|
||||
|
||||
samples: dict[str, list[float]] = {path: [] for path in paths}
|
||||
errors: list[dict[str, Any]] = []
|
||||
with ThreadPoolExecutor(max_workers=4) as pool:
|
||||
futures = [pool.submit(request, path) for _ in range(4) for path in paths]
|
||||
for future in as_completed(futures):
|
||||
path, elapsed, status = future.result()
|
||||
samples[path].append(elapsed)
|
||||
if status != 200:
|
||||
errors.append({"path": path, "status": status, "elapsed_ms": elapsed})
|
||||
result = {
|
||||
path: {
|
||||
"count": len(values),
|
||||
"p50_ms": percentile(values, 0.5),
|
||||
"p95_ms": percentile(values, 0.95),
|
||||
"max_ms": max(values, default=0),
|
||||
}
|
||||
for path, values in samples.items()
|
||||
}
|
||||
all_values = [value for values in samples.values() for value in values]
|
||||
result["summary"] = {
|
||||
"requests": len(all_values),
|
||||
"concurrency": 4,
|
||||
"errors": errors,
|
||||
"p50_ms": percentile(all_values, 0.5),
|
||||
"p95_ms": percentile(all_values, 0.95),
|
||||
"max_ms": max(all_values, default=0),
|
||||
}
|
||||
if errors or result["summary"]["p95_ms"] > 5000 or result["summary"]["max_ms"] > 10000:
|
||||
raise AssertionError(f"live performance sample exceeded its guardrail: {result['summary']}")
|
||||
return result
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--base-url", required=True)
|
||||
auth = parser.add_mutually_exclusive_group(required=True)
|
||||
auth.add_argument("--token-file", type=Path)
|
||||
auth.add_argument("--password")
|
||||
parser.add_argument("--run-dir", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
state = json.loads((args.run_dir / "state.json").read_text())
|
||||
run_id = state["run_id"]
|
||||
source_id = state["source_id"]
|
||||
base_url = args.base_url.rstrip("/")
|
||||
headers = {}
|
||||
if args.token_file is not None:
|
||||
headers["Authorization"] = f"Bearer {args.token_file.read_text().strip()}"
|
||||
client = httpx.Client(
|
||||
base_url=base_url,
|
||||
headers=headers,
|
||||
timeout=httpx.Timeout(60, connect=10),
|
||||
)
|
||||
if args.password is not None:
|
||||
login = call(
|
||||
client,
|
||||
"POST",
|
||||
"/api/v1/auth/login",
|
||||
json={"password": args.password, "remember_device": False},
|
||||
).json()
|
||||
csrf = str(login.get("csrf_token") or "")
|
||||
if not csrf:
|
||||
raise AssertionError("login response omitted CSRF token")
|
||||
client.headers["X-CSRF-Token"] = csrf
|
||||
|
||||
status = call(client, "GET", "/api/v1/status").json()
|
||||
if status.get("version") != "0.5.45" or status.get("access_mode") != "direct":
|
||||
raise AssertionError(f"unexpected live status: {status}")
|
||||
unauthenticated = httpx.get(base_url + "/api/v1/profile", timeout=10)
|
||||
invalid_token = httpx.get(
|
||||
base_url + "/api/v1/profile",
|
||||
headers={"Authorization": "Bearer invalid-final-acceptance-token"},
|
||||
timeout=10,
|
||||
)
|
||||
if (unauthenticated.status_code, invalid_token.status_code) != (401, 401):
|
||||
raise AssertionError("profile endpoint did not reject missing and invalid credentials")
|
||||
|
||||
page_one = call(client, "GET", "/api/v1/jobs?page=1&page_size=10").json()
|
||||
page_two = call(client, "GET", "/api/v1/jobs?page=2&page_size=10").json()
|
||||
first_ids = {item["id"] for item in page_one["items"]}
|
||||
second_ids = {item["id"] for item in page_two["items"]}
|
||||
if first_ids & second_ids or page_one["page"] != 1 or page_two["page"] != 2:
|
||||
raise AssertionError("background job pagination returned overlapping or incorrect pages")
|
||||
lane_totals = {
|
||||
lane: call(client, "GET", f"/api/v1/jobs?page=1&page_size=10&lane={lane}").json()["total"]
|
||||
for lane in ("ai", "transfer", "download", "scan")
|
||||
}
|
||||
if sum(lane_totals.values()) != page_one["total"]:
|
||||
raise AssertionError(f"job lane totals do not match all jobs: {lane_totals} != {page_one['total']}")
|
||||
|
||||
cancel_payload = b"cancelled upload acceptance payload"
|
||||
cancel_name = f"{run_id}-cancel-before-complete.mp4"
|
||||
cancel_upload = call(
|
||||
client,
|
||||
"POST",
|
||||
"/api/v1/uploads",
|
||||
json={
|
||||
"source_id": source_id,
|
||||
"relative_path": "ingest",
|
||||
"filename": cancel_name,
|
||||
"size_bytes": len(cancel_payload),
|
||||
"sha256": hashlib.sha256(cancel_payload).hexdigest(),
|
||||
"conflict": "skip",
|
||||
},
|
||||
).json()
|
||||
cancelled = client.delete(f"/api/v1/uploads/{cancel_upload['id']}")
|
||||
if cancelled.status_code != 204 or find_video(client, source_id, cancel_name):
|
||||
raise AssertionError("cancelled receiving upload leaked into the video catalog")
|
||||
|
||||
invalid_payload = b"ImageFind final acceptance: intentionally invalid mp4\n"
|
||||
invalid_name = f"{run_id}-manual-retry-invalid.mp4"
|
||||
before_job_ids = {item["id"] for item in call(client, "GET", "/api/v1/jobs?limit=500").json()}
|
||||
invalid_upload = call(
|
||||
client,
|
||||
"POST",
|
||||
"/api/v1/uploads",
|
||||
json={
|
||||
"source_id": source_id,
|
||||
"relative_path": "ingest",
|
||||
"filename": invalid_name,
|
||||
"title": "后台任务手动重试验收",
|
||||
"size_bytes": len(invalid_payload),
|
||||
"sha256": hashlib.sha256(invalid_payload).hexdigest(),
|
||||
"conflict": "skip",
|
||||
},
|
||||
).json()
|
||||
call(
|
||||
client,
|
||||
"PUT",
|
||||
f"/api/v1/uploads/{invalid_upload['id']}/chunks/0",
|
||||
content=invalid_payload,
|
||||
headers={"X-Chunk-SHA256": hashlib.sha256(invalid_payload).hexdigest()},
|
||||
)
|
||||
call(client, "POST", f"/api/v1/uploads/{invalid_upload['id']}/complete")
|
||||
upload_result = wait_upload(client, invalid_upload["id"])
|
||||
if upload_result["status"] != "completed":
|
||||
raise AssertionError(f"invalid fixture did not reach catalog processing: {upload_result}")
|
||||
|
||||
deadline = time.monotonic() + 180
|
||||
failed_job: dict[str, Any] | None = None
|
||||
invalid_video: dict[str, Any] | None = None
|
||||
while time.monotonic() < deadline:
|
||||
invalid_video = find_video(client, source_id, invalid_name)
|
||||
jobs = call(client, "GET", "/api/v1/jobs?limit=500").json()
|
||||
failed_job = next(
|
||||
(
|
||||
job
|
||||
for job in jobs
|
||||
if job["id"] not in before_job_ids
|
||||
and job["kind"] == "index_video"
|
||||
and job["status"] == "failed"
|
||||
),
|
||||
None,
|
||||
)
|
||||
if invalid_video and failed_job:
|
||||
break
|
||||
time.sleep(1)
|
||||
if not invalid_video or not failed_job:
|
||||
raise AssertionError("invalid video did not produce a run-owned failed index task")
|
||||
retry = call(client, "POST", f"/api/v1/jobs/{failed_job['id']}/retry", json={}).json()
|
||||
if retry.get("retried_from") != failed_job["id"] or retry.get("job_id") == failed_job["id"]:
|
||||
raise AssertionError(f"manual job retry response mismatch: {retry}")
|
||||
retried_job = wait_job(client, retry["job_id"])
|
||||
if retried_job["status"] != "failed" or int(retried_job.get("attempts") or 0) < 1:
|
||||
raise AssertionError(f"retried invalid index job did not execute: {retried_job}")
|
||||
|
||||
deleted = call(client, "DELETE", f"/api/v1/videos/{invalid_video['id']}?delete_source=true").json()
|
||||
if deleted.get("trash_id"):
|
||||
call(client, "DELETE", f"/api/v1/trash/{deleted['trash_id']}")
|
||||
|
||||
resources = call(client, "GET", "/api/v1/system/resources").json()
|
||||
if resources.get("database", {}).get("engine") != "postgresql":
|
||||
raise AssertionError(f"unexpected database resource status: {resources.get('database')}")
|
||||
models = call(client, "GET", "/api/v1/models").json()
|
||||
if not all(models.get("operational_components", {}).get(name) for name in ("visual", "ocr", "faces", "audio")):
|
||||
raise AssertionError(f"one or more AI components are not operational: {models.get('operational_components')}")
|
||||
downloads = call(client, "GET", "/api/v1/downloads/runtime").json()
|
||||
performance = performance_sample(base_url, client)
|
||||
|
||||
report = {
|
||||
"status": status,
|
||||
"authentication": {"missing": 401, "invalid": 401},
|
||||
"jobs": {
|
||||
"total": page_one["total"],
|
||||
"pages": page_one["pages"],
|
||||
"page_size": page_one["page_size"],
|
||||
"pages_disjoint": True,
|
||||
"lane_totals": lane_totals,
|
||||
"failed_job": failed_job,
|
||||
"retry": retry,
|
||||
"retried_job": retried_job,
|
||||
},
|
||||
"cancelled_upload": {"id": cancel_upload["id"], "hidden_from_videos": True},
|
||||
"database": resources.get("database"),
|
||||
"resource_lanes": resources.get("lanes"),
|
||||
"aria2": downloads,
|
||||
"models": models.get("operational_components"),
|
||||
"performance": performance,
|
||||
}
|
||||
(args.run_dir / "final-gaps-report.json").write_text(
|
||||
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
|
||||
)
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,162 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Read-only live acceptance for upload pagination and core API latency."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import statistics
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
|
||||
def call(client: httpx.Client, path: str) -> tuple[Any, float]:
|
||||
started = time.perf_counter()
|
||||
response = client.get(path)
|
||||
elapsed_ms = (time.perf_counter() - started) * 1000
|
||||
if response.is_error:
|
||||
raise RuntimeError(f"GET {path} -> {response.status_code}: {response.text[:500]}")
|
||||
return response.json(), elapsed_ms
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--base-url", required=True)
|
||||
auth = parser.add_mutually_exclusive_group(required=True)
|
||||
auth.add_argument("--token-file", type=Path)
|
||||
auth.add_argument("--password-stdin", action="store_true")
|
||||
parser.add_argument("--report", type=Path)
|
||||
args = parser.parse_args()
|
||||
|
||||
headers: dict[str, str] = {}
|
||||
if args.token_file:
|
||||
headers["Authorization"] = f"Bearer {args.token_file.read_text(encoding='utf-8').strip()}"
|
||||
client = httpx.Client(
|
||||
base_url=args.base_url.rstrip("/"),
|
||||
headers=headers,
|
||||
timeout=httpx.Timeout(30, connect=10),
|
||||
)
|
||||
if args.password_stdin:
|
||||
password = sys.stdin.readline().rstrip("\r\n")
|
||||
login = client.post(
|
||||
"/api/v1/auth/login",
|
||||
json={"password": password, "remember_device": False},
|
||||
)
|
||||
password = ""
|
||||
if login.is_error:
|
||||
raise RuntimeError(f"login -> {login.status_code}: {login.text[:500]}")
|
||||
|
||||
status, status_ms = call(client, "/api/v1/status")
|
||||
if status.get("version") != "0.5.45" or not status.get("configured"):
|
||||
raise AssertionError(f"unexpected status: {status}")
|
||||
|
||||
legacy, legacy_ms = call(client, "/api/v1/uploads?limit=4")
|
||||
if not isinstance(legacy, list) or len(legacy) > 4:
|
||||
raise AssertionError("legacy upload response is no longer a bounded array")
|
||||
|
||||
first, first_ms = call(client, "/api/v1/uploads?page=1&page_size=10")
|
||||
required = {
|
||||
"items",
|
||||
"status_items",
|
||||
"active_count",
|
||||
"failed_count",
|
||||
"page",
|
||||
"page_size",
|
||||
"total",
|
||||
"pages",
|
||||
}
|
||||
if not isinstance(first, dict) or not required.issubset(first):
|
||||
actual = sorted(first) if isinstance(first, dict) else type(first)
|
||||
raise AssertionError(f"upload page contract mismatch: {actual}")
|
||||
if first["page"] != 1 or first["page_size"] != 10 or len(first["items"]) > 10:
|
||||
raise AssertionError("upload first page bounds are invalid")
|
||||
if first["pages"] < 1 or first["total"] < len(first["items"]):
|
||||
raise AssertionError("upload pagination totals are invalid")
|
||||
order = [(str(item.get("created_at") or ""), str(item["id"])) for item in first["items"]]
|
||||
if order != sorted(order, reverse=True):
|
||||
raise AssertionError("upload page ordering is unstable")
|
||||
allowed_statuses = {"receiving", "queued", "transferring", "indexing", "failed"}
|
||||
if any(item.get("status") not in allowed_statuses for item in first["status_items"]):
|
||||
raise AssertionError("upload status summary contains terminal history")
|
||||
active_visible = sum(item.get("status") != "failed" for item in first["status_items"])
|
||||
failed_visible = sum(item.get("status") == "failed" for item in first["status_items"])
|
||||
if first["active_count"] < active_visible or first["failed_count"] < failed_visible:
|
||||
raise AssertionError("upload status counters are smaller than their visible summaries")
|
||||
|
||||
second_ids: list[str] = []
|
||||
second_ms: float | None = None
|
||||
if first["pages"] > 1:
|
||||
second, second_ms = call(client, "/api/v1/uploads?page=2&page_size=10")
|
||||
second_ids = [str(item["id"]) for item in second["items"]]
|
||||
if set(second_ids) & {str(item["id"]) for item in first["items"]}:
|
||||
raise AssertionError("upload pages overlap")
|
||||
|
||||
latency_paths = {
|
||||
"uploads": "/api/v1/uploads?page=1&page_size=10",
|
||||
"videos": "/api/v1/videos?limit=20",
|
||||
"profile": "/api/v1/profile",
|
||||
"resources": "/api/v1/system/resources",
|
||||
}
|
||||
latency: dict[str, dict[str, float]] = {}
|
||||
resource_state: dict[str, Any] = {}
|
||||
for name, path in latency_paths.items():
|
||||
samples: list[float] = []
|
||||
value: Any = None
|
||||
for _ in range(3):
|
||||
value, elapsed_ms = call(client, path)
|
||||
samples.append(elapsed_ms)
|
||||
latency[name] = {
|
||||
"median_ms": round(statistics.median(samples), 2),
|
||||
"max_ms": round(max(samples), 2),
|
||||
}
|
||||
if name == "resources" and isinstance(value, dict):
|
||||
database = value.get("database") or {}
|
||||
resource_state = {
|
||||
"cpu_percent": value.get("cpu_percent"),
|
||||
"memory_available_bytes": value.get("memory_available_bytes"),
|
||||
"running_jobs": value.get("running_jobs"),
|
||||
"paused_jobs": value.get("paused_jobs"),
|
||||
"database": {
|
||||
"engine": database.get("engine"),
|
||||
"pool_in_use": database.get("pool_in_use"),
|
||||
"pool_waiters": database.get("pool_waiters"),
|
||||
"slow_transaction_count": database.get("slow_transaction_count"),
|
||||
},
|
||||
}
|
||||
|
||||
report = {
|
||||
"status": status,
|
||||
"contract": {
|
||||
"legacy_count": len(legacy),
|
||||
"page": first["page"],
|
||||
"page_size": first["page_size"],
|
||||
"page_items": len(first["items"]),
|
||||
"total": first["total"],
|
||||
"pages": first["pages"],
|
||||
"second_page_items": len(second_ids),
|
||||
"status_items": len(first["status_items"]),
|
||||
"active_count": first["active_count"],
|
||||
"failed_count": first["failed_count"],
|
||||
},
|
||||
"initial_latency_ms": {
|
||||
"status": round(status_ms, 2),
|
||||
"legacy_uploads": round(legacy_ms, 2),
|
||||
"first_page": round(first_ms, 2),
|
||||
"second_page": round(second_ms, 2) if second_ms is not None else None,
|
||||
},
|
||||
"latency": latency,
|
||||
"resources": resource_state,
|
||||
}
|
||||
encoded = json.dumps(report, ensure_ascii=False, indent=2) + "\n"
|
||||
if args.report:
|
||||
args.report.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.report.write_text(encoded, encoding="utf-8")
|
||||
print(encoded, end="")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,219 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { createRequire } from "node:module";
|
||||
|
||||
const require = createRequire(new URL("../frontend/package.json", import.meta.url));
|
||||
const { chromium } = require("playwright");
|
||||
|
||||
const baseUrl = (process.env.IMAGEFIND_LIVE_URL || "").replace(/\/$/, "");
|
||||
const tokenFile = process.env.IMAGEFIND_LIVE_TOKEN_FILE || "";
|
||||
const token = tokenFile ? fs.readFileSync(tokenFile, "utf8").trim() : "";
|
||||
const password = process.env.IMAGEFIND_LIVE_PASSWORD || "";
|
||||
const targetVideoText = process.env.IMAGEFIND_LIVE_VIDEO_TEXT || "星河验收影片 729";
|
||||
const mutateMarkers = process.env.IMAGEFIND_LIVE_MUTATE_MARKERS !== "false";
|
||||
const outputDir = path.resolve(process.env.IMAGEFIND_LIVE_OUTPUT || "dist/live-player-acceptance");
|
||||
if (!baseUrl || (!token && !password)) {
|
||||
throw new Error("IMAGEFIND_LIVE_URL and either IMAGEFIND_LIVE_TOKEN_FILE or IMAGEFIND_LIVE_PASSWORD are required");
|
||||
}
|
||||
fs.mkdirSync(outputDir, { recursive: true });
|
||||
|
||||
async function newPage(browser, viewport) {
|
||||
const context = await browser.newContext({ viewport, locale: "zh-CN", acceptDownloads: true });
|
||||
const origin = new URL(baseUrl).origin;
|
||||
if (token) {
|
||||
await context.route("**/*", async route => {
|
||||
const url = new URL(route.request().url());
|
||||
if (url.origin === origin && url.pathname.includes("/api/")) {
|
||||
await route.continue({
|
||||
headers: { ...route.request().headers(), authorization: `Bearer ${token}` },
|
||||
});
|
||||
return;
|
||||
}
|
||||
await route.continue();
|
||||
});
|
||||
}
|
||||
const page = await context.newPage();
|
||||
await page.goto(baseUrl, { waitUntil: "domcontentloaded", timeout: 30_000 });
|
||||
if (password) {
|
||||
const passwordInput = page.getByLabel("管理员密码");
|
||||
if (await passwordInput.isVisible().catch(() => false)) {
|
||||
await passwordInput.fill(password);
|
||||
await page.getByRole("button", { name: "登录", exact: true }).click();
|
||||
}
|
||||
}
|
||||
await page.getByRole("button", { name: "首页", exact: true }).waitFor({ timeout: 30_000 });
|
||||
await page.locator("article.video-card").first().waitFor({ timeout: 30_000 });
|
||||
return { context, page };
|
||||
}
|
||||
|
||||
async function showControls(page) {
|
||||
const stage = page.locator(".custom-player");
|
||||
if ((await stage.getAttribute("class"))?.includes("controls-hidden")) {
|
||||
await page.locator(".player-gesture-surface").click({ position: { x: 20, y: 20 } });
|
||||
await page.waitForTimeout(350);
|
||||
}
|
||||
}
|
||||
|
||||
function attachNetworkAudit(page, errors) {
|
||||
const issues = [];
|
||||
page.on("response", response => {
|
||||
if (response.status() < 400) return;
|
||||
const request = response.request();
|
||||
const url = new URL(response.url());
|
||||
const issue = {
|
||||
method: request.method(),
|
||||
status: response.status(),
|
||||
path: url.pathname,
|
||||
resourceType: request.resourceType(),
|
||||
};
|
||||
issues.push(issue);
|
||||
errors.push(`response: ${issue.method} ${issue.status} ${issue.path} [${issue.resourceType}]`);
|
||||
});
|
||||
page.on("requestfailed", request => {
|
||||
const url = new URL(request.url());
|
||||
const issue = {
|
||||
method: request.method(),
|
||||
status: null,
|
||||
path: url.pathname,
|
||||
resourceType: request.resourceType(),
|
||||
failure: request.failure()?.errorText || "unknown network failure",
|
||||
};
|
||||
issues.push(issue);
|
||||
errors.push(`requestfailed: ${issue.method} ${issue.path} [${issue.resourceType}] ${issue.failure}`);
|
||||
});
|
||||
return issues;
|
||||
}
|
||||
|
||||
async function inspectPlayer(browser, viewport, name, mutateMarkers) {
|
||||
const { context, page } = await newPage(browser, viewport);
|
||||
const errors = [];
|
||||
page.on("pageerror", error => errors.push(`pageerror: ${error.message}`));
|
||||
page.on("console", message => {
|
||||
if (message.type() === "error") errors.push(`console: ${message.text()}`);
|
||||
});
|
||||
const networkIssues = attachNetworkAudit(page, errors);
|
||||
|
||||
const positive = page.locator("article.video-card").filter({ hasText: targetVideoText });
|
||||
if (!(await positive.count())) throw new Error(`${name}: target H.264 fixture was not found: ${targetVideoText}`);
|
||||
await positive.first().click();
|
||||
await page.locator(".player-page").waitFor({ timeout: 20_000 });
|
||||
const video = page.locator(".custom-player video");
|
||||
await page.waitForFunction(() => {
|
||||
const element = document.querySelector(".custom-player video");
|
||||
return element instanceof HTMLVideoElement && element.readyState >= 2 && element.duration > 0;
|
||||
}, null, { timeout: 90_000 });
|
||||
await showControls(page);
|
||||
|
||||
const media = async () => video.evaluate(element => ({
|
||||
paused: element.paused,
|
||||
currentTime: element.currentTime,
|
||||
duration: element.duration,
|
||||
playbackRate: element.playbackRate,
|
||||
readyState: element.readyState,
|
||||
}));
|
||||
const footerPlay = page.locator('.player-control-row > button[aria-label="播放"],.player-control-row > button[aria-label="暂停"]').first();
|
||||
if (!(await media()).paused) {
|
||||
await footerPlay.click();
|
||||
await page.waitForFunction(() => document.querySelector(".custom-player video")?.paused === true);
|
||||
}
|
||||
const beforePlay = await media();
|
||||
await footerPlay.click();
|
||||
await page.waitForFunction(() => document.querySelector(".custom-player video")?.paused === false);
|
||||
await page.waitForTimeout(1_000);
|
||||
const duringPlay = await media();
|
||||
if (duringPlay.currentTime <= beforePlay.currentTime) throw new Error(`${name}: playback time did not advance`);
|
||||
await footerPlay.click();
|
||||
await page.waitForFunction(() => document.querySelector(".custom-player video")?.paused === true);
|
||||
|
||||
await showControls(page);
|
||||
const seekStart = (await media()).currentTime;
|
||||
await page.getByRole("button", { name: "快进 10 秒" }).click();
|
||||
const seekForward = (await media()).currentTime;
|
||||
if (seekForward <= seekStart) throw new Error(`${name}: forward button did not seek`);
|
||||
await page.getByRole("button", { name: "快退 10 秒" }).click();
|
||||
const seekBack = (await media()).currentTime;
|
||||
if (seekBack >= seekForward) throw new Error(`${name}: back button did not seek`);
|
||||
|
||||
await page.locator(".player-rate > button").click();
|
||||
await page.getByRole("menuitem", { name: "1.5×", exact: true }).click();
|
||||
if (Math.abs((await media()).playbackRate - 1.5) > 0.01) throw new Error(`${name}: rate did not change to 1.5x`);
|
||||
|
||||
await page.getByRole("button", { name: "全屏" }).click();
|
||||
await page.waitForFunction(() => Boolean(document.fullscreenElement), null, { timeout: 10_000 });
|
||||
const fullscreen = await page.evaluate(() => document.fullscreenElement?.classList.contains("custom-player"));
|
||||
await page.evaluate(() => document.exitFullscreen());
|
||||
await page.waitForFunction(() => !document.fullscreenElement);
|
||||
|
||||
await showControls(page);
|
||||
await page.getByRole("button", { name: "视频时间点" }).click();
|
||||
const drawer = page.locator(".player-marker-drawer");
|
||||
await drawer.waitFor({ state: "visible" });
|
||||
const initialDots = await page.locator(".player-marker-dot").count();
|
||||
let markerCreated = false;
|
||||
let markerRenamed = false;
|
||||
let markerDeleted = false;
|
||||
if (mutateMarkers) {
|
||||
await drawer.getByRole("button", { name: "标记当前位置" }).click();
|
||||
await page.waitForFunction(count => document.querySelectorAll(".player-marker-dot").length > count, initialDots);
|
||||
markerCreated = true;
|
||||
const newest = drawer.locator(".player-marker-list article").last();
|
||||
await newest.getByRole("button", { name: "编辑时间点" }).click();
|
||||
const dialog = page.getByRole("alertdialog");
|
||||
await dialog.getByLabel("请输入").fill(`自动验收时间点-${Date.now()}`);
|
||||
await dialog.getByRole("button", { name: "确定" }).click();
|
||||
await dialog.waitFor({ state: "hidden" });
|
||||
markerRenamed = true;
|
||||
await drawer.locator(".player-marker-list article").last().getByRole("button", { name: "删除时间点" }).click();
|
||||
const confirm = page.getByRole("alertdialog");
|
||||
await confirm.getByRole("button", { name: "继续" }).click();
|
||||
await page.waitForFunction(count => document.querySelectorAll(".player-marker-dot").length === count, initialDots);
|
||||
markerDeleted = true;
|
||||
}
|
||||
|
||||
const downloadPromise = page.waitForEvent("download", { timeout: 30_000 });
|
||||
await page.locator(".player-actions a.action").filter({ hasText: "下载" }).click();
|
||||
const download = await downloadPromise;
|
||||
const downloadName = download.suggestedFilename();
|
||||
await download.cancel();
|
||||
const auditedNetworkIssues = networkIssues.map(issue => ({
|
||||
...issue,
|
||||
expected: issue.path.endsWith("/download") && issue.failure === "net::ERR_ABORTED",
|
||||
}));
|
||||
const auditedErrors = [...new Set(errors)].filter(
|
||||
value => !(value.includes("/download") && value.includes("net::ERR_ABORTED")),
|
||||
);
|
||||
|
||||
await page.screenshot({ path: path.join(outputDir, `${name}.png`), fullPage: true });
|
||||
const report = {
|
||||
name,
|
||||
viewport,
|
||||
media: await media(),
|
||||
playbackAdvancedSeconds: Number((duringPlay.currentTime - beforePlay.currentTime).toFixed(2)),
|
||||
seek: { before: seekStart, forward: seekForward, back: seekBack },
|
||||
fullscreen,
|
||||
markerDrawer: await drawer.isVisible(),
|
||||
markerCreated,
|
||||
markerRenamed,
|
||||
markerDeleted,
|
||||
downloadName,
|
||||
horizontalOverflow: await page.evaluate(() => document.documentElement.scrollWidth > document.documentElement.clientWidth),
|
||||
networkIssues: auditedNetworkIssues,
|
||||
errors: auditedErrors,
|
||||
};
|
||||
await context.close();
|
||||
return report;
|
||||
}
|
||||
|
||||
const browser = await chromium.launch({ headless: true });
|
||||
try {
|
||||
const results = [
|
||||
await inspectPlayer(browser, { width: 1440, height: 900 }, "desktop", mutateMarkers),
|
||||
await inspectPlayer(browser, { width: 390, height: 844 }, "mobile-390", false),
|
||||
];
|
||||
fs.writeFileSync(path.join(outputDir, "report.json"), `${JSON.stringify(results, null, 2)}\n`);
|
||||
console.log(JSON.stringify(results, null, 2));
|
||||
} finally {
|
||||
await browser.close();
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { createRequire } from "node:module";
|
||||
|
||||
const require = createRequire(new URL("../frontend/package.json", import.meta.url));
|
||||
const { chromium, request } = require("playwright");
|
||||
|
||||
const baseUrl = (process.env.IMAGEFIND_LIVE_URL || "").replace(/\/$/, "");
|
||||
const tokenFile = process.env.IMAGEFIND_LIVE_TOKEN_FILE || "";
|
||||
const token = tokenFile ? fs.readFileSync(tokenFile, "utf8").trim() : "";
|
||||
const password = process.env.IMAGEFIND_LIVE_PASSWORD || "";
|
||||
const runPrefix = process.env.IMAGEFIND_LIVE_RUN_PREFIX || "";
|
||||
const outputDir = path.resolve(process.env.IMAGEFIND_LIVE_OUTPUT || "dist/live-profile-acceptance");
|
||||
if (!baseUrl || (!token && !password)) {
|
||||
throw new Error("IMAGEFIND_LIVE_URL and either IMAGEFIND_LIVE_TOKEN_FILE or IMAGEFIND_LIVE_PASSWORD are required");
|
||||
}
|
||||
fs.mkdirSync(outputDir, { recursive: true });
|
||||
|
||||
const authHeaders = token ? { authorization: `Bearer ${token}` } : {};
|
||||
const api = await request.newContext({ baseURL: baseUrl, extraHTTPHeaders: authHeaders });
|
||||
let csrfToken = "";
|
||||
if (password) {
|
||||
const login = await api.post("/api/v1/auth/login", {
|
||||
data: { password, remember_device: false },
|
||||
});
|
||||
if (!login.ok()) throw new Error(`login failed: ${login.status()} ${await login.text()}`);
|
||||
csrfToken = String((await login.json()).csrf_token || "");
|
||||
if (!csrfToken) throw new Error("login response omitted CSRF token");
|
||||
}
|
||||
const mutationHeaders = csrfToken ? { "X-CSRF-Token": csrfToken } : {};
|
||||
const response = await api.get("/api/v1/videos?limit=500");
|
||||
if (!response.ok()) throw new Error(`unable to list videos: ${response.status()}`);
|
||||
const videos = await response.json();
|
||||
if (videos.length < 2) throw new Error("at least two disposable videos are required");
|
||||
const selectedVideos = (runPrefix
|
||||
? videos.filter(video => String(video.source_key || "").includes(runPrefix))
|
||||
: videos
|
||||
).slice(0, 2);
|
||||
if (selectedVideos.length < 2) throw new Error("at least two matching disposable videos are required");
|
||||
const snapshots = videos.map(video => ({
|
||||
id: video.id,
|
||||
liked: Boolean(video.liked),
|
||||
favorited: Boolean(video.favorited),
|
||||
progress_ms: video.progress_ms || 0,
|
||||
completed: Boolean(video.completed),
|
||||
}));
|
||||
|
||||
async function patchState(id, state) {
|
||||
const result = await api.patch(`/api/v1/videos/${id}/state`, { data: state, headers: mutationHeaders });
|
||||
if (!result.ok()) throw new Error(`state patch failed: ${result.status()} ${await result.text()}`);
|
||||
}
|
||||
|
||||
async function seed(state) {
|
||||
await Promise.all(selectedVideos.map(video => patchState(video.id, state)));
|
||||
}
|
||||
|
||||
async function openPage(browser, viewport) {
|
||||
const context = await browser.newContext({ viewport, locale: "zh-CN" });
|
||||
const origin = new URL(baseUrl).origin;
|
||||
if (token) {
|
||||
await context.route("**/*", async route => {
|
||||
const url = new URL(route.request().url());
|
||||
if (url.origin === origin && url.pathname.includes("/api/")) {
|
||||
await route.continue({ headers: { ...route.request().headers(), ...authHeaders } });
|
||||
return;
|
||||
}
|
||||
await route.continue();
|
||||
});
|
||||
}
|
||||
const page = await context.newPage();
|
||||
await page.goto(baseUrl, { waitUntil: "domcontentloaded", timeout: 30_000 });
|
||||
await page.waitForTimeout(1_000);
|
||||
if (password) {
|
||||
const passwordInput = page.getByLabel("管理员密码");
|
||||
if (await passwordInput.isVisible().catch(() => false)) {
|
||||
await passwordInput.fill(password);
|
||||
await page.getByRole("button", { name: "登录", exact: true }).click();
|
||||
}
|
||||
}
|
||||
await page.locator(".app-shell").waitFor({ timeout: 30_000 });
|
||||
if (viewport.width <= 720) {
|
||||
await page.locator(".mobile-profile-button").click();
|
||||
} else {
|
||||
await page.getByRole("button", { name: "账户菜单" }).click();
|
||||
await page.getByRole("button", { name: "个人中心", exact: true }).click();
|
||||
}
|
||||
await page.locator(".profile-page").waitFor({ timeout: 20_000 });
|
||||
return { context, page };
|
||||
}
|
||||
|
||||
async function bulkAction(page, tab, count, actionLabel) {
|
||||
await page.getByRole("button", { name: tab, exact: true }).click();
|
||||
const section = page.locator(".profile-videos");
|
||||
await section.locator("article.video-card").first().waitFor({ timeout: 20_000 });
|
||||
const before = await section.locator("article.video-card").count();
|
||||
await section.getByRole("button", { name: "管理", exact: true }).click();
|
||||
const choices = section.locator('article.video-card[aria-label^="选择 "]');
|
||||
if ((await choices.count()) < count) throw new Error(`${tab}: not enough selectable cards`);
|
||||
for (let index = 0; index < count; index += 1) await choices.nth(index).click();
|
||||
await section.getByRole("button", { name: actionLabel, exact: true }).click();
|
||||
const dialog = page.getByRole("alertdialog");
|
||||
await dialog.getByRole("button", { name: "继续", exact: true }).click();
|
||||
await section.locator(".loading").waitFor({ state: "hidden", timeout: 20_000 }).catch(() => {});
|
||||
await page.waitForFunction(
|
||||
({ selector, expected }) => document.querySelectorAll(selector).length === expected,
|
||||
{ selector: ".profile-videos article.video-card", expected: before - count },
|
||||
);
|
||||
return { before, after: await section.locator("article.video-card").count(), selected: count };
|
||||
}
|
||||
|
||||
const browser = await chromium.launch({ headless: true });
|
||||
const reports = [];
|
||||
try {
|
||||
await seed({ favorited: true });
|
||||
{
|
||||
const { context, page } = await openPage(browser, { width: 1440, height: 900 });
|
||||
const errors = [];
|
||||
page.on("pageerror", error => errors.push(error.message));
|
||||
const result = await bulkAction(page, "收藏", 1, "取消收藏");
|
||||
await page.screenshot({ path: path.join(outputDir, "desktop-favorite-single.png"), fullPage: true });
|
||||
reports.push({ name: "desktop-favorite-single", ...result, errors });
|
||||
await context.close();
|
||||
}
|
||||
|
||||
await seed({ liked: true });
|
||||
{
|
||||
const { context, page } = await openPage(browser, { width: 390, height: 844 });
|
||||
const errors = [];
|
||||
page.on("pageerror", error => errors.push(error.message));
|
||||
const likes = await bulkAction(page, "喜欢", 2, "取消喜欢");
|
||||
await page.screenshot({ path: path.join(outputDir, "mobile-like-multi.png"), fullPage: true });
|
||||
reports.push({ name: "mobile-like-multi", ...likes, errors });
|
||||
await context.close();
|
||||
}
|
||||
|
||||
await seed({ progress_ms: 3_000, completed: false });
|
||||
{
|
||||
const { context, page } = await openPage(browser, { width: 390, height: 844 });
|
||||
const errors = [];
|
||||
page.on("pageerror", error => errors.push(error.message));
|
||||
const history = await bulkAction(page, "观看记录", 2, "清除记录");
|
||||
await page.screenshot({ path: path.join(outputDir, "mobile-history-multi.png"), fullPage: true });
|
||||
reports.push({ name: "mobile-history-multi", ...history, errors });
|
||||
await context.close();
|
||||
}
|
||||
} finally {
|
||||
await Promise.all(snapshots.map(snapshot => patchState(snapshot.id, snapshot)));
|
||||
await browser.close();
|
||||
await api.dispose();
|
||||
}
|
||||
|
||||
fs.writeFileSync(path.join(outputDir, "report.json"), `${JSON.stringify(reports, null, 2)}\n`);
|
||||
console.log(JSON.stringify(reports, null, 2));
|
||||
@@ -0,0 +1,135 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Observe queue drain, API responsiveness, and inference idle reaping live."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
|
||||
def call(client: httpx.Client, path: str) -> tuple[dict[str, Any], float]:
|
||||
started = time.perf_counter()
|
||||
response = client.get(path)
|
||||
elapsed_ms = round((time.perf_counter() - started) * 1000, 1)
|
||||
if response.is_error:
|
||||
raise RuntimeError(f"GET {path} -> {response.status_code}: {response.text[:1000]}")
|
||||
return response.json(), elapsed_ms
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--base-url", required=True)
|
||||
parser.add_argument("--password", required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
parser.add_argument("--timeout", type=int, default=900)
|
||||
parser.add_argument("--idle-seconds", type=int, default=135)
|
||||
args = parser.parse_args()
|
||||
|
||||
client = httpx.Client(
|
||||
base_url=args.base_url.rstrip("/"),
|
||||
timeout=httpx.Timeout(30, connect=10),
|
||||
)
|
||||
login = client.post(
|
||||
"/api/v1/auth/login",
|
||||
json={"password": args.password, "remember_device": False},
|
||||
)
|
||||
login.raise_for_status()
|
||||
|
||||
samples: list[dict[str, Any]] = []
|
||||
idle_started: float | None = None
|
||||
deadline = time.monotonic() + args.timeout
|
||||
last_report = 0.0
|
||||
while time.monotonic() < deadline:
|
||||
resources, resources_ms = call(client, "/api/v1/system/resources")
|
||||
diagnostics, diagnostics_ms = call(client, "/api/v1/system/diagnostics")
|
||||
status, status_ms = call(client, "/api/v1/status")
|
||||
lanes = resources.get("lanes") or {}
|
||||
active = sum(
|
||||
int(lane.get("running") or 0) + int(lane.get("queued") or 0)
|
||||
for lane in lanes.values()
|
||||
)
|
||||
now = time.monotonic()
|
||||
if active:
|
||||
idle_started = None
|
||||
elif idle_started is None:
|
||||
idle_started = now
|
||||
idle_elapsed = 0 if idle_started is None else round(now - idle_started, 1)
|
||||
sample = {
|
||||
"at": time.time(),
|
||||
"version": status.get("version"),
|
||||
"active_jobs": active,
|
||||
"lanes": lanes,
|
||||
"cpu_percent": resources.get("cpu_percent"),
|
||||
"memory_available_bytes": resources.get("memory_available_bytes"),
|
||||
"process_rss_bytes": diagnostics.get("process_rss_bytes"),
|
||||
"event_loop_lag_ms": diagnostics.get("event_loop_lag_ms"),
|
||||
"event_loop_max_lag_ms": diagnostics.get("event_loop_max_lag_ms"),
|
||||
"request_p95_ms": diagnostics.get("request_p95_ms"),
|
||||
"inference": diagnostics.get("inference"),
|
||||
"database": diagnostics.get("database"),
|
||||
"latency_ms": {
|
||||
"resources": resources_ms,
|
||||
"diagnostics": diagnostics_ms,
|
||||
"status": status_ms,
|
||||
},
|
||||
"idle_elapsed_seconds": idle_elapsed,
|
||||
}
|
||||
samples.append(sample)
|
||||
if now - last_report >= 10:
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"active_jobs": active,
|
||||
"idle_seconds": idle_elapsed,
|
||||
"cpu": sample["cpu_percent"],
|
||||
"rss_mb": round(int(sample["process_rss_bytes"] or 0) / 1024**2, 1),
|
||||
"inference_running": bool((sample["inference"] or {}).get("running")),
|
||||
"max_request_ms": max(sample["latency_ms"].values()),
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
flush=True,
|
||||
)
|
||||
last_report = now
|
||||
if idle_elapsed >= args.idle_seconds and not (sample["inference"] or {}).get("running"):
|
||||
break
|
||||
time.sleep(2)
|
||||
else:
|
||||
raise TimeoutError("background queues did not drain and release inference before timeout")
|
||||
|
||||
final = samples[-1]
|
||||
all_latencies = [value for sample in samples for value in sample["latency_ms"].values()]
|
||||
report = {
|
||||
"samples": len(samples),
|
||||
"duration_seconds": round(samples[-1]["at"] - samples[0]["at"], 1),
|
||||
"active_jobs_initial": samples[0]["active_jobs"],
|
||||
"active_jobs_final": final["active_jobs"],
|
||||
"cpu_max_percent": max(float(sample["cpu_percent"] or 0) for sample in samples),
|
||||
"memory_available_min_gb": round(
|
||||
min(int(sample["memory_available_bytes"] or 0) for sample in samples) / 1024**3,
|
||||
2,
|
||||
),
|
||||
"process_rss_initial_mb": round(int(samples[0]["process_rss_bytes"] or 0) / 1024**2, 1),
|
||||
"process_rss_final_mb": round(int(final["process_rss_bytes"] or 0) / 1024**2, 1),
|
||||
"event_loop_lag_final_ms": final["event_loop_lag_ms"],
|
||||
"event_loop_max_lag_ms": final["event_loop_max_lag_ms"],
|
||||
"request_latency_max_ms": max(all_latencies),
|
||||
"inference_final": final["inference"],
|
||||
"database_final": final["database"],
|
||||
}
|
||||
if final["active_jobs"] != 0 or (final["inference"] or {}).get("running"):
|
||||
raise AssertionError(f"resources were not released: {report}")
|
||||
if max(all_latencies) > 5000:
|
||||
raise AssertionError(f"API latency exceeded 5 seconds during soak: {report}")
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,233 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Verify the six recognition filters against run-owned live videos."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
|
||||
def call(client: httpx.Client, method: str, path: str, **kwargs: Any) -> Any:
|
||||
response = client.request(method, path, **kwargs)
|
||||
if response.is_error:
|
||||
raise RuntimeError(f"{method} {path} -> {response.status_code}: {response.text[:1000]}")
|
||||
return response.json()
|
||||
|
||||
|
||||
def find_owned(videos: list[dict[str, Any]], source_id: str, filename: str) -> dict[str, Any]:
|
||||
match = next(
|
||||
(
|
||||
item
|
||||
for item in videos
|
||||
if item.get("source_id") == source_id
|
||||
and str(item.get("source_key") or "").rsplit("/", 1)[-1] == filename
|
||||
),
|
||||
None,
|
||||
)
|
||||
if match is None:
|
||||
raise AssertionError(f"run-owned search fixture is missing: {filename}")
|
||||
return match
|
||||
|
||||
|
||||
def search(
|
||||
client: httpx.Client,
|
||||
*,
|
||||
text: str,
|
||||
recognition_type: str,
|
||||
expected_video_id: str | None,
|
||||
expected_source: str,
|
||||
forbidden_video_id: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
result = call(
|
||||
client,
|
||||
"POST",
|
||||
"/api/v1/search",
|
||||
json={"text": text, "recognition_types": [recognition_type], "limit": 100},
|
||||
)
|
||||
items = result.get("items", [])
|
||||
ids = [item["video_id"] for item in items]
|
||||
if expected_video_id is not None:
|
||||
if expected_video_id not in ids:
|
||||
raise AssertionError(f"{recognition_type} search omitted expected video for {text!r}")
|
||||
matched = next(item for item in items if item["video_id"] == expected_video_id)
|
||||
if expected_source not in matched.get("match_sources", []):
|
||||
raise AssertionError(
|
||||
f"{recognition_type} search returned the target without {expected_source!r}: "
|
||||
f"{matched.get('match_sources')}"
|
||||
)
|
||||
if forbidden_video_id is not None and forbidden_video_id in ids:
|
||||
raise AssertionError(f"{recognition_type} search returned a forbidden video for {text!r}")
|
||||
return {
|
||||
"query": text,
|
||||
"result_count": len(items),
|
||||
"matched_expected": expected_video_id in ids if expected_video_id else not items,
|
||||
"expected_rank": ids.index(expected_video_id) + 1 if expected_video_id in ids else None,
|
||||
"match_sources": sorted(
|
||||
{
|
||||
source
|
||||
for item in items
|
||||
if expected_video_id is None or item["video_id"] == expected_video_id
|
||||
for source in item.get("match_sources", [])
|
||||
}
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--base-url", required=True)
|
||||
parser.add_argument("--token-file", type=Path, required=True)
|
||||
parser.add_argument("--run-dir", type=Path, required=True)
|
||||
parser.add_argument("--timeout", type=int, default=600)
|
||||
args = parser.parse_args()
|
||||
|
||||
state = json.loads((args.run_dir / "state.json").read_text())
|
||||
client = httpx.Client(
|
||||
base_url=args.base_url.rstrip("/"),
|
||||
headers={"Authorization": f"Bearer {args.token_file.read_text().strip()}"},
|
||||
timeout=httpx.Timeout(60, connect=10),
|
||||
)
|
||||
run_id = state["run_id"]
|
||||
source_id = state["source_id"]
|
||||
expected_names = {
|
||||
"positive": f"{run_id}-e2e-positive.mp4",
|
||||
"negative": f"{run_id}-e2e-negative.mp4",
|
||||
"subtitle": f"{run_id}-subtitle-e2e-subtitle.mp4",
|
||||
"audio": f"{run_id}-e2e-speech.mp4",
|
||||
}
|
||||
deadline = time.monotonic() + args.timeout
|
||||
while True:
|
||||
videos = call(client, "GET", "/api/v1/videos?limit=500")
|
||||
fixtures = {name: find_owned(videos, source_id, filename) for name, filename in expected_names.items()}
|
||||
terminal = all(
|
||||
item.get("index_state", {}).get("basic") in {"ready", "failed"}
|
||||
and item.get("index_state", {}).get("visual") in {"ready", "failed", "skipped"}
|
||||
and item.get("index_state", {}).get("ocr") in {"ready", "failed", "skipped"}
|
||||
for item in fixtures.values()
|
||||
)
|
||||
if terminal:
|
||||
break
|
||||
if time.monotonic() >= deadline:
|
||||
raise TimeoutError("search fixtures did not reach terminal basic/visual/OCR states")
|
||||
time.sleep(2)
|
||||
|
||||
positive = fixtures["positive"]["id"]
|
||||
negative = fixtures["negative"]["id"]
|
||||
subtitle = fixtures["subtitle"]["id"]
|
||||
audio = fixtures["audio"]["id"]
|
||||
report: dict[str, Any] = {
|
||||
"fixtures": {name: item["id"] for name, item in fixtures.items()},
|
||||
"index_state": {name: item.get("index_state") for name, item in fixtures.items()},
|
||||
"recognition": {},
|
||||
}
|
||||
report["recognition"]["visual_positive"] = search(
|
||||
client,
|
||||
text="海边日落与蓝色海洋",
|
||||
recognition_type="visual",
|
||||
expected_video_id=positive,
|
||||
expected_source="semantic",
|
||||
)
|
||||
report["recognition"]["visual_negative_fixture"] = search(
|
||||
client,
|
||||
text="绿色山脉与森林",
|
||||
recognition_type="visual",
|
||||
expected_video_id=negative,
|
||||
expected_source="semantic",
|
||||
)
|
||||
report["recognition"]["ocr_positive"] = search(
|
||||
client,
|
||||
text="星河测试 729",
|
||||
recognition_type="ocr",
|
||||
expected_video_id=positive,
|
||||
expected_source="ocr",
|
||||
forbidden_video_id=negative,
|
||||
)
|
||||
report["recognition"]["ocr_negative_fixture"] = search(
|
||||
client,
|
||||
text="山谷样本 314",
|
||||
recognition_type="ocr",
|
||||
expected_video_id=negative,
|
||||
expected_source="ocr",
|
||||
forbidden_video_id=positive,
|
||||
)
|
||||
report["recognition"]["subtitle_positive"] = search(
|
||||
client,
|
||||
text="字幕验证 星河测试729",
|
||||
recognition_type="subtitle",
|
||||
expected_video_id=subtitle,
|
||||
expected_source="subtitle",
|
||||
forbidden_video_id=negative,
|
||||
)
|
||||
report["recognition"]["subtitle_non_match"] = search(
|
||||
client,
|
||||
text="quartz zeppelin 9834721",
|
||||
recognition_type="subtitle",
|
||||
expected_video_id=None,
|
||||
expected_source="subtitle",
|
||||
forbidden_video_id=subtitle,
|
||||
)
|
||||
report["recognition"]["metadata_positive"] = search(
|
||||
client,
|
||||
text="IF-E2E-729",
|
||||
recognition_type="metadata",
|
||||
expected_video_id=positive,
|
||||
expected_source="metadata",
|
||||
forbidden_video_id=negative,
|
||||
)
|
||||
report["recognition"]["metadata_non_match"] = search(
|
||||
client,
|
||||
text="不存在的资料验收词 9834721",
|
||||
recognition_type="metadata",
|
||||
expected_video_id=None,
|
||||
expected_source="metadata",
|
||||
forbidden_video_id=positive,
|
||||
)
|
||||
report["recognition"]["audio_positive"] = search(
|
||||
client,
|
||||
text="image",
|
||||
recognition_type="audio",
|
||||
expected_video_id=audio,
|
||||
expected_source="audio",
|
||||
forbidden_video_id=negative,
|
||||
)
|
||||
report["recognition"]["audio_non_match"] = search(
|
||||
client,
|
||||
text="pineapple submarine 9834721",
|
||||
recognition_type="audio",
|
||||
expected_video_id=None,
|
||||
expected_source="audio",
|
||||
forbidden_video_id=audio,
|
||||
)
|
||||
face_report_path = args.run_dir / "face-report.json"
|
||||
if face_report_path.exists():
|
||||
face = json.loads(face_report_path.read_text())
|
||||
report["recognition"]["person_positive"] = search(
|
||||
client,
|
||||
text=face["person_name"],
|
||||
recognition_type="person",
|
||||
expected_video_id=face["video_id"],
|
||||
expected_source="person",
|
||||
)
|
||||
report["recognition"]["person_non_match"] = search(
|
||||
client,
|
||||
text=f"{run_id}-不存在人物-9834721",
|
||||
recognition_type="person",
|
||||
expected_video_id=None,
|
||||
expected_source="person",
|
||||
forbidden_video_id=face["video_id"],
|
||||
)
|
||||
else:
|
||||
report["recognition"]["person"] = {"status": "pending face acceptance"}
|
||||
|
||||
output = args.run_dir / "search-report.json"
|
||||
output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n")
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,299 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Verify live audio indexing with a run-owned video containing clear speech."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import atexit
|
||||
import hashlib
|
||||
import json
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
|
||||
def call(client: httpx.Client, method: str, path: str, **kwargs: Any) -> httpx.Response:
|
||||
response = client.request(method, path, **kwargs)
|
||||
if response.is_error:
|
||||
raise RuntimeError(f"{method} {path} -> {response.status_code}: {response.text[:1000]}")
|
||||
return response
|
||||
|
||||
|
||||
def sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
while block := handle.read(1024 * 1024):
|
||||
digest.update(block)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def find_video(client: httpx.Client, source_id: str, filename: str) -> dict[str, Any] | None:
|
||||
videos = call(client, "GET", "/api/v1/videos?limit=500").json()
|
||||
return next(
|
||||
(
|
||||
item
|
||||
for item in videos
|
||||
if item.get("source_id") == source_id
|
||||
and item.get("source_key", "").rsplit("/", 1)[-1] == filename
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
|
||||
def upload_fixture(
|
||||
client: httpx.Client,
|
||||
fixture: Path,
|
||||
state: dict[str, Any],
|
||||
) -> tuple[dict[str, Any], list[float]]:
|
||||
filename = f"{state['run_id']}-{fixture.name}"
|
||||
uploads = call(client, "GET", "/api/v1/uploads?limit=500").json()
|
||||
upload = next((item for item in uploads if item.get("filename") == filename), None)
|
||||
if upload is None or upload.get("status") in {"failed", "cancelled"}:
|
||||
upload = call(
|
||||
client,
|
||||
"POST",
|
||||
"/api/v1/uploads",
|
||||
json={
|
||||
"source_id": state["source_id"],
|
||||
"relative_path": "ingest",
|
||||
"filename": filename,
|
||||
"title": "音频 GPU 验收",
|
||||
"collection_id": state["collection_id"],
|
||||
"tag_ids": [state["tag_id"]],
|
||||
"size_bytes": fixture.stat().st_size,
|
||||
"sha256": sha256(fixture),
|
||||
"conflict": "skip",
|
||||
},
|
||||
).json()
|
||||
|
||||
latencies: list[float] = []
|
||||
if upload.get("status") != "completed":
|
||||
hidden = find_video(client, state["source_id"], filename)
|
||||
if hidden is not None:
|
||||
raise AssertionError("unfinished speech upload leaked into /videos")
|
||||
chunk_size = int(upload["chunk_size"])
|
||||
received = {int(index) for index in upload.get("received_chunks", upload.get("received", []))}
|
||||
with fixture.open("rb") as handle:
|
||||
for index in range(int(upload["total_chunks"])):
|
||||
data = handle.read(chunk_size)
|
||||
if index in received:
|
||||
continue
|
||||
started = time.perf_counter()
|
||||
call(
|
||||
client,
|
||||
"PUT",
|
||||
f"/api/v1/uploads/{upload['id']}/chunks/{index}",
|
||||
content=data,
|
||||
headers={"X-Chunk-SHA256": hashlib.sha256(data).hexdigest()},
|
||||
)
|
||||
latencies.append(round((time.perf_counter() - started) * 1000, 1))
|
||||
upload_id = upload["id"]
|
||||
upload = call(client, "POST", f"/api/v1/uploads/{upload_id}/complete").json()
|
||||
else:
|
||||
upload_id = upload["id"]
|
||||
|
||||
deadline = time.monotonic() + 240
|
||||
while upload.get("status") not in {"completed", "failed", "cancelled"}:
|
||||
if time.monotonic() >= deadline:
|
||||
raise TimeoutError(f"speech upload did not finish: {upload.get('status')}")
|
||||
time.sleep(1)
|
||||
uploads = call(client, "GET", "/api/v1/uploads?limit=500").json()
|
||||
upload = next(item for item in uploads if item["id"] == upload_id)
|
||||
if upload["status"] != "completed":
|
||||
raise AssertionError(f"speech upload failed: {upload.get('message')}")
|
||||
return upload, latencies
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--base-url", required=True)
|
||||
auth = parser.add_mutually_exclusive_group(required=True)
|
||||
auth.add_argument("--token-file", type=Path)
|
||||
auth.add_argument("--password")
|
||||
parser.add_argument("--run-dir", type=Path, required=True)
|
||||
parser.add_argument("--fixture", default="e2e-speech.mp4")
|
||||
parser.add_argument(
|
||||
"--query",
|
||||
action="append",
|
||||
dest="queries",
|
||||
help="Required audio-search keyword; repeat to verify multiple words (default: Americans)",
|
||||
)
|
||||
parser.add_argument("--timeout", type=int, default=1200)
|
||||
args = parser.parse_args()
|
||||
|
||||
state_path = args.run_dir / "state.json"
|
||||
state = json.loads(state_path.read_text())
|
||||
fixture = args.run_dir / args.fixture
|
||||
if not fixture.is_file():
|
||||
raise FileNotFoundError(fixture)
|
||||
headers = {}
|
||||
if args.token_file is not None:
|
||||
headers["Authorization"] = f"Bearer {args.token_file.read_text().strip()}"
|
||||
client = httpx.Client(
|
||||
base_url=args.base_url.rstrip("/"),
|
||||
headers=headers,
|
||||
timeout=httpx.Timeout(60, connect=10),
|
||||
)
|
||||
if args.password is not None:
|
||||
login = call(
|
||||
client,
|
||||
"POST",
|
||||
"/api/v1/auth/login",
|
||||
json={"password": args.password, "remember_device": False},
|
||||
).json()
|
||||
csrf = str(login.get("csrf_token") or "")
|
||||
if not csrf:
|
||||
raise AssertionError("login response omitted CSRF token")
|
||||
client.headers["X-CSRF-Token"] = csrf
|
||||
|
||||
def logout() -> None:
|
||||
try:
|
||||
client.post("/api/v1/auth/logout")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
atexit.register(logout)
|
||||
|
||||
# The repository's default live fixture contains the JFK inauguration line
|
||||
# "And so, my fellow Americans...". Keep the default assertion tied to
|
||||
# the actual fixture; callers using another recording can repeat --query.
|
||||
required_queries = [value.strip() for value in (args.queries or ["Americans"]) if value.strip()]
|
||||
if not required_queries:
|
||||
raise ValueError("at least one non-empty --query is required")
|
||||
|
||||
initial_models = call(client, "GET", "/api/v1/models").json()
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"audio_model": (initial_models.get("manifest") or {})
|
||||
.get("components", {})
|
||||
.get("audio"),
|
||||
"audio_health": (initial_models.get("component_health") or {}).get("audio"),
|
||||
"audio_accelerator": (initial_models.get("accelerator") or {})
|
||||
.get("components", {})
|
||||
.get("audio"),
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
flush=True,
|
||||
)
|
||||
|
||||
upload, chunk_latencies = upload_fixture(client, fixture, state)
|
||||
filename = upload["filename"]
|
||||
deadline = time.monotonic() + args.timeout
|
||||
observed_devices: list[str] = []
|
||||
latest_message = "等待视频入库"
|
||||
positives: dict[str, dict[str, Any]] = {}
|
||||
video: dict[str, Any] | None = None
|
||||
last_report_at = 0.0
|
||||
while time.monotonic() < deadline:
|
||||
video = find_video(client, state["source_id"], filename)
|
||||
models = call(client, "GET", "/api/v1/models").json()
|
||||
audio = models.get("accelerator", {}).get("components", {}).get("audio", {})
|
||||
device = audio.get("actual_device") or audio.get("device")
|
||||
if device and device not in observed_devices:
|
||||
observed_devices.append(device)
|
||||
if video is not None:
|
||||
for query in required_queries:
|
||||
if query in positives:
|
||||
continue
|
||||
result = call(
|
||||
client,
|
||||
"POST",
|
||||
"/api/v1/search",
|
||||
json={"text": query, "recognition_types": ["audio"]},
|
||||
).json()
|
||||
positive = next(
|
||||
(item for item in result.get("items", []) if item.get("video_id") == video["id"]),
|
||||
None,
|
||||
)
|
||||
if positive is not None:
|
||||
positives[query] = positive
|
||||
if len(positives) == len(required_queries):
|
||||
break
|
||||
jobs = call(client, "GET", "/api/v1/jobs?page=1&page_size=10").json()["items"]
|
||||
audio_jobs = [item for item in jobs if item.get("kind") == "transcribe_audio"]
|
||||
if audio_jobs:
|
||||
latest_message = audio_jobs[0].get("message") or audio_jobs[0].get("status", "")
|
||||
if audio_jobs[0].get("status") == "failed":
|
||||
raise AssertionError(f"audio job failed: {audio_jobs[0].get('error')}")
|
||||
now = time.monotonic()
|
||||
if now - last_report_at >= 10:
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"waiting": latest_message,
|
||||
"video_found": video is not None,
|
||||
"observed_devices": observed_devices,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
flush=True,
|
||||
)
|
||||
last_report_at = now
|
||||
time.sleep(2)
|
||||
if video is None:
|
||||
raise AssertionError("speech video never appeared in /videos")
|
||||
missing_queries = [query for query in required_queries if query not in positives]
|
||||
if missing_queries:
|
||||
raise TimeoutError(
|
||||
"audio search never matched required keywords "
|
||||
f"{missing_queries}; matched={sorted(positives)}; latest={latest_message}"
|
||||
)
|
||||
for query, positive in positives.items():
|
||||
if positive.get("segment_start_ms") is None or positive.get("segment_end_ms") is None:
|
||||
raise AssertionError(f"audio match omitted time segment for {query!r}: {positive}")
|
||||
match_details = positive.get("match_details", [])
|
||||
audio_text = " ".join(
|
||||
str(detail.get("text") or "")
|
||||
for detail in match_details
|
||||
if detail.get("type") == "audio"
|
||||
).strip()
|
||||
if not audio_text:
|
||||
raise AssertionError(f"audio match omitted recognized text for {query!r}: {positive}")
|
||||
if len([character for character in audio_text if character.isalnum()]) < 8:
|
||||
raise AssertionError(
|
||||
f"audio transcript is implausibly short for {query!r}: {audio_text!r}"
|
||||
)
|
||||
negative = call(
|
||||
client,
|
||||
"POST",
|
||||
"/api/v1/search",
|
||||
json={"text": "pineapple submarine 9834721", "recognition_types": ["audio"]},
|
||||
).json()
|
||||
if any(item.get("video_id") == video["id"] for item in negative.get("items", [])):
|
||||
raise AssertionError("non-matching audio search unexpectedly returned the speech video")
|
||||
|
||||
jobs = call(client, "GET", "/api/v1/jobs?page=1&page_size=10").json()["items"]
|
||||
latest_audio = next((item for item in jobs if item.get("kind") == "transcribe_audio"), None)
|
||||
report = {
|
||||
"upload_id": upload["id"],
|
||||
"video_id": video["id"],
|
||||
"chunk_latency_ms": chunk_latencies,
|
||||
"observed_audio_devices": observed_devices,
|
||||
"audio_job": latest_audio,
|
||||
"matches": {
|
||||
query: {
|
||||
"segment_start_ms": positive.get("segment_start_ms"),
|
||||
"segment_end_ms": positive.get("segment_end_ms"),
|
||||
"details": positive.get("match_details", []),
|
||||
}
|
||||
for query, positive in positives.items()
|
||||
},
|
||||
"negative_matches": len(negative.get("items", [])),
|
||||
}
|
||||
if not any(str(device).upper().startswith("GPU") for device in observed_devices):
|
||||
raise AssertionError(f"GPU was not observed during speech indexing: {observed_devices}")
|
||||
state[f"speech_upload_id_{fixture.stem}"] = upload["id"]
|
||||
state[f"speech_video_id_{fixture.stem}"] = video["id"]
|
||||
state_path.write_text(json.dumps(state, ensure_ascii=False, indent=2) + "\n")
|
||||
(args.run_dir / "speech-report.json").write_text(
|
||||
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
|
||||
)
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,99 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Issue or revoke a short-lived API token used by live acceptance scripts."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
|
||||
|
||||
def call(client: httpx.Client, method: str, path: str, **kwargs) -> httpx.Response:
|
||||
response = client.request(method, path, **kwargs)
|
||||
if response.is_error:
|
||||
raise RuntimeError(f"{method} {path} -> {response.status_code}: {response.text[:1000]}")
|
||||
return response
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--base-url", required=True)
|
||||
password = parser.add_mutually_exclusive_group()
|
||||
password.add_argument("--password")
|
||||
password.add_argument("--password-stdin", action="store_true")
|
||||
password.add_argument("--storage-state", type=Path)
|
||||
parser.add_argument("--token-file", type=Path, required=True)
|
||||
parser.add_argument("--id-file", type=Path, required=True)
|
||||
parser.add_argument("--name", default="ImageFind live acceptance")
|
||||
parser.add_argument("--revoke", action="store_true")
|
||||
args = parser.parse_args()
|
||||
token_revoke_ready = args.revoke and args.token_file.exists() and args.id_file.exists()
|
||||
if not token_revoke_ready and not (args.password or args.password_stdin or args.storage_state):
|
||||
parser.error("one of --password, --password-stdin or --storage-state is required")
|
||||
password_value = sys.stdin.readline().rstrip("\r\n") if args.password_stdin else args.password
|
||||
|
||||
client = httpx.Client(
|
||||
base_url=args.base_url.rstrip("/"),
|
||||
timeout=httpx.Timeout(30, connect=10),
|
||||
)
|
||||
if token_revoke_ready:
|
||||
client.headers["Authorization"] = (
|
||||
f"Bearer {args.token_file.read_text(encoding='utf-8').strip()}"
|
||||
)
|
||||
token_id = args.id_file.read_text(encoding="utf-8").strip()
|
||||
call(client, "DELETE", f"/api/v1/tokens/{token_id}")
|
||||
args.token_file.unlink(missing_ok=True)
|
||||
args.id_file.unlink(missing_ok=True)
|
||||
print(f"revoked live acceptance token {token_id}")
|
||||
return
|
||||
|
||||
if args.storage_state:
|
||||
state = json.loads(args.storage_state.read_text(encoding="utf-8"))
|
||||
for cookie in state.get("cookies", []):
|
||||
client.cookies.set(
|
||||
str(cookie["name"]),
|
||||
str(cookie["value"]),
|
||||
domain=str(cookie.get("domain") or ""),
|
||||
path=str(cookie.get("path") or "/"),
|
||||
)
|
||||
login = call(client, "GET", "/api/v1/auth/me").json()
|
||||
else:
|
||||
login = call(
|
||||
client,
|
||||
"POST",
|
||||
"/api/v1/auth/login",
|
||||
json={"password": password_value, "remember_device": False},
|
||||
).json()
|
||||
password_value = ""
|
||||
csrf = str(login.get("csrf_token") or "")
|
||||
if not csrf:
|
||||
raise AssertionError("login response omitted CSRF token")
|
||||
client.headers["X-CSRF-Token"] = csrf
|
||||
|
||||
if args.revoke:
|
||||
token_id = args.id_file.read_text(encoding="utf-8").strip()
|
||||
call(client, "DELETE", f"/api/v1/tokens/{token_id}")
|
||||
args.token_file.unlink(missing_ok=True)
|
||||
args.id_file.unlink(missing_ok=True)
|
||||
print(f"revoked live acceptance token {token_id}")
|
||||
return
|
||||
|
||||
result = call(
|
||||
client,
|
||||
"POST",
|
||||
"/api/v1/tokens",
|
||||
json={"name": args.name, "scopes": ["admin"]},
|
||||
).json()
|
||||
args.token_file.write_text(str(result["token"]), encoding="utf-8")
|
||||
args.id_file.write_text(str(result["id"]), encoding="utf-8")
|
||||
os.chmod(args.token_file, 0o600)
|
||||
os.chmod(args.id_file, 0o600)
|
||||
print(f"issued live acceptance token {result['id']}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,235 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run the resumable browser-upload acceptance path against a disposable live server.
|
||||
|
||||
The API token is read from a mode-0600 file and is never included in output. The
|
||||
script only creates objects prefixed by the supplied run id and records their ids
|
||||
so a later cleanup can be narrowly scoped.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
|
||||
def request(client: httpx.Client, method: str, path: str, **kwargs: Any) -> httpx.Response:
|
||||
started = time.perf_counter()
|
||||
response = client.request(method, path, **kwargs)
|
||||
elapsed_ms = round((time.perf_counter() - started) * 1000, 1)
|
||||
if response.is_error:
|
||||
detail = response.text[:1000]
|
||||
raise RuntimeError(f"{method} {path} -> {response.status_code} in {elapsed_ms} ms: {detail}")
|
||||
return response
|
||||
|
||||
|
||||
def sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
while block := handle.read(1024 * 1024):
|
||||
digest.update(block)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def ensure_metadata(client: httpx.Client, run_id: str) -> tuple[str, str, str]:
|
||||
collection_name = f"{run_id}-合集"
|
||||
collections = request(client, "GET", "/api/v1/collections").json()
|
||||
collection = next((item for item in collections if item["name"] == collection_name), None)
|
||||
if collection is None:
|
||||
collection = request(
|
||||
client,
|
||||
"POST",
|
||||
"/api/v1/collections",
|
||||
json={"name": collection_name, "description": "ImageFind 自动验收临时合集"},
|
||||
).json()
|
||||
|
||||
group_name = f"{run_id}-分类"
|
||||
groups = request(client, "GET", "/api/v1/tag-groups").json()
|
||||
group = next((item for item in groups if item["name"] == group_name), None)
|
||||
if group is None:
|
||||
group = request(
|
||||
client,
|
||||
"POST",
|
||||
"/api/v1/tag-groups",
|
||||
json={"name": group_name, "selection_mode": "multi", "sort_order": 9999},
|
||||
).json()
|
||||
group_id = group["id"]
|
||||
else:
|
||||
group_id = group["id"]
|
||||
|
||||
tag_name = f"{run_id}-样本"
|
||||
tags = request(client, "GET", f"/api/v1/tags?group_id={group_id}").json()
|
||||
tag = next((item for item in tags if item["name"] == tag_name), None)
|
||||
if tag is None:
|
||||
tag = request(
|
||||
client,
|
||||
"POST",
|
||||
"/api/v1/tags",
|
||||
json={"group_id": group_id, "name": tag_name},
|
||||
).json()
|
||||
return collection["id"], group_id, tag["id"]
|
||||
|
||||
|
||||
def upload_one(
|
||||
client: httpx.Client,
|
||||
fixture: Path,
|
||||
source_id: str,
|
||||
collection_id: str,
|
||||
tag_id: str,
|
||||
run_id: str,
|
||||
case_id: str,
|
||||
) -> tuple[dict[str, Any], list[float]]:
|
||||
case_prefix = f"-{case_id}" if case_id else ""
|
||||
filename = f"{run_id}{case_prefix}-{fixture.name}"
|
||||
existing = request(client, "GET", "/api/v1/uploads?limit=500").json()
|
||||
upload = next(
|
||||
(
|
||||
item
|
||||
for item in existing
|
||||
if item.get("filename") == filename
|
||||
and item.get("status") in {"receiving", "queued", "transferring", "indexing"}
|
||||
),
|
||||
None,
|
||||
)
|
||||
latencies: list[float] = []
|
||||
if upload is None:
|
||||
upload = request(
|
||||
client,
|
||||
"POST",
|
||||
"/api/v1/uploads",
|
||||
json={
|
||||
"source_id": source_id,
|
||||
"relative_path": "ingest",
|
||||
"filename": filename,
|
||||
"title": f"自动验收 · {fixture.stem}",
|
||||
"collection_id": collection_id,
|
||||
"tag_ids": [tag_id],
|
||||
"size_bytes": fixture.stat().st_size,
|
||||
"sha256": sha256(fixture),
|
||||
"conflict": "skip",
|
||||
},
|
||||
).json()
|
||||
|
||||
upload_id = upload["id"]
|
||||
if upload.get("status") == "completed":
|
||||
return upload, latencies
|
||||
hidden = request(client, "GET", "/api/v1/videos?limit=500").json()
|
||||
if any(item.get("source_key", "").endswith(filename) for item in hidden):
|
||||
raise AssertionError(f"unfinished upload leaked into /videos: {filename}")
|
||||
|
||||
chunk_size = int(upload["chunk_size"])
|
||||
received = {int(index) for index in upload.get("received_chunks", upload.get("received", []))}
|
||||
with fixture.open("rb") as handle:
|
||||
for index in range(int(upload["total_chunks"])):
|
||||
data = handle.read(chunk_size)
|
||||
if index in received:
|
||||
continue
|
||||
started = time.perf_counter()
|
||||
request(
|
||||
client,
|
||||
"PUT",
|
||||
f"/api/v1/uploads/{upload_id}/chunks/{index}",
|
||||
content=data,
|
||||
headers={"X-Chunk-SHA256": hashlib.sha256(data).hexdigest()},
|
||||
)
|
||||
latencies.append(round((time.perf_counter() - started) * 1000, 1))
|
||||
videos = request(client, "GET", "/api/v1/videos?limit=500").json()
|
||||
if any(item.get("source_key", "").endswith(filename) for item in videos):
|
||||
raise AssertionError(f"partially received upload leaked into /videos: {filename}")
|
||||
|
||||
upload = request(client, "POST", f"/api/v1/uploads/{upload_id}/complete").json()
|
||||
deadline = time.monotonic() + 180
|
||||
while upload.get("status") not in {"completed", "failed", "cancelled"}:
|
||||
if time.monotonic() >= deadline:
|
||||
raise TimeoutError(f"upload did not finish: {filename} status={upload.get('status')}")
|
||||
time.sleep(1)
|
||||
uploads = request(client, "GET", "/api/v1/uploads?limit=500").json()
|
||||
upload = next(item for item in uploads if item["id"] == upload_id)
|
||||
if upload["status"] != "completed":
|
||||
raise AssertionError(f"upload failed: {filename}: {upload.get('message')}")
|
||||
return upload, latencies
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--base-url", required=True)
|
||||
auth = parser.add_mutually_exclusive_group(required=True)
|
||||
auth.add_argument("--token-file", type=Path)
|
||||
auth.add_argument("--password")
|
||||
parser.add_argument("--run-dir", type=Path, required=True)
|
||||
parser.add_argument("--case-id", default="")
|
||||
parser.add_argument("--fixture", action="append", default=[])
|
||||
args = parser.parse_args()
|
||||
|
||||
state_path = args.run_dir / "state.json"
|
||||
state = json.loads(state_path.read_text())
|
||||
headers = {}
|
||||
if args.token_file is not None:
|
||||
token = args.token_file.read_text().strip()
|
||||
if not token:
|
||||
raise RuntimeError("empty API token")
|
||||
headers["Authorization"] = f"Bearer {token}"
|
||||
client = httpx.Client(
|
||||
base_url=args.base_url.rstrip("/"),
|
||||
headers=headers,
|
||||
timeout=httpx.Timeout(60, connect=10),
|
||||
)
|
||||
if args.password is not None:
|
||||
login = request(
|
||||
client,
|
||||
"POST",
|
||||
"/api/v1/auth/login",
|
||||
json={"password": args.password, "remember_device": False},
|
||||
).json()
|
||||
csrf = str(login.get("csrf_token") or "")
|
||||
if not csrf:
|
||||
raise AssertionError("login response omitted CSRF token")
|
||||
client.headers["X-CSRF-Token"] = csrf
|
||||
collection_id, group_id, tag_id = ensure_metadata(client, state["run_id"])
|
||||
state.update({"collection_id": collection_id, "tag_group_id": group_id, "tag_id": tag_id})
|
||||
results = []
|
||||
all_latencies: list[float] = []
|
||||
fixture_names = args.fixture or ["e2e-positive.mp4", "e2e-negative.mp4", "e2e-fallback.mpg"]
|
||||
for name in fixture_names:
|
||||
upload, latencies = upload_one(
|
||||
client,
|
||||
args.run_dir / name,
|
||||
state["source_id"],
|
||||
collection_id,
|
||||
tag_id,
|
||||
state["run_id"],
|
||||
args.case_id.strip(),
|
||||
)
|
||||
results.append(
|
||||
{
|
||||
"id": upload["id"],
|
||||
"filename": upload["filename"],
|
||||
"status": upload["status"],
|
||||
"progress": upload["progress"],
|
||||
"message": upload.get("message"),
|
||||
}
|
||||
)
|
||||
all_latencies.extend(latencies)
|
||||
state["upload_ids"] = [item["id"] for item in results]
|
||||
state_path.write_text(json.dumps(state, ensure_ascii=False, indent=2) + "\n")
|
||||
report = {
|
||||
"uploads": results,
|
||||
"chunk_latency_ms": {
|
||||
"count": len(all_latencies),
|
||||
"max": max(all_latencies, default=0),
|
||||
"average": round(sum(all_latencies) / len(all_latencies), 1) if all_latencies else 0,
|
||||
},
|
||||
}
|
||||
(args.run_dir / "upload-report.json").write_text(
|
||||
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
|
||||
)
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,241 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Exercise ImageFind's WebDAV server through its public direct listener."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import quote
|
||||
|
||||
import httpx
|
||||
|
||||
|
||||
def call(client: httpx.Client, method: str, path: str, **kwargs: Any) -> httpx.Response:
|
||||
response = client.request(method, path, **kwargs)
|
||||
if response.is_error:
|
||||
raise RuntimeError(f"{method} {path} -> {response.status_code}: {response.text[:1000]}")
|
||||
return response
|
||||
|
||||
|
||||
def digest(data: bytes) -> str:
|
||||
return hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
def encoded(parts: list[str]) -> str:
|
||||
return "/webdav/" + "/".join(quote(part, safe="") for part in parts)
|
||||
|
||||
|
||||
def videos(client: httpx.Client) -> list[dict[str, Any]]:
|
||||
return call(client, "GET", "/api/v1/videos?limit=500").json()
|
||||
|
||||
|
||||
def visible_video(client: httpx.Client, filename: str) -> dict[str, Any] | None:
|
||||
return next((item for item in videos(client) if item.get("source_key", "").endswith("/" + filename)), None)
|
||||
|
||||
|
||||
def wait_upload(client: httpx.Client, upload_id: str, timeout: int = 240) -> dict[str, Any]:
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
rows = call(client, "GET", "/api/v1/uploads?limit=500").json()
|
||||
upload = next((item for item in rows if item["id"] == upload_id), None)
|
||||
if upload and upload["status"] in {"completed", "failed", "cancelled"}:
|
||||
if upload["status"] != "completed":
|
||||
raise AssertionError(f"WebDAV upload failed: {upload.get('message') or upload.get('error')}")
|
||||
return upload
|
||||
time.sleep(1)
|
||||
raise TimeoutError(f"WebDAV upload did not complete: {upload_id}")
|
||||
|
||||
|
||||
def wait_video(client: httpx.Client, filename: str, timeout: int = 90) -> dict[str, Any]:
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
if video := visible_video(client, filename):
|
||||
return video
|
||||
time.sleep(1)
|
||||
raise TimeoutError(f"WebDAV video did not appear: {filename}")
|
||||
|
||||
|
||||
def upload_id(response: httpx.Response) -> str:
|
||||
value = response.headers.get("x-imagefind-upload-id")
|
||||
if not value:
|
||||
raise AssertionError(f"WebDAV response omitted upload id: {dict(response.headers)}")
|
||||
return value
|
||||
|
||||
|
||||
def collection_video_path(nodes: list[dict[str, Any]], video_id: str, parents: list[str] | None = None):
|
||||
parents = list(parents or [])
|
||||
for node in nodes:
|
||||
if node.get("kind") == "group":
|
||||
found = collection_video_path(node.get("children") or [], video_id, [*parents, node["name"]])
|
||||
if found is not None:
|
||||
return found
|
||||
elif node.get("video_id") == video_id:
|
||||
return parents
|
||||
return None
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--base-url", required=True)
|
||||
parser.add_argument("--token-file", type=Path, required=True)
|
||||
parser.add_argument("--run-dir", type=Path, required=True)
|
||||
parser.add_argument("--case-id", default="v045")
|
||||
args = parser.parse_args()
|
||||
|
||||
state = json.loads((args.run_dir / "state.json").read_text())
|
||||
token = args.token_file.read_text().strip()
|
||||
prefix = f"{state['run_id']}-{args.case_id.strip()}-dav"
|
||||
payload = (args.run_dir / "e2e-positive.mp4").read_bytes()
|
||||
checksum = digest(payload)
|
||||
api = httpx.Client(
|
||||
base_url=args.base_url.rstrip("/"),
|
||||
headers={"Authorization": f"Bearer {token}"},
|
||||
timeout=httpx.Timeout(60, connect=10),
|
||||
)
|
||||
dav = httpx.Client(
|
||||
base_url=args.base_url.rstrip("/"),
|
||||
auth=httpx.BasicAuth("imagefind", token),
|
||||
timeout=httpx.Timeout(120, connect=10, write=120, read=120),
|
||||
)
|
||||
|
||||
config = call(api, "GET", "/api/v1/webdav/config").json()
|
||||
if not config.get("enabled") or config.get("source_id") != state["source_id"]:
|
||||
raise AssertionError(f"unexpected WebDAV config: {config}")
|
||||
options = call(dav, "OPTIONS", "/webdav/")
|
||||
if "PROPFIND" not in options.headers.get("allow", ""):
|
||||
raise AssertionError("WebDAV OPTIONS omitted PROPFIND")
|
||||
root = call(dav, "PROPFIND", "/webdav/", headers={"Depth": "1"})
|
||||
if "multistatus" not in root.text:
|
||||
raise AssertionError("WebDAV root did not return a multistatus document")
|
||||
|
||||
root_name = f"{prefix}-root.mp4"
|
||||
root_response = call(
|
||||
dav,
|
||||
"PUT",
|
||||
encoded([root_name]),
|
||||
content=payload,
|
||||
headers={"X-Content-SHA256": checksum},
|
||||
)
|
||||
root_task = wait_upload(api, upload_id(root_response))
|
||||
root_video = wait_video(api, root_name)
|
||||
|
||||
collection_name = f"{prefix}-collection"
|
||||
groups = [f"level-{number:02d}" for number in range(1, 6)]
|
||||
path: list[str] = [collection_name]
|
||||
for part in [collection_name, *groups]:
|
||||
if part != collection_name:
|
||||
path.append(part)
|
||||
response = dav.request("MKCOL", encoded(path))
|
||||
if response.status_code not in {201, 405}:
|
||||
raise RuntimeError(f"MKCOL {path} -> {response.status_code}: {response.text[:1000]}")
|
||||
|
||||
resume_name = f"{prefix}-resume.mp4"
|
||||
resume_path = encoded([collection_name, *groups, resume_name])
|
||||
midpoint = len(payload) // 2
|
||||
first = call(
|
||||
dav,
|
||||
"PUT",
|
||||
resume_path,
|
||||
content=payload[:midpoint],
|
||||
headers={
|
||||
"Content-Range": f"bytes 0-{midpoint - 1}/{len(payload)}",
|
||||
"X-Content-SHA256": checksum,
|
||||
},
|
||||
)
|
||||
if first.status_code != 204 or int(first.headers.get("upload-offset", 0)) != midpoint:
|
||||
raise AssertionError(f"unexpected partial PUT response: {first.status_code} {dict(first.headers)}")
|
||||
if visible_video(api, resume_name):
|
||||
raise AssertionError("partial WebDAV upload leaked into /videos")
|
||||
staged = call(dav, "HEAD", resume_path)
|
||||
if int(staged.headers.get("upload-offset", 0)) != midpoint:
|
||||
raise AssertionError(f"HEAD did not expose resume offset: {dict(staged.headers)}")
|
||||
second = call(
|
||||
dav,
|
||||
"PUT",
|
||||
resume_path,
|
||||
content=payload[midpoint:],
|
||||
headers={
|
||||
"Content-Range": f"bytes {midpoint}-{len(payload) - 1}/{len(payload)}",
|
||||
"X-Content-SHA256": checksum,
|
||||
},
|
||||
)
|
||||
resume_task = wait_upload(api, upload_id(second))
|
||||
resume_video = wait_video(api, resume_name)
|
||||
|
||||
part_name = f"{prefix}-move.mp4.part"
|
||||
final_name = f"{prefix}-move.mp4"
|
||||
part_path = encoded([collection_name, *groups, part_name])
|
||||
final_path = encoded([collection_name, *groups, final_name])
|
||||
temporary = call(
|
||||
dav,
|
||||
"PUT",
|
||||
part_path,
|
||||
content=payload,
|
||||
headers={"X-Content-SHA256": checksum},
|
||||
)
|
||||
if temporary.headers.get("x-imagefind-upload-id"):
|
||||
raise AssertionError("temporary WebDAV name was committed before MOVE")
|
||||
moved = call(
|
||||
dav,
|
||||
"MOVE",
|
||||
part_path,
|
||||
headers={"Destination": args.base_url.rstrip("/") + final_path, "Overwrite": "F"},
|
||||
)
|
||||
move_task = wait_upload(api, upload_id(moved))
|
||||
move_video = wait_video(api, final_name)
|
||||
|
||||
started = time.perf_counter()
|
||||
duplicate = call(
|
||||
dav,
|
||||
"PUT",
|
||||
resume_path,
|
||||
content=payload,
|
||||
headers={"X-Content-SHA256": checksum, "Expect": "100-continue"},
|
||||
)
|
||||
dedupe_ms = round((time.perf_counter() - started) * 1000, 1)
|
||||
if duplicate.status_code != 204 or duplicate.headers.get("x-imagefind-deduplicated") != "true":
|
||||
raise AssertionError(f"same-path duplicate was not acknowledged: {dict(duplicate.headers)}")
|
||||
if duplicate.headers.get("x-imagefind-upload-id") != resume_task["id"]:
|
||||
raise AssertionError("deduplicated PUT did not reference the original upload task")
|
||||
|
||||
ranged = call(dav, "GET", final_path, headers={"Range": "bytes=0-63"})
|
||||
if ranged.status_code != 206 or ranged.content != payload[:64]:
|
||||
raise AssertionError("WebDAV Range read returned unexpected bytes")
|
||||
|
||||
collections = call(api, "GET", "/api/v1/collections").json()
|
||||
collection = next((item for item in collections if item["name"] == collection_name), None)
|
||||
if not collection:
|
||||
raise AssertionError("WebDAV directory did not create a collection")
|
||||
detail = call(api, "GET", f"/api/v1/collections/{collection['id']}").json()
|
||||
for video in (resume_video, move_video):
|
||||
actual_path = collection_video_path(detail.get("items") or [], video["id"])
|
||||
if actual_path != groups:
|
||||
raise AssertionError(f"collection hierarchy mismatch: {video['id']} -> {actual_path}")
|
||||
if any(item["id"] == root_video["id"] for item in detail.get("videos") or []):
|
||||
raise AssertionError("root WebDAV upload unexpectedly joined a collection")
|
||||
|
||||
report = {
|
||||
"root_upload": {"id": root_task["id"], "video_id": root_video["id"]},
|
||||
"resumable_upload": {
|
||||
"id": resume_task["id"],
|
||||
"video_id": resume_video["id"],
|
||||
"first_offset": midpoint,
|
||||
"final_size": len(payload),
|
||||
},
|
||||
"temporary_move": {"id": move_task["id"], "video_id": move_video["id"]},
|
||||
"deduplicated": True,
|
||||
"dedupe_latency_ms": dedupe_ms,
|
||||
"range_read": len(ranged.content),
|
||||
"collection_id": collection["id"],
|
||||
"collection_depth": len(groups),
|
||||
}
|
||||
(args.run_dir / "webdav-report.json").write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n")
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,279 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Stress ImageFind WebDAV reception without retaining generated large files."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import threading
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import quote
|
||||
|
||||
import httpx
|
||||
|
||||
BLOCK = bytes(1024 * 1024)
|
||||
|
||||
|
||||
def padded_hash(prefix: bytes, size: int) -> str:
|
||||
value = hashlib.sha256(prefix)
|
||||
remaining = size - len(prefix)
|
||||
while remaining > 0:
|
||||
length = min(len(BLOCK), remaining)
|
||||
value.update(BLOCK[:length])
|
||||
remaining -= length
|
||||
return value.hexdigest()
|
||||
|
||||
|
||||
def padded_body(prefix: bytes, size: int):
|
||||
yield prefix
|
||||
remaining = size - len(prefix)
|
||||
while remaining > 0:
|
||||
length = min(len(BLOCK), remaining)
|
||||
yield BLOCK[:length]
|
||||
remaining -= length
|
||||
|
||||
|
||||
def request(client: httpx.Client, method: str, path: str, **kwargs: Any) -> httpx.Response:
|
||||
response = client.request(method, path, **kwargs)
|
||||
if response.is_error:
|
||||
raise RuntimeError(f"{method} {path} -> {response.status_code}: {response.text[:1000]}")
|
||||
return response
|
||||
|
||||
|
||||
def encoded(parts: list[str]) -> str:
|
||||
return "/webdav/" + "/".join(quote(part, safe="") for part in parts)
|
||||
|
||||
|
||||
def wait_uploads(api: httpx.Client, ids: set[str], timeout: int) -> dict[str, dict[str, Any]]:
|
||||
deadline = time.monotonic() + timeout
|
||||
terminal: dict[str, dict[str, Any]] = {}
|
||||
while time.monotonic() < deadline:
|
||||
rows = request(api, "GET", "/api/v1/uploads?limit=500").json()
|
||||
by_id = {item["id"]: item for item in rows if item["id"] in ids}
|
||||
terminal = {
|
||||
key: value
|
||||
for key, value in by_id.items()
|
||||
if value["status"] in {"completed", "failed", "cancelled"}
|
||||
}
|
||||
if len(terminal) == len(ids):
|
||||
failures = [item for item in terminal.values() if item["status"] != "completed"]
|
||||
if failures:
|
||||
raise AssertionError(f"committed WebDAV uploads failed: {failures}")
|
||||
return terminal
|
||||
time.sleep(1)
|
||||
raise TimeoutError(f"WebDAV transfers did not finish: {ids - set(terminal)}")
|
||||
|
||||
|
||||
def wait_video(api: httpx.Client, filename: str, timeout: int = 120) -> dict[str, Any]:
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
rows = request(api, "GET", "/api/v1/videos?limit=500").json()
|
||||
match = next((item for item in rows if item.get("source_key", "").endswith("/" + filename)), None)
|
||||
if match:
|
||||
return match
|
||||
time.sleep(1)
|
||||
raise TimeoutError(f"committed stress video did not appear: {filename}")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--base-url", required=True)
|
||||
parser.add_argument("--token-file", type=Path, required=True)
|
||||
parser.add_argument("--run-dir", type=Path, required=True)
|
||||
parser.add_argument("--case-id", required=True)
|
||||
parser.add_argument("--size-mb", type=int, required=True)
|
||||
parser.add_argument("--count", type=int, required=True)
|
||||
parser.add_argument("--concurrency", type=int, default=4)
|
||||
parser.add_argument("--commit-count", type=int, default=1)
|
||||
parser.add_argument("--timeout", type=int, default=1800)
|
||||
args = parser.parse_args()
|
||||
|
||||
state = json.loads((args.run_dir / "state.json").read_text())
|
||||
token = args.token_file.read_text().strip()
|
||||
prefix = f"{state['run_id']}-{args.case_id}-stress"
|
||||
collection = f"{prefix}-collection"
|
||||
group = "uploads"
|
||||
size = args.size_mb * 1024**2
|
||||
fixture = (args.run_dir / "e2e-positive.mp4").read_bytes()
|
||||
if size < len(fixture):
|
||||
raise ValueError("stress size must be at least the fixture size")
|
||||
checksum = padded_hash(fixture, size)
|
||||
auth = httpx.BasicAuth("imagefind", token)
|
||||
api = httpx.Client(
|
||||
base_url=args.base_url.rstrip("/"),
|
||||
headers={"Authorization": f"Bearer {token}"},
|
||||
timeout=httpx.Timeout(60, connect=10),
|
||||
)
|
||||
dav = httpx.Client(
|
||||
base_url=args.base_url.rstrip("/"),
|
||||
auth=auth,
|
||||
timeout=httpx.Timeout(args.timeout, connect=15, read=args.timeout, write=args.timeout),
|
||||
)
|
||||
for parts in ([collection], [collection, group]):
|
||||
response = dav.request("MKCOL", encoded(parts))
|
||||
if response.status_code not in {201, 405}:
|
||||
raise RuntimeError(f"MKCOL {parts} -> {response.status_code}: {response.text[:1000]}")
|
||||
|
||||
stop = threading.Event()
|
||||
samples: list[dict[str, Any]] = []
|
||||
|
||||
def monitor() -> None:
|
||||
with httpx.Client(
|
||||
base_url=args.base_url.rstrip("/"),
|
||||
headers={"Authorization": f"Bearer {token}"},
|
||||
timeout=10,
|
||||
) as client:
|
||||
while not stop.wait(1):
|
||||
started = time.perf_counter()
|
||||
try:
|
||||
status = client.get("/api/v1/status")
|
||||
status.raise_for_status()
|
||||
latency = round((time.perf_counter() - started) * 1000, 1)
|
||||
resources = client.get("/api/v1/system/resources")
|
||||
resources.raise_for_status()
|
||||
payload = resources.json()
|
||||
samples.append(
|
||||
{
|
||||
"api_ms": latency,
|
||||
"cpu": payload.get("cpu_percent"),
|
||||
"memory_available": payload.get("memory_available_bytes"),
|
||||
"writer_wait_ms": (payload.get("database") or {}).get("last_wait_ms"),
|
||||
"ok": True,
|
||||
}
|
||||
)
|
||||
except Exception as exc:
|
||||
samples.append({"ok": False, "error": type(exc).__name__})
|
||||
|
||||
monitor_thread = threading.Thread(target=monitor, name="imagefind-webdav-monitor", daemon=True)
|
||||
monitor_thread.start()
|
||||
|
||||
files = [f"{prefix}-{index + 1:03d}.mp4" for index in range(args.count)]
|
||||
temporary_paths = {name: encoded([collection, group, name + ".part"]) for name in files}
|
||||
|
||||
def put_one(filename: str) -> dict[str, Any]:
|
||||
path = temporary_paths[filename]
|
||||
started = time.perf_counter()
|
||||
with httpx.Client(
|
||||
base_url=args.base_url.rstrip("/"),
|
||||
auth=auth,
|
||||
timeout=httpx.Timeout(args.timeout, connect=15, read=args.timeout, write=args.timeout),
|
||||
) as client:
|
||||
response = client.put(
|
||||
path,
|
||||
content=padded_body(fixture, size),
|
||||
headers={
|
||||
"Content-Length": str(size),
|
||||
"X-Content-SHA256": checksum,
|
||||
},
|
||||
)
|
||||
if response.is_error:
|
||||
raise RuntimeError(f"PUT {filename} -> {response.status_code}: {response.text[:1000]}")
|
||||
if response.status_code not in {201, 204}:
|
||||
raise RuntimeError(f"PUT {filename} returned {response.status_code}")
|
||||
return {
|
||||
"filename": filename,
|
||||
"status": response.status_code,
|
||||
"elapsed_ms": round((time.perf_counter() - started) * 1000, 1),
|
||||
"offset": int(response.headers.get("upload-offset", 0)),
|
||||
}
|
||||
|
||||
started_all = time.perf_counter()
|
||||
results: list[dict[str, Any]] = []
|
||||
error: Exception | None = None
|
||||
committed: list[tuple[str, str]] = []
|
||||
videos: list[dict[str, Any]] = []
|
||||
try:
|
||||
with ThreadPoolExecutor(max_workers=args.concurrency) as executor:
|
||||
futures = {executor.submit(put_one, filename): filename for filename in files}
|
||||
for future in as_completed(futures):
|
||||
results.append(future.result())
|
||||
if any(item["offset"] != size for item in results):
|
||||
raise AssertionError("one or more WebDAV PUT responses reported an incomplete offset")
|
||||
|
||||
for filename in files[: min(args.commit_count, len(files))]:
|
||||
source_path = temporary_paths[filename]
|
||||
destination = encoded([collection, group, filename])
|
||||
moved = request(
|
||||
dav,
|
||||
"MOVE",
|
||||
source_path,
|
||||
headers={
|
||||
"Destination": args.base_url.rstrip("/") + destination,
|
||||
"Overwrite": "F",
|
||||
},
|
||||
)
|
||||
upload_id = moved.headers.get("x-imagefind-upload-id")
|
||||
if not upload_id:
|
||||
raise AssertionError(f"MOVE omitted upload id for {filename}")
|
||||
committed.append((filename, upload_id))
|
||||
|
||||
if committed:
|
||||
wait_uploads(api, {upload_id for _, upload_id in committed}, args.timeout)
|
||||
for filename, _ in committed:
|
||||
video = wait_video(api, filename)
|
||||
ranged = request(
|
||||
dav,
|
||||
"GET",
|
||||
encoded([collection, group, filename]),
|
||||
headers={"Range": "bytes=0-63"},
|
||||
)
|
||||
if ranged.status_code != 206 or ranged.content != fixture[:64]:
|
||||
raise AssertionError(f"committed range read failed: {filename}")
|
||||
videos.append(video)
|
||||
except Exception as exc:
|
||||
error = exc
|
||||
finally:
|
||||
for filename in files[len(committed) :]:
|
||||
try:
|
||||
dav.delete(temporary_paths[filename])
|
||||
except Exception:
|
||||
pass
|
||||
for video in videos:
|
||||
try:
|
||||
removed = request(api, "DELETE", f"/api/v1/videos/{video['id']}?delete_source=true").json()
|
||||
if removed.get("trash_id"):
|
||||
request(api, "DELETE", f"/api/v1/trash/{removed['trash_id']}")
|
||||
except Exception:
|
||||
pass
|
||||
stop.set()
|
||||
monitor_thread.join(timeout=15)
|
||||
if error is not None:
|
||||
raise error
|
||||
|
||||
elapsed = time.perf_counter() - started_all
|
||||
successful_samples = [item for item in samples if item.get("ok")]
|
||||
report = {
|
||||
"case_id": args.case_id,
|
||||
"size_mb": args.size_mb,
|
||||
"count": args.count,
|
||||
"concurrency": args.concurrency,
|
||||
"committed": len(committed),
|
||||
"elapsed_seconds": round(elapsed, 2),
|
||||
"throughput_mib_s": round(args.size_mb * args.count / max(elapsed, 0.001), 2),
|
||||
"put_latency_ms": {
|
||||
"minimum": min(item["elapsed_ms"] for item in results),
|
||||
"maximum": max(item["elapsed_ms"] for item in results),
|
||||
},
|
||||
"monitor": {
|
||||
"samples": len(samples),
|
||||
"failed_samples": sum(1 for item in samples if not item.get("ok")),
|
||||
"max_api_ms": max((item["api_ms"] for item in successful_samples), default=0),
|
||||
"max_cpu_percent": max((item["cpu"] for item in successful_samples), default=0),
|
||||
"min_memory_available_gb": round(
|
||||
min((item["memory_available"] for item in successful_samples), default=0) / 1024**3,
|
||||
2,
|
||||
),
|
||||
"max_writer_wait_ms": max((item["writer_wait_ms"] for item in successful_samples), default=0),
|
||||
},
|
||||
}
|
||||
report_path = args.run_dir / f"webdav-stress-{args.case_id}.json"
|
||||
report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n")
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,45 @@
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from PIL import Image, ImageDraw
|
||||
|
||||
|
||||
def build(path: Path, size: int) -> None:
|
||||
# The fnOS mobile launcher already applies its own icon mask. Transparent
|
||||
# corners are composited against black in the app, so keep the full canvas
|
||||
# opaque and let the branded background run all the way to every edge.
|
||||
image = Image.new("RGBA", (size, size), (103, 145, 244, 255))
|
||||
draw = ImageDraw.Draw(image)
|
||||
center = size // 2
|
||||
radius = size // 5
|
||||
draw.ellipse(
|
||||
(center - radius, center - radius, center + radius, center + radius),
|
||||
outline=(255, 255, 255, 245),
|
||||
width=max(2, size // 25),
|
||||
)
|
||||
sparkle = size // 13
|
||||
draw.polygon(
|
||||
[
|
||||
(size * 3 // 4, size // 5),
|
||||
(size * 3 // 4 + sparkle, size // 3),
|
||||
(size * 7 // 8, size // 3 + sparkle),
|
||||
(size * 3 // 4 + sparkle, size // 3 + sparkle * 2),
|
||||
(size * 3 // 4, size // 2),
|
||||
(size * 3 // 4 - sparkle, size // 3 + sparkle * 2),
|
||||
(size * 5 // 8, size // 3 + sparkle),
|
||||
(size * 3 // 4 - sparkle, size // 3),
|
||||
],
|
||||
fill=(255, 255, 255, 255),
|
||||
)
|
||||
image.save(path, "PNG")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
destination = Path(sys.argv[1])
|
||||
destination.mkdir(parents=True, exist_ok=True)
|
||||
build(destination / "ICON.PNG", 128)
|
||||
build(destination / "ICON_256.PNG", 256)
|
||||
ui_images = destination / "app" / "ui" / "images"
|
||||
ui_images.mkdir(parents=True, exist_ok=True)
|
||||
build(ui_images / "icon_64.png", 64)
|
||||
build(ui_images / "icon_256.png", 256)
|
||||
Executable
+29
@@ -0,0 +1,29 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
|
||||
PYTHON_BIN=${PYTHON:-python3}
|
||||
WHEELHOUSE=${IMAGEFIND_WHEEL_CACHE:-$ROOT_DIR/.fnos-wheel-cache/python312}
|
||||
DOWNLOAD_ROOT=${IMAGEFIND_DEPENDENCY_TMPDIR:-${TMPDIR:-/tmp}}
|
||||
OPENCL_URL=https://deb.debian.org/debian/pool/main/o/ocl-icd/ocl-icd-libopencl1_2.2.14-2_amd64.deb
|
||||
OPENCL_SHA256=f3367b78f2548e6211e23081b3e0f144babfcc906c5bdbbe0d5e41915de70128
|
||||
|
||||
command -v "$PYTHON_BIN" >/dev/null
|
||||
for command_name in curl dpkg-deb find install sha256sum; do command -v "$command_name" >/dev/null; done
|
||||
mkdir -p "$WHEELHOUSE" "$ROOT_DIR/vendor" "$DOWNLOAD_ROOT"
|
||||
|
||||
"$PYTHON_BIN" -m pip download --disable-pip-version-check --only-binary=:all: \
|
||||
--implementation cp --python-version 312 --abi cp312 \
|
||||
--platform manylinux_2_28_x86_64 --platform manylinux_2_27_x86_64 \
|
||||
--platform manylinux2014_x86_64 --platform manylinux2010_x86_64 --platform manylinux1_x86_64 \
|
||||
--dest "$WHEELHOUSE" --requirement "$ROOT_DIR/requirements/runtime-core.txt"
|
||||
|
||||
work_dir=$(mktemp -d "${DOWNLOAD_ROOT%/}/imagefind-opencl.XXXXXX")
|
||||
trap 'rm -rf -- "$work_dir"' EXIT
|
||||
curl --fail --location --retry 4 --retry-all-errors "$OPENCL_URL" --output "$work_dir/opencl.deb"
|
||||
dpkg-deb -x "$work_dir/opencl.deb" "$work_dir/root"
|
||||
source_file=$(find "$work_dir/root" -type f -name 'libOpenCL.so.1.*' -print -quit)
|
||||
test -n "$source_file"
|
||||
printf '%s %s\n' "$OPENCL_SHA256" "$source_file" | sha256sum --check --status
|
||||
install -m 0644 "$source_file" "$ROOT_DIR/vendor/libOpenCL.so.1"
|
||||
printf 'Prepared %s and %s\n' "$WHEELHOUSE" "$ROOT_DIR/vendor/libOpenCL.so.1"
|
||||
@@ -0,0 +1,4 @@
|
||||
from imagefind.main import run
|
||||
|
||||
if __name__ == "__main__":
|
||||
run()
|
||||
Executable
+137
@@ -0,0 +1,137 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
PROJECT_ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
|
||||
APP_ROOT="${1:-$PROJECT_ROOT/.build-fnos/imagefind/app}"
|
||||
# Keep the default data root on the project filesystem. Resource admission is
|
||||
# intentionally disk-aware; placing smoke data on a small /tmp tmpfs can make
|
||||
# a healthy transfer worker wait forever at the production 5 GiB reserve.
|
||||
DATA_ROOT="${2:-$PROJECT_ROOT/.smoke-fnos}"
|
||||
PORT="${3:-18765}"
|
||||
PACKAGE_ROOT="${4:-$PROJECT_ROOT/fnos}"
|
||||
PYTHON_PATH="${5:-${IMAGEFIND_PYTHON_PATH:-}}"
|
||||
CONTROL="$PACKAGE_ROOT/cmd/main"
|
||||
INSTALL_CALLBACK="$PACKAGE_ROOT/cmd/install_callback"
|
||||
CONFIG_CALLBACK="$PACKAGE_ROOT/cmd/config_callback"
|
||||
SMOKE_PASSWORD="imagefind-smoke-password"
|
||||
SMOKE_POSTGRES_TOKEN="${IMAGEFIND_SMOKE_POSTGRES_TOKEN:-}"
|
||||
COOKIE_JAR="$DATA_ROOT/run/smoke-cookie.txt"
|
||||
BACKUP_FILE="$DATA_ROOT/run/smoke.ifbackup"
|
||||
EXPECTED_VERSION=$(sed -n '1p' "$APP_ROOT/runtime/VERSION")
|
||||
|
||||
APP_ROOT=$(CDPATH= cd -- "$APP_ROOT" && pwd)
|
||||
[ "${#SMOKE_POSTGRES_TOKEN}" -ge 20 ] || {
|
||||
printf '%s\n' 'Set IMAGEFIND_SMOKE_POSTGRES_TOKEN to a valid nxsir.postgresql enrollment token.' >&2
|
||||
exit 2
|
||||
}
|
||||
mkdir -p "$DATA_ROOT"
|
||||
DATA_ROOT=$(CDPATH= cd -- "$DATA_ROOT" && pwd)
|
||||
|
||||
cleanup() {
|
||||
IMAGEFIND_PYTHON_PATH="$PYTHON_PATH" TRIM_APPDEST="$APP_ROOT" TRIM_PKGVAR="$DATA_ROOT" TRIM_SERVICE_PORT="$PORT" \
|
||||
"$CONTROL" stop
|
||||
rm -f "$COOKIE_JAR" "$BACKUP_FILE"
|
||||
}
|
||||
|
||||
trap cleanup EXIT HUP INT TERM
|
||||
IMAGEFIND_PYTHON_PATH="$PYTHON_PATH" TRIM_APPDEST="$APP_ROOT" TRIM_PKGVAR="$DATA_ROOT" \
|
||||
wizard_admin_password="$SMOKE_PASSWORD" wizard_admin_password_confirm="$SMOKE_PASSWORD" \
|
||||
wizard_postgres_enrollment_token="$SMOKE_POSTGRES_TOKEN" \
|
||||
"$INSTALL_CALLBACK"
|
||||
TRIM_PKGVAR="$DATA_ROOT" wizard_direct_access=true wizard_direct_port="$PORT" \
|
||||
"$CONFIG_CALLBACK"
|
||||
IMAGEFIND_PYTHON_PATH="$PYTHON_PATH" TRIM_APPDEST="$APP_ROOT" TRIM_PKGVAR="$DATA_ROOT" TRIM_SERVICE_PORT="$PORT" \
|
||||
"$CONTROL" start
|
||||
|
||||
STATUS_BODY=""
|
||||
ATTEMPT=0
|
||||
while [ "$ATTEMPT" -lt 90 ]; do
|
||||
if STATUS_BODY=$(curl -fsS "http://127.0.0.1:$PORT/api/v1/status" 2>/dev/null); then
|
||||
break
|
||||
fi
|
||||
ATTEMPT=$((ATTEMPT + 1))
|
||||
sleep 1
|
||||
done
|
||||
|
||||
if [ -z "$STATUS_BODY" ]; then
|
||||
printf 'ImageFind did not become ready; application log follows:\n' >&2
|
||||
tail -100 "$DATA_ROOT/log/imagefind.log" >&2 || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
case "$STATUS_BODY" in
|
||||
*'"configured":true'*"\"version\":\"$EXPECTED_VERSION\""*) ;;
|
||||
*) printf 'Unexpected ImageFind status: %s\n' "$STATUS_BODY" >&2; exit 1 ;;
|
||||
esac
|
||||
|
||||
RUNTIME_PYTHON="$DATA_ROOT/runtime/current/bin/python"
|
||||
RUNTIME_VERSION=$("$RUNTIME_PYTHON" -m imagefind.main --version)
|
||||
CORE_RUNTIME=$(
|
||||
LD_LIBRARY_PATH="$APP_ROOT/bin${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" \
|
||||
XDG_CACHE_HOME="$DATA_ROOT/cache" \
|
||||
"$RUNTIME_PYTHON" -c \
|
||||
'import fastapi,httpx,imagefind,psycopg,pydantic,uvicorn; print("core-runtime-ok")'
|
||||
)
|
||||
case "$CORE_RUNTIME" in
|
||||
*'core-runtime-ok'*) ;;
|
||||
*) printf 'Offline core runtime check failed: %s\n' "$CORE_RUNTIME" >&2; exit 1 ;;
|
||||
esac
|
||||
FRONTEND_HEADERS=$(curl -fsS -D - -o /dev/null "http://127.0.0.1:$PORT/" | sed -n '1,8p')
|
||||
LOGIN_BODY=$(curl -fsS -c "$COOKIE_JAR" -H 'Content-Type: application/json' \
|
||||
--data "{\"password\":\"$SMOKE_PASSWORD\",\"remember_device\":false}" \
|
||||
"http://127.0.0.1:$PORT/api/v1/auth/login")
|
||||
CSRF_TOKEN=$(printf '%s' "$LOGIN_BODY" | sed -n 's/.*"csrf_token":"\([^"]*\)".*/\1/p')
|
||||
if [ -z "$CSRF_TOKEN" ]; then
|
||||
printf 'Unable to obtain CSRF token from packaged server.\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
BACKUP_STATUS=$(curl -fsS -b "$COOKIE_JAR" "http://127.0.0.1:$PORT/api/v1/backups/status")
|
||||
MODEL_STATUS=$(curl -fsS -b "$COOKIE_JAR" "http://127.0.0.1:$PORT/api/v1/models")
|
||||
case "$MODEL_STATUS" in
|
||||
*'"runtime_environment":{"state":"missing"'*'"source":"on_demand"'*) ;;
|
||||
*) printf 'AI runtime was not reported as on-demand: %s\n' "$MODEL_STATUS" >&2; exit 1 ;;
|
||||
esac
|
||||
BACKUP_QUEUE=$(curl -fsS -b "$COOKIE_JAR" -H "X-CSRF-Token: $CSRF_TOKEN" -H 'Content-Type: application/json' \
|
||||
--data "{\"scope\":\"keys\",\"password\":\"$SMOKE_PASSWORD\"}" \
|
||||
"http://127.0.0.1:$PORT/api/v1/backups")
|
||||
BACKUP_ID=$(printf '%s' "$BACKUP_QUEUE" | sed -n 's/.*"id":"\([^"]*\)".*/\1/p')
|
||||
if [ -z "$BACKUP_ID" ]; then
|
||||
printf 'Unable to queue packaged background backup: %s\n' "$BACKUP_QUEUE" >&2
|
||||
exit 1
|
||||
fi
|
||||
BACKUP_EXPORTS=""
|
||||
ATTEMPT=0
|
||||
while [ "$ATTEMPT" -lt 90 ]; do
|
||||
BACKUP_EXPORTS=$(curl -fsS -b "$COOKIE_JAR" "http://127.0.0.1:$PORT/api/v1/backups")
|
||||
case "$BACKUP_EXPORTS" in
|
||||
*"\"id\":\"$BACKUP_ID\""*'"status":"completed"'*) break ;;
|
||||
*"\"id\":\"$BACKUP_ID\""*'"status":"failed"'*)
|
||||
printf 'Packaged background backup failed: %s\n' "$BACKUP_EXPORTS" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
ATTEMPT=$((ATTEMPT + 1))
|
||||
sleep 1
|
||||
done
|
||||
case "$BACKUP_EXPORTS" in
|
||||
*"\"id\":\"$BACKUP_ID\""*'"status":"completed"'*) ;;
|
||||
*) printf 'Packaged background backup timed out: %s\n' "$BACKUP_EXPORTS" >&2; exit 1 ;;
|
||||
esac
|
||||
curl -fsS -b "$COOKIE_JAR" -o "$BACKUP_FILE" \
|
||||
"http://127.0.0.1:$PORT/api/v1/backups/$BACKUP_ID/download"
|
||||
BACKUP_MAGIC=$(dd if="$BACKUP_FILE" bs=1 count=8 2>/dev/null)
|
||||
if [ "$BACKUP_MAGIC" != "IFBACKUP" ]; then
|
||||
printf 'Packaged backup has an invalid header.\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
RESTORE_BODY=$(curl -fsS -b "$COOKIE_JAR" -H "X-CSRF-Token: $CSRF_TOKEN" \
|
||||
-F "file=@$BACKUP_FILE;type=application/vnd.imagefind.backup" \
|
||||
-F "password=$SMOKE_PASSWORD" -F 'confirmed=true' \
|
||||
"http://127.0.0.1:$PORT/api/v1/backups/restore")
|
||||
case "$RESTORE_BODY" in
|
||||
*'"scope":"keys"'*'"counts"'*'"scan_jobs":[]'*) ;;
|
||||
*) printf 'Unexpected packaged backup restore result: %s\n' "$RESTORE_BODY" >&2; exit 1 ;;
|
||||
esac
|
||||
printf 'status=%s\nruntime=%s\ncore_runtime=%s\nbackup_status=%s\nbackup_restore=%s\nfrontend_headers:\n%s\n' \
|
||||
"$STATUS_BODY" "$RUNTIME_VERSION" "$CORE_RUNTIME" "$BACKUP_STATUS" "$RESTORE_BODY" \
|
||||
"$FRONTEND_HEADERS"
|
||||
Executable
+122
@@ -0,0 +1,122 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
PROJECT_ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
|
||||
OLD_PACKAGE="${1:-$PROJECT_ROOT/dist/imagefind-0.3.9-x86_64.fpk}"
|
||||
NEW_APP_ROOT="${2:-$PROJECT_ROOT/.build-fnos/imagefind/app}"
|
||||
PORT="${3:-18766}"
|
||||
NEW_PACKAGE_ROOT="${4:-$PROJECT_ROOT/fnos}"
|
||||
PYTHON_PATH="${5:-${IMAGEFIND_PYTHON_PATH:-}}"
|
||||
PASSWORD="imagefind-upgrade-smoke-password"
|
||||
|
||||
[ -f "$OLD_PACKAGE" ] || { printf 'missing previous package: %s\n' "$OLD_PACKAGE" >&2; exit 1; }
|
||||
[ -f "$NEW_APP_ROOT/runtime/VERSION" ] && [ -d "$NEW_APP_ROOT/runtime/wheels" ] || {
|
||||
printf 'missing new packaged Python runtime: %s\n' "$NEW_APP_ROOT/runtime" >&2
|
||||
exit 1
|
||||
}
|
||||
OLD_PACKAGE=$(CDPATH= cd -- "$(dirname -- "$OLD_PACKAGE")" && pwd)/$(basename -- "$OLD_PACKAGE")
|
||||
NEW_APP_ROOT=$(CDPATH= cd -- "$NEW_APP_ROOT" && pwd)
|
||||
NEW_PACKAGE_ROOT=$(CDPATH= cd -- "$NEW_PACKAGE_ROOT" && pwd)
|
||||
|
||||
WORK_ROOT=$(mktemp -d "$PROJECT_ROOT/.upgrade-smoke.XXXXXX")
|
||||
OLD_PACKAGE_ROOT="$WORK_ROOT/old-package"
|
||||
ACTIVE_APP_ROOT="$WORK_ROOT/app"
|
||||
OLD_APP_ROOT="$WORK_ROOT/old-app"
|
||||
DATA_ROOT="$WORK_ROOT/var"
|
||||
LIFECYCLE_LOG="$WORK_ROOT/fnos-upgrade.log"
|
||||
COOKIE_JAR="$WORK_ROOT/cookie.txt"
|
||||
OLD_CONTROL="$OLD_PACKAGE_ROOT/cmd/main"
|
||||
NEW_CONTROL="$NEW_PACKAGE_ROOT/cmd/main"
|
||||
|
||||
cleanup() {
|
||||
set +e
|
||||
if [ -x "$NEW_CONTROL" ]; then
|
||||
IMAGEFIND_PYTHON_PATH="$PYTHON_PATH" TRIM_APPDEST="$ACTIVE_APP_ROOT" TRIM_PKGVAR="$DATA_ROOT" \
|
||||
"$NEW_CONTROL" stop >/dev/null 2>&1
|
||||
fi
|
||||
if [ -x "$OLD_CONTROL" ]; then
|
||||
TRIM_APPDEST="$ACTIVE_APP_ROOT" TRIM_PKGVAR="$DATA_ROOT" "$OLD_CONTROL" stop >/dev/null 2>&1
|
||||
fi
|
||||
if [ "${IMAGEFIND_KEEP_UPGRADE_SMOKE:-false}" = "true" ]; then
|
||||
printf 'upgrade smoke workspace preserved at %s\n' "$WORK_ROOT" >&2
|
||||
return
|
||||
fi
|
||||
case "$WORK_ROOT" in
|
||||
"$PROJECT_ROOT"/.upgrade-smoke.*) rm -rf -- "$WORK_ROOT" ;;
|
||||
*) printf 'refusing to clear unexpected upgrade smoke path: %s\n' "$WORK_ROOT" >&2 ;;
|
||||
esac
|
||||
}
|
||||
trap cleanup EXIT HUP INT TERM
|
||||
|
||||
mkdir -p "$OLD_PACKAGE_ROOT" "$ACTIVE_APP_ROOT"
|
||||
tar xzf "$OLD_PACKAGE" -C "$OLD_PACKAGE_ROOT" cmd manifest
|
||||
tar xOf "$OLD_PACKAGE" app.tgz | tar xzf - -C "$ACTIVE_APP_ROOT"
|
||||
OLD_VERSION=$(sed -n 's/^version[[:space:]]*=[[:space:]]*//p' "$OLD_PACKAGE_ROOT/manifest" | head -n 1)
|
||||
NEW_VERSION=$(sed -n 's/^version[[:space:]]*=[[:space:]]*//p' "$NEW_PACKAGE_ROOT/manifest" | head -n 1)
|
||||
[ -n "$OLD_VERSION" ] && [ -n "$NEW_VERSION" ] || { printf 'unable to read package versions\n' >&2; exit 1; }
|
||||
[ "$OLD_VERSION" != "$NEW_VERSION" ] || { printf 'upgrade smoke requires two different versions\n' >&2; exit 1; }
|
||||
|
||||
wait_for_version() {
|
||||
EXPECTED="$1"
|
||||
ATTEMPT=0
|
||||
while [ "$ATTEMPT" -lt 90 ]; do
|
||||
STATUS=$(curl -fsS "http://127.0.0.1:$PORT/api/v1/status" 2>/dev/null || true)
|
||||
case "$STATUS" in
|
||||
*'"configured":true'*'"version":"'"$EXPECTED"'"'*) return 0 ;;
|
||||
esac
|
||||
ATTEMPT=$((ATTEMPT + 1))
|
||||
sleep 1
|
||||
done
|
||||
printf 'version %s did not become ready; lifecycle log follows:\n' "$EXPECTED" >&2
|
||||
tail -100 "$DATA_ROOT/log/imagefind.log" >&2 || true
|
||||
return 1
|
||||
}
|
||||
|
||||
IMAGEFIND_PYTHON_PATH="$PYTHON_PATH" TRIM_APPDEST="$ACTIVE_APP_ROOT" TRIM_PKGVAR="$DATA_ROOT" \
|
||||
TRIM_TEMP_LOGFILE="$LIFECYCLE_LOG" \
|
||||
wizard_admin_password="$PASSWORD" wizard_admin_password_confirm="$PASSWORD" \
|
||||
"$OLD_PACKAGE_ROOT/cmd/install_callback"
|
||||
TRIM_PKGVAR="$DATA_ROOT" wizard_direct_access=true wizard_direct_port="$PORT" \
|
||||
"$OLD_PACKAGE_ROOT/cmd/config_callback"
|
||||
# A 0.3.15 PyInstaller service can be precisely what is broken on the target
|
||||
# NAS. Upgrade verification must therefore begin from its persisted data and
|
||||
# lifecycle callbacks, not require the old HTTP listener to become healthy.
|
||||
if [ "${IMAGEFIND_SMOKE_START_OLD:-false}" = "true" ]; then
|
||||
IMAGEFIND_PYTHON_PATH="$PYTHON_PATH" TRIM_APPDEST="$ACTIVE_APP_ROOT" TRIM_PKGVAR="$DATA_ROOT" \
|
||||
"$OLD_CONTROL" start
|
||||
wait_for_version "$OLD_VERSION"
|
||||
fi
|
||||
printf '%s\n' 'preserve-across-upgrade' >"$DATA_ROOT/data/upgrade-smoke-marker"
|
||||
IMAGEFIND_PYTHON_PATH="$PYTHON_PATH" TRIM_APPDEST="$ACTIVE_APP_ROOT" TRIM_PKGVAR="$DATA_ROOT" \
|
||||
"$OLD_CONTROL" stop
|
||||
|
||||
TRIM_APPDEST="$ACTIVE_APP_ROOT" TRIM_PKGVAR="$DATA_ROOT" TRIM_TEMP_LOGFILE="$LIFECYCLE_LOG" \
|
||||
"$NEW_PACKAGE_ROOT/cmd/upgrade_init"
|
||||
mv "$ACTIVE_APP_ROOT" "$OLD_APP_ROOT"
|
||||
ln -s "$NEW_APP_ROOT" "$ACTIVE_APP_ROOT"
|
||||
# Deliberately pass stale mismatched values from the previous wizard.
|
||||
# The upgrade must preserve the existing password and still succeed.
|
||||
TRIM_APPDEST="$ACTIVE_APP_ROOT" TRIM_PKGVAR="$DATA_ROOT" TRIM_TEMP_LOGFILE="$LIFECYCLE_LOG" \
|
||||
IMAGEFIND_PYTHON_PATH="$PYTHON_PATH" \
|
||||
wizard_admin_password=placeholder wizard_admin_password_confirm=different \
|
||||
"$NEW_PACKAGE_ROOT/cmd/upgrade_callback"
|
||||
IMAGEFIND_PYTHON_PATH="$PYTHON_PATH" TRIM_APPDEST="$ACTIVE_APP_ROOT" TRIM_PKGVAR="$DATA_ROOT" \
|
||||
"$NEW_CONTROL" start
|
||||
wait_for_version "$NEW_VERSION"
|
||||
|
||||
[ "$(sed -n '1p' "$DATA_ROOT/data/upgrade-smoke-marker")" = "preserve-across-upgrade" ] || {
|
||||
printf 'upgrade did not preserve application data\n' >&2
|
||||
exit 1
|
||||
}
|
||||
LOGIN=$(curl -fsS -c "$COOKIE_JAR" -H 'Content-Type: application/json' \
|
||||
--data "{\"password\":\"$PASSWORD\",\"remember_device\":false}" \
|
||||
"http://127.0.0.1:$PORT/api/v1/auth/login")
|
||||
case "$LOGIN" in
|
||||
*'"csrf_token"'*) ;;
|
||||
*) printf 'upgrade did not preserve the administrator password\n' >&2; exit 1 ;;
|
||||
esac
|
||||
grep -q '升级准备完成' "$LIFECYCLE_LOG"
|
||||
grep -q '升级数据检查完成' "$LIFECYCLE_LOG"
|
||||
|
||||
printf 'fnOS upgrade smoke passed: %s -> %s; data, access settings and password preserved\n' \
|
||||
"$OLD_VERSION" "$NEW_VERSION"
|
||||
Executable
+200
@@ -0,0 +1,200 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
PROJECT_ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
|
||||
PYTHON_BIN="${PYTHON:-$PROJECT_ROOT/.venv/bin/python}"
|
||||
FFMPEG_BIN="${FFMPEG:-$PROJECT_ROOT/vendor/ffmpeg}"
|
||||
FFPROBE_BIN="${FFPROBE:-$PROJECT_ROOT/vendor/ffprobe}"
|
||||
PORT="${1:-18766}"
|
||||
SMOKE_ROOT=$(mktemp -d "${TMPDIR:-/tmp}/imagefind-http-smoke.XXXXXX")
|
||||
DATA_ROOT="$SMOKE_ROOT/data"
|
||||
MEDIA_ROOT="$SMOKE_ROOT/media"
|
||||
COOKIE_JAR="$SMOKE_ROOT/cookies"
|
||||
SERVER_LOG="$SMOKE_ROOT/server.log"
|
||||
PASSWORD="imagefind-http-smoke-password"
|
||||
SERVER_PID=""
|
||||
|
||||
cleanup() {
|
||||
STATUS=$?
|
||||
trap - EXIT HUP INT TERM
|
||||
if [ -n "$SERVER_PID" ] && kill -0 "$SERVER_PID" 2>/dev/null; then
|
||||
kill "$SERVER_PID" 2>/dev/null || true
|
||||
wait "$SERVER_PID" 2>/dev/null || true
|
||||
fi
|
||||
if [ "$STATUS" -ne 0 ]; then
|
||||
printf '%s\n' 'HTTP smoke failed; application log follows:' >&2
|
||||
tail -100 "$SERVER_LOG" >&2 || true
|
||||
cp -p "$SERVER_LOG" "$PROJECT_ROOT/dist/http-smoke-failure.log" 2>/dev/null || true
|
||||
fi
|
||||
rm -rf -- "$SMOKE_ROOT"
|
||||
exit "$STATUS"
|
||||
}
|
||||
trap cleanup EXIT HUP INT TERM
|
||||
|
||||
mkdir -p "$DATA_ROOT" "$MEDIA_ROOT"
|
||||
"$FFMPEG_BIN" -hide_banner -loglevel error -f lavfi -i "testsrc=size=320x180:rate=24" \
|
||||
-t 2 -pix_fmt yuv420p -c:v libx264 -movflags +faststart "$SMOKE_ROOT/ABC-999.mp4"
|
||||
|
||||
env PYTHONPATH="$PROJECT_ROOT/backend" \
|
||||
IMAGEFIND_DATA_DIR="$DATA_ROOT" \
|
||||
IMAGEFIND_FRONTEND_DIR="$PROJECT_ROOT/frontend/dist" \
|
||||
IMAGEFIND_HOST="127.0.0.1" \
|
||||
IMAGEFIND_PORT="$PORT" \
|
||||
IMAGEFIND_UPLOAD_RESERVE_GB="0" \
|
||||
IMAGEFIND_RESOURCE_DISK_RESERVE_GB="0.5" \
|
||||
IMAGEFIND_EMBEDDING_BACKEND="hash" \
|
||||
IMAGEFIND_FFMPEG_PATH="$FFMPEG_BIN" \
|
||||
IMAGEFIND_FFPROBE_PATH="$FFPROBE_BIN" \
|
||||
"$PYTHON_BIN" -m imagefind.main >"$SERVER_LOG" 2>&1 &
|
||||
SERVER_PID=$!
|
||||
|
||||
ATTEMPT=0
|
||||
until curl -fsS "http://127.0.0.1:$PORT/api/v1/status" >/dev/null 2>&1; do
|
||||
ATTEMPT=$((ATTEMPT + 1))
|
||||
if [ "$ATTEMPT" -ge 60 ]; then
|
||||
printf '%s\n' 'ImageFind HTTP service did not become ready.' >&2
|
||||
exit 1
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
SETUP=$(curl -fsS -c "$COOKIE_JAR" -H 'Content-Type: application/json' \
|
||||
-d "{\"password\":\"$PASSWORD\"}" "http://127.0.0.1:$PORT/api/v1/setup")
|
||||
CSRF=$(printf '%s' "$SETUP" | "$PYTHON_BIN" -c 'import json,sys; print(json.load(sys.stdin)["csrf_token"])')
|
||||
SOURCE=$(curl -fsS -b "$COOKIE_JAR" -H 'Content-Type: application/json' -H "X-CSRF-Token: $CSRF" \
|
||||
-d "{\"name\":\"HTTP Smoke\",\"path\":\"$MEDIA_ROOT\"}" \
|
||||
"http://127.0.0.1:$PORT/api/v1/sources/local")
|
||||
SOURCE_ID=$(printf '%s' "$SOURCE" | "$PYTHON_BIN" -c 'import json,sys; print(json.load(sys.stdin)["id"])')
|
||||
curl -fsS -b "$COOKIE_JAR" -H 'Content-Type: application/json' -H "X-CSRF-Token: $CSRF" \
|
||||
-X PATCH -d '{"writable":true}' \
|
||||
"http://127.0.0.1:$PORT/api/v1/sources/$SOURCE_ID/writable" >/dev/null
|
||||
|
||||
SIZE=$(wc -c <"$SMOKE_ROOT/ABC-999.mp4" | tr -d ' ')
|
||||
UPLOAD=$(curl -fsS -b "$COOKIE_JAR" -H 'Content-Type: application/json' -H "X-CSRF-Token: $CSRF" \
|
||||
-d "{\"source_id\":\"$SOURCE_ID\",\"relative_path\":\"imports\",\"filename\":\"ABC-999.mp4\",\"size_bytes\":$SIZE}" \
|
||||
"http://127.0.0.1:$PORT/api/v1/uploads")
|
||||
UPLOAD_ID=$(printf '%s' "$UPLOAD" | "$PYTHON_BIN" -c 'import json,sys; print(json.load(sys.stdin)["id"])')
|
||||
DIGEST=$(sha256sum "$SMOKE_ROOT/ABC-999.mp4" | cut -d ' ' -f 1)
|
||||
curl -fsS -b "$COOKIE_JAR" -H 'Content-Type: application/octet-stream' -H "X-CSRF-Token: $CSRF" \
|
||||
-H "X-Chunk-SHA256: $DIGEST" -X PUT --data-binary "@$SMOKE_ROOT/ABC-999.mp4" \
|
||||
"http://127.0.0.1:$PORT/api/v1/uploads/$UPLOAD_ID/chunks/0" >/dev/null
|
||||
curl -fsS -b "$COOKIE_JAR" -H 'Content-Type: application/json' -H "X-CSRF-Token: $CSRF" \
|
||||
-d '{}' "http://127.0.0.1:$PORT/api/v1/uploads/$UPLOAD_ID/complete" >/dev/null
|
||||
|
||||
ATTEMPT=0
|
||||
while :; do
|
||||
UPLOADS=$(curl -fsS -b "$COOKIE_JAR" "http://127.0.0.1:$PORT/api/v1/uploads")
|
||||
STATE=$(printf '%s' "$UPLOADS" | "$PYTHON_BIN" -c \
|
||||
'import json,sys; items=json.load(sys.stdin); uid=sys.argv[1]; print(next(x["status"] for x in items if x["id"] == uid))' \
|
||||
"$UPLOAD_ID")
|
||||
[ "$STATE" = "completed" ] && break
|
||||
[ "$STATE" = "failed" ] && { printf '%s\n' "$UPLOADS" >&2; exit 1; }
|
||||
ATTEMPT=$((ATTEMPT + 1))
|
||||
[ "$ATTEMPT" -lt 60 ] || { printf '%s\n' "Upload stayed in state $STATE" >&2; exit 1; }
|
||||
sleep 1
|
||||
done
|
||||
|
||||
VIDEOS=$(curl -fsS -b "$COOKIE_JAR" "http://127.0.0.1:$PORT/api/v1/videos")
|
||||
VIDEO_ID=$(printf '%s' "$VIDEOS" | "$PYTHON_BIN" -c \
|
||||
'import json,sys; print(next(x["id"] for x in json.load(sys.stdin) if x["display_name"] == "ABC-999.mp4"))')
|
||||
curl -fsS -b "$COOKIE_JAR" "http://127.0.0.1:$PORT/api/v1/videos/$VIDEO_ID/download" \
|
||||
-o "$SMOKE_ROOT/download.mp4"
|
||||
cmp "$SMOKE_ROOT/ABC-999.mp4" "$SMOKE_ROOT/download.mp4"
|
||||
|
||||
TOKEN_RESPONSE=$(curl -fsS -b "$COOKIE_JAR" -H 'Content-Type: application/json' \
|
||||
-H "X-CSRF-Token: $CSRF" -d '{"name":"HTTP WebDAV smoke"}' \
|
||||
"http://127.0.0.1:$PORT/api/v1/tokens")
|
||||
API_TOKEN=$(printf '%s' "$TOKEN_RESPONSE" | "$PYTHON_BIN" -c \
|
||||
'import json,sys; print(json.load(sys.stdin)["token"])')
|
||||
curl -fsS -b "$COOKIE_JAR" -H 'Content-Type: application/json' -H "X-CSRF-Token: $CSRF" \
|
||||
-X PATCH \
|
||||
-d "{\"enabled\":true,\"source_id\":\"$SOURCE_ID\",\"relative_path\":\"webdav-smoke\"}" \
|
||||
"http://127.0.0.1:$PORT/api/v1/webdav/config" >/dev/null
|
||||
|
||||
DAV_SIZE=$SIZE
|
||||
DAV_SPLIT=$((DAV_SIZE / 2))
|
||||
DAV_REMAINING=$((DAV_SIZE - DAV_SPLIT))
|
||||
dd if="$SMOKE_ROOT/ABC-999.mp4" of="$SMOKE_ROOT/dav-first.bin" bs=1 count="$DAV_SPLIT" 2>/dev/null
|
||||
dd if="$SMOKE_ROOT/ABC-999.mp4" of="$SMOKE_ROOT/dav-second.bin" bs=1 skip="$DAV_SPLIT" 2>/dev/null
|
||||
DAV_URL="http://127.0.0.1:$PORT/webdav/DAV-RESUME.mp4"
|
||||
FIRST_STATUS=$(curl -sS -u "imagefind:$API_TOKEN" -X PUT \
|
||||
-H "Content-Range: bytes 0-$((DAV_SPLIT - 1))/$DAV_SIZE" \
|
||||
-H "Content-Length: $DAV_SPLIT" --data-binary "@$SMOKE_ROOT/dav-first.bin" \
|
||||
-D "$SMOKE_ROOT/dav-first.headers" -o "$SMOKE_ROOT/dav-first.body" -w '%{http_code}' "$DAV_URL")
|
||||
[ "$FIRST_STATUS" = "204" ]
|
||||
FIRST_OFFSET=$(tr -d '\r' <"$SMOKE_ROOT/dav-first.headers" | awk \
|
||||
'tolower($1)=="upload-offset:" {print $2; exit}')
|
||||
[ "$FIRST_OFFSET" = "$DAV_SPLIT" ]
|
||||
HEAD_STATUS=$(curl -sS -u "imagefind:$API_TOKEN" -I \
|
||||
-D "$SMOKE_ROOT/dav-head.headers" -o /dev/null -w '%{http_code}' "$DAV_URL")
|
||||
[ "$HEAD_STATUS" = "200" ]
|
||||
HEAD_OFFSET=$(tr -d '\r' <"$SMOKE_ROOT/dav-head.headers" | awk \
|
||||
'tolower($1)=="upload-offset:" {print $2; exit}')
|
||||
[ "$HEAD_OFFSET" = "$DAV_SPLIT" ]
|
||||
SECOND_STATUS=$(curl -sS -u "imagefind:$API_TOKEN" -X PUT \
|
||||
-H "Content-Range: bytes $DAV_SPLIT-$((DAV_SIZE - 1))/$DAV_SIZE" \
|
||||
-H "Content-Length: $DAV_REMAINING" --data-binary "@$SMOKE_ROOT/dav-second.bin" \
|
||||
-D "$SMOKE_ROOT/dav-second.headers" -o "$SMOKE_ROOT/dav-second.body" -w '%{http_code}' "$DAV_URL")
|
||||
[ "$SECOND_STATUS" = "204" ]
|
||||
DAV_UPLOAD_ID=$(tr -d '\r' <"$SMOKE_ROOT/dav-second.headers" | awk \
|
||||
'tolower($1)=="x-imagefind-upload-id:" {print $2; exit}')
|
||||
[ -n "$DAV_UPLOAD_ID" ]
|
||||
|
||||
ATTEMPT=0
|
||||
while :; do
|
||||
UPLOADS=$(curl -fsS -b "$COOKIE_JAR" "http://127.0.0.1:$PORT/api/v1/uploads")
|
||||
STATE=$(printf '%s' "$UPLOADS" | "$PYTHON_BIN" -c \
|
||||
'import json,sys; items=json.load(sys.stdin); uid=sys.argv[1]; print(next(x["status"] for x in items if x["id"] == uid))' \
|
||||
"$DAV_UPLOAD_ID")
|
||||
[ "$STATE" = "completed" ] && break
|
||||
[ "$STATE" = "failed" ] && { printf '%s\n' "$UPLOADS" >&2; exit 1; }
|
||||
ATTEMPT=$((ATTEMPT + 1))
|
||||
[ "$ATTEMPT" -lt 60 ] || { printf '%s\n' "WebDAV upload stayed in state $STATE" >&2; exit 1; }
|
||||
sleep 1
|
||||
done
|
||||
cmp "$SMOKE_ROOT/ABC-999.mp4" "$MEDIA_ROOT/webdav-smoke/DAV-RESUME.mp4"
|
||||
|
||||
DEDUP_STATUS=$(curl -sS -u "imagefind:$API_TOKEN" -X PUT \
|
||||
-H "Content-Length: $DAV_SIZE" -H "X-Content-SHA256: $DIGEST" \
|
||||
--data-binary "@$SMOKE_ROOT/ABC-999.mp4" -D "$SMOKE_ROOT/dav-dedup.headers" \
|
||||
-o "$SMOKE_ROOT/dav-dedup.body" -w '%{http_code}' "$DAV_URL")
|
||||
[ "$DEDUP_STATUS" = "204" ]
|
||||
DEDUPLICATED=$(tr -d '\r' <"$SMOKE_ROOT/dav-dedup.headers" | awk \
|
||||
'tolower($1)=="x-imagefind-deduplicated:" {print tolower($2); exit}')
|
||||
[ "$DEDUPLICATED" = "true" ]
|
||||
|
||||
if [ "${IMAGEFIND_RUN_WEBDAV_STRESS:-0}" = "1" ]; then
|
||||
if [ "${IMAGEFIND_RUN_WEBDAV_STRESS_FULL:-0}" = "1" ]; then
|
||||
"$PYTHON_BIN" "$PROJECT_ROOT/scripts/stress-webdav.py" \
|
||||
--base-url "http://127.0.0.1:$PORT" --token "$API_TOKEN" --workers 6 \
|
||||
--timeout 7200 --media-prefix "$SMOKE_ROOT/ABC-999.mp4" --full
|
||||
elif [ "${IMAGEFIND_RUN_WEBDAV_STRESS_LARGE:-0}" = "1" ]; then
|
||||
"$PYTHON_BIN" "$PROJECT_ROOT/scripts/stress-webdav.py" \
|
||||
--base-url "http://127.0.0.1:$PORT" --token "$API_TOKEN" --workers 4 \
|
||||
--timeout 1800 --media-prefix "$SMOKE_ROOT/ABC-999.mp4" --large
|
||||
else
|
||||
"$PYTHON_BIN" "$PROJECT_ROOT/scripts/stress-webdav.py" \
|
||||
--base-url "http://127.0.0.1:$PORT" --token "$API_TOKEN" --workers 4 \
|
||||
--timeout 600 --media-prefix "$SMOKE_ROOT/ABC-999.mp4"
|
||||
fi
|
||||
fi
|
||||
|
||||
RANGE_STATUS=$(curl -sS -b "$COOKIE_JAR" -H 'Range: bytes=128-1023' \
|
||||
-o "$SMOKE_ROOT/range.bin" -w '%{http_code}' \
|
||||
"http://127.0.0.1:$PORT/api/v1/videos/$VIDEO_ID/stream")
|
||||
[ "$RANGE_STATUS" = "206" ]
|
||||
dd if="$SMOKE_ROOT/ABC-999.mp4" of="$SMOKE_ROOT/expected-range.bin" bs=1 skip=128 count=896 2>/dev/null
|
||||
cmp "$SMOKE_ROOT/expected-range.bin" "$SMOKE_ROOT/range.bin"
|
||||
|
||||
PREVIEW=$(curl -fsS -b "$COOKIE_JAR" -H 'Content-Type: application/json' -H "X-CSRF-Token: $CSRF" \
|
||||
-d '{"start_ms":0}' "http://127.0.0.1:$PORT/api/v1/videos/$VIDEO_ID/preview")
|
||||
PLAYLIST_URL=$(printf '%s' "$PREVIEW" | "$PYTHON_BIN" -c \
|
||||
'import json,sys; print(json.load(sys.stdin)["playlist_url"])')
|
||||
PLAYLIST=$(curl -fsS -b "$COOKIE_JAR" "http://127.0.0.1:$PORT$PLAYLIST_URL")
|
||||
case "$PLAYLIST" in
|
||||
*'#EXTM3U'*'segment-00000.ts'*) ;;
|
||||
*) printf '%s\n' 'HLS playlist did not contain the first segment.' >&2; exit 1 ;;
|
||||
esac
|
||||
|
||||
printf 'upload=%s video=%s bytes=%s range=206 preview=ok webdav_resume=%s webdav_dedup=true sha256=%s\n' \
|
||||
"$UPLOAD_ID" "$VIDEO_ID" "$SIZE" "$DAV_UPLOAD_ID" "$DIGEST"
|
||||
Executable
+345
@@ -0,0 +1,345 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { createRequire } from "node:module";
|
||||
|
||||
const require = createRequire(new URL("../frontend/package.json", import.meta.url));
|
||||
const { chromium } = require("playwright");
|
||||
|
||||
const baseUrl = (process.env.IMAGEFIND_LIVE_URL || "").replace(/\/$/, "");
|
||||
const password = process.env.IMAGEFIND_LIVE_PASSWORD || "";
|
||||
const tokenFile = process.env.IMAGEFIND_LIVE_TOKEN_FILE || "";
|
||||
const apiToken = tokenFile ? fs.readFileSync(tokenFile, "utf8").trim() : "";
|
||||
const outputDir = path.resolve(process.env.IMAGEFIND_LIVE_OUTPUT || "dist/live-ui-audit");
|
||||
if (!baseUrl || (!password && !apiToken)) {
|
||||
throw new Error("IMAGEFIND_LIVE_URL and either IMAGEFIND_LIVE_PASSWORD or IMAGEFIND_LIVE_TOKEN_FILE are required");
|
||||
}
|
||||
fs.mkdirSync(outputDir, { recursive: true });
|
||||
|
||||
const auditPages = ["search", "library", "series", "uploads", "tags", "actors", "settings"];
|
||||
const mobileMenuNames = {
|
||||
library: "资料库",
|
||||
series: "合集",
|
||||
uploads: "上传中心",
|
||||
tags: "分类与标签",
|
||||
actors: "演员与人物",
|
||||
settings: "设置",
|
||||
};
|
||||
const desktopSelectors = {
|
||||
library: ".nav-item-3",
|
||||
series: ".nav-item-4",
|
||||
uploads: ".sidebar nav > .upload-status",
|
||||
tags: ".nav-item-6",
|
||||
actors: ".nav-item-7",
|
||||
settings: ".nav-item-8",
|
||||
};
|
||||
|
||||
async function login(page, name) {
|
||||
await page.goto(baseUrl, { waitUntil: "domcontentloaded", timeout: 30_000 });
|
||||
await page.waitForTimeout(1_000);
|
||||
const passwordInput = page.getByLabel("管理员密码");
|
||||
if (await passwordInput.isVisible().catch(() => false)) {
|
||||
await passwordInput.fill(password);
|
||||
await page.getByRole("button", { name: "登录", exact: true }).click();
|
||||
}
|
||||
try {
|
||||
await page.locator(".app-shell").waitFor({ state: "visible", timeout: 30_000 });
|
||||
} catch (error) {
|
||||
await page.screenshot({ path: path.join(outputDir, `${name}-login-error.png`), fullPage: true });
|
||||
const text = (await page.locator("body").innerText().catch(() => "")).slice(0, 2_000);
|
||||
throw new Error(`login did not reach the application: url=${page.url()} body=${JSON.stringify(text)}`, {
|
||||
cause: error,
|
||||
});
|
||||
}
|
||||
await page.waitForTimeout(2_000);
|
||||
}
|
||||
|
||||
async function resetHome(page) {
|
||||
await page.evaluate(() => {
|
||||
history.replaceState(
|
||||
{ ...history.state, imagefindNavigation: { kind: "page", page: "home", scrollY: 0 } },
|
||||
"",
|
||||
);
|
||||
});
|
||||
await page.reload({ waitUntil: "domcontentloaded", timeout: 30_000 });
|
||||
await page.locator(".app-shell").waitFor({ timeout: 30_000 });
|
||||
}
|
||||
|
||||
function attachNetworkAudit(page, errors) {
|
||||
const issues = [];
|
||||
let allowUnauthenticatedProbe = true;
|
||||
page.on("response", response => {
|
||||
if (response.status() < 400) return;
|
||||
const request = response.request();
|
||||
const url = new URL(response.url());
|
||||
const expectedProbe = allowUnauthenticatedProbe
|
||||
&& response.status() === 401
|
||||
&& request.method() === "GET"
|
||||
&& url.pathname === "/api/v1/auth/me";
|
||||
if (expectedProbe) return;
|
||||
const issue = {
|
||||
method: request.method(),
|
||||
status: response.status(),
|
||||
path: url.pathname,
|
||||
resourceType: request.resourceType(),
|
||||
};
|
||||
issues.push(issue);
|
||||
errors.push(`response: ${issue.method} ${issue.status} ${issue.path} [${issue.resourceType}]`);
|
||||
});
|
||||
page.on("requestfailed", request => {
|
||||
const url = new URL(request.url());
|
||||
const issue = {
|
||||
method: request.method(),
|
||||
status: null,
|
||||
path: url.pathname,
|
||||
resourceType: request.resourceType(),
|
||||
failure: request.failure()?.errorText || "unknown network failure",
|
||||
};
|
||||
issues.push(issue);
|
||||
errors.push(`requestfailed: ${issue.method} ${issue.path} [${issue.resourceType}] ${issue.failure}`);
|
||||
});
|
||||
return {
|
||||
issues,
|
||||
markAuthenticated() {
|
||||
allowUnauthenticatedProbe = false;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function inspect(browser, viewport, name) {
|
||||
const context = await browser.newContext({ viewport, locale: "zh-CN" });
|
||||
if (apiToken) {
|
||||
const origin = new URL(baseUrl).origin;
|
||||
await context.route("**/*", async route => {
|
||||
const url = new URL(route.request().url());
|
||||
if (url.origin !== origin || !url.pathname.includes("/api/")) {
|
||||
await route.continue();
|
||||
return;
|
||||
}
|
||||
await route.continue({
|
||||
headers: { ...route.request().headers(), authorization: `Bearer ${apiToken}` },
|
||||
});
|
||||
});
|
||||
}
|
||||
const page = await context.newPage();
|
||||
const errors = [];
|
||||
page.on("pageerror", error => errors.push(`pageerror: ${error.message}`));
|
||||
page.on("console", message => {
|
||||
if (message.type() === "error") errors.push(`console: ${message.text()}`);
|
||||
});
|
||||
const networkAudit = attachNetworkAudit(page, errors);
|
||||
await login(page, name);
|
||||
networkAudit.markAuthenticated();
|
||||
const home = await page.evaluate(() => ({
|
||||
scrollWidth: document.documentElement.scrollWidth,
|
||||
clientWidth: document.documentElement.clientWidth,
|
||||
scrollHeight: document.documentElement.scrollHeight,
|
||||
clientHeight: document.documentElement.clientHeight,
|
||||
}));
|
||||
const cards = page.locator("article.video-card");
|
||||
const cardCount = await cards.count();
|
||||
const coverErrors = await page.locator(".thumb-placeholder.error").count();
|
||||
await page.screenshot({ path: path.join(outputDir, `${name}-home.png`), fullPage: true });
|
||||
|
||||
const pageAudits = [];
|
||||
for (const destination of auditPages) {
|
||||
await resetHome(page);
|
||||
if (destination === "search") {
|
||||
if (viewport.width <= 720) {
|
||||
await page.getByRole("button", { name: "搜索", exact: true }).first().click();
|
||||
} else {
|
||||
await page.locator(".nav-item-2").click();
|
||||
}
|
||||
} else if (viewport.width <= 720) {
|
||||
await page.locator(".mobile-more-button").click();
|
||||
await page.locator(".mobile-more-menu")
|
||||
.getByRole("button", { name: new RegExp(mobileMenuNames[destination]) })
|
||||
.first()
|
||||
.click();
|
||||
} else {
|
||||
await page.locator(desktopSelectors[destination]).click();
|
||||
}
|
||||
await page.locator("main .page").waitFor({ state: "visible", timeout: 20_000 });
|
||||
await page.waitForTimeout(500);
|
||||
const geometry = await page.evaluate(() => {
|
||||
const content = document.querySelector(".content");
|
||||
const pagination = document.querySelector(".transfer-pagination");
|
||||
return {
|
||||
documentWidth: document.documentElement.scrollWidth,
|
||||
viewportWidth: document.documentElement.clientWidth,
|
||||
bodyWidth: document.body.scrollWidth,
|
||||
contentWidth: content?.scrollWidth || 0,
|
||||
contentClientWidth: content?.clientWidth || 0,
|
||||
contentHeight: content?.scrollHeight || 0,
|
||||
contentClientHeight: content?.clientHeight || 0,
|
||||
transferRows: document.querySelectorAll(".transfer-list > article").length,
|
||||
pagination: pagination?.textContent?.replace(/\s+/g, " ").trim() || null,
|
||||
paginationButtonHeights: pagination
|
||||
? [...pagination.querySelectorAll("button")].map(button => Math.round(button.getBoundingClientRect().height))
|
||||
: [],
|
||||
};
|
||||
});
|
||||
let paginationNavigation = null;
|
||||
if (destination === "uploads" && geometry.pagination) {
|
||||
const before = await page.locator(".transfer-list > article").first().innerText();
|
||||
const next = page.locator(".transfer-pagination").getByRole("button", { name: "下一页" });
|
||||
if (await next.isEnabled()) {
|
||||
await next.click();
|
||||
await page.waitForFunction(() => document.querySelector(".transfer-pagination")?.textContent?.includes("第 2 /"));
|
||||
const after = await page.locator(".transfer-list > article").first().innerText();
|
||||
paginationNavigation = {
|
||||
changed: before !== after,
|
||||
rows: await page.locator(".transfer-list > article").count(),
|
||||
text: (await page.locator(".transfer-pagination").innerText()).replace(/\s+/g, " ").trim(),
|
||||
};
|
||||
}
|
||||
}
|
||||
pageAudits.push({ destination, ...geometry, paginationNavigation });
|
||||
await page.screenshot({
|
||||
path: path.join(outputDir, `${name}-${destination}.png`),
|
||||
// Hundreds of historical tasks can make this page >17,000px tall. The
|
||||
// UI geometry is sampled above; keep the screenshot bounded to avoid a
|
||||
// Chromium bitmap allocation crash on low-memory test hosts.
|
||||
fullPage: destination !== "uploads",
|
||||
});
|
||||
}
|
||||
|
||||
await resetHome(page);
|
||||
|
||||
let scrollY = 0;
|
||||
let mobileMenu = null;
|
||||
if (viewport.width <= 720) {
|
||||
await page.evaluate(() => window.scrollTo(0, document.documentElement.scrollHeight));
|
||||
await page.waitForTimeout(500);
|
||||
scrollY = await page.evaluate(() => window.scrollY);
|
||||
await page.evaluate(() => window.scrollTo(0, 0));
|
||||
await page.locator(".mobile-more-button").click();
|
||||
const sheet = page.locator(".mobile-more-menu");
|
||||
await sheet.waitFor({ state: "visible" });
|
||||
// Visibility is reported at the first frame of the slide-in transition.
|
||||
// Geometry sampled there includes the temporary translateY and used to be
|
||||
// misreported as a sheet extending below the viewport.
|
||||
await page.waitForTimeout(300);
|
||||
mobileMenu = await sheet.evaluate(element => {
|
||||
const box = element.getBoundingClientRect();
|
||||
const buttons = [...element.querySelectorAll(".mobile-more-item")].map(button => {
|
||||
const rect = button.getBoundingClientRect();
|
||||
const style = getComputedStyle(button);
|
||||
return {
|
||||
text: button.textContent?.trim(),
|
||||
contentCenterDelta: Math.round(Math.max(0, ...[...button.children].map(child => {
|
||||
const childRect = child.getBoundingClientRect();
|
||||
return Math.abs(rect.left + rect.width / 2 - childRect.left - childRect.width / 2);
|
||||
}))),
|
||||
background: style.backgroundColor,
|
||||
};
|
||||
});
|
||||
return {
|
||||
top: Math.round(box.top),
|
||||
bottom: Math.round(box.bottom),
|
||||
viewportBottom: window.innerHeight,
|
||||
transform: getComputedStyle(element).transform,
|
||||
buttons,
|
||||
};
|
||||
});
|
||||
await page.screenshot({ path: path.join(outputDir, `${name}-more.png`), fullPage: true });
|
||||
await page.getByRole("button", { name: "关闭", exact: true }).click();
|
||||
await page.locator(".mobile-more-menu").waitFor({ state: "hidden" });
|
||||
await page.locator(".mobile-profile-button").click();
|
||||
await page.locator(".profile-page").waitFor({ state: "visible" });
|
||||
await page.screenshot({ path: path.join(outputDir, `${name}-profile.png`), fullPage: true });
|
||||
await page.getByRole("button", { name: "首页", exact: true }).click();
|
||||
}
|
||||
|
||||
let player = null;
|
||||
if (cardCount) {
|
||||
const preferred = cards.filter({ hasText: "e2e-positive" });
|
||||
await ((await preferred.count()) ? preferred.first() : cards.first()).click();
|
||||
await page.locator(".player-page").waitFor({ state: "visible", timeout: 20_000 });
|
||||
await page.waitForTimeout(2_000);
|
||||
const status = await page.locator(".player-center-status").textContent().catch(() => null);
|
||||
const stage = page.locator(".custom-player");
|
||||
if ((await stage.getAttribute("class"))?.includes("controls-hidden")) {
|
||||
await page.locator(".player-gesture-surface").click({ position: { x: 12, y: 12 } });
|
||||
await page.waitForTimeout(350);
|
||||
}
|
||||
const markerButton = page.getByRole("button", { name: "视频时间点" });
|
||||
const hitTest = await markerButton.evaluate(button => {
|
||||
const rect = button.getBoundingClientRect();
|
||||
const x = rect.left + rect.width / 2;
|
||||
const y = rect.top + rect.height / 2;
|
||||
const controls = button.closest(".player-controls");
|
||||
const controlsStyle = controls ? getComputedStyle(controls) : null;
|
||||
return {
|
||||
stageClass: button.closest(".custom-player")?.className,
|
||||
controls: controlsStyle ? {
|
||||
opacity: controlsStyle.opacity,
|
||||
pointerEvents: controlsStyle.pointerEvents,
|
||||
zIndex: controlsStyle.zIndex,
|
||||
} : null,
|
||||
stack: document.elementsFromPoint(x, y).slice(0, 6).map(element => ({
|
||||
tag: element.tagName,
|
||||
className: element.className,
|
||||
pointerEvents: getComputedStyle(element).pointerEvents,
|
||||
zIndex: getComputedStyle(element).zIndex,
|
||||
})),
|
||||
};
|
||||
});
|
||||
await markerButton.click();
|
||||
await page.locator(".player-marker-drawer").waitFor({ state: "visible" });
|
||||
await page.screenshot({ path: path.join(outputDir, `${name}-player.png`), fullPage: true });
|
||||
player = {
|
||||
status: status?.trim() || "ready",
|
||||
hitTest,
|
||||
markerDrawer: await page.locator(".player-marker-drawer").isVisible(),
|
||||
progressVisible: await page.getByLabel("播放进度").isVisible(),
|
||||
horizontalOverflow: await page.evaluate(
|
||||
() => document.documentElement.scrollWidth > document.documentElement.clientWidth,
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
await context.close();
|
||||
return {
|
||||
name,
|
||||
viewport,
|
||||
home,
|
||||
cardCount,
|
||||
coverErrors,
|
||||
pageAudits,
|
||||
scrollY,
|
||||
mobileMenu,
|
||||
player,
|
||||
networkIssues: networkAudit.issues,
|
||||
errors: [...new Set(errors)].slice(0, 20),
|
||||
};
|
||||
}
|
||||
|
||||
const cases = [
|
||||
[{ width: 320, height: 780 }, "mobile-320"],
|
||||
[{ width: 390, height: 844 }, "mobile-390"],
|
||||
[{ width: 768, height: 900 }, "tablet-768"],
|
||||
[{ width: 1024, height: 900 }, "desktop-1024"],
|
||||
[{ width: 1440, height: 900 }, "desktop-1440"],
|
||||
];
|
||||
const requested = new Set(
|
||||
(process.env.IMAGEFIND_LIVE_VIEWPORTS || "")
|
||||
.split(",")
|
||||
.map(value => value.trim())
|
||||
.filter(Boolean),
|
||||
);
|
||||
const results = [];
|
||||
for (const [viewport, name] of cases) {
|
||||
if (requested.size && !requested.has(name)) continue;
|
||||
// Chromium retains decoded full-page screenshots in process caches. A fresh
|
||||
// process per viewport keeps the five-size audit bounded on small CI hosts.
|
||||
const browser = await chromium.launch({ headless: true });
|
||||
try {
|
||||
results.push(await inspect(browser, viewport, name));
|
||||
fs.writeFileSync(path.join(outputDir, "report.json"), JSON.stringify(results, null, 2));
|
||||
} finally {
|
||||
await browser.close();
|
||||
}
|
||||
}
|
||||
console.log(JSON.stringify(results, null, 2));
|
||||
Executable
+376
@@ -0,0 +1,376 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Orthogonal ImageFind WebDAV upload stress test with isolated cleanup.
|
||||
|
||||
The default profile is CI-friendly. Pass ``--full`` to execute the requested
|
||||
10 MiB / 1 GiB / 5 GiB singles, ten-file batch and fifty 10 MiB-file batch.
|
||||
The API token is used for both WebDAV Basic authentication and cleanup.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import concurrent.futures
|
||||
import hashlib
|
||||
import os
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from collections.abc import Iterator
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from urllib.parse import quote
|
||||
|
||||
import httpx
|
||||
|
||||
MIB = 1024**2
|
||||
GIB = 1024**3
|
||||
CHUNK = MIB
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Case:
|
||||
virtual_path: str
|
||||
size: int
|
||||
resume: bool = False
|
||||
|
||||
|
||||
@dataclass
|
||||
class TimedBody:
|
||||
name: str
|
||||
size: int
|
||||
prefix: bytes = b""
|
||||
start: int = 0
|
||||
end: int | None = None
|
||||
finished_at: float | None = None
|
||||
|
||||
def __iter__(self) -> Iterator[bytes]:
|
||||
yield from blocks(
|
||||
self.name,
|
||||
self.size,
|
||||
prefix=self.prefix,
|
||||
start=self.start,
|
||||
end=self.end,
|
||||
)
|
||||
# httpx resumes the iterator after it has handed the final request
|
||||
# bytes to the transport. The remaining interval is server response
|
||||
# latency, including any accidental post-PUT whole-file work.
|
||||
self.finished_at = time.monotonic()
|
||||
|
||||
|
||||
def blocks(
|
||||
name: str,
|
||||
size: int,
|
||||
*,
|
||||
prefix: bytes = b"",
|
||||
start: int = 0,
|
||||
end: int | None = None,
|
||||
) -> Iterator[bytes]:
|
||||
end = size if end is None else end
|
||||
seed = hashlib.sha256(name.encode()).digest()
|
||||
block = (seed * (CHUNK // len(seed) + 1))[:CHUNK]
|
||||
position = start
|
||||
while position < end:
|
||||
if position < len(prefix):
|
||||
length = min(len(prefix) - position, end - position)
|
||||
yield prefix[position : position + length]
|
||||
position += length
|
||||
continue
|
||||
offset = position % CHUNK
|
||||
length = min(CHUNK - offset, end - position)
|
||||
yield block[offset : offset + length]
|
||||
position += length
|
||||
|
||||
|
||||
def encoded_path(path: str) -> str:
|
||||
return "/".join(quote(part, safe="") for part in path.split("/"))
|
||||
|
||||
|
||||
class StressRun:
|
||||
def __init__(
|
||||
self,
|
||||
base_url: str,
|
||||
token: str,
|
||||
full: bool,
|
||||
large: bool,
|
||||
workers: int,
|
||||
timeout: float,
|
||||
probe_timeout: float,
|
||||
media_prefix: bytes = b"",
|
||||
):
|
||||
self.base_url = base_url.rstrip("/")
|
||||
self.token = token
|
||||
self.full = full
|
||||
self.large = large
|
||||
self.workers = workers
|
||||
self.timeout = timeout
|
||||
self.probe_timeout = probe_timeout
|
||||
self.media_prefix = media_prefix
|
||||
self.run_id = f"ifstress-{uuid.uuid4().hex[:10]}"
|
||||
basic = base64.b64encode(f"imagefind:{token}".encode()).decode()
|
||||
self.dav_headers = {"Authorization": f"Basic {basic}"}
|
||||
self.api_headers = {"Authorization": f"Bearer {token}"}
|
||||
self.upload_ids: set[str] = set()
|
||||
self.request_timeout = httpx.Timeout(connect=15, write=60, read=120, pool=30)
|
||||
self.probe_latencies: list[float] = []
|
||||
self.probe_errors: list[str] = []
|
||||
|
||||
def probe(self, stopping: threading.Event) -> None:
|
||||
timeout = httpx.Timeout(connect=3, write=3, read=self.probe_timeout, pool=3)
|
||||
while not stopping.is_set():
|
||||
started = time.monotonic()
|
||||
try:
|
||||
response = httpx.get(f"{self.base_url}/api/v1/status", timeout=timeout)
|
||||
response.raise_for_status()
|
||||
self.probe_latencies.append(time.monotonic() - started)
|
||||
except Exception as exc:
|
||||
if len(self.probe_errors) < 10:
|
||||
self.probe_errors.append(f"{type(exc).__name__}: {exc}")
|
||||
stopping.wait(0.25)
|
||||
|
||||
def cases(self) -> list[Case]:
|
||||
if self.large:
|
||||
return [
|
||||
Case(f"{self.run_id}/large-{index}.mp4", 256 * MIB)
|
||||
for index in range(4)
|
||||
]
|
||||
single_sizes = [10 * MIB, GIB, 5 * GIB] if self.full else [10 * MIB]
|
||||
cases = [
|
||||
Case(f"{self.run_id}/single-{size}.mp4", size, resume=index == 0)
|
||||
for index, size in enumerate(single_sizes)
|
||||
]
|
||||
cases.append(Case(f"root-{self.run_id}.mp4", 10 * MIB if self.full else MIB))
|
||||
for index in range(10):
|
||||
depth = (1, 5, 10)[index % 3]
|
||||
groups = "/".join(f"level-{level}" for level in range(1, depth + 1))
|
||||
size = 10 * MIB if self.full else MIB
|
||||
cases.append(Case(f"{self.run_id}/{groups}/{self.run_id}-ten-{index:02d}.mp4", size))
|
||||
for index in range(50):
|
||||
size = 10 * MIB if self.full else 256 * 1024
|
||||
cases.append(Case(f"{self.run_id}/fifty/{self.run_id}-fifty-{index:02d}.mp4", size))
|
||||
return cases
|
||||
|
||||
def put(self, case: Case) -> tuple[str, int, str | None, float]:
|
||||
url = f"{self.base_url}/webdav/{encoded_path(case.virtual_path)}"
|
||||
headers = dict(self.dav_headers)
|
||||
upload_id = None
|
||||
response_body: TimedBody
|
||||
with httpx.Client(timeout=self.request_timeout) as client:
|
||||
if case.resume:
|
||||
split = case.size // 2
|
||||
first_body = TimedBody(
|
||||
case.virtual_path, case.size, self.media_prefix, end=split
|
||||
)
|
||||
first = client.put(
|
||||
url,
|
||||
headers={
|
||||
**headers,
|
||||
"Content-Length": str(split),
|
||||
"Content-Range": f"bytes 0-{split - 1}/{case.size}",
|
||||
},
|
||||
content=first_body,
|
||||
)
|
||||
first.raise_for_status()
|
||||
if first.headers.get("upload-offset") != str(split):
|
||||
raise RuntimeError(f"{case.virtual_path}: first offset was not persisted")
|
||||
head = client.head(url, headers=headers)
|
||||
head.raise_for_status()
|
||||
if head.headers.get("upload-offset") != str(split):
|
||||
raise RuntimeError(f"{case.virtual_path}: HEAD did not report the partial offset")
|
||||
response_body = TimedBody(
|
||||
case.virtual_path, case.size, self.media_prefix, start=split
|
||||
)
|
||||
response = client.put(
|
||||
url,
|
||||
headers={
|
||||
**headers,
|
||||
"Content-Length": str(case.size - split),
|
||||
"Content-Range": f"bytes {split}-{case.size - 1}/{case.size}",
|
||||
},
|
||||
content=response_body,
|
||||
)
|
||||
else:
|
||||
response_body = TimedBody(case.virtual_path, case.size, self.media_prefix)
|
||||
response = client.put(
|
||||
url,
|
||||
headers={**headers, "Content-Length": str(case.size)},
|
||||
content=response_body,
|
||||
)
|
||||
response.raise_for_status()
|
||||
upload_id = response.headers.get("x-imagefind-upload-id")
|
||||
response_latency = time.monotonic() - (response_body.finished_at or time.monotonic())
|
||||
return case.virtual_path, case.size, upload_id, response_latency
|
||||
|
||||
def wait(self, expected: int) -> None:
|
||||
deadline = time.monotonic() + self.timeout
|
||||
while time.monotonic() < deadline:
|
||||
response = httpx.get(
|
||||
f"{self.base_url}/api/v1/uploads?limit=500",
|
||||
headers=self.api_headers,
|
||||
timeout=30,
|
||||
)
|
||||
response.raise_for_status()
|
||||
rows = [row for row in response.json() if row.get("id") in self.upload_ids]
|
||||
failures = [row for row in rows if row.get("status") == "failed"]
|
||||
if failures:
|
||||
raise RuntimeError(f"upload failures: {failures}")
|
||||
if len(rows) == expected and all(row.get("status") == "completed" for row in rows):
|
||||
return
|
||||
time.sleep(2)
|
||||
raise TimeoutError("uploads did not complete before the stress timeout")
|
||||
|
||||
def upload_rows(self) -> list[dict]:
|
||||
response = httpx.get(
|
||||
f"{self.base_url}/api/v1/uploads?limit=500",
|
||||
headers=self.api_headers,
|
||||
timeout=self.request_timeout,
|
||||
)
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
|
||||
def verify_dedup(self, case: Case) -> None:
|
||||
url = f"{self.base_url}/webdav/{encoded_path(case.virtual_path)}"
|
||||
before = {row["id"] for row in self.upload_rows()}
|
||||
body = TimedBody(case.virtual_path, case.size, self.media_prefix)
|
||||
response = httpx.put(
|
||||
url,
|
||||
headers={
|
||||
**self.dav_headers,
|
||||
"Content-Length": str(case.size),
|
||||
},
|
||||
content=body,
|
||||
timeout=self.request_timeout,
|
||||
)
|
||||
response.raise_for_status()
|
||||
if response.headers.get("x-imagefind-deduplicated") != "true":
|
||||
raise RuntimeError("full retry after a simulated lost response was not deduplicated")
|
||||
after = {row["id"] for row in self.upload_rows()}
|
||||
if after != before:
|
||||
raise RuntimeError("full retry created another upload task")
|
||||
latency = time.monotonic() - (body.finished_at or time.monotonic())
|
||||
print(
|
||||
f"deduplicated lost-response retry response_latency={latency:.3f}s",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
def cleanup(self) -> None:
|
||||
try:
|
||||
videos = httpx.get(
|
||||
f"{self.base_url}/api/v1/videos?limit=500", headers=self.api_headers, timeout=30
|
||||
).json()
|
||||
for video in videos:
|
||||
if self.run_id in str(video.get("display_name")) or self.run_id in str(video.get("source_key")):
|
||||
response = httpx.delete(
|
||||
f"{self.base_url}/api/v1/videos/{video['id']}?delete_source=true",
|
||||
headers=self.api_headers,
|
||||
timeout=60,
|
||||
)
|
||||
if response.status_code != 404:
|
||||
response.raise_for_status()
|
||||
collections = httpx.get(
|
||||
f"{self.base_url}/api/v1/collections", headers=self.api_headers, timeout=30
|
||||
).json()
|
||||
for collection in collections:
|
||||
if collection.get("name") == self.run_id:
|
||||
httpx.delete(
|
||||
f"{self.base_url}/api/v1/collections/{collection['id']}",
|
||||
headers=self.api_headers,
|
||||
timeout=30,
|
||||
).raise_for_status()
|
||||
trash = httpx.get(
|
||||
f"{self.base_url}/api/v1/trash", headers=self.api_headers, timeout=30
|
||||
).json()
|
||||
for item in trash:
|
||||
if self.run_id in str(item.get("display_name")) or self.run_id in str(item.get("original_key")):
|
||||
response = httpx.delete(
|
||||
f"{self.base_url}/api/v1/trash/{item['id']}",
|
||||
headers=self.api_headers,
|
||||
timeout=60,
|
||||
)
|
||||
if response.status_code != 404:
|
||||
response.raise_for_status()
|
||||
except Exception as exc:
|
||||
print(f"cleanup warning: {exc}", file=sys.stderr)
|
||||
|
||||
def execute(self, keep: bool) -> None:
|
||||
cases = self.cases()
|
||||
started = time.monotonic()
|
||||
probe_stopping = threading.Event()
|
||||
probe_thread = threading.Thread(
|
||||
target=self.probe,
|
||||
args=(probe_stopping,),
|
||||
name="imagefind-stress-probe",
|
||||
daemon=True,
|
||||
)
|
||||
probe_thread.start()
|
||||
try:
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=self.workers) as executor:
|
||||
futures = [executor.submit(self.put, case) for case in cases]
|
||||
for future in concurrent.futures.as_completed(futures):
|
||||
path, size, upload_id, response_latency = future.result()
|
||||
if upload_id:
|
||||
self.upload_ids.add(upload_id)
|
||||
print(
|
||||
f"received {path} ({size} bytes) "
|
||||
f"response_latency={response_latency:.3f}s",
|
||||
flush=True,
|
||||
)
|
||||
self.wait(len(self.upload_ids))
|
||||
self.verify_dedup(cases[0])
|
||||
if self.probe_errors:
|
||||
raise RuntimeError(
|
||||
"ImageFind API became unreachable during WebDAV transfer: "
|
||||
+ "; ".join(self.probe_errors)
|
||||
)
|
||||
maximum_probe = max(self.probe_latencies, default=0)
|
||||
print(
|
||||
f"PASS run={self.run_id} files={len(cases)} bytes={sum(case.size for case in cases)} "
|
||||
f"seconds={time.monotonic() - started:.1f} max_probe_latency={maximum_probe:.3f}s",
|
||||
flush=True,
|
||||
)
|
||||
finally:
|
||||
probe_stopping.set()
|
||||
probe_thread.join(timeout=self.probe_timeout + 5)
|
||||
if not keep:
|
||||
self.cleanup()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--base-url", required=True, help="ImageFind native URL, e.g. http://nas:8765")
|
||||
parser.add_argument("--token", default=os.getenv("IMAGEFIND_WEBDAV_TOKEN"))
|
||||
profile = parser.add_mutually_exclusive_group()
|
||||
profile.add_argument("--full", action="store_true", help="include 1 GiB, 5 GiB and full-size batches")
|
||||
profile.add_argument("--large", action="store_true", help="run four concurrent 256 MiB uploads")
|
||||
parser.add_argument("--workers", type=int, default=8)
|
||||
parser.add_argument("--timeout", type=float, default=7200)
|
||||
parser.add_argument(
|
||||
"--probe-timeout",
|
||||
type=float,
|
||||
default=10,
|
||||
help="fail when the API is unreachable for this many seconds during transfers",
|
||||
)
|
||||
parser.add_argument("--keep", action="store_true", help="keep isolated stress data for inspection")
|
||||
parser.add_argument(
|
||||
"--media-prefix",
|
||||
help="optional valid media file prepended before deterministic padding",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
if not args.token:
|
||||
parser.error("--token or IMAGEFIND_WEBDAV_TOKEN is required")
|
||||
media_prefix = Path(args.media_prefix).read_bytes() if args.media_prefix else b""
|
||||
StressRun(
|
||||
args.base_url,
|
||||
args.token,
|
||||
args.full,
|
||||
args.large,
|
||||
min(10, max(1, args.workers)),
|
||||
args.timeout,
|
||||
max(1, args.probe_timeout),
|
||||
media_prefix,
|
||||
).execute(args.keep)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+189
@@ -0,0 +1,189 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
FILE=${1:?usage: upload-openlist-artifact.sh file [remote-directory]}
|
||||
REMOTE_DIR=${2:-/yidongpan/构建产物/liverecorder}
|
||||
OPENLIST_BASE_URL=${OPENLIST_BASE_URL:-https://openlist.nxsir.cn}
|
||||
OPENLIST_USERNAME=${OPENLIST_USERNAME:?OPENLIST_USERNAME is required}
|
||||
OPENLIST_PASSWORD=${OPENLIST_PASSWORD:?OPENLIST_PASSWORD is required}
|
||||
OPENLIST_UPLOAD_ATTEMPTS=${OPENLIST_UPLOAD_ATTEMPTS:-6}
|
||||
|
||||
test -f "$FILE" || { printf 'artifact does not exist: %s\n' "$FILE" >&2; exit 1; }
|
||||
for required_command in curl python3 stat basename sleep; do
|
||||
command -v "$required_command" >/dev/null 2>&1 || {
|
||||
printf 'required upload command is missing: %s\n' "$required_command" >&2
|
||||
exit 1
|
||||
}
|
||||
done
|
||||
|
||||
OPENLIST_BASE_URL=${OPENLIST_BASE_URL%/}
|
||||
REMOTE_DIR="/${REMOTE_DIR#/}"
|
||||
REMOTE_DIR=${REMOTE_DIR%/}
|
||||
FILE_NAME=$(basename -- "$FILE")
|
||||
BUILD_MARKER=${BUILD_TAG:-${BUILD_ID:-$$}}
|
||||
BUILD_MARKER=$(printf '%s' "$BUILD_MARKER" | tr -c 'A-Za-z0-9._-' '-')
|
||||
TEMP_NAME=".${FILE_NAME}.uploading-${BUILD_MARKER}"
|
||||
TEMP_PATH="$REMOTE_DIR/$TEMP_NAME"
|
||||
FINAL_PATH="$REMOTE_DIR/$FILE_NAME"
|
||||
LOCAL_SIZE=$(stat -c '%s' "$FILE")
|
||||
TOKEN=
|
||||
|
||||
json_string() {
|
||||
python3 -c 'import json,sys; print(json.dumps(sys.argv[1], ensure_ascii=False))' "$1"
|
||||
}
|
||||
|
||||
envelope_code() {
|
||||
python3 -c 'import json,sys
|
||||
try:
|
||||
value=json.load(sys.stdin)
|
||||
print(value.get("code", -1))
|
||||
except Exception:
|
||||
print(-1)'
|
||||
}
|
||||
|
||||
envelope_message() {
|
||||
python3 -c 'import json,sys
|
||||
try:
|
||||
value=json.load(sys.stdin)
|
||||
print(value.get("message", "unknown OpenList response"))
|
||||
except Exception:
|
||||
print("invalid OpenList response")'
|
||||
}
|
||||
|
||||
login() {
|
||||
local payload response code
|
||||
payload=$(printf '{"username":%s,"password":%s}' \
|
||||
"$(json_string "$OPENLIST_USERNAME")" \
|
||||
"$(json_string "$OPENLIST_PASSWORD")")
|
||||
response=$(curl --silent --show-error --location --fail-with-body \
|
||||
--connect-timeout 30 --max-time 120 \
|
||||
--request POST "$OPENLIST_BASE_URL/api/auth/login" \
|
||||
--header 'Content-Type: application/json' \
|
||||
--data-raw "$payload") || return 1
|
||||
code=$(printf '%s' "$response" | envelope_code)
|
||||
[ "$code" = "200" ] || {
|
||||
printf 'OpenList login failed: %s\n' "$(printf '%s' "$response" | envelope_message)" >&2
|
||||
return 1
|
||||
}
|
||||
TOKEN=$(printf '%s' "$response" | python3 -c 'import json,sys; print((json.load(sys.stdin).get("data") or {}).get("token", ""))')
|
||||
[ -n "$TOKEN" ]
|
||||
}
|
||||
|
||||
post_json() {
|
||||
local endpoint=$1 payload=$2
|
||||
curl --silent --show-error --location --fail-with-body \
|
||||
--connect-timeout 30 --max-time 180 \
|
||||
--request POST "$OPENLIST_BASE_URL$endpoint" \
|
||||
--header "Authorization: $TOKEN" \
|
||||
--header 'Content-Type: application/json' \
|
||||
--data-raw "$payload"
|
||||
}
|
||||
|
||||
remote_size() {
|
||||
local path=$1 response code
|
||||
response=$(post_json /api/fs/get "{\"path\":$(json_string "$path"),\"password\":\"\"}") || return 1
|
||||
code=$(printf '%s' "$response" | envelope_code)
|
||||
[ "$code" = "200" ] || return 1
|
||||
printf '%s' "$response" | python3 -c 'import json,sys; print((json.load(sys.stdin).get("data") or {}).get("size", -1))'
|
||||
}
|
||||
|
||||
ensure_remote_directory() {
|
||||
local current= segment response code
|
||||
while IFS= read -r segment; do
|
||||
[ -n "$segment" ] || continue
|
||||
current="$current/$segment"
|
||||
response=$(post_json /api/fs/list "{\"path\":$(json_string "$current"),\"password\":\"\",\"refresh\":false,\"page\":1,\"per_page\":1}" || true)
|
||||
code=$(printf '%s' "$response" | envelope_code)
|
||||
if [ "$code" = "200" ]; then
|
||||
continue
|
||||
fi
|
||||
response=$(post_json /api/fs/mkdir "{\"path\":$(json_string "$current")}") || return 1
|
||||
code=$(printf '%s' "$response" | envelope_code)
|
||||
[ "$code" = "200" ] || {
|
||||
printf 'OpenList mkdir failed for %s: %s\n' "$current" "$(printf '%s' "$response" | envelope_message)" >&2
|
||||
return 1
|
||||
}
|
||||
done < <(python3 -c 'import sys; print("\n".join(part for part in sys.argv[1].split("/") if part))' "$REMOTE_DIR")
|
||||
}
|
||||
|
||||
remove_temporary_file() {
|
||||
local response
|
||||
[ -n "$TOKEN" ] || login >/dev/null 2>&1 || return 0
|
||||
response=$(post_json /api/fs/remove "{\"dir\":$(json_string "$REMOTE_DIR"),\"names\":[$(json_string "$TEMP_NAME")]}" 2>/dev/null || true)
|
||||
[ "$(printf '%s' "$response" | envelope_code)" = "200" ] || true
|
||||
}
|
||||
|
||||
publish_temporary_file() {
|
||||
local response code
|
||||
response=$(post_json /api/fs/rename "{\"path\":$(json_string "$TEMP_PATH"),\"name\":$(json_string "$FILE_NAME")}") || return 1
|
||||
code=$(printf '%s' "$response" | envelope_code)
|
||||
[ "$code" = "200" ] || {
|
||||
printf 'OpenList rename failed: %s\n' "$(printf '%s' "$response" | envelope_message)" >&2
|
||||
return 1
|
||||
}
|
||||
}
|
||||
|
||||
upload_once() {
|
||||
local encoded_path response code uploaded_size
|
||||
encoded_path=$(python3 -c 'import sys,urllib.parse; print(urllib.parse.quote(sys.argv[1], safe=""))' "$TEMP_PATH")
|
||||
response=$(curl --silent --show-error --location --fail-with-body \
|
||||
--connect-timeout 30 --max-time 3600 --speed-limit 1024 --speed-time 120 \
|
||||
--request PUT "$OPENLIST_BASE_URL/api/fs/put" \
|
||||
--header "Authorization: $TOKEN" \
|
||||
--header "File-Path: $encoded_path" \
|
||||
--header 'As-Task: false' \
|
||||
--header 'Content-Type: application/octet-stream' \
|
||||
--header 'Expect:' \
|
||||
--data-binary "@$FILE") || return 1
|
||||
code=$(printf '%s' "$response" | envelope_code)
|
||||
[ "$code" = "200" ] || {
|
||||
printf 'OpenList upload failed: %s\n' "$(printf '%s' "$response" | envelope_message)" >&2
|
||||
return 1
|
||||
}
|
||||
uploaded_size=$(remote_size "$TEMP_PATH") || return 1
|
||||
[ "$uploaded_size" = "$LOCAL_SIZE" ] || {
|
||||
printf 'OpenList size mismatch for temporary upload: local=%s remote=%s\n' "$LOCAL_SIZE" "$uploaded_size" >&2
|
||||
return 1
|
||||
}
|
||||
}
|
||||
|
||||
printf 'Uploading %s (%s bytes) to OpenList %s\n' "$FILE_NAME" "$LOCAL_SIZE" "$REMOTE_DIR"
|
||||
|
||||
for attempt in $(seq 1 "$OPENLIST_UPLOAD_ATTEMPTS"); do
|
||||
TOKEN=
|
||||
upload_ready=1
|
||||
if login && ensure_remote_directory; then
|
||||
existing_size=$(remote_size "$FINAL_PATH" 2>/dev/null || true)
|
||||
if [ "$existing_size" = "$LOCAL_SIZE" ]; then
|
||||
printf 'OpenList artifact already exists with matching size: %s\n' "$FINAL_PATH"
|
||||
exit 0
|
||||
fi
|
||||
if [ -n "$existing_size" ]; then
|
||||
remove_response=$(post_json /api/fs/remove "{\"dir\":$(json_string "$REMOTE_DIR"),\"names\":[$(json_string "$FILE_NAME")]}" || true)
|
||||
if [ "$(printf '%s' "$remove_response" | envelope_code)" != "200" ]; then
|
||||
printf 'Unable to remove mismatched OpenList artifact before retry: %s\n' "$FINAL_PATH" >&2
|
||||
upload_ready=0
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ "$upload_ready" = "1" ] && upload_once && publish_temporary_file; then
|
||||
final_size=$(remote_size "$FINAL_PATH") || final_size=-1
|
||||
if [ "$final_size" = "$LOCAL_SIZE" ]; then
|
||||
printf 'OpenList upload verified: %s (%s bytes)\n' "$FINAL_PATH" "$final_size"
|
||||
exit 0
|
||||
fi
|
||||
printf 'OpenList final size mismatch: local=%s remote=%s\n' "$LOCAL_SIZE" "$final_size" >&2
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ "$attempt" -lt "$OPENLIST_UPLOAD_ATTEMPTS" ]; then
|
||||
delay=$((5 * (1 << (attempt - 1))))
|
||||
[ "$delay" -le 120 ] || delay=120
|
||||
printf 'OpenList upload attempt %s/%s failed; retrying in %ss\n' "$attempt" "$OPENLIST_UPLOAD_ATTEMPTS" "$delay" >&2
|
||||
sleep "$delay"
|
||||
fi
|
||||
done
|
||||
|
||||
remove_temporary_file
|
||||
printf 'OpenList upload failed after %s attempts: %s\n' "$OPENLIST_UPLOAD_ATTEMPTS" "$FILE_NAME" >&2
|
||||
exit 1
|
||||
Executable
+35
@@ -0,0 +1,35 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
PACKAGE=${1:?usage: verify-fnos-package.sh package.fpk [expected-version]}
|
||||
EXPECTED_VERSION=${2:-}
|
||||
VERIFY_ROOT=${IMAGEFIND_VERIFY_TMPDIR:-${TMPDIR:-/tmp}}
|
||||
mkdir -p "$VERIFY_ROOT"
|
||||
work_dir=$(mktemp -d "${VERIFY_ROOT%/}/imagefind-fnos-verify.XXXXXX")
|
||||
trap 'rm -rf -- "$work_dir"' EXIT
|
||||
|
||||
tar -xzf "$PACKAGE" -C "$work_dir"
|
||||
value() { sed -n "s/^$1[[:space:]]*=[[:space:]]*//p" "$work_dir/manifest" | head -n 1 | tr -d '\r'; }
|
||||
test "$(value appname)" = imagefind
|
||||
version=$(value version)
|
||||
test -n "$version"
|
||||
if [ -n "$EXPECTED_VERSION" ]; then test "$version" = "$EXPECTED_VERSION"; fi
|
||||
test "$(value platform)" = x86
|
||||
test "$(value install_dep_apps)" = python312,nxsir.postgresql
|
||||
test -x "$work_dir/cmd/main"
|
||||
test -s "$work_dir/app.tgz"
|
||||
gzip -t "$work_dir/app.tgz"
|
||||
tar -tzf "$work_dir/app.tgz" >"$work_dir/files.txt"
|
||||
grep -q '^frontend/index.html$' "$work_dir/files.txt"
|
||||
grep -q '^runtime/imagefind-.*\.whl$' "$work_dir/files.txt"
|
||||
grep -q '^runtime/wheels/.*\.whl$' "$work_dir/files.txt"
|
||||
grep -q '^bin/libOpenCL.so.1$' "$work_dir/files.txt"
|
||||
runtime_version=$(tar -xOzf "$work_dir/app.tgz" runtime/VERSION | sed -n '1p')
|
||||
test "$runtime_version" = "$version"
|
||||
if grep -Eq '^(data|models|cache|postgresql)/' "$work_dir/files.txt"; then
|
||||
printf 'persistent data must not be included in ImageFind app.tgz\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
test -s "$PACKAGE.sha256"
|
||||
(cd "$(dirname "$PACKAGE")" && sha256sum --check --status "$(basename "$PACKAGE").sha256")
|
||||
printf 'ImageFind fnOS package verified: %s (%s)\n' "$PACKAGE" "$version"
|
||||
Reference in New Issue
Block a user