feat: add ImageFind application and release pipelines

This commit is contained in:
2026-08-11 18:02:40 +08:00
commit 16239d7525
270 changed files with 59163 additions and 0 deletions
+104
View File
@@ -0,0 +1,104 @@
#!/bin/sh
set -eu
PROJECT_ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
BASE_VERSION=$(sed -n 's/^version[[:space:]]*=[[:space:]]*//p' "$PROJECT_ROOT/fnos/manifest" | head -n 1 | tr -d '\r')
PACKAGE_VERSION="${PACKAGE_VERSION:-$BASE_VERSION}"
case "$PACKAGE_VERSION" in
''|*[!0-9.]*|.*|*..*|*.) printf 'invalid fnOS package version: %s\n' "$PACKAGE_VERSION" >&2; exit 1 ;;
esac
[ "$(printf '%s' "$PACKAGE_VERSION" | awk -F. '{ print NF }')" -eq 3 ] || {
printf 'fnOS package version must contain exactly three numeric components: %s\n' "$PACKAGE_VERSION" >&2
exit 1
}
BUILD_ROOT="$PROJECT_ROOT/.build-fnos"
STAGE="$BUILD_ROOT/imagefind"
PACKED_ROOT="$BUILD_ROOT/packed"
FNPACK_TMP_ROOT="$BUILD_ROOT/fnpack-tmp"
WHEEL_BUILD_ROOT="$BUILD_ROOT/application-wheel"
OUTPUT="${1:-$PROJECT_ROOT/dist/imagefind-${PACKAGE_VERSION}-x86_64.fpk}"
WHEEL_CACHE="${IMAGEFIND_WHEEL_CACHE:-$PROJECT_ROOT/.fnos-wheel-cache/python312}"
if [ -n "${FNPACK:-}" ]; then
FNPACK_BIN="$FNPACK"
elif [ -x "$PROJECT_ROOT/.tools/fnpack" ]; then
FNPACK_BIN="$PROJECT_ROOT/.tools/fnpack"
else
FNPACK_BIN=fnpack
fi
if [ -n "${PYTHON:-}" ]; then
PYTHON_BIN="$PYTHON"
elif [ -x "$PROJECT_ROOT/.release-venv/bin/python" ]; then
PYTHON_BIN="$PROJECT_ROOT/.release-venv/bin/python"
elif [ -x "$PROJECT_ROOT/.venv/bin/python" ]; then
PYTHON_BIN="$PROJECT_ROOT/.venv/bin/python"
else
PYTHON_BIN=python3
fi
require_file() {
if [ ! -e "$1" ]; then
printf 'missing required release artifact: %s\n' "$1" >&2
exit 1
fi
}
FNPACK_BIN=$(command -v "$FNPACK_BIN") || {
printf 'fnpack is required; install it from the fnOS developer portal.\n' >&2
exit 1
}
PYTHON_BIN=$(command -v "$PYTHON_BIN") || {
printf 'Python with hatchling is required to assemble the release.\n' >&2
exit 1
}
command -v npm >/dev/null 2>&1 || { printf 'npm is required.\n' >&2; exit 1; }
"$PYTHON_BIN" -c 'import hatchling' >/dev/null 2>&1 || {
printf 'hatchling is required in the release Python environment.\n' >&2
exit 1
}
for artifact in libOpenCL.so.1; do
require_file "$PROJECT_ROOT/vendor/$artifact"
done
require_file "$PROJECT_ROOT/requirements/runtime-core.txt"
require_file "$PROJECT_ROOT/requirements/runtime-ai/constraints-cp312.txt"
[ -d "$WHEEL_CACHE" ] || { printf 'missing Python 3.12 core wheelhouse: %s\n' "$WHEEL_CACHE" >&2; exit 1; }
rm -rf "$BUILD_ROOT"
mkdir -p "$STAGE/app/bin" "$STAGE/app/frontend" "$STAGE/app/runtime/ai" \
"$STAGE/app/runtime/wheels" "$PACKED_ROOT" "$FNPACK_TMP_ROOT" \
"$WHEEL_BUILD_ROOT" "$(dirname "$OUTPUT")"
npm run build --prefix "$PROJECT_ROOT/frontend"
"$PYTHON_BIN" -m hatchling build -t wheel -d "$WHEEL_BUILD_ROOT"
cp -R "$PROJECT_ROOT/fnos/." "$STAGE/"
sed -i -E "s/^version[[:space:]]*=.*/version=${PACKAGE_VERSION}/" "$STAGE/manifest"
cp -R "$PROJECT_ROOT/frontend/dist/." "$STAGE/app/frontend/"
cp -p "$PROJECT_ROOT/vendor/libOpenCL.so.1" "$STAGE/app/bin/libOpenCL.so.1"
cp -p "$PROJECT_ROOT/requirements/runtime-core.txt" "$STAGE/app/runtime/runtime-core.txt"
cp -p "$PROJECT_ROOT/requirements/runtime-ai/"*.txt "$STAGE/app/runtime/ai/"
cp -p "$WHEEL_CACHE/"*.whl "$STAGE/app/runtime/wheels/"
cp -p "$WHEEL_BUILD_ROOT/"imagefind-*.whl "$STAGE/app/runtime/"
STAGED_APP_WHEEL=""
for candidate in "$STAGE/app/runtime/"imagefind-*.whl; do
if [ -f "$candidate" ]; then
STAGED_APP_WHEEL="$candidate"
break
fi
done
[ -n "$STAGED_APP_WHEEL" ] || { printf 'built application wheel is missing\n' >&2; exit 1; }
PACKAGE_ID=$("$PYTHON_BIN" -c \
'import hashlib,sys; h=hashlib.sha256(); [h.update(open(path,"rb").read()) for path in sys.argv[1:]]; print(h.hexdigest())' \
"$STAGED_APP_WHEEL" "$STAGE/app/runtime/runtime-core.txt")
printf '%s\n%s\n' "$PACKAGE_VERSION" "$PACKAGE_ID" >"$STAGE/app/runtime/VERSION"
"$PYTHON_BIN" "$PROJECT_ROOT/scripts/make_icons.py" "$STAGE"
chmod 0755 "$STAGE/cmd/"*
(
cd "$PACKED_ROOT"
TMPDIR="$FNPACK_TMP_ROOT" "$FNPACK_BIN" build --directory "$STAGE"
)
mv "$PACKED_ROOT/imagefind.fpk" "$OUTPUT"
(
cd "$(dirname "$OUTPUT")"
sha256sum "$(basename "$OUTPUT")" >"$(basename "$OUTPUT").sha256"
)
printf 'created %s\n' "$OUTPUT"
+84
View File
@@ -0,0 +1,84 @@
from __future__ import annotations
import argparse
import hashlib
import io
import json
import tarfile
from datetime import UTC, datetime
from pathlib import Path
def digest(path: Path) -> str:
value = hashlib.sha256()
with path.open("rb") as handle:
while chunk := handle.read(1024 * 1024):
value.update(chunk)
return value.hexdigest()
def main() -> None:
parser = argparse.ArgumentParser(description="Build a versioned ImageFind model bundle")
parser.add_argument("source", type=Path)
parser.add_argument("output", type=Path)
parser.add_argument("--version", default="1")
args = parser.parse_args()
source = args.source.resolve()
output = args.output.resolve()
for required in ("visual/image", "visual/text"):
if not (source / required).is_dir():
parser.error(f"missing {required}")
optional_components = {
"ocr": ("det.onnx", "rec.onnx", "cls.onnx"),
"faces": ("detector.xml", "detector.bin", "reidentification.xml", "reidentification.bin"),
}
for component, required_files in optional_components.items():
root = source / component
if root.exists():
missing = [name for name in required_files if not (root / name).is_file()]
if missing:
parser.error(f"incomplete {component}: missing {', '.join(missing)}")
audio_root = source / "audio"
if audio_root.exists() and (not (audio_root / "config.json").is_file() or not list(audio_root.glob("*.xml"))):
parser.error("incomplete audio: missing config.json or OpenVINO XML files")
try:
output.relative_to(source)
except ValueError:
pass
else:
parser.error("output must be outside the source model directory")
files = sorted(
path for path in source.rglob("*") if path.is_file() and path.relative_to(source).as_posix() != "manifest.json"
)
created_at = datetime.now(UTC)
manifest = {
"format_version": 2,
"version": args.version,
"created_at": created_at.isoformat(),
"source": "imagefind-build-model-bundle",
"components": {
"visual": {"version": args.version},
**({"ocr": {"version": args.version}} if (source / "ocr").is_dir() else {}),
**({"faces": {"version": args.version}} if (source / "faces").is_dir() else {}),
**({"audio": {"version": args.version}} if audio_root.is_dir() else {}),
},
"files": {path.relative_to(source).as_posix(): digest(path) for path in files},
}
manifest_bytes = (json.dumps(manifest, ensure_ascii=False, indent=2) + "\n").encode()
output.parent.mkdir(parents=True, exist_ok=True)
with tarfile.open(output, "w:gz") as archive:
for path in sorted(source.rglob("*")):
if path.relative_to(source).as_posix() == "manifest.json":
continue
archive.add(path, path.relative_to(source).as_posix(), recursive=False)
info = tarfile.TarInfo("manifest.json")
info.size = len(manifest_bytes)
info.mode = 0o644
info.mtime = int(created_at.timestamp())
archive.addfile(info, io.BytesIO(manifest_bytes))
print(f"bundle={output} sha256={digest(output)}")
if __name__ == "__main__":
main()
+9
View File
@@ -0,0 +1,9 @@
#!/usr/bin/env bash
set -euo pipefail
PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
python3 -m compileall -q "$PROJECT_ROOT/scripts" "$PROJECT_ROOT/backend/imagefind"
while IFS= read -r script; do
node --check "$script"
done < <(find "$PROJECT_ROOT/scripts" -maxdepth 1 -type f -name '*.mjs' -print | sort)
+10
View File
@@ -0,0 +1,10 @@
#!/usr/bin/env bash
set -euo pipefail
: "${JENKINS_NODE_PASSWORD:?JENKINS_NODE_PASSWORD is required}"
printf '%s\n' "$JENKINS_NODE_PASSWORD" | sudo -S -p '' -v
sudo -n -- \
/usr/bin/env "DOCKER_CONFIG=${DOCKER_CONFIG:-}" \
/usr/bin/docker "$@"
+58
View File
@@ -0,0 +1,58 @@
#!/usr/bin/env bash
set -euo pipefail
PG_BIN=/usr/lib/postgresql/15/bin
PG_DATA=/data/postgresql
PG_LOG=/data/postgresql.log
APP_DATA=/data/imagefind
PG_CONF=/data/postgres-client.conf
ADMIN_MARKER=/data/.imagefind-admin-initialized
mkdir -p "$PG_DATA" "$APP_DATA" /data/runtime
chown -R postgres:postgres "$PG_DATA"
chown -R imagefind:imagefind "$APP_DATA" /data/runtime
if [ ! -s "$PG_DATA/PG_VERSION" ]; then
runuser -u postgres -- "$PG_BIN/initdb" -D "$PG_DATA" --auth-local=trust --auth-host=scram-sha-256
printf '%s\n' "listen_addresses = '127.0.0.1'" "port = 5432" >>"$PG_DATA/postgresql.conf"
fi
runuser -u postgres -- "$PG_BIN/pg_ctl" -D "$PG_DATA" -l "$PG_LOG" -w start
stop_postgres() {
runuser -u postgres -- "$PG_BIN/pg_ctl" -D "$PG_DATA" -m fast -w stop >/dev/null 2>&1 || true
}
trap stop_postgres EXIT TERM INT HUP
if [ ! -s "$PG_CONF" ]; then
role_exists=$(runuser -u postgres -- psql -Atqc "SELECT 1 FROM pg_roles WHERE rolname='imagefind'" postgres)
db_exists=$(runuser -u postgres -- psql -Atqc "SELECT 1 FROM pg_database WHERE datname='imagefind'" postgres)
if [ -n "$role_exists" ] || [ -n "$db_exists" ]; then
printf 'PostgreSQL data exists but %s is missing; refusing to replace credentials.\n' "$PG_CONF" >&2
exit 1
fi
db_password=$(python -c 'import secrets; print(secrets.token_urlsafe(48))')
runuser -u postgres -- psql -v ON_ERROR_STOP=1 -v password="$db_password" postgres <<'SQL'
CREATE ROLE imagefind LOGIN PASSWORD :'password';
SQL
runuser -u postgres -- createdb --owner=imagefind imagefind
runuser -u postgres -- psql -v ON_ERROR_STOP=1 imagefind -c 'CREATE EXTENSION IF NOT EXISTS vector; CREATE EXTENSION IF NOT EXISTS pg_trgm;'
umask 077
printf 'host=127.0.0.1\nport=5432\ndatabase=imagefind\nusername=imagefind\npassword=%s\nsslmode=Disable\n' \
"$db_password" >"$PG_CONF"
chown imagefind:imagefind "$PG_CONF"
fi
if [ ! -e "$ADMIN_MARKER" ]; then
: "${IMAGEFIND_ADMIN_PASSWORD:?IMAGEFIND_ADMIN_PASSWORD is required for first initialization}"
printf '%s' "$IMAGEFIND_ADMIN_PASSWORD" | runuser -u imagefind --preserve-environment -- imagefind admin-password --stdin
install -o imagefind -g imagefind -m 0600 /dev/null "$ADMIN_MARKER"
fi
unset IMAGEFIND_ADMIN_PASSWORD || true
runuser -u imagefind --preserve-environment -- imagefind &
app_pid=$!
set +e
wait "$app_pid"
status=$?
set -e
exit "$status"
+161
View File
@@ -0,0 +1,161 @@
#!/usr/bin/env python3
"""Read-only live acceptance for audio search and transcript APIs."""
from __future__ import annotations
import argparse
import json
import re
import sys
from pathlib import Path
import httpx
from zhconv import convert
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--base-url", required=True)
auth = parser.add_mutually_exclusive_group(required=True)
auth.add_argument("--password-stdin", action="store_true")
auth.add_argument("--token-file", type=Path)
args = parser.parse_args()
password = sys.stdin.readline().rstrip("\r\n") if args.password_stdin else ""
headers = {}
if args.token_file is not None:
headers["Authorization"] = f"Bearer {args.token_file.read_text().strip()}"
client = httpx.Client(
base_url=args.base_url.rstrip("/"), headers=headers, timeout=httpx.Timeout(30, connect=10)
)
try:
if args.password_stdin:
login = client.post(
"/api/v1/auth/login", json={"password": password, "remember_device": False}
)
login.raise_for_status()
csrf = str(login.json().get("csrf_token") or "")
if not csrf:
raise AssertionError("login response omitted CSRF token")
client.headers["X-CSRF-Token"] = csrf
password = ""
status = client.get("/api/v1/status")
status.raise_for_status()
if status.json().get("version") != "0.5.45":
raise AssertionError(f"unexpected version: {status.json().get('version')}")
coverage_response = client.get("/api/v1/search/coverage")
coverage_response.raise_for_status()
coverage = coverage_response.json()
required = {
"total",
"indexed",
"searchable",
"empty",
"queued",
"running",
"failed",
"percent",
"current_model_percent",
}
if not required <= coverage.keys():
raise AssertionError(f"coverage fields missing: {sorted(required - coverage.keys())}")
videos_response = client.get("/api/v1/videos?limit=200")
videos_response.raise_for_status()
videos = videos_response.json()
selected = None
transcript = None
selected_item = None
fallback = None
for video in videos:
response = client.get(f"/api/v1/videos/{video['id']}/transcript?page=1&page_size=30")
response.raise_for_status()
candidate = response.json()
if candidate.get("items"):
fallback = fallback or (video, candidate, candidate["items"][0])
chinese_item = next(
(
item
for item in candidate["items"]
if re.search(r"[\u3400-\u9fff]{2,}", str(item.get("raw_text") or ""))
),
None,
)
if chinese_item:
selected, transcript, selected_item = video, candidate, chinese_item
break
if selected is None and fallback:
selected, transcript, selected_item = fallback
checks: dict[str, object] = {
"version": status.json()["version"],
"coverage": {key: coverage[key] for key in sorted(required)},
"transcript_video_found": bool(selected),
}
resources_response = client.get("/api/v1/system/resources")
resources_response.raise_for_status()
resources = resources_response.json()
diagnostics_response = client.get("/api/v1/system/diagnostics")
diagnostics_response.raise_for_status()
diagnostics = diagnostics_response.json()
models_response = client.get("/api/v1/models")
models_response.raise_for_status()
models = models_response.json()
checks["runtime"] = {
"cpu_percent": resources.get("cpu_percent"),
"memory_available_bytes": resources.get("memory_available_bytes"),
"io": resources.get("io"),
"ai_lane": (resources.get("lanes") or {}).get("ai"),
"database": diagnostics.get("database"),
"event_loop_lag_ms": diagnostics.get("event_loop_lag_ms"),
"request_p95_ms": diagnostics.get("request_p95_ms"),
"audio_accelerator": ((models.get("accelerator") or {}).get("components") or {}).get("audio"),
"audio_health": (models.get("component_health") or {}).get("audio"),
}
if selected and transcript:
first = selected_item or transcript["items"][0]
cjk_runs = re.findall(r"[\u3400-\u9fff]{2,}", str(first.get("raw_text") or ""))
if cjk_runs:
query = cjk_runs[0][: min(6, len(cjk_runs[0]))]
variants = list(dict.fromkeys((query, convert(query, "zh-cn"), convert(query, "zh-tw"))))
results = []
for variant in variants:
audio = client.post(
"/api/v1/search",
json={"text": variant, "recognition_types": ["audio"], "limit": 50},
)
audio.raise_for_status()
audio_ids = {item.get("video_id") for item in audio.json().get("items", [])}
combined = client.post("/api/v1/search", json={"text": variant, "limit": 50})
combined.raise_for_status()
combined_ids = {item.get("video_id") for item in combined.json().get("items", [])}
results.append(
{
"variant_changed": variant != query,
"audio_match": selected["id"] in audio_ids,
"combined_match": selected["id"] in combined_ids,
}
)
if not all(item["audio_match"] and item["combined_match"] for item in results):
raise AssertionError(f"live Chinese audio search mismatch: {results}")
checks["chinese_query_length"] = len(query)
checks["variant_checks"] = results
checks["transcript"] = {
"status": transcript.get("status"),
"total": transcript.get("total"),
"pages": transcript.get("pages"),
"items_returned": len(transcript.get("items", [])),
}
print(json.dumps(checks, ensure_ascii=False, indent=2), flush=True)
finally:
password = ""
try:
if args.password_stdin:
client.post("/api/v1/auth/logout")
finally:
client.close()
if __name__ == "__main__":
main()
+77
View File
@@ -0,0 +1,77 @@
#!/usr/bin/env node
import { mkdir } from "node:fs/promises";
import process from "node:process";
import playwright from "../frontend/node_modules/@playwright/test/index.js";
const { chromium } = playwright;
const chunks = [];
for await (const chunk of process.stdin) chunks.push(chunk);
const password = Buffer.concat(chunks).toString("utf8").trim();
if (!password) throw new Error("password is required on stdin");
const baseURL = String(process.argv[2] || "").replace(/\/$/, "");
if (!baseURL) throw new Error("base URL is required");
const output = new URL("../.playwright-results/live-0516/", import.meta.url).pathname;
await mkdir(output, { recursive: true });
const browser = await chromium.launch({ headless: true });
const reports = [];
try {
for (const viewport of [{ name: "desktop", width: 1440, height: 900 }, { name: "mobile", width: 390, height: 844 }]) {
const context = await browser.newContext({ viewport });
const page = await context.newPage();
await page.goto(baseURL, { waitUntil: "domcontentloaded" });
const passwordInput = page.getByLabel("管理员密码");
if (await passwordInput.isVisible().catch(() => false)) {
await passwordInput.fill(password);
const remember = page.locator('.remember-device input[type="checkbox"]');
if (await remember.isChecked()) await remember.uncheck();
await page.getByRole("button", { name: "登录", exact: true }).click();
}
try {
await page.locator(".app-shell").waitFor({ state: "visible", timeout: 30_000 });
await page.locator(".video-card,.home-empty").first().waitFor({ state: "visible", timeout: 30_000 });
} catch (error) {
const message = (await page.locator("body").innerText()).replace(/\s+/g, " ").slice(0, 500);
throw new Error(`${viewport.name} app shell did not become ready: ${message}`, { cause: error });
}
await page.locator(".nav-item-2").click();
await page.locator(".search-coverage").waitFor({ state: "visible", timeout: 15_000 });
await page.waitForFunction(
() => !document.querySelector(".search-coverage")?.textContent?.includes("正在读取"),
undefined,
{ timeout: 30_000 },
);
const coverage = (await page.locator(".search-coverage").innerText()).replace(/\s+/g, " ").trim();
const searchGeometry = await page.evaluate(() => ({
viewport: innerWidth,
document: document.documentElement.scrollWidth,
content: document.querySelector(".content")?.scrollWidth || 0,
}));
if (searchGeometry.document > searchGeometry.viewport + 1 || searchGeometry.content > searchGeometry.viewport + 1) {
throw new Error(`${viewport.name} search page overflows horizontally: ${JSON.stringify(searchGeometry)}`);
}
await page.screenshot({ path: `${output}/${viewport.name}-search.png`, fullPage: true });
await page.locator(".nav-item-1").click();
await page.locator(".video-card").first().click();
await page.locator(".transcript-panel").waitFor({ state: "visible", timeout: 15_000 });
await page.locator(".transcript-toggle").click();
await page.waitForFunction(
() => !document.querySelector(".transcript-toggle")?.textContent?.includes("正在读取"),
undefined,
{ timeout: 30_000 },
);
const transcriptStatus = (await page.locator(".transcript-toggle").innerText()).replace(/\s+/g, " ").trim();
const transcriptItems = await page.locator(".transcript-list>button").count();
await page.screenshot({ path: `${output}/${viewport.name}-player.png`, fullPage: true });
reports.push({ viewport: viewport.name, coverage, transcriptStatus, transcriptItems, searchGeometry });
await page.evaluate(async () => {
const csrf = sessionStorage.getItem("imagefind:csrf") || "";
await fetch("/api/v1/auth/logout", { method: "POST", headers: { "X-CSRF-Token": csrf } });
});
await context.close();
}
} finally {
await browser.close();
}
console.log(JSON.stringify(reports, null, 2));
+190
View File
@@ -0,0 +1,190 @@
#!/usr/bin/env python3
"""Exercise mutable live APIs using only run-owned ImageFind records."""
from __future__ import annotations
import argparse
import json
import time
from pathlib import Path
from typing import Any
import httpx
def call(client: httpx.Client, method: str, path: str, **kwargs: Any) -> httpx.Response:
response = client.request(method, path, **kwargs)
if response.is_error:
raise RuntimeError(f"{method} {path} -> {response.status_code}: {response.text[:1000]}")
return response
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--base-url", required=True)
auth = parser.add_mutually_exclusive_group(required=True)
auth.add_argument("--token-file", type=Path)
auth.add_argument("--password")
parser.add_argument("--run-dir", type=Path, required=True)
parser.add_argument("--case-id", default="")
args = parser.parse_args()
state = json.loads((args.run_dir / "state.json").read_text())
headers = {}
if args.token_file is not None:
headers["Authorization"] = f"Bearer {args.token_file.read_text().strip()}"
client = httpx.Client(
base_url=args.base_url.rstrip("/"),
headers=headers,
timeout=httpx.Timeout(60, connect=10),
)
if args.password is not None:
login = call(
client,
"POST",
"/api/v1/auth/login",
json={"password": args.password, "remember_device": False},
).json()
csrf = str(login.get("csrf_token") or "")
if not csrf:
raise AssertionError("login response omitted CSRF token")
client.headers["X-CSRF-Token"] = csrf
case_prefix = f"-{args.case_id.strip()}" if args.case_id.strip() else ""
expected_names = {
f"{state['run_id']}{case_prefix}-e2e-positive.mp4",
f"{state['run_id']}{case_prefix}-e2e-negative.mp4",
f"{state['run_id']}{case_prefix}-e2e-fallback.mpg",
}
videos = call(client, "GET", "/api/v1/videos?limit=500").json()
owned = {
item["source_key"].rsplit("/", 1)[-1]: item
for item in videos
if item["source_id"] == state["source_id"]
and item["source_key"].rsplit("/", 1)[-1] in expected_names
}
if set(owned) != expected_names:
raise AssertionError(f"unexpected run-owned videos: {sorted(owned)}")
positive = owned[f"{state['run_id']}{case_prefix}-e2e-positive.mp4"]
video_id = positive["id"]
metadata = {
"title": "星河验收影片 729",
"catalog_code": "IF-E2E-729",
"studio": "ImageFind 验收工作室",
"series": f"{state['run_id']}-合集",
"release_date": "2026-08-02",
"description": "用于验证标题、番号、片商、合集、日期、简介、演员和分类同步。",
"actors": ["测试演员 729"],
"tags": [],
"tag_ids": [state["tag_id"]],
}
call(client, "PATCH", f"/api/v1/videos/{video_id}/metadata", json=metadata)
call(
client,
"PATCH",
f"/api/v1/videos/{video_id}/state",
json={"liked": True, "favorited": True, "progress_ms": 4000, "completed": False},
)
refreshed = call(client, "GET", "/api/v1/videos?limit=500").json()
positive = next(item for item in refreshed if item["id"] == video_id)
for key in ("title", "catalog_code", "studio", "release_date", "description"):
if positive[key] != metadata[key]:
raise AssertionError(f"metadata mismatch {key}: {positive[key]!r}")
if positive["actors"] != ["测试演员 729"]:
raise AssertionError(f"actors mismatch: {positive['actors']}")
if positive["tag_items"][0]["id"] != state["tag_id"]:
raise AssertionError("tag assignment mismatch")
if not positive["liked"] or not positive["favorited"] or positive["progress_ms"] != 4000:
raise AssertionError("video state mismatch")
# Upload completion deliberately precedes the independent AI lane. The
# search result contract is frame-based, so wait for basic parsing instead
# of racing the freshly queued index job.
deadline = time.monotonic() + 300
while positive.get("index_state", {}).get("basic") == "pending":
if time.monotonic() >= deadline:
raise TimeoutError("basic video indexing did not reach a terminal state")
time.sleep(2)
refreshed = call(client, "GET", "/api/v1/videos?limit=500").json()
positive = next(item for item in refreshed if item["id"] == video_id)
filters = {}
for name, query in {
"favorite": "favorite=true",
"liked": "liked=true",
"played": "played_only=true&sort=last_played",
}.items():
result = call(client, "GET", f"/api/v1/videos?{query}&limit=500").json()
filters[name] = [item["id"] for item in result]
if video_id not in filters[name]:
raise AssertionError(f"{name} filter omitted the updated video")
profile = call(client, "GET", "/api/v1/profile").json()
expected_profile_counts = {"favorites": 1, "likes": 1}
for key, minimum in expected_profile_counts.items():
if int(profile.get("counts", {}).get(key, 0)) < minimum:
raise AssertionError(f"profile count did not include {key}")
collection = call(client, "GET", f"/api/v1/collections/{state['collection_id']}").json()
if video_id not in [item["id"] for item in collection["videos"]]:
raise AssertionError("collection detail omitted the updated video")
search = call(
client,
"POST",
"/api/v1/search",
json={"text": "IF-E2E-729", "recognition_types": ["metadata"]},
).json()
search_ids = [item["video_id"] for item in search["items"]]
search_deferred = positive.get("index_state", {}).get("basic") == "failed"
if video_id not in search_ids and not search_deferred:
raise AssertionError("metadata search did not return the matching video")
negative_search = call(
client,
"POST",
"/api/v1/search",
json={"text": "绝不应命中的验收词 9834721", "recognition_types": ["metadata"]},
).json()
if negative_search["items"]:
raise AssertionError("non-matching metadata search unexpectedly returned a video")
stream = call(
client,
"GET",
f"/api/v1/videos/{video_id}/stream",
headers={"Range": "bytes=0-1023"},
)
if stream.status_code != 206 or len(stream.content) != 1024:
raise AssertionError(f"range stream mismatch: {stream.status_code}, {len(stream.content)}")
with client.stream("GET", f"/api/v1/videos/{video_id}/download") as download:
if download.status_code != 200:
raise AssertionError(f"download status {download.status_code}")
disposition = download.headers.get("content-disposition", "")
if "attachment" not in disposition.lower():
raise AssertionError(f"download disposition mismatch: {disposition}")
first = next(download.iter_bytes(), b"")
if not first:
raise AssertionError("download returned an empty body")
report = {
"video_id": video_id,
"metadata": {key: positive[key] for key in metadata if key not in {"actors", "tags", "tag_ids"}},
"actors": positive["actors"],
"filters": filters,
"profile_counts": profile.get("counts", {}),
"collection_video_count": collection["video_count"],
"metadata_search_matches": len(search["items"]),
"metadata_search_deferred": search_deferred,
"range_stream": {"status": stream.status_code, "bytes": len(stream.content)},
"download": {"status": 200, "content_disposition": disposition},
}
(args.run_dir / "api-report.json").write_text(
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
)
state["positive_video_id"] = video_id
(args.run_dir / "state.json").write_text(json.dumps(state, ensure_ascii=False, indent=2) + "\n")
print(json.dumps(report, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()
+89
View File
@@ -0,0 +1,89 @@
#!/usr/bin/env node
import fs from "node:fs";
import path from "node:path";
import { createRequire } from "node:module";
const require = createRequire(new URL("../frontend/package.json", import.meta.url));
const { chromium } = require("playwright");
const baseUrl = String(process.env.IMAGEFIND_LIVE_URL || "").replace(/\/$/, "");
const tokenFile = process.env.IMAGEFIND_LIVE_TOKEN_FILE || "";
const outputDir = path.resolve(process.env.IMAGEFIND_LIVE_OUTPUT || "dist/postdeploy-0.5.45-bulk-retry");
if (!baseUrl || !tokenFile) throw new Error("IMAGEFIND_LIVE_URL and IMAGEFIND_LIVE_TOKEN_FILE are required");
const token = fs.readFileSync(tokenFile, "utf8").trim();
fs.mkdirSync(outputDir, { recursive: true });
async function openTasks(browser, viewport) {
const context = await browser.newContext({ viewport, locale: "zh-CN" });
const origin = new URL(baseUrl).origin;
await context.route("**/*", async route => {
const url = new URL(route.request().url());
if (url.origin === origin && url.pathname.includes("/api/")) {
await route.continue({
headers: { ...route.request().headers(), authorization: `Bearer ${token}` },
});
return;
}
await route.continue();
});
const page = await context.newPage();
await page.goto(baseUrl, { waitUntil: "domcontentloaded", timeout: 30_000 });
await page.locator(".app-shell").waitFor({ timeout: 30_000 });
if (viewport.width <= 720) {
await page.getByRole("button", { name: "更多" }).click();
await page.locator(".mobile-more-menu").getByRole("button", { name: "设置" }).click();
} else {
await page.locator(".nav-item-8").click();
}
await page.getByRole("heading", { name: "设置" }).waitFor({ timeout: 20_000 });
await page.getByRole("button", { name: "任务与偏好" }).click();
await page.locator(".jobs-panel").waitFor({ timeout: 20_000 });
await page
.getByRole("navigation", { name: "后台任务通道" })
.getByRole("button", { name: "上传转存" })
.click();
await page.locator(".job-retry-all").waitFor({ timeout: 20_000 });
return { context, page };
}
const browser = await chromium.launch({ headless: true });
const report = {};
try {
{
const { context, page } = await openTasks(browser, { width: 390, height: 844 });
const button = page.locator(".job-retry-all");
const box = await button.boundingBox();
report.mobile = {
button: (await button.innerText()).replace(/\s+/g, " ").trim(),
min_touch_target: Boolean(box && box.height >= 44),
overflow: await page.evaluate(() => document.documentElement.scrollWidth - innerWidth),
};
await page.screenshot({ path: path.join(outputDir, "mobile-transfer-retry.png"), fullPage: true });
await context.close();
}
{
const { context, page } = await openTasks(browser, { width: 1440, height: 900 });
const button = page.locator(".job-retry-all");
report.desktop = { button: (await button.innerText()).replace(/\s+/g, " ").trim() };
await button.click();
const dialog = page.getByRole("alertdialog");
report.desktop.confirmation = (await dialog.innerText()).replace(/\s+/g, " ").trim();
await page.screenshot({ path: path.join(outputDir, "desktop-confirmation.png"), fullPage: true });
const retryResponse = page.waitForResponse(
response => response.url().includes("/api/v1/jobs/retry-failed") && response.request().method() === "POST",
);
await dialog.getByRole("button", { name: "继续" }).click();
const response = await retryResponse;
if (!response.ok()) throw new Error(`bulk retry failed: ${response.status()} ${await response.text()}`);
report.desktop.result = await response.json();
await page.waitForTimeout(1_000);
report.desktop.button_visible_after_retry = (await page.locator(".job-retry-all").count()) > 0;
await page.screenshot({ path: path.join(outputDir, "desktop-result.png"), fullPage: true });
await context.close();
}
} finally {
await browser.close();
}
fs.writeFileSync(path.join(outputDir, "report.json"), `${JSON.stringify(report, null, 2)}\n`);
console.log(JSON.stringify(report, null, 2));
+116
View File
@@ -0,0 +1,116 @@
#!/usr/bin/env python3
"""Remove narrowly prefixed records created by live acceptance runs."""
from __future__ import annotations
import argparse
import json
from pathlib import Path
from typing import Any
import httpx
def call(client: httpx.Client, method: str, path: str, **kwargs: Any) -> httpx.Response:
response = client.request(method, path, **kwargs)
if response.is_error:
raise RuntimeError(f"{method} {path} -> {response.status_code}: {response.text[:1000]}")
return response
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--base-url", required=True)
auth = parser.add_mutually_exclusive_group(required=True)
auth.add_argument("--token-file", type=Path)
auth.add_argument("--password")
parser.add_argument("--prefix", required=True)
parser.add_argument("--keep", action="append", default=[])
parser.add_argument("--report", type=Path)
args = parser.parse_args()
if len(args.prefix) < 12 or not args.prefix.startswith("imagefind-e2e-"):
raise ValueError("cleanup prefix must be a narrowly scoped imagefind-e2e-* value")
headers = {}
if args.token_file is not None:
headers["Authorization"] = f"Bearer {args.token_file.read_text().strip()}"
client = httpx.Client(
base_url=args.base_url.rstrip("/"),
headers=headers,
timeout=httpx.Timeout(120, connect=10),
)
if args.password is not None:
login = call(
client,
"POST",
"/api/v1/auth/login",
json={"password": args.password, "remember_device": False},
).json()
csrf = str(login.get("csrf_token") or "")
if not csrf:
raise AssertionError("login response omitted CSRF token")
client.headers["X-CSRF-Token"] = csrf
keep = set(args.keep)
report: dict[str, Any] = {
"prefix": args.prefix,
"videos": [],
"collections": [],
"tags": [],
"tag_groups": [],
"people_hidden": [],
}
rows = call(client, "GET", "/api/v1/videos?limit=500").json()
for video in rows:
filename = str(video.get("source_key") or "").rsplit("/", 1)[-1]
if args.prefix not in str(video.get("source_key") or "") or filename in keep:
continue
deleted = call(client, "DELETE", f"/api/v1/videos/{video['id']}?delete_source=true").json()
if trash_id := deleted.get("trash_id"):
call(client, "DELETE", f"/api/v1/trash/{trash_id}")
report["videos"].append({"id": video["id"], "filename": filename, **deleted})
for collection in call(client, "GET", "/api/v1/collections").json():
if not str(collection.get("name") or "").startswith(args.prefix):
continue
detail = call(client, "GET", f"/api/v1/collections/{collection['id']}").json()
if detail.get("video_count"):
continue
result = call(client, "DELETE", f"/api/v1/collections/{collection['id']}").json()
report["collections"].append(result)
for group in call(client, "GET", "/api/v1/tag-groups").json():
if not str(group.get("name") or "").startswith(args.prefix):
continue
tags = call(client, "GET", f"/api/v1/tags?group_id={group['id']}").json()
if any(int(tag.get("video_count") or 0) for tag in tags):
continue
for tag in tags:
call(client, "DELETE", f"/api/v1/tags/{tag['id']}")
report["tags"].append(tag["id"])
call(client, "DELETE", f"/api/v1/tag-groups/{group['id']}")
report["tag_groups"].append(group["id"])
for source in call(client, "GET", "/api/v1/sources").json():
for item in call(client, "GET", f"/api/v1/trash?source_id={source['id']}").json():
if args.prefix in str(item.get("display_name") or ""):
call(client, "DELETE", f"/api/v1/trash/{item['id']}")
# Named people are deliberately preserved when their last face is removed.
# There is no destructive people endpoint, so hide narrowly prefixed test
# identities after their fixture videos have been deleted.
for person in call(client, "GET", "/api/v1/people").json():
name = str(person.get("name") or "")
if not name.startswith(args.prefix) or person.get("hidden"):
continue
call(client, "PATCH", f"/api/v1/people/{person['id']}", json={"name": name, "hidden": True})
report["people_hidden"].append(person["id"])
encoded = json.dumps(report, ensure_ascii=False, indent=2) + "\n"
if args.report:
args.report.parent.mkdir(parents=True, exist_ok=True)
args.report.write_text(encoded)
print(encoded, end="")
if __name__ == "__main__":
main()
+286
View File
@@ -0,0 +1,286 @@
#!/usr/bin/env python3
"""Exercise synchronous video deletion and the live recycle-bin workflow."""
from __future__ import annotations
import argparse
import hashlib
import json
import time
from pathlib import Path
from typing import Any
import httpx
def call(client: httpx.Client, method: str, path: str, **kwargs: Any) -> httpx.Response:
response = client.request(method, path, **kwargs)
if response.is_error:
raise RuntimeError(f"{method} {path} -> {response.status_code}: {response.text[:1000]}")
return response
def digest(path: Path) -> str:
value = hashlib.sha256()
with path.open("rb") as handle:
while block := handle.read(1024 * 1024):
value.update(block)
return value.hexdigest()
def videos(client: httpx.Client) -> list[dict[str, Any]]:
return call(client, "GET", "/api/v1/videos?limit=500").json()
def find_video(client: httpx.Client, source_id: str, filename: str) -> dict[str, Any] | None:
return next(
(
item
for item in videos(client)
if item.get("source_id") == source_id
and item.get("source_key", "").rsplit("/", 1)[-1] == filename
),
None,
)
def upload(
client: httpx.Client,
fixture: Path,
source_id: str,
filename: str,
title: str,
) -> dict[str, Any]:
visible = find_video(client, source_id, filename)
if visible:
return visible
existing = call(client, "GET", "/api/v1/uploads?limit=500").json()
task = next(
(
item
for item in existing
if item.get("filename") == filename
and item.get("status") in {"receiving", "queued", "transferring", "indexing"}
),
None,
)
if task is None:
task = call(
client,
"POST",
"/api/v1/uploads",
json={
"source_id": source_id,
"relative_path": "ingest",
"filename": filename,
"title": title,
"size_bytes": fixture.stat().st_size,
"sha256": digest(fixture),
"conflict": "skip",
},
).json()
upload_id = task["id"]
if task.get("status") != "completed":
chunk_size = int(task["chunk_size"])
received = {int(value) for value in task.get("received_chunks", task.get("received", []))}
with fixture.open("rb") as handle:
for index in range(int(task["total_chunks"])):
block = handle.read(chunk_size)
if index in received:
continue
call(
client,
"PUT",
f"/api/v1/uploads/{upload_id}/chunks/{index}",
content=block,
headers={"X-Chunk-SHA256": hashlib.sha256(block).hexdigest()},
)
call(client, "POST", f"/api/v1/uploads/{upload_id}/complete")
deadline = time.monotonic() + 180
while time.monotonic() < deadline:
tasks = call(client, "GET", "/api/v1/uploads?limit=500").json()
task = next(item for item in tasks if item["id"] == upload_id)
if task["status"] == "completed":
break
if task["status"] in {"failed", "cancelled"}:
raise AssertionError(f"upload failed: {task.get('message') or task.get('error')}")
time.sleep(1)
else:
raise TimeoutError(f"upload did not complete: {filename}")
deadline = time.monotonic() + 60
while time.monotonic() < deadline:
video = find_video(client, source_id, filename)
if video:
return video
time.sleep(1)
raise TimeoutError(f"completed upload did not appear in /videos: {filename}")
def purge_source_key(client: httpx.Client, source_id: str, key: str) -> None:
try:
result = call(
client,
"POST",
"/api/v1/files/trash",
json={"source_id": source_id, "keys": [key]},
).json()
except RuntimeError as exc:
if "404" in str(exc):
return
raise
for trash_id in result.get("ids", []):
call(client, "DELETE", f"/api/v1/trash/{trash_id}")
def search_ids(client: httpx.Client, text: str) -> list[str]:
result = call(
client,
"POST",
"/api/v1/search",
json={"text": text, "recognition_types": ["metadata"]},
).json()
return [item["video_id"] for item in result.get("items", [])]
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--base-url", required=True)
auth = parser.add_mutually_exclusive_group(required=True)
auth.add_argument("--token-file", type=Path)
auth.add_argument("--password")
parser.add_argument("--run-dir", type=Path, required=True)
args = parser.parse_args()
state = json.loads((args.run_dir / "state.json").read_text())
headers = {}
if args.token_file is not None:
headers["Authorization"] = f"Bearer {args.token_file.read_text().strip()}"
client = httpx.Client(
base_url=args.base_url.rstrip("/"),
headers=headers,
timeout=httpx.Timeout(60, connect=10),
)
if args.password is not None:
login = call(
client,
"POST",
"/api/v1/auth/login",
json={"password": args.password, "remember_device": False},
).json()
csrf = str(login.get("csrf_token") or "")
if not csrf:
raise AssertionError("login response omitted CSRF token")
client.headers["X-CSRF-Token"] = csrf
source_id = state["source_id"]
run_id = state["run_id"]
cases = [
{
"mode": "trash",
"fixture": args.run_dir / "e2e-positive.mp4",
"filename": f"{run_id}-delete-trash.mp4",
"title": "删除回收验收 481",
"key": f"ingest/{run_id}-delete-trash.mp4",
},
{
"mode": "tombstone",
"fixture": args.run_dir / "e2e-negative.mp4",
"filename": f"{run_id}-delete-only.mp4",
"title": "仅移出验收 592",
"key": f"ingest/{run_id}-delete-only.mp4",
},
]
report: dict[str, Any] = {}
try:
trash_case = cases[0]
first = upload(
client,
trash_case["fixture"],
source_id,
trash_case["filename"],
trash_case["title"],
)
deleted = call(client, "DELETE", f"/api/v1/videos/{first['id']}?delete_source=true").json()
if not deleted.get("source_deleted") or not deleted.get("trash_id"):
raise AssertionError(f"source deletion result mismatch: {deleted}")
if find_video(client, source_id, trash_case["filename"]):
raise AssertionError("source-deleted video remained visible")
if first["id"] in search_ids(client, "删除回收验收 481"):
raise AssertionError("source-deleted video remained in metadata search")
trash_rows = call(client, "GET", f"/api/v1/trash?source_id={source_id}").json()
trash_item = next((item for item in trash_rows if item["id"] == deleted["trash_id"]), None)
if not trash_item:
raise AssertionError("deleted source was not listed in recycle bin")
restored = call(client, "POST", f"/api/v1/trash/{trash_item['id']}/restore", json={}).json()
deadline = time.monotonic() + 90
restored_video = None
while time.monotonic() < deadline:
restored_video = find_video(client, source_id, trash_case["filename"])
if restored_video:
break
time.sleep(1)
if not restored_video:
raise AssertionError("restored source did not return to /videos")
removed_again = call(
client,
"DELETE",
f"/api/v1/videos/{restored_video['id']}?delete_source=true",
).json()
call(client, "DELETE", f"/api/v1/trash/{removed_again['trash_id']}")
report["trash"] = {
"deleted": deleted,
"restored_key": restored["key"],
"restored_video_id": restored_video["id"],
"purged": True,
}
tombstone_case = cases[1]
second = upload(
client,
tombstone_case["fixture"],
source_id,
tombstone_case["filename"],
tombstone_case["title"],
)
removed = call(client, "DELETE", f"/api/v1/videos/{second['id']}?delete_source=false").json()
if removed.get("source_deleted") or removed.get("trash_id"):
raise AssertionError(f"library-only deletion result mismatch: {removed}")
scan = call(client, "POST", f"/api/v1/sources/{source_id}/scan", json={}).json()
time.sleep(5)
if find_video(client, source_id, tombstone_case["filename"]):
raise AssertionError("tombstoned video reappeared after source scan")
if second["id"] in search_ids(client, "仅移出验收 592"):
raise AssertionError("tombstoned video remained in metadata search")
purge_source_key(client, source_id, tombstone_case["key"])
report["tombstone"] = {
"deleted": removed,
"scan_job_id": scan.get("job_id"),
"remained_hidden_after_scan": True,
"source_cleaned": True,
}
finally:
for case in cases:
video = find_video(client, source_id, case["filename"])
if video:
try:
deletion = call(
client,
"DELETE",
f"/api/v1/videos/{video['id']}?delete_source=true",
).json()
if deletion.get("trash_id"):
call(client, "DELETE", f"/api/v1/trash/{deletion['trash_id']}")
except Exception:
pass
else:
try:
purge_source_key(client, source_id, case["key"])
except Exception:
pass
(args.run_dir / "delete-report.json").write_text(
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
)
print(json.dumps(report, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()
+119
View File
@@ -0,0 +1,119 @@
#!/usr/bin/env python3
"""Verify visual/OCR/face GPU indexing and named-person search on live data."""
from __future__ import annotations
import argparse
import json
import time
from pathlib import Path
from typing import Any
import httpx
def call(client: httpx.Client, method: str, path: str, **kwargs: Any) -> httpx.Response:
response = client.request(method, path, **kwargs)
if response.is_error:
raise RuntimeError(f"{method} {path} -> {response.status_code}: {response.text[:1000]}")
return response
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--base-url", required=True)
parser.add_argument("--token-file", type=Path, required=True)
parser.add_argument("--run-dir", type=Path, required=True)
parser.add_argument("--case-id", default="face")
parser.add_argument("--timeout", type=int, default=600)
args = parser.parse_args()
state = json.loads((args.run_dir / "state.json").read_text())
token = args.token_file.read_text().strip()
client = httpx.Client(
base_url=args.base_url.rstrip("/"),
headers={"Authorization": f"Bearer {token}"},
timeout=httpx.Timeout(60, connect=10),
)
filename = f"{state['run_id']}-{args.case_id}-e2e-face.mp4"
observed: dict[str, list[str]] = {name: [] for name in ("visual", "ocr", "faces")}
deadline = time.monotonic() + args.timeout
video: dict[str, Any] | None = None
while time.monotonic() < deadline:
models = call(client, "GET", "/api/v1/models").json()
components = models.get("accelerator", {}).get("components", {})
for component in observed:
device = components.get(component, {}).get("actual_device")
if device and device not in observed[component]:
observed[component].append(device)
videos = call(client, "GET", "/api/v1/videos?limit=500").json()
video = next(
(
item
for item in videos
if item.get("source_id") == state["source_id"]
and str(item.get("source_key") or "").rsplit("/", 1)[-1] == filename
),
None,
)
if video and video.get("index_state", {}).get("faces") in {"ready", "failed"}:
break
time.sleep(1)
if not video:
raise AssertionError("face fixture never appeared in the video list")
if video.get("index_state", {}).get("faces") != "ready":
raise AssertionError(f"face indexing failed: {video.get('index_state')} {video.get('error')}")
matched_person: dict[str, Any] | None = None
matched_faces: list[dict[str, Any]] = []
for person in call(client, "GET", "/api/v1/people").json():
faces = call(client, "GET", f"/api/v1/people/{person['id']}/faces?limit=500").json()
owned = [face for face in faces if face.get("video_id") == video["id"]]
if owned:
matched_person = person
matched_faces = owned
break
if not matched_person:
raise AssertionError("the official OpenCV face fixture did not produce a person cluster")
person_name = f"{state['run_id']}-人物-729"
call(client, "PATCH", f"/api/v1/people/{matched_person['id']}", json={"name": person_name})
positive = call(
client,
"POST",
"/api/v1/search",
json={"text": person_name, "recognition_types": ["person"], "limit": 100},
).json()
if video["id"] not in {item["video_id"] for item in positive.get("items", [])}:
raise AssertionError("named-person search omitted the face fixture video")
negative = call(
client,
"POST",
"/api/v1/search",
json={
"text": f"{state['run_id']}-不存在人物-9834721",
"recognition_types": ["person"],
"limit": 100,
},
).json()
if video["id"] in {item["video_id"] for item in negative.get("items", [])}:
raise AssertionError("non-matching person search returned the face fixture video")
non_gpu = {name: devices for name, devices in observed.items() if not any("GPU" in d.upper() for d in devices)}
if non_gpu:
raise AssertionError(f"GPU was not observed for all frame components: {non_gpu}")
report = {
"video_id": video["id"],
"person_id": matched_person["id"],
"person_name": person_name,
"face_count": len(matched_faces),
"observed_devices": observed,
"positive_matches": len(positive.get("items", [])),
"negative_matches": len(negative.get("items", [])),
}
(args.run_dir / "face-report.json").write_text(
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
)
print(json.dumps(report, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()
+294
View File
@@ -0,0 +1,294 @@
#!/usr/bin/env python3
"""Cover final live gaps without touching records outside the supplied run prefix."""
from __future__ import annotations
import argparse
import hashlib
import json
import math
import time
from concurrent.futures import ThreadPoolExecutor, as_completed
from pathlib import Path
from typing import Any
import httpx
def call(client: httpx.Client, method: str, path: str, **kwargs: Any) -> httpx.Response:
response = client.request(method, path, **kwargs)
if response.is_error:
raise RuntimeError(f"{method} {path} -> {response.status_code}: {response.text[:1000]}")
return response
def percentile(values: list[float], ratio: float) -> float:
ordered = sorted(values)
return ordered[max(0, math.ceil(len(ordered) * ratio) - 1)] if ordered else 0.0
def find_video(client: httpx.Client, source_id: str, filename: str) -> dict[str, Any] | None:
return next(
(
video
for video in call(client, "GET", "/api/v1/videos?limit=500").json()
if video.get("source_id") == source_id
and str(video.get("source_key") or "").rsplit("/", 1)[-1] == filename
),
None,
)
def wait_upload(client: httpx.Client, upload_id: str, timeout: int = 180) -> dict[str, Any]:
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
upload = next(
(item for item in call(client, "GET", "/api/v1/uploads?limit=500").json() if item["id"] == upload_id),
None,
)
if upload and upload["status"] in {"completed", "failed", "cancelled"}:
return upload
time.sleep(1)
raise TimeoutError(f"upload did not reach a terminal state: {upload_id}")
def wait_job(client: httpx.Client, job_id: str, timeout: int = 180) -> dict[str, Any]:
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
job = next(
(item for item in call(client, "GET", "/api/v1/jobs?limit=500").json() if item["id"] == job_id),
None,
)
if job and job["status"] in {"completed", "failed", "cancelled"}:
return job
time.sleep(1)
raise TimeoutError(f"job did not reach a terminal state: {job_id}")
def performance_sample(base_url: str, client: httpx.Client) -> dict[str, Any]:
paths = (
"/api/v1/status",
"/api/v1/videos?limit=50",
"/api/v1/jobs?page=1&page_size=10",
"/api/v1/profile",
"/api/v1/models",
"/api/v1/system/resources",
)
def request(path: str) -> tuple[str, float, int]:
started = time.perf_counter()
response = httpx.get(
base_url + path,
headers=dict(client.headers),
cookies=dict(client.cookies),
timeout=httpx.Timeout(20, connect=5),
)
return path, round((time.perf_counter() - started) * 1000, 1), response.status_code
samples: dict[str, list[float]] = {path: [] for path in paths}
errors: list[dict[str, Any]] = []
with ThreadPoolExecutor(max_workers=4) as pool:
futures = [pool.submit(request, path) for _ in range(4) for path in paths]
for future in as_completed(futures):
path, elapsed, status = future.result()
samples[path].append(elapsed)
if status != 200:
errors.append({"path": path, "status": status, "elapsed_ms": elapsed})
result = {
path: {
"count": len(values),
"p50_ms": percentile(values, 0.5),
"p95_ms": percentile(values, 0.95),
"max_ms": max(values, default=0),
}
for path, values in samples.items()
}
all_values = [value for values in samples.values() for value in values]
result["summary"] = {
"requests": len(all_values),
"concurrency": 4,
"errors": errors,
"p50_ms": percentile(all_values, 0.5),
"p95_ms": percentile(all_values, 0.95),
"max_ms": max(all_values, default=0),
}
if errors or result["summary"]["p95_ms"] > 5000 or result["summary"]["max_ms"] > 10000:
raise AssertionError(f"live performance sample exceeded its guardrail: {result['summary']}")
return result
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--base-url", required=True)
auth = parser.add_mutually_exclusive_group(required=True)
auth.add_argument("--token-file", type=Path)
auth.add_argument("--password")
parser.add_argument("--run-dir", type=Path, required=True)
args = parser.parse_args()
state = json.loads((args.run_dir / "state.json").read_text())
run_id = state["run_id"]
source_id = state["source_id"]
base_url = args.base_url.rstrip("/")
headers = {}
if args.token_file is not None:
headers["Authorization"] = f"Bearer {args.token_file.read_text().strip()}"
client = httpx.Client(
base_url=base_url,
headers=headers,
timeout=httpx.Timeout(60, connect=10),
)
if args.password is not None:
login = call(
client,
"POST",
"/api/v1/auth/login",
json={"password": args.password, "remember_device": False},
).json()
csrf = str(login.get("csrf_token") or "")
if not csrf:
raise AssertionError("login response omitted CSRF token")
client.headers["X-CSRF-Token"] = csrf
status = call(client, "GET", "/api/v1/status").json()
if status.get("version") != "0.5.45" or status.get("access_mode") != "direct":
raise AssertionError(f"unexpected live status: {status}")
unauthenticated = httpx.get(base_url + "/api/v1/profile", timeout=10)
invalid_token = httpx.get(
base_url + "/api/v1/profile",
headers={"Authorization": "Bearer invalid-final-acceptance-token"},
timeout=10,
)
if (unauthenticated.status_code, invalid_token.status_code) != (401, 401):
raise AssertionError("profile endpoint did not reject missing and invalid credentials")
page_one = call(client, "GET", "/api/v1/jobs?page=1&page_size=10").json()
page_two = call(client, "GET", "/api/v1/jobs?page=2&page_size=10").json()
first_ids = {item["id"] for item in page_one["items"]}
second_ids = {item["id"] for item in page_two["items"]}
if first_ids & second_ids or page_one["page"] != 1 or page_two["page"] != 2:
raise AssertionError("background job pagination returned overlapping or incorrect pages")
lane_totals = {
lane: call(client, "GET", f"/api/v1/jobs?page=1&page_size=10&lane={lane}").json()["total"]
for lane in ("ai", "transfer", "download", "scan")
}
if sum(lane_totals.values()) != page_one["total"]:
raise AssertionError(f"job lane totals do not match all jobs: {lane_totals} != {page_one['total']}")
cancel_payload = b"cancelled upload acceptance payload"
cancel_name = f"{run_id}-cancel-before-complete.mp4"
cancel_upload = call(
client,
"POST",
"/api/v1/uploads",
json={
"source_id": source_id,
"relative_path": "ingest",
"filename": cancel_name,
"size_bytes": len(cancel_payload),
"sha256": hashlib.sha256(cancel_payload).hexdigest(),
"conflict": "skip",
},
).json()
cancelled = client.delete(f"/api/v1/uploads/{cancel_upload['id']}")
if cancelled.status_code != 204 or find_video(client, source_id, cancel_name):
raise AssertionError("cancelled receiving upload leaked into the video catalog")
invalid_payload = b"ImageFind final acceptance: intentionally invalid mp4\n"
invalid_name = f"{run_id}-manual-retry-invalid.mp4"
before_job_ids = {item["id"] for item in call(client, "GET", "/api/v1/jobs?limit=500").json()}
invalid_upload = call(
client,
"POST",
"/api/v1/uploads",
json={
"source_id": source_id,
"relative_path": "ingest",
"filename": invalid_name,
"title": "后台任务手动重试验收",
"size_bytes": len(invalid_payload),
"sha256": hashlib.sha256(invalid_payload).hexdigest(),
"conflict": "skip",
},
).json()
call(
client,
"PUT",
f"/api/v1/uploads/{invalid_upload['id']}/chunks/0",
content=invalid_payload,
headers={"X-Chunk-SHA256": hashlib.sha256(invalid_payload).hexdigest()},
)
call(client, "POST", f"/api/v1/uploads/{invalid_upload['id']}/complete")
upload_result = wait_upload(client, invalid_upload["id"])
if upload_result["status"] != "completed":
raise AssertionError(f"invalid fixture did not reach catalog processing: {upload_result}")
deadline = time.monotonic() + 180
failed_job: dict[str, Any] | None = None
invalid_video: dict[str, Any] | None = None
while time.monotonic() < deadline:
invalid_video = find_video(client, source_id, invalid_name)
jobs = call(client, "GET", "/api/v1/jobs?limit=500").json()
failed_job = next(
(
job
for job in jobs
if job["id"] not in before_job_ids
and job["kind"] == "index_video"
and job["status"] == "failed"
),
None,
)
if invalid_video and failed_job:
break
time.sleep(1)
if not invalid_video or not failed_job:
raise AssertionError("invalid video did not produce a run-owned failed index task")
retry = call(client, "POST", f"/api/v1/jobs/{failed_job['id']}/retry", json={}).json()
if retry.get("retried_from") != failed_job["id"] or retry.get("job_id") == failed_job["id"]:
raise AssertionError(f"manual job retry response mismatch: {retry}")
retried_job = wait_job(client, retry["job_id"])
if retried_job["status"] != "failed" or int(retried_job.get("attempts") or 0) < 1:
raise AssertionError(f"retried invalid index job did not execute: {retried_job}")
deleted = call(client, "DELETE", f"/api/v1/videos/{invalid_video['id']}?delete_source=true").json()
if deleted.get("trash_id"):
call(client, "DELETE", f"/api/v1/trash/{deleted['trash_id']}")
resources = call(client, "GET", "/api/v1/system/resources").json()
if resources.get("database", {}).get("engine") != "postgresql":
raise AssertionError(f"unexpected database resource status: {resources.get('database')}")
models = call(client, "GET", "/api/v1/models").json()
if not all(models.get("operational_components", {}).get(name) for name in ("visual", "ocr", "faces", "audio")):
raise AssertionError(f"one or more AI components are not operational: {models.get('operational_components')}")
downloads = call(client, "GET", "/api/v1/downloads/runtime").json()
performance = performance_sample(base_url, client)
report = {
"status": status,
"authentication": {"missing": 401, "invalid": 401},
"jobs": {
"total": page_one["total"],
"pages": page_one["pages"],
"page_size": page_one["page_size"],
"pages_disjoint": True,
"lane_totals": lane_totals,
"failed_job": failed_job,
"retry": retry,
"retried_job": retried_job,
},
"cancelled_upload": {"id": cancel_upload["id"], "hidden_from_videos": True},
"database": resources.get("database"),
"resource_lanes": resources.get("lanes"),
"aria2": downloads,
"models": models.get("operational_components"),
"performance": performance,
}
(args.run_dir / "final-gaps-report.json").write_text(
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
)
print(json.dumps(report, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()
+162
View File
@@ -0,0 +1,162 @@
#!/usr/bin/env python3
"""Read-only live acceptance for upload pagination and core API latency."""
from __future__ import annotations
import argparse
import json
import statistics
import sys
import time
from pathlib import Path
from typing import Any
import httpx
def call(client: httpx.Client, path: str) -> tuple[Any, float]:
started = time.perf_counter()
response = client.get(path)
elapsed_ms = (time.perf_counter() - started) * 1000
if response.is_error:
raise RuntimeError(f"GET {path} -> {response.status_code}: {response.text[:500]}")
return response.json(), elapsed_ms
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--base-url", required=True)
auth = parser.add_mutually_exclusive_group(required=True)
auth.add_argument("--token-file", type=Path)
auth.add_argument("--password-stdin", action="store_true")
parser.add_argument("--report", type=Path)
args = parser.parse_args()
headers: dict[str, str] = {}
if args.token_file:
headers["Authorization"] = f"Bearer {args.token_file.read_text(encoding='utf-8').strip()}"
client = httpx.Client(
base_url=args.base_url.rstrip("/"),
headers=headers,
timeout=httpx.Timeout(30, connect=10),
)
if args.password_stdin:
password = sys.stdin.readline().rstrip("\r\n")
login = client.post(
"/api/v1/auth/login",
json={"password": password, "remember_device": False},
)
password = ""
if login.is_error:
raise RuntimeError(f"login -> {login.status_code}: {login.text[:500]}")
status, status_ms = call(client, "/api/v1/status")
if status.get("version") != "0.5.45" or not status.get("configured"):
raise AssertionError(f"unexpected status: {status}")
legacy, legacy_ms = call(client, "/api/v1/uploads?limit=4")
if not isinstance(legacy, list) or len(legacy) > 4:
raise AssertionError("legacy upload response is no longer a bounded array")
first, first_ms = call(client, "/api/v1/uploads?page=1&page_size=10")
required = {
"items",
"status_items",
"active_count",
"failed_count",
"page",
"page_size",
"total",
"pages",
}
if not isinstance(first, dict) or not required.issubset(first):
actual = sorted(first) if isinstance(first, dict) else type(first)
raise AssertionError(f"upload page contract mismatch: {actual}")
if first["page"] != 1 or first["page_size"] != 10 or len(first["items"]) > 10:
raise AssertionError("upload first page bounds are invalid")
if first["pages"] < 1 or first["total"] < len(first["items"]):
raise AssertionError("upload pagination totals are invalid")
order = [(str(item.get("created_at") or ""), str(item["id"])) for item in first["items"]]
if order != sorted(order, reverse=True):
raise AssertionError("upload page ordering is unstable")
allowed_statuses = {"receiving", "queued", "transferring", "indexing", "failed"}
if any(item.get("status") not in allowed_statuses for item in first["status_items"]):
raise AssertionError("upload status summary contains terminal history")
active_visible = sum(item.get("status") != "failed" for item in first["status_items"])
failed_visible = sum(item.get("status") == "failed" for item in first["status_items"])
if first["active_count"] < active_visible or first["failed_count"] < failed_visible:
raise AssertionError("upload status counters are smaller than their visible summaries")
second_ids: list[str] = []
second_ms: float | None = None
if first["pages"] > 1:
second, second_ms = call(client, "/api/v1/uploads?page=2&page_size=10")
second_ids = [str(item["id"]) for item in second["items"]]
if set(second_ids) & {str(item["id"]) for item in first["items"]}:
raise AssertionError("upload pages overlap")
latency_paths = {
"uploads": "/api/v1/uploads?page=1&page_size=10",
"videos": "/api/v1/videos?limit=20",
"profile": "/api/v1/profile",
"resources": "/api/v1/system/resources",
}
latency: dict[str, dict[str, float]] = {}
resource_state: dict[str, Any] = {}
for name, path in latency_paths.items():
samples: list[float] = []
value: Any = None
for _ in range(3):
value, elapsed_ms = call(client, path)
samples.append(elapsed_ms)
latency[name] = {
"median_ms": round(statistics.median(samples), 2),
"max_ms": round(max(samples), 2),
}
if name == "resources" and isinstance(value, dict):
database = value.get("database") or {}
resource_state = {
"cpu_percent": value.get("cpu_percent"),
"memory_available_bytes": value.get("memory_available_bytes"),
"running_jobs": value.get("running_jobs"),
"paused_jobs": value.get("paused_jobs"),
"database": {
"engine": database.get("engine"),
"pool_in_use": database.get("pool_in_use"),
"pool_waiters": database.get("pool_waiters"),
"slow_transaction_count": database.get("slow_transaction_count"),
},
}
report = {
"status": status,
"contract": {
"legacy_count": len(legacy),
"page": first["page"],
"page_size": first["page_size"],
"page_items": len(first["items"]),
"total": first["total"],
"pages": first["pages"],
"second_page_items": len(second_ids),
"status_items": len(first["status_items"]),
"active_count": first["active_count"],
"failed_count": first["failed_count"],
},
"initial_latency_ms": {
"status": round(status_ms, 2),
"legacy_uploads": round(legacy_ms, 2),
"first_page": round(first_ms, 2),
"second_page": round(second_ms, 2) if second_ms is not None else None,
},
"latency": latency,
"resources": resource_state,
}
encoded = json.dumps(report, ensure_ascii=False, indent=2) + "\n"
if args.report:
args.report.parent.mkdir(parents=True, exist_ok=True)
args.report.write_text(encoded, encoding="utf-8")
print(encoded, end="")
if __name__ == "__main__":
main()
+219
View File
@@ -0,0 +1,219 @@
#!/usr/bin/env node
import fs from "node:fs";
import path from "node:path";
import { createRequire } from "node:module";
const require = createRequire(new URL("../frontend/package.json", import.meta.url));
const { chromium } = require("playwright");
const baseUrl = (process.env.IMAGEFIND_LIVE_URL || "").replace(/\/$/, "");
const tokenFile = process.env.IMAGEFIND_LIVE_TOKEN_FILE || "";
const token = tokenFile ? fs.readFileSync(tokenFile, "utf8").trim() : "";
const password = process.env.IMAGEFIND_LIVE_PASSWORD || "";
const targetVideoText = process.env.IMAGEFIND_LIVE_VIDEO_TEXT || "星河验收影片 729";
const mutateMarkers = process.env.IMAGEFIND_LIVE_MUTATE_MARKERS !== "false";
const outputDir = path.resolve(process.env.IMAGEFIND_LIVE_OUTPUT || "dist/live-player-acceptance");
if (!baseUrl || (!token && !password)) {
throw new Error("IMAGEFIND_LIVE_URL and either IMAGEFIND_LIVE_TOKEN_FILE or IMAGEFIND_LIVE_PASSWORD are required");
}
fs.mkdirSync(outputDir, { recursive: true });
async function newPage(browser, viewport) {
const context = await browser.newContext({ viewport, locale: "zh-CN", acceptDownloads: true });
const origin = new URL(baseUrl).origin;
if (token) {
await context.route("**/*", async route => {
const url = new URL(route.request().url());
if (url.origin === origin && url.pathname.includes("/api/")) {
await route.continue({
headers: { ...route.request().headers(), authorization: `Bearer ${token}` },
});
return;
}
await route.continue();
});
}
const page = await context.newPage();
await page.goto(baseUrl, { waitUntil: "domcontentloaded", timeout: 30_000 });
if (password) {
const passwordInput = page.getByLabel("管理员密码");
if (await passwordInput.isVisible().catch(() => false)) {
await passwordInput.fill(password);
await page.getByRole("button", { name: "登录", exact: true }).click();
}
}
await page.getByRole("button", { name: "首页", exact: true }).waitFor({ timeout: 30_000 });
await page.locator("article.video-card").first().waitFor({ timeout: 30_000 });
return { context, page };
}
async function showControls(page) {
const stage = page.locator(".custom-player");
if ((await stage.getAttribute("class"))?.includes("controls-hidden")) {
await page.locator(".player-gesture-surface").click({ position: { x: 20, y: 20 } });
await page.waitForTimeout(350);
}
}
function attachNetworkAudit(page, errors) {
const issues = [];
page.on("response", response => {
if (response.status() < 400) return;
const request = response.request();
const url = new URL(response.url());
const issue = {
method: request.method(),
status: response.status(),
path: url.pathname,
resourceType: request.resourceType(),
};
issues.push(issue);
errors.push(`response: ${issue.method} ${issue.status} ${issue.path} [${issue.resourceType}]`);
});
page.on("requestfailed", request => {
const url = new URL(request.url());
const issue = {
method: request.method(),
status: null,
path: url.pathname,
resourceType: request.resourceType(),
failure: request.failure()?.errorText || "unknown network failure",
};
issues.push(issue);
errors.push(`requestfailed: ${issue.method} ${issue.path} [${issue.resourceType}] ${issue.failure}`);
});
return issues;
}
async function inspectPlayer(browser, viewport, name, mutateMarkers) {
const { context, page } = await newPage(browser, viewport);
const errors = [];
page.on("pageerror", error => errors.push(`pageerror: ${error.message}`));
page.on("console", message => {
if (message.type() === "error") errors.push(`console: ${message.text()}`);
});
const networkIssues = attachNetworkAudit(page, errors);
const positive = page.locator("article.video-card").filter({ hasText: targetVideoText });
if (!(await positive.count())) throw new Error(`${name}: target H.264 fixture was not found: ${targetVideoText}`);
await positive.first().click();
await page.locator(".player-page").waitFor({ timeout: 20_000 });
const video = page.locator(".custom-player video");
await page.waitForFunction(() => {
const element = document.querySelector(".custom-player video");
return element instanceof HTMLVideoElement && element.readyState >= 2 && element.duration > 0;
}, null, { timeout: 90_000 });
await showControls(page);
const media = async () => video.evaluate(element => ({
paused: element.paused,
currentTime: element.currentTime,
duration: element.duration,
playbackRate: element.playbackRate,
readyState: element.readyState,
}));
const footerPlay = page.locator('.player-control-row > button[aria-label="播放"],.player-control-row > button[aria-label="暂停"]').first();
if (!(await media()).paused) {
await footerPlay.click();
await page.waitForFunction(() => document.querySelector(".custom-player video")?.paused === true);
}
const beforePlay = await media();
await footerPlay.click();
await page.waitForFunction(() => document.querySelector(".custom-player video")?.paused === false);
await page.waitForTimeout(1_000);
const duringPlay = await media();
if (duringPlay.currentTime <= beforePlay.currentTime) throw new Error(`${name}: playback time did not advance`);
await footerPlay.click();
await page.waitForFunction(() => document.querySelector(".custom-player video")?.paused === true);
await showControls(page);
const seekStart = (await media()).currentTime;
await page.getByRole("button", { name: "快进 10 秒" }).click();
const seekForward = (await media()).currentTime;
if (seekForward <= seekStart) throw new Error(`${name}: forward button did not seek`);
await page.getByRole("button", { name: "快退 10 秒" }).click();
const seekBack = (await media()).currentTime;
if (seekBack >= seekForward) throw new Error(`${name}: back button did not seek`);
await page.locator(".player-rate > button").click();
await page.getByRole("menuitem", { name: "1.5×", exact: true }).click();
if (Math.abs((await media()).playbackRate - 1.5) > 0.01) throw new Error(`${name}: rate did not change to 1.5x`);
await page.getByRole("button", { name: "全屏" }).click();
await page.waitForFunction(() => Boolean(document.fullscreenElement), null, { timeout: 10_000 });
const fullscreen = await page.evaluate(() => document.fullscreenElement?.classList.contains("custom-player"));
await page.evaluate(() => document.exitFullscreen());
await page.waitForFunction(() => !document.fullscreenElement);
await showControls(page);
await page.getByRole("button", { name: "视频时间点" }).click();
const drawer = page.locator(".player-marker-drawer");
await drawer.waitFor({ state: "visible" });
const initialDots = await page.locator(".player-marker-dot").count();
let markerCreated = false;
let markerRenamed = false;
let markerDeleted = false;
if (mutateMarkers) {
await drawer.getByRole("button", { name: "标记当前位置" }).click();
await page.waitForFunction(count => document.querySelectorAll(".player-marker-dot").length > count, initialDots);
markerCreated = true;
const newest = drawer.locator(".player-marker-list article").last();
await newest.getByRole("button", { name: "编辑时间点" }).click();
const dialog = page.getByRole("alertdialog");
await dialog.getByLabel("请输入").fill(`自动验收时间点-${Date.now()}`);
await dialog.getByRole("button", { name: "确定" }).click();
await dialog.waitFor({ state: "hidden" });
markerRenamed = true;
await drawer.locator(".player-marker-list article").last().getByRole("button", { name: "删除时间点" }).click();
const confirm = page.getByRole("alertdialog");
await confirm.getByRole("button", { name: "继续" }).click();
await page.waitForFunction(count => document.querySelectorAll(".player-marker-dot").length === count, initialDots);
markerDeleted = true;
}
const downloadPromise = page.waitForEvent("download", { timeout: 30_000 });
await page.locator(".player-actions a.action").filter({ hasText: "下载" }).click();
const download = await downloadPromise;
const downloadName = download.suggestedFilename();
await download.cancel();
const auditedNetworkIssues = networkIssues.map(issue => ({
...issue,
expected: issue.path.endsWith("/download") && issue.failure === "net::ERR_ABORTED",
}));
const auditedErrors = [...new Set(errors)].filter(
value => !(value.includes("/download") && value.includes("net::ERR_ABORTED")),
);
await page.screenshot({ path: path.join(outputDir, `${name}.png`), fullPage: true });
const report = {
name,
viewport,
media: await media(),
playbackAdvancedSeconds: Number((duringPlay.currentTime - beforePlay.currentTime).toFixed(2)),
seek: { before: seekStart, forward: seekForward, back: seekBack },
fullscreen,
markerDrawer: await drawer.isVisible(),
markerCreated,
markerRenamed,
markerDeleted,
downloadName,
horizontalOverflow: await page.evaluate(() => document.documentElement.scrollWidth > document.documentElement.clientWidth),
networkIssues: auditedNetworkIssues,
errors: auditedErrors,
};
await context.close();
return report;
}
const browser = await chromium.launch({ headless: true });
try {
const results = [
await inspectPlayer(browser, { width: 1440, height: 900 }, "desktop", mutateMarkers),
await inspectPlayer(browser, { width: 390, height: 844 }, "mobile-390", false),
];
fs.writeFileSync(path.join(outputDir, "report.json"), `${JSON.stringify(results, null, 2)}\n`);
console.log(JSON.stringify(results, null, 2));
} finally {
await browser.close();
}
+155
View File
@@ -0,0 +1,155 @@
#!/usr/bin/env node
import fs from "node:fs";
import path from "node:path";
import { createRequire } from "node:module";
const require = createRequire(new URL("../frontend/package.json", import.meta.url));
const { chromium, request } = require("playwright");
const baseUrl = (process.env.IMAGEFIND_LIVE_URL || "").replace(/\/$/, "");
const tokenFile = process.env.IMAGEFIND_LIVE_TOKEN_FILE || "";
const token = tokenFile ? fs.readFileSync(tokenFile, "utf8").trim() : "";
const password = process.env.IMAGEFIND_LIVE_PASSWORD || "";
const runPrefix = process.env.IMAGEFIND_LIVE_RUN_PREFIX || "";
const outputDir = path.resolve(process.env.IMAGEFIND_LIVE_OUTPUT || "dist/live-profile-acceptance");
if (!baseUrl || (!token && !password)) {
throw new Error("IMAGEFIND_LIVE_URL and either IMAGEFIND_LIVE_TOKEN_FILE or IMAGEFIND_LIVE_PASSWORD are required");
}
fs.mkdirSync(outputDir, { recursive: true });
const authHeaders = token ? { authorization: `Bearer ${token}` } : {};
const api = await request.newContext({ baseURL: baseUrl, extraHTTPHeaders: authHeaders });
let csrfToken = "";
if (password) {
const login = await api.post("/api/v1/auth/login", {
data: { password, remember_device: false },
});
if (!login.ok()) throw new Error(`login failed: ${login.status()} ${await login.text()}`);
csrfToken = String((await login.json()).csrf_token || "");
if (!csrfToken) throw new Error("login response omitted CSRF token");
}
const mutationHeaders = csrfToken ? { "X-CSRF-Token": csrfToken } : {};
const response = await api.get("/api/v1/videos?limit=500");
if (!response.ok()) throw new Error(`unable to list videos: ${response.status()}`);
const videos = await response.json();
if (videos.length < 2) throw new Error("at least two disposable videos are required");
const selectedVideos = (runPrefix
? videos.filter(video => String(video.source_key || "").includes(runPrefix))
: videos
).slice(0, 2);
if (selectedVideos.length < 2) throw new Error("at least two matching disposable videos are required");
const snapshots = videos.map(video => ({
id: video.id,
liked: Boolean(video.liked),
favorited: Boolean(video.favorited),
progress_ms: video.progress_ms || 0,
completed: Boolean(video.completed),
}));
async function patchState(id, state) {
const result = await api.patch(`/api/v1/videos/${id}/state`, { data: state, headers: mutationHeaders });
if (!result.ok()) throw new Error(`state patch failed: ${result.status()} ${await result.text()}`);
}
async function seed(state) {
await Promise.all(selectedVideos.map(video => patchState(video.id, state)));
}
async function openPage(browser, viewport) {
const context = await browser.newContext({ viewport, locale: "zh-CN" });
const origin = new URL(baseUrl).origin;
if (token) {
await context.route("**/*", async route => {
const url = new URL(route.request().url());
if (url.origin === origin && url.pathname.includes("/api/")) {
await route.continue({ headers: { ...route.request().headers(), ...authHeaders } });
return;
}
await route.continue();
});
}
const page = await context.newPage();
await page.goto(baseUrl, { waitUntil: "domcontentloaded", timeout: 30_000 });
await page.waitForTimeout(1_000);
if (password) {
const passwordInput = page.getByLabel("管理员密码");
if (await passwordInput.isVisible().catch(() => false)) {
await passwordInput.fill(password);
await page.getByRole("button", { name: "登录", exact: true }).click();
}
}
await page.locator(".app-shell").waitFor({ timeout: 30_000 });
if (viewport.width <= 720) {
await page.locator(".mobile-profile-button").click();
} else {
await page.getByRole("button", { name: "账户菜单" }).click();
await page.getByRole("button", { name: "个人中心", exact: true }).click();
}
await page.locator(".profile-page").waitFor({ timeout: 20_000 });
return { context, page };
}
async function bulkAction(page, tab, count, actionLabel) {
await page.getByRole("button", { name: tab, exact: true }).click();
const section = page.locator(".profile-videos");
await section.locator("article.video-card").first().waitFor({ timeout: 20_000 });
const before = await section.locator("article.video-card").count();
await section.getByRole("button", { name: "管理", exact: true }).click();
const choices = section.locator('article.video-card[aria-label^="选择 "]');
if ((await choices.count()) < count) throw new Error(`${tab}: not enough selectable cards`);
for (let index = 0; index < count; index += 1) await choices.nth(index).click();
await section.getByRole("button", { name: actionLabel, exact: true }).click();
const dialog = page.getByRole("alertdialog");
await dialog.getByRole("button", { name: "继续", exact: true }).click();
await section.locator(".loading").waitFor({ state: "hidden", timeout: 20_000 }).catch(() => {});
await page.waitForFunction(
({ selector, expected }) => document.querySelectorAll(selector).length === expected,
{ selector: ".profile-videos article.video-card", expected: before - count },
);
return { before, after: await section.locator("article.video-card").count(), selected: count };
}
const browser = await chromium.launch({ headless: true });
const reports = [];
try {
await seed({ favorited: true });
{
const { context, page } = await openPage(browser, { width: 1440, height: 900 });
const errors = [];
page.on("pageerror", error => errors.push(error.message));
const result = await bulkAction(page, "收藏", 1, "取消收藏");
await page.screenshot({ path: path.join(outputDir, "desktop-favorite-single.png"), fullPage: true });
reports.push({ name: "desktop-favorite-single", ...result, errors });
await context.close();
}
await seed({ liked: true });
{
const { context, page } = await openPage(browser, { width: 390, height: 844 });
const errors = [];
page.on("pageerror", error => errors.push(error.message));
const likes = await bulkAction(page, "喜欢", 2, "取消喜欢");
await page.screenshot({ path: path.join(outputDir, "mobile-like-multi.png"), fullPage: true });
reports.push({ name: "mobile-like-multi", ...likes, errors });
await context.close();
}
await seed({ progress_ms: 3_000, completed: false });
{
const { context, page } = await openPage(browser, { width: 390, height: 844 });
const errors = [];
page.on("pageerror", error => errors.push(error.message));
const history = await bulkAction(page, "观看记录", 2, "清除记录");
await page.screenshot({ path: path.join(outputDir, "mobile-history-multi.png"), fullPage: true });
reports.push({ name: "mobile-history-multi", ...history, errors });
await context.close();
}
} finally {
await Promise.all(snapshots.map(snapshot => patchState(snapshot.id, snapshot)));
await browser.close();
await api.dispose();
}
fs.writeFileSync(path.join(outputDir, "report.json"), `${JSON.stringify(reports, null, 2)}\n`);
console.log(JSON.stringify(reports, null, 2));
+135
View File
@@ -0,0 +1,135 @@
#!/usr/bin/env python3
"""Observe queue drain, API responsiveness, and inference idle reaping live."""
from __future__ import annotations
import argparse
import json
import time
from pathlib import Path
from typing import Any
import httpx
def call(client: httpx.Client, path: str) -> tuple[dict[str, Any], float]:
started = time.perf_counter()
response = client.get(path)
elapsed_ms = round((time.perf_counter() - started) * 1000, 1)
if response.is_error:
raise RuntimeError(f"GET {path} -> {response.status_code}: {response.text[:1000]}")
return response.json(), elapsed_ms
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--base-url", required=True)
parser.add_argument("--password", required=True)
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--timeout", type=int, default=900)
parser.add_argument("--idle-seconds", type=int, default=135)
args = parser.parse_args()
client = httpx.Client(
base_url=args.base_url.rstrip("/"),
timeout=httpx.Timeout(30, connect=10),
)
login = client.post(
"/api/v1/auth/login",
json={"password": args.password, "remember_device": False},
)
login.raise_for_status()
samples: list[dict[str, Any]] = []
idle_started: float | None = None
deadline = time.monotonic() + args.timeout
last_report = 0.0
while time.monotonic() < deadline:
resources, resources_ms = call(client, "/api/v1/system/resources")
diagnostics, diagnostics_ms = call(client, "/api/v1/system/diagnostics")
status, status_ms = call(client, "/api/v1/status")
lanes = resources.get("lanes") or {}
active = sum(
int(lane.get("running") or 0) + int(lane.get("queued") or 0)
for lane in lanes.values()
)
now = time.monotonic()
if active:
idle_started = None
elif idle_started is None:
idle_started = now
idle_elapsed = 0 if idle_started is None else round(now - idle_started, 1)
sample = {
"at": time.time(),
"version": status.get("version"),
"active_jobs": active,
"lanes": lanes,
"cpu_percent": resources.get("cpu_percent"),
"memory_available_bytes": resources.get("memory_available_bytes"),
"process_rss_bytes": diagnostics.get("process_rss_bytes"),
"event_loop_lag_ms": diagnostics.get("event_loop_lag_ms"),
"event_loop_max_lag_ms": diagnostics.get("event_loop_max_lag_ms"),
"request_p95_ms": diagnostics.get("request_p95_ms"),
"inference": diagnostics.get("inference"),
"database": diagnostics.get("database"),
"latency_ms": {
"resources": resources_ms,
"diagnostics": diagnostics_ms,
"status": status_ms,
},
"idle_elapsed_seconds": idle_elapsed,
}
samples.append(sample)
if now - last_report >= 10:
print(
json.dumps(
{
"active_jobs": active,
"idle_seconds": idle_elapsed,
"cpu": sample["cpu_percent"],
"rss_mb": round(int(sample["process_rss_bytes"] or 0) / 1024**2, 1),
"inference_running": bool((sample["inference"] or {}).get("running")),
"max_request_ms": max(sample["latency_ms"].values()),
},
ensure_ascii=False,
),
flush=True,
)
last_report = now
if idle_elapsed >= args.idle_seconds and not (sample["inference"] or {}).get("running"):
break
time.sleep(2)
else:
raise TimeoutError("background queues did not drain and release inference before timeout")
final = samples[-1]
all_latencies = [value for sample in samples for value in sample["latency_ms"].values()]
report = {
"samples": len(samples),
"duration_seconds": round(samples[-1]["at"] - samples[0]["at"], 1),
"active_jobs_initial": samples[0]["active_jobs"],
"active_jobs_final": final["active_jobs"],
"cpu_max_percent": max(float(sample["cpu_percent"] or 0) for sample in samples),
"memory_available_min_gb": round(
min(int(sample["memory_available_bytes"] or 0) for sample in samples) / 1024**3,
2,
),
"process_rss_initial_mb": round(int(samples[0]["process_rss_bytes"] or 0) / 1024**2, 1),
"process_rss_final_mb": round(int(final["process_rss_bytes"] or 0) / 1024**2, 1),
"event_loop_lag_final_ms": final["event_loop_lag_ms"],
"event_loop_max_lag_ms": final["event_loop_max_lag_ms"],
"request_latency_max_ms": max(all_latencies),
"inference_final": final["inference"],
"database_final": final["database"],
}
if final["active_jobs"] != 0 or (final["inference"] or {}).get("running"):
raise AssertionError(f"resources were not released: {report}")
if max(all_latencies) > 5000:
raise AssertionError(f"API latency exceeded 5 seconds during soak: {report}")
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print(json.dumps(report, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()
+233
View File
@@ -0,0 +1,233 @@
#!/usr/bin/env python3
"""Verify the six recognition filters against run-owned live videos."""
from __future__ import annotations
import argparse
import json
import time
from pathlib import Path
from typing import Any
import httpx
def call(client: httpx.Client, method: str, path: str, **kwargs: Any) -> Any:
response = client.request(method, path, **kwargs)
if response.is_error:
raise RuntimeError(f"{method} {path} -> {response.status_code}: {response.text[:1000]}")
return response.json()
def find_owned(videos: list[dict[str, Any]], source_id: str, filename: str) -> dict[str, Any]:
match = next(
(
item
for item in videos
if item.get("source_id") == source_id
and str(item.get("source_key") or "").rsplit("/", 1)[-1] == filename
),
None,
)
if match is None:
raise AssertionError(f"run-owned search fixture is missing: {filename}")
return match
def search(
client: httpx.Client,
*,
text: str,
recognition_type: str,
expected_video_id: str | None,
expected_source: str,
forbidden_video_id: str | None = None,
) -> dict[str, Any]:
result = call(
client,
"POST",
"/api/v1/search",
json={"text": text, "recognition_types": [recognition_type], "limit": 100},
)
items = result.get("items", [])
ids = [item["video_id"] for item in items]
if expected_video_id is not None:
if expected_video_id not in ids:
raise AssertionError(f"{recognition_type} search omitted expected video for {text!r}")
matched = next(item for item in items if item["video_id"] == expected_video_id)
if expected_source not in matched.get("match_sources", []):
raise AssertionError(
f"{recognition_type} search returned the target without {expected_source!r}: "
f"{matched.get('match_sources')}"
)
if forbidden_video_id is not None and forbidden_video_id in ids:
raise AssertionError(f"{recognition_type} search returned a forbidden video for {text!r}")
return {
"query": text,
"result_count": len(items),
"matched_expected": expected_video_id in ids if expected_video_id else not items,
"expected_rank": ids.index(expected_video_id) + 1 if expected_video_id in ids else None,
"match_sources": sorted(
{
source
for item in items
if expected_video_id is None or item["video_id"] == expected_video_id
for source in item.get("match_sources", [])
}
),
}
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--base-url", required=True)
parser.add_argument("--token-file", type=Path, required=True)
parser.add_argument("--run-dir", type=Path, required=True)
parser.add_argument("--timeout", type=int, default=600)
args = parser.parse_args()
state = json.loads((args.run_dir / "state.json").read_text())
client = httpx.Client(
base_url=args.base_url.rstrip("/"),
headers={"Authorization": f"Bearer {args.token_file.read_text().strip()}"},
timeout=httpx.Timeout(60, connect=10),
)
run_id = state["run_id"]
source_id = state["source_id"]
expected_names = {
"positive": f"{run_id}-e2e-positive.mp4",
"negative": f"{run_id}-e2e-negative.mp4",
"subtitle": f"{run_id}-subtitle-e2e-subtitle.mp4",
"audio": f"{run_id}-e2e-speech.mp4",
}
deadline = time.monotonic() + args.timeout
while True:
videos = call(client, "GET", "/api/v1/videos?limit=500")
fixtures = {name: find_owned(videos, source_id, filename) for name, filename in expected_names.items()}
terminal = all(
item.get("index_state", {}).get("basic") in {"ready", "failed"}
and item.get("index_state", {}).get("visual") in {"ready", "failed", "skipped"}
and item.get("index_state", {}).get("ocr") in {"ready", "failed", "skipped"}
for item in fixtures.values()
)
if terminal:
break
if time.monotonic() >= deadline:
raise TimeoutError("search fixtures did not reach terminal basic/visual/OCR states")
time.sleep(2)
positive = fixtures["positive"]["id"]
negative = fixtures["negative"]["id"]
subtitle = fixtures["subtitle"]["id"]
audio = fixtures["audio"]["id"]
report: dict[str, Any] = {
"fixtures": {name: item["id"] for name, item in fixtures.items()},
"index_state": {name: item.get("index_state") for name, item in fixtures.items()},
"recognition": {},
}
report["recognition"]["visual_positive"] = search(
client,
text="海边日落与蓝色海洋",
recognition_type="visual",
expected_video_id=positive,
expected_source="semantic",
)
report["recognition"]["visual_negative_fixture"] = search(
client,
text="绿色山脉与森林",
recognition_type="visual",
expected_video_id=negative,
expected_source="semantic",
)
report["recognition"]["ocr_positive"] = search(
client,
text="星河测试 729",
recognition_type="ocr",
expected_video_id=positive,
expected_source="ocr",
forbidden_video_id=negative,
)
report["recognition"]["ocr_negative_fixture"] = search(
client,
text="山谷样本 314",
recognition_type="ocr",
expected_video_id=negative,
expected_source="ocr",
forbidden_video_id=positive,
)
report["recognition"]["subtitle_positive"] = search(
client,
text="字幕验证 星河测试729",
recognition_type="subtitle",
expected_video_id=subtitle,
expected_source="subtitle",
forbidden_video_id=negative,
)
report["recognition"]["subtitle_non_match"] = search(
client,
text="quartz zeppelin 9834721",
recognition_type="subtitle",
expected_video_id=None,
expected_source="subtitle",
forbidden_video_id=subtitle,
)
report["recognition"]["metadata_positive"] = search(
client,
text="IF-E2E-729",
recognition_type="metadata",
expected_video_id=positive,
expected_source="metadata",
forbidden_video_id=negative,
)
report["recognition"]["metadata_non_match"] = search(
client,
text="不存在的资料验收词 9834721",
recognition_type="metadata",
expected_video_id=None,
expected_source="metadata",
forbidden_video_id=positive,
)
report["recognition"]["audio_positive"] = search(
client,
text="image",
recognition_type="audio",
expected_video_id=audio,
expected_source="audio",
forbidden_video_id=negative,
)
report["recognition"]["audio_non_match"] = search(
client,
text="pineapple submarine 9834721",
recognition_type="audio",
expected_video_id=None,
expected_source="audio",
forbidden_video_id=audio,
)
face_report_path = args.run_dir / "face-report.json"
if face_report_path.exists():
face = json.loads(face_report_path.read_text())
report["recognition"]["person_positive"] = search(
client,
text=face["person_name"],
recognition_type="person",
expected_video_id=face["video_id"],
expected_source="person",
)
report["recognition"]["person_non_match"] = search(
client,
text=f"{run_id}-不存在人物-9834721",
recognition_type="person",
expected_video_id=None,
expected_source="person",
forbidden_video_id=face["video_id"],
)
else:
report["recognition"]["person"] = {"status": "pending face acceptance"}
output = args.run_dir / "search-report.json"
output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n")
print(json.dumps(report, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()
+299
View File
@@ -0,0 +1,299 @@
#!/usr/bin/env python3
"""Verify live audio indexing with a run-owned video containing clear speech."""
from __future__ import annotations
import argparse
import atexit
import hashlib
import json
import time
from pathlib import Path
from typing import Any
import httpx
def call(client: httpx.Client, method: str, path: str, **kwargs: Any) -> httpx.Response:
response = client.request(method, path, **kwargs)
if response.is_error:
raise RuntimeError(f"{method} {path} -> {response.status_code}: {response.text[:1000]}")
return response
def sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
while block := handle.read(1024 * 1024):
digest.update(block)
return digest.hexdigest()
def find_video(client: httpx.Client, source_id: str, filename: str) -> dict[str, Any] | None:
videos = call(client, "GET", "/api/v1/videos?limit=500").json()
return next(
(
item
for item in videos
if item.get("source_id") == source_id
and item.get("source_key", "").rsplit("/", 1)[-1] == filename
),
None,
)
def upload_fixture(
client: httpx.Client,
fixture: Path,
state: dict[str, Any],
) -> tuple[dict[str, Any], list[float]]:
filename = f"{state['run_id']}-{fixture.name}"
uploads = call(client, "GET", "/api/v1/uploads?limit=500").json()
upload = next((item for item in uploads if item.get("filename") == filename), None)
if upload is None or upload.get("status") in {"failed", "cancelled"}:
upload = call(
client,
"POST",
"/api/v1/uploads",
json={
"source_id": state["source_id"],
"relative_path": "ingest",
"filename": filename,
"title": "音频 GPU 验收",
"collection_id": state["collection_id"],
"tag_ids": [state["tag_id"]],
"size_bytes": fixture.stat().st_size,
"sha256": sha256(fixture),
"conflict": "skip",
},
).json()
latencies: list[float] = []
if upload.get("status") != "completed":
hidden = find_video(client, state["source_id"], filename)
if hidden is not None:
raise AssertionError("unfinished speech upload leaked into /videos")
chunk_size = int(upload["chunk_size"])
received = {int(index) for index in upload.get("received_chunks", upload.get("received", []))}
with fixture.open("rb") as handle:
for index in range(int(upload["total_chunks"])):
data = handle.read(chunk_size)
if index in received:
continue
started = time.perf_counter()
call(
client,
"PUT",
f"/api/v1/uploads/{upload['id']}/chunks/{index}",
content=data,
headers={"X-Chunk-SHA256": hashlib.sha256(data).hexdigest()},
)
latencies.append(round((time.perf_counter() - started) * 1000, 1))
upload_id = upload["id"]
upload = call(client, "POST", f"/api/v1/uploads/{upload_id}/complete").json()
else:
upload_id = upload["id"]
deadline = time.monotonic() + 240
while upload.get("status") not in {"completed", "failed", "cancelled"}:
if time.monotonic() >= deadline:
raise TimeoutError(f"speech upload did not finish: {upload.get('status')}")
time.sleep(1)
uploads = call(client, "GET", "/api/v1/uploads?limit=500").json()
upload = next(item for item in uploads if item["id"] == upload_id)
if upload["status"] != "completed":
raise AssertionError(f"speech upload failed: {upload.get('message')}")
return upload, latencies
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--base-url", required=True)
auth = parser.add_mutually_exclusive_group(required=True)
auth.add_argument("--token-file", type=Path)
auth.add_argument("--password")
parser.add_argument("--run-dir", type=Path, required=True)
parser.add_argument("--fixture", default="e2e-speech.mp4")
parser.add_argument(
"--query",
action="append",
dest="queries",
help="Required audio-search keyword; repeat to verify multiple words (default: Americans)",
)
parser.add_argument("--timeout", type=int, default=1200)
args = parser.parse_args()
state_path = args.run_dir / "state.json"
state = json.loads(state_path.read_text())
fixture = args.run_dir / args.fixture
if not fixture.is_file():
raise FileNotFoundError(fixture)
headers = {}
if args.token_file is not None:
headers["Authorization"] = f"Bearer {args.token_file.read_text().strip()}"
client = httpx.Client(
base_url=args.base_url.rstrip("/"),
headers=headers,
timeout=httpx.Timeout(60, connect=10),
)
if args.password is not None:
login = call(
client,
"POST",
"/api/v1/auth/login",
json={"password": args.password, "remember_device": False},
).json()
csrf = str(login.get("csrf_token") or "")
if not csrf:
raise AssertionError("login response omitted CSRF token")
client.headers["X-CSRF-Token"] = csrf
def logout() -> None:
try:
client.post("/api/v1/auth/logout")
except Exception:
pass
atexit.register(logout)
# The repository's default live fixture contains the JFK inauguration line
# "And so, my fellow Americans...". Keep the default assertion tied to
# the actual fixture; callers using another recording can repeat --query.
required_queries = [value.strip() for value in (args.queries or ["Americans"]) if value.strip()]
if not required_queries:
raise ValueError("at least one non-empty --query is required")
initial_models = call(client, "GET", "/api/v1/models").json()
print(
json.dumps(
{
"audio_model": (initial_models.get("manifest") or {})
.get("components", {})
.get("audio"),
"audio_health": (initial_models.get("component_health") or {}).get("audio"),
"audio_accelerator": (initial_models.get("accelerator") or {})
.get("components", {})
.get("audio"),
},
ensure_ascii=False,
),
flush=True,
)
upload, chunk_latencies = upload_fixture(client, fixture, state)
filename = upload["filename"]
deadline = time.monotonic() + args.timeout
observed_devices: list[str] = []
latest_message = "等待视频入库"
positives: dict[str, dict[str, Any]] = {}
video: dict[str, Any] | None = None
last_report_at = 0.0
while time.monotonic() < deadline:
video = find_video(client, state["source_id"], filename)
models = call(client, "GET", "/api/v1/models").json()
audio = models.get("accelerator", {}).get("components", {}).get("audio", {})
device = audio.get("actual_device") or audio.get("device")
if device and device not in observed_devices:
observed_devices.append(device)
if video is not None:
for query in required_queries:
if query in positives:
continue
result = call(
client,
"POST",
"/api/v1/search",
json={"text": query, "recognition_types": ["audio"]},
).json()
positive = next(
(item for item in result.get("items", []) if item.get("video_id") == video["id"]),
None,
)
if positive is not None:
positives[query] = positive
if len(positives) == len(required_queries):
break
jobs = call(client, "GET", "/api/v1/jobs?page=1&page_size=10").json()["items"]
audio_jobs = [item for item in jobs if item.get("kind") == "transcribe_audio"]
if audio_jobs:
latest_message = audio_jobs[0].get("message") or audio_jobs[0].get("status", "")
if audio_jobs[0].get("status") == "failed":
raise AssertionError(f"audio job failed: {audio_jobs[0].get('error')}")
now = time.monotonic()
if now - last_report_at >= 10:
print(
json.dumps(
{
"waiting": latest_message,
"video_found": video is not None,
"observed_devices": observed_devices,
},
ensure_ascii=False,
),
flush=True,
)
last_report_at = now
time.sleep(2)
if video is None:
raise AssertionError("speech video never appeared in /videos")
missing_queries = [query for query in required_queries if query not in positives]
if missing_queries:
raise TimeoutError(
"audio search never matched required keywords "
f"{missing_queries}; matched={sorted(positives)}; latest={latest_message}"
)
for query, positive in positives.items():
if positive.get("segment_start_ms") is None or positive.get("segment_end_ms") is None:
raise AssertionError(f"audio match omitted time segment for {query!r}: {positive}")
match_details = positive.get("match_details", [])
audio_text = " ".join(
str(detail.get("text") or "")
for detail in match_details
if detail.get("type") == "audio"
).strip()
if not audio_text:
raise AssertionError(f"audio match omitted recognized text for {query!r}: {positive}")
if len([character for character in audio_text if character.isalnum()]) < 8:
raise AssertionError(
f"audio transcript is implausibly short for {query!r}: {audio_text!r}"
)
negative = call(
client,
"POST",
"/api/v1/search",
json={"text": "pineapple submarine 9834721", "recognition_types": ["audio"]},
).json()
if any(item.get("video_id") == video["id"] for item in negative.get("items", [])):
raise AssertionError("non-matching audio search unexpectedly returned the speech video")
jobs = call(client, "GET", "/api/v1/jobs?page=1&page_size=10").json()["items"]
latest_audio = next((item for item in jobs if item.get("kind") == "transcribe_audio"), None)
report = {
"upload_id": upload["id"],
"video_id": video["id"],
"chunk_latency_ms": chunk_latencies,
"observed_audio_devices": observed_devices,
"audio_job": latest_audio,
"matches": {
query: {
"segment_start_ms": positive.get("segment_start_ms"),
"segment_end_ms": positive.get("segment_end_ms"),
"details": positive.get("match_details", []),
}
for query, positive in positives.items()
},
"negative_matches": len(negative.get("items", [])),
}
if not any(str(device).upper().startswith("GPU") for device in observed_devices):
raise AssertionError(f"GPU was not observed during speech indexing: {observed_devices}")
state[f"speech_upload_id_{fixture.stem}"] = upload["id"]
state[f"speech_video_id_{fixture.stem}"] = video["id"]
state_path.write_text(json.dumps(state, ensure_ascii=False, indent=2) + "\n")
(args.run_dir / "speech-report.json").write_text(
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
)
print(json.dumps(report, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()
+99
View File
@@ -0,0 +1,99 @@
#!/usr/bin/env python3
"""Issue or revoke a short-lived API token used by live acceptance scripts."""
from __future__ import annotations
import argparse
import json
import os
import sys
from pathlib import Path
import httpx
def call(client: httpx.Client, method: str, path: str, **kwargs) -> httpx.Response:
response = client.request(method, path, **kwargs)
if response.is_error:
raise RuntimeError(f"{method} {path} -> {response.status_code}: {response.text[:1000]}")
return response
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--base-url", required=True)
password = parser.add_mutually_exclusive_group()
password.add_argument("--password")
password.add_argument("--password-stdin", action="store_true")
password.add_argument("--storage-state", type=Path)
parser.add_argument("--token-file", type=Path, required=True)
parser.add_argument("--id-file", type=Path, required=True)
parser.add_argument("--name", default="ImageFind live acceptance")
parser.add_argument("--revoke", action="store_true")
args = parser.parse_args()
token_revoke_ready = args.revoke and args.token_file.exists() and args.id_file.exists()
if not token_revoke_ready and not (args.password or args.password_stdin or args.storage_state):
parser.error("one of --password, --password-stdin or --storage-state is required")
password_value = sys.stdin.readline().rstrip("\r\n") if args.password_stdin else args.password
client = httpx.Client(
base_url=args.base_url.rstrip("/"),
timeout=httpx.Timeout(30, connect=10),
)
if token_revoke_ready:
client.headers["Authorization"] = (
f"Bearer {args.token_file.read_text(encoding='utf-8').strip()}"
)
token_id = args.id_file.read_text(encoding="utf-8").strip()
call(client, "DELETE", f"/api/v1/tokens/{token_id}")
args.token_file.unlink(missing_ok=True)
args.id_file.unlink(missing_ok=True)
print(f"revoked live acceptance token {token_id}")
return
if args.storage_state:
state = json.loads(args.storage_state.read_text(encoding="utf-8"))
for cookie in state.get("cookies", []):
client.cookies.set(
str(cookie["name"]),
str(cookie["value"]),
domain=str(cookie.get("domain") or ""),
path=str(cookie.get("path") or "/"),
)
login = call(client, "GET", "/api/v1/auth/me").json()
else:
login = call(
client,
"POST",
"/api/v1/auth/login",
json={"password": password_value, "remember_device": False},
).json()
password_value = ""
csrf = str(login.get("csrf_token") or "")
if not csrf:
raise AssertionError("login response omitted CSRF token")
client.headers["X-CSRF-Token"] = csrf
if args.revoke:
token_id = args.id_file.read_text(encoding="utf-8").strip()
call(client, "DELETE", f"/api/v1/tokens/{token_id}")
args.token_file.unlink(missing_ok=True)
args.id_file.unlink(missing_ok=True)
print(f"revoked live acceptance token {token_id}")
return
result = call(
client,
"POST",
"/api/v1/tokens",
json={"name": args.name, "scopes": ["admin"]},
).json()
args.token_file.write_text(str(result["token"]), encoding="utf-8")
args.id_file.write_text(str(result["id"]), encoding="utf-8")
os.chmod(args.token_file, 0o600)
os.chmod(args.id_file, 0o600)
print(f"issued live acceptance token {result['id']}")
if __name__ == "__main__":
main()
+235
View File
@@ -0,0 +1,235 @@
#!/usr/bin/env python3
"""Run the resumable browser-upload acceptance path against a disposable live server.
The API token is read from a mode-0600 file and is never included in output. The
script only creates objects prefixed by the supplied run id and records their ids
so a later cleanup can be narrowly scoped.
"""
from __future__ import annotations
import argparse
import hashlib
import json
import time
from pathlib import Path
from typing import Any
import httpx
def request(client: httpx.Client, method: str, path: str, **kwargs: Any) -> httpx.Response:
started = time.perf_counter()
response = client.request(method, path, **kwargs)
elapsed_ms = round((time.perf_counter() - started) * 1000, 1)
if response.is_error:
detail = response.text[:1000]
raise RuntimeError(f"{method} {path} -> {response.status_code} in {elapsed_ms} ms: {detail}")
return response
def sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
while block := handle.read(1024 * 1024):
digest.update(block)
return digest.hexdigest()
def ensure_metadata(client: httpx.Client, run_id: str) -> tuple[str, str, str]:
collection_name = f"{run_id}-合集"
collections = request(client, "GET", "/api/v1/collections").json()
collection = next((item for item in collections if item["name"] == collection_name), None)
if collection is None:
collection = request(
client,
"POST",
"/api/v1/collections",
json={"name": collection_name, "description": "ImageFind 自动验收临时合集"},
).json()
group_name = f"{run_id}-分类"
groups = request(client, "GET", "/api/v1/tag-groups").json()
group = next((item for item in groups if item["name"] == group_name), None)
if group is None:
group = request(
client,
"POST",
"/api/v1/tag-groups",
json={"name": group_name, "selection_mode": "multi", "sort_order": 9999},
).json()
group_id = group["id"]
else:
group_id = group["id"]
tag_name = f"{run_id}-样本"
tags = request(client, "GET", f"/api/v1/tags?group_id={group_id}").json()
tag = next((item for item in tags if item["name"] == tag_name), None)
if tag is None:
tag = request(
client,
"POST",
"/api/v1/tags",
json={"group_id": group_id, "name": tag_name},
).json()
return collection["id"], group_id, tag["id"]
def upload_one(
client: httpx.Client,
fixture: Path,
source_id: str,
collection_id: str,
tag_id: str,
run_id: str,
case_id: str,
) -> tuple[dict[str, Any], list[float]]:
case_prefix = f"-{case_id}" if case_id else ""
filename = f"{run_id}{case_prefix}-{fixture.name}"
existing = request(client, "GET", "/api/v1/uploads?limit=500").json()
upload = next(
(
item
for item in existing
if item.get("filename") == filename
and item.get("status") in {"receiving", "queued", "transferring", "indexing"}
),
None,
)
latencies: list[float] = []
if upload is None:
upload = request(
client,
"POST",
"/api/v1/uploads",
json={
"source_id": source_id,
"relative_path": "ingest",
"filename": filename,
"title": f"自动验收 · {fixture.stem}",
"collection_id": collection_id,
"tag_ids": [tag_id],
"size_bytes": fixture.stat().st_size,
"sha256": sha256(fixture),
"conflict": "skip",
},
).json()
upload_id = upload["id"]
if upload.get("status") == "completed":
return upload, latencies
hidden = request(client, "GET", "/api/v1/videos?limit=500").json()
if any(item.get("source_key", "").endswith(filename) for item in hidden):
raise AssertionError(f"unfinished upload leaked into /videos: {filename}")
chunk_size = int(upload["chunk_size"])
received = {int(index) for index in upload.get("received_chunks", upload.get("received", []))}
with fixture.open("rb") as handle:
for index in range(int(upload["total_chunks"])):
data = handle.read(chunk_size)
if index in received:
continue
started = time.perf_counter()
request(
client,
"PUT",
f"/api/v1/uploads/{upload_id}/chunks/{index}",
content=data,
headers={"X-Chunk-SHA256": hashlib.sha256(data).hexdigest()},
)
latencies.append(round((time.perf_counter() - started) * 1000, 1))
videos = request(client, "GET", "/api/v1/videos?limit=500").json()
if any(item.get("source_key", "").endswith(filename) for item in videos):
raise AssertionError(f"partially received upload leaked into /videos: {filename}")
upload = request(client, "POST", f"/api/v1/uploads/{upload_id}/complete").json()
deadline = time.monotonic() + 180
while upload.get("status") not in {"completed", "failed", "cancelled"}:
if time.monotonic() >= deadline:
raise TimeoutError(f"upload did not finish: {filename} status={upload.get('status')}")
time.sleep(1)
uploads = request(client, "GET", "/api/v1/uploads?limit=500").json()
upload = next(item for item in uploads if item["id"] == upload_id)
if upload["status"] != "completed":
raise AssertionError(f"upload failed: {filename}: {upload.get('message')}")
return upload, latencies
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--base-url", required=True)
auth = parser.add_mutually_exclusive_group(required=True)
auth.add_argument("--token-file", type=Path)
auth.add_argument("--password")
parser.add_argument("--run-dir", type=Path, required=True)
parser.add_argument("--case-id", default="")
parser.add_argument("--fixture", action="append", default=[])
args = parser.parse_args()
state_path = args.run_dir / "state.json"
state = json.loads(state_path.read_text())
headers = {}
if args.token_file is not None:
token = args.token_file.read_text().strip()
if not token:
raise RuntimeError("empty API token")
headers["Authorization"] = f"Bearer {token}"
client = httpx.Client(
base_url=args.base_url.rstrip("/"),
headers=headers,
timeout=httpx.Timeout(60, connect=10),
)
if args.password is not None:
login = request(
client,
"POST",
"/api/v1/auth/login",
json={"password": args.password, "remember_device": False},
).json()
csrf = str(login.get("csrf_token") or "")
if not csrf:
raise AssertionError("login response omitted CSRF token")
client.headers["X-CSRF-Token"] = csrf
collection_id, group_id, tag_id = ensure_metadata(client, state["run_id"])
state.update({"collection_id": collection_id, "tag_group_id": group_id, "tag_id": tag_id})
results = []
all_latencies: list[float] = []
fixture_names = args.fixture or ["e2e-positive.mp4", "e2e-negative.mp4", "e2e-fallback.mpg"]
for name in fixture_names:
upload, latencies = upload_one(
client,
args.run_dir / name,
state["source_id"],
collection_id,
tag_id,
state["run_id"],
args.case_id.strip(),
)
results.append(
{
"id": upload["id"],
"filename": upload["filename"],
"status": upload["status"],
"progress": upload["progress"],
"message": upload.get("message"),
}
)
all_latencies.extend(latencies)
state["upload_ids"] = [item["id"] for item in results]
state_path.write_text(json.dumps(state, ensure_ascii=False, indent=2) + "\n")
report = {
"uploads": results,
"chunk_latency_ms": {
"count": len(all_latencies),
"max": max(all_latencies, default=0),
"average": round(sum(all_latencies) / len(all_latencies), 1) if all_latencies else 0,
},
}
(args.run_dir / "upload-report.json").write_text(
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
)
print(json.dumps(report, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()
+241
View File
@@ -0,0 +1,241 @@
#!/usr/bin/env python3
"""Exercise ImageFind's WebDAV server through its public direct listener."""
from __future__ import annotations
import argparse
import hashlib
import json
import time
from pathlib import Path
from typing import Any
from urllib.parse import quote
import httpx
def call(client: httpx.Client, method: str, path: str, **kwargs: Any) -> httpx.Response:
response = client.request(method, path, **kwargs)
if response.is_error:
raise RuntimeError(f"{method} {path} -> {response.status_code}: {response.text[:1000]}")
return response
def digest(data: bytes) -> str:
return hashlib.sha256(data).hexdigest()
def encoded(parts: list[str]) -> str:
return "/webdav/" + "/".join(quote(part, safe="") for part in parts)
def videos(client: httpx.Client) -> list[dict[str, Any]]:
return call(client, "GET", "/api/v1/videos?limit=500").json()
def visible_video(client: httpx.Client, filename: str) -> dict[str, Any] | None:
return next((item for item in videos(client) if item.get("source_key", "").endswith("/" + filename)), None)
def wait_upload(client: httpx.Client, upload_id: str, timeout: int = 240) -> dict[str, Any]:
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
rows = call(client, "GET", "/api/v1/uploads?limit=500").json()
upload = next((item for item in rows if item["id"] == upload_id), None)
if upload and upload["status"] in {"completed", "failed", "cancelled"}:
if upload["status"] != "completed":
raise AssertionError(f"WebDAV upload failed: {upload.get('message') or upload.get('error')}")
return upload
time.sleep(1)
raise TimeoutError(f"WebDAV upload did not complete: {upload_id}")
def wait_video(client: httpx.Client, filename: str, timeout: int = 90) -> dict[str, Any]:
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
if video := visible_video(client, filename):
return video
time.sleep(1)
raise TimeoutError(f"WebDAV video did not appear: {filename}")
def upload_id(response: httpx.Response) -> str:
value = response.headers.get("x-imagefind-upload-id")
if not value:
raise AssertionError(f"WebDAV response omitted upload id: {dict(response.headers)}")
return value
def collection_video_path(nodes: list[dict[str, Any]], video_id: str, parents: list[str] | None = None):
parents = list(parents or [])
for node in nodes:
if node.get("kind") == "group":
found = collection_video_path(node.get("children") or [], video_id, [*parents, node["name"]])
if found is not None:
return found
elif node.get("video_id") == video_id:
return parents
return None
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--base-url", required=True)
parser.add_argument("--token-file", type=Path, required=True)
parser.add_argument("--run-dir", type=Path, required=True)
parser.add_argument("--case-id", default="v045")
args = parser.parse_args()
state = json.loads((args.run_dir / "state.json").read_text())
token = args.token_file.read_text().strip()
prefix = f"{state['run_id']}-{args.case_id.strip()}-dav"
payload = (args.run_dir / "e2e-positive.mp4").read_bytes()
checksum = digest(payload)
api = httpx.Client(
base_url=args.base_url.rstrip("/"),
headers={"Authorization": f"Bearer {token}"},
timeout=httpx.Timeout(60, connect=10),
)
dav = httpx.Client(
base_url=args.base_url.rstrip("/"),
auth=httpx.BasicAuth("imagefind", token),
timeout=httpx.Timeout(120, connect=10, write=120, read=120),
)
config = call(api, "GET", "/api/v1/webdav/config").json()
if not config.get("enabled") or config.get("source_id") != state["source_id"]:
raise AssertionError(f"unexpected WebDAV config: {config}")
options = call(dav, "OPTIONS", "/webdav/")
if "PROPFIND" not in options.headers.get("allow", ""):
raise AssertionError("WebDAV OPTIONS omitted PROPFIND")
root = call(dav, "PROPFIND", "/webdav/", headers={"Depth": "1"})
if "multistatus" not in root.text:
raise AssertionError("WebDAV root did not return a multistatus document")
root_name = f"{prefix}-root.mp4"
root_response = call(
dav,
"PUT",
encoded([root_name]),
content=payload,
headers={"X-Content-SHA256": checksum},
)
root_task = wait_upload(api, upload_id(root_response))
root_video = wait_video(api, root_name)
collection_name = f"{prefix}-collection"
groups = [f"level-{number:02d}" for number in range(1, 6)]
path: list[str] = [collection_name]
for part in [collection_name, *groups]:
if part != collection_name:
path.append(part)
response = dav.request("MKCOL", encoded(path))
if response.status_code not in {201, 405}:
raise RuntimeError(f"MKCOL {path} -> {response.status_code}: {response.text[:1000]}")
resume_name = f"{prefix}-resume.mp4"
resume_path = encoded([collection_name, *groups, resume_name])
midpoint = len(payload) // 2
first = call(
dav,
"PUT",
resume_path,
content=payload[:midpoint],
headers={
"Content-Range": f"bytes 0-{midpoint - 1}/{len(payload)}",
"X-Content-SHA256": checksum,
},
)
if first.status_code != 204 or int(first.headers.get("upload-offset", 0)) != midpoint:
raise AssertionError(f"unexpected partial PUT response: {first.status_code} {dict(first.headers)}")
if visible_video(api, resume_name):
raise AssertionError("partial WebDAV upload leaked into /videos")
staged = call(dav, "HEAD", resume_path)
if int(staged.headers.get("upload-offset", 0)) != midpoint:
raise AssertionError(f"HEAD did not expose resume offset: {dict(staged.headers)}")
second = call(
dav,
"PUT",
resume_path,
content=payload[midpoint:],
headers={
"Content-Range": f"bytes {midpoint}-{len(payload) - 1}/{len(payload)}",
"X-Content-SHA256": checksum,
},
)
resume_task = wait_upload(api, upload_id(second))
resume_video = wait_video(api, resume_name)
part_name = f"{prefix}-move.mp4.part"
final_name = f"{prefix}-move.mp4"
part_path = encoded([collection_name, *groups, part_name])
final_path = encoded([collection_name, *groups, final_name])
temporary = call(
dav,
"PUT",
part_path,
content=payload,
headers={"X-Content-SHA256": checksum},
)
if temporary.headers.get("x-imagefind-upload-id"):
raise AssertionError("temporary WebDAV name was committed before MOVE")
moved = call(
dav,
"MOVE",
part_path,
headers={"Destination": args.base_url.rstrip("/") + final_path, "Overwrite": "F"},
)
move_task = wait_upload(api, upload_id(moved))
move_video = wait_video(api, final_name)
started = time.perf_counter()
duplicate = call(
dav,
"PUT",
resume_path,
content=payload,
headers={"X-Content-SHA256": checksum, "Expect": "100-continue"},
)
dedupe_ms = round((time.perf_counter() - started) * 1000, 1)
if duplicate.status_code != 204 or duplicate.headers.get("x-imagefind-deduplicated") != "true":
raise AssertionError(f"same-path duplicate was not acknowledged: {dict(duplicate.headers)}")
if duplicate.headers.get("x-imagefind-upload-id") != resume_task["id"]:
raise AssertionError("deduplicated PUT did not reference the original upload task")
ranged = call(dav, "GET", final_path, headers={"Range": "bytes=0-63"})
if ranged.status_code != 206 or ranged.content != payload[:64]:
raise AssertionError("WebDAV Range read returned unexpected bytes")
collections = call(api, "GET", "/api/v1/collections").json()
collection = next((item for item in collections if item["name"] == collection_name), None)
if not collection:
raise AssertionError("WebDAV directory did not create a collection")
detail = call(api, "GET", f"/api/v1/collections/{collection['id']}").json()
for video in (resume_video, move_video):
actual_path = collection_video_path(detail.get("items") or [], video["id"])
if actual_path != groups:
raise AssertionError(f"collection hierarchy mismatch: {video['id']} -> {actual_path}")
if any(item["id"] == root_video["id"] for item in detail.get("videos") or []):
raise AssertionError("root WebDAV upload unexpectedly joined a collection")
report = {
"root_upload": {"id": root_task["id"], "video_id": root_video["id"]},
"resumable_upload": {
"id": resume_task["id"],
"video_id": resume_video["id"],
"first_offset": midpoint,
"final_size": len(payload),
},
"temporary_move": {"id": move_task["id"], "video_id": move_video["id"]},
"deduplicated": True,
"dedupe_latency_ms": dedupe_ms,
"range_read": len(ranged.content),
"collection_id": collection["id"],
"collection_depth": len(groups),
}
(args.run_dir / "webdav-report.json").write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n")
print(json.dumps(report, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()
+279
View File
@@ -0,0 +1,279 @@
#!/usr/bin/env python3
"""Stress ImageFind WebDAV reception without retaining generated large files."""
from __future__ import annotations
import argparse
import hashlib
import json
import threading
import time
from concurrent.futures import ThreadPoolExecutor, as_completed
from pathlib import Path
from typing import Any
from urllib.parse import quote
import httpx
BLOCK = bytes(1024 * 1024)
def padded_hash(prefix: bytes, size: int) -> str:
value = hashlib.sha256(prefix)
remaining = size - len(prefix)
while remaining > 0:
length = min(len(BLOCK), remaining)
value.update(BLOCK[:length])
remaining -= length
return value.hexdigest()
def padded_body(prefix: bytes, size: int):
yield prefix
remaining = size - len(prefix)
while remaining > 0:
length = min(len(BLOCK), remaining)
yield BLOCK[:length]
remaining -= length
def request(client: httpx.Client, method: str, path: str, **kwargs: Any) -> httpx.Response:
response = client.request(method, path, **kwargs)
if response.is_error:
raise RuntimeError(f"{method} {path} -> {response.status_code}: {response.text[:1000]}")
return response
def encoded(parts: list[str]) -> str:
return "/webdav/" + "/".join(quote(part, safe="") for part in parts)
def wait_uploads(api: httpx.Client, ids: set[str], timeout: int) -> dict[str, dict[str, Any]]:
deadline = time.monotonic() + timeout
terminal: dict[str, dict[str, Any]] = {}
while time.monotonic() < deadline:
rows = request(api, "GET", "/api/v1/uploads?limit=500").json()
by_id = {item["id"]: item for item in rows if item["id"] in ids}
terminal = {
key: value
for key, value in by_id.items()
if value["status"] in {"completed", "failed", "cancelled"}
}
if len(terminal) == len(ids):
failures = [item for item in terminal.values() if item["status"] != "completed"]
if failures:
raise AssertionError(f"committed WebDAV uploads failed: {failures}")
return terminal
time.sleep(1)
raise TimeoutError(f"WebDAV transfers did not finish: {ids - set(terminal)}")
def wait_video(api: httpx.Client, filename: str, timeout: int = 120) -> dict[str, Any]:
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
rows = request(api, "GET", "/api/v1/videos?limit=500").json()
match = next((item for item in rows if item.get("source_key", "").endswith("/" + filename)), None)
if match:
return match
time.sleep(1)
raise TimeoutError(f"committed stress video did not appear: {filename}")
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--base-url", required=True)
parser.add_argument("--token-file", type=Path, required=True)
parser.add_argument("--run-dir", type=Path, required=True)
parser.add_argument("--case-id", required=True)
parser.add_argument("--size-mb", type=int, required=True)
parser.add_argument("--count", type=int, required=True)
parser.add_argument("--concurrency", type=int, default=4)
parser.add_argument("--commit-count", type=int, default=1)
parser.add_argument("--timeout", type=int, default=1800)
args = parser.parse_args()
state = json.loads((args.run_dir / "state.json").read_text())
token = args.token_file.read_text().strip()
prefix = f"{state['run_id']}-{args.case_id}-stress"
collection = f"{prefix}-collection"
group = "uploads"
size = args.size_mb * 1024**2
fixture = (args.run_dir / "e2e-positive.mp4").read_bytes()
if size < len(fixture):
raise ValueError("stress size must be at least the fixture size")
checksum = padded_hash(fixture, size)
auth = httpx.BasicAuth("imagefind", token)
api = httpx.Client(
base_url=args.base_url.rstrip("/"),
headers={"Authorization": f"Bearer {token}"},
timeout=httpx.Timeout(60, connect=10),
)
dav = httpx.Client(
base_url=args.base_url.rstrip("/"),
auth=auth,
timeout=httpx.Timeout(args.timeout, connect=15, read=args.timeout, write=args.timeout),
)
for parts in ([collection], [collection, group]):
response = dav.request("MKCOL", encoded(parts))
if response.status_code not in {201, 405}:
raise RuntimeError(f"MKCOL {parts} -> {response.status_code}: {response.text[:1000]}")
stop = threading.Event()
samples: list[dict[str, Any]] = []
def monitor() -> None:
with httpx.Client(
base_url=args.base_url.rstrip("/"),
headers={"Authorization": f"Bearer {token}"},
timeout=10,
) as client:
while not stop.wait(1):
started = time.perf_counter()
try:
status = client.get("/api/v1/status")
status.raise_for_status()
latency = round((time.perf_counter() - started) * 1000, 1)
resources = client.get("/api/v1/system/resources")
resources.raise_for_status()
payload = resources.json()
samples.append(
{
"api_ms": latency,
"cpu": payload.get("cpu_percent"),
"memory_available": payload.get("memory_available_bytes"),
"writer_wait_ms": (payload.get("database") or {}).get("last_wait_ms"),
"ok": True,
}
)
except Exception as exc:
samples.append({"ok": False, "error": type(exc).__name__})
monitor_thread = threading.Thread(target=monitor, name="imagefind-webdav-monitor", daemon=True)
monitor_thread.start()
files = [f"{prefix}-{index + 1:03d}.mp4" for index in range(args.count)]
temporary_paths = {name: encoded([collection, group, name + ".part"]) for name in files}
def put_one(filename: str) -> dict[str, Any]:
path = temporary_paths[filename]
started = time.perf_counter()
with httpx.Client(
base_url=args.base_url.rstrip("/"),
auth=auth,
timeout=httpx.Timeout(args.timeout, connect=15, read=args.timeout, write=args.timeout),
) as client:
response = client.put(
path,
content=padded_body(fixture, size),
headers={
"Content-Length": str(size),
"X-Content-SHA256": checksum,
},
)
if response.is_error:
raise RuntimeError(f"PUT {filename} -> {response.status_code}: {response.text[:1000]}")
if response.status_code not in {201, 204}:
raise RuntimeError(f"PUT {filename} returned {response.status_code}")
return {
"filename": filename,
"status": response.status_code,
"elapsed_ms": round((time.perf_counter() - started) * 1000, 1),
"offset": int(response.headers.get("upload-offset", 0)),
}
started_all = time.perf_counter()
results: list[dict[str, Any]] = []
error: Exception | None = None
committed: list[tuple[str, str]] = []
videos: list[dict[str, Any]] = []
try:
with ThreadPoolExecutor(max_workers=args.concurrency) as executor:
futures = {executor.submit(put_one, filename): filename for filename in files}
for future in as_completed(futures):
results.append(future.result())
if any(item["offset"] != size for item in results):
raise AssertionError("one or more WebDAV PUT responses reported an incomplete offset")
for filename in files[: min(args.commit_count, len(files))]:
source_path = temporary_paths[filename]
destination = encoded([collection, group, filename])
moved = request(
dav,
"MOVE",
source_path,
headers={
"Destination": args.base_url.rstrip("/") + destination,
"Overwrite": "F",
},
)
upload_id = moved.headers.get("x-imagefind-upload-id")
if not upload_id:
raise AssertionError(f"MOVE omitted upload id for {filename}")
committed.append((filename, upload_id))
if committed:
wait_uploads(api, {upload_id for _, upload_id in committed}, args.timeout)
for filename, _ in committed:
video = wait_video(api, filename)
ranged = request(
dav,
"GET",
encoded([collection, group, filename]),
headers={"Range": "bytes=0-63"},
)
if ranged.status_code != 206 or ranged.content != fixture[:64]:
raise AssertionError(f"committed range read failed: {filename}")
videos.append(video)
except Exception as exc:
error = exc
finally:
for filename in files[len(committed) :]:
try:
dav.delete(temporary_paths[filename])
except Exception:
pass
for video in videos:
try:
removed = request(api, "DELETE", f"/api/v1/videos/{video['id']}?delete_source=true").json()
if removed.get("trash_id"):
request(api, "DELETE", f"/api/v1/trash/{removed['trash_id']}")
except Exception:
pass
stop.set()
monitor_thread.join(timeout=15)
if error is not None:
raise error
elapsed = time.perf_counter() - started_all
successful_samples = [item for item in samples if item.get("ok")]
report = {
"case_id": args.case_id,
"size_mb": args.size_mb,
"count": args.count,
"concurrency": args.concurrency,
"committed": len(committed),
"elapsed_seconds": round(elapsed, 2),
"throughput_mib_s": round(args.size_mb * args.count / max(elapsed, 0.001), 2),
"put_latency_ms": {
"minimum": min(item["elapsed_ms"] for item in results),
"maximum": max(item["elapsed_ms"] for item in results),
},
"monitor": {
"samples": len(samples),
"failed_samples": sum(1 for item in samples if not item.get("ok")),
"max_api_ms": max((item["api_ms"] for item in successful_samples), default=0),
"max_cpu_percent": max((item["cpu"] for item in successful_samples), default=0),
"min_memory_available_gb": round(
min((item["memory_available"] for item in successful_samples), default=0) / 1024**3,
2,
),
"max_writer_wait_ms": max((item["writer_wait_ms"] for item in successful_samples), default=0),
},
}
report_path = args.run_dir / f"webdav-stress-{args.case_id}.json"
report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n")
print(json.dumps(report, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()
+45
View File
@@ -0,0 +1,45 @@
import sys
from pathlib import Path
from PIL import Image, ImageDraw
def build(path: Path, size: int) -> None:
# The fnOS mobile launcher already applies its own icon mask. Transparent
# corners are composited against black in the app, so keep the full canvas
# opaque and let the branded background run all the way to every edge.
image = Image.new("RGBA", (size, size), (103, 145, 244, 255))
draw = ImageDraw.Draw(image)
center = size // 2
radius = size // 5
draw.ellipse(
(center - radius, center - radius, center + radius, center + radius),
outline=(255, 255, 255, 245),
width=max(2, size // 25),
)
sparkle = size // 13
draw.polygon(
[
(size * 3 // 4, size // 5),
(size * 3 // 4 + sparkle, size // 3),
(size * 7 // 8, size // 3 + sparkle),
(size * 3 // 4 + sparkle, size // 3 + sparkle * 2),
(size * 3 // 4, size // 2),
(size * 3 // 4 - sparkle, size // 3 + sparkle * 2),
(size * 5 // 8, size // 3 + sparkle),
(size * 3 // 4 - sparkle, size // 3),
],
fill=(255, 255, 255, 255),
)
image.save(path, "PNG")
if __name__ == "__main__":
destination = Path(sys.argv[1])
destination.mkdir(parents=True, exist_ok=True)
build(destination / "ICON.PNG", 128)
build(destination / "ICON_256.PNG", 256)
ui_images = destination / "app" / "ui" / "images"
ui_images.mkdir(parents=True, exist_ok=True)
build(ui_images / "icon_64.png", 64)
build(ui_images / "icon_256.png", 256)
+29
View File
@@ -0,0 +1,29 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
PYTHON_BIN=${PYTHON:-python3}
WHEELHOUSE=${IMAGEFIND_WHEEL_CACHE:-$ROOT_DIR/.fnos-wheel-cache/python312}
DOWNLOAD_ROOT=${IMAGEFIND_DEPENDENCY_TMPDIR:-${TMPDIR:-/tmp}}
OPENCL_URL=https://deb.debian.org/debian/pool/main/o/ocl-icd/ocl-icd-libopencl1_2.2.14-2_amd64.deb
OPENCL_SHA256=f3367b78f2548e6211e23081b3e0f144babfcc906c5bdbbe0d5e41915de70128
command -v "$PYTHON_BIN" >/dev/null
for command_name in curl dpkg-deb find install sha256sum; do command -v "$command_name" >/dev/null; done
mkdir -p "$WHEELHOUSE" "$ROOT_DIR/vendor" "$DOWNLOAD_ROOT"
"$PYTHON_BIN" -m pip download --disable-pip-version-check --only-binary=:all: \
--implementation cp --python-version 312 --abi cp312 \
--platform manylinux_2_28_x86_64 --platform manylinux_2_27_x86_64 \
--platform manylinux2014_x86_64 --platform manylinux2010_x86_64 --platform manylinux1_x86_64 \
--dest "$WHEELHOUSE" --requirement "$ROOT_DIR/requirements/runtime-core.txt"
work_dir=$(mktemp -d "${DOWNLOAD_ROOT%/}/imagefind-opencl.XXXXXX")
trap 'rm -rf -- "$work_dir"' EXIT
curl --fail --location --retry 4 --retry-all-errors "$OPENCL_URL" --output "$work_dir/opencl.deb"
dpkg-deb -x "$work_dir/opencl.deb" "$work_dir/root"
source_file=$(find "$work_dir/root" -type f -name 'libOpenCL.so.1.*' -print -quit)
test -n "$source_file"
printf '%s %s\n' "$OPENCL_SHA256" "$source_file" | sha256sum --check --status
install -m 0644 "$source_file" "$ROOT_DIR/vendor/libOpenCL.so.1"
printf 'Prepared %s and %s\n' "$WHEELHOUSE" "$ROOT_DIR/vendor/libOpenCL.so.1"
+4
View File
@@ -0,0 +1,4 @@
from imagefind.main import run
if __name__ == "__main__":
run()
+137
View File
@@ -0,0 +1,137 @@
#!/bin/sh
set -eu
PROJECT_ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
APP_ROOT="${1:-$PROJECT_ROOT/.build-fnos/imagefind/app}"
# Keep the default data root on the project filesystem. Resource admission is
# intentionally disk-aware; placing smoke data on a small /tmp tmpfs can make
# a healthy transfer worker wait forever at the production 5 GiB reserve.
DATA_ROOT="${2:-$PROJECT_ROOT/.smoke-fnos}"
PORT="${3:-18765}"
PACKAGE_ROOT="${4:-$PROJECT_ROOT/fnos}"
PYTHON_PATH="${5:-${IMAGEFIND_PYTHON_PATH:-}}"
CONTROL="$PACKAGE_ROOT/cmd/main"
INSTALL_CALLBACK="$PACKAGE_ROOT/cmd/install_callback"
CONFIG_CALLBACK="$PACKAGE_ROOT/cmd/config_callback"
SMOKE_PASSWORD="imagefind-smoke-password"
SMOKE_POSTGRES_TOKEN="${IMAGEFIND_SMOKE_POSTGRES_TOKEN:-}"
COOKIE_JAR="$DATA_ROOT/run/smoke-cookie.txt"
BACKUP_FILE="$DATA_ROOT/run/smoke.ifbackup"
EXPECTED_VERSION=$(sed -n '1p' "$APP_ROOT/runtime/VERSION")
APP_ROOT=$(CDPATH= cd -- "$APP_ROOT" && pwd)
[ "${#SMOKE_POSTGRES_TOKEN}" -ge 20 ] || {
printf '%s\n' 'Set IMAGEFIND_SMOKE_POSTGRES_TOKEN to a valid nxsir.postgresql enrollment token.' >&2
exit 2
}
mkdir -p "$DATA_ROOT"
DATA_ROOT=$(CDPATH= cd -- "$DATA_ROOT" && pwd)
cleanup() {
IMAGEFIND_PYTHON_PATH="$PYTHON_PATH" TRIM_APPDEST="$APP_ROOT" TRIM_PKGVAR="$DATA_ROOT" TRIM_SERVICE_PORT="$PORT" \
"$CONTROL" stop
rm -f "$COOKIE_JAR" "$BACKUP_FILE"
}
trap cleanup EXIT HUP INT TERM
IMAGEFIND_PYTHON_PATH="$PYTHON_PATH" TRIM_APPDEST="$APP_ROOT" TRIM_PKGVAR="$DATA_ROOT" \
wizard_admin_password="$SMOKE_PASSWORD" wizard_admin_password_confirm="$SMOKE_PASSWORD" \
wizard_postgres_enrollment_token="$SMOKE_POSTGRES_TOKEN" \
"$INSTALL_CALLBACK"
TRIM_PKGVAR="$DATA_ROOT" wizard_direct_access=true wizard_direct_port="$PORT" \
"$CONFIG_CALLBACK"
IMAGEFIND_PYTHON_PATH="$PYTHON_PATH" TRIM_APPDEST="$APP_ROOT" TRIM_PKGVAR="$DATA_ROOT" TRIM_SERVICE_PORT="$PORT" \
"$CONTROL" start
STATUS_BODY=""
ATTEMPT=0
while [ "$ATTEMPT" -lt 90 ]; do
if STATUS_BODY=$(curl -fsS "http://127.0.0.1:$PORT/api/v1/status" 2>/dev/null); then
break
fi
ATTEMPT=$((ATTEMPT + 1))
sleep 1
done
if [ -z "$STATUS_BODY" ]; then
printf 'ImageFind did not become ready; application log follows:\n' >&2
tail -100 "$DATA_ROOT/log/imagefind.log" >&2 || true
exit 1
fi
case "$STATUS_BODY" in
*'"configured":true'*"\"version\":\"$EXPECTED_VERSION\""*) ;;
*) printf 'Unexpected ImageFind status: %s\n' "$STATUS_BODY" >&2; exit 1 ;;
esac
RUNTIME_PYTHON="$DATA_ROOT/runtime/current/bin/python"
RUNTIME_VERSION=$("$RUNTIME_PYTHON" -m imagefind.main --version)
CORE_RUNTIME=$(
LD_LIBRARY_PATH="$APP_ROOT/bin${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" \
XDG_CACHE_HOME="$DATA_ROOT/cache" \
"$RUNTIME_PYTHON" -c \
'import fastapi,httpx,imagefind,psycopg,pydantic,uvicorn; print("core-runtime-ok")'
)
case "$CORE_RUNTIME" in
*'core-runtime-ok'*) ;;
*) printf 'Offline core runtime check failed: %s\n' "$CORE_RUNTIME" >&2; exit 1 ;;
esac
FRONTEND_HEADERS=$(curl -fsS -D - -o /dev/null "http://127.0.0.1:$PORT/" | sed -n '1,8p')
LOGIN_BODY=$(curl -fsS -c "$COOKIE_JAR" -H 'Content-Type: application/json' \
--data "{\"password\":\"$SMOKE_PASSWORD\",\"remember_device\":false}" \
"http://127.0.0.1:$PORT/api/v1/auth/login")
CSRF_TOKEN=$(printf '%s' "$LOGIN_BODY" | sed -n 's/.*"csrf_token":"\([^"]*\)".*/\1/p')
if [ -z "$CSRF_TOKEN" ]; then
printf 'Unable to obtain CSRF token from packaged server.\n' >&2
exit 1
fi
BACKUP_STATUS=$(curl -fsS -b "$COOKIE_JAR" "http://127.0.0.1:$PORT/api/v1/backups/status")
MODEL_STATUS=$(curl -fsS -b "$COOKIE_JAR" "http://127.0.0.1:$PORT/api/v1/models")
case "$MODEL_STATUS" in
*'"runtime_environment":{"state":"missing"'*'"source":"on_demand"'*) ;;
*) printf 'AI runtime was not reported as on-demand: %s\n' "$MODEL_STATUS" >&2; exit 1 ;;
esac
BACKUP_QUEUE=$(curl -fsS -b "$COOKIE_JAR" -H "X-CSRF-Token: $CSRF_TOKEN" -H 'Content-Type: application/json' \
--data "{\"scope\":\"keys\",\"password\":\"$SMOKE_PASSWORD\"}" \
"http://127.0.0.1:$PORT/api/v1/backups")
BACKUP_ID=$(printf '%s' "$BACKUP_QUEUE" | sed -n 's/.*"id":"\([^"]*\)".*/\1/p')
if [ -z "$BACKUP_ID" ]; then
printf 'Unable to queue packaged background backup: %s\n' "$BACKUP_QUEUE" >&2
exit 1
fi
BACKUP_EXPORTS=""
ATTEMPT=0
while [ "$ATTEMPT" -lt 90 ]; do
BACKUP_EXPORTS=$(curl -fsS -b "$COOKIE_JAR" "http://127.0.0.1:$PORT/api/v1/backups")
case "$BACKUP_EXPORTS" in
*"\"id\":\"$BACKUP_ID\""*'"status":"completed"'*) break ;;
*"\"id\":\"$BACKUP_ID\""*'"status":"failed"'*)
printf 'Packaged background backup failed: %s\n' "$BACKUP_EXPORTS" >&2
exit 1
;;
esac
ATTEMPT=$((ATTEMPT + 1))
sleep 1
done
case "$BACKUP_EXPORTS" in
*"\"id\":\"$BACKUP_ID\""*'"status":"completed"'*) ;;
*) printf 'Packaged background backup timed out: %s\n' "$BACKUP_EXPORTS" >&2; exit 1 ;;
esac
curl -fsS -b "$COOKIE_JAR" -o "$BACKUP_FILE" \
"http://127.0.0.1:$PORT/api/v1/backups/$BACKUP_ID/download"
BACKUP_MAGIC=$(dd if="$BACKUP_FILE" bs=1 count=8 2>/dev/null)
if [ "$BACKUP_MAGIC" != "IFBACKUP" ]; then
printf 'Packaged backup has an invalid header.\n' >&2
exit 1
fi
RESTORE_BODY=$(curl -fsS -b "$COOKIE_JAR" -H "X-CSRF-Token: $CSRF_TOKEN" \
-F "file=@$BACKUP_FILE;type=application/vnd.imagefind.backup" \
-F "password=$SMOKE_PASSWORD" -F 'confirmed=true' \
"http://127.0.0.1:$PORT/api/v1/backups/restore")
case "$RESTORE_BODY" in
*'"scope":"keys"'*'"counts"'*'"scan_jobs":[]'*) ;;
*) printf 'Unexpected packaged backup restore result: %s\n' "$RESTORE_BODY" >&2; exit 1 ;;
esac
printf 'status=%s\nruntime=%s\ncore_runtime=%s\nbackup_status=%s\nbackup_restore=%s\nfrontend_headers:\n%s\n' \
"$STATUS_BODY" "$RUNTIME_VERSION" "$CORE_RUNTIME" "$BACKUP_STATUS" "$RESTORE_BODY" \
"$FRONTEND_HEADERS"
+122
View File
@@ -0,0 +1,122 @@
#!/bin/sh
set -eu
PROJECT_ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
OLD_PACKAGE="${1:-$PROJECT_ROOT/dist/imagefind-0.3.9-x86_64.fpk}"
NEW_APP_ROOT="${2:-$PROJECT_ROOT/.build-fnos/imagefind/app}"
PORT="${3:-18766}"
NEW_PACKAGE_ROOT="${4:-$PROJECT_ROOT/fnos}"
PYTHON_PATH="${5:-${IMAGEFIND_PYTHON_PATH:-}}"
PASSWORD="imagefind-upgrade-smoke-password"
[ -f "$OLD_PACKAGE" ] || { printf 'missing previous package: %s\n' "$OLD_PACKAGE" >&2; exit 1; }
[ -f "$NEW_APP_ROOT/runtime/VERSION" ] && [ -d "$NEW_APP_ROOT/runtime/wheels" ] || {
printf 'missing new packaged Python runtime: %s\n' "$NEW_APP_ROOT/runtime" >&2
exit 1
}
OLD_PACKAGE=$(CDPATH= cd -- "$(dirname -- "$OLD_PACKAGE")" && pwd)/$(basename -- "$OLD_PACKAGE")
NEW_APP_ROOT=$(CDPATH= cd -- "$NEW_APP_ROOT" && pwd)
NEW_PACKAGE_ROOT=$(CDPATH= cd -- "$NEW_PACKAGE_ROOT" && pwd)
WORK_ROOT=$(mktemp -d "$PROJECT_ROOT/.upgrade-smoke.XXXXXX")
OLD_PACKAGE_ROOT="$WORK_ROOT/old-package"
ACTIVE_APP_ROOT="$WORK_ROOT/app"
OLD_APP_ROOT="$WORK_ROOT/old-app"
DATA_ROOT="$WORK_ROOT/var"
LIFECYCLE_LOG="$WORK_ROOT/fnos-upgrade.log"
COOKIE_JAR="$WORK_ROOT/cookie.txt"
OLD_CONTROL="$OLD_PACKAGE_ROOT/cmd/main"
NEW_CONTROL="$NEW_PACKAGE_ROOT/cmd/main"
cleanup() {
set +e
if [ -x "$NEW_CONTROL" ]; then
IMAGEFIND_PYTHON_PATH="$PYTHON_PATH" TRIM_APPDEST="$ACTIVE_APP_ROOT" TRIM_PKGVAR="$DATA_ROOT" \
"$NEW_CONTROL" stop >/dev/null 2>&1
fi
if [ -x "$OLD_CONTROL" ]; then
TRIM_APPDEST="$ACTIVE_APP_ROOT" TRIM_PKGVAR="$DATA_ROOT" "$OLD_CONTROL" stop >/dev/null 2>&1
fi
if [ "${IMAGEFIND_KEEP_UPGRADE_SMOKE:-false}" = "true" ]; then
printf 'upgrade smoke workspace preserved at %s\n' "$WORK_ROOT" >&2
return
fi
case "$WORK_ROOT" in
"$PROJECT_ROOT"/.upgrade-smoke.*) rm -rf -- "$WORK_ROOT" ;;
*) printf 'refusing to clear unexpected upgrade smoke path: %s\n' "$WORK_ROOT" >&2 ;;
esac
}
trap cleanup EXIT HUP INT TERM
mkdir -p "$OLD_PACKAGE_ROOT" "$ACTIVE_APP_ROOT"
tar xzf "$OLD_PACKAGE" -C "$OLD_PACKAGE_ROOT" cmd manifest
tar xOf "$OLD_PACKAGE" app.tgz | tar xzf - -C "$ACTIVE_APP_ROOT"
OLD_VERSION=$(sed -n 's/^version[[:space:]]*=[[:space:]]*//p' "$OLD_PACKAGE_ROOT/manifest" | head -n 1)
NEW_VERSION=$(sed -n 's/^version[[:space:]]*=[[:space:]]*//p' "$NEW_PACKAGE_ROOT/manifest" | head -n 1)
[ -n "$OLD_VERSION" ] && [ -n "$NEW_VERSION" ] || { printf 'unable to read package versions\n' >&2; exit 1; }
[ "$OLD_VERSION" != "$NEW_VERSION" ] || { printf 'upgrade smoke requires two different versions\n' >&2; exit 1; }
wait_for_version() {
EXPECTED="$1"
ATTEMPT=0
while [ "$ATTEMPT" -lt 90 ]; do
STATUS=$(curl -fsS "http://127.0.0.1:$PORT/api/v1/status" 2>/dev/null || true)
case "$STATUS" in
*'"configured":true'*'"version":"'"$EXPECTED"'"'*) return 0 ;;
esac
ATTEMPT=$((ATTEMPT + 1))
sleep 1
done
printf 'version %s did not become ready; lifecycle log follows:\n' "$EXPECTED" >&2
tail -100 "$DATA_ROOT/log/imagefind.log" >&2 || true
return 1
}
IMAGEFIND_PYTHON_PATH="$PYTHON_PATH" TRIM_APPDEST="$ACTIVE_APP_ROOT" TRIM_PKGVAR="$DATA_ROOT" \
TRIM_TEMP_LOGFILE="$LIFECYCLE_LOG" \
wizard_admin_password="$PASSWORD" wizard_admin_password_confirm="$PASSWORD" \
"$OLD_PACKAGE_ROOT/cmd/install_callback"
TRIM_PKGVAR="$DATA_ROOT" wizard_direct_access=true wizard_direct_port="$PORT" \
"$OLD_PACKAGE_ROOT/cmd/config_callback"
# A 0.3.15 PyInstaller service can be precisely what is broken on the target
# NAS. Upgrade verification must therefore begin from its persisted data and
# lifecycle callbacks, not require the old HTTP listener to become healthy.
if [ "${IMAGEFIND_SMOKE_START_OLD:-false}" = "true" ]; then
IMAGEFIND_PYTHON_PATH="$PYTHON_PATH" TRIM_APPDEST="$ACTIVE_APP_ROOT" TRIM_PKGVAR="$DATA_ROOT" \
"$OLD_CONTROL" start
wait_for_version "$OLD_VERSION"
fi
printf '%s\n' 'preserve-across-upgrade' >"$DATA_ROOT/data/upgrade-smoke-marker"
IMAGEFIND_PYTHON_PATH="$PYTHON_PATH" TRIM_APPDEST="$ACTIVE_APP_ROOT" TRIM_PKGVAR="$DATA_ROOT" \
"$OLD_CONTROL" stop
TRIM_APPDEST="$ACTIVE_APP_ROOT" TRIM_PKGVAR="$DATA_ROOT" TRIM_TEMP_LOGFILE="$LIFECYCLE_LOG" \
"$NEW_PACKAGE_ROOT/cmd/upgrade_init"
mv "$ACTIVE_APP_ROOT" "$OLD_APP_ROOT"
ln -s "$NEW_APP_ROOT" "$ACTIVE_APP_ROOT"
# Deliberately pass stale mismatched values from the previous wizard.
# The upgrade must preserve the existing password and still succeed.
TRIM_APPDEST="$ACTIVE_APP_ROOT" TRIM_PKGVAR="$DATA_ROOT" TRIM_TEMP_LOGFILE="$LIFECYCLE_LOG" \
IMAGEFIND_PYTHON_PATH="$PYTHON_PATH" \
wizard_admin_password=placeholder wizard_admin_password_confirm=different \
"$NEW_PACKAGE_ROOT/cmd/upgrade_callback"
IMAGEFIND_PYTHON_PATH="$PYTHON_PATH" TRIM_APPDEST="$ACTIVE_APP_ROOT" TRIM_PKGVAR="$DATA_ROOT" \
"$NEW_CONTROL" start
wait_for_version "$NEW_VERSION"
[ "$(sed -n '1p' "$DATA_ROOT/data/upgrade-smoke-marker")" = "preserve-across-upgrade" ] || {
printf 'upgrade did not preserve application data\n' >&2
exit 1
}
LOGIN=$(curl -fsS -c "$COOKIE_JAR" -H 'Content-Type: application/json' \
--data "{\"password\":\"$PASSWORD\",\"remember_device\":false}" \
"http://127.0.0.1:$PORT/api/v1/auth/login")
case "$LOGIN" in
*'"csrf_token"'*) ;;
*) printf 'upgrade did not preserve the administrator password\n' >&2; exit 1 ;;
esac
grep -q '升级准备完成' "$LIFECYCLE_LOG"
grep -q '升级数据检查完成' "$LIFECYCLE_LOG"
printf 'fnOS upgrade smoke passed: %s -> %s; data, access settings and password preserved\n' \
"$OLD_VERSION" "$NEW_VERSION"
+200
View File
@@ -0,0 +1,200 @@
#!/bin/sh
set -eu
PROJECT_ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
PYTHON_BIN="${PYTHON:-$PROJECT_ROOT/.venv/bin/python}"
FFMPEG_BIN="${FFMPEG:-$PROJECT_ROOT/vendor/ffmpeg}"
FFPROBE_BIN="${FFPROBE:-$PROJECT_ROOT/vendor/ffprobe}"
PORT="${1:-18766}"
SMOKE_ROOT=$(mktemp -d "${TMPDIR:-/tmp}/imagefind-http-smoke.XXXXXX")
DATA_ROOT="$SMOKE_ROOT/data"
MEDIA_ROOT="$SMOKE_ROOT/media"
COOKIE_JAR="$SMOKE_ROOT/cookies"
SERVER_LOG="$SMOKE_ROOT/server.log"
PASSWORD="imagefind-http-smoke-password"
SERVER_PID=""
cleanup() {
STATUS=$?
trap - EXIT HUP INT TERM
if [ -n "$SERVER_PID" ] && kill -0 "$SERVER_PID" 2>/dev/null; then
kill "$SERVER_PID" 2>/dev/null || true
wait "$SERVER_PID" 2>/dev/null || true
fi
if [ "$STATUS" -ne 0 ]; then
printf '%s\n' 'HTTP smoke failed; application log follows:' >&2
tail -100 "$SERVER_LOG" >&2 || true
cp -p "$SERVER_LOG" "$PROJECT_ROOT/dist/http-smoke-failure.log" 2>/dev/null || true
fi
rm -rf -- "$SMOKE_ROOT"
exit "$STATUS"
}
trap cleanup EXIT HUP INT TERM
mkdir -p "$DATA_ROOT" "$MEDIA_ROOT"
"$FFMPEG_BIN" -hide_banner -loglevel error -f lavfi -i "testsrc=size=320x180:rate=24" \
-t 2 -pix_fmt yuv420p -c:v libx264 -movflags +faststart "$SMOKE_ROOT/ABC-999.mp4"
env PYTHONPATH="$PROJECT_ROOT/backend" \
IMAGEFIND_DATA_DIR="$DATA_ROOT" \
IMAGEFIND_FRONTEND_DIR="$PROJECT_ROOT/frontend/dist" \
IMAGEFIND_HOST="127.0.0.1" \
IMAGEFIND_PORT="$PORT" \
IMAGEFIND_UPLOAD_RESERVE_GB="0" \
IMAGEFIND_RESOURCE_DISK_RESERVE_GB="0.5" \
IMAGEFIND_EMBEDDING_BACKEND="hash" \
IMAGEFIND_FFMPEG_PATH="$FFMPEG_BIN" \
IMAGEFIND_FFPROBE_PATH="$FFPROBE_BIN" \
"$PYTHON_BIN" -m imagefind.main >"$SERVER_LOG" 2>&1 &
SERVER_PID=$!
ATTEMPT=0
until curl -fsS "http://127.0.0.1:$PORT/api/v1/status" >/dev/null 2>&1; do
ATTEMPT=$((ATTEMPT + 1))
if [ "$ATTEMPT" -ge 60 ]; then
printf '%s\n' 'ImageFind HTTP service did not become ready.' >&2
exit 1
fi
sleep 1
done
SETUP=$(curl -fsS -c "$COOKIE_JAR" -H 'Content-Type: application/json' \
-d "{\"password\":\"$PASSWORD\"}" "http://127.0.0.1:$PORT/api/v1/setup")
CSRF=$(printf '%s' "$SETUP" | "$PYTHON_BIN" -c 'import json,sys; print(json.load(sys.stdin)["csrf_token"])')
SOURCE=$(curl -fsS -b "$COOKIE_JAR" -H 'Content-Type: application/json' -H "X-CSRF-Token: $CSRF" \
-d "{\"name\":\"HTTP Smoke\",\"path\":\"$MEDIA_ROOT\"}" \
"http://127.0.0.1:$PORT/api/v1/sources/local")
SOURCE_ID=$(printf '%s' "$SOURCE" | "$PYTHON_BIN" -c 'import json,sys; print(json.load(sys.stdin)["id"])')
curl -fsS -b "$COOKIE_JAR" -H 'Content-Type: application/json' -H "X-CSRF-Token: $CSRF" \
-X PATCH -d '{"writable":true}' \
"http://127.0.0.1:$PORT/api/v1/sources/$SOURCE_ID/writable" >/dev/null
SIZE=$(wc -c <"$SMOKE_ROOT/ABC-999.mp4" | tr -d ' ')
UPLOAD=$(curl -fsS -b "$COOKIE_JAR" -H 'Content-Type: application/json' -H "X-CSRF-Token: $CSRF" \
-d "{\"source_id\":\"$SOURCE_ID\",\"relative_path\":\"imports\",\"filename\":\"ABC-999.mp4\",\"size_bytes\":$SIZE}" \
"http://127.0.0.1:$PORT/api/v1/uploads")
UPLOAD_ID=$(printf '%s' "$UPLOAD" | "$PYTHON_BIN" -c 'import json,sys; print(json.load(sys.stdin)["id"])')
DIGEST=$(sha256sum "$SMOKE_ROOT/ABC-999.mp4" | cut -d ' ' -f 1)
curl -fsS -b "$COOKIE_JAR" -H 'Content-Type: application/octet-stream' -H "X-CSRF-Token: $CSRF" \
-H "X-Chunk-SHA256: $DIGEST" -X PUT --data-binary "@$SMOKE_ROOT/ABC-999.mp4" \
"http://127.0.0.1:$PORT/api/v1/uploads/$UPLOAD_ID/chunks/0" >/dev/null
curl -fsS -b "$COOKIE_JAR" -H 'Content-Type: application/json' -H "X-CSRF-Token: $CSRF" \
-d '{}' "http://127.0.0.1:$PORT/api/v1/uploads/$UPLOAD_ID/complete" >/dev/null
ATTEMPT=0
while :; do
UPLOADS=$(curl -fsS -b "$COOKIE_JAR" "http://127.0.0.1:$PORT/api/v1/uploads")
STATE=$(printf '%s' "$UPLOADS" | "$PYTHON_BIN" -c \
'import json,sys; items=json.load(sys.stdin); uid=sys.argv[1]; print(next(x["status"] for x in items if x["id"] == uid))' \
"$UPLOAD_ID")
[ "$STATE" = "completed" ] && break
[ "$STATE" = "failed" ] && { printf '%s\n' "$UPLOADS" >&2; exit 1; }
ATTEMPT=$((ATTEMPT + 1))
[ "$ATTEMPT" -lt 60 ] || { printf '%s\n' "Upload stayed in state $STATE" >&2; exit 1; }
sleep 1
done
VIDEOS=$(curl -fsS -b "$COOKIE_JAR" "http://127.0.0.1:$PORT/api/v1/videos")
VIDEO_ID=$(printf '%s' "$VIDEOS" | "$PYTHON_BIN" -c \
'import json,sys; print(next(x["id"] for x in json.load(sys.stdin) if x["display_name"] == "ABC-999.mp4"))')
curl -fsS -b "$COOKIE_JAR" "http://127.0.0.1:$PORT/api/v1/videos/$VIDEO_ID/download" \
-o "$SMOKE_ROOT/download.mp4"
cmp "$SMOKE_ROOT/ABC-999.mp4" "$SMOKE_ROOT/download.mp4"
TOKEN_RESPONSE=$(curl -fsS -b "$COOKIE_JAR" -H 'Content-Type: application/json' \
-H "X-CSRF-Token: $CSRF" -d '{"name":"HTTP WebDAV smoke"}' \
"http://127.0.0.1:$PORT/api/v1/tokens")
API_TOKEN=$(printf '%s' "$TOKEN_RESPONSE" | "$PYTHON_BIN" -c \
'import json,sys; print(json.load(sys.stdin)["token"])')
curl -fsS -b "$COOKIE_JAR" -H 'Content-Type: application/json' -H "X-CSRF-Token: $CSRF" \
-X PATCH \
-d "{\"enabled\":true,\"source_id\":\"$SOURCE_ID\",\"relative_path\":\"webdav-smoke\"}" \
"http://127.0.0.1:$PORT/api/v1/webdav/config" >/dev/null
DAV_SIZE=$SIZE
DAV_SPLIT=$((DAV_SIZE / 2))
DAV_REMAINING=$((DAV_SIZE - DAV_SPLIT))
dd if="$SMOKE_ROOT/ABC-999.mp4" of="$SMOKE_ROOT/dav-first.bin" bs=1 count="$DAV_SPLIT" 2>/dev/null
dd if="$SMOKE_ROOT/ABC-999.mp4" of="$SMOKE_ROOT/dav-second.bin" bs=1 skip="$DAV_SPLIT" 2>/dev/null
DAV_URL="http://127.0.0.1:$PORT/webdav/DAV-RESUME.mp4"
FIRST_STATUS=$(curl -sS -u "imagefind:$API_TOKEN" -X PUT \
-H "Content-Range: bytes 0-$((DAV_SPLIT - 1))/$DAV_SIZE" \
-H "Content-Length: $DAV_SPLIT" --data-binary "@$SMOKE_ROOT/dav-first.bin" \
-D "$SMOKE_ROOT/dav-first.headers" -o "$SMOKE_ROOT/dav-first.body" -w '%{http_code}' "$DAV_URL")
[ "$FIRST_STATUS" = "204" ]
FIRST_OFFSET=$(tr -d '\r' <"$SMOKE_ROOT/dav-first.headers" | awk \
'tolower($1)=="upload-offset:" {print $2; exit}')
[ "$FIRST_OFFSET" = "$DAV_SPLIT" ]
HEAD_STATUS=$(curl -sS -u "imagefind:$API_TOKEN" -I \
-D "$SMOKE_ROOT/dav-head.headers" -o /dev/null -w '%{http_code}' "$DAV_URL")
[ "$HEAD_STATUS" = "200" ]
HEAD_OFFSET=$(tr -d '\r' <"$SMOKE_ROOT/dav-head.headers" | awk \
'tolower($1)=="upload-offset:" {print $2; exit}')
[ "$HEAD_OFFSET" = "$DAV_SPLIT" ]
SECOND_STATUS=$(curl -sS -u "imagefind:$API_TOKEN" -X PUT \
-H "Content-Range: bytes $DAV_SPLIT-$((DAV_SIZE - 1))/$DAV_SIZE" \
-H "Content-Length: $DAV_REMAINING" --data-binary "@$SMOKE_ROOT/dav-second.bin" \
-D "$SMOKE_ROOT/dav-second.headers" -o "$SMOKE_ROOT/dav-second.body" -w '%{http_code}' "$DAV_URL")
[ "$SECOND_STATUS" = "204" ]
DAV_UPLOAD_ID=$(tr -d '\r' <"$SMOKE_ROOT/dav-second.headers" | awk \
'tolower($1)=="x-imagefind-upload-id:" {print $2; exit}')
[ -n "$DAV_UPLOAD_ID" ]
ATTEMPT=0
while :; do
UPLOADS=$(curl -fsS -b "$COOKIE_JAR" "http://127.0.0.1:$PORT/api/v1/uploads")
STATE=$(printf '%s' "$UPLOADS" | "$PYTHON_BIN" -c \
'import json,sys; items=json.load(sys.stdin); uid=sys.argv[1]; print(next(x["status"] for x in items if x["id"] == uid))' \
"$DAV_UPLOAD_ID")
[ "$STATE" = "completed" ] && break
[ "$STATE" = "failed" ] && { printf '%s\n' "$UPLOADS" >&2; exit 1; }
ATTEMPT=$((ATTEMPT + 1))
[ "$ATTEMPT" -lt 60 ] || { printf '%s\n' "WebDAV upload stayed in state $STATE" >&2; exit 1; }
sleep 1
done
cmp "$SMOKE_ROOT/ABC-999.mp4" "$MEDIA_ROOT/webdav-smoke/DAV-RESUME.mp4"
DEDUP_STATUS=$(curl -sS -u "imagefind:$API_TOKEN" -X PUT \
-H "Content-Length: $DAV_SIZE" -H "X-Content-SHA256: $DIGEST" \
--data-binary "@$SMOKE_ROOT/ABC-999.mp4" -D "$SMOKE_ROOT/dav-dedup.headers" \
-o "$SMOKE_ROOT/dav-dedup.body" -w '%{http_code}' "$DAV_URL")
[ "$DEDUP_STATUS" = "204" ]
DEDUPLICATED=$(tr -d '\r' <"$SMOKE_ROOT/dav-dedup.headers" | awk \
'tolower($1)=="x-imagefind-deduplicated:" {print tolower($2); exit}')
[ "$DEDUPLICATED" = "true" ]
if [ "${IMAGEFIND_RUN_WEBDAV_STRESS:-0}" = "1" ]; then
if [ "${IMAGEFIND_RUN_WEBDAV_STRESS_FULL:-0}" = "1" ]; then
"$PYTHON_BIN" "$PROJECT_ROOT/scripts/stress-webdav.py" \
--base-url "http://127.0.0.1:$PORT" --token "$API_TOKEN" --workers 6 \
--timeout 7200 --media-prefix "$SMOKE_ROOT/ABC-999.mp4" --full
elif [ "${IMAGEFIND_RUN_WEBDAV_STRESS_LARGE:-0}" = "1" ]; then
"$PYTHON_BIN" "$PROJECT_ROOT/scripts/stress-webdav.py" \
--base-url "http://127.0.0.1:$PORT" --token "$API_TOKEN" --workers 4 \
--timeout 1800 --media-prefix "$SMOKE_ROOT/ABC-999.mp4" --large
else
"$PYTHON_BIN" "$PROJECT_ROOT/scripts/stress-webdav.py" \
--base-url "http://127.0.0.1:$PORT" --token "$API_TOKEN" --workers 4 \
--timeout 600 --media-prefix "$SMOKE_ROOT/ABC-999.mp4"
fi
fi
RANGE_STATUS=$(curl -sS -b "$COOKIE_JAR" -H 'Range: bytes=128-1023' \
-o "$SMOKE_ROOT/range.bin" -w '%{http_code}' \
"http://127.0.0.1:$PORT/api/v1/videos/$VIDEO_ID/stream")
[ "$RANGE_STATUS" = "206" ]
dd if="$SMOKE_ROOT/ABC-999.mp4" of="$SMOKE_ROOT/expected-range.bin" bs=1 skip=128 count=896 2>/dev/null
cmp "$SMOKE_ROOT/expected-range.bin" "$SMOKE_ROOT/range.bin"
PREVIEW=$(curl -fsS -b "$COOKIE_JAR" -H 'Content-Type: application/json' -H "X-CSRF-Token: $CSRF" \
-d '{"start_ms":0}' "http://127.0.0.1:$PORT/api/v1/videos/$VIDEO_ID/preview")
PLAYLIST_URL=$(printf '%s' "$PREVIEW" | "$PYTHON_BIN" -c \
'import json,sys; print(json.load(sys.stdin)["playlist_url"])')
PLAYLIST=$(curl -fsS -b "$COOKIE_JAR" "http://127.0.0.1:$PORT$PLAYLIST_URL")
case "$PLAYLIST" in
*'#EXTM3U'*'segment-00000.ts'*) ;;
*) printf '%s\n' 'HLS playlist did not contain the first segment.' >&2; exit 1 ;;
esac
printf 'upload=%s video=%s bytes=%s range=206 preview=ok webdav_resume=%s webdav_dedup=true sha256=%s\n' \
"$UPLOAD_ID" "$VIDEO_ID" "$SIZE" "$DAV_UPLOAD_ID" "$DIGEST"
+345
View File
@@ -0,0 +1,345 @@
#!/usr/bin/env node
import fs from "node:fs";
import path from "node:path";
import { createRequire } from "node:module";
const require = createRequire(new URL("../frontend/package.json", import.meta.url));
const { chromium } = require("playwright");
const baseUrl = (process.env.IMAGEFIND_LIVE_URL || "").replace(/\/$/, "");
const password = process.env.IMAGEFIND_LIVE_PASSWORD || "";
const tokenFile = process.env.IMAGEFIND_LIVE_TOKEN_FILE || "";
const apiToken = tokenFile ? fs.readFileSync(tokenFile, "utf8").trim() : "";
const outputDir = path.resolve(process.env.IMAGEFIND_LIVE_OUTPUT || "dist/live-ui-audit");
if (!baseUrl || (!password && !apiToken)) {
throw new Error("IMAGEFIND_LIVE_URL and either IMAGEFIND_LIVE_PASSWORD or IMAGEFIND_LIVE_TOKEN_FILE are required");
}
fs.mkdirSync(outputDir, { recursive: true });
const auditPages = ["search", "library", "series", "uploads", "tags", "actors", "settings"];
const mobileMenuNames = {
library: "资料库",
series: "合集",
uploads: "上传中心",
tags: "分类与标签",
actors: "演员与人物",
settings: "设置",
};
const desktopSelectors = {
library: ".nav-item-3",
series: ".nav-item-4",
uploads: ".sidebar nav > .upload-status",
tags: ".nav-item-6",
actors: ".nav-item-7",
settings: ".nav-item-8",
};
async function login(page, name) {
await page.goto(baseUrl, { waitUntil: "domcontentloaded", timeout: 30_000 });
await page.waitForTimeout(1_000);
const passwordInput = page.getByLabel("管理员密码");
if (await passwordInput.isVisible().catch(() => false)) {
await passwordInput.fill(password);
await page.getByRole("button", { name: "登录", exact: true }).click();
}
try {
await page.locator(".app-shell").waitFor({ state: "visible", timeout: 30_000 });
} catch (error) {
await page.screenshot({ path: path.join(outputDir, `${name}-login-error.png`), fullPage: true });
const text = (await page.locator("body").innerText().catch(() => "")).slice(0, 2_000);
throw new Error(`login did not reach the application: url=${page.url()} body=${JSON.stringify(text)}`, {
cause: error,
});
}
await page.waitForTimeout(2_000);
}
async function resetHome(page) {
await page.evaluate(() => {
history.replaceState(
{ ...history.state, imagefindNavigation: { kind: "page", page: "home", scrollY: 0 } },
"",
);
});
await page.reload({ waitUntil: "domcontentloaded", timeout: 30_000 });
await page.locator(".app-shell").waitFor({ timeout: 30_000 });
}
function attachNetworkAudit(page, errors) {
const issues = [];
let allowUnauthenticatedProbe = true;
page.on("response", response => {
if (response.status() < 400) return;
const request = response.request();
const url = new URL(response.url());
const expectedProbe = allowUnauthenticatedProbe
&& response.status() === 401
&& request.method() === "GET"
&& url.pathname === "/api/v1/auth/me";
if (expectedProbe) return;
const issue = {
method: request.method(),
status: response.status(),
path: url.pathname,
resourceType: request.resourceType(),
};
issues.push(issue);
errors.push(`response: ${issue.method} ${issue.status} ${issue.path} [${issue.resourceType}]`);
});
page.on("requestfailed", request => {
const url = new URL(request.url());
const issue = {
method: request.method(),
status: null,
path: url.pathname,
resourceType: request.resourceType(),
failure: request.failure()?.errorText || "unknown network failure",
};
issues.push(issue);
errors.push(`requestfailed: ${issue.method} ${issue.path} [${issue.resourceType}] ${issue.failure}`);
});
return {
issues,
markAuthenticated() {
allowUnauthenticatedProbe = false;
},
};
}
async function inspect(browser, viewport, name) {
const context = await browser.newContext({ viewport, locale: "zh-CN" });
if (apiToken) {
const origin = new URL(baseUrl).origin;
await context.route("**/*", async route => {
const url = new URL(route.request().url());
if (url.origin !== origin || !url.pathname.includes("/api/")) {
await route.continue();
return;
}
await route.continue({
headers: { ...route.request().headers(), authorization: `Bearer ${apiToken}` },
});
});
}
const page = await context.newPage();
const errors = [];
page.on("pageerror", error => errors.push(`pageerror: ${error.message}`));
page.on("console", message => {
if (message.type() === "error") errors.push(`console: ${message.text()}`);
});
const networkAudit = attachNetworkAudit(page, errors);
await login(page, name);
networkAudit.markAuthenticated();
const home = await page.evaluate(() => ({
scrollWidth: document.documentElement.scrollWidth,
clientWidth: document.documentElement.clientWidth,
scrollHeight: document.documentElement.scrollHeight,
clientHeight: document.documentElement.clientHeight,
}));
const cards = page.locator("article.video-card");
const cardCount = await cards.count();
const coverErrors = await page.locator(".thumb-placeholder.error").count();
await page.screenshot({ path: path.join(outputDir, `${name}-home.png`), fullPage: true });
const pageAudits = [];
for (const destination of auditPages) {
await resetHome(page);
if (destination === "search") {
if (viewport.width <= 720) {
await page.getByRole("button", { name: "搜索", exact: true }).first().click();
} else {
await page.locator(".nav-item-2").click();
}
} else if (viewport.width <= 720) {
await page.locator(".mobile-more-button").click();
await page.locator(".mobile-more-menu")
.getByRole("button", { name: new RegExp(mobileMenuNames[destination]) })
.first()
.click();
} else {
await page.locator(desktopSelectors[destination]).click();
}
await page.locator("main .page").waitFor({ state: "visible", timeout: 20_000 });
await page.waitForTimeout(500);
const geometry = await page.evaluate(() => {
const content = document.querySelector(".content");
const pagination = document.querySelector(".transfer-pagination");
return {
documentWidth: document.documentElement.scrollWidth,
viewportWidth: document.documentElement.clientWidth,
bodyWidth: document.body.scrollWidth,
contentWidth: content?.scrollWidth || 0,
contentClientWidth: content?.clientWidth || 0,
contentHeight: content?.scrollHeight || 0,
contentClientHeight: content?.clientHeight || 0,
transferRows: document.querySelectorAll(".transfer-list > article").length,
pagination: pagination?.textContent?.replace(/\s+/g, " ").trim() || null,
paginationButtonHeights: pagination
? [...pagination.querySelectorAll("button")].map(button => Math.round(button.getBoundingClientRect().height))
: [],
};
});
let paginationNavigation = null;
if (destination === "uploads" && geometry.pagination) {
const before = await page.locator(".transfer-list > article").first().innerText();
const next = page.locator(".transfer-pagination").getByRole("button", { name: "下一页" });
if (await next.isEnabled()) {
await next.click();
await page.waitForFunction(() => document.querySelector(".transfer-pagination")?.textContent?.includes("第 2 /"));
const after = await page.locator(".transfer-list > article").first().innerText();
paginationNavigation = {
changed: before !== after,
rows: await page.locator(".transfer-list > article").count(),
text: (await page.locator(".transfer-pagination").innerText()).replace(/\s+/g, " ").trim(),
};
}
}
pageAudits.push({ destination, ...geometry, paginationNavigation });
await page.screenshot({
path: path.join(outputDir, `${name}-${destination}.png`),
// Hundreds of historical tasks can make this page >17,000px tall. The
// UI geometry is sampled above; keep the screenshot bounded to avoid a
// Chromium bitmap allocation crash on low-memory test hosts.
fullPage: destination !== "uploads",
});
}
await resetHome(page);
let scrollY = 0;
let mobileMenu = null;
if (viewport.width <= 720) {
await page.evaluate(() => window.scrollTo(0, document.documentElement.scrollHeight));
await page.waitForTimeout(500);
scrollY = await page.evaluate(() => window.scrollY);
await page.evaluate(() => window.scrollTo(0, 0));
await page.locator(".mobile-more-button").click();
const sheet = page.locator(".mobile-more-menu");
await sheet.waitFor({ state: "visible" });
// Visibility is reported at the first frame of the slide-in transition.
// Geometry sampled there includes the temporary translateY and used to be
// misreported as a sheet extending below the viewport.
await page.waitForTimeout(300);
mobileMenu = await sheet.evaluate(element => {
const box = element.getBoundingClientRect();
const buttons = [...element.querySelectorAll(".mobile-more-item")].map(button => {
const rect = button.getBoundingClientRect();
const style = getComputedStyle(button);
return {
text: button.textContent?.trim(),
contentCenterDelta: Math.round(Math.max(0, ...[...button.children].map(child => {
const childRect = child.getBoundingClientRect();
return Math.abs(rect.left + rect.width / 2 - childRect.left - childRect.width / 2);
}))),
background: style.backgroundColor,
};
});
return {
top: Math.round(box.top),
bottom: Math.round(box.bottom),
viewportBottom: window.innerHeight,
transform: getComputedStyle(element).transform,
buttons,
};
});
await page.screenshot({ path: path.join(outputDir, `${name}-more.png`), fullPage: true });
await page.getByRole("button", { name: "关闭", exact: true }).click();
await page.locator(".mobile-more-menu").waitFor({ state: "hidden" });
await page.locator(".mobile-profile-button").click();
await page.locator(".profile-page").waitFor({ state: "visible" });
await page.screenshot({ path: path.join(outputDir, `${name}-profile.png`), fullPage: true });
await page.getByRole("button", { name: "首页", exact: true }).click();
}
let player = null;
if (cardCount) {
const preferred = cards.filter({ hasText: "e2e-positive" });
await ((await preferred.count()) ? preferred.first() : cards.first()).click();
await page.locator(".player-page").waitFor({ state: "visible", timeout: 20_000 });
await page.waitForTimeout(2_000);
const status = await page.locator(".player-center-status").textContent().catch(() => null);
const stage = page.locator(".custom-player");
if ((await stage.getAttribute("class"))?.includes("controls-hidden")) {
await page.locator(".player-gesture-surface").click({ position: { x: 12, y: 12 } });
await page.waitForTimeout(350);
}
const markerButton = page.getByRole("button", { name: "视频时间点" });
const hitTest = await markerButton.evaluate(button => {
const rect = button.getBoundingClientRect();
const x = rect.left + rect.width / 2;
const y = rect.top + rect.height / 2;
const controls = button.closest(".player-controls");
const controlsStyle = controls ? getComputedStyle(controls) : null;
return {
stageClass: button.closest(".custom-player")?.className,
controls: controlsStyle ? {
opacity: controlsStyle.opacity,
pointerEvents: controlsStyle.pointerEvents,
zIndex: controlsStyle.zIndex,
} : null,
stack: document.elementsFromPoint(x, y).slice(0, 6).map(element => ({
tag: element.tagName,
className: element.className,
pointerEvents: getComputedStyle(element).pointerEvents,
zIndex: getComputedStyle(element).zIndex,
})),
};
});
await markerButton.click();
await page.locator(".player-marker-drawer").waitFor({ state: "visible" });
await page.screenshot({ path: path.join(outputDir, `${name}-player.png`), fullPage: true });
player = {
status: status?.trim() || "ready",
hitTest,
markerDrawer: await page.locator(".player-marker-drawer").isVisible(),
progressVisible: await page.getByLabel("播放进度").isVisible(),
horizontalOverflow: await page.evaluate(
() => document.documentElement.scrollWidth > document.documentElement.clientWidth,
),
};
}
await context.close();
return {
name,
viewport,
home,
cardCount,
coverErrors,
pageAudits,
scrollY,
mobileMenu,
player,
networkIssues: networkAudit.issues,
errors: [...new Set(errors)].slice(0, 20),
};
}
const cases = [
[{ width: 320, height: 780 }, "mobile-320"],
[{ width: 390, height: 844 }, "mobile-390"],
[{ width: 768, height: 900 }, "tablet-768"],
[{ width: 1024, height: 900 }, "desktop-1024"],
[{ width: 1440, height: 900 }, "desktop-1440"],
];
const requested = new Set(
(process.env.IMAGEFIND_LIVE_VIEWPORTS || "")
.split(",")
.map(value => value.trim())
.filter(Boolean),
);
const results = [];
for (const [viewport, name] of cases) {
if (requested.size && !requested.has(name)) continue;
// Chromium retains decoded full-page screenshots in process caches. A fresh
// process per viewport keeps the five-size audit bounded on small CI hosts.
const browser = await chromium.launch({ headless: true });
try {
results.push(await inspect(browser, viewport, name));
fs.writeFileSync(path.join(outputDir, "report.json"), JSON.stringify(results, null, 2));
} finally {
await browser.close();
}
}
console.log(JSON.stringify(results, null, 2));
+376
View File
@@ -0,0 +1,376 @@
#!/usr/bin/env python3
"""Orthogonal ImageFind WebDAV upload stress test with isolated cleanup.
The default profile is CI-friendly. Pass ``--full`` to execute the requested
10 MiB / 1 GiB / 5 GiB singles, ten-file batch and fifty 10 MiB-file batch.
The API token is used for both WebDAV Basic authentication and cleanup.
"""
from __future__ import annotations
import argparse
import base64
import concurrent.futures
import hashlib
import os
import sys
import threading
import time
import uuid
from collections.abc import Iterator
from dataclasses import dataclass
from pathlib import Path
from urllib.parse import quote
import httpx
MIB = 1024**2
GIB = 1024**3
CHUNK = MIB
@dataclass(frozen=True)
class Case:
virtual_path: str
size: int
resume: bool = False
@dataclass
class TimedBody:
name: str
size: int
prefix: bytes = b""
start: int = 0
end: int | None = None
finished_at: float | None = None
def __iter__(self) -> Iterator[bytes]:
yield from blocks(
self.name,
self.size,
prefix=self.prefix,
start=self.start,
end=self.end,
)
# httpx resumes the iterator after it has handed the final request
# bytes to the transport. The remaining interval is server response
# latency, including any accidental post-PUT whole-file work.
self.finished_at = time.monotonic()
def blocks(
name: str,
size: int,
*,
prefix: bytes = b"",
start: int = 0,
end: int | None = None,
) -> Iterator[bytes]:
end = size if end is None else end
seed = hashlib.sha256(name.encode()).digest()
block = (seed * (CHUNK // len(seed) + 1))[:CHUNK]
position = start
while position < end:
if position < len(prefix):
length = min(len(prefix) - position, end - position)
yield prefix[position : position + length]
position += length
continue
offset = position % CHUNK
length = min(CHUNK - offset, end - position)
yield block[offset : offset + length]
position += length
def encoded_path(path: str) -> str:
return "/".join(quote(part, safe="") for part in path.split("/"))
class StressRun:
def __init__(
self,
base_url: str,
token: str,
full: bool,
large: bool,
workers: int,
timeout: float,
probe_timeout: float,
media_prefix: bytes = b"",
):
self.base_url = base_url.rstrip("/")
self.token = token
self.full = full
self.large = large
self.workers = workers
self.timeout = timeout
self.probe_timeout = probe_timeout
self.media_prefix = media_prefix
self.run_id = f"ifstress-{uuid.uuid4().hex[:10]}"
basic = base64.b64encode(f"imagefind:{token}".encode()).decode()
self.dav_headers = {"Authorization": f"Basic {basic}"}
self.api_headers = {"Authorization": f"Bearer {token}"}
self.upload_ids: set[str] = set()
self.request_timeout = httpx.Timeout(connect=15, write=60, read=120, pool=30)
self.probe_latencies: list[float] = []
self.probe_errors: list[str] = []
def probe(self, stopping: threading.Event) -> None:
timeout = httpx.Timeout(connect=3, write=3, read=self.probe_timeout, pool=3)
while not stopping.is_set():
started = time.monotonic()
try:
response = httpx.get(f"{self.base_url}/api/v1/status", timeout=timeout)
response.raise_for_status()
self.probe_latencies.append(time.monotonic() - started)
except Exception as exc:
if len(self.probe_errors) < 10:
self.probe_errors.append(f"{type(exc).__name__}: {exc}")
stopping.wait(0.25)
def cases(self) -> list[Case]:
if self.large:
return [
Case(f"{self.run_id}/large-{index}.mp4", 256 * MIB)
for index in range(4)
]
single_sizes = [10 * MIB, GIB, 5 * GIB] if self.full else [10 * MIB]
cases = [
Case(f"{self.run_id}/single-{size}.mp4", size, resume=index == 0)
for index, size in enumerate(single_sizes)
]
cases.append(Case(f"root-{self.run_id}.mp4", 10 * MIB if self.full else MIB))
for index in range(10):
depth = (1, 5, 10)[index % 3]
groups = "/".join(f"level-{level}" for level in range(1, depth + 1))
size = 10 * MIB if self.full else MIB
cases.append(Case(f"{self.run_id}/{groups}/{self.run_id}-ten-{index:02d}.mp4", size))
for index in range(50):
size = 10 * MIB if self.full else 256 * 1024
cases.append(Case(f"{self.run_id}/fifty/{self.run_id}-fifty-{index:02d}.mp4", size))
return cases
def put(self, case: Case) -> tuple[str, int, str | None, float]:
url = f"{self.base_url}/webdav/{encoded_path(case.virtual_path)}"
headers = dict(self.dav_headers)
upload_id = None
response_body: TimedBody
with httpx.Client(timeout=self.request_timeout) as client:
if case.resume:
split = case.size // 2
first_body = TimedBody(
case.virtual_path, case.size, self.media_prefix, end=split
)
first = client.put(
url,
headers={
**headers,
"Content-Length": str(split),
"Content-Range": f"bytes 0-{split - 1}/{case.size}",
},
content=first_body,
)
first.raise_for_status()
if first.headers.get("upload-offset") != str(split):
raise RuntimeError(f"{case.virtual_path}: first offset was not persisted")
head = client.head(url, headers=headers)
head.raise_for_status()
if head.headers.get("upload-offset") != str(split):
raise RuntimeError(f"{case.virtual_path}: HEAD did not report the partial offset")
response_body = TimedBody(
case.virtual_path, case.size, self.media_prefix, start=split
)
response = client.put(
url,
headers={
**headers,
"Content-Length": str(case.size - split),
"Content-Range": f"bytes {split}-{case.size - 1}/{case.size}",
},
content=response_body,
)
else:
response_body = TimedBody(case.virtual_path, case.size, self.media_prefix)
response = client.put(
url,
headers={**headers, "Content-Length": str(case.size)},
content=response_body,
)
response.raise_for_status()
upload_id = response.headers.get("x-imagefind-upload-id")
response_latency = time.monotonic() - (response_body.finished_at or time.monotonic())
return case.virtual_path, case.size, upload_id, response_latency
def wait(self, expected: int) -> None:
deadline = time.monotonic() + self.timeout
while time.monotonic() < deadline:
response = httpx.get(
f"{self.base_url}/api/v1/uploads?limit=500",
headers=self.api_headers,
timeout=30,
)
response.raise_for_status()
rows = [row for row in response.json() if row.get("id") in self.upload_ids]
failures = [row for row in rows if row.get("status") == "failed"]
if failures:
raise RuntimeError(f"upload failures: {failures}")
if len(rows) == expected and all(row.get("status") == "completed" for row in rows):
return
time.sleep(2)
raise TimeoutError("uploads did not complete before the stress timeout")
def upload_rows(self) -> list[dict]:
response = httpx.get(
f"{self.base_url}/api/v1/uploads?limit=500",
headers=self.api_headers,
timeout=self.request_timeout,
)
response.raise_for_status()
return response.json()
def verify_dedup(self, case: Case) -> None:
url = f"{self.base_url}/webdav/{encoded_path(case.virtual_path)}"
before = {row["id"] for row in self.upload_rows()}
body = TimedBody(case.virtual_path, case.size, self.media_prefix)
response = httpx.put(
url,
headers={
**self.dav_headers,
"Content-Length": str(case.size),
},
content=body,
timeout=self.request_timeout,
)
response.raise_for_status()
if response.headers.get("x-imagefind-deduplicated") != "true":
raise RuntimeError("full retry after a simulated lost response was not deduplicated")
after = {row["id"] for row in self.upload_rows()}
if after != before:
raise RuntimeError("full retry created another upload task")
latency = time.monotonic() - (body.finished_at or time.monotonic())
print(
f"deduplicated lost-response retry response_latency={latency:.3f}s",
flush=True,
)
def cleanup(self) -> None:
try:
videos = httpx.get(
f"{self.base_url}/api/v1/videos?limit=500", headers=self.api_headers, timeout=30
).json()
for video in videos:
if self.run_id in str(video.get("display_name")) or self.run_id in str(video.get("source_key")):
response = httpx.delete(
f"{self.base_url}/api/v1/videos/{video['id']}?delete_source=true",
headers=self.api_headers,
timeout=60,
)
if response.status_code != 404:
response.raise_for_status()
collections = httpx.get(
f"{self.base_url}/api/v1/collections", headers=self.api_headers, timeout=30
).json()
for collection in collections:
if collection.get("name") == self.run_id:
httpx.delete(
f"{self.base_url}/api/v1/collections/{collection['id']}",
headers=self.api_headers,
timeout=30,
).raise_for_status()
trash = httpx.get(
f"{self.base_url}/api/v1/trash", headers=self.api_headers, timeout=30
).json()
for item in trash:
if self.run_id in str(item.get("display_name")) or self.run_id in str(item.get("original_key")):
response = httpx.delete(
f"{self.base_url}/api/v1/trash/{item['id']}",
headers=self.api_headers,
timeout=60,
)
if response.status_code != 404:
response.raise_for_status()
except Exception as exc:
print(f"cleanup warning: {exc}", file=sys.stderr)
def execute(self, keep: bool) -> None:
cases = self.cases()
started = time.monotonic()
probe_stopping = threading.Event()
probe_thread = threading.Thread(
target=self.probe,
args=(probe_stopping,),
name="imagefind-stress-probe",
daemon=True,
)
probe_thread.start()
try:
with concurrent.futures.ThreadPoolExecutor(max_workers=self.workers) as executor:
futures = [executor.submit(self.put, case) for case in cases]
for future in concurrent.futures.as_completed(futures):
path, size, upload_id, response_latency = future.result()
if upload_id:
self.upload_ids.add(upload_id)
print(
f"received {path} ({size} bytes) "
f"response_latency={response_latency:.3f}s",
flush=True,
)
self.wait(len(self.upload_ids))
self.verify_dedup(cases[0])
if self.probe_errors:
raise RuntimeError(
"ImageFind API became unreachable during WebDAV transfer: "
+ "; ".join(self.probe_errors)
)
maximum_probe = max(self.probe_latencies, default=0)
print(
f"PASS run={self.run_id} files={len(cases)} bytes={sum(case.size for case in cases)} "
f"seconds={time.monotonic() - started:.1f} max_probe_latency={maximum_probe:.3f}s",
flush=True,
)
finally:
probe_stopping.set()
probe_thread.join(timeout=self.probe_timeout + 5)
if not keep:
self.cleanup()
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--base-url", required=True, help="ImageFind native URL, e.g. http://nas:8765")
parser.add_argument("--token", default=os.getenv("IMAGEFIND_WEBDAV_TOKEN"))
profile = parser.add_mutually_exclusive_group()
profile.add_argument("--full", action="store_true", help="include 1 GiB, 5 GiB and full-size batches")
profile.add_argument("--large", action="store_true", help="run four concurrent 256 MiB uploads")
parser.add_argument("--workers", type=int, default=8)
parser.add_argument("--timeout", type=float, default=7200)
parser.add_argument(
"--probe-timeout",
type=float,
default=10,
help="fail when the API is unreachable for this many seconds during transfers",
)
parser.add_argument("--keep", action="store_true", help="keep isolated stress data for inspection")
parser.add_argument(
"--media-prefix",
help="optional valid media file prepended before deterministic padding",
)
args = parser.parse_args()
if not args.token:
parser.error("--token or IMAGEFIND_WEBDAV_TOKEN is required")
media_prefix = Path(args.media_prefix).read_bytes() if args.media_prefix else b""
StressRun(
args.base_url,
args.token,
args.full,
args.large,
min(10, max(1, args.workers)),
args.timeout,
max(1, args.probe_timeout),
media_prefix,
).execute(args.keep)
if __name__ == "__main__":
main()
+189
View File
@@ -0,0 +1,189 @@
#!/usr/bin/env bash
set -euo pipefail
FILE=${1:?usage: upload-openlist-artifact.sh file [remote-directory]}
REMOTE_DIR=${2:-/yidongpan/构建产物/liverecorder}
OPENLIST_BASE_URL=${OPENLIST_BASE_URL:-https://openlist.nxsir.cn}
OPENLIST_USERNAME=${OPENLIST_USERNAME:?OPENLIST_USERNAME is required}
OPENLIST_PASSWORD=${OPENLIST_PASSWORD:?OPENLIST_PASSWORD is required}
OPENLIST_UPLOAD_ATTEMPTS=${OPENLIST_UPLOAD_ATTEMPTS:-6}
test -f "$FILE" || { printf 'artifact does not exist: %s\n' "$FILE" >&2; exit 1; }
for required_command in curl python3 stat basename sleep; do
command -v "$required_command" >/dev/null 2>&1 || {
printf 'required upload command is missing: %s\n' "$required_command" >&2
exit 1
}
done
OPENLIST_BASE_URL=${OPENLIST_BASE_URL%/}
REMOTE_DIR="/${REMOTE_DIR#/}"
REMOTE_DIR=${REMOTE_DIR%/}
FILE_NAME=$(basename -- "$FILE")
BUILD_MARKER=${BUILD_TAG:-${BUILD_ID:-$$}}
BUILD_MARKER=$(printf '%s' "$BUILD_MARKER" | tr -c 'A-Za-z0-9._-' '-')
TEMP_NAME=".${FILE_NAME}.uploading-${BUILD_MARKER}"
TEMP_PATH="$REMOTE_DIR/$TEMP_NAME"
FINAL_PATH="$REMOTE_DIR/$FILE_NAME"
LOCAL_SIZE=$(stat -c '%s' "$FILE")
TOKEN=
json_string() {
python3 -c 'import json,sys; print(json.dumps(sys.argv[1], ensure_ascii=False))' "$1"
}
envelope_code() {
python3 -c 'import json,sys
try:
value=json.load(sys.stdin)
print(value.get("code", -1))
except Exception:
print(-1)'
}
envelope_message() {
python3 -c 'import json,sys
try:
value=json.load(sys.stdin)
print(value.get("message", "unknown OpenList response"))
except Exception:
print("invalid OpenList response")'
}
login() {
local payload response code
payload=$(printf '{"username":%s,"password":%s}' \
"$(json_string "$OPENLIST_USERNAME")" \
"$(json_string "$OPENLIST_PASSWORD")")
response=$(curl --silent --show-error --location --fail-with-body \
--connect-timeout 30 --max-time 120 \
--request POST "$OPENLIST_BASE_URL/api/auth/login" \
--header 'Content-Type: application/json' \
--data-raw "$payload") || return 1
code=$(printf '%s' "$response" | envelope_code)
[ "$code" = "200" ] || {
printf 'OpenList login failed: %s\n' "$(printf '%s' "$response" | envelope_message)" >&2
return 1
}
TOKEN=$(printf '%s' "$response" | python3 -c 'import json,sys; print((json.load(sys.stdin).get("data") or {}).get("token", ""))')
[ -n "$TOKEN" ]
}
post_json() {
local endpoint=$1 payload=$2
curl --silent --show-error --location --fail-with-body \
--connect-timeout 30 --max-time 180 \
--request POST "$OPENLIST_BASE_URL$endpoint" \
--header "Authorization: $TOKEN" \
--header 'Content-Type: application/json' \
--data-raw "$payload"
}
remote_size() {
local path=$1 response code
response=$(post_json /api/fs/get "{\"path\":$(json_string "$path"),\"password\":\"\"}") || return 1
code=$(printf '%s' "$response" | envelope_code)
[ "$code" = "200" ] || return 1
printf '%s' "$response" | python3 -c 'import json,sys; print((json.load(sys.stdin).get("data") or {}).get("size", -1))'
}
ensure_remote_directory() {
local current= segment response code
while IFS= read -r segment; do
[ -n "$segment" ] || continue
current="$current/$segment"
response=$(post_json /api/fs/list "{\"path\":$(json_string "$current"),\"password\":\"\",\"refresh\":false,\"page\":1,\"per_page\":1}" || true)
code=$(printf '%s' "$response" | envelope_code)
if [ "$code" = "200" ]; then
continue
fi
response=$(post_json /api/fs/mkdir "{\"path\":$(json_string "$current")}") || return 1
code=$(printf '%s' "$response" | envelope_code)
[ "$code" = "200" ] || {
printf 'OpenList mkdir failed for %s: %s\n' "$current" "$(printf '%s' "$response" | envelope_message)" >&2
return 1
}
done < <(python3 -c 'import sys; print("\n".join(part for part in sys.argv[1].split("/") if part))' "$REMOTE_DIR")
}
remove_temporary_file() {
local response
[ -n "$TOKEN" ] || login >/dev/null 2>&1 || return 0
response=$(post_json /api/fs/remove "{\"dir\":$(json_string "$REMOTE_DIR"),\"names\":[$(json_string "$TEMP_NAME")]}" 2>/dev/null || true)
[ "$(printf '%s' "$response" | envelope_code)" = "200" ] || true
}
publish_temporary_file() {
local response code
response=$(post_json /api/fs/rename "{\"path\":$(json_string "$TEMP_PATH"),\"name\":$(json_string "$FILE_NAME")}") || return 1
code=$(printf '%s' "$response" | envelope_code)
[ "$code" = "200" ] || {
printf 'OpenList rename failed: %s\n' "$(printf '%s' "$response" | envelope_message)" >&2
return 1
}
}
upload_once() {
local encoded_path response code uploaded_size
encoded_path=$(python3 -c 'import sys,urllib.parse; print(urllib.parse.quote(sys.argv[1], safe=""))' "$TEMP_PATH")
response=$(curl --silent --show-error --location --fail-with-body \
--connect-timeout 30 --max-time 3600 --speed-limit 1024 --speed-time 120 \
--request PUT "$OPENLIST_BASE_URL/api/fs/put" \
--header "Authorization: $TOKEN" \
--header "File-Path: $encoded_path" \
--header 'As-Task: false' \
--header 'Content-Type: application/octet-stream' \
--header 'Expect:' \
--data-binary "@$FILE") || return 1
code=$(printf '%s' "$response" | envelope_code)
[ "$code" = "200" ] || {
printf 'OpenList upload failed: %s\n' "$(printf '%s' "$response" | envelope_message)" >&2
return 1
}
uploaded_size=$(remote_size "$TEMP_PATH") || return 1
[ "$uploaded_size" = "$LOCAL_SIZE" ] || {
printf 'OpenList size mismatch for temporary upload: local=%s remote=%s\n' "$LOCAL_SIZE" "$uploaded_size" >&2
return 1
}
}
printf 'Uploading %s (%s bytes) to OpenList %s\n' "$FILE_NAME" "$LOCAL_SIZE" "$REMOTE_DIR"
for attempt in $(seq 1 "$OPENLIST_UPLOAD_ATTEMPTS"); do
TOKEN=
upload_ready=1
if login && ensure_remote_directory; then
existing_size=$(remote_size "$FINAL_PATH" 2>/dev/null || true)
if [ "$existing_size" = "$LOCAL_SIZE" ]; then
printf 'OpenList artifact already exists with matching size: %s\n' "$FINAL_PATH"
exit 0
fi
if [ -n "$existing_size" ]; then
remove_response=$(post_json /api/fs/remove "{\"dir\":$(json_string "$REMOTE_DIR"),\"names\":[$(json_string "$FILE_NAME")]}" || true)
if [ "$(printf '%s' "$remove_response" | envelope_code)" != "200" ]; then
printf 'Unable to remove mismatched OpenList artifact before retry: %s\n' "$FINAL_PATH" >&2
upload_ready=0
fi
fi
if [ "$upload_ready" = "1" ] && upload_once && publish_temporary_file; then
final_size=$(remote_size "$FINAL_PATH") || final_size=-1
if [ "$final_size" = "$LOCAL_SIZE" ]; then
printf 'OpenList upload verified: %s (%s bytes)\n' "$FINAL_PATH" "$final_size"
exit 0
fi
printf 'OpenList final size mismatch: local=%s remote=%s\n' "$LOCAL_SIZE" "$final_size" >&2
fi
fi
if [ "$attempt" -lt "$OPENLIST_UPLOAD_ATTEMPTS" ]; then
delay=$((5 * (1 << (attempt - 1))))
[ "$delay" -le 120 ] || delay=120
printf 'OpenList upload attempt %s/%s failed; retrying in %ss\n' "$attempt" "$OPENLIST_UPLOAD_ATTEMPTS" "$delay" >&2
sleep "$delay"
fi
done
remove_temporary_file
printf 'OpenList upload failed after %s attempts: %s\n' "$OPENLIST_UPLOAD_ATTEMPTS" "$FILE_NAME" >&2
exit 1
+35
View File
@@ -0,0 +1,35 @@
#!/usr/bin/env bash
set -euo pipefail
PACKAGE=${1:?usage: verify-fnos-package.sh package.fpk [expected-version]}
EXPECTED_VERSION=${2:-}
VERIFY_ROOT=${IMAGEFIND_VERIFY_TMPDIR:-${TMPDIR:-/tmp}}
mkdir -p "$VERIFY_ROOT"
work_dir=$(mktemp -d "${VERIFY_ROOT%/}/imagefind-fnos-verify.XXXXXX")
trap 'rm -rf -- "$work_dir"' EXIT
tar -xzf "$PACKAGE" -C "$work_dir"
value() { sed -n "s/^$1[[:space:]]*=[[:space:]]*//p" "$work_dir/manifest" | head -n 1 | tr -d '\r'; }
test "$(value appname)" = imagefind
version=$(value version)
test -n "$version"
if [ -n "$EXPECTED_VERSION" ]; then test "$version" = "$EXPECTED_VERSION"; fi
test "$(value platform)" = x86
test "$(value install_dep_apps)" = python312,nxsir.postgresql
test -x "$work_dir/cmd/main"
test -s "$work_dir/app.tgz"
gzip -t "$work_dir/app.tgz"
tar -tzf "$work_dir/app.tgz" >"$work_dir/files.txt"
grep -q '^frontend/index.html$' "$work_dir/files.txt"
grep -q '^runtime/imagefind-.*\.whl$' "$work_dir/files.txt"
grep -q '^runtime/wheels/.*\.whl$' "$work_dir/files.txt"
grep -q '^bin/libOpenCL.so.1$' "$work_dir/files.txt"
runtime_version=$(tar -xOzf "$work_dir/app.tgz" runtime/VERSION | sed -n '1p')
test "$runtime_version" = "$version"
if grep -Eq '^(data|models|cache|postgresql)/' "$work_dir/files.txt"; then
printf 'persistent data must not be included in ImageFind app.tgz\n' >&2
exit 1
fi
test -s "$PACKAGE.sha256"
(cd "$(dirname "$PACKAGE")" && sha256sum --check --status "$(basename "$PACKAGE").sha256")
printf 'ImageFind fnOS package verified: %s (%s)\n' "$PACKAGE" "$version"