From 1babec1b85545c1da3cbebdc4904a25520c128b9 Mon Sep 17 00:00:00 2001 From: nanxun Date: Wed, 12 Aug 2026 17:33:05 +0800 Subject: [PATCH] fix(fnos): reuse content-addressed core runtime --- fnos/cmd/runtime_bootstrap | 38 +++++++++++++++++++++++++++------- scripts/build-fnos.sh | 3 ++- scripts/smoke-fnos-package.sh | 2 +- scripts/verify-fnos-package.sh | 5 +++++ tests/test_fnos_lifecycle.py | 27 ++++++++++++++++++++++-- 5 files changed, 63 insertions(+), 12 deletions(-) diff --git a/fnos/cmd/runtime_bootstrap b/fnos/cmd/runtime_bootstrap index bea2ffb..7f117e6 100755 --- a/fnos/cmd/runtime_bootstrap +++ b/fnos/cmd/runtime_bootstrap @@ -18,11 +18,15 @@ fail() { [ -d "$WHEEL_ROOT" ] || fail "ImageFind 运行环境初始化失败:安装包缺少核心 wheelhouse。" [ -f "$CORE_REQUIREMENTS" ] || fail "ImageFind 运行环境初始化失败:安装包缺少核心依赖锁文件。" -VERSION=$(sed -n '1p' "$VERSION_FILE") -case "$VERSION" in +PACKAGE_VERSION=$(sed -n '1p' "$VERSION_FILE") +case "$PACKAGE_VERSION" in ''|*[!0-9.]* ) fail "ImageFind 运行环境初始化失败:版本信息无效。" ;; esac -PACKAGE_ID=$(sed -n '2p' "$VERSION_FILE") +APP_VERSION=$(sed -n '2p' "$VERSION_FILE") +case "$APP_VERSION" in + ''|*[!0-9.]* ) fail "ImageFind 运行环境初始化失败:应用版本信息无效。" ;; +esac +PACKAGE_ID=$(sed -n '3p' "$VERSION_FILE") case "$PACKAGE_ID" in ''|*[!0-9a-f]* ) fail "ImageFind 运行环境初始化失败:安装包指纹无效。" ;; esac @@ -39,8 +43,9 @@ PYTHON_VERSION=$($SYSTEM_PYTHON -c 'import sys; print(f"{sys.version_info.major} [ "$PYTHON_VERSION" = "3.12" ] || fail "ImageFind 需要 Python 3.12,当前为 $PYTHON_VERSION。" mkdir -p "$RUNTIME_ROOT" -TARGET="$RUNTIME_ROOT/core-$VERSION" -STAGE="$RUNTIME_ROOT/.core-$VERSION.tmp.$$" +RUNTIME_ID=$(printf '%s' "$PACKAGE_ID" | cut -c1-16) +TARGET="$RUNTIME_ROOT/core-$APP_VERSION-$RUNTIME_ID" +STAGE="$RUNTIME_ROOT/.core-$APP_VERSION-$RUNTIME_ID.tmp.$$" CURRENT="$RUNTIME_ROOT/current" runtime_valid() { @@ -51,7 +56,7 @@ runtime_valid() { IMAGEFIND_DATA_DIR="$DATA_ROOT/data" "$1/bin/python" -c \ 'import imagefind; print(imagefind.__version__)' 2>"$RUNTIME_ROOT/version-check.err" ) || return 1 - [ "$RUNTIME_VERSION" = "$VERSION" ] + [ "$RUNTIME_VERSION" = "$APP_VERSION" ] } if ! runtime_valid "$TARGET"; then @@ -83,7 +88,7 @@ if ! runtime_valid "$TARGET"; then fail "ImageFind 核心运行环境校验失败。" } if [ -e "$TARGET" ]; then - BACKUP="$RUNTIME_ROOT/core-$VERSION.invalid.$(date +%s)" + BACKUP="$RUNTIME_ROOT/core-$APP_VERSION-$RUNTIME_ID.invalid.$(date +%s)" mv "$TARGET" "$BACKUP" fi mv "$STAGE" "$TARGET" @@ -122,5 +127,22 @@ if ! runtime_valid "$CURRENT"; then else ACTUAL_VERSION='Python 不可执行' fi - fail "ImageFind 核心运行环境切换失败:期望版本 $VERSION,current=$CURRENT_LINK,实际版本=$ACTUAL_VERSION。" + fail "ImageFind 核心运行环境切换失败:期望版本 $APP_VERSION,current=$CURRENT_LINK,实际版本=$ACTUAL_VERSION。" fi + +# Keep the active runtime plus one most-recent rollback candidate. fnOS package +# build numbers change on every release, so retaining every historical venv +# would otherwise grow the application data directory without bound. +ROLLBACK_KEPT=0 +for OLD_RUNTIME in $(ls -dt "$RUNTIME_ROOT"/core-* 2>/dev/null || true); do + [ -d "$OLD_RUNTIME" ] || continue + [ "$OLD_RUNTIME" = "$TARGET" ] && continue + case "$OLD_RUNTIME" in + *.invalid.*) rm -rf -- "$OLD_RUNTIME"; continue ;; + esac + if [ "$ROLLBACK_KEPT" -eq 0 ]; then + ROLLBACK_KEPT=1 + else + rm -rf -- "$OLD_RUNTIME" + fi +done diff --git a/scripts/build-fnos.sh b/scripts/build-fnos.sh index 783842c..8e3c117 100755 --- a/scripts/build-fnos.sh +++ b/scripts/build-fnos.sh @@ -50,6 +50,7 @@ PYTHON_BIN=$(command -v "$PYTHON_BIN") || { printf 'Python with hatchling is required to assemble the release.\n' >&2 exit 1 } +APP_VERSION=$("$PYTHON_BIN" -c 'import pathlib,sys,tomllib; print(tomllib.loads(pathlib.Path(sys.argv[1]).read_text())["project"]["version"])' "$PROJECT_ROOT/pyproject.toml") command -v npm >/dev/null 2>&1 || { printf 'npm is required.\n' >&2; exit 1; } "$PYTHON_BIN" -c 'import hatchling' >/dev/null 2>&1 || { printf 'hatchling is required in the release Python environment.\n' >&2 @@ -89,7 +90,7 @@ done PACKAGE_ID=$("$PYTHON_BIN" -c \ 'import hashlib,sys; h=hashlib.sha256(); [h.update(open(path,"rb").read()) for path in sys.argv[1:]]; print(h.hexdigest())' \ "$STAGED_APP_WHEEL" "$STAGE/app/runtime/runtime-core.txt") -printf '%s\n%s\n' "$PACKAGE_VERSION" "$PACKAGE_ID" >"$STAGE/app/runtime/VERSION" +printf '%s\n%s\n%s\n' "$PACKAGE_VERSION" "$APP_VERSION" "$PACKAGE_ID" >"$STAGE/app/runtime/VERSION" "$PYTHON_BIN" "$PROJECT_ROOT/scripts/make_icons.py" "$STAGE" chmod 0755 "$STAGE/cmd/"* ( diff --git a/scripts/smoke-fnos-package.sh b/scripts/smoke-fnos-package.sh index 73c9131..1280d24 100755 --- a/scripts/smoke-fnos-package.sh +++ b/scripts/smoke-fnos-package.sh @@ -17,7 +17,7 @@ SMOKE_PASSWORD="imagefind-smoke-password" SMOKE_POSTGRES_TOKEN="${IMAGEFIND_SMOKE_POSTGRES_TOKEN:-}" COOKIE_JAR="$DATA_ROOT/run/smoke-cookie.txt" BACKUP_FILE="$DATA_ROOT/run/smoke.ifbackup" -EXPECTED_VERSION=$(sed -n '1p' "$APP_ROOT/runtime/VERSION") +EXPECTED_VERSION=$(sed -n '2p' "$APP_ROOT/runtime/VERSION") APP_ROOT=$(CDPATH= cd -- "$APP_ROOT" && pwd) [ "${#SMOKE_POSTGRES_TOKEN}" -ge 20 ] || { diff --git a/scripts/verify-fnos-package.sh b/scripts/verify-fnos-package.sh index 86564d2..9c66c44 100755 --- a/scripts/verify-fnos-package.sh +++ b/scripts/verify-fnos-package.sh @@ -26,6 +26,11 @@ grep -q '^runtime/wheels/.*\.whl$' "$work_dir/files.txt" grep -q '^bin/libOpenCL.so.1$' "$work_dir/files.txt" runtime_version=$(tar -xOzf "$work_dir/app.tgz" runtime/VERSION | sed -n '1p') test "$runtime_version" = "$version" +runtime_app_version=$(tar -xOzf "$work_dir/app.tgz" runtime/VERSION | sed -n '2p') +runtime_package_id=$(tar -xOzf "$work_dir/app.tgz" runtime/VERSION | sed -n '3p') +test -n "$runtime_app_version" +test "${#runtime_package_id}" -eq 64 +grep -q "^runtime/imagefind-${runtime_app_version}-.*\\.whl$" "$work_dir/files.txt" if grep -Eq '^(data|models|cache|postgresql)/' "$work_dir/files.txt"; then printf 'persistent data must not be included in ImageFind app.tgz\n' >&2 exit 1 diff --git a/tests/test_fnos_lifecycle.py b/tests/test_fnos_lifecycle.py index e0bc636..e6f3c7c 100644 --- a/tests/test_fnos_lifecycle.py +++ b/tests/test_fnos_lifecycle.py @@ -19,7 +19,7 @@ def _mock_app(tmp_path: Path) -> tuple[Path, Path, Path]: app_root = tmp_path / "app" runtime = app_root / "runtime" (runtime / "wheels").mkdir(parents=True) - (runtime / "VERSION").write_text("0.4.3\n" + "a" * 64 + "\n") + (runtime / "VERSION").write_text("0.4.30001\n0.4.3\n" + "a" * 64 + "\n") (runtime / "runtime-core.txt").write_text("# mocked offline core\n") (runtime / "imagefind-0.4.3-py3-none-any.whl").write_bytes(b"mock wheel") (runtime / "wheels" / "dependency.whl").write_bytes(b"mock dependency") @@ -312,7 +312,7 @@ def test_runtime_bootstrap_atomically_replaces_existing_current( assert result.returncode == 0, result.stderr assert current.is_symlink() - assert current.resolve() == runtime_root / "core-0.4.3" + assert current.resolve() == runtime_root / "core-0.4.3-aaaaaaaaaaaaaaaa" version = subprocess.run( [str(current / "bin" / "python"), "-m", "imagefind.main", "--version"], env=env, @@ -332,6 +332,29 @@ def test_runtime_bootstrap_atomically_replaces_existing_current( assert (previous[0] / "sentinel").read_text() == "preserve old runtime" +def test_runtime_bootstrap_reuses_content_id_across_package_builds_and_prunes_old_runtimes(tmp_path: Path): + app_root, args_file, password_file = _mock_app(tmp_path) + env = _environment(tmp_path, app_root, args_file, password_file) + runtime_root = tmp_path / "var" / "runtime" + runtime_root.mkdir(parents=True) + for name in ("core-0.3.14-old", "core-0.3.16-newer"): + old_runtime = runtime_root / name + old_runtime.mkdir() + (old_runtime / "sentinel").write_text(name) + + first = _run("runtime_bootstrap", env) + target = runtime_root / "core-0.4.3-aaaaaaaaaaaaaaaa" + assert first.returncode == 0, first.stderr + assert target.is_dir() + assert len([path for path in runtime_root.glob("core-*") if path.is_dir()]) == 2 + + (app_root / "runtime" / "VERSION").write_text("0.4.30002\n0.4.3\n" + "a" * 64 + "\n") + second = _run("runtime_bootstrap", env) + assert second.returncode == 0, second.stderr + assert (runtime_root / "current").resolve() == target + assert len([path for path in runtime_root.glob("core-*") if path.is_dir()]) == 2 + + def test_config_callback_writes_validated_direct_access_atomically(tmp_path: Path): data_root = tmp_path / "var" env = {