nanxun/imagefind
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(fnos): reuse content-addressed core runtime

Browse Source
This commit is contained in:
nanxun
2026-08-12 17:33:05 +08:00
parent eec1227179
commit 1babec1b85
5 changed files with 63 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fnos/cmd/runtime_bootstrap
+30 -8
View File
@@ -18,11 +18,15 @@ fail() {
[ -d "$WHEEL_ROOT" ] || fail "ImageFind 运行环境初始化失败：安装包缺少核心 wheelhouse。"
[ -f "$CORE_REQUIREMENTS" ] || fail "ImageFind 运行环境初始化失败：安装包缺少核心依赖锁文件。"
VERSION=$(sed -n '1p' "$VERSION_FILE")
case "$VERSION" in
PACKAGE_VERSION=$(sed -n '1p' "$VERSION_FILE")
case "$PACKAGE_VERSION" in
''|*[!0-9.]* ) fail "ImageFind 运行环境初始化失败：版本信息无效。" ;;
esac
PACKAGE_ID=$(sed -n '2p' "$VERSION_FILE")
APP_VERSION=$(sed -n '2p' "$VERSION_FILE")
case "$APP_VERSION" in
''|*[!0-9.]* ) fail "ImageFind 运行环境初始化失败：应用版本信息无效。" ;;
esac
PACKAGE_ID=$(sed -n '3p' "$VERSION_FILE")
case "$PACKAGE_ID" in
''|*[!0-9a-f]* ) fail "ImageFind 运行环境初始化失败：安装包指纹无效。" ;;
esac
@@ -39,8 +43,9 @@ PYTHON_VERSION=$($SYSTEM_PYTHON -c 'import sys; print(f"{sys.version_info.major}
[ "$PYTHON_VERSION" = "3.12" ] || fail "ImageFind 需要 Python 3.12，当前为 $PYTHON_VERSION。"
mkdir -p "$RUNTIME_ROOT"
TARGET="$RUNTIME_ROOT/core-$VERSION"
STAGE="$RUNTIME_ROOT/.core-$VERSION.tmp.$$"
RUNTIME_ID=$(printf '%s' "$PACKAGE_ID" | cut -c1-16)
TARGET="$RUNTIME_ROOT/core-$APP_VERSION-$RUNTIME_ID"
STAGE="$RUNTIME_ROOT/.core-$APP_VERSION-$RUNTIME_ID.tmp.$$"
CURRENT="$RUNTIME_ROOT/current"
runtime_valid() {
@@ -51,7 +56,7 @@ runtime_valid() {
IMAGEFIND_DATA_DIR="$DATA_ROOT/data" "$1/bin/python" -c \
'import imagefind; print(imagefind.__version__)' 2>"$RUNTIME_ROOT/version-check.err"
) || return 1
[ "$RUNTIME_VERSION" = "$VERSION" ]
[ "$RUNTIME_VERSION" = "$APP_VERSION" ]
}
if ! runtime_valid "$TARGET"; then
@@ -83,7 +88,7 @@ if ! runtime_valid "$TARGET"; then
fail "ImageFind 核心运行环境校验失败。"
}
if [ -e "$TARGET" ]; then
BACKUP="$RUNTIME_ROOT/core-$VERSION.invalid.$(date +%s)"
BACKUP="$RUNTIME_ROOT/core-$APP_VERSION-$RUNTIME_ID.invalid.$(date +%s)"
mv "$TARGET" "$BACKUP"
fi
mv "$STAGE" "$TARGET"
@@ -122,5 +127,22 @@ if ! runtime_valid "$CURRENT"; then
else
ACTUAL_VERSION='Python 不可执行'
fi
fail "ImageFind 核心运行环境切换失败：期望版本 $VERSION，current=$CURRENT_LINK，实际版本=$ACTUAL_VERSION。"
fail "ImageFind 核心运行环境切换失败：期望版本 $APP_VERSION，current=$CURRENT_LINK，实际版本=$ACTUAL_VERSION。"
fi
# Keep the active runtime plus one most-recent rollback candidate. fnOS package
# build numbers change on every release, so retaining every historical venv
# would otherwise grow the application data directory without bound.
ROLLBACK_KEPT=0
for OLD_RUNTIME in $(ls -dt "$RUNTIME_ROOT"/core-* 2>/dev/null || true); do
[ -d "$OLD_RUNTIME" ] || continue
[ "$OLD_RUNTIME" = "$TARGET" ] && continue
case "$OLD_RUNTIME" in
*.invalid.*) rm -rf -- "$OLD_RUNTIME"; continue ;;
esac
if [ "$ROLLBACK_KEPT" -eq 0 ]; then
ROLLBACK_KEPT=1
else
rm -rf -- "$OLD_RUNTIME"
fi
done
scripts/build-fnos.sh
+2 -1
View File
@@ -50,6 +50,7 @@ PYTHON_BIN=$(command -v "$PYTHON_BIN") || {
printf 'Python with hatchling is required to assemble the release.\n' >&2
exit 1
}
APP_VERSION=$("$PYTHON_BIN" -c 'import pathlib,sys,tomllib; print(tomllib.loads(pathlib.Path(sys.argv[1]).read_text())["project"]["version"])' "$PROJECT_ROOT/pyproject.toml")
command -v npm >/dev/null 2>&1 || { printf 'npm is required.\n' >&2; exit 1; }
"$PYTHON_BIN" -c 'import hatchling' >/dev/null 2>&1 || {
printf 'hatchling is required in the release Python environment.\n' >&2
@@ -89,7 +90,7 @@ done
PACKAGE_ID=$("$PYTHON_BIN" -c \
'import hashlib,sys; h=hashlib.sha256(); [h.update(open(path,"rb").read()) for path in sys.argv[1:]]; print(h.hexdigest())' \
"$STAGED_APP_WHEEL" "$STAGE/app/runtime/runtime-core.txt")
printf '%s\n%s\n' "$PACKAGE_VERSION" "$PACKAGE_ID" >"$STAGE/app/runtime/VERSION"
printf '%s\n%s\n%s\n' "$PACKAGE_VERSION" "$APP_VERSION" "$PACKAGE_ID" >"$STAGE/app/runtime/VERSION"
"$PYTHON_BIN" "$PROJECT_ROOT/scripts/make_icons.py" "$STAGE"
chmod 0755 "$STAGE/cmd/"*
(
scripts/smoke-fnos-package.sh
+1 -1
View File
@@ -17,7 +17,7 @@ SMOKE_PASSWORD="imagefind-smoke-password"
SMOKE_POSTGRES_TOKEN="${IMAGEFIND_SMOKE_POSTGRES_TOKEN:-}"
COOKIE_JAR="$DATA_ROOT/run/smoke-cookie.txt"
BACKUP_FILE="$DATA_ROOT/run/smoke.ifbackup"
EXPECTED_VERSION=$(sed -n '1p' "$APP_ROOT/runtime/VERSION")
EXPECTED_VERSION=$(sed -n '2p' "$APP_ROOT/runtime/VERSION")
APP_ROOT=$(CDPATH= cd -- "$APP_ROOT" && pwd)
[ "${#SMOKE_POSTGRES_TOKEN}" -ge 20 ] || {
scripts/verify-fnos-package.sh
+5
View File
@@ -26,6 +26,11 @@ grep -q '^runtime/wheels/.*\.whl$' "$work_dir/files.txt"
grep -q '^bin/libOpenCL.so.1$' "$work_dir/files.txt"
runtime_version=$(tar -xOzf "$work_dir/app.tgz" runtime/VERSION | sed -n '1p')
test "$runtime_version" = "$version"
runtime_app_version=$(tar -xOzf "$work_dir/app.tgz" runtime/VERSION | sed -n '2p')
runtime_package_id=$(tar -xOzf "$work_dir/app.tgz" runtime/VERSION | sed -n '3p')
test -n "$runtime_app_version"
test "${#runtime_package_id}" -eq 64
grep -q "^runtime/imagefind-${runtime_app_version}-.*\\.whl$" "$work_dir/files.txt"
if grep -Eq '^(data|models|cache|postgresql)/' "$work_dir/files.txt"; then
printf 'persistent data must not be included in ImageFind app.tgz\n' >&2
exit 1
tests/test_fnos_lifecycle.py
+25 -2
View File
@@ -19,7 +19,7 @@ def _mock_app(tmp_path: Path) -> tuple[Path, Path, Path]:
app_root = tmp_path / "app"
runtime = app_root / "runtime"
(runtime / "wheels").mkdir(parents=True)
(runtime / "VERSION").write_text("0.4.3\n" + "a" * 64 + "\n")
(runtime / "VERSION").write_text("0.4.30001\n0.4.3\n" + "a" * 64 + "\n")
(runtime / "runtime-core.txt").write_text("# mocked offline core\n")
(runtime / "imagefind-0.4.3-py3-none-any.whl").write_bytes(b"mock wheel")
(runtime / "wheels" / "dependency.whl").write_bytes(b"mock dependency")
@@ -312,7 +312,7 @@ def test_runtime_bootstrap_atomically_replaces_existing_current(
assert result.returncode == 0, result.stderr
assert current.is_symlink()
assert current.resolve() == runtime_root / "core-0.4.3"
assert current.resolve() == runtime_root / "core-0.4.3-aaaaaaaaaaaaaaaa"
version = subprocess.run(
[str(current / "bin" / "python"), "-m", "imagefind.main", "--version"],
env=env,
@@ -332,6 +332,29 @@ def test_runtime_bootstrap_atomically_replaces_existing_current(
assert (previous[0] / "sentinel").read_text() == "preserve old runtime"
def test_runtime_bootstrap_reuses_content_id_across_package_builds_and_prunes_old_runtimes(tmp_path: Path):
app_root, args_file, password_file = _mock_app(tmp_path)
env = _environment(tmp_path, app_root, args_file, password_file)
runtime_root = tmp_path / "var" / "runtime"
runtime_root.mkdir(parents=True)
for name in ("core-0.3.14-old", "core-0.3.16-newer"):
old_runtime = runtime_root / name
old_runtime.mkdir()
(old_runtime / "sentinel").write_text(name)
first = _run("runtime_bootstrap", env)
target = runtime_root / "core-0.4.3-aaaaaaaaaaaaaaaa"
assert first.returncode == 0, first.stderr
assert target.is_dir()
assert len([path for path in runtime_root.glob("core-*") if path.is_dir()]) == 2
(app_root / "runtime" / "VERSION").write_text("0.4.30002\n0.4.3\n" + "a" * 64 + "\n")
second = _run("runtime_bootstrap", env)
assert second.returncode == 0, second.stderr
assert (runtime_root / "current").resolve() == target
assert len([path for path in runtime_root.glob("core-*") if path.is_dir()]) == 2
def test_config_callback_writes_validated_direct_access_atomically(tmp_path: Path):
data_root = tmp_path / "var"
env = {