fix(fnos): reuse content-addressed core runtime
This commit is contained in:
@@ -18,11 +18,15 @@ fail() {
|
|||||||
[ -d "$WHEEL_ROOT" ] || fail "ImageFind 运行环境初始化失败:安装包缺少核心 wheelhouse。"
|
[ -d "$WHEEL_ROOT" ] || fail "ImageFind 运行环境初始化失败:安装包缺少核心 wheelhouse。"
|
||||||
[ -f "$CORE_REQUIREMENTS" ] || fail "ImageFind 运行环境初始化失败:安装包缺少核心依赖锁文件。"
|
[ -f "$CORE_REQUIREMENTS" ] || fail "ImageFind 运行环境初始化失败:安装包缺少核心依赖锁文件。"
|
||||||
|
|
||||||
VERSION=$(sed -n '1p' "$VERSION_FILE")
|
PACKAGE_VERSION=$(sed -n '1p' "$VERSION_FILE")
|
||||||
case "$VERSION" in
|
case "$PACKAGE_VERSION" in
|
||||||
''|*[!0-9.]* ) fail "ImageFind 运行环境初始化失败:版本信息无效。" ;;
|
''|*[!0-9.]* ) fail "ImageFind 运行环境初始化失败:版本信息无效。" ;;
|
||||||
esac
|
esac
|
||||||
PACKAGE_ID=$(sed -n '2p' "$VERSION_FILE")
|
APP_VERSION=$(sed -n '2p' "$VERSION_FILE")
|
||||||
|
case "$APP_VERSION" in
|
||||||
|
''|*[!0-9.]* ) fail "ImageFind 运行环境初始化失败:应用版本信息无效。" ;;
|
||||||
|
esac
|
||||||
|
PACKAGE_ID=$(sed -n '3p' "$VERSION_FILE")
|
||||||
case "$PACKAGE_ID" in
|
case "$PACKAGE_ID" in
|
||||||
''|*[!0-9a-f]* ) fail "ImageFind 运行环境初始化失败:安装包指纹无效。" ;;
|
''|*[!0-9a-f]* ) fail "ImageFind 运行环境初始化失败:安装包指纹无效。" ;;
|
||||||
esac
|
esac
|
||||||
@@ -39,8 +43,9 @@ PYTHON_VERSION=$($SYSTEM_PYTHON -c 'import sys; print(f"{sys.version_info.major}
|
|||||||
[ "$PYTHON_VERSION" = "3.12" ] || fail "ImageFind 需要 Python 3.12,当前为 $PYTHON_VERSION。"
|
[ "$PYTHON_VERSION" = "3.12" ] || fail "ImageFind 需要 Python 3.12,当前为 $PYTHON_VERSION。"
|
||||||
|
|
||||||
mkdir -p "$RUNTIME_ROOT"
|
mkdir -p "$RUNTIME_ROOT"
|
||||||
TARGET="$RUNTIME_ROOT/core-$VERSION"
|
RUNTIME_ID=$(printf '%s' "$PACKAGE_ID" | cut -c1-16)
|
||||||
STAGE="$RUNTIME_ROOT/.core-$VERSION.tmp.$$"
|
TARGET="$RUNTIME_ROOT/core-$APP_VERSION-$RUNTIME_ID"
|
||||||
|
STAGE="$RUNTIME_ROOT/.core-$APP_VERSION-$RUNTIME_ID.tmp.$$"
|
||||||
CURRENT="$RUNTIME_ROOT/current"
|
CURRENT="$RUNTIME_ROOT/current"
|
||||||
|
|
||||||
runtime_valid() {
|
runtime_valid() {
|
||||||
@@ -51,7 +56,7 @@ runtime_valid() {
|
|||||||
IMAGEFIND_DATA_DIR="$DATA_ROOT/data" "$1/bin/python" -c \
|
IMAGEFIND_DATA_DIR="$DATA_ROOT/data" "$1/bin/python" -c \
|
||||||
'import imagefind; print(imagefind.__version__)' 2>"$RUNTIME_ROOT/version-check.err"
|
'import imagefind; print(imagefind.__version__)' 2>"$RUNTIME_ROOT/version-check.err"
|
||||||
) || return 1
|
) || return 1
|
||||||
[ "$RUNTIME_VERSION" = "$VERSION" ]
|
[ "$RUNTIME_VERSION" = "$APP_VERSION" ]
|
||||||
}
|
}
|
||||||
|
|
||||||
if ! runtime_valid "$TARGET"; then
|
if ! runtime_valid "$TARGET"; then
|
||||||
@@ -83,7 +88,7 @@ if ! runtime_valid "$TARGET"; then
|
|||||||
fail "ImageFind 核心运行环境校验失败。"
|
fail "ImageFind 核心运行环境校验失败。"
|
||||||
}
|
}
|
||||||
if [ -e "$TARGET" ]; then
|
if [ -e "$TARGET" ]; then
|
||||||
BACKUP="$RUNTIME_ROOT/core-$VERSION.invalid.$(date +%s)"
|
BACKUP="$RUNTIME_ROOT/core-$APP_VERSION-$RUNTIME_ID.invalid.$(date +%s)"
|
||||||
mv "$TARGET" "$BACKUP"
|
mv "$TARGET" "$BACKUP"
|
||||||
fi
|
fi
|
||||||
mv "$STAGE" "$TARGET"
|
mv "$STAGE" "$TARGET"
|
||||||
@@ -122,5 +127,22 @@ if ! runtime_valid "$CURRENT"; then
|
|||||||
else
|
else
|
||||||
ACTUAL_VERSION='Python 不可执行'
|
ACTUAL_VERSION='Python 不可执行'
|
||||||
fi
|
fi
|
||||||
fail "ImageFind 核心运行环境切换失败:期望版本 $VERSION,current=$CURRENT_LINK,实际版本=$ACTUAL_VERSION。"
|
fail "ImageFind 核心运行环境切换失败:期望版本 $APP_VERSION,current=$CURRENT_LINK,实际版本=$ACTUAL_VERSION。"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# Keep the active runtime plus one most-recent rollback candidate. fnOS package
|
||||||
|
# build numbers change on every release, so retaining every historical venv
|
||||||
|
# would otherwise grow the application data directory without bound.
|
||||||
|
ROLLBACK_KEPT=0
|
||||||
|
for OLD_RUNTIME in $(ls -dt "$RUNTIME_ROOT"/core-* 2>/dev/null || true); do
|
||||||
|
[ -d "$OLD_RUNTIME" ] || continue
|
||||||
|
[ "$OLD_RUNTIME" = "$TARGET" ] && continue
|
||||||
|
case "$OLD_RUNTIME" in
|
||||||
|
*.invalid.*) rm -rf -- "$OLD_RUNTIME"; continue ;;
|
||||||
|
esac
|
||||||
|
if [ "$ROLLBACK_KEPT" -eq 0 ]; then
|
||||||
|
ROLLBACK_KEPT=1
|
||||||
|
else
|
||||||
|
rm -rf -- "$OLD_RUNTIME"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|||||||
@@ -50,6 +50,7 @@ PYTHON_BIN=$(command -v "$PYTHON_BIN") || {
|
|||||||
printf 'Python with hatchling is required to assemble the release.\n' >&2
|
printf 'Python with hatchling is required to assemble the release.\n' >&2
|
||||||
exit 1
|
exit 1
|
||||||
}
|
}
|
||||||
|
APP_VERSION=$("$PYTHON_BIN" -c 'import pathlib,sys,tomllib; print(tomllib.loads(pathlib.Path(sys.argv[1]).read_text())["project"]["version"])' "$PROJECT_ROOT/pyproject.toml")
|
||||||
command -v npm >/dev/null 2>&1 || { printf 'npm is required.\n' >&2; exit 1; }
|
command -v npm >/dev/null 2>&1 || { printf 'npm is required.\n' >&2; exit 1; }
|
||||||
"$PYTHON_BIN" -c 'import hatchling' >/dev/null 2>&1 || {
|
"$PYTHON_BIN" -c 'import hatchling' >/dev/null 2>&1 || {
|
||||||
printf 'hatchling is required in the release Python environment.\n' >&2
|
printf 'hatchling is required in the release Python environment.\n' >&2
|
||||||
@@ -89,7 +90,7 @@ done
|
|||||||
PACKAGE_ID=$("$PYTHON_BIN" -c \
|
PACKAGE_ID=$("$PYTHON_BIN" -c \
|
||||||
'import hashlib,sys; h=hashlib.sha256(); [h.update(open(path,"rb").read()) for path in sys.argv[1:]]; print(h.hexdigest())' \
|
'import hashlib,sys; h=hashlib.sha256(); [h.update(open(path,"rb").read()) for path in sys.argv[1:]]; print(h.hexdigest())' \
|
||||||
"$STAGED_APP_WHEEL" "$STAGE/app/runtime/runtime-core.txt")
|
"$STAGED_APP_WHEEL" "$STAGE/app/runtime/runtime-core.txt")
|
||||||
printf '%s\n%s\n' "$PACKAGE_VERSION" "$PACKAGE_ID" >"$STAGE/app/runtime/VERSION"
|
printf '%s\n%s\n%s\n' "$PACKAGE_VERSION" "$APP_VERSION" "$PACKAGE_ID" >"$STAGE/app/runtime/VERSION"
|
||||||
"$PYTHON_BIN" "$PROJECT_ROOT/scripts/make_icons.py" "$STAGE"
|
"$PYTHON_BIN" "$PROJECT_ROOT/scripts/make_icons.py" "$STAGE"
|
||||||
chmod 0755 "$STAGE/cmd/"*
|
chmod 0755 "$STAGE/cmd/"*
|
||||||
(
|
(
|
||||||
|
|||||||
@@ -17,7 +17,7 @@ SMOKE_PASSWORD="imagefind-smoke-password"
|
|||||||
SMOKE_POSTGRES_TOKEN="${IMAGEFIND_SMOKE_POSTGRES_TOKEN:-}"
|
SMOKE_POSTGRES_TOKEN="${IMAGEFIND_SMOKE_POSTGRES_TOKEN:-}"
|
||||||
COOKIE_JAR="$DATA_ROOT/run/smoke-cookie.txt"
|
COOKIE_JAR="$DATA_ROOT/run/smoke-cookie.txt"
|
||||||
BACKUP_FILE="$DATA_ROOT/run/smoke.ifbackup"
|
BACKUP_FILE="$DATA_ROOT/run/smoke.ifbackup"
|
||||||
EXPECTED_VERSION=$(sed -n '1p' "$APP_ROOT/runtime/VERSION")
|
EXPECTED_VERSION=$(sed -n '2p' "$APP_ROOT/runtime/VERSION")
|
||||||
|
|
||||||
APP_ROOT=$(CDPATH= cd -- "$APP_ROOT" && pwd)
|
APP_ROOT=$(CDPATH= cd -- "$APP_ROOT" && pwd)
|
||||||
[ "${#SMOKE_POSTGRES_TOKEN}" -ge 20 ] || {
|
[ "${#SMOKE_POSTGRES_TOKEN}" -ge 20 ] || {
|
||||||
|
|||||||
@@ -26,6 +26,11 @@ grep -q '^runtime/wheels/.*\.whl$' "$work_dir/files.txt"
|
|||||||
grep -q '^bin/libOpenCL.so.1$' "$work_dir/files.txt"
|
grep -q '^bin/libOpenCL.so.1$' "$work_dir/files.txt"
|
||||||
runtime_version=$(tar -xOzf "$work_dir/app.tgz" runtime/VERSION | sed -n '1p')
|
runtime_version=$(tar -xOzf "$work_dir/app.tgz" runtime/VERSION | sed -n '1p')
|
||||||
test "$runtime_version" = "$version"
|
test "$runtime_version" = "$version"
|
||||||
|
runtime_app_version=$(tar -xOzf "$work_dir/app.tgz" runtime/VERSION | sed -n '2p')
|
||||||
|
runtime_package_id=$(tar -xOzf "$work_dir/app.tgz" runtime/VERSION | sed -n '3p')
|
||||||
|
test -n "$runtime_app_version"
|
||||||
|
test "${#runtime_package_id}" -eq 64
|
||||||
|
grep -q "^runtime/imagefind-${runtime_app_version}-.*\\.whl$" "$work_dir/files.txt"
|
||||||
if grep -Eq '^(data|models|cache|postgresql)/' "$work_dir/files.txt"; then
|
if grep -Eq '^(data|models|cache|postgresql)/' "$work_dir/files.txt"; then
|
||||||
printf 'persistent data must not be included in ImageFind app.tgz\n' >&2
|
printf 'persistent data must not be included in ImageFind app.tgz\n' >&2
|
||||||
exit 1
|
exit 1
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ def _mock_app(tmp_path: Path) -> tuple[Path, Path, Path]:
|
|||||||
app_root = tmp_path / "app"
|
app_root = tmp_path / "app"
|
||||||
runtime = app_root / "runtime"
|
runtime = app_root / "runtime"
|
||||||
(runtime / "wheels").mkdir(parents=True)
|
(runtime / "wheels").mkdir(parents=True)
|
||||||
(runtime / "VERSION").write_text("0.4.3\n" + "a" * 64 + "\n")
|
(runtime / "VERSION").write_text("0.4.30001\n0.4.3\n" + "a" * 64 + "\n")
|
||||||
(runtime / "runtime-core.txt").write_text("# mocked offline core\n")
|
(runtime / "runtime-core.txt").write_text("# mocked offline core\n")
|
||||||
(runtime / "imagefind-0.4.3-py3-none-any.whl").write_bytes(b"mock wheel")
|
(runtime / "imagefind-0.4.3-py3-none-any.whl").write_bytes(b"mock wheel")
|
||||||
(runtime / "wheels" / "dependency.whl").write_bytes(b"mock dependency")
|
(runtime / "wheels" / "dependency.whl").write_bytes(b"mock dependency")
|
||||||
@@ -312,7 +312,7 @@ def test_runtime_bootstrap_atomically_replaces_existing_current(
|
|||||||
|
|
||||||
assert result.returncode == 0, result.stderr
|
assert result.returncode == 0, result.stderr
|
||||||
assert current.is_symlink()
|
assert current.is_symlink()
|
||||||
assert current.resolve() == runtime_root / "core-0.4.3"
|
assert current.resolve() == runtime_root / "core-0.4.3-aaaaaaaaaaaaaaaa"
|
||||||
version = subprocess.run(
|
version = subprocess.run(
|
||||||
[str(current / "bin" / "python"), "-m", "imagefind.main", "--version"],
|
[str(current / "bin" / "python"), "-m", "imagefind.main", "--version"],
|
||||||
env=env,
|
env=env,
|
||||||
@@ -332,6 +332,29 @@ def test_runtime_bootstrap_atomically_replaces_existing_current(
|
|||||||
assert (previous[0] / "sentinel").read_text() == "preserve old runtime"
|
assert (previous[0] / "sentinel").read_text() == "preserve old runtime"
|
||||||
|
|
||||||
|
|
||||||
|
def test_runtime_bootstrap_reuses_content_id_across_package_builds_and_prunes_old_runtimes(tmp_path: Path):
|
||||||
|
app_root, args_file, password_file = _mock_app(tmp_path)
|
||||||
|
env = _environment(tmp_path, app_root, args_file, password_file)
|
||||||
|
runtime_root = tmp_path / "var" / "runtime"
|
||||||
|
runtime_root.mkdir(parents=True)
|
||||||
|
for name in ("core-0.3.14-old", "core-0.3.16-newer"):
|
||||||
|
old_runtime = runtime_root / name
|
||||||
|
old_runtime.mkdir()
|
||||||
|
(old_runtime / "sentinel").write_text(name)
|
||||||
|
|
||||||
|
first = _run("runtime_bootstrap", env)
|
||||||
|
target = runtime_root / "core-0.4.3-aaaaaaaaaaaaaaaa"
|
||||||
|
assert first.returncode == 0, first.stderr
|
||||||
|
assert target.is_dir()
|
||||||
|
assert len([path for path in runtime_root.glob("core-*") if path.is_dir()]) == 2
|
||||||
|
|
||||||
|
(app_root / "runtime" / "VERSION").write_text("0.4.30002\n0.4.3\n" + "a" * 64 + "\n")
|
||||||
|
second = _run("runtime_bootstrap", env)
|
||||||
|
assert second.returncode == 0, second.stderr
|
||||||
|
assert (runtime_root / "current").resolve() == target
|
||||||
|
assert len([path for path in runtime_root.glob("core-*") if path.is_dir()]) == 2
|
||||||
|
|
||||||
|
|
||||||
def test_config_callback_writes_validated_direct_access_atomically(tmp_path: Path):
|
def test_config_callback_writes_validated_direct_access_atomically(tmp_path: Path):
|
||||||
data_root = tmp_path / "var"
|
data_root = tmp_path / "var"
|
||||||
env = {
|
env = {
|
||||||
|
|||||||
Reference in New Issue
Block a user