imagefind/scripts/smoke-fnos-package.sh

#!/bin/sh
set -eu

PROJECT_ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
APP_ROOT="${1:-$PROJECT_ROOT/.build-fnos/imagefind/app}"
# Keep the default data root on the project filesystem. Resource admission is
# intentionally disk-aware; placing smoke data on a small /tmp tmpfs can make
# a healthy transfer worker wait forever at the production 5 GiB reserve.
DATA_ROOT="${2:-$PROJECT_ROOT/.smoke-fnos}"
PORT="${3:-18765}"
PACKAGE_ROOT="${4:-$PROJECT_ROOT/fnos}"
PYTHON_PATH="${5:-${IMAGEFIND_PYTHON_PATH:-}}"
CONTROL="$PACKAGE_ROOT/cmd/main"
INSTALL_CALLBACK="$PACKAGE_ROOT/cmd/install_callback"
CONFIG_CALLBACK="$PACKAGE_ROOT/cmd/config_callback"
SMOKE_PASSWORD="imagefind-smoke-password"
SMOKE_POSTGRES_TOKEN="${IMAGEFIND_SMOKE_POSTGRES_TOKEN:-}"
COOKIE_JAR="$DATA_ROOT/run/smoke-cookie.txt"
BACKUP_FILE="$DATA_ROOT/run/smoke.ifbackup"
EXPECTED_VERSION=$(sed -n '2p' "$APP_ROOT/runtime/VERSION")

APP_ROOT=$(CDPATH= cd -- "$APP_ROOT" && pwd)
[ "${#SMOKE_POSTGRES_TOKEN}" -ge 20 ] || {
    printf '%s\n' 'Set IMAGEFIND_SMOKE_POSTGRES_TOKEN to a valid nxsir.postgresql enrollment token.' >&2
    exit 2
}
mkdir -p "$DATA_ROOT"
DATA_ROOT=$(CDPATH= cd -- "$DATA_ROOT" && pwd)

cleanup() {
    IMAGEFIND_PYTHON_PATH="$PYTHON_PATH" TRIM_APPDEST="$APP_ROOT" TRIM_PKGVAR="$DATA_ROOT" TRIM_SERVICE_PORT="$PORT" \
        "$CONTROL" stop
    rm -f "$COOKIE_JAR" "$BACKUP_FILE"
}

trap cleanup EXIT HUP INT TERM
IMAGEFIND_PYTHON_PATH="$PYTHON_PATH" TRIM_APPDEST="$APP_ROOT" TRIM_PKGVAR="$DATA_ROOT" \
    wizard_admin_password="$SMOKE_PASSWORD" wizard_admin_password_confirm="$SMOKE_PASSWORD" \
    wizard_postgres_enrollment_token="$SMOKE_POSTGRES_TOKEN" \
    "$INSTALL_CALLBACK"
TRIM_PKGVAR="$DATA_ROOT" wizard_direct_access=true wizard_direct_port="$PORT" \
    "$CONFIG_CALLBACK"
IMAGEFIND_PYTHON_PATH="$PYTHON_PATH" TRIM_APPDEST="$APP_ROOT" TRIM_PKGVAR="$DATA_ROOT" TRIM_SERVICE_PORT="$PORT" \
    "$CONTROL" start

STATUS_BODY=""
ATTEMPT=0
while [ "$ATTEMPT" -lt 90 ]; do
    if STATUS_BODY=$(curl -fsS "http://127.0.0.1:$PORT/api/v1/status" 2>/dev/null); then
        break
    fi
    ATTEMPT=$((ATTEMPT + 1))
    sleep 1
done

if [ -z "$STATUS_BODY" ]; then
    printf 'ImageFind did not become ready; application log follows:\n' >&2
    tail -100 "$DATA_ROOT/log/imagefind.log" >&2 || true
    exit 1
fi

case "$STATUS_BODY" in
    *'"configured":true'*"\"version\":\"$EXPECTED_VERSION\""*) ;;
    *) printf 'Unexpected ImageFind status: %s\n' "$STATUS_BODY" >&2; exit 1 ;;
esac

RUNTIME_PYTHON="$DATA_ROOT/runtime/current/bin/python"
RUNTIME_VERSION=$("$RUNTIME_PYTHON" -m imagefind.main --version)
CORE_RUNTIME=$(
    LD_LIBRARY_PATH="$APP_ROOT/bin${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" \
    XDG_CACHE_HOME="$DATA_ROOT/cache" \
        "$RUNTIME_PYTHON" -c \
        'import fastapi,httpx,imagefind,psycopg,pydantic,uvicorn; print("core-runtime-ok")'
)
case "$CORE_RUNTIME" in
    *'core-runtime-ok'*) ;;
    *) printf 'Offline core runtime check failed: %s\n' "$CORE_RUNTIME" >&2; exit 1 ;;
esac
FRONTEND_HEADERS=$(curl -fsS -D - -o /dev/null "http://127.0.0.1:$PORT/" | sed -n '1,8p')
LOGIN_BODY=$(curl -fsS -c "$COOKIE_JAR" -H 'Content-Type: application/json' \
    --data "{\"password\":\"$SMOKE_PASSWORD\",\"remember_device\":false}" \
    "http://127.0.0.1:$PORT/api/v1/auth/login")
CSRF_TOKEN=$(printf '%s' "$LOGIN_BODY" | sed -n 's/.*"csrf_token":"\([^"]*\)".*/\1/p')
if [ -z "$CSRF_TOKEN" ]; then
    printf 'Unable to obtain CSRF token from packaged server.\n' >&2
    exit 1
fi
BACKUP_STATUS=$(curl -fsS -b "$COOKIE_JAR" "http://127.0.0.1:$PORT/api/v1/backups/status")
MODEL_STATUS=$(curl -fsS -b "$COOKIE_JAR" "http://127.0.0.1:$PORT/api/v1/models")
case "$MODEL_STATUS" in
    *'"runtime_environment":{"state":"missing"'*'"source":"on_demand"'*) ;;
    *) printf 'AI runtime was not reported as on-demand: %s\n' "$MODEL_STATUS" >&2; exit 1 ;;
esac
BACKUP_QUEUE=$(curl -fsS -b "$COOKIE_JAR" -H "X-CSRF-Token: $CSRF_TOKEN" -H 'Content-Type: application/json' \
    --data "{\"scope\":\"keys\",\"password\":\"$SMOKE_PASSWORD\"}" \
    "http://127.0.0.1:$PORT/api/v1/backups")
BACKUP_ID=$(printf '%s' "$BACKUP_QUEUE" | sed -n 's/.*"id":"\([^"]*\)".*/\1/p')
if [ -z "$BACKUP_ID" ]; then
    printf 'Unable to queue packaged background backup: %s\n' "$BACKUP_QUEUE" >&2
    exit 1
fi
BACKUP_EXPORTS=""
ATTEMPT=0
while [ "$ATTEMPT" -lt 90 ]; do
    BACKUP_EXPORTS=$(curl -fsS -b "$COOKIE_JAR" "http://127.0.0.1:$PORT/api/v1/backups")
    case "$BACKUP_EXPORTS" in
        *"\"id\":\"$BACKUP_ID\""*'"status":"completed"'*) break ;;
        *"\"id\":\"$BACKUP_ID\""*'"status":"failed"'*)
            printf 'Packaged background backup failed: %s\n' "$BACKUP_EXPORTS" >&2
            exit 1
            ;;
    esac
    ATTEMPT=$((ATTEMPT + 1))
    sleep 1
done
case "$BACKUP_EXPORTS" in
    *"\"id\":\"$BACKUP_ID\""*'"status":"completed"'*) ;;
    *) printf 'Packaged background backup timed out: %s\n' "$BACKUP_EXPORTS" >&2; exit 1 ;;
esac
curl -fsS -b "$COOKIE_JAR" -o "$BACKUP_FILE" \
    "http://127.0.0.1:$PORT/api/v1/backups/$BACKUP_ID/download"
BACKUP_MAGIC=$(dd if="$BACKUP_FILE" bs=1 count=8 2>/dev/null)
if [ "$BACKUP_MAGIC" != "IFBACKUP" ]; then
    printf 'Packaged backup has an invalid header.\n' >&2
    exit 1
fi
RESTORE_BODY=$(curl -fsS -b "$COOKIE_JAR" -H "X-CSRF-Token: $CSRF_TOKEN" \
    -F "file=@$BACKUP_FILE;type=application/vnd.imagefind.backup" \
    -F "password=$SMOKE_PASSWORD" -F 'confirmed=true' \
    "http://127.0.0.1:$PORT/api/v1/backups/restore")
case "$RESTORE_BODY" in
    *'"scope":"keys"'*'"counts"'*'"scan_jobs":[]'*) ;;
    *) printf 'Unexpected packaged backup restore result: %s\n' "$RESTORE_BODY" >&2; exit 1 ;;
esac
printf 'status=%s\nruntime=%s\ncore_runtime=%s\nbackup_status=%s\nbackup_restore=%s\nfrontend_headers:\n%s\n' \
    "$STATUS_BODY" "$RUNTIME_VERSION" "$CORE_RUNTIME" "$BACKUP_STATUS" "$RESTORE_BODY" \
    "$FRONTEND_HEADERS"