refactor: move PostgreSQL shared service to dedicated repository

This commit is contained in:
2026-08-03 23:59:25 +08:00
parent ecc737f0bd
commit e0d3969e46
34 changed files with 18 additions and 2637 deletions
-177
View File
@@ -1,177 +0,0 @@
#!/bin/bash
set -euo pipefail
ROOT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
VERSION=15.1.1
PGVECTOR_VERSION=0.8.6
PGVECTOR_PACKAGE_VERSION=0.8.6-1.pgdg12%2B1
PGVECTOR_SHA256=b27ff894d1e2d23ebd7528fcb986923391977cbd5c5379ed74527875246854ca
OUTPUT="${1:-$ROOT_DIR/artifacts/fnos/nxsir-postgresql-${VERSION}-x86_64.fpk}"
WORKSPACE_CACHE=$(CDPATH= cd -- "$ROOT_DIR/.." && pwd)
DOTNET_BIN="${DOTNET:-$WORKSPACE_CACHE/.dotnet8/dotnet}"
NUGET_FEED="${POSTGRES_SERVICE_NUGET_FEED:-$WORKSPACE_CACHE/.nuget-feed}"
NUGET_PACKAGES="${NUGET_PACKAGES:-$WORKSPACE_CACHE/.nuget-packages}"
DOTNET_CLI_HOME="${DOTNET_CLI_HOME:-$WORKSPACE_CACHE/.dotnet-cli-home}"
BUILD_TMP_ROOT="${POSTGRES_SERVICE_BUILD_TMPDIR:-$WORKSPACE_CACHE/.fnos-build-tmp}"
if [ -n "${FNPACK:-}" ]; then
FNPACK_BIN="$FNPACK"
elif [ -x "$ROOT_DIR/.tools/fnpack" ]; then
FNPACK_BIN="$ROOT_DIR/.tools/fnpack"
else
FNPACK_BIN=fnpack
fi
FNPACK_BIN=$(command -v "$FNPACK_BIN") || {
printf 'fnpack is required; install it from the fnOS developer portal or set FNPACK.\n' >&2
exit 1
}
for command_name in npm apt-get curl dpkg-deb node sha256sum tar find realpath; do
command -v "$command_name" >/dev/null 2>&1 || {
printf 'required build command is missing: %s\n' "$command_name" >&2
exit 1
}
done
test -x "$DOTNET_BIN" || { printf 'missing .NET SDK: %s\n' "$DOTNET_BIN" >&2; exit 1; }
test -d "$NUGET_FEED" || { printf 'missing offline NuGet feed: %s\n' "$NUGET_FEED" >&2; exit 1; }
mkdir -p "$BUILD_TMP_ROOT" "$(dirname -- "$OUTPUT")"
WORK_DIR=$(mktemp -d "${BUILD_TMP_ROOT%/}/postgres-service-fnos-build.XXXXXX")
trap 'rm -rf -- "$WORK_DIR"' EXIT
STAGE="$WORK_DIR/stage"
PACKED_ROOT="$WORK_DIR/packed"
FNPACK_TMP_ROOT="$WORK_DIR/fnpack-tmp"
RUNTIME_ROOT="$STAGE/app/runtime"
EXTRACT_ROOT="$WORK_DIR/runtime-extract"
mkdir -p "$STAGE/app/server/wwwroot" "$RUNTIME_ROOT" "$PACKED_ROOT" "$FNPACK_TMP_ROOT" "$EXTRACT_ROOT"
cp -a "$ROOT_DIR/fnos-postgresql/." "$STAGE/"
printf 'Building PostgreSQL management frontend...\n'
npm run build:postgres-admin --prefix "$ROOT_DIR/frontend"
cp -a "$ROOT_DIR/frontend/dist-postgres/." "$STAGE/app/server/wwwroot/"
printf 'Publishing self-contained PostgreSQL management API...\n'
export NUGET_PACKAGES DOTNET_CLI_HOME
"$DOTNET_BIN" restore "$ROOT_DIR/src/PostgresService.WebApi/PostgresService.WebApi.csproj" \
-r linux-x64 \
--source "$NUGET_FEED" \
--disable-parallel
"$DOTNET_BIN" publish "$ROOT_DIR/src/PostgresService.WebApi/PostgresService.WebApi.csproj" \
-c Release \
-r linux-x64 \
--self-contained true \
--no-restore \
-p:DebugType=None \
-p:DebugSymbols=false \
-p:PublishSingleFile=false \
-p:PublishReadyToRun=false \
-o "$STAGE/app/server" \
/maxcpucount:1
rm -f "$STAGE/app/server/"*.pdb
printf 'Downloading pinned Debian Bookworm PostgreSQL 15 runtime...\n'
APT_ROOT="$WORK_DIR/apt"
mkdir -p \
"$APT_ROOT/etc/apt" \
"$APT_ROOT/var/lib/apt/lists/partial" \
"$APT_ROOT/var/lib/dpkg" \
"$APT_ROOT/var/cache/apt/archives/partial"
cp "$ROOT_DIR/scripts/fnos-bookworm.sources.list" "$APT_ROOT/etc/apt/sources.list"
touch "$APT_ROOT/var/lib/dpkg/status"
APT_OPTIONS=(
-o "Dir::Etc::sourcelist=$APT_ROOT/etc/apt/sources.list"
-o "Dir::Etc::sourceparts=-"
-o "Dir::State::status=$APT_ROOT/var/lib/dpkg/status"
-o "Dir::State::lists=$APT_ROOT/var/lib/apt/lists"
-o "Dir::Cache=$APT_ROOT/var/cache/apt"
-o "Dir::Cache::archives=$APT_ROOT/var/cache/apt/archives"
-o "Debug::NoLocking=1"
-o "APT::Architecture=amd64"
-o "Acquire::Languages=none"
)
apt-get "${APT_OPTIONS[@]}" update
apt-get "${APT_OPTIONS[@]}" --download-only --no-install-recommends --yes install \
postgresql-15 postgresql-client-15
shopt -s nullglob
runtime_packages=("$APT_ROOT"/var/cache/apt/archives/*.deb)
test "${#runtime_packages[@]}" -gt 0 || { printf 'APT did not download PostgreSQL runtime packages\n' >&2; exit 1; }
for package_file in "${runtime_packages[@]}"; do
dpkg-deb -x "$package_file" "$EXTRACT_ROOT"
done
shopt -u nullglob
printf 'Downloading pinned pgvector %s extension...\n' "$PGVECTOR_VERSION"
PGVECTOR_DEB="$WORK_DIR/postgresql-15-pgvector.deb"
curl --fail --location --retry 3 \
"https://apt.postgresql.org/pub/repos/apt/pool/main/p/pgvector/postgresql-15-pgvector_${PGVECTOR_PACKAGE_VERSION}_amd64.deb" \
--output "$PGVECTOR_DEB"
printf '%s %s\n' "$PGVECTOR_SHA256" "$PGVECTOR_DEB" | sha256sum --check --status
test "$(dpkg-deb -f "$PGVECTOR_DEB" Package)" = "postgresql-15-pgvector"
test "$(dpkg-deb -f "$PGVECTOR_DEB" Version)" = "${PGVECTOR_PACKAGE_VERSION//%2B/+}"
dpkg-deb -x "$PGVECTOR_DEB" "$EXTRACT_ROOT"
printf 'Assembling relocatable PostgreSQL runtime...\n'
cp -a "$EXTRACT_ROOT/." "$RUNTIME_ROOT/"
rm -rf \
"$RUNTIME_ROOT/usr/share/doc" \
"$RUNTIME_ROOT/usr/share/man" \
"$RUNTIME_ROOT/usr/share/locale" \
"$RUNTIME_ROOT/var" \
"$RUNTIME_ROOT/etc/init.d" \
"$RUNTIME_ROOT/usr/sbin"
# fnOS provides the matching glibc and loader. Keep other Debian libraries but
# never package a second libc implementation into the application runtime.
find "$RUNTIME_ROOT" -type f \( \
-name 'ld-linux-*.so.*' -o -name 'libc.so.*' -o -name 'libm.so.*' -o \
-name 'libmvec.so.*' -o -name 'libpthread.so.*' -o -name 'libdl.so.*' -o \
-name 'librt.so.*' -o -name 'libresolv.so.*' -o -name 'libutil.so.*' \
\) -delete
# Official fnOS packages reject absolute links. Rewrite Debian links to remain
# inside the staged runtime and fail if a link target was not packaged.
while IFS= read -r -d '' link_path; do
target=$(readlink -- "$link_path")
case "$target" in
/*)
staged_target="$RUNTIME_ROOT$target"
if [ ! -e "$staged_target" ]; then
rm "$link_path"
continue
fi
relative_target=$(realpath --relative-to="$(dirname -- "$link_path")" "$staged_target")
rm "$link_path"
ln -s "$relative_target" "$link_path"
;;
esac
done < <(find "$RUNTIME_ROOT" -type l -print0)
for required_file in \
"$RUNTIME_ROOT/usr/lib/postgresql/15/bin/postgres" \
"$RUNTIME_ROOT/usr/lib/postgresql/15/bin/initdb" \
"$RUNTIME_ROOT/usr/lib/postgresql/15/bin/pg_ctl" \
"$RUNTIME_ROOT/usr/lib/postgresql/15/bin/pg_dump" \
"$RUNTIME_ROOT/usr/lib/postgresql/15/lib/vector.so" \
"$RUNTIME_ROOT/usr/share/postgresql/15/extension/vector.control"; do
test -e "$required_file" || { printf 'runtime file is missing: %s\n' "$required_file" >&2; exit 1; }
done
node "$ROOT_DIR/scripts/generate-fnos-icons.mjs" "$STAGE" "$STAGE/app/ui/images"
chmod 0755 "$STAGE/cmd/"*
printf 'Packing PostgreSQL service with official fnOS fnpack...\n'
(
cd "$PACKED_ROOT"
TMPDIR="$FNPACK_TMP_ROOT" "$FNPACK_BIN" build --directory "$STAGE"
)
built_package=$(find "$PACKED_ROOT" -maxdepth 1 -type f -name '*.fpk' -print -quit)
test -n "$built_package" || { printf 'fnpack did not create an FPK\n' >&2; exit 1; }
mv "$built_package" "$OUTPUT"
(
cd "$(dirname -- "$OUTPUT")"
sha256sum "$(basename -- "$OUTPUT")" >"$(basename -- "$OUTPUT").sha256"
)
"$ROOT_DIR/scripts/verify-fnos-package.sh" "$OUTPUT"
printf 'Built %s\n' "$OUTPUT"
-149
View File
@@ -1,149 +0,0 @@
#!/bin/bash
set -euo pipefail
PACKAGE=${1:?usage: smoke-postgresql-fnos-package.sh postgresql-service.fpk [temporary-directory]}
SMOKE_TMP_ROOT="${2:-${POSTGRES_SERVICE_SMOKE_TMPDIR:-${TMPDIR:-/tmp}}}"
mkdir -p "$SMOKE_TMP_ROOT"
SMOKE_TMP_ROOT=$(CDPATH= cd -- "$SMOKE_TMP_ROOT" && pwd)
WORK_DIR=$(mktemp -d "${SMOKE_TMP_ROOT%/}/postgres-service-fnos-smoke.XXXXXX")
PACKAGE_ROOT="$WORK_DIR/package"
APP_ROOT="$WORK_DIR/app"
DATA_ROOT="$WORK_DIR/var"
VOLUME_ROOT="$WORK_DIR/volume"
API_PORT=${POSTGRES_SERVICE_SMOKE_API_PORT:-19433}
PG_PORT=${POSTGRES_SERVICE_SMOKE_PG_PORT:-19432}
ADMIN_PASSWORD='Postgres-Admin-Smoke-2026!'
ENROLLMENT_TOKEN='Postgres-Enrollment-Smoke-2026!'
CONTROL="$PACKAGE_ROOT/cmd/main"
cleanup() {
status=$?
if [ -x "$CONTROL" ]; then
TRIM_APPDEST="$APP_ROOT" TRIM_PKGVAR="$DATA_ROOT" TRIM_APPDEST_VOL="$VOLUME_ROOT" \
TRIM_SERVICE_PORT="$API_PORT" POSTGRES_SERVICE_PORT="$PG_PORT" \
"$CONTROL" stop >/dev/null 2>&1 || true
fi
if [ "$status" -ne 0 ]; then
printf '%s\n' 'PostgreSQL fnOS smoke test failed; service logs follow:' >&2
for log_file in "$DATA_ROOT/log/postgresql.log" "$DATA_ROOT/log/postgres-service.log"; do
if [ -f "$log_file" ]; then
printf '%s\n' "--- $log_file ---" >&2
tail -n 160 "$log_file" >&2 || true
fi
done
fi
rm -rf -- "$WORK_DIR"
return "$status"
}
trap cleanup EXIT HUP INT TERM
mkdir -p "$PACKAGE_ROOT" "$APP_ROOT" "$VOLUME_ROOT"
tar -xzf "$PACKAGE" -C "$PACKAGE_ROOT"
tar -xzf "$PACKAGE_ROOT/app.tgz" -C "$APP_ROOT"
if [ -n "${POSTGRES_SERVICE_SMOKE_SERVER_OVERLAY:-}" ]; then
test -x "$POSTGRES_SERVICE_SMOKE_SERVER_OVERLAY/PostgresService.WebApi" || {
printf 'invalid API overlay: %s\n' "$POSTGRES_SERVICE_SMOKE_SERVER_OVERLAY" >&2
exit 1
}
cp -a "$POSTGRES_SERVICE_SMOKE_SERVER_OVERLAY/." "$APP_ROOT/server/"
fi
export TRIM_APPDEST="$APP_ROOT"
export TRIM_PKGVAR="$DATA_ROOT"
export TRIM_APPDEST_VOL="$VOLUME_ROOT"
export TRIM_SERVICE_PORT="$API_PORT"
export POSTGRES_SERVICE_PORT="$PG_PORT"
wizard_postgres_admin_password="$ADMIN_PASSWORD" \
wizard_postgres_admin_password_confirm="$ADMIN_PASSWORD" \
wizard_postgres_enrollment_token="$ENROLLMENT_TOKEN" \
wizard_postgres_enrollment_token_confirm="$ENROLLMENT_TOKEN" \
"$PACKAGE_ROOT/cmd/install_callback"
"$CONTROL" start
"$CONTROL" status
BASE_URL="http://127.0.0.1:$API_PORT"
curl -fsS "$BASE_URL/health/ready" | grep -q '"status":"ready"'
curl -fsS "$BASE_URL/" | grep -q '<div id="app"></div>'
COOKIE_JAR="$WORK_DIR/cookies.txt"
curl -fsS -c "$COOKIE_JAR" -H 'Content-Type: application/json' \
--data "{\"username\":\"admin\",\"password\":\"$ADMIN_PASSWORD\"}" \
"$BASE_URL/api/v1/auth/login" >/dev/null
curl -fsS -b "$COOKIE_JAR" "$BASE_URL/api/v1/overview" | grep -q '"version"'
curl -fsS -b "$COOKIE_JAR" "$BASE_URL/api/v1/sessions" | grep -q '^\['
enroll() {
app_id=$1
display_name=$2
extensions=$3
destination=$4
curl -fsS \
-H "Authorization: Bearer $ENROLLMENT_TOKEN" \
-H 'Content-Type: application/json' \
--data "{\"appId\":\"$app_id\",\"displayName\":\"$display_name\",\"requestedExtensions\":$extensions}" \
"$BASE_URL/internal/v1/enroll" >"$destination"
}
enroll liverecorder 'Live Recorder' '[]' "$WORK_DIR/live.json"
enroll imagefind-test 'ImageFind Test Client' '["vector"]' "$WORK_DIR/image.json"
json_field() {
field=$1
file=$2
sed -n "s/.*\"$field\":\"\([^\"]*\)\".*/\1/p" "$file"
}
LIVE_DB=$(json_field database "$WORK_DIR/live.json")
LIVE_USER=$(json_field username "$WORK_DIR/live.json")
LIVE_PASSWORD=$(json_field password "$WORK_DIR/live.json")
IMAGE_DB=$(json_field database "$WORK_DIR/image.json")
IMAGE_USER=$(json_field username "$WORK_DIR/image.json")
IMAGE_PASSWORD=$(json_field password "$WORK_DIR/image.json")
test -n "$LIVE_DB" && test -n "$LIVE_USER" && test -n "$LIVE_PASSWORD"
test -n "$IMAGE_DB" && test -n "$IMAGE_USER" && test -n "$IMAGE_PASSWORD"
test "$LIVE_DB" != "$IMAGE_DB"
test "$LIVE_USER" != "$IMAGE_USER"
PG_BIN="$APP_ROOT/runtime/usr/lib/postgresql/15/bin"
PG_LIB="$APP_ROOT/runtime/usr/lib/postgresql/15/lib"
RUNTIME_LIBS="$APP_ROOT/runtime/usr/lib/x86_64-linux-gnu:$APP_ROOT/runtime/lib/x86_64-linux-gnu:$PG_LIB"
run_client_psql() {
password=$1
shift
env LD_LIBRARY_PATH="$RUNTIME_LIBS" PGPASSWORD="$password" "$PG_BIN/psql" "$@"
}
run_client_psql "$LIVE_PASSWORD" -h 127.0.0.1 -p "$PG_PORT" -U "$LIVE_USER" -d "$LIVE_DB" \
-v ON_ERROR_STOP=1 -c 'CREATE TABLE smoke_live(id integer PRIMARY KEY, value text); INSERT INTO smoke_live VALUES (1, '\''live'\'');' >/dev/null
run_client_psql "$IMAGE_PASSWORD" -h 127.0.0.1 -p "$PG_PORT" -U "$IMAGE_USER" -d "$IMAGE_DB" \
-v ON_ERROR_STOP=1 -c 'CREATE TABLE smoke_vectors(id integer PRIMARY KEY, embedding vector(3)); INSERT INTO smoke_vectors VALUES (1, '\''[1,2,3]'\''); SELECT embedding <-> '\''[1,2,4]'\'' FROM smoke_vectors;' >/dev/null
if run_client_psql "$LIVE_PASSWORD" -h 127.0.0.1 -p "$PG_PORT" -U "$LIVE_USER" -d "$IMAGE_DB" -Atqc 'SELECT 1' >/dev/null 2>&1; then
printf '%s\n' 'role isolation failed: Live Recorder connected to ImageFind database' >&2
exit 1
fi
curl -fsS -b "$COOKIE_JAR" -H 'Content-Type: application/json' \
--data "{\"database\":\"$IMAGE_DB\",\"sql\":\"SELECT count(*) FROM smoke_vectors\"}" \
"$BASE_URL/api/v1/query" | grep -q '"rowCount":1'
if curl -fsS -b "$COOKIE_JAR" -H 'Content-Type: application/json' \
--data "{\"database\":\"$IMAGE_DB\",\"sql\":\"DELETE FROM smoke_vectors\"}" \
"$BASE_URL/api/v1/query" >/dev/null 2>&1; then
printf '%s\n' 'read-only SQL console accepted a write statement' >&2
exit 1
fi
curl -fsS -b "$COOKIE_JAR" -H 'Content-Type: application/json' \
--data "{\"database\":\"$LIVE_DB\"}" "$BASE_URL/api/v1/backups" | grep -q '"sha256"'
POSTMASTER_PID=$(sed -n '1p' "$DATA_ROOT/postgres/postmaster.pid")
test -n "$POSTMASTER_PID"
kill -0 "$POSTMASTER_PID"
"$CONTROL" restart
"$CONTROL" status
curl -fsS "$BASE_URL/health/ready" | grep -q '"status":"ready"'
run_client_psql "$LIVE_PASSWORD" -h 127.0.0.1 -p "$PG_PORT" -U "$LIVE_USER" -d "$LIVE_DB" -Atqc 'SELECT value FROM smoke_live WHERE id = 1' | grep -q '^live$'
printf '%s\n' 'PostgreSQL fnOS smoke test passed: shared process, SCRAM isolation, pgvector, read-only SQL, backup and restart are ready'
+9 -22
View File
@@ -15,10 +15,7 @@ manifest_value() {
tar -xzf "$PACKAGE" -C "$WORK_DIR"
appname=$(manifest_value appname)
version=$(manifest_value version)
case "$appname" in
liverecorder|nxsir.postgresql) ;;
*) printf 'unexpected fnOS appname: %s\n' "$appname" >&2; exit 1 ;;
esac
test "$appname" = "liverecorder" || { printf 'unexpected fnOS appname: %s\n' "$appname" >&2; exit 1; }
test -n "$version"
test "$(manifest_value platform)" = "x86"
test -x "$WORK_DIR/cmd/main"
@@ -76,24 +73,14 @@ grep -q '^server/wwwroot/index.html$' "$WORK_DIR/app-files.txt"
grep -q '^ui/config$' "$WORK_DIR/app-files.txt"
grep -q '^ui/images/icon_64.png$' "$WORK_DIR/app-files.txt"
if [ "$appname" = "liverecorder" ]; then
grep -q '^server/LiveRecorder.WebApi$' "$WORK_DIR/app-files.txt"
grep -q '^server/Platforms/Douyin/Signing/sign-xbogus.js$' "$WORK_DIR/app-files.txt"
grep -q '^runtime/bin/curl$' "$WORK_DIR/app-files.txt"
grep -q '^runtime/etc/ssl/certs/ca-certificates.crt$' "$WORK_DIR/app-files.txt"
test "$(manifest_value install_dep_apps)" = "nxsir.postgresql:nodejs_v22"
if grep -Eq '^runtime/(bin/node|usr/(lib|share)/postgresql/|lib/(libLLVM|libz3))' "$WORK_DIR/app-files.txt"; then
printf 'Live Recorder must use shared PostgreSQL and the fnOS nodejs_v22 dependency\n' >&2
exit 1
fi
else
grep -q '^server/PostgresService.WebApi$' "$WORK_DIR/app-files.txt"
grep -q '^runtime/usr/lib/postgresql/15/bin/postgres$' "$WORK_DIR/app-files.txt"
grep -q '^runtime/usr/lib/postgresql/15/bin/initdb$' "$WORK_DIR/app-files.txt"
grep -q '^runtime/usr/lib/postgresql/15/bin/pg_ctl$' "$WORK_DIR/app-files.txt"
grep -q '^runtime/usr/share/postgresql/15/postgresql.conf.sample$' "$WORK_DIR/app-files.txt"
grep -q '^runtime/usr/lib/postgresql/15/lib/vector.so$' "$WORK_DIR/app-files.txt"
grep -q '^runtime/usr/share/postgresql/15/extension/vector.control$' "$WORK_DIR/app-files.txt"
grep -q '^server/LiveRecorder.WebApi$' "$WORK_DIR/app-files.txt"
grep -q '^server/Platforms/Douyin/Signing/sign-xbogus.js$' "$WORK_DIR/app-files.txt"
grep -q '^runtime/bin/curl$' "$WORK_DIR/app-files.txt"
grep -q '^runtime/etc/ssl/certs/ca-certificates.crt$' "$WORK_DIR/app-files.txt"
test "$(manifest_value install_dep_apps)" = "nxsir.postgresql:nodejs_v22"
if grep -Eq '^runtime/(bin/node|usr/(lib|share)/postgresql/|lib/(libLLVM|libz3))' "$WORK_DIR/app-files.txt"; then
printf 'Live Recorder must use shared PostgreSQL and the fnOS nodejs_v22 dependency\n' >&2
exit 1
fi
if grep -Eq '^runtime/.*/(ffmpeg|ffprobe)$' "$WORK_DIR/app-files.txt"; then