diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 0000000..79b3be8 --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,129 @@ +pipeline { + agent { + node { + label '构建机1' + customWorkspace '/home/nanxunai/goujian/workspace/postgresshare-release' + } + } + + options { + timestamps() + disableConcurrentBuilds() + skipDefaultCheckout(true) + timeout(time: 240, unit: 'MINUTES') + buildDiscarder(logRotator(daysToKeepStr: '30', numToKeepStr: '15')) + } + + environment { + PATH = '/home/nanxunai/.local/bin:/usr/local/bin:/usr/bin:/bin' + DOTNET = '/home/nanxunai/.local/bin/dotnet' + FNPACK_BIN = '/home/nanxunai/.local/bin/fnpack' + OPENLIST_CREDENTIALS = 'openlist_key' + OPENLIST_BASE_URL = 'https://openlist.nxsir.cn' + OPENLIST_REMOTE_DIR = '/yidongpan/构建产物/PostgresShare' + NPM_CONFIG_CACHE = "${WORKSPACE}/.ci/npm-cache" + NUGET_PACKAGES = "${WORKSPACE}/.ci/nuget-packages" + DOTNET_CLI_HOME = "${WORKSPACE}/.ci/dotnet-home" + POSTGRES_SERVICE_BUILD_TMPDIR = "${WORKSPACE}/.ci/fnos-build" + LIVERECORDER_VERIFY_TMPDIR = "${WORKSPACE}/.ci/fnos-verify" + POSTGRES_SERVICE_SMOKE_TMPDIR = "${WORKSPACE}/.ci/fnos-smoke" + } + + stages { + stage('Checkout') { + steps { + deleteDir() + checkout scm + } + } + + stage('Metadata And Preflight') { + steps { + script { + env.APP_VERSION = sh(script: "sed -n 's/^version=//p' fnos-postgresql/manifest | head -n 1", returnStdout: true).trim() + env.SHORT_SHA = sh(script: 'git rev-parse --short=8 HEAD', returnStdout: true).trim() + env.FNOS_VERSION = sh(script: ''' + set -euo pipefail + base="$APP_VERSION"; major=${base%%.*}; remainder=${base#*.} + minor=${remainder%%.*}; patch=${remainder#*.} + test "$base" = "$major.$minor.$patch" && test "$BUILD_NUMBER" -lt 100000 + printf '%s.%s.%s\n' "$major" "$minor" "$((patch * 100000 + BUILD_NUMBER))" + ''', returnStdout: true).trim() + env.IMMUTABLE_TAG = "${env.APP_VERSION}-b${env.BUILD_NUMBER}-${env.SHORT_SHA}" + env.FPK_BASENAME = "nxsir-postgresql-${env.FNOS_VERSION}-x86_64.fpk" + env.FPK_PATH = "${env.WORKSPACE}/artifacts/${env.FPK_BASENAME}" + env.MANIFEST_PATH = "${env.WORKSPACE}/artifacts/PostgresShare-${env.IMMUTABLE_TAG}-build-manifest.json" + currentBuild.displayName = "#${env.BUILD_NUMBER} ${env.IMMUTABLE_TAG}" + currentBuild.description = "fnOS ${env.FNOS_VERSION}" + } + sh ''' + set -euo pipefail + test "$(uname -m)" = x86_64 + for command_name in git curl python3 npm node sha256sum tar apt-get dpkg-deb; do command -v "$command_name" >/dev/null; done + test -x "$DOTNET" && test -x "$FNPACK_BIN" + mkdir -p "$NPM_CONFIG_CACHE" "$NUGET_PACKAGES" "$DOTNET_CLI_HOME" \ + "$POSTGRES_SERVICE_BUILD_TMPDIR" "$LIVERECORDER_VERIFY_TMPDIR" "$POSTGRES_SERVICE_SMOKE_TMPDIR" artifacts + available_kb=$(df -Pk "$WORKSPACE" | awk 'NR == 2 { print $4 }') + test "$available_kb" -ge 4194304 + ''' + } + } + + stage('Restore And Test') { + steps { + sh ''' + set -euo pipefail + npm ci --prefix frontend + npm run build:postgres-admin --prefix frontend + "$DOTNET" restore tests/PostgresService.Tests/PostgresService.Tests.csproj --disable-parallel + "$DOTNET" test tests/PostgresService.Tests/PostgresService.Tests.csproj -c Release --no-restore \ + --logger 'console;verbosity=normal' /maxcpucount:1 + ''' + } + } + + stage('Build Verify And Smoke fnOS x64') { + steps { + sh ''' + set -euo pipefail + PACKAGE_VERSION="$FNOS_VERSION" FNPACK="$FNPACK_BIN" DOTNET="$DOTNET" \ + ./scripts/build-postgresql-fnos-package.sh "$FPK_PATH" + ./scripts/verify-fnos-package.sh "$FPK_PATH" "$FNOS_VERSION" + ./scripts/smoke-postgresql-fnos-package.sh "$FPK_PATH" "$POSTGRES_SERVICE_SMOKE_TMPDIR" + fpk_sha=$(awk '{print $1}' "$FPK_PATH.sha256"); export fpk_sha + python3 - <<'PY' +import json, os, pathlib, subprocess +path = pathlib.Path(os.environ['MANIFEST_PATH']) +payload = { + 'project':'PostgresShare','buildNumber':os.environ['BUILD_NUMBER'], + 'commit':subprocess.check_output(['git','rev-parse','HEAD'],text=True).strip(), + 'productVersion':os.environ['APP_VERSION'],'fnosVersion':os.environ['FNOS_VERSION'], + 'fpk':pathlib.Path(os.environ['FPK_PATH']).name,'fpkSha256':os.environ['fpk_sha'], + 'runtime':'PostgreSQL 15 + pgvector 0.8.6' +} +path.write_text(json.dumps(payload,ensure_ascii=False,indent=2)+'\n',encoding='utf-8') +PY + sha256sum "$MANIFEST_PATH" >"$MANIFEST_PATH.sha256" + ''' + } + } + + stage('Upload Artifacts') { + steps { + withCredentials([usernamePassword(credentialsId: "${OPENLIST_CREDENTIALS}", usernameVariable: 'OPENLIST_USERNAME', passwordVariable: 'OPENLIST_PASSWORD')]) { + sh ''' + set -euo pipefail + for artifact in "$FPK_PATH" "$FPK_PATH.sha256" "$MANIFEST_PATH" "$MANIFEST_PATH.sha256"; do + ./scripts/upload-openlist-artifact.sh "$artifact" "$OPENLIST_REMOTE_DIR" + done + ''' + } + } + } + } + + post { + success { archiveArtifacts artifacts: 'artifacts/*.fpk,artifacts/*.sha256,artifacts/*.json', fingerprint: true } + always { cleanWs(deleteDirs: true, notFailBuild: true) } + } +} diff --git a/scripts/build-postgresql-fnos-package.sh b/scripts/build-postgresql-fnos-package.sh index 9c38c1b..511331d 100755 --- a/scripts/build-postgresql-fnos-package.sh +++ b/scripts/build-postgresql-fnos-package.sh @@ -2,11 +2,19 @@ set -euo pipefail ROOT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) -VERSION=15.1.1 +BASE_VERSION=$(sed -n 's/^version[[:space:]]*=[[:space:]]*//p' "$ROOT_DIR/fnos-postgresql/manifest" | head -n 1 | tr -d '\r') +PACKAGE_VERSION="${PACKAGE_VERSION:-$BASE_VERSION}" +case "$PACKAGE_VERSION" in + ''|*[!0-9.]*|.*|*..*|*.) printf 'invalid fnOS package version: %s\n' "$PACKAGE_VERSION" >&2; exit 1 ;; +esac +[ "$(printf '%s' "$PACKAGE_VERSION" | awk -F. '{ print NF }')" -eq 3 ] || { + printf 'fnOS package version must contain exactly three numeric components: %s\n' "$PACKAGE_VERSION" >&2 + exit 1 +} PGVECTOR_VERSION=0.8.6 PGVECTOR_PACKAGE_VERSION=0.8.6-1.pgdg12%2B1 PGVECTOR_SHA256=b27ff894d1e2d23ebd7528fcb986923391977cbd5c5379ed74527875246854ca -OUTPUT="${1:-$ROOT_DIR/artifacts/fnos/nxsir-postgresql-${VERSION}-x86_64.fpk}" +OUTPUT="${1:-$ROOT_DIR/artifacts/fnos/nxsir-postgresql-${PACKAGE_VERSION}-x86_64.fpk}" DOTNET_BIN="${DOTNET:-dotnet}" NUGET_FEED="${POSTGRES_SERVICE_NUGET_FEED:-}" NUGET_PACKAGES="${NUGET_PACKAGES:-$ROOT_DIR/.cache/nuget-packages}" @@ -47,6 +55,7 @@ RUNTIME_ROOT="$STAGE/app/runtime" EXTRACT_ROOT="$WORK_DIR/runtime-extract" mkdir -p "$STAGE/app/server/wwwroot" "$RUNTIME_ROOT" "$PACKED_ROOT" "$FNPACK_TMP_ROOT" "$EXTRACT_ROOT" cp -a "$ROOT_DIR/fnos-postgresql/." "$STAGE/" +sed -i -E "s/^version[[:space:]]*=.*/version=${PACKAGE_VERSION}/" "$STAGE/manifest" printf 'Building PostgreSQL management frontend...\n' npm run build:postgres-admin --prefix "$ROOT_DIR/frontend" @@ -176,5 +185,5 @@ mv "$built_package" "$OUTPUT" sha256sum "$(basename -- "$OUTPUT")" >"$(basename -- "$OUTPUT").sha256" ) -"$ROOT_DIR/scripts/verify-fnos-package.sh" "$OUTPUT" +"$ROOT_DIR/scripts/verify-fnos-package.sh" "$OUTPUT" "$PACKAGE_VERSION" printf 'Built %s\n' "$OUTPUT" diff --git a/scripts/upload-openlist-artifact.sh b/scripts/upload-openlist-artifact.sh new file mode 100755 index 0000000..8dfe4e9 --- /dev/null +++ b/scripts/upload-openlist-artifact.sh @@ -0,0 +1,189 @@ +#!/usr/bin/env bash +set -euo pipefail + +FILE=${1:?usage: upload-openlist-artifact.sh file [remote-directory]} +REMOTE_DIR=${2:-/yidongpan/构建产物/liverecorder} +OPENLIST_BASE_URL=${OPENLIST_BASE_URL:-https://openlist.nxsir.cn} +OPENLIST_USERNAME=${OPENLIST_USERNAME:?OPENLIST_USERNAME is required} +OPENLIST_PASSWORD=${OPENLIST_PASSWORD:?OPENLIST_PASSWORD is required} +OPENLIST_UPLOAD_ATTEMPTS=${OPENLIST_UPLOAD_ATTEMPTS:-6} + +test -f "$FILE" || { printf 'artifact does not exist: %s\n' "$FILE" >&2; exit 1; } +for required_command in curl python3 stat basename sleep; do + command -v "$required_command" >/dev/null 2>&1 || { + printf 'required upload command is missing: %s\n' "$required_command" >&2 + exit 1 + } +done + +OPENLIST_BASE_URL=${OPENLIST_BASE_URL%/} +REMOTE_DIR="/${REMOTE_DIR#/}" +REMOTE_DIR=${REMOTE_DIR%/} +FILE_NAME=$(basename -- "$FILE") +BUILD_MARKER=${BUILD_TAG:-${BUILD_ID:-$$}} +BUILD_MARKER=$(printf '%s' "$BUILD_MARKER" | tr -c 'A-Za-z0-9._-' '-') +TEMP_NAME=".${FILE_NAME}.uploading-${BUILD_MARKER}" +TEMP_PATH="$REMOTE_DIR/$TEMP_NAME" +FINAL_PATH="$REMOTE_DIR/$FILE_NAME" +LOCAL_SIZE=$(stat -c '%s' "$FILE") +TOKEN= + +json_string() { + python3 -c 'import json,sys; print(json.dumps(sys.argv[1], ensure_ascii=False))' "$1" +} + +envelope_code() { + python3 -c 'import json,sys +try: + value=json.load(sys.stdin) + print(value.get("code", -1)) +except Exception: + print(-1)' +} + +envelope_message() { + python3 -c 'import json,sys +try: + value=json.load(sys.stdin) + print(value.get("message", "unknown OpenList response")) +except Exception: + print("invalid OpenList response")' +} + +login() { + local payload response code + payload=$(printf '{"username":%s,"password":%s}' \ + "$(json_string "$OPENLIST_USERNAME")" \ + "$(json_string "$OPENLIST_PASSWORD")") + response=$(curl --silent --show-error --location --fail-with-body \ + --connect-timeout 30 --max-time 120 \ + --request POST "$OPENLIST_BASE_URL/api/auth/login" \ + --header 'Content-Type: application/json' \ + --data-raw "$payload") || return 1 + code=$(printf '%s' "$response" | envelope_code) + [ "$code" = "200" ] || { + printf 'OpenList login failed: %s\n' "$(printf '%s' "$response" | envelope_message)" >&2 + return 1 + } + TOKEN=$(printf '%s' "$response" | python3 -c 'import json,sys; print((json.load(sys.stdin).get("data") or {}).get("token", ""))') + [ -n "$TOKEN" ] +} + +post_json() { + local endpoint=$1 payload=$2 + curl --silent --show-error --location --fail-with-body \ + --connect-timeout 30 --max-time 180 \ + --request POST "$OPENLIST_BASE_URL$endpoint" \ + --header "Authorization: $TOKEN" \ + --header 'Content-Type: application/json' \ + --data-raw "$payload" +} + +remote_size() { + local path=$1 response code + response=$(post_json /api/fs/get "{\"path\":$(json_string "$path"),\"password\":\"\"}") || return 1 + code=$(printf '%s' "$response" | envelope_code) + [ "$code" = "200" ] || return 1 + printf '%s' "$response" | python3 -c 'import json,sys; print((json.load(sys.stdin).get("data") or {}).get("size", -1))' +} + +ensure_remote_directory() { + local current= segment response code + while IFS= read -r segment; do + [ -n "$segment" ] || continue + current="$current/$segment" + response=$(post_json /api/fs/list "{\"path\":$(json_string "$current"),\"password\":\"\",\"refresh\":false,\"page\":1,\"per_page\":1}" || true) + code=$(printf '%s' "$response" | envelope_code) + if [ "$code" = "200" ]; then + continue + fi + response=$(post_json /api/fs/mkdir "{\"path\":$(json_string "$current")}") || return 1 + code=$(printf '%s' "$response" | envelope_code) + [ "$code" = "200" ] || { + printf 'OpenList mkdir failed for %s: %s\n' "$current" "$(printf '%s' "$response" | envelope_message)" >&2 + return 1 + } + done < <(python3 -c 'import sys; print("\n".join(part for part in sys.argv[1].split("/") if part))' "$REMOTE_DIR") +} + +remove_temporary_file() { + local response + [ -n "$TOKEN" ] || login >/dev/null 2>&1 || return 0 + response=$(post_json /api/fs/remove "{\"dir\":$(json_string "$REMOTE_DIR"),\"names\":[$(json_string "$TEMP_NAME")]}" 2>/dev/null || true) + [ "$(printf '%s' "$response" | envelope_code)" = "200" ] || true +} + +publish_temporary_file() { + local response code + response=$(post_json /api/fs/rename "{\"path\":$(json_string "$TEMP_PATH"),\"name\":$(json_string "$FILE_NAME")}") || return 1 + code=$(printf '%s' "$response" | envelope_code) + [ "$code" = "200" ] || { + printf 'OpenList rename failed: %s\n' "$(printf '%s' "$response" | envelope_message)" >&2 + return 1 + } +} + +upload_once() { + local encoded_path response code uploaded_size + encoded_path=$(python3 -c 'import sys,urllib.parse; print(urllib.parse.quote(sys.argv[1], safe=""))' "$TEMP_PATH") + response=$(curl --silent --show-error --location --fail-with-body \ + --connect-timeout 30 --max-time 3600 --speed-limit 1024 --speed-time 120 \ + --request PUT "$OPENLIST_BASE_URL/api/fs/put" \ + --header "Authorization: $TOKEN" \ + --header "File-Path: $encoded_path" \ + --header 'As-Task: false' \ + --header 'Content-Type: application/octet-stream' \ + --header 'Expect:' \ + --data-binary "@$FILE") || return 1 + code=$(printf '%s' "$response" | envelope_code) + [ "$code" = "200" ] || { + printf 'OpenList upload failed: %s\n' "$(printf '%s' "$response" | envelope_message)" >&2 + return 1 + } + uploaded_size=$(remote_size "$TEMP_PATH") || return 1 + [ "$uploaded_size" = "$LOCAL_SIZE" ] || { + printf 'OpenList size mismatch for temporary upload: local=%s remote=%s\n' "$LOCAL_SIZE" "$uploaded_size" >&2 + return 1 + } +} + +printf 'Uploading %s (%s bytes) to OpenList %s\n' "$FILE_NAME" "$LOCAL_SIZE" "$REMOTE_DIR" + +for attempt in $(seq 1 "$OPENLIST_UPLOAD_ATTEMPTS"); do + TOKEN= + upload_ready=1 + if login && ensure_remote_directory; then + existing_size=$(remote_size "$FINAL_PATH" 2>/dev/null || true) + if [ "$existing_size" = "$LOCAL_SIZE" ]; then + printf 'OpenList artifact already exists with matching size: %s\n' "$FINAL_PATH" + exit 0 + fi + if [ -n "$existing_size" ]; then + remove_response=$(post_json /api/fs/remove "{\"dir\":$(json_string "$REMOTE_DIR"),\"names\":[$(json_string "$FILE_NAME")]}" || true) + if [ "$(printf '%s' "$remove_response" | envelope_code)" != "200" ]; then + printf 'Unable to remove mismatched OpenList artifact before retry: %s\n' "$FINAL_PATH" >&2 + upload_ready=0 + fi + fi + + if [ "$upload_ready" = "1" ] && upload_once && publish_temporary_file; then + final_size=$(remote_size "$FINAL_PATH") || final_size=-1 + if [ "$final_size" = "$LOCAL_SIZE" ]; then + printf 'OpenList upload verified: %s (%s bytes)\n' "$FINAL_PATH" "$final_size" + exit 0 + fi + printf 'OpenList final size mismatch: local=%s remote=%s\n' "$LOCAL_SIZE" "$final_size" >&2 + fi + fi + + if [ "$attempt" -lt "$OPENLIST_UPLOAD_ATTEMPTS" ]; then + delay=$((5 * (1 << (attempt - 1)))) + [ "$delay" -le 120 ] || delay=120 + printf 'OpenList upload attempt %s/%s failed; retrying in %ss\n' "$attempt" "$OPENLIST_UPLOAD_ATTEMPTS" "$delay" >&2 + sleep "$delay" + fi +done + +remove_temporary_file +printf 'OpenList upload failed after %s attempts: %s\n' "$OPENLIST_UPLOAD_ATTEMPTS" "$FILE_NAME" >&2 +exit 1 diff --git a/scripts/verify-fnos-package.sh b/scripts/verify-fnos-package.sh index 30e443f..116f8b6 100755 --- a/scripts/verify-fnos-package.sh +++ b/scripts/verify-fnos-package.sh @@ -1,7 +1,8 @@ #!/bin/bash set -euo pipefail -PACKAGE=${1:?usage: verify-fnos-package.sh package.fpk} +PACKAGE=${1:?usage: verify-fnos-package.sh package.fpk [expected-version]} +EXPECTED_VERSION=${2:-} VERIFY_TMP_ROOT="${LIVERECORDER_VERIFY_TMPDIR:-${TMPDIR:-/tmp}}" MAX_APP_UNCOMPRESSED_BYTES=$((512 * 1024 * 1024)) mkdir -p "$VERIFY_TMP_ROOT" @@ -17,6 +18,10 @@ appname=$(manifest_value appname) version=$(manifest_value version) test "$appname" = "nxsir.postgresql" || { printf 'unexpected fnOS appname: %s\n' "$appname" >&2; exit 1; } test -n "$version" +if [ -n "$EXPECTED_VERSION" ] && [ "$version" != "$EXPECTED_VERSION" ]; then + printf 'unexpected fnOS version: expected %s, got %s\n' "$EXPECTED_VERSION" "$version" >&2 + exit 1 +fi test "$(manifest_value platform)" = "x86" test -x "$WORK_DIR/cmd/main" test -x "$WORK_DIR/cmd/install_callback"