ci: add fnOS release pipeline and build versions
This commit is contained in:
Vendored
+129
@@ -0,0 +1,129 @@
|
||||
pipeline {
|
||||
agent {
|
||||
node {
|
||||
label '构建机1'
|
||||
customWorkspace '/home/nanxunai/goujian/workspace/postgresshare-release'
|
||||
}
|
||||
}
|
||||
|
||||
options {
|
||||
timestamps()
|
||||
disableConcurrentBuilds()
|
||||
skipDefaultCheckout(true)
|
||||
timeout(time: 240, unit: 'MINUTES')
|
||||
buildDiscarder(logRotator(daysToKeepStr: '30', numToKeepStr: '15'))
|
||||
}
|
||||
|
||||
environment {
|
||||
PATH = '/home/nanxunai/.local/bin:/usr/local/bin:/usr/bin:/bin'
|
||||
DOTNET = '/home/nanxunai/.local/bin/dotnet'
|
||||
FNPACK_BIN = '/home/nanxunai/.local/bin/fnpack'
|
||||
OPENLIST_CREDENTIALS = 'openlist_key'
|
||||
OPENLIST_BASE_URL = 'https://openlist.nxsir.cn'
|
||||
OPENLIST_REMOTE_DIR = '/yidongpan/构建产物/PostgresShare'
|
||||
NPM_CONFIG_CACHE = "${WORKSPACE}/.ci/npm-cache"
|
||||
NUGET_PACKAGES = "${WORKSPACE}/.ci/nuget-packages"
|
||||
DOTNET_CLI_HOME = "${WORKSPACE}/.ci/dotnet-home"
|
||||
POSTGRES_SERVICE_BUILD_TMPDIR = "${WORKSPACE}/.ci/fnos-build"
|
||||
LIVERECORDER_VERIFY_TMPDIR = "${WORKSPACE}/.ci/fnos-verify"
|
||||
POSTGRES_SERVICE_SMOKE_TMPDIR = "${WORKSPACE}/.ci/fnos-smoke"
|
||||
}
|
||||
|
||||
stages {
|
||||
stage('Checkout') {
|
||||
steps {
|
||||
deleteDir()
|
||||
checkout scm
|
||||
}
|
||||
}
|
||||
|
||||
stage('Metadata And Preflight') {
|
||||
steps {
|
||||
script {
|
||||
env.APP_VERSION = sh(script: "sed -n 's/^version=//p' fnos-postgresql/manifest | head -n 1", returnStdout: true).trim()
|
||||
env.SHORT_SHA = sh(script: 'git rev-parse --short=8 HEAD', returnStdout: true).trim()
|
||||
env.FNOS_VERSION = sh(script: '''
|
||||
set -euo pipefail
|
||||
base="$APP_VERSION"; major=${base%%.*}; remainder=${base#*.}
|
||||
minor=${remainder%%.*}; patch=${remainder#*.}
|
||||
test "$base" = "$major.$minor.$patch" && test "$BUILD_NUMBER" -lt 100000
|
||||
printf '%s.%s.%s\n' "$major" "$minor" "$((patch * 100000 + BUILD_NUMBER))"
|
||||
''', returnStdout: true).trim()
|
||||
env.IMMUTABLE_TAG = "${env.APP_VERSION}-b${env.BUILD_NUMBER}-${env.SHORT_SHA}"
|
||||
env.FPK_BASENAME = "nxsir-postgresql-${env.FNOS_VERSION}-x86_64.fpk"
|
||||
env.FPK_PATH = "${env.WORKSPACE}/artifacts/${env.FPK_BASENAME}"
|
||||
env.MANIFEST_PATH = "${env.WORKSPACE}/artifacts/PostgresShare-${env.IMMUTABLE_TAG}-build-manifest.json"
|
||||
currentBuild.displayName = "#${env.BUILD_NUMBER} ${env.IMMUTABLE_TAG}"
|
||||
currentBuild.description = "fnOS ${env.FNOS_VERSION}"
|
||||
}
|
||||
sh '''
|
||||
set -euo pipefail
|
||||
test "$(uname -m)" = x86_64
|
||||
for command_name in git curl python3 npm node sha256sum tar apt-get dpkg-deb; do command -v "$command_name" >/dev/null; done
|
||||
test -x "$DOTNET" && test -x "$FNPACK_BIN"
|
||||
mkdir -p "$NPM_CONFIG_CACHE" "$NUGET_PACKAGES" "$DOTNET_CLI_HOME" \
|
||||
"$POSTGRES_SERVICE_BUILD_TMPDIR" "$LIVERECORDER_VERIFY_TMPDIR" "$POSTGRES_SERVICE_SMOKE_TMPDIR" artifacts
|
||||
available_kb=$(df -Pk "$WORKSPACE" | awk 'NR == 2 { print $4 }')
|
||||
test "$available_kb" -ge 4194304
|
||||
'''
|
||||
}
|
||||
}
|
||||
|
||||
stage('Restore And Test') {
|
||||
steps {
|
||||
sh '''
|
||||
set -euo pipefail
|
||||
npm ci --prefix frontend
|
||||
npm run build:postgres-admin --prefix frontend
|
||||
"$DOTNET" restore tests/PostgresService.Tests/PostgresService.Tests.csproj --disable-parallel
|
||||
"$DOTNET" test tests/PostgresService.Tests/PostgresService.Tests.csproj -c Release --no-restore \
|
||||
--logger 'console;verbosity=normal' /maxcpucount:1
|
||||
'''
|
||||
}
|
||||
}
|
||||
|
||||
stage('Build Verify And Smoke fnOS x64') {
|
||||
steps {
|
||||
sh '''
|
||||
set -euo pipefail
|
||||
PACKAGE_VERSION="$FNOS_VERSION" FNPACK="$FNPACK_BIN" DOTNET="$DOTNET" \
|
||||
./scripts/build-postgresql-fnos-package.sh "$FPK_PATH"
|
||||
./scripts/verify-fnos-package.sh "$FPK_PATH" "$FNOS_VERSION"
|
||||
./scripts/smoke-postgresql-fnos-package.sh "$FPK_PATH" "$POSTGRES_SERVICE_SMOKE_TMPDIR"
|
||||
fpk_sha=$(awk '{print $1}' "$FPK_PATH.sha256"); export fpk_sha
|
||||
python3 - <<'PY'
|
||||
import json, os, pathlib, subprocess
|
||||
path = pathlib.Path(os.environ['MANIFEST_PATH'])
|
||||
payload = {
|
||||
'project':'PostgresShare','buildNumber':os.environ['BUILD_NUMBER'],
|
||||
'commit':subprocess.check_output(['git','rev-parse','HEAD'],text=True).strip(),
|
||||
'productVersion':os.environ['APP_VERSION'],'fnosVersion':os.environ['FNOS_VERSION'],
|
||||
'fpk':pathlib.Path(os.environ['FPK_PATH']).name,'fpkSha256':os.environ['fpk_sha'],
|
||||
'runtime':'PostgreSQL 15 + pgvector 0.8.6'
|
||||
}
|
||||
path.write_text(json.dumps(payload,ensure_ascii=False,indent=2)+'\n',encoding='utf-8')
|
||||
PY
|
||||
sha256sum "$MANIFEST_PATH" >"$MANIFEST_PATH.sha256"
|
||||
'''
|
||||
}
|
||||
}
|
||||
|
||||
stage('Upload Artifacts') {
|
||||
steps {
|
||||
withCredentials([usernamePassword(credentialsId: "${OPENLIST_CREDENTIALS}", usernameVariable: 'OPENLIST_USERNAME', passwordVariable: 'OPENLIST_PASSWORD')]) {
|
||||
sh '''
|
||||
set -euo pipefail
|
||||
for artifact in "$FPK_PATH" "$FPK_PATH.sha256" "$MANIFEST_PATH" "$MANIFEST_PATH.sha256"; do
|
||||
./scripts/upload-openlist-artifact.sh "$artifact" "$OPENLIST_REMOTE_DIR"
|
||||
done
|
||||
'''
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
post {
|
||||
success { archiveArtifacts artifacts: 'artifacts/*.fpk,artifacts/*.sha256,artifacts/*.json', fingerprint: true }
|
||||
always { cleanWs(deleteDirs: true, notFailBuild: true) }
|
||||
}
|
||||
}
|
||||
@@ -2,11 +2,19 @@
|
||||
set -euo pipefail
|
||||
|
||||
ROOT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
|
||||
VERSION=15.1.1
|
||||
BASE_VERSION=$(sed -n 's/^version[[:space:]]*=[[:space:]]*//p' "$ROOT_DIR/fnos-postgresql/manifest" | head -n 1 | tr -d '\r')
|
||||
PACKAGE_VERSION="${PACKAGE_VERSION:-$BASE_VERSION}"
|
||||
case "$PACKAGE_VERSION" in
|
||||
''|*[!0-9.]*|.*|*..*|*.) printf 'invalid fnOS package version: %s\n' "$PACKAGE_VERSION" >&2; exit 1 ;;
|
||||
esac
|
||||
[ "$(printf '%s' "$PACKAGE_VERSION" | awk -F. '{ print NF }')" -eq 3 ] || {
|
||||
printf 'fnOS package version must contain exactly three numeric components: %s\n' "$PACKAGE_VERSION" >&2
|
||||
exit 1
|
||||
}
|
||||
PGVECTOR_VERSION=0.8.6
|
||||
PGVECTOR_PACKAGE_VERSION=0.8.6-1.pgdg12%2B1
|
||||
PGVECTOR_SHA256=b27ff894d1e2d23ebd7528fcb986923391977cbd5c5379ed74527875246854ca
|
||||
OUTPUT="${1:-$ROOT_DIR/artifacts/fnos/nxsir-postgresql-${VERSION}-x86_64.fpk}"
|
||||
OUTPUT="${1:-$ROOT_DIR/artifacts/fnos/nxsir-postgresql-${PACKAGE_VERSION}-x86_64.fpk}"
|
||||
DOTNET_BIN="${DOTNET:-dotnet}"
|
||||
NUGET_FEED="${POSTGRES_SERVICE_NUGET_FEED:-}"
|
||||
NUGET_PACKAGES="${NUGET_PACKAGES:-$ROOT_DIR/.cache/nuget-packages}"
|
||||
@@ -47,6 +55,7 @@ RUNTIME_ROOT="$STAGE/app/runtime"
|
||||
EXTRACT_ROOT="$WORK_DIR/runtime-extract"
|
||||
mkdir -p "$STAGE/app/server/wwwroot" "$RUNTIME_ROOT" "$PACKED_ROOT" "$FNPACK_TMP_ROOT" "$EXTRACT_ROOT"
|
||||
cp -a "$ROOT_DIR/fnos-postgresql/." "$STAGE/"
|
||||
sed -i -E "s/^version[[:space:]]*=.*/version=${PACKAGE_VERSION}/" "$STAGE/manifest"
|
||||
|
||||
printf 'Building PostgreSQL management frontend...\n'
|
||||
npm run build:postgres-admin --prefix "$ROOT_DIR/frontend"
|
||||
@@ -176,5 +185,5 @@ mv "$built_package" "$OUTPUT"
|
||||
sha256sum "$(basename -- "$OUTPUT")" >"$(basename -- "$OUTPUT").sha256"
|
||||
)
|
||||
|
||||
"$ROOT_DIR/scripts/verify-fnos-package.sh" "$OUTPUT"
|
||||
"$ROOT_DIR/scripts/verify-fnos-package.sh" "$OUTPUT" "$PACKAGE_VERSION"
|
||||
printf 'Built %s\n' "$OUTPUT"
|
||||
|
||||
Executable
+189
@@ -0,0 +1,189 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
FILE=${1:?usage: upload-openlist-artifact.sh file [remote-directory]}
|
||||
REMOTE_DIR=${2:-/yidongpan/构建产物/liverecorder}
|
||||
OPENLIST_BASE_URL=${OPENLIST_BASE_URL:-https://openlist.nxsir.cn}
|
||||
OPENLIST_USERNAME=${OPENLIST_USERNAME:?OPENLIST_USERNAME is required}
|
||||
OPENLIST_PASSWORD=${OPENLIST_PASSWORD:?OPENLIST_PASSWORD is required}
|
||||
OPENLIST_UPLOAD_ATTEMPTS=${OPENLIST_UPLOAD_ATTEMPTS:-6}
|
||||
|
||||
test -f "$FILE" || { printf 'artifact does not exist: %s\n' "$FILE" >&2; exit 1; }
|
||||
for required_command in curl python3 stat basename sleep; do
|
||||
command -v "$required_command" >/dev/null 2>&1 || {
|
||||
printf 'required upload command is missing: %s\n' "$required_command" >&2
|
||||
exit 1
|
||||
}
|
||||
done
|
||||
|
||||
OPENLIST_BASE_URL=${OPENLIST_BASE_URL%/}
|
||||
REMOTE_DIR="/${REMOTE_DIR#/}"
|
||||
REMOTE_DIR=${REMOTE_DIR%/}
|
||||
FILE_NAME=$(basename -- "$FILE")
|
||||
BUILD_MARKER=${BUILD_TAG:-${BUILD_ID:-$$}}
|
||||
BUILD_MARKER=$(printf '%s' "$BUILD_MARKER" | tr -c 'A-Za-z0-9._-' '-')
|
||||
TEMP_NAME=".${FILE_NAME}.uploading-${BUILD_MARKER}"
|
||||
TEMP_PATH="$REMOTE_DIR/$TEMP_NAME"
|
||||
FINAL_PATH="$REMOTE_DIR/$FILE_NAME"
|
||||
LOCAL_SIZE=$(stat -c '%s' "$FILE")
|
||||
TOKEN=
|
||||
|
||||
json_string() {
|
||||
python3 -c 'import json,sys; print(json.dumps(sys.argv[1], ensure_ascii=False))' "$1"
|
||||
}
|
||||
|
||||
envelope_code() {
|
||||
python3 -c 'import json,sys
|
||||
try:
|
||||
value=json.load(sys.stdin)
|
||||
print(value.get("code", -1))
|
||||
except Exception:
|
||||
print(-1)'
|
||||
}
|
||||
|
||||
envelope_message() {
|
||||
python3 -c 'import json,sys
|
||||
try:
|
||||
value=json.load(sys.stdin)
|
||||
print(value.get("message", "unknown OpenList response"))
|
||||
except Exception:
|
||||
print("invalid OpenList response")'
|
||||
}
|
||||
|
||||
login() {
|
||||
local payload response code
|
||||
payload=$(printf '{"username":%s,"password":%s}' \
|
||||
"$(json_string "$OPENLIST_USERNAME")" \
|
||||
"$(json_string "$OPENLIST_PASSWORD")")
|
||||
response=$(curl --silent --show-error --location --fail-with-body \
|
||||
--connect-timeout 30 --max-time 120 \
|
||||
--request POST "$OPENLIST_BASE_URL/api/auth/login" \
|
||||
--header 'Content-Type: application/json' \
|
||||
--data-raw "$payload") || return 1
|
||||
code=$(printf '%s' "$response" | envelope_code)
|
||||
[ "$code" = "200" ] || {
|
||||
printf 'OpenList login failed: %s\n' "$(printf '%s' "$response" | envelope_message)" >&2
|
||||
return 1
|
||||
}
|
||||
TOKEN=$(printf '%s' "$response" | python3 -c 'import json,sys; print((json.load(sys.stdin).get("data") or {}).get("token", ""))')
|
||||
[ -n "$TOKEN" ]
|
||||
}
|
||||
|
||||
post_json() {
|
||||
local endpoint=$1 payload=$2
|
||||
curl --silent --show-error --location --fail-with-body \
|
||||
--connect-timeout 30 --max-time 180 \
|
||||
--request POST "$OPENLIST_BASE_URL$endpoint" \
|
||||
--header "Authorization: $TOKEN" \
|
||||
--header 'Content-Type: application/json' \
|
||||
--data-raw "$payload"
|
||||
}
|
||||
|
||||
remote_size() {
|
||||
local path=$1 response code
|
||||
response=$(post_json /api/fs/get "{\"path\":$(json_string "$path"),\"password\":\"\"}") || return 1
|
||||
code=$(printf '%s' "$response" | envelope_code)
|
||||
[ "$code" = "200" ] || return 1
|
||||
printf '%s' "$response" | python3 -c 'import json,sys; print((json.load(sys.stdin).get("data") or {}).get("size", -1))'
|
||||
}
|
||||
|
||||
ensure_remote_directory() {
|
||||
local current= segment response code
|
||||
while IFS= read -r segment; do
|
||||
[ -n "$segment" ] || continue
|
||||
current="$current/$segment"
|
||||
response=$(post_json /api/fs/list "{\"path\":$(json_string "$current"),\"password\":\"\",\"refresh\":false,\"page\":1,\"per_page\":1}" || true)
|
||||
code=$(printf '%s' "$response" | envelope_code)
|
||||
if [ "$code" = "200" ]; then
|
||||
continue
|
||||
fi
|
||||
response=$(post_json /api/fs/mkdir "{\"path\":$(json_string "$current")}") || return 1
|
||||
code=$(printf '%s' "$response" | envelope_code)
|
||||
[ "$code" = "200" ] || {
|
||||
printf 'OpenList mkdir failed for %s: %s\n' "$current" "$(printf '%s' "$response" | envelope_message)" >&2
|
||||
return 1
|
||||
}
|
||||
done < <(python3 -c 'import sys; print("\n".join(part for part in sys.argv[1].split("/") if part))' "$REMOTE_DIR")
|
||||
}
|
||||
|
||||
remove_temporary_file() {
|
||||
local response
|
||||
[ -n "$TOKEN" ] || login >/dev/null 2>&1 || return 0
|
||||
response=$(post_json /api/fs/remove "{\"dir\":$(json_string "$REMOTE_DIR"),\"names\":[$(json_string "$TEMP_NAME")]}" 2>/dev/null || true)
|
||||
[ "$(printf '%s' "$response" | envelope_code)" = "200" ] || true
|
||||
}
|
||||
|
||||
publish_temporary_file() {
|
||||
local response code
|
||||
response=$(post_json /api/fs/rename "{\"path\":$(json_string "$TEMP_PATH"),\"name\":$(json_string "$FILE_NAME")}") || return 1
|
||||
code=$(printf '%s' "$response" | envelope_code)
|
||||
[ "$code" = "200" ] || {
|
||||
printf 'OpenList rename failed: %s\n' "$(printf '%s' "$response" | envelope_message)" >&2
|
||||
return 1
|
||||
}
|
||||
}
|
||||
|
||||
upload_once() {
|
||||
local encoded_path response code uploaded_size
|
||||
encoded_path=$(python3 -c 'import sys,urllib.parse; print(urllib.parse.quote(sys.argv[1], safe=""))' "$TEMP_PATH")
|
||||
response=$(curl --silent --show-error --location --fail-with-body \
|
||||
--connect-timeout 30 --max-time 3600 --speed-limit 1024 --speed-time 120 \
|
||||
--request PUT "$OPENLIST_BASE_URL/api/fs/put" \
|
||||
--header "Authorization: $TOKEN" \
|
||||
--header "File-Path: $encoded_path" \
|
||||
--header 'As-Task: false' \
|
||||
--header 'Content-Type: application/octet-stream' \
|
||||
--header 'Expect:' \
|
||||
--data-binary "@$FILE") || return 1
|
||||
code=$(printf '%s' "$response" | envelope_code)
|
||||
[ "$code" = "200" ] || {
|
||||
printf 'OpenList upload failed: %s\n' "$(printf '%s' "$response" | envelope_message)" >&2
|
||||
return 1
|
||||
}
|
||||
uploaded_size=$(remote_size "$TEMP_PATH") || return 1
|
||||
[ "$uploaded_size" = "$LOCAL_SIZE" ] || {
|
||||
printf 'OpenList size mismatch for temporary upload: local=%s remote=%s\n' "$LOCAL_SIZE" "$uploaded_size" >&2
|
||||
return 1
|
||||
}
|
||||
}
|
||||
|
||||
printf 'Uploading %s (%s bytes) to OpenList %s\n' "$FILE_NAME" "$LOCAL_SIZE" "$REMOTE_DIR"
|
||||
|
||||
for attempt in $(seq 1 "$OPENLIST_UPLOAD_ATTEMPTS"); do
|
||||
TOKEN=
|
||||
upload_ready=1
|
||||
if login && ensure_remote_directory; then
|
||||
existing_size=$(remote_size "$FINAL_PATH" 2>/dev/null || true)
|
||||
if [ "$existing_size" = "$LOCAL_SIZE" ]; then
|
||||
printf 'OpenList artifact already exists with matching size: %s\n' "$FINAL_PATH"
|
||||
exit 0
|
||||
fi
|
||||
if [ -n "$existing_size" ]; then
|
||||
remove_response=$(post_json /api/fs/remove "{\"dir\":$(json_string "$REMOTE_DIR"),\"names\":[$(json_string "$FILE_NAME")]}" || true)
|
||||
if [ "$(printf '%s' "$remove_response" | envelope_code)" != "200" ]; then
|
||||
printf 'Unable to remove mismatched OpenList artifact before retry: %s\n' "$FINAL_PATH" >&2
|
||||
upload_ready=0
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ "$upload_ready" = "1" ] && upload_once && publish_temporary_file; then
|
||||
final_size=$(remote_size "$FINAL_PATH") || final_size=-1
|
||||
if [ "$final_size" = "$LOCAL_SIZE" ]; then
|
||||
printf 'OpenList upload verified: %s (%s bytes)\n' "$FINAL_PATH" "$final_size"
|
||||
exit 0
|
||||
fi
|
||||
printf 'OpenList final size mismatch: local=%s remote=%s\n' "$LOCAL_SIZE" "$final_size" >&2
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ "$attempt" -lt "$OPENLIST_UPLOAD_ATTEMPTS" ]; then
|
||||
delay=$((5 * (1 << (attempt - 1))))
|
||||
[ "$delay" -le 120 ] || delay=120
|
||||
printf 'OpenList upload attempt %s/%s failed; retrying in %ss\n' "$attempt" "$OPENLIST_UPLOAD_ATTEMPTS" "$delay" >&2
|
||||
sleep "$delay"
|
||||
fi
|
||||
done
|
||||
|
||||
remove_temporary_file
|
||||
printf 'OpenList upload failed after %s attempts: %s\n' "$OPENLIST_UPLOAD_ATTEMPTS" "$FILE_NAME" >&2
|
||||
exit 1
|
||||
@@ -1,7 +1,8 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
PACKAGE=${1:?usage: verify-fnos-package.sh package.fpk}
|
||||
PACKAGE=${1:?usage: verify-fnos-package.sh package.fpk [expected-version]}
|
||||
EXPECTED_VERSION=${2:-}
|
||||
VERIFY_TMP_ROOT="${LIVERECORDER_VERIFY_TMPDIR:-${TMPDIR:-/tmp}}"
|
||||
MAX_APP_UNCOMPRESSED_BYTES=$((512 * 1024 * 1024))
|
||||
mkdir -p "$VERIFY_TMP_ROOT"
|
||||
@@ -17,6 +18,10 @@ appname=$(manifest_value appname)
|
||||
version=$(manifest_value version)
|
||||
test "$appname" = "nxsir.postgresql" || { printf 'unexpected fnOS appname: %s\n' "$appname" >&2; exit 1; }
|
||||
test -n "$version"
|
||||
if [ -n "$EXPECTED_VERSION" ] && [ "$version" != "$EXPECTED_VERSION" ]; then
|
||||
printf 'unexpected fnOS version: expected %s, got %s\n' "$EXPECTED_VERSION" "$version" >&2
|
||||
exit 1
|
||||
fi
|
||||
test "$(manifest_value platform)" = "x86"
|
||||
test -x "$WORK_DIR/cmd/main"
|
||||
test -x "$WORK_DIR/cmd/install_callback"
|
||||
|
||||
Reference in New Issue
Block a user