254 lines
13 KiB
Groovy
254 lines
13 KiB
Groovy
pipeline {
|
|
agent {
|
|
node {
|
|
label '构建机1'
|
|
customWorkspace '/home/nanxunai/goujian/workspace/imagefind-release'
|
|
}
|
|
}
|
|
|
|
options {
|
|
timestamps()
|
|
disableConcurrentBuilds()
|
|
skipDefaultCheckout(true)
|
|
timeout(time: 240, unit: 'MINUTES')
|
|
buildDiscarder(logRotator(daysToKeepStr: '30', numToKeepStr: '15'))
|
|
}
|
|
|
|
environment {
|
|
PATH = '/home/nanxunai/.local/bin:/usr/local/bin:/usr/bin:/bin'
|
|
FNPACK_BIN = '/home/nanxunai/.local/bin/fnpack'
|
|
REGISTRY_URL = 'reg.nxsir.cn'
|
|
IMAGE_REPO = 'reg.nxsir.cn/imagefind'
|
|
HARBOR_CREDENTIALS = 'imagefind_key'
|
|
OPENLIST_CREDENTIALS = 'openlist_key'
|
|
NODE_CREDENTIALS = 'bbb939ea-4f01-4b47-aecb-c5ee2a551ef4'
|
|
OPENLIST_BASE_URL = 'https://openlist.nxsir.cn'
|
|
OPENLIST_REMOTE_DIR = '/yidongpan/构建产物/imagefind'
|
|
HTTP_PROXY_URL = 'http://192.168.5.200:7890'
|
|
NO_PROXY_HOSTS = '127.0.0.1,localhost,reg.nxsir.cn,gitea.nxsir.cn,openlist.nxsir.cn,pypi.tuna.tsinghua.edu.cn'
|
|
DOCKER_CONFIG = "${WORKSPACE}/.ci/docker"
|
|
NPM_CONFIG_CACHE = "${WORKSPACE}/.ci/npm-cache"
|
|
PIP_CACHE_DIR = "${WORKSPACE}/.ci/pip-cache"
|
|
PIP_INDEX_URL = 'https://pypi.tuna.tsinghua.edu.cn/simple'
|
|
PIP_DEFAULT_TIMEOUT = '60'
|
|
IMAGEFIND_WHEEL_CACHE = "${WORKSPACE}/.ci/wheelhouse-python312"
|
|
IMAGEFIND_DEPENDENCY_TMPDIR = "${WORKSPACE}/.ci/dependency-tmp"
|
|
IMAGEFIND_VERIFY_TMPDIR = "${WORKSPACE}/.ci/fnos-verify"
|
|
}
|
|
|
|
stages {
|
|
stage('Checkout') {
|
|
steps {
|
|
deleteDir()
|
|
checkout scm
|
|
}
|
|
}
|
|
|
|
stage('Metadata And Preflight') {
|
|
steps {
|
|
script {
|
|
env.APP_VERSION = sh(script: "sed -n 's/^version=//p' fnos/manifest | head -n 1", returnStdout: true).trim()
|
|
env.SHORT_SHA = sh(script: 'git rev-parse --short=8 HEAD', returnStdout: true).trim()
|
|
env.FNOS_VERSION = sh(script: '''
|
|
set -euo pipefail
|
|
base="$APP_VERSION"; major=${base%%.*}; remainder=${base#*.}
|
|
minor=${remainder%%.*}; patch=${remainder#*.}
|
|
test "$base" = "$major.$minor.$patch" && test "$BUILD_NUMBER" -lt 100000
|
|
printf '%s.%s.%s\n' "$major" "$minor" "$((patch * 100000 + BUILD_NUMBER))"
|
|
''', returnStdout: true).trim()
|
|
env.IMMUTABLE_TAG = "${env.APP_VERSION}-b${env.BUILD_NUMBER}-${env.SHORT_SHA}"
|
|
env.IMAGE_REF = "${env.IMAGE_REPO}:${env.IMMUTABLE_TAG}"
|
|
env.FPK_BASENAME = "imagefind-${env.FNOS_VERSION}-x86_64.fpk"
|
|
env.FPK_PATH = "${env.WORKSPACE}/artifacts/${env.FPK_BASENAME}"
|
|
env.MANIFEST_PATH = "${env.WORKSPACE}/artifacts/imagefind-${env.IMMUTABLE_TAG}-build-manifest.json"
|
|
currentBuild.displayName = "#${env.BUILD_NUMBER} ${env.IMMUTABLE_TAG}"
|
|
currentBuild.description = "fnOS ${env.FNOS_VERSION}"
|
|
}
|
|
withCredentials([usernamePassword(credentialsId: "${NODE_CREDENTIALS}", usernameVariable: 'JENKINS_NODE_USERNAME', passwordVariable: 'JENKINS_NODE_PASSWORD')]) {
|
|
sh '''
|
|
set -euo pipefail
|
|
if ! python3 -c 'import ensurepip' >/dev/null 2>&1; then
|
|
printf '%s\n' "$JENKINS_NODE_PASSWORD" | sudo -S -p '' apt-get update
|
|
printf '%s\n' "$JENKINS_NODE_PASSWORD" | sudo -S -p '' \
|
|
apt-get install -y --no-install-recommends python3.13-venv
|
|
printf '%s\n' "$JENKINS_NODE_PASSWORD" | sudo -S -p '' apt-get clean
|
|
fi
|
|
python3 -c 'import ensurepip'
|
|
'''
|
|
}
|
|
sh '''
|
|
set -euo pipefail
|
|
test "$(uname -m)" = x86_64
|
|
for command_name in git sudo docker curl python3 npm node sha256sum tar dpkg-deb; do command -v "$command_name" >/dev/null; done
|
|
test -x "$FNPACK_BIN"
|
|
mkdir -p "$DOCKER_CONFIG" "$NPM_CONFIG_CACHE" "$PIP_CACHE_DIR" "$IMAGEFIND_WHEEL_CACHE" \
|
|
"$IMAGEFIND_DEPENDENCY_TMPDIR" "$IMAGEFIND_VERIFY_TMPDIR" artifacts
|
|
chmod 700 "$DOCKER_CONFIG"
|
|
available_kb=$(df -Pk "$WORKSPACE" | awk 'NR == 2 { print $4 }')
|
|
test "$available_kb" -ge 3145728 || {
|
|
printf 'at least 3 GiB free disk space is required; available: %s KiB\n' "$available_kb" >&2
|
|
exit 1
|
|
}
|
|
'''
|
|
}
|
|
}
|
|
|
|
stage('Backend And Frontend Tests') {
|
|
steps {
|
|
sh '''
|
|
set -euo pipefail
|
|
python3 -m venv .ci/venv
|
|
.ci/venv/bin/python -m pip install --disable-pip-version-check --upgrade pip
|
|
.ci/venv/bin/python -m pip install --disable-pip-version-check '.[dev]'
|
|
.ci/venv/bin/ruff check backend tests
|
|
# Unit tests that do not require the fnOS shared PostgreSQL service.
|
|
# PostgreSQL integration is covered by the real-container smoke stage below.
|
|
PYTHONPATH=. .ci/venv/bin/pytest -q \
|
|
tests/test_openlist_client.py \
|
|
tests/test_workflow_contracts.py \
|
|
tests/test_accelerator_runtime.py \
|
|
tests/test_inference.py
|
|
npm ci --prefix frontend
|
|
npm run lint --prefix frontend
|
|
npm run build --prefix frontend
|
|
'''
|
|
}
|
|
}
|
|
|
|
stage('Prepare And Build fnOS x64') {
|
|
steps {
|
|
sh '''
|
|
set -euo pipefail
|
|
PYTHON=.ci/venv/bin/python ./scripts/prepare-fnos-dependencies.sh
|
|
PACKAGE_VERSION="$FNOS_VERSION" PYTHON=.ci/venv/bin/python FNPACK="$FNPACK_BIN" \
|
|
./scripts/build-fnos.sh "$FPK_PATH"
|
|
./scripts/verify-fnos-package.sh "$FPK_PATH" "$FNOS_VERSION"
|
|
'''
|
|
}
|
|
}
|
|
|
|
stage('Build And Smoke Docker amd64') {
|
|
steps {
|
|
withCredentials([usernamePassword(credentialsId: "${NODE_CREDENTIALS}", usernameVariable: 'JENKINS_NODE_USERNAME', passwordVariable: 'JENKINS_NODE_PASSWORD')]) {
|
|
sh '''
|
|
set -euo pipefail
|
|
DOCKER_BUILDKIT=1 ./scripts/ci-docker.sh build --network host --progress=plain \
|
|
--build-arg PIP_INDEX_URL="$PIP_INDEX_URL" \
|
|
--build-arg HTTP_PROXY="$HTTP_PROXY_URL" --build-arg HTTPS_PROXY="$HTTP_PROXY_URL" \
|
|
--build-arg NO_PROXY="$NO_PROXY_HOSTS" -t "$IMAGE_REF" .
|
|
smoke_name="imagefind-smoke-$BUILD_NUMBER"
|
|
mkdir -p .ci/docker-data
|
|
./scripts/ci-docker.sh rm -f "$smoke_name" >/dev/null 2>&1 || true
|
|
./scripts/ci-docker.sh run -d --name "$smoke_name" -p 127.0.0.1::8765 \
|
|
-e IMAGEFIND_ADMIN_PASSWORD='ImageFind-Smoke-2026!' \
|
|
-v "$WORKSPACE/.ci/docker-data:/data" "$IMAGE_REF"
|
|
host_port=$(./scripts/ci-docker.sh port "$smoke_name" 8765/tcp | sed -n 's/.*://p' | head -n 1)
|
|
test -n "$host_port"
|
|
ready=0
|
|
for attempt in $(seq 1 90); do
|
|
if curl -fsS "http://127.0.0.1:$host_port/" >/dev/null; then ready=1; break; fi
|
|
sleep 2
|
|
done
|
|
test "$ready" = 1 || { ./scripts/ci-docker.sh logs "$smoke_name" || true; exit 1; }
|
|
curl -fsS -H 'Content-Type: application/json' \
|
|
--data '{"password":"ImageFind-Smoke-2026!"}' \
|
|
"http://127.0.0.1:$host_port/api/v1/auth/login" | grep -q 'csrf'
|
|
./scripts/ci-docker.sh restart "$smoke_name" >/dev/null
|
|
ready=0
|
|
for attempt in $(seq 1 60); do
|
|
if curl -fsS "http://127.0.0.1:$host_port/" >/dev/null; then ready=1; break; fi
|
|
sleep 2
|
|
done
|
|
test "$ready" = 1
|
|
curl -fsS -H 'Content-Type: application/json' \
|
|
--data '{"password":"ImageFind-Smoke-2026!"}' \
|
|
"http://127.0.0.1:$host_port/api/v1/auth/login" | grep -q 'csrf'
|
|
test -s .ci/docker-data/postgresql/PG_VERSION
|
|
test -s .ci/docker-data/postgres-client.conf
|
|
test -e .ci/docker-data/.imagefind-admin-initialized
|
|
./scripts/ci-docker.sh rm -f "$smoke_name"
|
|
'''
|
|
}
|
|
}
|
|
}
|
|
|
|
stage('Push Immutable And Describe') {
|
|
steps {
|
|
withCredentials([
|
|
usernamePassword(credentialsId: "${NODE_CREDENTIALS}", usernameVariable: 'JENKINS_NODE_USERNAME', passwordVariable: 'JENKINS_NODE_PASSWORD'),
|
|
usernamePassword(credentialsId: "${HARBOR_CREDENTIALS}", usernameVariable: 'HARBOR_USERNAME', passwordVariable: 'HARBOR_PASSWORD')
|
|
]) {
|
|
sh '''
|
|
set -euo pipefail
|
|
set +x
|
|
auth=$(printf '%s:%s' "$HARBOR_USERNAME" "$HARBOR_PASSWORD" | base64 -w0)
|
|
printf '{"auths":{"%s":{"auth":"%s"}}}\n' "$REGISTRY_URL" "$auth" >"$DOCKER_CONFIG/config.json"
|
|
chmod 600 "$DOCKER_CONFIG/config.json"
|
|
unset auth
|
|
set -x
|
|
./scripts/ci-docker.sh push "$IMAGE_REF"
|
|
digest=$(./scripts/ci-docker.sh image inspect --format '{{index .RepoDigests 0}}' "$IMAGE_REF" | sed -n 's/.*@//p'); test -n "$digest"; export digest
|
|
fpk_sha=$(awk '{print $1}' "$FPK_PATH.sha256"); export fpk_sha
|
|
python3 - <<'PY'
|
|
import json, os, pathlib, subprocess
|
|
path = pathlib.Path(os.environ['MANIFEST_PATH'])
|
|
payload = {
|
|
'project':'imagefind','buildNumber':os.environ['BUILD_NUMBER'],
|
|
'commit':subprocess.check_output(['git','rev-parse','HEAD'],text=True).strip(),
|
|
'productVersion':os.environ['APP_VERSION'],'fnosVersion':os.environ['FNOS_VERSION'],
|
|
'fpk':pathlib.Path(os.environ['FPK_PATH']).name,'fpkSha256':os.environ['fpk_sha'],
|
|
'image':os.environ['IMAGE_REF'],'digest':os.environ['digest'],'platform':'linux/amd64',
|
|
'database':'PostgreSQL 15 + pgvector 0.8.6 (embedded Docker runtime)'
|
|
}
|
|
path.write_text(json.dumps(payload,ensure_ascii=False,indent=2)+chr(10),encoding='utf-8')
|
|
PY
|
|
sha256sum "$MANIFEST_PATH" >"$MANIFEST_PATH.sha256"
|
|
'''
|
|
}
|
|
}
|
|
}
|
|
|
|
stage('Upload Artifacts') {
|
|
steps {
|
|
withCredentials([usernamePassword(credentialsId: "${OPENLIST_CREDENTIALS}", usernameVariable: 'OPENLIST_USERNAME', passwordVariable: 'OPENLIST_PASSWORD')]) {
|
|
sh '''
|
|
set -euo pipefail
|
|
for artifact in "$FPK_PATH" "$FPK_PATH.sha256" "$MANIFEST_PATH" "$MANIFEST_PATH.sha256"; do
|
|
./scripts/upload-openlist-artifact.sh "$artifact" "$OPENLIST_REMOTE_DIR"
|
|
done
|
|
'''
|
|
}
|
|
}
|
|
}
|
|
|
|
stage('Publish Stable Image Tags') {
|
|
steps {
|
|
withCredentials([usernamePassword(credentialsId: "${NODE_CREDENTIALS}", usernameVariable: 'JENKINS_NODE_USERNAME', passwordVariable: 'JENKINS_NODE_PASSWORD')]) {
|
|
sh '''
|
|
set -euo pipefail
|
|
for tag in "$APP_VERSION" latest; do
|
|
./scripts/ci-docker.sh tag "$IMAGE_REF" "$IMAGE_REPO:$tag"
|
|
./scripts/ci-docker.sh push "$IMAGE_REPO:$tag"
|
|
done
|
|
'''
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
post {
|
|
success { archiveArtifacts artifacts: 'artifacts/*.fpk,artifacts/*.sha256,artifacts/*.json', fingerprint: true }
|
|
always {
|
|
withCredentials([usernamePassword(credentialsId: "${NODE_CREDENTIALS}", usernameVariable: 'JENKINS_NODE_USERNAME', passwordVariable: 'JENKINS_NODE_PASSWORD')]) {
|
|
sh '''
|
|
set +e
|
|
./scripts/ci-docker.sh rm -f "imagefind-smoke-$BUILD_NUMBER" >/dev/null 2>&1 || true
|
|
for tag in "$IMMUTABLE_TAG" "$APP_VERSION" latest; do ./scripts/ci-docker.sh rmi "$IMAGE_REPO:$tag" >/dev/null 2>&1 || true; done
|
|
: >"$DOCKER_CONFIG/config.json" 2>/dev/null || true
|
|
'''
|
|
}
|
|
cleanWs(deleteDirs: true, notFailBuild: true)
|
|
}
|
|
}
|
|
}
|