feat: add ImageFind application and release pipelines

2026-08-11 18:02:40 +08:00
commit 16239d7525
270 changed files with 59163 additions and 0 deletions
@@ -0,0 +1,200 @@
+#!/bin/sh
+set -eu
+
+PROJECT_ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
+PYTHON_BIN="${PYTHON:-$PROJECT_ROOT/.venv/bin/python}"
+FFMPEG_BIN="${FFMPEG:-$PROJECT_ROOT/vendor/ffmpeg}"
+FFPROBE_BIN="${FFPROBE:-$PROJECT_ROOT/vendor/ffprobe}"
+PORT="${1:-18766}"
+SMOKE_ROOT=$(mktemp -d "${TMPDIR:-/tmp}/imagefind-http-smoke.XXXXXX")
+DATA_ROOT="$SMOKE_ROOT/data"
+MEDIA_ROOT="$SMOKE_ROOT/media"
+COOKIE_JAR="$SMOKE_ROOT/cookies"
+SERVER_LOG="$SMOKE_ROOT/server.log"
+PASSWORD="imagefind-http-smoke-password"
+SERVER_PID=""
+
+cleanup() {
+    STATUS=$?
+    trap - EXIT HUP INT TERM
+    if [ -n "$SERVER_PID" ] && kill -0 "$SERVER_PID" 2>/dev/null; then
+        kill "$SERVER_PID" 2>/dev/null || true
+        wait "$SERVER_PID" 2>/dev/null || true
+    fi
+    if [ "$STATUS" -ne 0 ]; then
+        printf '%s\n' 'HTTP smoke failed; application log follows:' >&2
+        tail -100 "$SERVER_LOG" >&2 || true
+        cp -p "$SERVER_LOG" "$PROJECT_ROOT/dist/http-smoke-failure.log" 2>/dev/null || true
+    fi
+    rm -rf -- "$SMOKE_ROOT"
+    exit "$STATUS"
+}
+trap cleanup EXIT HUP INT TERM
+
+mkdir -p "$DATA_ROOT" "$MEDIA_ROOT"
+"$FFMPEG_BIN" -hide_banner -loglevel error -f lavfi -i "testsrc=size=320x180:rate=24" \
+    -t 2 -pix_fmt yuv420p -c:v libx264 -movflags +faststart "$SMOKE_ROOT/ABC-999.mp4"
+
+env PYTHONPATH="$PROJECT_ROOT/backend" \
+    IMAGEFIND_DATA_DIR="$DATA_ROOT" \
+    IMAGEFIND_FRONTEND_DIR="$PROJECT_ROOT/frontend/dist" \
+    IMAGEFIND_HOST="127.0.0.1" \
+    IMAGEFIND_PORT="$PORT" \
+    IMAGEFIND_UPLOAD_RESERVE_GB="0" \
+    IMAGEFIND_RESOURCE_DISK_RESERVE_GB="0.5" \
+    IMAGEFIND_EMBEDDING_BACKEND="hash" \
+    IMAGEFIND_FFMPEG_PATH="$FFMPEG_BIN" \
+    IMAGEFIND_FFPROBE_PATH="$FFPROBE_BIN" \
+    "$PYTHON_BIN" -m imagefind.main >"$SERVER_LOG" 2>&1 &
+SERVER_PID=$!
+
+ATTEMPT=0
+until curl -fsS "http://127.0.0.1:$PORT/api/v1/status" >/dev/null 2>&1; do
+    ATTEMPT=$((ATTEMPT + 1))
+    if [ "$ATTEMPT" -ge 60 ]; then
+        printf '%s\n' 'ImageFind HTTP service did not become ready.' >&2
+        exit 1
+    fi
+    sleep 1
+done
+
+SETUP=$(curl -fsS -c "$COOKIE_JAR" -H 'Content-Type: application/json' \
+    -d "{\"password\":\"$PASSWORD\"}" "http://127.0.0.1:$PORT/api/v1/setup")
+CSRF=$(printf '%s' "$SETUP" | "$PYTHON_BIN" -c 'import json,sys; print(json.load(sys.stdin)["csrf_token"])')
+SOURCE=$(curl -fsS -b "$COOKIE_JAR" -H 'Content-Type: application/json' -H "X-CSRF-Token: $CSRF" \
+    -d "{\"name\":\"HTTP Smoke\",\"path\":\"$MEDIA_ROOT\"}" \
+    "http://127.0.0.1:$PORT/api/v1/sources/local")
+SOURCE_ID=$(printf '%s' "$SOURCE" | "$PYTHON_BIN" -c 'import json,sys; print(json.load(sys.stdin)["id"])')
+curl -fsS -b "$COOKIE_JAR" -H 'Content-Type: application/json' -H "X-CSRF-Token: $CSRF" \
+    -X PATCH -d '{"writable":true}' \
+    "http://127.0.0.1:$PORT/api/v1/sources/$SOURCE_ID/writable" >/dev/null
+
+SIZE=$(wc -c <"$SMOKE_ROOT/ABC-999.mp4" | tr -d ' ')
+UPLOAD=$(curl -fsS -b "$COOKIE_JAR" -H 'Content-Type: application/json' -H "X-CSRF-Token: $CSRF" \
+    -d "{\"source_id\":\"$SOURCE_ID\",\"relative_path\":\"imports\",\"filename\":\"ABC-999.mp4\",\"size_bytes\":$SIZE}" \
+    "http://127.0.0.1:$PORT/api/v1/uploads")
+UPLOAD_ID=$(printf '%s' "$UPLOAD" | "$PYTHON_BIN" -c 'import json,sys; print(json.load(sys.stdin)["id"])')
+DIGEST=$(sha256sum "$SMOKE_ROOT/ABC-999.mp4" | cut -d ' ' -f 1)
+curl -fsS -b "$COOKIE_JAR" -H 'Content-Type: application/octet-stream' -H "X-CSRF-Token: $CSRF" \
+    -H "X-Chunk-SHA256: $DIGEST" -X PUT --data-binary "@$SMOKE_ROOT/ABC-999.mp4" \
+    "http://127.0.0.1:$PORT/api/v1/uploads/$UPLOAD_ID/chunks/0" >/dev/null
+curl -fsS -b "$COOKIE_JAR" -H 'Content-Type: application/json' -H "X-CSRF-Token: $CSRF" \
+    -d '{}' "http://127.0.0.1:$PORT/api/v1/uploads/$UPLOAD_ID/complete" >/dev/null
+
+ATTEMPT=0
+while :; do
+    UPLOADS=$(curl -fsS -b "$COOKIE_JAR" "http://127.0.0.1:$PORT/api/v1/uploads")
+    STATE=$(printf '%s' "$UPLOADS" | "$PYTHON_BIN" -c \
+        'import json,sys; items=json.load(sys.stdin); uid=sys.argv[1]; print(next(x["status"] for x in items if x["id"] == uid))' \
+        "$UPLOAD_ID")
+    [ "$STATE" = "completed" ] && break
+    [ "$STATE" = "failed" ] && { printf '%s\n' "$UPLOADS" >&2; exit 1; }
+    ATTEMPT=$((ATTEMPT + 1))
+    [ "$ATTEMPT" -lt 60 ] || { printf '%s\n' "Upload stayed in state $STATE" >&2; exit 1; }
+    sleep 1
+done
+
+VIDEOS=$(curl -fsS -b "$COOKIE_JAR" "http://127.0.0.1:$PORT/api/v1/videos")
+VIDEO_ID=$(printf '%s' "$VIDEOS" | "$PYTHON_BIN" -c \
+    'import json,sys; print(next(x["id"] for x in json.load(sys.stdin) if x["display_name"] == "ABC-999.mp4"))')
+curl -fsS -b "$COOKIE_JAR" "http://127.0.0.1:$PORT/api/v1/videos/$VIDEO_ID/download" \
+    -o "$SMOKE_ROOT/download.mp4"
+cmp "$SMOKE_ROOT/ABC-999.mp4" "$SMOKE_ROOT/download.mp4"
+
+TOKEN_RESPONSE=$(curl -fsS -b "$COOKIE_JAR" -H 'Content-Type: application/json' \
+    -H "X-CSRF-Token: $CSRF" -d '{"name":"HTTP WebDAV smoke"}' \
+    "http://127.0.0.1:$PORT/api/v1/tokens")
+API_TOKEN=$(printf '%s' "$TOKEN_RESPONSE" | "$PYTHON_BIN" -c \
+    'import json,sys; print(json.load(sys.stdin)["token"])')
+curl -fsS -b "$COOKIE_JAR" -H 'Content-Type: application/json' -H "X-CSRF-Token: $CSRF" \
+    -X PATCH \
+    -d "{\"enabled\":true,\"source_id\":\"$SOURCE_ID\",\"relative_path\":\"webdav-smoke\"}" \
+    "http://127.0.0.1:$PORT/api/v1/webdav/config" >/dev/null
+
+DAV_SIZE=$SIZE
+DAV_SPLIT=$((DAV_SIZE / 2))
+DAV_REMAINING=$((DAV_SIZE - DAV_SPLIT))
+dd if="$SMOKE_ROOT/ABC-999.mp4" of="$SMOKE_ROOT/dav-first.bin" bs=1 count="$DAV_SPLIT" 2>/dev/null
+dd if="$SMOKE_ROOT/ABC-999.mp4" of="$SMOKE_ROOT/dav-second.bin" bs=1 skip="$DAV_SPLIT" 2>/dev/null
+DAV_URL="http://127.0.0.1:$PORT/webdav/DAV-RESUME.mp4"
+FIRST_STATUS=$(curl -sS -u "imagefind:$API_TOKEN" -X PUT \
+    -H "Content-Range: bytes 0-$((DAV_SPLIT - 1))/$DAV_SIZE" \
+    -H "Content-Length: $DAV_SPLIT" --data-binary "@$SMOKE_ROOT/dav-first.bin" \
+    -D "$SMOKE_ROOT/dav-first.headers" -o "$SMOKE_ROOT/dav-first.body" -w '%{http_code}' "$DAV_URL")
+[ "$FIRST_STATUS" = "204" ]
+FIRST_OFFSET=$(tr -d '\r' <"$SMOKE_ROOT/dav-first.headers" | awk \
+    'tolower($1)=="upload-offset:" {print $2; exit}')
+[ "$FIRST_OFFSET" = "$DAV_SPLIT" ]
+HEAD_STATUS=$(curl -sS -u "imagefind:$API_TOKEN" -I \
+    -D "$SMOKE_ROOT/dav-head.headers" -o /dev/null -w '%{http_code}' "$DAV_URL")
+[ "$HEAD_STATUS" = "200" ]
+HEAD_OFFSET=$(tr -d '\r' <"$SMOKE_ROOT/dav-head.headers" | awk \
+    'tolower($1)=="upload-offset:" {print $2; exit}')
+[ "$HEAD_OFFSET" = "$DAV_SPLIT" ]
+SECOND_STATUS=$(curl -sS -u "imagefind:$API_TOKEN" -X PUT \
+    -H "Content-Range: bytes $DAV_SPLIT-$((DAV_SIZE - 1))/$DAV_SIZE" \
+    -H "Content-Length: $DAV_REMAINING" --data-binary "@$SMOKE_ROOT/dav-second.bin" \
+    -D "$SMOKE_ROOT/dav-second.headers" -o "$SMOKE_ROOT/dav-second.body" -w '%{http_code}' "$DAV_URL")
+[ "$SECOND_STATUS" = "204" ]
+DAV_UPLOAD_ID=$(tr -d '\r' <"$SMOKE_ROOT/dav-second.headers" | awk \
+    'tolower($1)=="x-imagefind-upload-id:" {print $2; exit}')
+[ -n "$DAV_UPLOAD_ID" ]
+
+ATTEMPT=0
+while :; do
+    UPLOADS=$(curl -fsS -b "$COOKIE_JAR" "http://127.0.0.1:$PORT/api/v1/uploads")
+    STATE=$(printf '%s' "$UPLOADS" | "$PYTHON_BIN" -c \
+        'import json,sys; items=json.load(sys.stdin); uid=sys.argv[1]; print(next(x["status"] for x in items if x["id"] == uid))' \
+        "$DAV_UPLOAD_ID")
+    [ "$STATE" = "completed" ] && break
+    [ "$STATE" = "failed" ] && { printf '%s\n' "$UPLOADS" >&2; exit 1; }
+    ATTEMPT=$((ATTEMPT + 1))
+    [ "$ATTEMPT" -lt 60 ] || { printf '%s\n' "WebDAV upload stayed in state $STATE" >&2; exit 1; }
+    sleep 1
+done
+cmp "$SMOKE_ROOT/ABC-999.mp4" "$MEDIA_ROOT/webdav-smoke/DAV-RESUME.mp4"
+
+DEDUP_STATUS=$(curl -sS -u "imagefind:$API_TOKEN" -X PUT \
+    -H "Content-Length: $DAV_SIZE" -H "X-Content-SHA256: $DIGEST" \
+    --data-binary "@$SMOKE_ROOT/ABC-999.mp4" -D "$SMOKE_ROOT/dav-dedup.headers" \
+    -o "$SMOKE_ROOT/dav-dedup.body" -w '%{http_code}' "$DAV_URL")
+[ "$DEDUP_STATUS" = "204" ]
+DEDUPLICATED=$(tr -d '\r' <"$SMOKE_ROOT/dav-dedup.headers" | awk \
+    'tolower($1)=="x-imagefind-deduplicated:" {print tolower($2); exit}')
+[ "$DEDUPLICATED" = "true" ]
+
+if [ "${IMAGEFIND_RUN_WEBDAV_STRESS:-0}" = "1" ]; then
+    if [ "${IMAGEFIND_RUN_WEBDAV_STRESS_FULL:-0}" = "1" ]; then
+        "$PYTHON_BIN" "$PROJECT_ROOT/scripts/stress-webdav.py" \
+            --base-url "http://127.0.0.1:$PORT" --token "$API_TOKEN" --workers 6 \
+            --timeout 7200 --media-prefix "$SMOKE_ROOT/ABC-999.mp4" --full
+    elif [ "${IMAGEFIND_RUN_WEBDAV_STRESS_LARGE:-0}" = "1" ]; then
+        "$PYTHON_BIN" "$PROJECT_ROOT/scripts/stress-webdav.py" \
+            --base-url "http://127.0.0.1:$PORT" --token "$API_TOKEN" --workers 4 \
+            --timeout 1800 --media-prefix "$SMOKE_ROOT/ABC-999.mp4" --large
+    else
+        "$PYTHON_BIN" "$PROJECT_ROOT/scripts/stress-webdav.py" \
+            --base-url "http://127.0.0.1:$PORT" --token "$API_TOKEN" --workers 4 \
+            --timeout 600 --media-prefix "$SMOKE_ROOT/ABC-999.mp4"
+    fi
+fi
+
+RANGE_STATUS=$(curl -sS -b "$COOKIE_JAR" -H 'Range: bytes=128-1023' \
+    -o "$SMOKE_ROOT/range.bin" -w '%{http_code}' \
+    "http://127.0.0.1:$PORT/api/v1/videos/$VIDEO_ID/stream")
+[ "$RANGE_STATUS" = "206" ]
+dd if="$SMOKE_ROOT/ABC-999.mp4" of="$SMOKE_ROOT/expected-range.bin" bs=1 skip=128 count=896 2>/dev/null
+cmp "$SMOKE_ROOT/expected-range.bin" "$SMOKE_ROOT/range.bin"
+
+PREVIEW=$(curl -fsS -b "$COOKIE_JAR" -H 'Content-Type: application/json' -H "X-CSRF-Token: $CSRF" \
+    -d '{"start_ms":0}' "http://127.0.0.1:$PORT/api/v1/videos/$VIDEO_ID/preview")
+PLAYLIST_URL=$(printf '%s' "$PREVIEW" | "$PYTHON_BIN" -c \
+    'import json,sys; print(json.load(sys.stdin)["playlist_url"])')
+PLAYLIST=$(curl -fsS -b "$COOKIE_JAR" "http://127.0.0.1:$PORT$PLAYLIST_URL")
+case "$PLAYLIST" in
+    *'#EXTM3U'*'segment-00000.ts'*) ;;
+    *) printf '%s\n' 'HLS playlist did not contain the first segment.' >&2; exit 1 ;;
+esac
+
+printf 'upload=%s video=%s bytes=%s range=206 preview=ok webdav_resume=%s webdav_dedup=true sha256=%s\n' \
+    "$UPLOAD_ID" "$VIDEO_ID" "$SIZE" "$DAV_UPLOAD_ID" "$DIGEST"