nanxun/imagefind
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
22b4e6a691e6e10c862e3b9d0d6fed5d2a0f3dcd
imagefind/scripts/smoke-http-upload.sh
T

201 lines
9.8 KiB
Bash
Executable File
Raw Blame History

#!/bin/sh
set -eu
PROJECT_ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
PYTHON_BIN="${PYTHON:-$PROJECT_ROOT/.venv/bin/python}"
FFMPEG_BIN="${FFMPEG:-$PROJECT_ROOT/vendor/ffmpeg}"
FFPROBE_BIN="${FFPROBE:-$PROJECT_ROOT/vendor/ffprobe}"
PORT="${1:-18766}"
SMOKE_ROOT=$(mktemp -d "${TMPDIR:-/tmp}/imagefind-http-smoke.XXXXXX")
DATA_ROOT="$SMOKE_ROOT/data"
MEDIA_ROOT="$SMOKE_ROOT/media"
COOKIE_JAR="$SMOKE_ROOT/cookies"
SERVER_LOG="$SMOKE_ROOT/server.log"
PASSWORD="imagefind-http-smoke-password"
SERVER_PID=""
cleanup() {
STATUS=$?
trap - EXIT HUP INT TERM
if [ -n "$SERVER_PID" ] && kill -0 "$SERVER_PID" 2>/dev/null; then
kill "$SERVER_PID" 2>/dev/null || true
wait "$SERVER_PID" 2>/dev/null || true
fi
if [ "$STATUS" -ne 0 ]; then
printf '%s\n' 'HTTP smoke failed; application log follows:' >&2
tail -100 "$SERVER_LOG" >&2 || true
cp -p "$SERVER_LOG" "$PROJECT_ROOT/dist/http-smoke-failure.log" 2>/dev/null || true
fi
rm -rf -- "$SMOKE_ROOT"
exit "$STATUS"
}
trap cleanup EXIT HUP INT TERM
mkdir -p "$DATA_ROOT" "$MEDIA_ROOT"
"$FFMPEG_BIN" -hide_banner -loglevel error -f lavfi -i "testsrc=size=320x180:rate=24" \
-t 2 -pix_fmt yuv420p -c:v libx264 -movflags +faststart "$SMOKE_ROOT/ABC-999.mp4"
env PYTHONPATH="$PROJECT_ROOT/backend" \
IMAGEFIND_DATA_DIR="$DATA_ROOT" \
IMAGEFIND_FRONTEND_DIR="$PROJECT_ROOT/frontend/dist" \
IMAGEFIND_HOST="127.0.0.1" \
IMAGEFIND_PORT="$PORT" \
IMAGEFIND_UPLOAD_RESERVE_GB="0" \
IMAGEFIND_RESOURCE_DISK_RESERVE_GB="0.5" \
IMAGEFIND_EMBEDDING_BACKEND="hash" \
IMAGEFIND_FFMPEG_PATH="$FFMPEG_BIN" \
IMAGEFIND_FFPROBE_PATH="$FFPROBE_BIN" \
"$PYTHON_BIN" -m imagefind.main >"$SERVER_LOG" 2>&1 &
SERVER_PID=$!
ATTEMPT=0
until curl -fsS "http://127.0.0.1:$PORT/api/v1/status" >/dev/null 2>&1; do
ATTEMPT=$((ATTEMPT + 1))
if [ "$ATTEMPT" -ge 60 ]; then
printf '%s\n' 'ImageFind HTTP service did not become ready.' >&2
exit 1
fi
sleep 1
done
SETUP=$(curl -fsS -c "$COOKIE_JAR" -H 'Content-Type: application/json' \
-d "{\"password\":\"$PASSWORD\"}" "http://127.0.0.1:$PORT/api/v1/setup")
CSRF=$(printf '%s' "$SETUP" | "$PYTHON_BIN" -c 'import json,sys; print(json.load(sys.stdin)["csrf_token"])')
SOURCE=$(curl -fsS -b "$COOKIE_JAR" -H 'Content-Type: application/json' -H "X-CSRF-Token: $CSRF" \
-d "{\"name\":\"HTTP Smoke\",\"path\":\"$MEDIA_ROOT\"}" \
"http://127.0.0.1:$PORT/api/v1/sources/local")
SOURCE_ID=$(printf '%s' "$SOURCE" | "$PYTHON_BIN" -c 'import json,sys; print(json.load(sys.stdin)["id"])')
curl -fsS -b "$COOKIE_JAR" -H 'Content-Type: application/json' -H "X-CSRF-Token: $CSRF" \
-X PATCH -d '{"writable":true}' \
"http://127.0.0.1:$PORT/api/v1/sources/$SOURCE_ID/writable" >/dev/null
SIZE=$(wc -c <"$SMOKE_ROOT/ABC-999.mp4" | tr -d ' ')
UPLOAD=$(curl -fsS -b "$COOKIE_JAR" -H 'Content-Type: application/json' -H "X-CSRF-Token: $CSRF" \
-d "{\"source_id\":\"$SOURCE_ID\",\"relative_path\":\"imports\",\"filename\":\"ABC-999.mp4\",\"size_bytes\":$SIZE}" \
"http://127.0.0.1:$PORT/api/v1/uploads")
UPLOAD_ID=$(printf '%s' "$UPLOAD" | "$PYTHON_BIN" -c 'import json,sys; print(json.load(sys.stdin)["id"])')
DIGEST=$(sha256sum "$SMOKE_ROOT/ABC-999.mp4" | cut -d ' ' -f 1)
curl -fsS -b "$COOKIE_JAR" -H 'Content-Type: application/octet-stream' -H "X-CSRF-Token: $CSRF" \
-H "X-Chunk-SHA256: $DIGEST" -X PUT --data-binary "@$SMOKE_ROOT/ABC-999.mp4" \
"http://127.0.0.1:$PORT/api/v1/uploads/$UPLOAD_ID/chunks/0" >/dev/null
curl -fsS -b "$COOKIE_JAR" -H 'Content-Type: application/json' -H "X-CSRF-Token: $CSRF" \
-d '{}' "http://127.0.0.1:$PORT/api/v1/uploads/$UPLOAD_ID/complete" >/dev/null
ATTEMPT=0
while :; do
UPLOADS=$(curl -fsS -b "$COOKIE_JAR" "http://127.0.0.1:$PORT/api/v1/uploads")
STATE=$(printf '%s' "$UPLOADS" | "$PYTHON_BIN" -c \
'import json,sys; items=json.load(sys.stdin); uid=sys.argv[1]; print(next(x["status"] for x in items if x["id"] == uid))' \
"$UPLOAD_ID")
[ "$STATE" = "completed" ] && break
[ "$STATE" = "failed" ] && { printf '%s\n' "$UPLOADS" >&2; exit 1; }
ATTEMPT=$((ATTEMPT + 1))
[ "$ATTEMPT" -lt 60 ] || { printf '%s\n' "Upload stayed in state $STATE" >&2; exit 1; }
sleep 1
done
VIDEOS=$(curl -fsS -b "$COOKIE_JAR" "http://127.0.0.1:$PORT/api/v1/videos")
VIDEO_ID=$(printf '%s' "$VIDEOS" | "$PYTHON_BIN" -c \
'import json,sys; print(next(x["id"] for x in json.load(sys.stdin) if x["display_name"] == "ABC-999.mp4"))')
curl -fsS -b "$COOKIE_JAR" "http://127.0.0.1:$PORT/api/v1/videos/$VIDEO_ID/download" \
-o "$SMOKE_ROOT/download.mp4"
cmp "$SMOKE_ROOT/ABC-999.mp4" "$SMOKE_ROOT/download.mp4"
TOKEN_RESPONSE=$(curl -fsS -b "$COOKIE_JAR" -H 'Content-Type: application/json' \
-H "X-CSRF-Token: $CSRF" -d '{"name":"HTTP WebDAV smoke"}' \
"http://127.0.0.1:$PORT/api/v1/tokens")
API_TOKEN=$(printf '%s' "$TOKEN_RESPONSE" | "$PYTHON_BIN" -c \
'import json,sys; print(json.load(sys.stdin)["token"])')
curl -fsS -b "$COOKIE_JAR" -H 'Content-Type: application/json' -H "X-CSRF-Token: $CSRF" \
-X PATCH \
-d "{\"enabled\":true,\"source_id\":\"$SOURCE_ID\",\"relative_path\":\"webdav-smoke\"}" \
"http://127.0.0.1:$PORT/api/v1/webdav/config" >/dev/null
DAV_SIZE=$SIZE
DAV_SPLIT=$((DAV_SIZE / 2))
DAV_REMAINING=$((DAV_SIZE - DAV_SPLIT))
dd if="$SMOKE_ROOT/ABC-999.mp4" of="$SMOKE_ROOT/dav-first.bin" bs=1 count="$DAV_SPLIT" 2>/dev/null
dd if="$SMOKE_ROOT/ABC-999.mp4" of="$SMOKE_ROOT/dav-second.bin" bs=1 skip="$DAV_SPLIT" 2>/dev/null
DAV_URL="http://127.0.0.1:$PORT/webdav/DAV-RESUME.mp4"
FIRST_STATUS=$(curl -sS -u "imagefind:$API_TOKEN" -X PUT \
-H "Content-Range: bytes 0-$((DAV_SPLIT - 1))/$DAV_SIZE" \
-H "Content-Length: $DAV_SPLIT" --data-binary "@$SMOKE_ROOT/dav-first.bin" \
-D "$SMOKE_ROOT/dav-first.headers" -o "$SMOKE_ROOT/dav-first.body" -w '%{http_code}' "$DAV_URL")
[ "$FIRST_STATUS" = "204" ]
FIRST_OFFSET=$(tr -d '\r' <"$SMOKE_ROOT/dav-first.headers" | awk \
'tolower($1)=="upload-offset:" {print $2; exit}')
[ "$FIRST_OFFSET" = "$DAV_SPLIT" ]
HEAD_STATUS=$(curl -sS -u "imagefind:$API_TOKEN" -I \
-D "$SMOKE_ROOT/dav-head.headers" -o /dev/null -w '%{http_code}' "$DAV_URL")
[ "$HEAD_STATUS" = "200" ]
HEAD_OFFSET=$(tr -d '\r' <"$SMOKE_ROOT/dav-head.headers" | awk \
'tolower($1)=="upload-offset:" {print $2; exit}')
[ "$HEAD_OFFSET" = "$DAV_SPLIT" ]
SECOND_STATUS=$(curl -sS -u "imagefind:$API_TOKEN" -X PUT \
-H "Content-Range: bytes $DAV_SPLIT-$((DAV_SIZE - 1))/$DAV_SIZE" \
-H "Content-Length: $DAV_REMAINING" --data-binary "@$SMOKE_ROOT/dav-second.bin" \
-D "$SMOKE_ROOT/dav-second.headers" -o "$SMOKE_ROOT/dav-second.body" -w '%{http_code}' "$DAV_URL")
[ "$SECOND_STATUS" = "204" ]
DAV_UPLOAD_ID=$(tr -d '\r' <"$SMOKE_ROOT/dav-second.headers" | awk \
'tolower($1)=="x-imagefind-upload-id:" {print $2; exit}')
[ -n "$DAV_UPLOAD_ID" ]
ATTEMPT=0
while :; do
UPLOADS=$(curl -fsS -b "$COOKIE_JAR" "http://127.0.0.1:$PORT/api/v1/uploads")
STATE=$(printf '%s' "$UPLOADS" | "$PYTHON_BIN" -c \
'import json,sys; items=json.load(sys.stdin); uid=sys.argv[1]; print(next(x["status"] for x in items if x["id"] == uid))' \
"$DAV_UPLOAD_ID")
[ "$STATE" = "completed" ] && break
[ "$STATE" = "failed" ] && { printf '%s\n' "$UPLOADS" >&2; exit 1; }
ATTEMPT=$((ATTEMPT + 1))
[ "$ATTEMPT" -lt 60 ] || { printf '%s\n' "WebDAV upload stayed in state $STATE" >&2; exit 1; }
sleep 1
done
cmp "$SMOKE_ROOT/ABC-999.mp4" "$MEDIA_ROOT/webdav-smoke/DAV-RESUME.mp4"
DEDUP_STATUS=$(curl -sS -u "imagefind:$API_TOKEN" -X PUT \
-H "Content-Length: $DAV_SIZE" -H "X-Content-SHA256: $DIGEST" \
--data-binary "@$SMOKE_ROOT/ABC-999.mp4" -D "$SMOKE_ROOT/dav-dedup.headers" \
-o "$SMOKE_ROOT/dav-dedup.body" -w '%{http_code}' "$DAV_URL")
[ "$DEDUP_STATUS" = "204" ]
DEDUPLICATED=$(tr -d '\r' <"$SMOKE_ROOT/dav-dedup.headers" | awk \
'tolower($1)=="x-imagefind-deduplicated:" {print tolower($2); exit}')
[ "$DEDUPLICATED" = "true" ]
if [ "${IMAGEFIND_RUN_WEBDAV_STRESS:-0}" = "1" ]; then
if [ "${IMAGEFIND_RUN_WEBDAV_STRESS_FULL:-0}" = "1" ]; then
"$PYTHON_BIN" "$PROJECT_ROOT/scripts/stress-webdav.py" \
--base-url "http://127.0.0.1:$PORT" --token "$API_TOKEN" --workers 6 \
--timeout 7200 --media-prefix "$SMOKE_ROOT/ABC-999.mp4" --full
elif [ "${IMAGEFIND_RUN_WEBDAV_STRESS_LARGE:-0}" = "1" ]; then
"$PYTHON_BIN" "$PROJECT_ROOT/scripts/stress-webdav.py" \
--base-url "http://127.0.0.1:$PORT" --token "$API_TOKEN" --workers 4 \
--timeout 1800 --media-prefix "$SMOKE_ROOT/ABC-999.mp4" --large
else
"$PYTHON_BIN" "$PROJECT_ROOT/scripts/stress-webdav.py" \
--base-url "http://127.0.0.1:$PORT" --token "$API_TOKEN" --workers 4 \
--timeout 600 --media-prefix "$SMOKE_ROOT/ABC-999.mp4"
fi
fi
RANGE_STATUS=$(curl -sS -b "$COOKIE_JAR" -H 'Range: bytes=128-1023' \
-o "$SMOKE_ROOT/range.bin" -w '%{http_code}' \
"http://127.0.0.1:$PORT/api/v1/videos/$VIDEO_ID/stream")
[ "$RANGE_STATUS" = "206" ]
dd if="$SMOKE_ROOT/ABC-999.mp4" of="$SMOKE_ROOT/expected-range.bin" bs=1 skip=128 count=896 2>/dev/null
cmp "$SMOKE_ROOT/expected-range.bin" "$SMOKE_ROOT/range.bin"
PREVIEW=$(curl -fsS -b "$COOKIE_JAR" -H 'Content-Type: application/json' -H "X-CSRF-Token: $CSRF" \
-d '{"start_ms":0}' "http://127.0.0.1:$PORT/api/v1/videos/$VIDEO_ID/preview")
PLAYLIST_URL=$(printf '%s' "$PREVIEW" | "$PYTHON_BIN" -c \
'import json,sys; print(json.load(sys.stdin)["playlist_url"])')
PLAYLIST=$(curl -fsS -b "$COOKIE_JAR" "http://127.0.0.1:$PORT$PLAYLIST_URL")
case "$PLAYLIST" in
*'#EXTM3U'*'segment-00000.ts'*) ;;
*) printf '%s\n' 'HLS playlist did not contain the first segment.' >&2; exit 1 ;;
esac
printf 'upload=%s video=%s bytes=%s range=206 preview=ok webdav_resume=%s webdav_dedup=true sha256=%s\n' \
"$UPLOAD_ID" "$VIDEO_ID" "$SIZE" "$DAV_UPLOAD_ID" "$DIGEST"
Reference in New Issue View Git Blame Copy Permalink