1716 lines
70 KiB
Python
1716 lines
70 KiB
Python
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import base64
|
|
import ctypes
|
|
import errno
|
|
import hashlib
|
|
import mimetypes
|
|
import os
|
|
import posixpath
|
|
import re
|
|
import shutil
|
|
import threading
|
|
import time
|
|
import uuid
|
|
import xml.etree.ElementTree as ET
|
|
from datetime import UTC, datetime
|
|
from email.utils import format_datetime
|
|
from pathlib import Path
|
|
from typing import Annotated
|
|
from urllib.parse import quote, unquote, urlsplit, urlunsplit
|
|
|
|
from fastapi import APIRouter, Depends, HTTPException, Request
|
|
from fastapi.responses import Response, StreamingResponse
|
|
from pydantic import BaseModel, Field
|
|
|
|
from .api import require_auth, services, stream_video
|
|
from .collections import (
|
|
assign_videos,
|
|
collection_by_name,
|
|
create_collection,
|
|
create_group,
|
|
delete_group,
|
|
move_item,
|
|
rename_group,
|
|
)
|
|
from .container import Services
|
|
from .database import utcnow
|
|
from .remote import safe_relative_path
|
|
from .sources import VIDEO_EXTENSIONS
|
|
|
|
router = APIRouter()
|
|
api_router = APIRouter(prefix="/api")
|
|
|
|
DAV = "DAV:"
|
|
IMAGEFIND_DAV = "urn:imagefind:webdav"
|
|
ET.register_namespace("d", DAV)
|
|
ET.register_namespace("if", IMAGEFIND_DAV)
|
|
TEMP_SUFFIXES = {".part", ".tmp", ".download", ".upload", ".crdownload"}
|
|
_CONTENT_RANGE = re.compile(r"^bytes\s+(\d+)-(\d+)/(\d+|\*)$", re.IGNORECASE)
|
|
WEBDAV_WRITE_QUANTUM = 512 * 1024
|
|
WEBDAV_WRITEBACK_INTERVAL = 8 * 1024**2
|
|
WEBDAV_DISK_CHECK_INTERVAL = 64 * 1024**2
|
|
WEBDAV_PROGRESS_INTERVAL_SECONDS = 10
|
|
WEBDAV_PROGRESS_INTERVAL_BYTES = 256 * 1024**2
|
|
_SYNC_FILE_RANGE_WRITE = 2
|
|
try:
|
|
_libc = ctypes.CDLL(None, use_errno=True)
|
|
_sync_file_range = _libc.sync_file_range
|
|
_sync_file_range.argtypes = [ctypes.c_int, ctypes.c_longlong, ctypes.c_longlong, ctypes.c_uint]
|
|
_sync_file_range.restype = ctypes.c_int
|
|
except (AttributeError, OSError): # pragma: no cover - non-Linux fallback
|
|
_sync_file_range = None
|
|
_path_locks_guard = threading.Lock()
|
|
_path_locks: dict[str, threading.Lock] = {}
|
|
|
|
|
|
def _path_lock(virtual_path: str) -> threading.Lock:
|
|
with _path_locks_guard:
|
|
return _path_locks.setdefault(virtual_path, threading.Lock())
|
|
|
|
|
|
def _release_path_lock(virtual_path: str, lock: threading.Lock) -> None:
|
|
lock.release()
|
|
with _path_locks_guard:
|
|
if _path_locks.get(virtual_path) is lock and not lock.locked():
|
|
_path_locks.pop(virtual_path, None)
|
|
|
|
|
|
async def _background_io(app: Services, function, /, *args, **kwargs):
|
|
# WebDAV disk and SQLite calls must never run on the ASGI loop. This is
|
|
# independent of the AI embedding backend: hash-mode development servers
|
|
# can receive the same multi-gigabyte uploads as fnOS/OpenVINO installs.
|
|
return await asyncio.to_thread(function, *args, **kwargs)
|
|
|
|
|
|
class WebDavConfigBody(BaseModel):
|
|
enabled: bool = False
|
|
source_id: str | None = None
|
|
relative_path: str = Field(default="", max_length=4096)
|
|
|
|
|
|
def _config(app: Services) -> dict:
|
|
value = app.db.setting("webdav_server", {})
|
|
value = value if isinstance(value, dict) else {}
|
|
return {
|
|
"enabled": bool(value.get("enabled", False)),
|
|
"source_id": value.get("source_id") or None,
|
|
"relative_path": str(value.get("relative_path") or ""),
|
|
}
|
|
|
|
|
|
def _webdav_url(request: Request, app: Services) -> str:
|
|
"""Return the native HTTP listener, never the fnOS HTML gateway.
|
|
|
|
fnOS' gateway understands browser sessions but does not forward WebDAV
|
|
authentication and methods. Keeping the request hostname is useful when
|
|
this endpoint is queried through that gateway, while the configured native
|
|
listener port makes the result usable by a real DAV client.
|
|
"""
|
|
|
|
forwarded_host = (request.headers.get("x-forwarded-host") or "").split(",", 1)[0].strip()
|
|
authority = forwarded_host or request.url.netloc
|
|
try:
|
|
parsed = urlsplit(f"//{authority}")
|
|
hostname = parsed.hostname or request.url.hostname or "127.0.0.1"
|
|
except ValueError:
|
|
hostname = request.url.hostname or "127.0.0.1"
|
|
host = f"[{hostname}]" if ":" in hostname and not hostname.startswith("[") else hostname
|
|
return urlunsplit(("http", f"{host}:{app.settings.port}", "/webdav/", "", ""))
|
|
|
|
|
|
def _public_config(request: Request, app: Services, value: dict) -> dict:
|
|
return {
|
|
**value,
|
|
"url": _webdav_url(request, app),
|
|
"username": "imagefind",
|
|
"direct_access": app.settings.direct_access,
|
|
"direct_port": app.settings.port,
|
|
"gateway_supported": False,
|
|
}
|
|
|
|
|
|
@api_router.get("/v1/webdav/config")
|
|
async def get_webdav_config(
|
|
request: Request,
|
|
app: Annotated[Services, Depends(services)],
|
|
_: Annotated[dict, Depends(require_auth)],
|
|
):
|
|
return _public_config(request, app, _config(app))
|
|
|
|
|
|
@api_router.patch("/v1/webdav/config")
|
|
async def update_webdav_config(
|
|
body: WebDavConfigBody,
|
|
request: Request,
|
|
app: Annotated[Services, Depends(services)],
|
|
_: Annotated[dict, Depends(require_auth)],
|
|
):
|
|
relative = safe_relative_path(body.relative_path)
|
|
if body.enabled and not app.settings.direct_access:
|
|
raise HTTPException(
|
|
409,
|
|
"WebDAV 只能通过直接 Web/API 访问；请先在飞牛应用配置中开启直接访问",
|
|
)
|
|
if body.enabled and not body.source_id:
|
|
raise HTTPException(400, "启用 WebDAV 前必须选择默认可写媒体库")
|
|
if body.source_id:
|
|
try:
|
|
app.storage.require_writable(body.source_id)
|
|
except KeyError as exc:
|
|
raise HTTPException(404, "目标媒体库不存在") from exc
|
|
except PermissionError as exc:
|
|
raise HTTPException(403, str(exc)) from exc
|
|
value = {"enabled": body.enabled, "source_id": body.source_id, "relative_path": relative}
|
|
app.db.set_setting("webdav_server", value)
|
|
return _public_config(request, app, value)
|
|
|
|
|
|
def _challenge(detail: str = "需要有效的 ImageFind API Token") -> HTTPException:
|
|
return HTTPException(
|
|
401,
|
|
detail,
|
|
headers={"WWW-Authenticate": 'Basic realm="ImageFind WebDAV", charset="UTF-8"'},
|
|
)
|
|
|
|
|
|
async def _authenticate(request: Request, app: Services) -> None:
|
|
value = request.headers.get("authorization", "")
|
|
if not value.lower().startswith("basic "):
|
|
raise _challenge()
|
|
try:
|
|
username, token = base64.b64decode(value.split(None, 1)[1], validate=True).decode().split(":", 1)
|
|
except (ValueError, UnicodeDecodeError) as exc:
|
|
raise _challenge("WebDAV Basic 凭据格式无效") from exc
|
|
if username != "imagefind" or not app.auth.verify_api_token(token, "webdav"):
|
|
raise _challenge("WebDAV 用户名或 API Token 无效")
|
|
|
|
|
|
def _parts(raw_path: str) -> list[str]:
|
|
decoded = unquote(raw_path or "").strip("/")
|
|
if not decoded:
|
|
return []
|
|
safe = safe_relative_path(decoded, allow_empty=False)
|
|
parts = safe.split("/")
|
|
if any(not value.strip() for value in parts):
|
|
raise HTTPException(409, "WebDAV 路径中不能包含空目录名")
|
|
return parts
|
|
|
|
|
|
def _href(request: Request, parts: list[str], *, directory: bool = False) -> str:
|
|
root = str(request.scope.get("root_path") or "").rstrip("/")
|
|
suffix = "/".join(quote(part, safe="") for part in parts)
|
|
value = f"{root}/webdav/" + suffix if root else "/webdav/" + suffix
|
|
if directory and not value.endswith("/"):
|
|
value += "/"
|
|
return value
|
|
|
|
|
|
def _collection(app: Services, name: str, *, create: bool = False) -> dict | None:
|
|
with app.db.transaction() if create else app.db.read() as conn:
|
|
row = collection_by_name(conn, name)
|
|
if not row and create:
|
|
collection_id = create_collection(conn, name)
|
|
row = conn.execute("SELECT * FROM collections WHERE id=?", (collection_id,)).fetchone()
|
|
return dict(row) if row else None
|
|
|
|
|
|
def _group(
|
|
app: Services,
|
|
collection_id: str,
|
|
names: list[str],
|
|
*,
|
|
create: bool = False,
|
|
) -> dict | None:
|
|
parent_id: str | None = None
|
|
with app.db.transaction() if create else app.db.read() as conn:
|
|
for name in names:
|
|
condition = "parent_id IS NULL" if parent_id is None else "parent_id=?"
|
|
args: tuple[object, ...] = (
|
|
(collection_id, name) if parent_id is None else (collection_id, parent_id, name)
|
|
)
|
|
row = conn.execute(
|
|
f"SELECT * FROM collection_items WHERE collection_id=? AND {condition} "
|
|
"AND kind='group' AND lower(name)=lower(?)",
|
|
args,
|
|
).fetchone()
|
|
if not row and create:
|
|
group_id = create_group(conn, collection_id, name, parent_id)
|
|
row = conn.execute("SELECT * FROM collection_items WHERE id=?", (group_id,)).fetchone()
|
|
if not row:
|
|
return None
|
|
parent_id = str(row["id"])
|
|
if not names:
|
|
return {"id": None, "collection_id": collection_id, "name": ""}
|
|
return dict(row) # type: ignore[arg-type]
|
|
|
|
|
|
def _video(
|
|
app: Services, collection_id: str, parent_id: str | None, filename: str
|
|
) -> dict | None:
|
|
with app.db.read() as conn:
|
|
condition = "ci.parent_id IS NULL" if parent_id is None else "ci.parent_id=?"
|
|
args: tuple[object, ...] = (
|
|
(collection_id, filename) if parent_id is None else (collection_id, parent_id, filename)
|
|
)
|
|
row = conn.execute(
|
|
"SELECT v.*,ci.id AS collection_item_id,ci.parent_id AS collection_parent_id "
|
|
"FROM collection_items ci JOIN videos v ON v.id=ci.video_id "
|
|
f"WHERE ci.collection_id=? AND {condition} AND v.display_name=? AND v.available=1 "
|
|
"ORDER BY ci.position,v.updated_at DESC LIMIT 1",
|
|
args,
|
|
).fetchone()
|
|
return dict(row) if row else None
|
|
|
|
|
|
def _root_video(app: Services, config: dict, filename: str) -> dict | None:
|
|
key = posixpath.join(config["relative_path"], filename) if config["relative_path"] else filename
|
|
with app.db.read() as conn:
|
|
row = conn.execute(
|
|
"SELECT v.* FROM videos v WHERE v.source_id=? AND v.source_key=? AND v.available=1 "
|
|
"AND NOT EXISTS(SELECT 1 FROM collection_videos cv WHERE cv.video_id=v.id) "
|
|
"ORDER BY v.updated_at DESC LIMIT 1",
|
|
(config["source_id"], key),
|
|
).fetchone()
|
|
return dict(row) if row else None
|
|
|
|
|
|
def _root_videos(app: Services, config: dict) -> list[dict]:
|
|
directory = config["relative_path"]
|
|
with app.db.read() as conn:
|
|
rows = conn.execute(
|
|
"SELECT v.* FROM videos v WHERE v.source_id=? AND v.available=1 "
|
|
"AND NOT EXISTS(SELECT 1 FROM collection_videos cv WHERE cv.video_id=v.id) "
|
|
"ORDER BY v.display_name",
|
|
(config["source_id"],),
|
|
).fetchall()
|
|
return [
|
|
dict(row)
|
|
for row in rows
|
|
if posixpath.dirname(str(row["source_key"])) == directory
|
|
]
|
|
|
|
|
|
def _normalize_staging(row) -> dict:
|
|
item = dict(row)
|
|
item["_resource_origin"] = "staging"
|
|
path = Path(item["temp_path"])
|
|
if path.is_file():
|
|
actual_size = path.stat().st_size
|
|
expected = item.get("expected_size")
|
|
if expected is None or actual_size <= int(expected):
|
|
# Disk is authoritative after a process crash: the last pwrite can
|
|
# be durable even if its two-second SQLite checkpoint was delayed.
|
|
item["received_bytes"] = actual_size
|
|
item["size_bytes"] = actual_size
|
|
return item
|
|
|
|
|
|
def _raw_staging(app: Services, virtual_path: str) -> dict | None:
|
|
with app.db.read() as conn:
|
|
row = conn.execute(
|
|
"SELECT * FROM webdav_staging WHERE virtual_path=?", (virtual_path,)
|
|
).fetchone()
|
|
return _normalize_staging(row) if row else None
|
|
|
|
|
|
def _pending_upload_resource(row) -> dict:
|
|
item = dict(row)
|
|
status = str(item.pop("upload_status"))
|
|
size = int(item.get("size_bytes") or 0)
|
|
item.update(
|
|
{
|
|
"_resource_origin": "upload",
|
|
"expected_size": size,
|
|
"received_bytes": size,
|
|
"state": "error" if status == "failed" else "indexing" if status == "indexing" else "accepted",
|
|
}
|
|
)
|
|
return item
|
|
|
|
|
|
def _pending_uploads(
|
|
app: Services,
|
|
config: dict,
|
|
*,
|
|
virtual_path: str | None = None,
|
|
collection_id: str | None = None,
|
|
parent_id: str | None = None,
|
|
) -> list[dict]:
|
|
filters = [
|
|
"u.source_id=?",
|
|
"u.origin='webdav'",
|
|
"u.webdav_path IS NOT NULL",
|
|
"u.status IN ('queued','transferring','indexing','failed')",
|
|
"(u.status<>'failed' OR u.failure_stage='catalog')",
|
|
"v.id IS NULL",
|
|
]
|
|
parameters: list[object] = [config["source_id"]]
|
|
if virtual_path is not None:
|
|
filters.append("u.webdav_path=?")
|
|
parameters.append(virtual_path)
|
|
if collection_id is None:
|
|
filters.append("u.collection_id IS NULL")
|
|
else:
|
|
filters.append("u.collection_id=?")
|
|
parameters.append(collection_id)
|
|
if parent_id is None:
|
|
filters.append("u.collection_parent_id IS NULL")
|
|
else:
|
|
filters.append("u.collection_parent_id=?")
|
|
parameters.append(parent_id)
|
|
with app.db.read() as conn:
|
|
rows = conn.execute(
|
|
"SELECT u.id AS upload_id,u.webdav_path AS virtual_path,u.temp_path,u.size_bytes,"
|
|
"u.content_sha256,u.content_sha256_verified,u.status AS upload_status,u.error,"
|
|
"u.collection_id,u.collection_parent_id,"
|
|
"u.created_at,u.updated_at FROM uploads u LEFT JOIN videos v "
|
|
"ON v.source_id=u.source_id AND v.source_key=u.target_key AND v.available=1 WHERE "
|
|
+ " AND ".join(filters)
|
|
+ " ORDER BY u.created_at DESC",
|
|
parameters,
|
|
).fetchall()
|
|
seen: set[str] = set()
|
|
result: list[dict] = []
|
|
for row in rows:
|
|
item = _pending_upload_resource(row)
|
|
if item["virtual_path"] in seen:
|
|
continue
|
|
seen.add(str(item["virtual_path"]))
|
|
result.append(item)
|
|
return result
|
|
|
|
|
|
def _staging(app: Services, virtual_path: str, config: dict | None = None) -> dict | None:
|
|
staged = _raw_staging(app, virtual_path)
|
|
if staged:
|
|
return staged
|
|
pending = _pending_uploads(
|
|
app,
|
|
config or _config(app),
|
|
virtual_path=virtual_path,
|
|
collection_id=None,
|
|
parent_id=None,
|
|
)
|
|
if pending:
|
|
return pending[0]
|
|
# A virtual path inside a collection cannot be filtered without resolving
|
|
# its current group ids. Exact-path lookup is still unambiguous and safe.
|
|
with app.db.read() as conn:
|
|
row = conn.execute(
|
|
"SELECT u.id AS upload_id,u.webdav_path AS virtual_path,u.temp_path,u.size_bytes,"
|
|
"u.content_sha256,u.content_sha256_verified,u.status AS upload_status,u.error,"
|
|
"u.collection_id,u.collection_parent_id,"
|
|
"u.created_at,u.updated_at FROM uploads u LEFT JOIN videos v "
|
|
"ON v.source_id=u.source_id AND v.source_key=u.target_key AND v.available=1 "
|
|
"WHERE u.source_id=? AND u.origin='webdav' AND u.webdav_path=? "
|
|
"AND u.status IN ('queued','transferring','indexing','failed') "
|
|
"AND (u.status<>'failed' OR u.failure_stage='catalog') AND v.id IS NULL "
|
|
"ORDER BY u.created_at DESC LIMIT 1",
|
|
((config or _config(app))["source_id"], virtual_path),
|
|
).fetchone()
|
|
return _pending_upload_resource(row) if row else None
|
|
|
|
|
|
def _staging_rows(
|
|
app: Services,
|
|
config: dict,
|
|
collection_id: str | None,
|
|
parent_id: str | None,
|
|
) -> list[dict]:
|
|
collection_condition = "collection_id IS NULL" if collection_id is None else "collection_id=?"
|
|
parent_condition = "collection_parent_id IS NULL" if parent_id is None else "collection_parent_id=?"
|
|
parameters = tuple(value for value in (collection_id, parent_id) if value is not None)
|
|
with app.db.read() as conn:
|
|
staged = [
|
|
_normalize_staging(row)
|
|
for row in conn.execute(
|
|
f"SELECT * FROM webdav_staging WHERE {collection_condition} AND {parent_condition} "
|
|
"ORDER BY virtual_path",
|
|
parameters,
|
|
).fetchall()
|
|
]
|
|
known = {str(item["virtual_path"]) for item in staged}
|
|
staged.extend(
|
|
item
|
|
for item in _pending_uploads(
|
|
app,
|
|
config,
|
|
collection_id=collection_id,
|
|
parent_id=parent_id,
|
|
)
|
|
if str(item["virtual_path"]) not in known
|
|
)
|
|
return staged
|
|
|
|
|
|
def _upload_mapping(
|
|
app: Services, parts: list[str], *, create: bool
|
|
) -> tuple[dict | None, str | None]:
|
|
if len(parts) == 1:
|
|
return None, None
|
|
collection = _collection(app, parts[0], create=create)
|
|
if not collection:
|
|
raise HTTPException(409 if create else 404, "合集不存在")
|
|
parent = _group(app, collection["id"], parts[1:-1], create=create)
|
|
if parts[1:-1] and not parent:
|
|
raise HTTPException(409 if create else 404, "合集分组不存在")
|
|
return collection, str(parent["id"]) if parent and parent.get("id") else None
|
|
|
|
|
|
def _dav_response(
|
|
href: str,
|
|
display_name: str,
|
|
*,
|
|
directory: bool,
|
|
size: int = 0,
|
|
modified: str | None = None,
|
|
content_type: str | None = None,
|
|
etag: str | None = None,
|
|
upload_offset: int | None = None,
|
|
upload_length: int | None = None,
|
|
upload_state: str | None = None,
|
|
) -> ET.Element:
|
|
response = ET.Element(f"{{{DAV}}}response")
|
|
ET.SubElement(response, f"{{{DAV}}}href").text = href
|
|
propstat = ET.SubElement(response, f"{{{DAV}}}propstat")
|
|
prop = ET.SubElement(propstat, f"{{{DAV}}}prop")
|
|
ET.SubElement(prop, f"{{{DAV}}}displayname").text = display_name
|
|
resource = ET.SubElement(prop, f"{{{DAV}}}resourcetype")
|
|
if directory:
|
|
ET.SubElement(resource, f"{{{DAV}}}collection")
|
|
else:
|
|
ET.SubElement(prop, f"{{{DAV}}}getcontentlength").text = str(max(0, size))
|
|
ET.SubElement(prop, f"{{{DAV}}}getcontenttype").text = content_type or "application/octet-stream"
|
|
ET.SubElement(prop, f"{{{DAV}}}getetag").text = f'"{etag or size}"'
|
|
if upload_offset is not None:
|
|
ET.SubElement(prop, f"{{{IMAGEFIND_DAV}}}upload-offset").text = str(upload_offset)
|
|
if upload_length is not None:
|
|
ET.SubElement(prop, f"{{{IMAGEFIND_DAV}}}upload-length").text = str(upload_length)
|
|
if upload_state:
|
|
ET.SubElement(prop, f"{{{IMAGEFIND_DAV}}}upload-state").text = upload_state
|
|
if modified:
|
|
try:
|
|
parsed = datetime.fromisoformat(modified.replace("Z", "+00:00"))
|
|
ET.SubElement(prop, f"{{{DAV}}}getlastmodified").text = format_datetime(
|
|
parsed.astimezone(UTC), usegmt=True
|
|
)
|
|
except ValueError:
|
|
pass
|
|
ET.SubElement(propstat, f"{{{DAV}}}status").text = "HTTP/1.1 200 OK"
|
|
return response
|
|
|
|
|
|
def _propfind(request: Request, app: Services, config: dict, parts: list[str]) -> Response:
|
|
depth = request.headers.get("depth", "infinity").lower()
|
|
if depth not in {"0", "1"}:
|
|
return Response(
|
|
content=b'<?xml version="1.0"?><d:error xmlns:d="DAV:"><d:propfind-finite-depth/></d:error>',
|
|
status_code=403,
|
|
media_type="application/xml",
|
|
)
|
|
root = ET.Element(f"{{{DAV}}}multistatus")
|
|
if not parts:
|
|
root.append(_dav_response(_href(request, [], directory=True), "ImageFind", directory=True))
|
|
if depth == "1":
|
|
with app.db.read() as conn:
|
|
rows = conn.execute("SELECT id,name,updated_at FROM collections ORDER BY name").fetchall()
|
|
staged_rows = _staging_rows(app, config, None, None)
|
|
known: set[str] = set()
|
|
for row in rows:
|
|
known.add(str(row["name"]))
|
|
root.append(
|
|
_dav_response(
|
|
_href(request, [row["name"]], directory=True),
|
|
row["name"],
|
|
directory=True,
|
|
modified=row["updated_at"],
|
|
)
|
|
)
|
|
for row in _root_videos(app, config):
|
|
known.add(str(row["display_name"]))
|
|
root.append(
|
|
_dav_response(
|
|
_href(request, [row["display_name"]]),
|
|
row["display_name"],
|
|
directory=False,
|
|
size=int(row["size_bytes"] or 0),
|
|
modified=row["modified_at"] or row["updated_at"],
|
|
content_type=mimetypes.guess_type(row["display_name"])[0],
|
|
etag=row["fingerprint"] or row["id"],
|
|
)
|
|
)
|
|
for row in staged_rows:
|
|
filename = str(row["virtual_path"])
|
|
if "/" in filename or filename in known:
|
|
continue
|
|
root.append(
|
|
_dav_response(
|
|
_href(request, [filename]),
|
|
filename,
|
|
directory=False,
|
|
size=int(row.get("received_bytes") or row.get("size_bytes") or 0),
|
|
modified=row["updated_at"],
|
|
content_type=mimetypes.guess_type(filename)[0],
|
|
etag=row.get("content_sha256") or row["updated_at"],
|
|
upload_offset=int(row.get("received_bytes") or 0),
|
|
upload_length=row.get("expected_size"),
|
|
upload_state=row.get("state"),
|
|
)
|
|
)
|
|
else:
|
|
collection = _collection(app, parts[0])
|
|
if not collection:
|
|
if len(parts) != 1:
|
|
raise HTTPException(404, "合集不存在")
|
|
staged = _staging(app, parts[0])
|
|
video = _root_video(app, config, parts[0])
|
|
resource = staged or video
|
|
if not resource:
|
|
raise HTTPException(404, "视频不存在")
|
|
root.append(
|
|
_dav_response(
|
|
_href(request, parts),
|
|
parts[-1],
|
|
directory=False,
|
|
size=int(
|
|
resource.get("received_bytes")
|
|
or resource.get("size_bytes")
|
|
or 0
|
|
),
|
|
modified=(
|
|
resource["updated_at"]
|
|
if staged
|
|
else resource["modified_at"] or resource["updated_at"]
|
|
),
|
|
content_type=mimetypes.guess_type(parts[-1])[0],
|
|
etag=(
|
|
resource.get("content_sha256") or resource["updated_at"]
|
|
if staged
|
|
else resource["fingerprint"] or resource["id"]
|
|
),
|
|
upload_offset=int(resource.get("received_bytes") or 0) if staged else None,
|
|
upload_length=resource.get("expected_size") if staged else None,
|
|
upload_state=resource.get("state") if staged else None,
|
|
)
|
|
)
|
|
content = ET.tostring(root, encoding="utf-8", xml_declaration=True)
|
|
return Response(content=content, status_code=207, media_type="application/xml; charset=utf-8")
|
|
group = _group(app, collection["id"], parts[1:])
|
|
if len(parts) == 1 or group:
|
|
parent_id = str(group["id"]) if group and group.get("id") else None
|
|
modified = group.get("updated_at") if group else collection["updated_at"]
|
|
root.append(
|
|
_dav_response(
|
|
_href(request, parts, directory=True),
|
|
parts[-1],
|
|
directory=True,
|
|
modified=modified,
|
|
)
|
|
)
|
|
if depth == "1":
|
|
condition = "parent_id IS NULL" if parent_id is None else "parent_id=?"
|
|
args: tuple[object, ...] = (
|
|
(collection["id"],) if parent_id is None else (collection["id"], parent_id)
|
|
)
|
|
with app.db.read() as conn:
|
|
groups = conn.execute(
|
|
f"SELECT id,name,updated_at FROM collection_items WHERE collection_id=? AND {condition} "
|
|
"AND kind='group' ORDER BY position,name",
|
|
args,
|
|
).fetchall()
|
|
videos = conn.execute(
|
|
"SELECT v.id,v.display_name,v.size_bytes,v.modified_at,v.updated_at,v.fingerprint "
|
|
"FROM collection_items ci JOIN videos v ON v.id=ci.video_id "
|
|
f"WHERE ci.collection_id=? AND {condition.replace('parent_id', 'ci.parent_id')} "
|
|
"AND ci.kind='video' AND v.available=1 ORDER BY ci.position,v.display_name",
|
|
args,
|
|
).fetchall()
|
|
staged = _staging_rows(app, config, str(collection["id"]), parent_id)
|
|
for child in groups:
|
|
root.append(
|
|
_dav_response(
|
|
_href(request, [*parts, child["name"]], directory=True),
|
|
child["name"],
|
|
directory=True,
|
|
modified=child["updated_at"],
|
|
)
|
|
)
|
|
known: set[str] = set()
|
|
for row in videos:
|
|
known.add(str(row["display_name"]))
|
|
root.append(
|
|
_dav_response(
|
|
_href(request, [*parts, row["display_name"]]),
|
|
row["display_name"],
|
|
directory=False,
|
|
size=int(row["size_bytes"] or 0),
|
|
modified=row["modified_at"] or row["updated_at"],
|
|
content_type=mimetypes.guess_type(row["display_name"])[0],
|
|
etag=row["fingerprint"] or row["id"],
|
|
)
|
|
)
|
|
for row in staged:
|
|
filename = str(row["virtual_path"]).rsplit("/", 1)[-1]
|
|
if filename in known:
|
|
continue
|
|
root.append(
|
|
_dav_response(
|
|
_href(request, [*parts, filename]),
|
|
filename,
|
|
directory=False,
|
|
size=int(row["received_bytes"] or row["size_bytes"] or 0),
|
|
modified=row["updated_at"],
|
|
content_type=mimetypes.guess_type(filename)[0],
|
|
etag=row["content_sha256"] or row["updated_at"],
|
|
upload_offset=int(row["received_bytes"] or 0),
|
|
upload_length=row["expected_size"],
|
|
upload_state=row["state"],
|
|
)
|
|
)
|
|
else:
|
|
parent = _group(app, collection["id"], parts[1:-1])
|
|
if parts[1:-1] and not parent:
|
|
raise HTTPException(404, "合集分组不存在")
|
|
parent_id = str(parent["id"]) if parent and parent.get("id") else None
|
|
virtual = "/".join(parts)
|
|
staged = _staging(app, virtual)
|
|
video = _video(app, collection["id"], parent_id, parts[-1])
|
|
resource = staged or video
|
|
if not resource:
|
|
raise HTTPException(404, "视频不存在")
|
|
root.append(
|
|
_dav_response(
|
|
_href(request, parts),
|
|
parts[-1],
|
|
directory=False,
|
|
size=int(
|
|
resource.get("received_bytes")
|
|
or resource.get("size_bytes")
|
|
or 0
|
|
),
|
|
modified=resource["updated_at"] if staged else resource["modified_at"] or resource["updated_at"],
|
|
content_type=mimetypes.guess_type(parts[-1])[0],
|
|
etag=(
|
|
resource.get("content_sha256") or resource["updated_at"]
|
|
if staged
|
|
else resource["fingerprint"] or resource["id"]
|
|
),
|
|
upload_offset=int(resource.get("received_bytes") or 0) if staged else None,
|
|
upload_length=resource.get("expected_size") if staged else None,
|
|
upload_state=resource.get("state") if staged else None,
|
|
)
|
|
)
|
|
content = ET.tostring(root, encoding="utf-8", xml_declaration=True)
|
|
return Response(content=content, status_code=207, media_type="application/xml; charset=utf-8")
|
|
|
|
|
|
def _is_video(filename: str) -> bool:
|
|
return Path(filename).suffix.lower() in VIDEO_EXTENSIONS
|
|
|
|
|
|
def _is_temporary(filename: str) -> bool:
|
|
lower = filename.lower()
|
|
return any(lower.endswith(suffix) for suffix in TEMP_SUFFIXES) or not _is_video(filename)
|
|
|
|
|
|
def _target_directory(config: dict, hierarchy: list[str]) -> str:
|
|
return safe_relative_path(posixpath.join(config["relative_path"], *hierarchy))
|
|
|
|
|
|
def _put_position(request: Request) -> tuple[int, int | None, int | None]:
|
|
declared_text = request.headers.get("content-length")
|
|
if declared_text is not None and not declared_text.isdigit():
|
|
raise HTTPException(400, "Content-Length 无效")
|
|
declared = int(declared_text) if declared_text is not None else None
|
|
value = request.headers.get("content-range")
|
|
if value:
|
|
match = _CONTENT_RANGE.fullmatch(value.strip())
|
|
if not match:
|
|
raise HTTPException(400, "Content-Range 无效")
|
|
start, end = int(match.group(1)), int(match.group(2))
|
|
total = None if match.group(3) == "*" else int(match.group(3))
|
|
if end < start or (declared is not None and declared != end - start + 1):
|
|
raise HTTPException(400, "Content-Range 与请求长度不一致")
|
|
if total is not None and end >= total:
|
|
raise HTTPException(400, "Content-Range 超出文件大小")
|
|
return start, total, end
|
|
offset_text = request.headers.get("upload-offset")
|
|
total_text = request.headers.get("upload-length")
|
|
if offset_text is not None and not offset_text.isdigit():
|
|
raise HTTPException(400, "Upload-Offset 无效")
|
|
if total_text is not None and not total_text.isdigit():
|
|
raise HTTPException(400, "Upload-Length 无效")
|
|
start = int(offset_text or 0)
|
|
total = int(total_text) if total_text is not None else (declared if start == 0 else None)
|
|
return start, total, None
|
|
|
|
|
|
def _request_sha256(request: Request) -> str | None:
|
|
for name in ("x-content-sha256", "x-checksum-sha256"):
|
|
value = (request.headers.get(name) or "").strip().lower()
|
|
if re.fullmatch(r"[0-9a-f]{64}", value):
|
|
return value
|
|
digest = request.headers.get("content-digest") or request.headers.get("digest") or ""
|
|
for part in digest.split(","):
|
|
name, separator, encoded = part.strip().partition("=")
|
|
if separator and name.lower() in {"sha-256", "sha256"}:
|
|
try:
|
|
raw = base64.b64decode(encoded.strip().strip(":"), validate=True)
|
|
except ValueError:
|
|
continue
|
|
if len(raw) == 32:
|
|
return raw.hex()
|
|
return None
|
|
|
|
|
|
def _persist_staging(
|
|
app: Services,
|
|
virtual: str,
|
|
temporary: Path,
|
|
received: int,
|
|
expected: int | None,
|
|
content_sha256: str | None,
|
|
state: str,
|
|
error: str | None,
|
|
collection_id: str | None,
|
|
parent_id: str | None,
|
|
content_sha256_verified: bool = False,
|
|
) -> None:
|
|
now = utcnow()
|
|
|
|
def write(conn):
|
|
conn.execute(
|
|
"INSERT INTO webdav_staging(virtual_path,temp_path,size_bytes,expected_size,received_bytes,"
|
|
"content_sha256,content_sha256_verified,state,error,collection_id,collection_parent_id,"
|
|
"created_at,updated_at) "
|
|
"VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(virtual_path) DO UPDATE SET "
|
|
"temp_path=excluded.temp_path,size_bytes=excluded.size_bytes,expected_size=excluded.expected_size,"
|
|
"received_bytes=excluded.received_bytes,content_sha256=excluded.content_sha256,state=excluded.state,"
|
|
"content_sha256_verified=excluded.content_sha256_verified,error=excluded.error,"
|
|
"collection_id=excluded.collection_id,"
|
|
"collection_parent_id=excluded.collection_parent_id,updated_at=excluded.updated_at",
|
|
(
|
|
virtual,
|
|
str(temporary),
|
|
received,
|
|
expected,
|
|
received,
|
|
content_sha256,
|
|
int(content_sha256_verified),
|
|
state,
|
|
error,
|
|
collection_id,
|
|
parent_id,
|
|
now,
|
|
now,
|
|
),
|
|
)
|
|
|
|
app.db.write_with_retry(write)
|
|
|
|
|
|
def _pwrite_all(file_descriptor: int, data: bytes, offset: int) -> int:
|
|
view = memoryview(data)
|
|
written_total = 0
|
|
while written_total < len(view):
|
|
written = os.pwrite(file_descriptor, view[written_total:], offset + written_total)
|
|
if written <= 0:
|
|
raise OSError("WebDAV 暂存文件写入未取得进展")
|
|
written_total += written
|
|
return written_total
|
|
|
|
|
|
def _queue_writeback(file_descriptor: int, offset: int, length: int) -> bool:
|
|
"""Start Linux writeback for a received range without waiting for durability.
|
|
|
|
This keeps the process below the global dirty-page throttle that otherwise
|
|
pauses every concurrent PUT at once. Unsupported kernels/filesystems simply
|
|
retain the normal buffered-write behavior.
|
|
"""
|
|
|
|
if _sync_file_range is None or length <= 0:
|
|
return False
|
|
if _sync_file_range(file_descriptor, offset, length, _SYNC_FILE_RANGE_WRITE) == 0:
|
|
return True
|
|
error_number = ctypes.get_errno()
|
|
if error_number in {errno.EINVAL, errno.ENOSYS, errno.EOPNOTSUPP}:
|
|
return False
|
|
raise OSError(error_number, os.strerror(error_number))
|
|
|
|
|
|
def _open_staging(path: Path, start: int) -> int:
|
|
"""Open a staging file without request-path durability barriers.
|
|
|
|
O_DSYNC looked attractive as a way to avoid a final writeback cliff, but on
|
|
slow or busy volumes every small write then becomes a durability barrier.
|
|
With several clients those barriers serialize and stop later request
|
|
sockets being read for longer than common DAV client write timeouts. The
|
|
receiver instead uses bounded, fairly interleaved buffered writes and the
|
|
transfer worker subsequently consumes and atomically commits the staging
|
|
file. A power loss may therefore leave a resumable partial upload, but it
|
|
cannot expose an incomplete video as completed.
|
|
"""
|
|
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
file_descriptor = os.open(path, os.O_RDWR | os.O_CREAT, 0o600)
|
|
try:
|
|
actual_size = os.fstat(file_descriptor).st_size
|
|
if actual_size < start:
|
|
raise OSError(
|
|
f"WebDAV 暂存文件短于已确认续传偏移：文件 {actual_size} 字节，偏移 {start} 字节"
|
|
)
|
|
# A process may stop after pwrite completed but before the matching
|
|
# database checkpoint committed. Only the persisted offset is safe to
|
|
# acknowledge to a resumed DAV client, so discard that unconfirmed
|
|
# tail instead of producing a duplicated/corrupted range.
|
|
if actual_size != start:
|
|
os.ftruncate(file_descriptor, start)
|
|
return file_descriptor
|
|
except Exception:
|
|
os.close(file_descriptor)
|
|
raise
|
|
|
|
|
|
def _staging_reservation(app: Services, virtual: str) -> int:
|
|
with app.db.read() as conn:
|
|
return int(
|
|
conn.execute(
|
|
"SELECT coalesce(sum(size_bytes),0) FROM uploads "
|
|
"WHERE status IN ('receiving','queued','transferring')"
|
|
).fetchone()[0]
|
|
) + int(
|
|
conn.execute(
|
|
"SELECT coalesce(sum(size_bytes),0) FROM webdav_staging WHERE virtual_path<>?",
|
|
(virtual,),
|
|
).fetchone()[0]
|
|
)
|
|
|
|
|
|
def _assign_duplicate_video(
|
|
app: Services,
|
|
source_id: str,
|
|
duplicate_key: str,
|
|
collection_id: str | None,
|
|
parent_id: str | None,
|
|
) -> None:
|
|
if not collection_id:
|
|
return
|
|
with app.db.transaction() as conn:
|
|
video = conn.execute(
|
|
"SELECT id FROM videos WHERE source_id=? AND source_key=? AND available=1",
|
|
(source_id, duplicate_key),
|
|
).fetchone()
|
|
if video:
|
|
assign_videos(conn, collection_id, [video["id"]], parent_id=parent_id)
|
|
|
|
|
|
def _delete_staging(app: Services, virtual: str) -> None:
|
|
app.db.write_with_retry(
|
|
lambda conn: conn.execute("DELETE FROM webdav_staging WHERE virtual_path=?", (virtual,))
|
|
)
|
|
|
|
|
|
async def _put(request: Request, app: Services, config: dict, parts: list[str]) -> Response:
|
|
if not parts:
|
|
raise HTTPException(409, "WebDAV 根目录不能作为文件覆盖")
|
|
virtual = "/".join(parts)
|
|
lock = _path_lock(virtual)
|
|
if not lock.acquire(blocking=False):
|
|
current = await _background_io(app, _staging, app, virtual, config)
|
|
offset = int(current.get("received_bytes") or 0) if current else 0
|
|
raise HTTPException(
|
|
423,
|
|
"同一路径正在上传，请稍后继续",
|
|
headers={"Upload-Offset": str(offset), "Retry-After": "2"},
|
|
)
|
|
try:
|
|
# Do not create collections/groups before the request body is read.
|
|
# A DAV client commonly waits for ``100 Continue`` at this point; a
|
|
# contended SQLite writer would otherwise delay the first body byte for
|
|
# tens of seconds and make the client report a network rollback.
|
|
try:
|
|
collection, parent_id = await _background_io(
|
|
app, _upload_mapping, app, parts, create=False
|
|
)
|
|
except HTTPException as exc:
|
|
if exc.status_code != 404:
|
|
raise
|
|
collection, parent_id = None, None
|
|
collection_id = str(collection["id"]) if collection else None
|
|
start, expected, range_end = _put_position(request)
|
|
expected_hash = _request_sha256(request)
|
|
maximum = 200 * 1024**3
|
|
if start > maximum or (expected is not None and (expected <= 0 or expected > maximum)):
|
|
raise HTTPException(413, "WebDAV 单文件最大 200 GB")
|
|
target_directory = _target_directory(config, parts[:-1])
|
|
if expected_hash and expected is not None and start == 0 and not _is_temporary(parts[-1]):
|
|
duplicate_upload = await _background_io(
|
|
app,
|
|
app.uploads.matching_webdav_upload,
|
|
config["source_id"],
|
|
virtual,
|
|
size_bytes=expected,
|
|
content_sha256=expected_hash,
|
|
)
|
|
if duplicate_upload:
|
|
return Response(
|
|
status_code=204,
|
|
headers={
|
|
"X-ImageFind-Deduplicated": "true",
|
|
"X-ImageFind-Upload-Id": str(duplicate_upload["id"]),
|
|
"Upload-Offset": str(expected),
|
|
"Upload-Length": str(expected),
|
|
"ETag": f'"sha256:{expected_hash}"',
|
|
},
|
|
)
|
|
duplicate_key = await _background_io(
|
|
app,
|
|
app.storage.matching_content_key,
|
|
config["source_id"],
|
|
target_directory,
|
|
parts[-1],
|
|
size_bytes=expected,
|
|
content_sha256=expected_hash,
|
|
)
|
|
if duplicate_key:
|
|
if collection_id:
|
|
with app.db.transaction() as conn:
|
|
video = conn.execute(
|
|
"SELECT id FROM videos WHERE source_id=? AND source_key=? AND available=1",
|
|
(config["source_id"], duplicate_key),
|
|
).fetchone()
|
|
if video:
|
|
assign_videos(conn, collection_id, [video["id"]], parent_id=parent_id)
|
|
return Response(
|
|
status_code=204,
|
|
headers={
|
|
"X-ImageFind-Deduplicated": "true",
|
|
"Upload-Offset": str(expected),
|
|
"Upload-Length": str(expected),
|
|
"ETag": f'"sha256:{expected_hash}"',
|
|
},
|
|
)
|
|
|
|
old = await _background_io(app, _staging, app, virtual, config)
|
|
old_received = int(old.get("received_bytes") or old.get("size_bytes") or 0) if old else 0
|
|
temporary = (
|
|
Path(old["temp_path"])
|
|
if old and Path(old["temp_path"]).is_file()
|
|
else app.settings.upload_staging_dir / f"webdav-{uuid.uuid4().hex}.part"
|
|
)
|
|
if start and (not old or not temporary.is_file() or old_received != start):
|
|
raise HTTPException(
|
|
409,
|
|
f"续传偏移不一致，服务器当前偏移为 {old_received if temporary.is_file() else 0}",
|
|
headers={"Upload-Offset": str(old_received if temporary.is_file() else 0)},
|
|
)
|
|
if start > 0 and old and expected is None:
|
|
expected = old.get("expected_size")
|
|
if start > 0 and old and expected_hash is None:
|
|
expected_hash = old.get("content_sha256")
|
|
if expected is not None and start > expected:
|
|
raise HTTPException(409, "续传偏移超过文件大小", headers={"Upload-Offset": str(old_received)})
|
|
|
|
reserved = await _background_io(app, _staging_reservation, app, virtual)
|
|
quota = int(app.settings.upload_staging_gb * 1024**3)
|
|
anticipated = expected if expected is not None else start
|
|
if reserved + anticipated > quota:
|
|
raise HTTPException(507, "上传暂存配额不足")
|
|
reserve = int(app.settings.upload_reserve_gb * 1024**3)
|
|
file_descriptor = await _background_io(app, _open_staging, temporary, start)
|
|
received = start
|
|
hasher = hashlib.sha256() if start == 0 else None
|
|
write_buffer = bytearray()
|
|
write_buffer_start = start
|
|
writeback_start = start
|
|
writeback_enabled = True
|
|
last_saved_at = time.monotonic()
|
|
last_saved_bytes = start
|
|
last_disk_check_bytes = start
|
|
staging_persisted = False
|
|
|
|
async def flush_buffer(*, complete: bool = False) -> None:
|
|
nonlocal write_buffer_start, writeback_start, writeback_enabled
|
|
# Never turn a large ASGI body chunk into one unbounded disk
|
|
# operation. Each PUT writes its own file in a worker thread; a
|
|
# device-level dirty-page stall in one request must not hold a
|
|
# process-wide lock and stop all other client sockets being read.
|
|
while len(write_buffer) >= WEBDAV_WRITE_QUANTUM or (complete and write_buffer):
|
|
length = min(len(write_buffer), WEBDAV_WRITE_QUANTUM)
|
|
data = bytes(write_buffer[:length])
|
|
written = await _background_io(
|
|
app, _pwrite_all, file_descriptor, data, write_buffer_start
|
|
)
|
|
if written != len(data):
|
|
raise OSError("WebDAV 暂存文件写入不完整")
|
|
write_buffer_start += written
|
|
del write_buffer[:written]
|
|
if (
|
|
writeback_enabled
|
|
and write_buffer_start - writeback_start >= WEBDAV_WRITEBACK_INTERVAL
|
|
):
|
|
writeback_enabled = await _background_io(
|
|
app,
|
|
_queue_writeback,
|
|
file_descriptor,
|
|
writeback_start,
|
|
write_buffer_start - writeback_start,
|
|
)
|
|
writeback_start = write_buffer_start
|
|
|
|
try:
|
|
async for chunk in request.stream():
|
|
if not chunk:
|
|
continue
|
|
if not staging_persisted:
|
|
# Enter the request stream before the first SQLite write so
|
|
# servers can acknowledge Expect: 100-continue promptly.
|
|
await _background_io(
|
|
app,
|
|
_persist_staging,
|
|
app,
|
|
virtual,
|
|
temporary,
|
|
received,
|
|
expected,
|
|
expected_hash,
|
|
"receiving",
|
|
None,
|
|
collection_id,
|
|
parent_id,
|
|
)
|
|
staging_persisted = True
|
|
received += len(chunk)
|
|
if received > maximum or (expected is not None and received > expected):
|
|
raise HTTPException(413, "上传内容超过声明的文件大小")
|
|
if reserved + received > quota:
|
|
raise HTTPException(507, "上传暂存配额不足")
|
|
if received - last_disk_check_bytes >= WEBDAV_DISK_CHECK_INTERVAL:
|
|
if (await _background_io(app, shutil.disk_usage, temporary.parent)).free < reserve:
|
|
raise HTTPException(507, "NAS 可用空间不足")
|
|
last_disk_check_bytes = received
|
|
if hasher is not None:
|
|
hasher.update(chunk)
|
|
write_buffer.extend(chunk)
|
|
if len(write_buffer) >= WEBDAV_WRITE_QUANTUM:
|
|
await flush_buffer()
|
|
now = time.monotonic()
|
|
if (
|
|
now - last_saved_at >= WEBDAV_PROGRESS_INTERVAL_SECONDS
|
|
or received - last_saved_bytes >= WEBDAV_PROGRESS_INTERVAL_BYTES
|
|
):
|
|
await flush_buffer(complete=True)
|
|
await _background_io(
|
|
app,
|
|
_persist_staging,
|
|
app,
|
|
virtual,
|
|
temporary,
|
|
received,
|
|
expected,
|
|
expected_hash,
|
|
"receiving",
|
|
None,
|
|
collection_id,
|
|
parent_id,
|
|
)
|
|
last_saved_at, last_saved_bytes = now, received
|
|
await flush_buffer(complete=True)
|
|
if writeback_enabled and write_buffer_start > writeback_start:
|
|
await _background_io(
|
|
app,
|
|
_queue_writeback,
|
|
file_descriptor,
|
|
writeback_start,
|
|
write_buffer_start - writeback_start,
|
|
)
|
|
# A synchronous fsync here used to hold the HTTP response for up
|
|
# to several minutes when multiple GiB PUTs completed together.
|
|
# Worse, the resulting device-wide writeback stopped the server
|
|
# reading other request bodies long enough for DAV clients to hit
|
|
# their socket write timeout. pwrite completion plus the strict
|
|
# fstat check below is sufficient to acknowledge receipt; the
|
|
# single transfer worker then consumes/atomically commits this
|
|
# staging file without making client connectivity depend on NAS
|
|
# flush latency.
|
|
final_stat = await _background_io(app, os.fstat, file_descriptor)
|
|
actual_size = final_stat.st_size
|
|
if actual_size != received:
|
|
raise OSError(
|
|
f"WebDAV 暂存文件尺寸异常：已接收 {received} 字节，磁盘记录 {actual_size} 字节"
|
|
)
|
|
except Exception as exc:
|
|
received = os.fstat(file_descriptor).st_size
|
|
await _background_io(
|
|
app,
|
|
_persist_staging,
|
|
app,
|
|
virtual,
|
|
temporary,
|
|
received,
|
|
expected,
|
|
expected_hash,
|
|
"partial",
|
|
str(exc)[:4000],
|
|
collection_id,
|
|
parent_id,
|
|
)
|
|
raise
|
|
finally:
|
|
os.close(file_descriptor)
|
|
|
|
if received <= 0:
|
|
raise HTTPException(400, "不能上传空文件")
|
|
if range_end is not None and received != range_end + 1:
|
|
await _background_io(
|
|
app,
|
|
_persist_staging,
|
|
app,
|
|
virtual,
|
|
temporary,
|
|
received,
|
|
expected,
|
|
expected_hash,
|
|
"partial",
|
|
"本次分块长度不足",
|
|
collection_id,
|
|
parent_id,
|
|
)
|
|
raise HTTPException(400, "Content-Range 与实际上传长度不一致")
|
|
if expected is not None and received < expected:
|
|
await _background_io(
|
|
app,
|
|
_persist_staging,
|
|
app,
|
|
virtual,
|
|
temporary,
|
|
received,
|
|
expected,
|
|
expected_hash,
|
|
"partial",
|
|
None,
|
|
collection_id,
|
|
parent_id,
|
|
)
|
|
return Response(
|
|
status_code=204,
|
|
headers={"Upload-Offset": str(received), "Upload-Length": str(expected)},
|
|
)
|
|
|
|
content_hash = hasher.hexdigest() if hasher is not None else expected_hash
|
|
if expected_hash and hasher is not None and content_hash != expected_hash:
|
|
await _background_io(
|
|
app,
|
|
_persist_staging,
|
|
app,
|
|
virtual,
|
|
temporary,
|
|
received,
|
|
expected,
|
|
content_hash,
|
|
"error",
|
|
"文件 SHA-256 校验失败",
|
|
collection_id,
|
|
parent_id,
|
|
)
|
|
raise HTTPException(422, "文件 SHA-256 校验失败")
|
|
expected = expected or received
|
|
if not _is_temporary(parts[-1]):
|
|
# Collection/group creation is deliberately after body receipt;
|
|
# see the request-stream comment above. Existing directories are
|
|
# still resolved before the body when possible, so normal uploads
|
|
# retain their collection mapping without delaying 100 Continue.
|
|
collection, parent_id = await _background_io(
|
|
app, _upload_mapping, app, parts, create=True
|
|
)
|
|
collection_id = str(collection["id"]) if collection else None
|
|
await _background_io(
|
|
app,
|
|
_persist_staging,
|
|
app,
|
|
virtual,
|
|
temporary,
|
|
received,
|
|
expected,
|
|
content_hash,
|
|
"complete",
|
|
None,
|
|
collection_id,
|
|
parent_id,
|
|
hasher is not None,
|
|
)
|
|
etag = (
|
|
f'"sha256:{content_hash}"'
|
|
if content_hash
|
|
else f'"upload:{received}:{final_stat.st_mtime_ns}"'
|
|
)
|
|
common_headers = {
|
|
"Upload-Offset": str(received),
|
|
"Upload-Length": str(expected),
|
|
"ETag": etag,
|
|
}
|
|
if _is_temporary(parts[-1]):
|
|
return Response(status_code=201 if old is None else 204, headers=common_headers)
|
|
|
|
if content_hash:
|
|
duplicate_upload = await _background_io(
|
|
app,
|
|
app.uploads.matching_webdav_upload,
|
|
config["source_id"],
|
|
virtual,
|
|
size_bytes=received,
|
|
content_sha256=content_hash,
|
|
)
|
|
if duplicate_upload:
|
|
await _background_io(app, temporary.unlink, True)
|
|
await _background_io(app, _delete_staging, app, virtual)
|
|
return Response(
|
|
status_code=204,
|
|
headers={
|
|
**common_headers,
|
|
"X-ImageFind-Deduplicated": "true",
|
|
"X-ImageFind-Upload-Id": str(duplicate_upload["id"]),
|
|
},
|
|
)
|
|
duplicate_key = await _background_io(
|
|
app,
|
|
app.storage.matching_content_key,
|
|
config["source_id"],
|
|
target_directory,
|
|
parts[-1],
|
|
size_bytes=received,
|
|
content_sha256=content_hash,
|
|
)
|
|
if duplicate_key:
|
|
await _background_io(
|
|
app,
|
|
_assign_duplicate_video,
|
|
app,
|
|
config["source_id"],
|
|
duplicate_key,
|
|
collection_id,
|
|
parent_id,
|
|
)
|
|
await _background_io(app, temporary.unlink, True)
|
|
await _background_io(app, _delete_staging, app, virtual)
|
|
return Response(
|
|
status_code=204,
|
|
headers={**common_headers, "X-ImageFind-Deduplicated": "true"},
|
|
)
|
|
try:
|
|
item = await _background_io(
|
|
app,
|
|
app.uploads.adopt_staged,
|
|
config["source_id"],
|
|
target_directory,
|
|
parts[-1],
|
|
temporary,
|
|
collection_id=collection_id,
|
|
collection_parent_id=parent_id,
|
|
content_sha256=content_hash,
|
|
webdav_path=virtual,
|
|
content_sha256_verified=hasher is not None,
|
|
)
|
|
except Exception as exc:
|
|
await _background_io(
|
|
app,
|
|
_persist_staging,
|
|
app,
|
|
virtual,
|
|
temporary,
|
|
received,
|
|
expected,
|
|
content_hash,
|
|
"error",
|
|
str(exc)[:4000],
|
|
collection_id,
|
|
parent_id,
|
|
)
|
|
raise
|
|
await _background_io(app, _delete_staging, app, virtual)
|
|
return Response(
|
|
status_code=201 if old is None else 204,
|
|
headers={**common_headers, "X-ImageFind-Upload-Id": item["id"]},
|
|
)
|
|
finally:
|
|
_release_path_lock(virtual, lock)
|
|
|
|
|
|
def _destination_parts(request: Request) -> list[str]:
|
|
value = request.headers.get("destination")
|
|
if not value:
|
|
raise HTTPException(400, "MOVE 缺少 Destination")
|
|
path = unquote(urlsplit(value).path)
|
|
marker = "/webdav/"
|
|
if marker not in path:
|
|
raise HTTPException(400, "Destination 不属于 ImageFind WebDAV")
|
|
return _parts(path.split(marker, 1)[1])
|
|
|
|
|
|
def _exists(app: Services, config: dict, parts: list[str]) -> bool:
|
|
if not parts:
|
|
return True
|
|
collection = _collection(app, parts[0])
|
|
if not collection:
|
|
return len(parts) == 1 and bool(
|
|
_staging(app, parts[0]) or _root_video(app, config, parts[0])
|
|
)
|
|
if len(parts) == 1 or _group(app, collection["id"], parts[1:]):
|
|
return True
|
|
parent = _group(app, collection["id"], parts[1:-1])
|
|
if parts[1:-1] and not parent:
|
|
return False
|
|
parent_id = str(parent["id"]) if parent and parent.get("id") else None
|
|
return bool(_staging(app, "/".join(parts)) or _video(app, collection["id"], parent_id, parts[-1]))
|
|
|
|
|
|
def _move(request: Request, app: Services, config: dict, parts: list[str]) -> Response:
|
|
if not parts:
|
|
raise HTTPException(403, "不能移动 WebDAV 根目录")
|
|
source_virtual = "/".join(parts)
|
|
staged = _raw_staging(app, source_virtual)
|
|
destination = _destination_parts(request)
|
|
if not destination:
|
|
raise HTTPException(409, "目标文件路径无效")
|
|
if request.headers.get("overwrite", "T").upper() == "F" and _exists(app, config, destination):
|
|
raise HTTPException(412, "目标已存在")
|
|
|
|
if staged:
|
|
collection, destination_parent_id = _upload_mapping(app, destination, create=True)
|
|
destination_collection_id = str(collection["id"]) if collection else None
|
|
destination_virtual = "/".join(destination)
|
|
if _is_temporary(destination[-1]):
|
|
with app.db.transaction() as conn:
|
|
if destination_virtual != source_virtual:
|
|
existing = conn.execute(
|
|
"SELECT temp_path FROM webdav_staging WHERE virtual_path=?", (destination_virtual,)
|
|
).fetchone()
|
|
if existing:
|
|
Path(existing["temp_path"]).unlink(missing_ok=True)
|
|
conn.execute("DELETE FROM webdav_staging WHERE virtual_path=?", (destination_virtual,))
|
|
conn.execute(
|
|
"UPDATE webdav_staging SET virtual_path=?,collection_id=?,collection_parent_id=?,"
|
|
"updated_at=? WHERE virtual_path=?",
|
|
(
|
|
destination_virtual,
|
|
collection["id"] if collection else None,
|
|
destination_parent_id,
|
|
utcnow(),
|
|
source_virtual,
|
|
),
|
|
)
|
|
return Response(status_code=201)
|
|
if staged.get("state") != "complete":
|
|
raise HTTPException(409, "文件尚未上传完整，不能提交入库")
|
|
content_hash = staged.get("content_sha256")
|
|
if content_hash:
|
|
duplicate_upload = app.uploads.matching_webdav_upload(
|
|
config["source_id"],
|
|
destination_virtual,
|
|
size_bytes=int(staged.get("received_bytes") or staged.get("size_bytes") or 0),
|
|
content_sha256=content_hash,
|
|
)
|
|
if duplicate_upload:
|
|
Path(staged["temp_path"]).unlink(missing_ok=True)
|
|
_delete_staging(app, source_virtual)
|
|
return Response(
|
|
status_code=204,
|
|
headers={
|
|
"X-ImageFind-Deduplicated": "true",
|
|
"X-ImageFind-Upload-Id": str(duplicate_upload["id"]),
|
|
},
|
|
)
|
|
duplicate_key = app.storage.matching_content_key(
|
|
config["source_id"],
|
|
_target_directory(config, destination[:-1]),
|
|
destination[-1],
|
|
size_bytes=int(staged.get("received_bytes") or staged.get("size_bytes") or 0),
|
|
content_sha256=content_hash,
|
|
)
|
|
if duplicate_key:
|
|
if destination_collection_id:
|
|
with app.db.transaction() as conn:
|
|
video = conn.execute(
|
|
"SELECT id FROM videos WHERE source_id=? AND source_key=? AND available=1",
|
|
(config["source_id"], duplicate_key),
|
|
).fetchone()
|
|
if video:
|
|
assign_videos(
|
|
conn,
|
|
destination_collection_id,
|
|
[video["id"]],
|
|
parent_id=destination_parent_id,
|
|
)
|
|
Path(staged["temp_path"]).unlink(missing_ok=True)
|
|
_delete_staging(app, source_virtual)
|
|
return Response(
|
|
status_code=204,
|
|
headers={"X-ImageFind-Deduplicated": "true"},
|
|
)
|
|
item = app.uploads.adopt_staged(
|
|
config["source_id"],
|
|
_target_directory(config, destination[:-1]),
|
|
destination[-1],
|
|
Path(staged["temp_path"]),
|
|
collection_id=destination_collection_id,
|
|
collection_parent_id=destination_parent_id,
|
|
content_sha256=content_hash,
|
|
webdav_path=destination_virtual,
|
|
content_sha256_verified=bool(staged.get("content_sha256_verified")),
|
|
)
|
|
_delete_staging(app, source_virtual)
|
|
return Response(status_code=201, headers={"X-ImageFind-Upload-Id": item["id"]})
|
|
|
|
pending = _staging(app, source_virtual, config)
|
|
if pending and pending.get("_resource_origin") == "upload":
|
|
raise HTTPException(
|
|
423,
|
|
"文件已接收并正在入库，暂时不能移动",
|
|
headers={
|
|
"Upload-Offset": str(int(pending.get("received_bytes") or 0)),
|
|
"Retry-After": "2",
|
|
},
|
|
)
|
|
destination_pending = _staging(app, "/".join(destination), config)
|
|
if destination_pending and destination_pending.get("_resource_origin") == "upload":
|
|
headers = {"X-ImageFind-Deduplicated": "true"}
|
|
if destination_pending.get("upload_id"):
|
|
headers["X-ImageFind-Upload-Id"] = str(destination_pending["upload_id"])
|
|
return Response(status_code=204, headers=headers)
|
|
if _is_temporary(parts[-1]) and _exists(app, config, destination):
|
|
return Response(status_code=204, headers={"X-ImageFind-Deduplicated": "true"})
|
|
|
|
if len(parts) < 2:
|
|
raise HTTPException(405, "已入库根目录视频请在 ImageFind 中管理")
|
|
if len(destination) < 2:
|
|
raise HTTPException(409, "已入库合集内容不能移动到根目录")
|
|
source_collection = _collection(app, parts[0])
|
|
if not source_collection:
|
|
raise HTTPException(404, "合集不存在")
|
|
collection = _collection(app, destination[0], create=True)
|
|
assert collection is not None
|
|
if collection["id"] != source_collection["id"]:
|
|
raise HTTPException(409, "已入库节点暂不支持跨合集移动")
|
|
destination_parent = _group(app, collection["id"], destination[1:-1])
|
|
if destination[1:-1] and not destination_parent:
|
|
raise HTTPException(409, "目标分组不存在")
|
|
destination_parent_id = (
|
|
str(destination_parent["id"]) if destination_parent and destination_parent.get("id") else None
|
|
)
|
|
source_group = _group(app, source_collection["id"], parts[1:])
|
|
with app.db.transaction() as conn:
|
|
if source_group:
|
|
rename_group(conn, source_collection["id"], str(source_group["id"]), destination[-1])
|
|
move_item(
|
|
conn,
|
|
source_collection["id"],
|
|
str(source_group["id"]),
|
|
destination_parent_id,
|
|
2**31 - 1,
|
|
)
|
|
else:
|
|
source_parent = _group(app, source_collection["id"], parts[1:-1])
|
|
source_parent_id = (
|
|
str(source_parent["id"]) if source_parent and source_parent.get("id") else None
|
|
)
|
|
video = _video(app, source_collection["id"], source_parent_id, parts[-1])
|
|
if not video:
|
|
raise HTTPException(404, "视频不存在")
|
|
if destination[-1] != parts[-1]:
|
|
raise HTTPException(409, "移动已入库视频时不能修改源文件名")
|
|
move_item(
|
|
conn,
|
|
source_collection["id"],
|
|
str(video["collection_item_id"]),
|
|
destination_parent_id,
|
|
2**31 - 1,
|
|
)
|
|
return Response(status_code=201)
|
|
|
|
|
|
def _staged_response(request: Request, staged: dict, filename: str) -> Response:
|
|
path = Path(staged["temp_path"])
|
|
if not path.is_file():
|
|
if staged.get("_resource_origin") != "upload":
|
|
raise HTTPException(404, "暂存文件不存在")
|
|
size = int(staged.get("received_bytes") or staged.get("size_bytes") or 0)
|
|
headers = {
|
|
"Accept-Ranges": "bytes",
|
|
"Content-Type": mimetypes.guess_type(filename)[0] or "application/octet-stream",
|
|
"Upload-Offset": str(size),
|
|
"Upload-Length": str(int(staged.get("expected_size") or size)),
|
|
"X-ImageFind-Upload-State": str(staged.get("state") or "accepted"),
|
|
"Retry-After": "2",
|
|
}
|
|
if request.method == "HEAD":
|
|
headers["Content-Length"] = str(size)
|
|
return Response(status_code=200, headers=headers)
|
|
raise HTTPException(503, "文件已接收，正在写入媒体库", headers=headers)
|
|
size = path.stat().st_size
|
|
start, end, status = 0, max(0, size - 1), 200
|
|
value = request.headers.get("range")
|
|
if value and value.startswith("bytes="):
|
|
try:
|
|
left, right = value[6:].split("-", 1)
|
|
if left:
|
|
start = int(left)
|
|
end = min(int(right) if right else size - 1, size - 1)
|
|
else:
|
|
suffix = min(int(right), size)
|
|
start = size - suffix
|
|
if start < 0 or start > end:
|
|
raise ValueError
|
|
status = 206
|
|
except ValueError as exc:
|
|
raise HTTPException(416, "无效的 Range") from exc
|
|
headers = {
|
|
"Accept-Ranges": "bytes",
|
|
"Content-Length": str(0 if size == 0 else end - start + 1),
|
|
"Content-Type": mimetypes.guess_type(filename)[0] or "application/octet-stream",
|
|
"Upload-Offset": str(int(staged.get("received_bytes") or size)),
|
|
"X-ImageFind-Upload-State": str(staged.get("state") or "complete"),
|
|
}
|
|
if staged.get("expected_size") is not None:
|
|
headers["Upload-Length"] = str(staged["expected_size"])
|
|
if status == 206:
|
|
headers["Content-Range"] = f"bytes {start}-{end}/{size}"
|
|
if request.method == "HEAD":
|
|
return Response(status_code=status, headers=headers)
|
|
|
|
async def body():
|
|
with path.open("rb") as handle:
|
|
handle.seek(start)
|
|
remaining = end - start + 1
|
|
while remaining:
|
|
chunk = handle.read(min(1024 * 1024, remaining))
|
|
if not chunk:
|
|
break
|
|
remaining -= len(chunk)
|
|
yield chunk
|
|
|
|
return StreamingResponse(body(), status_code=status, headers=headers)
|
|
|
|
|
|
def _resolve_read(request: Request, app: Services, parts: list[str]) -> Response | str:
|
|
if not parts:
|
|
raise HTTPException(405, "目录只能使用 PROPFIND 浏览")
|
|
staged = _staging(app, "/".join(parts))
|
|
if staged:
|
|
return _staged_response(request, staged, parts[-1])
|
|
config = _config(app)
|
|
collection = _collection(app, parts[0])
|
|
if not collection:
|
|
if len(parts) != 1:
|
|
raise HTTPException(404, "合集不存在")
|
|
video = _root_video(app, config, parts[0])
|
|
if not video:
|
|
raise HTTPException(404, "视频不存在")
|
|
return str(video["id"])
|
|
parent = _group(app, collection["id"], parts[1:-1])
|
|
if parts[1:-1] and not parent:
|
|
raise HTTPException(404, "合集分组不存在")
|
|
parent_id = str(parent["id"]) if parent and parent.get("id") else None
|
|
video = _video(app, collection["id"], parent_id, parts[-1])
|
|
if not video:
|
|
raise HTTPException(404, "视频不存在")
|
|
return str(video["id"])
|
|
|
|
|
|
async def _read(request: Request, app: Services, parts: list[str]) -> Response:
|
|
resolved = await _background_io(app, _resolve_read, request, app, parts)
|
|
if isinstance(resolved, Response):
|
|
return resolved
|
|
return await stream_video(resolved, request, app, {"kind": "api"}, mode="proxy")
|
|
|
|
|
|
def _mkcol(app: Services, parts: list[str]) -> Response:
|
|
if not parts:
|
|
raise HTTPException(405, "WebDAV 根目录已存在")
|
|
if len(parts) == 1:
|
|
existed = _collection(app, parts[0]) is not None
|
|
_collection(app, parts[0], create=True)
|
|
return Response(status_code=405 if existed else 201)
|
|
collection = _collection(app, parts[0])
|
|
if not collection:
|
|
raise HTTPException(409, "请先创建合集目录")
|
|
if _group(app, collection["id"], parts[1:]):
|
|
return Response(status_code=405)
|
|
parent = _group(app, collection["id"], parts[1:-1])
|
|
if parts[1:-1] and not parent:
|
|
raise HTTPException(409, "请先创建上级分组")
|
|
parent_id = str(parent["id"]) if parent and parent.get("id") else None
|
|
try:
|
|
with app.db.transaction() as conn:
|
|
create_group(conn, collection["id"], parts[-1], parent_id)
|
|
except ValueError as exc:
|
|
raise HTTPException(409, str(exc)) from exc
|
|
return Response(status_code=201)
|
|
|
|
|
|
def _delete(app: Services, config: dict, parts: list[str]) -> Response:
|
|
virtual = "/".join(parts)
|
|
staged = _staging(app, virtual, config)
|
|
if staged:
|
|
if staged.get("_resource_origin") == "upload":
|
|
raise HTTPException(
|
|
423,
|
|
"文件已接收并正在入库，请在上传中心管理该任务",
|
|
headers={"Retry-After": "2"},
|
|
)
|
|
Path(staged["temp_path"]).unlink(missing_ok=True)
|
|
_delete_staging(app, virtual)
|
|
return Response(status_code=204)
|
|
if len(parts) < 2:
|
|
raise HTTPException(405, "不能通过 WebDAV 删除根目录或合集")
|
|
collection = _collection(app, parts[0])
|
|
group = _group(app, collection["id"], parts[1:]) if collection else None
|
|
if not collection or not group:
|
|
raise HTTPException(405, "已入库视频必须在 ImageFind 中删除")
|
|
with app.db.transaction() as conn:
|
|
delete_group(conn, collection["id"], str(group["id"]))
|
|
return Response(status_code=204)
|
|
|
|
|
|
@router.api_route(
|
|
"/webdav",
|
|
methods=["OPTIONS", "PROPFIND", "MKCOL", "PUT", "MOVE", "GET", "HEAD", "DELETE"],
|
|
include_in_schema=False,
|
|
)
|
|
@router.api_route(
|
|
"/webdav/{dav_path:path}",
|
|
methods=["OPTIONS", "PROPFIND", "MKCOL", "PUT", "MOVE", "GET", "HEAD", "DELETE"],
|
|
include_in_schema=False,
|
|
)
|
|
async def webdav(request: Request, dav_path: str = ""):
|
|
app: Services = request.app.state.services
|
|
if request.method == "OPTIONS":
|
|
return Response(
|
|
status_code=200,
|
|
headers={
|
|
"DAV": "1",
|
|
"Allow": "OPTIONS, PROPFIND, MKCOL, PUT, MOVE, GET, HEAD, DELETE",
|
|
"MS-Author-Via": "DAV",
|
|
"X-ImageFind-Resumable": "content-range,upload-offset",
|
|
},
|
|
)
|
|
config = await _background_io(app, _config, app)
|
|
if not config["enabled"] or not config["source_id"]:
|
|
raise HTTPException(404, "ImageFind WebDAV 未启用")
|
|
await _authenticate(request, app)
|
|
parts = _parts(dav_path)
|
|
if request.method == "PROPFIND":
|
|
return await _background_io(app, _propfind, request, app, config, parts)
|
|
if request.method == "MKCOL":
|
|
return await _background_io(app, _mkcol, app, parts)
|
|
if request.method == "PUT":
|
|
return await _put(request, app, config, parts)
|
|
if request.method == "MOVE":
|
|
return await _background_io(app, _move, request, app, config, parts)
|
|
if request.method in {"GET", "HEAD"}:
|
|
return await _read(request, app, parts)
|
|
if request.method == "DELETE":
|
|
return await _background_io(app, _delete, app, config, parts)
|
|
raise HTTPException(405)
|