594 lines
21 KiB
Python
594 lines
21 KiB
Python
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import posixpath
|
|
import re
|
|
import secrets
|
|
import uuid
|
|
import xml.etree.ElementTree as ET
|
|
from collections import deque
|
|
from collections.abc import Callable, Iterator
|
|
from dataclasses import dataclass
|
|
from datetime import UTC, datetime
|
|
from email.utils import parsedate_to_datetime
|
|
from pathlib import Path, PurePosixPath
|
|
from urllib.parse import quote, unquote, urljoin, urlparse
|
|
|
|
import httpx
|
|
|
|
from .config import Settings
|
|
from .database import Database, utcnow
|
|
from .remote import AlistClient, RcloneManager, alist_webdav_url, quoted_path, safe_relative_path
|
|
from .runtime import RuntimeToolManager
|
|
from .security import SecretStore
|
|
|
|
VIDEO_EXTENSIONS = {
|
|
".3gp",
|
|
".asf",
|
|
".avi",
|
|
".flv",
|
|
".m2ts",
|
|
".m4v",
|
|
".mkv",
|
|
".mov",
|
|
".mp4",
|
|
".mpeg",
|
|
".mpg",
|
|
".mts",
|
|
".ogv",
|
|
".ts",
|
|
".vob",
|
|
".webm",
|
|
".wmv",
|
|
}
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class SourceItem:
|
|
key: str
|
|
display_name: str
|
|
location: str
|
|
size_bytes: int
|
|
modified_at: str | None
|
|
etag: str | None
|
|
fingerprint: str
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class RemoteAccess:
|
|
url: str
|
|
username: str
|
|
password: str
|
|
verify_tls: bool
|
|
_release: Callable[[], None] | None = None
|
|
_closed: bool = False
|
|
|
|
def close(self) -> None:
|
|
if self._closed:
|
|
return
|
|
self._closed = True
|
|
if self._release is not None:
|
|
self._release()
|
|
|
|
def as_tuple(self) -> tuple[str, str, str, bool]:
|
|
return self.url, self.username, self.password, self.verify_tls
|
|
|
|
def __enter__(self) -> RemoteAccess:
|
|
return self
|
|
|
|
def __exit__(self, *_args) -> None:
|
|
self.close()
|
|
|
|
|
|
def _fingerprint(*parts: object) -> str:
|
|
value = "\0".join("" if part is None else str(part) for part in parts)
|
|
return hashlib.sha256(value.encode()).hexdigest()
|
|
|
|
|
|
class LocalConnector:
|
|
def __init__(self, root: str):
|
|
self.root = Path(root).expanduser().resolve(strict=True)
|
|
if not self.root.is_dir():
|
|
raise ValueError("本地数据源必须是目录")
|
|
|
|
def items(self) -> Iterator[SourceItem]:
|
|
for current, directories, filenames in os.walk(self.root, followlinks=False):
|
|
directories[:] = [
|
|
name
|
|
for name in directories
|
|
if name != ".imagefind-trash" and not (Path(current) / name).is_symlink()
|
|
]
|
|
for filename in filenames:
|
|
path = Path(current) / filename
|
|
if path.suffix.lower() not in VIDEO_EXTENSIONS or path.is_symlink():
|
|
continue
|
|
try:
|
|
stat = path.stat()
|
|
resolved = path.resolve(strict=True)
|
|
resolved.relative_to(self.root)
|
|
except (OSError, ValueError):
|
|
continue
|
|
key = resolved.relative_to(self.root).as_posix()
|
|
modified = datetime.fromtimestamp(stat.st_mtime, UTC).isoformat()
|
|
yield SourceItem(
|
|
key=key,
|
|
display_name=path.name,
|
|
location=str(resolved),
|
|
size_bytes=stat.st_size,
|
|
modified_at=modified,
|
|
etag=None,
|
|
fingerprint=_fingerprint(stat.st_dev, stat.st_ino, stat.st_size, stat.st_mtime_ns),
|
|
)
|
|
|
|
def item(self, key: str) -> SourceItem:
|
|
relative = safe_relative_path(key, allow_empty=False)
|
|
path = (self.root / relative).resolve(strict=True)
|
|
path.relative_to(self.root)
|
|
if not path.is_file() or path.is_symlink() or path.suffix.lower() not in VIDEO_EXTENSIONS:
|
|
raise KeyError(relative)
|
|
stat = path.stat()
|
|
modified = datetime.fromtimestamp(stat.st_mtime, UTC).isoformat()
|
|
return SourceItem(
|
|
key=relative,
|
|
display_name=path.name,
|
|
location=str(path),
|
|
size_bytes=stat.st_size,
|
|
modified_at=modified,
|
|
etag=None,
|
|
fingerprint=_fingerprint(stat.st_dev, stat.st_ino, stat.st_size, stat.st_mtime_ns),
|
|
)
|
|
|
|
|
|
class WebDavConnector:
|
|
def __init__(
|
|
self,
|
|
base_url: str,
|
|
username: str,
|
|
password: str,
|
|
*,
|
|
verify_tls: bool = True,
|
|
timeout: float = 30,
|
|
on_close: Callable[[], None] | None = None,
|
|
):
|
|
self.base_url = base_url.rstrip("/") + "/"
|
|
parsed = urlparse(self.base_url)
|
|
if parsed.scheme not in {"http", "https"} or not parsed.netloc:
|
|
raise ValueError("WebDAV 地址必须是 http 或 https URL")
|
|
self._base = parsed
|
|
self.client = httpx.Client(
|
|
auth=httpx.BasicAuth(username, password),
|
|
verify=verify_tls,
|
|
timeout=httpx.Timeout(timeout),
|
|
follow_redirects=True,
|
|
trust_env=False,
|
|
)
|
|
self._on_close = on_close
|
|
self._closed = False
|
|
|
|
def close(self) -> None:
|
|
if self._closed:
|
|
return
|
|
self._closed = True
|
|
self.client.close()
|
|
if self._on_close is not None:
|
|
self._on_close()
|
|
|
|
def _safe_url(self, href: str) -> str | None:
|
|
url = urljoin(self.base_url, href)
|
|
parsed = urlparse(url)
|
|
base_path = posixpath.normpath(unquote(self._base.path))
|
|
path = posixpath.normpath(unquote(parsed.path))
|
|
if parsed.scheme != self._base.scheme or parsed.netloc != self._base.netloc:
|
|
return None
|
|
if path != base_path.rstrip("/") and not path.startswith(base_path.rstrip("/") + "/"):
|
|
return None
|
|
return url
|
|
|
|
@staticmethod
|
|
def _parse_multistatus(content: bytes) -> ET.Element:
|
|
try:
|
|
return ET.fromstring(content)
|
|
except ET.ParseError as original:
|
|
text = content.decode("utf-8", errors="replace")
|
|
match = re.search(r"<(?P<prefix>[A-Za-z_][\w.-]*:)?multistatus\b", text)
|
|
if not match:
|
|
raise RuntimeError(f"WebDAV PROPFIND 返回 XML 无法解析：{original}") from original
|
|
prefix = match.group("prefix") or ""
|
|
closing = f"</{prefix}multistatus>"
|
|
end = text.find(closing, match.end())
|
|
if end < 0:
|
|
raise RuntimeError(f"WebDAV PROPFIND 返回 XML 无法解析：{original}") from original
|
|
fragment = text[match.start() : end + len(closing)]
|
|
try:
|
|
return ET.fromstring(fragment.encode("utf-8"))
|
|
except ET.ParseError as exc:
|
|
raise RuntimeError(f"WebDAV PROPFIND 返回 XML 无法解析：{exc}") from exc
|
|
|
|
def _list(self, url: str, depth: str = "1") -> list[tuple[str, bool, int, str | None, str | None]]:
|
|
response = self.client.request(
|
|
"PROPFIND",
|
|
url,
|
|
headers={"Depth": depth, "Content-Type": "application/xml; charset=utf-8"},
|
|
content=(
|
|
"<?xml version='1.0' encoding='utf-8'?>"
|
|
"<d:propfind xmlns:d='DAV:'><d:prop>"
|
|
"<d:resourcetype/><d:getcontentlength/><d:getlastmodified/><d:getetag/>"
|
|
"</d:prop></d:propfind>"
|
|
),
|
|
)
|
|
if response.status_code != 207:
|
|
raise RuntimeError(f"WebDAV PROPFIND 失败：HTTP {response.status_code}")
|
|
root = self._parse_multistatus(response.content)
|
|
rows: list[tuple[str, bool, int, str | None, str | None]] = []
|
|
for item in root.findall("{DAV:}response"):
|
|
href_node = item.find("{DAV:}href")
|
|
if href_node is None or not href_node.text:
|
|
continue
|
|
safe_url = self._safe_url(href_node.text)
|
|
if not safe_url:
|
|
continue
|
|
props = None
|
|
for propstat in item.findall("{DAV:}propstat"):
|
|
status = propstat.findtext("{DAV:}status", "")
|
|
if " 200 " in status:
|
|
props = propstat.find("{DAV:}prop")
|
|
break
|
|
if props is None:
|
|
continue
|
|
resource_type = props.find("{DAV:}resourcetype")
|
|
is_dir = resource_type is not None and resource_type.find("{DAV:}collection") is not None
|
|
try:
|
|
size = int(props.findtext("{DAV:}getcontentlength", "0"))
|
|
except ValueError:
|
|
size = 0
|
|
modified = props.findtext("{DAV:}getlastmodified")
|
|
if modified:
|
|
try:
|
|
modified = parsedate_to_datetime(modified).astimezone(UTC).isoformat()
|
|
except (TypeError, ValueError):
|
|
pass
|
|
etag = props.findtext("{DAV:}getetag")
|
|
rows.append((safe_url, is_dir, size, modified, etag))
|
|
return rows
|
|
|
|
def items(self) -> Iterator[SourceItem]:
|
|
queue = deque([self.base_url])
|
|
visited: set[str] = set()
|
|
base_path = unquote(self._base.path).rstrip("/") + "/"
|
|
while queue:
|
|
directory = queue.popleft()
|
|
normalized_directory = directory.rstrip("/") + "/"
|
|
if normalized_directory in visited:
|
|
continue
|
|
visited.add(normalized_directory)
|
|
for url, is_dir, size, modified, etag in self._list(directory):
|
|
normalized = url.rstrip("/") + "/" if is_dir else url
|
|
if normalized == normalized_directory:
|
|
continue
|
|
parsed = urlparse(url)
|
|
relative = unquote(parsed.path)
|
|
if relative.startswith(base_path):
|
|
relative = relative[len(base_path) :]
|
|
relative = relative.strip("/")
|
|
if not relative:
|
|
continue
|
|
if relative == ".imagefind-trash" or relative.startswith(".imagefind-trash/"):
|
|
continue
|
|
if is_dir:
|
|
queue.append(normalized)
|
|
continue
|
|
if Path(relative).suffix.lower() not in VIDEO_EXTENSIONS:
|
|
continue
|
|
yield SourceItem(
|
|
key=relative,
|
|
display_name=posixpath.basename(relative),
|
|
location=url,
|
|
size_bytes=size,
|
|
modified_at=modified,
|
|
etag=etag,
|
|
fingerprint=_fingerprint(etag, size, modified),
|
|
)
|
|
|
|
def url_for(self, key: str) -> str:
|
|
relative = safe_relative_path(key, allow_empty=False)
|
|
encoded = "/".join(quote(part, safe="") for part in PurePosixPath(relative).parts)
|
|
return urljoin(self.base_url, encoded)
|
|
|
|
def item(self, key: str) -> SourceItem:
|
|
relative = safe_relative_path(key, allow_empty=False)
|
|
url = self.url_for(relative)
|
|
rows = self._list(url, depth="0")
|
|
row = next((value for value in rows if not value[1]), None)
|
|
if not row:
|
|
raise KeyError(relative)
|
|
_, _, size, modified, etag = row
|
|
return SourceItem(
|
|
key=relative,
|
|
display_name=posixpath.basename(relative),
|
|
location=url,
|
|
size_bytes=size,
|
|
modified_at=modified,
|
|
etag=etag,
|
|
fingerprint=_fingerprint(etag, size, modified),
|
|
)
|
|
|
|
|
|
class SourceService:
|
|
def __init__(
|
|
self,
|
|
db: Database,
|
|
settings: Settings,
|
|
secrets: SecretStore,
|
|
tools: RuntimeToolManager | None = None,
|
|
):
|
|
self.db = db
|
|
self.settings = settings
|
|
self.secrets = secrets
|
|
self.rclone = RcloneManager(settings, tools)
|
|
|
|
def add_local(self, name: str, path: str) -> str:
|
|
connector = LocalConnector(path)
|
|
source_id = str(uuid.uuid4())
|
|
now = utcnow()
|
|
config = {"path": str(connector.root)}
|
|
with self.db.transaction() as conn:
|
|
conn.execute(
|
|
"INSERT INTO sources(id,kind,name,config_json,created_at,updated_at) VALUES(?,?,?,?,?,?)",
|
|
(source_id, "local", name.strip() or connector.root.name, json.dumps(config), now, now),
|
|
)
|
|
return source_id
|
|
|
|
def add_webdav(
|
|
self,
|
|
name: str,
|
|
base_url: str,
|
|
username: str,
|
|
password: str,
|
|
verify_tls: bool = True,
|
|
) -> str:
|
|
probe = WebDavConnector(
|
|
base_url,
|
|
username,
|
|
password,
|
|
verify_tls=verify_tls,
|
|
timeout=self.settings.remote_timeout_seconds,
|
|
)
|
|
try:
|
|
probe._list(probe.base_url)
|
|
finally:
|
|
probe.close()
|
|
source_id = str(uuid.uuid4())
|
|
now = utcnow()
|
|
config = {"base_url": probe.base_url, "username": username, "verify_tls": verify_tls}
|
|
secret_blob = self.secrets.encrypt_json({"password": password})
|
|
with self.db.transaction() as conn:
|
|
conn.execute(
|
|
"INSERT INTO sources(id,kind,name,config_json,secret_blob,created_at,updated_at) VALUES(?,?,?,?,?,?,?)",
|
|
(source_id, "webdav", name.strip() or "WebDAV", json.dumps(config), secret_blob, now, now),
|
|
)
|
|
return source_id
|
|
|
|
def add_alist(
|
|
self,
|
|
name: str,
|
|
base_url: str,
|
|
root_path: str,
|
|
username: str,
|
|
password: str,
|
|
*,
|
|
mode: str,
|
|
verify_tls: bool = True,
|
|
) -> tuple[str, dict | None]:
|
|
return self._create_alist(
|
|
name,
|
|
base_url,
|
|
root_path,
|
|
username,
|
|
password,
|
|
mode=mode,
|
|
verify_tls=verify_tls,
|
|
)
|
|
|
|
def _create_alist(
|
|
self,
|
|
name: str,
|
|
base_url: str,
|
|
root_path: str,
|
|
username: str,
|
|
password: str,
|
|
*,
|
|
mode: str,
|
|
verify_tls: bool,
|
|
crypt_password: str | None = None,
|
|
crypt_salt: str | None = None,
|
|
) -> tuple[str, dict | None]:
|
|
if mode not in {"direct", "encrypted"}:
|
|
raise ValueError("AList 模式必须是 direct 或 encrypted")
|
|
root_path = safe_relative_path(root_path)
|
|
control = AlistClient(
|
|
base_url,
|
|
username,
|
|
password,
|
|
root_path=root_path,
|
|
verify_tls=verify_tls,
|
|
timeout=self.settings.remote_timeout_seconds,
|
|
)
|
|
try:
|
|
control.probe()
|
|
finally:
|
|
control.close()
|
|
source_id = str(uuid.uuid4())
|
|
now = utcnow()
|
|
config = {
|
|
"driver": "alist",
|
|
"mode": mode,
|
|
"base_url": base_url.rstrip("/"),
|
|
"root_path": root_path,
|
|
"username": username,
|
|
"verify_tls": verify_tls,
|
|
"writable": True,
|
|
}
|
|
recovery = None
|
|
secret = {"password": password}
|
|
if mode == "encrypted":
|
|
secret["crypt_password"] = crypt_password or secrets.token_urlsafe(32)
|
|
secret["crypt_salt"] = crypt_salt or secrets.token_urlsafe(24)
|
|
recovery = {
|
|
"format": "imagefind-rclone-crypt-v1",
|
|
"base_url": config["base_url"],
|
|
"root_path": root_path,
|
|
"username": username,
|
|
"crypt_password": secret["crypt_password"],
|
|
"crypt_salt": secret["crypt_salt"],
|
|
}
|
|
secret_blob = self.secrets.encrypt_json(secret)
|
|
with self.db.transaction() as conn:
|
|
conn.execute(
|
|
"INSERT INTO sources(id,kind,name,config_json,secret_blob,created_at,updated_at) VALUES(?,?,?,?,?,?,?)",
|
|
(source_id, "webdav", name.strip() or "AList", json.dumps(config), secret_blob, now, now),
|
|
)
|
|
try:
|
|
connector = self.connector(source_id)
|
|
try:
|
|
connector._list(connector.base_url)
|
|
finally:
|
|
connector.close()
|
|
except Exception:
|
|
self.rclone.stop(source_id)
|
|
with self.db.transaction() as conn:
|
|
conn.execute("DELETE FROM sources WHERE id=?", (source_id,))
|
|
raise
|
|
return source_id, recovery
|
|
|
|
def restore_alist(
|
|
self,
|
|
name: str,
|
|
base_url: str,
|
|
root_path: str,
|
|
username: str,
|
|
password: str,
|
|
crypt_password: str,
|
|
crypt_salt: str,
|
|
*,
|
|
verify_tls: bool = True,
|
|
) -> str:
|
|
if not crypt_password or not crypt_salt:
|
|
raise ValueError("恢复文件缺少加密口令")
|
|
source_id, _ = self._create_alist(
|
|
name,
|
|
base_url,
|
|
root_path,
|
|
username,
|
|
password,
|
|
mode="encrypted",
|
|
verify_tls=verify_tls,
|
|
crypt_password=crypt_password,
|
|
crypt_salt=crypt_salt,
|
|
)
|
|
return source_id
|
|
|
|
def list_sources(self) -> list[dict]:
|
|
with self.db.read() as conn:
|
|
rows = conn.execute(
|
|
"SELECT id,kind,name,config_json,secret_blob,enabled,status,last_scan_at,last_error,created_at "
|
|
"FROM sources ORDER BY created_at"
|
|
).fetchall()
|
|
result = []
|
|
for row in rows:
|
|
item = dict(row)
|
|
config = json.loads(item.pop("config_json"))
|
|
item["has_password"] = bool(item.pop("secret_blob", None))
|
|
if item["kind"] == "webdav":
|
|
config.pop("username", None)
|
|
item["config"] = config
|
|
item["enabled"] = bool(item["enabled"])
|
|
result.append(item)
|
|
return result
|
|
|
|
def get(self, source_id: str) -> dict:
|
|
with self.db.read() as conn:
|
|
row = conn.execute("SELECT * FROM sources WHERE id=?", (source_id,)).fetchone()
|
|
if not row:
|
|
raise KeyError(source_id)
|
|
item = dict(row)
|
|
item["config"] = json.loads(item.pop("config_json"))
|
|
item["secrets"] = self.secrets.decrypt_json(item.pop("secret_blob"))
|
|
return item
|
|
|
|
def connector(self, source_id: str) -> LocalConnector | WebDavConnector:
|
|
source = self.get(source_id)
|
|
if source["kind"] == "local":
|
|
return LocalConnector(source["config"]["path"])
|
|
if source["config"].get("driver") == "alist":
|
|
if source["config"].get("mode") == "encrypted":
|
|
endpoint = self.rclone.acquire(source)
|
|
return WebDavConnector(
|
|
endpoint.url,
|
|
endpoint.username,
|
|
endpoint.password,
|
|
verify_tls=True,
|
|
timeout=self.settings.remote_timeout_seconds,
|
|
on_close=endpoint.close,
|
|
)
|
|
return WebDavConnector(
|
|
alist_webdav_url(source["config"]["base_url"], source["config"].get("root_path", "")),
|
|
source["config"].get("username", ""),
|
|
source["secrets"].get("password", ""),
|
|
verify_tls=source["config"].get("verify_tls", True),
|
|
timeout=self.settings.remote_timeout_seconds,
|
|
)
|
|
return WebDavConnector(
|
|
source["config"]["base_url"],
|
|
source["config"]["username"],
|
|
source["secrets"]["password"],
|
|
verify_tls=source["config"].get("verify_tls", True),
|
|
timeout=self.settings.remote_timeout_seconds,
|
|
)
|
|
|
|
def open_remote_access(self, source_id: str, key: str) -> RemoteAccess:
|
|
source = self.get(source_id)
|
|
if source["kind"] != "webdav":
|
|
raise ValueError("数据源不是远程媒体库")
|
|
config = source["config"]
|
|
secret = source["secrets"]
|
|
if config.get("driver") == "alist":
|
|
if config.get("mode") == "encrypted":
|
|
endpoint = self.rclone.acquire(source)
|
|
return RemoteAccess(
|
|
urljoin(endpoint.url, quoted_path(key)),
|
|
endpoint.username,
|
|
endpoint.password,
|
|
True,
|
|
endpoint.close,
|
|
)
|
|
base = alist_webdav_url(config["base_url"], config.get("root_path", ""))
|
|
return RemoteAccess(
|
|
urljoin(base, quoted_path(key)),
|
|
config.get("username", ""),
|
|
secret.get("password", ""),
|
|
config.get("verify_tls", True),
|
|
)
|
|
return RemoteAccess(
|
|
urljoin(config["base_url"], quoted_path(key)),
|
|
config.get("username", ""),
|
|
secret.get("password", ""),
|
|
config.get("verify_tls", True),
|
|
)
|
|
|
|
def remote_access(self, source_id: str, key: str) -> tuple[str, str, str, bool]:
|
|
access = self.open_remote_access(source_id, key)
|
|
try:
|
|
return access.as_tuple()
|
|
finally:
|
|
access.close()
|
|
|
|
def delete(self, source_id: str) -> None:
|
|
self.rclone.stop(source_id)
|
|
with self.db.transaction() as conn:
|
|
conn.execute("DELETE FROM sources WHERE id=?", (source_id,))
|
|
|
|
def close(self) -> None:
|
|
self.rclone.stop_all()
|