1453 lines
58 KiB
Python
1453 lines
58 KiB
Python
from __future__ import annotations
|
|
|
|
import configparser
|
|
import contextlib
|
|
import json
|
|
import os
|
|
import re
|
|
import sqlite3
|
|
import threading
|
|
import time
|
|
import uuid
|
|
from collections.abc import Iterable, Iterator, Sequence
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
SCHEMA_VERSION = 24
|
|
DEFAULT_TAG_GROUP_ID = "00000000-0000-0000-0000-000000000001"
|
|
|
|
|
|
def utcnow() -> str:
|
|
return datetime.now(UTC).isoformat()
|
|
|
|
|
|
class Row:
|
|
def __init__(self, columns: Sequence[str], values: Sequence[Any]):
|
|
self._columns = list(columns)
|
|
self._values = tuple(values)
|
|
self._mapping = {column: values[index] for index, column in enumerate(columns)}
|
|
|
|
def __getitem__(self, key: int | str) -> Any:
|
|
if isinstance(key, int):
|
|
return self._values[key]
|
|
return self._mapping[key]
|
|
|
|
def __iter__(self):
|
|
return iter(self._values)
|
|
|
|
def __len__(self) -> int:
|
|
return len(self._values)
|
|
|
|
def __eq__(self, other: object) -> bool:
|
|
if isinstance(other, tuple):
|
|
return self._values == other
|
|
return super().__eq__(other)
|
|
|
|
def keys(self):
|
|
return self._mapping.keys()
|
|
|
|
def items(self):
|
|
return self._mapping.items()
|
|
|
|
def get(self, key: str, default: Any = None) -> Any:
|
|
return self._mapping.get(key, default)
|
|
|
|
|
|
class _WriteCoordinator:
|
|
slow_transaction_seconds = 0.25
|
|
|
|
def __init__(self):
|
|
self._metrics = threading.Lock()
|
|
self._waiters = 0
|
|
self._active = 0
|
|
self._last_wait_ms = 0.0
|
|
self._max_wait_ms = 0.0
|
|
self._lock_retry_count = 0
|
|
self._slow_transaction_count = 0
|
|
|
|
def acquire(self) -> float:
|
|
acquired_at = time.monotonic()
|
|
with self._metrics:
|
|
self._active += 1
|
|
return acquired_at
|
|
|
|
def release(self, acquired_at: float) -> None:
|
|
duration = max(0.0, time.monotonic() - acquired_at)
|
|
with self._metrics:
|
|
self._active = max(0, self._active - 1)
|
|
if duration >= self.slow_transaction_seconds:
|
|
self._slow_transaction_count += 1
|
|
|
|
def record_lock_retry(self) -> None:
|
|
with self._metrics:
|
|
self._lock_retry_count += 1
|
|
|
|
def status(self) -> dict[str, int | float | bool]:
|
|
with self._metrics:
|
|
return {
|
|
"writer_queue_depth": self._waiters,
|
|
"writer_active": self._active > 0,
|
|
"last_wait_ms": round(self._last_wait_ms, 2),
|
|
"max_wait_ms": round(self._max_wait_ms, 2),
|
|
"lock_retry_count": self._lock_retry_count,
|
|
"slow_transaction_count": self._slow_transaction_count,
|
|
}
|
|
|
|
|
|
class _ConnectionPool:
|
|
"""Small bounded psycopg pool tailored for the single-process fnOS service.
|
|
|
|
Keeping this local avoids pulling another runtime package into the offline
|
|
fnOS wheelhouse. Connections are always autocommit while idle, so a pool
|
|
entry can never retain an MVCC snapshot between requests. Explicit write
|
|
transactions are opened only by :meth:`Database.transaction`.
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
factory,
|
|
*,
|
|
min_size: int = 1,
|
|
max_size: int = 8,
|
|
timeout: float = 5.0,
|
|
idle_timeout: float = 60.0,
|
|
):
|
|
self.factory = factory
|
|
self.min_size = max(0, min(min_size, max_size))
|
|
self.max_size = max(1, max_size)
|
|
self.timeout = max(0.1, timeout)
|
|
self.idle_timeout = max(1.0, idle_timeout)
|
|
self._idle: list[tuple[Any, float]] = []
|
|
self._guard = threading.Condition()
|
|
self._total = 0
|
|
self._in_use = 0
|
|
self._waiters = 0
|
|
self._wait_count = 0
|
|
self._last_wait_ms = 0.0
|
|
self._max_wait_ms = 0.0
|
|
self._closed = False
|
|
|
|
@staticmethod
|
|
def _usable(connection) -> bool:
|
|
return not bool(getattr(connection, "closed", True))
|
|
|
|
def open(self) -> None:
|
|
for _ in range(self.min_size):
|
|
with self._guard:
|
|
if self._closed or self._total >= self.max_size:
|
|
return
|
|
self._total += 1
|
|
try:
|
|
connection = self.factory()
|
|
except Exception:
|
|
with self._guard:
|
|
self._total = max(0, self._total - 1)
|
|
self._guard.notify()
|
|
raise
|
|
with self._guard:
|
|
self._idle.append((connection, time.monotonic()))
|
|
self._guard.notify()
|
|
|
|
def acquire(self, timeout: float | None = None):
|
|
timeout = self.timeout if timeout is None else max(0.1, timeout)
|
|
started = time.monotonic()
|
|
deadline = started + timeout
|
|
while True:
|
|
connection = None
|
|
check_health = False
|
|
create_connection = False
|
|
with self._guard:
|
|
if self._closed:
|
|
raise RuntimeError("PostgreSQL 连接池已经关闭")
|
|
while self._idle:
|
|
candidate, released_at = self._idle.pop()
|
|
expired = (
|
|
self._total > self.min_size
|
|
and time.monotonic() - released_at >= self.idle_timeout
|
|
)
|
|
if expired or not self._usable(candidate):
|
|
try:
|
|
candidate.close()
|
|
finally:
|
|
self._total = max(0, self._total - 1)
|
|
continue
|
|
connection = candidate
|
|
check_health = time.monotonic() - released_at >= 5
|
|
break
|
|
if connection is not None:
|
|
self._in_use += 1
|
|
waited = max(0.0, (time.monotonic() - started) * 1000)
|
|
self._last_wait_ms = waited
|
|
self._max_wait_ms = max(self._max_wait_ms, waited)
|
|
elif self._total < self.max_size:
|
|
self._total += 1
|
|
create_connection = True
|
|
else:
|
|
remaining = deadline - time.monotonic()
|
|
if remaining <= 0:
|
|
raise TimeoutError("等待 PostgreSQL 连接超时")
|
|
self._waiters += 1
|
|
self._wait_count += 1
|
|
try:
|
|
self._guard.wait(timeout=remaining)
|
|
finally:
|
|
self._waiters = max(0, self._waiters - 1)
|
|
continue
|
|
if connection is not None:
|
|
if check_health:
|
|
try:
|
|
connection.execute("SELECT 1")
|
|
except Exception:
|
|
self.release(connection, discard=True)
|
|
continue
|
|
return connection
|
|
if create_connection:
|
|
break
|
|
try:
|
|
connection = self.factory()
|
|
except Exception:
|
|
with self._guard:
|
|
self._total = max(0, self._total - 1)
|
|
self._guard.notify()
|
|
raise
|
|
with self._guard:
|
|
self._in_use += 1
|
|
waited = max(0.0, (time.monotonic() - started) * 1000)
|
|
self._last_wait_ms = waited
|
|
self._max_wait_ms = max(self._max_wait_ms, waited)
|
|
return connection
|
|
|
|
def release(self, connection, *, discard: bool = False) -> None:
|
|
with self._guard:
|
|
self._in_use = max(0, self._in_use - 1)
|
|
closed = self._closed
|
|
if discard or closed or not self._usable(connection):
|
|
try:
|
|
connection.close()
|
|
finally:
|
|
with self._guard:
|
|
self._total = max(0, self._total - 1)
|
|
self._guard.notify()
|
|
return
|
|
with self._guard:
|
|
self._idle.append((connection, time.monotonic()))
|
|
self._guard.notify()
|
|
|
|
def close(self) -> None:
|
|
with self._guard:
|
|
self._closed = True
|
|
self._guard.notify_all()
|
|
while True:
|
|
with self._guard:
|
|
if not self._idle:
|
|
break
|
|
connection, _ = self._idle.pop()
|
|
try:
|
|
connection.close()
|
|
finally:
|
|
with self._guard:
|
|
self._total = max(0, self._total - 1)
|
|
|
|
def status(self) -> dict[str, int | float]:
|
|
with self._guard:
|
|
return {
|
|
"pool_min": self.min_size,
|
|
"pool_max": self.max_size,
|
|
"pool_size": self._total,
|
|
"pool_in_use": self._in_use,
|
|
"pool_idle": len(self._idle),
|
|
"pool_waiters": self._waiters,
|
|
"pool_wait_count": self._wait_count,
|
|
"pool_last_wait_ms": round(self._last_wait_ms, 2),
|
|
"pool_max_wait_ms": round(self._max_wait_ms, 2),
|
|
}
|
|
|
|
|
|
def _load_psycopg():
|
|
try:
|
|
import psycopg
|
|
except ImportError as exc:
|
|
raise RuntimeError("ImageFind 需要 psycopg PostgreSQL 客户端运行时") from exc
|
|
return psycopg
|
|
|
|
|
|
def _postgres_conf_from_path(path: Path) -> dict[str, str]:
|
|
parser = configparser.ConfigParser()
|
|
content = path.read_text(encoding="utf-8")
|
|
parser.read_string("[postgres]\n" + content)
|
|
return {key: value for key, value in parser.items("postgres")}
|
|
|
|
|
|
def _replace_placeholders(sql: str, *, escape_percent: bool = False) -> str:
|
|
pieces: list[str] = []
|
|
in_single = False
|
|
in_double = False
|
|
index = 0
|
|
while index < len(sql):
|
|
char = sql[index]
|
|
if char == "'" and not in_double:
|
|
in_single = not in_single
|
|
pieces.append(char)
|
|
elif char == '"' and not in_single:
|
|
in_double = not in_double
|
|
pieces.append(char)
|
|
elif char == "?" and not in_single and not in_double:
|
|
pieces.append("%s")
|
|
elif char == "%" and escape_percent:
|
|
# psycopg uses percent-style parameter binding even when the
|
|
# percent sign is part of a quoted LIKE pattern. All percent
|
|
# signs reaching this function came from the SQLite-style source
|
|
# query (generated placeholders are added in the branch above),
|
|
# so quote them before passing a parameter sequence to psycopg.
|
|
pieces.append("%%")
|
|
else:
|
|
pieces.append(char)
|
|
index += 1
|
|
return "".join(pieces)
|
|
|
|
|
|
def _strip_collations(sql: str) -> str:
|
|
return re.sub(r"\s+COLLATE\s+NOCASE", "", sql, flags=re.IGNORECASE)
|
|
|
|
|
|
def _translate_insert_or_ignore(sql: str) -> str:
|
|
if re.match(r"^\s*INSERT\s+OR\s+IGNORE\s+INTO\s+", sql, flags=re.IGNORECASE):
|
|
sql = re.sub(r"^\s*INSERT\s+OR\s+IGNORE\s+INTO\s+", "INSERT INTO ", sql, flags=re.IGNORECASE)
|
|
if "ON CONFLICT" not in sql.upper():
|
|
sql = sql.rstrip().rstrip(";") + " ON CONFLICT DO NOTHING"
|
|
return sql
|
|
|
|
|
|
def _translate_query(sql: str, params: Sequence[Any] | None) -> tuple[str, Sequence[Any] | None]:
|
|
original = sql
|
|
stripped = sql.strip()
|
|
if not stripped:
|
|
return sql, params
|
|
upper = stripped.upper()
|
|
if upper.startswith("PRAGMA "):
|
|
return "SELECT 1", ()
|
|
if "SQLITE_MASTER" in upper:
|
|
return (
|
|
"SELECT 1 WHERE EXISTS ("
|
|
"SELECT 1 FROM information_schema.tables WHERE table_schema='public' AND table_name=%s"
|
|
")",
|
|
(params[0] if params else "",),
|
|
)
|
|
sql = _strip_collations(sql)
|
|
sql = _translate_insert_or_ignore(sql)
|
|
sql = re.sub(r"^\s*INSERT\s+OR\s+REPLACE\s+INTO\s+", "INSERT INTO ", sql, flags=re.IGNORECASE)
|
|
sql = re.sub(r"\bbm25\s*\(\s*text_fts\s*\)", "0.0", sql, flags=re.IGNORECASE)
|
|
sql = re.sub(r"\babs\s*\(", "ABS(", sql, flags=re.IGNORECASE)
|
|
match_marker = "text_fts MATCH ?"
|
|
if match_marker in original:
|
|
parameter_index = _placeholder_count(original[: original.index(match_marker)])
|
|
sql = sql.replace(
|
|
match_marker,
|
|
"string_to_array(text_fts.tokens, ' ') && ?::text[]",
|
|
)
|
|
if params is not None and parameter_index < len(params):
|
|
params = list(params)
|
|
params[parameter_index] = _fts_tokens(str(params[parameter_index]))
|
|
sql = _replace_placeholders(sql, escape_percent=params is not None)
|
|
return sql, params
|
|
|
|
|
|
def _placeholder_count(sql: str) -> int:
|
|
count = 0
|
|
in_single = False
|
|
in_double = False
|
|
for char in sql:
|
|
if char == "'" and not in_double:
|
|
in_single = not in_single
|
|
elif char == '"' and not in_single:
|
|
in_double = not in_double
|
|
elif char == "?" and not in_single and not in_double:
|
|
count += 1
|
|
return count
|
|
|
|
|
|
def _fts_tokens(value: str) -> list[str]:
|
|
quoted = [token.replace('""', '"') for token in re.findall(r'"((?:""|[^"])*)"', value)]
|
|
if quoted:
|
|
return list(dict.fromkeys(token for token in quoted if token))
|
|
return list(
|
|
dict.fromkeys(
|
|
part.strip('"* ')
|
|
for part in re.split(r"\s+(?:OR\s+)?", value, flags=re.IGNORECASE)
|
|
if part.strip('"* ') and part.upper() != "OR"
|
|
)
|
|
)
|
|
|
|
|
|
def _split_script(script: str) -> list[str]:
|
|
statements: list[str] = []
|
|
current: list[str] = []
|
|
in_single = False
|
|
in_double = False
|
|
for char in script:
|
|
if char == "'" and not in_double:
|
|
in_single = not in_single
|
|
elif char == '"' and not in_single:
|
|
in_double = not in_double
|
|
if char == ";" and not in_single and not in_double:
|
|
statement = "".join(current).strip()
|
|
if statement:
|
|
statements.append(statement)
|
|
current = []
|
|
else:
|
|
current.append(char)
|
|
statement = "".join(current).strip()
|
|
if statement:
|
|
statements.append(statement)
|
|
return statements
|
|
|
|
|
|
class Cursor:
|
|
def __init__(self, raw):
|
|
self._raw = raw
|
|
self._columns: list[str] = []
|
|
|
|
@property
|
|
def rowcount(self) -> int:
|
|
return self._raw.rowcount
|
|
|
|
def execute(self, sql: str, params: Sequence[Any] | None = None):
|
|
translated, translated_params = _translate_query(sql, params)
|
|
try:
|
|
self._raw.execute(translated, translated_params)
|
|
except Exception as exc:
|
|
_raise_sqlite_compatible(exc)
|
|
self._columns = [item.name for item in (self._raw.description or [])]
|
|
return self
|
|
|
|
def fetchone(self) -> Row | None:
|
|
row = self._raw.fetchone()
|
|
return Row(self._columns, row) if row is not None else None
|
|
|
|
def fetchall(self) -> list[Row]:
|
|
return [Row(self._columns, row) for row in self._raw.fetchall()]
|
|
|
|
def __iter__(self):
|
|
for row in self._raw:
|
|
yield Row(self._columns, row)
|
|
|
|
|
|
class Connection:
|
|
def __init__(self, raw):
|
|
self._raw = raw
|
|
|
|
def execute(self, sql: str, params: Sequence[Any] | None = None) -> Cursor:
|
|
cursor = Cursor(self._raw.cursor())
|
|
return cursor.execute(sql, params)
|
|
|
|
def executemany(self, sql: str, rows: Iterable[Sequence[Any]]) -> Cursor:
|
|
translated, _ = _translate_query(sql, None)
|
|
cursor = Cursor(self._raw.cursor())
|
|
try:
|
|
cursor._raw.executemany(translated, rows)
|
|
except Exception as exc:
|
|
_raise_sqlite_compatible(exc)
|
|
return cursor
|
|
|
|
def executescript(self, script: str) -> None:
|
|
for statement in _split_script(script):
|
|
translated = _translate_ddl(statement)
|
|
if translated:
|
|
self.execute(translated)
|
|
|
|
def commit(self) -> None:
|
|
self._raw.commit()
|
|
|
|
def rollback(self) -> None:
|
|
self._raw.rollback()
|
|
|
|
def close(self) -> None:
|
|
self._raw.close()
|
|
|
|
|
|
class DatabaseTransientError(RuntimeError):
|
|
"""A PostgreSQL failure for which retrying a complete transaction is safe."""
|
|
|
|
|
|
def _sqlstate(exc: BaseException) -> str | None:
|
|
current: BaseException | None = exc
|
|
while current is not None:
|
|
value = getattr(current, "sqlstate", None)
|
|
if value:
|
|
return str(value)
|
|
current = current.__cause__
|
|
return None
|
|
|
|
|
|
def _raise_sqlite_compatible(exc: Exception) -> None:
|
|
sqlstate = _sqlstate(exc)
|
|
name = type(exc).__name__.lower()
|
|
message = str(exc)
|
|
if sqlstate in {"40001", "40P01", "55P03"}:
|
|
raise DatabaseTransientError(message) from exc
|
|
if sqlstate and sqlstate.startswith("23"):
|
|
raise sqlite3.IntegrityError(message) from exc
|
|
if "unique" in name or "unique" in message.lower() or "foreignkey" in name or "integrity" in name:
|
|
raise sqlite3.IntegrityError(message) from exc
|
|
raise exc
|
|
|
|
|
|
def _translate_ddl(sql: str) -> str:
|
|
upper = sql.strip().upper()
|
|
if upper.startswith("CREATE VIRTUAL TABLE"):
|
|
return "CREATE TABLE IF NOT EXISTS text_fts (entry_id TEXT PRIMARY KEY, tokens TEXT NOT NULL)"
|
|
if upper.startswith("CREATE TRIGGER"):
|
|
return ""
|
|
if upper.startswith("DROP TRIGGER"):
|
|
return ""
|
|
sql = _strip_collations(sql)
|
|
sql = re.sub(r"\bINTEGER\s+PRIMARY\s+KEY\s+CHECK\s*\([^)]+\)", "INTEGER PRIMARY KEY", sql, flags=re.IGNORECASE)
|
|
sql = re.sub(r"\bBLOB\b", "BYTEA", sql, flags=re.IGNORECASE)
|
|
sql = re.sub(r"\bREAL\b", "DOUBLE PRECISION", sql, flags=re.IGNORECASE)
|
|
sql = re.sub(r"\bAUTOINCREMENT\b", "", sql, flags=re.IGNORECASE)
|
|
return sql
|
|
|
|
|
|
class Database:
|
|
def __init__(self, path: Path):
|
|
self.path = path
|
|
self._metrics = _WriteCoordinator()
|
|
self._pool_guard = threading.Lock()
|
|
self._pool: _ConnectionPool | None = None
|
|
self._pool_min = max(0, int(os.environ.get("IMAGEFIND_DB_POOL_MIN", "1")))
|
|
self._pool_max = max(1, int(os.environ.get("IMAGEFIND_DB_POOL_MAX", "8")))
|
|
self._pool_timeout = max(0.1, float(os.environ.get("IMAGEFIND_DB_POOL_TIMEOUT", "5")))
|
|
self._pool_idle_timeout = max(1.0, float(os.environ.get("IMAGEFIND_DB_POOL_IDLE_TIMEOUT", "60")))
|
|
|
|
@property
|
|
def postgres_conf_path(self) -> Path:
|
|
return Path(os.environ.get("IMAGEFIND_POSTGRES_CONF") or self.path.parent.parent / "postgres-client.conf")
|
|
|
|
def _connect_raw(self, *, readonly: bool = False, timeout_seconds: int = 15, autocommit: bool = False):
|
|
psycopg = _load_psycopg()
|
|
if not self.postgres_conf_path.exists():
|
|
raise RuntimeError(f"PostgreSQL 凭据不存在：{self.postgres_conf_path}")
|
|
conf = _postgres_conf_from_path(self.postgres_conf_path)
|
|
# Configure the session in autocommit mode. Otherwise these SET
|
|
# statements open an implicit transaction before a caller can choose
|
|
# an isolation level for its own explicit BEGIN.
|
|
conn = psycopg.connect(
|
|
host=conf.get("host", "127.0.0.1"),
|
|
port=int(conf.get("port", "15432")),
|
|
dbname=conf["database"],
|
|
user=conf["username"],
|
|
password=conf["password"],
|
|
sslmode=conf.get("sslmode", conf.get("sslMode", "disable")).lower(),
|
|
connect_timeout=timeout_seconds,
|
|
autocommit=True,
|
|
)
|
|
# PostgreSQL clusters created by third-party fnOS packages should be
|
|
# UTF-8, but explicitly selecting the client encoding also keeps the
|
|
# SQLite-compatible row API stable against older SQL_ASCII clusters.
|
|
# Without it psycopg may expose TEXT values as bytes, which then
|
|
# leaks into JSON and search responses.
|
|
conn.execute("SET client_encoding TO 'UTF8'")
|
|
conn.execute("SET statement_timeout = '120s'")
|
|
conn.execute("SET idle_in_transaction_session_timeout = '120s'")
|
|
if readonly:
|
|
conn.execute("SET default_transaction_read_only = on")
|
|
if not autocommit and not readonly:
|
|
conn.autocommit = False
|
|
return conn
|
|
|
|
def _connection_pool(self) -> _ConnectionPool:
|
|
with self._pool_guard:
|
|
if self._pool is None:
|
|
self._pool = _ConnectionPool(
|
|
lambda: self._connect_raw(timeout_seconds=5, autocommit=True),
|
|
min_size=min(self._pool_min, self._pool_max),
|
|
max_size=self._pool_max,
|
|
timeout=self._pool_timeout,
|
|
idle_timeout=self._pool_idle_timeout,
|
|
)
|
|
self._pool.open()
|
|
return self._pool
|
|
|
|
def connect(self, busy_timeout_ms: int = 5000) -> Connection:
|
|
return Connection(self._connect_raw(timeout_seconds=max(1, int(busy_timeout_ms / 1000))))
|
|
|
|
def close(self) -> None:
|
|
with self._pool_guard:
|
|
pool = self._pool
|
|
self._pool = None
|
|
if pool is not None:
|
|
pool.close()
|
|
|
|
@contextlib.contextmanager
|
|
def transaction(self) -> Iterator[Connection]:
|
|
pool = self._connection_pool()
|
|
raw = pool.acquire()
|
|
conn = Connection(raw)
|
|
acquired_at = self._metrics.acquire()
|
|
discard = False
|
|
try:
|
|
with raw.transaction():
|
|
yield conn
|
|
except Exception:
|
|
discard = bool(getattr(raw, "closed", False))
|
|
raise
|
|
finally:
|
|
self._metrics.release(acquired_at)
|
|
pool.release(raw, discard=discard)
|
|
|
|
@contextlib.contextmanager
|
|
def critical_transaction(self, *, timeout_seconds: float = 30.0) -> Iterator[Connection]:
|
|
with self.transaction() as conn:
|
|
yield conn
|
|
|
|
@contextlib.contextmanager
|
|
def read(self) -> Iterator[Connection]:
|
|
pool = self._connection_pool()
|
|
raw = pool.acquire()
|
|
conn = Connection(raw)
|
|
discard = False
|
|
try:
|
|
raw.execute("SET default_transaction_read_only = on")
|
|
yield conn
|
|
except Exception:
|
|
discard = bool(getattr(raw, "closed", False))
|
|
raise
|
|
finally:
|
|
if not discard:
|
|
try:
|
|
raw.execute("SET default_transaction_read_only = off")
|
|
except Exception:
|
|
discard = True
|
|
pool.release(raw, discard=discard)
|
|
|
|
def write_with_retry(self, operation, *, timeout_seconds: float = 30.0):
|
|
deadline = time.monotonic() + max(0.1, timeout_seconds)
|
|
delay = 0.025
|
|
while True:
|
|
try:
|
|
with self.transaction() as conn:
|
|
return operation(conn)
|
|
except DatabaseTransientError:
|
|
if time.monotonic() >= deadline:
|
|
raise
|
|
self._metrics.record_lock_retry()
|
|
time.sleep(min(delay, max(0, deadline - time.monotonic())))
|
|
delay = min(delay * 2, 0.75)
|
|
|
|
def status(self) -> dict[str, int | float | bool | str]:
|
|
with self._pool_guard:
|
|
pool = self._pool
|
|
pool_status = pool.status() if pool is not None else {
|
|
"pool_min": self._pool_min,
|
|
"pool_max": self._pool_max,
|
|
"pool_size": 0,
|
|
"pool_in_use": 0,
|
|
"pool_idle": 0,
|
|
"pool_waiters": 0,
|
|
"pool_wait_count": 0,
|
|
"pool_last_wait_ms": 0.0,
|
|
"pool_max_wait_ms": 0.0,
|
|
}
|
|
metrics = self._metrics.status()
|
|
return {
|
|
"engine": "postgresql",
|
|
"journal_mode": "server",
|
|
**metrics,
|
|
"writer_queue_depth": int(pool_status["pool_waiters"]),
|
|
"writer_active": bool(pool_status["pool_in_use"]),
|
|
"last_wait_ms": float(pool_status["pool_last_wait_ms"]),
|
|
"max_wait_ms": float(pool_status["pool_max_wait_ms"]),
|
|
**pool_status,
|
|
}
|
|
|
|
def size_bytes(self) -> int:
|
|
with self.read() as conn:
|
|
row = conn.execute("SELECT pg_database_size(current_database())").fetchone()
|
|
return int(row[0] or 0)
|
|
|
|
def activity(self) -> dict[str, Any]:
|
|
with self.read() as conn:
|
|
states = conn.execute(
|
|
"SELECT coalesce(state,'unknown') AS state,count(*) AS count,"
|
|
"coalesce(max(extract(epoch FROM (clock_timestamp()-xact_start))),0) AS oldest_xact_seconds "
|
|
"FROM pg_stat_activity WHERE datname=current_database() AND pid<>pg_backend_pid() GROUP BY state"
|
|
).fetchall()
|
|
waiting = conn.execute(
|
|
"SELECT count(*) FROM pg_stat_activity WHERE datname=current_database() "
|
|
"AND pid<>pg_backend_pid() AND state='active' AND wait_event_type='Lock'"
|
|
).fetchone()
|
|
return {
|
|
"states": {
|
|
str(row["state"]): {
|
|
"count": int(row["count"]),
|
|
"oldest_transaction_seconds": round(float(row["oldest_xact_seconds"] or 0), 2),
|
|
}
|
|
for row in states
|
|
},
|
|
"waiting": int(waiting[0] or 0),
|
|
}
|
|
|
|
def _current_schema_version(self) -> int:
|
|
try:
|
|
with self.read() as conn:
|
|
row = conn.execute("SELECT value FROM settings WHERE key=?", ("schema_version",)).fetchone()
|
|
except Exception:
|
|
return 0
|
|
if row is None:
|
|
return 0
|
|
try:
|
|
return int(json.loads(row["value"]))
|
|
except (TypeError, ValueError, json.JSONDecodeError):
|
|
return int(row["value"])
|
|
|
|
def initialize(self) -> None:
|
|
self.close()
|
|
with self.transaction() as conn:
|
|
conn.execute("CREATE EXTENSION IF NOT EXISTS vector")
|
|
conn.execute("CREATE EXTENSION IF NOT EXISTS pg_trgm")
|
|
conn.executescript(POSTGRES_DDL)
|
|
conn.execute(
|
|
"ALTER TABLE api_tokens ADD COLUMN IF NOT EXISTS scopes_json TEXT NOT NULL DEFAULT '[\"admin\"]'"
|
|
)
|
|
conn.execute("ALTER TABLE jobs ADD COLUMN IF NOT EXISTS priority INTEGER NOT NULL DEFAULT 50")
|
|
conn.execute("ALTER TABLE jobs ADD COLUMN IF NOT EXISTS diagnostics_json TEXT NOT NULL DEFAULT '{}'")
|
|
conn.execute(
|
|
"UPDATE jobs SET priority=CASE kind "
|
|
"WHEN 'prepare_ai_runtime' THEN -10 WHEN 'install_models' THEN -10 "
|
|
"WHEN 'refresh_path' THEN 10 WHEN 'transfer_upload' THEN 10 "
|
|
"WHEN 'aria2_download' THEN 10 WHEN 'scan_source' THEN 10 "
|
|
"WHEN 'index_video' THEN 10 WHEN 'transcribe_audio' THEN 20 "
|
|
"WHEN 'suggest_tags' THEN 40 ELSE 50 END WHERE priority=50"
|
|
)
|
|
conn.execute(
|
|
"UPDATE jobs SET priority=10 "
|
|
"WHERE kind='index_video' AND status='queued' AND priority>=20"
|
|
)
|
|
conn.execute(
|
|
"UPDATE jobs SET priority=20 WHERE kind='transcribe_audio' AND status='queued' AND priority=30"
|
|
)
|
|
conn.execute(
|
|
"CREATE INDEX IF NOT EXISTS idx_jobs_queue_priority "
|
|
"ON jobs(status,run_after,priority,created_at)"
|
|
)
|
|
conn.execute(
|
|
"ALTER TABLE text_entries ADD COLUMN IF NOT EXISTS search_vector tsvector "
|
|
"GENERATED ALWAYS AS (to_tsvector('simple',coalesce(tokens,''))) STORED"
|
|
)
|
|
conn.execute("ALTER TABLE text_entries ADD COLUMN IF NOT EXISTS language TEXT")
|
|
conn.execute("ALTER TABLE text_entries ADD COLUMN IF NOT EXISTS quality_score DOUBLE PRECISION")
|
|
conn.execute("ALTER TABLE videos ADD COLUMN IF NOT EXISTS audio_index_revision INTEGER NOT NULL DEFAULT 0")
|
|
conn.execute("ALTER TABLE videos ADD COLUMN IF NOT EXISTS audio_detected_language TEXT")
|
|
conn.execute("ALTER TABLE videos ADD COLUMN IF NOT EXISTS audio_quality_score DOUBLE PRECISION")
|
|
conn.execute(
|
|
"ALTER TABLE videos ADD COLUMN IF NOT EXISTS audio_rejected_segments INTEGER NOT NULL DEFAULT 0"
|
|
)
|
|
conn.execute(
|
|
"ALTER TABLE videos ADD COLUMN IF NOT EXISTS audio_quality_flags_json TEXT NOT NULL DEFAULT '[]'"
|
|
)
|
|
conn.execute(
|
|
"ALTER TABLE videos ADD COLUMN IF NOT EXISTS audio_quality_repair_revision INTEGER NOT NULL DEFAULT 0"
|
|
)
|
|
conn.execute(
|
|
"ALTER TABLE videos ADD COLUMN IF NOT EXISTS storage_backend TEXT NOT NULL DEFAULT 'legacy_webdav'"
|
|
)
|
|
conn.execute("ALTER TABLE videos ADD COLUMN IF NOT EXISTS physical_path TEXT")
|
|
conn.execute("ALTER TABLE videos ADD COLUMN IF NOT EXISTS physical_size_bytes BIGINT")
|
|
conn.execute(
|
|
"ALTER TABLE trash ADD COLUMN IF NOT EXISTS storage_backend TEXT NOT NULL DEFAULT 'legacy_webdav'"
|
|
)
|
|
conn.execute("ALTER TABLE trash ADD COLUMN IF NOT EXISTS physical_original_path TEXT")
|
|
conn.execute("ALTER TABLE trash ADD COLUMN IF NOT EXISTS physical_trash_path TEXT")
|
|
conn.execute("ALTER TABLE trash ADD COLUMN IF NOT EXISTS physical_size_bytes BIGINT")
|
|
conn.execute("ALTER TABLE uploads ADD COLUMN IF NOT EXISTS phase TEXT NOT NULL DEFAULT 'receiving'")
|
|
conn.execute(
|
|
"ALTER TABLE uploads ADD COLUMN IF NOT EXISTS storage_backend TEXT NOT NULL DEFAULT 'legacy_webdav'"
|
|
)
|
|
conn.execute("ALTER TABLE uploads ADD COLUMN IF NOT EXISTS external_task_id TEXT")
|
|
conn.execute("ALTER TABLE uploads ADD COLUMN IF NOT EXISTS external_state INTEGER")
|
|
conn.execute(
|
|
"ALTER TABLE uploads ADD COLUMN IF NOT EXISTS external_progress DOUBLE PRECISION NOT NULL DEFAULT 0"
|
|
)
|
|
conn.execute("ALTER TABLE uploads ADD COLUMN IF NOT EXISTS external_status TEXT")
|
|
conn.execute("ALTER TABLE uploads ADD COLUMN IF NOT EXISTS external_error TEXT")
|
|
conn.execute("ALTER TABLE uploads ADD COLUMN IF NOT EXISTS external_started_at TEXT")
|
|
conn.execute("ALTER TABLE uploads ADD COLUMN IF NOT EXISTS external_last_progress_at TEXT")
|
|
conn.execute("ALTER TABLE uploads ADD COLUMN IF NOT EXISTS external_source_path TEXT")
|
|
conn.execute("ALTER TABLE uploads ADD COLUMN IF NOT EXISTS external_staged_path TEXT")
|
|
conn.execute("ALTER TABLE uploads ADD COLUMN IF NOT EXISTS external_target_path TEXT")
|
|
conn.execute("ALTER TABLE uploads ADD COLUMN IF NOT EXISTS external_local_path TEXT")
|
|
conn.execute("ALTER TABLE uploads ADD COLUMN IF NOT EXISTS external_size_bytes BIGINT")
|
|
conn.execute("ALTER TABLE uploads ADD COLUMN IF NOT EXISTS external_attempts INTEGER NOT NULL DEFAULT 0")
|
|
conn.execute(
|
|
"ALTER TABLE uploads ADD COLUMN IF NOT EXISTS recovery_state TEXT NOT NULL DEFAULT 'receiving'"
|
|
)
|
|
conn.execute("ALTER TABLE uploads ADD COLUMN IF NOT EXISTS recovery_mode TEXT")
|
|
conn.execute(
|
|
"UPDATE videos v SET storage_backend=u.storage_backend,physical_path=u.external_target_path,"
|
|
"physical_size_bytes=coalesce(u.external_size_bytes,v.size_bytes) FROM ("
|
|
"SELECT DISTINCT ON (source_id,target_key) source_id,target_key,storage_backend,"
|
|
"external_target_path,external_size_bytes FROM uploads "
|
|
"WHERE status='completed' AND storage_backend='openlist_native' "
|
|
"AND target_key IS NOT NULL AND external_target_path IS NOT NULL "
|
|
"ORDER BY source_id,target_key,updated_at DESC) u "
|
|
"WHERE v.source_id=u.source_id AND v.source_key=u.target_key "
|
|
"AND (v.physical_path IS NULL OR v.storage_backend<>'openlist_native')"
|
|
)
|
|
conn.execute(
|
|
"UPDATE uploads SET phase=CASE status WHEN 'receiving' THEN 'receiving' WHEN 'queued' THEN 'queued' "
|
|
"WHEN 'transferring' THEN 'transferring' WHEN 'indexing' THEN 'indexing' "
|
|
"WHEN 'completed' THEN 'ai_queued' WHEN 'failed' THEN 'failed' ELSE 'cancelled' END "
|
|
"WHERE phase IS NULL OR phase='receiving' AND status<>'receiving' "
|
|
"OR status='completed' AND phase<>'ai_queued'"
|
|
)
|
|
conn.execute("CREATE INDEX IF NOT EXISTS idx_text_entries_search ON text_entries USING GIN(search_vector)")
|
|
conn.execute(
|
|
"CREATE INDEX IF NOT EXISTS idx_text_entries_raw_text_trgm "
|
|
"ON text_entries USING GIN(lower(raw_text) gin_trgm_ops)"
|
|
)
|
|
conn.execute("CREATE INDEX IF NOT EXISTS idx_query_images_expiry ON query_images(expires_at)")
|
|
conn.execute(
|
|
"CREATE INDEX IF NOT EXISTS idx_videos_available_updated ON videos(available,updated_at DESC,id)"
|
|
)
|
|
conn.execute(
|
|
"CREATE INDEX IF NOT EXISTS idx_video_metadata_title_trgm "
|
|
"ON video_metadata USING GIN(lower(coalesce(title,'')) gin_trgm_ops)"
|
|
)
|
|
conn.execute(
|
|
"CREATE INDEX IF NOT EXISTS idx_people_name_trgm "
|
|
"ON people USING GIN(lower(coalesce(name,'')) gin_trgm_ops)"
|
|
)
|
|
conn.execute(
|
|
"CREATE TABLE IF NOT EXISTS job_resources("
|
|
"job_id TEXT NOT NULL REFERENCES jobs(id) ON DELETE CASCADE,"
|
|
"resource_type TEXT NOT NULL,resource_id TEXT NOT NULL,"
|
|
"PRIMARY KEY(job_id,resource_type,resource_id))"
|
|
)
|
|
conn.execute(
|
|
"CREATE INDEX IF NOT EXISTS idx_job_resources_lookup "
|
|
"ON job_resources(resource_type,resource_id,job_id)"
|
|
)
|
|
conn.execute(
|
|
"INSERT INTO job_resources(job_id,resource_type,resource_id) "
|
|
"SELECT id,'video',payload_json::jsonb->>'video_id' FROM jobs "
|
|
"WHERE payload_json::jsonb->>'video_id' IS NOT NULL "
|
|
"AND payload_json::jsonb->>'video_id'<>'' "
|
|
"ON CONFLICT DO NOTHING"
|
|
)
|
|
conn.execute(
|
|
"INSERT INTO job_resources(job_id,resource_type,resource_id) "
|
|
"SELECT jobs.id,'video',items.value FROM jobs "
|
|
"CROSS JOIN LATERAL jsonb_array_elements_text("
|
|
"CASE WHEN jsonb_typeof(payload_json::jsonb->'video_ids')='array' "
|
|
"THEN payload_json::jsonb->'video_ids' ELSE '[]'::jsonb END) AS items(value) "
|
|
"ON CONFLICT DO NOTHING"
|
|
)
|
|
conn.execute(
|
|
"INSERT INTO tag_groups(id,name,selection_mode,sort_order,created_at,updated_at) "
|
|
"VALUES(?,?,'multi',0,?,?) ON CONFLICT DO NOTHING",
|
|
(DEFAULT_TAG_GROUP_ID, "未分组", utcnow(), utcnow()),
|
|
)
|
|
conn.execute(
|
|
"INSERT INTO settings(key,value,updated_at) VALUES('schema_version',?,?) "
|
|
"ON CONFLICT(key) DO UPDATE SET value=excluded.value, updated_at=excluded.updated_at",
|
|
(str(SCHEMA_VERSION), utcnow()),
|
|
)
|
|
with self.read() as conn:
|
|
conn.execute("SELECT 1 FROM settings LIMIT 1").fetchone()
|
|
|
|
@staticmethod
|
|
def _migrate_collections_v8(conn: Connection) -> None:
|
|
"""Materialize legacy ``video_metadata.series`` values as collections.
|
|
|
|
Runtime storage is PostgreSQL-only, but logical v1 backups and the
|
|
legacy ``/series`` API still use the old metadata field as their input.
|
|
This idempotent bridge is therefore a data-model compatibility helper,
|
|
not a SQLite database migration.
|
|
"""
|
|
|
|
rows = conn.execute(
|
|
"SELECT vm.video_id,trim(vm.series) AS series,vm.updated_at,v.created_at "
|
|
"FROM video_metadata vm JOIN videos v ON v.id=vm.video_id "
|
|
"WHERE vm.series IS NOT NULL AND trim(vm.series)<>'' "
|
|
"ORDER BY lower(trim(vm.series)),v.created_at,vm.video_id"
|
|
).fetchall()
|
|
groups: dict[str, list[Row]] = {}
|
|
names: dict[str, str] = {}
|
|
for row in rows:
|
|
normalized = str(row["series"]).casefold()
|
|
groups.setdefault(normalized, []).append(row)
|
|
names.setdefault(normalized, str(row["series"]))
|
|
for normalized, videos in groups.items():
|
|
name = names[normalized]
|
|
collection = conn.execute(
|
|
"SELECT id,name FROM collections WHERE lower(name)=lower(?)",
|
|
(name,),
|
|
).fetchone()
|
|
if collection:
|
|
collection_id = str(collection["id"])
|
|
canonical_name = str(collection["name"])
|
|
else:
|
|
collection_id = str(
|
|
uuid.uuid5(uuid.NAMESPACE_URL, f"imagefind:legacy-series:{normalized}")
|
|
)
|
|
created_at = min(str(row["created_at"]) for row in videos)
|
|
updated_at = max(str(row["updated_at"]) for row in videos)
|
|
conn.execute(
|
|
"INSERT INTO collections(id,name,description,created_at,updated_at) "
|
|
"VALUES(?,?,'',?,?) ON CONFLICT DO NOTHING",
|
|
(collection_id, name, created_at, updated_at),
|
|
)
|
|
collection = conn.execute(
|
|
"SELECT id,name FROM collections WHERE lower(name)=lower(?)",
|
|
(name,),
|
|
).fetchone()
|
|
if not collection:
|
|
continue
|
|
collection_id = str(collection["id"])
|
|
canonical_name = str(collection["name"])
|
|
next_position = int(
|
|
conn.execute(
|
|
"SELECT coalesce(max(position),-1)+1 FROM collection_videos WHERE collection_id=?",
|
|
(collection_id,),
|
|
).fetchone()[0]
|
|
)
|
|
for row in videos:
|
|
membership = conn.execute(
|
|
"SELECT 1 FROM collection_videos WHERE video_id=?",
|
|
(row["video_id"],),
|
|
).fetchone()
|
|
if not membership:
|
|
conn.execute(
|
|
"INSERT INTO collection_videos(collection_id,video_id,position,added_at) "
|
|
"VALUES(?,?,?,?)",
|
|
(collection_id, row["video_id"], next_position, row["created_at"]),
|
|
)
|
|
next_position += 1
|
|
if str(row["series"]) != canonical_name:
|
|
conn.execute(
|
|
"UPDATE video_metadata SET series=?,updated_at=? WHERE video_id=?",
|
|
(canonical_name, row["updated_at"], row["video_id"]),
|
|
)
|
|
|
|
@staticmethod
|
|
def _migrate_collection_items_v14(conn: Connection) -> None:
|
|
"""Populate hierarchy roots for flat collection memberships."""
|
|
|
|
rows = conn.execute(
|
|
"SELECT cv.collection_id,cv.video_id,cv.position,cv.added_at "
|
|
"FROM collection_videos cv LEFT JOIN collection_items ci ON ci.video_id=cv.video_id "
|
|
"WHERE ci.id IS NULL ORDER BY cv.collection_id,cv.position,cv.added_at,cv.video_id"
|
|
).fetchall()
|
|
for row in rows:
|
|
item_id = str(
|
|
uuid.uuid5(uuid.NAMESPACE_URL, f"imagefind:collection-item:{row['video_id']}")
|
|
)
|
|
conn.execute(
|
|
"INSERT INTO collection_items(id,collection_id,parent_id,kind,name,video_id,position,"
|
|
"created_at,updated_at) VALUES(?,?,NULL,'video',NULL,?,?,?,?) ON CONFLICT DO NOTHING",
|
|
(
|
|
item_id,
|
|
row["collection_id"],
|
|
row["video_id"],
|
|
int(row["position"] or 0),
|
|
row["added_at"],
|
|
row["added_at"],
|
|
),
|
|
)
|
|
|
|
def setting(self, key: str, default: Any = None) -> Any:
|
|
with self.read() as conn:
|
|
row = conn.execute("SELECT value FROM settings WHERE key=?", (key,)).fetchone()
|
|
if row is None:
|
|
return default
|
|
try:
|
|
return json.loads(row["value"])
|
|
except json.JSONDecodeError:
|
|
return row["value"]
|
|
|
|
def set_setting(self, key: str, value: Any) -> None:
|
|
encoded = json.dumps(value, ensure_ascii=False)
|
|
with self.transaction() as conn:
|
|
conn.execute(
|
|
"INSERT INTO settings(key,value,updated_at) VALUES(?,?,?) "
|
|
"ON CONFLICT(key) DO UPDATE SET value=excluded.value, updated_at=excluded.updated_at",
|
|
(key, encoded, utcnow()),
|
|
)
|
|
|
|
|
|
POSTGRES_DDL = """
|
|
CREATE TABLE IF NOT EXISTS settings (
|
|
key TEXT PRIMARY KEY,
|
|
value TEXT NOT NULL,
|
|
updated_at TEXT NOT NULL
|
|
);
|
|
CREATE TABLE IF NOT EXISTS admin (
|
|
id INTEGER PRIMARY KEY CHECK (id = 1),
|
|
password_hash TEXT NOT NULL,
|
|
created_at TEXT NOT NULL,
|
|
updated_at TEXT NOT NULL
|
|
);
|
|
CREATE TABLE IF NOT EXISTS sessions (
|
|
token_hash TEXT PRIMARY KEY,
|
|
csrf_token TEXT NOT NULL,
|
|
expires_at TEXT NOT NULL,
|
|
created_at TEXT NOT NULL,
|
|
auth_source TEXT NOT NULL DEFAULT 'local',
|
|
external_user_id TEXT,
|
|
external_username TEXT
|
|
);
|
|
CREATE TABLE IF NOT EXISTS gateway_media_tokens (
|
|
token_hash TEXT PRIMARY KEY,
|
|
session_token_hash TEXT NOT NULL REFERENCES sessions(token_hash) ON DELETE CASCADE,
|
|
expires_at TEXT NOT NULL,
|
|
created_at TEXT NOT NULL
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_gateway_media_tokens_session ON gateway_media_tokens(session_token_hash,created_at DESC);
|
|
CREATE TABLE IF NOT EXISTS api_tokens (
|
|
id TEXT PRIMARY KEY,
|
|
name TEXT NOT NULL,
|
|
token_hash TEXT NOT NULL UNIQUE,
|
|
scopes_json TEXT NOT NULL DEFAULT '["admin"]',
|
|
created_at TEXT NOT NULL,
|
|
last_used_at TEXT,
|
|
revoked_at TEXT
|
|
);
|
|
CREATE TABLE IF NOT EXISTS sources (
|
|
id TEXT PRIMARY KEY,
|
|
kind TEXT NOT NULL CHECK(kind IN ('local', 'webdav')),
|
|
name TEXT NOT NULL,
|
|
config_json TEXT NOT NULL,
|
|
secret_blob TEXT,
|
|
enabled INTEGER NOT NULL DEFAULT 1,
|
|
status TEXT NOT NULL DEFAULT 'idle',
|
|
last_scan_at TEXT,
|
|
last_error TEXT,
|
|
created_at TEXT NOT NULL,
|
|
updated_at TEXT NOT NULL
|
|
);
|
|
CREATE TABLE IF NOT EXISTS videos (
|
|
id TEXT PRIMARY KEY,
|
|
source_id TEXT NOT NULL REFERENCES sources(id) ON DELETE CASCADE,
|
|
source_key TEXT NOT NULL,
|
|
display_name TEXT NOT NULL,
|
|
location TEXT NOT NULL,
|
|
size_bytes BIGINT NOT NULL DEFAULT 0,
|
|
modified_at TEXT,
|
|
etag TEXT,
|
|
fingerprint TEXT NOT NULL,
|
|
duration_ms INTEGER,
|
|
width INTEGER,
|
|
height INTEGER,
|
|
codec TEXT,
|
|
container TEXT,
|
|
status TEXT NOT NULL DEFAULT 'pending',
|
|
available INTEGER NOT NULL DEFAULT 1,
|
|
indexed_fingerprint TEXT,
|
|
basic_fingerprint TEXT,
|
|
visual_model_version TEXT,
|
|
ocr_model_version TEXT,
|
|
faces_model_version TEXT,
|
|
audio_model_version TEXT,
|
|
audio_index_revision INTEGER NOT NULL DEFAULT 0,
|
|
audio_detected_language TEXT,
|
|
audio_quality_score DOUBLE PRECISION,
|
|
audio_rejected_segments INTEGER NOT NULL DEFAULT 0,
|
|
audio_quality_flags_json TEXT NOT NULL DEFAULT '[]',
|
|
audio_quality_repair_revision INTEGER NOT NULL DEFAULT 0,
|
|
storage_backend TEXT NOT NULL DEFAULT 'legacy_webdav',
|
|
physical_path TEXT,
|
|
physical_size_bytes BIGINT,
|
|
error TEXT,
|
|
seen_scan_id TEXT,
|
|
content_sha256 TEXT,
|
|
content_fingerprint TEXT,
|
|
created_at TEXT NOT NULL,
|
|
updated_at TEXT NOT NULL,
|
|
UNIQUE(source_id, source_key)
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_videos_source ON videos(source_id, available);
|
|
CREATE INDEX IF NOT EXISTS idx_videos_status ON videos(status);
|
|
CREATE TABLE IF NOT EXISTS frames (
|
|
id TEXT PRIMARY KEY,
|
|
video_id TEXT NOT NULL REFERENCES videos(id) ON DELETE CASCADE,
|
|
timestamp_ms INTEGER NOT NULL,
|
|
segment_start_ms INTEGER NOT NULL,
|
|
segment_end_ms INTEGER NOT NULL,
|
|
thumbnail_path TEXT NOT NULL,
|
|
perceptual_hash TEXT,
|
|
vector_blob BYTEA,
|
|
embedding vector,
|
|
created_at TEXT NOT NULL
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_frames_video_time ON frames(video_id, timestamp_ms);
|
|
CREATE TABLE IF NOT EXISTS text_entries (
|
|
id TEXT PRIMARY KEY,
|
|
video_id TEXT NOT NULL REFERENCES videos(id) ON DELETE CASCADE,
|
|
frame_id TEXT REFERENCES frames(id) ON DELETE CASCADE,
|
|
kind TEXT NOT NULL CHECK(kind IN ('ocr','subtitle','audio','filename','path')),
|
|
start_ms INTEGER NOT NULL DEFAULT 0,
|
|
end_ms INTEGER NOT NULL DEFAULT 0,
|
|
raw_text TEXT NOT NULL,
|
|
tokens TEXT NOT NULL,
|
|
language TEXT,
|
|
quality_score DOUBLE PRECISION,
|
|
created_at TEXT NOT NULL
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_text_video ON text_entries(video_id, start_ms);
|
|
CREATE TABLE IF NOT EXISTS text_fts (
|
|
entry_id TEXT PRIMARY KEY,
|
|
tokens TEXT NOT NULL
|
|
);
|
|
CREATE TABLE IF NOT EXISTS people (
|
|
id TEXT PRIMARY KEY,
|
|
name TEXT NOT NULL,
|
|
normalized_name TEXT NOT NULL,
|
|
is_named INTEGER NOT NULL DEFAULT 0,
|
|
hidden INTEGER NOT NULL DEFAULT 0,
|
|
centroid_blob BYTEA,
|
|
face_count INTEGER NOT NULL DEFAULT 0,
|
|
created_at TEXT NOT NULL,
|
|
updated_at TEXT NOT NULL
|
|
);
|
|
CREATE TABLE IF NOT EXISTS faces (
|
|
id TEXT PRIMARY KEY,
|
|
frame_id TEXT NOT NULL REFERENCES frames(id) ON DELETE CASCADE,
|
|
video_id TEXT NOT NULL REFERENCES videos(id) ON DELETE CASCADE,
|
|
person_id TEXT NOT NULL REFERENCES people(id),
|
|
bbox_json TEXT NOT NULL,
|
|
confidence DOUBLE PRECISION NOT NULL,
|
|
thumbnail_path TEXT,
|
|
vector_blob BYTEA,
|
|
embedding vector,
|
|
created_at TEXT NOT NULL
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_faces_person ON faces(person_id);
|
|
CREATE INDEX IF NOT EXISTS idx_faces_video ON faces(video_id);
|
|
CREATE TABLE IF NOT EXISTS jobs (
|
|
id TEXT PRIMARY KEY,
|
|
kind TEXT NOT NULL,
|
|
payload_json TEXT NOT NULL,
|
|
dedupe_key TEXT,
|
|
status TEXT NOT NULL DEFAULT 'queued',
|
|
progress DOUBLE PRECISION NOT NULL DEFAULT 0,
|
|
message TEXT,
|
|
attempts INTEGER NOT NULL DEFAULT 0,
|
|
priority INTEGER NOT NULL DEFAULT 50,
|
|
run_after TEXT NOT NULL,
|
|
created_at TEXT NOT NULL,
|
|
started_at TEXT,
|
|
finished_at TEXT,
|
|
error TEXT,
|
|
diagnostics_json TEXT NOT NULL DEFAULT '{}',
|
|
lease_owner TEXT,
|
|
heartbeat_at TEXT,
|
|
pause_reason TEXT,
|
|
cancel_requested INTEGER NOT NULL DEFAULT 0
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_jobs_queue ON jobs(status, run_after, created_at);
|
|
CREATE UNIQUE INDEX IF NOT EXISTS idx_jobs_dedupe_active
|
|
ON jobs(dedupe_key)
|
|
WHERE dedupe_key IS NOT NULL AND status IN ('queued','running');
|
|
CREATE TABLE IF NOT EXISTS job_resources (
|
|
job_id TEXT NOT NULL REFERENCES jobs(id) ON DELETE CASCADE,
|
|
resource_type TEXT NOT NULL,
|
|
resource_id TEXT NOT NULL,
|
|
PRIMARY KEY(job_id,resource_type,resource_id)
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_job_resources_lookup
|
|
ON job_resources(resource_type,resource_id,job_id);
|
|
CREATE TABLE IF NOT EXISTS query_images (
|
|
id TEXT PRIMARY KEY,
|
|
path TEXT NOT NULL,
|
|
content_type TEXT NOT NULL,
|
|
expires_at TEXT NOT NULL,
|
|
created_at TEXT NOT NULL
|
|
);
|
|
CREATE TABLE IF NOT EXISTS backup_exports (
|
|
id TEXT PRIMARY KEY,
|
|
scope TEXT NOT NULL CHECK(scope IN ('keys','full')),
|
|
status TEXT NOT NULL DEFAULT 'queued',
|
|
filename TEXT,
|
|
path TEXT,
|
|
size_bytes BIGINT NOT NULL DEFAULT 0,
|
|
secret_blob TEXT,
|
|
error TEXT,
|
|
created_at TEXT NOT NULL,
|
|
finished_at TEXT,
|
|
expires_at TEXT NOT NULL
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_backup_exports_expiry ON backup_exports(expires_at,status);
|
|
CREATE TABLE IF NOT EXISTS collections (
|
|
id TEXT PRIMARY KEY,
|
|
name TEXT NOT NULL UNIQUE,
|
|
description TEXT NOT NULL DEFAULT '',
|
|
cover_video_id TEXT REFERENCES videos(id) ON DELETE SET NULL,
|
|
created_at TEXT NOT NULL,
|
|
updated_at TEXT NOT NULL
|
|
);
|
|
CREATE UNIQUE INDEX IF NOT EXISTS idx_collections_name ON collections(lower(name));
|
|
CREATE TABLE IF NOT EXISTS collection_videos (
|
|
collection_id TEXT NOT NULL REFERENCES collections(id) ON DELETE CASCADE,
|
|
video_id TEXT NOT NULL UNIQUE REFERENCES videos(id) ON DELETE CASCADE,
|
|
position INTEGER NOT NULL DEFAULT 0 CHECK(position>=0),
|
|
added_at TEXT NOT NULL,
|
|
PRIMARY KEY(collection_id,video_id)
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_collection_videos_order ON collection_videos(collection_id,position,added_at);
|
|
CREATE TABLE IF NOT EXISTS collection_items (
|
|
id TEXT PRIMARY KEY,
|
|
collection_id TEXT NOT NULL REFERENCES collections(id) ON DELETE CASCADE,
|
|
parent_id TEXT REFERENCES collection_items(id) ON DELETE CASCADE,
|
|
kind TEXT NOT NULL CHECK(kind IN ('group','video')),
|
|
name TEXT,
|
|
video_id TEXT UNIQUE REFERENCES videos(id) ON DELETE CASCADE,
|
|
position INTEGER NOT NULL DEFAULT 0 CHECK(position>=0),
|
|
created_at TEXT NOT NULL,
|
|
updated_at TEXT NOT NULL,
|
|
CHECK(
|
|
(kind='group' AND name IS NOT NULL AND video_id IS NULL)
|
|
OR (kind='video' AND name IS NULL AND video_id IS NOT NULL)
|
|
)
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_collection_items_parent ON collection_items(collection_id,parent_id,position,created_at);
|
|
CREATE UNIQUE INDEX IF NOT EXISTS idx_collection_items_group_root_name
|
|
ON collection_items(collection_id,lower(name))
|
|
WHERE kind='group' AND parent_id IS NULL;
|
|
CREATE UNIQUE INDEX IF NOT EXISTS idx_collection_items_group_child_name
|
|
ON collection_items(collection_id,parent_id,lower(name))
|
|
WHERE kind='group' AND parent_id IS NOT NULL;
|
|
CREATE TABLE IF NOT EXISTS tag_groups (
|
|
id TEXT PRIMARY KEY,
|
|
name TEXT NOT NULL UNIQUE,
|
|
selection_mode TEXT NOT NULL DEFAULT 'multi' CHECK(selection_mode IN ('single','multi')),
|
|
sort_order INTEGER NOT NULL DEFAULT 0,
|
|
created_at TEXT NOT NULL,
|
|
updated_at TEXT NOT NULL
|
|
);
|
|
CREATE TABLE IF NOT EXISTS tags (
|
|
id TEXT PRIMARY KEY,
|
|
group_id TEXT NOT NULL REFERENCES tag_groups(id) ON DELETE RESTRICT,
|
|
name TEXT NOT NULL,
|
|
ai_enabled INTEGER NOT NULL DEFAULT 0,
|
|
ai_method TEXT NOT NULL DEFAULT 'visual' CHECK(ai_method IN ('visual','text')),
|
|
ai_description TEXT NOT NULL DEFAULT '',
|
|
ai_threshold DOUBLE PRECISION NOT NULL DEFAULT 0.28 CHECK(ai_threshold>=-1 AND ai_threshold<=1),
|
|
match_terms_json TEXT NOT NULL DEFAULT '[]',
|
|
created_at TEXT NOT NULL,
|
|
updated_at TEXT NOT NULL,
|
|
UNIQUE(group_id,name)
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_tags_group ON tags(group_id,name);
|
|
CREATE UNIQUE INDEX IF NOT EXISTS idx_tags_group_name ON tags(group_id,lower(name));
|
|
CREATE TABLE IF NOT EXISTS collection_tags (
|
|
collection_id TEXT NOT NULL REFERENCES collections(id) ON DELETE CASCADE,
|
|
tag_id TEXT NOT NULL REFERENCES tags(id) ON DELETE CASCADE,
|
|
created_at TEXT NOT NULL,
|
|
PRIMARY KEY(collection_id,tag_id)
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_collection_tags_tag ON collection_tags(tag_id,collection_id);
|
|
CREATE TABLE IF NOT EXISTS uploads (
|
|
id TEXT PRIMARY KEY,
|
|
source_id TEXT NOT NULL REFERENCES sources(id) ON DELETE CASCADE,
|
|
relative_path TEXT NOT NULL,
|
|
filename TEXT NOT NULL,
|
|
title TEXT,
|
|
collection_id TEXT REFERENCES collections(id) ON DELETE SET NULL,
|
|
collection_parent_id TEXT REFERENCES collection_items(id) ON DELETE SET NULL,
|
|
tag_ids_json TEXT NOT NULL DEFAULT '[]',
|
|
origin TEXT NOT NULL DEFAULT 'browser',
|
|
webdav_path TEXT,
|
|
size_bytes BIGINT NOT NULL,
|
|
chunk_size INTEGER NOT NULL,
|
|
total_chunks INTEGER NOT NULL,
|
|
received_json TEXT NOT NULL DEFAULT '[]',
|
|
bytes_received BIGINT NOT NULL DEFAULT 0,
|
|
temp_path TEXT NOT NULL,
|
|
conflict TEXT NOT NULL DEFAULT 'rename' CHECK(conflict IN ('rename','replace','skip')),
|
|
status TEXT NOT NULL DEFAULT 'receiving'
|
|
CHECK(status IN ('receiving','queued','transferring','indexing','completed','failed','cancelled')),
|
|
phase TEXT NOT NULL DEFAULT 'receiving',
|
|
storage_backend TEXT NOT NULL DEFAULT 'legacy_webdav',
|
|
progress DOUBLE PRECISION NOT NULL DEFAULT 0,
|
|
message TEXT,
|
|
error TEXT,
|
|
job_id TEXT,
|
|
target_key TEXT,
|
|
failure_stage TEXT,
|
|
content_sha256 TEXT,
|
|
content_sha256_verified INTEGER NOT NULL DEFAULT 0,
|
|
deduplicated INTEGER NOT NULL DEFAULT 0,
|
|
retry_count INTEGER NOT NULL DEFAULT 0,
|
|
next_retry_at TEXT,
|
|
transferred_bytes BIGINT NOT NULL DEFAULT 0,
|
|
resume_mode TEXT,
|
|
external_task_id TEXT,
|
|
external_state INTEGER,
|
|
external_progress DOUBLE PRECISION NOT NULL DEFAULT 0,
|
|
external_status TEXT,
|
|
external_error TEXT,
|
|
external_started_at TEXT,
|
|
external_last_progress_at TEXT,
|
|
external_source_path TEXT,
|
|
external_staged_path TEXT,
|
|
external_target_path TEXT,
|
|
external_local_path TEXT,
|
|
external_size_bytes BIGINT,
|
|
external_attempts INTEGER NOT NULL DEFAULT 0,
|
|
recovery_state TEXT NOT NULL DEFAULT 'receiving',
|
|
recovery_mode TEXT,
|
|
created_at TEXT NOT NULL,
|
|
updated_at TEXT NOT NULL,
|
|
expires_at TEXT NOT NULL
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_uploads_status ON uploads(status, updated_at);
|
|
CREATE INDEX IF NOT EXISTS idx_uploads_created ON uploads(created_at DESC,id DESC);
|
|
CREATE INDEX IF NOT EXISTS idx_uploads_status_created ON uploads(status,created_at DESC,id DESC);
|
|
CREATE TABLE IF NOT EXISTS trash (
|
|
id TEXT PRIMARY KEY,
|
|
source_id TEXT NOT NULL REFERENCES sources(id) ON DELETE CASCADE,
|
|
original_key TEXT NOT NULL,
|
|
trash_key TEXT NOT NULL,
|
|
display_name TEXT NOT NULL,
|
|
size_bytes BIGINT NOT NULL DEFAULT 0,
|
|
storage_backend TEXT NOT NULL DEFAULT 'legacy_webdav',
|
|
physical_original_path TEXT,
|
|
physical_trash_path TEXT,
|
|
physical_size_bytes BIGINT,
|
|
deleted_at TEXT NOT NULL,
|
|
expires_at TEXT NOT NULL
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_trash_source ON trash(source_id, deleted_at);
|
|
CREATE TABLE IF NOT EXISTS downloads (
|
|
id TEXT PRIMARY KEY,
|
|
gid TEXT,
|
|
kind TEXT NOT NULL CHECK(kind IN ('url','magnet','torrent')),
|
|
source_uri TEXT NOT NULL,
|
|
source_id TEXT NOT NULL REFERENCES sources(id) ON DELETE CASCADE,
|
|
relative_path TEXT NOT NULL DEFAULT '',
|
|
staging_path TEXT NOT NULL,
|
|
display_name TEXT NOT NULL,
|
|
status TEXT NOT NULL DEFAULT 'queued'
|
|
CHECK(status IN ('queued','downloading','paused','transferring','completed','failed','cancelled')),
|
|
progress DOUBLE PRECISION NOT NULL DEFAULT 0,
|
|
total_bytes BIGINT NOT NULL DEFAULT 0,
|
|
completed_bytes BIGINT NOT NULL DEFAULT 0,
|
|
download_speed BIGINT NOT NULL DEFAULT 0,
|
|
files_json TEXT NOT NULL DEFAULT '[]',
|
|
error TEXT,
|
|
job_id TEXT,
|
|
created_at TEXT NOT NULL,
|
|
updated_at TEXT NOT NULL,
|
|
finished_at TEXT
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_downloads_status ON downloads(status,updated_at);
|
|
CREATE TABLE IF NOT EXISTS actors (
|
|
id TEXT PRIMARY KEY,
|
|
name TEXT NOT NULL,
|
|
aliases_json TEXT NOT NULL DEFAULT '[]',
|
|
person_id TEXT REFERENCES people(id) ON DELETE SET NULL,
|
|
created_at TEXT NOT NULL,
|
|
updated_at TEXT NOT NULL
|
|
);
|
|
CREATE UNIQUE INDEX IF NOT EXISTS idx_actors_name ON actors(lower(name));
|
|
CREATE TABLE IF NOT EXISTS video_metadata (
|
|
video_id TEXT PRIMARY KEY REFERENCES videos(id) ON DELETE CASCADE,
|
|
title TEXT,
|
|
catalog_code TEXT,
|
|
studio TEXT,
|
|
series TEXT,
|
|
release_date TEXT,
|
|
description TEXT,
|
|
updated_at TEXT NOT NULL
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_video_metadata_code ON video_metadata(catalog_code);
|
|
CREATE TABLE IF NOT EXISTS video_actors (
|
|
video_id TEXT NOT NULL REFERENCES videos(id) ON DELETE CASCADE,
|
|
actor_id TEXT NOT NULL REFERENCES actors(id) ON DELETE CASCADE,
|
|
PRIMARY KEY(video_id, actor_id)
|
|
);
|
|
CREATE TABLE IF NOT EXISTS video_tags (
|
|
video_id TEXT NOT NULL REFERENCES videos(id) ON DELETE CASCADE,
|
|
tag_id TEXT NOT NULL REFERENCES tags(id) ON DELETE CASCADE,
|
|
PRIMARY KEY(video_id, tag_id)
|
|
);
|
|
CREATE TABLE IF NOT EXISTS tag_suggestions (
|
|
id TEXT PRIMARY KEY,
|
|
video_id TEXT NOT NULL REFERENCES videos(id) ON DELETE CASCADE,
|
|
tag_id TEXT NOT NULL REFERENCES tags(id) ON DELETE CASCADE,
|
|
confidence DOUBLE PRECISION NOT NULL,
|
|
evidence_json TEXT NOT NULL DEFAULT '[]',
|
|
status TEXT NOT NULL DEFAULT 'pending' CHECK(status IN ('pending','accepted','rejected')),
|
|
video_fingerprint TEXT NOT NULL,
|
|
tag_revision TEXT NOT NULL,
|
|
created_at TEXT NOT NULL,
|
|
updated_at TEXT NOT NULL,
|
|
UNIQUE(video_id,tag_id)
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_tag_suggestions_video ON tag_suggestions(video_id,status,confidence DESC);
|
|
CREATE TABLE IF NOT EXISTS video_state (
|
|
video_id TEXT PRIMARY KEY REFERENCES videos(id) ON DELETE CASCADE,
|
|
liked INTEGER NOT NULL DEFAULT 0,
|
|
favorited INTEGER NOT NULL DEFAULT 0,
|
|
progress_ms INTEGER NOT NULL DEFAULT 0,
|
|
completed INTEGER NOT NULL DEFAULT 0,
|
|
last_played_at TEXT,
|
|
updated_at TEXT NOT NULL
|
|
);
|
|
CREATE TABLE IF NOT EXISTS video_markers (
|
|
id TEXT PRIMARY KEY,
|
|
video_id TEXT NOT NULL REFERENCES videos(id) ON DELETE CASCADE,
|
|
position_ms INTEGER NOT NULL CHECK(position_ms>=0),
|
|
title TEXT,
|
|
created_at TEXT NOT NULL,
|
|
updated_at TEXT NOT NULL,
|
|
UNIQUE(video_id,position_ms)
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_video_markers_time ON video_markers(video_id,position_ms);
|
|
CREATE TABLE IF NOT EXISTS video_tombstones (
|
|
source_id TEXT NOT NULL REFERENCES sources(id) ON DELETE CASCADE,
|
|
source_key TEXT NOT NULL,
|
|
video_id TEXT,
|
|
display_name TEXT NOT NULL,
|
|
source_deleted INTEGER NOT NULL DEFAULT 0 CHECK(source_deleted IN (0,1)),
|
|
deleted_at TEXT NOT NULL,
|
|
PRIMARY KEY(source_id,source_key)
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_video_tombstones_deleted ON video_tombstones(deleted_at);
|
|
CREATE TABLE IF NOT EXISTS webdav_staging (
|
|
virtual_path TEXT PRIMARY KEY,
|
|
temp_path TEXT NOT NULL UNIQUE,
|
|
size_bytes BIGINT NOT NULL DEFAULT 0,
|
|
expected_size BIGINT,
|
|
received_bytes BIGINT NOT NULL DEFAULT 0,
|
|
content_sha256 TEXT,
|
|
content_sha256_verified INTEGER NOT NULL DEFAULT 0,
|
|
state TEXT NOT NULL DEFAULT 'complete',
|
|
error TEXT,
|
|
collection_id TEXT REFERENCES collections(id) ON DELETE CASCADE,
|
|
collection_parent_id TEXT REFERENCES collection_items(id) ON DELETE SET NULL,
|
|
upload_id TEXT REFERENCES uploads(id) ON DELETE SET NULL,
|
|
created_at TEXT NOT NULL,
|
|
updated_at TEXT NOT NULL
|
|
);
|
|
CREATE TABLE IF NOT EXISTS ingest_guards (
|
|
source_id TEXT NOT NULL REFERENCES sources(id) ON DELETE CASCADE,
|
|
target_key TEXT NOT NULL,
|
|
upload_id TEXT NOT NULL REFERENCES uploads(id) ON DELETE CASCADE,
|
|
created_at TEXT NOT NULL,
|
|
PRIMARY KEY(source_id,target_key)
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_ingest_guards_upload ON ingest_guards(upload_id);
|
|
CREATE TABLE IF NOT EXISTS webdav_upload_receipts (
|
|
source_id TEXT NOT NULL REFERENCES sources(id) ON DELETE CASCADE,
|
|
virtual_path TEXT NOT NULL,
|
|
content_sha256 TEXT NOT NULL,
|
|
upload_id TEXT REFERENCES uploads(id) ON DELETE SET NULL,
|
|
size_bytes BIGINT NOT NULL DEFAULT 0,
|
|
created_at TEXT NOT NULL,
|
|
updated_at TEXT NOT NULL,
|
|
PRIMARY KEY(source_id,virtual_path,content_sha256)
|
|
);
|
|
"""
|