nanxun/imagefind
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: preserve upload recovery and harden speech indexing

Browse Source
This commit is contained in:
nanxun
2026-08-12 19:49:25 +08:00
parent 10f2c078ec
commit a0a185b96c
16 changed files with 13234 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/imagefind/api.py
+131 -1
View File
@@ -297,6 +297,11 @@ class UploadUpdateBody(BaseModel):
tag_ids: list[str] | None = Field(default=None, max_length=500)
class UploadBulkActionBody(BaseModel):
upload_ids: list[str] = Field(min_length=1, max_length=200)
action: Literal["cancel", "retry"]
class DownloadCreateBody(BaseModel):
url: str = Field(min_length=1, max_length=8192)
source_id: str
@@ -1896,6 +1901,34 @@ async def events(
async def search(
body: SearchBody, app: Annotated[Services, Depends(services)], _: Annotated[dict, Depends(require_auth)]
):
rebuild = app.db.setting("audio_rebuild", {})
if isinstance(rebuild, dict) and rebuild.get("status") in {"running", "failed"}:
if rebuild.get("status") == "running":
with app.db.read() as conn:
remaining = int(conn.execute(
"SELECT count(*) FROM videos WHERE available=1 AND audio_index_revision<?",
(SPEECH_INDEX_REVISION,),
).fetchone()[0])
active = int(conn.execute(
"SELECT count(*) FROM jobs WHERE kind='transcribe_audio' AND status IN ('queued','running')"
).fetchone()[0])
failed = int(conn.execute(
"SELECT count(*) FROM jobs WHERE kind='transcribe_audio' AND status='failed' AND finished_at>=?",
(rebuild.get("started_at") or "",),
).fetchone()[0])
if active == 0:
status = "completed" if remaining == 0 else "failed"
rebuild = {**rebuild, "status": status, "remaining": remaining, "failed_jobs": failed, "finished_at": utcnow()}
app.db.set_setting("audio_rebuild", rebuild)
if rebuild.get("status") == "completed":
rebuild = {}
else:
requested = set(body.recognition_types or [])
if "audio" in requested:
message = "音频索引全库重建存在失败任务，修复前音频搜索保持暂停" if rebuild.get("status") == "failed" else "音频索引正在全库重建，音频搜索暂时不可用"
raise HTTPException(409, message)
if body.recognition_types is None:
body.recognition_types = ["visual", "ocr", "person", "subtitle", "metadata"]
image_path = None
if body.image_id:
if not app.embeddings.status()["visual_ready"]:
@@ -2091,6 +2124,61 @@ def reconcile_speech_quality(
return {**result, "queued": len(queued)}
@router.post("/v1/speech/rebuild", status_code=202)
def rebuild_all_speech(
app: Annotated[Services, Depends(services)],
_: Annotated[dict, Depends(require_auth)],
):
if not app.models.runnable_component_versions().get("audio"):
raise HTTPException(409, "Small 音频模型尚未安装")
if app.settings.audio_model_variant != "small":
try:
app.models.set_audio_variant("small")
except (ValueError, ModelUnavailable, SpeechStageError) as exc:
raise HTTPException(409, str(exc)) from exc
cancelled = app.jobs.cancel_by_kind("transcribe_audio")
with app.db.transaction() as conn:
total = int(conn.execute("SELECT count(*) FROM videos WHERE available=1").fetchone()[0])
conn.execute("UPDATE videos SET audio_model_version=NULL,audio_index_revision=0 WHERE available=1")
state = {
"status": "running",
"model_variant": "small",
"index_revision": SPEECH_INDEX_REVISION,
"total": total,
"started_at": utcnow(),
}
app.db.set_setting("audio_rebuild", state)
queued = app.reconcile_ai()
return {**state, "cancelled": len(cancelled), "queued": len(queued)}
@router.get("/v1/speech/rebuild")
def speech_rebuild_status(
app: Annotated[Services, Depends(services)],
_: Annotated[dict, Depends(require_auth)],
):
state = app.db.setting("audio_rebuild", {"status": "idle"})
if not isinstance(state, dict) or state.get("status") != "running":
return state
with app.db.read() as conn:
completed = int(conn.execute(
"SELECT count(*) FROM videos WHERE available=1 AND audio_index_revision>=?",
(SPEECH_INDEX_REVISION,),
).fetchone()[0])
remaining = int(conn.execute(
"SELECT count(*) FROM videos WHERE available=1 AND audio_index_revision<?",
(SPEECH_INDEX_REVISION,),
).fetchone()[0])
active = int(conn.execute(
"SELECT count(*) FROM jobs WHERE kind='transcribe_audio' AND status IN ('queued','running')"
).fetchone()[0])
state = {**state, "completed": completed, "remaining": remaining, "active_jobs": active}
if remaining == 0 and active == 0:
state = {**state, "status": "completed", "finished_at": utcnow()}
app.db.set_setting("audio_rebuild", state)
return state
@router.get("/v1/videos/{video_id}/transcript")
def video_transcript(
video_id: str,
@@ -2556,10 +2644,19 @@ async def list_uploads(
limit: int = 100,
page: int | None = None,
page_size: int = 10,
status: str = "all",
source_id: str | None = None,
query: str = "",
):
if page is not None:
return await _background_api_with_fallback(
app.uploads.paginate, app.uploads.cached_paginate, page, page_size
app.uploads.paginate,
app.uploads.cached_paginate,
page,
page_size,
status,
source_id,
query,
)
return await _background_api_with_fallback(
app.uploads.list, app.uploads.cached_list, limit
@@ -2660,6 +2757,39 @@ def cancel_upload(
raise HTTPException(409, str(exc)) from exc
@router.delete("/v1/uploads/{upload_id}/recovery", status_code=204)
def discard_upload_recovery(
upload_id: str,
app: Annotated[Services, Depends(services)],
_: Annotated[dict, Depends(require_auth)],
):
try:
app.uploads.discard(upload_id)
except KeyError as exc:
raise HTTPException(404, "上传任务不存在") from exc
except ValueError as exc:
raise HTTPException(409, str(exc)) from exc
@router.post("/v1/uploads/bulk-actions")
def bulk_upload_actions(
body: UploadBulkActionBody,
app: Annotated[Services, Depends(services)],
_: Annotated[dict, Depends(require_auth)],
):
results = []
for upload_id in dict.fromkeys(body.upload_ids):
try:
if body.action == "cancel":
app.uploads.cancel(upload_id)
else:
app.uploads.retry(upload_id)
results.append({"id": upload_id, "ok": True})
except (KeyError, ValueError) as exc:
results.append({"id": upload_id, "ok": False, "error": str(exc) or "上传任务不存在"})
return {"results": results}
@router.get("/v1/downloads/runtime")
def download_runtime(
app: Annotated[Services, Depends(services)],
backend/imagefind/database.py
+6
View File
@@ -779,6 +779,10 @@ class Database:
conn.execute("ALTER TABLE uploads ADD COLUMN IF NOT EXISTS external_local_path TEXT")
conn.execute("ALTER TABLE uploads ADD COLUMN IF NOT EXISTS external_size_bytes BIGINT")
conn.execute("ALTER TABLE uploads ADD COLUMN IF NOT EXISTS external_attempts INTEGER NOT NULL DEFAULT 0")
conn.execute(
"ALTER TABLE uploads ADD COLUMN IF NOT EXISTS recovery_state TEXT NOT NULL DEFAULT 'receiving'"
)
conn.execute("ALTER TABLE uploads ADD COLUMN IF NOT EXISTS recovery_mode TEXT")
conn.execute(
"UPDATE videos v SET storage_backend=u.storage_backend,physical_path=u.external_target_path,"
"physical_size_bytes=coalesce(u.external_size_bytes,v.size_bytes) FROM ("
@@ -1291,6 +1295,8 @@ CREATE TABLE IF NOT EXISTS uploads (
external_local_path TEXT,
external_size_bytes BIGINT,
external_attempts INTEGER NOT NULL DEFAULT 0,
recovery_state TEXT NOT NULL DEFAULT 'receiving',
recovery_mode TEXT,
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL,
expires_at TEXT NOT NULL
backend/imagefind/openlist_native.py
+5 -7
View File
@@ -379,7 +379,6 @@ class OpenListNativeService:
if config.encrypted:
encrypted_root = job_root / "encrypted"
encrypted_file = self.sources.rclone.encrypt_to_local(source, input_path, target_key, encrypted_root)
input_path.unlink(missing_ok=True)
physical_relative = encrypted_file.relative_to(encrypted_root).as_posix()
local_file = encrypted_file
source_path = _remote_join(config.source_path, upload_id, "encrypted", physical_relative)
@@ -389,7 +388,7 @@ class OpenListNativeService:
direct_root.mkdir(parents=True, exist_ok=True)
local_file = direct_root / PurePosixPath(target_key).name
if input_path.resolve() != local_file.resolve():
os.replace(input_path, local_file)
shutil.copy2(input_path, local_file)
physical_relative = target_key
source_path = _remote_join(config.source_path, upload_id, "direct", local_file.name)
target_path = _remote_join(config.target_path, target_key)
@@ -648,8 +647,7 @@ class OpenListNativeService:
shutil.rmtree(job_root, ignore_errors=True)
def cancel(self, source_id: str, upload_id: str, task_id: str | None, staged_path: str | None) -> None:
try:
if task_id:
self.client(source_id).cancel_copy_task(task_id)
finally:
self.cleanup(source_id, upload_id, staged_path)
# Cancellation stops work but preserves recovery copies until an
# administrator explicitly discards the task and its data.
if task_id:
self.client(source_id).cancel_copy_task(task_id)
backend/imagefind/scanner.py
+22 -1
View File
@@ -151,7 +151,7 @@ class Scanner:
with self.db.read() as conn:
upload_row = conn.execute(
"SELECT title,collection_id,collection_parent_id,tag_ids_json,content_sha256,deduplicated,"
"storage_backend,external_target_path,external_size_bytes,size_bytes "
"storage_backend,external_staged_path,external_target_path,external_size_bytes,size_bytes "
"FROM uploads WHERE id=?",
(upload_id,),
).fetchone()
@@ -307,6 +307,27 @@ class Scanner:
)
conn.execute("DELETE FROM ingest_guards WHERE upload_id=?", (upload_id,))
self.jobs.update(job_id, 1, "文件已加入媒体库，视频资料解析已排队")
if upload_id and upload_row:
upload_data = dict(upload_row)
try:
if upload_data.get("storage_backend") == "openlist_native" and self.openlist_native:
self.openlist_native.cleanup(
source_id, upload_id, upload_data.get("external_staged_path")
)
with self.db.transaction() as conn:
row = conn.execute("SELECT temp_path FROM uploads WHERE id=?", (upload_id,)).fetchone()
if row:
from pathlib import Path
Path(row["temp_path"]).unlink(missing_ok=True)
conn.execute(
"UPDATE uploads SET recovery_state='released',recovery_mode=NULL,updated_at=? WHERE id=?",
(utcnow(), upload_id),
)
except OSError:
# Cataloging is already committed. Keep the recovery state
# visible so an administrator can explicitly discard it.
pass
except Exception as exc:
if upload_id:
with self.db.transaction() as conn:
backend/imagefind/speech_quality.py
+13 -1
View File
@@ -89,9 +89,21 @@ def aggregate_transcript_quality(
if short_ratio >= 0.65 and len(texts) >= 10:
flags.append("whole_short_segment_dominance")
penalty += 0.35
if any(text in _ENDING_HALLUCINATIONS for text, _count in repeated):
ending_counts = tuple(
sorted((text, count) for text, count in counts.items() if text in _ENDING_HALLUCINATIONS)
)
weak_speech_ending = any(
_phrase_key(item.get("text")) in _ENDING_HALLUCINATIONS
and item.get("speech_ratio") is not None
and float(item.get("speech_ratio") or 0) < 0.20
for item in segments
)
if any(count >= 2 for _text, count in ending_counts) or weak_speech_ending:
flags.append("whole_ending_hallucination")
penalty += 0.45
reported = dict(repeated)
reported.update(dict(ending_counts))
repeated = tuple(sorted(reported.items(), key=lambda item: (-item[1], item[0])))[:12]
flags.extend(f"repeated_phrase:{text}:{count}" for text, count in repeated)
score = max(0.0, min(1.0, float(1.0 if base_score is None else base_score) - penalty))
return AggregateTranscriptQuality(round(score, 3), tuple(dict.fromkeys(flags)), repeated, round(short_ratio, 3))
backend/imagefind/uploads.py
+98 -18
View File
@@ -58,7 +58,7 @@ class UploadService:
self._progress_state: dict[str, tuple[float, int]] = {}
self._list_cache_lock = threading.Lock()
self._list_cache: list[dict] = []
self._page_cache: dict[tuple[int, int], dict] = {}
self._page_cache: dict[tuple[int, int, str, str, str], dict] = {}
self.storage_usage = None
def _lock(self, upload_id: str) -> threading.Lock:
@@ -82,6 +82,18 @@ class UploadService:
@staticmethod
def _public(item: dict) -> dict:
result = dict(item)
status = result["status"]
recovery_state = str(result.get("recovery_state") or "")
if status in {"failed", "cancelled"} and recovery_state in {"", "receiving"}:
original = Path(str(result.get("temp_path") or "")).is_file()
encrypted = bool(
result.get("external_local_path")
and Path(str(result["external_local_path"])).is_file()
)
catalog = bool(result.get("failure_stage") == "catalog" and result.get("target_key"))
recovery_state = "available" if original or encrypted else "catalog" if catalog else "missing"
result["recovery_state"] = recovery_state
result["recovery_mode"] = "original" if original else "encrypted" if encrypted else "catalog" if catalog else None
result["deduplicated"] = bool(result.get("deduplicated", False))
result.pop("temp_path", None)
result.pop("target_key", None)
@@ -95,7 +107,6 @@ class UploadService:
result.pop("external_error", None)
external_task_id = result.pop("external_task_id", None)
result["external_task_ref"] = external_task_id[-8:] if external_task_id else None
status = result["status"]
failure_stage = result.get("failure_stage")
phase = result.get("phase") or status
if phase == "indexing":
@@ -160,8 +171,11 @@ class UploadService:
"label": "加入媒体库并排队 AI",
},
]
recovery_state = str(result.get("recovery_state") or "")
result["can_cancel"] = status in {"receiving", "queued", "transferring"} and failure_stage != "commit"
result["can_retry"] = status == "failed"
result["can_retry"] = status == "failed" and recovery_state in {"available", "catalog"}
result["can_discard"] = status in {"failed", "cancelled", "completed"}
result["requires_reupload"] = status == "failed" and recovery_state == "missing"
return result
def list(self, limit: int = 100) -> list[dict]:
@@ -193,23 +207,51 @@ class UploadService:
"status_items": [dict(item) for item in value.get("status_items", [])],
}
def paginate(self, page: int = 1, page_size: int = 10) -> dict:
def paginate(
self,
page: int = 1,
page_size: int = 10,
status: str = "all",
source_id: str | None = None,
query: str = "",
) -> dict:
"""Return one history page plus the small live-state set used by global UI."""
page = max(1, page)
page_size = min(max(page_size, 1), 50)
status = status if status in {"all", "active", "failed", "completed", "cancelled"} else "all"
clauses: list[str] = []
params: list[object] = []
status_values = {
"active": ("receiving", "queued", "transferring", "indexing"),
"failed": ("failed",),
"completed": ("completed",),
"cancelled": ("cancelled",),
}.get(status)
if status_values:
clauses.append("status IN (" + ",".join("?" for _ in status_values) + ")")
params.extend(status_values)
if source_id:
clauses.append("source_id=?")
params.append(source_id)
query = query.strip()[:200]
if query:
clauses.append("(filename ILIKE ? OR coalesce(title,'') ILIKE ?)")
params.extend((f"%{query}%", f"%{query}%"))
where = " WHERE " + " AND ".join(clauses) if clauses else ""
with self.db.read() as conn:
counts = conn.execute(
"SELECT count(*) AS total,"
"count(*) FILTER (WHERE status IN ('receiving','queued','transferring','indexing')) AS active_count,"
"count(*) FILTER (WHERE status='failed') AS failed_count FROM uploads"
"count(*) FILTER (WHERE status='failed') AS failed_count FROM uploads" + where,
tuple(params),
).fetchone()
total = int(counts["total"])
pages = max(1, (total + page_size - 1) // page_size)
page = min(page, pages)
rows = conn.execute(
"SELECT * FROM uploads ORDER BY created_at DESC,id DESC LIMIT ? OFFSET ?",
(page_size, (page - 1) * page_size),
"SELECT * FROM uploads" + where + " ORDER BY created_at DESC,id DESC LIMIT ? OFFSET ?",
(*params, page_size, (page - 1) * page_size),
).fetchall()
status_rows = conn.execute(
"SELECT * FROM uploads "
@@ -229,20 +271,27 @@ class UploadService:
"pages": pages,
}
with self._list_cache_lock:
cache_key = (page, page_size)
cache_key = (page, page_size, status, source_id or "", query)
self._page_cache.pop(cache_key, None)
self._page_cache[cache_key] = self._copy_page(result)
while len(self._page_cache) > 16:
self._page_cache.pop(next(iter(self._page_cache)))
return result
def cached_paginate(self, page: int = 1, page_size: int = 10) -> dict:
def cached_paginate(
self,
page: int = 1,
page_size: int = 10,
status: str = "all",
source_id: str | None = None,
query: str = "",
) -> dict:
"""Return the last matching page snapshot during database pressure."""
page = max(1, page)
page_size = min(max(page_size, 1), 50)
with self._list_cache_lock:
cached = self._page_cache.get((page, page_size))
cached = self._page_cache.get((page, page_size, status, source_id or "", query.strip()[:200]))
if cached is not None:
return self._copy_page(cached)
return {
@@ -631,13 +680,47 @@ class UploadService:
"UPDATE uploads SET status='cancelled',phase='cancelled',message='已取消',updated_at=? WHERE id=?",
(utcnow(), upload_id),
)
if upload["status"] in {"receiving", "queued", "failed"}:
Path(upload["temp_path"]).unlink(missing_ok=True)
self._refresh_recovery_state(upload_id)
def _refresh_recovery_state(self, upload_id: str) -> str:
upload = self._get(upload_id)
original = Path(upload["temp_path"]).is_file()
encrypted = bool(
upload.get("external_local_path") and Path(str(upload["external_local_path"])).is_file()
)
catalog = bool(upload.get("failure_stage") == "catalog" and upload.get("target_key"))
state = "available" if original or encrypted else "catalog" if catalog else "missing"
mode = "original" if original else "encrypted" if encrypted else "catalog" if catalog else None
self.db.write_with_retry(
lambda conn: conn.execute(
"UPDATE uploads SET recovery_state=?,recovery_mode=?,updated_at=? WHERE id=?",
(state, mode, utcnow(), upload_id),
)
)
return state
def discard(self, upload_id: str) -> None:
"""Explicitly delete a terminal task and all retained recovery copies."""
with self._lock(upload_id):
upload = self._get(upload_id)
if upload["status"] not in {"failed", "cancelled", "completed"}:
raise ValueError("任务仍在执行，请先取消")
if upload.get("storage_backend") == "openlist_native" and self.openlist_native is not None:
self.openlist_native.cleanup(
upload["source_id"], upload_id, upload.get("external_staged_path")
)
Path(upload["temp_path"]).unlink(missing_ok=True)
with self.db.transaction() as conn:
conn.execute("DELETE FROM ingest_guards WHERE upload_id=?", (upload_id,))
conn.execute("DELETE FROM uploads WHERE id=?", (upload_id,))
def retry(self, upload_id: str) -> str:
upload = self._get(upload_id)
if upload["status"] != "failed":
raise ValueError("该上传任务不能重试")
if self._refresh_recovery_state(upload_id) == "missing":
raise ValueError("恢复副本不存在，请重新上传源文件")
upload = self._get(upload_id)
with self.db.transaction() as conn:
if upload.get("failure_stage") == "catalog" and upload.get("target_key"):
job_id = self.jobs.enqueue(
@@ -1108,9 +1191,6 @@ class UploadService:
# OpenList move succeeded but before refresh_path was queued.
self._before_commit(upload_id, str(target_key))
self._queue_refresh(upload, upload_id, str(target_key), deduplicated=False)
self.openlist_native.cleanup(
upload["source_id"], upload_id, upload.get("external_staged_path")
)
return
if not staged_ready:
raise RuntimeError("OpenList 暂存目标不存在或长度不一致")
@@ -1123,7 +1203,6 @@ class UploadService:
external_size,
)
self._queue_refresh(upload, upload_id, str(target_key), deduplicated=False)
self.openlist_native.cleanup(upload["source_id"], upload_id, upload.get("external_staged_path"))
except JobRetry:
raise
except (JobCancelled, TransferCancelled):
@@ -1304,6 +1383,7 @@ class UploadService:
)
self.db.write_with_retry(fail)
self._refresh_recovery_state(upload_id)
raise
finally:
with self._progress_guard:
@@ -1469,14 +1549,14 @@ class UploadService:
with self.db.read() as conn:
rows = conn.execute(
"SELECT id,temp_path FROM uploads WHERE expires_at<? "
"AND status IN ('receiving','failed','cancelled','completed')",
"AND status='completed' AND recovery_state='released'",
(now,),
).fetchall()
for row in rows:
Path(row["temp_path"]).unlink(missing_ok=True)
with self.db.transaction() as conn:
removed = conn.execute(
"DELETE FROM uploads WHERE expires_at<? AND status IN ('receiving','failed','cancelled','completed')",
"DELETE FROM uploads WHERE expires_at<? AND status='completed' AND recovery_state='released'",
(now,),
).rowcount
cutoff = (datetime.now(UTC) - timedelta(days=self.settings.upload_incomplete_days)).isoformat()