diff --git a/backend/imagefind/api.py b/backend/imagefind/api.py index 726081a..5afbad6 100644 --- a/backend/imagefind/api.py +++ b/backend/imagefind/api.py @@ -297,6 +297,11 @@ class UploadUpdateBody(BaseModel): tag_ids: list[str] | None = Field(default=None, max_length=500) +class UploadBulkActionBody(BaseModel): + upload_ids: list[str] = Field(min_length=1, max_length=200) + action: Literal["cancel", "retry"] + + class DownloadCreateBody(BaseModel): url: str = Field(min_length=1, max_length=8192) source_id: str @@ -1896,6 +1901,34 @@ async def events( async def search( body: SearchBody, app: Annotated[Services, Depends(services)], _: Annotated[dict, Depends(require_auth)] ): + rebuild = app.db.setting("audio_rebuild", {}) + if isinstance(rebuild, dict) and rebuild.get("status") in {"running", "failed"}: + if rebuild.get("status") == "running": + with app.db.read() as conn: + remaining = int(conn.execute( + "SELECT count(*) FROM videos WHERE available=1 AND audio_index_revision=?", + (rebuild.get("started_at") or "",), + ).fetchone()[0]) + if active == 0: + status = "completed" if remaining == 0 else "failed" + rebuild = {**rebuild, "status": status, "remaining": remaining, "failed_jobs": failed, "finished_at": utcnow()} + app.db.set_setting("audio_rebuild", rebuild) + if rebuild.get("status") == "completed": + rebuild = {} + else: + requested = set(body.recognition_types or []) + if "audio" in requested: + message = "音频索引全库重建存在失败任务，修复前音频搜索保持暂停" if rebuild.get("status") == "failed" else "音频索引正在全库重建，音频搜索暂时不可用" + raise HTTPException(409, message) + if body.recognition_types is None: + body.recognition_types = ["visual", "ocr", "person", "subtitle", "metadata"] image_path = None if body.image_id: if not app.embeddings.status()["visual_ready"]: @@ -2091,6 +2124,61 @@ def reconcile_speech_quality( return {**result, "queued": len(queued)} +@router.post("/v1/speech/rebuild", status_code=202) +def rebuild_all_speech( + app: Annotated[Services, Depends(services)], + _: Annotated[dict, Depends(require_auth)], +): + if not app.models.runnable_component_versions().get("audio"): + raise HTTPException(409, "Small 音频模型尚未安装") + if app.settings.audio_model_variant != "small": + try: + app.models.set_audio_variant("small") + except (ValueError, ModelUnavailable, SpeechStageError) as exc: + raise HTTPException(409, str(exc)) from exc + cancelled = app.jobs.cancel_by_kind("transcribe_audio") + with app.db.transaction() as conn: + total = int(conn.execute("SELECT count(*) FROM videos WHERE available=1").fetchone()[0]) + conn.execute("UPDATE videos SET audio_model_version=NULL,audio_index_revision=0 WHERE available=1") + state = { + "status": "running", + "model_variant": "small", + "index_revision": SPEECH_INDEX_REVISION, + "total": total, + "started_at": utcnow(), + } + app.db.set_setting("audio_rebuild", state) + queued = app.reconcile_ai() + return {**state, "cancelled": len(cancelled), "queued": len(queued)} + + +@router.get("/v1/speech/rebuild") +def speech_rebuild_status( + app: Annotated[Services, Depends(services)], + _: Annotated[dict, Depends(require_auth)], +): + state = app.db.setting("audio_rebuild", {"status": "idle"}) + if not isinstance(state, dict) or state.get("status") != "running": + return state + with app.db.read() as conn: + completed = int(conn.execute( + "SELECT count(*) FROM videos WHERE available=1 AND audio_index_revision>=?", + (SPEECH_INDEX_REVISION,), + ).fetchone()[0]) + remaining = int(conn.execute( + "SELECT count(*) FROM videos WHERE available=1 AND audio_index_revision None: - try: - if task_id: - self.client(source_id).cancel_copy_task(task_id) - finally: - self.cleanup(source_id, upload_id, staged_path) + # Cancellation stops work but preserves recovery copies until an + # administrator explicitly discards the task and its data. + if task_id: + self.client(source_id).cancel_copy_task(task_id) diff --git a/backend/imagefind/scanner.py b/backend/imagefind/scanner.py index 46260a3..7d16a49 100644 --- a/backend/imagefind/scanner.py +++ b/backend/imagefind/scanner.py @@ -151,7 +151,7 @@ class Scanner: with self.db.read() as conn: upload_row = conn.execute( "SELECT title,collection_id,collection_parent_id,tag_ids_json,content_sha256,deduplicated," - "storage_backend,external_target_path,external_size_bytes,size_bytes " + "storage_backend,external_staged_path,external_target_path,external_size_bytes,size_bytes " "FROM uploads WHERE id=?", (upload_id,), ).fetchone() @@ -307,6 +307,27 @@ class Scanner: ) conn.execute("DELETE FROM ingest_guards WHERE upload_id=?", (upload_id,)) self.jobs.update(job_id, 1, "文件已加入媒体库，视频资料解析已排队") + if upload_id and upload_row: + upload_data = dict(upload_row) + try: + if upload_data.get("storage_backend") == "openlist_native" and self.openlist_native: + self.openlist_native.cleanup( + source_id, upload_id, upload_data.get("external_staged_path") + ) + with self.db.transaction() as conn: + row = conn.execute("SELECT temp_path FROM uploads WHERE id=?", (upload_id,)).fetchone() + if row: + from pathlib import Path + + Path(row["temp_path"]).unlink(missing_ok=True) + conn.execute( + "UPDATE uploads SET recovery_state='released',recovery_mode=NULL,updated_at=? WHERE id=?", + (utcnow(), upload_id), + ) + except OSError: + # Cataloging is already committed. Keep the recovery state + # visible so an administrator can explicitly discard it. + pass except Exception as exc: if upload_id: with self.db.transaction() as conn: diff --git a/backend/imagefind/speech_quality.py b/backend/imagefind/speech_quality.py index f0706a5..dd05b86 100644 --- a/backend/imagefind/speech_quality.py +++ b/backend/imagefind/speech_quality.py @@ -89,9 +89,21 @@ def aggregate_transcript_quality( if short_ratio >= 0.65 and len(texts) >= 10: flags.append("whole_short_segment_dominance") penalty += 0.35 - if any(text in _ENDING_HALLUCINATIONS for text, _count in repeated): + ending_counts = tuple( + sorted((text, count) for text, count in counts.items() if text in _ENDING_HALLUCINATIONS) + ) + weak_speech_ending = any( + _phrase_key(item.get("text")) in _ENDING_HALLUCINATIONS + and item.get("speech_ratio") is not None + and float(item.get("speech_ratio") or 0) < 0.20 + for item in segments + ) + if any(count >= 2 for _text, count in ending_counts) or weak_speech_ending: flags.append("whole_ending_hallucination") penalty += 0.45 + reported = dict(repeated) + reported.update(dict(ending_counts)) + repeated = tuple(sorted(reported.items(), key=lambda item: (-item[1], item[0])))[:12] flags.extend(f"repeated_phrase:{text}:{count}" for text, count in repeated) score = max(0.0, min(1.0, float(1.0 if base_score is None else base_score) - penalty)) return AggregateTranscriptQuality(round(score, 3), tuple(dict.fromkeys(flags)), repeated, round(short_ratio, 3)) diff --git a/backend/imagefind/uploads.py b/backend/imagefind/uploads.py index 6f2c622..2bf688d 100644 --- a/backend/imagefind/uploads.py +++ b/backend/imagefind/uploads.py @@ -58,7 +58,7 @@ class UploadService: self._progress_state: dict[str, tuple[float, int]] = {} self._list_cache_lock = threading.Lock() self._list_cache: list[dict] = [] - self._page_cache: dict[tuple[int, int], dict] = {} + self._page_cache: dict[tuple[int, int, str, str, str], dict] = {} self.storage_usage = None def _lock(self, upload_id: str) -> threading.Lock: @@ -82,6 +82,18 @@ class UploadService: @staticmethod def _public(item: dict) -> dict: result = dict(item) + status = result["status"] + recovery_state = str(result.get("recovery_state") or "") + if status in {"failed", "cancelled"} and recovery_state in {"", "receiving"}: + original = Path(str(result.get("temp_path") or "")).is_file() + encrypted = bool( + result.get("external_local_path") + and Path(str(result["external_local_path"])).is_file() + ) + catalog = bool(result.get("failure_stage") == "catalog" and result.get("target_key")) + recovery_state = "available" if original or encrypted else "catalog" if catalog else "missing" + result["recovery_state"] = recovery_state + result["recovery_mode"] = "original" if original else "encrypted" if encrypted else "catalog" if catalog else None result["deduplicated"] = bool(result.get("deduplicated", False)) result.pop("temp_path", None) result.pop("target_key", None) @@ -95,7 +107,6 @@ class UploadService: result.pop("external_error", None) external_task_id = result.pop("external_task_id", None) result["external_task_ref"] = external_task_id[-8:] if external_task_id else None - status = result["status"] failure_stage = result.get("failure_stage") phase = result.get("phase") or status if phase == "indexing": @@ -160,8 +171,11 @@ class UploadService: "label": "加入媒体库并排队 AI", }, ] + recovery_state = str(result.get("recovery_state") or "") result["can_cancel"] = status in {"receiving", "queued", "transferring"} and failure_stage != "commit" - result["can_retry"] = status == "failed" + result["can_retry"] = status == "failed" and recovery_state in {"available", "catalog"} + result["can_discard"] = status in {"failed", "cancelled", "completed"} + result["requires_reupload"] = status == "failed" and recovery_state == "missing" return result def list(self, limit: int = 100) -> list[dict]: @@ -193,23 +207,51 @@ class UploadService: "status_items": [dict(item) for item in value.get("status_items", [])], } - def paginate(self, page: int = 1, page_size: int = 10) -> dict: + def paginate( + self, + page: int = 1, + page_size: int = 10, + status: str = "all", + source_id: str | None = None, + query: str = "", + ) -> dict: """Return one history page plus the small live-state set used by global UI.""" page = max(1, page) page_size = min(max(page_size, 1), 50) + status = status if status in {"all", "active", "failed", "completed", "cancelled"} else "all" + clauses: list[str] = [] + params: list[object] = [] + status_values = { + "active": ("receiving", "queued", "transferring", "indexing"), + "failed": ("failed",), + "completed": ("completed",), + "cancelled": ("cancelled",), + }.get(status) + if status_values: + clauses.append("status IN (" + ",".join("?" for _ in status_values) + ")") + params.extend(status_values) + if source_id: + clauses.append("source_id=?") + params.append(source_id) + query = query.strip()[:200] + if query: + clauses.append("(filename ILIKE ? OR coalesce(title,'') ILIKE ?)") + params.extend((f"%{query}%", f"%{query}%")) + where = " WHERE " + " AND ".join(clauses) if clauses else "" with self.db.read() as conn: counts = conn.execute( "SELECT count(*) AS total," "count(*) FILTER (WHERE status IN ('receiving','queued','transferring','indexing')) AS active_count," - "count(*) FILTER (WHERE status='failed') AS failed_count FROM uploads" + "count(*) FILTER (WHERE status='failed') AS failed_count FROM uploads" + where, + tuple(params), ).fetchone() total = int(counts["total"]) pages = max(1, (total + page_size - 1) // page_size) page = min(page, pages) rows = conn.execute( - "SELECT * FROM uploads ORDER BY created_at DESC,id DESC LIMIT ? OFFSET ?", - (page_size, (page - 1) * page_size), + "SELECT * FROM uploads" + where + " ORDER BY created_at DESC,id DESC LIMIT ? OFFSET ?", + (*params, page_size, (page - 1) * page_size), ).fetchall() status_rows = conn.execute( "SELECT * FROM uploads " @@ -229,20 +271,27 @@ class UploadService: "pages": pages, } with self._list_cache_lock: - cache_key = (page, page_size) + cache_key = (page, page_size, status, source_id or "", query) self._page_cache.pop(cache_key, None) self._page_cache[cache_key] = self._copy_page(result) while len(self._page_cache) > 16: self._page_cache.pop(next(iter(self._page_cache))) return result - def cached_paginate(self, page: int = 1, page_size: int = 10) -> dict: + def cached_paginate( + self, + page: int = 1, + page_size: int = 10, + status: str = "all", + source_id: str | None = None, + query: str = "", + ) -> dict: """Return the last matching page snapshot during database pressure.""" page = max(1, page) page_size = min(max(page_size, 1), 50) with self._list_cache_lock: - cached = self._page_cache.get((page, page_size)) + cached = self._page_cache.get((page, page_size, status, source_id or "", query.strip()[:200])) if cached is not None: return self._copy_page(cached) return { @@ -631,13 +680,47 @@ class UploadService: "UPDATE uploads SET status='cancelled',phase='cancelled',message='已取消',updated_at=? WHERE id=?", (utcnow(), upload_id), ) - if upload["status"] in {"receiving", "queued", "failed"}: - Path(upload["temp_path"]).unlink(missing_ok=True) + self._refresh_recovery_state(upload_id) + + def _refresh_recovery_state(self, upload_id: str) -> str: + upload = self._get(upload_id) + original = Path(upload["temp_path"]).is_file() + encrypted = bool( + upload.get("external_local_path") and Path(str(upload["external_local_path"])).is_file() + ) + catalog = bool(upload.get("failure_stage") == "catalog" and upload.get("target_key")) + state = "available" if original or encrypted else "catalog" if catalog else "missing" + mode = "original" if original else "encrypted" if encrypted else "catalog" if catalog else None + self.db.write_with_retry( + lambda conn: conn.execute( + "UPDATE uploads SET recovery_state=?,recovery_mode=?,updated_at=? WHERE id=?", + (state, mode, utcnow(), upload_id), + ) + ) + return state + + def discard(self, upload_id: str) -> None: + """Explicitly delete a terminal task and all retained recovery copies.""" + with self._lock(upload_id): + upload = self._get(upload_id) + if upload["status"] not in {"failed", "cancelled", "completed"}: + raise ValueError("任务仍在执行，请先取消") + if upload.get("storage_backend") == "openlist_native" and self.openlist_native is not None: + self.openlist_native.cleanup( + upload["source_id"], upload_id, upload.get("external_staged_path") + ) + Path(upload["temp_path"]).unlink(missing_ok=True) + with self.db.transaction() as conn: + conn.execute("DELETE FROM ingest_guards WHERE upload_id=?", (upload_id,)) + conn.execute("DELETE FROM uploads WHERE id=?", (upload_id,)) def retry(self, upload_id: str) -> str: upload = self._get(upload_id) if upload["status"] != "failed": raise ValueError("该上传任务不能重试") + if self._refresh_recovery_state(upload_id) == "missing": + raise ValueError("恢复副本不存在，请重新上传源文件") + upload = self._get(upload_id) with self.db.transaction() as conn: if upload.get("failure_stage") == "catalog" and upload.get("target_key"): job_id = self.jobs.enqueue( @@ -1108,9 +1191,6 @@ class UploadService: # OpenList move succeeded but before refresh_path was queued. self._before_commit(upload_id, str(target_key)) self._queue_refresh(upload, upload_id, str(target_key), deduplicated=False) - self.openlist_native.cleanup( - upload["source_id"], upload_id, upload.get("external_staged_path") - ) return if not staged_ready: raise RuntimeError("OpenList 暂存目标不存在或长度不一致") @@ -1123,7 +1203,6 @@ class UploadService: external_size, ) self._queue_refresh(upload, upload_id, str(target_key), deduplicated=False) - self.openlist_native.cleanup(upload["source_id"], upload_id, upload.get("external_staged_path")) except JobRetry: raise except (JobCancelled, TransferCancelled): @@ -1304,6 +1383,7 @@ class UploadService: ) self.db.write_with_retry(fail) + self._refresh_recovery_state(upload_id) raise finally: with self._progress_guard: @@ -1469,14 +1549,14 @@ class UploadService: with self.db.read() as conn: rows = conn.execute( "SELECT id,temp_path FROM uploads WHERE expires_at;failure_stage?:"transfer"|"commit"|"catalog"; - can_cancel?:boolean;can_retry?:boolean; + can_cancel?:boolean;can_retry?:boolean;can_discard?:boolean;requires_reupload?:boolean; + recovery_state?:"receiving"|"available"|"catalog"|"missing"|"released";recovery_mode?:"original"|"encrypted"|"catalog"; content_sha256?:string;deduplicated?:boolean;retry_count?:number;next_retry_at?:string; transferred_bytes?:number;resume_mode?:"restart"|"offset"|"deduplicated"; phase?:"receiving"|"queued"|"encrypting"|"transferring"|"external_copying"|"external_verifying"|"indexing"|"completed"|"failed"|"cancelled"; @@ -57,6 +58,7 @@ type UploadTask = { collection_id?:string;collection_parent_id?:string;tag_ids?:string[];origin?:"browser"|"webdav"; }; type UploadPage = {items:UploadTask[];status_items:UploadTask[];active_count:number;failed_count:number;page:number;page_size:number;total:number;pages:number}; +type UploadFilters = {status:"all"|"active"|"failed"|"completed"|"cancelled";source_id:string;query:string}; type DownloadTask = { id:string;kind:"url"|"magnet"|"torrent";display_name:string;source_id:string;relative_path:string; status:string;progress:number;total_bytes:number;completed_bytes:number;download_speed:number; @@ -957,8 +959,9 @@ function LibraryPage({ sources, videos, uploads, uploadCounts, downloads, transf } function UploadsPage({ uploads, uploadPage, downloads, onDownloads, sources, reload, onPage, onUpload }: { uploads: UploadTask[];uploadPage:UploadPage;downloads:DownloadTask[];onDownloads:(items:DownloadTask[])=>void;sources:Source[];reload: () => void;onPage:(page:number)=>Promise;onUpload:()=>void }) { - uploads=uploads.map(task=>({...task,can_retry:task.can_retry??task.status==="failed",can_cancel:task.can_cancel??!["completed","cancelled","indexing"].includes(task.status)})); - const [tab,setTab]=useState<"uploads"|"downloads">("uploads");const [paging,setPaging]=useState(false);const [runtime,setRuntime]=useState<{available:boolean;running:boolean}|null>(null);const [runtimeLoad,setRuntimeLoad]=useState(idleLoad());const [showAdd,setShowAdd]=useState(false);const [url,setUrl]=useState("");const [sourceId,setSourceId]=useState("");const [relativePath,setRelativePath]=useState("");const [busy,setBusy]=useState(false);const [error,setError]=useState("");const torrentInput=useRef(null); + const [filteredPage,setFilteredPage]=useState(null);const visiblePage=filteredPage||uploadPage; + const visibleUploads=(filteredPage?.items||uploads).map(task=>({...task,can_retry:task.can_retry??task.status==="failed",can_cancel:task.can_cancel??!["completed","cancelled","indexing"].includes(task.status)})); + const [tab,setTab]=useState<"uploads"|"downloads">("uploads");const [paging,setPaging]=useState(false);const [runtime,setRuntime]=useState<{available:boolean;running:boolean}|null>(null);const [runtimeLoad,setRuntimeLoad]=useState(idleLoad());const [showAdd,setShowAdd]=useState(false);const [url,setUrl]=useState("");const [sourceId,setSourceId]=useState("");const [relativePath,setRelativePath]=useState("");const [busy,setBusy]=useState(false);const [error,setError]=useState("");const [filters,setFilters]=useState({status:"all",source_id:"",query:""});const [queryDraft,setQueryDraft]=useState("");const [selected,setSelected]=useState>(new Set());const [taskBusy,setTaskBusy]=useState>(new Set());const torrentInput=useRef(null); const writable=useMemo(()=>sources.filter(source=>source.config.writable||source.config.driver==="alist"||source.config.storage_backend==="openlist_native"),[sources]); const loadDownloads=useCallback(async()=>{setRuntimeLoad(current=>({phase:current.phase==="ready"?"ready":"loading",refreshing:current.phase==="ready"}));setError("");try{const [items,state]=await Promise.all([api("/downloads"),api<{available:boolean;running:boolean}>("/downloads/runtime")]);onDownloads(items);setRuntime(state);setRuntimeLoad({phase:"ready",refreshing:false})}catch(reason){const message=(reason as Error).message;setError(message);setRuntimeLoad(current=>current.phase==="ready"?{phase:"ready",refreshing:false,error:message}:{phase:"error",refreshing:false,error:message})}},[onDownloads]); useEffect(()=>{if(tab!=="downloads")return;void loadDownloads();const refresh=()=>{if(!document.hidden)void loadDownloads()};const timer=window.setInterval(refresh,2000);document.addEventListener("visibilitychange",refresh);return()=>{window.clearInterval(timer);document.removeEventListener("visibilitychange",refresh)}},[tab,loadDownloads]); @@ -968,16 +971,36 @@ function UploadsPage({ uploads, uploadPage, downloads, onDownloads, sources, rel async function createTorrent(file?:File){if(!file||!sourceId)return;setBusy(true);setError("");try{const form=new FormData();form.append("file",file);form.append("source_id",sourceId);form.append("relative_path",relativePath);await api("/downloads/torrent",{method:"POST",body:form});setShowAdd(false);await loadDownloads()}catch(reason){setError((reason as Error).message)}finally{setBusy(false);if(torrentInput.current)torrentInput.current.value=""}} async function action(task:DownloadTask,name:"pause"|"resume"|"retry"){try{await api(`/downloads/${task.id}/${name}`,{method:"POST",body:"{}"});await loadDownloads()}catch(reason){setError((reason as Error).message)}} async function removeDownload(task:DownloadTask){if(!await appConfirm(`移除“${task.display_name}”`,"未完成的暂存文件会一并删除；已写入媒体库的文件不会受影响。",true))return;await api(`/downloads/${task.id}`,{method:"DELETE"});await loadDownloads()} - async function changePage(next:number){if(paging||next<1||next>uploadPage.pages||next===uploadPage.page)return;setPaging(true);try{await onPage(next);document.querySelector(".transfer-center")?.scrollIntoView({block:"start",behavior:window.matchMedia("(prefers-reduced-motion: reduce)").matches?"auto":"smooth"})}finally{setPaging(false)}} - const attentionCount=uploadPage.active_count+uploadPage.failed_count; - const historyCount=Math.max(0,uploadPage.total-attentionCount); + async function fetchFiltered(page:number,next:UploadFilters){const params=new URLSearchParams({page:String(page),page_size:"10",status:next.status});if(next.source_id)params.set("source_id",next.source_id);if(next.query)params.set("query",next.query);setFilteredPage(await api(`/uploads?${params}`))} + async function refreshUploadView(){if(filteredPage)await fetchFiltered(filteredPage.page,filters);else reload()} + async function applyFilters(next:UploadFilters){setFilters(next);setSelected(new Set());setPaging(true);try{await fetchFiltered(1,next)}catch(reason){setError((reason as Error).message)}finally{setPaging(false)}} + async function taskAction(task:UploadTask,action:"cancel"|"retry"|"discard"){ + if(taskBusy.has(task.id))return; + const title=task.title?.trim()||cleanMediaTitle(task.filename); + if(action==="cancel"&&!await appConfirm(`取消“${title}”`,"任务会停止，但恢复副本会保留，之后仍可重试。",true))return; + if(action==="discard"&&!await appConfirm(`删除“${title}”及恢复副本`,"这会永久删除上传暂存和任务记录，无法撤销。",true))return; + setTaskBusy(current=>new Set(current).add(task.id));setError(""); + try{await api(action==="discard"?`/uploads/${task.id}/recovery`:action==="retry"?`/uploads/${task.id}/retry`:`/uploads/${task.id}`,{method:action==="retry"?"POST":"DELETE",body:action==="retry"?"{}":undefined});await refreshUploadView()}catch(reason){setError((reason as Error).message)}finally{setTaskBusy(current=>{const next=new Set(current);next.delete(task.id);return next})} + } + async function bulkAction(action:"cancel"|"retry"){ + const ids=[...selected].filter(id=>{const task=visibleUploads.find(item=>item.id===id);return action==="cancel"?task?.can_cancel:task?.can_retry});if(!ids.length)return; + if(action==="cancel"&&!await appConfirm(`取消 ${ids.length} 个上传任务`,"任务会停止，所有可恢复副本都会保留。",true))return; + setBusy(true);setError("");try{const response=await api<{results:Array<{id:string;ok:boolean;error?:string}>}>("/uploads/bulk-actions",{method:"POST",body:JSON.stringify({upload_ids:ids,action})});const failures=response.results.filter(item=>!item.ok);if(failures.length)setError(`${response.results.length-failures.length} 条成功，${failures.length} 条失败：${failures[0].error||"状态不允许"}`);setSelected(new Set());await refreshUploadView()}catch(reason){setError((reason as Error).message)}finally{setBusy(false)} + } + async function changePage(next:number){if(paging||next<1||next>visiblePage.pages||next===visiblePage.page)return;setPaging(true);try{if(filteredPage)await fetchFiltered(next,filters);else await onPage(next);document.querySelector(".transfer-center")?.scrollIntoView({block:"start",behavior:window.matchMedia("(prefers-reduced-motion: reduce)").matches?"auto":"smooth"})}finally{setPaging(false)}} + const attentionCount=visiblePage.active_count+visiblePage.failed_count; + const historyCount=Math.max(0,visiblePage.total-attentionCount); + const uploadControls=<>
{event.preventDefault();void applyFilters({...filters,query:queryDraft.trim()})}}>
{selected.size>0&&
已选择 {selected.size} 条
}; function renderUploadTask(task:UploadTask){ const target=sources.find(source=>source.id===task.source_id);const completed=task.status==="completed";const autoRetry=task.status==="queued"&&!!task.retry_count;const transferred=task.transferred_bytes?`${formatSize(task.transferred_bytes)} / ${formatSize(task.size_bytes)}`:"";const title=task.title?.trim()||cleanMediaTitle(task.filename);const stages=task.stages||[]; - return
{completed?:task.status==="failed"?:}
{title}{task.filename} · {task.stage_label||({receiving:"正在接收",queued:"等待上传",transferring:"正在转存",indexing:"正在加入媒体库",completed:"已完成",failed:"失败",cancelled:"已取消"} as Record)[task.status]||"上传处理中"} · {formatSize(task.size_bytes)}

目标：{target?.name||"媒体库"} · {target?sourceLabel(target):"媒体来源"}

{completed?

已加入媒体库，AI 识别已排队

:
{stages.map(step=>{step.label}{step.state==="completed"?"完成":step.state==="active"?`${Math.round((step.progress||0)*100)}%`:step.state==="failed"?"失败":step.state==="cancelled"?"已取消":"等待"})}
}
{task.deduplicated&&SHA-256 安全秒传}{task.resume_mode==="offset"&&断点续传}{autoRetry&&自动重试中}{transferred&&{transferred}}
{!completed&&

{task.error||task.message||"等待更新"}

}
{task.can_retry&&}{task.can_cancel&&}
; + const checked=selected.has(task.id);const operating=taskBusy.has(task.id); + // The ternary mutates a local Set copy and returns the boolean mutation result unused. + // eslint-disable-next-line @typescript-eslint/no-unused-expressions + return
{completed?:task.status==="failed"?:}
{title}{task.filename} · {task.stage_label||({receiving:"正在接收",queued:"等待上传",transferring:"正在转存",indexing:"正在加入媒体库",completed:"已完成",failed:"失败",cancelled:"已取消"} as Record)[task.status]||"上传处理中"} · {formatSize(task.size_bytes)}

目标：{target?.name||"媒体库"} · {target?sourceLabel(target):"媒体来源"}

{completed?

已加入媒体库，AI 识别已排队

:
{stages.map(step=>{step.label}{step.state==="completed"?"完成":step.state==="active"?`${Math.round((step.progress||0)*100)}%`:step.state==="failed"?"失败":step.state==="cancelled"?"已取消":"等待"})}
}
{task.deduplicated&&SHA-256 安全秒传}{task.resume_mode==="offset"&&断点续传}{task.recovery_state==="available"&&恢复副本已保留}{task.requires_reupload&&源文件已丢失，需重新上传}{autoRetry&&自动重试中}{transferred&&{transferred}}
{!completed&&

{task.error||task.message||"等待更新"}

}
{task.can_retry&&}{task.can_cancel&&}{task.can_discard&&}
; } return

传输中心

传输中心

统一查看浏览器上传、aria2 后台下载、媒体库写入和资料解析。

{tab==="uploads"?:}
- {error&&
{error}
} + {error&&
{error}
}{tab==="uploads"&&uploadControls} {tab==="uploads"?<>
进行中 {uploadPage.active_count}失败 {uploadPage.failed_count}历史 {historyCount}
{uploads.map(renderUploadTask)}{!uploads.length&&

还没有上传记录

点击“继续上传”，把视频导入本地或网盘媒体库。

}
{uploadPage.pages>1&&}:
{runtimeLoad.phase==="loading"||runtimeLoad.phase==="idle"?:runtimeLoad.phase==="error"?

后台下载状态加载失败

{runtimeLoad.error}

:<>{runtime&&!runtime.available&&
系统 aria2c 不可用请先在飞牛中安装 aria2；ImageFind 不会私自下载另一套运行时。
}{downloads.map(task=>{const target=sources.find(source=>source.id===task.source_id);return
{task.status==="completed"?:task.status==="failed"?:}
{task.display_name}{labels[task.status]||task.status} · {Math.round(task.progress*100)}%

目标：{target?.name||"媒体库"}/{task.relative_path} · {task.download_speed?`${formatSize(task.download_speed)}/s`:formatSize(task.total_bytes)}

{task.error||`${formatSize(task.completed_bytes)} / ${formatSize(task.total_bytes)}`}

{task.status==="downloading"&&}{task.status==="paused"&&}{task.status==="failed"&&}
})}{!downloads.length&&

还没有后台下载

支持 HTTP/HTTPS/FTP、磁力链接和 Torrent。

}}
} {showAdd&&

后台下载

新建后台下载

createTorrent(event.target.files?.[0])}/>{runtimeLoad.phase!=="ready"?:!runtime?.available&&
系统未安装 aria2c
}
}
; @@ -1460,11 +1483,14 @@ export default function App() { const loadSources=useCallback((quiet=false)=>runResource("sources",()=>api("/sources"),setSources,quiet),[runResource]); const loadHome=useCallback((quiet=false)=>runResource("home",()=>api("/home?item_limit=10&tag_limit=3"),setHomeFeed,quiet),[runResource]); const loadVideos=useCallback((quiet=false)=>runResource("videos",()=>api("/videos?limit=200"),setVideos,quiet),[runResource]); - const loadUploads=useCallback((quiet=false,requestedPage?:number)=>{ + const uploadFiltersRef=useRef({status:"all",source_id:"",query:""}); + const loadUploads=useCallback((quiet=false,requestedPage?:number,nextFilters?:UploadFilters)=>{ const target=requestedPage??uploadPageNumber.current; + if(nextFilters)uploadFiltersRef.current=nextFilters;const filters=uploadFiltersRef.current; const requestSequence=++uploadRequestSequence.current; if(requestedPage!==undefined)uploadPageNumber.current=target; - return runResource("uploads",()=>api(`/uploads?page=${target}&page_size=10`),value=>{ + const params=new URLSearchParams({page:String(target),page_size:"10",status:filters.status});if(filters.source_id)params.set("source_id",filters.source_id);if(filters.query)params.set("query",filters.query); + return runResource("uploads",()=>api(`/uploads?${params}`),value=>{ if(requestSequence!==uploadRequestSequence.current)return; const normalized=normalizeUploadPage(value,target);uploadPageNumber.current=normalized.page;setUploadPage(normalized);setUploads(normalized.status_items) },quiet); diff --git a/frontend/src/styles.css b/frontend/src/styles.css index 317c447..859d813 100644 --- a/frontend/src/styles.css +++ b/frontend/src/styles.css @@ -615,6 +615,12 @@ html[data-theme="dark"] :is(.upload-batch-metadata,.upload-tag-groups fieldset,. @media(max-width:720px){.upload-batch-metadata{padding:11px}.upload-batch-metadata>header{align-items:flex-start}.upload-batch-metadata>header button{min-height:44px}.upload-tag-groups button,.collection-tag-rules fieldset button{min-height:44px}.collection-hero>div:last-child{grid-template-columns:1fr 1fr}.collection-tag-rules>div{grid-template-columns:1fr}.collection-tag-rules>footer{align-items:stretch;flex-direction:column}.collection-tag-rules>footer .primary{min-height:44px}.webdav-connection{grid-template-columns:1fr}.webdav-connection>button{min-height:44px}.player-marker-drawer{top:8px;right:8px;bottom:52px;width:calc(100% - 16px);max-height:none;padding:9px}.player-marker-drawer>header>button,.player-marker-list article>button:not(.marker-jump){width:40px;height:40px}.player-marker-add{min-height:40px}.player-marker-list article{grid-template-columns:minmax(0,1fr) 40px 40px}.marker-jump{min-height:40px}.pull-refresh{top:0}} @media(prefers-reduced-motion:reduce){.player-marker-dot:after,.pull-refresh{transition:none}.pull-refresh svg{transform:none}} +/* Upload task management and mobile-safe controls */ +.upload-task-filters{display:grid;grid-template-columns:minmax(150px,190px) minmax(180px,240px) minmax(260px,1fr);gap:10px;align-items:end;margin-bottom:14px;padding:13px;border:1px solid var(--line);border-radius:14px;background:var(--surface)}.upload-task-filters label{min-width:0;display:grid;gap:6px;color:var(--muted);font-size:11px;font-weight:700}.upload-task-filters select,.upload-task-filters input{width:100%;height:42px;padding:0 11px;color:var(--text);border:1px solid var(--line);border-radius:9px;outline:0;background:var(--surface);font:inherit}.upload-task-filters :is(select,input):focus{border-color:var(--brand);box-shadow:0 0 0 3px color-mix(in srgb,var(--brand) 14%,transparent)}.upload-task-search{grid-template-columns:minmax(0,1fr) auto}.upload-task-search>span{grid-column:1/-1}.upload-task-search button{height:42px}.upload-selection-bar{position:sticky;z-index:12;top:calc(var(--top) + 8px);display:grid;grid-template-columns:auto 1fr auto auto 40px;gap:8px;align-items:center;margin-bottom:12px;padding:9px 10px;border:1px solid #bed0f1;border-radius:12px;background:color-mix(in srgb,var(--surface) 94%,var(--brand-soft));box-shadow:0 8px 24px #24324c14}.upload-selection-bar strong{font-size:12px}.upload-selection-bar button{min-height:40px}.upload-task-list>article{grid-template-columns:24px 42px minmax(0,1fr) auto}.upload-task-list>article.selected{border-color:#93b3ec;background:color-mix(in srgb,var(--surface) 92%,var(--brand-soft))}.transfer-select{width:24px;height:44px;display:grid;place-items:center;align-self:start}.transfer-select input{width:18px;height:18px;accent-color:var(--brand)}.transfer-meta b.failed{color:var(--red);background:#fff0f1}.transfer-actions{flex-wrap:wrap;justify-content:flex-end}.transfer-actions button:disabled{cursor:wait;opacity:.55} +html[data-theme="dark"] .upload-task-filters{border-color:var(--line);background:var(--surface)}html[data-theme="dark"] .upload-selection-bar{border-color:#30476f;background:#1d2b47}html[data-theme="dark"] .transfer-meta b.failed{color:#ff9aa4;background:#381d22} +@media(max-width:720px){.mobile-page-uploads{--upload-action-height:52px;--upload-action-gap:14px;padding-bottom:calc(var(--upload-action-height) + var(--upload-action-gap) + 62px + env(safe-area-inset-bottom))}.upload-task-filters{grid-template-columns:1fr 1fr;padding:11px}.upload-task-search{grid-column:1/-1}.upload-task-filters select,.upload-task-filters input{min-height:44px;font-size:16px}.upload-task-search button{min-width:76px;height:44px}.upload-selection-bar{top:8px;grid-template-columns:1fr 44px;margin-right:-2px;margin-left:-2px}.upload-selection-bar>span{display:none}.upload-selection-bar>button:not(.icon-button){min-height:44px}.upload-selection-bar .secondary,.upload-selection-bar .danger-text{grid-row:2}.upload-selection-bar .secondary{grid-column:1}.upload-selection-bar .danger-text{grid-column:2}.upload-selection-bar .icon-button{grid-row:1;grid-column:2}.upload-task-list>article{grid-template-columns:24px 40px minmax(0,1fr)}.upload-task-list .transfer-main{grid-column:3}.upload-task-list .transfer-actions{grid-column:2/4;justify-content:stretch}.upload-task-list .transfer-actions button{flex:1;min-width:min(100%,130px)}.transfer-pagination{position:relative;z-index:4;margin-bottom:calc(var(--upload-action-height) + var(--upload-action-gap));padding:8px}.mobile-page-uploads .continue-upload{bottom:calc(62px + max(12px,env(safe-area-inset-bottom)))} } +@media(max-width:360px){.upload-task-filters{grid-template-columns:1fr}.upload-task-search{grid-column:auto}.transfer-pagination{gap:6px}.transfer-pagination button{min-width:70px;padding-inline:8px}} + /* 0.3.40 mobile refresh, native WebDAV and player interaction polish */ .pull-refresh svg{transform:rotate(var(--pull-rotation))} .webdav-fields label{display:grid;gap:6px;color:var(--muted);font-size:11px;font-weight:700}.webdav-fields input,.webdav-fields select{width:100%;height:44px;padding:0 36px 0 11px;color:var(--text);border:1px solid var(--line);border-radius:10px;outline:0;background:var(--surface);font:inherit}.webdav-fields input{padding-right:11px}.webdav-fields input:focus,.webdav-fields select:focus{border-color:var(--brand);box-shadow:0 0 0 3px color-mix(in srgb,var(--brand) 14%,transparent)}.webdav-connection>span:nth-of-type(2){grid-column:auto}.webdav-token-create{align-self:end}.webdav-token-once{grid-column:1/-1;display:grid;grid-template-columns:minmax(0,1fr) auto;gap:7px;align-items:center;padding:10px;color:#176447;border:1px solid #bfe5d4;border-radius:9px;background:#edfaf4}.webdav-token-once>strong{grid-column:1/-1;font-size:10px}.webdav-token-once>code{min-width:0;overflow-wrap:anywhere;font-size:10px}.webdav-token-once>button{min-height:38px}.webdav-connection p b{color:var(--text)}.preference-item.disabled{opacity:.62}.preference-switch:disabled{cursor:not-allowed}.job-row-actions{display:flex;align-items:center;gap:7px} diff --git a/scripts/audio_small_ab.py b/scripts/audio_small_ab.py index a3da34d..fa87316 100644 --- a/scripts/audio_small_ab.py +++ b/scripts/audio_small_ab.py @@ -29,22 +29,44 @@ def arguments() -> argparse.Namespace: return parser.parse_args() -def worker_run(data_dir: Path, wav: Path, window: dict, combination: tuple[str, ...]) -> dict: +def _write_window_batch(source: Path, destination: Path, windows: list[dict]) -> int: + """Concatenate equal-sized diagnostic windows without reloading Whisper.""" + with wave.open(str(source), "rb") as handle: + params = handle.getparams() + sample_rate = handle.getframerate() + frame_count = handle.getnframes() + durations = {int(item["end_ms"]) - int(item["start_ms"]) for item in windows} + if len(durations) != 1: + raise ValueError("all diagnostic windows for one media item must have equal duration") + duration_ms = durations.pop() + frames_per_window = duration_ms * sample_rate // 1000 + with wave.open(str(destination), "wb") as output: + output.setparams(params) + for item in windows: + start_frame = int(item["start_ms"]) * sample_rate // 1000 + handle.setpos(min(start_frame, frame_count)) + raw = handle.readframes(frames_per_window) + expected = frames_per_window * params.nchannels * params.sampwidth + if len(raw) < expected: + raw += b"\0" * (expected - len(raw)) + output.writeframesraw(raw) + return duration_ms + + +def worker_run(data_dir: Path, wav: Path, windows: list[dict], combination: tuple[str, ...]) -> list[dict]: name, model, device, segmentation = combination with wave.open(str(wav), "rb") as handle: - sample_rate = handle.getframerate() - start_ms = int(window["start_ms"]) - duration_ms = int(window["end_ms"]) - start_ms + duration_ms = round(handle.getnframes() / handle.getframerate() * 1000 / len(windows)) command = [ sys.executable, "-m", "imagefind.audio_worker", "--data-dir", str(data_dir), "--wav", str(wav), "--device", device, "--model-variant", model, "--segmentation", segmentation, "--quality-profile", "accuracy", - "--language-policy", str(window.get("language_policy", "zh_priority")), + "--language-policy", str(windows[0].get("language_policy", "zh_priority")), "--chunk-seconds", str(max(15, min(30, (duration_ms + 999) // 1000))), "--overlap-seconds", "0", - "--start-frame", str(start_ms * sample_rate // 1000), - "--max-chunks", "1", "--cpu-threads", "2", + "--start-frame", "0", + "--max-chunks", str(len(windows)), "--cpu-threads", "2", ] started = time.monotonic() process = subprocess.run(command, text=True, capture_output=True, encoding="utf-8", errors="replace") @@ -55,19 +77,21 @@ def worker_run(data_dir: Path, wav: Path, window: dict, combination: tuple[str, events.append(json.loads(line)) except json.JSONDecodeError: pass - return { - "combination": name, - "model": model, - "device": device, - "beam": 1 if device == "GPU" else 5, - "segmentation": segmentation, - "elapsed_seconds": round(elapsed, 3), - "window_seconds": round(duration_ms / 1000, 3), - "rtf": round(elapsed / max(0.001, duration_ms / 1000), 3), - "exit_code": process.returncode, - "events": events, - "stderr_tail": process.stderr[-2000:], - } + results = [] + for index, window in enumerate(windows, 1): + selected = [event for event in events if event.get("chunk_index") == index] + if index == len(windows): + selected.extend(event for event in events if event.get("event") in {"complete", "error"}) + results.append({ + "combination": name, "model": model, "device": device, + "beam": 1 if device == "GPU" else 5, "segmentation": segmentation, + "elapsed_seconds": round(elapsed, 3), "window_seconds": round(duration_ms / 1000, 3), + "batch_window_count": len(windows), + "batch_rtf": round(elapsed / max(0.001, duration_ms * len(windows) / 1000), 3), + "exit_code": process.returncode, "events": selected, + "stderr_tail": process.stderr[-2000:] if process.returncode else "", "window": window, + }) + return results def main() -> int: @@ -95,10 +119,11 @@ def main() -> int: selected = windows.get(media.name) if not isinstance(selected, list) or len(selected) != 8: raise SystemExit(f"{media.name}: windows JSON must contain exactly 8 labelled windows") - for window in selected: - for combination in COMBINATIONS: - result = worker_run(args.data_dir, wav, window, combination) - result.update({"media": media.name, "window": window}) + batched_wav = Path(work) / f"{media.stem}-windows.wav" + _write_window_batch(wav, batched_wav, selected) + for combination in COMBINATIONS: + for result in worker_run(args.data_dir, batched_wav, selected, combination): + result["media"] = media.name artifact["runs"].append(result) args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text(json.dumps(artifact, ensure_ascii=False, indent=2), encoding="utf-8") diff --git a/tests/test_audio_index.py b/tests/test_audio_index.py index a378e02..f332446 100644 --- a/tests/test_audio_index.py +++ b/tests/test_audio_index.py @@ -16,9 +16,32 @@ from imagefind.jobs import JobRetry from imagefind.main import create_app from imagefind.media import MediaInput from imagefind.speech import SPEECH_INDEX_REVISION, SpeechService, SpeechStageError +from imagefind.speech_quality import aggregate_transcript_quality from imagefind.text import fts_query +def test_ending_hallucination_is_risky_twice_or_once_in_weak_speech(): + repeated = aggregate_transcript_quality([ + {"text": "拜拜"}, + {"text": "正常对话"}, + {"text": "拜拜"}, + ]) + assert "whole_ending_hallucination" in repeated.flags + assert ("拜拜", 2) in repeated.repeated_phrases + + weak = aggregate_transcript_quality([ + {"text": "谢谢大家收看", "speech_ratio": 0.1}, + {"text": "正常对话", "speech_ratio": 0.8}, + ]) + assert "whole_ending_hallucination" in weak.flags + + single = aggregate_transcript_quality([ + {"text": "拜拜", "speech_ratio": 0.8}, + {"text": "正常对话", "speech_ratio": 0.8}, + ]) + assert "whole_ending_hallucination" not in single.flags + + def _app(tmp_path: Path): settings = Settings(data_dir=tmp_path / "data", embedding_backend="hash", upload_reserve_gb=0) settings.prepare() diff --git a/tests/test_openlist_client.py b/tests/test_openlist_client.py index a2cf286..fd9f5bd 100644 --- a/tests/test_openlist_client.py +++ b/tests/test_openlist_client.py @@ -1,4 +1,5 @@ from contextlib import nullcontext +from pathlib import Path from types import SimpleNamespace import httpx @@ -114,6 +115,39 @@ def test_native_catalog_uses_physical_object_but_keeps_logical_media_identity(): assert item.fingerprint +def test_native_prepare_and_cancel_preserve_original_and_local_recovery(tmp_path: Path): + original = tmp_path / "upload.part" + original.write_bytes(b"data") + staging = tmp_path / "native" + + class Sources: + rclone = SimpleNamespace() + + def get(self, _source_id): + return {} + + service = object.__new__(OpenListNativeService) + service.sources = Sources() + service.configuration = lambda _source: SimpleNamespace( + local_staging_path=staging, + encrypted=False, + source_path="local-stage", + target_path="cloud", + ) + paths = service.prepare("source", "upload-id", "folder/movie.mp4", original) + assert original.read_bytes() == b"data" + assert Path(paths["local_path"]).read_bytes() == b"data" + + cancelled: list[str] = [] + service.client = lambda _source_id: SimpleNamespace( + cancel_copy_task=lambda task_id: cancelled.append(task_id) + ) + service.cancel("source", "upload-id", "task-id", paths["staged_path"]) + assert cancelled == ["task-id"] + assert original.is_file() + assert Path(paths["local_path"]).is_file() + + class _MovingClient: def __init__(self, files: dict[str, int], *, fail_after_move: bool = False): self.files = dict(files) diff --git a/tests/test_uploads_and_storage.py b/tests/test_uploads_and_storage.py index 8c2f95d..79d6d67 100644 --- a/tests/test_uploads_and_storage.py +++ b/tests/test_uploads_and_storage.py @@ -221,6 +221,11 @@ def test_upload_cancel_is_consistent_and_commit_window_returns_conflict(tmp_path job_state = conn.execute("SELECT status FROM jobs WHERE id=?", (job_id,)).fetchone() assert upload_state["status"] == "cancelled" assert job_state["status"] == "cancelled" + with app.state.services.db.read() as conn: + retained_path = Path( + conn.execute("SELECT temp_path FROM uploads WHERE id=?", (upload["id"],)).fetchone()[0] + ) + assert retained_path.read_bytes() == b"data" committed = app.state.services.uploads.create(source_id, "", "committed.mp4", 4) with app.state.services.db.transaction() as conn: @@ -239,6 +244,30 @@ def test_upload_cancel_is_consistent_and_commit_window_returns_conflict(tmp_path asyncio.run(scenario()) +def test_cancelled_upload_requires_explicit_discard_to_delete_recovery_copy(tmp_path: Path): + app, _, source_id, headers = _app(tmp_path) + uploads = app.state.services.uploads + upload = uploads.create(source_id, "", "recoverable.mp4", 4) + uploads.receive_chunk(upload["id"], 0, b"data") + with uploads.db.read() as conn: + staging = Path(conn.execute("SELECT temp_path FROM uploads WHERE id=?", (upload["id"],)).fetchone()[0]) + uploads.cancel(upload["id"]) + assert staging.is_file() + + async def scenario(): + transport = httpx.ASGITransport(app=app) + async with httpx.AsyncClient(transport=transport, base_url="http://test") as client: + response = await client.delete( + f"/api/v1/uploads/{upload['id']}/recovery", headers=headers + ) + assert response.status_code == 204 + + asyncio.run(scenario()) + assert not staging.exists() + with uploads.db.read() as conn: + assert conn.execute("SELECT 1 FROM uploads WHERE id=?", (upload["id"],)).fetchone() is None + + def test_upload_catalog_failure_retries_without_retransmitting(tmp_path: Path, monkeypatch): app, media, source_id, _ = _app(tmp_path) uploads = app.state.services.uploads