fix: preserve upload recovery and harden speech indexing

This commit is contained in:
2026-08-12 19:49:25 +08:00
parent 10f2c078ec
commit a0a185b96c
16 changed files with 13234 additions and 62 deletions
+131 -1
View File
@@ -297,6 +297,11 @@ class UploadUpdateBody(BaseModel):
tag_ids: list[str] | None = Field(default=None, max_length=500) tag_ids: list[str] | None = Field(default=None, max_length=500)
class UploadBulkActionBody(BaseModel):
upload_ids: list[str] = Field(min_length=1, max_length=200)
action: Literal["cancel", "retry"]
class DownloadCreateBody(BaseModel): class DownloadCreateBody(BaseModel):
url: str = Field(min_length=1, max_length=8192) url: str = Field(min_length=1, max_length=8192)
source_id: str source_id: str
@@ -1896,6 +1901,34 @@ async def events(
async def search( async def search(
body: SearchBody, app: Annotated[Services, Depends(services)], _: Annotated[dict, Depends(require_auth)] body: SearchBody, app: Annotated[Services, Depends(services)], _: Annotated[dict, Depends(require_auth)]
): ):
rebuild = app.db.setting("audio_rebuild", {})
if isinstance(rebuild, dict) and rebuild.get("status") in {"running", "failed"}:
if rebuild.get("status") == "running":
with app.db.read() as conn:
remaining = int(conn.execute(
"SELECT count(*) FROM videos WHERE available=1 AND audio_index_revision<?",
(SPEECH_INDEX_REVISION,),
).fetchone()[0])
active = int(conn.execute(
"SELECT count(*) FROM jobs WHERE kind='transcribe_audio' AND status IN ('queued','running')"
).fetchone()[0])
failed = int(conn.execute(
"SELECT count(*) FROM jobs WHERE kind='transcribe_audio' AND status='failed' AND finished_at>=?",
(rebuild.get("started_at") or "",),
).fetchone()[0])
if active == 0:
status = "completed" if remaining == 0 else "failed"
rebuild = {**rebuild, "status": status, "remaining": remaining, "failed_jobs": failed, "finished_at": utcnow()}
app.db.set_setting("audio_rebuild", rebuild)
if rebuild.get("status") == "completed":
rebuild = {}
else:
requested = set(body.recognition_types or [])
if "audio" in requested:
message = "音频索引全库重建存在失败任务,修复前音频搜索保持暂停" if rebuild.get("status") == "failed" else "音频索引正在全库重建,音频搜索暂时不可用"
raise HTTPException(409, message)
if body.recognition_types is None:
body.recognition_types = ["visual", "ocr", "person", "subtitle", "metadata"]
image_path = None image_path = None
if body.image_id: if body.image_id:
if not app.embeddings.status()["visual_ready"]: if not app.embeddings.status()["visual_ready"]:
@@ -2091,6 +2124,61 @@ def reconcile_speech_quality(
return {**result, "queued": len(queued)} return {**result, "queued": len(queued)}
@router.post("/v1/speech/rebuild", status_code=202)
def rebuild_all_speech(
app: Annotated[Services, Depends(services)],
_: Annotated[dict, Depends(require_auth)],
):
if not app.models.runnable_component_versions().get("audio"):
raise HTTPException(409, "Small 音频模型尚未安装")
if app.settings.audio_model_variant != "small":
try:
app.models.set_audio_variant("small")
except (ValueError, ModelUnavailable, SpeechStageError) as exc:
raise HTTPException(409, str(exc)) from exc
cancelled = app.jobs.cancel_by_kind("transcribe_audio")
with app.db.transaction() as conn:
total = int(conn.execute("SELECT count(*) FROM videos WHERE available=1").fetchone()[0])
conn.execute("UPDATE videos SET audio_model_version=NULL,audio_index_revision=0 WHERE available=1")
state = {
"status": "running",
"model_variant": "small",
"index_revision": SPEECH_INDEX_REVISION,
"total": total,
"started_at": utcnow(),
}
app.db.set_setting("audio_rebuild", state)
queued = app.reconcile_ai()
return {**state, "cancelled": len(cancelled), "queued": len(queued)}
@router.get("/v1/speech/rebuild")
def speech_rebuild_status(
app: Annotated[Services, Depends(services)],
_: Annotated[dict, Depends(require_auth)],
):
state = app.db.setting("audio_rebuild", {"status": "idle"})
if not isinstance(state, dict) or state.get("status") != "running":
return state
with app.db.read() as conn:
completed = int(conn.execute(
"SELECT count(*) FROM videos WHERE available=1 AND audio_index_revision>=?",
(SPEECH_INDEX_REVISION,),
).fetchone()[0])
remaining = int(conn.execute(
"SELECT count(*) FROM videos WHERE available=1 AND audio_index_revision<?",
(SPEECH_INDEX_REVISION,),
).fetchone()[0])
active = int(conn.execute(
"SELECT count(*) FROM jobs WHERE kind='transcribe_audio' AND status IN ('queued','running')"
).fetchone()[0])
state = {**state, "completed": completed, "remaining": remaining, "active_jobs": active}
if remaining == 0 and active == 0:
state = {**state, "status": "completed", "finished_at": utcnow()}
app.db.set_setting("audio_rebuild", state)
return state
@router.get("/v1/videos/{video_id}/transcript") @router.get("/v1/videos/{video_id}/transcript")
def video_transcript( def video_transcript(
video_id: str, video_id: str,
@@ -2556,10 +2644,19 @@ async def list_uploads(
limit: int = 100, limit: int = 100,
page: int | None = None, page: int | None = None,
page_size: int = 10, page_size: int = 10,
status: str = "all",
source_id: str | None = None,
query: str = "",
): ):
if page is not None: if page is not None:
return await _background_api_with_fallback( return await _background_api_with_fallback(
app.uploads.paginate, app.uploads.cached_paginate, page, page_size app.uploads.paginate,
app.uploads.cached_paginate,
page,
page_size,
status,
source_id,
query,
) )
return await _background_api_with_fallback( return await _background_api_with_fallback(
app.uploads.list, app.uploads.cached_list, limit app.uploads.list, app.uploads.cached_list, limit
@@ -2660,6 +2757,39 @@ def cancel_upload(
raise HTTPException(409, str(exc)) from exc raise HTTPException(409, str(exc)) from exc
@router.delete("/v1/uploads/{upload_id}/recovery", status_code=204)
def discard_upload_recovery(
upload_id: str,
app: Annotated[Services, Depends(services)],
_: Annotated[dict, Depends(require_auth)],
):
try:
app.uploads.discard(upload_id)
except KeyError as exc:
raise HTTPException(404, "上传任务不存在") from exc
except ValueError as exc:
raise HTTPException(409, str(exc)) from exc
@router.post("/v1/uploads/bulk-actions")
def bulk_upload_actions(
body: UploadBulkActionBody,
app: Annotated[Services, Depends(services)],
_: Annotated[dict, Depends(require_auth)],
):
results = []
for upload_id in dict.fromkeys(body.upload_ids):
try:
if body.action == "cancel":
app.uploads.cancel(upload_id)
else:
app.uploads.retry(upload_id)
results.append({"id": upload_id, "ok": True})
except (KeyError, ValueError) as exc:
results.append({"id": upload_id, "ok": False, "error": str(exc) or "上传任务不存在"})
return {"results": results}
@router.get("/v1/downloads/runtime") @router.get("/v1/downloads/runtime")
def download_runtime( def download_runtime(
app: Annotated[Services, Depends(services)], app: Annotated[Services, Depends(services)],
+6
View File
@@ -779,6 +779,10 @@ class Database:
conn.execute("ALTER TABLE uploads ADD COLUMN IF NOT EXISTS external_local_path TEXT") conn.execute("ALTER TABLE uploads ADD COLUMN IF NOT EXISTS external_local_path TEXT")
conn.execute("ALTER TABLE uploads ADD COLUMN IF NOT EXISTS external_size_bytes BIGINT") conn.execute("ALTER TABLE uploads ADD COLUMN IF NOT EXISTS external_size_bytes BIGINT")
conn.execute("ALTER TABLE uploads ADD COLUMN IF NOT EXISTS external_attempts INTEGER NOT NULL DEFAULT 0") conn.execute("ALTER TABLE uploads ADD COLUMN IF NOT EXISTS external_attempts INTEGER NOT NULL DEFAULT 0")
conn.execute(
"ALTER TABLE uploads ADD COLUMN IF NOT EXISTS recovery_state TEXT NOT NULL DEFAULT 'receiving'"
)
conn.execute("ALTER TABLE uploads ADD COLUMN IF NOT EXISTS recovery_mode TEXT")
conn.execute( conn.execute(
"UPDATE videos v SET storage_backend=u.storage_backend,physical_path=u.external_target_path," "UPDATE videos v SET storage_backend=u.storage_backend,physical_path=u.external_target_path,"
"physical_size_bytes=coalesce(u.external_size_bytes,v.size_bytes) FROM (" "physical_size_bytes=coalesce(u.external_size_bytes,v.size_bytes) FROM ("
@@ -1291,6 +1295,8 @@ CREATE TABLE IF NOT EXISTS uploads (
external_local_path TEXT, external_local_path TEXT,
external_size_bytes BIGINT, external_size_bytes BIGINT,
external_attempts INTEGER NOT NULL DEFAULT 0, external_attempts INTEGER NOT NULL DEFAULT 0,
recovery_state TEXT NOT NULL DEFAULT 'receiving',
recovery_mode TEXT,
created_at TEXT NOT NULL, created_at TEXT NOT NULL,
updated_at TEXT NOT NULL, updated_at TEXT NOT NULL,
expires_at TEXT NOT NULL expires_at TEXT NOT NULL
+5 -7
View File
@@ -379,7 +379,6 @@ class OpenListNativeService:
if config.encrypted: if config.encrypted:
encrypted_root = job_root / "encrypted" encrypted_root = job_root / "encrypted"
encrypted_file = self.sources.rclone.encrypt_to_local(source, input_path, target_key, encrypted_root) encrypted_file = self.sources.rclone.encrypt_to_local(source, input_path, target_key, encrypted_root)
input_path.unlink(missing_ok=True)
physical_relative = encrypted_file.relative_to(encrypted_root).as_posix() physical_relative = encrypted_file.relative_to(encrypted_root).as_posix()
local_file = encrypted_file local_file = encrypted_file
source_path = _remote_join(config.source_path, upload_id, "encrypted", physical_relative) source_path = _remote_join(config.source_path, upload_id, "encrypted", physical_relative)
@@ -389,7 +388,7 @@ class OpenListNativeService:
direct_root.mkdir(parents=True, exist_ok=True) direct_root.mkdir(parents=True, exist_ok=True)
local_file = direct_root / PurePosixPath(target_key).name local_file = direct_root / PurePosixPath(target_key).name
if input_path.resolve() != local_file.resolve(): if input_path.resolve() != local_file.resolve():
os.replace(input_path, local_file) shutil.copy2(input_path, local_file)
physical_relative = target_key physical_relative = target_key
source_path = _remote_join(config.source_path, upload_id, "direct", local_file.name) source_path = _remote_join(config.source_path, upload_id, "direct", local_file.name)
target_path = _remote_join(config.target_path, target_key) target_path = _remote_join(config.target_path, target_key)
@@ -648,8 +647,7 @@ class OpenListNativeService:
shutil.rmtree(job_root, ignore_errors=True) shutil.rmtree(job_root, ignore_errors=True)
def cancel(self, source_id: str, upload_id: str, task_id: str | None, staged_path: str | None) -> None: def cancel(self, source_id: str, upload_id: str, task_id: str | None, staged_path: str | None) -> None:
try: # Cancellation stops work but preserves recovery copies until an
if task_id: # administrator explicitly discards the task and its data.
self.client(source_id).cancel_copy_task(task_id) if task_id:
finally: self.client(source_id).cancel_copy_task(task_id)
self.cleanup(source_id, upload_id, staged_path)
+22 -1
View File
@@ -151,7 +151,7 @@ class Scanner:
with self.db.read() as conn: with self.db.read() as conn:
upload_row = conn.execute( upload_row = conn.execute(
"SELECT title,collection_id,collection_parent_id,tag_ids_json,content_sha256,deduplicated," "SELECT title,collection_id,collection_parent_id,tag_ids_json,content_sha256,deduplicated,"
"storage_backend,external_target_path,external_size_bytes,size_bytes " "storage_backend,external_staged_path,external_target_path,external_size_bytes,size_bytes "
"FROM uploads WHERE id=?", "FROM uploads WHERE id=?",
(upload_id,), (upload_id,),
).fetchone() ).fetchone()
@@ -307,6 +307,27 @@ class Scanner:
) )
conn.execute("DELETE FROM ingest_guards WHERE upload_id=?", (upload_id,)) conn.execute("DELETE FROM ingest_guards WHERE upload_id=?", (upload_id,))
self.jobs.update(job_id, 1, "文件已加入媒体库,视频资料解析已排队") self.jobs.update(job_id, 1, "文件已加入媒体库,视频资料解析已排队")
if upload_id and upload_row:
upload_data = dict(upload_row)
try:
if upload_data.get("storage_backend") == "openlist_native" and self.openlist_native:
self.openlist_native.cleanup(
source_id, upload_id, upload_data.get("external_staged_path")
)
with self.db.transaction() as conn:
row = conn.execute("SELECT temp_path FROM uploads WHERE id=?", (upload_id,)).fetchone()
if row:
from pathlib import Path
Path(row["temp_path"]).unlink(missing_ok=True)
conn.execute(
"UPDATE uploads SET recovery_state='released',recovery_mode=NULL,updated_at=? WHERE id=?",
(utcnow(), upload_id),
)
except OSError:
# Cataloging is already committed. Keep the recovery state
# visible so an administrator can explicitly discard it.
pass
except Exception as exc: except Exception as exc:
if upload_id: if upload_id:
with self.db.transaction() as conn: with self.db.transaction() as conn:
+13 -1
View File
@@ -89,9 +89,21 @@ def aggregate_transcript_quality(
if short_ratio >= 0.65 and len(texts) >= 10: if short_ratio >= 0.65 and len(texts) >= 10:
flags.append("whole_short_segment_dominance") flags.append("whole_short_segment_dominance")
penalty += 0.35 penalty += 0.35
if any(text in _ENDING_HALLUCINATIONS for text, _count in repeated): ending_counts = tuple(
sorted((text, count) for text, count in counts.items() if text in _ENDING_HALLUCINATIONS)
)
weak_speech_ending = any(
_phrase_key(item.get("text")) in _ENDING_HALLUCINATIONS
and item.get("speech_ratio") is not None
and float(item.get("speech_ratio") or 0) < 0.20
for item in segments
)
if any(count >= 2 for _text, count in ending_counts) or weak_speech_ending:
flags.append("whole_ending_hallucination") flags.append("whole_ending_hallucination")
penalty += 0.45 penalty += 0.45
reported = dict(repeated)
reported.update(dict(ending_counts))
repeated = tuple(sorted(reported.items(), key=lambda item: (-item[1], item[0])))[:12]
flags.extend(f"repeated_phrase:{text}:{count}" for text, count in repeated) flags.extend(f"repeated_phrase:{text}:{count}" for text, count in repeated)
score = max(0.0, min(1.0, float(1.0 if base_score is None else base_score) - penalty)) score = max(0.0, min(1.0, float(1.0 if base_score is None else base_score) - penalty))
return AggregateTranscriptQuality(round(score, 3), tuple(dict.fromkeys(flags)), repeated, round(short_ratio, 3)) return AggregateTranscriptQuality(round(score, 3), tuple(dict.fromkeys(flags)), repeated, round(short_ratio, 3))
+98 -18
View File
@@ -58,7 +58,7 @@ class UploadService:
self._progress_state: dict[str, tuple[float, int]] = {} self._progress_state: dict[str, tuple[float, int]] = {}
self._list_cache_lock = threading.Lock() self._list_cache_lock = threading.Lock()
self._list_cache: list[dict] = [] self._list_cache: list[dict] = []
self._page_cache: dict[tuple[int, int], dict] = {} self._page_cache: dict[tuple[int, int, str, str, str], dict] = {}
self.storage_usage = None self.storage_usage = None
def _lock(self, upload_id: str) -> threading.Lock: def _lock(self, upload_id: str) -> threading.Lock:
@@ -82,6 +82,18 @@ class UploadService:
@staticmethod @staticmethod
def _public(item: dict) -> dict: def _public(item: dict) -> dict:
result = dict(item) result = dict(item)
status = result["status"]
recovery_state = str(result.get("recovery_state") or "")
if status in {"failed", "cancelled"} and recovery_state in {"", "receiving"}:
original = Path(str(result.get("temp_path") or "")).is_file()
encrypted = bool(
result.get("external_local_path")
and Path(str(result["external_local_path"])).is_file()
)
catalog = bool(result.get("failure_stage") == "catalog" and result.get("target_key"))
recovery_state = "available" if original or encrypted else "catalog" if catalog else "missing"
result["recovery_state"] = recovery_state
result["recovery_mode"] = "original" if original else "encrypted" if encrypted else "catalog" if catalog else None
result["deduplicated"] = bool(result.get("deduplicated", False)) result["deduplicated"] = bool(result.get("deduplicated", False))
result.pop("temp_path", None) result.pop("temp_path", None)
result.pop("target_key", None) result.pop("target_key", None)
@@ -95,7 +107,6 @@ class UploadService:
result.pop("external_error", None) result.pop("external_error", None)
external_task_id = result.pop("external_task_id", None) external_task_id = result.pop("external_task_id", None)
result["external_task_ref"] = external_task_id[-8:] if external_task_id else None result["external_task_ref"] = external_task_id[-8:] if external_task_id else None
status = result["status"]
failure_stage = result.get("failure_stage") failure_stage = result.get("failure_stage")
phase = result.get("phase") or status phase = result.get("phase") or status
if phase == "indexing": if phase == "indexing":
@@ -160,8 +171,11 @@ class UploadService:
"label": "加入媒体库并排队 AI", "label": "加入媒体库并排队 AI",
}, },
] ]
recovery_state = str(result.get("recovery_state") or "")
result["can_cancel"] = status in {"receiving", "queued", "transferring"} and failure_stage != "commit" result["can_cancel"] = status in {"receiving", "queued", "transferring"} and failure_stage != "commit"
result["can_retry"] = status == "failed" result["can_retry"] = status == "failed" and recovery_state in {"available", "catalog"}
result["can_discard"] = status in {"failed", "cancelled", "completed"}
result["requires_reupload"] = status == "failed" and recovery_state == "missing"
return result return result
def list(self, limit: int = 100) -> list[dict]: def list(self, limit: int = 100) -> list[dict]:
@@ -193,23 +207,51 @@ class UploadService:
"status_items": [dict(item) for item in value.get("status_items", [])], "status_items": [dict(item) for item in value.get("status_items", [])],
} }
def paginate(self, page: int = 1, page_size: int = 10) -> dict: def paginate(
self,
page: int = 1,
page_size: int = 10,
status: str = "all",
source_id: str | None = None,
query: str = "",
) -> dict:
"""Return one history page plus the small live-state set used by global UI.""" """Return one history page plus the small live-state set used by global UI."""
page = max(1, page) page = max(1, page)
page_size = min(max(page_size, 1), 50) page_size = min(max(page_size, 1), 50)
status = status if status in {"all", "active", "failed", "completed", "cancelled"} else "all"
clauses: list[str] = []
params: list[object] = []
status_values = {
"active": ("receiving", "queued", "transferring", "indexing"),
"failed": ("failed",),
"completed": ("completed",),
"cancelled": ("cancelled",),
}.get(status)
if status_values:
clauses.append("status IN (" + ",".join("?" for _ in status_values) + ")")
params.extend(status_values)
if source_id:
clauses.append("source_id=?")
params.append(source_id)
query = query.strip()[:200]
if query:
clauses.append("(filename ILIKE ? OR coalesce(title,'') ILIKE ?)")
params.extend((f"%{query}%", f"%{query}%"))
where = " WHERE " + " AND ".join(clauses) if clauses else ""
with self.db.read() as conn: with self.db.read() as conn:
counts = conn.execute( counts = conn.execute(
"SELECT count(*) AS total," "SELECT count(*) AS total,"
"count(*) FILTER (WHERE status IN ('receiving','queued','transferring','indexing')) AS active_count," "count(*) FILTER (WHERE status IN ('receiving','queued','transferring','indexing')) AS active_count,"
"count(*) FILTER (WHERE status='failed') AS failed_count FROM uploads" "count(*) FILTER (WHERE status='failed') AS failed_count FROM uploads" + where,
tuple(params),
).fetchone() ).fetchone()
total = int(counts["total"]) total = int(counts["total"])
pages = max(1, (total + page_size - 1) // page_size) pages = max(1, (total + page_size - 1) // page_size)
page = min(page, pages) page = min(page, pages)
rows = conn.execute( rows = conn.execute(
"SELECT * FROM uploads ORDER BY created_at DESC,id DESC LIMIT ? OFFSET ?", "SELECT * FROM uploads" + where + " ORDER BY created_at DESC,id DESC LIMIT ? OFFSET ?",
(page_size, (page - 1) * page_size), (*params, page_size, (page - 1) * page_size),
).fetchall() ).fetchall()
status_rows = conn.execute( status_rows = conn.execute(
"SELECT * FROM uploads " "SELECT * FROM uploads "
@@ -229,20 +271,27 @@ class UploadService:
"pages": pages, "pages": pages,
} }
with self._list_cache_lock: with self._list_cache_lock:
cache_key = (page, page_size) cache_key = (page, page_size, status, source_id or "", query)
self._page_cache.pop(cache_key, None) self._page_cache.pop(cache_key, None)
self._page_cache[cache_key] = self._copy_page(result) self._page_cache[cache_key] = self._copy_page(result)
while len(self._page_cache) > 16: while len(self._page_cache) > 16:
self._page_cache.pop(next(iter(self._page_cache))) self._page_cache.pop(next(iter(self._page_cache)))
return result return result
def cached_paginate(self, page: int = 1, page_size: int = 10) -> dict: def cached_paginate(
self,
page: int = 1,
page_size: int = 10,
status: str = "all",
source_id: str | None = None,
query: str = "",
) -> dict:
"""Return the last matching page snapshot during database pressure.""" """Return the last matching page snapshot during database pressure."""
page = max(1, page) page = max(1, page)
page_size = min(max(page_size, 1), 50) page_size = min(max(page_size, 1), 50)
with self._list_cache_lock: with self._list_cache_lock:
cached = self._page_cache.get((page, page_size)) cached = self._page_cache.get((page, page_size, status, source_id or "", query.strip()[:200]))
if cached is not None: if cached is not None:
return self._copy_page(cached) return self._copy_page(cached)
return { return {
@@ -631,13 +680,47 @@ class UploadService:
"UPDATE uploads SET status='cancelled',phase='cancelled',message='已取消',updated_at=? WHERE id=?", "UPDATE uploads SET status='cancelled',phase='cancelled',message='已取消',updated_at=? WHERE id=?",
(utcnow(), upload_id), (utcnow(), upload_id),
) )
if upload["status"] in {"receiving", "queued", "failed"}: self._refresh_recovery_state(upload_id)
Path(upload["temp_path"]).unlink(missing_ok=True)
def _refresh_recovery_state(self, upload_id: str) -> str:
upload = self._get(upload_id)
original = Path(upload["temp_path"]).is_file()
encrypted = bool(
upload.get("external_local_path") and Path(str(upload["external_local_path"])).is_file()
)
catalog = bool(upload.get("failure_stage") == "catalog" and upload.get("target_key"))
state = "available" if original or encrypted else "catalog" if catalog else "missing"
mode = "original" if original else "encrypted" if encrypted else "catalog" if catalog else None
self.db.write_with_retry(
lambda conn: conn.execute(
"UPDATE uploads SET recovery_state=?,recovery_mode=?,updated_at=? WHERE id=?",
(state, mode, utcnow(), upload_id),
)
)
return state
def discard(self, upload_id: str) -> None:
"""Explicitly delete a terminal task and all retained recovery copies."""
with self._lock(upload_id):
upload = self._get(upload_id)
if upload["status"] not in {"failed", "cancelled", "completed"}:
raise ValueError("任务仍在执行,请先取消")
if upload.get("storage_backend") == "openlist_native" and self.openlist_native is not None:
self.openlist_native.cleanup(
upload["source_id"], upload_id, upload.get("external_staged_path")
)
Path(upload["temp_path"]).unlink(missing_ok=True)
with self.db.transaction() as conn:
conn.execute("DELETE FROM ingest_guards WHERE upload_id=?", (upload_id,))
conn.execute("DELETE FROM uploads WHERE id=?", (upload_id,))
def retry(self, upload_id: str) -> str: def retry(self, upload_id: str) -> str:
upload = self._get(upload_id) upload = self._get(upload_id)
if upload["status"] != "failed": if upload["status"] != "failed":
raise ValueError("该上传任务不能重试") raise ValueError("该上传任务不能重试")
if self._refresh_recovery_state(upload_id) == "missing":
raise ValueError("恢复副本不存在,请重新上传源文件")
upload = self._get(upload_id)
with self.db.transaction() as conn: with self.db.transaction() as conn:
if upload.get("failure_stage") == "catalog" and upload.get("target_key"): if upload.get("failure_stage") == "catalog" and upload.get("target_key"):
job_id = self.jobs.enqueue( job_id = self.jobs.enqueue(
@@ -1108,9 +1191,6 @@ class UploadService:
# OpenList move succeeded but before refresh_path was queued. # OpenList move succeeded but before refresh_path was queued.
self._before_commit(upload_id, str(target_key)) self._before_commit(upload_id, str(target_key))
self._queue_refresh(upload, upload_id, str(target_key), deduplicated=False) self._queue_refresh(upload, upload_id, str(target_key), deduplicated=False)
self.openlist_native.cleanup(
upload["source_id"], upload_id, upload.get("external_staged_path")
)
return return
if not staged_ready: if not staged_ready:
raise RuntimeError("OpenList 暂存目标不存在或长度不一致") raise RuntimeError("OpenList 暂存目标不存在或长度不一致")
@@ -1123,7 +1203,6 @@ class UploadService:
external_size, external_size,
) )
self._queue_refresh(upload, upload_id, str(target_key), deduplicated=False) self._queue_refresh(upload, upload_id, str(target_key), deduplicated=False)
self.openlist_native.cleanup(upload["source_id"], upload_id, upload.get("external_staged_path"))
except JobRetry: except JobRetry:
raise raise
except (JobCancelled, TransferCancelled): except (JobCancelled, TransferCancelled):
@@ -1304,6 +1383,7 @@ class UploadService:
) )
self.db.write_with_retry(fail) self.db.write_with_retry(fail)
self._refresh_recovery_state(upload_id)
raise raise
finally: finally:
with self._progress_guard: with self._progress_guard:
@@ -1469,14 +1549,14 @@ class UploadService:
with self.db.read() as conn: with self.db.read() as conn:
rows = conn.execute( rows = conn.execute(
"SELECT id,temp_path FROM uploads WHERE expires_at<? " "SELECT id,temp_path FROM uploads WHERE expires_at<? "
"AND status IN ('receiving','failed','cancelled','completed')", "AND status='completed' AND recovery_state='released'",
(now,), (now,),
).fetchall() ).fetchall()
for row in rows: for row in rows:
Path(row["temp_path"]).unlink(missing_ok=True) Path(row["temp_path"]).unlink(missing_ok=True)
with self.db.transaction() as conn: with self.db.transaction() as conn:
removed = conn.execute( removed = conn.execute(
"DELETE FROM uploads WHERE expires_at<? AND status IN ('receiving','failed','cancelled','completed')", "DELETE FROM uploads WHERE expires_at<? AND status='completed' AND recovery_state='released'",
(now,), (now,),
).rowcount ).rowcount
cutoff = (datetime.now(UTC) - timedelta(days=self.settings.upload_incomplete_days)).isoformat() cutoff = (datetime.now(UTC) - timedelta(days=self.settings.upload_incomplete_days)).isoformat()
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+128
View File
@@ -0,0 +1,128 @@
{
"M3316 直男与0-勾引直男体育生2.mp4": [
{
"label": "clear_dialogue_1",
"start_ms": 105000,
"end_ms": 125000
},
{
"label": "clear_dialogue_2",
"start_ms": 198000,
"end_ms": 218000
},
{
"label": "short_dense_1",
"start_ms": 315000,
"end_ms": 335000
},
{
"label": "short_dense_2",
"start_ms": 375000,
"end_ms": 395000
},
{
"label": "suspected_hallucination_1",
"start_ms": 268000,
"end_ms": 288000
},
{
"label": "suspected_hallucination_2",
"start_ms": 438000,
"end_ms": 458000
},
{
"label": "vad_miss_1",
"start_ms": 125000,
"end_ms": 145000
},
{
"label": "vad_miss_2",
"start_ms": 148000,
"end_ms": 168000
}
],
"M3316 直男与0-勾引直男体育生3.mp4": [
{
"label": "clear_dialogue_1",
"start_ms": 145000,
"end_ms": 165000
},
{
"label": "clear_dialogue_2",
"start_ms": 348000,
"end_ms": 368000
},
{
"label": "short_dense_1",
"start_ms": 0,
"end_ms": 20000
},
{
"label": "short_dense_2",
"start_ms": 85000,
"end_ms": 105000
},
{
"label": "suspected_hallucination_1",
"start_ms": 50000,
"end_ms": 70000
},
{
"label": "suspected_hallucination_2",
"start_ms": 310000,
"end_ms": 330000
},
{
"label": "vad_miss_1",
"start_ms": 188000,
"end_ms": 208000
},
{
"label": "vad_miss_2",
"start_ms": 240000,
"end_ms": 260000
}
],
"g4ww_FBcUUc7YTPo.mp4": [
{
"label": "clear_dialogue_1",
"start_ms": 25000,
"end_ms": 45000
},
{
"label": "clear_dialogue_2",
"start_ms": 292000,
"end_ms": 312000
},
{
"label": "short_dense_1",
"start_ms": 520000,
"end_ms": 540000
},
{
"label": "short_dense_2",
"start_ms": 545000,
"end_ms": 565000
},
{
"label": "suspected_hallucination_1",
"start_ms": 548000,
"end_ms": 568000
},
{
"label": "suspected_hallucination_2",
"start_ms": 333000,
"end_ms": 353000
},
{
"label": "vad_miss_1",
"start_ms": 200000,
"end_ms": 220000
},
{
"label": "vad_miss_2",
"start_ms": 400000,
"end_ms": 420000
}
]
}
+36 -10
View File
@@ -48,7 +48,8 @@ type UploadTask = {
bytes_received: number; size_bytes: number; message?: string; error?: string; bytes_received: number; size_bytes: number; message?: string; error?: string;
relative_path: string; chunk_size: number; total_chunks: number; received_chunks: number[]; relative_path: string; chunk_size: number; total_chunks: number; received_chunks: number[];
stage?:string;stage_label?:string;stages?:Array<{key:string;state:"pending"|"active"|"completed"|"failed"|"cancelled";progress:number;label:string}>;failure_stage?:"transfer"|"commit"|"catalog"; stage?:string;stage_label?:string;stages?:Array<{key:string;state:"pending"|"active"|"completed"|"failed"|"cancelled";progress:number;label:string}>;failure_stage?:"transfer"|"commit"|"catalog";
can_cancel?:boolean;can_retry?:boolean; can_cancel?:boolean;can_retry?:boolean;can_discard?:boolean;requires_reupload?:boolean;
recovery_state?:"receiving"|"available"|"catalog"|"missing"|"released";recovery_mode?:"original"|"encrypted"|"catalog";
content_sha256?:string;deduplicated?:boolean;retry_count?:number;next_retry_at?:string; content_sha256?:string;deduplicated?:boolean;retry_count?:number;next_retry_at?:string;
transferred_bytes?:number;resume_mode?:"restart"|"offset"|"deduplicated"; transferred_bytes?:number;resume_mode?:"restart"|"offset"|"deduplicated";
phase?:"receiving"|"queued"|"encrypting"|"transferring"|"external_copying"|"external_verifying"|"indexing"|"completed"|"failed"|"cancelled"; phase?:"receiving"|"queued"|"encrypting"|"transferring"|"external_copying"|"external_verifying"|"indexing"|"completed"|"failed"|"cancelled";
@@ -57,6 +58,7 @@ type UploadTask = {
collection_id?:string;collection_parent_id?:string;tag_ids?:string[];origin?:"browser"|"webdav"; collection_id?:string;collection_parent_id?:string;tag_ids?:string[];origin?:"browser"|"webdav";
}; };
type UploadPage = {items:UploadTask[];status_items:UploadTask[];active_count:number;failed_count:number;page:number;page_size:number;total:number;pages:number}; type UploadPage = {items:UploadTask[];status_items:UploadTask[];active_count:number;failed_count:number;page:number;page_size:number;total:number;pages:number};
type UploadFilters = {status:"all"|"active"|"failed"|"completed"|"cancelled";source_id:string;query:string};
type DownloadTask = { type DownloadTask = {
id:string;kind:"url"|"magnet"|"torrent";display_name:string;source_id:string;relative_path:string; id:string;kind:"url"|"magnet"|"torrent";display_name:string;source_id:string;relative_path:string;
status:string;progress:number;total_bytes:number;completed_bytes:number;download_speed:number; status:string;progress:number;total_bytes:number;completed_bytes:number;download_speed:number;
@@ -957,8 +959,9 @@ function LibraryPage({ sources, videos, uploads, uploadCounts, downloads, transf
} }
function UploadsPage({ uploads, uploadPage, downloads, onDownloads, sources, reload, onPage, onUpload }: { uploads: UploadTask[];uploadPage:UploadPage;downloads:DownloadTask[];onDownloads:(items:DownloadTask[])=>void;sources:Source[];reload: () => void;onPage:(page:number)=>Promise<unknown>;onUpload:()=>void }) { function UploadsPage({ uploads, uploadPage, downloads, onDownloads, sources, reload, onPage, onUpload }: { uploads: UploadTask[];uploadPage:UploadPage;downloads:DownloadTask[];onDownloads:(items:DownloadTask[])=>void;sources:Source[];reload: () => void;onPage:(page:number)=>Promise<unknown>;onUpload:()=>void }) {
uploads=uploads.map(task=>({...task,can_retry:task.can_retry??task.status==="failed",can_cancel:task.can_cancel??!["completed","cancelled","indexing"].includes(task.status)})); const [filteredPage,setFilteredPage]=useState<UploadPage|null>(null);const visiblePage=filteredPage||uploadPage;
const [tab,setTab]=useState<"uploads"|"downloads">("uploads");const [paging,setPaging]=useState(false);const [runtime,setRuntime]=useState<{available:boolean;running:boolean}|null>(null);const [runtimeLoad,setRuntimeLoad]=useState<LoadStatus>(idleLoad());const [showAdd,setShowAdd]=useState(false);const [url,setUrl]=useState("");const [sourceId,setSourceId]=useState("");const [relativePath,setRelativePath]=useState("");const [busy,setBusy]=useState(false);const [error,setError]=useState("");const torrentInput=useRef<HTMLInputElement>(null); const visibleUploads=(filteredPage?.items||uploads).map(task=>({...task,can_retry:task.can_retry??task.status==="failed",can_cancel:task.can_cancel??!["completed","cancelled","indexing"].includes(task.status)}));
const [tab,setTab]=useState<"uploads"|"downloads">("uploads");const [paging,setPaging]=useState(false);const [runtime,setRuntime]=useState<{available:boolean;running:boolean}|null>(null);const [runtimeLoad,setRuntimeLoad]=useState<LoadStatus>(idleLoad());const [showAdd,setShowAdd]=useState(false);const [url,setUrl]=useState("");const [sourceId,setSourceId]=useState("");const [relativePath,setRelativePath]=useState("");const [busy,setBusy]=useState(false);const [error,setError]=useState("");const [filters,setFilters]=useState<UploadFilters>({status:"all",source_id:"",query:""});const [queryDraft,setQueryDraft]=useState("");const [selected,setSelected]=useState<Set<string>>(new Set());const [taskBusy,setTaskBusy]=useState<Set<string>>(new Set());const torrentInput=useRef<HTMLInputElement>(null);
const writable=useMemo(()=>sources.filter(source=>source.config.writable||source.config.driver==="alist"||source.config.storage_backend==="openlist_native"),[sources]); const writable=useMemo(()=>sources.filter(source=>source.config.writable||source.config.driver==="alist"||source.config.storage_backend==="openlist_native"),[sources]);
const loadDownloads=useCallback(async()=>{setRuntimeLoad(current=>({phase:current.phase==="ready"?"ready":"loading",refreshing:current.phase==="ready"}));setError("");try{const [items,state]=await Promise.all([api<DownloadTask[]>("/downloads"),api<{available:boolean;running:boolean}>("/downloads/runtime")]);onDownloads(items);setRuntime(state);setRuntimeLoad({phase:"ready",refreshing:false})}catch(reason){const message=(reason as Error).message;setError(message);setRuntimeLoad(current=>current.phase==="ready"?{phase:"ready",refreshing:false,error:message}:{phase:"error",refreshing:false,error:message})}},[onDownloads]); const loadDownloads=useCallback(async()=>{setRuntimeLoad(current=>({phase:current.phase==="ready"?"ready":"loading",refreshing:current.phase==="ready"}));setError("");try{const [items,state]=await Promise.all([api<DownloadTask[]>("/downloads"),api<{available:boolean;running:boolean}>("/downloads/runtime")]);onDownloads(items);setRuntime(state);setRuntimeLoad({phase:"ready",refreshing:false})}catch(reason){const message=(reason as Error).message;setError(message);setRuntimeLoad(current=>current.phase==="ready"?{phase:"ready",refreshing:false,error:message}:{phase:"error",refreshing:false,error:message})}},[onDownloads]);
useEffect(()=>{if(tab!=="downloads")return;void loadDownloads();const refresh=()=>{if(!document.hidden)void loadDownloads()};const timer=window.setInterval(refresh,2000);document.addEventListener("visibilitychange",refresh);return()=>{window.clearInterval(timer);document.removeEventListener("visibilitychange",refresh)}},[tab,loadDownloads]); useEffect(()=>{if(tab!=="downloads")return;void loadDownloads();const refresh=()=>{if(!document.hidden)void loadDownloads()};const timer=window.setInterval(refresh,2000);document.addEventListener("visibilitychange",refresh);return()=>{window.clearInterval(timer);document.removeEventListener("visibilitychange",refresh)}},[tab,loadDownloads]);
@@ -968,16 +971,36 @@ function UploadsPage({ uploads, uploadPage, downloads, onDownloads, sources, rel
async function createTorrent(file?:File){if(!file||!sourceId)return;setBusy(true);setError("");try{const form=new FormData();form.append("file",file);form.append("source_id",sourceId);form.append("relative_path",relativePath);await api("/downloads/torrent",{method:"POST",body:form});setShowAdd(false);await loadDownloads()}catch(reason){setError((reason as Error).message)}finally{setBusy(false);if(torrentInput.current)torrentInput.current.value=""}} async function createTorrent(file?:File){if(!file||!sourceId)return;setBusy(true);setError("");try{const form=new FormData();form.append("file",file);form.append("source_id",sourceId);form.append("relative_path",relativePath);await api("/downloads/torrent",{method:"POST",body:form});setShowAdd(false);await loadDownloads()}catch(reason){setError((reason as Error).message)}finally{setBusy(false);if(torrentInput.current)torrentInput.current.value=""}}
async function action(task:DownloadTask,name:"pause"|"resume"|"retry"){try{await api(`/downloads/${task.id}/${name}`,{method:"POST",body:"{}"});await loadDownloads()}catch(reason){setError((reason as Error).message)}} async function action(task:DownloadTask,name:"pause"|"resume"|"retry"){try{await api(`/downloads/${task.id}/${name}`,{method:"POST",body:"{}"});await loadDownloads()}catch(reason){setError((reason as Error).message)}}
async function removeDownload(task:DownloadTask){if(!await appConfirm(`移除“${task.display_name}`,"未完成的暂存文件会一并删除;已写入媒体库的文件不会受影响。",true))return;await api(`/downloads/${task.id}`,{method:"DELETE"});await loadDownloads()} async function removeDownload(task:DownloadTask){if(!await appConfirm(`移除“${task.display_name}`,"未完成的暂存文件会一并删除;已写入媒体库的文件不会受影响。",true))return;await api(`/downloads/${task.id}`,{method:"DELETE"});await loadDownloads()}
async function changePage(next:number){if(paging||next<1||next>uploadPage.pages||next===uploadPage.page)return;setPaging(true);try{await onPage(next);document.querySelector(".transfer-center")?.scrollIntoView({block:"start",behavior:window.matchMedia("(prefers-reduced-motion: reduce)").matches?"auto":"smooth"})}finally{setPaging(false)}} async function fetchFiltered(page:number,next:UploadFilters){const params=new URLSearchParams({page:String(page),page_size:"10",status:next.status});if(next.source_id)params.set("source_id",next.source_id);if(next.query)params.set("query",next.query);setFilteredPage(await api<UploadPage>(`/uploads?${params}`))}
const attentionCount=uploadPage.active_count+uploadPage.failed_count; async function refreshUploadView(){if(filteredPage)await fetchFiltered(filteredPage.page,filters);else reload()}
const historyCount=Math.max(0,uploadPage.total-attentionCount); async function applyFilters(next:UploadFilters){setFilters(next);setSelected(new Set());setPaging(true);try{await fetchFiltered(1,next)}catch(reason){setError((reason as Error).message)}finally{setPaging(false)}}
async function taskAction(task:UploadTask,action:"cancel"|"retry"|"discard"){
if(taskBusy.has(task.id))return;
const title=task.title?.trim()||cleanMediaTitle(task.filename);
if(action==="cancel"&&!await appConfirm(`取消“${title}`,"任务会停止,但恢复副本会保留,之后仍可重试。",true))return;
if(action==="discard"&&!await appConfirm(`删除“${title}”及恢复副本`,"这会永久删除上传暂存和任务记录,无法撤销。",true))return;
setTaskBusy(current=>new Set(current).add(task.id));setError("");
try{await api(action==="discard"?`/uploads/${task.id}/recovery`:action==="retry"?`/uploads/${task.id}/retry`:`/uploads/${task.id}`,{method:action==="retry"?"POST":"DELETE",body:action==="retry"?"{}":undefined});await refreshUploadView()}catch(reason){setError((reason as Error).message)}finally{setTaskBusy(current=>{const next=new Set(current);next.delete(task.id);return next})}
}
async function bulkAction(action:"cancel"|"retry"){
const ids=[...selected].filter(id=>{const task=visibleUploads.find(item=>item.id===id);return action==="cancel"?task?.can_cancel:task?.can_retry});if(!ids.length)return;
if(action==="cancel"&&!await appConfirm(`取消 ${ids.length} 个上传任务`,"任务会停止,所有可恢复副本都会保留。",true))return;
setBusy(true);setError("");try{const response=await api<{results:Array<{id:string;ok:boolean;error?:string}>}>("/uploads/bulk-actions",{method:"POST",body:JSON.stringify({upload_ids:ids,action})});const failures=response.results.filter(item=>!item.ok);if(failures.length)setError(`${response.results.length-failures.length} 条成功,${failures.length} 条失败:${failures[0].error||"状态不允许"}`);setSelected(new Set());await refreshUploadView()}catch(reason){setError((reason as Error).message)}finally{setBusy(false)}
}
async function changePage(next:number){if(paging||next<1||next>visiblePage.pages||next===visiblePage.page)return;setPaging(true);try{if(filteredPage)await fetchFiltered(next,filters);else await onPage(next);document.querySelector(".transfer-center")?.scrollIntoView({block:"start",behavior:window.matchMedia("(prefers-reduced-motion: reduce)").matches?"auto":"smooth"})}finally{setPaging(false)}}
const attentionCount=visiblePage.active_count+visiblePage.failed_count;
const historyCount=Math.max(0,visiblePage.total-attentionCount);
const uploadControls=<><form className="upload-task-filters" onSubmit={event=>{event.preventDefault();void applyFilters({...filters,query:queryDraft.trim()})}}><label><span></span><select value={filters.status} onChange={event=>void applyFilters({...filters,status:event.target.value as UploadFilters["status"]})}><option value="all"></option><option value="active"></option><option value="failed"></option><option value="completed"></option><option value="cancelled"></option></select></label><label><span></span><select value={filters.source_id} onChange={event=>void applyFilters({...filters,source_id:event.target.value})}><option value=""></option>{sources.map(source=><option key={source.id} value={source.id}>{source.name}</option>)}</select></label><label className="upload-task-search"><span></span><input value={queryDraft} onChange={event=>setQueryDraft(event.target.value)} placeholder="文件名或标题"/><button className="secondary" disabled={paging}><Search/></button></label></form>{selected.size>0&&<div className="upload-selection-bar" role="status"><strong> {selected.size} </strong><span/><button className="secondary" disabled={busy||![...selected].some(id=>uploads.find(item=>item.id===id)?.can_retry)} onClick={()=>void bulkAction("retry")}><RefreshCw/></button><button className="danger-text" disabled={busy||![...selected].some(id=>uploads.find(item=>item.id===id)?.can_cancel)} onClick={()=>void bulkAction("cancel")}></button><button className="icon-button" aria-label="清除选择" onClick={()=>setSelected(new Set())}><X/></button></div>}</>;
function renderUploadTask(task:UploadTask){ function renderUploadTask(task:UploadTask){
const target=sources.find(source=>source.id===task.source_id);const completed=task.status==="completed";const autoRetry=task.status==="queued"&&!!task.retry_count;const transferred=task.transferred_bytes?`${formatSize(task.transferred_bytes)} / ${formatSize(task.size_bytes)}`:"";const title=task.title?.trim()||cleanMediaTitle(task.filename);const stages=task.stages||[]; const target=sources.find(source=>source.id===task.source_id);const completed=task.status==="completed";const autoRetry=task.status==="queued"&&!!task.retry_count;const transferred=task.transferred_bytes?`${formatSize(task.transferred_bytes)} / ${formatSize(task.size_bytes)}`:"";const title=task.title?.trim()||cleanMediaTitle(task.filename);const stages=task.stages||[];
return <article className={completed?"completed-history":""} key={task.id}><span className={`transfer-icon ${task.status}`}>{completed?<CheckCircle2/>:task.status==="failed"?<AlertCircle/>:<Upload/>}</span><div className="transfer-main"><div><strong>{title}</strong><span>{task.filename} · {task.stage_label||({receiving:"正在接收",queued:"等待上传",transferring:"正在转存",indexing:"正在加入媒体库",completed:"已完成",failed:"失败",cancelled:"已取消"} as Record<string,string>)[task.status]||"上传处理中"} · {formatSize(task.size_bytes)}</span></div><p className="transfer-target">{target?.name||"媒体库"} · {target?sourceLabel(target):"媒体来源"}</p>{completed?<p className="transfer-completed"><CheckCircle2/>AI </p>:<div className="transfer-stages">{stages.map(step=><span className={step.state==="completed"?"done":step.state} key={step.key}><i><b style={{transform:`scaleX(${Math.max(0,Math.min(1,step.progress||0))})`}}/></i><small>{step.label}</small><em>{step.state==="completed"?"完成":step.state==="active"?`${Math.round((step.progress||0)*100)}%`:step.state==="failed"?"失败":step.state==="cancelled"?"已取消":"等待"}</em></span>)}</div>}<div className="transfer-meta">{task.deduplicated&&<b>SHA-256 </b>}{task.resume_mode==="offset"&&<b></b>}{autoRetry&&<b></b>}{transferred&&<span>{transferred}</span>}</div>{!completed&&<p>{task.error||task.message||"等待更新"}</p>}</div><div className="transfer-actions">{task.can_retry&&<button onClick={()=>api(`/uploads/${task.id}/retry`,{method:"POST",body:"{}"}).then(reload)}><RefreshCw/>{task.failure_stage==="catalog"?"重新入库":"重试"}</button>}{task.can_cancel&&<button className="danger-text" onClick={()=>api(`/uploads/${task.id}`,{method:"DELETE"}).then(reload)}></button>}</div></article>; const checked=selected.has(task.id);const operating=taskBusy.has(task.id);
// The ternary mutates a local Set copy and returns the boolean mutation result unused.
// eslint-disable-next-line @typescript-eslint/no-unused-expressions
return <article className={`${completed?"completed-history ":""}${checked?"selected":""}`} key={task.id}><label className="transfer-select"><input type="checkbox" checked={checked} onChange={()=>setSelected(current=>{const next=new Set(current);next.has(task.id)?next.delete(task.id):next.add(task.id);return next})}/><span className="sr-only"> {title}</span></label><span className={`transfer-icon ${task.status}`}>{completed?<CheckCircle2/>:task.status==="failed"?<AlertCircle/>:<Upload/>}</span><div className="transfer-main"><div><strong>{title}</strong><span>{task.filename} · {task.stage_label||({receiving:"正在接收",queued:"等待上传",transferring:"正在转存",indexing:"正在加入媒体库",completed:"已完成",failed:"失败",cancelled:"已取消"} as Record<string,string>)[task.status]||"上传处理中"} · {formatSize(task.size_bytes)}</span></div><p className="transfer-target">{target?.name||"媒体库"} · {target?sourceLabel(target):"媒体来源"}</p>{completed?<p className="transfer-completed"><CheckCircle2/>AI </p>:<div className="transfer-stages">{stages.map(step=><span className={step.state==="completed"?"done":step.state} key={step.key}><i><b style={{transform:`scaleX(${Math.max(0,Math.min(1,step.progress||0))})`}}/></i><small>{step.label}</small><em>{step.state==="completed"?"完成":step.state==="active"?`${Math.round((step.progress||0)*100)}%`:step.state==="failed"?"失败":step.state==="cancelled"?"已取消":"等待"}</em></span>)}</div>}<div className="transfer-meta">{task.deduplicated&&<b>SHA-256 </b>}{task.resume_mode==="offset"&&<b></b>}{task.recovery_state==="available"&&<b></b>}{task.requires_reupload&&<b className="failed"></b>}{autoRetry&&<b></b>}{transferred&&<span>{transferred}</span>}</div>{!completed&&<p>{task.error||task.message||"等待更新"}</p>}</div><div className="transfer-actions">{task.can_retry&&<button disabled={operating} onClick={()=>void taskAction(task,"retry")}><RefreshCw className={operating?"spin":""}/>{task.failure_stage==="catalog"?"重新入库":"重试"}</button>}{task.can_cancel&&<button disabled={operating} className="danger-text" onClick={()=>void taskAction(task,"cancel")}></button>}{task.can_discard&&<button disabled={operating} className="danger-text" onClick={()=>void taskAction(task,"discard")}><Trash2/></button>}</div></article>;
} }
return <div className="page transfer-center"><header className="page-heading"><button className="icon-button page-back" aria-label="返回" onClick={()=>window.history.back()}><ArrowLeft/></button><div><p className="eyebrow"></p><h1></h1><p>aria2 </p></div><div className="page-heading-actions"><button className="secondary" onClick={()=>tab==="uploads"?reload():loadDownloads()}><RefreshCw/></button>{tab==="uploads"?<button className="primary continue-upload" onClick={onUpload}><Upload/></button>:<button className="primary" onClick={()=>setShowAdd(true)}><Download/></button>}</div></header> return <div className="page transfer-center"><header className="page-heading"><button className="icon-button page-back" aria-label="返回" onClick={()=>window.history.back()}><ArrowLeft/></button><div><p className="eyebrow"></p><h1></h1><p>aria2 </p></div><div className="page-heading-actions"><button className="secondary" onClick={()=>tab==="uploads"?reload():loadDownloads()}><RefreshCw/></button>{tab==="uploads"?<button className="primary continue-upload" onClick={onUpload}><Upload/></button>:<button className="primary" onClick={()=>setShowAdd(true)}><Download/></button>}</div></header>
<nav className="segment-nav transfer-tabs"><button className={tab==="uploads"?"active":""} onClick={()=>setTab("uploads")}> {attentionCount>0&&<b aria-label={`${attentionCount} 条需要关注`}>{attentionCount}</b>}</button><button className={tab==="downloads"?"active":""} onClick={()=>setTab("downloads")}> <b>{downloads.length}</b></button></nav> <nav className="segment-nav transfer-tabs"><button className={tab==="uploads"?"active":""} onClick={()=>setTab("uploads")}> {attentionCount>0&&<b aria-label={`${attentionCount} 条需要关注`}>{attentionCount}</b>}</button><button className={tab==="downloads"?"active":""} onClick={()=>setTab("downloads")}> <b>{downloads.length}</b></button></nav>
{error&&<div className="error banner"><AlertCircle/>{error}</div>} {error&&<div className="error banner"><AlertCircle/>{error}</div>}{tab==="uploads"&&uploadControls}
{tab==="uploads"?<><div className="transfer-summary" role="status" aria-label={`上传记录:进行中 ${uploadPage.active_count},失败 ${uploadPage.failed_count},历史 ${historyCount}`}><span className="active"><Upload/> <b>{uploadPage.active_count}</b></span><span className={uploadPage.failed_count?"failed":""}><AlertCircle/> <b>{uploadPage.failed_count}</b></span><span><Clock3/> <b>{historyCount}</b></span></div><div className="transfer-list">{uploads.map(renderUploadTask)}{!uploads.length&&<div className="empty"><Upload/><h3></h3><p></p></div>}</div>{uploadPage.pages>1&&<nav className="transfer-pagination" aria-label="上传任务分页"><button className="secondary" disabled={paging||uploadPage.page<=1} onClick={()=>void changePage(uploadPage.page-1)}></button><span> {uploadPage.page} / {uploadPage.pages} <small> {uploadPage.total} </small></span><button className="secondary" disabled={paging||uploadPage.page>=uploadPage.pages} onClick={()=>void changePage(uploadPage.page+1)}></button></nav>}</>:<div className="transfer-list download-list">{runtimeLoad.phase==="loading"||runtimeLoad.phase==="idle"?<AsyncNotice label="正在检查 aria2 与后台下载…"/>:runtimeLoad.phase==="error"?<div className="empty compact async-error"><AlertCircle/><h3></h3><p>{runtimeLoad.error}</p><button className="secondary" onClick={()=>void loadDownloads()}><RefreshCw/></button></div>:<>{runtime&&!runtime.available&&<div className="warning-callout"><AlertCircle/><span><strong> aria2c </strong> aria2ImageFind </span></div>}{downloads.map(task=>{const target=sources.find(source=>source.id===task.source_id);return <article key={task.id}><span className={`transfer-icon ${task.status}`}>{task.status==="completed"?<CheckCircle2/>:task.status==="failed"?<AlertCircle/>:<Download/>}</span><div className="transfer-main"><div><strong>{task.display_name}</strong><span>{labels[task.status]||task.status} · {Math.round(task.progress*100)}%</span></div><p className="transfer-target">{target?.name||"媒体库"}/{task.relative_path} · {task.download_speed?`${formatSize(task.download_speed)}/s`:formatSize(task.total_bytes)}</p><i className="download-progress"><b style={{transform:`scaleX(${Math.max(0,Math.min(1,task.progress))})`}}/></i><p>{task.error||`${formatSize(task.completed_bytes)} / ${formatSize(task.total_bytes)}`}</p></div><div className="transfer-actions">{task.status==="downloading"&&<button onClick={()=>action(task,"pause")}><Pause/></button>}{task.status==="paused"&&<button onClick={()=>action(task,"resume")}><Play/></button>}{task.status==="failed"&&<button onClick={()=>action(task,"retry")}><RefreshCw/></button>}<button className="danger-text" onClick={()=>removeDownload(task)}><Trash2/></button></div></article>})}{!downloads.length&&<div className="empty"><Download/><h3></h3><p> HTTP/HTTPS/FTP Torrent</p></div>}</>}</div>} {tab==="uploads"?<><div className="transfer-summary" role="status" aria-label={`上传记录:进行中 ${uploadPage.active_count},失败 ${uploadPage.failed_count},历史 ${historyCount}`}><span className="active"><Upload/> <b>{uploadPage.active_count}</b></span><span className={uploadPage.failed_count?"failed":""}><AlertCircle/> <b>{uploadPage.failed_count}</b></span><span><Clock3/> <b>{historyCount}</b></span></div><div className="transfer-list">{uploads.map(renderUploadTask)}{!uploads.length&&<div className="empty"><Upload/><h3></h3><p></p></div>}</div>{uploadPage.pages>1&&<nav className="transfer-pagination" aria-label="上传任务分页"><button className="secondary" disabled={paging||uploadPage.page<=1} onClick={()=>void changePage(uploadPage.page-1)}></button><span> {uploadPage.page} / {uploadPage.pages} <small> {uploadPage.total} </small></span><button className="secondary" disabled={paging||uploadPage.page>=uploadPage.pages} onClick={()=>void changePage(uploadPage.page+1)}></button></nav>}</>:<div className="transfer-list download-list">{runtimeLoad.phase==="loading"||runtimeLoad.phase==="idle"?<AsyncNotice label="正在检查 aria2 与后台下载…"/>:runtimeLoad.phase==="error"?<div className="empty compact async-error"><AlertCircle/><h3></h3><p>{runtimeLoad.error}</p><button className="secondary" onClick={()=>void loadDownloads()}><RefreshCw/></button></div>:<>{runtime&&!runtime.available&&<div className="warning-callout"><AlertCircle/><span><strong> aria2c </strong> aria2ImageFind </span></div>}{downloads.map(task=>{const target=sources.find(source=>source.id===task.source_id);return <article key={task.id}><span className={`transfer-icon ${task.status}`}>{task.status==="completed"?<CheckCircle2/>:task.status==="failed"?<AlertCircle/>:<Download/>}</span><div className="transfer-main"><div><strong>{task.display_name}</strong><span>{labels[task.status]||task.status} · {Math.round(task.progress*100)}%</span></div><p className="transfer-target">{target?.name||"媒体库"}/{task.relative_path} · {task.download_speed?`${formatSize(task.download_speed)}/s`:formatSize(task.total_bytes)}</p><i className="download-progress"><b style={{transform:`scaleX(${Math.max(0,Math.min(1,task.progress))})`}}/></i><p>{task.error||`${formatSize(task.completed_bytes)} / ${formatSize(task.total_bytes)}`}</p></div><div className="transfer-actions">{task.status==="downloading"&&<button onClick={()=>action(task,"pause")}><Pause/></button>}{task.status==="paused"&&<button onClick={()=>action(task,"resume")}><Play/></button>}{task.status==="failed"&&<button onClick={()=>action(task,"retry")}><RefreshCw/></button>}<button className="danger-text" onClick={()=>removeDownload(task)}><Trash2/></button></div></article>})}{!downloads.length&&<div className="empty"><Download/><h3></h3><p> HTTP/HTTPS/FTP Torrent</p></div>}</>}</div>}
{showAdd&&<div className="modal-backdrop"><form className="dialog download-dialog" onSubmit={createDownload}><header><div><p className="eyebrow"></p><h2></h2></div><button type="button" className="icon-button" onClick={()=>setShowAdd(false)}><X/></button></header><label><input autoFocus value={url} onChange={event=>setUrl(event.target.value)} placeholder="https://…、ftp://… 或 magnet:?…"/></label><label><select value={sourceId} onChange={event=>setSourceId(event.target.value)}>{writable.map(source=><option key={source.id} value={source.id}>{source.name}</option>)}</select></label><label><input value={relativePath} onChange={event=>setRelativePath(event.target.value)} placeholder="可留空,Torrent 会保留安全目录结构"/></label><button type="button" className="secondary wide" onClick={()=>torrentInput.current?.click()}><Folder/> .torrent </button><input ref={torrentInput} hidden type="file" accept=".torrent,application/x-bittorrent" onChange={event=>createTorrent(event.target.files?.[0])}/>{runtimeLoad.phase!=="ready"?<AsyncNotice compact label="正在检查 aria2…"/>:!runtime?.available&&<div className="error"><AlertCircle/> aria2c</div>}<footer><button type="button" className="secondary" onClick={()=>setShowAdd(false)}></button><button className="primary" disabled={busy||runtimeLoad.phase!=="ready"||!runtime?.available||!url.trim()||!sourceId}>{busy?<RefreshCw className="spin"/>:<Download/>}</button></footer></form></div>} {showAdd&&<div className="modal-backdrop"><form className="dialog download-dialog" onSubmit={createDownload}><header><div><p className="eyebrow"></p><h2></h2></div><button type="button" className="icon-button" onClick={()=>setShowAdd(false)}><X/></button></header><label><input autoFocus value={url} onChange={event=>setUrl(event.target.value)} placeholder="https://…、ftp://… 或 magnet:?…"/></label><label><select value={sourceId} onChange={event=>setSourceId(event.target.value)}>{writable.map(source=><option key={source.id} value={source.id}>{source.name}</option>)}</select></label><label><input value={relativePath} onChange={event=>setRelativePath(event.target.value)} placeholder="可留空,Torrent 会保留安全目录结构"/></label><button type="button" className="secondary wide" onClick={()=>torrentInput.current?.click()}><Folder/> .torrent </button><input ref={torrentInput} hidden type="file" accept=".torrent,application/x-bittorrent" onChange={event=>createTorrent(event.target.files?.[0])}/>{runtimeLoad.phase!=="ready"?<AsyncNotice compact label="正在检查 aria2…"/>:!runtime?.available&&<div className="error"><AlertCircle/> aria2c</div>}<footer><button type="button" className="secondary" onClick={()=>setShowAdd(false)}></button><button className="primary" disabled={busy||runtimeLoad.phase!=="ready"||!runtime?.available||!url.trim()||!sourceId}>{busy?<RefreshCw className="spin"/>:<Download/>}</button></footer></form></div>}
</div>; </div>;
@@ -1460,11 +1483,14 @@ export default function App() {
const loadSources=useCallback((quiet=false)=>runResource("sources",()=>api<Source[]>("/sources"),setSources,quiet),[runResource]); const loadSources=useCallback((quiet=false)=>runResource("sources",()=>api<Source[]>("/sources"),setSources,quiet),[runResource]);
const loadHome=useCallback((quiet=false)=>runResource("home",()=>api<HomeFeed>("/home?item_limit=10&tag_limit=3"),setHomeFeed,quiet),[runResource]); const loadHome=useCallback((quiet=false)=>runResource("home",()=>api<HomeFeed>("/home?item_limit=10&tag_limit=3"),setHomeFeed,quiet),[runResource]);
const loadVideos=useCallback((quiet=false)=>runResource("videos",()=>api<Video[]>("/videos?limit=200"),setVideos,quiet),[runResource]); const loadVideos=useCallback((quiet=false)=>runResource("videos",()=>api<Video[]>("/videos?limit=200"),setVideos,quiet),[runResource]);
const loadUploads=useCallback((quiet=false,requestedPage?:number)=>{ const uploadFiltersRef=useRef<UploadFilters>({status:"all",source_id:"",query:""});
const loadUploads=useCallback((quiet=false,requestedPage?:number,nextFilters?:UploadFilters)=>{
const target=requestedPage??uploadPageNumber.current; const target=requestedPage??uploadPageNumber.current;
if(nextFilters)uploadFiltersRef.current=nextFilters;const filters=uploadFiltersRef.current;
const requestSequence=++uploadRequestSequence.current; const requestSequence=++uploadRequestSequence.current;
if(requestedPage!==undefined)uploadPageNumber.current=target; if(requestedPage!==undefined)uploadPageNumber.current=target;
return runResource("uploads",()=>api<UploadPage|UploadTask[]>(`/uploads?page=${target}&page_size=10`),value=>{ const params=new URLSearchParams({page:String(target),page_size:"10",status:filters.status});if(filters.source_id)params.set("source_id",filters.source_id);if(filters.query)params.set("query",filters.query);
return runResource("uploads",()=>api<UploadPage|UploadTask[]>(`/uploads?${params}`),value=>{
if(requestSequence!==uploadRequestSequence.current)return; if(requestSequence!==uploadRequestSequence.current)return;
const normalized=normalizeUploadPage(value,target);uploadPageNumber.current=normalized.page;setUploadPage(normalized);setUploads(normalized.status_items) const normalized=normalizeUploadPage(value,target);uploadPageNumber.current=normalized.page;setUploadPage(normalized);setUploads(normalized.status_items)
},quiet); },quiet);
+6
View File
@@ -615,6 +615,12 @@ html[data-theme="dark"] :is(.upload-batch-metadata,.upload-tag-groups fieldset,.
@media(max-width:720px){.upload-batch-metadata{padding:11px}.upload-batch-metadata>header{align-items:flex-start}.upload-batch-metadata>header button{min-height:44px}.upload-tag-groups button,.collection-tag-rules fieldset button{min-height:44px}.collection-hero>div:last-child{grid-template-columns:1fr 1fr}.collection-tag-rules>div{grid-template-columns:1fr}.collection-tag-rules>footer{align-items:stretch;flex-direction:column}.collection-tag-rules>footer .primary{min-height:44px}.webdav-connection{grid-template-columns:1fr}.webdav-connection>button{min-height:44px}.player-marker-drawer{top:8px;right:8px;bottom:52px;width:calc(100% - 16px);max-height:none;padding:9px}.player-marker-drawer>header>button,.player-marker-list article>button:not(.marker-jump){width:40px;height:40px}.player-marker-add{min-height:40px}.player-marker-list article{grid-template-columns:minmax(0,1fr) 40px 40px}.marker-jump{min-height:40px}.pull-refresh{top:0}} @media(max-width:720px){.upload-batch-metadata{padding:11px}.upload-batch-metadata>header{align-items:flex-start}.upload-batch-metadata>header button{min-height:44px}.upload-tag-groups button,.collection-tag-rules fieldset button{min-height:44px}.collection-hero>div:last-child{grid-template-columns:1fr 1fr}.collection-tag-rules>div{grid-template-columns:1fr}.collection-tag-rules>footer{align-items:stretch;flex-direction:column}.collection-tag-rules>footer .primary{min-height:44px}.webdav-connection{grid-template-columns:1fr}.webdav-connection>button{min-height:44px}.player-marker-drawer{top:8px;right:8px;bottom:52px;width:calc(100% - 16px);max-height:none;padding:9px}.player-marker-drawer>header>button,.player-marker-list article>button:not(.marker-jump){width:40px;height:40px}.player-marker-add{min-height:40px}.player-marker-list article{grid-template-columns:minmax(0,1fr) 40px 40px}.marker-jump{min-height:40px}.pull-refresh{top:0}}
@media(prefers-reduced-motion:reduce){.player-marker-dot:after,.pull-refresh{transition:none}.pull-refresh svg{transform:none}} @media(prefers-reduced-motion:reduce){.player-marker-dot:after,.pull-refresh{transition:none}.pull-refresh svg{transform:none}}
/* Upload task management and mobile-safe controls */
.upload-task-filters{display:grid;grid-template-columns:minmax(150px,190px) minmax(180px,240px) minmax(260px,1fr);gap:10px;align-items:end;margin-bottom:14px;padding:13px;border:1px solid var(--line);border-radius:14px;background:var(--surface)}.upload-task-filters label{min-width:0;display:grid;gap:6px;color:var(--muted);font-size:11px;font-weight:700}.upload-task-filters select,.upload-task-filters input{width:100%;height:42px;padding:0 11px;color:var(--text);border:1px solid var(--line);border-radius:9px;outline:0;background:var(--surface);font:inherit}.upload-task-filters :is(select,input):focus{border-color:var(--brand);box-shadow:0 0 0 3px color-mix(in srgb,var(--brand) 14%,transparent)}.upload-task-search{grid-template-columns:minmax(0,1fr) auto}.upload-task-search>span{grid-column:1/-1}.upload-task-search button{height:42px}.upload-selection-bar{position:sticky;z-index:12;top:calc(var(--top) + 8px);display:grid;grid-template-columns:auto 1fr auto auto 40px;gap:8px;align-items:center;margin-bottom:12px;padding:9px 10px;border:1px solid #bed0f1;border-radius:12px;background:color-mix(in srgb,var(--surface) 94%,var(--brand-soft));box-shadow:0 8px 24px #24324c14}.upload-selection-bar strong{font-size:12px}.upload-selection-bar button{min-height:40px}.upload-task-list>article{grid-template-columns:24px 42px minmax(0,1fr) auto}.upload-task-list>article.selected{border-color:#93b3ec;background:color-mix(in srgb,var(--surface) 92%,var(--brand-soft))}.transfer-select{width:24px;height:44px;display:grid;place-items:center;align-self:start}.transfer-select input{width:18px;height:18px;accent-color:var(--brand)}.transfer-meta b.failed{color:var(--red);background:#fff0f1}.transfer-actions{flex-wrap:wrap;justify-content:flex-end}.transfer-actions button:disabled{cursor:wait;opacity:.55}
html[data-theme="dark"] .upload-task-filters{border-color:var(--line);background:var(--surface)}html[data-theme="dark"] .upload-selection-bar{border-color:#30476f;background:#1d2b47}html[data-theme="dark"] .transfer-meta b.failed{color:#ff9aa4;background:#381d22}
@media(max-width:720px){.mobile-page-uploads{--upload-action-height:52px;--upload-action-gap:14px;padding-bottom:calc(var(--upload-action-height) + var(--upload-action-gap) + 62px + env(safe-area-inset-bottom))}.upload-task-filters{grid-template-columns:1fr 1fr;padding:11px}.upload-task-search{grid-column:1/-1}.upload-task-filters select,.upload-task-filters input{min-height:44px;font-size:16px}.upload-task-search button{min-width:76px;height:44px}.upload-selection-bar{top:8px;grid-template-columns:1fr 44px;margin-right:-2px;margin-left:-2px}.upload-selection-bar>span{display:none}.upload-selection-bar>button:not(.icon-button){min-height:44px}.upload-selection-bar .secondary,.upload-selection-bar .danger-text{grid-row:2}.upload-selection-bar .secondary{grid-column:1}.upload-selection-bar .danger-text{grid-column:2}.upload-selection-bar .icon-button{grid-row:1;grid-column:2}.upload-task-list>article{grid-template-columns:24px 40px minmax(0,1fr)}.upload-task-list .transfer-main{grid-column:3}.upload-task-list .transfer-actions{grid-column:2/4;justify-content:stretch}.upload-task-list .transfer-actions button{flex:1;min-width:min(100%,130px)}.transfer-pagination{position:relative;z-index:4;margin-bottom:calc(var(--upload-action-height) + var(--upload-action-gap));padding:8px}.mobile-page-uploads .continue-upload{bottom:calc(62px + max(12px,env(safe-area-inset-bottom)))} }
@media(max-width:360px){.upload-task-filters{grid-template-columns:1fr}.upload-task-search{grid-column:auto}.transfer-pagination{gap:6px}.transfer-pagination button{min-width:70px;padding-inline:8px}}
/* 0.3.40 mobile refresh, native WebDAV and player interaction polish */ /* 0.3.40 mobile refresh, native WebDAV and player interaction polish */
.pull-refresh svg{transform:rotate(var(--pull-rotation))} .pull-refresh svg{transform:rotate(var(--pull-rotation))}
.webdav-fields label{display:grid;gap:6px;color:var(--muted);font-size:11px;font-weight:700}.webdav-fields input,.webdav-fields select{width:100%;height:44px;padding:0 36px 0 11px;color:var(--text);border:1px solid var(--line);border-radius:10px;outline:0;background:var(--surface);font:inherit}.webdav-fields input{padding-right:11px}.webdav-fields input:focus,.webdav-fields select:focus{border-color:var(--brand);box-shadow:0 0 0 3px color-mix(in srgb,var(--brand) 14%,transparent)}.webdav-connection>span:nth-of-type(2){grid-column:auto}.webdav-token-create{align-self:end}.webdav-token-once{grid-column:1/-1;display:grid;grid-template-columns:minmax(0,1fr) auto;gap:7px;align-items:center;padding:10px;color:#176447;border:1px solid #bfe5d4;border-radius:9px;background:#edfaf4}.webdav-token-once>strong{grid-column:1/-1;font-size:10px}.webdav-token-once>code{min-width:0;overflow-wrap:anywhere;font-size:10px}.webdav-token-once>button{min-height:38px}.webdav-connection p b{color:var(--text)}.preference-item.disabled{opacity:.62}.preference-switch:disabled{cursor:not-allowed}.job-row-actions{display:flex;align-items:center;gap:7px} .webdav-fields label{display:grid;gap:6px;color:var(--muted);font-size:11px;font-weight:700}.webdav-fields input,.webdav-fields select{width:100%;height:44px;padding:0 36px 0 11px;color:var(--text);border:1px solid var(--line);border-radius:10px;outline:0;background:var(--surface);font:inherit}.webdav-fields input{padding-right:11px}.webdav-fields input:focus,.webdav-fields select:focus{border-color:var(--brand);box-shadow:0 0 0 3px color-mix(in srgb,var(--brand) 14%,transparent)}.webdav-connection>span:nth-of-type(2){grid-column:auto}.webdav-token-create{align-self:end}.webdav-token-once{grid-column:1/-1;display:grid;grid-template-columns:minmax(0,1fr) auto;gap:7px;align-items:center;padding:10px;color:#176447;border:1px solid #bfe5d4;border-radius:9px;background:#edfaf4}.webdav-token-once>strong{grid-column:1/-1;font-size:10px}.webdav-token-once>code{min-width:0;overflow-wrap:anywhere;font-size:10px}.webdav-token-once>button{min-height:38px}.webdav-connection p b{color:var(--text)}.preference-item.disabled{opacity:.62}.preference-switch:disabled{cursor:not-allowed}.job-row-actions{display:flex;align-items:center;gap:7px}
+49 -24
View File
@@ -29,22 +29,44 @@ def arguments() -> argparse.Namespace:
return parser.parse_args() return parser.parse_args()
def worker_run(data_dir: Path, wav: Path, window: dict, combination: tuple[str, ...]) -> dict: def _write_window_batch(source: Path, destination: Path, windows: list[dict]) -> int:
"""Concatenate equal-sized diagnostic windows without reloading Whisper."""
with wave.open(str(source), "rb") as handle:
params = handle.getparams()
sample_rate = handle.getframerate()
frame_count = handle.getnframes()
durations = {int(item["end_ms"]) - int(item["start_ms"]) for item in windows}
if len(durations) != 1:
raise ValueError("all diagnostic windows for one media item must have equal duration")
duration_ms = durations.pop()
frames_per_window = duration_ms * sample_rate // 1000
with wave.open(str(destination), "wb") as output:
output.setparams(params)
for item in windows:
start_frame = int(item["start_ms"]) * sample_rate // 1000
handle.setpos(min(start_frame, frame_count))
raw = handle.readframes(frames_per_window)
expected = frames_per_window * params.nchannels * params.sampwidth
if len(raw) < expected:
raw += b"\0" * (expected - len(raw))
output.writeframesraw(raw)
return duration_ms
def worker_run(data_dir: Path, wav: Path, windows: list[dict], combination: tuple[str, ...]) -> list[dict]:
name, model, device, segmentation = combination name, model, device, segmentation = combination
with wave.open(str(wav), "rb") as handle: with wave.open(str(wav), "rb") as handle:
sample_rate = handle.getframerate() duration_ms = round(handle.getnframes() / handle.getframerate() * 1000 / len(windows))
start_ms = int(window["start_ms"])
duration_ms = int(window["end_ms"]) - start_ms
command = [ command = [
sys.executable, "-m", "imagefind.audio_worker", sys.executable, "-m", "imagefind.audio_worker",
"--data-dir", str(data_dir), "--wav", str(wav), "--data-dir", str(data_dir), "--wav", str(wav),
"--device", device, "--model-variant", model, "--device", device, "--model-variant", model,
"--segmentation", segmentation, "--quality-profile", "accuracy", "--segmentation", segmentation, "--quality-profile", "accuracy",
"--language-policy", str(window.get("language_policy", "zh_priority")), "--language-policy", str(windows[0].get("language_policy", "zh_priority")),
"--chunk-seconds", str(max(15, min(30, (duration_ms + 999) // 1000))), "--chunk-seconds", str(max(15, min(30, (duration_ms + 999) // 1000))),
"--overlap-seconds", "0", "--overlap-seconds", "0",
"--start-frame", str(start_ms * sample_rate // 1000), "--start-frame", "0",
"--max-chunks", "1", "--cpu-threads", "2", "--max-chunks", str(len(windows)), "--cpu-threads", "2",
] ]
started = time.monotonic() started = time.monotonic()
process = subprocess.run(command, text=True, capture_output=True, encoding="utf-8", errors="replace") process = subprocess.run(command, text=True, capture_output=True, encoding="utf-8", errors="replace")
@@ -55,19 +77,21 @@ def worker_run(data_dir: Path, wav: Path, window: dict, combination: tuple[str,
events.append(json.loads(line)) events.append(json.loads(line))
except json.JSONDecodeError: except json.JSONDecodeError:
pass pass
return { results = []
"combination": name, for index, window in enumerate(windows, 1):
"model": model, selected = [event for event in events if event.get("chunk_index") == index]
"device": device, if index == len(windows):
"beam": 1 if device == "GPU" else 5, selected.extend(event for event in events if event.get("event") in {"complete", "error"})
"segmentation": segmentation, results.append({
"elapsed_seconds": round(elapsed, 3), "combination": name, "model": model, "device": device,
"window_seconds": round(duration_ms / 1000, 3), "beam": 1 if device == "GPU" else 5, "segmentation": segmentation,
"rtf": round(elapsed / max(0.001, duration_ms / 1000), 3), "elapsed_seconds": round(elapsed, 3), "window_seconds": round(duration_ms / 1000, 3),
"exit_code": process.returncode, "batch_window_count": len(windows),
"events": events, "batch_rtf": round(elapsed / max(0.001, duration_ms * len(windows) / 1000), 3),
"stderr_tail": process.stderr[-2000:], "exit_code": process.returncode, "events": selected,
} "stderr_tail": process.stderr[-2000:] if process.returncode else "", "window": window,
})
return results
def main() -> int: def main() -> int:
@@ -95,10 +119,11 @@ def main() -> int:
selected = windows.get(media.name) selected = windows.get(media.name)
if not isinstance(selected, list) or len(selected) != 8: if not isinstance(selected, list) or len(selected) != 8:
raise SystemExit(f"{media.name}: windows JSON must contain exactly 8 labelled windows") raise SystemExit(f"{media.name}: windows JSON must contain exactly 8 labelled windows")
for window in selected: batched_wav = Path(work) / f"{media.stem}-windows.wav"
for combination in COMBINATIONS: _write_window_batch(wav, batched_wav, selected)
result = worker_run(args.data_dir, wav, window, combination) for combination in COMBINATIONS:
result.update({"media": media.name, "window": window}) for result in worker_run(args.data_dir, batched_wav, selected, combination):
result["media"] = media.name
artifact["runs"].append(result) artifact["runs"].append(result)
args.output.parent.mkdir(parents=True, exist_ok=True) args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(artifact, ensure_ascii=False, indent=2), encoding="utf-8") args.output.write_text(json.dumps(artifact, ensure_ascii=False, indent=2), encoding="utf-8")
+23
View File
@@ -16,9 +16,32 @@ from imagefind.jobs import JobRetry
from imagefind.main import create_app from imagefind.main import create_app
from imagefind.media import MediaInput from imagefind.media import MediaInput
from imagefind.speech import SPEECH_INDEX_REVISION, SpeechService, SpeechStageError from imagefind.speech import SPEECH_INDEX_REVISION, SpeechService, SpeechStageError
from imagefind.speech_quality import aggregate_transcript_quality
from imagefind.text import fts_query from imagefind.text import fts_query
def test_ending_hallucination_is_risky_twice_or_once_in_weak_speech():
repeated = aggregate_transcript_quality([
{"text": "拜拜"},
{"text": "正常对话"},
{"text": "拜拜"},
])
assert "whole_ending_hallucination" in repeated.flags
assert ("拜拜", 2) in repeated.repeated_phrases
weak = aggregate_transcript_quality([
{"text": "谢谢大家收看", "speech_ratio": 0.1},
{"text": "正常对话", "speech_ratio": 0.8},
])
assert "whole_ending_hallucination" in weak.flags
single = aggregate_transcript_quality([
{"text": "拜拜", "speech_ratio": 0.8},
{"text": "正常对话", "speech_ratio": 0.8},
])
assert "whole_ending_hallucination" not in single.flags
def _app(tmp_path: Path): def _app(tmp_path: Path):
settings = Settings(data_dir=tmp_path / "data", embedding_backend="hash", upload_reserve_gb=0) settings = Settings(data_dir=tmp_path / "data", embedding_backend="hash", upload_reserve_gb=0)
settings.prepare() settings.prepare()
+34
View File
@@ -1,4 +1,5 @@
from contextlib import nullcontext from contextlib import nullcontext
from pathlib import Path
from types import SimpleNamespace from types import SimpleNamespace
import httpx import httpx
@@ -114,6 +115,39 @@ def test_native_catalog_uses_physical_object_but_keeps_logical_media_identity():
assert item.fingerprint assert item.fingerprint
def test_native_prepare_and_cancel_preserve_original_and_local_recovery(tmp_path: Path):
original = tmp_path / "upload.part"
original.write_bytes(b"data")
staging = tmp_path / "native"
class Sources:
rclone = SimpleNamespace()
def get(self, _source_id):
return {}
service = object.__new__(OpenListNativeService)
service.sources = Sources()
service.configuration = lambda _source: SimpleNamespace(
local_staging_path=staging,
encrypted=False,
source_path="local-stage",
target_path="cloud",
)
paths = service.prepare("source", "upload-id", "folder/movie.mp4", original)
assert original.read_bytes() == b"data"
assert Path(paths["local_path"]).read_bytes() == b"data"
cancelled: list[str] = []
service.client = lambda _source_id: SimpleNamespace(
cancel_copy_task=lambda task_id: cancelled.append(task_id)
)
service.cancel("source", "upload-id", "task-id", paths["staged_path"])
assert cancelled == ["task-id"]
assert original.is_file()
assert Path(paths["local_path"]).is_file()
class _MovingClient: class _MovingClient:
def __init__(self, files: dict[str, int], *, fail_after_move: bool = False): def __init__(self, files: dict[str, int], *, fail_after_move: bool = False):
self.files = dict(files) self.files = dict(files)
+29
View File
@@ -221,6 +221,11 @@ def test_upload_cancel_is_consistent_and_commit_window_returns_conflict(tmp_path
job_state = conn.execute("SELECT status FROM jobs WHERE id=?", (job_id,)).fetchone() job_state = conn.execute("SELECT status FROM jobs WHERE id=?", (job_id,)).fetchone()
assert upload_state["status"] == "cancelled" assert upload_state["status"] == "cancelled"
assert job_state["status"] == "cancelled" assert job_state["status"] == "cancelled"
with app.state.services.db.read() as conn:
retained_path = Path(
conn.execute("SELECT temp_path FROM uploads WHERE id=?", (upload["id"],)).fetchone()[0]
)
assert retained_path.read_bytes() == b"data"
committed = app.state.services.uploads.create(source_id, "", "committed.mp4", 4) committed = app.state.services.uploads.create(source_id, "", "committed.mp4", 4)
with app.state.services.db.transaction() as conn: with app.state.services.db.transaction() as conn:
@@ -239,6 +244,30 @@ def test_upload_cancel_is_consistent_and_commit_window_returns_conflict(tmp_path
asyncio.run(scenario()) asyncio.run(scenario())
def test_cancelled_upload_requires_explicit_discard_to_delete_recovery_copy(tmp_path: Path):
app, _, source_id, headers = _app(tmp_path)
uploads = app.state.services.uploads
upload = uploads.create(source_id, "", "recoverable.mp4", 4)
uploads.receive_chunk(upload["id"], 0, b"data")
with uploads.db.read() as conn:
staging = Path(conn.execute("SELECT temp_path FROM uploads WHERE id=?", (upload["id"],)).fetchone()[0])
uploads.cancel(upload["id"])
assert staging.is_file()
async def scenario():
transport = httpx.ASGITransport(app=app)
async with httpx.AsyncClient(transport=transport, base_url="http://test") as client:
response = await client.delete(
f"/api/v1/uploads/{upload['id']}/recovery", headers=headers
)
assert response.status_code == 204
asyncio.run(scenario())
assert not staging.exists()
with uploads.db.read() as conn:
assert conn.execute("SELECT 1 FROM uploads WHERE id=?", (upload["id"],)).fetchone() is None
def test_upload_catalog_failure_retries_without_retransmitting(tmp_path: Path, monkeypatch): def test_upload_catalog_failure_retries_without_retransmitting(tmp_path: Path, monkeypatch):
app, media, source_id, _ = _app(tmp_path) app, media, source_id, _ = _app(tmp_path)
uploads = app.state.services.uploads uploads = app.state.services.uploads