From 8960b68ac823e67584b7a39c6802ed28e9394bf3 Mon Sep 17 00:00:00 2001 From: nanxun Date: Sat, 19 Sep 2026 10:25:10 +0800 Subject: [PATCH] fix: harden auto uploads and table layouts --- README.md | 2 +- frontend/playwright.config.ts | 4 +- frontend/src/styles/main.css | 21 +-- frontend/src/types.ts | 9 ++ frontend/src/views/SettingsView.vue | 73 +++++++++- frontend/tests/e2e/ui-regression.spec.ts | 55 +++++++- .../Persistence/PersistenceContracts.cs | 4 +- .../Models/Settings/SettingsModels.cs | 19 +++ .../Persistence/Repositories/Repositories.cs | 27 +++- .../Services/CompletionDispatchService.cs | 68 ++++++--- .../Services/OpenListClient.cs | 62 +++++++- .../Services/OpenListUploadHealthState.cs | 61 +++++++- .../Services/OpenListUploadQueueService.cs | 72 +++++++++- src/LiveRecorder.WebApi/Program.cs | 24 +++- .../LiveRecorder.Tests/OpenListUploadTests.cs | 133 ++++++++++++++++++ 15 files changed, 586 insertions(+), 48 deletions(-) diff --git a/README.md b/README.md index e6a3742..49eda26 100644 --- a/README.md +++ b/README.md @@ -284,7 +284,7 @@ docker compose up -d - `dotnet build LiveRecorder.sln --no-restore -m:1` - `npm run build`（frontend 目录） -- `npm run test:e2e`（frontend 目录，覆盖 1920 / 1366 / 1024 / 768 / 390） +- `npm run test:e2e`（frontend 目录，覆盖 1920 / 1552 / 1366 / 1024 / 768 / 390） - Docker Compose 完整部署验证通过 ## 后续规划 diff --git a/frontend/playwright.config.ts b/frontend/playwright.config.ts index 1b1c24c..aa82f58 100644 --- a/frontend/playwright.config.ts +++ b/frontend/playwright.config.ts @@ -2,6 +2,7 @@ import { defineConfig } from "@playwright/test"; const viewports = [ { name: "desktop-1920", width: 1920, height: 1080 }, + { name: "desktop-1552", width: 1552, height: 761 }, { name: "desktop-1366", width: 1366, height: 768 }, { name: "tablet-1024", width: 1024, height: 768 }, { name: "tablet-768", width: 768, height: 1024 }, @@ -21,7 +22,8 @@ export default defineConfig({ trace: "retain-on-failure" }, webServer: { - command: "VITE_DISABLE_DEVTOOLS=1 npm run dev -- --host 127.0.0.1 --port 47173 --strictPort", + command: "npm run dev -- --host 127.0.0.1 --port 47173 --strictPort", + env: { VITE_DISABLE_DEVTOOLS: "1" }, url: "http://127.0.0.1:47173", reuseExistingServer: false, timeout: 120_000 diff --git a/frontend/src/styles/main.css b/frontend/src/styles/main.css index 5dbf0dd..c92cf9d 100644 --- a/frontend/src/styles/main.css +++ b/frontend/src/styles/main.css @@ -355,19 +355,24 @@ button { font-family: inherit; cursor: pointer; } /* tables */ .el-table { - --el-table-border-color: transparent; --el-table-header-bg-color: transparent; - --el-table-bg-color: transparent; --el-table-row-hover-bg-color: var(--surface-hover); - --el-table-current-row-bg-color: var(--accent-soft); --el-fill-color-blank: transparent; - background: transparent; + --el-table-border-color: transparent; --el-table-header-bg-color: var(--surface-muted); + --el-table-bg-color: var(--surface); --el-table-tr-bg-color: var(--surface); + --el-table-row-hover-bg-color: var(--surface-hover); + --el-table-current-row-bg-color: var(--accent-soft); --el-fill-color-blank: var(--surface); + background: var(--surface); } .el-table::before, .el-table__inner-wrapper::before { display: none; } .el-table th.el-table__cell { padding: 11px 16px; border-bottom: 1px solid var(--border-subtle); background: var(--surface-muted); } html[data-theme="dark"] .el-table th.el-table__cell { background: var(--surface-muted); } .el-table th.el-table__cell > .cell { color: var(--text-muted); font-size: 12px; font-weight: 700; letter-spacing: .02em; text-transform: uppercase; } -.el-table td.el-table__cell { padding: 13px 16px; border-bottom: 1px solid var(--border-subtle); background: transparent; } -.el-table tr { background: transparent; } -.el-table tbody tr:nth-child(even) { background: rgba(15, 23, 42, 0.018); } -html[data-theme="dark"] .el-table tbody tr:nth-child(even) { background: rgba(255, 255, 255, 0.02); } +.el-table td.el-table__cell { padding: 13px 16px; border-bottom: 1px solid var(--border-subtle); background: var(--surface); } +.el-table tr { background: var(--surface); } +.el-table tbody tr:nth-child(even) > td.el-table__cell { background: color-mix(in srgb, var(--surface-strong) 24%, var(--surface)); } +.el-table tbody tr:hover > td.el-table__cell { background: var(--surface-hover); } +.el-table .el-table-fixed-column--right, +.el-table .el-table-fixed-column--left { background: inherit; } +.el-table .el-table-fixed-column--right.is-first-column { box-shadow: -10px 0 18px -16px rgba(23, 35, 58, .55); } +html[data-theme="dark"] .el-table .el-table-fixed-column--right.is-first-column { box-shadow: -10px 0 20px -15px rgba(0, 0, 0, .9); } .premium-table { width: 100%; } /* descriptions */ diff --git a/frontend/src/types.ts b/frontend/src/types.ts index b273865..f3692dc 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -653,6 +653,15 @@ export interface OpenListConnectionTestResult { success: boolean; version?: string; message: string; + sourcePath?: OpenListPathCheck; + destinationPath?: OpenListPathCheck; +} + +export interface OpenListPathCheck { + path: string; + success: boolean; + canWrite: boolean; + message: string; } export interface OpenListDirectoryItem { diff --git a/frontend/src/views/SettingsView.vue b/frontend/src/views/SettingsView.vue index 0907c5a..0320ff6 100644 --- a/frontend/src/views/SettingsView.vue +++ b/frontend/src/views/SettingsView.vue @@ -53,6 +53,7 @@ const changingPassword = ref(false); const testingEmail = ref(false); const testingWebhook = ref(false); const testingOpenList = ref(false); +const openListTestResult = ref(null); const openListDirectoryLoading = ref(false); const openListDirectoryDialogVisible = ref(false); const openListDirectoryPickerTarget = ref<"source" | "destination">("source"); @@ -568,7 +569,9 @@ function buildOpenListConnectionPayload() { return { baseUrl: form.openListUpload.baseUrl, username: form.openListUpload.username, - password: form.openListUpload.password + password: form.openListUpload.password, + sourcePath: form.openListUpload.sourcePath, + destinationPath: form.openListUpload.destinationPath }; } @@ -580,8 +583,10 @@ async function testOpenListConnection() { "/settings/openlist/test", buildOpenListConnectionPayload() ); + openListTestResult.value = data; ElMessage[data.success ? "success" : "warning"](data.message); } catch (error) { + openListTestResult.value = null; ElMessage.error(getApiErrorMessage(error, "OpenList 连接测试失败")); } finally { testingOpenList.value = false; @@ -1646,6 +1651,29 @@ onBeforeRouteLeave(async () => { 测试连接 +
+
+ + {{ openListTestResult.success ? "配置可用" : "配置异常" }} + + {{ openListTestResult.message }} +
+
+ 源目录 + {{ openListTestResult.sourcePath.path }} + {{ openListTestResult.sourcePath.message }} +
+
+ 目标目录 + {{ openListTestResult.destinationPath.path }} + {{ openListTestResult.destinationPath.message }} +
+
+ @@ -1965,6 +1993,14 @@ onBeforeRouteLeave(async () => { + @@ -2828,6 +2864,41 @@ onBeforeRouteLeave(async () => { color: var(--text-secondary); } +.openlist-test-result { + display: grid; + gap: 10px; + margin-bottom: 16px; + padding: 12px 14px; + border: 1px solid color-mix(in srgb, var(--success) 30%, var(--border-subtle)); + border-radius: var(--radius-sm); + background: var(--success-soft); +} + +.openlist-test-result--failed { + border-color: color-mix(in srgb, var(--warning) 34%, var(--border-subtle)); + background: var(--warning-soft); +} + +.openlist-test-result__summary, +.openlist-test-result__path { + display: flex; + align-items: center; + flex-wrap: wrap; + gap: 8px; + color: var(--text-secondary); + font-size: 12px; + line-height: 1.55; +} + +.openlist-test-result__path code { + color: var(--text-primary); + overflow-wrap: anywhere; +} + +.event-script-conflict-alert { + margin-bottom: 16px; +} + .template-help { margin-top: 8px; padding: 16px 18px; diff --git a/frontend/tests/e2e/ui-regression.spec.ts b/frontend/tests/e2e/ui-regression.spec.ts index f9fe2fa..fcc5c1f 100644 --- a/frontend/tests/e2e/ui-regression.spec.ts +++ b/frontend/tests/e2e/ui-regression.spec.ts @@ -377,6 +377,13 @@ test("live room table, drawer and dialog retain their final actions", async ({ p await expect(actionHeader).toBeVisible(); const box = await actionHeader.boundingBox(); expect(box && box.x + box.width).toBeLessThanOrEqual((page.viewportSize()?.width ?? 0) + 1); + const fixedActionCell = page.locator("td.el-table-fixed-column--right").first(); + await expect(fixedActionCell).toBeVisible(); + const fixedBackground = await fixedActionCell.evaluate((element) => + window.getComputedStyle(element).backgroundColor + ); + expect(fixedBackground).not.toBe("transparent"); + expect(fixedBackground).not.toMatch(/rgba\([^)]*,\s*0\s*\)$/); } await page.getByRole("button", { name: /^查看/ }).first().click(); @@ -677,7 +684,7 @@ test("recording failure recovery adapts its actions and long paths to each viewp await expect(page.getByText(recordingFailure.filePath, { exact: true })).toBeVisible(); await expect(page.getByRole("button", { name: "确认有效" })).toBeVisible(); - if ((page.viewportSize()?.width ?? 0) <= 767) { + if ((page.viewportSize()?.width ?? 0) <= 768) { await expect(page.locator(".failure-card")).toHaveCount(1); await expect(page.getByRole("columnheader", { name: "操作" })).toHaveCount(0); } else { @@ -794,3 +801,49 @@ test("settings layer advanced controls without clipping the quick bar", async ({ await expectNoDocumentOverflow(page); await capture(page, testInfo, "settings"); }); + +test("OpenList test distinguishes login success from a broken destination mount", async ({ page }) => { + await page.route("**/api/settings", async (route) => { + await route.fulfill({ + status: 200, + contentType: "application/json", + body: JSON.stringify({ + enableFileUpload: true, + enableAutoUpload: true, + uploadTarget: 3, + openListUpload: { + baseUrl: "https://openlist.example.com", + username: "tester", + password: "secret", + basePath: "/archive", + sourcePath: "/source", + destinationPath: "/archive" + } + }) + }); + }); + await page.route("**/api/settings/openlist/test", async (route) => { + const request = route.request().postDataJSON(); + expect(request.sourcePath).toBe("/source"); + expect(request.destinationPath).toBe("/archive"); + await route.fulfill({ + status: 200, + contentType: "application/json", + body: JSON.stringify({ + success: false, + version: "4.1.10", + message: "OpenList 登录成功，但一个或多个配置目录不可用。", + sourcePath: { path: "/source", success: true, canWrite: false, message: "目录可访问。" }, + destinationPath: { path: "/archive", success: false, canWrite: false, message: "provider timeout" } + }) + }); + }); + + await page.goto("/settings/upload"); + await expect(page.getByRole("heading", { name: "OpenList 服务端复制" })).toBeVisible(); + await page.getByRole("button", { name: "测试连接" }).click(); + await expect(page.getByText("配置异常", { exact: true })).toBeVisible(); + await expect(page.getByText("目录可访问。", { exact: true })).toBeVisible(); + await expect(page.getByText("provider timeout", { exact: true })).toBeVisible(); + await expectNoDocumentOverflow(page); +}); diff --git a/src/LiveRecorder.Application/Abstractions/Persistence/PersistenceContracts.cs b/src/LiveRecorder.Application/Abstractions/Persistence/PersistenceContracts.cs index 2180a9b..12ec1ed 100644 --- a/src/LiveRecorder.Application/Abstractions/Persistence/PersistenceContracts.cs +++ b/src/LiveRecorder.Application/Abstractions/Persistence/PersistenceContracts.cs @@ -180,7 +180,9 @@ public sealed record OperationsMetricsSnapshot( DateTimeOffset? OldestTranscodeUpdatedAt, DateTimeOffset? OldestUploadProgressAt, int StalledUploadCount, - int CleanupFailureCount); + int CleanupFailureCount, + DateTimeOffset? OldestActionableUploadAt, + int PendingUploadRecoveryCount); public interface IUserAccountRepository { diff --git a/src/LiveRecorder.Application/Models/Settings/SettingsModels.cs b/src/LiveRecorder.Application/Models/Settings/SettingsModels.cs index 170ca24..3f836c4 100644 --- a/src/LiveRecorder.Application/Models/Settings/SettingsModels.cs +++ b/src/LiveRecorder.Application/Models/Settings/SettingsModels.cs @@ -104,6 +104,10 @@ public class OpenListConnectionRequest public string Username { get; set; } = string.Empty; public string Password { get; set; } = string.Empty; + + public string? SourcePath { get; set; } + + public string? DestinationPath { get; set; } } public sealed class OpenListDirectoryRequest : OpenListConnectionRequest @@ -134,6 +138,21 @@ public sealed class OpenListConnectionTestDto public string? Version { get; init; } public required string Message { get; init; } + + public OpenListPathCheckDto? SourcePath { get; init; } + + public OpenListPathCheckDto? DestinationPath { get; init; } +} + +public sealed class OpenListPathCheckDto +{ + public required string Path { get; init; } + + public bool Success { get; init; } + + public bool CanWrite { get; init; } + + public required string Message { get; init; } } public sealed class SystemSettingsDto diff --git a/src/LiveRecorder.Infrastructure/Persistence/Repositories/Repositories.cs b/src/LiveRecorder.Infrastructure/Persistence/Repositories/Repositories.cs index a6a11b5..7065dce 100644 --- a/src/LiveRecorder.Infrastructure/Persistence/Repositories/Repositories.cs +++ b/src/LiveRecorder.Infrastructure/Persistence/Repositories/Repositories.cs @@ -171,7 +171,8 @@ public sealed class OperationsMetricsRepository : IOperationsMetricsRepository public async Task GetAsync(CancellationToken cancellationToken = default) { - var stalledBefore = DateTimeOffset.UtcNow.Subtract(UploadStallThreshold); + var now = DateTimeOffset.UtcNow; + var stalledBefore = now.Subtract(UploadStallThreshold); var oldestTranscodeUpdatedAt = await _dbContext.RecordTasks .AsNoTracking() .Where(static task => task.Status == RecordTaskStatus.Processing) @@ -183,9 +184,27 @@ public sealed class OperationsMetricsRepository : IOperationsMetricsRepository .Select(job => (DateTimeOffset?)(job.LastProgressAt ?? job.ExternalTaskStartedAt ?? job.UpdatedAt)) .MinAsync(cancellationToken); var stalledUploadCount = await _dbContext.RecordUploadJobs.CountAsync( - job => job.Status == RecordArtifactUploadStatus.Uploading && + job => (job.Status == RecordArtifactUploadStatus.Queued || + job.Status == RecordArtifactUploadStatus.Uploading || + (job.Status == RecordArtifactUploadStatus.WaitingRetry && + (!job.NextAttemptAt.HasValue || job.NextAttemptAt <= now))) && (job.LastProgressAt ?? job.ExternalTaskStartedAt ?? job.UpdatedAt) < stalledBefore, cancellationToken); + var oldestActionableUploadAt = await _dbContext.RecordUploadJobs + .AsNoTracking() + .Where(job => job.Status == RecordArtifactUploadStatus.Queued || + job.Status == RecordArtifactUploadStatus.Uploading || + (job.Status == RecordArtifactUploadStatus.WaitingRetry && + (!job.NextAttemptAt.HasValue || job.NextAttemptAt <= now))) + .Select(job => (DateTimeOffset?)(job.LastProgressAt ?? job.ExternalTaskStartedAt ?? job.UpdatedAt)) + .MinAsync(cancellationToken); + var pendingUploadRecoveryCount = await _dbContext.RecordTasks.CountAsync( + task => (task.Status == RecordTaskStatus.Completed || task.Status == RecordTaskStatus.Stopped) && + !task.IsHiddenArtifactSource && + task.Result != null && + task.Result.UploadStatus == RecordArtifactUploadStatus.NotUploaded && + task.Result.FilePath != string.Empty, + cancellationToken); var cleanupFailureCount = await _dbContext.RecordUploadJobs.CountAsync( static job => job.Status == RecordArtifactUploadStatus.WaitingRetry && job.CurrentArtifact == RecordUploadArtifactStage.Completed, @@ -194,7 +213,9 @@ public sealed class OperationsMetricsRepository : IOperationsMetricsRepository oldestTranscodeUpdatedAt, oldestUploadProgressAt, stalledUploadCount, - cleanupFailureCount); + cleanupFailureCount, + oldestActionableUploadAt, + pendingUploadRecoveryCount); } } diff --git a/src/LiveRecorder.Infrastructure/Services/CompletionDispatchService.cs b/src/LiveRecorder.Infrastructure/Services/CompletionDispatchService.cs index 054baf7..665af0c 100644 --- a/src/LiveRecorder.Infrastructure/Services/CompletionDispatchService.cs +++ b/src/LiveRecorder.Infrastructure/Services/CompletionDispatchService.cs @@ -115,34 +115,68 @@ public sealed class CompletionDispatchService var recordSession = task.RecordSession ?? throw new InvalidOperationException(); var recordResult = task.Result ?? throw new InvalidOperationException(); + var dispatchErrors = new List(); if (!dispatch.ScriptDispatched) { - var scriptResult = await _eventScriptService.RunSegmentCompletedAsync( - task.LiveRoom, - recordSession, - task, - recordResult, - recordResult.FilePath, - task.EndedAt ?? DateTimeOffset.UtcNow, - eventId: dispatch.Id, - cancellationToken: cancellationToken); - if (scriptResult is null || scriptResult.Success) + try { - dispatch.MarkScriptDispatched(DateTimeOffset.UtcNow); - await _dbContext.SaveChangesAsync(cancellationToken); + var scriptResult = await _eventScriptService.RunSegmentCompletedAsync( + task.LiveRoom, + recordSession, + task, + recordResult, + recordResult.FilePath, + task.EndedAt ?? DateTimeOffset.UtcNow, + eventId: dispatch.Id, + cancellationToken: cancellationToken); + if (scriptResult is null || scriptResult.Success) + { + dispatch.MarkScriptDispatched(DateTimeOffset.UtcNow); + } + else + { + dispatchErrors.Add($"事件脚本：{scriptResult.Message}"); + } } - else + catch (Exception ex) when (ex is not OperationCanceledException || !cancellationToken.IsCancellationRequested) { - throw new InvalidOperationException(scriptResult.Message); + dispatchErrors.Add($"事件脚本：{ex.Message}"); } } if (!dispatch.UploadDispatched) { - _ = await _recordUploadService.TryAutoUploadTaskAsync(task.Id, cancellationToken); - dispatch.MarkUploadDispatched(DateTimeOffset.UtcNow); - await _dbContext.SaveChangesAsync(cancellationToken); + try + { + var uploadResult = await _recordUploadService.TryAutoUploadTaskAsync(task.Id, cancellationToken); + if (uploadResult is null || uploadResult.Success) + { + dispatch.MarkUploadDispatched(DateTimeOffset.UtcNow); + } + else + { + dispatchErrors.Add($"自动上传：{uploadResult.Message}"); + } + } + catch (Exception ex) when (ex is not OperationCanceledException || !cancellationToken.IsCancellationRequested) + { + dispatchErrors.Add($"自动上传：{ex.Message}"); + } } + + if (dispatchErrors.Count > 0) + { + dispatch.ScheduleRetry( + string.Join("；", dispatchErrors), + DateTimeOffset.UtcNow.Add(RetryDelay), + DateTimeOffset.UtcNow); + } + + await _dbContext.SaveChangesAsync(cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; } catch (Exception ex) { diff --git a/src/LiveRecorder.Infrastructure/Services/OpenListClient.cs b/src/LiveRecorder.Infrastructure/Services/OpenListClient.cs index 6e05279..72aa4ee 100644 --- a/src/LiveRecorder.Infrastructure/Services/OpenListClient.cs +++ b/src/LiveRecorder.Infrastructure/Services/OpenListClient.cs @@ -170,16 +170,70 @@ public sealed class OpenListClient : IOpenListClient version = versionElement.GetString(); } + var sourcePath = await TestDirectoryAsync(connection, connection.SourcePath, requireWrite: false, cancellationToken); + var destinationPath = await TestDirectoryAsync(connection, connection.DestinationPath, requireWrite: true, cancellationToken); + var success = (sourcePath?.Success ?? true) && (destinationPath?.Success ?? true); + return new OpenListConnectionTestDto { - Success = true, + Success = success, Version = version, - Message = string.IsNullOrWhiteSpace(version) - ? "OpenList 连接和登录成功。" - : $"OpenList 连接和登录成功：{version}" + Message = success + ? (string.IsNullOrWhiteSpace(version) + ? "OpenList 登录和目录检查成功。" + : $"OpenList 登录和目录检查成功：{version}") + : "OpenList 登录成功，但一个或多个配置目录不可用。", + SourcePath = sourcePath, + DestinationPath = destinationPath }; } + private async Task TestDirectoryAsync( + OpenListConnectionRequest connection, + string? path, + bool requireWrite, + CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(path)) + { + return null; + } + + var normalizedPath = NormalizePath(path); + try + { + var directory = await ListDirectoriesAsync( + new OpenListDirectoryRequest + { + BaseUrl = connection.BaseUrl, + Username = connection.Username, + Password = connection.Password, + Path = normalizedPath + }, + cancellationToken); + var success = !requireWrite || directory.CanWrite; + return new OpenListPathCheckDto + { + Path = normalizedPath, + Success = success, + CanWrite = directory.CanWrite, + Message = success + ? (requireWrite ? "目录可访问且允许写入。" : "目录可访问。") + : "目录可访问，但 OpenList 未授予写入权限。" + }; + } + catch (Exception ex) when (ex is not OperationCanceledException || !cancellationToken.IsCancellationRequested) + { + return new OpenListPathCheckDto + { + Path = normalizedPath, + Success = false, + CanWrite = false, + Message = ex.Message + }; + } + } + public async Task ListDirectoriesAsync( OpenListDirectoryRequest request, CancellationToken cancellationToken = default) diff --git a/src/LiveRecorder.Infrastructure/Services/OpenListUploadHealthState.cs b/src/LiveRecorder.Infrastructure/Services/OpenListUploadHealthState.cs index b98f2d8..52e9852 100644 --- a/src/LiveRecorder.Infrastructure/Services/OpenListUploadHealthState.cs +++ b/src/LiveRecorder.Infrastructure/Services/OpenListUploadHealthState.cs @@ -15,6 +15,16 @@ public sealed record OpenListQueueHealthSnapshot( DateTimeOffset? LastErrorAt) { public bool IsPaused => Status != OpenListQueueHealthStatus.Healthy; + + public DateTimeOffset? LastWorkerIterationAt { get; init; } + + public DateTimeOffset? LastRecoveryAt { get; init; } + + public int LastRecoveredCount { get; init; } + + public string? LastWorkerError { get; init; } + + public string? LastRecoveryError { get; init; } } /// @@ -32,12 +42,61 @@ public sealed class OpenListUploadHealthState private DateTimeOffset? _retryAt; private DateTimeOffset? _lastErrorAt; private int _consecutiveRateLimits; + private DateTimeOffset? _lastWorkerIterationAt; + private DateTimeOffset? _lastRecoveryAt; + private int _lastRecoveredCount; + private string? _lastWorkerError; + private string? _lastRecoveryError; public OpenListQueueHealthSnapshot GetSnapshot() { lock (_gate) { - return new OpenListQueueHealthSnapshot(_status, _reason, _retryAt, _lastErrorAt); + return new OpenListQueueHealthSnapshot(_status, _reason, _retryAt, _lastErrorAt) + { + LastWorkerIterationAt = _lastWorkerIterationAt, + LastRecoveryAt = _lastRecoveryAt, + LastRecoveredCount = _lastRecoveredCount, + LastWorkerError = _lastWorkerError, + LastRecoveryError = _lastRecoveryError + }; + } + } + + public void MarkWorkerIteration(DateTimeOffset occurredAt) + { + lock (_gate) + { + _lastWorkerIterationAt = occurredAt; + _lastWorkerError = null; + } + } + + public void MarkRecovery(DateTimeOffset occurredAt, int recoveredCount) + { + lock (_gate) + { + _lastRecoveryAt = occurredAt; + _lastRecoveredCount = Math.Max(0, recoveredCount); + _lastRecoveryError = null; + } + } + + public void MarkWorkerError(DateTimeOffset occurredAt, string error) + { + lock (_gate) + { + _lastWorkerIterationAt = occurredAt; + _lastWorkerError = string.IsNullOrWhiteSpace(error) ? null : error.Trim(); + } + } + + public void MarkRecoveryError(DateTimeOffset occurredAt, string error) + { + lock (_gate) + { + _lastRecoveryAt = occurredAt; + _lastRecoveryError = string.IsNullOrWhiteSpace(error) ? null : error.Trim(); } } diff --git a/src/LiveRecorder.Infrastructure/Services/OpenListUploadQueueService.cs b/src/LiveRecorder.Infrastructure/Services/OpenListUploadQueueService.cs index 009ec58..d792171 100644 --- a/src/LiveRecorder.Infrastructure/Services/OpenListUploadQueueService.cs +++ b/src/LiveRecorder.Infrastructure/Services/OpenListUploadQueueService.cs @@ -153,6 +153,16 @@ public sealed class OpenListUploadQueueService { recovered++; } + else if (!result.Success) + { + await _systemLogService.WriteAsync( + SystemLogLevel.Warning, + "Upload", + "跳过一个无法恢复的 OpenList 自动上传任务。", + $"recordTaskId={taskId}; error={result.Message}", + recordTaskId: taskId, + cancellationToken: cancellationToken); + } } catch (Exception ex) { @@ -1346,21 +1356,22 @@ public sealed class OpenListUploadQueueService public sealed class OpenListUploadBackgroundService : BackgroundService { private static readonly TimeSpan IdleDelay = TimeSpan.FromSeconds(2); - private static readonly TimeSpan RecoveryInterval = TimeSpan.FromMinutes(1); private readonly IServiceScopeFactory _scopeFactory; + private readonly OpenListUploadHealthState _healthState; private readonly ILogger _logger; public OpenListUploadBackgroundService( IServiceScopeFactory scopeFactory, + OpenListUploadHealthState healthState, ILogger logger) { _scopeFactory = scopeFactory; + _healthState = healthState; _logger = logger; } protected override async Task ExecuteAsync(CancellationToken stoppingToken) { - var nextRecoveryAt = DateTimeOffset.MinValue; while (!stoppingToken.IsCancellationRequested) { try @@ -1368,11 +1379,7 @@ public sealed class OpenListUploadBackgroundService : BackgroundService using var scope = _scopeFactory.CreateScope(); var queue = scope.ServiceProvider.GetRequiredService(); _ = await queue.ProcessNextAsync(stoppingToken); - if (DateTimeOffset.UtcNow >= nextRecoveryAt) - { - await queue.RecoverPendingAutomaticUploadsAsync(cancellationToken: stoppingToken); - nextRecoveryAt = DateTimeOffset.UtcNow.Add(RecoveryInterval); - } + _healthState.MarkWorkerIteration(DateTimeOffset.UtcNow); await Task.Delay(IdleDelay, stoppingToken); } @@ -1382,6 +1389,7 @@ public sealed class OpenListUploadBackgroundService : BackgroundService } catch (Exception ex) { + _healthState.MarkWorkerError(DateTimeOffset.UtcNow, ex.Message); _logger.LogError(ex, "OpenList upload background worker failed"); try { @@ -1395,3 +1403,53 @@ public sealed class OpenListUploadBackgroundService : BackgroundService } } } + +public sealed class OpenListAutomaticRecoveryBackgroundService : BackgroundService +{ + private static readonly TimeSpan RecoveryInterval = TimeSpan.FromMinutes(1); + private readonly IServiceScopeFactory _scopeFactory; + private readonly OpenListUploadHealthState _healthState; + private readonly ILogger _logger; + + public OpenListAutomaticRecoveryBackgroundService( + IServiceScopeFactory scopeFactory, + OpenListUploadHealthState healthState, + ILogger logger) + { + _scopeFactory = scopeFactory; + _healthState = healthState; + _logger = logger; + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + while (!stoppingToken.IsCancellationRequested) + { + try + { + using var scope = _scopeFactory.CreateScope(); + var queue = scope.ServiceProvider.GetRequiredService(); + var recovered = await queue.RecoverPendingAutomaticUploadsAsync(cancellationToken: stoppingToken); + _healthState.MarkRecovery(DateTimeOffset.UtcNow, recovered); + await Task.Delay(RecoveryInterval, stoppingToken); + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + break; + } + catch (Exception ex) + { + _healthState.MarkRecoveryError(DateTimeOffset.UtcNow, ex.Message); + _logger.LogError(ex, "OpenList automatic upload recovery failed"); + try + { + await Task.Delay(TimeSpan.FromSeconds(10), stoppingToken); + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + break; + } + } + } + } +} diff --git a/src/LiveRecorder.WebApi/Program.cs b/src/LiveRecorder.WebApi/Program.cs index 585caad..b120949 100644 --- a/src/LiveRecorder.WebApi/Program.cs +++ b/src/LiveRecorder.WebApi/Program.cs @@ -253,6 +253,7 @@ builder.Services.AddHostedService(provider => provider.GetRequiredService(); builder.Services.AddHostedService(); builder.Services.AddHostedService(); +builder.Services.AddHostedService(); builder.Services.AddHostedService(); builder.Services.AddHostedService(); @@ -338,18 +339,34 @@ app.MapGet("/api/operations/health", async ( IOperationsMetricsRepository metricsRepository, ISystemSettingsService settingsService, IStorageGuardService storageGuardService, + OpenListUploadHealthState uploadHealthState, CancellationToken cancellationToken) => { var metrics = await metricsRepository.GetAsync(cancellationToken); var settings = await settingsService.GetAsync(cancellationToken); var storage = storageGuardService.CheckCanStartOrResume(settings); + var uploadQueue = uploadHealthState.GetSnapshot(); + var now = DateTimeOffset.UtcNow; + var monitorOpenList = settings.EnableFileUpload && + settings.EnableAutoUpload && + settings.UploadTarget == UploadTargetType.OpenList; + var workerHeartbeatStale = monitorOpenList && + uploadQueue.LastWorkerIterationAt.HasValue && + uploadQueue.LastWorkerIterationAt < now.AddMinutes(-5); + var recoveryHeartbeatStale = monitorOpenList && + uploadQueue.LastRecoveryAt.HasValue && + uploadQueue.LastRecoveryAt < now.AddMinutes(-5); var degraded = storage.Tier == StorageTier.Red || metrics.StalledUploadCount > 0 || - metrics.CleanupFailureCount > 0; + metrics.CleanupFailureCount > 0 || + !string.IsNullOrWhiteSpace(uploadQueue.LastWorkerError) || + !string.IsNullOrWhiteSpace(uploadQueue.LastRecoveryError) || + workerHeartbeatStale || + recoveryHeartbeatStale; return Results.Ok(new { status = degraded ? "degraded" : "healthy", - timestamp = DateTimeOffset.UtcNow, + timestamp = now, storage = new { tier = storage.Tier.ToString(), @@ -357,7 +374,8 @@ app.MapGet("/api/operations/health", async ( storage.AvailableBytes, storage.Message }, - operations = metrics + operations = metrics, + uploadQueue }); }); diff --git a/tests/LiveRecorder.Tests/OpenListUploadTests.cs b/tests/LiveRecorder.Tests/OpenListUploadTests.cs index 81161a4..00df7e6 100644 --- a/tests/LiveRecorder.Tests/OpenListUploadTests.cs +++ b/tests/LiveRecorder.Tests/OpenListUploadTests.cs @@ -4,6 +4,7 @@ using System.Text; using System.Text.Json; using LiveRecorder.Application.Abstractions.Logging; using LiveRecorder.Application.Abstractions.Recording; +using LiveRecorder.Application.Abstractions.Scripting; using LiveRecorder.Application.Abstractions.Settings; using LiveRecorder.Application.Models.Logs; using LiveRecorder.Application.Models.Settings; @@ -14,11 +15,83 @@ using LiveRecorder.Infrastructure.Persistence.Repositories; using LiveRecorder.Infrastructure.Services; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Storage; +using Microsoft.Extensions.Logging.Abstractions; namespace LiveRecorder.Tests; public sealed class OpenListUploadTests { + [Fact] + public async Task TestConnection_ReportsDestinationMountFailureAfterSuccessfulLogin() + { + var handler = new StubHttpMessageHandler(async request => + { + var path = request.RequestUri!.AbsolutePath; + if (path.EndsWith("/api/auth/login", StringComparison.Ordinal)) + { + return JsonResponse("""{"code":200,"message":"success","data":{"token":"test-token"}}"""); + } + + if (path.EndsWith("/api/public/settings", StringComparison.Ordinal)) + { + return JsonResponse("""{"code":200,"message":"success","data":{"version":"4.1.10"}}"""); + } + + var body = JsonDocument.Parse(await request.Content!.ReadAsStringAsync()).RootElement; + return body.GetProperty("path").GetString() == "/source" + ? JsonResponse("""{"code":200,"message":"success","data":{"content":[],"write":false}}""") + : JsonResponse("""{"code":500,"message":"provider timeout","data":null}"""); + }); + var client = CreateClient(handler); + var connection = Connection(); + connection.SourcePath = "/source"; + connection.DestinationPath = "/destination"; + + var result = await client.TestConnectionAsync(connection); + + Assert.False(result.Success); + Assert.Equal("4.1.10", result.Version); + Assert.True(result.SourcePath?.Success); + Assert.False(result.DestinationPath?.Success); + Assert.Contains("provider timeout", result.DestinationPath?.Message); + } + + [Fact] + public async Task CompletionDispatch_ScriptFailureDoesNotBlockAutomaticUpload() + { + await using var fixture = await QueueFixture.CreateAsync(); + await fixture.PrepareCompletionDispatchAsync(); + var service = fixture.CreateCompletionDispatchService(new FixedEventScriptService( + new EventScriptExecutionResultDto { Success = false, Message = "script failed" })); + + await service.TryDispatchTaskAsync(fixture.RecordTaskId); + + var dispatch = await fixture.Context.RecordCompletionDispatches.SingleAsync(); + Assert.False(dispatch.ScriptDispatched); + Assert.True(dispatch.UploadDispatched); + Assert.Null(dispatch.CompletedAt); + Assert.Contains("script failed", dispatch.LastError); + Assert.NotNull(await fixture.Context.RecordUploadJobs.SingleOrDefaultAsync()); + } + + [Fact] + public async Task CompletionDispatch_UploadRejectionRemainsRetryable() + { + await using var fixture = await QueueFixture.CreateAsync( + new VideoMetadata(0.18, 1920, 1080, "h264", "aac", 30, 4_000_000)); + await fixture.PrepareCompletionDispatchAsync(); + var service = fixture.CreateCompletionDispatchService(new FixedEventScriptService(null)); + + await service.TryDispatchTaskAsync(fixture.RecordTaskId); + + var dispatch = await fixture.Context.RecordCompletionDispatches.SingleAsync(); + Assert.True(dispatch.ScriptDispatched); + Assert.False(dispatch.UploadDispatched); + Assert.Null(dispatch.CompletedAt); + Assert.Contains("自动上传", dispatch.LastError); + Assert.True(dispatch.NextAttemptAt > DateTimeOffset.UtcNow); + } + [Fact] public async Task Enqueue_RejectsShortMediaAndKeepsItNotUploaded() { @@ -763,6 +836,30 @@ public sealed class OpenListUploadTests public Guid RecordTaskId { get; } + public async Task PrepareCompletionDispatchAsync() + { + var session = await Context.RecordSessions.SingleAsync(); + session.MarkCompleted(DateTimeOffset.UtcNow); + if (!await Context.RecordCompletionDispatches.AnyAsync()) + { + Context.RecordCompletionDispatches.Add( + new RecordCompletionDispatch(RecordTaskId, DateTimeOffset.UtcNow)); + } + + await Context.SaveChangesAsync(); + } + + public CompletionDispatchService CreateCompletionDispatchService(IEventScriptService eventScriptService) + { + var uploadService = new RecordUploadService(Context, _settingsService, new NullSystemLogService(), Queue); + var consolidation = new ShortFragmentConsolidationService( + Context, + _settingsService, + _videoMetadataService, + NullLogger.Instance); + return new CompletionDispatchService(Context, eventScriptService, uploadService, consolidation); + } + public static async Task CreateAsync(VideoMetadata? metadata = null) { var temporaryRoot = Path.Combine(Path.GetTempPath(), $"live-recorder-openlist-{Guid.NewGuid():N}"); @@ -886,6 +983,42 @@ public sealed class OpenListUploadTests CancellationToken cancellationToken = default) => Task.FromResult(null); } + private sealed class FixedEventScriptService : IEventScriptService + { + private readonly EventScriptExecutionResultDto? _segmentResult; + + public FixedEventScriptService(EventScriptExecutionResultDto? segmentResult) + { + _segmentResult = segmentResult; + } + + public Task RunLiveStartedAsync( + LiveRoom liveRoom, + DateTimeOffset occurredAt, + CancellationToken cancellationToken = default) => Task.FromResult(null); + + public Task RunLiveEndedAsync( + LiveRoom liveRoom, + DateTimeOffset occurredAt, + CancellationToken cancellationToken = default) => Task.FromResult(null); + + public Task RunSegmentCompletedAsync( + LiveRoom? liveRoom, + RecordSession recordSession, + RecordTask recordTask, + RecordResult? recordResult, + string segmentFilePath, + DateTimeOffset occurredAt, + bool forceRun = false, + Guid? eventId = null, + CancellationToken cancellationToken = default) => Task.FromResult(_segmentResult); + + public Task TestAsync( + TestEventScriptRequest request, + CancellationToken cancellationToken = default) => + Task.FromResult(new EventScriptTestResultDto { Success = true, Message = "ok" }); + } + private sealed class FixedSettingsService : ISystemSettingsService { private readonly SystemSettingsDto _settings;