2 Commits
Author SHA1 Message Date
nanxun 8960b68ac8 fix: harden auto uploads and table layouts 2026-09-19 10:25:10 +08:00
nanxun f0fb4db755 ci: bypass proxy for daocloud image layers 2026-09-16 23:40:56 +08:00
16 changed files with 588 additions and 48 deletions
Vendored
+2
View File
@@ -273,6 +273,8 @@ pipeline {
--driver-opt "env.https_proxy=$HTTP_PROXY_URL" \ --driver-opt "env.https_proxy=$HTTP_PROXY_URL" \
--driver-opt "env.HTTP_PROXY=$HTTP_PROXY_URL" \ --driver-opt "env.HTTP_PROXY=$HTTP_PROXY_URL" \
--driver-opt "env.HTTPS_PROXY=$HTTP_PROXY_URL" \ --driver-opt "env.HTTPS_PROXY=$HTTP_PROXY_URL" \
--driver-opt "env.no_proxy=.daocloud.vip" \
--driver-opt "env.NO_PROXY=.daocloud.vip" \
--use --use
fi fi
./scripts/ci-docker.sh buildx inspect "$BUILDER_NAME" --bootstrap ./scripts/ci-docker.sh buildx inspect "$BUILDER_NAME" --bootstrap
+1 -1
View File
@@ -284,7 +284,7 @@ docker compose up -d
- `dotnet build LiveRecorder.sln --no-restore -m:1` - `dotnet build LiveRecorder.sln --no-restore -m:1`
- `npm run build`frontend 目录) - `npm run build`frontend 目录)
- `npm run test:e2e`frontend 目录,覆盖 1920 / 1366 / 1024 / 768 / 390 - `npm run test:e2e`frontend 目录,覆盖 1920 / 1552 / 1366 / 1024 / 768 / 390
- Docker Compose 完整部署验证通过 - Docker Compose 完整部署验证通过
## 后续规划 ## 后续规划
+3 -1
View File
@@ -2,6 +2,7 @@ import { defineConfig } from "@playwright/test";
const viewports = [ const viewports = [
{ name: "desktop-1920", width: 1920, height: 1080 }, { name: "desktop-1920", width: 1920, height: 1080 },
{ name: "desktop-1552", width: 1552, height: 761 },
{ name: "desktop-1366", width: 1366, height: 768 }, { name: "desktop-1366", width: 1366, height: 768 },
{ name: "tablet-1024", width: 1024, height: 768 }, { name: "tablet-1024", width: 1024, height: 768 },
{ name: "tablet-768", width: 768, height: 1024 }, { name: "tablet-768", width: 768, height: 1024 },
@@ -21,7 +22,8 @@ export default defineConfig({
trace: "retain-on-failure" trace: "retain-on-failure"
}, },
webServer: { webServer: {
command: "VITE_DISABLE_DEVTOOLS=1 npm run dev -- --host 127.0.0.1 --port 47173 --strictPort", command: "npm run dev -- --host 127.0.0.1 --port 47173 --strictPort",
env: { VITE_DISABLE_DEVTOOLS: "1" },
url: "http://127.0.0.1:47173", url: "http://127.0.0.1:47173",
reuseExistingServer: false, reuseExistingServer: false,
timeout: 120_000 timeout: 120_000
+13 -8
View File
@@ -355,19 +355,24 @@ button { font-family: inherit; cursor: pointer; }
/* tables */ /* tables */
.el-table { .el-table {
--el-table-border-color: transparent; --el-table-header-bg-color: transparent; --el-table-border-color: transparent; --el-table-header-bg-color: var(--surface-muted);
--el-table-bg-color: transparent; --el-table-row-hover-bg-color: var(--surface-hover); --el-table-bg-color: var(--surface); --el-table-tr-bg-color: var(--surface);
--el-table-current-row-bg-color: var(--accent-soft); --el-fill-color-blank: transparent; --el-table-row-hover-bg-color: var(--surface-hover);
background: transparent; --el-table-current-row-bg-color: var(--accent-soft); --el-fill-color-blank: var(--surface);
background: var(--surface);
} }
.el-table::before, .el-table__inner-wrapper::before { display: none; } .el-table::before, .el-table__inner-wrapper::before { display: none; }
.el-table th.el-table__cell { padding: 11px 16px; border-bottom: 1px solid var(--border-subtle); background: var(--surface-muted); } .el-table th.el-table__cell { padding: 11px 16px; border-bottom: 1px solid var(--border-subtle); background: var(--surface-muted); }
html[data-theme="dark"] .el-table th.el-table__cell { background: var(--surface-muted); } html[data-theme="dark"] .el-table th.el-table__cell { background: var(--surface-muted); }
.el-table th.el-table__cell > .cell { color: var(--text-muted); font-size: 12px; font-weight: 700; letter-spacing: .02em; text-transform: uppercase; } .el-table th.el-table__cell > .cell { color: var(--text-muted); font-size: 12px; font-weight: 700; letter-spacing: .02em; text-transform: uppercase; }
.el-table td.el-table__cell { padding: 13px 16px; border-bottom: 1px solid var(--border-subtle); background: transparent; } .el-table td.el-table__cell { padding: 13px 16px; border-bottom: 1px solid var(--border-subtle); background: var(--surface); }
.el-table tr { background: transparent; } .el-table tr { background: var(--surface); }
.el-table tbody tr:nth-child(even) { background: rgba(15, 23, 42, 0.018); } .el-table tbody tr:nth-child(even) > td.el-table__cell { background: color-mix(in srgb, var(--surface-strong) 24%, var(--surface)); }
html[data-theme="dark"] .el-table tbody tr:nth-child(even) { background: rgba(255, 255, 255, 0.02); } .el-table tbody tr:hover > td.el-table__cell { background: var(--surface-hover); }
.el-table .el-table-fixed-column--right,
.el-table .el-table-fixed-column--left { background: inherit; }
.el-table .el-table-fixed-column--right.is-first-column { box-shadow: -10px 0 18px -16px rgba(23, 35, 58, .55); }
html[data-theme="dark"] .el-table .el-table-fixed-column--right.is-first-column { box-shadow: -10px 0 20px -15px rgba(0, 0, 0, .9); }
.premium-table { width: 100%; } .premium-table { width: 100%; }
/* descriptions */ /* descriptions */
+9
View File
@@ -653,6 +653,15 @@ export interface OpenListConnectionTestResult {
success: boolean; success: boolean;
version?: string; version?: string;
message: string; message: string;
sourcePath?: OpenListPathCheck;
destinationPath?: OpenListPathCheck;
}
export interface OpenListPathCheck {
path: string;
success: boolean;
canWrite: boolean;
message: string;
} }
export interface OpenListDirectoryItem { export interface OpenListDirectoryItem {
+72 -1
View File
@@ -53,6 +53,7 @@ const changingPassword = ref(false);
const testingEmail = ref(false); const testingEmail = ref(false);
const testingWebhook = ref(false); const testingWebhook = ref(false);
const testingOpenList = ref(false); const testingOpenList = ref(false);
const openListTestResult = ref<OpenListConnectionTestResult | null>(null);
const openListDirectoryLoading = ref(false); const openListDirectoryLoading = ref(false);
const openListDirectoryDialogVisible = ref(false); const openListDirectoryDialogVisible = ref(false);
const openListDirectoryPickerTarget = ref<"source" | "destination">("source"); const openListDirectoryPickerTarget = ref<"source" | "destination">("source");
@@ -568,7 +569,9 @@ function buildOpenListConnectionPayload() {
return { return {
baseUrl: form.openListUpload.baseUrl, baseUrl: form.openListUpload.baseUrl,
username: form.openListUpload.username, username: form.openListUpload.username,
password: form.openListUpload.password password: form.openListUpload.password,
sourcePath: form.openListUpload.sourcePath,
destinationPath: form.openListUpload.destinationPath
}; };
} }
@@ -580,8 +583,10 @@ async function testOpenListConnection() {
"/settings/openlist/test", "/settings/openlist/test",
buildOpenListConnectionPayload() buildOpenListConnectionPayload()
); );
openListTestResult.value = data;
ElMessage[data.success ? "success" : "warning"](data.message); ElMessage[data.success ? "success" : "warning"](data.message);
} catch (error) { } catch (error) {
openListTestResult.value = null;
ElMessage.error(getApiErrorMessage(error, "OpenList 连接测试失败")); ElMessage.error(getApiErrorMessage(error, "OpenList 连接测试失败"));
} finally { } finally {
testingOpenList.value = false; testingOpenList.value = false;
@@ -1646,6 +1651,29 @@ onBeforeRouteLeave(async () => {
<el-button :loading="testingOpenList" @click="testOpenListConnection">测试连接</el-button> <el-button :loading="testingOpenList" @click="testOpenListConnection">测试连接</el-button>
</div> </div>
<div
v-if="openListTestResult"
class="openlist-test-result"
:class="{ 'openlist-test-result--failed': !openListTestResult.success }"
>
<div class="openlist-test-result__summary">
<el-tag :type="openListTestResult.success ? 'success' : 'warning'">
{{ openListTestResult.success ? "配置可用" : "配置异常" }}
</el-tag>
<span>{{ openListTestResult.message }}</span>
</div>
<div v-if="openListTestResult.sourcePath" class="openlist-test-result__path">
<el-tag :type="openListTestResult.sourcePath.success ? 'success' : 'danger'" size="small">源目录</el-tag>
<code>{{ openListTestResult.sourcePath.path }}</code>
<span>{{ openListTestResult.sourcePath.message }}</span>
</div>
<div v-if="openListTestResult.destinationPath" class="openlist-test-result__path">
<el-tag :type="openListTestResult.destinationPath.success ? 'success' : 'danger'" size="small">目标目录</el-tag>
<code>{{ openListTestResult.destinationPath.path }}</code>
<span>{{ openListTestResult.destinationPath.message }}</span>
</div>
</div>
<el-form label-position="top"> <el-form label-position="top">
<el-row :gutter="16"> <el-row :gutter="16">
<el-col :span="12"> <el-col :span="12">
@@ -1965,6 +1993,14 @@ onBeforeRouteLeave(async () => {
</el-button> </el-button>
</div> </div>
</div> </div>
<el-alert
v-if="form.enableFileUpload && form.enableAutoUpload && form.enableSegmentCompletedScript"
class="event-script-conflict-alert"
type="warning"
:closable="false"
show-icon
title="内置自动上传与分片完成脚本会独立执行。请勿在该脚本中重复实现上传,以免重复提交或因脚本故障干扰完成事件。"
/>
<el-form-item v-if="form.segmentCompletedScriptMode === 'path'" label="脚本路径"> <el-form-item v-if="form.segmentCompletedScriptMode === 'path'" label="脚本路径">
<el-input v-model="form.segmentCompletedScriptPath" placeholder="/app/scripts/segment-completed.sh" /> <el-input v-model="form.segmentCompletedScriptPath" placeholder="/app/scripts/segment-completed.sh" />
</el-form-item> </el-form-item>
@@ -2828,6 +2864,41 @@ onBeforeRouteLeave(async () => {
color: var(--text-secondary); color: var(--text-secondary);
} }
.openlist-test-result {
display: grid;
gap: 10px;
margin-bottom: 16px;
padding: 12px 14px;
border: 1px solid color-mix(in srgb, var(--success) 30%, var(--border-subtle));
border-radius: var(--radius-sm);
background: var(--success-soft);
}
.openlist-test-result--failed {
border-color: color-mix(in srgb, var(--warning) 34%, var(--border-subtle));
background: var(--warning-soft);
}
.openlist-test-result__summary,
.openlist-test-result__path {
display: flex;
align-items: center;
flex-wrap: wrap;
gap: 8px;
color: var(--text-secondary);
font-size: 12px;
line-height: 1.55;
}
.openlist-test-result__path code {
color: var(--text-primary);
overflow-wrap: anywhere;
}
.event-script-conflict-alert {
margin-bottom: 16px;
}
.template-help { .template-help {
margin-top: 8px; margin-top: 8px;
padding: 16px 18px; padding: 16px 18px;
+54 -1
View File
@@ -377,6 +377,13 @@ test("live room table, drawer and dialog retain their final actions", async ({ p
await expect(actionHeader).toBeVisible(); await expect(actionHeader).toBeVisible();
const box = await actionHeader.boundingBox(); const box = await actionHeader.boundingBox();
expect(box && box.x + box.width).toBeLessThanOrEqual((page.viewportSize()?.width ?? 0) + 1); expect(box && box.x + box.width).toBeLessThanOrEqual((page.viewportSize()?.width ?? 0) + 1);
const fixedActionCell = page.locator("td.el-table-fixed-column--right").first();
await expect(fixedActionCell).toBeVisible();
const fixedBackground = await fixedActionCell.evaluate((element) =>
window.getComputedStyle(element).backgroundColor
);
expect(fixedBackground).not.toBe("transparent");
expect(fixedBackground).not.toMatch(/rgba\([^)]*,\s*0\s*\)$/);
} }
await page.getByRole("button", { name: /^查看/ }).first().click(); await page.getByRole("button", { name: /^查看/ }).first().click();
@@ -677,7 +684,7 @@ test("recording failure recovery adapts its actions and long paths to each viewp
await expect(page.getByText(recordingFailure.filePath, { exact: true })).toBeVisible(); await expect(page.getByText(recordingFailure.filePath, { exact: true })).toBeVisible();
await expect(page.getByRole("button", { name: "确认有效" })).toBeVisible(); await expect(page.getByRole("button", { name: "确认有效" })).toBeVisible();
if ((page.viewportSize()?.width ?? 0) <= 767) { if ((page.viewportSize()?.width ?? 0) <= 768) {
await expect(page.locator(".failure-card")).toHaveCount(1); await expect(page.locator(".failure-card")).toHaveCount(1);
await expect(page.getByRole("columnheader", { name: "操作" })).toHaveCount(0); await expect(page.getByRole("columnheader", { name: "操作" })).toHaveCount(0);
} else { } else {
@@ -794,3 +801,49 @@ test("settings layer advanced controls without clipping the quick bar", async ({
await expectNoDocumentOverflow(page); await expectNoDocumentOverflow(page);
await capture(page, testInfo, "settings"); await capture(page, testInfo, "settings");
}); });
test("OpenList test distinguishes login success from a broken destination mount", async ({ page }) => {
await page.route("**/api/settings", async (route) => {
await route.fulfill({
status: 200,
contentType: "application/json",
body: JSON.stringify({
enableFileUpload: true,
enableAutoUpload: true,
uploadTarget: 3,
openListUpload: {
baseUrl: "https://openlist.example.com",
username: "tester",
password: "secret",
basePath: "/archive",
sourcePath: "/source",
destinationPath: "/archive"
}
})
});
});
await page.route("**/api/settings/openlist/test", async (route) => {
const request = route.request().postDataJSON();
expect(request.sourcePath).toBe("/source");
expect(request.destinationPath).toBe("/archive");
await route.fulfill({
status: 200,
contentType: "application/json",
body: JSON.stringify({
success: false,
version: "4.1.10",
message: "OpenList 登录成功,但一个或多个配置目录不可用。",
sourcePath: { path: "/source", success: true, canWrite: false, message: "目录可访问。" },
destinationPath: { path: "/archive", success: false, canWrite: false, message: "provider timeout" }
})
});
});
await page.goto("/settings/upload");
await expect(page.getByRole("heading", { name: "OpenList 服务端复制" })).toBeVisible();
await page.getByRole("button", { name: "测试连接" }).click();
await expect(page.getByText("配置异常", { exact: true })).toBeVisible();
await expect(page.getByText("目录可访问。", { exact: true })).toBeVisible();
await expect(page.getByText("provider timeout", { exact: true })).toBeVisible();
await expectNoDocumentOverflow(page);
});
@@ -180,7 +180,9 @@ public sealed record OperationsMetricsSnapshot(
DateTimeOffset? OldestTranscodeUpdatedAt, DateTimeOffset? OldestTranscodeUpdatedAt,
DateTimeOffset? OldestUploadProgressAt, DateTimeOffset? OldestUploadProgressAt,
int StalledUploadCount, int StalledUploadCount,
int CleanupFailureCount); int CleanupFailureCount,
DateTimeOffset? OldestActionableUploadAt,
int PendingUploadRecoveryCount);
public interface IUserAccountRepository public interface IUserAccountRepository
{ {
@@ -104,6 +104,10 @@ public class OpenListConnectionRequest
public string Username { get; set; } = string.Empty; public string Username { get; set; } = string.Empty;
public string Password { get; set; } = string.Empty; public string Password { get; set; } = string.Empty;
public string? SourcePath { get; set; }
public string? DestinationPath { get; set; }
} }
public sealed class OpenListDirectoryRequest : OpenListConnectionRequest public sealed class OpenListDirectoryRequest : OpenListConnectionRequest
@@ -134,6 +138,21 @@ public sealed class OpenListConnectionTestDto
public string? Version { get; init; } public string? Version { get; init; }
public required string Message { get; init; } public required string Message { get; init; }
public OpenListPathCheckDto? SourcePath { get; init; }
public OpenListPathCheckDto? DestinationPath { get; init; }
}
public sealed class OpenListPathCheckDto
{
public required string Path { get; init; }
public bool Success { get; init; }
public bool CanWrite { get; init; }
public required string Message { get; init; }
} }
public sealed class SystemSettingsDto public sealed class SystemSettingsDto
@@ -171,7 +171,8 @@ public sealed class OperationsMetricsRepository : IOperationsMetricsRepository
public async Task<OperationsMetricsSnapshot> GetAsync(CancellationToken cancellationToken = default) public async Task<OperationsMetricsSnapshot> GetAsync(CancellationToken cancellationToken = default)
{ {
var stalledBefore = DateTimeOffset.UtcNow.Subtract(UploadStallThreshold); var now = DateTimeOffset.UtcNow;
var stalledBefore = now.Subtract(UploadStallThreshold);
var oldestTranscodeUpdatedAt = await _dbContext.RecordTasks var oldestTranscodeUpdatedAt = await _dbContext.RecordTasks
.AsNoTracking() .AsNoTracking()
.Where(static task => task.Status == RecordTaskStatus.Processing) .Where(static task => task.Status == RecordTaskStatus.Processing)
@@ -183,9 +184,27 @@ public sealed class OperationsMetricsRepository : IOperationsMetricsRepository
.Select(job => (DateTimeOffset?)(job.LastProgressAt ?? job.ExternalTaskStartedAt ?? job.UpdatedAt)) .Select(job => (DateTimeOffset?)(job.LastProgressAt ?? job.ExternalTaskStartedAt ?? job.UpdatedAt))
.MinAsync(cancellationToken); .MinAsync(cancellationToken);
var stalledUploadCount = await _dbContext.RecordUploadJobs.CountAsync( var stalledUploadCount = await _dbContext.RecordUploadJobs.CountAsync(
job => job.Status == RecordArtifactUploadStatus.Uploading && job => (job.Status == RecordArtifactUploadStatus.Queued ||
job.Status == RecordArtifactUploadStatus.Uploading ||
(job.Status == RecordArtifactUploadStatus.WaitingRetry &&
(!job.NextAttemptAt.HasValue || job.NextAttemptAt <= now))) &&
(job.LastProgressAt ?? job.ExternalTaskStartedAt ?? job.UpdatedAt) < stalledBefore, (job.LastProgressAt ?? job.ExternalTaskStartedAt ?? job.UpdatedAt) < stalledBefore,
cancellationToken); cancellationToken);
var oldestActionableUploadAt = await _dbContext.RecordUploadJobs
.AsNoTracking()
.Where(job => job.Status == RecordArtifactUploadStatus.Queued ||
job.Status == RecordArtifactUploadStatus.Uploading ||
(job.Status == RecordArtifactUploadStatus.WaitingRetry &&
(!job.NextAttemptAt.HasValue || job.NextAttemptAt <= now)))
.Select(job => (DateTimeOffset?)(job.LastProgressAt ?? job.ExternalTaskStartedAt ?? job.UpdatedAt))
.MinAsync(cancellationToken);
var pendingUploadRecoveryCount = await _dbContext.RecordTasks.CountAsync(
task => (task.Status == RecordTaskStatus.Completed || task.Status == RecordTaskStatus.Stopped) &&
!task.IsHiddenArtifactSource &&
task.Result != null &&
task.Result.UploadStatus == RecordArtifactUploadStatus.NotUploaded &&
task.Result.FilePath != string.Empty,
cancellationToken);
var cleanupFailureCount = await _dbContext.RecordUploadJobs.CountAsync( var cleanupFailureCount = await _dbContext.RecordUploadJobs.CountAsync(
static job => job.Status == RecordArtifactUploadStatus.WaitingRetry && static job => job.Status == RecordArtifactUploadStatus.WaitingRetry &&
job.CurrentArtifact == RecordUploadArtifactStage.Completed, job.CurrentArtifact == RecordUploadArtifactStage.Completed,
@@ -194,7 +213,9 @@ public sealed class OperationsMetricsRepository : IOperationsMetricsRepository
oldestTranscodeUpdatedAt, oldestTranscodeUpdatedAt,
oldestUploadProgressAt, oldestUploadProgressAt,
stalledUploadCount, stalledUploadCount,
cleanupFailureCount); cleanupFailureCount,
oldestActionableUploadAt,
pendingUploadRecoveryCount);
} }
} }
@@ -115,34 +115,68 @@ public sealed class CompletionDispatchService
var recordSession = task.RecordSession ?? throw new InvalidOperationException(); var recordSession = task.RecordSession ?? throw new InvalidOperationException();
var recordResult = task.Result ?? throw new InvalidOperationException(); var recordResult = task.Result ?? throw new InvalidOperationException();
var dispatchErrors = new List<string>();
if (!dispatch.ScriptDispatched) if (!dispatch.ScriptDispatched)
{ {
var scriptResult = await _eventScriptService.RunSegmentCompletedAsync( try
task.LiveRoom,
recordSession,
task,
recordResult,
recordResult.FilePath,
task.EndedAt ?? DateTimeOffset.UtcNow,
eventId: dispatch.Id,
cancellationToken: cancellationToken);
if (scriptResult is null || scriptResult.Success)
{ {
dispatch.MarkScriptDispatched(DateTimeOffset.UtcNow); var scriptResult = await _eventScriptService.RunSegmentCompletedAsync(
await _dbContext.SaveChangesAsync(cancellationToken); task.LiveRoom,
recordSession,
task,
recordResult,
recordResult.FilePath,
task.EndedAt ?? DateTimeOffset.UtcNow,
eventId: dispatch.Id,
cancellationToken: cancellationToken);
if (scriptResult is null || scriptResult.Success)
{
dispatch.MarkScriptDispatched(DateTimeOffset.UtcNow);
}
else
{
dispatchErrors.Add($"事件脚本:{scriptResult.Message}");
}
} }
else catch (Exception ex) when (ex is not OperationCanceledException || !cancellationToken.IsCancellationRequested)
{ {
throw new InvalidOperationException(scriptResult.Message); dispatchErrors.Add($"事件脚本:{ex.Message}");
} }
} }
if (!dispatch.UploadDispatched) if (!dispatch.UploadDispatched)
{ {
_ = await _recordUploadService.TryAutoUploadTaskAsync(task.Id, cancellationToken); try
dispatch.MarkUploadDispatched(DateTimeOffset.UtcNow); {
await _dbContext.SaveChangesAsync(cancellationToken); var uploadResult = await _recordUploadService.TryAutoUploadTaskAsync(task.Id, cancellationToken);
if (uploadResult is null || uploadResult.Success)
{
dispatch.MarkUploadDispatched(DateTimeOffset.UtcNow);
}
else
{
dispatchErrors.Add($"自动上传:{uploadResult.Message}");
}
}
catch (Exception ex) when (ex is not OperationCanceledException || !cancellationToken.IsCancellationRequested)
{
dispatchErrors.Add($"自动上传:{ex.Message}");
}
} }
if (dispatchErrors.Count > 0)
{
dispatch.ScheduleRetry(
string.Join("", dispatchErrors),
DateTimeOffset.UtcNow.Add(RetryDelay),
DateTimeOffset.UtcNow);
}
await _dbContext.SaveChangesAsync(cancellationToken);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
throw;
} }
catch (Exception ex) catch (Exception ex)
{ {
@@ -170,16 +170,70 @@ public sealed class OpenListClient : IOpenListClient
version = versionElement.GetString(); version = versionElement.GetString();
} }
var sourcePath = await TestDirectoryAsync(connection, connection.SourcePath, requireWrite: false, cancellationToken);
var destinationPath = await TestDirectoryAsync(connection, connection.DestinationPath, requireWrite: true, cancellationToken);
var success = (sourcePath?.Success ?? true) && (destinationPath?.Success ?? true);
return new OpenListConnectionTestDto return new OpenListConnectionTestDto
{ {
Success = true, Success = success,
Version = version, Version = version,
Message = string.IsNullOrWhiteSpace(version) Message = success
? "OpenList 连接和登录成功。" ? (string.IsNullOrWhiteSpace(version)
: $"OpenList 连接和登录成功:{version}" ? "OpenList 登录和目录检查成功。"
: $"OpenList 登录和目录检查成功:{version}")
: "OpenList 登录成功,但一个或多个配置目录不可用。",
SourcePath = sourcePath,
DestinationPath = destinationPath
}; };
} }
private async Task<OpenListPathCheckDto?> TestDirectoryAsync(
OpenListConnectionRequest connection,
string? path,
bool requireWrite,
CancellationToken cancellationToken)
{
if (string.IsNullOrWhiteSpace(path))
{
return null;
}
var normalizedPath = NormalizePath(path);
try
{
var directory = await ListDirectoriesAsync(
new OpenListDirectoryRequest
{
BaseUrl = connection.BaseUrl,
Username = connection.Username,
Password = connection.Password,
Path = normalizedPath
},
cancellationToken);
var success = !requireWrite || directory.CanWrite;
return new OpenListPathCheckDto
{
Path = normalizedPath,
Success = success,
CanWrite = directory.CanWrite,
Message = success
? (requireWrite ? "目录可访问且允许写入。" : "目录可访问。")
: "目录可访问,但 OpenList 未授予写入权限。"
};
}
catch (Exception ex) when (ex is not OperationCanceledException || !cancellationToken.IsCancellationRequested)
{
return new OpenListPathCheckDto
{
Path = normalizedPath,
Success = false,
CanWrite = false,
Message = ex.Message
};
}
}
public async Task<OpenListDirectoryListDto> ListDirectoriesAsync( public async Task<OpenListDirectoryListDto> ListDirectoriesAsync(
OpenListDirectoryRequest request, OpenListDirectoryRequest request,
CancellationToken cancellationToken = default) CancellationToken cancellationToken = default)
@@ -15,6 +15,16 @@ public sealed record OpenListQueueHealthSnapshot(
DateTimeOffset? LastErrorAt) DateTimeOffset? LastErrorAt)
{ {
public bool IsPaused => Status != OpenListQueueHealthStatus.Healthy; public bool IsPaused => Status != OpenListQueueHealthStatus.Healthy;
public DateTimeOffset? LastWorkerIterationAt { get; init; }
public DateTimeOffset? LastRecoveryAt { get; init; }
public int LastRecoveredCount { get; init; }
public string? LastWorkerError { get; init; }
public string? LastRecoveryError { get; init; }
} }
/// <summary> /// <summary>
@@ -32,12 +42,61 @@ public sealed class OpenListUploadHealthState
private DateTimeOffset? _retryAt; private DateTimeOffset? _retryAt;
private DateTimeOffset? _lastErrorAt; private DateTimeOffset? _lastErrorAt;
private int _consecutiveRateLimits; private int _consecutiveRateLimits;
private DateTimeOffset? _lastWorkerIterationAt;
private DateTimeOffset? _lastRecoveryAt;
private int _lastRecoveredCount;
private string? _lastWorkerError;
private string? _lastRecoveryError;
public OpenListQueueHealthSnapshot GetSnapshot() public OpenListQueueHealthSnapshot GetSnapshot()
{ {
lock (_gate) lock (_gate)
{ {
return new OpenListQueueHealthSnapshot(_status, _reason, _retryAt, _lastErrorAt); return new OpenListQueueHealthSnapshot(_status, _reason, _retryAt, _lastErrorAt)
{
LastWorkerIterationAt = _lastWorkerIterationAt,
LastRecoveryAt = _lastRecoveryAt,
LastRecoveredCount = _lastRecoveredCount,
LastWorkerError = _lastWorkerError,
LastRecoveryError = _lastRecoveryError
};
}
}
public void MarkWorkerIteration(DateTimeOffset occurredAt)
{
lock (_gate)
{
_lastWorkerIterationAt = occurredAt;
_lastWorkerError = null;
}
}
public void MarkRecovery(DateTimeOffset occurredAt, int recoveredCount)
{
lock (_gate)
{
_lastRecoveryAt = occurredAt;
_lastRecoveredCount = Math.Max(0, recoveredCount);
_lastRecoveryError = null;
}
}
public void MarkWorkerError(DateTimeOffset occurredAt, string error)
{
lock (_gate)
{
_lastWorkerIterationAt = occurredAt;
_lastWorkerError = string.IsNullOrWhiteSpace(error) ? null : error.Trim();
}
}
public void MarkRecoveryError(DateTimeOffset occurredAt, string error)
{
lock (_gate)
{
_lastRecoveryAt = occurredAt;
_lastRecoveryError = string.IsNullOrWhiteSpace(error) ? null : error.Trim();
} }
} }
@@ -153,6 +153,16 @@ public sealed class OpenListUploadQueueService
{ {
recovered++; recovered++;
} }
else if (!result.Success)
{
await _systemLogService.WriteAsync(
SystemLogLevel.Warning,
"Upload",
"跳过一个无法恢复的 OpenList 自动上传任务。",
$"recordTaskId={taskId}; error={result.Message}",
recordTaskId: taskId,
cancellationToken: cancellationToken);
}
} }
catch (Exception ex) catch (Exception ex)
{ {
@@ -1346,21 +1356,22 @@ public sealed class OpenListUploadQueueService
public sealed class OpenListUploadBackgroundService : BackgroundService public sealed class OpenListUploadBackgroundService : BackgroundService
{ {
private static readonly TimeSpan IdleDelay = TimeSpan.FromSeconds(2); private static readonly TimeSpan IdleDelay = TimeSpan.FromSeconds(2);
private static readonly TimeSpan RecoveryInterval = TimeSpan.FromMinutes(1);
private readonly IServiceScopeFactory _scopeFactory; private readonly IServiceScopeFactory _scopeFactory;
private readonly OpenListUploadHealthState _healthState;
private readonly ILogger<OpenListUploadBackgroundService> _logger; private readonly ILogger<OpenListUploadBackgroundService> _logger;
public OpenListUploadBackgroundService( public OpenListUploadBackgroundService(
IServiceScopeFactory scopeFactory, IServiceScopeFactory scopeFactory,
OpenListUploadHealthState healthState,
ILogger<OpenListUploadBackgroundService> logger) ILogger<OpenListUploadBackgroundService> logger)
{ {
_scopeFactory = scopeFactory; _scopeFactory = scopeFactory;
_healthState = healthState;
_logger = logger; _logger = logger;
} }
protected override async Task ExecuteAsync(CancellationToken stoppingToken) protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{ {
var nextRecoveryAt = DateTimeOffset.MinValue;
while (!stoppingToken.IsCancellationRequested) while (!stoppingToken.IsCancellationRequested)
{ {
try try
@@ -1368,11 +1379,7 @@ public sealed class OpenListUploadBackgroundService : BackgroundService
using var scope = _scopeFactory.CreateScope(); using var scope = _scopeFactory.CreateScope();
var queue = scope.ServiceProvider.GetRequiredService<OpenListUploadQueueService>(); var queue = scope.ServiceProvider.GetRequiredService<OpenListUploadQueueService>();
_ = await queue.ProcessNextAsync(stoppingToken); _ = await queue.ProcessNextAsync(stoppingToken);
if (DateTimeOffset.UtcNow >= nextRecoveryAt) _healthState.MarkWorkerIteration(DateTimeOffset.UtcNow);
{
await queue.RecoverPendingAutomaticUploadsAsync(cancellationToken: stoppingToken);
nextRecoveryAt = DateTimeOffset.UtcNow.Add(RecoveryInterval);
}
await Task.Delay(IdleDelay, stoppingToken); await Task.Delay(IdleDelay, stoppingToken);
} }
@@ -1382,6 +1389,7 @@ public sealed class OpenListUploadBackgroundService : BackgroundService
} }
catch (Exception ex) catch (Exception ex)
{ {
_healthState.MarkWorkerError(DateTimeOffset.UtcNow, ex.Message);
_logger.LogError(ex, "OpenList upload background worker failed"); _logger.LogError(ex, "OpenList upload background worker failed");
try try
{ {
@@ -1395,3 +1403,53 @@ public sealed class OpenListUploadBackgroundService : BackgroundService
} }
} }
} }
public sealed class OpenListAutomaticRecoveryBackgroundService : BackgroundService
{
private static readonly TimeSpan RecoveryInterval = TimeSpan.FromMinutes(1);
private readonly IServiceScopeFactory _scopeFactory;
private readonly OpenListUploadHealthState _healthState;
private readonly ILogger<OpenListAutomaticRecoveryBackgroundService> _logger;
public OpenListAutomaticRecoveryBackgroundService(
IServiceScopeFactory scopeFactory,
OpenListUploadHealthState healthState,
ILogger<OpenListAutomaticRecoveryBackgroundService> logger)
{
_scopeFactory = scopeFactory;
_healthState = healthState;
_logger = logger;
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
while (!stoppingToken.IsCancellationRequested)
{
try
{
using var scope = _scopeFactory.CreateScope();
var queue = scope.ServiceProvider.GetRequiredService<OpenListUploadQueueService>();
var recovered = await queue.RecoverPendingAutomaticUploadsAsync(cancellationToken: stoppingToken);
_healthState.MarkRecovery(DateTimeOffset.UtcNow, recovered);
await Task.Delay(RecoveryInterval, stoppingToken);
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
{
break;
}
catch (Exception ex)
{
_healthState.MarkRecoveryError(DateTimeOffset.UtcNow, ex.Message);
_logger.LogError(ex, "OpenList automatic upload recovery failed");
try
{
await Task.Delay(TimeSpan.FromSeconds(10), stoppingToken);
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
{
break;
}
}
}
}
}
+21 -3
View File
@@ -253,6 +253,7 @@ builder.Services.AddHostedService(provider => provider.GetRequiredService<LiveRo
builder.Services.AddHostedService<CleanupOperationBackgroundService>(); builder.Services.AddHostedService<CleanupOperationBackgroundService>();
builder.Services.AddHostedService<RetentionCleanupBackgroundService>(); builder.Services.AddHostedService<RetentionCleanupBackgroundService>();
builder.Services.AddHostedService<OpenListUploadBackgroundService>(); builder.Services.AddHostedService<OpenListUploadBackgroundService>();
builder.Services.AddHostedService<OpenListAutomaticRecoveryBackgroundService>();
builder.Services.AddHostedService<CompletionDispatchBackgroundService>(); builder.Services.AddHostedService<CompletionDispatchBackgroundService>();
builder.Services.AddHostedService<SystemLogRetentionBackgroundService>(); builder.Services.AddHostedService<SystemLogRetentionBackgroundService>();
@@ -338,18 +339,34 @@ app.MapGet("/api/operations/health", async (
IOperationsMetricsRepository metricsRepository, IOperationsMetricsRepository metricsRepository,
ISystemSettingsService settingsService, ISystemSettingsService settingsService,
IStorageGuardService storageGuardService, IStorageGuardService storageGuardService,
OpenListUploadHealthState uploadHealthState,
CancellationToken cancellationToken) => CancellationToken cancellationToken) =>
{ {
var metrics = await metricsRepository.GetAsync(cancellationToken); var metrics = await metricsRepository.GetAsync(cancellationToken);
var settings = await settingsService.GetAsync(cancellationToken); var settings = await settingsService.GetAsync(cancellationToken);
var storage = storageGuardService.CheckCanStartOrResume(settings); var storage = storageGuardService.CheckCanStartOrResume(settings);
var uploadQueue = uploadHealthState.GetSnapshot();
var now = DateTimeOffset.UtcNow;
var monitorOpenList = settings.EnableFileUpload &&
settings.EnableAutoUpload &&
settings.UploadTarget == UploadTargetType.OpenList;
var workerHeartbeatStale = monitorOpenList &&
uploadQueue.LastWorkerIterationAt.HasValue &&
uploadQueue.LastWorkerIterationAt < now.AddMinutes(-5);
var recoveryHeartbeatStale = monitorOpenList &&
uploadQueue.LastRecoveryAt.HasValue &&
uploadQueue.LastRecoveryAt < now.AddMinutes(-5);
var degraded = storage.Tier == StorageTier.Red || var degraded = storage.Tier == StorageTier.Red ||
metrics.StalledUploadCount > 0 || metrics.StalledUploadCount > 0 ||
metrics.CleanupFailureCount > 0; metrics.CleanupFailureCount > 0 ||
!string.IsNullOrWhiteSpace(uploadQueue.LastWorkerError) ||
!string.IsNullOrWhiteSpace(uploadQueue.LastRecoveryError) ||
workerHeartbeatStale ||
recoveryHeartbeatStale;
return Results.Ok(new return Results.Ok(new
{ {
status = degraded ? "degraded" : "healthy", status = degraded ? "degraded" : "healthy",
timestamp = DateTimeOffset.UtcNow, timestamp = now,
storage = new storage = new
{ {
tier = storage.Tier.ToString(), tier = storage.Tier.ToString(),
@@ -357,7 +374,8 @@ app.MapGet("/api/operations/health", async (
storage.AvailableBytes, storage.AvailableBytes,
storage.Message storage.Message
}, },
operations = metrics operations = metrics,
uploadQueue
}); });
}); });
@@ -4,6 +4,7 @@ using System.Text;
using System.Text.Json; using System.Text.Json;
using LiveRecorder.Application.Abstractions.Logging; using LiveRecorder.Application.Abstractions.Logging;
using LiveRecorder.Application.Abstractions.Recording; using LiveRecorder.Application.Abstractions.Recording;
using LiveRecorder.Application.Abstractions.Scripting;
using LiveRecorder.Application.Abstractions.Settings; using LiveRecorder.Application.Abstractions.Settings;
using LiveRecorder.Application.Models.Logs; using LiveRecorder.Application.Models.Logs;
using LiveRecorder.Application.Models.Settings; using LiveRecorder.Application.Models.Settings;
@@ -14,11 +15,83 @@ using LiveRecorder.Infrastructure.Persistence.Repositories;
using LiveRecorder.Infrastructure.Services; using LiveRecorder.Infrastructure.Services;
using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Storage; using Microsoft.EntityFrameworkCore.Storage;
using Microsoft.Extensions.Logging.Abstractions;
namespace LiveRecorder.Tests; namespace LiveRecorder.Tests;
public sealed class OpenListUploadTests public sealed class OpenListUploadTests
{ {
[Fact]
public async Task TestConnection_ReportsDestinationMountFailureAfterSuccessfulLogin()
{
var handler = new StubHttpMessageHandler(async request =>
{
var path = request.RequestUri!.AbsolutePath;
if (path.EndsWith("/api/auth/login", StringComparison.Ordinal))
{
return JsonResponse("""{"code":200,"message":"success","data":{"token":"test-token"}}""");
}
if (path.EndsWith("/api/public/settings", StringComparison.Ordinal))
{
return JsonResponse("""{"code":200,"message":"success","data":{"version":"4.1.10"}}""");
}
var body = JsonDocument.Parse(await request.Content!.ReadAsStringAsync()).RootElement;
return body.GetProperty("path").GetString() == "/source"
? JsonResponse("""{"code":200,"message":"success","data":{"content":[],"write":false}}""")
: JsonResponse("""{"code":500,"message":"provider timeout","data":null}""");
});
var client = CreateClient(handler);
var connection = Connection();
connection.SourcePath = "/source";
connection.DestinationPath = "/destination";
var result = await client.TestConnectionAsync(connection);
Assert.False(result.Success);
Assert.Equal("4.1.10", result.Version);
Assert.True(result.SourcePath?.Success);
Assert.False(result.DestinationPath?.Success);
Assert.Contains("provider timeout", result.DestinationPath?.Message);
}
[Fact]
public async Task CompletionDispatch_ScriptFailureDoesNotBlockAutomaticUpload()
{
await using var fixture = await QueueFixture.CreateAsync();
await fixture.PrepareCompletionDispatchAsync();
var service = fixture.CreateCompletionDispatchService(new FixedEventScriptService(
new EventScriptExecutionResultDto { Success = false, Message = "script failed" }));
await service.TryDispatchTaskAsync(fixture.RecordTaskId);
var dispatch = await fixture.Context.RecordCompletionDispatches.SingleAsync();
Assert.False(dispatch.ScriptDispatched);
Assert.True(dispatch.UploadDispatched);
Assert.Null(dispatch.CompletedAt);
Assert.Contains("script failed", dispatch.LastError);
Assert.NotNull(await fixture.Context.RecordUploadJobs.SingleOrDefaultAsync());
}
[Fact]
public async Task CompletionDispatch_UploadRejectionRemainsRetryable()
{
await using var fixture = await QueueFixture.CreateAsync(
new VideoMetadata(0.18, 1920, 1080, "h264", "aac", 30, 4_000_000));
await fixture.PrepareCompletionDispatchAsync();
var service = fixture.CreateCompletionDispatchService(new FixedEventScriptService(null));
await service.TryDispatchTaskAsync(fixture.RecordTaskId);
var dispatch = await fixture.Context.RecordCompletionDispatches.SingleAsync();
Assert.True(dispatch.ScriptDispatched);
Assert.False(dispatch.UploadDispatched);
Assert.Null(dispatch.CompletedAt);
Assert.Contains("自动上传", dispatch.LastError);
Assert.True(dispatch.NextAttemptAt > DateTimeOffset.UtcNow);
}
[Fact] [Fact]
public async Task Enqueue_RejectsShortMediaAndKeepsItNotUploaded() public async Task Enqueue_RejectsShortMediaAndKeepsItNotUploaded()
{ {
@@ -763,6 +836,30 @@ public sealed class OpenListUploadTests
public Guid RecordTaskId { get; } public Guid RecordTaskId { get; }
public async Task PrepareCompletionDispatchAsync()
{
var session = await Context.RecordSessions.SingleAsync();
session.MarkCompleted(DateTimeOffset.UtcNow);
if (!await Context.RecordCompletionDispatches.AnyAsync())
{
Context.RecordCompletionDispatches.Add(
new RecordCompletionDispatch(RecordTaskId, DateTimeOffset.UtcNow));
}
await Context.SaveChangesAsync();
}
public CompletionDispatchService CreateCompletionDispatchService(IEventScriptService eventScriptService)
{
var uploadService = new RecordUploadService(Context, _settingsService, new NullSystemLogService(), Queue);
var consolidation = new ShortFragmentConsolidationService(
Context,
_settingsService,
_videoMetadataService,
NullLogger<ShortFragmentConsolidationService>.Instance);
return new CompletionDispatchService(Context, eventScriptService, uploadService, consolidation);
}
public static async Task<QueueFixture> CreateAsync(VideoMetadata? metadata = null) public static async Task<QueueFixture> CreateAsync(VideoMetadata? metadata = null)
{ {
var temporaryRoot = Path.Combine(Path.GetTempPath(), $"live-recorder-openlist-{Guid.NewGuid():N}"); var temporaryRoot = Path.Combine(Path.GetTempPath(), $"live-recorder-openlist-{Guid.NewGuid():N}");
@@ -886,6 +983,42 @@ public sealed class OpenListUploadTests
CancellationToken cancellationToken = default) => Task.FromResult<string?>(null); CancellationToken cancellationToken = default) => Task.FromResult<string?>(null);
} }
private sealed class FixedEventScriptService : IEventScriptService
{
private readonly EventScriptExecutionResultDto? _segmentResult;
public FixedEventScriptService(EventScriptExecutionResultDto? segmentResult)
{
_segmentResult = segmentResult;
}
public Task<EventScriptExecutionResultDto?> RunLiveStartedAsync(
LiveRoom liveRoom,
DateTimeOffset occurredAt,
CancellationToken cancellationToken = default) => Task.FromResult<EventScriptExecutionResultDto?>(null);
public Task<EventScriptExecutionResultDto?> RunLiveEndedAsync(
LiveRoom liveRoom,
DateTimeOffset occurredAt,
CancellationToken cancellationToken = default) => Task.FromResult<EventScriptExecutionResultDto?>(null);
public Task<EventScriptExecutionResultDto?> RunSegmentCompletedAsync(
LiveRoom? liveRoom,
RecordSession recordSession,
RecordTask recordTask,
RecordResult? recordResult,
string segmentFilePath,
DateTimeOffset occurredAt,
bool forceRun = false,
Guid? eventId = null,
CancellationToken cancellationToken = default) => Task.FromResult(_segmentResult);
public Task<EventScriptTestResultDto> TestAsync(
TestEventScriptRequest request,
CancellationToken cancellationToken = default) =>
Task.FromResult(new EventScriptTestResultDto { Success = true, Message = "ok" });
}
private sealed class FixedSettingsService : ISystemSettingsService private sealed class FixedSettingsService : ISystemSettingsService
{ {
private readonly SystemSettingsDto _settings; private readonly SystemSettingsDto _settings;