Compare commits
2
Commits
c056a3e741
...
8960b68ac8
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8960b68ac8 | ||
|
|
f0fb4db755 |
Vendored
+2
@@ -273,6 +273,8 @@ pipeline {
|
|||||||
--driver-opt "env.https_proxy=$HTTP_PROXY_URL" \
|
--driver-opt "env.https_proxy=$HTTP_PROXY_URL" \
|
||||||
--driver-opt "env.HTTP_PROXY=$HTTP_PROXY_URL" \
|
--driver-opt "env.HTTP_PROXY=$HTTP_PROXY_URL" \
|
||||||
--driver-opt "env.HTTPS_PROXY=$HTTP_PROXY_URL" \
|
--driver-opt "env.HTTPS_PROXY=$HTTP_PROXY_URL" \
|
||||||
|
--driver-opt "env.no_proxy=.daocloud.vip" \
|
||||||
|
--driver-opt "env.NO_PROXY=.daocloud.vip" \
|
||||||
--use
|
--use
|
||||||
fi
|
fi
|
||||||
./scripts/ci-docker.sh buildx inspect "$BUILDER_NAME" --bootstrap
|
./scripts/ci-docker.sh buildx inspect "$BUILDER_NAME" --bootstrap
|
||||||
|
|||||||
@@ -284,7 +284,7 @@ docker compose up -d
|
|||||||
|
|
||||||
- `dotnet build LiveRecorder.sln --no-restore -m:1`
|
- `dotnet build LiveRecorder.sln --no-restore -m:1`
|
||||||
- `npm run build`(frontend 目录)
|
- `npm run build`(frontend 目录)
|
||||||
- `npm run test:e2e`(frontend 目录,覆盖 1920 / 1366 / 1024 / 768 / 390)
|
- `npm run test:e2e`(frontend 目录,覆盖 1920 / 1552 / 1366 / 1024 / 768 / 390)
|
||||||
- Docker Compose 完整部署验证通过
|
- Docker Compose 完整部署验证通过
|
||||||
|
|
||||||
## 后续规划
|
## 后续规划
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ import { defineConfig } from "@playwright/test";
|
|||||||
|
|
||||||
const viewports = [
|
const viewports = [
|
||||||
{ name: "desktop-1920", width: 1920, height: 1080 },
|
{ name: "desktop-1920", width: 1920, height: 1080 },
|
||||||
|
{ name: "desktop-1552", width: 1552, height: 761 },
|
||||||
{ name: "desktop-1366", width: 1366, height: 768 },
|
{ name: "desktop-1366", width: 1366, height: 768 },
|
||||||
{ name: "tablet-1024", width: 1024, height: 768 },
|
{ name: "tablet-1024", width: 1024, height: 768 },
|
||||||
{ name: "tablet-768", width: 768, height: 1024 },
|
{ name: "tablet-768", width: 768, height: 1024 },
|
||||||
@@ -21,7 +22,8 @@ export default defineConfig({
|
|||||||
trace: "retain-on-failure"
|
trace: "retain-on-failure"
|
||||||
},
|
},
|
||||||
webServer: {
|
webServer: {
|
||||||
command: "VITE_DISABLE_DEVTOOLS=1 npm run dev -- --host 127.0.0.1 --port 47173 --strictPort",
|
command: "npm run dev -- --host 127.0.0.1 --port 47173 --strictPort",
|
||||||
|
env: { VITE_DISABLE_DEVTOOLS: "1" },
|
||||||
url: "http://127.0.0.1:47173",
|
url: "http://127.0.0.1:47173",
|
||||||
reuseExistingServer: false,
|
reuseExistingServer: false,
|
||||||
timeout: 120_000
|
timeout: 120_000
|
||||||
|
|||||||
@@ -355,19 +355,24 @@ button { font-family: inherit; cursor: pointer; }
|
|||||||
|
|
||||||
/* tables */
|
/* tables */
|
||||||
.el-table {
|
.el-table {
|
||||||
--el-table-border-color: transparent; --el-table-header-bg-color: transparent;
|
--el-table-border-color: transparent; --el-table-header-bg-color: var(--surface-muted);
|
||||||
--el-table-bg-color: transparent; --el-table-row-hover-bg-color: var(--surface-hover);
|
--el-table-bg-color: var(--surface); --el-table-tr-bg-color: var(--surface);
|
||||||
--el-table-current-row-bg-color: var(--accent-soft); --el-fill-color-blank: transparent;
|
--el-table-row-hover-bg-color: var(--surface-hover);
|
||||||
background: transparent;
|
--el-table-current-row-bg-color: var(--accent-soft); --el-fill-color-blank: var(--surface);
|
||||||
|
background: var(--surface);
|
||||||
}
|
}
|
||||||
.el-table::before, .el-table__inner-wrapper::before { display: none; }
|
.el-table::before, .el-table__inner-wrapper::before { display: none; }
|
||||||
.el-table th.el-table__cell { padding: 11px 16px; border-bottom: 1px solid var(--border-subtle); background: var(--surface-muted); }
|
.el-table th.el-table__cell { padding: 11px 16px; border-bottom: 1px solid var(--border-subtle); background: var(--surface-muted); }
|
||||||
html[data-theme="dark"] .el-table th.el-table__cell { background: var(--surface-muted); }
|
html[data-theme="dark"] .el-table th.el-table__cell { background: var(--surface-muted); }
|
||||||
.el-table th.el-table__cell > .cell { color: var(--text-muted); font-size: 12px; font-weight: 700; letter-spacing: .02em; text-transform: uppercase; }
|
.el-table th.el-table__cell > .cell { color: var(--text-muted); font-size: 12px; font-weight: 700; letter-spacing: .02em; text-transform: uppercase; }
|
||||||
.el-table td.el-table__cell { padding: 13px 16px; border-bottom: 1px solid var(--border-subtle); background: transparent; }
|
.el-table td.el-table__cell { padding: 13px 16px; border-bottom: 1px solid var(--border-subtle); background: var(--surface); }
|
||||||
.el-table tr { background: transparent; }
|
.el-table tr { background: var(--surface); }
|
||||||
.el-table tbody tr:nth-child(even) { background: rgba(15, 23, 42, 0.018); }
|
.el-table tbody tr:nth-child(even) > td.el-table__cell { background: color-mix(in srgb, var(--surface-strong) 24%, var(--surface)); }
|
||||||
html[data-theme="dark"] .el-table tbody tr:nth-child(even) { background: rgba(255, 255, 255, 0.02); }
|
.el-table tbody tr:hover > td.el-table__cell { background: var(--surface-hover); }
|
||||||
|
.el-table .el-table-fixed-column--right,
|
||||||
|
.el-table .el-table-fixed-column--left { background: inherit; }
|
||||||
|
.el-table .el-table-fixed-column--right.is-first-column { box-shadow: -10px 0 18px -16px rgba(23, 35, 58, .55); }
|
||||||
|
html[data-theme="dark"] .el-table .el-table-fixed-column--right.is-first-column { box-shadow: -10px 0 20px -15px rgba(0, 0, 0, .9); }
|
||||||
.premium-table { width: 100%; }
|
.premium-table { width: 100%; }
|
||||||
|
|
||||||
/* descriptions */
|
/* descriptions */
|
||||||
|
|||||||
@@ -653,6 +653,15 @@ export interface OpenListConnectionTestResult {
|
|||||||
success: boolean;
|
success: boolean;
|
||||||
version?: string;
|
version?: string;
|
||||||
message: string;
|
message: string;
|
||||||
|
sourcePath?: OpenListPathCheck;
|
||||||
|
destinationPath?: OpenListPathCheck;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface OpenListPathCheck {
|
||||||
|
path: string;
|
||||||
|
success: boolean;
|
||||||
|
canWrite: boolean;
|
||||||
|
message: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface OpenListDirectoryItem {
|
export interface OpenListDirectoryItem {
|
||||||
|
|||||||
@@ -53,6 +53,7 @@ const changingPassword = ref(false);
|
|||||||
const testingEmail = ref(false);
|
const testingEmail = ref(false);
|
||||||
const testingWebhook = ref(false);
|
const testingWebhook = ref(false);
|
||||||
const testingOpenList = ref(false);
|
const testingOpenList = ref(false);
|
||||||
|
const openListTestResult = ref<OpenListConnectionTestResult | null>(null);
|
||||||
const openListDirectoryLoading = ref(false);
|
const openListDirectoryLoading = ref(false);
|
||||||
const openListDirectoryDialogVisible = ref(false);
|
const openListDirectoryDialogVisible = ref(false);
|
||||||
const openListDirectoryPickerTarget = ref<"source" | "destination">("source");
|
const openListDirectoryPickerTarget = ref<"source" | "destination">("source");
|
||||||
@@ -568,7 +569,9 @@ function buildOpenListConnectionPayload() {
|
|||||||
return {
|
return {
|
||||||
baseUrl: form.openListUpload.baseUrl,
|
baseUrl: form.openListUpload.baseUrl,
|
||||||
username: form.openListUpload.username,
|
username: form.openListUpload.username,
|
||||||
password: form.openListUpload.password
|
password: form.openListUpload.password,
|
||||||
|
sourcePath: form.openListUpload.sourcePath,
|
||||||
|
destinationPath: form.openListUpload.destinationPath
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -580,8 +583,10 @@ async function testOpenListConnection() {
|
|||||||
"/settings/openlist/test",
|
"/settings/openlist/test",
|
||||||
buildOpenListConnectionPayload()
|
buildOpenListConnectionPayload()
|
||||||
);
|
);
|
||||||
|
openListTestResult.value = data;
|
||||||
ElMessage[data.success ? "success" : "warning"](data.message);
|
ElMessage[data.success ? "success" : "warning"](data.message);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
openListTestResult.value = null;
|
||||||
ElMessage.error(getApiErrorMessage(error, "OpenList 连接测试失败"));
|
ElMessage.error(getApiErrorMessage(error, "OpenList 连接测试失败"));
|
||||||
} finally {
|
} finally {
|
||||||
testingOpenList.value = false;
|
testingOpenList.value = false;
|
||||||
@@ -1646,6 +1651,29 @@ onBeforeRouteLeave(async () => {
|
|||||||
<el-button :loading="testingOpenList" @click="testOpenListConnection">测试连接</el-button>
|
<el-button :loading="testingOpenList" @click="testOpenListConnection">测试连接</el-button>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<div
|
||||||
|
v-if="openListTestResult"
|
||||||
|
class="openlist-test-result"
|
||||||
|
:class="{ 'openlist-test-result--failed': !openListTestResult.success }"
|
||||||
|
>
|
||||||
|
<div class="openlist-test-result__summary">
|
||||||
|
<el-tag :type="openListTestResult.success ? 'success' : 'warning'">
|
||||||
|
{{ openListTestResult.success ? "配置可用" : "配置异常" }}
|
||||||
|
</el-tag>
|
||||||
|
<span>{{ openListTestResult.message }}</span>
|
||||||
|
</div>
|
||||||
|
<div v-if="openListTestResult.sourcePath" class="openlist-test-result__path">
|
||||||
|
<el-tag :type="openListTestResult.sourcePath.success ? 'success' : 'danger'" size="small">源目录</el-tag>
|
||||||
|
<code>{{ openListTestResult.sourcePath.path }}</code>
|
||||||
|
<span>{{ openListTestResult.sourcePath.message }}</span>
|
||||||
|
</div>
|
||||||
|
<div v-if="openListTestResult.destinationPath" class="openlist-test-result__path">
|
||||||
|
<el-tag :type="openListTestResult.destinationPath.success ? 'success' : 'danger'" size="small">目标目录</el-tag>
|
||||||
|
<code>{{ openListTestResult.destinationPath.path }}</code>
|
||||||
|
<span>{{ openListTestResult.destinationPath.message }}</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
<el-form label-position="top">
|
<el-form label-position="top">
|
||||||
<el-row :gutter="16">
|
<el-row :gutter="16">
|
||||||
<el-col :span="12">
|
<el-col :span="12">
|
||||||
@@ -1965,6 +1993,14 @@ onBeforeRouteLeave(async () => {
|
|||||||
</el-button>
|
</el-button>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
<el-alert
|
||||||
|
v-if="form.enableFileUpload && form.enableAutoUpload && form.enableSegmentCompletedScript"
|
||||||
|
class="event-script-conflict-alert"
|
||||||
|
type="warning"
|
||||||
|
:closable="false"
|
||||||
|
show-icon
|
||||||
|
title="内置自动上传与分片完成脚本会独立执行。请勿在该脚本中重复实现上传,以免重复提交或因脚本故障干扰完成事件。"
|
||||||
|
/>
|
||||||
<el-form-item v-if="form.segmentCompletedScriptMode === 'path'" label="脚本路径">
|
<el-form-item v-if="form.segmentCompletedScriptMode === 'path'" label="脚本路径">
|
||||||
<el-input v-model="form.segmentCompletedScriptPath" placeholder="/app/scripts/segment-completed.sh" />
|
<el-input v-model="form.segmentCompletedScriptPath" placeholder="/app/scripts/segment-completed.sh" />
|
||||||
</el-form-item>
|
</el-form-item>
|
||||||
@@ -2828,6 +2864,41 @@ onBeforeRouteLeave(async () => {
|
|||||||
color: var(--text-secondary);
|
color: var(--text-secondary);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.openlist-test-result {
|
||||||
|
display: grid;
|
||||||
|
gap: 10px;
|
||||||
|
margin-bottom: 16px;
|
||||||
|
padding: 12px 14px;
|
||||||
|
border: 1px solid color-mix(in srgb, var(--success) 30%, var(--border-subtle));
|
||||||
|
border-radius: var(--radius-sm);
|
||||||
|
background: var(--success-soft);
|
||||||
|
}
|
||||||
|
|
||||||
|
.openlist-test-result--failed {
|
||||||
|
border-color: color-mix(in srgb, var(--warning) 34%, var(--border-subtle));
|
||||||
|
background: var(--warning-soft);
|
||||||
|
}
|
||||||
|
|
||||||
|
.openlist-test-result__summary,
|
||||||
|
.openlist-test-result__path {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
flex-wrap: wrap;
|
||||||
|
gap: 8px;
|
||||||
|
color: var(--text-secondary);
|
||||||
|
font-size: 12px;
|
||||||
|
line-height: 1.55;
|
||||||
|
}
|
||||||
|
|
||||||
|
.openlist-test-result__path code {
|
||||||
|
color: var(--text-primary);
|
||||||
|
overflow-wrap: anywhere;
|
||||||
|
}
|
||||||
|
|
||||||
|
.event-script-conflict-alert {
|
||||||
|
margin-bottom: 16px;
|
||||||
|
}
|
||||||
|
|
||||||
.template-help {
|
.template-help {
|
||||||
margin-top: 8px;
|
margin-top: 8px;
|
||||||
padding: 16px 18px;
|
padding: 16px 18px;
|
||||||
|
|||||||
@@ -377,6 +377,13 @@ test("live room table, drawer and dialog retain their final actions", async ({ p
|
|||||||
await expect(actionHeader).toBeVisible();
|
await expect(actionHeader).toBeVisible();
|
||||||
const box = await actionHeader.boundingBox();
|
const box = await actionHeader.boundingBox();
|
||||||
expect(box && box.x + box.width).toBeLessThanOrEqual((page.viewportSize()?.width ?? 0) + 1);
|
expect(box && box.x + box.width).toBeLessThanOrEqual((page.viewportSize()?.width ?? 0) + 1);
|
||||||
|
const fixedActionCell = page.locator("td.el-table-fixed-column--right").first();
|
||||||
|
await expect(fixedActionCell).toBeVisible();
|
||||||
|
const fixedBackground = await fixedActionCell.evaluate((element) =>
|
||||||
|
window.getComputedStyle(element).backgroundColor
|
||||||
|
);
|
||||||
|
expect(fixedBackground).not.toBe("transparent");
|
||||||
|
expect(fixedBackground).not.toMatch(/rgba\([^)]*,\s*0\s*\)$/);
|
||||||
}
|
}
|
||||||
|
|
||||||
await page.getByRole("button", { name: /^查看/ }).first().click();
|
await page.getByRole("button", { name: /^查看/ }).first().click();
|
||||||
@@ -677,7 +684,7 @@ test("recording failure recovery adapts its actions and long paths to each viewp
|
|||||||
await expect(page.getByText(recordingFailure.filePath, { exact: true })).toBeVisible();
|
await expect(page.getByText(recordingFailure.filePath, { exact: true })).toBeVisible();
|
||||||
await expect(page.getByRole("button", { name: "确认有效" })).toBeVisible();
|
await expect(page.getByRole("button", { name: "确认有效" })).toBeVisible();
|
||||||
|
|
||||||
if ((page.viewportSize()?.width ?? 0) <= 767) {
|
if ((page.viewportSize()?.width ?? 0) <= 768) {
|
||||||
await expect(page.locator(".failure-card")).toHaveCount(1);
|
await expect(page.locator(".failure-card")).toHaveCount(1);
|
||||||
await expect(page.getByRole("columnheader", { name: "操作" })).toHaveCount(0);
|
await expect(page.getByRole("columnheader", { name: "操作" })).toHaveCount(0);
|
||||||
} else {
|
} else {
|
||||||
@@ -794,3 +801,49 @@ test("settings layer advanced controls without clipping the quick bar", async ({
|
|||||||
await expectNoDocumentOverflow(page);
|
await expectNoDocumentOverflow(page);
|
||||||
await capture(page, testInfo, "settings");
|
await capture(page, testInfo, "settings");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("OpenList test distinguishes login success from a broken destination mount", async ({ page }) => {
|
||||||
|
await page.route("**/api/settings", async (route) => {
|
||||||
|
await route.fulfill({
|
||||||
|
status: 200,
|
||||||
|
contentType: "application/json",
|
||||||
|
body: JSON.stringify({
|
||||||
|
enableFileUpload: true,
|
||||||
|
enableAutoUpload: true,
|
||||||
|
uploadTarget: 3,
|
||||||
|
openListUpload: {
|
||||||
|
baseUrl: "https://openlist.example.com",
|
||||||
|
username: "tester",
|
||||||
|
password: "secret",
|
||||||
|
basePath: "/archive",
|
||||||
|
sourcePath: "/source",
|
||||||
|
destinationPath: "/archive"
|
||||||
|
}
|
||||||
|
})
|
||||||
|
});
|
||||||
|
});
|
||||||
|
await page.route("**/api/settings/openlist/test", async (route) => {
|
||||||
|
const request = route.request().postDataJSON();
|
||||||
|
expect(request.sourcePath).toBe("/source");
|
||||||
|
expect(request.destinationPath).toBe("/archive");
|
||||||
|
await route.fulfill({
|
||||||
|
status: 200,
|
||||||
|
contentType: "application/json",
|
||||||
|
body: JSON.stringify({
|
||||||
|
success: false,
|
||||||
|
version: "4.1.10",
|
||||||
|
message: "OpenList 登录成功,但一个或多个配置目录不可用。",
|
||||||
|
sourcePath: { path: "/source", success: true, canWrite: false, message: "目录可访问。" },
|
||||||
|
destinationPath: { path: "/archive", success: false, canWrite: false, message: "provider timeout" }
|
||||||
|
})
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
await page.goto("/settings/upload");
|
||||||
|
await expect(page.getByRole("heading", { name: "OpenList 服务端复制" })).toBeVisible();
|
||||||
|
await page.getByRole("button", { name: "测试连接" }).click();
|
||||||
|
await expect(page.getByText("配置异常", { exact: true })).toBeVisible();
|
||||||
|
await expect(page.getByText("目录可访问。", { exact: true })).toBeVisible();
|
||||||
|
await expect(page.getByText("provider timeout", { exact: true })).toBeVisible();
|
||||||
|
await expectNoDocumentOverflow(page);
|
||||||
|
});
|
||||||
|
|||||||
@@ -180,7 +180,9 @@ public sealed record OperationsMetricsSnapshot(
|
|||||||
DateTimeOffset? OldestTranscodeUpdatedAt,
|
DateTimeOffset? OldestTranscodeUpdatedAt,
|
||||||
DateTimeOffset? OldestUploadProgressAt,
|
DateTimeOffset? OldestUploadProgressAt,
|
||||||
int StalledUploadCount,
|
int StalledUploadCount,
|
||||||
int CleanupFailureCount);
|
int CleanupFailureCount,
|
||||||
|
DateTimeOffset? OldestActionableUploadAt,
|
||||||
|
int PendingUploadRecoveryCount);
|
||||||
|
|
||||||
public interface IUserAccountRepository
|
public interface IUserAccountRepository
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -104,6 +104,10 @@ public class OpenListConnectionRequest
|
|||||||
public string Username { get; set; } = string.Empty;
|
public string Username { get; set; } = string.Empty;
|
||||||
|
|
||||||
public string Password { get; set; } = string.Empty;
|
public string Password { get; set; } = string.Empty;
|
||||||
|
|
||||||
|
public string? SourcePath { get; set; }
|
||||||
|
|
||||||
|
public string? DestinationPath { get; set; }
|
||||||
}
|
}
|
||||||
|
|
||||||
public sealed class OpenListDirectoryRequest : OpenListConnectionRequest
|
public sealed class OpenListDirectoryRequest : OpenListConnectionRequest
|
||||||
@@ -134,6 +138,21 @@ public sealed class OpenListConnectionTestDto
|
|||||||
public string? Version { get; init; }
|
public string? Version { get; init; }
|
||||||
|
|
||||||
public required string Message { get; init; }
|
public required string Message { get; init; }
|
||||||
|
|
||||||
|
public OpenListPathCheckDto? SourcePath { get; init; }
|
||||||
|
|
||||||
|
public OpenListPathCheckDto? DestinationPath { get; init; }
|
||||||
|
}
|
||||||
|
|
||||||
|
public sealed class OpenListPathCheckDto
|
||||||
|
{
|
||||||
|
public required string Path { get; init; }
|
||||||
|
|
||||||
|
public bool Success { get; init; }
|
||||||
|
|
||||||
|
public bool CanWrite { get; init; }
|
||||||
|
|
||||||
|
public required string Message { get; init; }
|
||||||
}
|
}
|
||||||
|
|
||||||
public sealed class SystemSettingsDto
|
public sealed class SystemSettingsDto
|
||||||
|
|||||||
@@ -171,7 +171,8 @@ public sealed class OperationsMetricsRepository : IOperationsMetricsRepository
|
|||||||
|
|
||||||
public async Task<OperationsMetricsSnapshot> GetAsync(CancellationToken cancellationToken = default)
|
public async Task<OperationsMetricsSnapshot> GetAsync(CancellationToken cancellationToken = default)
|
||||||
{
|
{
|
||||||
var stalledBefore = DateTimeOffset.UtcNow.Subtract(UploadStallThreshold);
|
var now = DateTimeOffset.UtcNow;
|
||||||
|
var stalledBefore = now.Subtract(UploadStallThreshold);
|
||||||
var oldestTranscodeUpdatedAt = await _dbContext.RecordTasks
|
var oldestTranscodeUpdatedAt = await _dbContext.RecordTasks
|
||||||
.AsNoTracking()
|
.AsNoTracking()
|
||||||
.Where(static task => task.Status == RecordTaskStatus.Processing)
|
.Where(static task => task.Status == RecordTaskStatus.Processing)
|
||||||
@@ -183,9 +184,27 @@ public sealed class OperationsMetricsRepository : IOperationsMetricsRepository
|
|||||||
.Select(job => (DateTimeOffset?)(job.LastProgressAt ?? job.ExternalTaskStartedAt ?? job.UpdatedAt))
|
.Select(job => (DateTimeOffset?)(job.LastProgressAt ?? job.ExternalTaskStartedAt ?? job.UpdatedAt))
|
||||||
.MinAsync(cancellationToken);
|
.MinAsync(cancellationToken);
|
||||||
var stalledUploadCount = await _dbContext.RecordUploadJobs.CountAsync(
|
var stalledUploadCount = await _dbContext.RecordUploadJobs.CountAsync(
|
||||||
job => job.Status == RecordArtifactUploadStatus.Uploading &&
|
job => (job.Status == RecordArtifactUploadStatus.Queued ||
|
||||||
|
job.Status == RecordArtifactUploadStatus.Uploading ||
|
||||||
|
(job.Status == RecordArtifactUploadStatus.WaitingRetry &&
|
||||||
|
(!job.NextAttemptAt.HasValue || job.NextAttemptAt <= now))) &&
|
||||||
(job.LastProgressAt ?? job.ExternalTaskStartedAt ?? job.UpdatedAt) < stalledBefore,
|
(job.LastProgressAt ?? job.ExternalTaskStartedAt ?? job.UpdatedAt) < stalledBefore,
|
||||||
cancellationToken);
|
cancellationToken);
|
||||||
|
var oldestActionableUploadAt = await _dbContext.RecordUploadJobs
|
||||||
|
.AsNoTracking()
|
||||||
|
.Where(job => job.Status == RecordArtifactUploadStatus.Queued ||
|
||||||
|
job.Status == RecordArtifactUploadStatus.Uploading ||
|
||||||
|
(job.Status == RecordArtifactUploadStatus.WaitingRetry &&
|
||||||
|
(!job.NextAttemptAt.HasValue || job.NextAttemptAt <= now)))
|
||||||
|
.Select(job => (DateTimeOffset?)(job.LastProgressAt ?? job.ExternalTaskStartedAt ?? job.UpdatedAt))
|
||||||
|
.MinAsync(cancellationToken);
|
||||||
|
var pendingUploadRecoveryCount = await _dbContext.RecordTasks.CountAsync(
|
||||||
|
task => (task.Status == RecordTaskStatus.Completed || task.Status == RecordTaskStatus.Stopped) &&
|
||||||
|
!task.IsHiddenArtifactSource &&
|
||||||
|
task.Result != null &&
|
||||||
|
task.Result.UploadStatus == RecordArtifactUploadStatus.NotUploaded &&
|
||||||
|
task.Result.FilePath != string.Empty,
|
||||||
|
cancellationToken);
|
||||||
var cleanupFailureCount = await _dbContext.RecordUploadJobs.CountAsync(
|
var cleanupFailureCount = await _dbContext.RecordUploadJobs.CountAsync(
|
||||||
static job => job.Status == RecordArtifactUploadStatus.WaitingRetry &&
|
static job => job.Status == RecordArtifactUploadStatus.WaitingRetry &&
|
||||||
job.CurrentArtifact == RecordUploadArtifactStage.Completed,
|
job.CurrentArtifact == RecordUploadArtifactStage.Completed,
|
||||||
@@ -194,7 +213,9 @@ public sealed class OperationsMetricsRepository : IOperationsMetricsRepository
|
|||||||
oldestTranscodeUpdatedAt,
|
oldestTranscodeUpdatedAt,
|
||||||
oldestUploadProgressAt,
|
oldestUploadProgressAt,
|
||||||
stalledUploadCount,
|
stalledUploadCount,
|
||||||
cleanupFailureCount);
|
cleanupFailureCount,
|
||||||
|
oldestActionableUploadAt,
|
||||||
|
pendingUploadRecoveryCount);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -115,34 +115,68 @@ public sealed class CompletionDispatchService
|
|||||||
var recordSession = task.RecordSession ?? throw new InvalidOperationException();
|
var recordSession = task.RecordSession ?? throw new InvalidOperationException();
|
||||||
var recordResult = task.Result ?? throw new InvalidOperationException();
|
var recordResult = task.Result ?? throw new InvalidOperationException();
|
||||||
|
|
||||||
|
var dispatchErrors = new List<string>();
|
||||||
if (!dispatch.ScriptDispatched)
|
if (!dispatch.ScriptDispatched)
|
||||||
{
|
{
|
||||||
var scriptResult = await _eventScriptService.RunSegmentCompletedAsync(
|
try
|
||||||
task.LiveRoom,
|
|
||||||
recordSession,
|
|
||||||
task,
|
|
||||||
recordResult,
|
|
||||||
recordResult.FilePath,
|
|
||||||
task.EndedAt ?? DateTimeOffset.UtcNow,
|
|
||||||
eventId: dispatch.Id,
|
|
||||||
cancellationToken: cancellationToken);
|
|
||||||
if (scriptResult is null || scriptResult.Success)
|
|
||||||
{
|
{
|
||||||
dispatch.MarkScriptDispatched(DateTimeOffset.UtcNow);
|
var scriptResult = await _eventScriptService.RunSegmentCompletedAsync(
|
||||||
await _dbContext.SaveChangesAsync(cancellationToken);
|
task.LiveRoom,
|
||||||
|
recordSession,
|
||||||
|
task,
|
||||||
|
recordResult,
|
||||||
|
recordResult.FilePath,
|
||||||
|
task.EndedAt ?? DateTimeOffset.UtcNow,
|
||||||
|
eventId: dispatch.Id,
|
||||||
|
cancellationToken: cancellationToken);
|
||||||
|
if (scriptResult is null || scriptResult.Success)
|
||||||
|
{
|
||||||
|
dispatch.MarkScriptDispatched(DateTimeOffset.UtcNow);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
dispatchErrors.Add($"事件脚本:{scriptResult.Message}");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
else
|
catch (Exception ex) when (ex is not OperationCanceledException || !cancellationToken.IsCancellationRequested)
|
||||||
{
|
{
|
||||||
throw new InvalidOperationException(scriptResult.Message);
|
dispatchErrors.Add($"事件脚本:{ex.Message}");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!dispatch.UploadDispatched)
|
if (!dispatch.UploadDispatched)
|
||||||
{
|
{
|
||||||
_ = await _recordUploadService.TryAutoUploadTaskAsync(task.Id, cancellationToken);
|
try
|
||||||
dispatch.MarkUploadDispatched(DateTimeOffset.UtcNow);
|
{
|
||||||
await _dbContext.SaveChangesAsync(cancellationToken);
|
var uploadResult = await _recordUploadService.TryAutoUploadTaskAsync(task.Id, cancellationToken);
|
||||||
|
if (uploadResult is null || uploadResult.Success)
|
||||||
|
{
|
||||||
|
dispatch.MarkUploadDispatched(DateTimeOffset.UtcNow);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
dispatchErrors.Add($"自动上传:{uploadResult.Message}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (Exception ex) when (ex is not OperationCanceledException || !cancellationToken.IsCancellationRequested)
|
||||||
|
{
|
||||||
|
dispatchErrors.Add($"自动上传:{ex.Message}");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (dispatchErrors.Count > 0)
|
||||||
|
{
|
||||||
|
dispatch.ScheduleRetry(
|
||||||
|
string.Join(";", dispatchErrors),
|
||||||
|
DateTimeOffset.UtcNow.Add(RetryDelay),
|
||||||
|
DateTimeOffset.UtcNow);
|
||||||
|
}
|
||||||
|
|
||||||
|
await _dbContext.SaveChangesAsync(cancellationToken);
|
||||||
|
}
|
||||||
|
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||||
|
{
|
||||||
|
throw;
|
||||||
}
|
}
|
||||||
catch (Exception ex)
|
catch (Exception ex)
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -170,16 +170,70 @@ public sealed class OpenListClient : IOpenListClient
|
|||||||
version = versionElement.GetString();
|
version = versionElement.GetString();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var sourcePath = await TestDirectoryAsync(connection, connection.SourcePath, requireWrite: false, cancellationToken);
|
||||||
|
var destinationPath = await TestDirectoryAsync(connection, connection.DestinationPath, requireWrite: true, cancellationToken);
|
||||||
|
var success = (sourcePath?.Success ?? true) && (destinationPath?.Success ?? true);
|
||||||
|
|
||||||
return new OpenListConnectionTestDto
|
return new OpenListConnectionTestDto
|
||||||
{
|
{
|
||||||
Success = true,
|
Success = success,
|
||||||
Version = version,
|
Version = version,
|
||||||
Message = string.IsNullOrWhiteSpace(version)
|
Message = success
|
||||||
? "OpenList 连接和登录成功。"
|
? (string.IsNullOrWhiteSpace(version)
|
||||||
: $"OpenList 连接和登录成功:{version}"
|
? "OpenList 登录和目录检查成功。"
|
||||||
|
: $"OpenList 登录和目录检查成功:{version}")
|
||||||
|
: "OpenList 登录成功,但一个或多个配置目录不可用。",
|
||||||
|
SourcePath = sourcePath,
|
||||||
|
DestinationPath = destinationPath
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private async Task<OpenListPathCheckDto?> TestDirectoryAsync(
|
||||||
|
OpenListConnectionRequest connection,
|
||||||
|
string? path,
|
||||||
|
bool requireWrite,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
if (string.IsNullOrWhiteSpace(path))
|
||||||
|
{
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
var normalizedPath = NormalizePath(path);
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var directory = await ListDirectoriesAsync(
|
||||||
|
new OpenListDirectoryRequest
|
||||||
|
{
|
||||||
|
BaseUrl = connection.BaseUrl,
|
||||||
|
Username = connection.Username,
|
||||||
|
Password = connection.Password,
|
||||||
|
Path = normalizedPath
|
||||||
|
},
|
||||||
|
cancellationToken);
|
||||||
|
var success = !requireWrite || directory.CanWrite;
|
||||||
|
return new OpenListPathCheckDto
|
||||||
|
{
|
||||||
|
Path = normalizedPath,
|
||||||
|
Success = success,
|
||||||
|
CanWrite = directory.CanWrite,
|
||||||
|
Message = success
|
||||||
|
? (requireWrite ? "目录可访问且允许写入。" : "目录可访问。")
|
||||||
|
: "目录可访问,但 OpenList 未授予写入权限。"
|
||||||
|
};
|
||||||
|
}
|
||||||
|
catch (Exception ex) when (ex is not OperationCanceledException || !cancellationToken.IsCancellationRequested)
|
||||||
|
{
|
||||||
|
return new OpenListPathCheckDto
|
||||||
|
{
|
||||||
|
Path = normalizedPath,
|
||||||
|
Success = false,
|
||||||
|
CanWrite = false,
|
||||||
|
Message = ex.Message
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
public async Task<OpenListDirectoryListDto> ListDirectoriesAsync(
|
public async Task<OpenListDirectoryListDto> ListDirectoriesAsync(
|
||||||
OpenListDirectoryRequest request,
|
OpenListDirectoryRequest request,
|
||||||
CancellationToken cancellationToken = default)
|
CancellationToken cancellationToken = default)
|
||||||
|
|||||||
@@ -15,6 +15,16 @@ public sealed record OpenListQueueHealthSnapshot(
|
|||||||
DateTimeOffset? LastErrorAt)
|
DateTimeOffset? LastErrorAt)
|
||||||
{
|
{
|
||||||
public bool IsPaused => Status != OpenListQueueHealthStatus.Healthy;
|
public bool IsPaused => Status != OpenListQueueHealthStatus.Healthy;
|
||||||
|
|
||||||
|
public DateTimeOffset? LastWorkerIterationAt { get; init; }
|
||||||
|
|
||||||
|
public DateTimeOffset? LastRecoveryAt { get; init; }
|
||||||
|
|
||||||
|
public int LastRecoveredCount { get; init; }
|
||||||
|
|
||||||
|
public string? LastWorkerError { get; init; }
|
||||||
|
|
||||||
|
public string? LastRecoveryError { get; init; }
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
@@ -32,12 +42,61 @@ public sealed class OpenListUploadHealthState
|
|||||||
private DateTimeOffset? _retryAt;
|
private DateTimeOffset? _retryAt;
|
||||||
private DateTimeOffset? _lastErrorAt;
|
private DateTimeOffset? _lastErrorAt;
|
||||||
private int _consecutiveRateLimits;
|
private int _consecutiveRateLimits;
|
||||||
|
private DateTimeOffset? _lastWorkerIterationAt;
|
||||||
|
private DateTimeOffset? _lastRecoveryAt;
|
||||||
|
private int _lastRecoveredCount;
|
||||||
|
private string? _lastWorkerError;
|
||||||
|
private string? _lastRecoveryError;
|
||||||
|
|
||||||
public OpenListQueueHealthSnapshot GetSnapshot()
|
public OpenListQueueHealthSnapshot GetSnapshot()
|
||||||
{
|
{
|
||||||
lock (_gate)
|
lock (_gate)
|
||||||
{
|
{
|
||||||
return new OpenListQueueHealthSnapshot(_status, _reason, _retryAt, _lastErrorAt);
|
return new OpenListQueueHealthSnapshot(_status, _reason, _retryAt, _lastErrorAt)
|
||||||
|
{
|
||||||
|
LastWorkerIterationAt = _lastWorkerIterationAt,
|
||||||
|
LastRecoveryAt = _lastRecoveryAt,
|
||||||
|
LastRecoveredCount = _lastRecoveredCount,
|
||||||
|
LastWorkerError = _lastWorkerError,
|
||||||
|
LastRecoveryError = _lastRecoveryError
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void MarkWorkerIteration(DateTimeOffset occurredAt)
|
||||||
|
{
|
||||||
|
lock (_gate)
|
||||||
|
{
|
||||||
|
_lastWorkerIterationAt = occurredAt;
|
||||||
|
_lastWorkerError = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void MarkRecovery(DateTimeOffset occurredAt, int recoveredCount)
|
||||||
|
{
|
||||||
|
lock (_gate)
|
||||||
|
{
|
||||||
|
_lastRecoveryAt = occurredAt;
|
||||||
|
_lastRecoveredCount = Math.Max(0, recoveredCount);
|
||||||
|
_lastRecoveryError = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void MarkWorkerError(DateTimeOffset occurredAt, string error)
|
||||||
|
{
|
||||||
|
lock (_gate)
|
||||||
|
{
|
||||||
|
_lastWorkerIterationAt = occurredAt;
|
||||||
|
_lastWorkerError = string.IsNullOrWhiteSpace(error) ? null : error.Trim();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void MarkRecoveryError(DateTimeOffset occurredAt, string error)
|
||||||
|
{
|
||||||
|
lock (_gate)
|
||||||
|
{
|
||||||
|
_lastRecoveryAt = occurredAt;
|
||||||
|
_lastRecoveryError = string.IsNullOrWhiteSpace(error) ? null : error.Trim();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -153,6 +153,16 @@ public sealed class OpenListUploadQueueService
|
|||||||
{
|
{
|
||||||
recovered++;
|
recovered++;
|
||||||
}
|
}
|
||||||
|
else if (!result.Success)
|
||||||
|
{
|
||||||
|
await _systemLogService.WriteAsync(
|
||||||
|
SystemLogLevel.Warning,
|
||||||
|
"Upload",
|
||||||
|
"跳过一个无法恢复的 OpenList 自动上传任务。",
|
||||||
|
$"recordTaskId={taskId}; error={result.Message}",
|
||||||
|
recordTaskId: taskId,
|
||||||
|
cancellationToken: cancellationToken);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
catch (Exception ex)
|
catch (Exception ex)
|
||||||
{
|
{
|
||||||
@@ -1346,21 +1356,22 @@ public sealed class OpenListUploadQueueService
|
|||||||
public sealed class OpenListUploadBackgroundService : BackgroundService
|
public sealed class OpenListUploadBackgroundService : BackgroundService
|
||||||
{
|
{
|
||||||
private static readonly TimeSpan IdleDelay = TimeSpan.FromSeconds(2);
|
private static readonly TimeSpan IdleDelay = TimeSpan.FromSeconds(2);
|
||||||
private static readonly TimeSpan RecoveryInterval = TimeSpan.FromMinutes(1);
|
|
||||||
private readonly IServiceScopeFactory _scopeFactory;
|
private readonly IServiceScopeFactory _scopeFactory;
|
||||||
|
private readonly OpenListUploadHealthState _healthState;
|
||||||
private readonly ILogger<OpenListUploadBackgroundService> _logger;
|
private readonly ILogger<OpenListUploadBackgroundService> _logger;
|
||||||
|
|
||||||
public OpenListUploadBackgroundService(
|
public OpenListUploadBackgroundService(
|
||||||
IServiceScopeFactory scopeFactory,
|
IServiceScopeFactory scopeFactory,
|
||||||
|
OpenListUploadHealthState healthState,
|
||||||
ILogger<OpenListUploadBackgroundService> logger)
|
ILogger<OpenListUploadBackgroundService> logger)
|
||||||
{
|
{
|
||||||
_scopeFactory = scopeFactory;
|
_scopeFactory = scopeFactory;
|
||||||
|
_healthState = healthState;
|
||||||
_logger = logger;
|
_logger = logger;
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||||
{
|
{
|
||||||
var nextRecoveryAt = DateTimeOffset.MinValue;
|
|
||||||
while (!stoppingToken.IsCancellationRequested)
|
while (!stoppingToken.IsCancellationRequested)
|
||||||
{
|
{
|
||||||
try
|
try
|
||||||
@@ -1368,11 +1379,7 @@ public sealed class OpenListUploadBackgroundService : BackgroundService
|
|||||||
using var scope = _scopeFactory.CreateScope();
|
using var scope = _scopeFactory.CreateScope();
|
||||||
var queue = scope.ServiceProvider.GetRequiredService<OpenListUploadQueueService>();
|
var queue = scope.ServiceProvider.GetRequiredService<OpenListUploadQueueService>();
|
||||||
_ = await queue.ProcessNextAsync(stoppingToken);
|
_ = await queue.ProcessNextAsync(stoppingToken);
|
||||||
if (DateTimeOffset.UtcNow >= nextRecoveryAt)
|
_healthState.MarkWorkerIteration(DateTimeOffset.UtcNow);
|
||||||
{
|
|
||||||
await queue.RecoverPendingAutomaticUploadsAsync(cancellationToken: stoppingToken);
|
|
||||||
nextRecoveryAt = DateTimeOffset.UtcNow.Add(RecoveryInterval);
|
|
||||||
}
|
|
||||||
|
|
||||||
await Task.Delay(IdleDelay, stoppingToken);
|
await Task.Delay(IdleDelay, stoppingToken);
|
||||||
}
|
}
|
||||||
@@ -1382,6 +1389,7 @@ public sealed class OpenListUploadBackgroundService : BackgroundService
|
|||||||
}
|
}
|
||||||
catch (Exception ex)
|
catch (Exception ex)
|
||||||
{
|
{
|
||||||
|
_healthState.MarkWorkerError(DateTimeOffset.UtcNow, ex.Message);
|
||||||
_logger.LogError(ex, "OpenList upload background worker failed");
|
_logger.LogError(ex, "OpenList upload background worker failed");
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
@@ -1395,3 +1403,53 @@ public sealed class OpenListUploadBackgroundService : BackgroundService
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public sealed class OpenListAutomaticRecoveryBackgroundService : BackgroundService
|
||||||
|
{
|
||||||
|
private static readonly TimeSpan RecoveryInterval = TimeSpan.FromMinutes(1);
|
||||||
|
private readonly IServiceScopeFactory _scopeFactory;
|
||||||
|
private readonly OpenListUploadHealthState _healthState;
|
||||||
|
private readonly ILogger<OpenListAutomaticRecoveryBackgroundService> _logger;
|
||||||
|
|
||||||
|
public OpenListAutomaticRecoveryBackgroundService(
|
||||||
|
IServiceScopeFactory scopeFactory,
|
||||||
|
OpenListUploadHealthState healthState,
|
||||||
|
ILogger<OpenListAutomaticRecoveryBackgroundService> logger)
|
||||||
|
{
|
||||||
|
_scopeFactory = scopeFactory;
|
||||||
|
_healthState = healthState;
|
||||||
|
_logger = logger;
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||||
|
{
|
||||||
|
while (!stoppingToken.IsCancellationRequested)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
using var scope = _scopeFactory.CreateScope();
|
||||||
|
var queue = scope.ServiceProvider.GetRequiredService<OpenListUploadQueueService>();
|
||||||
|
var recovered = await queue.RecoverPendingAutomaticUploadsAsync(cancellationToken: stoppingToken);
|
||||||
|
_healthState.MarkRecovery(DateTimeOffset.UtcNow, recovered);
|
||||||
|
await Task.Delay(RecoveryInterval, stoppingToken);
|
||||||
|
}
|
||||||
|
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||||
|
{
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
_healthState.MarkRecoveryError(DateTimeOffset.UtcNow, ex.Message);
|
||||||
|
_logger.LogError(ex, "OpenList automatic upload recovery failed");
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await Task.Delay(TimeSpan.FromSeconds(10), stoppingToken);
|
||||||
|
}
|
||||||
|
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||||
|
{
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -253,6 +253,7 @@ builder.Services.AddHostedService(provider => provider.GetRequiredService<LiveRo
|
|||||||
builder.Services.AddHostedService<CleanupOperationBackgroundService>();
|
builder.Services.AddHostedService<CleanupOperationBackgroundService>();
|
||||||
builder.Services.AddHostedService<RetentionCleanupBackgroundService>();
|
builder.Services.AddHostedService<RetentionCleanupBackgroundService>();
|
||||||
builder.Services.AddHostedService<OpenListUploadBackgroundService>();
|
builder.Services.AddHostedService<OpenListUploadBackgroundService>();
|
||||||
|
builder.Services.AddHostedService<OpenListAutomaticRecoveryBackgroundService>();
|
||||||
builder.Services.AddHostedService<CompletionDispatchBackgroundService>();
|
builder.Services.AddHostedService<CompletionDispatchBackgroundService>();
|
||||||
builder.Services.AddHostedService<SystemLogRetentionBackgroundService>();
|
builder.Services.AddHostedService<SystemLogRetentionBackgroundService>();
|
||||||
|
|
||||||
@@ -338,18 +339,34 @@ app.MapGet("/api/operations/health", async (
|
|||||||
IOperationsMetricsRepository metricsRepository,
|
IOperationsMetricsRepository metricsRepository,
|
||||||
ISystemSettingsService settingsService,
|
ISystemSettingsService settingsService,
|
||||||
IStorageGuardService storageGuardService,
|
IStorageGuardService storageGuardService,
|
||||||
|
OpenListUploadHealthState uploadHealthState,
|
||||||
CancellationToken cancellationToken) =>
|
CancellationToken cancellationToken) =>
|
||||||
{
|
{
|
||||||
var metrics = await metricsRepository.GetAsync(cancellationToken);
|
var metrics = await metricsRepository.GetAsync(cancellationToken);
|
||||||
var settings = await settingsService.GetAsync(cancellationToken);
|
var settings = await settingsService.GetAsync(cancellationToken);
|
||||||
var storage = storageGuardService.CheckCanStartOrResume(settings);
|
var storage = storageGuardService.CheckCanStartOrResume(settings);
|
||||||
|
var uploadQueue = uploadHealthState.GetSnapshot();
|
||||||
|
var now = DateTimeOffset.UtcNow;
|
||||||
|
var monitorOpenList = settings.EnableFileUpload &&
|
||||||
|
settings.EnableAutoUpload &&
|
||||||
|
settings.UploadTarget == UploadTargetType.OpenList;
|
||||||
|
var workerHeartbeatStale = monitorOpenList &&
|
||||||
|
uploadQueue.LastWorkerIterationAt.HasValue &&
|
||||||
|
uploadQueue.LastWorkerIterationAt < now.AddMinutes(-5);
|
||||||
|
var recoveryHeartbeatStale = monitorOpenList &&
|
||||||
|
uploadQueue.LastRecoveryAt.HasValue &&
|
||||||
|
uploadQueue.LastRecoveryAt < now.AddMinutes(-5);
|
||||||
var degraded = storage.Tier == StorageTier.Red ||
|
var degraded = storage.Tier == StorageTier.Red ||
|
||||||
metrics.StalledUploadCount > 0 ||
|
metrics.StalledUploadCount > 0 ||
|
||||||
metrics.CleanupFailureCount > 0;
|
metrics.CleanupFailureCount > 0 ||
|
||||||
|
!string.IsNullOrWhiteSpace(uploadQueue.LastWorkerError) ||
|
||||||
|
!string.IsNullOrWhiteSpace(uploadQueue.LastRecoveryError) ||
|
||||||
|
workerHeartbeatStale ||
|
||||||
|
recoveryHeartbeatStale;
|
||||||
return Results.Ok(new
|
return Results.Ok(new
|
||||||
{
|
{
|
||||||
status = degraded ? "degraded" : "healthy",
|
status = degraded ? "degraded" : "healthy",
|
||||||
timestamp = DateTimeOffset.UtcNow,
|
timestamp = now,
|
||||||
storage = new
|
storage = new
|
||||||
{
|
{
|
||||||
tier = storage.Tier.ToString(),
|
tier = storage.Tier.ToString(),
|
||||||
@@ -357,7 +374,8 @@ app.MapGet("/api/operations/health", async (
|
|||||||
storage.AvailableBytes,
|
storage.AvailableBytes,
|
||||||
storage.Message
|
storage.Message
|
||||||
},
|
},
|
||||||
operations = metrics
|
operations = metrics,
|
||||||
|
uploadQueue
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ using System.Text;
|
|||||||
using System.Text.Json;
|
using System.Text.Json;
|
||||||
using LiveRecorder.Application.Abstractions.Logging;
|
using LiveRecorder.Application.Abstractions.Logging;
|
||||||
using LiveRecorder.Application.Abstractions.Recording;
|
using LiveRecorder.Application.Abstractions.Recording;
|
||||||
|
using LiveRecorder.Application.Abstractions.Scripting;
|
||||||
using LiveRecorder.Application.Abstractions.Settings;
|
using LiveRecorder.Application.Abstractions.Settings;
|
||||||
using LiveRecorder.Application.Models.Logs;
|
using LiveRecorder.Application.Models.Logs;
|
||||||
using LiveRecorder.Application.Models.Settings;
|
using LiveRecorder.Application.Models.Settings;
|
||||||
@@ -14,11 +15,83 @@ using LiveRecorder.Infrastructure.Persistence.Repositories;
|
|||||||
using LiveRecorder.Infrastructure.Services;
|
using LiveRecorder.Infrastructure.Services;
|
||||||
using Microsoft.EntityFrameworkCore;
|
using Microsoft.EntityFrameworkCore;
|
||||||
using Microsoft.EntityFrameworkCore.Storage;
|
using Microsoft.EntityFrameworkCore.Storage;
|
||||||
|
using Microsoft.Extensions.Logging.Abstractions;
|
||||||
|
|
||||||
namespace LiveRecorder.Tests;
|
namespace LiveRecorder.Tests;
|
||||||
|
|
||||||
public sealed class OpenListUploadTests
|
public sealed class OpenListUploadTests
|
||||||
{
|
{
|
||||||
|
[Fact]
|
||||||
|
public async Task TestConnection_ReportsDestinationMountFailureAfterSuccessfulLogin()
|
||||||
|
{
|
||||||
|
var handler = new StubHttpMessageHandler(async request =>
|
||||||
|
{
|
||||||
|
var path = request.RequestUri!.AbsolutePath;
|
||||||
|
if (path.EndsWith("/api/auth/login", StringComparison.Ordinal))
|
||||||
|
{
|
||||||
|
return JsonResponse("""{"code":200,"message":"success","data":{"token":"test-token"}}""");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (path.EndsWith("/api/public/settings", StringComparison.Ordinal))
|
||||||
|
{
|
||||||
|
return JsonResponse("""{"code":200,"message":"success","data":{"version":"4.1.10"}}""");
|
||||||
|
}
|
||||||
|
|
||||||
|
var body = JsonDocument.Parse(await request.Content!.ReadAsStringAsync()).RootElement;
|
||||||
|
return body.GetProperty("path").GetString() == "/source"
|
||||||
|
? JsonResponse("""{"code":200,"message":"success","data":{"content":[],"write":false}}""")
|
||||||
|
: JsonResponse("""{"code":500,"message":"provider timeout","data":null}""");
|
||||||
|
});
|
||||||
|
var client = CreateClient(handler);
|
||||||
|
var connection = Connection();
|
||||||
|
connection.SourcePath = "/source";
|
||||||
|
connection.DestinationPath = "/destination";
|
||||||
|
|
||||||
|
var result = await client.TestConnectionAsync(connection);
|
||||||
|
|
||||||
|
Assert.False(result.Success);
|
||||||
|
Assert.Equal("4.1.10", result.Version);
|
||||||
|
Assert.True(result.SourcePath?.Success);
|
||||||
|
Assert.False(result.DestinationPath?.Success);
|
||||||
|
Assert.Contains("provider timeout", result.DestinationPath?.Message);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task CompletionDispatch_ScriptFailureDoesNotBlockAutomaticUpload()
|
||||||
|
{
|
||||||
|
await using var fixture = await QueueFixture.CreateAsync();
|
||||||
|
await fixture.PrepareCompletionDispatchAsync();
|
||||||
|
var service = fixture.CreateCompletionDispatchService(new FixedEventScriptService(
|
||||||
|
new EventScriptExecutionResultDto { Success = false, Message = "script failed" }));
|
||||||
|
|
||||||
|
await service.TryDispatchTaskAsync(fixture.RecordTaskId);
|
||||||
|
|
||||||
|
var dispatch = await fixture.Context.RecordCompletionDispatches.SingleAsync();
|
||||||
|
Assert.False(dispatch.ScriptDispatched);
|
||||||
|
Assert.True(dispatch.UploadDispatched);
|
||||||
|
Assert.Null(dispatch.CompletedAt);
|
||||||
|
Assert.Contains("script failed", dispatch.LastError);
|
||||||
|
Assert.NotNull(await fixture.Context.RecordUploadJobs.SingleOrDefaultAsync());
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task CompletionDispatch_UploadRejectionRemainsRetryable()
|
||||||
|
{
|
||||||
|
await using var fixture = await QueueFixture.CreateAsync(
|
||||||
|
new VideoMetadata(0.18, 1920, 1080, "h264", "aac", 30, 4_000_000));
|
||||||
|
await fixture.PrepareCompletionDispatchAsync();
|
||||||
|
var service = fixture.CreateCompletionDispatchService(new FixedEventScriptService(null));
|
||||||
|
|
||||||
|
await service.TryDispatchTaskAsync(fixture.RecordTaskId);
|
||||||
|
|
||||||
|
var dispatch = await fixture.Context.RecordCompletionDispatches.SingleAsync();
|
||||||
|
Assert.True(dispatch.ScriptDispatched);
|
||||||
|
Assert.False(dispatch.UploadDispatched);
|
||||||
|
Assert.Null(dispatch.CompletedAt);
|
||||||
|
Assert.Contains("自动上传", dispatch.LastError);
|
||||||
|
Assert.True(dispatch.NextAttemptAt > DateTimeOffset.UtcNow);
|
||||||
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
public async Task Enqueue_RejectsShortMediaAndKeepsItNotUploaded()
|
public async Task Enqueue_RejectsShortMediaAndKeepsItNotUploaded()
|
||||||
{
|
{
|
||||||
@@ -763,6 +836,30 @@ public sealed class OpenListUploadTests
|
|||||||
|
|
||||||
public Guid RecordTaskId { get; }
|
public Guid RecordTaskId { get; }
|
||||||
|
|
||||||
|
public async Task PrepareCompletionDispatchAsync()
|
||||||
|
{
|
||||||
|
var session = await Context.RecordSessions.SingleAsync();
|
||||||
|
session.MarkCompleted(DateTimeOffset.UtcNow);
|
||||||
|
if (!await Context.RecordCompletionDispatches.AnyAsync())
|
||||||
|
{
|
||||||
|
Context.RecordCompletionDispatches.Add(
|
||||||
|
new RecordCompletionDispatch(RecordTaskId, DateTimeOffset.UtcNow));
|
||||||
|
}
|
||||||
|
|
||||||
|
await Context.SaveChangesAsync();
|
||||||
|
}
|
||||||
|
|
||||||
|
public CompletionDispatchService CreateCompletionDispatchService(IEventScriptService eventScriptService)
|
||||||
|
{
|
||||||
|
var uploadService = new RecordUploadService(Context, _settingsService, new NullSystemLogService(), Queue);
|
||||||
|
var consolidation = new ShortFragmentConsolidationService(
|
||||||
|
Context,
|
||||||
|
_settingsService,
|
||||||
|
_videoMetadataService,
|
||||||
|
NullLogger<ShortFragmentConsolidationService>.Instance);
|
||||||
|
return new CompletionDispatchService(Context, eventScriptService, uploadService, consolidation);
|
||||||
|
}
|
||||||
|
|
||||||
public static async Task<QueueFixture> CreateAsync(VideoMetadata? metadata = null)
|
public static async Task<QueueFixture> CreateAsync(VideoMetadata? metadata = null)
|
||||||
{
|
{
|
||||||
var temporaryRoot = Path.Combine(Path.GetTempPath(), $"live-recorder-openlist-{Guid.NewGuid():N}");
|
var temporaryRoot = Path.Combine(Path.GetTempPath(), $"live-recorder-openlist-{Guid.NewGuid():N}");
|
||||||
@@ -886,6 +983,42 @@ public sealed class OpenListUploadTests
|
|||||||
CancellationToken cancellationToken = default) => Task.FromResult<string?>(null);
|
CancellationToken cancellationToken = default) => Task.FromResult<string?>(null);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private sealed class FixedEventScriptService : IEventScriptService
|
||||||
|
{
|
||||||
|
private readonly EventScriptExecutionResultDto? _segmentResult;
|
||||||
|
|
||||||
|
public FixedEventScriptService(EventScriptExecutionResultDto? segmentResult)
|
||||||
|
{
|
||||||
|
_segmentResult = segmentResult;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Task<EventScriptExecutionResultDto?> RunLiveStartedAsync(
|
||||||
|
LiveRoom liveRoom,
|
||||||
|
DateTimeOffset occurredAt,
|
||||||
|
CancellationToken cancellationToken = default) => Task.FromResult<EventScriptExecutionResultDto?>(null);
|
||||||
|
|
||||||
|
public Task<EventScriptExecutionResultDto?> RunLiveEndedAsync(
|
||||||
|
LiveRoom liveRoom,
|
||||||
|
DateTimeOffset occurredAt,
|
||||||
|
CancellationToken cancellationToken = default) => Task.FromResult<EventScriptExecutionResultDto?>(null);
|
||||||
|
|
||||||
|
public Task<EventScriptExecutionResultDto?> RunSegmentCompletedAsync(
|
||||||
|
LiveRoom? liveRoom,
|
||||||
|
RecordSession recordSession,
|
||||||
|
RecordTask recordTask,
|
||||||
|
RecordResult? recordResult,
|
||||||
|
string segmentFilePath,
|
||||||
|
DateTimeOffset occurredAt,
|
||||||
|
bool forceRun = false,
|
||||||
|
Guid? eventId = null,
|
||||||
|
CancellationToken cancellationToken = default) => Task.FromResult(_segmentResult);
|
||||||
|
|
||||||
|
public Task<EventScriptTestResultDto> TestAsync(
|
||||||
|
TestEventScriptRequest request,
|
||||||
|
CancellationToken cancellationToken = default) =>
|
||||||
|
Task.FromResult(new EventScriptTestResultDto { Success = true, Message = "ok" });
|
||||||
|
}
|
||||||
|
|
||||||
private sealed class FixedSettingsService : ISystemSettingsService
|
private sealed class FixedSettingsService : ISystemSettingsService
|
||||||
{
|
{
|
||||||
private readonly SystemSettingsDto _settings;
|
private readonly SystemSettingsDto _settings;
|
||||||
|
|||||||
Reference in New Issue
Block a user