fix: improve recovery and auto-start flows

This commit is contained in:
2026-04-25 00:06:25 +08:00
parent 39db8c012f
commit 9701c88c8b
5 changed files with 207 additions and 14 deletions
+99 -1
View File
@@ -115,6 +115,26 @@ const emailTemplateTokens = [
"{{occurredAtUtc}}" "{{occurredAtUtc}}"
]; ];
const eventScriptEnvironmentExamples = [
{ name: "LIVE_RECORDER_EVENT", example: "segment_completed", scope: "全部事件" },
{ name: "LIVE_RECORDER_PLATFORM", example: "Douyin", scope: "全部事件" },
{ name: "LIVE_RECORDER_LIVE_ROOM_ID", example: "6f73a2f2-1d4c-4e7a-a9b1-3d29d54ed901", scope: "全部事件" },
{ name: "LIVE_RECORDER_ROOM_ID", example: "676493068539", scope: "全部事件" },
{ name: "LIVE_RECORDER_TITLE", example: "今晚划水聊天", scope: "全部事件" },
{ name: "LIVE_RECORDER_ANCHOR", example: "布莱克", scope: "全部事件" },
{ name: "LIVE_RECORDER_SOURCE_URL", example: "https://live.douyin.com/676493068539", scope: "全部事件" },
{ name: "LIVE_RECORDER_OCCURRED_AT_UTC", example: "2026-04-25T12:34:56.7890000+00:00", scope: "全部事件" },
{ name: "LIVE_RECORDER_RECORD_SESSION_ID", example: "8e2e9c64-b8f6-4d15-b6cb-1d4ce0adab77", scope: "仅分片完成" },
{ name: "LIVE_RECORDER_RECORD_TASK_ID", example: "2a4810a2-7ef4-4a22-90d4-0211b90cc54c", scope: "仅分片完成" },
{ name: "LIVE_RECORDER_SEGMENT_INDEX", example: "1", scope: "仅分片完成" },
{ name: "LIVE_RECORDER_SEGMENT_FILE_PATH", example: "/app/records/Douyin/布莱克/2026-04-25/203000_今晚划水聊天__00001.mp4", scope: "仅分片完成" },
{ name: "LIVE_RECORDER_DANMAKU_FILE_PATH", example: "/app/records/Douyin/布莱克/2026-04-25/203000_今晚划水聊天__00001.xml", scope: "仅分片完成" },
{ name: "LIVE_RECORDER_DURATION_SECONDS", example: "2185.1", scope: "仅分片完成" },
{ name: "LIVE_RECORDER_FILE_SIZE_BYTES", example: "734003200", scope: "仅分片完成" },
{ name: "LIVE_RECORDER_TASK_STATUS", example: "Completed", scope: "仅分片完成" },
{ name: "LIVE_RECORDER_SESSION_STATUS", example: "Running", scope: "仅分片完成" }
];
const canSendTestEmail = computed(() => const canSendTestEmail = computed(() =>
Boolean(form.emailSmtpHost.trim() && form.emailFromAddress.trim() && form.emailToAddresses.trim()) Boolean(form.emailSmtpHost.trim() && form.emailFromAddress.trim() && form.emailToAddresses.trim())
); );
@@ -480,7 +500,25 @@ onMounted(loadSettings);
</el-form> </el-form>
<div class="helper-panel"> <div class="helper-panel">
常用环境变量LIVE_RECORDER_EVENTLIVE_RECORDER_PLATFORMLIVE_RECORDER_ROOM_IDLIVE_RECORDER_ANCHORLIVE_RECORDER_RECORD_SESSION_IDLIVE_RECORDER_RECORD_TASK_IDLIVE_RECORDER_SEGMENT_INDEXLIVE_RECORDER_SEGMENT_FILE_PATHLIVE_RECORDER_DANMAKU_FILE_PATH <div class="event-script-help__intro">
开播/下播脚本会带全部事件里的变量分片完成脚本会额外带分片路径弹幕路径时长文件大小任务状态这些值取不到时会传空字符串
</div>
<div class="event-script-example">
<div class="event-script-example__label">环境变量示例值</div>
<div class="event-script-example__grid">
<div v-for="item in eventScriptEnvironmentExamples" :key="item.name" class="event-script-example__row">
<code>{{ item.name }}</code>
<code class="event-script-example__value">{{ item.example }}</code>
<span class="event-script-example__scope">{{ item.scope }}</span>
</div>
</div>
</div>
<div class="event-script-help__usage">
<div><strong>Bash:</strong> <code>echo "$LIVE_RECORDER_EVENT $LIVE_RECORDER_SEGMENT_FILE_PATH"</code></div>
<div><strong>PowerShell:</strong> <code>Write-Host "$env:LIVE_RECORDER_EVENT $env:LIVE_RECORDER_SEGMENT_FILE_PATH"</code></div>
</div>
</div> </div>
</el-card> </el-card>
@@ -692,6 +730,57 @@ onMounted(loadSettings);
line-height: 1.7; line-height: 1.7;
} }
.event-script-help__intro {
margin-bottom: 12px;
}
.event-script-example {
display: grid;
gap: 10px;
margin-top: 12px;
padding: 14px;
border-radius: 10px;
background: rgba(255, 255, 255, 0.72);
border: 1px solid rgba(91, 110, 129, 0.12);
}
.event-script-example__label {
font-size: 12px;
font-weight: 700;
color: #687381;
letter-spacing: 0.06em;
text-transform: uppercase;
}
.event-script-example__grid {
display: grid;
gap: 8px;
}
.event-script-example__row {
display: grid;
grid-template-columns: minmax(220px, 1.2fr) minmax(0, 1.8fr) auto;
gap: 12px;
align-items: start;
font-size: 12px;
}
.event-script-example__value {
overflow-wrap: anywhere;
color: #44505c;
}
.event-script-example__scope {
color: #687381;
white-space: nowrap;
}
.event-script-help__usage {
display: grid;
gap: 6px;
margin-top: 12px;
}
.template-section { .template-section {
padding: 18px 18px 4px; padding: 18px 18px 4px;
border-radius: 12px; border-radius: 12px;
@@ -770,6 +859,15 @@ onMounted(loadSettings);
flex-direction: column; flex-direction: column;
align-items: flex-start; align-items: flex-start;
} }
.event-script-example__row {
grid-template-columns: 1fr;
gap: 4px;
}
.event-script-example__scope {
white-space: normal;
}
} }
@media (max-width: 768px) { @media (max-width: 768px) {
@@ -4,6 +4,7 @@ using LiveRecorder.Application.Abstractions.Platforms;
using LiveRecorder.Application.Abstractions.Recording; using LiveRecorder.Application.Abstractions.Recording;
using LiveRecorder.Application.Abstractions.Settings; using LiveRecorder.Application.Abstractions.Settings;
using LiveRecorder.Application.Models.LiveRooms; using LiveRecorder.Application.Models.LiveRooms;
using LiveRecorder.Application.Models.RecordTasks;
using LiveRecorder.Domain.Entities; using LiveRecorder.Domain.Entities;
using LiveRecorder.Domain.Enums; using LiveRecorder.Domain.Enums;
@@ -17,6 +18,7 @@ public sealed class LiveRoomService
private readonly LiveRoomRecordingSettingsResolver _liveRoomRecordingSettingsResolver; private readonly LiveRoomRecordingSettingsResolver _liveRoomRecordingSettingsResolver;
private readonly ISystemSettingsService _systemSettingsService; private readonly ISystemSettingsService _systemSettingsService;
private readonly StoppedOrphanRecordSessionCleanupService _stoppedOrphanRecordSessionCleanupService; private readonly StoppedOrphanRecordSessionCleanupService _stoppedOrphanRecordSessionCleanupService;
private readonly RecordService _recordService;
private readonly IUnitOfWork _unitOfWork; private readonly IUnitOfWork _unitOfWork;
private readonly ISystemLogService _systemLogService; private readonly ISystemLogService _systemLogService;
@@ -27,6 +29,7 @@ public sealed class LiveRoomService
LiveRoomRecordingSettingsResolver liveRoomRecordingSettingsResolver, LiveRoomRecordingSettingsResolver liveRoomRecordingSettingsResolver,
ISystemSettingsService systemSettingsService, ISystemSettingsService systemSettingsService,
StoppedOrphanRecordSessionCleanupService stoppedOrphanRecordSessionCleanupService, StoppedOrphanRecordSessionCleanupService stoppedOrphanRecordSessionCleanupService,
RecordService recordService,
IUnitOfWork unitOfWork, IUnitOfWork unitOfWork,
ISystemLogService systemLogService) ISystemLogService systemLogService)
{ {
@@ -36,6 +39,7 @@ public sealed class LiveRoomService
_liveRoomRecordingSettingsResolver = liveRoomRecordingSettingsResolver; _liveRoomRecordingSettingsResolver = liveRoomRecordingSettingsResolver;
_systemSettingsService = systemSettingsService; _systemSettingsService = systemSettingsService;
_stoppedOrphanRecordSessionCleanupService = stoppedOrphanRecordSessionCleanupService; _stoppedOrphanRecordSessionCleanupService = stoppedOrphanRecordSessionCleanupService;
_recordService = recordService;
_unitOfWork = unitOfWork; _unitOfWork = unitOfWork;
_systemLogService = systemLogService; _systemLogService = systemLogService;
} }
@@ -214,6 +218,7 @@ public sealed class LiveRoomService
$"Live room resolved: {liveRoom.RoomId} ({liveRoom.Platform}).", $"Live room resolved: {liveRoom.RoomId} ({liveRoom.Platform}).",
liveRoomId: liveRoom.Id, liveRoomId: liveRoom.Id,
cancellationToken: cancellationToken); cancellationToken: cancellationToken);
await TryAutoStartRecordingAsync(liveRoom, liveStatus, cancellationToken);
var effectiveSettings = await _liveRoomRecordingSettingsResolver.ResolveAsync(liveRoom, cancellationToken); var effectiveSettings = await _liveRoomRecordingSettingsResolver.ResolveAsync(liveRoom, cancellationToken);
return (liveRoom, effectiveSettings, created); return (liveRoom, effectiveSettings, created);
@@ -238,6 +243,7 @@ public sealed class LiveRoomService
detail: $"status={liveStatus.StatusCode}, rawStatus={liveStatus.RawStatus}", detail: $"status={liveStatus.StatusCode}, rawStatus={liveStatus.RawStatus}",
liveRoomId: room.Id, liveRoomId: room.Id,
cancellationToken: cancellationToken); cancellationToken: cancellationToken);
await TryAutoStartRecordingAsync(room, liveStatus, cancellationToken);
var effectiveSettings = await _liveRoomRecordingSettingsResolver.ResolveAsync(room, cancellationToken); var effectiveSettings = await _liveRoomRecordingSettingsResolver.ResolveAsync(room, cancellationToken);
return Map(room, effectiveSettings); return Map(room, effectiveSettings);
@@ -405,6 +411,56 @@ public sealed class LiveRoomService
return rooms.ToDictionary(item => item.Id, item => _liveRoomRecordingSettingsResolver.Resolve(item, systemSettings)); return rooms.ToDictionary(item => item.Id, item => _liveRoomRecordingSettingsResolver.Resolve(item, systemSettings));
} }
private async Task TryAutoStartRecordingAsync(
LiveRoom liveRoom,
LiveStatusSnapshot liveStatus,
CancellationToken cancellationToken)
{
if (!liveStatus.IsLive || !liveRoom.IsEnabled)
{
return;
}
var settings = await _systemSettingsService.GetAsync(cancellationToken);
if (!settings.AutoStartRecordingOnLive)
{
await _systemLogService.WriteAsync(
SystemLogLevel.Info,
"LiveRoom",
"Live room is already online, but immediate auto-start is disabled by settings.",
liveRoomId: liveRoom.Id,
cancellationToken: cancellationToken);
return;
}
await _systemLogService.WriteAsync(
SystemLogLevel.Info,
"LiveRoom",
"Live room is already online. Attempting immediate auto-start after resolve/refresh.",
liveRoomId: liveRoom.Id,
cancellationToken: cancellationToken);
try
{
await _recordService.StartAsync(
new StartRecordTaskRequest
{
LiveRoomId = liveRoom.Id
},
cancellationToken);
}
catch (Exception ex)
{
await _systemLogService.WriteAsync(
ex is InvalidOperationException ? SystemLogLevel.Warning : SystemLogLevel.Error,
"LiveRoom",
"Immediate auto-start after resolve/refresh did not start a recording session.",
ex.Message,
liveRoomId: liveRoom.Id,
cancellationToken: cancellationToken);
}
}
private LiveRoomDto Map(LiveRoom room, RecordingExecutionSettings effectiveSettings) => new() private LiveRoomDto Map(LiveRoom room, RecordingExecutionSettings effectiveSettings) => new()
{ {
Id = room.Id, Id = room.Id,
@@ -80,7 +80,9 @@ public sealed partial class FfmpegService
var stderrLines = new Queue<string>(); var stderrLines = new Queue<string>();
var progressSync = new object(); var progressSync = new object();
double? processedSeconds = null; double? processedSeconds = null;
var lastReportedWholePercent = -1; double? lastReportedProgressPercent = null;
string? lastReportedStage = null;
string? lastReportedDetail = null;
var attemptStartedAt = DateTimeOffset.UtcNow; var attemptStartedAt = DateTimeOffset.UtcNow;
var activeStage = "Finalizing MP4"; var activeStage = "Finalizing MP4";
var maxDuration = TimeSpan.FromMinutes(Math.Clamp(mp4FinalizeTimeoutMinutes, 1, 1440)); var maxDuration = TimeSpan.FromMinutes(Math.Clamp(mp4FinalizeTimeoutMinutes, 1, 1440));
@@ -114,19 +116,28 @@ public sealed partial class FfmpegService
detail = $"Processed {processedSeconds.Value:F1}s"; detail = $"Processed {processedSeconds.Value:F1}s";
} }
var wholePercent = progressPercent.HasValue ? (int)Math.Floor(progressPercent.Value) : -1; var effectiveStage = stageOverride ?? activeStage;
if (stageOverride is null && wholePercent == lastReportedWholePercent) var normalizedProgressPercent = progressPercent.HasValue
? Math.Round(progressPercent.Value, 1, MidpointRounding.AwayFromZero)
: (double?)null;
var effectiveDetail = detail ?? $"Optimizing MP4 index for {Path.GetFileName(targetPath)}";
if (string.Equals(effectiveStage, lastReportedStage, StringComparison.Ordinal) &&
string.Equals(effectiveDetail, lastReportedDetail, StringComparison.Ordinal) &&
Nullable.Equals(normalizedProgressPercent, lastReportedProgressPercent))
{ {
return; return;
} }
lastReportedWholePercent = wholePercent; lastReportedStage = effectiveStage;
lastReportedDetail = effectiveDetail;
lastReportedProgressPercent = normalizedProgressPercent;
SetPostProcessState( SetPostProcessState(
recordSessionId, recordSessionId,
recordTaskId, recordTaskId,
stageOverride ?? activeStage, effectiveStage,
progressPercent, normalizedProgressPercent,
detail ?? $"Optimizing MP4 index for {Path.GetFileName(targetPath)}"); effectiveDetail);
} }
} }
@@ -159,7 +170,9 @@ public sealed partial class FfmpegService
ClearErrorDetail(); ClearErrorDetail();
activeStage = stage; activeStage = stage;
processedSeconds = null; processedSeconds = null;
lastReportedWholePercent = -1; lastReportedProgressPercent = null;
lastReportedStage = null;
lastReportedDetail = null;
attemptStartedAt = DateTimeOffset.UtcNow; attemptStartedAt = DateTimeOffset.UtcNow;
System.Threading.Interlocked.Exchange(ref lastActivityTicks, attemptStartedAt.UtcTicks); System.Threading.Interlocked.Exchange(ref lastActivityTicks, attemptStartedAt.UtcTicks);
@@ -242,7 +242,7 @@ public sealed partial class FfmpegService : IFfmpegService
public async Task<bool> TryReconcileInactiveSessionAsync(Guid recordSessionId, CancellationToken cancellationToken = default) public async Task<bool> TryReconcileInactiveSessionAsync(Guid recordSessionId, CancellationToken cancellationToken = default)
{ {
if (IsRunning(recordSessionId) || IsSessionUnderPostProcessing(recordSessionId)) if (IsRunning(recordSessionId))
{ {
return false; return false;
} }
@@ -269,11 +269,23 @@ public sealed partial class FfmpegService : IFfmpegService
.ThenBy(static item => item.CreatedAt) .ThenBy(static item => item.CreatedAt)
.ToList(); .ToList();
var anyUsableOutput = false; var anyUsableOutput = tasks.Any(static item => item.Status == RecordTaskStatus.Completed);
var hasBackgroundPostProcessing = false;
string? sessionFinalizationError = null; string? sessionFinalizationError = null;
foreach (var task in tasks.Where(item => IsActiveTaskStatus(item.Status))) foreach (var task in tasks)
{ {
if (IsTaskUnderPostProcessing(task.Id))
{
hasBackgroundPostProcessing = true;
continue;
}
if (!IsActiveTaskStatus(task.Status))
{
continue;
}
var durationSeconds = task.StartedAt.HasValue var durationSeconds = task.StartedAt.HasValue
? (double?)Math.Max(0, (endedAt - task.StartedAt.Value).TotalSeconds) ? (double?)Math.Max(0, (endedAt - task.StartedAt.Value).TotalSeconds)
: null; : null;
@@ -299,6 +311,7 @@ public sealed partial class FfmpegService : IFfmpegService
if (IsLowStoragePauseError(taskFinalizationError)) if (IsLowStoragePauseError(taskFinalizationError))
{ {
task.MarkProcessing(taskFinalizationError, endedAt); task.MarkProcessing(taskFinalizationError, endedAt);
hasBackgroundPostProcessing = true;
} }
else if (!string.IsNullOrWhiteSpace(taskFinalizationError)) else if (!string.IsNullOrWhiteSpace(taskFinalizationError))
{ {
@@ -323,6 +336,12 @@ public sealed partial class FfmpegService : IFfmpegService
{ {
recordSession.MarkFailed(sessionFinalizationError, endedAt); recordSession.MarkFailed(sessionFinalizationError, endedAt);
} }
else if (hasBackgroundPostProcessing)
{
recordSession.MarkStopped(
endedAt,
"Recording process was no longer running when the session was reconciled. Post-processing will continue in background.");
}
else if (anyUsableOutput) else if (anyUsableOutput)
{ {
recordSession.MarkCompleted(endedAt); recordSession.MarkCompleted(endedAt);
@@ -594,8 +613,8 @@ public sealed partial class FfmpegService : IFfmpegService
private void ClearPostProcessState(Guid recordTaskId) => private void ClearPostProcessState(Guid recordTaskId) =>
_postProcessStates.TryRemove(recordTaskId, out _); _postProcessStates.TryRemove(recordTaskId, out _);
private bool IsSessionUnderPostProcessing(Guid recordSessionId) => private bool IsTaskUnderPostProcessing(Guid recordTaskId) =>
_postProcessStates.Values.Any(entry => entry.RecordSessionId == recordSessionId); _postProcessStates.ContainsKey(recordTaskId);
private async Task<IDisposable> AcquireTranscodeSlotAsync(int maxConcurrentTasks, CancellationToken cancellationToken) private async Task<IDisposable> AcquireTranscodeSlotAsync(int maxConcurrentTasks, CancellationToken cancellationToken)
{ {
@@ -93,7 +93,14 @@ public sealed class LiveRoomPollingBackgroundService : BackgroundService
try try
{ {
using var notificationScope = _serviceScopeFactory.CreateScope(); using var notificationScope = _serviceScopeFactory.CreateScope();
var logService = notificationScope.ServiceProvider.GetRequiredService<ISystemLogService>();
var emailNotificationService = notificationScope.ServiceProvider.GetRequiredService<IEmailNotificationService>(); var emailNotificationService = notificationScope.ServiceProvider.GetRequiredService<IEmailNotificationService>();
await logService.WriteAsync(
SystemLogLevel.Error,
"Scheduler",
"Background live room polling failed.",
ex.ToString(),
cancellationToken: CancellationToken.None);
if (ShouldSendExceptionEmail(BuildExceptionEmailKey("background-loop", ex))) if (ShouldSendExceptionEmail(BuildExceptionEmailKey("background-loop", ex)))
{ {
await emailNotificationService.SendExceptionAsync( await emailNotificationService.SendExceptionAsync(