ActivityPub-federated video streaming platform using P2P directly in your web browser
More robust concurrent live handler
Chocobozzzaff09e05f831
diff --git a/packages/tests/src/api/live/live.ts b/packages/tests/src/api/live/live.tsindex 5c49dfbd65..e4c9c97c1a 100644--- a/packages/tests/src/api/live/live.ts+++ b/packages/tests/src/api/live/live.ts@@ -519,23 +519,27 @@ describe('Test live', function () {await testFfmpegStreamError(command, true)})- it('Should succeed with the correct params', async function () {+ it('Should succeed with the correct params and list the live', async function () {this.timeout(60000)const command = sendRTMPStream({ rtmpBaseUrl: rtmpUrl + '/live', streamKey: liveVideo.streamKey })- await testFfmpegStreamError(command, false)- })- it('Should list this live now someone stream into it', async function () {- for (const server of servers) {- const { total, data } = await server.videos.list()+ try {+ await waitUntilLivePublishedOnAllServers(servers, liveVideo.uuid)- expect(total).to.equal(1)- expect(data).to.have.lengthOf(1)+ for (const server of servers) {+ const { total, data } = await server.videos.list()- const video = data[0]- expect(video.name).to.equal('user live')- expect(video.isLive).to.be.true+ expect(total).to.equal(1)+ expect(data).to.have.lengthOf(1)++ const video = data[0]+ expect(video.name).to.equal('user live')+ expect(video.isLive).to.be.true+ }+ } finally {+ // Don't leak the ffmpeg process in the next tests if an assertion above failed+ await stopFfmpeg(command)}})diff --git a/server/core/initializers/constants.ts b/server/core/initializers/constants.tsindex 89735f7d8e..e4e6f12bb5 100644--- a/server/core/initializers/constants.ts+++ b/server/core/initializers/constants.ts@@ -1176,6 +1176,17 @@ export const RESUMABLE_UPLOAD_SESSION_LIFETIME = SCHEDULER_INTERVALS_MS.REMOVE_Dexport const VIDEO_LIVE = {EXTENSION: '.ts',CLEANUP_DELAY: 1000 * 60 * 5, // 5 minutes+ // Delay before aborting a session on RTMP disconnection, so we kill ffmpeg even if it still has data to process+ ABORT_DELAY_ON_RTMP_DISCONNECT: 2000, // 2 seconds+ // Max time we wait for ffmpeg to exit after we sent it a SIGINT, before killing it+ FFMPEG_EXIT_TIMEOUT: 10000, // 10 seconds+ // Time we give a remote runner to flush its last chunks after we aborted the session+ // If this delay is too short, the last chunks of this session can land in the directory *after* we told everyone we released it+ REMOTE_RUNNER_FLUSH_DELAY: 5000, // 5 seconds+ // Max time a new session waits for the previous session of a permanent live to release the live directory+ PREVIOUS_SESSION_CLEANUP_TIMEOUT: 30000, // 30 seconds+ // Max time we wait for ffmpeg to fill the live master playlist it just created before giving up on it+ MASTER_PLAYLIST_READ_TIMEOUT: 5000, // 5 secondsSEGMENT_TIME_SECONDS: {DEFAULT_LATENCY: 4, // 4 secondsSMALL_LATENCY: 2 // 2 secondsdiff --git a/server/core/lib/live/live-manager.ts b/server/core/lib/live/live-manager.tsindex 94a1a0f79e..f9e2cec727 100644--- a/server/core/lib/live/live-manager.ts+++ b/server/core/lib/live/live-manager.ts@@ -1,4 +1,4 @@-import { pick, wait } from '@peertube/peertube-core-utils'+import { pick, timeoutPromise, wait } from '@peertube/peertube-core-utils'import {ffprobePromise,getVideoStreamBitrate,@@ -62,6 +62,10 @@ class LiveManager {private readonly muxingSessions = new Map<string, MuxingSession>()private readonly videoSessions = new Map<string, string>()+ // Video UUID -> cleanup of its current/last muxing session+ // A permanent live can be streamed again before the previous session finished to write its files on disk+ private readonly sessionCleanups = new Map<string, { sessionId: string, cleanup: Promise<void> }>()+private rtmpServer: Serverprivate rtmpsServer: ServerTLS@@ -94,7 +98,7 @@ class LiveManager {logger.info('Live session ended.', { sessionId, ...lTags(sessionId) })// Force session aborting, so we kill ffmpeg even if it still has data to process (slow CPU)- setTimeout(() => this.abortSession(sessionId), 2000)+ setTimeout(() => this.abortSession(sessionId), VIDEO_LIVE.ABORT_DELAY_ON_RTMP_DISCONNECT)})registerConfigChangedHandler(() => {@@ -231,6 +235,13 @@ class LiveManager {return context}+ // Don't remove the cleanup of a more recent session of the same video+ private deleteSessionCleanup (videoUUID: string, sessionId: string) {+ if (this.sessionCleanups.get(videoUUID)?.sessionId !== sessionId) return++ this.sessionCleanups.delete(videoUUID)+ }+private abortSession (sessionId: string) {const session = this.getContext().sessions.get(sessionId)if (session) {@@ -300,6 +311,30 @@ class LiveManager {return this.abortSession(sessionId)}+ // A previous session of this permanent live may still be writing its last segments on disk+ // Wait for it, so we don't cleanup/write the same files from two sessions at the same time+ const previous = this.sessionCleanups.get(video.uuid)+ if (previous) {+ logger.info(+ 'Waiting for the previous live session %s of video %s to be cleaned up.',+ previous.sessionId,+ video.uuid,+ lTags(sessionId, video.uuid)+ )++ // Never block this live forever if the previous cleanup is stuck (a hanging object storage upload for example)+ try {+ await timeoutPromise(previous.cleanup, VIDEO_LIVE.PREVIOUS_SESSION_CLEANUP_TIMEOUT)+ } catch (err) {+ logger.warn(+ 'Previous live session %s of video %s is still not cleaned up, starting the new session anyway.',+ previous.sessionId,+ video.uuid,+ { err, ...lTags(sessionId, video.uuid) }+ )+ }+ }+// Cleanup old potential live (could happen with a permanent live)const oldStreamingPlaylist = await VideoStreamingPlaylistModel.loadHLSByVideo(video.id)if (oldStreamingPlaylist) {@@ -463,6 +498,9 @@ class LiveManager {])})+ // Track the cleanup from the session creation: a new session must not write in the live directory before this one flushed its files+ this.sessionCleanups.set(videoUUID, { sessionId, cleanup: muxingSession.waitForCleanup() })+muxingSession.on('live-ready', () => this.publishAndFederateLive({ live: videoLive, ratio, audioOnlyOutput, localLTags }))const safeStopSession = (error: LiveVideoErrorType) => {@@ -493,7 +531,7 @@ class LiveManager {safeStopSession(LiveVideoError.QUOTA_EXCEEDED)})- muxingSession.on('transcoding-error', ({ videoUUID }) => {+ muxingSession.on('transcoding-error', () => {safeStopSession(LiveVideoError.FFMPEG_ERROR)})@@ -503,6 +541,7 @@ class LiveManager {muxingSession.on('after-cleanup', ({ videoUUID }) => {this.muxingSessions.delete(sessionId)+ this.deleteSessionCleanup(videoUUID, sessionId)LiveQuotaStore.Instance.removeLive(user.id, sessionId)@@ -519,9 +558,11 @@ class LiveManager {logger.error('Cannot run muxing.', { err, ...localLTags })this.muxingSessions.delete(sessionId)+ this.deleteSessionCleanup(videoUUID, sessionId)LiveQuotaStore.Instance.removeLive(user.id, sessionId)+ // Resolves the cleanup promise, so a new session of this permanent live is not blockedmuxingSession.destroy()this.stopSessionOfVideo({@@ -574,8 +615,9 @@ class LiveManager {}private onMuxingFFmpegEnd (videoUUID: string, sessionId: string) {- // Session already cleaned up- if (!this.videoSessions.has(videoUUID)) return+ // Session already cleaned up, or a more recent session of this permanent live already registered itself:+ // the transcoding process of an ending session can exit long after the next one started+ if (this.videoSessions.get(videoUUID) !== sessionId) returnthis.videoSessions.delete(videoUUID)diff --git a/server/core/lib/live/live-utils.ts b/server/core/lib/live/live-utils.tsindex 86c2948114..2df2939eba 100644--- a/server/core/lib/live/live-utils.ts+++ b/server/core/lib/live/live-utils.ts@@ -4,6 +4,7 @@ import { logger } from '@server/helpers/logger.js'import { CONFIG } from '@server/initializers/config.js'import { VIDEO_LIVE } from '@server/initializers/constants.js'import { MStreamingPlaylist, MStreamingPlaylistVideo, MVideo } from '@server/types/models/index.js'+import Bluebird from 'bluebird'import { pathExists, remove } from 'fs-extra/esm'import { readdir, rmdir } from 'fs/promises'import { basename, join } from 'path'@@ -17,7 +18,7 @@ export function buildConcatenatedName (segmentOrPlaylistPath: string) {}export async function cleanupAndDestroyPermanentLive (video: MVideo, streamingPlaylist: MStreamingPlaylist) {- await cleanupTMPLiveFiles(video, streamingPlaylist)+ const cleaned = await cleanupTMPLiveFiles(video, streamingPlaylist)if (video.state === VideoState.WAITING_FOR_LIVE) {// Try to delete local filesystem empty paths@@ -31,6 +32,15 @@ export async function cleanupAndDestroyPermanentLive (video: MVideo, streamingPl}}+ // Keep the playlist instead, so the next session retries the cleanup before writing in the directory+ if (!cleaned) {+ logger.warn(+ 'Not destroying the streaming playlist of %s: some live files could not be removed, keeping it so the cleanup is retried.',+ video.uuid+ )+ return+ }+await streamingPlaylist.destroy()}@@ -47,10 +57,11 @@ export async function cleanupUnsavedNormalLive (video: MVideo, streamingPlaylistawait streamingPlaylist.destroy()}+// Returns false if some files could not be removedexport async function cleanupTMPLiveFiles (video: MVideo, streamingPlaylist: MStreamingPlaylist) {await cleanupTMPLiveFilesFromObjectStorage(streamingPlaylist.withVideo(video))- await cleanupTMPLiveFilesFromFilesystem(video)+ return cleanupTMPLiveFilesFromFilesystem(video)}export function getLiveSegmentTime (latencyMode: LiveVideoLatencyModeType) {@@ -90,23 +101,32 @@ function isTMPLiveFile (name: string) {name.endsWith('.tmp')}+// Returns false if some files could not be removedasync function cleanupTMPLiveFilesFromFilesystem (video: MVideo) {const hlsDirectory = getLiveDirectory(video)- if (!await pathExists(hlsDirectory)) return+ if (!await pathExists(hlsDirectory)) return truelogger.info('Cleanup TMP live files from filesystem of %s.', hlsDirectory)const files = await readdir(hlsDirectory)- for (const filename of files) {- if (isTMPLiveFile(filename)) {- const p = join(hlsDirectory, filename)+ let cleaned = true- remove(p)- .catch(err => logger.error('Cannot remove %s.', p, { err }))- }- }+ // Await the removals: the caller can start a new live session in this directory just after+ // A live directory can hold thousands of segments with a big DVR window, so don't remove them all at once+ await Bluebird.map(files.filter(filename => isTMPLiveFile(filename)), filename => {+ const p = join(hlsDirectory, filename)++ return remove(p)+ .catch(err => {+ logger.error('Cannot remove %s.', p, { err })++ cleaned = false+ })+ }, { concurrency: 10 })++ return cleaned}async function cleanupTMPLiveFilesFromObjectStorage (streamingPlaylist: MStreamingPlaylistVideo) {diff --git a/server/core/lib/live/shared/muxing-session.ts b/server/core/lib/live/shared/muxing-session.tsindex feed6ecbb8..a8825e40cf 100644--- a/server/core/lib/live/shared/muxing-session.ts+++ b/server/core/lib/live/shared/muxing-session.ts@@ -8,7 +8,7 @@ import {VideoStreamingPlaylistType} from '@peertube/peertube-models'import { computeOutputFPS } from '@server/helpers/ffmpeg/index.js'-import { LoggerTagsFn, logger, loggerTagsFactory } from '@server/helpers/logger.js'+import { logger, loggerTagsFactory, LoggerTagsFn } from '@server/helpers/logger.js'import { CONFIG } from '@server/initializers/config.js'import { MEMOIZE_TTL, P2P_MEDIA_LOADER_PEER_VERSION, VIDEO_LIVE } from '@server/initializers/constants.js'import { removeHLSFileObjectStorageByPath, storeHLSFileFromContent, storeHLSFileFromPath } from '@server/lib/object-storage/index.js'@@ -20,7 +20,7 @@ import { FSWatcher, watch } from 'chokidar'import { EventEmitter } from 'events'import { FfprobeData } from 'fluent-ffmpeg'import { ensureDir } from 'fs-extra/esm'-import { appendFile, readFile, stat } from 'fs/promises'+import { appendFile, readdir, readFile, stat } from 'fs/promises'import memoizee from 'memoizee'import PQueue from 'p-queue'import { basename, join } from 'path'@@ -100,6 +100,23 @@ class MuxingSession extends EventEmitter implements MuxingSession {private segmentsToProcessPerPlaylist: { [playlistId: string]: string[] } = {}+ // A segment can be seen twice: by the files watcher and by the cleanup task+ private readonly processedSegments = new Set<string>()++ // Playlist ID -> Queue+ // Segments of a same playlist are processed in order, and the cleanup can wait for the pending ones+ private readonly segmentProcessingQueues = new Map<string, PQueue>()++ // Closing the files watcher stops new events but not the async handlers it already started, that can still write+ // in the live directory (or in object storage): the cleanup waits for them+ private readonly pendingWatcherHandlers = new Set<Promise<void>>()++ private cleanupResolve: () => void+ // Settled by runCleanup(), and by destroy() as a safety net so waitForCleanup() callers can never hang+ private readonly cleanupPromise = new Promise<void>(resolve => {+ this.cleanupResolve = resolve+ })+private streamingPlaylist: MStreamingPlaylistVideoprivate liveSegmentShaStore: LiveSegmentShaStore@@ -181,13 +198,10 @@ class MuxingSession extends EventEmitter implements MuxingSession {await this.prepareDirectories()- this.transcodingWrapper = this.buildTranscodingWrapper(toTranscode)-- this.transcodingWrapper.on('end', () => this.onTranscodedEnded())- this.transcodingWrapper.on('error', () => this.onTranscodingError())-- await this.transcodingWrapper.run()-+ // Watch the directory *before* running the transcoding process, so the watcher can never be created after the+ // session was cleaned up and destroyed: run() can abort and emit 'end' before it returns, and the cleanup would+ // then close a watcher that does not exist yet+ // Chokidar also emits 'add' for the files it finds on init, so we don't miss anything by watching earlierthis.filesWatcher = watch(this.outDirectory, {// Ignore 'segments-sha256.json' and 'segments-sha256.json.tmp' files that are frequently updated and not usefulignored: path => path.endsWith('.json') || path.endsWith('json.tmp'),@@ -196,13 +210,36 @@ class MuxingSession extends EventEmitter implements MuxingSession {this.watchMasterFile()this.watchTSFiles()++ this.transcodingWrapper = this.buildTranscodingWrapper(toTranscode)++ this.transcodingWrapper.on('end', () => this.onTranscodedEnded())+ this.transcodingWrapper.on('error', () => this.onTranscodingError())++ // abort() was called before the wrapper existed+ // Abort the wrapper anyway, so it emits 'end' and the cleanup of this session is scheduled+ if (this.aborted) {+ logger.debug('Live muxing of %s was aborted before the transcoding process started.', this.videoUUID, this.lTags())++ this.transcodingWrapper.abort()+ return+ }++ await this.transcodingWrapper.run()}abort () {- if (!this.transcodingWrapper) return-+ if (this.aborted) returnthis.aborted = true- this.transcodingWrapper.abort()++ // The wrapper may not exist yet: runMuxing() checks this flag before running it+ this.transcodingWrapper?.abort()+ }++ // Resolves when this session does not write in the live directory anymore+ // The transcoding process has exited and all its segments have been hashed/stored+ waitForCleanup () {+ return this.cleanupPromise}destroy () {@@ -211,6 +248,13 @@ class MuxingSession extends EventEmitter implements MuxingSession {this.closeWatcher().catch(err => logger.error('Cannot close files watcher of %s.', this.outDirectory, { err, ...this.lTags() }))+ // We removed its listeners below, so don't leave a pending timer that would emit an event nobody handles+ this.transcodingWrapper?.destroy()++ // Safety net: this session is over, so unblock everyone waiting for its cleanup even if it was never scheduled+ // (e.g. runMuxing() threw before the transcoding wrapper was able to run)+ this.cleanupResolve()+this.removeAllListeners()this.isAbleToUploadVideoWithCache.clear()this.hasClientSocketInBadHealthWithCache.clear()@@ -225,22 +269,27 @@ class MuxingSession extends EventEmitter implements MuxingSession {return watcher.close()}+ // Watcher handlers are async and fire and forget: remember them so the cleanup can wait for the pending ones+ private trackWatcherHandler (handler: Promise<void>) {+ const tracked: Promise<void> = handler+ .catch(err => {+ logger.error('Error in live files watcher handler of %s.', this.outDirectory, { err, ...this.lTags() })+ })+ .finally(() => {+ this.pendingWatcherHandlers.delete(tracked)+ })++ this.pendingWatcherHandlers.add(tracked)+ }+private watchMasterFile () {- this.filesWatcher.on('add', async path => {+ const addHandler = async (path: string) => {if (path !== join(this.outDirectory, this.streamingPlaylist.playlistFilename)) returnif (this.masterPlaylistCreated === true) returntry {if (this.streamingPlaylist.storage === FileStorage.OBJECT_STORAGE) {- let masterContent = await readFile(path, 'utf-8')-- // If the disk sync is slow, don't upload an empty master playlist on object storage- // Wait for ffmpeg to correctly fill it- while (!masterContent) {- await wait(100)-- masterContent = await readFile(path, 'utf-8')- }+ const masterContent = await this.readNonEmptyMasterPlaylist(path)logger.debug('Uploading live master playlist on object storage for %s', this.videoUUID, { masterContent, ...this.lTags() })@@ -266,12 +315,38 @@ class MuxingSession extends EventEmitter implements MuxingSession {await this.streamingPlaylist.save()} catch (err) {logger.error('Cannot update streaming playlist.', { err, ...this.lTags() })++ // Don't set masterPlaylistCreated: without a stored master playlist the live would be published but unplayable+ // Stop the session instead, so we don't federate a broken live+ this.stopBrokenSession()+ return}this.masterPlaylistCreated = truelogger.info('Master playlist file for %s has been created', this.videoUUID, this.lTags())- })+ }++ this.filesWatcher.on('add', path => this.trackWatcherHandler(addHandler(path)))+ }++ // Throws if ffmpeg did not fill the master playlist in time: we don't want to store an empty one+ private async readNonEmptyMasterPlaylist (path: string) {+ const deadline = Date.now() + VIDEO_LIVE.MASTER_PLAYLIST_READ_TIMEOUT++ do {+ const content = await readFile(path, 'utf-8')+ if (content) return content++ // The session is over: bail out instead of making the cleanup wait for us+ if (this.cleanupScheduled) break++ // If the disk sync is slow, don't upload an empty master playlist on object storage+ // Wait for ffmpeg to correctly fill it+ await wait(100)+ } while (Date.now() < deadline)++ throw new Error(`Live master playlist ${path} is still empty after ${VIDEO_LIVE.MASTER_PLAYLIST_READ_TIMEOUT} ms`)}private watchTSFiles () {@@ -283,9 +358,14 @@ class MuxingSession extends EventEmitter implements MuxingSession {logger.debug('Live add handler of TS file %s.', segmentPath, this.lTags())const playlistId = this.getPlaylistIdFromTS(segmentPath)+ if (!playlistId) {+ logger.warn('Cannot get the playlist id of live segment %s, ignoring it.', segmentPath, this.lTags())+ return+ }const segmentsToProcess = this.segmentsToProcessPerPlaylist[playlistId] || []- this.processSegments(segmentsToProcess)+ // The cleanup waits for the queue, so we don't need to await it here+ void this.processSegments(playlistId, segmentsToProcess)this.segmentsToProcessPerPlaylist[playlistId] = [ segmentPath ]@@ -302,6 +382,9 @@ class MuxingSession extends EventEmitter implements MuxingSession {logger.debug('Live delete handler of TS file %s.', segmentPath, this.lTags())+ // The segment does not exist anymore so the cleanup won't list it: don't track it forever+ this.processedSegments.delete(segmentPath)+this.liveSegmentShaStore.removeSegmentSha(segmentPath)if (this.streamingPlaylist.storage === FileStorage.OBJECT_STORAGE) {@@ -314,7 +397,7 @@ class MuxingSession extends EventEmitter implements MuxingSession {}this.filesWatcher.on('add', p => addHandler(p))- this.filesWatcher.on('unlink', p => deleteHandler(p))+ this.filesWatcher.on('unlink', p => this.trackWatcherHandler(deleteHandler(p)))}private async isQuotaExceeded (segmentPath: string) {@@ -373,8 +456,22 @@ class MuxingSession extends EventEmitter implements MuxingSession {return now <= max}- private processSegments (segmentPaths: string[]) {- Bluebird.mapSeries(segmentPaths, previousSegment => this.processSegment(previousSegment))+ // All the segments must belong to the playlist of `playlistId`+ private processSegments (playlistId: string, segmentPaths: string[]) {+ if (segmentPaths.length === 0) return Promise.resolve()++ return this.getSegmentProcessingQueue(playlistId)+ .add(() =>+ // Catch per segment: mapSeries would abandon the next ones, and the cleanup is the last chance to process them+ Bluebird.mapSeries(segmentPaths, segmentPath => {+ return this.processSegment(segmentPath)+ .catch(err => {+ if (this.aborted) return++ logger.error('Cannot process segment %s.', segmentPath, { err, ...this.lTags() })+ })+ })+ ).catch(err => {if (this.aborted) return@@ -382,28 +479,52 @@ class MuxingSession extends EventEmitter implements MuxingSession {})}- private async processSegment (segmentPath: string) {- // Check user quota if the user enabled replay saving- if (await this.isQuotaExceeded(segmentPath) === true) {- this.emit('quota-exceeded', { videoUUID: this.videoUUID })- return+ private getSegmentProcessingQueue (playlistId: string) {+ if (!this.segmentProcessingQueues.has(playlistId)) {+ this.segmentProcessingQueues.set(playlistId, new PQueue({ concurrency: 1 }))}- // Add sha hash of previous segments, because ffmpeg should have finished generating them- await this.liveSegmentShaStore.addSegmentSha(segmentPath)+ return this.segmentProcessingQueues.get(playlistId)+ }- if (this.saveReplay) {- await this.addSegmentToReplay(segmentPath)- }+ private async processSegment (segmentPath: string) {+ // Already processed by the files watcher or by the cleanup+ if (this.processedSegments.has(segmentPath)) return+ // Mark it before processing it so the cleanup doesn't process it again while we're working on it+ this.processedSegments.add(segmentPath)- if (this.streamingPlaylist.storage === FileStorage.OBJECT_STORAGE) {- try {- await storeHLSFileFromPath(this.streamingPlaylist.Video, segmentPath)+ // The replay is an append only file: retrying a segment we already appended would duplicate it+ let appendedToReplay = false- await this.processM3U8ToObjectStorage(segmentPath)- } catch (err) {- logger.error('Cannot store TS segment %s in object storage', segmentPath, { err, ...this.lTags() })+ try {+ // Check user quota if the user enabled replay saving+ if (await this.isQuotaExceeded(segmentPath) === true) {+ this.emit('quota-exceeded', { videoUUID: this.videoUUID })+ return+ }++ // Add sha hash of previous segments, because ffmpeg should have finished generating them+ await this.liveSegmentShaStore.addSegmentSha(segmentPath)++ if (this.saveReplay) {+ await this.addSegmentToReplay(segmentPath)+ appendedToReplay = true}++ if (this.streamingPlaylist.storage === FileStorage.OBJECT_STORAGE) {+ try {+ await storeHLSFileFromPath(this.streamingPlaylist.Video, segmentPath)++ await this.processM3U8ToObjectStorage(segmentPath)+ } catch (err) {+ logger.error('Cannot store TS segment %s in object storage', segmentPath, { err, ...this.lTags() })+ }+ }+ } catch (err) {+ // Forget the segment so the cleanup can retry it, unless retrying it would duplicate it in the replay+ if (!appendedToReplay) this.processedSegments.delete(segmentPath)++ throw err}// Master playlist and segment JSON file are created, live is ready@@ -444,20 +565,28 @@ class MuxingSession extends EventEmitter implements MuxingSession {}private onTranscodingError () {- this.emit('transcoding-error', { videoUUID: this.videoUUID })-// On ffmpeg error the transcoding wrapper abort() short-circuits and never emits 'end'// So schedule the cleanup here too+ // Schedule it before emitting so listeners can wait for the cleanup to completethis.scheduleCleanup()++ this.emit('transcoding-error', { videoUUID: this.videoUUID })}- private onTranscodedEnded () {- this.emit('transcoding-end', { videoUUID: this.videoUUID })+ // The live is broken but the transcoding process is still running and writing files+ // Contrary to onTranscodingError() don't schedule the cleanup here+ private stopBrokenSession () {+ this.emit('transcoding-error', { videoUUID: this.videoUUID })+ }+ private onTranscodedEnded () {// Don't log the input URL, which contains the stream key (a long lived secret)logger.info('RTMP transmuxing for video %s ended. Scheduling cleanup', this.videoUUID, this.lTags())+ // Schedule it before emitting so listeners can wait for the cleanup to completethis.scheduleCleanup()++ this.emit('transcoding-end', { videoUUID: this.videoUUID })}private scheduleCleanup () {@@ -465,25 +594,68 @@ class MuxingSession extends EventEmitter implements MuxingSession {if (this.cleanupScheduled) returnthis.cleanupScheduled = true- setTimeout(() => {- // Wait latest segments generation, and close watchers- this.closeWatcher()- .then(() => {- // Process remaining segments hash- for (const key of Object.keys(this.segmentsToProcessPerPlaylist)) {- this.processSegments(this.segmentsToProcessPerPlaylist[key])- }- })- .catch(err => {- logger.error(- 'Cannot close watchers of %s or process remaining hash segments.',- this.outDirectory,- { err, ...this.lTags() }- )- })+ this.runCleanup()+ .catch(err => logger.error('Cannot run cleanup of %s.', this.outDirectory, { err, ...this.lTags() }))+ }++ private async runCleanup () {+ try {+ // The transcoding process exited so no new segment will be generated: we can close the watcher+ await this.closeWatcher()++ // Closing the watcher does not wait for the handlers it already started, that can still write files+ await Promise.all(this.pendingWatcherHandlers)++ // Wait for the segments the watcher already notified us about+ await Promise.all(Array.from(this.segmentProcessingQueues.values(), queue => queue.onIdle()))++ // Watcher events are asynchronous, so the last segments generated by the transcoding process may never have been+ // notified to us: list the output directory instead of relying on the events we received+ await this.processRemainingSegments()+ } catch (err) {+ logger.error(+ 'Cannot close watchers of %s or process remaining hash segments.',+ this.outDirectory,+ { err, ...this.lTags() }+ )+ }++ this.cleanupResolve()++ this.emit('after-cleanup', { videoUUID: this.videoUUID })+ }++ private async processRemainingSegments () {+ const filenames = await readdir(this.outDirectory)++ const segmentPaths = filenames+ // Segments are generated with a zero padded counter (%v-%06d.ts) so sorting them by name keeps the live order+ .filter(f => f.endsWith(VIDEO_LIVE.EXTENSION))+ .sort()+ .map(f => join(this.outDirectory, f))+ .filter(p => !this.processedSegments.has(p))++ let remainingCount = 0+ const perPlaylist = new Map<string, string[]>()++ for (const segmentPath of segmentPaths) {+ const playlistId = this.getPlaylistIdFromTS(segmentPath)+ // Not a segment generated by our transcoding process+ if (!playlistId) continue++ if (!perPlaylist.has(playlistId)) perPlaylist.set(playlistId, [])+ perPlaylist.get(playlistId).push(segmentPath)++ remainingCount+++ }++ if (remainingCount === 0) return++ logger.debug('Processing %d remaining live segments of %s.', remainingCount, this.videoUUID, this.lTags())- this.emit('after-cleanup', { videoUUID: this.videoUUID })- }, 1000)+ await Promise.all(+ Array.from(perPlaylist, ([ playlistId, paths ]) => this.processSegments(playlistId, paths))+ )}private hasClientSocketInBadHealth (sessionId: string) {@@ -583,10 +755,11 @@ class MuxingSession extends EventEmitter implements MuxingSession {: new FFmpegTranscodingWrapper(options)}+ // Returns undefined if the file was not generated by our transcoding processprivate getPlaylistIdFromTS (segmentPath: string) {- const playlistIdMatcher = /^([\d+])-/+ const playlistIdMatcher = /^(\d+)-/- return basename(segmentPath).match(playlistIdMatcher)[1]+ return basename(segmentPath).match(playlistIdMatcher)?.[1]}private getPlaylistNameFromTS (segmentPath: string) {diff --git a/server/core/lib/live/shared/transcoding-wrapper/abstract-transcoding-wrapper.ts b/server/core/lib/live/shared/transcoding-wrapper/abstract-transcoding-wrapper.tsindex 6f0579cb84..5ae8ae2b77 100644--- a/server/core/lib/live/shared/transcoding-wrapper/abstract-transcoding-wrapper.ts+++ b/server/core/lib/live/shared/transcoding-wrapper/abstract-transcoding-wrapper.ts@@ -111,9 +111,11 @@ abstract class AbstractTranscodingWrapper extends EventEmitter {abstract run (): Promise<void>abstract abort (abortError?: LiveVideoErrorType): void++ abstract destroy (): void}export {- type AbstractTranscodingWrapperOptions,- AbstractTranscodingWrapper+ AbstractTranscodingWrapper,+ type AbstractTranscodingWrapperOptions}diff --git a/server/core/lib/live/shared/transcoding-wrapper/ffmpeg-transcoding-wrapper.ts b/server/core/lib/live/shared/transcoding-wrapper/ffmpeg-transcoding-wrapper.tsindex b432153e2a..0d0373584c 100644--- a/server/core/lib/live/shared/transcoding-wrapper/ffmpeg-transcoding-wrapper.ts+++ b/server/core/lib/live/shared/transcoding-wrapper/ffmpeg-transcoding-wrapper.ts@@ -2,6 +2,7 @@ import { FFmpegLive } from '@peertube/peertube-ffmpeg'import { getFFmpegCommandWrapperOptions } from '@server/helpers/ffmpeg/index.js'import { logger } from '@server/helpers/logger.js'import { CONFIG } from '@server/initializers/config.js'+import { VIDEO_LIVE } from '@server/initializers/constants.js'import { VideoTranscodingProfilesManager } from '@server/lib/transcoding/default-transcoding-profiles.js'import { FfmpegCommand } from 'fluent-ffmpeg'import { getLiveSegmentTime } from '../../live-utils.js'@@ -14,8 +15,10 @@ export class FFmpegTranscodingWrapper extends AbstractTranscodingWrapper {private errored = falseprivate ended = false+ private exitTimeout: NodeJS.Timeout+async run () {- this.ffmpegCommand = CONFIG.LIVE.TRANSCODING.ENABLED+ const ffmpegCommand = CONFIG.LIVE.TRANSCODING.ENABLED? await this.buildFFmpegLive().getLiveTranscodingCommand({inputUrl: this.inputLocalUrl,@@ -46,6 +49,14 @@ export class FFmpegTranscodingWrapper extends AbstractTranscodingWrapper {segmentDuration: getLiveSegmentTime(this.videoLive.latencyMode)})+ // abort() may have been called while we were building the command: don't spawn an ffmpeg process nobody watches+ if (this.aborted || this.ended || this.errored) {+ logger.debug('Live transcoding of %s was aborted before ffmpeg started.', this.videoUUID, this.lTags())+ return+ }++ this.ffmpegCommand = ffmpegCommand+logger.info('Running local live muxing/transcoding for %s.', this.videoUUID, this.lTags())let ffmpegShellCommand: string@@ -69,14 +80,30 @@ export class FFmpegTranscodingWrapper extends AbstractTranscodingWrapper {abort () {if (this.ended || this.errored || this.aborted) return- logger.debug('Killing ffmpeg after live abort of ' + this.videoUUID, this.lTags())+ this.aborted = true- if (this.ffmpegCommand) {- this.ffmpegCommand.kill('SIGINT')+ // ffmpeg was never started: there is nothing to wait for+ if (!this.ffmpegCommand) {+ this.emitEnded()+ return}- this.aborted = true- this.emit('end')+ logger.debug('Killing ffmpeg after live abort of ' + this.videoUUID, this.lTags())++ this.ffmpegCommand.kill('SIGINT')++ // Don't emit 'end' yet: on SIGINT ffmpeg still has to write its last segments and update the playlists+ // We wait for the process to actually exit so listeners know the live directory is not written anymore+ this.exitTimeout = setTimeout(() => {+ logger.warn('FFmpeg did not exit %d ms after SIGINT of %s, killing it.', VIDEO_LIVE.FFMPEG_EXIT_TIMEOUT, this.videoUUID, this.lTags())++ this.ffmpegCommand.kill('SIGKILL')+ this.emitEnded()+ }, VIDEO_LIVE.FFMPEG_EXIT_TIMEOUT)+ }++ destroy () {+ clearTimeout(this.exitTimeout)}private onFFmpegError (options: {@@ -87,9 +114,20 @@ export class FFmpegTranscodingWrapper extends AbstractTranscodingWrapper {}) {const { err, stdout, stderr, ffmpegShellCommand } = options+ // We killed ffmpeg ourselves: it exits in error but has flushed its last segments, so this is a normal end for us+ if (this.aborted) {+ logger.debug(+ 'Ignoring ffmpeg error of %s because we aborted the live.',+ this.videoUUID,+ { err, stdout, stderr, ffmpegShellCommand, ...this.lTags() }+ )++ return this.emitEnded()+ }+// Don't care that we killed the ffmpeg processif (err?.message?.includes('Exiting normally')) return- if (this.ended || this.errored || this.aborted) return+ if (this.ended || this.errored) returnlogger.error('FFmpeg transcoding error.', { err, stdout, stderr, ffmpegShellCommand, ...this.lTags() })@@ -98,11 +136,19 @@ export class FFmpegTranscodingWrapper extends AbstractTranscodingWrapper {}private onFFmpegEnded () {- if (this.ended || this.errored || this.aborted) return+ if (this.errored) returnlogger.debug('Live ffmpeg transcoding ended for ' + this.videoUUID, this.lTags())+ this.emitEnded()+ }++ private emitEnded () {+ if (this.ended) returnthis.ended = true++ clearTimeout(this.exitTimeout)+this.emit('end')}diff --git a/server/core/lib/live/shared/transcoding-wrapper/remote-transcoding-wrapper.ts b/server/core/lib/live/shared/transcoding-wrapper/remote-transcoding-wrapper.tsindex 4e375a749f..1b77311f2f 100644--- a/server/core/lib/live/shared/transcoding-wrapper/remote-transcoding-wrapper.ts+++ b/server/core/lib/live/shared/transcoding-wrapper/remote-transcoding-wrapper.ts@@ -1,9 +1,16 @@+import { logger } from '@server/helpers/logger.js'+import { VIDEO_LIVE } from '@server/initializers/constants.js'import { LiveRTMPHLSTranscodingJobHandler } from '@server/lib/runners/index.js'+import { MRunnerJob } from '@server/types/models/runners/index.js'import { AbstractTranscodingWrapper } from './abstract-transcoding-wrapper.js'export class RemoteTranscodingWrapper extends AbstractTranscodingWrapper {+ private aborted = false++ private flushTimeout: NodeJS.Timeout+async run () {- await new LiveRTMPHLSTranscodingJobHandler().create({+ const runnerJob = await new LiveRTMPHLSTranscodingJobHandler().create({rtmpUrl: this.inputPublicUrl,sessionId: this.sessionId,toTranscode: this.toTranscode,@@ -13,9 +20,35 @@ export class RemoteTranscodingWrapper extends AbstractTranscodingWrapper {segmentListSize: this.segmentListSize,segmentDuration: this.segmentDuration})++ // abort() may have been called while we were creating the job+ if (this.aborted) {+ logger.debug('Cancelling remote live transcoding job of %s aborted before a runner processed it.', this.videoUUID, this.lTags())++ await this.cancelRunnerJob(runnerJob)+ }}abort () {- this.emit('end')+ if (this.aborted) return+ this.aborted = true++ logger.debug('Waiting for the remote runner of %s to flush its last chunks.', this.videoUUID, this.lTags())++ // The runner uploads its chunks with HTTP requests that the API writes in our output directory, independently of this wrapper+ // Add a delay to flush its last chunks before listeners consider the directory is not written anymore+ this.flushTimeout = setTimeout(() => this.emit('end'), VIDEO_LIVE.REMOTE_RUNNER_FLUSH_DELAY)+ }++ destroy () {+ clearTimeout(this.flushTimeout)+ }++ private async cancelRunnerJob (runnerJob: MRunnerJob) {+ try {+ await new LiveRTMPHLSTranscodingJobHandler().cancel({ runnerJob })+ } catch (err) {+ logger.error('Cannot cancel remote live transcoding job of %s.', this.videoUUID, { err, ...this.lTags() })+ }}}