Skip to content

ActivityPub-federated video streaming platform using P2P directly in your web browser

More robust concurrent live handler

Chocobozzzaff09e05f831
diff --git a/packages/tests/src/api/live/live.ts b/packages/tests/src/api/live/live.ts
index 5c49dfbd65..e4c9c97c1a 100644
--- a/packages/tests/src/api/live/live.ts
+++ b/packages/tests/src/api/live/live.ts
@@ -519,23 +519,27 @@ describe('Test live', function () {
await testFfmpegStreamError(command, true)
})
- it('Should succeed with the correct params', async function () {
+ it('Should succeed with the correct params and list the live', async function () {
this.timeout(60000)
const command = sendRTMPStream({ rtmpBaseUrl: rtmpUrl + '/live', streamKey: liveVideo.streamKey })
- await testFfmpegStreamError(command, false)
- })
- it('Should list this live now someone stream into it', async function () {
- for (const server of servers) {
- const { total, data } = await server.videos.list()
+ try {
+ await waitUntilLivePublishedOnAllServers(servers, liveVideo.uuid)
- expect(total).to.equal(1)
- expect(data).to.have.lengthOf(1)
+ for (const server of servers) {
+ const { total, data } = await server.videos.list()
- const video = data[0]
- expect(video.name).to.equal('user live')
- expect(video.isLive).to.be.true
+ expect(total).to.equal(1)
+ expect(data).to.have.lengthOf(1)
+
+ const video = data[0]
+ expect(video.name).to.equal('user live')
+ expect(video.isLive).to.be.true
+ }
+ } finally {
+ // Don't leak the ffmpeg process in the next tests if an assertion above failed
+ await stopFfmpeg(command)
}
})
diff --git a/server/core/initializers/constants.ts b/server/core/initializers/constants.ts
index 89735f7d8e..e4e6f12bb5 100644
--- a/server/core/initializers/constants.ts
+++ b/server/core/initializers/constants.ts
@@ -1176,6 +1176,17 @@ export const RESUMABLE_UPLOAD_SESSION_LIFETIME = SCHEDULER_INTERVALS_MS.REMOVE_D
export const VIDEO_LIVE = {
EXTENSION: '.ts',
CLEANUP_DELAY: 1000 * 60 * 5, // 5 minutes
+ // Delay before aborting a session on RTMP disconnection, so we kill ffmpeg even if it still has data to process
+ ABORT_DELAY_ON_RTMP_DISCONNECT: 2000, // 2 seconds
+ // Max time we wait for ffmpeg to exit after we sent it a SIGINT, before killing it
+ FFMPEG_EXIT_TIMEOUT: 10000, // 10 seconds
+ // Time we give a remote runner to flush its last chunks after we aborted the session
+ // If this delay is too short, the last chunks of this session can land in the directory *after* we told everyone we released it
+ REMOTE_RUNNER_FLUSH_DELAY: 5000, // 5 seconds
+ // Max time a new session waits for the previous session of a permanent live to release the live directory
+ PREVIOUS_SESSION_CLEANUP_TIMEOUT: 30000, // 30 seconds
+ // Max time we wait for ffmpeg to fill the live master playlist it just created before giving up on it
+ MASTER_PLAYLIST_READ_TIMEOUT: 5000, // 5 seconds
SEGMENT_TIME_SECONDS: {
DEFAULT_LATENCY: 4, // 4 seconds
SMALL_LATENCY: 2 // 2 seconds
diff --git a/server/core/lib/live/live-manager.ts b/server/core/lib/live/live-manager.ts
index 94a1a0f79e..f9e2cec727 100644
--- a/server/core/lib/live/live-manager.ts
+++ b/server/core/lib/live/live-manager.ts
@@ -1,4 +1,4 @@
-import { pick, wait } from '@peertube/peertube-core-utils'
+import { pick, timeoutPromise, wait } from '@peertube/peertube-core-utils'
import {
ffprobePromise,
getVideoStreamBitrate,
@@ -62,6 +62,10 @@ class LiveManager {
private readonly muxingSessions = new Map<string, MuxingSession>()
private readonly videoSessions = new Map<string, string>()
+ // Video UUID -> cleanup of its current/last muxing session
+ // A permanent live can be streamed again before the previous session finished to write its files on disk
+ private readonly sessionCleanups = new Map<string, { sessionId: string, cleanup: Promise<void> }>()
+
private rtmpServer: Server
private rtmpsServer: ServerTLS
@@ -94,7 +98,7 @@ class LiveManager {
logger.info('Live session ended.', { sessionId, ...lTags(sessionId) })
// Force session aborting, so we kill ffmpeg even if it still has data to process (slow CPU)
- setTimeout(() => this.abortSession(sessionId), 2000)
+ setTimeout(() => this.abortSession(sessionId), VIDEO_LIVE.ABORT_DELAY_ON_RTMP_DISCONNECT)
})
registerConfigChangedHandler(() => {
@@ -231,6 +235,13 @@ class LiveManager {
return context
}
+ // Don't remove the cleanup of a more recent session of the same video
+ private deleteSessionCleanup (videoUUID: string, sessionId: string) {
+ if (this.sessionCleanups.get(videoUUID)?.sessionId !== sessionId) return
+
+ this.sessionCleanups.delete(videoUUID)
+ }
+
private abortSession (sessionId: string) {
const session = this.getContext().sessions.get(sessionId)
if (session) {
@@ -300,6 +311,30 @@ class LiveManager {
return this.abortSession(sessionId)
}
+ // A previous session of this permanent live may still be writing its last segments on disk
+ // Wait for it, so we don't cleanup/write the same files from two sessions at the same time
+ const previous = this.sessionCleanups.get(video.uuid)
+ if (previous) {
+ logger.info(
+ 'Waiting for the previous live session %s of video %s to be cleaned up.',
+ previous.sessionId,
+ video.uuid,
+ lTags(sessionId, video.uuid)
+ )
+
+ // Never block this live forever if the previous cleanup is stuck (a hanging object storage upload for example)
+ try {
+ await timeoutPromise(previous.cleanup, VIDEO_LIVE.PREVIOUS_SESSION_CLEANUP_TIMEOUT)
+ } catch (err) {
+ logger.warn(
+ 'Previous live session %s of video %s is still not cleaned up, starting the new session anyway.',
+ previous.sessionId,
+ video.uuid,
+ { err, ...lTags(sessionId, video.uuid) }
+ )
+ }
+ }
+
// Cleanup old potential live (could happen with a permanent live)
const oldStreamingPlaylist = await VideoStreamingPlaylistModel.loadHLSByVideo(video.id)
if (oldStreamingPlaylist) {
@@ -463,6 +498,9 @@ class LiveManager {
])
})
+ // Track the cleanup from the session creation: a new session must not write in the live directory before this one flushed its files
+ this.sessionCleanups.set(videoUUID, { sessionId, cleanup: muxingSession.waitForCleanup() })
+
muxingSession.on('live-ready', () => this.publishAndFederateLive({ live: videoLive, ratio, audioOnlyOutput, localLTags }))
const safeStopSession = (error: LiveVideoErrorType) => {
@@ -493,7 +531,7 @@ class LiveManager {
safeStopSession(LiveVideoError.QUOTA_EXCEEDED)
})
- muxingSession.on('transcoding-error', ({ videoUUID }) => {
+ muxingSession.on('transcoding-error', () => {
safeStopSession(LiveVideoError.FFMPEG_ERROR)
})
@@ -503,6 +541,7 @@ class LiveManager {
muxingSession.on('after-cleanup', ({ videoUUID }) => {
this.muxingSessions.delete(sessionId)
+ this.deleteSessionCleanup(videoUUID, sessionId)
LiveQuotaStore.Instance.removeLive(user.id, sessionId)
@@ -519,9 +558,11 @@ class LiveManager {
logger.error('Cannot run muxing.', { err, ...localLTags })
this.muxingSessions.delete(sessionId)
+ this.deleteSessionCleanup(videoUUID, sessionId)
LiveQuotaStore.Instance.removeLive(user.id, sessionId)
+ // Resolves the cleanup promise, so a new session of this permanent live is not blocked
muxingSession.destroy()
this.stopSessionOfVideo({
@@ -574,8 +615,9 @@ class LiveManager {
}
private onMuxingFFmpegEnd (videoUUID: string, sessionId: string) {
- // Session already cleaned up
- if (!this.videoSessions.has(videoUUID)) return
+ // Session already cleaned up, or a more recent session of this permanent live already registered itself:
+ // the transcoding process of an ending session can exit long after the next one started
+ if (this.videoSessions.get(videoUUID) !== sessionId) return
this.videoSessions.delete(videoUUID)
diff --git a/server/core/lib/live/live-utils.ts b/server/core/lib/live/live-utils.ts
index 86c2948114..2df2939eba 100644
--- a/server/core/lib/live/live-utils.ts
+++ b/server/core/lib/live/live-utils.ts
@@ -4,6 +4,7 @@ import { logger } from '@server/helpers/logger.js'
import { CONFIG } from '@server/initializers/config.js'
import { VIDEO_LIVE } from '@server/initializers/constants.js'
import { MStreamingPlaylist, MStreamingPlaylistVideo, MVideo } from '@server/types/models/index.js'
+import Bluebird from 'bluebird'
import { pathExists, remove } from 'fs-extra/esm'
import { readdir, rmdir } from 'fs/promises'
import { basename, join } from 'path'
@@ -17,7 +18,7 @@ export function buildConcatenatedName (segmentOrPlaylistPath: string) {
}
export async function cleanupAndDestroyPermanentLive (video: MVideo, streamingPlaylist: MStreamingPlaylist) {
- await cleanupTMPLiveFiles(video, streamingPlaylist)
+ const cleaned = await cleanupTMPLiveFiles(video, streamingPlaylist)
if (video.state === VideoState.WAITING_FOR_LIVE) {
// Try to delete local filesystem empty paths
@@ -31,6 +32,15 @@ export async function cleanupAndDestroyPermanentLive (video: MVideo, streamingPl
}
}
+ // Keep the playlist instead, so the next session retries the cleanup before writing in the directory
+ if (!cleaned) {
+ logger.warn(
+ 'Not destroying the streaming playlist of %s: some live files could not be removed, keeping it so the cleanup is retried.',
+ video.uuid
+ )
+ return
+ }
+
await streamingPlaylist.destroy()
}
@@ -47,10 +57,11 @@ export async function cleanupUnsavedNormalLive (video: MVideo, streamingPlaylist
await streamingPlaylist.destroy()
}
+// Returns false if some files could not be removed
export async function cleanupTMPLiveFiles (video: MVideo, streamingPlaylist: MStreamingPlaylist) {
await cleanupTMPLiveFilesFromObjectStorage(streamingPlaylist.withVideo(video))
- await cleanupTMPLiveFilesFromFilesystem(video)
+ return cleanupTMPLiveFilesFromFilesystem(video)
}
export function getLiveSegmentTime (latencyMode: LiveVideoLatencyModeType) {
@@ -90,23 +101,32 @@ function isTMPLiveFile (name: string) {
name.endsWith('.tmp')
}
+// Returns false if some files could not be removed
async function cleanupTMPLiveFilesFromFilesystem (video: MVideo) {
const hlsDirectory = getLiveDirectory(video)
- if (!await pathExists(hlsDirectory)) return
+ if (!await pathExists(hlsDirectory)) return true
logger.info('Cleanup TMP live files from filesystem of %s.', hlsDirectory)
const files = await readdir(hlsDirectory)
- for (const filename of files) {
- if (isTMPLiveFile(filename)) {
- const p = join(hlsDirectory, filename)
+ let cleaned = true
- remove(p)
- .catch(err => logger.error('Cannot remove %s.', p, { err }))
- }
- }
+ // Await the removals: the caller can start a new live session in this directory just after
+ // A live directory can hold thousands of segments with a big DVR window, so don't remove them all at once
+ await Bluebird.map(files.filter(filename => isTMPLiveFile(filename)), filename => {
+ const p = join(hlsDirectory, filename)
+
+ return remove(p)
+ .catch(err => {
+ logger.error('Cannot remove %s.', p, { err })
+
+ cleaned = false
+ })
+ }, { concurrency: 10 })
+
+ return cleaned
}
async function cleanupTMPLiveFilesFromObjectStorage (streamingPlaylist: MStreamingPlaylistVideo) {
diff --git a/server/core/lib/live/shared/muxing-session.ts b/server/core/lib/live/shared/muxing-session.ts
index feed6ecbb8..a8825e40cf 100644
--- a/server/core/lib/live/shared/muxing-session.ts
+++ b/server/core/lib/live/shared/muxing-session.ts
@@ -8,7 +8,7 @@ import {
VideoStreamingPlaylistType
} from '@peertube/peertube-models'
import { computeOutputFPS } from '@server/helpers/ffmpeg/index.js'
-import { LoggerTagsFn, logger, loggerTagsFactory } from '@server/helpers/logger.js'
+import { logger, loggerTagsFactory, LoggerTagsFn } from '@server/helpers/logger.js'
import { CONFIG } from '@server/initializers/config.js'
import { MEMOIZE_TTL, P2P_MEDIA_LOADER_PEER_VERSION, VIDEO_LIVE } from '@server/initializers/constants.js'
import { removeHLSFileObjectStorageByPath, storeHLSFileFromContent, storeHLSFileFromPath } from '@server/lib/object-storage/index.js'
@@ -20,7 +20,7 @@ import { FSWatcher, watch } from 'chokidar'
import { EventEmitter } from 'events'
import { FfprobeData } from 'fluent-ffmpeg'
import { ensureDir } from 'fs-extra/esm'
-import { appendFile, readFile, stat } from 'fs/promises'
+import { appendFile, readdir, readFile, stat } from 'fs/promises'
import memoizee from 'memoizee'
import PQueue from 'p-queue'
import { basename, join } from 'path'
@@ -100,6 +100,23 @@ class MuxingSession extends EventEmitter implements MuxingSession {
private segmentsToProcessPerPlaylist: { [playlistId: string]: string[] } = {}
+ // A segment can be seen twice: by the files watcher and by the cleanup task
+ private readonly processedSegments = new Set<string>()
+
+ // Playlist ID -> Queue
+ // Segments of a same playlist are processed in order, and the cleanup can wait for the pending ones
+ private readonly segmentProcessingQueues = new Map<string, PQueue>()
+
+ // Closing the files watcher stops new events but not the async handlers it already started, that can still write
+ // in the live directory (or in object storage): the cleanup waits for them
+ private readonly pendingWatcherHandlers = new Set<Promise<void>>()
+
+ private cleanupResolve: () => void
+ // Settled by runCleanup(), and by destroy() as a safety net so waitForCleanup() callers can never hang
+ private readonly cleanupPromise = new Promise<void>(resolve => {
+ this.cleanupResolve = resolve
+ })
+
private streamingPlaylist: MStreamingPlaylistVideo
private liveSegmentShaStore: LiveSegmentShaStore
@@ -181,13 +198,10 @@ class MuxingSession extends EventEmitter implements MuxingSession {
await this.prepareDirectories()
- this.transcodingWrapper = this.buildTranscodingWrapper(toTranscode)
-
- this.transcodingWrapper.on('end', () => this.onTranscodedEnded())
- this.transcodingWrapper.on('error', () => this.onTranscodingError())
-
- await this.transcodingWrapper.run()
-
+ // Watch the directory *before* running the transcoding process, so the watcher can never be created after the
+ // session was cleaned up and destroyed: run() can abort and emit 'end' before it returns, and the cleanup would
+ // then close a watcher that does not exist yet
+ // Chokidar also emits 'add' for the files it finds on init, so we don't miss anything by watching earlier
this.filesWatcher = watch(this.outDirectory, {
// Ignore 'segments-sha256.json' and 'segments-sha256.json.tmp' files that are frequently updated and not useful
ignored: path => path.endsWith('.json') || path.endsWith('json.tmp'),
@@ -196,13 +210,36 @@ class MuxingSession extends EventEmitter implements MuxingSession {
this.watchMasterFile()
this.watchTSFiles()
+
+ this.transcodingWrapper = this.buildTranscodingWrapper(toTranscode)
+
+ this.transcodingWrapper.on('end', () => this.onTranscodedEnded())
+ this.transcodingWrapper.on('error', () => this.onTranscodingError())
+
+ // abort() was called before the wrapper existed
+ // Abort the wrapper anyway, so it emits 'end' and the cleanup of this session is scheduled
+ if (this.aborted) {
+ logger.debug('Live muxing of %s was aborted before the transcoding process started.', this.videoUUID, this.lTags())
+
+ this.transcodingWrapper.abort()
+ return
+ }
+
+ await this.transcodingWrapper.run()
}
abort () {
- if (!this.transcodingWrapper) return
-
+ if (this.aborted) return
this.aborted = true
- this.transcodingWrapper.abort()
+
+ // The wrapper may not exist yet: runMuxing() checks this flag before running it
+ this.transcodingWrapper?.abort()
+ }
+
+ // Resolves when this session does not write in the live directory anymore
+ // The transcoding process has exited and all its segments have been hashed/stored
+ waitForCleanup () {
+ return this.cleanupPromise
}
destroy () {
@@ -211,6 +248,13 @@ class MuxingSession extends EventEmitter implements MuxingSession {
this.closeWatcher()
.catch(err => logger.error('Cannot close files watcher of %s.', this.outDirectory, { err, ...this.lTags() }))
+ // We removed its listeners below, so don't leave a pending timer that would emit an event nobody handles
+ this.transcodingWrapper?.destroy()
+
+ // Safety net: this session is over, so unblock everyone waiting for its cleanup even if it was never scheduled
+ // (e.g. runMuxing() threw before the transcoding wrapper was able to run)
+ this.cleanupResolve()
+
this.removeAllListeners()
this.isAbleToUploadVideoWithCache.clear()
this.hasClientSocketInBadHealthWithCache.clear()
@@ -225,22 +269,27 @@ class MuxingSession extends EventEmitter implements MuxingSession {
return watcher.close()
}
+ // Watcher handlers are async and fire and forget: remember them so the cleanup can wait for the pending ones
+ private trackWatcherHandler (handler: Promise<void>) {
+ const tracked: Promise<void> = handler
+ .catch(err => {
+ logger.error('Error in live files watcher handler of %s.', this.outDirectory, { err, ...this.lTags() })
+ })
+ .finally(() => {
+ this.pendingWatcherHandlers.delete(tracked)
+ })
+
+ this.pendingWatcherHandlers.add(tracked)
+ }
+
private watchMasterFile () {
- this.filesWatcher.on('add', async path => {
+ const addHandler = async (path: string) => {
if (path !== join(this.outDirectory, this.streamingPlaylist.playlistFilename)) return
if (this.masterPlaylistCreated === true) return
try {
if (this.streamingPlaylist.storage === FileStorage.OBJECT_STORAGE) {
- let masterContent = await readFile(path, 'utf-8')
-
- // If the disk sync is slow, don't upload an empty master playlist on object storage
- // Wait for ffmpeg to correctly fill it
- while (!masterContent) {
- await wait(100)
-
- masterContent = await readFile(path, 'utf-8')
- }
+ const masterContent = await this.readNonEmptyMasterPlaylist(path)
logger.debug('Uploading live master playlist on object storage for %s', this.videoUUID, { masterContent, ...this.lTags() })
@@ -266,12 +315,38 @@ class MuxingSession extends EventEmitter implements MuxingSession {
await this.streamingPlaylist.save()
} catch (err) {
logger.error('Cannot update streaming playlist.', { err, ...this.lTags() })
+
+ // Don't set masterPlaylistCreated: without a stored master playlist the live would be published but unplayable
+ // Stop the session instead, so we don't federate a broken live
+ this.stopBrokenSession()
+ return
}
this.masterPlaylistCreated = true
logger.info('Master playlist file for %s has been created', this.videoUUID, this.lTags())
- })
+ }
+
+ this.filesWatcher.on('add', path => this.trackWatcherHandler(addHandler(path)))
+ }
+
+ // Throws if ffmpeg did not fill the master playlist in time: we don't want to store an empty one
+ private async readNonEmptyMasterPlaylist (path: string) {
+ const deadline = Date.now() + VIDEO_LIVE.MASTER_PLAYLIST_READ_TIMEOUT
+
+ do {
+ const content = await readFile(path, 'utf-8')
+ if (content) return content
+
+ // The session is over: bail out instead of making the cleanup wait for us
+ if (this.cleanupScheduled) break
+
+ // If the disk sync is slow, don't upload an empty master playlist on object storage
+ // Wait for ffmpeg to correctly fill it
+ await wait(100)
+ } while (Date.now() < deadline)
+
+ throw new Error(`Live master playlist ${path} is still empty after ${VIDEO_LIVE.MASTER_PLAYLIST_READ_TIMEOUT} ms`)
}
private watchTSFiles () {
@@ -283,9 +358,14 @@ class MuxingSession extends EventEmitter implements MuxingSession {
logger.debug('Live add handler of TS file %s.', segmentPath, this.lTags())
const playlistId = this.getPlaylistIdFromTS(segmentPath)
+ if (!playlistId) {
+ logger.warn('Cannot get the playlist id of live segment %s, ignoring it.', segmentPath, this.lTags())
+ return
+ }
const segmentsToProcess = this.segmentsToProcessPerPlaylist[playlistId] || []
- this.processSegments(segmentsToProcess)
+ // The cleanup waits for the queue, so we don't need to await it here
+ void this.processSegments(playlistId, segmentsToProcess)
this.segmentsToProcessPerPlaylist[playlistId] = [ segmentPath ]
@@ -302,6 +382,9 @@ class MuxingSession extends EventEmitter implements MuxingSession {
logger.debug('Live delete handler of TS file %s.', segmentPath, this.lTags())
+ // The segment does not exist anymore so the cleanup won't list it: don't track it forever
+ this.processedSegments.delete(segmentPath)
+
this.liveSegmentShaStore.removeSegmentSha(segmentPath)
if (this.streamingPlaylist.storage === FileStorage.OBJECT_STORAGE) {
@@ -314,7 +397,7 @@ class MuxingSession extends EventEmitter implements MuxingSession {
}
this.filesWatcher.on('add', p => addHandler(p))
- this.filesWatcher.on('unlink', p => deleteHandler(p))
+ this.filesWatcher.on('unlink', p => this.trackWatcherHandler(deleteHandler(p)))
}
private async isQuotaExceeded (segmentPath: string) {
@@ -373,8 +456,22 @@ class MuxingSession extends EventEmitter implements MuxingSession {
return now <= max
}
- private processSegments (segmentPaths: string[]) {
- Bluebird.mapSeries(segmentPaths, previousSegment => this.processSegment(previousSegment))
+ // All the segments must belong to the playlist of `playlistId`
+ private processSegments (playlistId: string, segmentPaths: string[]) {
+ if (segmentPaths.length === 0) return Promise.resolve()
+
+ return this.getSegmentProcessingQueue(playlistId)
+ .add(() =>
+ // Catch per segment: mapSeries would abandon the next ones, and the cleanup is the last chance to process them
+ Bluebird.mapSeries(segmentPaths, segmentPath => {
+ return this.processSegment(segmentPath)
+ .catch(err => {
+ if (this.aborted) return
+
+ logger.error('Cannot process segment %s.', segmentPath, { err, ...this.lTags() })
+ })
+ })
+ )
.catch(err => {
if (this.aborted) return
@@ -382,28 +479,52 @@ class MuxingSession extends EventEmitter implements MuxingSession {
})
}
- private async processSegment (segmentPath: string) {
- // Check user quota if the user enabled replay saving
- if (await this.isQuotaExceeded(segmentPath) === true) {
- this.emit('quota-exceeded', { videoUUID: this.videoUUID })
- return
+ private getSegmentProcessingQueue (playlistId: string) {
+ if (!this.segmentProcessingQueues.has(playlistId)) {
+ this.segmentProcessingQueues.set(playlistId, new PQueue({ concurrency: 1 }))
}
- // Add sha hash of previous segments, because ffmpeg should have finished generating them
- await this.liveSegmentShaStore.addSegmentSha(segmentPath)
+ return this.segmentProcessingQueues.get(playlistId)
+ }
- if (this.saveReplay) {
- await this.addSegmentToReplay(segmentPath)
- }
+ private async processSegment (segmentPath: string) {
+ // Already processed by the files watcher or by the cleanup
+ if (this.processedSegments.has(segmentPath)) return
+ // Mark it before processing it so the cleanup doesn't process it again while we're working on it
+ this.processedSegments.add(segmentPath)
- if (this.streamingPlaylist.storage === FileStorage.OBJECT_STORAGE) {
- try {
- await storeHLSFileFromPath(this.streamingPlaylist.Video, segmentPath)
+ // The replay is an append only file: retrying a segment we already appended would duplicate it
+ let appendedToReplay = false
- await this.processM3U8ToObjectStorage(segmentPath)
- } catch (err) {
- logger.error('Cannot store TS segment %s in object storage', segmentPath, { err, ...this.lTags() })
+ try {
+ // Check user quota if the user enabled replay saving
+ if (await this.isQuotaExceeded(segmentPath) === true) {
+ this.emit('quota-exceeded', { videoUUID: this.videoUUID })
+ return
+ }
+
+ // Add sha hash of previous segments, because ffmpeg should have finished generating them
+ await this.liveSegmentShaStore.addSegmentSha(segmentPath)
+
+ if (this.saveReplay) {
+ await this.addSegmentToReplay(segmentPath)
+ appendedToReplay = true
}
+
+ if (this.streamingPlaylist.storage === FileStorage.OBJECT_STORAGE) {
+ try {
+ await storeHLSFileFromPath(this.streamingPlaylist.Video, segmentPath)
+
+ await this.processM3U8ToObjectStorage(segmentPath)
+ } catch (err) {
+ logger.error('Cannot store TS segment %s in object storage', segmentPath, { err, ...this.lTags() })
+ }
+ }
+ } catch (err) {
+ // Forget the segment so the cleanup can retry it, unless retrying it would duplicate it in the replay
+ if (!appendedToReplay) this.processedSegments.delete(segmentPath)
+
+ throw err
}
// Master playlist and segment JSON file are created, live is ready
@@ -444,20 +565,28 @@ class MuxingSession extends EventEmitter implements MuxingSession {
}
private onTranscodingError () {
- this.emit('transcoding-error', { videoUUID: this.videoUUID })
-
// On ffmpeg error the transcoding wrapper abort() short-circuits and never emits 'end'
// So schedule the cleanup here too
+ // Schedule it before emitting so listeners can wait for the cleanup to complete
this.scheduleCleanup()
+
+ this.emit('transcoding-error', { videoUUID: this.videoUUID })
}
- private onTranscodedEnded () {
- this.emit('transcoding-end', { videoUUID: this.videoUUID })
+ // The live is broken but the transcoding process is still running and writing files
+ // Contrary to onTranscodingError() don't schedule the cleanup here
+ private stopBrokenSession () {
+ this.emit('transcoding-error', { videoUUID: this.videoUUID })
+ }
+ private onTranscodedEnded () {
// Don't log the input URL, which contains the stream key (a long lived secret)
logger.info('RTMP transmuxing for video %s ended. Scheduling cleanup', this.videoUUID, this.lTags())
+ // Schedule it before emitting so listeners can wait for the cleanup to complete
this.scheduleCleanup()
+
+ this.emit('transcoding-end', { videoUUID: this.videoUUID })
}
private scheduleCleanup () {
@@ -465,25 +594,68 @@ class MuxingSession extends EventEmitter implements MuxingSession {
if (this.cleanupScheduled) return
this.cleanupScheduled = true
- setTimeout(() => {
- // Wait latest segments generation, and close watchers
- this.closeWatcher()
- .then(() => {
- // Process remaining segments hash
- for (const key of Object.keys(this.segmentsToProcessPerPlaylist)) {
- this.processSegments(this.segmentsToProcessPerPlaylist[key])
- }
- })
- .catch(err => {
- logger.error(
- 'Cannot close watchers of %s or process remaining hash segments.',
- this.outDirectory,
- { err, ...this.lTags() }
- )
- })
+ this.runCleanup()
+ .catch(err => logger.error('Cannot run cleanup of %s.', this.outDirectory, { err, ...this.lTags() }))
+ }
+
+ private async runCleanup () {
+ try {
+ // The transcoding process exited so no new segment will be generated: we can close the watcher
+ await this.closeWatcher()
+
+ // Closing the watcher does not wait for the handlers it already started, that can still write files
+ await Promise.all(this.pendingWatcherHandlers)
+
+ // Wait for the segments the watcher already notified us about
+ await Promise.all(Array.from(this.segmentProcessingQueues.values(), queue => queue.onIdle()))
+
+ // Watcher events are asynchronous, so the last segments generated by the transcoding process may never have been
+ // notified to us: list the output directory instead of relying on the events we received
+ await this.processRemainingSegments()
+ } catch (err) {
+ logger.error(
+ 'Cannot close watchers of %s or process remaining hash segments.',
+ this.outDirectory,
+ { err, ...this.lTags() }
+ )
+ }
+
+ this.cleanupResolve()
+
+ this.emit('after-cleanup', { videoUUID: this.videoUUID })
+ }
+
+ private async processRemainingSegments () {
+ const filenames = await readdir(this.outDirectory)
+
+ const segmentPaths = filenames
+ // Segments are generated with a zero padded counter (%v-%06d.ts) so sorting them by name keeps the live order
+ .filter(f => f.endsWith(VIDEO_LIVE.EXTENSION))
+ .sort()
+ .map(f => join(this.outDirectory, f))
+ .filter(p => !this.processedSegments.has(p))
+
+ let remainingCount = 0
+ const perPlaylist = new Map<string, string[]>()
+
+ for (const segmentPath of segmentPaths) {
+ const playlistId = this.getPlaylistIdFromTS(segmentPath)
+ // Not a segment generated by our transcoding process
+ if (!playlistId) continue
+
+ if (!perPlaylist.has(playlistId)) perPlaylist.set(playlistId, [])
+ perPlaylist.get(playlistId).push(segmentPath)
+
+ remainingCount++
+ }
+
+ if (remainingCount === 0) return
+
+ logger.debug('Processing %d remaining live segments of %s.', remainingCount, this.videoUUID, this.lTags())
- this.emit('after-cleanup', { videoUUID: this.videoUUID })
- }, 1000)
+ await Promise.all(
+ Array.from(perPlaylist, ([ playlistId, paths ]) => this.processSegments(playlistId, paths))
+ )
}
private hasClientSocketInBadHealth (sessionId: string) {
@@ -583,10 +755,11 @@ class MuxingSession extends EventEmitter implements MuxingSession {
: new FFmpegTranscodingWrapper(options)
}
+ // Returns undefined if the file was not generated by our transcoding process
private getPlaylistIdFromTS (segmentPath: string) {
- const playlistIdMatcher = /^([\d+])-/
+ const playlistIdMatcher = /^(\d+)-/
- return basename(segmentPath).match(playlistIdMatcher)[1]
+ return basename(segmentPath).match(playlistIdMatcher)?.[1]
}
private getPlaylistNameFromTS (segmentPath: string) {
diff --git a/server/core/lib/live/shared/transcoding-wrapper/abstract-transcoding-wrapper.ts b/server/core/lib/live/shared/transcoding-wrapper/abstract-transcoding-wrapper.ts
index 6f0579cb84..5ae8ae2b77 100644
--- a/server/core/lib/live/shared/transcoding-wrapper/abstract-transcoding-wrapper.ts
+++ b/server/core/lib/live/shared/transcoding-wrapper/abstract-transcoding-wrapper.ts
@@ -111,9 +111,11 @@ abstract class AbstractTranscodingWrapper extends EventEmitter {
abstract run (): Promise<void>
abstract abort (abortError?: LiveVideoErrorType): void
+
+ abstract destroy (): void
}
export {
- type AbstractTranscodingWrapperOptions,
- AbstractTranscodingWrapper
+ AbstractTranscodingWrapper,
+ type AbstractTranscodingWrapperOptions
}
diff --git a/server/core/lib/live/shared/transcoding-wrapper/ffmpeg-transcoding-wrapper.ts b/server/core/lib/live/shared/transcoding-wrapper/ffmpeg-transcoding-wrapper.ts
index b432153e2a..0d0373584c 100644
--- a/server/core/lib/live/shared/transcoding-wrapper/ffmpeg-transcoding-wrapper.ts
+++ b/server/core/lib/live/shared/transcoding-wrapper/ffmpeg-transcoding-wrapper.ts
@@ -2,6 +2,7 @@ import { FFmpegLive } from '@peertube/peertube-ffmpeg'
import { getFFmpegCommandWrapperOptions } from '@server/helpers/ffmpeg/index.js'
import { logger } from '@server/helpers/logger.js'
import { CONFIG } from '@server/initializers/config.js'
+import { VIDEO_LIVE } from '@server/initializers/constants.js'
import { VideoTranscodingProfilesManager } from '@server/lib/transcoding/default-transcoding-profiles.js'
import { FfmpegCommand } from 'fluent-ffmpeg'
import { getLiveSegmentTime } from '../../live-utils.js'
@@ -14,8 +15,10 @@ export class FFmpegTranscodingWrapper extends AbstractTranscodingWrapper {
private errored = false
private ended = false
+ private exitTimeout: NodeJS.Timeout
+
async run () {
- this.ffmpegCommand = CONFIG.LIVE.TRANSCODING.ENABLED
+ const ffmpegCommand = CONFIG.LIVE.TRANSCODING.ENABLED
? await this.buildFFmpegLive().getLiveTranscodingCommand({
inputUrl: this.inputLocalUrl,
@@ -46,6 +49,14 @@ export class FFmpegTranscodingWrapper extends AbstractTranscodingWrapper {
segmentDuration: getLiveSegmentTime(this.videoLive.latencyMode)
})
+ // abort() may have been called while we were building the command: don't spawn an ffmpeg process nobody watches
+ if (this.aborted || this.ended || this.errored) {
+ logger.debug('Live transcoding of %s was aborted before ffmpeg started.', this.videoUUID, this.lTags())
+ return
+ }
+
+ this.ffmpegCommand = ffmpegCommand
+
logger.info('Running local live muxing/transcoding for %s.', this.videoUUID, this.lTags())
let ffmpegShellCommand: string
@@ -69,14 +80,30 @@ export class FFmpegTranscodingWrapper extends AbstractTranscodingWrapper {
abort () {
if (this.ended || this.errored || this.aborted) return
- logger.debug('Killing ffmpeg after live abort of ' + this.videoUUID, this.lTags())
+ this.aborted = true
- if (this.ffmpegCommand) {
- this.ffmpegCommand.kill('SIGINT')
+ // ffmpeg was never started: there is nothing to wait for
+ if (!this.ffmpegCommand) {
+ this.emitEnded()
+ return
}
- this.aborted = true
- this.emit('end')
+ logger.debug('Killing ffmpeg after live abort of ' + this.videoUUID, this.lTags())
+
+ this.ffmpegCommand.kill('SIGINT')
+
+ // Don't emit 'end' yet: on SIGINT ffmpeg still has to write its last segments and update the playlists
+ // We wait for the process to actually exit so listeners know the live directory is not written anymore
+ this.exitTimeout = setTimeout(() => {
+ logger.warn('FFmpeg did not exit %d ms after SIGINT of %s, killing it.', VIDEO_LIVE.FFMPEG_EXIT_TIMEOUT, this.videoUUID, this.lTags())
+
+ this.ffmpegCommand.kill('SIGKILL')
+ this.emitEnded()
+ }, VIDEO_LIVE.FFMPEG_EXIT_TIMEOUT)
+ }
+
+ destroy () {
+ clearTimeout(this.exitTimeout)
}
private onFFmpegError (options: {
@@ -87,9 +114,20 @@ export class FFmpegTranscodingWrapper extends AbstractTranscodingWrapper {
}) {
const { err, stdout, stderr, ffmpegShellCommand } = options
+ // We killed ffmpeg ourselves: it exits in error but has flushed its last segments, so this is a normal end for us
+ if (this.aborted) {
+ logger.debug(
+ 'Ignoring ffmpeg error of %s because we aborted the live.',
+ this.videoUUID,
+ { err, stdout, stderr, ffmpegShellCommand, ...this.lTags() }
+ )
+
+ return this.emitEnded()
+ }
+
// Don't care that we killed the ffmpeg process
if (err?.message?.includes('Exiting normally')) return
- if (this.ended || this.errored || this.aborted) return
+ if (this.ended || this.errored) return
logger.error('FFmpeg transcoding error.', { err, stdout, stderr, ffmpegShellCommand, ...this.lTags() })
@@ -98,11 +136,19 @@ export class FFmpegTranscodingWrapper extends AbstractTranscodingWrapper {
}
private onFFmpegEnded () {
- if (this.ended || this.errored || this.aborted) return
+ if (this.errored) return
logger.debug('Live ffmpeg transcoding ended for ' + this.videoUUID, this.lTags())
+ this.emitEnded()
+ }
+
+ private emitEnded () {
+ if (this.ended) return
this.ended = true
+
+ clearTimeout(this.exitTimeout)
+
this.emit('end')
}
diff --git a/server/core/lib/live/shared/transcoding-wrapper/remote-transcoding-wrapper.ts b/server/core/lib/live/shared/transcoding-wrapper/remote-transcoding-wrapper.ts
index 4e375a749f..1b77311f2f 100644
--- a/server/core/lib/live/shared/transcoding-wrapper/remote-transcoding-wrapper.ts
+++ b/server/core/lib/live/shared/transcoding-wrapper/remote-transcoding-wrapper.ts
@@ -1,9 +1,16 @@
+import { logger } from '@server/helpers/logger.js'
+import { VIDEO_LIVE } from '@server/initializers/constants.js'
import { LiveRTMPHLSTranscodingJobHandler } from '@server/lib/runners/index.js'
+import { MRunnerJob } from '@server/types/models/runners/index.js'
import { AbstractTranscodingWrapper } from './abstract-transcoding-wrapper.js'
export class RemoteTranscodingWrapper extends AbstractTranscodingWrapper {
+ private aborted = false
+
+ private flushTimeout: NodeJS.Timeout
+
async run () {
- await new LiveRTMPHLSTranscodingJobHandler().create({
+ const runnerJob = await new LiveRTMPHLSTranscodingJobHandler().create({
rtmpUrl: this.inputPublicUrl,
sessionId: this.sessionId,
toTranscode: this.toTranscode,
@@ -13,9 +20,35 @@ export class RemoteTranscodingWrapper extends AbstractTranscodingWrapper {
segmentListSize: this.segmentListSize,
segmentDuration: this.segmentDuration
})
+
+ // abort() may have been called while we were creating the job
+ if (this.aborted) {
+ logger.debug('Cancelling remote live transcoding job of %s aborted before a runner processed it.', this.videoUUID, this.lTags())
+
+ await this.cancelRunnerJob(runnerJob)
+ }
}
abort () {
- this.emit('end')
+ if (this.aborted) return
+ this.aborted = true
+
+ logger.debug('Waiting for the remote runner of %s to flush its last chunks.', this.videoUUID, this.lTags())
+
+ // The runner uploads its chunks with HTTP requests that the API writes in our output directory, independently of this wrapper
+ // Add a delay to flush its last chunks before listeners consider the directory is not written anymore
+ this.flushTimeout = setTimeout(() => this.emit('end'), VIDEO_LIVE.REMOTE_RUNNER_FLUSH_DELAY)
+ }
+
+ destroy () {
+ clearTimeout(this.flushTimeout)
+ }
+
+ private async cancelRunnerJob (runnerJob: MRunnerJob) {
+ try {
+ await new LiveRTMPHLSTranscodingJobHandler().cancel({ runnerJob })
+ } catch (err) {
+ logger.error('Cannot cancel remote live transcoding job of %s.', this.videoUUID, { err, ...this.lTags() })
+ }
}
}