ActivityPub-federated video streaming platform using P2P directly in your web browser
Move away chokidar for files watching
Chocobozzz80a1c0b7ff4f
diff --git a/apps/peertube-runner/src/server/process/shared/process-live.ts b/apps/peertube-runner/src/server/process/shared/process-live.tsindex 3b054b2cb5..67c6dd102f 100644--- a/apps/peertube-runner/src/server/process/shared/process-live.ts+++ b/apps/peertube-runner/src/server/process/shared/process-live.ts@@ -1,4 +1,11 @@-import { wait } from '@peertube/peertube-core-utils'+import {+ compareLiveFilenames,+ getLivePlaylistIdFromSegmentPath,+ getLivePlaylistNameFromSegmentPath,+ isLivePlaylistPath,+ isLiveSegmentPath,+ wait+} from '@peertube/peertube-core-utils'import {ffprobePromise,getVideoStreamBitrate,@@ -13,8 +20,7 @@ import {RunnerJobLiveRTMPHLSTranscodingPayload,ServerErrorCode} from '@peertube/peertube-models'-import { buildUUID } from '@peertube/peertube-node-utils'-import { FSWatcher, watch } from 'chokidar'+import { buildUUID, DirectoryWatcher } from '@peertube/peertube-node-utils'import { FfmpegCommand } from 'fluent-ffmpeg'import { ensureDir, remove } from 'fs-extra/esm'import { readFile } from 'fs/promises'@@ -29,11 +35,15 @@ type CustomLiveRTMPHLSTranscodingUpdatePayload = Omit<LiveRTMPHLSTranscodingUpdaexport class ProcessLiveRTMPHLSTranscoding {private readonly outputPath: string- private readonly fsWatchers: FSWatcher[] = []+ private fsWatcher: DirectoryWatcher- // Playlist name -> chunks+ // Playlist ID -> chunksprivate readonly pendingChunksPerPlaylist = new Map<string, string[]>()+ // Closing the files watcher stops new events but not the async handlers it already started, that can still read+ // the chunks and the playlists of the output directory: the cleanup waits for them+ private readonly pendingWatcherHandlers = new Set<Promise<void>>()+private readonly playlistsCreated = new Set<string>()private allPlaylistsCreated = false@@ -68,45 +78,63 @@ export class ProcessLiveRTMPHLSTranscoding {const bitrate = await getVideoStreamBitrate(this.payload.input.rtmpUrl, probe)const { ratio } = await getVideoStreamDimensionsInfo(this.payload.input.rtmpUrl, probe)- const m3u8Watcher = watch(this.outputPath, { ignored: path => path !== this.outputPath && !path.endsWith('.m3u8') })- this.fsWatchers.push(m3u8Watcher)-- const tsWatcher = watch(this.outputPath, { ignored: path => path !== this.outputPath && !path.endsWith('.ts') })- this.fsWatchers.push(tsWatcher)-- m3u8Watcher.on('change', p => {- logger.debug(`${p} m3u8 playlist changed`)- })-- m3u8Watcher.on('add', p => {+ const onPlaylistAdded = (p: string) => {this.playlistsCreated.add(p)if (this.playlistsCreated.size === this.options.job.payload.output.toTranscode.length + 1) {this.allPlaylistsCreated = truelogger.info('All m3u8 playlists are created.')}- })+ }- tsWatcher.on('add', async p => {+ const onChunkAdded = async (p: string) => {try {await this.sendPendingChunks()} catch (err) {this.onUpdateError({ err, rej, res })}- const playlistName = this.getPlaylistIdFromTS(p)+ const playlistId = getLivePlaylistIdFromSegmentPath(p)+ if (!playlistId) {+ logger.warn(`Cannot get the playlist id of live chunk ${p}, ignoring it`)+ return+ }- const pendingChunks = this.pendingChunksPerPlaylist.get(playlistName) || []+ const pendingChunks = this.pendingChunksPerPlaylist.get(playlistId) || []pendingChunks.push(p)- this.pendingChunksPerPlaylist.set(playlistName, pendingChunks)+ this.pendingChunksPerPlaylist.set(playlistId, pendingChunks)+ }++ this.fsWatcher = new DirectoryWatcher({+ directory: this.outputPath,+ filter: filename => isLiveSegmentPath(filename) || isLivePlaylistPath(filename),+ // A listing has to rebuild the order in which ffmpeg generated the files+ sort: compareLiveFilenames})- tsWatcher.on('unlink', p => {- this.sendDeletedChunkUpdate(p)- .catch(err => this.onUpdateError({ err, rej, res }))+ this.fsWatcher.on('error', err => logger.error({ err }, `Error in watcher of ${this.outputPath}`))++ this.fsWatcher.on('add', p => {+ if (isLivePlaylistPath(p)) return onPlaylistAdded(p)++ this.trackWatcherHandler(+ onChunkAdded(p)+ .catch(err => logger.error({ err }, `Error in add handler of ${p}`))+ )})+ this.fsWatcher.on('unlink', p => {+ if (!isLiveSegmentPath(p)) return++ this.trackWatcherHandler(+ this.sendDeletedChunkUpdate(p)+ .catch(err => this.onUpdateError({ err, rej, res }))+ )+ })++ this.fsWatcher.watch()+this.ffmpegCommand = await buildFFmpegLive().getLiveTranscodingCommand({inputUrl: this.payload.input.rtmpUrl,@@ -229,6 +257,11 @@ export class ProcessLiveRTMPHLSTranscoding {await wait(1500)try {+ // ffmpeg exited so no new chunk will be generated, but the OS watcher may never have notified us of the last+ // ones: ask for a final listing, and wait for the handlers that queue them+ await this.fsWatcher?.flush()+ await Promise.all(this.pendingWatcherHandlers)+await this.sendPendingChunks()} catch (err) {logger.error(err, 'Cannot send latest chunks after ffmpeg ended')@@ -256,8 +289,8 @@ export class ProcessLiveRTMPHLSTranscoding {// ---------------------------------------------------------------------------- private sendDeletedChunkUpdate (deletedChunk: string): Promise<any> {- if (this.ended) return Promise.resolve()+ private async sendDeletedChunkUpdate (deletedChunk: string): Promise<any> {+ if (this.ended) returnlogger.debug(`Sending removed live chunk ${deletedChunk} update`)@@ -268,9 +301,10 @@ export class ProcessLiveRTMPHLSTranscoding {videoChunkFilename}- if (this.allPlaylistsCreated) {- const playlistName = this.getPlaylistName(videoChunkFilename)+ const playlistName = getLivePlaylistNameFromSegmentPath(videoChunkFilename)+ // We may never have been able to read the content of the playlist that references this chunk+ if (this.allPlaylistsCreated && this.latestFilteredPlaylistContent[playlistName]) {payload = {...payload,@@ -303,9 +337,9 @@ export class ProcessLiveRTMPHLSTranscoding {videoChunkFile: chunk}- if (this.allPlaylistsCreated) {- const playlistName = this.getPlaylistName(videoChunkFilename)+ const playlistName = getLivePlaylistNameFromSegmentPath(videoChunkFilename)+ if (this.allPlaylistsCreated && playlistName) {try {await this.updatePlaylistContent(playlistName, videoChunkFilename)@@ -359,16 +393,6 @@ export class ProcessLiveRTMPHLSTranscoding {}}- private getPlaylistName (videoChunkFilename: string) {- return `${videoChunkFilename.split('-')[0]}.m3u8`- }-- private getPlaylistIdFromTS (segmentPath: string) {- const playlistIdMatcher = /^([\d+])-/-- return basename(segmentPath).match(playlistIdMatcher)[1]- }-private async updatePlaylistContent (playlistName: string, latestChunkFilename: string) {const m3u8Path = join(this.outputPath, playlistName)let playlistContent = await readFile(m3u8Path, 'utf-8')@@ -396,12 +420,27 @@ export class ProcessLiveRTMPHLSTranscoding {private cleanup () {logger.debug(`Cleaning up job ${this.options.job.uuid}`)- for (const fsWatcher of this.fsWatchers) {- fsWatcher.close()- .catch(err => logger.error({ err }, 'Cannot close watcher'))- }-- remove(this.outputPath)+ this.removeOutputDirectory().catch(err => logger.error({ err }, `Cannot remove ${this.outputPath}`))}++ private async removeOutputDirectory () {+ // Wait for the watcher to stop reading the directory before removing it+ await this.fsWatcher?.close()++ await Promise.all(this.pendingWatcherHandlers)++ await remove(this.outputPath)+ }++ // Watcher handlers are async and fire and forget: remember them so the cleanup can wait for the pending ones+ private trackWatcherHandler (handler: Promise<void>) {+ const tracked: Promise<void> = handler+ .catch(err => logger.error({ err }, 'Error in live files watcher handler'))+ .finally(() => {+ this.pendingWatcherHandlers.delete(tracked)+ })++ this.pendingWatcherHandlers.add(tracked)+ }}diff --git a/package.json b/package.jsonindex 9f46475ac0..5a9807ebd6 100644--- a/package.json+++ b/package.json@@ -133,7 +133,6 @@"bluebird": "^3.7.2","bullmq": "^6.0.11","bytes": "^3.1.2",- "chokidar": "^5.0.0","commander": "^15.0.0","config": "^5.0.0","content-disposition": "^2.0.1",diff --git a/packages/core-utils/src/videos/index.ts b/packages/core-utils/src/videos/index.tsindex 8b936c7463..79beba44b2 100644--- a/packages/core-utils/src/videos/index.ts+++ b/packages/core-utils/src/videos/index.ts@@ -1,4 +1,5 @@export * from './bitrate.js'export * from './common.js'+export * from './live.js'export * from './p2p.js'export * from './state.js'diff --git a/packages/core-utils/src/videos/live.ts b/packages/core-utils/src/videos/live.tsnew file mode 100644index 0000000000..a4ab5c3ee4--- /dev/null+++ b/packages/core-utils/src/videos/live.ts@@ -0,0 +1,71 @@+// One definition of the files a live directory contains+// Shared by the server and runners++export const LIVE_SEGMENT_EXTENSION = '.ts'+export const LIVE_PLAYLIST_EXTENSION = '.m3u8'++export function isLiveSegmentPath (path: string) {+ return path.endsWith(LIVE_SEGMENT_EXTENSION)+}++export function isLivePlaylistPath (path: string) {+ return path.endsWith(LIVE_PLAYLIST_EXTENSION)+}++// Our transcoding process generates the segments of a live as `<playlist id>-<counter>.ts`+// Returns undefined if the file was not generated by our transcoding process+export function getLivePlaylistIdFromSegmentPath (segmentPath: string) {+ return parseLiveSegmentFilename(getFilename(segmentPath))?.playlistId+}++// Returns undefined if the file was not generated by our transcoding process+export function getLivePlaylistNameFromSegmentPath (segmentPath: string) {+ const playlistId = getLivePlaylistIdFromSegmentPath(segmentPath)+ if (playlistId === undefined) return undefined++ return playlistId + LIVE_PLAYLIST_EXTENSION+}++// Order the files of a live directory the way the transcoding process generates them:+// * the playlists first, because the handlers of a segment need the playlist that references it+// * then the segments of a same playlist by their counter: it is zero padded (%v-%06d.ts) but it grows past 6 digits+// after 999999 segments, so a lexicographic sort would put the newest segments of a permanent live first+export function compareLiveFilenames (a: string, b: string) {+ const aSegment = parseLiveSegmentFilename(a)+ const bSegment = parseLiveSegmentFilename(b)++ if (!aSegment || !bSegment) {+ if (aSegment) return 1+ if (bSegment) return -1++ return compareStrings(a, b)+ }++ if (aSegment.playlistId !== bSegment.playlistId) return compareStrings(aSegment.playlistId, bSegment.playlistId)++ return aSegment.counter - bSegment.counter+}++// ---------------------------------------------------------------------------+// Private+// ---------------------------------------------------------------------------++function parseLiveSegmentFilename (filename: string) {+ if (!isLiveSegmentPath(filename)) return undefined++ const matches = filename.match(/^(\d+)-(\d+)\.ts$/)+ if (!matches) return undefined++ return { playlistId: matches[1], counter: parseInt(matches[2], 10) }+}++function getFilename (path: string) {+ return path.split(/[/\\]/).pop()+}++function compareStrings (a: string, b: string) {+ if (a < b) return -1+ if (a > b) return 1++ return 0+}diff --git a/packages/node-utils/src/directory-watcher.ts b/packages/node-utils/src/directory-watcher.tsnew file mode 100644index 0000000000..712ceb67eb--- /dev/null+++ b/packages/node-utils/src/directory-watcher.ts@@ -0,0 +1,241 @@+import { FSWatcher, watch } from 'fs'+import { readdir, stat } from 'fs/promises'+import { join } from 'path'+import { TypedEventEmitter } from './typed-event-emitter.js'++// Delay between the full directory listings used to recover the events the OS watcher may have dropped+const RECONCILE_INTERVAL = 10000+// A directory the OS watcher keeps reporting is unlikely to have dropped an event, and listing it is expensive when it+// holds thousands of files: reconcile it less often, but still often enough to recover an event it dropped anyway+const BUSY_RECONCILE_INTERVAL = 60000++export interface DirectoryWatcherOptions {+ directory: string++ // Only the filenames accepted by this filter emit events+ filter: (filename: string) => boolean++ // The OS watcher notifies us of the files in the order they appeared+ // But a full directory listing has to rebuild that order (default: by filename)+ sort?: (a: string, b: string) => number+}++interface DirectoryWatcherEvents {+ 'add': (path: string) => void+ 'unlink': (path: string) => void++ 'error': (err: Error) => void+}++// Emit an 'add'/'unlink' event with the full path of the files of a directory that appear/disappear+class DirectoryWatcher extends TypedEventEmitter<DirectoryWatcherEvents> {+ private readonly directory: string+ private readonly filter: (filename: string) => boolean+ private readonly sort: (a: string, b: string) => number++ // Filenames we emitted an 'add' for, so we don't emit it twice and know which 'unlink' we have to emit+ private readonly knownFilenames = new Set<string>()++ // Filenames the OS watcher notified us about, waiting to be checked+ // A Set keeps them in event order, which the consumers rely on to process the files in the order they appeared+ private readonly pendingFilenames = new Set<string>()++ private watcher: FSWatcher++ private reconcileTimer: NodeJS.Timeout+ // The OS watcher did not tell us which file changed, or we may have missed events: list the whole directory+ private reconcileRequested = true+ private lastReconcileDate = 0+ private osEventSinceLastReconcile = false++ private processingPromise: Promise<void>+ private closePromise: Promise<void>+ private closed = false++ constructor (options: DirectoryWatcherOptions) {+ super()++ this.directory = options.directory+ this.filter = options.filter+ this.sort = options.sort+ }++ // Starts watching the directory, and emits an 'add' for the files it already contains+ watch () {+ if (this.watcher !== undefined || this.closed) return++ this.watcher = watch(this.directory, { persistent: true })++ this.watcher.on('change', (_type, filename) => {+ // Some platforms don't always provide the filename: we don't know what changed, so list the directory instead+ if (!filename) this.reconcileRequested = true+ else if (this.filter(filename.toString())) {+ this.pendingFilenames.add(filename.toString())+ this.osEventSinceLastReconcile = true+ } else return++ // Process already handles promise rejection+ void this.process()+ })++ this.watcher.on('error', err => this.emit('error', err))++ this.reconcileTimer = setInterval(() => {+ // The OS watcher is doing its job: don't list the whole directory on every tick, BUSY_RECONCILE_INTERVAL is+ // enough to recover the events it may have dropped anyway+ if (this.osEventSinceLastReconcile && Date.now() - this.lastReconcileDate < BUSY_RECONCILE_INTERVAL) return++ this.reconcileRequested = true+ void this.process()+ }, RECONCILE_INTERVAL)++ // The OS watcher already keeps the event loop alive: don't let a forgotten watcher hold the process+ this.reconcileTimer.unref()++ // Default reconcileRequested is true, so this lists the directory and emits an 'add' for the files it already contains+ // It runs *after* the OS watcher was installed, so we can't miss a file created in between+ void this.process()+ }++ // List the whole directory and emit the events of the files the OS watcher never notified us about+ // Resolves once every file we know about has been emitted, so callers can rely on having seen everything+ async flush () {+ this.reconcileRequested = true++ // Process until nothing is left to process+ while (!this.closed && (this.reconcileRequested || this.pendingFilenames.size !== 0)) {+ await this.process()+ }+ }++ // Resolves once the checks that were already running have finished, so the caller can touch the directory+ close () {+ if (this.closePromise !== undefined) return this.closePromise++ this.closed = true++ clearInterval(this.reconcileTimer)++ this.watcher?.close()+ this.watcher = undefined++ this.closePromise = this.processingPromise ?? Promise.resolve()++ return this.closePromise+ }++ // ---------------------------------------------------------------------------+ // Private+ // ---------------------------------------------------------------------------++ // Check the pending filenames one at a time, so we always emit the events of a directory in order+ private process () {+ if (this.processingPromise !== undefined) return this.processingPromise++ this.processingPromise = this.processPending()+ .catch(err => {+ this.emit('error', err)+ })+ .finally(() => {+ this.processingPromise = undefined+ })++ return this.processingPromise+ }++ private async processPending () {+ while (!this.closed && (this.reconcileRequested || this.pendingFilenames.size !== 0)) {+ if (this.reconcileRequested) {+ this.reconcileRequested = false++ await this.runStep(() => this.reconcile())+ continue+ }++ const filename = this.pendingFilenames.values().next().value+ this.pendingFilenames.delete(filename)++ await this.runStep(() => this.checkFilename(filename))+ }+ }++ // A file we cannot stat, or a listing we cannot read, must not abandon the events we still have to emit:+ // report the error and carry on with the next filename+ private async runStep (step: () => Promise<void>) {+ try {+ await step()+ } catch (err) {+ this.emit('error', err)+ }+ }++ private async checkFilename (filename: string) {+ if (await this.exists(join(this.directory, filename))) this.emitAdd(filename)+ else this.emitUnlink(filename)+ }++ // Contrary to chokidar we don't stat the entries, so this stays cheap even on a directory that holds thousands of files+ private async reconcile () {+ this.lastReconcileDate = Date.now()+ this.osEventSinceLastReconcile = false++ let filenames: string[]++ try {+ filenames = (await readdir(this.directory)).filter(f => this.filter(f))+ } catch (err) {+ // The directory was removed: there is nothing left to reconcile+ if (err.code === 'ENOENT') return++ throw err+ }++ // The listing is more up to date than the individual checks we had queued, so drop the filenames it contains+ // We keep the others: they may have been created after the listing, and we still have to check them+ for (const filename of filenames) {+ this.pendingFilenames.delete(filename)+ }++ const added = filenames.filter(f => !this.knownFilenames.has(f))+ for (const filename of added.sort(this.sort)) {+ this.emitAdd(filename)+ }++ const current = new Set(filenames)+ // Iterate a copy: emitUnlink() removes the filename from the set+ for (const filename of Array.from(this.knownFilenames)) {+ if (!current.has(filename)) this.emitUnlink(filename)+ }+ }++ private emitAdd (filename: string) {+ if (this.closed) return+ if (this.knownFilenames.has(filename)) return++ this.knownFilenames.add(filename)+ this.emit('add', join(this.directory, filename))+ }++ private emitUnlink (filename: string) {+ if (this.closed) return+ if (!this.knownFilenames.delete(filename)) return++ this.emit('unlink', join(this.directory, filename))+ }++ private async exists (path: string) {+ try {+ await stat(path)++ return true+ } catch (err) {+ if (err.code === 'ENOENT') return false++ throw err+ }+ }+}++export {+ DirectoryWatcher+}diff --git a/packages/node-utils/src/index.ts b/packages/node-utils/src/index.tsindex 4152934749..c74dd2bc0b 100644--- a/packages/node-utils/src/index.ts+++ b/packages/node-utils/src/index.ts@@ -1,6 +1,8 @@export * from './crypto.js'+export * from './directory-watcher.js'export * from './env.js'export * from './file.js'export * from './p2p.js'export * from './path.js'+export * from './typed-event-emitter.js'export * from './uuid.js'diff --git a/packages/node-utils/src/typed-event-emitter.ts b/packages/node-utils/src/typed-event-emitter.tsnew file mode 100644index 0000000000..0a486801f5--- /dev/null+++ b/packages/node-utils/src/typed-event-emitter.ts@@ -0,0 +1,54 @@+import { EventEmitter } from 'events'++// EventEmitter that types its event names and the arguments of their listeners+// Extend it instead of repeating the class/interface declaration merging a typed emitter would otherwise need+// Every method that takes an event name is overridden, so none of them can be called with an unknown event+export class TypedEventEmitter<Events extends Record<keyof Events, (...args: any[]) => void>> extends EventEmitter {+ on<U extends keyof Events & string> (event: U, listener: Events[U]): this {+ return super.on(event, listener)+ }++ addListener<U extends keyof Events & string> (event: U, listener: Events[U]): this {+ return super.addListener(event, listener)+ }++ once<U extends keyof Events & string> (event: U, listener: Events[U]): this {+ return super.once(event, listener)+ }++ prependListener<U extends keyof Events & string> (event: U, listener: Events[U]): this {+ return super.prependListener(event, listener)+ }++ prependOnceListener<U extends keyof Events & string> (event: U, listener: Events[U]): this {+ return super.prependOnceListener(event, listener)+ }++ off<U extends keyof Events & string> (event: U, listener: Events[U]): this {+ return super.off(event, listener)+ }++ removeListener<U extends keyof Events & string> (event: U, listener: Events[U]): this {+ return super.removeListener(event, listener)+ }++ removeAllListeners<U extends keyof Events & string> (event?: U): this {+ return super.removeAllListeners(event)+ }++ listeners<U extends keyof Events & string> (event: U): Events[U][] {+ return super.listeners(event) as Events[U][]+ }++ rawListeners<U extends keyof Events & string> (event: U): Events[U][] {+ return super.rawListeners(event) as Events[U][]+ }++ listenerCount<U extends keyof Events & string> (event: U, listener?: Events[U]): number {+ return super.listenerCount(event, listener)+ }++ emit<U extends keyof Events & string> (event: U, ...args: Parameters<Events[U]>): boolean {+ return super.emit(event, ...args)+ }+}diff --git a/packages/tests/fixtures/video_short.ts b/packages/tests/fixtures/video_short.tsnew file mode 100644index 0000000000..e604360f9cBinary files /dev/null and b/packages/tests/fixtures/video_short.ts differdiff --git a/packages/tests/src/api/transcoding/transcoder.ts b/packages/tests/src/api/transcoding/transcoder.tsindex 11877b648b..fd97c16210 100644--- a/packages/tests/src/api/transcoding/transcoder.ts+++ b/packages/tests/src/api/transcoding/transcoder.ts@@ -153,7 +153,7 @@ describe('Test video transcoding', function () {it('Should accept and transcode additional extensions', async function () {this.timeout(300_000)- for (const fixture of [ 'video_short.mkv', 'video_short.avi' ]) {+ for (const fixture of [ 'video_short.mkv', 'video_short.avi', 'video_short.ts' ]) {const attributes = {name: fixture,fixturediff --git a/packages/tests/src/server-helpers/directory-watcher.ts b/packages/tests/src/server-helpers/directory-watcher.tsnew file mode 100644index 0000000000..f1be3beb71--- /dev/null+++ b/packages/tests/src/server-helpers/directory-watcher.ts@@ -0,0 +1,237 @@+/* oxlint-disable @typescript-eslint/no-unused-expressions */++import { compareLiveFilenames, wait } from '@peertube/peertube-core-utils'+import { DirectoryWatcher } from '@peertube/peertube-node-utils'+import { expect } from 'chai'+import { mkdtemp, rename, rm, symlink, unlink, writeFile } from 'fs/promises'+import { tmpdir } from 'os'+import { basename, join } from 'path'++describe('Directory watcher', function () {+ let directory: string+ let watcher: DirectoryWatcher++ let added: string[]+ let unlinked: string[]+ let errors: Error[]++ async function buildWatcher (options: {+ filter?: (filename: string) => boolean+ sort?: (a: string, b: string) => number+ } = {}) {+ added = []+ unlinked = []+ errors = []++ watcher = new DirectoryWatcher({+ directory,+ filter: options.filter ?? (filename => filename.endsWith('.ts')),+ sort: options.sort+ })++ watcher.on('add', p => added.push(basename(p)))+ watcher.on('unlink', p => unlinked.push(basename(p)))+ watcher.on('error', err => errors.push(err))++ watcher.watch()++ // Let the initial listing run+ await watcher.flush()++ return watcher+ }++ // The events of the OS watcher are asynchronous: don't rely on a fixed delay, that a loaded CI runner may exceed+ async function waitUntil (condition: () => boolean) {+ const deadline = Date.now() + 20000++ while (!condition() && Date.now() < deadline) {+ await wait(100)+ }+ }++ function segmentName (playlistId: number, counter: number) {+ return `${playlistId}-${counter.toString().padStart(6, '0')}.ts`+ }++ function writeSegment (playlistId: number, counter: number) {+ return writeFile(join(directory, segmentName(playlistId, counter)), 'segment')+ }++ beforeEach(async function () {+ directory = await mkdtemp(join(tmpdir(), 'peertube-directory-watcher-'))+ })++ afterEach(async function () {+ await watcher?.close()+ watcher = undefined++ await rm(directory, { recursive: true, force: true })+ })++ it('Should emit an add for the files the directory already contains', async function () {+ await writeSegment(0, 1)+ await writeSegment(0, 2)+ await writeFile(join(directory, 'segments-sha256.json'), '{}')++ await buildWatcher()++ expect(added).to.deep.equal([ '0-000001.ts', '0-000002.ts' ])+ expect(unlinked).to.be.empty+ expect(errors).to.be.empty+ })++ it('Should not emit an event for a filtered out file', async function () {+ await buildWatcher()++ await writeFile(join(directory, 'segments-sha256.json'), '{}')+ await writeFile(join(directory, '0-000001.ts.tmp'), 'incomplete')+ await watcher.flush()++ expect(added).to.be.empty+ expect(unlinked).to.be.empty+ })++ it('Should emit an add and an unlink of a file, in the order they appeared', async function () {+ await buildWatcher()++ for (let i = 1; i <= 20; i++) {+ await writeSegment(0, i)+ }+ await watcher.flush()++ expect(added).to.deep.equal(new Array(20).fill(null).map((_v, i) => segmentName(0, i + 1)))++ await unlink(join(directory, segmentName(0, 1)))+ await unlink(join(directory, segmentName(0, 2)))+ await watcher.flush()++ expect(unlinked).to.deep.equal([ segmentName(0, 1), segmentName(0, 2) ])+ expect(errors).to.be.empty+ })++ it('Should emit a single add for a file that is rewritten', async function () {+ await buildWatcher({ filter: filename => filename.endsWith('.m3u8') })++ // ffmpeg writes its playlists in a temporary file that it renames on the target+ for (let i = 0; i < 3; i++) {+ await writeFile(join(directory, '0.m3u8.tmp'), 'playlist v' + i)+ await rename(join(directory, '0.m3u8.tmp'), join(directory, '0.m3u8'))+ await watcher.flush()+ }++ expect(added).to.deep.equal([ '0.m3u8' ])+ expect(unlinked).to.be.empty+ })++ it('Should emit an add again for a file that is recreated', async function () {+ await buildWatcher()++ await writeSegment(0, 1)+ await watcher.flush()+ await unlink(join(directory, segmentName(0, 1)))+ await watcher.flush()+ await writeSegment(0, 1)+ await watcher.flush()++ expect(added).to.deep.equal([ segmentName(0, 1), segmentName(0, 1) ])+ expect(unlinked).to.deep.equal([ segmentName(0, 1) ])+ })++ it('Should emit the files of a listing in the order of the sort option', async function () {+ // The counter of a segment is zero padded, but it grows past its padding on a long lasting live+ for (const counter of [ 1000000, 2, 1 ]) {+ await writeSegment(0, counter)+ }++ await writeFile(join(directory, '0.m3u8'), 'playlist')+ await writeFile(join(directory, 'master.m3u8'), 'master playlist')++ await buildWatcher({+ filter: filename => filename.endsWith('.ts') || filename.endsWith('.m3u8'),+ sort: compareLiveFilenames+ })++ // The playlists first: the handlers of a segment need the playlist that references it+ expect(added).to.deep.equal([+ '0.m3u8',+ 'master.m3u8',+ segmentName(0, 1),+ segmentName(0, 2),+ segmentName(0, 1000000)+ ])+ expect(errors).to.be.empty+ })++ it('Should recover the events the OS watcher never sent us', async function () {+ await buildWatcher()++ // Simulate a dropped event: the file exists but nothing notified the watcher about it+ await watcher.close()+ await writeSegment(0, 1)++ await buildWatcher()++ expect(added).to.deep.equal([ segmentName(0, 1) ])+ })++ it('Should keep emitting the other files after one of them cannot be checked', async function () {+ await buildWatcher()++ // Create everything in the same tick, so the checks of the segments are queued behind the one that fails+ // A symlink pointing at itself is listed by the directory, but stat() fails on it with ELOOP+ await Promise.all([+ symlink('loop.ts', join(directory, 'loop.ts')),+ ...new Array(20).fill(null).map((_v, i) => writeSegment(0, i + 1))+ ])++ // Don't flush: a full listing does not stat the entries, so only the event path can hit the error+ // The OS can notify us more than once of the symlink, so we can stat it (and fail) more than once+ await waitUntil(() => errors.length !== 0 && added.length === 20)++ expect(errors).to.not.be.empty+ for (const err of errors) {+ expect((err as any).code).to.equal('ELOOP')+ }++ // The files queued behind the one we cannot check must still be emitted+ expect(added.sort()).to.deep.equal(new Array(20).fill(null).map((_v, i) => segmentName(0, i + 1)))+ })++ it('Should not emit anything anymore once closed', async function () {+ await buildWatcher()++ await watcher.close()++ await writeSegment(0, 1)+ await wait(500)++ expect(added).to.be.empty+ expect(errors).to.be.empty+ })++ it('Should be able to be closed twice', async function () {+ await buildWatcher()++ await watcher.close()+ await watcher.close()+ })++ it('Should resolve every close() call and let the directory be removed', async function () {+ await buildWatcher()++ for (let i = 1; i <= 50; i++) {+ await writeSegment(0, i)+ }++ // A first caller that does not wait for the checks in flight must not let the next ones resolve before they end+ void watcher.close()++ await watcher.close()+ await rm(directory, { recursive: true, force: true })++ await wait(500)++ expect(errors).to.be.empty+ })+})diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex 8347bcea3a..7bffff9358 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -149,9 +149,6 @@ importers:bytes:specifier: ^3.1.2version: 3.1.2- chokidar:- specifier: ^5.0.0- version: 5.0.0commander:specifier: ^15.0.0version: 15.0.0diff --git a/server/core/initializers/constants.ts b/server/core/initializers/constants.tsindex b936b820b8..b72627a836 100644--- a/server/core/initializers/constants.ts+++ b/server/core/initializers/constants.ts@@ -1,4 +1,4 @@-import { randomInt } from '@peertube/peertube-core-utils'+import { LIVE_SEGMENT_EXTENSION, randomInt } from '@peertube/peertube-core-utils'import {AbuseState,AbuseStateType,@@ -1214,7 +1214,7 @@ export const DIRECTORIES = {export const RESUMABLE_UPLOAD_SESSION_LIFETIME = SCHEDULER_INTERVALS_MS.REMOVE_DANGLING_RESUMABLE_UPLOADSexport const VIDEO_LIVE = {- EXTENSION: '.ts',+ EXTENSION: LIVE_SEGMENT_EXTENSION,CLEANUP_DELAY: 1000 * 60 * 5, // 5 minutes// Delay before aborting a session on RTMP disconnection, so we kill ffmpeg even if it still has data to processABORT_DELAY_ON_RTMP_DISCONNECT: 2000, // 2 seconds@@ -1608,6 +1608,10 @@ function buildVideoMimetypeExt () {'video/m2ts': '.m2ts',+ // Some OS recognize .ts files as text, so we need to add a specific mimetype for them+ // See https://stackoverflow.com/questions/14230396/ts-files-always-get-recognized-as-text-vnd-trolltech-linguist-and-never-as-vide+ 'text/vnd.trolltech.linguist': '.ts',+// Old formats reliant on MPEG-1/MPEG-2'video/mpv': '.mpv','video/mpeg2': '.m2v',diff --git a/server/core/lib/internal-event-emitter.ts b/server/core/lib/internal-event-emitter.tsindex d1a7ad6d07..9999ee3aad 100644--- a/server/core/lib/internal-event-emitter.ts+++ b/server/core/lib/internal-event-emitter.ts@@ -1,5 +1,5 @@+import { TypedEventEmitter } from '@peertube/peertube-node-utils'import { MChannel, MVideo, MVideoImmutable, MVideoPlaylist, MVideoPlaylistElement } from '@server/types/models/index.js'-import { EventEmitter } from 'events'export interface PeerTubeInternalEvents {'video-created': (options: { video: MVideo }) => void@@ -21,21 +21,7 @@ export interface PeerTubeInternalEvents {'chapters-updated': (options: { video: MVideoImmutable }) => void}-// oxlint-disable-next-line @typescript-eslint/no-unsafe-declaration-merging-declare interface InternalEventEmitter {- on<U extends keyof PeerTubeInternalEvents>(- event: U,- listener: PeerTubeInternalEvents[U]- ): this-- emit<U extends keyof PeerTubeInternalEvents>(- event: U,- ...args: Parameters<PeerTubeInternalEvents[U]>- ): boolean-}--// oxlint-disable-next-line @typescript-eslint/no-unsafe-declaration-merging-class InternalEventEmitter extends EventEmitter {+class InternalEventEmitter extends TypedEventEmitter<PeerTubeInternalEvents> {private static instance: InternalEventEmitterstatic get Instance () {diff --git a/server/core/lib/live/live-utils.ts b/server/core/lib/live/live-utils.tsindex 45d1043744..37c4d279d1 100644--- a/server/core/lib/live/live-utils.ts+++ b/server/core/lib/live/live-utils.ts@@ -1,4 +1,4 @@-import { wait } from '@peertube/peertube-core-utils'+import { LIVE_PLAYLIST_EXTENSION, LIVE_SEGMENT_EXTENSION, wait } from '@peertube/peertube-core-utils'import { FileStorage, LiveVideoLatencyMode, LiveVideoLatencyModeType, VideoState } from '@peertube/peertube-models'import { createLogger } from '@server/helpers/logger.js'import { CONFIG } from '@server/initializers/config.js'@@ -95,8 +95,8 @@ export function getLiveSegmentListSize (options: {// ---------------------------------------------------------------------------function isTMPLiveFile (name: string) {- return name.endsWith('.ts') ||- name.endsWith('.m3u8') ||+ return name.endsWith(LIVE_PLAYLIST_EXTENSION) ||+ name.endsWith(LIVE_SEGMENT_EXTENSION) ||name.endsWith('.json') ||name.endsWith('.mpd') ||name.endsWith('.m4s') ||diff --git a/server/core/lib/live/shared/muxing-session.ts b/server/core/lib/live/shared/muxing-session.tsindex 33f72c786e..3c7ee60e1d 100644--- a/server/core/lib/live/shared/muxing-session.ts+++ b/server/core/lib/live/shared/muxing-session.ts@@ -1,4 +1,11 @@-import { wait } from '@peertube/peertube-core-utils'+import {+ compareLiveFilenames,+ getLivePlaylistIdFromSegmentPath,+ getLivePlaylistNameFromSegmentPath,+ isLivePlaylistPath,+ isLiveSegmentPath,+ wait+} from '@peertube/peertube-core-utils'import {FileStorage,LiveVideoError,@@ -7,6 +14,7 @@ import {VideoResolution,VideoStreamingPlaylistType} from '@peertube/peertube-models'+import { DirectoryWatcher, TypedEventEmitter } from '@peertube/peertube-node-utils'import { computeOutputFPS } from '@server/helpers/ffmpeg/index.js'import { createLogger } from '@server/helpers/logger.js'import { CONFIG } from '@server/initializers/config.js'@@ -16,8 +24,6 @@ import { VideoFileModel } from '@server/models/video/video-file.js'import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist.js'import { MStreamingPlaylistVideo, MUserId, MVideoLiveVideo } from '@server/types/models/index.js'import Bluebird from 'bluebird'-import { FSWatcher, watch } from 'chokidar'-import { EventEmitter } from 'events'import { FfprobeData } from 'fluent-ffmpeg'import { ensureDir } from 'fs-extra/esm'import { appendFile, readdir, readFile, stat } from 'fs/promises'@@ -51,21 +57,7 @@ interface MuxingSessionEvents {'after-cleanup': (options: { videoUUID: string }) => void}-// oxlint-disable-next-line @typescript-eslint/no-unsafe-declaration-merging-declare interface MuxingSession {- on<U extends keyof MuxingSessionEvents>(- event: U,- listener: MuxingSessionEvents[U]- ): this-- emit<U extends keyof MuxingSessionEvents>(- event: U,- ...args: Parameters<MuxingSessionEvents[U]>- ): boolean-}--// oxlint-disable-next-line @typescript-eslint/no-unsafe-declaration-merging-class MuxingSession extends EventEmitter implements MuxingSession {+class MuxingSession extends TypedEventEmitter<MuxingSessionEvents> {private transcodingWrapper: AbstractTranscodingWrapperprivate readonly context: any@@ -120,7 +112,7 @@ class MuxingSession extends EventEmitter implements MuxingSession {private streamingPlaylist: MStreamingPlaylistVideoprivate liveSegmentShaStore: LiveSegmentShaStore- private filesWatcher: FSWatcher+ private filesWatcher: DirectoryWatcherprivate masterPlaylistCreated = falseprivate liveReady = false@@ -199,16 +191,23 @@ class MuxingSession extends EventEmitter implements MuxingSession {// Watch the directory *before* running the transcoding process, so the watcher can never be created after the// session was cleaned up and destroyed: run() can abort and emit 'end' before it returns, and the cleanup would// then close a watcher that does not exist yet- // Chokidar also emits 'add' for the files it finds on init, so we don't miss anything by watching earlier- this.filesWatcher = watch(this.outDirectory, {- // Ignore 'segments-sha256.json' and 'segments-sha256.json.tmp' files that are frequently updated and not useful- ignored: path => path.endsWith('.json') || path.endsWith('json.tmp'),- depth: 0+ // The watcher also emits 'add' for the files it finds on init, so we don't miss anything by watching earlier+ this.filesWatcher = new DirectoryWatcher({+ directory: this.outDirectory,+ // Only the segments and the playlists interest us: in particular 'segments-sha256.json' is frequently updated by+ // this session itself, and the '.tmp' files of the transcoding process are incomplete+ filter: filename => isLiveSegmentPath(filename) || isLivePlaylistPath(filename),+ // A listing has to rebuild the order in which the transcoding process generated the files+ sort: compareLiveFilenames})+ this.filesWatcher.on('error', err => logger.error('Error in live files watcher of %s.', this.outDirectory, { err }))+this.watchMasterFile()this.watchTSFiles()+ this.filesWatcher.watch()+this.transcodingWrapper = this.buildTranscodingWrapper(toTranscode)this.transcodingWrapper.on('end', () => this.onTranscodedEnded())@@ -243,7 +242,8 @@ class MuxingSession extends EventEmitter implements MuxingSession {destroy () {// Ensure the files watcher is always closed, even when the cleanup was never scheduled// (e.g. an ffmpeg error makes the transcoding wrapper abort() short-circuit without emitting 'end')- this.closeWatcher()+ // Every caller gets the same close promise, so runCleanup() can still close it itself+ this.filesWatcher?.close().catch(err => logger.error('Cannot close files watcher of %s.', this.outDirectory, { err }))// We removed its listeners below, so don't leave a pending timer that would emit an event nobody handles@@ -258,15 +258,6 @@ class MuxingSession extends EventEmitter implements MuxingSession {this.hasClientSocketInBadHealthWithCache.clear()}- private closeWatcher () {- if (!this.filesWatcher) return Promise.resolve()-- const watcher = this.filesWatcher- this.filesWatcher = undefined-- return watcher.close()- }-// Watcher handlers are async and fire and forget: remember them so the cleanup can wait for the pending onesprivate trackWatcherHandler (handler: Promise<void>) {const tracked: Promise<void> = handler@@ -351,11 +342,11 @@ class MuxingSession extends EventEmitter implements MuxingSession {const startStreamDateTime = new Date().getTime()const addHandler = (segmentPath: string) => {- if (segmentPath.endsWith('.ts') !== true) return+ if (!isLiveSegmentPath(segmentPath)) returnlogger.debug('Live add handler of TS file %s.', segmentPath)- const playlistId = this.getPlaylistIdFromTS(segmentPath)+ const playlistId = getLivePlaylistIdFromSegmentPath(segmentPath)if (!playlistId) {logger.warn('Cannot get the playlist id of live segment %s, ignoring it.', segmentPath)return@@ -376,7 +367,7 @@ class MuxingSession extends EventEmitter implements MuxingSession {}const deleteHandler = async (segmentPath: string) => {- if (segmentPath.endsWith('.ts') !== true) return+ if (!isLiveSegmentPath(segmentPath)) returnlogger.debug('Live delete handler of TS file %s.', segmentPath)@@ -534,7 +525,7 @@ class MuxingSession extends EventEmitter implements MuxingSession {}private async processM3U8ToObjectStorage (segmentPath: string) {- const m3u8Path = join(this.outDirectory, this.getPlaylistNameFromTS(segmentPath))+ const m3u8Path = join(this.outDirectory, getLivePlaylistNameFromSegmentPath(segmentPath))logger.debug('Process M3U8 file %s.', m3u8Path)@@ -598,8 +589,10 @@ class MuxingSession extends EventEmitter implements MuxingSession {private async runCleanup () {try {- // The transcoding process exited so no new segment will be generated: we can close the watcher- await this.closeWatcher()+ // The transcoding process exited so no new segment will be generated, but the OS watcher may never have+ // notified us of the last ones: ask for a final listing before closing the watcher+ await this.filesWatcher?.flush()+ await this.filesWatcher?.close()// Closing the watcher does not wait for the handlers it already started, that can still write filesawait Promise.all(this.pendingWatcherHandlers)@@ -607,8 +600,6 @@ class MuxingSession extends EventEmitter implements MuxingSession {// Wait for the segments the watcher already notified us aboutawait Promise.all(Array.from(this.segmentProcessingQueues.values(), queue => queue.onIdle()))- // Watcher events are asynchronous, so the last segments generated by the transcoding process may never have been- // notified to us: list the output directory instead of relying on the events we receivedawait this.processRemainingSegments()} catch (err) {logger.error(@@ -623,13 +614,14 @@ class MuxingSession extends EventEmitter implements MuxingSession {this.emit('after-cleanup', { videoUUID: this.videoUUID })}+ // Process the segments still on disk that no handler took care ofprivate async processRemainingSegments () {const filenames = await readdir(this.outDirectory)const segmentPaths = filenames- // Segments are generated with a zero padded counter (%v-%06d.ts) so sorting them by name keeps the live order- .filter(f => f.endsWith(VIDEO_LIVE.EXTENSION))- .sort()+ .filter(f => isLiveSegmentPath(f))+ // The listing order of the filesystem is arbitrary: process the segments in the order they were generated+ .sort(compareLiveFilenames).map(f => join(this.outDirectory, f)).filter(p => !this.processedSegments.has(p))@@ -637,7 +629,7 @@ class MuxingSession extends EventEmitter implements MuxingSession {const perPlaylist = new Map<string, string[]>()for (const segmentPath of segmentPaths) {- const playlistId = this.getPlaylistIdFromTS(segmentPath)+ const playlistId = getLivePlaylistIdFromSegmentPath(segmentPath)// Not a segment generated by our transcoding processif (!playlistId) continue@@ -751,17 +743,6 @@ class MuxingSession extends EventEmitter implements MuxingSession {: new FFmpegTranscodingWrapper(options)}- // Returns undefined if the file was not generated by our transcoding process- private getPlaylistIdFromTS (segmentPath: string) {- const playlistIdMatcher = /^(\d+)-/-- return basename(segmentPath).match(playlistIdMatcher)?.[1]- }-- private getPlaylistNameFromTS (segmentPath: string) {- return `${this.getPlaylistIdFromTS(segmentPath)}.m3u8`- }-private buildToTranscode () {return this.allResolutions.map(resolution => {let toTranscodeFPS: numberdiff --git a/server/core/lib/live/shared/transcoding-wrapper/abstract-transcoding-wrapper.ts b/server/core/lib/live/shared/transcoding-wrapper/abstract-transcoding-wrapper.tsindex 40423d0214..35b9defcd9 100644--- a/server/core/lib/live/shared/transcoding-wrapper/abstract-transcoding-wrapper.ts+++ b/server/core/lib/live/shared/transcoding-wrapper/abstract-transcoding-wrapper.ts@@ -1,6 +1,6 @@import { LiveVideoErrorType } from '@peertube/peertube-models'+import { TypedEventEmitter } from '@peertube/peertube-node-utils'import { MStreamingPlaylistVideo, MVideoLiveVideo } from '@server/types/models/index.js'-import EventEmitter from 'events'import { FfprobeData } from 'fluent-ffmpeg'interface TranscodingWrapperEvents {@@ -9,19 +9,6 @@ interface TranscodingWrapperEvents {'error': (options: { err: Error }) => void}-// oxlint-disable-next-line @typescript-eslint/no-unsafe-declaration-merging-declare interface AbstractTranscodingWrapper {- on<U extends keyof TranscodingWrapperEvents>(- event: U,- listener: TranscodingWrapperEvents[U]- ): this-- emit<U extends keyof TranscodingWrapperEvents>(- event: U,- ...args: Parameters<TranscodingWrapperEvents[U]>- ): boolean-}-interface AbstractTranscodingWrapperOptions {streamingPlaylist: MStreamingPlaylistVideovideoLive: MVideoLiveVideo@@ -48,8 +35,7 @@ interface AbstractTranscodingWrapperOptions {outDirectory: string}-// oxlint-disable-next-line @typescript-eslint/no-unsafe-declaration-merging-abstract class AbstractTranscodingWrapper extends EventEmitter {+abstract class AbstractTranscodingWrapper extends TypedEventEmitter<TranscodingWrapperEvents> {protected readonly videoLive: MVideoLiveVideoprotected readonly toTranscode: {