Skip to content

ActivityPub-federated video streaming platform using P2P directly in your web browser

Abort ffmpeg and HTTP requests on timeout

Chocobozzz481546defd75
diff --git a/packages/core-utils/src/common/promises.ts b/packages/core-utils/src/common/promises.ts
index bb90175fdc..77a3f12b17 100644
--- a/packages/core-utils/src/common/promises.ts
+++ b/packages/core-utils/src/common/promises.ts
@@ -18,6 +18,27 @@ export function timeoutPromise<T> (promise: Promise<T>, timeoutMs: number) {
]).finally(() => clearTimeout(timer))
}
+// Timeout promise with cleanup callback to kill orphaned processes and cancel HTTP requests
+export function timeoutPromiseWithCleanup<T> (
+ promise: Promise<T>,
+ timeoutMs: number,
+ onTimeoutCleanup?: () => void
+) {
+ let timer: ReturnType<typeof setTimeout>
+
+ return Promise.race([
+ promise,
+
+ new Promise<T>((_res, rej) => {
+ timer = setTimeout(() => {
+ if (onTimeoutCleanup) onTimeoutCleanup()
+
+ rej(new Error('Timeout'))
+ }, timeoutMs)
+ })
+ ]).finally(() => clearTimeout(timer))
+}
+
export function promisify0<A> (func: (cb: (err: any, result: A) => void) => void): () => Promise<A> {
return function promisified (): Promise<A> {
return new Promise<A>((resolve: (arg: A) => void, reject: (err: any) => void) => {
diff --git a/packages/ffmpeg/src/ffmpeg-command-wrapper.ts b/packages/ffmpeg/src/ffmpeg-command-wrapper.ts
index a2f464d68d..dc04e49824 100644
--- a/packages/ffmpeg/src/ffmpeg-command-wrapper.ts
+++ b/packages/ffmpeg/src/ffmpeg-command-wrapper.ts
@@ -24,6 +24,9 @@ export interface FFmpegCommandWrapperOptions {
updateJobProgress?: (progress?: number) => void
onEnd?: () => void
onError?: (err: Error) => void
+
+ // Optional abort signal to cancel the ffmpeg process on timeout or other cancellation
+ abortSignal?: AbortSignal
}
export class FFmpegCommandWrapper {
@@ -42,6 +45,7 @@ export class FFmpegCommandWrapper {
private readonly updateJobProgress: (progress?: number) => void
private readonly onEnd?: () => void
private readonly onError?: (err: Error) => void
+ private readonly abortSignal?: AbortSignal
private command: FfmpegCommand
@@ -58,6 +62,7 @@ export class FFmpegCommandWrapper {
this.onEnd = options.onEnd
this.onError = options.onError
+ this.abortSignal = options.abortSignal
}
getAvailableEncoders () {
@@ -84,6 +89,20 @@ export class FFmpegCommandWrapper {
this.command = undefined
}
+ // Kill the ffmpeg process (called on timeout to prevent orphaned processes)
+ killCommand (signal = 'SIGKILL') {
+ if (!this.command) return
+
+ this.logger.debug(`Killing ffmpeg process with ${signal}`, this.lTags)
+
+ try {
+ this.command.kill(signal)
+ this.command = undefined
+ } catch (err) {
+ this.logger.warn(`Failed to kill ffmpeg process`, { err, ...this.lTags })
+ }
+ }
+
buildCommand (inputs: (string | Readable)[] | string | Readable, inputFileMutexReleaser?: MutexInterface.Releaser) {
if (this.command) throw new Error('Command is already built')
@@ -155,6 +174,22 @@ export class FFmpegCommandWrapper {
})
}
+ // Listen for abort signal to kill the ffmpeg process on timeout
+ if (this.abortSignal) {
+ if (this.abortSignal.aborted) {
+ // Signal already aborted before command started
+ this.killCommand('SIGKILL')
+ rej(new Error('FFmpeg process aborted'))
+ return
+ }
+
+ this.abortSignal.addEventListener('abort', () => {
+ this.logger.debug('Abort signal received, killing ffmpeg process', this.lTags)
+ this.killCommand('SIGKILL')
+ rej(new Error('FFmpeg process aborted'))
+ }, { once: true })
+ }
+
this.command.run()
})
}
diff --git a/packages/models/src/server/job.model.ts b/packages/models/src/server/job.model.ts
index cb7f627ef9..cd382cfad0 100644
--- a/packages/models/src/server/job.model.ts
+++ b/packages/models/src/server/job.model.ts
@@ -36,6 +36,7 @@ export type JobType =
| 'video-transcription'
| 'videos-stats'
+// Client API
export interface Job {
id: number | string
state: JobState | 'unknown'
@@ -53,6 +54,8 @@ export interface Job {
}
}
+// ---------------------------------------------------------------------------
+
export type ActivitypubHttpBroadcastPayload = {
uris: string[]
contextType: ContextType
@@ -74,6 +77,7 @@ export type ActivitypubHttpFetcherPayload = {
type: FetchType
videoId?: number
accountId?: number
+ abortSignal?: AbortSignal
}
export type ActivitypubHttpUnicastPayload = {
@@ -148,6 +152,7 @@ interface BaseTranscodingPayload {
videoUUID: string
canMoveVideoState: boolean
isNewVideo?: boolean
+ abortSignal?: AbortSignal
}
export interface HLSTranscodingPayload extends BaseTranscodingPayload {
diff --git a/server/core/helpers/requests.ts b/server/core/helpers/requests.ts
index c23846d5fe..44da15b582 100644
--- a/server/core/helpers/requests.ts
+++ b/server/core/helpers/requests.ts
@@ -39,6 +39,9 @@ export type PeerTubeRequestOptions = {
jsonResponse?: boolean
followRedirect?: boolean
+
+ // Support AbortSignal for request cancellation (e.g., on job timeout)
+ signal?: AbortSignal
} & Pick<OptionsInit, 'headers' | 'json' | 'method' | 'searchParams'>
export const unsafeSSRFGot = got.extend({
@@ -273,6 +276,7 @@ function buildGotOptions (options: PeerTubeRequestOptions): OptionsOfUnknownResp
json: options.json,
searchParams: options.searchParams,
followRedirect: options.followRedirect,
+ signal: options.signal,
retry: {
limit: 2
},
diff --git a/server/core/lib/activitypub/crawl.ts b/server/core/lib/activitypub/crawl.ts
index d7c645fbd9..aedff98a85 100644
--- a/server/core/lib/activitypub/crawl.ts
+++ b/server/core/lib/activitypub/crawl.ts
@@ -9,20 +9,30 @@ import { fetchAP } from './activity.js'
type HandlerFunction<T> = (items: T[]) => Promise<any> | Bluebird<any>
type CleanerFunction = (startedDate: Date) => Promise<any>
-export async function crawlCollectionPage<T> (argUrl: string, handler: HandlerFunction<T>, cleaner?: CleanerFunction) {
+export async function crawlCollectionPage<T> (
+ argUrl: string,
+ handler: HandlerFunction<T>,
+ cleaner?: CleanerFunction,
+ abortSignal?: AbortSignal
+) {
let url = argUrl
logger.info('Crawling ActivityPub data on %s.', url)
const startDate = new Date()
- const response = await fetchAP<ActivityPubOrderedCollection<T>>(url)
+ const response = await fetchAP<ActivityPubOrderedCollection<T>>(url, { signal: abortSignal })
const firstBody = response.body
const limit = ACTIVITY_PUB.FETCH_PAGE_LIMIT
let i = 0
let nextLink = firstBody.first
while (nextLink && i < limit) {
+ // Check if abort signal has been triggered
+ if (abortSignal?.aborted) {
+ throw new Error('ActivityPub crawl aborted')
+ }
+
i++
let body: any
@@ -34,7 +44,7 @@ export async function crawlCollectionPage<T> (argUrl: string, handler: HandlerFu
url = nextLink
- const res = await fetchAP<ActivityPubOrderedCollection<T>>(url)
+ const res = await fetchAP<ActivityPubOrderedCollection<T>>(url, { signal: abortSignal })
body = res.body
} else {
// nextLink is already the object we want
diff --git a/server/core/lib/job-queue/handlers/activitypub-http-fetcher.ts b/server/core/lib/job-queue/handlers/activitypub-http-fetcher.ts
index 7cfdccbc72..6ba84effbf 100644
--- a/server/core/lib/job-queue/handlers/activitypub-http-fetcher.ts
+++ b/server/core/lib/job-queue/handlers/activitypub-http-fetcher.ts
@@ -35,7 +35,7 @@ async function processActivityPubHttpFetcher (job: Job) {
'video-comments': crawlStartDate => VideoCommentModel.cleanOldCommentsOf(video.id, crawlStartDate)
}
- return crawlCollectionPage(payload.uri, fetcherType[payload.type], cleanerType[payload.type])
+ return crawlCollectionPage(payload.uri, fetcherType[payload.type], cleanerType[payload.type], payload.abortSignal)
}
// ---------------------------------------------------------------------------
diff --git a/server/core/lib/job-queue/handlers/video-transcoding.ts b/server/core/lib/job-queue/handlers/video-transcoding.ts
index 14e000d688..9316dbfa2b 100644
--- a/server/core/lib/job-queue/handlers/video-transcoding.ts
+++ b/server/core/lib/job-queue/handlers/video-transcoding.ts
@@ -77,7 +77,12 @@ export {
// Job handlers
// ---------------------------------------------------------------------------
-async function handleWebVideoMergeAudioJob (job: Job, payload: MergeAudioTranscodingPayload, video: MVideoFull, user: MUserId) {
+async function handleWebVideoMergeAudioJob (
+ job: Job<MergeAudioTranscodingPayload>,
+ payload: MergeAudioTranscodingPayload,
+ video: MVideoFull,
+ user: MUserId
+) {
logger.info('Handling merge audio transcoding job for %s.', video.uuid, lTags(video.uuid), { payload })
await mergeAudioVideofile({ video, resolution: payload.resolution, fps: payload.fps, job })
@@ -87,7 +92,12 @@ async function handleWebVideoMergeAudioJob (job: Job, payload: MergeAudioTransco
await onTranscodingEnded({ isNewVideo: payload.isNewVideo, moveVideoToNextState: payload.canMoveVideoState, video })
}
-async function handleWebVideoOptimizeJob (job: Job, payload: OptimizeTranscodingPayload, video: MVideoFull, user: MUserId) {
+async function handleWebVideoOptimizeJob (
+ job: Job<OptimizeTranscodingPayload>,
+ payload: OptimizeTranscodingPayload,
+ video: MVideoFull,
+ user: MUserId
+) {
logger.info('Handling optimize transcoding job for %s.', video.uuid, lTags(video.uuid), { payload })
await optimizeOriginalVideofile({ video, job })
@@ -99,7 +109,11 @@ async function handleWebVideoOptimizeJob (job: Job, payload: OptimizeTranscoding
// ---------------------------------------------------------------------------
-async function handleNewWebVideoResolutionJob (job: Job, payload: NewWebVideoResolutionTranscodingPayload, video: MVideoFull) {
+async function handleNewWebVideoResolutionJob (
+ job: Job<NewWebVideoResolutionTranscodingPayload>,
+ payload: NewWebVideoResolutionTranscodingPayload,
+ video: MVideoFull
+) {
logger.info('Handling Web Video transcoding job for %s.', video.uuid, lTags(video.uuid), { payload })
await transcodeNewWebVideoResolution({ video, resolution: payload.resolution, fps: payload.fps, job })
@@ -112,7 +126,7 @@ async function handleNewWebVideoResolutionJob (job: Job, payload: NewWebVideoRes
// ---------------------------------------------------------------------------
-async function handleHLSJob (job: Job, payload: HLSTranscodingPayload, videoArg: MVideoFull) {
+async function handleHLSJob (job: Job<HLSTranscodingPayload>, payload: HLSTranscodingPayload, videoArg: MVideoFull) {
logger.info('Handling HLS transcoding job for %s.', videoArg.uuid, lTags(videoArg.uuid), { payload })
const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(videoArg.uuid)
diff --git a/server/core/lib/job-queue/job-queue.ts b/server/core/lib/job-queue/job-queue.ts
index 7aeb4f73f2..638990db7f 100644
--- a/server/core/lib/job-queue/job-queue.ts
+++ b/server/core/lib/job-queue/job-queue.ts
@@ -1,4 +1,4 @@
-import { pick, timeoutPromise } from '@peertube/peertube-core-utils'
+import { pick, timeoutPromiseWithCleanup } from '@peertube/peertube-core-utils'
import {
ActivitypubFollowPayload,
ActivitypubHttpBroadcastPayload,
@@ -73,10 +73,10 @@ import { processVideoChannelImport } from './handlers/video-channel-import.js'
import { processVideoFileImport } from './handlers/video-file-import.js'
import { processVideoImport } from './handlers/video-import.js'
import { processVideoLiveEnding } from './handlers/video-live-ending.js'
+import { processVideosStats } from './handlers/video-stats.js'
import { processVideoStudioEdition } from './handlers/video-studio-edition.js'
import { processVideoTranscoding } from './handlers/video-transcoding.js'
import { processVideoTranscription } from './handlers/video-transcription.js'
-import { processVideosStats } from './handlers/video-stats.js'
export type CreateJobTypeAndPayload =
| { type: 'build-automatic-tags', payload: BuildAutomaticTagsPayload }
@@ -243,11 +243,28 @@ class JobQueue {
const handler = function (job: Job) {
const timeout = JOB_TTL[handlerName]
- const p = handlers[handlerName](job)
-
- if (!timeout) return p
-
- return timeoutPromise(p, timeout)
+ if (!timeout) return handlers[handlerName](job)
+
+ const controller = new AbortController()
+
+ // Store abort signal in job immediately so handlers can access it
+ job.data.abortSignal = controller.signal
+
+ // Create timeout with cleanup callback to kill orphaned processes
+ return timeoutPromiseWithCleanup(
+ handlers[handlerName](job),
+ timeout,
+ () => {
+ logger.warn(
+ 'Job %s in queue %s exceeded timeout of %d ms, triggering cleanup',
+ job.id,
+ handlerName,
+ timeout
+ )
+
+ controller.abort()
+ }
+ )
}
const processor = async (jobArg: Job) => {
diff --git a/server/core/lib/transcoding/hls-transcoding.ts b/server/core/lib/transcoding/hls-transcoding.ts
index 7ebb1faf24..a70a6a788c 100644
--- a/server/core/lib/transcoding/hls-transcoding.ts
+++ b/server/core/lib/transcoding/hls-transcoding.ts
@@ -38,7 +38,7 @@ export async function generateHlsPlaylistResolutionFromTS (options: {
}
// Generate an HLS playlist from an input file, and update the master playlist
-export function generateHlsPlaylistResolution (options: {
+export function generateHlsPlaylistResolution<JobData extends { abortSignal?: AbortSignal }> (options: {
video: MVideo
videoInputPath: string
@@ -48,7 +48,7 @@ export function generateHlsPlaylistResolution (options: {
fps: number
inputFileMutexReleaser: MutexInterface.Releaser
separatedAudio: boolean
- job?: Job
+ job?: Job<JobData>
}) {
return generateHlsPlaylistCommon({
type: 'hls' as 'hls',
@@ -143,7 +143,7 @@ export async function onHLSVideoFileTranscoding (options: {
// Private
// ---------------------------------------------------------------------------
-async function generateHlsPlaylistCommon (options: {
+async function generateHlsPlaylistCommon<JobData extends { abortSignal?: AbortSignal }> (options: {
type: 'hls' | 'hls-from-ts'
video: MVideo
@@ -160,7 +160,7 @@ async function generateHlsPlaylistCommon (options: {
isAAC?: boolean
- job?: Job
+ job?: Job<JobData>
}) {
const {
type,
diff --git a/server/core/lib/transcoding/shared/ffmpeg-builder.ts b/server/core/lib/transcoding/shared/ffmpeg-builder.ts
index b28ffdb1f4..3256aa3dd3 100644
--- a/server/core/lib/transcoding/shared/ffmpeg-builder.ts
+++ b/server/core/lib/transcoding/shared/ffmpeg-builder.ts
@@ -1,10 +1,10 @@
-import { Job } from 'bullmq'
+import { FFmpegVOD } from '@peertube/peertube-ffmpeg'
import { getFFmpegCommandWrapperOptions } from '@server/helpers/ffmpeg/index.js'
import { logger } from '@server/helpers/logger.js'
-import { FFmpegVOD } from '@peertube/peertube-ffmpeg'
+import { Job } from 'bullmq'
import { VideoTranscodingProfilesManager } from '../default-transcoding-profiles.js'
-export function buildFFmpegVOD (job?: Job) {
+export function buildFFmpegVOD<T extends Job<{ abortSignal?: AbortSignal }>> (job?: T) {
return new FFmpegVOD({
...getFFmpegCommandWrapperOptions('vod', VideoTranscodingProfilesManager.Instance.getAvailableEncoders()),
@@ -13,6 +13,9 @@ export function buildFFmpegVOD (job?: Job) {
job.updateProgress(progress)
.catch(err => logger.error('Cannot update ffmpeg job progress', { err }))
- }
+ },
+
+ // Pass the abort signal from the job so the ffmpeg process can be killed on timeout
+ abortSignal: job?.data?.abortSignal
})
}