ActivityPub-federated video streaming platform using P2P directly in your web browser
Abort ffmpeg and HTTP requests on timeout
Chocobozzz481546defd75
diff --git a/packages/core-utils/src/common/promises.ts b/packages/core-utils/src/common/promises.tsindex bb90175fdc..77a3f12b17 100644--- a/packages/core-utils/src/common/promises.ts+++ b/packages/core-utils/src/common/promises.ts@@ -18,6 +18,27 @@ export function timeoutPromise<T> (promise: Promise<T>, timeoutMs: number) {]).finally(() => clearTimeout(timer))}+// Timeout promise with cleanup callback to kill orphaned processes and cancel HTTP requests+export function timeoutPromiseWithCleanup<T> (+ promise: Promise<T>,+ timeoutMs: number,+ onTimeoutCleanup?: () => void+) {+ let timer: ReturnType<typeof setTimeout>++ return Promise.race([+ promise,++ new Promise<T>((_res, rej) => {+ timer = setTimeout(() => {+ if (onTimeoutCleanup) onTimeoutCleanup()++ rej(new Error('Timeout'))+ }, timeoutMs)+ })+ ]).finally(() => clearTimeout(timer))+}+export function promisify0<A> (func: (cb: (err: any, result: A) => void) => void): () => Promise<A> {return function promisified (): Promise<A> {return new Promise<A>((resolve: (arg: A) => void, reject: (err: any) => void) => {diff --git a/packages/ffmpeg/src/ffmpeg-command-wrapper.ts b/packages/ffmpeg/src/ffmpeg-command-wrapper.tsindex a2f464d68d..dc04e49824 100644--- a/packages/ffmpeg/src/ffmpeg-command-wrapper.ts+++ b/packages/ffmpeg/src/ffmpeg-command-wrapper.ts@@ -24,6 +24,9 @@ export interface FFmpegCommandWrapperOptions {updateJobProgress?: (progress?: number) => voidonEnd?: () => voidonError?: (err: Error) => void++ // Optional abort signal to cancel the ffmpeg process on timeout or other cancellation+ abortSignal?: AbortSignal}export class FFmpegCommandWrapper {@@ -42,6 +45,7 @@ export class FFmpegCommandWrapper {private readonly updateJobProgress: (progress?: number) => voidprivate readonly onEnd?: () => voidprivate readonly onError?: (err: Error) => void+ private readonly abortSignal?: AbortSignalprivate command: FfmpegCommand@@ -58,6 +62,7 @@ export class FFmpegCommandWrapper {this.onEnd = options.onEndthis.onError = options.onError+ this.abortSignal = options.abortSignal}getAvailableEncoders () {@@ -84,6 +89,20 @@ export class FFmpegCommandWrapper {this.command = undefined}+ // Kill the ffmpeg process (called on timeout to prevent orphaned processes)+ killCommand (signal = 'SIGKILL') {+ if (!this.command) return++ this.logger.debug(`Killing ffmpeg process with ${signal}`, this.lTags)++ try {+ this.command.kill(signal)+ this.command = undefined+ } catch (err) {+ this.logger.warn(`Failed to kill ffmpeg process`, { err, ...this.lTags })+ }+ }+buildCommand (inputs: (string | Readable)[] | string | Readable, inputFileMutexReleaser?: MutexInterface.Releaser) {if (this.command) throw new Error('Command is already built')@@ -155,6 +174,22 @@ export class FFmpegCommandWrapper {})}+ // Listen for abort signal to kill the ffmpeg process on timeout+ if (this.abortSignal) {+ if (this.abortSignal.aborted) {+ // Signal already aborted before command started+ this.killCommand('SIGKILL')+ rej(new Error('FFmpeg process aborted'))+ return+ }++ this.abortSignal.addEventListener('abort', () => {+ this.logger.debug('Abort signal received, killing ffmpeg process', this.lTags)+ this.killCommand('SIGKILL')+ rej(new Error('FFmpeg process aborted'))+ }, { once: true })+ }+this.command.run()})}diff --git a/packages/models/src/server/job.model.ts b/packages/models/src/server/job.model.tsindex cb7f627ef9..cd382cfad0 100644--- a/packages/models/src/server/job.model.ts+++ b/packages/models/src/server/job.model.ts@@ -36,6 +36,7 @@ export type JobType =| 'video-transcription'| 'videos-stats'+// Client APIexport interface Job {id: number | stringstate: JobState | 'unknown'@@ -53,6 +54,8 @@ export interface Job {}}+// ---------------------------------------------------------------------------+export type ActivitypubHttpBroadcastPayload = {uris: string[]contextType: ContextType@@ -74,6 +77,7 @@ export type ActivitypubHttpFetcherPayload = {type: FetchTypevideoId?: numberaccountId?: number+ abortSignal?: AbortSignal}export type ActivitypubHttpUnicastPayload = {@@ -148,6 +152,7 @@ interface BaseTranscodingPayload {videoUUID: stringcanMoveVideoState: booleanisNewVideo?: boolean+ abortSignal?: AbortSignal}export interface HLSTranscodingPayload extends BaseTranscodingPayload {diff --git a/server/core/helpers/requests.ts b/server/core/helpers/requests.tsindex c23846d5fe..44da15b582 100644--- a/server/core/helpers/requests.ts+++ b/server/core/helpers/requests.ts@@ -39,6 +39,9 @@ export type PeerTubeRequestOptions = {jsonResponse?: booleanfollowRedirect?: boolean++ // Support AbortSignal for request cancellation (e.g., on job timeout)+ signal?: AbortSignal} & Pick<OptionsInit, 'headers' | 'json' | 'method' | 'searchParams'>export const unsafeSSRFGot = got.extend({@@ -273,6 +276,7 @@ function buildGotOptions (options: PeerTubeRequestOptions): OptionsOfUnknownRespjson: options.json,searchParams: options.searchParams,followRedirect: options.followRedirect,+ signal: options.signal,retry: {limit: 2},diff --git a/server/core/lib/activitypub/crawl.ts b/server/core/lib/activitypub/crawl.tsindex d7c645fbd9..aedff98a85 100644--- a/server/core/lib/activitypub/crawl.ts+++ b/server/core/lib/activitypub/crawl.ts@@ -9,20 +9,30 @@ import { fetchAP } from './activity.js'type HandlerFunction<T> = (items: T[]) => Promise<any> | Bluebird<any>type CleanerFunction = (startedDate: Date) => Promise<any>-export async function crawlCollectionPage<T> (argUrl: string, handler: HandlerFunction<T>, cleaner?: CleanerFunction) {+export async function crawlCollectionPage<T> (+ argUrl: string,+ handler: HandlerFunction<T>,+ cleaner?: CleanerFunction,+ abortSignal?: AbortSignal+) {let url = argUrllogger.info('Crawling ActivityPub data on %s.', url)const startDate = new Date()- const response = await fetchAP<ActivityPubOrderedCollection<T>>(url)+ const response = await fetchAP<ActivityPubOrderedCollection<T>>(url, { signal: abortSignal })const firstBody = response.bodyconst limit = ACTIVITY_PUB.FETCH_PAGE_LIMITlet i = 0let nextLink = firstBody.firstwhile (nextLink && i < limit) {+ // Check if abort signal has been triggered+ if (abortSignal?.aborted) {+ throw new Error('ActivityPub crawl aborted')+ }+i++let body: any@@ -34,7 +44,7 @@ export async function crawlCollectionPage<T> (argUrl: string, handler: HandlerFuurl = nextLink- const res = await fetchAP<ActivityPubOrderedCollection<T>>(url)+ const res = await fetchAP<ActivityPubOrderedCollection<T>>(url, { signal: abortSignal })body = res.body} else {// nextLink is already the object we wantdiff --git a/server/core/lib/job-queue/handlers/activitypub-http-fetcher.ts b/server/core/lib/job-queue/handlers/activitypub-http-fetcher.tsindex 7cfdccbc72..6ba84effbf 100644--- a/server/core/lib/job-queue/handlers/activitypub-http-fetcher.ts+++ b/server/core/lib/job-queue/handlers/activitypub-http-fetcher.ts@@ -35,7 +35,7 @@ async function processActivityPubHttpFetcher (job: Job) {'video-comments': crawlStartDate => VideoCommentModel.cleanOldCommentsOf(video.id, crawlStartDate)}- return crawlCollectionPage(payload.uri, fetcherType[payload.type], cleanerType[payload.type])+ return crawlCollectionPage(payload.uri, fetcherType[payload.type], cleanerType[payload.type], payload.abortSignal)}// ---------------------------------------------------------------------------diff --git a/server/core/lib/job-queue/handlers/video-transcoding.ts b/server/core/lib/job-queue/handlers/video-transcoding.tsindex 14e000d688..9316dbfa2b 100644--- a/server/core/lib/job-queue/handlers/video-transcoding.ts+++ b/server/core/lib/job-queue/handlers/video-transcoding.ts@@ -77,7 +77,12 @@ export {// Job handlers// ----------------------------------------------------------------------------async function handleWebVideoMergeAudioJob (job: Job, payload: MergeAudioTranscodingPayload, video: MVideoFull, user: MUserId) {+async function handleWebVideoMergeAudioJob (+ job: Job<MergeAudioTranscodingPayload>,+ payload: MergeAudioTranscodingPayload,+ video: MVideoFull,+ user: MUserId+) {logger.info('Handling merge audio transcoding job for %s.', video.uuid, lTags(video.uuid), { payload })await mergeAudioVideofile({ video, resolution: payload.resolution, fps: payload.fps, job })@@ -87,7 +92,12 @@ async function handleWebVideoMergeAudioJob (job: Job, payload: MergeAudioTranscoawait onTranscodingEnded({ isNewVideo: payload.isNewVideo, moveVideoToNextState: payload.canMoveVideoState, video })}-async function handleWebVideoOptimizeJob (job: Job, payload: OptimizeTranscodingPayload, video: MVideoFull, user: MUserId) {+async function handleWebVideoOptimizeJob (+ job: Job<OptimizeTranscodingPayload>,+ payload: OptimizeTranscodingPayload,+ video: MVideoFull,+ user: MUserId+) {logger.info('Handling optimize transcoding job for %s.', video.uuid, lTags(video.uuid), { payload })await optimizeOriginalVideofile({ video, job })@@ -99,7 +109,11 @@ async function handleWebVideoOptimizeJob (job: Job, payload: OptimizeTranscoding// ----------------------------------------------------------------------------async function handleNewWebVideoResolutionJob (job: Job, payload: NewWebVideoResolutionTranscodingPayload, video: MVideoFull) {+async function handleNewWebVideoResolutionJob (+ job: Job<NewWebVideoResolutionTranscodingPayload>,+ payload: NewWebVideoResolutionTranscodingPayload,+ video: MVideoFull+) {logger.info('Handling Web Video transcoding job for %s.', video.uuid, lTags(video.uuid), { payload })await transcodeNewWebVideoResolution({ video, resolution: payload.resolution, fps: payload.fps, job })@@ -112,7 +126,7 @@ async function handleNewWebVideoResolutionJob (job: Job, payload: NewWebVideoRes// ----------------------------------------------------------------------------async function handleHLSJob (job: Job, payload: HLSTranscodingPayload, videoArg: MVideoFull) {+async function handleHLSJob (job: Job<HLSTranscodingPayload>, payload: HLSTranscodingPayload, videoArg: MVideoFull) {logger.info('Handling HLS transcoding job for %s.', videoArg.uuid, lTags(videoArg.uuid), { payload })const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(videoArg.uuid)diff --git a/server/core/lib/job-queue/job-queue.ts b/server/core/lib/job-queue/job-queue.tsindex 7aeb4f73f2..638990db7f 100644--- a/server/core/lib/job-queue/job-queue.ts+++ b/server/core/lib/job-queue/job-queue.ts@@ -1,4 +1,4 @@-import { pick, timeoutPromise } from '@peertube/peertube-core-utils'+import { pick, timeoutPromiseWithCleanup } from '@peertube/peertube-core-utils'import {ActivitypubFollowPayload,ActivitypubHttpBroadcastPayload,@@ -73,10 +73,10 @@ import { processVideoChannelImport } from './handlers/video-channel-import.js'import { processVideoFileImport } from './handlers/video-file-import.js'import { processVideoImport } from './handlers/video-import.js'import { processVideoLiveEnding } from './handlers/video-live-ending.js'+import { processVideosStats } from './handlers/video-stats.js'import { processVideoStudioEdition } from './handlers/video-studio-edition.js'import { processVideoTranscoding } from './handlers/video-transcoding.js'import { processVideoTranscription } from './handlers/video-transcription.js'-import { processVideosStats } from './handlers/video-stats.js'export type CreateJobTypeAndPayload =| { type: 'build-automatic-tags', payload: BuildAutomaticTagsPayload }@@ -243,11 +243,28 @@ class JobQueue {const handler = function (job: Job) {const timeout = JOB_TTL[handlerName]- const p = handlers[handlerName](job)-- if (!timeout) return p-- return timeoutPromise(p, timeout)+ if (!timeout) return handlers[handlerName](job)++ const controller = new AbortController()++ // Store abort signal in job immediately so handlers can access it+ job.data.abortSignal = controller.signal++ // Create timeout with cleanup callback to kill orphaned processes+ return timeoutPromiseWithCleanup(+ handlers[handlerName](job),+ timeout,+ () => {+ logger.warn(+ 'Job %s in queue %s exceeded timeout of %d ms, triggering cleanup',+ job.id,+ handlerName,+ timeout+ )++ controller.abort()+ }+ )}const processor = async (jobArg: Job) => {diff --git a/server/core/lib/transcoding/hls-transcoding.ts b/server/core/lib/transcoding/hls-transcoding.tsindex 7ebb1faf24..a70a6a788c 100644--- a/server/core/lib/transcoding/hls-transcoding.ts+++ b/server/core/lib/transcoding/hls-transcoding.ts@@ -38,7 +38,7 @@ export async function generateHlsPlaylistResolutionFromTS (options: {}// Generate an HLS playlist from an input file, and update the master playlist-export function generateHlsPlaylistResolution (options: {+export function generateHlsPlaylistResolution<JobData extends { abortSignal?: AbortSignal }> (options: {video: MVideovideoInputPath: string@@ -48,7 +48,7 @@ export function generateHlsPlaylistResolution (options: {fps: numberinputFileMutexReleaser: MutexInterface.ReleaserseparatedAudio: boolean- job?: Job+ job?: Job<JobData>}) {return generateHlsPlaylistCommon({type: 'hls' as 'hls',@@ -143,7 +143,7 @@ export async function onHLSVideoFileTranscoding (options: {// Private// ----------------------------------------------------------------------------async function generateHlsPlaylistCommon (options: {+async function generateHlsPlaylistCommon<JobData extends { abortSignal?: AbortSignal }> (options: {type: 'hls' | 'hls-from-ts'video: MVideo@@ -160,7 +160,7 @@ async function generateHlsPlaylistCommon (options: {isAAC?: boolean- job?: Job+ job?: Job<JobData>}) {const {type,diff --git a/server/core/lib/transcoding/shared/ffmpeg-builder.ts b/server/core/lib/transcoding/shared/ffmpeg-builder.tsindex b28ffdb1f4..3256aa3dd3 100644--- a/server/core/lib/transcoding/shared/ffmpeg-builder.ts+++ b/server/core/lib/transcoding/shared/ffmpeg-builder.ts@@ -1,10 +1,10 @@-import { Job } from 'bullmq'+import { FFmpegVOD } from '@peertube/peertube-ffmpeg'import { getFFmpegCommandWrapperOptions } from '@server/helpers/ffmpeg/index.js'import { logger } from '@server/helpers/logger.js'-import { FFmpegVOD } from '@peertube/peertube-ffmpeg'+import { Job } from 'bullmq'import { VideoTranscodingProfilesManager } from '../default-transcoding-profiles.js'-export function buildFFmpegVOD (job?: Job) {+export function buildFFmpegVOD<T extends Job<{ abortSignal?: AbortSignal }>> (job?: T) {return new FFmpegVOD({...getFFmpegCommandWrapperOptions('vod', VideoTranscodingProfilesManager.Instance.getAvailableEncoders()),@@ -13,6 +13,9 @@ export function buildFFmpegVOD (job?: Job) {job.updateProgress(progress).catch(err => logger.error('Cannot update ffmpeg job progress', { err }))- }+ },++ // Pass the abort signal from the job so the ffmpeg process can be killed on timeout+ abortSignal: job?.data?.abortSignal})}