ActivityPub-federated video streaming platform using P2P directly in your web browser
Add ability to cancel transcoding job
Chocobozzzebf728415d8f
diff --git a/.dprint.json b/.dprint.jsonindex dbf61f9988..faa816894d 100644--- a/.dprint.json+++ b/.dprint.json@@ -14,6 +14,7 @@"trailingCommas": "never","operatorPosition": "maintain","preferHanging": false,+ "parameters.preferHanging": "always","preferSingleLine": false,"arrowFunction.useParentheses": "preferNone","binaryExpression.linePerExpression": false,diff --git a/client/src/app/+admin/system/jobs/job.service.ts b/client/src/app/+admin/system/jobs/job.service.tsindex 45550e88d0..76d3fc104e 100644--- a/client/src/app/+admin/system/jobs/job.service.ts+++ b/client/src/app/+admin/system/jobs/job.service.ts@@ -37,6 +37,13 @@ export class JobService {)}+ cancelJob (options: { jobId: string | number, jobType: JobType }) {+ const url = `${JobService.BASE_JOB_URL}/${options.jobType}/${options.jobId}/cancel`++ return this.authHttp.post(url, {})+ .pipe(catchError(err => this.restExtractor.handleError(err)))+ }+private prettyPrintData (obj: Job) {const data = JSON.stringify(obj.data, null, 2)diff --git a/client/src/app/+admin/system/jobs/jobs.component.html b/client/src/app/+admin/system/jobs/jobs.component.htmlindex 5e717a9f45..f26faecea3 100644--- a/client/src/app/+admin/system/jobs/jobs.component.html+++ b/client/src/app/+admin/system/jobs/jobs.component.html@@ -63,6 +63,21 @@<div>{{ job.finishedOn }}</div></td>}++ @if (table.isColumnDisplayed('actions')) {+ <td class="job-actions">+ @if (job.canCancel) {+ <my-button+ type="button"+ theme="secondary"+ i18n-label+ label="Cancel"+ i18n-title title="Cancel this job and abort its underlying process"+ (click)="cancelJob(job)"+ ></my-button>+ }+ </td>+ }</ng-template><ng-template #expandedRow let-job>diff --git a/client/src/app/+admin/system/jobs/jobs.component.ts b/client/src/app/+admin/system/jobs/jobs.component.tsindex bcfa7934ab..3853912f89 100644--- a/client/src/app/+admin/system/jobs/jobs.component.ts+++ b/client/src/app/+admin/system/jobs/jobs.component.ts@@ -2,8 +2,9 @@ import { CommonModule } from '@angular/common'import { Component, inject, viewChild } from '@angular/core'import { FormsModule } from '@angular/forms'import { RouterLink } from '@angular/router'-import { RestPagination, ServerService } from '@app/core'+import { ConfirmService, Notifier, RestPagination, ServerService } from '@app/core'import { AdvancedFilterDef } from '@app/shared/shared-forms/advanced-input-filter.component'+import { ButtonComponent } from '@app/shared/shared-main/buttons/button.component'import { PeerTubeBadgeService } from '@app/shared/shared-main/common/peertube-badge.service'import { Job, JobState, JobType } from '@peertube/peertube-models'import { arrayOfAllFactory } from '@peertube/peertube-typescript-utils'@@ -12,7 +13,7 @@ import { NumberFormatterPipe } from '../../../shared/shared-main/common/number-fimport { TableColumnInfo, TableComponent } from '../../../shared/shared-tables/table.component'import { JobService } from './job.service'-type ColumnName = 'id' | 'type' | 'priority' | 'state' | 'progress' | 'createdAt' | 'processed'+type ColumnName = 'id' | 'type' | 'priority' | 'state' | 'progress' | 'createdAt' | 'processed' | 'actions'type DataLoaderParameter = Parameters<JobsComponent['_dataLoader']>[0]@Component({@@ -24,13 +25,16 @@ type DataLoaderParameter = Parameters<JobsComponent['_dataLoader']>[0]FormsModule,TableComponent,NumberFormatterPipe,- RouterLink+ RouterLink,+ ButtonComponent]})export class JobsComponent {private server = inject(ServerService)private jobsService = inject(JobService)private peertubeBadgeService = inject(PeerTubeBadgeService)+ private notifier = inject(Notifier)+ private confirm = inject(ConfirmService)readonly table = viewChild<TableComponent<Job, DataLoaderParameter, ColumnName>>('table')@@ -101,7 +105,8 @@ export class JobsComponent {{ id: 'state', class: 'job-state', label: $localize`State`, sortable: false },{ id: 'progress', class: 'job-progress', label: $localize`Progress`, isDisplayed: () => this.displayGlobalProgress, sortable: false },{ id: 'createdAt', class: 'job-date', label: $localize`Created`, sortable: true },- { id: 'processed', label: $localize`Processed/Finished`, sortable: false }+ { id: 'processed', label: $localize`Processed/Finished`, sortable: false },+ { id: 'actions', label: $localize`Actions`, sortable: false }]dataLoader: typeof this._dataLoader@@ -166,6 +171,26 @@ export class JobsComponent {return this.server.isRemoteRunnersEnabled()}+ async cancelJob (job: Job) {+ if (!job.canCancel) return++ const res = await this.confirm.confirm($localize`Are you sure you want to cancel job ${job.id}?`, $localize`Cancel job`, {+ confirmButtonText: $localize`Cancel job`,+ cancelButtonText: $localize`Keep job`+ })++ if (!res) return++ this.jobsService.cancelJob({ jobId: job.id, jobType: job.type })+ .subscribe({+ next: () => {+ this.notifier.success($localize`Job ${job.id} cancellation requested`)+ this.table()?.loadData()+ },+ error: err => this.notifier.error(err.message, $localize`Cannot cancel job ${job.id}`)+ })+ }+private _dataLoader (options: {pagination: RestPaginationsort: SortMetadiff --git a/packages/models/src/server/job.model.ts b/packages/models/src/server/job.model.tsindex cd382cfad0..74e2a636dd 100644--- a/packages/models/src/server/job.model.ts+++ b/packages/models/src/server/job.model.ts@@ -48,6 +48,7 @@ export interface Job {createdAt: Date | stringfinishedOn: Date | stringprocessedOn: Date | string+ canCancel: booleanparent?: {id: string@@ -152,7 +153,6 @@ interface BaseTranscodingPayload {videoUUID: stringcanMoveVideoState: booleanisNewVideo?: boolean- abortSignal?: AbortSignal}export interface HLSTranscodingPayload extends BaseTranscodingPayload {diff --git a/packages/server-commands/src/server/jobs-command.ts b/packages/server-commands/src/server/jobs-command.tsindex 18aa0cd95a..9eed9536e2 100644--- a/packages/server-commands/src/server/jobs-command.ts+++ b/packages/server-commands/src/server/jobs-command.ts@@ -3,7 +3,6 @@ import { HttpStatusCode, Job, JobState, JobType, ResultList } from '@peertube/peimport { AbstractCommand, OverrideCommandOptions } from '../shared/index.js'export class JobsCommand extends AbstractCommand {-async getLatest (options: OverrideCommandOptions & {jobType: JobType}) {@@ -38,6 +37,21 @@ export class JobsCommand extends AbstractCommand {})}+ cancel (options: OverrideCommandOptions & {+ jobId: string | number+ jobType: JobType+ }) {+ const path = `/api/v1/jobs/${options.jobType}/${options.jobId}/cancel`++ return this.postBodyRequest({+ ...options,++ path,+ implicitToken: true,+ defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204+ })+ }+list (options: OverrideCommandOptions & {state?: JobStatejobType?: JobTypediff --git a/packages/tests/src/api/check-params/jobs.ts b/packages/tests/src/api/check-params/jobs.tsindex b4f36055f9..67b8bd5cc9 100644--- a/packages/tests/src/api/check-params/jobs.ts+++ b/packages/tests/src/api/check-params/jobs.ts@@ -1,6 +1,5 @@/* oxlint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */-import { checkBadCountPagination, checkBadSort, checkBadStartPagination } from '@tests/shared/checks.js'import { HttpStatusCode } from '@peertube/peertube-models'import {cleanupTests,@@ -10,6 +9,7 @@ import {PeerTubeServer,setAccessTokensToServers} from '@peertube/peertube-server-commands'+import { checkBadCountPagination, checkBadSort, checkBadStartPagination } from '@tests/shared/checks.js'describe('Test jobs API validators', function () {const path = '/api/v1/jobs/failed'@@ -118,6 +118,45 @@ describe('Test jobs API validators', function () {})})+ describe('When cancelling a job', function () {+ const path = '/api/v1/jobs/video-transcoding/123/cancel'++ it('Should fail with a non authenticated user', async function () {+ await makePostBodyRequest({+ url: server.url,+ path,+ expectedStatus: HttpStatusCode.UNAUTHORIZED_401+ })+ })++ it('Should fail with a non admin user', async function () {+ await makePostBodyRequest({+ url: server.url,+ path,+ token: userAccessToken,+ expectedStatus: HttpStatusCode.FORBIDDEN_403+ })+ })++ it('Should fail with a bad job type', async function () {+ await makePostBodyRequest({+ url: server.url,+ path: '/api/v1/jobs/not-a-job-type/123/cancel',+ token: server.accessToken,+ expectedStatus: HttpStatusCode.BAD_REQUEST_400+ })+ })++ it('Should fail when the job does not exist', async function () {+ await makePostBodyRequest({+ url: server.url,+ path: '/api/v1/jobs/video-transcoding/9999999/cancel',+ token: server.accessToken,+ expectedStatus: HttpStatusCode.NOT_FOUND_404+ })+ })+ })+after(async function () {await cleanupTests([ server ])})diff --git a/packages/tests/src/api/server/cancel-job.ts b/packages/tests/src/api/server/cancel-job.tsnew file mode 100644index 0000000000..d9137152a6--- /dev/null+++ b/packages/tests/src/api/server/cancel-job.ts@@ -0,0 +1,162 @@+/* oxlint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */++import { wait } from '@peertube/peertube-core-utils'+import { Job, JobState, JobType, VideoDetails, VideoPrivacy, VideoStudioTask } from '@peertube/peertube-models'+import { cleanupTests, createSingleServer, PeerTubeServer, setAccessTokensToServers } from '@peertube/peertube-server-commands'+import { expect } from 'chai'++async function waitForActiveJob (server: PeerTubeServer, uuid: string, jobType: JobType) {+ let activeJob: Job++ while (!activeJob) {+ activeJob = await findActiveJobByUUID(server, uuid, jobType)+ if (!activeJob) await wait(300)+ }++ return activeJob+}++async function waitForJobCancellation (server: PeerTubeServer, uuid: string, jobType: JobType) {+ let stillActive = true++ while (stillActive) {+ const job = await findActiveJobByUUID(server, uuid, jobType)+ stillActive = !!job++ if (stillActive) await wait(300)+ }+}++async function findActiveJobByUUID (server: PeerTubeServer, uuid: string, jobType: JobType) {+ const { data } = await server.jobs.list({ state: 'active' as JobState, jobType })++ return data.find(j => j.data?.videoUUID === uuid)+}++describe('Test cancelling running jobs', function () {+ let server: PeerTubeServer+ let video: VideoDetails++ before(async function () {+ this.timeout(120_000)++ server = await createSingleServer(1)+ await setAccessTokensToServers([ server ])++ // Enable transcoding with a single concurrent worker so the job stays in "active" long enough for us to cancel it.+ await server.config.enableTranscoding({ resolutions: 'max', hls: true, webVideo: true })+ await server.config.setTranscodingConcurrency(1)+ })++ it('Should cancel a running transcoding job and abort its ffmpeg process', async function () {+ // Upload a video without waiting for transcoding jobs+ const { uuid } = await server.videos.upload({+ attributes: {+ name: 'to cancel',+ privacy: VideoPrivacy.PUBLIC,+ fixture: 'video_high_bitrate_1080p.mp4'+ },+ waitTorrentGeneration: false+ })++ const activeJob = await waitForActiveJob(server, uuid, 'video-transcoding')++ expect(activeJob.canCancel).to.be.true++ await server.jobs.cancel({ jobId: activeJob.id, jobType: 'video-transcoding' })++ await waitForJobCancellation(server, uuid, 'video-transcoding')++ // Make sure no other transcoding job has been re-spawned for this video+ video = await server.videos.get({ id: uuid })++ const playlistFiles = video.streamingPlaylists.flatMap(p => p.files)+ expect(playlistFiles).to.have.lengthOf(0)+ })++ // --------------------------------------------------------------------------+ // Transcription cancel+ // --------------------------------------------------------------------------++ it('Should cancel a running transcription job', async function () {+ await server.config.enableTranscription()++ const { uuid } = await server.videos.upload({+ attributes: {+ name: 'to cancel transcription',+ privacy: VideoPrivacy.PUBLIC,+ fixture: 'video_high_bitrate_1080p.mp4'+ },+ waitTorrentGeneration: false+ })++ const activeJob = await waitForActiveJob(server, uuid, 'video-transcription')++ expect(activeJob.canCancel).to.be.true++ await server.jobs.cancel({ jobId: activeJob.id, jobType: 'video-transcription' })++ await waitForJobCancellation(server, uuid, 'video-transcription')+ })++ // --------------------------------------------------------------------------+ // Storyboard cancel+ // --------------------------------------------------------------------------++ it('Should cancel a running storyboard generation job and abort its ffmpeg process', async function () {+ // Upload a long video so storyboard generation takes enough time+ const { uuid } = await server.videos.upload({+ attributes: {+ name: 'to cancel storyboard',+ privacy: VideoPrivacy.PUBLIC,+ fixture: 'video_very_long_10p.mp4'+ },+ waitTorrentGeneration: false+ })++ const activeJob = await waitForActiveJob(server, uuid, 'generate-video-storyboard')++ expect(activeJob.canCancel).to.be.true++ await server.jobs.cancel({ jobId: activeJob.id, jobType: 'generate-video-storyboard' })++ await waitForJobCancellation(server, uuid, 'generate-video-storyboard')+ })++ // --------------------------------------------------------------------------+ // Studio edition cancel+ // --------------------------------------------------------------------------++ it('Should cancel a running video studio edition job and abort its ffmpeg process', async function () {+ await server.config.disableTranscoding()++ // Upload a video and wait for initial processing+ const { uuid, id } = await server.videos.quickUpload({ name: 'to cancel studio', fixture: 'video_short.webm' })+ await server.config.enableMinimumTranscoding()+ await server.config.enableStudio()++ // Trigger a simple cut task that requires ffmpeg processing+ const tasks: VideoStudioTask[] = [+ {+ name: 'add-intro',+ options: {+ file: 'video_very_long_10p.mp4'+ }+ }+ ]++ await server.videoStudio.createEditionTasks({ videoId: id, tasks })++ const activeJob = await waitForActiveJob(server, uuid, 'video-studio-edition')++ expect(activeJob.canCancel).to.be.true++ await server.jobs.cancel({ jobId: activeJob.id, jobType: 'video-studio-edition' })++ await waitForJobCancellation(server, uuid, 'video-studio-edition')+ })++ after(async function () {+ await cleanupTests([ server ])+ })+})diff --git a/packages/transcription/src/abstract-transcriber.ts b/packages/transcription/src/abstract-transcriber.tsindex 181a305c06..6c708686b5 100644--- a/packages/transcription/src/abstract-transcriber.ts+++ b/packages/transcription/src/abstract-transcriber.ts@@ -16,6 +16,7 @@ export interface TranscribeArgs {language?: stringrunId?: string+ signal?: AbortSignal}export abstract class AbstractTranscriber {@@ -76,7 +77,12 @@ export abstract class AbstractTranscriber {return this.engine.command}- protected getExec (env?: { [id: string]: string }) {+ protected getExec (options: {+ signal?: AbortSignal+ env?: { [id: string]: string }+ } = {}) {+ const { signal, env } = options+const logLevels = {command: 'debug',output: 'debug',@@ -92,6 +98,8 @@ export abstract class AbstractTranscriber {this.logger[level](message, verboseObject)},+ cancelSignal: signal,+env})}diff --git a/packages/transcription/src/whisper/transcriber/ctranslate2-transcriber.ts b/packages/transcription/src/whisper/transcriber/ctranslate2-transcriber.tsindex 6a55b10e32..8500c995af 100644--- a/packages/transcription/src/whisper/transcriber/ctranslate2-transcriber.ts+++ b/packages/transcription/src/whisper/transcriber/ctranslate2-transcriber.ts@@ -8,18 +8,18 @@ import { WhisperBuiltinModel } from '../whisper-builtin-model.js'import { OpenaiTranscriber } from './openai-transcriber.js'export class Ctranslate2Transcriber extends OpenaiTranscriber {-async transcribe ({mediaFilePath,model = new WhisperBuiltinModel('tiny'),language,format,transcriptDirectory,- runId = buildSUUID()+ runId = buildSUUID(),+ signal}: TranscribeArgs): Promise<TranscriptFile> {this.assertLanguageDetectionAvailable(language)- const $$ = this.getExec(this.getExecEnv())+ const $$ = this.getExec({ signal, env: this.getExecEnv() })if (model.path) {assert(await lstat(model.path).then(stats => stats.isDirectory()), 'Model path must be a path to a directory.')diff --git a/packages/transcription/src/whisper/transcriber/openai-transcriber.ts b/packages/transcription/src/whisper/transcriber/openai-transcriber.tsindex afad29c48e..2396a8b4f2 100644--- a/packages/transcription/src/whisper/transcriber/openai-transcriber.ts+++ b/packages/transcription/src/whisper/transcriber/openai-transcriber.ts@@ -6,18 +6,18 @@ import { AbstractTranscriber, TranscribeArgs } from '../../abstract-transcriber.import { TranscriptFile, TranscriptFormat } from '../../transcript-file.js'export class OpenaiTranscriber extends AbstractTranscriber {-async transcribe ({mediaFilePath,model,language,format,transcriptDirectory,- runId = buildSUUID()+ runId = buildSUUID(),+ signal}: TranscribeArgs): Promise<TranscriptFile> {this.assertLanguageDetectionAvailable(language)- const $$ = this.getExec(this.getExecEnv())+ const $$ = this.getExec({ signal, env: this.getExecEnv() })const languageArgs = language ? [ '--language', language ] : []diff --git a/server/core/controllers/api/jobs.ts b/server/core/controllers/api/jobs.tsindex ffaca4a9e9..91f02f5078 100644--- a/server/core/controllers/api/jobs.ts+++ b/server/core/controllers/api/jobs.ts@@ -7,6 +7,7 @@ import {apiRateLimiter,asyncMiddleware,authenticate,+ cancelJobValidator,ensureUserHasRight,jobsSortValidator,openapiOperationDoc,@@ -20,19 +21,22 @@ const jobsRouter = express.Router()jobsRouter.use(apiRateLimiter)-jobsRouter.post('/pause',+jobsRouter.post(+ '/pause',authenticate,ensureUserHasRight(UserRight.MANAGE_JOBS),asyncMiddleware(pauseJobQueue))-jobsRouter.post('/resume',+jobsRouter.post(+ '/resume',authenticate,ensureUserHasRight(UserRight.MANAGE_JOBS),resumeJobQueue)-jobsRouter.get('/:state?',+jobsRouter.get(+ '/:state?',openapiOperationDoc({ operationId: 'getJobs' }),authenticate,ensureUserHasRight(UserRight.MANAGE_JOBS),@@ -44,6 +48,15 @@ jobsRouter.get('/:state?',asyncMiddleware(listJobs))+jobsRouter.post(+ '/:jobType/:jobId/cancel',+ openapiOperationDoc({ operationId: 'cancelJob' }),+ authenticate,+ ensureUserHasRight(UserRight.MANAGE_JOBS),+ cancelJobValidator,+ cancelJob+)+// ---------------------------------------------------------------------------export {@@ -64,6 +77,14 @@ function resumeJobQueue (req: express.Request, res: express.Response) {return res.sendStatus(HttpStatusCode.NO_CONTENT_204)}+function cancelJob (req: express.Request, res: express.Response) {+ const job = res.locals.job++ JobQueue.Instance.cancelJob(job.queueName as JobType, job)++ return res.sendStatus(HttpStatusCode.NO_CONTENT_204)+}+async function listJobs (req: express.Request, res: express.Response) {const state = req.params.state as JobStateconst asc = req.query.sort === 'createdAt'@@ -100,7 +121,8 @@ async function formatJob (job: BullJob, state?: JobState): Promise<Job> {error: getJobError(job),createdAt: new Date(job.timestamp),finishedOn: new Date(job.finishedOn),- processedOn: new Date(job.processedOn)+ processedOn: new Date(job.processedOn),+ canCancel: await JobQueue.Instance.canCancelJob(job.queueName as JobType, job)}}diff --git a/server/core/lib/job-queue/handlers/generate-storyboard.ts b/server/core/lib/job-queue/handlers/generate-storyboard.tsindex c1e52dc293..04012cca29 100644--- a/server/core/lib/job-queue/handlers/generate-storyboard.ts+++ b/server/core/lib/job-queue/handlers/generate-storyboard.ts@@ -10,90 +10,101 @@ import { VideoModel } from '@server/models/video/video.js'import { Job } from 'bullmq'import { join } from 'path'import { buildSpriteSize, buildTotalSprites, findGridSize, insertStoryboardInDatabase } from '../../storyboard.js'+import { buildPromiseForAbortSignal } from './shared/job-helpers.js'const lTagsBase = loggerTagsFactory('storyboard')-export async function processGenerateStoryboard (job: Job): Promise<void> {- const payload = job.data as GenerateStoryboardPayload- const lTags = lTagsBase(payload.videoUUID)+export async function processGenerateStoryboard (job: Job, abortSignal?: AbortSignal) {+ const abortPromise = buildPromiseForAbortSignal(abortSignal)- logger.info(`Processing generate storyboard of ${payload.videoUUID} in job ${job.id}.`, lTags)+ const run = async () => {+ const payload = job.data as GenerateStoryboardPayload+ const lTags = lTagsBase(payload.videoUUID)- if (CONFIG.STORYBOARDS.ENABLED !== true) {- logger.info(`Storyboard disabled, do not process storyboard of ${payload.videoUUID} in job ${job.id}.`, lTags)- return- }-- const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(payload.videoUUID)+ logger.info(`Processing generate storyboard of ${payload.videoUUID} in job ${job.id}.`, lTags)- try {- const video = await VideoModel.loadFull(payload.videoUUID)- if (!video) {- logger.info(`Video ${payload.videoUUID} does not exist anymore, skipping storyboard generation.`, lTags)+ if (CONFIG.STORYBOARDS.ENABLED !== true) {+ logger.info(`Storyboard disabled, do not process storyboard of ${payload.videoUUID} in job ${job.id}.`, lTags)return}- const inputFile = video.getMaxQualityFile(VideoFileStream.VIDEO)- if (!inputFile) {- logger.info(`Do not generate a storyboard of ${payload.videoUUID} since the video does not have a video stream`, lTags)- return- }+ const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(payload.videoUUID)- await VideoPathManager.Instance.makeAvailableVideoFile(inputFile, async videoPath => {- const { spriteHeight, spriteWidth } = await buildSpriteSize(videoPath)+ try {+ const video = await VideoModel.loadFull(payload.videoUUID)+ if (!video) {+ logger.info(`Video ${payload.videoUUID} does not exist anymore, skipping storyboard generation.`, lTags)+ return+ }- const { totalSprites, spriteDuration } = buildTotalSprites(video)- if (totalSprites === 0) {- logger.info(`Do not generate a storyboard of ${payload.videoUUID} because the video is not long enough`, lTags)+ const inputFile = video.getMaxQualityFile(VideoFileStream.VIDEO)+ if (!inputFile) {+ logger.info(`Do not generate a storyboard of ${payload.videoUUID} since the video does not have a video stream`, lTags)return}- const spritesCount = findGridSize({- toFind: totalSprites,- maxEdgeCount: STORYBOARD.SPRITES_MAX_EDGE_COUNT- })+ await VideoPathManager.Instance.makeAvailableVideoFile(inputFile, async videoPath => {+ const { spriteHeight, spriteWidth } = await buildSpriteSize(videoPath)- const filename = generateImageFilename()- const destination = join(CONFIG.STORAGE.STORYBOARDS_DIR, filename)-- logger.debug(- `Generating storyboard from video of ${video.uuid} to ${destination}`,- { ...lTags, totalSprites, spritesCount, spriteDuration, videoDuration: video.duration, spriteHeight, spriteWidth }- )-- const ffmpeg = new FFmpegImage(getFFmpegCommandWrapperOptions('thumbnail'))-- await ffmpeg.generateStoryboardFromVideo({- destination,- path: videoPath,- inputFileMutexReleaser,- sprites: {- size: {- height: spriteHeight,- width: spriteWidth- },- count: spritesCount,- duration: spriteDuration+ const { totalSprites, spriteDuration } = buildTotalSprites(video)+ if (totalSprites === 0) {+ logger.info(`Do not generate a storyboard of ${payload.videoUUID} because the video is not long enough`, lTags)+ return}- })-- await insertStoryboardInDatabase({- videoUUID: video.uuid,- lTags,-- filename,- destination,-- imageSize: await getImageSize(destination),- spriteHeight,- spriteWidth,- spriteDuration,-- federate: payload.federate+ const spritesCount = findGridSize({+ toFind: totalSprites,+ maxEdgeCount: STORYBOARD.SPRITES_MAX_EDGE_COUNT+ })++ const filename = generateImageFilename()+ const destination = join(CONFIG.STORAGE.STORYBOARDS_DIR, filename)++ logger.debug(+ `Generating storyboard from video of ${video.uuid} to ${destination}`,+ { ...lTags, totalSprites, spritesCount, spriteDuration, videoDuration: video.duration, spriteHeight, spriteWidth }+ )++ const ffmpeg = new FFmpegImage({+ ...getFFmpegCommandWrapperOptions('thumbnail'),++ abortSignal+ })++ await ffmpeg.generateStoryboardFromVideo({+ destination,+ path: videoPath,+ inputFileMutexReleaser,+ sprites: {+ size: {+ height: spriteHeight,+ width: spriteWidth+ },+ count: spritesCount,+ duration: spriteDuration+ }+ })++ await insertStoryboardInDatabase({+ videoUUID: video.uuid,+ lTags,++ filename,+ destination,++ imageSize: await getImageSize(destination),++ spriteHeight,+ spriteWidth,+ spriteDuration,++ federate: payload.federate+ })})- })- } finally {- inputFileMutexReleaser()+ } finally {+ inputFileMutexReleaser()+ }}++ return Promise.race([ run(), abortPromise ])}diff --git a/server/core/lib/job-queue/handlers/shared/job-helpers.ts b/server/core/lib/job-queue/handlers/shared/job-helpers.tsnew file mode 100644index 0000000000..1870d1c798--- /dev/null+++ b/server/core/lib/job-queue/handlers/shared/job-helpers.ts@@ -0,0 +1,13 @@+import { UnrecoverableError } from 'bullmq'++export function buildPromiseForAbortSignal (abortSignal?: AbortSignal) {+ return new Promise((_res, rej) => {+ if (abortSignal?.aborted) {+ return rej(new UnrecoverableError('Job has been canceled'))+ }++ abortSignal?.addEventListener('abort', () => {+ rej(new UnrecoverableError('Job has been canceled'))+ })+ })+}diff --git a/server/core/lib/job-queue/handlers/video-studio-edition.ts b/server/core/lib/job-queue/handlers/video-studio-edition.tsindex c7c217e2c2..97221c7b5f 100644--- a/server/core/lib/job-queue/handlers/video-studio-edition.ts+++ b/server/core/lib/job-queue/handlers/video-studio-edition.ts@@ -27,89 +27,98 @@ import { Job } from 'bullmq'import { remove } from 'fs-extra/esm'import { extname, join } from 'path'import { logger, loggerTagsFactory } from '../../../helpers/logger.js'+import { buildPromiseForAbortSignal } from './shared/job-helpers.js'const lTagsBase = loggerTagsFactory('video-studio')-async function processVideoStudioEdition (job: Job) {- const payload = job.data as VideoStudioEditionPayload- const lTags = lTagsBase(payload.videoUUID)+async function processVideoStudioEdition (job: Job, abortSignal?: AbortSignal) {+ const abortPromise = buildPromiseForAbortSignal(abortSignal)- logger.info('Process video studio edition of %s in job %s.', payload.videoUUID, job.id, lTags)+ const run = async () => {+ const payload = job.data as VideoStudioEditionPayload+ const lTags = lTagsBase(payload.videoUUID)- let inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(payload.videoUUID)+ logger.info('Process video studio edition of %s in job %s.', payload.videoUUID, job.id, lTags)- try {- const video = await VideoModel.loadFull(payload.videoUUID)+ let inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(payload.videoUUID)- // No video, maybe deleted?- if (!video) {- logger.info('Can\'t process job %d, video does not exist.', job.id, lTags)+ try {+ const video = await VideoModel.loadFull(payload.videoUUID)- await safeCleanupStudioTMPFiles(payload.tasks)- return undefined- }+ // No video, maybe deleted?+ if (!video) {+ logger.info('Can\'t process job %d, video does not exist.', job.id, lTags)- await checkUserQuotaOrThrow(video, payload)+ await safeCleanupStudioTMPFiles(payload.tasks)+ return undefined+ }- await video.reload()+ await checkUserQuotaOrThrow(video, payload)- const editionResultPath = await VideoPathManager.Instance.makeAvailableMaxQualityFiles(video, async ({- videoPath: originalVideoFilePath,- separatedAudioPath- }) => {- let tmpInputFilePath: string- let outputPath: string+ await video.reload()- for (const task of payload.tasks) {- const outputFilename = buildUUID() + extname(originalVideoFilePath)- outputPath = join(CONFIG.STORAGE.TMP_DIR, outputFilename)+ const editionResultPath = await VideoPathManager.Instance.makeAvailableMaxQualityFiles(video, async ({+ videoPath: originalVideoFilePath,+ separatedAudioPath+ }) => {+ let tmpInputFilePath: string+ let outputPath: string- await processTask({- videoInputPath: tmpInputFilePath ?? originalVideoFilePath,+ for (const task of payload.tasks) {+ const outputFilename = buildUUID() + extname(originalVideoFilePath)+ outputPath = join(CONFIG.STORAGE.TMP_DIR, outputFilename)- separatedAudioInputPath: tmpInputFilePath- ? undefined- : separatedAudioPath,+ await processTask({+ videoInputPath: tmpInputFilePath ?? originalVideoFilePath,- inputFileMutexReleaser,+ separatedAudioInputPath: tmpInputFilePath+ ? undefined+ : separatedAudioPath,- video,- outputPath,- task,- lTags- })+ inputFileMutexReleaser,- if (tmpInputFilePath) await remove(tmpInputFilePath)+ video,+ outputPath,+ task,+ lTags,- // For the next iteration- tmpInputFilePath = outputPath- inputFileMutexReleaser = undefined- }+ abortSignal+ })- return outputPath- })+ if (tmpInputFilePath) await remove(tmpInputFilePath)- logger.info('Video edition ended for video %s.', video.uuid, lTags)+ // For the next iteration+ tmpInputFilePath = outputPath+ inputFileMutexReleaser = undefined+ }- await onVideoStudioEnded({ video, editionResultPath, tasks: payload.tasks })- } catch (err) {- await safeCleanupStudioTMPFiles(payload.tasks)+ return outputPath+ })- try {- await sequelizeTypescript.transaction(async transaction => {- const video = await VideoModel.load(payload.videoUUID, transaction)- if (!video || video.state === VideoState.PUBLISHED) return+ logger.info('Video edition ended for video %s.', video.uuid, lTags)- await video.setNewState(VideoState.PUBLISHED, false, transaction)- })+ await onVideoStudioEnded({ video, editionResultPath, tasks: payload.tasks })} catch (err) {- logger.error('Cannot reset video state after studio error', { err, ...lTags })- }+ await safeCleanupStudioTMPFiles(payload.tasks)++ try {+ await sequelizeTypescript.transaction(async transaction => {+ const video = await VideoModel.load(payload.videoUUID, transaction)+ if (!video || video.state === VideoState.PUBLISHED) return++ await video.setNewState(VideoState.PUBLISHED, false, transaction)+ })+ } catch (err) {+ logger.error('Cannot reset video state after studio error', { err, ...lTags })+ }- throw err- } finally {- if (inputFileMutexReleaser) inputFileMutexReleaser()+ throw err+ } finally {+ if (inputFileMutexReleaser) inputFileMutexReleaser()+ }}++ return Promise.race([ run(), abortPromise ])}// ---------------------------------------------------------------------------@@ -130,6 +139,8 @@ type TaskProcessorOptions<T extends VideoStudioTaskPayload = VideoStudioTaskPaylvideo: MVideotask: TlTags: { tags: (string | number)[] }++ abortSignal?: AbortSignal}const taskProcessors: { [id in VideoStudioTask['name']]: (options: TaskProcessorOptions) => Promise<any> } = {@@ -152,11 +163,11 @@ async function processTask (options: TaskProcessorOptions) {}function processAddIntroOutro (options: TaskProcessorOptions<VideoStudioTaskIntroPayload | VideoStudioTaskOutroPayload>) {- const { task, lTags } = options+ const { task, lTags, abortSignal } = optionslogger.debug('Will add intro/outro to the video.', { options, ...lTags })- return buildFFmpegEdition().addIntroOutro({+ return buildFFmpegEdition(abortSignal).addIntroOutro({...pick(options, [ 'inputFileMutexReleaser', 'videoInputPath', 'separatedAudioInputPath', 'outputPath' ]),introOutroPath: task.options.file,@@ -167,11 +178,11 @@ function processAddIntroOutro (options: TaskProcessorOptions<VideoStudioTaskIntr}function processCut (options: TaskProcessorOptions<VideoStudioTaskCutPayload>) {- const { task, lTags } = options+ const { task, lTags, abortSignal } = optionslogger.debug('Will cut the video.', { options, ...lTags })- return buildFFmpegEdition().cutVideo({+ return buildFFmpegEdition(abortSignal).cutVideo({...pick(options, [ 'inputFileMutexReleaser', 'videoInputPath', 'separatedAudioInputPath', 'outputPath' ]),start: task.options.start,@@ -180,11 +191,11 @@ function processCut (options: TaskProcessorOptions<VideoStudioTaskCutPayload>) {}function processRemoveSegments (options: TaskProcessorOptions<VideoStudioTaskRemoveSegmentsPayload>) {- const { task, lTags } = options+ const { task, lTags, abortSignal } = optionslogger.debug('Will remove segments from the video.', { options, ...lTags })- return buildFFmpegEdition().removeSegments({+ return buildFFmpegEdition(abortSignal).removeSegments({...pick(options, [ 'inputFileMutexReleaser', 'videoInputPath', 'separatedAudioInputPath', 'outputPath' ]),segments: task.options.segments@@ -192,11 +203,11 @@ function processRemoveSegments (options: TaskProcessorOptions<VideoStudioTaskRem}function processAddWatermark (options: TaskProcessorOptions<VideoStudioTaskWatermarkPayload>) {- const { task, lTags } = options+ const { task, lTags, abortSignal } = optionslogger.debug('Will add watermark to the video.', { options, ...lTags })- return buildFFmpegEdition().addWatermark({+ return buildFFmpegEdition(abortSignal).addWatermark({...pick(options, [ 'inputFileMutexReleaser', 'videoInputPath', 'separatedAudioInputPath', 'outputPath' ]),watermarkPath: task.options.file,@@ -222,6 +233,10 @@ async function checkUserQuotaOrThrow (video: MVideoFull, payload: VideoStudioEdi}}-function buildFFmpegEdition () {- return new FFmpegEdition(getFFmpegCommandWrapperOptions('vod', VideoTranscodingProfilesManager.Instance.getAvailableEncoders()))+function buildFFmpegEdition (abortSignal?: AbortSignal) {+ return new FFmpegEdition({+ ...getFFmpegCommandWrapperOptions('vod', VideoTranscodingProfilesManager.Instance.getAvailableEncoders()),++ abortSignal+ })}diff --git a/server/core/lib/job-queue/handlers/video-transcoding.ts b/server/core/lib/job-queue/handlers/video-transcoding.tsindex 9316dbfa2b..db26505534 100644--- a/server/core/lib/job-queue/handlers/video-transcoding.ts+++ b/server/core/lib/job-queue/handlers/video-transcoding.ts@@ -16,12 +16,19 @@ import { VideoPathManager } from '@server/lib/video-path-manager.js'import { moveToFailedTranscodingState } from '@server/lib/video-state.js'import { UserModel } from '@server/models/user/user.js'import { VideoJobInfoModel } from '@server/models/video/video-job-info.js'-import { MUser, MUserId, MVideoFull } from '@server/types/models/index.js'+import { MUser, MVideoFull } from '@server/types/models/index.js'import { Job } from 'bullmq'import { logger, loggerTagsFactory } from '../../../helpers/logger.js'import { VideoModel } from '../../../models/video/video.js'+import { buildPromiseForAbortSignal } from './shared/job-helpers.js'-type HandlerFunction = (job: Job, payload: VideoTranscodingPayload, video: MVideoFull, user: MUser) => Promise<void>+type HandlerFunction = (options: {+ job: Job+ payload: VideoTranscodingPayload+ video: MVideoFull+ user: MUser+ abortSignal: AbortSignal+}) => Promise<void>const handlers: { [id in VideoTranscodingPayload['type']]: HandlerFunction } = {'new-resolution-to-hls': handleHLSJob,@@ -32,39 +39,45 @@ const handlers: { [id in VideoTranscodingPayload['type']]: HandlerFunction } = {const lTags = loggerTagsFactory('transcoding')-async function processVideoTranscoding (job: Job) {- const payload = job.data as VideoTranscodingPayload- logger.info('Processing transcoding job %s.', job.id, lTags(payload.videoUUID))+async function processVideoTranscoding (job: Job, abortSignal: AbortSignal) {+ const abortPromise = buildPromiseForAbortSignal(abortSignal)- const video = await VideoModel.loadFull(payload.videoUUID)- // No video, maybe deleted?- if (!video) {- logger.info(`Do not process job ${job.id}, video does not exist.`, lTags(payload.videoUUID))- return undefined- }+ const run = async () => {+ const payload = job.data as VideoTranscodingPayload+ logger.info('Processing transcoding job %s.', job.id, lTags(payload.videoUUID))- const user = await UserModel.loadByChannelActorId(video.VideoChannel.Actor.id)+ const video = await VideoModel.loadFull(payload.videoUUID)+ // No video, maybe deleted?+ if (!video) {+ logger.info(`Do not process job ${job.id}, video does not exist.`, lTags(payload.videoUUID))+ return undefined+ }- const handler = handlers[payload.type]+ const user = await UserModel.loadByChannelActorId(video.VideoChannel.Actor.id)- if (!handler) {- await moveToFailedTranscodingState(video)- await VideoJobInfoModel.decrease(video.uuid, 'pendingTranscode')+ const handler = handlers[payload.type]- throw new Error('Cannot find transcoding handler for ' + payload.type)- }+ if (!handler) {+ await moveToFailedTranscodingState(video)+ await VideoJobInfoModel.decrease(video.uuid, 'pendingTranscode')- try {- await handler(job, payload, video, user)- } catch (error) {- await moveToFailedTranscodingState(video)+ throw new Error('Cannot find transcoding handler for ' + payload.type)+ }- await VideoJobInfoModel.decrease(video.uuid, 'pendingTranscode')+ try {+ await handler({ job, payload, video, user, abortSignal })+ } catch (error) {+ await moveToFailedTranscodingState(video)- throw error+ await VideoJobInfoModel.decrease(video.uuid, 'pendingTranscode')++ throw error+ }++ return video}- return video+ return Promise.race([ run(), abortPromise ])}// ---------------------------------------------------------------------------@@ -77,30 +90,36 @@ export {// Job handlers// ----------------------------------------------------------------------------async function handleWebVideoMergeAudioJob (- job: Job<MergeAudioTranscodingPayload>,- payload: MergeAudioTranscodingPayload,- video: MVideoFull,- user: MUserId-) {+async function handleWebVideoMergeAudioJob (options: {+ job: Job<MergeAudioTranscodingPayload>+ payload: MergeAudioTranscodingPayload+ video: MVideoFull+ user: MUser+ abortSignal: AbortSignal+}) {+ const { job, payload, video, abortSignal } = options+logger.info('Handling merge audio transcoding job for %s.', video.uuid, lTags(video.uuid), { payload })- await mergeAudioVideofile({ video, resolution: payload.resolution, fps: payload.fps, job })+ await mergeAudioVideofile({ video, resolution: payload.resolution, fps: payload.fps, job, abortSignal })logger.info('Merge audio transcoding job for %s ended.', video.uuid, lTags(video.uuid), { payload })await onTranscodingEnded({ isNewVideo: payload.isNewVideo, moveVideoToNextState: payload.canMoveVideoState, video })}-async function handleWebVideoOptimizeJob (- job: Job<OptimizeTranscodingPayload>,- payload: OptimizeTranscodingPayload,- video: MVideoFull,- user: MUserId-) {+async function handleWebVideoOptimizeJob (options: {+ job: Job<OptimizeTranscodingPayload>+ payload: OptimizeTranscodingPayload+ video: MVideoFull+ user: MUser+ abortSignal: AbortSignal+}) {+ const { job, payload, video, abortSignal } = options+logger.info('Handling optimize transcoding job for %s.', video.uuid, lTags(video.uuid), { payload })- await optimizeOriginalVideofile({ video, job })+ await optimizeOriginalVideofile({ video, job, abortSignal })logger.info('Optimize transcoding job for %s ended.', video.uuid, lTags(video.uuid), { payload })@@ -109,14 +128,18 @@ async function handleWebVideoOptimizeJob (// ----------------------------------------------------------------------------async function handleNewWebVideoResolutionJob (- job: Job<NewWebVideoResolutionTranscodingPayload>,- payload: NewWebVideoResolutionTranscodingPayload,+async function handleNewWebVideoResolutionJob (options: {+ job: Job<NewWebVideoResolutionTranscodingPayload>+ payload: NewWebVideoResolutionTranscodingPayloadvideo: MVideoFull-) {+ user: MUser+ abortSignal: AbortSignal+}) {+ const { job, payload, video, abortSignal } = options+logger.info('Handling Web Video transcoding job for %s.', video.uuid, lTags(video.uuid), { payload })- await transcodeNewWebVideoResolution({ video, resolution: payload.resolution, fps: payload.fps, job })+ await transcodeNewWebVideoResolution({ video, resolution: payload.resolution, fps: payload.fps, job, abortSignal })logger.info('Web Video transcoding job for %s ended.', video.uuid, lTags(video.uuid), { payload })@@ -126,7 +149,15 @@ async function handleNewWebVideoResolutionJob (// ----------------------------------------------------------------------------async function handleHLSJob (job: Job<HLSTranscodingPayload>, payload: HLSTranscodingPayload, videoArg: MVideoFull) {+async function handleHLSJob (options: {+ job: Job<HLSTranscodingPayload>+ payload: HLSTranscodingPayload+ video: MVideoFull+ user: MUser+ abortSignal: AbortSignal+}) {+ const { job, payload, video: videoArg, abortSignal } = options+logger.info('Handling HLS transcoding job for %s.', videoArg.uuid, lTags(videoArg.uuid), { payload })const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(videoArg.uuid)@@ -153,7 +184,8 @@ async function handleHLSJob (job: Job<HLSTranscodingPayload>, payload: HLSTranscresolution: payload.resolution,fps: payload.fps,separatedAudio: payload.separatedAudio,- job+ job,+ abortSignal})})} finally {diff --git a/server/core/lib/job-queue/handlers/video-transcription.ts b/server/core/lib/job-queue/handlers/video-transcription.tsindex 47ae8e3374..027a3b7da8 100644--- a/server/core/lib/job-queue/handlers/video-transcription.ts+++ b/server/core/lib/job-queue/handlers/video-transcription.ts@@ -3,19 +3,26 @@ import { generateSubtitle } from '@server/lib/video-captions.js'import { Job } from 'bullmq'import { logger, loggerTagsFactory } from '../../../helpers/logger.js'import { VideoModel } from '../../../models/video/video.js'+import { buildPromiseForAbortSignal } from './shared/job-helpers.js'const lTags = loggerTagsFactory('transcription')-export async function processVideoTranscription (job: Job) {- const payload = job.data as VideoTranscriptionPayload+export async function processVideoTranscription (job: Job, abortSignal?: AbortSignal) {+ const abortPromise = buildPromiseForAbortSignal(abortSignal)- logger.info('Processing video transcription in job %s.', job.id)+ const run = async () => {+ const payload = job.data as VideoTranscriptionPayload- const video = await VideoModel.load(payload.videoUUID)- if (!video) {- logger.info('Do not process transcription job %d, video does not exist.', job.id, lTags(payload.videoUUID))- return+ logger.info('Processing video transcription in job %s.', job.id)++ const video = await VideoModel.load(payload.videoUUID)+ if (!video) {+ logger.info('Do not process transcription job %d, video does not exist.', job.id, lTags(payload.videoUUID))+ return+ }++ return generateSubtitle({ video, signal: abortSignal })}- return generateSubtitle({ video })+ return Promise.race([ run(), abortPromise ])}diff --git a/server/core/lib/job-queue/job-queue.ts b/server/core/lib/job-queue/job-queue.tsindex 638990db7f..0ee7cbb6a6 100644--- a/server/core/lib/job-queue/job-queue.ts+++ b/server/core/lib/job-queue/job-queue.ts@@ -1,4 +1,4 @@-import { pick, timeoutPromiseWithCleanup } from '@peertube/peertube-core-utils'+import { pick } from '@peertube/peertube-core-utils'import {ActivitypubFollowPayload,ActivitypubHttpBroadcastPayload,@@ -116,7 +116,7 @@ export type CreateJobOptions = {deduplicationId?: string}-const handlers: { [id in JobType]: (job: Job) => Promise<any> } = {+const handlers: { [id in JobType]: (job: Job, signal?: AbortSignal) => Promise<any> } = {'build-automatic-tags': processBuildAutomaticTags,'activitypub-cleaner': processActivityPubCleaner,'activitypub-follow': processActivityPubFollow,@@ -185,6 +185,8 @@ const jobTypes: JobType[] = ['video-transcoding']+const cancelableJobTypes: JobType[] = [ 'video-transcoding', 'video-transcription', 'video-studio-edition', 'generate-video-storyboard' ]+const silentFailure = new Set<JobType>([ 'activitypub-http-unicast' ])class JobQueue {@@ -233,6 +235,8 @@ class JobQueue {}private buildWorker (handlerName: JobType) {+ let worker: Worker+const workerOptions: WorkerOptions = {autorun: false,concurrency: this.getJobConcurrency(handlerName),@@ -241,39 +245,27 @@ class JobQueue {maxStalledCount: 10}- const handler = function (job: Job) {+ const handler = function (options: { job: Job, signal: AbortSignal }) {+ const { job, signal } = options+const timeout = JOB_TTL[handlerName]- if (!timeout) return handlers[handlerName](job)-- const controller = new AbortController()-- // Store abort signal in job immediately so handlers can access it- job.data.abortSignal = controller.signal-- // Create timeout with cleanup callback to kill orphaned processes- return timeoutPromiseWithCleanup(- handlers[handlerName](job),- timeout,- () => {- logger.warn(- 'Job %s in queue %s exceeded timeout of %d ms, triggering cleanup',- job.id,- handlerName,- timeout- )-- controller.abort()- }- )+ if (!timeout) return handlers[handlerName](job, signal)++ const timeoutId = setTimeout(() => {+ worker.cancelJob(job.id, 'Timeout exceeded')+ }, timeout)++ return handlers[handlerName](job, signal)+ .finally(() => clearTimeout(timeoutId))}- const processor = async (jobArg: Job) => {+ const processor = async (jobArg: Job, _, signal: AbortSignal) => {const job = await Hooks.wrapObject(jobArg, 'filter:job-queue.process.params', { type: handlerName })- return Hooks.wrapPromiseFun(handler, job, 'filter:job-queue.process.result')+ return Hooks.wrapPromiseFun(handler, { job, signal }, 'filter:job-queue.process.result')}- const worker = new Worker(handlerName, processor, workerOptions)+ worker = new Worker(handlerName, processor, workerOptions)worker.on('failed', (job, err) => {const logLevel = silentFailure.has(handlerName)@@ -520,6 +512,30 @@ class JobQueue {return total}+ async getJob (jobType: JobType, jobId: string): Promise<Job> {+ const queue = this.queues[jobType]+ if (!queue) throw new Error(`Unknown queue ${jobType}`)++ return queue.getJob(jobId)+ }++ async canCancelJob (jobType: JobType, job: Job) {+ if (!cancelableJobTypes.includes(jobType)) return false++ const isActive = await job.isActive()++ return isActive+ }++ cancelJob (jobType: JobType, job: Job) {+ logger.info('Cancelling job %s in queue %s.', job.id, job.queueName)++ const worker = this.workers[jobType]+ if (!worker) throw new Error(`Unknown queue ${jobType}`)++ return worker.cancelJob(job.id, 'Job cancelled by admin')+ }+private buildStateFilter (state?: JobState) {if (!state) return Array.from(jobStates)diff --git a/server/core/lib/transcoding/hls-transcoding.ts b/server/core/lib/transcoding/hls-transcoding.tsindex a70a6a788c..cb3b62891b 100644--- a/server/core/lib/transcoding/hls-transcoding.ts+++ b/server/core/lib/transcoding/hls-transcoding.ts@@ -38,7 +38,7 @@ export async function generateHlsPlaylistResolutionFromTS (options: {}// Generate an HLS playlist from an input file, and update the master playlist-export function generateHlsPlaylistResolution<JobData extends { abortSignal?: AbortSignal }> (options: {+export function generateHlsPlaylistResolution (options: {video: MVideovideoInputPath: string@@ -48,7 +48,9 @@ export function generateHlsPlaylistResolution<JobData extends { abortSignal?: Abfps: numberinputFileMutexReleaser: MutexInterface.ReleaserseparatedAudio: boolean- job?: Job<JobData>++ job: Job+ abortSignal: AbortSignal}) {return generateHlsPlaylistCommon({type: 'hls' as 'hls',@@ -61,7 +63,8 @@ export function generateHlsPlaylistResolution<JobData extends { abortSignal?: Ab'fps','separatedAudio','inputFileMutexReleaser',- 'job'+ 'job',+ 'abortSignal'])})}@@ -143,7 +146,7 @@ export async function onHLSVideoFileTranscoding (options: {// Private// ----------------------------------------------------------------------------async function generateHlsPlaylistCommon<JobData extends { abortSignal?: AbortSignal }> (options: {+async function generateHlsPlaylistCommon (options: {type: 'hls' | 'hls-from-ts'video: MVideo@@ -160,7 +163,8 @@ async function generateHlsPlaylistCommon<JobData extends { abortSignal?: AbortSiisAAC?: boolean- job?: Job<JobData>+ job?: Job+ abortSignal?: AbortSignal}) {const {type,@@ -173,7 +177,8 @@ async function generateHlsPlaylistCommon<JobData extends { abortSignal?: AbortSiisAAC,job,inputFileMutexReleaser,- preventInputFileLocking+ preventInputFileLocking,+ abortSignal} = optionsconst transcodeDirectory = CONFIG.STORAGE.TMP_DIR@@ -211,7 +216,7 @@ async function generateHlsPlaylistCommon<JobData extends { abortSignal?: AbortSi}}- await buildFFmpegVOD(job).transcode(transcodeOptions)+ await buildFFmpegVOD({ job, abortSignal }).transcode(transcodeOptions)// Ensure the mutex is released if the ffmpeg command failed and did not release itif (inputFileMutexReleaser) inputFileMutexReleaser()diff --git a/server/core/lib/transcoding/shared/ffmpeg-builder.ts b/server/core/lib/transcoding/shared/ffmpeg-builder.tsindex 3256aa3dd3..acb51619c5 100644--- a/server/core/lib/transcoding/shared/ffmpeg-builder.ts+++ b/server/core/lib/transcoding/shared/ffmpeg-builder.ts@@ -4,7 +4,12 @@ import { logger } from '@server/helpers/logger.js'import { Job } from 'bullmq'import { VideoTranscodingProfilesManager } from '../default-transcoding-profiles.js'-export function buildFFmpegVOD<T extends Job<{ abortSignal?: AbortSignal }>> (job?: T) {+export function buildFFmpegVOD (options: {+ job?: Job+ abortSignal?: AbortSignal+} = {}) {+ const { job, abortSignal } = options+return new FFmpegVOD({...getFFmpegCommandWrapperOptions('vod', VideoTranscodingProfilesManager.Instance.getAvailableEncoders()),@@ -16,6 +21,6 @@ export function buildFFmpegVOD<T extends Job<{ abortSignal?: AbortSignal }>> (jo},// Pass the abort signal from the job so the ffmpeg process can be killed on timeout- abortSignal: job?.data?.abortSignal+ abortSignal})}diff --git a/server/core/lib/transcoding/web-transcoding.ts b/server/core/lib/transcoding/web-transcoding.tsindex af0683ab76..c14d519388 100644--- a/server/core/lib/transcoding/web-transcoding.ts+++ b/server/core/lib/transcoding/web-transcoding.ts@@ -28,8 +28,9 @@ import { buildOriginalFileResolution } from './transcoding-resolutions.js'export async function optimizeOriginalVideofile (options: {video: MVideoFulljob: Job+ abortSignal: AbortSignal}) {- const { job } = options+ const { job, abortSignal } = optionsconst transcodeDirectory = CONFIG.STORAGE.TMP_DIRconst newExtname = '.mp4'@@ -52,7 +53,7 @@ export async function optimizeOriginalVideofile (options: {const fps = computeOutputFPS({ inputFPS: inputVideoFile.fps, resolution, isOriginResolution: true, type: 'vod' })// Could be very long!- await buildFFmpegVOD(job).transcode({+ await buildFFmpegVOD({ job, abortSignal }).transcode({type: transcodeType,videoInputPath,@@ -81,8 +82,9 @@ export async function transcodeNewWebVideoResolution (options: {resolution: numberfps: numberjob: Job+ abortSignal: AbortSignal}) {- const { video: videoArg, resolution, fps, job } = options+ const { video: videoArg, resolution, fps, job, abortSignal } = optionsconst transcodeDirectory = CONFIG.STORAGE.TMP_DIRconst newExtname = '.mp4'@@ -110,7 +112,7 @@ export async function transcodeNewWebVideoResolution (options: {fps}- await buildFFmpegVOD(job).transcode(transcodeOptions)+ await buildFFmpegVOD({ job, abortSignal }).transcode(transcodeOptions)return onWebVideoFileTranscoding({ video, videoOutputPath })})@@ -127,8 +129,9 @@ export async function mergeAudioVideofile (options: {resolution: numberfps: numberjob: Job+ abortSignal: AbortSignal}) {- const { video: videoArg, resolution, fps, job } = options+ const { video: videoArg, resolution, fps, job, abortSignal } = optionsconst transcodeDirectory = CONFIG.STORAGE.TMP_DIRconst newExtname = '.mp4'@@ -162,7 +165,7 @@ export async function mergeAudioVideofile (options: {}try {- await buildFFmpegVOD(job).transcode(transcodeOptions)+ await buildFFmpegVOD({ job, abortSignal }).transcode(transcodeOptions)await remove(tmpThumbnailPath)} catch (err) {diff --git a/server/core/lib/video-captions.ts b/server/core/lib/video-captions.tsindex 08a4af675d..d796096ee5 100644--- a/server/core/lib/video-captions.ts+++ b/server/core/lib/video-captions.ts@@ -141,6 +141,7 @@ let transcriber: AbstractTranscriberexport async function generateSubtitle (options: {video: MVideoUUID+ signal?: AbortSignal}) {const outputPath = join(CONFIG.STORAGE.TMP_DIR, 'transcription', buildSUUID())@@ -200,7 +201,9 @@ export async function generateSubtitle (options: {transcriptDirectory: outputPath,- format: 'vtt'+ format: 'vtt',++ signal: options.signal})const refreshedVideo = await VideoModel.loadFull(video.uuid)diff --git a/server/core/middlewares/validators/jobs.ts b/server/core/middlewares/validators/jobs.tsindex 070cef32e7..53d34c8781 100644--- a/server/core/middlewares/validators/jobs.ts+++ b/server/core/middlewares/validators/jobs.ts@@ -1,12 +1,14 @@+import { HttpStatusCode, JobType } from '@peertube/peertube-models'import express from 'express'import { param, query } from 'express-validator'import { isValidJobState, isValidJobType } from '../../helpers/custom-validators/jobs.js'import { loggerTagsFactory } from '../../helpers/logger.js'+import { JobQueue } from '../../lib/job-queue/index.js'import { areValidationErrors } from './shared/index.js'const lTags = loggerTagsFactory('validators', 'jobs')-const listJobsValidator = [+export const listJobsValidator = [param('state').optional().custom(isValidJobState),@@ -22,8 +24,28 @@ const listJobsValidator = [}]-// ---------------------------------------------------------------------------+export const cancelJobValidator = [+ param('jobType').custom(isValidJobType),+ param('jobId').isString().not().isEmpty(),-export {- listJobsValidator-}+ async (req: express.Request, res: express.Response, next: express.NextFunction) => {+ if (areValidationErrors(req, res, lTags())) return++ const jobType = req.params.jobType as JobType+ const jobId = req.params.jobId++ const job = await JobQueue.Instance.getJob(jobType, jobId)+ if (!job) return res.sendStatus(HttpStatusCode.NOT_FOUND_404)++ if (!await JobQueue.Instance.canCancelJob(job.queueName as JobType, job)) {+ return res.fail({+ status: HttpStatusCode.CONFLICT_409,+ message: req.t(`This job cannot be cancelled`)+ })+ }++ res.locals.job = job++ return next()+ }+]diff --git a/server/core/types/express.d.ts b/server/core/types/express.d.tsindex d35389ebe9..f292e7bd19 100644--- a/server/core/types/express.d.ts+++ b/server/core/types/express.d.ts@@ -6,6 +6,7 @@ import {ServerLogLevel,VideoCreate} from '@peertube/peertube-models'+import { SignupMode } from '@server/lib/signup.ts'import { RegisterServerAuthExternalOptions } from '@server/types/index.js'import {MAbuseMessage,@@ -13,6 +14,8 @@ import {MAccountBlocklist,MActorFollowActorsDefault,MActorUrl,+ MBlocklistSubscription,+ MChangeOwnershipFull,MChannelBannerAccountDefault,MChannelCollaboratorAccount,MChannelSyncChannel,@@ -21,7 +24,6 @@ import {MStreamingPlaylist,MUserAccountUrl,MUserExport,- MChangeOwnershipFull,MVideoEmbedDomain,MVideoFile,MVideoFormattableDetails,@@ -36,8 +38,7 @@ import {MVideoWithBlacklist,MVideoWithRights,MWatchedWordsList,- MWatchedWordsSubscription,- MBlocklistSubscription+ MWatchedWordsSubscription} from '@server/types/models/index.js'import { MOAuthToken, MOAuthTokenUser } from '@server/types/models/oauth/oauth-token.js'import { MPlugin, MServer, MServerBlocklist } from '@server/types/models/server.js'@@ -45,6 +46,7 @@ import { MVideoImportDefault } from '@server/types/models/video/video-import.js'import { MVideoPlaylistElement, MVideoPlaylistElementVideoUrlPlaylistPrivacy } from '@server/types/models/video/video-playlist-element.js'import { MAccountVideoRateAccountVideo } from '@server/types/models/video/video-rate.js'import { Metadata, File as UploadXFile } from '@uploadx/core'+import { Job as BullJob } from 'bullmq'import { FfprobeData } from 'fluent-ffmpeg'import { OutgoingHttpHeaders } from 'http'import { Writable } from 'stream'@@ -65,7 +67,6 @@ import {} from './models/index.js'import { MRunner, MRunnerJobRunner, MRunnerRegistrationToken } from './models/runners/index.js'import { MVideoSource } from './models/video/video-source.js'-import { SignupMode } from '@server/lib/signup.ts'declare module 'express' {export interface Request {@@ -268,6 +269,8 @@ declare module 'express' {channelCollaborator?: MChannelCollaboratorAccountblocklistSubscription?: MBlocklistSubscription++ job?: BullJob}}}diff --git a/support/doc/api/openapi.yaml b/support/doc/api/openapi.yamlindex 8c536a9dba..4e55ed0dad 100644--- a/support/doc/api/openapi.yaml+++ b/support/doc/api/openapi.yaml@@ -1234,6 +1234,37 @@ paths:items:$ref: '#/components/schemas/Job'+ /api/v1/jobs/{jobType}/{jobId}/cancel:+ post:+ summary: Cancel a job+ description: Abort a running job and its underlying ffmpeg/child process. Only active jobs that support cancellation can be cancelled.+ operationId: cancelJob+ security:+ - OAuth2:+ - admin+ tags:+ - Job+ parameters:+ - name: jobType+ in: path+ required: true+ description: The type of the job to cancel+ schema:+ type: string+ - name: jobId+ in: path+ required: true+ description: The id of the job to cancel+ schema:+ type: string+ responses:+ '204':+ description: successful operation+ '404':+ description: job not found+ '409':+ description: job cannot be cancelled (not active or not cancellable)+/api/v1/server/followers:get:tags:@@ -12336,6 +12367,9 @@ components:processedOn:type: stringformat: date-time+ canCancel:+ description: Whether this job is currently running and supports cancellation+ type: booleanAddUserResponse:properties:user: