Skip to content

ActivityPub-federated video streaming platform using P2P directly in your web browser

Add ability to cancel transcoding job

Chocobozzzebf728415d8f
diff --git a/.dprint.json b/.dprint.json
index dbf61f9988..faa816894d 100644
--- a/.dprint.json
+++ b/.dprint.json
@@ -14,6 +14,7 @@
"trailingCommas": "never",
"operatorPosition": "maintain",
"preferHanging": false,
+ "parameters.preferHanging": "always",
"preferSingleLine": false,
"arrowFunction.useParentheses": "preferNone",
"binaryExpression.linePerExpression": false,
diff --git a/client/src/app/+admin/system/jobs/job.service.ts b/client/src/app/+admin/system/jobs/job.service.ts
index 45550e88d0..76d3fc104e 100644
--- a/client/src/app/+admin/system/jobs/job.service.ts
+++ b/client/src/app/+admin/system/jobs/job.service.ts
@@ -37,6 +37,13 @@ export class JobService {
)
}
+ cancelJob (options: { jobId: string | number, jobType: JobType }) {
+ const url = `${JobService.BASE_JOB_URL}/${options.jobType}/${options.jobId}/cancel`
+
+ return this.authHttp.post(url, {})
+ .pipe(catchError(err => this.restExtractor.handleError(err)))
+ }
+
private prettyPrintData (obj: Job) {
const data = JSON.stringify(obj.data, null, 2)
diff --git a/client/src/app/+admin/system/jobs/jobs.component.html b/client/src/app/+admin/system/jobs/jobs.component.html
index 5e717a9f45..f26faecea3 100644
--- a/client/src/app/+admin/system/jobs/jobs.component.html
+++ b/client/src/app/+admin/system/jobs/jobs.component.html
@@ -63,6 +63,21 @@
<div>{{ job.finishedOn }}</div>
</td>
}
+
+ @if (table.isColumnDisplayed('actions')) {
+ <td class="job-actions">
+ @if (job.canCancel) {
+ <my-button
+ type="button"
+ theme="secondary"
+ i18n-label
+ label="Cancel"
+ i18n-title title="Cancel this job and abort its underlying process"
+ (click)="cancelJob(job)"
+ ></my-button>
+ }
+ </td>
+ }
</ng-template>
<ng-template #expandedRow let-job>
diff --git a/client/src/app/+admin/system/jobs/jobs.component.ts b/client/src/app/+admin/system/jobs/jobs.component.ts
index bcfa7934ab..3853912f89 100644
--- a/client/src/app/+admin/system/jobs/jobs.component.ts
+++ b/client/src/app/+admin/system/jobs/jobs.component.ts
@@ -2,8 +2,9 @@ import { CommonModule } from '@angular/common'
import { Component, inject, viewChild } from '@angular/core'
import { FormsModule } from '@angular/forms'
import { RouterLink } from '@angular/router'
-import { RestPagination, ServerService } from '@app/core'
+import { ConfirmService, Notifier, RestPagination, ServerService } from '@app/core'
import { AdvancedFilterDef } from '@app/shared/shared-forms/advanced-input-filter.component'
+import { ButtonComponent } from '@app/shared/shared-main/buttons/button.component'
import { PeerTubeBadgeService } from '@app/shared/shared-main/common/peertube-badge.service'
import { Job, JobState, JobType } from '@peertube/peertube-models'
import { arrayOfAllFactory } from '@peertube/peertube-typescript-utils'
@@ -12,7 +13,7 @@ import { NumberFormatterPipe } from '../../../shared/shared-main/common/number-f
import { TableColumnInfo, TableComponent } from '../../../shared/shared-tables/table.component'
import { JobService } from './job.service'
-type ColumnName = 'id' | 'type' | 'priority' | 'state' | 'progress' | 'createdAt' | 'processed'
+type ColumnName = 'id' | 'type' | 'priority' | 'state' | 'progress' | 'createdAt' | 'processed' | 'actions'
type DataLoaderParameter = Parameters<JobsComponent['_dataLoader']>[0]
@Component({
@@ -24,13 +25,16 @@ type DataLoaderParameter = Parameters<JobsComponent['_dataLoader']>[0]
FormsModule,
TableComponent,
NumberFormatterPipe,
- RouterLink
+ RouterLink,
+ ButtonComponent
]
})
export class JobsComponent {
private server = inject(ServerService)
private jobsService = inject(JobService)
private peertubeBadgeService = inject(PeerTubeBadgeService)
+ private notifier = inject(Notifier)
+ private confirm = inject(ConfirmService)
readonly table = viewChild<TableComponent<Job, DataLoaderParameter, ColumnName>>('table')
@@ -101,7 +105,8 @@ export class JobsComponent {
{ id: 'state', class: 'job-state', label: $localize`State`, sortable: false },
{ id: 'progress', class: 'job-progress', label: $localize`Progress`, isDisplayed: () => this.displayGlobalProgress, sortable: false },
{ id: 'createdAt', class: 'job-date', label: $localize`Created`, sortable: true },
- { id: 'processed', label: $localize`Processed/Finished`, sortable: false }
+ { id: 'processed', label: $localize`Processed/Finished`, sortable: false },
+ { id: 'actions', label: $localize`Actions`, sortable: false }
]
dataLoader: typeof this._dataLoader
@@ -166,6 +171,26 @@ export class JobsComponent {
return this.server.isRemoteRunnersEnabled()
}
+ async cancelJob (job: Job) {
+ if (!job.canCancel) return
+
+ const res = await this.confirm.confirm($localize`Are you sure you want to cancel job ${job.id}?`, $localize`Cancel job`, {
+ confirmButtonText: $localize`Cancel job`,
+ cancelButtonText: $localize`Keep job`
+ })
+
+ if (!res) return
+
+ this.jobsService.cancelJob({ jobId: job.id, jobType: job.type })
+ .subscribe({
+ next: () => {
+ this.notifier.success($localize`Job ${job.id} cancellation requested`)
+ this.table()?.loadData()
+ },
+ error: err => this.notifier.error(err.message, $localize`Cannot cancel job ${job.id}`)
+ })
+ }
+
private _dataLoader (options: {
pagination: RestPagination
sort: SortMeta
diff --git a/packages/models/src/server/job.model.ts b/packages/models/src/server/job.model.ts
index cd382cfad0..74e2a636dd 100644
--- a/packages/models/src/server/job.model.ts
+++ b/packages/models/src/server/job.model.ts
@@ -48,6 +48,7 @@ export interface Job {
createdAt: Date | string
finishedOn: Date | string
processedOn: Date | string
+ canCancel: boolean
parent?: {
id: string
@@ -152,7 +153,6 @@ interface BaseTranscodingPayload {
videoUUID: string
canMoveVideoState: boolean
isNewVideo?: boolean
- abortSignal?: AbortSignal
}
export interface HLSTranscodingPayload extends BaseTranscodingPayload {
diff --git a/packages/server-commands/src/server/jobs-command.ts b/packages/server-commands/src/server/jobs-command.ts
index 18aa0cd95a..9eed9536e2 100644
--- a/packages/server-commands/src/server/jobs-command.ts
+++ b/packages/server-commands/src/server/jobs-command.ts
@@ -3,7 +3,6 @@ import { HttpStatusCode, Job, JobState, JobType, ResultList } from '@peertube/pe
import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js'
export class JobsCommand extends AbstractCommand {
-
async getLatest (options: OverrideCommandOptions & {
jobType: JobType
}) {
@@ -38,6 +37,21 @@ export class JobsCommand extends AbstractCommand {
})
}
+ cancel (options: OverrideCommandOptions & {
+ jobId: string | number
+ jobType: JobType
+ }) {
+ const path = `/api/v1/jobs/${options.jobType}/${options.jobId}/cancel`
+
+ return this.postBodyRequest({
+ ...options,
+
+ path,
+ implicitToken: true,
+ defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204
+ })
+ }
+
list (options: OverrideCommandOptions & {
state?: JobState
jobType?: JobType
diff --git a/packages/tests/src/api/check-params/jobs.ts b/packages/tests/src/api/check-params/jobs.ts
index b4f36055f9..67b8bd5cc9 100644
--- a/packages/tests/src/api/check-params/jobs.ts
+++ b/packages/tests/src/api/check-params/jobs.ts
@@ -1,6 +1,5 @@
/* oxlint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
-import { checkBadCountPagination, checkBadSort, checkBadStartPagination } from '@tests/shared/checks.js'
import { HttpStatusCode } from '@peertube/peertube-models'
import {
cleanupTests,
@@ -10,6 +9,7 @@ import {
PeerTubeServer,
setAccessTokensToServers
} from '@peertube/peertube-server-commands'
+import { checkBadCountPagination, checkBadSort, checkBadStartPagination } from '@tests/shared/checks.js'
describe('Test jobs API validators', function () {
const path = '/api/v1/jobs/failed'
@@ -118,6 +118,45 @@ describe('Test jobs API validators', function () {
})
})
+ describe('When cancelling a job', function () {
+ const path = '/api/v1/jobs/video-transcoding/123/cancel'
+
+ it('Should fail with a non authenticated user', async function () {
+ await makePostBodyRequest({
+ url: server.url,
+ path,
+ expectedStatus: HttpStatusCode.UNAUTHORIZED_401
+ })
+ })
+
+ it('Should fail with a non admin user', async function () {
+ await makePostBodyRequest({
+ url: server.url,
+ path,
+ token: userAccessToken,
+ expectedStatus: HttpStatusCode.FORBIDDEN_403
+ })
+ })
+
+ it('Should fail with a bad job type', async function () {
+ await makePostBodyRequest({
+ url: server.url,
+ path: '/api/v1/jobs/not-a-job-type/123/cancel',
+ token: server.accessToken,
+ expectedStatus: HttpStatusCode.BAD_REQUEST_400
+ })
+ })
+
+ it('Should fail when the job does not exist', async function () {
+ await makePostBodyRequest({
+ url: server.url,
+ path: '/api/v1/jobs/video-transcoding/9999999/cancel',
+ token: server.accessToken,
+ expectedStatus: HttpStatusCode.NOT_FOUND_404
+ })
+ })
+ })
+
after(async function () {
await cleanupTests([ server ])
})
diff --git a/packages/tests/src/api/server/cancel-job.ts b/packages/tests/src/api/server/cancel-job.ts
new file mode 100644
index 0000000000..d9137152a6
--- /dev/null
+++ b/packages/tests/src/api/server/cancel-job.ts
@@ -0,0 +1,162 @@
+/* oxlint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
+
+import { wait } from '@peertube/peertube-core-utils'
+import { Job, JobState, JobType, VideoDetails, VideoPrivacy, VideoStudioTask } from '@peertube/peertube-models'
+import { cleanupTests, createSingleServer, PeerTubeServer, setAccessTokensToServers } from '@peertube/peertube-server-commands'
+import { expect } from 'chai'
+
+async function waitForActiveJob (server: PeerTubeServer, uuid: string, jobType: JobType) {
+ let activeJob: Job
+
+ while (!activeJob) {
+ activeJob = await findActiveJobByUUID(server, uuid, jobType)
+ if (!activeJob) await wait(300)
+ }
+
+ return activeJob
+}
+
+async function waitForJobCancellation (server: PeerTubeServer, uuid: string, jobType: JobType) {
+ let stillActive = true
+
+ while (stillActive) {
+ const job = await findActiveJobByUUID(server, uuid, jobType)
+ stillActive = !!job
+
+ if (stillActive) await wait(300)
+ }
+}
+
+async function findActiveJobByUUID (server: PeerTubeServer, uuid: string, jobType: JobType) {
+ const { data } = await server.jobs.list({ state: 'active' as JobState, jobType })
+
+ return data.find(j => j.data?.videoUUID === uuid)
+}
+
+describe('Test cancelling running jobs', function () {
+ let server: PeerTubeServer
+ let video: VideoDetails
+
+ before(async function () {
+ this.timeout(120_000)
+
+ server = await createSingleServer(1)
+ await setAccessTokensToServers([ server ])
+
+ // Enable transcoding with a single concurrent worker so the job stays in "active" long enough for us to cancel it.
+ await server.config.enableTranscoding({ resolutions: 'max', hls: true, webVideo: true })
+ await server.config.setTranscodingConcurrency(1)
+ })
+
+ it('Should cancel a running transcoding job and abort its ffmpeg process', async function () {
+ // Upload a video without waiting for transcoding jobs
+ const { uuid } = await server.videos.upload({
+ attributes: {
+ name: 'to cancel',
+ privacy: VideoPrivacy.PUBLIC,
+ fixture: 'video_high_bitrate_1080p.mp4'
+ },
+ waitTorrentGeneration: false
+ })
+
+ const activeJob = await waitForActiveJob(server, uuid, 'video-transcoding')
+
+ expect(activeJob.canCancel).to.be.true
+
+ await server.jobs.cancel({ jobId: activeJob.id, jobType: 'video-transcoding' })
+
+ await waitForJobCancellation(server, uuid, 'video-transcoding')
+
+ // Make sure no other transcoding job has been re-spawned for this video
+ video = await server.videos.get({ id: uuid })
+
+ const playlistFiles = video.streamingPlaylists.flatMap(p => p.files)
+ expect(playlistFiles).to.have.lengthOf(0)
+ })
+
+ // --------------------------------------------------------------------------
+ // Transcription cancel
+ // --------------------------------------------------------------------------
+
+ it('Should cancel a running transcription job', async function () {
+ await server.config.enableTranscription()
+
+ const { uuid } = await server.videos.upload({
+ attributes: {
+ name: 'to cancel transcription',
+ privacy: VideoPrivacy.PUBLIC,
+ fixture: 'video_high_bitrate_1080p.mp4'
+ },
+ waitTorrentGeneration: false
+ })
+
+ const activeJob = await waitForActiveJob(server, uuid, 'video-transcription')
+
+ expect(activeJob.canCancel).to.be.true
+
+ await server.jobs.cancel({ jobId: activeJob.id, jobType: 'video-transcription' })
+
+ await waitForJobCancellation(server, uuid, 'video-transcription')
+ })
+
+ // --------------------------------------------------------------------------
+ // Storyboard cancel
+ // --------------------------------------------------------------------------
+
+ it('Should cancel a running storyboard generation job and abort its ffmpeg process', async function () {
+ // Upload a long video so storyboard generation takes enough time
+ const { uuid } = await server.videos.upload({
+ attributes: {
+ name: 'to cancel storyboard',
+ privacy: VideoPrivacy.PUBLIC,
+ fixture: 'video_very_long_10p.mp4'
+ },
+ waitTorrentGeneration: false
+ })
+
+ const activeJob = await waitForActiveJob(server, uuid, 'generate-video-storyboard')
+
+ expect(activeJob.canCancel).to.be.true
+
+ await server.jobs.cancel({ jobId: activeJob.id, jobType: 'generate-video-storyboard' })
+
+ await waitForJobCancellation(server, uuid, 'generate-video-storyboard')
+ })
+
+ // --------------------------------------------------------------------------
+ // Studio edition cancel
+ // --------------------------------------------------------------------------
+
+ it('Should cancel a running video studio edition job and abort its ffmpeg process', async function () {
+ await server.config.disableTranscoding()
+
+ // Upload a video and wait for initial processing
+ const { uuid, id } = await server.videos.quickUpload({ name: 'to cancel studio', fixture: 'video_short.webm' })
+ await server.config.enableMinimumTranscoding()
+ await server.config.enableStudio()
+
+ // Trigger a simple cut task that requires ffmpeg processing
+ const tasks: VideoStudioTask[] = [
+ {
+ name: 'add-intro',
+ options: {
+ file: 'video_very_long_10p.mp4'
+ }
+ }
+ ]
+
+ await server.videoStudio.createEditionTasks({ videoId: id, tasks })
+
+ const activeJob = await waitForActiveJob(server, uuid, 'video-studio-edition')
+
+ expect(activeJob.canCancel).to.be.true
+
+ await server.jobs.cancel({ jobId: activeJob.id, jobType: 'video-studio-edition' })
+
+ await waitForJobCancellation(server, uuid, 'video-studio-edition')
+ })
+
+ after(async function () {
+ await cleanupTests([ server ])
+ })
+})
diff --git a/packages/transcription/src/abstract-transcriber.ts b/packages/transcription/src/abstract-transcriber.ts
index 181a305c06..6c708686b5 100644
--- a/packages/transcription/src/abstract-transcriber.ts
+++ b/packages/transcription/src/abstract-transcriber.ts
@@ -16,6 +16,7 @@ export interface TranscribeArgs {
language?: string
runId?: string
+ signal?: AbortSignal
}
export abstract class AbstractTranscriber {
@@ -76,7 +77,12 @@ export abstract class AbstractTranscriber {
return this.engine.command
}
- protected getExec (env?: { [id: string]: string }) {
+ protected getExec (options: {
+ signal?: AbortSignal
+ env?: { [id: string]: string }
+ } = {}) {
+ const { signal, env } = options
+
const logLevels = {
command: 'debug',
output: 'debug',
@@ -92,6 +98,8 @@ export abstract class AbstractTranscriber {
this.logger[level](message, verboseObject)
},
+ cancelSignal: signal,
+
env
})
}
diff --git a/packages/transcription/src/whisper/transcriber/ctranslate2-transcriber.ts b/packages/transcription/src/whisper/transcriber/ctranslate2-transcriber.ts
index 6a55b10e32..8500c995af 100644
--- a/packages/transcription/src/whisper/transcriber/ctranslate2-transcriber.ts
+++ b/packages/transcription/src/whisper/transcriber/ctranslate2-transcriber.ts
@@ -8,18 +8,18 @@ import { WhisperBuiltinModel } from '../whisper-builtin-model.js'
import { OpenaiTranscriber } from './openai-transcriber.js'
export class Ctranslate2Transcriber extends OpenaiTranscriber {
-
async transcribe ({
mediaFilePath,
model = new WhisperBuiltinModel('tiny'),
language,
format,
transcriptDirectory,
- runId = buildSUUID()
+ runId = buildSUUID(),
+ signal
}: TranscribeArgs): Promise<TranscriptFile> {
this.assertLanguageDetectionAvailable(language)
- const $$ = this.getExec(this.getExecEnv())
+ const $$ = this.getExec({ signal, env: this.getExecEnv() })
if (model.path) {
assert(await lstat(model.path).then(stats => stats.isDirectory()), 'Model path must be a path to a directory.')
diff --git a/packages/transcription/src/whisper/transcriber/openai-transcriber.ts b/packages/transcription/src/whisper/transcriber/openai-transcriber.ts
index afad29c48e..2396a8b4f2 100644
--- a/packages/transcription/src/whisper/transcriber/openai-transcriber.ts
+++ b/packages/transcription/src/whisper/transcriber/openai-transcriber.ts
@@ -6,18 +6,18 @@ import { AbstractTranscriber, TranscribeArgs } from '../../abstract-transcriber.
import { TranscriptFile, TranscriptFormat } from '../../transcript-file.js'
export class OpenaiTranscriber extends AbstractTranscriber {
-
async transcribe ({
mediaFilePath,
model,
language,
format,
transcriptDirectory,
- runId = buildSUUID()
+ runId = buildSUUID(),
+ signal
}: TranscribeArgs): Promise<TranscriptFile> {
this.assertLanguageDetectionAvailable(language)
- const $$ = this.getExec(this.getExecEnv())
+ const $$ = this.getExec({ signal, env: this.getExecEnv() })
const languageArgs = language ? [ '--language', language ] : []
diff --git a/server/core/controllers/api/jobs.ts b/server/core/controllers/api/jobs.ts
index ffaca4a9e9..91f02f5078 100644
--- a/server/core/controllers/api/jobs.ts
+++ b/server/core/controllers/api/jobs.ts
@@ -7,6 +7,7 @@ import {
apiRateLimiter,
asyncMiddleware,
authenticate,
+ cancelJobValidator,
ensureUserHasRight,
jobsSortValidator,
openapiOperationDoc,
@@ -20,19 +21,22 @@ const jobsRouter = express.Router()
jobsRouter.use(apiRateLimiter)
-jobsRouter.post('/pause',
+jobsRouter.post(
+ '/pause',
authenticate,
ensureUserHasRight(UserRight.MANAGE_JOBS),
asyncMiddleware(pauseJobQueue)
)
-jobsRouter.post('/resume',
+jobsRouter.post(
+ '/resume',
authenticate,
ensureUserHasRight(UserRight.MANAGE_JOBS),
resumeJobQueue
)
-jobsRouter.get('/:state?',
+jobsRouter.get(
+ '/:state?',
openapiOperationDoc({ operationId: 'getJobs' }),
authenticate,
ensureUserHasRight(UserRight.MANAGE_JOBS),
@@ -44,6 +48,15 @@ jobsRouter.get('/:state?',
asyncMiddleware(listJobs)
)
+jobsRouter.post(
+ '/:jobType/:jobId/cancel',
+ openapiOperationDoc({ operationId: 'cancelJob' }),
+ authenticate,
+ ensureUserHasRight(UserRight.MANAGE_JOBS),
+ cancelJobValidator,
+ cancelJob
+)
+
// ---------------------------------------------------------------------------
export {
@@ -64,6 +77,14 @@ function resumeJobQueue (req: express.Request, res: express.Response) {
return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
}
+function cancelJob (req: express.Request, res: express.Response) {
+ const job = res.locals.job
+
+ JobQueue.Instance.cancelJob(job.queueName as JobType, job)
+
+ return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
+}
+
async function listJobs (req: express.Request, res: express.Response) {
const state = req.params.state as JobState
const asc = req.query.sort === 'createdAt'
@@ -100,7 +121,8 @@ async function formatJob (job: BullJob, state?: JobState): Promise<Job> {
error: getJobError(job),
createdAt: new Date(job.timestamp),
finishedOn: new Date(job.finishedOn),
- processedOn: new Date(job.processedOn)
+ processedOn: new Date(job.processedOn),
+ canCancel: await JobQueue.Instance.canCancelJob(job.queueName as JobType, job)
}
}
diff --git a/server/core/lib/job-queue/handlers/generate-storyboard.ts b/server/core/lib/job-queue/handlers/generate-storyboard.ts
index c1e52dc293..04012cca29 100644
--- a/server/core/lib/job-queue/handlers/generate-storyboard.ts
+++ b/server/core/lib/job-queue/handlers/generate-storyboard.ts
@@ -10,90 +10,101 @@ import { VideoModel } from '@server/models/video/video.js'
import { Job } from 'bullmq'
import { join } from 'path'
import { buildSpriteSize, buildTotalSprites, findGridSize, insertStoryboardInDatabase } from '../../storyboard.js'
+import { buildPromiseForAbortSignal } from './shared/job-helpers.js'
const lTagsBase = loggerTagsFactory('storyboard')
-export async function processGenerateStoryboard (job: Job): Promise<void> {
- const payload = job.data as GenerateStoryboardPayload
- const lTags = lTagsBase(payload.videoUUID)
+export async function processGenerateStoryboard (job: Job, abortSignal?: AbortSignal) {
+ const abortPromise = buildPromiseForAbortSignal(abortSignal)
- logger.info(`Processing generate storyboard of ${payload.videoUUID} in job ${job.id}.`, lTags)
+ const run = async () => {
+ const payload = job.data as GenerateStoryboardPayload
+ const lTags = lTagsBase(payload.videoUUID)
- if (CONFIG.STORYBOARDS.ENABLED !== true) {
- logger.info(`Storyboard disabled, do not process storyboard of ${payload.videoUUID} in job ${job.id}.`, lTags)
- return
- }
-
- const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(payload.videoUUID)
+ logger.info(`Processing generate storyboard of ${payload.videoUUID} in job ${job.id}.`, lTags)
- try {
- const video = await VideoModel.loadFull(payload.videoUUID)
- if (!video) {
- logger.info(`Video ${payload.videoUUID} does not exist anymore, skipping storyboard generation.`, lTags)
+ if (CONFIG.STORYBOARDS.ENABLED !== true) {
+ logger.info(`Storyboard disabled, do not process storyboard of ${payload.videoUUID} in job ${job.id}.`, lTags)
return
}
- const inputFile = video.getMaxQualityFile(VideoFileStream.VIDEO)
- if (!inputFile) {
- logger.info(`Do not generate a storyboard of ${payload.videoUUID} since the video does not have a video stream`, lTags)
- return
- }
+ const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(payload.videoUUID)
- await VideoPathManager.Instance.makeAvailableVideoFile(inputFile, async videoPath => {
- const { spriteHeight, spriteWidth } = await buildSpriteSize(videoPath)
+ try {
+ const video = await VideoModel.loadFull(payload.videoUUID)
+ if (!video) {
+ logger.info(`Video ${payload.videoUUID} does not exist anymore, skipping storyboard generation.`, lTags)
+ return
+ }
- const { totalSprites, spriteDuration } = buildTotalSprites(video)
- if (totalSprites === 0) {
- logger.info(`Do not generate a storyboard of ${payload.videoUUID} because the video is not long enough`, lTags)
+ const inputFile = video.getMaxQualityFile(VideoFileStream.VIDEO)
+ if (!inputFile) {
+ logger.info(`Do not generate a storyboard of ${payload.videoUUID} since the video does not have a video stream`, lTags)
return
}
- const spritesCount = findGridSize({
- toFind: totalSprites,
- maxEdgeCount: STORYBOARD.SPRITES_MAX_EDGE_COUNT
- })
+ await VideoPathManager.Instance.makeAvailableVideoFile(inputFile, async videoPath => {
+ const { spriteHeight, spriteWidth } = await buildSpriteSize(videoPath)
- const filename = generateImageFilename()
- const destination = join(CONFIG.STORAGE.STORYBOARDS_DIR, filename)
-
- logger.debug(
- `Generating storyboard from video of ${video.uuid} to ${destination}`,
- { ...lTags, totalSprites, spritesCount, spriteDuration, videoDuration: video.duration, spriteHeight, spriteWidth }
- )
-
- const ffmpeg = new FFmpegImage(getFFmpegCommandWrapperOptions('thumbnail'))
-
- await ffmpeg.generateStoryboardFromVideo({
- destination,
- path: videoPath,
- inputFileMutexReleaser,
- sprites: {
- size: {
- height: spriteHeight,
- width: spriteWidth
- },
- count: spritesCount,
- duration: spriteDuration
+ const { totalSprites, spriteDuration } = buildTotalSprites(video)
+ if (totalSprites === 0) {
+ logger.info(`Do not generate a storyboard of ${payload.videoUUID} because the video is not long enough`, lTags)
+ return
}
- })
-
- await insertStoryboardInDatabase({
- videoUUID: video.uuid,
- lTags,
-
- filename,
- destination,
-
- imageSize: await getImageSize(destination),
- spriteHeight,
- spriteWidth,
- spriteDuration,
-
- federate: payload.federate
+ const spritesCount = findGridSize({
+ toFind: totalSprites,
+ maxEdgeCount: STORYBOARD.SPRITES_MAX_EDGE_COUNT
+ })
+
+ const filename = generateImageFilename()
+ const destination = join(CONFIG.STORAGE.STORYBOARDS_DIR, filename)
+
+ logger.debug(
+ `Generating storyboard from video of ${video.uuid} to ${destination}`,
+ { ...lTags, totalSprites, spritesCount, spriteDuration, videoDuration: video.duration, spriteHeight, spriteWidth }
+ )
+
+ const ffmpeg = new FFmpegImage({
+ ...getFFmpegCommandWrapperOptions('thumbnail'),
+
+ abortSignal
+ })
+
+ await ffmpeg.generateStoryboardFromVideo({
+ destination,
+ path: videoPath,
+ inputFileMutexReleaser,
+ sprites: {
+ size: {
+ height: spriteHeight,
+ width: spriteWidth
+ },
+ count: spritesCount,
+ duration: spriteDuration
+ }
+ })
+
+ await insertStoryboardInDatabase({
+ videoUUID: video.uuid,
+ lTags,
+
+ filename,
+ destination,
+
+ imageSize: await getImageSize(destination),
+
+ spriteHeight,
+ spriteWidth,
+ spriteDuration,
+
+ federate: payload.federate
+ })
})
- })
- } finally {
- inputFileMutexReleaser()
+ } finally {
+ inputFileMutexReleaser()
+ }
}
+
+ return Promise.race([ run(), abortPromise ])
}
diff --git a/server/core/lib/job-queue/handlers/shared/job-helpers.ts b/server/core/lib/job-queue/handlers/shared/job-helpers.ts
new file mode 100644
index 0000000000..1870d1c798
--- /dev/null
+++ b/server/core/lib/job-queue/handlers/shared/job-helpers.ts
@@ -0,0 +1,13 @@
+import { UnrecoverableError } from 'bullmq'
+
+export function buildPromiseForAbortSignal (abortSignal?: AbortSignal) {
+ return new Promise((_res, rej) => {
+ if (abortSignal?.aborted) {
+ return rej(new UnrecoverableError('Job has been canceled'))
+ }
+
+ abortSignal?.addEventListener('abort', () => {
+ rej(new UnrecoverableError('Job has been canceled'))
+ })
+ })
+}
diff --git a/server/core/lib/job-queue/handlers/video-studio-edition.ts b/server/core/lib/job-queue/handlers/video-studio-edition.ts
index c7c217e2c2..97221c7b5f 100644
--- a/server/core/lib/job-queue/handlers/video-studio-edition.ts
+++ b/server/core/lib/job-queue/handlers/video-studio-edition.ts
@@ -27,89 +27,98 @@ import { Job } from 'bullmq'
import { remove } from 'fs-extra/esm'
import { extname, join } from 'path'
import { logger, loggerTagsFactory } from '../../../helpers/logger.js'
+import { buildPromiseForAbortSignal } from './shared/job-helpers.js'
const lTagsBase = loggerTagsFactory('video-studio')
-async function processVideoStudioEdition (job: Job) {
- const payload = job.data as VideoStudioEditionPayload
- const lTags = lTagsBase(payload.videoUUID)
+async function processVideoStudioEdition (job: Job, abortSignal?: AbortSignal) {
+ const abortPromise = buildPromiseForAbortSignal(abortSignal)
- logger.info('Process video studio edition of %s in job %s.', payload.videoUUID, job.id, lTags)
+ const run = async () => {
+ const payload = job.data as VideoStudioEditionPayload
+ const lTags = lTagsBase(payload.videoUUID)
- let inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(payload.videoUUID)
+ logger.info('Process video studio edition of %s in job %s.', payload.videoUUID, job.id, lTags)
- try {
- const video = await VideoModel.loadFull(payload.videoUUID)
+ let inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(payload.videoUUID)
- // No video, maybe deleted?
- if (!video) {
- logger.info('Can\'t process job %d, video does not exist.', job.id, lTags)
+ try {
+ const video = await VideoModel.loadFull(payload.videoUUID)
- await safeCleanupStudioTMPFiles(payload.tasks)
- return undefined
- }
+ // No video, maybe deleted?
+ if (!video) {
+ logger.info('Can\'t process job %d, video does not exist.', job.id, lTags)
- await checkUserQuotaOrThrow(video, payload)
+ await safeCleanupStudioTMPFiles(payload.tasks)
+ return undefined
+ }
- await video.reload()
+ await checkUserQuotaOrThrow(video, payload)
- const editionResultPath = await VideoPathManager.Instance.makeAvailableMaxQualityFiles(video, async ({
- videoPath: originalVideoFilePath,
- separatedAudioPath
- }) => {
- let tmpInputFilePath: string
- let outputPath: string
+ await video.reload()
- for (const task of payload.tasks) {
- const outputFilename = buildUUID() + extname(originalVideoFilePath)
- outputPath = join(CONFIG.STORAGE.TMP_DIR, outputFilename)
+ const editionResultPath = await VideoPathManager.Instance.makeAvailableMaxQualityFiles(video, async ({
+ videoPath: originalVideoFilePath,
+ separatedAudioPath
+ }) => {
+ let tmpInputFilePath: string
+ let outputPath: string
- await processTask({
- videoInputPath: tmpInputFilePath ?? originalVideoFilePath,
+ for (const task of payload.tasks) {
+ const outputFilename = buildUUID() + extname(originalVideoFilePath)
+ outputPath = join(CONFIG.STORAGE.TMP_DIR, outputFilename)
- separatedAudioInputPath: tmpInputFilePath
- ? undefined
- : separatedAudioPath,
+ await processTask({
+ videoInputPath: tmpInputFilePath ?? originalVideoFilePath,
- inputFileMutexReleaser,
+ separatedAudioInputPath: tmpInputFilePath
+ ? undefined
+ : separatedAudioPath,
- video,
- outputPath,
- task,
- lTags
- })
+ inputFileMutexReleaser,
- if (tmpInputFilePath) await remove(tmpInputFilePath)
+ video,
+ outputPath,
+ task,
+ lTags,
- // For the next iteration
- tmpInputFilePath = outputPath
- inputFileMutexReleaser = undefined
- }
+ abortSignal
+ })
- return outputPath
- })
+ if (tmpInputFilePath) await remove(tmpInputFilePath)
- logger.info('Video edition ended for video %s.', video.uuid, lTags)
+ // For the next iteration
+ tmpInputFilePath = outputPath
+ inputFileMutexReleaser = undefined
+ }
- await onVideoStudioEnded({ video, editionResultPath, tasks: payload.tasks })
- } catch (err) {
- await safeCleanupStudioTMPFiles(payload.tasks)
+ return outputPath
+ })
- try {
- await sequelizeTypescript.transaction(async transaction => {
- const video = await VideoModel.load(payload.videoUUID, transaction)
- if (!video || video.state === VideoState.PUBLISHED) return
+ logger.info('Video edition ended for video %s.', video.uuid, lTags)
- await video.setNewState(VideoState.PUBLISHED, false, transaction)
- })
+ await onVideoStudioEnded({ video, editionResultPath, tasks: payload.tasks })
} catch (err) {
- logger.error('Cannot reset video state after studio error', { err, ...lTags })
- }
+ await safeCleanupStudioTMPFiles(payload.tasks)
+
+ try {
+ await sequelizeTypescript.transaction(async transaction => {
+ const video = await VideoModel.load(payload.videoUUID, transaction)
+ if (!video || video.state === VideoState.PUBLISHED) return
+
+ await video.setNewState(VideoState.PUBLISHED, false, transaction)
+ })
+ } catch (err) {
+ logger.error('Cannot reset video state after studio error', { err, ...lTags })
+ }
- throw err
- } finally {
- if (inputFileMutexReleaser) inputFileMutexReleaser()
+ throw err
+ } finally {
+ if (inputFileMutexReleaser) inputFileMutexReleaser()
+ }
}
+
+ return Promise.race([ run(), abortPromise ])
}
// ---------------------------------------------------------------------------
@@ -130,6 +139,8 @@ type TaskProcessorOptions<T extends VideoStudioTaskPayload = VideoStudioTaskPayl
video: MVideo
task: T
lTags: { tags: (string | number)[] }
+
+ abortSignal?: AbortSignal
}
const taskProcessors: { [id in VideoStudioTask['name']]: (options: TaskProcessorOptions) => Promise<any> } = {
@@ -152,11 +163,11 @@ async function processTask (options: TaskProcessorOptions) {
}
function processAddIntroOutro (options: TaskProcessorOptions<VideoStudioTaskIntroPayload | VideoStudioTaskOutroPayload>) {
- const { task, lTags } = options
+ const { task, lTags, abortSignal } = options
logger.debug('Will add intro/outro to the video.', { options, ...lTags })
- return buildFFmpegEdition().addIntroOutro({
+ return buildFFmpegEdition(abortSignal).addIntroOutro({
...pick(options, [ 'inputFileMutexReleaser', 'videoInputPath', 'separatedAudioInputPath', 'outputPath' ]),
introOutroPath: task.options.file,
@@ -167,11 +178,11 @@ function processAddIntroOutro (options: TaskProcessorOptions<VideoStudioTaskIntr
}
function processCut (options: TaskProcessorOptions<VideoStudioTaskCutPayload>) {
- const { task, lTags } = options
+ const { task, lTags, abortSignal } = options
logger.debug('Will cut the video.', { options, ...lTags })
- return buildFFmpegEdition().cutVideo({
+ return buildFFmpegEdition(abortSignal).cutVideo({
...pick(options, [ 'inputFileMutexReleaser', 'videoInputPath', 'separatedAudioInputPath', 'outputPath' ]),
start: task.options.start,
@@ -180,11 +191,11 @@ function processCut (options: TaskProcessorOptions<VideoStudioTaskCutPayload>) {
}
function processRemoveSegments (options: TaskProcessorOptions<VideoStudioTaskRemoveSegmentsPayload>) {
- const { task, lTags } = options
+ const { task, lTags, abortSignal } = options
logger.debug('Will remove segments from the video.', { options, ...lTags })
- return buildFFmpegEdition().removeSegments({
+ return buildFFmpegEdition(abortSignal).removeSegments({
...pick(options, [ 'inputFileMutexReleaser', 'videoInputPath', 'separatedAudioInputPath', 'outputPath' ]),
segments: task.options.segments
@@ -192,11 +203,11 @@ function processRemoveSegments (options: TaskProcessorOptions<VideoStudioTaskRem
}
function processAddWatermark (options: TaskProcessorOptions<VideoStudioTaskWatermarkPayload>) {
- const { task, lTags } = options
+ const { task, lTags, abortSignal } = options
logger.debug('Will add watermark to the video.', { options, ...lTags })
- return buildFFmpegEdition().addWatermark({
+ return buildFFmpegEdition(abortSignal).addWatermark({
...pick(options, [ 'inputFileMutexReleaser', 'videoInputPath', 'separatedAudioInputPath', 'outputPath' ]),
watermarkPath: task.options.file,
@@ -222,6 +233,10 @@ async function checkUserQuotaOrThrow (video: MVideoFull, payload: VideoStudioEdi
}
}
-function buildFFmpegEdition () {
- return new FFmpegEdition(getFFmpegCommandWrapperOptions('vod', VideoTranscodingProfilesManager.Instance.getAvailableEncoders()))
+function buildFFmpegEdition (abortSignal?: AbortSignal) {
+ return new FFmpegEdition({
+ ...getFFmpegCommandWrapperOptions('vod', VideoTranscodingProfilesManager.Instance.getAvailableEncoders()),
+
+ abortSignal
+ })
}
diff --git a/server/core/lib/job-queue/handlers/video-transcoding.ts b/server/core/lib/job-queue/handlers/video-transcoding.ts
index 9316dbfa2b..db26505534 100644
--- a/server/core/lib/job-queue/handlers/video-transcoding.ts
+++ b/server/core/lib/job-queue/handlers/video-transcoding.ts
@@ -16,12 +16,19 @@ import { VideoPathManager } from '@server/lib/video-path-manager.js'
import { moveToFailedTranscodingState } from '@server/lib/video-state.js'
import { UserModel } from '@server/models/user/user.js'
import { VideoJobInfoModel } from '@server/models/video/video-job-info.js'
-import { MUser, MUserId, MVideoFull } from '@server/types/models/index.js'
+import { MUser, MVideoFull } from '@server/types/models/index.js'
import { Job } from 'bullmq'
import { logger, loggerTagsFactory } from '../../../helpers/logger.js'
import { VideoModel } from '../../../models/video/video.js'
+import { buildPromiseForAbortSignal } from './shared/job-helpers.js'
-type HandlerFunction = (job: Job, payload: VideoTranscodingPayload, video: MVideoFull, user: MUser) => Promise<void>
+type HandlerFunction = (options: {
+ job: Job
+ payload: VideoTranscodingPayload
+ video: MVideoFull
+ user: MUser
+ abortSignal: AbortSignal
+}) => Promise<void>
const handlers: { [id in VideoTranscodingPayload['type']]: HandlerFunction } = {
'new-resolution-to-hls': handleHLSJob,
@@ -32,39 +39,45 @@ const handlers: { [id in VideoTranscodingPayload['type']]: HandlerFunction } = {
const lTags = loggerTagsFactory('transcoding')
-async function processVideoTranscoding (job: Job) {
- const payload = job.data as VideoTranscodingPayload
- logger.info('Processing transcoding job %s.', job.id, lTags(payload.videoUUID))
+async function processVideoTranscoding (job: Job, abortSignal: AbortSignal) {
+ const abortPromise = buildPromiseForAbortSignal(abortSignal)
- const video = await VideoModel.loadFull(payload.videoUUID)
- // No video, maybe deleted?
- if (!video) {
- logger.info(`Do not process job ${job.id}, video does not exist.`, lTags(payload.videoUUID))
- return undefined
- }
+ const run = async () => {
+ const payload = job.data as VideoTranscodingPayload
+ logger.info('Processing transcoding job %s.', job.id, lTags(payload.videoUUID))
- const user = await UserModel.loadByChannelActorId(video.VideoChannel.Actor.id)
+ const video = await VideoModel.loadFull(payload.videoUUID)
+ // No video, maybe deleted?
+ if (!video) {
+ logger.info(`Do not process job ${job.id}, video does not exist.`, lTags(payload.videoUUID))
+ return undefined
+ }
- const handler = handlers[payload.type]
+ const user = await UserModel.loadByChannelActorId(video.VideoChannel.Actor.id)
- if (!handler) {
- await moveToFailedTranscodingState(video)
- await VideoJobInfoModel.decrease(video.uuid, 'pendingTranscode')
+ const handler = handlers[payload.type]
- throw new Error('Cannot find transcoding handler for ' + payload.type)
- }
+ if (!handler) {
+ await moveToFailedTranscodingState(video)
+ await VideoJobInfoModel.decrease(video.uuid, 'pendingTranscode')
- try {
- await handler(job, payload, video, user)
- } catch (error) {
- await moveToFailedTranscodingState(video)
+ throw new Error('Cannot find transcoding handler for ' + payload.type)
+ }
- await VideoJobInfoModel.decrease(video.uuid, 'pendingTranscode')
+ try {
+ await handler({ job, payload, video, user, abortSignal })
+ } catch (error) {
+ await moveToFailedTranscodingState(video)
- throw error
+ await VideoJobInfoModel.decrease(video.uuid, 'pendingTranscode')
+
+ throw error
+ }
+
+ return video
}
- return video
+ return Promise.race([ run(), abortPromise ])
}
// ---------------------------------------------------------------------------
@@ -77,30 +90,36 @@ export {
// Job handlers
// ---------------------------------------------------------------------------
-async function handleWebVideoMergeAudioJob (
- job: Job<MergeAudioTranscodingPayload>,
- payload: MergeAudioTranscodingPayload,
- video: MVideoFull,
- user: MUserId
-) {
+async function handleWebVideoMergeAudioJob (options: {
+ job: Job<MergeAudioTranscodingPayload>
+ payload: MergeAudioTranscodingPayload
+ video: MVideoFull
+ user: MUser
+ abortSignal: AbortSignal
+}) {
+ const { job, payload, video, abortSignal } = options
+
logger.info('Handling merge audio transcoding job for %s.', video.uuid, lTags(video.uuid), { payload })
- await mergeAudioVideofile({ video, resolution: payload.resolution, fps: payload.fps, job })
+ await mergeAudioVideofile({ video, resolution: payload.resolution, fps: payload.fps, job, abortSignal })
logger.info('Merge audio transcoding job for %s ended.', video.uuid, lTags(video.uuid), { payload })
await onTranscodingEnded({ isNewVideo: payload.isNewVideo, moveVideoToNextState: payload.canMoveVideoState, video })
}
-async function handleWebVideoOptimizeJob (
- job: Job<OptimizeTranscodingPayload>,
- payload: OptimizeTranscodingPayload,
- video: MVideoFull,
- user: MUserId
-) {
+async function handleWebVideoOptimizeJob (options: {
+ job: Job<OptimizeTranscodingPayload>
+ payload: OptimizeTranscodingPayload
+ video: MVideoFull
+ user: MUser
+ abortSignal: AbortSignal
+}) {
+ const { job, payload, video, abortSignal } = options
+
logger.info('Handling optimize transcoding job for %s.', video.uuid, lTags(video.uuid), { payload })
- await optimizeOriginalVideofile({ video, job })
+ await optimizeOriginalVideofile({ video, job, abortSignal })
logger.info('Optimize transcoding job for %s ended.', video.uuid, lTags(video.uuid), { payload })
@@ -109,14 +128,18 @@ async function handleWebVideoOptimizeJob (
// ---------------------------------------------------------------------------
-async function handleNewWebVideoResolutionJob (
- job: Job<NewWebVideoResolutionTranscodingPayload>,
- payload: NewWebVideoResolutionTranscodingPayload,
+async function handleNewWebVideoResolutionJob (options: {
+ job: Job<NewWebVideoResolutionTranscodingPayload>
+ payload: NewWebVideoResolutionTranscodingPayload
video: MVideoFull
-) {
+ user: MUser
+ abortSignal: AbortSignal
+}) {
+ const { job, payload, video, abortSignal } = options
+
logger.info('Handling Web Video transcoding job for %s.', video.uuid, lTags(video.uuid), { payload })
- await transcodeNewWebVideoResolution({ video, resolution: payload.resolution, fps: payload.fps, job })
+ await transcodeNewWebVideoResolution({ video, resolution: payload.resolution, fps: payload.fps, job, abortSignal })
logger.info('Web Video transcoding job for %s ended.', video.uuid, lTags(video.uuid), { payload })
@@ -126,7 +149,15 @@ async function handleNewWebVideoResolutionJob (
// ---------------------------------------------------------------------------
-async function handleHLSJob (job: Job<HLSTranscodingPayload>, payload: HLSTranscodingPayload, videoArg: MVideoFull) {
+async function handleHLSJob (options: {
+ job: Job<HLSTranscodingPayload>
+ payload: HLSTranscodingPayload
+ video: MVideoFull
+ user: MUser
+ abortSignal: AbortSignal
+}) {
+ const { job, payload, video: videoArg, abortSignal } = options
+
logger.info('Handling HLS transcoding job for %s.', videoArg.uuid, lTags(videoArg.uuid), { payload })
const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(videoArg.uuid)
@@ -153,7 +184,8 @@ async function handleHLSJob (job: Job<HLSTranscodingPayload>, payload: HLSTransc
resolution: payload.resolution,
fps: payload.fps,
separatedAudio: payload.separatedAudio,
- job
+ job,
+ abortSignal
})
})
} finally {
diff --git a/server/core/lib/job-queue/handlers/video-transcription.ts b/server/core/lib/job-queue/handlers/video-transcription.ts
index 47ae8e3374..027a3b7da8 100644
--- a/server/core/lib/job-queue/handlers/video-transcription.ts
+++ b/server/core/lib/job-queue/handlers/video-transcription.ts
@@ -3,19 +3,26 @@ import { generateSubtitle } from '@server/lib/video-captions.js'
import { Job } from 'bullmq'
import { logger, loggerTagsFactory } from '../../../helpers/logger.js'
import { VideoModel } from '../../../models/video/video.js'
+import { buildPromiseForAbortSignal } from './shared/job-helpers.js'
const lTags = loggerTagsFactory('transcription')
-export async function processVideoTranscription (job: Job) {
- const payload = job.data as VideoTranscriptionPayload
+export async function processVideoTranscription (job: Job, abortSignal?: AbortSignal) {
+ const abortPromise = buildPromiseForAbortSignal(abortSignal)
- logger.info('Processing video transcription in job %s.', job.id)
+ const run = async () => {
+ const payload = job.data as VideoTranscriptionPayload
- const video = await VideoModel.load(payload.videoUUID)
- if (!video) {
- logger.info('Do not process transcription job %d, video does not exist.', job.id, lTags(payload.videoUUID))
- return
+ logger.info('Processing video transcription in job %s.', job.id)
+
+ const video = await VideoModel.load(payload.videoUUID)
+ if (!video) {
+ logger.info('Do not process transcription job %d, video does not exist.', job.id, lTags(payload.videoUUID))
+ return
+ }
+
+ return generateSubtitle({ video, signal: abortSignal })
}
- return generateSubtitle({ video })
+ return Promise.race([ run(), abortPromise ])
}
diff --git a/server/core/lib/job-queue/job-queue.ts b/server/core/lib/job-queue/job-queue.ts
index 638990db7f..0ee7cbb6a6 100644
--- a/server/core/lib/job-queue/job-queue.ts
+++ b/server/core/lib/job-queue/job-queue.ts
@@ -1,4 +1,4 @@
-import { pick, timeoutPromiseWithCleanup } from '@peertube/peertube-core-utils'
+import { pick } from '@peertube/peertube-core-utils'
import {
ActivitypubFollowPayload,
ActivitypubHttpBroadcastPayload,
@@ -116,7 +116,7 @@ export type CreateJobOptions = {
deduplicationId?: string
}
-const handlers: { [id in JobType]: (job: Job) => Promise<any> } = {
+const handlers: { [id in JobType]: (job: Job, signal?: AbortSignal) => Promise<any> } = {
'build-automatic-tags': processBuildAutomaticTags,
'activitypub-cleaner': processActivityPubCleaner,
'activitypub-follow': processActivityPubFollow,
@@ -185,6 +185,8 @@ const jobTypes: JobType[] = [
'video-transcoding'
]
+const cancelableJobTypes: JobType[] = [ 'video-transcoding', 'video-transcription', 'video-studio-edition', 'generate-video-storyboard' ]
+
const silentFailure = new Set<JobType>([ 'activitypub-http-unicast' ])
class JobQueue {
@@ -233,6 +235,8 @@ class JobQueue {
}
private buildWorker (handlerName: JobType) {
+ let worker: Worker
+
const workerOptions: WorkerOptions = {
autorun: false,
concurrency: this.getJobConcurrency(handlerName),
@@ -241,39 +245,27 @@ class JobQueue {
maxStalledCount: 10
}
- const handler = function (job: Job) {
+ const handler = function (options: { job: Job, signal: AbortSignal }) {
+ const { job, signal } = options
+
const timeout = JOB_TTL[handlerName]
- if (!timeout) return handlers[handlerName](job)
-
- const controller = new AbortController()
-
- // Store abort signal in job immediately so handlers can access it
- job.data.abortSignal = controller.signal
-
- // Create timeout with cleanup callback to kill orphaned processes
- return timeoutPromiseWithCleanup(
- handlers[handlerName](job),
- timeout,
- () => {
- logger.warn(
- 'Job %s in queue %s exceeded timeout of %d ms, triggering cleanup',
- job.id,
- handlerName,
- timeout
- )
-
- controller.abort()
- }
- )
+ if (!timeout) return handlers[handlerName](job, signal)
+
+ const timeoutId = setTimeout(() => {
+ worker.cancelJob(job.id, 'Timeout exceeded')
+ }, timeout)
+
+ return handlers[handlerName](job, signal)
+ .finally(() => clearTimeout(timeoutId))
}
- const processor = async (jobArg: Job) => {
+ const processor = async (jobArg: Job, _, signal: AbortSignal) => {
const job = await Hooks.wrapObject(jobArg, 'filter:job-queue.process.params', { type: handlerName })
- return Hooks.wrapPromiseFun(handler, job, 'filter:job-queue.process.result')
+ return Hooks.wrapPromiseFun(handler, { job, signal }, 'filter:job-queue.process.result')
}
- const worker = new Worker(handlerName, processor, workerOptions)
+ worker = new Worker(handlerName, processor, workerOptions)
worker.on('failed', (job, err) => {
const logLevel = silentFailure.has(handlerName)
@@ -520,6 +512,30 @@ class JobQueue {
return total
}
+ async getJob (jobType: JobType, jobId: string): Promise<Job> {
+ const queue = this.queues[jobType]
+ if (!queue) throw new Error(`Unknown queue ${jobType}`)
+
+ return queue.getJob(jobId)
+ }
+
+ async canCancelJob (jobType: JobType, job: Job) {
+ if (!cancelableJobTypes.includes(jobType)) return false
+
+ const isActive = await job.isActive()
+
+ return isActive
+ }
+
+ cancelJob (jobType: JobType, job: Job) {
+ logger.info('Cancelling job %s in queue %s.', job.id, job.queueName)
+
+ const worker = this.workers[jobType]
+ if (!worker) throw new Error(`Unknown queue ${jobType}`)
+
+ return worker.cancelJob(job.id, 'Job cancelled by admin')
+ }
+
private buildStateFilter (state?: JobState) {
if (!state) return Array.from(jobStates)
diff --git a/server/core/lib/transcoding/hls-transcoding.ts b/server/core/lib/transcoding/hls-transcoding.ts
index a70a6a788c..cb3b62891b 100644
--- a/server/core/lib/transcoding/hls-transcoding.ts
+++ b/server/core/lib/transcoding/hls-transcoding.ts
@@ -38,7 +38,7 @@ export async function generateHlsPlaylistResolutionFromTS (options: {
}
// Generate an HLS playlist from an input file, and update the master playlist
-export function generateHlsPlaylistResolution<JobData extends { abortSignal?: AbortSignal }> (options: {
+export function generateHlsPlaylistResolution (options: {
video: MVideo
videoInputPath: string
@@ -48,7 +48,9 @@ export function generateHlsPlaylistResolution<JobData extends { abortSignal?: Ab
fps: number
inputFileMutexReleaser: MutexInterface.Releaser
separatedAudio: boolean
- job?: Job<JobData>
+
+ job: Job
+ abortSignal: AbortSignal
}) {
return generateHlsPlaylistCommon({
type: 'hls' as 'hls',
@@ -61,7 +63,8 @@ export function generateHlsPlaylistResolution<JobData extends { abortSignal?: Ab
'fps',
'separatedAudio',
'inputFileMutexReleaser',
- 'job'
+ 'job',
+ 'abortSignal'
])
})
}
@@ -143,7 +146,7 @@ export async function onHLSVideoFileTranscoding (options: {
// Private
// ---------------------------------------------------------------------------
-async function generateHlsPlaylistCommon<JobData extends { abortSignal?: AbortSignal }> (options: {
+async function generateHlsPlaylistCommon (options: {
type: 'hls' | 'hls-from-ts'
video: MVideo
@@ -160,7 +163,8 @@ async function generateHlsPlaylistCommon<JobData extends { abortSignal?: AbortSi
isAAC?: boolean
- job?: Job<JobData>
+ job?: Job
+ abortSignal?: AbortSignal
}) {
const {
type,
@@ -173,7 +177,8 @@ async function generateHlsPlaylistCommon<JobData extends { abortSignal?: AbortSi
isAAC,
job,
inputFileMutexReleaser,
- preventInputFileLocking
+ preventInputFileLocking,
+ abortSignal
} = options
const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
@@ -211,7 +216,7 @@ async function generateHlsPlaylistCommon<JobData extends { abortSignal?: AbortSi
}
}
- await buildFFmpegVOD(job).transcode(transcodeOptions)
+ await buildFFmpegVOD({ job, abortSignal }).transcode(transcodeOptions)
// Ensure the mutex is released if the ffmpeg command failed and did not release it
if (inputFileMutexReleaser) inputFileMutexReleaser()
diff --git a/server/core/lib/transcoding/shared/ffmpeg-builder.ts b/server/core/lib/transcoding/shared/ffmpeg-builder.ts
index 3256aa3dd3..acb51619c5 100644
--- a/server/core/lib/transcoding/shared/ffmpeg-builder.ts
+++ b/server/core/lib/transcoding/shared/ffmpeg-builder.ts
@@ -4,7 +4,12 @@ import { logger } from '@server/helpers/logger.js'
import { Job } from 'bullmq'
import { VideoTranscodingProfilesManager } from '../default-transcoding-profiles.js'
-export function buildFFmpegVOD<T extends Job<{ abortSignal?: AbortSignal }>> (job?: T) {
+export function buildFFmpegVOD (options: {
+ job?: Job
+ abortSignal?: AbortSignal
+} = {}) {
+ const { job, abortSignal } = options
+
return new FFmpegVOD({
...getFFmpegCommandWrapperOptions('vod', VideoTranscodingProfilesManager.Instance.getAvailableEncoders()),
@@ -16,6 +21,6 @@ export function buildFFmpegVOD<T extends Job<{ abortSignal?: AbortSignal }>> (jo
},
// Pass the abort signal from the job so the ffmpeg process can be killed on timeout
- abortSignal: job?.data?.abortSignal
+ abortSignal
})
}
diff --git a/server/core/lib/transcoding/web-transcoding.ts b/server/core/lib/transcoding/web-transcoding.ts
index af0683ab76..c14d519388 100644
--- a/server/core/lib/transcoding/web-transcoding.ts
+++ b/server/core/lib/transcoding/web-transcoding.ts
@@ -28,8 +28,9 @@ import { buildOriginalFileResolution } from './transcoding-resolutions.js'
export async function optimizeOriginalVideofile (options: {
video: MVideoFull
job: Job
+ abortSignal: AbortSignal
}) {
- const { job } = options
+ const { job, abortSignal } = options
const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
const newExtname = '.mp4'
@@ -52,7 +53,7 @@ export async function optimizeOriginalVideofile (options: {
const fps = computeOutputFPS({ inputFPS: inputVideoFile.fps, resolution, isOriginResolution: true, type: 'vod' })
// Could be very long!
- await buildFFmpegVOD(job).transcode({
+ await buildFFmpegVOD({ job, abortSignal }).transcode({
type: transcodeType,
videoInputPath,
@@ -81,8 +82,9 @@ export async function transcodeNewWebVideoResolution (options: {
resolution: number
fps: number
job: Job
+ abortSignal: AbortSignal
}) {
- const { video: videoArg, resolution, fps, job } = options
+ const { video: videoArg, resolution, fps, job, abortSignal } = options
const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
const newExtname = '.mp4'
@@ -110,7 +112,7 @@ export async function transcodeNewWebVideoResolution (options: {
fps
}
- await buildFFmpegVOD(job).transcode(transcodeOptions)
+ await buildFFmpegVOD({ job, abortSignal }).transcode(transcodeOptions)
return onWebVideoFileTranscoding({ video, videoOutputPath })
})
@@ -127,8 +129,9 @@ export async function mergeAudioVideofile (options: {
resolution: number
fps: number
job: Job
+ abortSignal: AbortSignal
}) {
- const { video: videoArg, resolution, fps, job } = options
+ const { video: videoArg, resolution, fps, job, abortSignal } = options
const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
const newExtname = '.mp4'
@@ -162,7 +165,7 @@ export async function mergeAudioVideofile (options: {
}
try {
- await buildFFmpegVOD(job).transcode(transcodeOptions)
+ await buildFFmpegVOD({ job, abortSignal }).transcode(transcodeOptions)
await remove(tmpThumbnailPath)
} catch (err) {
diff --git a/server/core/lib/video-captions.ts b/server/core/lib/video-captions.ts
index 08a4af675d..d796096ee5 100644
--- a/server/core/lib/video-captions.ts
+++ b/server/core/lib/video-captions.ts
@@ -141,6 +141,7 @@ let transcriber: AbstractTranscriber
export async function generateSubtitle (options: {
video: MVideoUUID
+ signal?: AbortSignal
}) {
const outputPath = join(CONFIG.STORAGE.TMP_DIR, 'transcription', buildSUUID())
@@ -200,7 +201,9 @@ export async function generateSubtitle (options: {
transcriptDirectory: outputPath,
- format: 'vtt'
+ format: 'vtt',
+
+ signal: options.signal
})
const refreshedVideo = await VideoModel.loadFull(video.uuid)
diff --git a/server/core/middlewares/validators/jobs.ts b/server/core/middlewares/validators/jobs.ts
index 070cef32e7..53d34c8781 100644
--- a/server/core/middlewares/validators/jobs.ts
+++ b/server/core/middlewares/validators/jobs.ts
@@ -1,12 +1,14 @@
+import { HttpStatusCode, JobType } from '@peertube/peertube-models'
import express from 'express'
import { param, query } from 'express-validator'
import { isValidJobState, isValidJobType } from '../../helpers/custom-validators/jobs.js'
import { loggerTagsFactory } from '../../helpers/logger.js'
+import { JobQueue } from '../../lib/job-queue/index.js'
import { areValidationErrors } from './shared/index.js'
const lTags = loggerTagsFactory('validators', 'jobs')
-const listJobsValidator = [
+export const listJobsValidator = [
param('state')
.optional()
.custom(isValidJobState),
@@ -22,8 +24,28 @@ const listJobsValidator = [
}
]
-// ---------------------------------------------------------------------------
+export const cancelJobValidator = [
+ param('jobType').custom(isValidJobType),
+ param('jobId').isString().not().isEmpty(),
-export {
- listJobsValidator
-}
+ async (req: express.Request, res: express.Response, next: express.NextFunction) => {
+ if (areValidationErrors(req, res, lTags())) return
+
+ const jobType = req.params.jobType as JobType
+ const jobId = req.params.jobId
+
+ const job = await JobQueue.Instance.getJob(jobType, jobId)
+ if (!job) return res.sendStatus(HttpStatusCode.NOT_FOUND_404)
+
+ if (!await JobQueue.Instance.canCancelJob(job.queueName as JobType, job)) {
+ return res.fail({
+ status: HttpStatusCode.CONFLICT_409,
+ message: req.t(`This job cannot be cancelled`)
+ })
+ }
+
+ res.locals.job = job
+
+ return next()
+ }
+]
diff --git a/server/core/types/express.d.ts b/server/core/types/express.d.ts
index d35389ebe9..f292e7bd19 100644
--- a/server/core/types/express.d.ts
+++ b/server/core/types/express.d.ts
@@ -6,6 +6,7 @@ import {
ServerLogLevel,
VideoCreate
} from '@peertube/peertube-models'
+import { SignupMode } from '@server/lib/signup.ts'
import { RegisterServerAuthExternalOptions } from '@server/types/index.js'
import {
MAbuseMessage,
@@ -13,6 +14,8 @@ import {
MAccountBlocklist,
MActorFollowActorsDefault,
MActorUrl,
+ MBlocklistSubscription,
+ MChangeOwnershipFull,
MChannelBannerAccountDefault,
MChannelCollaboratorAccount,
MChannelSyncChannel,
@@ -21,7 +24,6 @@ import {
MStreamingPlaylist,
MUserAccountUrl,
MUserExport,
- MChangeOwnershipFull,
MVideoEmbedDomain,
MVideoFile,
MVideoFormattableDetails,
@@ -36,8 +38,7 @@ import {
MVideoWithBlacklist,
MVideoWithRights,
MWatchedWordsList,
- MWatchedWordsSubscription,
- MBlocklistSubscription
+ MWatchedWordsSubscription
} from '@server/types/models/index.js'
import { MOAuthToken, MOAuthTokenUser } from '@server/types/models/oauth/oauth-token.js'
import { MPlugin, MServer, MServerBlocklist } from '@server/types/models/server.js'
@@ -45,6 +46,7 @@ import { MVideoImportDefault } from '@server/types/models/video/video-import.js'
import { MVideoPlaylistElement, MVideoPlaylistElementVideoUrlPlaylistPrivacy } from '@server/types/models/video/video-playlist-element.js'
import { MAccountVideoRateAccountVideo } from '@server/types/models/video/video-rate.js'
import { Metadata, File as UploadXFile } from '@uploadx/core'
+import { Job as BullJob } from 'bullmq'
import { FfprobeData } from 'fluent-ffmpeg'
import { OutgoingHttpHeaders } from 'http'
import { Writable } from 'stream'
@@ -65,7 +67,6 @@ import {
} from './models/index.js'
import { MRunner, MRunnerJobRunner, MRunnerRegistrationToken } from './models/runners/index.js'
import { MVideoSource } from './models/video/video-source.js'
-import { SignupMode } from '@server/lib/signup.ts'
declare module 'express' {
export interface Request {
@@ -268,6 +269,8 @@ declare module 'express' {
channelCollaborator?: MChannelCollaboratorAccount
blocklistSubscription?: MBlocklistSubscription
+
+ job?: BullJob
}
}
}
diff --git a/support/doc/api/openapi.yaml b/support/doc/api/openapi.yaml
index 8c536a9dba..4e55ed0dad 100644
--- a/support/doc/api/openapi.yaml
+++ b/support/doc/api/openapi.yaml
@@ -1234,6 +1234,37 @@ paths:
items:
$ref: '#/components/schemas/Job'
+ /api/v1/jobs/{jobType}/{jobId}/cancel:
+ post:
+ summary: Cancel a job
+ description: Abort a running job and its underlying ffmpeg/child process. Only active jobs that support cancellation can be cancelled.
+ operationId: cancelJob
+ security:
+ - OAuth2:
+ - admin
+ tags:
+ - Job
+ parameters:
+ - name: jobType
+ in: path
+ required: true
+ description: The type of the job to cancel
+ schema:
+ type: string
+ - name: jobId
+ in: path
+ required: true
+ description: The id of the job to cancel
+ schema:
+ type: string
+ responses:
+ '204':
+ description: successful operation
+ '404':
+ description: job not found
+ '409':
+ description: job cannot be cancelled (not active or not cancellable)
+
/api/v1/server/followers:
get:
tags:
@@ -12336,6 +12367,9 @@ components:
processedOn:
type: string
format: date-time
+ canCancel:
+ description: Whether this job is currently running and supports cancellation
+ type: boolean
AddUserResponse:
properties:
user: