ActivityPub-federated video streaming platform using P2P directly in your web browser
Feature for runners - handle storyboard-generation-job (#7191)
ilfarprodd52e8b89ea9
diff --git a/apps/peertube-runner/src/server/process/process.ts b/apps/peertube-runner/src/server/process/process.tsindex b20c2297d1..b6d9be5bac 100644--- a/apps/peertube-runner/src/server/process/process.ts+++ b/apps/peertube-runner/src/server/process/process.ts@@ -7,7 +7,13 @@ import {RunnerJobVODWebVideoTranscodingPayload} from '@peertube/peertube-models'import { logger } from '../../shared/index.js'-import { processAudioMergeTranscoding, processHLSTranscoding, ProcessOptions, processWebVideoTranscoding } from './shared/index.js'+import {+ processAudioMergeTranscoding,+ processGenerateStoryboard,+ processHLSTranscoding,+ ProcessOptions,+ processWebVideoTranscoding+} from './shared/index.js'import { ProcessLiveRTMPHLSTranscoding } from './shared/process-live.js'import { processStudioTranscoding } from './shared/process-studio.js'import { processVideoTranscription } from './shared/process-transcription.js'@@ -42,6 +48,10 @@ export async function processJob (options: ProcessOptions) {await processVideoTranscription(options as ProcessOptions<RunnerJobTranscriptionPayload>)break+ case 'generate-video-storyboard':+ await processGenerateStoryboard(options as any)+ break+default:logger.error(`Unknown job ${job.type} to process`)returndiff --git a/apps/peertube-runner/src/server/process/shared/common.ts b/apps/peertube-runner/src/server/process/shared/common.tsindex 8e7ba4583c..45d6b58665 100644--- a/apps/peertube-runner/src/server/process/shared/common.ts+++ b/apps/peertube-runner/src/server/process/shared/common.ts@@ -1,5 +1,12 @@import { pick } from '@peertube/peertube-core-utils'-import { FFmpegEdition, FFmpegLive, FFmpegVOD, getDefaultAvailableEncoders, getDefaultEncodersToTry } from '@peertube/peertube-ffmpeg'+import {+ FFmpegEdition,+ FFmpegImage,+ FFmpegLive,+ FFmpegVOD,+ getDefaultAvailableEncoders,+ getDefaultEncodersToTry+} from '@peertube/peertube-ffmpeg'import { RunnerJob, RunnerJobPayload } from '@peertube/peertube-models'import { buildUUID } from '@peertube/peertube-node-utils'import { PeerTubeServer } from '@peertube/peertube-server-commands'@@ -8,9 +15,9 @@ import { join } from 'path'import { ConfigManager, downloadFile, logger } from '../../../shared/index.js'import { getWinstonLogger } from './winston-logger.js'-export type JobWithToken <T extends RunnerJobPayload = RunnerJobPayload> = RunnerJob<T> & { jobToken: string }+export type JobWithToken<T extends RunnerJobPayload = RunnerJobPayload> = RunnerJob<T> & { jobToken: string }-export type ProcessOptions <T extends RunnerJobPayload = RunnerJobPayload> = {+export type ProcessOptions<T extends RunnerJobPayload = RunnerJobPayload> = {server: PeerTubeServerjob: JobWithToken<T>runnerToken: string@@ -108,6 +115,10 @@ export function buildFFmpegEdition () {return new FFmpegEdition(getCommonFFmpegOptions())}+export function buildFFmpegImage () {+ return new FFmpegImage(getCommonFFmpegOptions())+}+function getCommonFFmpegOptions () {const config = ConfigManager.Instance.getConfig()diff --git a/apps/peertube-runner/src/server/process/shared/index.ts b/apps/peertube-runner/src/server/process/shared/index.tsindex 67d556f915..b6d05d3a9d 100644--- a/apps/peertube-runner/src/server/process/shared/index.ts+++ b/apps/peertube-runner/src/server/process/shared/index.ts@@ -1,3 +1,4 @@export * from './common.js'export * from './process-vod.js'export * from './winston-logger.js'+export * from './process-storyboard.js'diff --git a/apps/peertube-runner/src/server/process/shared/process-storyboard.ts b/apps/peertube-runner/src/server/process/shared/process-storyboard.tsnew file mode 100644index 0000000000..fdd48a972a--- /dev/null+++ b/apps/peertube-runner/src/server/process/shared/process-storyboard.ts@@ -0,0 +1,60 @@+import { RunnerJobGenerateStoryboardPayload, GenerateStoryboardSuccess } from '@peertube/peertube-models'+import { buildUUID } from '@peertube/peertube-node-utils'+import { remove } from 'fs-extra/esm'+import { join } from 'path'+import { ConfigManager } from '../../../shared/config-manager.js'+import { logger } from '../../../shared/index.js'+import { buildFFmpegImage, downloadInputFile, ProcessOptions, scheduleTranscodingProgress } from './common.js'++export async function processGenerateStoryboard (options: ProcessOptions<RunnerJobGenerateStoryboardPayload>) {+ const { server, job, runnerToken } = options++ const payload = job.payload++ let ffmpegProgress: number+ let videoInputPath: string++ const outputPath = join(ConfigManager.Instance.getStoryboardDirectory(), `storyboard-${buildUUID()}.jpg`)++ const updateProgressInterval = scheduleTranscodingProgress({+ job,+ server,+ runnerToken,+ progressGetter: () => ffmpegProgress+ })++ try {+ logger.info(`Downloading input file ${payload.input.videoFileUrl} for storyboard job ${job.jobToken}`)++ videoInputPath = await downloadInputFile({ url: payload.input.videoFileUrl, runnerToken, job })++ logger.info(`Downloaded input file ${payload.input.videoFileUrl} for job ${job.jobToken}. Generating storyboard.`)++ const ffmpegImage = buildFFmpegImage()++ await ffmpegImage.generateStoryboardFromVideo({+ path: videoInputPath,+ destination: outputPath,+ inputFileMutexReleaser: () => {},+ sprites: payload.sprites+ })++ const successBody: GenerateStoryboardSuccess = {+ storyboardFile: outputPath+ }++ await server.runnerJobs.success({+ jobToken: job.jobToken,+ jobUUID: job.uuid,+ runnerToken,+ payload: successBody,+ reqPayload: payload+ })+ } finally {+ if (videoInputPath) await remove(videoInputPath)+ if (outputPath) await remove(outputPath)+ if (updateProgressInterval) clearInterval(updateProgressInterval)+ }+}++diff --git a/apps/peertube-runner/src/server/server.ts b/apps/peertube-runner/src/server/server.tsindex 62a7bce64e..8928ced3ad 100644--- a/apps/peertube-runner/src/server/server.ts+++ b/apps/peertube-runner/src/server/server.ts@@ -74,9 +74,11 @@ export class RunnerServer {// Process jobsawait ensureDir(ConfigManager.Instance.getTranscodingDirectory())+ await ensureDir(ConfigManager.Instance.getStoryboardDirectory())await this.cleanupTMP()logger.info(`Using ${ConfigManager.Instance.getTranscodingDirectory()} for transcoding directory`)+ logger.info(`Using ${ConfigManager.Instance.getStoryboardDirectory()} for storyboard directory`)this.initialized = trueawait this.checkAvailableJobs()diff --git a/apps/peertube-runner/src/server/shared/supported-job.ts b/apps/peertube-runner/src/server/shared/supported-job.tsindex 6be753cd74..bf137669f6 100644--- a/apps/peertube-runner/src/server/shared/supported-job.ts+++ b/apps/peertube-runner/src/server/shared/supported-job.ts@@ -7,7 +7,8 @@ import {RunnerJobVODAudioMergeTranscodingPayload,RunnerJobVODHLSTranscodingPayload,RunnerJobVODWebVideoTranscodingPayload,- VideoStudioTaskPayload+ VideoStudioTaskPayload,+ RunnerJobGenerateStoryboardPayload} from '@peertube/peertube-models'const supportedMatrix: { [ id in RunnerJobType ]: (payload: RunnerJobPayload) => boolean } = {@@ -33,7 +34,8 @@ const supportedMatrix: { [ id in RunnerJobType ]: (payload: RunnerJobPayload) =>},'video-transcription': (_payload: RunnerJobTranscriptionPayload) => {return true- }+ },+ 'generate-video-storyboard': (_payload: RunnerJobGenerateStoryboardPayload) => true}export function isJobSupported (job: { type: RunnerJobType, payload: RunnerJobPayload }, enabledJobs?: Set<RunnerJobType>) {diff --git a/apps/peertube-runner/src/shared/config-manager.ts b/apps/peertube-runner/src/shared/config-manager.tsindex 1c1550f551..a4d01a2026 100644--- a/apps/peertube-runner/src/shared/config-manager.ts+++ b/apps/peertube-runner/src/shared/config-manager.ts@@ -108,6 +108,10 @@ export class ConfigManager {// ---------------------------------------------------------------------------+ getStoryboardDirectory () {+ return join(paths.cache, this.id, 'storyboard')+ }+getTranscodingDirectory () {return join(paths.cache, this.id, 'transcoding')}diff --git a/client/src/app/+admin/config/pages/admin-config-general.component.html b/client/src/app/+admin/config/pages/admin-config-general.component.htmlindex 3570f57f78..a3e37c9551 100644--- a/client/src/app/+admin/config/pages/admin-config-general.component.html+++ b/client/src/app/+admin/config/pages/admin-config-general.component.html@@ -395,6 +395,20 @@<ng-container ngProjectAs="description"><span i18n>Generate storyboards of local videos using ffmpeg so users can see the video preview in the player while scrubbing the video</span></ng-container>++ <ng-container ngProjectAs="extra">+ <div class="form-group" formGroupName="remoteRunners" [ngClass]="getStoryboardRunnerDisabledClass()">+ <my-peertube-checkbox+ inputName="storyboardsRemoteRunnersEnabled" formControlName="enabled"+ i18n-labelText labelText="Enable remote runners to generate storyboards"+ >+ <ng-container ngProjectAs="description">+ <div i18n>Use <a routerLink="/admin/settings/system/runners/runners-list">remote runners</a> to generate storyboards.</div>+ <div i18n>Remote runners have to register on your instance first.</div>+ </ng-container>+ </my-peertube-checkbox>+ </div>+ </ng-container></my-peertube-checkbox></div></ng-container>@@ -416,10 +430,8 @@i18n-labelText labelText="Enable remote runners for transcription"><ng-container ngProjectAs="description">- <span i18n>- Use <a routerLink="/admin/settings/system/runners/runners-list">remote runners</a> to process transcription tasks.- Remote runners has to register on your instance first.- </span>+ <div i18n>Use <a routerLink="/admin/settings/system/runners/runners-list">remote runners</a> to process transcription tasks.</div>+ <div i18n>Remote runners have to register on your instance first.</div></ng-container></my-peertube-checkbox></div>diff --git a/client/src/app/+admin/config/pages/admin-config-general.component.ts b/client/src/app/+admin/config/pages/admin-config-general.component.tsindex 1cf99599b2..48136ab237 100644--- a/client/src/app/+admin/config/pages/admin-config-general.component.ts+++ b/client/src/app/+admin/config/pages/admin-config-general.component.ts@@ -182,6 +182,10 @@ type Form = {storyboards: FormGroup<{enabled: FormControl<boolean>++ remoteRunners: FormGroup<{+ enabled: FormControl<boolean>+ }>}>defaults: FormGroup<{@@ -423,7 +427,10 @@ export class AdminConfigGeneralComponent implements OnInit, OnDestroy, CanCompon}},storyboards: {- enabled: null+ enabled: null,+ remoteRunners: {+ enabled: null+ }},defaults: {publish: {@@ -536,6 +543,16 @@ export class AdminConfigGeneralComponent implements OnInit, OnDestroy, CanCompon// ---------------------------------------------------------------------------+ isStoryboardEnabled () {+ return this.form.value.storyboards.enabled === true+ }++ getStoryboardRunnerDisabledClass () {+ return { 'disabled-checkbox-extra': !this.isStoryboardEnabled() }+ }++ // ---------------------------------------------------------------------------+isAutoFollowIndexEnabled () {return this.form.value.followings.instance.autoFollowIndex.enabled === true}diff --git a/client/src/app/+admin/config/pages/admin-config-live.component.html b/client/src/app/+admin/config/pages/admin-config-live.component.htmlindex 2bd0b15afa..96b85d3eb5 100644--- a/client/src/app/+admin/config/pages/admin-config-live.component.html+++ b/client/src/app/+admin/config/pages/admin-config-live.component.html@@ -167,10 +167,8 @@i18n-labelText labelText="Enable remote runners for lives"><ng-container ngProjectAs="description">- <span i18n>- Use <a routerLink="/admin/settings/system/runners/runners-list">remote runners</a> to process live transcoding.- Remote runners has to register on your instance first.- </span>+ <div i18n>Use <a routerLink="/admin/settings/system/runners/runners-list">remote runners</a> to process live transcoding.</div>+ <div i18n>Remote runners have to register on your instance first.</div></ng-container></my-peertube-checkbox></div>diff --git a/client/src/app/+admin/config/pages/admin-config-vod.component.html b/client/src/app/+admin/config/pages/admin-config-vod.component.htmlindex 791b3da455..fe05587345 100644--- a/client/src/app/+admin/config/pages/admin-config-vod.component.html+++ b/client/src/app/+admin/config/pages/admin-config-vod.component.html@@ -185,10 +185,8 @@i18n-labelText labelText="Enable remote runners for VOD"><ng-container ngProjectAs="description">- <span i18n>- Use <a routerLink="/admin/settings/system/runners/runners-list">remote runners</a> to process VOD transcoding.- Remote runners has to register on your instance first.- </span>+ <div i18n>Use <a routerLink="/admin/settings/system/runners/runners-list">remote runners</a> to process VOD transcoding.</div>+ <div i18n>Remote runners have to register on your instance first.</div></ng-container></my-peertube-checkbox></div>@@ -262,19 +260,19 @@<ng-container ngProjectAs="description" *ngIf="!isTranscodingEnabled()"><span i18n>⚠️ You need to enable transcoding first to enable video studio</span></ng-container>- </my-peertube-checkbox>- </div>- <div class="form-group" formGroupName="remoteRunners" [ngClass]="getStudioDisabledClass()">- <my-peertube-checkbox- inputName="videoStudioRemoteRunnersEnabled" formControlName="enabled"- i18n-labelText labelText="Enable remote runners for studio"- >- <ng-container ngProjectAs="description">- <span i18n>- Use <a routerLink="/admin/settings/system/runners/runners-list">remote runners</a> to process studio transcoding tasks.- Remote runners has to register on your instance first.- </span>+ <ng-container ngProjectAs="extra">+ <div class="form-group" formGroupName="remoteRunners" [ngClass]="getStudioRunnerDisabledClass()">+ <my-peertube-checkbox+ inputName="videoStudioRemoteRunnersEnabled" formControlName="enabled"+ i18n-labelText labelText="Enable remote runners for studio"+ >+ <ng-container ngProjectAs="description">+ <div i18n>Use <a routerLink="/admin/settings/system/runners/runners-list">remote runners</a> to process studio transcoding tasks.</div>+ <div i18n>Remote runners have to register on your instance first.</div>+ </ng-container>+ </my-peertube-checkbox>+ </div></ng-container></my-peertube-checkbox></div>diff --git a/client/src/app/+admin/config/pages/admin-config-vod.component.ts b/client/src/app/+admin/config/pages/admin-config-vod.component.tsindex 1a5f14037a..ee2f1de458 100644--- a/client/src/app/+admin/config/pages/admin-config-vod.component.ts+++ b/client/src/app/+admin/config/pages/admin-config-vod.component.ts@@ -226,7 +226,7 @@ export class AdminConfigVODComponent implements OnInit, OnDestroy, CanComponentDreturn { 'disabled-checkbox-extra': !this.isTranscodingEnabled() || this.isRemoteRunnerVODEnabled() }}- getStudioDisabledClass () {+ getStudioRunnerDisabledClass () {return { 'disabled-checkbox-extra': !this.isStudioEnabled() }}diff --git a/config/default.yaml b/config/default.yamlindex f6be2eb4cc..740f9c0659 100644--- a/config/default.yaml+++ b/config/default.yaml@@ -1137,6 +1137,10 @@ storyboards:# Generate storyboards of local videos using ffmpeg so users can see the video preview in the player while scrubbing the videoenabled: true+ remote_runners:+ # Use remote runners to generate storyboards instead of processing them locally+ enabled: false+# Update default PeerTube values# Set by API when the field is not provided and put as default value in clientdefaults:diff --git a/config/production.yaml.example b/config/production.yaml.exampleindex 6901b8ea74..6648813f33 100644--- a/config/production.yaml.example+++ b/config/production.yaml.example@@ -340,6 +340,9 @@ security:powered_by_header:enabled: true+http_server:+ request_timeout: '5 minutes'+tracker:# If you disable the tracker, you disable the P2P on your PeerTube instanceenabled: true@@ -1144,6 +1147,10 @@ storyboards:# Generate storyboards of local videos using ffmpeg so users can see the video preview in the player while scrubbing the videoenabled: true+ remote_runners:+ # Use remote runners to generate storyboards instead of processing them locally+ enabled: false+# Update default PeerTube values# Set by API when the field is not provided and put as default value in clientdefaults:diff --git a/packages/models/src/runners/runner-jobs/runner-job-payload.model.ts b/packages/models/src/runners/runner-jobs/runner-job-payload.model.tsindex ddc8a33ec5..6c9db15dde 100644--- a/packages/models/src/runners/runner-jobs/runner-job-payload.model.ts+++ b/packages/models/src/runners/runner-jobs/runner-job-payload.model.ts@@ -14,7 +14,8 @@ export type RunnerJobPayload =RunnerJobVODPayload |RunnerJobLiveRTMPHLSTranscodingPayload |RunnerJobStudioTranscodingPayload |- RunnerJobTranscriptionPayload+ RunnerJobTranscriptionPayload |+ RunnerJobGenerateStoryboardPayload// ---------------------------------------------------------------------------@@ -90,6 +91,24 @@ export interface RunnerJobTranscriptionPayload {}}+export interface RunnerJobGenerateStoryboardPayload {+ input: {+ videoFileUrl: string+ }++ // Computed server-side for consistency across runners+ sprites: {+ size: { width: number, height: number }+ count: { width: number, height: number }+ duration: number+ }++ output: {+ // To upload on an external URL+ storyboardFileCustomUpload?: RunnerJobCustomUpload+ }+}+// ---------------------------------------------------------------------------export function isAudioMergeTranscodingPayload (payload: RunnerJobPayload): payload is RunnerJobVODAudioMergeTranscodingPayload {diff --git a/packages/models/src/runners/runner-jobs/runner-job-private-payload.model.ts b/packages/models/src/runners/runner-jobs/runner-job-private-payload.model.tsindex 6caf38fcd0..4a2f679f30 100644--- a/packages/models/src/runners/runner-jobs/runner-job-private-payload.model.ts+++ b/packages/models/src/runners/runner-jobs/runner-job-private-payload.model.ts@@ -1,15 +1,16 @@import { VideoStudioTaskPayload } from '../../server/index.js'export type RunnerJobVODPrivatePayload =- RunnerJobVODWebVideoTranscodingPrivatePayload |- RunnerJobVODAudioMergeTranscodingPrivatePayload |- RunnerJobVODHLSTranscodingPrivatePayload+ | RunnerJobVODWebVideoTranscodingPrivatePayload+ | RunnerJobVODAudioMergeTranscodingPrivatePayload+ | RunnerJobVODHLSTranscodingPrivatePayloadexport type RunnerJobPrivatePayload =- RunnerJobVODPrivatePayload |- RunnerJobLiveRTMPHLSTranscodingPrivatePayload |- RunnerJobVideoStudioTranscodingPrivatePayload |- RunnerJobTranscriptionPrivatePayload+ | RunnerJobVODPrivatePayload+ | RunnerJobLiveRTMPHLSTranscodingPrivatePayload+ | RunnerJobVideoStudioTranscodingPrivatePayload+ | RunnerJobTranscriptionPrivatePayload+ | RunnerJobGenerateStoryboardPrivatePayload// ---------------------------------------------------------------------------@@ -52,3 +53,7 @@ export interface RunnerJobVideoStudioTranscodingPrivatePayload {export interface RunnerJobTranscriptionPrivatePayload {videoUUID: string}++export interface RunnerJobGenerateStoryboardPrivatePayload {+ videoUUID: string+}diff --git a/packages/models/src/runners/runner-jobs/runner-job-success-body.model.ts b/packages/models/src/runners/runner-jobs/runner-job-success-body.model.tsindex 541f815ca3..a7e3010c13 100644--- a/packages/models/src/runners/runner-jobs/runner-job-success-body.model.ts+++ b/packages/models/src/runners/runner-jobs/runner-job-success-body.model.ts@@ -13,7 +13,8 @@ export type RunnerJobSuccessPayload =VODAudioMergeTranscodingSuccess |LiveRTMPHLSTranscodingSuccess |VideoStudioTranscodingSuccess |- TranscriptionSuccess+ TranscriptionSuccess |+ GenerateStoryboardSuccessexport interface VODWebVideoTranscodingSuccess {videoFile: Blob | string@@ -42,6 +43,10 @@ export interface TranscriptionSuccess {vttFile: Blob | string}+export interface GenerateStoryboardSuccess {+ storyboardFile: Blob | string+}+export function isWebVideoOrAudioMergeTranscodingPayloadSuccess (payload: RunnerJobSuccessPayload): payload is VODHLSTranscodingSuccess | VODAudioMergeTranscodingSuccess {@@ -55,3 +60,7 @@ export function isHLSTranscodingPayloadSuccess (payload: RunnerJobSuccessPayloadexport function isTranscriptionPayloadSuccess (payload: RunnerJobSuccessPayload): payload is TranscriptionSuccess {return !!(payload as TranscriptionSuccess)?.vttFile}++export function isGenerateStoryboardSuccess (payload: RunnerJobSuccessPayload): payload is GenerateStoryboardSuccess {+ return !!(payload as GenerateStoryboardSuccess)?.storyboardFile+}diff --git a/packages/models/src/runners/runner-jobs/runner-job-type.type.ts b/packages/models/src/runners/runner-jobs/runner-job-type.type.tsindex 0456e788cf..81dee08113 100644--- a/packages/models/src/runners/runner-jobs/runner-job-type.type.ts+++ b/packages/models/src/runners/runner-jobs/runner-job-type.type.ts@@ -4,4 +4,5 @@ export type RunnerJobType ='vod-audio-merge-transcoding' |'live-rtmp-hls-transcoding' |'video-studio-transcoding' |- 'video-transcription'+ 'video-transcription' |+ 'generate-video-storyboard'diff --git a/packages/models/src/server/custom-config.model.ts b/packages/models/src/server/custom-config.model.tsindex 993d7bdd86..fb24334941 100644--- a/packages/models/src/server/custom-config.model.ts+++ b/packages/models/src/server/custom-config.model.ts@@ -345,6 +345,9 @@ export interface CustomConfig {storyboards: {enabled: boolean+ remoteRunners: {+ enabled: boolean+ }}defaults: {diff --git a/packages/models/src/server/server-config.model.ts b/packages/models/src/server/server-config.model.tsindex 5d8aeac070..4bb84d4067 100644--- a/packages/models/src/server/server-config.model.ts+++ b/packages/models/src/server/server-config.model.ts@@ -418,6 +418,9 @@ export interface ServerConfig {storyboards: {enabled: boolean+ remoteRunners: {+ enabled: boolean+ }}videoTranscription: {diff --git a/packages/server-commands/src/runners/runner-jobs-command.ts b/packages/server-commands/src/runners/runner-jobs-command.tsindex b727c89606..73426d09b9 100644--- a/packages/server-commands/src/runners/runner-jobs-command.ts+++ b/packages/server-commands/src/runners/runner-jobs-command.ts@@ -4,6 +4,7 @@ import {AcceptRunnerJobBody,AcceptRunnerJobResult,ErrorRunnerJobBody,+ GenerateStoryboardSuccess,HttpStatusCode,ListRunnerJobsQuery,RequestRunnerJobBody,@@ -11,6 +12,7 @@ import {ResultList,RunnerJobAdmin,RunnerJobCustomUpload,+ RunnerJobGenerateStoryboardPayload,RunnerJobLiveRTMPHLSTranscodingPayload,RunnerJobPayload,RunnerJobState,@@ -26,6 +28,7 @@ import {TranscriptionSuccess,VODHLSTranscodingSuccess,VODWebVideoTranscodingSuccess,+ isGenerateStoryboardSuccess,isHLSTranscodingPayloadSuccess,isLiveRTMPHLSTranscodingUpdatePayload,isTranscriptionPayloadSuccess,@@ -265,6 +268,22 @@ export class RunnerJobsCommand extends AbstractCommand {payloadWithoutFiles = omit(payloadWithoutFiles as TranscriptionSuccess, [ 'vttFile' ])}+ // Generate storyboard success payload contains a storyboard image file+ if (isGenerateStoryboardSuccess(payload) && payload.storyboardFile) {+ const reqPayload = options.reqPayload as RunnerJobGenerateStoryboardPayload++ this.updateUploadPayloads({+ attachesStore: attaches,+ customUploadsStore: customUploads,++ file: payload.storyboardFile,+ attachName: 'storyboardFile',+ customUpload: reqPayload?.output?.storyboardFileCustomUpload+ })++ payloadWithoutFiles = omit(payloadWithoutFiles as GenerateStoryboardSuccess, [ 'storyboardFile' ])+ }+return this.uploadRunnerJobRequest({...options,@@ -367,12 +386,17 @@ export class RunnerJobsCommand extends AbstractCommand {if (!jobUUID) {const { availableJobs } = await this.request({ runnerToken })- jobUUID = availableJobs[0].uuid+ // Find a web video transcoding job specifically+ const webVideoJob = availableJobs.find(j => j.type === 'vod-web-video-transcoding')+ if (!webVideoJob) throw new Error('No web video transcoding jobs available')++ jobUUID = webVideoJob.uuid}const { job } = await this.accept({ runnerToken, jobUUID })const jobToken = job.jobToken+ // Use a proper fixture file path for testingconst payload: RunnerJobSuccessPayload = { videoFile: 'video_short.mp4' }await this.success({ runnerToken, jobUUID, jobToken, payload, reqPayload: undefined })diff --git a/packages/tests/src/api/runners/index.ts b/packages/tests/src/api/runners/index.tsindex ed82228920..d9c8bbedab 100644--- a/packages/tests/src/api/runners/index.ts+++ b/packages/tests/src/api/runners/index.ts@@ -1,4 +1,5 @@export * from './runner-common.js'+export * from './runner-generate-storyboard.js'export * from './runner-live-transcoding.js'export * from './runner-socket.js'export * from './runner-studio-transcoding.js'diff --git a/packages/tests/src/api/runners/runner-generate-storyboard.ts b/packages/tests/src/api/runners/runner-generate-storyboard.tsnew file mode 100644index 0000000000..a8acdded9f--- /dev/null+++ b/packages/tests/src/api/runners/runner-generate-storyboard.ts@@ -0,0 +1,92 @@+/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */++import { GenerateStoryboardSuccess, RunnerJobGenerateStoryboardPayload } from '@peertube/peertube-models'+import {+ PeerTubeServer,+ cleanupTests,+ createMultipleServers,+ doubleFollow,+ setAccessTokensToServers,+ setDefaultVideoChannel,+ waitJobs+} from '@peertube/peertube-server-commands'+import { testImage } from '@tests/shared/checks.js'+import { checkPersistentTmpIsEmpty } from '@tests/shared/directories.js'+import { expect } from 'chai'++describe('Test runner generate storyboard', function () {+ let servers: PeerTubeServer[] = []+ let runnerToken: string++ before(async function () {+ this.timeout(120_000)++ servers = await createMultipleServers(2)++ await setAccessTokensToServers(servers)+ await setDefaultVideoChannel(servers)++ await doubleFollow(servers[0], servers[1])++ await servers[0].config.updateExistingConfig({+ newConfig: {+ storyboards: {+ enabled: true,+ remoteRunners: {+ enabled: true+ }+ }+ }+ })++ runnerToken = await servers[0].runners.autoRegisterRunner()+ })++ async function upload () {+ const { uuid } = await servers[0].videos.upload({ attributes: { name: 'video', language: undefined } })+ await waitJobs(servers)++ const { availableJobs } = await servers[0].runnerJobs.request({ runnerToken })+ expect(availableJobs).to.have.lengthOf(1)++ const jobUUID = availableJobs[0].uuid++ const { job } = await servers[0].runnerJobs.accept<RunnerJobGenerateStoryboardPayload>({ runnerToken, jobUUID })+ return { uuid, job }+ }++ it('Should execute a remote generate storyboard job', async function () {+ this.timeout(240_000)++ const { uuid, job } = await upload()++ expect(job.type === 'generate-video-storyboard')+ expect(job.payload.input.videoFileUrl).to.exist++ // Check video input file+ {+ await servers[0].runnerJobs.getJobFile({ url: job.payload.input.videoFileUrl, jobToken: job.jobToken, runnerToken })+ }++ const payload: GenerateStoryboardSuccess = {+ storyboardFile: 'banner.jpg'+ }++ await servers[0].runnerJobs.success({ runnerToken, jobUUID: job.uuid, jobToken: job.jobToken, payload })++ await waitJobs(servers)++ for (const server of servers) {+ const { storyboards } = await server.storyboard.list({ id: uuid })++ expect(storyboards).to.have.lengthOf(1)++ await testImage({ name: 'banner.jpg', url: storyboards[0].fileUrl })+ }+ await checkPersistentTmpIsEmpty(servers[0])+ })++ after(async function () {+ await cleanupTests(servers)+ })+})diff --git a/packages/tests/src/api/server/config.ts b/packages/tests/src/api/server/config.tsindex 463e00eeaf..1aa30e24c4 100644--- a/packages/tests/src/api/server/config.ts+++ b/packages/tests/src/api/server/config.ts@@ -443,7 +443,10 @@ function buildNewCustomConfig (server: PeerTubeServer): CustomConfig {}},storyboards: {- enabled: false+ enabled: false,+ remoteRunners: {+ enabled: true+ }},export: {users: {diff --git a/packages/tests/src/api/videos/index.ts b/packages/tests/src/api/videos/index.tsindex aa714c44ab..6e1348548f 100644--- a/packages/tests/src/api/videos/index.ts+++ b/packages/tests/src/api/videos/index.ts@@ -20,6 +20,7 @@ import './video-schedule-update.js'import './video-source.js'import './video-static-file-privacy.js'import './video-storyboard.js'+import './video-storyboard-remote-runner.js'import './video-transcription.js'import './videos-common-filters.js'import './videos-history.js'diff --git a/packages/tests/src/api/videos/video-storyboard.ts b/packages/tests/src/api/videos/video-storyboard.tsindex 2ede57a1e0..501213fd61 100644--- a/packages/tests/src/api/videos/video-storyboard.ts+++ b/packages/tests/src/api/videos/video-storyboard.ts@@ -1,17 +1,11 @@/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */-import { expect } from 'chai'-import { readdir } from 'fs/promises'-import { basename } from 'path'-import { FIXTURE_URLS } from '@tests/shared/fixture-urls.js'+import { VideoPrivacy } from '@peertube/peertube-models'import { areHttpImportTestsDisabled } from '@peertube/peertube-node-utils'-import { HttpStatusCode, VideoPrivacy } from '@peertube/peertube-models'import {cleanupTests,createMultipleServers,doubleFollow,- makeGetRequest,- makeRawRequest,PeerTubeServer,sendRTMPStream,setAccessTokensToServers,@@ -19,44 +13,11 @@ import {stopFfmpeg,waitJobs} from '@peertube/peertube-server-commands'--async function checkStoryboard (options: {- server: PeerTubeServer- uuid: string- spriteHeight?: number- spriteWidth?: number- tilesCount?: number- minSize?: number- spriteDuration?: number-}) {- const { server, uuid, tilesCount, spriteDuration = 1, spriteHeight = 108, spriteWidth = 192, minSize = 1000 } = options-- const { storyboards } = await server.storyboard.list({ id: uuid })-- expect(storyboards).to.have.lengthOf(1)-- const storyboard = storyboards[0]-- expect(storyboard.spriteDuration).to.equal(spriteDuration)- expect(storyboard.spriteHeight).to.equal(spriteHeight)- expect(storyboard.spriteWidth).to.equal(spriteWidth)- expect(storyboard.storyboardPath).to.exist-- if (tilesCount) {- expect(storyboard.totalWidth).to.equal(spriteWidth * Math.min(tilesCount, 11))- expect(storyboard.totalHeight).to.equal(spriteHeight * Math.max((tilesCount / 11), 1))- }-- {- const { body } = await makeGetRequest({ url: server.url, path: storyboard.storyboardPath, expectedStatus: HttpStatusCode.OK_200 })- expect(body.length).to.be.above(minSize)- }-- {- const { body } = await makeRawRequest({ url: storyboard.fileUrl, expectedStatus: HttpStatusCode.OK_200 })- expect(body.length).to.be.above(minSize)- }-}+import { FIXTURE_URLS } from '@tests/shared/fixture-urls.js'+import { checkStoryboard } from '@tests/shared/storyboard.js'+import { expect } from 'chai'+import { readdir } from 'fs/promises'+import { basename } from 'path'describe('Test video storyboard', function () {let servers: PeerTubeServer[]@@ -205,16 +166,16 @@ describe('Test video storyboard', function () {}{- const storyboads = await listFiles()- expect(storyboads).to.include(storyboardName)+ const storyboards = await listFiles()+ expect(storyboards).to.include(storyboardName)}await servers[0].videos.remove({ id: baseUUID })await waitJobs(servers){- const storyboads = await listFiles()- expect(storyboads).to.not.include(storyboardName)+ const storyboards = await listFiles()+ expect(storyboards).to.not.include(storyboardName)}})diff --git a/packages/tests/src/peertube-runner/generate-storyboard.ts b/packages/tests/src/peertube-runner/generate-storyboard.tsnew file mode 100644index 0000000000..4cfbc453fa--- /dev/null+++ b/packages/tests/src/peertube-runner/generate-storyboard.ts@@ -0,0 +1,189 @@+/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */++import { wait } from '@peertube/peertube-core-utils'+import { RunnerJobState, VideoPrivacy } from '@peertube/peertube-models'+import { areHttpImportTestsDisabled } from '@peertube/peertube-node-utils'+import {+ PeerTubeServer,+ cleanupTests,+ createMultipleServers,+ doubleFollow,+ sendRTMPStream,+ setAccessTokensToServers,+ setDefaultVideoChannel,+ stopFfmpeg,+ waitJobs+} from '@peertube/peertube-server-commands'+import { checkPeerTubeRunnerCacheIsEmpty } from '@tests/shared/directories.js'+import { FIXTURE_URLS } from '@tests/shared/fixture-urls.js'+import { PeerTubeRunnerProcess } from '@tests/shared/peertube-runner-process.js'+import { checkNoStoryboard, checkStoryboard } from '@tests/shared/storyboard.js'+import { expect } from 'chai'++describe('Test generate storyboard in peertube-runner program', function () {+ let servers: PeerTubeServer[] = []+ let peertubeRunner: PeerTubeRunnerProcess++ before(async function () {+ this.timeout(120_000)++ servers = await createMultipleServers(2)++ await setAccessTokensToServers(servers)+ await setDefaultVideoChannel(servers)++ await doubleFollow(servers[0], servers[1])++ await servers[0].config.updateExistingConfig({+ newConfig: {+ storyboards: {+ enabled: true,+ remoteRunners: {+ enabled: true+ }+ }+ }+ })++ const registrationToken = await servers[0].runnerRegistrationTokens.getFirstRegistrationToken()++ peertubeRunner = new PeerTubeRunnerProcess(servers[0])+ await peertubeRunner.runServer({ jobType: 'generate-video-storyboard' })+ await peertubeRunner.registerPeerTubeInstance({ registrationToken, runnerName: 'runner' })+ })++ describe('Running storyboard generation', function () {+ describe('Common on filesystem', function () {+ it('Should run generate storyboard on classic file without transcoding', async function () {+ this.timeout(360000)++ const { uuid } = await servers[0].videos.quickUpload({ name: 'video' })+ await waitJobs(servers, { runnerJobs: true })++ for (const server of servers) {+ await checkStoryboard({ server, uuid, tilesCount: 5 })+ }+ })++ it('Should run generate storyboard on classic file with transcoding', async function () {+ this.timeout(360000)++ await servers[0].config.enableMinimumTranscoding()++ const { uuid } = await servers[0].videos.quickUpload({ name: 'video' })+ await waitJobs(servers, { runnerJobs: true })++ for (const server of servers) {+ await checkStoryboard({ server, uuid, tilesCount: 5 })+ }++ await servers[0].config.disableTranscoding()+ })++ it('Should generate a storyboard after HTTP import', async function () {+ this.timeout(120000)++ if (areHttpImportTestsDisabled()) return++ // 3s video+ const { video } = await servers[0].videoImports.importVideo({+ attributes: {+ targetUrl: FIXTURE_URLS.goodVideo,+ channelId: servers[0].store.channel.id,+ privacy: VideoPrivacy.PUBLIC+ }+ })+ await waitJobs(servers, { runnerJobs: true })++ for (const server of servers) {+ await checkStoryboard({ server, uuid: video.uuid, spriteHeight: 144, tilesCount: 3 })+ }+ })++ it('Should generate a storyboard after torrent import', async function () {+ this.timeout(240000)++ if (areHttpImportTestsDisabled()) return++ // 10s video+ const { video } = await servers[0].videoImports.importVideo({+ attributes: {+ magnetUri: FIXTURE_URLS.magnet,+ channelId: servers[0].store.channel.id,+ privacy: VideoPrivacy.PUBLIC+ }+ })+ await waitJobs(servers, { runnerJobs: true })++ for (const server of servers) {+ await checkStoryboard({ server, uuid: video.uuid, tilesCount: 10 })+ }+ })++ it('Should generate a storyboard after a live', async function () {+ this.timeout(240000)++ await servers[0].config.enableMinimumTranscoding()+ await servers[0].config.enableLive({ allowReplay: true, transcoding: true, resolutions: 'min' })++ const { live, video } = await servers[0].live.quickCreate({+ saveReplay: true,+ permanentLive: false,+ privacy: VideoPrivacy.PUBLIC+ })++ const ffmpegCommand = sendRTMPStream({ rtmpBaseUrl: live.rtmpUrl, streamKey: live.streamKey })+ await servers[0].live.waitUntilPublished({ videoId: video.id })++ await stopFfmpeg(ffmpegCommand)++ await servers[0].live.waitUntilReplacedByReplay({ videoId: video.id })+ await waitJobs(servers, { runnerJobs: true })++ for (const server of servers) {+ await checkStoryboard({ server, uuid: video.uuid })+ }+ })+ })++ describe('When generate storyboard is not enabled in runner', function () {+ before(async function () {+ await peertubeRunner.unregisterPeerTubeInstance({ runnerName: 'runner' })+ peertubeRunner.kill()+ await wait(500)++ const registrationToken = await servers[0].runnerRegistrationTokens.getFirstRegistrationToken()+ await peertubeRunner.runServer({ jobType: 'live-rtmp-hls-transcoding' })+ await peertubeRunner.registerPeerTubeInstance({ registrationToken, runnerName: 'runner' })+ })++ it('Should not run generate storyboard', async function () {+ this.timeout(60000)++ const { uuid } = await servers[0].videos.quickUpload({ name: 'video' })+ await waitJobs(servers)+ await wait(2000)++ const { data } = await servers[0].runnerJobs.list({ stateOneOf: [ RunnerJobState.PENDING ] })+ expect(data.some(j => j.type === 'generate-video-storyboard')).to.be.true++ await checkNoStoryboard({ server: servers[0], uuid })+ })+ })++ describe('Check cleanup', function () {+ it('Should have an empty cache directory', async function () {+ await checkPeerTubeRunnerCacheIsEmpty(peertubeRunner, 'storyboard')+ })+ })+ })++ after(async function () {+ if (peertubeRunner) {+ await peertubeRunner.unregisterPeerTubeInstance({ runnerName: 'runner' })+ peertubeRunner.kill()+ }++ await cleanupTests(servers)+ })+})diff --git a/packages/tests/src/peertube-runner/index.ts b/packages/tests/src/peertube-runner/index.tsindex 41758deec2..9afba566a2 100644--- a/packages/tests/src/peertube-runner/index.ts+++ b/packages/tests/src/peertube-runner/index.ts@@ -1,5 +1,6 @@export * from './client-cli.js'export * from './custom-upload.js'+export * from './generate-storyboard.js'export * from './live-transcoding.js'export * from './replace-file.js'export * from './shutdown.js'diff --git a/packages/tests/src/shared/directories.ts b/packages/tests/src/shared/directories.tsindex e9be0c77d6..d7d7cbfbc9 100644--- a/packages/tests/src/shared/directories.ts+++ b/packages/tests/src/shared/directories.ts@@ -32,7 +32,10 @@ export async function checkDirectoryIsEmpty (server: PeerTubeServer, directory:expect(filtered).to.have.lengthOf(0)}-export async function checkPeerTubeRunnerCacheIsEmpty (runner: PeerTubeRunnerProcess, subDir: 'transcoding' | 'transcription') {+export async function checkPeerTubeRunnerCacheIsEmpty (+ runner: PeerTubeRunnerProcess,+ subDir: 'transcoding' | 'transcription' | 'storyboard'+) {const directoryPath = join(homedir(), '.cache', 'peertube-runner-nodejs', runner.getId(), subDir)const directoryExists = await pathExists(directoryPath)diff --git a/packages/tests/src/shared/storyboard.ts b/packages/tests/src/shared/storyboard.tsnew file mode 100644index 0000000000..801cdf8ac6--- /dev/null+++ b/packages/tests/src/shared/storyboard.ts@@ -0,0 +1,53 @@+/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */++import { HttpStatusCode } from '@peertube/peertube-models'+import { makeGetRequest, makeRawRequest, PeerTubeServer } from '@peertube/peertube-server-commands'+import { expect } from 'chai'++export async function checkStoryboard (options: {+ server: PeerTubeServer+ uuid: string+ spriteHeight?: number+ spriteWidth?: number+ tilesCount?: number+ minSize?: number+ spriteDuration?: number+}) {+ const { server, uuid, tilesCount, spriteDuration = 1, spriteHeight = 108, spriteWidth = 192, minSize = 1000 } = options++ const { storyboards } = await server.storyboard.list({ id: uuid })+ expect(storyboards).to.have.lengthOf(1)++ const storyboard = storyboards[0]++ expect(storyboard.spriteDuration).to.equal(spriteDuration)+ expect(storyboard.spriteHeight).to.equal(spriteHeight)+ expect(storyboard.spriteWidth).to.equal(spriteWidth)+ expect(storyboard.storyboardPath).to.exist++ if (tilesCount) {+ expect(storyboard.totalWidth).to.equal(spriteWidth * Math.min(tilesCount, 11))+ expect(storyboard.totalHeight).to.equal(spriteHeight * Math.max(tilesCount / 11, 1))+ }++ {+ const { body } = await makeGetRequest({ url: server.url, path: storyboard.storyboardPath, expectedStatus: HttpStatusCode.OK_200 })+ expect(body.length).to.be.above(minSize)+ }++ {+ const { body } = await makeRawRequest({ url: storyboard.fileUrl, expectedStatus: HttpStatusCode.OK_200 })+ expect(body.length).to.be.above(minSize)+ }+}++export async function checkNoStoryboard (options: {+ server: PeerTubeServer+ uuid: string+}) {+ const { server, uuid } = options++ const { storyboards } = await server.storyboard.list({ id: uuid })++ expect(storyboards).to.have.lengthOf(0)+}diff --git a/server/core/controllers/api/config.ts b/server/core/controllers/api/config.tsindex df01b5ab56..20e2130f34 100644--- a/server/core/controllers/api/config.ts+++ b/server/core/controllers/api/config.ts@@ -582,7 +582,10 @@ function customConfig (): CustomConfig {}},storyboards: {- enabled: CONFIG.STORYBOARDS.ENABLED+ enabled: CONFIG.STORYBOARDS.ENABLED,+ remoteRunners: {+ enabled: CONFIG.STORYBOARDS.REMOTE_RUNNERS.ENABLED+ }},defaults: {publish: {diff --git a/server/core/controllers/api/runners/jobs.ts b/server/core/controllers/api/runners/jobs.tsindex ea0a87ac79..e7052ef7ec 100644--- a/server/core/controllers/api/runners/jobs.ts+++ b/server/core/controllers/api/runners/jobs.ts@@ -15,6 +15,7 @@ import {RunnerJobUpdatePayload,ServerErrorCode,TranscriptionSuccess,+ GenerateStoryboardSuccess,UserRight,VODAudioMergeTranscodingSuccess,VODHLSTranscodingSuccess,@@ -56,8 +57,13 @@ import { RunnerModel } from '@server/models/runner/runner.js'import express, { UploadFiles } from 'express'const postRunnerJobSuccessVideoFiles = createReqFiles(- [ 'payload[videoFile]', 'payload[resolutionPlaylistFile]', 'payload[vttFile]' ],- { ...MIMETYPES.VIDEO.MIMETYPE_EXT, ...MIMETYPES.M3U8.MIMETYPE_EXT, ...MIMETYPES.VIDEO_CAPTIONS.MIMETYPE_EXT }+ [ 'payload[videoFile]', 'payload[resolutionPlaylistFile]', 'payload[vttFile]', 'payload[storyboardFile]' ],+ {+ ...MIMETYPES.VIDEO.MIMETYPE_EXT,+ ...MIMETYPES.M3U8.MIMETYPE_EXT,+ ...MIMETYPES.VIDEO_CAPTIONS.MIMETYPE_EXT,+ ...MIMETYPES.IMAGE.MIMETYPE_EXT+ })const runnerJobUpdateVideoFiles = createReqFiles(@@ -384,6 +390,14 @@ const jobSuccessPayloadBuilders: {vttFile: files['payload[vttFile]'][0].path}+ },++ 'generate-video-storyboard': (payload: GenerateStoryboardSuccess, files) => {+ return {+ ...payload,++ storyboardFile: files['payload[storyboardFile]'][0].path+ }}}diff --git a/server/core/controllers/api/videos/source.ts b/server/core/controllers/api/videos/source.tsindex d0b8ebe9f3..375a6b96e4 100644--- a/server/core/controllers/api/videos/source.ts+++ b/server/core/controllers/api/videos/source.ts@@ -8,7 +8,7 @@ import { setupUploadResumableRoutes } from '@server/lib/uploadx.js'import { autoBlacklistVideoIfNeeded } from '@server/lib/video-blacklist.js'import { regenerateTranscriptionTaskIfNeeded } from '@server/lib/video-captions.js'import { buildNewFile, createVideoSource } from '@server/lib/video-file.js'-import { buildMoveVideoJob, buildStoryboardJobIfNeeded } from '@server/lib/video-jobs.js'+import { addRemoteStoryboardJobIfNeeded, buildLocalStoryboardJobIfNeeded, buildMoveVideoJob } from '@server/lib/video-jobs.js'import { VideoPathManager } from '@server/lib/video-path-manager.js'import { buildNextVideoState } from '@server/lib/video-state.js'import { openapiOperationDoc } from '@server/middlewares/doc.js'@@ -172,7 +172,7 @@ async function addVideoJobsAfterUpload (video: MVideoFullLight, videoFile: MVide}},- buildStoryboardJobIfNeeded({ video, federate: false }),+ buildLocalStoryboardJobIfNeeded({ video, federate: false }),{type: 'federate-video' as const,@@ -201,6 +201,7 @@ async function addVideoJobsAfterUpload (video: MVideoFullLight, videoFile: MVideawait JobQueue.Instance.createSequentialJobFlow(...jobs)+ await addRemoteStoryboardJobIfNeeded(video)await regenerateTranscriptionTaskIfNeeded(video)}diff --git a/server/core/helpers/custom-validators/runners/jobs.ts b/server/core/helpers/custom-validators/runners/jobs.tsindex ec0238b77b..c9097cfd7f 100644--- a/server/core/helpers/custom-validators/runners/jobs.ts+++ b/server/core/helpers/custom-validators/runners/jobs.ts@@ -7,7 +7,8 @@ import {VODAudioMergeTranscodingSuccess,VODHLSTranscodingSuccess,VODWebVideoTranscodingSuccess,- VideoStudioTranscodingSuccess+ VideoStudioTranscodingSuccess,+ GenerateStoryboardSuccess} from '@peertube/peertube-models'import { CONSTRAINTS_FIELDS, RUNNER_JOB_STATES } from '@server/initializers/constants.js'import { UploadFilesForCheck } from 'express'@@ -16,7 +17,15 @@ import { exists, isArray, isFileValid, isSafeFilename } from '../misc.js'const RUNNER_JOBS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.RUNNER_JOBS-const runnerJobTypes = new Set([ 'vod-hls-transcoding', 'vod-web-video-transcoding', 'vod-audio-merge-transcoding' ])+const runnerJobTypes = new Set([+ 'vod-hls-transcoding',+ 'vod-web-video-transcoding',+ 'vod-audio-merge-transcoding',+ 'live-rtmp-hls-transcoding',+ 'video-studio-transcoding',+ 'video-transcription',+ 'generate-video-storyboard'+])export function isRunnerJobTypeValid (value: RunnerJobType) {return runnerJobTypes.has(value)}@@ -27,7 +36,8 @@ export function isRunnerJobSuccessPayloadValid (value: RunnerJobSuccessPayload,isRunnerJobVODAudioMergeResultPayloadValid(value as VODHLSTranscodingSuccess, type, files) ||isRunnerJobLiveRTMPHLSResultPayloadValid(value as LiveRTMPHLSTranscodingSuccess, type) ||isRunnerJobVideoStudioResultPayloadValid(value as VideoStudioTranscodingSuccess, type, files) ||- isRunnerJobTranscriptionResultPayloadValid(value as TranscriptionSuccess, type, files)+ isRunnerJobTranscriptionResultPayloadValid(value as TranscriptionSuccess, type, files) ||+ isRunnerJobGenerateStoryboardResultPayloadValid(value as GenerateStoryboardSuccess, type, files)}// ---------------------------------------------------------------------------@@ -42,7 +52,8 @@ export function isRunnerJobUpdatePayloadValid (value: RunnerJobUpdatePayload, tyisRunnerJobVideoStudioUpdatePayloadValid(value, type, files) ||isRunnerJobVODAudioMergeUpdatePayloadValid(value, type, files) ||isRunnerJobLiveRTMPHLSUpdatePayloadValid(value, type, files) ||- isRunnerJobTranscriptionUpdatePayloadValid(value, type, files)+ isRunnerJobTranscriptionUpdatePayloadValid(value, type, files) ||+ isRunnerJobGenerateStoryboardUpdatePayloadValid(value, type, files)}// ---------------------------------------------------------------------------@@ -103,7 +114,7 @@ function isRunnerJobLiveRTMPHLSResultPayloadValid (value: LiveRTMPHLSTranscodingSuccess,type: RunnerJobType) {- return type === 'live-rtmp-hls-transcoding' && (!value || (typeof value === 'object' && Object.keys(value).length === 0))+ return type === 'live-rtmp-hls-transcoding' && isEmptyUpdatePayload(value)}function isRunnerJobVideoStudioResultPayloadValid (@@ -124,6 +135,15 @@ function isRunnerJobTranscriptionResultPayloadValid (isFileValid({ files, field: 'payload[vttFile]', mimeTypeRegex: null, maxSize: null })}+function isRunnerJobGenerateStoryboardResultPayloadValid (+ value: GenerateStoryboardSuccess,+ type: RunnerJobType,+ files: UploadFilesForCheck+) {+ return type === 'generate-video-storyboard' &&+ isFileValid({ files, field: 'payload[storyboardFile]', mimeTypeRegex: null, maxSize: null })+}+// ---------------------------------------------------------------------------function isRunnerJobVODWebVideoUpdatePayloadValid (@@ -131,8 +151,7 @@ function isRunnerJobVODWebVideoUpdatePayloadValid (type: RunnerJobType,_files: UploadFilesForCheck) {- return type === 'vod-web-video-transcoding' &&- (!value || (typeof value === 'object' && Object.keys(value).length === 0))+ return type === 'vod-web-video-transcoding' && isEmptyUpdatePayload(value)}function isRunnerJobVODHLSUpdatePayloadValid (@@ -140,8 +159,7 @@ function isRunnerJobVODHLSUpdatePayloadValid (type: RunnerJobType,_files: UploadFilesForCheck) {- return type === 'vod-hls-transcoding' &&- (!value || (typeof value === 'object' && Object.keys(value).length === 0))+ return type === 'vod-hls-transcoding' && isEmptyUpdatePayload(value)}function isRunnerJobVODAudioMergeUpdatePayloadValid (@@ -149,8 +167,7 @@ function isRunnerJobVODAudioMergeUpdatePayloadValid (type: RunnerJobType,_files: UploadFilesForCheck) {- return type === 'vod-audio-merge-transcoding' &&- (!value || (typeof value === 'object' && Object.keys(value).length === 0))+ return type === 'vod-audio-merge-transcoding' && isEmptyUpdatePayload(value)}function isRunnerJobTranscriptionUpdatePayloadValid (@@ -158,8 +175,7 @@ function isRunnerJobTranscriptionUpdatePayloadValid (type: RunnerJobType,_files: UploadFilesForCheck) {- return type === 'video-transcription' &&- (!value || (typeof value === 'object' && Object.keys(value).length === 0))+ return type === 'video-transcription' && isEmptyUpdatePayload(value)}function isRunnerJobLiveRTMPHLSUpdatePayloadValid (@@ -201,6 +217,17 @@ function isRunnerJobVideoStudioUpdatePayloadValid (type: RunnerJobType,_files: UploadFilesForCheck) {- return type === 'video-studio-transcoding' &&- (!value || (typeof value === 'object' && Object.keys(value).length === 0))+ return type === 'video-studio-transcoding' && isEmptyUpdatePayload(value)+}++function isRunnerJobGenerateStoryboardUpdatePayloadValid (+ value: RunnerJobUpdatePayload,+ type: RunnerJobType,+ _files: UploadFilesForCheck+) {+ return type === 'generate-video-storyboard' && isEmptyUpdatePayload(value)+}++function isEmptyUpdatePayload (value: any): boolean {+ return !value || (typeof value === 'object' && Object.keys(value).length === 0)}diff --git a/server/core/initializers/config.ts b/server/core/initializers/config.tsindex 409495d1d2..db5a549d93 100644--- a/server/core/initializers/config.ts+++ b/server/core/initializers/config.ts@@ -1099,6 +1099,11 @@ const CONFIG = {STORYBOARDS: {get ENABLED () {return config.get<boolean>('storyboards.enabled')+ },+ REMOTE_RUNNERS: {+ get ENABLED () {+ return config.get<boolean>('storyboards.remote_runners.enabled')+ }}},EMAIL: {diff --git a/server/core/initializers/constants.ts b/server/core/initializers/constants.tsindex c87f0fc214..aaa353938c 100644--- a/server/core/initializers/constants.ts+++ b/server/core/initializers/constants.ts@@ -291,6 +291,7 @@ export const REPEAT_JOBS: { [id in JobType]?: RepeatOptions } = {}}export const JOB_PRIORITY = {+ STORYBOARD: 95,TRANSCODING: 100,VIDEO_STUDIO: 150,TRANSCRIPTION: 200diff --git a/server/core/lib/job-queue/handlers/generate-storyboard.ts b/server/core/lib/job-queue/handlers/generate-storyboard.tsindex ffabac9dd9..dfe47ab504 100644--- a/server/core/lib/job-queue/handlers/generate-storyboard.ts+++ b/server/core/lib/job-queue/handlers/generate-storyboard.ts@@ -1,25 +1,20 @@-import { FFmpegImage, ffprobePromise, getVideoStreamDimensionsInfo } from '@peertube/peertube-ffmpeg'+import { FFmpegImage } from '@peertube/peertube-ffmpeg'import { GenerateStoryboardPayload, VideoFileStream } from '@peertube/peertube-models'-import { retryTransactionWrapper } from '@server/helpers/database-utils.js'import { getFFmpegCommandWrapperOptions } from '@server/helpers/ffmpeg/index.js'import { generateImageFilename } from '@server/helpers/image-utils.js'import { logger, loggerTagsFactory } from '@server/helpers/logger.js'-import { deleteFileAndCatch } from '@server/helpers/utils.js'import { CONFIG } from '@server/initializers/config.js'import { STORYBOARD } from '@server/initializers/constants.js'-import { sequelizeTypescript } from '@server/initializers/database.js'-import { federateVideoIfNeeded } from '@server/lib/activitypub/videos/index.js'import { VideoPathManager } from '@server/lib/video-path-manager.js'import { getImageSizeFromWorker } from '@server/lib/worker/parent-process.js'-import { StoryboardModel } from '@server/models/video/storyboard.js'import { VideoModel } from '@server/models/video/video.js'-import { MVideo } from '@server/types/models/index.js'import { Job } from 'bullmq'import { join } from 'path'+import { buildSpriteSize, buildTotalSprites, findGridSize, insertStoryboardInDatabase } from '../../storyboard.js'const lTagsBase = loggerTagsFactory('storyboard')-async function processGenerateStoryboard (job: Job): Promise<void> {+export async function processGenerateStoryboard (job: Job): Promise<void> {const payload = job.data as GenerateStoryboardPayloadconst lTags = lTagsBase(payload.videoUUID)@@ -46,26 +41,9 @@ async function processGenerateStoryboard (job: Job): Promise<void> {}await VideoPathManager.Instance.makeAvailableVideoFile(inputFile, async videoPath => {- const probe = await ffprobePromise(videoPath)-- const videoStreamInfo = await getVideoStreamDimensionsInfo(videoPath, probe)- let spriteHeight: number- let spriteWidth: number-- if (videoStreamInfo.isPortraitMode) {- spriteHeight = STORYBOARD.SPRITE_MAX_SIZE- spriteWidth = Math.round(spriteHeight * videoStreamInfo.ratio)- } else {- spriteWidth = STORYBOARD.SPRITE_MAX_SIZE- spriteHeight = Math.round(spriteWidth / videoStreamInfo.ratio)- }-- const ffmpeg = new FFmpegImage(getFFmpegCommandWrapperOptions('thumbnail'))-- const filename = generateImageFilename()- const destination = join(CONFIG.STORAGE.STORYBOARDS_DIR, filename)+ const { spriteHeight, spriteWidth } = await buildSpriteSize(videoPath)- const { totalSprites, spriteDuration } = buildSpritesMetadata({ video })+ const { totalSprites, spriteDuration } = buildTotalSprites(video)if (totalSprites === 0) {logger.info(`Do not generate a storyboard of ${payload.videoUUID} because the video is not long enough`, lTags)return@@ -76,11 +54,16 @@ async function processGenerateStoryboard (job: Job): Promise<void> {maxEdgeCount: STORYBOARD.SPRITES_MAX_EDGE_COUNT})+ const filename = generateImageFilename()+ const destination = join(CONFIG.STORAGE.STORYBOARDS_DIR, filename)+logger.debug(`Generating storyboard from video of ${video.uuid} to ${destination}`,{ ...lTags, totalSprites, spritesCount, spriteDuration, videoDuration: video.duration, spriteHeight, spriteWidth })+ const ffmpeg = new FFmpegImage(getFFmpegCommandWrapperOptions('thumbnail'))+await ffmpeg.generateStoryboardFromVideo({destination,path: videoPath,@@ -95,78 +78,23 @@ async function processGenerateStoryboard (job: Job): Promise<void> {}})- const imageSize = await getImageSizeFromWorker(destination)-- await retryTransactionWrapper(() => {- return sequelizeTypescript.transaction(async transaction => {- const videoStillExists = await VideoModel.load(video.id, transaction)- if (!videoStillExists) {- logger.info(`Video ${payload.videoUUID} does not exist anymore, skipping storyboard generation.`, lTags)- deleteFileAndCatch(destination)- return- }-- const existing = await StoryboardModel.loadByVideo(video.id, transaction)- if (existing) await existing.destroy({ transaction })-- await StoryboardModel.create({- filename,- totalHeight: imageSize.height,- totalWidth: imageSize.width,- spriteHeight,- spriteWidth,- spriteDuration,- videoId: video.id- }, { transaction })-- logger.info(`Storyboard generation ${destination} ended for video ${video.uuid}.`, lTags)-- if (payload.federate) {- await federateVideoIfNeeded(video, false, transaction)- }- })- })- })- } finally {- inputFileMutexReleaser()- }-}--// -----------------------------------------------------------------------------export {- processGenerateStoryboard-}--function buildSpritesMetadata (options: {- video: MVideo-}) {- const { video } = options-- if (video.duration < 3) return { spriteDuration: undefined, totalSprites: 0 }-- const maxSprites = Math.min(Math.ceil(video.duration), STORYBOARD.SPRITES_MAX_EDGE_COUNT * STORYBOARD.SPRITES_MAX_EDGE_COUNT)-- const spriteDuration = Math.ceil(video.duration / maxSprites)- const totalSprites = Math.ceil(video.duration / spriteDuration)+ await insertStoryboardInDatabase({+ videoUUID: video.uuid,+ lTags,- // We can generate a single line so we don't need a prime number- if (totalSprites <= STORYBOARD.SPRITES_MAX_EDGE_COUNT) return { spriteDuration, totalSprites }+ filename,+ destination,- return { spriteDuration, totalSprites }-}+ imageSize: await getImageSizeFromWorker(destination),-function findGridSize (options: {- toFind: number- maxEdgeCount: number-}) {- const { toFind, maxEdgeCount } = options+ spriteHeight,+ spriteWidth,+ spriteDuration,- for (let i = 1; i <= maxEdgeCount; i++) {- for (let j = i; j <= maxEdgeCount; j++) {- if (toFind <= i * j) return { width: j, height: i }- }+ federate: payload.federate+ })+ })+ } finally {+ inputFileMutexReleaser()}-- throw new Error(`Could not find grid size (to find: ${toFind}, max edge count: ${maxEdgeCount}`)}diff --git a/server/core/lib/job-queue/handlers/video-import.ts b/server/core/lib/job-queue/handlers/video-import.tsindex df08043f26..a5c8fe05b4 100644--- a/server/core/lib/job-queue/handlers/video-import.ts+++ b/server/core/lib/job-queue/handlers/video-import.ts@@ -1,8 +1,5 @@import { buildAspectRatio } from '@peertube/peertube-core-utils'-import {- ffprobePromise,- getChaptersFromContainer, getVideoStreamDuration-} from '@peertube/peertube-ffmpeg'+import { ffprobePromise, getChaptersFromContainer, getVideoStreamDuration } from '@peertube/peertube-ffmpeg'import {ThumbnailType,ThumbnailType_Type,@@ -12,7 +9,8 @@ import {VideoImportTorrentPayload,VideoImportTorrentPayloadType,VideoImportYoutubeDLPayload,- VideoImportYoutubeDLPayloadType, VideoState+ VideoImportYoutubeDLPayloadType,+ VideoState} from '@peertube/peertube-models'import { retryTransactionWrapper } from '@server/helpers/database-utils.js'import { YoutubeDLWrapper } from '@server/helpers/youtube-dl/index.js'@@ -27,7 +25,7 @@ import { isUserQuotaValid } from '@server/lib/user.js'import { createTranscriptionTaskIfNeeded } from '@server/lib/video-captions.js'import { replaceChaptersIfNotExist } from '@server/lib/video-chapters.js'import { buildNewFile } from '@server/lib/video-file.js'-import { buildMoveVideoJob, buildStoryboardJobIfNeeded } from '@server/lib/video-jobs.js'+import { addLocalOrRemoteStoryboardJobIfNeeded, buildMoveVideoJob } from '@server/lib/video-jobs.js'import { VideoPathManager } from '@server/lib/video-path-manager.js'import { buildNextVideoState } from '@server/lib/video-state.js'import { VideoCaptionModel } from '@server/models/video/video-caption.js'@@ -305,7 +303,7 @@ async function afterImportSuccess (options: {}// Generate the storyboard in the job queue, and don't forget to federate an update after- await JobQueue.Instance.createJob(buildStoryboardJobIfNeeded({ video, federate: true }))+ await addLocalOrRemoteStoryboardJobIfNeeded({ video, federate: true })if (await VideoCaptionModel.hasVideoCaption(video.id) !== true && generateTranscription === true) {await createTranscriptionTaskIfNeeded(video)diff --git a/server/core/lib/job-queue/handlers/video-live-ending.ts b/server/core/lib/job-queue/handlers/video-live-ending.tsindex 3f899db3eb..678884b928 100644--- a/server/core/lib/job-queue/handlers/video-live-ending.ts+++ b/server/core/lib/job-queue/handlers/video-live-ending.ts@@ -14,7 +14,7 @@ import {import { generateLocalVideoMiniature, regenerateMiniaturesIfNeeded, updateLocalVideoMiniatureFromExisting } from '@server/lib/thumbnail.js'import { generateHlsPlaylistResolutionFromTS } from '@server/lib/transcoding/hls-transcoding.js'import { createTranscriptionTaskIfNeeded } from '@server/lib/video-captions.js'-import { buildStoryboardJobIfNeeded } from '@server/lib/video-jobs.js'+import { addLocalOrRemoteStoryboardJobIfNeeded } from '@server/lib/video-jobs.js'import { VideoPathManager } from '@server/lib/video-path-manager.js'import { isVideoInPublicDirectory } from '@server/lib/video-privacy.js'import { moveToNextState } from '@server/lib/video-state.js'@@ -41,7 +41,6 @@ import { pathExists, remove } from 'fs-extra/esm'import { readdir } from 'fs/promises'import { isAbsolute, join } from 'path'import { logger, loggerTagsFactory } from '../../../helpers/logger.js'-import { JobQueue } from '../job-queue.js'const lTags = loggerTagsFactory('live', 'job')@@ -362,7 +361,7 @@ async function cleanupLiveAndFederate (options: {}function createStoryboardJob (video: MVideo) {- return JobQueue.Instance.createJob(buildStoryboardJobIfNeeded({ video, federate: true }))+ return addLocalOrRemoteStoryboardJobIfNeeded({ video, federate: true })}async function hasReplayFiles (replayDirectory: string) {diff --git a/server/core/lib/runners/job-handlers/abstract-job-handler.ts b/server/core/lib/runners/job-handlers/abstract-job-handler.tsindex 35069bf34e..a5e1457e2a 100644--- a/server/core/lib/runners/job-handlers/abstract-job-handler.ts+++ b/server/core/lib/runners/job-handlers/abstract-job-handler.ts@@ -1,5 +1,7 @@import { pick } from '@peertube/peertube-core-utils'import {+ RunnerJobGenerateStoryboardPayload,+ RunnerJobGenerateStoryboardPrivatePayload,RunnerJobLiveRTMPHLSTranscodingPayload,RunnerJobLiveRTMPHLSTranscodingPrivatePayload,RunnerJobState,@@ -28,50 +30,56 @@ import { setAsUpdated } from '@server/models/shared/update.js'import { MRunnerJob } from '@server/types/models/runners/index.js'type CreateRunnerJobArg =- {+ | {type: Extract<RunnerJobType, 'vod-web-video-transcoding'>payload: RunnerJobVODWebVideoTranscodingPayloadprivatePayload: RunnerJobVODWebVideoTranscodingPrivatePayload- } |- {+ }+ | {type: Extract<RunnerJobType, 'vod-hls-transcoding'>payload: RunnerJobVODHLSTranscodingPayloadprivatePayload: RunnerJobVODHLSTranscodingPrivatePayload- } |- {+ }+ | {type: Extract<RunnerJobType, 'vod-audio-merge-transcoding'>payload: RunnerJobVODAudioMergeTranscodingPayloadprivatePayload: RunnerJobVODAudioMergeTranscodingPrivatePayload- } |- {+ }+ | {type: Extract<RunnerJobType, 'live-rtmp-hls-transcoding'>payload: RunnerJobLiveRTMPHLSTranscodingPayloadprivatePayload: RunnerJobLiveRTMPHLSTranscodingPrivatePayload- } |- {+ }+ | {type: Extract<RunnerJobType, 'video-studio-transcoding'>payload: RunnerJobStudioTranscodingPayloadprivatePayload: RunnerJobVideoStudioTranscodingPrivatePayload- } |- {+ }+ | {+ type: Extract<RunnerJobType, 'generate-video-storyboard'>+ payload: RunnerJobGenerateStoryboardPayload+ privatePayload: RunnerJobGenerateStoryboardPrivatePayload+ }+ | {type: Extract<RunnerJobType, 'video-transcription'>payload: RunnerJobTranscriptionPayloadprivatePayload: RunnerJobTranscriptionPrivatePayload}-export abstract class AbstractJobHandler <C, U extends RunnerJobUpdatePayload, S extends RunnerJobSuccessPayload> {-+export abstract class AbstractJobHandler<C, U extends RunnerJobUpdatePayload, S extends RunnerJobSuccessPayload> {protected readonly lTags = loggerTagsFactory('runner')// ---------------------------------------------------------------------------abstract create (options: C): Promise<MRunnerJob>- protected async createRunnerJob (options: CreateRunnerJobArg & {- jobUUID: string- priority: number- dependsOnRunnerJob?: MRunnerJob- }): Promise<MRunnerJob> {+ protected async createRunnerJob (+ options: CreateRunnerJobArg & {+ jobUUID: string+ priority: number+ dependsOnRunnerJob?: MRunnerJob+ }+ ): Promise<MRunnerJob> {const { priority, dependsOnRunnerJob } = optionslogger.debug('Creating runner job', { options, dependsOnRunnerJob, ...this.lTags(options.type) })diff --git a/server/core/lib/runners/job-handlers/index.ts b/server/core/lib/runners/job-handlers/index.tsindex a9fb79160a..87496a0255 100644--- a/server/core/lib/runners/job-handlers/index.ts+++ b/server/core/lib/runners/job-handlers/index.ts@@ -6,3 +6,4 @@ export * from './video-studio-transcoding-job-handler.js'export * from './vod-audio-merge-transcoding-job-handler.js'export * from './vod-hls-transcoding-job-handler.js'export * from './vod-web-video-transcoding-job-handler.js'+export * from './video-storyboard-job-handler.js'diff --git a/server/core/lib/runners/job-handlers/runner-job-handlers.ts b/server/core/lib/runners/job-handlers/runner-job-handlers.tsindex 771bf9d6ac..b85aaec194 100644--- a/server/core/lib/runners/job-handlers/runner-job-handlers.ts+++ b/server/core/lib/runners/job-handlers/runner-job-handlers.ts@@ -7,6 +7,7 @@ import { VideoStudioTranscodingJobHandler } from './video-studio-transcoding-jobimport { VODAudioMergeTranscodingJobHandler } from './vod-audio-merge-transcoding-job-handler.js'import { VODHLSTranscodingJobHandler } from './vod-hls-transcoding-job-handler.js'import { VODWebVideoTranscodingJobHandler } from './vod-web-video-transcoding-job-handler.js'+import { VideoStoryboardJobHandler } from './video-storyboard-job-handler.js'const processors: Record<RunnerJobType, new() => AbstractJobHandler<unknown, RunnerJobUpdatePayload, RunnerJobSuccessPayload>> = {'vod-web-video-transcoding': VODWebVideoTranscodingJobHandler,@@ -14,7 +15,8 @@ const processors: Record<RunnerJobType, new() => AbstractJobHandler<unknown, Run'vod-audio-merge-transcoding': VODAudioMergeTranscodingJobHandler,'live-rtmp-hls-transcoding': LiveRTMPHLSTranscodingJobHandler,'video-studio-transcoding': VideoStudioTranscodingJobHandler,- 'video-transcription': TranscriptionJobHandler+ 'video-transcription': TranscriptionJobHandler,+ 'generate-video-storyboard': VideoStoryboardJobHandler}export function getRunnerJobHandlerClass (job: MRunnerJob) {diff --git a/server/core/lib/runners/job-handlers/transcription-job-handler.ts b/server/core/lib/runners/job-handlers/transcription-job-handler.tsindex cda557a7ba..cba7b5d48a 100644--- a/server/core/lib/runners/job-handlers/transcription-job-handler.ts+++ b/server/core/lib/runners/job-handlers/transcription-job-handler.ts@@ -72,7 +72,7 @@ export class TranscriptionJobHandler extends AbstractJobHandler<CreateOptions, RjobUUID,payload,privatePayload,- priority: JOB_PRIORITY.TRANSCODING+ priority: JOB_PRIORITY.TRANSCRIPTION})return jobdiff --git a/server/core/lib/runners/job-handlers/video-storyboard-job-handler.ts b/server/core/lib/runners/job-handlers/video-storyboard-job-handler.tsnew file mode 100644index 0000000000..1a4c8cbdba--- /dev/null+++ b/server/core/lib/runners/job-handlers/video-storyboard-job-handler.ts@@ -0,0 +1,118 @@+import {+ GenerateStoryboardSuccess,+ RunnerJobGenerateStoryboardPayload,+ RunnerJobGenerateStoryboardPrivatePayload,+ RunnerJobUpdatePayload,+ VideoFileStream+} from '@peertube/peertube-models'+import { buildUUID } from '@peertube/peertube-node-utils'+import { generateImageFilename } from '@server/helpers/image-utils.js'+import { logger, loggerTagsFactory } from '@server/helpers/logger.js'+import { CONFIG } from '@server/initializers/config.js'+import { JOB_PRIORITY, STORYBOARD } from '@server/initializers/constants.js'+import { VideoPathManager } from '@server/lib/video-path-manager.js'+import { getImageSizeFromWorker } from '@server/lib/worker/parent-process.js'+import { VideoModel } from '@server/models/video/video.js'+import { MRunnerJob } from '@server/types/models/runners/index.js'+import { move } from 'fs-extra/esm'+import { join } from 'path'+import { buildSpriteSize, buildTotalSprites, findGridSize, insertStoryboardInDatabase } from '../../storyboard.js'+import { generateRunnerTranscodingVideoInputFileUrl } from '../runner-urls.js'+import { AbstractJobHandler } from './abstract-job-handler.js'++const lTagsBase = loggerTagsFactory('storyboard', 'runners')++type CreateOptions = {+ videoUUID: string+}++export class VideoStoryboardJobHandler extends AbstractJobHandler<CreateOptions, RunnerJobUpdatePayload, GenerateStoryboardSuccess> {+ async create (options: CreateOptions) {+ const { videoUUID } = options+ const lTags = lTagsBase(videoUUID)++ const jobUUID = buildUUID()++ const video = await VideoModel.loadFull(videoUUID)+ const inputFile = video.getMaxQualityFile(VideoFileStream.VIDEO)++ return VideoPathManager.Instance.makeAvailableVideoFile(inputFile, async videoPath => {+ const { spriteHeight, spriteWidth } = await buildSpriteSize(videoPath)++ const { spriteDuration, totalSprites } = buildTotalSprites(video)+ if (totalSprites === 0) {+ logger.info(`Do not generate remote storyboard job of ${videoUUID} because the video is not long enough`, lTags)+ return+ }++ const spritesCount = findGridSize({ toFind: totalSprites, maxEdgeCount: STORYBOARD.SPRITES_MAX_EDGE_COUNT })++ const payload: RunnerJobGenerateStoryboardPayload = {+ input: {+ videoFileUrl: generateRunnerTranscodingVideoInputFileUrl(jobUUID, videoUUID)+ },+ sprites: {+ size: { height: spriteHeight, width: spriteWidth },+ count: spritesCount,+ duration: spriteDuration+ },+ output: {}+ }++ const privatePayload: RunnerJobGenerateStoryboardPrivatePayload = { videoUUID }++ const job = await this.createRunnerJob({+ type: 'generate-video-storyboard',+ jobUUID,+ payload,+ privatePayload,+ priority: JOB_PRIORITY.STORYBOARD+ })++ return job+ })+ }++ protected isAbortSupported () {+ return true+ }++ protected specificUpdate (_options: { runnerJob: MRunnerJob }) {}+ protected specificAbort (_options: { runnerJob: MRunnerJob }) {}++ // When runner returns the storyboard image, finish the server-side creation like local job would+ protected async specificComplete (options: { runnerJob: MRunnerJob, resultPayload: GenerateStoryboardSuccess }) {+ const { runnerJob, resultPayload } = options++ const video = await VideoModel.loadFull(runnerJob.privatePayload.videoUUID)+ if (!video) return++ const destinationFilename = generateImageFilename()+ const destinationPath = join(CONFIG.STORAGE.STORYBOARDS_DIR, destinationFilename)++ await move(resultPayload.storyboardFile as string, destinationPath)++ const { sprites } = runnerJob.payload as RunnerJobGenerateStoryboardPayload++ await insertStoryboardInDatabase({+ videoUUID: video.uuid,+ lTags: this.lTags(video.uuid, runnerJob.uuid),++ filename: destinationFilename,+ destination: destinationPath,++ imageSize: await getImageSizeFromWorker(destinationPath),++ spriteHeight: sprites.size.height,+ spriteWidth: sprites.size.width,+ spriteDuration: sprites.duration,++ federate: true+ })++ logger.info('Runner storyboard job %s for %s ended.', runnerJob.uuid, video.uuid, this.lTags(video.uuid, runnerJob.uuid))+ }++ protected specificError (_options: { runnerJob: MRunnerJob }) {}+ protected specificCancel (_options: { runnerJob: MRunnerJob }) {}+}diff --git a/server/core/lib/server-config-manager.ts b/server/core/lib/server-config-manager.tsindex 2d30359f95..1a809d7072 100644--- a/server/core/lib/server-config-manager.ts+++ b/server/core/lib/server-config-manager.ts@@ -384,7 +384,10 @@ class ServerConfigManager {},storyboards: {- enabled: CONFIG.STORYBOARDS.ENABLED+ enabled: CONFIG.STORYBOARDS.ENABLED,+ remoteRunners: {+ enabled: CONFIG.STORYBOARDS.REMOTE_RUNNERS.ENABLED+ }},webrtc: {diff --git a/server/core/lib/storyboard.ts b/server/core/lib/storyboard.tsnew file mode 100644index 0000000000..15276a8b4f--- /dev/null+++ b/server/core/lib/storyboard.ts@@ -0,0 +1,98 @@+import { ffprobePromise, getVideoStreamDimensionsInfo } from '@peertube/peertube-ffmpeg'+import { retryTransactionWrapper } from '@server/helpers/database-utils.js'+import { LoggerTags, logger } from '@server/helpers/logger.js'+import { deleteFileAndCatch } from '@server/helpers/utils.js'+import { STORYBOARD } from '@server/initializers/constants.js'+import { sequelizeTypescript } from '@server/initializers/database.js'+import { StoryboardModel } from '@server/models/video/storyboard.js'+import { VideoModel } from '@server/models/video/video.js'+import { MVideo } from '@server/types/models/index.js'+import { federateVideoIfNeeded } from './activitypub/videos/federate.js'++export async function buildSpriteSize (videoPath: string) {+ const probe = await ffprobePromise(videoPath)+ const videoStreamInfo = await getVideoStreamDimensionsInfo(videoPath, probe)++ if (videoStreamInfo.isPortraitMode) {+ return {+ spriteHeight: STORYBOARD.SPRITE_MAX_SIZE,+ spriteWidth: Math.round(STORYBOARD.SPRITE_MAX_SIZE * videoStreamInfo.ratio)+ }+ }++ return {+ spriteWidth: STORYBOARD.SPRITE_MAX_SIZE,+ spriteHeight: Math.round(STORYBOARD.SPRITE_MAX_SIZE / videoStreamInfo.ratio)+ }+}++export function buildTotalSprites (video: MVideo) {+ if (video.duration < 3) return { spriteDuration: undefined, totalSprites: 0 }++ const maxSprites = Math.min(Math.ceil(video.duration), STORYBOARD.SPRITES_MAX_EDGE_COUNT * STORYBOARD.SPRITES_MAX_EDGE_COUNT)++ const spriteDuration = Math.ceil(video.duration / maxSprites)+ const totalSprites = Math.ceil(video.duration / spriteDuration)++ // We can generate a single line so we don't need a prime number+ if (totalSprites <= STORYBOARD.SPRITES_MAX_EDGE_COUNT) return { spriteDuration, totalSprites }++ return { spriteDuration, totalSprites }+}++export function findGridSize (options: {+ toFind: number+ maxEdgeCount: number+}) {+ const { toFind, maxEdgeCount } = options++ for (let i = 1; i <= maxEdgeCount; i++) {+ for (let j = i; j <= maxEdgeCount; j++) {+ if (toFind <= i * j) return { width: j, height: i }+ }+ }++ throw new Error(`Could not find grid size (to find: ${toFind}, max edge count: ${maxEdgeCount}`)+}++export async function insertStoryboardInDatabase (options: {+ videoUUID: string+ lTags: LoggerTags+ filename: string+ destination: string+ imageSize: { width: number, height: number }+ spriteHeight: number+ spriteWidth: number+ spriteDuration: number+ federate: boolean+}) {+ const { videoUUID, lTags, imageSize, spriteHeight, spriteWidth, spriteDuration, destination, filename, federate } = options++ await retryTransactionWrapper(() => {+ return sequelizeTypescript.transaction(async transaction => {+ const video = await VideoModel.loadFull(videoUUID, transaction)+ if (!video) {+ logger.info(`Video ${videoUUID} does not exist anymore, skipping storyboard generation.`, lTags)+ deleteFileAndCatch(destination)+ return+ }++ const existing = await StoryboardModel.loadByVideo(video.id, transaction)+ if (existing) await existing.destroy({ transaction })++ await StoryboardModel.create({+ filename,+ totalHeight: imageSize.height,+ totalWidth: imageSize.width,+ spriteHeight,+ spriteWidth,+ spriteDuration,+ videoId: video.id+ }, { transaction })++ if (federate) {+ await federateVideoIfNeeded(video, false, transaction)+ }+ })+ })+}diff --git a/server/core/lib/transcoding/web-transcoding.ts b/server/core/lib/transcoding/web-transcoding.tsindex 1df6a2ba68..8d9a011262 100644--- a/server/core/lib/transcoding/web-transcoding.ts+++ b/server/core/lib/transcoding/web-transcoding.ts@@ -16,10 +16,9 @@ import { copyFile } from 'fs/promises'import { basename, join } from 'path'import { CONFIG } from '../../initializers/config.js'import { VideoFileModel } from '../../models/video/video-file.js'-import { JobQueue } from '../job-queue/index.js'import { generateWebVideoFilename } from '../paths.js'import { buildNewFile, saveNewOriginalFileIfNeeded } from '../video-file.js'-import { buildStoryboardJobIfNeeded } from '../video-jobs.js'+import { addLocalOrRemoteStoryboardJobIfNeeded } from '../video-jobs.js'import { VideoPathManager } from '../video-path-manager.js'import { buildFFmpegVOD } from './shared/index.js'import { buildOriginalFileResolution } from './transcoding-resolutions.js'@@ -229,7 +228,7 @@ export async function onWebVideoFileTranscoding (options: {video.VideoFiles = await video.$get('VideoFiles')if (wasAudioFile) {- await JobQueue.Instance.createJob(buildStoryboardJobIfNeeded({ video, federate: false }))+ await addLocalOrRemoteStoryboardJobIfNeeded({ video, federate: false })}return { video, videoFile }diff --git a/server/core/lib/video-jobs.ts b/server/core/lib/video-jobs.tsindex 3e2bcbe79a..e082f5be52 100644--- a/server/core/lib/video-jobs.ts+++ b/server/core/lib/video-jobs.ts@@ -3,6 +3,7 @@ import { CONFIG } from '@server/initializers/config.js'import { VideoJobInfoModel } from '@server/models/video/video-job-info.js'import { MVideo, MVideoFile, MVideoFullLight, MVideoUUID } from '@server/types/models/index.js'import { CreateJobArgument, CreateJobOptions, JobQueue } from './job-queue/job-queue.js'+import { VideoStoryboardJobHandler } from './runners/index.js'import { createTranscriptionTaskIfNeeded } from './video-captions.js'import { moveFilesIfPrivacyChanged } from './video-privacy.js'@@ -26,13 +27,13 @@ export async function buildMoveVideoJob (options: {}}-export function buildStoryboardJobIfNeeded (options: {+export function buildLocalStoryboardJobIfNeeded (options: {video: MVideofederate: boolean}) {const { video, federate } = options- if (CONFIG.STORYBOARDS.ENABLED) {+ if (CONFIG.STORYBOARDS.ENABLED && !CONFIG.STORYBOARDS.REMOTE_RUNNERS.ENABLED) {return {type: 'generate-video-storyboard' as 'generate-video-storyboard',payload: {@@ -55,6 +56,28 @@ export function buildStoryboardJobIfNeeded (options: {return undefined}+export function addRemoteStoryboardJobIfNeeded (video: MVideo) {+ if (CONFIG.STORYBOARDS.ENABLED !== true) return+ if (CONFIG.STORYBOARDS.REMOTE_RUNNERS.ENABLED !== true) return++ return new VideoStoryboardJobHandler().create({ videoUUID: video.uuid })+}++export async function addLocalOrRemoteStoryboardJobIfNeeded (options: {+ video: MVideo+ federate: boolean+}) {+ const { video, federate } = options++ if (CONFIG.STORYBOARDS.ENABLED !== true) return++ if (CONFIG.STORYBOARDS.REMOTE_RUNNERS.ENABLED === true) {+ await addRemoteStoryboardJobIfNeeded(video)+ } else {+ await JobQueue.Instance.createJob(buildLocalStoryboardJobIfNeeded({ video, federate }))+ }+}+export async function addVideoJobsAfterCreation (options: {video: MVideovideoFile: MVideoFile@@ -72,7 +95,7 @@ export async function addVideoJobsAfterCreation (options: {}},- buildStoryboardJobIfNeeded({ video, federate: false }),+ buildLocalStoryboardJobIfNeeded({ video, federate: false }),{type: 'notify',@@ -109,6 +132,8 @@ export async function addVideoJobsAfterCreation (options: {await JobQueue.Instance.createSequentialJobFlow(...jobs)+ await addRemoteStoryboardJobIfNeeded(video)+if (generateTranscription === true) {await createTranscriptionTaskIfNeeded(video)}diff --git a/server/core/lib/video-studio.ts b/server/core/lib/video-studio.tsindex 256fe86353..4dc713bc7c 100644--- a/server/core/lib/video-studio.ts+++ b/server/core/lib/video-studio.ts@@ -12,7 +12,7 @@ import { VideoStudioTranscodingJobHandler } from './runners/index.js'import { getTranscodingJobPriority } from './transcoding/transcoding-priority.js'import { regenerateTranscriptionTaskIfNeeded } from './video-captions.js'import { buildNewFile, removeHLSPlaylist, removeWebVideoFile } from './video-file.js'-import { buildStoryboardJobIfNeeded } from './video-jobs.js'+import { addRemoteStoryboardJobIfNeeded, buildLocalStoryboardJobIfNeeded } from './video-jobs.js'import { VideoPathManager } from './video-path-manager.js'const lTags = loggerTagsFactory('video-studio')@@ -110,7 +110,7 @@ export async function onVideoStudioEnded (options: {await video.save()await JobQueue.Instance.createSequentialJobFlow(- buildStoryboardJobIfNeeded({ video, federate: false }),+ buildLocalStoryboardJobIfNeeded({ video, federate: false }),{type: 'federate-video' as 'federate-video',payload: {@@ -129,6 +129,7 @@ export async function onVideoStudioEnded (options: {})+ await addRemoteStoryboardJobIfNeeded(video)await regenerateTranscriptionTaskIfNeeded(video)}diff --git a/server/scripts/create-generate-storyboard-job.ts b/server/scripts/create-generate-storyboard-job.tsindex c33b7e3312..c051e3732f 100644--- a/server/scripts/create-generate-storyboard-job.ts+++ b/server/scripts/create-generate-storyboard-job.ts@@ -1,10 +1,10 @@-import { program } from 'commander'import { toCompleteUUID } from '@server/helpers/custom-validators/misc.js'import { initDatabaseModels } from '@server/initializers/database.js'import { JobQueue } from '@server/lib/job-queue/index.js'+import { addLocalOrRemoteStoryboardJobIfNeeded } from '@server/lib/video-jobs.js'import { StoryboardModel } from '@server/models/video/storyboard.js'import { VideoModel } from '@server/models/video/video.js'-import { buildStoryboardJobIfNeeded } from '@server/lib/video-jobs.js'+import { program } from 'commander'program.description('Generate videos storyboard')@@ -61,7 +61,7 @@ async function run () {if (videoFull.isLive) continue- await JobQueue.Instance.createJob(buildStoryboardJobIfNeeded({ video: videoFull, federate: true }))+ await addLocalOrRemoteStoryboardJobIfNeeded({ video: videoFull, federate: true })console.log(`Created generate-storyboard job for ${videoFull.name}.`)}