Skip to content

ActivityPub-federated video streaming platform using P2P directly in your web browser

Optimize transcoding job queue

Chocobozzz0920f5fa822f
diff --git a/apps/peertube-runner/src/server/process/shared/process-vod.ts b/apps/peertube-runner/src/server/process/shared/process-vod.ts
index d792626b77..99c897ebaf 100644
--- a/apps/peertube-runner/src/server/process/shared/process-vod.ts
+++ b/apps/peertube-runner/src/server/process/shared/process-vod.ts
@@ -1,3 +1,4 @@
+import { canCopyForHLS } from '@peertube/peertube-ffmpeg'
import {
RunnerJobVODAudioMergeTranscodingPayload,
RunnerJobVODHLSTranscodingPayload,
@@ -18,13 +19,6 @@ import {
ProcessOptions,
scheduleTranscodingProgress
} from './common.js'
-import {
- canDoQuickAudioTranscode,
- canDoQuickVideoTranscode,
- ffprobePromise,
- getVideoStreamDimensionsInfo,
- getVideoStreamFPS
-} from '@peertube/peertube-ffmpeg'
export async function processWebVideoTranscoding (options: ProcessOptions<RunnerJobVODWebVideoTranscodingPayload>) {
const { server, job, runnerToken } = options
@@ -117,16 +111,11 @@ export async function processHLSTranscoding (options: ProcessOptions<RunnerJobVO
videoInputPath = await downloadInputFile({ url: payload.input.videoFileUrl, runnerToken, job })
separatedAudioInputPath = await downloadSeparatedAudioFileIfNeeded({ urls: payload.input.separatedAudioFileUrl, runnerToken, job })
- const inputProbe = await ffprobePromise(videoInputPath)
- const { resolution } = await getVideoStreamDimensionsInfo(videoInputPath, inputProbe)
- const fps = await getVideoStreamFPS(videoInputPath, inputProbe)
-
- // Copy codecs if the input file can be quick transcoded (appropriate bitrate, codecs, etc.)
- // And if the input resolution/fps are the same as the output resolution/fps
- const copyCodecs = await canDoQuickAudioTranscode(videoInputPath, inputProbe) &&
- await canDoQuickVideoTranscode(videoInputPath, fps) &&
- resolution === payload.output.resolution &&
- (!resolution || fps === payload.output.fps)
+ const copyCodecs = await canCopyForHLS({
+ fps: payload.output.fps,
+ resolution: payload.output.resolution,
+ path: videoInputPath
+ })
logger.info(`Downloaded input file ${payload.input.videoFileUrl} for job ${job.jobToken}. Running HLS transcoding.`)
diff --git a/client/src/app/+admin/config/pages/admin-config-general.component.html b/client/src/app/+admin/config/pages/admin-config-general.component.html
index 4602d7b477..b2e656ef92 100644
--- a/client/src/app/+admin/config/pages/admin-config-general.component.html
+++ b/client/src/app/+admin/config/pages/admin-config-general.component.html
@@ -287,7 +287,7 @@
<ng-container formGroupName="videos">
<div class="form-group">
<label i18n for="importConcurrency">Import jobs concurrency</label>
- <span i18n class="small muted ms-1">allows to import multiple videos in parallel. ⚠️ Requires a PeerTube restart</span>
+ <span i18n class="small muted ms-1">allows to import multiple videos in parallel</span>
<div class="number-with-unit">
<input type="number" id="importConcurrency" formControlName="concurrency" />
diff --git a/client/src/app/+admin/config/pages/admin-config-vod.component.html b/client/src/app/+admin/config/pages/admin-config-vod.component.html
index fe87e86102..375aa499ed 100644
--- a/client/src/app/+admin/config/pages/admin-config-vod.component.html
+++ b/client/src/app/+admin/config/pages/admin-config-vod.component.html
@@ -230,7 +230,7 @@
<div class="form-group" [ngClass]="getLocalTranscodingDisabledClass()">
<label i18n for="transcodingConcurrency">Transcoding jobs concurrency</label>
- <span class="small muted ms-1" i18n>allows to transcode multiple files in parallel. ⚠️ Requires a PeerTube restart</span>
+ <span class="small muted ms-1" i18n>allows to transcode multiple files in parallel</span>
<div class="number-with-unit">
<input type="number" id="transcodingConcurrency" formControlName="concurrency" />
diff --git a/client/src/app/+admin/system/jobs/jobs.component.ts b/client/src/app/+admin/system/jobs/jobs.component.ts
index c13777c0cd..15b023f70a 100644
--- a/client/src/app/+admin/system/jobs/jobs.component.ts
+++ b/client/src/app/+admin/system/jobs/jobs.component.ts
@@ -108,21 +108,32 @@ export class JobsComponent implements OnInit {
this.loadJobStateAndType()
}
- getJobStateClasses (state: JobStateClient) {
+ getJobStateClasses (state: JobStateClient): string[] {
switch (state) {
+ case 'all':
+ return []
+
case 'active':
return [ 'pt-badge', 'badge-blue' ]
+
case 'completed':
return [ 'pt-badge', 'badge-green' ]
+
case 'delayed':
+ case 'prioritized':
+ case 'paused':
return [ 'pt-badge', 'badge-brown' ]
+
case 'failed':
return [ 'pt-badge', 'badge-red' ]
+
case 'waiting':
+ case 'waiting-children':
return [ 'pt-badge', 'badge-yellow' ]
}
- return []
+ // Do not remove, to ensure all cases are handled by the switch
+ return state
}
onJobStateOrTypeChanged () {
diff --git a/packages/ffmpeg/src/ffmpeg-default-transcoding-profile.ts b/packages/ffmpeg/src/ffmpeg-default-transcoding-profile.ts
index 7e0743a2d5..1105d14f1a 100644
--- a/packages/ffmpeg/src/ffmpeg-default-transcoding-profile.ts
+++ b/packages/ffmpeg/src/ffmpeg-default-transcoding-profile.ts
@@ -1,6 +1,7 @@
import { getAverageTheoreticalBitrate, getMaxTheoreticalBitrate, getMinTheoreticalBitrate } from '@peertube/peertube-core-utils'
import {
buildStreamSuffix,
+ ffprobePromise,
getAudioStream,
getMaxAudioKBitrate,
getVideoStream,
@@ -147,6 +148,25 @@ export async function canDoQuickVideoTranscode (path: string, maxFPS: number, pr
return true
}
+// Copy codecs if the input file can be quick transcoded (appropriate bitrate, codecs, etc.)
+// And if the input resolution/fps are the same as the output resolution/fps
+export async function canCopyForHLS (options: {
+ path: string
+ fps: number
+ resolution: number
+}, probe?: FfprobeData): Promise<boolean> {
+ const { path, fps, resolution } = options
+
+ const inputProbe = probe ?? await ffprobePromise(path)
+ const { resolution: inputResolution } = await getVideoStreamDimensionsInfo(path, inputProbe)
+ const inputFPS = await getVideoStreamFPS(path, inputProbe)
+
+ return await canDoQuickAudioTranscode(path, probe) &&
+ await canDoQuickVideoTranscode(path, fps, probe) &&
+ resolution === inputResolution &&
+ (!inputResolution || fps === inputFPS)
+}
+
// ---------------------------------------------------------------------------
function getTargetBitrate (options: {
diff --git a/packages/models/src/runners/runner-jobs/runner-job-private-payload.model.ts b/packages/models/src/runners/runner-jobs/runner-job-private-payload.model.ts
index 4a2f679f30..a3612577dc 100644
--- a/packages/models/src/runners/runner-jobs/runner-job-private-payload.model.ts
+++ b/packages/models/src/runners/runner-jobs/runner-job-private-payload.model.ts
@@ -1,4 +1,5 @@
import { VideoStudioTaskPayload } from '../../server/index.js'
+import { VideoFileStreamType } from '../../videos/file/video-file-stream.enum.js'
export type RunnerJobVODPrivatePayload =
| RunnerJobVODWebVideoTranscodingPrivatePayload
@@ -18,18 +19,26 @@ export interface RunnerJobVODWebVideoTranscodingPrivatePayload {
videoUUID: string
isNewVideo: boolean
deleteInputFileId: number | null
+
+ canMoveVideoState: boolean
}
export interface RunnerJobVODAudioMergeTranscodingPrivatePayload {
videoUUID: string
isNewVideo: boolean
deleteInputFileId: number | null
+
+ canMoveVideoState: boolean
}
export interface RunnerJobVODHLSTranscodingPrivatePayload {
videoUUID: string
isNewVideo: boolean
deleteWebVideoFiles: boolean
+
+ canMoveVideoState: boolean
+ inputStreams: VideoFileStreamType[]
+ transcodingRequestAt: string
}
// ---------------------------------------------------------------------------
diff --git a/packages/models/src/server/job.model.ts b/packages/models/src/server/job.model.ts
index 95f8775f90..c94b09d563 100644
--- a/packages/models/src/server/job.model.ts
+++ b/packages/models/src/server/job.model.ts
@@ -1,5 +1,5 @@
import { ContextType } from '../activitypub/context.js'
-import { VideoStateType } from '../videos/index.js'
+import { VideoFileStreamType, VideoStateType } from '../videos/index.js'
import { VideoStudioTaskCut } from '../videos/studio/index.js'
import { SendEmailOptions } from './emailer.model.js'
@@ -144,7 +144,7 @@ export type ManageVideoTorrentPayload = {
interface BaseTranscodingPayload {
videoUUID: string
- hasChildren?: boolean
+ canMoveVideoState: boolean
isNewVideo?: boolean
}
@@ -152,11 +152,14 @@ export interface HLSTranscodingPayload extends BaseTranscodingPayload {
type: 'new-resolution-to-hls'
resolution: number
fps: number
- copyCodecs: boolean
separatedAudio: boolean
deleteWebVideoFiles: boolean
+
+ inputStreams: VideoFileStreamType[]
+
+ transcodingRequestAt: string
}
export interface NewWebVideoResolutionTranscodingPayload extends BaseTranscodingPayload {
@@ -174,8 +177,6 @@ export interface MergeAudioTranscodingPayload extends BaseTranscodingPayload {
export interface OptimizeTranscodingPayload extends BaseTranscodingPayload {
type: 'optimize-to-web-video'
-
- quickTranscode: boolean
}
export type VideoTranscodingPayload =
@@ -203,8 +204,15 @@ export type MoveStoragePayload = MoveVideoStoragePayload | MoveCaptionPayload
export interface MoveVideoStoragePayload {
videoUUID: string
- isNewVideo: boolean
- previousVideoState: VideoStateType
+
+ // FIXME: old API compatibility, remove in PeerTube v9
+ isNewVideo?: boolean
+ previousVideoState?: VideoStateType
+
+ moveVideoState?: {
+ isNewVideo: boolean
+ previousVideoState: VideoStateType
+ }
}
export interface MoveCaptionPayload {
diff --git a/packages/tests/src/api/object-storage/videos.ts b/packages/tests/src/api/object-storage/videos.ts
index c1ff9b8412..ef3a9170f1 100644
--- a/packages/tests/src/api/object-storage/videos.ts
+++ b/packages/tests/src/api/object-storage/videos.ts
@@ -15,7 +15,7 @@ import {
setAccessTokensToServers,
waitJobs
} from '@peertube/peertube-server-commands'
-import { expectLogDoesNotContain, expectStartWith } from '@tests/shared/checks.js'
+import { expectStartWith } from '@tests/shared/checks.js'
import { checkTmpIsEmpty } from '@tests/shared/directories.js'
import { generateHighBitrateVideo } from '@tests/shared/generate.js'
import { MockObjectStorageProxy } from '@tests/shared/mock-servers/mock-object-storage.js'
@@ -304,12 +304,6 @@ function runTestSuite (options: {
}
})
- it('Should not have downloaded files from object storage', async function () {
- for (const server of servers) {
- await expectLogDoesNotContain(server, 'from object storage')
- }
- })
-
after(async function () {
await mockObjectStorageProxy.terminate()
await objectStorage.cleanupMock()
diff --git a/packages/tests/src/api/runners/runner-vod-transcoding.ts b/packages/tests/src/api/runners/runner-vod-transcoding.ts
index 433e355a86..5673c7c2d8 100644
--- a/packages/tests/src/api/runners/runner-vod-transcoding.ts
+++ b/packages/tests/src/api/runners/runner-vod-transcoding.ts
@@ -64,7 +64,6 @@ describe('Test runner VOD transcoding', function () {
})
describe('Without transcoding', function () {
-
before(async function () {
this.timeout(60000)
@@ -81,7 +80,6 @@ describe('Test runner VOD transcoding', function () {
})
describe('With classic transcoding enabled', function () {
-
before(async function () {
this.timeout(60000)
@@ -176,15 +174,15 @@ describe('Test runner VOD transcoding', function () {
await waitJobs(servers)
})
- it('Should have the video updated', async function () {
- for (const server of servers) {
- const video = await server.videos.get({ id: videoUUID })
- expect(video.files).to.have.lengthOf(1)
- expect(video.streamingPlaylists).to.have.lengthOf(0)
+ it('Should have the video updated but not federated', async function () {
+ const video = await servers[0].videos.get({ id: videoUUID })
+ expect(video.files).to.have.lengthOf(1)
+ expect(video.streamingPlaylists).to.have.lengthOf(0)
- const { body } = await makeRawRequest({ url: video.files[0].fileUrl, expectedStatus: HttpStatusCode.OK_200 })
- expect(body).to.deep.equal(await readFile(buildAbsoluteFixturePath('video_short.mp4')))
- }
+ const { body } = await makeRawRequest({ url: video.files[0].fileUrl, expectedStatus: HttpStatusCode.OK_200 })
+ expect(body).to.deep.equal(await readFile(buildAbsoluteFixturePath('video_short.mp4')))
+
+ await servers[1].videos.get({ id: videoUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
})
it('Should have 4 lower resolution to transcode', async function () {
@@ -213,6 +211,16 @@ describe('Test runner VOD transcoding', function () {
await servers[0].runnerJobs.success({ runnerToken, jobUUID, jobToken, payload })
})
+ it('Should have federated the video', async function () {
+ await waitJobs(servers)
+
+ for (const server of servers) {
+ const video = await server.videos.get({ id: videoUUID })
+ expect(video.files).to.have.lengthOf(2)
+ expect(video.streamingPlaylists).to.have.lengthOf(0)
+ }
+ })
+
it('Should process all other jobs', async function () {
const { availableJobs } = await servers[0].runnerJobs.requestVOD({ runnerToken })
expect(availableJobs).to.have.lengthOf(3)
@@ -382,7 +390,6 @@ describe('Test runner VOD transcoding', function () {
})
describe('Web video and HLS transcoding', function () {
-
before(async function () {
this.timeout(60000)
@@ -482,15 +489,14 @@ describe('Test runner VOD transcoding', function () {
await waitJobs(servers)
})
- it('Should have the video updated', async function () {
- for (const server of servers) {
- const video = await server.videos.get({ id: videoUUID })
- expect(video.files).to.have.lengthOf(1)
- expect(video.streamingPlaylists).to.have.lengthOf(0)
+ it('Should have the video updated but not federated', async function () {
+ const video = await servers[0].videos.get({ id: videoUUID })
+ expect(video.files).to.have.lengthOf(1)
+ expect(video.streamingPlaylists).to.have.lengthOf(0)
+ const { body } = await makeRawRequest({ url: video.files[0].fileUrl, expectedStatus: HttpStatusCode.OK_200 })
+ expect(body).to.deep.equal(await readFile(buildAbsoluteFixturePath('video_short_480p.mp4')))
- const { body } = await makeRawRequest({ url: video.files[0].fileUrl, expectedStatus: HttpStatusCode.OK_200 })
- expect(body).to.deep.equal(await readFile(buildAbsoluteFixturePath('video_short_480p.mp4')))
- }
+ await servers[1].videos.get({ id: videoUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
})
it('Should have 4 lower resolutions to transcode', async function () {
@@ -524,7 +530,7 @@ describe('Test runner VOD transcoding', function () {
await waitJobs(servers)
})
- it('Should have the video updated', async function () {
+ it('Should have the video updated and federated', async function () {
for (const server of servers) {
const video = await server.videos.get({ id: videoUUID })
diff --git a/packages/tests/src/api/server/jobs.ts b/packages/tests/src/api/server/jobs.ts
index 8f71759b23..e975fbac70 100644
--- a/packages/tests/src/api/server/jobs.ts
+++ b/packages/tests/src/api/server/jobs.ts
@@ -99,15 +99,13 @@ describe('Test jobs', function () {
await wait(5000)
- {
- const body = await servers[1].jobs.list({ state: 'waiting', jobType: 'video-transcoding' })
- // root transcoding
- expect(body.data).to.have.lengthOf(1)
+ for (const state of [ 'waiting', 'waiting-children' ] as const) {
+ const body = await servers[1].jobs.list({ state, jobType: 'video-transcoding' })
+ expect(body.data).to.have.lengthOf(0)
}
{
- const body = await servers[1].jobs.list({ state: 'waiting-children', jobType: 'transcoding-job-builder' })
- // next transcoding jobs
+ const body = await servers[1].jobs.list({ state: 'waiting', jobType: 'transcoding-job-builder' })
expect(body.data).to.have.lengthOf(1)
}
})
@@ -119,8 +117,17 @@ describe('Test jobs', function () {
await waitJobs(servers)
- const body = await servers[1].jobs.list({ state: 'waiting', jobType: 'video-transcoding' })
- expect(body.data).to.have.lengthOf(0)
+ for (const state of [ 'waiting', 'waiting-children' ] as const) {
+ {
+ const body = await servers[1].jobs.list({ state, jobType: 'video-transcoding' })
+ expect(body.data).to.have.lengthOf(0)
+ }
+
+ {
+ const body = await servers[1].jobs.list({ state, jobType: 'transcoding-job-builder' })
+ expect(body.data).to.have.lengthOf(0)
+ }
+ }
})
after(async function () {
diff --git a/packages/tests/src/api/transcoding/create-transcoding.ts b/packages/tests/src/api/transcoding/create-transcoding.ts
index ffc39a5ced..aafa3b21c8 100644
--- a/packages/tests/src/api/transcoding/create-transcoding.ts
+++ b/packages/tests/src/api/transcoding/create-transcoding.ts
@@ -82,7 +82,6 @@ function runTests (options: {
})
describe('Common transcoding', function () {
-
it('Should generate HLS', async function () {
this.timeout(60000)
@@ -306,7 +305,6 @@ function runTests (options: {
})
describe('With split audio and video', function () {
-
async function runTest (options: {
audio: boolean
hls: boolean
@@ -408,10 +406,8 @@ function runTests (options: {
}
describe('Test create transcoding jobs from API', function () {
-
for (const concurrency of [ 1, 2 ]) {
describe('With concurrency ' + concurrency, function () {
-
describe('On filesystem', function () {
runTests({ concurrency, enableObjectStorage: false })
})
diff --git a/packages/tests/src/api/transcoding/split-audio-and-video.ts b/packages/tests/src/api/transcoding/split-audio-and-video.ts
index b9b9ba82dc..9d6ecdc040 100644
--- a/packages/tests/src/api/transcoding/split-audio-and-video.ts
+++ b/packages/tests/src/api/transcoding/split-audio-and-video.ts
@@ -52,29 +52,20 @@ describe('Test HLS with audio and video splitted', function () {
await completeCheckHlsPlaylist({ servers, videoUUID: uuid, hlsOnly, splittedAudio: true, objectStorageBaseUrl })
})
- if (concurrency === 1) {
- it('Should have processed audio just after first video resolution', async function () {
- const { data } = await servers[0].jobs.list({ jobType: 'video-transcoding' })
+ it('Should have processed audio just after first video resolution', async function () {
+ const { data } = await servers[0].jobs.list({ jobType: 'video-transcoding' })
- const hlsJobs = data.filter(job => {
- const data = job.data as HLSTranscodingPayload
+ const hlsJobs = data.filter(job => {
+ const data = job.data as HLSTranscodingPayload
- return data.videoUUID === videoUUIDs[0] && data.type === 'new-resolution-to-hls'
- })
-
- const dataForVideo = sortBy(hlsJobs, 'processedOn')
+ return data.videoUUID === videoUUIDs[0] && data.type === 'new-resolution-to-hls'
+ })
- expect((dataForVideo[0].data as HLSTranscodingPayload).resolution).to.equal(720)
+ const dataForVideo = sortBy(hlsJobs, 'processedOn')
- // FIXME: next job after parent succeeded should be audio but bullmq seems to fire another one when it updates the child state
- // It's the reason why another video stream transcoding is executed before the audio
- try {
- expect((dataForVideo[1].data as HLSTranscodingPayload).resolution).to.equal(0)
- } catch (err) {
- expect((dataForVideo[2].data as HLSTranscodingPayload).resolution).to.equal(0)
- }
- })
- }
+ expect((dataForVideo[0].data as HLSTranscodingPayload).resolution).to.equal(720)
+ expect((dataForVideo[1].data as HLSTranscodingPayload).resolution).to.equal(0)
+ })
it('Should upload an audio file and transcode it to HLS', async function () {
this.timeout(120000)
@@ -169,7 +160,6 @@ describe('Test HLS with audio and video splitted', function () {
for (const concurrency of [ 1, 2 ]) {
describe(`With concurrency ${concurrency}`, function () {
-
describe('With Web Video & HLS enabled', function () {
runTestSuite({ hlsOnly: false, concurrency })
})
diff --git a/packages/tests/src/api/transcoding/transcoder.ts b/packages/tests/src/api/transcoding/transcoder.ts
index e1f9e05e96..9d00e5006e 100644
--- a/packages/tests/src/api/transcoding/transcoder.ts
+++ b/packages/tests/src/api/transcoding/transcoder.ts
@@ -1,11 +1,7 @@
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
import { getAllFiles, omit } from '@peertube/peertube-core-utils'
-import {
- ffprobePromise,
- getAudioStream,
- hasAudioStream
-} from '@peertube/peertube-ffmpeg'
+import { ffprobePromise, getAudioStream, hasAudioStream } from '@peertube/peertube-ffmpeg'
import { HttpStatusCode, VideoFileMetadata, VideoState } from '@peertube/peertube-models'
import { buildAbsoluteFixturePath } from '@peertube/peertube-node-utils'
import {
@@ -45,7 +41,6 @@ describe('Test video transcoding', function () {
})
describe('Common transcoding', function () {
-
it('Should not transcode video on server 1', async function () {
this.timeout(60_000)
@@ -209,7 +204,6 @@ describe('Test video transcoding', function () {
})
describe('Audio transcoding', function () {
-
it('Should transcode high bit rate mp3 to proper bit rate', async function () {
this.timeout(120_000)
@@ -304,9 +298,7 @@ describe('Test video transcoding', function () {
})
describe('Audio upload', function () {
-
function runSuite (mode: 'legacy' | 'resumable') {
-
before(async function () {
await servers[1].config.enableTranscoding({ hls: true, webVideo: true, resolutions: [] })
})
@@ -405,7 +397,6 @@ describe('Test video transcoding', function () {
})
describe('FFprobe', function () {
-
it('Should provide valid ffprobe data', async function () {
this.timeout(160_000)
@@ -423,25 +414,29 @@ describe('Test video transcoding', function () {
const metadata = new VideoFileMetadata(probe)
// expected format properties
- for (const p of [
- 'tags.encoder',
- 'format_long_name',
- 'size',
- 'bit_rate'
- ]) {
+ for (
+ const p of [
+ 'tags.encoder',
+ 'format_long_name',
+ 'size',
+ 'bit_rate'
+ ]
+ ) {
expect(metadata.format).to.have.nested.property(p)
}
// expected stream properties
- for (const p of [
- 'codec_long_name',
- 'profile',
- 'width',
- 'height',
- 'display_aspect_ratio',
- 'avg_frame_rate',
- 'pix_fmt'
- ]) {
+ for (
+ const p of [
+ 'codec_long_name',
+ 'profile',
+ 'width',
+ 'height',
+ 'display_aspect_ratio',
+ 'avg_frame_rate',
+ 'pix_fmt'
+ ]
+ ) {
expect(metadata.streams[0]).to.have.nested.property(p)
}
@@ -475,7 +470,6 @@ describe('Test video transcoding', function () {
})
describe('Transcoding job queue', function () {
-
it('Should have the appropriate priorities for transcoding jobs', async function () {
const body = await servers[1].jobs.list({
start: 0,
@@ -497,9 +491,14 @@ describe('Test video transcoding', function () {
expect(webVideoJobs).to.have.lengthOf(7)
expect(optimizeJobs).to.have.lengthOf(1)
- for (const j of optimizeJobs.concat(hlsJobs.concat(webVideoJobs))) {
- expect(j.priority).to.be.greaterThan(100)
- expect(j.priority).to.be.lessThan(150)
+ expect(optimizeJobs[0].priority).to.be.greaterThan(100).and.lessThan(150)
+
+ for (const job of [ ...webVideoJobs, ...hlsJobs ]) {
+ if (job.data.resolution === 2160) {
+ expect(job.priority).to.be.greaterThan(100).and.lessThan(150)
+ } else {
+ expect(job.priority).to.be.greaterThan(10000).and.lessThan(10050)
+ }
}
})
})
diff --git a/packages/tests/src/api/videos/index.ts b/packages/tests/src/api/videos/index.ts
index 87158f0de8..bbb5157ec8 100644
--- a/packages/tests/src/api/videos/index.ts
+++ b/packages/tests/src/api/videos/index.ts
@@ -20,6 +20,7 @@ import './video-playlists.js'
import './video-privacy.js'
import './video-schedule-update.js'
import './video-source.js'
+import './video-state.js'
import './video-static-file-privacy.js'
import './video-storyboard.js'
import './video-transcription.js'
diff --git a/packages/tests/src/api/videos/video-state.ts b/packages/tests/src/api/videos/video-state.ts
new file mode 100644
index 0000000000..e89724c8dc
--- /dev/null
+++ b/packages/tests/src/api/videos/video-state.ts
@@ -0,0 +1,321 @@
+/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
+import { sortBy } from '@peertube/peertube-core-utils'
+import { MoveVideoStoragePayload, VideoState, VideoTranscodingPayload } from '@peertube/peertube-models'
+import { areMockObjectStorageTestsDisabled } from '@peertube/peertube-node-utils'
+import {
+ ObjectStorageCommand,
+ PeerTubeServer,
+ cleanupTests,
+ createMultipleServers,
+ doubleFollow,
+ setAccessTokensToServers,
+ waitJobs
+} from '@peertube/peertube-server-commands'
+import { expect } from 'chai'
+
+describe('Test video state on local job queue', function () {
+ let servers: PeerTubeServer[] = []
+ const resolutions = [ 720, 240, 144 ]
+
+ async function prepareVideo () {
+ const { uuid } = await servers[0].videos.quickUpload({ name: 'video', fixture: 'video_short.mp4' })
+ await waitJobs(servers)
+
+ return uuid
+ }
+
+ before(async function () {
+ this.timeout(120000)
+
+ servers = await createMultipleServers(2)
+
+ await setAccessTokensToServers(servers)
+ await doubleFollow(servers[0], servers[1])
+
+ await servers[0].config.setTranscodingConcurrency(1)
+ })
+
+ describe('Without object storage', function () {
+ async function checkPublication (options: {
+ uuid: string
+ publishedJobFinder: (data: VideoTranscodingPayload) => boolean
+ }) {
+ const { uuid, publishedJobFinder } = options
+
+ const { data } = await servers[0].jobs.list({ jobType: 'video-transcoding' })
+ const publishedJob = data.find(j => {
+ const data = j.data as VideoTranscodingPayload
+
+ return data.videoUUID === uuid && publishedJobFinder(data)
+ })
+
+ for (const server of servers) {
+ const video = await server.videos.get({ id: uuid })
+ expect(video.state.id).to.equal(VideoState.PUBLISHED)
+
+ expect(publishedJob.processedOn).to.exist
+ expect(publishedJob.finishedOn).to.exist
+
+ expect(new Date(video.publishedAt)).to.be.greaterThan(new Date(publishedJob.processedOn))
+ expect(new Date(video.publishedAt)).to.be.below(new Date(publishedJob.finishedOn))
+ }
+ }
+
+ async function checkJobOrder (options: {
+ uuid: string
+ order: { type: VideoTranscodingPayload['type'], resolution?: number }[]
+ }) {
+ const { uuid, order } = options
+
+ const { data } = await servers[0].jobs.list({ jobType: 'video-transcoding' })
+
+ const jobs = sortBy(data.filter(j => (j.data as VideoTranscodingPayload).videoUUID === uuid), 'processedOn')
+ .map(j => {
+ const data = j.data as VideoTranscodingPayload
+
+ if (data.type === 'optimize-to-web-video') return { type: data.type }
+
+ return { type: data.type, resolution: data.resolution }
+ })
+
+ expect(jobs).to.deep.equal(order)
+ }
+
+ it('Should directly publish a video after upload without transcoding', async function () {
+ await servers[0].config.disableTranscoding()
+
+ const { uuid } = await servers[0].videos.quickUpload({ name: 'video' })
+ await waitJobs(servers)
+
+ for (const server of servers) {
+ const video = await server.videos.get({ id: uuid })
+ expect(video.state.id).to.equal(VideoState.PUBLISHED)
+ }
+ })
+
+ it('Should publish after web video transcoding only, when the optimize job ends', async function () {
+ await servers[0].config.enableTranscoding({ hls: false, webVideo: true, resolutions: [] })
+ const uuid = await prepareVideo()
+
+ await checkPublication({
+ uuid,
+ publishedJobFinder: data => data.type === 'optimize-to-web-video'
+ })
+
+ await checkJobOrder({
+ uuid,
+ order: [ { type: 'optimize-to-web-video' } ]
+ })
+ })
+
+ it('Should publish after web video transcoding only, when the first child resolution job ends', async function () {
+ await servers[0].config.enableTranscoding({ hls: false, webVideo: true, resolutions })
+ const uuid = await prepareVideo()
+
+ await checkPublication({
+ uuid,
+ publishedJobFinder: data => data.type === 'new-resolution-to-web-video' && data.resolution === 240
+ })
+
+ await checkJobOrder({
+ uuid,
+ order: [
+ { type: 'optimize-to-web-video' },
+ { type: 'new-resolution-to-web-video', resolution: 240 },
+ { type: 'new-resolution-to-web-video', resolution: 144 }
+ ]
+ })
+ })
+
+ it('Should publish after web & hls transcodings, when the max HLS resolution ends', async function () {
+ await servers[0].config.enableTranscoding({ hls: true, webVideo: true, resolutions })
+ const uuid = await prepareVideo()
+
+ await checkPublication({
+ uuid,
+ publishedJobFinder: data => data.type === 'new-resolution-to-hls' && data.resolution === 720
+ })
+
+ await checkJobOrder({
+ uuid,
+ order: [
+ { type: 'optimize-to-web-video' },
+ { type: 'new-resolution-to-hls', resolution: 720 },
+ { type: 'new-resolution-to-web-video', resolution: 240 },
+ { type: 'new-resolution-to-web-video', resolution: 144 },
+ { type: 'new-resolution-to-hls', resolution: 240 },
+ { type: 'new-resolution-to-hls', resolution: 144 }
+ ]
+ })
+ })
+
+ it('Should publish after HLS transcoding only, when the first child resolution job ends', async function () {
+ await servers[0].config.enableTranscoding({ hls: true, webVideo: false, resolutions: [] })
+ const uuid = await prepareVideo()
+
+ await checkPublication({
+ uuid,
+ publishedJobFinder: data => data.type === 'new-resolution-to-hls' && data.resolution === 720
+ })
+
+ await checkJobOrder({
+ uuid,
+ order: [
+ { type: 'optimize-to-web-video' },
+ { type: 'new-resolution-to-hls', resolution: 720 }
+ ]
+ })
+ })
+
+ it('Should publish after hls transcoding only, when the max HLS resolution ends', async function () {
+ await servers[0].config.enableTranscoding({ hls: true, webVideo: false, resolutions })
+ const uuid = await prepareVideo()
+
+ await checkPublication({
+ uuid,
+ publishedJobFinder: data => data.type === 'new-resolution-to-hls' && data.resolution === 720
+ })
+
+ await checkJobOrder({
+ uuid,
+ order: [
+ { type: 'optimize-to-web-video' },
+ { type: 'new-resolution-to-hls', resolution: 720 },
+ { type: 'new-resolution-to-hls', resolution: 240 },
+ { type: 'new-resolution-to-hls', resolution: 144 }
+ ]
+ })
+ })
+
+ it('Should publish after hls transcoding only with audio and video splitted, when audio & video streams are ready', async function () {
+ await servers[0].config.enableTranscoding({ hls: true, webVideo: false, resolutions, splitAudioAndVideo: true })
+ const uuid = await prepareVideo()
+
+ await checkPublication({
+ uuid,
+ publishedJobFinder: data => data.type === 'new-resolution-to-hls' && data.resolution === 0
+ })
+
+ await checkJobOrder({
+ uuid,
+ order: [
+ { type: 'optimize-to-web-video' },
+ { type: 'new-resolution-to-hls', resolution: 720 },
+ { type: 'new-resolution-to-hls', resolution: 0 },
+ { type: 'new-resolution-to-hls', resolution: 240 },
+ { type: 'new-resolution-to-hls', resolution: 144 }
+ ]
+ })
+ })
+ })
+
+ describe('With object storage', function () {
+ if (areMockObjectStorageTestsDisabled()) return
+
+ const objectStorage = new ObjectStorageCommand()
+
+ async function checkPublication (options: {
+ uuid: string
+ publishedJobFinder: (data: VideoTranscodingPayload) => boolean
+ }) {
+ const { uuid, publishedJobFinder } = options
+
+ const { data: transcodingJobs } = await servers[0].jobs.list({ jobType: 'video-transcoding' })
+ const transcodingJob = transcodingJobs.find(j => {
+ const data = j.data as VideoTranscodingPayload
+
+ return data.videoUUID === uuid && publishedJobFinder(data)
+ })
+
+ // Video is published in the next move to object storage job
+ const { data: objectStorageJobs } = await servers[0].jobs.list({ jobType: 'move-to-object-storage', sort: 'createdAt', count: 100 })
+ const publishedJob = objectStorageJobs.find(j => {
+ const data = j.data as MoveVideoStoragePayload
+
+ return data.videoUUID === uuid && new Date(j.processedOn) > new Date(transcodingJob.processedOn)
+ })
+
+ for (const server of servers) {
+ const video = await server.videos.get({ id: uuid })
+ expect(video.state.id).to.equal(VideoState.PUBLISHED)
+
+ expect(publishedJob.processedOn).to.exist
+ expect(publishedJob.finishedOn).to.exist
+
+ expect(new Date(video.publishedAt)).to.be.greaterThan(new Date(publishedJob.processedOn))
+ expect(new Date(video.publishedAt)).to.be.below(new Date(publishedJob.finishedOn))
+ }
+ }
+
+ before(async function () {
+ await objectStorage.prepareDefaultMockBuckets()
+
+ await servers[0].kill()
+ await servers[0].run(objectStorage.getDefaultMockConfig())
+ })
+
+ it('Should publish after web video transcoding only, when the optimize job ends', async function () {
+ await servers[0].config.enableTranscoding({ hls: false, webVideo: true, resolutions: [] })
+ const uuid = await prepareVideo()
+
+ await checkPublication({
+ uuid,
+ publishedJobFinder: data => data.type === 'optimize-to-web-video'
+ })
+ })
+
+ it('Should publish after web video transcoding only, when the first child resolution job ends', async function () {
+ await servers[0].config.enableTranscoding({ hls: false, webVideo: true, resolutions })
+ const uuid = await prepareVideo()
+
+ await checkPublication({
+ uuid,
+ publishedJobFinder: data => data.type === 'new-resolution-to-web-video' && data.resolution === 240
+ })
+ })
+
+ it('Should publish after web & hls transcodings, when the max HLS resolution ends', async function () {
+ await servers[0].config.enableTranscoding({ hls: true, webVideo: true, resolutions })
+ const uuid = await prepareVideo()
+
+ await checkPublication({
+ uuid,
+ publishedJobFinder: data => data.type === 'new-resolution-to-hls' && data.resolution === 720
+ })
+ })
+
+ it('Should publish after HLS transcoding only, when the first child resolution job ends', async function () {
+ await servers[0].config.enableTranscoding({ hls: true, webVideo: false, resolutions: [] })
+ const uuid = await prepareVideo()
+
+ await checkPublication({
+ uuid,
+ publishedJobFinder: data => data.type === 'new-resolution-to-hls' && data.resolution === 720
+ })
+ })
+
+ it('Should publish after hls transcoding only, when the max HLS resolution ends', async function () {
+ await servers[0].config.enableTranscoding({ hls: true, webVideo: false, resolutions })
+ const uuid = await prepareVideo()
+
+ await checkPublication({
+ uuid,
+ publishedJobFinder: data => data.type === 'new-resolution-to-hls' && data.resolution === 720
+ })
+ })
+
+ it('Should publish after hls transcoding only with audio and video splitted, when audio & video streams are ready', async function () {
+ await servers[0].config.enableTranscoding({ hls: true, webVideo: false, resolutions, splitAudioAndVideo: true })
+ const uuid = await prepareVideo()
+
+ await checkPublication({
+ uuid,
+ publishedJobFinder: data => data.type === 'new-resolution-to-hls' && data.resolution === 0
+ })
+ })
+ })
+
+ after(async function () {
+ await cleanupTests(servers)
+ })
+})
diff --git a/packages/tests/src/peertube-runner/custom-upload.ts b/packages/tests/src/peertube-runner/custom-upload.ts
index cc4a9a5299..a045fa5897 100644
--- a/packages/tests/src/peertube-runner/custom-upload.ts
+++ b/packages/tests/src/peertube-runner/custom-upload.ts
@@ -24,6 +24,8 @@ describe('Test peertube-runner custom upload', function () {
let mockUploadServerUrl: string
let transcoded: string
+ const resolutions = [ 720 ]
+
const mockUpload = new MockUpload()
async function registerRunner () {
@@ -84,7 +86,7 @@ describe('Test peertube-runner custom upload', function () {
})
it('Should upload the file on another endpoint for web video', async function () {
- await server.config.enableTranscoding({ hls: false, webVideo: true })
+ await server.config.enableTranscoding({ hls: false, webVideo: true, resolutions })
await server.videos.quickUpload({ name: 'video 1' })
await server.videos.quickUpload({ name: 'video 2' })
diff --git a/packages/tests/src/peertube-runner/index.ts b/packages/tests/src/peertube-runner/index.ts
index 9afba566a2..30e22508b8 100644
--- a/packages/tests/src/peertube-runner/index.ts
+++ b/packages/tests/src/peertube-runner/index.ts
@@ -5,5 +5,6 @@ export * from './live-transcoding.js'
export * from './replace-file.js'
export * from './shutdown.js'
export * from './studio-transcoding.js'
+export * from './video-state.js'
export * from './video-transcription.js'
export * from './vod-transcoding.js'
diff --git a/packages/tests/src/peertube-runner/video-state.ts b/packages/tests/src/peertube-runner/video-state.ts
new file mode 100644
index 0000000000..a972980cd9
--- /dev/null
+++ b/packages/tests/src/peertube-runner/video-state.ts
@@ -0,0 +1,327 @@
+/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
+import { sortBy } from '@peertube/peertube-core-utils'
+import { MoveVideoStoragePayload, RunnerJobType, RunnerJobVODPayload, VideoState } from '@peertube/peertube-models'
+import { areMockObjectStorageTestsDisabled } from '@peertube/peertube-node-utils'
+import {
+ ObjectStorageCommand,
+ PeerTubeServer,
+ cleanupTests,
+ createMultipleServers,
+ doubleFollow,
+ setAccessTokensToServers,
+ waitJobs
+} from '@peertube/peertube-server-commands'
+import { PeerTubeRunnerProcess } from '@tests/shared/peertube-runner-process.js'
+import { expect } from 'chai'
+
+describe('Test video state on peertube runners', function () {
+ let servers: PeerTubeServer[] = []
+ const resolutions = [ 720, 240, 144 ]
+ let peertubeRunner: PeerTubeRunnerProcess
+ const transcodingTypes = new Set<RunnerJobType>([ 'vod-audio-merge-transcoding', 'vod-hls-transcoding', 'vod-web-video-transcoding' ])
+
+ async function prepareVideo () {
+ const { uuid } = await servers[0].videos.quickUpload({ name: 'video', fixture: 'video_short.mp4' })
+ await waitJobs(servers, { runnerJobs: true })
+
+ return uuid
+ }
+
+ before(async function () {
+ this.timeout(120000)
+
+ servers = await createMultipleServers(2)
+
+ await setAccessTokensToServers(servers)
+ await doubleFollow(servers[0], servers[1])
+
+ const registrationToken = await servers[0].runnerRegistrationTokens.getFirstRegistrationToken()
+
+ peertubeRunner = new PeerTubeRunnerProcess(servers[0])
+ await peertubeRunner.runServer()
+ await peertubeRunner.registerPeerTubeInstance({ registrationToken, runnerName: 'runner' })
+
+ await servers[0].config.setTranscodingConcurrency(1)
+ await servers[0].config.enableRemoteTranscoding()
+ })
+
+ describe('Without object storage', function () {
+ async function checkPublication (options: {
+ uuid: string
+ publishedJobFinder: (jobType: RunnerJobType, data: RunnerJobVODPayload) => boolean
+ }) {
+ const { uuid, publishedJobFinder } = options
+
+ const { data } = await servers[0].runnerJobs.list({ count: 100 })
+ const publishedJob = data
+ .filter(j => transcodingTypes.has(j.type))
+ .find(j => {
+ return j.privatePayload.videoUUID === uuid && publishedJobFinder(j.type, j.payload as RunnerJobVODPayload)
+ })
+
+ for (const server of servers) {
+ const video = await server.videos.get({ id: uuid })
+ expect(video.state.id).to.equal(VideoState.PUBLISHED)
+
+ expect(publishedJob.startedAt).to.exist
+ expect(publishedJob.finishedAt).to.exist
+
+ expect(new Date(video.publishedAt)).to.be.greaterThan(new Date(publishedJob.startedAt))
+ expect(new Date(video.publishedAt)).to.be.below(new Date(publishedJob.finishedAt))
+ }
+ }
+
+ async function checkJobOrder (options: {
+ uuid: string
+ order: { type: RunnerJobType, resolution: number }[]
+ }) {
+ const { uuid, order } = options
+
+ const { data } = await servers[0].runnerJobs.list({ count: 100 })
+ const filteredJobs = sortBy(data.filter(j => transcodingTypes.has(j.type) && j.privatePayload.videoUUID === uuid), 'startedAt')
+
+ const jobs = filteredJobs.map(j => {
+ return { type: j.type, resolution: (j.payload as RunnerJobVODPayload).output.resolution }
+ })
+
+ expect(jobs).to.deep.equal(order)
+ }
+
+ it('Should directly publish a video after upload without transcoding', async function () {
+ await servers[0].config.disableTranscoding()
+
+ const { uuid } = await servers[0].videos.quickUpload({ name: 'video' })
+ await waitJobs(servers)
+
+ for (const server of servers) {
+ const video = await server.videos.get({ id: uuid })
+ expect(video.state.id).to.equal(VideoState.PUBLISHED)
+ }
+ })
+
+ it('Should publish after web video transcoding only, when the optimize job ends', async function () {
+ await servers[0].config.enableTranscoding({ hls: false, webVideo: true, resolutions: [] })
+ const uuid = await prepareVideo()
+
+ await checkPublication({
+ uuid,
+ publishedJobFinder: (type, data) => type === 'vod-web-video-transcoding' && data.output.resolution === 720
+ })
+
+ await checkJobOrder({
+ uuid,
+ order: [ { type: 'vod-web-video-transcoding', resolution: 720 } ]
+ })
+ })
+
+ it('Should publish after web video transcoding only, when the first child resolution job ends', async function () {
+ await servers[0].config.enableTranscoding({ hls: false, webVideo: true, resolutions })
+ const uuid = await prepareVideo()
+
+ await checkPublication({
+ uuid,
+ publishedJobFinder: (type, data) => type === 'vod-web-video-transcoding' && data.output.resolution === 240
+ })
+
+ await checkJobOrder({
+ uuid,
+ order: [
+ { type: 'vod-web-video-transcoding', resolution: 720 },
+ { type: 'vod-web-video-transcoding', resolution: 240 },
+ { type: 'vod-web-video-transcoding', resolution: 144 }
+ ]
+ })
+ })
+
+ it('Should publish after web & hls transcodings, when the max HLS resolution ends', async function () {
+ await servers[0].config.enableTranscoding({ hls: true, webVideo: true, resolutions })
+ const uuid = await prepareVideo()
+
+ await checkPublication({
+ uuid,
+ publishedJobFinder: (type, data) => type === 'vod-hls-transcoding' && data.output.resolution === 720
+ })
+
+ await checkJobOrder({
+ uuid,
+ order: [
+ { type: 'vod-web-video-transcoding', resolution: 720 },
+ { type: 'vod-hls-transcoding', resolution: 720 },
+ { type: 'vod-web-video-transcoding', resolution: 240 },
+ { type: 'vod-web-video-transcoding', resolution: 144 },
+ { type: 'vod-hls-transcoding', resolution: 240 },
+ { type: 'vod-hls-transcoding', resolution: 144 }
+ ]
+ })
+ })
+
+ it('Should publish after HLS transcoding only, when the first child resolution job ends', async function () {
+ await servers[0].config.enableTranscoding({ hls: true, webVideo: false, resolutions: [] })
+ const uuid = await prepareVideo()
+
+ await checkPublication({
+ uuid,
+ publishedJobFinder: (type, data) => type === 'vod-hls-transcoding' && data.output.resolution === 720
+ })
+
+ await checkJobOrder({
+ uuid,
+ order: [
+ { type: 'vod-web-video-transcoding', resolution: 720 },
+ { type: 'vod-hls-transcoding', resolution: 720 }
+ ]
+ })
+ })
+
+ it('Should publish after hls transcoding only, when the max HLS resolution ends', async function () {
+ await servers[0].config.enableTranscoding({ hls: true, webVideo: false, resolutions })
+ const uuid = await prepareVideo()
+
+ await checkPublication({
+ uuid,
+ publishedJobFinder: (type, data) => type === 'vod-hls-transcoding' && data.output.resolution === 720
+ })
+
+ await checkJobOrder({
+ uuid,
+ order: [
+ { type: 'vod-web-video-transcoding', resolution: 720 },
+ { type: 'vod-hls-transcoding', resolution: 720 },
+ { type: 'vod-hls-transcoding', resolution: 240 },
+ { type: 'vod-hls-transcoding', resolution: 144 }
+ ]
+ })
+ })
+
+ it('Should publish after hls transcoding only with audio and video splitted, when audio & video streams are ready', async function () {
+ await servers[0].config.enableTranscoding({ hls: true, webVideo: false, resolutions, splitAudioAndVideo: true })
+ const uuid = await prepareVideo()
+
+ await checkPublication({
+ uuid,
+ publishedJobFinder: (type, data) => type === 'vod-hls-transcoding' && data.output.resolution === 0
+ })
+
+ await checkJobOrder({
+ uuid,
+ order: [
+ { type: 'vod-web-video-transcoding', resolution: 720 },
+ { type: 'vod-hls-transcoding', resolution: 720 },
+ { type: 'vod-hls-transcoding', resolution: 0 },
+ { type: 'vod-hls-transcoding', resolution: 240 },
+ { type: 'vod-hls-transcoding', resolution: 144 }
+ ]
+ })
+ })
+ })
+
+ describe('With object storage', function () {
+ if (areMockObjectStorageTestsDisabled()) return
+
+ const objectStorage = new ObjectStorageCommand()
+
+ async function checkPublication (options: {
+ uuid: string
+ publishedJobFinder: (jobType: RunnerJobType, data: RunnerJobVODPayload) => boolean
+ }) {
+ const { uuid, publishedJobFinder } = options
+
+ const { data } = await servers[0].runnerJobs.list({ count: 100 })
+ const transcodingJob = data
+ .filter(j => transcodingTypes.has(j.type))
+ .find(j => {
+ return j.privatePayload.videoUUID === uuid && publishedJobFinder(j.type, j.payload as RunnerJobVODPayload)
+ })
+
+ // Video is published in the next move to object storage job
+ const { data: objectStorageJobs } = await servers[0].jobs.list({ jobType: 'move-to-object-storage', sort: 'createdAt', count: 100 })
+ const publishedJob = objectStorageJobs.find(j => {
+ const data = j.data as MoveVideoStoragePayload
+
+ return data.videoUUID === uuid && new Date(j.processedOn) > new Date(transcodingJob.startedAt)
+ })
+
+ for (const server of servers) {
+ const video = await server.videos.get({ id: uuid })
+ expect(video.state.id).to.equal(VideoState.PUBLISHED)
+
+ expect(publishedJob.processedOn).to.exist
+ expect(publishedJob.finishedOn).to.exist
+
+ expect(new Date(video.publishedAt)).to.be.greaterThan(new Date(publishedJob.processedOn))
+ expect(new Date(video.publishedAt)).to.be.below(new Date(publishedJob.finishedOn))
+ }
+ }
+
+ before(async function () {
+ await objectStorage.prepareDefaultMockBuckets()
+
+ await servers[0].kill()
+ await servers[0].run(objectStorage.getDefaultMockConfig())
+ })
+
+ it('Should publish after web video transcoding only, when the optimize job ends', async function () {
+ await servers[0].config.enableTranscoding({ hls: false, webVideo: true, resolutions: [] })
+ const uuid = await prepareVideo()
+
+ await checkPublication({
+ uuid,
+ publishedJobFinder: (type, data) => type === 'vod-web-video-transcoding' && data.output.resolution === 720
+ })
+ })
+
+ it('Should publish after web video transcoding only, when the first child resolution job ends', async function () {
+ await servers[0].config.enableTranscoding({ hls: false, webVideo: true, resolutions })
+ const uuid = await prepareVideo()
+
+ await checkPublication({
+ uuid,
+ publishedJobFinder: (type, data) => type === 'vod-web-video-transcoding' && data.output.resolution === 720
+ })
+ })
+
+ it('Should publish after web & hls transcodings, when the max HLS resolution ends', async function () {
+ await servers[0].config.enableTranscoding({ hls: true, webVideo: true, resolutions })
+ const uuid = await prepareVideo()
+
+ await checkPublication({
+ uuid,
+ publishedJobFinder: (type, data) => type === 'vod-web-video-transcoding' && data.output.resolution === 720
+ })
+ })
+
+ it('Should publish after HLS transcoding only, when the first child resolution job ends', async function () {
+ await servers[0].config.enableTranscoding({ hls: true, webVideo: false, resolutions: [] })
+ const uuid = await prepareVideo()
+
+ await checkPublication({
+ uuid,
+ publishedJobFinder: (type, data) => type === 'vod-hls-transcoding' && data.output.resolution === 720
+ })
+ })
+
+ it('Should publish after hls transcoding only, when the max HLS resolution ends', async function () {
+ await servers[0].config.enableTranscoding({ hls: true, webVideo: false, resolutions })
+ const uuid = await prepareVideo()
+
+ await checkPublication({
+ uuid,
+ publishedJobFinder: (type, data) => type === 'vod-hls-transcoding' && data.output.resolution === 720
+ })
+ })
+
+ it('Should publish after hls transcoding only with audio and video splitted, when audio & video streams are ready', async function () {
+ await servers[0].config.enableTranscoding({ hls: true, webVideo: false, resolutions, splitAudioAndVideo: true })
+ const uuid = await prepareVideo()
+
+ await checkPublication({
+ uuid,
+ publishedJobFinder: (type, data) => type === 'vod-hls-transcoding' && data.output.resolution === 0
+ })
+ })
+ })
+
+ after(async function () {
+ await cleanupTests(servers)
+ })
+})
diff --git a/server/core/controllers/api/videos/source.ts b/server/core/controllers/api/videos/source.ts
index aec4688cee..5cba7efb2b 100644
--- a/server/core/controllers/api/videos/source.ts
+++ b/server/core/controllers/api/videos/source.ts
@@ -200,7 +200,16 @@ async function addVideoJobsAfterUpload (video: MVideoFullLight, videoFile: MVide
]
if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) {
- jobs.push(await buildMoveVideoJob({ video, isNewVideo: false, previousVideoState: undefined, type: 'move-to-object-storage' }))
+ jobs.push(
+ await buildMoveVideoJob({
+ type: 'move-to-object-storage',
+ video,
+ moveVideoState: {
+ isNewVideo: false,
+ previousVideoState: undefined
+ }
+ })
+ )
}
if (video.state === VideoState.TO_TRANSCODE) {
diff --git a/server/core/lib/hls.ts b/server/core/lib/hls.ts
index c9a855259a..3ab01f4c22 100644
--- a/server/core/lib/hls.ts
+++ b/server/core/lib/hls.ts
@@ -148,6 +148,8 @@ function updateMasterHLSPlaylist (video: MVideo, playlistArg: MStreamingPlaylist
const masterPlaylists = [ '#EXTM3U', '#EXT-X-VERSION:3', '', ...extMediaSubtitle, '', ...extMediaAudio, '', ...extStreamInfo ]
+ await playlist.reload()
+
if (playlist.playlistFilename) {
await video.removeStreamingPlaylistFile(playlist, playlist.playlistFilename)
}
diff --git a/server/core/lib/job-queue/handlers/move-to-file-system.ts b/server/core/lib/job-queue/handlers/move-to-file-system.ts
index 0e72f39283..a27303421f 100644
--- a/server/core/lib/job-queue/handlers/move-to-file-system.ts
+++ b/server/core/lib/job-queue/handlers/move-to-file-system.ts
@@ -1,28 +1,7 @@
-import { FileStorage, isMoveCaptionPayload, isMoveVideoStoragePayload, MoveStoragePayload, VideoStateType } from '@peertube/peertube-models'
+import { isMoveCaptionPayload, isMoveVideoStoragePayload, MoveStoragePayload } from '@peertube/peertube-models'
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
-import { updateTorrentMetadata } from '@server/lib/webtorrent.js'
-import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants.js'
-import {
- makeCaptionFileAvailable,
- makeHLSFileAvailable,
- makeOriginalFileAvailable,
- makeWebVideoFileAvailable,
- removeCaptionObjectStorage,
- removeHLSFileObjectStorageByFilename,
- removeHLSObjectStorage,
- removeOriginalFileObjectStorage,
- removeWebVideoObjectStorage
-} from '@server/lib/object-storage/index.js'
-import { getHLSDirectory, getHLSResolutionPlaylistFilename } from '@server/lib/paths.js'
-import { updateHLSMasterOnCaptionChange, upsertCaptionPlaylistOnFS } from '@server/lib/video-captions.js'
-import { VideoPathManager } from '@server/lib/video-path-manager.js'
-import { moveToFailedMoveToFileSystemState, moveToNextState } from '@server/lib/video-state.js'
-import { MStreamingPlaylistVideo, MVideo, MVideoCaption, MVideoFile, MVideoWithAllFiles } from '@server/types/models/index.js'
-import { MVideoSource } from '@server/types/models/video/video-source.js'
+import { moveCaptionToFS, moveVideoToFS, onMoveVideoToFSFailure } from '@server/lib/move-storage/move-to-file-system.js'
import { Job } from 'bullmq'
-import { join } from 'path'
-import { moveCaptionToStorageJob } from './shared/move-caption.js'
-import { moveVideoToStorageJob, onMoveVideoToStorageFailure } from './shared/move-video.js'
const lTagsBase = loggerTagsFactory('move-file-system')
@@ -30,28 +9,23 @@ export async function processMoveToFileSystem (job: Job) {
const payload = job.data as MoveStoragePayload
if (isMoveVideoStoragePayload(payload)) { // Move all video related files
- logger.info('Moving video %s to file system in job %s.', payload.videoUUID, job.id)
+ logger.info(`Moving video ${payload.videoUUID} to file system in job ${job.id}.`, lTagsBase(payload.videoUUID))
- await moveVideoToStorageJob({
- jobId: job.id,
- videoUUID: payload.videoUUID,
- loggerTags: lTagsBase().tags,
-
- moveWebVideoFiles,
- moveHLSFiles,
- moveVideoSourceFile,
- moveCaptionFiles,
+ const moveVideoState = payload.isNewVideo !== undefined
+ ? { isNewVideo: payload.isNewVideo, previousVideoState: payload.previousVideoState }
+ : payload.moveVideoState
- doAfterLastMove: video => doAfterLastMove({ video, previousVideoState: payload.previousVideoState, isNewVideo: payload.isNewVideo })
+ return moveVideoToFS({
+ videoUUID: payload.videoUUID,
+ moveVideoState,
+ loggerTags: lTagsBase().tags
})
} else if (isMoveCaptionPayload(payload)) { // Only caption file
- logger.info(`Moving video caption ${payload.captionId} to file system in job ${job.id}.`)
+ logger.info(`Moving video caption ${payload.captionId} to file system in job ${job.id}.`, lTagsBase(payload.captionId))
- await moveCaptionToStorageJob({
- jobId: job.id,
+ return moveCaptionToFS({
captionId: payload.captionId,
- loggerTags: lTagsBase().tags,
- moveCaptionFiles
+ loggerTags: lTagsBase().tags
})
} else {
throw new Error('Unknown payload type')
@@ -63,168 +37,9 @@ export async function onMoveToFileSystemFailure (job: Job, err: any) {
if (!isMoveVideoStoragePayload(payload)) return
- await onMoveVideoToStorageFailure({
+ await onMoveVideoToFSFailure({
videoUUID: payload.videoUUID,
err,
- lTags: lTagsBase(),
- moveToFailedState: moveToFailedMoveToFileSystemState
+ loggerTags: lTagsBase().tags
})
}
-
-// ---------------------------------------------------------------------------
-// Private
-// ---------------------------------------------------------------------------
-
-async function moveVideoSourceFile (source: MVideoSource) {
- if (source.storage === FileStorage.FILE_SYSTEM) return
-
- await makeOriginalFileAvailable(
- source.keptOriginalFilename,
- VideoPathManager.Instance.getFSOriginalVideoFilePath(source.keptOriginalFilename)
- )
-
- const oldFileUrl = source.fileUrl
-
- source.fileUrl = null
- source.storage = FileStorage.FILE_SYSTEM
- await source.save()
-
- logger.debug('Removing original video file %s because it\'s now on file system', oldFileUrl, lTagsBase())
-
- await removeOriginalFileObjectStorage(source)
-}
-
-async function moveWebVideoFiles (video: MVideoWithAllFiles) {
- for (const file of video.VideoFiles) {
- if (file.storage === FileStorage.FILE_SYSTEM) continue
-
- await makeWebVideoFileAvailable(file.filename, VideoPathManager.Instance.getFSVideoFileOutputPath(video, file))
- await onVideoFileMoved({
- videoOrPlaylist: video,
- file,
- objetStorageRemover: () => removeWebVideoObjectStorage(file)
- })
- }
-}
-
-async function moveHLSFiles (video: MVideoWithAllFiles) {
- for (const playlist of video.VideoStreamingPlaylists) {
- const playlistWithVideo = playlist.withVideo(video)
-
- for (const file of playlist.VideoFiles) {
- if (file.storage === FileStorage.FILE_SYSTEM) continue
-
- // Resolution playlist
- const playlistFilename = getHLSResolutionPlaylistFilename(file.filename)
- await makeHLSFileAvailable(playlistWithVideo, playlistFilename, join(getHLSDirectory(video), playlistFilename))
- await makeHLSFileAvailable(playlistWithVideo, file.filename, join(getHLSDirectory(video), file.filename))
-
- await onVideoFileMoved({
- videoOrPlaylist: playlistWithVideo,
- file,
- objetStorageRemover: async () => {
- await removeHLSFileObjectStorageByFilename(playlistWithVideo, playlistFilename)
- await removeHLSFileObjectStorageByFilename(playlistWithVideo, file.filename)
- }
- })
- }
- }
-}
-
-async function onVideoFileMoved (options: {
- videoOrPlaylist: MVideo | MStreamingPlaylistVideo
- file: MVideoFile
- objetStorageRemover: () => Promise<any>
-}) {
- const { videoOrPlaylist, file, objetStorageRemover } = options
-
- const oldFileUrl = file.fileUrl
-
- file.fileUrl = null
- file.storage = FileStorage.FILE_SYSTEM
-
- await updateTorrentMetadata(videoOrPlaylist, file)
- await file.save()
-
- logger.debug('Removing web video file %s because it\'s now on file system', oldFileUrl, lTagsBase())
- await objetStorageRemover()
-}
-
-// ---------------------------------------------------------------------------
-
-async function moveCaptionFiles (captions: MVideoCaption[], hls: MStreamingPlaylistVideo) {
- let hlsUpdated = false
-
- for (const caption of captions) {
- if (caption.storage === FileStorage.OBJECT_STORAGE) {
- const oldFileUrl = caption.fileUrl
-
- await makeCaptionFileAvailable(caption.filename, caption.getFSFilePath())
-
- // Assign new values before building the m3u8 file
- caption.fileUrl = null
- caption.storage = FileStorage.FILE_SYSTEM
-
- await caption.save()
-
- logger.debug('Removing caption file %s because it\'s now on file system', oldFileUrl, lTagsBase())
- await removeCaptionObjectStorage(caption)
- }
-
- if (hls && (!caption.m3u8Filename || caption.m3u8Url)) {
- hlsUpdated = true
-
- const oldM3U8Url = caption.m3u8Url
- const oldM3U8Filename = caption.m3u8Filename
-
- // Caption link has been updated, so we must also update the HLS caption playlist
- caption.m3u8Filename = await upsertCaptionPlaylistOnFS(caption, hls.Video)
- caption.m3u8Url = null
-
- await caption.save()
-
- if (oldM3U8Url) {
- logger.debug(`Removing video caption playlist file ${oldM3U8Url} because it's now on file system`, lTagsBase())
-
- await removeHLSFileObjectStorageByFilename(hls, oldM3U8Filename)
- }
- }
- }
-
- if (hlsUpdated) {
- await updateHLSMasterOnCaptionChange(hls.Video, hls)
- }
-}
-
-// ---------------------------------------------------------------------------
-
-async function doAfterLastMove (options: {
- video: MVideoWithAllFiles
- previousVideoState: VideoStateType
- isNewVideo: boolean
-}) {
- const { video, previousVideoState, isNewVideo } = options
-
- for (const playlist of video.VideoStreamingPlaylists) {
- if (playlist.storage === FileStorage.FILE_SYSTEM) continue
-
- const playlistWithVideo = playlist.withVideo(video)
-
- for (const filename of [ playlist.playlistFilename, playlist.segmentsSha256Filename ]) {
- await makeHLSFileAvailable(playlistWithVideo, filename, join(getHLSDirectory(video), filename))
- }
-
- playlist.playlistUrl = null
- playlist.segmentsSha256Url = null
- playlist.storage = FileStorage.FILE_SYSTEM
-
- playlist.assignP2PMediaLoaderInfoHashes(video, playlist.VideoFiles)
- playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION
-
- await playlist.save()
-
- await removeHLSObjectStorage(playlistWithVideo)
- }
-
- await moveToNextState({ video, previousVideoState, isNewVideo })
-}
diff --git a/server/core/lib/job-queue/handlers/move-to-object-storage.ts b/server/core/lib/job-queue/handlers/move-to-object-storage.ts
index a7d5ce9a3a..e3c047f128 100644
--- a/server/core/lib/job-queue/handlers/move-to-object-storage.ts
+++ b/server/core/lib/job-queue/handlers/move-to-object-storage.ts
@@ -1,27 +1,11 @@
-import { FileStorage, isMoveCaptionPayload, isMoveVideoStoragePayload, MoveStoragePayload, VideoStateType } from '@peertube/peertube-models'
+import { isMoveCaptionPayload, isMoveVideoStoragePayload, MoveStoragePayload } from '@peertube/peertube-models'
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
-import { updateTorrentMetadata } from '@server/lib/webtorrent.js'
-import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants.js'
-import { buildCaptionM3U8Content } from '@server/lib/hls.js'
import {
- storeHLSFileFromContent,
- storeHLSFileFromFilename,
- storeOriginalVideoFile,
- storeVideoCaption,
- storeWebVideoFile
-} from '@server/lib/object-storage/index.js'
-import { getHLSDirectory, getHLSResolutionPlaylistFilename } from '@server/lib/paths.js'
-import { updateHLSMasterOnCaptionChange } from '@server/lib/video-captions.js'
-import { VideoPathManager } from '@server/lib/video-path-manager.js'
-import { moveToFailedMoveToObjectStorageState, moveToNextState } from '@server/lib/video-state.js'
-import { MStreamingPlaylistVideo, MVideo, MVideoCaption, MVideoFile, MVideoWithAllFiles } from '@server/types/models/index.js'
-import { MVideoSource } from '@server/types/models/video/video-source.js'
+ moveCaptionToObjectStorage,
+ moveVideoToObjectStorage,
+ onMoveVideoToObjectStorageFailure
+} from '@server/lib/move-storage/move-to-object-storage.js'
import { Job } from 'bullmq'
-import { remove } from 'fs-extra/esm'
-import { join } from 'path'
-import { moveCaptionToStorageJob } from './shared/move-caption.js'
-import { moveVideoToStorageJob, onMoveVideoToStorageFailure } from './shared/move-video.js'
-import { VideoCaptionModel } from '@server/models/video/video-caption.js'
const lTagsBase = loggerTagsFactory('move-object-storage')
@@ -31,28 +15,21 @@ export async function processMoveToObjectStorage (job: Job) {
if (isMoveVideoStoragePayload(payload)) { // Move all video related files
logger.info(`Moving video ${payload.videoUUID} to object storage in job ${job.id}`, lTagsBase(payload.videoUUID))
- await moveVideoToStorageJob({
- jobId: job.id,
- videoUUID: payload.videoUUID,
- loggerTags: lTagsBase().tags,
-
- moveWebVideoFiles,
- moveHLSFiles,
- moveVideoSourceFile,
- moveCaptionFiles,
+ const moveVideoState = payload.isNewVideo !== undefined
+ ? { isNewVideo: payload.isNewVideo, previousVideoState: payload.previousVideoState }
+ : payload.moveVideoState
- doAfterLastMove: video => {
- return doAfterLastVideoMove({ video, previousVideoState: payload.previousVideoState, isNewVideo: payload.isNewVideo })
- }
+ await moveVideoToObjectStorage({
+ videoUUID: payload.videoUUID,
+ moveVideoState,
+ loggerTags: lTagsBase().tags
})
} else if (isMoveCaptionPayload(payload)) { // Only caption file
logger.info(`Moving video caption ${payload.captionId} to object storage in job ${job.id}.`, lTagsBase(payload.captionId))
- await moveCaptionToStorageJob({
- jobId: job.id,
+ return moveCaptionToObjectStorage({
captionId: payload.captionId,
- loggerTags: lTagsBase().tags,
- moveCaptionFiles
+ loggerTags: lTagsBase().tags
})
} else {
throw new Error('Unknown payload type')
@@ -64,155 +41,9 @@ export async function onMoveToObjectStorageFailure (job: Job, err: any) {
if (!isMoveVideoStoragePayload(payload)) return
- await onMoveVideoToStorageFailure({
+ await onMoveVideoToObjectStorageFailure({
videoUUID: payload.videoUUID,
err,
- lTags: lTagsBase(),
- moveToFailedState: moveToFailedMoveToObjectStorageState
+ loggerTags: lTagsBase().tags
})
}
-
-// ---------------------------------------------------------------------------
-
-async function moveVideoSourceFile (source: MVideoSource) {
- if (source.storage !== FileStorage.FILE_SYSTEM) return
-
- const sourcePath = VideoPathManager.Instance.getFSOriginalVideoFilePath(source.keptOriginalFilename)
- const fileUrl = await storeOriginalVideoFile(sourcePath, source.keptOriginalFilename)
-
- source.storage = FileStorage.OBJECT_STORAGE
- source.fileUrl = fileUrl
- await source.save()
-
- logger.debug('Removing original video file ' + sourcePath + ' because it\'s now on object storage', lTagsBase())
-
- await remove(sourcePath)
-}
-
-// ---------------------------------------------------------------------------
-
-async function moveCaptionFiles (captions: MVideoCaption[], hls: MStreamingPlaylistVideo) {
- let hlsUpdated = false
-
- for (const caption of captions) {
- if (caption.storage === FileStorage.FILE_SYSTEM) {
- const captionPath = caption.getFSFilePath()
-
- // Assign new values before building the m3u8 file
- caption.fileUrl = await storeVideoCaption(captionPath, caption.filename)
- caption.storage = FileStorage.OBJECT_STORAGE
-
- await caption.save()
-
- logger.debug(`Removing video caption file ${captionPath} because it's now on object storage`, lTagsBase())
- await remove(captionPath)
- }
-
- if (hls && (!caption.m3u8Filename || !caption.m3u8Url)) {
- hlsUpdated = true
-
- const m3u8PathToRemove = caption.getFSM3U8Path(hls.Video)
-
- // Caption link has been updated, so we must also update the HLS caption playlist
- const content = buildCaptionM3U8Content({ video: hls.Video, caption })
-
- caption.m3u8Filename = VideoCaptionModel.generateM3U8Filename(caption.filename)
- caption.m3u8Url = await storeHLSFileFromContent({
- playlist: hls,
- pathOrFilename: caption.m3u8Filename,
- content
- })
-
- await caption.save()
-
- if (m3u8PathToRemove) {
- logger.debug(`Removing video caption playlist file ${m3u8PathToRemove} because it's now on object storage`, lTagsBase())
- await remove(m3u8PathToRemove)
- }
- }
- }
-
- if (hlsUpdated) {
- await updateHLSMasterOnCaptionChange(hls.Video, hls)
- }
-}
-
-// ---------------------------------------------------------------------------
-
-async function moveWebVideoFiles (video: MVideoWithAllFiles) {
- for (const file of video.VideoFiles) {
- if (file.storage !== FileStorage.FILE_SYSTEM) continue
-
- const fileUrl = await storeWebVideoFile(video, file)
-
- const oldPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, file)
- await onVideoFileMoved({ videoOrPlaylist: video, file, fileUrl, oldPath })
- }
-}
-
-async function moveHLSFiles (video: MVideoWithAllFiles) {
- for (const playlist of video.VideoStreamingPlaylists) {
- const playlistWithVideo = playlist.withVideo(video)
-
- for (const file of playlist.VideoFiles) {
- if (file.storage !== FileStorage.FILE_SYSTEM) continue
-
- // Resolution playlist
- const playlistFilename = getHLSResolutionPlaylistFilename(file.filename)
- await storeHLSFileFromFilename(playlistWithVideo, playlistFilename)
-
- // Resolution fragmented file
- const fileUrl = await storeHLSFileFromFilename(playlistWithVideo, file.filename)
-
- const oldPath = join(getHLSDirectory(video), file.filename)
-
- await onVideoFileMoved({ videoOrPlaylist: Object.assign(playlist, { Video: video }), file, fileUrl, oldPath })
- }
- }
-}
-
-async function onVideoFileMoved (options: {
- videoOrPlaylist: MVideo | MStreamingPlaylistVideo
- file: MVideoFile
- fileUrl: string
- oldPath: string
-}) {
- const { videoOrPlaylist, file, fileUrl, oldPath } = options
-
- file.fileUrl = fileUrl
- file.storage = FileStorage.OBJECT_STORAGE
-
- await updateTorrentMetadata(videoOrPlaylist, file)
- await file.save()
-
- logger.debug('Removing %s because it\'s now on object storage', oldPath, lTagsBase())
- await remove(oldPath)
-}
-
-// ---------------------------------------------------------------------------
-
-async function doAfterLastVideoMove (options: {
- video: MVideoWithAllFiles
- previousVideoState: VideoStateType
- isNewVideo: boolean
-}) {
- const { video, previousVideoState, isNewVideo } = options
-
- for (const playlist of video.VideoStreamingPlaylists) {
- if (playlist.storage === FileStorage.OBJECT_STORAGE) continue
-
- const playlistWithVideo = playlist.withVideo(video)
-
- playlist.playlistUrl = await storeHLSFileFromFilename(playlistWithVideo, playlist.playlistFilename)
- playlist.segmentsSha256Url = await storeHLSFileFromFilename(playlistWithVideo, playlist.segmentsSha256Filename)
- playlist.storage = FileStorage.OBJECT_STORAGE
-
- playlist.assignP2PMediaLoaderInfoHashes(video, playlist.VideoFiles)
- playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION
-
- await playlist.save()
- }
-
- await remove(getHLSDirectory(video))
- await moveToNextState({ video, previousVideoState, isNewVideo })
-}
diff --git a/server/core/lib/job-queue/handlers/transcoding-job-builder.ts b/server/core/lib/job-queue/handlers/transcoding-job-builder.ts
index 185739bd67..41bb749686 100644
--- a/server/core/lib/job-queue/handlers/transcoding-job-builder.ts
+++ b/server/core/lib/job-queue/handlers/transcoding-job-builder.ts
@@ -8,6 +8,12 @@ import { Job } from 'bullmq'
import { logger } from '../../../helpers/logger.js'
import { JobQueue } from '../job-queue.js'
+/**
+ * Create transcoding jobs.
+ * They are not created directly so we can keep parent/children relationships using bullmq features
+ * while having the ability to use remote runner jobs that are not part of bullmq
+ */
+
async function processTranscodingJobBuilder (job: Job) {
const payload = job.data as TranscodingJobBuilderPayload
@@ -23,8 +29,7 @@ async function processTranscodingJobBuilder (job: Job) {
video,
videoFile,
- user,
- videoFileAlreadyLocked: false
+ user
})
}
diff --git a/server/core/lib/job-queue/handlers/video-file-import.ts b/server/core/lib/job-queue/handlers/video-file-import.ts
index 8741b0e039..7c39ca4152 100644
--- a/server/core/lib/job-queue/handlers/video-file-import.ts
+++ b/server/core/lib/job-queue/handlers/video-file-import.ts
@@ -27,7 +27,16 @@ async function processVideoFileImport (job: Job) {
await updateVideoFile(video, payload.filePath)
if (CONFIG.OBJECT_STORAGE.ENABLED) {
- await JobQueue.Instance.createJob(await buildMoveVideoJob({ video, previousVideoState: video.state, type: 'move-to-object-storage' }))
+ await JobQueue.Instance.createJob(
+ await buildMoveVideoJob({
+ type: 'move-to-object-storage',
+ video,
+ moveVideoState: {
+ isNewVideo: false,
+ previousVideoState: video.state
+ }
+ })
+ )
} else {
await federateVideoIfNeeded(video, false)
}
diff --git a/server/core/lib/job-queue/handlers/video-import.ts b/server/core/lib/job-queue/handlers/video-import.ts
index 74394e0de0..c2572d4725 100644
--- a/server/core/lib/job-queue/handlers/video-import.ts
+++ b/server/core/lib/job-queue/handlers/video-import.ts
@@ -253,8 +253,7 @@ async function processFile (downloader: () => Promise<string>, videoImport: MVid
video,
videoFile,
user: videoImport.User,
- generateTranscription: options.generateTranscription,
- videoFileAlreadyLocked: true
+ generateTranscription: options.generateTranscription
})
} finally {
videoFileLockReleaser()
@@ -305,11 +304,9 @@ async function afterImportSuccess (options: {
videoFile: MVideoFile
user: MUserId
- videoFileAlreadyLocked: boolean
-
generateTranscription: boolean
}) {
- const { video, videoFile, videoImport, user, generateTranscription, videoFileAlreadyLocked } = options
+ const { video, videoFile, videoImport, user, generateTranscription } = options
Notifier.Instance.notifyOnFinishedVideoImport({ videoImport: Object.assign(videoImport, { Video: video }), success: true })
@@ -330,13 +327,20 @@ async function afterImportSuccess (options: {
if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) {
await JobQueue.Instance.createJob(
- await buildMoveVideoJob({ video, previousVideoState: VideoState.TO_IMPORT, type: 'move-to-object-storage' })
+ await buildMoveVideoJob({
+ type: 'move-to-object-storage',
+ video,
+ moveVideoState: {
+ isNewVideo: true,
+ previousVideoState: VideoState.TO_IMPORT
+ }
+ })
)
return
}
if (video.state === VideoState.TO_TRANSCODE) { // Create transcoding jobs?
- await createOptimizeOrMergeAudioJobs({ video, videoFile, isNewVideo: true, user, videoFileAlreadyLocked })
+ await createOptimizeOrMergeAudioJobs({ video, videoFile, isNewVideo: true, user })
}
}
diff --git a/server/core/lib/job-queue/handlers/video-transcoding.ts b/server/core/lib/job-queue/handlers/video-transcoding.ts
index 9ab7143a8f..829e8d0060 100644
--- a/server/core/lib/job-queue/handlers/video-transcoding.ts
+++ b/server/core/lib/job-queue/handlers/video-transcoding.ts
@@ -5,7 +5,7 @@ import {
OptimizeTranscodingPayload,
VideoTranscodingPayload
} from '@peertube/peertube-models'
-import { isHLSAudioMissing } from '@server/lib/runners/job-handlers/shared/utils.js'
+import { hasMissingHLSStreams } from '@server/lib/runners/job-handlers/shared/utils.js'
import { onTranscodingEnded } from '@server/lib/transcoding/ended-transcoding.js'
import { generateHlsPlaylistResolution } from '@server/lib/transcoding/hls-transcoding.js'
import { mergeAudioVideofile, optimizeOriginalVideofile, transcodeNewWebVideoResolution } from '@server/lib/transcoding/web-transcoding.js'
@@ -82,17 +82,17 @@ async function handleWebVideoMergeAudioJob (job: Job, payload: MergeAudioTransco
logger.info('Merge audio transcoding job for %s ended.', video.uuid, lTags(video.uuid), { payload })
- await onTranscodingEnded({ isNewVideo: payload.isNewVideo, moveVideoToNextState: !payload.hasChildren, video })
+ await onTranscodingEnded({ isNewVideo: payload.isNewVideo, moveVideoToNextState: payload.canMoveVideoState, video })
}
async function handleWebVideoOptimizeJob (job: Job, payload: OptimizeTranscodingPayload, video: MVideoFullLight, user: MUserId) {
logger.info('Handling optimize transcoding job for %s.', video.uuid, lTags(video.uuid), { payload })
- await optimizeOriginalVideofile({ video, quickTranscode: payload.quickTranscode, job })
+ await optimizeOriginalVideofile({ video, job })
logger.info('Optimize transcoding job for %s ended.', video.uuid, lTags(video.uuid), { payload })
- await onTranscodingEnded({ isNewVideo: payload.isNewVideo, moveVideoToNextState: !payload.hasChildren, video })
+ await onTranscodingEnded({ isNewVideo: payload.isNewVideo, moveVideoToNextState: payload.canMoveVideoState, video })
}
// ---------------------------------------------------------------------------
@@ -104,7 +104,8 @@ async function handleNewWebVideoResolutionJob (job: Job, payload: NewWebVideoRes
logger.info('Web Video transcoding job for %s ended.', video.uuid, lTags(video.uuid), { payload })
- await onTranscodingEnded({ isNewVideo: payload.isNewVideo, moveVideoToNextState: !payload.hasChildren, video })
+ // Always move video to next state, we're ready enough with this resolution
+ await onTranscodingEnded({ isNewVideo: payload.isNewVideo, moveVideoToNextState: payload.canMoveVideoState, video })
}
// ---------------------------------------------------------------------------
@@ -119,9 +120,10 @@ async function handleHLSJob (job: Job, payload: HLSTranscodingPayload, videoArg:
video = await VideoModel.loadFull(videoArg.uuid)
const { videoFile, separatedAudioFile } = video.getMaxQualityAudioAndVideoFiles()
+ const webVideoFile = video.getWebVideoFileResolution(payload.resolution)
- const videoFileInputs = payload.copyCodecs
- ? [ video.getWebVideoFileMinResolution(payload.resolution) ]
+ const videoFileInputs = webVideoFile
+ ? [ webVideoFile ]
: [ videoFile, separatedAudioFile ].filter(v => !!v)
await VideoPathManager.Instance.makeAvailableVideoFiles(videoFileInputs, ([ videoPath, separatedAudioPath ]) => {
@@ -134,7 +136,6 @@ async function handleHLSJob (job: Job, payload: HLSTranscodingPayload, videoArg:
inputFileMutexReleaser,
resolution: payload.resolution,
fps: payload.fps,
- copyCodecs: payload.copyCodecs,
separatedAudio: payload.separatedAudio,
job
})
@@ -145,22 +146,20 @@ async function handleHLSJob (job: Job, payload: HLSTranscodingPayload, videoArg:
logger.info('HLS transcoding job for %s ended.', video.uuid, lTags(video.uuid), { payload })
- const missingAudio = await isHLSAudioMissing({
- resolution: payload.resolution,
- separatedAudio: payload.separatedAudio,
+ const missingStream = await hasMissingHLSStreams({
+ inputStreams: payload.inputStreams,
+ transcodingRequestAt: payload.transcodingRequestAt,
videoId: videoArg.uuid
})
- if (!missingAudio && payload.deleteWebVideoFiles === true) {
+ if (!missingStream && payload.deleteWebVideoFiles === true) {
logger.info('Removing Web Video files of %s now we have a HLS version of it.', video.uuid, lTags(video.uuid))
await removeAllWebVideoFiles(video)
}
- // Splitted audio, wait audio generation before moving the video in its next state
- const moveVideoToNextState = missingAudio
- ? false
- : !payload.hasChildren
+ // Splitted audio, wait audio & video generation before moving the video in its next state
+ const moveVideoToNextState = payload.canMoveVideoState && !missingStream
await onTranscodingEnded({ isNewVideo: payload.isNewVideo, moveVideoToNextState, video })
}
diff --git a/server/core/lib/job-queue/job-queue.ts b/server/core/lib/job-queue/job-queue.ts
index 356516f152..6ac3753a73 100644
--- a/server/core/lib/job-queue/job-queue.ts
+++ b/server/core/lib/job-queue/job-queue.ts
@@ -28,7 +28,7 @@ import {
VideoTranscriptionPayload
} from '@peertube/peertube-models'
import { jobStates } from '@server/helpers/custom-validators/jobs.js'
-import { CONFIG } from '@server/initializers/config.js'
+import { CONFIG, registerConfigChangedHandler } from '@server/initializers/config.js'
import { processVideoRedundancy } from '@server/lib/job-queue/handlers/video-redundancy.js'
import {
FlowJob,
@@ -218,6 +218,12 @@ class JobQueue {
})
this.addRepeatableJobs()
+
+ registerConfigChangedHandler(() => {
+ for (const handlerName of Object.keys(handlers)) {
+ this.workers[handlerName].concurrency = this.getJobConcurrency(handlerName)
+ }
+ })
}
private buildWorker (handlerName: JobType) {
diff --git a/server/core/lib/move-storage/move-to-file-system.ts b/server/core/lib/move-storage/move-to-file-system.ts
new file mode 100644
index 0000000000..9a69a0e79b
--- /dev/null
+++ b/server/core/lib/move-storage/move-to-file-system.ts
@@ -0,0 +1,240 @@
+import { FileStorage, VideoStateType } from '@peertube/peertube-models'
+import { logger, LoggerTags, loggerTagsFactory } from '@server/helpers/logger.js'
+import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants.js'
+import {
+ makeCaptionFileAvailable,
+ makeHLSFileAvailable,
+ makeOriginalFileAvailable,
+ makeWebVideoFileAvailable,
+ removeCaptionObjectStorage,
+ removeHLSFileObjectStorageByFilename,
+ removeOriginalFileObjectStorage,
+ removeWebVideoObjectStorage
+} from '@server/lib/object-storage/index.js'
+import { getHLSDirectory, getHLSResolutionPlaylistFilename } from '@server/lib/paths.js'
+import { updateHLSMasterOnCaptionChange, upsertCaptionPlaylistOnFS } from '@server/lib/video-captions.js'
+import { VideoPathManager } from '@server/lib/video-path-manager.js'
+import { moveToFailedMoveToFileSystemState, moveToNextState } from '@server/lib/video-state.js'
+import { updateTorrentMetadata } from '@server/lib/webtorrent.js'
+import { VideoModel } from '@server/models/video/video.js'
+import { MStreamingPlaylistVideo, MVideo, MVideoCaption, MVideoFile, MVideoWithAllFiles } from '@server/types/models/index.js'
+import { MVideoSource } from '@server/types/models/video/video-source.js'
+import { join } from 'path'
+import { moveCaptionToStorage } from './shared/move-caption.js'
+import { moveVideoToStorage, onMoveVideoToStorageFailure } from './shared/move-video.js'
+
+const lTagsBase = loggerTagsFactory('move-file-system')
+
+export async function moveVideoToFS (options: {
+ videoUUID: string
+
+ moveVideoState?: {
+ isNewVideo: boolean
+ previousVideoState: VideoStateType
+ }
+
+ loggerTags: LoggerTags['tags']
+}) {
+ const { videoUUID, moveVideoState, loggerTags } = options
+
+ await moveVideoToStorage({
+ videoUUID,
+ loggerTags: [ ...lTagsBase().tags, ...loggerTags ],
+
+ targetStorage: FileStorage.FILE_SYSTEM,
+
+ moveWebVideoFiles,
+ moveHLSFiles,
+ moveVideoSourceFile,
+ moveCaptionFiles
+ })
+
+ if (options.moveVideoState) {
+ const video = await VideoModel.load(videoUUID)
+
+ await moveToNextState({ video, ...moveVideoState })
+ }
+}
+
+export function moveCaptionToFS (options: {
+ captionId: number
+ loggerTags: LoggerTags['tags']
+}) {
+ const { captionId, loggerTags } = options
+
+ return moveCaptionToStorage({
+ captionId,
+ loggerTags: [ ...lTagsBase().tags, ...loggerTags ],
+ moveCaptionFiles
+ })
+}
+
+export async function onMoveVideoToFSFailure (options: {
+ videoUUID: string
+ loggerTags: LoggerTags['tags']
+ err: Error
+}) {
+ const { videoUUID, err, loggerTags } = options
+
+ await onMoveVideoToStorageFailure({
+ videoUUID,
+ err,
+ loggerTags: [ ...lTagsBase().tags, ...loggerTags ],
+ moveToFailedState: moveToFailedMoveToFileSystemState
+ })
+}
+
+// ---------------------------------------------------------------------------
+// Private
+// ---------------------------------------------------------------------------
+
+async function moveVideoSourceFile (source: MVideoSource) {
+ if (source.storage === FileStorage.FILE_SYSTEM) return
+
+ await makeOriginalFileAvailable(
+ source.keptOriginalFilename,
+ VideoPathManager.Instance.getFSOriginalVideoFilePath(source.keptOriginalFilename)
+ )
+
+ const oldFileUrl = source.fileUrl
+
+ source.fileUrl = null
+ source.storage = FileStorage.FILE_SYSTEM
+ await source.save()
+
+ logger.debug('Removing original video file %s because it\'s now on file system', oldFileUrl, lTagsBase())
+
+ await removeOriginalFileObjectStorage(source)
+}
+
+async function moveWebVideoFiles (video: MVideoWithAllFiles) {
+ for (const file of video.VideoFiles) {
+ if (file.storage === FileStorage.FILE_SYSTEM) continue
+
+ await makeWebVideoFileAvailable(file.filename, VideoPathManager.Instance.getFSVideoFileOutputPath(video, file))
+ await onVideoFileMoved({
+ videoOrPlaylist: video,
+ file,
+ objetStorageRemover: () => removeWebVideoObjectStorage(file)
+ })
+ }
+}
+
+async function moveHLSFiles (video: MVideoWithAllFiles) {
+ for (const playlist of video.VideoStreamingPlaylists) {
+ const playlistWithVideo = playlist.withVideo(video)
+
+ let updatedFile = false
+
+ for (const file of playlist.VideoFiles) {
+ if (file.storage === FileStorage.FILE_SYSTEM) continue
+
+ updatedFile = true
+
+ // Resolution playlist
+ const playlistFilename = getHLSResolutionPlaylistFilename(file.filename)
+ await makeHLSFileAvailable(playlistWithVideo, playlistFilename, join(getHLSDirectory(video), playlistFilename))
+ await makeHLSFileAvailable(playlistWithVideo, file.filename, join(getHLSDirectory(video), file.filename))
+
+ await onVideoFileMoved({
+ videoOrPlaylist: playlistWithVideo,
+ file,
+ objetStorageRemover: async () => {
+ await removeHLSFileObjectStorageByFilename(playlistWithVideo, playlistFilename)
+ await removeHLSFileObjectStorageByFilename(playlistWithVideo, file.filename)
+ }
+ })
+ }
+
+ if (playlistWithVideo.storage !== FileStorage.FILE_SYSTEM) {
+ await makeHLSFileAvailable(playlistWithVideo, playlist.playlistFilename, join(getHLSDirectory(video), playlist.playlistFilename))
+
+ await makeHLSFileAvailable(
+ playlistWithVideo,
+ playlist.segmentsSha256Filename,
+ join(getHLSDirectory(video), playlist.segmentsSha256Filename)
+ )
+
+ playlist.playlistUrl = null
+ playlist.segmentsSha256Url = null
+ playlist.storage = FileStorage.FILE_SYSTEM
+
+ await playlist.save()
+
+ await removeHLSFileObjectStorageByFilename(playlistWithVideo, playlist.playlistFilename)
+ await removeHLSFileObjectStorageByFilename(playlistWithVideo, playlist.segmentsSha256Filename)
+ }
+
+ if (updatedFile === true) {
+ playlist.assignP2PMediaLoaderInfoHashes(video, playlist.VideoFiles)
+ playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION
+
+ await playlist.save()
+ }
+ }
+}
+
+async function onVideoFileMoved (options: {
+ videoOrPlaylist: MVideo | MStreamingPlaylistVideo
+ file: MVideoFile
+ objetStorageRemover: () => Promise<any>
+}) {
+ const { videoOrPlaylist, file, objetStorageRemover } = options
+
+ const oldFileUrl = file.fileUrl
+
+ file.fileUrl = null
+ file.storage = FileStorage.FILE_SYSTEM
+
+ await updateTorrentMetadata(videoOrPlaylist, file)
+ await file.save()
+
+ logger.debug('Removing web video file %s because it\'s now on file system', oldFileUrl, lTagsBase())
+ await objetStorageRemover()
+}
+
+// ---------------------------------------------------------------------------
+
+async function moveCaptionFiles (captions: MVideoCaption[], hls: MStreamingPlaylistVideo) {
+ let hlsUpdated = false
+
+ for (const caption of captions) {
+ if (caption.storage === FileStorage.OBJECT_STORAGE) {
+ const oldFileUrl = caption.fileUrl
+
+ await makeCaptionFileAvailable(caption.filename, caption.getFSFilePath())
+
+ // Assign new values before building the m3u8 file
+ caption.fileUrl = null
+ caption.storage = FileStorage.FILE_SYSTEM
+
+ await caption.save()
+
+ logger.debug('Removing caption file %s because it\'s now on file system', oldFileUrl, lTagsBase())
+ await removeCaptionObjectStorage(caption)
+ }
+
+ if (hls && (!caption.m3u8Filename || caption.m3u8Url)) {
+ hlsUpdated = true
+
+ const oldM3U8Url = caption.m3u8Url
+ const oldM3U8Filename = caption.m3u8Filename
+
+ // Caption link has been updated, so we must also update the HLS caption playlist
+ caption.m3u8Filename = await upsertCaptionPlaylistOnFS(caption, hls.Video)
+ caption.m3u8Url = null
+
+ await caption.save()
+
+ if (oldM3U8Url) {
+ logger.debug(`Removing video caption playlist file ${oldM3U8Url} because it's now on file system`, lTagsBase())
+
+ await removeHLSFileObjectStorageByFilename(hls, oldM3U8Filename)
+ }
+ }
+ }
+
+ if (hlsUpdated) {
+ await updateHLSMasterOnCaptionChange(hls.Video, hls)
+ }
+}
diff --git a/server/core/lib/move-storage/move-to-object-storage.ts b/server/core/lib/move-storage/move-to-object-storage.ts
new file mode 100644
index 0000000000..9ad2ed7e93
--- /dev/null
+++ b/server/core/lib/move-storage/move-to-object-storage.ts
@@ -0,0 +1,235 @@
+import { FileStorage, VideoStateType } from '@peertube/peertube-models'
+import { logger, LoggerTags, loggerTagsFactory } from '@server/helpers/logger.js'
+import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants.js'
+import { buildCaptionM3U8Content } from '@server/lib/hls.js'
+import {
+ storeHLSFileFromContent,
+ storeHLSFileFromFilename,
+ storeOriginalVideoFile,
+ storeVideoCaption,
+ storeWebVideoFile
+} from '@server/lib/object-storage/index.js'
+import { getHLSDirectory, getHLSResolutionPlaylistFilename } from '@server/lib/paths.js'
+import { updateHLSMasterOnCaptionChange } from '@server/lib/video-captions.js'
+import { VideoPathManager } from '@server/lib/video-path-manager.js'
+import { moveToFailedMoveToObjectStorageState, moveToNextState } from '@server/lib/video-state.js'
+import { updateTorrentMetadata } from '@server/lib/webtorrent.js'
+import { VideoCaptionModel } from '@server/models/video/video-caption.js'
+import { VideoModel } from '@server/models/video/video.js'
+import { MStreamingPlaylistVideo, MVideo, MVideoCaption, MVideoFile, MVideoWithAllFiles } from '@server/types/models/index.js'
+import { MVideoSource } from '@server/types/models/video/video-source.js'
+import { remove } from 'fs-extra/esm'
+import { rmdir } from 'fs/promises'
+import { join } from 'path'
+import { federateVideoIfNeeded } from '../activitypub/videos/federate.js'
+import { moveCaptionToStorage } from './shared/move-caption.js'
+import { moveVideoToStorage, onMoveVideoToStorageFailure } from './shared/move-video.js'
+
+const lTagsBase = loggerTagsFactory('object-storage', 'move-object-storage')
+
+export async function moveVideoToObjectStorage (options: {
+ videoUUID: string
+
+ moveVideoState?: {
+ isNewVideo: boolean
+ previousVideoState: VideoStateType
+ }
+
+ loggerTags: LoggerTags['tags']
+}) {
+ const { videoUUID, moveVideoState, loggerTags } = options
+
+ await moveVideoToStorage({
+ videoUUID,
+ loggerTags: [ ...lTagsBase().tags, ...loggerTags ],
+
+ targetStorage: FileStorage.OBJECT_STORAGE,
+
+ moveWebVideoFiles,
+ moveHLSFiles,
+ moveVideoSourceFile,
+ moveCaptionFiles
+ })
+
+ if (options.moveVideoState) {
+ await moveToNextState({ video: { uuid: videoUUID }, ...moveVideoState })
+ } else {
+ const videoFull = await VideoModel.loadFull(videoUUID)
+ await federateVideoIfNeeded(videoFull, false, undefined)
+ }
+}
+
+export function moveCaptionToObjectStorage (options: {
+ captionId: number
+ loggerTags: LoggerTags['tags']
+}) {
+ const { captionId, loggerTags } = options
+
+ return moveCaptionToStorage({
+ captionId,
+ loggerTags: [ ...lTagsBase().tags, ...loggerTags ],
+ moveCaptionFiles
+ })
+}
+
+export async function onMoveVideoToObjectStorageFailure (options: {
+ videoUUID: string
+ loggerTags: LoggerTags['tags']
+ err: Error
+}) {
+ const { videoUUID, err, loggerTags } = options
+
+ await onMoveVideoToStorageFailure({
+ videoUUID,
+ err,
+ loggerTags: [ ...lTagsBase().tags, ...loggerTags ],
+ moveToFailedState: moveToFailedMoveToObjectStorageState
+ })
+}
+
+// ---------------------------------------------------------------------------
+
+async function moveVideoSourceFile (source: MVideoSource) {
+ if (source.storage !== FileStorage.FILE_SYSTEM) return
+
+ const sourcePath = VideoPathManager.Instance.getFSOriginalVideoFilePath(source.keptOriginalFilename)
+ const fileUrl = await storeOriginalVideoFile(sourcePath, source.keptOriginalFilename)
+
+ source.storage = FileStorage.OBJECT_STORAGE
+ source.fileUrl = fileUrl
+ await source.save()
+
+ logger.debug('Removing original video file ' + sourcePath + ' because it\'s now on object storage', lTagsBase())
+
+ await remove(sourcePath)
+}
+
+// ---------------------------------------------------------------------------
+
+async function moveCaptionFiles (captions: MVideoCaption[], hls: MStreamingPlaylistVideo) {
+ let hlsUpdated = false
+
+ for (const caption of captions) {
+ if (caption.storage === FileStorage.FILE_SYSTEM) {
+ const captionPath = caption.getFSFilePath()
+
+ // Assign new values before building the m3u8 file
+ caption.fileUrl = await storeVideoCaption(captionPath, caption.filename)
+ caption.storage = FileStorage.OBJECT_STORAGE
+
+ await caption.save()
+
+ logger.debug(`Removing video caption file ${captionPath} because it's now on object storage`, lTagsBase())
+ await remove(captionPath)
+ }
+
+ if (hls && (!caption.m3u8Filename || !caption.m3u8Url)) {
+ hlsUpdated = true
+
+ const m3u8PathToRemove = caption.getFSM3U8Path(hls.Video)
+
+ // Caption file URL has been updated, so we must also update the HLS caption playlist
+ const content = buildCaptionM3U8Content({ video: hls.Video, caption })
+
+ caption.m3u8Filename = VideoCaptionModel.generateM3U8Filename(caption.filename)
+ caption.m3u8Url = await storeHLSFileFromContent({
+ playlist: hls,
+ pathOrFilename: caption.m3u8Filename,
+ content
+ })
+
+ await caption.save()
+
+ if (m3u8PathToRemove) {
+ logger.debug(`Removing video caption playlist file ${m3u8PathToRemove} because it's now on object storage`, lTagsBase())
+ await remove(m3u8PathToRemove)
+ }
+ }
+ }
+
+ if (hlsUpdated) {
+ await updateHLSMasterOnCaptionChange(hls.Video, hls)
+ }
+}
+
+// ---------------------------------------------------------------------------
+
+async function moveWebVideoFiles (video: MVideoWithAllFiles) {
+ for (const file of video.VideoFiles) {
+ if (file.storage !== FileStorage.FILE_SYSTEM) continue
+
+ const fileUrl = await storeWebVideoFile(video, file)
+
+ const oldPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, file)
+ await onVideoFileMoved({ videoOrPlaylist: video, file, fileUrl, oldPath })
+ }
+}
+
+async function moveHLSFiles (video: MVideoWithAllFiles) {
+ for (const playlist of video.VideoStreamingPlaylists) {
+ const playlistWithVideo = playlist.withVideo(video)
+
+ let updatedFile = false
+
+ for (const file of playlist.VideoFiles) {
+ if (file.storage !== FileStorage.FILE_SYSTEM) continue
+
+ updatedFile = true
+
+ // Resolution playlist
+ const playlistFilename = getHLSResolutionPlaylistFilename(file.filename)
+ await storeHLSFileFromFilename(playlistWithVideo, playlistFilename)
+
+ // Resolution fragmented file
+ const fileUrl = await storeHLSFileFromFilename(playlistWithVideo, file.filename)
+
+ const oldPath = join(getHLSDirectory(video), file.filename)
+
+ await onVideoFileMoved({ videoOrPlaylist: Object.assign(playlist, { Video: video }), file, fileUrl, oldPath })
+
+ await remove(join(getHLSDirectory(video), playlistFilename))
+ }
+
+ if (playlistWithVideo.storage === FileStorage.FILE_SYSTEM) {
+ playlist.playlistUrl = await storeHLSFileFromFilename(playlistWithVideo, playlist.playlistFilename)
+ playlist.segmentsSha256Url = await storeHLSFileFromFilename(playlistWithVideo, playlist.segmentsSha256Filename)
+ playlist.storage = FileStorage.OBJECT_STORAGE
+
+ await playlist.save()
+
+ await remove(join(getHLSDirectory(video), playlist.playlistFilename))
+ await remove(join(getHLSDirectory(video), playlist.segmentsSha256Filename))
+ }
+
+ if (updatedFile === true) {
+ playlist.assignP2PMediaLoaderInfoHashes(video, playlist.VideoFiles)
+ playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION
+
+ await playlist.save()
+ }
+ }
+
+ try {
+ await rmdir(getHLSDirectory(video))
+ } catch {
+ // Nothing to do, directory may be not empty if there is a transcoding in progress
+ }
+}
+
+async function onVideoFileMoved (options: {
+ videoOrPlaylist: MVideo | MStreamingPlaylistVideo
+ file: MVideoFile
+ fileUrl: string
+ oldPath: string
+}) {
+ const { videoOrPlaylist, file, fileUrl, oldPath } = options
+
+ file.fileUrl = fileUrl
+ file.storage = FileStorage.OBJECT_STORAGE
+
+ await updateTorrentMetadata(videoOrPlaylist, file)
+ await file.save()
+
+ logger.debug('Removing %s because it\'s now on object storage', oldPath, lTagsBase())
+ await remove(oldPath)
+}
diff --git a/server/core/lib/job-queue/handlers/shared/move-caption.ts b/server/core/lib/move-storage/shared/move-caption.ts
similarity index 84%
rename from server/core/lib/job-queue/handlers/shared/move-caption.ts
rename to server/core/lib/move-storage/shared/move-caption.ts
index bf03bb7588..7158c60512 100644
--- a/server/core/lib/job-queue/handlers/shared/move-caption.ts
+++ b/server/core/lib/move-storage/shared/move-caption.ts
@@ -1,5 +1,5 @@
import { retryTransactionWrapper } from '@server/helpers/database-utils.js'
-import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
+import { logger, LoggerTags, loggerTagsFactory } from '@server/helpers/logger.js'
import { sequelizeTypescript } from '@server/initializers/database.js'
import { federateVideoIfNeeded } from '@server/lib/activitypub/videos/federate.js'
import { VideoPathManager } from '@server/lib/video-path-manager.js'
@@ -8,15 +8,13 @@ import { VideoStreamingPlaylistModel } from '@server/models/video/video-streamin
import { VideoModel } from '@server/models/video/video.js'
import { MStreamingPlaylistVideoUUID, MVideoCaption } from '@server/types/models/index.js'
-export async function moveCaptionToStorageJob (options: {
- jobId: string
+export async function moveCaptionToStorage (options: {
captionId: number
- loggerTags: (number | string)[]
+ loggerTags: LoggerTags['tags']
moveCaptionFiles: (captions: MVideoCaption[], hls: MStreamingPlaylistVideoUUID) => Promise<void>
}) {
const {
- jobId,
loggerTags,
captionId,
moveCaptionFiles
@@ -27,7 +25,7 @@ export async function moveCaptionToStorageJob (options: {
const caption = await VideoCaptionModel.loadWithVideo(captionId)
if (!caption) {
- logger.info(`Can't process job ${jobId}, caption does not exist anymore.`, lTagsBase())
+ logger.info(`Can't process caption ${captionId}, caption does not exist anymore.`, lTagsBase())
return
}
diff --git a/server/core/lib/job-queue/handlers/shared/move-video.ts b/server/core/lib/move-storage/shared/move-video.ts
similarity index 52%
rename from server/core/lib/job-queue/handlers/shared/move-video.ts
rename to server/core/lib/move-storage/shared/move-video.ts
index e0bdb1b2d3..aa2ae6e403 100644
--- a/server/core/lib/job-queue/handlers/shared/move-video.ts
+++ b/server/core/lib/move-storage/shared/move-video.ts
@@ -1,33 +1,32 @@
+import { FileStorage, FileStorageType } from '@peertube/peertube-models'
import { LoggerTags, logger, loggerTagsFactory } from '@server/helpers/logger.js'
import { VideoPathManager } from '@server/lib/video-path-manager.js'
import { VideoCaptionModel } from '@server/models/video/video-caption.js'
import { VideoJobInfoModel } from '@server/models/video/video-job-info.js'
import { VideoSourceModel } from '@server/models/video/video-source.js'
import { VideoModel } from '@server/models/video/video.js'
-import { MStreamingPlaylistVideoUUID, MVideoCaption, MVideoWithAllFiles } from '@server/types/models/index.js'
+import { MStreamingPlaylistVideoUUID, MVideo, MVideoCaption, MVideoWithAllFiles } from '@server/types/models/index.js'
import { MVideoSource } from '@server/types/models/video/video-source.js'
-export async function moveVideoToStorageJob (options: {
- jobId: string
+export async function moveVideoToStorage (options: {
videoUUID: string
- loggerTags: (number | string)[]
+ loggerTags: LoggerTags['tags']
+
+ targetStorage: FileStorageType
moveWebVideoFiles: (video: MVideoWithAllFiles) => Promise<void>
moveHLSFiles: (video: MVideoWithAllFiles) => Promise<void>
moveVideoSourceFile: (source: MVideoSource) => Promise<void>
moveCaptionFiles: (captions: MVideoCaption[], hls: MStreamingPlaylistVideoUUID) => Promise<void>
-
- doAfterLastMove: (video: MVideoWithAllFiles) => Promise<void>
}) {
const {
- jobId,
loggerTags,
videoUUID,
moveVideoSourceFile,
moveHLSFiles,
moveWebVideoFiles,
moveCaptionFiles,
- doAfterLastMove
+ targetStorage
} = options
const lTagsBase = loggerTagsFactory(...loggerTags)
@@ -37,7 +36,7 @@ export async function moveVideoToStorageJob (options: {
const video = await VideoModel.loadWithFiles(videoUUID)
// No video, maybe deleted?
if (!video) {
- logger.info(`Can't process job ${jobId}, video does not exist.`, lTagsBase(videoUUID))
+ logger.info(`Can't move video ${videoUUID}, video does not exist.`, lTagsBase(videoUUID))
fileMutexReleaser()
return undefined
}
@@ -45,42 +44,36 @@ export async function moveVideoToStorageJob (options: {
const lTags = lTagsBase(video.uuid, video.url)
try {
- const source = await VideoSourceModel.loadLatest(video.id)
- if (source?.keptOriginalFilename) {
+ const { source, captions, hls, webFiles } = await filterVideoResourcesToBeMoved(video, targetStorage)
+
+ if (captions.length !== 0) {
+ logger.debug(`Moving ${captions.length} captions of ${video.uuid}.`, lTags)
+
+ const hls = video.getHLSPlaylist()
+ await moveCaptionFiles(captions, hls)
+ }
+
+ if (source) {
logger.debug(`Moving video source ${source.keptOriginalFilename} file of video ${video.uuid}`, lTags)
await moveVideoSourceFile(source)
}
- if (video.VideoFiles) {
- logger.debug(`Moving ${video.VideoFiles.length} web video files for video ${video.uuid}.`, lTags)
+ if (webFiles.length !== 0) {
+ logger.debug(`Moving ${webFiles.length} web video files for video ${video.uuid}.`, lTags)
await moveWebVideoFiles(video)
}
- if (video.VideoStreamingPlaylists) {
+ if (hls) {
logger.debug(`Moving HLS playlist of ${video.uuid}.`, lTags)
await moveHLSFiles(video)
}
- const captions = await VideoCaptionModel.listVideoCaptions(video.id)
- if (captions.length !== 0) {
- logger.debug(`Moving ${captions.length} captions of ${video.uuid}.`, lTags)
-
- const hls = video.getHLSPlaylist()
- await moveCaptionFiles(captions, hls)
- }
-
const pendingMove = await VideoJobInfoModel.decrease(video.uuid, 'pendingMove')
- logger.info(`Moved video ${video.uuid}. Checking pending move.`, lTags, { pendingMove })
-
- if (pendingMove === 0) {
- logger.info(`Running cleanup after moving files (video ${video.uuid} in job ${jobId})`, lTags)
-
- await doAfterLastMove(video)
- }
+ logger.info(`Moved video ${video.uuid}. Remaining pending move: ${pendingMove}.`, lTags)
} finally { // Error handling is managed by the job queue
fileMutexReleaser()
}
@@ -89,16 +82,54 @@ export async function moveVideoToStorageJob (options: {
export async function onMoveVideoToStorageFailure (options: {
videoUUID: string
err: any
- lTags: LoggerTags
+ loggerTags: LoggerTags['tags']
moveToFailedState: (video: MVideoWithAllFiles) => Promise<void>
}) {
- const { videoUUID, err, lTags, moveToFailedState } = options
+ const { videoUUID, err, loggerTags, moveToFailedState } = options
const video = await VideoModel.loadWithFiles(videoUUID)
if (!video) return
- logger.error(`Cannot move video ${video.url} storage.`, { err, ...lTags })
+ logger.error(`Cannot move video ${video.url} storage.`, { err, tags: loggerTags })
await moveToFailedState(video)
await VideoJobInfoModel.abortAllTasks(video.uuid, 'pendingMove')
}
+
+export async function filterVideoResourcesToBeMoved (videoArg: MVideo, targetStorage: FileStorageType) {
+ const video = await VideoModel.loadFull(videoArg.id)
+ const captions = await VideoCaptionModel.listVideoCaptions(video.id)
+ const source = await VideoSourceModel.loadLatest(video.id)
+
+ const hls = video.getHLSPlaylist()
+
+ const moveHLS = hls && (hls.storage !== targetStorage || hls.VideoFiles.some(f => f.storage !== targetStorage))
+
+ return {
+ source: source?.keptOriginalFilename && source.storage !== targetStorage
+ ? source
+ : undefined,
+
+ hls: moveHLS
+ ? hls
+ : undefined,
+
+ webFiles: video.VideoFiles.filter(f => f.storage !== targetStorage),
+ captions: captions.filter(c => {
+ if (c.storage !== targetStorage) return true
+
+ if (hls) {
+ if (targetStorage === FileStorage.OBJECT_STORAGE) return !c.m3u8Filename || !c.m3u8Url
+ else if (targetStorage === FileStorage.FILE_SYSTEM) return !c.m3u8Filename || c.m3u8Url
+ }
+
+ return false
+ })
+ }
+}
+
+export async function hasVideoResourcesToBeMoved (video: MVideo, targetStorage: FileStorageType) {
+ const { captions, hls, source, webFiles } = await filterVideoResourcesToBeMoved(video, targetStorage)
+
+ return captions.length !== 0 || !!hls || !!source || webFiles.length !== 0
+}
diff --git a/server/core/lib/runners/job-handlers/live-rtmp-hls-transcoding-job-handler.ts b/server/core/lib/runners/job-handlers/live-rtmp-hls-transcoding-job-handler.ts
index e0fd1f78db..a596c83a2c 100644
--- a/server/core/lib/runners/job-handlers/live-rtmp-hls-transcoding-job-handler.ts
+++ b/server/core/lib/runners/job-handlers/live-rtmp-hls-transcoding-job-handler.ts
@@ -36,8 +36,9 @@ type CreateOptions = {
}
// eslint-disable-next-line max-len
-export class LiveRTMPHLSTranscodingJobHandler extends AbstractJobHandler<CreateOptions, LiveRTMPHLSTranscodingUpdatePayload, LiveRTMPHLSTranscodingSuccess> {
-
+export class LiveRTMPHLSTranscodingJobHandler
+ extends AbstractJobHandler<CreateOptions, LiveRTMPHLSTranscodingUpdatePayload, LiveRTMPHLSTranscodingSuccess>
+{
async create (options: CreateOptions) {
const { video, rtmpUrl, toTranscode, playlist, segmentDuration, segmentListSize, outputDirectory, sessionId } = options
@@ -65,7 +66,7 @@ export class LiveRTMPHLSTranscodingJobHandler extends AbstractJobHandler<CreateO
jobUUID,
payload,
privatePayload,
- priority: JOB_PRIORITY.TRANSCODING
+ priority: JOB_PRIORITY.REQUIRED_TRANSCODING
})
return job
@@ -106,7 +107,9 @@ export class LiveRTMPHLSTranscodingJobHandler extends AbstractJobHandler<CreateO
logger.debug(
'Runner live RTMP to HLS job %s for %s updated.',
- runnerJob.uuid, videoUUID, { updatePayload, ...this.lTags(videoUUID, runnerJob.uuid) }
+ runnerJob.uuid,
+ videoUUID,
+ { updatePayload, ...this.lTags(videoUUID, runnerJob.uuid) }
)
}
diff --git a/server/core/lib/runners/job-handlers/shared/utils.ts b/server/core/lib/runners/job-handlers/shared/utils.ts
index fa2bdfd7cc..102dbdc374 100644
--- a/server/core/lib/runners/job-handlers/shared/utils.ts
+++ b/server/core/lib/runners/job-handlers/shared/utils.ts
@@ -1,7 +1,7 @@
import {
RunnerJobVODAudioMergeTranscodingPrivatePayload,
RunnerJobVODWebVideoTranscodingPrivatePayload,
- VideoResolution
+ VideoFileStreamType
} from '@peertube/peertube-models'
import { logger, LoggerTagsFn } from '@server/helpers/logger.js'
import { onTranscodingEnded } from '@server/lib/transcoding/ended-transcoding.js'
@@ -24,7 +24,7 @@ export async function onVODWebVideoOrAudioMergeTranscodingJob (options: {
await onWebVideoFileTranscoding({ video, videoOutputPath: videoFilePath, deleteWebInputVideoFile, wasAudioFile })
- await onTranscodingEnded({ isNewVideo: privatePayload.isNewVideo, moveVideoToNextState: true, video })
+ await onTranscodingEnded({ isNewVideo: privatePayload.isNewVideo, moveVideoToNextState: privatePayload.canMoveVideoState, video })
}
export async function loadRunnerVideo (runnerJob: MRunnerJob, lTags: LoggerTagsFn) {
@@ -39,18 +39,24 @@ export async function loadRunnerVideo (runnerJob: MRunnerJob, lTags: LoggerTagsF
return video
}
-export async function isHLSAudioMissing (options: {
- resolution: number
- separatedAudio: boolean
+export async function hasMissingHLSStreams (options: {
+ inputStreams: VideoFileStreamType[]
+ transcodingRequestAt: string
videoId: string | number
}) {
- if (!options.separatedAudio) return false
+ const { videoId, inputStreams, transcodingRequestAt } = options
- if (options.resolution !== VideoResolution.H_NOVIDEO) {
- const video = await VideoModel.loadFull(options.videoId)
+ const video = await VideoModel.loadFull(videoId)
+ const hlsFiles = video.getHLSPlaylist().VideoFiles
- // Video doesn't have audio file yet
- if (video.hasAudio() !== true) return true
+ for (const inputStream of inputStreams) {
+ const hasStream = hlsFiles.some(f => {
+ // Compare creation dates to avoid using files created before the root job (e.g., from a previous transcoding)
+ return new Date(f.createdAt).getTime() >= new Date(transcodingRequestAt).getTime() &&
+ (f.streams & inputStream)
+ })
+
+ if (!hasStream) return true
}
return false
diff --git a/server/core/lib/runners/job-handlers/vod-audio-merge-transcoding-job-handler.ts b/server/core/lib/runners/job-handlers/vod-audio-merge-transcoding-job-handler.ts
index 5933501f51..de5035aeb8 100644
--- a/server/core/lib/runners/job-handlers/vod-audio-merge-transcoding-job-handler.ts
+++ b/server/core/lib/runners/job-handlers/vod-audio-merge-transcoding-job-handler.ts
@@ -21,12 +21,14 @@ type CreateOptions = {
fps: number
priority: number
deleteInputFileId: number | null
+ canMoveVideoState: boolean
dependsOnRunnerJob?: MRunnerJob
}
// eslint-disable-next-line max-len
-export class VODAudioMergeTranscodingJobHandler extends AbstractVODTranscodingJobHandler<CreateOptions, RunnerJobUpdatePayload, VODAudioMergeTranscodingSuccess> {
-
+export class VODAudioMergeTranscodingJobHandler
+ extends AbstractVODTranscodingJobHandler<CreateOptions, RunnerJobUpdatePayload, VODAudioMergeTranscodingSuccess>
+{
async create (options: CreateOptions) {
const { video, resolution, fps, priority, dependsOnRunnerJob } = options
@@ -43,7 +45,7 @@ export class VODAudioMergeTranscodingJobHandler extends AbstractVODTranscodingJo
}
const privatePayload: RunnerJobVODWebVideoTranscodingPrivatePayload = {
- ...pick(options, [ 'isNewVideo', 'deleteInputFileId' ]),
+ ...pick(options, [ 'isNewVideo', 'deleteInputFileId', 'canMoveVideoState' ]),
videoUUID: video.uuid
}
@@ -80,7 +82,9 @@ export class VODAudioMergeTranscodingJobHandler extends AbstractVODTranscodingJo
logger.info(
'Runner VOD audio merge transcoding job %s for %s ended.',
- runnerJob.uuid, video.uuid, this.lTags(video.uuid, runnerJob.uuid)
+ runnerJob.uuid,
+ video.uuid,
+ this.lTags(video.uuid, runnerJob.uuid)
)
}
}
diff --git a/server/core/lib/runners/job-handlers/vod-hls-transcoding-job-handler.ts b/server/core/lib/runners/job-handlers/vod-hls-transcoding-job-handler.ts
index 6ac3c67f0f..bee6b8376e 100644
--- a/server/core/lib/runners/job-handlers/vod-hls-transcoding-job-handler.ts
+++ b/server/core/lib/runners/job-handlers/vod-hls-transcoding-job-handler.ts
@@ -3,6 +3,7 @@ import {
RunnerJobUpdatePayload,
RunnerJobVODHLSTranscodingPayload,
RunnerJobVODHLSTranscodingPrivatePayload,
+ VideoFileStreamType,
VODHLSTranscodingSuccess
} from '@peertube/peertube-models'
import { buildUUID } from '@peertube/peertube-node-utils'
@@ -15,7 +16,7 @@ import { MVideoWithFile } from '@server/types/models/index.js'
import { MRunnerJob } from '@server/types/models/runners/index.js'
import { generateRunnerTranscodingAudioInputFileUrl, generateRunnerTranscodingVideoInputFileUrl } from '../runner-urls.js'
import { AbstractVODTranscodingJobHandler } from './abstract-vod-transcoding-job-handler.js'
-import { isHLSAudioMissing, loadRunnerVideo } from './shared/utils.js'
+import { hasMissingHLSStreams, loadRunnerVideo } from './shared/utils.js'
type CreateOptions = {
video: MVideoWithFile
@@ -25,6 +26,11 @@ type CreateOptions = {
fps: number
priority: number
separatedAudio: boolean
+
+ inputStreams: VideoFileStreamType[]
+ transcodingRequestAt: string
+ canMoveVideoState: boolean
+
dependsOnRunnerJob?: MRunnerJob
}
@@ -55,7 +61,7 @@ export class VODHLSTranscodingJobHandler
}
const privatePayload: RunnerJobVODHLSTranscodingPrivatePayload = {
- ...pick(options, [ 'isNewVideo', 'deleteWebVideoFiles' ]),
+ ...pick(options, [ 'isNewVideo', 'deleteWebVideoFiles', 'inputStreams', 'canMoveVideoState', 'transcodingRequestAt' ]),
videoUUID: video.uuid
}
@@ -81,7 +87,6 @@ export class VODHLSTranscodingJobHandler
resultPayload: VODHLSTranscodingSuccess
}) {
const { runnerJob, resultPayload } = options
- const payload = runnerJob.payload as RunnerJobVODHLSTranscodingPayload
const privatePayload = runnerJob.privatePayload as RunnerJobVODHLSTranscodingPrivatePayload
const video = await loadRunnerVideo(runnerJob, this.lTags)
@@ -97,20 +102,24 @@ export class VODHLSTranscodingJobHandler
})
// Splitted audio? Wait audio generation before moving the video in its next state
- const missingAudio = await isHLSAudioMissing({
- resolution: payload.output.resolution,
- separatedAudio: payload.output.separatedAudio,
+ const missingStream = await hasMissingHLSStreams({
+ inputStreams: privatePayload.inputStreams,
+ transcodingRequestAt: privatePayload.transcodingRequestAt,
videoId: video.uuid
})
- await onTranscodingEnded({ isNewVideo: privatePayload.isNewVideo, moveVideoToNextState: !missingAudio, video })
-
- if (!missingAudio && privatePayload.deleteWebVideoFiles === true) {
+ if (!missingStream && privatePayload.deleteWebVideoFiles === true) {
logger.info('Removing web video files of %s now we have a HLS version of it.', video.uuid, this.lTags(video.uuid))
await removeAllWebVideoFiles(video)
}
+ await onTranscodingEnded({
+ isNewVideo: privatePayload.isNewVideo,
+ moveVideoToNextState: privatePayload.canMoveVideoState && !missingStream,
+ video
+ })
+
logger.info('Runner VOD HLS job %s for %s ended.', runnerJob.uuid, video.uuid, this.lTags(runnerJob.uuid, video.uuid))
}
}
diff --git a/server/core/lib/runners/job-handlers/vod-web-video-transcoding-job-handler.ts b/server/core/lib/runners/job-handlers/vod-web-video-transcoding-job-handler.ts
index 3be7306624..881c31a065 100644
--- a/server/core/lib/runners/job-handlers/vod-web-video-transcoding-job-handler.ts
+++ b/server/core/lib/runners/job-handlers/vod-web-video-transcoding-job-handler.ts
@@ -21,12 +21,14 @@ type CreateOptions = {
fps: number
priority: number
deleteInputFileId: number | null
+ canMoveVideoState: boolean
dependsOnRunnerJob?: MRunnerJob
}
// eslint-disable-next-line max-len
-export class VODWebVideoTranscodingJobHandler extends AbstractVODTranscodingJobHandler<CreateOptions, RunnerJobUpdatePayload, VODWebVideoTranscodingSuccess> {
-
+export class VODWebVideoTranscodingJobHandler
+ extends AbstractVODTranscodingJobHandler<CreateOptions, RunnerJobUpdatePayload, VODWebVideoTranscodingSuccess>
+{
async create (options: CreateOptions) {
const { video, resolution, fps, priority, dependsOnRunnerJob } = options
@@ -48,7 +50,7 @@ export class VODWebVideoTranscodingJobHandler extends AbstractVODTranscodingJobH
}
const privatePayload: RunnerJobVODWebVideoTranscodingPrivatePayload = {
- ...pick(options, [ 'isNewVideo', 'deleteInputFileId' ]),
+ ...pick(options, [ 'isNewVideo', 'deleteInputFileId', 'canMoveVideoState' ]),
videoUUID: video.uuid
}
@@ -85,7 +87,9 @@ export class VODWebVideoTranscodingJobHandler extends AbstractVODTranscodingJobH
logger.info(
'Runner VOD web video transcoding job %s for %s ended.',
- runnerJob.uuid, video.uuid, this.lTags(video.uuid, runnerJob.uuid)
+ runnerJob.uuid,
+ video.uuid,
+ this.lTags(video.uuid, runnerJob.uuid)
)
}
}
diff --git a/server/core/lib/transcoding/create-transcoding-job.ts b/server/core/lib/transcoding/create-transcoding-job.ts
index b13d2b8ac9..f2c70c624c 100644
--- a/server/core/lib/transcoding/create-transcoding-job.ts
+++ b/server/core/lib/transcoding/create-transcoding-job.ts
@@ -7,7 +7,6 @@ export function createOptimizeOrMergeAudioJobs (options: {
videoFile: MVideoFile
isNewVideo: boolean
user: MUserId
- videoFileAlreadyLocked: boolean
}) {
return getJobBuilder().createOptimizeOrMergeAudioJobs(options)
}
diff --git a/server/core/lib/transcoding/ended-transcoding.ts b/server/core/lib/transcoding/ended-transcoding.ts
index 468a3aa830..b9ee961379 100644
--- a/server/core/lib/transcoding/ended-transcoding.ts
+++ b/server/core/lib/transcoding/ended-transcoding.ts
@@ -1,6 +1,11 @@
+import { FileStorage } from '@peertube/peertube-models'
import { retryTransactionWrapper } from '@server/helpers/database-utils.js'
+import { CONFIG } from '@server/initializers/config.js'
import { VideoJobInfoModel } from '@server/models/video/video-job-info.js'
import { MVideo } from '@server/types/models/index.js'
+import { JobQueue } from '../job-queue/job-queue.js'
+import { hasVideoResourcesToBeMoved } from '../move-storage/shared/move-video.js'
+import { buildMoveVideoJob } from '../video-jobs.js'
import { moveToNextState } from '../video-state.js'
export async function onTranscodingEnded (options: {
@@ -13,6 +18,11 @@ export async function onTranscodingEnded (options: {
await VideoJobInfoModel.decrease(video.uuid, 'pendingTranscode')
if (moveVideoToNextState) {
- await retryTransactionWrapper(moveToNextState, { video, isNewVideo })
+ const changedState = await retryTransactionWrapper(moveToNextState, { video, isNewVideo })
+
+ // Still send the transcoded file to external storage if needed
+ if (!changedState && CONFIG.OBJECT_STORAGE.ENABLED && await hasVideoResourcesToBeMoved(video, FileStorage.OBJECT_STORAGE)) {
+ await JobQueue.Instance.createJob(await buildMoveVideoJob({ type: 'move-to-object-storage', video }))
+ }
}
}
diff --git a/server/core/lib/transcoding/hls-transcoding.ts b/server/core/lib/transcoding/hls-transcoding.ts
index bc1b2211bc..f25af760c3 100644
--- a/server/core/lib/transcoding/hls-transcoding.ts
+++ b/server/core/lib/transcoding/hls-transcoding.ts
@@ -1,8 +1,8 @@
import { pick } from '@peertube/peertube-core-utils'
-import { getVideoStreamDuration, HLSFromTSTranscodeOptions, HLSTranscodeOptions } from '@peertube/peertube-ffmpeg'
+import { canCopyForHLS, getVideoStreamDuration, HLSFromTSTranscodeOptions, HLSTranscodeOptions } from '@peertube/peertube-ffmpeg'
import { retryTransactionWrapper } from '@server/helpers/database-utils.js'
-import { createTorrentAndSetInfoHash } from '@server/lib/webtorrent.js'
import { sequelizeTypescript } from '@server/initializers/database.js'
+import { createTorrentAndSetInfoHash } from '@server/lib/webtorrent.js'
import { MVideo } from '@server/types/models/index.js'
import { MutexInterface } from 'async-mutex'
import { Job } from 'bullmq'
@@ -13,10 +13,10 @@ import { VideoFileModel } from '../../models/video/video-file.js'
import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist.js'
import { renameVideoFileInPlaylist, updateM3U8AndShaPlaylist } from '../hls.js'
import { generateHLSVideoFilename, getHLSResolutionPlaylistFilename } from '../paths.js'
+import { createAllCaptionPlaylistsOnFSIfNeeded } from '../video-captions.js'
import { buildNewFile } from '../video-file.js'
import { VideoPathManager } from '../video-path-manager.js'
import { buildFFmpegVOD } from './shared/index.js'
-import { createAllCaptionPlaylistsOnFSIfNeeded } from '../video-captions.js'
// Concat TS segments from a live video to a fragmented mp4 HLS playlist
export async function generateHlsPlaylistResolutionFromTS (options: {
@@ -45,7 +45,6 @@ export function generateHlsPlaylistResolution (options: {
resolution: number
fps: number
- copyCodecs: boolean
inputFileMutexReleaser: MutexInterface.Releaser
separatedAudio: boolean
job?: Job
@@ -59,7 +58,6 @@ export function generateHlsPlaylistResolution (options: {
'video',
'resolution',
'fps',
- 'copyCodecs',
'separatedAudio',
'inputFileMutexReleaser',
'job'
@@ -158,7 +156,6 @@ async function generateHlsPlaylistCommon (options: {
separatedAudio?: boolean
- copyCodecs?: boolean
isAAC?: boolean
job?: Job
@@ -170,7 +167,6 @@ async function generateHlsPlaylistCommon (options: {
separatedAudioInputPath,
resolution,
fps,
- copyCodecs,
separatedAudio,
isAAC,
job,
@@ -198,7 +194,9 @@ async function generateHlsPlaylistCommon (options: {
resolution,
fps,
- copyCodecs,
+
+ copyCodecs: !separatedAudioInputPath && await canCopyForHLS({ fps, resolution, path: videoInputPath }),
+
separatedAudio,
isAAC,
diff --git a/server/core/lib/transcoding/shared/job-builders/abstract-job-builder.ts b/server/core/lib/transcoding/shared/job-builders/abstract-job-builder.ts
index 90d19c005b..b972205a88 100644
--- a/server/core/lib/transcoding/shared/job-builders/abstract-job-builder.ts
+++ b/server/core/lib/transcoding/shared/job-builders/abstract-job-builder.ts
@@ -1,136 +1,134 @@
-import { ffprobePromise } from '@peertube/peertube-ffmpeg'
-import { VideoResolution } from '@peertube/peertube-models'
+import { VideoFileStreamType, VideoResolution } from '@peertube/peertube-models'
import { computeOutputFPS } from '@server/helpers/ffmpeg/framerate.js'
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
import { CONFIG } from '@server/initializers/config.js'
import { DEFAULT_AUDIO_MERGE_RESOLUTION, DEFAULT_AUDIO_RESOLUTION } from '@server/initializers/constants.js'
import { Hooks } from '@server/lib/plugins/hooks.js'
-import { VideoPathManager } from '@server/lib/video-path-manager.js'
import { MUserId, MVideoFile, MVideoFullLight } from '@server/types/models/index.js'
-import { canDoQuickTranscode } from '../../transcoding-quick-transcode.js'
import { buildOriginalFileResolution, computeResolutionsToTranscode } from '../../transcoding-resolutions.js'
const lTags = loggerTagsFactory('transcoding')
-export abstract class AbstractJobBuilder<P> {
+export type TranscodingPriorityType = 'required' | 'optional'
+
+export abstract class AbstractJobBuilder<P extends { transcodingPriority: TranscodingPriorityType }> {
async createOptimizeOrMergeAudioJobs (options: {
video: MVideoFullLight
videoFile: MVideoFile
isNewVideo: boolean
user: MUserId
- videoFileAlreadyLocked: boolean
}) {
- const { video, videoFile, isNewVideo, user, videoFileAlreadyLocked } = options
+ const { video, videoFile, isNewVideo, user } = options
- let mergeOrOptimizePayload: P & { higherPriority?: boolean }
- let children: (P & { higherPriority?: boolean })[][] = []
+ let mergeOrOptimizePayload: P
+ let children: P[][] = []
- const mutexReleaser = videoFileAlreadyLocked
- ? () => {}
- : await VideoPathManager.Instance.lockFiles(video.uuid)
+ const inputStreams = video.getStreamTypes()
+ const transcodingRequestAt = new Date().toISOString()
- try {
- await video.reload()
- await videoFile.reload()
+ let maxFPS: number
+ let maxResolution: number
- await VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(video), async videoFilePath => {
- const probe = await ffprobePromise(videoFilePath)
- const quickTranscode = await canDoQuickTranscode(videoFilePath, CONFIG.TRANSCODING.FPS.MAX, probe)
+ let hlsAudioAlreadyGenerated = false
- let maxFPS: number
- let maxResolution: number
+ if (videoFile.isAudio()) {
+ // The first transcoding job will transcode to this FPS value
+ maxFPS = Math.min(DEFAULT_AUDIO_MERGE_RESOLUTION, CONFIG.TRANSCODING.FPS.MAX)
+ maxResolution = DEFAULT_AUDIO_RESOLUTION
- let hlsAudioAlreadyGenerated = false
+ mergeOrOptimizePayload = this.buildMergeAudioPayload({
+ video,
+ isNewVideo,
+ inputFile: videoFile,
+ resolution: maxResolution,
+ fps: maxFPS,
- if (videoFile.isAudio()) {
- // The first transcoding job will transcode to this FPS value
- maxFPS = Math.min(DEFAULT_AUDIO_MERGE_RESOLUTION, CONFIG.TRANSCODING.FPS.MAX)
- maxResolution = DEFAULT_AUDIO_RESOLUTION
+ transcodingPriority: 'required',
+ canMoveVideoState: null // Will be set below
+ })
+ } else {
+ const inputFPS = videoFile.fps
+ maxResolution = buildOriginalFileResolution(videoFile.resolution)
+ maxFPS = computeOutputFPS({ inputFPS, resolution: maxResolution, isOriginResolution: true, type: 'vod' })
+
+ mergeOrOptimizePayload = this.buildOptimizePayload({
+ video,
+ isNewVideo,
+ inputFile: videoFile,
+ resolution: maxResolution,
+ fps: maxFPS,
+
+ transcodingPriority: 'required',
+ canMoveVideoState: null // Will be set below
+ })
+ }
- mergeOrOptimizePayload = this.buildMergeAudioPayload({
- video,
- isNewVideo,
- inputFile: videoFile,
- resolution: maxResolution,
- fps: maxFPS
- })
- } else {
- const inputFPS = videoFile.fps
- maxResolution = buildOriginalFileResolution(videoFile.resolution)
- maxFPS = computeOutputFPS({ inputFPS, resolution: maxResolution, isOriginResolution: true, type: 'vod' })
+ // HLS version of max resolution
+ if (CONFIG.TRANSCODING.HLS.ENABLED === true) {
+ const hasSplitAudioTranscoding = CONFIG.TRANSCODING.HLS.SPLIT_AUDIO_AND_VIDEO && videoFile.hasAudio()
- mergeOrOptimizePayload = this.buildOptimizePayload({
- video,
- isNewVideo,
- quickTranscode,
- inputFile: videoFile,
- resolution: maxResolution,
- fps: maxFPS
- })
- }
+ children.push([
+ this.buildHLSJobPayload({
+ deleteWebVideoFiles: !CONFIG.TRANSCODING.WEB_VIDEOS.ENABLED,
- // HLS version of max resolution
- if (CONFIG.TRANSCODING.HLS.ENABLED === true) {
- const hasSplitAudioTranscoding = CONFIG.TRANSCODING.HLS.SPLIT_AUDIO_AND_VIDEO && videoFile.hasAudio()
+ separatedAudio: hasSplitAudioTranscoding,
- const hlsPayloads: (P & { higherPriority?: boolean })[] = []
+ resolution: maxResolution,
+ fps: maxFPS,
+ video,
+ isNewVideo,
- hlsPayloads.push(
- this.buildHLSJobPayload({
- deleteWebVideoFiles: !CONFIG.TRANSCODING.WEB_VIDEOS.ENABLED,
+ inputStreams,
+ transcodingRequestAt,
- separatedAudio: hasSplitAudioTranscoding,
+ transcodingPriority: 'required',
+ canMoveVideoState: true
+ })
+ ])
- copyCodecs: true,
+ if (hasSplitAudioTranscoding) {
+ hlsAudioAlreadyGenerated = true
- resolution: maxResolution,
- fps: maxFPS,
- video,
- isNewVideo
- })
- )
-
- if (hasSplitAudioTranscoding) {
- hlsAudioAlreadyGenerated = true
-
- hlsPayloads.unshift(
- {
- higherPriority: true,
-
- ...this.buildHLSJobPayload({
- deleteWebVideoFiles: false,
- separatedAudio: hasSplitAudioTranscoding,
-
- copyCodecs: true,
- resolution: 0,
- fps: 0,
- video,
- isNewVideo
- })
- }
- )
- }
-
- children.push(hlsPayloads)
- }
+ children.push([
+ this.buildHLSJobPayload({
+ deleteWebVideoFiles: !CONFIG.TRANSCODING.WEB_VIDEOS.ENABLED,
+ separatedAudio: hasSplitAudioTranscoding,
- const lowerResolutionJobPayloads = await this.buildLowerResolutionJobPayloads({
- video,
- inputVideoResolution: maxResolution,
- inputVideoFPS: maxFPS,
- hasAudio: videoFile.hasAudio(),
- isNewVideo,
- hlsAudioAlreadyGenerated
- })
+ resolution: 0,
+ fps: 0,
+ video,
+ isNewVideo,
- children = children.concat(lowerResolutionJobPayloads)
- })
- } finally {
- mutexReleaser()
+ inputStreams,
+ transcodingRequestAt,
+
+ transcodingPriority: 'required',
+ canMoveVideoState: true
+ })
+ ])
+ }
}
+ const lowerResolutionJobPayloads = await this.buildLowerResolutionJobPayloads({
+ video,
+ inputStreams,
+ transcodingRequestAt,
+ inputVideoResolution: maxResolution,
+ inputVideoFPS: maxFPS,
+ hasAudio: videoFile.hasAudio(),
+ isNewVideo,
+ hlsAudioAlreadyGenerated
+ })
+
+ children = children.concat(lowerResolutionJobPayloads)
+
+ this.reassignCanMoveVideoState(mergeOrOptimizePayload, children.length === 0)
+
await this.createJobs({
- payloads: [ [ mergeOrOptimizePayload ], ...children ],
+ payloads: {
+ parent: mergeOrOptimizePayload,
+ children
+ },
user,
video
})
@@ -145,15 +143,16 @@ export abstract class AbstractJobBuilder<P> {
}) {
const { video, transcodingType, resolutions, isNewVideo } = options
const separatedAudio = CONFIG.TRANSCODING.HLS.SPLIT_AUDIO_AND_VIDEO
+ const transcodingRequestAt = new Date().toISOString()
+ const inputStreams = video.getStreamTypes()
const maxResolution = Math.max(...resolutions)
- const childrenResolutions = resolutions.filter(r => r !== maxResolution)
- logger.info('Manually creating transcoding jobs for %s.', transcodingType, { childrenResolutions, maxResolution, ...lTags(video.uuid) })
+ logger.info(`Manually creating transcoding jobs for ${transcodingType}`, { resolutions, maxResolution, ...lTags(video.uuid) })
const inputFPS = video.getMaxFPS()
- const children = childrenResolutions
+ const children = resolutions
.map(resolution => {
const fps = computeOutputFPS({ inputFPS, resolution, isOriginResolution: maxResolution === resolution, type: 'vod' })
@@ -161,11 +160,32 @@ export abstract class AbstractJobBuilder<P> {
// We'll generate audio resolution in a parent job
if (resolution === VideoResolution.H_NOVIDEO && separatedAudio) return undefined
- return this.buildHLSJobPayload({ video, resolution, fps, isNewVideo, separatedAudio })
+ return [
+ this.buildHLSJobPayload({
+ video,
+ resolution,
+ fps,
+ isNewVideo,
+ separatedAudio,
+ canMoveVideoState: true,
+ inputStreams,
+ transcodingRequestAt,
+ transcodingPriority: 'optional'
+ })
+ ]
}
if (transcodingType === 'web-video') {
- return this.buildWebVideoJobPayload({ video, resolution, fps, isNewVideo })
+ return [
+ this.buildWebVideoJobPayload({
+ video,
+ resolution,
+ fps,
+ isNewVideo,
+ canMoveVideoState: true,
+ transcodingPriority: 'optional'
+ })
+ ]
}
throw new Error('Unknown transcoding type')
@@ -174,32 +194,44 @@ export abstract class AbstractJobBuilder<P> {
const fps = computeOutputFPS({ inputFPS, resolution: maxResolution, isOriginResolution: true, type: 'vod' })
- const parent = transcodingType === 'hls'
- ? this.buildHLSJobPayload({ video, resolution: maxResolution, fps, isNewVideo, separatedAudio })
- : this.buildWebVideoJobPayload({ video, resolution: maxResolution, fps, isNewVideo })
-
- // Low resolutions use the biggest one as ffmpeg input so we need to process max resolution (with audio) independently
- const payloads: [[P], ...(P[][])] = [ [ parent ] ]
-
// Process audio first to not override the max resolution where the audio stream will be removed
- if (transcodingType === 'hls' && separatedAudio) {
- payloads.unshift([ this.buildHLSJobPayload({ video, resolution: VideoResolution.H_NOVIDEO, fps, isNewVideo, separatedAudio }) ])
- }
-
- if (children && children.length !== 0) payloads.push(children)
+ const parent = transcodingType === 'hls' && separatedAudio
+ ? this.buildHLSJobPayload({
+ video,
+ resolution: VideoResolution.H_NOVIDEO,
+ fps,
+ isNewVideo,
+ separatedAudio,
+ canMoveVideoState: true,
+ transcodingRequestAt,
+ inputStreams,
+ transcodingPriority: 'optional'
+ })
+ : undefined
- await this.createJobs({ video, payloads, user: null })
+ await this.createJobs({ video, payloads: { parent, children }, user: null })
}
private async buildLowerResolutionJobPayloads (options: {
video: MVideoFullLight
inputVideoResolution: number
inputVideoFPS: number
+ inputStreams: VideoFileStreamType[]
hasAudio: boolean
isNewVideo: boolean
hlsAudioAlreadyGenerated: boolean
+ transcodingRequestAt: string
}) {
- const { video, inputVideoResolution, inputVideoFPS, isNewVideo, hlsAudioAlreadyGenerated, hasAudio } = options
+ const {
+ video,
+ inputVideoResolution,
+ inputVideoFPS,
+ inputStreams,
+ isNewVideo,
+ hlsAudioAlreadyGenerated,
+ hasAudio,
+ transcodingRequestAt
+ } = options
// Create transcoding jobs if there are enabled resolutions
const resolutionsEnabled = await Hooks.wrapObject(
@@ -231,7 +263,9 @@ export abstract class AbstractJobBuilder<P> {
video,
resolution,
fps,
- isNewVideo
+ isNewVideo,
+ canMoveVideoState: true,
+ transcodingPriority: 'optional'
})
)
}
@@ -245,7 +279,10 @@ export abstract class AbstractJobBuilder<P> {
fps,
isNewVideo,
separatedAudio: hasAudio && CONFIG.TRANSCODING.HLS.SPLIT_AUDIO_AND_VIDEO,
- copyCodecs: CONFIG.TRANSCODING.WEB_VIDEOS.ENABLED
+ canMoveVideoState: true,
+ transcodingPriority: 'optional',
+ transcodingRequestAt,
+ inputStreams
})
)
}
@@ -262,8 +299,14 @@ export abstract class AbstractJobBuilder<P> {
protected abstract createJobs (options: {
video: MVideoFullLight
- // Array of sequential jobs to create that depend on parent job
- payloads: [[(P & { higherPriority?: boolean })], ...((P & { higherPriority?: boolean })[][])]
+ payloads: {
+ parent: P
+
+ // Parallel of sequential jobs to execute
+ // [ [ Parallel1 ], [ Parallel2 ], ... ]
+ // [ [ Seq1, Seq2, ... ], [ Seq1, ... ]
+ children: P[][]
+ }
user: MUserId | null
}): Promise<void>
@@ -273,15 +316,20 @@ export abstract class AbstractJobBuilder<P> {
isNewVideo: boolean
resolution: number
fps: number
+
+ transcodingPriority: TranscodingPriorityType
+ canMoveVideoState: boolean
}): P
protected abstract buildOptimizePayload (options: {
video: MVideoFullLight
isNewVideo: boolean
- quickTranscode: boolean
inputFile: MVideoFile
resolution: number
fps: number
+
+ transcodingPriority: TranscodingPriorityType
+ canMoveVideoState: boolean
}): P
protected abstract buildHLSJobPayload (options: {
@@ -290,8 +338,14 @@ export abstract class AbstractJobBuilder<P> {
fps: number
isNewVideo: boolean
separatedAudio: boolean
+
deleteWebVideoFiles?: boolean // default false
- copyCodecs?: boolean // default false
+
+ inputStreams: VideoFileStreamType[]
+ canMoveVideoState: boolean
+ transcodingRequestAt: string
+
+ transcodingPriority: TranscodingPriorityType
}): P
protected abstract buildWebVideoJobPayload (options: {
@@ -299,5 +353,12 @@ export abstract class AbstractJobBuilder<P> {
resolution: number
fps: number
isNewVideo: boolean
+
+ transcodingPriority: TranscodingPriorityType
+ canMoveVideoState: boolean
}): P
+
+ // ---------------------------------------------------------------------------
+
+ protected abstract reassignCanMoveVideoState (payload: P, canMoveVideoState: boolean): void
}
diff --git a/server/core/lib/transcoding/shared/job-builders/transcoding-job-queue-builder.ts b/server/core/lib/transcoding/shared/job-builders/transcoding-job-queue-builder.ts
index fa4b39a2a8..a531ef83ac 100644
--- a/server/core/lib/transcoding/shared/job-builders/transcoding-job-queue-builder.ts
+++ b/server/core/lib/transcoding/shared/job-builders/transcoding-job-queue-builder.ts
@@ -1,44 +1,48 @@
+import { pick } from '@peertube/peertube-core-utils'
import {
HLSTranscodingPayload,
MergeAudioTranscodingPayload,
NewWebVideoResolutionTranscodingPayload,
OptimizeTranscodingPayload,
+ VideoFileStreamType,
VideoTranscodingPayload
} from '@peertube/peertube-models'
import { CreateJobArgument, JobQueue } from '@server/lib/job-queue/index.js'
import { VideoJobInfoModel } from '@server/models/video/video-job-info.js'
import { MUserId, MVideo } from '@server/types/models/index.js'
import { getTranscodingJobPriority } from '../../transcoding-priority.js'
-import { AbstractJobBuilder } from './abstract-job-builder.js'
+import { AbstractJobBuilder, TranscodingPriorityType } from './abstract-job-builder.js'
-type Payload =
- MergeAudioTranscodingPayload |
- OptimizeTranscodingPayload |
- NewWebVideoResolutionTranscodingPayload |
- HLSTranscodingPayload
+type BasePayload =
+ | MergeAudioTranscodingPayload
+ | OptimizeTranscodingPayload
+ | NewWebVideoResolutionTranscodingPayload
+ | HLSTranscodingPayload
-type PayloadWithPriority = Payload & { higherPriority?: boolean }
-
-export class TranscodingJobQueueBuilder extends AbstractJobBuilder <Payload> {
+type FullPayload = BasePayload & { transcodingPriority: TranscodingPriorityType }
+export class TranscodingJobQueueBuilder extends AbstractJobBuilder<FullPayload> {
protected async createJobs (options: {
video: MVideo
- // Array of sequential jobs to create that depend on parent job
- payloads: [ [ PayloadWithPriority ], ...(PayloadWithPriority[][]) ]
+ payloads: {
+ parent: FullPayload | null
+ children: FullPayload[][]
+ }
user: MUserId | null
}): Promise<void> {
- const { video, payloads, user } = options
-
- const priority = await getTranscodingJobPriority({ user, type: 'vod', fallback: undefined })
+ const { video, payloads: { parent, children }, user } = options
- const parent = payloads[0][0]
- payloads.shift()
+ const requiredPriority = await getTranscodingJobPriority({ user, type: 'vod-required' })
+ const optionalPriority = await getTranscodingJobPriority({ user, type: 'vod-optional' })
- const nextTranscodingSequentialJobs = payloads.map(p => {
+ const nextTranscodingSequentialJobs = children.map(p => {
return p.map(payload => {
return this.buildTranscodingJob({
payload,
- priority: payload.higherPriority ? priority - 1 : priority
+
+ priority: payload.transcodingPriority === 'required'
+ ? requiredPriority
+ : optionalPriority
})
})
})
@@ -51,29 +55,34 @@ export class TranscodingJobQueueBuilder extends AbstractJobBuilder <Payload> {
}
}
- const parentJob = this.buildTranscodingJob({
- payload: parent,
- priority: parent.higherPriority ? priority - 1 : priority,
- hasChildren: payloads.length !== 0
- })
+ const parentJob = parent
+ ? this.buildTranscodingJob({
+ payload: parent,
+
+ priority: parent.transcodingPriority === 'required'
+ ? requiredPriority
+ : optionalPriority
+ })
+ : undefined
await JobQueue.Instance.createSequentialJobFlow(parentJob, transcodingJobBuilderJob)
// transcoding-job-builder job will increase pendingTranscode
- await VideoJobInfoModel.increaseOrCreate(video.uuid, 'pendingTranscode')
+ if (parentJob) {
+ await VideoJobInfoModel.increaseOrCreate(video.uuid, 'pendingTranscode')
+ }
}
private buildTranscodingJob (options: {
payload: VideoTranscodingPayload
- hasChildren?: boolean
priority: number
}) {
- const { priority, payload, hasChildren = false } = options
+ const { priority, payload } = options
return {
type: 'video-transcoding' as 'video-transcoding',
priority,
- payload: { ...payload, hasChildren }
+ payload
}
}
@@ -85,19 +94,32 @@ export class TranscodingJobQueueBuilder extends AbstractJobBuilder <Payload> {
fps: number
isNewVideo: boolean
separatedAudio: boolean
+
+ transcodingPriority: TranscodingPriorityType
+
+ transcodingRequestAt: string
+ canMoveVideoState: boolean
+ inputStreams: VideoFileStreamType[]
+
deleteWebVideoFiles?: boolean // default false
- copyCodecs?: boolean // default false
- }): HLSTranscodingPayload {
- const { video, resolution, fps, isNewVideo, separatedAudio, deleteWebVideoFiles = false, copyCodecs = false } = options
+ }): HLSTranscodingPayload & { transcodingPriority: TranscodingPriorityType } {
+ const { video, deleteWebVideoFiles = false } = options
return {
type: 'new-resolution-to-hls',
videoUUID: video.uuid,
- resolution,
- fps,
- copyCodecs,
- isNewVideo,
- separatedAudio,
+
+ ...pick(options, [
+ 'resolution',
+ 'fps',
+ 'isNewVideo',
+ 'separatedAudio',
+ 'canMoveVideoState',
+ 'transcodingPriority',
+ 'transcodingRequestAt',
+ 'inputStreams'
+ ]),
+
deleteWebVideoFiles
}
}
@@ -107,15 +129,23 @@ export class TranscodingJobQueueBuilder extends AbstractJobBuilder <Payload> {
resolution: number
fps: number
isNewVideo: boolean
- }): NewWebVideoResolutionTranscodingPayload {
- const { video, resolution, fps, isNewVideo } = options
+
+ transcodingPriority: TranscodingPriorityType
+ canMoveVideoState: boolean
+ }): NewWebVideoResolutionTranscodingPayload & { transcodingPriority: TranscodingPriorityType } {
+ const { video } = options
return {
type: 'new-resolution-to-web-video',
videoUUID: video.uuid,
- isNewVideo,
- resolution,
- fps
+
+ ...pick(options, [
+ 'isNewVideo',
+ 'resolution',
+ 'fps',
+ 'transcodingPriority',
+ 'canMoveVideoState'
+ ])
}
}
@@ -124,40 +154,51 @@ export class TranscodingJobQueueBuilder extends AbstractJobBuilder <Payload> {
isNewVideo: boolean
fps: number
resolution: number
- }): MergeAudioTranscodingPayload {
- const { video, isNewVideo, resolution, fps } = options
+
+ transcodingPriority: TranscodingPriorityType
+ canMoveVideoState: boolean
+ }): MergeAudioTranscodingPayload & { transcodingPriority: TranscodingPriorityType } {
+ const { video } = options
return {
type: 'merge-audio-to-web-video',
- resolution,
- fps,
videoUUID: video.uuid,
- // Will be set later
- hasChildren: undefined,
-
- isNewVideo
+ ...pick(options, [
+ 'resolution',
+ 'fps',
+ 'isNewVideo',
+ 'transcodingPriority',
+ 'canMoveVideoState'
+ ])
}
}
protected buildOptimizePayload (options: {
video: MVideo
- quickTranscode: boolean
isNewVideo: boolean
- }): OptimizeTranscodingPayload {
- const { video, quickTranscode, isNewVideo } = options
+
+ transcodingPriority: TranscodingPriorityType
+ canMoveVideoState: boolean
+ }): OptimizeTranscodingPayload & { transcodingPriority: TranscodingPriorityType } {
+ const { video } = options
return {
type: 'optimize-to-web-video',
videoUUID: video.uuid,
- isNewVideo,
-
- // Will be set later
- hasChildren: undefined,
- quickTranscode
+ ...pick(options, [
+ 'isNewVideo',
+ 'transcodingPriority',
+ 'canMoveVideoState'
+ ])
}
}
+ // ---------------------------------------------------------------------------
+
+ protected reassignCanMoveVideoState (payload: FullPayload, canMoveVideoState: boolean): void {
+ payload.canMoveVideoState = canMoveVideoState
+ }
}
diff --git a/server/core/lib/transcoding/shared/job-builders/transcoding-runner-job-builder.ts b/server/core/lib/transcoding/shared/job-builders/transcoding-runner-job-builder.ts
index a12a7b3b54..4c8bd95281 100644
--- a/server/core/lib/transcoding/shared/job-builders/transcoding-runner-job-builder.ts
+++ b/server/core/lib/transcoding/shared/job-builders/transcoding-runner-job-builder.ts
@@ -1,3 +1,5 @@
+import { pick } from '@peertube/peertube-core-utils'
+import { VideoFileStreamType } from '@peertube/peertube-models'
import {
VODAudioMergeTranscodingJobHandler,
VODHLSTranscodingJobHandler,
@@ -6,13 +8,13 @@ import {
import { MUserId, MVideo, MVideoFile, MVideoFullLight } from '@server/types/models/index.js'
import { MRunnerJob } from '@server/types/models/runners/runner-job.js'
import { getTranscodingJobPriority } from '../../transcoding-priority.js'
-import { AbstractJobBuilder } from './abstract-job-builder.js'
+import { AbstractJobBuilder, TranscodingPriorityType } from './abstract-job-builder.js'
/**
* Class to build transcoding job in the local job queue
*/
-type Payload = {
+type BasePayload = {
Builder: new() => VODHLSTranscodingJobHandler
options: Omit<Parameters<VODHLSTranscodingJobHandler['create']>[0], 'priority'>
} | {
@@ -23,34 +25,43 @@ type Payload = {
options: Omit<Parameters<VODWebVideoTranscodingJobHandler['create']>[0], 'priority'>
}
-type PayloadWithPriority = Payload & { higherPriority?: boolean }
+type FullPayload = BasePayload & { transcodingPriority: TranscodingPriorityType }
-// eslint-disable-next-line max-len
-export class TranscodingRunnerJobBuilder extends AbstractJobBuilder<Payload> {
+export class TranscodingRunnerJobBuilder extends AbstractJobBuilder<FullPayload> {
protected async createJobs (options: {
video: MVideo
- payloads: [[PayloadWithPriority], ...(PayloadWithPriority[][])] // Array of sequential jobs to create that depend on parent job
+ payloads: {
+ parent: FullPayload | null
+ children: FullPayload[][]
+ }
user: MUserId | null
}): Promise<void> {
- const { payloads, user } = options
+ const { payloads: { parent, children }, user } = options
- const parent = payloads[0][0]
- payloads.shift()
+ const requiredPriority = await getTranscodingJobPriority({ user, type: 'vod-required' })
+ const optionalPriority = await getTranscodingJobPriority({ user, type: 'vod-optional' })
- const priority = await getTranscodingJobPriority({ user, type: 'vod', fallback: 0 })
+ const parentJob = parent
+ ? await this.createJob({
+ payload: parent,
- const parentJob = await this.createJob({
- payload: parent,
- priority: parent.higherPriority ? priority - 1 : priority
- })
+ priority: parent.transcodingPriority === 'required'
+ ? requiredPriority
+ : optionalPriority
+ })
+ : undefined
- for (const parallelPayloads of payloads) {
+ for (const parallelPayloads of children) {
let lastJob = parentJob
- for (const parallelPayload of parallelPayloads) {
+ for (const sequentialPayload of parallelPayloads) {
lastJob = await this.createJob({
- payload: parallelPayload,
- priority: parallelPayload.higherPriority ? priority - 1 : priority,
+ payload: sequentialPayload,
+
+ priority: sequentialPayload.transcodingPriority === 'required'
+ ? requiredPriority
+ : optionalPriority,
+
dependsOnRunnerJob: lastJob
})
}
@@ -60,7 +71,7 @@ export class TranscodingRunnerJobBuilder extends AbstractJobBuilder<Payload> {
}
private createJob (options: {
- payload: Payload
+ payload: FullPayload
priority: number
dependsOnRunnerJob?: MRunnerJob
}) {
@@ -84,22 +95,36 @@ export class TranscodingRunnerJobBuilder extends AbstractJobBuilder<Payload> {
fps: number
isNewVideo: boolean
separatedAudio: boolean
+
+ transcodingPriority: TranscodingPriorityType
+
+ canMoveVideoState: boolean
+ inputStreams: VideoFileStreamType[]
+ transcodingRequestAt: string
+
deleteWebVideoFiles?: boolean // default false
- copyCodecs?: boolean // default false
- }): Payload {
- const { video, resolution, fps, isNewVideo, separatedAudio, deleteWebVideoFiles = false } = options
+ }): FullPayload {
+ const { deleteWebVideoFiles = false } = options
return {
Builder: VODHLSTranscodingJobHandler,
options: {
- video,
- resolution,
- fps,
- isNewVideo,
- separatedAudio,
+ ...pick(options, [
+ 'video',
+ 'resolution',
+ 'fps',
+ 'isNewVideo',
+ 'separatedAudio',
+ 'canMoveVideoState',
+ 'transcodingRequestAt',
+ 'inputStreams'
+ ]),
+
deleteWebVideoFiles
- }
+ },
+
+ ...pick(options, [ 'transcodingPriority' ])
}
}
@@ -108,19 +133,28 @@ export class TranscodingRunnerJobBuilder extends AbstractJobBuilder<Payload> {
resolution: number
fps: number
isNewVideo: boolean
- }): Payload {
- const { video, resolution, fps, isNewVideo } = options
+ transcodingPriority: TranscodingPriorityType
+ canMoveVideoState: boolean
+ }): FullPayload {
return {
Builder: VODWebVideoTranscodingJobHandler,
options: {
- video,
- resolution,
- fps,
- isNewVideo,
+ ...pick(options, [
+ 'video',
+ 'resolution',
+ 'fps',
+ 'isNewVideo',
+ 'canMoveVideoState',
+ 'transcodingPriority',
+ 'canMoveVideoState'
+ ]),
+
deleteInputFileId: null
- }
+ },
+
+ ...pick(options, [ 'transcodingPriority' ])
}
}
@@ -130,40 +164,63 @@ export class TranscodingRunnerJobBuilder extends AbstractJobBuilder<Payload> {
isNewVideo: boolean
fps: number
resolution: number
- }): Payload {
- const { video, isNewVideo, inputFile, resolution, fps } = options
+
+ transcodingPriority: TranscodingPriorityType
+ canMoveVideoState: boolean
+ }): FullPayload {
+ const { inputFile } = options
return {
Builder: VODAudioMergeTranscodingJobHandler,
+
options: {
- video,
- resolution,
- fps,
- isNewVideo,
+ ...pick(options, [
+ 'video',
+ 'resolution',
+ 'fps',
+ 'isNewVideo',
+ 'canMoveVideoState'
+ ]),
+
deleteInputFileId: inputFile.id
- }
+ },
+
+ ...pick(options, [ 'transcodingPriority' ])
}
}
protected buildOptimizePayload (options: {
video: MVideoFullLight
inputFile: MVideoFile
- quickTranscode: boolean
isNewVideo: boolean
fps: number
resolution: number
- }): Payload {
- const { video, isNewVideo, inputFile, fps, resolution } = options
+
+ transcodingPriority: TranscodingPriorityType
+ canMoveVideoState: boolean
+ }): FullPayload {
+ const { inputFile } = options
return {
Builder: VODWebVideoTranscodingJobHandler,
+
options: {
- video,
- resolution,
- fps,
- isNewVideo,
+ ...pick(options, [
+ 'video',
+ 'resolution',
+ 'fps',
+ 'isNewVideo',
+ 'canMoveVideoState'
+ ]),
+
deleteInputFileId: inputFile.id
- }
+ },
+
+ ...pick(options, [ 'transcodingPriority' ])
}
}
+
+ protected reassignCanMoveVideoState (payload: FullPayload, canMoveVideoState: boolean) {
+ payload.options.canMoveVideoState = canMoveVideoState
+ }
}
diff --git a/server/core/lib/transcoding/transcoding-priority.ts b/server/core/lib/transcoding/transcoding-priority.ts
index 42061d0f96..d591316adb 100644
--- a/server/core/lib/transcoding/transcoding-priority.ts
+++ b/server/core/lib/transcoding/transcoding-priority.ts
@@ -4,21 +4,24 @@ import { MUserId } from '@server/types/models/index.js'
export async function getTranscodingJobPriority (options: {
user: MUserId
- fallback: number
- type: 'vod' | 'studio'
+ type: 'vod-required' | 'vod-optional' | 'studio'
}) {
- const { user, fallback, type } = options
+ const { user, type } = options
- if (!user) return fallback
+ const priorityMap = {
+ 'vod-required': JOB_PRIORITY.REQUIRED_TRANSCODING,
+ 'vod-optional': JOB_PRIORITY.OPTIONAL_TRANSCODING,
+ 'studio': JOB_PRIORITY.VIDEO_STUDIO
+ }
+
+ const priority = priorityMap[type]
+
+ if (!user) return priority
const now = new Date()
const lastWeek = new Date(now.getFullYear(), now.getMonth(), now.getDate() - 7)
const videoUploadedByUser = await VideoModel.countVideosUploadedByUserSince(user.id, lastWeek)
- const base = type === 'vod'
- ? JOB_PRIORITY.TRANSCODING
- : JOB_PRIORITY.VIDEO_STUDIO
-
- return base + videoUploadedByUser
+ return priority + videoUploadedByUser
}
diff --git a/server/core/lib/transcoding/web-transcoding.ts b/server/core/lib/transcoding/web-transcoding.ts
index f843f8a497..b8cb295846 100644
--- a/server/core/lib/transcoding/web-transcoding.ts
+++ b/server/core/lib/transcoding/web-transcoding.ts
@@ -22,14 +22,14 @@ import { addLocalOrRemoteStoryboardJobIfNeeded } from '../video-jobs.js'
import { VideoPathManager } from '../video-path-manager.js'
import { buildFFmpegVOD } from './shared/index.js'
import { buildOriginalFileResolution } from './transcoding-resolutions.js'
+import { canDoQuickTranscode } from './transcoding-quick-transcode.js'
// Optimize the original video file and replace it. The resolution is not changed.
export async function optimizeOriginalVideofile (options: {
video: MVideoFullLight
- quickTranscode: boolean
job: Job
}) {
- const { quickTranscode, job } = options
+ const { job } = options
const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
const newExtname = '.mp4'
@@ -44,7 +44,7 @@ export async function optimizeOriginalVideofile (options: {
const result = await VideoPathManager.Instance.makeAvailableVideoFile(inputVideoFile, async videoInputPath => {
const videoOutputPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname)
- const transcodeType: TranscodeVODOptionsType = quickTranscode
+ const transcodeType: TranscodeVODOptionsType = await canDoQuickTranscode(videoInputPath, CONFIG.TRANSCODING.FPS.MAX)
? 'quick-transcode'
: 'video'
diff --git a/server/core/lib/video-captions.ts b/server/core/lib/video-captions.ts
index 9e317b9d30..4c1e3539d5 100644
--- a/server/core/lib/video-captions.ts
+++ b/server/core/lib/video-captions.ts
@@ -16,7 +16,7 @@ import { MStreamingPlaylist, MVideo, MVideoCaption, MVideoFullLight, MVideoId, M
import { MutexInterface } from 'async-mutex'
import { ensureDir, remove } from 'fs-extra/esm'
import { writeFile } from 'fs/promises'
-import { join } from 'path'
+import { dirname, join } from 'path'
import { federateVideoIfNeeded } from './activitypub/videos/federate.js'
import { buildCaptionM3U8Content, updateM3U8AndShaPlaylist } from './hls.js'
import { JobQueue } from './job-queue/job-queue.js'
@@ -272,6 +272,7 @@ export async function upsertCaptionPlaylistOnFS (caption: MVideoCaption, video:
logger.debug(`Creating caption playlist ${m3u8Destination} of video ${video.uuid}`, lTags(video.uuid))
const content = buildCaptionM3U8Content({ video, caption })
+ await ensureDir(dirname(m3u8Destination))
await writeFile(m3u8Destination, content, 'utf8')
return m3u8Filename
diff --git a/server/core/lib/video-jobs.ts b/server/core/lib/video-jobs.ts
index a0db46174a..3f50ff736a 100644
--- a/server/core/lib/video-jobs.ts
+++ b/server/core/lib/video-jobs.ts
@@ -17,11 +17,14 @@ import { moveFilesIfPrivacyChanged } from './video-privacy.js'
export async function buildMoveVideoJob (options: {
video: MVideoUUID
- previousVideoState: VideoStateType
type: 'move-to-object-storage' | 'move-to-file-system'
- isNewVideo?: boolean // Default true
+
+ moveVideoState?: {
+ isNewVideo: boolean
+ previousVideoState: VideoStateType
+ }
}) {
- const { video, previousVideoState, isNewVideo = true, type } = options
+ const { video, moveVideoState, type } = options
await VideoJobInfoModel.increaseOrCreate(video.uuid, 'pendingMove')
@@ -29,8 +32,7 @@ export async function buildMoveVideoJob (options: {
type,
payload: {
videoUUID: video.uuid,
- isNewVideo,
- previousVideoState
+ moveVideoState
}
}
}
@@ -133,8 +135,18 @@ export async function addVideoJobsAfterCreation (options: {
}
]
+ // No transcoding, move the file directly on object storage
if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) {
- jobs.push(await buildMoveVideoJob({ video, previousVideoState: undefined, type: 'move-to-object-storage' }))
+ jobs.push(
+ await buildMoveVideoJob({
+ type: 'move-to-object-storage',
+ video,
+ moveVideoState: {
+ isNewVideo: true,
+ previousVideoState: undefined
+ }
+ })
+ )
}
if (video.state === VideoState.TO_TRANSCODE) {
diff --git a/server/core/lib/video-state.ts b/server/core/lib/video-state.ts
index 1ea463002a..edc4238043 100644
--- a/server/core/lib/video-state.ts
+++ b/server/core/lib/video-state.ts
@@ -1,18 +1,19 @@
-import { Transaction } from 'sequelize'
import { VideoState, VideoStateType } from '@peertube/peertube-models'
import { retryTransactionWrapper } from '@server/helpers/database-utils.js'
-import { logger } from '@server/helpers/logger.js'
+import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
import { CONFIG } from '@server/initializers/config.js'
import { sequelizeTypescript } from '@server/initializers/database.js'
-import { VideoJobInfoModel } from '@server/models/video/video-job-info.js'
import { VideoModel } from '@server/models/video/video.js'
import { MVideo, MVideoFullLight, MVideoUUID } from '@server/types/models/index.js'
+import { Transaction } from 'sequelize'
import { federateVideoIfNeeded } from './activitypub/videos/index.js'
import { JobQueue } from './job-queue/index.js'
import { Notifier } from './notifier/index.js'
import { buildMoveVideoJob } from './video-jobs.js'
-function buildNextVideoState (currentState?: VideoStateType) {
+const lTags = loggerTagsFactory('video-state')
+
+export function buildNextVideoState (currentState?: VideoStateType) {
if (currentState === VideoState.PUBLISHED) {
throw new Error('Video is already in its final state')
}
@@ -38,7 +39,7 @@ function buildNextVideoState (currentState?: VideoStateType) {
return VideoState.PUBLISHED
}
-function moveToNextState (options: {
+export function moveToNextState (options: {
video: MVideoUUID
previousVideoState?: VideoStateType
isNewVideo?: boolean // Default true
@@ -54,57 +55,65 @@ function moveToNextState (options: {
// Already in its final state
if (videoDatabase.state === VideoState.PUBLISHED) {
- return federateVideoIfNeeded(videoDatabase, false, t)
+ await federateVideoIfNeeded(videoDatabase, false, t)
+
+ logger.debug(`Video ${videoDatabase.uuid} is already published, no state change.`, lTags(videoDatabase.uuid))
+
+ return false
}
const newState = buildNextVideoState(videoDatabase.state)
if (newState === VideoState.PUBLISHED) {
- return moveToPublishedState({ video: videoDatabase, previousVideoState, isNewVideo, transaction: t })
+ await moveToPublishedState({ video: videoDatabase, previousVideoState, isNewVideo, transaction: t })
+ return true
}
if (newState === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) {
- return moveToExternalStorageState({ video: videoDatabase, isNewVideo, transaction: t })
+ await moveToExternalStorageState({ video: videoDatabase, isNewVideo, transaction: t })
+ return true
}
+
+ throw new Error('Unknown next state for video ' + videoDatabase.uuid + ': ' + newState)
})
})
}
// ---------------------------------------------------------------------------
-async function moveToExternalStorageState (options: {
+export async function moveToExternalStorageState (options: {
video: MVideoFullLight
isNewVideo: boolean
transaction: Transaction
}) {
const { video, isNewVideo, transaction } = options
- const videoJobInfo = await VideoJobInfoModel.load(video.id, transaction)
- const pendingTranscode = videoJobInfo?.pendingTranscode || 0
-
- // We want to wait all transcoding jobs before moving the video on an external storage
- if (pendingTranscode !== 0) return false
-
const previousVideoState = video.state
if (video.state !== VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) {
await video.setNewState(VideoState.TO_MOVE_TO_EXTERNAL_STORAGE, isNewVideo, transaction)
}
- logger.info('Creating external storage move job for video %s.', video.uuid, { tags: [ video.uuid ] })
+ logger.info('Creating external storage move job for video %s.', video.uuid, lTags(video.uuid))
try {
- await JobQueue.Instance.createJob(await buildMoveVideoJob({ video, previousVideoState, isNewVideo, type: 'move-to-object-storage' }))
+ await JobQueue.Instance.createJob(
+ await buildMoveVideoJob({
+ type: 'move-to-object-storage',
+ video,
+ moveVideoState: { isNewVideo, previousVideoState }
+ })
+ )
return true
} catch (err) {
- logger.error('Cannot add move to object storage job', { err })
+ logger.error('Cannot add move to object storage job', { err, ...lTags(video.uuid) })
return false
}
}
-async function moveToFileSystemState (options: {
+export async function moveToFileSystemState (options: {
video: MVideoFullLight
isNewVideo: boolean
transaction: Transaction
@@ -120,11 +129,20 @@ async function moveToFileSystemState (options: {
logger.info('Creating move to file system job for video %s.', video.uuid, { tags: [ video.uuid ] })
try {
- await JobQueue.Instance.createJob(await buildMoveVideoJob({ video, previousVideoState, isNewVideo, type: 'move-to-file-system' }))
+ await JobQueue.Instance.createJob(
+ await buildMoveVideoJob({
+ type: 'move-to-file-system',
+ video,
+ moveVideoState: {
+ previousVideoState,
+ isNewVideo
+ }
+ })
+ )
return true
} catch (err) {
- logger.error('Cannot add move to file system job', { err })
+ logger.error('Cannot add move to file system job', { err, ...lTags(video.uuid) })
return false
}
@@ -132,36 +150,26 @@ async function moveToFileSystemState (options: {
// ---------------------------------------------------------------------------
-function moveToFailedTranscodingState (video: MVideo) {
+export function moveToFailedTranscodingState (video: MVideo) {
if (video.state === VideoState.TRANSCODING_FAILED) return
return video.setNewState(VideoState.TRANSCODING_FAILED, false, undefined)
}
-function moveToFailedMoveToObjectStorageState (video: MVideo) {
+export function moveToFailedMoveToObjectStorageState (video: MVideo) {
if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE_FAILED) return
return video.setNewState(VideoState.TO_MOVE_TO_EXTERNAL_STORAGE_FAILED, false, undefined)
}
-function moveToFailedMoveToFileSystemState (video: MVideo) {
+export function moveToFailedMoveToFileSystemState (video: MVideo) {
if (video.state === VideoState.TO_MOVE_TO_FILE_SYSTEM_FAILED) return
return video.setNewState(VideoState.TO_MOVE_TO_FILE_SYSTEM_FAILED, false, undefined)
}
// ---------------------------------------------------------------------------
-
-export {
- buildNextVideoState,
- moveToFailedMoveToFileSystemState,
- moveToExternalStorageState,
- moveToFileSystemState,
- moveToFailedTranscodingState,
- moveToFailedMoveToObjectStorageState,
- moveToNextState
-}
-
+// Private
// ---------------------------------------------------------------------------
async function moveToPublishedState (options: {
@@ -173,7 +181,7 @@ async function moveToPublishedState (options: {
const { video, isNewVideo, transaction, previousVideoState } = options
const previousState = previousVideoState ?? video.state
- logger.info('Publishing video %s.', video.uuid, { isNewVideo, previousState, tags: [ video.uuid ] })
+ logger.info('Publishing video %s.', video.uuid, { isNewVideo, previousState, ...lTags(video.uuid) })
await video.setNewState(VideoState.PUBLISHED, isNewVideo, transaction)
diff --git a/server/core/lib/video-studio.ts b/server/core/lib/video-studio.ts
index d5b089594a..0adbf1b5b6 100644
--- a/server/core/lib/video-studio.ts
+++ b/server/core/lib/video-studio.ts
@@ -75,7 +75,7 @@ export async function createVideoStudioJob (options: {
}) {
const { video, user, payload } = options
- const priority = await getTranscodingJobPriority({ user, type: 'studio', fallback: 0 })
+ const priority = await getTranscodingJobPriority({ user, type: 'studio' })
if (CONFIG.VIDEO_STUDIO.REMOTE_RUNNERS.ENABLED) {
await new VideoStudioTranscodingJobHandler().create({ video, tasks: payload.tasks, priority })
diff --git a/server/core/models/video/video.ts b/server/core/models/video/video.ts
index 42469e0b27..c6a34394d7 100644
--- a/server/core/models/video/video.ts
+++ b/server/core/models/video/video.ts
@@ -1,4 +1,4 @@
-import { buildVideoEmbedPath, buildVideoWatchPath, maxBy, pick, sortBy, wait } from '@peertube/peertube-core-utils'
+import { buildVideoEmbedPath, buildVideoWatchPath, maxBy, pick, wait } from '@peertube/peertube-core-utils'
import {
FileStorage,
ResultList,
@@ -1851,6 +1851,15 @@ export class VideoModel extends SequelizeModel<VideoModel> {
return !!this.getMaxQualityFile(VideoFileStream.VIDEO)
}
+ getStreamTypes<T extends MVideoWithFile> (this: T) {
+ const streamTypes: VideoFileStreamType[] = []
+
+ if (this.hasAudio()) streamTypes.push(VideoFileStream.AUDIO)
+ if (this.hasVideo()) streamTypes.push(VideoFileStream.VIDEO)
+
+ return streamTypes
+ }
+
static loadHasStream (videoId: number, stream: VideoFileStreamType) {
const query = 'SELECT 1 FROM "videoFile" WHERE "videoId" = $videoId AND ("streams" & $stream) = $stream ' +
'UNION ALL ' +
@@ -1868,16 +1877,13 @@ export class VideoModel extends SequelizeModel<VideoModel> {
// ---------------------------------------------------------------------------
- getWebVideoFileMinResolution<T extends MVideoWithFile> (this: T, resolution: number): MVideoFileVideo {
+ getWebVideoFileResolution<T extends MVideoWithFile> (this: T, resolution: number): MVideoFileVideo {
if (Array.isArray(this.VideoFiles) === false) return undefined
- for (const file of sortBy(this.VideoFiles, 'resolution')) {
- if (file.resolution < resolution) continue
-
- return Object.assign(file, { Video: this })
- }
+ const file = this.VideoFiles.find(f => f.resolution === resolution)
+ if (!file) return undefined
- return undefined
+ return Object.assign(file, { Video: this })
}
hasWebVideoFiles () {
diff --git a/server/scripts/create-move-video-storage-job.ts b/server/scripts/create-move-video-storage-job.ts
index 6741ed5479..2c4e90f8e1 100644
--- a/server/scripts/create-move-video-storage-job.ts
+++ b/server/scripts/create-move-video-storage-job.ts
@@ -1,15 +1,13 @@
import { createCommand } from '@commander-js/extra-typings'
-import { FileStorage, VideoState } from '@peertube/peertube-models'
+import { FileStorage, FileStorageType, VideoState } from '@peertube/peertube-models'
import { toCompleteUUID } from '@server/helpers/custom-validators/misc.js'
import { CONFIG } from '@server/initializers/config.js'
import { initDatabaseModels } from '@server/initializers/database.js'
import { JobQueue } from '@server/lib/job-queue/index.js'
+import { hasVideoResourcesToBeMoved } from '@server/lib/move-storage/shared/move-video.js'
import { moveToExternalStorageState, moveToFileSystemState } from '@server/lib/video-state.js'
-import { VideoCaptionModel } from '@server/models/video/video-caption.js'
-import { VideoSourceModel } from '@server/models/video/video-source.js'
import { VideoModel } from '@server/models/video/video.js'
-import { MStreamingPlaylist, MVideoCaption, MVideoFile, MVideoFullLight } from '@server/types/models/index.js'
-import { MVideoSource } from '@server/types/models/video/video-source.js'
+import { MVideoFullLight } from '@server/types/models/index.js'
const program = createCommand()
.description('Move videos to another storage.')
@@ -89,15 +87,7 @@ async function run () {
if (options.toObjectStorage) {
await createMoveJobIfNeeded({
video: videoFull,
- type: 'to object storage',
- canProcessVideo: options => {
- const { files, hls, source, captions } = options
-
- return files.some(f => f.storage === FileStorage.FILE_SYSTEM) ||
- hls?.storage === FileStorage.FILE_SYSTEM ||
- source?.storage === FileStorage.FILE_SYSTEM ||
- captions.some(c => c.storage === FileStorage.FILE_SYSTEM)
- },
+ targetStorage: FileStorage.OBJECT_STORAGE,
handler: () => moveToExternalStorageState({ video: videoFull, isNewVideo: false, transaction: undefined })
})
@@ -107,16 +97,7 @@ async function run () {
if (options.toFileSystem) {
await createMoveJobIfNeeded({
video: videoFull,
- type: 'to file system',
-
- canProcessVideo: options => {
- const { files, hls, source, captions } = options
-
- return files.some(f => f.storage === FileStorage.OBJECT_STORAGE) ||
- hls?.storage === FileStorage.OBJECT_STORAGE ||
- source?.storage === FileStorage.OBJECT_STORAGE ||
- captions.some(c => c.storage === FileStorage.OBJECT_STORAGE)
- },
+ targetStorage: FileStorage.FILE_SYSTEM,
handler: () => moveToFileSystemState({ video: videoFull, isNewVideo: false, transaction: undefined })
})
@@ -126,26 +107,17 @@ async function run () {
async function createMoveJobIfNeeded (options: {
video: MVideoFullLight
- type: 'to object storage' | 'to file system'
-
- canProcessVideo: (options: {
- files: MVideoFile[]
- hls: MStreamingPlaylist
- source: MVideoSource
- captions: MVideoCaption[]
- }) => boolean
+ targetStorage: FileStorageType
handler: () => Promise<any>
}) {
- const { video, type, canProcessVideo, handler } = options
-
- const files = video.VideoFiles || []
- const hls = video.getHLSPlaylist()
+ const { video, targetStorage, handler } = options
- const source = await VideoSourceModel.loadLatest(video.id)
- const captions = await VideoCaptionModel.listVideoCaptions(video.id)
+ if (await hasVideoResourcesToBeMoved(video, targetStorage)) {
+ const type = targetStorage === FileStorage.OBJECT_STORAGE
+ ? 'to object storage'
+ : 'to file system'
- if (canProcessVideo({ files, hls, source, captions })) {
console.log(`Moving ${type} video ${video.name}`)
const success = await handler()