Skip to content

ActivityPub-federated video streaming platform using P2P directly in your web browser

Don't create too many refresh jobs

Chocobozzz94420191f731
diff --git a/server/core/controllers/api/accounts.ts b/server/core/controllers/api/accounts.ts
index fd2f01bec0..0f52730be4 100644
--- a/server/core/controllers/api/accounts.ts
+++ b/server/core/controllers/api/accounts.ts
@@ -1,12 +1,12 @@
import { VideoPlaylistForAccountListQuery } from '@peertube/peertube-models'
import { pickCommonVideoQuery } from '@server/helpers/query.js'
+import { scheduleActorRefreshIfNeeded } from '@server/lib/activitypub/actors/refresh.js'
import { ActorFollowModel } from '@server/models/actor/actor-follow.js'
import { getServerActor } from '@server/models/application/application.js'
import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync.js'
import express from 'express'
import { buildNSFWFilters, getCountVideos, isUserAbleToSearchRemoteURI } from '../../helpers/express-utils.js'
import { getFormattedObjects } from '../../helpers/utils.js'
-import { JobQueue } from '../../lib/job-queue/index.js'
import { Hooks } from '../../lib/plugins/hooks.js'
import {
apiRateLimiter,
@@ -146,9 +146,7 @@ export {
function getAccount (req: express.Request, res: express.Response) {
const account = res.locals.account
- if (account.isOutdated()) {
- JobQueue.Instance.createJobAsync({ type: 'activitypub-refresher', payload: { type: 'actor', url: account.Actor.url } })
- }
+ scheduleActorRefreshIfNeeded(account.Actor)
return res.json(account.toFormattedJSON())
}
diff --git a/server/core/controllers/api/video-channels/index.ts b/server/core/controllers/api/video-channels/index.ts
index dc406c5e1d..6573c5cedb 100644
--- a/server/core/controllers/api/video-channels/index.ts
+++ b/server/core/controllers/api/video-channels/index.ts
@@ -60,6 +60,7 @@ import { VideoPlaylistModel } from '../../../models/video/video-playlist.js'
import { VideoModel } from '../../../models/video/video.js'
import { channelCollaborators } from './video-channel-collaborators.js'
import { videoChannelLogosRouter } from './video-channel-logos.js'
+import { scheduleActorRefreshIfNeeded } from '@server/lib/activitypub/actors/refresh.js'
const auditLogger = auditLoggerFactory('channels')
@@ -307,9 +308,7 @@ async function getVideoChannel (req: express.Request, res: express.Response) {
const id = res.locals.videoChannel.id
const videoChannel = await Hooks.wrapObject(res.locals.videoChannel, 'filter:api.video-channel.get.result', { id })
- if (videoChannel.isOutdated()) {
- JobQueue.Instance.createJobAsync({ type: 'activitypub-refresher', payload: { type: 'actor', url: videoChannel.Actor.url } })
- }
+ scheduleActorRefreshIfNeeded(videoChannel.Actor)
return res.json(videoChannel.toFormattedJSON())
}
diff --git a/server/core/controllers/api/video-playlist.ts b/server/core/controllers/api/video-playlist.ts
index 5f6a9bba15..af9d62d9e9 100644
--- a/server/core/controllers/api/video-playlist.ts
+++ b/server/core/controllers/api/video-playlist.ts
@@ -13,7 +13,7 @@ import {
VideoPlaylistUpdate
} from '@peertube/peertube-models'
import { uuidToShort } from '@peertube/peertube-node-utils'
-import { scheduleRefreshIfNeeded } from '@server/lib/activitypub/playlists/index.js'
+import { schedulePlaylistRefreshIfNeeded } from '@server/lib/activitypub/playlists/index.js'
import { Hooks } from '@server/lib/plugins/hooks.js'
import {
generateThumbnailForPlaylist,
@@ -170,7 +170,7 @@ async function listVideoPlaylists (req: express.Request, res: express.Response)
function getVideoPlaylist (req: express.Request, res: express.Response) {
const videoPlaylist = res.locals.videoPlaylistSummary
- scheduleRefreshIfNeeded(videoPlaylist)
+ schedulePlaylistRefreshIfNeeded(videoPlaylist)
return res.json(videoPlaylist.toFormattedJSON())
}
diff --git a/server/core/controllers/api/videos/index.ts b/server/core/controllers/api/videos/index.ts
index 086b343dc1..7040c59333 100644
--- a/server/core/controllers/api/videos/index.ts
+++ b/server/core/controllers/api/videos/index.ts
@@ -1,5 +1,6 @@
import { HttpStatusCode, VideoChannelActivityAction } from '@peertube/peertube-models'
import { pickCommonVideoQuery } from '@server/helpers/query.js'
+import { scheduleVideoRefreshIfNeeded } from '@server/lib/activitypub/videos/index.js'
import { openapiOperationDoc } from '@server/middlewares/doc.js'
import { getServerActor } from '@server/models/application/application.js'
import { VideoChannelActivityModel } from '@server/models/video/video-channel-activity.js'
@@ -10,7 +11,6 @@ import { logger } from '../../../helpers/logger.js'
import { getFormattedObjects } from '../../../helpers/utils.js'
import { VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES, VIDEO_PRIVACIES } from '../../../initializers/constants.js'
import { sequelizeTypescript } from '../../../initializers/database.js'
-import { JobQueue } from '../../../lib/job-queue/index.js'
import { Hooks } from '../../../lib/plugins/hooks.js'
import {
apiRateLimiter,
@@ -142,9 +142,7 @@ async function getVideo (req: express.Request, res: express.Response) {
// Filter may return null/undefined value to forbid video access
if (!video) return res.sendStatus(HttpStatusCode.NOT_FOUND_404)
- if (video.isOutdated()) {
- JobQueue.Instance.createJobAsync({ type: 'activitypub-refresher', payload: { type: 'video', url: video.url } })
- }
+ scheduleVideoRefreshIfNeeded(video)
return res.json(video.toFormattedDetailsJSON())
}
diff --git a/server/core/controllers/api/videos/source.ts b/server/core/controllers/api/videos/source.ts
index 02f4cfd210..e887f19ef5 100644
--- a/server/core/controllers/api/videos/source.ts
+++ b/server/core/controllers/api/videos/source.ts
@@ -1,7 +1,7 @@
import { buildAspectRatio } from '@peertube/peertube-core-utils'
import { HttpStatusCode, VideoChannelActivityAction, VideoState } from '@peertube/peertube-models'
import { sequelizeTypescript } from '@server/initializers/database.js'
-import { CreateJobArgument, CreateJobOptions, JobQueue } from '@server/lib/job-queue/index.js'
+import { CreateJobTypeAndPayload, CreateJobOptions, JobQueue } from '@server/lib/job-queue/index.js'
import { Hooks } from '@server/lib/plugins/hooks.js'
import { regenerateLocalVideoThumbnailsFromVideoIfNeeded } from '@server/lib/thumbnail.js'
import { setupUploadResumableRoutes } from '@server/lib/uploadx.js'
@@ -178,7 +178,7 @@ async function replaceVideoSourceResumable (req: express.Request, res: express.R
}
async function addVideoJobsAfterUpload (video: MVideoFullLight, videoFile: MVideoFile) {
- const jobs: (CreateJobArgument & CreateJobOptions)[] = [
+ const jobs: (CreateJobTypeAndPayload & CreateJobOptions)[] = [
{
type: 'manage-video-torrent' as const,
payload: {
diff --git a/server/core/lib/activitypub/actors/refresh.ts b/server/core/lib/activitypub/actors/refresh.ts
index 75b566f893..1e13bef165 100644
--- a/server/core/lib/activitypub/actors/refresh.ts
+++ b/server/core/lib/activitypub/actors/refresh.ts
@@ -2,9 +2,10 @@ import { HttpStatusCode } from '@peertube/peertube-models'
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
import { CachePromiseFactory } from '@server/helpers/promise-cache.js'
import { PeerTubeRequestError } from '@server/helpers/requests.js'
+import { JobQueue } from '@server/lib/job-queue/job-queue.js'
import { ActorLoadByUrlType } from '@server/lib/model-loaders/index.js'
import { ActorModel } from '@server/models/actor/actor.js'
-import { MActorFull, MActorOutdated } from '@server/types/models/index.js'
+import { MActorFull, MActorOutdated, MActorUrl } from '@server/types/models/index.js'
import { fetchRemoteActor } from './shared/index.js'
import { APActorUpdater } from './updater.js'
import { getUrlFromWebfinger } from './webfinger.js'
@@ -16,20 +17,28 @@ type RefreshOptions<T> = {
fetchedType: Extract<ActorLoadByUrlType, 'all'> | 'partial'
}
+// ---------------------------------------------------------------------------
+
const promiseCache = new CachePromiseFactory(
doRefresh,
(options: RefreshOptions<MActorFull | MActorOutdated>) => options.actor.id + ''
)
-function refreshActorIfNeeded<T extends MActorFull | MActorOutdated> (options: RefreshOptions<T>): RefreshResult<T> {
+export function refreshActorIfNeeded<T extends MActorFull | MActorOutdated> (options: RefreshOptions<T>): RefreshResult<T> {
const actorArg = options.actor
if (!actorArg.isOutdated()) return Promise.resolve({ actor: actorArg, refreshed: false })
return promiseCache.run(options)
}
-export {
- refreshActorIfNeeded
+export function scheduleActorRefreshIfNeeded (actor: MActorOutdated & MActorUrl) {
+ if (!actor.isOutdated()) return
+
+ JobQueue.Instance.createJobAsync({
+ type: 'activitypub-refresher',
+ payload: { type: 'actor', url: actor.url },
+ deduplicationId: `refresh-actor-${actor.url}`
+ })
}
// ---------------------------------------------------------------------------
diff --git a/server/core/lib/activitypub/playlists/get.ts b/server/core/lib/activitypub/playlists/get.ts
index 32d1f4b6e2..eb7567c297 100644
--- a/server/core/lib/activitypub/playlists/get.ts
+++ b/server/core/lib/activitypub/playlists/get.ts
@@ -2,14 +2,14 @@ import { VideoPlaylistModel } from '@server/models/video/video-playlist.js'
import { MVideoPlaylistFullSummary } from '@server/types/models/index.js'
import { getAPId } from '../activity.js'
import { createOrUpdateVideoPlaylist } from './create-update.js'
-import { scheduleRefreshIfNeeded } from './refresh.js'
+import { schedulePlaylistRefreshIfNeeded } from './refresh.js'
import { fetchRemoteVideoPlaylist } from './shared/index.js'
export async function getOrCreateAPVideoPlaylist (playlistUrl: string): Promise<MVideoPlaylistFullSummary> {
const playlistFromDatabase = await VideoPlaylistModel.loadByUrlWithAccountAndChannelSummary(playlistUrl)
if (playlistFromDatabase) {
- scheduleRefreshIfNeeded(playlistFromDatabase)
+ schedulePlaylistRefreshIfNeeded(playlistFromDatabase)
return playlistFromDatabase
}
diff --git a/server/core/lib/activitypub/playlists/refresh.ts b/server/core/lib/activitypub/playlists/refresh.ts
index 1e749f2810..625cc9ef24 100644
--- a/server/core/lib/activitypub/playlists/refresh.ts
+++ b/server/core/lib/activitypub/playlists/refresh.ts
@@ -6,10 +6,14 @@ import { MVideoPlaylist, MVideoPlaylistOwnerDefault } from '@server/types/models
import { createOrUpdateVideoPlaylist } from './create-update.js'
import { fetchRemoteVideoPlaylist } from './shared/index.js'
-function scheduleRefreshIfNeeded (playlist: MVideoPlaylist) {
+function schedulePlaylistRefreshIfNeeded (playlist: MVideoPlaylist) {
if (!playlist.isOutdated()) return
- JobQueue.Instance.createJobAsync({ type: 'activitypub-refresher', payload: { type: 'video-playlist', url: playlist.url } })
+ JobQueue.Instance.createJobAsync({
+ type: 'activitypub-refresher',
+ payload: { type: 'video-playlist', url: playlist.url },
+ deduplicationId: `refresh-video-playlist-${playlist.url}`
+ })
}
async function refreshVideoPlaylistIfNeeded (videoPlaylist: MVideoPlaylistOwnerDefault): Promise<MVideoPlaylistOwnerDefault> {
@@ -51,5 +55,5 @@ async function refreshVideoPlaylistIfNeeded (videoPlaylist: MVideoPlaylistOwnerD
export {
refreshVideoPlaylistIfNeeded,
- scheduleRefreshIfNeeded
+ schedulePlaylistRefreshIfNeeded
}
diff --git a/server/core/lib/activitypub/videos/get.ts b/server/core/lib/activitypub/videos/get.ts
index d667b433f9..13d697f2c6 100644
--- a/server/core/lib/activitypub/videos/get.ts
+++ b/server/core/lib/activitypub/videos/get.ts
@@ -1,8 +1,7 @@
import { APObjectId } from '@peertube/peertube-models'
import { retryTransactionWrapper } from '@server/helpers/database-utils.js'
import { logger } from '@server/helpers/logger.js'
-import { JobQueue } from '@server/lib/job-queue/index.js'
-import { loadVideoByUrl, VideoLoadByUrlType } from '@server/lib/model-loaders/index.js'
+import { loadVideoByUrl } from '@server/lib/model-loaders/index.js'
import {
MVideoAccountLightBlacklistAllFiles,
MVideoImmutable,
@@ -10,10 +9,10 @@ import {
MVideoThumbnailBlacklist
} from '@server/types/models/index.js'
import { getAPId } from '../activity.js'
-import { refreshVideoIfNeeded } from './refresh.js'
+import { refreshVideoIfNeeded, scheduleVideoRefreshIfNeeded } from './refresh.js'
import { APVideoCreator, fetchRemoteVideo, SyncParam, syncVideoExternalAttributes } from './shared/index.js'
-type GetVideoResult <T> = Promise<{
+type GetVideoResult<T> = Promise<{
video: T
created: boolean
autoBlacklisted?: boolean
@@ -55,12 +54,22 @@ export async function getOrCreateAPVideo (
// Get video url
const videoUrl = getAPId(options.videoObject)
- let videoFromDatabase = await loadVideoByUrl(videoUrl, fetchType)
+ const videoFromDatabase = await loadVideoByUrl(videoUrl, fetchType)
if (videoFromDatabase) {
- if (allowRefresh === true) {
- // Typings ensure allowRefresh === false in unsafe-only-immutable-attributes fetch type
- videoFromDatabase = await scheduleRefresh(videoFromDatabase as MVideoThumbnail, fetchType, syncParam)
+ // We know that allowRefresh === false on `unsafe-only-immutable-attributes` fetch type because of type definitions
+ let video = videoFromDatabase as MVideoThumbnail
+
+ if (allowRefresh === true && video.isOutdated()) {
+ if (syncParam.refreshVideo === true) {
+ video = await refreshVideoIfNeeded({
+ video,
+ fetchedType: fetchType,
+ syncParam
+ })
+ } else {
+ scheduleVideoRefreshIfNeeded(video)
+ }
}
return { video: videoFromDatabase, created: false }
@@ -107,28 +116,3 @@ export async function maybeGetOrCreateAPVideo (options: GetVideoParamAll | GetVi
return { video: undefined, created: false }
}
}
-
-// ---------------------------------------------------------------------------
-// Private
-// ---------------------------------------------------------------------------
-
-async function scheduleRefresh (video: MVideoThumbnail, fetchType: VideoLoadByUrlType, syncParam: SyncParam) {
- if (!video.isOutdated()) return video
-
- const refreshOptions = {
- video,
- fetchedType: fetchType,
- syncParam
- }
-
- if (syncParam.refreshVideo === true) {
- return refreshVideoIfNeeded(refreshOptions)
- }
-
- await JobQueue.Instance.createJob({
- type: 'activitypub-refresher',
- payload: { type: 'video', url: video.url }
- })
-
- return video
-}
diff --git a/server/core/lib/activitypub/videos/refresh.ts b/server/core/lib/activitypub/videos/refresh.ts
index a407843d00..e7e3ec012a 100644
--- a/server/core/lib/activitypub/videos/refresh.ts
+++ b/server/core/lib/activitypub/videos/refresh.ts
@@ -1,14 +1,25 @@
+import { HttpStatusCode } from '@peertube/peertube-models'
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
import { PeerTubeRequestError } from '@server/helpers/requests.js'
+import { JobQueue } from '@server/lib/job-queue/job-queue.js'
import { VideoLoadByUrlType } from '@server/lib/model-loaders/index.js'
import { VideoModel } from '@server/models/video/video.js'
-import { MVideoAccountLightBlacklistAllFiles, MVideoThumbnail } from '@server/types/models/index.js'
-import { HttpStatusCode } from '@peertube/peertube-models'
+import { MVideo, MVideoFullLight, MVideoThumbnail } from '@server/types/models/index.js'
import { ActorFollowHealthCache } from '../../actor-follow-health-cache.js'
import { fetchRemoteVideo, SyncParam, syncVideoExternalAttributes } from './shared/index.js'
import { APVideoUpdater } from './updater.js'
-async function refreshVideoIfNeeded (options: {
+export function scheduleVideoRefreshIfNeeded (video: MVideo) {
+ if (!video.isOutdated()) return
+
+ JobQueue.Instance.createJobAsync({
+ type: 'activitypub-refresher',
+ deduplicationId: `video-refresh-${video.url}`,
+ payload: { type: 'video', url: video.url }
+ })
+}
+
+export async function refreshVideoIfNeeded (options: {
video: MVideoThumbnail
fetchedType: VideoLoadByUrlType
syncParam: SyncParam
@@ -17,7 +28,7 @@ async function refreshVideoIfNeeded (options: {
// We need more attributes if the argument video was fetched with not enough joints
const video = options.fetchedType === 'all'
- ? options.video as MVideoAccountLightBlacklistAllFiles
+ ? options.video as MVideoFullLight
: await VideoModel.loadByUrlAndPopulateAccountAndFiles(options.video.url)
const lTags = loggerTagsFactory('ap', 'video', 'refresh', video.uuid, video.url)
@@ -62,9 +73,3 @@ async function refreshVideoIfNeeded (options: {
return video
}
}
-
-// ---------------------------------------------------------------------------
-
-export {
- refreshVideoIfNeeded
-}
diff --git a/server/core/lib/files-cache/video-captions-file-cache.ts b/server/core/lib/files-cache/video-captions-file-cache.ts
index 2a7c1e6585..76ce2491b3 100644
--- a/server/core/lib/files-cache/video-captions-file-cache.ts
+++ b/server/core/lib/files-cache/video-captions-file-cache.ts
@@ -2,7 +2,7 @@ import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
import { VideoCaptionModel } from '@server/models/video/video-caption.js'
import { VideoModel } from '@server/models/video/video.js'
import { MVideoCaption } from '@server/types/models/index.js'
-import { JobQueue } from '../job-queue/job-queue.js'
+import { scheduleVideoRefreshIfNeeded } from '../activitypub/videos/refresh.js'
import { AbstractFileCache } from './shared/abstract-file-cache.js'
const lTags = loggerTagsFactory('lazy-load', 'video-captions')
@@ -24,7 +24,7 @@ export class VideoCaptionsFileCache extends AbstractFileCache<MVideoCaption> {
try {
const video = await VideoModel.load(model.videoId)
- JobQueue.Instance.createJobAsync({ type: 'activitypub-refresher', payload: { type: 'video', url: video.url } })
+ scheduleVideoRefreshIfNeeded(video)
} catch (err) {
logger.error('Error while refreshing video for lazy fetch', { ...lTags(), err })
}
diff --git a/server/core/lib/files-cache/video-storyboards-image-file-cache.ts b/server/core/lib/files-cache/video-storyboards-image-file-cache.ts
index a51b8228aa..ca38041a8f 100644
--- a/server/core/lib/files-cache/video-storyboards-image-file-cache.ts
+++ b/server/core/lib/files-cache/video-storyboards-image-file-cache.ts
@@ -2,7 +2,7 @@ import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
import { StoryboardModel } from '@server/models/video/storyboard.js'
import { VideoModel } from '@server/models/video/video.js'
import { MStoryboard } from '@server/types/models/index.js'
-import { JobQueue } from '../job-queue/job-queue.js'
+import { scheduleVideoRefreshIfNeeded } from '../activitypub/videos/index.js'
import { AbstractImageFileCache } from './shared/abstract-image-file-cache.js'
const lTags = loggerTagsFactory('lazy-load', 'video-storyboards')
@@ -24,7 +24,7 @@ export class VideoStoryboardsImageFileCache extends AbstractImageFileCache<MStor
try {
const video = await VideoModel.load(model.videoId)
- JobQueue.Instance.createJobAsync({ type: 'activitypub-refresher', payload: { type: 'video', url: video.url } })
+ scheduleVideoRefreshIfNeeded(video)
} catch (err) {
logger.error('Error while refreshing video for lazy fetch', { ...lTags(), err })
}
diff --git a/server/core/lib/files-cache/video-thumbnails-image-file-cache.ts b/server/core/lib/files-cache/video-thumbnails-image-file-cache.ts
index 8fedd42347..9985bbcfc0 100644
--- a/server/core/lib/files-cache/video-thumbnails-image-file-cache.ts
+++ b/server/core/lib/files-cache/video-thumbnails-image-file-cache.ts
@@ -2,7 +2,7 @@ import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
import { ThumbnailModel } from '@server/models/video/thumbnail.js'
import { VideoModel } from '@server/models/video/video.js'
import { MThumbnail } from '@server/types/models/index.js'
-import { JobQueue } from '../job-queue/job-queue.js'
+import { scheduleVideoRefreshIfNeeded } from '../activitypub/videos/refresh.js'
import { AbstractImageFileCache } from './shared/abstract-image-file-cache.js'
const lTags = loggerTagsFactory('lazy-load', 'video-thumbnails')
@@ -24,7 +24,7 @@ export class VideoThumbnailsImageFileCache extends AbstractImageFileCache<MThumb
try {
const video = await VideoModel.load(model.videoId)
- JobQueue.Instance.createJobAsync({ type: 'activitypub-refresher', payload: { type: 'video', url: video.url } })
+ scheduleVideoRefreshIfNeeded(video)
} catch (err) {
logger.error('Error while refreshing video for lazy fetch', { ...lTags(), err })
}
diff --git a/server/core/lib/job-queue/job-queue.ts b/server/core/lib/job-queue/job-queue.ts
index 6ac3753a73..4a1107c35b 100644
--- a/server/core/lib/job-queue/job-queue.ts
+++ b/server/core/lib/job-queue/job-queue.ts
@@ -76,7 +76,7 @@ import { processVideoTranscoding } from './handlers/video-transcoding.js'
import { processVideoTranscription } from './handlers/video-transcription.js'
import { processVideosViewsStats } from './handlers/video-views-stats.js'
-export type CreateJobArgument =
+export type CreateJobTypeAndPayload =
| { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload }
| { type: 'activitypub-http-broadcast-parallel', payload: ActivitypubHttpBroadcastPayload }
| { type: 'activitypub-http-unicast', payload: ActivitypubHttpUnicastPayload }
@@ -110,6 +110,7 @@ export type CreateJobOptions = {
delay?: number
priority?: number
failParentOnFailure?: boolean
+ deduplicationId?: string
}
const handlers: { [id in JobType]: (job: Job) => Promise<any> } = {
@@ -356,12 +357,12 @@ class JobQueue {
// ---------------------------------------------------------------------------
- createJobAsync (options: CreateJobArgument & CreateJobOptions): void {
+ createJobAsync (options: CreateJobTypeAndPayload & CreateJobOptions): void {
this.createJob(options)
.catch(err => logger.error('Cannot create job.', { err, options }))
}
- createJob (options: CreateJobArgument & CreateJobOptions | undefined) {
+ createJob (options: CreateJobTypeAndPayload & CreateJobOptions | undefined) {
if (!options) return
const queue: Queue = this.queues[options.type]
@@ -375,7 +376,7 @@ class JobQueue {
return queue.add('job', options.payload, jobOptions)
}
- createSequentialJobFlow (...jobs: ((CreateJobArgument & CreateJobOptions) | undefined)[]) {
+ createSequentialJobFlow (...jobs: ((CreateJobTypeAndPayload & CreateJobOptions) | undefined)[]) {
let lastJob: FlowJob
logger.debug('Creating jobs in local job queue', { jobs })
@@ -395,7 +396,7 @@ class JobQueue {
return this.flowProducer.add(lastJob)
}
- createJobWithChildren (parent: CreateJobArgument & CreateJobOptions, children: (CreateJobArgument & CreateJobOptions)[]) {
+ createJobWithChildren (parent: CreateJobTypeAndPayload & CreateJobOptions, children: (CreateJobTypeAndPayload & CreateJobOptions)[]) {
return this.flowProducer.add({
...this.buildJobFlowOption(parent),
@@ -403,7 +404,7 @@ class JobQueue {
})
}
- private buildJobFlowOption (job: CreateJobArgument & CreateJobOptions): FlowJob {
+ private buildJobFlowOption (job: CreateJobTypeAndPayload & CreateJobOptions): FlowJob {
return {
name: 'job',
data: job.payload,
@@ -423,6 +424,12 @@ class JobQueue {
priority: options.priority,
delay: options.delay,
+ deduplication: options.deduplicationId
+ ? {
+ id: options.deduplicationId
+ }
+ : undefined,
+
...this.buildJobRemovalOptions(type)
}
}
diff --git a/server/core/lib/sync-channel.ts b/server/core/lib/sync-channel.ts
index 11fc929145..09190247e1 100644
--- a/server/core/lib/sync-channel.ts
+++ b/server/core/lib/sync-channel.ts
@@ -6,7 +6,7 @@ import { buildYoutubeDLImport } from '@server/lib/video-pre-import.js'
import { UserModel } from '@server/models/user/user.js'
import { VideoImportModel } from '@server/models/video/video-import.js'
import { MChannelAccountDefault, MChannelSync } from '@server/types/models/index.js'
-import { CreateJobArgument, JobQueue } from './job-queue/index.js'
+import { CreateJobTypeAndPayload, JobQueue } from './job-queue/index.js'
import { ServerConfigManager } from './server-config-manager.js'
import { buildRetryImportJob } from './video-post-import.js'
import { getLeastPrivatePrivacy } from './video.js'
@@ -49,7 +49,7 @@ export async function synchronizeChannel (options: {
{ targetUrls, ...lTags() }
)
- const children: CreateJobArgument[] = []
+ const children: CreateJobTypeAndPayload[] = []
let buildJobErrors = 0
@@ -108,7 +108,7 @@ export async function synchronizeChannel (options: {
}
// Will update the channel sync status
- const parent: CreateJobArgument = {
+ const parent: CreateJobTypeAndPayload = {
type: 'after-video-channel-import',
payload: {
channelSyncId: channelSync?.id,
diff --git a/server/core/lib/transcoding/shared/job-builders/transcoding-job-queue-builder.ts b/server/core/lib/transcoding/shared/job-builders/transcoding-job-queue-builder.ts
index a531ef83ac..870a5de689 100644
--- a/server/core/lib/transcoding/shared/job-builders/transcoding-job-queue-builder.ts
+++ b/server/core/lib/transcoding/shared/job-builders/transcoding-job-queue-builder.ts
@@ -7,7 +7,7 @@ import {
VideoFileStreamType,
VideoTranscodingPayload
} from '@peertube/peertube-models'
-import { CreateJobArgument, JobQueue } from '@server/lib/job-queue/index.js'
+import { CreateJobTypeAndPayload, JobQueue } from '@server/lib/job-queue/index.js'
import { VideoJobInfoModel } from '@server/models/video/video-job-info.js'
import { MUserId, MVideo } from '@server/types/models/index.js'
import { getTranscodingJobPriority } from '../../transcoding-priority.js'
@@ -47,7 +47,7 @@ export class TranscodingJobQueueBuilder extends AbstractJobBuilder<FullPayload>
})
})
- const transcodingJobBuilderJob: CreateJobArgument = {
+ const transcodingJobBuilderJob: CreateJobTypeAndPayload = {
type: 'transcoding-job-builder',
payload: {
videoUUID: video.uuid,
diff --git a/server/core/lib/video-jobs.ts b/server/core/lib/video-jobs.ts
index 3f50ff736a..60b7dd71ed 100644
--- a/server/core/lib/video-jobs.ts
+++ b/server/core/lib/video-jobs.ts
@@ -10,7 +10,7 @@ import { CONFIG } from '@server/initializers/config.js'
import { VideoJobInfoModel } from '@server/models/video/video-job-info.js'
import { VideoModel } from '@server/models/video/video.js'
import { MVideo, MVideoFile, MVideoFullLight, MVideoUUID } from '@server/types/models/index.js'
-import { CreateJobArgument, CreateJobOptions, JobQueue } from './job-queue/job-queue.js'
+import { CreateJobTypeAndPayload, CreateJobOptions, JobQueue } from './job-queue/job-queue.js'
import { VideoStoryboardJobHandler } from './runners/index.js'
import { createTranscriptionTaskIfNeeded } from './video-captions.js'
import { moveFilesIfPrivacyChanged } from './video-privacy.js'
@@ -106,7 +106,7 @@ export async function addVideoJobsAfterCreation (options: {
}) {
const { video, videoFile, generateTranscription } = options
- const jobs: (CreateJobArgument & CreateJobOptions)[] = [
+ const jobs: (CreateJobTypeAndPayload & CreateJobOptions)[] = [
{
type: 'manage-video-torrent' as 'manage-video-torrent',
payload: {
@@ -178,7 +178,7 @@ export async function addVideoJobsAfterUpdate (options: {
oldPrivacy: VideoPrivacyType
}) {
const { video, nameChanged, oldPrivacy, isNewVideoForFederation } = options
- const jobs: CreateJobArgument[] = []
+ const jobs: CreateJobTypeAndPayload[] = []
const filePathChanged = await moveFilesIfPrivacyChanged(video, oldPrivacy)
const hls = video.getHLSPlaylist()