Skip to content

ActivityPub-federated video streaming platform using P2P directly in your web browser

Prevent duplicated thumbnails

Chocobozzz1d2f40ad76d7
diff --git a/server/core/controllers/api/video-playlist.ts b/server/core/controllers/api/video-playlist.ts
index 05a70328de..1dcd447692 100644
--- a/server/core/controllers/api/video-playlist.ts
+++ b/server/core/controllers/api/video-playlist.ts
@@ -16,13 +16,14 @@ import { uuidToShort } from '@peertube/peertube-node-utils'
import { schedulePlaylistRefreshIfNeeded } from '@server/lib/activitypub/playlists/index.js'
import { Hooks } from '@server/lib/plugins/hooks.js'
import {
- generateThumbnailForPlaylist,
+ generateThumbnailForPlaylistIfNeeded,
+ regeneratePlaylistThumbnailIfNeeded,
reorderPlaylistOrElementsPosition,
sendPlaylistPositionUpdateOfChannel
} from '@server/lib/video-playlist.js'
import { getServerActor } from '@server/models/application/application.js'
import { VideoChannelActivityModel } from '@server/models/video/video-channel-activity.js'
-import { MVideoPlaylistFull, MVideoPlaylistThumbnail } from '@server/types/models/index.js'
+import { MVideoPlaylistFull } from '@server/types/models/index.js'
import express from 'express'
import { resetSequelizeInstance, retryTransactionWrapper } from '../../helpers/database-utils.js'
import { createReqFiles } from '../../helpers/express-utils.js'
@@ -91,7 +92,7 @@ videoPlaylistRouter.put(
authenticate,
reqThumbnailFile,
asyncMiddleware(videoPlaylistsUpdateValidator),
- asyncRetryTransactionMiddleware(updateVideoPlaylist)
+ asyncMiddleware(updateVideoPlaylist)
)
videoPlaylistRouter.delete(
@@ -255,112 +256,119 @@ async function updateVideoPlaylist (req: express.Request, res: express.Response)
const wasPrivatePlaylist = playlist.privacy === VideoPlaylistPrivacy.PRIVATE
const wasNotPrivatePlaylist = playlist.privacy !== VideoPlaylistPrivacy.PRIVATE
- let removedFromChannel: { id: number, position: number }
-
const thumbnailField = req.files?.['thumbnailfile']
- const thumbnailModels = thumbnailField
- ? await createLocalPlaylistThumbnailsFromImage({
- inputPath: thumbnailField[0].path,
- playlist,
- automaticallyGenerated: false
- })
- : undefined
try {
- await sequelizeTypescript.transaction(async t => {
- const newChannel = res.locals.videoChannel
- const user = res.locals.oauth.token.User
-
- // Had a channel, but the user changed it (to null or another channel)
- if (playlist.videoChannelId && body.videoChannelId !== undefined && body.videoChannelId !== playlist.videoChannelId) {
- await VideoChannelActivityModel.addPlaylistActivity({
- action: VideoChannelActivityAction.REMOVE_CHANNEL_OWNERSHIP,
- user,
- channel: playlist.VideoChannel,
+ // A concurrent request updating the same playlist thumbnails can hit the unique index on thumbnail size
+ await retryTransactionWrapper(async () => {
+ // Build the thumbnails here so it's rebuilt on retry
+ const thumbnailModels = thumbnailField
+ ? await createLocalPlaylistThumbnailsFromImage({
+ inputPath: thumbnailField[0].path,
playlist,
- transaction: t
+ automaticallyGenerated: false
})
-
- removedFromChannel = {
- id: playlist.videoChannelId,
- position: playlist.videoChannelPosition
+ : undefined
+
+ return sequelizeTypescript.transaction(async t => {
+ // If this is a retried attempt, sequelize would otherwise think this instance has not changed
+ await playlist.reload({ transaction: t })
+
+ let removedFromChannel: { id: number, position: number }
+
+ const newChannel = res.locals.videoChannel
+ const user = res.locals.oauth.token.User
+
+ // Had a channel, but the user changed it (to null or another channel)
+ if (playlist.videoChannelId && body.videoChannelId !== undefined && body.videoChannelId !== playlist.videoChannelId) {
+ await VideoChannelActivityModel.addPlaylistActivity({
+ action: VideoChannelActivityAction.REMOVE_CHANNEL_OWNERSHIP,
+ user,
+ channel: playlist.VideoChannel,
+ playlist,
+ transaction: t
+ })
+
+ removedFromChannel = {
+ id: playlist.videoChannelId,
+ position: playlist.videoChannelPosition
+ }
+
+ playlist.videoChannelId = null
+ playlist.VideoChannel = null
}
- playlist.videoChannelId = null
- playlist.VideoChannel = null
- }
-
- if (newChannel && newChannel.id !== playlist.videoChannelId) {
- await VideoChannelActivityModel.addPlaylistActivity({
- action: VideoChannelActivityAction.CREATE_CHANNEL_OWNERSHIP,
- user,
- channel: newChannel,
- playlist,
- transaction: t
- })
-
- playlist.videoChannelPosition = await VideoPlaylistModel.getNextPositionOf({
- videoChannelId: newChannel.id,
- transaction: t
- })
-
- playlist.videoChannelId = newChannel.id
- playlist.VideoChannel = newChannel
- } else if (newChannel) {
- await VideoChannelActivityModel.addPlaylistActivity({
- action: VideoChannelActivityAction.UPDATE,
- user: res.locals.oauth.token.User,
- channel: newChannel,
- playlist,
- transaction: t
- })
- }
+ if (newChannel && newChannel.id !== playlist.videoChannelId) {
+ await VideoChannelActivityModel.addPlaylistActivity({
+ action: VideoChannelActivityAction.CREATE_CHANNEL_OWNERSHIP,
+ user,
+ channel: newChannel,
+ playlist,
+ transaction: t
+ })
+
+ playlist.videoChannelPosition = await VideoPlaylistModel.getNextPositionOf({
+ videoChannelId: newChannel.id,
+ transaction: t
+ })
+
+ playlist.videoChannelId = newChannel.id
+ playlist.VideoChannel = newChannel
+ } else if (newChannel) {
+ await VideoChannelActivityModel.addPlaylistActivity({
+ action: VideoChannelActivityAction.UPDATE,
+ user: res.locals.oauth.token.User,
+ channel: newChannel,
+ playlist,
+ transaction: t
+ })
+ }
- if (body.displayName !== undefined) playlist.name = body.displayName
- if (body.description !== undefined) playlist.description = body.description
+ if (body.displayName !== undefined) playlist.name = body.displayName
+ if (body.description !== undefined) playlist.description = body.description
- if (body.privacy !== undefined) {
- playlist.privacy = forceNumber(body.privacy) as VideoPlaylistPrivacyType
+ if (body.privacy !== undefined) {
+ playlist.privacy = forceNumber(body.privacy) as VideoPlaylistPrivacyType
- if (wasNotPrivatePlaylist === true && playlist.privacy === VideoPlaylistPrivacy.PRIVATE) {
- await sendDeleteVideoPlaylist(playlist, t)
+ if (wasNotPrivatePlaylist === true && playlist.privacy === VideoPlaylistPrivacy.PRIVATE) {
+ await sendDeleteVideoPlaylist(playlist, t)
+ }
}
- }
- const playlistUpdated = await playlist.save({ transaction: t })
+ const playlistUpdated = await playlist.save({ transaction: t })
- if (thumbnailModels) {
- await playlistUpdated.replaceAndSaveThumbnails(thumbnailModels, t)
- }
+ if (thumbnailModels) {
+ await playlistUpdated.replaceAndSaveThumbnails(thumbnailModels, t)
+ }
- const isNewPlaylist = wasPrivatePlaylist && playlistUpdated.privacy !== VideoPlaylistPrivacy.PRIVATE
+ const isNewPlaylist = wasPrivatePlaylist && playlistUpdated.privacy !== VideoPlaylistPrivacy.PRIVATE
- if (isNewPlaylist) {
- await sendCreateVideoPlaylist(playlistUpdated, t)
- } else {
- await sendUpdateVideoPlaylist(playlistUpdated, t)
- }
+ if (isNewPlaylist) {
+ await sendCreateVideoPlaylist(playlistUpdated, t)
+ } else {
+ await sendUpdateVideoPlaylist(playlistUpdated, t)
+ }
- if (removedFromChannel) {
- await VideoPlaylistModel.increasePositionOf({
- videoChannelId: removedFromChannel.id,
- fromPosition: removedFromChannel.position,
- by: -1,
- transaction: t
- })
+ if (removedFromChannel) {
+ await VideoPlaylistModel.increasePositionOf({
+ videoChannelId: removedFromChannel.id,
+ fromPosition: removedFromChannel.position,
+ by: -1,
+ transaction: t
+ })
- await sendPlaylistPositionUpdateOfChannel(removedFromChannel.id, t)
- }
+ await sendPlaylistPositionUpdateOfChannel(removedFromChannel.id, t)
+ }
- logger.info('Video playlist %s updated.', playlist.uuid)
+ logger.info('Video playlist %s updated.', playlist.uuid)
- return playlistUpdated
- })
+ return playlistUpdated
+ })
+ }, { retryUniqueConstraintViolation: true })
} catch (err) {
logger.debug('Cannot update the video playlist.', { err })
- // If the transaction is retried, sequelize will think the object has not changed
- // So we need to restore the previous fields
+ // Leave the instance clean for any other code path that might reuse it after this handler returns
await resetSequelizeInstance(playlist)
throw err
@@ -447,9 +455,7 @@ async function addVideoInPlaylist (req: express.Request, res: express.Response)
})
// If the user did not set a thumbnail, automatically take the video thumbnail
- if (videoPlaylist.shouldGenerateThumbnailWithNewElement(playlistElement)) {
- await generateThumbnailForPlaylist(videoPlaylist, video)
- }
+ await generateThumbnailForPlaylistIfNeeded({ videoPlaylist, element: playlistElement, video })
sendUpdateVideoPlaylist(videoPlaylist, undefined)
.catch(err => logger.error('Cannot send video playlist update.', { err }))
@@ -532,8 +538,8 @@ async function removeVideoFromPlaylist (req: express.Request, res: express.Respo
})
// Do we need to regenerate the default thumbnail?
- if (positionToDelete === 1 && videoPlaylist.hasGeneratedThumbnail()) {
- await regeneratePlaylistThumbnail(videoPlaylist)
+ if (positionToDelete === 1) {
+ await regeneratePlaylistThumbnailIfNeeded(videoPlaylist)
}
sendUpdateVideoPlaylist(videoPlaylist, undefined)
@@ -569,8 +575,8 @@ async function reorderVideosOfPlaylist (req: express.Request, res: express.Respo
})
// The first element changed
- if ((start === 1 || insertAfter === 0) && videoPlaylist.hasGeneratedThumbnail()) {
- await regeneratePlaylistThumbnail(videoPlaylist)
+ if (start === 1 || insertAfter === 0) {
+ await regeneratePlaylistThumbnailIfNeeded(videoPlaylist)
}
await sendUpdateVideoPlaylist(videoPlaylist, undefined)
@@ -608,10 +614,3 @@ async function listVideosOfPlaylist (req: express.Request, res: express.Response
const options = { accountId: user?.Account?.id }
return res.json(getFormattedObjects(resultList.data, resultList.total, options))
}
-
-async function regeneratePlaylistThumbnail (videoPlaylist: MVideoPlaylistThumbnail) {
- await videoPlaylist.removeThumbnails(undefined)
-
- const firstElement = await VideoPlaylistElementModel.loadFirstElementWithVideoThumbnail(videoPlaylist.id)
- if (firstElement) await generateThumbnailForPlaylist(videoPlaylist, firstElement.Video)
-}
diff --git a/server/core/helpers/custom-validators/activitypub/misc.ts b/server/core/helpers/custom-validators/activitypub/misc.ts
index 9c2b59b732..8185ab9ed2 100644
--- a/server/core/helpers/custom-validators/activitypub/misc.ts
+++ b/server/core/helpers/custom-validators/activitypub/misc.ts
@@ -66,12 +66,23 @@ export function setValidRemoteIcon (entity: any) {
if (entity.icon && !isArray(entity.icon)) entity.icon = [ entity.icon ]
if (!entity.icon) entity.icon = []
+ const existingSizes = new Set<string>()
+
entity.icon = entity.icon.filter(icon => {
- return icon.type === 'Image' &&
+ const isValid = icon.type === 'Image' &&
isActivityPubUrlValid(icon.url) &&
!!MIMETYPES.IMAGE.MIMETYPE_EXT[icon.mediaType] &&
validator.default.isInt(icon.width + '', { min: 0 }) &&
validator.default.isInt(icon.height + '', { min: 0 })
+
+ if (!isValid) return false
+
+ // We store at most one icon per size, so ignore the next ones the remote instance may have sent
+ const size = `${icon.width}x${icon.height}`
+ if (existingSizes.has(size)) return false
+ existingSizes.add(size)
+
+ return true
})
return true
diff --git a/server/core/helpers/database-utils.ts b/server/core/helpers/database-utils.ts
index 87edf05ee0..7a05084ebf 100644
--- a/server/core/helpers/database-utils.ts
+++ b/server/core/helpers/database-utils.ts
@@ -1,71 +1,40 @@
-import retry from 'async/retry.js'
-import Bluebird from 'bluebird'
+import { sequelizeTypescript } from '@server/initializers/database.js'
import { Transaction } from 'sequelize'
import { Model } from 'sequelize-typescript'
-import { sequelizeTypescript } from '@server/initializers/database.js'
import { logger } from './logger.js'
-export function retryTransactionWrapper<T, A, B, C, D> (
- functionToRetry: (arg1: A, arg2: B, arg3: C, arg4: D) => Promise<T>,
- arg1: A,
- arg2: B,
- arg3: C,
- arg4: D
-): Promise<T>
-
-export function retryTransactionWrapper<T, A, B, C> (
- functionToRetry: (arg1: A, arg2: B, arg3: C) => Promise<T>,
- arg1: A,
- arg2: B,
- arg3: C
-): Promise<T>
-
-export function retryTransactionWrapper<T, A, B> (
- functionToRetry: (arg1: A, arg2: B) => Promise<T>,
- arg1: A,
- arg2: B
-): Promise<T>
-
-export function retryTransactionWrapper<T, A> (
- functionToRetry: (arg1: A) => Promise<T>,
- arg1: A
-): Promise<T>
-
-export function retryTransactionWrapper<T> (
- functionToRetry: () => Promise<T> | Bluebird<T>
-): Promise<T>
-
-export function retryTransactionWrapper<T> (
- functionToRetry: (...args: any[]) => Promise<T>,
- ...args: any[]
-): Promise<T> {
- return transactionRetryer<T>(callback => {
- functionToRetry.apply(null, args)
- .then((result: T) => callback(null, result))
- .catch(err => callback(err))
- })
- .catch(err => {
- logger.warn(`Cannot execute ${functionToRetry.name || 'function'} with many retries.`, { err })
- throw err
- })
+export type RetryTransactionWrapperOptions = {
+ // Also retry when the transaction failed because of a unique constraint violation
+ retryUniqueConstraintViolation?: boolean // default false
}
-export function transactionRetryer<T> (func: (err: any, data: T) => any) {
- return new Promise<T>((res, rej) => {
- retry(
- {
- times: 5,
+export async function retryTransactionWrapper<T> (
+ functionToRetry: () => Promise<T>,
+ options: RetryTransactionWrapperOptions = {}
+): Promise<T> {
+ const maxAttempts = 5
+ let attempts = 0
- errorFilter: err => {
- const willRetry = err.name === 'SequelizeDatabaseError'
- logger.debug('Maybe retrying the transaction function.', { willRetry, err, tags: [ 'sql', 'retry' ] })
- return willRetry
- }
- },
- func,
- (err, data) => err ? rej(err) : res(data)
- )
- })
+ while (true) {
+ try {
+ return await functionToRetry()
+ } catch (err) {
+ attempts++
+
+ const willRetry = attempts < maxAttempts && (
+ err?.name === 'SequelizeDatabaseError' ||
+ (options?.retryUniqueConstraintViolation === true && err?.name === 'SequelizeUniqueConstraintError')
+ )
+
+ logger.debug('Maybe retrying the transaction function.', { willRetry, err, tags: [ 'sql', 'retry' ] })
+
+ if (!willRetry) {
+ logger.warn(`Cannot execute function with many retries.`, { err, attempts, stack: err?.stack })
+
+ throw err
+ }
+ }
+ }
}
export function saveInTransactionWithRetries<T extends Pick<Model, 'save' | 'changed'>> (
diff --git a/server/core/initializers/checker-after-init.ts b/server/core/initializers/checker-after-init.ts
index ea4b96cb00..39aa98de54 100644
--- a/server/core/initializers/checker-after-init.ts
+++ b/server/core/initializers/checker-after-init.ts
@@ -386,6 +386,12 @@ function checkThumbnailsConfig () {
if (!isArray(CONFIG.THUMBNAILS.SIZES) || CONFIG.THUMBNAILS.SIZES.length === 0) {
throw new Error('thumbnails.sizes must not be empty')
}
+
+ // A video/playlist can only have one thumbnail of a given size
+ const sizes = CONFIG.THUMBNAILS.SIZES.map(s => `${s.width}x${s.height}`)
+ if (new Set(sizes).size !== sizes.length) {
+ throw new Error('thumbnails.sizes must not contain multiple sizes with the same width and height')
+ }
}
function checkBrowseVideosConfig () {
diff --git a/server/core/initializers/constants.ts b/server/core/initializers/constants.ts
index 01b0143f22..f4c2f4e4a1 100644
--- a/server/core/initializers/constants.ts
+++ b/server/core/initializers/constants.ts
@@ -62,7 +62,7 @@ import { CONFIG, registerConfigChangedHandler } from './config.js'
// ---------------------------------------------------------------------------
-export const LAST_MIGRATION_VERSION = 1090
+export const LAST_MIGRATION_VERSION = 1095
// ---------------------------------------------------------------------------
diff --git a/server/core/initializers/migrations/1095-thumbnail-unique-size.ts b/server/core/initializers/migrations/1095-thumbnail-unique-size.ts
new file mode 100644
index 0000000000..891ee9549b
--- /dev/null
+++ b/server/core/initializers/migrations/1095-thumbnail-unique-size.ts
@@ -0,0 +1,29 @@
+import * as Sequelize from 'sequelize'
+
+async function up (utils: {
+ transaction: Sequelize.Transaction
+ queryInterface: Sequelize.QueryInterface
+ sequelize: Sequelize.Sequelize
+}): Promise<void> {
+ const { transaction } = utils
+
+ // Only keep the most recent one of each size to create unique indexes
+ for (const column of [ 'videoId', 'videoPlaylistId' ]) {
+ const query = 'DELETE FROM "thumbnail" t1 ' +
+ `USING (SELECT MAX("id") AS "id", "${column}", "width", "height" FROM "thumbnail" ` +
+ `WHERE "${column}" IS NOT NULL AND "width" IS NOT NULL AND "height" IS NOT NULL ` +
+ `GROUP BY "${column}", "width", "height" HAVING COUNT(*) > 1) t2 ` +
+ `WHERE t1."${column}" = t2."${column}" AND t1."width" = t2."width" AND t1."height" = t2."height" AND t1."id" <> t2."id"`
+
+ await utils.sequelize.query(query, { transaction })
+ }
+}
+
+function down (options) {
+ throw new Error('Not implemented.')
+}
+
+export {
+ down,
+ up
+}
diff --git a/server/core/lib/activitypub/actors/get.ts b/server/core/lib/activitypub/actors/get.ts
index f554bdb611..80bb7c03ac 100644
--- a/server/core/lib/activitypub/actors/get.ts
+++ b/server/core/lib/activitypub/actors/get.ts
@@ -59,7 +59,7 @@ async function getOrCreateAPActor (
}
const creator = new APActorCreator(actorObject, ownerActor)
- actor = await retryTransactionWrapper(creator.create.bind(creator))
+ actor = await retryTransactionWrapper(() => creator.create())
created = true
accountPlaylistsUrl = actorObject.playlists
}
diff --git a/server/core/lib/activitypub/crawl.ts b/server/core/lib/activitypub/crawl.ts
index 7ea31aec7d..faf6c62add 100644
--- a/server/core/lib/activitypub/crawl.ts
+++ b/server/core/lib/activitypub/crawl.ts
@@ -1,7 +1,7 @@
-import Bluebird from 'bluebird'
-import { URL } from 'url'
import { ActivityPubOrderedCollection } from '@peertube/peertube-models'
import { retryTransactionWrapper } from '@server/helpers/database-utils.js'
+import Bluebird from 'bluebird'
+import { URL } from 'url'
import { logger } from '../../helpers/logger.js'
import { ACTIVITY_PUB, WEBSERVER } from '../../initializers/constants.js'
import { fetchAP } from './activity.js'
@@ -61,5 +61,5 @@ export async function crawlCollectionPage<T> (
}
}
- if (cleaner) await retryTransactionWrapper(cleaner, startDate)
+ if (cleaner) await retryTransactionWrapper(() => cleaner(startDate))
}
diff --git a/server/core/lib/activitypub/process/process-announce.ts b/server/core/lib/activitypub/process/process-announce.ts
index 5b48919049..971f1da1a2 100644
--- a/server/core/lib/activitypub/process/process-announce.ts
+++ b/server/core/lib/activitypub/process/process-announce.ts
@@ -17,7 +17,7 @@ async function processAnnounceActivity (options: APProcessorOptions<ActivityAnno
// Announces by accounts are not supported
if (actorAnnouncer.type !== 'Application' && actorAnnouncer.type !== 'Group') return
- return retryTransactionWrapper(processVideoShare, actorAnnouncer, activity, notify)
+ return retryTransactionWrapper(() => processVideoShare(actorAnnouncer, activity, notify))
}
// ---------------------------------------------------------------------------
diff --git a/server/core/lib/activitypub/process/process-create.ts b/server/core/lib/activitypub/process/process-create.ts
index f5664bf167..d72ba821f2 100644
--- a/server/core/lib/activitypub/process/process-create.ts
+++ b/server/core/lib/activitypub/process/process-create.ts
@@ -46,19 +46,25 @@ async function processCreateActivity (options: APProcessorOptions<ActivityCreate
// Comments will be fetched from videos
if (options.fromFetch) return
- return retryTransactionWrapper(processCreateVideoComment, activity, activityObject, byActor, options.fromFetch)
+ return retryTransactionWrapper(() => {
+ return processCreateVideoComment(activity as ActivityCreate<VideoCommentObject | string>, activityObject, byActor, false)
+ })
}
if (activityType === 'WatchAction') {
- return retryTransactionWrapper(processCreateWatchAction, activityObject)
+ return retryTransactionWrapper(() => processCreateWatchAction(activityObject))
}
if (activityType === 'CacheFile') {
- return retryTransactionWrapper(processCreateCacheFile, activity, activityObject, byActor)
+ return retryTransactionWrapper(() => {
+ return processCreateCacheFile(activity as ActivityCreate<CacheFileObject | string>, activityObject, byActor)
+ })
}
if (activityType === 'Playlist') {
- return retryTransactionWrapper(processCreatePlaylist, activity, activityObject, byActor)
+ return retryTransactionWrapper(() => {
+ return processCreatePlaylist(activity as ActivityCreate<PlaylistObject | string>, activityObject, byActor)
+ })
}
logger.warn('Unknown activity object type %s when creating activity.', activityType, { activity: activity.id })
diff --git a/server/core/lib/activitypub/process/process-delete.ts b/server/core/lib/activitypub/process/process-delete.ts
index 2663435507..945554670f 100644
--- a/server/core/lib/activitypub/process/process-delete.ts
+++ b/server/core/lib/activitypub/process/process-delete.ts
@@ -15,7 +15,9 @@ import {
MActorSignature,
MChannelAccountActor,
MChannelActor,
- MCommentOwnerVideo
+ MCommentOwnerVideo,
+ MVideoAccountLightBlacklistAllFiles,
+ MVideoPlaylistAccountThumbnail
} from '../../../types/models/index.js'
import { forwardVideoRelatedActivity } from '../send/shared/send-utils.js'
@@ -35,20 +37,20 @@ async function processDeleteActivity (options: APProcessorOptions<ActivityDelete
const accountToDelete = byActorFull.Account as MAccountActor
accountToDelete.Actor = byActorFull
- return retryTransactionWrapper(processDeleteAccount, accountToDelete)
+ return retryTransactionWrapper(() => processDeleteAccount(accountToDelete))
} else if (isChannelActor(byActorFull.type)) {
if (!byActorFull.VideoChannel) throw new Error('Actor ' + byActorFull.url + ' is a group but we cannot find it in database.')
const channelToDelete = byActorFull.VideoChannel as MChannelAccountActor & { Actor: MActorFull }
channelToDelete.Actor = byActorFull
- return retryTransactionWrapper(processDeleteVideoChannel, channelToDelete)
+ return retryTransactionWrapper(() => processDeleteVideoChannel(channelToDelete))
}
}
{
const videoCommentInstance = await VideoCommentModel.loadByUrlAndPopulateAccountAndVideoAndReply(objectUrl)
if (videoCommentInstance) {
- return retryTransactionWrapper(processDeleteVideoComment, byActor, videoCommentInstance, activity)
+ return retryTransactionWrapper(() => processDeleteVideoComment(byActor, videoCommentInstance, activity))
}
}
@@ -57,7 +59,7 @@ async function processDeleteActivity (options: APProcessorOptions<ActivityDelete
if (videoInstance) {
if (videoInstance.isLocal()) throw new Error(`Remote instance cannot delete owned video ${videoInstance.url}.`)
- return retryTransactionWrapper(processDeleteVideo, byActor, videoInstance)
+ return retryTransactionWrapper(() => processDeleteVideo(byActor, videoInstance))
}
}
@@ -66,7 +68,7 @@ async function processDeleteActivity (options: APProcessorOptions<ActivityDelete
if (videoPlaylist) {
if (videoPlaylist.isLocal()) throw new Error(`Remote instance cannot delete owned playlist ${videoPlaylist.url}.`)
- return retryTransactionWrapper(processDeleteVideoPlaylist, byActor, videoPlaylist)
+ return retryTransactionWrapper(() => processDeleteVideoPlaylist(byActor, videoPlaylist))
}
}
@@ -81,7 +83,7 @@ export {
// ---------------------------------------------------------------------------
-async function processDeleteVideo (actor: MActor, videoToDelete: VideoModel) {
+async function processDeleteVideo (actor: MActor, videoToDelete: MVideoAccountLightBlacklistAllFiles) {
logger.debug('Removing remote video "%s".', videoToDelete.uuid)
await sequelizeTypescript.transaction(async t => {
@@ -95,7 +97,7 @@ async function processDeleteVideo (actor: MActor, videoToDelete: VideoModel) {
logger.info('Remote video with uuid %s removed.', videoToDelete.uuid)
}
-async function processDeleteVideoPlaylist (actor: MActor, playlistToDelete: VideoPlaylistModel) {
+async function processDeleteVideoPlaylist (actor: MActor, playlistToDelete: MVideoPlaylistAccountThumbnail) {
logger.debug('Removing remote video playlist "%s".', playlistToDelete.uuid)
await sequelizeTypescript.transaction(async t => {
diff --git a/server/core/lib/activitypub/process/process-dislike.ts b/server/core/lib/activitypub/process/process-dislike.ts
index 0876e692bf..8666e73d27 100644
--- a/server/core/lib/activitypub/process/process-dislike.ts
+++ b/server/core/lib/activitypub/process/process-dislike.ts
@@ -10,7 +10,8 @@ import { canVideoBeFederated, federateVideoIfNeeded, maybeGetOrCreateAPVideo } f
async function processDislikeActivity (options: APProcessorOptions<ActivityDislike>) {
const { activity, byActor } = options
- return retryTransactionWrapper(processDislike, activity, byActor)
+
+ return retryTransactionWrapper(() => processDislike(activity, byActor))
}
// ---------------------------------------------------------------------------
diff --git a/server/core/lib/activitypub/process/process-flag.ts b/server/core/lib/activitypub/process/process-flag.ts
index 45cfa70248..c5ecef059f 100644
--- a/server/core/lib/activitypub/process/process-flag.ts
+++ b/server/core/lib/activitypub/process/process-flag.ts
@@ -1,9 +1,9 @@
+import { abusePredefinedReasonsMap } from '@peertube/peertube-core-utils'
+import { AbuseState, ActivityFlag } from '@peertube/peertube-models'
import { createAccountAbuse, createVideoAbuse, createVideoCommentAbuse } from '@server/lib/moderation.js'
import { AccountModel } from '@server/models/account/account.js'
import { VideoCommentModel } from '@server/models/video/video-comment.js'
import { VideoModel } from '@server/models/video/video.js'
-import { abusePredefinedReasonsMap } from '@peertube/peertube-core-utils'
-import { AbuseState, ActivityFlag } from '@peertube/peertube-models'
import { retryTransactionWrapper } from '../../../helpers/database-utils.js'
import { logger } from '../../../helpers/logger.js'
import { sequelizeTypescript } from '../../../initializers/database.js'
@@ -14,7 +14,7 @@ import { MAccountDefault, MActorSignature, MCommentOwnerVideo } from '../../../t
async function processFlagActivity (options: APProcessorOptions<ActivityFlag>) {
const { activity, byActor } = options
- return retryTransactionWrapper(processCreateAbuse, activity, byActor)
+ return retryTransactionWrapper(() => processCreateAbuse(activity, byActor))
}
// ---------------------------------------------------------------------------
@@ -35,7 +35,7 @@ async function processCreateAbuse (flag: ActivityFlag, byActor: MActorSignature)
const tags = Array.isArray(flag.tag) ? flag.tag : []
const predefinedReasons = tags.map(tag => abusePredefinedReasonsMap[tag.name])
- .filter(v => !isNaN(v))
+ .filter(v => !isNaN(v))
const startAt = flag.startAt
const endAt = flag.endAt
diff --git a/server/core/lib/activitypub/process/process-follow.ts b/server/core/lib/activitypub/process/process-follow.ts
index aa807dc758..a9f7081e22 100644
--- a/server/core/lib/activitypub/process/process-follow.ts
+++ b/server/core/lib/activitypub/process/process-follow.ts
@@ -1,8 +1,8 @@
-import { Transaction } from 'sequelize'
import { ActivityFollow } from '@peertube/peertube-models'
import { isBlockedByServerOrAccount } from '@server/lib/blocklist.js'
import { AccountModel } from '@server/models/account/account.js'
import { getServerActor } from '@server/models/application/application.js'
+import { Transaction } from 'sequelize'
import { retryTransactionWrapper } from '../../../helpers/database-utils.js'
import { logger } from '../../../helpers/logger.js'
import { CONFIG } from '../../../initializers/config.js'
@@ -22,7 +22,7 @@ async function processFollowActivity (options: APProcessorOptions<ActivityFollow
const activityId = activity.id
const objectId = getAPId(activity.object)
- return retryTransactionWrapper(processFollow, byActor, activityId, objectId)
+ return retryTransactionWrapper(() => processFollow(byActor, activityId, objectId))
}
// ---------------------------------------------------------------------------
diff --git a/server/core/lib/activitypub/process/process-like.ts b/server/core/lib/activitypub/process/process-like.ts
index 8c7c77230e..996909f284 100644
--- a/server/core/lib/activitypub/process/process-like.ts
+++ b/server/core/lib/activitypub/process/process-like.ts
@@ -12,7 +12,7 @@ import { canVideoBeFederated, federateVideoIfNeeded, maybeGetOrCreateAPVideo } f
async function processLikeActivity (options: APProcessorOptions<ActivityLike>) {
const { activity, byActor } = options
- return retryTransactionWrapper(processLikeVideo, byActor, activity)
+ return retryTransactionWrapper(() => processLikeVideo(byActor, activity))
}
// ---------------------------------------------------------------------------
diff --git a/server/core/lib/activitypub/process/process-undo.ts b/server/core/lib/activitypub/process/process-undo.ts
index 84d45d14d1..3cde321334 100644
--- a/server/core/lib/activitypub/process/process-undo.ts
+++ b/server/core/lib/activitypub/process/process-undo.ts
@@ -28,27 +28,29 @@ async function processUndoActivity (options: APProcessorOptions<ActivityUndo<Act
const activityToUndo = activity.object
if (activityToUndo.type === 'Like') {
- return retryTransactionWrapper(processUndoLike, byActor, activity)
+ return retryTransactionWrapper(() => processUndoLike(byActor, activity as ActivityUndo<ActivityLike>))
}
if (activityToUndo.type === 'Create') {
const objectToUndo = await fetchAPObjectIfNeeded<CacheFileObject>(activityToUndo.object)
if (objectToUndo.type === 'CacheFile') {
- return retryTransactionWrapper(processUndoCacheFile, byActor, activity, objectToUndo)
+ return retryTransactionWrapper(() => {
+ return processUndoCacheFile(byActor, activity as ActivityUndo<ActivityCreate<CacheFileObject>>, objectToUndo)
+ })
}
}
if (activityToUndo.type === 'Dislike') {
- return retryTransactionWrapper(processUndoDislike, byActor, activity)
+ return retryTransactionWrapper(() => processUndoDislike(byActor, activity as ActivityUndo<ActivityDislike>))
}
if (activityToUndo.type === 'Follow') {
- return retryTransactionWrapper(processUndoFollow, byActor, activityToUndo)
+ return retryTransactionWrapper(() => processUndoFollow(byActor, activityToUndo))
}
if (activityToUndo.type === 'Announce') {
- return retryTransactionWrapper(processUndoAnnounce, byActor, activityToUndo)
+ return retryTransactionWrapper(() => processUndoAnnounce(byActor, activityToUndo))
}
logger.warn('Unknown activity object type %s -> %s when undo activity.', activityToUndo.type, { activity: activity.id })
diff --git a/server/core/lib/activitypub/process/process-update.ts b/server/core/lib/activitypub/process/process-update.ts
index bb5b0283c0..12335d4c94 100644
--- a/server/core/lib/activitypub/process/process-update.ts
+++ b/server/core/lib/activitypub/process/process-update.ts
@@ -26,8 +26,8 @@ import { createOrUpdateCacheFile } from '../cache-file.js'
import { upsertAPPlayerSettings } from '../player-settings.js'
import { createOrUpdateVideoPlaylist } from '../playlists/index.js'
import { forwardVideoRelatedActivity } from '../send/shared/send-utils.js'
-import { APVideoUpdater, canVideoBeFederated, getOrCreateAPVideo, maybeGetOrCreateAPVideo } from '../videos/index.js'
import { checkUrlsSameHost } from '../url.js'
+import { APVideoUpdater, canVideoBeFederated, getOrCreateAPVideo, maybeGetOrCreateAPVideo } from '../videos/index.js'
async function processUpdateActivity (options: APProcessorOptions<ActivityUpdate<ActivityUpdateObject>>) {
const { activity, byActor } = options
@@ -36,27 +36,27 @@ async function processUpdateActivity (options: APProcessorOptions<ActivityUpdate
const objectType = object.type
if (objectType === 'Video') {
- return retryTransactionWrapper(processUpdateVideo, byActor, activity)
+ return retryTransactionWrapper(() => processUpdateVideo(byActor, activity as ActivityUpdate<VideoObject | string>))
}
if (isActorTypeValid(objectType as ActivityPubActorType)) {
// We need more attributes
const byActorFull = await ActorModel.loadByUrlAndPopulateAccountAndChannel(byActor.url)
- return retryTransactionWrapper(processUpdateActor, byActorFull, object)
+ return retryTransactionWrapper(() => processUpdateActor(byActorFull, object as ActivityPubActor))
}
if (objectType === 'CacheFile') {
// We need more attributes
const byActorFull = await ActorModel.loadByUrlAndPopulateAccountAndChannel(byActor.url)
- return retryTransactionWrapper(processUpdateCacheFile, byActorFull, activity, object)
+ return retryTransactionWrapper(() => processUpdateCacheFile(byActorFull, activity as ActivityUpdate<CacheFileObject | string>, object))
}
if (objectType === 'Playlist') {
- return retryTransactionWrapper(processUpdatePlaylist, byActor, activity, object)
+ return retryTransactionWrapper(() => processUpdatePlaylist(byActor, activity as ActivityUpdate<PlaylistObject | string>, object))
}
if (objectType === 'PlayerSettings') {
- return retryTransactionWrapper(processUpdatePlayerSettings, byActor, object)
+ return retryTransactionWrapper(() => processUpdatePlayerSettings(byActor, object))
}
return undefined
diff --git a/server/core/lib/activitypub/videos/get.ts b/server/core/lib/activitypub/videos/get.ts
index e80ec7067a..1332d46bdc 100644
--- a/server/core/lib/activitypub/videos/get.ts
+++ b/server/core/lib/activitypub/videos/get.ts
@@ -78,7 +78,7 @@ export async function getOrCreateAPVideo (
try {
const creator = new APVideoCreator(videoObject)
- const { autoBlacklisted, videoCreated } = await retryTransactionWrapper(creator.create.bind(creator))
+ const { autoBlacklisted, videoCreated } = await retryTransactionWrapper(() => creator.create())
await syncVideoExternalAttributes(videoCreated, videoObject, syncParam)
diff --git a/server/core/lib/job-queue/handlers/activitypub-follow.ts b/server/core/lib/job-queue/handlers/activitypub-follow.ts
index b3571beb82..d0f1a01eaa 100644
--- a/server/core/lib/job-queue/handlers/activitypub-follow.ts
+++ b/server/core/lib/job-queue/handlers/activitypub-follow.ts
@@ -1,18 +1,18 @@
-import { Job } from 'bullmq'
-import { getLocalActorFollowActivityPubUrl } from '@server/lib/activitypub/url.js'
import { ActivitypubFollowPayload } from '@peertube/peertube-models'
+import { getApplicationActorOfHost } from '@server/helpers/activity-pub-utils.js'
+import { getLocalActorFollowActivityPubUrl } from '@server/lib/activitypub/url.js'
+import { Job } from 'bullmq'
import { sanitizeHost } from '../../../helpers/core-utils.js'
import { retryTransactionWrapper } from '../../../helpers/database-utils.js'
import { logger } from '../../../helpers/logger.js'
import { REMOTE_SCHEME, SERVER_ACTOR_NAME, WEBSERVER } from '../../../initializers/constants.js'
import { sequelizeTypescript } from '../../../initializers/database.js'
-import { ActorModel } from '../../../models/actor/actor.js'
import { ActorFollowModel } from '../../../models/actor/actor-follow.js'
+import { ActorModel } from '../../../models/actor/actor.js'
import { MActor, MActorFull } from '../../../types/models/index.js'
import { getOrCreateAPActor, loadActorUrlOrGetFromWebfinger } from '../../activitypub/actors/index.js'
import { sendFollow } from '../../activitypub/send/index.js'
import { Notifier } from '../../notifier/index.js'
-import { getApplicationActorOfHost } from '@server/helpers/activity-pub-utils.js'
async function processActivityPubFollow (job: Job) {
const payload = job.data as ActivitypubFollowPayload
@@ -56,7 +56,7 @@ async function processActivityPubFollow (job: Job) {
const fromActor = await ActorModel.load(payload.followerActorId)
- return retryTransactionWrapper(follow, fromActor, targetActor, payload.isAutoFollow)
+ return retryTransactionWrapper(() => follow(fromActor, targetActor, payload.isAutoFollow))
}
// ---------------------------------------------------------------------------
diff --git a/server/core/lib/runners/job-handlers/abstract-vod-transcoding-job-handler.ts b/server/core/lib/runners/job-handlers/abstract-vod-transcoding-job-handler.ts
index f588cb1c55..e350e5c919 100644
--- a/server/core/lib/runners/job-handlers/abstract-vod-transcoding-job-handler.ts
+++ b/server/core/lib/runners/job-handlers/abstract-vod-transcoding-job-handler.ts
@@ -59,7 +59,7 @@ export abstract class AbstractVODTranscodingJobHandler<C, U extends RunnerJobUpd
this.lTags(video.uuid)
)
- await retryTransactionWrapper(moveToNextState, { video })
+ await retryTransactionWrapper(() => moveToNextState({ video }))
}
}
}
diff --git a/server/core/lib/transcoding/ended-transcoding.ts b/server/core/lib/transcoding/ended-transcoding.ts
index 8bbde3d618..212ba500ce 100644
--- a/server/core/lib/transcoding/ended-transcoding.ts
+++ b/server/core/lib/transcoding/ended-transcoding.ts
@@ -17,7 +17,7 @@ export async function onTranscodingEnded (options: {
await VideoJobInfoModel.decrease(video.uuid, 'pendingTranscode')
if (moveVideoToNextState) {
- const changedState = await retryTransactionWrapper(moveToNextState, { video })
+ const changedState = await retryTransactionWrapper(() => moveToNextState({ video }))
// Still send the transcoded file to external storage if needed
if (!changedState && CONFIG.OBJECT_STORAGE.ENABLED && await hasVideoResourcesToBeMoved(video, FileStorage.OBJECT_STORAGE)) {
diff --git a/server/core/lib/user-import-export/importers/video-playlists-importer.ts b/server/core/lib/user-import-export/importers/video-playlists-importer.ts
index 0c86d883b9..f4beb8d22f 100644
--- a/server/core/lib/user-import-export/importers/video-playlists-importer.ts
+++ b/server/core/lib/user-import-export/importers/video-playlists-importer.ts
@@ -1,17 +1,9 @@
+import { pick } from '@peertube/peertube-core-utils'
import { VideoPlaylistPrivacy, VideoPlaylistType, VideoPlaylistsExportJSON } from '@peertube/peertube-models'
-import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
import { buildUUID } from '@peertube/peertube-node-utils'
-import { MChannelBannerAccountDefault, MVideoPlaylistFull, MVideoPlaylistThumbnail } from '@server/types/models/index.js'
-import { getLocalVideoPlaylistActivityPubUrl, getLocalVideoPlaylistElementActivityPubUrl } from '@server/lib/activitypub/url.js'
-import { VideoChannelModel } from '@server/models/video/video-channel.js'
-import { VideoPlaylistModel } from '@server/models/video/video-playlist.js'
-import { AbstractUserImporter } from './abstract-user-importer.js'
-import { sendCreateVideoPlaylist } from '@server/lib/activitypub/send/send-create.js'
-import { sequelizeTypescript } from '@server/initializers/database.js'
-import { createLocalPlaylistThumbnailsFromImage } from '@server/lib/thumbnail.js'
-import { CONSTRAINTS_FIELDS, USER_IMPORT } from '@server/initializers/constants.js'
-import { VideoPlaylistElementModel } from '@server/models/video/video-playlist-element.js'
-import { loadOrCreateVideoIfAllowedForUser } from '@server/lib/model-loaders/video.js'
+import { isActorPreferredUsernameValid } from '@server/helpers/custom-validators/activitypub/actor.js'
+import { isUrlValid } from '@server/helpers/custom-validators/activitypub/misc.js'
+import { isArray } from '@server/helpers/custom-validators/misc.js'
import {
isVideoPlaylistDescriptionValid,
isVideoPlaylistNameValid,
@@ -19,13 +11,21 @@ import {
isVideoPlaylistTimestampValid,
isVideoPlaylistTypeValid
} from '@server/helpers/custom-validators/video-playlists.js'
-import { isActorPreferredUsernameValid } from '@server/helpers/custom-validators/activitypub/actor.js'
import { saveInTransactionWithRetries } from '@server/helpers/database-utils.js'
-import { isArray } from '@server/helpers/custom-validators/misc.js'
-import { isUrlValid } from '@server/helpers/custom-validators/activitypub/misc.js'
-import { pick } from '@peertube/peertube-core-utils'
+import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
+import { CONSTRAINTS_FIELDS, USER_IMPORT } from '@server/initializers/constants.js'
+import { sequelizeTypescript } from '@server/initializers/database.js'
+import { sendCreateVideoPlaylist } from '@server/lib/activitypub/send/send-create.js'
+import { getLocalVideoPlaylistActivityPubUrl, getLocalVideoPlaylistElementActivityPubUrl } from '@server/lib/activitypub/url.js'
+import { loadOrCreateVideoIfAllowedForUser } from '@server/lib/model-loaders/video.js'
+import { createLocalPlaylistThumbnailsFromImage } from '@server/lib/thumbnail.js'
+import { generateThumbnailForPlaylistIfNeeded } from '@server/lib/video-playlist.js'
+import { VideoChannelModel } from '@server/models/video/video-channel.js'
+import { VideoPlaylistElementModel } from '@server/models/video/video-playlist-element.js'
+import { VideoPlaylistModel } from '@server/models/video/video-playlist.js'
import { VideoModel } from '@server/models/video/video.js'
-import { generateThumbnailForPlaylist } from '@server/lib/video-playlist.js'
+import { MChannelBannerAccountDefault, MVideoPlaylistFull, MVideoPlaylistThumbnail } from '@server/types/models/index.js'
+import { AbstractUserImporter } from './abstract-user-importer.js'
const lTags = loggerTagsFactory('user-import')
@@ -149,8 +149,9 @@ export class VideoPlaylistsImporter extends AbstractUserImporter<VideoPlaylistsE
})
}
- await sequelizeTypescript.transaction(async t => {
+ const createdElements = await sequelizeTypescript.transaction(async t => {
let position = await VideoPlaylistElementModel.getNextPositionOf(playlist.id, t)
+ const createdElements: VideoPlaylistElementModel[] = []
for (const elementToCreate of elementsToCreate) {
const playlistElement = new VideoPlaylistElementModel({
@@ -165,15 +166,27 @@ export class VideoPlaylistsImporter extends AbstractUserImporter<VideoPlaylistsE
playlistElement.url = getLocalVideoPlaylistElementActivityPubUrl(playlist, playlistElement)
await playlistElement.save({ transaction: t })
- if (playlist.shouldGenerateThumbnailWithNewElement(playlistElement)) {
- const video = await VideoModel.loadFull(elementToCreate.videoId)
-
- generateThumbnailForPlaylist(playlist, video)
- .catch(err => logger.error('Cannot generate thumbnail from playlist', { err }))
- }
+ createdElements.push(playlistElement)
position++
}
+
+ return createdElements
})
+
+ // Generate the playlist thumbnail outside of the transaction, from the first element that has one
+ await playlist.reloadThumbnails()
+
+ for (const element of createdElements) {
+ if (playlist.hasThumbnail()) break
+
+ const video = await VideoModel.loadFull(element.videoId)
+
+ try {
+ await generateThumbnailForPlaylistIfNeeded({ videoPlaylist: playlist, element, video })
+ } catch (err) {
+ logger.error('Cannot generate thumbnail of playlist', { err, ...lTags() })
+ }
+ }
}
}
diff --git a/server/core/lib/video-playlist.ts b/server/core/lib/video-playlist.ts
index dd05f8aa3d..a0ded9bcea 100644
--- a/server/core/lib/video-playlist.ts
+++ b/server/core/lib/video-playlist.ts
@@ -1,14 +1,17 @@
import { VideoPlaylistPrivacy, VideoPlaylistType } from '@peertube/peertube-models'
+import { retryTransactionWrapper } from '@server/helpers/database-utils.js'
import { generateImageFilename } from '@server/helpers/image-utils.js'
import { logger } from '@server/helpers/logger.js'
import { CONFIG } from '@server/initializers/config.js'
+import { sequelizeTypescript } from '@server/initializers/database.js'
import { SequelizeModel } from '@server/models/shared/sequelize-type.js'
import { VideoPlaylistElementModel } from '@server/models/video/video-playlist-element.js'
-import { copy } from 'fs-extra/esm'
+import { Mutex } from 'async-mutex'
+import { copy, remove } from 'fs-extra/esm'
import { extname, join } from 'path'
import { Transaction } from 'sequelize'
import { VideoPlaylistModel } from '../models/video/video-playlist.js'
-import { MAccount, MVideoThumbnails } from '../types/models/index.js'
+import { MAccount, MVideoPlaylistElement, MVideoThumbnails } from '../types/models/index.js'
import { MVideoPlaylistOwner, MVideoPlaylistThumbnail } from '../types/models/video/video-playlist.js'
import { sendUpdateVideoPlaylist } from './activitypub/send/send-update.js'
import { getLocalVideoPlaylistActivityPubUrl } from './activitypub/url.js'
@@ -32,7 +35,34 @@ export async function createWatchLaterPlaylist (account: MAccount, t: Transactio
return videoPlaylist
}
-export async function generateThumbnailForPlaylist (videoPlaylist: MVideoPlaylistThumbnail, video: MVideoThumbnails) {
+// Generate the playlist thumbnail from the thumbnail of the video that has just been added, if the playlist needs one
+export function generateThumbnailForPlaylistIfNeeded (options: {
+ videoPlaylist: MVideoPlaylistThumbnail
+ element: MVideoPlaylistElement
+ video: MVideoThumbnails
+}) {
+ const { videoPlaylist, element, video } = options
+
+ return runInPlaylistThumbnailMutex(videoPlaylist, async () => {
+ if (videoPlaylist.shouldGenerateThumbnailWithNewElement(element) !== true) return
+
+ await generateThumbnailForPlaylist(videoPlaylist, video)
+ })
+}
+
+// The first element of the playlist changed: regenerate the automatic thumbnail from the new first element
+export function regeneratePlaylistThumbnailIfNeeded (videoPlaylist: MVideoPlaylistThumbnail) {
+ return runInPlaylistThumbnailMutex(videoPlaylist, async () => {
+ if (videoPlaylist.hasGeneratedThumbnail() !== true) return
+
+ await videoPlaylist.removeThumbnails(undefined)
+
+ const firstElement = await VideoPlaylistElementModel.loadFirstElementWithVideoThumbnail(videoPlaylist.id)
+ if (firstElement) await generateThumbnailForPlaylist(videoPlaylist, firstElement.Video)
+ })
+}
+
+async function generateThumbnailForPlaylist (videoPlaylist: MVideoPlaylistThumbnail, video: MVideoThumbnails) {
logger.info('Generating default thumbnail to playlist %s.', videoPlaylist.url)
const videoThumbnail = video.getBestThumbnail('16:9')
@@ -42,9 +72,10 @@ export async function generateThumbnailForPlaylist (videoPlaylist: MVideoPlaylis
}
const tmpImageName = generateImageFilename(extname(videoThumbnail.filename))
+ const tmpImagePath = join(CONFIG.STORAGE.TMP_DIR, tmpImageName)
if (video.isLocal() === true) {
- await copy(videoThumbnail.getFSPath(), join(CONFIG.STORAGE.TMP_DIR, tmpImageName))
+ await copy(videoThumbnail.getFSPath(), tmpImagePath)
} else {
await downloadImage({
url: videoThumbnail.fileUrl,
@@ -53,14 +84,67 @@ export async function generateThumbnailForPlaylist (videoPlaylist: MVideoPlaylis
})
}
- const thumbnails = await createLocalPlaylistThumbnailsFromImage({
- inputPath: join(CONFIG.STORAGE.TMP_DIR, tmpImageName),
- playlist: videoPlaylist,
- automaticallyGenerated: true,
- keepOriginal: false
- })
+ try {
+ // Another process (or a thumbnail uploaded by the user) can replace the thumbnails of this playlist at the same time:
+ // the unique index on the thumbnail size is what prevents duplicates, so just retry and let the last writer win
+ await retryTransactionWrapper(async () => {
+ // Build the thumbnails inside the retry: sequelize would not insert again the models of a rolled back attempt
+ const thumbnails = await createLocalPlaylistThumbnailsFromImage({
+ inputPath: tmpImagePath,
+ playlist: videoPlaylist,
+ automaticallyGenerated: true,
+ keepOriginal: true
+ })
+
+ try {
+ await sequelizeTypescript.transaction(t => videoPlaylist.replaceAndSaveThumbnails(thumbnails, t))
+ } catch (err) {
+ // The next attempt generates new files, so don't leak the ones we just created
+ await Promise.all(
+ thumbnails.map(thumbnail => {
+ return thumbnail.removeFile()
+ .catch(removeErr => logger.error('Cannot remove thumbnail file %s.', thumbnail.filename, { err: removeErr }))
+ })
+ )
+
+ throw err
+ }
+ }, { retryUniqueConstraintViolation: true })
+ } finally {
+ await remove(tmpImagePath)
+ }
+}
+
+// Multiple requests can concurrently add/remove/reorder elements of the same playlist and so try to generate
+// its automatic thumbnail at the same time
+const playlistThumbnailMutexes = new Map<number, { mutex: Mutex, users: number }>()
+
+async function runInPlaylistThumbnailMutex<T> (videoPlaylist: MVideoPlaylistThumbnail, fn: () => Promise<T>) {
+ const playlistId = videoPlaylist.id
+
+ let entry = playlistThumbnailMutexes.get(playlistId)
+ if (!entry) {
+ entry = { mutex: new Mutex(), users: 0 }
+ playlistThumbnailMutexes.set(playlistId, entry)
+ }
- await videoPlaylist.replaceAndSaveThumbnails(thumbnails, undefined)
+ // Count ourselves before waiting for the lock so the entry cannot be removed under us
+ entry.users++
+
+ const releaser = await entry.mutex.acquire()
+
+ try {
+ await videoPlaylist.reloadThumbnails()
+
+ return await fn()
+ } finally {
+ releaser()
+
+ entry.users--
+
+ // Don't leak mutexes of playlists that are not updated anymore
+ if (entry.users === 0) playlistThumbnailMutexes.delete(playlistId)
+ }
}
export async function reorderPlaylistOrElementsPosition<T extends typeof VideoPlaylistElementModel | typeof VideoPlaylistModel> (options: {
diff --git a/server/core/middlewares/async.ts b/server/core/middlewares/async.ts
index 04d85a95cb..280a856561 100644
--- a/server/core/middlewares/async.ts
+++ b/server/core/middlewares/async.ts
@@ -7,7 +7,7 @@ import { retryTransactionWrapper } from '../helpers/database-utils.js'
export type RequestPromiseHandler = ValidationChain | ExpressPromiseHandler
-function asyncMiddleware (fun: RequestPromiseHandler | RequestPromiseHandler[]) {
+export function asyncMiddleware (fun: RequestPromiseHandler | RequestPromiseHandler[]) {
return async (req: Request, res: Response, next: NextFunction) => {
if (Array.isArray(fun) !== true) {
return Promise.resolve((fun as RequestHandler)(req, res, next))
@@ -32,17 +32,8 @@ function asyncMiddleware (fun: RequestPromiseHandler | RequestPromiseHandler[])
}
}
-function asyncRetryTransactionMiddleware (fun: (req: Request, res: Response, next: NextFunction) => Promise<any>) {
+export function asyncRetryTransactionMiddleware (fun: (req: Request, res: Response, next: NextFunction) => Promise<any>) {
return (req: Request, res: Response, next: NextFunction) => {
- return Promise.resolve(
- retryTransactionWrapper(fun, req, res, next)
- ).catch(err => next(err))
+ return Promise.resolve(retryTransactionWrapper(() => fun(req, res, next))).catch(err => next(err))
}
}
-
-// ---------------------------------------------------------------------------
-
-export {
- asyncMiddleware,
- asyncRetryTransactionMiddleware
-}
diff --git a/server/core/models/video/thumbnail.ts b/server/core/models/video/thumbnail.ts
index f956ddbdd5..8f9ff7445b 100644
--- a/server/core/models/video/thumbnail.ts
+++ b/server/core/models/video/thumbnail.ts
@@ -1,10 +1,11 @@
import { ActivityIconObject, Thumbnail, type ThumbnailAspectRatio } from '@peertube/peertube-models'
import { AttributesOnly } from '@peertube/peertube-typescript-utils'
+import { afterCommitIfTransaction } from '@server/helpers/database-utils.js'
import { CONFIG } from '@server/initializers/config.js'
import { MThumbnail } from '@server/types/models/index.js'
import { remove } from 'fs-extra/esm'
import { extname, join } from 'path'
-import { Op } from 'sequelize'
+import { Op, Transaction } from 'sequelize'
import {
AfterDestroy,
AllowNull,
@@ -24,6 +25,18 @@ import { buildSQLAttributes } from '../shared/table.js'
import { VideoPlaylistModel } from './video-playlist.js'
import { VideoModel } from './video.js'
+// A thumbnail always belongs to a video or to a playlist, never to both
+type ThumbnailOwner =
+ | { videoId: number, videoPlaylistId?: never }
+ | { videoPlaylistId: number, videoId?: never }
+
+function pickOwner (owner: ThumbnailOwner): ThumbnailOwner {
+ if (owner.videoId !== undefined) return { videoId: owner.videoId }
+ if (owner.videoPlaylistId !== undefined) return { videoPlaylistId: owner.videoPlaylistId }
+
+ throw new Error('Cannot build thumbnail query without a video id or a video playlist id')
+}
+
export const thumbnailAPIAttributes = [
'filename',
'fileUrl',
@@ -44,6 +57,25 @@ export const thumbnailAPIAttributes = [
{
fields: [ 'filename' ],
unique: true
+ },
+ // A video/playlist can only have one thumbnail of a given size
+ {
+ fields: [ 'videoId', 'width', 'height' ],
+ where: {
+ videoId: {
+ [Op.ne]: null
+ }
+ },
+ unique: true
+ },
+ {
+ fields: [ 'videoPlaylistId', 'width', 'height' ],
+ where: {
+ videoPlaylistId: {
+ [Op.ne]: null
+ }
+ },
+ unique: true
}
]
})
@@ -110,10 +142,11 @@ export class ThumbnailModel extends SequelizeModel<ThumbnailModel> {
declare updatedAt: Date
@AfterDestroy
- static removeFiles (instance: ThumbnailModel) {
- // Don't block the transaction
- instance.removeFile()
- .catch(err => logger.error('Cannot remove thumbnail file %s.', instance.filename, { err }))
+ static removeFiles (instance: ThumbnailModel, options: { transaction?: Transaction }) {
+ afterCommitIfTransaction(options.transaction, () => {
+ instance.removeFile()
+ .catch(err => logger.error('Cannot remove thumbnail file %s.', instance.filename, { err }))
+ })
}
// ---------------------------------------------------------------------------
@@ -149,6 +182,55 @@ export class ThumbnailModel extends SequelizeModel<ThumbnailModel> {
// ---------------------------------------------------------------------------
+ static listOf (options: ThumbnailOwner & { transaction?: Transaction }) {
+ return ThumbnailModel.findAll({
+ where: pickOwner(options),
+ transaction: options.transaction
+ })
+ }
+
+ // Always fetch the thumbnails from the database to prevent concurrency issues
+ static async removeAllOf (options: ThumbnailOwner & { transaction?: Transaction }) {
+ for (const thumbnail of await ThumbnailModel.listOf(options)) {
+ await thumbnail.destroy({ transaction: options.transaction })
+ }
+ }
+
+ static async replaceAllOf (options: ThumbnailOwner & {
+ thumbnails: MThumbnail[]
+ transaction?: Transaction
+ }) {
+ const { thumbnails, transaction } = options
+ const owner = pickOwner(options)
+
+ if (thumbnails.length === 0) {
+ throw new Error('Cannot replace thumbnails with an empty array, at least one thumbnail is required')
+ }
+
+ // Thumbnails that are updated in place instead of being re-created
+ const keptIds = new Set(thumbnails.map(t => t.id).filter(id => !!id))
+
+ // Remove the old thumbnails first, otherwise the new ones would conflict with the unique constraint on their size
+ for (const thumbnail of await ThumbnailModel.listOf({ ...owner, transaction })) {
+ if (keptIds.has(thumbnail.id)) continue
+
+ await thumbnail.destroy({ transaction })
+ }
+
+ const savedThumbnails: ThumbnailModel[] = []
+
+ for (const thumbnail of thumbnails) {
+ if (owner.videoId !== undefined) thumbnail.videoId = owner.videoId
+ else thumbnail.videoPlaylistId = owner.videoPlaylistId
+
+ savedThumbnails.push(await thumbnail.save({ transaction }))
+ }
+
+ return savedThumbnails
+ }
+
+ // ---------------------------------------------------------------------------
+
getFSPath () {
return join(CONFIG.STORAGE.THUMBNAILS_DIR, this.filename)
}
diff --git a/server/core/models/video/video-playlist.ts b/server/core/models/video/video-playlist.ts
index 12c45daab4..8d636f59b7 100644
--- a/server/core/models/video/video-playlist.ts
+++ b/server/core/models/video/video-playlist.ts
@@ -622,38 +622,19 @@ export class VideoPlaylistModel extends SequelizeModel<VideoPlaylistModel> {
// ---------------------------------------------------------------------------
async replaceAndSaveThumbnails (thumbnails: MThumbnail[], transaction?: Transaction) {
- if (thumbnails.length === 0) {
- throw new Error('Cannot replace thumbnails with an empty array, at least one thumbnail is required')
- }
-
- let oldThumbnails = Array.isArray(this.Thumbnails)
- ? [ ...this.Thumbnails ]
- : []
-
- this.Thumbnails = []
-
- for (const thumbnail of thumbnails) {
- thumbnail.videoPlaylistId = this.id
-
- const savedThumbnail = await thumbnail.save({ transaction })
- oldThumbnails = oldThumbnails.filter(t => t.id !== savedThumbnail.id)
-
- this.Thumbnails.push(savedThumbnail)
- }
-
- for (const oldThumbnail of oldThumbnails) {
- await oldThumbnail.destroy({ transaction })
- }
+ this.Thumbnails = await ThumbnailModel.replaceAllOf({ thumbnails, videoPlaylistId: this.id, transaction })
}
async removeThumbnails (t: Transaction) {
- for (const thumbnail of this.Thumbnails) {
- await thumbnail.destroy({ transaction: t })
- }
+ await ThumbnailModel.removeAllOf({ videoPlaylistId: this.id, transaction: t })
this.Thumbnails = []
}
+ async reloadThumbnails (t?: Transaction) {
+ this.Thumbnails = await ThumbnailModel.listOf({ videoPlaylistId: this.id, transaction: t })
+ }
+
hasThumbnail () {
return this.Thumbnails.length !== 0
}
diff --git a/server/core/models/video/video.ts b/server/core/models/video/video.ts
index cc326ace8d..16334db952 100644
--- a/server/core/models/video/video.ts
+++ b/server/core/models/video/video.ts
@@ -1998,28 +1998,7 @@ export class VideoModel extends SequelizeModel<VideoModel> {
// ---------------------------------------------------------------------------
async replaceAndSaveThumbnails (thumbnails: MThumbnail[], transaction?: Transaction) {
- if (thumbnails.length === 0) {
- throw new Error('Cannot replace thumbnails with an empty array, at least one thumbnail is required')
- }
-
- let oldThumbnails = Array.isArray(this.Thumbnails)
- ? [ ...this.Thumbnails ]
- : []
-
- this.Thumbnails = []
-
- for (const thumbnail of thumbnails) {
- thumbnail.videoId = this.id
-
- const savedThumbnail = await thumbnail.save({ transaction })
- oldThumbnails = oldThumbnails.filter(t => t.id !== savedThumbnail.id)
-
- this.Thumbnails.push(savedThumbnail)
- }
-
- for (const oldThumbnail of oldThumbnails) {
- await oldThumbnail.destroy({ transaction })
- }
+ this.Thumbnails = await ThumbnailModel.replaceAllOf({ thumbnails, videoId: this.id, transaction })
}
// ---------------------------------------------------------------------------