ActivityPub-federated video streaming platform using P2P directly in your web browser
Prevent duplicated thumbnails
Chocobozzz1d2f40ad76d7
diff --git a/server/core/controllers/api/video-playlist.ts b/server/core/controllers/api/video-playlist.tsindex 05a70328de..1dcd447692 100644--- a/server/core/controllers/api/video-playlist.ts+++ b/server/core/controllers/api/video-playlist.ts@@ -16,13 +16,14 @@ import { uuidToShort } from '@peertube/peertube-node-utils'import { schedulePlaylistRefreshIfNeeded } from '@server/lib/activitypub/playlists/index.js'import { Hooks } from '@server/lib/plugins/hooks.js'import {- generateThumbnailForPlaylist,+ generateThumbnailForPlaylistIfNeeded,+ regeneratePlaylistThumbnailIfNeeded,reorderPlaylistOrElementsPosition,sendPlaylistPositionUpdateOfChannel} from '@server/lib/video-playlist.js'import { getServerActor } from '@server/models/application/application.js'import { VideoChannelActivityModel } from '@server/models/video/video-channel-activity.js'-import { MVideoPlaylistFull, MVideoPlaylistThumbnail } from '@server/types/models/index.js'+import { MVideoPlaylistFull } from '@server/types/models/index.js'import express from 'express'import { resetSequelizeInstance, retryTransactionWrapper } from '../../helpers/database-utils.js'import { createReqFiles } from '../../helpers/express-utils.js'@@ -91,7 +92,7 @@ videoPlaylistRouter.put(authenticate,reqThumbnailFile,asyncMiddleware(videoPlaylistsUpdateValidator),- asyncRetryTransactionMiddleware(updateVideoPlaylist)+ asyncMiddleware(updateVideoPlaylist))videoPlaylistRouter.delete(@@ -255,112 +256,119 @@ async function updateVideoPlaylist (req: express.Request, res: express.Response)const wasPrivatePlaylist = playlist.privacy === VideoPlaylistPrivacy.PRIVATEconst wasNotPrivatePlaylist = playlist.privacy !== VideoPlaylistPrivacy.PRIVATE- let removedFromChannel: { id: number, position: number }-const thumbnailField = req.files?.['thumbnailfile']- const thumbnailModels = thumbnailField- ? await createLocalPlaylistThumbnailsFromImage({- inputPath: thumbnailField[0].path,- playlist,- automaticallyGenerated: false- })- : undefinedtry {- await sequelizeTypescript.transaction(async t => {- const newChannel = res.locals.videoChannel- const user = res.locals.oauth.token.User-- // Had a channel, but the user changed it (to null or another channel)- if (playlist.videoChannelId && body.videoChannelId !== undefined && body.videoChannelId !== playlist.videoChannelId) {- await VideoChannelActivityModel.addPlaylistActivity({- action: VideoChannelActivityAction.REMOVE_CHANNEL_OWNERSHIP,- user,- channel: playlist.VideoChannel,+ // A concurrent request updating the same playlist thumbnails can hit the unique index on thumbnail size+ await retryTransactionWrapper(async () => {+ // Build the thumbnails here so it's rebuilt on retry+ const thumbnailModels = thumbnailField+ ? await createLocalPlaylistThumbnailsFromImage({+ inputPath: thumbnailField[0].path,playlist,- transaction: t+ automaticallyGenerated: false})-- removedFromChannel = {- id: playlist.videoChannelId,- position: playlist.videoChannelPosition+ : undefined++ return sequelizeTypescript.transaction(async t => {+ // If this is a retried attempt, sequelize would otherwise think this instance has not changed+ await playlist.reload({ transaction: t })++ let removedFromChannel: { id: number, position: number }++ const newChannel = res.locals.videoChannel+ const user = res.locals.oauth.token.User++ // Had a channel, but the user changed it (to null or another channel)+ if (playlist.videoChannelId && body.videoChannelId !== undefined && body.videoChannelId !== playlist.videoChannelId) {+ await VideoChannelActivityModel.addPlaylistActivity({+ action: VideoChannelActivityAction.REMOVE_CHANNEL_OWNERSHIP,+ user,+ channel: playlist.VideoChannel,+ playlist,+ transaction: t+ })++ removedFromChannel = {+ id: playlist.videoChannelId,+ position: playlist.videoChannelPosition+ }++ playlist.videoChannelId = null+ playlist.VideoChannel = null}- playlist.videoChannelId = null- playlist.VideoChannel = null- }-- if (newChannel && newChannel.id !== playlist.videoChannelId) {- await VideoChannelActivityModel.addPlaylistActivity({- action: VideoChannelActivityAction.CREATE_CHANNEL_OWNERSHIP,- user,- channel: newChannel,- playlist,- transaction: t- })-- playlist.videoChannelPosition = await VideoPlaylistModel.getNextPositionOf({- videoChannelId: newChannel.id,- transaction: t- })-- playlist.videoChannelId = newChannel.id- playlist.VideoChannel = newChannel- } else if (newChannel) {- await VideoChannelActivityModel.addPlaylistActivity({- action: VideoChannelActivityAction.UPDATE,- user: res.locals.oauth.token.User,- channel: newChannel,- playlist,- transaction: t- })- }+ if (newChannel && newChannel.id !== playlist.videoChannelId) {+ await VideoChannelActivityModel.addPlaylistActivity({+ action: VideoChannelActivityAction.CREATE_CHANNEL_OWNERSHIP,+ user,+ channel: newChannel,+ playlist,+ transaction: t+ })++ playlist.videoChannelPosition = await VideoPlaylistModel.getNextPositionOf({+ videoChannelId: newChannel.id,+ transaction: t+ })++ playlist.videoChannelId = newChannel.id+ playlist.VideoChannel = newChannel+ } else if (newChannel) {+ await VideoChannelActivityModel.addPlaylistActivity({+ action: VideoChannelActivityAction.UPDATE,+ user: res.locals.oauth.token.User,+ channel: newChannel,+ playlist,+ transaction: t+ })+ }- if (body.displayName !== undefined) playlist.name = body.displayName- if (body.description !== undefined) playlist.description = body.description+ if (body.displayName !== undefined) playlist.name = body.displayName+ if (body.description !== undefined) playlist.description = body.description- if (body.privacy !== undefined) {- playlist.privacy = forceNumber(body.privacy) as VideoPlaylistPrivacyType+ if (body.privacy !== undefined) {+ playlist.privacy = forceNumber(body.privacy) as VideoPlaylistPrivacyType- if (wasNotPrivatePlaylist === true && playlist.privacy === VideoPlaylistPrivacy.PRIVATE) {- await sendDeleteVideoPlaylist(playlist, t)+ if (wasNotPrivatePlaylist === true && playlist.privacy === VideoPlaylistPrivacy.PRIVATE) {+ await sendDeleteVideoPlaylist(playlist, t)+ }}- }- const playlistUpdated = await playlist.save({ transaction: t })+ const playlistUpdated = await playlist.save({ transaction: t })- if (thumbnailModels) {- await playlistUpdated.replaceAndSaveThumbnails(thumbnailModels, t)- }+ if (thumbnailModels) {+ await playlistUpdated.replaceAndSaveThumbnails(thumbnailModels, t)+ }- const isNewPlaylist = wasPrivatePlaylist && playlistUpdated.privacy !== VideoPlaylistPrivacy.PRIVATE+ const isNewPlaylist = wasPrivatePlaylist && playlistUpdated.privacy !== VideoPlaylistPrivacy.PRIVATE- if (isNewPlaylist) {- await sendCreateVideoPlaylist(playlistUpdated, t)- } else {- await sendUpdateVideoPlaylist(playlistUpdated, t)- }+ if (isNewPlaylist) {+ await sendCreateVideoPlaylist(playlistUpdated, t)+ } else {+ await sendUpdateVideoPlaylist(playlistUpdated, t)+ }- if (removedFromChannel) {- await VideoPlaylistModel.increasePositionOf({- videoChannelId: removedFromChannel.id,- fromPosition: removedFromChannel.position,- by: -1,- transaction: t- })+ if (removedFromChannel) {+ await VideoPlaylistModel.increasePositionOf({+ videoChannelId: removedFromChannel.id,+ fromPosition: removedFromChannel.position,+ by: -1,+ transaction: t+ })- await sendPlaylistPositionUpdateOfChannel(removedFromChannel.id, t)- }+ await sendPlaylistPositionUpdateOfChannel(removedFromChannel.id, t)+ }- logger.info('Video playlist %s updated.', playlist.uuid)+ logger.info('Video playlist %s updated.', playlist.uuid)- return playlistUpdated- })+ return playlistUpdated+ })+ }, { retryUniqueConstraintViolation: true })} catch (err) {logger.debug('Cannot update the video playlist.', { err })- // If the transaction is retried, sequelize will think the object has not changed- // So we need to restore the previous fields+ // Leave the instance clean for any other code path that might reuse it after this handler returnsawait resetSequelizeInstance(playlist)throw err@@ -447,9 +455,7 @@ async function addVideoInPlaylist (req: express.Request, res: express.Response)})// If the user did not set a thumbnail, automatically take the video thumbnail- if (videoPlaylist.shouldGenerateThumbnailWithNewElement(playlistElement)) {- await generateThumbnailForPlaylist(videoPlaylist, video)- }+ await generateThumbnailForPlaylistIfNeeded({ videoPlaylist, element: playlistElement, video })sendUpdateVideoPlaylist(videoPlaylist, undefined).catch(err => logger.error('Cannot send video playlist update.', { err }))@@ -532,8 +538,8 @@ async function removeVideoFromPlaylist (req: express.Request, res: express.Respo})// Do we need to regenerate the default thumbnail?- if (positionToDelete === 1 && videoPlaylist.hasGeneratedThumbnail()) {- await regeneratePlaylistThumbnail(videoPlaylist)+ if (positionToDelete === 1) {+ await regeneratePlaylistThumbnailIfNeeded(videoPlaylist)}sendUpdateVideoPlaylist(videoPlaylist, undefined)@@ -569,8 +575,8 @@ async function reorderVideosOfPlaylist (req: express.Request, res: express.Respo})// The first element changed- if ((start === 1 || insertAfter === 0) && videoPlaylist.hasGeneratedThumbnail()) {- await regeneratePlaylistThumbnail(videoPlaylist)+ if (start === 1 || insertAfter === 0) {+ await regeneratePlaylistThumbnailIfNeeded(videoPlaylist)}await sendUpdateVideoPlaylist(videoPlaylist, undefined)@@ -608,10 +614,3 @@ async function listVideosOfPlaylist (req: express.Request, res: express.Responseconst options = { accountId: user?.Account?.id }return res.json(getFormattedObjects(resultList.data, resultList.total, options))}--async function regeneratePlaylistThumbnail (videoPlaylist: MVideoPlaylistThumbnail) {- await videoPlaylist.removeThumbnails(undefined)-- const firstElement = await VideoPlaylistElementModel.loadFirstElementWithVideoThumbnail(videoPlaylist.id)- if (firstElement) await generateThumbnailForPlaylist(videoPlaylist, firstElement.Video)-}diff --git a/server/core/helpers/custom-validators/activitypub/misc.ts b/server/core/helpers/custom-validators/activitypub/misc.tsindex 9c2b59b732..8185ab9ed2 100644--- a/server/core/helpers/custom-validators/activitypub/misc.ts+++ b/server/core/helpers/custom-validators/activitypub/misc.ts@@ -66,12 +66,23 @@ export function setValidRemoteIcon (entity: any) {if (entity.icon && !isArray(entity.icon)) entity.icon = [ entity.icon ]if (!entity.icon) entity.icon = []+ const existingSizes = new Set<string>()+entity.icon = entity.icon.filter(icon => {- return icon.type === 'Image' &&+ const isValid = icon.type === 'Image' &&isActivityPubUrlValid(icon.url) &&!!MIMETYPES.IMAGE.MIMETYPE_EXT[icon.mediaType] &&validator.default.isInt(icon.width + '', { min: 0 }) &&validator.default.isInt(icon.height + '', { min: 0 })++ if (!isValid) return false++ // We store at most one icon per size, so ignore the next ones the remote instance may have sent+ const size = `${icon.width}x${icon.height}`+ if (existingSizes.has(size)) return false+ existingSizes.add(size)++ return true})return truediff --git a/server/core/helpers/database-utils.ts b/server/core/helpers/database-utils.tsindex 87edf05ee0..7a05084ebf 100644--- a/server/core/helpers/database-utils.ts+++ b/server/core/helpers/database-utils.ts@@ -1,71 +1,40 @@-import retry from 'async/retry.js'-import Bluebird from 'bluebird'+import { sequelizeTypescript } from '@server/initializers/database.js'import { Transaction } from 'sequelize'import { Model } from 'sequelize-typescript'-import { sequelizeTypescript } from '@server/initializers/database.js'import { logger } from './logger.js'-export function retryTransactionWrapper<T, A, B, C, D> (- functionToRetry: (arg1: A, arg2: B, arg3: C, arg4: D) => Promise<T>,- arg1: A,- arg2: B,- arg3: C,- arg4: D-): Promise<T>--export function retryTransactionWrapper<T, A, B, C> (- functionToRetry: (arg1: A, arg2: B, arg3: C) => Promise<T>,- arg1: A,- arg2: B,- arg3: C-): Promise<T>--export function retryTransactionWrapper<T, A, B> (- functionToRetry: (arg1: A, arg2: B) => Promise<T>,- arg1: A,- arg2: B-): Promise<T>--export function retryTransactionWrapper<T, A> (- functionToRetry: (arg1: A) => Promise<T>,- arg1: A-): Promise<T>--export function retryTransactionWrapper<T> (- functionToRetry: () => Promise<T> | Bluebird<T>-): Promise<T>--export function retryTransactionWrapper<T> (- functionToRetry: (...args: any[]) => Promise<T>,- ...args: any[]-): Promise<T> {- return transactionRetryer<T>(callback => {- functionToRetry.apply(null, args)- .then((result: T) => callback(null, result))- .catch(err => callback(err))- })- .catch(err => {- logger.warn(`Cannot execute ${functionToRetry.name || 'function'} with many retries.`, { err })- throw err- })+export type RetryTransactionWrapperOptions = {+ // Also retry when the transaction failed because of a unique constraint violation+ retryUniqueConstraintViolation?: boolean // default false}-export function transactionRetryer<T> (func: (err: any, data: T) => any) {- return new Promise<T>((res, rej) => {- retry(- {- times: 5,+export async function retryTransactionWrapper<T> (+ functionToRetry: () => Promise<T>,+ options: RetryTransactionWrapperOptions = {}+): Promise<T> {+ const maxAttempts = 5+ let attempts = 0- errorFilter: err => {- const willRetry = err.name === 'SequelizeDatabaseError'- logger.debug('Maybe retrying the transaction function.', { willRetry, err, tags: [ 'sql', 'retry' ] })- return willRetry- }- },- func,- (err, data) => err ? rej(err) : res(data)- )- })+ while (true) {+ try {+ return await functionToRetry()+ } catch (err) {+ attempts++++ const willRetry = attempts < maxAttempts && (+ err?.name === 'SequelizeDatabaseError' ||+ (options?.retryUniqueConstraintViolation === true && err?.name === 'SequelizeUniqueConstraintError')+ )++ logger.debug('Maybe retrying the transaction function.', { willRetry, err, tags: [ 'sql', 'retry' ] })++ if (!willRetry) {+ logger.warn(`Cannot execute function with many retries.`, { err, attempts, stack: err?.stack })++ throw err+ }+ }+ }}export function saveInTransactionWithRetries<T extends Pick<Model, 'save' | 'changed'>> (diff --git a/server/core/initializers/checker-after-init.ts b/server/core/initializers/checker-after-init.tsindex ea4b96cb00..39aa98de54 100644--- a/server/core/initializers/checker-after-init.ts+++ b/server/core/initializers/checker-after-init.ts@@ -386,6 +386,12 @@ function checkThumbnailsConfig () {if (!isArray(CONFIG.THUMBNAILS.SIZES) || CONFIG.THUMBNAILS.SIZES.length === 0) {throw new Error('thumbnails.sizes must not be empty')}++ // A video/playlist can only have one thumbnail of a given size+ const sizes = CONFIG.THUMBNAILS.SIZES.map(s => `${s.width}x${s.height}`)+ if (new Set(sizes).size !== sizes.length) {+ throw new Error('thumbnails.sizes must not contain multiple sizes with the same width and height')+ }}function checkBrowseVideosConfig () {diff --git a/server/core/initializers/constants.ts b/server/core/initializers/constants.tsindex 01b0143f22..f4c2f4e4a1 100644--- a/server/core/initializers/constants.ts+++ b/server/core/initializers/constants.ts@@ -62,7 +62,7 @@ import { CONFIG, registerConfigChangedHandler } from './config.js'// ----------------------------------------------------------------------------export const LAST_MIGRATION_VERSION = 1090+export const LAST_MIGRATION_VERSION = 1095// ---------------------------------------------------------------------------diff --git a/server/core/initializers/migrations/1095-thumbnail-unique-size.ts b/server/core/initializers/migrations/1095-thumbnail-unique-size.tsnew file mode 100644index 0000000000..891ee9549b--- /dev/null+++ b/server/core/initializers/migrations/1095-thumbnail-unique-size.ts@@ -0,0 +1,29 @@+import * as Sequelize from 'sequelize'++async function up (utils: {+ transaction: Sequelize.Transaction+ queryInterface: Sequelize.QueryInterface+ sequelize: Sequelize.Sequelize+}): Promise<void> {+ const { transaction } = utils++ // Only keep the most recent one of each size to create unique indexes+ for (const column of [ 'videoId', 'videoPlaylistId' ]) {+ const query = 'DELETE FROM "thumbnail" t1 ' ++ `USING (SELECT MAX("id") AS "id", "${column}", "width", "height" FROM "thumbnail" ` ++ `WHERE "${column}" IS NOT NULL AND "width" IS NOT NULL AND "height" IS NOT NULL ` ++ `GROUP BY "${column}", "width", "height" HAVING COUNT(*) > 1) t2 ` ++ `WHERE t1."${column}" = t2."${column}" AND t1."width" = t2."width" AND t1."height" = t2."height" AND t1."id" <> t2."id"`++ await utils.sequelize.query(query, { transaction })+ }+}++function down (options) {+ throw new Error('Not implemented.')+}++export {+ down,+ up+}diff --git a/server/core/lib/activitypub/actors/get.ts b/server/core/lib/activitypub/actors/get.tsindex f554bdb611..80bb7c03ac 100644--- a/server/core/lib/activitypub/actors/get.ts+++ b/server/core/lib/activitypub/actors/get.ts@@ -59,7 +59,7 @@ async function getOrCreateAPActor (}const creator = new APActorCreator(actorObject, ownerActor)- actor = await retryTransactionWrapper(creator.create.bind(creator))+ actor = await retryTransactionWrapper(() => creator.create())created = trueaccountPlaylistsUrl = actorObject.playlists}diff --git a/server/core/lib/activitypub/crawl.ts b/server/core/lib/activitypub/crawl.tsindex 7ea31aec7d..faf6c62add 100644--- a/server/core/lib/activitypub/crawl.ts+++ b/server/core/lib/activitypub/crawl.ts@@ -1,7 +1,7 @@-import Bluebird from 'bluebird'-import { URL } from 'url'import { ActivityPubOrderedCollection } from '@peertube/peertube-models'import { retryTransactionWrapper } from '@server/helpers/database-utils.js'+import Bluebird from 'bluebird'+import { URL } from 'url'import { logger } from '../../helpers/logger.js'import { ACTIVITY_PUB, WEBSERVER } from '../../initializers/constants.js'import { fetchAP } from './activity.js'@@ -61,5 +61,5 @@ export async function crawlCollectionPage<T> (}}- if (cleaner) await retryTransactionWrapper(cleaner, startDate)+ if (cleaner) await retryTransactionWrapper(() => cleaner(startDate))}diff --git a/server/core/lib/activitypub/process/process-announce.ts b/server/core/lib/activitypub/process/process-announce.tsindex 5b48919049..971f1da1a2 100644--- a/server/core/lib/activitypub/process/process-announce.ts+++ b/server/core/lib/activitypub/process/process-announce.ts@@ -17,7 +17,7 @@ async function processAnnounceActivity (options: APProcessorOptions<ActivityAnno// Announces by accounts are not supportedif (actorAnnouncer.type !== 'Application' && actorAnnouncer.type !== 'Group') return- return retryTransactionWrapper(processVideoShare, actorAnnouncer, activity, notify)+ return retryTransactionWrapper(() => processVideoShare(actorAnnouncer, activity, notify))}// ---------------------------------------------------------------------------diff --git a/server/core/lib/activitypub/process/process-create.ts b/server/core/lib/activitypub/process/process-create.tsindex f5664bf167..d72ba821f2 100644--- a/server/core/lib/activitypub/process/process-create.ts+++ b/server/core/lib/activitypub/process/process-create.ts@@ -46,19 +46,25 @@ async function processCreateActivity (options: APProcessorOptions<ActivityCreate// Comments will be fetched from videosif (options.fromFetch) return- return retryTransactionWrapper(processCreateVideoComment, activity, activityObject, byActor, options.fromFetch)+ return retryTransactionWrapper(() => {+ return processCreateVideoComment(activity as ActivityCreate<VideoCommentObject | string>, activityObject, byActor, false)+ })}if (activityType === 'WatchAction') {- return retryTransactionWrapper(processCreateWatchAction, activityObject)+ return retryTransactionWrapper(() => processCreateWatchAction(activityObject))}if (activityType === 'CacheFile') {- return retryTransactionWrapper(processCreateCacheFile, activity, activityObject, byActor)+ return retryTransactionWrapper(() => {+ return processCreateCacheFile(activity as ActivityCreate<CacheFileObject | string>, activityObject, byActor)+ })}if (activityType === 'Playlist') {- return retryTransactionWrapper(processCreatePlaylist, activity, activityObject, byActor)+ return retryTransactionWrapper(() => {+ return processCreatePlaylist(activity as ActivityCreate<PlaylistObject | string>, activityObject, byActor)+ })}logger.warn('Unknown activity object type %s when creating activity.', activityType, { activity: activity.id })diff --git a/server/core/lib/activitypub/process/process-delete.ts b/server/core/lib/activitypub/process/process-delete.tsindex 2663435507..945554670f 100644--- a/server/core/lib/activitypub/process/process-delete.ts+++ b/server/core/lib/activitypub/process/process-delete.ts@@ -15,7 +15,9 @@ import {MActorSignature,MChannelAccountActor,MChannelActor,- MCommentOwnerVideo+ MCommentOwnerVideo,+ MVideoAccountLightBlacklistAllFiles,+ MVideoPlaylistAccountThumbnail} from '../../../types/models/index.js'import { forwardVideoRelatedActivity } from '../send/shared/send-utils.js'@@ -35,20 +37,20 @@ async function processDeleteActivity (options: APProcessorOptions<ActivityDeleteconst accountToDelete = byActorFull.Account as MAccountActoraccountToDelete.Actor = byActorFull- return retryTransactionWrapper(processDeleteAccount, accountToDelete)+ return retryTransactionWrapper(() => processDeleteAccount(accountToDelete))} else if (isChannelActor(byActorFull.type)) {if (!byActorFull.VideoChannel) throw new Error('Actor ' + byActorFull.url + ' is a group but we cannot find it in database.')const channelToDelete = byActorFull.VideoChannel as MChannelAccountActor & { Actor: MActorFull }channelToDelete.Actor = byActorFull- return retryTransactionWrapper(processDeleteVideoChannel, channelToDelete)+ return retryTransactionWrapper(() => processDeleteVideoChannel(channelToDelete))}}{const videoCommentInstance = await VideoCommentModel.loadByUrlAndPopulateAccountAndVideoAndReply(objectUrl)if (videoCommentInstance) {- return retryTransactionWrapper(processDeleteVideoComment, byActor, videoCommentInstance, activity)+ return retryTransactionWrapper(() => processDeleteVideoComment(byActor, videoCommentInstance, activity))}}@@ -57,7 +59,7 @@ async function processDeleteActivity (options: APProcessorOptions<ActivityDeleteif (videoInstance) {if (videoInstance.isLocal()) throw new Error(`Remote instance cannot delete owned video ${videoInstance.url}.`)- return retryTransactionWrapper(processDeleteVideo, byActor, videoInstance)+ return retryTransactionWrapper(() => processDeleteVideo(byActor, videoInstance))}}@@ -66,7 +68,7 @@ async function processDeleteActivity (options: APProcessorOptions<ActivityDeleteif (videoPlaylist) {if (videoPlaylist.isLocal()) throw new Error(`Remote instance cannot delete owned playlist ${videoPlaylist.url}.`)- return retryTransactionWrapper(processDeleteVideoPlaylist, byActor, videoPlaylist)+ return retryTransactionWrapper(() => processDeleteVideoPlaylist(byActor, videoPlaylist))}}@@ -81,7 +83,7 @@ export {// ----------------------------------------------------------------------------async function processDeleteVideo (actor: MActor, videoToDelete: VideoModel) {+async function processDeleteVideo (actor: MActor, videoToDelete: MVideoAccountLightBlacklistAllFiles) {logger.debug('Removing remote video "%s".', videoToDelete.uuid)await sequelizeTypescript.transaction(async t => {@@ -95,7 +97,7 @@ async function processDeleteVideo (actor: MActor, videoToDelete: VideoModel) {logger.info('Remote video with uuid %s removed.', videoToDelete.uuid)}-async function processDeleteVideoPlaylist (actor: MActor, playlistToDelete: VideoPlaylistModel) {+async function processDeleteVideoPlaylist (actor: MActor, playlistToDelete: MVideoPlaylistAccountThumbnail) {logger.debug('Removing remote video playlist "%s".', playlistToDelete.uuid)await sequelizeTypescript.transaction(async t => {diff --git a/server/core/lib/activitypub/process/process-dislike.ts b/server/core/lib/activitypub/process/process-dislike.tsindex 0876e692bf..8666e73d27 100644--- a/server/core/lib/activitypub/process/process-dislike.ts+++ b/server/core/lib/activitypub/process/process-dislike.ts@@ -10,7 +10,8 @@ import { canVideoBeFederated, federateVideoIfNeeded, maybeGetOrCreateAPVideo } fasync function processDislikeActivity (options: APProcessorOptions<ActivityDislike>) {const { activity, byActor } = options- return retryTransactionWrapper(processDislike, activity, byActor)++ return retryTransactionWrapper(() => processDislike(activity, byActor))}// ---------------------------------------------------------------------------diff --git a/server/core/lib/activitypub/process/process-flag.ts b/server/core/lib/activitypub/process/process-flag.tsindex 45cfa70248..c5ecef059f 100644--- a/server/core/lib/activitypub/process/process-flag.ts+++ b/server/core/lib/activitypub/process/process-flag.ts@@ -1,9 +1,9 @@+import { abusePredefinedReasonsMap } from '@peertube/peertube-core-utils'+import { AbuseState, ActivityFlag } from '@peertube/peertube-models'import { createAccountAbuse, createVideoAbuse, createVideoCommentAbuse } from '@server/lib/moderation.js'import { AccountModel } from '@server/models/account/account.js'import { VideoCommentModel } from '@server/models/video/video-comment.js'import { VideoModel } from '@server/models/video/video.js'-import { abusePredefinedReasonsMap } from '@peertube/peertube-core-utils'-import { AbuseState, ActivityFlag } from '@peertube/peertube-models'import { retryTransactionWrapper } from '../../../helpers/database-utils.js'import { logger } from '../../../helpers/logger.js'import { sequelizeTypescript } from '../../../initializers/database.js'@@ -14,7 +14,7 @@ import { MAccountDefault, MActorSignature, MCommentOwnerVideo } from '../../../tasync function processFlagActivity (options: APProcessorOptions<ActivityFlag>) {const { activity, byActor } = options- return retryTransactionWrapper(processCreateAbuse, activity, byActor)+ return retryTransactionWrapper(() => processCreateAbuse(activity, byActor))}// ---------------------------------------------------------------------------@@ -35,7 +35,7 @@ async function processCreateAbuse (flag: ActivityFlag, byActor: MActorSignature)const tags = Array.isArray(flag.tag) ? flag.tag : []const predefinedReasons = tags.map(tag => abusePredefinedReasonsMap[tag.name])- .filter(v => !isNaN(v))+ .filter(v => !isNaN(v))const startAt = flag.startAtconst endAt = flag.endAtdiff --git a/server/core/lib/activitypub/process/process-follow.ts b/server/core/lib/activitypub/process/process-follow.tsindex aa807dc758..a9f7081e22 100644--- a/server/core/lib/activitypub/process/process-follow.ts+++ b/server/core/lib/activitypub/process/process-follow.ts@@ -1,8 +1,8 @@-import { Transaction } from 'sequelize'import { ActivityFollow } from '@peertube/peertube-models'import { isBlockedByServerOrAccount } from '@server/lib/blocklist.js'import { AccountModel } from '@server/models/account/account.js'import { getServerActor } from '@server/models/application/application.js'+import { Transaction } from 'sequelize'import { retryTransactionWrapper } from '../../../helpers/database-utils.js'import { logger } from '../../../helpers/logger.js'import { CONFIG } from '../../../initializers/config.js'@@ -22,7 +22,7 @@ async function processFollowActivity (options: APProcessorOptions<ActivityFollowconst activityId = activity.idconst objectId = getAPId(activity.object)- return retryTransactionWrapper(processFollow, byActor, activityId, objectId)+ return retryTransactionWrapper(() => processFollow(byActor, activityId, objectId))}// ---------------------------------------------------------------------------diff --git a/server/core/lib/activitypub/process/process-like.ts b/server/core/lib/activitypub/process/process-like.tsindex 8c7c77230e..996909f284 100644--- a/server/core/lib/activitypub/process/process-like.ts+++ b/server/core/lib/activitypub/process/process-like.ts@@ -12,7 +12,7 @@ import { canVideoBeFederated, federateVideoIfNeeded, maybeGetOrCreateAPVideo } fasync function processLikeActivity (options: APProcessorOptions<ActivityLike>) {const { activity, byActor } = options- return retryTransactionWrapper(processLikeVideo, byActor, activity)+ return retryTransactionWrapper(() => processLikeVideo(byActor, activity))}// ---------------------------------------------------------------------------diff --git a/server/core/lib/activitypub/process/process-undo.ts b/server/core/lib/activitypub/process/process-undo.tsindex 84d45d14d1..3cde321334 100644--- a/server/core/lib/activitypub/process/process-undo.ts+++ b/server/core/lib/activitypub/process/process-undo.ts@@ -28,27 +28,29 @@ async function processUndoActivity (options: APProcessorOptions<ActivityUndo<Actconst activityToUndo = activity.objectif (activityToUndo.type === 'Like') {- return retryTransactionWrapper(processUndoLike, byActor, activity)+ return retryTransactionWrapper(() => processUndoLike(byActor, activity as ActivityUndo<ActivityLike>))}if (activityToUndo.type === 'Create') {const objectToUndo = await fetchAPObjectIfNeeded<CacheFileObject>(activityToUndo.object)if (objectToUndo.type === 'CacheFile') {- return retryTransactionWrapper(processUndoCacheFile, byActor, activity, objectToUndo)+ return retryTransactionWrapper(() => {+ return processUndoCacheFile(byActor, activity as ActivityUndo<ActivityCreate<CacheFileObject>>, objectToUndo)+ })}}if (activityToUndo.type === 'Dislike') {- return retryTransactionWrapper(processUndoDislike, byActor, activity)+ return retryTransactionWrapper(() => processUndoDislike(byActor, activity as ActivityUndo<ActivityDislike>))}if (activityToUndo.type === 'Follow') {- return retryTransactionWrapper(processUndoFollow, byActor, activityToUndo)+ return retryTransactionWrapper(() => processUndoFollow(byActor, activityToUndo))}if (activityToUndo.type === 'Announce') {- return retryTransactionWrapper(processUndoAnnounce, byActor, activityToUndo)+ return retryTransactionWrapper(() => processUndoAnnounce(byActor, activityToUndo))}logger.warn('Unknown activity object type %s -> %s when undo activity.', activityToUndo.type, { activity: activity.id })diff --git a/server/core/lib/activitypub/process/process-update.ts b/server/core/lib/activitypub/process/process-update.tsindex bb5b0283c0..12335d4c94 100644--- a/server/core/lib/activitypub/process/process-update.ts+++ b/server/core/lib/activitypub/process/process-update.ts@@ -26,8 +26,8 @@ import { createOrUpdateCacheFile } from '../cache-file.js'import { upsertAPPlayerSettings } from '../player-settings.js'import { createOrUpdateVideoPlaylist } from '../playlists/index.js'import { forwardVideoRelatedActivity } from '../send/shared/send-utils.js'-import { APVideoUpdater, canVideoBeFederated, getOrCreateAPVideo, maybeGetOrCreateAPVideo } from '../videos/index.js'import { checkUrlsSameHost } from '../url.js'+import { APVideoUpdater, canVideoBeFederated, getOrCreateAPVideo, maybeGetOrCreateAPVideo } from '../videos/index.js'async function processUpdateActivity (options: APProcessorOptions<ActivityUpdate<ActivityUpdateObject>>) {const { activity, byActor } = options@@ -36,27 +36,27 @@ async function processUpdateActivity (options: APProcessorOptions<ActivityUpdateconst objectType = object.typeif (objectType === 'Video') {- return retryTransactionWrapper(processUpdateVideo, byActor, activity)+ return retryTransactionWrapper(() => processUpdateVideo(byActor, activity as ActivityUpdate<VideoObject | string>))}if (isActorTypeValid(objectType as ActivityPubActorType)) {// We need more attributesconst byActorFull = await ActorModel.loadByUrlAndPopulateAccountAndChannel(byActor.url)- return retryTransactionWrapper(processUpdateActor, byActorFull, object)+ return retryTransactionWrapper(() => processUpdateActor(byActorFull, object as ActivityPubActor))}if (objectType === 'CacheFile') {// We need more attributesconst byActorFull = await ActorModel.loadByUrlAndPopulateAccountAndChannel(byActor.url)- return retryTransactionWrapper(processUpdateCacheFile, byActorFull, activity, object)+ return retryTransactionWrapper(() => processUpdateCacheFile(byActorFull, activity as ActivityUpdate<CacheFileObject | string>, object))}if (objectType === 'Playlist') {- return retryTransactionWrapper(processUpdatePlaylist, byActor, activity, object)+ return retryTransactionWrapper(() => processUpdatePlaylist(byActor, activity as ActivityUpdate<PlaylistObject | string>, object))}if (objectType === 'PlayerSettings') {- return retryTransactionWrapper(processUpdatePlayerSettings, byActor, object)+ return retryTransactionWrapper(() => processUpdatePlayerSettings(byActor, object))}return undefineddiff --git a/server/core/lib/activitypub/videos/get.ts b/server/core/lib/activitypub/videos/get.tsindex e80ec7067a..1332d46bdc 100644--- a/server/core/lib/activitypub/videos/get.ts+++ b/server/core/lib/activitypub/videos/get.ts@@ -78,7 +78,7 @@ export async function getOrCreateAPVideo (try {const creator = new APVideoCreator(videoObject)- const { autoBlacklisted, videoCreated } = await retryTransactionWrapper(creator.create.bind(creator))+ const { autoBlacklisted, videoCreated } = await retryTransactionWrapper(() => creator.create())await syncVideoExternalAttributes(videoCreated, videoObject, syncParam)diff --git a/server/core/lib/job-queue/handlers/activitypub-follow.ts b/server/core/lib/job-queue/handlers/activitypub-follow.tsindex b3571beb82..d0f1a01eaa 100644--- a/server/core/lib/job-queue/handlers/activitypub-follow.ts+++ b/server/core/lib/job-queue/handlers/activitypub-follow.ts@@ -1,18 +1,18 @@-import { Job } from 'bullmq'-import { getLocalActorFollowActivityPubUrl } from '@server/lib/activitypub/url.js'import { ActivitypubFollowPayload } from '@peertube/peertube-models'+import { getApplicationActorOfHost } from '@server/helpers/activity-pub-utils.js'+import { getLocalActorFollowActivityPubUrl } from '@server/lib/activitypub/url.js'+import { Job } from 'bullmq'import { sanitizeHost } from '../../../helpers/core-utils.js'import { retryTransactionWrapper } from '../../../helpers/database-utils.js'import { logger } from '../../../helpers/logger.js'import { REMOTE_SCHEME, SERVER_ACTOR_NAME, WEBSERVER } from '../../../initializers/constants.js'import { sequelizeTypescript } from '../../../initializers/database.js'-import { ActorModel } from '../../../models/actor/actor.js'import { ActorFollowModel } from '../../../models/actor/actor-follow.js'+import { ActorModel } from '../../../models/actor/actor.js'import { MActor, MActorFull } from '../../../types/models/index.js'import { getOrCreateAPActor, loadActorUrlOrGetFromWebfinger } from '../../activitypub/actors/index.js'import { sendFollow } from '../../activitypub/send/index.js'import { Notifier } from '../../notifier/index.js'-import { getApplicationActorOfHost } from '@server/helpers/activity-pub-utils.js'async function processActivityPubFollow (job: Job) {const payload = job.data as ActivitypubFollowPayload@@ -56,7 +56,7 @@ async function processActivityPubFollow (job: Job) {const fromActor = await ActorModel.load(payload.followerActorId)- return retryTransactionWrapper(follow, fromActor, targetActor, payload.isAutoFollow)+ return retryTransactionWrapper(() => follow(fromActor, targetActor, payload.isAutoFollow))}// ---------------------------------------------------------------------------diff --git a/server/core/lib/runners/job-handlers/abstract-vod-transcoding-job-handler.ts b/server/core/lib/runners/job-handlers/abstract-vod-transcoding-job-handler.tsindex f588cb1c55..e350e5c919 100644--- a/server/core/lib/runners/job-handlers/abstract-vod-transcoding-job-handler.ts+++ b/server/core/lib/runners/job-handlers/abstract-vod-transcoding-job-handler.ts@@ -59,7 +59,7 @@ export abstract class AbstractVODTranscodingJobHandler<C, U extends RunnerJobUpdthis.lTags(video.uuid))- await retryTransactionWrapper(moveToNextState, { video })+ await retryTransactionWrapper(() => moveToNextState({ video }))}}}diff --git a/server/core/lib/transcoding/ended-transcoding.ts b/server/core/lib/transcoding/ended-transcoding.tsindex 8bbde3d618..212ba500ce 100644--- a/server/core/lib/transcoding/ended-transcoding.ts+++ b/server/core/lib/transcoding/ended-transcoding.ts@@ -17,7 +17,7 @@ export async function onTranscodingEnded (options: {await VideoJobInfoModel.decrease(video.uuid, 'pendingTranscode')if (moveVideoToNextState) {- const changedState = await retryTransactionWrapper(moveToNextState, { video })+ const changedState = await retryTransactionWrapper(() => moveToNextState({ video }))// Still send the transcoded file to external storage if neededif (!changedState && CONFIG.OBJECT_STORAGE.ENABLED && await hasVideoResourcesToBeMoved(video, FileStorage.OBJECT_STORAGE)) {diff --git a/server/core/lib/user-import-export/importers/video-playlists-importer.ts b/server/core/lib/user-import-export/importers/video-playlists-importer.tsindex 0c86d883b9..f4beb8d22f 100644--- a/server/core/lib/user-import-export/importers/video-playlists-importer.ts+++ b/server/core/lib/user-import-export/importers/video-playlists-importer.ts@@ -1,17 +1,9 @@+import { pick } from '@peertube/peertube-core-utils'import { VideoPlaylistPrivacy, VideoPlaylistType, VideoPlaylistsExportJSON } from '@peertube/peertube-models'-import { logger, loggerTagsFactory } from '@server/helpers/logger.js'import { buildUUID } from '@peertube/peertube-node-utils'-import { MChannelBannerAccountDefault, MVideoPlaylistFull, MVideoPlaylistThumbnail } from '@server/types/models/index.js'-import { getLocalVideoPlaylistActivityPubUrl, getLocalVideoPlaylistElementActivityPubUrl } from '@server/lib/activitypub/url.js'-import { VideoChannelModel } from '@server/models/video/video-channel.js'-import { VideoPlaylistModel } from '@server/models/video/video-playlist.js'-import { AbstractUserImporter } from './abstract-user-importer.js'-import { sendCreateVideoPlaylist } from '@server/lib/activitypub/send/send-create.js'-import { sequelizeTypescript } from '@server/initializers/database.js'-import { createLocalPlaylistThumbnailsFromImage } from '@server/lib/thumbnail.js'-import { CONSTRAINTS_FIELDS, USER_IMPORT } from '@server/initializers/constants.js'-import { VideoPlaylistElementModel } from '@server/models/video/video-playlist-element.js'-import { loadOrCreateVideoIfAllowedForUser } from '@server/lib/model-loaders/video.js'+import { isActorPreferredUsernameValid } from '@server/helpers/custom-validators/activitypub/actor.js'+import { isUrlValid } from '@server/helpers/custom-validators/activitypub/misc.js'+import { isArray } from '@server/helpers/custom-validators/misc.js'import {isVideoPlaylistDescriptionValid,isVideoPlaylistNameValid,@@ -19,13 +11,21 @@ import {isVideoPlaylistTimestampValid,isVideoPlaylistTypeValid} from '@server/helpers/custom-validators/video-playlists.js'-import { isActorPreferredUsernameValid } from '@server/helpers/custom-validators/activitypub/actor.js'import { saveInTransactionWithRetries } from '@server/helpers/database-utils.js'-import { isArray } from '@server/helpers/custom-validators/misc.js'-import { isUrlValid } from '@server/helpers/custom-validators/activitypub/misc.js'-import { pick } from '@peertube/peertube-core-utils'+import { logger, loggerTagsFactory } from '@server/helpers/logger.js'+import { CONSTRAINTS_FIELDS, USER_IMPORT } from '@server/initializers/constants.js'+import { sequelizeTypescript } from '@server/initializers/database.js'+import { sendCreateVideoPlaylist } from '@server/lib/activitypub/send/send-create.js'+import { getLocalVideoPlaylistActivityPubUrl, getLocalVideoPlaylistElementActivityPubUrl } from '@server/lib/activitypub/url.js'+import { loadOrCreateVideoIfAllowedForUser } from '@server/lib/model-loaders/video.js'+import { createLocalPlaylistThumbnailsFromImage } from '@server/lib/thumbnail.js'+import { generateThumbnailForPlaylistIfNeeded } from '@server/lib/video-playlist.js'+import { VideoChannelModel } from '@server/models/video/video-channel.js'+import { VideoPlaylistElementModel } from '@server/models/video/video-playlist-element.js'+import { VideoPlaylistModel } from '@server/models/video/video-playlist.js'import { VideoModel } from '@server/models/video/video.js'-import { generateThumbnailForPlaylist } from '@server/lib/video-playlist.js'+import { MChannelBannerAccountDefault, MVideoPlaylistFull, MVideoPlaylistThumbnail } from '@server/types/models/index.js'+import { AbstractUserImporter } from './abstract-user-importer.js'const lTags = loggerTagsFactory('user-import')@@ -149,8 +149,9 @@ export class VideoPlaylistsImporter extends AbstractUserImporter<VideoPlaylistsE})}- await sequelizeTypescript.transaction(async t => {+ const createdElements = await sequelizeTypescript.transaction(async t => {let position = await VideoPlaylistElementModel.getNextPositionOf(playlist.id, t)+ const createdElements: VideoPlaylistElementModel[] = []for (const elementToCreate of elementsToCreate) {const playlistElement = new VideoPlaylistElementModel({@@ -165,15 +166,27 @@ export class VideoPlaylistsImporter extends AbstractUserImporter<VideoPlaylistsEplaylistElement.url = getLocalVideoPlaylistElementActivityPubUrl(playlist, playlistElement)await playlistElement.save({ transaction: t })- if (playlist.shouldGenerateThumbnailWithNewElement(playlistElement)) {- const video = await VideoModel.loadFull(elementToCreate.videoId)-- generateThumbnailForPlaylist(playlist, video)- .catch(err => logger.error('Cannot generate thumbnail from playlist', { err }))- }+ createdElements.push(playlistElement)position++}++ return createdElements})++ // Generate the playlist thumbnail outside of the transaction, from the first element that has one+ await playlist.reloadThumbnails()++ for (const element of createdElements) {+ if (playlist.hasThumbnail()) break++ const video = await VideoModel.loadFull(element.videoId)++ try {+ await generateThumbnailForPlaylistIfNeeded({ videoPlaylist: playlist, element, video })+ } catch (err) {+ logger.error('Cannot generate thumbnail of playlist', { err, ...lTags() })+ }+ }}}diff --git a/server/core/lib/video-playlist.ts b/server/core/lib/video-playlist.tsindex dd05f8aa3d..a0ded9bcea 100644--- a/server/core/lib/video-playlist.ts+++ b/server/core/lib/video-playlist.ts@@ -1,14 +1,17 @@import { VideoPlaylistPrivacy, VideoPlaylistType } from '@peertube/peertube-models'+import { retryTransactionWrapper } from '@server/helpers/database-utils.js'import { generateImageFilename } from '@server/helpers/image-utils.js'import { logger } from '@server/helpers/logger.js'import { CONFIG } from '@server/initializers/config.js'+import { sequelizeTypescript } from '@server/initializers/database.js'import { SequelizeModel } from '@server/models/shared/sequelize-type.js'import { VideoPlaylistElementModel } from '@server/models/video/video-playlist-element.js'-import { copy } from 'fs-extra/esm'+import { Mutex } from 'async-mutex'+import { copy, remove } from 'fs-extra/esm'import { extname, join } from 'path'import { Transaction } from 'sequelize'import { VideoPlaylistModel } from '../models/video/video-playlist.js'-import { MAccount, MVideoThumbnails } from '../types/models/index.js'+import { MAccount, MVideoPlaylistElement, MVideoThumbnails } from '../types/models/index.js'import { MVideoPlaylistOwner, MVideoPlaylistThumbnail } from '../types/models/video/video-playlist.js'import { sendUpdateVideoPlaylist } from './activitypub/send/send-update.js'import { getLocalVideoPlaylistActivityPubUrl } from './activitypub/url.js'@@ -32,7 +35,34 @@ export async function createWatchLaterPlaylist (account: MAccount, t: Transactioreturn videoPlaylist}-export async function generateThumbnailForPlaylist (videoPlaylist: MVideoPlaylistThumbnail, video: MVideoThumbnails) {+// Generate the playlist thumbnail from the thumbnail of the video that has just been added, if the playlist needs one+export function generateThumbnailForPlaylistIfNeeded (options: {+ videoPlaylist: MVideoPlaylistThumbnail+ element: MVideoPlaylistElement+ video: MVideoThumbnails+}) {+ const { videoPlaylist, element, video } = options++ return runInPlaylistThumbnailMutex(videoPlaylist, async () => {+ if (videoPlaylist.shouldGenerateThumbnailWithNewElement(element) !== true) return++ await generateThumbnailForPlaylist(videoPlaylist, video)+ })+}++// The first element of the playlist changed: regenerate the automatic thumbnail from the new first element+export function regeneratePlaylistThumbnailIfNeeded (videoPlaylist: MVideoPlaylistThumbnail) {+ return runInPlaylistThumbnailMutex(videoPlaylist, async () => {+ if (videoPlaylist.hasGeneratedThumbnail() !== true) return++ await videoPlaylist.removeThumbnails(undefined)++ const firstElement = await VideoPlaylistElementModel.loadFirstElementWithVideoThumbnail(videoPlaylist.id)+ if (firstElement) await generateThumbnailForPlaylist(videoPlaylist, firstElement.Video)+ })+}++async function generateThumbnailForPlaylist (videoPlaylist: MVideoPlaylistThumbnail, video: MVideoThumbnails) {logger.info('Generating default thumbnail to playlist %s.', videoPlaylist.url)const videoThumbnail = video.getBestThumbnail('16:9')@@ -42,9 +72,10 @@ export async function generateThumbnailForPlaylist (videoPlaylist: MVideoPlaylis}const tmpImageName = generateImageFilename(extname(videoThumbnail.filename))+ const tmpImagePath = join(CONFIG.STORAGE.TMP_DIR, tmpImageName)if (video.isLocal() === true) {- await copy(videoThumbnail.getFSPath(), join(CONFIG.STORAGE.TMP_DIR, tmpImageName))+ await copy(videoThumbnail.getFSPath(), tmpImagePath)} else {await downloadImage({url: videoThumbnail.fileUrl,@@ -53,14 +84,67 @@ export async function generateThumbnailForPlaylist (videoPlaylist: MVideoPlaylis})}- const thumbnails = await createLocalPlaylistThumbnailsFromImage({- inputPath: join(CONFIG.STORAGE.TMP_DIR, tmpImageName),- playlist: videoPlaylist,- automaticallyGenerated: true,- keepOriginal: false- })+ try {+ // Another process (or a thumbnail uploaded by the user) can replace the thumbnails of this playlist at the same time:+ // the unique index on the thumbnail size is what prevents duplicates, so just retry and let the last writer win+ await retryTransactionWrapper(async () => {+ // Build the thumbnails inside the retry: sequelize would not insert again the models of a rolled back attempt+ const thumbnails = await createLocalPlaylistThumbnailsFromImage({+ inputPath: tmpImagePath,+ playlist: videoPlaylist,+ automaticallyGenerated: true,+ keepOriginal: true+ })++ try {+ await sequelizeTypescript.transaction(t => videoPlaylist.replaceAndSaveThumbnails(thumbnails, t))+ } catch (err) {+ // The next attempt generates new files, so don't leak the ones we just created+ await Promise.all(+ thumbnails.map(thumbnail => {+ return thumbnail.removeFile()+ .catch(removeErr => logger.error('Cannot remove thumbnail file %s.', thumbnail.filename, { err: removeErr }))+ })+ )++ throw err+ }+ }, { retryUniqueConstraintViolation: true })+ } finally {+ await remove(tmpImagePath)+ }+}++// Multiple requests can concurrently add/remove/reorder elements of the same playlist and so try to generate+// its automatic thumbnail at the same time+const playlistThumbnailMutexes = new Map<number, { mutex: Mutex, users: number }>()++async function runInPlaylistThumbnailMutex<T> (videoPlaylist: MVideoPlaylistThumbnail, fn: () => Promise<T>) {+ const playlistId = videoPlaylist.id++ let entry = playlistThumbnailMutexes.get(playlistId)+ if (!entry) {+ entry = { mutex: new Mutex(), users: 0 }+ playlistThumbnailMutexes.set(playlistId, entry)+ }- await videoPlaylist.replaceAndSaveThumbnails(thumbnails, undefined)+ // Count ourselves before waiting for the lock so the entry cannot be removed under us+ entry.users++++ const releaser = await entry.mutex.acquire()++ try {+ await videoPlaylist.reloadThumbnails()++ return await fn()+ } finally {+ releaser()++ entry.users--++ // Don't leak mutexes of playlists that are not updated anymore+ if (entry.users === 0) playlistThumbnailMutexes.delete(playlistId)+ }}export async function reorderPlaylistOrElementsPosition<T extends typeof VideoPlaylistElementModel | typeof VideoPlaylistModel> (options: {diff --git a/server/core/middlewares/async.ts b/server/core/middlewares/async.tsindex 04d85a95cb..280a856561 100644--- a/server/core/middlewares/async.ts+++ b/server/core/middlewares/async.ts@@ -7,7 +7,7 @@ import { retryTransactionWrapper } from '../helpers/database-utils.js'export type RequestPromiseHandler = ValidationChain | ExpressPromiseHandler-function asyncMiddleware (fun: RequestPromiseHandler | RequestPromiseHandler[]) {+export function asyncMiddleware (fun: RequestPromiseHandler | RequestPromiseHandler[]) {return async (req: Request, res: Response, next: NextFunction) => {if (Array.isArray(fun) !== true) {return Promise.resolve((fun as RequestHandler)(req, res, next))@@ -32,17 +32,8 @@ function asyncMiddleware (fun: RequestPromiseHandler | RequestPromiseHandler[])}}-function asyncRetryTransactionMiddleware (fun: (req: Request, res: Response, next: NextFunction) => Promise<any>) {+export function asyncRetryTransactionMiddleware (fun: (req: Request, res: Response, next: NextFunction) => Promise<any>) {return (req: Request, res: Response, next: NextFunction) => {- return Promise.resolve(- retryTransactionWrapper(fun, req, res, next)- ).catch(err => next(err))+ return Promise.resolve(retryTransactionWrapper(() => fun(req, res, next))).catch(err => next(err))}}--// -----------------------------------------------------------------------------export {- asyncMiddleware,- asyncRetryTransactionMiddleware-}diff --git a/server/core/models/video/thumbnail.ts b/server/core/models/video/thumbnail.tsindex f956ddbdd5..8f9ff7445b 100644--- a/server/core/models/video/thumbnail.ts+++ b/server/core/models/video/thumbnail.ts@@ -1,10 +1,11 @@import { ActivityIconObject, Thumbnail, type ThumbnailAspectRatio } from '@peertube/peertube-models'import { AttributesOnly } from '@peertube/peertube-typescript-utils'+import { afterCommitIfTransaction } from '@server/helpers/database-utils.js'import { CONFIG } from '@server/initializers/config.js'import { MThumbnail } from '@server/types/models/index.js'import { remove } from 'fs-extra/esm'import { extname, join } from 'path'-import { Op } from 'sequelize'+import { Op, Transaction } from 'sequelize'import {AfterDestroy,AllowNull,@@ -24,6 +25,18 @@ import { buildSQLAttributes } from '../shared/table.js'import { VideoPlaylistModel } from './video-playlist.js'import { VideoModel } from './video.js'+// A thumbnail always belongs to a video or to a playlist, never to both+type ThumbnailOwner =+ | { videoId: number, videoPlaylistId?: never }+ | { videoPlaylistId: number, videoId?: never }++function pickOwner (owner: ThumbnailOwner): ThumbnailOwner {+ if (owner.videoId !== undefined) return { videoId: owner.videoId }+ if (owner.videoPlaylistId !== undefined) return { videoPlaylistId: owner.videoPlaylistId }++ throw new Error('Cannot build thumbnail query without a video id or a video playlist id')+}+export const thumbnailAPIAttributes = ['filename','fileUrl',@@ -44,6 +57,25 @@ export const thumbnailAPIAttributes = [{fields: [ 'filename' ],unique: true+ },+ // A video/playlist can only have one thumbnail of a given size+ {+ fields: [ 'videoId', 'width', 'height' ],+ where: {+ videoId: {+ [Op.ne]: null+ }+ },+ unique: true+ },+ {+ fields: [ 'videoPlaylistId', 'width', 'height' ],+ where: {+ videoPlaylistId: {+ [Op.ne]: null+ }+ },+ unique: true}]})@@ -110,10 +142,11 @@ export class ThumbnailModel extends SequelizeModel<ThumbnailModel> {declare updatedAt: Date@AfterDestroy- static removeFiles (instance: ThumbnailModel) {- // Don't block the transaction- instance.removeFile()- .catch(err => logger.error('Cannot remove thumbnail file %s.', instance.filename, { err }))+ static removeFiles (instance: ThumbnailModel, options: { transaction?: Transaction }) {+ afterCommitIfTransaction(options.transaction, () => {+ instance.removeFile()+ .catch(err => logger.error('Cannot remove thumbnail file %s.', instance.filename, { err }))+ })}// ---------------------------------------------------------------------------@@ -149,6 +182,55 @@ export class ThumbnailModel extends SequelizeModel<ThumbnailModel> {// ---------------------------------------------------------------------------+ static listOf (options: ThumbnailOwner & { transaction?: Transaction }) {+ return ThumbnailModel.findAll({+ where: pickOwner(options),+ transaction: options.transaction+ })+ }++ // Always fetch the thumbnails from the database to prevent concurrency issues+ static async removeAllOf (options: ThumbnailOwner & { transaction?: Transaction }) {+ for (const thumbnail of await ThumbnailModel.listOf(options)) {+ await thumbnail.destroy({ transaction: options.transaction })+ }+ }++ static async replaceAllOf (options: ThumbnailOwner & {+ thumbnails: MThumbnail[]+ transaction?: Transaction+ }) {+ const { thumbnails, transaction } = options+ const owner = pickOwner(options)++ if (thumbnails.length === 0) {+ throw new Error('Cannot replace thumbnails with an empty array, at least one thumbnail is required')+ }++ // Thumbnails that are updated in place instead of being re-created+ const keptIds = new Set(thumbnails.map(t => t.id).filter(id => !!id))++ // Remove the old thumbnails first, otherwise the new ones would conflict with the unique constraint on their size+ for (const thumbnail of await ThumbnailModel.listOf({ ...owner, transaction })) {+ if (keptIds.has(thumbnail.id)) continue++ await thumbnail.destroy({ transaction })+ }++ const savedThumbnails: ThumbnailModel[] = []++ for (const thumbnail of thumbnails) {+ if (owner.videoId !== undefined) thumbnail.videoId = owner.videoId+ else thumbnail.videoPlaylistId = owner.videoPlaylistId++ savedThumbnails.push(await thumbnail.save({ transaction }))+ }++ return savedThumbnails+ }++ // ---------------------------------------------------------------------------+getFSPath () {return join(CONFIG.STORAGE.THUMBNAILS_DIR, this.filename)}diff --git a/server/core/models/video/video-playlist.ts b/server/core/models/video/video-playlist.tsindex 12c45daab4..8d636f59b7 100644--- a/server/core/models/video/video-playlist.ts+++ b/server/core/models/video/video-playlist.ts@@ -622,38 +622,19 @@ export class VideoPlaylistModel extends SequelizeModel<VideoPlaylistModel> {// ---------------------------------------------------------------------------async replaceAndSaveThumbnails (thumbnails: MThumbnail[], transaction?: Transaction) {- if (thumbnails.length === 0) {- throw new Error('Cannot replace thumbnails with an empty array, at least one thumbnail is required')- }-- let oldThumbnails = Array.isArray(this.Thumbnails)- ? [ ...this.Thumbnails ]- : []-- this.Thumbnails = []-- for (const thumbnail of thumbnails) {- thumbnail.videoPlaylistId = this.id-- const savedThumbnail = await thumbnail.save({ transaction })- oldThumbnails = oldThumbnails.filter(t => t.id !== savedThumbnail.id)-- this.Thumbnails.push(savedThumbnail)- }-- for (const oldThumbnail of oldThumbnails) {- await oldThumbnail.destroy({ transaction })- }+ this.Thumbnails = await ThumbnailModel.replaceAllOf({ thumbnails, videoPlaylistId: this.id, transaction })}async removeThumbnails (t: Transaction) {- for (const thumbnail of this.Thumbnails) {- await thumbnail.destroy({ transaction: t })- }+ await ThumbnailModel.removeAllOf({ videoPlaylistId: this.id, transaction: t })this.Thumbnails = []}+ async reloadThumbnails (t?: Transaction) {+ this.Thumbnails = await ThumbnailModel.listOf({ videoPlaylistId: this.id, transaction: t })+ }+hasThumbnail () {return this.Thumbnails.length !== 0}diff --git a/server/core/models/video/video.ts b/server/core/models/video/video.tsindex cc326ace8d..16334db952 100644--- a/server/core/models/video/video.ts+++ b/server/core/models/video/video.ts@@ -1998,28 +1998,7 @@ export class VideoModel extends SequelizeModel<VideoModel> {// ---------------------------------------------------------------------------async replaceAndSaveThumbnails (thumbnails: MThumbnail[], transaction?: Transaction) {- if (thumbnails.length === 0) {- throw new Error('Cannot replace thumbnails with an empty array, at least one thumbnail is required')- }-- let oldThumbnails = Array.isArray(this.Thumbnails)- ? [ ...this.Thumbnails ]- : []-- this.Thumbnails = []-- for (const thumbnail of thumbnails) {- thumbnail.videoId = this.id-- const savedThumbnail = await thumbnail.save({ transaction })- oldThumbnails = oldThumbnails.filter(t => t.id !== savedThumbnail.id)-- this.Thumbnails.push(savedThumbnail)- }-- for (const oldThumbnail of oldThumbnails) {- await oldThumbnail.destroy({ transaction })- }+ this.Thumbnails = await ThumbnailModel.replaceAllOf({ thumbnails, videoId: this.id, transaction })}// ---------------------------------------------------------------------------