ActivityPub-federated video streaming platform using P2P directly in your web browser
Refactor a little bit auto tags
Chocobozzzd3766d0773fc
diff --git a/packages/tests/src/api/moderation/automatic-tags.ts b/packages/tests/src/api/moderation/automatic-tags.tsindex d3909df359..5d53f1477a 100644--- a/packages/tests/src/api/moderation/automatic-tags.ts+++ b/packages/tests/src/api/moderation/automatic-tags.ts@@ -468,7 +468,6 @@ describe('Test automatic tags', function () {await waitJobs(servers)const { total, data } = await servers[0].videos.listAllForAdmin({ autoTagOneOf: [ 'mickey list v2' ] })- console.log(data)expect(total).to.equal(0)expect(data).to.have.lengthOf(0)})diff --git a/server/core/controllers/api/videos/comment.ts b/server/core/controllers/api/videos/comment.tsindex 7df873fb2f..a4bd483d77 100644--- a/server/core/controllers/api/videos/comment.ts+++ b/server/core/controllers/api/videos/comment.ts@@ -287,7 +287,7 @@ async function listVideoCommentReplies (req: express.Request, res: express.Respoasync function addVideoCommentThread (req: express.Request, res: express.Response) {const videoCommentInfo: VideoCommentCreate = req.body- const { comment, pendingAutomaticTags } = await createLocalVideoComment({+ const { comment, holdStatus } = await createLocalVideoComment({text: videoCommentInfo.text,inReplyToComment: null,video: res.locals.videoWithRights,@@ -295,7 +295,8 @@ async function addVideoCommentThread (req: express.Request, res: express.Respons})// The `build-object-automatic-tags` job notifies once the held status of the comment is final- if (!pendingAutomaticTags) Notifier.Instance.notifyOnNewComment(comment)+ if (holdStatus !== 'held-for-auto-tags') Notifier.Instance.notifyOnNewComment(comment)+auditLogger.create(getAuditIdFromRes(res), new CommentAuditView(comment.toFormattedJSON()))Hooks.runAction('action:api.video-thread.created', { comment, req, res })@@ -306,7 +307,7 @@ async function addVideoCommentThread (req: express.Request, res: express.Responsasync function addVideoCommentReply (req: express.Request, res: express.Response) {const videoCommentInfo: VideoCommentCreate = req.body- const { comment, pendingAutomaticTags } = await createLocalVideoComment({+ const { comment, holdStatus } = await createLocalVideoComment({text: videoCommentInfo.text,inReplyToComment: res.locals.videoCommentFull,video: res.locals.videoWithRights,@@ -314,7 +315,8 @@ async function addVideoCommentReply (req: express.Request, res: express.Response})// The `build-object-automatic-tags` job notifies once the held status of the comment is final- if (!pendingAutomaticTags) Notifier.Instance.notifyOnNewComment(comment)+ if (holdStatus !== 'held-for-auto-tags') Notifier.Instance.notifyOnNewComment(comment)+auditLogger.create(getAuditIdFromRes(res), new CommentAuditView(comment.toFormattedJSON()))Hooks.runAction('action:api.video-comment-reply.created', { comment, req, res })diff --git a/server/core/controllers/api/videos/source.ts b/server/core/controllers/api/videos/source.tsindex 4e54894213..8f5a6b6c74 100644--- a/server/core/controllers/api/videos/source.ts+++ b/server/core/controllers/api/videos/source.ts@@ -2,11 +2,11 @@ import { buildAspectRatio } from '@peertube/peertube-core-utils'import { HttpStatusCode, VideoChannelActivityAction, VideoState } from '@peertube/peertube-models'import { sequelizeTypescript } from '@server/initializers/database.js'import { buildNonDuplicatedFederateVideoJob } from '@server/lib/activitypub/videos/federate.js'+import { buildNonDuplicatedVideoAutomaticTagsJob } from '@server/lib/automatic-tags/automatic-tags.js'import { CreateJobOptions, CreateJobTypeAndPayload, JobQueue } from '@server/lib/job-queue/index.js'import { Hooks } from '@server/lib/plugins/hooks.js'import { regenerateLocalVideoThumbnailsFromVideoIfNeeded } from '@server/lib/thumbnail.js'import { setupUploadResumableRoutes } from '@server/lib/uploadx.js'-import { buildNonDuplicatedVideoAutomaticTagsJob } from '@server/lib/automatic-tags/automatic-tags.js'import { autoBlacklistVideoIfNeeded } from '@server/lib/video-blacklist.js'import { regenerateTranscriptionTaskIfNeeded } from '@server/lib/video-captions.js'import { buildNewFile, createVideoSource } from '@server/lib/video-file.js'@@ -147,7 +147,7 @@ async function doReplaceVideoSourceResumable (req: express.Request, res: expressvideo,user,// The name and the description of the video did not change, so its automatic tags are still up to date- automaticTagsPending: false,+ holdIfAutoTagPolicy: false,isRemote: false,isNew: false,isNewFile: true,diff --git a/server/core/lib/activitypub/process/process-create.ts b/server/core/lib/activitypub/process/process-create.tsindex bab67a792b..ee4b6fdadd 100644--- a/server/core/lib/activitypub/process/process-create.ts+++ b/server/core/lib/activitypub/process/process-create.ts@@ -151,7 +151,7 @@ async function processCreateVideoComment (let video: MVideoAccountLightBlacklistAllFileslet created: booleanlet comment: MCommentOwnerVideo- let heldPendingAutomaticTags: boolean+ let heldForAutoTags: booleantry {const resolveThreadResult = await resolveThread({ url: commentObject.id, isVideo: false })@@ -160,7 +160,7 @@ async function processCreateVideoComment (video = resolveThreadResult.videocreated = resolveThreadResult.commentCreatedcomment = resolveThreadResult.comment- heldPendingAutomaticTags = resolveThreadResult.heldPendingAutomaticTags+ heldForAutoTags = resolveThreadResult.heldForAutoTags} catch (err) {logger.debug('Cannot process video comment because we could not resolve thread %s. Maybe it was not a video thread, so skip it.',@@ -198,7 +198,7 @@ async function processCreateVideoComment (}// The `build-object-automatic-tags` job notifies once the held status of the comment is final- if (created && !heldPendingAutomaticTags) Notifier.Instance.notifyOnNewComment(comment)+ if (created && !heldForAutoTags) Notifier.Instance.notifyOnNewComment(comment)}// The origin instance re-sends us the comment when we approved the replydiff --git a/server/core/lib/activitypub/video-comments.ts b/server/core/lib/activitypub/video-comments.tsindex 35339b7881..82a46975c6 100644--- a/server/core/lib/activitypub/video-comments.ts+++ b/server/core/lib/activitypub/video-comments.ts@@ -15,7 +15,7 @@ import {import { createCommentAutomaticTagsJob } from '../automatic-tags/automatic-tags.js'import { isRemoteVideoCommentAccepted } from '../moderation.js'import { Hooks } from '../plugins/hooks.js'-import { CommentReviewDecision, getCommentReviewDecision } from '../video-comment.js'+import { CommentHoldStatus, getCommentHoldStatus } from '../video-comment.js'import { fetchAP } from './activity.js'import { getOrCreateAPActor } from './actors/index.js'import { checkUrlsSameHost } from './url.js'@@ -33,7 +33,7 @@ type ResolveThreadResult = Promise<{video: MVideoAccountLightBlacklistAllFilescomment: MCommentOwnerVideocommentCreated: boolean- heldPendingAutomaticTags: boolean+ heldForAutoTags: boolean}>export async function addVideoComments (commentUrls: string[]) {@@ -55,6 +55,11 @@ export async function addVideoComments (commentUrls: string[]) {}, { concurrency: CRAWL_REQUEST_CONCURRENCY })}+// Resolves a full AP comment thread starting from `url`, walking up the `inReplyTo` chain until it hits+// either a comment already stored locally or the video the thread is attached to.+//+// `params.comments` accumulates the chain: index 0 ends up being the comment `url` originally pointed to+// and the last index is the comment directly replying to the video (the thread root)export async function resolveThread (params: ResolveThreadParams): ResolveThreadResult {const { url, isVideo } = params@@ -92,13 +97,14 @@ async function resolveCommentFromDB (params: ResolveThreadParams) {let parentComments = comments.concat([ commentFromDatabase ])- // Speed up things and resolve directly the thread+ // The rest of the thread above this comment is already in DB too: append it instead of re-fetching each ancestor from the remote instanceif (commentFromDatabase.InReplyToVideoComment) {const data = await VideoCommentModel.listThreadParentComments({ comment: commentFromDatabase, order: 'DESC' })parentComments = parentComments.concat(data)}+ // We know the video already, so skip straight to it instead of walking inReplyTo comment by commentreturn resolveThread({url: commentFromDatabase.Video.url,comments: parentComments,@@ -109,6 +115,7 @@ async function resolveCommentFromDB (params: ResolveThreadParams) {// ---------------------------------------------------------------------------+// Once the video is reached the whole chain is known and gets persisted root-firstasync function tryToResolveThreadFromVideo (params: ResolveThreadParams) {const { url, comments, commentCreated } = params@@ -128,9 +135,11 @@ async function tryToResolveThreadFromVideo (params: ResolveThreadParams) {let resultComment: MCommentOwnerVideo// comments[0] is the comment the thread was resolved for: only its held status is reported back to the caller,// the other comments in the array are ancestors synthesized to complete the thread- let resultCommentHeldPendingAutomaticTags = false+ let resultCommentHeldForAutoTags = falseif (comments.length !== 0) {+ // Process root-first (comments.length - 1 down to 0): each comment needs its parent's real DB id for+ // inReplyToCommentId/originCommentId, which only exists once the parent has been savedconst firstReply = comments[comments.length - 1] as MCommentOwnerVideofirstReply.inReplyToCommentId = nullfirstReply.originCommentId = null@@ -142,15 +151,16 @@ async function tryToResolveThreadFromVideo (params: ResolveThreadParams) {return undefined}- const firstReplyDecision = await assignReviewIfNew(firstReply, video)+ const { isNew: firstReplyIsNew, holdStatus: firstReplyHoldStatus } = await assignReviewIfNew(firstReply, video)comments[comments.length - 1] = await firstReply.save()- if (firstReplyDecision) {- if (comments.length === 1) resultCommentHeldPendingAutomaticTags = firstReplyDecision.pendingAutomaticTags+ // New comment: process auto tags+ if (firstReplyIsNew) {+ if (comments.length === 1) resultCommentHeldForAutoTags = firstReplyHoldStatus === 'held-for-auto-tags'createCommentAutomaticTagsJob({comment: firstReply,- moderation: firstReplyDecision.pendingAutomaticTags+ moderation: firstReplyHoldStatus === 'held-for-auto-tags'? 'release-hold': 'none',// Only the comment the thread was resolved for (comments[0]) must notify the video owner: the other ones@@ -171,16 +181,17 @@ async function tryToResolveThreadFromVideo (params: ResolveThreadParams) {return undefined}- const decision = await assignReviewIfNew(comment, video)+ const { isNew, holdStatus } = await assignReviewIfNew(comment, video)comments[i] = await comment.save()- if (decision) {- if (i === 0) resultCommentHeldPendingAutomaticTags = decision.pendingAutomaticTags+ // New comment: process auto tags+ if (isNew) {+ if (i === 0) resultCommentHeldForAutoTags = holdStatus === 'held-for-auto-tags'createCommentAutomaticTagsJob({comment,- moderation: decision.pendingAutomaticTags+ moderation: holdStatus === 'held-for-auto-tags'? 'release-hold': 'none',// comments[0] is the comment the thread was resolved for: only it must notify the video owner@@ -192,31 +203,38 @@ async function tryToResolveThreadFromVideo (params: ResolveThreadParams) {resultComment = comments[0] as MCommentOwnerVideo}- return { video, comment: resultComment, commentCreated, heldPendingAutomaticTags: resultCommentHeldPendingAutomaticTags }+ return { video, comment: resultComment, commentCreated, heldForAutoTags: resultCommentHeldForAutoTags }}-// Assign the held for review status of the comment if it's a new one, and return the decision (undefined if not new)+// Assign the held for review status of the comment if it's a new one+// `holdStatus` is only set for a new comment: an existing one keeps the status it already has in database// Automatic tags are built by a job, so a comment is held as soon as the account has a review policy// The job then releases it if the tags it ends up with don't match any of them-async function assignReviewIfNew (comment: MComment, video: MVideoAccountLight): Promise<CommentReviewDecision> {+async function assignReviewIfNew (comment: MComment, video: MVideoAccountLight): Promise<{+ isNew: boolean+ holdStatus?: CommentHoldStatus+}> {// Remote comment already exists in database -> we don't need to rebuild automatic tags- if (comment.id) return undefined+ if (comment.id) return { isNew: false }// Third parties rely on origin, so if origin has the comment it's not held for review- const decision = video.isLocal() || comment.isLocal()- ? await getCommentReviewDecision({ user: null, video, automaticTagsPending: true })- : { heldForReview: false, pendingAutomaticTags: false }+ const holdStatus: CommentHoldStatus = video.isLocal() || comment.isLocal()+ ? await getCommentHoldStatus({ user: null, video, holdIfAutoTagPolicy: true })+ : 'not-held'- comment.heldForReview = decision.heldForReview+ comment.heldForReview = holdStatus !== 'not-held'- return decision+ return { isNew: true, holdStatus }}// ---------------------------------------------------------------------------+// Despite the name, this also resolves the original leaf/target comment on the first call, not just+// its ancestors: it's the fallback once DB lookup and video resolution have both failed for `url`async function resolveRemoteParentComment (params: ResolveThreadParams) {const { url, comments } = params+ // Guard against unbounded/malicious inReplyTo chainsif (comments.length > ACTIVITY_PUB.MAX_RECURSION_COMMENTS) {throw new Error('Recursion limit reached when resolving a thread')}@@ -230,10 +248,12 @@ async function resolveRemoteParentComment (params: ResolveThreadParams) {const actorUrl = body.attributedToif (!actorUrl && body.type !== 'Tombstone') throw new Error('Miss attributed to in comment')+ // An actor hosted on another server can't author a comment on this one (identity spoofing guard)if (actorUrl && checkUrlsSameHost(url, actorUrl) !== true) {throw new Error(`Actor url ${actorUrl} has not the same host than the comment url ${url}`)}+ // The fetched object must claim an id on the same host it was fetched fromif (checkUrlsSameHost(body.id, url) !== true) {throw new Error(`Comment url ${url} host is different from the AP object id ${body.id}`)}diff --git a/server/core/lib/activitypub/videos/get.ts b/server/core/lib/activitypub/videos/get.tsindex f1eacffa1f..39247a90fb 100644--- a/server/core/lib/activitypub/videos/get.ts+++ b/server/core/lib/activitypub/videos/get.ts@@ -2,6 +2,7 @@ import { APObjectId } from '@peertube/peertube-models'import { retryTransactionWrapper } from '@server/helpers/database-utils.js'import { createLogger } from '@server/helpers/logger.js'import { loadVideoByUrl } from '@server/lib/model-loaders/index.js'+import { AutoBlacklistStatus } from '@server/lib/video-blacklist.js'import { MVideoAccountLightBlacklistAllFiles, MVideoImmutable, MVideoThumbnails, MVideoWithBlacklist } from '@server/types/models/index.js'import { getAPId } from '../activity.js'import { refreshVideoIfNeeded, scheduleVideoRefreshIfNeeded } from './refresh.js'@@ -12,7 +13,7 @@ const logger = createLogger()type GetVideoResult<T> = Promise<{video: Tcreated: boolean- autoBlacklisted?: boolean+ autoBlacklistStatus?: AutoBlacklistStatus}>type GetVideoParamAll = {@@ -82,11 +83,11 @@ export async function getOrCreateAPVideo (try {const creator = new APVideoCreator(videoObject)- const { autoBlacklisted, videoCreated } = await retryTransactionWrapper(() => creator.create())+ const { autoBlacklistStatus, videoCreated } = await retryTransactionWrapper(() => creator.create())await syncVideoExternalAttributes(videoCreated, videoObject, syncParam)- return { video: videoCreated, created: true, autoBlacklisted }+ return { video: videoCreated, created: true, autoBlacklistStatus }} catch (err) {// Maybe a concurrent getOrCreateAPVideo call created this videoif (err.name === 'SequelizeUniqueConstraintError') {diff --git a/server/core/lib/activitypub/videos/shared/abstract-builder.ts b/server/core/lib/activitypub/videos/shared/abstract-builder.tsindex 63676bf7db..7730255791 100644--- a/server/core/lib/activitypub/videos/shared/abstract-builder.ts+++ b/server/core/lib/activitypub/videos/shared/abstract-builder.ts@@ -235,6 +235,7 @@ export abstract class APVideoAbstractBuilder {}// The automatic tags are built from the name and the description, so they only have to be rebuilt when one changed+ // TODO: add support if remote file changedprotected automaticTagsNeedRebuild (options: {video: MVideooldVideo?: Pick<MVideo, 'name' | 'description'>diff --git a/server/core/lib/activitypub/videos/shared/creator.ts b/server/core/lib/activitypub/videos/shared/creator.tsindex 647552807f..e9893057f0 100644--- a/server/core/lib/activitypub/videos/shared/creator.ts+++ b/server/core/lib/activitypub/videos/shared/creator.ts@@ -30,7 +30,7 @@ export class APVideoCreator extends APVideoAbstractBuilder {const videoData = getVideoAttributesFromObject(channel, this.videoObject, this.videoObject.to)const video = VideoModel.build({ ...videoData, likes: 0, dislikes: 0 }) as MVideoThumbnails- const { autoBlacklisted, videoCreated } = await sequelizeTypescript.transaction(async t => {+ const { autoBlacklistStatus, videoCreated } = await sequelizeTypescript.transaction(async t => {const videoCreated = await video.save({ transaction: t }) as MVideoFullvideoCreated.VideoChannel = channel@@ -46,9 +46,9 @@ export class APVideoCreator extends APVideoAbstractBuilder {// We added a video in this channel, set it as updatedawait channel.setAsUpdated(t)- const { blacklisted: autoBlacklisted, pendingAutomaticTags } = await autoBlacklistVideoIfNeeded({+ const autoBlacklistStatus = await autoBlacklistVideoIfNeeded({video: videoCreated,- automaticTagsPending: true,+ holdIfAutoTagPolicy: true,user: undefined,isRemote: true,isNew: true,@@ -58,7 +58,7 @@ export class APVideoCreator extends APVideoAbstractBuilder {createVideoAutomaticTagsJob({video: videoCreated,- moderation: pendingAutomaticTags+ moderation: autoBlacklistStatus === 'held-for-auto-tags'? 'release-hold': 'apply',transaction: t@@ -68,12 +68,12 @@ export class APVideoCreator extends APVideoAbstractBuilder {Hooks.runAction('action:activity-pub.remote-video.created', { video: videoCreated, videoAPObject: this.videoObject })- return { autoBlacklisted, videoCreated }+ return { autoBlacklistStatus, videoCreated }})await this.updateChapters(videoCreated)await this.upsertPlayerSettings(videoCreated)- return { autoBlacklisted, videoCreated }+ return { autoBlacklistStatus, videoCreated }}}diff --git a/server/core/lib/activitypub/videos/updater.ts b/server/core/lib/activitypub/videos/updater.tsindex c3c844fc05..809ce133ce 100644--- a/server/core/lib/activitypub/videos/updater.ts+++ b/server/core/lib/activitypub/videos/updater.ts@@ -91,12 +91,10 @@ export class APVideoUpdater extends APVideoAbstractBuilder {await this.updateChapters(videoUpdated)await this.upsertPlayerSettings(videoUpdated)- // The video is already published: don't hold it while its automatic tags are rebuilt, or a simple metadata- // update would block it and re-announce it to the subscribers once released- // The `build-object-automatic-tags` job applies the auto tag block policies afterwardsawait autoBlacklistVideoIfNeeded({video: videoUpdated,- automaticTagsPending: false,+ // Already published: don't hold it while its automatic tags are rebuilt+ holdIfAutoTagPolicy: false,user: undefined,isRemote: true,isNew: false,diff --git a/server/core/lib/automatic-tags/automatic-tags.ts b/server/core/lib/automatic-tags/automatic-tags.tsindex 631e5ae9f3..8380bd0deb 100644--- a/server/core/lib/automatic-tags/automatic-tags.ts+++ b/server/core/lib/automatic-tags/automatic-tags.ts@@ -1,5 +1,6 @@import { AutomaticTagPolicyType, AutomaticTagsModeration, BuildObjectAutomaticTagsPayload } from '@peertube/peertube-models'-import { afterCommitIfTransaction } from '@server/helpers/database-utils.js'+import { afterCommitIfTransaction, retryTransactionWrapper } from '@server/helpers/database-utils.js'+import { sequelizeTypescript } from '@server/initializers/database.js'import { getServerAccount } from '@server/models/application/application.js'import { AccountAutomaticTagPolicyModel } from '@server/models/automatic-tag/account-automatic-tag-policy.js'import { AutomaticTagModel } from '@server/models/automatic-tag/automatic-tag.js'@@ -65,33 +66,37 @@ export async function setAndSaveVideoAutomaticTags (options: {if (Object.keys(automaticTagsByAccount).length === 0) return- const { toCreateItems, toDeleteItems } = await _buildAutomaticTagItems({- automaticTagsByAccount,+ await retryTransactionWrapper(() => {+ return sequelizeTypescript.transaction(async transaction => {+ const { toCreateItems, toDeleteItems } = await _buildAutomaticTagItems({+ automaticTagsByAccount,- existingAutomaticTagsGetter: accountIds => {- return VideoAutomaticTagModel.listByAccountIdsAndVideoId({ videoId: video.id, accountIds })- }- })+ existingAutomaticTagsGetter: accountIds => {+ return VideoAutomaticTagModel.listByAccountIdsAndVideoId({ videoId: video.id, accountIds, transaction })+ }+ })- for (const item of toDeleteItems) {- await item.destroy()- }+ for (const item of toDeleteItems) {+ await item.destroy({ transaction })+ }- const videoAutomaticTags: MVideoAutomaticTagWithTag[] = []+ const videoAutomaticTags: MVideoAutomaticTagWithTag[] = []- for (const tag of toCreateItems) {- const automaticTagInstance = await AutomaticTagModel.findOrCreateAutomaticTag({ tag: tag.name })+ for (const tag of toCreateItems) {+ const automaticTagInstance = await AutomaticTagModel.findOrCreateAutomaticTag({ tag: tag.name, transaction })- const [ videoAutomaticTag ] = await VideoAutomaticTagModel.upsert({- accountId: tag.accountId,- automaticTagId: automaticTagInstance.id,- videoId: video.id- })+ const [ videoAutomaticTag ] = await VideoAutomaticTagModel.upsert({+ accountId: tag.accountId,+ automaticTagId: automaticTagInstance.id,+ videoId: video.id+ }, { transaction })- videoAutomaticTag.AutomaticTag = automaticTagInstance+ videoAutomaticTag.AutomaticTag = automaticTagInstance- videoAutomaticTags.push(videoAutomaticTag)- }+ videoAutomaticTags.push(videoAutomaticTag)+ }+ })+ })}async function _buildAutomaticTagItems<T extends MCommentAutomaticTagWithTag | MVideoAutomaticTagWithTag> (options: {diff --git a/server/core/lib/job-queue/handlers/build-object-automatic-tags.ts b/server/core/lib/job-queue/handlers/build-object-automatic-tags.tsindex 01090141bc..7b466e7f8b 100644--- a/server/core/lib/job-queue/handlers/build-object-automatic-tags.ts+++ b/server/core/lib/job-queue/handlers/build-object-automatic-tags.ts@@ -5,7 +5,7 @@ import { sequelizeTypescript } from '@server/initializers/database.js'import { buildAndSaveCommentAutomaticTags, buildAndSaveVideoAutomaticTags } from '@server/lib/automatic-tags/automatic-tags.js'import { Notifier } from '@server/lib/notifier/index.js'import { autoBlacklistVideoByAutoTagPolicyIfNeeded, resolvePendingAutoTagBlacklist } from '@server/lib/video-blacklist.js'-import { approveComment, getCommentReviewDecision } from '@server/lib/video-comment.js'+import { approveComment, getCommentHoldStatus } from '@server/lib/video-comment.js'import { UserModel } from '@server/models/user/user.js'import { VideoCommentModel } from '@server/models/video/video-comment.js'import { VideoModel } from '@server/models/video/video.js'@@ -80,17 +80,17 @@ async function buildOfComment (payload: BuildObjectAutomaticTagsPayload) {if (commentReloaded.heldForReview) {const video = commentReloaded.Video- const { heldForReview } = await getCommentReviewDecision({+ const holdStatus = await getCommentHoldStatus({user: commentReloaded.accountId? await UserModel.loadByAccountId(commentReloaded.accountId, t): null,video,- automaticTagsPending: false,+ holdIfAutoTagPolicy: false,ownerAutomaticTags: automaticTagsByAccount[video.VideoChannel.accountId],transaction: t})- if (!heldForReview) {+ if (holdStatus === 'not-held') {// The comment was only held while waiting for its automatic tags: its author must not be notified of an approvalawait approveComment(commentReloaded, { notify: false, transaction: t })commentReloaded.heldForReview = false@@ -100,8 +100,7 @@ async function buildOfComment (payload: BuildObjectAutomaticTagsPayload) {}// The video owner has not been notified at creation, because the held status of the comment was not final yet- // Notify it even if the comment is still held for review, so it can approve it (same as a comment held by- // a `commentsPolicy: REQUIRES_APPROVAL` video, which is notified synchronously at creation)+ // Notify it even if the comment is still held for review, so it can approve it// `notify` is false for a comment that only exists to complete a federated thread (see `resolveThread`)if (payload.notify !== false) {afterCommitIfTransaction(t, () => Notifier.Instance.notifyOnNewComment(commentReloaded))diff --git a/server/core/lib/job-queue/handlers/video-import.ts b/server/core/lib/job-queue/handlers/video-import.tsindex 28051d5f0f..0c4442841f 100644--- a/server/core/lib/job-queue/handlers/video-import.ts+++ b/server/core/lib/job-queue/handlers/video-import.ts@@ -309,13 +309,10 @@ async function afterImportSuccess (options: {const { video, videoFile, videoImport, user, generateTranscription } = options// The file now exists, so plugin auto taggers can analyze it- // They can be slow, so let the `build-object-automatic-tags` job run them instead of blocking this queue- // Hold the video until it has confirmed or released the hold: the video has not been announced yet- // Note the video may already be blocked by the instance policy, applied by `insertFromImportIntoDB`- const { pendingAutomaticTags } = await autoBlacklistVideoIfNeeded({+ const autoBlacklistStatus = await autoBlacklistVideoIfNeeded({video,user,- automaticTagsPending: true,+ holdIfAutoTagPolicy: true,isRemote: false,isNew: true,isNewFile: true,@@ -325,7 +322,7 @@ async function afterImportSuccess (options: {createVideoAutomaticTagsJob({video,- moderation: pendingAutomaticTags+ moderation: autoBlacklistStatus === 'held-for-auto-tags'? 'release-hold': 'apply'})@@ -334,14 +331,12 @@ async function afterImportSuccess (options: {// A video held for its automatic tags is announced by the job instead, which also notifies the moderators if it// ends up confirming the block- if (!pendingAutomaticTags) {- if (video.isBlacklisted()) {- const videoBlacklist = Object.assign(video.VideoBlacklist, { Video: video })+ if (autoBlacklistStatus === 'auto-blacklisted') {+ const videoBlacklist = Object.assign(video.VideoBlacklist, { Video: video })- Notifier.Instance.notifyOnVideoAutoBlacklist(videoBlacklist)- } else {- Notifier.Instance.notifyOnNewVideoOrLiveIfNeeded(video)- }+ Notifier.Instance.notifyOnVideoAutoBlacklist(videoBlacklist)+ } else if (autoBlacklistStatus === 'not-auto-blacklisted') {+ Notifier.Instance.notifyOnNewVideoOrLiveIfNeeded(video)}// Generate the storyboard in the job queue, and don't forget to federate an update afterdiff --git a/server/core/lib/local-video-creator.ts b/server/core/lib/local-video-creator.tsindex b8bf9291b5..5cf58529d1 100644--- a/server/core/lib/local-video-creator.ts+++ b/server/core/lib/local-video-creator.ts@@ -194,24 +194,23 @@ export class LocalVideoCreator {}}- // A video with a file has its automatic tags built by `addVideoJobsAfterCreation`, before it is announced.- // A video without one (live) is federated right away, so hold it until the tagging job has run- const holdForAutomaticTags = !this.videoFile-- const { pendingAutomaticTags } = await autoBlacklistVideoIfNeeded({+ const autoBlacklistStatus = await autoBlacklistVideoIfNeeded({video: this.video,user: this.options.user,- automaticTagsPending: holdForAutomaticTags,+ // A video with a file has its automatic tags built by `addVideoJobsAfterCreation`, before it is announced.+ // A video without one (live) is federated right away, so hold it until the tagging job has run+ holdIfAutoTagPolicy: !this.videoFile,isRemote: false,isNew: true,isNewFile: true,transaction})- if (holdForAutomaticTags) {+ // Keep it sync with `holdIfAutoTagPolicy` above+ if (!this.videoFile) {createVideoAutomaticTagsJob({video: this.video,- moderation: pendingAutomaticTags+ moderation: autoBlacklistStatus === 'held-for-auto-tags'? 'release-hold': 'apply',transactiondiff --git a/server/core/lib/local-video-updater.ts b/server/core/lib/local-video-updater.tsindex 9a04c83de9..3732211ac6 100644--- a/server/core/lib/local-video-updater.ts+++ b/server/core/lib/local-video-updater.ts@@ -209,7 +209,7 @@ export class LocalVideoUpdater {isRemote: false,isNew: false,isNewFile: false,- automaticTagsPending: false,+ holdIfAutoTagPolicy: false,transaction: t})diff --git a/server/core/lib/video-blacklist.ts b/server/core/lib/video-blacklist.tsindex c3e9a101a0..e6ce729420 100644--- a/server/core/lib/video-blacklist.ts+++ b/server/core/lib/video-blacklist.ts@@ -33,6 +33,8 @@ import { Hooks } from './plugins/hooks.js'const logger = createLogger('blacklist')+export type AutoBlacklistStatus = 'auto-blacklisted' | 'held-for-auto-tags' | 'not-auto-blacklisted'+export async function autoBlacklistVideoIfNeeded (parameters: {video: MVideoWithBlacklistLightisRemote: boolean@@ -40,17 +42,17 @@ export async function autoBlacklistVideoIfNeeded (parameters: {isNewFile: boolean// The automatic tags of the video have not been built yet: we don't know which tags it will have- automaticTagsPending: boolean+ holdIfAutoTagPolicy: booleanautomaticTagsByAccount?: Record<number, string[]>user?: MUsernotify?: booleantransaction?: Transaction-}) {- const { video, user, isRemote, isNew, isNewFile, automaticTagsPending, automaticTagsByAccount, notify = true, transaction } = parameters+}): Promise<AutoBlacklistStatus> {+ const { video, user, isRemote, isNew, isNewFile, holdIfAutoTagPolicy, automaticTagsByAccount, notify = true, transaction } = parameters// Already blacklisted- if (video.VideoBlacklist) return { blacklisted: true, pendingAutomaticTags: false }+ if (video.VideoBlacklist) return 'auto-blacklisted'const doAutoBlacklistByInstancePolicy = await Hooks.wrapFun(_autoBlacklistByInstancePolicyNeeded,@@ -58,27 +60,42 @@ export async function autoBlacklistVideoIfNeeded (parameters: {'filter:video.auto-blacklist.result')+ // Global instance policyif (doAutoBlacklistByInstancePolicy) {await _autoBlacklist({ video, notify, user, transaction, type: VideoBlacklistType.AUTO_BY_INSTANCE_POLICY })- return { blacklisted: true, pendingAutomaticTags: false }+ return 'auto-blacklisted'+ }++ // Special hold mode if the instance has auto tag policies that could block the video,+ // and the automatic tags of the video have not been built: block the video if the instance could want to review it+ // Let the `build-object-automatic-tags` job release it if the tags it ends up with don't match any of its auto block policies+ if (holdIfAutoTagPolicy && await _shouldHoldForAutoTagPolicy({ video, user, transaction })) {+ await _autoBlacklist({+ video,+ notify: false, // Don't notify, build auto tag job will notify if the video is really blocked+ user,+ transaction,+ type: VideoBlacklistType.AUTO_BY_AUTO_TAG_POLICY+ })++ return 'held-for-auto-tags'}- if (await _autoBlacklistByAutoTagPolicyNeeded({ video, user, automaticTagsPending, automaticTagsByAccount, transaction })) {- // We don't know yet if the video really matches a policy- // So don't notify the moderators before the `build-object-automatic-tags` job has confirmed it+ // Not in hold mode: auto blacklist the video if its automatic tags match an auto block policy+ if (await _shouldAutoBlacklistByAutoTagPolicy({ video, user, automaticTagsByAccount, transaction })) {await _autoBlacklist({video,- notify: notify && !automaticTagsPending,+ notify,user,transaction,type: VideoBlacklistType.AUTO_BY_AUTO_TAG_POLICY})- return { blacklisted: true, pendingAutomaticTags: automaticTagsPending }+ return 'auto-blacklisted'}- return { blacklisted: false, pendingAutomaticTags: false }+ return 'not-auto-blacklisted'}// Called by the `build-object-automatic-tags` job for a video that was not put on hold@@ -93,9 +110,7 @@ export async function autoBlacklistVideoByAutoTagPolicyIfNeeded (options: {const { video, automaticTagsByAccount, user, notify = true, transaction } = optionsif (video.VideoBlacklist) return false- if (!await _autoBlacklistByAutoTagPolicyNeeded({ video, user, automaticTagsPending: false, automaticTagsByAccount, transaction })) {- return false- }+ if (!await _shouldAutoBlacklistByAutoTagPolicy({ video, user, automaticTagsByAccount, transaction })) return falseawait _autoBlacklist({video,@@ -122,7 +137,7 @@ export async function resolvePendingAutoTagBlacklist (options: {const videoBlacklist = await VideoBlacklistModel.loadByVideoId(video.id, transaction)if (!videoBlacklist || videoBlacklist.type !== VideoBlacklistType.AUTO_BY_AUTO_TAG_POLICY) return- if (await _autoBlacklistByAutoTagPolicyNeeded({ video, automaticTagsPending: false, automaticTagsByAccount, transaction })) {+ if (await _shouldAutoBlacklistByAutoTagPolicy({ video, automaticTagsByAccount, transaction })) {const videoBlacklistWithVideo = videoBlacklist as MVideoBlacklistVideovideoBlacklistWithVideo.Video = video@@ -188,31 +203,38 @@ function _autoBlacklistByInstancePolicyNeeded (parameters: {return true}-async function _autoBlacklistByAutoTagPolicyNeeded (options: {+async function _shouldHoldForAutoTagPolicy (options: {+ video: MVideoWithBlacklistLight+ transaction?: Transaction+ user?: MUser+}) {+ const { user, video, transaction } = options++ if (video.isLocal() && user?.hasRight(UserRight.MANAGE_VIDEO_BLACKLIST)) return false++ const accountId = (await getServerAccount()).id++ return AccountAutomaticTagPolicyModel.hasPolicy({+ accountId,+ policy: AutomaticTagPolicy.AUTO_BLACKLIST_VIDEO,+ transaction+ })+}++async function _shouldAutoBlacklistByAutoTagPolicy (options: {video: MVideoWithBlacklistLight- automaticTagsPending: booleanautomaticTagsByAccount?: Record<number, string[]>transaction?: Transactionuser?: MUser}) {- const { user, video, transaction, automaticTagsPending, automaticTagsByAccount } = options+ const { user, video, transaction, automaticTagsByAccount } = options- if (!automaticTagsPending && !automaticTagsByAccount) return false+ if (!automaticTagsByAccount) return falseif (video.isLocal() && user?.hasRight(UserRight.MANAGE_VIDEO_BLACKLIST)) return falseconst accountId = (await getServerAccount()).id- // Block the video if the instance could want to review it- // Let the `build-object-automatic-tags` job release it if the tags it ends up with don't match any of its auto block policies- if (automaticTagsPending) {- return AccountAutomaticTagPolicyModel.hasPolicy({- accountId,- policy: AutomaticTagPolicy.AUTO_BLACKLIST_VIDEO,- transaction- })- }-const tags = automaticTagsByAccount?.[accountId]if (!tags || tags.length === 0) return falsediff --git a/server/core/lib/video-comment.ts b/server/core/lib/video-comment.tsindex 7abc37fedf..bc4665dcfe 100644--- a/server/core/lib/video-comment.ts+++ b/server/core/lib/video-comment.ts@@ -94,10 +94,10 @@ export async function createLocalVideoComment (options: {return sequelizeTypescript.transaction(async transaction => {const account = await AccountModel.load(user.Account.id, transaction)- const { heldForReview, pendingAutomaticTags } = await getCommentReviewDecision({+ const holdStatus = await getCommentHoldStatus({user,video,- automaticTagsPending: true,+ holdIfAutoTagPolicy: true,transaction})@@ -107,7 +107,7 @@ export async function createLocalVideoComment (options: {inReplyToCommentId,videoId: video.id,accountId: account.id,- heldForReview,+ heldForReview: holdStatus !== 'not-held',url: new Date().toISOString()}, { transaction, validate: false })@@ -117,7 +117,7 @@ export async function createLocalVideoComment (options: {createCommentAutomaticTagsJob({comment: savedComment,- moderation: pendingAutomaticTags+ moderation: holdStatus === 'held-for-auto-tags'? 'release-hold': 'none',notify: true,@@ -130,7 +130,7 @@ export async function createLocalVideoComment (options: {await sendCreateVideoCommentIfNeeded(savedComment, transaction)- return { comment: savedComment, pendingAutomaticTags }+ return { comment: savedComment, holdStatus }})}@@ -186,59 +186,59 @@ export function buildFormattedCommentTree (options: {}}-export type CommentReviewDecision = {- heldForReview: boolean+export type CommentHoldStatus = 'held-for-review' | 'held-for-auto-tags' | 'not-held'- // The comment is only held because its automatic tags are not built yet- // The `build-object-automatic-tags` job confirms or releases that hold, and is in charge of notifying the video owner of the new comment- pendingAutomaticTags: boolean-}--export async function getCommentReviewDecision (options: {+export async function getCommentHoldStatus (options: {user: MUserAccountIdvideo: MVideoAccountIdUrl// The automatic tags of the comment have not been built yet: we don't know which tags it will have- automaticTagsPending: boolean+ holdIfAutoTagPolicy: booleanownerAutomaticTags?: string[]transaction?: Transaction-}): Promise<CommentReviewDecision> {- const { user, video, transaction, automaticTagsPending, ownerAutomaticTags } = options-- const notHeld = { heldForReview: false, pendingAutomaticTags: false }+}): Promise<CommentHoldStatus> {+ const { user, video, transaction, holdIfAutoTagPolicy, ownerAutomaticTags } = options+ // User bypass checkif (video.isLocal() && user) {- if (user.hasRight(UserRight.MANAGE_ANY_VIDEO_COMMENT)) return notHeld- if (user.Account.id === video.VideoChannel.accountId) return notHeld+ if (user.hasRight(UserRight.MANAGE_ANY_VIDEO_COMMENT)) return 'not-held'+ if (user.Account.id === video.VideoChannel.accountId) return 'not-held'}+ // Global owner policyif (video.commentsPolicy === VideoCommentPolicy.REQUIRES_APPROVAL) {- return { heldForReview: true, pendingAutomaticTags: false }+ return 'held-for-review'}- if (video.isLocal() !== true) return notHeld+ // Don't check auto tags policy on remote videos+ if (video.isLocal() !== true) return 'not-held'// Hold the comment if the account could want to review it// Let the `build-object-automatic-tags` job release it if the tags it ends up with don't match any of its review policies- if (automaticTagsPending) {+ if (holdIfAutoTagPolicy) {const hasPolicy = await AccountAutomaticTagPolicyModel.hasPolicy({accountId: video.VideoChannel.accountId,policy: AutomaticTagPolicy.REVIEW_COMMENT,transaction})- return { heldForReview: hasPolicy, pendingAutomaticTags: hasPolicy }+ return hasPolicy+ ? 'held-for-auto-tags'+ : 'not-held'}- if (!ownerAutomaticTags || ownerAutomaticTags.length === 0) return notHeld+ if (!ownerAutomaticTags || ownerAutomaticTags.length === 0) return 'not-held'- const heldForReview = await AccountAutomaticTagPolicyModel.hasPolicyOnTags({+ // Check on specific auto tags provided+ const hasPolicy = await AccountAutomaticTagPolicyModel.hasPolicyOnTags({accountId: video.VideoChannel.accountId,policy: AutomaticTagPolicy.REVIEW_COMMENT,tags: ownerAutomaticTags,transaction})- return { heldForReview, pendingAutomaticTags: false }+ return hasPolicy+ ? 'held-for-review'+ : 'not-held'}diff --git a/server/core/lib/video-pre-import.ts b/server/core/lib/video-pre-import.tsindex 111e1d4fa7..181307d63a 100644--- a/server/core/lib/video-pre-import.ts+++ b/server/core/lib/video-pre-import.ts@@ -79,7 +79,7 @@ export async function insertFromImportIntoDB (parameters: {// The video stays in the `TO_IMPORT` state until then, so it is neither federated nor notifiedawait autoBlacklistVideoIfNeeded({video: videoCreated,- automaticTagsPending: false,+ holdIfAutoTagPolicy: false,user,notify: false,isRemote: false,