Skip to content

ActivityPub-federated video streaming platform using P2P directly in your web browser

Refactor a little bit auto tags

Chocobozzzd3766d0773fc
diff --git a/packages/tests/src/api/moderation/automatic-tags.ts b/packages/tests/src/api/moderation/automatic-tags.ts
index d3909df359..5d53f1477a 100644
--- a/packages/tests/src/api/moderation/automatic-tags.ts
+++ b/packages/tests/src/api/moderation/automatic-tags.ts
@@ -468,7 +468,6 @@ describe('Test automatic tags', function () {
await waitJobs(servers)
const { total, data } = await servers[0].videos.listAllForAdmin({ autoTagOneOf: [ 'mickey list v2' ] })
- console.log(data)
expect(total).to.equal(0)
expect(data).to.have.lengthOf(0)
})
diff --git a/server/core/controllers/api/videos/comment.ts b/server/core/controllers/api/videos/comment.ts
index 7df873fb2f..a4bd483d77 100644
--- a/server/core/controllers/api/videos/comment.ts
+++ b/server/core/controllers/api/videos/comment.ts
@@ -287,7 +287,7 @@ async function listVideoCommentReplies (req: express.Request, res: express.Respo
async function addVideoCommentThread (req: express.Request, res: express.Response) {
const videoCommentInfo: VideoCommentCreate = req.body
- const { comment, pendingAutomaticTags } = await createLocalVideoComment({
+ const { comment, holdStatus } = await createLocalVideoComment({
text: videoCommentInfo.text,
inReplyToComment: null,
video: res.locals.videoWithRights,
@@ -295,7 +295,8 @@ async function addVideoCommentThread (req: express.Request, res: express.Respons
})
// The `build-object-automatic-tags` job notifies once the held status of the comment is final
- if (!pendingAutomaticTags) Notifier.Instance.notifyOnNewComment(comment)
+ if (holdStatus !== 'held-for-auto-tags') Notifier.Instance.notifyOnNewComment(comment)
+
auditLogger.create(getAuditIdFromRes(res), new CommentAuditView(comment.toFormattedJSON()))
Hooks.runAction('action:api.video-thread.created', { comment, req, res })
@@ -306,7 +307,7 @@ async function addVideoCommentThread (req: express.Request, res: express.Respons
async function addVideoCommentReply (req: express.Request, res: express.Response) {
const videoCommentInfo: VideoCommentCreate = req.body
- const { comment, pendingAutomaticTags } = await createLocalVideoComment({
+ const { comment, holdStatus } = await createLocalVideoComment({
text: videoCommentInfo.text,
inReplyToComment: res.locals.videoCommentFull,
video: res.locals.videoWithRights,
@@ -314,7 +315,8 @@ async function addVideoCommentReply (req: express.Request, res: express.Response
})
// The `build-object-automatic-tags` job notifies once the held status of the comment is final
- if (!pendingAutomaticTags) Notifier.Instance.notifyOnNewComment(comment)
+ if (holdStatus !== 'held-for-auto-tags') Notifier.Instance.notifyOnNewComment(comment)
+
auditLogger.create(getAuditIdFromRes(res), new CommentAuditView(comment.toFormattedJSON()))
Hooks.runAction('action:api.video-comment-reply.created', { comment, req, res })
diff --git a/server/core/controllers/api/videos/source.ts b/server/core/controllers/api/videos/source.ts
index 4e54894213..8f5a6b6c74 100644
--- a/server/core/controllers/api/videos/source.ts
+++ b/server/core/controllers/api/videos/source.ts
@@ -2,11 +2,11 @@ import { buildAspectRatio } from '@peertube/peertube-core-utils'
import { HttpStatusCode, VideoChannelActivityAction, VideoState } from '@peertube/peertube-models'
import { sequelizeTypescript } from '@server/initializers/database.js'
import { buildNonDuplicatedFederateVideoJob } from '@server/lib/activitypub/videos/federate.js'
+import { buildNonDuplicatedVideoAutomaticTagsJob } from '@server/lib/automatic-tags/automatic-tags.js'
import { CreateJobOptions, CreateJobTypeAndPayload, JobQueue } from '@server/lib/job-queue/index.js'
import { Hooks } from '@server/lib/plugins/hooks.js'
import { regenerateLocalVideoThumbnailsFromVideoIfNeeded } from '@server/lib/thumbnail.js'
import { setupUploadResumableRoutes } from '@server/lib/uploadx.js'
-import { buildNonDuplicatedVideoAutomaticTagsJob } from '@server/lib/automatic-tags/automatic-tags.js'
import { autoBlacklistVideoIfNeeded } from '@server/lib/video-blacklist.js'
import { regenerateTranscriptionTaskIfNeeded } from '@server/lib/video-captions.js'
import { buildNewFile, createVideoSource } from '@server/lib/video-file.js'
@@ -147,7 +147,7 @@ async function doReplaceVideoSourceResumable (req: express.Request, res: express
video,
user,
// The name and the description of the video did not change, so its automatic tags are still up to date
- automaticTagsPending: false,
+ holdIfAutoTagPolicy: false,
isRemote: false,
isNew: false,
isNewFile: true,
diff --git a/server/core/lib/activitypub/process/process-create.ts b/server/core/lib/activitypub/process/process-create.ts
index bab67a792b..ee4b6fdadd 100644
--- a/server/core/lib/activitypub/process/process-create.ts
+++ b/server/core/lib/activitypub/process/process-create.ts
@@ -151,7 +151,7 @@ async function processCreateVideoComment (
let video: MVideoAccountLightBlacklistAllFiles
let created: boolean
let comment: MCommentOwnerVideo
- let heldPendingAutomaticTags: boolean
+ let heldForAutoTags: boolean
try {
const resolveThreadResult = await resolveThread({ url: commentObject.id, isVideo: false })
@@ -160,7 +160,7 @@ async function processCreateVideoComment (
video = resolveThreadResult.video
created = resolveThreadResult.commentCreated
comment = resolveThreadResult.comment
- heldPendingAutomaticTags = resolveThreadResult.heldPendingAutomaticTags
+ heldForAutoTags = resolveThreadResult.heldForAutoTags
} catch (err) {
logger.debug(
'Cannot process video comment because we could not resolve thread %s. Maybe it was not a video thread, so skip it.',
@@ -198,7 +198,7 @@ async function processCreateVideoComment (
}
// The `build-object-automatic-tags` job notifies once the held status of the comment is final
- if (created && !heldPendingAutomaticTags) Notifier.Instance.notifyOnNewComment(comment)
+ if (created && !heldForAutoTags) Notifier.Instance.notifyOnNewComment(comment)
}
// The origin instance re-sends us the comment when we approved the reply
diff --git a/server/core/lib/activitypub/video-comments.ts b/server/core/lib/activitypub/video-comments.ts
index 35339b7881..82a46975c6 100644
--- a/server/core/lib/activitypub/video-comments.ts
+++ b/server/core/lib/activitypub/video-comments.ts
@@ -15,7 +15,7 @@ import {
import { createCommentAutomaticTagsJob } from '../automatic-tags/automatic-tags.js'
import { isRemoteVideoCommentAccepted } from '../moderation.js'
import { Hooks } from '../plugins/hooks.js'
-import { CommentReviewDecision, getCommentReviewDecision } from '../video-comment.js'
+import { CommentHoldStatus, getCommentHoldStatus } from '../video-comment.js'
import { fetchAP } from './activity.js'
import { getOrCreateAPActor } from './actors/index.js'
import { checkUrlsSameHost } from './url.js'
@@ -33,7 +33,7 @@ type ResolveThreadResult = Promise<{
video: MVideoAccountLightBlacklistAllFiles
comment: MCommentOwnerVideo
commentCreated: boolean
- heldPendingAutomaticTags: boolean
+ heldForAutoTags: boolean
}>
export async function addVideoComments (commentUrls: string[]) {
@@ -55,6 +55,11 @@ export async function addVideoComments (commentUrls: string[]) {
}, { concurrency: CRAWL_REQUEST_CONCURRENCY })
}
+// Resolves a full AP comment thread starting from `url`, walking up the `inReplyTo` chain until it hits
+// either a comment already stored locally or the video the thread is attached to.
+//
+// `params.comments` accumulates the chain: index 0 ends up being the comment `url` originally pointed to
+// and the last index is the comment directly replying to the video (the thread root)
export async function resolveThread (params: ResolveThreadParams): ResolveThreadResult {
const { url, isVideo } = params
@@ -92,13 +97,14 @@ async function resolveCommentFromDB (params: ResolveThreadParams) {
let parentComments = comments.concat([ commentFromDatabase ])
- // Speed up things and resolve directly the thread
+ // The rest of the thread above this comment is already in DB too: append it instead of re-fetching each ancestor from the remote instance
if (commentFromDatabase.InReplyToVideoComment) {
const data = await VideoCommentModel.listThreadParentComments({ comment: commentFromDatabase, order: 'DESC' })
parentComments = parentComments.concat(data)
}
+ // We know the video already, so skip straight to it instead of walking inReplyTo comment by comment
return resolveThread({
url: commentFromDatabase.Video.url,
comments: parentComments,
@@ -109,6 +115,7 @@ async function resolveCommentFromDB (params: ResolveThreadParams) {
// ---------------------------------------------------------------------------
+// Once the video is reached the whole chain is known and gets persisted root-first
async function tryToResolveThreadFromVideo (params: ResolveThreadParams) {
const { url, comments, commentCreated } = params
@@ -128,9 +135,11 @@ async function tryToResolveThreadFromVideo (params: ResolveThreadParams) {
let resultComment: MCommentOwnerVideo
// comments[0] is the comment the thread was resolved for: only its held status is reported back to the caller,
// the other comments in the array are ancestors synthesized to complete the thread
- let resultCommentHeldPendingAutomaticTags = false
+ let resultCommentHeldForAutoTags = false
if (comments.length !== 0) {
+ // Process root-first (comments.length - 1 down to 0): each comment needs its parent's real DB id for
+ // inReplyToCommentId/originCommentId, which only exists once the parent has been saved
const firstReply = comments[comments.length - 1] as MCommentOwnerVideo
firstReply.inReplyToCommentId = null
firstReply.originCommentId = null
@@ -142,15 +151,16 @@ async function tryToResolveThreadFromVideo (params: ResolveThreadParams) {
return undefined
}
- const firstReplyDecision = await assignReviewIfNew(firstReply, video)
+ const { isNew: firstReplyIsNew, holdStatus: firstReplyHoldStatus } = await assignReviewIfNew(firstReply, video)
comments[comments.length - 1] = await firstReply.save()
- if (firstReplyDecision) {
- if (comments.length === 1) resultCommentHeldPendingAutomaticTags = firstReplyDecision.pendingAutomaticTags
+ // New comment: process auto tags
+ if (firstReplyIsNew) {
+ if (comments.length === 1) resultCommentHeldForAutoTags = firstReplyHoldStatus === 'held-for-auto-tags'
createCommentAutomaticTagsJob({
comment: firstReply,
- moderation: firstReplyDecision.pendingAutomaticTags
+ moderation: firstReplyHoldStatus === 'held-for-auto-tags'
? 'release-hold'
: 'none',
// Only the comment the thread was resolved for (comments[0]) must notify the video owner: the other ones
@@ -171,16 +181,17 @@ async function tryToResolveThreadFromVideo (params: ResolveThreadParams) {
return undefined
}
- const decision = await assignReviewIfNew(comment, video)
+ const { isNew, holdStatus } = await assignReviewIfNew(comment, video)
comments[i] = await comment.save()
- if (decision) {
- if (i === 0) resultCommentHeldPendingAutomaticTags = decision.pendingAutomaticTags
+ // New comment: process auto tags
+ if (isNew) {
+ if (i === 0) resultCommentHeldForAutoTags = holdStatus === 'held-for-auto-tags'
createCommentAutomaticTagsJob({
comment,
- moderation: decision.pendingAutomaticTags
+ moderation: holdStatus === 'held-for-auto-tags'
? 'release-hold'
: 'none',
// comments[0] is the comment the thread was resolved for: only it must notify the video owner
@@ -192,31 +203,38 @@ async function tryToResolveThreadFromVideo (params: ResolveThreadParams) {
resultComment = comments[0] as MCommentOwnerVideo
}
- return { video, comment: resultComment, commentCreated, heldPendingAutomaticTags: resultCommentHeldPendingAutomaticTags }
+ return { video, comment: resultComment, commentCreated, heldForAutoTags: resultCommentHeldForAutoTags }
}
-// Assign the held for review status of the comment if it's a new one, and return the decision (undefined if not new)
+// Assign the held for review status of the comment if it's a new one
+// `holdStatus` is only set for a new comment: an existing one keeps the status it already has in database
// Automatic tags are built by a job, so a comment is held as soon as the account has a review policy
// The job then releases it if the tags it ends up with don't match any of them
-async function assignReviewIfNew (comment: MComment, video: MVideoAccountLight): Promise<CommentReviewDecision> {
+async function assignReviewIfNew (comment: MComment, video: MVideoAccountLight): Promise<{
+ isNew: boolean
+ holdStatus?: CommentHoldStatus
+}> {
// Remote comment already exists in database -> we don't need to rebuild automatic tags
- if (comment.id) return undefined
+ if (comment.id) return { isNew: false }
// Third parties rely on origin, so if origin has the comment it's not held for review
- const decision = video.isLocal() || comment.isLocal()
- ? await getCommentReviewDecision({ user: null, video, automaticTagsPending: true })
- : { heldForReview: false, pendingAutomaticTags: false }
+ const holdStatus: CommentHoldStatus = video.isLocal() || comment.isLocal()
+ ? await getCommentHoldStatus({ user: null, video, holdIfAutoTagPolicy: true })
+ : 'not-held'
- comment.heldForReview = decision.heldForReview
+ comment.heldForReview = holdStatus !== 'not-held'
- return decision
+ return { isNew: true, holdStatus }
}
// ---------------------------------------------------------------------------
+// Despite the name, this also resolves the original leaf/target comment on the first call, not just
+// its ancestors: it's the fallback once DB lookup and video resolution have both failed for `url`
async function resolveRemoteParentComment (params: ResolveThreadParams) {
const { url, comments } = params
+ // Guard against unbounded/malicious inReplyTo chains
if (comments.length > ACTIVITY_PUB.MAX_RECURSION_COMMENTS) {
throw new Error('Recursion limit reached when resolving a thread')
}
@@ -230,10 +248,12 @@ async function resolveRemoteParentComment (params: ResolveThreadParams) {
const actorUrl = body.attributedTo
if (!actorUrl && body.type !== 'Tombstone') throw new Error('Miss attributed to in comment')
+ // An actor hosted on another server can't author a comment on this one (identity spoofing guard)
if (actorUrl && checkUrlsSameHost(url, actorUrl) !== true) {
throw new Error(`Actor url ${actorUrl} has not the same host than the comment url ${url}`)
}
+ // The fetched object must claim an id on the same host it was fetched from
if (checkUrlsSameHost(body.id, url) !== true) {
throw new Error(`Comment url ${url} host is different from the AP object id ${body.id}`)
}
diff --git a/server/core/lib/activitypub/videos/get.ts b/server/core/lib/activitypub/videos/get.ts
index f1eacffa1f..39247a90fb 100644
--- a/server/core/lib/activitypub/videos/get.ts
+++ b/server/core/lib/activitypub/videos/get.ts
@@ -2,6 +2,7 @@ import { APObjectId } from '@peertube/peertube-models'
import { retryTransactionWrapper } from '@server/helpers/database-utils.js'
import { createLogger } from '@server/helpers/logger.js'
import { loadVideoByUrl } from '@server/lib/model-loaders/index.js'
+import { AutoBlacklistStatus } from '@server/lib/video-blacklist.js'
import { MVideoAccountLightBlacklistAllFiles, MVideoImmutable, MVideoThumbnails, MVideoWithBlacklist } from '@server/types/models/index.js'
import { getAPId } from '../activity.js'
import { refreshVideoIfNeeded, scheduleVideoRefreshIfNeeded } from './refresh.js'
@@ -12,7 +13,7 @@ const logger = createLogger()
type GetVideoResult<T> = Promise<{
video: T
created: boolean
- autoBlacklisted?: boolean
+ autoBlacklistStatus?: AutoBlacklistStatus
}>
type GetVideoParamAll = {
@@ -82,11 +83,11 @@ export async function getOrCreateAPVideo (
try {
const creator = new APVideoCreator(videoObject)
- const { autoBlacklisted, videoCreated } = await retryTransactionWrapper(() => creator.create())
+ const { autoBlacklistStatus, videoCreated } = await retryTransactionWrapper(() => creator.create())
await syncVideoExternalAttributes(videoCreated, videoObject, syncParam)
- return { video: videoCreated, created: true, autoBlacklisted }
+ return { video: videoCreated, created: true, autoBlacklistStatus }
} catch (err) {
// Maybe a concurrent getOrCreateAPVideo call created this video
if (err.name === 'SequelizeUniqueConstraintError') {
diff --git a/server/core/lib/activitypub/videos/shared/abstract-builder.ts b/server/core/lib/activitypub/videos/shared/abstract-builder.ts
index 63676bf7db..7730255791 100644
--- a/server/core/lib/activitypub/videos/shared/abstract-builder.ts
+++ b/server/core/lib/activitypub/videos/shared/abstract-builder.ts
@@ -235,6 +235,7 @@ export abstract class APVideoAbstractBuilder {
}
// The automatic tags are built from the name and the description, so they only have to be rebuilt when one changed
+ // TODO: add support if remote file changed
protected automaticTagsNeedRebuild (options: {
video: MVideo
oldVideo?: Pick<MVideo, 'name' | 'description'>
diff --git a/server/core/lib/activitypub/videos/shared/creator.ts b/server/core/lib/activitypub/videos/shared/creator.ts
index 647552807f..e9893057f0 100644
--- a/server/core/lib/activitypub/videos/shared/creator.ts
+++ b/server/core/lib/activitypub/videos/shared/creator.ts
@@ -30,7 +30,7 @@ export class APVideoCreator extends APVideoAbstractBuilder {
const videoData = getVideoAttributesFromObject(channel, this.videoObject, this.videoObject.to)
const video = VideoModel.build({ ...videoData, likes: 0, dislikes: 0 }) as MVideoThumbnails
- const { autoBlacklisted, videoCreated } = await sequelizeTypescript.transaction(async t => {
+ const { autoBlacklistStatus, videoCreated } = await sequelizeTypescript.transaction(async t => {
const videoCreated = await video.save({ transaction: t }) as MVideoFull
videoCreated.VideoChannel = channel
@@ -46,9 +46,9 @@ export class APVideoCreator extends APVideoAbstractBuilder {
// We added a video in this channel, set it as updated
await channel.setAsUpdated(t)
- const { blacklisted: autoBlacklisted, pendingAutomaticTags } = await autoBlacklistVideoIfNeeded({
+ const autoBlacklistStatus = await autoBlacklistVideoIfNeeded({
video: videoCreated,
- automaticTagsPending: true,
+ holdIfAutoTagPolicy: true,
user: undefined,
isRemote: true,
isNew: true,
@@ -58,7 +58,7 @@ export class APVideoCreator extends APVideoAbstractBuilder {
createVideoAutomaticTagsJob({
video: videoCreated,
- moderation: pendingAutomaticTags
+ moderation: autoBlacklistStatus === 'held-for-auto-tags'
? 'release-hold'
: 'apply',
transaction: t
@@ -68,12 +68,12 @@ export class APVideoCreator extends APVideoAbstractBuilder {
Hooks.runAction('action:activity-pub.remote-video.created', { video: videoCreated, videoAPObject: this.videoObject })
- return { autoBlacklisted, videoCreated }
+ return { autoBlacklistStatus, videoCreated }
})
await this.updateChapters(videoCreated)
await this.upsertPlayerSettings(videoCreated)
- return { autoBlacklisted, videoCreated }
+ return { autoBlacklistStatus, videoCreated }
}
}
diff --git a/server/core/lib/activitypub/videos/updater.ts b/server/core/lib/activitypub/videos/updater.ts
index c3c844fc05..809ce133ce 100644
--- a/server/core/lib/activitypub/videos/updater.ts
+++ b/server/core/lib/activitypub/videos/updater.ts
@@ -91,12 +91,10 @@ export class APVideoUpdater extends APVideoAbstractBuilder {
await this.updateChapters(videoUpdated)
await this.upsertPlayerSettings(videoUpdated)
- // The video is already published: don't hold it while its automatic tags are rebuilt, or a simple metadata
- // update would block it and re-announce it to the subscribers once released
- // The `build-object-automatic-tags` job applies the auto tag block policies afterwards
await autoBlacklistVideoIfNeeded({
video: videoUpdated,
- automaticTagsPending: false,
+ // Already published: don't hold it while its automatic tags are rebuilt
+ holdIfAutoTagPolicy: false,
user: undefined,
isRemote: true,
isNew: false,
diff --git a/server/core/lib/automatic-tags/automatic-tags.ts b/server/core/lib/automatic-tags/automatic-tags.ts
index 631e5ae9f3..8380bd0deb 100644
--- a/server/core/lib/automatic-tags/automatic-tags.ts
+++ b/server/core/lib/automatic-tags/automatic-tags.ts
@@ -1,5 +1,6 @@
import { AutomaticTagPolicyType, AutomaticTagsModeration, BuildObjectAutomaticTagsPayload } from '@peertube/peertube-models'
-import { afterCommitIfTransaction } from '@server/helpers/database-utils.js'
+import { afterCommitIfTransaction, retryTransactionWrapper } from '@server/helpers/database-utils.js'
+import { sequelizeTypescript } from '@server/initializers/database.js'
import { getServerAccount } from '@server/models/application/application.js'
import { AccountAutomaticTagPolicyModel } from '@server/models/automatic-tag/account-automatic-tag-policy.js'
import { AutomaticTagModel } from '@server/models/automatic-tag/automatic-tag.js'
@@ -65,33 +66,37 @@ export async function setAndSaveVideoAutomaticTags (options: {
if (Object.keys(automaticTagsByAccount).length === 0) return
- const { toCreateItems, toDeleteItems } = await _buildAutomaticTagItems({
- automaticTagsByAccount,
+ await retryTransactionWrapper(() => {
+ return sequelizeTypescript.transaction(async transaction => {
+ const { toCreateItems, toDeleteItems } = await _buildAutomaticTagItems({
+ automaticTagsByAccount,
- existingAutomaticTagsGetter: accountIds => {
- return VideoAutomaticTagModel.listByAccountIdsAndVideoId({ videoId: video.id, accountIds })
- }
- })
+ existingAutomaticTagsGetter: accountIds => {
+ return VideoAutomaticTagModel.listByAccountIdsAndVideoId({ videoId: video.id, accountIds, transaction })
+ }
+ })
- for (const item of toDeleteItems) {
- await item.destroy()
- }
+ for (const item of toDeleteItems) {
+ await item.destroy({ transaction })
+ }
- const videoAutomaticTags: MVideoAutomaticTagWithTag[] = []
+ const videoAutomaticTags: MVideoAutomaticTagWithTag[] = []
- for (const tag of toCreateItems) {
- const automaticTagInstance = await AutomaticTagModel.findOrCreateAutomaticTag({ tag: tag.name })
+ for (const tag of toCreateItems) {
+ const automaticTagInstance = await AutomaticTagModel.findOrCreateAutomaticTag({ tag: tag.name, transaction })
- const [ videoAutomaticTag ] = await VideoAutomaticTagModel.upsert({
- accountId: tag.accountId,
- automaticTagId: automaticTagInstance.id,
- videoId: video.id
- })
+ const [ videoAutomaticTag ] = await VideoAutomaticTagModel.upsert({
+ accountId: tag.accountId,
+ automaticTagId: automaticTagInstance.id,
+ videoId: video.id
+ }, { transaction })
- videoAutomaticTag.AutomaticTag = automaticTagInstance
+ videoAutomaticTag.AutomaticTag = automaticTagInstance
- videoAutomaticTags.push(videoAutomaticTag)
- }
+ videoAutomaticTags.push(videoAutomaticTag)
+ }
+ })
+ })
}
async function _buildAutomaticTagItems<T extends MCommentAutomaticTagWithTag | MVideoAutomaticTagWithTag> (options: {
diff --git a/server/core/lib/job-queue/handlers/build-object-automatic-tags.ts b/server/core/lib/job-queue/handlers/build-object-automatic-tags.ts
index 01090141bc..7b466e7f8b 100644
--- a/server/core/lib/job-queue/handlers/build-object-automatic-tags.ts
+++ b/server/core/lib/job-queue/handlers/build-object-automatic-tags.ts
@@ -5,7 +5,7 @@ import { sequelizeTypescript } from '@server/initializers/database.js'
import { buildAndSaveCommentAutomaticTags, buildAndSaveVideoAutomaticTags } from '@server/lib/automatic-tags/automatic-tags.js'
import { Notifier } from '@server/lib/notifier/index.js'
import { autoBlacklistVideoByAutoTagPolicyIfNeeded, resolvePendingAutoTagBlacklist } from '@server/lib/video-blacklist.js'
-import { approveComment, getCommentReviewDecision } from '@server/lib/video-comment.js'
+import { approveComment, getCommentHoldStatus } from '@server/lib/video-comment.js'
import { UserModel } from '@server/models/user/user.js'
import { VideoCommentModel } from '@server/models/video/video-comment.js'
import { VideoModel } from '@server/models/video/video.js'
@@ -80,17 +80,17 @@ async function buildOfComment (payload: BuildObjectAutomaticTagsPayload) {
if (commentReloaded.heldForReview) {
const video = commentReloaded.Video
- const { heldForReview } = await getCommentReviewDecision({
+ const holdStatus = await getCommentHoldStatus({
user: commentReloaded.accountId
? await UserModel.loadByAccountId(commentReloaded.accountId, t)
: null,
video,
- automaticTagsPending: false,
+ holdIfAutoTagPolicy: false,
ownerAutomaticTags: automaticTagsByAccount[video.VideoChannel.accountId],
transaction: t
})
- if (!heldForReview) {
+ if (holdStatus === 'not-held') {
// The comment was only held while waiting for its automatic tags: its author must not be notified of an approval
await approveComment(commentReloaded, { notify: false, transaction: t })
commentReloaded.heldForReview = false
@@ -100,8 +100,7 @@ async function buildOfComment (payload: BuildObjectAutomaticTagsPayload) {
}
// The video owner has not been notified at creation, because the held status of the comment was not final yet
- // Notify it even if the comment is still held for review, so it can approve it (same as a comment held by
- // a `commentsPolicy: REQUIRES_APPROVAL` video, which is notified synchronously at creation)
+ // Notify it even if the comment is still held for review, so it can approve it
// `notify` is false for a comment that only exists to complete a federated thread (see `resolveThread`)
if (payload.notify !== false) {
afterCommitIfTransaction(t, () => Notifier.Instance.notifyOnNewComment(commentReloaded))
diff --git a/server/core/lib/job-queue/handlers/video-import.ts b/server/core/lib/job-queue/handlers/video-import.ts
index 28051d5f0f..0c4442841f 100644
--- a/server/core/lib/job-queue/handlers/video-import.ts
+++ b/server/core/lib/job-queue/handlers/video-import.ts
@@ -309,13 +309,10 @@ async function afterImportSuccess (options: {
const { video, videoFile, videoImport, user, generateTranscription } = options
// The file now exists, so plugin auto taggers can analyze it
- // They can be slow, so let the `build-object-automatic-tags` job run them instead of blocking this queue
- // Hold the video until it has confirmed or released the hold: the video has not been announced yet
- // Note the video may already be blocked by the instance policy, applied by `insertFromImportIntoDB`
- const { pendingAutomaticTags } = await autoBlacklistVideoIfNeeded({
+ const autoBlacklistStatus = await autoBlacklistVideoIfNeeded({
video,
user,
- automaticTagsPending: true,
+ holdIfAutoTagPolicy: true,
isRemote: false,
isNew: true,
isNewFile: true,
@@ -325,7 +322,7 @@ async function afterImportSuccess (options: {
createVideoAutomaticTagsJob({
video,
- moderation: pendingAutomaticTags
+ moderation: autoBlacklistStatus === 'held-for-auto-tags'
? 'release-hold'
: 'apply'
})
@@ -334,14 +331,12 @@ async function afterImportSuccess (options: {
// A video held for its automatic tags is announced by the job instead, which also notifies the moderators if it
// ends up confirming the block
- if (!pendingAutomaticTags) {
- if (video.isBlacklisted()) {
- const videoBlacklist = Object.assign(video.VideoBlacklist, { Video: video })
+ if (autoBlacklistStatus === 'auto-blacklisted') {
+ const videoBlacklist = Object.assign(video.VideoBlacklist, { Video: video })
- Notifier.Instance.notifyOnVideoAutoBlacklist(videoBlacklist)
- } else {
- Notifier.Instance.notifyOnNewVideoOrLiveIfNeeded(video)
- }
+ Notifier.Instance.notifyOnVideoAutoBlacklist(videoBlacklist)
+ } else if (autoBlacklistStatus === 'not-auto-blacklisted') {
+ Notifier.Instance.notifyOnNewVideoOrLiveIfNeeded(video)
}
// Generate the storyboard in the job queue, and don't forget to federate an update after
diff --git a/server/core/lib/local-video-creator.ts b/server/core/lib/local-video-creator.ts
index b8bf9291b5..5cf58529d1 100644
--- a/server/core/lib/local-video-creator.ts
+++ b/server/core/lib/local-video-creator.ts
@@ -194,24 +194,23 @@ export class LocalVideoCreator {
}
}
- // A video with a file has its automatic tags built by `addVideoJobsAfterCreation`, before it is announced.
- // A video without one (live) is federated right away, so hold it until the tagging job has run
- const holdForAutomaticTags = !this.videoFile
-
- const { pendingAutomaticTags } = await autoBlacklistVideoIfNeeded({
+ const autoBlacklistStatus = await autoBlacklistVideoIfNeeded({
video: this.video,
user: this.options.user,
- automaticTagsPending: holdForAutomaticTags,
+ // A video with a file has its automatic tags built by `addVideoJobsAfterCreation`, before it is announced.
+ // A video without one (live) is federated right away, so hold it until the tagging job has run
+ holdIfAutoTagPolicy: !this.videoFile,
isRemote: false,
isNew: true,
isNewFile: true,
transaction
})
- if (holdForAutomaticTags) {
+ // Keep it sync with `holdIfAutoTagPolicy` above
+ if (!this.videoFile) {
createVideoAutomaticTagsJob({
video: this.video,
- moderation: pendingAutomaticTags
+ moderation: autoBlacklistStatus === 'held-for-auto-tags'
? 'release-hold'
: 'apply',
transaction
diff --git a/server/core/lib/local-video-updater.ts b/server/core/lib/local-video-updater.ts
index 9a04c83de9..3732211ac6 100644
--- a/server/core/lib/local-video-updater.ts
+++ b/server/core/lib/local-video-updater.ts
@@ -209,7 +209,7 @@ export class LocalVideoUpdater {
isRemote: false,
isNew: false,
isNewFile: false,
- automaticTagsPending: false,
+ holdIfAutoTagPolicy: false,
transaction: t
})
diff --git a/server/core/lib/video-blacklist.ts b/server/core/lib/video-blacklist.ts
index c3e9a101a0..e6ce729420 100644
--- a/server/core/lib/video-blacklist.ts
+++ b/server/core/lib/video-blacklist.ts
@@ -33,6 +33,8 @@ import { Hooks } from './plugins/hooks.js'
const logger = createLogger('blacklist')
+export type AutoBlacklistStatus = 'auto-blacklisted' | 'held-for-auto-tags' | 'not-auto-blacklisted'
+
export async function autoBlacklistVideoIfNeeded (parameters: {
video: MVideoWithBlacklistLight
isRemote: boolean
@@ -40,17 +42,17 @@ export async function autoBlacklistVideoIfNeeded (parameters: {
isNewFile: boolean
// The automatic tags of the video have not been built yet: we don't know which tags it will have
- automaticTagsPending: boolean
+ holdIfAutoTagPolicy: boolean
automaticTagsByAccount?: Record<number, string[]>
user?: MUser
notify?: boolean
transaction?: Transaction
-}) {
- const { video, user, isRemote, isNew, isNewFile, automaticTagsPending, automaticTagsByAccount, notify = true, transaction } = parameters
+}): Promise<AutoBlacklistStatus> {
+ const { video, user, isRemote, isNew, isNewFile, holdIfAutoTagPolicy, automaticTagsByAccount, notify = true, transaction } = parameters
// Already blacklisted
- if (video.VideoBlacklist) return { blacklisted: true, pendingAutomaticTags: false }
+ if (video.VideoBlacklist) return 'auto-blacklisted'
const doAutoBlacklistByInstancePolicy = await Hooks.wrapFun(
_autoBlacklistByInstancePolicyNeeded,
@@ -58,27 +60,42 @@ export async function autoBlacklistVideoIfNeeded (parameters: {
'filter:video.auto-blacklist.result'
)
+ // Global instance policy
if (doAutoBlacklistByInstancePolicy) {
await _autoBlacklist({ video, notify, user, transaction, type: VideoBlacklistType.AUTO_BY_INSTANCE_POLICY })
- return { blacklisted: true, pendingAutomaticTags: false }
+ return 'auto-blacklisted'
+ }
+
+ // Special hold mode if the instance has auto tag policies that could block the video,
+ // and the automatic tags of the video have not been built: block the video if the instance could want to review it
+ // Let the `build-object-automatic-tags` job release it if the tags it ends up with don't match any of its auto block policies
+ if (holdIfAutoTagPolicy && await _shouldHoldForAutoTagPolicy({ video, user, transaction })) {
+ await _autoBlacklist({
+ video,
+ notify: false, // Don't notify, build auto tag job will notify if the video is really blocked
+ user,
+ transaction,
+ type: VideoBlacklistType.AUTO_BY_AUTO_TAG_POLICY
+ })
+
+ return 'held-for-auto-tags'
}
- if (await _autoBlacklistByAutoTagPolicyNeeded({ video, user, automaticTagsPending, automaticTagsByAccount, transaction })) {
- // We don't know yet if the video really matches a policy
- // So don't notify the moderators before the `build-object-automatic-tags` job has confirmed it
+ // Not in hold mode: auto blacklist the video if its automatic tags match an auto block policy
+ if (await _shouldAutoBlacklistByAutoTagPolicy({ video, user, automaticTagsByAccount, transaction })) {
await _autoBlacklist({
video,
- notify: notify && !automaticTagsPending,
+ notify,
user,
transaction,
type: VideoBlacklistType.AUTO_BY_AUTO_TAG_POLICY
})
- return { blacklisted: true, pendingAutomaticTags: automaticTagsPending }
+ return 'auto-blacklisted'
}
- return { blacklisted: false, pendingAutomaticTags: false }
+ return 'not-auto-blacklisted'
}
// Called by the `build-object-automatic-tags` job for a video that was not put on hold
@@ -93,9 +110,7 @@ export async function autoBlacklistVideoByAutoTagPolicyIfNeeded (options: {
const { video, automaticTagsByAccount, user, notify = true, transaction } = options
if (video.VideoBlacklist) return false
- if (!await _autoBlacklistByAutoTagPolicyNeeded({ video, user, automaticTagsPending: false, automaticTagsByAccount, transaction })) {
- return false
- }
+ if (!await _shouldAutoBlacklistByAutoTagPolicy({ video, user, automaticTagsByAccount, transaction })) return false
await _autoBlacklist({
video,
@@ -122,7 +137,7 @@ export async function resolvePendingAutoTagBlacklist (options: {
const videoBlacklist = await VideoBlacklistModel.loadByVideoId(video.id, transaction)
if (!videoBlacklist || videoBlacklist.type !== VideoBlacklistType.AUTO_BY_AUTO_TAG_POLICY) return
- if (await _autoBlacklistByAutoTagPolicyNeeded({ video, automaticTagsPending: false, automaticTagsByAccount, transaction })) {
+ if (await _shouldAutoBlacklistByAutoTagPolicy({ video, automaticTagsByAccount, transaction })) {
const videoBlacklistWithVideo = videoBlacklist as MVideoBlacklistVideo
videoBlacklistWithVideo.Video = video
@@ -188,31 +203,38 @@ function _autoBlacklistByInstancePolicyNeeded (parameters: {
return true
}
-async function _autoBlacklistByAutoTagPolicyNeeded (options: {
+async function _shouldHoldForAutoTagPolicy (options: {
+ video: MVideoWithBlacklistLight
+ transaction?: Transaction
+ user?: MUser
+}) {
+ const { user, video, transaction } = options
+
+ if (video.isLocal() && user?.hasRight(UserRight.MANAGE_VIDEO_BLACKLIST)) return false
+
+ const accountId = (await getServerAccount()).id
+
+ return AccountAutomaticTagPolicyModel.hasPolicy({
+ accountId,
+ policy: AutomaticTagPolicy.AUTO_BLACKLIST_VIDEO,
+ transaction
+ })
+}
+
+async function _shouldAutoBlacklistByAutoTagPolicy (options: {
video: MVideoWithBlacklistLight
- automaticTagsPending: boolean
automaticTagsByAccount?: Record<number, string[]>
transaction?: Transaction
user?: MUser
}) {
- const { user, video, transaction, automaticTagsPending, automaticTagsByAccount } = options
+ const { user, video, transaction, automaticTagsByAccount } = options
- if (!automaticTagsPending && !automaticTagsByAccount) return false
+ if (!automaticTagsByAccount) return false
if (video.isLocal() && user?.hasRight(UserRight.MANAGE_VIDEO_BLACKLIST)) return false
const accountId = (await getServerAccount()).id
- // Block the video if the instance could want to review it
- // Let the `build-object-automatic-tags` job release it if the tags it ends up with don't match any of its auto block policies
- if (automaticTagsPending) {
- return AccountAutomaticTagPolicyModel.hasPolicy({
- accountId,
- policy: AutomaticTagPolicy.AUTO_BLACKLIST_VIDEO,
- transaction
- })
- }
-
const tags = automaticTagsByAccount?.[accountId]
if (!tags || tags.length === 0) return false
diff --git a/server/core/lib/video-comment.ts b/server/core/lib/video-comment.ts
index 7abc37fedf..bc4665dcfe 100644
--- a/server/core/lib/video-comment.ts
+++ b/server/core/lib/video-comment.ts
@@ -94,10 +94,10 @@ export async function createLocalVideoComment (options: {
return sequelizeTypescript.transaction(async transaction => {
const account = await AccountModel.load(user.Account.id, transaction)
- const { heldForReview, pendingAutomaticTags } = await getCommentReviewDecision({
+ const holdStatus = await getCommentHoldStatus({
user,
video,
- automaticTagsPending: true,
+ holdIfAutoTagPolicy: true,
transaction
})
@@ -107,7 +107,7 @@ export async function createLocalVideoComment (options: {
inReplyToCommentId,
videoId: video.id,
accountId: account.id,
- heldForReview,
+ heldForReview: holdStatus !== 'not-held',
url: new Date().toISOString()
}, { transaction, validate: false })
@@ -117,7 +117,7 @@ export async function createLocalVideoComment (options: {
createCommentAutomaticTagsJob({
comment: savedComment,
- moderation: pendingAutomaticTags
+ moderation: holdStatus === 'held-for-auto-tags'
? 'release-hold'
: 'none',
notify: true,
@@ -130,7 +130,7 @@ export async function createLocalVideoComment (options: {
await sendCreateVideoCommentIfNeeded(savedComment, transaction)
- return { comment: savedComment, pendingAutomaticTags }
+ return { comment: savedComment, holdStatus }
})
}
@@ -186,59 +186,59 @@ export function buildFormattedCommentTree (options: {
}
}
-export type CommentReviewDecision = {
- heldForReview: boolean
+export type CommentHoldStatus = 'held-for-review' | 'held-for-auto-tags' | 'not-held'
- // The comment is only held because its automatic tags are not built yet
- // The `build-object-automatic-tags` job confirms or releases that hold, and is in charge of notifying the video owner of the new comment
- pendingAutomaticTags: boolean
-}
-
-export async function getCommentReviewDecision (options: {
+export async function getCommentHoldStatus (options: {
user: MUserAccountId
video: MVideoAccountIdUrl
// The automatic tags of the comment have not been built yet: we don't know which tags it will have
- automaticTagsPending: boolean
+ holdIfAutoTagPolicy: boolean
ownerAutomaticTags?: string[]
transaction?: Transaction
-}): Promise<CommentReviewDecision> {
- const { user, video, transaction, automaticTagsPending, ownerAutomaticTags } = options
-
- const notHeld = { heldForReview: false, pendingAutomaticTags: false }
+}): Promise<CommentHoldStatus> {
+ const { user, video, transaction, holdIfAutoTagPolicy, ownerAutomaticTags } = options
+ // User bypass check
if (video.isLocal() && user) {
- if (user.hasRight(UserRight.MANAGE_ANY_VIDEO_COMMENT)) return notHeld
- if (user.Account.id === video.VideoChannel.accountId) return notHeld
+ if (user.hasRight(UserRight.MANAGE_ANY_VIDEO_COMMENT)) return 'not-held'
+ if (user.Account.id === video.VideoChannel.accountId) return 'not-held'
}
+ // Global owner policy
if (video.commentsPolicy === VideoCommentPolicy.REQUIRES_APPROVAL) {
- return { heldForReview: true, pendingAutomaticTags: false }
+ return 'held-for-review'
}
- if (video.isLocal() !== true) return notHeld
+ // Don't check auto tags policy on remote videos
+ if (video.isLocal() !== true) return 'not-held'
// Hold the comment if the account could want to review it
// Let the `build-object-automatic-tags` job release it if the tags it ends up with don't match any of its review policies
- if (automaticTagsPending) {
+ if (holdIfAutoTagPolicy) {
const hasPolicy = await AccountAutomaticTagPolicyModel.hasPolicy({
accountId: video.VideoChannel.accountId,
policy: AutomaticTagPolicy.REVIEW_COMMENT,
transaction
})
- return { heldForReview: hasPolicy, pendingAutomaticTags: hasPolicy }
+ return hasPolicy
+ ? 'held-for-auto-tags'
+ : 'not-held'
}
- if (!ownerAutomaticTags || ownerAutomaticTags.length === 0) return notHeld
+ if (!ownerAutomaticTags || ownerAutomaticTags.length === 0) return 'not-held'
- const heldForReview = await AccountAutomaticTagPolicyModel.hasPolicyOnTags({
+ // Check on specific auto tags provided
+ const hasPolicy = await AccountAutomaticTagPolicyModel.hasPolicyOnTags({
accountId: video.VideoChannel.accountId,
policy: AutomaticTagPolicy.REVIEW_COMMENT,
tags: ownerAutomaticTags,
transaction
})
- return { heldForReview, pendingAutomaticTags: false }
+ return hasPolicy
+ ? 'held-for-review'
+ : 'not-held'
}
diff --git a/server/core/lib/video-pre-import.ts b/server/core/lib/video-pre-import.ts
index 111e1d4fa7..181307d63a 100644
--- a/server/core/lib/video-pre-import.ts
+++ b/server/core/lib/video-pre-import.ts
@@ -79,7 +79,7 @@ export async function insertFromImportIntoDB (parameters: {
// The video stays in the `TO_IMPORT` state until then, so it is neither federated nor notified
await autoBlacklistVideoIfNeeded({
video: videoCreated,
- automaticTagsPending: false,
+ holdIfAutoTagPolicy: false,
user,
notify: false,
isRemote: false,