Skip to content

ActivityPub-federated video streaming platform using P2P directly in your web browser

Refactor video updater

Chocobozzz0578b8bfa02a
diff --git a/server/core/controllers/api/videos/update.ts b/server/core/controllers/api/videos/update.ts
index 6081a70716..721e98b98b 100644
--- a/server/core/controllers/api/videos/update.ts
+++ b/server/core/controllers/api/videos/update.ts
@@ -6,12 +6,10 @@ import { openapiOperationDoc } from '@server/middlewares/doc.js'
import { MVideoThumbnails } from '@server/types/models/index.js'
import express from 'express'
import { createReqFiles } from '../../../helpers/express-utils.js'
-import { loggerTagsFactory } from '../../../helpers/logger.js'
import { MIMETYPES } from '../../../initializers/constants.js'
import { Hooks } from '../../../lib/plugins/hooks.js'
import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate, videosUpdateValidator } from '../../../middlewares/index.js'
-const lTags = loggerTagsFactory('api', 'video')
const updateRouter = express.Router()
const reqVideoFileUpdate = createReqFiles([ 'thumbnailfile', 'previewfile' ], MIMETYPES.IMAGE.MIMETYPE_EXT)
@@ -40,9 +38,9 @@ async function updateVideo (req: express.Request, res: express.Response) {
const thumbnails = await buildVideoThumbnailsFromReq(videoFromReq, req)
- const localVideoUpdater = new LocalVideoUpdater({ user, lTags, video: videoFromReq })
+ const localVideoUpdater = new LocalVideoUpdater({ user, tags: [ 'api', 'video' ], video: videoFromReq })
- const video = await localVideoUpdater.update({ ...body, thumbnails })
+ const video = await localVideoUpdater.update({ ...body, thumbnails, channel: res.locals.videoChannel })
Hooks.runAction('action:api.video.updated', { video, body: req.body, req, res })
diff --git a/server/core/lib/local-video-updater.ts b/server/core/lib/local-video-updater.ts
new file mode 100644
index 0000000000..de5689bf72
--- /dev/null
+++ b/server/core/lib/local-video-updater.ts
@@ -0,0 +1,401 @@
+import { exists, forceNumber } from '@peertube/peertube-core-utils'
+import {
+ ManageVideoTorrentPayload,
+ NSFWFlag,
+ VideoChannelActivityAction,
+ VideoPrivacy,
+ VideoPrivacyType,
+ VideoUpdate
+} from '@peertube/peertube-models'
+import { auditLoggerFactory, getAuditIdFromUser, VideoAuditView } from '@server/helpers/audit-logger.js'
+import { logger, loggerTagsFactory, LoggerTagsFn } from '@server/helpers/logger.js'
+import { sequelizeTypescript } from '@server/initializers/database.js'
+import { getServerAccount } from '@server/models/application/application.js'
+import { ScheduleVideoUpdateModel } from '@server/models/video/schedule-video-update.js'
+import { VideoChannelActivityModel } from '@server/models/video/video-channel-activity.js'
+import { VideoPasswordModel } from '@server/models/video/video-password.js'
+import { VideoModel } from '@server/models/video/video.js'
+import { MChannelBannerAccountDefault, MThumbnail, MUserAccountUrl, MVideo, MVideoFull, MVideoUUID } from '@server/types/models/index.js'
+import { FilteredModelAttributes } from '@server/types/sequelize.js'
+import { Transaction } from 'sequelize'
+import { sendDeleteVideo } from './activitypub/send/send-delete.js'
+import { changeVideoChannelShare } from './activitypub/share.js'
+import { isPrivacyForFederation } from './activitypub/videos/index.js'
+import { AutomaticTagger } from './automatic-tags/automatic-tagger.js'
+import { setAndSaveVideoAutomaticTags } from './automatic-tags/automatic-tags.js'
+import { CreateJobTypeAndPayload, JobQueue } from './job-queue/job-queue.js'
+import { autoBlacklistVideoIfNeeded } from './video-blacklist.js'
+import { replaceChaptersFromDescriptionIfNeeded } from './video-chapters.js'
+import { VideoPathManager } from './video-path-manager.js'
+import { isNewVideoForSubscription, moveFilesIfPrivacyChanged } from './video-privacy.js'
+import { setVideoTags } from './video.js'
+
+const auditLogger = auditLoggerFactory('videos')
+
+export class LocalVideoUpdater {
+ private readonly lTags: LoggerTagsFn
+ private readonly oldVideo: MVideoFull
+ private readonly user: MUserAccountUrl | null
+
+ private readonly oldPrivacy: VideoPrivacyType
+ private readonly hadPrivacyForFederation: boolean
+
+ private readonly oldVideoAuditView: VideoAuditView
+
+ constructor (options: {
+ tags: (string | number)[]
+
+ video: MVideoFull
+ user: MUserAccountUrl | null
+ }) {
+ this.user = options.user
+ this.oldVideo = options.video
+ this.lTags = loggerTagsFactory(...options.tags, options.video.uuid)
+
+ this.hadPrivacyForFederation = isPrivacyForFederation(this.oldVideo.privacy)
+ this.oldPrivacy = this.oldVideo.privacy
+
+ this.oldVideoAuditView = new VideoAuditView(this.oldVideo.toFormattedDetailsJSON())
+ }
+
+ async update (options: Omit<VideoUpdate, 'thumbnailfile' | 'previewfile'> & {
+ thumbnails?: MThumbnail[]
+ channel?: MChannelBannerAccountDefault
+ }) {
+ const videoFileLockReleaser = await VideoPathManager.Instance.lockFiles(this.oldVideo.uuid)
+
+ try {
+ const { video, newVideoForSubscription } = await sequelizeTypescript.transaction(async t => {
+ // Refresh video to prevent concurrent updates
+ const video = await VideoModel.loadFull(this.oldVideo.id, t)
+
+ const oldName = video.name
+ const oldDescription = video.description
+ const oldVideoChannel = video.VideoChannel
+
+ const keysToUpdate: (keyof VideoUpdate & FilteredModelAttributes<VideoModel>)[] = [
+ 'name',
+ 'category',
+ 'licence',
+ 'language',
+ 'nsfw',
+ 'nsfwFlags',
+ 'nsfwSummary',
+ 'waitTranscoding',
+ 'support',
+ 'description',
+ 'downloadEnabled',
+ 'commentsPolicy'
+ ]
+
+ for (const key of keysToUpdate) {
+ if (options[key] !== undefined) video.set(key, options[key])
+ }
+
+ if (video.nsfw !== true) {
+ video.nsfwFlags = NSFWFlag.NONE
+ video.nsfwSummary = null
+ }
+
+ if (options.originallyPublishedAt !== undefined) {
+ video.originallyPublishedAt = options.originallyPublishedAt
+ ? new Date(options.originallyPublishedAt)
+ : null
+ }
+
+ // Privacy update?
+ let newVideoForSubscription = false
+
+ if (options.privacy !== undefined) {
+ newVideoForSubscription = isNewVideoForSubscription({
+ currentPrivacy: video.privacy,
+ newPrivacy: options.privacy,
+ firstPublishedAt: video.firstPublishedAt
+ })
+
+ await this.updateVideoPrivacy({
+ video,
+ privacy: options.privacy,
+ passwords: options.videoPasswords,
+ hadPrivacyForFederation: this.hadPrivacyForFederation,
+ transaction: t
+ })
+ }
+
+ // Force updatedAt attribute change
+ if (!video.changed()) {
+ await video.setAsRefreshed(t)
+ }
+
+ await video.save({ transaction: t })
+
+ if (options.thumbnails && options.thumbnails.length !== 0) {
+ await video.replaceAndSaveThumbnails(options.thumbnails, t)
+ }
+
+ if (options.tags !== undefined) {
+ await setVideoTags({ video, tags: options.tags, transaction: t })
+ }
+
+ // Channel update
+ if (options.channel && video.channelId !== options.channel.id) {
+ const oldChannel = video.VideoChannel
+ const newChannel = options.channel
+
+ if (this.user) {
+ await VideoChannelActivityModel.addVideoActivity({
+ action: VideoChannelActivityAction.REMOVE_CHANNEL_OWNERSHIP,
+ user: this.user,
+ channel: oldChannel,
+ video,
+ transaction: t
+ })
+
+ await VideoChannelActivityModel.addVideoActivity({
+ action: VideoChannelActivityAction.CREATE_CHANNEL_OWNERSHIP,
+ user: this.user,
+ channel: newChannel,
+ video,
+ transaction: t
+ })
+ }
+
+ await video.$set('VideoChannel', newChannel, { transaction: t })
+ video.VideoChannel = newChannel
+
+ if (this.hadPrivacyForFederation === true) {
+ await changeVideoChannelShare(video, oldVideoChannel, t)
+ }
+ } else if (this.user) {
+ await VideoChannelActivityModel.addVideoActivity({
+ action: VideoChannelActivityAction.UPDATE,
+ user: this.user,
+ channel: video.VideoChannel,
+ video,
+ transaction: t
+ })
+ }
+
+ // Schedule an update?
+ await this.updateScheduleIfNeeded({
+ video,
+ scheduleUpdate: options.scheduleUpdate,
+ transaction: t
+ })
+
+ if (oldDescription !== video.description) {
+ await replaceChaptersFromDescriptionIfNeeded({
+ video,
+ newDescription: video.description,
+ transaction: t,
+ oldDescription
+ })
+ }
+
+ let automaticTagsByAccount: Record<number, string[]>
+ if (oldName !== video.name || oldDescription !== video.description) {
+ automaticTagsByAccount = await new AutomaticTagger().buildVideoAutomaticTags({
+ serverAccount: await getServerAccount(),
+ video,
+ transaction: t
+ })
+ await setAndSaveVideoAutomaticTags({ video, automaticTagsByAccount, transaction: t })
+ }
+
+ await autoBlacklistVideoIfNeeded({
+ video,
+ user: this.user,
+ isRemote: false,
+ isNew: false,
+ isNewFile: false,
+ automaticTagsByAccount,
+ transaction: t
+ })
+
+ if (this.user) {
+ auditLogger.update(
+ getAuditIdFromUser(this.user),
+ new VideoAuditView(video.toFormattedDetailsJSON()),
+ this.oldVideoAuditView
+ )
+ }
+
+ logger.info('Video with name %s and uuid %s updated.', video.name, video.uuid, this.lTags())
+
+ return { video, newVideoForSubscription }
+ })
+
+ const nameChanged = exists(options.name)
+
+ const filePathChanged = exists(this.oldPrivacy)
+ ? await moveFilesIfPrivacyChanged(video, this.oldPrivacy)
+ : false
+
+ await this.rebuildHLSInfoHashesIfNeeded({ video, filePathChanged })
+
+ const jobs: CreateJobTypeAndPayload[] = []
+
+ jobs.push(...this.buildUpdateTorrentJobIfNeeded({ video, nameChanged, filePathChanged }))
+ jobs.push(...this.buildFederationJob({ video }))
+ jobs.push(...this.buildNotifyJobIfNeeded({ video, newVideoForSubscription }))
+
+ await JobQueue.Instance.createSequentialJobFlow(...jobs)
+
+ return video
+ } finally {
+ videoFileLockReleaser()
+ }
+ }
+
+ // Return a boolean indicating if the video is considered as "new" for remote instances in the federation
+ private async updateVideoPrivacy (options: {
+ video: MVideoFull
+ privacy: VideoUpdate['privacy']
+ passwords: VideoUpdate['videoPasswords']
+ hadPrivacyForFederation: boolean
+ transaction: Transaction
+ }) {
+ const { video, privacy, passwords, hadPrivacyForFederation, transaction } = options
+
+ const newPrivacy = forceNumber(privacy) as VideoPrivacyType
+ this.setVideoPrivacy(video, newPrivacy)
+
+ // Delete passwords if video is not anymore password protected
+ if (video.privacy === VideoPrivacy.PASSWORD_PROTECTED && newPrivacy !== VideoPrivacy.PASSWORD_PROTECTED) {
+ await VideoPasswordModel.deleteAllPasswords(video.id, transaction)
+ }
+
+ if (newPrivacy === VideoPrivacy.PASSWORD_PROTECTED && exists(passwords)) {
+ await VideoPasswordModel.deleteAllPasswords(video.id, transaction)
+ await VideoPasswordModel.addPasswords(passwords, video.id, transaction)
+ }
+
+ // Unfederate the video if the new privacy is not compatible with federation
+ if (hadPrivacyForFederation && !isPrivacyForFederation(video.privacy)) {
+ await sendDeleteVideo({ video, deleteForPrivacyChange: true, transaction })
+ }
+ }
+
+ private async updateScheduleIfNeeded (options: {
+ video: MVideoFull
+ scheduleUpdate: VideoUpdate['scheduleUpdate']
+ transaction: Transaction
+ }) {
+ const { video, scheduleUpdate, transaction } = options
+
+ if (scheduleUpdate) {
+ const updateAt = new Date(scheduleUpdate.updateAt)
+
+ video.publishedAt = updateAt
+ await video.save({ transaction })
+
+ await ScheduleVideoUpdateModel.upsert({
+ videoId: video.id,
+ updateAt,
+ privacy: scheduleUpdate.privacy || null
+ }, { transaction })
+
+ return
+ }
+
+ if (scheduleUpdate === null) {
+ const deleted = await ScheduleVideoUpdateModel.deleteByVideoId(video.id, transaction)
+
+ if (deleted) {
+ video.publishedAt = new Date()
+ await video.save({ transaction })
+ }
+ }
+ }
+
+ private setVideoPrivacy (video: MVideo, newPrivacy: VideoPrivacyType) {
+ if (video.privacy === VideoPrivacy.PRIVATE && newPrivacy !== VideoPrivacy.PRIVATE) {
+ video.publishedAt = new Date()
+
+ if (!video.firstPublishedAt) {
+ video.firstPublishedAt = video.publishedAt
+ }
+ }
+
+ video.privacy = newPrivacy
+ }
+
+ private async rebuildHLSInfoHashesIfNeeded (options: {
+ video: MVideoFull
+ filePathChanged: boolean
+ }) {
+ const { video, filePathChanged } = options
+
+ const hls = video.getHLSPlaylist()
+
+ if (filePathChanged && hls) {
+ logger.debug('Updating HLS playlist file paths after privacy change', this.lTags())
+
+ hls.assignP2PMediaLoaderInfoHashes(video, hls.VideoFiles)
+ await hls.save()
+ }
+ }
+
+ private buildUpdateTorrentJobIfNeeded (options: {
+ video: MVideoFull
+ nameChanged: boolean
+ filePathChanged: boolean
+ }) {
+ const { video, nameChanged, filePathChanged } = options
+
+ const jobs: CreateJobTypeAndPayload[] = []
+
+ if (!video.isLive && (nameChanged || filePathChanged)) {
+ logger.debug('Updating video torrent metadata after name or file path change', this.lTags())
+
+ for (const file of (video.VideoFiles || [])) {
+ const payload: ManageVideoTorrentPayload = { action: 'update-metadata', videoId: video.id, videoFileId: file.id }
+
+ jobs.push({ type: 'manage-video-torrent', payload })
+ }
+
+ const hls = video.getHLSPlaylist()
+
+ for (const file of (hls?.VideoFiles || [])) {
+ const payload: ManageVideoTorrentPayload = { action: 'update-metadata', streamingPlaylistId: hls.id, videoFileId: file.id }
+
+ jobs.push({ type: 'manage-video-torrent', payload })
+ }
+ }
+
+ return jobs
+ }
+
+ private buildFederationJob (options: {
+ video: MVideoUUID
+ }): CreateJobTypeAndPayload[] {
+ const { video } = options
+
+ return [
+ {
+ type: 'federate-video',
+ payload: { videoUUID: video.uuid }
+ }
+ ]
+ }
+
+ private buildNotifyJobIfNeeded (options: {
+ video: MVideoUUID
+ newVideoForSubscription: boolean
+ }): CreateJobTypeAndPayload[] {
+ const { video, newVideoForSubscription } = options
+ if (!newVideoForSubscription) return []
+
+ logger.debug('Video is considered new for subscriptions: create the notification job', this.lTags())
+
+ return [
+ {
+ type: 'notify',
+ payload: {
+ action: 'new-video',
+ videoUUID: video.uuid
+ }
+ }
+ ]
+ }
+}
diff --git a/server/core/lib/schedulers/update-videos-scheduler.ts b/server/core/lib/schedulers/update-videos-scheduler.ts
index fdb15d9bea..e1e061e509 100644
--- a/server/core/lib/schedulers/update-videos-scheduler.ts
+++ b/server/core/lib/schedulers/update-videos-scheduler.ts
@@ -3,12 +3,9 @@ import { VideoModel } from '@server/models/video/video.js'
import { MScheduleVideoUpdate } from '@server/types/models/index.js'
import { logger, loggerTagsFactory } from '../../helpers/logger.js'
import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants.js'
-import { sequelizeTypescript } from '../../initializers/database.js'
import { ScheduleVideoUpdateModel } from '../../models/video/schedule-video-update.js'
+import { LocalVideoUpdater } from '../local-video-updater.js'
import { Notifier } from '../notifier/index.js'
-import { onVideoLocalUpdate } from '../video-jobs.js'
-import { VideoPathManager } from '../video-path-manager.js'
-import { isNewVideoForSubscription, setVideoPrivacy } from '../video-privacy.js'
import { AbstractScheduler } from './abstract-scheduler.js'
const lTags = loggerTagsFactory('schedulers', 'update-videos')
@@ -37,63 +34,38 @@ export class UpdateVideosScheduler extends AbstractScheduler {
const videoOnly = await VideoModel.load(schedule.videoId)
if (!videoOnly) continue
- const mutexReleaser = await VideoPathManager.Instance.lockFiles(videoOnly.uuid)
-
try {
const { video, published } = await this.updateAVideo(schedule)
if (published) Notifier.Instance.notifyOnVideoPublishedAfterScheduledUpdate(video)
} catch (err) {
logger.error('Cannot update video ' + videoOnly.uuid, { err, ...lTags(videoOnly.uuid) })
- } finally {
- mutexReleaser()
}
}
}
private async updateAVideo (schedule: MScheduleVideoUpdate) {
let oldPrivacy: VideoPrivacyType
- let newVideoForSubscriptions = false
let published = false
- const video = await sequelizeTypescript.transaction(async t => {
- const video = await VideoModel.loadFull(schedule.videoId, t)
- if (video.state === VideoState.TO_TRANSCODE) return null
+ let video = await VideoModel.loadFull(schedule.videoId)
+ if (video.state === VideoState.TO_TRANSCODE) return { video, published: false }
- logger.info('Executing scheduled video update on ' + video.uuid, lTags(video.uuid))
+ logger.info('Executing scheduled video update on ' + video.uuid, lTags(video.uuid))
- if (schedule.privacy) {
- newVideoForSubscriptions = isNewVideoForSubscription({
- currentPrivacy: video.privacy,
- newPrivacy: schedule.privacy,
- firstPublishedAt: video.firstPublishedAt
- })
+ if (schedule.privacy) {
+ oldPrivacy = video.privacy
- oldPrivacy = video.privacy
+ const updater = new LocalVideoUpdater({ user: null, tags: lTags().tags, video })
- setVideoPrivacy(video, schedule.privacy)
- await video.save({ transaction: t })
+ video = await updater.update({ privacy: schedule.privacy })
- if (oldPrivacy === VideoPrivacy.PRIVATE) {
- published = true
- }
+ if (oldPrivacy === VideoPrivacy.PRIVATE) {
+ published = true
}
-
- await schedule.destroy({ transaction: t })
-
- return video
- })
-
- if (!video) {
- return { video, published: false }
}
- await onVideoLocalUpdate({
- video,
- oldPrivacy,
- isNewVideoForSubscription: newVideoForSubscriptions,
- nameChanged: false
- })
+ await schedule.destroy()
return { video, published }
}
diff --git a/server/core/lib/video-jobs.ts b/server/core/lib/video-jobs.ts
index 988fa0c79a..9bb3670518 100644
--- a/server/core/lib/video-jobs.ts
+++ b/server/core/lib/video-jobs.ts
@@ -1,15 +1,11 @@
-import { exists } from '@peertube/peertube-core-utils'
-import { ManageVideoTorrentPayload, VideoFileStream, VideoPrivacyType, VideoState, VideoStateType } from '@peertube/peertube-models'
-import { logger } from '@server/helpers/logger.js'
+import { VideoFileStream, VideoState, VideoStateType } from '@peertube/peertube-models'
import { CONFIG } from '@server/initializers/config.js'
import { VideoJobInfoModel } from '@server/models/video/video-job-info.js'
import { VideoModel } from '@server/models/video/video.js'
-import { MVideo, MVideoFile, MVideoFull, MVideoUUID } from '@server/types/models/index.js'
+import { MVideo, MVideoFile, MVideoUUID } from '@server/types/models/index.js'
import { CreateJobOptions, CreateJobTypeAndPayload, JobQueue } from './job-queue/job-queue.js'
-import { lTags } from './object-storage/shared/index.js'
import { VideoStoryboardJobHandler } from './runners/index.js'
import { createTranscriptionTaskIfNeeded } from './video-captions.js'
-import { moveFilesIfPrivacyChanged } from './video-privacy.js'
export async function buildMoveVideoJob (options: {
video: MVideoUUID
@@ -155,65 +151,3 @@ export async function addVideoJobsAfterCreation (options: {
await createTranscriptionTaskIfNeeded(video)
}
}
-
-export async function onVideoLocalUpdate (options: {
- video: MVideoFull
-
- isNewVideoForSubscription: boolean
-
- nameChanged: boolean
- oldPrivacy?: VideoPrivacyType
-}) {
- const { video, nameChanged, oldPrivacy, isNewVideoForSubscription } = options
- const jobs: CreateJobTypeAndPayload[] = []
-
- const filePathChanged = exists(oldPrivacy)
- ? await moveFilesIfPrivacyChanged(video, oldPrivacy)
- : false
-
- const hls = video.getHLSPlaylist()
-
- if (filePathChanged && hls) {
- logger.debug('Updating HLS playlist file paths after privacy change', lTags(video.uuid))
-
- hls.assignP2PMediaLoaderInfoHashes(video, hls.VideoFiles)
- await hls.save()
- }
-
- if (!video.isLive && (nameChanged || filePathChanged)) {
- logger.debug('Updating video torrent metadata after name or file path change', lTags(video.uuid))
-
- for (const file of (video.VideoFiles || [])) {
- const payload: ManageVideoTorrentPayload = { action: 'update-metadata', videoId: video.id, videoFileId: file.id }
-
- jobs.push({ type: 'manage-video-torrent', payload })
- }
-
- const hls = video.getHLSPlaylist()
-
- for (const file of (hls?.VideoFiles || [])) {
- const payload: ManageVideoTorrentPayload = { action: 'update-metadata', streamingPlaylistId: hls.id, videoFileId: file.id }
-
- jobs.push({ type: 'manage-video-torrent', payload })
- }
- }
-
- jobs.push({
- type: 'federate-video',
- payload: { videoUUID: video.uuid }
- })
-
- if (isNewVideoForSubscription) {
- logger.debug('Video is considered new for subscriptions: create the notification job', lTags(video.uuid))
-
- jobs.push({
- type: 'notify',
- payload: {
- action: 'new-video',
- videoUUID: video.uuid
- }
- })
- }
-
- return JobQueue.Instance.createSequentialJobFlow(...jobs)
-}
diff --git a/server/core/lib/video-privacy.ts b/server/core/lib/video-privacy.ts
index 6c824493c5..4c3d148626 100644
--- a/server/core/lib/video-privacy.ts
+++ b/server/core/lib/video-privacy.ts
@@ -14,18 +14,6 @@ const validPrivacySet = new Set<VideoPrivacyType>([
VideoPrivacy.PASSWORD_PROTECTED
])
-export function setVideoPrivacy (video: MVideo, newPrivacy: VideoPrivacyType) {
- if (video.privacy === VideoPrivacy.PRIVATE && newPrivacy !== VideoPrivacy.PRIVATE) {
- video.publishedAt = new Date()
-
- if (!video.firstPublishedAt) {
- video.firstPublishedAt = video.publishedAt
- }
- }
-
- video.privacy = newPrivacy
-}
-
export function isVideoInPrivateDirectory (privacy: VideoPrivacyType) {
return validPrivacySet.has(privacy)
}