ActivityPub-federated video streaming platform using P2P directly in your web browser
Refactor video updater
Chocobozzz0578b8bfa02a
diff --git a/server/core/controllers/api/videos/update.ts b/server/core/controllers/api/videos/update.tsindex 6081a70716..721e98b98b 100644--- a/server/core/controllers/api/videos/update.ts+++ b/server/core/controllers/api/videos/update.ts@@ -6,12 +6,10 @@ import { openapiOperationDoc } from '@server/middlewares/doc.js'import { MVideoThumbnails } from '@server/types/models/index.js'import express from 'express'import { createReqFiles } from '../../../helpers/express-utils.js'-import { loggerTagsFactory } from '../../../helpers/logger.js'import { MIMETYPES } from '../../../initializers/constants.js'import { Hooks } from '../../../lib/plugins/hooks.js'import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate, videosUpdateValidator } from '../../../middlewares/index.js'-const lTags = loggerTagsFactory('api', 'video')const updateRouter = express.Router()const reqVideoFileUpdate = createReqFiles([ 'thumbnailfile', 'previewfile' ], MIMETYPES.IMAGE.MIMETYPE_EXT)@@ -40,9 +38,9 @@ async function updateVideo (req: express.Request, res: express.Response) {const thumbnails = await buildVideoThumbnailsFromReq(videoFromReq, req)- const localVideoUpdater = new LocalVideoUpdater({ user, lTags, video: videoFromReq })+ const localVideoUpdater = new LocalVideoUpdater({ user, tags: [ 'api', 'video' ], video: videoFromReq })- const video = await localVideoUpdater.update({ ...body, thumbnails })+ const video = await localVideoUpdater.update({ ...body, thumbnails, channel: res.locals.videoChannel })Hooks.runAction('action:api.video.updated', { video, body: req.body, req, res })diff --git a/server/core/lib/local-video-updater.ts b/server/core/lib/local-video-updater.tsnew file mode 100644index 0000000000..de5689bf72--- /dev/null+++ b/server/core/lib/local-video-updater.ts@@ -0,0 +1,401 @@+import { exists, forceNumber } from '@peertube/peertube-core-utils'+import {+ ManageVideoTorrentPayload,+ NSFWFlag,+ VideoChannelActivityAction,+ VideoPrivacy,+ VideoPrivacyType,+ VideoUpdate+} from '@peertube/peertube-models'+import { auditLoggerFactory, getAuditIdFromUser, VideoAuditView } from '@server/helpers/audit-logger.js'+import { logger, loggerTagsFactory, LoggerTagsFn } from '@server/helpers/logger.js'+import { sequelizeTypescript } from '@server/initializers/database.js'+import { getServerAccount } from '@server/models/application/application.js'+import { ScheduleVideoUpdateModel } from '@server/models/video/schedule-video-update.js'+import { VideoChannelActivityModel } from '@server/models/video/video-channel-activity.js'+import { VideoPasswordModel } from '@server/models/video/video-password.js'+import { VideoModel } from '@server/models/video/video.js'+import { MChannelBannerAccountDefault, MThumbnail, MUserAccountUrl, MVideo, MVideoFull, MVideoUUID } from '@server/types/models/index.js'+import { FilteredModelAttributes } from '@server/types/sequelize.js'+import { Transaction } from 'sequelize'+import { sendDeleteVideo } from './activitypub/send/send-delete.js'+import { changeVideoChannelShare } from './activitypub/share.js'+import { isPrivacyForFederation } from './activitypub/videos/index.js'+import { AutomaticTagger } from './automatic-tags/automatic-tagger.js'+import { setAndSaveVideoAutomaticTags } from './automatic-tags/automatic-tags.js'+import { CreateJobTypeAndPayload, JobQueue } from './job-queue/job-queue.js'+import { autoBlacklistVideoIfNeeded } from './video-blacklist.js'+import { replaceChaptersFromDescriptionIfNeeded } from './video-chapters.js'+import { VideoPathManager } from './video-path-manager.js'+import { isNewVideoForSubscription, moveFilesIfPrivacyChanged } from './video-privacy.js'+import { setVideoTags } from './video.js'++const auditLogger = auditLoggerFactory('videos')++export class LocalVideoUpdater {+ private readonly lTags: LoggerTagsFn+ private readonly oldVideo: MVideoFull+ private readonly user: MUserAccountUrl | null++ private readonly oldPrivacy: VideoPrivacyType+ private readonly hadPrivacyForFederation: boolean++ private readonly oldVideoAuditView: VideoAuditView++ constructor (options: {+ tags: (string | number)[]++ video: MVideoFull+ user: MUserAccountUrl | null+ }) {+ this.user = options.user+ this.oldVideo = options.video+ this.lTags = loggerTagsFactory(...options.tags, options.video.uuid)++ this.hadPrivacyForFederation = isPrivacyForFederation(this.oldVideo.privacy)+ this.oldPrivacy = this.oldVideo.privacy++ this.oldVideoAuditView = new VideoAuditView(this.oldVideo.toFormattedDetailsJSON())+ }++ async update (options: Omit<VideoUpdate, 'thumbnailfile' | 'previewfile'> & {+ thumbnails?: MThumbnail[]+ channel?: MChannelBannerAccountDefault+ }) {+ const videoFileLockReleaser = await VideoPathManager.Instance.lockFiles(this.oldVideo.uuid)++ try {+ const { video, newVideoForSubscription } = await sequelizeTypescript.transaction(async t => {+ // Refresh video to prevent concurrent updates+ const video = await VideoModel.loadFull(this.oldVideo.id, t)++ const oldName = video.name+ const oldDescription = video.description+ const oldVideoChannel = video.VideoChannel++ const keysToUpdate: (keyof VideoUpdate & FilteredModelAttributes<VideoModel>)[] = [+ 'name',+ 'category',+ 'licence',+ 'language',+ 'nsfw',+ 'nsfwFlags',+ 'nsfwSummary',+ 'waitTranscoding',+ 'support',+ 'description',+ 'downloadEnabled',+ 'commentsPolicy'+ ]++ for (const key of keysToUpdate) {+ if (options[key] !== undefined) video.set(key, options[key])+ }++ if (video.nsfw !== true) {+ video.nsfwFlags = NSFWFlag.NONE+ video.nsfwSummary = null+ }++ if (options.originallyPublishedAt !== undefined) {+ video.originallyPublishedAt = options.originallyPublishedAt+ ? new Date(options.originallyPublishedAt)+ : null+ }++ // Privacy update?+ let newVideoForSubscription = false++ if (options.privacy !== undefined) {+ newVideoForSubscription = isNewVideoForSubscription({+ currentPrivacy: video.privacy,+ newPrivacy: options.privacy,+ firstPublishedAt: video.firstPublishedAt+ })++ await this.updateVideoPrivacy({+ video,+ privacy: options.privacy,+ passwords: options.videoPasswords,+ hadPrivacyForFederation: this.hadPrivacyForFederation,+ transaction: t+ })+ }++ // Force updatedAt attribute change+ if (!video.changed()) {+ await video.setAsRefreshed(t)+ }++ await video.save({ transaction: t })++ if (options.thumbnails && options.thumbnails.length !== 0) {+ await video.replaceAndSaveThumbnails(options.thumbnails, t)+ }++ if (options.tags !== undefined) {+ await setVideoTags({ video, tags: options.tags, transaction: t })+ }++ // Channel update+ if (options.channel && video.channelId !== options.channel.id) {+ const oldChannel = video.VideoChannel+ const newChannel = options.channel++ if (this.user) {+ await VideoChannelActivityModel.addVideoActivity({+ action: VideoChannelActivityAction.REMOVE_CHANNEL_OWNERSHIP,+ user: this.user,+ channel: oldChannel,+ video,+ transaction: t+ })++ await VideoChannelActivityModel.addVideoActivity({+ action: VideoChannelActivityAction.CREATE_CHANNEL_OWNERSHIP,+ user: this.user,+ channel: newChannel,+ video,+ transaction: t+ })+ }++ await video.$set('VideoChannel', newChannel, { transaction: t })+ video.VideoChannel = newChannel++ if (this.hadPrivacyForFederation === true) {+ await changeVideoChannelShare(video, oldVideoChannel, t)+ }+ } else if (this.user) {+ await VideoChannelActivityModel.addVideoActivity({+ action: VideoChannelActivityAction.UPDATE,+ user: this.user,+ channel: video.VideoChannel,+ video,+ transaction: t+ })+ }++ // Schedule an update?+ await this.updateScheduleIfNeeded({+ video,+ scheduleUpdate: options.scheduleUpdate,+ transaction: t+ })++ if (oldDescription !== video.description) {+ await replaceChaptersFromDescriptionIfNeeded({+ video,+ newDescription: video.description,+ transaction: t,+ oldDescription+ })+ }++ let automaticTagsByAccount: Record<number, string[]>+ if (oldName !== video.name || oldDescription !== video.description) {+ automaticTagsByAccount = await new AutomaticTagger().buildVideoAutomaticTags({+ serverAccount: await getServerAccount(),+ video,+ transaction: t+ })+ await setAndSaveVideoAutomaticTags({ video, automaticTagsByAccount, transaction: t })+ }++ await autoBlacklistVideoIfNeeded({+ video,+ user: this.user,+ isRemote: false,+ isNew: false,+ isNewFile: false,+ automaticTagsByAccount,+ transaction: t+ })++ if (this.user) {+ auditLogger.update(+ getAuditIdFromUser(this.user),+ new VideoAuditView(video.toFormattedDetailsJSON()),+ this.oldVideoAuditView+ )+ }++ logger.info('Video with name %s and uuid %s updated.', video.name, video.uuid, this.lTags())++ return { video, newVideoForSubscription }+ })++ const nameChanged = exists(options.name)++ const filePathChanged = exists(this.oldPrivacy)+ ? await moveFilesIfPrivacyChanged(video, this.oldPrivacy)+ : false++ await this.rebuildHLSInfoHashesIfNeeded({ video, filePathChanged })++ const jobs: CreateJobTypeAndPayload[] = []++ jobs.push(...this.buildUpdateTorrentJobIfNeeded({ video, nameChanged, filePathChanged }))+ jobs.push(...this.buildFederationJob({ video }))+ jobs.push(...this.buildNotifyJobIfNeeded({ video, newVideoForSubscription }))++ await JobQueue.Instance.createSequentialJobFlow(...jobs)++ return video+ } finally {+ videoFileLockReleaser()+ }+ }++ // Return a boolean indicating if the video is considered as "new" for remote instances in the federation+ private async updateVideoPrivacy (options: {+ video: MVideoFull+ privacy: VideoUpdate['privacy']+ passwords: VideoUpdate['videoPasswords']+ hadPrivacyForFederation: boolean+ transaction: Transaction+ }) {+ const { video, privacy, passwords, hadPrivacyForFederation, transaction } = options++ const newPrivacy = forceNumber(privacy) as VideoPrivacyType+ this.setVideoPrivacy(video, newPrivacy)++ // Delete passwords if video is not anymore password protected+ if (video.privacy === VideoPrivacy.PASSWORD_PROTECTED && newPrivacy !== VideoPrivacy.PASSWORD_PROTECTED) {+ await VideoPasswordModel.deleteAllPasswords(video.id, transaction)+ }++ if (newPrivacy === VideoPrivacy.PASSWORD_PROTECTED && exists(passwords)) {+ await VideoPasswordModel.deleteAllPasswords(video.id, transaction)+ await VideoPasswordModel.addPasswords(passwords, video.id, transaction)+ }++ // Unfederate the video if the new privacy is not compatible with federation+ if (hadPrivacyForFederation && !isPrivacyForFederation(video.privacy)) {+ await sendDeleteVideo({ video, deleteForPrivacyChange: true, transaction })+ }+ }++ private async updateScheduleIfNeeded (options: {+ video: MVideoFull+ scheduleUpdate: VideoUpdate['scheduleUpdate']+ transaction: Transaction+ }) {+ const { video, scheduleUpdate, transaction } = options++ if (scheduleUpdate) {+ const updateAt = new Date(scheduleUpdate.updateAt)++ video.publishedAt = updateAt+ await video.save({ transaction })++ await ScheduleVideoUpdateModel.upsert({+ videoId: video.id,+ updateAt,+ privacy: scheduleUpdate.privacy || null+ }, { transaction })++ return+ }++ if (scheduleUpdate === null) {+ const deleted = await ScheduleVideoUpdateModel.deleteByVideoId(video.id, transaction)++ if (deleted) {+ video.publishedAt = new Date()+ await video.save({ transaction })+ }+ }+ }++ private setVideoPrivacy (video: MVideo, newPrivacy: VideoPrivacyType) {+ if (video.privacy === VideoPrivacy.PRIVATE && newPrivacy !== VideoPrivacy.PRIVATE) {+ video.publishedAt = new Date()++ if (!video.firstPublishedAt) {+ video.firstPublishedAt = video.publishedAt+ }+ }++ video.privacy = newPrivacy+ }++ private async rebuildHLSInfoHashesIfNeeded (options: {+ video: MVideoFull+ filePathChanged: boolean+ }) {+ const { video, filePathChanged } = options++ const hls = video.getHLSPlaylist()++ if (filePathChanged && hls) {+ logger.debug('Updating HLS playlist file paths after privacy change', this.lTags())++ hls.assignP2PMediaLoaderInfoHashes(video, hls.VideoFiles)+ await hls.save()+ }+ }++ private buildUpdateTorrentJobIfNeeded (options: {+ video: MVideoFull+ nameChanged: boolean+ filePathChanged: boolean+ }) {+ const { video, nameChanged, filePathChanged } = options++ const jobs: CreateJobTypeAndPayload[] = []++ if (!video.isLive && (nameChanged || filePathChanged)) {+ logger.debug('Updating video torrent metadata after name or file path change', this.lTags())++ for (const file of (video.VideoFiles || [])) {+ const payload: ManageVideoTorrentPayload = { action: 'update-metadata', videoId: video.id, videoFileId: file.id }++ jobs.push({ type: 'manage-video-torrent', payload })+ }++ const hls = video.getHLSPlaylist()++ for (const file of (hls?.VideoFiles || [])) {+ const payload: ManageVideoTorrentPayload = { action: 'update-metadata', streamingPlaylistId: hls.id, videoFileId: file.id }++ jobs.push({ type: 'manage-video-torrent', payload })+ }+ }++ return jobs+ }++ private buildFederationJob (options: {+ video: MVideoUUID+ }): CreateJobTypeAndPayload[] {+ const { video } = options++ return [+ {+ type: 'federate-video',+ payload: { videoUUID: video.uuid }+ }+ ]+ }++ private buildNotifyJobIfNeeded (options: {+ video: MVideoUUID+ newVideoForSubscription: boolean+ }): CreateJobTypeAndPayload[] {+ const { video, newVideoForSubscription } = options+ if (!newVideoForSubscription) return []++ logger.debug('Video is considered new for subscriptions: create the notification job', this.lTags())++ return [+ {+ type: 'notify',+ payload: {+ action: 'new-video',+ videoUUID: video.uuid+ }+ }+ ]+ }+}diff --git a/server/core/lib/schedulers/update-videos-scheduler.ts b/server/core/lib/schedulers/update-videos-scheduler.tsindex fdb15d9bea..e1e061e509 100644--- a/server/core/lib/schedulers/update-videos-scheduler.ts+++ b/server/core/lib/schedulers/update-videos-scheduler.ts@@ -3,12 +3,9 @@ import { VideoModel } from '@server/models/video/video.js'import { MScheduleVideoUpdate } from '@server/types/models/index.js'import { logger, loggerTagsFactory } from '../../helpers/logger.js'import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants.js'-import { sequelizeTypescript } from '../../initializers/database.js'import { ScheduleVideoUpdateModel } from '../../models/video/schedule-video-update.js'+import { LocalVideoUpdater } from '../local-video-updater.js'import { Notifier } from '../notifier/index.js'-import { onVideoLocalUpdate } from '../video-jobs.js'-import { VideoPathManager } from '../video-path-manager.js'-import { isNewVideoForSubscription, setVideoPrivacy } from '../video-privacy.js'import { AbstractScheduler } from './abstract-scheduler.js'const lTags = loggerTagsFactory('schedulers', 'update-videos')@@ -37,63 +34,38 @@ export class UpdateVideosScheduler extends AbstractScheduler {const videoOnly = await VideoModel.load(schedule.videoId)if (!videoOnly) continue- const mutexReleaser = await VideoPathManager.Instance.lockFiles(videoOnly.uuid)-try {const { video, published } = await this.updateAVideo(schedule)if (published) Notifier.Instance.notifyOnVideoPublishedAfterScheduledUpdate(video)} catch (err) {logger.error('Cannot update video ' + videoOnly.uuid, { err, ...lTags(videoOnly.uuid) })- } finally {- mutexReleaser()}}}private async updateAVideo (schedule: MScheduleVideoUpdate) {let oldPrivacy: VideoPrivacyType- let newVideoForSubscriptions = falselet published = false- const video = await sequelizeTypescript.transaction(async t => {- const video = await VideoModel.loadFull(schedule.videoId, t)- if (video.state === VideoState.TO_TRANSCODE) return null+ let video = await VideoModel.loadFull(schedule.videoId)+ if (video.state === VideoState.TO_TRANSCODE) return { video, published: false }- logger.info('Executing scheduled video update on ' + video.uuid, lTags(video.uuid))+ logger.info('Executing scheduled video update on ' + video.uuid, lTags(video.uuid))- if (schedule.privacy) {- newVideoForSubscriptions = isNewVideoForSubscription({- currentPrivacy: video.privacy,- newPrivacy: schedule.privacy,- firstPublishedAt: video.firstPublishedAt- })+ if (schedule.privacy) {+ oldPrivacy = video.privacy- oldPrivacy = video.privacy+ const updater = new LocalVideoUpdater({ user: null, tags: lTags().tags, video })- setVideoPrivacy(video, schedule.privacy)- await video.save({ transaction: t })+ video = await updater.update({ privacy: schedule.privacy })- if (oldPrivacy === VideoPrivacy.PRIVATE) {- published = true- }+ if (oldPrivacy === VideoPrivacy.PRIVATE) {+ published = true}-- await schedule.destroy({ transaction: t })-- return video- })-- if (!video) {- return { video, published: false }}- await onVideoLocalUpdate({- video,- oldPrivacy,- isNewVideoForSubscription: newVideoForSubscriptions,- nameChanged: false- })+ await schedule.destroy()return { video, published }}diff --git a/server/core/lib/video-jobs.ts b/server/core/lib/video-jobs.tsindex 988fa0c79a..9bb3670518 100644--- a/server/core/lib/video-jobs.ts+++ b/server/core/lib/video-jobs.ts@@ -1,15 +1,11 @@-import { exists } from '@peertube/peertube-core-utils'-import { ManageVideoTorrentPayload, VideoFileStream, VideoPrivacyType, VideoState, VideoStateType } from '@peertube/peertube-models'-import { logger } from '@server/helpers/logger.js'+import { VideoFileStream, VideoState, VideoStateType } from '@peertube/peertube-models'import { CONFIG } from '@server/initializers/config.js'import { VideoJobInfoModel } from '@server/models/video/video-job-info.js'import { VideoModel } from '@server/models/video/video.js'-import { MVideo, MVideoFile, MVideoFull, MVideoUUID } from '@server/types/models/index.js'+import { MVideo, MVideoFile, MVideoUUID } from '@server/types/models/index.js'import { CreateJobOptions, CreateJobTypeAndPayload, JobQueue } from './job-queue/job-queue.js'-import { lTags } from './object-storage/shared/index.js'import { VideoStoryboardJobHandler } from './runners/index.js'import { createTranscriptionTaskIfNeeded } from './video-captions.js'-import { moveFilesIfPrivacyChanged } from './video-privacy.js'export async function buildMoveVideoJob (options: {video: MVideoUUID@@ -155,65 +151,3 @@ export async function addVideoJobsAfterCreation (options: {await createTranscriptionTaskIfNeeded(video)}}--export async function onVideoLocalUpdate (options: {- video: MVideoFull-- isNewVideoForSubscription: boolean-- nameChanged: boolean- oldPrivacy?: VideoPrivacyType-}) {- const { video, nameChanged, oldPrivacy, isNewVideoForSubscription } = options- const jobs: CreateJobTypeAndPayload[] = []-- const filePathChanged = exists(oldPrivacy)- ? await moveFilesIfPrivacyChanged(video, oldPrivacy)- : false-- const hls = video.getHLSPlaylist()-- if (filePathChanged && hls) {- logger.debug('Updating HLS playlist file paths after privacy change', lTags(video.uuid))-- hls.assignP2PMediaLoaderInfoHashes(video, hls.VideoFiles)- await hls.save()- }-- if (!video.isLive && (nameChanged || filePathChanged)) {- logger.debug('Updating video torrent metadata after name or file path change', lTags(video.uuid))-- for (const file of (video.VideoFiles || [])) {- const payload: ManageVideoTorrentPayload = { action: 'update-metadata', videoId: video.id, videoFileId: file.id }-- jobs.push({ type: 'manage-video-torrent', payload })- }-- const hls = video.getHLSPlaylist()-- for (const file of (hls?.VideoFiles || [])) {- const payload: ManageVideoTorrentPayload = { action: 'update-metadata', streamingPlaylistId: hls.id, videoFileId: file.id }-- jobs.push({ type: 'manage-video-torrent', payload })- }- }-- jobs.push({- type: 'federate-video',- payload: { videoUUID: video.uuid }- })-- if (isNewVideoForSubscription) {- logger.debug('Video is considered new for subscriptions: create the notification job', lTags(video.uuid))-- jobs.push({- type: 'notify',- payload: {- action: 'new-video',- videoUUID: video.uuid- }- })- }-- return JobQueue.Instance.createSequentialJobFlow(...jobs)-}diff --git a/server/core/lib/video-privacy.ts b/server/core/lib/video-privacy.tsindex 6c824493c5..4c3d148626 100644--- a/server/core/lib/video-privacy.ts+++ b/server/core/lib/video-privacy.ts@@ -14,18 +14,6 @@ const validPrivacySet = new Set<VideoPrivacyType>([VideoPrivacy.PASSWORD_PROTECTED])-export function setVideoPrivacy (video: MVideo, newPrivacy: VideoPrivacyType) {- if (video.privacy === VideoPrivacy.PRIVATE && newPrivacy !== VideoPrivacy.PRIVATE) {- video.publishedAt = new Date()-- if (!video.firstPublishedAt) {- video.firstPublishedAt = video.publishedAt- }- }-- video.privacy = newPrivacy-}-export function isVideoInPrivateDirectory (privacy: VideoPrivacyType) {return validPrivacySet.has(privacy)}