ActivityPub-federated video streaming platform using P2P directly in your web browser
More robust sync
Chocobozzzfaf7b28cf292
diff --git a/server/core/lib/sync-channel.ts b/server/core/lib/sync-channel.tsindex f86470a86f..bdea42cb61 100644--- a/server/core/lib/sync-channel.ts+++ b/server/core/lib/sync-channel.ts@@ -1,4 +1,4 @@-import { VideoChannelSyncState } from '@peertube/peertube-models'+import { VideoChannelSyncState, VideoImportState } from '@peertube/peertube-models'import { logger, loggerTagsFactory, LoggerTagsFn } from '@server/helpers/logger.js'import { YoutubeDlImportError, YoutubeDlImportErrorCode, YoutubeDLWrapper } from '@server/helpers/youtube-dl/index.js'import { CONFIG } from '@server/initializers/config.js'@@ -50,6 +50,9 @@ export async function synchronizeChannel (options: {)const children: CreateJobTypeAndPayload[] = []+ // Ids of video imports already persisted in DB+ // If job creation fails, these must be reverted to FAILED so they are picked up by the retry mechanism instead of staying stuck+ const touchedVideoImportIds: number[] = []let buildJobErrors = 0@@ -59,7 +62,7 @@ export async function synchronizeChannel (options: {try {if (await skipImport({ channel, channelSync, targetUrl, lTags })) continue- const { job } = await buildYoutubeDLImport({+ const { job, videoImport } = await buildYoutubeDLImport({user,channel,targetUrl,@@ -72,6 +75,7 @@ export async function synchronizeChannel (options: {})children.push(job)+ touchedVideoImportIds.push(videoImport.id)} catch (err) {if (err instanceof YoutubeDlImportError) {if (@@ -106,6 +110,7 @@ export async function synchronizeChannel (options: {)children.push(await buildRetryImportJob(videoImport))+ touchedVideoImportIds.push(videoImport.id)}}@@ -118,7 +123,20 @@ export async function synchronizeChannel (options: {}}- await JobQueue.Instance.createJobWithChildren(parent, children)+ try {+ await JobQueue.Instance.createJobWithChildren(parent, children)+ } catch (err) {+ try {+ await VideoImportModel.updateStateByIds(touchedVideoImportIds, VideoImportState.FAILED, 'Failed to create the video import job')+ } catch (updateErr) {+ logger.error(`Failed to update state of video imports to FAILED after failing to create the video import job`, {+ updateErr,+ ...rootLTags()+ })+ }++ throw err+ }} catch (err) {logger.error(`Failed to import ${externalChannelUrl} in channel ${channelUsername}`, { err, ...rootLTags() })diff --git a/server/core/lib/video-pre-import.ts b/server/core/lib/video-pre-import.tsindex 196bff59bd..e652c5d40e 100644--- a/server/core/lib/video-pre-import.ts+++ b/server/core/lib/video-pre-import.ts@@ -239,50 +239,66 @@ export async function buildYoutubeDLImport (options: {videoPasswords: importDataOverride.videoPasswords})- await sequelizeTypescript.transaction(async transaction => {- // Priority to explicitly set description- if (importDataOverride.description) {- const inserted = await replaceChaptersFromDescriptionIfNeeded({ newDescription: importDataOverride.description, video, transaction })- if (inserted) return- }+ try {+ await sequelizeTypescript.transaction(async transaction => {+ // Priority to explicitly set description+ if (importDataOverride.description) {+ const inserted = await replaceChaptersFromDescriptionIfNeeded({+ newDescription: importDataOverride.description,+ video,+ transaction+ })+ if (inserted) return+ }- // Then priority to youtube-dl chapters- if (youtubeDLInfo.chapters.length !== 0) {- logger.info(- `Inserting chapters in video ${video.uuid} from youtube-dl`,- { chapters: youtubeDLInfo.chapters, tags: [ 'chapters', video.uuid ] }- )+ // Then priority to youtube-dl chapters+ if (youtubeDLInfo.chapters.length !== 0) {+ logger.info(+ `Inserting chapters in video ${video.uuid} from youtube-dl`,+ { chapters: youtubeDLInfo.chapters, tags: [ 'chapters', video.uuid ] }+ )- await replaceChapters({ video, chapters: youtubeDLInfo.chapters, transaction })- return- }+ await replaceChapters({ video, chapters: youtubeDLInfo.chapters, transaction })+ return+ }- if (video.description) {- await replaceChaptersFromDescriptionIfNeeded({ newDescription: video.description, video, transaction })- }- })+ if (video.description) {+ await replaceChaptersFromDescriptionIfNeeded({ newDescription: video.description, video, transaction })+ }+ })- // Get video subtitles- await processYoutubeSubtitles({ youtubeDL, targetUrl, video, userLanguage })+ // Get video subtitles+ await processYoutubeSubtitles({ youtubeDL, targetUrl, video, userLanguage })- let fileExt = `.${youtubeDLInfo.ext}`- if (!isVideoFileExtnameValid(fileExt)) fileExt = '.mp4'+ let fileExt = `.${youtubeDLInfo.ext}`+ if (!isVideoFileExtnameValid(fileExt)) fileExt = '.mp4'- const payload: VideoImportPayload = {- type: 'youtube-dl' as 'youtube-dl',- videoImportId: videoImport.id,- fileExt,- generateTranscription: importDataOverride.generateTranscription ?? true,- // If part of a sync process, there is a parent job that will aggregate children results- preventException: !!channelSync- }+ const payload: VideoImportPayload = {+ type: 'youtube-dl' as 'youtube-dl',+ videoImportId: videoImport.id,+ fileExt,+ generateTranscription: importDataOverride.generateTranscription ?? true,+ // If part of a sync process, there is a parent job that will aggregate children results+ preventException: !!channelSync+ }- videoImport.payload = payload- await videoImport.save()+ videoImport.payload = payload+ await videoImport.save()++ return {+ videoImport,+ job: { type: 'video-import' as 'video-import', payload }+ }+ } catch (err) {+ // Auto destroy video import to not keep a "PENDING" import that never gets a job+ try {+ await videoImport.Video.destroy()+ await videoImport.destroy()+ } catch (cleanupErr) {+ logger.error(`Cannot cleanup video import for ${targetUrl} after a build error.`, { err: cleanupErr })+ }- return {- videoImport,- job: { type: 'video-import' as 'video-import', payload }+ throw err}}diff --git a/server/core/models/video/video-import.ts b/server/core/models/video/video-import.tsindex caf6e476e7..8f40234410 100644--- a/server/core/models/video/video-import.ts+++ b/server/core/models/video/video-import.ts@@ -237,6 +237,20 @@ export class VideoImportModel extends SequelizeModel<VideoImportModel> {})}+ static updateStateByIds (ids: number[], state: VideoImportStateType, error?: string) {+ if (ids.length === 0) return+ return VideoImportModel.update(+ { state, error },+ {+ where: {+ id: {+ [Op.in]: ids+ }+ }+ }+ )+ }+// ---------------------------------------------------------------------------getTargetIdentifier () {