ActivityPub-federated video streaming platform using P2P directly in your web browser
Optimize and more reliable channel sync
Chocobozzz73b6a95600ee
diff --git a/config/default.yaml b/config/default.yamlindex 5a95c32b97..b3be3bc3bc 100644--- a/config/default.yaml+++ b/config/default.yaml@@ -811,7 +811,7 @@ import:timeout: '2 hours'# Number of attempts a user or the channel sync process can do to import a video if an error occurs during the import process- max_attempts: 3+ max_attempts: 5# Classic HTTP or all sites supported by youtube-dl https://rg3.github.io/youtube-dl/supportedsites.htmlhttp:diff --git a/config/production.yaml.example b/config/production.yaml.exampleindex f45b868896..273d745f07 100644--- a/config/production.yaml.example+++ b/config/production.yaml.example@@ -821,7 +821,7 @@ import:timeout: '2 hours'# Number of attempts a user or the channel sync process can do to import a video if an error occurs during the import process- max_attempts: 3+ max_attempts: 5# Classic HTTP or all sites supported by youtube-dl https://rg3.github.io/youtube-dl/supportedsites.htmlhttp:diff --git a/packages/models/src/server/job.model.ts b/packages/models/src/server/job.model.tsindex c94b09d563..296753396c 100644--- a/packages/models/src/server/job.model.ts+++ b/packages/models/src/server/job.model.ts@@ -281,6 +281,7 @@ export interface VideoChannelImportPayload {export interface AfterVideoChannelImportPayload {channelSyncId: number+ buildJobErrors: number}// ---------------------------------------------------------------------------diff --git a/packages/tests/src/api/videos/video-channel-syncs.ts b/packages/tests/src/api/videos/video-channel-syncs.tsindex cff1f33f35..daff8481aa 100644--- a/packages/tests/src/api/videos/video-channel-syncs.ts+++ b/packages/tests/src/api/videos/video-channel-syncs.ts@@ -235,7 +235,7 @@ describe('Test channel synchronizations', function () {await sqlCommands[0].setImportUrl(importId, 'http://fail.example.com')await sqlCommands[0].setImportState(importId, VideoImportState.FAILED)- for (let i = 2; i < 7; i++) {+ for (let i = 2; i < 6; i++) {await servers[0].debug.sendCommand({body: {command: 'process-video-channel-sync-latest'@@ -246,11 +246,34 @@ describe('Test channel synchronizations', function () {{const { data } = await servers[0].videoImports.listMyVideoImports({ videoChannelSyncId: rootChannelSyncId })- expect(data[0].state.id).to.equal(VideoImportState.FAILED)- expect(data[0].error).to.include('fail.example.com')- expect(data[0].attempts).to.equal(Math.min(3, i))+ const failedImport = data.find(i => i.id === importId)++ expect(failedImport.state.id).to.equal(VideoImportState.FAILED)+ expect(failedImport.error).to.include('fail.example.com')+ expect(failedImport.attempts).to.equal(i)+ }++ {+ const { data } = await servers[0].channelSyncs.listByAccount({ accountName: 'root' })+ const sync = data.find(s => s.id === rootChannelSyncId)+ expect(sync.state.id).to.equal(VideoChannelSyncState.FAILED)}}++ // Last one, sync stopped trying to import the failed job, so the sync is a success+ await servers[0].debug.sendCommand({+ body: {+ command: 'process-video-channel-sync-latest'+ }+ })++ await waitJobs(servers)++ {+ const { data } = await servers[0].channelSyncs.listByAccount({ accountName: 'root' })+ const sync = data.find(s => s.id === rootChannelSyncId)+ expect(sync.state.id).to.equal(VideoChannelSyncState.SYNCED)+ }})// FIXME: youtube-dl/yt-dlp doesn't work when speicifying a port after the hostnamediff --git a/packages/tests/src/api/videos/video-imports.ts b/packages/tests/src/api/videos/video-imports.tsindex 12fa9687c8..06efa45ea3 100644--- a/packages/tests/src/api/videos/video-imports.ts+++ b/packages/tests/src/api/videos/video-imports.ts@@ -124,8 +124,13 @@ describe('Test video imports', function () {expect(video.name).to.equal('small video - youtube'){- expect(video.thumbnailPath).to.match(new RegExp(`^/lazy-static/thumbnails/.+.webp$`))- expect(video.previewPath).to.match(new RegExp(`^/lazy-static/previews/.+.webp$`))+ try {+ expect(video.thumbnailPath).to.match(new RegExp(`^/lazy-static/thumbnails/.+.webp$`))+ expect(video.previewPath).to.match(new RegExp(`^/lazy-static/previews/.+.webp$`))+ } catch {+ expect(video.thumbnailPath).to.match(new RegExp(`^/lazy-static/thumbnails/.+.jpg$`))+ expect(video.previewPath).to.match(new RegExp(`^/lazy-static/previews/.+.jpg$`))+ }// FIXME: re-enable when we'll support webp pixel match// const suffix = mode === 'yt-dlp'diff --git a/server/core/controllers/api/videos/import.ts b/server/core/controllers/api/videos/import.tsindex 30506efa2b..315e241d0f 100644--- a/server/core/controllers/api/videos/import.ts+++ b/server/core/controllers/api/videos/import.ts@@ -7,8 +7,9 @@ import {VideoImportPayload,VideoImportState} from '@peertube/peertube-models'-import { retryImport } from '@server/lib/video-post-import.js'-import { buildVideoFromImport, buildYoutubeDLImport, insertFromImportIntoDB, YoutubeDlImportError } from '@server/lib/video-pre-import.js'+import { YoutubeDlImportError, YoutubeDlImportErrorCode } from '@server/helpers/youtube-dl/youtube-dl-wrapper.js'+import { buildRetryImportJob } from '@server/lib/video-post-import.js'+import { buildVideoFromImport, buildYoutubeDLImport, insertFromImportIntoDB } from '@server/lib/video-pre-import.js'import { MThumbnail, MVideoThumbnail } from '@server/types/models/index.js'import express from 'express'import { move } from 'fs-extra/esm'@@ -100,7 +101,7 @@ async function cancelVideoImport (req: express.Request, res: express.Response) {async function retryVideoImport (req: express.Request, res: express.Response) {const videoImport = res.locals.videoImport- await retryImport(videoImport)+ await JobQueue.Instance.createJob(await buildRetryImportJob(videoImport))return res.sendStatus(HttpStatusCode.NO_CONTENT_204)}@@ -179,10 +180,11 @@ async function handleTorrentImport (req: express.Request, res: express.Response,function statusFromYtDlImportError (err: YoutubeDlImportError): HttpStatusCodeType {switch (err.code) {- case YoutubeDlImportError.CODE.NOT_ONLY_UNICAST_URL:+ case YoutubeDlImportErrorCode.NOT_ONLY_UNICAST_URL:return HttpStatusCode.FORBIDDEN_403- case YoutubeDlImportError.CODE.FETCH_ERROR:+ case YoutubeDlImportErrorCode.FETCH_ERROR:+ case YoutubeDlImportErrorCode.IS_LIVE:return HttpStatusCode.BAD_REQUEST_400default:diff --git a/server/core/helpers/youtube-dl/youtube-dl-wrapper.ts b/server/core/helpers/youtube-dl/youtube-dl-wrapper.tsindex 38eaeb7d12..9ac43533b4 100644--- a/server/core/helpers/youtube-dl/youtube-dl-wrapper.ts+++ b/server/core/helpers/youtube-dl/youtube-dl-wrapper.ts@@ -1,10 +1,12 @@+import { VideoResolutionType } from '@peertube/peertube-models'+import { CONFIG } from '@server/initializers/config.js'+import { ExecaError } from 'execa'import { move, pathExists, remove } from 'fs-extra/esm'import { readdir } from 'fs/promises'import { dirname, join } from 'path'import { inspect } from 'util'-import { VideoResolutionType } from '@peertube/peertube-models'-import { CONFIG } from '@server/initializers/config.js'import { isVideoFileExtnameValid } from '../custom-validators/videos.js'+import { t } from '../i18n.js'import { logger, loggerTagsFactory } from '../logger.js'import { generateVideoImportTmpPath } from '../utils.js'import { YoutubeDLCLI } from './youtube-dl-cli.js'@@ -12,6 +14,60 @@ import { YoutubeDLInfo, YoutubeDLInfoBuilder } from './youtube-dl-info-builder.jconst lTags = loggerTagsFactory('youtube-dl')+export const YoutubeDlImportErrorCode = {+ FETCH_ERROR: 0,+ NOT_ONLY_UNICAST_URL: 1,+ SKIP_PUBLICATION_DATE: 2,+ IS_LIVE: 3+}++export type YoutubeDlImportErrorCodeType = typeof YoutubeDlImportErrorCode[keyof typeof YoutubeDlImportErrorCode]++export class YoutubeDlImportError extends Error {+ static fromError (err: Error, code: YoutubeDlImportErrorCodeType, message?: string) {+ const ytDlErr = new this({ message: message ?? err.message, code })+ ytDlErr.cause = err++ return ytDlErr+ }++ code: YoutubeDlImportErrorCodeType++ constructor ({ message, code }) {+ super(message)++ this.code = code+ }++ isUnavailableVideoError () {+ const stderr = this.getStderr()++ if (stderr.includes('Video unavailable') || stderr.includes(' 429 ')) {+ return true+ }++ return false+ }++ isRateLimitError () {+ const stderr = this.getStderr()++ if (stderr.includes('Sign in to confirm you’re not a bot')) {+ return true+ }++ return false+ }++ private getStderr () {+ const stderr = (this.cause as ExecaError)?.stderr++ if (typeof stderr === 'string') return stderr++ return ''+ }+}+export type YoutubeDLSubs = {language: stringfilename: string@@ -22,7 +78,7 @@ const processOptions = {maxBuffer: 1024 * 1024 * 30 // 30MB}-class YoutubeDLWrapper {+export class YoutubeDLWrapper {constructor (private readonly url: string,private readonly enabledResolutions: VideoResolutionType[],@@ -30,7 +86,12 @@ class YoutubeDLWrapper {) {}- async getInfoForDownload (youtubeDLArgs: string[] = []): Promise<YoutubeDLInfo> {+ async getInfoForDownload (options: {+ userLanguage: string+ youtubeDLArgs?: string[]+ }): Promise<YoutubeDLInfo> {+ const { userLanguage, youtubeDLArgs = [] } = options+const youtubeDL = await YoutubeDLCLI.safeGet()const info = await youtubeDL.getInfo({@@ -40,9 +101,19 @@ class YoutubeDLWrapper {processOptions})- if (!info) throw new Error(`YoutubeDL could not get info from ${this.url}`)+ if (!info) {+ throw new YoutubeDlImportError({+ message: t(`Cannot fetch information from import for URL {targetUrl}`, userLanguage, { targetUrl: this.url }),+ code: YoutubeDlImportErrorCode.FETCH_ERROR+ })+ }- if (info.is_live === true) throw new Error('Cannot download a live streaming.')+ if (info.is_live === true || [ 'is_live', 'post_live', 'is_upcoming' ].includes(info.live_status)) {+ throw new YoutubeDlImportError({+ message: t('Cannot download a live streaming for URL {targetUrl}', userLanguage, { targetUrl: this.url }),+ code: YoutubeDlImportErrorCode.IS_LIVE+ })+ }const infoBuilder = new YoutubeDLInfoBuilder(info)@@ -50,8 +121,11 @@ class YoutubeDLWrapper {}async getInfoForListImport (options: {+ userLanguage: stringlatestVideosCount?: number}) {+ const { userLanguage } = options+const youtubeDL = await YoutubeDLCLI.safeGet()const list = await youtubeDL.getListInfo({@@ -60,7 +134,14 @@ class YoutubeDLWrapper {processOptions})- if (!Array.isArray(list)) throw new Error(`YoutubeDL could not get list info from ${this.url}: ${inspect(list)}`)+ if (!Array.isArray(list)) {+ logger.debug(`List info from youtube-dl is not an array for ${this.url}.`, { info: inspect(list), ...lTags() })++ throw new YoutubeDlImportError({+ message: t(`YoutubeDL could not get list info from ${this.url}: ${inspect(list)}`, userLanguage, { targetUrl: this.url }),+ code: YoutubeDlImportErrorCode.FETCH_ERROR+ })+ }return list.map(info => info.webpage_url)}@@ -114,13 +195,20 @@ class YoutubeDLWrapper {await move(pathWithoutExtension, pathWithoutExtension + '.mp4')}- return this.guessVideoPathWithExtension(pathWithoutExtension, fileExt)+ const path = await this.guessVideoPathWithExtension(pathWithoutExtension, fileExt)+ if (!path) {+ const directoryContent = await readdir(dirname(pathWithoutExtension))++ throw new Error(`Cannot guess path of ${pathWithoutExtension}. Directory content: ${directoryContent.join(', ')}`)+ }++ return path} catch (err) {this.guessVideoPathWithExtension(pathWithoutExtension, fileExt).then(path => {- logger.debug('Error in youtube-dl import, deleting file %s.', path, { err, ...lTags() })+ logger.debug('Error in youtube-dl import, deleting file if exists.', { err, pathToDelete: path, ...lTags() })- return remove(path)+ if (path) return remove(path)}).catch(innerErr => logger.error('Cannot remove file in youtubeDL error.', { innerErr, ...lTags() }))@@ -141,14 +229,6 @@ class YoutubeDLWrapper {if (await pathExists(path)) return path}- const directoryContent = await readdir(dirname(tmpPath))-- throw new Error(`Cannot guess path of ${tmpPath}. Directory content: ${directoryContent.join(', ')}`)+ return undefined}}--// -----------------------------------------------------------------------------export {- YoutubeDLWrapper-}diff --git a/server/core/lib/job-queue/handlers/after-video-channel-import.ts b/server/core/lib/job-queue/handlers/after-video-channel-import.tsindex 91d889dcb0..58b80b8273 100644--- a/server/core/lib/job-queue/handlers/after-video-channel-import.ts+++ b/server/core/lib/job-queue/handlers/after-video-channel-import.ts@@ -1,7 +1,9 @@-import { Job } from 'bullmq'-import { logger } from '@server/helpers/logger.js'-import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync.js'import { AfterVideoChannelImportPayload, VideoChannelSyncState, VideoImportPreventExceptionResult } from '@peertube/peertube-models'+import { logger, loggerTagsFactory } from '@server/helpers/logger.js'+import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync.js'+import { Job } from 'bullmq'++const lTags = loggerTagsFactory('channel-synchronization')export async function processAfterVideoChannelImport (job: Job) {const payload = job.data as AfterVideoChannelImportPayload@@ -17,7 +19,7 @@ export async function processAfterVideoChannelImport (job: Job) {const childrenValues = await job.getChildrenValues<VideoImportPreventExceptionResult>()- let errors = 0+ let errors = payload.buildJobErrors || 0let successes = 0for (const value of Object.values(childrenValues)) {@@ -27,10 +29,10 @@ export async function processAfterVideoChannelImport (job: Job) {if (errors > 0) {sync.state = VideoChannelSyncState.FAILED- logger.error(`Finished synchronizing "${sync.VideoChannel.Actor.preferredUsername}" with failures.`, { errors, successes })+ logger.error(`Finished synchronizing "${sync.VideoChannel.Actor.preferredUsername}" with failures.`, { errors, successes, ...lTags() })} else {sync.state = VideoChannelSyncState.SYNCED- logger.info(`Finished synchronizing "${sync.VideoChannel.Actor.preferredUsername}" successfully.`, { successes })+ logger.info(`Finished synchronizing "${sync.VideoChannel.Actor.preferredUsername}" successfully.`, { successes, ...lTags() })}await sync.save()diff --git a/server/core/lib/job-queue/job-queue.ts b/server/core/lib/job-queue/job-queue.tsindex 6ac3753a73..24459492c0 100644--- a/server/core/lib/job-queue/job-queue.ts+++ b/server/core/lib/job-queue/job-queue.ts@@ -409,7 +409,7 @@ class JobQueue {data: job.payload,queueName: job.type,opts: {- failParentOnFailure: true,+ failParentOnFailure: false,...this.buildJobOptions(job.type as JobType, pick(job, [ 'priority', 'delay', 'failParentOnFailure' ]))}diff --git a/server/core/lib/schedulers/video-channel-sync-latest-scheduler.ts b/server/core/lib/schedulers/video-channel-sync-latest-scheduler.tsindex 36ca2321a8..a1b2f26c85 100644--- a/server/core/lib/schedulers/video-channel-sync-latest-scheduler.ts+++ b/server/core/lib/schedulers/video-channel-sync-latest-scheduler.ts@@ -5,7 +5,6 @@ import { VideoChannelModel } from '@server/models/video/video-channel.js'import { VideoImportModel } from '@server/models/video/video-import.js'import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants.js'import { synchronizeChannel } from '../sync-channel.js'-import { retryImport } from '../video-post-import.js'import { AbstractScheduler } from './abstract-scheduler.js'const lTags = loggerTagsFactory('schedulers', 'channel-synchronization')@@ -31,33 +30,27 @@ export class VideoChannelSyncLatestScheduler extends AbstractScheduler {for (const sync of channelSyncs) {const channel = await VideoChannelModel.loadAndPopulateAccount(sync.videoChannelId)- logger.info(- `Creating video import jobs for "${channel.Actor.preferredUsername}" sync with external channel "${sync.externalChannelUrl}"`,- lTags()- )-// We can't rely on publication date for playlist elements// For example, an old video may have been added to a playlist since the last sync- const skipPublishedBefore = this.isPlaylistUrl(sync.externalChannelUrl)- ? undefined- : sync.lastSyncAt || sync.createdAt+ let skipPublishedBeforeOrEq: Date++ if (!this.isPlaylistUrl(sync.externalChannelUrl)) {+ const lastImport = await VideoImportModel.loadLastImportBySyncId({ channelSyncId: sync.id })++ if (lastImport && lastImport.Video?.originallyPublishedAt) {+ skipPublishedBeforeOrEq = lastImport.Video.originallyPublishedAt+ } else {+ skipPublishedBeforeOrEq = sync.lastSyncAt || sync.createdAt+ }+ }await synchronizeChannel({channel,externalChannelUrl: sync.externalChannelUrl,videosCountLimit: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.VIDEOS_LIMIT_PER_SYNCHRONIZATION,channelSync: sync,- skipPublishedBefore+ skipPublishedBeforeOrEq})-- const failed = await VideoImportModel.listFailedBySyncId({ channelSyncId: sync.id })- for (const videoImport of failed) {- logger.info(- `Retrying failed video import (id: ${videoImport.id}) for channel "${channel.Actor.preferredUsername}"`,- lTags()- )- await retryImport(videoImport)- }}}diff --git a/server/core/lib/sync-channel.ts b/server/core/lib/sync-channel.tsindex 71547aa4ef..07a304da23 100644--- a/server/core/lib/sync-channel.ts+++ b/server/core/lib/sync-channel.ts@@ -1,6 +1,6 @@import { VideoChannelSyncState } from '@peertube/peertube-models'-import { logger, loggerTagsFactory } from '@server/helpers/logger.js'-import { YoutubeDLWrapper } from '@server/helpers/youtube-dl/index.js'+import { logger, loggerTagsFactory, LoggerTagsFn } from '@server/helpers/logger.js'+import { YoutubeDlImportError, YoutubeDlImportErrorCode, YoutubeDLWrapper } from '@server/helpers/youtube-dl/index.js'import { CONFIG } from '@server/initializers/config.js'import { buildYoutubeDLImport } from '@server/lib/video-pre-import.js'import { UserModel } from '@server/models/user/user.js'@@ -8,18 +8,20 @@ import { VideoImportModel } from '@server/models/video/video-import.js'import { MChannelAccountDefault, MChannelSync } from '@server/types/models/index.js'import { CreateJobArgument, JobQueue } from './job-queue/index.js'import { ServerConfigManager } from './server-config-manager.js'+import { buildRetryImportJob } from './video-post-import.js'import { getLeastPrivatePrivacy } from './video.js'-const lTags = loggerTagsFactory('channel-synchronization')+const rootLTags = loggerTagsFactory('channel-synchronization')export async function synchronizeChannel (options: {channel: MChannelAccountDefaultexternalChannelUrl: stringvideosCountLimit: numberchannelSync?: MChannelSync- skipPublishedBefore?: Date+ skipPublishedBeforeOrEq?: Date}) {- const { channel, externalChannelUrl, videosCountLimit, skipPublishedBefore, channelSync } = options+ const { channel, externalChannelUrl, videosCountLimit, skipPublishedBeforeOrEq, channelSync } = options+ const channelUsername = channel.Actor.preferredUsernameif (channelSync) {channelSync.state = VideoChannelSyncState.PROCESSING@@ -35,37 +37,34 @@ export async function synchronizeChannel (options: {CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION)- const targetUrls = await youtubeDL.getInfoForListImport({ latestVideosCount: videosCountLimit })+ const lTags = loggerTagsFactory(...rootLTags().tags, channelSync.externalChannelUrl, channelUsername)++ const targetUrls = await youtubeDL.getInfoForListImport({+ userLanguage: user.getLanguage(),+ latestVideosCount: videosCountLimit+ })logger.info(- 'Fetched %d candidate URLs for sync channel %s.',- targetUrls.length,- channel.Actor.preferredUsername,+ `Fetched ${targetUrls.length} candidate URLs for sync channel ${channelSync.externalChannelUrl}.`,{ targetUrls, ...lTags() })- if (targetUrls.length === 0) {- if (channelSync) {- channelSync.state = VideoChannelSyncState.SYNCED- await channelSync.save()- }-- return- }-const children: CreateJobArgument[] = []+ let buildJobErrors = 0+for (const targetUrl of targetUrls) {logger.debug(`Import candidate: ${targetUrl}`, lTags())try {- if (await skipImport({ channel, channelSync, targetUrl, skipPublishedBefore })) continue+ if (await skipImport({ channel, channelSync, targetUrl, lTags })) continueconst { job } = await buildYoutubeDLImport({user,channel,targetUrl,channelSync,+ skipPublishedBeforeOrEq,importDataOverride: {privacy: getLeastPrivatePrivacy(),support: channel.support@@ -74,21 +73,49 @@ export async function synchronizeChannel (options: {children.push(job)} catch (err) {- logger.error(`Cannot build import for ${targetUrl} in channel ${channel.name}`, { err, ...lTags() })+ if (err instanceof YoutubeDlImportError) {+ if (+ err.code === YoutubeDlImportErrorCode.SKIP_PUBLICATION_DATE ||+ err.code === YoutubeDlImportErrorCode.IS_LIVE ||+ err.isUnavailableVideoError()+ ) {+ continue+ }++ if (err.isRateLimitError()) {+ logger.info(`Stopping synchronization due to rate limit error in channel ${channelUsername}.`, { err, ...lTags() })+ break+ }+ }++ buildJobErrors++++ logger.error(`Cannot build import for ${targetUrl} in channel ${channelUsername}`, { err, ...lTags() })}}+ const failed = await VideoImportModel.listFailedBySyncId({ channelSyncId: channelSync.id })+ for (const videoImport of failed) {+ logger.info(+ `Retrying failed video import (id: ${videoImport.id}) for channel "${channel.Actor.preferredUsername}"`,+ rootLTags()+ )++ children.push(await buildRetryImportJob(videoImport))+ }+// Will update the channel sync statusconst parent: CreateJobArgument = {type: 'after-video-channel-import',payload: {- channelSyncId: channelSync?.id+ channelSyncId: channelSync?.id,+ buildJobErrors}}await JobQueue.Instance.createJobWithChildren(parent, children)} catch (err) {- logger.error(`Failed to import ${externalChannelUrl} in channel ${channel.name}`, { err, ...lTags() })+ logger.error(`Failed to import ${externalChannelUrl} in channel ${channelUsername}`, { err, ...rootLTags() })channelSync.state = VideoChannelSyncState.FAILEDawait channelSync.save()}@@ -100,33 +127,17 @@ async function skipImport (options: {channel: MChannelAccountDefaultchannelSync: MChannelSynctargetUrl: string- skipPublishedBefore?: Date+ lTags: LoggerTagsFn}) {- const { channel, channelSync, targetUrl, skipPublishedBefore } = options+ const { channel, channelSync, targetUrl, lTags } = optionsif (await VideoImportModel.urlAlreadyImported({ channelId: channel.id, channelSyncId: channelSync?.id, targetUrl })) {logger.debug(`${targetUrl} is already imported for channel ${channel.name}, skipping video channel synchronization.`,{ channelSync, ...lTags() })- return true- }-- if (skipPublishedBefore) {- const youtubeDL = new YoutubeDLWrapper(- targetUrl,- ServerConfigManager.Instance.getEnabledResolutions('vod'),- CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION- )-- const videoInfo = await youtubeDL.getInfoForDownload()- const onlyAfterWithoutTime = new Date(skipPublishedBefore)- onlyAfterWithoutTime.setHours(0, 0, 0, 0)-- if (videoInfo.originallyPublishedAtWithoutTime.getTime() < onlyAfterWithoutTime.getTime()) {- return true- }+ return true}return falsediff --git a/server/core/lib/video-post-import.ts b/server/core/lib/video-post-import.tsindex 511a0ef536..2a6bc789ca 100644--- a/server/core/lib/video-post-import.ts+++ b/server/core/lib/video-post-import.ts@@ -1,9 +1,8 @@import { VideoImportPayload, VideoImportState, VideoImportYoutubeDLPayload } from '@peertube/peertube-models'import { CONFIG } from '@server/initializers/config.js'import { MVideoImport } from '@server/types/models/video/video-import.js'-import { JobQueue } from './job-queue/job-queue.js'-export async function retryImport (videoImport: MVideoImport) {+export async function buildRetryImportJob (videoImport: MVideoImport) {let type: VideoImportPayload['type']if (videoImport.magnetUri) type = 'magnet-uri'@@ -21,8 +20,8 @@ export async function retryImport (videoImport: MVideoImport) {fileExt: (videoImport.payload as VideoImportYoutubeDLPayload)?.fileExt}- await JobQueue.Instance.createJob({ type: 'video-import', payload })-videoImport.state = VideoImportState.PENDINGawait videoImport.save()++ return { type: 'video-import' as const, payload }}diff --git a/server/core/lib/video-pre-import.ts b/server/core/lib/video-pre-import.tsindex f24aaec5f8..63392383a6 100644--- a/server/core/lib/video-pre-import.ts+++ b/server/core/lib/video-pre-import.ts@@ -12,8 +12,9 @@ import {import { isVTTFileValid } from '@server/helpers/custom-validators/video-captions.js'import { isVideoFileExtnameValid } from '@server/helpers/custom-validators/videos.js'import { isResolvingToUnicastOnly } from '@server/helpers/dns.js'+import { guessLanguageFromReq, t } from '@server/helpers/i18n.js'import { logger } from '@server/helpers/logger.js'-import { YoutubeDLInfo, YoutubeDLWrapper } from '@server/helpers/youtube-dl/index.js'+import { YoutubeDlImportError, YoutubeDlImportErrorCode, YoutubeDLInfo, YoutubeDLWrapper } from '@server/helpers/youtube-dl/index.js'import { CONFIG } from '@server/initializers/config.js'import { sequelizeTypescript } from '@server/initializers/database.js'import { Hooks } from '@server/lib/plugins/hooks.js'@@ -37,38 +38,16 @@ import {MVideoThumbnail,MVideoWithBlacklistLight} from '@server/types/models/index.js'+import express from 'express'import { remove } from 'fs-extra/esm'import { getLocalVideoActivityPubUrl } from './activitypub/url.js'import { updateLocalVideoMiniatureFromExisting, updateLocalVideoMiniatureFromUrl } from './thumbnail.js'import { createLocalCaption } from './video-captions.js'import { replaceChapters, replaceChaptersFromDescriptionIfNeeded } from './video-chapters.js'-class YoutubeDlImportError extends Error {- code: YoutubeDlImportError.CODE- cause?: Error // Property to remove once ES2022 is used- constructor ({ message, code }) {- super(message)- this.code = code- }-- static fromError (err: Error, code: YoutubeDlImportError.CODE, message?: string) {- const ytDlErr = new this({ message: message ?? err.message, code })- ytDlErr.cause = err- ytDlErr.stack = err.stack // Useless once ES2022 is used- return ytDlErr- }-}--namespace YoutubeDlImportError {- export enum CODE {- FETCH_ERROR,- NOT_ONLY_UNICAST_URL- }-}-// ----------------------------------------------------------------------------async function insertFromImportIntoDB (parameters: {+export async function insertFromImportIntoDB (parameters: {video: MVideoThumbnailthumbnailModel: MThumbnailpreviewModel: MThumbnail@@ -129,7 +108,7 @@ async function insertFromImportIntoDB (parameters: {return videoImport}-async function buildVideoFromImport ({ channelId, importData, importDataOverride, importType }: {+export async function buildVideoFromImport ({ channelId, importData, importDataOverride, importType }: {channelId: numberimportData: YoutubeDLInfoimportDataOverride?: Partial<VideoImportCreate>@@ -171,16 +150,37 @@ async function buildVideoFromImport ({ channelId, importData, importDataOverridereturn video}-async function buildYoutubeDLImport (options: {+export async function buildYoutubeDLImport (options: {targetUrl: stringchannel: MChannelAccountDefaultuser: MUserAccountId+channelSync?: MChannelSyncimportDataOverride?: Partial<VideoImportCreate>thumbnailFilePath?: stringpreviewFilePath?: string++ skipPublishedBeforeOrEq?: Date++ req?: express.Request+ res?: express.Response}) {- const { targetUrl, channel, channelSync, importDataOverride, thumbnailFilePath, previewFilePath, user } = options+ const {+ targetUrl,+ channel,+ channelSync,+ importDataOverride,+ thumbnailFilePath,+ previewFilePath,+ user,+ skipPublishedBeforeOrEq,+ req,+ res+ } = options++ const userLanguage = req && res+ ? guessLanguageFromReq(req, res)+ : user.getLanguage()const youtubeDL = new YoutubeDLWrapper(targetUrl,@@ -189,21 +189,27 @@ async function buildYoutubeDLImport (options: {)// Get video infos- let youtubeDLInfo: YoutubeDLInfo- try {- youtubeDLInfo = await youtubeDL.getInfoForDownload()- } catch (err) {- throw YoutubeDlImportError.fromError(- err,- YoutubeDlImportError.CODE.FETCH_ERROR,- `Cannot fetch information from import for URL ${targetUrl}`- )+ const youtubeDLInfo = await youtubeDL.getInfoForDownload({ userLanguage })++ if (skipPublishedBeforeOrEq) {+ const onlyAfterWithoutTime = new Date(skipPublishedBeforeOrEq)+ onlyAfterWithoutTime.setHours(0, 0, 0, 0)++ if (youtubeDLInfo.originallyPublishedAtWithoutTime.getTime() < onlyAfterWithoutTime.getTime()) {+ throw new YoutubeDlImportError({+ message: t(`Video originally published at {publishedAt} is before the limit of {limit}`, userLanguage, {+ publishedAt: youtubeDLInfo.originallyPublishedAtWithoutTime.toLocaleString(userLanguage),+ limit: onlyAfterWithoutTime.toLocaleString(userLanguage)+ }),+ code: YoutubeDlImportErrorCode.SKIP_PUBLICATION_DATE+ })+ }}if (!await hasUnicastURLsOnly(youtubeDLInfo)) {throw new YoutubeDlImportError({- message: 'Cannot use non unicast IP as targetUrl.',- code: YoutubeDlImportError.CODE.NOT_ONLY_UNICAST_URL+ message: t('Cannot use non unicast IP as targetUrl.', userLanguage),+ code: YoutubeDlImportErrorCode.NOT_ONLY_UNICAST_URL})}@@ -292,9 +298,7 @@ async function buildYoutubeDLImport (options: {}// -----------------------------------------------------------------------------export { buildVideoFromImport, buildYoutubeDLImport, insertFromImportIntoDB, YoutubeDlImportError }-+// Private// ---------------------------------------------------------------------------async function forgeThumbnail ({ inputPath, video, downloadUrl, type }: {diff --git a/server/core/models/user/user.ts b/server/core/models/user/user.tsindex b762d64b9a..1ed51f0a41 100644--- a/server/core/models/user/user.ts+++ b/server/core/models/user/user.ts@@ -14,6 +14,7 @@ import {type UserRoleType} from '@peertube/peertube-models'import { isNSFWFlagsValid } from '@server/helpers/custom-validators/videos.js'+import { englishLanguage } from '@server/helpers/i18n.js'import { CONFIG } from '@server/initializers/config.js'import { TokensCache } from '@server/lib/auth/tokens-cache.js'import { LiveQuotaStore } from '@server/lib/live/index.js'@@ -983,7 +984,7 @@ export class UserModel extends SequelizeModel<UserModel> {}getLanguage () {- return this.language || CONFIG.INSTANCE.DEFAULT_LANGUAGE+ return this.language || CONFIG.INSTANCE.DEFAULT_LANGUAGE || englishLanguage}toFormattedJSON (this: MUserFormattable, parameters: { withAdminFlags?: boolean } = {}): User {diff --git a/server/core/models/video/video-import.ts b/server/core/models/video/video-import.tsindex 97083fe5e3..ebd71fe9c9 100644--- a/server/core/models/video/video-import.ts+++ b/server/core/models/video/video-import.ts@@ -1,4 +1,5 @@import { VideoImport, type VideoImportPayload, VideoImportState, type VideoImportStateType } from '@peertube/peertube-models'+import { CONFIG } from '@server/initializers/config.js'import { MVideoImport, MVideoImportDefault, MVideoImportFormattable } from '@server/types/models/video/video-import.js'import { Op } from 'sequelize'import {@@ -17,12 +18,11 @@ import {import { isVideoImportStateValid, isVideoImportTargetUrlValid } from '../../helpers/custom-validators/video-imports.js'import { isVideoMagnetUriValid } from '../../helpers/custom-validators/videos.js'import { CONSTRAINTS_FIELDS, VIDEO_IMPORT_STATES } from '../../initializers/constants.js'-import { buildSQLAttributes, SequelizeModel, throwIfNotValid } from '../shared/index.js'+import { buildSQLAttributes, getSort, SequelizeModel, throwIfNotValid } from '../shared/index.js'import { UserModel } from '../user/user.js'import { ListVideoImportsOptions, VideoImportListQueryBuilder } from './sql/import/video-import-list-query-builder.js'import { VideoChannelSyncModel } from './video-channel-sync.js'import { VideoModel, ScopeNames as VideoModelScopeNames } from './video.js'-import { CONFIG } from '@server/initializers/config.js'const defaultVideoScope = () => {return VideoModel.scope([@@ -223,6 +223,17 @@ export class VideoImportModel extends SequelizeModel<VideoImportModel> {})}+ static loadLastImportBySyncId (options: {+ channelSyncId: number+ }) {+ return VideoImportModel.findOne<MVideoImportDefault>({+ where: {+ videoChannelSyncId: options.channelSyncId+ },+ order: getSort('-createdAt')+ })+ }+// ---------------------------------------------------------------------------getTargetIdentifier () {