Skip to content

ActivityPub-federated video streaming platform using P2P directly in your web browser

Optimize and more reliable channel sync

Chocobozzz73b6a95600ee
diff --git a/config/default.yaml b/config/default.yaml
index 5a95c32b97..b3be3bc3bc 100644
--- a/config/default.yaml
+++ b/config/default.yaml
@@ -811,7 +811,7 @@ import:
timeout: '2 hours'
# Number of attempts a user or the channel sync process can do to import a video if an error occurs during the import process
- max_attempts: 3
+ max_attempts: 5
# Classic HTTP or all sites supported by youtube-dl https://rg3.github.io/youtube-dl/supportedsites.html
http:
diff --git a/config/production.yaml.example b/config/production.yaml.example
index f45b868896..273d745f07 100644
--- a/config/production.yaml.example
+++ b/config/production.yaml.example
@@ -821,7 +821,7 @@ import:
timeout: '2 hours'
# Number of attempts a user or the channel sync process can do to import a video if an error occurs during the import process
- max_attempts: 3
+ max_attempts: 5
# Classic HTTP or all sites supported by youtube-dl https://rg3.github.io/youtube-dl/supportedsites.html
http:
diff --git a/packages/models/src/server/job.model.ts b/packages/models/src/server/job.model.ts
index c94b09d563..296753396c 100644
--- a/packages/models/src/server/job.model.ts
+++ b/packages/models/src/server/job.model.ts
@@ -281,6 +281,7 @@ export interface VideoChannelImportPayload {
export interface AfterVideoChannelImportPayload {
channelSyncId: number
+ buildJobErrors: number
}
// ---------------------------------------------------------------------------
diff --git a/packages/tests/src/api/videos/video-channel-syncs.ts b/packages/tests/src/api/videos/video-channel-syncs.ts
index cff1f33f35..daff8481aa 100644
--- a/packages/tests/src/api/videos/video-channel-syncs.ts
+++ b/packages/tests/src/api/videos/video-channel-syncs.ts
@@ -235,7 +235,7 @@ describe('Test channel synchronizations', function () {
await sqlCommands[0].setImportUrl(importId, 'http://fail.example.com')
await sqlCommands[0].setImportState(importId, VideoImportState.FAILED)
- for (let i = 2; i < 7; i++) {
+ for (let i = 2; i < 6; i++) {
await servers[0].debug.sendCommand({
body: {
command: 'process-video-channel-sync-latest'
@@ -246,11 +246,34 @@ describe('Test channel synchronizations', function () {
{
const { data } = await servers[0].videoImports.listMyVideoImports({ videoChannelSyncId: rootChannelSyncId })
- expect(data[0].state.id).to.equal(VideoImportState.FAILED)
- expect(data[0].error).to.include('fail.example.com')
- expect(data[0].attempts).to.equal(Math.min(3, i))
+ const failedImport = data.find(i => i.id === importId)
+
+ expect(failedImport.state.id).to.equal(VideoImportState.FAILED)
+ expect(failedImport.error).to.include('fail.example.com')
+ expect(failedImport.attempts).to.equal(i)
+ }
+
+ {
+ const { data } = await servers[0].channelSyncs.listByAccount({ accountName: 'root' })
+ const sync = data.find(s => s.id === rootChannelSyncId)
+ expect(sync.state.id).to.equal(VideoChannelSyncState.FAILED)
}
}
+
+ // Last one, sync stopped trying to import the failed job, so the sync is a success
+ await servers[0].debug.sendCommand({
+ body: {
+ command: 'process-video-channel-sync-latest'
+ }
+ })
+
+ await waitJobs(servers)
+
+ {
+ const { data } = await servers[0].channelSyncs.listByAccount({ accountName: 'root' })
+ const sync = data.find(s => s.id === rootChannelSyncId)
+ expect(sync.state.id).to.equal(VideoChannelSyncState.SYNCED)
+ }
})
// FIXME: youtube-dl/yt-dlp doesn't work when speicifying a port after the hostname
diff --git a/packages/tests/src/api/videos/video-imports.ts b/packages/tests/src/api/videos/video-imports.ts
index 12fa9687c8..06efa45ea3 100644
--- a/packages/tests/src/api/videos/video-imports.ts
+++ b/packages/tests/src/api/videos/video-imports.ts
@@ -124,8 +124,13 @@ describe('Test video imports', function () {
expect(video.name).to.equal('small video - youtube')
{
- expect(video.thumbnailPath).to.match(new RegExp(`^/lazy-static/thumbnails/.+.webp$`))
- expect(video.previewPath).to.match(new RegExp(`^/lazy-static/previews/.+.webp$`))
+ try {
+ expect(video.thumbnailPath).to.match(new RegExp(`^/lazy-static/thumbnails/.+.webp$`))
+ expect(video.previewPath).to.match(new RegExp(`^/lazy-static/previews/.+.webp$`))
+ } catch {
+ expect(video.thumbnailPath).to.match(new RegExp(`^/lazy-static/thumbnails/.+.jpg$`))
+ expect(video.previewPath).to.match(new RegExp(`^/lazy-static/previews/.+.jpg$`))
+ }
// FIXME: re-enable when we'll support webp pixel match
// const suffix = mode === 'yt-dlp'
diff --git a/server/core/controllers/api/videos/import.ts b/server/core/controllers/api/videos/import.ts
index 30506efa2b..315e241d0f 100644
--- a/server/core/controllers/api/videos/import.ts
+++ b/server/core/controllers/api/videos/import.ts
@@ -7,8 +7,9 @@ import {
VideoImportPayload,
VideoImportState
} from '@peertube/peertube-models'
-import { retryImport } from '@server/lib/video-post-import.js'
-import { buildVideoFromImport, buildYoutubeDLImport, insertFromImportIntoDB, YoutubeDlImportError } from '@server/lib/video-pre-import.js'
+import { YoutubeDlImportError, YoutubeDlImportErrorCode } from '@server/helpers/youtube-dl/youtube-dl-wrapper.js'
+import { buildRetryImportJob } from '@server/lib/video-post-import.js'
+import { buildVideoFromImport, buildYoutubeDLImport, insertFromImportIntoDB } from '@server/lib/video-pre-import.js'
import { MThumbnail, MVideoThumbnail } from '@server/types/models/index.js'
import express from 'express'
import { move } from 'fs-extra/esm'
@@ -100,7 +101,7 @@ async function cancelVideoImport (req: express.Request, res: express.Response) {
async function retryVideoImport (req: express.Request, res: express.Response) {
const videoImport = res.locals.videoImport
- await retryImport(videoImport)
+ await JobQueue.Instance.createJob(await buildRetryImportJob(videoImport))
return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
}
@@ -179,10 +180,11 @@ async function handleTorrentImport (req: express.Request, res: express.Response,
function statusFromYtDlImportError (err: YoutubeDlImportError): HttpStatusCodeType {
switch (err.code) {
- case YoutubeDlImportError.CODE.NOT_ONLY_UNICAST_URL:
+ case YoutubeDlImportErrorCode.NOT_ONLY_UNICAST_URL:
return HttpStatusCode.FORBIDDEN_403
- case YoutubeDlImportError.CODE.FETCH_ERROR:
+ case YoutubeDlImportErrorCode.FETCH_ERROR:
+ case YoutubeDlImportErrorCode.IS_LIVE:
return HttpStatusCode.BAD_REQUEST_400
default:
diff --git a/server/core/helpers/youtube-dl/youtube-dl-wrapper.ts b/server/core/helpers/youtube-dl/youtube-dl-wrapper.ts
index 38eaeb7d12..9ac43533b4 100644
--- a/server/core/helpers/youtube-dl/youtube-dl-wrapper.ts
+++ b/server/core/helpers/youtube-dl/youtube-dl-wrapper.ts
@@ -1,10 +1,12 @@
+import { VideoResolutionType } from '@peertube/peertube-models'
+import { CONFIG } from '@server/initializers/config.js'
+import { ExecaError } from 'execa'
import { move, pathExists, remove } from 'fs-extra/esm'
import { readdir } from 'fs/promises'
import { dirname, join } from 'path'
import { inspect } from 'util'
-import { VideoResolutionType } from '@peertube/peertube-models'
-import { CONFIG } from '@server/initializers/config.js'
import { isVideoFileExtnameValid } from '../custom-validators/videos.js'
+import { t } from '../i18n.js'
import { logger, loggerTagsFactory } from '../logger.js'
import { generateVideoImportTmpPath } from '../utils.js'
import { YoutubeDLCLI } from './youtube-dl-cli.js'
@@ -12,6 +14,60 @@ import { YoutubeDLInfo, YoutubeDLInfoBuilder } from './youtube-dl-info-builder.j
const lTags = loggerTagsFactory('youtube-dl')
+export const YoutubeDlImportErrorCode = {
+ FETCH_ERROR: 0,
+ NOT_ONLY_UNICAST_URL: 1,
+ SKIP_PUBLICATION_DATE: 2,
+ IS_LIVE: 3
+}
+
+export type YoutubeDlImportErrorCodeType = typeof YoutubeDlImportErrorCode[keyof typeof YoutubeDlImportErrorCode]
+
+export class YoutubeDlImportError extends Error {
+ static fromError (err: Error, code: YoutubeDlImportErrorCodeType, message?: string) {
+ const ytDlErr = new this({ message: message ?? err.message, code })
+ ytDlErr.cause = err
+
+ return ytDlErr
+ }
+
+ code: YoutubeDlImportErrorCodeType
+
+ constructor ({ message, code }) {
+ super(message)
+
+ this.code = code
+ }
+
+ isUnavailableVideoError () {
+ const stderr = this.getStderr()
+
+ if (stderr.includes('Video unavailable') || stderr.includes(' 429 ')) {
+ return true
+ }
+
+ return false
+ }
+
+ isRateLimitError () {
+ const stderr = this.getStderr()
+
+ if (stderr.includes('Sign in to confirm you’re not a bot')) {
+ return true
+ }
+
+ return false
+ }
+
+ private getStderr () {
+ const stderr = (this.cause as ExecaError)?.stderr
+
+ if (typeof stderr === 'string') return stderr
+
+ return ''
+ }
+}
+
export type YoutubeDLSubs = {
language: string
filename: string
@@ -22,7 +78,7 @@ const processOptions = {
maxBuffer: 1024 * 1024 * 30 // 30MB
}
-class YoutubeDLWrapper {
+export class YoutubeDLWrapper {
constructor (
private readonly url: string,
private readonly enabledResolutions: VideoResolutionType[],
@@ -30,7 +86,12 @@ class YoutubeDLWrapper {
) {
}
- async getInfoForDownload (youtubeDLArgs: string[] = []): Promise<YoutubeDLInfo> {
+ async getInfoForDownload (options: {
+ userLanguage: string
+ youtubeDLArgs?: string[]
+ }): Promise<YoutubeDLInfo> {
+ const { userLanguage, youtubeDLArgs = [] } = options
+
const youtubeDL = await YoutubeDLCLI.safeGet()
const info = await youtubeDL.getInfo({
@@ -40,9 +101,19 @@ class YoutubeDLWrapper {
processOptions
})
- if (!info) throw new Error(`YoutubeDL could not get info from ${this.url}`)
+ if (!info) {
+ throw new YoutubeDlImportError({
+ message: t(`Cannot fetch information from import for URL {targetUrl}`, userLanguage, { targetUrl: this.url }),
+ code: YoutubeDlImportErrorCode.FETCH_ERROR
+ })
+ }
- if (info.is_live === true) throw new Error('Cannot download a live streaming.')
+ if (info.is_live === true || [ 'is_live', 'post_live', 'is_upcoming' ].includes(info.live_status)) {
+ throw new YoutubeDlImportError({
+ message: t('Cannot download a live streaming for URL {targetUrl}', userLanguage, { targetUrl: this.url }),
+ code: YoutubeDlImportErrorCode.IS_LIVE
+ })
+ }
const infoBuilder = new YoutubeDLInfoBuilder(info)
@@ -50,8 +121,11 @@ class YoutubeDLWrapper {
}
async getInfoForListImport (options: {
+ userLanguage: string
latestVideosCount?: number
}) {
+ const { userLanguage } = options
+
const youtubeDL = await YoutubeDLCLI.safeGet()
const list = await youtubeDL.getListInfo({
@@ -60,7 +134,14 @@ class YoutubeDLWrapper {
processOptions
})
- if (!Array.isArray(list)) throw new Error(`YoutubeDL could not get list info from ${this.url}: ${inspect(list)}`)
+ if (!Array.isArray(list)) {
+ logger.debug(`List info from youtube-dl is not an array for ${this.url}.`, { info: inspect(list), ...lTags() })
+
+ throw new YoutubeDlImportError({
+ message: t(`YoutubeDL could not get list info from ${this.url}: ${inspect(list)}`, userLanguage, { targetUrl: this.url }),
+ code: YoutubeDlImportErrorCode.FETCH_ERROR
+ })
+ }
return list.map(info => info.webpage_url)
}
@@ -114,13 +195,20 @@ class YoutubeDLWrapper {
await move(pathWithoutExtension, pathWithoutExtension + '.mp4')
}
- return this.guessVideoPathWithExtension(pathWithoutExtension, fileExt)
+ const path = await this.guessVideoPathWithExtension(pathWithoutExtension, fileExt)
+ if (!path) {
+ const directoryContent = await readdir(dirname(pathWithoutExtension))
+
+ throw new Error(`Cannot guess path of ${pathWithoutExtension}. Directory content: ${directoryContent.join(', ')}`)
+ }
+
+ return path
} catch (err) {
this.guessVideoPathWithExtension(pathWithoutExtension, fileExt)
.then(path => {
- logger.debug('Error in youtube-dl import, deleting file %s.', path, { err, ...lTags() })
+ logger.debug('Error in youtube-dl import, deleting file if exists.', { err, pathToDelete: path, ...lTags() })
- return remove(path)
+ if (path) return remove(path)
})
.catch(innerErr => logger.error('Cannot remove file in youtubeDL error.', { innerErr, ...lTags() }))
@@ -141,14 +229,6 @@ class YoutubeDLWrapper {
if (await pathExists(path)) return path
}
- const directoryContent = await readdir(dirname(tmpPath))
-
- throw new Error(`Cannot guess path of ${tmpPath}. Directory content: ${directoryContent.join(', ')}`)
+ return undefined
}
}
-
-// ---------------------------------------------------------------------------
-
-export {
- YoutubeDLWrapper
-}
diff --git a/server/core/lib/job-queue/handlers/after-video-channel-import.ts b/server/core/lib/job-queue/handlers/after-video-channel-import.ts
index 91d889dcb0..58b80b8273 100644
--- a/server/core/lib/job-queue/handlers/after-video-channel-import.ts
+++ b/server/core/lib/job-queue/handlers/after-video-channel-import.ts
@@ -1,7 +1,9 @@
-import { Job } from 'bullmq'
-import { logger } from '@server/helpers/logger.js'
-import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync.js'
import { AfterVideoChannelImportPayload, VideoChannelSyncState, VideoImportPreventExceptionResult } from '@peertube/peertube-models'
+import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
+import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync.js'
+import { Job } from 'bullmq'
+
+const lTags = loggerTagsFactory('channel-synchronization')
export async function processAfterVideoChannelImport (job: Job) {
const payload = job.data as AfterVideoChannelImportPayload
@@ -17,7 +19,7 @@ export async function processAfterVideoChannelImport (job: Job) {
const childrenValues = await job.getChildrenValues<VideoImportPreventExceptionResult>()
- let errors = 0
+ let errors = payload.buildJobErrors || 0
let successes = 0
for (const value of Object.values(childrenValues)) {
@@ -27,10 +29,10 @@ export async function processAfterVideoChannelImport (job: Job) {
if (errors > 0) {
sync.state = VideoChannelSyncState.FAILED
- logger.error(`Finished synchronizing "${sync.VideoChannel.Actor.preferredUsername}" with failures.`, { errors, successes })
+ logger.error(`Finished synchronizing "${sync.VideoChannel.Actor.preferredUsername}" with failures.`, { errors, successes, ...lTags() })
} else {
sync.state = VideoChannelSyncState.SYNCED
- logger.info(`Finished synchronizing "${sync.VideoChannel.Actor.preferredUsername}" successfully.`, { successes })
+ logger.info(`Finished synchronizing "${sync.VideoChannel.Actor.preferredUsername}" successfully.`, { successes, ...lTags() })
}
await sync.save()
diff --git a/server/core/lib/job-queue/job-queue.ts b/server/core/lib/job-queue/job-queue.ts
index 6ac3753a73..24459492c0 100644
--- a/server/core/lib/job-queue/job-queue.ts
+++ b/server/core/lib/job-queue/job-queue.ts
@@ -409,7 +409,7 @@ class JobQueue {
data: job.payload,
queueName: job.type,
opts: {
- failParentOnFailure: true,
+ failParentOnFailure: false,
...this.buildJobOptions(job.type as JobType, pick(job, [ 'priority', 'delay', 'failParentOnFailure' ]))
}
diff --git a/server/core/lib/schedulers/video-channel-sync-latest-scheduler.ts b/server/core/lib/schedulers/video-channel-sync-latest-scheduler.ts
index 36ca2321a8..a1b2f26c85 100644
--- a/server/core/lib/schedulers/video-channel-sync-latest-scheduler.ts
+++ b/server/core/lib/schedulers/video-channel-sync-latest-scheduler.ts
@@ -5,7 +5,6 @@ import { VideoChannelModel } from '@server/models/video/video-channel.js'
import { VideoImportModel } from '@server/models/video/video-import.js'
import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants.js'
import { synchronizeChannel } from '../sync-channel.js'
-import { retryImport } from '../video-post-import.js'
import { AbstractScheduler } from './abstract-scheduler.js'
const lTags = loggerTagsFactory('schedulers', 'channel-synchronization')
@@ -31,33 +30,27 @@ export class VideoChannelSyncLatestScheduler extends AbstractScheduler {
for (const sync of channelSyncs) {
const channel = await VideoChannelModel.loadAndPopulateAccount(sync.videoChannelId)
- logger.info(
- `Creating video import jobs for "${channel.Actor.preferredUsername}" sync with external channel "${sync.externalChannelUrl}"`,
- lTags()
- )
-
// We can't rely on publication date for playlist elements
// For example, an old video may have been added to a playlist since the last sync
- const skipPublishedBefore = this.isPlaylistUrl(sync.externalChannelUrl)
- ? undefined
- : sync.lastSyncAt || sync.createdAt
+ let skipPublishedBeforeOrEq: Date
+
+ if (!this.isPlaylistUrl(sync.externalChannelUrl)) {
+ const lastImport = await VideoImportModel.loadLastImportBySyncId({ channelSyncId: sync.id })
+
+ if (lastImport && lastImport.Video?.originallyPublishedAt) {
+ skipPublishedBeforeOrEq = lastImport.Video.originallyPublishedAt
+ } else {
+ skipPublishedBeforeOrEq = sync.lastSyncAt || sync.createdAt
+ }
+ }
await synchronizeChannel({
channel,
externalChannelUrl: sync.externalChannelUrl,
videosCountLimit: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.VIDEOS_LIMIT_PER_SYNCHRONIZATION,
channelSync: sync,
- skipPublishedBefore
+ skipPublishedBeforeOrEq
})
-
- const failed = await VideoImportModel.listFailedBySyncId({ channelSyncId: sync.id })
- for (const videoImport of failed) {
- logger.info(
- `Retrying failed video import (id: ${videoImport.id}) for channel "${channel.Actor.preferredUsername}"`,
- lTags()
- )
- await retryImport(videoImport)
- }
}
}
diff --git a/server/core/lib/sync-channel.ts b/server/core/lib/sync-channel.ts
index 71547aa4ef..07a304da23 100644
--- a/server/core/lib/sync-channel.ts
+++ b/server/core/lib/sync-channel.ts
@@ -1,6 +1,6 @@
import { VideoChannelSyncState } from '@peertube/peertube-models'
-import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
-import { YoutubeDLWrapper } from '@server/helpers/youtube-dl/index.js'
+import { logger, loggerTagsFactory, LoggerTagsFn } from '@server/helpers/logger.js'
+import { YoutubeDlImportError, YoutubeDlImportErrorCode, YoutubeDLWrapper } from '@server/helpers/youtube-dl/index.js'
import { CONFIG } from '@server/initializers/config.js'
import { buildYoutubeDLImport } from '@server/lib/video-pre-import.js'
import { UserModel } from '@server/models/user/user.js'
@@ -8,18 +8,20 @@ import { VideoImportModel } from '@server/models/video/video-import.js'
import { MChannelAccountDefault, MChannelSync } from '@server/types/models/index.js'
import { CreateJobArgument, JobQueue } from './job-queue/index.js'
import { ServerConfigManager } from './server-config-manager.js'
+import { buildRetryImportJob } from './video-post-import.js'
import { getLeastPrivatePrivacy } from './video.js'
-const lTags = loggerTagsFactory('channel-synchronization')
+const rootLTags = loggerTagsFactory('channel-synchronization')
export async function synchronizeChannel (options: {
channel: MChannelAccountDefault
externalChannelUrl: string
videosCountLimit: number
channelSync?: MChannelSync
- skipPublishedBefore?: Date
+ skipPublishedBeforeOrEq?: Date
}) {
- const { channel, externalChannelUrl, videosCountLimit, skipPublishedBefore, channelSync } = options
+ const { channel, externalChannelUrl, videosCountLimit, skipPublishedBeforeOrEq, channelSync } = options
+ const channelUsername = channel.Actor.preferredUsername
if (channelSync) {
channelSync.state = VideoChannelSyncState.PROCESSING
@@ -35,37 +37,34 @@ export async function synchronizeChannel (options: {
CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION
)
- const targetUrls = await youtubeDL.getInfoForListImport({ latestVideosCount: videosCountLimit })
+ const lTags = loggerTagsFactory(...rootLTags().tags, channelSync.externalChannelUrl, channelUsername)
+
+ const targetUrls = await youtubeDL.getInfoForListImport({
+ userLanguage: user.getLanguage(),
+ latestVideosCount: videosCountLimit
+ })
logger.info(
- 'Fetched %d candidate URLs for sync channel %s.',
- targetUrls.length,
- channel.Actor.preferredUsername,
+ `Fetched ${targetUrls.length} candidate URLs for sync channel ${channelSync.externalChannelUrl}.`,
{ targetUrls, ...lTags() }
)
- if (targetUrls.length === 0) {
- if (channelSync) {
- channelSync.state = VideoChannelSyncState.SYNCED
- await channelSync.save()
- }
-
- return
- }
-
const children: CreateJobArgument[] = []
+ let buildJobErrors = 0
+
for (const targetUrl of targetUrls) {
logger.debug(`Import candidate: ${targetUrl}`, lTags())
try {
- if (await skipImport({ channel, channelSync, targetUrl, skipPublishedBefore })) continue
+ if (await skipImport({ channel, channelSync, targetUrl, lTags })) continue
const { job } = await buildYoutubeDLImport({
user,
channel,
targetUrl,
channelSync,
+ skipPublishedBeforeOrEq,
importDataOverride: {
privacy: getLeastPrivatePrivacy(),
support: channel.support
@@ -74,21 +73,49 @@ export async function synchronizeChannel (options: {
children.push(job)
} catch (err) {
- logger.error(`Cannot build import for ${targetUrl} in channel ${channel.name}`, { err, ...lTags() })
+ if (err instanceof YoutubeDlImportError) {
+ if (
+ err.code === YoutubeDlImportErrorCode.SKIP_PUBLICATION_DATE ||
+ err.code === YoutubeDlImportErrorCode.IS_LIVE ||
+ err.isUnavailableVideoError()
+ ) {
+ continue
+ }
+
+ if (err.isRateLimitError()) {
+ logger.info(`Stopping synchronization due to rate limit error in channel ${channelUsername}.`, { err, ...lTags() })
+ break
+ }
+ }
+
+ buildJobErrors++
+
+ logger.error(`Cannot build import for ${targetUrl} in channel ${channelUsername}`, { err, ...lTags() })
}
}
+ const failed = await VideoImportModel.listFailedBySyncId({ channelSyncId: channelSync.id })
+ for (const videoImport of failed) {
+ logger.info(
+ `Retrying failed video import (id: ${videoImport.id}) for channel "${channel.Actor.preferredUsername}"`,
+ rootLTags()
+ )
+
+ children.push(await buildRetryImportJob(videoImport))
+ }
+
// Will update the channel sync status
const parent: CreateJobArgument = {
type: 'after-video-channel-import',
payload: {
- channelSyncId: channelSync?.id
+ channelSyncId: channelSync?.id,
+ buildJobErrors
}
}
await JobQueue.Instance.createJobWithChildren(parent, children)
} catch (err) {
- logger.error(`Failed to import ${externalChannelUrl} in channel ${channel.name}`, { err, ...lTags() })
+ logger.error(`Failed to import ${externalChannelUrl} in channel ${channelUsername}`, { err, ...rootLTags() })
channelSync.state = VideoChannelSyncState.FAILED
await channelSync.save()
}
@@ -100,33 +127,17 @@ async function skipImport (options: {
channel: MChannelAccountDefault
channelSync: MChannelSync
targetUrl: string
- skipPublishedBefore?: Date
+ lTags: LoggerTagsFn
}) {
- const { channel, channelSync, targetUrl, skipPublishedBefore } = options
+ const { channel, channelSync, targetUrl, lTags } = options
if (await VideoImportModel.urlAlreadyImported({ channelId: channel.id, channelSyncId: channelSync?.id, targetUrl })) {
logger.debug(
`${targetUrl} is already imported for channel ${channel.name}, skipping video channel synchronization.`,
{ channelSync, ...lTags() }
)
- return true
- }
-
- if (skipPublishedBefore) {
- const youtubeDL = new YoutubeDLWrapper(
- targetUrl,
- ServerConfigManager.Instance.getEnabledResolutions('vod'),
- CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION
- )
-
- const videoInfo = await youtubeDL.getInfoForDownload()
- const onlyAfterWithoutTime = new Date(skipPublishedBefore)
- onlyAfterWithoutTime.setHours(0, 0, 0, 0)
-
- if (videoInfo.originallyPublishedAtWithoutTime.getTime() < onlyAfterWithoutTime.getTime()) {
- return true
- }
+ return true
}
return false
diff --git a/server/core/lib/video-post-import.ts b/server/core/lib/video-post-import.ts
index 511a0ef536..2a6bc789ca 100644
--- a/server/core/lib/video-post-import.ts
+++ b/server/core/lib/video-post-import.ts
@@ -1,9 +1,8 @@
import { VideoImportPayload, VideoImportState, VideoImportYoutubeDLPayload } from '@peertube/peertube-models'
import { CONFIG } from '@server/initializers/config.js'
import { MVideoImport } from '@server/types/models/video/video-import.js'
-import { JobQueue } from './job-queue/job-queue.js'
-export async function retryImport (videoImport: MVideoImport) {
+export async function buildRetryImportJob (videoImport: MVideoImport) {
let type: VideoImportPayload['type']
if (videoImport.magnetUri) type = 'magnet-uri'
@@ -21,8 +20,8 @@ export async function retryImport (videoImport: MVideoImport) {
fileExt: (videoImport.payload as VideoImportYoutubeDLPayload)?.fileExt
}
- await JobQueue.Instance.createJob({ type: 'video-import', payload })
-
videoImport.state = VideoImportState.PENDING
await videoImport.save()
+
+ return { type: 'video-import' as const, payload }
}
diff --git a/server/core/lib/video-pre-import.ts b/server/core/lib/video-pre-import.ts
index f24aaec5f8..63392383a6 100644
--- a/server/core/lib/video-pre-import.ts
+++ b/server/core/lib/video-pre-import.ts
@@ -12,8 +12,9 @@ import {
import { isVTTFileValid } from '@server/helpers/custom-validators/video-captions.js'
import { isVideoFileExtnameValid } from '@server/helpers/custom-validators/videos.js'
import { isResolvingToUnicastOnly } from '@server/helpers/dns.js'
+import { guessLanguageFromReq, t } from '@server/helpers/i18n.js'
import { logger } from '@server/helpers/logger.js'
-import { YoutubeDLInfo, YoutubeDLWrapper } from '@server/helpers/youtube-dl/index.js'
+import { YoutubeDlImportError, YoutubeDlImportErrorCode, YoutubeDLInfo, YoutubeDLWrapper } from '@server/helpers/youtube-dl/index.js'
import { CONFIG } from '@server/initializers/config.js'
import { sequelizeTypescript } from '@server/initializers/database.js'
import { Hooks } from '@server/lib/plugins/hooks.js'
@@ -37,38 +38,16 @@ import {
MVideoThumbnail,
MVideoWithBlacklistLight
} from '@server/types/models/index.js'
+import express from 'express'
import { remove } from 'fs-extra/esm'
import { getLocalVideoActivityPubUrl } from './activitypub/url.js'
import { updateLocalVideoMiniatureFromExisting, updateLocalVideoMiniatureFromUrl } from './thumbnail.js'
import { createLocalCaption } from './video-captions.js'
import { replaceChapters, replaceChaptersFromDescriptionIfNeeded } from './video-chapters.js'
-class YoutubeDlImportError extends Error {
- code: YoutubeDlImportError.CODE
- cause?: Error // Property to remove once ES2022 is used
- constructor ({ message, code }) {
- super(message)
- this.code = code
- }
-
- static fromError (err: Error, code: YoutubeDlImportError.CODE, message?: string) {
- const ytDlErr = new this({ message: message ?? err.message, code })
- ytDlErr.cause = err
- ytDlErr.stack = err.stack // Useless once ES2022 is used
- return ytDlErr
- }
-}
-
-namespace YoutubeDlImportError {
- export enum CODE {
- FETCH_ERROR,
- NOT_ONLY_UNICAST_URL
- }
-}
-
// ---------------------------------------------------------------------------
-async function insertFromImportIntoDB (parameters: {
+export async function insertFromImportIntoDB (parameters: {
video: MVideoThumbnail
thumbnailModel: MThumbnail
previewModel: MThumbnail
@@ -129,7 +108,7 @@ async function insertFromImportIntoDB (parameters: {
return videoImport
}
-async function buildVideoFromImport ({ channelId, importData, importDataOverride, importType }: {
+export async function buildVideoFromImport ({ channelId, importData, importDataOverride, importType }: {
channelId: number
importData: YoutubeDLInfo
importDataOverride?: Partial<VideoImportCreate>
@@ -171,16 +150,37 @@ async function buildVideoFromImport ({ channelId, importData, importDataOverride
return video
}
-async function buildYoutubeDLImport (options: {
+export async function buildYoutubeDLImport (options: {
targetUrl: string
channel: MChannelAccountDefault
user: MUserAccountId
+
channelSync?: MChannelSync
importDataOverride?: Partial<VideoImportCreate>
thumbnailFilePath?: string
previewFilePath?: string
+
+ skipPublishedBeforeOrEq?: Date
+
+ req?: express.Request
+ res?: express.Response
}) {
- const { targetUrl, channel, channelSync, importDataOverride, thumbnailFilePath, previewFilePath, user } = options
+ const {
+ targetUrl,
+ channel,
+ channelSync,
+ importDataOverride,
+ thumbnailFilePath,
+ previewFilePath,
+ user,
+ skipPublishedBeforeOrEq,
+ req,
+ res
+ } = options
+
+ const userLanguage = req && res
+ ? guessLanguageFromReq(req, res)
+ : user.getLanguage()
const youtubeDL = new YoutubeDLWrapper(
targetUrl,
@@ -189,21 +189,27 @@ async function buildYoutubeDLImport (options: {
)
// Get video infos
- let youtubeDLInfo: YoutubeDLInfo
- try {
- youtubeDLInfo = await youtubeDL.getInfoForDownload()
- } catch (err) {
- throw YoutubeDlImportError.fromError(
- err,
- YoutubeDlImportError.CODE.FETCH_ERROR,
- `Cannot fetch information from import for URL ${targetUrl}`
- )
+ const youtubeDLInfo = await youtubeDL.getInfoForDownload({ userLanguage })
+
+ if (skipPublishedBeforeOrEq) {
+ const onlyAfterWithoutTime = new Date(skipPublishedBeforeOrEq)
+ onlyAfterWithoutTime.setHours(0, 0, 0, 0)
+
+ if (youtubeDLInfo.originallyPublishedAtWithoutTime.getTime() < onlyAfterWithoutTime.getTime()) {
+ throw new YoutubeDlImportError({
+ message: t(`Video originally published at {publishedAt} is before the limit of {limit}`, userLanguage, {
+ publishedAt: youtubeDLInfo.originallyPublishedAtWithoutTime.toLocaleString(userLanguage),
+ limit: onlyAfterWithoutTime.toLocaleString(userLanguage)
+ }),
+ code: YoutubeDlImportErrorCode.SKIP_PUBLICATION_DATE
+ })
+ }
}
if (!await hasUnicastURLsOnly(youtubeDLInfo)) {
throw new YoutubeDlImportError({
- message: 'Cannot use non unicast IP as targetUrl.',
- code: YoutubeDlImportError.CODE.NOT_ONLY_UNICAST_URL
+ message: t('Cannot use non unicast IP as targetUrl.', userLanguage),
+ code: YoutubeDlImportErrorCode.NOT_ONLY_UNICAST_URL
})
}
@@ -292,9 +298,7 @@ async function buildYoutubeDLImport (options: {
}
// ---------------------------------------------------------------------------
-
-export { buildVideoFromImport, buildYoutubeDLImport, insertFromImportIntoDB, YoutubeDlImportError }
-
+// Private
// ---------------------------------------------------------------------------
async function forgeThumbnail ({ inputPath, video, downloadUrl, type }: {
diff --git a/server/core/models/user/user.ts b/server/core/models/user/user.ts
index b762d64b9a..1ed51f0a41 100644
--- a/server/core/models/user/user.ts
+++ b/server/core/models/user/user.ts
@@ -14,6 +14,7 @@ import {
type UserRoleType
} from '@peertube/peertube-models'
import { isNSFWFlagsValid } from '@server/helpers/custom-validators/videos.js'
+import { englishLanguage } from '@server/helpers/i18n.js'
import { CONFIG } from '@server/initializers/config.js'
import { TokensCache } from '@server/lib/auth/tokens-cache.js'
import { LiveQuotaStore } from '@server/lib/live/index.js'
@@ -983,7 +984,7 @@ export class UserModel extends SequelizeModel<UserModel> {
}
getLanguage () {
- return this.language || CONFIG.INSTANCE.DEFAULT_LANGUAGE
+ return this.language || CONFIG.INSTANCE.DEFAULT_LANGUAGE || englishLanguage
}
toFormattedJSON (this: MUserFormattable, parameters: { withAdminFlags?: boolean } = {}): User {
diff --git a/server/core/models/video/video-import.ts b/server/core/models/video/video-import.ts
index 97083fe5e3..ebd71fe9c9 100644
--- a/server/core/models/video/video-import.ts
+++ b/server/core/models/video/video-import.ts
@@ -1,4 +1,5 @@
import { VideoImport, type VideoImportPayload, VideoImportState, type VideoImportStateType } from '@peertube/peertube-models'
+import { CONFIG } from '@server/initializers/config.js'
import { MVideoImport, MVideoImportDefault, MVideoImportFormattable } from '@server/types/models/video/video-import.js'
import { Op } from 'sequelize'
import {
@@ -17,12 +18,11 @@ import {
import { isVideoImportStateValid, isVideoImportTargetUrlValid } from '../../helpers/custom-validators/video-imports.js'
import { isVideoMagnetUriValid } from '../../helpers/custom-validators/videos.js'
import { CONSTRAINTS_FIELDS, VIDEO_IMPORT_STATES } from '../../initializers/constants.js'
-import { buildSQLAttributes, SequelizeModel, throwIfNotValid } from '../shared/index.js'
+import { buildSQLAttributes, getSort, SequelizeModel, throwIfNotValid } from '../shared/index.js'
import { UserModel } from '../user/user.js'
import { ListVideoImportsOptions, VideoImportListQueryBuilder } from './sql/import/video-import-list-query-builder.js'
import { VideoChannelSyncModel } from './video-channel-sync.js'
import { VideoModel, ScopeNames as VideoModelScopeNames } from './video.js'
-import { CONFIG } from '@server/initializers/config.js'
const defaultVideoScope = () => {
return VideoModel.scope([
@@ -223,6 +223,17 @@ export class VideoImportModel extends SequelizeModel<VideoImportModel> {
})
}
+ static loadLastImportBySyncId (options: {
+ channelSyncId: number
+ }) {
+ return VideoImportModel.findOne<MVideoImportDefault>({
+ where: {
+ videoChannelSyncId: options.channelSyncId
+ },
+ order: getSort('-createdAt')
+ })
+ }
+
// ---------------------------------------------------------------------------
getTargetIdentifier () {