Skip to content

ActivityPub-federated video streaming platform using P2P directly in your web browser

Fix fetching latest elements of a playlist

Chocobozzzb4df49b87f84
diff --git a/server/core/lib/schedulers/video-channel-sync-latest-scheduler.ts b/server/core/lib/schedulers/video-channel-sync-latest-scheduler.ts
index 2a3ad18b19..2273265ac5 100644
--- a/server/core/lib/schedulers/video-channel-sync-latest-scheduler.ts
+++ b/server/core/lib/schedulers/video-channel-sync-latest-scheduler.ts
@@ -29,17 +29,22 @@ export class VideoChannelSyncLatestScheduler extends AbstractScheduler {
logger.info(
'Creating video import jobs for "%s" sync with external channel "%s"',
- channel.Actor.preferredUsername, sync.externalChannelUrl
+ channel.Actor.preferredUsername,
+ sync.externalChannelUrl
)
- const onlyAfter = sync.lastSyncAt || sync.createdAt
+ // We can't rely on publication date for playlist elements
+ // For example, an old video may have been added to a playlist since the last sync
+ const skipPublishedBefore = this.isPlaylistUrl(sync.externalChannelUrl)
+ ? undefined
+ : sync.lastSyncAt || sync.createdAt
await synchronizeChannel({
channel,
externalChannelUrl: sync.externalChannelUrl,
videosCountLimit: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.VIDEOS_LIMIT_PER_SYNCHRONIZATION,
channelSync: sync,
- onlyAfter
+ skipPublishedBefore
})
}
}
@@ -47,4 +52,14 @@ export class VideoChannelSyncLatestScheduler extends AbstractScheduler {
static get Instance () {
return this.instance || (this.instance = new this())
}
+
+ private isPlaylistUrl (url: string): boolean {
+ const parsed = new URL(url)
+ const pathname = parsed.pathname.toLowerCase()
+
+ return pathname.startsWith('/playlist/') || // Dailymotion playlist
+ pathname.startsWith('/showcase/') || // Vimeo playlist
+ pathname.startsWith('/playlist?') || // YouTube playlist
+ pathname.startsWith('/w/p/') // PeerTube playlist
+ }
}
diff --git a/server/core/lib/sync-channel.ts b/server/core/lib/sync-channel.ts
index d83a26043a..ad642e4810 100644
--- a/server/core/lib/sync-channel.ts
+++ b/server/core/lib/sync-channel.ts
@@ -16,9 +16,9 @@ export async function synchronizeChannel (options: {
externalChannelUrl: string
videosCountLimit: number
channelSync?: MChannelSync
- onlyAfter?: Date
+ skipPublishedBefore?: Date
}) {
- const { channel, externalChannelUrl, videosCountLimit, onlyAfter, channelSync } = options
+ const { channel, externalChannelUrl, videosCountLimit, skipPublishedBefore, channelSync } = options
if (channelSync) {
channelSync.state = VideoChannelSyncState.PROCESSING
@@ -58,7 +58,7 @@ export async function synchronizeChannel (options: {
logger.debug(`Import candidate: ${targetUrl}`, lTags())
try {
- if (await skipImport({ channel, channelSync, targetUrl, onlyAfter })) continue
+ if (await skipImport({ channel, channelSync, targetUrl, skipPublishedBefore })) continue
const { job } = await buildYoutubeDLImport({
user,
@@ -98,9 +98,9 @@ async function skipImport (options: {
channel: MChannelAccountDefault
channelSync: MChannelSync
targetUrl: string
- onlyAfter?: Date
+ skipPublishedBefore?: Date
}) {
- const { channel, channelSync, targetUrl, onlyAfter } = options
+ const { channel, channelSync, targetUrl, skipPublishedBefore } = options
if (await VideoImportModel.urlAlreadyImported({ channelId: channel.id, channelSyncId: channelSync?.id, targetUrl })) {
logger.debug(
@@ -110,7 +110,7 @@ async function skipImport (options: {
return true
}
- if (onlyAfter) {
+ if (skipPublishedBefore) {
const youtubeDL = new YoutubeDLWrapper(
targetUrl,
ServerConfigManager.Instance.getEnabledResolutions('vod'),
@@ -119,7 +119,7 @@ async function skipImport (options: {
const videoInfo = await youtubeDL.getInfoForDownload()
- const onlyAfterWithoutTime = new Date(onlyAfter)
+ const onlyAfterWithoutTime = new Date(skipPublishedBefore)
onlyAfterWithoutTime.setHours(0, 0, 0, 0)
if (videoInfo.originallyPublishedAtWithoutTime.getTime() < onlyAfterWithoutTime.getTime()) {