Skip to content

ActivityPub-federated video streaming platform using P2P directly in your web browser

Don't miss videos on full sync error

Chocobozzze70cd8f792af
diff --git a/server/core/initializers/migrations/1085-video-channel-sync-full-sync-cutoff.ts b/server/core/initializers/migrations/1085-video-channel-sync-full-sync-cutoff.ts
new file mode 100644
index 0000000000..eb68c89a85
--- /dev/null
+++ b/server/core/initializers/migrations/1085-video-channel-sync-full-sync-cutoff.ts
@@ -0,0 +1,24 @@
+import * as Sequelize from 'sequelize'
+
+async function up (utils: {
+ transaction: Sequelize.Transaction
+ queryInterface: Sequelize.QueryInterface
+ sequelize: Sequelize.Sequelize
+}): Promise<void> {
+ const { transaction } = utils
+
+ await utils.queryInterface.addColumn('videoChannelSync', 'fullSyncCutoffAt', {
+ type: Sequelize.DATE,
+ allowNull: true,
+ defaultValue: null
+ }, { transaction })
+}
+
+function down () {
+ throw new Error('Not implemented.')
+}
+
+export {
+ down,
+ up
+}
diff --git a/server/core/lib/schedulers/video-channel-sync-latest-scheduler.ts b/server/core/lib/schedulers/video-channel-sync-latest-scheduler.ts
index a1b2f26c85..3972215a4c 100644
--- a/server/core/lib/schedulers/video-channel-sync-latest-scheduler.ts
+++ b/server/core/lib/schedulers/video-channel-sync-latest-scheduler.ts
@@ -30,6 +30,24 @@ export class VideoChannelSyncLatestScheduler extends AbstractScheduler {
for (const sync of channelSyncs) {
const channel = await VideoChannelModel.loadAndPopulateAccount(sync.videoChannelId)
+ // A previous full run got interrupted: retry where we stopped to not miss videos
+ if (sync.fullSyncCutoffAt) {
+ logger.info(
+ `Channel sync ${sync.id} has an unfinished full sync (cutoff ${sync.fullSyncCutoffAt.toISOString()}), continuing it`,
+ lTags()
+ )
+
+ await synchronizeChannel({
+ channel,
+ externalChannelUrl: sync.externalChannelUrl,
+ videosCountLimit: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.FULL_SYNC_VIDEOS_LIMIT,
+ channelSync: sync,
+ skipPublishedBeforeOrEq: sync.fullSyncCutoffAt
+ })
+
+ continue
+ }
+
// We can't rely on publication date for playlist elements
// For example, an old video may have been added to a playlist since the last sync
let skipPublishedBeforeOrEq: Date
diff --git a/server/core/lib/sync-channel.ts b/server/core/lib/sync-channel.ts
index e17775cccc..76802677f8 100644
--- a/server/core/lib/sync-channel.ts
+++ b/server/core/lib/sync-channel.ts
@@ -55,6 +55,8 @@ export async function synchronizeChannel (options: {
const touchedVideoImportIds: number[] = []
let buildJobErrors = 0
+ let stoppedByRateLimit = false
+ let lastAttemptedPublishedAt: Date | undefined
for (const targetUrl of targetUrls) {
logger.debug(`Import candidate: ${targetUrl}`, lTags())
@@ -75,6 +77,10 @@ export async function synchronizeChannel (options: {
})
children.push(job)
+
+ if (videoImport.Video?.originallyPublishedAt) {
+ lastAttemptedPublishedAt = videoImport.Video.originallyPublishedAt
+ }
touchedVideoImportIds.push(videoImport.id)
} catch (err) {
if (err instanceof YoutubeDlImportError) {
@@ -90,6 +96,7 @@ export async function synchronizeChannel (options: {
if (err.isRateLimitError()) {
logger.info(`Stopping synchronization due to rate limit error in channel ${channelUsername}.`, { err, ...lTags() })
+ stoppedByRateLimit = true
break
}
}
@@ -100,8 +107,18 @@ export async function synchronizeChannel (options: {
}
}
- // Retry failed imports from this sync (if any)
if (channelSync) {
+ // Remember how far we got so we retry a full sync next time
+ if (stoppedByRateLimit) {
+ if (lastAttemptedPublishedAt) channelSync.fullSyncCutoffAt = lastAttemptedPublishedAt
+ } else {
+ // Not interrupted, reset full sync date
+ channelSync.fullSyncCutoffAt = null
+ }
+
+ await channelSync.save()
+
+ // Retry failed imports from this sync (if any)
const failed = await VideoImportModel.listFailedBySyncId({ channelSyncId: channelSync.id })
for (const videoImport of failed) {
logger.info(
diff --git a/server/core/models/video/video-channel-sync.ts b/server/core/models/video/video-channel-sync.ts
index 897180f13c..b8efbebcc8 100644
--- a/server/core/models/video/video-channel-sync.ts
+++ b/server/core/models/video/video-channel-sync.ts
@@ -60,6 +60,11 @@ export class VideoChannelSyncModel extends SequelizeModel<VideoChannelSyncModel>
@Column(DataType.DATE)
declare lastSyncAt: Date
+ // Publish date of the oldest video a full sync attempted to reach before being interrupted
+ @AllowNull(true)
+ @Column(DataType.DATE)
+ declare fullSyncCutoffAt: Date
+
@CreatedAt
declare createdAt: Date