ActivityPub-federated video streaming platform using P2P directly in your web browser
Don't miss videos on full sync error
Chocobozzze70cd8f792af
diff --git a/server/core/initializers/migrations/1085-video-channel-sync-full-sync-cutoff.ts b/server/core/initializers/migrations/1085-video-channel-sync-full-sync-cutoff.tsnew file mode 100644index 0000000000..eb68c89a85--- /dev/null+++ b/server/core/initializers/migrations/1085-video-channel-sync-full-sync-cutoff.ts@@ -0,0 +1,24 @@+import * as Sequelize from 'sequelize'++async function up (utils: {+ transaction: Sequelize.Transaction+ queryInterface: Sequelize.QueryInterface+ sequelize: Sequelize.Sequelize+}): Promise<void> {+ const { transaction } = utils++ await utils.queryInterface.addColumn('videoChannelSync', 'fullSyncCutoffAt', {+ type: Sequelize.DATE,+ allowNull: true,+ defaultValue: null+ }, { transaction })+}++function down () {+ throw new Error('Not implemented.')+}++export {+ down,+ up+}diff --git a/server/core/lib/schedulers/video-channel-sync-latest-scheduler.ts b/server/core/lib/schedulers/video-channel-sync-latest-scheduler.tsindex a1b2f26c85..3972215a4c 100644--- a/server/core/lib/schedulers/video-channel-sync-latest-scheduler.ts+++ b/server/core/lib/schedulers/video-channel-sync-latest-scheduler.ts@@ -30,6 +30,24 @@ export class VideoChannelSyncLatestScheduler extends AbstractScheduler {for (const sync of channelSyncs) {const channel = await VideoChannelModel.loadAndPopulateAccount(sync.videoChannelId)+ // A previous full run got interrupted: retry where we stopped to not miss videos+ if (sync.fullSyncCutoffAt) {+ logger.info(+ `Channel sync ${sync.id} has an unfinished full sync (cutoff ${sync.fullSyncCutoffAt.toISOString()}), continuing it`,+ lTags()+ )++ await synchronizeChannel({+ channel,+ externalChannelUrl: sync.externalChannelUrl,+ videosCountLimit: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.FULL_SYNC_VIDEOS_LIMIT,+ channelSync: sync,+ skipPublishedBeforeOrEq: sync.fullSyncCutoffAt+ })++ continue+ }+// We can't rely on publication date for playlist elements// For example, an old video may have been added to a playlist since the last synclet skipPublishedBeforeOrEq: Datediff --git a/server/core/lib/sync-channel.ts b/server/core/lib/sync-channel.tsindex e17775cccc..76802677f8 100644--- a/server/core/lib/sync-channel.ts+++ b/server/core/lib/sync-channel.ts@@ -55,6 +55,8 @@ export async function synchronizeChannel (options: {const touchedVideoImportIds: number[] = []let buildJobErrors = 0+ let stoppedByRateLimit = false+ let lastAttemptedPublishedAt: Date | undefinedfor (const targetUrl of targetUrls) {logger.debug(`Import candidate: ${targetUrl}`, lTags())@@ -75,6 +77,10 @@ export async function synchronizeChannel (options: {})children.push(job)++ if (videoImport.Video?.originallyPublishedAt) {+ lastAttemptedPublishedAt = videoImport.Video.originallyPublishedAt+ }touchedVideoImportIds.push(videoImport.id)} catch (err) {if (err instanceof YoutubeDlImportError) {@@ -90,6 +96,7 @@ export async function synchronizeChannel (options: {if (err.isRateLimitError()) {logger.info(`Stopping synchronization due to rate limit error in channel ${channelUsername}.`, { err, ...lTags() })+ stoppedByRateLimit = truebreak}}@@ -100,8 +107,18 @@ export async function synchronizeChannel (options: {}}- // Retry failed imports from this sync (if any)if (channelSync) {+ // Remember how far we got so we retry a full sync next time+ if (stoppedByRateLimit) {+ if (lastAttemptedPublishedAt) channelSync.fullSyncCutoffAt = lastAttemptedPublishedAt+ } else {+ // Not interrupted, reset full sync date+ channelSync.fullSyncCutoffAt = null+ }++ await channelSync.save()++ // Retry failed imports from this sync (if any)const failed = await VideoImportModel.listFailedBySyncId({ channelSyncId: channelSync.id })for (const videoImport of failed) {logger.info(diff --git a/server/core/models/video/video-channel-sync.ts b/server/core/models/video/video-channel-sync.tsindex 897180f13c..b8efbebcc8 100644--- a/server/core/models/video/video-channel-sync.ts+++ b/server/core/models/video/video-channel-sync.ts@@ -60,6 +60,11 @@ export class VideoChannelSyncModel extends SequelizeModel<VideoChannelSyncModel>@Column(DataType.DATE)declare lastSyncAt: Date+ // Publish date of the oldest video a full sync attempted to reach before being interrupted+ @AllowNull(true)+ @Column(DataType.DATE)+ declare fullSyncCutoffAt: Date+@CreatedAtdeclare createdAt: Date