Skip to content

ActivityPub-federated video streaming platform using P2P directly in your web browser

Log when manual migration script hasn't been run

Chocobozzz83e263eaa9ef
diff --git a/server/core/initializers/constants.ts b/server/core/initializers/constants.ts
index e4e6f12bb5..56fe0d4879 100644
--- a/server/core/initializers/constants.ts
+++ b/server/core/initializers/constants.ts
@@ -62,7 +62,7 @@ import { CONFIG, registerConfigChangedHandler } from './config.js'
// ---------------------------------------------------------------------------
-export const LAST_MIGRATION_VERSION = 1115
+export const LAST_MIGRATION_VERSION = 1120
// ---------------------------------------------------------------------------
@@ -382,8 +382,20 @@ export const SCHEDULER_INTERVALS_MS = {
CHANNEL_SYNC_CHECK_INTERVAL: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.CHECK_INTERVAL,
BLOCKLIST_SUBSCRIPTIONS_SYNC: 60000 * 60, // 1 hour
WATCHED_WORDS_SUBSCRIPTIONS_SYNC: 60000 * 60, // 1 hour
- REMOVE_OLD_USER_LOGIN_DEVICES: 60000 * 60 * 24 // 1 day
-}
+ REMOVE_OLD_USER_LOGIN_DEVICES: 60000 * 60 * 24, // 1 day
+ CHECK_MANUAL_MIGRATION_SCRIPTS: 60000 * 60 * 12 // 12 hours
+}
+
+export const MANUAL_MIGRATION_SCRIPTS = [
+ 'peertube-4.0',
+ 'peertube-4.2',
+ 'peertube-5.0',
+ 'peertube-6.3',
+ 'peertube-7.2',
+ 'peertube-8.0',
+ 'peertube-8.1',
+ 'peertube-8.3'
+]
// Devices not seen again after this delay are forgotten, so a login from that IP/user-agent pair will be treated as new again
export const USER_LOGIN_DEVICE_MAX_AGE = 60000 * 60 * 24 * 365 // 1 year
diff --git a/server/core/initializers/installer.ts b/server/core/initializers/installer.ts
index 3d3571f833..f565c7780e 100644
--- a/server/core/initializers/installer.ts
+++ b/server/core/initializers/installer.ts
@@ -174,7 +174,17 @@ async function createApplicationIfNotExist () {
const application = await ApplicationModel.create({
migrationVersion: LAST_MIGRATION_VERSION,
nodeVersion: process.version,
- nodeABIVersion: getNodeABIVersion()
+ nodeABIVersion: getNodeABIVersion(),
+ // Scripts that existed before the manual migration tracking system was introduced: assume already run
+ manualMigrationScriptsRun: [
+ 'peertube-4.0',
+ 'peertube-4.2',
+ 'peertube-5.0',
+ 'peertube-6.3',
+ 'peertube-7.2',
+ 'peertube-8.0',
+ 'peertube-8.1'
+ ]
})
return createApplicationActor(application.id)
diff --git a/server/core/initializers/migrations/1120-manual-migration-scripts-run.ts b/server/core/initializers/migrations/1120-manual-migration-scripts-run.ts
new file mode 100644
index 0000000000..9f5ac2dc1f
--- /dev/null
+++ b/server/core/initializers/migrations/1120-manual-migration-scripts-run.ts
@@ -0,0 +1,32 @@
+import * as Sequelize from 'sequelize'
+
+async function up (utils: {
+ transaction: Sequelize.Transaction
+ queryInterface: Sequelize.QueryInterface
+ sequelize: Sequelize.Sequelize
+}): Promise<void> {
+ const { transaction } = utils
+
+ await utils.queryInterface.addColumn('application', 'manualMigrationScriptsRun', {
+ type: Sequelize.ARRAY(Sequelize.STRING),
+ allowNull: false,
+ defaultValue: []
+ }, { transaction })
+
+ // Scripts that existed before this tracking system was introduced: assume already run
+ const alreadyRun = [ 'peertube-4.0', 'peertube-4.2', 'peertube-5.0', 'peertube-6.3', 'peertube-7.2', 'peertube-8.0', 'peertube-8.1' ]
+
+ await utils.sequelize.query(
+ `UPDATE "application" SET "manualMigrationScriptsRun" = ARRAY[${alreadyRun.map(s => `'${s}'`).join(', ')}]`,
+ { transaction }
+ )
+}
+
+function down () {
+ throw new Error('Not implemented.')
+}
+
+export {
+ down,
+ up
+}
diff --git a/server/core/lib/schedulers/manual-migration-scripts-scheduler.ts b/server/core/lib/schedulers/manual-migration-scripts-scheduler.ts
new file mode 100644
index 0000000000..927971902e
--- /dev/null
+++ b/server/core/lib/schedulers/manual-migration-scripts-scheduler.ts
@@ -0,0 +1,37 @@
+import { ApplicationModel } from '@server/models/application/application.js'
+import { createLogger } from '../../helpers/logger.js'
+import { MANUAL_MIGRATION_SCRIPTS, SCHEDULER_INTERVALS_MS } from '../../initializers/constants.js'
+import { AbstractScheduler } from './abstract-scheduler.js'
+
+const logger = createLogger('schedulers')
+
+export class ManualMigrationScriptsScheduler extends AbstractScheduler {
+ private static instance: AbstractScheduler
+
+ protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.CHECK_MANUAL_MIGRATION_SCRIPTS
+
+ private constructor () {
+ super({ randomRunOnEnable: true })
+ }
+
+ protected async internalExecute () {
+ return this.checkMigrationScriptsRun()
+ }
+
+ private async checkMigrationScriptsRun () {
+ const application = await ApplicationModel.load()
+
+ const missing = MANUAL_MIGRATION_SCRIPTS.filter(script => !application.manualMigrationScriptsRun.includes(script))
+ if (missing.length === 0) return
+
+ logger.warn(
+ 'The following manual migration scripts have not been run yet: %s. ' +
+ 'Check the CHANGELOG for instructions on how to run them.',
+ missing.join(', ')
+ )
+ }
+
+ static get Instance () {
+ return this.instance || (this.instance = new this())
+ }
+}
diff --git a/server/core/models/application/application.ts b/server/core/models/application/application.ts
index c0d9726441..a625f1aca8 100644
--- a/server/core/models/application/application.ts
+++ b/server/core/models/application/application.ts
@@ -66,6 +66,11 @@ export class ApplicationModel extends SequelizeModel<ApplicationModel> {
@Column(DataType.JSONB)
declare configPart: ConfigPart
+ @AllowNull(false)
+ @Default([])
+ @Column(DataType.ARRAY(DataType.STRING))
+ declare manualMigrationScriptsRun: string[]
+
@HasOne(() => AccountModel, {
foreignKey: {
allowNull: true
@@ -100,6 +105,20 @@ export class ApplicationModel extends SequelizeModel<ApplicationModel> {
return configPart?.OBJECT_STORAGE.STREAMING_PLAYLISTS.BASE_URL !== CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.BASE_URL
}
+ static async hasManualMigrationScriptRun (scriptName: string) {
+ const application = await this.load()
+
+ return application.manualMigrationScriptsRun.includes(scriptName)
+ }
+
+ static async setManualMigrationScriptRun (scriptName: string) {
+ const application = await this.load()
+ if (application.manualMigrationScriptsRun.includes(scriptName)) return
+
+ application.manualMigrationScriptsRun = [ ...application.manualMigrationScriptsRun, scriptName ]
+ await application.save()
+ }
+
static async updateNodeVersionsOrConfig () {
const application = await this.load()
diff --git a/server/scripts/migrations/peertube-4.0.ts b/server/scripts/migrations/peertube-4.0.ts
index 0b4c04adfa..ed2fdefad8 100644
--- a/server/scripts/migrations/peertube-4.0.ts
+++ b/server/scripts/migrations/peertube-4.0.ts
@@ -7,6 +7,7 @@ import {
getHLSResolutionPlaylistFilename
} from '@server/lib/paths.js'
import { VideoPathManager } from '@server/lib/video-path-manager.js'
+import { ApplicationModel } from '@server/models/application/application.js'
import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist.js'
import { VideoModel } from '@server/models/video/video.js'
import Bluebird from 'bluebird'
@@ -14,6 +15,8 @@ import { move } from 'fs-extra/esm'
import { readFile, writeFile } from 'fs/promises'
import { join } from 'path'
+const MIGRATION_NAME = 'peertube-4.0'
+
run()
.then(() => process.exit(0))
.catch(err => {
@@ -22,10 +25,10 @@ run()
})
async function run () {
- console.log('Migrate old HLS paths to new format.')
-
await initDatabaseModels(true)
+ console.log('Migrate old HLS paths to new format.')
+
JobQueue.Instance.init()
const ids = await VideoModel.listLocalIds()
@@ -38,6 +41,8 @@ async function run () {
}
}, { concurrency: 5 })
+ await ApplicationModel.setManualMigrationScriptRun(MIGRATION_NAME)
+
console.log('Migration finished!')
}
diff --git a/server/scripts/migrations/peertube-4.2.ts b/server/scripts/migrations/peertube-4.2.ts
index 163ae3b3ab..1b0ffd29cd 100644
--- a/server/scripts/migrations/peertube-4.2.ts
+++ b/server/scripts/migrations/peertube-4.2.ts
@@ -10,10 +10,13 @@ import { sendUpdateActor } from '@server/lib/activitypub/send/index.js'
import { JobQueue } from '@server/lib/job-queue/index.js'
import { AccountModel } from '@server/models/account/account.js'
import { ActorModel } from '@server/models/actor/actor.js'
+import { ApplicationModel } from '@server/models/application/application.js'
import { VideoChannelModel } from '@server/models/video/video-channel.js'
import { MAccountDefault, MActorDefault, MChannelDefault } from '@server/types/models/index.js'
import { join } from 'path'
+const MIGRATION_NAME = 'peertube-4.2'
+
run()
.then(() => process.exit(0))
.catch(err => {
@@ -22,9 +25,10 @@ run()
})
async function run () {
+ await initDatabaseModels(true)
+
console.log('Generate avatar miniatures from existing avatars.')
- await initDatabaseModels(true)
JobQueue.Instance.init()
const accounts: AccountModel[] = await AccountModel.findAll({
@@ -65,6 +69,8 @@ async function run () {
}
}
+ await ApplicationModel.setManualMigrationScriptRun(MIGRATION_NAME)
+
console.log('Generation finished!')
}
diff --git a/server/scripts/migrations/peertube-5.0.ts b/server/scripts/migrations/peertube-5.0.ts
index 64505b133e..80184b6df5 100644
--- a/server/scripts/migrations/peertube-5.0.ts
+++ b/server/scripts/migrations/peertube-5.0.ts
@@ -3,11 +3,14 @@ import { DIRECTORIES } from '@server/initializers/constants.js'
import { initDatabaseModels } from '@server/initializers/database.js'
import { moveFilesIfPrivacyChanged } from '@server/lib/video-privacy.js'
import { updateTorrentForFileAndSave } from '@server/lib/webtorrent.js'
+import { ApplicationModel } from '@server/models/application/application.js'
import { VideoModel } from '@server/models/video/video.js'
import { MVideoFull } from '@server/types/models/index.js'
import { ensureDir } from 'fs-extra/esm'
import { Op } from 'sequelize'
+const MIGRATION_NAME = 'peertube-5.0'
+
run()
.then(() => process.exit(0))
.catch(err => {
@@ -16,13 +19,13 @@ run()
})
async function run () {
+ await initDatabaseModels(true)
+
console.log('Moving private video files in dedicated folders.')
await ensureDir(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE)
await ensureDir(DIRECTORIES.WEB_VIDEOS.PRIVATE)
- await initDatabaseModels(true)
-
const videos = await VideoModel.unscoped().findAll({
attributes: [ 'uuid' ],
where: {
@@ -53,6 +56,8 @@ async function run () {
console.error('Cannot process video %s.', uuid, err)
}
}
+
+ await ApplicationModel.setManualMigrationScriptRun(MIGRATION_NAME)
}
async function updateTorrents (video: MVideoFull) {
diff --git a/server/scripts/migrations/peertube-6.3.ts b/server/scripts/migrations/peertube-6.3.ts
index ad78ac7065..222bc89b73 100644
--- a/server/scripts/migrations/peertube-6.3.ts
+++ b/server/scripts/migrations/peertube-6.3.ts
@@ -3,12 +3,15 @@ import { VideoFileStream } from '@peertube/peertube-models'
import { initDatabaseModels, sequelizeTypescript } from '@server/initializers/database.js'
import { buildFileMetadata } from '@server/lib/video-file.js'
import { VideoPathManager } from '@server/lib/video-path-manager.js'
+import { ApplicationModel } from '@server/models/application/application.js'
import { VideoFileModel } from '@server/models/video/video-file.js'
import { VideoModel } from '@server/models/video/video.js'
import Bluebird from 'bluebird'
import { pathExists } from 'fs-extra/esm'
import { QueryTypes } from 'sequelize'
+const MIGRATION_NAME = 'peertube-6.3'
+
run()
.then(() => process.exit(0))
.catch(err => {
@@ -68,6 +71,8 @@ async function run () {
}
}, { concurrency: 5 })
+ await ApplicationModel.setManualMigrationScriptRun(MIGRATION_NAME)
+
console.log('\n## Migration finished! ##')
}
diff --git a/server/scripts/migrations/peertube-7.2.ts b/server/scripts/migrations/peertube-7.2.ts
index 0adc2b0754..615b2a5565 100644
--- a/server/scripts/migrations/peertube-7.2.ts
+++ b/server/scripts/migrations/peertube-7.2.ts
@@ -1,4 +1,7 @@
import { initDatabaseModels, sequelizeTypescript } from '@server/initializers/database.js'
+import { ApplicationModel } from '@server/models/application/application.js'
+
+const MIGRATION_NAME = 'peertube-7.2'
run()
.then(() => process.exit(0))
@@ -16,5 +19,7 @@ async function run () {
await sequelizeTypescript.query(query)
}
+ await ApplicationModel.setManualMigrationScriptRun(MIGRATION_NAME)
+
console.log('Done!')
}
diff --git a/server/scripts/migrations/peertube-8.0.ts b/server/scripts/migrations/peertube-8.0.ts
index 53e4baf98e..0b0cb720bc 100644
--- a/server/scripts/migrations/peertube-8.0.ts
+++ b/server/scripts/migrations/peertube-8.0.ts
@@ -2,6 +2,8 @@ import { initDatabaseModels, sequelizeTypescript } from '@server/initializers/da
import { ApplicationModel } from '@server/models/application/application.js'
import { QueryTypes } from 'sequelize'
+const MIGRATION_NAME = 'peertube-8.0'
+
run()
.then(() => process.exit(0))
.catch(err => {
@@ -41,5 +43,7 @@ async function run () {
i++
}
+ await ApplicationModel.setManualMigrationScriptRun(MIGRATION_NAME)
+
console.log('Done!')
}
diff --git a/server/scripts/migrations/peertube-8.1.ts b/server/scripts/migrations/peertube-8.1.ts
index d7d4b79e61..9d111a0f45 100644
--- a/server/scripts/migrations/peertube-8.1.ts
+++ b/server/scripts/migrations/peertube-8.1.ts
@@ -1,10 +1,13 @@
import { CONFIG } from '@server/initializers/config.js'
-import { sequelizeTypescript } from '@server/initializers/database.js'
+import { initDatabaseModels, sequelizeTypescript } from '@server/initializers/database.js'
+import { ApplicationModel } from '@server/models/application/application.js'
import Bluebird from 'bluebird'
import { move } from 'fs-extra'
import { readdir } from 'fs/promises'
import { join } from 'path'
+const MIGRATION_NAME = 'peertube-8.1'
+
run()
.then(() => process.exit(0))
.catch(err => {
@@ -13,6 +16,8 @@ run()
})
async function run () {
+ await initDatabaseModels(true)
+
try {
await movePreviewsToThumbnails()
} catch (err) {
@@ -48,6 +53,8 @@ async function run () {
} catch (err) {
console.error('An error occurred while updating local video caption URLs:', err)
}
+
+ await ApplicationModel.setManualMigrationScriptRun(MIGRATION_NAME)
}
async function movePreviewsToThumbnails () {
diff --git a/server/scripts/migrations/peertube-8.3.ts b/server/scripts/migrations/peertube-8.3.ts
index e6d03c3fde..0c5309f8bd 100644
--- a/server/scripts/migrations/peertube-8.3.ts
+++ b/server/scripts/migrations/peertube-8.3.ts
@@ -1,4 +1,7 @@
-import { sequelizeTypescript } from '@server/initializers/database.js'
+import { initDatabaseModels, sequelizeTypescript } from '@server/initializers/database.js'
+import { ApplicationModel } from '@server/models/application/application.js'
+
+const MIGRATION_NAME = 'peertube-8.3'
run()
.then(() => process.exit(0))
@@ -8,8 +11,12 @@ run()
})
async function run () {
+ await initDatabaseModels(true)
+
await fillVideoSearchTable()
await migrateVideoInfohashes()
+
+ await ApplicationModel.setManualMigrationScriptRun(MIGRATION_NAME)
}
async function fillVideoSearchTable () {
diff --git a/server/server.ts b/server/server.ts
index f705a299fa..3427f987ef 100644
--- a/server/server.ts
+++ b/server/server.ts
@@ -150,6 +150,7 @@ import { ActorFollowScheduler } from './core/lib/schedulers/actor-follow-schedul
import { AutoFollowIndexInstances } from './core/lib/schedulers/auto-follow-index-instances.js'
import { BlocklistSubscriptionsScheduler } from './core/lib/schedulers/blocklist-subscriptions-scheduler.js'
import { GeoIPUpdateScheduler } from './core/lib/schedulers/geo-ip-update-scheduler.js'
+import { ManualMigrationScriptsScheduler } from './core/lib/schedulers/manual-migration-scripts-scheduler.js'
import { PeerTubeVersionCheckScheduler } from './core/lib/schedulers/peertube-version-check-scheduler.js'
import { PluginsCheckScheduler } from './core/lib/schedulers/plugins-check-scheduler.js'
import { RemoveDanglingResumableUploadsScheduler } from './core/lib/schedulers/remove-dangling-resumable-uploads-scheduler.js'
@@ -345,6 +346,7 @@ async function startApplication () {
RemoveExpiredUserExportsScheduler.Instance.enable()
UpdateTokenSessionScheduler.Instance.enable()
RemoveOldUserLoginDevicesScheduler.Instance.enable()
+ ManualMigrationScriptsScheduler.Instance.enable()
OpenTelemetryMetrics.Instance.registerMetrics({ trackerServer })