ActivityPub-federated video streaming platform using P2P directly in your web browser
Log when manual migration script hasn't been run
Chocobozzz83e263eaa9ef
diff --git a/server/core/initializers/constants.ts b/server/core/initializers/constants.tsindex e4e6f12bb5..56fe0d4879 100644--- a/server/core/initializers/constants.ts+++ b/server/core/initializers/constants.ts@@ -62,7 +62,7 @@ import { CONFIG, registerConfigChangedHandler } from './config.js'// ----------------------------------------------------------------------------export const LAST_MIGRATION_VERSION = 1115+export const LAST_MIGRATION_VERSION = 1120// ---------------------------------------------------------------------------@@ -382,8 +382,20 @@ export const SCHEDULER_INTERVALS_MS = {CHANNEL_SYNC_CHECK_INTERVAL: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.CHECK_INTERVAL,BLOCKLIST_SUBSCRIPTIONS_SYNC: 60000 * 60, // 1 hourWATCHED_WORDS_SUBSCRIPTIONS_SYNC: 60000 * 60, // 1 hour- REMOVE_OLD_USER_LOGIN_DEVICES: 60000 * 60 * 24 // 1 day-}+ REMOVE_OLD_USER_LOGIN_DEVICES: 60000 * 60 * 24, // 1 day+ CHECK_MANUAL_MIGRATION_SCRIPTS: 60000 * 60 * 12 // 12 hours+}++export const MANUAL_MIGRATION_SCRIPTS = [+ 'peertube-4.0',+ 'peertube-4.2',+ 'peertube-5.0',+ 'peertube-6.3',+ 'peertube-7.2',+ 'peertube-8.0',+ 'peertube-8.1',+ 'peertube-8.3'+]// Devices not seen again after this delay are forgotten, so a login from that IP/user-agent pair will be treated as new againexport const USER_LOGIN_DEVICE_MAX_AGE = 60000 * 60 * 24 * 365 // 1 yeardiff --git a/server/core/initializers/installer.ts b/server/core/initializers/installer.tsindex 3d3571f833..f565c7780e 100644--- a/server/core/initializers/installer.ts+++ b/server/core/initializers/installer.ts@@ -174,7 +174,17 @@ async function createApplicationIfNotExist () {const application = await ApplicationModel.create({migrationVersion: LAST_MIGRATION_VERSION,nodeVersion: process.version,- nodeABIVersion: getNodeABIVersion()+ nodeABIVersion: getNodeABIVersion(),+ // Scripts that existed before the manual migration tracking system was introduced: assume already run+ manualMigrationScriptsRun: [+ 'peertube-4.0',+ 'peertube-4.2',+ 'peertube-5.0',+ 'peertube-6.3',+ 'peertube-7.2',+ 'peertube-8.0',+ 'peertube-8.1'+ ]})return createApplicationActor(application.id)diff --git a/server/core/initializers/migrations/1120-manual-migration-scripts-run.ts b/server/core/initializers/migrations/1120-manual-migration-scripts-run.tsnew file mode 100644index 0000000000..9f5ac2dc1f--- /dev/null+++ b/server/core/initializers/migrations/1120-manual-migration-scripts-run.ts@@ -0,0 +1,32 @@+import * as Sequelize from 'sequelize'++async function up (utils: {+ transaction: Sequelize.Transaction+ queryInterface: Sequelize.QueryInterface+ sequelize: Sequelize.Sequelize+}): Promise<void> {+ const { transaction } = utils++ await utils.queryInterface.addColumn('application', 'manualMigrationScriptsRun', {+ type: Sequelize.ARRAY(Sequelize.STRING),+ allowNull: false,+ defaultValue: []+ }, { transaction })++ // Scripts that existed before this tracking system was introduced: assume already run+ const alreadyRun = [ 'peertube-4.0', 'peertube-4.2', 'peertube-5.0', 'peertube-6.3', 'peertube-7.2', 'peertube-8.0', 'peertube-8.1' ]++ await utils.sequelize.query(+ `UPDATE "application" SET "manualMigrationScriptsRun" = ARRAY[${alreadyRun.map(s => `'${s}'`).join(', ')}]`,+ { transaction }+ )+}++function down () {+ throw new Error('Not implemented.')+}++export {+ down,+ up+}diff --git a/server/core/lib/schedulers/manual-migration-scripts-scheduler.ts b/server/core/lib/schedulers/manual-migration-scripts-scheduler.tsnew file mode 100644index 0000000000..927971902e--- /dev/null+++ b/server/core/lib/schedulers/manual-migration-scripts-scheduler.ts@@ -0,0 +1,37 @@+import { ApplicationModel } from '@server/models/application/application.js'+import { createLogger } from '../../helpers/logger.js'+import { MANUAL_MIGRATION_SCRIPTS, SCHEDULER_INTERVALS_MS } from '../../initializers/constants.js'+import { AbstractScheduler } from './abstract-scheduler.js'++const logger = createLogger('schedulers')++export class ManualMigrationScriptsScheduler extends AbstractScheduler {+ private static instance: AbstractScheduler++ protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.CHECK_MANUAL_MIGRATION_SCRIPTS++ private constructor () {+ super({ randomRunOnEnable: true })+ }++ protected async internalExecute () {+ return this.checkMigrationScriptsRun()+ }++ private async checkMigrationScriptsRun () {+ const application = await ApplicationModel.load()++ const missing = MANUAL_MIGRATION_SCRIPTS.filter(script => !application.manualMigrationScriptsRun.includes(script))+ if (missing.length === 0) return++ logger.warn(+ 'The following manual migration scripts have not been run yet: %s. ' ++ 'Check the CHANGELOG for instructions on how to run them.',+ missing.join(', ')+ )+ }++ static get Instance () {+ return this.instance || (this.instance = new this())+ }+}diff --git a/server/core/models/application/application.ts b/server/core/models/application/application.tsindex c0d9726441..a625f1aca8 100644--- a/server/core/models/application/application.ts+++ b/server/core/models/application/application.ts@@ -66,6 +66,11 @@ export class ApplicationModel extends SequelizeModel<ApplicationModel> {@Column(DataType.JSONB)declare configPart: ConfigPart+ @AllowNull(false)+ @Default([])+ @Column(DataType.ARRAY(DataType.STRING))+ declare manualMigrationScriptsRun: string[]+@HasOne(() => AccountModel, {foreignKey: {allowNull: true@@ -100,6 +105,20 @@ export class ApplicationModel extends SequelizeModel<ApplicationModel> {return configPart?.OBJECT_STORAGE.STREAMING_PLAYLISTS.BASE_URL !== CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.BASE_URL}+ static async hasManualMigrationScriptRun (scriptName: string) {+ const application = await this.load()++ return application.manualMigrationScriptsRun.includes(scriptName)+ }++ static async setManualMigrationScriptRun (scriptName: string) {+ const application = await this.load()+ if (application.manualMigrationScriptsRun.includes(scriptName)) return++ application.manualMigrationScriptsRun = [ ...application.manualMigrationScriptsRun, scriptName ]+ await application.save()+ }+static async updateNodeVersionsOrConfig () {const application = await this.load()diff --git a/server/scripts/migrations/peertube-4.0.ts b/server/scripts/migrations/peertube-4.0.tsindex 0b4c04adfa..ed2fdefad8 100644--- a/server/scripts/migrations/peertube-4.0.ts+++ b/server/scripts/migrations/peertube-4.0.ts@@ -7,6 +7,7 @@ import {getHLSResolutionPlaylistFilename} from '@server/lib/paths.js'import { VideoPathManager } from '@server/lib/video-path-manager.js'+import { ApplicationModel } from '@server/models/application/application.js'import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist.js'import { VideoModel } from '@server/models/video/video.js'import Bluebird from 'bluebird'@@ -14,6 +15,8 @@ import { move } from 'fs-extra/esm'import { readFile, writeFile } from 'fs/promises'import { join } from 'path'+const MIGRATION_NAME = 'peertube-4.0'+run().then(() => process.exit(0)).catch(err => {@@ -22,10 +25,10 @@ run()})async function run () {- console.log('Migrate old HLS paths to new format.')-await initDatabaseModels(true)+ console.log('Migrate old HLS paths to new format.')+JobQueue.Instance.init()const ids = await VideoModel.listLocalIds()@@ -38,6 +41,8 @@ async function run () {}}, { concurrency: 5 })+ await ApplicationModel.setManualMigrationScriptRun(MIGRATION_NAME)+console.log('Migration finished!')}diff --git a/server/scripts/migrations/peertube-4.2.ts b/server/scripts/migrations/peertube-4.2.tsindex 163ae3b3ab..1b0ffd29cd 100644--- a/server/scripts/migrations/peertube-4.2.ts+++ b/server/scripts/migrations/peertube-4.2.ts@@ -10,10 +10,13 @@ import { sendUpdateActor } from '@server/lib/activitypub/send/index.js'import { JobQueue } from '@server/lib/job-queue/index.js'import { AccountModel } from '@server/models/account/account.js'import { ActorModel } from '@server/models/actor/actor.js'+import { ApplicationModel } from '@server/models/application/application.js'import { VideoChannelModel } from '@server/models/video/video-channel.js'import { MAccountDefault, MActorDefault, MChannelDefault } from '@server/types/models/index.js'import { join } from 'path'+const MIGRATION_NAME = 'peertube-4.2'+run().then(() => process.exit(0)).catch(err => {@@ -22,9 +25,10 @@ run()})async function run () {+ await initDatabaseModels(true)+console.log('Generate avatar miniatures from existing avatars.')- await initDatabaseModels(true)JobQueue.Instance.init()const accounts: AccountModel[] = await AccountModel.findAll({@@ -65,6 +69,8 @@ async function run () {}}+ await ApplicationModel.setManualMigrationScriptRun(MIGRATION_NAME)+console.log('Generation finished!')}diff --git a/server/scripts/migrations/peertube-5.0.ts b/server/scripts/migrations/peertube-5.0.tsindex 64505b133e..80184b6df5 100644--- a/server/scripts/migrations/peertube-5.0.ts+++ b/server/scripts/migrations/peertube-5.0.ts@@ -3,11 +3,14 @@ import { DIRECTORIES } from '@server/initializers/constants.js'import { initDatabaseModels } from '@server/initializers/database.js'import { moveFilesIfPrivacyChanged } from '@server/lib/video-privacy.js'import { updateTorrentForFileAndSave } from '@server/lib/webtorrent.js'+import { ApplicationModel } from '@server/models/application/application.js'import { VideoModel } from '@server/models/video/video.js'import { MVideoFull } from '@server/types/models/index.js'import { ensureDir } from 'fs-extra/esm'import { Op } from 'sequelize'+const MIGRATION_NAME = 'peertube-5.0'+run().then(() => process.exit(0)).catch(err => {@@ -16,13 +19,13 @@ run()})async function run () {+ await initDatabaseModels(true)+console.log('Moving private video files in dedicated folders.')await ensureDir(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE)await ensureDir(DIRECTORIES.WEB_VIDEOS.PRIVATE)- await initDatabaseModels(true)-const videos = await VideoModel.unscoped().findAll({attributes: [ 'uuid' ],where: {@@ -53,6 +56,8 @@ async function run () {console.error('Cannot process video %s.', uuid, err)}}++ await ApplicationModel.setManualMigrationScriptRun(MIGRATION_NAME)}async function updateTorrents (video: MVideoFull) {diff --git a/server/scripts/migrations/peertube-6.3.ts b/server/scripts/migrations/peertube-6.3.tsindex ad78ac7065..222bc89b73 100644--- a/server/scripts/migrations/peertube-6.3.ts+++ b/server/scripts/migrations/peertube-6.3.ts@@ -3,12 +3,15 @@ import { VideoFileStream } from '@peertube/peertube-models'import { initDatabaseModels, sequelizeTypescript } from '@server/initializers/database.js'import { buildFileMetadata } from '@server/lib/video-file.js'import { VideoPathManager } from '@server/lib/video-path-manager.js'+import { ApplicationModel } from '@server/models/application/application.js'import { VideoFileModel } from '@server/models/video/video-file.js'import { VideoModel } from '@server/models/video/video.js'import Bluebird from 'bluebird'import { pathExists } from 'fs-extra/esm'import { QueryTypes } from 'sequelize'+const MIGRATION_NAME = 'peertube-6.3'+run().then(() => process.exit(0)).catch(err => {@@ -68,6 +71,8 @@ async function run () {}}, { concurrency: 5 })+ await ApplicationModel.setManualMigrationScriptRun(MIGRATION_NAME)+console.log('\n## Migration finished! ##')}diff --git a/server/scripts/migrations/peertube-7.2.ts b/server/scripts/migrations/peertube-7.2.tsindex 0adc2b0754..615b2a5565 100644--- a/server/scripts/migrations/peertube-7.2.ts+++ b/server/scripts/migrations/peertube-7.2.ts@@ -1,4 +1,7 @@import { initDatabaseModels, sequelizeTypescript } from '@server/initializers/database.js'+import { ApplicationModel } from '@server/models/application/application.js'++const MIGRATION_NAME = 'peertube-7.2'run().then(() => process.exit(0))@@ -16,5 +19,7 @@ async function run () {await sequelizeTypescript.query(query)}+ await ApplicationModel.setManualMigrationScriptRun(MIGRATION_NAME)+console.log('Done!')}diff --git a/server/scripts/migrations/peertube-8.0.ts b/server/scripts/migrations/peertube-8.0.tsindex 53e4baf98e..0b0cb720bc 100644--- a/server/scripts/migrations/peertube-8.0.ts+++ b/server/scripts/migrations/peertube-8.0.ts@@ -2,6 +2,8 @@ import { initDatabaseModels, sequelizeTypescript } from '@server/initializers/daimport { ApplicationModel } from '@server/models/application/application.js'import { QueryTypes } from 'sequelize'+const MIGRATION_NAME = 'peertube-8.0'+run().then(() => process.exit(0)).catch(err => {@@ -41,5 +43,7 @@ async function run () {i++}+ await ApplicationModel.setManualMigrationScriptRun(MIGRATION_NAME)+console.log('Done!')}diff --git a/server/scripts/migrations/peertube-8.1.ts b/server/scripts/migrations/peertube-8.1.tsindex d7d4b79e61..9d111a0f45 100644--- a/server/scripts/migrations/peertube-8.1.ts+++ b/server/scripts/migrations/peertube-8.1.ts@@ -1,10 +1,13 @@import { CONFIG } from '@server/initializers/config.js'-import { sequelizeTypescript } from '@server/initializers/database.js'+import { initDatabaseModels, sequelizeTypescript } from '@server/initializers/database.js'+import { ApplicationModel } from '@server/models/application/application.js'import Bluebird from 'bluebird'import { move } from 'fs-extra'import { readdir } from 'fs/promises'import { join } from 'path'+const MIGRATION_NAME = 'peertube-8.1'+run().then(() => process.exit(0)).catch(err => {@@ -13,6 +16,8 @@ run()})async function run () {+ await initDatabaseModels(true)+try {await movePreviewsToThumbnails()} catch (err) {@@ -48,6 +53,8 @@ async function run () {} catch (err) {console.error('An error occurred while updating local video caption URLs:', err)}++ await ApplicationModel.setManualMigrationScriptRun(MIGRATION_NAME)}async function movePreviewsToThumbnails () {diff --git a/server/scripts/migrations/peertube-8.3.ts b/server/scripts/migrations/peertube-8.3.tsindex e6d03c3fde..0c5309f8bd 100644--- a/server/scripts/migrations/peertube-8.3.ts+++ b/server/scripts/migrations/peertube-8.3.ts@@ -1,4 +1,7 @@-import { sequelizeTypescript } from '@server/initializers/database.js'+import { initDatabaseModels, sequelizeTypescript } from '@server/initializers/database.js'+import { ApplicationModel } from '@server/models/application/application.js'++const MIGRATION_NAME = 'peertube-8.3'run().then(() => process.exit(0))@@ -8,8 +11,12 @@ run()})async function run () {+ await initDatabaseModels(true)+await fillVideoSearchTable()await migrateVideoInfohashes()++ await ApplicationModel.setManualMigrationScriptRun(MIGRATION_NAME)}async function fillVideoSearchTable () {diff --git a/server/server.ts b/server/server.tsindex f705a299fa..3427f987ef 100644--- a/server/server.ts+++ b/server/server.ts@@ -150,6 +150,7 @@ import { ActorFollowScheduler } from './core/lib/schedulers/actor-follow-schedulimport { AutoFollowIndexInstances } from './core/lib/schedulers/auto-follow-index-instances.js'import { BlocklistSubscriptionsScheduler } from './core/lib/schedulers/blocklist-subscriptions-scheduler.js'import { GeoIPUpdateScheduler } from './core/lib/schedulers/geo-ip-update-scheduler.js'+import { ManualMigrationScriptsScheduler } from './core/lib/schedulers/manual-migration-scripts-scheduler.js'import { PeerTubeVersionCheckScheduler } from './core/lib/schedulers/peertube-version-check-scheduler.js'import { PluginsCheckScheduler } from './core/lib/schedulers/plugins-check-scheduler.js'import { RemoveDanglingResumableUploadsScheduler } from './core/lib/schedulers/remove-dangling-resumable-uploads-scheduler.js'@@ -345,6 +346,7 @@ async function startApplication () {RemoveExpiredUserExportsScheduler.Instance.enable()UpdateTokenSessionScheduler.Instance.enable()RemoveOldUserLoginDevicesScheduler.Instance.enable()+ ManualMigrationScriptsScheduler.Instance.enable()OpenTelemetryMetrics.Instance.registerMetrics({ trackerServer })