ActivityPub-federated video streaming platform using P2P directly in your web browser
Fix PeerTube quitting cleanup
Chocobozzz6287c26c760d
diff --git a/packages/server-commands/src/server/server.ts b/packages/server-commands/src/server/server.tsindex eeda5513c4..56b1873cf3 100644--- a/packages/server-commands/src/server/server.ts+++ b/packages/server-commands/src/server/server.ts@@ -338,14 +338,33 @@ export class PeerTubeServer {})}+ // PeerTube gracefully shuts down on SIGTERM, so wait for the process to actually exit to be sure it released its portskill () {if (!this.app) return Promise.resolve()- process.kill(this.app.pid)-+ const app = this.appthis.app = null- return Promise.resolve()+ return new Promise<void>(res => {+ if (app.exitCode !== null || app.signalCode !== null) return res()++ const timeout = setTimeout(() => {+ app.kill('SIGKILL')+ res()+ }, 10000)++ app.once('exit', () => {+ clearTimeout(timeout)+ res()+ })++ try {+ process.kill(app.pid)+ } catch {+ clearTimeout(timeout)+ res()+ }+ })}private randomServer () {diff --git a/server/core/initializers/constants.ts b/server/core/initializers/constants.tsindex febb40f15b..0863e8e833 100644--- a/server/core/initializers/constants.ts+++ b/server/core/initializers/constants.ts@@ -378,6 +378,14 @@ export const REQUEST_TIMEOUTS = {REDUNDANCY: JOB_TTL['video-redundancy']}+// Container runtimes send a SIGKILL if we take too long to exit (docker waits 10 seconds, kubernetes 30 seconds)+export const SHUTDOWN_TIMEOUTS = {+ // Time we let in flight HTTP requests complete before destroying their sockets+ HTTP_CONNECTIONS: 2000, // 2 seconds+ // Time we let the whole graceful shutdown complete before exiting anyway+ GLOBAL: 8000 // 8 seconds+}+export const SCHEDULER_INTERVALS_MS = {RUNNER_JOB_WATCH_DOG: Math.min(CONFIG.REMOTE_RUNNERS.STALLED_JOBS.VOD, CONFIG.REMOTE_RUNNERS.STALLED_JOBS.LIVE),ACTOR_FOLLOW_SCORES: 60000 * 60 * 20, // 20 hoursdiff --git a/server/core/lib/job-queue/job-queue.ts b/server/core/lib/job-queue/job-queue.tsindex 4e29f431bf..5c1b00ebba 100644--- a/server/core/lib/job-queue/job-queue.ts+++ b/server/core/lib/job-queue/job-queue.ts@@ -329,7 +329,8 @@ class JobQueue {// ---------------------------------------------------------------------------- async terminate () {+ // Use force: true to not wait for active jobs to complete (they will be retried when detected as stalled)+ async terminate (options: { force: boolean }) {const promises = Object.keys(this.workers).map(handlerName => {const worker: Worker = this.workers[handlerName]@@ -337,7 +338,7 @@ class JobQueue {const queueEvent: QueueEvents = this.queueEvents[handlerName]return Promise.all([- worker.close(false),+ worker.close(options.force),queue.close(),queueEvent.close()])diff --git a/server/core/lib/redis.ts b/server/core/lib/redis.tsindex e7f96b16e8..682b1bc0a6 100644--- a/server/core/lib/redis.ts+++ b/server/core/lib/redis.ts@@ -26,6 +26,7 @@ class Redis {private static instance: Redisprivate initialized = falseprivate connected = false+ private quitting = falseprivate client: IoRedisprivate prefix: string@@ -51,17 +52,33 @@ class Redis {logger.error(`Reconnecting to redis in ${ms}.`)})this.client.on('close', () => {- logger.error('Connection to redis has closed.')+ // Expected when we shut down PeerTube+ if (this.quitting !== true) logger.error('Connection to redis has closed.')+this.connected = false})this.client.on('end', () => {+ if (this.quitting === true) {+ logger.info('Connection to redis has closed.')+ return+ }+logger.error('Connection to redis has closed and no more reconnects will be done.')})this.prefix = 'redis-' + WEBSERVER.HOST + '-'}+ async quit () {+ if (this.initialized !== true) return++ this.initialized = false+ this.quitting = true++ await this.client.quit()+ }+static getRedisClientOptions (name?: string, options: RedisOptions = {}, logOptions = false): RedisOptions {const connectionName = [ 'PeerTube', name ].join('')const connectTimeout = 20000 // Could be slow since node use sync call to compile PeerTubediff --git a/server/core/lib/schedulers/abstract-scheduler.ts b/server/core/lib/schedulers/abstract-scheduler.tsindex bf92da46ee..aff6fb7981 100644--- a/server/core/lib/schedulers/abstract-scheduler.ts+++ b/server/core/lib/schedulers/abstract-scheduler.ts@@ -5,6 +5,8 @@ import { createLogger } from '../../helpers/logger.js'const logger = createLogger('schedulers')export abstract class AbstractScheduler {+ private static readonly enabledSchedulers = new Set<AbstractScheduler>()+protected abstract schedulerIntervalMs: numberprivate interval: NodeJS.Timeout@@ -21,9 +23,17 @@ export abstract class AbstractScheduler {}}+ static disableAll () {+ for (const scheduler of this.enabledSchedulers) {+ scheduler.disable()+ }+ }+enable () {if (!this.schedulerIntervalMs) throw new Error('Interval is not correctly set.')+ AbstractScheduler.enabledSchedulers.add(this)+if (this.randomRunOnEnable === true) {const randomDelay = randomInt(0, Math.floor(this.schedulerIntervalMs / 2))@@ -43,6 +53,8 @@ export abstract class AbstractScheduler {}disable () {+ AbstractScheduler.enabledSchedulers.delete(this)+if (this.firstRunTimeout) {clearTimeout(this.firstRunTimeout)this.firstRunTimeout = undefineddiff --git a/server/core/lib/shutdown.ts b/server/core/lib/shutdown.tsnew file mode 100644index 0000000000..56a23f7f6f--- /dev/null+++ b/server/core/lib/shutdown.ts@@ -0,0 +1,89 @@+import { Server as HTTPServer } from 'node:http'+import { createLogger } from '../helpers/logger.js'+import { SHUTDOWN_TIMEOUTS } from '../initializers/constants.js'+import { sequelizeTypescript } from '../initializers/database.js'+import { JobQueue } from './job-queue/job-queue.js'+import { LiveManager } from './live/live-manager.js'+import { Redis } from './redis.js'+import { AbstractScheduler } from './schedulers/abstract-scheduler.js'++const logger = createLogger()++let shuttingDown = false++/**+ * A signal that has no listener keeps its default disposition, and the kernel never delivers such a signal to the init process+ * of a PID namespace. PeerTube is PID 1 in our docker image, so without an explicit listener SIGTERM is silently discarded and+ * the container is SIGKILLed after the runtime grace period. So always listen to the signals we want to handle.+ */+export function registerGracefulShutdown (server: HTTPServer) {+ for (const signal of [ 'SIGINT', 'SIGTERM' ] as const) {+ process.on(signal, () => {+ if (shuttingDown === true) {+ logger.info(`Received a second ${signal} signal, exiting now.`)++ return process.exit(0)+ }++ shuttingDown = true+ logger.info(`Received ${signal}, gracefully shutting down PeerTube.`)++ shutdown(server)+ .catch(err => logger.error('Error in graceful shutdown.', { err }))+ .finally(() => process.exit(0))+ })+ }+}++// ---------------------------------------------------------------------------+// Private+// ---------------------------------------------------------------------------++async function shutdown (server: HTTPServer) {+ const timeout = setTimeout(() => {+ logger.warn(`Graceful shutdown did not complete in ${SHUTDOWN_TIMEOUTS.GLOBAL}ms, exiting now.`)++ process.exit(0)+ }, SHUTDOWN_TIMEOUTS.GLOBAL)+ timeout.unref()++ // Stop scheduling new work first, so nothing can grab a database connection we are about to close+ AbstractScheduler.disableAll()+ LiveManager.Instance.stop()++ await Promise.all([+ closeHTTPServer(server),++ // Active jobs are not awaited: a transcoding job can run for hours. They are retried when detected as stalled+ JobQueue.Instance.terminate({ force: true })+ .catch(err => logger.error('Cannot terminate job queue.', { err }))+ ])++ await Promise.all([+ sequelizeTypescript.close()+ .catch(err => logger.error('Cannot close database connection.', { err })),++ Redis.Instance.quit()+ .catch(err => logger.error('Cannot close redis connection.', { err }))+ ])++ clearTimeout(timeout)++ logger.info('PeerTube gracefully shut down.')+}++function closeHTTPServer (server: HTTPServer) {+ return new Promise<void>(res => {+ server.close(err => {+ if (err) logger.error('Cannot close HTTP server.', { err })++ res()+ })++ // server.close() only stops accepting new connections: also close the idle keep alive/websocket ones+ server.closeIdleConnections()++ // And then destroy the remaining ones so a long running request cannot delay the shutdown+ setTimeout(() => server.closeAllConnections(), SHUTDOWN_TIMEOUTS.HTTP_CONNECTIONS).unref()+ })+}diff --git a/server/server.ts b/server/server.tsindex 3427f987ef..b8b3faa506 100644--- a/server/server.ts+++ b/server/server.ts@@ -45,7 +45,7 @@ try {// ----------- Database -----------// Initialize database and models-import { checkDatabaseConnectionOrDie, initDatabaseModels, sequelizeTypescript } from './core/initializers/database.js'+import { checkDatabaseConnectionOrDie, initDatabaseModels } from './core/initializers/database.js'checkDatabaseConnectionOrDie()import { migrate } from './core/initializers/migrator.js'@@ -163,6 +163,7 @@ import { VideoStatsBufferScheduler } from './core/lib/schedulers/video-stats-bufimport { VideosRedundancyScheduler } from './core/lib/schedulers/videos-redundancy-scheduler.js'import { WatchedWordsSubscriptionsScheduler } from './core/lib/schedulers/watched-words-subscriptions-scheduler.js'import { YoutubeDlUpdateScheduler } from './core/lib/schedulers/youtube-dl-update-scheduler.js'+import { registerGracefulShutdown } from './core/lib/shutdown.js'import { advertiseDoNotTrack } from './core/middlewares/dnt.js'import { apiFailMiddleware } from './core/middlewares/error.js'@@ -299,6 +300,9 @@ const { server, trackerServer } = createWebsocketTrackerServer(app)server.requestTimeout = CONFIG.HTTP_TIMEOUTS.REQUEST+// Register it before the application is started so the process can always be stopped, even during a long migration+registerGracefulShutdown(server)+// ----------- Run -----------async function startApplication () {@@ -393,14 +397,4 @@ async function startApplication () {if (cliOptions['benchmarkStartup']) process.exit(0)})-- process.on('exit', () => {- sequelizeTypescript.close()- .catch(err => logger.error('Cannot close database connection.', { err }))-- JobQueue.Instance.terminate()- .catch(err => logger.error('Cannot terminate job queue.', { err }))- })-- process.on('SIGINT', () => process.exit(0))}