Skip to content

ActivityPub-federated video streaming platform using P2P directly in your web browser

Fix PeerTube quitting cleanup

Chocobozzz6287c26c760d
diff --git a/packages/server-commands/src/server/server.ts b/packages/server-commands/src/server/server.ts
index eeda5513c4..56b1873cf3 100644
--- a/packages/server-commands/src/server/server.ts
+++ b/packages/server-commands/src/server/server.ts
@@ -338,14 +338,33 @@ export class PeerTubeServer {
})
}
+ // PeerTube gracefully shuts down on SIGTERM, so wait for the process to actually exit to be sure it released its ports
kill () {
if (!this.app) return Promise.resolve()
- process.kill(this.app.pid)
-
+ const app = this.app
this.app = null
- return Promise.resolve()
+ return new Promise<void>(res => {
+ if (app.exitCode !== null || app.signalCode !== null) return res()
+
+ const timeout = setTimeout(() => {
+ app.kill('SIGKILL')
+ res()
+ }, 10000)
+
+ app.once('exit', () => {
+ clearTimeout(timeout)
+ res()
+ })
+
+ try {
+ process.kill(app.pid)
+ } catch {
+ clearTimeout(timeout)
+ res()
+ }
+ })
}
private randomServer () {
diff --git a/server/core/initializers/constants.ts b/server/core/initializers/constants.ts
index febb40f15b..0863e8e833 100644
--- a/server/core/initializers/constants.ts
+++ b/server/core/initializers/constants.ts
@@ -378,6 +378,14 @@ export const REQUEST_TIMEOUTS = {
REDUNDANCY: JOB_TTL['video-redundancy']
}
+// Container runtimes send a SIGKILL if we take too long to exit (docker waits 10 seconds, kubernetes 30 seconds)
+export const SHUTDOWN_TIMEOUTS = {
+ // Time we let in flight HTTP requests complete before destroying their sockets
+ HTTP_CONNECTIONS: 2000, // 2 seconds
+ // Time we let the whole graceful shutdown complete before exiting anyway
+ GLOBAL: 8000 // 8 seconds
+}
+
export const SCHEDULER_INTERVALS_MS = {
RUNNER_JOB_WATCH_DOG: Math.min(CONFIG.REMOTE_RUNNERS.STALLED_JOBS.VOD, CONFIG.REMOTE_RUNNERS.STALLED_JOBS.LIVE),
ACTOR_FOLLOW_SCORES: 60000 * 60 * 20, // 20 hours
diff --git a/server/core/lib/job-queue/job-queue.ts b/server/core/lib/job-queue/job-queue.ts
index 4e29f431bf..5c1b00ebba 100644
--- a/server/core/lib/job-queue/job-queue.ts
+++ b/server/core/lib/job-queue/job-queue.ts
@@ -329,7 +329,8 @@ class JobQueue {
// ---------------------------------------------------------------------------
- async terminate () {
+ // Use force: true to not wait for active jobs to complete (they will be retried when detected as stalled)
+ async terminate (options: { force: boolean }) {
const promises = Object.keys(this.workers)
.map(handlerName => {
const worker: Worker = this.workers[handlerName]
@@ -337,7 +338,7 @@ class JobQueue {
const queueEvent: QueueEvents = this.queueEvents[handlerName]
return Promise.all([
- worker.close(false),
+ worker.close(options.force),
queue.close(),
queueEvent.close()
])
diff --git a/server/core/lib/redis.ts b/server/core/lib/redis.ts
index e7f96b16e8..682b1bc0a6 100644
--- a/server/core/lib/redis.ts
+++ b/server/core/lib/redis.ts
@@ -26,6 +26,7 @@ class Redis {
private static instance: Redis
private initialized = false
private connected = false
+ private quitting = false
private client: IoRedis
private prefix: string
@@ -51,17 +52,33 @@ class Redis {
logger.error(`Reconnecting to redis in ${ms}.`)
})
this.client.on('close', () => {
- logger.error('Connection to redis has closed.')
+ // Expected when we shut down PeerTube
+ if (this.quitting !== true) logger.error('Connection to redis has closed.')
+
this.connected = false
})
this.client.on('end', () => {
+ if (this.quitting === true) {
+ logger.info('Connection to redis has closed.')
+ return
+ }
+
logger.error('Connection to redis has closed and no more reconnects will be done.')
})
this.prefix = 'redis-' + WEBSERVER.HOST + '-'
}
+ async quit () {
+ if (this.initialized !== true) return
+
+ this.initialized = false
+ this.quitting = true
+
+ await this.client.quit()
+ }
+
static getRedisClientOptions (name?: string, options: RedisOptions = {}, logOptions = false): RedisOptions {
const connectionName = [ 'PeerTube', name ].join('')
const connectTimeout = 20000 // Could be slow since node use sync call to compile PeerTube
diff --git a/server/core/lib/schedulers/abstract-scheduler.ts b/server/core/lib/schedulers/abstract-scheduler.ts
index bf92da46ee..aff6fb7981 100644
--- a/server/core/lib/schedulers/abstract-scheduler.ts
+++ b/server/core/lib/schedulers/abstract-scheduler.ts
@@ -5,6 +5,8 @@ import { createLogger } from '../../helpers/logger.js'
const logger = createLogger('schedulers')
export abstract class AbstractScheduler {
+ private static readonly enabledSchedulers = new Set<AbstractScheduler>()
+
protected abstract schedulerIntervalMs: number
private interval: NodeJS.Timeout
@@ -21,9 +23,17 @@ export abstract class AbstractScheduler {
}
}
+ static disableAll () {
+ for (const scheduler of this.enabledSchedulers) {
+ scheduler.disable()
+ }
+ }
+
enable () {
if (!this.schedulerIntervalMs) throw new Error('Interval is not correctly set.')
+ AbstractScheduler.enabledSchedulers.add(this)
+
if (this.randomRunOnEnable === true) {
const randomDelay = randomInt(0, Math.floor(this.schedulerIntervalMs / 2))
@@ -43,6 +53,8 @@ export abstract class AbstractScheduler {
}
disable () {
+ AbstractScheduler.enabledSchedulers.delete(this)
+
if (this.firstRunTimeout) {
clearTimeout(this.firstRunTimeout)
this.firstRunTimeout = undefined
diff --git a/server/core/lib/shutdown.ts b/server/core/lib/shutdown.ts
new file mode 100644
index 0000000000..56a23f7f6f
--- /dev/null
+++ b/server/core/lib/shutdown.ts
@@ -0,0 +1,89 @@
+import { Server as HTTPServer } from 'node:http'
+import { createLogger } from '../helpers/logger.js'
+import { SHUTDOWN_TIMEOUTS } from '../initializers/constants.js'
+import { sequelizeTypescript } from '../initializers/database.js'
+import { JobQueue } from './job-queue/job-queue.js'
+import { LiveManager } from './live/live-manager.js'
+import { Redis } from './redis.js'
+import { AbstractScheduler } from './schedulers/abstract-scheduler.js'
+
+const logger = createLogger()
+
+let shuttingDown = false
+
+/**
+ * A signal that has no listener keeps its default disposition, and the kernel never delivers such a signal to the init process
+ * of a PID namespace. PeerTube is PID 1 in our docker image, so without an explicit listener SIGTERM is silently discarded and
+ * the container is SIGKILLed after the runtime grace period. So always listen to the signals we want to handle.
+ */
+export function registerGracefulShutdown (server: HTTPServer) {
+ for (const signal of [ 'SIGINT', 'SIGTERM' ] as const) {
+ process.on(signal, () => {
+ if (shuttingDown === true) {
+ logger.info(`Received a second ${signal} signal, exiting now.`)
+
+ return process.exit(0)
+ }
+
+ shuttingDown = true
+ logger.info(`Received ${signal}, gracefully shutting down PeerTube.`)
+
+ shutdown(server)
+ .catch(err => logger.error('Error in graceful shutdown.', { err }))
+ .finally(() => process.exit(0))
+ })
+ }
+}
+
+// ---------------------------------------------------------------------------
+// Private
+// ---------------------------------------------------------------------------
+
+async function shutdown (server: HTTPServer) {
+ const timeout = setTimeout(() => {
+ logger.warn(`Graceful shutdown did not complete in ${SHUTDOWN_TIMEOUTS.GLOBAL}ms, exiting now.`)
+
+ process.exit(0)
+ }, SHUTDOWN_TIMEOUTS.GLOBAL)
+ timeout.unref()
+
+ // Stop scheduling new work first, so nothing can grab a database connection we are about to close
+ AbstractScheduler.disableAll()
+ LiveManager.Instance.stop()
+
+ await Promise.all([
+ closeHTTPServer(server),
+
+ // Active jobs are not awaited: a transcoding job can run for hours. They are retried when detected as stalled
+ JobQueue.Instance.terminate({ force: true })
+ .catch(err => logger.error('Cannot terminate job queue.', { err }))
+ ])
+
+ await Promise.all([
+ sequelizeTypescript.close()
+ .catch(err => logger.error('Cannot close database connection.', { err })),
+
+ Redis.Instance.quit()
+ .catch(err => logger.error('Cannot close redis connection.', { err }))
+ ])
+
+ clearTimeout(timeout)
+
+ logger.info('PeerTube gracefully shut down.')
+}
+
+function closeHTTPServer (server: HTTPServer) {
+ return new Promise<void>(res => {
+ server.close(err => {
+ if (err) logger.error('Cannot close HTTP server.', { err })
+
+ res()
+ })
+
+ // server.close() only stops accepting new connections: also close the idle keep alive/websocket ones
+ server.closeIdleConnections()
+
+ // And then destroy the remaining ones so a long running request cannot delay the shutdown
+ setTimeout(() => server.closeAllConnections(), SHUTDOWN_TIMEOUTS.HTTP_CONNECTIONS).unref()
+ })
+}
diff --git a/server/server.ts b/server/server.ts
index 3427f987ef..b8b3faa506 100644
--- a/server/server.ts
+++ b/server/server.ts
@@ -45,7 +45,7 @@ try {
// ----------- Database -----------
// Initialize database and models
-import { checkDatabaseConnectionOrDie, initDatabaseModels, sequelizeTypescript } from './core/initializers/database.js'
+import { checkDatabaseConnectionOrDie, initDatabaseModels } from './core/initializers/database.js'
checkDatabaseConnectionOrDie()
import { migrate } from './core/initializers/migrator.js'
@@ -163,6 +163,7 @@ import { VideoStatsBufferScheduler } from './core/lib/schedulers/video-stats-buf
import { VideosRedundancyScheduler } from './core/lib/schedulers/videos-redundancy-scheduler.js'
import { WatchedWordsSubscriptionsScheduler } from './core/lib/schedulers/watched-words-subscriptions-scheduler.js'
import { YoutubeDlUpdateScheduler } from './core/lib/schedulers/youtube-dl-update-scheduler.js'
+import { registerGracefulShutdown } from './core/lib/shutdown.js'
import { advertiseDoNotTrack } from './core/middlewares/dnt.js'
import { apiFailMiddleware } from './core/middlewares/error.js'
@@ -299,6 +300,9 @@ const { server, trackerServer } = createWebsocketTrackerServer(app)
server.requestTimeout = CONFIG.HTTP_TIMEOUTS.REQUEST
+// Register it before the application is started so the process can always be stopped, even during a long migration
+registerGracefulShutdown(server)
+
// ----------- Run -----------
async function startApplication () {
@@ -393,14 +397,4 @@ async function startApplication () {
if (cliOptions['benchmarkStartup']) process.exit(0)
})
-
- process.on('exit', () => {
- sequelizeTypescript.close()
- .catch(err => logger.error('Cannot close database connection.', { err }))
-
- JobQueue.Instance.terminate()
- .catch(err => logger.error('Cannot terminate job queue.', { err }))
- })
-
- process.on('SIGINT', () => process.exit(0))
}