Skip to content

ActivityPub-federated video streaming platform using P2P directly in your web browser

Expose runner and runner job queue metrics (#7469)

Lachlan Stevensa87677574693
diff --git a/packages/tests/src/api/server/open-telemetry.ts b/packages/tests/src/api/server/open-telemetry.ts
index 6de8ad05b4..01f9ed79c6 100644
--- a/packages/tests/src/api/server/open-telemetry.ts
+++ b/packages/tests/src/api/server/open-telemetry.ts
@@ -57,6 +57,44 @@ describe('Open Telemetry', function () {
expect(res.text).to.contain('http_request_duration_ms_bucket{')
})
+ it('Should have runner observer metrics', async function () {
+ await setAccessTokensToServers([ server ])
+
+ const runnerName = 'otel-runner'
+ const runnerVersion = '1.2.3'
+
+ const { data } = await server.runnerRegistrationTokens.list({ sort: 'createdAt' })
+ const registrationToken = data[0].registrationToken
+ const { runnerToken } = await server.runners.register({
+ name: runnerName,
+ version: runnerVersion,
+ registrationToken
+ })
+
+ try {
+ const res = await makeRawRequest({ url: metricsUrl, expectedStatus: HttpStatusCode.OK_200 })
+
+ expect(res.text).to.match(/peertube_runner_count\{[^}]*\} 1(\.0)?/)
+ expect(res.text).to.match(
+ new RegExp(`peertube_runner_info\\{[^}]*runnerName="${runnerName}"[^}]*version="${runnerVersion}"[^}]*\\} 1(\\.0)?`)
+ )
+
+ const secondsMatch = res.text.match(
+ new RegExp(`peertube_runner_seconds_since_last_contact\\{[^}]*runnerName="${runnerName}"[^}]*\\} ([0-9]+(?:\\.[0-9]+)?)`)
+ )
+ expect(secondsMatch).to.not.be.null
+
+ if (!secondsMatch) {
+ throw new Error('Could not find runner contact metric in OpenTelemetry output')
+ }
+
+ const seconds = Number.parseFloat(secondsMatch[1])
+ expect(seconds).to.be.at.least(0)
+ } finally {
+ await server.runners.unregister({ runnerToken })
+ }
+ })
+
it('Should have playback metrics', async function () {
await setAccessTokensToServers([ server ])
diff --git a/server/core/lib/opentelemetry/metric-helpers/index.ts b/server/core/lib/opentelemetry/metric-helpers/index.ts
index ec9c354080..d41c0e6d98 100644
--- a/server/core/lib/opentelemetry/metric-helpers/index.ts
+++ b/server/core/lib/opentelemetry/metric-helpers/index.ts
@@ -3,5 +3,7 @@ export * from './lives-observers-builder.js'
export * from './job-queue-observers-builder.js'
export * from './nodejs-observers-builder.js'
export * from './playback-metrics.js'
+export * from './runner-job-queue-observers-builder.js'
+export * from './runner-observers-builder.js'
export * from './stats-observers-builder.js'
export * from './viewers-observers-builder.js'
diff --git a/server/core/lib/opentelemetry/metric-helpers/runner-job-queue-observers-builder.ts b/server/core/lib/opentelemetry/metric-helpers/runner-job-queue-observers-builder.ts
new file mode 100644
index 0000000000..354a08b4a7
--- /dev/null
+++ b/server/core/lib/opentelemetry/metric-helpers/runner-job-queue-observers-builder.ts
@@ -0,0 +1,35 @@
+import { Meter } from '@opentelemetry/api'
+import { RunnerJobState } from '@peertube/peertube-models'
+import { runnerJobTypes } from '@server/lib/runners/job-handlers/runner-job-handlers.js'
+import { RunnerJobModel } from '@server/models/runner/runner-job.js'
+
+const stateLabels = Object.fromEntries(
+ Object.entries(RunnerJobState).map(([ key, value ]) => [ value, key.toLowerCase() ])
+) as Record<number, string>
+
+export class RunnerJobQueueObserversBuilder {
+
+ constructor (private readonly meter: Meter) {
+
+ }
+
+ buildObservers () {
+ this.meter.createObservableGauge('peertube_runner_job_queue_total', {
+ description: 'Total jobs in the PeerTube runner job queue'
+ }).addCallback(async observableResult => {
+ const stats = await RunnerJobModel.getStats()
+
+ const countMap = new Map(
+ stats.map(({ jobType, state, count }) => [ `${jobType}:${state}`, count ])
+ )
+
+ for (const jobType of runnerJobTypes) {
+ for (const [ stateNum, stateLabel ] of Object.entries(stateLabels)) {
+ const count = countMap.get(`${jobType}:${stateNum}`) ?? 0
+ observableResult.observe(count, { jobType, state: stateLabel })
+ }
+ }
+ })
+ }
+
+}
diff --git a/server/core/lib/opentelemetry/metric-helpers/runner-observers-builder.ts b/server/core/lib/opentelemetry/metric-helpers/runner-observers-builder.ts
new file mode 100644
index 0000000000..551b323401
--- /dev/null
+++ b/server/core/lib/opentelemetry/metric-helpers/runner-observers-builder.ts
@@ -0,0 +1,44 @@
+import { Meter } from '@opentelemetry/api'
+import { RunnerModel } from '@server/models/runner/runner.js'
+
+export class RunnerObserversBuilder {
+
+ constructor (private readonly meter: Meter) {
+
+ }
+
+ buildObservers () {
+ const runnerCount = this.meter.createObservableGauge('peertube_runner_count', {
+ description: 'Number of registered PeerTube runners'
+ })
+
+ const runnerInfo = this.meter.createObservableGauge('peertube_runner_info', {
+ description: 'PeerTube runner metadata'
+ })
+
+ const secondsSinceLastContact = this.meter.createObservableGauge('peertube_runner_seconds_since_last_contact', {
+ description: 'Seconds since the PeerTube runner last checked in',
+ unit: 's'
+ })
+
+ this.meter.addBatchObservableCallback(async observableResult => {
+ const runners = await RunnerModel.listAll()
+
+ observableResult.observe(runnerCount, runners.length)
+
+ for (const runner of runners) {
+ const secondsSinceContact = (Date.now() - runner.lastContact.getTime()) / 1000
+
+ observableResult.observe(secondsSinceLastContact, secondsSinceContact, { runnerName: runner.name })
+ observableResult.observe(runnerInfo, 1, {
+ runnerName: runner.name,
+ description: runner.description ?? '',
+ ip: runner.ip,
+ version: runner.version ?? 'unknown',
+ createdAt: runner.createdAt.toISOString()
+ })
+ }
+ }, [ runnerCount, runnerInfo, secondsSinceLastContact ])
+ }
+
+}
diff --git a/server/core/lib/opentelemetry/metrics.ts b/server/core/lib/opentelemetry/metrics.ts
index 2a57c681bd..f7a6db7bb1 100644
--- a/server/core/lib/opentelemetry/metrics.ts
+++ b/server/core/lib/opentelemetry/metrics.ts
@@ -12,6 +12,8 @@ import {
LivesObserversBuilder,
NodeJSObserversBuilder,
PlaybackMetrics,
+ RunnerJobQueueObserversBuilder,
+ RunnerObserversBuilder,
StatsObserversBuilder,
ViewersObserversBuilder
} from './metric-helpers/index.js'
@@ -72,6 +74,8 @@ class OpenTelemetryMetrics {
new NodeJSObserversBuilder(this.meter).buildObservers()
new JobQueueObserversBuilder(this.meter).buildObservers()
+ new RunnerJobQueueObserversBuilder(this.meter).buildObservers()
+ new RunnerObserversBuilder(this.meter).buildObservers()
new StatsObserversBuilder(this.meter).buildObservers()
new LivesObserversBuilder(this.meter).buildObservers()
new ViewersObserversBuilder(this.meter).buildObservers()
diff --git a/server/core/lib/runners/job-handlers/runner-job-handlers.ts b/server/core/lib/runners/job-handlers/runner-job-handlers.ts
index b85aaec194..4051fd1297 100644
--- a/server/core/lib/runners/job-handlers/runner-job-handlers.ts
+++ b/server/core/lib/runners/job-handlers/runner-job-handlers.ts
@@ -19,6 +19,8 @@ const processors: Record<RunnerJobType, new() => AbstractJobHandler<unknown, Run
'generate-video-storyboard': VideoStoryboardJobHandler
}
+export const runnerJobTypes = Object.keys(processors)
+
export function getRunnerJobHandlerClass (job: MRunnerJob) {
return processors[job.type]
}
diff --git a/server/core/models/runner/runner-job.ts b/server/core/models/runner/runner-job.ts
index 1076cde935..70068bda0c 100644
--- a/server/core/models/runner/runner-job.ts
+++ b/server/core/models/runner/runner-job.ts
@@ -10,7 +10,7 @@ import {
import { isArray, isUUIDValid } from '@server/helpers/custom-validators/misc.js'
import { CONSTRAINTS_FIELDS, RUNNER_JOB_STATES } from '@server/initializers/constants.js'
import { MRunnerJob, MRunnerJobRunner, MRunnerJobRunnerParent } from '@server/types/models/runners/index.js'
-import { Op, Transaction } from 'sequelize'
+import { literal, Op, Transaction } from 'sequelize'
import {
AllowNull,
BelongsTo,
@@ -24,7 +24,7 @@ import {
Table,
UpdatedAt
} from 'sequelize-typescript'
-import { SequelizeModel, getSort, searchAttribute } from '../shared/index.js'
+import { SequelizeModel, getSort, parseAggregateResult, searchAttribute } from '../shared/index.js'
import { RunnerModel } from './runner.js'
enum ScopeNames {
@@ -290,6 +290,24 @@ export class RunnerJobModel extends SequelizeModel<RunnerJobModel> {
]).then(([ total, data ]) => ({ total, data }))
}
+ static getStats () {
+ return RunnerJobModel.findAll<MRunnerJob>({
+ attributes: [
+ 'type',
+ 'state',
+ [ literal('COUNT(*)'), 'count' ]
+ ],
+ group: [ 'type', 'state' ],
+ raw: true
+ }).then(rows => {
+ return (rows as any[]).map(row => ({
+ jobType: row.type as RunnerJobType,
+ state: row.state as RunnerJobStateType,
+ count: parseAggregateResult(row.count)
+ }))
+ })
+ }
+
static updateDependantJobsOf (runnerJob: MRunnerJob) {
const where = {
dependsOnRunnerJobId: runnerJob.id
diff --git a/server/core/models/runner/runner.ts b/server/core/models/runner/runner.ts
index 924dacca8d..ef4606e685 100644
--- a/server/core/models/runner/runner.ts
+++ b/server/core/models/runner/runner.ts
@@ -88,6 +88,10 @@ export class RunnerModel extends SequelizeModel<RunnerModel> {
return RunnerModel.findOne(query)
}
+ static listAll () {
+ return RunnerModel.findAll<MRunner>()
+ }
+
static listForApi (options: {
start: number
count: number