ActivityPub-federated video streaming platform using P2P directly in your web browser
Expose runner and runner job queue metrics (#7469)
Lachlan Stevensa87677574693
diff --git a/packages/tests/src/api/server/open-telemetry.ts b/packages/tests/src/api/server/open-telemetry.tsindex 6de8ad05b4..01f9ed79c6 100644--- a/packages/tests/src/api/server/open-telemetry.ts+++ b/packages/tests/src/api/server/open-telemetry.ts@@ -57,6 +57,44 @@ describe('Open Telemetry', function () {expect(res.text).to.contain('http_request_duration_ms_bucket{')})+ it('Should have runner observer metrics', async function () {+ await setAccessTokensToServers([ server ])++ const runnerName = 'otel-runner'+ const runnerVersion = '1.2.3'++ const { data } = await server.runnerRegistrationTokens.list({ sort: 'createdAt' })+ const registrationToken = data[0].registrationToken+ const { runnerToken } = await server.runners.register({+ name: runnerName,+ version: runnerVersion,+ registrationToken+ })++ try {+ const res = await makeRawRequest({ url: metricsUrl, expectedStatus: HttpStatusCode.OK_200 })++ expect(res.text).to.match(/peertube_runner_count\{[^}]*\} 1(\.0)?/)+ expect(res.text).to.match(+ new RegExp(`peertube_runner_info\\{[^}]*runnerName="${runnerName}"[^}]*version="${runnerVersion}"[^}]*\\} 1(\\.0)?`)+ )++ const secondsMatch = res.text.match(+ new RegExp(`peertube_runner_seconds_since_last_contact\\{[^}]*runnerName="${runnerName}"[^}]*\\} ([0-9]+(?:\\.[0-9]+)?)`)+ )+ expect(secondsMatch).to.not.be.null++ if (!secondsMatch) {+ throw new Error('Could not find runner contact metric in OpenTelemetry output')+ }++ const seconds = Number.parseFloat(secondsMatch[1])+ expect(seconds).to.be.at.least(0)+ } finally {+ await server.runners.unregister({ runnerToken })+ }+ })+it('Should have playback metrics', async function () {await setAccessTokensToServers([ server ])diff --git a/server/core/lib/opentelemetry/metric-helpers/index.ts b/server/core/lib/opentelemetry/metric-helpers/index.tsindex ec9c354080..d41c0e6d98 100644--- a/server/core/lib/opentelemetry/metric-helpers/index.ts+++ b/server/core/lib/opentelemetry/metric-helpers/index.ts@@ -3,5 +3,7 @@ export * from './lives-observers-builder.js'export * from './job-queue-observers-builder.js'export * from './nodejs-observers-builder.js'export * from './playback-metrics.js'+export * from './runner-job-queue-observers-builder.js'+export * from './runner-observers-builder.js'export * from './stats-observers-builder.js'export * from './viewers-observers-builder.js'diff --git a/server/core/lib/opentelemetry/metric-helpers/runner-job-queue-observers-builder.ts b/server/core/lib/opentelemetry/metric-helpers/runner-job-queue-observers-builder.tsnew file mode 100644index 0000000000..354a08b4a7--- /dev/null+++ b/server/core/lib/opentelemetry/metric-helpers/runner-job-queue-observers-builder.ts@@ -0,0 +1,35 @@+import { Meter } from '@opentelemetry/api'+import { RunnerJobState } from '@peertube/peertube-models'+import { runnerJobTypes } from '@server/lib/runners/job-handlers/runner-job-handlers.js'+import { RunnerJobModel } from '@server/models/runner/runner-job.js'++const stateLabels = Object.fromEntries(+ Object.entries(RunnerJobState).map(([ key, value ]) => [ value, key.toLowerCase() ])+) as Record<number, string>++export class RunnerJobQueueObserversBuilder {++ constructor (private readonly meter: Meter) {++ }++ buildObservers () {+ this.meter.createObservableGauge('peertube_runner_job_queue_total', {+ description: 'Total jobs in the PeerTube runner job queue'+ }).addCallback(async observableResult => {+ const stats = await RunnerJobModel.getStats()++ const countMap = new Map(+ stats.map(({ jobType, state, count }) => [ `${jobType}:${state}`, count ])+ )++ for (const jobType of runnerJobTypes) {+ for (const [ stateNum, stateLabel ] of Object.entries(stateLabels)) {+ const count = countMap.get(`${jobType}:${stateNum}`) ?? 0+ observableResult.observe(count, { jobType, state: stateLabel })+ }+ }+ })+ }++}diff --git a/server/core/lib/opentelemetry/metric-helpers/runner-observers-builder.ts b/server/core/lib/opentelemetry/metric-helpers/runner-observers-builder.tsnew file mode 100644index 0000000000..551b323401--- /dev/null+++ b/server/core/lib/opentelemetry/metric-helpers/runner-observers-builder.ts@@ -0,0 +1,44 @@+import { Meter } from '@opentelemetry/api'+import { RunnerModel } from '@server/models/runner/runner.js'++export class RunnerObserversBuilder {++ constructor (private readonly meter: Meter) {++ }++ buildObservers () {+ const runnerCount = this.meter.createObservableGauge('peertube_runner_count', {+ description: 'Number of registered PeerTube runners'+ })++ const runnerInfo = this.meter.createObservableGauge('peertube_runner_info', {+ description: 'PeerTube runner metadata'+ })++ const secondsSinceLastContact = this.meter.createObservableGauge('peertube_runner_seconds_since_last_contact', {+ description: 'Seconds since the PeerTube runner last checked in',+ unit: 's'+ })++ this.meter.addBatchObservableCallback(async observableResult => {+ const runners = await RunnerModel.listAll()++ observableResult.observe(runnerCount, runners.length)++ for (const runner of runners) {+ const secondsSinceContact = (Date.now() - runner.lastContact.getTime()) / 1000++ observableResult.observe(secondsSinceLastContact, secondsSinceContact, { runnerName: runner.name })+ observableResult.observe(runnerInfo, 1, {+ runnerName: runner.name,+ description: runner.description ?? '',+ ip: runner.ip,+ version: runner.version ?? 'unknown',+ createdAt: runner.createdAt.toISOString()+ })+ }+ }, [ runnerCount, runnerInfo, secondsSinceLastContact ])+ }++}diff --git a/server/core/lib/opentelemetry/metrics.ts b/server/core/lib/opentelemetry/metrics.tsindex 2a57c681bd..f7a6db7bb1 100644--- a/server/core/lib/opentelemetry/metrics.ts+++ b/server/core/lib/opentelemetry/metrics.ts@@ -12,6 +12,8 @@ import {LivesObserversBuilder,NodeJSObserversBuilder,PlaybackMetrics,+ RunnerJobQueueObserversBuilder,+ RunnerObserversBuilder,StatsObserversBuilder,ViewersObserversBuilder} from './metric-helpers/index.js'@@ -72,6 +74,8 @@ class OpenTelemetryMetrics {new NodeJSObserversBuilder(this.meter).buildObservers()new JobQueueObserversBuilder(this.meter).buildObservers()+ new RunnerJobQueueObserversBuilder(this.meter).buildObservers()+ new RunnerObserversBuilder(this.meter).buildObservers()new StatsObserversBuilder(this.meter).buildObservers()new LivesObserversBuilder(this.meter).buildObservers()new ViewersObserversBuilder(this.meter).buildObservers()diff --git a/server/core/lib/runners/job-handlers/runner-job-handlers.ts b/server/core/lib/runners/job-handlers/runner-job-handlers.tsindex b85aaec194..4051fd1297 100644--- a/server/core/lib/runners/job-handlers/runner-job-handlers.ts+++ b/server/core/lib/runners/job-handlers/runner-job-handlers.ts@@ -19,6 +19,8 @@ const processors: Record<RunnerJobType, new() => AbstractJobHandler<unknown, Run'generate-video-storyboard': VideoStoryboardJobHandler}+export const runnerJobTypes = Object.keys(processors)+export function getRunnerJobHandlerClass (job: MRunnerJob) {return processors[job.type]}diff --git a/server/core/models/runner/runner-job.ts b/server/core/models/runner/runner-job.tsindex 1076cde935..70068bda0c 100644--- a/server/core/models/runner/runner-job.ts+++ b/server/core/models/runner/runner-job.ts@@ -10,7 +10,7 @@ import {import { isArray, isUUIDValid } from '@server/helpers/custom-validators/misc.js'import { CONSTRAINTS_FIELDS, RUNNER_JOB_STATES } from '@server/initializers/constants.js'import { MRunnerJob, MRunnerJobRunner, MRunnerJobRunnerParent } from '@server/types/models/runners/index.js'-import { Op, Transaction } from 'sequelize'+import { literal, Op, Transaction } from 'sequelize'import {AllowNull,BelongsTo,@@ -24,7 +24,7 @@ import {Table,UpdatedAt} from 'sequelize-typescript'-import { SequelizeModel, getSort, searchAttribute } from '../shared/index.js'+import { SequelizeModel, getSort, parseAggregateResult, searchAttribute } from '../shared/index.js'import { RunnerModel } from './runner.js'enum ScopeNames {@@ -290,6 +290,24 @@ export class RunnerJobModel extends SequelizeModel<RunnerJobModel> {]).then(([ total, data ]) => ({ total, data }))}+ static getStats () {+ return RunnerJobModel.findAll<MRunnerJob>({+ attributes: [+ 'type',+ 'state',+ [ literal('COUNT(*)'), 'count' ]+ ],+ group: [ 'type', 'state' ],+ raw: true+ }).then(rows => {+ return (rows as any[]).map(row => ({+ jobType: row.type as RunnerJobType,+ state: row.state as RunnerJobStateType,+ count: parseAggregateResult(row.count)+ }))+ })+ }+static updateDependantJobsOf (runnerJob: MRunnerJob) {const where = {dependsOnRunnerJobId: runnerJob.iddiff --git a/server/core/models/runner/runner.ts b/server/core/models/runner/runner.tsindex 924dacca8d..ef4606e685 100644--- a/server/core/models/runner/runner.ts+++ b/server/core/models/runner/runner.ts@@ -88,6 +88,10 @@ export class RunnerModel extends SequelizeModel<RunnerModel> {return RunnerModel.findOne(query)}+ static listAll () {+ return RunnerModel.findAll<MRunner>()+ }+static listForApi (options: {start: numbercount: number