Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(nodejs): push schema check results to the pushgateway (#130766)

Julian Beza7761f6a2577
diff --git a/docs/published/handbook/engineering/safe-django-migrations.md b/docs/published/handbook/engineering/safe-django-migrations.md
index 5e7a75a81409..7e69790e62b2 100644
--- a/docs/published/handbook/engineering/safe-django-migrations.md
+++ b/docs/published/handbook/engineering/safe-django-migrations.md
@@ -1008,6 +1008,7 @@ The check only covers Django tables in the main database.
Node Jest CI fails when Node SQL reads a table of an app that `products/db_routing.yaml` sends to a product database, or a table with an app prefix that no migration creates.
ClickHouse tables are out of scope.
The sqlx databases (persons, cyclotron, behavioral cohorts) are checked in shadow mode: the hook logs their state but does not hold the rollout yet.
+The hook pushes its Django and sqlx results to the Prometheus Pushgateway as `node_schema_check_` metrics, so alerts can fire on a missing migration that the logs alone would hide.
Column and table drops are the opposite case and stay code-first: remove every reader, deploy, then drop.
The check cannot help there, because Django applies the drop while the old Node pods still read the column.
diff --git a/nodejs/src/common/config.ts b/nodejs/src/common/config.ts
index 52bfb1118678..41c2566c2495 100644
--- a/nodejs/src/common/config.ts
+++ b/nodejs/src/common/config.ts
@@ -117,6 +117,10 @@ export type CommonConfig = BaseServerConfig & {
POSTGRES_BEHAVIORAL_COHORTS_USER: string
POSTGRES_BEHAVIORAL_COHORTS_PASSWORD: string
+ // Schema check hook Job (nodejs/src/schema-check)
+ SCHEMA_CHECK_PUSHGATEWAY_URL: string
+ SCHEMA_CHECK_APP: string
+
// PersonHog gRPC
PERSONHOG_ENABLED: boolean
/**
@@ -345,6 +349,10 @@ export function getDefaultCommonConfig(): CommonConfig {
POSTGRES_BEHAVIORAL_COHORTS_USER: 'postgres',
POSTGRES_BEHAVIORAL_COHORTS_PASSWORD: '',
+ // Schema check hook Job (nodejs/src/schema-check)
+ SCHEMA_CHECK_PUSHGATEWAY_URL: '',
+ SCHEMA_CHECK_APP: '',
+
// PersonHog gRPC
PERSONHOG_ENABLED: false,
PERSONHOG_ADDR: '',
diff --git a/nodejs/src/schema-check/check.test.ts b/nodejs/src/schema-check/check.test.ts
index 26fb4e92d5c0..4a5db4ce0eab 100644
--- a/nodejs/src/schema-check/check.test.ts
+++ b/nodejs/src/schema-check/check.test.ts
@@ -7,11 +7,13 @@ import {
MigrationStore,
SqlxMigrationRow,
SqlxMigrationStore,
+ SqlxShadowResult,
evaluateSqlxRows,
findMissingRequirements,
} from './check'
import { CliDependencies, EXIT_ERROR, EXIT_MISSING, EXIT_SATISFIED, runCli } from './cli'
import { DjangoRequirement, SqlxRequirement, buildManifest } from './manifest'
+import { renderMetrics } from './metrics-push'
const WORKFLOWS: DjangoRequirement = {
kind: 'django',
@@ -105,6 +107,46 @@ describe('schema check', () => {
})
})
+ describe('renderMetrics', () => {
+ const sqlxResult = (requirement: SqlxRequirement, status: SqlxShadowResult['status']): SqlxShadowResult => ({
+ requirement,
+ status,
+ detail: '',
+ })
+
+ it.each<[string, number, SqlxShadowResult[], string[]]>([
+ [
+ 'satisfied',
+ 0,
+ [sqlxResult(CYCLOTRON, 'satisfied')],
+ ['node_schema_check_sqlx_missing{database="cyclotron-node"} 0'],
+ ],
+ [
+ 'missing',
+ 0,
+ [sqlxResult(CYCLOTRON, 'missing')],
+ ['node_schema_check_sqlx_missing{database="cyclotron-node"} 1'],
+ ],
+ [
+ 'skipped, errored and dirty databases omitted',
+ 1,
+ [sqlxResult(PERSONS, 'skipped'), sqlxResult(CYCLOTRON, 'error'), sqlxResult(PERSONS, 'dirty')],
+ [],
+ ],
+ ['no sqlx database checked', 2, [sqlxResult(PERSONS, 'skipped')], []],
+ ])('%s', (_case, djangoExitCode, sqlx, expectedSqlxSamples) => {
+ const samples = renderMetrics(djangoExitCode, sqlx, 1700000000)
+ .split('\n')
+ .filter((line) => line && !line.startsWith('#'))
+
+ expect(samples).toEqual([
+ 'node_schema_check_last_run_timestamp_seconds 1700000000',
+ `node_schema_check_django_exit_code ${djangoExitCode}`,
+ ...expectedSqlxSamples,
+ ])
+ })
+ })
+
describe('runCli', () => {
let dir: string
let output: string[]
@@ -122,6 +164,7 @@ describe('schema check', () => {
): CliDependencies => ({
createStore: () => store,
createSqlxStore: () => sqlxStore,
+ fetch: jest.fn(),
sleep: (ms) => {
if (clock) {
clock.now += ms
@@ -202,6 +245,32 @@ describe('schema check', () => {
expect(output.some((line) => line.includes(expectedLog))).toBe(true)
})
+ it.each<[string, () => Promise<Response>]>([
+ ['the Pushgateway is unreachable', () => Promise.reject(new Error('connect ECONNREFUSED'))],
+ ['the Pushgateway answers 500', () => Promise.resolve(new Response('', { status: 500 }))],
+ ])('keeps the exit code when %s', async (_case, respond) => {
+ const manifestPath = writeManifest(JSON.stringify(buildManifest([WORKFLOWS], null)))
+ const store = new FakeMigrationStore([[]])
+ const fetchMock = jest.fn(respond)
+
+ const code = await runCli(
+ ['--manifest', manifestPath],
+ {
+ DATABASE_URL: 'postgres://x',
+ SCHEMA_CHECK_PUSHGATEWAY_URL: 'http://pushgateway.example.com:9091/',
+ SCHEMA_CHECK_APP: 'ingestion events',
+ },
+ { ...makeDeps(store), fetch: fetchMock }
+ )
+
+ expect(code).toBe(EXIT_MISSING)
+ expect(fetchMock).toHaveBeenCalledWith(
+ 'http://pushgateway.example.com:9091/metrics/job/node_schema_check/node_app/ingestion%20events',
+ expect.objectContaining({ method: 'PUT' })
+ )
+ expect(output.some((line) => line.startsWith('Warning: schema check metrics push failed'))).toBe(true)
+ })
+
it('keeps polling until the migration lands within --wait-seconds', async () => {
const clock = { now: 0 }
jest.spyOn(Date, 'now').mockImplementation(() => clock.now)
diff --git a/nodejs/src/schema-check/cli.ts b/nodejs/src/schema-check/cli.ts
index 0b86623216bc..2150d7758f68 100644
--- a/nodejs/src/schema-check/cli.ts
+++ b/nodejs/src/schema-check/cli.ts
@@ -7,6 +7,7 @@ import { buildBehavioralCohortsDatabaseUrl, buildPosthogDatabaseUrl } from '~/co
import {
MigrationStore,
SqlxMigrationStore,
+ SqlxShadowResult,
checkSqlxShadow,
describeError,
describeMissing,
@@ -20,6 +21,7 @@ import {
readManifest,
sqlxRequirements,
} from './manifest'
+import { FetchFunction, pushMetrics, renderMetrics } from './metrics-push'
import { PostgresMigrationStore, PostgresSqlxMigrationStore } from './postgres-migration-store'
export const EXIT_SATISFIED = 0
@@ -43,12 +45,14 @@ Reads DATABASE_URL, or POSTHOG_DB_NAME, POSTHOG_DB_USER, POSTHOG_DB_PASSWORD,
POSTHOG_POSTGRES_HOST and POSTHOG_POSTGRES_PORT, the same as the Node services.
sqlx databases use PERSONS_DATABASE_URL (or PERSONS_READONLY_DATABASE_URL),
CYCLOTRON_NODE_DATABASE_URL and BEHAVIORAL_COHORTS_DATABASE_URL (or
-POSTGRES_BEHAVIORAL_COHORTS_HOST, _USER and _PASSWORD).`
+POSTGRES_BEHAVIORAL_COHORTS_HOST, _USER and _PASSWORD).
+The result is pushed to SCHEMA_CHECK_PUSHGATEWAY_URL under SCHEMA_CHECK_APP when both are set.`
export interface CliDependencies {
createStore: (databaseUrl: string, connectTimeoutMs: number, statementTimeoutMs: number) => MigrationStore
createSqlxStore: (databaseUrl: string, connectTimeoutMs: number, statementTimeoutMs: number) => SqlxMigrationStore
sleep?: (ms: number) => Promise<void>
+ fetch: FetchFunction
log: (message: string) => void
logError: (message: string) => void
}
@@ -58,6 +62,9 @@ const defaultDependencies: CliDependencies = {
new PostgresMigrationStore({ databaseUrl, connectTimeoutMs, statementTimeoutMs }),
createSqlxStore: (databaseUrl, connectTimeoutMs, statementTimeoutMs) =>
new PostgresSqlxMigrationStore({ databaseUrl, connectTimeoutMs, statementTimeoutMs }),
+ // A one-shot Job, so it skips the request util's pooled agents and metrics that long-running services need.
+ // eslint-disable-next-line no-restricted-globals
+ fetch: (url, init) => fetch(url, init),
log: (message) => console.log(message),
logError: (message) => console.error(message),
}
@@ -83,6 +90,8 @@ const PERSONS_URL_KEYS = [
'PERSONS_READONLY_DATABASE_URL',
] as const satisfies readonly (keyof CommonConfig)[]
const CYCLOTRON_NODE_URL_KEY = 'CYCLOTRON_NODE_DATABASE_URL' satisfies keyof CdpConfig
+const PUSHGATEWAY_URL_KEY = 'SCHEMA_CHECK_PUSHGATEWAY_URL' satisfies keyof CommonConfig
+const PUSH_APP_KEY = 'SCHEMA_CHECK_APP' satisfies keyof CommonConfig
// Reads the same variables as the Node services, without the dev defaults, so an empty result means the app does not use the database.
export function resolveSqlxDatabaseUrl(database: SqlxDatabase, env: Record<string, string | undefined>): string {
@@ -107,7 +116,7 @@ async function logSqlxShadow(
env: Record<string, string | undefined>,
options: { connectTimeoutMs: number; statementTimeoutMs: number },
deps: CliDependencies
-): Promise<void> {
+): Promise<SqlxShadowResult[]> {
const results = await Promise.all(
requirements.map((requirement) => {
const databaseUrl = resolveSqlxDatabaseUrl(requirement.database, env)
@@ -122,6 +131,7 @@ async function logSqlxShadow(
`Shadow sqlx check (does not block): ${result.requirement.database} ${result.status}, ${result.detail}`
)
}
+ return results
}
function parseSeconds(name: string, value: string | undefined, fallback: number): number {
@@ -135,6 +145,24 @@ function parseSeconds(name: string, value: string | undefined, fallback: number)
return seconds
}
+// Earlier exits push nothing, because the alerts only read runs that reached the database.
+async function pushResult(
+ djangoExitCode: number,
+ sqlx: SqlxShadowResult[],
+ env: Record<string, string | undefined>,
+ deps: CliDependencies
+): Promise<number> {
+ const pushgatewayUrl = env[PUSHGATEWAY_URL_KEY]
+ const app = env[PUSH_APP_KEY]
+ if (!pushgatewayUrl || !app) {
+ deps.log('Schema check metrics push skipped, SCHEMA_CHECK_PUSHGATEWAY_URL or SCHEMA_CHECK_APP is not set')
+ return djangoExitCode
+ }
+ const body = renderMetrics(djangoExitCode, sqlx, Math.floor(Date.now() / 1000))
+ await pushMetrics(body, { pushgatewayUrl, app }, deps.fetch, deps.logError)
+ return djangoExitCode
+}
+
export async function runCli(
argv: string[],
env: Record<string, string | undefined>,
@@ -201,7 +229,7 @@ export async function runCli(
log: deps.log,
})
// Runs after the Django wait so that the log shows the sqlx state at the moment the Deployment would roll.
- await logSqlxShadow(
+ const sqlxResults = await logSqlxShadow(
sqlx,
env,
{
@@ -213,15 +241,15 @@ export async function runCli(
if (outcome.status === 'satisfied') {
deps.log('All required Django migrations are applied.')
- return EXIT_SATISFIED
+ return pushResult(EXIT_SATISFIED, sqlxResults, env, deps)
}
if (outcome.status === 'error') {
deps.logError(`Schema check error: could not read django_migrations: ${describeError(outcome.error)}`)
- return EXIT_ERROR
+ return pushResult(EXIT_ERROR, sqlxResults, env, deps)
}
deps.logError('Required Django migrations are not applied yet:')
describeMissing(outcome.missing, outcome.latestApplied).forEach((line) => deps.logError(` ${line}`))
- return EXIT_MISSING
+ return pushResult(EXIT_MISSING, sqlxResults, env, deps)
}
if (require.main === module) {
diff --git a/nodejs/src/schema-check/metrics-push.ts b/nodejs/src/schema-check/metrics-push.ts
new file mode 100644
index 000000000000..622bd00827c3
--- /dev/null
+++ b/nodejs/src/schema-check/metrics-push.ts
@@ -0,0 +1,55 @@
+import { SqlxShadowResult, describeError } from './check'
+
+export const PUSH_TIMEOUT_MS = 3000
+
+export interface PushTarget {
+ pushgatewayUrl: string
+ app: string
+}
+
+export type FetchFunction = (url: string, init: RequestInit) => Promise<Response>
+
+// The charts alert rules read these names. Error, dirty and skipped results get no sample, so they
+// can never replace an earlier "missing" with 0.
+export function renderMetrics(djangoExitCode: number, sqlx: SqlxShadowResult[], nowSeconds: number): string {
+ const sqlxSamples = sqlx
+ .filter((result) => result.status === 'satisfied' || result.status === 'missing')
+ .map(
+ (result) =>
+ `node_schema_check_sqlx_missing{database="${result.requirement.database}"} ${result.status === 'missing' ? 1 : 0}`
+ )
+ return (
+ [
+ '# TYPE node_schema_check_last_run_timestamp_seconds gauge',
+ `node_schema_check_last_run_timestamp_seconds ${nowSeconds}`,
+ '# TYPE node_schema_check_django_exit_code gauge',
+ `node_schema_check_django_exit_code ${djangoExitCode}`,
+ ...(sqlxSamples.length > 0 ? ['# TYPE node_schema_check_sqlx_missing gauge', ...sqlxSamples] : []),
+ ].join('\n') + '\n'
+ )
+}
+
+// A failed push only logs, because the Pushgateway being down must never block a rollout.
+export async function pushMetrics(
+ body: string,
+ target: PushTarget,
+ fetch: FetchFunction,
+ logError: (message: string) => void
+): Promise<void> {
+ const url = `${target.pushgatewayUrl.replace(/\/+$/, '')}/metrics/job/node_schema_check/node_app/${encodeURIComponent(target.app)}`
+ try {
+ const response = await fetch(url, {
+ method: 'PUT',
+ headers: { 'Content-Type': 'text/plain; version=0.0.4' },
+ body,
+ signal: AbortSignal.timeout(PUSH_TIMEOUT_MS),
+ })
+ if (!response.ok) {
+ logError(`Warning: schema check metrics push failed with HTTP ${response.status}`)
+ }
+ } catch (error) {
+ logError(
+ `Warning: schema check metrics push failed: ${describeError(error instanceof Error ? error : new Error(String(error)))}`
+ )
+ }
+}