:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(nodejs): push schema check results to the pushgateway (#130766)
Julian Beza7761f6a2577
diff --git a/docs/published/handbook/engineering/safe-django-migrations.md b/docs/published/handbook/engineering/safe-django-migrations.mdindex 5e7a75a81409..7e69790e62b2 100644--- a/docs/published/handbook/engineering/safe-django-migrations.md+++ b/docs/published/handbook/engineering/safe-django-migrations.md@@ -1008,6 +1008,7 @@ The check only covers Django tables in the main database.Node Jest CI fails when Node SQL reads a table of an app that `products/db_routing.yaml` sends to a product database, or a table with an app prefix that no migration creates.ClickHouse tables are out of scope.The sqlx databases (persons, cyclotron, behavioral cohorts) are checked in shadow mode: the hook logs their state but does not hold the rollout yet.+The hook pushes its Django and sqlx results to the Prometheus Pushgateway as `node_schema_check_` metrics, so alerts can fire on a missing migration that the logs alone would hide.Column and table drops are the opposite case and stay code-first: remove every reader, deploy, then drop.The check cannot help there, because Django applies the drop while the old Node pods still read the column.diff --git a/nodejs/src/common/config.ts b/nodejs/src/common/config.tsindex 52bfb1118678..41c2566c2495 100644--- a/nodejs/src/common/config.ts+++ b/nodejs/src/common/config.ts@@ -117,6 +117,10 @@ export type CommonConfig = BaseServerConfig & {POSTGRES_BEHAVIORAL_COHORTS_USER: stringPOSTGRES_BEHAVIORAL_COHORTS_PASSWORD: string+ // Schema check hook Job (nodejs/src/schema-check)+ SCHEMA_CHECK_PUSHGATEWAY_URL: string+ SCHEMA_CHECK_APP: string+// PersonHog gRPCPERSONHOG_ENABLED: boolean/**@@ -345,6 +349,10 @@ export function getDefaultCommonConfig(): CommonConfig {POSTGRES_BEHAVIORAL_COHORTS_USER: 'postgres',POSTGRES_BEHAVIORAL_COHORTS_PASSWORD: '',+ // Schema check hook Job (nodejs/src/schema-check)+ SCHEMA_CHECK_PUSHGATEWAY_URL: '',+ SCHEMA_CHECK_APP: '',+// PersonHog gRPCPERSONHOG_ENABLED: false,PERSONHOG_ADDR: '',diff --git a/nodejs/src/schema-check/check.test.ts b/nodejs/src/schema-check/check.test.tsindex 26fb4e92d5c0..4a5db4ce0eab 100644--- a/nodejs/src/schema-check/check.test.ts+++ b/nodejs/src/schema-check/check.test.ts@@ -7,11 +7,13 @@ import {MigrationStore,SqlxMigrationRow,SqlxMigrationStore,+ SqlxShadowResult,evaluateSqlxRows,findMissingRequirements,} from './check'import { CliDependencies, EXIT_ERROR, EXIT_MISSING, EXIT_SATISFIED, runCli } from './cli'import { DjangoRequirement, SqlxRequirement, buildManifest } from './manifest'+import { renderMetrics } from './metrics-push'const WORKFLOWS: DjangoRequirement = {kind: 'django',@@ -105,6 +107,46 @@ describe('schema check', () => {})})+ describe('renderMetrics', () => {+ const sqlxResult = (requirement: SqlxRequirement, status: SqlxShadowResult['status']): SqlxShadowResult => ({+ requirement,+ status,+ detail: '',+ })++ it.each<[string, number, SqlxShadowResult[], string[]]>([+ [+ 'satisfied',+ 0,+ [sqlxResult(CYCLOTRON, 'satisfied')],+ ['node_schema_check_sqlx_missing{database="cyclotron-node"} 0'],+ ],+ [+ 'missing',+ 0,+ [sqlxResult(CYCLOTRON, 'missing')],+ ['node_schema_check_sqlx_missing{database="cyclotron-node"} 1'],+ ],+ [+ 'skipped, errored and dirty databases omitted',+ 1,+ [sqlxResult(PERSONS, 'skipped'), sqlxResult(CYCLOTRON, 'error'), sqlxResult(PERSONS, 'dirty')],+ [],+ ],+ ['no sqlx database checked', 2, [sqlxResult(PERSONS, 'skipped')], []],+ ])('%s', (_case, djangoExitCode, sqlx, expectedSqlxSamples) => {+ const samples = renderMetrics(djangoExitCode, sqlx, 1700000000)+ .split('\n')+ .filter((line) => line && !line.startsWith('#'))++ expect(samples).toEqual([+ 'node_schema_check_last_run_timestamp_seconds 1700000000',+ `node_schema_check_django_exit_code ${djangoExitCode}`,+ ...expectedSqlxSamples,+ ])+ })+ })+describe('runCli', () => {let dir: stringlet output: string[]@@ -122,6 +164,7 @@ describe('schema check', () => {): CliDependencies => ({createStore: () => store,createSqlxStore: () => sqlxStore,+ fetch: jest.fn(),sleep: (ms) => {if (clock) {clock.now += ms@@ -202,6 +245,32 @@ describe('schema check', () => {expect(output.some((line) => line.includes(expectedLog))).toBe(true)})+ it.each<[string, () => Promise<Response>]>([+ ['the Pushgateway is unreachable', () => Promise.reject(new Error('connect ECONNREFUSED'))],+ ['the Pushgateway answers 500', () => Promise.resolve(new Response('', { status: 500 }))],+ ])('keeps the exit code when %s', async (_case, respond) => {+ const manifestPath = writeManifest(JSON.stringify(buildManifest([WORKFLOWS], null)))+ const store = new FakeMigrationStore([[]])+ const fetchMock = jest.fn(respond)++ const code = await runCli(+ ['--manifest', manifestPath],+ {+ DATABASE_URL: 'postgres://x',+ SCHEMA_CHECK_PUSHGATEWAY_URL: 'http://pushgateway.example.com:9091/',+ SCHEMA_CHECK_APP: 'ingestion events',+ },+ { ...makeDeps(store), fetch: fetchMock }+ )++ expect(code).toBe(EXIT_MISSING)+ expect(fetchMock).toHaveBeenCalledWith(+ 'http://pushgateway.example.com:9091/metrics/job/node_schema_check/node_app/ingestion%20events',+ expect.objectContaining({ method: 'PUT' })+ )+ expect(output.some((line) => line.startsWith('Warning: schema check metrics push failed'))).toBe(true)+ })+it('keeps polling until the migration lands within --wait-seconds', async () => {const clock = { now: 0 }jest.spyOn(Date, 'now').mockImplementation(() => clock.now)diff --git a/nodejs/src/schema-check/cli.ts b/nodejs/src/schema-check/cli.tsindex 0b86623216bc..2150d7758f68 100644--- a/nodejs/src/schema-check/cli.ts+++ b/nodejs/src/schema-check/cli.ts@@ -7,6 +7,7 @@ import { buildBehavioralCohortsDatabaseUrl, buildPosthogDatabaseUrl } from '~/coimport {MigrationStore,SqlxMigrationStore,+ SqlxShadowResult,checkSqlxShadow,describeError,describeMissing,@@ -20,6 +21,7 @@ import {readManifest,sqlxRequirements,} from './manifest'+import { FetchFunction, pushMetrics, renderMetrics } from './metrics-push'import { PostgresMigrationStore, PostgresSqlxMigrationStore } from './postgres-migration-store'export const EXIT_SATISFIED = 0@@ -43,12 +45,14 @@ Reads DATABASE_URL, or POSTHOG_DB_NAME, POSTHOG_DB_USER, POSTHOG_DB_PASSWORD,POSTHOG_POSTGRES_HOST and POSTHOG_POSTGRES_PORT, the same as the Node services.sqlx databases use PERSONS_DATABASE_URL (or PERSONS_READONLY_DATABASE_URL),CYCLOTRON_NODE_DATABASE_URL and BEHAVIORAL_COHORTS_DATABASE_URL (or-POSTGRES_BEHAVIORAL_COHORTS_HOST, _USER and _PASSWORD).`+POSTGRES_BEHAVIORAL_COHORTS_HOST, _USER and _PASSWORD).+The result is pushed to SCHEMA_CHECK_PUSHGATEWAY_URL under SCHEMA_CHECK_APP when both are set.`export interface CliDependencies {createStore: (databaseUrl: string, connectTimeoutMs: number, statementTimeoutMs: number) => MigrationStorecreateSqlxStore: (databaseUrl: string, connectTimeoutMs: number, statementTimeoutMs: number) => SqlxMigrationStoresleep?: (ms: number) => Promise<void>+ fetch: FetchFunctionlog: (message: string) => voidlogError: (message: string) => void}@@ -58,6 +62,9 @@ const defaultDependencies: CliDependencies = {new PostgresMigrationStore({ databaseUrl, connectTimeoutMs, statementTimeoutMs }),createSqlxStore: (databaseUrl, connectTimeoutMs, statementTimeoutMs) =>new PostgresSqlxMigrationStore({ databaseUrl, connectTimeoutMs, statementTimeoutMs }),+ // A one-shot Job, so it skips the request util's pooled agents and metrics that long-running services need.+ // eslint-disable-next-line no-restricted-globals+ fetch: (url, init) => fetch(url, init),log: (message) => console.log(message),logError: (message) => console.error(message),}@@ -83,6 +90,8 @@ const PERSONS_URL_KEYS = ['PERSONS_READONLY_DATABASE_URL',] as const satisfies readonly (keyof CommonConfig)[]const CYCLOTRON_NODE_URL_KEY = 'CYCLOTRON_NODE_DATABASE_URL' satisfies keyof CdpConfig+const PUSHGATEWAY_URL_KEY = 'SCHEMA_CHECK_PUSHGATEWAY_URL' satisfies keyof CommonConfig+const PUSH_APP_KEY = 'SCHEMA_CHECK_APP' satisfies keyof CommonConfig// Reads the same variables as the Node services, without the dev defaults, so an empty result means the app does not use the database.export function resolveSqlxDatabaseUrl(database: SqlxDatabase, env: Record<string, string | undefined>): string {@@ -107,7 +116,7 @@ async function logSqlxShadow(env: Record<string, string | undefined>,options: { connectTimeoutMs: number; statementTimeoutMs: number },deps: CliDependencies-): Promise<void> {+): Promise<SqlxShadowResult[]> {const results = await Promise.all(requirements.map((requirement) => {const databaseUrl = resolveSqlxDatabaseUrl(requirement.database, env)@@ -122,6 +131,7 @@ async function logSqlxShadow(`Shadow sqlx check (does not block): ${result.requirement.database} ${result.status}, ${result.detail}`)}+ return results}function parseSeconds(name: string, value: string | undefined, fallback: number): number {@@ -135,6 +145,24 @@ function parseSeconds(name: string, value: string | undefined, fallback: number)return seconds}+// Earlier exits push nothing, because the alerts only read runs that reached the database.+async function pushResult(+ djangoExitCode: number,+ sqlx: SqlxShadowResult[],+ env: Record<string, string | undefined>,+ deps: CliDependencies+): Promise<number> {+ const pushgatewayUrl = env[PUSHGATEWAY_URL_KEY]+ const app = env[PUSH_APP_KEY]+ if (!pushgatewayUrl || !app) {+ deps.log('Schema check metrics push skipped, SCHEMA_CHECK_PUSHGATEWAY_URL or SCHEMA_CHECK_APP is not set')+ return djangoExitCode+ }+ const body = renderMetrics(djangoExitCode, sqlx, Math.floor(Date.now() / 1000))+ await pushMetrics(body, { pushgatewayUrl, app }, deps.fetch, deps.logError)+ return djangoExitCode+}+export async function runCli(argv: string[],env: Record<string, string | undefined>,@@ -201,7 +229,7 @@ export async function runCli(log: deps.log,})// Runs after the Django wait so that the log shows the sqlx state at the moment the Deployment would roll.- await logSqlxShadow(+ const sqlxResults = await logSqlxShadow(sqlx,env,{@@ -213,15 +241,15 @@ export async function runCli(if (outcome.status === 'satisfied') {deps.log('All required Django migrations are applied.')- return EXIT_SATISFIED+ return pushResult(EXIT_SATISFIED, sqlxResults, env, deps)}if (outcome.status === 'error') {deps.logError(`Schema check error: could not read django_migrations: ${describeError(outcome.error)}`)- return EXIT_ERROR+ return pushResult(EXIT_ERROR, sqlxResults, env, deps)}deps.logError('Required Django migrations are not applied yet:')describeMissing(outcome.missing, outcome.latestApplied).forEach((line) => deps.logError(` ${line}`))- return EXIT_MISSING+ return pushResult(EXIT_MISSING, sqlxResults, env, deps)}if (require.main === module) {diff --git a/nodejs/src/schema-check/metrics-push.ts b/nodejs/src/schema-check/metrics-push.tsnew file mode 100644index 000000000000..622bd00827c3--- /dev/null+++ b/nodejs/src/schema-check/metrics-push.ts@@ -0,0 +1,55 @@+import { SqlxShadowResult, describeError } from './check'++export const PUSH_TIMEOUT_MS = 3000++export interface PushTarget {+ pushgatewayUrl: string+ app: string+}++export type FetchFunction = (url: string, init: RequestInit) => Promise<Response>++// The charts alert rules read these names. Error, dirty and skipped results get no sample, so they+// can never replace an earlier "missing" with 0.+export function renderMetrics(djangoExitCode: number, sqlx: SqlxShadowResult[], nowSeconds: number): string {+ const sqlxSamples = sqlx+ .filter((result) => result.status === 'satisfied' || result.status === 'missing')+ .map(+ (result) =>+ `node_schema_check_sqlx_missing{database="${result.requirement.database}"} ${result.status === 'missing' ? 1 : 0}`+ )+ return (+ [+ '# TYPE node_schema_check_last_run_timestamp_seconds gauge',+ `node_schema_check_last_run_timestamp_seconds ${nowSeconds}`,+ '# TYPE node_schema_check_django_exit_code gauge',+ `node_schema_check_django_exit_code ${djangoExitCode}`,+ ...(sqlxSamples.length > 0 ? ['# TYPE node_schema_check_sqlx_missing gauge', ...sqlxSamples] : []),+ ].join('\n') + '\n'+ )+}++// A failed push only logs, because the Pushgateway being down must never block a rollout.+export async function pushMetrics(+ body: string,+ target: PushTarget,+ fetch: FetchFunction,+ logError: (message: string) => void+): Promise<void> {+ const url = `${target.pushgatewayUrl.replace(/\/+$/, '')}/metrics/job/node_schema_check/node_app/${encodeURIComponent(target.app)}`+ try {+ const response = await fetch(url, {+ method: 'PUT',+ headers: { 'Content-Type': 'text/plain; version=0.0.4' },+ body,+ signal: AbortSignal.timeout(PUSH_TIMEOUT_MS),+ })+ if (!response.ok) {+ logError(`Warning: schema check metrics push failed with HTTP ${response.status}`)+ }+ } catch (error) {+ logError(+ `Warning: schema check metrics push failed: ${describeError(error instanceof Error ? error : new Error(String(error)))}`+ )+ }+}