:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(nodejs): check every sqlx version and read persons through the replica url (#130062)
Julian Bezefe6fbcf3179
diff --git a/nodejs/src/schema-check/check.test.ts b/nodejs/src/schema-check/check.test.tsindex b866c6a61a26..26fb4e92d5c0 100644--- a/nodejs/src/schema-check/check.test.ts+++ b/nodejs/src/schema-check/check.test.ts@@ -27,7 +27,12 @@ const TRACING_SQUASH: DjangoRequirement = {replaces: ['0001_initial', '0002_span_kind'],}-const CYCLOTRON: SqlxRequirement = { kind: 'sqlx', database: 'cyclotron-node', version: 20260201000001 }+const CYCLOTRON: SqlxRequirement = {+ kind: 'sqlx',+ database: 'cyclotron-node',+ versions: [20260101000001, 20260201000001],+}+const PERSONS: SqlxRequirement = { kind: 'sqlx', database: 'persons', versions: [20250101000001] }class FakeSqlxMigrationStore implements SqlxMigrationStore {constructor(private readonly rows: SqlxMigrationRow[]) {}@@ -77,8 +82,16 @@ describe('schema check', () => {describe('evaluateSqlxRows', () => {it.each<[string, SqlxMigrationRow[], string]>([- ['version applied', [{ version: 20260201000001, success: true }], 'satisfied'],- ['version not applied', [], 'missing'],+ [+ 'every version applied',+ [+ { version: 20260101000001, success: true },+ { version: 20260201000001, success: true },+ ],+ 'satisfied',+ ],+ ['a lower version merged late is not applied', [{ version: 20260201000001, success: true }], 'missing'],+ ['no version applied', [], 'missing'],['a failed migration on the database',[@@ -137,6 +150,14 @@ describe('schema check', () => {'of an unknown kind',JSON.stringify({ schema_version: 1, digest: 'x', requirements: [{ ...WORKFLOWS, kind: 'flyway' }] }),],+ [+ 'a sqlx entry without versions',+ JSON.stringify({+ schema_version: 1,+ digest: 'x',+ requirements: [WORKFLOWS, { ...CYCLOTRON, versions: [] }],+ }),+ ],])('exits with an error when the manifest is %s', async (_case, content) => {const manifestPath = content === null ? path.join(dir, 'absent.json') : writeManifest(content)const store = new FakeMigrationStore([[]])@@ -166,8 +187,9 @@ describe('schema check', () => {'cyclotron-node dirty',],['not configured for this app', {}, [], 'cyclotron-node skipped'],+ ['replica-only', { PERSONS_READONLY_DATABASE_URL: 'postgres://p' }, [], 'persons missing'],])('logs a %s sqlx database and still exits 0', async (_case, extraEnv, sqlxRows, expectedLog) => {- const manifestPath = writeManifest(JSON.stringify(buildManifest([WORKFLOWS, CYCLOTRON], null)))+ const manifestPath = writeManifest(JSON.stringify(buildManifest([WORKFLOWS, CYCLOTRON, PERSONS], null)))const store = new FakeMigrationStore([[{ app: 'workflows', name: '0040_frequency_cap_hold' }]])const code = await runCli(diff --git a/nodejs/src/schema-check/check.ts b/nodejs/src/schema-check/check.tsindex 571a1a2e778f..19baeddfb785 100644--- a/nodejs/src/schema-check/check.ts+++ b/nodejs/src/schema-check/check.ts@@ -138,10 +138,17 @@ export function evaluateSqlxRows(requirement: SqlxRequirement, rows: SqlxMigratiif (failed.length > 0) {return { requirement, status: 'dirty', detail: `failed migrations recorded: ${failed.join(', ')}` }}- if (rows.some((row) => row.version === requirement.version)) {- return { requirement, status: 'satisfied', detail: `version ${requirement.version} is applied` }+ const applied = new Set(rows.map((row) => row.version))+ const missing = requirement.versions.filter((version) => !applied.has(version))+ if (missing.length === 0) {+ const latest = requirement.versions[requirement.versions.length - 1]+ return {+ requirement,+ status: 'satisfied',+ detail: `all ${requirement.versions.length} versions are applied, latest ${latest}`,+ }}- return { requirement, status: 'missing', detail: `version ${requirement.version} is not applied` }+ return { requirement, status: 'missing', detail: `versions not applied: ${missing.join(', ')}` }}export async function checkSqlxShadow(@@ -153,7 +160,7 @@ export async function checkSqlxShadow(return { requirement, status: 'skipped', detail: 'no database URL in this environment' }}try {- return evaluateSqlxRows(requirement, await store.fetchRows([requirement.version]))+ return evaluateSqlxRows(requirement, await store.fetchRows(requirement.versions))} catch (error) {return {requirement,diff --git a/nodejs/src/schema-check/cli.ts b/nodejs/src/schema-check/cli.tsindex 0f9d5288d3d6..0b86623216bc 100644--- a/nodejs/src/schema-check/cli.ts+++ b/nodejs/src/schema-check/cli.ts@@ -1,6 +1,7 @@import { parseArgs } from 'util'-import { getDefaultCommonConfig } from '~/common/config'+import type { CdpConfig } from '~/cdp/config'+import { CommonConfig, getDefaultCommonConfig } from '~/common/config'import { buildBehavioralCohortsDatabaseUrl, buildPosthogDatabaseUrl } from '~/common/config/database-url'import {@@ -40,8 +41,9 @@ Options:Reads DATABASE_URL, or POSTHOG_DB_NAME, POSTHOG_DB_USER, POSTHOG_DB_PASSWORD,POSTHOG_POSTGRES_HOST and POSTHOG_POSTGRES_PORT, the same as the Node services.-sqlx databases use PERSONS_DATABASE_URL, CYCLOTRON_NODE_DATABASE_URL and-BEHAVIORAL_COHORTS_DATABASE_URL (or POSTGRES_BEHAVIORAL_COHORTS_HOST, _USER and _PASSWORD).`+sqlx databases use PERSONS_DATABASE_URL (or PERSONS_READONLY_DATABASE_URL),+CYCLOTRON_NODE_DATABASE_URL and BEHAVIORAL_COHORTS_DATABASE_URL (or+POSTGRES_BEHAVIORAL_COHORTS_HOST, _USER and _PASSWORD).`export interface CliDependencies {createStore: (databaseUrl: string, connectTimeoutMs: number, statementTimeoutMs: number) => MigrationStore@@ -75,13 +77,22 @@ export function resolveDatabaseUrl(env: Record<string, string | undefined>): str})}+// The keys must stay names of the Node config, so a rename there fails to compile here instead of turning into a skip.+const PERSONS_URL_KEYS = [+ 'PERSONS_DATABASE_URL',+ 'PERSONS_READONLY_DATABASE_URL',+] as const satisfies readonly (keyof CommonConfig)[]+const CYCLOTRON_NODE_URL_KEY = 'CYCLOTRON_NODE_DATABASE_URL' satisfies keyof CdpConfig+// Reads the same variables as the Node services, without the dev defaults, so an empty result means the app does not use the database.export function resolveSqlxDatabaseUrl(database: SqlxDatabase, env: Record<string, string | undefined>): string {if (database === 'persons') {- return env.PERSONS_DATABASE_URL ?? ''+ // Most ingestion apps get only the read-only persons URL. Replica lag can only make the check wait longer.+ const key = PERSONS_URL_KEYS.find((name) => env[name])+ return key ? env[key]! : ''}if (database === 'cyclotron-node') {- return env.CYCLOTRON_NODE_DATABASE_URL ?? ''+ return env[CYCLOTRON_NODE_URL_KEY] || ''}return buildBehavioralCohortsDatabaseUrl({BEHAVIORAL_COHORTS_DATABASE_URL: env.BEHAVIORAL_COHORTS_DATABASE_URL ?? '',diff --git a/nodejs/src/schema-check/derive-requirements.test.ts b/nodejs/src/schema-check/derive-requirements.test.tsindex 471de25fb07d..0d403f70fcbe 100644--- a/nodejs/src/schema-check/derive-requirements.test.ts+++ b/nodejs/src/schema-check/derive-requirements.test.ts@@ -110,7 +110,7 @@ describe('deriveSchemaRequirements', () => {replaces: ['0001_initial', '0002_span_kind'],},{ kind: 'django', database: 'default', app: 'widgets', migration: '0002_widget_table' },- { kind: 'sqlx', database: 'cyclotron-node', version: 20260201000001 },+ { kind: 'sqlx', database: 'cyclotron-node', versions: [20260101000001, 20260201000001] },])expect(Object.fromEntries(tablesByRequirement)).toEqual({legacy: ['legacy_widget'],diff --git a/nodejs/src/schema-check/derive-requirements.ts b/nodejs/src/schema-check/derive-requirements.tsindex 4dac95978f1f..031f87f81cca 100644--- a/nodejs/src/schema-check/derive-requirements.ts+++ b/nodejs/src/schema-check/derive-requirements.ts@@ -49,7 +49,7 @@ export interface DerivedSchemaRequirements {export interface SqlxMigrationDir {database: SqlxDatabase- latestVersion: number+ versions: number[]tables: Set<string>}@@ -190,22 +190,22 @@ export function readSqlxMigrationDir(repoRoot: string, database: SqlxDatabase):if (!existsSync(dir)) {throw new Error(`${dir} does not exist`)}- let latestVersion = 0+ const versions = new Set<number>()const tables = new Set<string>()for (const fileName of readdirSync(dir)) {const version = fileName.match(SQLX_MIGRATION_FILE)?.[1]if (version === undefined) {continue}- latestVersion = Math.max(latestVersion, Number(version))+ versions.add(Number(version))for (const table of matchAll(readFileSync(path.join(dir, fileName), 'utf8'), CREATE_TABLE)) {tables.add(table.toLowerCase())}}- if (latestVersion === 0) {+ if (versions.size === 0) {throw new Error(`${dir} has no sqlx migrations`)}- return { database, latestVersion, tables }+ return { database, versions: [...versions].sort((a, b) => a - b), tables }}export function deriveSchemaRequirements(repoRoot: string): DerivedSchemaRequirements {@@ -249,7 +249,7 @@ export function deriveSchemaRequirements(repoRoot: string): DerivedSchemaRequirecontinue}tablesByRequirement.set(`sqlx:${database}`, tables)- sqlxRequirements.push({ kind: 'sqlx', database, version: migrationDir.latestVersion })+ sqlxRequirements.push({ kind: 'sqlx', database, versions: migrationDir.versions })}return { requirements: [...djangoRequirements, ...sqlxRequirements], tablesByRequirement }diff --git a/nodejs/src/schema-check/generate.ts b/nodejs/src/schema-check/generate.tsindex 62a2fa2adf75..71857819f225 100644--- a/nodejs/src/schema-check/generate.ts+++ b/nodejs/src/schema-check/generate.ts@@ -40,7 +40,7 @@ function main(): number {const target =requirement.kind === 'django'? `${requirement.app}.${requirement.migration}`- : `sqlx ${requirement.database}@${requirement.version}`+ : `sqlx ${requirement.database} (${requirement.versions.length} versions, latest ${requirement.versions[requirement.versions.length - 1]})`console.log(` ${target}: ${(tablesByRequirement.get(key) ?? []).join(', ')}`)}return 0diff --git a/nodejs/src/schema-check/manifest.ts b/nodejs/src/schema-check/manifest.tsindex f4eba842ecef..7c3348ecbc33 100644--- a/nodejs/src/schema-check/manifest.ts+++ b/nodejs/src/schema-check/manifest.ts@@ -25,7 +25,9 @@ export interface DjangoRequirement {export interface SqlxRequirement {kind: 'sqlx'database: SqlxDatabase- version: number+ // Every version in the migration directory, ascending. A version is the timestamp in the file name, so a+ // late merge can add a version below the highest one, and `sqlx migrate run` still applies it.+ versions: number[]}export type Requirement = DjangoRequirement | SqlxRequirement@@ -84,10 +86,15 @@ function parseSqlxRequirement(item: Record<string, unknown>, index: number): Sqlif (!database) {throw new ManifestError(`requirements[${index}].database must be one of ${SQLX_DATABASES.join(', ')}`)}- if (typeof item.version !== 'number' || !Number.isSafeInteger(item.version) || item.version <= 0) {- throw new ManifestError(`requirements[${index}].version must be a positive integer`)+ const versions = item.versions+ if (+ !Array.isArray(versions) ||+ versions.length === 0 ||+ !versions.every((version) => typeof version === 'number' && Number.isSafeInteger(version) && version > 0)+ ) {+ throw new ManifestError(`requirements[${index}].versions must be a non-empty list of positive integers`)}- return { kind: 'sqlx', database, version: item.version }+ return { kind: 'sqlx', database, versions: [...new Set<number>(versions)].sort((a, b) => a - b) }}// An unknown kind is an error. Skipping it would report a pass for a requirement that nothing checked.