Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(nodejs): fail ci when node sql reads a table the schema check cannot cover (#130063)

Julian Bez6a6138fa5d04
diff --git a/.github/workflows/ci-nodejs.yml b/.github/workflows/ci-nodejs.yml
index 8d0c591a8f7b..1bf0dbc6a661 100644
--- a/.github/workflows/ci-nodejs.yml
+++ b/.github/workflows/ci-nodejs.yml
@@ -141,6 +141,13 @@ jobs:
# A migration change makes the test job build Postgres from the Django
# models, and the Jest integration tests read product tables too.
- 'products/*/backend/migrations/**'
+ # The schema-check test reads every Django app's migrations and the routing
+ # file to prove that each table Node SQL reads has a migration the check covers.
+ - 'posthog/*/migrations/**'
+ - 'products/db_routing.yaml'
+ - 'rust/persons_migrations/**'
+ - 'rust/cyclotron-node-migrations/**'
+ - 'rust/behavioral_cohorts_migrations/**'
- 'posthog/plugins/**'
# Django reads this to decide which filter globals it accepts, and the only
# check that it still matches the runtime type is a jest test.
@@ -336,6 +343,9 @@ jobs:
count=$(printf '%s\n' $NODE_SRC_FILES | grep -c . || true)
[ "$count" -le 200 ] || full over_ceiling
[ -n "${files// /}" ] || full empty_file_list
+ # This test reads Node SQL and the migrations from disk, so no import connects it to the
+ # changed files and --findRelatedTests would never select it.
+ files="$files src/schema-check/derive-requirements.test.ts"
echo "::notice::selective jest over $count changed Node source file(s)"
{
diff --git a/docs/published/handbook/engineering/safe-django-migrations.md b/docs/published/handbook/engineering/safe-django-migrations.md
index 820636934e4f..2adc7309f5dc 100644
--- a/docs/published/handbook/engineering/safe-django-migrations.md
+++ b/docs/published/handbook/engineering/safe-django-migrations.md
@@ -996,10 +996,16 @@ Before deploying risky migrations:
Node and Rust services deploy on their own schedule, often before the Django deploy that runs `bin/migrate`, so a merged migration is not yet live when their new image starts.
Code in those services that reads a new table or column needs the migration applied in production first: merge the migration in its own PR and wait for it to deploy, or make the reader tolerate the missing column.
-For Node, the schema check in `nodejs/src/schema-check/` enforces this once its deploy hook is enabled.
+For Node, the schema check in `nodejs/src/schema-check/` enforces this on every Node app that connects to the main Postgres.
+An app without a write role on it opts out explicitly in the charts repo, because the hook reads the primary.
Each Node image records the Django migrations it needs at build time, and the hook holds the rollout until `django_migrations` has all of them.
If Django has not applied them when the hook's wait ends, the sync fails and the old pods keep serving until a later sync passes.
+The check only covers Django tables in the main database.
+Node Jest CI fails when Node SQL reads a table of an app that `products/db_routing.yaml` sends to a product database, or a table with an app prefix that no migration creates.
+ClickHouse tables are out of scope.
+The sqlx databases (persons, cyclotron, behavioral cohorts) are checked in shadow mode: the hook logs their state but does not hold the rollout yet.
Column and table drops are the opposite case and stay code-first: remove every reader, deploy, then drop.
+The check cannot help there, because Django applies the drop while the old Node pods still read the column.
## Getting Help
diff --git a/nodejs/src/schema-check/derive-requirements.test.ts b/nodejs/src/schema-check/derive-requirements.test.ts
index 0d403f70fcbe..27e085e380cd 100644
--- a/nodejs/src/schema-check/derive-requirements.test.ts
+++ b/nodejs/src/schema-check/derive-requirements.test.ts
@@ -49,9 +49,25 @@ const FIXTURE_FILES: Record<string, string> = {
'products/stamphog/backend/apps.py':
'class StamphogConfig(AppConfig):\n name = "products.stamphog.backend"\n label = "stamphog"\n',
'products/stamphog/backend/migrations/0001_initial.py': 'migrations.CreateModel(name="Review", fields=[])\n',
- 'products/stamphog/backend/migrations/max_migration.txt': '0001_initial\n',
+ 'products/stamphog/backend/migrations/0002_take_widget.py':
+ 'migrations.CreateModel(name="Widget", fields=[], options={"db_table": "legacy_widget"})\n',
+ 'products/stamphog/backend/migrations/max_migration.txt': '0002_take_widget\n',
'products/db_routing.yaml': 'routes:\n - app_label: stamphog\n database: stamphog\n',
+ 'products/replay/backend/apps.py':
+ 'class ReplayConfig(AppConfig):\n name = "products.replay.backend"\n label = "replay"\n',
+ 'products/replay/backend/migrations/0001_initial.py': 'migrations.CreateModel(name="Lens", fields=[])\n',
+ 'products/replay/backend/migrations/0002_rename.py':
+ 'migrations.RenameModel(old_name="Lens", new_name="Observation")\n',
+ 'products/replay/backend/migrations/max_migration.txt': '0002_rename\n',
+
+ 'products/rawsql/backend/apps.py':
+ 'class RawSqlConfig(AppConfig):\n name = "products.rawsql.backend"\n label = "rawsql"\n',
+ 'products/rawsql/backend/migrations/0001_initial.py':
+ '# A CREATE TABLE for each team would be slow.\n' +
+ 'migrations.RunSQL("CREATE TABLE IF NOT EXISTS rawsql_store (id bigint)")\n',
+ 'products/rawsql/backend/migrations/max_migration.txt': '0001_initial\n',
+
'products/unused/backend/apps.py':
'class UnusedConfig(AppConfig):\n name = "products.unused.backend"\n label = "unused"\n',
'products/unused/backend/migrations/0001_initial.py': 'migrations.CreateModel(name="Thing", fields=[])\n',
@@ -63,18 +79,31 @@ const FIXTURE_FILES: Record<string, string> = {
'CREATE TABLE IF NOT EXISTS cyclotron_jobs (id uuid);\n',
// The latest version creates no table, but Node still needs the database at that version.
'rust/cyclotron-node-migrations/20260201000001_add_index.sql': 'CREATE INDEX idx_jobs ON cyclotron_jobs (id);\n',
+ 'rust/cyclotron-node-migrations/20260301000001_add.batch.index.sql':
+ 'CREATE INDEX idx_batch ON cyclotron_jobs (id);\n',
+ 'rust/cyclotron-node-migrations/20260301000001_add.batch.index.down.sql': 'DROP INDEX idx_batch;\n',
'rust/behavioral_cohorts_migrations/20260301000001_initial.sql': 'CREATE TABLE cohort_membership (id bigint);\n',
'nodejs/src/reader.ts': [
+ "import { PostgresUse } from './postgres'",
+ 'const glob = `${KEY}/*`',
"const a = 'SELECT * FROM posthog_team'",
"const b = 'SELECT * FROM posthog_session_session'",
"const c = 'SELECT * FROM posthog_survey'",
"const d = 'SELECT * FROM legacy_widget'",
"const e = 'SELECT * FROM tracing_span'",
"const f = 'SELECT * FROM clickhouse_events'",
- "const g = 'SELECT * FROM stamphog_review'",
+ 'const g = \'SELECT * FROM posthog_team AS "t", "public"."stamphog_review" r\'',
"const i = 'SELECT * FROM cyclotron_jobs'",
+ 'for (const row of rows) {}',
+ "const j = 'SELECT * FROM replay_observation'",
+ "const k = 'INSERT INTO RAWSQL_STORE VALUES (1)'",
+ 'const l = "SELECT \'//\' AS marker FROM posthog_ghost"',
+ '// Rows move from posthog_ghost_comment once the job runs.',
].join('\n'),
+ 'nodejs/src/clickhouse-reader.ts':
+ "import { ClickHouseClient } from '@clickhouse/client'\nconst q = 'SELECT * FROM tracing_spans_distributed'\n",
+ 'nodejs/src/pool-reader.ts': "/* note */ const q = 'SELECT * FROM posthog_pool_ghost'\n/* next */\n",
'nodejs/src/reader.test.ts': "const h = 'SELECT * FROM unused_thing'\n",
}
@@ -95,12 +124,14 @@ describe('deriveSchemaRequirements', () => {
})
it('requires the leaf of every Django app and sqlx database whose tables nodejs/src reads', () => {
- const { requirements, tablesByRequirement } = deriveSchemaRequirements(repoRoot)
+ const { requirements, tablesByRequirement, findings } = deriveSchemaRequirements(repoRoot)
expect(requirements).toEqual([
{ kind: 'django', database: 'default', app: 'legacy', migration: '0002_move_widget' },
{ kind: 'django', database: 'default', app: 'posthog', migration: '0002_team_flag' },
{ kind: 'django', database: 'default', app: 'posthog_session', migration: '0001_initial' },
+ { kind: 'django', database: 'default', app: 'rawsql', migration: '0001_initial' },
+ { kind: 'django', database: 'default', app: 'replay', migration: '0002_rename' },
{ kind: 'django', database: 'default', app: 'surveys', migration: '0001_initial' },
{
kind: 'django',
@@ -110,24 +141,34 @@ describe('deriveSchemaRequirements', () => {
replaces: ['0001_initial', '0002_span_kind'],
},
{ kind: 'django', database: 'default', app: 'widgets', migration: '0002_widget_table' },
- { kind: 'sqlx', database: 'cyclotron-node', versions: [20260101000001, 20260201000001] },
+ { kind: 'sqlx', database: 'cyclotron-node', versions: [20260101000001, 20260201000001, 20260301000001] },
])
expect(Object.fromEntries(tablesByRequirement)).toEqual({
legacy: ['legacy_widget'],
posthog: ['posthog_team'],
posthog_session: ['posthog_session_session'],
+ rawsql: ['rawsql_store'],
+ replay: ['replay_observation'],
surveys: ['posthog_survey'],
tracing: ['tracing_span'],
widgets: ['legacy_widget'],
'sqlx:cyclotron-node': ['cyclotron_jobs'],
})
+ // clickhouse_events has no app prefix, so it is out of scope. The comment adds no reference.
+ expect(findings).toEqual([
+ 'legacy_widget: owned by stamphog, which products/db_routing.yaml sends to a product database',
+ 'posthog_ghost: no scanned migration creates it, although its name starts with posthog_',
+ 'posthog_pool_ghost: no scanned migration creates it, although its name starts with posthog_',
+ 'stamphog_review: owned by stamphog, which products/db_routing.yaml sends to a product database',
+ ])
})
- it('finds the workflows config, team and cyclotron job tables on the real repo', () => {
- const { tablesByRequirement } = deriveSchemaRequirements(path.resolve(__dirname, '../../..'))
+ it('finds the workflows config, team and cyclotron job tables on the real repo, and no uncovered table', () => {
+ const { tablesByRequirement, findings } = deriveSchemaRequirements(path.resolve(__dirname, '../../..'))
expect(tablesByRequirement.get('workflows')).toContain('workflows_teamworkflowsconfig')
expect(tablesByRequirement.get('posthog')).toContain('posthog_team')
expect(tablesByRequirement.get('sqlx:cyclotron-node')).toContain('cyclotron_jobs')
+ expect(findings).toEqual([])
})
})
diff --git a/nodejs/src/schema-check/derive-requirements.ts b/nodejs/src/schema-check/derive-requirements.ts
index 031f87f81cca..651639bb4fe1 100644
--- a/nodejs/src/schema-check/derive-requirements.ts
+++ b/nodejs/src/schema-check/derive-requirements.ts
@@ -23,9 +23,12 @@ const SKIPPED_DIR_NAMES = new Set(['node_modules', '__pycache__', 'dist', 'front
const MIGRATION_FILE = /^\d{4}_\w+\.py$/
// sqlx takes the version from the file name prefix. A .down.sql file reverts its .up.sql twin and adds no version.
-const SQLX_MIGRATION_FILE = /^(\d+)_[\w-]+(?:\.up)?\.sql$/
-const CREATE_TABLE = /\bCREATE\s+TABLE\s+(?:IF\s+NOT\s+EXISTS\s+)?(?:public\.)?"?(\w+)"?/gi
+const SQLX_MIGRATION_FILE = /^(\d+)_(?!.*\.down\.sql$).+\.sql$/
+// The lookahead stops a backtrack from taking IF as the name, for example in `CREATE TABLE IF NOT EXISTS {name}`.
+const CREATE_TABLE = /\bCREATE\s+TABLE\s+(?:IF\s+NOT\s+EXISTS\s+)?(?:public\.)?"?(?!(?:if|only)\b)(\w+)"?/gi
const CREATE_MODEL_NAME = /CreateModel\(\s*name\s*=\s*['"](\w+)['"]/g
+const RENAME_MODEL_CALL = /RenameModel\(([^)]*)\)/g
+const NEW_NAME_ARGUMENT = /\bnew_name\s*=\s*['"](\w+)['"]/
const DB_TABLE_OPTION = /['"]db_table['"]\s*:\s*['"](\w+)['"]/g
const TABLE_ARGUMENT = /\b(?:db_)?table\s*=\s*['"](\w+)['"]/g
const APP_CONFIG_LABEL = /^\s*label\s*=\s*['"](\w+)['"]/m
@@ -34,6 +37,18 @@ const SQUASH_REPLACES = /^\s*replaces\s*=\s*\[([\s\S]*?)\]/m
const MIGRATION_KEY = /\(\s*['"](\w+)['"]\s*,\s*['"](\w+)['"]\s*\)/g
const ROUTED_APP_LABEL = /app_label:\s*['"]?(\w+)['"]?/g
const LOWERCASE_IDENTIFIER = /\b[a-z][a-z0-9_]+\b/g
+const SQL_TABLE_NAME = String.raw`(?:"?public"?\.)?"?(?!(?:if|only)\b)[a-z_]\w*"?`
+const SQL_TABLE_REFERENCE = new RegExp(
+ String.raw`\b(?:FROM|JOIN|INTO|UPDATE|TABLE|TRUNCATE)\s+(?:ONLY\s+)?(?:IF\s+(?:NOT\s+)?EXISTS\s+)?` +
+ String.raw`((?:${SQL_TABLE_NAME}(?:\s+(?:AS\s+)?"?\w+"?)?\s*,\s*)*${SQL_TABLE_NAME})`,
+ 'gi'
+)
+const SQL_TABLE_LIST_ITEM = new RegExp(String.raw`^\s*(${SQL_TABLE_NAME})`, 'i')
+// Whole comment lines only: prose reuses SQL keywords, and a `//` inside a SQL string must not cut the query.
+const TS_COMMENT_LINE = /^\s*(?:\/\/.*|\/\*(?:(?!\*\/)[\s\S])*\*\/)$/gm
+// ClickHouse tables share app prefixes, so a file that names ClickHouse and no Postgres client gives no findings.
+const POSTGRES_CLIENT_USE = /\bPostgresUse\b|\bPostgresRouter\b|\bTransactionClient\b|\bLazyPool\b|from 'pg'/
+const CLICKHOUSE_USE = /clickhouse/i
export interface DjangoApp {
label: string
@@ -45,6 +60,7 @@ export interface DerivedSchemaRequirements {
requirements: Requirement[]
// Keyed by Django app label, or by "sqlx:<database>" for sqlx requirements.
tablesByRequirement: Map<string, string[]>
+ findings: string[]
}
export interface SqlxMigrationDir {
@@ -135,10 +151,18 @@ export function collectTableOwners(apps: DjangoApp[]): Map<string, Set<string>>
const migrationFiles = readdirSync(app.migrationsDir).filter((name) => MIGRATION_FILE.test(name))
for (const fileName of migrationFiles) {
const text = readFileSync(path.join(app.migrationsDir, fileName), 'utf8')
- for (const modelName of matchAll(text, CREATE_MODEL_NAME)) {
+ const renamedModels = matchAll(text, RENAME_MODEL_CALL).flatMap(
+ (call) => call.match(NEW_NAME_ARGUMENT)?.[1] ?? []
+ )
+ for (const modelName of [...matchAll(text, CREATE_MODEL_NAME), ...renamedModels]) {
addOwner(`${app.label}_${modelName.toLowerCase()}`, app.label)
}
- for (const table of [...matchAll(text, DB_TABLE_OPTION), ...matchAll(text, TABLE_ARGUMENT)]) {
+ const tables = [...matchAll(text, DB_TABLE_OPTION), ...matchAll(text, TABLE_ARGUMENT)]
+ // A RunSQL can create a table. Prose like "a CREATE TABLE would ..." has no underscore, unlike a Django table.
+ const rawSqlTables = matchAll(text, CREATE_TABLE)
+ .map((name) => name.toLowerCase())
+ .filter((name) => name.includes('_'))
+ for (const table of [...tables, ...rawSqlTables]) {
addOwner(table, app.label)
}
}
@@ -164,6 +188,68 @@ export function collectNodeIdentifiers(nodeSourceDir: string): Set<string> {
return identifiers
}
+export interface NodeSqlTables {
+ all: Set<string>
+ // Without the files that only query ClickHouse.
+ postgres: Set<string>
+}
+
+export function collectNodeSqlTables(nodeSourceDir: string): NodeSqlTables {
+ const sourceFiles = listFilesRecursively(
+ nodeSourceDir,
+ (name) => name.endsWith('.ts') && !name.includes('.test.')
+ ).filter((file) => !file.split(path.sep).includes('__tests__'))
+
+ const tables: NodeSqlTables = { all: new Set(), postgres: new Set() }
+ for (const file of sourceFiles) {
+ const code = readFileSync(file, 'utf8').replace(TS_COMMENT_LINE, '')
+ const clickhouseOnly = CLICKHOUSE_USE.test(code) && !POSTGRES_CLIENT_USE.test(code)
+ for (const list of matchAll(code, SQL_TABLE_REFERENCE)) {
+ for (const item of list.split(',')) {
+ const name = item.match(SQL_TABLE_LIST_ITEM)?.[1] ?? ''
+ const table = name
+ .replace(/"/g, '')
+ .replace(/^public\./i, '')
+ .toLowerCase()
+ tables.all.add(table)
+ if (!clickhouseOnly) {
+ tables.postgres.add(table)
+ }
+ }
+ }
+ }
+ return tables
+}
+
+// Flags routed tables (product database, not checked) and prefixed tables no scanned migration creates.
+export function findUncoveredTables(
+ nodeSqlTables: Set<string>,
+ tableOwners: Map<string, Set<string>>,
+ routedTableOwners: Map<string, Set<string>>,
+ sqlxTables: Set<string>,
+ appLabels: string[]
+): string[] {
+ const findings: string[] = []
+ for (const table of [...nodeSqlTables].sort()) {
+ // A model moved into a routed app can keep its old db_table, so a routed owner wins over an old one.
+ const routedOwners = routedTableOwners.get(table)
+ if (routedOwners) {
+ findings.push(
+ `${table}: owned by ${[...routedOwners].sort().join(', ')}, which products/db_routing.yaml sends to a product database`
+ )
+ continue
+ }
+ if (tableOwners.has(table) || sqlxTables.has(table)) {
+ continue
+ }
+ const prefixLabel = appLabels.find((label) => table.startsWith(`${label}_`))
+ if (prefixLabel) {
+ findings.push(`${table}: no scanned migration creates it, although its name starts with ${prefixLabel}_`)
+ }
+ }
+ return findings
+}
+
export function readSquashReplaces(app: DjangoApp): string[] | undefined {
const text = readFileSync(path.join(app.migrationsDir, `${app.leafMigration}.py`), 'utf8')
const replacesBlock = text.match(SQUASH_REPLACES)?.[1]
@@ -210,9 +296,15 @@ export function readSqlxMigrationDir(repoRoot: string, database: SqlxDatabase):
export function deriveSchemaRequirements(repoRoot: string): DerivedSchemaRequirements {
const routedLabels = readProductRoutedLabels(repoRoot)
- const apps = findDjangoApps(repoRoot).filter((app) => !routedLabels.has(app.label))
+ const allApps = findDjangoApps(repoRoot)
+ const apps = allApps.filter((app) => !routedLabels.has(app.label))
const tableOwners = collectTableOwners(apps)
- const nodeIdentifiers = collectNodeIdentifiers(path.join(repoRoot, NODEJS_SOURCE_DIR))
+ const nodeSqlTables = collectNodeSqlTables(path.join(repoRoot, NODEJS_SOURCE_DIR))
+ // Postgres folds unquoted names to lowercase, so `FROM POSTHOG_TEAM` reads posthog_team too.
+ const nodeIdentifiers = new Set([
+ ...collectNodeIdentifiers(path.join(repoRoot, NODEJS_SOURCE_DIR)),
+ ...nodeSqlTables.all,
+ ])
const tablesByRequirement = new Map<string, string[]>()
for (const table of [...tableOwners.keys()].sort()) {
@@ -242,8 +334,10 @@ export function deriveSchemaRequirements(repoRoot: string): DerivedSchemaRequire
})
const sqlxRequirements: SqlxRequirement[] = []
+ const sqlxTables = new Set<string>()
for (const database of SQLX_DATABASES) {
const migrationDir = readSqlxMigrationDir(repoRoot, database)
+ migrationDir.tables.forEach((table) => sqlxTables.add(table))
const tables = [...migrationDir.tables].filter((table) => nodeIdentifiers.has(table)).sort()
if (tables.length === 0) {
continue
@@ -252,5 +346,13 @@ export function deriveSchemaRequirements(repoRoot: string): DerivedSchemaRequire
sqlxRequirements.push({ kind: 'sqlx', database, versions: migrationDir.versions })
}
- return { requirements: [...djangoRequirements, ...sqlxRequirements], tablesByRequirement }
+ const findings = findUncoveredTables(
+ nodeSqlTables.postgres,
+ tableOwners,
+ collectTableOwners(allApps.filter((app) => routedLabels.has(app.label))),
+ sqlxTables,
+ allApps.map((app) => app.label)
+ )
+
+ return { requirements: [...djangoRequirements, ...sqlxRequirements], tablesByRequirement, findings }
}
diff --git a/nodejs/src/schema-check/generate.ts b/nodejs/src/schema-check/generate.ts
index 71857819f225..977d297b1c3d 100644
--- a/nodejs/src/schema-check/generate.ts
+++ b/nodejs/src/schema-check/generate.ts
@@ -6,8 +6,10 @@ import { deriveSchemaRequirements } from './derive-requirements'
import { buildManifest } from './manifest'
const USAGE =
- 'Usage: node nodejs/dist/schema-check/generate.js --repo-root PATH --output PATH [--source-commit SHA]\n' +
- 'Writes the Django migrations this Node build needs, for nodejs/dist/schema-check/cli.js to check.'
+ 'Usage: node nodejs/dist/schema-check/generate.js --repo-root PATH --output PATH [--source-commit SHA] [--strict]\n' +
+ 'Writes the Django migrations this Node build needs, for nodejs/dist/schema-check/cli.js to check.\n' +
+ '--strict exits 1 when Node SQL reads a table that the check cannot cover. The image build runs without it and only\n' +
+ 'warns; the real-repo Jest test asserts the same rule in CI.'
function main(): number {
const { values } = parseArgs({
@@ -15,6 +17,7 @@ function main(): number {
'repo-root': { type: 'string' },
output: { type: 'string' },
'source-commit': { type: 'string' },
+ strict: { type: 'boolean' },
help: { type: 'boolean' },
},
})
@@ -23,7 +26,7 @@ function main(): number {
return values.help ? 0 : 2
}
- const { requirements, tablesByRequirement } = deriveSchemaRequirements(path.resolve(values['repo-root']))
+ const { requirements, tablesByRequirement, findings } = deriveSchemaRequirements(path.resolve(values['repo-root']))
if (!requirements.some((requirement) => requirement.kind === 'django')) {
console.error('Found no Django tables in nodejs/src. The source mounts are probably wrong.')
return 1
@@ -43,7 +46,13 @@ function main(): number {
: `sqlx ${requirement.database} (${requirement.versions.length} versions, latest ${requirement.versions[requirement.versions.length - 1]})`
console.log(` ${target}: ${(tablesByRequirement.get(key) ?? []).join(', ')}`)
}
- return 0
+
+ if (findings.length === 0) {
+ return 0
+ }
+ console.error(`Node SQL reads ${findings.length} tables that the schema check cannot cover:`)
+ findings.forEach((finding) => console.error(` ${finding}`))
+ return values.strict ? 1 : 0
}
process.exitCode = main()