:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(nodejs): add a build-time migration manifest and schema check (#128242)
Julian Bez8e87f32eed4b
diff --git a/Dockerfile.node b/Dockerfile.nodeindex 5d5fe54065dc..0a30cabc2752 100644--- a/Dockerfile.node+++ b/Dockerfile.node@@ -85,6 +85,22 @@ RUN NODE_OPTIONS="--max-old-space-size=16384" bin/turbo --filter=@posthog/replay# Then build the nodejs servicesRUN NODE_OPTIONS="--max-old-space-size=16384" bin/turbo --filter=@posthog/nodejs build+# Record the Django and sqlx migrations this build reads, so the deploy-time schema check can wait for them+# (nodejs/src/schema-check). Bind mounts give the generator the migration files without adding the+# Python and Rust trees to any layer, and BuildKit reruns this step when any mounted file changes.+ARG COMMIT_HASH+RUN --mount=type=bind,source=posthog,target=/schema-src/posthog \+ --mount=type=bind,source=ee,target=/schema-src/ee \+ --mount=type=bind,source=products,target=/schema-src/products \+ --mount=type=bind,source=nodejs/src,target=/schema-src/nodejs/src \+ --mount=type=bind,source=rust/persons_migrations,target=/schema-src/rust/persons_migrations \+ --mount=type=bind,source=rust/cyclotron-node-migrations,target=/schema-src/rust/cyclotron-node-migrations \+ --mount=type=bind,source=rust/behavioral_cohorts_migrations,target=/schema-src/rust/behavioral_cohorts_migrations \+ node nodejs/dist/schema-check/generate.js \+ --repo-root /schema-src \+ --output nodejs/schema-requirements.json \+ --source-commit "${COMMIT_HASH:-unknown}"+## Runtime stage: Minimal image with only nodejs runtime dependencies@@ -150,6 +166,7 @@ COPY --from=nodejs-build --chown=posthog:posthog /code/nodejs/dist /code/nodejs/COPY --from=nodejs-build --chown=posthog:posthog /code/node_modules /code/node_modulesCOPY --from=nodejs-build --chown=posthog:posthog /code/nodejs/node_modules /code/nodejs/node_modulesCOPY --from=nodejs-build --chown=posthog:posthog /code/nodejs/package.json /code/nodejs/package.json+COPY --from=nodejs-build --chown=posthog:posthog /code/nodejs/schema-requirements.json /code/nodejs/schema-requirements.jsonCOPY --from=nodejs-build --chown=posthog:posthog /code/common/replay-headless/dist /code/common/replay-headless/distCOPY --from=nodejs-build --chown=posthog:posthog /code/common/replay-headless/package.json /code/common/replay-headless/package.jsonCOPY --from=nodejs-build --chown=posthog:posthog /code/nodejs/assets /code/nodejs/assetsdiff --git a/docs/published/handbook/engineering/safe-django-migrations.md b/docs/published/handbook/engineering/safe-django-migrations.mdindex b7ac2b574aff..820636934e4f 100644--- a/docs/published/handbook/engineering/safe-django-migrations.md+++ b/docs/published/handbook/engineering/safe-django-migrations.md@@ -992,6 +992,15 @@ Before deploying risky migrations:- **Important:** With `atomic=False`, migrations may partially apply changes. If a migration fails, Django won't automatically roll back the changes. Always verify schema consistency after failed runs and be prepared to manually fix partial states.- Remember that infra team may roll back deployments at any time for any reason (performance issues, alerts, etc.) - plan for this+### 5. Apply the Migration Before Node or Rust Code Reads the Column++Node and Rust services deploy on their own schedule, often before the Django deploy that runs `bin/migrate`, so a merged migration is not yet live when their new image starts.+Code in those services that reads a new table or column needs the migration applied in production first: merge the migration in its own PR and wait for it to deploy, or make the reader tolerate the missing column.+For Node, the schema check in `nodejs/src/schema-check/` enforces this once its deploy hook is enabled.+Each Node image records the Django migrations it needs at build time, and the hook holds the rollout until `django_migrations` has all of them.+If Django has not applied them when the hook's wait ends, the sync fails and the old pods keep serving until a later sync passes.+Column and table drops are the opposite case and stay code-first: remove every reader, deploy, then drop.+## Getting HelpIf you're unsure about a migration:diff --git a/nodejs/src/common/config/config.ts b/nodejs/src/common/config/config.tsindex bc895f2ba550..acf5234d88a0 100644--- a/nodejs/src/common/config/config.ts+++ b/nodejs/src/common/config/config.ts@@ -10,6 +10,7 @@ import { getDefaultLogsIngestionConsumerConfig, getDefaultTracesIngestionConsumeimport { PluginsServerConfig, ValueMatcher, stringToPluginServerMode } from '~/types'import { stringToBoolean } from '../utils/env-utils'+import { buildBehavioralCohortsDatabaseUrl, buildPosthogDatabaseUrl } from './database-url'// Re-export for backwards compatibilityexport { DEFAULT_HTTP_SERVER_PORT } from '~/common/config'@@ -56,28 +57,9 @@ export function overrideWithEnv(}const newConfig: PluginsServerConfig = { ...tmpConfig }- if (!newConfig.DATABASE_URL && !newConfig.POSTHOG_DB_NAME) {- throw Error(- 'You must specify either DATABASE_URL or the database options POSTHOG_DB_NAME, POSTHOG_DB_USER, POSTHOG_DB_PASSWORD, POSTHOG_POSTGRES_HOST, POSTHOG_POSTGRES_PORT!'- )- }-- if (!newConfig.DATABASE_URL) {- const encodedUser = encodeURIComponent(newConfig.POSTHOG_DB_USER)- const encodedPassword = encodeURIComponent(newConfig.POSTHOG_DB_PASSWORD)- newConfig.DATABASE_URL = `postgres://${encodedUser}:${encodedPassword}@${newConfig.POSTHOG_POSTGRES_HOST}:${newConfig.POSTHOG_POSTGRES_PORT}/${newConfig.POSTHOG_DB_NAME}`- }+ newConfig.DATABASE_URL = buildPosthogDatabaseUrl(newConfig)- if (- !newConfig.BEHAVIORAL_COHORTS_DATABASE_URL &&- newConfig.POSTGRES_BEHAVIORAL_COHORTS_HOST &&- newConfig.POSTGRES_BEHAVIORAL_COHORTS_USER &&- newConfig.POSTGRES_BEHAVIORAL_COHORTS_PASSWORD- ) {- const encodedUser = encodeURIComponent(newConfig.POSTGRES_BEHAVIORAL_COHORTS_USER)- const encodedPassword = encodeURIComponent(newConfig.POSTGRES_BEHAVIORAL_COHORTS_PASSWORD)- newConfig.BEHAVIORAL_COHORTS_DATABASE_URL = `postgres://${encodedUser}:${encodedPassword}@${newConfig.POSTGRES_BEHAVIORAL_COHORTS_HOST}:5432/behavioral_cohorts`- }+ newConfig.BEHAVIORAL_COHORTS_DATABASE_URL = buildBehavioralCohortsDatabaseUrl(newConfig)return newConfig}diff --git a/nodejs/src/common/config/database-url.ts b/nodejs/src/common/config/database-url.tsnew file mode 100644index 000000000000..419f9f6b73ac--- /dev/null+++ b/nodejs/src/common/config/database-url.ts@@ -0,0 +1,49 @@+import type { CommonConfig } from '~/common/config'++export type PosthogDatabaseUrlConfig = Pick<+ CommonConfig,+ | 'DATABASE_URL'+ | 'POSTHOG_DB_NAME'+ | 'POSTHOG_DB_USER'+ | 'POSTHOG_DB_PASSWORD'+ | 'POSTHOG_POSTGRES_HOST'+ | 'POSTHOG_POSTGRES_PORT'+>++export type BehavioralCohortsDatabaseUrlConfig = Pick<+ CommonConfig,+ | 'BEHAVIORAL_COHORTS_DATABASE_URL'+ | 'POSTGRES_BEHAVIORAL_COHORTS_HOST'+ | 'POSTGRES_BEHAVIORAL_COHORTS_USER'+ | 'POSTGRES_BEHAVIORAL_COHORTS_PASSWORD'+>++export function buildPosthogDatabaseUrl(config: PosthogDatabaseUrlConfig): string {+ if (config.DATABASE_URL) {+ return config.DATABASE_URL+ }++ if (!config.POSTHOG_DB_NAME) {+ throw Error(+ 'You must specify either DATABASE_URL or the database options POSTHOG_DB_NAME, POSTHOG_DB_USER, POSTHOG_DB_PASSWORD, POSTHOG_POSTGRES_HOST, POSTHOG_POSTGRES_PORT!'+ )+ }++ const encodedUser = encodeURIComponent(config.POSTHOG_DB_USER)+ const encodedPassword = encodeURIComponent(config.POSTHOG_DB_PASSWORD)+ return `postgres://${encodedUser}:${encodedPassword}@${config.POSTHOG_POSTGRES_HOST}:${config.POSTHOG_POSTGRES_PORT}/${config.POSTHOG_DB_NAME}`+}++export function buildBehavioralCohortsDatabaseUrl(config: BehavioralCohortsDatabaseUrlConfig): string {+ if (+ config.BEHAVIORAL_COHORTS_DATABASE_URL ||+ !config.POSTGRES_BEHAVIORAL_COHORTS_HOST ||+ !config.POSTGRES_BEHAVIORAL_COHORTS_USER ||+ !config.POSTGRES_BEHAVIORAL_COHORTS_PASSWORD+ ) {+ return config.BEHAVIORAL_COHORTS_DATABASE_URL+ }+ const encodedUser = encodeURIComponent(config.POSTGRES_BEHAVIORAL_COHORTS_USER)+ const encodedPassword = encodeURIComponent(config.POSTGRES_BEHAVIORAL_COHORTS_PASSWORD)+ return `postgres://${encodedUser}:${encodedPassword}@${config.POSTGRES_BEHAVIORAL_COHORTS_HOST}:5432/behavioral_cohorts`+}diff --git a/nodejs/src/schema-check/check.test.ts b/nodejs/src/schema-check/check.test.tsnew file mode 100644index 000000000000..b866c6a61a26--- /dev/null+++ b/nodejs/src/schema-check/check.test.ts@@ -0,0 +1,216 @@+import { mkdtempSync, rmSync, writeFileSync } from 'fs'+import { tmpdir } from 'os'+import path from 'path'++import {+ AppliedMigration,+ MigrationStore,+ SqlxMigrationRow,+ SqlxMigrationStore,+ evaluateSqlxRows,+ findMissingRequirements,+} from './check'+import { CliDependencies, EXIT_ERROR, EXIT_MISSING, EXIT_SATISFIED, runCli } from './cli'+import { DjangoRequirement, SqlxRequirement, buildManifest } from './manifest'++const WORKFLOWS: DjangoRequirement = {+ kind: 'django',+ database: 'default',+ app: 'workflows',+ migration: '0040_frequency_cap_hold',+}+const TRACING_SQUASH: DjangoRequirement = {+ kind: 'django',+ database: 'default',+ app: 'tracing',+ migration: '0001_squash_initial',+ replaces: ['0001_initial', '0002_span_kind'],+}++const CYCLOTRON: SqlxRequirement = { kind: 'sqlx', database: 'cyclotron-node', version: 20260201000001 }++class FakeSqlxMigrationStore implements SqlxMigrationStore {+ constructor(private readonly rows: SqlxMigrationRow[]) {}++ fetchRows(): Promise<SqlxMigrationRow[]> {+ return Promise.resolve(this.rows)+ }+}++class FakeMigrationStore implements MigrationStore {+ fetchAppliedCalls = 0++ constructor(private readonly appliedPerPoll: AppliedMigration[][]) {}++ fetchApplied(): Promise<AppliedMigration[]> {+ const index = Math.min(this.fetchAppliedCalls, this.appliedPerPoll.length - 1)+ this.fetchAppliedCalls += 1+ return Promise.resolve(this.appliedPerPoll[index])+ }++ fetchLatestApplied(): Promise<Map<string, string>> {+ return Promise.resolve(new Map([['workflows', '0039_previous']]))+ }+}++describe('schema check', () => {+ describe('findMissingRequirements', () => {+ it.each<[string, DjangoRequirement, AppliedMigration[], boolean]>([+ ['leaf applied', WORKFLOWS, [{ app: 'workflows', name: '0040_frequency_cap_hold' }], false],+ ['leaf not applied', WORKFLOWS, [{ app: 'workflows', name: '0039_previous' }], true],+ ['same name in another app', WORKFLOWS, [{ app: 'posthog', name: '0040_frequency_cap_hold' }], true],+ ['squash row applied', TRACING_SQUASH, [{ app: 'tracing', name: '0001_squash_initial' }], false],+ [+ 'every replaced migration applied',+ TRACING_SQUASH,+ [+ { app: 'tracing', name: '0001_initial' },+ { app: 'tracing', name: '0002_span_kind' },+ ],+ false,+ ],+ ['only some replaced migrations applied', TRACING_SQUASH, [{ app: 'tracing', name: '0001_initial' }], true],+ ])('%s', (_case, requirement, applied, expectMissing) => {+ expect(findMissingRequirements([requirement], applied)).toEqual(expectMissing ? [requirement] : [])+ })+ })++ describe('evaluateSqlxRows', () => {+ it.each<[string, SqlxMigrationRow[], string]>([+ ['version applied', [{ version: 20260201000001, success: true }], 'satisfied'],+ ['version not applied', [], 'missing'],+ [+ 'a failed migration on the database',+ [+ { version: 20260201000001, success: true },+ { version: 20260301000001, success: false },+ ],+ 'dirty',+ ],+ ])('%s', (_case, rows, expectedStatus) => {+ expect(evaluateSqlxRows(CYCLOTRON, rows).status).toBe(expectedStatus)+ })+ })++ describe('runCli', () => {+ let dir: string+ let output: string[]++ const writeManifest = (content: string): string => {+ const manifestPath = path.join(dir, 'schema-requirements.json')+ writeFileSync(manifestPath, content)+ return manifestPath+ }++ const makeDeps = (+ store: MigrationStore,+ clock?: { now: number },+ sqlxStore: SqlxMigrationStore = new FakeSqlxMigrationStore([])+ ): CliDependencies => ({+ createStore: () => store,+ createSqlxStore: () => sqlxStore,+ sleep: (ms) => {+ if (clock) {+ clock.now += ms+ }+ return Promise.resolve()+ },+ log: (message) => output.push(message),+ logError: (message) => output.push(message),+ })++ beforeEach(() => {+ dir = mkdtempSync(path.join(tmpdir(), 'schema-check-cli-'))+ output = []+ })++ afterEach(() => {+ rmSync(dir, { recursive: true, force: true })+ jest.restoreAllMocks()+ })++ it.each([+ ['missing', null],+ ['unparseable', '{not json'],+ ['empty', JSON.stringify({ schema_version: 1, digest: 'x', requirements: [] })],+ [+ 'of an unknown kind',+ JSON.stringify({ schema_version: 1, digest: 'x', requirements: [{ ...WORKFLOWS, kind: 'flyway' }] }),+ ],+ ])('exits with an error when the manifest is %s', async (_case, content) => {+ const manifestPath = content === null ? path.join(dir, 'absent.json') : writeManifest(content)+ const store = new FakeMigrationStore([[]])++ const code = await runCli(['--manifest', manifestPath], { DATABASE_URL: 'postgres://x' }, makeDeps(store))++ expect(code).toBe(EXIT_ERROR)+ expect(store.fetchAppliedCalls).toBe(0)+ })++ it('exits 1 and names the missing migration and the latest applied one', async () => {+ const manifestPath = writeManifest(JSON.stringify(buildManifest([WORKFLOWS], 'abc123')))+ const store = new FakeMigrationStore([[{ app: 'workflows', name: '0039_previous' }]])++ const code = await runCli(['--manifest', manifestPath], { DATABASE_URL: 'postgres://x' }, makeDeps(store))++ expect(code).toBe(EXIT_MISSING)+ expect(output).toContain(' needs workflows.0040_frequency_cap_hold, has up to 0039_previous')+ })++ it.each<[string, Record<string, string>, SqlxMigrationRow[], string]>([+ ['missing', { CYCLOTRON_NODE_DATABASE_URL: 'postgres://c' }, [], 'cyclotron-node missing'],+ [+ 'dirty',+ { CYCLOTRON_NODE_DATABASE_URL: 'postgres://c' },+ [{ version: 1, success: false }],+ 'cyclotron-node dirty',+ ],+ ['not configured for this app', {}, [], 'cyclotron-node skipped'],+ ])('logs a %s sqlx database and still exits 0', async (_case, extraEnv, sqlxRows, expectedLog) => {+ const manifestPath = writeManifest(JSON.stringify(buildManifest([WORKFLOWS, CYCLOTRON], null)))+ const store = new FakeMigrationStore([[{ app: 'workflows', name: '0040_frequency_cap_hold' }]])++ const code = await runCli(+ ['--manifest', manifestPath],+ { DATABASE_URL: 'postgres://x', ...extraEnv },+ makeDeps(store, undefined, new FakeSqlxMigrationStore(sqlxRows))+ )++ expect(code).toBe(EXIT_SATISFIED)+ expect(output.some((line) => line.includes(expectedLog))).toBe(true)+ })++ it('keeps polling until the migration lands within --wait-seconds', async () => {+ const clock = { now: 0 }+ jest.spyOn(Date, 'now').mockImplementation(() => clock.now)+ const manifestPath = writeManifest(JSON.stringify(buildManifest([WORKFLOWS], null)))+ const store = new FakeMigrationStore([[], [], [{ app: 'workflows', name: '0040_frequency_cap_hold' }]])++ const code = await runCli(+ ['--manifest', manifestPath, '--wait-seconds', '60', '--poll-interval-seconds', '15'],+ { DATABASE_URL: 'postgres://x' },+ makeDeps(store, clock)+ )++ expect(code).toBe(EXIT_SATISFIED)+ expect(store.fetchAppliedCalls).toBe(3)+ })++ it('gives up with exit 1 once --wait-seconds passes', async () => {+ const clock = { now: 0 }+ jest.spyOn(Date, 'now').mockImplementation(() => clock.now)+ const manifestPath = writeManifest(JSON.stringify(buildManifest([WORKFLOWS], null)))+ const store = new FakeMigrationStore([[]])++ const code = await runCli(+ ['--manifest', manifestPath, '--wait-seconds', '40', '--poll-interval-seconds', '15'],+ { DATABASE_URL: 'postgres://x' },+ makeDeps(store, clock)+ )++ expect(code).toBe(EXIT_MISSING)+ // Polls at 0 s, 15 s, 30 s and 40 s, where the last sleep is cut to the remaining 10 s.+ expect(store.fetchAppliedCalls).toBe(4)+ })+ })+})diff --git a/nodejs/src/schema-check/check.ts b/nodejs/src/schema-check/check.tsnew file mode 100644index 000000000000..571a1a2e778f--- /dev/null+++ b/nodejs/src/schema-check/check.ts@@ -0,0 +1,164 @@+import { DjangoRequirement, SqlxRequirement } from './manifest'++export interface AppliedMigration {+ app: string+ name: string+}++export interface MigrationStore {+ fetchApplied(keys: AppliedMigration[]): Promise<AppliedMigration[]>+ fetchLatestApplied(apps: string[]): Promise<Map<string, string>>+}++export type CheckOutcome =+ | { status: 'satisfied' }+ | { status: 'missing'; missing: DjangoRequirement[]; latestApplied: Map<string, string> }+ | { status: 'error'; error: Error }++export interface WaitOptions {+ waitSeconds: number+ pollIntervalMs: number+ now?: () => number+ sleep?: (ms: number) => Promise<void>+ log?: (message: string) => void+}++function migrationKey(app: string, name: string): string {+ return `${app}.${name}`+}++export function requiredMigrationKeys(requirements: DjangoRequirement[]): AppliedMigration[] {+ return requirements.flatMap((requirement) => [+ { app: requirement.app, name: requirement.migration },+ ...(requirement.replaces ?? []).map((name) => ({ app: requirement.app, name })),+ ])+}++// Django records a squash as applied once all migrations it replaces are applied, even without its own row.+export function findMissingRequirements(+ requirements: DjangoRequirement[],+ applied: AppliedMigration[]+): DjangoRequirement[] {+ const appliedKeys = new Set(applied.map((row) => migrationKey(row.app, row.name)))+ return requirements.filter((requirement) => {+ if (appliedKeys.has(migrationKey(requirement.app, requirement.migration))) {+ return false+ }+ const replaces = requirement.replaces ?? []+ const replacedAllApplied =+ replaces.length > 0 && replaces.every((name) => appliedKeys.has(migrationKey(requirement.app, name)))+ return !replacedAllApplied+ })+}++export function describeMissing(missing: DjangoRequirement[], latestApplied: Map<string, string>): string[] {+ return missing.map((requirement) => {+ const latest = latestApplied.get(requirement.app)+ const has = latest ? `has up to ${latest}` : 'has no migrations for this app'+ return `needs ${requirement.app}.${requirement.migration}, ${has}`+ })+}++// Node reports a refused connection to every resolved address as an AggregateError with an empty message.+export function describeError(error: Error): string {+ return error.message || (error as NodeJS.ErrnoException).code || error.name+}++export async function checkOnce(requirements: DjangoRequirement[], store: MigrationStore): Promise<CheckOutcome> {+ try {+ const applied = await store.fetchApplied(requiredMigrationKeys(requirements))+ const missing = findMissingRequirements(requirements, applied)+ if (missing.length === 0) {+ return { status: 'satisfied' }+ }+ const latestApplied = await store.fetchLatestApplied([...new Set(missing.map((item) => item.app))])+ return { status: 'missing', missing, latestApplied }+ } catch (error) {+ return { status: 'error', error: error instanceof Error ? error : new Error(String(error)) }+ }+}++function defaultSleep(ms: number): Promise<void> {+ return new Promise((resolve) => setTimeout(resolve, ms))+}++// Database errors count as "not yet" while time remains, because the hook can start before a failover settles.+export async function waitForRequirements(+ requirements: DjangoRequirement[],+ store: MigrationStore,+ options: WaitOptions+): Promise<CheckOutcome> {+ const now = options.now ?? Date.now+ const sleep = options.sleep ?? defaultSleep+ const log = options.log ?? ((): void => {})+ const deadline = now() + options.waitSeconds * 1000++ for (;;) {+ const outcome = await checkOnce(requirements, store)+ if (outcome.status === 'satisfied') {+ return outcome+ }++ const remainingMs = deadline - now()+ if (remainingMs <= 0) {+ return outcome+ }++ if (outcome.status === 'missing') {+ const names = outcome.missing.map((item) => `${item.app}.${item.migration}`)+ log(`Waiting for ${names.length} migrations: ${names.join(', ')}`)+ } else {+ log(`Waiting: could not read django_migrations: ${describeError(outcome.error)}`)+ }+ await sleep(Math.min(options.pollIntervalMs, remainingMs))+ }+}++export interface SqlxMigrationRow {+ version: number+ success: boolean+}++export interface SqlxMigrationStore {+ // Returns the rows for the given versions plus every row with success = false.+ fetchRows(versions: number[]): Promise<SqlxMigrationRow[]>+}++export type SqlxShadowStatus = 'satisfied' | 'missing' | 'dirty' | 'skipped' | 'error'++export interface SqlxShadowResult {+ requirement: SqlxRequirement+ status: SqlxShadowStatus+ detail: string+}++// sqlx writes success = false when a migration without a transaction fails halfway. sqlx refuses to run on such a database.+export function evaluateSqlxRows(requirement: SqlxRequirement, rows: SqlxMigrationRow[]): SqlxShadowResult {+ const failed = rows.filter((row) => !row.success).map((row) => row.version)+ if (failed.length > 0) {+ return { requirement, status: 'dirty', detail: `failed migrations recorded: ${failed.join(', ')}` }+ }+ if (rows.some((row) => row.version === requirement.version)) {+ return { requirement, status: 'satisfied', detail: `version ${requirement.version} is applied` }+ }+ return { requirement, status: 'missing', detail: `version ${requirement.version} is not applied` }+}++export async function checkSqlxShadow(+ requirement: SqlxRequirement,+ store: SqlxMigrationStore | null+): Promise<SqlxShadowResult> {+ // Node has no production default for these URLs, so an app without one does not use that database.+ if (store === null) {+ return { requirement, status: 'skipped', detail: 'no database URL in this environment' }+ }+ try {+ return evaluateSqlxRows(requirement, await store.fetchRows([requirement.version]))+ } catch (error) {+ return {+ requirement,+ status: 'error',+ detail: describeError(error instanceof Error ? error : new Error(String(error))),+ }+ }+}diff --git a/nodejs/src/schema-check/cli.ts b/nodejs/src/schema-check/cli.tsnew file mode 100644index 000000000000..0f9d5288d3d6--- /dev/null+++ b/nodejs/src/schema-check/cli.ts@@ -0,0 +1,220 @@+import { parseArgs } from 'util'++import { getDefaultCommonConfig } from '~/common/config'+import { buildBehavioralCohortsDatabaseUrl, buildPosthogDatabaseUrl } from '~/common/config/database-url'++import {+ MigrationStore,+ SqlxMigrationStore,+ checkSqlxShadow,+ describeError,+ describeMissing,+ waitForRequirements,+} from './check'+import {+ DEFAULT_MANIFEST_PATH,+ SqlxDatabase,+ SqlxRequirement,+ djangoRequirements,+ readManifest,+ sqlxRequirements,+} from './manifest'+import { PostgresMigrationStore, PostgresSqlxMigrationStore } from './postgres-migration-store'++export const EXIT_SATISFIED = 0+export const EXIT_MISSING = 1+export const EXIT_ERROR = 2++const USAGE = `Usage: node nodejs/dist/schema-check/cli.js [--manifest PATH] [--wait-seconds N]++Checks that the main Postgres database has every Django migration this Node build needs.+Exit codes: 0 all applied, 1 migrations missing, 2 the check could not run.+sqlx requirements run in shadow mode: the result is logged and never changes the exit code.++Options:+ --manifest PATH Requirements file (default: ${DEFAULT_MANIFEST_PATH})+ --wait-seconds N Poll until applied or N seconds pass (default: 0, check once)+ --poll-interval-seconds N Time between polls (default: 15)+ --connect-timeout-seconds N Postgres connect timeout (default: 5)+ --statement-timeout-seconds N Postgres statement timeout (default: 10)++Reads DATABASE_URL, or POSTHOG_DB_NAME, POSTHOG_DB_USER, POSTHOG_DB_PASSWORD,+POSTHOG_POSTGRES_HOST and POSTHOG_POSTGRES_PORT, the same as the Node services.+sqlx databases use PERSONS_DATABASE_URL, CYCLOTRON_NODE_DATABASE_URL and+BEHAVIORAL_COHORTS_DATABASE_URL (or POSTGRES_BEHAVIORAL_COHORTS_HOST, _USER and _PASSWORD).`++export interface CliDependencies {+ createStore: (databaseUrl: string, connectTimeoutMs: number, statementTimeoutMs: number) => MigrationStore+ createSqlxStore: (databaseUrl: string, connectTimeoutMs: number, statementTimeoutMs: number) => SqlxMigrationStore+ sleep?: (ms: number) => Promise<void>+ log: (message: string) => void+ logError: (message: string) => void+}++const defaultDependencies: CliDependencies = {+ createStore: (databaseUrl, connectTimeoutMs, statementTimeoutMs) =>+ new PostgresMigrationStore({ databaseUrl, connectTimeoutMs, statementTimeoutMs }),+ createSqlxStore: (databaseUrl, connectTimeoutMs, statementTimeoutMs) =>+ new PostgresSqlxMigrationStore({ databaseUrl, connectTimeoutMs, statementTimeoutMs }),+ log: (message) => console.log(message),+ logError: (message) => console.error(message),+}++// Uses the primary through DATABASE_URL. DATABASE_READONLY_URL is never read because replica lag hides fresh migrations.+export function resolveDatabaseUrl(env: Record<string, string | undefined>): string {+ const defaults = getDefaultCommonConfig()+ return buildPosthogDatabaseUrl({+ DATABASE_URL: env.DATABASE_URL ?? defaults.DATABASE_URL,+ POSTHOG_DB_NAME: env.POSTHOG_DB_NAME ?? defaults.POSTHOG_DB_NAME,+ POSTHOG_DB_USER: env.POSTHOG_DB_USER ?? defaults.POSTHOG_DB_USER,+ POSTHOG_DB_PASSWORD: env.POSTHOG_DB_PASSWORD ?? defaults.POSTHOG_DB_PASSWORD,+ POSTHOG_POSTGRES_HOST: env.POSTHOG_POSTGRES_HOST ?? defaults.POSTHOG_POSTGRES_HOST,+ POSTHOG_POSTGRES_PORT: env.POSTHOG_POSTGRES_PORT+ ? parseInt(env.POSTHOG_POSTGRES_PORT, 10)+ : defaults.POSTHOG_POSTGRES_PORT,+ })+}++// Reads the same variables as the Node services, without the dev defaults, so an empty result means the app does not use the database.+export function resolveSqlxDatabaseUrl(database: SqlxDatabase, env: Record<string, string | undefined>): string {+ if (database === 'persons') {+ return env.PERSONS_DATABASE_URL ?? ''+ }+ if (database === 'cyclotron-node') {+ return env.CYCLOTRON_NODE_DATABASE_URL ?? ''+ }+ return buildBehavioralCohortsDatabaseUrl({+ BEHAVIORAL_COHORTS_DATABASE_URL: env.BEHAVIORAL_COHORTS_DATABASE_URL ?? '',+ POSTGRES_BEHAVIORAL_COHORTS_HOST: env.POSTGRES_BEHAVIORAL_COHORTS_HOST ?? '',+ POSTGRES_BEHAVIORAL_COHORTS_USER: env.POSTGRES_BEHAVIORAL_COHORTS_USER ?? '',+ POSTGRES_BEHAVIORAL_COHORTS_PASSWORD: env.POSTGRES_BEHAVIORAL_COHORTS_PASSWORD ?? '',+ })+}++async function logSqlxShadow(+ requirements: SqlxRequirement[],+ env: Record<string, string | undefined>,+ options: { connectTimeoutMs: number; statementTimeoutMs: number },+ deps: CliDependencies+): Promise<void> {+ const results = await Promise.all(+ requirements.map((requirement) => {+ const databaseUrl = resolveSqlxDatabaseUrl(requirement.database, env)+ const store = databaseUrl+ ? deps.createSqlxStore(databaseUrl, options.connectTimeoutMs, options.statementTimeoutMs)+ : null+ return checkSqlxShadow(requirement, store)+ })+ )+ for (const result of results) {+ deps.log(+ `Shadow sqlx check (does not block): ${result.requirement.database} ${result.status}, ${result.detail}`+ )+ }+}++function parseSeconds(name: string, value: string | undefined, fallback: number): number {+ if (value === undefined) {+ return fallback+ }+ const seconds = Number(value)+ if (!Number.isFinite(seconds) || seconds < 0) {+ throw new Error(`--${name} must be a number of seconds, got "${value}"`)+ }+ return seconds+}++export async function runCli(+ argv: string[],+ env: Record<string, string | undefined>,+ deps: CliDependencies = defaultDependencies+): Promise<number> {+ let options: {+ manifestPath: string+ waitSeconds: number+ pollIntervalSeconds: number+ connectTimeoutSeconds: number+ statementTimeoutSeconds: number+ }+ try {+ const { values } = parseArgs({+ args: argv,+ options: {+ manifest: { type: 'string' },+ 'wait-seconds': { type: 'string' },+ 'poll-interval-seconds': { type: 'string' },+ 'connect-timeout-seconds': { type: 'string' },+ 'statement-timeout-seconds': { type: 'string' },+ help: { type: 'boolean' },+ },+ })+ if (values.help) {+ deps.log(USAGE)+ return EXIT_SATISFIED+ }+ options = {+ manifestPath: values.manifest ?? DEFAULT_MANIFEST_PATH,+ waitSeconds: parseSeconds('wait-seconds', values['wait-seconds'], 0),+ pollIntervalSeconds: parseSeconds('poll-interval-seconds', values['poll-interval-seconds'], 15),+ connectTimeoutSeconds: parseSeconds('connect-timeout-seconds', values['connect-timeout-seconds'], 5),+ statementTimeoutSeconds: parseSeconds('statement-timeout-seconds', values['statement-timeout-seconds'], 10),+ }+ } catch (error) {+ deps.logError(`Schema check error: ${error.message}\n\n${USAGE}`)+ return EXIT_ERROR+ }++ let store: MigrationStore+ let manifest: ReturnType<typeof readManifest>+ try {+ manifest = readManifest(options.manifestPath)+ store = deps.createStore(+ resolveDatabaseUrl(env),+ options.connectTimeoutSeconds * 1000,+ options.statementTimeoutSeconds * 1000+ )+ } catch (error) {+ deps.logError(`Schema check error: ${error.message}`)+ return EXIT_ERROR+ }++ const django = djangoRequirements(manifest)+ const sqlx = sqlxRequirements(manifest)+ deps.log(+ `Checking ${django.length} Django apps and ${sqlx.length} sqlx databases in shadow mode (manifest ${manifest.digest}, commit ${manifest.source_commit ?? 'unknown'})`+ )+ const outcome = await waitForRequirements(django, store, {+ waitSeconds: options.waitSeconds,+ pollIntervalMs: options.pollIntervalSeconds * 1000,+ sleep: deps.sleep,+ log: deps.log,+ })+ // Runs after the Django wait so that the log shows the sqlx state at the moment the Deployment would roll.+ await logSqlxShadow(+ sqlx,+ env,+ {+ connectTimeoutMs: options.connectTimeoutSeconds * 1000,+ statementTimeoutMs: options.statementTimeoutSeconds * 1000,+ },+ deps+ )++ if (outcome.status === 'satisfied') {+ deps.log('All required Django migrations are applied.')+ return EXIT_SATISFIED+ }+ if (outcome.status === 'error') {+ deps.logError(`Schema check error: could not read django_migrations: ${describeError(outcome.error)}`)+ return EXIT_ERROR+ }+ deps.logError('Required Django migrations are not applied yet:')+ describeMissing(outcome.missing, outcome.latestApplied).forEach((line) => deps.logError(` ${line}`))+ return EXIT_MISSING+}++if (require.main === module) {+ void runCli(process.argv.slice(2), process.env).then((code) => {+ process.exitCode = code+ })+}diff --git a/nodejs/src/schema-check/derive-requirements.test.ts b/nodejs/src/schema-check/derive-requirements.test.tsnew file mode 100644index 000000000000..471de25fb07d--- /dev/null+++ b/nodejs/src/schema-check/derive-requirements.test.ts@@ -0,0 +1,133 @@+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'fs'+import { tmpdir } from 'os'+import path from 'path'++import { deriveSchemaRequirements } from './derive-requirements'++const FIXTURE_FILES: Record<string, string> = {+ 'posthog/apps.py': 'class PostHogConfig(AppConfig):\n name = "posthog"\n',+ 'posthog/migrations/0001_initial.py': 'migrations.CreateModel(\n name="Team",\n fields=[],\n)\n',+ 'posthog/migrations/0002_team_flag.py': 'migrations.AddField(model_name="team", name="flag")\n',+ 'posthog/migrations/max_migration.txt': '0002_team_flag\n',++ // The label override makes the table posthog_session_session, not session_session.+ 'posthog/session/apps.py':+ 'class SessionConfig(AppConfig):\n name = "posthog.session"\n label = "posthog_session"\n',+ 'posthog/session/migrations/0001_initial.py': 'migrations.CreateModel(name="Session", fields=[])\n',+ 'posthog/session/migrations/max_migration.txt': '0001_initial\n',++ 'posthog/clickhouse/migrations/0001_initial.py': 'migrations.CreateModel(name="Events", fields=[])\n',+ 'posthog/clickhouse/migrations/max_migration.txt': '0001_initial\n',++ 'products/surveys/backend/apps.py':+ 'class SurveysConfig(AppConfig):\n name = "products.surveys.backend"\n label = "surveys"\n',+ 'products/surveys/backend/migrations/0001_initial.py':+ 'migrations.CreateModel(\n name="Survey",\n fields=[],\n options={"db_table": "posthog_survey"},\n)\n',+ 'products/surveys/backend/migrations/max_migration.txt': '0001_initial\n',++ 'products/legacy/backend/apps.py':+ 'class LegacyConfig(AppConfig):\n name = "products.legacy.backend"\n label = "legacy"\n',+ 'products/legacy/backend/migrations/0001_initial.py': 'migrations.CreateModel(name="Widget", fields=[])\n',+ 'products/legacy/backend/migrations/0002_move_widget.py': 'migrations.DeleteModel(name="Widget")\n',+ 'products/legacy/backend/migrations/max_migration.txt': '0002_move_widget\n',++ 'products/widgets/backend/apps.py':+ 'class WidgetsConfig(AppConfig):\n name = "products.widgets.backend"\n label = "widgets"\n',+ 'products/widgets/backend/migrations/0001_initial.py': 'migrations.CreateModel(name="Widget", fields=[])\n',+ 'products/widgets/backend/migrations/0002_widget_table.py':+ 'migrations.AlterModelTable(name="widget", table="legacy_widget")\n',+ 'products/widgets/backend/migrations/max_migration.txt': '0002_widget_table\n',++ 'products/tracing/backend/apps.py':+ 'class TracingConfig(AppConfig):\n name = "products.tracing.backend"\n label = "tracing"\n',+ 'products/tracing/backend/migrations/0001_squash_initial.py':+ 'class Migration(migrations.Migration):\n' ++ ' replaces = [\n ("tracing", "0001_initial"),\n ("tracing", "0002_span_kind"),\n ]\n' ++ ' operations = [migrations.CreateModel(name="Span", fields=[])]\n',+ 'products/tracing/backend/migrations/max_migration.txt': '0001_squash_initial\n',++ 'products/stamphog/backend/apps.py':+ 'class StamphogConfig(AppConfig):\n name = "products.stamphog.backend"\n label = "stamphog"\n',+ 'products/stamphog/backend/migrations/0001_initial.py': 'migrations.CreateModel(name="Review", fields=[])\n',+ 'products/stamphog/backend/migrations/max_migration.txt': '0001_initial\n',+ 'products/db_routing.yaml': 'routes:\n - app_label: stamphog\n database: stamphog\n',++ 'products/unused/backend/apps.py':+ 'class UnusedConfig(AppConfig):\n name = "products.unused.backend"\n label = "unused"\n',+ 'products/unused/backend/migrations/0001_initial.py': 'migrations.CreateModel(name="Thing", fields=[])\n',+ 'products/unused/backend/migrations/max_migration.txt': '0001_initial\n',++ 'rust/persons_migrations/20250101000001_initial.sql':+ 'CREATE TABLE IF NOT EXISTS posthog_persons_row (id bigint);\n',+ 'rust/cyclotron-node-migrations/20260101000001_initial.sql':+ 'CREATE TABLE IF NOT EXISTS cyclotron_jobs (id uuid);\n',+ // The latest version creates no table, but Node still needs the database at that version.+ 'rust/cyclotron-node-migrations/20260201000001_add_index.sql': 'CREATE INDEX idx_jobs ON cyclotron_jobs (id);\n',+ 'rust/behavioral_cohorts_migrations/20260301000001_initial.sql': 'CREATE TABLE cohort_membership (id bigint);\n',++ 'nodejs/src/reader.ts': [+ "const a = 'SELECT * FROM posthog_team'",+ "const b = 'SELECT * FROM posthog_session_session'",+ "const c = 'SELECT * FROM posthog_survey'",+ "const d = 'SELECT * FROM legacy_widget'",+ "const e = 'SELECT * FROM tracing_span'",+ "const f = 'SELECT * FROM clickhouse_events'",+ "const g = 'SELECT * FROM stamphog_review'",+ "const i = 'SELECT * FROM cyclotron_jobs'",+ ].join('\n'),+ 'nodejs/src/reader.test.ts': "const h = 'SELECT * FROM unused_thing'\n",+}++describe('deriveSchemaRequirements', () => {+ let repoRoot: string++ beforeAll(() => {+ repoRoot = mkdtempSync(path.join(tmpdir(), 'schema-check-'))+ for (const [relativePath, content] of Object.entries(FIXTURE_FILES)) {+ const filePath = path.join(repoRoot, relativePath)+ mkdirSync(path.dirname(filePath), { recursive: true })+ writeFileSync(filePath, content)+ }+ })++ afterAll(() => {+ rmSync(repoRoot, { recursive: true, force: true })+ })++ it('requires the leaf of every Django app and sqlx database whose tables nodejs/src reads', () => {+ const { requirements, tablesByRequirement } = deriveSchemaRequirements(repoRoot)++ expect(requirements).toEqual([+ { kind: 'django', database: 'default', app: 'legacy', migration: '0002_move_widget' },+ { kind: 'django', database: 'default', app: 'posthog', migration: '0002_team_flag' },+ { kind: 'django', database: 'default', app: 'posthog_session', migration: '0001_initial' },+ { kind: 'django', database: 'default', app: 'surveys', migration: '0001_initial' },+ {+ kind: 'django',+ database: 'default',+ app: 'tracing',+ migration: '0001_squash_initial',+ replaces: ['0001_initial', '0002_span_kind'],+ },+ { kind: 'django', database: 'default', app: 'widgets', migration: '0002_widget_table' },+ { kind: 'sqlx', database: 'cyclotron-node', version: 20260201000001 },+ ])+ expect(Object.fromEntries(tablesByRequirement)).toEqual({+ legacy: ['legacy_widget'],+ posthog: ['posthog_team'],+ posthog_session: ['posthog_session_session'],+ surveys: ['posthog_survey'],+ tracing: ['tracing_span'],+ widgets: ['legacy_widget'],+ 'sqlx:cyclotron-node': ['cyclotron_jobs'],+ })+ })++ it('finds the workflows config, team and cyclotron job tables on the real repo', () => {+ const { tablesByRequirement } = deriveSchemaRequirements(path.resolve(__dirname, '../../..'))++ expect(tablesByRequirement.get('workflows')).toContain('workflows_teamworkflowsconfig')+ expect(tablesByRequirement.get('posthog')).toContain('posthog_team')+ expect(tablesByRequirement.get('sqlx:cyclotron-node')).toContain('cyclotron_jobs')+ })+})diff --git a/nodejs/src/schema-check/derive-requirements.ts b/nodejs/src/schema-check/derive-requirements.tsnew file mode 100644index 000000000000..4dac95978f1f--- /dev/null+++ b/nodejs/src/schema-check/derive-requirements.ts@@ -0,0 +1,256 @@+import { existsSync, readFileSync, readdirSync } from 'fs'+import path from 'path'++import { DjangoRequirement, Requirement, SQLX_DATABASES, SqlxDatabase, SqlxRequirement } from './manifest'++// Top-level trees that hold Django apps. Dockerfile.node bind-mounts the same trees into the generator step.+export const DJANGO_SOURCE_ROOTS = ['posthog', 'ee', 'products']++// ClickHouse migrations are not Django apps and do not write to django_migrations in Postgres.+const EXCLUDED_MIGRATION_DIRS = ['posthog/clickhouse/migrations']++const PRODUCT_DB_ROUTING_FILE = 'products/db_routing.yaml'++// rust/Dockerfile.sqlx-migrate copies these directories, and the sqlx-migrate deploy applies them.+export const SQLX_MIGRATION_DIRS: Record<SqlxDatabase, string> = {+ persons: 'rust/persons_migrations',+ 'cyclotron-node': 'rust/cyclotron-node-migrations',+ 'behavioral-cohorts': 'rust/behavioral_cohorts_migrations',+}+const NODEJS_SOURCE_DIR = 'nodejs/src'++const SKIPPED_DIR_NAMES = new Set(['node_modules', '__pycache__', 'dist', 'frontend'])++const MIGRATION_FILE = /^\d{4}_\w+\.py$/+// sqlx takes the version from the file name prefix. A .down.sql file reverts its .up.sql twin and adds no version.+const SQLX_MIGRATION_FILE = /^(\d+)_[\w-]+(?:\.up)?\.sql$/+const CREATE_TABLE = /\bCREATE\s+TABLE\s+(?:IF\s+NOT\s+EXISTS\s+)?(?:public\.)?"?(\w+)"?/gi+const CREATE_MODEL_NAME = /CreateModel\(\s*name\s*=\s*['"](\w+)['"]/g+const DB_TABLE_OPTION = /['"]db_table['"]\s*:\s*['"](\w+)['"]/g+const TABLE_ARGUMENT = /\b(?:db_)?table\s*=\s*['"](\w+)['"]/g+const APP_CONFIG_LABEL = /^\s*label\s*=\s*['"](\w+)['"]/m+const APP_CONFIG_NAME = /^\s*name\s*=\s*['"]([\w.]+)['"]/m+const SQUASH_REPLACES = /^\s*replaces\s*=\s*\[([\s\S]*?)\]/m+const MIGRATION_KEY = /\(\s*['"](\w+)['"]\s*,\s*['"](\w+)['"]\s*\)/g+const ROUTED_APP_LABEL = /app_label:\s*['"]?(\w+)['"]?/g+const LOWERCASE_IDENTIFIER = /\b[a-z][a-z0-9_]+\b/g++export interface DjangoApp {+ label: string+ migrationsDir: string+ leafMigration: string+}++export interface DerivedSchemaRequirements {+ requirements: Requirement[]+ // Keyed by Django app label, or by "sqlx:<database>" for sqlx requirements.+ tablesByRequirement: Map<string, string[]>+}++export interface SqlxMigrationDir {+ database: SqlxDatabase+ latestVersion: number+ tables: Set<string>+}++function listFilesRecursively(dir: string, keep: (fileName: string) => boolean): string[] {+ if (!existsSync(dir)) {+ return []+ }+ const files: string[] = []+ for (const entry of readdirSync(dir, { withFileTypes: true })) {+ if (entry.name.startsWith('.') || SKIPPED_DIR_NAMES.has(entry.name)) {+ continue+ }+ const fullPath = path.join(dir, entry.name)+ if (entry.isDirectory()) {+ files.push(...listFilesRecursively(fullPath, keep))+ } else if (entry.isFile() && keep(entry.name)) {+ files.push(fullPath)+ }+ }+ return files+}++function matchAll(text: string, pattern: RegExp): string[] {+ return Array.from(text.matchAll(pattern), (match) => match[1])+}++// Mirrors Django: AppConfig.label wins, then the last part of AppConfig.name, then the package directory name.+export function readAppLabel(appDir: string): string {+ const appsPy = path.join(appDir, 'apps.py')+ if (existsSync(appsPy)) {+ const text = readFileSync(appsPy, 'utf8')+ const label = text.match(APP_CONFIG_LABEL)?.[1]+ if (label) {+ return label+ }+ const name = text.match(APP_CONFIG_NAME)?.[1]+ if (name) {+ return name.split('.').pop()!+ }+ }+ return path.basename(appDir)+}++export function findDjangoApps(repoRoot: string): DjangoApp[] {+ const excluded = new Set(EXCLUDED_MIGRATION_DIRS.map((dir) => path.join(repoRoot, dir)))+ const maxMigrationFiles = DJANGO_SOURCE_ROOTS.flatMap((root) =>+ listFilesRecursively(path.join(repoRoot, root), (name) => name === 'max_migration.txt')+ )++ const apps: DjangoApp[] = []+ for (const maxMigrationFile of maxMigrationFiles) {+ const migrationsDir = path.dirname(maxMigrationFile)+ if (path.basename(migrationsDir) !== 'migrations' || excluded.has(migrationsDir)) {+ continue+ }+ const leafMigration = readFileSync(maxMigrationFile, 'utf8').trim()+ if (!existsSync(path.join(migrationsDir, `${leafMigration}.py`))) {+ throw new Error(`${maxMigrationFile} names ${leafMigration}, but ${leafMigration}.py does not exist`)+ }+ apps.push({ label: readAppLabel(path.dirname(migrationsDir)), migrationsDir, leafMigration })+ }+ return apps+}++export function readProductRoutedLabels(repoRoot: string): Set<string> {+ const routingFile = path.join(repoRoot, PRODUCT_DB_ROUTING_FILE)+ if (!existsSync(routingFile)) {+ return new Set()+ }+ return new Set(matchAll(readFileSync(routingFile, 'utf8'), ROUTED_APP_LABEL))+}++// A table belongs to every app whose migrations name it, so a model moved between apps puts both apps in scope.+export function collectTableOwners(apps: DjangoApp[]): Map<string, Set<string>> {+ const owners = new Map<string, Set<string>>()+ const addOwner = (table: string, label: string): void => {+ const labels = owners.get(table) ?? new Set<string>()+ labels.add(label)+ owners.set(table, labels)+ }++ for (const app of apps) {+ const migrationFiles = readdirSync(app.migrationsDir).filter((name) => MIGRATION_FILE.test(name))+ for (const fileName of migrationFiles) {+ const text = readFileSync(path.join(app.migrationsDir, fileName), 'utf8')+ for (const modelName of matchAll(text, CREATE_MODEL_NAME)) {+ addOwner(`${app.label}_${modelName.toLowerCase()}`, app.label)+ }+ for (const table of [...matchAll(text, DB_TABLE_OPTION), ...matchAll(text, TABLE_ARGUMENT)]) {+ addOwner(table, app.label)+ }+ }+ }+ return owners+}++// The scan does not know which database a query uses. Node writes posthog_cohortpeople through the persons+// database, but the match still puts the cohorts app in scope, so a new cohorts migration holds Node deploys+// until Django applies it. This is a known false positive and it only adds wait time.+export function collectNodeIdentifiers(nodeSourceDir: string): Set<string> {+ const sourceFiles = listFilesRecursively(+ nodeSourceDir,+ (name) => name.endsWith('.ts') && !name.includes('.test.')+ ).filter((file) => !file.split(path.sep).includes('__tests__'))++ const identifiers = new Set<string>()+ for (const file of sourceFiles) {+ for (const identifier of readFileSync(file, 'utf8').match(LOWERCASE_IDENTIFIER) ?? []) {+ identifiers.add(identifier)+ }+ }+ return identifiers+}++export function readSquashReplaces(app: DjangoApp): string[] | undefined {+ const text = readFileSync(path.join(app.migrationsDir, `${app.leafMigration}.py`), 'utf8')+ const replacesBlock = text.match(SQUASH_REPLACES)?.[1]+ if (replacesBlock === undefined) {+ return undefined+ }++ const names: string[] = []+ for (const [, replacedApp, replacedName] of replacesBlock.matchAll(MIGRATION_KEY)) {+ // The check compares names within one app, so a cross-app replaces entry would be checked against the wrong app.+ if (replacedApp !== app.label) {+ throw new Error(+ `${app.label}.${app.leafMigration} replaces ${replacedApp}.${replacedName}, which is in another app`+ )+ }+ names.push(replacedName)+ }+ return names+}++// A missing directory throws, because a wrong bind mount in Dockerfile.node must fail the build and not drop the requirement.+export function readSqlxMigrationDir(repoRoot: string, database: SqlxDatabase): SqlxMigrationDir {+ const dir = path.join(repoRoot, SQLX_MIGRATION_DIRS[database])+ if (!existsSync(dir)) {+ throw new Error(`${dir} does not exist`)+ }+ let latestVersion = 0+ const tables = new Set<string>()+ for (const fileName of readdirSync(dir)) {+ const version = fileName.match(SQLX_MIGRATION_FILE)?.[1]+ if (version === undefined) {+ continue+ }+ latestVersion = Math.max(latestVersion, Number(version))+ for (const table of matchAll(readFileSync(path.join(dir, fileName), 'utf8'), CREATE_TABLE)) {+ tables.add(table.toLowerCase())+ }+ }+ if (latestVersion === 0) {+ throw new Error(`${dir} has no sqlx migrations`)+ }+ return { database, latestVersion, tables }+}++export function deriveSchemaRequirements(repoRoot: string): DerivedSchemaRequirements {+ const routedLabels = readProductRoutedLabels(repoRoot)+ const apps = findDjangoApps(repoRoot).filter((app) => !routedLabels.has(app.label))+ const tableOwners = collectTableOwners(apps)+ const nodeIdentifiers = collectNodeIdentifiers(path.join(repoRoot, NODEJS_SOURCE_DIR))++ const tablesByRequirement = new Map<string, string[]>()+ for (const table of [...tableOwners.keys()].sort()) {+ if (!nodeIdentifiers.has(table)) {+ continue+ }+ for (const label of tableOwners.get(table)!) {+ tablesByRequirement.set(label, [...(tablesByRequirement.get(label) ?? []), table])+ }+ }++ const djangoRequirements: DjangoRequirement[] = apps+ .filter((app) => tablesByRequirement.has(app.label))+ .sort((a, b) => a.label.localeCompare(b.label))+ .map((app) => {+ const requirement: DjangoRequirement = {+ kind: 'django',+ database: 'default',+ app: app.label,+ migration: app.leafMigration,+ }+ const replaces = readSquashReplaces(app)+ if (replaces !== undefined) {+ requirement.replaces = replaces+ }+ return requirement+ })++ const sqlxRequirements: SqlxRequirement[] = []+ for (const database of SQLX_DATABASES) {+ const migrationDir = readSqlxMigrationDir(repoRoot, database)+ const tables = [...migrationDir.tables].filter((table) => nodeIdentifiers.has(table)).sort()+ if (tables.length === 0) {+ continue+ }+ tablesByRequirement.set(`sqlx:${database}`, tables)+ sqlxRequirements.push({ kind: 'sqlx', database, version: migrationDir.latestVersion })+ }++ return { requirements: [...djangoRequirements, ...sqlxRequirements], tablesByRequirement }+}diff --git a/nodejs/src/schema-check/generate.ts b/nodejs/src/schema-check/generate.tsnew file mode 100644index 000000000000..62a2fa2adf75--- /dev/null+++ b/nodejs/src/schema-check/generate.ts@@ -0,0 +1,49 @@+import { writeFileSync } from 'fs'+import path from 'path'+import { parseArgs } from 'util'++import { deriveSchemaRequirements } from './derive-requirements'+import { buildManifest } from './manifest'++const USAGE =+ 'Usage: node nodejs/dist/schema-check/generate.js --repo-root PATH --output PATH [--source-commit SHA]\n' ++ 'Writes the Django migrations this Node build needs, for nodejs/dist/schema-check/cli.js to check.'++function main(): number {+ const { values } = parseArgs({+ options: {+ 'repo-root': { type: 'string' },+ output: { type: 'string' },+ 'source-commit': { type: 'string' },+ help: { type: 'boolean' },+ },+ })+ if (values.help || !values['repo-root'] || !values.output) {+ console.error(USAGE)+ return values.help ? 0 : 2+ }++ const { requirements, tablesByRequirement } = deriveSchemaRequirements(path.resolve(values['repo-root']))+ if (!requirements.some((requirement) => requirement.kind === 'django')) {+ console.error('Found no Django tables in nodejs/src. The source mounts are probably wrong.')+ return 1+ }++ const sourceCommit =+ values['source-commit'] && values['source-commit'] !== 'unknown' ? values['source-commit'] : null+ const manifest = buildManifest(requirements, sourceCommit)+ writeFileSync(values.output, JSON.stringify(manifest, null, 4) + '\n')++ console.log(`Wrote ${values.output} (digest ${manifest.digest}, ${requirements.length} requirements)`)+ for (const requirement of requirements) {+ const key = requirement.kind === 'django' ? requirement.app : `sqlx:${requirement.database}`+ const target =+ requirement.kind === 'django'+ ? `${requirement.app}.${requirement.migration}`+ : `sqlx ${requirement.database}@${requirement.version}`+ console.log(` ${target}: ${(tablesByRequirement.get(key) ?? []).join(', ')}`)+ }+ return 0+}++process.exitCode = main()diff --git a/nodejs/src/schema-check/manifest.ts b/nodejs/src/schema-check/manifest.tsnew file mode 100644index 000000000000..f4eba842ecef--- /dev/null+++ b/nodejs/src/schema-check/manifest.ts@@ -0,0 +1,157 @@+import { createHash } from 'crypto'+import { existsSync, readFileSync } from 'fs'+import path from 'path'++import { parseJSON } from '~/common/utils/json-parse'++export const MANIFEST_SCHEMA_VERSION = 1++// Resolves to /code/nodejs/schema-requirements.json in the Node image, next to nodejs/dist.+export const DEFAULT_MANIFEST_PATH = path.resolve(__dirname, '..', '..', 'schema-requirements.json')++// The names match the arguments of rust/bin/migrate-entry, which the sqlx-migrate deploy runs per database.+export const SQLX_DATABASES = ['persons', 'cyclotron-node', 'behavioral-cohorts'] as const+export type SqlxDatabase = (typeof SQLX_DATABASES)[number]++export interface DjangoRequirement {+ kind: 'django'+ database: 'default'+ app: string+ migration: string+ // Set when `migration` is a squash. Django can record either the squash or every migration it replaces.+ replaces?: string[]+}++export interface SqlxRequirement {+ kind: 'sqlx'+ database: SqlxDatabase+ version: number+}++export type Requirement = DjangoRequirement | SqlxRequirement++export interface SchemaRequirementsManifest {+ schema_version: number+ source_commit: string | null+ // Identifies the requirement list in logs. Consumers do not recompute it, because JSON.stringify output is not canonical across languages.+ digest: string+ requirements: Requirement[]+}++export class ManifestError extends Error {}++export function computeRequirementsDigest(requirements: Requirement[]): string {+ return createHash('sha256').update(JSON.stringify(requirements)).digest('hex').slice(0, 12)+}++export function buildManifest(requirements: Requirement[], sourceCommit: string | null): SchemaRequirementsManifest {+ return {+ schema_version: MANIFEST_SCHEMA_VERSION,+ source_commit: sourceCommit,+ digest: computeRequirementsDigest(requirements),+ requirements,+ }+}++function isStringArray(value: unknown): value is string[] {+ return Array.isArray(value) && value.every((item) => typeof item === 'string' && item.length > 0)+}++function parseDjangoRequirement(item: Record<string, unknown>, index: number): DjangoRequirement {+ if (item.database !== 'default') {+ throw new ManifestError(`requirements[${index}].database must be "default"`)+ }+ if (typeof item.app !== 'string' || !item.app || typeof item.migration !== 'string' || !item.migration) {+ throw new ManifestError(`requirements[${index}] needs a non-empty app and migration`)+ }+ if (item.replaces !== undefined && !isStringArray(item.replaces)) {+ throw new ManifestError(`requirements[${index}].replaces must be a list of migration names`)+ }+ const requirement: DjangoRequirement = {+ kind: 'django',+ database: 'default',+ app: item.app,+ migration: item.migration,+ }+ if (item.replaces !== undefined) {+ requirement.replaces = item.replaces+ }+ return requirement+}++function parseSqlxRequirement(item: Record<string, unknown>, index: number): SqlxRequirement {+ const database = SQLX_DATABASES.find((name) => name === item.database)+ if (!database) {+ throw new ManifestError(`requirements[${index}].database must be one of ${SQLX_DATABASES.join(', ')}`)+ }+ if (typeof item.version !== 'number' || !Number.isSafeInteger(item.version) || item.version <= 0) {+ throw new ManifestError(`requirements[${index}].version must be a positive integer`)+ }+ return { kind: 'sqlx', database, version: item.version }+}++// An unknown kind is an error. Skipping it would report a pass for a requirement that nothing checked.+function parseRequirement(value: unknown, index: number): Requirement {+ const item = value as Record<string, unknown> | null+ if (typeof item !== 'object' || item === null) {+ throw new ManifestError(`requirements[${index}] is not an object`)+ }+ if (item.kind === 'django') {+ return parseDjangoRequirement(item, index)+ }+ if (item.kind === 'sqlx') {+ return parseSqlxRequirement(item, index)+ }+ throw new ManifestError(`requirements[${index}].kind must be "django" or "sqlx", got ${JSON.stringify(item.kind)}`)+}++export function parseManifest(text: string): SchemaRequirementsManifest {+ let raw: unknown+ try {+ raw = parseJSON(text)+ } catch (error) {+ throw new ManifestError(`manifest is not valid JSON: ${error.message}`)+ }++ const data = raw as Record<string, unknown> | null+ if (typeof data !== 'object' || data === null) {+ throw new ManifestError('manifest is not a JSON object')+ }+ if (data.schema_version !== MANIFEST_SCHEMA_VERSION) {+ throw new ManifestError(+ `manifest schema_version is ${JSON.stringify(data.schema_version)}, expected ${MANIFEST_SCHEMA_VERSION}`+ )+ }+ if (!Array.isArray(data.requirements)) {+ throw new ManifestError('manifest has no requirements list')+ }+ const requirements = data.requirements.map(parseRequirement)+ // No Django entry means the generator found nothing, which is a broken build and not "no requirements".+ if (!requirements.some((requirement) => requirement.kind === 'django')) {+ throw new ManifestError('manifest has no Django requirements')+ }++ return {+ schema_version: MANIFEST_SCHEMA_VERSION,+ source_commit: typeof data.source_commit === 'string' ? data.source_commit : null,+ digest: typeof data.digest === 'string' ? data.digest : '',+ requirements,+ }+}++export function djangoRequirements(manifest: SchemaRequirementsManifest): DjangoRequirement[] {+ return manifest.requirements.filter(+ (requirement): requirement is DjangoRequirement => requirement.kind === 'django'+ )+}++export function sqlxRequirements(manifest: SchemaRequirementsManifest): SqlxRequirement[] {+ return manifest.requirements.filter((requirement): requirement is SqlxRequirement => requirement.kind === 'sqlx')+}++export function readManifest(manifestPath: string): SchemaRequirementsManifest {+ if (!existsSync(manifestPath)) {+ throw new ManifestError(`manifest not found at ${manifestPath}`)+ }+ return parseManifest(readFileSync(manifestPath, 'utf8'))+}diff --git a/nodejs/src/schema-check/postgres-migration-store.ts b/nodejs/src/schema-check/postgres-migration-store.tsnew file mode 100644index 000000000000..d0df9fae3d16--- /dev/null+++ b/nodejs/src/schema-check/postgres-migration-store.ts@@ -0,0 +1,85 @@+import { Client, QueryResultRow } from 'pg'++import { AppliedMigration, MigrationStore, SqlxMigrationRow, SqlxMigrationStore } from './check'++export interface PostgresMigrationStoreOptions {+ databaseUrl: string+ connectTimeoutMs: number+ statementTimeoutMs: number+}++const FETCH_APPLIED_SQL = `+ SELECT app, name FROM django_migrations+ WHERE (app, name) IN (SELECT * FROM unnest($1::text[], $2::text[]))+`++const FETCH_LATEST_APPLIED_SQL = `+ SELECT app, max(name) AS name FROM django_migrations+ WHERE app = ANY($1::text[])+ GROUP BY app+`++// pg returns BIGINT as a string. sqlx versions are timestamps far below 2^53, so Number() keeps them exact.+const FETCH_SQLX_ROWS_SQL = `+ SELECT version::text AS version, success FROM _sqlx_migrations+ WHERE version = ANY($1::bigint[]) OR NOT success+`++// Opens a fresh connection per query so that a dropped connection during a long wait does not stick.+async function queryInTransaction<Row extends QueryResultRow>(+ options: PostgresMigrationStoreOptions,+ sql: string,+ params: unknown[]+): Promise<Row[]> {+ // The statement timeout must not go into the Client config. pg sends it as a startup parameter,+ // and PgBouncer drops connections with startup parameters it does not track.+ const client = new Client({+ connectionString: options.databaseUrl,+ connectionTimeoutMillis: options.connectTimeoutMs,+ query_timeout: options.statementTimeoutMs,+ application_name: 'node-schema-check',+ })+ // pg emits connection errors as events too. Without a listener they crash the process instead of the query.+ client.on('error', () => {})+ try {+ await client.connect()+ // SET LOCAL keeps the timeout inside this transaction, so PgBouncer in transaction mode+ // does not hand the setting to the next client of the server connection.+ await client.query('BEGIN')+ await client.query("SELECT set_config('statement_timeout', $1, true)", [String(options.statementTimeoutMs)])+ const result = await client.query<Row>(sql, params)+ await client.query('COMMIT')+ return result.rows+ } finally {+ await client.end().catch(() => {})+ }+}++export class PostgresMigrationStore implements MigrationStore {+ constructor(private readonly options: PostgresMigrationStoreOptions) {}++ fetchApplied(keys: AppliedMigration[]): Promise<AppliedMigration[]> {+ return queryInTransaction<AppliedMigration>(this.options, FETCH_APPLIED_SQL, [+ keys.map((key) => key.app),+ keys.map((key) => key.name),+ ])+ }++ async fetchLatestApplied(apps: string[]): Promise<Map<string, string>> {+ const rows = await queryInTransaction<AppliedMigration>(this.options, FETCH_LATEST_APPLIED_SQL, [apps])+ return new Map(rows.map((row) => [row.app, row.name]))+ }+}++export class PostgresSqlxMigrationStore implements SqlxMigrationStore {+ constructor(private readonly options: PostgresMigrationStoreOptions) {}++ async fetchRows(versions: number[]): Promise<SqlxMigrationRow[]> {+ const rows = await queryInTransaction<{ version: string; success: boolean }>(+ this.options,+ FETCH_SQLX_ROWS_SQL,+ [versions.map(String)]+ )+ return rows.map((row) => ({ version: Number(row.version), success: row.success }))+ }+}