:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(cdp): rerun selects runs in the window of their first scheduled time (#129907)
Michael Reichenbach44035b076f6b
diff --git a/nodejs/src/cdp/cdp-api.ts b/nodejs/src/cdp/cdp-api.tsindex babfcb1e2fd5..d31569ef5ba9 100644--- a/nodejs/src/cdp/cdp-api.ts+++ b/nodejs/src/cdp/cdp-api.ts@@ -362,6 +362,7 @@ export class CdpApi {this.rerunJobManager = new RerunJobManager({dbUrl: this.config.CYCLOTRON_NODE_DATABASE_URL,maxCount: this.config.HOG_INVOCATION_RERUN_MAX_COUNT,+ runStartPagingTeams: this.config.CDP_RERUN_RUN_START_PAGING_TEAMS,})await this.rerunJobManager.connect()}diff --git a/nodejs/src/cdp/config.ts b/nodejs/src/cdp/config.tsindex 38d36652a100..0a798c47ce61 100644--- a/nodejs/src/cdp/config.ts+++ b/nodejs/src/cdp/config.ts@@ -141,6 +141,9 @@ export type CdpConfig = ClickhouseConfig & {EMAIL_STATUS_PRODUCER: CdpProducerNameEMAIL_STATUS_ENABLED: booleanHOG_INVOCATION_RERUN_MAX_COUNT: number+ // Comma-separated team ids, or "*", whose new rerun jobs page by run start. Enable it only once every+ // cdp-rerun-worker runs a release that reads the job's paging mode.+ CDP_RERUN_RUN_START_PAGING_TEAMS: string// How many rerun wrapper jobs the worker dequeues per cyclotron-v2 poll.// Kept small by default — each job runs a full ClickHouse query per page.CDP_RERUN_WORKER_BATCH_SIZE: number@@ -396,6 +399,7 @@ export function getDefaultCdpConfig(): CdpConfig {// Hard cap on rows a single rerun wrapper job will drain. Mirrors the// Django serializer's HOG_INVOCATION_RERUN_MAX_COUNT (same env var).HOG_INVOCATION_RERUN_MAX_COUNT: 10000,+ CDP_RERUN_RUN_START_PAGING_TEAMS: '',// Small by default — rerun jobs are heavy (a full ClickHouse query per// page), so a replica drains one wrapper job at a time unless tuned up.CDP_RERUN_WORKER_BATCH_SIZE: 1,diff --git a/nodejs/src/cdp/rerun/rerun-job.manager.test.ts b/nodejs/src/cdp/rerun/rerun-job.manager.test.tsindex 48b48a05e46d..b3822cbe4115 100644--- a/nodejs/src/cdp/rerun/rerun-job.manager.test.ts+++ b/nodejs/src/cdp/rerun/rerun-job.manager.test.ts@@ -15,7 +15,7 @@ describe('RerunJobManager', () => {beforeAll(async () => {assertPool = new Pool({ connectionString: DB_URL })- manager = new RerunJobManager({ dbUrl: DB_URL, maxCount: TEST_MAX_COUNT })+ manager = new RerunJobManager({ dbUrl: DB_URL, maxCount: TEST_MAX_COUNT, runStartPagingTeams: '7' })await manager.connect()})@@ -85,6 +85,7 @@ describe('RerunJobManager', () => {expect(state.function_id).toBe(functionId)expect(state.request.filter.invocation_ids).toEqual(ids)expect(state.request.filter.window_start).toBe(baseFilter.window_start)+ expect(state.paging).toBe('run_start')expect(state.progress).toEqual({queued: 0,skipped: 0,@@ -121,6 +122,7 @@ describe('RerunJobManager', () => {expect(state.request.filter).toMatchObject(filter)expect(state.request.filter.invocation_ids).toBeUndefined()expect(state.request.replay_code_sends).toBe(false)+ expect(state.paging).toBe('last_row')expect(state.progress.cursor).toBeUndefined()expect(state.progress.done).toBe(false)})diff --git a/nodejs/src/cdp/rerun/rerun-job.manager.ts b/nodejs/src/cdp/rerun/rerun-job.manager.tsindex d477f3791691..62aca112381a 100644--- a/nodejs/src/cdp/rerun/rerun-job.manager.ts+++ b/nodejs/src/cdp/rerun/rerun-job.manager.ts@@ -1,5 +1,6 @@import { v7 as uuidv7 } from 'uuid'+import { buildIntegerMatcher } from '~/common/config/matchers'import { logger } from '~/common/utils/logger'import { CyclotronV2Manager } from '../services/cyclotron-v2'@@ -18,6 +19,7 @@ export interface RerunJobManagerConfig {depthLimit?: number/** Mirror of the Django serializer cap (HOG_INVOCATION_RERUN_MAX_COUNT env var). */maxCount: number+ runStartPagingTeams?: string}/**@@ -31,8 +33,10 @@ export interface RerunJobManagerConfig {*/export class RerunJobManager {private manager: CyclotronV2Manager+ private pagesByRunStart: (teamId: number) => booleanconstructor(private config: RerunJobManagerConfig) {+ this.pagesByRunStart = buildIntegerMatcher(config.runStartPagingTeams, true)this.manager = new CyclotronV2Manager({pool: {dbUrl: config.dbUrl,@@ -83,6 +87,7 @@ export class RerunJobManager {const state: RerunJobState = {function_kind: functionKind,function_id: functionId,+ paging: this.pagesByRunStart(teamId) ? 'run_start' : 'last_row',request: { filter, replay_code_sends: request.replay_code_sends === true },progress: {queued: 0,diff --git a/nodejs/src/cdp/rerun/rerun-job.types.ts b/nodejs/src/cdp/rerun/rerun-job.types.tsindex 1bae853a2ff9..dbbe056bd6b1 100644--- a/nodejs/src/cdp/rerun/rerun-job.types.ts+++ b/nodejs/src/cdp/rerun/rerun-job.types.ts@@ -39,11 +39,8 @@ export type RerunStatusValue = 'running' | 'succeeded' | 'failed' | 'canceled'/*** Filter shape for a rerun request.*- * `window_start` / `window_end` are REQUIRED — the lifecycle table is- * partitioned by `toYYYYMMDD(scheduled_at)`, so a query without a time bound- * scans every partition (up to 30 of them, since the TTL drops parts older- * than that). The window also caps rerun to data that's still resident in the- * table — older rows are gone via TTL anyway.+ * `window_start` / `window_end` are REQUIRED and match the run's first scheduled time.+ * The window also caps rerun to data that's still resident in the table — older rows are gone via TTL anyway.** `invocation_ids` is an OPTIONAL additional restriction. When set, the* paginator pulls only those ids within the window — a UI "rerun these@@ -82,10 +79,16 @@ export interface RerunCursor {invocation_id: string}+/**+ * Fixed when the job is created, because a previous-release worker resuming a `run_start` cursor collapses+ * only part of a run's rows. A job without the field pages as `last_row`, as the previous release does.+ */+export type RerunPagingMode = 'run_start' | 'last_row'+export interface RerunJobProgress {queued: numberskipped: number- /** Keyset cursor on (scheduled_at, invocation_id). Undefined on first page; null when exhausted. */+ /** Keyset cursor on the paging mode's time and invocation_id. Undefined on first page; null when exhausted. */cursor?: RerunCursor | null/** True once the worker has processed every page or exhausted max_count. */done: boolean@@ -111,6 +114,7 @@ export interface RerunJobProgress {export interface RerunJobState {function_kind: RerunFunctionKindfunction_id: string+ paging?: RerunPagingModerequest: RerunRequestprogress: RerunJobProgress}diff --git a/nodejs/src/cdp/rerun/rerun-paginator.replay-fidelity.test.ts b/nodejs/src/cdp/rerun/rerun-paginator.replay-fidelity.test.tsindex 6b2f645ac8f5..23d7d5aaee28 100644--- a/nodejs/src/cdp/rerun/rerun-paginator.replay-fidelity.test.ts+++ b/nodejs/src/cdp/rerun/rerun-paginator.replay-fidelity.test.ts@@ -111,7 +111,7 @@ describe('RerunPaginatorService replay fidelity (hog_flow)', () => {parent_run_id: 'run-1',attempts: 0,last_scheduled_at: '2026-06-01 09:00:00.000000',- first_scheduled_at: '2026-06-01 09:00:00.000000',+ run_first_scheduled_at: '2026-06-01 09:00:00.000000',// decodeInvocationGlobals treats a leading '{' as raw JSON.invocation_globals: JSON.stringify(persistedState),},@@ -182,7 +182,7 @@ describe('RerunPaginatorService replay fidelity (hog_flow)', () => {parent_run_id: '',attempts: 0,last_scheduled_at: '2026-06-01 09:00:00.000000',- first_scheduled_at: '2026-06-01 09:00:00.000000',+ run_first_scheduled_at: '2026-06-01 09:00:00.000000',invocation_globals: JSON.stringify(persistedState),},]@@ -221,7 +221,7 @@ describe('RerunPaginatorService replay fidelity (hog_flow)', () => {parent_run_id: '',attempts: 0,last_scheduled_at: '2026-06-01 09:00:00.000000',- first_scheduled_at: '2026-06-01 09:00:00.000000',+ run_first_scheduled_at: '2026-06-01 09:00:00.000000',invocation_globals: JSON.stringify(persistedState),},]diff --git a/nodejs/src/cdp/rerun/rerun-paginator.routing.test.ts b/nodejs/src/cdp/rerun/rerun-paginator.routing.test.tsindex 93a41cdb4468..5da3332dec6c 100644--- a/nodejs/src/cdp/rerun/rerun-paginator.routing.test.ts+++ b/nodejs/src/cdp/rerun/rerun-paginator.routing.test.ts@@ -158,7 +158,7 @@ describe('RerunPaginatorService queue routing', () => {parent_run_id: '',attempts: 0,last_scheduled_at: '2026-01-01 00:00:00',- first_scheduled_at: '2026-01-01 00:00:00',+ run_first_scheduled_at: '2026-01-01 00:00:00',// Well-formed globals so the rerunnable-type cases pass rehydration's// schema validation — the type gate is what these tests exercise.invocation_globals: JSON.stringify({diff --git a/nodejs/src/cdp/rerun/rerun-paginator.service.test.ts b/nodejs/src/cdp/rerun/rerun-paginator.service.test.tsindex 3a794c4634ba..81afbc14c71a 100644--- a/nodejs/src/cdp/rerun/rerun-paginator.service.test.ts+++ b/nodejs/src/cdp/rerun/rerun-paginator.service.test.ts@@ -89,6 +89,7 @@ describe('RerunPaginatorService integration', () => {// max_attempts filter. Defaults to 0.rerunAttempts?: numberscheduledAt?: Date+ firstScheduledAt?: Date}>): Promise<void> => {for (const r of rows) {@@ -108,6 +109,7 @@ describe('RerunPaginatorService integration', () => {timings: [],attempts: 0,rerunAttempts: r.rerunAttempts,+ firstScheduledAt: r.firstScheduledAt?.toISOString().replace('T', ' ').replace('Z', '000'),},teamId: team.id,functionId: hogFunction.id,@@ -269,6 +271,9 @@ describe('RerunPaginatorService integration', () => {await closeHub(hub)})+ const enqueuedIds = (): string[] =>+ hogQueue.queueInvocations.mock.calls.flatMap(([invocations]) => invocations.map((i) => i.id))+const buildState = (overrides: Partial<RerunJobState> = {}): RerunJobState => ({function_kind: 'hog_function',function_id: hogFunction.id,@@ -530,6 +535,95 @@ describe('RerunPaginatorService integration', () => {expect(next.progress.skipped).toBe(1)})+ it('matches a run by the window of its first scheduled time, whatever day its last row lands on', async () => {+ const run = (id: string, start: string, end: string) => [+ {+ invocation_id: id,+ status: 'running' as const,+ scheduledAt: new Date(start),+ firstScheduledAt: new Date(start),+ },+ {+ invocation_id: id,+ status: 'failed' as const,+ error: new Error('boom'),+ scheduledAt: new Date(end),+ firstScheduledAt: new Date(start),+ },+ ]+ await seedRows([+ ...run('finishes-after-window', '2026-02-01T23:00:00Z', '2026-02-02T01:00:00Z'),+ ...run('started-before-window', '2026-01-31T23:00:00Z', '2026-02-01T01:00:00Z'),+ ])++ const state = buildState({+ paging: 'run_start',+ request: {+ filter: { window_start: '2026-02-01T00:00:00Z', window_end: '2026-02-02T00:00:00Z' },+ },+ })++ await paginator.processPage(team.id, state, { jobId: 'test-rerun-job', createdAt: DateTime.now() })+ expect(enqueuedIds()).toEqual(['finishes-after-window'])+ })++ it.each([+ {+ paging: 'run_start' as const,+ cursor: { scheduled_at: '2026-02-01 12:00:00.000000', invocation_id: 'later-start' },+ expectedIds: ['spans-days'],+ expectedCursor: { scheduled_at: '2026-02-01 10:00:00.000000', invocation_id: 'spans-days' },+ },+ {+ // A job created before the switch carries no paging mode and keeps paging by last row.+ paging: undefined,+ cursor: { scheduled_at: '2026-02-02 09:00:00.000000', invocation_id: 'spans-days' },+ expectedIds: ['later-start'],+ expectedCursor: { scheduled_at: '2026-02-01 12:00:00.000000', invocation_id: 'later-start' },+ },+ ])(+ 'resumes a $paging job from its cursor without skipping or repeating a run',+ async ({ paging, cursor, expectedIds, expectedCursor }) => {+ await seedRows([+ {+ invocation_id: 'spans-days',+ status: 'running',+ scheduledAt: new Date('2026-02-01T10:00:00Z'),+ firstScheduledAt: new Date('2026-02-01T10:00:00Z'),+ },+ {+ invocation_id: 'later-start',+ status: 'failed',+ error: new Error('boom'),+ scheduledAt: new Date('2026-02-01T12:00:00Z'),+ firstScheduledAt: new Date('2026-02-01T12:00:00Z'),+ },+ {+ invocation_id: 'spans-days',+ status: 'failed',+ error: new Error('boom'),+ scheduledAt: new Date('2026-02-02T09:00:00Z'),+ firstScheduledAt: new Date('2026-02-01T10:00:00Z'),+ },+ ])++ const state = buildState({+ paging,+ request: {+ filter: { window_start: '2026-02-01T00:00:00Z', window_end: '2026-02-03T00:00:00Z' },+ },+ progress: { queued: 0, skipped: 0, done: false, cursor },+ })++ const { state: next } = await paginator.processPage(team.id, state, {+ jobId: 'test-rerun-job',+ createdAt: DateTime.now(),+ })+ expect(enqueuedIds()).toEqual(expectedIds)+ expect(next.progress.cursor).toEqual(expectedCursor)+ }+ )+it('honours max_count by capping queued+skipped at the user-provided limit', async () => {await seedRows([{ invocation_id: 'a', status: 'failed', error: new Error('5xx') },diff --git a/nodejs/src/cdp/rerun/rerun-paginator.service.ts b/nodejs/src/cdp/rerun/rerun-paginator.service.tsindex 60c46883825b..991f375c3ad1 100644--- a/nodejs/src/cdp/rerun/rerun-paginator.service.ts+++ b/nodejs/src/cdp/rerun/rerun-paginator.service.ts@@ -32,6 +32,7 @@ import {RerunFunctionKind,RerunJobProgress,RerunJobState,+ RerunPagingMode,} from './rerun-job.types'const counterRerunPageProcessed = new Counter({@@ -69,6 +70,29 @@ const toClickhouseDateTime = (value: string): string => {.replace(/([+-]\d{2}):?(\d{2})$/, '')}+interface Paging {+ keyColumn: 'first_scheduled_at' | 'scheduled_at'+ orderColumn: 'run_first_scheduled_at' | 'last_scheduled_at'+ cursorTime: (row: InvocationRow) => string+}++const PAGING: Record<RerunPagingMode, Paging> = {+ // Every row of a run shares `first_scheduled_at`, so the window and cursor keep or drop whole runs.+ // There is no `scheduled_at` bound, because a rescheduled row can land before the run's start.+ run_start: {+ keyColumn: 'first_scheduled_at',+ orderColumn: 'run_first_scheduled_at',+ cursorTime: (row) => row.run_first_scheduled_at,+ },+ last_row: {+ keyColumn: 'scheduled_at',+ orderColumn: 'last_scheduled_at',+ cursorTime: (row) => row.last_scheduled_at,+ },+}++const pagingFor = (state: RerunJobState): Paging => PAGING[state.paging ?? 'last_row']+interface InvocationRow {invocation_id: stringparent_run_id: string@@ -76,7 +100,7 @@ interface InvocationRow {// shadowing the raw `attempts` column in the fetch query — see fetchPage.latest_attempts: numberlast_scheduled_at: string- first_scheduled_at: string+ run_first_scheduled_at: stringinvocation_globals: string}@@ -141,9 +165,8 @@ export class RerunPaginatorService {const remainingBudget = this.remainingBudget(state)const toProcess = rows.slice(0, remainingBudget)- // A historical window can't see the newer lifecycle rows a prior replay wrote —- // they land in the partition of their own scheduled_at, outside the window — so- // the in-window collapse can report 'failed' for an invocation that has since+ // A `last_row` page, or a row written without its run's first_scheduled_at, can miss the+ // newer lifecycle rows a prior replay wrote, so the collapse can report 'failed' for an invocation that has since// succeeded. Re-check the page against each invocation's latest status across// all partitions; without this, replaying the same window twice re-fires side// effects (emails, webhooks) for already-recovered runs.@@ -435,7 +458,7 @@ export class RerunPaginatorService {return null}const last = processed[processed.length - 1]- return { scheduled_at: last.last_scheduled_at, invocation_id: last.invocation_id }+ return { scheduled_at: pagingFor(state).cursorTime(last), invocation_id: last.invocation_id }}/**@@ -537,13 +560,15 @@ export class RerunPaginatorService {? toClickhouseDateTime(state.progress.cursor.scheduled_at): ''- // Keyset pagination on (scheduled_at, invocation_id). When the cursor+ const { keyColumn, orderColumn } = pagingFor(state)++ // Keyset pagination on (key, invocation_id). When the cursor// is undefined we start from the top of the window. ClickHouse needs// the tuple compared via a single AND-chained inequality.const cursor = state.progress.cursorconst cursorClause =cursor && cursor.scheduled_at- ? " AND (scheduled_at, invocation_id) < ({cursor_scheduled_at:DateTime64(6, 'UTC')}, {cursor_invocation_id:String})"+ ? ` AND (${keyColumn}, invocation_id) < ({cursor_scheduled_at:DateTime64(6, 'UTC')}, {cursor_invocation_id:String})`: ''const errorKindClause = filter.error_kind?.length? 'AND argMax(error_kind, version) IN {error_kind:Array(String)}'@@ -556,8 +581,7 @@ export class RerunPaginatorService {const maxAttemptsClause =filter.max_attempts !== undefined ? 'AND argMax(attempts, version) < {max_attempts:UInt8}' : ''// `invocation_ids` is an OPTIONAL additional restriction layered on top- // of the time window. The window is still required — it pins the query- // to a small set of date partitions instead of scanning everything.+ // of the time window, which is still required.const invocationIdsClause = filter.invocation_ids?.length? 'AND invocation_id IN {invocation_ids:Array(String)}': ''@@ -574,14 +598,15 @@ export class RerunPaginatorService {-- pill autodrain always does) would fail every page. Keep the names distinct.argMax(attempts, version) AS latest_attempts,argMax(invocation_globals, version) AS invocation_globals,- argMax(first_scheduled_at, version) AS first_scheduled_at,+ -- Not aliased 'first_scheduled_at': the alias would replace the raw column in the WHERE.+ argMax(first_scheduled_at, version) AS run_first_scheduled_at,max(scheduled_at) AS last_scheduled_atFROM hog_invocation_resultsWHERE team_id = {team_id:Int64}AND function_kind = {function_kind:String}AND function_id = {function_id:String}- AND scheduled_at >= {window_start:DateTime64(6, 'UTC')}- AND scheduled_at < {window_end:DateTime64(6, 'UTC')}+ AND ${keyColumn} >= {window_start:DateTime64(6, 'UTC')}+ AND ${keyColumn} < {window_end:DateTime64(6, 'UTC')}${invocationIdsClause}${cursorClause}GROUP BY invocation_id@@ -590,7 +615,7 @@ export class RerunPaginatorService {${errorKindClause}${errorMessageClause}${maxAttemptsClause}- ORDER BY last_scheduled_at DESC, invocation_id DESC+ ORDER BY ${orderColumn} DESC, invocation_id DESCLIMIT {limit:UInt32}`,query_params: {team_id: teamId,@@ -752,7 +777,7 @@ export class RerunPaginatorService {// Carry the original first-scheduled time forward — the// producer writes this verbatim on every retry's lifecycle// rows so ReplacingMergeTree doesn't collapse it away.- firstScheduledAt: row.first_scheduled_at,+ firstScheduledAt: row.run_first_scheduled_at,},teamId,functionId,@@ -869,7 +894,7 @@ export class RerunPaginatorService {// lifecycle row producer can derive `attempts` / `is_retry`// for flows too, and the `max_attempts` guard actually trips.rerunAttempts: (row.latest_attempts || 0) + 1,- firstScheduledAt: row.first_scheduled_at,+ firstScheduledAt: row.run_first_scheduled_at,}return invocation}