Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(cdp): rerun selects runs in the window of their first scheduled time (#129907)

Michael Reichenbach44035b076f6b
diff --git a/nodejs/src/cdp/cdp-api.ts b/nodejs/src/cdp/cdp-api.ts
index babfcb1e2fd5..d31569ef5ba9 100644
--- a/nodejs/src/cdp/cdp-api.ts
+++ b/nodejs/src/cdp/cdp-api.ts
@@ -362,6 +362,7 @@ export class CdpApi {
this.rerunJobManager = new RerunJobManager({
dbUrl: this.config.CYCLOTRON_NODE_DATABASE_URL,
maxCount: this.config.HOG_INVOCATION_RERUN_MAX_COUNT,
+ runStartPagingTeams: this.config.CDP_RERUN_RUN_START_PAGING_TEAMS,
})
await this.rerunJobManager.connect()
}
diff --git a/nodejs/src/cdp/config.ts b/nodejs/src/cdp/config.ts
index 38d36652a100..0a798c47ce61 100644
--- a/nodejs/src/cdp/config.ts
+++ b/nodejs/src/cdp/config.ts
@@ -141,6 +141,9 @@ export type CdpConfig = ClickhouseConfig & {
EMAIL_STATUS_PRODUCER: CdpProducerName
EMAIL_STATUS_ENABLED: boolean
HOG_INVOCATION_RERUN_MAX_COUNT: number
+ // Comma-separated team ids, or "*", whose new rerun jobs page by run start. Enable it only once every
+ // cdp-rerun-worker runs a release that reads the job's paging mode.
+ CDP_RERUN_RUN_START_PAGING_TEAMS: string
// How many rerun wrapper jobs the worker dequeues per cyclotron-v2 poll.
// Kept small by default — each job runs a full ClickHouse query per page.
CDP_RERUN_WORKER_BATCH_SIZE: number
@@ -396,6 +399,7 @@ export function getDefaultCdpConfig(): CdpConfig {
// Hard cap on rows a single rerun wrapper job will drain. Mirrors the
// Django serializer's HOG_INVOCATION_RERUN_MAX_COUNT (same env var).
HOG_INVOCATION_RERUN_MAX_COUNT: 10000,
+ CDP_RERUN_RUN_START_PAGING_TEAMS: '',
// Small by default — rerun jobs are heavy (a full ClickHouse query per
// page), so a replica drains one wrapper job at a time unless tuned up.
CDP_RERUN_WORKER_BATCH_SIZE: 1,
diff --git a/nodejs/src/cdp/rerun/rerun-job.manager.test.ts b/nodejs/src/cdp/rerun/rerun-job.manager.test.ts
index 48b48a05e46d..b3822cbe4115 100644
--- a/nodejs/src/cdp/rerun/rerun-job.manager.test.ts
+++ b/nodejs/src/cdp/rerun/rerun-job.manager.test.ts
@@ -15,7 +15,7 @@ describe('RerunJobManager', () => {
beforeAll(async () => {
assertPool = new Pool({ connectionString: DB_URL })
- manager = new RerunJobManager({ dbUrl: DB_URL, maxCount: TEST_MAX_COUNT })
+ manager = new RerunJobManager({ dbUrl: DB_URL, maxCount: TEST_MAX_COUNT, runStartPagingTeams: '7' })
await manager.connect()
})
@@ -85,6 +85,7 @@ describe('RerunJobManager', () => {
expect(state.function_id).toBe(functionId)
expect(state.request.filter.invocation_ids).toEqual(ids)
expect(state.request.filter.window_start).toBe(baseFilter.window_start)
+ expect(state.paging).toBe('run_start')
expect(state.progress).toEqual({
queued: 0,
skipped: 0,
@@ -121,6 +122,7 @@ describe('RerunJobManager', () => {
expect(state.request.filter).toMatchObject(filter)
expect(state.request.filter.invocation_ids).toBeUndefined()
expect(state.request.replay_code_sends).toBe(false)
+ expect(state.paging).toBe('last_row')
expect(state.progress.cursor).toBeUndefined()
expect(state.progress.done).toBe(false)
})
diff --git a/nodejs/src/cdp/rerun/rerun-job.manager.ts b/nodejs/src/cdp/rerun/rerun-job.manager.ts
index d477f3791691..62aca112381a 100644
--- a/nodejs/src/cdp/rerun/rerun-job.manager.ts
+++ b/nodejs/src/cdp/rerun/rerun-job.manager.ts
@@ -1,5 +1,6 @@
import { v7 as uuidv7 } from 'uuid'
+import { buildIntegerMatcher } from '~/common/config/matchers'
import { logger } from '~/common/utils/logger'
import { CyclotronV2Manager } from '../services/cyclotron-v2'
@@ -18,6 +19,7 @@ export interface RerunJobManagerConfig {
depthLimit?: number
/** Mirror of the Django serializer cap (HOG_INVOCATION_RERUN_MAX_COUNT env var). */
maxCount: number
+ runStartPagingTeams?: string
}
/**
@@ -31,8 +33,10 @@ export interface RerunJobManagerConfig {
*/
export class RerunJobManager {
private manager: CyclotronV2Manager
+ private pagesByRunStart: (teamId: number) => boolean
constructor(private config: RerunJobManagerConfig) {
+ this.pagesByRunStart = buildIntegerMatcher(config.runStartPagingTeams, true)
this.manager = new CyclotronV2Manager({
pool: {
dbUrl: config.dbUrl,
@@ -83,6 +87,7 @@ export class RerunJobManager {
const state: RerunJobState = {
function_kind: functionKind,
function_id: functionId,
+ paging: this.pagesByRunStart(teamId) ? 'run_start' : 'last_row',
request: { filter, replay_code_sends: request.replay_code_sends === true },
progress: {
queued: 0,
diff --git a/nodejs/src/cdp/rerun/rerun-job.types.ts b/nodejs/src/cdp/rerun/rerun-job.types.ts
index 1bae853a2ff9..dbbe056bd6b1 100644
--- a/nodejs/src/cdp/rerun/rerun-job.types.ts
+++ b/nodejs/src/cdp/rerun/rerun-job.types.ts
@@ -39,11 +39,8 @@ export type RerunStatusValue = 'running' | 'succeeded' | 'failed' | 'canceled'
/**
* Filter shape for a rerun request.
*
- * `window_start` / `window_end` are REQUIRED — the lifecycle table is
- * partitioned by `toYYYYMMDD(scheduled_at)`, so a query without a time bound
- * scans every partition (up to 30 of them, since the TTL drops parts older
- * than that). The window also caps rerun to data that's still resident in the
- * table — older rows are gone via TTL anyway.
+ * `window_start` / `window_end` are REQUIRED and match the run's first scheduled time.
+ * The window also caps rerun to data that's still resident in the table — older rows are gone via TTL anyway.
*
* `invocation_ids` is an OPTIONAL additional restriction. When set, the
* paginator pulls only those ids within the window — a UI "rerun these
@@ -82,10 +79,16 @@ export interface RerunCursor {
invocation_id: string
}
+/**
+ * Fixed when the job is created, because a previous-release worker resuming a `run_start` cursor collapses
+ * only part of a run's rows. A job without the field pages as `last_row`, as the previous release does.
+ */
+export type RerunPagingMode = 'run_start' | 'last_row'
+
export interface RerunJobProgress {
queued: number
skipped: number
- /** Keyset cursor on (scheduled_at, invocation_id). Undefined on first page; null when exhausted. */
+ /** Keyset cursor on the paging mode's time and invocation_id. Undefined on first page; null when exhausted. */
cursor?: RerunCursor | null
/** True once the worker has processed every page or exhausted max_count. */
done: boolean
@@ -111,6 +114,7 @@ export interface RerunJobProgress {
export interface RerunJobState {
function_kind: RerunFunctionKind
function_id: string
+ paging?: RerunPagingMode
request: RerunRequest
progress: RerunJobProgress
}
diff --git a/nodejs/src/cdp/rerun/rerun-paginator.replay-fidelity.test.ts b/nodejs/src/cdp/rerun/rerun-paginator.replay-fidelity.test.ts
index 6b2f645ac8f5..23d7d5aaee28 100644
--- a/nodejs/src/cdp/rerun/rerun-paginator.replay-fidelity.test.ts
+++ b/nodejs/src/cdp/rerun/rerun-paginator.replay-fidelity.test.ts
@@ -111,7 +111,7 @@ describe('RerunPaginatorService replay fidelity (hog_flow)', () => {
parent_run_id: 'run-1',
attempts: 0,
last_scheduled_at: '2026-06-01 09:00:00.000000',
- first_scheduled_at: '2026-06-01 09:00:00.000000',
+ run_first_scheduled_at: '2026-06-01 09:00:00.000000',
// decodeInvocationGlobals treats a leading '{' as raw JSON.
invocation_globals: JSON.stringify(persistedState),
},
@@ -182,7 +182,7 @@ describe('RerunPaginatorService replay fidelity (hog_flow)', () => {
parent_run_id: '',
attempts: 0,
last_scheduled_at: '2026-06-01 09:00:00.000000',
- first_scheduled_at: '2026-06-01 09:00:00.000000',
+ run_first_scheduled_at: '2026-06-01 09:00:00.000000',
invocation_globals: JSON.stringify(persistedState),
},
]
@@ -221,7 +221,7 @@ describe('RerunPaginatorService replay fidelity (hog_flow)', () => {
parent_run_id: '',
attempts: 0,
last_scheduled_at: '2026-06-01 09:00:00.000000',
- first_scheduled_at: '2026-06-01 09:00:00.000000',
+ run_first_scheduled_at: '2026-06-01 09:00:00.000000',
invocation_globals: JSON.stringify(persistedState),
},
]
diff --git a/nodejs/src/cdp/rerun/rerun-paginator.routing.test.ts b/nodejs/src/cdp/rerun/rerun-paginator.routing.test.ts
index 93a41cdb4468..5da3332dec6c 100644
--- a/nodejs/src/cdp/rerun/rerun-paginator.routing.test.ts
+++ b/nodejs/src/cdp/rerun/rerun-paginator.routing.test.ts
@@ -158,7 +158,7 @@ describe('RerunPaginatorService queue routing', () => {
parent_run_id: '',
attempts: 0,
last_scheduled_at: '2026-01-01 00:00:00',
- first_scheduled_at: '2026-01-01 00:00:00',
+ run_first_scheduled_at: '2026-01-01 00:00:00',
// Well-formed globals so the rerunnable-type cases pass rehydration's
// schema validation — the type gate is what these tests exercise.
invocation_globals: JSON.stringify({
diff --git a/nodejs/src/cdp/rerun/rerun-paginator.service.test.ts b/nodejs/src/cdp/rerun/rerun-paginator.service.test.ts
index 3a794c4634ba..81afbc14c71a 100644
--- a/nodejs/src/cdp/rerun/rerun-paginator.service.test.ts
+++ b/nodejs/src/cdp/rerun/rerun-paginator.service.test.ts
@@ -89,6 +89,7 @@ describe('RerunPaginatorService integration', () => {
// max_attempts filter. Defaults to 0.
rerunAttempts?: number
scheduledAt?: Date
+ firstScheduledAt?: Date
}>
): Promise<void> => {
for (const r of rows) {
@@ -108,6 +109,7 @@ describe('RerunPaginatorService integration', () => {
timings: [],
attempts: 0,
rerunAttempts: r.rerunAttempts,
+ firstScheduledAt: r.firstScheduledAt?.toISOString().replace('T', ' ').replace('Z', '000'),
},
teamId: team.id,
functionId: hogFunction.id,
@@ -269,6 +271,9 @@ describe('RerunPaginatorService integration', () => {
await closeHub(hub)
})
+ const enqueuedIds = (): string[] =>
+ hogQueue.queueInvocations.mock.calls.flatMap(([invocations]) => invocations.map((i) => i.id))
+
const buildState = (overrides: Partial<RerunJobState> = {}): RerunJobState => ({
function_kind: 'hog_function',
function_id: hogFunction.id,
@@ -530,6 +535,95 @@ describe('RerunPaginatorService integration', () => {
expect(next.progress.skipped).toBe(1)
})
+ it('matches a run by the window of its first scheduled time, whatever day its last row lands on', async () => {
+ const run = (id: string, start: string, end: string) => [
+ {
+ invocation_id: id,
+ status: 'running' as const,
+ scheduledAt: new Date(start),
+ firstScheduledAt: new Date(start),
+ },
+ {
+ invocation_id: id,
+ status: 'failed' as const,
+ error: new Error('boom'),
+ scheduledAt: new Date(end),
+ firstScheduledAt: new Date(start),
+ },
+ ]
+ await seedRows([
+ ...run('finishes-after-window', '2026-02-01T23:00:00Z', '2026-02-02T01:00:00Z'),
+ ...run('started-before-window', '2026-01-31T23:00:00Z', '2026-02-01T01:00:00Z'),
+ ])
+
+ const state = buildState({
+ paging: 'run_start',
+ request: {
+ filter: { window_start: '2026-02-01T00:00:00Z', window_end: '2026-02-02T00:00:00Z' },
+ },
+ })
+
+ await paginator.processPage(team.id, state, { jobId: 'test-rerun-job', createdAt: DateTime.now() })
+ expect(enqueuedIds()).toEqual(['finishes-after-window'])
+ })
+
+ it.each([
+ {
+ paging: 'run_start' as const,
+ cursor: { scheduled_at: '2026-02-01 12:00:00.000000', invocation_id: 'later-start' },
+ expectedIds: ['spans-days'],
+ expectedCursor: { scheduled_at: '2026-02-01 10:00:00.000000', invocation_id: 'spans-days' },
+ },
+ {
+ // A job created before the switch carries no paging mode and keeps paging by last row.
+ paging: undefined,
+ cursor: { scheduled_at: '2026-02-02 09:00:00.000000', invocation_id: 'spans-days' },
+ expectedIds: ['later-start'],
+ expectedCursor: { scheduled_at: '2026-02-01 12:00:00.000000', invocation_id: 'later-start' },
+ },
+ ])(
+ 'resumes a $paging job from its cursor without skipping or repeating a run',
+ async ({ paging, cursor, expectedIds, expectedCursor }) => {
+ await seedRows([
+ {
+ invocation_id: 'spans-days',
+ status: 'running',
+ scheduledAt: new Date('2026-02-01T10:00:00Z'),
+ firstScheduledAt: new Date('2026-02-01T10:00:00Z'),
+ },
+ {
+ invocation_id: 'later-start',
+ status: 'failed',
+ error: new Error('boom'),
+ scheduledAt: new Date('2026-02-01T12:00:00Z'),
+ firstScheduledAt: new Date('2026-02-01T12:00:00Z'),
+ },
+ {
+ invocation_id: 'spans-days',
+ status: 'failed',
+ error: new Error('boom'),
+ scheduledAt: new Date('2026-02-02T09:00:00Z'),
+ firstScheduledAt: new Date('2026-02-01T10:00:00Z'),
+ },
+ ])
+
+ const state = buildState({
+ paging,
+ request: {
+ filter: { window_start: '2026-02-01T00:00:00Z', window_end: '2026-02-03T00:00:00Z' },
+ },
+ progress: { queued: 0, skipped: 0, done: false, cursor },
+ })
+
+ const { state: next } = await paginator.processPage(team.id, state, {
+ jobId: 'test-rerun-job',
+ createdAt: DateTime.now(),
+ })
+ expect(enqueuedIds()).toEqual(expectedIds)
+ expect(next.progress.cursor).toEqual(expectedCursor)
+ }
+ )
+
it('honours max_count by capping queued+skipped at the user-provided limit', async () => {
await seedRows([
{ invocation_id: 'a', status: 'failed', error: new Error('5xx') },
diff --git a/nodejs/src/cdp/rerun/rerun-paginator.service.ts b/nodejs/src/cdp/rerun/rerun-paginator.service.ts
index 60c46883825b..991f375c3ad1 100644
--- a/nodejs/src/cdp/rerun/rerun-paginator.service.ts
+++ b/nodejs/src/cdp/rerun/rerun-paginator.service.ts
@@ -32,6 +32,7 @@ import {
RerunFunctionKind,
RerunJobProgress,
RerunJobState,
+ RerunPagingMode,
} from './rerun-job.types'
const counterRerunPageProcessed = new Counter({
@@ -69,6 +70,29 @@ const toClickhouseDateTime = (value: string): string => {
.replace(/([+-]\d{2}):?(\d{2})$/, '')
}
+interface Paging {
+ keyColumn: 'first_scheduled_at' | 'scheduled_at'
+ orderColumn: 'run_first_scheduled_at' | 'last_scheduled_at'
+ cursorTime: (row: InvocationRow) => string
+}
+
+const PAGING: Record<RerunPagingMode, Paging> = {
+ // Every row of a run shares `first_scheduled_at`, so the window and cursor keep or drop whole runs.
+ // There is no `scheduled_at` bound, because a rescheduled row can land before the run's start.
+ run_start: {
+ keyColumn: 'first_scheduled_at',
+ orderColumn: 'run_first_scheduled_at',
+ cursorTime: (row) => row.run_first_scheduled_at,
+ },
+ last_row: {
+ keyColumn: 'scheduled_at',
+ orderColumn: 'last_scheduled_at',
+ cursorTime: (row) => row.last_scheduled_at,
+ },
+}
+
+const pagingFor = (state: RerunJobState): Paging => PAGING[state.paging ?? 'last_row']
+
interface InvocationRow {
invocation_id: string
parent_run_id: string
@@ -76,7 +100,7 @@ interface InvocationRow {
// shadowing the raw `attempts` column in the fetch query — see fetchPage.
latest_attempts: number
last_scheduled_at: string
- first_scheduled_at: string
+ run_first_scheduled_at: string
invocation_globals: string
}
@@ -141,9 +165,8 @@ export class RerunPaginatorService {
const remainingBudget = this.remainingBudget(state)
const toProcess = rows.slice(0, remainingBudget)
- // A historical window can't see the newer lifecycle rows a prior replay wrote —
- // they land in the partition of their own scheduled_at, outside the window — so
- // the in-window collapse can report 'failed' for an invocation that has since
+ // A `last_row` page, or a row written without its run's first_scheduled_at, can miss the
+ // newer lifecycle rows a prior replay wrote, so the collapse can report 'failed' for an invocation that has since
// succeeded. Re-check the page against each invocation's latest status across
// all partitions; without this, replaying the same window twice re-fires side
// effects (emails, webhooks) for already-recovered runs.
@@ -435,7 +458,7 @@ export class RerunPaginatorService {
return null
}
const last = processed[processed.length - 1]
- return { scheduled_at: last.last_scheduled_at, invocation_id: last.invocation_id }
+ return { scheduled_at: pagingFor(state).cursorTime(last), invocation_id: last.invocation_id }
}
/**
@@ -537,13 +560,15 @@ export class RerunPaginatorService {
? toClickhouseDateTime(state.progress.cursor.scheduled_at)
: ''
- // Keyset pagination on (scheduled_at, invocation_id). When the cursor
+ const { keyColumn, orderColumn } = pagingFor(state)
+
+ // Keyset pagination on (key, invocation_id). When the cursor
// is undefined we start from the top of the window. ClickHouse needs
// the tuple compared via a single AND-chained inequality.
const cursor = state.progress.cursor
const cursorClause =
cursor && cursor.scheduled_at
- ? " AND (scheduled_at, invocation_id) < ({cursor_scheduled_at:DateTime64(6, 'UTC')}, {cursor_invocation_id:String})"
+ ? ` AND (${keyColumn}, invocation_id) < ({cursor_scheduled_at:DateTime64(6, 'UTC')}, {cursor_invocation_id:String})`
: ''
const errorKindClause = filter.error_kind?.length
? 'AND argMax(error_kind, version) IN {error_kind:Array(String)}'
@@ -556,8 +581,7 @@ export class RerunPaginatorService {
const maxAttemptsClause =
filter.max_attempts !== undefined ? 'AND argMax(attempts, version) < {max_attempts:UInt8}' : ''
// `invocation_ids` is an OPTIONAL additional restriction layered on top
- // of the time window. The window is still required — it pins the query
- // to a small set of date partitions instead of scanning everything.
+ // of the time window, which is still required.
const invocationIdsClause = filter.invocation_ids?.length
? 'AND invocation_id IN {invocation_ids:Array(String)}'
: ''
@@ -574,14 +598,15 @@ export class RerunPaginatorService {
-- pill autodrain always does) would fail every page. Keep the names distinct.
argMax(attempts, version) AS latest_attempts,
argMax(invocation_globals, version) AS invocation_globals,
- argMax(first_scheduled_at, version) AS first_scheduled_at,
+ -- Not aliased 'first_scheduled_at': the alias would replace the raw column in the WHERE.
+ argMax(first_scheduled_at, version) AS run_first_scheduled_at,
max(scheduled_at) AS last_scheduled_at
FROM hog_invocation_results
WHERE team_id = {team_id:Int64}
AND function_kind = {function_kind:String}
AND function_id = {function_id:String}
- AND scheduled_at >= {window_start:DateTime64(6, 'UTC')}
- AND scheduled_at < {window_end:DateTime64(6, 'UTC')}
+ AND ${keyColumn} >= {window_start:DateTime64(6, 'UTC')}
+ AND ${keyColumn} < {window_end:DateTime64(6, 'UTC')}
${invocationIdsClause}
${cursorClause}
GROUP BY invocation_id
@@ -590,7 +615,7 @@ export class RerunPaginatorService {
${errorKindClause}
${errorMessageClause}
${maxAttemptsClause}
- ORDER BY last_scheduled_at DESC, invocation_id DESC
+ ORDER BY ${orderColumn} DESC, invocation_id DESC
LIMIT {limit:UInt32}`,
query_params: {
team_id: teamId,
@@ -752,7 +777,7 @@ export class RerunPaginatorService {
// Carry the original first-scheduled time forward — the
// producer writes this verbatim on every retry's lifecycle
// rows so ReplacingMergeTree doesn't collapse it away.
- firstScheduledAt: row.first_scheduled_at,
+ firstScheduledAt: row.run_first_scheduled_at,
},
teamId,
functionId,
@@ -869,7 +894,7 @@ export class RerunPaginatorService {
// lifecycle row producer can derive `attempts` / `is_retry`
// for flows too, and the `max_attempts` guard actually trips.
rerunAttempts: (row.latest_attempts || 0) + 1,
- firstScheduledAt: row.first_scheduled_at,
+ firstScheduledAt: row.run_first_scheduled_at,
}
return invocation
}