:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { MockKafkaProducerWrapper } from '~/tests/helpers/mocks/producer.mock'
import { ClickHouseClient } from '@clickhouse/client'
import { DateTime } from 'luxon'
import { KAFKA_HOG_INVOCATION_RESULTS } from '~/common/config/kafka-topics'
import { KafkaProducerWrapper } from '~/common/kafka/producer'
import { closeHub, createHub } from '~/common/utils/db/hub'
import { UUIDT } from '~/common/utils/utils'
import { createCdpConsumerDeps } from '~/tests/helpers/cdp'
import { Clickhouse } from '~/tests/helpers/clickhouse'
import { waitForExpect } from '~/tests/helpers/expectations'
import { waitForHogInvocationResultsMvReady } from '~/tests/helpers/hog-invocation-results'
import { TEST_KAFKA_TOPICS, ensureKafkaTopics } from '~/tests/helpers/kafka'
import { createTestTeamFixture } from '~/tests/helpers/sql'
import { Hub, Team } from '../../types'
import { insertHogFunction as _insertHogFunction, createHogExecutionGlobals } from '../_tests/fixtures'
import { createCdpOutputsRegistry } from '../outputs/registry'
import { HogFlowManagerService } from '../services/hogflows/hogflow-manager.service'
import { CyclotronJobQueuePostgresV2 } from '../services/job-queue/job-queue-postgres-v2'
import { JobQueue } from '../services/job-queue/job-queue.interface'
import { HogFunctionManagerService } from '../services/managers/hog-function-manager.service'
import { HogFunctionMonitoringService } from '../services/monitoring/hog-function-monitoring.service'
import { HogInvocationResultsService } from '../services/monitoring/hog-invocation-results.service'
import { CyclotronJobInvocationHogFunction, HogFunctionInvocationGlobals, HogFunctionType } from '../types'
import { RERUN_MAX_CONSECUTIVE_PAGE_ERRORS, RERUN_PAGE_SIZE, RerunJobState } from './rerun-job.types'
import { RerunPaginatorService } from './rerun-paginator.service'
const ActualKafkaProducerWrapper = jest.requireActual('~/common/kafka/producer').KafkaProducerWrapper
/**
* Integration test for RerunPaginatorService.
*
* Instead of mocking the ClickHouse client, this test seeds real lifecycle rows
* through `HogInvocationResultsService.queueLifecycleRow` → Kafka → ClickHouse
* Kafka MV → `hog_invocation_results`, then runs the paginator against the real
* data. Only the side effects the paginator emits (the running lifecycle row +
* the cyclotron re-enqueue) are mocked, so we can assert what got produced.
*
* This proves the whole pipeline end-to-end: the queueLifecycleRow shape that
* the worker uses, the Kafka MV's JSON parsing, and the paginator's argMax
* collapse query all line up.
*/
describe('RerunPaginatorService integration', () => {
jest.setTimeout(180_000)
let hub: Hub
let kafkaProducer: KafkaProducerWrapper
let team: Team
let clickhouse: Clickhouse
let hogFunction: HogFunctionType
let seedingService: HogInvocationResultsService
let chClient: ClickHouseClient
let paginator: RerunPaginatorService
// hog functions re-enqueue to the kafka queue, hog flows to postgres-v2.
let hogQueue: jest.Mocked<JobQueue>
let hogflowQueue: jest.Mocked<CyclotronJobQueuePostgresV2>
let paginatorLifecycleService: jest.Mocked<HogInvocationResultsService>
let paginatorMonitoringService: jest.Mocked<HogFunctionMonitoringService>
let hogFunctionManager: HogFunctionManagerService
let hogFlowManager: jest.Mocked<HogFlowManagerService>
beforeAll(() => {
clickhouse = Clickhouse.create()
chClient = Clickhouse.createClient()
})
afterAll(async () => {
clickhouse?.close()
await chClient?.close()
})
let seededCount = 0
/**
* Seed `n` lifecycle rows for the given function. Each entry becomes a real
* Kafka produce + MV ingestion, so by the time this returns ClickHouse has
* the rows ready for the paginator to read.
*/
const seedRows = async (
rows: Array<{
invocation_id: string
status: 'running' | 'succeeded' | 'failed'
error?: Error
// Stable error_kind stamped on the row (e.g. 'janitor_poison_pill').
// When omitted it is derived from the error message, as before.
errorKind?: string
// Prior rerun count → the row's `attempts` column, for exercising the
// max_attempts filter. Defaults to 0.
rerunAttempts?: number
scheduledAt?: Date
firstScheduledAt?: Date
}>
): Promise<void> => {
for (const r of rows) {
const globals = createHogExecutionGlobals({
project: { id: team.id } as any,
event: {
uuid: `evt-${r.invocation_id}`,
event: '$pageview',
properties: {},
timestamp: '2026-05-10T09:00:00Z',
} as any,
})
const invocation: CyclotronJobInvocationHogFunction = {
id: r.invocation_id,
state: {
globals: globals as any,
timings: [],
attempts: 0,
rerunAttempts: r.rerunAttempts,
firstScheduledAt: r.firstScheduledAt?.toISOString().replace('T', ' ').replace('Z', '000'),
},
teamId: team.id,
functionId: hogFunction.id,
hogFunction,
queue: 'hog',
queuePriority: 0,
queueScheduledAt: r.scheduledAt ? ({ toJSDate: () => r.scheduledAt } as any) : undefined,
}
seedingService.queueLifecycleRow(invocation, r.status, { error: r.error, errorKind: r.errorKind })
}
await seedingService.flush()
await kafkaProducer.flush()
// Track cumulative seeded rows so calling seedRows twice in the same
// test waits for *all* rows (rather than trivially passing on the
// previous seed's count).
seededCount += rows.length
const expected = seededCount
// Seed visibility rides the shared Kafka -> ClickHouse pipe, whose consumer may still be
// chewing a backlog from whichever suite the shard ran just before this one (run order
// shifts whenever sibling files change size). The deadline is sized for that worst case;
// waitForExpect polls, so a healthy pipe still completes in seconds.
await waitForExpect(async () => {
const got = await clickhouse.query<{ c: number }>(
`SELECT count() AS c FROM hog_invocation_results
WHERE team_id = ${team.id}
AND function_id = '${hogFunction.id}'`
)
expect(Number(got[0]?.c ?? 0)).toBeGreaterThanOrEqual(expected)
}, 90_000)
}
// Produce a raw lifecycle row with a chosen (here: undecodable) invocation_globals,
// bypassing the seeding service so we can exercise the paginator's per-row
// rehydration-failure path — a poison-pill record whose stored globals can't be
// decoded must be skipped, not abort the whole recovery batch.
const seedRawRow = async (invocationId: string, invocationGlobals: string): Promise<void> => {
await kafkaProducer.queueMessages({
topic: KAFKA_HOG_INVOCATION_RESULTS,
messages: [
{
key: invocationId,
value: JSON.stringify({
team_id: team.id,
function_kind: 'hog_function',
function_id: hogFunction.id,
invocation_id: invocationId,
parent_run_id: '',
status: 'failed',
attempts: 0,
is_retry: 0,
scheduled_at: DateTime.utc().toFormat("yyyy-MM-dd HH:mm:ss.SSS'000'"),
started_at: null,
finished_at: null,
duration_ms: null,
error_kind: 'janitor_poison_pill',
error_message: 'poison pill',
event_uuid: '',
distinct_id: '',
person_id: '',
invocation_globals: invocationGlobals,
version: String(BigInt(Date.now()) * 1000n),
is_deleted: 0,
}),
},
],
})
await kafkaProducer.flush()
seededCount += 1
await waitForExpect(async () => {
const got = await clickhouse.query<{ c: number }>(
`SELECT count() AS c FROM hog_invocation_results
WHERE team_id = ${team.id} AND invocation_id = '${invocationId}'`
)
expect(Number(got[0]?.c ?? 0)).toBeGreaterThanOrEqual(1)
}, 90_000)
}
beforeAll(async () => {
MockKafkaProducerWrapper.create = jest.fn((...args: any[]) => ActualKafkaProducerWrapper.create(...args))
// Ensure all topics exist (idempotently, without deleting) so the ClickHouse Kafka engine
// consumers keep their connections. Includes KAFKA_HOG_INVOCATION_RESULTS, which this test's
// MV needs but the shared set does not cover.
await ensureKafkaTopics([...TEST_KAFKA_TOPICS, KAFKA_HOG_INVOCATION_RESULTS])
await waitForHogInvocationResultsMvReady(clickhouse)
})
beforeEach(async () => {
seededCount = 0
hub = await createHub()
kafkaProducer = await ActualKafkaProducerWrapper.create(hub.KAFKA_CLIENT_RACK)
team = (await createTestTeamFixture(hub.postgres)).team
// Real seeding path: outputs → kafka → MV → CH.
const deps = createCdpConsumerDeps(hub, kafkaProducer)
const outputs = createCdpOutputsRegistry().build(deps.cdpProducerRegistry, {
...hub,
HOG_INVOCATION_RESULTS_TOPIC: KAFKA_HOG_INVOCATION_RESULTS,
} as any)
seedingService = new HogInvocationResultsService(outputs, { HOG_INVOCATION_RESULTS_ENABLED: true })
hogFunction = await _insertHogFunction(hub.postgres, team.id, {
type: 'destination',
hog: 'print("hi")',
bytecode: [],
inputs_schema: [],
inputs: {},
})
// Real hog function manager so rehydrate can resolve the function from postgres.
hogFunctionManager = new HogFunctionManagerService(hub.postgres, hub.pubSub, hub.encryptedFields)
hogFunctionManager['onHogFunctionsReloaded'](team.id, [hogFunction.id])
// Hog flow manager stays mocked — this suite focuses on the hog-function
// path and the CH query/rehydrate logic. Hog flow rehydration is
// exercised structurally (rest of the same code path) but not end-to-end here.
hogFlowManager = {
getHogFlow: jest.fn().mockResolvedValue(null),
} as unknown as jest.Mocked<HogFlowManagerService>
hogQueue = {
queueInvocations: jest.fn().mockResolvedValue(undefined),
} as unknown as jest.Mocked<JobQueue>
hogflowQueue = {
queueInvocations: jest.fn().mockResolvedValue(undefined),
} as unknown as jest.Mocked<CyclotronJobQueuePostgresV2>
// The paginator's lifecycle service is mocked so we can assert what it
// got asked to write (the 'running' row for each rerun). The actual
// CH state for that running row is a flaky-on-Redpanda concern handled
// in the e2e test — here we only care that the paginator queues it.
paginatorLifecycleService = {
queueLifecycleRow: jest.fn(),
queueRerunWrapperRow: jest.fn(),
flush: jest.fn().mockResolvedValue(undefined),
dropQueuedRowsFor: jest.fn(),
} as unknown as jest.Mocked<HogInvocationResultsService>
paginatorMonitoringService = {
queueLogs: jest.fn(),
flush: jest.fn().mockResolvedValue(undefined),
} as unknown as jest.Mocked<HogFunctionMonitoringService>
paginator = new RerunPaginatorService(
chClient,
hogFunctionManager,
hogFlowManager,
paginatorLifecycleService,
{ hog_function: hogQueue, hog_flow: hogflowQueue },
paginatorMonitoringService,
10000
)
})
afterEach(async () => {
await kafkaProducer?.disconnect().catch(() => undefined)
await closeHub(hub)
})
const enqueuedIds = (): string[] =>
hogQueue.queueInvocations.mock.calls.flatMap(([invocations]) => invocations.map((i) => i.id))
const buildState = (overrides: Partial<RerunJobState> = {}): RerunJobState => ({
function_kind: 'hog_function',
function_id: hogFunction.id,
// Window is required by the unified rerun schema. Default to a wide
// year-long window so the seeded test data falls inside.
request: { filter: { window_start: '2026-01-01T00:00:00Z', window_end: '2027-01-01T00:00:00Z' } },
progress: { queued: 0, skipped: 0, done: false },
...overrides,
})
describe('by-ids mode', () => {
it('rehydrates and re-enqueues only the requested invocations, ignoring others', async () => {
await seedRows([
{ invocation_id: 'inv-a', status: 'failed', error: new Error('http_5xx error') },
{ invocation_id: 'inv-b', status: 'succeeded' },
{ invocation_id: 'inv-c', status: 'failed', error: new Error('timeout') },
])
const state = buildState({
request: {
filter: {
window_start: '2026-01-01T00:00:00Z',
window_end: '2027-01-01T00:00:00Z',
invocation_ids: ['inv-a', 'inv-c'],
},
},
})
const { state: next } = await paginator.processPage(team.id, state, {
jobId: 'test-rerun-job',
createdAt: DateTime.now(),
})
expect(hogQueue.queueInvocations).toHaveBeenCalledTimes(1)
const enqueued = hogQueue.queueInvocations.mock.calls[0][0] as CyclotronJobInvocationHogFunction[]
const enqueuedIds = enqueued.map((i) => i.id).sort()
expect(enqueuedIds).toEqual(['inv-a', 'inv-c'])
// Each rerun invocation gets a 'running' lifecycle row queued.
// is_retry / attempts are derived inside queueLifecycleRow from
// state.rerunAttempts (set by the rehydrator), not passed as an opt.
const runningCalls = paginatorLifecycleService.queueLifecycleRow.mock.calls.filter(
(c) => c[1] === 'running'
)
expect(runningCalls).toHaveLength(2)
// Each queued invocation's state should carry rerunAttempts=1.
for (const [invocation] of runningCalls) {
expect((invocation as CyclotronJobInvocationHogFunction).state.rerunAttempts).toBe(1)
}
expect(next.progress.queued).toBe(2)
expect(next.progress.skipped).toBe(0)
expect(next.progress.done).toBe(true)
})
it('skips invocation ids that have no matching row in clickhouse', async () => {
await seedRows([{ invocation_id: 'inv-real', status: 'failed', error: new Error('5xx') }])
const state = buildState({
request: {
filter: {
window_start: '2026-01-01T00:00:00Z',
window_end: '2027-01-01T00:00:00Z',
invocation_ids: ['inv-real', 'inv-missing'],
},
},
})
const { state: next } = await paginator.processPage(team.id, state, {
jobId: 'test-rerun-job',
createdAt: DateTime.now(),
})
const enqueued = hogQueue.queueInvocations.mock.calls[0][0] as CyclotronJobInvocationHogFunction[]
expect(enqueued.map((i) => i.id)).toEqual(['inv-real'])
// by-ids mode marks done once remaining_ids is drained, regardless of
// whether every id had a matching CH row.
expect(next.progress.done).toBe(true)
})
it('re-enqueues bare globals — inputs are not stored or pre-resolved by the paginator', async () => {
await seedRows([{ invocation_id: 'inv-rebuild', status: 'failed', error: new Error('boom') }])
const state = buildState({
request: {
filter: {
window_start: '2026-01-01T00:00:00Z',
window_end: '2027-01-01T00:00:00Z',
invocation_ids: ['inv-rebuild'],
},
},
})
await paginator.processPage(team.id, state, { jobId: 'test-rerun-job', createdAt: DateTime.now() })
// The re-enqueued invocation carries no resolved `inputs` — the
// executor rebuilds them from the current hog function config at
// run time, so the rerun never trusts a stored snapshot.
const enqueued = hogQueue.queueInvocations.mock.calls[0][0] as CyclotronJobInvocationHogFunction[]
expect(enqueued).toHaveLength(1)
expect(enqueued[0].state.globals).not.toHaveProperty('inputs')
})
it('skips a record that fails to rehydrate and still re-enqueues the rest of the batch', async () => {
await seedRows([{ invocation_id: 'inv-good', status: 'failed', error: new Error('5xx') }])
await seedRawRow('inv-bad', 'not-a-decodable-globals-blob')
const state = buildState({
request: {
filter: {
window_start: '2026-01-01T00:00:00Z',
window_end: '2027-01-01T00:00:00Z',
invocation_ids: ['inv-good', 'inv-bad'],
},
},
})
const { state: next } = await paginator.processPage(team.id, state, {
jobId: 'test-rerun-job',
createdAt: DateTime.now(),
})
// The undecodable record is skipped, not fatal — the good one still re-enqueues.
const enqueued = hogQueue.queueInvocations.mock.calls.flatMap(
(c) => c[0] as CyclotronJobInvocationHogFunction[]
)
expect(enqueued.map((i) => i.id)).toEqual(['inv-good'])
expect(next.progress.queued).toBe(1)
expect(next.progress.skipped).toBeGreaterThanOrEqual(1)
})
})
describe('by-filter mode', () => {
it('defaults to status=failed and only matches failed rows in the window', async () => {
await seedRows([
{ invocation_id: 'ok-1', status: 'succeeded' },
{ invocation_id: 'fail-1', status: 'failed', error: new Error('500 err') },
{ invocation_id: 'fail-2', status: 'failed', error: new Error('timeout') },
])
const state = buildState({
request: {
filter: { window_start: '2026-01-01T00:00:00Z', window_end: '2027-01-01T00:00:00Z' },
},
})
const { state: next } = await paginator.processPage(team.id, state, {
jobId: 'test-rerun-job',
createdAt: DateTime.now(),
})
const enqueued = hogQueue.queueInvocations.mock.calls[0][0] as CyclotronJobInvocationHogFunction[]
const ids = enqueued.map((i) => i.id).sort()
expect(ids).toEqual(['fail-1', 'fail-2'])
expect(next.progress.queued).toBe(2)
})
it('honours error_kind filter (only http_5xx rows are rerun)', async () => {
await seedRows([
{ invocation_id: 'inv-500', status: 'failed', error: new Error('500 server error') },
{ invocation_id: 'inv-timeout', status: 'failed', error: new Error('Request timed out') },
])
const state = buildState({
request: {
filter: {
window_start: '2026-01-01T00:00:00Z',
window_end: '2027-01-01T00:00:00Z',
error_kind: ['http_5xx'],
},
},
})
const { state: next } = await paginator.processPage(team.id, state, {
jobId: 'test-rerun-job',
createdAt: DateTime.now(),
})
const enqueued = hogQueue.queueInvocations.mock.calls[0]?.[0] as
| CyclotronJobInvocationHogFunction[]
| undefined
expect(enqueued?.map((i) => i.id)).toEqual(['inv-500'])
expect(next.progress.queued).toBe(1)
})
it('honours error_message_contains filter, case-insensitively', async () => {
await seedRows([
{
invocation_id: 'inv-sender',
status: 'failed',
error: new Error(
'The custom sender address "[email protected]" is not a valid email address.'
),
},
{ invocation_id: 'inv-other', status: 'failed', error: new Error('Some unrelated hog error') },
])
const state = buildState({
request: {
filter: {
window_start: '2026-01-01T00:00:00Z',
window_end: '2027-01-01T00:00:00Z',
// Deliberately lower-cased vs the stored message: a needle copied
// from logs must match regardless of capitalization.
error_message_contains: 'the custom sender address',
},
},
})
const { state: next } = await paginator.processPage(team.id, state, {
jobId: 'test-rerun-job',
createdAt: DateTime.now(),
})
const enqueued = hogQueue.queueInvocations.mock.calls[0]?.[0] as
| CyclotronJobInvocationHogFunction[]
| undefined
expect(enqueued?.map((i) => i.id)).toEqual(['inv-sender'])
expect(next.progress.queued).toBe(1)
})
it('skips invocations whose latest status outside the window no longer matches', async () => {
// Original failures inside a historical window...
await seedRows([
{
invocation_id: 'inv-replayed',
status: 'failed',
error: new Error('boom'),
scheduledAt: new Date('2026-02-01T10:00:00Z'),
},
{
invocation_id: 'inv-still-failed',
status: 'failed',
error: new Error('boom'),
scheduledAt: new Date('2026-02-01T11:00:00Z'),
},
])
// ...then a successful replay of one of them lands its lifecycle row in the
// partition of its own scheduled_at, outside the window. The windowed page
// query still sees the old failed row, so without the cross-partition status
// check a second rerun would re-enqueue it and re-fire its side effects.
await seedRows([
{ invocation_id: 'inv-replayed', status: 'succeeded', scheduledAt: new Date('2026-06-01T10:00:00Z') },
])
const state = buildState({
request: {
filter: { window_start: '2026-02-01T00:00:00Z', window_end: '2026-02-02T00:00:00Z' },
},
})
const { state: next } = await paginator.processPage(team.id, state, {
jobId: 'test-rerun-job',
createdAt: DateTime.now(),
})
const enqueued = hogQueue.queueInvocations.mock.calls[0]?.[0] as
| CyclotronJobInvocationHogFunction[]
| undefined
expect(enqueued?.map((i) => i.id)).toEqual(['inv-still-failed'])
expect(next.progress.queued).toBe(1)
expect(next.progress.skipped).toBe(1)
})
it('matches a run by the window of its first scheduled time, whatever day its last row lands on', async () => {
const run = (id: string, start: string, end: string) => [
{
invocation_id: id,
status: 'running' as const,
scheduledAt: new Date(start),
firstScheduledAt: new Date(start),
},
{
invocation_id: id,
status: 'failed' as const,
error: new Error('boom'),
scheduledAt: new Date(end),
firstScheduledAt: new Date(start),
},
]
await seedRows([
...run('finishes-after-window', '2026-02-01T23:00:00Z', '2026-02-02T01:00:00Z'),
...run('started-before-window', '2026-01-31T23:00:00Z', '2026-02-01T01:00:00Z'),
])
const state = buildState({
paging: 'run_start',
request: {
filter: { window_start: '2026-02-01T00:00:00Z', window_end: '2026-02-02T00:00:00Z' },
},
})
await paginator.processPage(team.id, state, { jobId: 'test-rerun-job', createdAt: DateTime.now() })
expect(enqueuedIds()).toEqual(['finishes-after-window'])
})
it.each([
{
paging: 'run_start' as const,
cursor: { scheduled_at: '2026-02-01 12:00:00.000000', invocation_id: 'later-start' },
expectedIds: ['spans-days'],
expectedCursor: { scheduled_at: '2026-02-01 10:00:00.000000', invocation_id: 'spans-days' },
},
{
// A job created before the switch carries no paging mode and keeps paging by last row.
paging: undefined,
cursor: { scheduled_at: '2026-02-02 09:00:00.000000', invocation_id: 'spans-days' },
expectedIds: ['later-start'],
expectedCursor: { scheduled_at: '2026-02-01 12:00:00.000000', invocation_id: 'later-start' },
},
])(
'resumes a $paging job from its cursor without skipping or repeating a run',
async ({ paging, cursor, expectedIds, expectedCursor }) => {
await seedRows([
{
invocation_id: 'spans-days',
status: 'running',
scheduledAt: new Date('2026-02-01T10:00:00Z'),
firstScheduledAt: new Date('2026-02-01T10:00:00Z'),
},
{
invocation_id: 'later-start',
status: 'failed',
error: new Error('boom'),
scheduledAt: new Date('2026-02-01T12:00:00Z'),
firstScheduledAt: new Date('2026-02-01T12:00:00Z'),
},
{
invocation_id: 'spans-days',
status: 'failed',
error: new Error('boom'),
scheduledAt: new Date('2026-02-02T09:00:00Z'),
firstScheduledAt: new Date('2026-02-01T10:00:00Z'),
},
])
const state = buildState({
paging,
request: {
filter: { window_start: '2026-02-01T00:00:00Z', window_end: '2026-02-03T00:00:00Z' },
},
progress: { queued: 0, skipped: 0, done: false, cursor },
})
const { state: next } = await paginator.processPage(team.id, state, {
jobId: 'test-rerun-job',
createdAt: DateTime.now(),
})
expect(enqueuedIds()).toEqual(expectedIds)
expect(next.progress.cursor).toEqual(expectedCursor)
}
)
it('honours max_count by capping queued+skipped at the user-provided limit', async () => {
await seedRows([
{ invocation_id: 'a', status: 'failed', error: new Error('5xx') },
{ invocation_id: 'b', status: 'failed', error: new Error('5xx') },
{ invocation_id: 'c', status: 'failed', error: new Error('5xx') },
])
const state = buildState({
request: {
filter: {
window_start: '2026-01-01T00:00:00Z',
window_end: '2027-01-01T00:00:00Z',
max_count: 2,
},
},
})
const { state: next } = await paginator.processPage(team.id, state, {
jobId: 'test-rerun-job',
createdAt: DateTime.now(),
})
expect(next.progress.queued).toBe(2)
expect(next.progress.done).toBe(true)
})
it('returns done=true with an empty page when no rows match the filter', async () => {
await seedRows([{ invocation_id: 'inv-ok', status: 'succeeded' }])
const state = buildState({
request: {
filter: { window_start: '2026-01-01T00:00:00Z', window_end: '2027-01-01T00:00:00Z' },
},
})
const { state: next } = await paginator.processPage(team.id, state, {
jobId: 'test-rerun-job',
createdAt: DateTime.now(),
})
expect(next.progress.queued).toBe(0)
expect(next.progress.done).toBe(true)
expect(hogQueue.queueInvocations).not.toHaveBeenCalled()
})
it('keeps a page successful when only the wrapper progress row fails to write', async () => {
// The page's invocations are committed before the wrapper row is
// written, so a failure there must not count toward the give-up
// streak — otherwise a lifecycle-write outage marks a rerun failed
// that actually ran, and holds the cursor back onto the same page.
await seedRows([{ invocation_id: 'inv-ok', status: 'succeeded' }])
jest.spyOn(paginatorLifecycleService, 'queueRerunWrapperRow').mockImplementation(() => {
throw new Error('lifecycle write boom')
})
const state = buildState({
request: {
filter: { window_start: '2026-01-01T00:00:00Z', window_end: '2027-01-01T00:00:00Z' },
},
})
const { state: next } = await paginator.processPage(team.id, state, {
jobId: 'test-rerun-job',
createdAt: DateTime.now(),
})
expect(next.progress.done).toBe(true)
expect(next.progress.last_error).toBeUndefined()
expect(next.progress.consecutive_errors).toBeUndefined()
})
})
describe('max_attempts filter (poison-pill autodrain path)', () => {
// Regression guard for the aggregate-inside-aggregate ClickHouse error: with
// max_attempts set, fetchPage adds `argMax(attempts, version) < {max_attempts}`
// to HAVING. If the SELECT alias is named `attempts` it shadows the raw column,
// so `attempts` inside the HAVING argMax resolves to the alias (itself an
// aggregate) → ClickHouse rejects every page. No other test sets max_attempts
// (the manual-rerun UI leaves it unset), so this whole clause was dormant until
// the autodrain became the first caller to set it unconditionally.
it('runs a valid query and honours the cap when max_attempts is set', async () => {
await seedRows([
{
invocation_id: 'pp-under-cap',
status: 'failed',
error: new Error('poison pill'),
errorKind: 'janitor_poison_pill',
rerunAttempts: 1,
},
{
invocation_id: 'pp-over-cap',
status: 'failed',
error: new Error('poison pill'),
errorKind: 'janitor_poison_pill',
rerunAttempts: 5,
},
])
// The exact filter the autodrain enqueues.
const state = buildState({
request: {
filter: {
window_start: '2026-01-01T00:00:00Z',
window_end: '2027-01-01T00:00:00Z',
status: ['failed'],
error_kind: ['janitor_poison_pill'],
max_attempts: 3,
},
},
})
const { state: next } = await paginator.processPage(team.id, state, {
jobId: 'test-rerun-job',
createdAt: DateTime.now(),
})
// The query executed — without the alias fix this is the CH
// aggregate-inside-aggregate error and every page fails here.
expect(next.progress.last_error).toBeUndefined()
// Under the cap is re-enqueued; over the cap is filtered by the HAVING clause.
const enqueued = hogQueue.queueInvocations.mock.calls[0]?.[0] as
| CyclotronJobInvocationHogFunction[]
| undefined
expect(enqueued?.map((i) => i.id)).toEqual(['pp-under-cap'])
expect(next.progress.queued).toBe(1)
expect(next.progress.done).toBe(true)
})
})
describe('rehydration', () => {
it('skips rows whose hog function has been deleted between original run and rerun', async () => {
const orphanedFn = await _insertHogFunction(hub.postgres, team.id, {
type: 'destination',
hog: 'print("orphan")',
bytecode: [],
})
// Seed under the orphaned function...
const prevFn = hogFunction
hogFunction = orphanedFn
await seedRows([{ invocation_id: 'inv-orphan', status: 'failed', error: new Error('5xx') }])
hogFunction = prevFn
// ...then ask the paginator to rerun it. The paginator's hog function
// manager (real) lazy-loads the orphaned function (still exists in PG)
// — we explicitly stub `getHogFunction` to return null to simulate the
// "deleted between original run and rerun" scenario.
const realGetHogFunction = hogFunctionManager.getHogFunction.bind(hogFunctionManager)
jest.spyOn(hogFunctionManager, 'getHogFunction').mockImplementation(async (id) => {
if (id === orphanedFn.id) {
return null
}
return realGetHogFunction(id)
})
const state: RerunJobState = {
function_kind: 'hog_function',
function_id: orphanedFn.id,
request: {
filter: {
window_start: '2026-01-01T00:00:00Z',
window_end: '2027-01-01T00:00:00Z',
invocation_ids: ['inv-orphan'],
},
},
progress: { queued: 0, skipped: 0, done: false },
}
const { state: next } = await paginator.processPage(team.id, state, {
jobId: 'test-rerun-job',
createdAt: DateTime.now(),
})
expect(next.progress.queued).toBe(0)
expect(next.progress.skipped).toBe(1)
expect(hogQueue.queueInvocations).not.toHaveBeenCalled()
})
})
describe('error handling', () => {
const buildBrokenPaginator = (): RerunPaginatorService => {
const brokenChClient = {
query: jest.fn().mockRejectedValue(new Error('clickhouse boom')),
} as unknown as ClickHouseClient
return new RerunPaginatorService(
brokenChClient,
hogFunctionManager,
hogFlowManager,
paginatorLifecycleService,
{ hog_function: hogQueue, hog_flow: hogflowQueue },
paginatorMonitoringService,
10000
)
}
const brokenState = () =>
buildState({
request: {
filter: {
window_start: '2026-01-01T00:00:00Z',
window_end: '2027-01-01T00:00:00Z',
invocation_ids: ['x'],
},
},
progress: { queued: 0, skipped: 0, done: false },
})
it('captures a page error on progress.last_error, bumps the streak, and keeps the job running', async () => {
const { state: next } = await buildBrokenPaginator().processPage(team.id, brokenState(), {
jobId: 'test-rerun-job',
createdAt: DateTime.now(),
})
expect(next.progress.done).toBe(false)
expect(next.progress.last_error).toContain('clickhouse boom')
// Streak tracked so a persistently-failing page eventually gives up
// instead of rescheduling forever (which overflows transition_count).
expect(next.progress.consecutive_errors).toBe(1)
expect(hogQueue.queueInvocations).not.toHaveBeenCalled()
})
it('gives up (throws) once a page has errored RERUN_MAX_CONSECUTIVE_PAGE_ERRORS times', async () => {
// One more errored page past the cap must throw so the worker fails
// the wrapper job terminally — the guard against an unbounded retry
// loop that would drive the SMALLINT transition_count to overflow.
const state = brokenState()
state.progress.consecutive_errors = RERUN_MAX_CONSECUTIVE_PAGE_ERRORS - 1
await expect(
buildBrokenPaginator().processPage(team.id, state, {
jobId: 'test-rerun-job',
createdAt: DateTime.now(),
})
).rejects.toThrow('clickhouse boom')
})
})
describe('done logic (real CH)', () => {
it('marks done when fewer rows came back than the page size (no further pages)', async () => {
await seedRows([{ invocation_id: 'one', status: 'failed', error: new Error('5xx') }])
const state = buildState({
request: {
filter: { window_start: '2026-01-01T00:00:00Z', window_end: '2027-01-01T00:00:00Z' },
},
})
const { state: next } = await paginator.processPage(team.id, state, {
jobId: 'test-rerun-job',
createdAt: DateTime.now(),
})
// 1 row < RERUN_PAGE_SIZE → done.
expect(next.progress.done).toBe(true)
// Sanity: RERUN_PAGE_SIZE constant didn't regress to a tiny value.
expect(RERUN_PAGE_SIZE).toBeGreaterThan(10)
})
it('carries first_scheduled_at from the stored row onto the rehydrated invocation state', async () => {
// The producer writes `first_scheduled_at` verbatim on every retry's
// lifecycle row so ReplacingMergeTree doesn't collapse it away. The
// paginator's contract here: read it from the stored row, stamp it
// onto the rehydrated state so the next retry's producer sees the
// same original time. Without this, every kafka retry would
// overwrite first_scheduled_at with the latest scheduled_at.
const id = 'inv-first-' + new UUIDT().toString()
const originalScheduledAt = new Date('2026-05-01T09:00:00Z')
await seedRows([
{
invocation_id: id,
status: 'failed',
error: new Error('5xx'),
scheduledAt: originalScheduledAt,
},
])
const state = buildState({
request: {
filter: {
window_start: '2026-01-01T00:00:00Z',
window_end: '2027-01-01T00:00:00Z',
invocation_ids: [id],
},
},
})
await paginator.processPage(team.id, state, {
jobId: 'test-rerun-job',
createdAt: DateTime.now(),
})
const enqueued = hogQueue.queueInvocations.mock.calls[0][0] as CyclotronJobInvocationHogFunction[]
expect(enqueued).toHaveLength(1)
// CH returns DateTime64 as 'YYYY-MM-DD HH:MM:SS.ffffff' — just check
// the date portion matches the original scheduled time. The exact
// serialization differs between paths but the day/time is preserved.
expect(enqueued[0].state.firstScheduledAt).toMatch(/^2026-05-01[T ]09:00:00/)
})
it('reads the latest row per invocation via argMax(_, version) collapse', async () => {
// Same invocation_id, multiple lifecycle rows simulating: original
// failed → user clicked rerun → rerun running → rerun succeeded.
// The paginator should see the LATEST `succeeded` state, not the
// earlier `failed`. (Filter mode defaults to status=failed, so an
// invocation that's now succeeded should NOT be picked up.)
const id = 'inv-collapse-' + new UUIDT().toString()
await seedRows([{ invocation_id: id, status: 'failed', error: new Error('5xx') }])
// Small sleep to ensure version monotonicity is observable.
await new Promise((r) => setTimeout(r, 5))
await seedRows([{ invocation_id: id, status: 'succeeded' }])
const state = buildState({
request: {
filter: { window_start: '2026-01-01T00:00:00Z', window_end: '2027-01-01T00:00:00Z' },
},
})
const { state: next } = await paginator.processPage(team.id, state, {
jobId: 'test-rerun-job',
createdAt: DateTime.now(),
})
expect(next.progress.queued).toBe(0)
})
})
// Silence unused-import noise for HogFunctionInvocationGlobals which is
// referenced indirectly via the rehydration assertion.
void (undefined as unknown as HogFunctionInvocationGlobals)
})