Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

42.3 KB
Raw Blame
import { MockKafkaProducerWrapper } from '~/tests/helpers/mocks/producer.mock'

import { ClickHouseClient } from '@clickhouse/client'
import { DateTime } from 'luxon'

import { KAFKA_HOG_INVOCATION_RESULTS } from '~/common/config/kafka-topics'
import { KafkaProducerWrapper } from '~/common/kafka/producer'
import { closeHub, createHub } from '~/common/utils/db/hub'
import { UUIDT } from '~/common/utils/utils'
import { createCdpConsumerDeps } from '~/tests/helpers/cdp'
import { Clickhouse } from '~/tests/helpers/clickhouse'
import { waitForExpect } from '~/tests/helpers/expectations'
import { waitForHogInvocationResultsMvReady } from '~/tests/helpers/hog-invocation-results'
import { TEST_KAFKA_TOPICS, ensureKafkaTopics } from '~/tests/helpers/kafka'
import { createTestTeamFixture } from '~/tests/helpers/sql'

import { Hub, Team } from '../../types'
import { insertHogFunction as _insertHogFunction, createHogExecutionGlobals } from '../_tests/fixtures'
import { createCdpOutputsRegistry } from '../outputs/registry'
import { HogFlowManagerService } from '../services/hogflows/hogflow-manager.service'
import { CyclotronJobQueuePostgresV2 } from '../services/job-queue/job-queue-postgres-v2'
import { JobQueue } from '../services/job-queue/job-queue.interface'
import { HogFunctionManagerService } from '../services/managers/hog-function-manager.service'
import { HogFunctionMonitoringService } from '../services/monitoring/hog-function-monitoring.service'
import { HogInvocationResultsService } from '../services/monitoring/hog-invocation-results.service'
import { CyclotronJobInvocationHogFunction, HogFunctionInvocationGlobals, HogFunctionType } from '../types'
import { RERUN_MAX_CONSECUTIVE_PAGE_ERRORS, RERUN_PAGE_SIZE, RerunJobState } from './rerun-job.types'
import { RerunPaginatorService } from './rerun-paginator.service'

const ActualKafkaProducerWrapper = jest.requireActual('~/common/kafka/producer').KafkaProducerWrapper

/**
 * Integration test for RerunPaginatorService.
 *
 * Instead of mocking the ClickHouse client, this test seeds real lifecycle rows
 * through `HogInvocationResultsService.queueLifecycleRow` → Kafka → ClickHouse
 * Kafka MV → `hog_invocation_results`, then runs the paginator against the real
 * data. Only the side effects the paginator emits (the running lifecycle row +
 * the cyclotron re-enqueue) are mocked, so we can assert what got produced.
 *
 * This proves the whole pipeline end-to-end: the queueLifecycleRow shape that
 * the worker uses, the Kafka MV's JSON parsing, and the paginator's argMax
 * collapse query all line up.
 */
describe('RerunPaginatorService integration', () => {
    jest.setTimeout(180_000)

    let hub: Hub
    let kafkaProducer: KafkaProducerWrapper
    let team: Team
    let clickhouse: Clickhouse
    let hogFunction: HogFunctionType
    let seedingService: HogInvocationResultsService
    let chClient: ClickHouseClient
    let paginator: RerunPaginatorService
    // hog functions re-enqueue to the kafka queue, hog flows to postgres-v2.
    let hogQueue: jest.Mocked<JobQueue>
    let hogflowQueue: jest.Mocked<CyclotronJobQueuePostgresV2>
    let paginatorLifecycleService: jest.Mocked<HogInvocationResultsService>
    let paginatorMonitoringService: jest.Mocked<HogFunctionMonitoringService>
    let hogFunctionManager: HogFunctionManagerService
    let hogFlowManager: jest.Mocked<HogFlowManagerService>

    beforeAll(() => {
        clickhouse = Clickhouse.create()
        chClient = Clickhouse.createClient()
    })

    afterAll(async () => {
        clickhouse?.close()
        await chClient?.close()
    })

    let seededCount = 0
    /**
     * Seed `n` lifecycle rows for the given function. Each entry becomes a real
     * Kafka produce + MV ingestion, so by the time this returns ClickHouse has
     * the rows ready for the paginator to read.
     */
    const seedRows = async (
        rows: Array<{
            invocation_id: string
            status: 'running' | 'succeeded' | 'failed'
            error?: Error
            // Stable error_kind stamped on the row (e.g. 'janitor_poison_pill').
            // When omitted it is derived from the error message, as before.
            errorKind?: string
            // Prior rerun count → the row's `attempts` column, for exercising the
            // max_attempts filter. Defaults to 0.
            rerunAttempts?: number
            scheduledAt?: Date
            firstScheduledAt?: Date
        }>
    ): Promise<void> => {
        for (const r of rows) {
            const globals = createHogExecutionGlobals({
                project: { id: team.id } as any,
                event: {
                    uuid: `evt-${r.invocation_id}`,
                    event: '$pageview',
                    properties: {},
                    timestamp: '2026-05-10T09:00:00Z',
                } as any,
            })
            const invocation: CyclotronJobInvocationHogFunction = {
                id: r.invocation_id,
                state: {
                    globals: globals as any,
                    timings: [],
                    attempts: 0,
                    rerunAttempts: r.rerunAttempts,
                    firstScheduledAt: r.firstScheduledAt?.toISOString().replace('T', ' ').replace('Z', '000'),
                },
                teamId: team.id,
                functionId: hogFunction.id,
                hogFunction,
                queue: 'hog',
                queuePriority: 0,
                queueScheduledAt: r.scheduledAt ? ({ toJSDate: () => r.scheduledAt } as any) : undefined,
            }
            seedingService.queueLifecycleRow(invocation, r.status, { error: r.error, errorKind: r.errorKind })
        }
        await seedingService.flush()
        await kafkaProducer.flush()

        // Track cumulative seeded rows so calling seedRows twice in the same
        // test waits for *all* rows (rather than trivially passing on the
        // previous seed's count).
        seededCount += rows.length
        const expected = seededCount

        // Seed visibility rides the shared Kafka -> ClickHouse pipe, whose consumer may still be
        // chewing a backlog from whichever suite the shard ran just before this one (run order
        // shifts whenever sibling files change size). The deadline is sized for that worst case;
        // waitForExpect polls, so a healthy pipe still completes in seconds.
        await waitForExpect(async () => {
            const got = await clickhouse.query<{ c: number }>(
                `SELECT count() AS c FROM hog_invocation_results
                 WHERE team_id = ${team.id}
                   AND function_id = '${hogFunction.id}'`
            )
            expect(Number(got[0]?.c ?? 0)).toBeGreaterThanOrEqual(expected)
        }, 90_000)
    }

    // Produce a raw lifecycle row with a chosen (here: undecodable) invocation_globals,
    // bypassing the seeding service so we can exercise the paginator's per-row
    // rehydration-failure path — a poison-pill record whose stored globals can't be
    // decoded must be skipped, not abort the whole recovery batch.
    const seedRawRow = async (invocationId: string, invocationGlobals: string): Promise<void> => {
        await kafkaProducer.queueMessages({
            topic: KAFKA_HOG_INVOCATION_RESULTS,
            messages: [
                {
                    key: invocationId,
                    value: JSON.stringify({
                        team_id: team.id,
                        function_kind: 'hog_function',
                        function_id: hogFunction.id,
                        invocation_id: invocationId,
                        parent_run_id: '',
                        status: 'failed',
                        attempts: 0,
                        is_retry: 0,
                        scheduled_at: DateTime.utc().toFormat("yyyy-MM-dd HH:mm:ss.SSS'000'"),
                        started_at: null,
                        finished_at: null,
                        duration_ms: null,
                        error_kind: 'janitor_poison_pill',
                        error_message: 'poison pill',
                        event_uuid: '',
                        distinct_id: '',
                        person_id: '',
                        invocation_globals: invocationGlobals,
                        version: String(BigInt(Date.now()) * 1000n),
                        is_deleted: 0,
                    }),
                },
            ],
        })
        await kafkaProducer.flush()
        seededCount += 1
        await waitForExpect(async () => {
            const got = await clickhouse.query<{ c: number }>(
                `SELECT count() AS c FROM hog_invocation_results
                 WHERE team_id = ${team.id} AND invocation_id = '${invocationId}'`
            )
            expect(Number(got[0]?.c ?? 0)).toBeGreaterThanOrEqual(1)
        }, 90_000)
    }

    beforeAll(async () => {
        MockKafkaProducerWrapper.create = jest.fn((...args: any[]) => ActualKafkaProducerWrapper.create(...args))
        // Ensure all topics exist (idempotently, without deleting) so the ClickHouse Kafka engine
        // consumers keep their connections. Includes KAFKA_HOG_INVOCATION_RESULTS, which this test's
        // MV needs but the shared set does not cover.
        await ensureKafkaTopics([...TEST_KAFKA_TOPICS, KAFKA_HOG_INVOCATION_RESULTS])
        await waitForHogInvocationResultsMvReady(clickhouse)
    })

    beforeEach(async () => {
        seededCount = 0

        hub = await createHub()
        kafkaProducer = await ActualKafkaProducerWrapper.create(hub.KAFKA_CLIENT_RACK)
        team = (await createTestTeamFixture(hub.postgres)).team

        // Real seeding path: outputs → kafka → MV → CH.
        const deps = createCdpConsumerDeps(hub, kafkaProducer)
        const outputs = createCdpOutputsRegistry().build(deps.cdpProducerRegistry, {
            ...hub,
            HOG_INVOCATION_RESULTS_TOPIC: KAFKA_HOG_INVOCATION_RESULTS,
        } as any)
        seedingService = new HogInvocationResultsService(outputs, { HOG_INVOCATION_RESULTS_ENABLED: true })

        hogFunction = await _insertHogFunction(hub.postgres, team.id, {
            type: 'destination',
            hog: 'print("hi")',
            bytecode: [],
            inputs_schema: [],
            inputs: {},
        })

        // Real hog function manager so rehydrate can resolve the function from postgres.
        hogFunctionManager = new HogFunctionManagerService(hub.postgres, hub.pubSub, hub.encryptedFields)
        hogFunctionManager['onHogFunctionsReloaded'](team.id, [hogFunction.id])

        // Hog flow manager stays mocked — this suite focuses on the hog-function
        // path and the CH query/rehydrate logic. Hog flow rehydration is
        // exercised structurally (rest of the same code path) but not end-to-end here.
        hogFlowManager = {
            getHogFlow: jest.fn().mockResolvedValue(null),
        } as unknown as jest.Mocked<HogFlowManagerService>

        hogQueue = {
            queueInvocations: jest.fn().mockResolvedValue(undefined),
        } as unknown as jest.Mocked<JobQueue>
        hogflowQueue = {
            queueInvocations: jest.fn().mockResolvedValue(undefined),
        } as unknown as jest.Mocked<CyclotronJobQueuePostgresV2>

        // The paginator's lifecycle service is mocked so we can assert what it
        // got asked to write (the 'running' row for each rerun). The actual
        // CH state for that running row is a flaky-on-Redpanda concern handled
        // in the e2e test — here we only care that the paginator queues it.
        paginatorLifecycleService = {
            queueLifecycleRow: jest.fn(),
            queueRerunWrapperRow: jest.fn(),
            flush: jest.fn().mockResolvedValue(undefined),
            dropQueuedRowsFor: jest.fn(),
        } as unknown as jest.Mocked<HogInvocationResultsService>

        paginatorMonitoringService = {
            queueLogs: jest.fn(),
            flush: jest.fn().mockResolvedValue(undefined),
        } as unknown as jest.Mocked<HogFunctionMonitoringService>

        paginator = new RerunPaginatorService(
            chClient,
            hogFunctionManager,
            hogFlowManager,
            paginatorLifecycleService,
            { hog_function: hogQueue, hog_flow: hogflowQueue },
            paginatorMonitoringService,
            10000
        )
    })

    afterEach(async () => {
        await kafkaProducer?.disconnect().catch(() => undefined)
        await closeHub(hub)
    })

    const enqueuedIds = (): string[] =>
        hogQueue.queueInvocations.mock.calls.flatMap(([invocations]) => invocations.map((i) => i.id))

    const buildState = (overrides: Partial<RerunJobState> = {}): RerunJobState => ({
        function_kind: 'hog_function',
        function_id: hogFunction.id,
        // Window is required by the unified rerun schema. Default to a wide
        // year-long window so the seeded test data falls inside.
        request: { filter: { window_start: '2026-01-01T00:00:00Z', window_end: '2027-01-01T00:00:00Z' } },
        progress: { queued: 0, skipped: 0, done: false },
        ...overrides,
    })

    describe('by-ids mode', () => {
        it('rehydrates and re-enqueues only the requested invocations, ignoring others', async () => {
            await seedRows([
                { invocation_id: 'inv-a', status: 'failed', error: new Error('http_5xx error') },
                { invocation_id: 'inv-b', status: 'succeeded' },
                { invocation_id: 'inv-c', status: 'failed', error: new Error('timeout') },
            ])

            const state = buildState({
                request: {
                    filter: {
                        window_start: '2026-01-01T00:00:00Z',
                        window_end: '2027-01-01T00:00:00Z',
                        invocation_ids: ['inv-a', 'inv-c'],
                    },
                },
            })

            const { state: next } = await paginator.processPage(team.id, state, {
                jobId: 'test-rerun-job',
                createdAt: DateTime.now(),
            })

            expect(hogQueue.queueInvocations).toHaveBeenCalledTimes(1)
            const enqueued = hogQueue.queueInvocations.mock.calls[0][0] as CyclotronJobInvocationHogFunction[]
            const enqueuedIds = enqueued.map((i) => i.id).sort()
            expect(enqueuedIds).toEqual(['inv-a', 'inv-c'])

            // Each rerun invocation gets a 'running' lifecycle row queued.
            // is_retry / attempts are derived inside queueLifecycleRow from
            // state.rerunAttempts (set by the rehydrator), not passed as an opt.
            const runningCalls = paginatorLifecycleService.queueLifecycleRow.mock.calls.filter(
                (c) => c[1] === 'running'
            )
            expect(runningCalls).toHaveLength(2)
            // Each queued invocation's state should carry rerunAttempts=1.
            for (const [invocation] of runningCalls) {
                expect((invocation as CyclotronJobInvocationHogFunction).state.rerunAttempts).toBe(1)
            }

            expect(next.progress.queued).toBe(2)
            expect(next.progress.skipped).toBe(0)
            expect(next.progress.done).toBe(true)
        })

        it('skips invocation ids that have no matching row in clickhouse', async () => {
            await seedRows([{ invocation_id: 'inv-real', status: 'failed', error: new Error('5xx') }])

            const state = buildState({
                request: {
                    filter: {
                        window_start: '2026-01-01T00:00:00Z',
                        window_end: '2027-01-01T00:00:00Z',
                        invocation_ids: ['inv-real', 'inv-missing'],
                    },
                },
            })

            const { state: next } = await paginator.processPage(team.id, state, {
                jobId: 'test-rerun-job',
                createdAt: DateTime.now(),
            })

            const enqueued = hogQueue.queueInvocations.mock.calls[0][0] as CyclotronJobInvocationHogFunction[]
            expect(enqueued.map((i) => i.id)).toEqual(['inv-real'])
            // by-ids mode marks done once remaining_ids is drained, regardless of
            // whether every id had a matching CH row.
            expect(next.progress.done).toBe(true)
        })

        it('re-enqueues bare globals — inputs are not stored or pre-resolved by the paginator', async () => {
            await seedRows([{ invocation_id: 'inv-rebuild', status: 'failed', error: new Error('boom') }])

            const state = buildState({
                request: {
                    filter: {
                        window_start: '2026-01-01T00:00:00Z',
                        window_end: '2027-01-01T00:00:00Z',
                        invocation_ids: ['inv-rebuild'],
                    },
                },
            })

            await paginator.processPage(team.id, state, { jobId: 'test-rerun-job', createdAt: DateTime.now() })

            // The re-enqueued invocation carries no resolved `inputs` — the
            // executor rebuilds them from the current hog function config at
            // run time, so the rerun never trusts a stored snapshot.
            const enqueued = hogQueue.queueInvocations.mock.calls[0][0] as CyclotronJobInvocationHogFunction[]
            expect(enqueued).toHaveLength(1)
            expect(enqueued[0].state.globals).not.toHaveProperty('inputs')
        })

        it('skips a record that fails to rehydrate and still re-enqueues the rest of the batch', async () => {
            await seedRows([{ invocation_id: 'inv-good', status: 'failed', error: new Error('5xx') }])
            await seedRawRow('inv-bad', 'not-a-decodable-globals-blob')

            const state = buildState({
                request: {
                    filter: {
                        window_start: '2026-01-01T00:00:00Z',
                        window_end: '2027-01-01T00:00:00Z',
                        invocation_ids: ['inv-good', 'inv-bad'],
                    },
                },
            })

            const { state: next } = await paginator.processPage(team.id, state, {
                jobId: 'test-rerun-job',
                createdAt: DateTime.now(),
            })

            // The undecodable record is skipped, not fatal — the good one still re-enqueues.
            const enqueued = hogQueue.queueInvocations.mock.calls.flatMap(
                (c) => c[0] as CyclotronJobInvocationHogFunction[]
            )
            expect(enqueued.map((i) => i.id)).toEqual(['inv-good'])
            expect(next.progress.queued).toBe(1)
            expect(next.progress.skipped).toBeGreaterThanOrEqual(1)
        })
    })

    describe('by-filter mode', () => {
        it('defaults to status=failed and only matches failed rows in the window', async () => {
            await seedRows([
                { invocation_id: 'ok-1', status: 'succeeded' },
                { invocation_id: 'fail-1', status: 'failed', error: new Error('500 err') },
                { invocation_id: 'fail-2', status: 'failed', error: new Error('timeout') },
            ])

            const state = buildState({
                request: {
                    filter: { window_start: '2026-01-01T00:00:00Z', window_end: '2027-01-01T00:00:00Z' },
                },
            })

            const { state: next } = await paginator.processPage(team.id, state, {
                jobId: 'test-rerun-job',
                createdAt: DateTime.now(),
            })

            const enqueued = hogQueue.queueInvocations.mock.calls[0][0] as CyclotronJobInvocationHogFunction[]
            const ids = enqueued.map((i) => i.id).sort()
            expect(ids).toEqual(['fail-1', 'fail-2'])
            expect(next.progress.queued).toBe(2)
        })

        it('honours error_kind filter (only http_5xx rows are rerun)', async () => {
            await seedRows([
                { invocation_id: 'inv-500', status: 'failed', error: new Error('500 server error') },
                { invocation_id: 'inv-timeout', status: 'failed', error: new Error('Request timed out') },
            ])

            const state = buildState({
                request: {
                    filter: {
                        window_start: '2026-01-01T00:00:00Z',
                        window_end: '2027-01-01T00:00:00Z',
                        error_kind: ['http_5xx'],
                    },
                },
            })

            const { state: next } = await paginator.processPage(team.id, state, {
                jobId: 'test-rerun-job',
                createdAt: DateTime.now(),
            })
            const enqueued = hogQueue.queueInvocations.mock.calls[0]?.[0] as
                | CyclotronJobInvocationHogFunction[]
                | undefined
            expect(enqueued?.map((i) => i.id)).toEqual(['inv-500'])
            expect(next.progress.queued).toBe(1)
        })

        it('honours error_message_contains filter, case-insensitively', async () => {
            await seedRows([
                {
                    invocation_id: 'inv-sender',
                    status: 'failed',
                    error: new Error(
                        'The custom sender address "[email protected]" is not a valid email address.'
                    ),
                },
                { invocation_id: 'inv-other', status: 'failed', error: new Error('Some unrelated hog error') },
            ])

            const state = buildState({
                request: {
                    filter: {
                        window_start: '2026-01-01T00:00:00Z',
                        window_end: '2027-01-01T00:00:00Z',
                        // Deliberately lower-cased vs the stored message: a needle copied
                        // from logs must match regardless of capitalization.
                        error_message_contains: 'the custom sender address',
                    },
                },
            })

            const { state: next } = await paginator.processPage(team.id, state, {
                jobId: 'test-rerun-job',
                createdAt: DateTime.now(),
            })
            const enqueued = hogQueue.queueInvocations.mock.calls[0]?.[0] as
                | CyclotronJobInvocationHogFunction[]
                | undefined
            expect(enqueued?.map((i) => i.id)).toEqual(['inv-sender'])
            expect(next.progress.queued).toBe(1)
        })

        it('skips invocations whose latest status outside the window no longer matches', async () => {
            // Original failures inside a historical window...
            await seedRows([
                {
                    invocation_id: 'inv-replayed',
                    status: 'failed',
                    error: new Error('boom'),
                    scheduledAt: new Date('2026-02-01T10:00:00Z'),
                },
                {
                    invocation_id: 'inv-still-failed',
                    status: 'failed',
                    error: new Error('boom'),
                    scheduledAt: new Date('2026-02-01T11:00:00Z'),
                },
            ])
            // ...then a successful replay of one of them lands its lifecycle row in the
            // partition of its own scheduled_at, outside the window. The windowed page
            // query still sees the old failed row, so without the cross-partition status
            // check a second rerun would re-enqueue it and re-fire its side effects.
            await seedRows([
                { invocation_id: 'inv-replayed', status: 'succeeded', scheduledAt: new Date('2026-06-01T10:00:00Z') },
            ])

            const state = buildState({
                request: {
                    filter: { window_start: '2026-02-01T00:00:00Z', window_end: '2026-02-02T00:00:00Z' },
                },
            })

            const { state: next } = await paginator.processPage(team.id, state, {
                jobId: 'test-rerun-job',
                createdAt: DateTime.now(),
            })
            const enqueued = hogQueue.queueInvocations.mock.calls[0]?.[0] as
                | CyclotronJobInvocationHogFunction[]
                | undefined
            expect(enqueued?.map((i) => i.id)).toEqual(['inv-still-failed'])
            expect(next.progress.queued).toBe(1)
            expect(next.progress.skipped).toBe(1)
        })

        it('matches a run by the window of its first scheduled time, whatever day its last row lands on', async () => {
            const run = (id: string, start: string, end: string) => [
                {
                    invocation_id: id,
                    status: 'running' as const,
                    scheduledAt: new Date(start),
                    firstScheduledAt: new Date(start),
                },
                {
                    invocation_id: id,
                    status: 'failed' as const,
                    error: new Error('boom'),
                    scheduledAt: new Date(end),
                    firstScheduledAt: new Date(start),
                },
            ]
            await seedRows([
                ...run('finishes-after-window', '2026-02-01T23:00:00Z', '2026-02-02T01:00:00Z'),
                ...run('started-before-window', '2026-01-31T23:00:00Z', '2026-02-01T01:00:00Z'),
            ])

            const state = buildState({
                paging: 'run_start',
                request: {
                    filter: { window_start: '2026-02-01T00:00:00Z', window_end: '2026-02-02T00:00:00Z' },
                },
            })

            await paginator.processPage(team.id, state, { jobId: 'test-rerun-job', createdAt: DateTime.now() })
            expect(enqueuedIds()).toEqual(['finishes-after-window'])
        })

        it.each([
            {
                paging: 'run_start' as const,
                cursor: { scheduled_at: '2026-02-01 12:00:00.000000', invocation_id: 'later-start' },
                expectedIds: ['spans-days'],
                expectedCursor: { scheduled_at: '2026-02-01 10:00:00.000000', invocation_id: 'spans-days' },
            },
            {
                // A job created before the switch carries no paging mode and keeps paging by last row.
                paging: undefined,
                cursor: { scheduled_at: '2026-02-02 09:00:00.000000', invocation_id: 'spans-days' },
                expectedIds: ['later-start'],
                expectedCursor: { scheduled_at: '2026-02-01 12:00:00.000000', invocation_id: 'later-start' },
            },
        ])(
            'resumes a $paging job from its cursor without skipping or repeating a run',
            async ({ paging, cursor, expectedIds, expectedCursor }) => {
                await seedRows([
                    {
                        invocation_id: 'spans-days',
                        status: 'running',
                        scheduledAt: new Date('2026-02-01T10:00:00Z'),
                        firstScheduledAt: new Date('2026-02-01T10:00:00Z'),
                    },
                    {
                        invocation_id: 'later-start',
                        status: 'failed',
                        error: new Error('boom'),
                        scheduledAt: new Date('2026-02-01T12:00:00Z'),
                        firstScheduledAt: new Date('2026-02-01T12:00:00Z'),
                    },
                    {
                        invocation_id: 'spans-days',
                        status: 'failed',
                        error: new Error('boom'),
                        scheduledAt: new Date('2026-02-02T09:00:00Z'),
                        firstScheduledAt: new Date('2026-02-01T10:00:00Z'),
                    },
                ])

                const state = buildState({
                    paging,
                    request: {
                        filter: { window_start: '2026-02-01T00:00:00Z', window_end: '2026-02-03T00:00:00Z' },
                    },
                    progress: { queued: 0, skipped: 0, done: false, cursor },
                })

                const { state: next } = await paginator.processPage(team.id, state, {
                    jobId: 'test-rerun-job',
                    createdAt: DateTime.now(),
                })
                expect(enqueuedIds()).toEqual(expectedIds)
                expect(next.progress.cursor).toEqual(expectedCursor)
            }
        )

        it('honours max_count by capping queued+skipped at the user-provided limit', async () => {
            await seedRows([
                { invocation_id: 'a', status: 'failed', error: new Error('5xx') },
                { invocation_id: 'b', status: 'failed', error: new Error('5xx') },
                { invocation_id: 'c', status: 'failed', error: new Error('5xx') },
            ])

            const state = buildState({
                request: {
                    filter: {
                        window_start: '2026-01-01T00:00:00Z',
                        window_end: '2027-01-01T00:00:00Z',
                        max_count: 2,
                    },
                },
            })

            const { state: next } = await paginator.processPage(team.id, state, {
                jobId: 'test-rerun-job',
                createdAt: DateTime.now(),
            })
            expect(next.progress.queued).toBe(2)
            expect(next.progress.done).toBe(true)
        })

        it('returns done=true with an empty page when no rows match the filter', async () => {
            await seedRows([{ invocation_id: 'inv-ok', status: 'succeeded' }])

            const state = buildState({
                request: {
                    filter: { window_start: '2026-01-01T00:00:00Z', window_end: '2027-01-01T00:00:00Z' },
                },
            })

            const { state: next } = await paginator.processPage(team.id, state, {
                jobId: 'test-rerun-job',
                createdAt: DateTime.now(),
            })
            expect(next.progress.queued).toBe(0)
            expect(next.progress.done).toBe(true)
            expect(hogQueue.queueInvocations).not.toHaveBeenCalled()
        })

        it('keeps a page successful when only the wrapper progress row fails to write', async () => {
            // The page's invocations are committed before the wrapper row is
            // written, so a failure there must not count toward the give-up
            // streak — otherwise a lifecycle-write outage marks a rerun failed
            // that actually ran, and holds the cursor back onto the same page.
            await seedRows([{ invocation_id: 'inv-ok', status: 'succeeded' }])
            jest.spyOn(paginatorLifecycleService, 'queueRerunWrapperRow').mockImplementation(() => {
                throw new Error('lifecycle write boom')
            })

            const state = buildState({
                request: {
                    filter: { window_start: '2026-01-01T00:00:00Z', window_end: '2027-01-01T00:00:00Z' },
                },
            })

            const { state: next } = await paginator.processPage(team.id, state, {
                jobId: 'test-rerun-job',
                createdAt: DateTime.now(),
            })

            expect(next.progress.done).toBe(true)
            expect(next.progress.last_error).toBeUndefined()
            expect(next.progress.consecutive_errors).toBeUndefined()
        })
    })

    describe('max_attempts filter (poison-pill autodrain path)', () => {
        // Regression guard for the aggregate-inside-aggregate ClickHouse error: with
        // max_attempts set, fetchPage adds `argMax(attempts, version) < {max_attempts}`
        // to HAVING. If the SELECT alias is named `attempts` it shadows the raw column,
        // so `attempts` inside the HAVING argMax resolves to the alias (itself an
        // aggregate) → ClickHouse rejects every page. No other test sets max_attempts
        // (the manual-rerun UI leaves it unset), so this whole clause was dormant until
        // the autodrain became the first caller to set it unconditionally.
        it('runs a valid query and honours the cap when max_attempts is set', async () => {
            await seedRows([
                {
                    invocation_id: 'pp-under-cap',
                    status: 'failed',
                    error: new Error('poison pill'),
                    errorKind: 'janitor_poison_pill',
                    rerunAttempts: 1,
                },
                {
                    invocation_id: 'pp-over-cap',
                    status: 'failed',
                    error: new Error('poison pill'),
                    errorKind: 'janitor_poison_pill',
                    rerunAttempts: 5,
                },
            ])

            // The exact filter the autodrain enqueues.
            const state = buildState({
                request: {
                    filter: {
                        window_start: '2026-01-01T00:00:00Z',
                        window_end: '2027-01-01T00:00:00Z',
                        status: ['failed'],
                        error_kind: ['janitor_poison_pill'],
                        max_attempts: 3,
                    },
                },
            })

            const { state: next } = await paginator.processPage(team.id, state, {
                jobId: 'test-rerun-job',
                createdAt: DateTime.now(),
            })

            // The query executed — without the alias fix this is the CH
            // aggregate-inside-aggregate error and every page fails here.
            expect(next.progress.last_error).toBeUndefined()

            // Under the cap is re-enqueued; over the cap is filtered by the HAVING clause.
            const enqueued = hogQueue.queueInvocations.mock.calls[0]?.[0] as
                | CyclotronJobInvocationHogFunction[]
                | undefined
            expect(enqueued?.map((i) => i.id)).toEqual(['pp-under-cap'])
            expect(next.progress.queued).toBe(1)
            expect(next.progress.done).toBe(true)
        })
    })

    describe('rehydration', () => {
        it('skips rows whose hog function has been deleted between original run and rerun', async () => {
            const orphanedFn = await _insertHogFunction(hub.postgres, team.id, {
                type: 'destination',
                hog: 'print("orphan")',
                bytecode: [],
            })
            // Seed under the orphaned function...
            const prevFn = hogFunction
            hogFunction = orphanedFn
            await seedRows([{ invocation_id: 'inv-orphan', status: 'failed', error: new Error('5xx') }])
            hogFunction = prevFn

            // ...then ask the paginator to rerun it. The paginator's hog function
            // manager (real) lazy-loads the orphaned function (still exists in PG)
            // — we explicitly stub `getHogFunction` to return null to simulate the
            // "deleted between original run and rerun" scenario.
            const realGetHogFunction = hogFunctionManager.getHogFunction.bind(hogFunctionManager)
            jest.spyOn(hogFunctionManager, 'getHogFunction').mockImplementation(async (id) => {
                if (id === orphanedFn.id) {
                    return null
                }
                return realGetHogFunction(id)
            })

            const state: RerunJobState = {
                function_kind: 'hog_function',
                function_id: orphanedFn.id,
                request: {
                    filter: {
                        window_start: '2026-01-01T00:00:00Z',
                        window_end: '2027-01-01T00:00:00Z',
                        invocation_ids: ['inv-orphan'],
                    },
                },
                progress: { queued: 0, skipped: 0, done: false },
            }

            const { state: next } = await paginator.processPage(team.id, state, {
                jobId: 'test-rerun-job',
                createdAt: DateTime.now(),
            })
            expect(next.progress.queued).toBe(0)
            expect(next.progress.skipped).toBe(1)
            expect(hogQueue.queueInvocations).not.toHaveBeenCalled()
        })
    })

    describe('error handling', () => {
        const buildBrokenPaginator = (): RerunPaginatorService => {
            const brokenChClient = {
                query: jest.fn().mockRejectedValue(new Error('clickhouse boom')),
            } as unknown as ClickHouseClient
            return new RerunPaginatorService(
                brokenChClient,
                hogFunctionManager,
                hogFlowManager,
                paginatorLifecycleService,
                { hog_function: hogQueue, hog_flow: hogflowQueue },
                paginatorMonitoringService,
                10000
            )
        }

        const brokenState = () =>
            buildState({
                request: {
                    filter: {
                        window_start: '2026-01-01T00:00:00Z',
                        window_end: '2027-01-01T00:00:00Z',
                        invocation_ids: ['x'],
                    },
                },
                progress: { queued: 0, skipped: 0, done: false },
            })

        it('captures a page error on progress.last_error, bumps the streak, and keeps the job running', async () => {
            const { state: next } = await buildBrokenPaginator().processPage(team.id, brokenState(), {
                jobId: 'test-rerun-job',
                createdAt: DateTime.now(),
            })
            expect(next.progress.done).toBe(false)
            expect(next.progress.last_error).toContain('clickhouse boom')
            // Streak tracked so a persistently-failing page eventually gives up
            // instead of rescheduling forever (which overflows transition_count).
            expect(next.progress.consecutive_errors).toBe(1)
            expect(hogQueue.queueInvocations).not.toHaveBeenCalled()
        })

        it('gives up (throws) once a page has errored RERUN_MAX_CONSECUTIVE_PAGE_ERRORS times', async () => {
            // One more errored page past the cap must throw so the worker fails
            // the wrapper job terminally — the guard against an unbounded retry
            // loop that would drive the SMALLINT transition_count to overflow.
            const state = brokenState()
            state.progress.consecutive_errors = RERUN_MAX_CONSECUTIVE_PAGE_ERRORS - 1

            await expect(
                buildBrokenPaginator().processPage(team.id, state, {
                    jobId: 'test-rerun-job',
                    createdAt: DateTime.now(),
                })
            ).rejects.toThrow('clickhouse boom')
        })
    })

    describe('done logic (real CH)', () => {
        it('marks done when fewer rows came back than the page size (no further pages)', async () => {
            await seedRows([{ invocation_id: 'one', status: 'failed', error: new Error('5xx') }])

            const state = buildState({
                request: {
                    filter: { window_start: '2026-01-01T00:00:00Z', window_end: '2027-01-01T00:00:00Z' },
                },
            })

            const { state: next } = await paginator.processPage(team.id, state, {
                jobId: 'test-rerun-job',
                createdAt: DateTime.now(),
            })
            // 1 row < RERUN_PAGE_SIZE → done.
            expect(next.progress.done).toBe(true)
            // Sanity: RERUN_PAGE_SIZE constant didn't regress to a tiny value.
            expect(RERUN_PAGE_SIZE).toBeGreaterThan(10)
        })

        it('carries first_scheduled_at from the stored row onto the rehydrated invocation state', async () => {
            // The producer writes `first_scheduled_at` verbatim on every retry's
            // lifecycle row so ReplacingMergeTree doesn't collapse it away. The
            // paginator's contract here: read it from the stored row, stamp it
            // onto the rehydrated state so the next retry's producer sees the
            // same original time. Without this, every kafka retry would
            // overwrite first_scheduled_at with the latest scheduled_at.
            const id = 'inv-first-' + new UUIDT().toString()
            const originalScheduledAt = new Date('2026-05-01T09:00:00Z')
            await seedRows([
                {
                    invocation_id: id,
                    status: 'failed',
                    error: new Error('5xx'),
                    scheduledAt: originalScheduledAt,
                },
            ])

            const state = buildState({
                request: {
                    filter: {
                        window_start: '2026-01-01T00:00:00Z',
                        window_end: '2027-01-01T00:00:00Z',
                        invocation_ids: [id],
                    },
                },
            })

            await paginator.processPage(team.id, state, {
                jobId: 'test-rerun-job',
                createdAt: DateTime.now(),
            })

            const enqueued = hogQueue.queueInvocations.mock.calls[0][0] as CyclotronJobInvocationHogFunction[]
            expect(enqueued).toHaveLength(1)
            // CH returns DateTime64 as 'YYYY-MM-DD HH:MM:SS.ffffff' — just check
            // the date portion matches the original scheduled time. The exact
            // serialization differs between paths but the day/time is preserved.
            expect(enqueued[0].state.firstScheduledAt).toMatch(/^2026-05-01[T ]09:00:00/)
        })

        it('reads the latest row per invocation via argMax(_, version) collapse', async () => {
            // Same invocation_id, multiple lifecycle rows simulating: original
            // failed → user clicked rerun → rerun running → rerun succeeded.
            // The paginator should see the LATEST `succeeded` state, not the
            // earlier `failed`. (Filter mode defaults to status=failed, so an
            // invocation that's now succeeded should NOT be picked up.)
            const id = 'inv-collapse-' + new UUIDT().toString()
            await seedRows([{ invocation_id: id, status: 'failed', error: new Error('5xx') }])
            // Small sleep to ensure version monotonicity is observable.
            await new Promise((r) => setTimeout(r, 5))
            await seedRows([{ invocation_id: id, status: 'succeeded' }])

            const state = buildState({
                request: {
                    filter: { window_start: '2026-01-01T00:00:00Z', window_end: '2027-01-01T00:00:00Z' },
                },
            })

            const { state: next } = await paginator.processPage(team.id, state, {
                jobId: 'test-rerun-job',
                createdAt: DateTime.now(),
            })
            expect(next.progress.queued).toBe(0)
        })
    })

    // Silence unused-import noise for HogFunctionInvocationGlobals which is
    // referenced indirectly via the rehydration assertion.
    void (undefined as unknown as HogFunctionInvocationGlobals)
})