Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

29.3 KB
Raw Blame
import { MockKafkaProducerWrapper } from '~/tests/helpers/mocks/producer.mock'
import { mockFetch } from '~/tests/helpers/mocks/request.mock'

import { KafkaProducerObserver } from '~/tests/helpers/mocks/producer.spy'

import { DateTime } from 'luxon'
import { Pool } from 'pg'

import { KAFKA_HOG_INVOCATION_RESULTS } from '~/common/config/kafka-topics'
import { KafkaProducerWrapper } from '~/common/kafka/producer'
import { PersonReadRepository } from '~/common/persons/repositories/person-repository'
import { closeHub, createHub } from '~/common/utils/db/hub'
import { createCdpConsumerDeps } from '~/tests/helpers/cdp'
import { Clickhouse } from '~/tests/helpers/clickhouse'
import { waitForExpect } from '~/tests/helpers/expectations'
import { waitForHogInvocationResultsMvReady } from '~/tests/helpers/hog-invocation-results'
import { TEST_KAFKA_TOPICS, ensureKafkaTopics } from '~/tests/helpers/kafka'
import { getFirstTeam, resetTestDatabase } from '~/tests/helpers/sql'

import { Hub, Team } from '../../types'
import { FixtureHogFlowBuilder } from '../_tests/builders/hogflow.builder'
import { HOG_FILTERS_EXAMPLES, HOG_INPUTS_EXAMPLES } from '../_tests/examples'
import {
    insertHogFunction as _insertHogFunction,
    createHogExecutionGlobals,
    insertHogFunctionTemplate,
} from '../_tests/fixtures'
import { insertHogFlow } from '../_tests/fixtures-hogflows'
import { CdpConsumerBaseDeps } from '../consumers/cdp-base.consumer'
import { CdpCyclotronWorkerHogFlow } from '../consumers/cdp-cyclotron-worker-hogflow.consumer'
import { CdpCyclotronWorker } from '../consumers/cdp-cyclotron-worker.consumer'
import { CdpEventsConsumer } from '../consumers/cdp-events.consumer'
import { CdpRerunWorkerConsumer } from '../consumers/cdp-rerun-worker.consumer'
import { CyclotronJobQueueKafka } from '../services/job-queue/job-queue-kafka'
import { CyclotronJobQueuePostgresV2 } from '../services/job-queue/job-queue-postgres-v2'
import { compileHog } from '../templates/compiler'
import { HogFunctionInvocationGlobals, HogFunctionType } from '../types'
import { RerunJobManager } from './rerun-job.manager'
import { RERUN_QUEUE_NAME } from './rerun-job.types'

const ActualKafkaProducerWrapper = jest.requireActual('~/common/kafka/producer').KafkaProducerWrapper

const NODE_DB_URL = 'postgres://posthog:posthog@localhost:5432/test_cyclotron_node'

interface PersistedRow {
    invocation_id: string
    status: string
    is_retry: number
    attempts: number
    error_kind: string
    function_kind: string
    invocation_globals: string
}

/**
 * End-to-end exercise of the runs/rerun pipeline.
 *
 * Everything is real: real Kafka, real ClickHouse Kafka MV, real cyclotron-v2
 * postgres, real cyclotron worker, real rerun wrapper-job loop. The ONLY thing
 * mocked is the inbound Django request — we call `RerunJobManager.enqueue(...)`
 * directly with the same payload the Django view would proxy through.
 *
 * Flow under test:
 *   1. Hog function runs -> producer writes a 'succeeded' lifecycle row to
 *      Kafka -> MV lands it in `hog_invocation_results`.
 *   2. Simulated Django POST: `rerunManager.enqueue({ invocation_ids })`.
 *   3. Real `CdpRerunWorkerConsumer` dequeues the wrapper job, queries the
 *      real `hog_invocation_results`, rehydrates globals (rebuilding inputs),
 *      and re-enqueues onto the regular cyclotron queue.
 *   4. The real cyclotron worker picks up the rerun invocation and writes a
 *      second lifecycle row, this time with `is_retry=1` and `attempts > 1`.
 */
describe('CDP hog invocation rerun e2e', () => {
    jest.setTimeout(60_000)

    let hub: Hub
    let kafkaProducer: KafkaProducerWrapper
    let team: Team
    let fnFetch: HogFunctionType
    let globals: HogFunctionInvocationGlobals
    let mockProducerObserver: KafkaProducerObserver
    let eventsConsumer: CdpEventsConsumer
    let cyclotronWorker: CdpCyclotronWorker
    let hogflowWorker: CdpCyclotronWorkerHogFlow
    let rerunManager: RerunJobManager
    let rerunWorker: CdpRerunWorkerConsumer
    let kafkaQueue: CyclotronJobQueueKafka
    let postgresV2Queue: CyclotronJobQueuePostgresV2
    let nodeAssertPool: Pool
    let clickhouse: Clickhouse
    let cdpDeps: CdpConsumerBaseDeps

    beforeAll(() => {
        clickhouse = Clickhouse.create()
    })

    afterAll(() => {
        clickhouse?.close()
        jest.useRealTimers()
    })

    beforeEach(async () => {
        // Use a real KafkaProducerWrapper per the cdp-e2e pattern — gives us a
        // real producer to observe via KafkaProducerObserver.
        MockKafkaProducerWrapper.create = jest.fn((...args: any[]) => ActualKafkaProducerWrapper.create(...args))

        // Ensure all topics exist (idempotently, without deleting) so the ClickHouse
        // Kafka engine consumers keep their connections. Includes KAFKA_HOG_INVOCATION_RESULTS,
        // which this test's MV needs but the shared set does not cover.
        await ensureKafkaTopics([...TEST_KAFKA_TOPICS, KAFKA_HOG_INVOCATION_RESULTS])
        await clickhouse.truncate('hog_invocation_results_data')
        await waitForHogInvocationResultsMvReady(clickhouse)
        await resetTestDatabase()
        await clickhouse.truncate('hog_invocation_results_data')

        hub = await createHub()
        kafkaProducer = await ActualKafkaProducerWrapper.create(hub.KAFKA_CLIENT_RACK)
        mockProducerObserver = new KafkaProducerObserver(kafkaProducer)

        team = await getFirstTeam(hub.postgres)

        // The rerun strips `person` from invocation_globals — the cyclotron
        // worker reloads it via getCyclotronPerson(distinct_id). Provide a mock
        // person so the rerun can resolve `{person}` in the function's inputs.
        const mockPersonRepo: jest.Mocked<PersonReadRepository> = {
            fetchPerson: jest.fn().mockResolvedValue(undefined),
            fetchPersonsByDistinctIds: jest.fn().mockResolvedValue([
                {
                    id: '1',
                    uuid: 'dd3d6f80-60ad-45c3-bd61-e2300f2ba7e1',
                    team_id: team.id,
                    properties: { email: '[email protected]' },
                    properties_last_updated_at: {},
                    properties_last_operation: null,
                    created_at: DateTime.utc(),
                    version: 1,
                    is_identified: true,
                    is_user_id: null,
                    last_seen_at: null,
                    distinct_id: 'distinct_id',
                },
            ]),
            fetchPersonsByPersonIds: jest.fn().mockResolvedValue([]),
            fetchDistinctIdsForPersons: jest.fn().mockResolvedValue({}),
        }

        mockProducerObserver.resetKafkaProducer()

        hub.CDP_FETCH_RETRIES = 0
        hub.CDP_FETCH_BACKOFF_BASE_MS = 1
        hub.CDP_CYCLOTRON_COMPRESS_KAFKA_DATA = true
        hub.CYCLOTRON_NODE_DATABASE_URL = NODE_DB_URL
        hub.HOG_INVOCATION_RESULTS_ENABLED = true

        // Clean any stale rerun wrapper jobs from prior runs.
        nodeAssertPool = new Pool({ connectionString: NODE_DB_URL })
        await nodeAssertPool.query('DELETE FROM cyclotron_jobs WHERE queue_name = $1', [RERUN_QUEUE_NAME])

        const hog = `
        let res := fetch(inputs.url, {
            'headers': { 'Content-Type': 'application/json' },
            'body': inputs.body,
            'method': inputs.method
        });
        print('Fetch response:', res);
        `

        fnFetch = await _insertHogFunction(hub.postgres, team.id, {
            type: 'destination',
            hog,
            bytecode: await compileHog(hog),
            inputs_schema: HOG_INPUTS_EXAMPLES.simple_fetch.inputs_schema ?? [],
            inputs: HOG_INPUTS_EXAMPLES.simple_fetch.inputs,
            ...HOG_FILTERS_EXAMPLES.no_filters,
        })

        kafkaQueue = new CyclotronJobQueueKafka(hub.KAFKA_CLIENT_RACK, hub, hub.CONSUMER_BATCH_SIZE)
        postgresV2Queue = new CyclotronJobQueuePostgresV2(hub.CONSUMER_BATCH_SIZE, hub)

        cdpDeps = { ...createCdpConsumerDeps(hub, kafkaProducer), personRepository: mockPersonRepo }

        eventsConsumer = new CdpEventsConsumer(hub, cdpDeps, {
            hogQueue: kafkaQueue,
            hogflowQueue: postgresV2Queue,
        })
        // We call processBatch directly — no need to actually join the kafka group.
        // Stubbing keeps the test off Redpanda's stale-group-protocol coordinator,
        // which otherwise fails the join with "Inconsistent group protocol".
        eventsConsumer['kafkaConsumer'] = {
            connect: jest.fn(),
            disconnect: jest.fn(),
            isHealthy: jest.fn(),
        } as any
        await eventsConsumer.start()

        cyclotronWorker = new CdpCyclotronWorker(hub, cdpDeps, kafkaQueue)
        await cyclotronWorker.start()

        rerunManager = new RerunJobManager({ dbUrl: NODE_DB_URL, maxCount: 10000 })
        await rerunManager.connect()

        globals = createHogExecutionGlobals({
            project: { id: team.id } as any,
            event: {
                uuid: '0d0ff14e-1b15-4afe-99e3-1ea83f0e3aab',
                event: '$pageview',
                properties: { $current_url: 'https://posthog.com' },
                timestamp: '2026-05-10T09:00:00Z',
            } as any,
        })
    })

    afterEach(async () => {
        await Promise.all([
            eventsConsumer?.stop().catch(() => undefined),
            cyclotronWorker?.stop().catch(() => undefined),
            hogflowWorker?.stop().catch(() => undefined),
            rerunWorker?.stop().catch(() => undefined),
            rerunManager?.disconnect().catch(() => undefined),
        ])
        await kafkaProducer?.disconnect()
        await closeHub(hub)
        await nodeAssertPool.end()
        mockProducerObserver?.resetKafkaProducer()
    })

    it('produces a succeeded lifecycle row, then reruns the invocation via the wrapper-job pipeline', async () => {
        // ── 1. Original invocation succeeds ─────────────────────────────────
        mockFetch.mockResolvedValue({
            status: 200,
            json: () => Promise.resolve({ ok: true }),
            text: () => Promise.resolve('{"ok":true}'),
            headers: { 'Content-Type': 'application/json' },
            dump: () => Promise.resolve(),
        })

        const { invocations } = await eventsConsumer.processBatch([globals])
        expect(invocations).toHaveLength(1)

        // Wait for the terminal 'succeeded' row to flow Kafka -> MV -> ClickHouse.
        await waitForExpect(async () => {
            const rows = await clickhouse.query<PersistedRow>(
                `SELECT invocation_id, status, is_retry, attempts, error_kind, function_kind
                 FROM hog_invocation_results
                 WHERE team_id = ${team.id} AND function_id = '${fnFetch.id}' AND status = 'succeeded'`
            )
            expect(rows.length).toBeGreaterThanOrEqual(1)
        }, 30_000)

        const originalRows = await clickhouse.query<PersistedRow>(
            `SELECT invocation_id, status, is_retry, attempts, error_kind, function_kind, invocation_globals
             FROM hog_invocation_results
             WHERE team_id = ${team.id} AND function_id = '${fnFetch.id}' AND status = 'succeeded'`
        )
        const originalInvocationId = originalRows[0].invocation_id
        expect(originalRows[0].is_retry).toBe(0)
        expect(originalRows[0].function_kind).toBe('hog_function')
        // invocation_globals is stored gzip+base64'd, not raw JSON — base64
        // never starts with `{`. The rerun below proves the round-trip: it can
        // only rehydrate and re-run if `decodeInvocationGlobals` decompresses
        // this value correctly.
        expect(originalRows[0].invocation_globals.length).toBeGreaterThan(0)
        expect(originalRows[0].invocation_globals.startsWith('{')).toBe(false)
        // The prior cyclotron_jobs row is in a terminal 'completed' state by
        // this point. The rerun path's `overwriteExisting: true` upsert
        // (cyclotron-v2 ON CONFLICT) handles the PK collision without us
        // having to manually delete the row.

        // ── 2. Mimic Django POST /rerun — only the request itself is faked ──────
        // The rerun request requires a time window; we use a wide one around
        // "now" (the lifecycle row's scheduled_at = current time) and restrict
        // to a specific invocation_id via the optional filter field.
        const windowStart = new Date(Date.now() - 7 * 24 * 60 * 60 * 1000).toISOString()
        const windowEnd = new Date(Date.now() + 24 * 60 * 60 * 1000).toISOString()
        const rerunJobId = await rerunManager.enqueue(team.id, 'hog_function', fnFetch.id, {
            filter: {
                window_start: windowStart,
                window_end: windowEnd,
                status: ['succeeded'],
                invocation_ids: [originalInvocationId],
            },
        })

        // Wrapper job is real and durable in test_cyclotron_node postgres.
        const wrapperRow = await nodeAssertPool.query(
            'SELECT id, queue_name, status FROM cyclotron_jobs WHERE id = $1',
            [rerunJobId]
        )
        expect(wrapperRow.rows[0]).toMatchObject({
            id: rerunJobId,
            queue_name: RERUN_QUEUE_NAME,
            status: 'available',
        })

        // ── 3. Rerun worker drains the wrapper job ────────────────────────────────
        rerunWorker = new CdpRerunWorkerConsumer(
            { ...hub, CDP_CYCLOTRON_JOB_QUEUE_CONSUMER_MODE: 'postgres' },
            cdpDeps,
            { hog_function: kafkaQueue, hog_flow: postgresV2Queue }
        )
        await rerunWorker.start()

        // ── 4. Wait for the wrapper job to complete ────────────────────────────────
        await waitForExpect(async () => {
            const res = await nodeAssertPool.query('SELECT status FROM cyclotron_jobs WHERE id = $1', [rerunJobId])
            // 'completed' = the paginator marked done=true and the worker acked.
            expect(res.rows[0]?.status).toBe('completed')
        }, 30_000)

        // ── 5. Reruned invocation flows through the real cyclotron worker ─────────
        // The paginator emits a 'running' row at re-enqueue time and the worker
        // emits the terminal row when the rerun invocation completes. Both
        // rows now correctly carry `is_retry=1` / `attempts=1` because the
        // paginator sets `state.rerunAttempts=1` on the rehydrated invocation
        // and `queueLifecycleRow` derives those columns from that field.
        await waitForExpect(async () => {
            const rows = await clickhouse.query<PersistedRow>(
                `SELECT invocation_id, status, is_retry, attempts, error_kind, function_kind
                 FROM hog_invocation_results
                 WHERE team_id = ${team.id}
                   AND function_id = '${fnFetch.id}'
                   AND invocation_id = '${originalInvocationId}'
                   AND is_retry = 1`
            )
            // Either the paginator's running row, the worker's terminal row,
            // or both should be present and tagged is_retry=1.
            expect(rows.length).toBeGreaterThanOrEqual(1)
            for (const row of rows) {
                expect(row.attempts).toBe(1)
            }
        }, 30_000)

        // ── 6. The hog function's fetch was called twice — once original, once rerun ─
        // The wait above can pass on the paginator's running row, before the worker executes; poll for the fetch.
        await waitForExpect(() => {
            expect(mockFetch.mock.calls.length).toBeGreaterThanOrEqual(2)
        }, 30_000)
    })

    it('reruns successfully when person rehydration misses and the input reads person.properties', async () => {
        // On the rerun path the paginator strips globals.person; the worker
        // re-hydrates via getCyclotronPerson, which returns null when the
        // distinct_id has no row in posthog_persondistinctid (cookieless
        // events by design, deleted persons, etc.). The worker must fall back
        // to an empty person stub — leaving globals.person as undefined halts
        // any input bytecode dereferencing `person.properties.*` with "Could
        // not execute bytecode", before coalesce fallbacks can traverse to an
        // event-level branch.
        //
        // If the worker's null-lookup fallback regresses, this test's rerun
        // fetch never fires and the count assertion fails.

        // Custom function whose input specifically dereferences
        // `person.properties.foo` — the shape that crashed pre-fix.
        const FALLBACK_WEBHOOK_URL = 'https://example.com/person-props-fallback'
        const gclidLikeHog = `
        let res := fetch(inputs.url, { 'method': 'POST', 'body': f'foo={inputs.foo}' });
        print('Fetch response:', res);
        `
        const personPropsFn = await _insertHogFunction(hub.postgres, team.id, {
            type: 'destination',
            hog: gclidLikeHog,
            bytecode: await compileHog(gclidLikeHog),
            inputs_schema: [
                { key: 'url', type: 'string', label: 'URL', secret: false, required: true },
                { key: 'foo', type: 'string', label: 'foo', secret: false, required: false },
            ],
            inputs: {
                url: {
                    value: FALLBACK_WEBHOOK_URL,
                    bytecode: ['_h', 32, FALLBACK_WEBHOOK_URL],
                },
                // Same shape as the Google Ads template: person.properties first,
                // event.properties as the tail fallback.
                foo: {
                    value: '{person.properties.foo ?? event.properties.foo}',
                    bytecode: await compileHog('return person.properties.foo ?? event.properties.foo'),
                },
            },
            ...HOG_FILTERS_EXAMPLES.no_filters,
        })

        mockFetch.mockResolvedValue({
            status: 200,
            json: () => Promise.resolve({ ok: true }),
            text: () => Promise.resolve('{"ok":true}'),
            headers: { 'Content-Type': 'application/json' },
            dump: () => Promise.resolve(),
        })

        // Simulate cookieless / missing-person for the rerun: personRepo returns
        // no rows for any distinct_id, so getCyclotronPerson returns null. The
        // ORIGINAL invocation doesn't call the repo — events consumer uses the
        // person we attach to data.person directly — so this only affects the
        // rerun path's rehydration.
        const personRepo = cdpDeps.personRepository as jest.Mocked<PersonReadRepository>
        personRepo.fetchPersonsByDistinctIds.mockResolvedValue([])

        // Original event carries the fallback value on event.properties. The
        // person we hand-attach here has no `foo`, so even on the original the
        // coalesce falls through to event.properties.foo — proving the
        // fallback wiring is real (and letting the rerun's stubbed-empty
        // person hit the same branch).
        const cookielessGlobals = createHogExecutionGlobals({
            project: { id: team.id } as any,
            event: {
                uuid: 'aaaa1111-bbbb-2222-cccc-333333333333',
                event: '$pageview',
                distinct_id: 'cookieless_stub',
                properties: { foo: 'from-event' },
                timestamp: '2026-07-02T09:00:00Z',
            } as any,
            person: {
                id: '',
                name: 'cookieless_stub',
                url: 'http://localhost:8000/persons/cookieless_stub',
                properties: {}, // no `foo` — coalesce must reach event branch
            },
        })

        const { invocations } = await eventsConsumer.processBatch([cookielessGlobals])
        // Both fnFetch (from beforeEach) and personPropsFn match a $pageview — we
        // care about ours.
        const personPropsInvocations = invocations.filter((i) => i.functionId === personPropsFn.id)
        expect(personPropsInvocations).toHaveLength(1)

        await waitForExpect(async () => {
            const rows = await clickhouse.query<PersistedRow>(
                `SELECT invocation_id, status FROM hog_invocation_results
                 WHERE team_id = ${team.id} AND function_id = '${personPropsFn.id}' AND status = 'succeeded'`
            )
            expect(rows.length).toBeGreaterThanOrEqual(1)
        }, 30_000)

        const originalRows = await clickhouse.query<PersistedRow>(
            `SELECT invocation_id, status FROM hog_invocation_results
             WHERE team_id = ${team.id} AND function_id = '${personPropsFn.id}' AND status = 'succeeded'`
        )
        const originalInvocationId = originalRows[0].invocation_id

        // Trigger the rerun scoped to just this invocation_id.
        const windowStart = new Date(Date.now() - 7 * 24 * 60 * 60 * 1000).toISOString()
        const windowEnd = new Date(Date.now() + 24 * 60 * 60 * 1000).toISOString()
        const rerunJobId = await rerunManager.enqueue(team.id, 'hog_function', personPropsFn.id, {
            filter: {
                window_start: windowStart,
                window_end: windowEnd,
                status: ['succeeded'],
                invocation_ids: [originalInvocationId],
            },
        })

        rerunWorker = new CdpRerunWorkerConsumer(
            { ...hub, CDP_CYCLOTRON_JOB_QUEUE_CONSUMER_MODE: 'postgres' },
            cdpDeps,
            { hog_function: kafkaQueue, hog_flow: postgresV2Queue }
        )
        await rerunWorker.start()

        await waitForExpect(async () => {
            const res = await nodeAssertPool.query('SELECT status FROM cyclotron_jobs WHERE id = $1', [rerunJobId])
            expect(res.rows[0]?.status).toBe('completed')
        }, 30_000)

        // The core assertion: the rerun re-fired the fetch to our specific URL,
        // with the value coming through the coalesce's event-level fallback.
        // Pre-fix, the input eval crashed on `undefined.properties` and this
        // count stayed at 1 (just the original). Post-fix, it's 2 (original +
        // rerun), and the rerun's body carries `foo=from-event`.
        // Job 'completed' only means the paginator finished re-enqueueing; the worker executes asynchronously, so poll.
        await waitForExpect(() => {
            expect(
                mockFetch.mock.calls.filter(([url]) => String(url).startsWith(FALLBACK_WEBHOOK_URL)).length
            ).toBeGreaterThanOrEqual(2)
        }, 30_000)
        const ourCalls = mockFetch.mock.calls.filter(([url]) => String(url).startsWith(FALLBACK_WEBHOOK_URL))
        const rerunBody = String((ourCalls[ourCalls.length - 1] as any)[1]?.body ?? '')
        expect(rerunBody).toContain('foo=from-event')
    })

    it('reruns a multi-action hog flow by resuming, not restarting, so completed actions do not re-fire', async () => {
        // The hog-flow counterpart to the hog_function reruns above, and the core
        // guard for #70792: a hog_function rerun re-executes the whole function
        // (fetch fires again, by design), but a hog FLOW rerun must RESUME from
        // where the recorded run left off — never restart from the trigger and
        // re-run already-completed actions. A completed flow's recorded state
        // sits at `exit`, so its rerun resumes there and the fetch action never
        // fires a second time. Pre-#70792 the rehydration dropped `currentAction`,
        // the flow restarted from the trigger, and the fetch re-fired — the
        // double-send this proves is gone. Distinct URL isolates the flow's fetch
        // from the beforeEach hog_function that also matches this event.
        const FLOW_WEBHOOK_URL = 'https://example.com/flow-resume-webhook'

        mockFetch.mockResolvedValue({
            status: 200,
            json: () => Promise.resolve({ ok: true }),
            text: () => Promise.resolve('{"ok":true}'),
            headers: { 'Content-Type': 'application/json' },
            dump: () => Promise.resolve(),
        })

        // The flow's function action runs a template, not inline hog — insert the same
        // fetch template the workflows-e2e suite uses.
        await insertHogFunctionTemplate(hub.postgres, {
            id: 'template-rerun-e2e-fetch',
            name: 'Rerun E2E Fetch',
            code: `
            let res := fetch(inputs.url, {'method': inputs.method});
            print('Fetch result:', res.status);
            `,
            inputs_schema: [
                { key: 'url', type: 'string', required: true },
                { key: 'method', type: 'string', required: false },
            ],
        })

        const flow = new FixtureHogFlowBuilder()
            .withTeamId(team.id)
            .withStatus('active')
            .withWorkflow({
                actions: {
                    trigger: {
                        type: 'trigger',
                        config: { type: 'event', filters: HOG_FILTERS_EXAMPLES.no_filters.filters ?? {} },
                    },
                    function_1: {
                        type: 'function',
                        config: {
                            template_id: 'template-rerun-e2e-fetch',
                            inputs: { url: { value: FLOW_WEBHOOK_URL }, method: { value: 'POST' } },
                        },
                    },
                    exit: { type: 'exit', config: {} },
                },
                edges: [
                    { from: 'trigger', to: 'function_1', type: 'continue' },
                    { from: 'function_1', to: 'exit', type: 'continue' },
                ],
            })
            .build()
        await insertHogFlow(hub.postgres, flow)

        // The hog-flow worker polls the same postgres-v2 backend the events consumer
        // routes flows to (hogflowQueue in beforeEach).
        hogflowWorker = new CdpCyclotronWorkerHogFlow(hub, cdpDeps, postgresV2Queue)
        await hogflowWorker.start()

        // ── 1. Original flow runs to completion — fetch fires exactly once ──────────
        const { backgroundTask } = await eventsConsumer.processBatch([globals])
        await backgroundTask

        await waitForExpect(() => {
            expect(mockFetch.mock.calls.filter(([url]) => String(url) === FLOW_WEBHOOK_URL).length).toBe(1)
        }, 15_000)

        // Terminal 'succeeded' hog_flow row lands in ClickHouse via Kafka -> MV.
        await waitForExpect(async () => {
            const rows = await clickhouse.query<PersistedRow>(
                `SELECT invocation_id, status, is_retry, function_kind
                 FROM hog_invocation_results
                 WHERE team_id = ${team.id} AND function_id = '${flow.id}' AND status = 'succeeded'`
            )
            expect(rows.length).toBeGreaterThanOrEqual(1)
        }, 30_000)

        const originalRows = await clickhouse.query<PersistedRow>(
            `SELECT invocation_id, status, is_retry, function_kind
             FROM hog_invocation_results
             WHERE team_id = ${team.id} AND function_id = '${flow.id}' AND status = 'succeeded'`
        )
        expect(originalRows[0].function_kind).toBe('hog_flow')
        expect(originalRows[0].is_retry).toBe(0)
        const originalInvocationId = originalRows[0].invocation_id

        // ── 2. Mimic Django POST /rerun for the flow, scoped to this invocation ─────
        const windowStart = new Date(Date.now() - 7 * 24 * 60 * 60 * 1000).toISOString()
        const windowEnd = new Date(Date.now() + 24 * 60 * 60 * 1000).toISOString()
        const rerunJobId = await rerunManager.enqueue(team.id, 'hog_flow', flow.id, {
            filter: {
                window_start: windowStart,
                window_end: windowEnd,
                status: ['succeeded'],
                invocation_ids: [originalInvocationId],
            },
        })

        // ── 3. Rerun worker drains the wrapper job (hog_flow -> postgres-v2) ────────
        rerunWorker = new CdpRerunWorkerConsumer(
            { ...hub, CDP_CYCLOTRON_JOB_QUEUE_CONSUMER_MODE: 'postgres' },
            cdpDeps,
            { hog_function: kafkaQueue, hog_flow: postgresV2Queue }
        )
        await rerunWorker.start()

        await waitForExpect(async () => {
            const res = await nodeAssertPool.query('SELECT status FROM cyclotron_jobs WHERE id = $1', [rerunJobId])
            expect(res.rows[0]?.status).toBe('completed')
        }, 30_000)

        // ── 4. Rerun invocation flows back through the real hog-flow worker ─────────
        // Wait for the rerun's terminal row (is_retry=1) so we know it fully executed
        // — by this point it would have re-fired the fetch if it were restarting.
        await waitForExpect(async () => {
            const rows = await clickhouse.query<PersistedRow>(
                `SELECT invocation_id, status, is_retry, function_kind
                 FROM hog_invocation_results
                 WHERE team_id = ${team.id}
                   AND function_id = '${flow.id}'
                   AND invocation_id = '${originalInvocationId}'
                   AND is_retry = 1`
            )
            expect(rows.length).toBeGreaterThanOrEqual(1)
        }, 30_000)

        // ── 5. The resume ran nothing already done: the fetch is still at ONE call ──
        // Restart-from-trigger (the pre-#70792 bug) would make this 2.
        expect(mockFetch.mock.calls.filter(([url]) => String(url) === FLOW_WEBHOOK_URL).length).toBe(1)
    })
})