Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

9.6 KB
Raw Blame
import { DateTime } from 'luxon'

import { closeHub, createHub } from '~/common/utils/db/hub'
import { parseJSON } from '~/common/utils/json-parse'

import { createCdpConsumerDeps } from '../../../tests/helpers/cdp'
import { Hub } from '../../types'
import { RERUN_PAGE_ERROR_BACKOFF_MAX_MS, RERUN_QUEUE_NAME, RerunJobState } from '../rerun/rerun-job.types'
import { RerunJobQueues } from '../rerun/rerun-paginator.service'
import { CyclotronV2DequeuedJob } from '../services/cyclotron-v2/types'
import { CdpRerunWorkerConsumer, pageRetryDelayMs } from './cdp-rerun-worker.consumer'

// Minimal stub for a job queue backend — the consumer only calls the producer
// lifecycle hooks; the paginator (which would call queueInvocations) is stubbed.
const buildMockJobQueues = (): RerunJobQueues => {
    const stub = () =>
        ({
            startAsProducer: jest.fn().mockResolvedValue(undefined),
            stopProducer: jest.fn().mockResolvedValue(undefined),
            queueInvocations: jest.fn().mockResolvedValue(undefined),
        }) as any
    return { hog_function: stub(), hog_flow: stub() }
}

const buildDequeuedJob = (overrides: Partial<CyclotronV2DequeuedJob> = {}): jest.Mocked<CyclotronV2DequeuedJob> => {
    return {
        id: overrides.id ?? 'wrapper-job-1',
        teamId: overrides.teamId ?? 7,
        functionId: overrides.functionId ?? 'fn-1',
        queueName: RERUN_QUEUE_NAME,
        priority: 0,
        scheduled: DateTime.now(),
        created: DateTime.now(),
        parentRunId: null,
        transitionCount: 1,
        state: overrides.state ?? null,
        distinctId: null,
        personId: null,
        actionId: null,
        ack: jest.fn().mockResolvedValue(undefined),
        fail: jest.fn().mockResolvedValue(undefined),
        reschedule: jest.fn().mockResolvedValue(undefined),
        cancel: jest.fn().mockResolvedValue(undefined),
        heartbeat: jest.fn().mockResolvedValue(undefined),
    } as unknown as jest.Mocked<CyclotronV2DequeuedJob>
}

const buildState = (overrides: Partial<RerunJobState> = {}): RerunJobState => ({
    function_kind: 'hog_function',
    function_id: 'fn-1',
    request: {
        filter: {
            window_start: '2026-01-01T00:00:00Z',
            window_end: '2027-01-01T00:00:00Z',
            invocation_ids: ['inv-1'],
        },
    },
    progress: { queued: 0, skipped: 0, done: false },
    ...overrides,
})

const jobWithState = (state: RerunJobState): jest.Mocked<CyclotronV2DequeuedJob> =>
    buildDequeuedJob({ state: Buffer.from(JSON.stringify(state)) })

describe('CdpRerunWorkerConsumer', () => {
    let consumer: CdpRerunWorkerConsumer
    let hub: Hub
    let mockProcessPage: jest.Mock

    beforeEach(async () => {
        hub = await createHub()
        // The worker needs the cyclotron-v2 node db to exist; pretend it does so the
        // consumer constructs cleanly. Real queue interactions are mocked below.
        hub.CYCLOTRON_NODE_DATABASE_URL = 'postgres://posthog:posthog@localhost:5432/test_cyclotron_node'

        consumer = new CdpRerunWorkerConsumer(hub, createCdpConsumerDeps(hub), buildMockJobQueues())

        // Replace the paginator with a stub so we exercise the consumer's
        // ack/reschedule/fail decisions without hitting ClickHouse. The catch
        // path also calls `writeWrapperFailure` so the stub stubs that too.
        mockProcessPage = jest.fn()
        consumer['paginator'] = {
            processPage: mockProcessPage,
            writeWrapperFailure: jest.fn().mockResolvedValue(undefined),
        } as any

        // Don't start the real cyclotron worker loop — job queue producers are
        // already stubbed via buildMockJobQueues.
        consumer['worker'] = null
    })

    afterEach(async () => {
        // stop() touches the worker and cyclotron client; both are stubbed/null above.
        await consumer.stop().catch(() => undefined)
        await closeHub(hub)
    })

    describe('handleJob', () => {
        it('acks the wrapper job when the paginator returns done=true', async () => {
            const state = buildState()
            const job = jobWithState(state)
            mockProcessPage.mockResolvedValue({
                state: { ...state, progress: { ...state.progress, done: true, queued: 1 } },
            })

            await consumer['handleJob'](job)

            expect(job.ack).toHaveBeenCalledTimes(1)
            expect(job.reschedule).not.toHaveBeenCalled()
            expect(job.fail).not.toHaveBeenCalled()
        })

        it('reschedules the wrapper job with updated state when the paginator returns done=false', async () => {
            const state = buildState()
            const job = jobWithState(state)
            const nextState: RerunJobState = {
                ...state,
                progress: {
                    ...state.progress,
                    cursor: { scheduled_at: '2026-05-01 00:00:00.000000', invocation_id: 'last' },
                    queued: 200,
                    done: false,
                },
            }
            mockProcessPage.mockResolvedValue({ state: nextState })

            await consumer['handleJob'](job)

            expect(job.ack).not.toHaveBeenCalled()
            expect(job.fail).not.toHaveBeenCalled()
            expect(job.reschedule).toHaveBeenCalledTimes(1)
            const rescheduleArg = job.reschedule.mock.calls[0][0]!
            expect(rescheduleArg.scheduledAt).toBeInstanceOf(Date)
            // Persisted state matches the new state returned by the paginator.
            const persisted = parseJSON(rescheduleArg.state!.toString('utf8')) as RerunJobState
            expect(persisted.progress.queued).toBe(200)
            expect(persisted.progress.cursor).toEqual(nextState.progress.cursor)
        })

        it('fails the wrapper job when state JSON is malformed', async () => {
            const job = buildDequeuedJob({ state: Buffer.from('not-json{') })

            await consumer['handleJob'](job)

            expect(job.fail).toHaveBeenCalledTimes(1)
            expect(mockProcessPage).not.toHaveBeenCalled()
        })

        it('fails the wrapper job when state is missing entirely', async () => {
            const job = buildDequeuedJob({ state: null })

            await consumer['handleJob'](job)

            expect(job.fail).toHaveBeenCalledTimes(1)
            expect(mockProcessPage).not.toHaveBeenCalled()
        })

        it('fails the wrapper job when the paginator throws', async () => {
            const state = buildState()
            const job = jobWithState(state)
            mockProcessPage.mockRejectedValue(new Error('paginator exploded'))

            await consumer['handleJob'](job)

            expect(job.fail).toHaveBeenCalledTimes(1)
            expect(job.ack).not.toHaveBeenCalled()
            expect(job.reschedule).not.toHaveBeenCalled()
        })

        it('heartbeats during a long-running page so the cyclotron lock does not expire', async () => {
            jest.useFakeTimers()
            const state = buildState()
            const job = jobWithState(state)

            // Make processPage take long enough for two heartbeat ticks.
            mockProcessPage.mockImplementation(async () => {
                await new Promise<void>((resolve) => {
                    setTimeout(() => resolve(), 25_000)
                })
                return { state: { ...state, progress: { ...state.progress, done: true } } }
            })

            const handlePromise = consumer['handleJob'](job)
            await jest.advanceTimersByTimeAsync(25_000)
            await handlePromise

            expect(job.heartbeat).toHaveBeenCalled()
            expect(job.heartbeat.mock.calls.length).toBeGreaterThanOrEqual(2)
            jest.useRealTimers()
        })

        it('fails the job when the paginator field is unset (defensive guard)', async () => {
            consumer['paginator'] = null
            const state = buildState()
            const job = jobWithState(state)

            await consumer['handleJob'](job)

            expect(job.fail).toHaveBeenCalledTimes(1)
        })
    })

    describe('handleBatch', () => {
        it('drives every job in the batch through handleJob in order', async () => {
            const state = buildState()
            mockProcessPage.mockResolvedValue({ state: { ...state, progress: { ...state.progress, done: true } } })

            const jobs = [jobWithState(state), jobWithState({ ...state, function_id: 'fn-2' })]
            await consumer['handleBatch'](jobs)

            for (const job of jobs) {
                expect(job.ack).toHaveBeenCalledTimes(1)
            }
            expect(mockProcessPage).toHaveBeenCalledTimes(2)
        })
    })

    describe('pageRetryDelayMs', () => {
        // A retry storm against a ClickHouse that is already degraded is what
        // produces the streak in the first place, so the delay has to grow with
        // it while a healthy page keeps the flat cadence.
        it.each([
            [0, 500],
            [1, 1000],
            [4, 8000],
            [20, RERUN_PAGE_ERROR_BACKOFF_MAX_MS],
        ])('delays %i consecutive errors by %ims', (consecutiveErrors, expected) => {
            expect(pageRetryDelayMs(consecutiveErrors)).toBe(expected)
        })
    })

    describe('isHealthy', () => {
        it('returns an error result when stopping', () => {
            consumer['isStopping'] = true
            expect(consumer.isHealthy().isError()).toBe(true)
        })

        it('returns an error result when the worker is missing', () => {
            consumer['isStopping'] = false
            consumer['worker'] = null
            expect(consumer.isHealthy().isError()).toBe(true)
        })
    })
})