Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

263 KB
Raw Blame
/**
 * Workflows E2E tests through postgres-v2 (Cyclotron node DB).
 *
 * These tests exercise the full hogflow lifecycle:
 *   event → CdpEventsConsumer → CyclotronJobQueuePostgresV2 (produces to v2 DB)
 *   → CdpCyclotronWorkerHogFlow (polls v2 DB) → HogFlowExecutorService
 *   → results written back to v2 DB → logs/metrics to Kafka
 *
 * Only `fetch` is mocked. Everything else is real: v2 database, Kafka
 * producers, Postgres, Redis, person loading, filter evaluation, and
 * state serialization.
 */
import { MockKafkaProducerWrapper } from '~/tests/helpers/mocks/producer.mock'
import { mockFetch, mockInternalFetch } from '~/tests/helpers/mocks/request.mock'

import { KafkaProducerObserver } from '~/tests/helpers/mocks/producer.spy'

import jsonwebtoken from 'jsonwebtoken'
import { DateTime } from 'luxon'
import { Pool } from 'pg'
import { register } from 'prom-client'
import supertest from 'supertest'
import express from 'ultimate-express'

import { HogFlow } from '~/cdp/schema/hogflow'
import { template as createTaskTemplate } from '~/cdp/templates/_destinations/posthog_tasks/posthog-create-task.template'
import { template as webhookTemplate } from '~/cdp/templates/_destinations/webhook/webhook.template'
import { setupExpressApp } from '~/common/api/router'
import {
    KAFKA_APP_METRICS_2,
    KAFKA_HOG_INVOCATION_RESULTS,
    KAFKA_LOG_ENTRIES,
    KAFKA_MESSAGE_ASSETS,
} from '~/common/config/kafka-topics'
import { KafkaProducerWrapper } from '~/common/kafka/producer'
import { InternalPersonWithDistinctId, PersonReadRepository } from '~/common/persons/repositories/person-repository'
import { deleteKeysWithPrefix } from '~/common/redis/_tests/redis'
import { InternalFetchService } from '~/common/services/internal-fetch'
import { closeHub, createHub } from '~/common/utils/db/hub'
import { PostgresUse } from '~/common/utils/db/postgres'
import { parseJSON } from '~/common/utils/json-parse'
import { UUIDT } from '~/common/utils/utils'
import { createCdpConsumerDeps } from '~/tests/helpers/cdp'
import { waitForExpect } from '~/tests/helpers/expectations'
import { TEST_KAFKA_TOPICS, ensureKafkaTopics } from '~/tests/helpers/kafka'
import { createTeam, getFirstTeam, resetBehavioralCohortsDatabase, resetTestDatabase } from '~/tests/helpers/sql'

import { Hub, Team } from '../../src/types'
import { createRedisV2PoolFromConfig } from '../common/redis/redis-v2'
import { FixtureHogFlowBuilder } from './_tests/builders/hogflow.builder'
import { HOG_FILTERS_EXAMPLES, HOG_FLOW_MASK_EXAMPLES } from './_tests/examples'
import {
    createHogExecutionGlobals,
    insertCohortMembership,
    insertHogFunctionTemplate,
    insertIntegration,
} from './_tests/fixtures'
import { insertHogFlow } from './_tests/fixtures-hogflows'
import { CdpApi } from './cdp-api'
import { CdpCyclotronWorkerBatchResolve } from './consumers/cdp-cyclotron-worker-batch-resolve.consumer'
import { CdpCyclotronWorkerEmail } from './consumers/cdp-cyclotron-worker-email.consumer'
import { CdpCyclotronWorkerHogFlow } from './consumers/cdp-cyclotron-worker-hogflow.consumer'
import { CdpDatawarehouseEventsConsumer } from './consumers/cdp-data-warehouse-events.consumer'
import { CdpEventsConsumer } from './consumers/cdp-events.consumer'
import { CdpHogflowSubscriptionMatcherConsumer } from './consumers/cdp-hogflow-subscription-matcher.consumer'
import { createCdpOutputsRegistry } from './outputs/registry'
import {
    CyclotronV2DequeuedJob,
    CyclotronV2Janitor,
    CyclotronV2Manager,
    CyclotronV2Worker,
    JANITOR_POISON_PILL_ERROR_KIND,
} from './services/cyclotron-v2'
import {
    HOGFLOW_BATCH_RESOLVE_QUEUE,
    MAX_RESOLVER_ATTEMPTS,
    serializeResolverState,
} from './services/hogflows/batch-resolver.types'
import { HogFlowBatchPersonQueryService } from './services/hogflows/hogflow-batch-person-query.service'
import { CyclotronJobQueueKafka } from './services/job-queue/job-queue-kafka'
import { CyclotronJobQueuePostgresV2 } from './services/job-queue/job-queue-postgres-v2'
import { CyclotronJobQueueRateLimitedPostgresV2 } from './services/job-queue/job-queue-rate-limited-postgres-v2'
import { JobQueue } from './services/job-queue/job-queue.interface'
import { createFrequencyCapValkeyPool } from './services/messaging/frequency-cap-valkey-pool'
import { HogInvocationResultsService } from './services/monitoring/hog-invocation-results.service'
import { RateLimiterService } from './services/rate-limiter/rate-limiter.service'
import { HogFunctionInvocationGlobals } from './types'
import { convertBatchHogFlowRequestToHogFunctionInvocationGlobals } from './utils'
import { convertToHogFunctionFilterGlobal } from './utils/hog-function-filtering'

const ActualKafkaProducerWrapper = jest.requireActual('~/common/kafka/producer').KafkaProducerWrapper

// DNS is mocked (like fetch) because EmailValidationService's MX lookups are a real
// network boundary: validation is fail-open, so a CI resolver hiccup would silently
// turn a "skipped hard bounce" assertion into a sent email — a guaranteed flake.
// Implementations are domain-aware and set in the email-queue block's beforeEach.
const mockDnsResolveMx = jest.fn()
const mockDnsResolve4 = jest.fn()
const mockDnsResolve6 = jest.fn()

jest.mock('node:dns/promises', () => ({
    Resolver: jest.fn().mockImplementation(() => ({
        resolveMx: mockDnsResolveMx,
        resolve4: mockDnsResolve4,
        resolve6: mockDnsResolve6,
    })),
}))

// Use the same env vars as config.ts (lines 221-229) so cleanup pools and hub target the same DBs
const CYCLOTRON_NODE_DB_URL =
    process.env.CYCLOTRON_NODE_DATABASE_URL ?? 'postgres://posthog:posthog@localhost:5432/test_cyclotron_node'

describe('Workflows E2E (postgres-v2)', () => {
    jest.setTimeout(30000)

    let eventsConsumer: CdpEventsConsumer
    let dwhConsumer: CdpDatawarehouseEventsConsumer
    let hogflowWorker: CdpCyclotronWorkerHogFlow
    let hogflowQueue: JobQueue

    let hub: Hub
    let kafkaProducer: KafkaProducerWrapper
    let mockProducerObserver: KafkaProducerObserver
    let team: Team
    let globals: HogFunctionInvocationGlobals
    let mockPersonRepo: jest.Mocked<PersonReadRepository>
    let cyclotronPool: Pool
    let deps: ReturnType<typeof createCdpConsumerDeps>

    beforeAll(() => {
        cyclotronPool = new Pool({
            connectionString: CYCLOTRON_NODE_DB_URL,
        })
    })

    afterAll(async () => {
        await cyclotronPool.end()
    })

    beforeEach(async () => {
        // Real Kafka producers for all CDP producer slots
        MockKafkaProducerWrapper.create = jest.fn((...args) => {
            return ActualKafkaProducerWrapper.create(...args)
        })

        await ensureKafkaTopics(TEST_KAFKA_TOPICS)
        await resetTestDatabase()
        await cyclotronPool.query('DELETE FROM cyclotron_jobs')

        hub = await createHub()

        kafkaProducer = await ActualKafkaProducerWrapper.create(hub.KAFKA_CLIENT_RACK)
        mockProducerObserver = new KafkaProducerObserver(kafkaProducer)

        team = await getFirstTeam(hub.postgres)
        mockProducerObserver.resetKafkaProducer()

        hub.CDP_CYCLOTRON_BATCH_DELAY_MS = 50
        hub.CDP_FETCH_RETRIES = 2
        hub.CDP_FETCH_BACKOFF_BASE_MS = 50

        // Insert a simple fetch template for function actions
        await insertHogFunctionTemplate(hub.postgres, {
            id: 'template-workflows-e2e-fetch',
            name: 'Workflows E2E Fetch',
            code: `
            let res := fetch(inputs.url, {'method': inputs.method});
            print('Fetch result:', res.status);
            `,
            inputs_schema: [
                { key: 'url', type: 'string', required: true },
                { key: 'method', type: 'string', required: false },
            ],
        })

        mockPersonRepo = {
            fetchPerson: jest.fn().mockResolvedValue(undefined),
            fetchPersonsByDistinctIds: jest.fn().mockResolvedValue([]),
            fetchPersonsByPersonIds: jest.fn().mockResolvedValue([]),
            fetchDistinctIdsForPersons: jest.fn().mockResolvedValue({}),
        }

        deps = { ...createCdpConsumerDeps(hub, kafkaProducer), personRepository: mockPersonRepo }

        const kafkaQueue = new CyclotronJobQueueKafka(hub.KAFKA_CLIENT_RACK, hub, hub.CONSUMER_BATCH_SIZE)

        hogflowQueue = new CyclotronJobQueuePostgresV2(hub.CONSUMER_BATCH_SIZE, hub)

        // Events consumer — only start as producer (skip Kafka consumer connection).
        // We call processBatch() directly so the Kafka consumer is not needed.
        eventsConsumer = new CdpEventsConsumer(hub, deps, {
            hogQueue: kafkaQueue,
            hogflowQueue,
        })
        // Drives the data-warehouse-table trigger path. We call processBatch() directly, so the
        // Kafka consumer is never connected — the shared queues below are the only producers.
        dwhConsumer = new CdpDatawarehouseEventsConsumer(hub, deps, {
            hogQueue: kafkaQueue,
            hogflowQueue,
        })
        await Promise.all([kafkaQueue.startAsProducer(), hogflowQueue.startAsProducer()])

        // Start hogflow worker (consumer side — polls the postgres-v2 backend)
        hogflowWorker = new CdpCyclotronWorkerHogFlow(hub, deps, hogflowQueue)
        await hogflowWorker.start()

        mockFetch.mockResolvedValue({
            status: 200,
            headers: { 'Content-Type': 'application/json' },
            json: () => Promise.resolve({ success: true }),
            text: () => Promise.resolve(JSON.stringify({ success: true })),
            dump: () => Promise.resolve(),
        })
    })

    afterEach(async () => {
        // dwhConsumer shares the already-stopped queues with eventsConsumer, so it has nothing to stop.
        await Promise.all([eventsConsumer?.stop() ?? Promise.resolve(), hogflowWorker?.stop() ?? Promise.resolve()])
        await kafkaProducer.disconnect()
        await closeHub(hub)
        mockProducerObserver.resetKafkaProducer()
    })

    // ── Helpers ──────────────────────────────────────────────────────

    function createGlobals(
        overrides: Partial<HogFunctionInvocationGlobals['event']> = {}
    ): HogFunctionInvocationGlobals {
        return createHogExecutionGlobals({
            project: { id: team.id } as any,
            event: {
                uuid: 'b3a1fe86-b10c-43cc-acaf-d208977608d0',
                event: '$pageview',
                properties: {
                    $current_url: 'https://posthog.com',
                    $lib_version: '1.0.0',
                },
                timestamp: '2024-09-03T09:00:00Z',
                ...overrides,
            } as any,
        })
    }

    // v2 stores job lifecycle in the `status` column.
    const statusColumn = 'status'

    async function queryCyclotronJobs(): Promise<any[]> {
        const result = await cyclotronPool.query(`SELECT * FROM cyclotron_jobs ORDER BY created ASC`)
        return result.rows
    }

    /** Send an event through the events consumer and wait for it to be queued to v2 */
    async function triggerWorkflow(eventGlobals: HogFunctionInvocationGlobals): Promise<void> {
        const { backgroundTask } = await eventsConsumer.processBatch([eventGlobals])
        await backgroundTask
    }

    /**
     * Mirrors what the hog flow API stores on save: a secret input is stripped out of the plaintext
     * `actions` and kept Fernet-encrypted in `encrypted_inputs`, keyed by action id and then input key.
     */
    async function setWorkflowEncryptedInputs(
        workflowId: string,
        secretsByActionId: Record<string, Record<string, { value: unknown }>>
    ): Promise<void> {
        await hub.postgres.query(
            PostgresUse.COMMON_WRITE,
            `UPDATE posthog_hogflow SET encrypted_inputs = $2 WHERE id = $1`,
            [workflowId, hub.encryptedFields.encrypt(JSON.stringify(secretsByActionId))],
            'setWorkflowEncryptedInputs'
        )
    }

    /**
     * Build the row-scoped globals the DWH consumer produces for a synced warehouse row: a synthetic
     * event named `$warehouse_source_row` with no real person, and the source table on
     * `event.properties.$source_table` so the consumer's eligibilityFn matches warehouse-table triggers.
     */
    function createDwhGlobals(
        tableName: string,
        rowProperties: Record<string, any> = {}
    ): HogFunctionInvocationGlobals {
        return createHogExecutionGlobals({
            project: { id: team.id } as any,
            event: {
                uuid: new UUIDT().toString(),
                event: '$warehouse_source_row',
                distinct_id: '',
                properties: { ...rowProperties, $source_table: tableName },
                timestamp: '2024-09-03T09:00:00Z',
            } as any,
        })
    }

    /** Send a synced warehouse row through the DWH consumer and wait for it to be queued */
    async function triggerDwhWorkflow(rowGlobals: HogFunctionInvocationGlobals): Promise<void> {
        const { backgroundTask } = await dwhConsumer.processBatch([rowGlobals])
        await backgroundTask
    }

    /** Insert an active hogflow for the current team */
    async function createWorkflow(
        workflow: Parameters<FixtureHogFlowBuilder['withWorkflow']>[0],
        opts?: { name?: string }
    ): Promise<string> {
        const flow = await createWorkflowFlow(workflow, opts)
        return flow.id
    }

    /** Same as createWorkflow but returns the full HogFlow object (useful for hand-built invocations) */
    async function createWorkflowFlow(
        workflow: Parameters<FixtureHogFlowBuilder['withWorkflow']>[0],
        opts?: { name?: string; conversion?: HogFlow['conversion']; exitCondition?: HogFlow['exit_condition'] }
    ): Promise<HogFlow> {
        const builder = new FixtureHogFlowBuilder().withTeamId(team.id).withStatus('active').withWorkflow(workflow)
        if (opts?.name) {
            builder.withName(opts.name)
        }
        if (opts?.conversion) {
            builder.withConversion(opts.conversion)
        }
        if (opts?.exitCondition) {
            builder.withExitCondition(opts.exitCondition)
        }
        const flow = builder.build()
        await insertHogFlow(hub.postgres, flow)
        return flow
    }

    /**
     * Construct and enqueue a batch-shaped CyclotronJobInvocation directly, mimicking what
     * the batch resolver's buildHogFlowInvocation would produce. Skips the blast-radius API
     * call so tests don't need to stand up the Django side.
     */
    async function triggerBatchWorkflow(hogFlow: HogFlow, personUuid: string): Promise<void> {
        const invocationGlobals = convertBatchHogFlowRequestToHogFunctionInvocationGlobals({
            team,
            personId: personUuid,
            siteUrl: hub.SITE_URL,
        })
        const filterGlobals = convertToHogFunctionFilterGlobal(invocationGlobals)
        const invocation = {
            id: new UUIDT().toString(),
            state: {
                event: invocationGlobals.event,
                personId: personUuid,
                actionStepCount: 0,
                variables: {},
            },
            teamId: team.id,
            functionId: hogFlow.id,
            parentRunId: new UUIDT().toString(),
            hogFlow,
            person: invocationGlobals.person,
            filterGlobals,
            queue: 'hogflow' as const,
            queuePriority: 1,
        }
        await hogflowQueue.queueInvocations([invocation])
    }

    // Reusable action configs
    const trigger = () =>
        ({
            type: 'trigger' as const,
            config: { type: 'event', filters: HOG_FILTERS_EXAMPLES.no_filters.filters ?? {} },
        }) as const

    const fetchAction = (url: string, method = 'POST') => ({
        type: 'function' as const,
        config: {
            template_id: 'template-workflows-e2e-fetch',
            inputs: { url: { value: url }, method: { value: method } },
        },
    })

    const delayAction = (duration: string) => ({
        type: 'delay' as const,
        config: { delay_duration: duration },
    })

    // Waits for a date carried by the event rather than a fixed span. Bytecode is what the HogQL compiler
    // emits for `properties.expires_at`, which is what HogFlowSerializer stores for this expression.
    const delayUntilAction = (offset?: string) => ({
        type: 'delay' as const,
        config: {
            delay_until: {
                expression: 'properties.expires_at',
                bytecode: ['_H', 1, 32, 'expires_at', 32, 'properties', 1, 2],
                ...(offset ? { offset } : {}),
            },
        },
    })

    const exitAction = () => ({ type: 'exit' as const, config: {} })

    // Mirrors what HogFlowSerializer compiles for {events: [{id: <name>}]}: a single
    // equality check on the `event` global. The matcher is fail-closed when bytecode
    // is absent, so fixtures must supply it the same way Django would on save.
    const eventNameBytecode = (eventName: string): any[] => ['_H', 1, 32, eventName, 32, 'event', 1, 1, 11]
    const eventNameFilter = (eventName: string) => ({
        filters: { events: [{ id: eventName }], bytecode: eventNameBytecode(eventName) },
    })
    // An action-based wait entry: the editor's Actions picker yields a filter with `actions` set and
    // `events` empty. Django compiles the action's match conditions into bytecode the same way.
    const actionFilter = (eventName: string, actionId: number) => ({
        filters: { actions: [{ id: actionId, type: 'actions' }], events: [], bytecode: eventNameBytecode(eventName) },
    })
    // The state left when the last event is removed from a wait entry in the UI: no events, no
    // actions. Empty filters compile to always-true bytecode (op 29), which must NOT wake on every
    // event. ['_H', 1, 29] is exactly what the Django compiler emits for empty filters.
    const emptyEventFilter = () => ({ filters: { events: [], bytecode: ['_H', 1, 29] } })
    // A wait CONDITION with no property filters: the state left when a condition's last filter is
    // removed, or one is added but never filled in. Django compiles it to the same always-true
    // bytecode (op 29). Unlike an events entry, the executor evaluates the condition on entry, so
    // without the guard this fires the wait immediately. Mirrors the serializer's compiled shape.
    const emptyConditionFilters = () => ({ bytecode: ['_H', 1, 29], properties: [] })
    // A wait CONDITION on a person property: `person.properties[key] == value`. Mirrors the bytecode
    // Django compiles for a single person-property equality filter. Used to prove that a person
    // mutation (no analytics event) can satisfy a parked condition wait.
    const personPropertyConditionFilters = (key: string, value: string): any => ({
        bytecode: ['_H', 1, 32, value, 32, key, 32, 'properties', 32, 'person', 1, 3, 11],
        properties: [{ key, value, type: 'person', operator: 'exact' }],
    })

    describe('simple workflow: trigger → function → exit', () => {
        let workflowId: string

        beforeEach(async () => {
            workflowId = await createWorkflow({
                actions: {
                    trigger: trigger(),
                    function_1: fetchAction('https://example.com/webhook'),
                    exit: exitAction(),
                },
                edges: [
                    { from: 'trigger', to: 'function_1', type: 'continue' },
                    { from: 'function_1', to: 'exit', type: 'continue' },
                ],
            })
            globals = createGlobals()
        })

        it('should execute the workflow end-to-end through v2', async () => {
            await triggerWorkflow(globals)

            await waitForExpect(() => {
                expect(mockFetch).toHaveBeenCalledTimes(1)
            }, 10000)

            expect(mockFetch).toHaveBeenCalledWith(
                'https://example.com/webhook',
                expect.objectContaining({ method: 'POST' })
            )

            // Verify metrics were produced to Kafka
            await waitForExpect(() => {
                const metrics = mockProducerObserver
                    .getProducedKafkaMessagesForTopic(KAFKA_APP_METRICS_2)
                    .filter((m: any) => m.value.app_source === 'hog_flow')
                expect(metrics.length).toBeGreaterThanOrEqual(1)
            }, 5000)

            // Verify logs were produced to Kafka
            await waitForExpect(() => {
                const logs = mockProducerObserver.getProducedKafkaMessagesForTopic(KAFKA_LOG_ENTRIES)
                expect(logs.length).toBeGreaterThanOrEqual(1)
            }, 5000)
        })

        it('mirrors every metric of the run under the workflow version that produced it', async () => {
            await triggerWorkflow(globals)

            const namesFor = (appSource: string, appSourceId: string): string[] =>
                mockProducerObserver
                    .getProducedKafkaMessagesForTopic(KAFKA_APP_METRICS_2)
                    .filter((m: any) => m.value.app_source === appSource && m.value.app_source_id === appSourceId)
                    .map((m: any) => m.value.metric_name)
                    .sort()

            // The version-scoped series has to carry the whole run, not just the run-level outcome —
            // a per-version step funnel is only readable if every step's metric is mirrored.
            await waitForExpect(() => {
                expect(namesFor('hog_flow', workflowId)).toContain('succeeded')
                expect(namesFor('hog_flow_version', `${workflowId}/1`)).toEqual(namesFor('hog_flow', workflowId))
            }, 10000)
        })
    })

    describe('workflow with a secret function input: trigger → function → exit', () => {
        const secretApiKey = 'workflows-e2e-secret-api-key-0000'
        let workflowId: string

        beforeEach(async () => {
            await insertHogFunctionTemplate(hub.postgres, {
                id: 'template-workflows-e2e-secret-fetch',
                name: 'Workflows E2E Secret Fetch',
                code: `
                let res := fetch(inputs.url, {'method': 'POST', 'headers': {'Authorization': f'Bearer {inputs.api_key}'}});
                print('Fetch result:', res.status);
                `,
                inputs_schema: [
                    { key: 'url', type: 'string', required: true },
                    { key: 'api_key', type: 'string', required: true, secret: true },
                ],
            })

            workflowId = await createWorkflow({
                actions: {
                    trigger: trigger(),
                    send_with_secret: {
                        type: 'function',
                        config: {
                            template_id: 'template-workflows-e2e-secret-fetch',
                            inputs: { url: { value: 'https://example.com/secret-webhook' } },
                        },
                    },
                    exit: exitAction(),
                },
                edges: [
                    { from: 'trigger', to: 'send_with_secret', type: 'continue' },
                    { from: 'send_with_secret', to: 'exit', type: 'continue' },
                ],
            })

            await setWorkflowEncryptedInputs(workflowId, { send_with_secret: { api_key: { value: secretApiKey } } })
            globals = createGlobals()
        })

        it('sends the decrypted secret and keeps it out of logs and stored jobs', async () => {
            await triggerWorkflow(globals)

            await waitForExpect(() => {
                expect(mockFetch).toHaveBeenCalledTimes(1)
            }, 10000)
            expect(mockFetch).toHaveBeenCalledWith(
                'https://example.com/secret-webhook',
                expect.objectContaining({
                    method: 'POST',
                    headers: expect.objectContaining({ Authorization: `Bearer ${secretApiKey}` }),
                })
            )

            await waitForExpect(() => {
                const logs = mockProducerObserver
                    .getProducedKafkaMessagesForTopic(KAFKA_LOG_ENTRIES)
                    .filter((m: any) => m.value.log_source_id === workflowId)
                expect(logs.map((m: any) => m.value.message)).toContainEqual(
                    expect.stringContaining('Fetch result:, 200')
                )
                expect(JSON.stringify(logs)).not.toContain(secretApiKey)
            }, 5000)

            expect(JSON.stringify(await queryCyclotronJobs())).not.toContain(secretApiKey)
        })
    })

    describe('workflow with webhook secret headers: trigger → webhook → exit', () => {
        const token = 'workflows-e2e-secret-headers-token-0000'
        const apiKey = 'workflows-e2e-secret-headers-api-key-0000'
        const webhookUrl = 'https://example.com/secret-headers-webhook'
        let workflowId: string
        let releaseRetry: () => void = () => {}

        const fetchResponse = (status: number) => ({
            status,
            json: () => Promise.resolve({}),
            text: () => Promise.resolve('{}'),
            headers: { 'Content-Type': 'application/json' },
            dump: () => Promise.resolve(),
        })

        beforeEach(async () => {
            await insertHogFunctionTemplate(hub.postgres, {
                id: 'template-webhook',
                name: webhookTemplate.name,
                code: webhookTemplate.code,
                inputs_schema: webhookTemplate.inputs_schema,
            })

            workflowId = await createWorkflow({
                actions: {
                    trigger: trigger(),
                    send_webhook: {
                        type: 'function',
                        config: {
                            template_id: 'template-webhook',
                            inputs: {
                                url: { value: webhookUrl },
                                method: { value: 'POST' },
                                body: { value: { event: '{event.event}' } },
                                // Same name as a secret header in another case, so the merge has to
                                // replace it instead of sending both.
                                headers: {
                                    value: {
                                        'Content-Type': 'application/json',
                                        authorization: 'plaintext-authorization',
                                    },
                                },
                                debug: { value: true },
                            },
                        },
                    },
                    exit: exitAction(),
                },
                edges: [
                    { from: 'trigger', to: 'send_webhook', type: 'continue' },
                    { from: 'send_webhook', to: 'exit', type: 'continue' },
                ],
            })
            await setWorkflowEncryptedInputs(workflowId, {
                send_webhook: { secret_headers: { value: { Authorization: `Bearer ${token}`, 'X-Api-Key': apiKey } } },
            })

            // The first attempt fails so the fetch parameters are written to the job row for the retry.
            // The retry is then held open until the test releases it, so the row can be read while it
            // still holds the state the worker resumed from.
            let attempts = 0
            const retryGate = new Promise<void>((resolve) => {
                releaseRetry = resolve
            })
            mockFetch.mockImplementation(async () => {
                attempts += 1
                if (attempts > 1) {
                    await retryGate
                }
                return fetchResponse(attempts === 1 ? 500 : 200)
            })
            globals = createGlobals()
        })

        it('merges the decrypted secret headers on both attempts and keeps them off the fetch parameters', async () => {
            try {
                await triggerWorkflow(globals)

                await waitForExpect(() => {
                    expect(mockFetch).toHaveBeenCalledTimes(2)
                }, 10000)

                // Only `queueParameters` is asserted on. `state.globals.inputs` carries every resolved input,
                // which is a property of the whole job payload and not of this template.
                const blobs = (await queryCyclotronJobs()).map((row: any) =>
                    parseJSON((row.state as Buffer).toString('utf-8'))
                )
                const fetchBlob = blobs.find((blob: any) => blob.queueParameters?.type === 'fetch')
                expect(fetchBlob).toBeDefined()
                expect(fetchBlob.queueParameters).toMatchObject({
                    secret_headers_input: 'secret_headers',
                    headers: { 'Content-Type': 'application/json', authorization: 'plaintext-authorization' },
                })
                expect(JSON.stringify(fetchBlob.queueParameters)).not.toContain(token)
                expect(JSON.stringify(fetchBlob.queueParameters)).not.toContain(apiKey)
            } finally {
                releaseRetry()
            }

            for (const [url, options] of mockFetch.mock.calls as [string, any][]) {
                expect(url).toEqual(webhookUrl)
                expect(options.headers).toMatchObject({
                    Authorization: `Bearer ${token}`,
                    'X-Api-Key': apiKey,
                    'Content-Type': 'application/json',
                })
                expect(Object.keys(options.headers).filter((name) => name.toLowerCase() === 'authorization')).toEqual([
                    'Authorization',
                ])
            }

            await waitForExpect(() => {
                const messages = mockProducerObserver
                    .getProducedKafkaMessagesForTopic(KAFKA_LOG_ENTRIES)
                    .filter((m: any) => m.value.log_source_id === workflowId)
                    .map((m: any) => m.value.message as string)
                expect(messages).toContainEqual(
                    expect.stringContaining('HTTP fetch failed on attempt 1 with status code 500. Retrying.')
                )
                expect(messages).toContainEqual(expect.stringContaining('Response, 200'))
                const requestLogs = messages.filter((message) => message.includes('Request, '))
                expect(requestLogs).toHaveLength(1)
                expect(requestLogs[0]).toContain('"secret_headers_input":"secret_headers"')
                // Log redaction masks values but not header names. The name is what appears if the
                // template merges the secret headers into `payload.headers` instead of passing the reference.
                expect(requestLogs[0]).not.toContain('X-Api-Key')
                expect(JSON.stringify(messages)).not.toContain(token)
            }, 5000)
        })
    })

    describe('delay workflow: trigger → delay → function → exit', () => {
        beforeEach(async () => {
            await createWorkflow({
                actions: {
                    trigger: trigger(),
                    delay_1: delayAction('0.5s'),
                    function_1: fetchAction('https://example.com/delayed-webhook'),
                    exit: exitAction(),
                },
                edges: [
                    { from: 'trigger', to: 'delay_1', type: 'continue' },
                    { from: 'delay_1', to: 'function_1', type: 'continue' },
                    { from: 'function_1', to: 'exit', type: 'continue' },
                ],
            })
            globals = createGlobals()
        })

        it('should reschedule on delay and execute function after delay passes', async () => {
            await triggerWorkflow(globals)

            // First: worker picks up job and hits the delay — job gets rescheduled
            await waitForExpect(async () => {
                const jobs = await queryCyclotronJobs()
                const rescheduled = jobs.filter(
                    (j: any) => j.status === 'available' && new Date(j.scheduled) > new Date()
                )
                expect(rescheduled.length).toBe(1)
            }, 5000)

            // Fetch should NOT have been called yet (delay hasn't passed)
            expect(mockFetch).not.toHaveBeenCalled()

            // Wait for the delay to pass and the worker to pick up the job again
            await waitForExpect(() => {
                expect(mockFetch).toHaveBeenCalledTimes(1)
            }, 10000)

            expect(mockFetch).toHaveBeenCalledWith(
                'https://example.com/delayed-webhook',
                expect.objectContaining({ method: 'POST' })
            )

            // Verify the job completed (transition_count > 1 due to reschedule)
            await waitForExpect(async () => {
                const jobs = await queryCyclotronJobs()
                const terminal = jobs.filter(
                    (j: any) => j.status === 'completed' || j.status === 'failed' || j.status === 'canceled'
                )
                expect(terminal.length).toBeGreaterThanOrEqual(1)
            }, 5000)
        })
    })

    describe('delay until a date carried by the data', () => {
        const workflowWaitingUntil = async (offset?: string): Promise<void> => {
            await createWorkflow({
                actions: {
                    trigger: trigger(),
                    delay_1: delayUntilAction(offset),
                    function_1: fetchAction('https://example.com/reminder'),
                    exit: exitAction(),
                },
                edges: [
                    { from: 'trigger', to: 'delay_1', type: 'continue' },
                    { from: 'delay_1', to: 'function_1', type: 'continue' },
                    { from: 'function_1', to: 'exit', type: 'continue' },
                ],
            })
        }

        it('parks until the date on the event, then continues', async () => {
            // A fixed duration cannot express this: the instant comes from the payload, so two runs of the
            // same workflow park to different times.
            const expiresAt = DateTime.utc().plus({ seconds: 0.5 })
            await workflowWaitingUntil()
            await triggerWorkflow(createGlobals({ properties: { expires_at: expiresAt.toISO() } } as any))
            await waitForExpect(async () => {
                const parked = (await queryCyclotronJobs()).filter(
                    (j: any) => j[statusColumn] === 'available' && new Date(j.scheduled) > new Date()
                )
                expect(parked).toHaveLength(1)
                // Parked to the instant from the data, not to some default span.
                const scheduled = DateTime.fromJSDate(new Date(parked[0].scheduled)).toUTC()
                expect(Math.abs(scheduled.diff(expiresAt).as('seconds'))).toBeLessThan(2)
            }, 5000)

            expect(mockFetch).not.toHaveBeenCalled()

            await waitForExpect(() => {
                expect(mockFetch).toHaveBeenCalledTimes(1)
            }, 15000)
            expect(mockFetch).toHaveBeenCalledWith('https://example.com/reminder', expect.anything())
        })

        it('continues straight away when the date has already passed', async () => {
            // The guard against a reminder for something that already happened firing days late.
            await workflowWaitingUntil()
            await triggerWorkflow(
                createGlobals({ properties: { expires_at: DateTime.utc().minus({ days: 5 }).toISO() } } as any)
            )

            await waitForExpect(() => {
                expect(mockFetch).toHaveBeenCalledTimes(1)
            }, 10000)
        })

        it('fires before the date when an offset asks for it', async () => {
            // The shape a "remind me N days before" workflow needs, and the reason an offset exists rather
            // than only a bare date: here the date is 1 hour out and the offset pulls the wait to now.
            await workflowWaitingUntil('-1h')
            await triggerWorkflow(
                createGlobals({ properties: { expires_at: DateTime.utc().plus({ hours: 1 }).toISO() } } as any)
            )

            await waitForExpect(() => {
                expect(mockFetch).toHaveBeenCalledTimes(1)
            }, 10000)
        })

        it('aborts instead of sending when the date cannot be worked out', async () => {
            // The outcome that matters: a "before it expires" message must not go out for someone with no
            // expiry. on_error defaults to 'continue', which would do exactly that, so an unresolvable date
            // aborts the run regardless of that setting.
            await workflowWaitingUntil()
            await triggerWorkflow(createGlobals({ properties: {} } as any))

            // Waiting for a terminal state, not merely "not available": a job being worked on is also not
            // available, so the looser check can pass in the window before the run would have sent.
            await waitForExpect(async () => {
                const jobs = await queryCyclotronJobs()
                expect(jobs.filter((j: any) => j[statusColumn] === 'failed')).toHaveLength(1)
            }, 10000)
            expect(mockFetch).not.toHaveBeenCalled()
        })
    })

    describe('conditional branch workflow', () => {
        beforeEach(async () => {
            await createWorkflow({
                actions: {
                    trigger: trigger(),
                    branch: {
                        type: 'conditional_branch',
                        config: {
                            conditions: [
                                { filters: HOG_FILTERS_EXAMPLES.pageview_or_autocapture_filter.filters },
                                { filters: HOG_FILTERS_EXAMPLES.elements_text_filter.filters },
                            ],
                        },
                    },
                    function_a: fetchAction('https://example.com/branch-a'),
                    function_b: fetchAction('https://example.com/branch-b'),
                    exit: exitAction(),
                },
                edges: [
                    { from: 'trigger', to: 'branch', type: 'continue' },
                    { from: 'branch', to: 'function_a', type: 'branch', index: 0 },
                    { from: 'branch', to: 'function_b', type: 'branch', index: 1 },
                    { from: 'function_a', to: 'exit', type: 'continue' },
                    { from: 'function_b', to: 'exit', type: 'continue' },
                ],
            })
        })

        it('should take branch A when event matches the first condition', async () => {
            globals = createGlobals({
                event: '$pageview',
                properties: { $current_url: 'https://posthog.com/pricing' },
            })
            await triggerWorkflow(globals)

            await waitForExpect(() => {
                expect(mockFetch).toHaveBeenCalledTimes(1)
            }, 10000)

            expect(mockFetch).toHaveBeenCalledWith('https://example.com/branch-a', expect.anything())
        })
    })

    describe('cohort condition workflow', () => {
        const COHORT_ID = 4242
        const PERSON_UUID = 'a5f478a1-1d34-4c8a-a0b5-6a3b62e3f8a1'

        // What the Django serializer compiles for a cohort condition filter; the shape is pinned
        // from the Python side by test_hog_flow_conditional_branch_cohort_filter_compiles
        const cohortConditionFilters = (cohortId: number): Record<string, any> => ({
            bytecode: ['_H', 1, 33, cohortId, 32, 'cohort_ids', 1, 1, 2, 'inCohort', 2],
            properties: [{ key: 'id', type: 'cohort', value: cohortId }],
        })

        beforeEach(async () => {
            await resetBehavioralCohortsDatabase(hub.postgres)
            // The person's uuid is the key the cohort_membership lookup runs against
            mockPersonRepo.fetchPersonsByDistinctIds.mockResolvedValue([
                {
                    id: '1',
                    uuid: PERSON_UUID,
                    team_id: team.id,
                    properties: {},
                    properties_last_updated_at: {},
                    properties_last_operation: null,
                    created_at: DateTime.utc(),
                    version: 1,
                    is_identified: true,
                    is_user_id: null,
                    last_seen_at: null,
                    distinct_id: 'distinct_id',
                },
            ])
            await createWorkflow({
                actions: {
                    trigger: trigger(),
                    branch: {
                        type: 'conditional_branch',
                        config: {
                            conditions: [{ filters: cohortConditionFilters(COHORT_ID) }],
                        },
                    },
                    function_member: fetchAction('https://example.com/in-cohort'),
                    function_non_member: fetchAction('https://example.com/not-in-cohort'),
                    exit: exitAction(),
                },
                edges: [
                    { from: 'trigger', to: 'branch', type: 'continue' },
                    { from: 'branch', to: 'function_member', type: 'branch', index: 0 },
                    { from: 'branch', to: 'function_non_member', type: 'continue' },
                    { from: 'function_member', to: 'exit', type: 'continue' },
                    { from: 'function_non_member', to: 'exit', type: 'continue' },
                ],
            })
            globals = createGlobals()
        })

        it.each([
            ['a membership row', true, 'https://example.com/in-cohort'],
            ['an in_cohort=false row (person left)', false, 'https://example.com/not-in-cohort'],
        ])('routes a person with %s down the right branch', async (_name, inCohort, expectedUrl) => {
            await insertCohortMembership(hub.postgres, {
                team_id: team.id,
                cohort_id: COHORT_ID,
                person_id: PERSON_UUID,
                in_cohort: inCohort,
            })

            await triggerWorkflow(globals)

            await waitForExpect(() => {
                expect(mockFetch).toHaveBeenCalledTimes(1)
            }, 10000)
            expect(mockFetch).toHaveBeenCalledWith(expectedUrl, expect.anything())
        })

        it('routes a person the pipeline never wrote a row for as a non-member', async () => {
            await triggerWorkflow(globals)

            await waitForExpect(() => {
                expect(mockFetch).toHaveBeenCalledTimes(1)
            }, 10000)
            expect(mockFetch).toHaveBeenCalledWith('https://example.com/not-in-cohort', expect.anything())
        })
    })

    describe('workflow disabled mid-execution', () => {
        let hogFlowId: string

        beforeEach(async () => {
            hogFlowId = await createWorkflow({
                actions: {
                    trigger: trigger(),
                    delay_1: delayAction('0.5s'),
                    function_1: fetchAction('https://example.com/should-not-fire'),
                    exit: exitAction(),
                },
                edges: [
                    { from: 'trigger', to: 'delay_1', type: 'continue' },
                    { from: 'delay_1', to: 'function_1', type: 'continue' },
                    { from: 'function_1', to: 'exit', type: 'continue' },
                ],
            })
            globals = createGlobals()
        })

        it('should cancel the job when workflow is archived during delay', async () => {
            await triggerWorkflow(globals)

            // Wait for the delay step to be hit (job rescheduled)
            await waitForExpect(async () => {
                const jobs = await queryCyclotronJobs()
                expect(jobs.some((j: any) => j.status === 'available' && new Date(j.scheduled) > new Date())).toBe(true)
            }, 5000)

            // Archive the hogflow while job is waiting
            await hub.postgres.query(
                PostgresUse.COMMON_WRITE,
                `UPDATE posthog_hogflow SET status = 'archived' WHERE id = $1`,
                [hogFlowId],
                'archiveHogFlow'
            )

            // Force the hogflow manager to reload
            ;(hogflowWorker as any).hogFlowManager.lazyLoader.markForRefresh(hogFlowId)

            // Wait for the delayed job to be picked up and canceled
            await waitForExpect(async () => {
                const jobs = await queryCyclotronJobs()
                const canceled = jobs.filter((j: any) => j.status === 'canceled')
                expect(canceled.length).toBe(1)
            }, 10000)

            // Function should NOT have been called
            expect(mockFetch).not.toHaveBeenCalled()

            // The run must terminate through the result pipeline, not a silent queue flip:
            // a 'canceled' metric lands and the run is not counted as 'succeeded'.
            await waitForExpect(() => {
                const metricNames = mockProducerObserver
                    .getProducedKafkaMessagesForTopic(KAFKA_APP_METRICS_2)
                    .filter((m: any) => m.value.app_source === 'hog_flow' && m.value.app_source_id === hogFlowId)
                    .map((m: any) => m.value.metric_name)
                expect(metricNames).toContain('canceled')
                expect(metricNames).not.toContain('succeeded')
            }, 5000)
        })
    })

    describe('live edit while a run is parked (follow-live contract)', () => {
        // The runs park on a 2m delay so they cannot wake before the edit lands; the tests then
        // pull the wake forward explicitly (the same scheduled-time update the subscription
        // matcher performs), so there is no race between the park deadline and the edit.

        /** Wait until a job is parked in the future (a delay/wait step was hit) */
        async function waitForParkedJob(): Promise<void> {
            await waitForExpect(async () => {
                const jobs = await queryCyclotronJobs()
                expect(jobs.some((j: any) => j.status === 'available' && new Date(j.scheduled) > new Date())).toBe(true)
            }, 5000)
        }

        /** Persist an edited actions/edges graph and bust the worker's config cache, like Django's save + pub/sub would */
        async function applyLiveEdit(flow: HogFlow): Promise<void> {
            await hub.postgres.query(
                PostgresUse.COMMON_WRITE,
                `UPDATE posthog_hogflow SET actions = $2, edges = $3, updated_at = NOW() WHERE id = $1`,
                [flow.id, JSON.stringify(flow.actions), JSON.stringify(flow.edges)],
                'liveEditHogFlow'
            )
            ;(hogflowWorker as any).hogFlowManager.lazyLoader.markForRefresh(flow.id)
        }

        /** Pull parked jobs' scheduled time forward so the worker picks them up now */
        async function wakeParkedJobsNow(): Promise<void> {
            await cyclotronPool.query(
                `UPDATE cyclotron_jobs SET scheduled = NOW() WHERE ${statusColumn} = 'available' AND scheduled > NOW()`
            )
        }

        /** Shorten the flow's delay so the woken run advances instead of re-parking */
        function shortenDelay(flow: HogFlow, actionId: string): void {
            const delay = flow.actions.find((a) => a.id === actionId)!
            ;(delay.config as any).delay_duration = '0.5s'
        }

        it('a content edit made while parked on an upstream delay is honored on wake', async () => {
            const flow = await createWorkflowFlow({
                actions: {
                    trigger: trigger(),
                    delay_1: delayAction('2m'),
                    function_1: fetchAction('https://example.com/content-v1'),
                    exit: exitAction(),
                },
                edges: [
                    { from: 'trigger', to: 'delay_1', type: 'continue' },
                    { from: 'delay_1', to: 'function_1', type: 'continue' },
                    { from: 'function_1', to: 'exit', type: 'continue' },
                ],
            })
            await triggerWorkflow(createGlobals())
            await waitForParkedJob()

            const functionAction = flow.actions.find((a) => a.id === 'function_1')!
            ;(functionAction.config as any).inputs.url.value = 'https://example.com/content-v2'
            shortenDelay(flow, 'delay_1')
            await applyLiveEdit(flow)
            await wakeParkedJobsNow()

            await waitForExpect(() => {
                expect(mockFetch).toHaveBeenCalledTimes(1)
            }, 10000)
            expect(mockFetch).toHaveBeenCalledWith('https://example.com/content-v2', expect.anything())
        })

        it("rerouting the parked step's edge does not re-run earlier steps or run the old target", async () => {
            const flow = await createWorkflowFlow({
                actions: {
                    trigger: trigger(),
                    function_a: fetchAction('https://example.com/step-a'),
                    delay_1: delayAction('2m'),
                    function_b: fetchAction('https://example.com/step-b'),
                    exit: exitAction(),
                },
                edges: [
                    { from: 'trigger', to: 'function_a', type: 'continue' },
                    { from: 'function_a', to: 'delay_1', type: 'continue' },
                    { from: 'delay_1', to: 'function_b', type: 'continue' },
                    { from: 'function_b', to: 'exit', type: 'continue' },
                ],
            })
            await triggerWorkflow(createGlobals())
            await waitForExpect(() => {
                expect(mockFetch).toHaveBeenCalledWith('https://example.com/step-a', expect.anything())
            }, 10000)
            await waitForParkedJob()

            // Reroute the delay's continue edge onto a newly added step
            const functionB = flow.actions.find((a) => a.id === 'function_b')!
            flow.actions.push({
                ...functionB,
                id: 'function_c',
                config: {
                    ...(functionB.config as any),
                    inputs: { url: { value: 'https://example.com/step-c' }, method: { value: 'POST' } },
                },
            })
            flow.edges = [
                { from: 'trigger', to: 'function_a', type: 'continue' },
                { from: 'function_a', to: 'delay_1', type: 'continue' },
                { from: 'delay_1', to: 'function_c', type: 'continue' },
                { from: 'function_c', to: 'exit', type: 'continue' },
            ]
            shortenDelay(flow, 'delay_1')
            await applyLiveEdit(flow)
            await wakeParkedJobsNow()

            await waitForExpect(() => {
                expect(mockFetch).toHaveBeenCalledWith('https://example.com/step-c', expect.anything())
            }, 10000)

            // Exactly one send per executed step: the step behind the run did not re-run, the old
            // target never ran, the new target ran once
            const urls = mockFetch.mock.calls.map((call) => call[0])
            expect(urls.filter((u) => u === 'https://example.com/step-a')).toHaveLength(1)
            expect(urls.filter((u) => u === 'https://example.com/step-b')).toHaveLength(0)
            expect(urls.filter((u) => u === 'https://example.com/step-c')).toHaveLength(1)
        })

        it('deleting the step a run is parked on exits the run gracefully on wake', async () => {
            const flow = await createWorkflowFlow({
                actions: {
                    trigger: trigger(),
                    delay_1: delayAction('2m'),
                    function_1: fetchAction('https://example.com/should-not-fire'),
                    exit: exitAction(),
                },
                edges: [
                    { from: 'trigger', to: 'delay_1', type: 'continue' },
                    { from: 'delay_1', to: 'function_1', type: 'continue' },
                    { from: 'function_1', to: 'exit', type: 'continue' },
                ],
            })
            await triggerWorkflow(createGlobals())
            await waitForParkedJob()

            flow.actions = flow.actions.filter((a) => a.id !== 'delay_1')
            flow.edges = [
                { from: 'trigger', to: 'function_1', type: 'continue' },
                { from: 'function_1', to: 'exit', type: 'continue' },
            ]
            await applyLiveEdit(flow)
            await wakeParkedJobsNow()

            // The parked run wakes, finds its step gone, and finishes as a deliberate exit
            await waitForExpect(async () => {
                const jobs = await queryCyclotronJobs()
                expect(jobs.filter((j: any) => j.status === 'completed')).toHaveLength(1)
            }, 10000)
            expect(mockFetch).not.toHaveBeenCalled()

            await waitForExpect(() => {
                const metricNames = mockProducerObserver
                    .getProducedKafkaMessagesForTopic(KAFKA_APP_METRICS_2)
                    .filter((m: any) => m.value.app_source === 'hog_flow')
                    .map((m: any) => m.value.metric_name)
                expect(metricNames).toContain('exited_workflow_changed')
                expect(metricNames).not.toContain('failed')
            }, 5000)
        })
    })

    describe('multiple workflows matching same event', () => {
        const simpleFetchWorkflow = (url: string) => ({
            actions: {
                trigger: trigger(),
                function_1: fetchAction(url),
                exit: exitAction(),
            },
            edges: [
                { from: 'trigger', to: 'function_1', type: 'continue' as const },
                { from: 'function_1', to: 'exit', type: 'continue' as const },
            ],
        })

        beforeEach(async () => {
            await createWorkflow(simpleFetchWorkflow('https://example.com/workflow-a'), { name: 'Workflow A' })
            await createWorkflow(simpleFetchWorkflow('https://example.com/workflow-b'), { name: 'Workflow B' })
            globals = createGlobals()
        })

        it('should execute both workflows independently', async () => {
            await triggerWorkflow(globals)

            await waitForExpect(() => {
                expect(mockFetch).toHaveBeenCalledTimes(2)
            }, 10000)

            const urls = mockFetch.mock.calls.map((call) => call[0])
            expect(urls).toContain('https://example.com/workflow-a')
            expect(urls).toContain('https://example.com/workflow-b')
        })
    })

    describe('wait_until_condition: condition matches immediately', () => {
        beforeEach(async () => {
            await createWorkflow({
                actions: {
                    trigger: trigger(),
                    wait_condition: {
                        type: 'wait_until_condition',
                        config: {
                            condition: {
                                // Matches $pageview with "posthog" in $current_url
                                filters: HOG_FILTERS_EXAMPLES.pageview_or_autocapture_filter.filters,
                            },
                            max_wait_duration: '10s',
                        },
                    },
                    function_matched: fetchAction('https://example.com/condition-matched'),
                    function_timeout: fetchAction('https://example.com/condition-timed-out'),
                    exit: exitAction(),
                },
                edges: [
                    { from: 'trigger', to: 'wait_condition', type: 'continue' },
                    { from: 'wait_condition', to: 'function_matched', type: 'branch', index: 0 },
                    { from: 'wait_condition', to: 'function_timeout', type: 'continue' },
                    { from: 'function_matched', to: 'exit', type: 'continue' },
                    { from: 'function_timeout', to: 'exit', type: 'continue' },
                ],
            })
            // Event matches the condition: $pageview with posthog in URL
            globals = createGlobals({
                event: '$pageview',
                properties: { $current_url: 'https://posthog.com' },
            })
        })

        it('should take the matched branch without rescheduling', async () => {
            await triggerWorkflow(globals)

            await waitForExpect(() => {
                expect(mockFetch).toHaveBeenCalledTimes(1)
            }, 10000)

            // Should hit the matched branch, not the timeout branch
            expect(mockFetch).toHaveBeenCalledWith('https://example.com/condition-matched', expect.anything())
        })
    })

    describe('wait_until_condition: condition never matches, times out', () => {
        beforeEach(async () => {
            await createWorkflow({
                actions: {
                    trigger: trigger(),
                    wait_condition: {
                        type: 'wait_until_condition',
                        config: {
                            condition: {
                                // Requires $autocapture with "reload" in elements_chain_texts — won't match
                                filters: HOG_FILTERS_EXAMPLES.elements_text_filter.filters,
                            },
                            max_wait_duration: '0.5s',
                        },
                    },
                    function_1: fetchAction('https://example.com/after-wait-timeout'),
                    exit: exitAction(),
                },
                edges: [
                    { from: 'trigger', to: 'wait_condition', type: 'continue' },
                    { from: 'wait_condition', to: 'exit', type: 'branch', index: 0 },
                    { from: 'wait_condition', to: 'function_1', type: 'continue' },
                    { from: 'function_1', to: 'exit', type: 'continue' },
                ],
            })
            globals = createGlobals()
        })

        it('should reschedule while polling, then continue after max_wait expires', async () => {
            await triggerWorkflow(globals)

            // Job should be rescheduled (condition doesn't match, waiting for next poll)
            await waitForExpect(async () => {
                const jobs = await queryCyclotronJobs()
                const rescheduled = jobs.filter(
                    (j: any) => j.status === 'available' && new Date(j.scheduled) > new Date()
                )
                expect(rescheduled.length).toBe(1)
            }, 5000)

            // Fetch should NOT be called yet — still waiting for condition
            expect(mockFetch).not.toHaveBeenCalled()

            // After max_wait expires, the condition times out and the workflow
            // continues to the function action via the continue edge
            await waitForExpect(() => {
                expect(mockFetch).toHaveBeenCalledTimes(1)
            }, 10000)

            expect(mockFetch).toHaveBeenCalledWith('https://example.com/after-wait-timeout', expect.anything())
        })
    })

    describe('wait_until_condition: subscription matcher wakes parked jobs', () => {
        let matcher: CdpHogflowSubscriptionMatcherConsumer

        // trigger → wait_condition → (matched branch | timeout continue) → exit
        const createWaitUntilWorkflow = (waitConfig: Record<string, any>): Promise<string> =>
            createWorkflow({
                actions: {
                    trigger: trigger(),
                    wait_condition: { type: 'wait_until_condition', config: waitConfig },
                    function_matched: fetchAction('https://example.com/condition-matched'),
                    function_timeout: fetchAction('https://example.com/timed-out'),
                    exit: exitAction(),
                },
                edges: [
                    { from: 'trigger', to: 'wait_condition', type: 'continue' },
                    { from: 'wait_condition', to: 'function_matched', type: 'branch', index: 0 },
                    { from: 'wait_condition', to: 'function_timeout', type: 'continue' },
                    { from: 'function_matched', to: 'exit', type: 'continue' },
                    { from: 'function_timeout', to: 'exit', type: 'continue' },
                ],
            })

        // The job is parked when it is available with a scheduled time in the future.
        const expectParked = async (): Promise<void> => {
            await waitForExpect(async () => {
                const jobs = await queryCyclotronJobs()
                expect(jobs.some((j: any) => j.status === 'available' && new Date(j.scheduled) > new Date())).toBe(true)
            }, 5000)
            expect(mockFetch).not.toHaveBeenCalled()
        }

        beforeEach(() => {
            matcher = new CdpHogflowSubscriptionMatcherConsumer({ ...hub }, deps)
        })

        afterEach(async () => {
            await matcher?.stop().catch(() => {})
        })

        it('wakes a parked job and takes the matched branch when a subscribed event fires', async () => {
            await createWaitUntilWorkflow({
                // Property condition never matches the trigger event, so the job parks.
                condition: { filters: HOG_FILTERS_EXAMPLES.elements_text_filter.filters },
                events: [eventNameFilter('wakeup_event')],
                max_wait_duration: '5m',
            })
            await triggerWorkflow(createGlobals())
            await expectParked()

            // A subscribed event fires for this person — the matcher wakes the job.
            await matcher.processBatch([createGlobals({ event: 'wakeup_event' })])

            await waitForExpect(() => {
                expect(mockFetch).toHaveBeenCalledTimes(1)
            }, 10000)
            expect(mockFetch).toHaveBeenCalledWith('https://example.com/condition-matched', expect.anything())
        })

        it('wakes a parked job whose wait entry is action-based (events empty, actions + bytecode set)', async () => {
            await createWaitUntilWorkflow({
                // Property condition never matches the trigger event, so the job parks.
                condition: { filters: HOG_FILTERS_EXAMPLES.elements_text_filter.filters },
                // "Events to wait for" entry targets a PostHog Action: filters.events is empty,
                // filters.actions is set, and the compiled bytecode matches the action's event.
                events: [actionFilter('action_wakeup_event', 3)],
                max_wait_duration: '5m',
            })
            await triggerWorkflow(createGlobals())
            await expectParked()

            // The action's underlying event fires — the matcher must wake the job via the action
            // entry even though filters.events is empty.
            await matcher.processBatch([createGlobals({ event: 'action_wakeup_event' })])

            await waitForExpect(() => {
                expect(mockFetch).toHaveBeenCalledTimes(1)
            }, 10000)
            expect(mockFetch).toHaveBeenCalledWith('https://example.com/condition-matched', expect.anything())
        })

        it('wakes a parked job when a later event satisfies the property condition', async () => {
            await createWaitUntilWorkflow({
                // No events list — only a property-based condition. The matcher evaluates the
                // condition against every incoming event, making property waits event-driven.
                condition: { filters: HOG_FILTERS_EXAMPLES.pageview_or_autocapture_filter.filters },
                max_wait_duration: '5m',
            })
            // Trigger with an event that does not satisfy the condition, so the job parks.
            await triggerWorkflow(createGlobals({ event: 'custom_trigger', properties: {} }))
            await expectParked()

            // A later $pageview with a posthog URL satisfies the property condition.
            await matcher.processBatch([
                createGlobals({ event: '$pageview', properties: { $current_url: 'https://posthog.com' } }),
            ])

            await waitForExpect(() => {
                expect(mockFetch).toHaveBeenCalledTimes(1)
            }, 10000)
            expect(mockFetch).toHaveBeenCalledWith('https://example.com/condition-matched', expect.anything())
        })

        it('takes the timeout branch when neither events nor the condition match before max_wait', async () => {
            await createWaitUntilWorkflow({
                condition: { filters: HOG_FILTERS_EXAMPLES.elements_text_filter.filters },
                events: [eventNameFilter('never_fires')],
                max_wait_duration: '0.5s',
            })
            await triggerWorkflow(createGlobals())

            // An unrelated event passes through the matcher but must not wake the job.
            await matcher.processBatch([createGlobals({ event: 'some_other_event' })])

            // After max_wait expires the job advances down the continue (timeout) branch.
            await waitForExpect(() => {
                expect(mockFetch).toHaveBeenCalledTimes(1)
            }, 15000)
            expect(mockFetch).toHaveBeenCalledWith('https://example.com/timed-out', expect.anything())
        })

        it('leaves the job parked when an event matches neither the events nor the condition', async () => {
            await createWaitUntilWorkflow({
                condition: { filters: HOG_FILTERS_EXAMPLES.elements_text_filter.filters },
                events: [eventNameFilter('wakeup_event')],
                max_wait_duration: '5m',
            })
            await triggerWorkflow(createGlobals())
            await expectParked()

            // An unrelated event must not wake the job.
            await matcher.processBatch([createGlobals({ event: 'some_other_event' })])

            const jobs = await queryCyclotronJobs()
            expect(jobs.every((j: any) => j.status === 'available' && new Date(j.scheduled) > new Date())).toBe(true)
            expect(mockFetch).not.toHaveBeenCalled()
        })

        it('does not wake on a completely empty "events to wait for" entry (always-true bytecode)', async () => {
            // The state left when the last event is removed from a wait entry: empty filters compile
            // to always-true bytecode. Inserted directly so it bypasses the serializer strip — this
            // guards the matcher itself, which must NOT fire the workflow on every incoming event.
            await createWaitUntilWorkflow({
                condition: { filters: HOG_FILTERS_EXAMPLES.elements_text_filter.filters },
                events: [emptyEventFilter()],
                max_wait_duration: '5m',
            })
            await triggerWorkflow(createGlobals())
            await expectParked()

            // An unrelated event must not wake the job — the always-true bytecode would otherwise match.
            await matcher.processBatch([createGlobals({ event: 'some_unrelated_event' })])

            const jobs = await queryCyclotronJobs()
            expect(jobs.every((j: any) => j.status === 'available' && new Date(j.scheduled) > new Date())).toBe(true)
            expect(mockFetch).not.toHaveBeenCalled()
        })

        it('does not fire on entry for an empty property condition; takes the timeout branch', async () => {
            // An empty condition compiles to always-true bytecode. The executor evaluates the
            // condition on entry, so without the guard the wait advances down the matched branch
            // immediately. With no events and no real condition it must park and time out instead.
            await createWaitUntilWorkflow({
                condition: { filters: emptyConditionFilters() },
                max_wait_duration: '0.5s',
            })
            await triggerWorkflow(createGlobals())

            // If it fired on entry this would be the matched branch; it must be the timeout branch.
            await waitForExpect(() => {
                expect(mockFetch).toHaveBeenCalledTimes(1)
            }, 15000)
            expect(mockFetch).toHaveBeenCalledWith('https://example.com/timed-out', expect.anything())
        })

        it('does not fire on entry when an empty condition coexists with a real events entry; still wakes on the event', async () => {
            // The reported bug: an empty (always-true) condition alongside a real "events to wait
            // for" entry. Without the guard the empty condition matches on entry and the wait fires
            // immediately, ignoring the configured event. It must park and only wake when the event
            // actually fires.
            await createWaitUntilWorkflow({
                condition: { filters: emptyConditionFilters() },
                events: [eventNameFilter('wakeup_event')],
                max_wait_duration: '5m',
            })
            await triggerWorkflow(createGlobals())
            await expectParked()

            // The configured event fires — the matcher wakes the job via the events entry.
            await matcher.processBatch([createGlobals({ event: 'wakeup_event' })])

            await waitForExpect(() => {
                expect(mockFetch).toHaveBeenCalledTimes(1)
            }, 10000)
            expect(mockFetch).toHaveBeenCalledWith('https://example.com/condition-matched', expect.anything())
        })

        it('does not collapse a delay before a wait when the wait event fires during the delay', async () => {
            // trigger -> delay(5m) -> wait_until_condition(wakeup_event) -> matched / timeout. The
            // wait's event firing DURING the delay must not wake the job: the delay handler no longer
            // pre-advances currentAction to the wait, so while parked the job is at the delay (not the
            // wait) and the matcher leaves it alone. The delay is honored.
            await createWorkflow({
                actions: {
                    trigger: trigger(),
                    delay: { type: 'delay', config: { delay_duration: '5m' } },
                    wait_condition: {
                        type: 'wait_until_condition',
                        config: {
                            events: [eventNameFilter('wakeup_event')],
                            condition: { filters: null },
                            max_wait_duration: '5m',
                        },
                    },
                    function_matched: fetchAction('https://example.com/matched'),
                    exit: exitAction(),
                },
                edges: [
                    { from: 'trigger', to: 'delay', type: 'continue' },
                    { from: 'delay', to: 'wait_condition', type: 'continue' },
                    { from: 'wait_condition', to: 'function_matched', type: 'branch', index: 0 },
                    { from: 'wait_condition', to: 'exit', type: 'continue' },
                    { from: 'function_matched', to: 'exit', type: 'continue' },
                ],
            })
            await triggerWorkflow(createGlobals())
            await expectParked()

            // The wait's event fires during the delay — the job must stay parked in the delay.
            await matcher.processBatch([createGlobals({ event: 'wakeup_event' })])

            const jobs = await queryCyclotronJobs()
            expect(jobs.every((j: any) => j.status === 'available' && new Date(j.scheduled) > new Date())).toBe(true)
            expect(mockFetch).not.toHaveBeenCalled()
        })

        it('does not run the next step early when a conversion event fires during a delay', async () => {
            // trigger -> delay -> fetch, with an event-based conversion goal used only for
            // measurement (exit_only_at_end). A conversion event arriving while the job is parked
            // in the delay must NOT wake it and run the fetch ~5 minutes early.
            await createWorkflowFlow(
                {
                    actions: {
                        trigger: trigger(),
                        delay: { type: 'delay', config: { delay_duration: '5m' } },
                        after_delay: fetchAction('https://example.com/after-delay'),
                        exit: exitAction(),
                    },
                    edges: [
                        { from: 'trigger', to: 'delay', type: 'continue' },
                        { from: 'delay', to: 'after_delay', type: 'continue' },
                        { from: 'after_delay', to: 'exit', type: 'continue' },
                    ],
                },
                {
                    exitCondition: 'exit_only_at_end',
                    conversion: {
                        filters: [],
                        bytecode: [],
                        events: [eventNameFilter('conversion_event')],
                    } as any,
                }
            )
            await triggerWorkflow(createGlobals())
            await expectParked()

            // The conversion event fires during the delay — it must not pull the job out early.
            await matcher.processBatch([createGlobals({ event: 'conversion_event' })])

            const jobs = await queryCyclotronJobs()
            expect(jobs.every((j: any) => j.status === 'available' && new Date(j.scheduled) > new Date())).toBe(true)
            expect(mockFetch).not.toHaveBeenCalled()
        })

        it('exits the workflow when an exit_on_conversion goal fires during a delay', async () => {
            // Same shape, but the workflow exits on conversion. The conversion event during the delay
            // SHOULD resume the job — to exit it early — but must still not run the next step (fetch).
            await createWorkflowFlow(
                {
                    actions: {
                        trigger: trigger(),
                        delay: { type: 'delay', config: { delay_duration: '5m' } },
                        after_delay: fetchAction('https://example.com/after-delay'),
                        exit: exitAction(),
                    },
                    edges: [
                        { from: 'trigger', to: 'delay', type: 'continue' },
                        { from: 'delay', to: 'after_delay', type: 'continue' },
                        { from: 'after_delay', to: 'exit', type: 'continue' },
                    ],
                },
                {
                    exitCondition: 'exit_on_conversion',
                    conversion: {
                        filters: [],
                        bytecode: [],
                        events: [eventNameFilter('conversion_event')],
                    } as any,
                }
            )
            await triggerWorkflow(createGlobals())
            await expectParked()

            // The conversion event fires during the delay — the workflow must exit early.
            await matcher.processBatch([createGlobals({ event: 'conversion_event' })])

            await waitForExpect(async () => {
                const jobs = await queryCyclotronJobs()
                expect(jobs.some((j: any) => ['completed', 'failed', 'canceled'].includes(j.status))).toBe(true)
            }, 10000)
            // Exited on conversion — the after-delay step never ran.
            expect(mockFetch).not.toHaveBeenCalled()
        })

        it('wakes a parked condition wait on a person-property change with no analytics event', async () => {
            // The core of removing the 10-minute poll: a property change that produces no analytics
            // event (only a clickhouse_person mutation) must still wake the wait. The matcher's
            // person stream synthesizes a $person_updated globals carrying the new person.properties,
            // which the wait's property condition then matches.
            await createWaitUntilWorkflow({
                // Person-property condition only — never satisfied by the trigger event, so it parks.
                condition: { filters: personPropertyConditionFilters('plan', 'enterprise') },
                max_wait_duration: '5m',
            })
            // person is not persisted in the job state — the worker re-resolves it from the person store
            // on dequeue (CyclotronPerson.id = person.uuid). Without a resolvable person the re-parked job
            // stores person_id = null and the person-stream matcher can never find it. Production resolves
            // person_id during ingestion before the job parks; mirror that so the parked job carries
            // person_id = 'uuid' — the same id the clickhouse_person mutation below targets. The resolved
            // person has no `plan`, so the condition still fails on entry and the job parks as expected.
            mockPersonRepo.fetchPersonsByDistinctIds.mockResolvedValue([
                {
                    id: '1',
                    uuid: 'uuid',
                    team_id: team.id,
                    properties: { email: '[email protected]' },
                    properties_last_updated_at: {},
                    properties_last_operation: null,
                    created_at: DateTime.utc(),
                    version: 1,
                    is_identified: true,
                    is_user_id: null,
                    last_seen_at: null,
                    distinct_id: 'distinct_id',
                },
            ])
            await triggerWorkflow(createGlobals())
            await expectParked()

            // A clickhouse_person mutation for that person — the person now matches `plan=enterprise`.
            // No analytics event is involved; this comes straight off the person topic.
            const personMessage = {
                value: Buffer.from(
                    JSON.stringify({
                        id: 'uuid',
                        team_id: team.id,
                        properties: JSON.stringify({ email: '[email protected]', plan: 'enterprise' }),
                        is_deleted: 0,
                        is_identified: 1,
                        created_at: '2024-09-03 09:00:00.000',
                        timestamp: '2024-09-03 09:00:00.000',
                        version: 2,
                    })
                ),
            }
            const personGlobals = await matcher._parsePersonBatch([personMessage as any])
            await matcher.processBatch(personGlobals)

            await waitForExpect(() => {
                expect(mockFetch).toHaveBeenCalledTimes(1)
            }, 10000)
            expect(mockFetch).toHaveBeenCalledWith('https://example.com/condition-matched', expect.anything())
        })

        // Wait conditions authored as HogQL expressions must be woken by the streams, not only by the
        // 10-minute poll — otherwise removing the poll would silently strand them. These three cover
        // the distinct HogQL condition shapes real production flows use; each bytecode is exactly what
        // the Django serializer compiles for that expression (verified via compile_filters_bytecode).
        // If HogQL compilation changes, or a stream stops carrying what the bytecode reads (e.g. the
        // person wake dropping properties), one of these fails.
        it('wakes a HogQL event-name wait condition via the events stream', async () => {
            // HogQL: event = 'billing_added'
            await createWaitUntilWorkflow({
                condition: {
                    filters: {
                        bytecode: ['_H', 1, 32, 'billing_added', 32, 'event', 1, 1, 11],
                        properties: [{ type: 'hogql', key: "event = 'billing_added'" }],
                    },
                },
                max_wait_duration: '5m',
            })
            await triggerWorkflow(createGlobals({ event: 'custom_trigger', properties: {} }))
            await expectParked()
            await matcher.processBatch([createGlobals({ event: 'billing_added', properties: {} })])
            await waitForExpect(() => {
                expect(mockFetch).toHaveBeenCalledTimes(1)
            }, 10000)
            expect(mockFetch).toHaveBeenCalledWith('https://example.com/condition-matched', expect.anything())
        })

        it('wakes a HogQL event+property wait condition only on the matching event and property', async () => {
            // HogQL: event = 'cal_booking' and properties.trigger_event = 'BOOKING_CREATED'
            await createWaitUntilWorkflow({
                condition: {
                    filters: {
                        bytecode: [
                            '_H',
                            1,
                            32,
                            'cal_booking',
                            32,
                            'event',
                            1,
                            1,
                            11,
                            32,
                            'BOOKING_CREATED',
                            32,
                            'trigger_event',
                            32,
                            'properties',
                            1,
                            2,
                            11,
                            3,
                            2,
                        ],
                        properties: [
                            {
                                type: 'hogql',
                                key: "event = 'cal_booking' and properties.trigger_event = 'BOOKING_CREATED'",
                            },
                        ],
                    },
                },
                max_wait_duration: '5m',
            })
            await triggerWorkflow(createGlobals({ event: 'custom_trigger', properties: {} }))
            await expectParked()
            // Right event, wrong property — must not wake.
            await matcher.processBatch([
                createGlobals({ event: 'cal_booking', properties: { trigger_event: 'OTHER' } }),
            ])
            const jobs = await queryCyclotronJobs()
            expect(jobs.every((j: any) => j.status === 'available' && new Date(j.scheduled) > new Date())).toBe(true)
            expect(mockFetch).not.toHaveBeenCalled()
            // Right event and property — wakes.
            await matcher.processBatch([
                createGlobals({ event: 'cal_booking', properties: { trigger_event: 'BOOKING_CREATED' } }),
            ])
            await waitForExpect(() => {
                expect(mockFetch).toHaveBeenCalledTimes(1)
            }, 10000)
            expect(mockFetch).toHaveBeenCalledWith('https://example.com/condition-matched', expect.anything())
        })

        it('wakes a HogQL dynamic-key person-property wait condition via the person stream', async () => {
            // HogQL: person.properties[concat(toString(person.properties.shop_id), '_billing_status')] = 'active'
            await createWaitUntilWorkflow({
                condition: {
                    filters: {
                        bytecode: [
                            '_H',
                            1,
                            32,
                            'active',
                            32,
                            'properties',
                            32,
                            'person',
                            1,
                            2,
                            32,
                            'shop_id',
                            32,
                            'properties',
                            32,
                            'person',
                            1,
                            3,
                            2,
                            'toString',
                            1,
                            32,
                            '_billing_status',
                            2,
                            'concat',
                            2,
                            45,
                            11,
                        ],
                        properties: [
                            {
                                type: 'hogql',
                                key: "person.properties[concat(toString(person.properties.shop_id), '_billing_status')] = 'active'",
                            },
                        ],
                    },
                },
                max_wait_duration: '5m',
            })
            // Parked person has shop_id but no billing status yet, so the condition is false and it parks.
            mockPersonRepo.fetchPersonsByDistinctIds.mockResolvedValue([
                {
                    id: '1',
                    uuid: 'uuid',
                    team_id: team.id,
                    properties: { shop_id: '42' },
                    properties_last_updated_at: {},
                    properties_last_operation: null,
                    created_at: DateTime.utc(),
                    version: 1,
                    is_identified: true,
                    is_user_id: null,
                    last_seen_at: null,
                    distinct_id: 'distinct_id',
                },
            ])
            await triggerWorkflow(createGlobals())
            await expectParked()
            // clickhouse_person mutation sets 42_billing_status = 'active' with no analytics event.
            const personMessage = {
                value: Buffer.from(
                    JSON.stringify({
                        id: 'uuid',
                        team_id: team.id,
                        properties: JSON.stringify({ shop_id: '42', '42_billing_status': 'active' }),
                        is_deleted: 0,
                        is_identified: 1,
                        created_at: '2024-09-03 09:00:00.000',
                        timestamp: '2024-09-03 09:00:00.000',
                        version: 2,
                    })
                ),
            }
            const personGlobals = await matcher._parsePersonBatch([personMessage as any])
            await matcher.processBatch(personGlobals)
            await waitForExpect(() => {
                expect(mockFetch).toHaveBeenCalledTimes(1)
            }, 10000)
            expect(mockFetch).toHaveBeenCalledWith('https://example.com/condition-matched', expect.anything())
        })

        // A parked wait keyed on an anonymous person can't be woken by the survivor's updates after a
        // merge — the survivor's person/event updates arrive under a new person_id the wait doesn't
        // reference. The matcher re-keys the wait off the clickhouse_person_distinct_id repoint so it
        // follows the survivor. These two tests cover the re-key and the wake it enables.
        // Factories (not eager consts): they read `team`, which is only assigned in beforeEach.
        const anonPersonRow = (): any => ({
            id: '1',
            uuid: 'old-uuid',
            team_id: team.id,
            properties: { email: '[email protected]' },
            properties_last_updated_at: {},
            properties_last_operation: null,
            created_at: DateTime.utc(),
            version: 1,
            is_identified: true,
            is_user_id: null,
            last_seen_at: null,
            distinct_id: 'distinct_id',
        })
        const distinctIdMoveMessage = (overrides: Record<string, any> = {}): any => ({
            value: Buffer.from(
                JSON.stringify({
                    team_id: team.id,
                    distinct_id: 'distinct_id',
                    person_id: 'new-uuid',
                    version: 2,
                    is_deleted: 0,
                    ...overrides,
                })
            ),
        })
        // The post-merge survivor the repoint points at, now carrying the merged plan=enterprise. The
        // re-keyed job resolves by this survivor personId (personIdRepointed), not the repointed distinct_id.
        const survivorPersonRow = (): any => ({
            id: '2',
            uuid: 'new-uuid',
            team_id: team.id,
            properties: { email: '[email protected]', plan: 'enterprise' },
            properties_last_updated_at: {},
            properties_last_operation: null,
            created_at: DateTime.utc(),
            version: 2,
            is_identified: true,
            is_user_id: null,
            last_seen_at: null,
        })

        it('re-keys a wait onto the survivor after a merge repoints its distinct_id, so the survivor wakes it', async () => {
            await createWaitUntilWorkflow({
                condition: { filters: personPropertyConditionFilters('plan', 'enterprise') },
                max_wait_duration: '5m',
            })
            // Parked keyed on the anon person ('old-uuid'), which has no `plan`, so the wait parks on entry.
            mockPersonRepo.fetchPersonsByDistinctIds.mockResolvedValue([anonPersonRow()])
            await triggerWorkflow(createGlobals())
            await expectParked()

            // The merge repoints 'distinct_id' onto survivor 'new-uuid' (version > 0). The matcher re-keys the
            // parked wait onto the survivor and wakes it (scheduled = now). The worker then resolves the job by
            // the survivor personId — not the repointed distinct_id — re-checks the condition against the
            // survivor's plan=enterprise, and advances down the matched branch, firing the fetch.
            mockPersonRepo.fetchPersonsByPersonIds.mockResolvedValue([survivorPersonRow()])
            await matcher.processMoveBatch(matcher._parsePersonDistinctIdBatch([distinctIdMoveMessage() as any]))

            await waitForExpect(() => {
                expect(mockFetch).toHaveBeenCalledTimes(1)
            }, 10000)
            expect(mockFetch).toHaveBeenCalledWith('https://example.com/condition-matched', expect.anything())
        })

        it('anchors and wakes a wait that parked before its distinct_id had a person', async () => {
            // Production shape: an event arrives for a distinct_id with no person yet, so the wait parks
            // with person_id NULL. Person wakes are keyed on person_id alone, so nothing can address that
            // job — before this was fixed, only the 10-minute polling re-check ever advanced it.
            await createWaitUntilWorkflow({
                condition: { filters: personPropertyConditionFilters('plan', 'enterprise') },
                max_wait_duration: '5m',
            })
            mockPersonRepo.fetchPersonsByDistinctIds.mockResolvedValue([])
            await triggerWorkflow(createGlobals())
            await expectParked()

            // The premise of the test: no anchor to wake.
            const parked = await cyclotronPool.query(
                `SELECT person_id FROM cyclotron_jobs WHERE ${statusColumn} = 'available'`
            )
            expect(parked.rows).toHaveLength(1)
            expect(parked.rows[0].person_id).toBeNull()

            // The distinct_id acquires a person for the first time (version 0), and that person already
            // satisfies the condition. The matcher fills the missing anchor and wakes the wait, which then
            // resolves by personId and advances down the matched branch.
            mockPersonRepo.fetchPersonsByPersonIds.mockResolvedValue([survivorPersonRow()])
            await matcher.processMoveBatch(
                matcher._parsePersonDistinctIdBatch([distinctIdMoveMessage({ version: 0 }) as any])
            )

            await waitForExpect(() => {
                expect(mockFetch).toHaveBeenCalledTimes(1)
            }, 10000)
            expect(mockFetch).toHaveBeenCalledWith('https://example.com/condition-matched', expect.anything())
        })

        it('anchors a wait whose condition is still false, so a later person update can wake it', async () => {
            // The shape the affected production runs actually take, and the one the test above does not
            // cover: the person exists by the time the anchor is filled, but does not satisfy the condition
            // yet. So the fill wakes the wait, the re-check fails, and it re-parks — this time WITH an
            // anchor. The property is then set, and that person update has to be able to find the job.
            // Without the anchor there is no key for the person stream to match on and only the polling
            // re-check would ever advance it.
            await createWaitUntilWorkflow({
                condition: { filters: personPropertyConditionFilters('plan', 'enterprise') },
                max_wait_duration: '5m',
            })
            mockPersonRepo.fetchPersonsByDistinctIds.mockResolvedValue([])
            await triggerWorkflow(createGlobals())
            await expectParked()

            const beforeFill = await cyclotronPool.query(
                `SELECT person_id FROM cyclotron_jobs WHERE ${statusColumn} = 'available'`
            )
            expect(beforeFill.rows[0].person_id).toBeNull()

            // First mapping arrives. The person exists now but has no `plan`, so the condition is false.
            const personWithoutPlan = { ...survivorPersonRow(), properties: { email: '[email protected]' } }
            mockPersonRepo.fetchPersonsByPersonIds.mockResolvedValue([personWithoutPlan])
            await matcher.processMoveBatch(
                matcher._parsePersonDistinctIdBatch([distinctIdMoveMessage({ version: 0 }) as any])
            )

            // It re-parked rather than advancing, and it now carries the anchor.
            await waitForExpect(async () => {
                const afterFill = await cyclotronPool.query(
                    `SELECT person_id FROM cyclotron_jobs WHERE ${statusColumn} = 'available'`
                )
                expect(afterFill.rows).toHaveLength(1)
                expect(afterFill.rows[0].person_id).toBe('new-uuid')
            }, 10000)
            expect(mockFetch).not.toHaveBeenCalled()

            // Now the property is set. This is a person mutation with no analytics event, so it can only be
            // matched on person_id — the anchor written above is what makes it findable.
            const personMessage = {
                value: Buffer.from(
                    JSON.stringify({
                        id: 'new-uuid',
                        team_id: team.id,
                        properties: JSON.stringify({ email: '[email protected]', plan: 'enterprise' }),
                        is_deleted: 0,
                        is_identified: 1,
                        created_at: '2024-09-03 09:00:00.000',
                        timestamp: '2024-09-03 09:00:00.000',
                        version: 3,
                    })
                ),
            }
            mockPersonRepo.fetchPersonsByPersonIds.mockResolvedValue([survivorPersonRow()])
            await matcher.processBatch(await matcher._parsePersonBatch([personMessage as any]))

            await waitForExpect(() => {
                expect(mockFetch).toHaveBeenCalledTimes(1)
            }, 10000)
            expect(mockFetch).toHaveBeenCalledWith('https://example.com/condition-matched', expect.anything())
        })

        it('wakes a parked wait from a cdp_internal_events signal with no analytics event', async () => {
            // CDP-generated signals (e.g. $insight_alert_firing) arrive on cdp_internal_events and never
            // hit the analytics events topic. The matcher parses them via _parseInternalEventsBatch and
            // wakes parked waits whose "events to wait for" name the signal, matched by distinct_id.
            await createWaitUntilWorkflow({
                // Property condition never matches the trigger event, so the job parks until the signal.
                condition: { filters: HOG_FILTERS_EXAMPLES.elements_text_filter.filters },
                events: [eventNameFilter('$insight_alert_firing')],
                max_wait_duration: '5m',
            })
            await triggerWorkflow(createGlobals())
            await expectParked()

            // A raw cdp_internal_events message for this person's distinct_id — no analytics event.
            const internalEventMessage = {
                value: Buffer.from(
                    JSON.stringify({
                        team_id: team.id,
                        event: {
                            uuid: new UUIDT().toString(),
                            event: '$insight_alert_firing',
                            distinct_id: 'distinct_id',
                            properties: {},
                            timestamp: '2024-09-03T09:00:00Z',
                        },
                    })
                ),
            }
            const internalGlobals = await matcher._parseInternalEventsBatch([internalEventMessage as any])
            await matcher.processBatch(internalGlobals)

            await waitForExpect(() => {
                expect(mockFetch).toHaveBeenCalledTimes(1)
            }, 10000)
            expect(mockFetch).toHaveBeenCalledWith('https://example.com/condition-matched', expect.anything())
        })

        it('counts an event-based conversion exactly once per run even when the event fires repeatedly', async () => {
            // Regression guard for conversion over-counting on measurement-only flows. The run stays
            // parked in the delay (exit_only_at_end), and the conversion event fires across three
            // separate matcher batches. The matcher must record exactly ONE `conversion` metric for
            // the run (deduped via conversionCounted), and must never wake the job.
            await createWorkflowFlow(
                {
                    actions: {
                        trigger: trigger(),
                        delay: { type: 'delay', config: { delay_duration: '5m' } },
                        after_delay: fetchAction('https://example.com/after-delay'),
                        exit: exitAction(),
                    },
                    edges: [
                        { from: 'trigger', to: 'delay', type: 'continue' },
                        { from: 'delay', to: 'after_delay', type: 'continue' },
                        { from: 'after_delay', to: 'exit', type: 'continue' },
                    ],
                },
                {
                    exitCondition: 'exit_only_at_end',
                    conversion: {
                        filters: [],
                        bytecode: [],
                        events: [eventNameFilter('conversion_event')],
                    } as any,
                }
            )
            await triggerWorkflow(createGlobals())
            await expectParked()

            const conversionCount = (): number =>
                mockProducerObserver
                    .getProducedKafkaMessagesForTopic(KAFKA_APP_METRICS_2)
                    .filter((m: any) => m.value.app_source === 'hog_flow')
                    .filter((m: any) => m.value.metric_name === 'conversion')
                    .reduce((sum: number, m: any) => sum + m.value.count, 0)

            // First match counts the conversion once.
            await matcher.processBatch([createGlobals({ event: 'conversion_event' })])
            await waitForExpect(() => {
                expect(conversionCount()).toBe(1)
            }, 5000)

            // The same conversion event firing again must NOT increment the count — the run already converted.
            await matcher.processBatch([createGlobals({ event: 'conversion_event' })])
            await matcher.processBatch([createGlobals({ event: 'conversion_event' })])
            expect(conversionCount()).toBe(1)

            // Measurement-only: the run stays parked and the after-delay step never runs early.
            const jobs = await queryCyclotronJobs()
            expect(jobs.every((j: any) => j.status === 'available' && new Date(j.scheduled) > new Date())).toBe(true)
            expect(mockFetch).not.toHaveBeenCalled()
        })
    })

    describe('wait_until_time_window: window in the future', () => {
        beforeEach(async () => {
            // A fixed daily window is "open" during its own minutes every day, so any
            // run landing inside it (e.g. CI at 09:5x UTC for a 23:5x UTC+14 window)
            // executes immediately instead of rescheduling. Derive the window a few
            // minutes ahead of now so it is always strictly in the future.
            const now = DateTime.utc()
            const windowStart = now.plus({ minutes: 10 }).toFormat('HH:mm')
            const windowEnd = now.plus({ minutes: 20 }).toFormat('HH:mm')
            await createWorkflow({
                actions: {
                    trigger: trigger(),
                    wait_window: {
                        type: 'wait_until_time_window',
                        config: { timezone: 'UTC', day: 'any', time: [windowStart, windowEnd] },
                    },
                    function_1: fetchAction('https://example.com/after-time-window'),
                    exit: exitAction(),
                },
                edges: [
                    { from: 'trigger', to: 'wait_window', type: 'continue' },
                    { from: 'wait_window', to: 'function_1', type: 'continue' },
                    { from: 'function_1', to: 'exit', type: 'continue' },
                ],
            })
            globals = createGlobals()
        })

        it('parks until the window opens and does not advance early on a premature resume', async () => {
            await triggerWorkflow(globals)

            // Job should be rescheduled to the future time window.
            await waitForExpect(async () => {
                const jobs = await queryCyclotronJobs()
                const rescheduled = jobs.filter(
                    (j: any) => j.status === 'available' && new Date(j.scheduled) > new Date()
                )
                expect(rescheduled.length).toBe(1)
            }, 5000)
            expect(mockFetch).not.toHaveBeenCalled()

            // A premature resume (the window is still in the future) must re-park, not advance to the
            // next step. The handler stays at the wait_until_time_window step and reschedules, so the
            // step that follows the window never runs early.
            await cyclotronPool.query(`UPDATE cyclotron_jobs SET scheduled = NOW() WHERE ${statusColumn} = 'available'`)
            await waitForExpect(async () => {
                const jobs = await queryCyclotronJobs()
                expect(jobs.some((j: any) => j.status === 'available' && new Date(j.scheduled) > new Date())).toBe(true)
            }, 10000)
            expect(mockFetch).not.toHaveBeenCalled()
        })
    })

    describe('wait_until_time_window: window currently open', () => {
        it('advances through the window and runs the next step', async () => {
            // day: 'any', time: 'any' is always open, so the step must advance and run the next action
            // instead of parking forever.
            await createWorkflow({
                actions: {
                    trigger: trigger(),
                    wait_window: {
                        type: 'wait_until_time_window',
                        config: { timezone: 'UTC', day: 'any', time: 'any' },
                    },
                    function_1: fetchAction('https://example.com/window-open'),
                    exit: exitAction(),
                },
                edges: [
                    { from: 'trigger', to: 'wait_window', type: 'continue' },
                    { from: 'wait_window', to: 'function_1', type: 'continue' },
                    { from: 'function_1', to: 'exit', type: 'continue' },
                ],
            })

            await triggerWorkflow(createGlobals())

            await waitForExpect(() => {
                expect(mockFetch).toHaveBeenCalledTimes(1)
            }, 10000)
            expect(mockFetch).toHaveBeenCalledWith('https://example.com/window-open', expect.anything())
        })
    })

    describe('fetch failure with retries', () => {
        beforeEach(async () => {
            await createWorkflow({
                actions: {
                    trigger: trigger(),
                    function_1: fetchAction('https://example.com/failing-endpoint'),
                    exit: exitAction(),
                },
                edges: [
                    { from: 'trigger', to: 'function_1', type: 'continue' },
                    { from: 'function_1', to: 'exit', type: 'continue' },
                ],
            })

            mockFetch.mockResolvedValue({
                status: 500,
                headers: {},
                json: () => Promise.resolve({ error: 'Server error' }),
                text: () => Promise.resolve(JSON.stringify({ error: 'Server error' })),
                dump: () => Promise.resolve(),
            })

            globals = createGlobals()
        })

        it('should retry the fetch and eventually complete with error', async () => {
            await triggerWorkflow(globals)

            // Hogflow function actions retry fetch within a single execution cycle.
            // We expect at least 2 calls (initial + retry) before the workflow completes.
            await waitForExpect(() => {
                expect(mockFetch.mock.calls.length).toBeGreaterThanOrEqual(2)
            }, 15000)

            // Verify the workflow eventually reaches a terminal state
            await waitForExpect(async () => {
                const jobs = await queryCyclotronJobs()
                const terminal = jobs.filter(
                    (j: any) => j.status === 'completed' || j.status === 'failed' || j.status === 'canceled'
                )
                expect(terminal.length).toBeGreaterThanOrEqual(1)
            }, 5000)
        })
    })

    describe('person data survives v2 round-trip', () => {
        beforeEach(async () => {
            const person: InternalPersonWithDistinctId = {
                id: '1',
                uuid: 'dd3d6f80-60ad-45c3-bd61-e2300f2ba7e1',
                team_id: team.id,
                properties: { email: '[email protected]', name: 'Test User', plan: 'enterprise' },
                properties_last_updated_at: {},
                properties_last_operation: null,
                created_at: DateTime.utc(),
                version: 1,
                is_identified: true,
                is_user_id: null,
                last_seen_at: null,
                distinct_id: 'test-distinct-id',
            }
            mockPersonRepo.fetchPersonsByDistinctIds.mockResolvedValue([person])

            await insertHogFunctionTemplate(hub.postgres, {
                id: 'template-workflows-e2e-person',
                name: 'Workflows E2E Person',
                code: `fetch(inputs.url, { 'method': 'POST', 'body': inputs.body });`,
                inputs_schema: [
                    { key: 'url', type: 'string', required: true },
                    { key: 'body', type: 'string', required: true },
                ],
            })

            await createWorkflow({
                actions: {
                    trigger: trigger(),
                    function_1: {
                        type: 'function',
                        config: {
                            template_id: 'template-workflows-e2e-person',
                            inputs: {
                                url: { value: 'https://example.com/person-test' },
                                body: {
                                    value: '{person.properties.email}',
                                    bytecode: ['_H', 1, 32, 'email', 32, 'properties', 32, 'person', 1, 3, 38],
                                },
                            },
                        },
                    },
                    exit: exitAction(),
                },
                edges: [
                    { from: 'trigger', to: 'function_1', type: 'continue' },
                    { from: 'function_1', to: 'exit', type: 'continue' },
                ],
            })

            globals = createGlobals({ distinct_id: 'test-distinct-id' })
        })

        it('should load person and pass properties to function action', async () => {
            await triggerWorkflow(globals)

            await waitForExpect(() => {
                expect(mockFetch).toHaveBeenCalledTimes(1)
            }, 10000)

            // The fetch body should contain the person's email, proving person data
            // was loaded by the worker after deserializing the v2 job
            expect(mockFetch).toHaveBeenCalledWith(
                'https://example.com/person-test',
                expect.objectContaining({
                    body: '[email protected]',
                    method: 'POST',
                })
            )
        })
    })

    describe('batch workflow: event.distinct_id resolved at the worker', () => {
        // A batch-triggered workflow enqueues invocations with personId but an empty
        // event.distinct_id (blast radius returns UUIDs only). The cyclotron worker is
        // responsible for resolving one distinct_id per person during its existing
        // postgres lookup and backfilling state.event.distinct_id — otherwise capture-based
        // templates defaulting to {event.distinct_id} would silently mint new person profiles.
        const personUuid = 'aaaaaaaa-1111-1111-1111-111111111111'
        const personDistinctId = 'batch-person-distinct-id'
        let hogFlow: HogFlow

        beforeEach(async () => {
            const person = {
                id: '2',
                uuid: personUuid,
                team_id: team.id,
                properties: { email: '[email protected]', plan: 'enterprise' },
                properties_last_updated_at: {},
                properties_last_operation: null,
                created_at: DateTime.utc(),
                version: 1,
                is_identified: true,
                is_user_id: null,
                last_seen_at: null,
            }
            mockPersonRepo.fetchPersonsByDistinctIds.mockResolvedValue([{ ...person, distinct_id: personDistinctId }])
            mockPersonRepo.fetchPersonsByPersonIds.mockResolvedValue([person])
            mockPersonRepo.fetchDistinctIdsForPersons.mockResolvedValue({
                [person.id]: [personDistinctId],
            })

            await insertHogFunctionTemplate(hub.postgres, {
                id: 'template-workflows-e2e-batch-distinct-id',
                name: 'Workflows E2E Batch Distinct Id',
                code: `fetch(inputs.url, { 'method': 'POST', 'body': inputs.distinct_id });`,
                inputs_schema: [
                    { key: 'url', type: 'string', required: true },
                    { key: 'distinct_id', type: 'string', required: true },
                ],
            })

            hogFlow = await createWorkflowFlow({
                actions: {
                    trigger: {
                        type: 'trigger',
                        config: {
                            // 'batch' trigger so the events consumer skips this workflow —
                            // it only fires when an invocation is queued directly.
                            type: 'batch',
                            filters: { properties: [{ key: 'plan', value: 'enterprise', type: 'person' }] },
                        },
                    },
                    function_1: {
                        type: 'function',
                        config: {
                            template_id: 'template-workflows-e2e-batch-distinct-id',
                            inputs: {
                                url: { value: 'https://example.com/batch-distinct-id' },
                                distinct_id: {
                                    value: '{event.distinct_id}',
                                    bytecode: ['_H', 1, 32, 'distinct_id', 32, 'event', 1, 2, 38],
                                },
                            },
                        },
                    },
                    exit: exitAction(),
                },
                edges: [
                    { from: 'trigger', to: 'function_1', type: 'continue' },
                    { from: 'function_1', to: 'exit', type: 'continue' },
                ],
            })
        })

        it("backfills {event.distinct_id} from the person's distinct_id at dequeue", async () => {
            await triggerBatchWorkflow(hogFlow, personUuid)

            await waitForExpect(() => {
                expect(mockFetch).toHaveBeenCalledTimes(1)
            }, 5000)

            // The hog template's `{event.distinct_id}` resolved at runtime to the value
            // the worker backfilled — proving the full chain: postgres lookup → CyclotronPerson.distinct_id
            // → state.event.distinct_id mutation → hog input resolution.
            expect(mockFetch).toHaveBeenCalledWith(
                'https://example.com/batch-distinct-id',
                expect.objectContaining({
                    body: personDistinctId,
                    method: 'POST',
                })
            )
        })

        it('does NOT call fetchDistinctIdsForPersons for event-triggered invocations', async () => {
            // Regression guard for the optimization: when event.distinct_id is set going in,
            // the persons-manager uses the by-distinct_id lookup which returns the distinct_id
            // as part of the lookup key — no separate fetchDistinctIdsForPersons RPC needed.
            // Without this guard, the by-person_id path could accidentally be used for event
            // triggers and pay an unnecessary postgres round-trip per worker tick.
            await insertHogFunctionTemplate(hub.postgres, {
                id: 'template-workflows-e2e-event-no-extra-rpc',
                name: 'Workflows E2E Event No Extra RPC',
                code: `fetch(inputs.url, { 'method': 'POST' });`,
                inputs_schema: [{ key: 'url', type: 'string', required: true }],
            })
            await createWorkflow({
                actions: {
                    trigger: trigger(),
                    function_1: fetchAction('https://example.com/event-trigger-no-extra-rpc'),
                    exit: exitAction(),
                },
                edges: [
                    { from: 'trigger', to: 'function_1', type: 'continue' },
                    { from: 'function_1', to: 'exit', type: 'continue' },
                ],
            })

            mockPersonRepo.fetchDistinctIdsForPersons.mockClear()

            await triggerWorkflow(createGlobals({ distinct_id: personDistinctId }))

            await waitForExpect(() => {
                expect(mockFetch).toHaveBeenCalled()
            }, 5000)

            expect(mockPersonRepo.fetchDistinctIdsForPersons).not.toHaveBeenCalled()
        })
    })

    describe('data warehouse table trigger workflow', () => {
        const TABLE_NAME = 'postgres.orders'

        // Row-scoped trigger that always matches (return-true bytecode), so the row payload alone
        // decides whether the flow fires.
        const dwhTrigger = (tableName: string) =>
            ({
                type: 'trigger' as const,
                config: {
                    type: 'data-warehouse-table',
                    table_name: tableName,
                    filters: { properties: [], bytecode: ['_h', 29] },
                },
            }) as any

        beforeEach(async () => {
            await createWorkflow({
                actions: {
                    trigger: dwhTrigger(TABLE_NAME),
                    function_1: fetchAction('https://example.com/warehouse-row'),
                    exit: exitAction(),
                },
                edges: [
                    { from: 'trigger', to: 'function_1', type: 'continue' },
                    { from: 'function_1', to: 'exit', type: 'continue' },
                ],
            })
        })

        it('runs the workflow end-to-end when a row syncs into the matching table', async () => {
            await triggerDwhWorkflow(createDwhGlobals(TABLE_NAME, { order_id: 42 }))

            await waitForExpect(() => {
                expect(mockFetch).toHaveBeenCalledTimes(1)
            }, 10000)

            expect(mockFetch).toHaveBeenCalledWith(
                'https://example.com/warehouse-row',
                expect.objectContaining({ method: 'POST' })
            )
        })

        it('does not run the workflow when the synced row belongs to a different table', async () => {
            await triggerDwhWorkflow(createDwhGlobals('postgres.other_table', { order_id: 1 }))

            // Give the worker a chance to (not) pick anything up before asserting it stayed idle.
            await waitForExpect(async () => {
                const jobs = await queryCyclotronJobs()
                expect(jobs).toHaveLength(0)
            }, 3000)

            expect(mockFetch).not.toHaveBeenCalled()
        })

        it('does not look up a person for warehouse rows (no synthetic distinct_id resolution)', async () => {
            // Regression guard: warehouse rows carry a made-up event.distinct_id that evals truthy.
            // The worker must skip the person lookup for data-warehouse-table triggers so we don't
            // pay a person round-trip (or accidentally resolve a bogus person) per synced row.
            const personSpy = jest.spyOn((hogflowWorker as any).personsManager, 'getCyclotronPerson')

            await triggerDwhWorkflow(createDwhGlobals(TABLE_NAME, { order_id: 7 }))

            await waitForExpect(() => {
                expect(mockFetch).toHaveBeenCalledTimes(1)
            }, 10000)

            expect(personSpy).not.toHaveBeenCalled()
        })
    })

    describe('posthog_ticket_tags input resolves templated values per element', () => {
        // Regression guard for the templating opt-in in products/cdp/backend/validation.py.
        // Reproduces the real user-reported case: a ticket tag set to
        // `zendesk/{variables.zendesk_ticketid}` used to ship to the runtime as
        // a literal placeholder string (because the type wasn't on the bytecode
        // opt-in list), so the ticket ended up tagged with the raw template text
        // instead of "zendesk/12345". The fix puts the per-element bytecode that
        // `generate_template_bytecode` already emits for lists into a shape that
        // `formatHogInput` can walk element-by-element.
        beforeEach(async () => {
            await insertHogFunctionTemplate(hub.postgres, {
                id: 'template-workflows-e2e-tags',
                name: 'Workflows E2E Tags',
                code: `fetch(inputs.url, { 'method': 'POST', 'body': jsonStringify(inputs.tags) });`,
                inputs_schema: [
                    { key: 'url', type: 'string', required: true },
                    { key: 'tags', type: 'posthog_ticket_tags', required: true },
                ],
            })

            const hogFlow = new FixtureHogFlowBuilder()
                .withTeamId(team.id)
                .withStatus('active')
                .withWorkflow({
                    actions: {
                        trigger: trigger(),
                        function_1: {
                            type: 'function',
                            config: {
                                template_id: 'template-workflows-e2e-tags',
                                inputs: {
                                    url: { value: 'https://example.com/tags' },
                                    tags: {
                                        // What the Python serializer now produces for a
                                        // `posthog_ticket_tags` value like
                                        // `["zendesk/{variables.zendesk_ticketid}"]`:
                                        // one outer array, one inner bytecode per element.
                                        // Inner bytecode is a concat of the literal prefix
                                        // and a variables.* field access — same shape Django
                                        // emits today for other templated string inputs
                                        // (cf. the production `text` input bytecode).
                                        value: ['zendesk/{variables.zendesk_ticketid}'],
                                        templating: 'hog',
                                        bytecode: [
                                            [
                                                '_H',
                                                1,
                                                32,
                                                'zendesk/',
                                                32,
                                                'zendesk_ticketid',
                                                32,
                                                'variables',
                                                1,
                                                2,
                                                2,
                                                'concat',
                                                2,
                                            ],
                                        ],
                                    },
                                },
                            },
                        },
                        exit: exitAction(),
                    },
                    edges: [
                        { from: 'trigger', to: 'function_1', type: 'continue' },
                        { from: 'function_1', to: 'exit', type: 'continue' },
                    ],
                })
                .build()
            // Workflow-defined variable with a default — populated into state.variables
            // by createHogFlowInvocation, so the function action sees it via globals.variables.
            // In a real workflow the value would be set by an earlier `Get ticket` action's
            // output_variable; for this test we pre-seed via the default to keep it focused.
            hogFlow.variables = [
                { key: 'zendesk_ticketid', type: 'string', label: 'Zendesk ticket id', default: '12345' },
            ]
            await insertHogFlow(hub.postgres, hogFlow)

            globals = createGlobals()
        })

        it('resolves zendesk/{variables.zendesk_ticketid} per element', async () => {
            await triggerWorkflow(globals)

            await waitForExpect(() => {
                expect(mockFetch).toHaveBeenCalledTimes(1)
            }, 10000)

            // The body proves the full chain end-to-end: per-element bytecode →
            // formatHogInput recurses into the list → executes against globals
            // populated with workflow variables → concat produces "zendesk/12345".
            // Pre-fix behaviour would emit `["zendesk/{variables.zendesk_ticketid}"]`.
            expect(mockFetch).toHaveBeenCalledWith(
                'https://example.com/tags',
                expect.objectContaining({
                    body: JSON.stringify(['zendesk/12345']),
                    method: 'POST',
                })
            )
        })
    })

    describe('create AI task action', () => {
        let flowId: string

        beforeEach(async () => {
            // The real template, not a fixture copy: this block exists to prove the shipped hog
            // compiles and drives the registered async function inside a running workflow, which
            // is the execution path a template-level test never touches.
            await insertHogFunctionTemplate(hub.postgres, {
                id: createTaskTemplate.id,
                name: createTaskTemplate.name,
                code: createTaskTemplate.code,
                inputs_schema: createTaskTemplate.inputs_schema,
            })

            const flow = await createWorkflowFlow({
                actions: {
                    trigger: trigger(),
                    function_1: {
                        type: 'function',
                        config: {
                            template_id: createTaskTemplate.id,
                            inputs: {
                                prompt: { value: 'Investigate the error spike' },
                                title: { value: 'Error spike' },
                                non_failure_status_codes: { value: [409] },
                            },
                        },
                        output_variable: { key: 'task' },
                    },
                    function_2: fetchAction('https://example.com/task-done'),
                    exit: exitAction(),
                },
                edges: [
                    { from: 'trigger', to: 'function_1', type: 'continue' },
                    { from: 'function_1', to: 'function_2', type: 'continue' },
                    { from: 'function_2', to: 'exit', type: 'continue' },
                ],
            })
            flowId = flow.id
            globals = createGlobals()
        })

        const runMetricNames = (): string[] =>
            mockProducerObserver
                .getProducedKafkaMessagesForTopic(KAFKA_APP_METRICS_2)
                .filter((m: any) => m.value.app_source === 'hog_flow' && m.value.app_source_id === flowId)
                .map((m: any) => m.value.metric_name)

        it('creates the task through the real registered async function', async () => {
            mockFetch.mockResolvedValue({
                status: 201,
                headers: { 'Content-Type': 'application/json' },
                json: () => Promise.resolve({ id: 'task-1', run_id: 'run-1' }),
                text: () => Promise.resolve(JSON.stringify({ id: 'task-1', run_id: 'run-1' })),
                dump: () => Promise.resolve(),
            } as any)

            await triggerWorkflow(globals)

            await waitForExpect(() => {
                expect(mockFetch).toHaveBeenCalledTimes(1)
            }, 10000)

            const [url, options] = mockFetch.mock.calls[0]
            expect(url).toEqual(`${hub.SITE_URL}/api/projects/${team.id}/workflow_tasks/`)
            expect(options.method).toEqual('POST')

            // Step-scoped dedupe key: run id + action id, so a second task step in the same run
            // would not collide with this one.
            expect(parseJSON(options.body)).toEqual({
                prompt: 'Investigate the error spike',
                title: 'Error spike',
                event: expect.objectContaining({
                    event: '$pageview',
                    uuid: 'b3a1fe86-b10c-43cc-acaf-d208977608d0',
                }),
                idempotency_key: expect.stringMatching(/^[0-9a-f-]{36}:function_1:0$/),
            })

            // The endpoint trusts these claims to resolve the workflow owner, so the token has to
            // verify against the shared dev secret with exactly this audience and flow id.
            const token = (options.headers['Authorization'] as string).replace('Bearer ', '')
            // The literal pins the cross-language contract: Django's TASKS_CREATE_JWT_SECRET dev default
            // (posthog/settings/data_stores.py) must match the nodejs one or local runs 401.
            // nosemgrep: javascript.jsonwebtoken.security.jwt-hardcode.hardcoded-jwt-secret
            const claims = jsonwebtoken.verify(token, 'local-dev-tasks-create-jwt', {
                audience: 'posthog:tasks:create',
                algorithms: ['HS256'],
            }) as jsonwebtoken.JwtPayload
            expect(claims.team_id).toEqual(team.id)
            expect(claims.hog_flow_id).toEqual(flowId)

            // The step parks until Django wakes it with the task's outcome.
            let originKey = ''
            await waitForExpect(async () => {
                const { rows } = await cyclotronPool.query(`SELECT id, status, state FROM cyclotron_jobs`)
                expect(rows).toHaveLength(1)
                expect(rows[0].status).toEqual('available')
                const state = parseJSON(rows[0].state.toString('utf-8')).state
                expect(state.currentAction.awaitingResume.key).toEqual(`${rows[0].id}:function_1:0`)
                originKey = state.currentAction.awaitingResume.key
            }, 10000)
            expect(mockFetch).toHaveBeenCalledTimes(1)
            expect(runMetricNames()).not.toContain('failed')

            // Django reports the run finished: the same internal event it produces, through the
            // matcher's own parser, wakes the parked step and the workflow moves on.
            const matcher = new CdpHogflowSubscriptionMatcherConsumer({ ...hub }, deps)
            const { resumes } = matcher._splitStepResumes([
                {
                    value: Buffer.from(
                        JSON.stringify({
                            team_id: team.id,
                            event: {
                                uuid: 'a5c2d9e1-3f4b-4c8d-9e0f-1a2b3c4d5e6f',
                                event: '$workflow_step_resume',
                                distinct_id: `team_${team.id}`,
                                properties: {
                                    origin_key: originKey,
                                    status: 'completed',
                                    result: { run_id: 'run-1', final_message: 'Found the cause' },
                                },
                                timestamp: '2024-01-01T00:00:00Z',
                            },
                        })
                    ),
                } as any,
            ])
            await matcher.processStepResumes(resumes)

            await waitForExpect(() => {
                expect(mockFetch).toHaveBeenCalledTimes(2)
            }, 10000)
            expect(mockFetch.mock.calls[1][0]).toEqual('https://example.com/task-done')
            const logs = mockProducerObserver
                .getProducedKafkaMessagesForTopic(KAFKA_LOG_ENTRIES)
                .map((m: any) => m.value.message as string)
            expect(logs).toContainEqual(expect.stringContaining('[Action:function_1] The task finished'))
            expect(runMetricNames()).not.toContain('failed')
        })

        it('fails the run when the task limit replies 409', async () => {
            mockFetch.mockResolvedValue({
                status: 409,
                headers: { 'Content-Type': 'application/json' },
                json: () => Promise.resolve({ detail: 'This workflow already has 5 tasks running' }),
                text: () => Promise.resolve(JSON.stringify({ detail: 'This workflow already has 5 tasks running' })),
                dump: () => Promise.resolve(),
            } as any)

            await triggerWorkflow(globals)

            await waitForExpect(() => {
                expect(runMetricNames()).toContain('failed')
            }, 10000)
            // 409 is terminal for the step: one request, no retry burning the engine's budget.
            const taskCreates = mockFetch.mock.calls.filter(([url]) => (url as string).includes('/workflow_tasks/'))
            expect(taskCreates).toHaveLength(1)
        })
    })

    describe('heartbeat during long batches', () => {
        let janitor: CyclotronV2Janitor

        beforeEach(async () => {
            // Restart the worker with a short heartbeat interval so the
            // interval fires several times inside the test's 600ms fetch delay.
            // Default is 10s — nothing would fire in a test.
            await hogflowWorker.stop()
            hub.CDP_CYCLOTRON_HEARTBEAT_INTERVAL_MS = 100
            hogflowWorker = new CdpCyclotronWorkerHogFlow(hub, deps, hogflowQueue)
            await hogflowWorker.start()

            // Aggressive stall/poison thresholds so the test observes the
            // exact regression the heartbeat guards against: without heartbeats
            // firing, a 600ms fetch would exceed the 300ms stall window and
            // the janitor would reset the row on the very first runOnce().
            // cleanupGraceMs is huge so terminal jobs stay queryable after
            // completion — we assert on their final janitor_touch_count.
            janitor = new CyclotronV2Janitor({
                pool: { dbUrl: CYCLOTRON_NODE_DB_URL },
                stallTimeoutMs: 300,
                maxTouchCount: 2,
                cleanupGraceMs: 99_999_000,
                cleanupBatchSize: 100,
                cleanupIntervalMs: 60_000,
            })

            await createWorkflow({
                actions: {
                    trigger: trigger(),
                    slow_fetch: fetchAction('https://example.com/slow'),
                    exit: exitAction(),
                },
                edges: [
                    { from: 'trigger', to: 'slow_fetch', type: 'continue' },
                    { from: 'slow_fetch', to: 'exit', type: 'continue' },
                ],
            })
            globals = createGlobals()
        })

        afterEach(async () => {
            await janitor?.stop()
        })

        it('heartbeats keep janitor_touch_count at 0 during a slow batch', async () => {
            // Fetch takes 600ms — 2x the stallTimeoutMs. The heartbeat
            // interval firing every 100ms is the only thing preventing
            // last_heartbeat from going stale.
            mockFetch.mockImplementation(
                () =>
                    new Promise((resolve) =>
                        setTimeout(
                            () =>
                                resolve({
                                    status: 200,
                                    headers: { 'Content-Type': 'application/json' },
                                    json: () => Promise.resolve({ ok: true }),
                                    text: () => Promise.resolve(JSON.stringify({ ok: true })),
                                    dump: () => Promise.resolve(),
                                }),
                            600
                        )
                    )
            )

            await triggerWorkflow(globals)

            // Wait until the fetch is in flight, then pass the stall cutoff. Without heartbeats,
            // last_heartbeat would be >300ms old and the janitor would mark it stalled.
            await waitForExpect(async () => {
                const jobs = await queryCyclotronJobs()
                expect(jobs.some((j) => j.status === 'running')).toBe(true)
            }, 5000)
            await new Promise((r) => setTimeout(r, 350))
            const midResult = await janitor.runOnce()
            expect(midResult.stalled).toBe(0)
            expect(midResult.poisoned).toBe(0)

            // Wait for the workflow to complete
            await waitForExpect(async () => {
                const jobs = await queryCyclotronJobs()
                expect(jobs.some((j) => j.status === 'completed')).toBe(true)
            }, 10000)

            // Final check: no row ever accumulated a touch. This is the
            // load-bearing assertion — regressions to the setInterval or
            // its cleanup would flip this to >= 1.
            const jobs = await queryCyclotronJobs()
            for (const row of jobs) {
                expect(row.janitor_touch_count).toBe(0)
            }
        })
    })
})

// Email queue routing: the email worker reschedules jobs between queue names on the same v2 backend.
describe('Workflows E2E (email queue)', () => {
    jest.setTimeout(30000)

    let eventsConsumer: CdpEventsConsumer
    let hogflowWorker: CdpCyclotronWorkerHogFlow
    let emailWorker: CdpCyclotronWorkerEmail
    let matcher: CdpHogflowSubscriptionMatcherConsumer | undefined

    let hub: Hub
    let kafkaProducer: KafkaProducerWrapper
    let mockProducerObserver: KafkaProducerObserver
    let team: Team
    let cyclotronPool: Pool
    let deps: ReturnType<typeof createCdpConsumerDeps>

    beforeAll(() => {
        cyclotronPool = new Pool({ connectionString: CYCLOTRON_NODE_DB_URL })
    })

    afterAll(async () => {
        await cyclotronPool.end()
    })

    beforeEach(async () => {
        MockKafkaProducerWrapper.create = jest.fn((...args) => {
            return ActualKafkaProducerWrapper.create(...args)
        })

        await ensureKafkaTopics(TEST_KAFKA_TOPICS)
        await resetTestDatabase()
        await cyclotronPool.query('DELETE FROM cyclotron_jobs')

        hub = await createHub()
        hub.CDP_CYCLOTRON_BATCH_DELAY_MS = 50
        // Without a Valkey host the SES rate limiter pool is null and every sending
        // limit in this block is silently disabled. Point it at the local test Redis.
        hub.SES_RATE_LIMITER_VALKEY_HOST = hub.CDP_REDIS_HOST || '127.0.0.1'
        if (hub.CDP_REDIS_PORT) {
            hub.SES_RATE_LIMITER_VALKEY_PORT = hub.CDP_REDIS_PORT
        }

        // `.invalid` domains are NXDOMAIN, everything else resolves as deliverable.
        const nxdomain = () => Promise.reject(Object.assign(new Error('queryMx ENOTFOUND'), { code: 'ENOTFOUND' }))
        mockDnsResolveMx.mockImplementation((domain: string) =>
            domain.endsWith('.invalid') ? nxdomain() : Promise.resolve([{ exchange: 'mx.example.com', priority: 10 }])
        )
        mockDnsResolve4.mockImplementation((domain: string) =>
            domain.endsWith('.invalid') ? nxdomain() : Promise.resolve(['1.2.3.4'])
        )
        mockDnsResolve6.mockImplementation(() => Promise.resolve([]))

        kafkaProducer = await ActualKafkaProducerWrapper.create(hub.KAFKA_CLIENT_RACK)
        mockProducerObserver = new KafkaProducerObserver(kafkaProducer)

        team = await getFirstTeam(hub.postgres)
        mockProducerObserver.resetKafkaProducer()

        // Email integration — provider 'maildev' routes to local SMTP (port 1025)
        await insertIntegration(hub.postgres, team.id, {
            id: 1,
            kind: 'email',
            config: {
                email: '[email protected]',
                name: 'Test Sender',
                domain: 'posthog.com',
                verified: true,
                provider: 'maildev',
            },
        })

        // Native-email template that the workflow's email action invokes
        await insertHogFunctionTemplate(hub.postgres, {
            id: 'template-workflows-e2e-email',
            name: 'Workflows E2E Email',
            code: `sendEmail(inputs.email)`,
            inputs_schema: [
                {
                    type: 'native_email',
                    key: 'email',
                    label: 'Email message',
                    integration: 'email',
                    required: true,
                    default: {
                        to: { email: '', name: '' },
                        from: { email: '', name: '' },
                        subject: '',
                        text: 'Hello!',
                        html: '<div>Hello!</div>',
                    },
                    secret: false,
                    description: '',
                    templating: 'liquid',
                },
            ],
        })

        matcher = undefined
        deps = createCdpConsumerDeps(hub, kafkaProducer)
        const kafkaQueue = new CyclotronJobQueueKafka(hub.KAFKA_CLIENT_RACK, hub, hub.CONSUMER_BATCH_SIZE)
        // Each consumer gets a dedicated CyclotronJobQueuePostgresV2 — sharing one
        // across two consumers collides on `this.worker` and the shared pg pool.
        // Mirrors the prod deployment model where each capability runs in its own pod.
        const eventsProducerQueue = new CyclotronJobQueuePostgresV2(hub.CONSUMER_BATCH_SIZE, hub)
        const hogflowConsumerQueue = new CyclotronJobQueuePostgresV2(hub.CONSUMER_BATCH_SIZE, hub)
        const emailConsumerQueue = new CyclotronJobQueuePostgresV2(hub.CONSUMER_BATCH_SIZE, hub)

        eventsConsumer = new CdpEventsConsumer(hub, deps, {
            hogQueue: kafkaQueue,
            hogflowQueue: eventsProducerQueue,
        })
        await Promise.all([kafkaQueue.startAsProducer(), eventsProducerQueue.startAsProducer()])

        // Hogflow worker polls jobs with queue_name='hogflow' and re-stamps email
        // jobs to queue_name='email' so the email worker picks them up
        hogflowWorker = new CdpCyclotronWorkerHogFlow(hub, deps, hogflowConsumerQueue)
        await hogflowWorker.start()

        // Email worker polls jobs with queue_name='email', sends via EmailService,
        // and continues the workflow inline (until it hits a fetch or terminates)
        emailWorker = new CdpCyclotronWorkerEmail(hub, deps, emailConsumerQueue)
        await emailWorker.start()
    })

    afterEach(async () => {
        await Promise.all([
            eventsConsumer?.stop() ?? Promise.resolve(),
            hogflowWorker?.stop() ?? Promise.resolve(),
            emailWorker?.stop() ?? Promise.resolve(),
            matcher?.stop().catch(() => {}) ?? Promise.resolve(),
        ])
        await kafkaProducer.disconnect()
        await closeHub(hub)
        mockProducerObserver.resetKafkaProducer()
    })

    async function queryCyclotronJobs(): Promise<any[]> {
        const result = await cyclotronPool.query(`SELECT *, status AS status FROM cyclotron_jobs ORDER BY created ASC`)
        return result.rows
    }

    function createGlobals(
        overrides: Partial<HogFunctionInvocationGlobals['event']> = {}
    ): HogFunctionInvocationGlobals {
        return createHogExecutionGlobals({
            project: { id: team.id } as any,
            event: {
                uuid: 'b3a1fe86-b10c-43cc-acaf-d208977608d0',
                event: '$pageview',
                properties: {
                    $current_url: 'https://posthog.com',
                    $lib_version: '1.0.0',
                },
                timestamp: '2024-09-03T09:00:00Z',
                ...overrides,
            } as any,
        })
    }

    // Mirrors what HogFlowSerializer compiles for {events: [{id: <name>}]}: a single
    // equality check on the `event` global. The matcher fails closed without bytecode.
    const eventNameFilter = (eventName: string) => ({
        filters: { events: [{ id: eventName }], bytecode: ['_H', 1, 32, eventName, 32, 'event', 1, 1, 11] as any[] },
    })

    it('routes the email through the dedicated queue and continues the workflow', async () => {
        const hogFlow = new FixtureHogFlowBuilder()
            .withTeamId(team.id)
            .withStatus('active')
            .withExitCondition('exit_only_at_end')
            .withWorkflow({
                actions: {
                    trigger: {
                        type: 'trigger',
                        config: { type: 'event', filters: HOG_FILTERS_EXAMPLES.no_filters.filters ?? {} },
                    },
                    email_1: {
                        type: 'function_email',
                        config: {
                            template_id: 'template-workflows-e2e-email',
                            inputs: {
                                email: {
                                    value: {
                                        to: { email: '[email protected]', name: 'Recipient' },
                                        from: { integrationId: 1, email: '[email protected]' },
                                        subject: 'Test Email',
                                        text: 'Test text',
                                        html: '<p>Test html</p>',
                                    },
                                },
                            },
                        },
                    },
                    exit: { type: 'exit', config: {} },
                },
                edges: [
                    { from: 'trigger', to: 'email_1', type: 'continue' },
                    { from: 'email_1', to: 'exit', type: 'continue' },
                ],
            })
            .build()
        await insertHogFlow(hub.postgres, hogFlow)

        // Trigger the workflow via the events consumer (real Kafka producer + v2 queue)
        const { backgroundTask } = await eventsConsumer.processBatch([createGlobals()])
        await backgroundTask

        // Verify both metric stages fire EXACTLY ONCE — regression guard against
        // a double-counting bug where each metric (incl. unrelated ones like the
        // exit_node 'succeeded') was being pushed twice per invocation. The
        // AppMetricsAggregator dedupes by key in-memory, so we sum `count` across
        // all messages rather than counting messages: one push at count=1 looks
        // identical to two pushes at count=1 unless we sum.
        await waitForExpect(() => {
            const sumCounts = (filter: (m: any) => boolean) =>
                mockProducerObserver
                    .getProducedKafkaMessagesForTopic(KAFKA_APP_METRICS_2)
                    .filter((m: any) => m.value.app_source === 'hog_flow')
                    .filter(filter)
                    .reduce((sum: number, m: any) => sum + m.value.count, 0)

            expect(sumCounts((m) => m.value.metric_name === 'email_queued')).toBe(1)
            expect(sumCounts((m) => m.value.metric_name === 'email_sent')).toBe(1)
            // The exit action's 'succeeded' metric has nothing to do with the email
            // pipeline — including it locks down that the doubling isn't email-specific.
            // (The instance_id matches the action key from FixtureHogFlowBuilder.)
            expect(sumCounts((m) => m.value.metric_name === 'succeeded' && m.value.instance_id === 'exit')).toBe(1)
        }, 15000)

        // Workflow should reach a terminal state once the email worker has continued through exit
        await waitForExpect(async () => {
            const jobs = await queryCyclotronJobs()
            const terminal = jobs.filter(
                (j: any) => j.status === 'completed' || j.status === 'failed' || j.status === 'canceled'
            )
            expect(terminal.length).toBeGreaterThanOrEqual(1)
        }, 10000)
    })

    it('skips a predicted hard bounce before the email queue and completes the workflow', async () => {
        // Locks down the pipeline sequencing the unit tests can't: the MX-validation
        // skip happens on the hogflow worker BEFORE routeEmailToQueue, so a dead-domain
        // recipient must produce no email_queued/email_sent, no billable_invocation,
        // exactly one email_bounce_prevented, and a workflow that still runs to exit
        // instead of wedging on the email queue.
        const hogFlow = new FixtureHogFlowBuilder()
            .withTeamId(team.id)
            .withStatus('active')
            .withExitCondition('exit_only_at_end')
            .withWorkflow({
                actions: {
                    trigger: {
                        type: 'trigger',
                        config: { type: 'event', filters: HOG_FILTERS_EXAMPLES.no_filters.filters ?? {} },
                    },
                    email_1: {
                        type: 'function_email',
                        config: {
                            template_id: 'template-workflows-e2e-email',
                            inputs: {
                                email: {
                                    value: {
                                        to: { email: '[email protected]', name: 'Recipient' },
                                        from: { integrationId: 1, email: '[email protected]' },
                                        subject: 'Predicted bounce',
                                        text: 'Should never send',
                                        html: '<p>Should never send</p>',
                                    },
                                },
                            },
                        },
                    },
                    exit: { type: 'exit', config: {} },
                },
                edges: [
                    { from: 'trigger', to: 'email_1', type: 'continue' },
                    { from: 'email_1', to: 'exit', type: 'continue' },
                ],
            })
            .build()
        await insertHogFlow(hub.postgres, hogFlow)

        const { backgroundTask } = await eventsConsumer.processBatch([createGlobals()])
        await backgroundTask

        await waitForExpect(() => {
            const sumCounts = (filter: (m: any) => boolean) =>
                mockProducerObserver
                    .getProducedKafkaMessagesForTopic(KAFKA_APP_METRICS_2)
                    .filter((m: any) => m.value.app_source === 'hog_flow')
                    .filter(filter)
                    .reduce((sum: number, m: any) => sum + m.value.count, 0)

            // Wait for the two positive signals first — once the exit 'succeeded' metric
            // has flushed, the absence of email metrics below is meaningful, since
            // email_queued would have been emitted earlier in the pipeline.
            expect(sumCounts((m) => m.value.metric_name === 'email_bounce_prevented')).toBe(1)
            expect(sumCounts((m) => m.value.metric_name === 'succeeded' && m.value.instance_id === 'exit')).toBe(1)

            expect(sumCounts((m) => m.value.metric_name === 'email_queued')).toBe(0)
            expect(sumCounts((m) => m.value.metric_name === 'email_sent')).toBe(0)
            expect(sumCounts((m) => m.value.metric_name === 'email_failed')).toBe(0)
            expect(sumCounts((m) => m.value.metric_name === 'billable_invocation')).toBe(0)
        }, 15000)

        await waitForExpect(async () => {
            const jobs = await queryCyclotronJobs()
            expect(jobs.filter((j: any) => j.status === 'completed').length).toBeGreaterThanOrEqual(1)
            expect(jobs.filter((j: any) => j.status === 'failed').length).toBe(0)
        }, 10000)
    })

    it('sends up to the frequency cap per person, then skips and completes the run', async () => {
        // The email step runs on the hogflow worker and again on the email worker. Each pass checks the cap,
        // so with a cap of one the first run only sends if the second pass reuses its slot.
        await hub.postgres.query(
            PostgresUse.COMMON_WRITE,
            `INSERT INTO workflows_teamworkflowsconfig
                (team_id, capture_workflows_engagement_events, email_tracking_consent_mode,
                 email_sending_suspension_reason, ses_tenant_sending_status, email_sending_tier,
                 marketing_frequency_cap_max_messages, marketing_frequency_cap_window_days)
             VALUES ($1, false, 'off', '', '', 0, 1, 1)`,
            [team.id],
            'set-team-frequency-cap'
        )
        const frequencyCapValkey = createFrequencyCapValkeyPool(hub)!
        await deleteKeysWithPrefix(frequencyCapValkey, `@posthog/workflows-frequency-cap/${team.id}/`)

        // The cap is per person, and these workers resolve the person from the event's distinct_id.
        const person: InternalPersonWithDistinctId = {
            id: '1',
            uuid: '5b0c4a6e-2f0c-4a43-9c1e-7f6a1d0b9e21',
            team_id: team.id,
            properties: { email: '[email protected]' },
            properties_last_updated_at: {},
            properties_last_operation: null,
            created_at: DateTime.utc(),
            version: 1,
            is_identified: true,
            is_user_id: null,
            last_seen_at: null,
            distinct_id: 'capped-distinct-id',
        }
        const personDeps = {
            ...deps,
            personRepository: {
                fetchPerson: jest.fn().mockResolvedValue(person),
                fetchPersonsByDistinctIds: jest.fn().mockResolvedValue([person]),
                fetchPersonsByPersonIds: jest.fn().mockResolvedValue([person]),
                fetchDistinctIdsForPersons: jest.fn().mockResolvedValue({}),
            },
        }
        await Promise.all([hogflowWorker.stop(), emailWorker.stop()])
        hogflowWorker = new CdpCyclotronWorkerHogFlow(
            hub,
            personDeps,
            new CyclotronJobQueuePostgresV2(hub.CONSUMER_BATCH_SIZE, hub)
        )
        emailWorker = new CdpCyclotronWorkerEmail(
            hub,
            personDeps,
            new CyclotronJobQueuePostgresV2(hub.CONSUMER_BATCH_SIZE, hub)
        )
        await Promise.all([hogflowWorker.start(), emailWorker.start()])

        const hogFlow = new FixtureHogFlowBuilder()
            .withTeamId(team.id)
            .withStatus('active')
            .withExitCondition('exit_only_at_end')
            .withWorkflow({
                actions: {
                    trigger: {
                        type: 'trigger',
                        config: { type: 'event', filters: HOG_FILTERS_EXAMPLES.no_filters.filters ?? {} },
                    },
                    email_1: {
                        type: 'function_email',
                        config: {
                            template_id: 'template-workflows-e2e-email',
                            inputs: {
                                email: {
                                    value: {
                                        to: { email: '[email protected]', name: 'Recipient' },
                                        from: { integrationId: 1, email: '[email protected]' },
                                        subject: 'Frequency cap',
                                        text: 'Test text',
                                        html: '<p>Test html</p>',
                                    },
                                },
                            },
                        },
                    },
                    exit: { type: 'exit', config: {} },
                },
                edges: [
                    { from: 'trigger', to: 'email_1', type: 'continue' },
                    { from: 'email_1', to: 'exit', type: 'continue' },
                ],
            })
            .build()
        await insertHogFlow(hub.postgres, hogFlow)

        const { backgroundTask } = await eventsConsumer.processBatch([
            createGlobals({ uuid: new UUIDT().toString(), distinct_id: 'capped-distinct-id' } as any),
            createGlobals({ uuid: new UUIDT().toString(), distinct_id: 'capped-distinct-id' } as any),
        ])
        await backgroundTask

        await waitForExpect(() => {
            const sumCounts = (filter: (m: any) => boolean) =>
                mockProducerObserver
                    .getProducedKafkaMessagesForTopic(KAFKA_APP_METRICS_2)
                    .filter((m: any) => m.value.app_source === 'hog_flow')
                    .filter(filter)
                    .reduce((sum: number, m: any) => sum + m.value.count, 0)

            expect(sumCounts((m) => m.value.metric_name === 'succeeded' && m.value.instance_id === 'exit')).toBe(2)
            expect(sumCounts((m) => m.value.metric_name === 'email_sent')).toBe(1)
            expect(sumCounts((m) => m.value.metric_name === 'message_frequency_capped')).toBe(1)
        }, 15000)

        const jobs = await queryCyclotronJobs()
        expect(jobs.filter((j: any) => j.status === 'completed')).toHaveLength(2)
        expect(jobs.filter((j: any) => j.status === 'failed')).toHaveLength(0)
    })

    it('does not emit duplicate Resuming / Executing / pause logs for the email-queue routing reschedule', async () => {
        // Email steps reschedule themselves once to switch onto the dedicated email queue
        // (see HogFunctionHandler.execute in actions/hog_function.ts). That second dequeue
        // continues the *same* action and would otherwise re-emit "Resuming workflow execution
        // at Email", "Executing action Email", and a "Workflow will pause until <basically
        // now>" line — leaking the internal queue routing into customer-visible logs.
        //
        // The fix tags the action state with `routingOnlyReschedule: true` on the rescheduling
        // dequeue and consumes it on the next dequeue to suppress those three lines. This test
        // is the regression guard: trigger → email → exit should produce exactly one trigger
        // log, one "Executing action [Action:email_1]" line, one "Email sent" line, and no
        // routing-flavored pause / resume noise.
        const hogFlow = new FixtureHogFlowBuilder()
            .withTeamId(team.id)
            .withStatus('active')
            .withExitCondition('exit_only_at_end')
            .withWorkflow({
                actions: {
                    trigger: {
                        type: 'trigger',
                        config: { type: 'event', filters: HOG_FILTERS_EXAMPLES.no_filters.filters ?? {} },
                    },
                    email_1: {
                        type: 'function_email',
                        config: {
                            template_id: 'template-workflows-e2e-email',
                            inputs: {
                                email: {
                                    value: {
                                        to: { email: '[email protected]', name: 'Recipient' },
                                        from: { integrationId: 1, email: '[email protected]' },
                                        subject: 'Routing-reschedule log test',
                                        text: 'Test text',
                                        html: '<p>Test html</p>',
                                    },
                                },
                            },
                        },
                    },
                    exit: { type: 'exit', config: {} },
                },
                edges: [
                    { from: 'trigger', to: 'email_1', type: 'continue' },
                    { from: 'email_1', to: 'exit', type: 'continue' },
                ],
            })
            .build()
        await insertHogFlow(hub.postgres, hogFlow)

        const { backgroundTask } = await eventsConsumer.processBatch([createGlobals()])
        await backgroundTask

        // Wait for the workflow to terminate so all logs from both dequeues have been produced.
        await waitForExpect(async () => {
            const jobs = await queryCyclotronJobs()
            const terminal = jobs.filter(
                (j: any) => j.status === 'completed' || j.status === 'failed' || j.status === 'canceled'
            )
            expect(terminal.length).toBeGreaterThanOrEqual(1)
        }, 15000)

        // Collect every log entry produced by this hogflow run from the Kafka topic.
        const logMessages = mockProducerObserver
            .getProducedKafkaMessagesForTopic(KAFKA_LOG_ENTRIES)
            .map((m: any) => m.value.message as string)

        // Sanity check: the test wired up correctly (email actually sent).
        expect(logMessages.some((msg) => msg.includes('Email sent to [email protected]'))).toBe(true)

        // The email action's "Executing action" debug log must fire EXACTLY ONCE despite the
        // two dequeues it takes to switch queues. Anchor on the action id ('email_1') so we
        // don't accidentally also match the trigger or exit action's lines.
        const executingEmailLogs = logMessages.filter((msg) => msg === 'Executing action [Action:email_1]')
        expect(executingEmailLogs).toHaveLength(1)

        // The "Resuming workflow execution at" log fires at most once per dequeue — and the
        // routing-continuation dequeue should be silent. So we should never see a Resuming
        // line anchored on the email action (the first dequeue Starts at the trigger, not the
        // email step).
        const resumingEmailLogs = logMessages.filter(
            (msg) => msg.includes('Resuming workflow execution at') && msg.includes('[Action:email_1]')
        )
        expect(resumingEmailLogs).toHaveLength(0)

        // No "Workflow will pause until" lines either — the only pause in this workflow is the
        // sub-millisecond routing reschedule, which the suppression should hide. Real pauses
        // (delays, wait_until_condition, SES throttle retries) still log normally; they're
        // covered by other tests in this file and aren't exercised here.
        const pauseLogs = logMessages.filter((msg) => msg.startsWith('Workflow will pause until'))
        expect(pauseLogs).toHaveLength(0)
    })

    it('re-routes between hogflow and email queues across email → fetch → email', async () => {
        // Exercises the full ping-pong:
        //   hogflow worker → email queue (email_1) → email worker sends → routes back to hogflow
        //   → hogflow worker does fetch → email queue (email_2) → email worker sends → exits
        // Proves queueMetadata.originQueue is honored on the return trip from the email worker.
        await insertHogFunctionTemplate(hub.postgres, {
            id: 'template-workflows-e2e-fetch',
            name: 'Workflows E2E Fetch',
            code: `
            let res := fetch(inputs.url, {'method': inputs.method});
            print('Fetch result:', res.status);
            `,
            inputs_schema: [
                { key: 'url', type: 'string', required: true },
                { key: 'method', type: 'string', required: false },
            ],
        })

        mockFetch.mockResolvedValue({
            status: 200,
            headers: { 'Content-Type': 'application/json' },
            json: () => Promise.resolve({ success: true }),
            text: () => Promise.resolve(JSON.stringify({ success: true })),
            dump: () => Promise.resolve(),
        })

        const emailAction = (label: string) => ({
            type: 'function_email' as const,
            config: {
                template_id: 'template-workflows-e2e-email',
                inputs: {
                    email: {
                        value: {
                            to: { email: '[email protected]', name: 'Recipient' },
                            from: { integrationId: 1, email: '[email protected]' },
                            subject: label,
                            text: label,
                            html: `<p>${label}</p>`,
                        },
                    },
                },
            },
        })

        const hogFlow = new FixtureHogFlowBuilder()
            .withTeamId(team.id)
            .withStatus('active')
            .withExitCondition('exit_only_at_end')
            .withWorkflow({
                actions: {
                    trigger: {
                        type: 'trigger',
                        config: { type: 'event', filters: HOG_FILTERS_EXAMPLES.no_filters.filters ?? {} },
                    },
                    email_1: emailAction('First email'),
                    fetch_1: {
                        type: 'function',
                        config: {
                            template_id: 'template-workflows-e2e-fetch',
                            inputs: {
                                url: { value: 'https://example.com/between-emails' },
                                method: { value: 'POST' },
                            },
                        },
                    },
                    email_2: emailAction('Second email'),
                    exit: { type: 'exit', config: {} },
                },
                edges: [
                    { from: 'trigger', to: 'email_1', type: 'continue' },
                    { from: 'email_1', to: 'fetch_1', type: 'continue' },
                    { from: 'fetch_1', to: 'email_2', type: 'continue' },
                    { from: 'email_2', to: 'exit', type: 'continue' },
                ],
            })
            .build()
        await insertHogFlow(hub.postgres, hogFlow)

        const { backgroundTask } = await eventsConsumer.processBatch([createGlobals()])
        await backgroundTask

        // The fetch must fire exactly once — happens on the hogflow worker between
        // the two email queue hops. If it never fires, the email worker is failing
        // to route back to hogflow after the first send.
        await waitForExpect(() => {
            expect(mockFetch).toHaveBeenCalledTimes(1)
            expect(mockFetch).toHaveBeenCalledWith(
                'https://example.com/between-emails',
                expect.objectContaining({ method: 'POST' })
            )
        }, 15000)

        // Two emails were queued and two were sent — one for each side of the fetch.
        // Sum `count` across all messages: with aggregator in-memory dedup we'd
        // miss a 2× per-send bug by only counting messages, not their counts.
        await waitForExpect(() => {
            const sumCounts = (filter: (m: any) => boolean) =>
                mockProducerObserver
                    .getProducedKafkaMessagesForTopic(KAFKA_APP_METRICS_2)
                    .filter((m: any) => m.value.app_source === 'hog_flow')
                    .filter(filter)
                    .reduce((sum: number, m: any) => sum + m.value.count, 0)

            expect(sumCounts((m) => m.value.metric_name === 'email_queued')).toBe(2)
            expect(sumCounts((m) => m.value.metric_name === 'email_sent')).toBe(2)
        }, 15000)

        await waitForExpect(async () => {
            const jobs = await queryCyclotronJobs()
            const terminal = jobs.filter(
                (j: any) => j.status === 'completed' || j.status === 'failed' || j.status === 'canceled'
            )
            expect(terminal.length).toBeGreaterThanOrEqual(1)
        }, 10000)
    })

    it('suppresses routing logs in both directions across an email → fetch → email ping-pong', async () => {
        // Companion regression guard to the single-email test above, extended to the full
        // ping-pong (`hogflow → email → hogflow → email → exit`). Both routing directions
        // — `routeEmailToQueue` (hogflow → email) and `routeToQueue` (email → hogflow,
        // taken when a fetch action follows an email send) — go through the same
        // `finished: false` + nullish `queueScheduledAt` branch in HogFunctionHandler, so
        // both set `routingOnlyReschedule` and both routing dequeues should be silent in
        // the logs. This test asserts that on a four-action workflow with two emails and
        // a fetch between them, we still see exactly one Executing line per action and
        // zero Resuming-at-email/fetch lines.
        await insertHogFunctionTemplate(hub.postgres, {
            id: 'template-workflows-e2e-fetch',
            name: 'Workflows E2E Fetch',
            code: `
            let res := fetch(inputs.url, {'method': inputs.method});
            print('Fetch result:', res.status);
            `,
            inputs_schema: [
                { key: 'url', type: 'string', required: true },
                { key: 'method', type: 'string', required: false },
            ],
        })

        mockFetch.mockResolvedValue({
            status: 200,
            headers: { 'Content-Type': 'application/json' },
            json: () => Promise.resolve({ success: true }),
            text: () => Promise.resolve(JSON.stringify({ success: true })),
            dump: () => Promise.resolve(),
        })

        const emailAction = (label: string) => ({
            type: 'function_email' as const,
            config: {
                template_id: 'template-workflows-e2e-email',
                inputs: {
                    email: {
                        value: {
                            to: { email: '[email protected]', name: 'Recipient' },
                            from: { integrationId: 1, email: '[email protected]' },
                            subject: label,
                            text: label,
                            html: `<p>${label}</p>`,
                        },
                    },
                },
            },
        })

        const hogFlow = new FixtureHogFlowBuilder()
            .withTeamId(team.id)
            .withStatus('active')
            .withExitCondition('exit_only_at_end')
            .withWorkflow({
                actions: {
                    trigger: {
                        type: 'trigger',
                        config: { type: 'event', filters: HOG_FILTERS_EXAMPLES.no_filters.filters ?? {} },
                    },
                    email_1: emailAction('Ping-pong email 1'),
                    fetch_1: {
                        type: 'function',
                        config: {
                            template_id: 'template-workflows-e2e-fetch',
                            inputs: {
                                url: { value: 'https://example.com/ping-pong-fetch' },
                                method: { value: 'POST' },
                            },
                        },
                    },
                    email_2: emailAction('Ping-pong email 2'),
                    exit: { type: 'exit', config: {} },
                },
                edges: [
                    { from: 'trigger', to: 'email_1', type: 'continue' },
                    { from: 'email_1', to: 'fetch_1', type: 'continue' },
                    { from: 'fetch_1', to: 'email_2', type: 'continue' },
                    { from: 'email_2', to: 'exit', type: 'continue' },
                ],
            })
            .build()
        await insertHogFlow(hub.postgres, hogFlow)

        const { backgroundTask } = await eventsConsumer.processBatch([createGlobals()])
        await backgroundTask

        // Wait for both emails sent + the workflow terminated, so all four routing
        // reschedules have happened and all their logs are in Kafka.
        await waitForExpect(() => {
            const sumCounts = (filter: (m: any) => boolean) =>
                mockProducerObserver
                    .getProducedKafkaMessagesForTopic(KAFKA_APP_METRICS_2)
                    .filter((m: any) => m.value.app_source === 'hog_flow')
                    .filter(filter)
                    .reduce((sum: number, m: any) => sum + m.value.count, 0)

            expect(sumCounts((m) => m.value.metric_name === 'email_sent')).toBe(2)
        }, 15000)

        await waitForExpect(async () => {
            const jobs = await queryCyclotronJobs()
            const terminal = jobs.filter(
                (j: any) => j.status === 'completed' || j.status === 'failed' || j.status === 'canceled'
            )
            expect(terminal.length).toBeGreaterThanOrEqual(1)
        }, 10000)

        const logMessages = mockProducerObserver
            .getProducedKafkaMessagesForTopic(KAFKA_LOG_ENTRIES)
            .map((m: any) => m.value.message as string)

        // Sanity: both emails actually sent (not a vacuous pass where suppression broke the
        // flow). The "Email sent" log lines are prefixed with `[Action:email_X]` via
        // `actionIdForLogging`, so we substring-match instead of equality-match.
        expect(logMessages.filter((msg) => msg.includes('Email sent to [email protected]'))).toHaveLength(2)

        // Each routed action runs across two dequeues but only the "real" execution should log.
        // - email_1 routes hogflow → email, sends on the email queue
        // - fetch_1 routes email → hogflow (because the next step is a non-email function),
        //   runs on the hogflow queue
        // - email_2 routes hogflow → email, sends on the email queue
        // - exit runs inline at the tail of email_2's dequeue.
        // Trigger is NOT in this list: `ensureCurrentAction` advances `currentAction` past
        // the trigger to its successor immediately, so the trigger action itself never
        // reaches the "Executing action" log site.
        for (const actionId of ['email_1', 'fetch_1', 'email_2', 'exit']) {
            const executingLogs = logMessages.filter((msg) => msg === `Executing action [Action:${actionId}]`)
            expect(executingLogs).toHaveLength(1)
        }

        // No `Resuming workflow execution at [Action:X]` lines for any of the routed actions.
        // The first dequeue Starts at the trigger; subsequent transitions are all routing
        // reschedules or in-loop next-action advances, none of which re-enter execute()
        // with a non-suppressed flag for these actions.
        for (const actionId of ['email_1', 'fetch_1', 'email_2']) {
            const resumingLogs = logMessages.filter(
                (msg) => msg.includes('Resuming workflow execution at') && msg.includes(`[Action:${actionId}]`)
            )
            expect(resumingLogs).toHaveLength(0)
        }

        // No `Workflow will pause until X` lines anywhere — the three routing reschedules
        // (email_1, fetch_1, email_2) are all sub-millisecond and have to be silenced.
        // The workflow has no delays or wait_until_condition steps so any pause log here
        // would be the routing leak we're guarding against.
        const pauseLogs = logMessages.filter((msg) => msg.startsWith('Workflow will pause until'))
        expect(pauseLogs).toHaveLength(0)
    })

    it('keeps logging real pauses (delay before email) while still suppressing the routing reschedule', async () => {
        // Counter-example test: the suppression must NOT silence real pauses. A workflow
        // with `trigger → delay → email → exit` produces two reschedules:
        //   1. The delay action returns an explicit `queueScheduledAt` 0.5s in the future
        //      (real pause — must keep logging "Workflow will pause until X" and the
        //      corresponding "Resuming workflow execution at [Action:delay_1]" on wake).
        //   2. The email action returns no `queueScheduledAt` (routing-only — must be
        //      silent in both directions).
        // If the fix over-reaches and suppresses real delay pauses, this test fails.
        const hogFlow = new FixtureHogFlowBuilder()
            .withTeamId(team.id)
            .withStatus('active')
            .withExitCondition('exit_only_at_end')
            .withWorkflow({
                actions: {
                    trigger: {
                        type: 'trigger',
                        config: { type: 'event', filters: HOG_FILTERS_EXAMPLES.no_filters.filters ?? {} },
                    },
                    delay_1: { type: 'delay', config: { delay_duration: '0.5s' } },
                    email_1: {
                        type: 'function_email',
                        config: {
                            template_id: 'template-workflows-e2e-email',
                            inputs: {
                                email: {
                                    value: {
                                        to: { email: '[email protected]', name: 'Recipient' },
                                        from: { integrationId: 1, email: '[email protected]' },
                                        subject: 'After-delay email',
                                        text: 'After-delay email',
                                        html: '<p>After-delay email</p>',
                                    },
                                },
                            },
                        },
                    },
                    exit: { type: 'exit', config: {} },
                },
                edges: [
                    { from: 'trigger', to: 'delay_1', type: 'continue' },
                    { from: 'delay_1', to: 'email_1', type: 'continue' },
                    { from: 'email_1', to: 'exit', type: 'continue' },
                ],
            })
            .build()
        await insertHogFlow(hub.postgres, hogFlow)

        const { backgroundTask } = await eventsConsumer.processBatch([createGlobals()])
        await backgroundTask

        await waitForExpect(() => {
            const sumCounts = (filter: (m: any) => boolean) =>
                mockProducerObserver
                    .getProducedKafkaMessagesForTopic(KAFKA_APP_METRICS_2)
                    .filter((m: any) => m.value.app_source === 'hog_flow')
                    .filter(filter)
                    .reduce((sum: number, m: any) => sum + m.value.count, 0)
            expect(sumCounts((m) => m.value.metric_name === 'email_sent')).toBe(1)
        }, 15000)

        await waitForExpect(async () => {
            const jobs = await queryCyclotronJobs()
            const terminal = jobs.filter(
                (j: any) => j.status === 'completed' || j.status === 'failed' || j.status === 'canceled'
            )
            expect(terminal.length).toBeGreaterThanOrEqual(1)
        }, 10000)

        const logMessages = mockProducerObserver
            .getProducedKafkaMessagesForTopic(KAFKA_LOG_ENTRIES)
            .map((m: any) => m.value.message as string)

        expect(logMessages.filter((msg) => msg.includes('Email sent to [email protected]'))).toHaveLength(1)

        // The delay is a genuine pause and must still be logged; the email's routing hop onto
        // the email queue must NOT add a duplicate. Assert exactly one pause and one resume line
        // overall, independent of which action each references — so the guard tests the
        // suppression's intent and stays valid regardless of whether the delay advances
        // currentAction before parking.
        const pauseLogs = logMessages.filter((msg) => msg.startsWith('Workflow will pause until'))
        expect(pauseLogs).toHaveLength(1)
        const resumeLogs = logMessages.filter((msg) => msg.includes('Resuming workflow execution at'))
        expect(resumeLogs).toHaveLength(1)
    })

    it('wakes a wait_until_condition parked on the email queue after an email step', async () => {
        // Reproduces the prod bug: an email step routes the invocation to the email queue, so the
        // following wait_until_condition parks on the email queue (not hogflow). The matcher must
        // still find and wake it there — otherwise a matching event never wakes the job and the
        // post-wait email is never sent.
        const emailAction = (label: string) => ({
            type: 'function_email' as const,
            config: {
                template_id: 'template-workflows-e2e-email',
                inputs: {
                    email: {
                        value: {
                            to: { email: '[email protected]', name: 'Recipient' },
                            from: { integrationId: 1, email: '[email protected]' },
                            subject: label,
                            text: label,
                            html: `<p>${label}</p>`,
                        },
                    },
                },
            },
        })

        const hogFlow = new FixtureHogFlowBuilder()
            .withTeamId(team.id)
            .withStatus('active')
            .withExitCondition('exit_only_at_end')
            .withWorkflow({
                actions: {
                    trigger: {
                        type: 'trigger',
                        config: { type: 'event', filters: HOG_FILTERS_EXAMPLES.no_filters.filters ?? {} },
                    },
                    email_1: emailAction('First email'),
                    wait_condition: {
                        type: 'wait_until_condition',
                        config: {
                            // Property condition never matches, so only the event can wake the job.
                            condition: { filters: HOG_FILTERS_EXAMPLES.elements_text_filter.filters },
                            events: [eventNameFilter('wakeup_event')],
                            // Long enough that the job stays parked for the whole test — the only way
                            // the second email sends is the matcher waking it, never a timeout.
                            max_wait_duration: '5m',
                        },
                    },
                    email_2: emailAction('Second email'),
                    exit: { type: 'exit', config: {} },
                },
                edges: [
                    { from: 'trigger', to: 'email_1', type: 'continue' },
                    { from: 'email_1', to: 'wait_condition', type: 'continue' },
                    { from: 'wait_condition', to: 'email_2', type: 'branch', index: 0 },
                    { from: 'wait_condition', to: 'exit', type: 'continue' },
                    { from: 'email_2', to: 'exit', type: 'continue' },
                ],
            })
            .build()
        await insertHogFlow(hub.postgres, hogFlow)

        const emailsSent = () =>
            mockProducerObserver
                .getProducedKafkaMessagesForTopic(KAFKA_APP_METRICS_2)
                .filter((m: any) => m.value.app_source === 'hog_flow')
                .filter((m: any) => m.value.metric_name === 'email_sent')
                .reduce((sum: number, m: any) => sum + m.value.count, 0)

        // Trigger: email_1 routes to the email queue, the email worker sends it and continues the
        // flow to the wait step, which parks.
        const { backgroundTask } = await eventsConsumer.processBatch([createGlobals()])
        await backgroundTask

        // The first email is sent and the job parks waiting for the event.
        await waitForExpect(async () => {
            expect(emailsSent()).toBe(1)
            const jobs = await queryCyclotronJobs()
            expect(jobs.some((j: any) => j.status === 'available' && new Date(j.scheduled) > new Date())).toBe(true)
        }, 15000)

        // The wait parks on the email queue (carried over from email_1). The matcher has to find
        // it there regardless of queue — that's exactly the scenario that was broken.
        const parked = (await queryCyclotronJobs()).find(
            (j: any) => j.status === 'available' && new Date(j.scheduled) > new Date()
        )
        expect(parked).toBeDefined()
        expect(parked?.queue_name).toBe('email')
        expect(emailsSent()).toBe(1)

        // The subscribed event fires for this person — the matcher wakes the parked job even though
        // it sits on the email queue, the email worker resumes it down the matched branch, and the
        // second email is sent. This is the end-to-end "the job wakes and the next step runs" check.
        matcher = new CdpHogflowSubscriptionMatcherConsumer({ ...hub }, deps)
        await matcher.processBatch([createGlobals({ event: 'wakeup_event' })])

        await waitForExpect(() => {
            expect(emailsSent()).toBe(2)
        }, 15000)
    })

    it("a workflow over its sending limit does not hold up another workflow's emails", async () => {
        // Two workflows on the same team and the same email queue. Workflow A has a
        // sending limit of 2 per minute and gets 8 sends queued at once, so all but the
        // first are denied. Workflow B has no limit and 3 sends. The whole pipeline is
        // real: events consumer -> hogflow worker -> email queue -> email worker ->
        // rate limiter -> reschedule. B's emails must go out while A's denied sends
        // park on their own future slots, each dequeued once, with the job row's
        // transition counter staying low.
        const buildEmailFlow = (triggerEvent: string, recipient: string): HogFlow =>
            new FixtureHogFlowBuilder()
                .withTeamId(team.id)
                .withStatus('active')
                .withExitCondition('exit_only_at_end')
                .withWorkflow({
                    actions: {
                        trigger: {
                            type: 'trigger',
                            config: { type: 'event', ...eventNameFilter(triggerEvent) },
                        },
                        email_1: {
                            type: 'function_email',
                            config: {
                                template_id: 'template-workflows-e2e-email',
                                inputs: {
                                    email: {
                                        value: {
                                            to: { email: recipient, name: 'Recipient' },
                                            from: { integrationId: 1, email: '[email protected]' },
                                            subject: `To ${recipient}`,
                                            text: 'Test text',
                                            html: '<p>Test html</p>',
                                        },
                                    },
                                },
                            },
                        },
                        exit: { type: 'exit', config: {} },
                    },
                    edges: [
                        { from: 'trigger', to: 'email_1', type: 'continue' },
                        { from: 'email_1', to: 'exit', type: 'continue' },
                    ],
                })
                .build()

        const flowA = buildEmailFlow('signup_a', '[email protected]')
        const flowB = buildEmailFlow('signup_b', '[email protected]')
        await insertHogFlow(hub.postgres, flowA)
        await insertHogFlow(hub.postgres, flowB)
        // 2 per minute: one burst send, then one slot every 30s. The denied sends park
        // 30s+ out, far past this test's clock, so they stay parked.
        await hub.postgres.query(
            PostgresUse.COMMON_WRITE,
            `UPDATE posthog_hogflow SET email_sending_rate_limit = $1 WHERE id = $2`,
            [JSON.stringify({ count: 2, period: 'minute' }), flowA.id],
            'set-email-sending-rate-limit'
        )

        const events = [
            ...Array.from({ length: 8 }, (_, i) =>
                createGlobals({ event: 'signup_a', uuid: new UUIDT().toString(), distinct_id: `person_a_${i}` } as any)
            ),
            ...Array.from({ length: 3 }, (_, i) =>
                createGlobals({ event: 'signup_b', uuid: new UUIDT().toString(), distinct_id: `person_b_${i}` } as any)
            ),
        ]
        const { backgroundTask } = await eventsConsumer.processBatch(events)
        await backgroundTask

        // B's 3 runs finish end to end, and A gets its one in-budget send out: 4 sends total.
        await waitForExpect(async () => {
            const jobs = await queryCyclotronJobs()
            const bJobs = jobs.filter((j: any) => j.function_id === flowB.id)
            expect(bJobs.length).toBe(3)
            expect(bJobs.every((j: any) => j.status === 'completed')).toBe(true)

            const sent = mockProducerObserver
                .getProducedKafkaMessagesForTopic(KAFKA_APP_METRICS_2)
                .filter((m: any) => m.value.app_source === 'hog_flow')
                .filter((m: any) => m.value.metric_name === 'email_sent')
                .reduce((sum: number, m: any) => sum + m.value.count, 0)
            expect(sent).toBe(4)
        }, 15000)

        // A's 7 denied sends are parked out of the way: still queued, each on its own
        // future slot, each dequeued exactly once on the email queue.
        const jobs = await queryCyclotronJobs()
        const parkedA = jobs.filter(
            (j: any) => j.function_id === flowA.id && j.queue_name === 'email' && j.status === 'available'
        )
        expect(parkedA.length).toBe(7)
        const slots = parkedA.map((j: any) => new Date(j.scheduled).getTime())
        expect(new Set(slots).size).toBe(7)
        for (const slot of slots) {
            expect(slot).toBeGreaterThan(Date.now() + 20_000)
        }
        // Trigger enqueue, hogflow pass (dequeue + route to email), email pass
        // (dequeue + park) is at most 4 transitions. Anything higher means a
        // denied send went around the loop again.
        for (const job of parkedA) {
            expect(job.transition_count).toBeLessThanOrEqual(4)
        }
    })

    it("a team over its tier cap does not hold up another team's emails", async () => {
        // Two teams, each with one email workflow. Team A sits on a tier that allows
        // 2 emails per day and gets 6 sends queued; team B is on a high tier with 3
        // sends. The whole pipeline is real and the tier caps are enforced: B's emails
        // must all go out while A's over-cap sends park on future slots from A's own
        // refill, without cycling.
        //
        // The default email worker was built with the caps off, so swap in one that
        // enforces them. Tier 0 allows 2/day, tier 1 is effectively unlimited.
        await emailWorker.stop()
        hub.EMAIL_TEAM_SENDING_CAP_MODE = 'enforce'
        hub.EMAIL_TEAM_SENDING_CAP_DAILY_BY_TIER = '2,240000'
        const enforcedQueue = new CyclotronJobQueuePostgresV2(hub.CONSUMER_BATCH_SIZE, hub)
        emailWorker = new CdpCyclotronWorkerEmail(hub, deps, enforcedQueue)
        await emailWorker.start()

        // The tier bucket is keyed by team id and the test Redis keeps data between
        // runs, so team A's bucket must start full or a previous run drains this one.
        const capValkey = createRedisV2PoolFromConfig({
            connection: hub.CDP_REDIS_HOST
                ? {
                      url: hub.CDP_REDIS_HOST,
                      options: { port: hub.CDP_REDIS_PORT, password: hub.CDP_REDIS_PASSWORD },
                  }
                : { url: hub.REDIS_URL },
            poolMinSize: hub.REDIS_POOL_MIN_SIZE,
            poolMaxSize: hub.REDIS_POOL_MAX_SIZE,
        })
        await deleteKeysWithPrefix(capValkey, '@posthog/team-email-rate-day')

        // Team A has no config row and defaults to tier 0. Team B gets tier 1.
        const teamBId = await createTeam(hub.postgres, team.organization_id)
        await hub.postgres.query(
            PostgresUse.COMMON_WRITE,
            `INSERT INTO workflows_teamworkflowsconfig
                (team_id, capture_workflows_engagement_events, email_tracking_consent_mode,
                 email_sending_suspension_reason, ses_tenant_sending_status, email_sending_tier)
             VALUES ($1, false, 'off', '', '', 1)`,
            [teamBId],
            'set-team-email-tier'
        )
        await insertIntegration(hub.postgres, teamBId, {
            id: 2,
            kind: 'email',
            config: {
                email: '[email protected]',
                name: 'Test Sender',
                domain: 'posthog.com',
                verified: true,
                provider: 'maildev',
            },
        })

        const buildFlow = (teamId: number, integrationId: number, triggerEvent: string, recipient: string): HogFlow =>
            new FixtureHogFlowBuilder()
                .withTeamId(teamId)
                .withStatus('active')
                .withExitCondition('exit_only_at_end')
                .withWorkflow({
                    actions: {
                        trigger: {
                            type: 'trigger',
                            config: { type: 'event', ...eventNameFilter(triggerEvent) },
                        },
                        email_1: {
                            type: 'function_email',
                            config: {
                                template_id: 'template-workflows-e2e-email',
                                inputs: {
                                    email: {
                                        value: {
                                            to: { email: recipient, name: 'Recipient' },
                                            from: { integrationId, email: '[email protected]' },
                                            subject: `To ${recipient}`,
                                            text: 'Test text',
                                            html: '<p>Test html</p>',
                                        },
                                    },
                                },
                            },
                        },
                        exit: { type: 'exit', config: {} },
                    },
                    edges: [
                        { from: 'trigger', to: 'email_1', type: 'continue' },
                        { from: 'email_1', to: 'exit', type: 'continue' },
                    ],
                })
                .build()

        const flowA = buildFlow(team.id, 1, 'tier_a', '[email protected]')
        const flowB = buildFlow(teamBId, 2, 'tier_b', '[email protected]')
        await insertHogFlow(hub.postgres, flowA)
        await insertHogFlow(hub.postgres, flowB)

        const teamBGlobals = (i: number): HogFunctionInvocationGlobals =>
            createHogExecutionGlobals({
                project: { id: teamBId } as any,
                event: {
                    uuid: new UUIDT().toString(),
                    event: 'tier_b',
                    distinct_id: `person_tier_b_${i}`,
                    properties: {},
                    timestamp: '2024-09-03T09:00:00Z',
                } as any,
            })
        const events = [
            ...Array.from({ length: 6 }, (_, i) =>
                createGlobals({
                    event: 'tier_a',
                    uuid: new UUIDT().toString(),
                    distinct_id: `person_tier_a_${i}`,
                } as any)
            ),
            ...Array.from({ length: 3 }, (_, i) => teamBGlobals(i)),
        ]
        const { backgroundTask } = await eventsConsumer.processBatch(events)
        await backgroundTask

        // Team B's 3 runs finish end to end; team A gets its 2 in-budget sends out.
        await waitForExpect(async () => {
            const jobs = await queryCyclotronJobs()
            const bJobs = jobs.filter((j: any) => j.function_id === flowB.id)
            expect(bJobs.length).toBe(3)
            expect(bJobs.every((j: any) => j.status === 'completed')).toBe(true)

            const sent = mockProducerObserver
                .getProducedKafkaMessagesForTopic(KAFKA_APP_METRICS_2)
                .filter((m: any) => m.value.app_source === 'hog_flow')
                .filter((m: any) => m.value.metric_name === 'email_sent')
                .reduce((sum: number, m: any) => sum + m.value.count, 0)
            expect(sent).toBe(5)
        }, 15000)

        // Team A's 4 over-cap sends are parked far in the future (its refill is one
        // token per 12 hours), each dequeued once on the email queue.
        const jobs = await queryCyclotronJobs()
        const parkedA = jobs.filter(
            (j: any) => j.function_id === flowA.id && j.queue_name === 'email' && j.status === 'available'
        )
        expect(parkedA.length).toBe(4)
        for (const job of parkedA) {
            expect(new Date(job.scheduled).getTime()).toBeGreaterThan(Date.now() + 25 * 60 * 1000)
            expect(job.transition_count).toBeLessThanOrEqual(4)
        }
    })

    it('rate-limited variant processes emails end-to-end through the dedicated bucket', async () => {
        // Verifies the inject-pattern wiring: CyclotronJobQueueRateLimitedPostgresV2
        // gates dequeue via a Valkey bucket, then the email worker processes the
        // job normally. Reuses the local test Redis as the bucket store (same
        // approach as rate-limiter.service.test.ts).
        await emailWorker.stop()

        const limiterValkey = createRedisV2PoolFromConfig({
            connection: hub.CDP_REDIS_HOST
                ? {
                      url: hub.CDP_REDIS_HOST,
                      options: { port: hub.CDP_REDIS_PORT, password: hub.CDP_REDIS_PASSWORD },
                  }
                : { url: hub.REDIS_URL },
            poolMinSize: hub.REDIS_POOL_MIN_SIZE,
            poolMaxSize: hub.REDIS_POOL_MAX_SIZE,
        })
        const bucketKey = '@posthog-test/ses-e2e/bucket'
        await deleteKeysWithPrefix(limiterValkey, '@posthog-test/ses-e2e/')

        const rateLimitedQueue = new CyclotronJobQueueRateLimitedPostgresV2(hub.CONSUMER_BATCH_SIZE, hub, {
            limiter: new RateLimiterService(limiterValkey, { name: 'ses-e2e' }),
            key: bucketKey,
            capacity: 10,
            refillPerSecond: 10,
            throttledPollDelayMs: 50,
        })
        emailWorker = new CdpCyclotronWorkerEmail(hub, deps, rateLimitedQueue)
        await emailWorker.start()

        const hogFlow = new FixtureHogFlowBuilder()
            .withTeamId(team.id)
            .withStatus('active')
            .withExitCondition('exit_only_at_end')
            .withWorkflow({
                actions: {
                    trigger: {
                        type: 'trigger',
                        config: { type: 'event', filters: HOG_FILTERS_EXAMPLES.no_filters.filters ?? {} },
                    },
                    email_1: {
                        type: 'function_email',
                        config: {
                            template_id: 'template-workflows-e2e-email',
                            inputs: {
                                email: {
                                    value: {
                                        to: { email: '[email protected]', name: 'Recipient' },
                                        from: { integrationId: 1, email: '[email protected]' },
                                        subject: 'Rate-limited email',
                                        text: 'Test text',
                                        html: '<p>Test html</p>',
                                    },
                                },
                            },
                        },
                    },
                    exit: { type: 'exit', config: {} },
                },
                edges: [
                    { from: 'trigger', to: 'email_1', type: 'continue' },
                    { from: 'email_1', to: 'exit', type: 'continue' },
                ],
            })
            .build()
        await insertHogFlow(hub.postgres, hogFlow)

        const { backgroundTask } = await eventsConsumer.processBatch([createGlobals()])
        await backgroundTask

        // The email is sent — the rate-limited queue gated, dequeued, and processed the job.
        await waitForExpect(() => {
            const emailSentCount = mockProducerObserver
                .getProducedKafkaMessagesForTopic(KAFKA_APP_METRICS_2)
                .filter((m: any) => m.value.app_source === 'hog_flow')
                .filter((m: any) => m.value.metric_name === 'email_sent')
                .reduce((sum: number, m: any) => sum + m.value.count, 0)
            expect(emailSentCount).toBe(1)
        }, 15000)

        // The bucket has been touched — proves the rate limiter was actually consulted,
        // not bypassed. `ts` and `pool` are written on every claim (cold start or refill).
        const bucket = await limiterValkey.useClient({ name: 'read-bucket' }, (client) => client.hgetall(bucketKey))
        expect(bucket?.ts).toBeTruthy()
        expect(bucket?.pool).toBeTruthy()
    })

    it('rate-limits dequeue when the bucket drains, then drains the queue as it refills', async () => {
        // Tiny bucket (capacity 1, refill 5/sec = 1 token every 200ms) so the
        // worker has to wait for refills between sends. With 3 emails enqueued
        // we should see the bucket get denied at least once while the worker
        // is waiting — and all 3 should still eventually go through.
        await emailWorker.stop()

        const limiterValkey = createRedisV2PoolFromConfig({
            connection: hub.CDP_REDIS_HOST
                ? {
                      url: hub.CDP_REDIS_HOST,
                      options: { port: hub.CDP_REDIS_PORT, password: hub.CDP_REDIS_PASSWORD },
                  }
                : { url: hub.REDIS_URL },
            poolMinSize: hub.REDIS_POOL_MIN_SIZE,
            poolMaxSize: hub.REDIS_POOL_MAX_SIZE,
        })
        const bucketKey = '@posthog-test/ses-e2e-throttled/bucket'
        await deleteKeysWithPrefix(limiterValkey, '@posthog-test/ses-e2e-throttled/')

        const limiterName = 'ses-e2e-throttled'
        const rateLimitedQueue = new CyclotronJobQueueRateLimitedPostgresV2(hub.CONSUMER_BATCH_SIZE, hub, {
            limiter: new RateLimiterService(limiterValkey, { name: limiterName }),
            key: bucketKey,
            capacity: 1,
            refillPerSecond: 5,
            throttledPollDelayMs: 50,
        })
        emailWorker = new CdpCyclotronWorkerEmail(hub, deps, rateLimitedQueue)
        await emailWorker.start()

        // Snapshot the denied counter so we measure only this test's claims.
        const readDeniedCount = async (): Promise<number> => {
            const metric = register.getSingleMetric('cdp_rate_limiter_claim_total')
            if (!metric) {
                return 0
            }
            const data = await metric.get()
            return data.values
                .filter((v: any) => v.labels.result === 'denied' && v.labels.limiter === limiterName)
                .reduce((sum: number, v: any) => sum + v.value, 0)
        }
        const deniedBefore = await readDeniedCount()

        const hogFlow = new FixtureHogFlowBuilder()
            .withTeamId(team.id)
            .withStatus('active')
            .withExitCondition('exit_only_at_end')
            .withWorkflow({
                actions: {
                    trigger: {
                        type: 'trigger',
                        config: { type: 'event', filters: HOG_FILTERS_EXAMPLES.no_filters.filters ?? {} },
                    },
                    email_1: {
                        type: 'function_email',
                        config: {
                            template_id: 'template-workflows-e2e-email',
                            inputs: {
                                email: {
                                    value: {
                                        to: { email: '[email protected]', name: 'Recipient' },
                                        from: { integrationId: 1, email: '[email protected]' },
                                        subject: 'Throttled email',
                                        text: 'Test text',
                                        html: '<p>Test html</p>',
                                    },
                                },
                            },
                        },
                    },
                    exit: { type: 'exit', config: {} },
                },
                edges: [
                    { from: 'trigger', to: 'email_1', type: 'continue' },
                    { from: 'email_1', to: 'exit', type: 'continue' },
                ],
            })
            .build()
        await insertHogFlow(hub.postgres, hogFlow)

        // Three distinct events → three email jobs queued near-simultaneously.
        const events = Array.from({ length: 3 }, () => createGlobals({ uuid: new UUIDT().toString() }))
        const { backgroundTask } = await eventsConsumer.processBatch(events)
        await backgroundTask

        // All three eventually send — generous timeout because the bucket only
        // refills 5 tokens/sec.
        await waitForExpect(() => {
            const emailSentCount = mockProducerObserver
                .getProducedKafkaMessagesForTopic(KAFKA_APP_METRICS_2)
                .filter((m: any) => m.value.app_source === 'hog_flow')
                .filter((m: any) => m.value.metric_name === 'email_sent')
                .reduce((sum: number, m: any) => sum + m.value.count, 0)
            expect(emailSentCount).toBe(3)
        }, 20000)

        // The bucket was denied at least once during processing — proves the
        // gating actually fired, not that we just dequeued 3 jobs in a row.
        // (With capacity=1 and three pending jobs, between sends the worker
        // polls many times finding bucket=0.)
        const deniedAfter = await readDeniedCount()
        expect(deniedAfter - deniedBefore).toBeGreaterThan(0)
    })

    it('claims only the visible row count (sparse traffic does not drain the bucket)', async () => {
        // Regression guard for the pre-size fix. Without it, a single ready
        // email would claim the bucket's full capacity — draining ~capacity-1
        // tokens of SES budget per actual send — even though the worker can
        // only dequeue one row. Pre-sizing asks the limiter for exactly the
        // number of rows the worker is about to dequeue.
        //
        // refillPerSecond=0 freezes the bucket between the claim and our
        // assertion, so the post-claim pool is a deterministic measure of
        // what was deducted (capacity minus tokens granted on the one claim).
        await emailWorker.stop()

        const limiterValkey = createRedisV2PoolFromConfig({
            connection: hub.CDP_REDIS_HOST
                ? {
                      url: hub.CDP_REDIS_HOST,
                      options: { port: hub.CDP_REDIS_PORT, password: hub.CDP_REDIS_PASSWORD },
                  }
                : { url: hub.REDIS_URL },
            poolMinSize: hub.REDIS_POOL_MIN_SIZE,
            poolMaxSize: hub.REDIS_POOL_MAX_SIZE,
        })
        const bucketKey = '@posthog-test/ses-e2e-presize/bucket'
        await deleteKeysWithPrefix(limiterValkey, '@posthog-test/ses-e2e-presize/')

        const capacity = 10
        const rateLimitedQueue = new CyclotronJobQueueRateLimitedPostgresV2(hub.CONSUMER_BATCH_SIZE, hub, {
            limiter: new RateLimiterService(limiterValkey, { name: 'ses-e2e-presize' }),
            key: bucketKey,
            capacity,
            refillPerSecond: 0,
            throttledPollDelayMs: 50,
        })
        emailWorker = new CdpCyclotronWorkerEmail(hub, deps, rateLimitedQueue)
        await emailWorker.start()

        const hogFlow = new FixtureHogFlowBuilder()
            .withTeamId(team.id)
            .withStatus('active')
            .withExitCondition('exit_only_at_end')
            .withWorkflow({
                actions: {
                    trigger: {
                        type: 'trigger',
                        config: { type: 'event', filters: HOG_FILTERS_EXAMPLES.no_filters.filters ?? {} },
                    },
                    email_1: {
                        type: 'function_email',
                        config: {
                            template_id: 'template-workflows-e2e-email',
                            inputs: {
                                email: {
                                    value: {
                                        to: { email: '[email protected]', name: 'Recipient' },
                                        from: { integrationId: 1, email: '[email protected]' },
                                        subject: 'Sparse-traffic email',
                                        text: 'Test text',
                                        html: '<p>Test html</p>',
                                    },
                                },
                            },
                        },
                    },
                    exit: { type: 'exit', config: {} },
                },
                edges: [
                    { from: 'trigger', to: 'email_1', type: 'continue' },
                    { from: 'email_1', to: 'exit', type: 'continue' },
                ],
            })
            .build()
        await insertHogFlow(hub.postgres, hogFlow)

        // Exactly ONE event → one email job — the sparse-traffic scenario.
        const { backgroundTask } = await eventsConsumer.processBatch([createGlobals({ uuid: new UUIDT().toString() })])
        await backgroundTask

        await waitForExpect(() => {
            const emailSentCount = mockProducerObserver
                .getProducedKafkaMessagesForTopic(KAFKA_APP_METRICS_2)
                .filter((m: any) => m.value.app_source === 'hog_flow')
                .filter((m: any) => m.value.metric_name === 'email_sent')
                .reduce((sum: number, m: any) => sum + m.value.count, 0)
            expect(emailSentCount).toBe(1)
        }, 15000)

        // Bucket should retain ~capacity-1 tokens — we claimed 1, not capacity.
        // Without the pre-size fix `pool` would be 0 here (the whole bucket
        // drained on the one claim).
        const bucket = await limiterValkey.useClient({ name: 'read-bucket' }, (client) => client.hgetall(bucketKey))
        const pool = parseFloat(bucket?.pool ?? '0')
        expect(pool).toBeGreaterThanOrEqual(capacity - 1)
    })

    // ---- Message-assets bulk flush at the batch boundary ----
    //
    // Email assets used to be produced one-at-a-time via a fire-and-forget Kafka call
    // from `email.service.ts → MessageAssetsService.captureSentEmail`. We've moved that
    // to a buffer-then-flush pattern that drains `result.messageAssets` at the batch
    // boundary and bulk-produces, gated on broker ack before the consumer commits
    // offsets. These tests pin the end-to-end behavior: one workflow → one asset row in
    // the `message_assets` Kafka topic with the right metadata, and a single batch with
    // multiple emails produces all rows.
    describe('message_assets bulk capture', () => {
        const buildEmailWorkflow = (subject: string) =>
            new FixtureHogFlowBuilder()
                .withTeamId(team.id)
                .withStatus('active')
                .withExitCondition('exit_only_at_end')
                .withWorkflow({
                    actions: {
                        trigger: {
                            type: 'trigger',
                            config: { type: 'event', filters: HOG_FILTERS_EXAMPLES.no_filters.filters ?? {} },
                        },
                        email_1: {
                            type: 'function_email',
                            config: {
                                template_id: 'template-workflows-e2e-email',
                                inputs: {
                                    email: {
                                        value: {
                                            to: { email: '[email protected]', name: 'Recipient' },
                                            from: { integrationId: 1, email: '[email protected]' },
                                            subject,
                                            text: 'plain text body',
                                            html: `<p>${subject}</p>`,
                                        },
                                    },
                                },
                            },
                        },
                        exit: { type: 'exit', config: {} },
                    },
                    edges: [
                        { from: 'trigger', to: 'email_1', type: 'continue' },
                        { from: 'email_1', to: 'exit', type: 'continue' },
                    ],
                })
                .build()

        const assetMessages = (): { key: string; value: any }[] =>
            mockProducerObserver.getProducedKafkaMessagesForTopic(KAFKA_MESSAGE_ASSETS) as any

        it('produces one message_assets row per workflow email with the rendered HTML and metadata', async () => {
            const hogFlow = buildEmailWorkflow('Asset bulk-capture single')
            await insertHogFlow(hub.postgres, hogFlow)

            const { backgroundTask } = await eventsConsumer.processBatch([createGlobals()])
            await backgroundTask

            // The asset row only lands once the email worker's batch flushes — wait for
            // the workflow to reach a terminal state, then assert against the topic.
            await waitForExpect(async () => {
                const jobs = await queryCyclotronJobs()
                const terminal = jobs.filter(
                    (j: any) => j.status === 'completed' || j.status === 'failed' || j.status === 'canceled'
                )
                expect(terminal.length).toBeGreaterThanOrEqual(1)
            }, 15000)

            await waitForExpect(() => {
                const rows = assetMessages()
                expect(rows).toHaveLength(1)

                const row = rows[0].value as Record<string, any>
                expect(row.team_id).toBe(team.id)
                expect(row.function_kind).toBe('hog_flow')
                expect(row.kind).toBe('email')
                expect(row.status).toBe('sent')
                expect(row.action_id).toBe('email_1')
                expect(row.recipient).toBe('[email protected]')
                expect(row.subject).toBe('Asset bulk-capture single')
                expect(row.html).toBe('<p>Asset bulk-capture single</p>')
                // Partition key must match invocation_id so retries collapse via the
                // destination ReplacingMergeTree(version).
                expect(rows[0].key).toBe(row.invocation_id)
            }, 15000)
        })

        it('bulk-captures every asset when multiple workflow runs share a batch', async () => {
            // Use distinct subjects so we can assert on row content regardless of the
            // partition-level ordering the Kafka producer chooses.
            const hogFlow = buildEmailWorkflow('Bulk asset capture')
            await insertHogFlow(hub.postgres, hogFlow)

            // Three globals dispatched in one `processBatch` call — they go through the
            // events consumer together. Each fires its own workflow run, each emits one
            // asset; the bulk-flush is what we're exercising.
            const { backgroundTask } = await eventsConsumer.processBatch([
                createGlobals({ uuid: 'aaaaaaaa-0000-0000-0000-000000000001' as any }),
                createGlobals({ uuid: 'aaaaaaaa-0000-0000-0000-000000000002' as any }),
                createGlobals({ uuid: 'aaaaaaaa-0000-0000-0000-000000000003' as any }),
            ])
            await backgroundTask

            await waitForExpect(async () => {
                const jobs = await queryCyclotronJobs()
                const terminal = jobs.filter(
                    (j: any) => j.status === 'completed' || j.status === 'failed' || j.status === 'canceled'
                )
                expect(terminal.length).toBeGreaterThanOrEqual(3)
            }, 20000)

            await waitForExpect(() => {
                const rows = assetMessages()
                expect(rows.length).toBeGreaterThanOrEqual(3)

                // Every produced row must carry distinct invocation_id values (one per
                // workflow run) — otherwise the buffer is dropping or aliasing rows.
                const invocationIds = new Set(rows.map((r) => (r.value as any).invocation_id))
                expect(invocationIds.size).toBeGreaterThanOrEqual(3)

                // Subject and HTML are constant across the three runs (same flow), so we
                // just sanity-check that every row carries the expected shape.
                for (const row of rows) {
                    const value = row.value as Record<string, any>
                    expect(value.subject).toBe('Bulk asset capture')
                    expect(value.html).toBe('<p>Bulk asset capture</p>')
                    expect(value.kind).toBe('email')
                    expect(value.status).toBe('sent')
                    expect(row.key).toBe(value.invocation_id)
                }
            }, 20000)
        })
    })
})

/**
 * E2E for the batch resolver dispatch path through the cdp-api HTTP boundary.
 *
 * Goes through real express + supertest, real CdpApi, real cyclotron-node
 * Postgres. Verifies that POST `/batch_invocations/<id>` creates a resolver
 * cyclotron job pointing at the right queue with the right state.
 *
 * The deep state-machine paths (page execution, terminal write, truncation,
 * Django down → resolver parks) are covered by the integration tests in
 * `consumers/cdp-cyclotron-worker-batch-resolve.consumer.test.ts`. This
 * test is the boundary backstop — it caught a real bug (wrong DB URL on
 * the resolver manager) during development.
 *
 * Hub lifecycle follows cdp-api.test.ts: one hub for the suite (beforeAll),
 * torn down once (afterAll). Per-test isolation comes from resetting the
 * postgres team data + truncating cyclotron_jobs in beforeEach.
 */

/**
 * E2E for the cyclotron batch resolver. POST through cdp-api, real consumer
 * loop, mocked Django endpoints, assert on the resulting cyclotron + Django
 * state. Mirrors how the system runs in prod — no manual dequeue plumbing.
 */
describe('Workflows E2E: batch resolver dispatch via cdp-api', () => {
    jest.setTimeout(60000)

    const CYCLOTRON_NODE_DB_URL =
        process.env.CYCLOTRON_NODE_DATABASE_URL ?? 'postgres://posthog:posthog@localhost:5432/test_cyclotron_node'

    let hub: Hub
    let team: Team
    let cyclotronPool: Pool
    let app: express.Application
    let server: any
    let api: CdpApi
    let batchResolverProducer: CyclotronV2Manager
    let deps: ReturnType<typeof createCdpConsumerDeps>
    let kafkaProducer: KafkaProducerWrapper
    let mockProducerObserver: KafkaProducerObserver
    // Fresh consumer per test — built in the it() body, stopped in afterEach.
    let resolverWorker: CdpCyclotronWorkerBatchResolve | undefined

    beforeAll(async () => {
        cyclotronPool = new Pool({ connectionString: CYCLOTRON_NODE_DB_URL })

        MockKafkaProducerWrapper.create = jest.fn((...args) => ActualKafkaProducerWrapper.create(...args))
        // KAFKA_HOG_INVOCATION_RESULTS isn't in TEST_KAFKA_TOPICS — the resolver produces a
        // running row per enrolled person there, so the topic has to exist or the produce fails.
        await ensureKafkaTopics([...TEST_KAFKA_TOPICS, KAFKA_HOG_INVOCATION_RESULTS])

        hub = await createHub({
            SITE_URL: 'http://localhost:8000',
        })
        // Lifecycle rows are gated on this flag, which is off by default outside dev — without
        // it the row assertions below would pass against an empty topic.
        hub.HOG_INVOCATION_RESULTS_ENABLED = true

        const { createMockJobQueue } = require('../../tests/helpers/mocks/job-queue.mock')
        kafkaProducer = await ActualKafkaProducerWrapper.create(hub.KAFKA_CLIENT_RACK)
        mockProducerObserver = new KafkaProducerObserver(kafkaProducer)
        deps = createCdpConsumerDeps(hub, kafkaProducer)
        batchResolverProducer = new CyclotronV2Manager({
            pool: { dbUrl: CYCLOTRON_NODE_DB_URL, maxConnections: 5 },
        })
        api = new CdpApi(
            hub,
            deps,
            { hogQueue: createMockJobQueue(), hogflowQueue: createMockJobQueue() },
            batchResolverProducer
        )
        app = setupExpressApp()
        app.use('/', api.router())
        server = app.listen(0, () => {})
    })

    afterAll(async () => {
        server?.close()
        await batchResolverProducer?.disconnect()
        await kafkaProducer?.disconnect()
        await closeHub(hub)
        await cyclotronPool.end()
    })

    beforeEach(async () => {
        await resetTestDatabase()
        await cyclotronPool.query(`DELETE FROM cyclotron_jobs`)
        team = await getFirstTeam(hub.postgres)
        resolverWorker = undefined
        mockProducerObserver.resetKafkaProducer()
    })

    afterEach(async () => {
        // If the test started the consumer, shut it down before the next test.
        // resolverWorker.stop() waits for any in-flight processBatch to settle,
        // then closes the pool — so afterAll's closeHub has no leftover work.
        await resolverWorker?.stop()
    })

    // Fresh worker per test — stop() disconnects the underlying Postgres pool,
    // so a shared instance can't survive past the first test's afterEach.
    // `wrapJob` lets a test intercept a dequeued job's methods (e.g. fail a commit once).
    function buildResolverConsumer(
        wrapJob?: (job: CyclotronV2DequeuedJob) => CyclotronV2DequeuedJob
    ): CdpCyclotronWorkerBatchResolve {
        const cyclotronWorker = new CyclotronV2Worker({
            pool: { dbUrl: CYCLOTRON_NODE_DB_URL, maxConnections: 10 },
            queueName: HOGFLOW_BATCH_RESOLVE_QUEUE,
            pollDelayMs: 100,
            batchMaxSize: 1,
            heartbeatTimeoutMs: hub.CDP_HOG_FLOW_BATCH_AUDIENCE_FETCH_TIMEOUT_MS + 30_000,
        })
        // The consumer only calls connect/disconnect/isHealthy on the worker.
        const workerForConsumer = wrapJob
            ? ({
                  connect: (callback: (jobs: CyclotronV2DequeuedJob[]) => Promise<void>) =>
                      cyclotronWorker.connect((jobs) => callback(jobs.map(wrapJob))),
                  disconnect: () => cyclotronWorker.disconnect(),
                  isHealthy: () => cyclotronWorker.isHealthy(),
              } as unknown as CyclotronV2Worker)
            : cyclotronWorker
        const internalFetchService = new InternalFetchService(hub.INTERNAL_API_BASE_URL, hub.INTERNAL_API_SECRET)
        const queryService = new HogFlowBatchPersonQueryService(
            internalFetchService,
            hub.CDP_HOG_FLOW_BATCH_AUDIENCE_FETCH_TIMEOUT_MS
        )
        return new CdpCyclotronWorkerBatchResolve(hub, deps, workerForConsumer, queryService, internalFetchService)
    }

    async function insertActiveBatchFlow(triggerMasking?: HogFlow['trigger_masking']): Promise<HogFlow> {
        const flow = new FixtureHogFlowBuilder()
            .withTeamId(team.id)
            .withStatus('active')
            .withTriggerMasking(triggerMasking)
            .withWorkflow({
                actions: {
                    trigger: {
                        type: 'trigger',
                        config: {
                            type: 'batch',
                            filters: { properties: [{ key: 'plan', value: 'enterprise', type: 'person' }] },
                        },
                    },
                    function_1: {
                        type: 'function',
                        config: {
                            template_id: 'template-workflows-e2e-fetch',
                            inputs: { url: { value: 'https://example.com/batch' } },
                        },
                    },
                },
                edges: [{ from: 'trigger', to: 'function_1', type: 'continue' }],
            })
            .build()
        return await insertHogFlow(hub.postgres, flow)
    }

    it('POST /batch_invocations with flag on creates a resolver cyclotron job with the right shape', async () => {
        const flow = await insertActiveBatchFlow()
        const parentRunId = new UUIDT().toString()

        const response = await supertest(app)
            .post(`/api/projects/${team.id}/hog_flows/${flow.id}/batch_invocations/${parentRunId}`)
            .send({
                filters: { filter_test_accounts: false },
                max_audience_size: 750,
                variables: { greeting: 'hi' },
                group_type_index: 2,
            })
            .expect(200)

        expect(response.body).toEqual({ status: 'queued' })

        const rows = await cyclotronPool.query<{
            id: string
            queue_name: string
            status: string
            parent_run_id: string
            team_id: number
            function_id: string
            state: Buffer | null
        }>(
            `SELECT id, queue_name, status::text AS status, parent_run_id, team_id, function_id, state
             FROM cyclotron_jobs
             WHERE queue_name = 'hogflow_batch_resolve' AND parent_run_id = $1`,
            [parentRunId]
        )
        expect(rows.rows).toHaveLength(1)
        const job = rows.rows[0]
        expect(job.status).toBe('available')
        expect(job.parent_run_id).toBe(parentRunId)
        expect(job.team_id).toBe(team.id)
        expect(job.function_id).toBe(flow.id)

        const state = parseJSON((job.state as Buffer).toString('utf-8')) as Record<string, unknown>
        expect(state).toMatchObject({
            batchJobId: parentRunId,
            teamId: team.id,
            hogFlowId: flow.id,
            maxAudienceSize: 750,
            variables: { greeting: 'hi' },
            groupTypeIndex: 2,
            cursor: null,
            totalEnqueued: 0,
            pagesProcessed: 0,
        })
        expect(state.pendingTerminal).toBeUndefined()
    })

    it('rejects with 400 when the workflow is not a batch trigger', async () => {
        const flow = new FixtureHogFlowBuilder()
            .withTeamId(team.id)
            .withStatus('active')
            .withWorkflow({
                actions: {
                    trigger: {
                        type: 'trigger',
                        config: { type: 'event', filters: {} },
                    },
                    function_1: {
                        type: 'function',
                        config: {
                            template_id: 'template-workflows-e2e-fetch',
                            inputs: { url: { value: 'https://example.com/' } },
                        },
                    },
                },
                edges: [{ from: 'trigger', to: 'function_1', type: 'continue' }],
            })
            .build()
        const inserted = await insertHogFlow(hub.postgres, flow)
        const parentRunId = new UUIDT().toString()

        await supertest(app)
            .post(`/api/projects/${team.id}/hog_flows/${inserted.id}/batch_invocations/${parentRunId}`)
            .send({ filters: { filter_test_accounts: false } })
            .expect(400)

        const rows = await cyclotronPool.query(`SELECT id FROM cyclotron_jobs WHERE parent_run_id = $1`, [parentRunId])
        expect(rows.rows).toHaveLength(0)
    })

    it('full lifecycle: HTTP POST → resolver chunks 3 pages → children enqueued → Django PUT status=completed', async () => {
        const flow = await insertActiveBatchFlow()
        const parentRunId = new UUIDT().toString()

        const personIds = [
            new UUIDT().toString(),
            new UUIDT().toString(),
            new UUIDT().toString(),
            new UUIDT().toString(),
            new UUIDT().toString(),
        ]
        const statusPuts: Array<{ status: string }> = []
        let personPageCalls = 0

        mockInternalFetch.mockImplementation((url: string, opts: any) => {
            if (url.includes('/user_blast_radius_persons')) {
                personPageCalls += 1
                const pages = [
                    { users_affected: personIds.slice(0, 2), cursor: 'c1', has_more: true },
                    { users_affected: personIds.slice(2, 4), cursor: 'c2', has_more: true },
                    { users_affected: personIds.slice(4), cursor: null, has_more: false },
                ]
                const page = pages[Math.min(personPageCalls - 1, pages.length - 1)]
                return Promise.resolve({
                    status: 200,
                    headers: {},
                    json: () => Promise.resolve({}),
                    text: () => Promise.resolve(JSON.stringify(page)),
                    dump: () => Promise.resolve(),
                })
            }
            if (url.includes('/batch_jobs/') && url.endsWith('/status')) {
                statusPuts.push(parseJSON(opts.body) as { status: string })
                return Promise.resolve({
                    status: 200,
                    headers: {},
                    json: () => Promise.resolve({}),
                    text: () => Promise.resolve('{}'),
                    dump: () => Promise.resolve(),
                })
            }
            return Promise.reject(new Error(`Unexpected internalFetch call to ${url}`))
        })

        await supertest(app)
            .post(`/api/projects/${team.id}/hog_flows/${flow.id}/batch_invocations/${parentRunId}`)
            .send({ filters: { filter_test_accounts: false }, max_audience_size: 1000 })
            .expect(200)

        // Start the consumer — it'll pick up the resolver job and process pages
        // on its own polling loop. waitForExpect waits for the terminal Django
        // PUT, which only happens after all 3 pages + the terminal-write phase.
        resolverWorker = buildResolverConsumer()
        await resolverWorker.start()

        await waitForExpect(() => {
            expect(statusPuts).toHaveLength(1)
        }, 20000)

        expect(personPageCalls).toBe(3)
        expect(statusPuts[0]).toEqual({ status: 'completed' })

        const children = await cyclotronPool.query(
            `SELECT id FROM cyclotron_jobs WHERE queue_name = 'hogflow' AND parent_run_id = $1`,
            [parentRunId]
        )
        expect(children.rows).toHaveLength(personIds.length)

        // Enrolling a person has to register as a started run at the same time as it enqueues
        // the work. Both of these used to be skipped here (only the event-triggered path wrote
        // them), which read on the workflow page as a batch that never started and had no runs
        // to list — for the whole of any delay the flow was parked on.
        await waitForExpect(() => {
            const triggered = mockProducerObserver
                .getProducedKafkaMessagesForTopic(KAFKA_APP_METRICS_2)
                .filter((m: any) => m.value.app_source === 'hog_flow' && m.value.metric_name === 'triggered')
            // Summed, not counted: the aggregator collapses a flush window's metrics onto one
            // message per key, so a 3-page resolve produces fewer messages than persons.
            expect(triggered.reduce((total, m: any) => total + m.value.count, 0)).toBe(personIds.length)
            for (const metric of triggered) {
                // Keyed on the batch run, which is what the batch metrics view queries. Keyed on
                // the workflow id instead, the count is real but the page still reads zero.
                expect((metric as any).value.app_source_id).toBe(parentRunId)
                // Run-level, so no step id — the started and in-progress counters filter on the
                // empty instance and would skip these otherwise.
                expect((metric as any).value.instance_id).toBe('')
            }

            const rows = mockProducerObserver
                .getProducedKafkaMessagesForTopic(KAFKA_HOG_INVOCATION_RESULTS)
                .map((m: any) => m.value)
            expect(rows).toHaveLength(personIds.length)
            for (const row of rows) {
                // hog_flow, not hog_function: the invocations API filters on this, so a
                // misclassified row is written but never listed.
                expect(row.function_kind).toBe('hog_flow')
                expect(row.function_id).toBe(flow.id)
                expect(row.parent_run_id).toBe(parentRunId)
                expect(row.status).toBe('running')
            }
        }, 10000)
    })

    it('keeps the job lock and worker health during a slow audience fetch', async () => {
        const flow = await insertActiveBatchFlow()
        const parentRunId = new UUIDT().toString()
        const personId = new UUIDT().toString()
        const originalTimeout = hub.CDP_HOG_FLOW_BATCH_AUDIENCE_FETCH_TIMEOUT_MS
        hub.CDP_HOG_FLOW_BATCH_AUDIENCE_FETCH_TIMEOUT_MS = 45_000
        const audiencePage = Promise.withResolvers<void>()
        const statusPuts: Array<{ status: string }> = []
        let audienceCalls = 0
        const janitor = new CyclotronV2Janitor({
            pool: { dbUrl: CYCLOTRON_NODE_DB_URL },
            stallTimeoutMs: 15_000,
            maxTouchCount: 2,
            cleanupGraceMs: 99_999_000,
            cleanupBatchSize: 100,
            cleanupIntervalMs: 60_000,
        })

        mockInternalFetch.mockImplementation(async (url: string, opts: any) => {
            if (url.includes('/user_blast_radius_persons')) {
                audienceCalls += 1
                expect(opts.timeoutMs).toBe(45_000)
                await audiencePage.promise
                return {
                    status: 200,
                    headers: {},
                    json: () => Promise.resolve({}),
                    text: () =>
                        Promise.resolve(JSON.stringify({ users_affected: [personId], cursor: null, has_more: false })),
                    dump: async () => {},
                }
            }
            if (url.includes('/batch_jobs/') && url.endsWith('/status')) {
                statusPuts.push(parseJSON(opts.body) as { status: string })
                return {
                    status: 200,
                    headers: {},
                    json: () => Promise.resolve({}),
                    text: () => Promise.resolve('{}'),
                    dump: async () => {},
                }
            }
            throw new Error(`Unexpected internalFetch call to ${url}`)
        })

        try {
            await supertest(app)
                .post(`/api/projects/${team.id}/hog_flows/${flow.id}/batch_invocations/${parentRunId}`)
                .send({ filters: { filter_test_accounts: false }, max_audience_size: 1000 })
                .expect(200)

            resolverWorker = buildResolverConsumer()
            await resolverWorker.start()
            await waitForExpect(() => expect(audienceCalls).toBe(1), 5000)
            await new Promise((resolve) => setTimeout(resolve, 32_000))

            expect(resolverWorker.isHealthy().status).toBe('ok')
            const sweep = await janitor.runOnce()
            expect(sweep.stalled).toBe(0)
            expect(sweep.poisoned).toBe(0)

            audiencePage.resolve()
            await waitForExpect(() => expect(statusPuts).toEqual([{ status: 'completed' }]), 10_000)
            expect(audienceCalls).toBe(1)
            const jobs = await cyclotronPool.query(
                'SELECT queue_name, janitor_touch_count FROM cyclotron_jobs WHERE parent_run_id = $1',
                [parentRunId]
            )
            expect(jobs.rows.filter((job) => job.queue_name === 'hogflow')).toHaveLength(1)
            for (const job of jobs.rows) {
                expect(job.janitor_touch_count).toBe(0)
            }
        } finally {
            audiencePage.resolve()
            hub.CDP_HOG_FLOW_BATCH_AUDIENCE_FETCH_TIMEOUT_MS = originalTimeout
            await janitor.stop()
        }
    }, 60_000)

    // Regression test: batch-resolved invocations used to skip trigger_masking entirely
    // (only the event-triggered pipeline applied it), so a scheduled batch workflow with
    // a masking TTL re-enrolled the same audience on every run.
    it('trigger_masking suppresses re-enrolling the same person across two batch runs', async () => {
        const flow = await insertActiveBatchFlow(HOG_FLOW_MASK_EXAMPLES.oncePerTimePeriod.trigger_masking)
        const personId = new UUIDT().toString()
        const statusPuts: Array<{ status: string }> = []

        mockInternalFetch.mockImplementation((url: string, opts: any) => {
            if (url.includes('/user_blast_radius_persons')) {
                return Promise.resolve({
                    status: 200,
                    headers: {},
                    json: () => Promise.resolve({}),
                    text: () =>
                        Promise.resolve(JSON.stringify({ users_affected: [personId], cursor: null, has_more: false })),
                    dump: () => Promise.resolve(),
                })
            }
            if (url.includes('/batch_jobs/') && url.endsWith('/status')) {
                statusPuts.push(parseJSON(opts.body) as { status: string })
                return Promise.resolve({
                    status: 200,
                    headers: {},
                    json: () => Promise.resolve({}),
                    text: () => Promise.resolve('{}'),
                    dump: () => Promise.resolve(),
                })
            }
            return Promise.reject(new Error(`Unexpected internalFetch call to ${url}`))
        })

        resolverWorker = buildResolverConsumer()
        await resolverWorker.start()

        // First run: the person hasn't been masked yet, so they get enrolled.
        const firstRunId = new UUIDT().toString()
        await supertest(app)
            .post(`/api/projects/${team.id}/hog_flows/${flow.id}/batch_invocations/${firstRunId}`)
            .send({ filters: { filter_test_accounts: false }, max_audience_size: 1000 })
            .expect(200)

        await waitForExpect(() => {
            expect(statusPuts).toHaveLength(1)
        }, 20000)

        const firstRunChildren = await cyclotronPool.query(
            `SELECT id FROM cyclotron_jobs WHERE queue_name = 'hogflow' AND parent_run_id = $1`,
            [firstRunId]
        )
        expect(firstRunChildren.rows).toHaveLength(1)

        // Second run (simulating the next scheduled re-enrollment): the same person is
        // masked, so no child invocation should be enqueued this time.
        const secondRunId = new UUIDT().toString()
        await supertest(app)
            .post(`/api/projects/${team.id}/hog_flows/${flow.id}/batch_invocations/${secondRunId}`)
            .send({ filters: { filter_test_accounts: false }, max_audience_size: 1000 })
            .expect(200)

        await waitForExpect(() => {
            expect(statusPuts).toHaveLength(2)
        }, 20000)

        const secondRunChildren = await cyclotronPool.query(
            `SELECT id FROM cyclotron_jobs WHERE queue_name = 'hogflow' AND parent_run_id = $1`,
            [secondRunId]
        )
        expect(secondRunChildren.rows).toHaveLength(0)
    })

    // The mask claims land in Redis before the page commits to Postgres. If the commit
    // fails without releasing them, the stall-recovery replay of the same cursor sees
    // the whole page as already-masked and silently drops every enrollment in it.
    it('failed page commit releases mask claims, so the stall-recovery replay still enrolls', async () => {
        const flow = await insertActiveBatchFlow(HOG_FLOW_MASK_EXAMPLES.oncePerTimePeriod.trigger_masking)
        const personId = new UUIDT().toString()
        const statusPuts: Array<{ status: string }> = []

        mockInternalFetch.mockImplementation((url: string, opts: any) => {
            if (url.includes('/user_blast_radius_persons')) {
                return Promise.resolve({
                    status: 200,
                    headers: {},
                    json: () => Promise.resolve({}),
                    text: () =>
                        Promise.resolve(JSON.stringify({ users_affected: [personId], cursor: null, has_more: false })),
                    dump: () => Promise.resolve(),
                })
            }
            if (url.includes('/batch_jobs/') && url.endsWith('/status')) {
                statusPuts.push(parseJSON(opts.body) as { status: string })
                return Promise.resolve({
                    status: 200,
                    headers: {},
                    json: () => Promise.resolve({}),
                    text: () => Promise.resolve('{}'),
                    dump: () => Promise.resolve(),
                })
            }
            return Promise.reject(new Error(`Unexpected internalFetch call to ${url}`))
        })

        let commitFailuresLeft = 1
        resolverWorker = buildResolverConsumer((job) => ({
            ...job,
            bulkCreateAndCheckIn: (input) => {
                if (commitFailuresLeft > 0) {
                    commitFailuresLeft--
                    return Promise.reject(new Error('injected commit failure'))
                }
                return job.bulkCreateAndCheckIn(input)
            },
        }))
        await resolverWorker.start()

        const parentRunId = new UUIDT().toString()
        await supertest(app)
            .post(`/api/projects/${team.id}/hog_flows/${flow.id}/batch_invocations/${parentRunId}`)
            .send({ filters: { filter_test_accounts: false }, max_audience_size: 1000 })
            .expect(200)

        // The first attempt claims the person's mask slot, then hits the injected commit
        // failure. Wait until the resolver has released the claim (counter back to 0) —
        // only then is the job's parked state the same as a real stall.
        const maskRedis = createRedisV2PoolFromConfig({
            connection: hub.CDP_REDIS_HOST
                ? {
                      url: hub.CDP_REDIS_HOST,
                      options: { port: hub.CDP_REDIS_PORT, password: hub.CDP_REDIS_PASSWORD },
                  }
                : { url: hub.REDIS_URL },
            poolMinSize: hub.REDIS_POOL_MIN_SIZE,
            poolMaxSize: hub.REDIS_POOL_MAX_SIZE,
        })
        await waitForExpect(async () => {
            expect(commitFailuresLeft).toBe(0)
            const maskCounters = await maskRedis.useClient({ name: 'mask-peek' }, async (client) => {
                const keys = await client.keys(`@posthog-test/hog-masker/mask/${flow.id}/*`)
                return Promise.all(keys.map((key) => client.get(key)))
            })
            expect(maskCounters).toEqual(['0'])
        }, 20000)

        // What the janitor's stall recovery does to the parked job, without waiting
        // out the stall timeout.
        await cyclotronPool.query(
            `UPDATE cyclotron_jobs SET status = 'available', lock_id = NULL, last_heartbeat = NULL
             WHERE queue_name = $1 AND parent_run_id = $2`,
            [HOGFLOW_BATCH_RESOLVE_QUEUE, parentRunId]
        )

        await waitForExpect(() => {
            expect(statusPuts).toEqual([{ status: 'completed' }])
        }, 20000)

        // The replay re-evaluated masking against the released claim and enrolled the
        // person — without the release it would classify them as masked and enqueue nothing.
        const children = await cyclotronPool.query(
            `SELECT id FROM cyclotron_jobs WHERE queue_name = 'hogflow' AND parent_run_id = $1`,
            [parentRunId]
        )
        expect(children.rows).toHaveLength(1)
    })

    it('failed lifecycle: workflow deleted before processing → resolver transitions to failed → Django PUT status=failed', async () => {
        const flow = await insertActiveBatchFlow()
        const parentRunId = new UUIDT().toString()
        const statusPuts: Array<{ status: string }> = []

        mockInternalFetch.mockImplementation((url: string, opts: any) => {
            if (url.includes('/batch_jobs/') && url.endsWith('/status')) {
                statusPuts.push(parseJSON(opts.body) as { status: string })
                return Promise.resolve({
                    status: 200,
                    headers: {},
                    json: () => Promise.resolve({}),
                    text: () => Promise.resolve('{}'),
                    dump: () => Promise.resolve(),
                })
            }
            // Audience fetch shouldn't be reached — the resolver bails on missing
            // hogflow before getting that far.
            return Promise.reject(new Error(`Unexpected internalFetch call to ${url}`))
        })

        // Dispatch BEFORE starting the consumer so we can sabotage the workflow
        // before any processing happens (otherwise the consumer would race us
        // and process the job before we delete the workflow).
        await supertest(app)
            .post(`/api/projects/${team.id}/hog_flows/${flow.id}/batch_invocations/${parentRunId}`)
            .send({ filters: { filter_test_accounts: false }, max_audience_size: 1000 })
            .expect(200)

        // Sabotage: customer deletes the workflow between dispatch and processing.
        await hub.postgres.query(
            PostgresUse.COMMON_WRITE,
            `DELETE FROM posthog_hogflow WHERE id = $1`,
            [flow.id],
            'delete-hogflow-for-test'
        )
        // The consumer we're about to build will have its own hogFlowManager
        // with an empty cache, so refresh isn't strictly needed — but be
        // explicit so the test isn't sensitive to cache lifecycle changes.

        resolverWorker = buildResolverConsumer()
        await resolverWorker.start()

        await waitForExpect(() => {
            expect(statusPuts).toHaveLength(1)
        }, 20000)

        expect(statusPuts[0]).toEqual({ status: 'failed' })

        const children = await cyclotronPool.query(
            `SELECT id FROM cyclotron_jobs WHERE queue_name = 'hogflow' AND parent_run_id = $1`,
            [parentRunId]
        )
        expect(children.rows).toHaveLength(0)
    })

    it('Django down: resolver retries the terminal PUT instead of acking', async () => {
        const flow = await insertActiveBatchFlow()
        const parentRunId = new UUIDT().toString()
        const personIds = [new UUIDT().toString()]
        let putAttempts = 0

        mockInternalFetch.mockImplementation((url: string) => {
            if (url.includes('/user_blast_radius_persons')) {
                return Promise.resolve({
                    status: 200,
                    headers: {},
                    json: () => Promise.resolve({}),
                    text: () =>
                        Promise.resolve(
                            JSON.stringify({
                                users_affected: personIds,
                                cursor: null,
                                has_more: false,
                            })
                        ),
                    dump: () => Promise.resolve(),
                })
            }
            if (url.includes('/batch_jobs/') && url.endsWith('/status')) {
                putAttempts += 1
                return Promise.resolve({
                    status: 503,
                    headers: {},
                    json: () => Promise.resolve({}),
                    text: () => Promise.resolve('Service Unavailable'),
                    dump: () => Promise.resolve(),
                })
            }
            return Promise.reject(new Error(`Unexpected internalFetch call to ${url}`))
        })

        await supertest(app)
            .post(`/api/projects/${team.id}/hog_flows/${flow.id}/batch_invocations/${parentRunId}`)
            .send({ filters: { filter_test_accounts: false }, max_audience_size: 1000 })
            .expect(200)

        resolverWorker = buildResolverConsumer()
        await resolverWorker.start()

        // Wait until at least one Django PUT attempt has happened — proves the
        // resolver reached the terminal-write phase. Then verify the resolver
        // job is parked (status='available' with pendingTerminal still set),
        // not acked.
        await waitForExpect(() => {
            expect(putAttempts).toBeGreaterThanOrEqual(1)
        }, 20000)

        const rows = await cyclotronPool.query<{ status: string; state: Buffer | null }>(
            `SELECT status::text AS status, state FROM cyclotron_jobs
             WHERE queue_name = 'hogflow_batch_resolve' AND parent_run_id = $1`,
            [parentRunId]
        )
        expect(rows.rows).toHaveLength(1)
        expect(rows.rows[0].status).toBe('available')
        const state = parseJSON((rows.rows[0].state as Buffer).toString('utf-8')) as { pendingTerminal?: string }
        expect(state.pendingTerminal).toBe('completed')
    })

    it('audience fetch failure: resolver reschedules with backoff, cursor unchanged, no children enqueued', async () => {
        const flow = await insertActiveBatchFlow()
        const parentRunId = new UUIDT().toString()
        let fetchAttempts = 0

        mockInternalFetch.mockImplementation((url: string) => {
            if (url.includes('/user_blast_radius_persons')) {
                fetchAttempts += 1
                // Audience fetch always fails — resolver should keep retrying,
                // never advance the cursor, never enqueue children.
                return Promise.reject(new Error('The operation was aborted due to timeout'))
            }
            return Promise.reject(new Error(`Unexpected internalFetch call to ${url}`))
        })

        await supertest(app)
            .post(`/api/projects/${team.id}/hog_flows/${flow.id}/batch_invocations/${parentRunId}`)
            .send({ filters: { filter_test_accounts: false }, max_audience_size: 1000 })
            .expect(200)

        resolverWorker = buildResolverConsumer()
        await resolverWorker.start()

        await waitForExpect(() => {
            expect(fetchAttempts).toBeGreaterThanOrEqual(1)
        }, 20000)

        const rows = await cyclotronPool.query<{ status: string; state: Buffer | null }>(
            `SELECT status::text AS status, state FROM cyclotron_jobs
             WHERE queue_name = 'hogflow_batch_resolve' AND parent_run_id = $1`,
            [parentRunId]
        )
        expect(rows.rows).toHaveLength(1)
        expect(rows.rows[0].status).toBe('available')
        const state = parseJSON((rows.rows[0].state as Buffer).toString('utf-8')) as {
            cursor: string | null
            totalEnqueued: number
            pagesProcessed: number
            pendingTerminal?: string
        }
        expect(state.cursor).toBeNull() // never advanced
        expect(state.totalEnqueued).toBe(0)
        expect(state.pagesProcessed).toBe(0)
        expect(state.pendingTerminal).toBeUndefined()

        const children = await cyclotronPool.query(
            `SELECT id FROM cyclotron_jobs WHERE queue_name = 'hogflow' AND parent_run_id = $1`,
            [parentRunId]
        )
        expect(children.rows).toHaveLength(0)
    })

    it('truncation: pre-existing job at maxAudienceSize skips fetch + emits customer log + writes status=completed', async () => {
        // Pre-populate a resolver job at the cap so the next dequeue takes
        // the truncation branch (audience fetch never runs).
        const flow = await insertActiveBatchFlow()
        const parentRunId = new UUIDT().toString()
        const statusPuts: Array<{ status: string }> = []

        mockInternalFetch.mockImplementation((url: string, opts: any) => {
            if (url.includes('/batch_jobs/') && url.endsWith('/status')) {
                statusPuts.push(parseJSON(opts.body) as { status: string })
                return Promise.resolve({
                    status: 200,
                    headers: {},
                    json: () => Promise.resolve({}),
                    text: () => Promise.resolve('{}'),
                    dump: () => Promise.resolve(),
                })
            }
            return Promise.reject(new Error(`Unexpected internalFetch call to ${url}`))
        })

        const cappedState = serializeResolverState({
            batchJobId: parentRunId,
            teamId: team.id,
            hogFlowId: flow.id,
            filters: { properties: [], filter_test_accounts: false },
            variables: {},
            maxAudienceSize: 100,
            cursor: 'some-cursor',
            totalEnqueued: 100,
            pagesProcessed: 1,
            attempts: 0,
            startedAt: new Date().toISOString(),
        })
        await batchResolverProducer.createJob({
            teamId: team.id,
            queueName: 'hogflow_batch_resolve',
            parentRunId,
            functionId: flow.id,
            state: cappedState,
        })

        resolverWorker = buildResolverConsumer()
        await resolverWorker.start()

        await waitForExpect(() => {
            expect(statusPuts).toHaveLength(1)
        }, 20000)

        // Status PUT was completed (truncation is still success, not failure)
        expect(statusPuts[0]).toEqual({ status: 'completed' })

        // No children — the resolver short-circuited before audience fetch
        const children = await cyclotronPool.query(
            `SELECT id FROM cyclotron_jobs WHERE queue_name = 'hogflow' AND parent_run_id = $1`,
            [parentRunId]
        )
        expect(children.rows).toHaveLength(0)
    })

    it('persistent fetch failure exhausts MAX_RESOLVER_ATTEMPTS → flips to pendingTerminal=failed and writes status=failed', async () => {
        // Pre-populate a resolver job already one retry below the cap so we
        // only need to drive a single more fetch failure to cross the
        // threshold — avoids burning 5x backoff windows in CI.
        const flow = await insertActiveBatchFlow()
        const parentRunId = new UUIDT().toString()
        const statusPuts: Array<{ status: string }> = []
        let fetchAttempts = 0

        mockInternalFetch.mockImplementation((url: string, opts: any) => {
            if (url.includes('/user_blast_radius_persons')) {
                fetchAttempts += 1
                return Promise.reject(new Error('ClickHouse permanently rejecting the audience query'))
            }
            if (url.includes('/batch_jobs/') && url.endsWith('/status')) {
                statusPuts.push(parseJSON(opts.body) as { status: string })
                return Promise.resolve({
                    status: 200,
                    headers: {},
                    json: () => Promise.resolve({}),
                    text: () => Promise.resolve('{}'),
                    dump: () => Promise.resolve(),
                })
            }
            return Promise.reject(new Error(`Unexpected internalFetch call to ${url}`))
        })

        const seededState = serializeResolverState({
            batchJobId: parentRunId,
            teamId: team.id,
            hogFlowId: flow.id,
            filters: { properties: [], filter_test_accounts: false },
            variables: {},
            maxAudienceSize: 1000,
            cursor: null,
            totalEnqueued: 0,
            pagesProcessed: 0,
            attempts: MAX_RESOLVER_ATTEMPTS - 1,
            startedAt: new Date().toISOString(),
        })
        await batchResolverProducer.createJob({
            teamId: team.id,
            queueName: HOGFLOW_BATCH_RESOLVE_QUEUE,
            parentRunId,
            functionId: flow.id,
            state: seededState,
        })

        resolverWorker = buildResolverConsumer()
        await resolverWorker.start()

        await waitForExpect(() => {
            expect(statusPuts).toHaveLength(1)
        }, 20000)

        expect(statusPuts[0]).toEqual({ status: 'failed' })
        expect(fetchAttempts).toBe(1) // one more retry consumed the last attempt budget

        // No children enqueued — the resolver bailed before ever returning a page
        const children = await cyclotronPool.query(
            `SELECT id FROM cyclotron_jobs WHERE queue_name = 'hogflow' AND parent_run_id = $1`,
            [parentRunId]
        )
        expect(children.rows).toHaveLength(0)
    })

    it('persistent Django 4xx on terminal write exhausts MAX_RESOLVER_ATTEMPTS → job.fail()', async () => {
        // Pre-populate a job already in the terminal-write phase with attempts
        // one below the cap. The 404 response is a permanent failure — no
        // amount of retrying will make Django accept it (the row is gone) —
        // so the resolver should give up and fail the cyclotron job instead
        // of looping forever.
        const flow = await insertActiveBatchFlow()
        const parentRunId = new UUIDT().toString()
        let putAttempts = 0

        mockInternalFetch.mockImplementation((url: string) => {
            if (url.includes('/batch_jobs/') && url.endsWith('/status')) {
                putAttempts += 1
                return Promise.resolve({
                    status: 404,
                    headers: {},
                    json: () => Promise.resolve({}),
                    text: () => Promise.resolve('Not Found'),
                    dump: () => Promise.resolve(),
                })
            }
            return Promise.reject(new Error(`Unexpected internalFetch call to ${url}`))
        })

        const pendingState = serializeResolverState({
            batchJobId: parentRunId,
            teamId: team.id,
            hogFlowId: flow.id,
            filters: { properties: [], filter_test_accounts: false },
            variables: {},
            maxAudienceSize: 1000,
            cursor: 'last-cursor',
            totalEnqueued: 5,
            pagesProcessed: 1,
            attempts: MAX_RESOLVER_ATTEMPTS - 1,
            startedAt: new Date().toISOString(),
            pendingTerminal: 'completed',
        })
        await batchResolverProducer.createJob({
            teamId: team.id,
            queueName: HOGFLOW_BATCH_RESOLVE_QUEUE,
            parentRunId,
            functionId: flow.id,
            state: pendingState,
        })

        resolverWorker = buildResolverConsumer()
        await resolverWorker.start()

        await waitForExpect(async () => {
            const r = await cyclotronPool.query<{ status: string }>(
                `SELECT status::text AS status FROM cyclotron_jobs
                 WHERE queue_name = 'hogflow_batch_resolve' AND parent_run_id = $1`,
                [parentRunId]
            )
            expect(r.rows[0]?.status).toBe('failed')
        }, 20000)

        // One more attempt consumed the budget, then the job failed
        expect(putAttempts).toBe(1)
    })

    it('hard cap: page that would cross maxAudienceSize is truncated before enqueue', async () => {
        // maxAudienceSize=4 with pages of 3. Without the hard cap the resolver
        // would enqueue 6 children (overshoot by 2) then notice and truncate.
        // With the hard cap the second page is truncated to 1 row so the
        // total never exceeds 4.
        const flow = await insertActiveBatchFlow()
        const parentRunId = new UUIDT().toString()
        const personIds = Array.from({ length: 6 }, () => new UUIDT().toString())
        const statusPuts: Array<{ status: string }> = []
        let personPageCalls = 0

        mockInternalFetch.mockImplementation((url: string, opts: any) => {
            if (url.includes('/user_blast_radius_persons')) {
                personPageCalls += 1
                const pages = [
                    { users_affected: personIds.slice(0, 3), cursor: 'c1', has_more: true },
                    { users_affected: personIds.slice(3, 6), cursor: 'c2', has_more: true },
                ]
                const page = pages[Math.min(personPageCalls - 1, pages.length - 1)]
                return Promise.resolve({
                    status: 200,
                    headers: {},
                    json: () => Promise.resolve({}),
                    text: () => Promise.resolve(JSON.stringify(page)),
                    dump: () => Promise.resolve(),
                })
            }
            if (url.includes('/batch_jobs/') && url.endsWith('/status')) {
                statusPuts.push(parseJSON(opts.body) as { status: string })
                return Promise.resolve({
                    status: 200,
                    headers: {},
                    json: () => Promise.resolve({}),
                    text: () => Promise.resolve('{}'),
                    dump: () => Promise.resolve(),
                })
            }
            return Promise.reject(new Error(`Unexpected internalFetch call to ${url}`))
        })

        await supertest(app)
            .post(`/api/projects/${team.id}/hog_flows/${flow.id}/batch_invocations/${parentRunId}`)
            .send({ filters: { filter_test_accounts: false }, max_audience_size: 4 })
            .expect(200)

        resolverWorker = buildResolverConsumer()
        await resolverWorker.start()

        await waitForExpect(() => {
            expect(statusPuts).toHaveLength(1)
        }, 20000)

        expect(statusPuts[0]).toEqual({ status: 'completed' })

        const children = await cyclotronPool.query(
            `SELECT id FROM cyclotron_jobs WHERE queue_name = 'hogflow' AND parent_run_id = $1`,
            [parentRunId]
        )
        // Hard cap: exactly 4 children, never 6
        expect(children.rows).toHaveLength(4)
    })

    it('invalid resolver state: garbage bytes → job.fail() (not retry)', async () => {
        // Insert a resolver job with state that doesn't match the Zod schema.
        // Simulates schema drift, corruption, or a row written by an
        // incompatible older deploy. The resolver must FAIL the job (terminal)
        // rather than reschedule — otherwise we'd retry-loop forever on a job
        // that can never succeed.
        const parentRunId = new UUIDT().toString()
        const malformedState = Buffer.from(JSON.stringify({ not: 'a valid resolver state' }))
        await batchResolverProducer.createJob({
            teamId: team.id,
            queueName: 'hogflow_batch_resolve',
            parentRunId,
            functionId: new UUIDT().toString(),
            state: malformedState,
        })

        mockInternalFetch.mockImplementation((url: string) =>
            Promise.reject(new Error(`Unexpected internalFetch call to ${url}`))
        )

        resolverWorker = buildResolverConsumer()
        await resolverWorker.start()

        await waitForExpect(async () => {
            const r = await cyclotronPool.query<{ status: string }>(
                `SELECT status::text AS status FROM cyclotron_jobs
                 WHERE queue_name = 'hogflow_batch_resolve' AND parent_run_id = $1`,
                [parentRunId]
            )
            expect(r.rows[0]?.status).toBe('failed')
        }, 20000)

        // No children, no Django PUT — the resolver failed before any work.
        const children = await cyclotronPool.query(
            `SELECT id FROM cyclotron_jobs WHERE queue_name = 'hogflow' AND parent_run_id = $1`,
            [parentRunId]
        )
        expect(children.rows).toHaveLength(0)
    })
})

// Janitor poison-pill recovery (postgres-v2). Exercises the REAL results service
// + real Kafka produce — the seam the mocked unit tests can't cover: an isolated
// poison pill is recorded as a failed, replayable result on hog_invocation_results
// BEFORE its cyclotron row is deleted, and with recovery disabled the janitor
// reverts to master's legacy path (marks the pill failed, no recovery record).
describe('Workflows E2E (janitor poison-pill recovery, postgres-v2)', () => {
    jest.setTimeout(30000)

    let hub: Hub
    let kafkaProducer: KafkaProducerWrapper
    let mockProducerObserver: KafkaProducerObserver
    let team: Team
    let cyclotronPool: Pool
    let invocationResults: HogInvocationResultsService
    let janitor: CyclotronV2Janitor | undefined

    beforeAll(() => {
        cyclotronPool = new Pool({ connectionString: CYCLOTRON_NODE_DB_URL })
    })

    afterAll(async () => {
        await cyclotronPool.end()
    })

    beforeEach(async () => {
        MockKafkaProducerWrapper.create = jest.fn((...args) => ActualKafkaProducerWrapper.create(...args))

        // KAFKA_HOG_INVOCATION_RESULTS isn't in TEST_KAFKA_TOPICS — the janitor
        // produces the give-up record there, so it must exist or the produce
        // fails and the row is (correctly) never deleted.
        await ensureKafkaTopics([...TEST_KAFKA_TOPICS, KAFKA_HOG_INVOCATION_RESULTS])
        await resetTestDatabase()
        await cyclotronPool.query('DELETE FROM cyclotron_jobs')

        hub = await createHub()
        // The give-up record path is gated on this flag (off by default outside dev).
        hub.HOG_INVOCATION_RESULTS_ENABLED = true

        kafkaProducer = await ActualKafkaProducerWrapper.create(hub.KAFKA_CLIENT_RACK)
        mockProducerObserver = new KafkaProducerObserver(kafkaProducer)
        team = await getFirstTeam(hub.postgres)
        mockProducerObserver.resetKafkaProducer()

        const deps = createCdpConsumerDeps(hub, kafkaProducer)
        const outputs = createCdpOutputsRegistry().build(deps.cdpProducerRegistry, hub)
        invocationResults = new HogInvocationResultsService(outputs, hub)
    })

    afterEach(async () => {
        await janitor?.stop().catch(() => {})
        janitor = undefined
        await kafkaProducer.disconnect()
        await closeHub(hub)
        mockProducerObserver.resetKafkaProducer()
    })

    function createJanitor(overrides: Record<string, unknown> = {}): CyclotronV2Janitor {
        return new CyclotronV2Janitor(
            {
                pool: { dbUrl: CYCLOTRON_NODE_DB_URL },
                cleanupGraceMs: 0,
                stallTimeoutMs: 1000,
                maxTouchCount: 3,
                ...overrides,
            },
            invocationResults
        )
    }

    // A hogflow job stuck 'running' with a long-stale heartbeat and a touch count
    // over the budget. The serialized state mirrors a wait_until_condition that
    // had already advanced past earlier actions.
    async function insertPoisonedHogflowJob(): Promise<string> {
        const id = new UUIDT().toString()
        const state = Buffer.from(
            JSON.stringify({
                state: {
                    event: { uuid: 'evt-poison', distinct_id: 'd-poison' },
                    actionStepCount: 2,
                    variables: {},
                    currentAction: { id: 'wait_condition', startedAtTimestamp: 1 },
                },
            })
        )
        await cyclotronPool.query(
            `INSERT INTO cyclotron_jobs
                (id, team_id, function_id, queue_name, status, priority, scheduled, created,
                 lock_id, last_heartbeat, janitor_touch_count, transition_count, last_transition, state)
             VALUES ($1, $2, $3, 'hogflow', 'running'::CyclotronJobStatus, 0, NOW(), NOW(),
                     $4, NOW() - INTERVAL '5 minutes', 5, 10, NOW(), $5)`,
            [id, team.id, new UUIDT().toString(), new UUIDT().toString(), state]
        )
        return id
    }

    it('records an isolated poison pill as a failed, replayable result and only then deletes it', async () => {
        const id = await insertPoisonedHogflowJob()

        janitor = createJanitor()
        const result = await janitor.runOnce()

        expect(result.poisonedIds).toEqual([id])

        // A failed lifecycle row reached the hog_invocation_results topic so the
        // run is discoverable in the Invocations UI and replayable by rerun.
        await waitForExpect(() => {
            const rows = mockProducerObserver.getProducedKafkaMessagesForTopic(KAFKA_HOG_INVOCATION_RESULTS)
            expect(rows.some((m: any) => m.value.invocation_id === id)).toBe(true)
        }, 10000)

        const row = mockProducerObserver
            .getProducedKafkaMessagesForTopic(KAFKA_HOG_INVOCATION_RESULTS)
            .find((m: any) => m.value.invocation_id === id)!.value
        expect(row.status).toBe('failed')
        expect(row.function_kind).toBe('hog_flow')
        expect(row.function_id).toBeTruthy()
        expect(row.error_kind).toBe(JANITOR_POISON_PILL_ERROR_KIND)

        // The cyclotron row is gone — but only because the record exists first.
        const remaining = await cyclotronPool.query('SELECT 1 FROM cyclotron_jobs WHERE id = $1', [id])
        expect(remaining.rowCount).toBe(0)
    })

    it('reverts to legacy mark-failed (records nothing) when recovery is disabled', async () => {
        const id = await insertPoisonedHogflowJob()

        janitor = createJanitor({ poisonRecoveryEnabled: false })
        const result = await janitor.runOnce()

        // Kill-switch off → master's legacy path: mark the pill failed and produce
        // no recovery record. The give-up is terminal (no infinite retry) but not
        // replayable — exactly master's pre-recovery behavior.
        expect(result.poisonedIds).toEqual([id])
        expect(mockProducerObserver.getProducedKafkaMessagesForTopic(KAFKA_HOG_INVOCATION_RESULTS)).toHaveLength(0)

        const rows = await cyclotronPool.query('SELECT status FROM cyclotron_jobs WHERE id = $1', [id])
        expect(rows.rowCount).toBe(1)
        expect(rows.rows[0].status).toBe('failed')
    })
})