Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

122 KB
Raw Blame
// sort-imports-ignore
import { PosthogJwtAudience } from '~/cdp/utils/jwt-utils'
import { ScopedServiceJwt } from '~/cdp/utils/scoped-service-jwt'
import { FetchOptions } from '~/common/utils/request'
import { createHmac } from 'node:crypto'
import { createServer } from 'http'
import { DateTime } from 'luxon'
import { AddressInfo } from 'net'

import { CyclotronInvocationQueueParametersFetchType } from '~/cdp/schema/cyclotron'
import { ACCESS_TOKEN_PLACEHOLDER } from '~/common/config/constants'
import { logger } from '~/common/utils/logger'
import { createTestTeamFixture } from '~/tests/helpers/sql'

import { HogExecutorAsyncService } from '../../../src/cdp/services/hog-executor-async.service'
import { HogExecutorService } from '../../../src/cdp/services/hog-executor.service'
import { HogInputsService } from '../../../src/cdp/services/hog-inputs.service'
import { RecipientsManagerService } from '../../../src/cdp/services/managers/recipients-manager.service'
import { TeamWorkflowsConfigService } from '../../../src/cdp/services/managers/team-workflows-config.service'
import { EmailService } from '../../../src/cdp/services/messaging/email.service'
import {
    EmailSuppressionService,
    emailSuppressionConfigFromEnv,
} from '../../../src/cdp/services/messaging/email-suppression.service'
import { EmailTrackingCodeSigner } from '../../../src/cdp/services/messaging/helpers/tracking-code'
import { RecipientTokensService } from '../../../src/cdp/services/messaging/recipient-tokens.service'
import { CyclotronJobInvocationHogFunction, HogFunctionType } from '../../../src/cdp/types'
import { Hub } from '../../../src/types'
import { closeHub, createHub } from '~/common/utils/db/hub'
import { parseJSON } from '~/common/utils/json-parse'
import { promisifyCallback } from '~/common/utils/utils'
import { compileHog } from '../templates/compiler'
import { HOG_EXAMPLES, HOG_FILTERS_EXAMPLES, HOG_INPUTS_EXAMPLES } from '../_tests/examples'
import {
    createExampleInvocation,
    createHogExecutionGlobals,
    createHogFunction,
    insertIntegration,
} from '../_tests/fixtures'
import { isConnectionLevelError } from '../utils/cdp-fetch'
import { EncryptedFields } from '../utils/encryption-utils'
import { EXTEND_OBJECT_KEY } from './hog-inputs.service'
import { SELF_LOOP_DEPTH_PROPERTY, selfLoopGuardCounter } from './self-loop-guard'

// Mock before importing fetch
jest.mock('~/common/utils/request', () => {
    const original = jest.requireActual('~/common/utils/request')
    return {
        ...original,
        fetch: jest.fn().mockImplementation((url, options) => {
            return original.fetch(url, options)
        }),
    }
})

import { fetch, SecureRequestError } from '~/common/utils/request'

const cleanLogs = (logs: string[]): string[] => {
    // Replaces the function time with a fixed value to simplify testing
    return logs.map((log) => {
        return log.replace(/Function completed in \d+(\.\d+)?ms/, 'Function completed in REPLACEDms')
    })
}

describe('Hog Executor', () => {
    jest.setTimeout(1000)
    let executor: HogExecutorAsyncService
    let hub: Hub

    const createExecutor = (encryptedFields: EncryptedFields): HogExecutorAsyncService => {
        const hogInputsService = new HogInputsService(
            hub.integrationManager,
            new RecipientTokensService(hub.ENCRYPTION_SALT_KEYS, hub.SITE_URL),
            encryptedFields
        )
        const emailService = new EmailService(
            {
                sesAccessKeyId: hub.SES_ACCESS_KEY_ID,
                sesSecretAccessKey: hub.SES_SECRET_ACCESS_KEY,
                sesRegion: hub.SES_REGION,
                sesEndpoint: hub.SES_ENDPOINT,
                sesTrackedConfigurationSet: hub.SES_TRACKED_CONFIGURATION_SET,
                sesUntrackedConfigurationSet: hub.SES_UNTRACKED_CONFIGURATION_SET,
            },
            hub.integrationManager,
            new TeamWorkflowsConfigService(hub.postgres, hub.pubSub),
            hub.ENCRYPTION_SALT_KEYS,
            hub.SITE_URL,
            new EmailTrackingCodeSigner(hub.ENCRYPTION_SALT_KEYS, hub.CDP_EMAIL_TRACKING_URL),
            new EmailSuppressionService(hub.postgres, emailSuppressionConfigFromEnv()),
            new RecipientsManagerService(hub.postgres),
            null,
            { report: jest.fn() }
        )
        const recipientTokensService = new RecipientTokensService(hub.ENCRYPTION_SALT_KEYS, hub.SITE_URL)
        return new HogExecutorAsyncService(
            new HogExecutorService({ executionTimeoutMs: hub.CDP_WATCHER_HOG_COST_TIMING_UPPER_MS }, hogInputsService),
            {
                googleAdwordsDeveloperToken: hub.CDP_GOOGLE_ADWORDS_DEVELOPER_TOKEN,
                fetchRetries: hub.CDP_FETCH_RETRIES,
                fetchBackoffBaseMs: hub.CDP_FETCH_BACKOFF_BASE_MS,
                fetchBackoffMaxMs: hub.CDP_FETCH_BACKOFF_MAX_MS,
                siteUrl: hub.SITE_URL,
                internalApiBaseUrl: hub.INTERNAL_API_BASE_URL,
            },
            {
                teamManager: hub.teamManager,
                conversationsTicketsJwt: new ScopedServiceJwt(
                    PosthogJwtAudience.CONVERSATIONS_TICKETS,
                    hub.CONVERSATIONS_TICKETS_JWT_SECRET
                ),
                customerAnalyticsAccountsJwt: new ScopedServiceJwt(
                    PosthogJwtAudience.CUSTOMER_ANALYTICS_ACCOUNTS,
                    hub.CUSTOMER_ANALYTICS_ACCOUNTS_JWT_SECRET
                ),
                hogInputsService,
                emailService,
                recipientTokensService,
                // No push sends in this suite - the push queue type is covered by push-notification.service.test.ts
                pushNotificationService: undefined as any,
            }
        )
    }

    beforeEach(async () => {
        const fixedTime = DateTime.fromObject({ year: 2025, month: 1, day: 1 }, { zone: 'UTC' })
        jest.spyOn(Date, 'now').mockReturnValue(fixedTime.toMillis())

        hub = await createHub()
        executor = createExecutor(hub.encryptedFields)
    })

    afterEach(async () => {
        // Ensure any spies (e.g., execHog, Math.random, Date.now) are restored between tests
        jest.restoreAllMocks()
        await closeHub(hub)
    })

    describe('getSensitiveValues', () => {
        it('masks the nested secrets of every integration in an integration_multi input', () => {
            const hogFunction = {
                inputs_schema: [{ type: 'integration_multi', key: 'channels' }],
            } as unknown as HogFunctionType
            const inputs = {
                channels: [
                    { $integration_id: 1, access_token_raw: 'fcm-secret-token' },
                    { $integration_id: 2, signing_key: 'apns-signing-key-secret' },
                ],
            }

            const values = executor.hogExecutor.getSensitiveValues(hogFunction, inputs)

            // Without integration_multi + array handling these secrets leak into team-visible logs.
            expect(values).toContain('fcm-secret-token')
            expect(values).toContain('apns-signing-key-secret')
        })
    })

    describe('general event processing', () => {
        let hogFunction: HogFunctionType
        beforeEach(() => {
            hogFunction = createHogFunction({
                name: 'Test hog function',
                ...HOG_EXAMPLES.simple_fetch,
                ...HOG_INPUTS_EXAMPLES.simple_fetch,
                ...HOG_FILTERS_EXAMPLES.no_filters,
            })
        })

        it('can execute an invocation', async () => {
            const invocation = createExampleInvocation(hogFunction)

            const result = await executor.execute(invocation)
            expect(result).toEqual({
                capturedPostHogEvents: [],
                warehouseWebhookPayloads: [],
                messageAssets: [],
                conversionWatchers: [],
                invocation: {
                    state: {
                        globals: invocation.state.globals,
                        timings: [
                            {
                                kind: 'hog',
                                duration_ms: expect.any(Number),
                            },
                        ],
                        vmState: expect.any(Object),
                        attempts: 0,
                    },
                    id: expect.any(String),
                    teamId: 1,
                    hogFunction: invocation.hogFunction,
                    functionId: invocation.functionId,
                    queue: 'hog',
                    queueMetadata: undefined,
                    queueScheduledAt: undefined,
                    queueSource: undefined,
                    queueParameters: expect.any(Object),
                    queuePriority: 0,
                },
                finished: false,
                logs: expect.any(Array),
                metrics: [],
            })
        })

        it('can handle null input values', async () => {
            hogFunction.inputs!.debug = null
            const invocation = createExampleInvocation(hogFunction)

            const result = await executor.execute(invocation)
            expect(result.finished).toBe(false)
            expect(result.error).toBeUndefined()
        })

        it('can handle selecting entire object', async () => {
            const invocation = createExampleInvocation({
                ...hogFunction,
                inputs: {
                    ...hogFunction.inputs,
                    headers: {
                        value: {
                            [EXTEND_OBJECT_KEY]: '{person.properties}',
                        },
                        templating: 'hog',
                        bytecode: {
                            [EXTEND_OBJECT_KEY]: ['_H', 1, 32, 'properties', 32, 'person', 1, 2],
                        },
                        order: 3,
                    },
                },
            })

            invocation.state.globals.event.timestamp = '2024-06-07T12:00:00.000Z'

            const result = await executor.execute(invocation)
            expect(result.invocation.queueParameters).toMatchInlineSnapshot(`
                {
                  "body": "{"event":{"uuid":"uuid","event":"test","elements_chain":"","distinct_id":"distinct_id","url":"http://localhost:8000/events/1","properties":{"$lib_version":"1.2.3"},"timestamp":"2024-06-07T12:00:00.000Z"},"groups":{},"nested":{"foo":"http://localhost:8000/events/1"},"person":{"id":"uuid","name":"test","url":"http://localhost:8000/persons/1","properties":{"email":"[email protected]","first_name":"Pumpkin"}},"event_url":"http://localhost:8000/events/1-test"}",
                  "headers": {
                    "email": "[email protected]",
                    "first_name": "Pumpkin",
                  },
                  "method": "POST",
                  "type": "fetch",
                  "url": "https://example.com/posthog-webhook",
                }
            `)
            expect(result.finished).toBe(false)
            expect(result.error).toBeUndefined()
        })

        it('can handle selecting entire object with overrides', async () => {
            const invocation = createExampleInvocation({
                ...hogFunction,
                inputs: {
                    ...hogFunction.inputs,
                    headers: {
                        value: {
                            [EXTEND_OBJECT_KEY]: '{person.properties}',
                            email: 'email-is-hidden',
                        },
                        templating: 'hog',
                        bytecode: {
                            [EXTEND_OBJECT_KEY]: ['_H', 1, 32, 'properties', 32, 'person', 1, 2],
                            email: ['_H', 1, 32, 'email-is-hidden'],
                        },
                        order: 3,
                    },
                },
            })

            invocation.state.globals.event.timestamp = '2024-06-07T12:00:00.000Z'

            const result = await executor.execute(invocation)
            expect(result.invocation.queueParameters).toMatchInlineSnapshot(`
                {
                  "body": "{"event":{"uuid":"uuid","event":"test","elements_chain":"","distinct_id":"distinct_id","url":"http://localhost:8000/events/1","properties":{"$lib_version":"1.2.3"},"timestamp":"2024-06-07T12:00:00.000Z"},"groups":{},"nested":{"foo":"http://localhost:8000/events/1"},"person":{"id":"uuid","name":"test","url":"http://localhost:8000/persons/1","properties":{"email":"[email protected]","first_name":"Pumpkin"}},"event_url":"http://localhost:8000/events/1-test"}",
                  "headers": {
                    "email": "email-is-hidden",
                    "first_name": "Pumpkin",
                  },
                  "method": "POST",
                  "type": "fetch",
                  "url": "https://example.com/posthog-webhook",
                }
            `)
            expect(result.finished).toBe(false)
            expect(result.error).toBeUndefined()
        })

        it('collects and redacts secret values from the logs', async () => {
            const fn = createHogFunction({
                ...HOG_EXAMPLES.input_printer,
                ...HOG_INPUTS_EXAMPLES.secret_inputs,
            })
            const invocation = createExampleInvocation(fn)
            const result = await executor.execute(invocation)

            expect(cleanLogs(result.logs.map((x) => x.message))).toMatchInlineSnapshot(`
                [
                  "test",
                  "{"nested":{"foo":"***REDACTED***","null":null,"bool":false}}",
                  "{"foo":"***REDACTED***","null":null,"bool":false}",
                  "substring: ***REDACTED***",
                  "{"input_1":"test","secret_input_2":{"foo":"***REDACTED***","null":null,"bool":false},"secret_input_3":"***REDACTED***"}",
                  "Function completed in REPLACEDms. Sync: 0ms. Mem: 0.17kb. Ops: 28. Event: 'http://localhost:8000/events/1'",
                ]
            `)
        })

        it('redacts secret values from the log when the inputs fail to build', async () => {
            const fn = createHogFunction({
                ...HOG_EXAMPLES.input_printer,
                ...HOG_INPUTS_EXAMPLES.secret_inputs,
            })
            const invocation = createExampleInvocation(fn)
            delete (invocation.state.globals as any).inputs
            jest.spyOn((executor as any).hogExecutor.hogInputsService, 'buildInputsWithGlobals').mockRejectedValueOnce(
                new Error('Unsupported unit for dateDiff: super secret')
            )

            const result = await executor.execute(invocation)

            const messages = result.logs.map((x) => x.message)
            expect(messages.join('\n')).not.toContain('super secret')
            // result.error is persisted and shown in the Invocations tab, so it must be masked too.
            expect(result.error).toContain('***REDACTED***')
            expect(result.error).not.toContain('super secret')
            expect(messages).toEqual(
                expect.arrayContaining([
                    expect.stringContaining(
                        'Error building inputs: Error: Unsupported unit for dateDiff: ***REDACTED***'
                    ),
                ])
            )
        })

        it('queues up an async function call', async () => {
            const invocation = createExampleInvocation(hogFunction)
            invocation.state.globals.event.timestamp = '2024-06-07T12:00:00.000Z'
            const result = await executor.execute(invocation)

            expect(result.invocation).toMatchObject({
                queue: 'hog',
                queueParameters: {
                    type: 'fetch',
                    url: 'https://example.com/posthog-webhook',
                    method: 'POST',
                    headers: { version: 'v=1.2.3' },
                },
            })

            const body = parseJSON((result.invocation.queueParameters as any).body!)
            expect(body).toEqual({
                event: {
                    uuid: 'uuid',
                    event: 'test',
                    elements_chain: '',
                    distinct_id: 'distinct_id',
                    url: 'http://localhost:8000/events/1',
                    properties: { $lib_version: '1.2.3' },
                    timestamp: '2024-06-07T12:00:00.000Z',
                },
                groups: {},
                nested: { foo: 'http://localhost:8000/events/1' },
                person: {
                    id: 'uuid',
                    name: 'test',
                    url: 'http://localhost:8000/persons/1',
                    properties: { email: '[email protected]', first_name: 'Pumpkin' },
                },
                event_url: 'http://localhost:8000/events/1-test',
            })
        })

        it('merges previousResult into execute() result', async () => {
            const invocation = createExampleInvocation(hogFunction)

            const previousResult = {
                finished: false,
                logs: [{ level: 'info', timestamp: DateTime.utc(), message: 'Prev log' }],
                metrics: [
                    {
                        team_id: 1,
                        app_source_id: invocation.functionId,
                        metric_kind: 'other',
                        metric_name: 'prev_metric',
                        count: 1,
                    },
                ],
                capturedPostHogEvents: [
                    {
                        team_id: 1,
                        timestamp: DateTime.utc().toISO(),
                        distinct_id: 'did',
                        event: 'prev_event',
                        properties: {},
                    },
                ],
                execResult: { foo: 'bar' },
            } as any

            const result = await executor.execute(invocation, undefined, previousResult)

            // No new logs are produced before async fetch, so previous logs/metrics/events should persist
            expect(result.logs.map((l) => l.message)).toEqual(['Prev log'])
            expect(result.metrics).toEqual(previousResult.metrics)
            expect(result.capturedPostHogEvents).toEqual(previousResult.capturedPostHogEvents)
            expect(result.execResult).toEqual({ foo: 'bar' })
        })
    })

    describe('mappings', () => {
        it('rebuilds mapping inputs when an invocation arrives without inputs (rerun path)', async () => {
            // The rerun path strips `inputs` from the persisted globals and lets
            // the executor rebuild them. For mapping destinations the mapping's
            // own inputs (e.g. Google Ads `gclid`) must be re-merged — otherwise
            // they resolve to nothing and the function early-exits on rerun.
            const hog = `return inputs.gclid`
            const mappingFn = createHogFunction({
                hog,
                bytecode: await compileHog(hog),
                ...HOG_FILTERS_EXAMPLES.no_filters,
                inputs_schema: [],
                mappings: [
                    {
                        ...HOG_FILTERS_EXAMPLES.no_filters,
                        inputs: {
                            gclid: {
                                order: 0,
                                value: '{person.properties.gclid ?? person.properties.$initial_gclid}',
                                bytecode: await compileHog(
                                    'return person.properties.gclid ?? person.properties.$initial_gclid'
                                ),
                            },
                        },
                    },
                ],
            })

            const invocation = createExampleInvocation(mappingFn, {
                person: {
                    id: 'uuid',
                    name: 'test',
                    url: 'http://localhost:8000/persons/1',
                    properties: { email: '[email protected]', $initial_gclid: 'INITIAL_TOKEN_ABC' },
                },
            })
            // Simulate the rerun blob: inputs are stripped before persistence.
            expect(invocation.state.globals.inputs).toBeUndefined()

            const res = await executor.execute(invocation)
            expect(res.error).toBeUndefined()
            expect(res.execResult).toBe('INITIAL_TOKEN_ABC')
        })
    })

    describe('slow functions', () => {
        it('limits the execution time and exits appropriately', async () => {
            jest.spyOn(Date, 'now').mockRestore()

            const fn = createHogFunction({
                ...HOG_EXAMPLES.malicious_function,
                ...HOG_INPUTS_EXAMPLES.simple_fetch,
                ...HOG_FILTERS_EXAMPLES.no_filters,
            })

            const result = await executor.execute(createExampleInvocation(fn))
            expect(result.error).toContain('Execution timed out after 0.55 seconds. Performed ')

            expect(result.logs.map((log) => log.message)).toEqual([
                'I AM FIBONACCI',
                'I AM FIBONACCI',
                'I AM FIBONACCI',
                'I AM FIBONACCI',
                'I AM FIBONACCI',
                'I AM FIBONACCI',
                'I AM FIBONACCI',
                'I AM FIBONACCI',
                'I AM FIBONACCI',
                'I AM FIBONACCI',
                'I AM FIBONACCI',
                'I AM FIBONACCI',
                'I AM FIBONACCI',
                'I AM FIBONACCI',
                'I AM FIBONACCI',
                'I AM FIBONACCI',
                'I AM FIBONACCI',
                'I AM FIBONACCI',
                'I AM FIBONACCI',
                'I AM FIBONACCI',
                'I AM FIBONACCI',
                'I AM FIBONACCI',
                'I AM FIBONACCI',
                'I AM FIBONACCI',
                'Function exceeded maximum log entries. No more logs will be collected. Event: uuid',
                expect.stringContaining(
                    'Error executing function on event uuid: HogVMException: Execution timed out after 0.55 seconds. Performed'
                ),
            ])
        })
    })

    describe('result handling', () => {
        it('does not set execResult when VM returns a falsy result', async () => {
            const fn = createHogFunction({
                ...HOG_EXAMPLES.simple_fetch,
                ...HOG_INPUTS_EXAMPLES.simple_fetch,
                ...HOG_FILTERS_EXAMPLES.no_filters,
            })

            const hogExecModule = require('../utils/hog-exec')
            jest.spyOn(hogExecModule, 'execHog').mockResolvedValue({
                execResult: {
                    finished: true,
                    result: null, // falsy value
                    state: { syncDuration: 0, maxMemUsed: 1024 * 0.17, ops: 28, stack: [] },
                },
                error: undefined,
                durationMs: 1,
            })

            const res = await executor.execute(createExampleInvocation(fn))
            expect(res.finished).toBe(true)
            expect(res.execResult).toBeUndefined()
            expect(cleanLogs(res.logs.map((x) => x.message))).toEqual([
                "Function completed in REPLACEDms. Sync: 0ms. Mem: 0.17kb. Ops: 28. Event: 'http://localhost:8000/events/1'",
            ])
        })

        it('sets execResult when VM returns an object synchronously', async () => {
            // This tests a simple return statement without any async functions
            const fn = createHogFunction({
                ...HOG_EXAMPLES.simple_return_object,
                ...HOG_FILTERS_EXAMPLES.no_filters,
            })

            const res = await executor.execute(createExampleInvocation(fn))
            expect(res.finished).toBe(true)
            expect(res.execResult).toEqual({
                status: 'pending',
                priority: 'high',
                ticket_number: 42,
            })
        })
    })

    describe('posthogCaptue', () => {
        it('captures events', async () => {
            const fn = createHogFunction({
                ...HOG_EXAMPLES.posthog_capture,
                ...HOG_INPUTS_EXAMPLES.simple_fetch,
                ...HOG_FILTERS_EXAMPLES.no_filters,
            })

            const result = await executor.execute(createExampleInvocation(fn))
            expect(result?.capturedPostHogEvents).toMatchInlineSnapshot(`
                [
                  {
                    "distinct_id": "distinct_id",
                    "event": "test (copy)",
                    "properties": {
                      "$hog_function_execution_count": 1,
                    },
                    "team_id": 1,
                    "timestamp": "2025-01-01T00:00:00.000Z",
                  },
                ]
            `)
        })

        it('falls back to person.id for distinct_id when event.distinct_id is empty (batch invocations)', async () => {
            const fn = createHogFunction({
                ...HOG_EXAMPLES.posthog_capture,
                ...HOG_INPUTS_EXAMPLES.simple_fetch,
                ...HOG_FILTERS_EXAMPLES.no_filters,
            })

            const globals = createHogExecutionGlobals({
                groups: {},
                event: {
                    distinct_id: '',
                } as any,
                person: {
                    id: 'person-uuid-123',
                    name: 'Batch Person',
                    url: 'http://localhost:8000/persons/1',
                    properties: { email: '[email protected]' },
                },
            } as any)
            const result = await executor.execute(createExampleInvocation(fn, globals))
            expect(result?.capturedPostHogEvents).toHaveLength(1)
            expect(result?.capturedPostHogEvents[0].distinct_id).toBe('person-uuid-123')
        })

        it('allows events that have already used their postHogCapture a maximum of 10 times', async () => {
            const fn = createHogFunction({
                ...HOG_EXAMPLES.posthog_capture,
                ...HOG_INPUTS_EXAMPLES.simple_fetch,
                ...HOG_FILTERS_EXAMPLES.no_filters,
            })

            const globals = createHogExecutionGlobals({
                groups: {},
                event: {
                    properties: {
                        $hog_function_execution_count: 9,
                    },
                },
            } as any)
            const result = await executor.execute(createExampleInvocation(fn, globals))
            expect(result?.capturedPostHogEvents).toMatchInlineSnapshot(`
                [
                  {
                    "distinct_id": "distinct_id",
                    "event": "test (copy)",
                    "properties": {
                      "$hog_function_execution_count": 10,
                    },
                    "team_id": 1,
                    "timestamp": "2025-01-01T00:00:00.000Z",
                  },
                ]
            `)
        })

        it('ignores events that have already used their postHogCapture 10 times', async () => {
            const fn = createHogFunction({
                ...HOG_EXAMPLES.posthog_capture,
                ...HOG_INPUTS_EXAMPLES.simple_fetch,
                ...HOG_FILTERS_EXAMPLES.no_filters,
            })

            const globals = createHogExecutionGlobals({
                groups: {},
                event: {
                    properties: {
                        $hog_function_execution_count: 10,
                    },
                },
            } as any)
            const result = await executor.execute(createExampleInvocation(fn, globals))
            expect(result?.capturedPostHogEvents).toEqual([])
            expect(cleanLogs(result?.logs.map((log) => log.message) ?? [])).toMatchInlineSnapshot(`
                [
                  "postHogCapture was called from an event that already executed this function 10 times previously. To prevent unbounded infinite loops, the event was not captured.",
                  "Function completed in REPLACEDms. Sync: 0ms. Mem: 0.1kb. Ops: 15. Event: 'http://localhost:8000/events/1'",
                ]
            `)
        })
    })

    describe('postHogGetTicket and postHogUpdateTicket', () => {
        const mockExecHogForAsyncFunction = (asyncFunctionName: string, asyncFunctionArgs: any[]) => {
            const hogExecModule = require('../utils/hog-exec')
            jest.spyOn(hogExecModule, 'execHog').mockResolvedValue({
                execResult: {
                    finished: false,
                    asyncFunctionName,
                    asyncFunctionArgs,
                    state: { syncDuration: 1, maxMemUsed: 100, ops: 10, stack: [] },
                },
                error: undefined,
                durationMs: 1,
            })
        }

        // Provide pre-built inputs so buildInputsWithGlobals is skipped
        const createTicketInvocation = () =>
            createExampleInvocation(
                createHogFunction({
                    ...HOG_EXAMPLES.simple_fetch,
                    ...HOG_INPUTS_EXAMPLES.simple_fetch,
                    ...HOG_FILTERS_EXAMPLES.no_filters,
                }),
                { inputs: {} }
            )

        const TICKET_UUID = '0198a5c1-2f6e-7c3a-9b41-b6d21c0aa111'
        const requestModule = require('~/common/utils/request')

        const mockInternalResponse = (status: number, body: unknown) => ({
            status,
            headers: {},
            text: () => Promise.resolve(JSON.stringify(body)),
            json: () => Promise.resolve(body),
            dump: () => Promise.resolve(),
        })

        // In the test env the secret defaults on (mirroring Django), so the JWT path is the
        // default; legacy tests opt out the same way an unprovisioned environment does.
        const disableTicketJwt = () => {
            ;(executor as any).deps.conversationsTicketsJwt = new ScopedServiceJwt(
                PosthogJwtAudience.CONVERSATIONS_TICKETS,
                ''
            )
        }

        describe('scoped JWT path', () => {
            it('postHogGetTicket mints a team+ticket pinned token for the internal route and needs no team secret', async () => {
                const fetchSpy = jest
                    .spyOn(requestModule, 'internalFetch')
                    .mockResolvedValue(mockInternalResponse(200, { id: TICKET_UUID, status: 'new' }))
                const getTeamSpy = jest.spyOn(hub.teamManager, 'getTeam')

                mockExecHogForAsyncFunction('postHogGetTicket', [{ ticket_id: TICKET_UUID }])
                const result = await executor.execute(createTicketInvocation())

                expect(fetchSpy).toHaveBeenCalledTimes(1)
                const [url, options] = fetchSpy.mock.calls[0] as unknown as [string, FetchOptions]
                expect(url).toEqual(
                    `${hub.INTERNAL_API_BASE_URL}/api/projects/1/internal/conversations/tickets/${TICKET_UUID}`
                )
                expect(options.method).toEqual('GET')

                const headers = options.headers as Record<string, string>
                const token = headers['Authorization'].replace('Bearer ', '')
                const claims = new ScopedServiceJwt(
                    PosthogJwtAudience.CONVERSATIONS_TICKETS,
                    hub.CONVERSATIONS_TICKETS_JWT_SECRET
                ).verify(token)
                expect(claims.team_id).toEqual(1)
                expect(claims.ticket_id).toEqual(TICKET_UUID)
                expect(claims.exp! - claims.iat!).toEqual(5 * 60)

                expect(result.invocation.state.vmState!.stack).toEqual([
                    { status: 200, body: { id: TICKET_UUID, status: 'new' } },
                ])
                // No credential is needed or persisted: the team secret is never read and
                // nothing lands in queueParameters (job rows stay free of auth material).
                expect(getTeamSpy).not.toHaveBeenCalled()
                expect(result.invocation.queueParameters).toBeUndefined()
            })

            it('postHogSendTicketMessage posts the message and only marks a private note when the flag is true', async () => {
                const fetchSpy = jest
                    .spyOn(requestModule, 'internalFetch')
                    .mockResolvedValueOnce(mockInternalResponse(409, { error: 'Message is still being created' }))
                    .mockResolvedValue(mockInternalResponse(201, { id: 'message-1', is_private: true }))

                mockExecHogForAsyncFunction('postHogSendTicketMessage', [
                    { ticket_id: TICKET_UUID, message: '  Internal only  ', is_private: true },
                ])
                const invocation = createTicketInvocation()
                invocation.state.actionId = 'send_message'
                invocation.state.actionStepCount = 0
                const result = await executor.execute(invocation)

                expect(fetchSpy).toHaveBeenCalledTimes(2)
                const [url, options] = fetchSpy.mock.calls[1] as unknown as [string, FetchOptions]
                expect(url).toEqual(
                    `${hub.INTERNAL_API_BASE_URL}/api/projects/1/internal/conversations/tickets/${TICKET_UUID}`
                )
                expect(options.method).toEqual('POST')
                expect(parseJSON(options.body as string)).toEqual({
                    message: 'Internal only',
                    is_private: true,
                    idempotency_key: `${invocation.id}:send_message:0`,
                })
                expect(result.invocation.state.vmState!.stack).toEqual([
                    { status: 201, body: { id: 'message-1', is_private: true } },
                ])
            })

            it('postHogUpdateTicket forwards the updates body and workflow attribution header', async () => {
                const fetchSpy = jest
                    .spyOn(requestModule, 'internalFetch')
                    .mockResolvedValue(mockInternalResponse(200, { ok: true }))

                mockExecHogForAsyncFunction('postHogUpdateTicket', [
                    { ticket_id: TICKET_UUID, updates: { status: 'resolved', priority: 'high' } },
                ])
                const invocation = createTicketInvocation()
                ;(invocation as any).hogFlow = { id: 'flow-123' }
                await executor.execute(invocation)

                const [, options] = fetchSpy.mock.calls[0] as unknown as [string, FetchOptions]
                expect(options.method).toEqual('PATCH')
                expect(options.body).toEqual(JSON.stringify({ status: 'resolved', priority: 'high' }))
                expect((options.headers as Record<string, string>)['X-PostHog-Hog-Flow-Id']).toEqual('flow-123')
            })

            it.each(['test-ticket-123', '../../../admin', `${TICKET_UUID}/extra`])(
                'refuses a non-UUID ticket_id (%s) before any token is minted',
                async (badTicketId) => {
                    const fetchSpy = jest.spyOn(requestModule, 'internalFetch')

                    mockExecHogForAsyncFunction('postHogGetTicket', [{ ticket_id: badTicketId }])
                    const result = await executor.execute(createTicketInvocation())

                    expect(result.error).toContain('must be a UUID')
                    expect(fetchSpy).not.toHaveBeenCalled()
                }
            )

            it('lowercases a mixed-case ticket_id for the URL and the claim', async () => {
                const fetchSpy = jest
                    .spyOn(requestModule, 'internalFetch')
                    .mockResolvedValue(mockInternalResponse(200, { id: TICKET_UUID }))

                mockExecHogForAsyncFunction('postHogGetTicket', [{ ticket_id: TICKET_UUID.toUpperCase() }])
                await executor.execute(createTicketInvocation())

                const [url, options] = fetchSpy.mock.calls[0] as unknown as [string, FetchOptions]
                expect(url).toContain(`/tickets/${TICKET_UUID}`)
                const headers = options.headers as Record<string, string>
                const claims = new ScopedServiceJwt(
                    PosthogJwtAudience.CONVERSATIONS_TICKETS,
                    hub.CONVERSATIONS_TICKETS_JWT_SECRET
                ).verify(headers['Authorization'].replace('Bearer ', ''))
                expect(claims.ticket_id).toEqual(TICKET_UUID)
            })

            it('passes a 4xx through to the Hog response without retrying', async () => {
                const fetchSpy = jest
                    .spyOn(requestModule, 'internalFetch')
                    .mockResolvedValue(mockInternalResponse(404, { error: 'Ticket not found' }))

                mockExecHogForAsyncFunction('postHogGetTicket', [{ ticket_id: TICKET_UUID }])
                const result = await executor.execute(createTicketInvocation())

                expect(fetchSpy).toHaveBeenCalledTimes(1)
                expect(result.invocation.state.vmState!.stack).toEqual([
                    { status: 404, body: { error: 'Ticket not found' } },
                ])
            })

            it('retries a dropped connection and returns the eventual response', async () => {
                const fetchSpy = jest
                    .spyOn(requestModule, 'internalFetch')
                    .mockRejectedValueOnce(new Error('socket hang up'))
                    .mockResolvedValue(mockInternalResponse(200, { id: TICKET_UUID }))

                mockExecHogForAsyncFunction('postHogGetTicket', [{ ticket_id: TICKET_UUID }])
                const result = await executor.execute(createTicketInvocation())

                expect(fetchSpy).toHaveBeenCalledTimes(2)
                expect(result.invocation.state.vmState!.stack).toEqual([{ status: 200, body: { id: TICKET_UUID } }])
                expect(result.logs.some((log) => log.message.includes('Retrying'))).toBe(true)
            })

            it('pushes a status-500 response after exhausting connection retries', async () => {
                const fetchSpy = jest
                    .spyOn(requestModule, 'internalFetch')
                    .mockRejectedValue(new Error('connect ECONNREFUSED'))

                mockExecHogForAsyncFunction('postHogGetTicket', [{ ticket_id: TICKET_UUID }])
                const result = await executor.execute(createTicketInvocation())

                expect(fetchSpy).toHaveBeenCalledTimes(3)
                const [pushed] = result.invocation.state.vmState!.stack as [{ status: number; body: string }]
                expect(pushed.status).toEqual(500)
                expect(pushed.body).toContain('ECONNREFUSED')
            })

            it('retries and then fails loudly when the body stream fails after a 200 header', async () => {
                const fetchSpy = jest.spyOn(requestModule, 'internalFetch').mockResolvedValue({
                    status: 200,
                    headers: {},
                    text: () => Promise.reject(new Error('other side closed')),
                    json: () => Promise.reject(new Error('other side closed')),
                    dump: () => Promise.resolve(),
                })

                mockExecHogForAsyncFunction('postHogGetTicket', [{ ticket_id: TICKET_UUID }])
                const result = await executor.execute(createTicketInvocation())

                // A 200 header with an unreadable body is not a success: it retries like any
                // other failed attempt instead of silently returning an empty ticket.
                expect(fetchSpy).toHaveBeenCalledTimes(3)
                const [pushed] = result.invocation.state.vmState!.stack as [{ status: number; body: string }]
                expect(pushed.status).toEqual(500)
                expect(pushed.body).toContain('other side closed')
            })

            it('preserves the upstream body on the final attempt of a retriable status', async () => {
                const fetchSpy = jest
                    .spyOn(requestModule, 'internalFetch')
                    .mockResolvedValue(mockInternalResponse(503, { error: 'database unavailable' }))

                mockExecHogForAsyncFunction('postHogGetTicket', [{ ticket_id: TICKET_UUID }])
                const result = await executor.execute(createTicketInvocation())

                expect(fetchSpy).toHaveBeenCalledTimes(3)
                expect(result.invocation.state.vmState!.stack).toEqual([
                    { status: 503, body: { error: 'database unavailable' } },
                ])
            })

            it('retries a transient 500 the same way the legacy fetch path does', async () => {
                const fetchSpy = jest
                    .spyOn(requestModule, 'internalFetch')
                    .mockResolvedValueOnce(mockInternalResponse(500, { error: 'boom' }))
                    .mockResolvedValue(mockInternalResponse(200, { id: TICKET_UUID }))

                mockExecHogForAsyncFunction('postHogGetTicket', [{ ticket_id: TICKET_UUID }])
                const result = await executor.execute(createTicketInvocation())

                expect(fetchSpy).toHaveBeenCalledTimes(2)
                expect(result.invocation.state.vmState!.stack).toEqual([{ status: 200, body: { id: TICKET_UUID } }])
            })

            it('rejects a second inline ticket call once the per-invocation async budget is spent', async () => {
                const fetchSpy = jest
                    .spyOn(requestModule, 'internalFetch')
                    .mockResolvedValue(mockInternalResponse(200, { id: TICKET_UUID }))

                const bytecode = await compileHog(`
                    let a := postHogGetTicket({'ticket_id': '${TICKET_UUID}'})
                    let b := postHogGetTicket({'ticket_id': '${TICKET_UUID}'})
                    return b
                `)
                const invocation = createExampleInvocation(createHogFunction({ bytecode }), { inputs: {} })

                const result = await executor.executeWithAsyncFunctions(invocation)

                // Neither ticket call sets queueParameters, so without a shared budget both
                // would run inline in the same worker slot regardless of maxAsyncFunctions.
                expect(fetchSpy).toHaveBeenCalledTimes(1)
                expect(result.error).toContain('Max async functions reached')
                expect(result.finished).toBe(true)
            })
        })

        describe('unprovisioned JWT secret', () => {
            it('fails the action with instructions naming the env var', async () => {
                disableTicketJwt()
                const fetchSpy = jest.spyOn(requestModule, 'internalFetch')

                mockExecHogForAsyncFunction('postHogGetTicket', [{ ticket_id: TICKET_UUID }])
                const result = await executor.execute(createTicketInvocation())

                expect(result.error).toContain('CONVERSATIONS_TICKETS_JWT_SECRET')
                expect(result.error).toContain('ticket workflow actions')
                expect(result.error).not.toContain('[')
                expect(fetchSpy).not.toHaveBeenCalled()
            })
        })

        it.each(['postHogGetTicket', 'postHogUpdateTicket', 'postHogSendTicketMessage'])(
            '%s errors when ticket_id is missing',
            async (name) => {
                const args =
                    name === 'postHogUpdateTicket'
                        ? [{ updates: { status: 'new' } }]
                        : name === 'postHogSendTicketMessage'
                          ? [{ message: 'hello' }]
                          : [{}]
                mockExecHogForAsyncFunction(name, args)

                const result = await executor.execute(createTicketInvocation())
                expect(result.error).toContain("missing 'ticket_id'")
            }
        )

        it('postHogSendTicketMessage errors when message is missing', async () => {
            mockExecHogForAsyncFunction('postHogSendTicketMessage', [{ ticket_id: TICKET_UUID, message: '   ' }])

            const result = await executor.execute(createTicketInvocation())
            expect(result.error).toContain("missing 'message'")
        })

        it('postHogSendTicketMessage errors outside a workflow step', async () => {
            mockExecHogForAsyncFunction('postHogSendTicketMessage', [{ ticket_id: TICKET_UUID, message: 'hello' }])

            const result = await executor.execute(createTicketInvocation())
            expect(result.error).toContain('only runs inside a workflow')
        })
    })

    describe('account workflow actions', () => {
        const mockExecHogForAsyncFunction = (asyncFunctionName: string, asyncFunctionArgs: any[]) => {
            const hogExecModule = require('../utils/hog-exec')
            jest.spyOn(hogExecModule, 'execHog').mockResolvedValue({
                execResult: {
                    finished: false,
                    asyncFunctionName,
                    asyncFunctionArgs,
                    state: { syncDuration: 1, maxMemUsed: 100, ops: 10, stack: [] },
                },
                error: undefined,
                durationMs: 1,
            })
        }

        const createAccountInvocation = () =>
            createExampleInvocation(
                createHogFunction({
                    ...HOG_EXAMPLES.simple_fetch,
                    ...HOG_INPUTS_EXAMPLES.simple_fetch,
                    ...HOG_FILTERS_EXAMPLES.no_filters,
                }),
                { inputs: {} }
            )

        const requestModule = require('~/common/utils/request')

        const mockInternalResponse = (status: number, body: unknown) => ({
            status,
            headers: {},
            text: () => Promise.resolve(JSON.stringify(body)),
            json: () => Promise.resolve(body),
            dump: () => Promise.resolve(),
        })

        const verifyAccountClaims = (headers: Record<string, string>) =>
            new ScopedServiceJwt(
                PosthogJwtAudience.CUSTOMER_ANALYTICS_ACCOUNTS,
                hub.CUSTOMER_ANALYTICS_ACCOUNTS_JWT_SECRET
            ).verify(headers['Authorization'].replace('Bearer ', ''))

        // In the test env the secret defaults on (mirroring Django), so the JWT path is the
        // default; legacy tests opt out the same way an unprovisioned environment does.
        const disableAccountJwt = () => {
            ;(executor as any).deps.customerAnalyticsAccountsJwt = new ScopedServiceJwt(
                PosthogJwtAudience.CUSTOMER_ANALYTICS_ACCOUNTS,
                ''
            )
        }

        describe('scoped JWT path', () => {
            // Transport behavior (retries, 4xx pass-through, async budget) is shared with the
            // ticket actions and covered by their suite; these pin the account-specific wiring.

            it('postHogGetAccount mints a team+account pinned token for the internal route and needs no team secret', async () => {
                const fetchSpy = jest
                    .spyOn(requestModule, 'internalFetch')
                    .mockResolvedValue(mockInternalResponse(200, { external_id: 'acme corp/1', name: 'Acme' }))
                const getTeamSpy = jest.spyOn(hub.teamManager, 'getTeam')

                mockExecHogForAsyncFunction('postHogGetAccount', [{ external_id: 'acme corp/1' }])
                const result = await executor.execute(createAccountInvocation())

                expect(fetchSpy).toHaveBeenCalledTimes(1)
                const [url, options] = fetchSpy.mock.calls[0] as unknown as [string, FetchOptions]
                // URL-encoded in the query string; the claim carries the raw value Django reads
                // back from the decoded query param.
                expect(url).toEqual(
                    `${hub.INTERNAL_API_BASE_URL}/api/projects/1/internal/customer_analytics/account?external_id=acme%20corp%2F1`
                )
                expect(options.method).toEqual('GET')

                const claims = verifyAccountClaims(options.headers as Record<string, string>)
                expect(claims.team_id).toEqual(1)
                expect(claims.external_id).toEqual('acme corp/1')
                expect(claims.exp! - claims.iat!).toEqual(5 * 60)

                expect(result.invocation.state.vmState!.stack).toEqual([
                    { status: 200, body: { external_id: 'acme corp/1', name: 'Acme' } },
                ])
                // No credential is needed or persisted: the team secret is never read and
                // nothing lands in queueParameters (job rows stay free of auth material).
                expect(getTeamSpy).not.toHaveBeenCalled()
                expect(result.invocation.queueParameters).toBeUndefined()
            })

            it('postHogUpdateAccount pins the claimed external_id over the updates body', async () => {
                const fetchSpy = jest
                    .spyOn(requestModule, 'internalFetch')
                    .mockResolvedValue(mockInternalResponse(200, { ok: true }))

                mockExecHogForAsyncFunction('postHogUpdateAccount', [
                    // A Hog-provided updates.external_id must not redirect the write away from
                    // the account the token is pinned to.
                    { external_id: 'acme-1', updates: { tags: ['enterprise'], external_id: 'other-account' } },
                ])
                const invocation = createAccountInvocation()
                ;(invocation as any).hogFlow = { id: 'flow-123' }
                await executor.execute(invocation)

                const [url, options] = fetchSpy.mock.calls[0] as unknown as [string, FetchOptions]
                expect(url).toEqual(`${hub.INTERNAL_API_BASE_URL}/api/projects/1/internal/customer_analytics/account`)
                expect(options.method).toEqual('PATCH')
                expect(options.body).toEqual(JSON.stringify({ tags: ['enterprise'], external_id: 'acme-1' }))
                expect((options.headers as Record<string, string>)['X-PostHog-Hog-Flow-Id']).toEqual('flow-123')
                const claims = verifyAccountClaims(options.headers as Record<string, string>)
                expect(claims.external_id).toEqual('acme-1')
            })

            it('postHogSetAccountProperties sends an explicit property clear as API null', async () => {
                const fetchSpy = jest
                    .spyOn(requestModule, 'internalFetch')
                    .mockResolvedValue(mockInternalResponse(200, { ok: true }))

                mockExecHogForAsyncFunction('postHogSetAccountProperties', [
                    { external_id: 'acme-1', properties: { 'def-1': { __posthog_clear_property: true } } },
                ])
                await executor.execute(createAccountInvocation())

                const [url, options] = fetchSpy.mock.calls[0] as unknown as [string, FetchOptions]
                expect(url).toEqual(
                    `${hub.INTERNAL_API_BASE_URL}/api/projects/1/internal/customer_analytics/account/custom_property_values`
                )
                expect(options.method).toEqual('PATCH')
                expect(options.body).toEqual(JSON.stringify({ external_id: 'acme-1', properties: { 'def-1': null } }))
            })

            it('postHogCreateAccount posts the claimed external_id with workflow attribution', async () => {
                const fetchSpy = jest
                    .spyOn(requestModule, 'internalFetch')
                    .mockResolvedValue(mockInternalResponse(201, { external_id: 'acme-new' }))

                mockExecHogForAsyncFunction('postHogCreateAccount', [{ external_id: 'acme-new' }])
                const invocation = createAccountInvocation()
                ;(invocation as any).hogFlow = { id: 'flow-123' }
                const result = await executor.execute(invocation)

                const [url, options] = fetchSpy.mock.calls[0] as unknown as [string, FetchOptions]
                expect(url).toEqual(`${hub.INTERNAL_API_BASE_URL}/api/projects/1/internal/customer_analytics/account`)
                expect(options.method).toEqual('POST')
                expect(options.body).toEqual(JSON.stringify({ external_id: 'acme-new' }))
                expect((options.headers as Record<string, string>)['X-PostHog-Hog-Flow-Id']).toEqual('flow-123')
                const claims = verifyAccountClaims(options.headers as Record<string, string>)
                expect(claims.external_id).toEqual('acme-new')
                expect(result.invocation.state.vmState!.stack).toEqual([
                    { status: 201, body: { external_id: 'acme-new' } },
                ])
            })
        })

        describe('unprovisioned JWT secret', () => {
            it('fails the action with instructions naming the env var', async () => {
                disableAccountJwt()
                const fetchSpy = jest.spyOn(requestModule, 'internalFetch')

                mockExecHogForAsyncFunction('postHogGetAccount', [{ external_id: 'acme-1' }])
                const result = await executor.execute(createAccountInvocation())

                expect(result.error).toContain('CUSTOMER_ANALYTICS_ACCOUNTS_JWT_SECRET')
                expect(result.error).toContain('account workflow actions')
                expect(result.error).not.toContain('[')
                expect(fetchSpy).not.toHaveBeenCalled()
            })
        })

        it.each(['postHogGetAccount', 'postHogUpdateAccount', 'postHogSetAccountProperties', 'postHogCreateAccount'])(
            '%s errors when external_id is missing',
            async (name) => {
                mockExecHogForAsyncFunction(name, [{}])

                const result = await executor.execute(createAccountInvocation())
                expect(result.error).toContain("missing 'external_id'")
            }
        )
    })

    describe('produceToWarehouseWebhooks', () => {
        const buildInvocation = async (code: string): Promise<CyclotronJobInvocationHogFunction> => {
            const bytecode = await compileHog(code)
            return createExampleInvocation(createHogFunction({ bytecode }), { inputs: {} })
        }

        // Regression test for a stack-empty crash when an async function with no
        // meaningful return value is called as an expression statement.
        // The bytecode compiler emits a trailing POP after every expression
        // statement, but the generic async function path in execute() never pushed
        // a return value onto the resumed VM stack — so when the cyclotron worker
        // resumed the invocation, the POP fired against an empty stack and raised
        // "Invalid HogQL bytecode, stack is empty, can not pop".
        it('finishes cleanly when called as an expression statement', async () => {
            const invocation = await buildInvocation(`produceToWarehouseWebhooks({'foo': 'bar'}, 'test-schema-id')`)

            const result = await executor.executeWithAsyncFunctions(invocation)

            expect(result.error).toBeUndefined()
            expect(result.finished).toBe(true)
            expect(result.warehouseWebhookPayloads).toHaveLength(1)
            expect(result.warehouseWebhookPayloads[0]).toMatchObject({
                schema_id: 'test-schema-id',
                payload: { foo: 'bar' },
            })
        })

        it('finishes cleanly when followed by another statement', async () => {
            const invocation = await buildInvocation(
                `produceToWarehouseWebhooks({'foo': 'bar'}, 'test-schema-id')
                 print('after produce')`
            )

            const result = await executor.executeWithAsyncFunctions(invocation)

            expect(result.error).toBeUndefined()
            expect(result.finished).toBe(true)
            expect(result.warehouseWebhookPayloads).toHaveLength(1)
        })
    })

    describe('fetch does not allow internal flag', () => {
        it('regular fetch call does not pass through internal flag', async () => {
            const invocation = createExampleInvocation(
                createHogFunction({
                    ...HOG_EXAMPLES.simple_fetch,
                    ...HOG_INPUTS_EXAMPLES.simple_fetch,
                    ...HOG_FILTERS_EXAMPLES.no_filters,
                })
            )

            const result = await executor.execute(invocation)

            // The fetch case handler only picks url/method/body/headers — internal is never passed
            expect(result.invocation.queueParameters).toBeDefined()
            expect((result.invocation.queueParameters as any).type).toBe('fetch')
            expect((result.invocation.queueParameters as any).internal).toBeUndefined()
        })
    })

    describe('executeFetch', () => {
        jest.setTimeout(10000)
        let server: any
        let baseUrl: string
        const mockRequest = jest.fn()
        let hogFunction: HogFunctionType

        beforeAll(async () => {
            server = createServer((req, res) => {
                mockRequest(req, res)
            })

            await promisifyCallback<void>((cb) => {
                server.listen(0, () => {
                    logger.info('Server listening')
                    cb(null, server)
                })
            })
            const address = server.address() as AddressInfo
            baseUrl = `http://localhost:${address.port}`

            hogFunction = createHogFunction({
                name: 'Test hog function',
                ...HOG_EXAMPLES.simple_fetch,
                ...HOG_INPUTS_EXAMPLES.simple_fetch,
                ...HOG_FILTERS_EXAMPLES.no_filters,
            })
        })

        afterAll(async () => {
            logger.info('Closing server')
            await promisifyCallback<void>((cb) => {
                logger.info('Closed server')
                server.close(cb)
            })
        })

        beforeEach(() => {
            jest.spyOn(Math, 'random').mockReturnValue(0.5)

            mockRequest.mockImplementation((req, res) => {
                res.writeHead(200, { 'Content-Type': 'text/plain' })
                res.end('Hello, world!')
            })
        })

        const createFetchInvocation = async (
            params: Omit<CyclotronInvocationQueueParametersFetchType, 'type'>
        ): Promise<CyclotronJobInvocationHogFunction> => {
            const invocation = createExampleInvocation(hogFunction)

            // Execute just to have an expecting stack
            const res = await executor.execute(invocation)
            expect(res.invocation.queueParameters?.type).toBe('fetch')

            // Simulate what the callback does
            invocation.queue = 'hog'
            invocation.queueParameters = {
                type: 'fetch',
                ...params,
            } as any
            return invocation
        }

        it('completes successful fetch', async () => {
            const invocation = await createFetchInvocation({
                url: `${baseUrl}/test`,
                method: 'GET',
                body: 'test body',
            })

            const result = await executor.executeFetch(invocation)

            expect(mockRequest).toHaveBeenCalledWith(
                expect.objectContaining({
                    method: 'GET',
                    url: '/test',
                }),
                expect.any(Object)
            )

            // General check for clearance of the invocation
            expect(result.finished).toBe(false)
            expect(result.error).toBeUndefined()
            expect(result.invocation.queue).toBe('hog')
            expect(result.invocation.queueParameters).toBeUndefined()
            expect(result.invocation.queueMetadata).toBeUndefined()
            expect(result.invocation.queuePriority).toEqual(0)
            expect(result.invocation.queueScheduledAt).toBeUndefined()

            // State checks
            expect(result.invocation.state.attempts).toBe(0)
            expect(result.invocation.state.timings.slice(-1)).toEqual([
                expect.objectContaining({
                    kind: 'async_function',
                    duration_ms: expect.any(Number),
                }),
            ])

            expect(result.invocation.state.vmState!.stack.slice(-1)).toEqual([
                {
                    status: 200,
                    body: 'Hello, world!',
                },
            ])

            // Now also exposed on the execResult for callers of execute()
            expect(result.execResult).toEqual({ status: 200, body: 'Hello, world!' })
        })

        it('handles failure status and retries', async () => {
            mockRequest.mockImplementation((req: any, res: any) => {
                res.writeHead(500, { 'Content-Type': 'text/plain' })
                res.end('test server error body')
            })

            const invocation = await createFetchInvocation({
                url: `${baseUrl}/test`,
                method: 'GET',
                max_tries: 2,
            })

            const vmStateStackLength = invocation.state.vmState!.stack.length

            let result = await executor.executeFetch(invocation)

            // Should be scheduled for retry
            expect(result.invocation.state.attempts).toBe(1)
            expect(result.logs.map((log) => log.message)).toEqual([
                'HTTP fetch failed on attempt 1 with status code 500. Retrying.',
            ])
            expect(result.invocation.queuePriority).toBe(1) // Priority decreased
            expect(result.invocation.queueScheduledAt?.toISO()).toMatchInlineSnapshot(`"2025-01-01T00:00:01.500Z"`)
            expect(result.invocation.state.vmState!.stack.length).toBe(vmStateStackLength)

            // Execute the retry
            result = await executor.executeFetch(result.invocation)
            expect(result.invocation.state.attempts).toBe(2)
            expect(result.logs.map((log) => log.message)).toEqual([
                'HTTP fetch failed on attempt 2 with status code 500. Retrying.',
            ])
            expect(result.invocation.queuePriority).toBe(2) // Priority decreased
            expect(result.invocation.queueScheduledAt?.toISO()).toMatchInlineSnapshot(`"2025-01-01T00:00:02.500Z"`)
            expect(result.invocation.state.vmState!.stack.length).toBe(vmStateStackLength)
            // Execute the final retry
            result = await executor.executeFetch(result.invocation)
            expect(result.logs.map((log) => log.message)).toEqual([
                'HTTP fetch failed on attempt 3 with status code 500.',
            ])
            // All values reset due to no longer retrying
            expect(result.invocation.state.attempts).toBe(0)
            expect(result.invocation.queuePriority).toBe(0) // Priority reset as we are no longer retrying
            expect(result.invocation.queueScheduledAt).toBeUndefined()
            // Should now be complete with failure response
            expect(result.invocation.state.vmState!.stack.length).toBe(vmStateStackLength + 1)
            const response = result.invocation.state.vmState!.stack.slice(-1)[0]
            expect(response).toMatchInlineSnapshot(`
                {
                  "body": "test server error body",
                  "status": 500,
                }
            `)
            expect(result.invocation.queue).toBe('hog')
        })

        it('sets result.error after retries are exhausted', async () => {
            mockRequest.mockImplementation((req: any, res: any) => {
                res.writeHead(500, { 'Content-Type': 'text/plain' })
                res.end('server error')
            })

            const invocation = await createFetchInvocation({
                url: `${baseUrl}/test`,
                method: 'GET',
            })

            const maxRetries = executor['config'].fetchRetries
            let result = await executor.executeFetch(invocation)

            for (let attempt = 1; attempt < maxRetries; attempt++) {
                expect(result.error).toBeUndefined()
                expect(result.invocation.state.attempts).toBe(attempt)
                expect(result.invocation.queueScheduledAt).toBeDefined()
                result = await executor.executeFetch(result.invocation)
            }

            expect(result.error).toBeInstanceOf(Error)
            expect(result.error.message).toContain(`HTTP fetch failed on attempt ${maxRetries}`)
            expect(result.error.message).toContain('with status code 500')
            expect(result.invocation.queueScheduledAt).toBeUndefined()
        })

        describe('aws_sigv4', () => {
            // `secret: true` HogFunction inputs land in `encrypted_inputs` after
            // Django's `move_secret_inputs` runs on save. The Node manager decrypts
            // `encrypted_inputs` in memory before the executor sees the function, so
            // by the time we reach the fetch path it's a plaintext map keyed by
            // input name. Seed *that* field — not `inputs` — so the tests mirror the
            // production data shape for the Kinesis template.
            const seedAwsCredentialInputs = (invocation: CyclotronJobInvocationHogFunction) => {
                invocation.hogFunction.encrypted_inputs = {
                    ...(invocation.hogFunction.encrypted_inputs ?? {}),
                    aws_access_key_id: { value: 'AKIDEXAMPLE' },
                    aws_secret_access_key: { value: 'wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY' },
                } as any
            }

            const sigv4Refs = {
                service: 'kinesis',
                region: 'us-east-1',
                access_key_id_input: 'aws_access_key_id',
                secret_access_key_input: 'aws_secret_access_key',
            }

            it('signs the request and Authorization header arrives at the upstream', async () => {
                let receivedAuth: string | undefined
                let receivedAmzDate: string | undefined
                mockRequest.mockImplementation((req: any, res: any) => {
                    receivedAuth = req.headers.authorization
                    receivedAmzDate = req.headers['x-amz-date']
                    res.writeHead(200, { 'Content-Type': 'text/plain' })
                    res.end('ok')
                })

                const invocation = await createFetchInvocation({
                    url: `${baseUrl}/`,
                    method: 'POST',
                    body: '{}',
                    headers: { 'Content-Type': 'application/x-amz-json-1.1' },
                    aws_sigv4: sigv4Refs,
                })
                seedAwsCredentialInputs(invocation)

                await executor.executeFetch(invocation)

                expect(receivedAmzDate).toBe('20250101T000000Z')
                expect(receivedAuth).toMatch(
                    /^AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE\/20250101\/us-east-1\/kinesis\/aws4_request, SignedHeaders=[a-z0-9;-]+, Signature=[a-f0-9]{64}$/
                )
            })

            // Customer impact this prevents: the original bug ticket had a request
            // signed at T, queued behind a timed-out attempt, then retried >5 min
            // later — AWS rejected the retry with InvalidSignatureException because
            // the X-Amz-Date inside the Authorization no longer matched server time.
            it('produces a fresh signature on retry instead of reusing the original', async () => {
                const receivedAuthHeaders: string[] = []
                const receivedAmzDates: string[] = []
                let callCount = 0
                mockRequest.mockImplementation((req: any, res: any) => {
                    receivedAuthHeaders.push(req.headers.authorization)
                    receivedAmzDates.push(req.headers['x-amz-date'])
                    callCount++
                    if (callCount === 1) {
                        res.writeHead(500, { 'Content-Type': 'text/plain' })
                        res.end('first attempt fails')
                    } else {
                        res.writeHead(200, { 'Content-Type': 'text/plain' })
                        res.end('second attempt ok')
                    }
                })

                const invocation = await createFetchInvocation({
                    url: `${baseUrl}/`,
                    method: 'POST',
                    body: '{}',
                    headers: { 'Content-Type': 'application/x-amz-json-1.1' },
                    aws_sigv4: sigv4Refs,
                })
                seedAwsCredentialInputs(invocation)

                let result = await executor.executeFetch(invocation)
                expect(result.invocation.state.attempts).toBe(1)

                // Simulate the cyclotron queue + backoff: by the time the retry
                // actually runs, >6 minutes have passed. With the old "sign once
                // in Hog" path this would push the signature past AWS's 5-minute
                // window and the retry would 400 with InvalidSignatureException.
                const retryTime = DateTime.fromObject({ year: 2025, month: 1, day: 1 }, { zone: 'UTC' }).plus({
                    minutes: 6,
                })
                jest.spyOn(Date, 'now').mockReturnValue(retryTime.toMillis())

                result = await executor.executeFetch(result.invocation)

                expect(receivedAmzDates[0]).toBe('20250101T000000Z')
                expect(receivedAmzDates[1]).toBe('20250101T000600Z')
                expect(receivedAuthHeaders[0]).not.toBe(receivedAuthHeaders[1])
                expect(result.error).toBeUndefined()
            })

            // Defense in depth: queue payloads should never carry a stale
            // Authorization, but if one ever leaks in (e.g. through a custom
            // template that sets it directly), it must not be used.
            it('overwrites any stale Authorization header sitting in the queue payload', async () => {
                let receivedAuth: string | undefined
                mockRequest.mockImplementation((req: any, res: any) => {
                    receivedAuth = req.headers.authorization
                    res.writeHead(200, { 'Content-Type': 'text/plain' })
                    res.end('ok')
                })

                const invocation = await createFetchInvocation({
                    url: `${baseUrl}/`,
                    method: 'POST',
                    body: '{}',
                    headers: {
                        'Content-Type': 'application/x-amz-json-1.1',
                        Authorization: 'AWS4-HMAC-SHA256 Credential=STALE/19700101/us-east-1/kinesis/aws4_request, ...',
                        'X-Amz-Date': '19700101T000000Z',
                    },
                    aws_sigv4: sigv4Refs,
                })
                seedAwsCredentialInputs(invocation)

                await executor.executeFetch(invocation)

                expect(receivedAuth).not.toContain('STALE')
                expect(receivedAuth).toContain('AKIDEXAMPLE/20250101/us-east-1/kinesis/aws4_request')
            })

            // Defense in depth for an unusual case: if a custom template wires
            // up the credential as a non-secret input (`secret: false`), the value
            // will live on `inputs` rather than `encrypted_inputs`. The lookup
            // must fall through to `inputs` so signing still works.
            it('falls back to plaintext inputs when encrypted_inputs does not carry the credential', async () => {
                let receivedAuth: string | undefined
                mockRequest.mockImplementation((req: any, res: any) => {
                    receivedAuth = req.headers.authorization
                    res.writeHead(200, { 'Content-Type': 'text/plain' })
                    res.end('ok')
                })

                const invocation = await createFetchInvocation({
                    url: `${baseUrl}/`,
                    method: 'POST',
                    body: '{}',
                    headers: { 'Content-Type': 'application/x-amz-json-1.1' },
                    aws_sigv4: sigv4Refs,
                })
                invocation.hogFunction.inputs = {
                    ...(invocation.hogFunction.inputs ?? {}),
                    aws_access_key_id: { value: 'AKIDEXAMPLE' },
                    aws_secret_access_key: { value: 'wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY' },
                } as any

                await executor.executeFetch(invocation)

                expect(receivedAuth).toContain('AKIDEXAMPLE/20250101/us-east-1/kinesis/aws4_request')
            })

            // If the input referenced by `*_input` is missing or non-string we must
            // NOT fall through and ship an unsigned request to AWS — that'd 403 and
            // potentially leak the request body in error logs.
            it('errors loudly when the referenced credential input is missing', async () => {
                mockRequest.mockImplementation((req: any, res: any) => {
                    res.writeHead(200, { 'Content-Type': 'text/plain' })
                    res.end('ok')
                })

                const invocation = await createFetchInvocation({
                    url: `${baseUrl}/`,
                    method: 'POST',
                    body: '{}',
                    headers: { 'Content-Type': 'application/x-amz-json-1.1' },
                    aws_sigv4: sigv4Refs,
                })
                // Intentionally do NOT seed inputs.

                const result = await executor.executeFetch(invocation)

                expect(mockRequest).not.toHaveBeenCalled()
                expect(result.error).toBeInstanceOf(Error)
                expect(result.error.message).toContain('AWS SigV4 signing failed')
                expect(result.error.message).toContain('aws_access_key_id')
                expect(result.error.message).toContain('aws_secret_access_key')
            })
        })

        describe('standard_webhooks', () => {
            // Secret from the Standard Webhooks spec's reference example. The tests
            // recompute the expected signature with the base64-decoded key, exactly
            // as a receiver's verification library would.
            const KEY = Buffer.from('MfKQ9r8GKYqrTwjUPD8ILPZIo2LaLaSw', 'base64')
            const BODY = '{"test": 2432232314}'
            // Shaped like what the fetch async function puts on the queue payload.
            const SIGNING_REFS = {
                secret_input: 'signing_secret',
                webhook_id: '4f6c9f2a-5b1e-4f5b-9d3c-2a7e8c1d0b64',
            }

            const seedSigningSecretInput = (invocation: CyclotronJobInvocationHogFunction) => {
                invocation.hogFunction.encrypted_inputs = {
                    ...(invocation.hogFunction.encrypted_inputs ?? {}),
                    signing_secret: { value: 'whsec_MfKQ9r8GKYqrTwjUPD8ILPZIo2LaLaSw' },
                } as any
            }

            const expectedSignature = (webhookId: string, timestamp: string): string =>
                `v1,${createHmac('sha256', KEY).update(`${webhookId}.${timestamp}.${BODY}`).digest('base64')}`

            it('signs the request so the reference verification succeeds upstream', async () => {
                let receivedId: string | undefined
                let receivedTimestamp: string | undefined
                let receivedSignature: string | undefined
                mockRequest.mockImplementation((req: any, res: any) => {
                    receivedId = req.headers['webhook-id']
                    receivedTimestamp = req.headers['webhook-timestamp']
                    receivedSignature = req.headers['webhook-signature']
                    res.writeHead(200, { 'Content-Type': 'text/plain' })
                    res.end('ok')
                })

                const invocation = await createFetchInvocation({
                    url: `${baseUrl}/`,
                    method: 'POST',
                    body: BODY,
                    headers: { 'Content-Type': 'application/json' },
                    standard_webhooks: SIGNING_REFS,
                })
                seedSigningSecretInput(invocation)

                const result = await executor.executeFetch(invocation)

                expect(result.error).toBeUndefined()
                expect(receivedId).toBe(SIGNING_REFS.webhook_id)
                // Date.now is mocked to 2025-01-01T00:00:00Z in beforeEach
                expect(receivedTimestamp).toBe('1735689600')
                expect(receivedSignature).toBe(expectedSignature(receivedId!, receivedTimestamp!))
            })

            // The spec wants webhook-id constant across retries (the receiver's
            // idempotency key) while the timestamp must be fresh so the retry
            // stays within the receiver's tolerance window.
            it('keeps webhook-id constant across retries while refreshing the timestamp and signature', async () => {
                const receivedIds: string[] = []
                const receivedTimestamps: string[] = []
                const receivedSignatures: string[] = []
                let callCount = 0
                mockRequest.mockImplementation((req: any, res: any) => {
                    receivedIds.push(req.headers['webhook-id'])
                    receivedTimestamps.push(req.headers['webhook-timestamp'])
                    receivedSignatures.push(req.headers['webhook-signature'])
                    callCount++
                    if (callCount === 1) {
                        res.writeHead(500, { 'Content-Type': 'text/plain' })
                        res.end('first attempt fails')
                    } else {
                        res.writeHead(200, { 'Content-Type': 'text/plain' })
                        res.end('second attempt ok')
                    }
                })

                const invocation = await createFetchInvocation({
                    url: `${baseUrl}/`,
                    method: 'POST',
                    body: BODY,
                    headers: { 'Content-Type': 'application/json' },
                    standard_webhooks: SIGNING_REFS,
                })
                seedSigningSecretInput(invocation)

                let result = await executor.executeFetch(invocation)
                expect(result.invocation.state.attempts).toBe(1)

                const retryTime = DateTime.fromObject({ year: 2025, month: 1, day: 1 }, { zone: 'UTC' }).plus({
                    minutes: 6,
                })
                jest.spyOn(Date, 'now').mockReturnValue(retryTime.toMillis())

                result = await executor.executeFetch(result.invocation)

                expect(result.error).toBeUndefined()
                expect(receivedIds[0]).toBe(receivedIds[1])
                expect(receivedTimestamps).toEqual(['1735689600', '1735689960'])
                expect(receivedSignatures[0]).not.toBe(receivedSignatures[1])
                expect(receivedSignatures[1]).toBe(expectedSignature(receivedIds[1], receivedTimestamps[1]))
            })

            it('errors loudly instead of sending an unsigned request when the secret input is missing', async () => {
                mockRequest.mockImplementation((req: any, res: any) => {
                    res.writeHead(200, { 'Content-Type': 'text/plain' })
                    res.end('ok')
                })

                const invocation = await createFetchInvocation({
                    url: `${baseUrl}/`,
                    method: 'POST',
                    body: BODY,
                    headers: { 'Content-Type': 'application/json' },
                    standard_webhooks: SIGNING_REFS,
                })
                // Intentionally do NOT seed inputs.

                const result = await executor.executeFetch(invocation)

                expect(mockRequest).not.toHaveBeenCalled()
                expect(result.error).toBeInstanceOf(Error)
                expect(result.error.message).toContain('Standard Webhooks signing failed')
                expect(result.error.message).toContain('signing_secret')
            })
        })

        describe('secret_headers_input', () => {
            const seedSecretHeadersInput = (invocation: CyclotronJobInvocationHogFunction) => {
                invocation.hogFunction.encrypted_inputs = {
                    ...(invocation.hogFunction.encrypted_inputs ?? {}),
                    secret_headers: { value: { authorization: 'Bearer sk_test_token', 'X-Api-Key': 'key_test_token' } },
                } as any
            }

            it('merges the resolved headers into every attempt without persisting them to the queue payload', async () => {
                const receivedHeaders: Record<string, string | string[] | undefined>[] = []
                let callCount = 0
                mockRequest.mockImplementation((req: any, res: any) => {
                    receivedHeaders.push(req.headers)
                    callCount++
                    res.writeHead(callCount === 1 ? 500 : 200, { 'Content-Type': 'text/plain' })
                    res.end('ok')
                })

                const invocation = await createFetchInvocation({
                    url: `${baseUrl}/`,
                    method: 'POST',
                    body: '{}',
                    headers: { 'Content-Type': 'application/json', Authorization: 'plaintext' },
                    secret_headers_input: 'secret_headers',
                })
                seedSecretHeadersInput(invocation)

                let result = await executor.executeFetch(invocation)
                expect(result.invocation.state.attempts).toBe(1)
                expect(JSON.stringify(result.invocation.queueParameters)).not.toContain('sk_test_token')

                result = await executor.executeFetch(result.invocation)

                expect(result.error).toBeUndefined()
                expect(receivedHeaders).toHaveLength(2)
                for (const headers of receivedHeaders) {
                    expect(headers['authorization']).toBe('Bearer sk_test_token')
                    expect(headers['x-api-key']).toBe('key_test_token')
                    expect(headers['content-type']).toBe('application/json')
                }
            })

            it('keeps the resolved headers inside an AWS SigV4 signed request', async () => {
                let received: Record<string, string | string[] | undefined> = {}
                mockRequest.mockImplementation((req: any, res: any) => {
                    received = req.headers
                    res.writeHead(200, { 'Content-Type': 'text/plain' })
                    res.end('ok')
                })

                const invocation = await createFetchInvocation({
                    url: `${baseUrl}/`,
                    method: 'POST',
                    body: '{}',
                    headers: { 'Content-Type': 'application/x-amz-json-1.1' },
                    aws_sigv4: {
                        service: 'kinesis',
                        region: 'us-east-1',
                        access_key_id_input: 'aws_access_key_id',
                        secret_access_key_input: 'aws_secret_access_key',
                    },
                    secret_headers_input: 'secret_headers',
                })
                seedSecretHeadersInput(invocation)
                invocation.hogFunction.encrypted_inputs = {
                    ...invocation.hogFunction.encrypted_inputs,
                    aws_access_key_id: { value: 'AKIDEXAMPLE' },
                    aws_secret_access_key: { value: 'wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY' },
                } as any

                const result = await executor.executeFetch(invocation)

                expect(result.error).toBeUndefined()
                expect(received['x-api-key']).toBe('key_test_token')
                expect(received['authorization']).toMatch(/^AWS4-HMAC-SHA256 .*SignedHeaders=[a-z0-9;-]*x-api-key/)
            })

            it.each([
                ['the secret headers input is missing', undefined],
                ['a secret header value is not text', { value: { Authorization: 12345 } }],
            ])('errors loudly instead of sending the request when %s', async (_name, secretHeaders) => {
                const invocation = await createFetchInvocation({
                    url: `${baseUrl}/`,
                    method: 'POST',
                    body: '{}',
                    secret_headers_input: 'secret_headers',
                })
                if (secretHeaders) {
                    invocation.hogFunction.encrypted_inputs = { secret_headers: secretHeaders } as any
                }

                const result = await executor.executeFetch(invocation)

                expect(mockRequest).not.toHaveBeenCalled()
                expect(result.error).toBeInstanceOf(Error)
                expect(result.error.message).toContain('secret_headers')
            })
        })

        it('respects maxFetchRetries option to disable retries', async () => {
            mockRequest.mockImplementation((req: any, res: any) => {
                res.writeHead(500, { 'Content-Type': 'text/plain' })
                res.end('server error')
            })

            const invocation = await createFetchInvocation({
                url: `${baseUrl}/test`,
                method: 'GET',
            })

            const result = await executor.executeWithAsyncFunctions(invocation, { maxFetchRetries: 0 })

            expect(result.finished).toBe(true)
            expect(result.error).toBeInstanceOf(Error)
            expect(result.error!.message).toContain('HTTP fetch failed on attempt 1')
            expect(result.invocation.queueScheduledAt).toBeUndefined()
        })

        it('handles request errors', async () => {
            const invocation = await createFetchInvocation({
                url: 'http://non-existent-host-name',
                method: 'GET',
            })

            const result = await executor.executeFetch(invocation)

            // Should be scheduled for retry
            expect(result.invocation.queue).toBe('hog')
            expect(result.invocation.queueScheduledAt).toMatchInlineSnapshot(`"2025-01-01T00:00:01.500Z"`)
            expect(result.logs.map((log) => log.message)).toMatchInlineSnapshot(`
                [
                  "HTTP fetch failed on attempt 1 with status code (none). Error: Invalid hostname. Retrying.",
                ]
            `)
        })

        it('handles security errors', async () => {
            jest.mocked(fetch).mockRejectedValueOnce(new SecureRequestError('Hostname is not allowed'))

            const invocation = await createFetchInvocation({
                url: 'http://localhost',
                method: 'GET',
            })

            const result = await executor.executeFetch(invocation)

            // Should not be scheduled for retry
            expect(result.invocation.queue).toBe('hog')
            expect(result.invocation.queueScheduledAt).toBeUndefined()
            expect(result.logs.map((log) => log.message)).toMatchInlineSnapshot(`
                [
                  "HTTP fetch failed on attempt 1 with status code (none). Error: Hostname is not allowed.",
                ]
            `)
        })

        it('handles timeouts', async () => {
            jest.mocked(fetch).mockRejectedValueOnce(new Error('The operation was aborted due to timeout'))

            const invocation = await createFetchInvocation({
                url: `${baseUrl}/test`,
                method: 'GET',
            })

            const result = await executor.executeFetch(invocation)

            expect(result.invocation.queue).toBe('hog')
            expect(result.invocation.queueScheduledAt).toMatchInlineSnapshot(`"2025-01-01T00:00:01.500Z"`)
            expect(result.logs.map((log) => log.message)).toMatchInlineSnapshot(`
                [
                  "HTTP fetch failed on attempt 1 with status code (none). Error: The operation was aborted due to timeout. Retrying.",
                ]
            `)
        })

        it('handles ResponseContentLengthMismatchError', async () => {
            jest.mocked(fetch).mockImplementationOnce(() => {
                const error = new Error('Response body length does not match content-length header')
                error.name = 'ResponseContentLengthMismatchError'
                return Promise.reject(error)
            })

            const invocation = await createFetchInvocation({
                url: `${baseUrl}/test`,
                method: 'GET',
            })

            const result = await executor.executeFetch(invocation)

            expect(result.invocation.queue).toBe('hog')
            expect(result.invocation.queueScheduledAt).toBeUndefined() // Should not retry
            expect(result.logs.map((log) => log.message)).toMatchInlineSnapshot(`
                [
                  "HTTP fetch failed on attempt 1 with status code (none). Error: Response body length does not match content-length header.",
                ]
            `)
        })

        it('completes fetch with headers', async () => {
            mockRequest.mockImplementation((req: any, res: any) => {
                if (req.headers['x-test'] === 'test') {
                    res.writeHead(200)
                } else {
                    res.writeHead(400)
                }
                res.end('Hello, world!')
            })

            const invocation = await createFetchInvocation({
                url: `${baseUrl}/test`,
                method: 'GET',
                headers: {
                    'X-Test': 'test',
                },
            })

            const result = await executor.executeFetch(invocation)
            const response = result.invocation.state.vmState!.stack.slice(-1)[0]

            expect(result.invocation.queue).toBe('hog')
            expect(response).toMatchInlineSnapshot(`
                {
                  "body": "Hello, world!",
                  "status": 200,
                }
            `)
        })

        it('completes fetch with body', async () => {
            mockRequest.mockImplementation((req: any, res: any) => {
                let body = ''
                req.on('data', (chunk: any) => {
                    body += chunk
                })
                req.on('end', () => {
                    expect(body).toBe('test body')
                    res.writeHead(200)
                    res.end('Hello, world!')
                })
            })

            const invocation = await createFetchInvocation({
                url: `${baseUrl}/test`,
                method: 'POST',
                body: 'test body',
            })

            const result = await executor.executeFetch(invocation)
            const response = result.invocation.state.vmState!.stack.slice(-1)[0]

            expect(result.invocation.queue).toBe('hog')
            expect(response).toMatchInlineSnapshot(`
                {
                  "body": "Hello, world!",
                  "status": 200,
                }
            `)
        })

        it('handles minimum parameters', async () => {
            mockRequest.mockImplementation((req: any, res: any) => {
                expect(req.method).toBe('GET')
                res.writeHead(200)
                res.end('Hello, world!')
            })

            const invocation = await createFetchInvocation({
                url: `${baseUrl}/test`,
                method: 'GET',
            })

            const result = await executor.executeFetch(invocation)
            const response = result.invocation.state.vmState!.stack.slice(-1)[0]

            expect(result.invocation.queue).toBe('hog')
            expect(response).toMatchInlineSnapshot(`
                {
                  "body": "Hello, world!",
                  "status": 200,
                }
            `)
        })

        it('adds secret headers for certain endpoints', async () => {
            jest.mocked(fetch).mockImplementation(() => {
                return Promise.resolve({
                    status: 200,
                    body: 'Hello, world!',
                    headers: {},
                    json: () => Promise.resolve({}),
                    text: () => Promise.resolve(''),
                    dump: () => Promise.resolve(),
                })
            })

            executor['config'].googleAdwordsDeveloperToken = 'ADWORDS_TOKEN'

            let invocation = await createFetchInvocation({
                url: 'https://googleads.googleapis.com/1234',
                method: 'POST',
                headers: {
                    'X-Test': 'test',
                },
            })

            await executor.executeFetch(invocation)
            expect((jest.mocked(fetch).mock.calls[0][1] as any).headers).toMatchInlineSnapshot(`
                {
                  "X-Test": "test",
                  "developer-token": "ADWORDS_TOKEN",
                }
            `)

            // Check it doesn't do it for redirect
            invocation = await createFetchInvocation({
                url: 'https://nasty.com?redirect=https://googleads.googleapis.com/1234',
                method: 'POST',
                headers: {
                    'X-Test': 'test',
                },
            })
            await executor.executeFetch(invocation)
            expect((jest.mocked(fetch).mock.calls[1][1] as any).headers).toMatchInlineSnapshot(`
                {
                  "X-Test": "test",
                }
            `)
        })

        it.each([
            ['the keys that built the inputs', (keys: string): string => keys],
            ['a key rotation that promoted a new key', (keys: string): string => `${'n'.repeat(32)},${keys}`],
        ])('puts each integration token only where the template placed it, under %s', async (_name, rotate) => {
            jest.mocked(fetch).mockImplementation(() => {
                return Promise.resolve({
                    status: 200,
                    body: 'Hello, world!',
                    headers: {},
                    json: () => Promise.resolve({}),
                    text: () => Promise.resolve(''),
                    dump: () => Promise.resolve(),
                })
            })

            const team = (await createTestTeamFixture(hub.postgres)).team
            const insertTokenIntegration = (token: string) =>
                insertIntegration(hub.postgres, team.id, {
                    kind: 'oauth',
                    sensitive_config: { access_token: hub.encryptedFields.encrypt(token) },
                })
            const crm = await insertTokenIntegration('crm_secret_token')
            const chat = await insertTokenIntegration('chat_secret_token')

            const invocation = createExampleInvocation({ ...hogFunction, team_id: team.id })
            invocation.hogFunction.inputs = { crm: { value: crm.id }, chat: { value: chat.id } }
            invocation.hogFunction.inputs_schema = [
                { key: 'crm', type: 'integration', required: true },
                { key: 'chat', type: 'integration', required: true },
            ]
            const { inputs } = await executor.hogExecutor.buildInputsWithGlobals(
                invocation.hogFunction,
                invocation.state.globals
            )
            const lookAlike = ACCESS_TOKEN_PLACEHOLDER + crm.id

            invocation.state.vmState = { stack: [] } as any
            invocation.queueParameters = {
                type: 'fetch',
                url: `https://example.com/test?q=${inputs.chat.access_token}`,
                method: 'POST',
                headers: { Authorization: `Bearer ${inputs.crm.access_token}`, 'X-Chat': inputs.chat.access_token },
                body: JSON.stringify({ token: inputs.crm.access_token, title: lookAlike }),
            } as any

            await createExecutor(new EncryptedFields(rotate(hub.ENCRYPTION_SALT_KEYS))).executeFetch(invocation)

            const [url, options] = jest.mocked(fetch).mock.calls[0] as any
            expect([url, options.headers, parseJSON(options.body)]).toEqual([
                'https://example.com/test?q=chat_secret_token',
                { Authorization: 'Bearer crm_secret_token', 'X-Chat': 'chat_secret_token' },
                { token: 'crm_secret_token', title: lookAlike },
            ])
        })

        it('does not substitute the access token of a warehouse source row', async () => {
            jest.mocked(fetch).mockImplementation(() =>
                Promise.resolve({
                    status: 200,
                    body: 'Hello, world!',
                    headers: {},
                    json: () => Promise.resolve({}),
                    text: () => Promise.resolve(''),
                    dump: () => Promise.resolve(),
                })
            )

            const team = (await createTestTeamFixture(hub.postgres)).team
            const warehouse = await insertIntegration(hub.postgres, team.id, {
                kind: 'postgresql' as any,
                config: { warehouse_source_type: 'Postgres' },
                sensitive_config: { access_token: hub.encryptedFields.encrypt('warehouse_secret_token') },
            })
            const placeholder = (
                new HogInputsService(
                    hub.integrationManager,
                    new RecipientTokensService(hub.ENCRYPTION_SALT_KEYS, hub.SITE_URL),
                    hub.encryptedFields
                ) as any
            ).accessTokenPlaceholder(warehouse.id)

            const invocation = createExampleInvocation({ ...hogFunction, team_id: team.id })
            invocation.hogFunction.inputs = { connection: { value: warehouse.id } }
            invocation.hogFunction.inputs_schema = [{ key: 'connection', type: 'integration', required: true }]
            invocation.state.vmState = { stack: [] } as any
            invocation.queueParameters = {
                type: 'fetch',
                url: 'https://example.com/test',
                method: 'POST',
                headers: { Authorization: `Bearer ${placeholder}` },
                body: JSON.stringify({ token: placeholder }),
            } as any

            await createExecutor(hub.encryptedFields).executeFetch(invocation)

            const [, options] = jest.mocked(fetch).mock.calls[0] as any
            expect(options.headers).toEqual({ Authorization: `Bearer ${placeholder}` })
            expect(options.body).not.toContain('warehouse_secret_token')
        })

        describe('with non_failure_status_codes', () => {
            beforeEach(() => {
                const actualRequest = jest.requireActual('~/common/utils/request') as { fetch: typeof fetch }
                jest.mocked(fetch).mockImplementation((url, options) => actualRequest.fetch(url, options))
            })

            const setNonFailureConfig = (
                invocation: CyclotronJobInvocationHogFunction,
                value: Array<number | string>
            ): void => {
                invocation.hogFunction.inputs_schema = [
                    ...(invocation.hogFunction.inputs_schema ?? []),
                    {
                        key: 'non_failure_status_codes',
                        type: 'non_failure_status_codes',
                        label: 'Non-failure response codes',
                        required: false,
                    },
                ]
                invocation.hogFunction.inputs = {
                    ...(invocation.hogFunction.inputs ?? {}),
                    non_failure_status_codes: { value },
                }
            }

            it('treats matched non-retriable 4xx as success (exact match)', async () => {
                mockRequest.mockImplementation((req: any, res: any) => {
                    res.writeHead(400, { 'Content-Type': 'text/plain' })
                    res.end('backdated consent')
                })

                const invocation = await createFetchInvocation({
                    url: `${baseUrl}/test`,
                    method: 'GET',
                })
                setNonFailureConfig(invocation, [400])

                const result = await executor.executeFetch(invocation)

                expect(result.error).toBeUndefined()
                expect(result.invocation.queueScheduledAt).toBeUndefined()
                expect(result.invocation.state.vmState!.stack.slice(-1)[0]).toEqual({
                    status: 400,
                    body: 'backdated consent',
                })
                expect(result.logs.map((l) => ({ level: l.level, message: l.message }))).toEqual([
                    {
                        level: 'info',
                        message: expect.stringContaining('status code 400'),
                    },
                ])
            })

            it('treats matched 4xx as success via wildcard', async () => {
                mockRequest.mockImplementation((req: any, res: any) => {
                    res.writeHead(404, { 'Content-Type': 'text/plain' })
                    res.end('not found')
                })

                const invocation = await createFetchInvocation({
                    url: `${baseUrl}/test`,
                    method: 'GET',
                })
                setNonFailureConfig(invocation, ['4xx'])

                const result = await executor.executeFetch(invocation)

                expect(result.error).toBeUndefined()
                expect(result.invocation.state.vmState!.stack.slice(-1)[0]).toEqual({
                    status: 404,
                    body: 'not found',
                })
            })

            it('does not match when ignore list does not cover the status', async () => {
                mockRequest.mockImplementation((req: any, res: any) => {
                    res.writeHead(401, { 'Content-Type': 'text/plain' })
                    res.end('unauthorized')
                })

                const invocation = await createFetchInvocation({
                    url: `${baseUrl}/test`,
                    method: 'GET',
                })
                setNonFailureConfig(invocation, [400, 403])

                const result = await executor.executeFetch(invocation)

                expect(result.error).toBeInstanceOf(Error)
                expect(result.error.message).toContain('status code 401')
                expect(result.logs[0].level).toBe('error')
            })

            it('still retries a retriable status that is in the ignore list, then succeeds without setting result.error', async () => {
                mockRequest.mockImplementation((req: any, res: any) => {
                    res.writeHead(500, { 'Content-Type': 'text/plain' })
                    res.end('server error')
                })

                const invocation = await createFetchInvocation({
                    url: `${baseUrl}/test`,
                    method: 'GET',
                })
                setNonFailureConfig(invocation, [500])

                const maxRetries = executor['config'].fetchRetries
                let result = await executor.executeFetch(invocation)
                // Verify every intermediate attempt also logged at 'info' — regression guard
                // against any future change that re-raises retry logs to 'error' when the
                // status is in the non-failure list.
                expect(result.logs.every((l) => l.level === 'info')).toBe(true)

                for (let attempt = 1; attempt < maxRetries; attempt++) {
                    expect(result.error).toBeUndefined()
                    expect(result.invocation.queueScheduledAt).not.toBeUndefined()
                    expect(result.invocation.state.attempts).toBe(attempt)
                    result = await executor.executeFetch(result.invocation)
                    expect(result.logs.every((l) => l.level === 'info')).toBe(true)
                }

                expect(result.error).toBeUndefined()
                expect(result.invocation.queueScheduledAt).toBeUndefined()
                expect(result.invocation.state.vmState!.stack.slice(-1)[0]).toEqual({
                    status: 500,
                    body: 'server error',
                })
                expect(result.logs.every((l) => l.level === 'info')).toBe(true)
            })

            it('mixed wildcard and number ignores both 4xx and the specific 5xx', async () => {
                mockRequest.mockImplementation((req: any, res: any) => {
                    res.writeHead(404, { 'Content-Type': 'text/plain' })
                    res.end('a')
                })
                let invocation = await createFetchInvocation({ url: `${baseUrl}/test`, method: 'GET' })
                setNonFailureConfig(invocation, ['4xx', 500])

                let result = await executor.executeFetch(invocation)
                expect(result.error).toBeUndefined()
                expect(result.invocation.state.vmState!.stack.slice(-1)[0]).toEqual({ status: 404, body: 'a' })

                mockRequest.mockImplementation((req: any, res: any) => {
                    res.writeHead(500, { 'Content-Type': 'text/plain' })
                    res.end('b')
                })
                invocation = await createFetchInvocation({ url: `${baseUrl}/test`, method: 'GET' })
                setNonFailureConfig(invocation, ['4xx', 500])

                // 500 is retriable — drain retries until terminal
                const maxRetries = executor['config'].fetchRetries
                result = await executor.executeFetch(invocation)
                for (let attempt = 1; attempt < maxRetries; attempt++) {
                    result = await executor.executeFetch(result.invocation)
                }
                expect(result.error).toBeUndefined()
                expect(result.invocation.state.vmState!.stack.slice(-1)[0]).toEqual({ status: 500, body: 'b' })
            })

            it('does not ignore a 502 when config is [4xx, 500]', async () => {
                mockRequest.mockImplementation((req: any, res: any) => {
                    res.writeHead(502, { 'Content-Type': 'text/plain' })
                    res.end('bad gateway')
                })
                const invocation = await createFetchInvocation({ url: `${baseUrl}/test`, method: 'GET' })
                setNonFailureConfig(invocation, ['4xx', 500])

                const maxRetries = executor['config'].fetchRetries
                let result = await executor.executeFetch(invocation)
                for (let attempt = 1; attempt < maxRetries; attempt++) {
                    result = await executor.executeFetch(result.invocation)
                }
                expect(result.error).toBeInstanceOf(Error)
                expect(result.error.message).toContain('status code 502')
            })

            it('does not affect successful responses', async () => {
                mockRequest.mockImplementation((req: any, res: any) => {
                    res.writeHead(200, { 'Content-Type': 'text/plain' })
                    res.end('ok')
                })
                const invocation = await createFetchInvocation({ url: `${baseUrl}/test`, method: 'GET' })
                setNonFailureConfig(invocation, ['4xx', 500])

                const result = await executor.executeFetch(invocation)
                expect(result.error).toBeUndefined()
                expect(result.invocation.state.vmState!.stack.slice(-1)[0]).toEqual({ status: 200, body: 'ok' })
            })

            it('is a no-op when ignore config is empty', async () => {
                mockRequest.mockImplementation((req: any, res: any) => {
                    res.writeHead(400, { 'Content-Type': 'text/plain' })
                    res.end('bad')
                })
                const invocation = await createFetchInvocation({ url: `${baseUrl}/test`, method: 'GET' })
                setNonFailureConfig(invocation, [])

                const result = await executor.executeFetch(invocation)
                expect(result.error).toBeInstanceOf(Error)
                expect(result.error.message).toContain('status code 400')
            })
        })

        describe('self-loop guard', () => {
            const OWN_TOKEN = 'phc_synthetic_own_0000000000000000'
            const INGEST_URL = 'https://us.i.posthog.com/capture/'

            const mockOwnTeam = (): void => {
                jest.spyOn(hub.teamManager, 'getTeam').mockResolvedValue({
                    id: 1,
                    api_token: OWN_TOKEN,
                    secret_api_token: null,
                } as any)
            }

            const ownTokenCaptureBody = (): string =>
                JSON.stringify({ api_key: OWN_TOKEN, event: 'replicated', distinct_id: 'u1', properties: {} })

            // Seed this destination's own self-loop depth (keyed by its function id).
            const setSelfLoopDepth = (invocation: CyclotronJobInvocationHogFunction, depth: number): void => {
                invocation.state.globals.event.properties = {
                    ...invocation.state.globals.event.properties,
                    [SELF_LOOP_DEPTH_PROPERTY]: { [invocation.hogFunction.id]: depth },
                }
            }

            // Seed a high depth for a DIFFERENT function - simulates an event that passed
            // through an unrelated deep chain. Must not count toward this destination.
            const setOtherFunctionDepth = (invocation: CyclotronJobInvocationHogFunction, depth: number): void => {
                invocation.state.globals.event.properties = {
                    ...invocation.state.globals.event.properties,
                    [SELF_LOOP_DEPTH_PROPERTY]: { 'some-other-function-id': depth },
                }
            }

            // Capture the body sent to the (mocked) ingest endpoint without a real network call.
            const captureIngestFetch = (): { getBody: () => string | undefined } => {
                let sentBody: string | undefined
                ;(fetch as jest.Mock).mockImplementationOnce((_url: string, options: any) => {
                    sentBody = options.body
                    return Promise.resolve({ status: 200, headers: {}, text: () => Promise.resolve('ok') })
                })
                return { getBody: () => sentBody }
            }

            const readActionCount = async (mode: string, action: string): Promise<number> => {
                const metric = await selfLoopGuardCounter.get()
                return metric.values.find((v) => v.labels.mode === mode && v.labels.action === action)?.value ?? 0
            }

            it('fails open: a team lookup error never breaks the fetch', async () => {
                jest.spyOn(hub.teamManager, 'getTeam').mockRejectedValue(new Error('db unavailable'))
                const invocation = await createFetchInvocation({
                    url: INGEST_URL,
                    method: 'POST',
                    body: ownTokenCaptureBody(),
                })
                ;(fetch as jest.Mock).mockImplementationOnce(() =>
                    Promise.resolve({ status: 200, headers: {}, text: () => Promise.resolve('ok') })
                )

                const result = await executor.executeFetch(invocation)

                // Detection failing must not surface as a destination error.
                expect(result.error).toBeUndefined()
            })

            // Every hop under the cap is allowed and its outgoing body stamped with this
            // destination's next depth - including hop 0 (a fresh external event), which a
            // legitimate run always is.
            it.each([
                { case: 'fresh hop 0 (a legitimate external run)', depth: 0, stampedTo: 1 },
                { case: 'mid-chain under the cap', depth: 2, stampedTo: 3 },
                { case: 'the last hop under the cap', depth: 9, stampedTo: 10 },
            ])('enforce: allows + stamps the next hop ($case)', async ({ depth, stampedTo }) => {
                mockOwnTeam()
                const invocation = await createFetchInvocation({
                    url: INGEST_URL,
                    method: 'POST',
                    body: ownTokenCaptureBody(),
                })
                setSelfLoopDepth(invocation, depth)
                const sent = captureIngestFetch()
                const blockedBefore = await readActionCount('enforce', 'blocked')

                const result = await executor.executeFetch(invocation)

                // Fetch proceeds, body carries this destination's incremented depth, nothing blocked.
                expect(result.error).toBeUndefined()
                expect(parseJSON(sent.getBody()!).properties[SELF_LOOP_DEPTH_PROPERTY][invocation.hogFunction.id]).toBe(
                    stampedTo
                )
                expect(await readActionCount('enforce', 'blocked')).toBe(blockedBefore)
            })

            // The whole point of per-function depth: an event that arrived carrying a huge
            // depth for a DIFFERENT function is treated as depth 0 here, so a legitimately
            // running destination is never blocked by an unrelated deep chain.
            it('enforce: does NOT block when the high depth belongs to another function', async () => {
                mockOwnTeam()
                const invocation = await createFetchInvocation({
                    url: INGEST_URL,
                    method: 'POST',
                    body: ownTokenCaptureBody(),
                })
                setOtherFunctionDepth(invocation, 50)
                const sent = captureIngestFetch()
                const blockedBefore = await readActionCount('enforce', 'blocked')

                const result = await executor.executeFetch(invocation)

                // Allowed, stamped as this destination's first hop, not blocked.
                expect(result.error).toBeUndefined()
                expect(parseJSON(sent.getBody()!).properties[SELF_LOOP_DEPTH_PROPERTY][invocation.hogFunction.id]).toBe(
                    1
                )
                expect(await readActionCount('enforce', 'blocked')).toBe(blockedBefore)
            })

            it('enforce: breaks the chain once it reaches the cap', async () => {
                mockOwnTeam()
                const invocation = await createFetchInvocation({
                    url: INGEST_URL,
                    method: 'POST',
                    body: ownTokenCaptureBody(),
                })
                setSelfLoopDepth(invocation, 10)
                mockRequest.mockClear()
                const blockedBefore = await readActionCount('enforce', 'blocked')

                const result = await executor.executeFetch(invocation)

                // Blocked: error set, finished, no fetch attempted, metric moved.
                expect(result.error).toBeInstanceOf(Error)
                expect(result.finished).toBe(true)
                expect(mockRequest).not.toHaveBeenCalled()
                expect(await readActionCount('enforce', 'blocked')).toBe(blockedBefore + 1)
                expect(cleanLogs(result.logs.map((l) => l.message))).toEqual(
                    expect.arrayContaining([expect.stringContaining('event-forwarding loop has already repeated')])
                )
            })

            it('enforce: leaves a normal external fetch untouched', async () => {
                mockOwnTeam()
                const invocation = await createFetchInvocation({
                    url: `${baseUrl}/test`,
                    method: 'POST',
                    body: ownTokenCaptureBody(),
                })
                mockRequest.mockClear()

                const result = await executor.executeFetch(invocation)

                expect(result.error).toBeUndefined()
                expect(mockRequest).toHaveBeenCalled()
            })
        })
    })

    describe('isConnectionLevelError', () => {
        it.each([
            [{ code: 'UND_ERR_SOCKET', message: 'other side closed' }, true],
            [{ code: 'ECONNRESET', message: 'read ECONNRESET' }, true],
            [{ code: 'EPIPE', message: 'write EPIPE' }, true],
            [{ code: undefined, message: 'other side closed' }, true],
            [{ code: undefined, message: 'socket hang up' }, true],
            [{ code: 'ENOTFOUND', message: 'getaddrinfo ENOTFOUND' }, false],
            [{ code: undefined, message: 'some other error' }, false],
            [null, false],
            [undefined, false],
        ])('returns %s for %j', (error, expected) => {
            expect(isConnectionLevelError(error)).toBe(expected)
        })
    })

    describe('routeEmailToQueue', () => {
        it('should route the invocation to the email queue', () => {
            const hogFunction = createHogFunction({
                name: 'Email function',
                metadata: { message_category_type: 'marketing' },
            })

            const invocation: CyclotronJobInvocationHogFunction = {
                ...createExampleInvocation(hogFunction),
                queue: 'hogflow',
                queueParameters: {
                    type: 'email',
                    to: { email: '[email protected]' },
                    from: { integrationId: 1 },
                    subject: 'Test',
                    text: 'Hello',
                    html: '<p>Hello</p>',
                },
            }
            invocation.state.vmState = { stack: [] } as any

            const result = (executor as any).routeEmailToQueue(invocation, invocation.queuePriority)

            expect(result.finished).toBe(false)
            expect(result.invocation.queue).toBe('email')
            expect(result.invocation.queueMetadata?.originQueue).toBe('hogflow')
            expect(result.invocation.queuePriority).toBe(1)
            expect(result.invocation.queueMetadata?.originPriority).toBe(invocation.queuePriority)
            expect(result.metrics).toContainEqual(
                expect.objectContaining({
                    metric_name: 'email_queued',
                    metric_kind: 'email',
                })
            )
        })

        it('should classify transactional sends into the fast priority class', () => {
            const hogFunction = createHogFunction({
                name: 'Email function',
                metadata: { message_category_type: 'transactional' },
            })

            const invocation: CyclotronJobInvocationHogFunction = {
                ...createExampleInvocation(hogFunction),
                queue: 'hogflow',
                queueParameters: {
                    type: 'email',
                    to: { email: '[email protected]' },
                    from: { integrationId: 1 },
                    subject: 'Test',
                    text: 'Hello',
                    html: '<p>Hello</p>',
                },
            }
            invocation.state.vmState = { stack: [] } as any

            const result = (executor as any).routeEmailToQueue(invocation, invocation.queuePriority)

            expect(result.invocation.queuePriority).toBe(0)
        })

        it('should restore the origin priority when routing back from the email queue', () => {
            const hogFunction = createHogFunction({ name: 'Email function' })
            const invocation: CyclotronJobInvocationHogFunction = {
                ...createExampleInvocation(hogFunction),
                queue: 'email',
                queuePriority: 1,
                // originPriority 0 also guards the restore against a `||`-style
                // fallback that would treat a falsy origin priority as absent.
                queueMetadata: { originQueue: 'hogflow', originPriority: 0 },
            }

            const result = (executor as any).routeToQueue(invocation, 'hogflow')

            expect(result.invocation.queue).toBe('hogflow')
            expect(result.invocation.queuePriority).toBe(0)
            expect(result.invocation.queueMetadata).toBeUndefined()
        })

        it('should preserve the same job ID (no new job created)', () => {
            const hogFunction = createHogFunction({ name: 'Email function' })
            const invocation: CyclotronJobInvocationHogFunction = {
                ...createExampleInvocation(hogFunction),
                queueParameters: {
                    type: 'email',
                    to: { email: '[email protected]' },
                    from: { integrationId: 1 },
                    subject: 'Test',
                    text: 'Hello',
                    html: '<p>Hello</p>',
                },
            }
            invocation.state.vmState = { stack: [] } as any

            const result = (executor as any).routeEmailToQueue(invocation, invocation.queuePriority)

            expect(result.invocation.id).toBe(invocation.id)
        })
    })

    describe('email queue routing', () => {
        const createEmailInvocation = (): CyclotronJobInvocationHogFunction => {
            const hogFunction = createHogFunction({ name: 'Email function', team_id: 123 })
            const invocation: CyclotronJobInvocationHogFunction = {
                ...createExampleInvocation(hogFunction),
                teamId: 123,
                queueParameters: {
                    type: 'email',
                    to: { email: '[email protected]' },
                    from: { integrationId: 1 },
                    subject: 'Test',
                    text: 'Hello',
                    html: '<p>Hello</p>',
                },
            }
            invocation.state.vmState = { stack: [] } as any
            return invocation
        }

        it('should route email sends to the dedicated email queue', async () => {
            const invocation = createEmailInvocation()

            const result = await executor.executeWithAsyncFunctions(invocation)

            expect(result.invocation.queue).toBe('email')
            expect(result.invocation.queueMetadata?.originQueue).toBeDefined()
            expect(result.finished).toBe(false)
        })

        it('should send inline when isTest is set', async () => {
            const invocation = createEmailInvocation()

            const result = await executor.executeWithAsyncFunctions(invocation, { isTest: true })

            expect(result.invocation.queue).not.toBe('email')
            expect(result.finished).toBe(true)
        })

        it('should stash the origin queue priority when the send happens mid-run', async () => {
            // A send from the hogflow queue runs the hog program first, which clones the
            // invocation and resets queuePriority to 0 before the email is detected and routed.
            // The stashed origin priority must come from the entry invocation, otherwise the job
            // returns to the hogflow queue at priority 0 and jumps ahead of every other run.
            const hogFunction = createHogFunction({ name: 'Email function' })
            const invocation: CyclotronJobInvocationHogFunction = {
                ...createExampleInvocation(hogFunction, { inputs: {} }, 'hogflow'),
                queuePriority: 2,
            }
            invocation.state.vmState = { stack: [] } as any

            const hogExecModule = require('../utils/hog-exec')
            jest.spyOn(hogExecModule, 'execHog').mockResolvedValue({
                execResult: {
                    finished: false,
                    asyncFunctionName: 'sendEmail',
                    asyncFunctionArgs: [
                        {
                            to: { email: '[email protected]' },
                            from: { integrationId: 1 },
                            subject: 'Test',
                            text: 'Hello',
                            html: '<p>Hello</p>',
                        },
                    ],
                    state: { syncDuration: 1, maxMemUsed: 100, ops: 10, stack: [] },
                },
                error: undefined,
                durationMs: 1,
            })

            const result = await executor.executeWithAsyncFunctions(invocation)

            expect(result.invocation.queue).toBe('email')
            expect(result.invocation.queueMetadata?.originPriority).toBe(2)
        })
    })
})