:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { DateTime } from 'luxon'
import { IngestionOutputs } from '~/common/outputs/ingestion-outputs'
import { parseJSON } from '~/common/utils/json-parse'
import { FixtureHogFlowBuilder } from '../../_tests/builders/hogflow.builder'
import { createExampleInvocation } from '../../_tests/fixtures'
import { createExampleHogFlowInvocation } from '../../_tests/fixtures-hogflows'
import { CdpOutput } from '../../cdp-services'
import {
HogInvocationResultRow,
HogInvocationResultsService,
decodeInvocationGlobals,
} from './hog-invocation-results.service'
const buildOutputsMock = (): jest.Mocked<IngestionOutputs<CdpOutput>> => {
return {
produce: jest.fn().mockResolvedValue(undefined),
} as unknown as jest.Mocked<IngestionOutputs<CdpOutput>>
}
const parseProducedRows = (outputs: jest.Mocked<IngestionOutputs<CdpOutput>>): HogInvocationResultRow[] => {
return outputs.produce.mock.calls.map((call) => {
const arg = call[1] as { key: Buffer | null; value: Buffer | null }
return parseJSON(arg.value!.toString('utf8')) as HogInvocationResultRow
})
}
describe('HogInvocationResultsService', () => {
let outputs: jest.Mocked<IngestionOutputs<CdpOutput>>
let service: HogInvocationResultsService
beforeEach(() => {
outputs = buildOutputsMock()
service = new HogInvocationResultsService(outputs, { HOG_INVOCATION_RESULTS_ENABLED: true })
})
describe('feature flag', () => {
it('produces nothing when HOG_INVOCATION_RESULTS_ENABLED is false', async () => {
service = new HogInvocationResultsService(outputs, { HOG_INVOCATION_RESULTS_ENABLED: false })
const invocation = createExampleInvocation()
service.queueLifecycleRow(invocation, 'running')
service.queueInvocationResults([
{
invocation,
finished: true,
error: undefined,
logs: [],
metrics: [],
capturedPostHogEvents: [],
warehouseWebhookPayloads: [],
messageAssets: [],
} as any,
])
await service.flush()
expect(outputs.produce).not.toHaveBeenCalled()
})
})
describe('recordTerminalFailureDurably', () => {
it('produces a failed row with an overridden error_kind and reports success', async () => {
const invocation = createExampleInvocation()
const ok = await service.recordTerminalFailureDurably(invocation, {
error: 'poison pill: stalled and reset 3 times without completing',
errorKind: 'janitor_poison_pill',
})
expect(ok).toBe(true)
const rows = parseProducedRows(outputs)
expect(rows).toHaveLength(1)
expect(rows[0].status).toBe('failed')
expect(rows[0].error_kind).toBe('janitor_poison_pill')
expect(rows[0].error_message).toContain('poison pill')
expect(rows[0].invocation_id).toBe(invocation.id)
})
it('returns false (does not throw) when the produce fails — caller must not delete', async () => {
outputs.produce.mockRejectedValueOnce(new Error('broker unavailable'))
const ok = await service.recordTerminalFailureDurably(createExampleInvocation(), {
error: 'boom',
errorKind: 'janitor_poison_pill',
})
expect(ok).toBe(false)
})
it('returns false when recording is disabled', async () => {
service = new HogInvocationResultsService(outputs, { HOG_INVOCATION_RESULTS_ENABLED: false })
const ok = await service.recordTerminalFailureDurably(createExampleInvocation(), { error: 'boom' })
expect(ok).toBe(false)
expect(outputs.produce).not.toHaveBeenCalled()
})
})
describe('queueLifecycleRow', () => {
it('writes a status=running row with start timing and no error fields', async () => {
const invocation = createExampleInvocation()
service.queueLifecycleRow(invocation, 'running')
await service.flush()
const rows = parseProducedRows(outputs)
expect(rows).toHaveLength(1)
const row = rows[0]
expect(row.status).toBe('running')
expect(row.function_kind).toBe('hog_function')
expect(row.team_id).toBe(invocation.teamId)
expect(row.function_id).toBe(invocation.functionId)
expect(row.invocation_id).toBe(invocation.id)
expect(row.is_retry).toBe(0)
expect(row.started_at).not.toBeNull()
expect(row.finished_at).toBeNull()
expect(row.error_kind).toBe('')
expect(row.error_message).toBe('')
})
it('strips inputs from invocation_globals — secrets must not be persisted', async () => {
const invocation = createExampleInvocation(
{ id: 'fn-1', team_id: 1 },
{
inputs: {
api_key: 'sk-super-secret',
nested: { token: 'abc123' },
} as any,
}
)
service.queueLifecycleRow(invocation, 'running')
await service.flush()
const rows = parseProducedRows(outputs)
expect(rows).toHaveLength(1)
const globals = (await decodeInvocationGlobals(rows[0].invocation_globals)) as Record<string, any>
// The event context we care about is preserved.
expect(globals.event?.uuid).toBeDefined()
expect(globals.project?.id).toBeDefined()
// But `inputs` is gone entirely.
expect(globals).not.toHaveProperty('inputs')
// And the decoded payload does not mention the secret anywhere
// (catches the case where a future schema change moves inputs
// under a different key without us noticing).
const decoded = JSON.stringify(globals)
expect(decoded).not.toContain('sk-super-secret')
expect(decoded).not.toContain('abc123')
})
const send = {
content: { subject: 'Your login code', html: '<p>Your code is 482913</p>' },
tracking: { captureEngagement: false },
}
const messageData = { variables: { code: '482913' }, send }
const noMessageData = { variables: undefined, send: undefined }
it.each([
['an ordinary email', false, {}, messageData],
['a sensitive email', true, {}, noMessageData],
['an ordinary push', false, {}, messageData, 'function_push'],
['a sensitive push', true, {}, noMessageData, 'function_push'],
['a recovery stub of a sensitive run', undefined, { sensitive: true }, noMessageData],
[
'a recovery stub of an unstamped transactional run',
undefined,
{ event: { event: '$transactional_email' } },
noMessageData,
],
['a recovery stub of an ordinary run', undefined, {}, messageData],
])(
'keeps the message data in a workflow snapshot only for %s',
async (_name, sensitive, state, expected, stepType = 'function_email') => {
const hogFlow =
sensitive === undefined
? ({ id: 'flow-1' } as any)
: new FixtureHogFlowBuilder()
.withWorkflow({
actions: {
message: {
type: stepType,
config: {
template_id:
stepType === 'function_push'
? 'template-native-push'
: 'template-email',
sensitive,
inputs: {},
},
} as any,
},
edges: [],
})
.build()
const invocation = createExampleHogFlowInvocation(hogFlow, { variables: { code: '482913' }, send })
// A step paused for a provider retry keeps the rendered message on its VM stack.
const vmState = { stack: [{ title: 'Your code is 482913' }] }
const person = { id: 'person-1', properties: { code: '482913' } }
Object.assign(invocation.state, state, {
event: { ...invocation.state.event, ...(state as any).event },
currentAction: {
id: 'message',
startedAtTimestamp: 0,
hogFunctionState: { vmState, globals: { person } },
},
})
await expect(service.recordTerminalFailureDurably(invocation)).resolves.toBe(true)
const globals = (await decodeInvocationGlobals(
parseProducedRows(outputs)[0].invocation_globals
)) as Record<string, any>
expect({ variables: globals.variables, send: globals.send }).toEqual(expected)
expect(globals.currentAction.hogFunctionState.vmState).toEqual(expected.variables && vmState)
expect(globals.currentAction.hogFunctionState.globals.person).toEqual(expected.variables && person)
}
)
it('strips groups and person from invocation_globals — rebuilt downstream on rerun', async () => {
const invocation = createExampleInvocation({}, {
groups: {
organization: {
id: 'org-1',
type: 'organization',
index: 0,
url: 'http://localhost',
properties: { plan: 'enterprise', seats: 500 },
},
},
} as any)
service.queueLifecycleRow(invocation, 'running')
await service.flush()
const rows = parseProducedRows(outputs)
const globals = (await decodeInvocationGlobals(rows[0].invocation_globals)) as Record<string, any>
expect(globals).not.toHaveProperty('groups')
expect(globals).not.toHaveProperty('person')
// The event — the rerun trigger — is kept.
expect(globals.event?.uuid).toBeDefined()
expect(globals.project?.id).toBeDefined()
})
it('marks is_retry=1 and attempts=N when state.rerunAttempts is set', async () => {
const invocation = createExampleInvocation()
invocation.state.rerunAttempts = 1
service.queueLifecycleRow(invocation, 'running')
await service.flush()
const rows = parseProducedRows(outputs)
expect(rows[0].is_retry).toBe(1)
expect(rows[0].attempts).toBe(1)
})
it('leaves is_retry=0 and attempts=0 when state.rerunAttempts is unset (original run)', async () => {
const invocation = createExampleInvocation()
// Fetch-retry counter is bumped by the executor — it must NOT bleed
// into the lifecycle row's `attempts`/`is_retry` fields.
invocation.state.attempts = 3
service.queueLifecycleRow(invocation, 'running')
await service.flush()
const rows = parseProducedRows(outputs)
expect(rows[0].is_retry).toBe(0)
expect(rows[0].attempts).toBe(0)
})
it('extracts event_uuid, distinct_id, and person_id into promoted columns', async () => {
const invocation = createExampleInvocation({}, {
event: {
uuid: 'event-uuid-123',
event: 'test',
elements_chain: '',
distinct_id: 'distinct-456',
url: 'http://localhost',
properties: {},
timestamp: '2026-05-01T00:00:00Z',
},
person: {
id: 'person-789',
name: 'x',
url: 'http://localhost',
properties: {},
},
} as any)
service.queueLifecycleRow(invocation, 'running')
await service.flush()
const rows = parseProducedRows(outputs)
expect(rows[0].event_uuid).toBe('event-uuid-123')
expect(rows[0].distinct_id).toBe('distinct-456')
expect(rows[0].person_id).toBe('person-789')
})
})
describe('queueInvocationResults', () => {
it('writes status=succeeded on a result with no error', async () => {
const invocation = createExampleInvocation()
service.queueInvocationResults([
{
invocation,
finished: true,
error: undefined,
logs: [],
metrics: [],
capturedPostHogEvents: [],
warehouseWebhookPayloads: [],
messageAssets: [],
} as any,
])
await service.flush()
const rows = parseProducedRows(outputs)
expect(rows).toHaveLength(1)
expect(rows[0].status).toBe('succeeded')
expect(rows[0].error_kind).toBe('')
})
it('writes status=failed and classifies the error kind', async () => {
const invocation = createExampleInvocation()
service.queueInvocationResults([
{
invocation,
finished: true,
error: new Error('Request timed out after 30s'),
logs: [],
metrics: [],
capturedPostHogEvents: [],
warehouseWebhookPayloads: [],
messageAssets: [],
} as any,
])
await service.flush()
const rows = parseProducedRows(outputs)
expect(rows).toHaveLength(1)
expect(rows[0].status).toBe('failed')
expect(rows[0].error_kind).toBe('timeout')
// Exact match: the message only, never the stack trace.
expect(rows[0].error_message).toBe('Request timed out after 30s')
})
it('writes status=canceled on a finished result flagged canceled, with no error fields', async () => {
const invocation = createExampleInvocation()
service.queueInvocationResults([
{
invocation,
finished: true,
canceled: true,
error: undefined,
logs: [],
metrics: [],
capturedPostHogEvents: [],
warehouseWebhookPayloads: [],
messageAssets: [],
} as any,
])
await service.flush()
const rows = parseProducedRows(outputs)
expect(rows).toHaveLength(1)
expect(rows[0].status).toBe('canceled')
expect(rows[0].error_kind).toBe('')
expect(rows[0].finished_at).not.toBeNull()
})
it('produces no row for an in-flight result that has not finished and has no error', async () => {
const invocation = createExampleInvocation()
service.queueInvocationResults([
{
invocation,
finished: false,
error: undefined,
logs: [],
metrics: [],
capturedPostHogEvents: [],
warehouseWebhookPayloads: [],
messageAssets: [],
} as any,
])
await service.flush()
expect(outputs.produce).not.toHaveBeenCalled()
})
})
describe('flush', () => {
it('uses the invocation_id as the Kafka partition key', async () => {
const invocation = createExampleInvocation({ id: 'fn-2' })
service.queueLifecycleRow(invocation, 'running')
await service.flush()
const arg = outputs.produce.mock.calls[0][1] as { key: Buffer | null; value: Buffer | null }
expect(arg.key?.toString('utf8')).toBe(invocation.id)
})
it('clears the queue after flush so a second flush is a no-op', async () => {
const invocation = createExampleInvocation()
service.queueLifecycleRow(invocation, 'running')
await service.flush()
await service.flush()
expect(outputs.produce).toHaveBeenCalledTimes(1)
})
})
describe('version monotonicity', () => {
it('produces strictly increasing version values across successive rows for the same invocation', async () => {
// Queued back-to-back with no delay: both rows land in the same wall-clock
// millisecond, so this only passes if version is clamped to strictly exceed the
// last value issued — the property that keeps ReplacingMergeTree from dropping the
// terminal row. No sleep, so it can't flake on wall-clock jitter.
const invocation = createExampleInvocation()
service.queueLifecycleRow(invocation, 'running')
service.queueLifecycleRow(invocation, 'succeeded')
await service.flush()
// flush() produces rows concurrently (Promise.all), so their produce order is not
// the queue order — match by status rather than array index.
const rows = parseProducedRows(outputs)
expect(rows).toHaveLength(2)
const running = rows.find((r) => r.status === 'running')!
const succeeded = rows.find((r) => r.status === 'succeeded')!
expect(BigInt(succeeded.version)).toBeGreaterThan(BigInt(running.version))
})
})
describe('invocation_globals compression', () => {
it('produces invocation_globals as a compressed blob that round-trips back to the event', async () => {
const invocation = createExampleInvocation({}, {
event: {
uuid: 'event-uuid-rt',
event: '$pageview',
elements_chain: '',
distinct_id: 'distinct-rt',
url: 'http://localhost',
properties: { $current_url: 'https://posthog.com', big: 'x'.repeat(5000) },
timestamp: '2026-05-01T00:00:00Z',
},
} as any)
service.queueLifecycleRow(invocation, 'running')
await service.flush()
const rows = parseProducedRows(outputs)
const stored = rows[0].invocation_globals
// On the wire the field is a base64 blob — never raw JSON.
expect(stored.startsWith('{')).toBe(false)
// A repetitive 5KB event compresses well below its raw size.
expect(stored.length).toBeLessThan(JSON.stringify(invocation.state.globals.event).length)
// And it decodes back to the original event payload intact.
const globals = (await decodeInvocationGlobals(stored)) as Record<string, any>
expect(globals.event).toEqual(invocation.state.globals.event)
})
it('decodes legacy uncompressed (raw JSON) rows unchanged', async () => {
const legacy = JSON.stringify({ event: { uuid: 'legacy-uuid' }, project: { id: 7 } })
expect(await decodeInvocationGlobals(legacy)).toEqual({
event: { uuid: 'legacy-uuid' },
project: { id: 7 },
})
})
})
describe('first_scheduled_at', () => {
it('stamps the original scheduled time onto invocation state on the first running row', async () => {
const invocation = createExampleInvocation()
invocation.queueScheduledAt = DateTime.utc(2026, 1, 1, 0, 0, 0)
service.queueLifecycleRow(invocation, 'running')
await service.flush()
const rows = parseProducedRows(outputs)
expect(rows[0].first_scheduled_at).toBe(rows[0].scheduled_at)
expect(invocation.state.firstScheduledAt).toBe(rows[0].scheduled_at)
})
it('keeps the original first_scheduled_at on the terminal row after a fetch retry reschedules', async () => {
const invocation = createExampleInvocation()
invocation.queueScheduledAt = DateTime.utc(2026, 1, 1, 0, 0, 0)
service.queueLifecycleRow(invocation, 'running')
const original = invocation.state.firstScheduledAt
// Simulate a cyclotron fetch retry: queueScheduledAt is overwritten
// with a later backoff time before the terminal row is produced.
invocation.queueScheduledAt = DateTime.utc(2026, 1, 1, 0, 5, 0)
service.queueLifecycleRow(invocation, 'succeeded')
await service.flush()
const terminal = parseProducedRows(outputs).find((r) => r.status === 'succeeded')!
expect(terminal.first_scheduled_at).toBe(original)
expect(terminal.scheduled_at).not.toBe(terminal.first_scheduled_at)
})
})
})