:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
/**
* Workflows E2E tests through postgres-v2 (Cyclotron node DB).
*
* These tests exercise the full hogflow lifecycle:
* event → CdpEventsConsumer → CyclotronJobQueuePostgresV2 (produces to v2 DB)
* → CdpCyclotronWorkerHogFlow (polls v2 DB) → HogFlowExecutorService
* → results written back to v2 DB → logs/metrics to Kafka
*
* Only `fetch` is mocked. Everything else is real: v2 database, Kafka
* producers, Postgres, Redis, person loading, filter evaluation, and
* state serialization.
*/
import { MockKafkaProducerWrapper } from '~/tests/helpers/mocks/producer.mock'
import { mockFetch, mockInternalFetch } from '~/tests/helpers/mocks/request.mock'
import { KafkaProducerObserver } from '~/tests/helpers/mocks/producer.spy'
import jsonwebtoken from 'jsonwebtoken'
import { DateTime } from 'luxon'
import { Pool } from 'pg'
import { register } from 'prom-client'
import supertest from 'supertest'
import express from 'ultimate-express'
import { HogFlow } from '~/cdp/schema/hogflow'
import { template as createTaskTemplate } from '~/cdp/templates/_destinations/posthog_tasks/posthog-create-task.template'
import { template as webhookTemplate } from '~/cdp/templates/_destinations/webhook/webhook.template'
import { setupExpressApp } from '~/common/api/router'
import {
KAFKA_APP_METRICS_2,
KAFKA_HOG_INVOCATION_RESULTS,
KAFKA_LOG_ENTRIES,
KAFKA_MESSAGE_ASSETS,
} from '~/common/config/kafka-topics'
import { KafkaProducerWrapper } from '~/common/kafka/producer'
import { InternalPersonWithDistinctId, PersonReadRepository } from '~/common/persons/repositories/person-repository'
import { deleteKeysWithPrefix } from '~/common/redis/_tests/redis'
import { InternalFetchService } from '~/common/services/internal-fetch'
import { closeHub, createHub } from '~/common/utils/db/hub'
import { PostgresUse } from '~/common/utils/db/postgres'
import { parseJSON } from '~/common/utils/json-parse'
import { UUIDT } from '~/common/utils/utils'
import { createCdpConsumerDeps } from '~/tests/helpers/cdp'
import { waitForExpect } from '~/tests/helpers/expectations'
import { TEST_KAFKA_TOPICS, ensureKafkaTopics } from '~/tests/helpers/kafka'
import { createTeam, getFirstTeam, resetBehavioralCohortsDatabase, resetTestDatabase } from '~/tests/helpers/sql'
import { Hub, Team } from '../../src/types'
import { createRedisV2PoolFromConfig } from '../common/redis/redis-v2'
import { FixtureHogFlowBuilder } from './_tests/builders/hogflow.builder'
import { HOG_FILTERS_EXAMPLES, HOG_FLOW_MASK_EXAMPLES } from './_tests/examples'
import {
createHogExecutionGlobals,
insertCohortMembership,
insertHogFunctionTemplate,
insertIntegration,
} from './_tests/fixtures'
import { insertHogFlow } from './_tests/fixtures-hogflows'
import { CdpApi } from './cdp-api'
import { CdpCyclotronWorkerBatchResolve } from './consumers/cdp-cyclotron-worker-batch-resolve.consumer'
import { CdpCyclotronWorkerEmail } from './consumers/cdp-cyclotron-worker-email.consumer'
import { CdpCyclotronWorkerHogFlow } from './consumers/cdp-cyclotron-worker-hogflow.consumer'
import { CdpDatawarehouseEventsConsumer } from './consumers/cdp-data-warehouse-events.consumer'
import { CdpEventsConsumer } from './consumers/cdp-events.consumer'
import { CdpHogflowSubscriptionMatcherConsumer } from './consumers/cdp-hogflow-subscription-matcher.consumer'
import { createCdpOutputsRegistry } from './outputs/registry'
import {
CyclotronV2DequeuedJob,
CyclotronV2Janitor,
CyclotronV2Manager,
CyclotronV2Worker,
JANITOR_POISON_PILL_ERROR_KIND,
} from './services/cyclotron-v2'
import {
HOGFLOW_BATCH_RESOLVE_QUEUE,
MAX_RESOLVER_ATTEMPTS,
serializeResolverState,
} from './services/hogflows/batch-resolver.types'
import { HogFlowBatchPersonQueryService } from './services/hogflows/hogflow-batch-person-query.service'
import { CyclotronJobQueueKafka } from './services/job-queue/job-queue-kafka'
import { CyclotronJobQueuePostgresV2 } from './services/job-queue/job-queue-postgres-v2'
import { CyclotronJobQueueRateLimitedPostgresV2 } from './services/job-queue/job-queue-rate-limited-postgres-v2'
import { JobQueue } from './services/job-queue/job-queue.interface'
import { createFrequencyCapValkeyPool } from './services/messaging/frequency-cap-valkey-pool'
import { HogInvocationResultsService } from './services/monitoring/hog-invocation-results.service'
import { RateLimiterService } from './services/rate-limiter/rate-limiter.service'
import { HogFunctionInvocationGlobals } from './types'
import { convertBatchHogFlowRequestToHogFunctionInvocationGlobals } from './utils'
import { convertToHogFunctionFilterGlobal } from './utils/hog-function-filtering'
const ActualKafkaProducerWrapper = jest.requireActual('~/common/kafka/producer').KafkaProducerWrapper
// DNS is mocked (like fetch) because EmailValidationService's MX lookups are a real
// network boundary: validation is fail-open, so a CI resolver hiccup would silently
// turn a "skipped hard bounce" assertion into a sent email — a guaranteed flake.
// Implementations are domain-aware and set in the email-queue block's beforeEach.
const mockDnsResolveMx = jest.fn()
const mockDnsResolve4 = jest.fn()
const mockDnsResolve6 = jest.fn()
jest.mock('node:dns/promises', () => ({
Resolver: jest.fn().mockImplementation(() => ({
resolveMx: mockDnsResolveMx,
resolve4: mockDnsResolve4,
resolve6: mockDnsResolve6,
})),
}))
// Use the same env vars as config.ts (lines 221-229) so cleanup pools and hub target the same DBs
const CYCLOTRON_NODE_DB_URL =
process.env.CYCLOTRON_NODE_DATABASE_URL ?? 'postgres://posthog:posthog@localhost:5432/test_cyclotron_node'
describe('Workflows E2E (postgres-v2)', () => {
jest.setTimeout(30000)
let eventsConsumer: CdpEventsConsumer
let dwhConsumer: CdpDatawarehouseEventsConsumer
let hogflowWorker: CdpCyclotronWorkerHogFlow
let hogflowQueue: JobQueue
let hub: Hub
let kafkaProducer: KafkaProducerWrapper
let mockProducerObserver: KafkaProducerObserver
let team: Team
let globals: HogFunctionInvocationGlobals
let mockPersonRepo: jest.Mocked<PersonReadRepository>
let cyclotronPool: Pool
let deps: ReturnType<typeof createCdpConsumerDeps>
beforeAll(() => {
cyclotronPool = new Pool({
connectionString: CYCLOTRON_NODE_DB_URL,
})
})
afterAll(async () => {
await cyclotronPool.end()
})
beforeEach(async () => {
// Real Kafka producers for all CDP producer slots
MockKafkaProducerWrapper.create = jest.fn((...args) => {
return ActualKafkaProducerWrapper.create(...args)
})
await ensureKafkaTopics(TEST_KAFKA_TOPICS)
await resetTestDatabase()
await cyclotronPool.query('DELETE FROM cyclotron_jobs')
hub = await createHub()
kafkaProducer = await ActualKafkaProducerWrapper.create(hub.KAFKA_CLIENT_RACK)
mockProducerObserver = new KafkaProducerObserver(kafkaProducer)
team = await getFirstTeam(hub.postgres)
mockProducerObserver.resetKafkaProducer()
hub.CDP_CYCLOTRON_BATCH_DELAY_MS = 50
hub.CDP_FETCH_RETRIES = 2
hub.CDP_FETCH_BACKOFF_BASE_MS = 50
// Insert a simple fetch template for function actions
await insertHogFunctionTemplate(hub.postgres, {
id: 'template-workflows-e2e-fetch',
name: 'Workflows E2E Fetch',
code: `
let res := fetch(inputs.url, {'method': inputs.method});
print('Fetch result:', res.status);
`,
inputs_schema: [
{ key: 'url', type: 'string', required: true },
{ key: 'method', type: 'string', required: false },
],
})
mockPersonRepo = {
fetchPerson: jest.fn().mockResolvedValue(undefined),
fetchPersonsByDistinctIds: jest.fn().mockResolvedValue([]),
fetchPersonsByPersonIds: jest.fn().mockResolvedValue([]),
fetchDistinctIdsForPersons: jest.fn().mockResolvedValue({}),
}
deps = { ...createCdpConsumerDeps(hub, kafkaProducer), personRepository: mockPersonRepo }
const kafkaQueue = new CyclotronJobQueueKafka(hub.KAFKA_CLIENT_RACK, hub, hub.CONSUMER_BATCH_SIZE)
hogflowQueue = new CyclotronJobQueuePostgresV2(hub.CONSUMER_BATCH_SIZE, hub)
// Events consumer — only start as producer (skip Kafka consumer connection).
// We call processBatch() directly so the Kafka consumer is not needed.
eventsConsumer = new CdpEventsConsumer(hub, deps, {
hogQueue: kafkaQueue,
hogflowQueue,
})
// Drives the data-warehouse-table trigger path. We call processBatch() directly, so the
// Kafka consumer is never connected — the shared queues below are the only producers.
dwhConsumer = new CdpDatawarehouseEventsConsumer(hub, deps, {
hogQueue: kafkaQueue,
hogflowQueue,
})
await Promise.all([kafkaQueue.startAsProducer(), hogflowQueue.startAsProducer()])
// Start hogflow worker (consumer side — polls the postgres-v2 backend)
hogflowWorker = new CdpCyclotronWorkerHogFlow(hub, deps, hogflowQueue)
await hogflowWorker.start()
mockFetch.mockResolvedValue({
status: 200,
headers: { 'Content-Type': 'application/json' },
json: () => Promise.resolve({ success: true }),
text: () => Promise.resolve(JSON.stringify({ success: true })),
dump: () => Promise.resolve(),
})
})
afterEach(async () => {
// dwhConsumer shares the already-stopped queues with eventsConsumer, so it has nothing to stop.
await Promise.all([eventsConsumer?.stop() ?? Promise.resolve(), hogflowWorker?.stop() ?? Promise.resolve()])
await kafkaProducer.disconnect()
await closeHub(hub)
mockProducerObserver.resetKafkaProducer()
})
// ── Helpers ──────────────────────────────────────────────────────
function createGlobals(
overrides: Partial<HogFunctionInvocationGlobals['event']> = {}
): HogFunctionInvocationGlobals {
return createHogExecutionGlobals({
project: { id: team.id } as any,
event: {
uuid: 'b3a1fe86-b10c-43cc-acaf-d208977608d0',
event: '$pageview',
properties: {
$current_url: 'https://posthog.com',
$lib_version: '1.0.0',
},
timestamp: '2024-09-03T09:00:00Z',
...overrides,
} as any,
})
}
// v2 stores job lifecycle in the `status` column.
const statusColumn = 'status'
async function queryCyclotronJobs(): Promise<any[]> {
const result = await cyclotronPool.query(`SELECT * FROM cyclotron_jobs ORDER BY created ASC`)
return result.rows
}
/** Send an event through the events consumer and wait for it to be queued to v2 */
async function triggerWorkflow(eventGlobals: HogFunctionInvocationGlobals): Promise<void> {
const { backgroundTask } = await eventsConsumer.processBatch([eventGlobals])
await backgroundTask
}
/**
* Mirrors what the hog flow API stores on save: a secret input is stripped out of the plaintext
* `actions` and kept Fernet-encrypted in `encrypted_inputs`, keyed by action id and then input key.
*/
async function setWorkflowEncryptedInputs(
workflowId: string,
secretsByActionId: Record<string, Record<string, { value: unknown }>>
): Promise<void> {
await hub.postgres.query(
PostgresUse.COMMON_WRITE,
`UPDATE posthog_hogflow SET encrypted_inputs = $2 WHERE id = $1`,
[workflowId, hub.encryptedFields.encrypt(JSON.stringify(secretsByActionId))],
'setWorkflowEncryptedInputs'
)
}
/**
* Build the row-scoped globals the DWH consumer produces for a synced warehouse row: a synthetic
* event named `$warehouse_source_row` with no real person, and the source table on
* `event.properties.$source_table` so the consumer's eligibilityFn matches warehouse-table triggers.
*/
function createDwhGlobals(
tableName: string,
rowProperties: Record<string, any> = {}
): HogFunctionInvocationGlobals {
return createHogExecutionGlobals({
project: { id: team.id } as any,
event: {
uuid: new UUIDT().toString(),
event: '$warehouse_source_row',
distinct_id: '',
properties: { ...rowProperties, $source_table: tableName },
timestamp: '2024-09-03T09:00:00Z',
} as any,
})
}
/** Send a synced warehouse row through the DWH consumer and wait for it to be queued */
async function triggerDwhWorkflow(rowGlobals: HogFunctionInvocationGlobals): Promise<void> {
const { backgroundTask } = await dwhConsumer.processBatch([rowGlobals])
await backgroundTask
}
/** Insert an active hogflow for the current team */
async function createWorkflow(
workflow: Parameters<FixtureHogFlowBuilder['withWorkflow']>[0],
opts?: { name?: string }
): Promise<string> {
const flow = await createWorkflowFlow(workflow, opts)
return flow.id
}
/** Same as createWorkflow but returns the full HogFlow object (useful for hand-built invocations) */
async function createWorkflowFlow(
workflow: Parameters<FixtureHogFlowBuilder['withWorkflow']>[0],
opts?: { name?: string; conversion?: HogFlow['conversion']; exitCondition?: HogFlow['exit_condition'] }
): Promise<HogFlow> {
const builder = new FixtureHogFlowBuilder().withTeamId(team.id).withStatus('active').withWorkflow(workflow)
if (opts?.name) {
builder.withName(opts.name)
}
if (opts?.conversion) {
builder.withConversion(opts.conversion)
}
if (opts?.exitCondition) {
builder.withExitCondition(opts.exitCondition)
}
const flow = builder.build()
await insertHogFlow(hub.postgres, flow)
return flow
}
/**
* Construct and enqueue a batch-shaped CyclotronJobInvocation directly, mimicking what
* the batch resolver's buildHogFlowInvocation would produce. Skips the blast-radius API
* call so tests don't need to stand up the Django side.
*/
async function triggerBatchWorkflow(hogFlow: HogFlow, personUuid: string): Promise<void> {
const invocationGlobals = convertBatchHogFlowRequestToHogFunctionInvocationGlobals({
team,
personId: personUuid,
siteUrl: hub.SITE_URL,
})
const filterGlobals = convertToHogFunctionFilterGlobal(invocationGlobals)
const invocation = {
id: new UUIDT().toString(),
state: {
event: invocationGlobals.event,
personId: personUuid,
actionStepCount: 0,
variables: {},
},
teamId: team.id,
functionId: hogFlow.id,
parentRunId: new UUIDT().toString(),
hogFlow,
person: invocationGlobals.person,
filterGlobals,
queue: 'hogflow' as const,
queuePriority: 1,
}
await hogflowQueue.queueInvocations([invocation])
}
// Reusable action configs
const trigger = () =>
({
type: 'trigger' as const,
config: { type: 'event', filters: HOG_FILTERS_EXAMPLES.no_filters.filters ?? {} },
}) as const
const fetchAction = (url: string, method = 'POST') => ({
type: 'function' as const,
config: {
template_id: 'template-workflows-e2e-fetch',
inputs: { url: { value: url }, method: { value: method } },
},
})
const delayAction = (duration: string) => ({
type: 'delay' as const,
config: { delay_duration: duration },
})
// Waits for a date carried by the event rather than a fixed span. Bytecode is what the HogQL compiler
// emits for `properties.expires_at`, which is what HogFlowSerializer stores for this expression.
const delayUntilAction = (offset?: string) => ({
type: 'delay' as const,
config: {
delay_until: {
expression: 'properties.expires_at',
bytecode: ['_H', 1, 32, 'expires_at', 32, 'properties', 1, 2],
...(offset ? { offset } : {}),
},
},
})
const exitAction = () => ({ type: 'exit' as const, config: {} })
// Mirrors what HogFlowSerializer compiles for {events: [{id: <name>}]}: a single
// equality check on the `event` global. The matcher is fail-closed when bytecode
// is absent, so fixtures must supply it the same way Django would on save.
const eventNameBytecode = (eventName: string): any[] => ['_H', 1, 32, eventName, 32, 'event', 1, 1, 11]
const eventNameFilter = (eventName: string) => ({
filters: { events: [{ id: eventName }], bytecode: eventNameBytecode(eventName) },
})
// An action-based wait entry: the editor's Actions picker yields a filter with `actions` set and
// `events` empty. Django compiles the action's match conditions into bytecode the same way.
const actionFilter = (eventName: string, actionId: number) => ({
filters: { actions: [{ id: actionId, type: 'actions' }], events: [], bytecode: eventNameBytecode(eventName) },
})
// The state left when the last event is removed from a wait entry in the UI: no events, no
// actions. Empty filters compile to always-true bytecode (op 29), which must NOT wake on every
// event. ['_H', 1, 29] is exactly what the Django compiler emits for empty filters.
const emptyEventFilter = () => ({ filters: { events: [], bytecode: ['_H', 1, 29] } })
// A wait CONDITION with no property filters: the state left when a condition's last filter is
// removed, or one is added but never filled in. Django compiles it to the same always-true
// bytecode (op 29). Unlike an events entry, the executor evaluates the condition on entry, so
// without the guard this fires the wait immediately. Mirrors the serializer's compiled shape.
const emptyConditionFilters = () => ({ bytecode: ['_H', 1, 29], properties: [] })
// A wait CONDITION on a person property: `person.properties[key] == value`. Mirrors the bytecode
// Django compiles for a single person-property equality filter. Used to prove that a person
// mutation (no analytics event) can satisfy a parked condition wait.
const personPropertyConditionFilters = (key: string, value: string): any => ({
bytecode: ['_H', 1, 32, value, 32, key, 32, 'properties', 32, 'person', 1, 3, 11],
properties: [{ key, value, type: 'person', operator: 'exact' }],
})
describe('simple workflow: trigger → function → exit', () => {
let workflowId: string
beforeEach(async () => {
workflowId = await createWorkflow({
actions: {
trigger: trigger(),
function_1: fetchAction('https://example.com/webhook'),
exit: exitAction(),
},
edges: [
{ from: 'trigger', to: 'function_1', type: 'continue' },
{ from: 'function_1', to: 'exit', type: 'continue' },
],
})
globals = createGlobals()
})
it('should execute the workflow end-to-end through v2', async () => {
await triggerWorkflow(globals)
await waitForExpect(() => {
expect(mockFetch).toHaveBeenCalledTimes(1)
}, 10000)
expect(mockFetch).toHaveBeenCalledWith(
'https://example.com/webhook',
expect.objectContaining({ method: 'POST' })
)
// Verify metrics were produced to Kafka
await waitForExpect(() => {
const metrics = mockProducerObserver
.getProducedKafkaMessagesForTopic(KAFKA_APP_METRICS_2)
.filter((m: any) => m.value.app_source === 'hog_flow')
expect(metrics.length).toBeGreaterThanOrEqual(1)
}, 5000)
// Verify logs were produced to Kafka
await waitForExpect(() => {
const logs = mockProducerObserver.getProducedKafkaMessagesForTopic(KAFKA_LOG_ENTRIES)
expect(logs.length).toBeGreaterThanOrEqual(1)
}, 5000)
})
it('mirrors every metric of the run under the workflow version that produced it', async () => {
await triggerWorkflow(globals)
const namesFor = (appSource: string, appSourceId: string): string[] =>
mockProducerObserver
.getProducedKafkaMessagesForTopic(KAFKA_APP_METRICS_2)
.filter((m: any) => m.value.app_source === appSource && m.value.app_source_id === appSourceId)
.map((m: any) => m.value.metric_name)
.sort()
// The version-scoped series has to carry the whole run, not just the run-level outcome —
// a per-version step funnel is only readable if every step's metric is mirrored.
await waitForExpect(() => {
expect(namesFor('hog_flow', workflowId)).toContain('succeeded')
expect(namesFor('hog_flow_version', `${workflowId}/1`)).toEqual(namesFor('hog_flow', workflowId))
}, 10000)
})
})
describe('workflow with a secret function input: trigger → function → exit', () => {
const secretApiKey = 'workflows-e2e-secret-api-key-0000'
let workflowId: string
beforeEach(async () => {
await insertHogFunctionTemplate(hub.postgres, {
id: 'template-workflows-e2e-secret-fetch',
name: 'Workflows E2E Secret Fetch',
code: `
let res := fetch(inputs.url, {'method': 'POST', 'headers': {'Authorization': f'Bearer {inputs.api_key}'}});
print('Fetch result:', res.status);
`,
inputs_schema: [
{ key: 'url', type: 'string', required: true },
{ key: 'api_key', type: 'string', required: true, secret: true },
],
})
workflowId = await createWorkflow({
actions: {
trigger: trigger(),
send_with_secret: {
type: 'function',
config: {
template_id: 'template-workflows-e2e-secret-fetch',
inputs: { url: { value: 'https://example.com/secret-webhook' } },
},
},
exit: exitAction(),
},
edges: [
{ from: 'trigger', to: 'send_with_secret', type: 'continue' },
{ from: 'send_with_secret', to: 'exit', type: 'continue' },
],
})
await setWorkflowEncryptedInputs(workflowId, { send_with_secret: { api_key: { value: secretApiKey } } })
globals = createGlobals()
})
it('sends the decrypted secret and keeps it out of logs and stored jobs', async () => {
await triggerWorkflow(globals)
await waitForExpect(() => {
expect(mockFetch).toHaveBeenCalledTimes(1)
}, 10000)
expect(mockFetch).toHaveBeenCalledWith(
'https://example.com/secret-webhook',
expect.objectContaining({
method: 'POST',
headers: expect.objectContaining({ Authorization: `Bearer ${secretApiKey}` }),
})
)
await waitForExpect(() => {
const logs = mockProducerObserver
.getProducedKafkaMessagesForTopic(KAFKA_LOG_ENTRIES)
.filter((m: any) => m.value.log_source_id === workflowId)
expect(logs.map((m: any) => m.value.message)).toContainEqual(
expect.stringContaining('Fetch result:, 200')
)
expect(JSON.stringify(logs)).not.toContain(secretApiKey)
}, 5000)
expect(JSON.stringify(await queryCyclotronJobs())).not.toContain(secretApiKey)
})
})
describe('workflow with webhook secret headers: trigger → webhook → exit', () => {
const token = 'workflows-e2e-secret-headers-token-0000'
const apiKey = 'workflows-e2e-secret-headers-api-key-0000'
const webhookUrl = 'https://example.com/secret-headers-webhook'
let workflowId: string
let releaseRetry: () => void = () => {}
const fetchResponse = (status: number) => ({
status,
json: () => Promise.resolve({}),
text: () => Promise.resolve('{}'),
headers: { 'Content-Type': 'application/json' },
dump: () => Promise.resolve(),
})
beforeEach(async () => {
await insertHogFunctionTemplate(hub.postgres, {
id: 'template-webhook',
name: webhookTemplate.name,
code: webhookTemplate.code,
inputs_schema: webhookTemplate.inputs_schema,
})
workflowId = await createWorkflow({
actions: {
trigger: trigger(),
send_webhook: {
type: 'function',
config: {
template_id: 'template-webhook',
inputs: {
url: { value: webhookUrl },
method: { value: 'POST' },
body: { value: { event: '{event.event}' } },
// Same name as a secret header in another case, so the merge has to
// replace it instead of sending both.
headers: {
value: {
'Content-Type': 'application/json',
authorization: 'plaintext-authorization',
},
},
debug: { value: true },
},
},
},
exit: exitAction(),
},
edges: [
{ from: 'trigger', to: 'send_webhook', type: 'continue' },
{ from: 'send_webhook', to: 'exit', type: 'continue' },
],
})
await setWorkflowEncryptedInputs(workflowId, {
send_webhook: { secret_headers: { value: { Authorization: `Bearer ${token}`, 'X-Api-Key': apiKey } } },
})
// The first attempt fails so the fetch parameters are written to the job row for the retry.
// The retry is then held open until the test releases it, so the row can be read while it
// still holds the state the worker resumed from.
let attempts = 0
const retryGate = new Promise<void>((resolve) => {
releaseRetry = resolve
})
mockFetch.mockImplementation(async () => {
attempts += 1
if (attempts > 1) {
await retryGate
}
return fetchResponse(attempts === 1 ? 500 : 200)
})
globals = createGlobals()
})
it('merges the decrypted secret headers on both attempts and keeps them off the fetch parameters', async () => {
try {
await triggerWorkflow(globals)
await waitForExpect(() => {
expect(mockFetch).toHaveBeenCalledTimes(2)
}, 10000)
// Only `queueParameters` is asserted on. `state.globals.inputs` carries every resolved input,
// which is a property of the whole job payload and not of this template.
const blobs = (await queryCyclotronJobs()).map((row: any) =>
parseJSON((row.state as Buffer).toString('utf-8'))
)
const fetchBlob = blobs.find((blob: any) => blob.queueParameters?.type === 'fetch')
expect(fetchBlob).toBeDefined()
expect(fetchBlob.queueParameters).toMatchObject({
secret_headers_input: 'secret_headers',
headers: { 'Content-Type': 'application/json', authorization: 'plaintext-authorization' },
})
expect(JSON.stringify(fetchBlob.queueParameters)).not.toContain(token)
expect(JSON.stringify(fetchBlob.queueParameters)).not.toContain(apiKey)
} finally {
releaseRetry()
}
for (const [url, options] of mockFetch.mock.calls as [string, any][]) {
expect(url).toEqual(webhookUrl)
expect(options.headers).toMatchObject({
Authorization: `Bearer ${token}`,
'X-Api-Key': apiKey,
'Content-Type': 'application/json',
})
expect(Object.keys(options.headers).filter((name) => name.toLowerCase() === 'authorization')).toEqual([
'Authorization',
])
}
await waitForExpect(() => {
const messages = mockProducerObserver
.getProducedKafkaMessagesForTopic(KAFKA_LOG_ENTRIES)
.filter((m: any) => m.value.log_source_id === workflowId)
.map((m: any) => m.value.message as string)
expect(messages).toContainEqual(
expect.stringContaining('HTTP fetch failed on attempt 1 with status code 500. Retrying.')
)
expect(messages).toContainEqual(expect.stringContaining('Response, 200'))
const requestLogs = messages.filter((message) => message.includes('Request, '))
expect(requestLogs).toHaveLength(1)
expect(requestLogs[0]).toContain('"secret_headers_input":"secret_headers"')
// Log redaction masks values but not header names. The name is what appears if the
// template merges the secret headers into `payload.headers` instead of passing the reference.
expect(requestLogs[0]).not.toContain('X-Api-Key')
expect(JSON.stringify(messages)).not.toContain(token)
}, 5000)
})
})
describe('delay workflow: trigger → delay → function → exit', () => {
beforeEach(async () => {
await createWorkflow({
actions: {
trigger: trigger(),
delay_1: delayAction('0.5s'),
function_1: fetchAction('https://example.com/delayed-webhook'),
exit: exitAction(),
},
edges: [
{ from: 'trigger', to: 'delay_1', type: 'continue' },
{ from: 'delay_1', to: 'function_1', type: 'continue' },
{ from: 'function_1', to: 'exit', type: 'continue' },
],
})
globals = createGlobals()
})
it('should reschedule on delay and execute function after delay passes', async () => {
await triggerWorkflow(globals)
// First: worker picks up job and hits the delay — job gets rescheduled
await waitForExpect(async () => {
const jobs = await queryCyclotronJobs()
const rescheduled = jobs.filter(
(j: any) => j.status === 'available' && new Date(j.scheduled) > new Date()
)
expect(rescheduled.length).toBe(1)
}, 5000)
// Fetch should NOT have been called yet (delay hasn't passed)
expect(mockFetch).not.toHaveBeenCalled()
// Wait for the delay to pass and the worker to pick up the job again
await waitForExpect(() => {
expect(mockFetch).toHaveBeenCalledTimes(1)
}, 10000)
expect(mockFetch).toHaveBeenCalledWith(
'https://example.com/delayed-webhook',
expect.objectContaining({ method: 'POST' })
)
// Verify the job completed (transition_count > 1 due to reschedule)
await waitForExpect(async () => {
const jobs = await queryCyclotronJobs()
const terminal = jobs.filter(
(j: any) => j.status === 'completed' || j.status === 'failed' || j.status === 'canceled'
)
expect(terminal.length).toBeGreaterThanOrEqual(1)
}, 5000)
})
})
describe('delay until a date carried by the data', () => {
const workflowWaitingUntil = async (offset?: string): Promise<void> => {
await createWorkflow({
actions: {
trigger: trigger(),
delay_1: delayUntilAction(offset),
function_1: fetchAction('https://example.com/reminder'),
exit: exitAction(),
},
edges: [
{ from: 'trigger', to: 'delay_1', type: 'continue' },
{ from: 'delay_1', to: 'function_1', type: 'continue' },
{ from: 'function_1', to: 'exit', type: 'continue' },
],
})
}
it('parks until the date on the event, then continues', async () => {
// A fixed duration cannot express this: the instant comes from the payload, so two runs of the
// same workflow park to different times.
const expiresAt = DateTime.utc().plus({ seconds: 0.5 })
await workflowWaitingUntil()
await triggerWorkflow(createGlobals({ properties: { expires_at: expiresAt.toISO() } } as any))
await waitForExpect(async () => {
const parked = (await queryCyclotronJobs()).filter(
(j: any) => j[statusColumn] === 'available' && new Date(j.scheduled) > new Date()
)
expect(parked).toHaveLength(1)
// Parked to the instant from the data, not to some default span.
const scheduled = DateTime.fromJSDate(new Date(parked[0].scheduled)).toUTC()
expect(Math.abs(scheduled.diff(expiresAt).as('seconds'))).toBeLessThan(2)
}, 5000)
expect(mockFetch).not.toHaveBeenCalled()
await waitForExpect(() => {
expect(mockFetch).toHaveBeenCalledTimes(1)
}, 15000)
expect(mockFetch).toHaveBeenCalledWith('https://example.com/reminder', expect.anything())
})
it('continues straight away when the date has already passed', async () => {
// The guard against a reminder for something that already happened firing days late.
await workflowWaitingUntil()
await triggerWorkflow(
createGlobals({ properties: { expires_at: DateTime.utc().minus({ days: 5 }).toISO() } } as any)
)
await waitForExpect(() => {
expect(mockFetch).toHaveBeenCalledTimes(1)
}, 10000)
})
it('fires before the date when an offset asks for it', async () => {
// The shape a "remind me N days before" workflow needs, and the reason an offset exists rather
// than only a bare date: here the date is 1 hour out and the offset pulls the wait to now.
await workflowWaitingUntil('-1h')
await triggerWorkflow(
createGlobals({ properties: { expires_at: DateTime.utc().plus({ hours: 1 }).toISO() } } as any)
)
await waitForExpect(() => {
expect(mockFetch).toHaveBeenCalledTimes(1)
}, 10000)
})
it('aborts instead of sending when the date cannot be worked out', async () => {
// The outcome that matters: a "before it expires" message must not go out for someone with no
// expiry. on_error defaults to 'continue', which would do exactly that, so an unresolvable date
// aborts the run regardless of that setting.
await workflowWaitingUntil()
await triggerWorkflow(createGlobals({ properties: {} } as any))
// Waiting for a terminal state, not merely "not available": a job being worked on is also not
// available, so the looser check can pass in the window before the run would have sent.
await waitForExpect(async () => {
const jobs = await queryCyclotronJobs()
expect(jobs.filter((j: any) => j[statusColumn] === 'failed')).toHaveLength(1)
}, 10000)
expect(mockFetch).not.toHaveBeenCalled()
})
})
describe('conditional branch workflow', () => {
beforeEach(async () => {
await createWorkflow({
actions: {
trigger: trigger(),
branch: {
type: 'conditional_branch',
config: {
conditions: [
{ filters: HOG_FILTERS_EXAMPLES.pageview_or_autocapture_filter.filters },
{ filters: HOG_FILTERS_EXAMPLES.elements_text_filter.filters },
],
},
},
function_a: fetchAction('https://example.com/branch-a'),
function_b: fetchAction('https://example.com/branch-b'),
exit: exitAction(),
},
edges: [
{ from: 'trigger', to: 'branch', type: 'continue' },
{ from: 'branch', to: 'function_a', type: 'branch', index: 0 },
{ from: 'branch', to: 'function_b', type: 'branch', index: 1 },
{ from: 'function_a', to: 'exit', type: 'continue' },
{ from: 'function_b', to: 'exit', type: 'continue' },
],
})
})
it('should take branch A when event matches the first condition', async () => {
globals = createGlobals({
event: '$pageview',
properties: { $current_url: 'https://posthog.com/pricing' },
})
await triggerWorkflow(globals)
await waitForExpect(() => {
expect(mockFetch).toHaveBeenCalledTimes(1)
}, 10000)
expect(mockFetch).toHaveBeenCalledWith('https://example.com/branch-a', expect.anything())
})
})
describe('cohort condition workflow', () => {
const COHORT_ID = 4242
const PERSON_UUID = 'a5f478a1-1d34-4c8a-a0b5-6a3b62e3f8a1'
// What the Django serializer compiles for a cohort condition filter; the shape is pinned
// from the Python side by test_hog_flow_conditional_branch_cohort_filter_compiles
const cohortConditionFilters = (cohortId: number): Record<string, any> => ({
bytecode: ['_H', 1, 33, cohortId, 32, 'cohort_ids', 1, 1, 2, 'inCohort', 2],
properties: [{ key: 'id', type: 'cohort', value: cohortId }],
})
beforeEach(async () => {
await resetBehavioralCohortsDatabase(hub.postgres)
// The person's uuid is the key the cohort_membership lookup runs against
mockPersonRepo.fetchPersonsByDistinctIds.mockResolvedValue([
{
id: '1',
uuid: PERSON_UUID,
team_id: team.id,
properties: {},
properties_last_updated_at: {},
properties_last_operation: null,
created_at: DateTime.utc(),
version: 1,
is_identified: true,
is_user_id: null,
last_seen_at: null,
distinct_id: 'distinct_id',
},
])
await createWorkflow({
actions: {
trigger: trigger(),
branch: {
type: 'conditional_branch',
config: {
conditions: [{ filters: cohortConditionFilters(COHORT_ID) }],
},
},
function_member: fetchAction('https://example.com/in-cohort'),
function_non_member: fetchAction('https://example.com/not-in-cohort'),
exit: exitAction(),
},
edges: [
{ from: 'trigger', to: 'branch', type: 'continue' },
{ from: 'branch', to: 'function_member', type: 'branch', index: 0 },
{ from: 'branch', to: 'function_non_member', type: 'continue' },
{ from: 'function_member', to: 'exit', type: 'continue' },
{ from: 'function_non_member', to: 'exit', type: 'continue' },
],
})
globals = createGlobals()
})
it.each([
['a membership row', true, 'https://example.com/in-cohort'],
['an in_cohort=false row (person left)', false, 'https://example.com/not-in-cohort'],
])('routes a person with %s down the right branch', async (_name, inCohort, expectedUrl) => {
await insertCohortMembership(hub.postgres, {
team_id: team.id,
cohort_id: COHORT_ID,
person_id: PERSON_UUID,
in_cohort: inCohort,
})
await triggerWorkflow(globals)
await waitForExpect(() => {
expect(mockFetch).toHaveBeenCalledTimes(1)
}, 10000)
expect(mockFetch).toHaveBeenCalledWith(expectedUrl, expect.anything())
})
it('routes a person the pipeline never wrote a row for as a non-member', async () => {
await triggerWorkflow(globals)
await waitForExpect(() => {
expect(mockFetch).toHaveBeenCalledTimes(1)
}, 10000)
expect(mockFetch).toHaveBeenCalledWith('https://example.com/not-in-cohort', expect.anything())
})
})
describe('workflow disabled mid-execution', () => {
let hogFlowId: string
beforeEach(async () => {
hogFlowId = await createWorkflow({
actions: {
trigger: trigger(),
delay_1: delayAction('0.5s'),
function_1: fetchAction('https://example.com/should-not-fire'),
exit: exitAction(),
},
edges: [
{ from: 'trigger', to: 'delay_1', type: 'continue' },
{ from: 'delay_1', to: 'function_1', type: 'continue' },
{ from: 'function_1', to: 'exit', type: 'continue' },
],
})
globals = createGlobals()
})
it('should cancel the job when workflow is archived during delay', async () => {
await triggerWorkflow(globals)
// Wait for the delay step to be hit (job rescheduled)
await waitForExpect(async () => {
const jobs = await queryCyclotronJobs()
expect(jobs.some((j: any) => j.status === 'available' && new Date(j.scheduled) > new Date())).toBe(true)
}, 5000)
// Archive the hogflow while job is waiting
await hub.postgres.query(
PostgresUse.COMMON_WRITE,
`UPDATE posthog_hogflow SET status = 'archived' WHERE id = $1`,
[hogFlowId],
'archiveHogFlow'
)
// Force the hogflow manager to reload
;(hogflowWorker as any).hogFlowManager.lazyLoader.markForRefresh(hogFlowId)
// Wait for the delayed job to be picked up and canceled
await waitForExpect(async () => {
const jobs = await queryCyclotronJobs()
const canceled = jobs.filter((j: any) => j.status === 'canceled')
expect(canceled.length).toBe(1)
}, 10000)
// Function should NOT have been called
expect(mockFetch).not.toHaveBeenCalled()
// The run must terminate through the result pipeline, not a silent queue flip:
// a 'canceled' metric lands and the run is not counted as 'succeeded'.
await waitForExpect(() => {
const metricNames = mockProducerObserver
.getProducedKafkaMessagesForTopic(KAFKA_APP_METRICS_2)
.filter((m: any) => m.value.app_source === 'hog_flow' && m.value.app_source_id === hogFlowId)
.map((m: any) => m.value.metric_name)
expect(metricNames).toContain('canceled')
expect(metricNames).not.toContain('succeeded')
}, 5000)
})
})
describe('live edit while a run is parked (follow-live contract)', () => {
// The runs park on a 2m delay so they cannot wake before the edit lands; the tests then
// pull the wake forward explicitly (the same scheduled-time update the subscription
// matcher performs), so there is no race between the park deadline and the edit.
/** Wait until a job is parked in the future (a delay/wait step was hit) */
async function waitForParkedJob(): Promise<void> {
await waitForExpect(async () => {
const jobs = await queryCyclotronJobs()
expect(jobs.some((j: any) => j.status === 'available' && new Date(j.scheduled) > new Date())).toBe(true)
}, 5000)
}
/** Persist an edited actions/edges graph and bust the worker's config cache, like Django's save + pub/sub would */
async function applyLiveEdit(flow: HogFlow): Promise<void> {
await hub.postgres.query(
PostgresUse.COMMON_WRITE,
`UPDATE posthog_hogflow SET actions = $2, edges = $3, updated_at = NOW() WHERE id = $1`,
[flow.id, JSON.stringify(flow.actions), JSON.stringify(flow.edges)],
'liveEditHogFlow'
)
;(hogflowWorker as any).hogFlowManager.lazyLoader.markForRefresh(flow.id)
}
/** Pull parked jobs' scheduled time forward so the worker picks them up now */
async function wakeParkedJobsNow(): Promise<void> {
await cyclotronPool.query(
`UPDATE cyclotron_jobs SET scheduled = NOW() WHERE ${statusColumn} = 'available' AND scheduled > NOW()`
)
}
/** Shorten the flow's delay so the woken run advances instead of re-parking */
function shortenDelay(flow: HogFlow, actionId: string): void {
const delay = flow.actions.find((a) => a.id === actionId)!
;(delay.config as any).delay_duration = '0.5s'
}
it('a content edit made while parked on an upstream delay is honored on wake', async () => {
const flow = await createWorkflowFlow({
actions: {
trigger: trigger(),
delay_1: delayAction('2m'),
function_1: fetchAction('https://example.com/content-v1'),
exit: exitAction(),
},
edges: [
{ from: 'trigger', to: 'delay_1', type: 'continue' },
{ from: 'delay_1', to: 'function_1', type: 'continue' },
{ from: 'function_1', to: 'exit', type: 'continue' },
],
})
await triggerWorkflow(createGlobals())
await waitForParkedJob()
const functionAction = flow.actions.find((a) => a.id === 'function_1')!
;(functionAction.config as any).inputs.url.value = 'https://example.com/content-v2'
shortenDelay(flow, 'delay_1')
await applyLiveEdit(flow)
await wakeParkedJobsNow()
await waitForExpect(() => {
expect(mockFetch).toHaveBeenCalledTimes(1)
}, 10000)
expect(mockFetch).toHaveBeenCalledWith('https://example.com/content-v2', expect.anything())
})
it("rerouting the parked step's edge does not re-run earlier steps or run the old target", async () => {
const flow = await createWorkflowFlow({
actions: {
trigger: trigger(),
function_a: fetchAction('https://example.com/step-a'),
delay_1: delayAction('2m'),
function_b: fetchAction('https://example.com/step-b'),
exit: exitAction(),
},
edges: [
{ from: 'trigger', to: 'function_a', type: 'continue' },
{ from: 'function_a', to: 'delay_1', type: 'continue' },
{ from: 'delay_1', to: 'function_b', type: 'continue' },
{ from: 'function_b', to: 'exit', type: 'continue' },
],
})
await triggerWorkflow(createGlobals())
await waitForExpect(() => {
expect(mockFetch).toHaveBeenCalledWith('https://example.com/step-a', expect.anything())
}, 10000)
await waitForParkedJob()
// Reroute the delay's continue edge onto a newly added step
const functionB = flow.actions.find((a) => a.id === 'function_b')!
flow.actions.push({
...functionB,
id: 'function_c',
config: {
...(functionB.config as any),
inputs: { url: { value: 'https://example.com/step-c' }, method: { value: 'POST' } },
},
})
flow.edges = [
{ from: 'trigger', to: 'function_a', type: 'continue' },
{ from: 'function_a', to: 'delay_1', type: 'continue' },
{ from: 'delay_1', to: 'function_c', type: 'continue' },
{ from: 'function_c', to: 'exit', type: 'continue' },
]
shortenDelay(flow, 'delay_1')
await applyLiveEdit(flow)
await wakeParkedJobsNow()
await waitForExpect(() => {
expect(mockFetch).toHaveBeenCalledWith('https://example.com/step-c', expect.anything())
}, 10000)
// Exactly one send per executed step: the step behind the run did not re-run, the old
// target never ran, the new target ran once
const urls = mockFetch.mock.calls.map((call) => call[0])
expect(urls.filter((u) => u === 'https://example.com/step-a')).toHaveLength(1)
expect(urls.filter((u) => u === 'https://example.com/step-b')).toHaveLength(0)
expect(urls.filter((u) => u === 'https://example.com/step-c')).toHaveLength(1)
})
it('deleting the step a run is parked on exits the run gracefully on wake', async () => {
const flow = await createWorkflowFlow({
actions: {
trigger: trigger(),
delay_1: delayAction('2m'),
function_1: fetchAction('https://example.com/should-not-fire'),
exit: exitAction(),
},
edges: [
{ from: 'trigger', to: 'delay_1', type: 'continue' },
{ from: 'delay_1', to: 'function_1', type: 'continue' },
{ from: 'function_1', to: 'exit', type: 'continue' },
],
})
await triggerWorkflow(createGlobals())
await waitForParkedJob()
flow.actions = flow.actions.filter((a) => a.id !== 'delay_1')
flow.edges = [
{ from: 'trigger', to: 'function_1', type: 'continue' },
{ from: 'function_1', to: 'exit', type: 'continue' },
]
await applyLiveEdit(flow)
await wakeParkedJobsNow()
// The parked run wakes, finds its step gone, and finishes as a deliberate exit
await waitForExpect(async () => {
const jobs = await queryCyclotronJobs()
expect(jobs.filter((j: any) => j.status === 'completed')).toHaveLength(1)
}, 10000)
expect(mockFetch).not.toHaveBeenCalled()
await waitForExpect(() => {
const metricNames = mockProducerObserver
.getProducedKafkaMessagesForTopic(KAFKA_APP_METRICS_2)
.filter((m: any) => m.value.app_source === 'hog_flow')
.map((m: any) => m.value.metric_name)
expect(metricNames).toContain('exited_workflow_changed')
expect(metricNames).not.toContain('failed')
}, 5000)
})
})
describe('multiple workflows matching same event', () => {
const simpleFetchWorkflow = (url: string) => ({
actions: {
trigger: trigger(),
function_1: fetchAction(url),
exit: exitAction(),
},
edges: [
{ from: 'trigger', to: 'function_1', type: 'continue' as const },
{ from: 'function_1', to: 'exit', type: 'continue' as const },
],
})
beforeEach(async () => {
await createWorkflow(simpleFetchWorkflow('https://example.com/workflow-a'), { name: 'Workflow A' })
await createWorkflow(simpleFetchWorkflow('https://example.com/workflow-b'), { name: 'Workflow B' })
globals = createGlobals()
})
it('should execute both workflows independently', async () => {
await triggerWorkflow(globals)
await waitForExpect(() => {
expect(mockFetch).toHaveBeenCalledTimes(2)
}, 10000)
const urls = mockFetch.mock.calls.map((call) => call[0])
expect(urls).toContain('https://example.com/workflow-a')
expect(urls).toContain('https://example.com/workflow-b')
})
})
describe('wait_until_condition: condition matches immediately', () => {
beforeEach(async () => {
await createWorkflow({
actions: {
trigger: trigger(),
wait_condition: {
type: 'wait_until_condition',
config: {
condition: {
// Matches $pageview with "posthog" in $current_url
filters: HOG_FILTERS_EXAMPLES.pageview_or_autocapture_filter.filters,
},
max_wait_duration: '10s',
},
},
function_matched: fetchAction('https://example.com/condition-matched'),
function_timeout: fetchAction('https://example.com/condition-timed-out'),
exit: exitAction(),
},
edges: [
{ from: 'trigger', to: 'wait_condition', type: 'continue' },
{ from: 'wait_condition', to: 'function_matched', type: 'branch', index: 0 },
{ from: 'wait_condition', to: 'function_timeout', type: 'continue' },
{ from: 'function_matched', to: 'exit', type: 'continue' },
{ from: 'function_timeout', to: 'exit', type: 'continue' },
],
})
// Event matches the condition: $pageview with posthog in URL
globals = createGlobals({
event: '$pageview',
properties: { $current_url: 'https://posthog.com' },
})
})
it('should take the matched branch without rescheduling', async () => {
await triggerWorkflow(globals)
await waitForExpect(() => {
expect(mockFetch).toHaveBeenCalledTimes(1)
}, 10000)
// Should hit the matched branch, not the timeout branch
expect(mockFetch).toHaveBeenCalledWith('https://example.com/condition-matched', expect.anything())
})
})
describe('wait_until_condition: condition never matches, times out', () => {
beforeEach(async () => {
await createWorkflow({
actions: {
trigger: trigger(),
wait_condition: {
type: 'wait_until_condition',
config: {
condition: {
// Requires $autocapture with "reload" in elements_chain_texts — won't match
filters: HOG_FILTERS_EXAMPLES.elements_text_filter.filters,
},
max_wait_duration: '0.5s',
},
},
function_1: fetchAction('https://example.com/after-wait-timeout'),
exit: exitAction(),
},
edges: [
{ from: 'trigger', to: 'wait_condition', type: 'continue' },
{ from: 'wait_condition', to: 'exit', type: 'branch', index: 0 },
{ from: 'wait_condition', to: 'function_1', type: 'continue' },
{ from: 'function_1', to: 'exit', type: 'continue' },
],
})
globals = createGlobals()
})
it('should reschedule while polling, then continue after max_wait expires', async () => {
await triggerWorkflow(globals)
// Job should be rescheduled (condition doesn't match, waiting for next poll)
await waitForExpect(async () => {
const jobs = await queryCyclotronJobs()
const rescheduled = jobs.filter(
(j: any) => j.status === 'available' && new Date(j.scheduled) > new Date()
)
expect(rescheduled.length).toBe(1)
}, 5000)
// Fetch should NOT be called yet — still waiting for condition
expect(mockFetch).not.toHaveBeenCalled()
// After max_wait expires, the condition times out and the workflow
// continues to the function action via the continue edge
await waitForExpect(() => {
expect(mockFetch).toHaveBeenCalledTimes(1)
}, 10000)
expect(mockFetch).toHaveBeenCalledWith('https://example.com/after-wait-timeout', expect.anything())
})
})
describe('wait_until_condition: subscription matcher wakes parked jobs', () => {
let matcher: CdpHogflowSubscriptionMatcherConsumer
// trigger → wait_condition → (matched branch | timeout continue) → exit
const createWaitUntilWorkflow = (waitConfig: Record<string, any>): Promise<string> =>
createWorkflow({
actions: {
trigger: trigger(),
wait_condition: { type: 'wait_until_condition', config: waitConfig },
function_matched: fetchAction('https://example.com/condition-matched'),
function_timeout: fetchAction('https://example.com/timed-out'),
exit: exitAction(),
},
edges: [
{ from: 'trigger', to: 'wait_condition', type: 'continue' },
{ from: 'wait_condition', to: 'function_matched', type: 'branch', index: 0 },
{ from: 'wait_condition', to: 'function_timeout', type: 'continue' },
{ from: 'function_matched', to: 'exit', type: 'continue' },
{ from: 'function_timeout', to: 'exit', type: 'continue' },
],
})
// The job is parked when it is available with a scheduled time in the future.
const expectParked = async (): Promise<void> => {
await waitForExpect(async () => {
const jobs = await queryCyclotronJobs()
expect(jobs.some((j: any) => j.status === 'available' && new Date(j.scheduled) > new Date())).toBe(true)
}, 5000)
expect(mockFetch).not.toHaveBeenCalled()
}
beforeEach(() => {
matcher = new CdpHogflowSubscriptionMatcherConsumer({ ...hub }, deps)
})
afterEach(async () => {
await matcher?.stop().catch(() => {})
})
it('wakes a parked job and takes the matched branch when a subscribed event fires', async () => {
await createWaitUntilWorkflow({
// Property condition never matches the trigger event, so the job parks.
condition: { filters: HOG_FILTERS_EXAMPLES.elements_text_filter.filters },
events: [eventNameFilter('wakeup_event')],
max_wait_duration: '5m',
})
await triggerWorkflow(createGlobals())
await expectParked()
// A subscribed event fires for this person — the matcher wakes the job.
await matcher.processBatch([createGlobals({ event: 'wakeup_event' })])
await waitForExpect(() => {
expect(mockFetch).toHaveBeenCalledTimes(1)
}, 10000)
expect(mockFetch).toHaveBeenCalledWith('https://example.com/condition-matched', expect.anything())
})
it('wakes a parked job whose wait entry is action-based (events empty, actions + bytecode set)', async () => {
await createWaitUntilWorkflow({
// Property condition never matches the trigger event, so the job parks.
condition: { filters: HOG_FILTERS_EXAMPLES.elements_text_filter.filters },
// "Events to wait for" entry targets a PostHog Action: filters.events is empty,
// filters.actions is set, and the compiled bytecode matches the action's event.
events: [actionFilter('action_wakeup_event', 3)],
max_wait_duration: '5m',
})
await triggerWorkflow(createGlobals())
await expectParked()
// The action's underlying event fires — the matcher must wake the job via the action
// entry even though filters.events is empty.
await matcher.processBatch([createGlobals({ event: 'action_wakeup_event' })])
await waitForExpect(() => {
expect(mockFetch).toHaveBeenCalledTimes(1)
}, 10000)
expect(mockFetch).toHaveBeenCalledWith('https://example.com/condition-matched', expect.anything())
})
it('wakes a parked job when a later event satisfies the property condition', async () => {
await createWaitUntilWorkflow({
// No events list — only a property-based condition. The matcher evaluates the
// condition against every incoming event, making property waits event-driven.
condition: { filters: HOG_FILTERS_EXAMPLES.pageview_or_autocapture_filter.filters },
max_wait_duration: '5m',
})
// Trigger with an event that does not satisfy the condition, so the job parks.
await triggerWorkflow(createGlobals({ event: 'custom_trigger', properties: {} }))
await expectParked()
// A later $pageview with a posthog URL satisfies the property condition.
await matcher.processBatch([
createGlobals({ event: '$pageview', properties: { $current_url: 'https://posthog.com' } }),
])
await waitForExpect(() => {
expect(mockFetch).toHaveBeenCalledTimes(1)
}, 10000)
expect(mockFetch).toHaveBeenCalledWith('https://example.com/condition-matched', expect.anything())
})
it('takes the timeout branch when neither events nor the condition match before max_wait', async () => {
await createWaitUntilWorkflow({
condition: { filters: HOG_FILTERS_EXAMPLES.elements_text_filter.filters },
events: [eventNameFilter('never_fires')],
max_wait_duration: '0.5s',
})
await triggerWorkflow(createGlobals())
// An unrelated event passes through the matcher but must not wake the job.
await matcher.processBatch([createGlobals({ event: 'some_other_event' })])
// After max_wait expires the job advances down the continue (timeout) branch.
await waitForExpect(() => {
expect(mockFetch).toHaveBeenCalledTimes(1)
}, 15000)
expect(mockFetch).toHaveBeenCalledWith('https://example.com/timed-out', expect.anything())
})
it('leaves the job parked when an event matches neither the events nor the condition', async () => {
await createWaitUntilWorkflow({
condition: { filters: HOG_FILTERS_EXAMPLES.elements_text_filter.filters },
events: [eventNameFilter('wakeup_event')],
max_wait_duration: '5m',
})
await triggerWorkflow(createGlobals())
await expectParked()
// An unrelated event must not wake the job.
await matcher.processBatch([createGlobals({ event: 'some_other_event' })])
const jobs = await queryCyclotronJobs()
expect(jobs.every((j: any) => j.status === 'available' && new Date(j.scheduled) > new Date())).toBe(true)
expect(mockFetch).not.toHaveBeenCalled()
})
it('does not wake on a completely empty "events to wait for" entry (always-true bytecode)', async () => {
// The state left when the last event is removed from a wait entry: empty filters compile
// to always-true bytecode. Inserted directly so it bypasses the serializer strip — this
// guards the matcher itself, which must NOT fire the workflow on every incoming event.
await createWaitUntilWorkflow({
condition: { filters: HOG_FILTERS_EXAMPLES.elements_text_filter.filters },
events: [emptyEventFilter()],
max_wait_duration: '5m',
})
await triggerWorkflow(createGlobals())
await expectParked()
// An unrelated event must not wake the job — the always-true bytecode would otherwise match.
await matcher.processBatch([createGlobals({ event: 'some_unrelated_event' })])
const jobs = await queryCyclotronJobs()
expect(jobs.every((j: any) => j.status === 'available' && new Date(j.scheduled) > new Date())).toBe(true)
expect(mockFetch).not.toHaveBeenCalled()
})
it('does not fire on entry for an empty property condition; takes the timeout branch', async () => {
// An empty condition compiles to always-true bytecode. The executor evaluates the
// condition on entry, so without the guard the wait advances down the matched branch
// immediately. With no events and no real condition it must park and time out instead.
await createWaitUntilWorkflow({
condition: { filters: emptyConditionFilters() },
max_wait_duration: '0.5s',
})
await triggerWorkflow(createGlobals())
// If it fired on entry this would be the matched branch; it must be the timeout branch.
await waitForExpect(() => {
expect(mockFetch).toHaveBeenCalledTimes(1)
}, 15000)
expect(mockFetch).toHaveBeenCalledWith('https://example.com/timed-out', expect.anything())
})
it('does not fire on entry when an empty condition coexists with a real events entry; still wakes on the event', async () => {
// The reported bug: an empty (always-true) condition alongside a real "events to wait
// for" entry. Without the guard the empty condition matches on entry and the wait fires
// immediately, ignoring the configured event. It must park and only wake when the event
// actually fires.
await createWaitUntilWorkflow({
condition: { filters: emptyConditionFilters() },
events: [eventNameFilter('wakeup_event')],
max_wait_duration: '5m',
})
await triggerWorkflow(createGlobals())
await expectParked()
// The configured event fires — the matcher wakes the job via the events entry.
await matcher.processBatch([createGlobals({ event: 'wakeup_event' })])
await waitForExpect(() => {
expect(mockFetch).toHaveBeenCalledTimes(1)
}, 10000)
expect(mockFetch).toHaveBeenCalledWith('https://example.com/condition-matched', expect.anything())
})
it('does not collapse a delay before a wait when the wait event fires during the delay', async () => {
// trigger -> delay(5m) -> wait_until_condition(wakeup_event) -> matched / timeout. The
// wait's event firing DURING the delay must not wake the job: the delay handler no longer
// pre-advances currentAction to the wait, so while parked the job is at the delay (not the
// wait) and the matcher leaves it alone. The delay is honored.
await createWorkflow({
actions: {
trigger: trigger(),
delay: { type: 'delay', config: { delay_duration: '5m' } },
wait_condition: {
type: 'wait_until_condition',
config: {
events: [eventNameFilter('wakeup_event')],
condition: { filters: null },
max_wait_duration: '5m',
},
},
function_matched: fetchAction('https://example.com/matched'),
exit: exitAction(),
},
edges: [
{ from: 'trigger', to: 'delay', type: 'continue' },
{ from: 'delay', to: 'wait_condition', type: 'continue' },
{ from: 'wait_condition', to: 'function_matched', type: 'branch', index: 0 },
{ from: 'wait_condition', to: 'exit', type: 'continue' },
{ from: 'function_matched', to: 'exit', type: 'continue' },
],
})
await triggerWorkflow(createGlobals())
await expectParked()
// The wait's event fires during the delay — the job must stay parked in the delay.
await matcher.processBatch([createGlobals({ event: 'wakeup_event' })])
const jobs = await queryCyclotronJobs()
expect(jobs.every((j: any) => j.status === 'available' && new Date(j.scheduled) > new Date())).toBe(true)
expect(mockFetch).not.toHaveBeenCalled()
})
it('does not run the next step early when a conversion event fires during a delay', async () => {
// trigger -> delay -> fetch, with an event-based conversion goal used only for
// measurement (exit_only_at_end). A conversion event arriving while the job is parked
// in the delay must NOT wake it and run the fetch ~5 minutes early.
await createWorkflowFlow(
{
actions: {
trigger: trigger(),
delay: { type: 'delay', config: { delay_duration: '5m' } },
after_delay: fetchAction('https://example.com/after-delay'),
exit: exitAction(),
},
edges: [
{ from: 'trigger', to: 'delay', type: 'continue' },
{ from: 'delay', to: 'after_delay', type: 'continue' },
{ from: 'after_delay', to: 'exit', type: 'continue' },
],
},
{
exitCondition: 'exit_only_at_end',
conversion: {
filters: [],
bytecode: [],
events: [eventNameFilter('conversion_event')],
} as any,
}
)
await triggerWorkflow(createGlobals())
await expectParked()
// The conversion event fires during the delay — it must not pull the job out early.
await matcher.processBatch([createGlobals({ event: 'conversion_event' })])
const jobs = await queryCyclotronJobs()
expect(jobs.every((j: any) => j.status === 'available' && new Date(j.scheduled) > new Date())).toBe(true)
expect(mockFetch).not.toHaveBeenCalled()
})
it('exits the workflow when an exit_on_conversion goal fires during a delay', async () => {
// Same shape, but the workflow exits on conversion. The conversion event during the delay
// SHOULD resume the job — to exit it early — but must still not run the next step (fetch).
await createWorkflowFlow(
{
actions: {
trigger: trigger(),
delay: { type: 'delay', config: { delay_duration: '5m' } },
after_delay: fetchAction('https://example.com/after-delay'),
exit: exitAction(),
},
edges: [
{ from: 'trigger', to: 'delay', type: 'continue' },
{ from: 'delay', to: 'after_delay', type: 'continue' },
{ from: 'after_delay', to: 'exit', type: 'continue' },
],
},
{
exitCondition: 'exit_on_conversion',
conversion: {
filters: [],
bytecode: [],
events: [eventNameFilter('conversion_event')],
} as any,
}
)
await triggerWorkflow(createGlobals())
await expectParked()
// The conversion event fires during the delay — the workflow must exit early.
await matcher.processBatch([createGlobals({ event: 'conversion_event' })])
await waitForExpect(async () => {
const jobs = await queryCyclotronJobs()
expect(jobs.some((j: any) => ['completed', 'failed', 'canceled'].includes(j.status))).toBe(true)
}, 10000)
// Exited on conversion — the after-delay step never ran.
expect(mockFetch).not.toHaveBeenCalled()
})
it('wakes a parked condition wait on a person-property change with no analytics event', async () => {
// The core of removing the 10-minute poll: a property change that produces no analytics
// event (only a clickhouse_person mutation) must still wake the wait. The matcher's
// person stream synthesizes a $person_updated globals carrying the new person.properties,
// which the wait's property condition then matches.
await createWaitUntilWorkflow({
// Person-property condition only — never satisfied by the trigger event, so it parks.
condition: { filters: personPropertyConditionFilters('plan', 'enterprise') },
max_wait_duration: '5m',
})
// person is not persisted in the job state — the worker re-resolves it from the person store
// on dequeue (CyclotronPerson.id = person.uuid). Without a resolvable person the re-parked job
// stores person_id = null and the person-stream matcher can never find it. Production resolves
// person_id during ingestion before the job parks; mirror that so the parked job carries
// person_id = 'uuid' — the same id the clickhouse_person mutation below targets. The resolved
// person has no `plan`, so the condition still fails on entry and the job parks as expected.
mockPersonRepo.fetchPersonsByDistinctIds.mockResolvedValue([
{
id: '1',
uuid: 'uuid',
team_id: team.id,
properties: { email: '[email protected]' },
properties_last_updated_at: {},
properties_last_operation: null,
created_at: DateTime.utc(),
version: 1,
is_identified: true,
is_user_id: null,
last_seen_at: null,
distinct_id: 'distinct_id',
},
])
await triggerWorkflow(createGlobals())
await expectParked()
// A clickhouse_person mutation for that person — the person now matches `plan=enterprise`.
// No analytics event is involved; this comes straight off the person topic.
const personMessage = {
value: Buffer.from(
JSON.stringify({
id: 'uuid',
team_id: team.id,
properties: JSON.stringify({ email: '[email protected]', plan: 'enterprise' }),
is_deleted: 0,
is_identified: 1,
created_at: '2024-09-03 09:00:00.000',
timestamp: '2024-09-03 09:00:00.000',
version: 2,
})
),
}
const personGlobals = await matcher._parsePersonBatch([personMessage as any])
await matcher.processBatch(personGlobals)
await waitForExpect(() => {
expect(mockFetch).toHaveBeenCalledTimes(1)
}, 10000)
expect(mockFetch).toHaveBeenCalledWith('https://example.com/condition-matched', expect.anything())
})
// Wait conditions authored as HogQL expressions must be woken by the streams, not only by the
// 10-minute poll — otherwise removing the poll would silently strand them. These three cover
// the distinct HogQL condition shapes real production flows use; each bytecode is exactly what
// the Django serializer compiles for that expression (verified via compile_filters_bytecode).
// If HogQL compilation changes, or a stream stops carrying what the bytecode reads (e.g. the
// person wake dropping properties), one of these fails.
it('wakes a HogQL event-name wait condition via the events stream', async () => {
// HogQL: event = 'billing_added'
await createWaitUntilWorkflow({
condition: {
filters: {
bytecode: ['_H', 1, 32, 'billing_added', 32, 'event', 1, 1, 11],
properties: [{ type: 'hogql', key: "event = 'billing_added'" }],
},
},
max_wait_duration: '5m',
})
await triggerWorkflow(createGlobals({ event: 'custom_trigger', properties: {} }))
await expectParked()
await matcher.processBatch([createGlobals({ event: 'billing_added', properties: {} })])
await waitForExpect(() => {
expect(mockFetch).toHaveBeenCalledTimes(1)
}, 10000)
expect(mockFetch).toHaveBeenCalledWith('https://example.com/condition-matched', expect.anything())
})
it('wakes a HogQL event+property wait condition only on the matching event and property', async () => {
// HogQL: event = 'cal_booking' and properties.trigger_event = 'BOOKING_CREATED'
await createWaitUntilWorkflow({
condition: {
filters: {
bytecode: [
'_H',
1,
32,
'cal_booking',
32,
'event',
1,
1,
11,
32,
'BOOKING_CREATED',
32,
'trigger_event',
32,
'properties',
1,
2,
11,
3,
2,
],
properties: [
{
type: 'hogql',
key: "event = 'cal_booking' and properties.trigger_event = 'BOOKING_CREATED'",
},
],
},
},
max_wait_duration: '5m',
})
await triggerWorkflow(createGlobals({ event: 'custom_trigger', properties: {} }))
await expectParked()
// Right event, wrong property — must not wake.
await matcher.processBatch([
createGlobals({ event: 'cal_booking', properties: { trigger_event: 'OTHER' } }),
])
const jobs = await queryCyclotronJobs()
expect(jobs.every((j: any) => j.status === 'available' && new Date(j.scheduled) > new Date())).toBe(true)
expect(mockFetch).not.toHaveBeenCalled()
// Right event and property — wakes.
await matcher.processBatch([
createGlobals({ event: 'cal_booking', properties: { trigger_event: 'BOOKING_CREATED' } }),
])
await waitForExpect(() => {
expect(mockFetch).toHaveBeenCalledTimes(1)
}, 10000)
expect(mockFetch).toHaveBeenCalledWith('https://example.com/condition-matched', expect.anything())
})
it('wakes a HogQL dynamic-key person-property wait condition via the person stream', async () => {
// HogQL: person.properties[concat(toString(person.properties.shop_id), '_billing_status')] = 'active'
await createWaitUntilWorkflow({
condition: {
filters: {
bytecode: [
'_H',
1,
32,
'active',
32,
'properties',
32,
'person',
1,
2,
32,
'shop_id',
32,
'properties',
32,
'person',
1,
3,
2,
'toString',
1,
32,
'_billing_status',
2,
'concat',
2,
45,
11,
],
properties: [
{
type: 'hogql',
key: "person.properties[concat(toString(person.properties.shop_id), '_billing_status')] = 'active'",
},
],
},
},
max_wait_duration: '5m',
})
// Parked person has shop_id but no billing status yet, so the condition is false and it parks.
mockPersonRepo.fetchPersonsByDistinctIds.mockResolvedValue([
{
id: '1',
uuid: 'uuid',
team_id: team.id,
properties: { shop_id: '42' },
properties_last_updated_at: {},
properties_last_operation: null,
created_at: DateTime.utc(),
version: 1,
is_identified: true,
is_user_id: null,
last_seen_at: null,
distinct_id: 'distinct_id',
},
])
await triggerWorkflow(createGlobals())
await expectParked()
// clickhouse_person mutation sets 42_billing_status = 'active' with no analytics event.
const personMessage = {
value: Buffer.from(
JSON.stringify({
id: 'uuid',
team_id: team.id,
properties: JSON.stringify({ shop_id: '42', '42_billing_status': 'active' }),
is_deleted: 0,
is_identified: 1,
created_at: '2024-09-03 09:00:00.000',
timestamp: '2024-09-03 09:00:00.000',
version: 2,
})
),
}
const personGlobals = await matcher._parsePersonBatch([personMessage as any])
await matcher.processBatch(personGlobals)
await waitForExpect(() => {
expect(mockFetch).toHaveBeenCalledTimes(1)
}, 10000)
expect(mockFetch).toHaveBeenCalledWith('https://example.com/condition-matched', expect.anything())
})
// A parked wait keyed on an anonymous person can't be woken by the survivor's updates after a
// merge — the survivor's person/event updates arrive under a new person_id the wait doesn't
// reference. The matcher re-keys the wait off the clickhouse_person_distinct_id repoint so it
// follows the survivor. These two tests cover the re-key and the wake it enables.
// Factories (not eager consts): they read `team`, which is only assigned in beforeEach.
const anonPersonRow = (): any => ({
id: '1',
uuid: 'old-uuid',
team_id: team.id,
properties: { email: '[email protected]' },
properties_last_updated_at: {},
properties_last_operation: null,
created_at: DateTime.utc(),
version: 1,
is_identified: true,
is_user_id: null,
last_seen_at: null,
distinct_id: 'distinct_id',
})
const distinctIdMoveMessage = (overrides: Record<string, any> = {}): any => ({
value: Buffer.from(
JSON.stringify({
team_id: team.id,
distinct_id: 'distinct_id',
person_id: 'new-uuid',
version: 2,
is_deleted: 0,
...overrides,
})
),
})
// The post-merge survivor the repoint points at, now carrying the merged plan=enterprise. The
// re-keyed job resolves by this survivor personId (personIdRepointed), not the repointed distinct_id.
const survivorPersonRow = (): any => ({
id: '2',
uuid: 'new-uuid',
team_id: team.id,
properties: { email: '[email protected]', plan: 'enterprise' },
properties_last_updated_at: {},
properties_last_operation: null,
created_at: DateTime.utc(),
version: 2,
is_identified: true,
is_user_id: null,
last_seen_at: null,
})
it('re-keys a wait onto the survivor after a merge repoints its distinct_id, so the survivor wakes it', async () => {
await createWaitUntilWorkflow({
condition: { filters: personPropertyConditionFilters('plan', 'enterprise') },
max_wait_duration: '5m',
})
// Parked keyed on the anon person ('old-uuid'), which has no `plan`, so the wait parks on entry.
mockPersonRepo.fetchPersonsByDistinctIds.mockResolvedValue([anonPersonRow()])
await triggerWorkflow(createGlobals())
await expectParked()
// The merge repoints 'distinct_id' onto survivor 'new-uuid' (version > 0). The matcher re-keys the
// parked wait onto the survivor and wakes it (scheduled = now). The worker then resolves the job by
// the survivor personId — not the repointed distinct_id — re-checks the condition against the
// survivor's plan=enterprise, and advances down the matched branch, firing the fetch.
mockPersonRepo.fetchPersonsByPersonIds.mockResolvedValue([survivorPersonRow()])
await matcher.processMoveBatch(matcher._parsePersonDistinctIdBatch([distinctIdMoveMessage() as any]))
await waitForExpect(() => {
expect(mockFetch).toHaveBeenCalledTimes(1)
}, 10000)
expect(mockFetch).toHaveBeenCalledWith('https://example.com/condition-matched', expect.anything())
})
it('anchors and wakes a wait that parked before its distinct_id had a person', async () => {
// Production shape: an event arrives for a distinct_id with no person yet, so the wait parks
// with person_id NULL. Person wakes are keyed on person_id alone, so nothing can address that
// job — before this was fixed, only the 10-minute polling re-check ever advanced it.
await createWaitUntilWorkflow({
condition: { filters: personPropertyConditionFilters('plan', 'enterprise') },
max_wait_duration: '5m',
})
mockPersonRepo.fetchPersonsByDistinctIds.mockResolvedValue([])
await triggerWorkflow(createGlobals())
await expectParked()
// The premise of the test: no anchor to wake.
const parked = await cyclotronPool.query(
`SELECT person_id FROM cyclotron_jobs WHERE ${statusColumn} = 'available'`
)
expect(parked.rows).toHaveLength(1)
expect(parked.rows[0].person_id).toBeNull()
// The distinct_id acquires a person for the first time (version 0), and that person already
// satisfies the condition. The matcher fills the missing anchor and wakes the wait, which then
// resolves by personId and advances down the matched branch.
mockPersonRepo.fetchPersonsByPersonIds.mockResolvedValue([survivorPersonRow()])
await matcher.processMoveBatch(
matcher._parsePersonDistinctIdBatch([distinctIdMoveMessage({ version: 0 }) as any])
)
await waitForExpect(() => {
expect(mockFetch).toHaveBeenCalledTimes(1)
}, 10000)
expect(mockFetch).toHaveBeenCalledWith('https://example.com/condition-matched', expect.anything())
})
it('anchors a wait whose condition is still false, so a later person update can wake it', async () => {
// The shape the affected production runs actually take, and the one the test above does not
// cover: the person exists by the time the anchor is filled, but does not satisfy the condition
// yet. So the fill wakes the wait, the re-check fails, and it re-parks — this time WITH an
// anchor. The property is then set, and that person update has to be able to find the job.
// Without the anchor there is no key for the person stream to match on and only the polling
// re-check would ever advance it.
await createWaitUntilWorkflow({
condition: { filters: personPropertyConditionFilters('plan', 'enterprise') },
max_wait_duration: '5m',
})
mockPersonRepo.fetchPersonsByDistinctIds.mockResolvedValue([])
await triggerWorkflow(createGlobals())
await expectParked()
const beforeFill = await cyclotronPool.query(
`SELECT person_id FROM cyclotron_jobs WHERE ${statusColumn} = 'available'`
)
expect(beforeFill.rows[0].person_id).toBeNull()
// First mapping arrives. The person exists now but has no `plan`, so the condition is false.
const personWithoutPlan = { ...survivorPersonRow(), properties: { email: '[email protected]' } }
mockPersonRepo.fetchPersonsByPersonIds.mockResolvedValue([personWithoutPlan])
await matcher.processMoveBatch(
matcher._parsePersonDistinctIdBatch([distinctIdMoveMessage({ version: 0 }) as any])
)
// It re-parked rather than advancing, and it now carries the anchor.
await waitForExpect(async () => {
const afterFill = await cyclotronPool.query(
`SELECT person_id FROM cyclotron_jobs WHERE ${statusColumn} = 'available'`
)
expect(afterFill.rows).toHaveLength(1)
expect(afterFill.rows[0].person_id).toBe('new-uuid')
}, 10000)
expect(mockFetch).not.toHaveBeenCalled()
// Now the property is set. This is a person mutation with no analytics event, so it can only be
// matched on person_id — the anchor written above is what makes it findable.
const personMessage = {
value: Buffer.from(
JSON.stringify({
id: 'new-uuid',
team_id: team.id,
properties: JSON.stringify({ email: '[email protected]', plan: 'enterprise' }),
is_deleted: 0,
is_identified: 1,
created_at: '2024-09-03 09:00:00.000',
timestamp: '2024-09-03 09:00:00.000',
version: 3,
})
),
}
mockPersonRepo.fetchPersonsByPersonIds.mockResolvedValue([survivorPersonRow()])
await matcher.processBatch(await matcher._parsePersonBatch([personMessage as any]))
await waitForExpect(() => {
expect(mockFetch).toHaveBeenCalledTimes(1)
}, 10000)
expect(mockFetch).toHaveBeenCalledWith('https://example.com/condition-matched', expect.anything())
})
it('wakes a parked wait from a cdp_internal_events signal with no analytics event', async () => {
// CDP-generated signals (e.g. $insight_alert_firing) arrive on cdp_internal_events and never
// hit the analytics events topic. The matcher parses them via _parseInternalEventsBatch and
// wakes parked waits whose "events to wait for" name the signal, matched by distinct_id.
await createWaitUntilWorkflow({
// Property condition never matches the trigger event, so the job parks until the signal.
condition: { filters: HOG_FILTERS_EXAMPLES.elements_text_filter.filters },
events: [eventNameFilter('$insight_alert_firing')],
max_wait_duration: '5m',
})
await triggerWorkflow(createGlobals())
await expectParked()
// A raw cdp_internal_events message for this person's distinct_id — no analytics event.
const internalEventMessage = {
value: Buffer.from(
JSON.stringify({
team_id: team.id,
event: {
uuid: new UUIDT().toString(),
event: '$insight_alert_firing',
distinct_id: 'distinct_id',
properties: {},
timestamp: '2024-09-03T09:00:00Z',
},
})
),
}
const internalGlobals = await matcher._parseInternalEventsBatch([internalEventMessage as any])
await matcher.processBatch(internalGlobals)
await waitForExpect(() => {
expect(mockFetch).toHaveBeenCalledTimes(1)
}, 10000)
expect(mockFetch).toHaveBeenCalledWith('https://example.com/condition-matched', expect.anything())
})
it('counts an event-based conversion exactly once per run even when the event fires repeatedly', async () => {
// Regression guard for conversion over-counting on measurement-only flows. The run stays
// parked in the delay (exit_only_at_end), and the conversion event fires across three
// separate matcher batches. The matcher must record exactly ONE `conversion` metric for
// the run (deduped via conversionCounted), and must never wake the job.
await createWorkflowFlow(
{
actions: {
trigger: trigger(),
delay: { type: 'delay', config: { delay_duration: '5m' } },
after_delay: fetchAction('https://example.com/after-delay'),
exit: exitAction(),
},
edges: [
{ from: 'trigger', to: 'delay', type: 'continue' },
{ from: 'delay', to: 'after_delay', type: 'continue' },
{ from: 'after_delay', to: 'exit', type: 'continue' },
],
},
{
exitCondition: 'exit_only_at_end',
conversion: {
filters: [],
bytecode: [],
events: [eventNameFilter('conversion_event')],
} as any,
}
)
await triggerWorkflow(createGlobals())
await expectParked()
const conversionCount = (): number =>
mockProducerObserver
.getProducedKafkaMessagesForTopic(KAFKA_APP_METRICS_2)
.filter((m: any) => m.value.app_source === 'hog_flow')
.filter((m: any) => m.value.metric_name === 'conversion')
.reduce((sum: number, m: any) => sum + m.value.count, 0)
// First match counts the conversion once.
await matcher.processBatch([createGlobals({ event: 'conversion_event' })])
await waitForExpect(() => {
expect(conversionCount()).toBe(1)
}, 5000)
// The same conversion event firing again must NOT increment the count — the run already converted.
await matcher.processBatch([createGlobals({ event: 'conversion_event' })])
await matcher.processBatch([createGlobals({ event: 'conversion_event' })])
expect(conversionCount()).toBe(1)
// Measurement-only: the run stays parked and the after-delay step never runs early.
const jobs = await queryCyclotronJobs()
expect(jobs.every((j: any) => j.status === 'available' && new Date(j.scheduled) > new Date())).toBe(true)
expect(mockFetch).not.toHaveBeenCalled()
})
})
describe('wait_until_time_window: window in the future', () => {
beforeEach(async () => {
// A fixed daily window is "open" during its own minutes every day, so any
// run landing inside it (e.g. CI at 09:5x UTC for a 23:5x UTC+14 window)
// executes immediately instead of rescheduling. Derive the window a few
// minutes ahead of now so it is always strictly in the future.
const now = DateTime.utc()
const windowStart = now.plus({ minutes: 10 }).toFormat('HH:mm')
const windowEnd = now.plus({ minutes: 20 }).toFormat('HH:mm')
await createWorkflow({
actions: {
trigger: trigger(),
wait_window: {
type: 'wait_until_time_window',
config: { timezone: 'UTC', day: 'any', time: [windowStart, windowEnd] },
},
function_1: fetchAction('https://example.com/after-time-window'),
exit: exitAction(),
},
edges: [
{ from: 'trigger', to: 'wait_window', type: 'continue' },
{ from: 'wait_window', to: 'function_1', type: 'continue' },
{ from: 'function_1', to: 'exit', type: 'continue' },
],
})
globals = createGlobals()
})
it('parks until the window opens and does not advance early on a premature resume', async () => {
await triggerWorkflow(globals)
// Job should be rescheduled to the future time window.
await waitForExpect(async () => {
const jobs = await queryCyclotronJobs()
const rescheduled = jobs.filter(
(j: any) => j.status === 'available' && new Date(j.scheduled) > new Date()
)
expect(rescheduled.length).toBe(1)
}, 5000)
expect(mockFetch).not.toHaveBeenCalled()
// A premature resume (the window is still in the future) must re-park, not advance to the
// next step. The handler stays at the wait_until_time_window step and reschedules, so the
// step that follows the window never runs early.
await cyclotronPool.query(`UPDATE cyclotron_jobs SET scheduled = NOW() WHERE ${statusColumn} = 'available'`)
await waitForExpect(async () => {
const jobs = await queryCyclotronJobs()
expect(jobs.some((j: any) => j.status === 'available' && new Date(j.scheduled) > new Date())).toBe(true)
}, 10000)
expect(mockFetch).not.toHaveBeenCalled()
})
})
describe('wait_until_time_window: window currently open', () => {
it('advances through the window and runs the next step', async () => {
// day: 'any', time: 'any' is always open, so the step must advance and run the next action
// instead of parking forever.
await createWorkflow({
actions: {
trigger: trigger(),
wait_window: {
type: 'wait_until_time_window',
config: { timezone: 'UTC', day: 'any', time: 'any' },
},
function_1: fetchAction('https://example.com/window-open'),
exit: exitAction(),
},
edges: [
{ from: 'trigger', to: 'wait_window', type: 'continue' },
{ from: 'wait_window', to: 'function_1', type: 'continue' },
{ from: 'function_1', to: 'exit', type: 'continue' },
],
})
await triggerWorkflow(createGlobals())
await waitForExpect(() => {
expect(mockFetch).toHaveBeenCalledTimes(1)
}, 10000)
expect(mockFetch).toHaveBeenCalledWith('https://example.com/window-open', expect.anything())
})
})
describe('fetch failure with retries', () => {
beforeEach(async () => {
await createWorkflow({
actions: {
trigger: trigger(),
function_1: fetchAction('https://example.com/failing-endpoint'),
exit: exitAction(),
},
edges: [
{ from: 'trigger', to: 'function_1', type: 'continue' },
{ from: 'function_1', to: 'exit', type: 'continue' },
],
})
mockFetch.mockResolvedValue({
status: 500,
headers: {},
json: () => Promise.resolve({ error: 'Server error' }),
text: () => Promise.resolve(JSON.stringify({ error: 'Server error' })),
dump: () => Promise.resolve(),
})
globals = createGlobals()
})
it('should retry the fetch and eventually complete with error', async () => {
await triggerWorkflow(globals)
// Hogflow function actions retry fetch within a single execution cycle.
// We expect at least 2 calls (initial + retry) before the workflow completes.
await waitForExpect(() => {
expect(mockFetch.mock.calls.length).toBeGreaterThanOrEqual(2)
}, 15000)
// Verify the workflow eventually reaches a terminal state
await waitForExpect(async () => {
const jobs = await queryCyclotronJobs()
const terminal = jobs.filter(
(j: any) => j.status === 'completed' || j.status === 'failed' || j.status === 'canceled'
)
expect(terminal.length).toBeGreaterThanOrEqual(1)
}, 5000)
})
})
describe('person data survives v2 round-trip', () => {
beforeEach(async () => {
const person: InternalPersonWithDistinctId = {
id: '1',
uuid: 'dd3d6f80-60ad-45c3-bd61-e2300f2ba7e1',
team_id: team.id,
properties: { email: '[email protected]', name: 'Test User', plan: 'enterprise' },
properties_last_updated_at: {},
properties_last_operation: null,
created_at: DateTime.utc(),
version: 1,
is_identified: true,
is_user_id: null,
last_seen_at: null,
distinct_id: 'test-distinct-id',
}
mockPersonRepo.fetchPersonsByDistinctIds.mockResolvedValue([person])
await insertHogFunctionTemplate(hub.postgres, {
id: 'template-workflows-e2e-person',
name: 'Workflows E2E Person',
code: `fetch(inputs.url, { 'method': 'POST', 'body': inputs.body });`,
inputs_schema: [
{ key: 'url', type: 'string', required: true },
{ key: 'body', type: 'string', required: true },
],
})
await createWorkflow({
actions: {
trigger: trigger(),
function_1: {
type: 'function',
config: {
template_id: 'template-workflows-e2e-person',
inputs: {
url: { value: 'https://example.com/person-test' },
body: {
value: '{person.properties.email}',
bytecode: ['_H', 1, 32, 'email', 32, 'properties', 32, 'person', 1, 3, 38],
},
},
},
},
exit: exitAction(),
},
edges: [
{ from: 'trigger', to: 'function_1', type: 'continue' },
{ from: 'function_1', to: 'exit', type: 'continue' },
],
})
globals = createGlobals({ distinct_id: 'test-distinct-id' })
})
it('should load person and pass properties to function action', async () => {
await triggerWorkflow(globals)
await waitForExpect(() => {
expect(mockFetch).toHaveBeenCalledTimes(1)
}, 10000)
// The fetch body should contain the person's email, proving person data
// was loaded by the worker after deserializing the v2 job
expect(mockFetch).toHaveBeenCalledWith(
'https://example.com/person-test',
expect.objectContaining({
body: '[email protected]',
method: 'POST',
})
)
})
})
describe('batch workflow: event.distinct_id resolved at the worker', () => {
// A batch-triggered workflow enqueues invocations with personId but an empty
// event.distinct_id (blast radius returns UUIDs only). The cyclotron worker is
// responsible for resolving one distinct_id per person during its existing
// postgres lookup and backfilling state.event.distinct_id — otherwise capture-based
// templates defaulting to {event.distinct_id} would silently mint new person profiles.
const personUuid = 'aaaaaaaa-1111-1111-1111-111111111111'
const personDistinctId = 'batch-person-distinct-id'
let hogFlow: HogFlow
beforeEach(async () => {
const person = {
id: '2',
uuid: personUuid,
team_id: team.id,
properties: { email: '[email protected]', plan: 'enterprise' },
properties_last_updated_at: {},
properties_last_operation: null,
created_at: DateTime.utc(),
version: 1,
is_identified: true,
is_user_id: null,
last_seen_at: null,
}
mockPersonRepo.fetchPersonsByDistinctIds.mockResolvedValue([{ ...person, distinct_id: personDistinctId }])
mockPersonRepo.fetchPersonsByPersonIds.mockResolvedValue([person])
mockPersonRepo.fetchDistinctIdsForPersons.mockResolvedValue({
[person.id]: [personDistinctId],
})
await insertHogFunctionTemplate(hub.postgres, {
id: 'template-workflows-e2e-batch-distinct-id',
name: 'Workflows E2E Batch Distinct Id',
code: `fetch(inputs.url, { 'method': 'POST', 'body': inputs.distinct_id });`,
inputs_schema: [
{ key: 'url', type: 'string', required: true },
{ key: 'distinct_id', type: 'string', required: true },
],
})
hogFlow = await createWorkflowFlow({
actions: {
trigger: {
type: 'trigger',
config: {
// 'batch' trigger so the events consumer skips this workflow —
// it only fires when an invocation is queued directly.
type: 'batch',
filters: { properties: [{ key: 'plan', value: 'enterprise', type: 'person' }] },
},
},
function_1: {
type: 'function',
config: {
template_id: 'template-workflows-e2e-batch-distinct-id',
inputs: {
url: { value: 'https://example.com/batch-distinct-id' },
distinct_id: {
value: '{event.distinct_id}',
bytecode: ['_H', 1, 32, 'distinct_id', 32, 'event', 1, 2, 38],
},
},
},
},
exit: exitAction(),
},
edges: [
{ from: 'trigger', to: 'function_1', type: 'continue' },
{ from: 'function_1', to: 'exit', type: 'continue' },
],
})
})
it("backfills {event.distinct_id} from the person's distinct_id at dequeue", async () => {
await triggerBatchWorkflow(hogFlow, personUuid)
await waitForExpect(() => {
expect(mockFetch).toHaveBeenCalledTimes(1)
}, 5000)
// The hog template's `{event.distinct_id}` resolved at runtime to the value
// the worker backfilled — proving the full chain: postgres lookup → CyclotronPerson.distinct_id
// → state.event.distinct_id mutation → hog input resolution.
expect(mockFetch).toHaveBeenCalledWith(
'https://example.com/batch-distinct-id',
expect.objectContaining({
body: personDistinctId,
method: 'POST',
})
)
})
it('does NOT call fetchDistinctIdsForPersons for event-triggered invocations', async () => {
// Regression guard for the optimization: when event.distinct_id is set going in,
// the persons-manager uses the by-distinct_id lookup which returns the distinct_id
// as part of the lookup key — no separate fetchDistinctIdsForPersons RPC needed.
// Without this guard, the by-person_id path could accidentally be used for event
// triggers and pay an unnecessary postgres round-trip per worker tick.
await insertHogFunctionTemplate(hub.postgres, {
id: 'template-workflows-e2e-event-no-extra-rpc',
name: 'Workflows E2E Event No Extra RPC',
code: `fetch(inputs.url, { 'method': 'POST' });`,
inputs_schema: [{ key: 'url', type: 'string', required: true }],
})
await createWorkflow({
actions: {
trigger: trigger(),
function_1: fetchAction('https://example.com/event-trigger-no-extra-rpc'),
exit: exitAction(),
},
edges: [
{ from: 'trigger', to: 'function_1', type: 'continue' },
{ from: 'function_1', to: 'exit', type: 'continue' },
],
})
mockPersonRepo.fetchDistinctIdsForPersons.mockClear()
await triggerWorkflow(createGlobals({ distinct_id: personDistinctId }))
await waitForExpect(() => {
expect(mockFetch).toHaveBeenCalled()
}, 5000)
expect(mockPersonRepo.fetchDistinctIdsForPersons).not.toHaveBeenCalled()
})
})
describe('data warehouse table trigger workflow', () => {
const TABLE_NAME = 'postgres.orders'
// Row-scoped trigger that always matches (return-true bytecode), so the row payload alone
// decides whether the flow fires.
const dwhTrigger = (tableName: string) =>
({
type: 'trigger' as const,
config: {
type: 'data-warehouse-table',
table_name: tableName,
filters: { properties: [], bytecode: ['_h', 29] },
},
}) as any
beforeEach(async () => {
await createWorkflow({
actions: {
trigger: dwhTrigger(TABLE_NAME),
function_1: fetchAction('https://example.com/warehouse-row'),
exit: exitAction(),
},
edges: [
{ from: 'trigger', to: 'function_1', type: 'continue' },
{ from: 'function_1', to: 'exit', type: 'continue' },
],
})
})
it('runs the workflow end-to-end when a row syncs into the matching table', async () => {
await triggerDwhWorkflow(createDwhGlobals(TABLE_NAME, { order_id: 42 }))
await waitForExpect(() => {
expect(mockFetch).toHaveBeenCalledTimes(1)
}, 10000)
expect(mockFetch).toHaveBeenCalledWith(
'https://example.com/warehouse-row',
expect.objectContaining({ method: 'POST' })
)
})
it('does not run the workflow when the synced row belongs to a different table', async () => {
await triggerDwhWorkflow(createDwhGlobals('postgres.other_table', { order_id: 1 }))
// Give the worker a chance to (not) pick anything up before asserting it stayed idle.
await waitForExpect(async () => {
const jobs = await queryCyclotronJobs()
expect(jobs).toHaveLength(0)
}, 3000)
expect(mockFetch).not.toHaveBeenCalled()
})
it('does not look up a person for warehouse rows (no synthetic distinct_id resolution)', async () => {
// Regression guard: warehouse rows carry a made-up event.distinct_id that evals truthy.
// The worker must skip the person lookup for data-warehouse-table triggers so we don't
// pay a person round-trip (or accidentally resolve a bogus person) per synced row.
const personSpy = jest.spyOn((hogflowWorker as any).personsManager, 'getCyclotronPerson')
await triggerDwhWorkflow(createDwhGlobals(TABLE_NAME, { order_id: 7 }))
await waitForExpect(() => {
expect(mockFetch).toHaveBeenCalledTimes(1)
}, 10000)
expect(personSpy).not.toHaveBeenCalled()
})
})
describe('posthog_ticket_tags input resolves templated values per element', () => {
// Regression guard for the templating opt-in in products/cdp/backend/validation.py.
// Reproduces the real user-reported case: a ticket tag set to
// `zendesk/{variables.zendesk_ticketid}` used to ship to the runtime as
// a literal placeholder string (because the type wasn't on the bytecode
// opt-in list), so the ticket ended up tagged with the raw template text
// instead of "zendesk/12345". The fix puts the per-element bytecode that
// `generate_template_bytecode` already emits for lists into a shape that
// `formatHogInput` can walk element-by-element.
beforeEach(async () => {
await insertHogFunctionTemplate(hub.postgres, {
id: 'template-workflows-e2e-tags',
name: 'Workflows E2E Tags',
code: `fetch(inputs.url, { 'method': 'POST', 'body': jsonStringify(inputs.tags) });`,
inputs_schema: [
{ key: 'url', type: 'string', required: true },
{ key: 'tags', type: 'posthog_ticket_tags', required: true },
],
})
const hogFlow = new FixtureHogFlowBuilder()
.withTeamId(team.id)
.withStatus('active')
.withWorkflow({
actions: {
trigger: trigger(),
function_1: {
type: 'function',
config: {
template_id: 'template-workflows-e2e-tags',
inputs: {
url: { value: 'https://example.com/tags' },
tags: {
// What the Python serializer now produces for a
// `posthog_ticket_tags` value like
// `["zendesk/{variables.zendesk_ticketid}"]`:
// one outer array, one inner bytecode per element.
// Inner bytecode is a concat of the literal prefix
// and a variables.* field access — same shape Django
// emits today for other templated string inputs
// (cf. the production `text` input bytecode).
value: ['zendesk/{variables.zendesk_ticketid}'],
templating: 'hog',
bytecode: [
[
'_H',
1,
32,
'zendesk/',
32,
'zendesk_ticketid',
32,
'variables',
1,
2,
2,
'concat',
2,
],
],
},
},
},
},
exit: exitAction(),
},
edges: [
{ from: 'trigger', to: 'function_1', type: 'continue' },
{ from: 'function_1', to: 'exit', type: 'continue' },
],
})
.build()
// Workflow-defined variable with a default — populated into state.variables
// by createHogFlowInvocation, so the function action sees it via globals.variables.
// In a real workflow the value would be set by an earlier `Get ticket` action's
// output_variable; for this test we pre-seed via the default to keep it focused.
hogFlow.variables = [
{ key: 'zendesk_ticketid', type: 'string', label: 'Zendesk ticket id', default: '12345' },
]
await insertHogFlow(hub.postgres, hogFlow)
globals = createGlobals()
})
it('resolves zendesk/{variables.zendesk_ticketid} per element', async () => {
await triggerWorkflow(globals)
await waitForExpect(() => {
expect(mockFetch).toHaveBeenCalledTimes(1)
}, 10000)
// The body proves the full chain end-to-end: per-element bytecode →
// formatHogInput recurses into the list → executes against globals
// populated with workflow variables → concat produces "zendesk/12345".
// Pre-fix behaviour would emit `["zendesk/{variables.zendesk_ticketid}"]`.
expect(mockFetch).toHaveBeenCalledWith(
'https://example.com/tags',
expect.objectContaining({
body: JSON.stringify(['zendesk/12345']),
method: 'POST',
})
)
})
})
describe('create AI task action', () => {
let flowId: string
beforeEach(async () => {
// The real template, not a fixture copy: this block exists to prove the shipped hog
// compiles and drives the registered async function inside a running workflow, which
// is the execution path a template-level test never touches.
await insertHogFunctionTemplate(hub.postgres, {
id: createTaskTemplate.id,
name: createTaskTemplate.name,
code: createTaskTemplate.code,
inputs_schema: createTaskTemplate.inputs_schema,
})
const flow = await createWorkflowFlow({
actions: {
trigger: trigger(),
function_1: {
type: 'function',
config: {
template_id: createTaskTemplate.id,
inputs: {
prompt: { value: 'Investigate the error spike' },
title: { value: 'Error spike' },
non_failure_status_codes: { value: [409] },
},
},
output_variable: { key: 'task' },
},
function_2: fetchAction('https://example.com/task-done'),
exit: exitAction(),
},
edges: [
{ from: 'trigger', to: 'function_1', type: 'continue' },
{ from: 'function_1', to: 'function_2', type: 'continue' },
{ from: 'function_2', to: 'exit', type: 'continue' },
],
})
flowId = flow.id
globals = createGlobals()
})
const runMetricNames = (): string[] =>
mockProducerObserver
.getProducedKafkaMessagesForTopic(KAFKA_APP_METRICS_2)
.filter((m: any) => m.value.app_source === 'hog_flow' && m.value.app_source_id === flowId)
.map((m: any) => m.value.metric_name)
it('creates the task through the real registered async function', async () => {
mockFetch.mockResolvedValue({
status: 201,
headers: { 'Content-Type': 'application/json' },
json: () => Promise.resolve({ id: 'task-1', run_id: 'run-1' }),
text: () => Promise.resolve(JSON.stringify({ id: 'task-1', run_id: 'run-1' })),
dump: () => Promise.resolve(),
} as any)
await triggerWorkflow(globals)
await waitForExpect(() => {
expect(mockFetch).toHaveBeenCalledTimes(1)
}, 10000)
const [url, options] = mockFetch.mock.calls[0]
expect(url).toEqual(`${hub.SITE_URL}/api/projects/${team.id}/workflow_tasks/`)
expect(options.method).toEqual('POST')
// Step-scoped dedupe key: run id + action id, so a second task step in the same run
// would not collide with this one.
expect(parseJSON(options.body)).toEqual({
prompt: 'Investigate the error spike',
title: 'Error spike',
event: expect.objectContaining({
event: '$pageview',
uuid: 'b3a1fe86-b10c-43cc-acaf-d208977608d0',
}),
idempotency_key: expect.stringMatching(/^[0-9a-f-]{36}:function_1:0$/),
})
// The endpoint trusts these claims to resolve the workflow owner, so the token has to
// verify against the shared dev secret with exactly this audience and flow id.
const token = (options.headers['Authorization'] as string).replace('Bearer ', '')
// The literal pins the cross-language contract: Django's TASKS_CREATE_JWT_SECRET dev default
// (posthog/settings/data_stores.py) must match the nodejs one or local runs 401.
// nosemgrep: javascript.jsonwebtoken.security.jwt-hardcode.hardcoded-jwt-secret
const claims = jsonwebtoken.verify(token, 'local-dev-tasks-create-jwt', {
audience: 'posthog:tasks:create',
algorithms: ['HS256'],
}) as jsonwebtoken.JwtPayload
expect(claims.team_id).toEqual(team.id)
expect(claims.hog_flow_id).toEqual(flowId)
// The step parks until Django wakes it with the task's outcome.
let originKey = ''
await waitForExpect(async () => {
const { rows } = await cyclotronPool.query(`SELECT id, status, state FROM cyclotron_jobs`)
expect(rows).toHaveLength(1)
expect(rows[0].status).toEqual('available')
const state = parseJSON(rows[0].state.toString('utf-8')).state
expect(state.currentAction.awaitingResume.key).toEqual(`${rows[0].id}:function_1:0`)
originKey = state.currentAction.awaitingResume.key
}, 10000)
expect(mockFetch).toHaveBeenCalledTimes(1)
expect(runMetricNames()).not.toContain('failed')
// Django reports the run finished: the same internal event it produces, through the
// matcher's own parser, wakes the parked step and the workflow moves on.
const matcher = new CdpHogflowSubscriptionMatcherConsumer({ ...hub }, deps)
const { resumes } = matcher._splitStepResumes([
{
value: Buffer.from(
JSON.stringify({
team_id: team.id,
event: {
uuid: 'a5c2d9e1-3f4b-4c8d-9e0f-1a2b3c4d5e6f',
event: '$workflow_step_resume',
distinct_id: `team_${team.id}`,
properties: {
origin_key: originKey,
status: 'completed',
result: { run_id: 'run-1', final_message: 'Found the cause' },
},
timestamp: '2024-01-01T00:00:00Z',
},
})
),
} as any,
])
await matcher.processStepResumes(resumes)
await waitForExpect(() => {
expect(mockFetch).toHaveBeenCalledTimes(2)
}, 10000)
expect(mockFetch.mock.calls[1][0]).toEqual('https://example.com/task-done')
const logs = mockProducerObserver
.getProducedKafkaMessagesForTopic(KAFKA_LOG_ENTRIES)
.map((m: any) => m.value.message as string)
expect(logs).toContainEqual(expect.stringContaining('[Action:function_1] The task finished'))
expect(runMetricNames()).not.toContain('failed')
})
it('fails the run when the task limit replies 409', async () => {
mockFetch.mockResolvedValue({
status: 409,
headers: { 'Content-Type': 'application/json' },
json: () => Promise.resolve({ detail: 'This workflow already has 5 tasks running' }),
text: () => Promise.resolve(JSON.stringify({ detail: 'This workflow already has 5 tasks running' })),
dump: () => Promise.resolve(),
} as any)
await triggerWorkflow(globals)
await waitForExpect(() => {
expect(runMetricNames()).toContain('failed')
}, 10000)
// 409 is terminal for the step: one request, no retry burning the engine's budget.
const taskCreates = mockFetch.mock.calls.filter(([url]) => (url as string).includes('/workflow_tasks/'))
expect(taskCreates).toHaveLength(1)
})
})
describe('heartbeat during long batches', () => {
let janitor: CyclotronV2Janitor
beforeEach(async () => {
// Restart the worker with a short heartbeat interval so the
// interval fires several times inside the test's 600ms fetch delay.
// Default is 10s — nothing would fire in a test.
await hogflowWorker.stop()
hub.CDP_CYCLOTRON_HEARTBEAT_INTERVAL_MS = 100
hogflowWorker = new CdpCyclotronWorkerHogFlow(hub, deps, hogflowQueue)
await hogflowWorker.start()
// Aggressive stall/poison thresholds so the test observes the
// exact regression the heartbeat guards against: without heartbeats
// firing, a 600ms fetch would exceed the 300ms stall window and
// the janitor would reset the row on the very first runOnce().
// cleanupGraceMs is huge so terminal jobs stay queryable after
// completion — we assert on their final janitor_touch_count.
janitor = new CyclotronV2Janitor({
pool: { dbUrl: CYCLOTRON_NODE_DB_URL },
stallTimeoutMs: 300,
maxTouchCount: 2,
cleanupGraceMs: 99_999_000,
cleanupBatchSize: 100,
cleanupIntervalMs: 60_000,
})
await createWorkflow({
actions: {
trigger: trigger(),
slow_fetch: fetchAction('https://example.com/slow'),
exit: exitAction(),
},
edges: [
{ from: 'trigger', to: 'slow_fetch', type: 'continue' },
{ from: 'slow_fetch', to: 'exit', type: 'continue' },
],
})
globals = createGlobals()
})
afterEach(async () => {
await janitor?.stop()
})
it('heartbeats keep janitor_touch_count at 0 during a slow batch', async () => {
// Fetch takes 600ms — 2x the stallTimeoutMs. The heartbeat
// interval firing every 100ms is the only thing preventing
// last_heartbeat from going stale.
mockFetch.mockImplementation(
() =>
new Promise((resolve) =>
setTimeout(
() =>
resolve({
status: 200,
headers: { 'Content-Type': 'application/json' },
json: () => Promise.resolve({ ok: true }),
text: () => Promise.resolve(JSON.stringify({ ok: true })),
dump: () => Promise.resolve(),
}),
600
)
)
)
await triggerWorkflow(globals)
// Wait until the fetch is in flight, then pass the stall cutoff. Without heartbeats,
// last_heartbeat would be >300ms old and the janitor would mark it stalled.
await waitForExpect(async () => {
const jobs = await queryCyclotronJobs()
expect(jobs.some((j) => j.status === 'running')).toBe(true)
}, 5000)
await new Promise((r) => setTimeout(r, 350))
const midResult = await janitor.runOnce()
expect(midResult.stalled).toBe(0)
expect(midResult.poisoned).toBe(0)
// Wait for the workflow to complete
await waitForExpect(async () => {
const jobs = await queryCyclotronJobs()
expect(jobs.some((j) => j.status === 'completed')).toBe(true)
}, 10000)
// Final check: no row ever accumulated a touch. This is the
// load-bearing assertion — regressions to the setInterval or
// its cleanup would flip this to >= 1.
const jobs = await queryCyclotronJobs()
for (const row of jobs) {
expect(row.janitor_touch_count).toBe(0)
}
})
})
})
// Email queue routing: the email worker reschedules jobs between queue names on the same v2 backend.
describe('Workflows E2E (email queue)', () => {
jest.setTimeout(30000)
let eventsConsumer: CdpEventsConsumer
let hogflowWorker: CdpCyclotronWorkerHogFlow
let emailWorker: CdpCyclotronWorkerEmail
let matcher: CdpHogflowSubscriptionMatcherConsumer | undefined
let hub: Hub
let kafkaProducer: KafkaProducerWrapper
let mockProducerObserver: KafkaProducerObserver
let team: Team
let cyclotronPool: Pool
let deps: ReturnType<typeof createCdpConsumerDeps>
beforeAll(() => {
cyclotronPool = new Pool({ connectionString: CYCLOTRON_NODE_DB_URL })
})
afterAll(async () => {
await cyclotronPool.end()
})
beforeEach(async () => {
MockKafkaProducerWrapper.create = jest.fn((...args) => {
return ActualKafkaProducerWrapper.create(...args)
})
await ensureKafkaTopics(TEST_KAFKA_TOPICS)
await resetTestDatabase()
await cyclotronPool.query('DELETE FROM cyclotron_jobs')
hub = await createHub()
hub.CDP_CYCLOTRON_BATCH_DELAY_MS = 50
// Without a Valkey host the SES rate limiter pool is null and every sending
// limit in this block is silently disabled. Point it at the local test Redis.
hub.SES_RATE_LIMITER_VALKEY_HOST = hub.CDP_REDIS_HOST || '127.0.0.1'
if (hub.CDP_REDIS_PORT) {
hub.SES_RATE_LIMITER_VALKEY_PORT = hub.CDP_REDIS_PORT
}
// `.invalid` domains are NXDOMAIN, everything else resolves as deliverable.
const nxdomain = () => Promise.reject(Object.assign(new Error('queryMx ENOTFOUND'), { code: 'ENOTFOUND' }))
mockDnsResolveMx.mockImplementation((domain: string) =>
domain.endsWith('.invalid') ? nxdomain() : Promise.resolve([{ exchange: 'mx.example.com', priority: 10 }])
)
mockDnsResolve4.mockImplementation((domain: string) =>
domain.endsWith('.invalid') ? nxdomain() : Promise.resolve(['1.2.3.4'])
)
mockDnsResolve6.mockImplementation(() => Promise.resolve([]))
kafkaProducer = await ActualKafkaProducerWrapper.create(hub.KAFKA_CLIENT_RACK)
mockProducerObserver = new KafkaProducerObserver(kafkaProducer)
team = await getFirstTeam(hub.postgres)
mockProducerObserver.resetKafkaProducer()
// Email integration — provider 'maildev' routes to local SMTP (port 1025)
await insertIntegration(hub.postgres, team.id, {
id: 1,
kind: 'email',
config: {
email: '[email protected]',
name: 'Test Sender',
domain: 'posthog.com',
verified: true,
provider: 'maildev',
},
})
// Native-email template that the workflow's email action invokes
await insertHogFunctionTemplate(hub.postgres, {
id: 'template-workflows-e2e-email',
name: 'Workflows E2E Email',
code: `sendEmail(inputs.email)`,
inputs_schema: [
{
type: 'native_email',
key: 'email',
label: 'Email message',
integration: 'email',
required: true,
default: {
to: { email: '', name: '' },
from: { email: '', name: '' },
subject: '',
text: 'Hello!',
html: '<div>Hello!</div>',
},
secret: false,
description: '',
templating: 'liquid',
},
],
})
matcher = undefined
deps = createCdpConsumerDeps(hub, kafkaProducer)
const kafkaQueue = new CyclotronJobQueueKafka(hub.KAFKA_CLIENT_RACK, hub, hub.CONSUMER_BATCH_SIZE)
// Each consumer gets a dedicated CyclotronJobQueuePostgresV2 — sharing one
// across two consumers collides on `this.worker` and the shared pg pool.
// Mirrors the prod deployment model where each capability runs in its own pod.
const eventsProducerQueue = new CyclotronJobQueuePostgresV2(hub.CONSUMER_BATCH_SIZE, hub)
const hogflowConsumerQueue = new CyclotronJobQueuePostgresV2(hub.CONSUMER_BATCH_SIZE, hub)
const emailConsumerQueue = new CyclotronJobQueuePostgresV2(hub.CONSUMER_BATCH_SIZE, hub)
eventsConsumer = new CdpEventsConsumer(hub, deps, {
hogQueue: kafkaQueue,
hogflowQueue: eventsProducerQueue,
})
await Promise.all([kafkaQueue.startAsProducer(), eventsProducerQueue.startAsProducer()])
// Hogflow worker polls jobs with queue_name='hogflow' and re-stamps email
// jobs to queue_name='email' so the email worker picks them up
hogflowWorker = new CdpCyclotronWorkerHogFlow(hub, deps, hogflowConsumerQueue)
await hogflowWorker.start()
// Email worker polls jobs with queue_name='email', sends via EmailService,
// and continues the workflow inline (until it hits a fetch or terminates)
emailWorker = new CdpCyclotronWorkerEmail(hub, deps, emailConsumerQueue)
await emailWorker.start()
})
afterEach(async () => {
await Promise.all([
eventsConsumer?.stop() ?? Promise.resolve(),
hogflowWorker?.stop() ?? Promise.resolve(),
emailWorker?.stop() ?? Promise.resolve(),
matcher?.stop().catch(() => {}) ?? Promise.resolve(),
])
await kafkaProducer.disconnect()
await closeHub(hub)
mockProducerObserver.resetKafkaProducer()
})
async function queryCyclotronJobs(): Promise<any[]> {
const result = await cyclotronPool.query(`SELECT *, status AS status FROM cyclotron_jobs ORDER BY created ASC`)
return result.rows
}
function createGlobals(
overrides: Partial<HogFunctionInvocationGlobals['event']> = {}
): HogFunctionInvocationGlobals {
return createHogExecutionGlobals({
project: { id: team.id } as any,
event: {
uuid: 'b3a1fe86-b10c-43cc-acaf-d208977608d0',
event: '$pageview',
properties: {
$current_url: 'https://posthog.com',
$lib_version: '1.0.0',
},
timestamp: '2024-09-03T09:00:00Z',
...overrides,
} as any,
})
}
// Mirrors what HogFlowSerializer compiles for {events: [{id: <name>}]}: a single
// equality check on the `event` global. The matcher fails closed without bytecode.
const eventNameFilter = (eventName: string) => ({
filters: { events: [{ id: eventName }], bytecode: ['_H', 1, 32, eventName, 32, 'event', 1, 1, 11] as any[] },
})
it('routes the email through the dedicated queue and continues the workflow', async () => {
const hogFlow = new FixtureHogFlowBuilder()
.withTeamId(team.id)
.withStatus('active')
.withExitCondition('exit_only_at_end')
.withWorkflow({
actions: {
trigger: {
type: 'trigger',
config: { type: 'event', filters: HOG_FILTERS_EXAMPLES.no_filters.filters ?? {} },
},
email_1: {
type: 'function_email',
config: {
template_id: 'template-workflows-e2e-email',
inputs: {
email: {
value: {
to: { email: '[email protected]', name: 'Recipient' },
from: { integrationId: 1, email: '[email protected]' },
subject: 'Test Email',
text: 'Test text',
html: '<p>Test html</p>',
},
},
},
},
},
exit: { type: 'exit', config: {} },
},
edges: [
{ from: 'trigger', to: 'email_1', type: 'continue' },
{ from: 'email_1', to: 'exit', type: 'continue' },
],
})
.build()
await insertHogFlow(hub.postgres, hogFlow)
// Trigger the workflow via the events consumer (real Kafka producer + v2 queue)
const { backgroundTask } = await eventsConsumer.processBatch([createGlobals()])
await backgroundTask
// Verify both metric stages fire EXACTLY ONCE — regression guard against
// a double-counting bug where each metric (incl. unrelated ones like the
// exit_node 'succeeded') was being pushed twice per invocation. The
// AppMetricsAggregator dedupes by key in-memory, so we sum `count` across
// all messages rather than counting messages: one push at count=1 looks
// identical to two pushes at count=1 unless we sum.
await waitForExpect(() => {
const sumCounts = (filter: (m: any) => boolean) =>
mockProducerObserver
.getProducedKafkaMessagesForTopic(KAFKA_APP_METRICS_2)
.filter((m: any) => m.value.app_source === 'hog_flow')
.filter(filter)
.reduce((sum: number, m: any) => sum + m.value.count, 0)
expect(sumCounts((m) => m.value.metric_name === 'email_queued')).toBe(1)
expect(sumCounts((m) => m.value.metric_name === 'email_sent')).toBe(1)
// The exit action's 'succeeded' metric has nothing to do with the email
// pipeline — including it locks down that the doubling isn't email-specific.
// (The instance_id matches the action key from FixtureHogFlowBuilder.)
expect(sumCounts((m) => m.value.metric_name === 'succeeded' && m.value.instance_id === 'exit')).toBe(1)
}, 15000)
// Workflow should reach a terminal state once the email worker has continued through exit
await waitForExpect(async () => {
const jobs = await queryCyclotronJobs()
const terminal = jobs.filter(
(j: any) => j.status === 'completed' || j.status === 'failed' || j.status === 'canceled'
)
expect(terminal.length).toBeGreaterThanOrEqual(1)
}, 10000)
})
it('skips a predicted hard bounce before the email queue and completes the workflow', async () => {
// Locks down the pipeline sequencing the unit tests can't: the MX-validation
// skip happens on the hogflow worker BEFORE routeEmailToQueue, so a dead-domain
// recipient must produce no email_queued/email_sent, no billable_invocation,
// exactly one email_bounce_prevented, and a workflow that still runs to exit
// instead of wedging on the email queue.
const hogFlow = new FixtureHogFlowBuilder()
.withTeamId(team.id)
.withStatus('active')
.withExitCondition('exit_only_at_end')
.withWorkflow({
actions: {
trigger: {
type: 'trigger',
config: { type: 'event', filters: HOG_FILTERS_EXAMPLES.no_filters.filters ?? {} },
},
email_1: {
type: 'function_email',
config: {
template_id: 'template-workflows-e2e-email',
inputs: {
email: {
value: {
to: { email: '[email protected]', name: 'Recipient' },
from: { integrationId: 1, email: '[email protected]' },
subject: 'Predicted bounce',
text: 'Should never send',
html: '<p>Should never send</p>',
},
},
},
},
},
exit: { type: 'exit', config: {} },
},
edges: [
{ from: 'trigger', to: 'email_1', type: 'continue' },
{ from: 'email_1', to: 'exit', type: 'continue' },
],
})
.build()
await insertHogFlow(hub.postgres, hogFlow)
const { backgroundTask } = await eventsConsumer.processBatch([createGlobals()])
await backgroundTask
await waitForExpect(() => {
const sumCounts = (filter: (m: any) => boolean) =>
mockProducerObserver
.getProducedKafkaMessagesForTopic(KAFKA_APP_METRICS_2)
.filter((m: any) => m.value.app_source === 'hog_flow')
.filter(filter)
.reduce((sum: number, m: any) => sum + m.value.count, 0)
// Wait for the two positive signals first — once the exit 'succeeded' metric
// has flushed, the absence of email metrics below is meaningful, since
// email_queued would have been emitted earlier in the pipeline.
expect(sumCounts((m) => m.value.metric_name === 'email_bounce_prevented')).toBe(1)
expect(sumCounts((m) => m.value.metric_name === 'succeeded' && m.value.instance_id === 'exit')).toBe(1)
expect(sumCounts((m) => m.value.metric_name === 'email_queued')).toBe(0)
expect(sumCounts((m) => m.value.metric_name === 'email_sent')).toBe(0)
expect(sumCounts((m) => m.value.metric_name === 'email_failed')).toBe(0)
expect(sumCounts((m) => m.value.metric_name === 'billable_invocation')).toBe(0)
}, 15000)
await waitForExpect(async () => {
const jobs = await queryCyclotronJobs()
expect(jobs.filter((j: any) => j.status === 'completed').length).toBeGreaterThanOrEqual(1)
expect(jobs.filter((j: any) => j.status === 'failed').length).toBe(0)
}, 10000)
})
it('sends up to the frequency cap per person, then skips and completes the run', async () => {
// The email step runs on the hogflow worker and again on the email worker. Each pass checks the cap,
// so with a cap of one the first run only sends if the second pass reuses its slot.
await hub.postgres.query(
PostgresUse.COMMON_WRITE,
`INSERT INTO workflows_teamworkflowsconfig
(team_id, capture_workflows_engagement_events, email_tracking_consent_mode,
email_sending_suspension_reason, ses_tenant_sending_status, email_sending_tier,
marketing_frequency_cap_max_messages, marketing_frequency_cap_window_days)
VALUES ($1, false, 'off', '', '', 0, 1, 1)`,
[team.id],
'set-team-frequency-cap'
)
const frequencyCapValkey = createFrequencyCapValkeyPool(hub)!
await deleteKeysWithPrefix(frequencyCapValkey, `@posthog/workflows-frequency-cap/${team.id}/`)
// The cap is per person, and these workers resolve the person from the event's distinct_id.
const person: InternalPersonWithDistinctId = {
id: '1',
uuid: '5b0c4a6e-2f0c-4a43-9c1e-7f6a1d0b9e21',
team_id: team.id,
properties: { email: '[email protected]' },
properties_last_updated_at: {},
properties_last_operation: null,
created_at: DateTime.utc(),
version: 1,
is_identified: true,
is_user_id: null,
last_seen_at: null,
distinct_id: 'capped-distinct-id',
}
const personDeps = {
...deps,
personRepository: {
fetchPerson: jest.fn().mockResolvedValue(person),
fetchPersonsByDistinctIds: jest.fn().mockResolvedValue([person]),
fetchPersonsByPersonIds: jest.fn().mockResolvedValue([person]),
fetchDistinctIdsForPersons: jest.fn().mockResolvedValue({}),
},
}
await Promise.all([hogflowWorker.stop(), emailWorker.stop()])
hogflowWorker = new CdpCyclotronWorkerHogFlow(
hub,
personDeps,
new CyclotronJobQueuePostgresV2(hub.CONSUMER_BATCH_SIZE, hub)
)
emailWorker = new CdpCyclotronWorkerEmail(
hub,
personDeps,
new CyclotronJobQueuePostgresV2(hub.CONSUMER_BATCH_SIZE, hub)
)
await Promise.all([hogflowWorker.start(), emailWorker.start()])
const hogFlow = new FixtureHogFlowBuilder()
.withTeamId(team.id)
.withStatus('active')
.withExitCondition('exit_only_at_end')
.withWorkflow({
actions: {
trigger: {
type: 'trigger',
config: { type: 'event', filters: HOG_FILTERS_EXAMPLES.no_filters.filters ?? {} },
},
email_1: {
type: 'function_email',
config: {
template_id: 'template-workflows-e2e-email',
inputs: {
email: {
value: {
to: { email: '[email protected]', name: 'Recipient' },
from: { integrationId: 1, email: '[email protected]' },
subject: 'Frequency cap',
text: 'Test text',
html: '<p>Test html</p>',
},
},
},
},
},
exit: { type: 'exit', config: {} },
},
edges: [
{ from: 'trigger', to: 'email_1', type: 'continue' },
{ from: 'email_1', to: 'exit', type: 'continue' },
],
})
.build()
await insertHogFlow(hub.postgres, hogFlow)
const { backgroundTask } = await eventsConsumer.processBatch([
createGlobals({ uuid: new UUIDT().toString(), distinct_id: 'capped-distinct-id' } as any),
createGlobals({ uuid: new UUIDT().toString(), distinct_id: 'capped-distinct-id' } as any),
])
await backgroundTask
await waitForExpect(() => {
const sumCounts = (filter: (m: any) => boolean) =>
mockProducerObserver
.getProducedKafkaMessagesForTopic(KAFKA_APP_METRICS_2)
.filter((m: any) => m.value.app_source === 'hog_flow')
.filter(filter)
.reduce((sum: number, m: any) => sum + m.value.count, 0)
expect(sumCounts((m) => m.value.metric_name === 'succeeded' && m.value.instance_id === 'exit')).toBe(2)
expect(sumCounts((m) => m.value.metric_name === 'email_sent')).toBe(1)
expect(sumCounts((m) => m.value.metric_name === 'message_frequency_capped')).toBe(1)
}, 15000)
const jobs = await queryCyclotronJobs()
expect(jobs.filter((j: any) => j.status === 'completed')).toHaveLength(2)
expect(jobs.filter((j: any) => j.status === 'failed')).toHaveLength(0)
})
it('does not emit duplicate Resuming / Executing / pause logs for the email-queue routing reschedule', async () => {
// Email steps reschedule themselves once to switch onto the dedicated email queue
// (see HogFunctionHandler.execute in actions/hog_function.ts). That second dequeue
// continues the *same* action and would otherwise re-emit "Resuming workflow execution
// at Email", "Executing action Email", and a "Workflow will pause until <basically
// now>" line — leaking the internal queue routing into customer-visible logs.
//
// The fix tags the action state with `routingOnlyReschedule: true` on the rescheduling
// dequeue and consumes it on the next dequeue to suppress those three lines. This test
// is the regression guard: trigger → email → exit should produce exactly one trigger
// log, one "Executing action [Action:email_1]" line, one "Email sent" line, and no
// routing-flavored pause / resume noise.
const hogFlow = new FixtureHogFlowBuilder()
.withTeamId(team.id)
.withStatus('active')
.withExitCondition('exit_only_at_end')
.withWorkflow({
actions: {
trigger: {
type: 'trigger',
config: { type: 'event', filters: HOG_FILTERS_EXAMPLES.no_filters.filters ?? {} },
},
email_1: {
type: 'function_email',
config: {
template_id: 'template-workflows-e2e-email',
inputs: {
email: {
value: {
to: { email: '[email protected]', name: 'Recipient' },
from: { integrationId: 1, email: '[email protected]' },
subject: 'Routing-reschedule log test',
text: 'Test text',
html: '<p>Test html</p>',
},
},
},
},
},
exit: { type: 'exit', config: {} },
},
edges: [
{ from: 'trigger', to: 'email_1', type: 'continue' },
{ from: 'email_1', to: 'exit', type: 'continue' },
],
})
.build()
await insertHogFlow(hub.postgres, hogFlow)
const { backgroundTask } = await eventsConsumer.processBatch([createGlobals()])
await backgroundTask
// Wait for the workflow to terminate so all logs from both dequeues have been produced.
await waitForExpect(async () => {
const jobs = await queryCyclotronJobs()
const terminal = jobs.filter(
(j: any) => j.status === 'completed' || j.status === 'failed' || j.status === 'canceled'
)
expect(terminal.length).toBeGreaterThanOrEqual(1)
}, 15000)
// Collect every log entry produced by this hogflow run from the Kafka topic.
const logMessages = mockProducerObserver
.getProducedKafkaMessagesForTopic(KAFKA_LOG_ENTRIES)
.map((m: any) => m.value.message as string)
// Sanity check: the test wired up correctly (email actually sent).
expect(logMessages.some((msg) => msg.includes('Email sent to [email protected]'))).toBe(true)
// The email action's "Executing action" debug log must fire EXACTLY ONCE despite the
// two dequeues it takes to switch queues. Anchor on the action id ('email_1') so we
// don't accidentally also match the trigger or exit action's lines.
const executingEmailLogs = logMessages.filter((msg) => msg === 'Executing action [Action:email_1]')
expect(executingEmailLogs).toHaveLength(1)
// The "Resuming workflow execution at" log fires at most once per dequeue — and the
// routing-continuation dequeue should be silent. So we should never see a Resuming
// line anchored on the email action (the first dequeue Starts at the trigger, not the
// email step).
const resumingEmailLogs = logMessages.filter(
(msg) => msg.includes('Resuming workflow execution at') && msg.includes('[Action:email_1]')
)
expect(resumingEmailLogs).toHaveLength(0)
// No "Workflow will pause until" lines either — the only pause in this workflow is the
// sub-millisecond routing reschedule, which the suppression should hide. Real pauses
// (delays, wait_until_condition, SES throttle retries) still log normally; they're
// covered by other tests in this file and aren't exercised here.
const pauseLogs = logMessages.filter((msg) => msg.startsWith('Workflow will pause until'))
expect(pauseLogs).toHaveLength(0)
})
it('re-routes between hogflow and email queues across email → fetch → email', async () => {
// Exercises the full ping-pong:
// hogflow worker → email queue (email_1) → email worker sends → routes back to hogflow
// → hogflow worker does fetch → email queue (email_2) → email worker sends → exits
// Proves queueMetadata.originQueue is honored on the return trip from the email worker.
await insertHogFunctionTemplate(hub.postgres, {
id: 'template-workflows-e2e-fetch',
name: 'Workflows E2E Fetch',
code: `
let res := fetch(inputs.url, {'method': inputs.method});
print('Fetch result:', res.status);
`,
inputs_schema: [
{ key: 'url', type: 'string', required: true },
{ key: 'method', type: 'string', required: false },
],
})
mockFetch.mockResolvedValue({
status: 200,
headers: { 'Content-Type': 'application/json' },
json: () => Promise.resolve({ success: true }),
text: () => Promise.resolve(JSON.stringify({ success: true })),
dump: () => Promise.resolve(),
})
const emailAction = (label: string) => ({
type: 'function_email' as const,
config: {
template_id: 'template-workflows-e2e-email',
inputs: {
email: {
value: {
to: { email: '[email protected]', name: 'Recipient' },
from: { integrationId: 1, email: '[email protected]' },
subject: label,
text: label,
html: `<p>${label}</p>`,
},
},
},
},
})
const hogFlow = new FixtureHogFlowBuilder()
.withTeamId(team.id)
.withStatus('active')
.withExitCondition('exit_only_at_end')
.withWorkflow({
actions: {
trigger: {
type: 'trigger',
config: { type: 'event', filters: HOG_FILTERS_EXAMPLES.no_filters.filters ?? {} },
},
email_1: emailAction('First email'),
fetch_1: {
type: 'function',
config: {
template_id: 'template-workflows-e2e-fetch',
inputs: {
url: { value: 'https://example.com/between-emails' },
method: { value: 'POST' },
},
},
},
email_2: emailAction('Second email'),
exit: { type: 'exit', config: {} },
},
edges: [
{ from: 'trigger', to: 'email_1', type: 'continue' },
{ from: 'email_1', to: 'fetch_1', type: 'continue' },
{ from: 'fetch_1', to: 'email_2', type: 'continue' },
{ from: 'email_2', to: 'exit', type: 'continue' },
],
})
.build()
await insertHogFlow(hub.postgres, hogFlow)
const { backgroundTask } = await eventsConsumer.processBatch([createGlobals()])
await backgroundTask
// The fetch must fire exactly once — happens on the hogflow worker between
// the two email queue hops. If it never fires, the email worker is failing
// to route back to hogflow after the first send.
await waitForExpect(() => {
expect(mockFetch).toHaveBeenCalledTimes(1)
expect(mockFetch).toHaveBeenCalledWith(
'https://example.com/between-emails',
expect.objectContaining({ method: 'POST' })
)
}, 15000)
// Two emails were queued and two were sent — one for each side of the fetch.
// Sum `count` across all messages: with aggregator in-memory dedup we'd
// miss a 2× per-send bug by only counting messages, not their counts.
await waitForExpect(() => {
const sumCounts = (filter: (m: any) => boolean) =>
mockProducerObserver
.getProducedKafkaMessagesForTopic(KAFKA_APP_METRICS_2)
.filter((m: any) => m.value.app_source === 'hog_flow')
.filter(filter)
.reduce((sum: number, m: any) => sum + m.value.count, 0)
expect(sumCounts((m) => m.value.metric_name === 'email_queued')).toBe(2)
expect(sumCounts((m) => m.value.metric_name === 'email_sent')).toBe(2)
}, 15000)
await waitForExpect(async () => {
const jobs = await queryCyclotronJobs()
const terminal = jobs.filter(
(j: any) => j.status === 'completed' || j.status === 'failed' || j.status === 'canceled'
)
expect(terminal.length).toBeGreaterThanOrEqual(1)
}, 10000)
})
it('suppresses routing logs in both directions across an email → fetch → email ping-pong', async () => {
// Companion regression guard to the single-email test above, extended to the full
// ping-pong (`hogflow → email → hogflow → email → exit`). Both routing directions
// — `routeEmailToQueue` (hogflow → email) and `routeToQueue` (email → hogflow,
// taken when a fetch action follows an email send) — go through the same
// `finished: false` + nullish `queueScheduledAt` branch in HogFunctionHandler, so
// both set `routingOnlyReschedule` and both routing dequeues should be silent in
// the logs. This test asserts that on a four-action workflow with two emails and
// a fetch between them, we still see exactly one Executing line per action and
// zero Resuming-at-email/fetch lines.
await insertHogFunctionTemplate(hub.postgres, {
id: 'template-workflows-e2e-fetch',
name: 'Workflows E2E Fetch',
code: `
let res := fetch(inputs.url, {'method': inputs.method});
print('Fetch result:', res.status);
`,
inputs_schema: [
{ key: 'url', type: 'string', required: true },
{ key: 'method', type: 'string', required: false },
],
})
mockFetch.mockResolvedValue({
status: 200,
headers: { 'Content-Type': 'application/json' },
json: () => Promise.resolve({ success: true }),
text: () => Promise.resolve(JSON.stringify({ success: true })),
dump: () => Promise.resolve(),
})
const emailAction = (label: string) => ({
type: 'function_email' as const,
config: {
template_id: 'template-workflows-e2e-email',
inputs: {
email: {
value: {
to: { email: '[email protected]', name: 'Recipient' },
from: { integrationId: 1, email: '[email protected]' },
subject: label,
text: label,
html: `<p>${label}</p>`,
},
},
},
},
})
const hogFlow = new FixtureHogFlowBuilder()
.withTeamId(team.id)
.withStatus('active')
.withExitCondition('exit_only_at_end')
.withWorkflow({
actions: {
trigger: {
type: 'trigger',
config: { type: 'event', filters: HOG_FILTERS_EXAMPLES.no_filters.filters ?? {} },
},
email_1: emailAction('Ping-pong email 1'),
fetch_1: {
type: 'function',
config: {
template_id: 'template-workflows-e2e-fetch',
inputs: {
url: { value: 'https://example.com/ping-pong-fetch' },
method: { value: 'POST' },
},
},
},
email_2: emailAction('Ping-pong email 2'),
exit: { type: 'exit', config: {} },
},
edges: [
{ from: 'trigger', to: 'email_1', type: 'continue' },
{ from: 'email_1', to: 'fetch_1', type: 'continue' },
{ from: 'fetch_1', to: 'email_2', type: 'continue' },
{ from: 'email_2', to: 'exit', type: 'continue' },
],
})
.build()
await insertHogFlow(hub.postgres, hogFlow)
const { backgroundTask } = await eventsConsumer.processBatch([createGlobals()])
await backgroundTask
// Wait for both emails sent + the workflow terminated, so all four routing
// reschedules have happened and all their logs are in Kafka.
await waitForExpect(() => {
const sumCounts = (filter: (m: any) => boolean) =>
mockProducerObserver
.getProducedKafkaMessagesForTopic(KAFKA_APP_METRICS_2)
.filter((m: any) => m.value.app_source === 'hog_flow')
.filter(filter)
.reduce((sum: number, m: any) => sum + m.value.count, 0)
expect(sumCounts((m) => m.value.metric_name === 'email_sent')).toBe(2)
}, 15000)
await waitForExpect(async () => {
const jobs = await queryCyclotronJobs()
const terminal = jobs.filter(
(j: any) => j.status === 'completed' || j.status === 'failed' || j.status === 'canceled'
)
expect(terminal.length).toBeGreaterThanOrEqual(1)
}, 10000)
const logMessages = mockProducerObserver
.getProducedKafkaMessagesForTopic(KAFKA_LOG_ENTRIES)
.map((m: any) => m.value.message as string)
// Sanity: both emails actually sent (not a vacuous pass where suppression broke the
// flow). The "Email sent" log lines are prefixed with `[Action:email_X]` via
// `actionIdForLogging`, so we substring-match instead of equality-match.
expect(logMessages.filter((msg) => msg.includes('Email sent to [email protected]'))).toHaveLength(2)
// Each routed action runs across two dequeues but only the "real" execution should log.
// - email_1 routes hogflow → email, sends on the email queue
// - fetch_1 routes email → hogflow (because the next step is a non-email function),
// runs on the hogflow queue
// - email_2 routes hogflow → email, sends on the email queue
// - exit runs inline at the tail of email_2's dequeue.
// Trigger is NOT in this list: `ensureCurrentAction` advances `currentAction` past
// the trigger to its successor immediately, so the trigger action itself never
// reaches the "Executing action" log site.
for (const actionId of ['email_1', 'fetch_1', 'email_2', 'exit']) {
const executingLogs = logMessages.filter((msg) => msg === `Executing action [Action:${actionId}]`)
expect(executingLogs).toHaveLength(1)
}
// No `Resuming workflow execution at [Action:X]` lines for any of the routed actions.
// The first dequeue Starts at the trigger; subsequent transitions are all routing
// reschedules or in-loop next-action advances, none of which re-enter execute()
// with a non-suppressed flag for these actions.
for (const actionId of ['email_1', 'fetch_1', 'email_2']) {
const resumingLogs = logMessages.filter(
(msg) => msg.includes('Resuming workflow execution at') && msg.includes(`[Action:${actionId}]`)
)
expect(resumingLogs).toHaveLength(0)
}
// No `Workflow will pause until X` lines anywhere — the three routing reschedules
// (email_1, fetch_1, email_2) are all sub-millisecond and have to be silenced.
// The workflow has no delays or wait_until_condition steps so any pause log here
// would be the routing leak we're guarding against.
const pauseLogs = logMessages.filter((msg) => msg.startsWith('Workflow will pause until'))
expect(pauseLogs).toHaveLength(0)
})
it('keeps logging real pauses (delay before email) while still suppressing the routing reschedule', async () => {
// Counter-example test: the suppression must NOT silence real pauses. A workflow
// with `trigger → delay → email → exit` produces two reschedules:
// 1. The delay action returns an explicit `queueScheduledAt` 0.5s in the future
// (real pause — must keep logging "Workflow will pause until X" and the
// corresponding "Resuming workflow execution at [Action:delay_1]" on wake).
// 2. The email action returns no `queueScheduledAt` (routing-only — must be
// silent in both directions).
// If the fix over-reaches and suppresses real delay pauses, this test fails.
const hogFlow = new FixtureHogFlowBuilder()
.withTeamId(team.id)
.withStatus('active')
.withExitCondition('exit_only_at_end')
.withWorkflow({
actions: {
trigger: {
type: 'trigger',
config: { type: 'event', filters: HOG_FILTERS_EXAMPLES.no_filters.filters ?? {} },
},
delay_1: { type: 'delay', config: { delay_duration: '0.5s' } },
email_1: {
type: 'function_email',
config: {
template_id: 'template-workflows-e2e-email',
inputs: {
email: {
value: {
to: { email: '[email protected]', name: 'Recipient' },
from: { integrationId: 1, email: '[email protected]' },
subject: 'After-delay email',
text: 'After-delay email',
html: '<p>After-delay email</p>',
},
},
},
},
},
exit: { type: 'exit', config: {} },
},
edges: [
{ from: 'trigger', to: 'delay_1', type: 'continue' },
{ from: 'delay_1', to: 'email_1', type: 'continue' },
{ from: 'email_1', to: 'exit', type: 'continue' },
],
})
.build()
await insertHogFlow(hub.postgres, hogFlow)
const { backgroundTask } = await eventsConsumer.processBatch([createGlobals()])
await backgroundTask
await waitForExpect(() => {
const sumCounts = (filter: (m: any) => boolean) =>
mockProducerObserver
.getProducedKafkaMessagesForTopic(KAFKA_APP_METRICS_2)
.filter((m: any) => m.value.app_source === 'hog_flow')
.filter(filter)
.reduce((sum: number, m: any) => sum + m.value.count, 0)
expect(sumCounts((m) => m.value.metric_name === 'email_sent')).toBe(1)
}, 15000)
await waitForExpect(async () => {
const jobs = await queryCyclotronJobs()
const terminal = jobs.filter(
(j: any) => j.status === 'completed' || j.status === 'failed' || j.status === 'canceled'
)
expect(terminal.length).toBeGreaterThanOrEqual(1)
}, 10000)
const logMessages = mockProducerObserver
.getProducedKafkaMessagesForTopic(KAFKA_LOG_ENTRIES)
.map((m: any) => m.value.message as string)
expect(logMessages.filter((msg) => msg.includes('Email sent to [email protected]'))).toHaveLength(1)
// The delay is a genuine pause and must still be logged; the email's routing hop onto
// the email queue must NOT add a duplicate. Assert exactly one pause and one resume line
// overall, independent of which action each references — so the guard tests the
// suppression's intent and stays valid regardless of whether the delay advances
// currentAction before parking.
const pauseLogs = logMessages.filter((msg) => msg.startsWith('Workflow will pause until'))
expect(pauseLogs).toHaveLength(1)
const resumeLogs = logMessages.filter((msg) => msg.includes('Resuming workflow execution at'))
expect(resumeLogs).toHaveLength(1)
})
it('wakes a wait_until_condition parked on the email queue after an email step', async () => {
// Reproduces the prod bug: an email step routes the invocation to the email queue, so the
// following wait_until_condition parks on the email queue (not hogflow). The matcher must
// still find and wake it there — otherwise a matching event never wakes the job and the
// post-wait email is never sent.
const emailAction = (label: string) => ({
type: 'function_email' as const,
config: {
template_id: 'template-workflows-e2e-email',
inputs: {
email: {
value: {
to: { email: '[email protected]', name: 'Recipient' },
from: { integrationId: 1, email: '[email protected]' },
subject: label,
text: label,
html: `<p>${label}</p>`,
},
},
},
},
})
const hogFlow = new FixtureHogFlowBuilder()
.withTeamId(team.id)
.withStatus('active')
.withExitCondition('exit_only_at_end')
.withWorkflow({
actions: {
trigger: {
type: 'trigger',
config: { type: 'event', filters: HOG_FILTERS_EXAMPLES.no_filters.filters ?? {} },
},
email_1: emailAction('First email'),
wait_condition: {
type: 'wait_until_condition',
config: {
// Property condition never matches, so only the event can wake the job.
condition: { filters: HOG_FILTERS_EXAMPLES.elements_text_filter.filters },
events: [eventNameFilter('wakeup_event')],
// Long enough that the job stays parked for the whole test — the only way
// the second email sends is the matcher waking it, never a timeout.
max_wait_duration: '5m',
},
},
email_2: emailAction('Second email'),
exit: { type: 'exit', config: {} },
},
edges: [
{ from: 'trigger', to: 'email_1', type: 'continue' },
{ from: 'email_1', to: 'wait_condition', type: 'continue' },
{ from: 'wait_condition', to: 'email_2', type: 'branch', index: 0 },
{ from: 'wait_condition', to: 'exit', type: 'continue' },
{ from: 'email_2', to: 'exit', type: 'continue' },
],
})
.build()
await insertHogFlow(hub.postgres, hogFlow)
const emailsSent = () =>
mockProducerObserver
.getProducedKafkaMessagesForTopic(KAFKA_APP_METRICS_2)
.filter((m: any) => m.value.app_source === 'hog_flow')
.filter((m: any) => m.value.metric_name === 'email_sent')
.reduce((sum: number, m: any) => sum + m.value.count, 0)
// Trigger: email_1 routes to the email queue, the email worker sends it and continues the
// flow to the wait step, which parks.
const { backgroundTask } = await eventsConsumer.processBatch([createGlobals()])
await backgroundTask
// The first email is sent and the job parks waiting for the event.
await waitForExpect(async () => {
expect(emailsSent()).toBe(1)
const jobs = await queryCyclotronJobs()
expect(jobs.some((j: any) => j.status === 'available' && new Date(j.scheduled) > new Date())).toBe(true)
}, 15000)
// The wait parks on the email queue (carried over from email_1). The matcher has to find
// it there regardless of queue — that's exactly the scenario that was broken.
const parked = (await queryCyclotronJobs()).find(
(j: any) => j.status === 'available' && new Date(j.scheduled) > new Date()
)
expect(parked).toBeDefined()
expect(parked?.queue_name).toBe('email')
expect(emailsSent()).toBe(1)
// The subscribed event fires for this person — the matcher wakes the parked job even though
// it sits on the email queue, the email worker resumes it down the matched branch, and the
// second email is sent. This is the end-to-end "the job wakes and the next step runs" check.
matcher = new CdpHogflowSubscriptionMatcherConsumer({ ...hub }, deps)
await matcher.processBatch([createGlobals({ event: 'wakeup_event' })])
await waitForExpect(() => {
expect(emailsSent()).toBe(2)
}, 15000)
})
it("a workflow over its sending limit does not hold up another workflow's emails", async () => {
// Two workflows on the same team and the same email queue. Workflow A has a
// sending limit of 2 per minute and gets 8 sends queued at once, so all but the
// first are denied. Workflow B has no limit and 3 sends. The whole pipeline is
// real: events consumer -> hogflow worker -> email queue -> email worker ->
// rate limiter -> reschedule. B's emails must go out while A's denied sends
// park on their own future slots, each dequeued once, with the job row's
// transition counter staying low.
const buildEmailFlow = (triggerEvent: string, recipient: string): HogFlow =>
new FixtureHogFlowBuilder()
.withTeamId(team.id)
.withStatus('active')
.withExitCondition('exit_only_at_end')
.withWorkflow({
actions: {
trigger: {
type: 'trigger',
config: { type: 'event', ...eventNameFilter(triggerEvent) },
},
email_1: {
type: 'function_email',
config: {
template_id: 'template-workflows-e2e-email',
inputs: {
email: {
value: {
to: { email: recipient, name: 'Recipient' },
from: { integrationId: 1, email: '[email protected]' },
subject: `To ${recipient}`,
text: 'Test text',
html: '<p>Test html</p>',
},
},
},
},
},
exit: { type: 'exit', config: {} },
},
edges: [
{ from: 'trigger', to: 'email_1', type: 'continue' },
{ from: 'email_1', to: 'exit', type: 'continue' },
],
})
.build()
const flowA = buildEmailFlow('signup_a', '[email protected]')
const flowB = buildEmailFlow('signup_b', '[email protected]')
await insertHogFlow(hub.postgres, flowA)
await insertHogFlow(hub.postgres, flowB)
// 2 per minute: one burst send, then one slot every 30s. The denied sends park
// 30s+ out, far past this test's clock, so they stay parked.
await hub.postgres.query(
PostgresUse.COMMON_WRITE,
`UPDATE posthog_hogflow SET email_sending_rate_limit = $1 WHERE id = $2`,
[JSON.stringify({ count: 2, period: 'minute' }), flowA.id],
'set-email-sending-rate-limit'
)
const events = [
...Array.from({ length: 8 }, (_, i) =>
createGlobals({ event: 'signup_a', uuid: new UUIDT().toString(), distinct_id: `person_a_${i}` } as any)
),
...Array.from({ length: 3 }, (_, i) =>
createGlobals({ event: 'signup_b', uuid: new UUIDT().toString(), distinct_id: `person_b_${i}` } as any)
),
]
const { backgroundTask } = await eventsConsumer.processBatch(events)
await backgroundTask
// B's 3 runs finish end to end, and A gets its one in-budget send out: 4 sends total.
await waitForExpect(async () => {
const jobs = await queryCyclotronJobs()
const bJobs = jobs.filter((j: any) => j.function_id === flowB.id)
expect(bJobs.length).toBe(3)
expect(bJobs.every((j: any) => j.status === 'completed')).toBe(true)
const sent = mockProducerObserver
.getProducedKafkaMessagesForTopic(KAFKA_APP_METRICS_2)
.filter((m: any) => m.value.app_source === 'hog_flow')
.filter((m: any) => m.value.metric_name === 'email_sent')
.reduce((sum: number, m: any) => sum + m.value.count, 0)
expect(sent).toBe(4)
}, 15000)
// A's 7 denied sends are parked out of the way: still queued, each on its own
// future slot, each dequeued exactly once on the email queue.
const jobs = await queryCyclotronJobs()
const parkedA = jobs.filter(
(j: any) => j.function_id === flowA.id && j.queue_name === 'email' && j.status === 'available'
)
expect(parkedA.length).toBe(7)
const slots = parkedA.map((j: any) => new Date(j.scheduled).getTime())
expect(new Set(slots).size).toBe(7)
for (const slot of slots) {
expect(slot).toBeGreaterThan(Date.now() + 20_000)
}
// Trigger enqueue, hogflow pass (dequeue + route to email), email pass
// (dequeue + park) is at most 4 transitions. Anything higher means a
// denied send went around the loop again.
for (const job of parkedA) {
expect(job.transition_count).toBeLessThanOrEqual(4)
}
})
it("a team over its tier cap does not hold up another team's emails", async () => {
// Two teams, each with one email workflow. Team A sits on a tier that allows
// 2 emails per day and gets 6 sends queued; team B is on a high tier with 3
// sends. The whole pipeline is real and the tier caps are enforced: B's emails
// must all go out while A's over-cap sends park on future slots from A's own
// refill, without cycling.
//
// The default email worker was built with the caps off, so swap in one that
// enforces them. Tier 0 allows 2/day, tier 1 is effectively unlimited.
await emailWorker.stop()
hub.EMAIL_TEAM_SENDING_CAP_MODE = 'enforce'
hub.EMAIL_TEAM_SENDING_CAP_DAILY_BY_TIER = '2,240000'
const enforcedQueue = new CyclotronJobQueuePostgresV2(hub.CONSUMER_BATCH_SIZE, hub)
emailWorker = new CdpCyclotronWorkerEmail(hub, deps, enforcedQueue)
await emailWorker.start()
// The tier bucket is keyed by team id and the test Redis keeps data between
// runs, so team A's bucket must start full or a previous run drains this one.
const capValkey = createRedisV2PoolFromConfig({
connection: hub.CDP_REDIS_HOST
? {
url: hub.CDP_REDIS_HOST,
options: { port: hub.CDP_REDIS_PORT, password: hub.CDP_REDIS_PASSWORD },
}
: { url: hub.REDIS_URL },
poolMinSize: hub.REDIS_POOL_MIN_SIZE,
poolMaxSize: hub.REDIS_POOL_MAX_SIZE,
})
await deleteKeysWithPrefix(capValkey, '@posthog/team-email-rate-day')
// Team A has no config row and defaults to tier 0. Team B gets tier 1.
const teamBId = await createTeam(hub.postgres, team.organization_id)
await hub.postgres.query(
PostgresUse.COMMON_WRITE,
`INSERT INTO workflows_teamworkflowsconfig
(team_id, capture_workflows_engagement_events, email_tracking_consent_mode,
email_sending_suspension_reason, ses_tenant_sending_status, email_sending_tier)
VALUES ($1, false, 'off', '', '', 1)`,
[teamBId],
'set-team-email-tier'
)
await insertIntegration(hub.postgres, teamBId, {
id: 2,
kind: 'email',
config: {
email: '[email protected]',
name: 'Test Sender',
domain: 'posthog.com',
verified: true,
provider: 'maildev',
},
})
const buildFlow = (teamId: number, integrationId: number, triggerEvent: string, recipient: string): HogFlow =>
new FixtureHogFlowBuilder()
.withTeamId(teamId)
.withStatus('active')
.withExitCondition('exit_only_at_end')
.withWorkflow({
actions: {
trigger: {
type: 'trigger',
config: { type: 'event', ...eventNameFilter(triggerEvent) },
},
email_1: {
type: 'function_email',
config: {
template_id: 'template-workflows-e2e-email',
inputs: {
email: {
value: {
to: { email: recipient, name: 'Recipient' },
from: { integrationId, email: '[email protected]' },
subject: `To ${recipient}`,
text: 'Test text',
html: '<p>Test html</p>',
},
},
},
},
},
exit: { type: 'exit', config: {} },
},
edges: [
{ from: 'trigger', to: 'email_1', type: 'continue' },
{ from: 'email_1', to: 'exit', type: 'continue' },
],
})
.build()
const flowA = buildFlow(team.id, 1, 'tier_a', '[email protected]')
const flowB = buildFlow(teamBId, 2, 'tier_b', '[email protected]')
await insertHogFlow(hub.postgres, flowA)
await insertHogFlow(hub.postgres, flowB)
const teamBGlobals = (i: number): HogFunctionInvocationGlobals =>
createHogExecutionGlobals({
project: { id: teamBId } as any,
event: {
uuid: new UUIDT().toString(),
event: 'tier_b',
distinct_id: `person_tier_b_${i}`,
properties: {},
timestamp: '2024-09-03T09:00:00Z',
} as any,
})
const events = [
...Array.from({ length: 6 }, (_, i) =>
createGlobals({
event: 'tier_a',
uuid: new UUIDT().toString(),
distinct_id: `person_tier_a_${i}`,
} as any)
),
...Array.from({ length: 3 }, (_, i) => teamBGlobals(i)),
]
const { backgroundTask } = await eventsConsumer.processBatch(events)
await backgroundTask
// Team B's 3 runs finish end to end; team A gets its 2 in-budget sends out.
await waitForExpect(async () => {
const jobs = await queryCyclotronJobs()
const bJobs = jobs.filter((j: any) => j.function_id === flowB.id)
expect(bJobs.length).toBe(3)
expect(bJobs.every((j: any) => j.status === 'completed')).toBe(true)
const sent = mockProducerObserver
.getProducedKafkaMessagesForTopic(KAFKA_APP_METRICS_2)
.filter((m: any) => m.value.app_source === 'hog_flow')
.filter((m: any) => m.value.metric_name === 'email_sent')
.reduce((sum: number, m: any) => sum + m.value.count, 0)
expect(sent).toBe(5)
}, 15000)
// Team A's 4 over-cap sends are parked far in the future (its refill is one
// token per 12 hours), each dequeued once on the email queue.
const jobs = await queryCyclotronJobs()
const parkedA = jobs.filter(
(j: any) => j.function_id === flowA.id && j.queue_name === 'email' && j.status === 'available'
)
expect(parkedA.length).toBe(4)
for (const job of parkedA) {
expect(new Date(job.scheduled).getTime()).toBeGreaterThan(Date.now() + 25 * 60 * 1000)
expect(job.transition_count).toBeLessThanOrEqual(4)
}
})
it('rate-limited variant processes emails end-to-end through the dedicated bucket', async () => {
// Verifies the inject-pattern wiring: CyclotronJobQueueRateLimitedPostgresV2
// gates dequeue via a Valkey bucket, then the email worker processes the
// job normally. Reuses the local test Redis as the bucket store (same
// approach as rate-limiter.service.test.ts).
await emailWorker.stop()
const limiterValkey = createRedisV2PoolFromConfig({
connection: hub.CDP_REDIS_HOST
? {
url: hub.CDP_REDIS_HOST,
options: { port: hub.CDP_REDIS_PORT, password: hub.CDP_REDIS_PASSWORD },
}
: { url: hub.REDIS_URL },
poolMinSize: hub.REDIS_POOL_MIN_SIZE,
poolMaxSize: hub.REDIS_POOL_MAX_SIZE,
})
const bucketKey = '@posthog-test/ses-e2e/bucket'
await deleteKeysWithPrefix(limiterValkey, '@posthog-test/ses-e2e/')
const rateLimitedQueue = new CyclotronJobQueueRateLimitedPostgresV2(hub.CONSUMER_BATCH_SIZE, hub, {
limiter: new RateLimiterService(limiterValkey, { name: 'ses-e2e' }),
key: bucketKey,
capacity: 10,
refillPerSecond: 10,
throttledPollDelayMs: 50,
})
emailWorker = new CdpCyclotronWorkerEmail(hub, deps, rateLimitedQueue)
await emailWorker.start()
const hogFlow = new FixtureHogFlowBuilder()
.withTeamId(team.id)
.withStatus('active')
.withExitCondition('exit_only_at_end')
.withWorkflow({
actions: {
trigger: {
type: 'trigger',
config: { type: 'event', filters: HOG_FILTERS_EXAMPLES.no_filters.filters ?? {} },
},
email_1: {
type: 'function_email',
config: {
template_id: 'template-workflows-e2e-email',
inputs: {
email: {
value: {
to: { email: '[email protected]', name: 'Recipient' },
from: { integrationId: 1, email: '[email protected]' },
subject: 'Rate-limited email',
text: 'Test text',
html: '<p>Test html</p>',
},
},
},
},
},
exit: { type: 'exit', config: {} },
},
edges: [
{ from: 'trigger', to: 'email_1', type: 'continue' },
{ from: 'email_1', to: 'exit', type: 'continue' },
],
})
.build()
await insertHogFlow(hub.postgres, hogFlow)
const { backgroundTask } = await eventsConsumer.processBatch([createGlobals()])
await backgroundTask
// The email is sent — the rate-limited queue gated, dequeued, and processed the job.
await waitForExpect(() => {
const emailSentCount = mockProducerObserver
.getProducedKafkaMessagesForTopic(KAFKA_APP_METRICS_2)
.filter((m: any) => m.value.app_source === 'hog_flow')
.filter((m: any) => m.value.metric_name === 'email_sent')
.reduce((sum: number, m: any) => sum + m.value.count, 0)
expect(emailSentCount).toBe(1)
}, 15000)
// The bucket has been touched — proves the rate limiter was actually consulted,
// not bypassed. `ts` and `pool` are written on every claim (cold start or refill).
const bucket = await limiterValkey.useClient({ name: 'read-bucket' }, (client) => client.hgetall(bucketKey))
expect(bucket?.ts).toBeTruthy()
expect(bucket?.pool).toBeTruthy()
})
it('rate-limits dequeue when the bucket drains, then drains the queue as it refills', async () => {
// Tiny bucket (capacity 1, refill 5/sec = 1 token every 200ms) so the
// worker has to wait for refills between sends. With 3 emails enqueued
// we should see the bucket get denied at least once while the worker
// is waiting — and all 3 should still eventually go through.
await emailWorker.stop()
const limiterValkey = createRedisV2PoolFromConfig({
connection: hub.CDP_REDIS_HOST
? {
url: hub.CDP_REDIS_HOST,
options: { port: hub.CDP_REDIS_PORT, password: hub.CDP_REDIS_PASSWORD },
}
: { url: hub.REDIS_URL },
poolMinSize: hub.REDIS_POOL_MIN_SIZE,
poolMaxSize: hub.REDIS_POOL_MAX_SIZE,
})
const bucketKey = '@posthog-test/ses-e2e-throttled/bucket'
await deleteKeysWithPrefix(limiterValkey, '@posthog-test/ses-e2e-throttled/')
const limiterName = 'ses-e2e-throttled'
const rateLimitedQueue = new CyclotronJobQueueRateLimitedPostgresV2(hub.CONSUMER_BATCH_SIZE, hub, {
limiter: new RateLimiterService(limiterValkey, { name: limiterName }),
key: bucketKey,
capacity: 1,
refillPerSecond: 5,
throttledPollDelayMs: 50,
})
emailWorker = new CdpCyclotronWorkerEmail(hub, deps, rateLimitedQueue)
await emailWorker.start()
// Snapshot the denied counter so we measure only this test's claims.
const readDeniedCount = async (): Promise<number> => {
const metric = register.getSingleMetric('cdp_rate_limiter_claim_total')
if (!metric) {
return 0
}
const data = await metric.get()
return data.values
.filter((v: any) => v.labels.result === 'denied' && v.labels.limiter === limiterName)
.reduce((sum: number, v: any) => sum + v.value, 0)
}
const deniedBefore = await readDeniedCount()
const hogFlow = new FixtureHogFlowBuilder()
.withTeamId(team.id)
.withStatus('active')
.withExitCondition('exit_only_at_end')
.withWorkflow({
actions: {
trigger: {
type: 'trigger',
config: { type: 'event', filters: HOG_FILTERS_EXAMPLES.no_filters.filters ?? {} },
},
email_1: {
type: 'function_email',
config: {
template_id: 'template-workflows-e2e-email',
inputs: {
email: {
value: {
to: { email: '[email protected]', name: 'Recipient' },
from: { integrationId: 1, email: '[email protected]' },
subject: 'Throttled email',
text: 'Test text',
html: '<p>Test html</p>',
},
},
},
},
},
exit: { type: 'exit', config: {} },
},
edges: [
{ from: 'trigger', to: 'email_1', type: 'continue' },
{ from: 'email_1', to: 'exit', type: 'continue' },
],
})
.build()
await insertHogFlow(hub.postgres, hogFlow)
// Three distinct events → three email jobs queued near-simultaneously.
const events = Array.from({ length: 3 }, () => createGlobals({ uuid: new UUIDT().toString() }))
const { backgroundTask } = await eventsConsumer.processBatch(events)
await backgroundTask
// All three eventually send — generous timeout because the bucket only
// refills 5 tokens/sec.
await waitForExpect(() => {
const emailSentCount = mockProducerObserver
.getProducedKafkaMessagesForTopic(KAFKA_APP_METRICS_2)
.filter((m: any) => m.value.app_source === 'hog_flow')
.filter((m: any) => m.value.metric_name === 'email_sent')
.reduce((sum: number, m: any) => sum + m.value.count, 0)
expect(emailSentCount).toBe(3)
}, 20000)
// The bucket was denied at least once during processing — proves the
// gating actually fired, not that we just dequeued 3 jobs in a row.
// (With capacity=1 and three pending jobs, between sends the worker
// polls many times finding bucket=0.)
const deniedAfter = await readDeniedCount()
expect(deniedAfter - deniedBefore).toBeGreaterThan(0)
})
it('claims only the visible row count (sparse traffic does not drain the bucket)', async () => {
// Regression guard for the pre-size fix. Without it, a single ready
// email would claim the bucket's full capacity — draining ~capacity-1
// tokens of SES budget per actual send — even though the worker can
// only dequeue one row. Pre-sizing asks the limiter for exactly the
// number of rows the worker is about to dequeue.
//
// refillPerSecond=0 freezes the bucket between the claim and our
// assertion, so the post-claim pool is a deterministic measure of
// what was deducted (capacity minus tokens granted on the one claim).
await emailWorker.stop()
const limiterValkey = createRedisV2PoolFromConfig({
connection: hub.CDP_REDIS_HOST
? {
url: hub.CDP_REDIS_HOST,
options: { port: hub.CDP_REDIS_PORT, password: hub.CDP_REDIS_PASSWORD },
}
: { url: hub.REDIS_URL },
poolMinSize: hub.REDIS_POOL_MIN_SIZE,
poolMaxSize: hub.REDIS_POOL_MAX_SIZE,
})
const bucketKey = '@posthog-test/ses-e2e-presize/bucket'
await deleteKeysWithPrefix(limiterValkey, '@posthog-test/ses-e2e-presize/')
const capacity = 10
const rateLimitedQueue = new CyclotronJobQueueRateLimitedPostgresV2(hub.CONSUMER_BATCH_SIZE, hub, {
limiter: new RateLimiterService(limiterValkey, { name: 'ses-e2e-presize' }),
key: bucketKey,
capacity,
refillPerSecond: 0,
throttledPollDelayMs: 50,
})
emailWorker = new CdpCyclotronWorkerEmail(hub, deps, rateLimitedQueue)
await emailWorker.start()
const hogFlow = new FixtureHogFlowBuilder()
.withTeamId(team.id)
.withStatus('active')
.withExitCondition('exit_only_at_end')
.withWorkflow({
actions: {
trigger: {
type: 'trigger',
config: { type: 'event', filters: HOG_FILTERS_EXAMPLES.no_filters.filters ?? {} },
},
email_1: {
type: 'function_email',
config: {
template_id: 'template-workflows-e2e-email',
inputs: {
email: {
value: {
to: { email: '[email protected]', name: 'Recipient' },
from: { integrationId: 1, email: '[email protected]' },
subject: 'Sparse-traffic email',
text: 'Test text',
html: '<p>Test html</p>',
},
},
},
},
},
exit: { type: 'exit', config: {} },
},
edges: [
{ from: 'trigger', to: 'email_1', type: 'continue' },
{ from: 'email_1', to: 'exit', type: 'continue' },
],
})
.build()
await insertHogFlow(hub.postgres, hogFlow)
// Exactly ONE event → one email job — the sparse-traffic scenario.
const { backgroundTask } = await eventsConsumer.processBatch([createGlobals({ uuid: new UUIDT().toString() })])
await backgroundTask
await waitForExpect(() => {
const emailSentCount = mockProducerObserver
.getProducedKafkaMessagesForTopic(KAFKA_APP_METRICS_2)
.filter((m: any) => m.value.app_source === 'hog_flow')
.filter((m: any) => m.value.metric_name === 'email_sent')
.reduce((sum: number, m: any) => sum + m.value.count, 0)
expect(emailSentCount).toBe(1)
}, 15000)
// Bucket should retain ~capacity-1 tokens — we claimed 1, not capacity.
// Without the pre-size fix `pool` would be 0 here (the whole bucket
// drained on the one claim).
const bucket = await limiterValkey.useClient({ name: 'read-bucket' }, (client) => client.hgetall(bucketKey))
const pool = parseFloat(bucket?.pool ?? '0')
expect(pool).toBeGreaterThanOrEqual(capacity - 1)
})
// ---- Message-assets bulk flush at the batch boundary ----
//
// Email assets used to be produced one-at-a-time via a fire-and-forget Kafka call
// from `email.service.ts → MessageAssetsService.captureSentEmail`. We've moved that
// to a buffer-then-flush pattern that drains `result.messageAssets` at the batch
// boundary and bulk-produces, gated on broker ack before the consumer commits
// offsets. These tests pin the end-to-end behavior: one workflow → one asset row in
// the `message_assets` Kafka topic with the right metadata, and a single batch with
// multiple emails produces all rows.
describe('message_assets bulk capture', () => {
const buildEmailWorkflow = (subject: string) =>
new FixtureHogFlowBuilder()
.withTeamId(team.id)
.withStatus('active')
.withExitCondition('exit_only_at_end')
.withWorkflow({
actions: {
trigger: {
type: 'trigger',
config: { type: 'event', filters: HOG_FILTERS_EXAMPLES.no_filters.filters ?? {} },
},
email_1: {
type: 'function_email',
config: {
template_id: 'template-workflows-e2e-email',
inputs: {
email: {
value: {
to: { email: '[email protected]', name: 'Recipient' },
from: { integrationId: 1, email: '[email protected]' },
subject,
text: 'plain text body',
html: `<p>${subject}</p>`,
},
},
},
},
},
exit: { type: 'exit', config: {} },
},
edges: [
{ from: 'trigger', to: 'email_1', type: 'continue' },
{ from: 'email_1', to: 'exit', type: 'continue' },
],
})
.build()
const assetMessages = (): { key: string; value: any }[] =>
mockProducerObserver.getProducedKafkaMessagesForTopic(KAFKA_MESSAGE_ASSETS) as any
it('produces one message_assets row per workflow email with the rendered HTML and metadata', async () => {
const hogFlow = buildEmailWorkflow('Asset bulk-capture single')
await insertHogFlow(hub.postgres, hogFlow)
const { backgroundTask } = await eventsConsumer.processBatch([createGlobals()])
await backgroundTask
// The asset row only lands once the email worker's batch flushes — wait for
// the workflow to reach a terminal state, then assert against the topic.
await waitForExpect(async () => {
const jobs = await queryCyclotronJobs()
const terminal = jobs.filter(
(j: any) => j.status === 'completed' || j.status === 'failed' || j.status === 'canceled'
)
expect(terminal.length).toBeGreaterThanOrEqual(1)
}, 15000)
await waitForExpect(() => {
const rows = assetMessages()
expect(rows).toHaveLength(1)
const row = rows[0].value as Record<string, any>
expect(row.team_id).toBe(team.id)
expect(row.function_kind).toBe('hog_flow')
expect(row.kind).toBe('email')
expect(row.status).toBe('sent')
expect(row.action_id).toBe('email_1')
expect(row.recipient).toBe('[email protected]')
expect(row.subject).toBe('Asset bulk-capture single')
expect(row.html).toBe('<p>Asset bulk-capture single</p>')
// Partition key must match invocation_id so retries collapse via the
// destination ReplacingMergeTree(version).
expect(rows[0].key).toBe(row.invocation_id)
}, 15000)
})
it('bulk-captures every asset when multiple workflow runs share a batch', async () => {
// Use distinct subjects so we can assert on row content regardless of the
// partition-level ordering the Kafka producer chooses.
const hogFlow = buildEmailWorkflow('Bulk asset capture')
await insertHogFlow(hub.postgres, hogFlow)
// Three globals dispatched in one `processBatch` call — they go through the
// events consumer together. Each fires its own workflow run, each emits one
// asset; the bulk-flush is what we're exercising.
const { backgroundTask } = await eventsConsumer.processBatch([
createGlobals({ uuid: 'aaaaaaaa-0000-0000-0000-000000000001' as any }),
createGlobals({ uuid: 'aaaaaaaa-0000-0000-0000-000000000002' as any }),
createGlobals({ uuid: 'aaaaaaaa-0000-0000-0000-000000000003' as any }),
])
await backgroundTask
await waitForExpect(async () => {
const jobs = await queryCyclotronJobs()
const terminal = jobs.filter(
(j: any) => j.status === 'completed' || j.status === 'failed' || j.status === 'canceled'
)
expect(terminal.length).toBeGreaterThanOrEqual(3)
}, 20000)
await waitForExpect(() => {
const rows = assetMessages()
expect(rows.length).toBeGreaterThanOrEqual(3)
// Every produced row must carry distinct invocation_id values (one per
// workflow run) — otherwise the buffer is dropping or aliasing rows.
const invocationIds = new Set(rows.map((r) => (r.value as any).invocation_id))
expect(invocationIds.size).toBeGreaterThanOrEqual(3)
// Subject and HTML are constant across the three runs (same flow), so we
// just sanity-check that every row carries the expected shape.
for (const row of rows) {
const value = row.value as Record<string, any>
expect(value.subject).toBe('Bulk asset capture')
expect(value.html).toBe('<p>Bulk asset capture</p>')
expect(value.kind).toBe('email')
expect(value.status).toBe('sent')
expect(row.key).toBe(value.invocation_id)
}
}, 20000)
})
})
})
/**
* E2E for the batch resolver dispatch path through the cdp-api HTTP boundary.
*
* Goes through real express + supertest, real CdpApi, real cyclotron-node
* Postgres. Verifies that POST `/batch_invocations/<id>` creates a resolver
* cyclotron job pointing at the right queue with the right state.
*
* The deep state-machine paths (page execution, terminal write, truncation,
* Django down → resolver parks) are covered by the integration tests in
* `consumers/cdp-cyclotron-worker-batch-resolve.consumer.test.ts`. This
* test is the boundary backstop — it caught a real bug (wrong DB URL on
* the resolver manager) during development.
*
* Hub lifecycle follows cdp-api.test.ts: one hub for the suite (beforeAll),
* torn down once (afterAll). Per-test isolation comes from resetting the
* postgres team data + truncating cyclotron_jobs in beforeEach.
*/
/**
* E2E for the cyclotron batch resolver. POST through cdp-api, real consumer
* loop, mocked Django endpoints, assert on the resulting cyclotron + Django
* state. Mirrors how the system runs in prod — no manual dequeue plumbing.
*/
describe('Workflows E2E: batch resolver dispatch via cdp-api', () => {
jest.setTimeout(60000)
const CYCLOTRON_NODE_DB_URL =
process.env.CYCLOTRON_NODE_DATABASE_URL ?? 'postgres://posthog:posthog@localhost:5432/test_cyclotron_node'
let hub: Hub
let team: Team
let cyclotronPool: Pool
let app: express.Application
let server: any
let api: CdpApi
let batchResolverProducer: CyclotronV2Manager
let deps: ReturnType<typeof createCdpConsumerDeps>
let kafkaProducer: KafkaProducerWrapper
let mockProducerObserver: KafkaProducerObserver
// Fresh consumer per test — built in the it() body, stopped in afterEach.
let resolverWorker: CdpCyclotronWorkerBatchResolve | undefined
beforeAll(async () => {
cyclotronPool = new Pool({ connectionString: CYCLOTRON_NODE_DB_URL })
MockKafkaProducerWrapper.create = jest.fn((...args) => ActualKafkaProducerWrapper.create(...args))
// KAFKA_HOG_INVOCATION_RESULTS isn't in TEST_KAFKA_TOPICS — the resolver produces a
// running row per enrolled person there, so the topic has to exist or the produce fails.
await ensureKafkaTopics([...TEST_KAFKA_TOPICS, KAFKA_HOG_INVOCATION_RESULTS])
hub = await createHub({
SITE_URL: 'http://localhost:8000',
})
// Lifecycle rows are gated on this flag, which is off by default outside dev — without
// it the row assertions below would pass against an empty topic.
hub.HOG_INVOCATION_RESULTS_ENABLED = true
const { createMockJobQueue } = require('../../tests/helpers/mocks/job-queue.mock')
kafkaProducer = await ActualKafkaProducerWrapper.create(hub.KAFKA_CLIENT_RACK)
mockProducerObserver = new KafkaProducerObserver(kafkaProducer)
deps = createCdpConsumerDeps(hub, kafkaProducer)
batchResolverProducer = new CyclotronV2Manager({
pool: { dbUrl: CYCLOTRON_NODE_DB_URL, maxConnections: 5 },
})
api = new CdpApi(
hub,
deps,
{ hogQueue: createMockJobQueue(), hogflowQueue: createMockJobQueue() },
batchResolverProducer
)
app = setupExpressApp()
app.use('/', api.router())
server = app.listen(0, () => {})
})
afterAll(async () => {
server?.close()
await batchResolverProducer?.disconnect()
await kafkaProducer?.disconnect()
await closeHub(hub)
await cyclotronPool.end()
})
beforeEach(async () => {
await resetTestDatabase()
await cyclotronPool.query(`DELETE FROM cyclotron_jobs`)
team = await getFirstTeam(hub.postgres)
resolverWorker = undefined
mockProducerObserver.resetKafkaProducer()
})
afterEach(async () => {
// If the test started the consumer, shut it down before the next test.
// resolverWorker.stop() waits for any in-flight processBatch to settle,
// then closes the pool — so afterAll's closeHub has no leftover work.
await resolverWorker?.stop()
})
// Fresh worker per test — stop() disconnects the underlying Postgres pool,
// so a shared instance can't survive past the first test's afterEach.
// `wrapJob` lets a test intercept a dequeued job's methods (e.g. fail a commit once).
function buildResolverConsumer(
wrapJob?: (job: CyclotronV2DequeuedJob) => CyclotronV2DequeuedJob
): CdpCyclotronWorkerBatchResolve {
const cyclotronWorker = new CyclotronV2Worker({
pool: { dbUrl: CYCLOTRON_NODE_DB_URL, maxConnections: 10 },
queueName: HOGFLOW_BATCH_RESOLVE_QUEUE,
pollDelayMs: 100,
batchMaxSize: 1,
heartbeatTimeoutMs: hub.CDP_HOG_FLOW_BATCH_AUDIENCE_FETCH_TIMEOUT_MS + 30_000,
})
// The consumer only calls connect/disconnect/isHealthy on the worker.
const workerForConsumer = wrapJob
? ({
connect: (callback: (jobs: CyclotronV2DequeuedJob[]) => Promise<void>) =>
cyclotronWorker.connect((jobs) => callback(jobs.map(wrapJob))),
disconnect: () => cyclotronWorker.disconnect(),
isHealthy: () => cyclotronWorker.isHealthy(),
} as unknown as CyclotronV2Worker)
: cyclotronWorker
const internalFetchService = new InternalFetchService(hub.INTERNAL_API_BASE_URL, hub.INTERNAL_API_SECRET)
const queryService = new HogFlowBatchPersonQueryService(
internalFetchService,
hub.CDP_HOG_FLOW_BATCH_AUDIENCE_FETCH_TIMEOUT_MS
)
return new CdpCyclotronWorkerBatchResolve(hub, deps, workerForConsumer, queryService, internalFetchService)
}
async function insertActiveBatchFlow(triggerMasking?: HogFlow['trigger_masking']): Promise<HogFlow> {
const flow = new FixtureHogFlowBuilder()
.withTeamId(team.id)
.withStatus('active')
.withTriggerMasking(triggerMasking)
.withWorkflow({
actions: {
trigger: {
type: 'trigger',
config: {
type: 'batch',
filters: { properties: [{ key: 'plan', value: 'enterprise', type: 'person' }] },
},
},
function_1: {
type: 'function',
config: {
template_id: 'template-workflows-e2e-fetch',
inputs: { url: { value: 'https://example.com/batch' } },
},
},
},
edges: [{ from: 'trigger', to: 'function_1', type: 'continue' }],
})
.build()
return await insertHogFlow(hub.postgres, flow)
}
it('POST /batch_invocations with flag on creates a resolver cyclotron job with the right shape', async () => {
const flow = await insertActiveBatchFlow()
const parentRunId = new UUIDT().toString()
const response = await supertest(app)
.post(`/api/projects/${team.id}/hog_flows/${flow.id}/batch_invocations/${parentRunId}`)
.send({
filters: { filter_test_accounts: false },
max_audience_size: 750,
variables: { greeting: 'hi' },
group_type_index: 2,
})
.expect(200)
expect(response.body).toEqual({ status: 'queued' })
const rows = await cyclotronPool.query<{
id: string
queue_name: string
status: string
parent_run_id: string
team_id: number
function_id: string
state: Buffer | null
}>(
`SELECT id, queue_name, status::text AS status, parent_run_id, team_id, function_id, state
FROM cyclotron_jobs
WHERE queue_name = 'hogflow_batch_resolve' AND parent_run_id = $1`,
[parentRunId]
)
expect(rows.rows).toHaveLength(1)
const job = rows.rows[0]
expect(job.status).toBe('available')
expect(job.parent_run_id).toBe(parentRunId)
expect(job.team_id).toBe(team.id)
expect(job.function_id).toBe(flow.id)
const state = parseJSON((job.state as Buffer).toString('utf-8')) as Record<string, unknown>
expect(state).toMatchObject({
batchJobId: parentRunId,
teamId: team.id,
hogFlowId: flow.id,
maxAudienceSize: 750,
variables: { greeting: 'hi' },
groupTypeIndex: 2,
cursor: null,
totalEnqueued: 0,
pagesProcessed: 0,
})
expect(state.pendingTerminal).toBeUndefined()
})
it('rejects with 400 when the workflow is not a batch trigger', async () => {
const flow = new FixtureHogFlowBuilder()
.withTeamId(team.id)
.withStatus('active')
.withWorkflow({
actions: {
trigger: {
type: 'trigger',
config: { type: 'event', filters: {} },
},
function_1: {
type: 'function',
config: {
template_id: 'template-workflows-e2e-fetch',
inputs: { url: { value: 'https://example.com/' } },
},
},
},
edges: [{ from: 'trigger', to: 'function_1', type: 'continue' }],
})
.build()
const inserted = await insertHogFlow(hub.postgres, flow)
const parentRunId = new UUIDT().toString()
await supertest(app)
.post(`/api/projects/${team.id}/hog_flows/${inserted.id}/batch_invocations/${parentRunId}`)
.send({ filters: { filter_test_accounts: false } })
.expect(400)
const rows = await cyclotronPool.query(`SELECT id FROM cyclotron_jobs WHERE parent_run_id = $1`, [parentRunId])
expect(rows.rows).toHaveLength(0)
})
it('full lifecycle: HTTP POST → resolver chunks 3 pages → children enqueued → Django PUT status=completed', async () => {
const flow = await insertActiveBatchFlow()
const parentRunId = new UUIDT().toString()
const personIds = [
new UUIDT().toString(),
new UUIDT().toString(),
new UUIDT().toString(),
new UUIDT().toString(),
new UUIDT().toString(),
]
const statusPuts: Array<{ status: string }> = []
let personPageCalls = 0
mockInternalFetch.mockImplementation((url: string, opts: any) => {
if (url.includes('/user_blast_radius_persons')) {
personPageCalls += 1
const pages = [
{ users_affected: personIds.slice(0, 2), cursor: 'c1', has_more: true },
{ users_affected: personIds.slice(2, 4), cursor: 'c2', has_more: true },
{ users_affected: personIds.slice(4), cursor: null, has_more: false },
]
const page = pages[Math.min(personPageCalls - 1, pages.length - 1)]
return Promise.resolve({
status: 200,
headers: {},
json: () => Promise.resolve({}),
text: () => Promise.resolve(JSON.stringify(page)),
dump: () => Promise.resolve(),
})
}
if (url.includes('/batch_jobs/') && url.endsWith('/status')) {
statusPuts.push(parseJSON(opts.body) as { status: string })
return Promise.resolve({
status: 200,
headers: {},
json: () => Promise.resolve({}),
text: () => Promise.resolve('{}'),
dump: () => Promise.resolve(),
})
}
return Promise.reject(new Error(`Unexpected internalFetch call to ${url}`))
})
await supertest(app)
.post(`/api/projects/${team.id}/hog_flows/${flow.id}/batch_invocations/${parentRunId}`)
.send({ filters: { filter_test_accounts: false }, max_audience_size: 1000 })
.expect(200)
// Start the consumer — it'll pick up the resolver job and process pages
// on its own polling loop. waitForExpect waits for the terminal Django
// PUT, which only happens after all 3 pages + the terminal-write phase.
resolverWorker = buildResolverConsumer()
await resolverWorker.start()
await waitForExpect(() => {
expect(statusPuts).toHaveLength(1)
}, 20000)
expect(personPageCalls).toBe(3)
expect(statusPuts[0]).toEqual({ status: 'completed' })
const children = await cyclotronPool.query(
`SELECT id FROM cyclotron_jobs WHERE queue_name = 'hogflow' AND parent_run_id = $1`,
[parentRunId]
)
expect(children.rows).toHaveLength(personIds.length)
// Enrolling a person has to register as a started run at the same time as it enqueues
// the work. Both of these used to be skipped here (only the event-triggered path wrote
// them), which read on the workflow page as a batch that never started and had no runs
// to list — for the whole of any delay the flow was parked on.
await waitForExpect(() => {
const triggered = mockProducerObserver
.getProducedKafkaMessagesForTopic(KAFKA_APP_METRICS_2)
.filter((m: any) => m.value.app_source === 'hog_flow' && m.value.metric_name === 'triggered')
// Summed, not counted: the aggregator collapses a flush window's metrics onto one
// message per key, so a 3-page resolve produces fewer messages than persons.
expect(triggered.reduce((total, m: any) => total + m.value.count, 0)).toBe(personIds.length)
for (const metric of triggered) {
// Keyed on the batch run, which is what the batch metrics view queries. Keyed on
// the workflow id instead, the count is real but the page still reads zero.
expect((metric as any).value.app_source_id).toBe(parentRunId)
// Run-level, so no step id — the started and in-progress counters filter on the
// empty instance and would skip these otherwise.
expect((metric as any).value.instance_id).toBe('')
}
const rows = mockProducerObserver
.getProducedKafkaMessagesForTopic(KAFKA_HOG_INVOCATION_RESULTS)
.map((m: any) => m.value)
expect(rows).toHaveLength(personIds.length)
for (const row of rows) {
// hog_flow, not hog_function: the invocations API filters on this, so a
// misclassified row is written but never listed.
expect(row.function_kind).toBe('hog_flow')
expect(row.function_id).toBe(flow.id)
expect(row.parent_run_id).toBe(parentRunId)
expect(row.status).toBe('running')
}
}, 10000)
})
it('keeps the job lock and worker health during a slow audience fetch', async () => {
const flow = await insertActiveBatchFlow()
const parentRunId = new UUIDT().toString()
const personId = new UUIDT().toString()
const originalTimeout = hub.CDP_HOG_FLOW_BATCH_AUDIENCE_FETCH_TIMEOUT_MS
hub.CDP_HOG_FLOW_BATCH_AUDIENCE_FETCH_TIMEOUT_MS = 45_000
const audiencePage = Promise.withResolvers<void>()
const statusPuts: Array<{ status: string }> = []
let audienceCalls = 0
const janitor = new CyclotronV2Janitor({
pool: { dbUrl: CYCLOTRON_NODE_DB_URL },
stallTimeoutMs: 15_000,
maxTouchCount: 2,
cleanupGraceMs: 99_999_000,
cleanupBatchSize: 100,
cleanupIntervalMs: 60_000,
})
mockInternalFetch.mockImplementation(async (url: string, opts: any) => {
if (url.includes('/user_blast_radius_persons')) {
audienceCalls += 1
expect(opts.timeoutMs).toBe(45_000)
await audiencePage.promise
return {
status: 200,
headers: {},
json: () => Promise.resolve({}),
text: () =>
Promise.resolve(JSON.stringify({ users_affected: [personId], cursor: null, has_more: false })),
dump: async () => {},
}
}
if (url.includes('/batch_jobs/') && url.endsWith('/status')) {
statusPuts.push(parseJSON(opts.body) as { status: string })
return {
status: 200,
headers: {},
json: () => Promise.resolve({}),
text: () => Promise.resolve('{}'),
dump: async () => {},
}
}
throw new Error(`Unexpected internalFetch call to ${url}`)
})
try {
await supertest(app)
.post(`/api/projects/${team.id}/hog_flows/${flow.id}/batch_invocations/${parentRunId}`)
.send({ filters: { filter_test_accounts: false }, max_audience_size: 1000 })
.expect(200)
resolverWorker = buildResolverConsumer()
await resolverWorker.start()
await waitForExpect(() => expect(audienceCalls).toBe(1), 5000)
await new Promise((resolve) => setTimeout(resolve, 32_000))
expect(resolverWorker.isHealthy().status).toBe('ok')
const sweep = await janitor.runOnce()
expect(sweep.stalled).toBe(0)
expect(sweep.poisoned).toBe(0)
audiencePage.resolve()
await waitForExpect(() => expect(statusPuts).toEqual([{ status: 'completed' }]), 10_000)
expect(audienceCalls).toBe(1)
const jobs = await cyclotronPool.query(
'SELECT queue_name, janitor_touch_count FROM cyclotron_jobs WHERE parent_run_id = $1',
[parentRunId]
)
expect(jobs.rows.filter((job) => job.queue_name === 'hogflow')).toHaveLength(1)
for (const job of jobs.rows) {
expect(job.janitor_touch_count).toBe(0)
}
} finally {
audiencePage.resolve()
hub.CDP_HOG_FLOW_BATCH_AUDIENCE_FETCH_TIMEOUT_MS = originalTimeout
await janitor.stop()
}
}, 60_000)
// Regression test: batch-resolved invocations used to skip trigger_masking entirely
// (only the event-triggered pipeline applied it), so a scheduled batch workflow with
// a masking TTL re-enrolled the same audience on every run.
it('trigger_masking suppresses re-enrolling the same person across two batch runs', async () => {
const flow = await insertActiveBatchFlow(HOG_FLOW_MASK_EXAMPLES.oncePerTimePeriod.trigger_masking)
const personId = new UUIDT().toString()
const statusPuts: Array<{ status: string }> = []
mockInternalFetch.mockImplementation((url: string, opts: any) => {
if (url.includes('/user_blast_radius_persons')) {
return Promise.resolve({
status: 200,
headers: {},
json: () => Promise.resolve({}),
text: () =>
Promise.resolve(JSON.stringify({ users_affected: [personId], cursor: null, has_more: false })),
dump: () => Promise.resolve(),
})
}
if (url.includes('/batch_jobs/') && url.endsWith('/status')) {
statusPuts.push(parseJSON(opts.body) as { status: string })
return Promise.resolve({
status: 200,
headers: {},
json: () => Promise.resolve({}),
text: () => Promise.resolve('{}'),
dump: () => Promise.resolve(),
})
}
return Promise.reject(new Error(`Unexpected internalFetch call to ${url}`))
})
resolverWorker = buildResolverConsumer()
await resolverWorker.start()
// First run: the person hasn't been masked yet, so they get enrolled.
const firstRunId = new UUIDT().toString()
await supertest(app)
.post(`/api/projects/${team.id}/hog_flows/${flow.id}/batch_invocations/${firstRunId}`)
.send({ filters: { filter_test_accounts: false }, max_audience_size: 1000 })
.expect(200)
await waitForExpect(() => {
expect(statusPuts).toHaveLength(1)
}, 20000)
const firstRunChildren = await cyclotronPool.query(
`SELECT id FROM cyclotron_jobs WHERE queue_name = 'hogflow' AND parent_run_id = $1`,
[firstRunId]
)
expect(firstRunChildren.rows).toHaveLength(1)
// Second run (simulating the next scheduled re-enrollment): the same person is
// masked, so no child invocation should be enqueued this time.
const secondRunId = new UUIDT().toString()
await supertest(app)
.post(`/api/projects/${team.id}/hog_flows/${flow.id}/batch_invocations/${secondRunId}`)
.send({ filters: { filter_test_accounts: false }, max_audience_size: 1000 })
.expect(200)
await waitForExpect(() => {
expect(statusPuts).toHaveLength(2)
}, 20000)
const secondRunChildren = await cyclotronPool.query(
`SELECT id FROM cyclotron_jobs WHERE queue_name = 'hogflow' AND parent_run_id = $1`,
[secondRunId]
)
expect(secondRunChildren.rows).toHaveLength(0)
})
// The mask claims land in Redis before the page commits to Postgres. If the commit
// fails without releasing them, the stall-recovery replay of the same cursor sees
// the whole page as already-masked and silently drops every enrollment in it.
it('failed page commit releases mask claims, so the stall-recovery replay still enrolls', async () => {
const flow = await insertActiveBatchFlow(HOG_FLOW_MASK_EXAMPLES.oncePerTimePeriod.trigger_masking)
const personId = new UUIDT().toString()
const statusPuts: Array<{ status: string }> = []
mockInternalFetch.mockImplementation((url: string, opts: any) => {
if (url.includes('/user_blast_radius_persons')) {
return Promise.resolve({
status: 200,
headers: {},
json: () => Promise.resolve({}),
text: () =>
Promise.resolve(JSON.stringify({ users_affected: [personId], cursor: null, has_more: false })),
dump: () => Promise.resolve(),
})
}
if (url.includes('/batch_jobs/') && url.endsWith('/status')) {
statusPuts.push(parseJSON(opts.body) as { status: string })
return Promise.resolve({
status: 200,
headers: {},
json: () => Promise.resolve({}),
text: () => Promise.resolve('{}'),
dump: () => Promise.resolve(),
})
}
return Promise.reject(new Error(`Unexpected internalFetch call to ${url}`))
})
let commitFailuresLeft = 1
resolverWorker = buildResolverConsumer((job) => ({
...job,
bulkCreateAndCheckIn: (input) => {
if (commitFailuresLeft > 0) {
commitFailuresLeft--
return Promise.reject(new Error('injected commit failure'))
}
return job.bulkCreateAndCheckIn(input)
},
}))
await resolverWorker.start()
const parentRunId = new UUIDT().toString()
await supertest(app)
.post(`/api/projects/${team.id}/hog_flows/${flow.id}/batch_invocations/${parentRunId}`)
.send({ filters: { filter_test_accounts: false }, max_audience_size: 1000 })
.expect(200)
// The first attempt claims the person's mask slot, then hits the injected commit
// failure. Wait until the resolver has released the claim (counter back to 0) —
// only then is the job's parked state the same as a real stall.
const maskRedis = createRedisV2PoolFromConfig({
connection: hub.CDP_REDIS_HOST
? {
url: hub.CDP_REDIS_HOST,
options: { port: hub.CDP_REDIS_PORT, password: hub.CDP_REDIS_PASSWORD },
}
: { url: hub.REDIS_URL },
poolMinSize: hub.REDIS_POOL_MIN_SIZE,
poolMaxSize: hub.REDIS_POOL_MAX_SIZE,
})
await waitForExpect(async () => {
expect(commitFailuresLeft).toBe(0)
const maskCounters = await maskRedis.useClient({ name: 'mask-peek' }, async (client) => {
const keys = await client.keys(`@posthog-test/hog-masker/mask/${flow.id}/*`)
return Promise.all(keys.map((key) => client.get(key)))
})
expect(maskCounters).toEqual(['0'])
}, 20000)
// What the janitor's stall recovery does to the parked job, without waiting
// out the stall timeout.
await cyclotronPool.query(
`UPDATE cyclotron_jobs SET status = 'available', lock_id = NULL, last_heartbeat = NULL
WHERE queue_name = $1 AND parent_run_id = $2`,
[HOGFLOW_BATCH_RESOLVE_QUEUE, parentRunId]
)
await waitForExpect(() => {
expect(statusPuts).toEqual([{ status: 'completed' }])
}, 20000)
// The replay re-evaluated masking against the released claim and enrolled the
// person — without the release it would classify them as masked and enqueue nothing.
const children = await cyclotronPool.query(
`SELECT id FROM cyclotron_jobs WHERE queue_name = 'hogflow' AND parent_run_id = $1`,
[parentRunId]
)
expect(children.rows).toHaveLength(1)
})
it('failed lifecycle: workflow deleted before processing → resolver transitions to failed → Django PUT status=failed', async () => {
const flow = await insertActiveBatchFlow()
const parentRunId = new UUIDT().toString()
const statusPuts: Array<{ status: string }> = []
mockInternalFetch.mockImplementation((url: string, opts: any) => {
if (url.includes('/batch_jobs/') && url.endsWith('/status')) {
statusPuts.push(parseJSON(opts.body) as { status: string })
return Promise.resolve({
status: 200,
headers: {},
json: () => Promise.resolve({}),
text: () => Promise.resolve('{}'),
dump: () => Promise.resolve(),
})
}
// Audience fetch shouldn't be reached — the resolver bails on missing
// hogflow before getting that far.
return Promise.reject(new Error(`Unexpected internalFetch call to ${url}`))
})
// Dispatch BEFORE starting the consumer so we can sabotage the workflow
// before any processing happens (otherwise the consumer would race us
// and process the job before we delete the workflow).
await supertest(app)
.post(`/api/projects/${team.id}/hog_flows/${flow.id}/batch_invocations/${parentRunId}`)
.send({ filters: { filter_test_accounts: false }, max_audience_size: 1000 })
.expect(200)
// Sabotage: customer deletes the workflow between dispatch and processing.
await hub.postgres.query(
PostgresUse.COMMON_WRITE,
`DELETE FROM posthog_hogflow WHERE id = $1`,
[flow.id],
'delete-hogflow-for-test'
)
// The consumer we're about to build will have its own hogFlowManager
// with an empty cache, so refresh isn't strictly needed — but be
// explicit so the test isn't sensitive to cache lifecycle changes.
resolverWorker = buildResolverConsumer()
await resolverWorker.start()
await waitForExpect(() => {
expect(statusPuts).toHaveLength(1)
}, 20000)
expect(statusPuts[0]).toEqual({ status: 'failed' })
const children = await cyclotronPool.query(
`SELECT id FROM cyclotron_jobs WHERE queue_name = 'hogflow' AND parent_run_id = $1`,
[parentRunId]
)
expect(children.rows).toHaveLength(0)
})
it('Django down: resolver retries the terminal PUT instead of acking', async () => {
const flow = await insertActiveBatchFlow()
const parentRunId = new UUIDT().toString()
const personIds = [new UUIDT().toString()]
let putAttempts = 0
mockInternalFetch.mockImplementation((url: string) => {
if (url.includes('/user_blast_radius_persons')) {
return Promise.resolve({
status: 200,
headers: {},
json: () => Promise.resolve({}),
text: () =>
Promise.resolve(
JSON.stringify({
users_affected: personIds,
cursor: null,
has_more: false,
})
),
dump: () => Promise.resolve(),
})
}
if (url.includes('/batch_jobs/') && url.endsWith('/status')) {
putAttempts += 1
return Promise.resolve({
status: 503,
headers: {},
json: () => Promise.resolve({}),
text: () => Promise.resolve('Service Unavailable'),
dump: () => Promise.resolve(),
})
}
return Promise.reject(new Error(`Unexpected internalFetch call to ${url}`))
})
await supertest(app)
.post(`/api/projects/${team.id}/hog_flows/${flow.id}/batch_invocations/${parentRunId}`)
.send({ filters: { filter_test_accounts: false }, max_audience_size: 1000 })
.expect(200)
resolverWorker = buildResolverConsumer()
await resolverWorker.start()
// Wait until at least one Django PUT attempt has happened — proves the
// resolver reached the terminal-write phase. Then verify the resolver
// job is parked (status='available' with pendingTerminal still set),
// not acked.
await waitForExpect(() => {
expect(putAttempts).toBeGreaterThanOrEqual(1)
}, 20000)
const rows = await cyclotronPool.query<{ status: string; state: Buffer | null }>(
`SELECT status::text AS status, state FROM cyclotron_jobs
WHERE queue_name = 'hogflow_batch_resolve' AND parent_run_id = $1`,
[parentRunId]
)
expect(rows.rows).toHaveLength(1)
expect(rows.rows[0].status).toBe('available')
const state = parseJSON((rows.rows[0].state as Buffer).toString('utf-8')) as { pendingTerminal?: string }
expect(state.pendingTerminal).toBe('completed')
})
it('audience fetch failure: resolver reschedules with backoff, cursor unchanged, no children enqueued', async () => {
const flow = await insertActiveBatchFlow()
const parentRunId = new UUIDT().toString()
let fetchAttempts = 0
mockInternalFetch.mockImplementation((url: string) => {
if (url.includes('/user_blast_radius_persons')) {
fetchAttempts += 1
// Audience fetch always fails — resolver should keep retrying,
// never advance the cursor, never enqueue children.
return Promise.reject(new Error('The operation was aborted due to timeout'))
}
return Promise.reject(new Error(`Unexpected internalFetch call to ${url}`))
})
await supertest(app)
.post(`/api/projects/${team.id}/hog_flows/${flow.id}/batch_invocations/${parentRunId}`)
.send({ filters: { filter_test_accounts: false }, max_audience_size: 1000 })
.expect(200)
resolverWorker = buildResolverConsumer()
await resolverWorker.start()
await waitForExpect(() => {
expect(fetchAttempts).toBeGreaterThanOrEqual(1)
}, 20000)
const rows = await cyclotronPool.query<{ status: string; state: Buffer | null }>(
`SELECT status::text AS status, state FROM cyclotron_jobs
WHERE queue_name = 'hogflow_batch_resolve' AND parent_run_id = $1`,
[parentRunId]
)
expect(rows.rows).toHaveLength(1)
expect(rows.rows[0].status).toBe('available')
const state = parseJSON((rows.rows[0].state as Buffer).toString('utf-8')) as {
cursor: string | null
totalEnqueued: number
pagesProcessed: number
pendingTerminal?: string
}
expect(state.cursor).toBeNull() // never advanced
expect(state.totalEnqueued).toBe(0)
expect(state.pagesProcessed).toBe(0)
expect(state.pendingTerminal).toBeUndefined()
const children = await cyclotronPool.query(
`SELECT id FROM cyclotron_jobs WHERE queue_name = 'hogflow' AND parent_run_id = $1`,
[parentRunId]
)
expect(children.rows).toHaveLength(0)
})
it('truncation: pre-existing job at maxAudienceSize skips fetch + emits customer log + writes status=completed', async () => {
// Pre-populate a resolver job at the cap so the next dequeue takes
// the truncation branch (audience fetch never runs).
const flow = await insertActiveBatchFlow()
const parentRunId = new UUIDT().toString()
const statusPuts: Array<{ status: string }> = []
mockInternalFetch.mockImplementation((url: string, opts: any) => {
if (url.includes('/batch_jobs/') && url.endsWith('/status')) {
statusPuts.push(parseJSON(opts.body) as { status: string })
return Promise.resolve({
status: 200,
headers: {},
json: () => Promise.resolve({}),
text: () => Promise.resolve('{}'),
dump: () => Promise.resolve(),
})
}
return Promise.reject(new Error(`Unexpected internalFetch call to ${url}`))
})
const cappedState = serializeResolverState({
batchJobId: parentRunId,
teamId: team.id,
hogFlowId: flow.id,
filters: { properties: [], filter_test_accounts: false },
variables: {},
maxAudienceSize: 100,
cursor: 'some-cursor',
totalEnqueued: 100,
pagesProcessed: 1,
attempts: 0,
startedAt: new Date().toISOString(),
})
await batchResolverProducer.createJob({
teamId: team.id,
queueName: 'hogflow_batch_resolve',
parentRunId,
functionId: flow.id,
state: cappedState,
})
resolverWorker = buildResolverConsumer()
await resolverWorker.start()
await waitForExpect(() => {
expect(statusPuts).toHaveLength(1)
}, 20000)
// Status PUT was completed (truncation is still success, not failure)
expect(statusPuts[0]).toEqual({ status: 'completed' })
// No children — the resolver short-circuited before audience fetch
const children = await cyclotronPool.query(
`SELECT id FROM cyclotron_jobs WHERE queue_name = 'hogflow' AND parent_run_id = $1`,
[parentRunId]
)
expect(children.rows).toHaveLength(0)
})
it('persistent fetch failure exhausts MAX_RESOLVER_ATTEMPTS → flips to pendingTerminal=failed and writes status=failed', async () => {
// Pre-populate a resolver job already one retry below the cap so we
// only need to drive a single more fetch failure to cross the
// threshold — avoids burning 5x backoff windows in CI.
const flow = await insertActiveBatchFlow()
const parentRunId = new UUIDT().toString()
const statusPuts: Array<{ status: string }> = []
let fetchAttempts = 0
mockInternalFetch.mockImplementation((url: string, opts: any) => {
if (url.includes('/user_blast_radius_persons')) {
fetchAttempts += 1
return Promise.reject(new Error('ClickHouse permanently rejecting the audience query'))
}
if (url.includes('/batch_jobs/') && url.endsWith('/status')) {
statusPuts.push(parseJSON(opts.body) as { status: string })
return Promise.resolve({
status: 200,
headers: {},
json: () => Promise.resolve({}),
text: () => Promise.resolve('{}'),
dump: () => Promise.resolve(),
})
}
return Promise.reject(new Error(`Unexpected internalFetch call to ${url}`))
})
const seededState = serializeResolverState({
batchJobId: parentRunId,
teamId: team.id,
hogFlowId: flow.id,
filters: { properties: [], filter_test_accounts: false },
variables: {},
maxAudienceSize: 1000,
cursor: null,
totalEnqueued: 0,
pagesProcessed: 0,
attempts: MAX_RESOLVER_ATTEMPTS - 1,
startedAt: new Date().toISOString(),
})
await batchResolverProducer.createJob({
teamId: team.id,
queueName: HOGFLOW_BATCH_RESOLVE_QUEUE,
parentRunId,
functionId: flow.id,
state: seededState,
})
resolverWorker = buildResolverConsumer()
await resolverWorker.start()
await waitForExpect(() => {
expect(statusPuts).toHaveLength(1)
}, 20000)
expect(statusPuts[0]).toEqual({ status: 'failed' })
expect(fetchAttempts).toBe(1) // one more retry consumed the last attempt budget
// No children enqueued — the resolver bailed before ever returning a page
const children = await cyclotronPool.query(
`SELECT id FROM cyclotron_jobs WHERE queue_name = 'hogflow' AND parent_run_id = $1`,
[parentRunId]
)
expect(children.rows).toHaveLength(0)
})
it('persistent Django 4xx on terminal write exhausts MAX_RESOLVER_ATTEMPTS → job.fail()', async () => {
// Pre-populate a job already in the terminal-write phase with attempts
// one below the cap. The 404 response is a permanent failure — no
// amount of retrying will make Django accept it (the row is gone) —
// so the resolver should give up and fail the cyclotron job instead
// of looping forever.
const flow = await insertActiveBatchFlow()
const parentRunId = new UUIDT().toString()
let putAttempts = 0
mockInternalFetch.mockImplementation((url: string) => {
if (url.includes('/batch_jobs/') && url.endsWith('/status')) {
putAttempts += 1
return Promise.resolve({
status: 404,
headers: {},
json: () => Promise.resolve({}),
text: () => Promise.resolve('Not Found'),
dump: () => Promise.resolve(),
})
}
return Promise.reject(new Error(`Unexpected internalFetch call to ${url}`))
})
const pendingState = serializeResolverState({
batchJobId: parentRunId,
teamId: team.id,
hogFlowId: flow.id,
filters: { properties: [], filter_test_accounts: false },
variables: {},
maxAudienceSize: 1000,
cursor: 'last-cursor',
totalEnqueued: 5,
pagesProcessed: 1,
attempts: MAX_RESOLVER_ATTEMPTS - 1,
startedAt: new Date().toISOString(),
pendingTerminal: 'completed',
})
await batchResolverProducer.createJob({
teamId: team.id,
queueName: HOGFLOW_BATCH_RESOLVE_QUEUE,
parentRunId,
functionId: flow.id,
state: pendingState,
})
resolverWorker = buildResolverConsumer()
await resolverWorker.start()
await waitForExpect(async () => {
const r = await cyclotronPool.query<{ status: string }>(
`SELECT status::text AS status FROM cyclotron_jobs
WHERE queue_name = 'hogflow_batch_resolve' AND parent_run_id = $1`,
[parentRunId]
)
expect(r.rows[0]?.status).toBe('failed')
}, 20000)
// One more attempt consumed the budget, then the job failed
expect(putAttempts).toBe(1)
})
it('hard cap: page that would cross maxAudienceSize is truncated before enqueue', async () => {
// maxAudienceSize=4 with pages of 3. Without the hard cap the resolver
// would enqueue 6 children (overshoot by 2) then notice and truncate.
// With the hard cap the second page is truncated to 1 row so the
// total never exceeds 4.
const flow = await insertActiveBatchFlow()
const parentRunId = new UUIDT().toString()
const personIds = Array.from({ length: 6 }, () => new UUIDT().toString())
const statusPuts: Array<{ status: string }> = []
let personPageCalls = 0
mockInternalFetch.mockImplementation((url: string, opts: any) => {
if (url.includes('/user_blast_radius_persons')) {
personPageCalls += 1
const pages = [
{ users_affected: personIds.slice(0, 3), cursor: 'c1', has_more: true },
{ users_affected: personIds.slice(3, 6), cursor: 'c2', has_more: true },
]
const page = pages[Math.min(personPageCalls - 1, pages.length - 1)]
return Promise.resolve({
status: 200,
headers: {},
json: () => Promise.resolve({}),
text: () => Promise.resolve(JSON.stringify(page)),
dump: () => Promise.resolve(),
})
}
if (url.includes('/batch_jobs/') && url.endsWith('/status')) {
statusPuts.push(parseJSON(opts.body) as { status: string })
return Promise.resolve({
status: 200,
headers: {},
json: () => Promise.resolve({}),
text: () => Promise.resolve('{}'),
dump: () => Promise.resolve(),
})
}
return Promise.reject(new Error(`Unexpected internalFetch call to ${url}`))
})
await supertest(app)
.post(`/api/projects/${team.id}/hog_flows/${flow.id}/batch_invocations/${parentRunId}`)
.send({ filters: { filter_test_accounts: false }, max_audience_size: 4 })
.expect(200)
resolverWorker = buildResolverConsumer()
await resolverWorker.start()
await waitForExpect(() => {
expect(statusPuts).toHaveLength(1)
}, 20000)
expect(statusPuts[0]).toEqual({ status: 'completed' })
const children = await cyclotronPool.query(
`SELECT id FROM cyclotron_jobs WHERE queue_name = 'hogflow' AND parent_run_id = $1`,
[parentRunId]
)
// Hard cap: exactly 4 children, never 6
expect(children.rows).toHaveLength(4)
})
it('invalid resolver state: garbage bytes → job.fail() (not retry)', async () => {
// Insert a resolver job with state that doesn't match the Zod schema.
// Simulates schema drift, corruption, or a row written by an
// incompatible older deploy. The resolver must FAIL the job (terminal)
// rather than reschedule — otherwise we'd retry-loop forever on a job
// that can never succeed.
const parentRunId = new UUIDT().toString()
const malformedState = Buffer.from(JSON.stringify({ not: 'a valid resolver state' }))
await batchResolverProducer.createJob({
teamId: team.id,
queueName: 'hogflow_batch_resolve',
parentRunId,
functionId: new UUIDT().toString(),
state: malformedState,
})
mockInternalFetch.mockImplementation((url: string) =>
Promise.reject(new Error(`Unexpected internalFetch call to ${url}`))
)
resolverWorker = buildResolverConsumer()
await resolverWorker.start()
await waitForExpect(async () => {
const r = await cyclotronPool.query<{ status: string }>(
`SELECT status::text AS status FROM cyclotron_jobs
WHERE queue_name = 'hogflow_batch_resolve' AND parent_run_id = $1`,
[parentRunId]
)
expect(r.rows[0]?.status).toBe('failed')
}, 20000)
// No children, no Django PUT — the resolver failed before any work.
const children = await cyclotronPool.query(
`SELECT id FROM cyclotron_jobs WHERE queue_name = 'hogflow' AND parent_run_id = $1`,
[parentRunId]
)
expect(children.rows).toHaveLength(0)
})
})
// Janitor poison-pill recovery (postgres-v2). Exercises the REAL results service
// + real Kafka produce — the seam the mocked unit tests can't cover: an isolated
// poison pill is recorded as a failed, replayable result on hog_invocation_results
// BEFORE its cyclotron row is deleted, and with recovery disabled the janitor
// reverts to master's legacy path (marks the pill failed, no recovery record).
describe('Workflows E2E (janitor poison-pill recovery, postgres-v2)', () => {
jest.setTimeout(30000)
let hub: Hub
let kafkaProducer: KafkaProducerWrapper
let mockProducerObserver: KafkaProducerObserver
let team: Team
let cyclotronPool: Pool
let invocationResults: HogInvocationResultsService
let janitor: CyclotronV2Janitor | undefined
beforeAll(() => {
cyclotronPool = new Pool({ connectionString: CYCLOTRON_NODE_DB_URL })
})
afterAll(async () => {
await cyclotronPool.end()
})
beforeEach(async () => {
MockKafkaProducerWrapper.create = jest.fn((...args) => ActualKafkaProducerWrapper.create(...args))
// KAFKA_HOG_INVOCATION_RESULTS isn't in TEST_KAFKA_TOPICS — the janitor
// produces the give-up record there, so it must exist or the produce
// fails and the row is (correctly) never deleted.
await ensureKafkaTopics([...TEST_KAFKA_TOPICS, KAFKA_HOG_INVOCATION_RESULTS])
await resetTestDatabase()
await cyclotronPool.query('DELETE FROM cyclotron_jobs')
hub = await createHub()
// The give-up record path is gated on this flag (off by default outside dev).
hub.HOG_INVOCATION_RESULTS_ENABLED = true
kafkaProducer = await ActualKafkaProducerWrapper.create(hub.KAFKA_CLIENT_RACK)
mockProducerObserver = new KafkaProducerObserver(kafkaProducer)
team = await getFirstTeam(hub.postgres)
mockProducerObserver.resetKafkaProducer()
const deps = createCdpConsumerDeps(hub, kafkaProducer)
const outputs = createCdpOutputsRegistry().build(deps.cdpProducerRegistry, hub)
invocationResults = new HogInvocationResultsService(outputs, hub)
})
afterEach(async () => {
await janitor?.stop().catch(() => {})
janitor = undefined
await kafkaProducer.disconnect()
await closeHub(hub)
mockProducerObserver.resetKafkaProducer()
})
function createJanitor(overrides: Record<string, unknown> = {}): CyclotronV2Janitor {
return new CyclotronV2Janitor(
{
pool: { dbUrl: CYCLOTRON_NODE_DB_URL },
cleanupGraceMs: 0,
stallTimeoutMs: 1000,
maxTouchCount: 3,
...overrides,
},
invocationResults
)
}
// A hogflow job stuck 'running' with a long-stale heartbeat and a touch count
// over the budget. The serialized state mirrors a wait_until_condition that
// had already advanced past earlier actions.
async function insertPoisonedHogflowJob(): Promise<string> {
const id = new UUIDT().toString()
const state = Buffer.from(
JSON.stringify({
state: {
event: { uuid: 'evt-poison', distinct_id: 'd-poison' },
actionStepCount: 2,
variables: {},
currentAction: { id: 'wait_condition', startedAtTimestamp: 1 },
},
})
)
await cyclotronPool.query(
`INSERT INTO cyclotron_jobs
(id, team_id, function_id, queue_name, status, priority, scheduled, created,
lock_id, last_heartbeat, janitor_touch_count, transition_count, last_transition, state)
VALUES ($1, $2, $3, 'hogflow', 'running'::CyclotronJobStatus, 0, NOW(), NOW(),
$4, NOW() - INTERVAL '5 minutes', 5, 10, NOW(), $5)`,
[id, team.id, new UUIDT().toString(), new UUIDT().toString(), state]
)
return id
}
it('records an isolated poison pill as a failed, replayable result and only then deletes it', async () => {
const id = await insertPoisonedHogflowJob()
janitor = createJanitor()
const result = await janitor.runOnce()
expect(result.poisonedIds).toEqual([id])
// A failed lifecycle row reached the hog_invocation_results topic so the
// run is discoverable in the Invocations UI and replayable by rerun.
await waitForExpect(() => {
const rows = mockProducerObserver.getProducedKafkaMessagesForTopic(KAFKA_HOG_INVOCATION_RESULTS)
expect(rows.some((m: any) => m.value.invocation_id === id)).toBe(true)
}, 10000)
const row = mockProducerObserver
.getProducedKafkaMessagesForTopic(KAFKA_HOG_INVOCATION_RESULTS)
.find((m: any) => m.value.invocation_id === id)!.value
expect(row.status).toBe('failed')
expect(row.function_kind).toBe('hog_flow')
expect(row.function_id).toBeTruthy()
expect(row.error_kind).toBe(JANITOR_POISON_PILL_ERROR_KIND)
// The cyclotron row is gone — but only because the record exists first.
const remaining = await cyclotronPool.query('SELECT 1 FROM cyclotron_jobs WHERE id = $1', [id])
expect(remaining.rowCount).toBe(0)
})
it('reverts to legacy mark-failed (records nothing) when recovery is disabled', async () => {
const id = await insertPoisonedHogflowJob()
janitor = createJanitor({ poisonRecoveryEnabled: false })
const result = await janitor.runOnce()
// Kill-switch off → master's legacy path: mark the pill failed and produce
// no recovery record. The give-up is terminal (no infinite retry) but not
// replayable — exactly master's pre-recovery behavior.
expect(result.poisonedIds).toEqual([id])
expect(mockProducerObserver.getProducedKafkaMessagesForTopic(KAFKA_HOG_INVOCATION_RESULTS)).toHaveLength(0)
const rows = await cyclotronPool.query('SELECT status FROM cyclotron_jobs WHERE id = $1', [id])
expect(rows.rowCount).toBe(1)
expect(rows.rows[0].status).toBe('failed')
})
})