:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { MockKafkaProducerWrapper } from '~/tests/helpers/mocks/producer.mock'
import { mockFetch } from '~/tests/helpers/mocks/request.mock'
import { KafkaProducerObserver } from '~/tests/helpers/mocks/producer.spy'
import { DateTime } from 'luxon'
import { Pool } from 'pg'
import { KAFKA_HOG_INVOCATION_RESULTS } from '~/common/config/kafka-topics'
import { KafkaProducerWrapper } from '~/common/kafka/producer'
import { PersonReadRepository } from '~/common/persons/repositories/person-repository'
import { closeHub, createHub } from '~/common/utils/db/hub'
import { createCdpConsumerDeps } from '~/tests/helpers/cdp'
import { Clickhouse } from '~/tests/helpers/clickhouse'
import { waitForExpect } from '~/tests/helpers/expectations'
import { waitForHogInvocationResultsMvReady } from '~/tests/helpers/hog-invocation-results'
import { TEST_KAFKA_TOPICS, ensureKafkaTopics } from '~/tests/helpers/kafka'
import { getFirstTeam, resetTestDatabase } from '~/tests/helpers/sql'
import { Hub, Team } from '../../types'
import { FixtureHogFlowBuilder } from '../_tests/builders/hogflow.builder'
import { HOG_FILTERS_EXAMPLES, HOG_INPUTS_EXAMPLES } from '../_tests/examples'
import {
insertHogFunction as _insertHogFunction,
createHogExecutionGlobals,
insertHogFunctionTemplate,
} from '../_tests/fixtures'
import { insertHogFlow } from '../_tests/fixtures-hogflows'
import { CdpConsumerBaseDeps } from '../consumers/cdp-base.consumer'
import { CdpCyclotronWorkerHogFlow } from '../consumers/cdp-cyclotron-worker-hogflow.consumer'
import { CdpCyclotronWorker } from '../consumers/cdp-cyclotron-worker.consumer'
import { CdpEventsConsumer } from '../consumers/cdp-events.consumer'
import { CdpRerunWorkerConsumer } from '../consumers/cdp-rerun-worker.consumer'
import { CyclotronJobQueueKafka } from '../services/job-queue/job-queue-kafka'
import { CyclotronJobQueuePostgresV2 } from '../services/job-queue/job-queue-postgres-v2'
import { compileHog } from '../templates/compiler'
import { HogFunctionInvocationGlobals, HogFunctionType } from '../types'
import { RerunJobManager } from './rerun-job.manager'
import { RERUN_QUEUE_NAME } from './rerun-job.types'
const ActualKafkaProducerWrapper = jest.requireActual('~/common/kafka/producer').KafkaProducerWrapper
const NODE_DB_URL = 'postgres://posthog:posthog@localhost:5432/test_cyclotron_node'
interface PersistedRow {
invocation_id: string
status: string
is_retry: number
attempts: number
error_kind: string
function_kind: string
invocation_globals: string
}
/**
* End-to-end exercise of the runs/rerun pipeline.
*
* Everything is real: real Kafka, real ClickHouse Kafka MV, real cyclotron-v2
* postgres, real cyclotron worker, real rerun wrapper-job loop. The ONLY thing
* mocked is the inbound Django request — we call `RerunJobManager.enqueue(...)`
* directly with the same payload the Django view would proxy through.
*
* Flow under test:
* 1. Hog function runs -> producer writes a 'succeeded' lifecycle row to
* Kafka -> MV lands it in `hog_invocation_results`.
* 2. Simulated Django POST: `rerunManager.enqueue({ invocation_ids })`.
* 3. Real `CdpRerunWorkerConsumer` dequeues the wrapper job, queries the
* real `hog_invocation_results`, rehydrates globals (rebuilding inputs),
* and re-enqueues onto the regular cyclotron queue.
* 4. The real cyclotron worker picks up the rerun invocation and writes a
* second lifecycle row, this time with `is_retry=1` and `attempts > 1`.
*/
describe('CDP hog invocation rerun e2e', () => {
jest.setTimeout(60_000)
let hub: Hub
let kafkaProducer: KafkaProducerWrapper
let team: Team
let fnFetch: HogFunctionType
let globals: HogFunctionInvocationGlobals
let mockProducerObserver: KafkaProducerObserver
let eventsConsumer: CdpEventsConsumer
let cyclotronWorker: CdpCyclotronWorker
let hogflowWorker: CdpCyclotronWorkerHogFlow
let rerunManager: RerunJobManager
let rerunWorker: CdpRerunWorkerConsumer
let kafkaQueue: CyclotronJobQueueKafka
let postgresV2Queue: CyclotronJobQueuePostgresV2
let nodeAssertPool: Pool
let clickhouse: Clickhouse
let cdpDeps: CdpConsumerBaseDeps
beforeAll(() => {
clickhouse = Clickhouse.create()
})
afterAll(() => {
clickhouse?.close()
jest.useRealTimers()
})
beforeEach(async () => {
// Use a real KafkaProducerWrapper per the cdp-e2e pattern — gives us a
// real producer to observe via KafkaProducerObserver.
MockKafkaProducerWrapper.create = jest.fn((...args: any[]) => ActualKafkaProducerWrapper.create(...args))
// Ensure all topics exist (idempotently, without deleting) so the ClickHouse
// Kafka engine consumers keep their connections. Includes KAFKA_HOG_INVOCATION_RESULTS,
// which this test's MV needs but the shared set does not cover.
await ensureKafkaTopics([...TEST_KAFKA_TOPICS, KAFKA_HOG_INVOCATION_RESULTS])
await clickhouse.truncate('hog_invocation_results_data')
await waitForHogInvocationResultsMvReady(clickhouse)
await resetTestDatabase()
await clickhouse.truncate('hog_invocation_results_data')
hub = await createHub()
kafkaProducer = await ActualKafkaProducerWrapper.create(hub.KAFKA_CLIENT_RACK)
mockProducerObserver = new KafkaProducerObserver(kafkaProducer)
team = await getFirstTeam(hub.postgres)
// The rerun strips `person` from invocation_globals — the cyclotron
// worker reloads it via getCyclotronPerson(distinct_id). Provide a mock
// person so the rerun can resolve `{person}` in the function's inputs.
const mockPersonRepo: jest.Mocked<PersonReadRepository> = {
fetchPerson: jest.fn().mockResolvedValue(undefined),
fetchPersonsByDistinctIds: jest.fn().mockResolvedValue([
{
id: '1',
uuid: 'dd3d6f80-60ad-45c3-bd61-e2300f2ba7e1',
team_id: team.id,
properties: { email: '[email protected]' },
properties_last_updated_at: {},
properties_last_operation: null,
created_at: DateTime.utc(),
version: 1,
is_identified: true,
is_user_id: null,
last_seen_at: null,
distinct_id: 'distinct_id',
},
]),
fetchPersonsByPersonIds: jest.fn().mockResolvedValue([]),
fetchDistinctIdsForPersons: jest.fn().mockResolvedValue({}),
}
mockProducerObserver.resetKafkaProducer()
hub.CDP_FETCH_RETRIES = 0
hub.CDP_FETCH_BACKOFF_BASE_MS = 1
hub.CDP_CYCLOTRON_COMPRESS_KAFKA_DATA = true
hub.CYCLOTRON_NODE_DATABASE_URL = NODE_DB_URL
hub.HOG_INVOCATION_RESULTS_ENABLED = true
// Clean any stale rerun wrapper jobs from prior runs.
nodeAssertPool = new Pool({ connectionString: NODE_DB_URL })
await nodeAssertPool.query('DELETE FROM cyclotron_jobs WHERE queue_name = $1', [RERUN_QUEUE_NAME])
const hog = `
let res := fetch(inputs.url, {
'headers': { 'Content-Type': 'application/json' },
'body': inputs.body,
'method': inputs.method
});
print('Fetch response:', res);
`
fnFetch = await _insertHogFunction(hub.postgres, team.id, {
type: 'destination',
hog,
bytecode: await compileHog(hog),
inputs_schema: HOG_INPUTS_EXAMPLES.simple_fetch.inputs_schema ?? [],
inputs: HOG_INPUTS_EXAMPLES.simple_fetch.inputs,
...HOG_FILTERS_EXAMPLES.no_filters,
})
kafkaQueue = new CyclotronJobQueueKafka(hub.KAFKA_CLIENT_RACK, hub, hub.CONSUMER_BATCH_SIZE)
postgresV2Queue = new CyclotronJobQueuePostgresV2(hub.CONSUMER_BATCH_SIZE, hub)
cdpDeps = { ...createCdpConsumerDeps(hub, kafkaProducer), personRepository: mockPersonRepo }
eventsConsumer = new CdpEventsConsumer(hub, cdpDeps, {
hogQueue: kafkaQueue,
hogflowQueue: postgresV2Queue,
})
// We call processBatch directly — no need to actually join the kafka group.
// Stubbing keeps the test off Redpanda's stale-group-protocol coordinator,
// which otherwise fails the join with "Inconsistent group protocol".
eventsConsumer['kafkaConsumer'] = {
connect: jest.fn(),
disconnect: jest.fn(),
isHealthy: jest.fn(),
} as any
await eventsConsumer.start()
cyclotronWorker = new CdpCyclotronWorker(hub, cdpDeps, kafkaQueue)
await cyclotronWorker.start()
rerunManager = new RerunJobManager({ dbUrl: NODE_DB_URL, maxCount: 10000 })
await rerunManager.connect()
globals = createHogExecutionGlobals({
project: { id: team.id } as any,
event: {
uuid: '0d0ff14e-1b15-4afe-99e3-1ea83f0e3aab',
event: '$pageview',
properties: { $current_url: 'https://posthog.com' },
timestamp: '2026-05-10T09:00:00Z',
} as any,
})
})
afterEach(async () => {
await Promise.all([
eventsConsumer?.stop().catch(() => undefined),
cyclotronWorker?.stop().catch(() => undefined),
hogflowWorker?.stop().catch(() => undefined),
rerunWorker?.stop().catch(() => undefined),
rerunManager?.disconnect().catch(() => undefined),
])
await kafkaProducer?.disconnect()
await closeHub(hub)
await nodeAssertPool.end()
mockProducerObserver?.resetKafkaProducer()
})
it('produces a succeeded lifecycle row, then reruns the invocation via the wrapper-job pipeline', async () => {
// ── 1. Original invocation succeeds ─────────────────────────────────
mockFetch.mockResolvedValue({
status: 200,
json: () => Promise.resolve({ ok: true }),
text: () => Promise.resolve('{"ok":true}'),
headers: { 'Content-Type': 'application/json' },
dump: () => Promise.resolve(),
})
const { invocations } = await eventsConsumer.processBatch([globals])
expect(invocations).toHaveLength(1)
// Wait for the terminal 'succeeded' row to flow Kafka -> MV -> ClickHouse.
await waitForExpect(async () => {
const rows = await clickhouse.query<PersistedRow>(
`SELECT invocation_id, status, is_retry, attempts, error_kind, function_kind
FROM hog_invocation_results
WHERE team_id = ${team.id} AND function_id = '${fnFetch.id}' AND status = 'succeeded'`
)
expect(rows.length).toBeGreaterThanOrEqual(1)
}, 30_000)
const originalRows = await clickhouse.query<PersistedRow>(
`SELECT invocation_id, status, is_retry, attempts, error_kind, function_kind, invocation_globals
FROM hog_invocation_results
WHERE team_id = ${team.id} AND function_id = '${fnFetch.id}' AND status = 'succeeded'`
)
const originalInvocationId = originalRows[0].invocation_id
expect(originalRows[0].is_retry).toBe(0)
expect(originalRows[0].function_kind).toBe('hog_function')
// invocation_globals is stored gzip+base64'd, not raw JSON — base64
// never starts with `{`. The rerun below proves the round-trip: it can
// only rehydrate and re-run if `decodeInvocationGlobals` decompresses
// this value correctly.
expect(originalRows[0].invocation_globals.length).toBeGreaterThan(0)
expect(originalRows[0].invocation_globals.startsWith('{')).toBe(false)
// The prior cyclotron_jobs row is in a terminal 'completed' state by
// this point. The rerun path's `overwriteExisting: true` upsert
// (cyclotron-v2 ON CONFLICT) handles the PK collision without us
// having to manually delete the row.
// ── 2. Mimic Django POST /rerun — only the request itself is faked ──────
// The rerun request requires a time window; we use a wide one around
// "now" (the lifecycle row's scheduled_at = current time) and restrict
// to a specific invocation_id via the optional filter field.
const windowStart = new Date(Date.now() - 7 * 24 * 60 * 60 * 1000).toISOString()
const windowEnd = new Date(Date.now() + 24 * 60 * 60 * 1000).toISOString()
const rerunJobId = await rerunManager.enqueue(team.id, 'hog_function', fnFetch.id, {
filter: {
window_start: windowStart,
window_end: windowEnd,
status: ['succeeded'],
invocation_ids: [originalInvocationId],
},
})
// Wrapper job is real and durable in test_cyclotron_node postgres.
const wrapperRow = await nodeAssertPool.query(
'SELECT id, queue_name, status FROM cyclotron_jobs WHERE id = $1',
[rerunJobId]
)
expect(wrapperRow.rows[0]).toMatchObject({
id: rerunJobId,
queue_name: RERUN_QUEUE_NAME,
status: 'available',
})
// ── 3. Rerun worker drains the wrapper job ────────────────────────────────
rerunWorker = new CdpRerunWorkerConsumer(
{ ...hub, CDP_CYCLOTRON_JOB_QUEUE_CONSUMER_MODE: 'postgres' },
cdpDeps,
{ hog_function: kafkaQueue, hog_flow: postgresV2Queue }
)
await rerunWorker.start()
// ── 4. Wait for the wrapper job to complete ────────────────────────────────
await waitForExpect(async () => {
const res = await nodeAssertPool.query('SELECT status FROM cyclotron_jobs WHERE id = $1', [rerunJobId])
// 'completed' = the paginator marked done=true and the worker acked.
expect(res.rows[0]?.status).toBe('completed')
}, 30_000)
// ── 5. Reruned invocation flows through the real cyclotron worker ─────────
// The paginator emits a 'running' row at re-enqueue time and the worker
// emits the terminal row when the rerun invocation completes. Both
// rows now correctly carry `is_retry=1` / `attempts=1` because the
// paginator sets `state.rerunAttempts=1` on the rehydrated invocation
// and `queueLifecycleRow` derives those columns from that field.
await waitForExpect(async () => {
const rows = await clickhouse.query<PersistedRow>(
`SELECT invocation_id, status, is_retry, attempts, error_kind, function_kind
FROM hog_invocation_results
WHERE team_id = ${team.id}
AND function_id = '${fnFetch.id}'
AND invocation_id = '${originalInvocationId}'
AND is_retry = 1`
)
// Either the paginator's running row, the worker's terminal row,
// or both should be present and tagged is_retry=1.
expect(rows.length).toBeGreaterThanOrEqual(1)
for (const row of rows) {
expect(row.attempts).toBe(1)
}
}, 30_000)
// ── 6. The hog function's fetch was called twice — once original, once rerun ─
// The wait above can pass on the paginator's running row, before the worker executes; poll for the fetch.
await waitForExpect(() => {
expect(mockFetch.mock.calls.length).toBeGreaterThanOrEqual(2)
}, 30_000)
})
it('reruns successfully when person rehydration misses and the input reads person.properties', async () => {
// On the rerun path the paginator strips globals.person; the worker
// re-hydrates via getCyclotronPerson, which returns null when the
// distinct_id has no row in posthog_persondistinctid (cookieless
// events by design, deleted persons, etc.). The worker must fall back
// to an empty person stub — leaving globals.person as undefined halts
// any input bytecode dereferencing `person.properties.*` with "Could
// not execute bytecode", before coalesce fallbacks can traverse to an
// event-level branch.
//
// If the worker's null-lookup fallback regresses, this test's rerun
// fetch never fires and the count assertion fails.
// Custom function whose input specifically dereferences
// `person.properties.foo` — the shape that crashed pre-fix.
const FALLBACK_WEBHOOK_URL = 'https://example.com/person-props-fallback'
const gclidLikeHog = `
let res := fetch(inputs.url, { 'method': 'POST', 'body': f'foo={inputs.foo}' });
print('Fetch response:', res);
`
const personPropsFn = await _insertHogFunction(hub.postgres, team.id, {
type: 'destination',
hog: gclidLikeHog,
bytecode: await compileHog(gclidLikeHog),
inputs_schema: [
{ key: 'url', type: 'string', label: 'URL', secret: false, required: true },
{ key: 'foo', type: 'string', label: 'foo', secret: false, required: false },
],
inputs: {
url: {
value: FALLBACK_WEBHOOK_URL,
bytecode: ['_h', 32, FALLBACK_WEBHOOK_URL],
},
// Same shape as the Google Ads template: person.properties first,
// event.properties as the tail fallback.
foo: {
value: '{person.properties.foo ?? event.properties.foo}',
bytecode: await compileHog('return person.properties.foo ?? event.properties.foo'),
},
},
...HOG_FILTERS_EXAMPLES.no_filters,
})
mockFetch.mockResolvedValue({
status: 200,
json: () => Promise.resolve({ ok: true }),
text: () => Promise.resolve('{"ok":true}'),
headers: { 'Content-Type': 'application/json' },
dump: () => Promise.resolve(),
})
// Simulate cookieless / missing-person for the rerun: personRepo returns
// no rows for any distinct_id, so getCyclotronPerson returns null. The
// ORIGINAL invocation doesn't call the repo — events consumer uses the
// person we attach to data.person directly — so this only affects the
// rerun path's rehydration.
const personRepo = cdpDeps.personRepository as jest.Mocked<PersonReadRepository>
personRepo.fetchPersonsByDistinctIds.mockResolvedValue([])
// Original event carries the fallback value on event.properties. The
// person we hand-attach here has no `foo`, so even on the original the
// coalesce falls through to event.properties.foo — proving the
// fallback wiring is real (and letting the rerun's stubbed-empty
// person hit the same branch).
const cookielessGlobals = createHogExecutionGlobals({
project: { id: team.id } as any,
event: {
uuid: 'aaaa1111-bbbb-2222-cccc-333333333333',
event: '$pageview',
distinct_id: 'cookieless_stub',
properties: { foo: 'from-event' },
timestamp: '2026-07-02T09:00:00Z',
} as any,
person: {
id: '',
name: 'cookieless_stub',
url: 'http://localhost:8000/persons/cookieless_stub',
properties: {}, // no `foo` — coalesce must reach event branch
},
})
const { invocations } = await eventsConsumer.processBatch([cookielessGlobals])
// Both fnFetch (from beforeEach) and personPropsFn match a $pageview — we
// care about ours.
const personPropsInvocations = invocations.filter((i) => i.functionId === personPropsFn.id)
expect(personPropsInvocations).toHaveLength(1)
await waitForExpect(async () => {
const rows = await clickhouse.query<PersistedRow>(
`SELECT invocation_id, status FROM hog_invocation_results
WHERE team_id = ${team.id} AND function_id = '${personPropsFn.id}' AND status = 'succeeded'`
)
expect(rows.length).toBeGreaterThanOrEqual(1)
}, 30_000)
const originalRows = await clickhouse.query<PersistedRow>(
`SELECT invocation_id, status FROM hog_invocation_results
WHERE team_id = ${team.id} AND function_id = '${personPropsFn.id}' AND status = 'succeeded'`
)
const originalInvocationId = originalRows[0].invocation_id
// Trigger the rerun scoped to just this invocation_id.
const windowStart = new Date(Date.now() - 7 * 24 * 60 * 60 * 1000).toISOString()
const windowEnd = new Date(Date.now() + 24 * 60 * 60 * 1000).toISOString()
const rerunJobId = await rerunManager.enqueue(team.id, 'hog_function', personPropsFn.id, {
filter: {
window_start: windowStart,
window_end: windowEnd,
status: ['succeeded'],
invocation_ids: [originalInvocationId],
},
})
rerunWorker = new CdpRerunWorkerConsumer(
{ ...hub, CDP_CYCLOTRON_JOB_QUEUE_CONSUMER_MODE: 'postgres' },
cdpDeps,
{ hog_function: kafkaQueue, hog_flow: postgresV2Queue }
)
await rerunWorker.start()
await waitForExpect(async () => {
const res = await nodeAssertPool.query('SELECT status FROM cyclotron_jobs WHERE id = $1', [rerunJobId])
expect(res.rows[0]?.status).toBe('completed')
}, 30_000)
// The core assertion: the rerun re-fired the fetch to our specific URL,
// with the value coming through the coalesce's event-level fallback.
// Pre-fix, the input eval crashed on `undefined.properties` and this
// count stayed at 1 (just the original). Post-fix, it's 2 (original +
// rerun), and the rerun's body carries `foo=from-event`.
// Job 'completed' only means the paginator finished re-enqueueing; the worker executes asynchronously, so poll.
await waitForExpect(() => {
expect(
mockFetch.mock.calls.filter(([url]) => String(url).startsWith(FALLBACK_WEBHOOK_URL)).length
).toBeGreaterThanOrEqual(2)
}, 30_000)
const ourCalls = mockFetch.mock.calls.filter(([url]) => String(url).startsWith(FALLBACK_WEBHOOK_URL))
const rerunBody = String((ourCalls[ourCalls.length - 1] as any)[1]?.body ?? '')
expect(rerunBody).toContain('foo=from-event')
})
it('reruns a multi-action hog flow by resuming, not restarting, so completed actions do not re-fire', async () => {
// The hog-flow counterpart to the hog_function reruns above, and the core
// guard for #70792: a hog_function rerun re-executes the whole function
// (fetch fires again, by design), but a hog FLOW rerun must RESUME from
// where the recorded run left off — never restart from the trigger and
// re-run already-completed actions. A completed flow's recorded state
// sits at `exit`, so its rerun resumes there and the fetch action never
// fires a second time. Pre-#70792 the rehydration dropped `currentAction`,
// the flow restarted from the trigger, and the fetch re-fired — the
// double-send this proves is gone. Distinct URL isolates the flow's fetch
// from the beforeEach hog_function that also matches this event.
const FLOW_WEBHOOK_URL = 'https://example.com/flow-resume-webhook'
mockFetch.mockResolvedValue({
status: 200,
json: () => Promise.resolve({ ok: true }),
text: () => Promise.resolve('{"ok":true}'),
headers: { 'Content-Type': 'application/json' },
dump: () => Promise.resolve(),
})
// The flow's function action runs a template, not inline hog — insert the same
// fetch template the workflows-e2e suite uses.
await insertHogFunctionTemplate(hub.postgres, {
id: 'template-rerun-e2e-fetch',
name: 'Rerun E2E Fetch',
code: `
let res := fetch(inputs.url, {'method': inputs.method});
print('Fetch result:', res.status);
`,
inputs_schema: [
{ key: 'url', type: 'string', required: true },
{ key: 'method', type: 'string', required: false },
],
})
const flow = new FixtureHogFlowBuilder()
.withTeamId(team.id)
.withStatus('active')
.withWorkflow({
actions: {
trigger: {
type: 'trigger',
config: { type: 'event', filters: HOG_FILTERS_EXAMPLES.no_filters.filters ?? {} },
},
function_1: {
type: 'function',
config: {
template_id: 'template-rerun-e2e-fetch',
inputs: { url: { value: FLOW_WEBHOOK_URL }, method: { value: 'POST' } },
},
},
exit: { type: 'exit', config: {} },
},
edges: [
{ from: 'trigger', to: 'function_1', type: 'continue' },
{ from: 'function_1', to: 'exit', type: 'continue' },
],
})
.build()
await insertHogFlow(hub.postgres, flow)
// The hog-flow worker polls the same postgres-v2 backend the events consumer
// routes flows to (hogflowQueue in beforeEach).
hogflowWorker = new CdpCyclotronWorkerHogFlow(hub, cdpDeps, postgresV2Queue)
await hogflowWorker.start()
// ── 1. Original flow runs to completion — fetch fires exactly once ──────────
const { backgroundTask } = await eventsConsumer.processBatch([globals])
await backgroundTask
await waitForExpect(() => {
expect(mockFetch.mock.calls.filter(([url]) => String(url) === FLOW_WEBHOOK_URL).length).toBe(1)
}, 15_000)
// Terminal 'succeeded' hog_flow row lands in ClickHouse via Kafka -> MV.
await waitForExpect(async () => {
const rows = await clickhouse.query<PersistedRow>(
`SELECT invocation_id, status, is_retry, function_kind
FROM hog_invocation_results
WHERE team_id = ${team.id} AND function_id = '${flow.id}' AND status = 'succeeded'`
)
expect(rows.length).toBeGreaterThanOrEqual(1)
}, 30_000)
const originalRows = await clickhouse.query<PersistedRow>(
`SELECT invocation_id, status, is_retry, function_kind
FROM hog_invocation_results
WHERE team_id = ${team.id} AND function_id = '${flow.id}' AND status = 'succeeded'`
)
expect(originalRows[0].function_kind).toBe('hog_flow')
expect(originalRows[0].is_retry).toBe(0)
const originalInvocationId = originalRows[0].invocation_id
// ── 2. Mimic Django POST /rerun for the flow, scoped to this invocation ─────
const windowStart = new Date(Date.now() - 7 * 24 * 60 * 60 * 1000).toISOString()
const windowEnd = new Date(Date.now() + 24 * 60 * 60 * 1000).toISOString()
const rerunJobId = await rerunManager.enqueue(team.id, 'hog_flow', flow.id, {
filter: {
window_start: windowStart,
window_end: windowEnd,
status: ['succeeded'],
invocation_ids: [originalInvocationId],
},
})
// ── 3. Rerun worker drains the wrapper job (hog_flow -> postgres-v2) ────────
rerunWorker = new CdpRerunWorkerConsumer(
{ ...hub, CDP_CYCLOTRON_JOB_QUEUE_CONSUMER_MODE: 'postgres' },
cdpDeps,
{ hog_function: kafkaQueue, hog_flow: postgresV2Queue }
)
await rerunWorker.start()
await waitForExpect(async () => {
const res = await nodeAssertPool.query('SELECT status FROM cyclotron_jobs WHERE id = $1', [rerunJobId])
expect(res.rows[0]?.status).toBe('completed')
}, 30_000)
// ── 4. Rerun invocation flows back through the real hog-flow worker ─────────
// Wait for the rerun's terminal row (is_retry=1) so we know it fully executed
// — by this point it would have re-fired the fetch if it were restarting.
await waitForExpect(async () => {
const rows = await clickhouse.query<PersistedRow>(
`SELECT invocation_id, status, is_retry, function_kind
FROM hog_invocation_results
WHERE team_id = ${team.id}
AND function_id = '${flow.id}'
AND invocation_id = '${originalInvocationId}'
AND is_retry = 1`
)
expect(rows.length).toBeGreaterThanOrEqual(1)
}, 30_000)
// ── 5. The resume ran nothing already done: the fetch is still at ONE call ──
// Restart-from-trigger (the pre-#70792 bug) would make this 2.
expect(mockFetch.mock.calls.filter(([url]) => String(url) === FLOW_WEBHOOK_URL).length).toBe(1)
})
})