:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { IngestionOutputs } from '~/common/outputs/ingestion-outputs'
import { parseJSON } from '~/common/utils/json-parse'
import { WAREHOUSE_SOURCE_WEBHOOKS_OUTPUT, WarehouseSourceWebhooksOutput } from '../../outputs/outputs'
import { CyclotronJobInvocationResult, WarehouseWebhookPayload } from '../../types'
import { WarehouseWebhooksService } from './warehouse-webhooks.service'
const buildPayload = (overrides: Partial<WarehouseWebhookPayload> = {}): WarehouseWebhookPayload => ({
team_id: 1,
schema_id: 'schema-a',
payload: { foo: 'bar' },
...overrides,
})
const buildResult = (warehouseWebhookPayloads: WarehouseWebhookPayload[]): CyclotronJobInvocationResult =>
({ warehouseWebhookPayloads }) as unknown as CyclotronJobInvocationResult
describe('WarehouseWebhooksService', () => {
let outputs: jest.Mocked<IngestionOutputs<WarehouseSourceWebhooksOutput>>
let service: WarehouseWebhooksService
beforeEach(() => {
outputs = {
produce: jest.fn().mockResolvedValue(undefined),
} as unknown as jest.Mocked<IngestionOutputs<WarehouseSourceWebhooksOutput>>
service = new WarehouseWebhooksService(outputs)
})
describe('queue + flush', () => {
it('produces queued payloads on flush, encoded with key=team:schema and JSON-serialised value', async () => {
const payload = buildPayload({ team_id: 42, schema_id: 'schema-x', payload: { a: 1, nested: { b: 2 } } })
service.queue([payload])
await service.flush()
expect(outputs.produce).toHaveBeenCalledTimes(1)
const call = outputs.produce.mock.calls[0]
expect(call[0]).toBe(WAREHOUSE_SOURCE_WEBHOOKS_OUTPUT)
expect(call[1].key).toEqual(Buffer.from('42:schema-x'))
expect(parseJSON((call[1].value as Buffer).toString())).toEqual({
schema_id: 'schema-x',
team_id: 42,
payload: JSON.stringify({ a: 1, nested: { b: 2 } }),
})
})
it('produces one message per queued payload', async () => {
service.queue([buildPayload({ schema_id: 'a' }), buildPayload({ schema_id: 'b' })])
service.queue([buildPayload({ schema_id: 'c' })])
await service.flush()
expect(outputs.produce).toHaveBeenCalledTimes(3)
})
it('clears the buffer after flush', async () => {
service.queue([buildPayload()])
await service.flush()
await service.flush()
expect(outputs.produce).toHaveBeenCalledTimes(1)
})
it('queue([]) is a no-op', async () => {
service.queue([])
await service.flush()
expect(outputs.produce).not.toHaveBeenCalled()
})
it('flush with empty buffer does not call outputs.produce', async () => {
await service.flush()
expect(outputs.produce).not.toHaveBeenCalled()
})
it('swallows produce errors so one bad payload does not block the rest', async () => {
outputs.produce.mockRejectedValueOnce(new Error('kafka down'))
service.queue([buildPayload({ schema_id: 'a' }), buildPayload({ schema_id: 'b' })])
await expect(service.flush()).resolves.toBeUndefined()
expect(outputs.produce).toHaveBeenCalledTimes(2)
})
})
describe('queueInvocationResults', () => {
it('extracts warehouseWebhookPayloads from each result and queues them', async () => {
const p1 = buildPayload({ schema_id: 'a' })
const p2 = buildPayload({ schema_id: 'b' })
const p3 = buildPayload({ schema_id: 'c' })
service.queueInvocationResults([buildResult([p1, p2]), buildResult([p3])])
await service.flush()
expect(outputs.produce).toHaveBeenCalledTimes(3)
})
it('skips results with empty / undefined warehouseWebhookPayloads', async () => {
service.queueInvocationResults([
{ warehouseWebhookPayloads: [] } as unknown as CyclotronJobInvocationResult,
{ warehouseWebhookPayloads: undefined } as unknown as CyclotronJobInvocationResult,
])
await service.flush()
expect(outputs.produce).not.toHaveBeenCalled()
})
it('handles an empty result array', async () => {
service.queueInvocationResults([])
await service.flush()
expect(outputs.produce).not.toHaveBeenCalled()
})
})
})