:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { Message } from 'node-rdkafka'
import { logger } from '~/common/utils/logger'
import { createMockPipeline } from '~/tests/helpers/mock-pipeline'
import { BaseChunkPipeline } from './base-chunk-pipeline'
import { BatchBudget } from './batch-budget'
import {
aggregateKafkaDebugContexts,
createBatch,
createContext,
createKafkaDebugContext,
createNewChunkPipeline,
createOkContext,
} from './helpers'
import { batchBudgetCheckpointCounter } from './metrics'
import { getBudgetCheckpoints } from './metrics.test-utils'
import { dlq, drop, isTimeoutResult, ok } from './results'
jest.mock('~/common/utils/logger', () => ({
logger: { debug: jest.fn(), info: jest.fn(), warn: jest.fn(), error: jest.fn() },
}))
function createTestMessage(overrides: Partial<Message> = {}): Message {
return {
value: Buffer.from('test'),
topic: 'test',
partition: 0,
offset: 1,
key: Buffer.from('key1'),
size: 4,
timestamp: Date.now(),
headers: [],
...overrides,
}
}
describe('BaseChunkPipeline', () => {
beforeEach(() => {
batchBudgetCheckpointCounter.reset()
})
describe('basic functionality', () => {
it('times out the exhausted elements of a mixed-budget chunk and runs the step on the rest', async () => {
const exhausted = BatchBudget.softDeadline(Date.now() - 1)
const healthy = BatchBudget.softDeadline(Date.now() + 5000)
let seen: { id: number }[] = []
const rootPipeline = createNewChunkPipeline<{ id: number }, { budget: BatchBudget }>().build()
const pipeline = new BaseChunkPipeline(function enrichChunk(items: { id: number }[]) {
seen = items
return Promise.resolve(items.map((item) => ok({ id: item.id * 10 })))
}, rootPipeline)
pipeline.feed([
createOkContext({ id: 1 }, { budget: exhausted }),
createOkContext({ id: 2 }, { budget: healthy }),
])
const results = await pipeline.next()
expect(seen).toEqual([{ id: 2 }])
expect(isTimeoutResult(results![0].result)).toBe(true)
expect((results![0].result as { reason: string }).reason).toBe('budget exceeded before enrichChunk')
expect(results![1].result).toEqual(ok({ id: 20 }))
expect(await getBudgetCheckpoints('chunk', 'enrichChunk')).toBe(1)
})
it('should process batch through pipeline', async () => {
const messages: Message[] = [
createTestMessage({ value: Buffer.from('test1'), offset: 1 }),
createTestMessage({ value: Buffer.from('test2'), offset: 2 }),
]
const batch = createBatch(messages.map((message) => ({ message })))
const rootPipeline = createNewChunkPipeline().build()
const pipeline = new BaseChunkPipeline((items: any[]) => {
return Promise.resolve(items.map((item: any) => ok({ processed: item.message.value?.toString() })))
}, rootPipeline)
pipeline.feed(batch)
const results = await pipeline.next()
expect(results).toEqual([
createContext(ok({ processed: 'test1' }), {
message: messages[0],
debugContext: createKafkaDebugContext(messages[0]),
lastStep: 'anonymousChunkStep',
}),
createContext(ok({ processed: 'test2' }), {
message: messages[1],
debugContext: createKafkaDebugContext(messages[1]),
lastStep: 'anonymousChunkStep',
}),
])
})
it('should handle empty batch', async () => {
const rootPipeline = createNewChunkPipeline().build()
const pipeline = new BaseChunkPipeline((items: any[]) => {
return Promise.resolve(items.map((item: any) => ok(item)))
}, rootPipeline)
pipeline.feed([])
const results = await pipeline.next()
expect(results).toEqual(null)
})
})
describe('chunk operations', () => {
it('should execute chunk step on all successful values', async () => {
const messages: Message[] = [
createTestMessage({ value: Buffer.from('1'), offset: 1 }),
createTestMessage({ value: Buffer.from('2'), offset: 2 }),
createTestMessage({ value: Buffer.from('3'), offset: 3 }),
]
const batch = createBatch(messages.map((message) => ({ message })))
const rootPipeline = createNewChunkPipeline().build()
const pipeline = new BaseChunkPipeline((items: any[]) => {
return Promise.resolve(
items.map((item: any) => ok({ count: parseInt(item.message.value?.toString() || '0') * 2 }))
)
}, rootPipeline)
pipeline.feed(batch)
const results = await pipeline.next()
expect(results).toEqual([
createContext(ok({ count: 2 }), {
message: messages[0],
debugContext: createKafkaDebugContext(messages[0]),
lastStep: 'anonymousChunkStep',
}),
createContext(ok({ count: 4 }), {
message: messages[1],
debugContext: createKafkaDebugContext(messages[1]),
lastStep: 'anonymousChunkStep',
}),
createContext(ok({ count: 6 }), {
message: messages[2],
debugContext: createKafkaDebugContext(messages[2]),
lastStep: 'anonymousChunkStep',
}),
])
})
it('should preserve non-success results and only process successful ones', async () => {
const messages: Message[] = [
createTestMessage({ value: Buffer.from('1'), offset: 1 }),
createTestMessage({ value: Buffer.from('drop'), offset: 2 }),
createTestMessage({ value: Buffer.from('3'), offset: 3 }),
createTestMessage({ value: Buffer.from('dlq'), offset: 4 }),
]
const batch = createBatch(messages.map((message) => ({ message })))
const rootPipeline = createNewChunkPipeline().build()
const firstPipeline = new BaseChunkPipeline((items: any[]) => {
return Promise.resolve(
items.map((item: any) => {
const value = item.message.value?.toString() || ''
if (value === 'drop') {
return drop('dropped item')
}
if (value === 'dlq') {
return dlq('dlq item', new Error('test error'))
}
return ok({ count: parseInt(value) })
})
)
}, rootPipeline)
const secondPipeline = new BaseChunkPipeline((items: any[]) => {
expect(items).toEqual([{ count: 1 }, { count: 3 }])
return Promise.resolve(items.map((item: any) => ok({ count: item.count * 2 })))
}, firstPipeline)
secondPipeline.feed(batch)
const results = await secondPipeline.next()
expect(results).toEqual([
createContext(ok({ count: 2 }), {
message: messages[0],
debugContext: createKafkaDebugContext(messages[0]),
lastStep: 'anonymousChunkStep',
}),
createContext(drop('dropped item'), {
message: messages[1],
debugContext: createKafkaDebugContext(messages[1]),
lastStep: 'anonymousChunkStep',
}),
createContext(ok({ count: 6 }), {
message: messages[2],
debugContext: createKafkaDebugContext(messages[2]),
lastStep: 'anonymousChunkStep',
}),
createContext(dlq('dlq item', new Error('test error')), {
message: messages[3],
debugContext: createKafkaDebugContext(messages[3]),
lastStep: 'anonymousChunkStep',
}),
])
})
})
describe('error handling', () => {
it('should propagate errors from chunk operations and log aggregated debug contexts', async () => {
const messages: Message[] = [
createTestMessage({ value: Buffer.from('1'), offset: 1 }),
createTestMessage({ value: Buffer.from('2'), offset: 2 }),
createTestMessage({ value: Buffer.from('3'), offset: 5 }),
]
const batch = createBatch(messages.map((message) => ({ message })))
const rootPipeline = createNewChunkPipeline().build()
const pipeline = new BaseChunkPipeline(
() => {
return Promise.reject(new Error('Chunk step failed'))
},
rootPipeline,
{ aggregateDebugContexts: aggregateKafkaDebugContexts }
)
pipeline.feed(batch)
await expect(pipeline.next()).rejects.toThrow('Chunk step failed')
expect(logger.error).toHaveBeenCalledWith(
'🔥',
expect.stringContaining('threw'),
expect.objectContaining({
error: 'Chunk step failed',
debugContext: { count: 3, offsets: ['test[0]:1-2,5'] },
})
)
})
})
describe('step name tracking', () => {
it('should include step name in context for successful results', async () => {
const messages: Message[] = [
createTestMessage({ value: Buffer.from('test1'), offset: 1 }),
createTestMessage({ value: Buffer.from('test2'), offset: 2 }),
]
const batch = createBatch(messages.map((message) => ({ message })))
const rootPipeline = createNewChunkPipeline().build()
function testChunkStep(items: any[]) {
return Promise.resolve(items.map((item: any) => ok({ processed: item.message.value?.toString() })))
}
const pipeline = new BaseChunkPipeline(testChunkStep, rootPipeline)
pipeline.feed(batch)
const results = await pipeline.next()
expect(results).toEqual([
createContext(ok({ processed: 'test1' }), {
message: messages[0],
debugContext: createKafkaDebugContext(messages[0]),
lastStep: 'testChunkStep',
}),
createContext(ok({ processed: 'test2' }), {
message: messages[1],
debugContext: createKafkaDebugContext(messages[1]),
lastStep: 'testChunkStep',
}),
])
})
it('should use anonymousChunkStep when step has no name', async () => {
const messages: Message[] = [createTestMessage({ value: Buffer.from('test1'), offset: 1 })]
const batch = createBatch(messages.map((message) => ({ message })))
const rootPipeline = createNewChunkPipeline().build()
const anonymousStep = (items: any[]) => {
return Promise.resolve(items.map((item: any) => ok({ processed: item.message.value?.toString() })))
}
const pipeline = new BaseChunkPipeline(anonymousStep, rootPipeline)
pipeline.feed(batch)
const results = await pipeline.next()
expect(results).toEqual([
createContext(ok({ processed: 'test1' }), {
message: messages[0],
debugContext: createKafkaDebugContext(messages[0]),
lastStep: 'anonymousStep',
}),
])
})
it('should not update lastStep for failed results', async () => {
const messages: Message[] = [
createTestMessage({ value: Buffer.from('test1'), offset: 1 }),
createTestMessage({ value: Buffer.from('drop'), offset: 2 }),
]
const batch = createBatch(messages.map((message) => ({ message })))
const rootPipeline = createNewChunkPipeline().build()
function testChunkStep(items: any[]) {
return Promise.resolve(
items.map((item: any) => {
const value = item.message.value?.toString() || ''
if (value === 'drop') {
return drop('dropped item')
}
return ok({ processed: value })
})
)
}
const pipeline = new BaseChunkPipeline(testChunkStep, rootPipeline)
pipeline.feed(batch)
const results = await pipeline.next()
expect(results).toEqual([
createContext(ok({ processed: 'test1' }), {
message: messages[0],
debugContext: createKafkaDebugContext(messages[0]),
lastStep: 'testChunkStep',
}),
createContext(drop('dropped item'), {
message: messages[1],
debugContext: createKafkaDebugContext(messages[1]),
lastStep: 'testChunkStep',
}),
])
})
})
describe('side effects accumulation', () => {
it('should accumulate side effects from previous context and current step result', async () => {
const messages: Message[] = [createTestMessage({ value: Buffer.from('test'), offset: 1 })]
const initialSideEffect1 = Promise.resolve('initial-side-effect-1')
const initialSideEffect2 = Promise.resolve('initial-side-effect-2')
const batch = [
createOkContext(
{ message: messages[0] },
{
message: messages[0],
sideEffects: [initialSideEffect1, initialSideEffect2],
}
),
]
const rootPipeline = createNewChunkPipeline().build()
const stepSideEffect1 = Promise.resolve('step-side-effect-1')
const stepSideEffect2 = Promise.resolve('step-side-effect-2')
const pipeline = new BaseChunkPipeline((items: any[]) => {
return Promise.resolve(items.map(() => ok({ processed: 'result' }, [stepSideEffect1, stepSideEffect2])))
}, rootPipeline)
pipeline.feed(batch)
const results = await pipeline.next()
expect(results).toHaveLength(1)
expect(results![0].context.sideEffects).toEqual([
initialSideEffect1,
initialSideEffect2,
stepSideEffect1,
stepSideEffect2,
])
})
it('should preserve context side effects when step returns no side effects', async () => {
const messages: Message[] = [createTestMessage({ value: Buffer.from('test'), offset: 1 })]
const existingSideEffect = Promise.resolve('existing-side-effect')
const batch = [
createOkContext(
{ message: messages[0] },
{
message: messages[0],
sideEffects: [existingSideEffect],
}
),
]
const rootPipeline = createNewChunkPipeline().build()
const pipeline = new BaseChunkPipeline((items: any[]) => {
return Promise.resolve(items.map(() => ok({ processed: 'result' })))
}, rootPipeline)
pipeline.feed(batch)
const results = await pipeline.next()
expect(results).toHaveLength(1)
expect(results![0].context.sideEffects).toEqual([existingSideEffect])
})
it('should add step side effects when context has no existing side effects', async () => {
const messages: Message[] = [createTestMessage({ value: Buffer.from('test'), offset: 1 })]
const batch = [
createOkContext(
{ message: messages[0] },
{
message: messages[0],
sideEffects: [],
}
),
]
const rootPipeline = createNewChunkPipeline().build()
const stepSideEffect = Promise.resolve('step-side-effect')
const pipeline = new BaseChunkPipeline((items: any[]) => {
return Promise.resolve(items.map(() => ok({ processed: 'result' }, [stepSideEffect])))
}, rootPipeline)
pipeline.feed(batch)
const results = await pipeline.next()
expect(results).toHaveLength(1)
expect(results![0].context.sideEffects).toEqual([stepSideEffect])
})
it('should handle multiple items with different side effect patterns', async () => {
const messages: Message[] = [
createTestMessage({ value: Buffer.from('item1'), offset: 1 }),
createTestMessage({ value: Buffer.from('item2'), offset: 2 }),
createTestMessage({ value: Buffer.from('item3'), offset: 3 }),
]
const sideEffect1 = Promise.resolve('context-1')
const sideEffect2 = Promise.resolve('context-2')
const sideEffect3 = Promise.resolve('context-3')
const batch = [
createOkContext(
{ message: messages[0] },
{
message: messages[0],
sideEffects: [sideEffect1],
}
),
createOkContext(
{ message: messages[1] },
{
message: messages[1],
sideEffects: [sideEffect2, sideEffect3],
}
),
createOkContext(
{ message: messages[2] },
{
message: messages[2],
sideEffects: [],
}
),
]
const rootPipeline = createNewChunkPipeline().build()
const step1SideEffect = Promise.resolve('step-1')
const step3aSideEffect = Promise.resolve('step-3a')
const step3bSideEffect = Promise.resolve('step-3b')
const pipeline = new BaseChunkPipeline((_: any[]) => {
return Promise.resolve([
ok({ processed: 'result1' }, [step1SideEffect]),
ok({ processed: 'result2' }), // No step side effects
ok({ processed: 'result3' }, [step3aSideEffect, step3bSideEffect]),
])
}, rootPipeline)
pipeline.feed(batch)
const results = await pipeline.next()
expect(results).toHaveLength(3)
// First item: context + step side effects
expect(results![0].context.sideEffects).toEqual([sideEffect1, step1SideEffect])
// Second item: only context side effects (step has none)
expect(results![1].context.sideEffects).toEqual([sideEffect2, sideEffect3])
// Third item: only step side effects (context has none)
expect(results![2].context.sideEffects).toEqual([step3aSideEffect, step3bSideEffect])
})
})
describe('warning accumulation', () => {
it('should accumulate warnings from step results', async () => {
const messages: Message[] = [createTestMessage({ value: Buffer.from('test1'), offset: 1 })]
const batch = createBatch(messages.map((message) => ({ message })))
const rootPipeline = createNewChunkPipeline().build()
const stepWarning = { type: 'merge_race_condition' as const, details: { message: 'step warning' } }
const pipeline = new BaseChunkPipeline((items: any[]) => {
return Promise.resolve(items.map(() => ok({ processed: 'result' }, [], [stepWarning])))
}, rootPipeline)
pipeline.feed(batch)
const results = await pipeline.next()
expect(results).toHaveLength(1)
expect(results![0].context.warnings).toEqual([stepWarning])
})
it('should merge context warnings with step warnings', async () => {
const messages: Message[] = [createTestMessage({ value: Buffer.from('test1'), offset: 1 })]
const contextWarning = { type: 'client_ingestion_warning' as const, details: { message: 'from context' } }
const batch = [
createOkContext(
{ message: messages[0] },
{
message: messages[0],
sideEffects: [],
warnings: [contextWarning],
}
),
]
const rootPipeline = createNewChunkPipeline().build()
const stepWarning = { type: 'schema_validation_failed' as const, details: { message: 'from step' } }
const pipeline = new BaseChunkPipeline((items: any[]) => {
return Promise.resolve(items.map(() => ok({ processed: 'result' }, [], [stepWarning])))
}, rootPipeline)
pipeline.feed(batch)
const results = await pipeline.next()
expect(results).toHaveLength(1)
expect(results![0].context.warnings).toEqual([contextWarning, stepWarning])
})
it('should handle multiple warnings from multiple items', async () => {
const messages: Message[] = [
createTestMessage({ value: Buffer.from('item1'), offset: 1 }),
createTestMessage({ value: Buffer.from('item2'), offset: 2 }),
createTestMessage({ value: Buffer.from('item3'), offset: 3 }),
]
const contextWarning1 = { type: 'ignored_invalid_timestamp' as const, details: { idx: 1 } }
const contextWarning2 = { type: 'invalid_heatmap_data' as const, details: { idx: 2 } }
const batch = [
createOkContext(
{ message: messages[0] },
{
message: messages[0],
sideEffects: [],
warnings: [contextWarning1],
}
),
createOkContext(
{ message: messages[1] },
{
message: messages[1],
sideEffects: [],
warnings: [contextWarning2],
}
),
createOkContext(
{ message: messages[2] },
{
message: messages[2],
sideEffects: [],
warnings: [],
}
),
]
const rootPipeline = createNewChunkPipeline().build()
const stepWarning1 = { type: 'message_size_too_large' as const, details: { result: 1 } }
const stepWarning3a = { type: 'group_key_too_long' as const, details: { result: 3 } }
const stepWarning3b = { type: 'event_dropped_too_old' as const, details: { result: 3 } }
const pipeline = new BaseChunkPipeline((_: any[]) => {
return Promise.resolve([
ok({ processed: 'result1' }, [], [stepWarning1]),
ok({ processed: 'result2' }), // No step warnings
ok({ processed: 'result3' }, [], [stepWarning3a, stepWarning3b]),
])
}, rootPipeline)
pipeline.feed(batch)
const results = await pipeline.next()
expect(results).toHaveLength(3)
// First item: context + step warnings
expect(results![0].context.warnings).toEqual([contextWarning1, stepWarning1])
// Second item: only context warnings (step has none)
expect(results![1].context.warnings).toEqual([contextWarning2])
// Third item: only step warnings (context has none)
expect(results![2].context.warnings).toEqual([stepWarning3a, stepWarning3b])
})
it('should preserve warnings for non-success results', async () => {
const messages: Message[] = [
createTestMessage({ value: Buffer.from('1'), offset: 1 }),
createTestMessage({ value: Buffer.from('drop'), offset: 2 }),
]
const contextWarning = { type: 'client_ingestion_warning' as const, details: { message: 'existing' } }
const batch = [
createOkContext(
{ message: messages[0] },
{
message: messages[0],
sideEffects: [],
warnings: [contextWarning],
}
),
createContext(drop('drop_reason'), {
message: messages[1],
sideEffects: [],
warnings: [contextWarning],
}),
]
const mockPrevious = createMockPipeline(batch)
const pipeline = new BaseChunkPipeline((items: any[]) => {
// Only process OK results
return Promise.resolve(items.map(() => ok({ processed: 'result' })))
}, mockPrevious)
const results = await pipeline.next()
expect(results).toHaveLength(2)
// OK result should have warnings accumulated
expect(results![0].context.warnings).toEqual([contextWarning])
// Drop result should preserve context warnings
expect(results![1].context.warnings).toEqual([contextWarning])
})
})
})