:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { GROUPS_OUTPUT, INGESTION_WARNINGS_OUTPUT } from '~/common/outputs'
import { PERSONS_OUTPUT, PERSON_DISTINCT_IDS_OUTPUT, PERSON_MERGE_EVENTS_OUTPUT } from '~/common/outputs'
import { MessageSizeTooLarge } from '~/common/utils/db/error'
import { BatchWritingGroupStore } from '~/ingestion/common/groups/batch-writing-group-store'
import { GroupFlushResult } from '~/ingestion/common/groups/group-store.interface'
import { emitIngestionWarning } from '~/ingestion/common/ingestion-warnings'
import { FlushResult, PersonsStore } from '~/ingestion/common/persons/persons-store'
import {
batchStoreFlushCacheEntriesHistogram,
batchStoreFlushDirtyEntriesHistogram,
batchStoreFlushKafkaMessagesHistogram,
batchStoreFlushLatencyHistogram,
batchStoreFlushOperationsCounter,
batchStoreFlushReferencedBatchesHistogram,
batchStoreFlushResultRecordsHistogram,
batchStoreFlushTriggerBatchSizeHistogram,
} from '~/ingestion/common/stores/metrics'
import { AfterBatchInput } from '~/ingestion/framework/batching-pipeline'
import { isOkResult, ok } from '~/ingestion/framework/results'
import { createMockIngestionOutputs } from '~/tests/helpers/mock-ingestion-outputs'
import {
FlushBatchStoresOutputs,
FlushBatchStoresStepConfig,
createFlushBatchStoresStep,
} from './flush-batch-stores-step'
jest.mock('~/ingestion/common/ingestion-warnings', () => ({
emitIngestionWarning: jest.fn(),
}))
jest.mock('~/ingestion/common/stores/metrics', () => ({
batchStoreFlushCacheEntriesHistogram: { observe: jest.fn() },
batchStoreFlushDirtyEntriesHistogram: { observe: jest.fn() },
batchStoreFlushKafkaMessagesHistogram: { observe: jest.fn() },
batchStoreFlushLatencyHistogram: { observe: jest.fn() },
batchStoreFlushOperationsCounter: { inc: jest.fn() },
batchStoreFlushReferencedBatchesHistogram: { observe: jest.fn() },
batchStoreFlushResultRecordsHistogram: { observe: jest.fn() },
batchStoreFlushTriggerBatchSizeHistogram: { observe: jest.fn() },
}))
describe('flush-batch-stores-step', () => {
let mockPersonsStore: jest.Mocked<PersonsStore>
let mockGroupStore: jest.Mocked<BatchWritingGroupStore>
let mockOutputs: FlushBatchStoresOutputs
let storesConfig: FlushBatchStoresStepConfig
beforeEach(() => {
mockPersonsStore = {
getFlushStats: jest.fn(() => ({
dirtyEntryCount: 0,
referencedBatchCount: 0,
cacheEntryCount: 0,
})),
flush: jest.fn(),
shutdown: jest.fn(),
releaseBatch: jest.fn(),
} as any
mockGroupStore = {
getFlushStats: jest.fn(() => ({
dirtyEntryCount: 0,
referencedBatchCount: 0,
cacheEntryCount: 0,
})),
flush: jest.fn(),
shutdown: jest.fn(),
releaseBatch: jest.fn(),
} as any
mockOutputs = createMockIngestionOutputs<
| typeof PERSONS_OUTPUT
| typeof PERSON_DISTINCT_IDS_OUTPUT
| typeof INGESTION_WARNINGS_OUTPUT
| typeof PERSON_MERGE_EVENTS_OUTPUT
| typeof GROUPS_OUTPUT
>()
storesConfig = {
personsStore: mockPersonsStore,
groupStore: mockGroupStore,
outputs: mockOutputs,
}
jest.clearAllMocks()
})
describe('createFlushBatchStoresStep', () => {
let step: ReturnType<typeof createFlushBatchStoresStep>
function makeInput(elementCount = 2): AfterBatchInput<void, any, NonNullable<unknown>> {
return {
elements: Array.from({ length: elementCount }, () => ({
result: ok(undefined),
context: { sideEffects: [], warnings: [] },
})),
batchContext: {},
batchId: 0,
}
}
beforeEach(() => {
step = createFlushBatchStoresStep(storesConfig)
})
it('should flush both stores in parallel', async () => {
mockPersonsStore.flush.mockResolvedValue([])
mockGroupStore.flush.mockResolvedValue([])
await step(makeInput())
expect(mockPersonsStore.flush).toHaveBeenCalledTimes(1)
expect(mockGroupStore.flush).toHaveBeenCalledTimes(1)
})
it('should return ok result with produce promises as side effects', async () => {
const personMessages: FlushResult[] = [
{
messages: [
{
output: PERSONS_OUTPUT,
value: Buffer.from('value1'),
},
],
teamId: 1,
distinctId: 'user1',
uuid: 'uuid1',
},
]
mockPersonsStore.flush.mockResolvedValue(personMessages)
mockGroupStore.flush.mockResolvedValue([])
const produceSpy = jest.spyOn(mockOutputs, 'produce')
const result = await step(makeInput())
expect(isOkResult(result)).toBe(true)
if (isOkResult(result)) {
expect(result.sideEffects).toHaveLength(1)
}
expect(produceSpy).toHaveBeenCalledWith(PERSONS_OUTPUT, {
key: null,
value: Buffer.from('value1'),
teamId: 1,
})
})
it('should handle multiple messages per flush result', async () => {
const personMessages: FlushResult[] = [
{
messages: [
{ output: PERSONS_OUTPUT, value: Buffer.from('value1') },
{ output: PERSONS_OUTPUT, value: Buffer.from('value2') },
],
teamId: 1,
distinctId: 'user1',
uuid: 'uuid1',
},
]
mockPersonsStore.flush.mockResolvedValue(personMessages)
mockGroupStore.flush.mockResolvedValue([])
const produceSpy = jest.spyOn(mockOutputs, 'produce')
const result = await step(makeInput())
expect(produceSpy).toHaveBeenCalledTimes(2)
expect(isOkResult(result)).toBe(true)
if (isOkResult(result)) {
expect(result.sideEffects).toHaveLength(2)
}
})
it('should handle multiple flush results with multiple messages each', async () => {
const personMessages: FlushResult[] = [
{
messages: [
{ output: PERSONS_OUTPUT, value: Buffer.from('value1') },
{ output: PERSONS_OUTPUT, value: Buffer.from('value2') },
],
teamId: 1,
distinctId: 'user1',
uuid: 'uuid1',
},
{
messages: [{ output: PERSON_DISTINCT_IDS_OUTPUT, value: Buffer.from('value3') }],
teamId: 2,
distinctId: 'user2',
uuid: 'uuid2',
},
]
mockPersonsStore.flush.mockResolvedValue(personMessages)
mockGroupStore.flush.mockResolvedValue([])
const produceSpy = jest.spyOn(mockOutputs, 'produce')
const result = await step(makeInput())
expect(produceSpy).toHaveBeenCalledTimes(3)
expect(isOkResult(result)).toBe(true)
if (isOkResult(result)) {
expect(result.sideEffects).toHaveLength(3)
}
})
it('should return empty side effects when no messages to produce', async () => {
mockPersonsStore.flush.mockResolvedValue([])
mockGroupStore.flush.mockResolvedValue([])
const result = await step(makeInput())
expect(isOkResult(result)).toBe(true)
if (isOkResult(result)) {
expect(result.sideEffects).toHaveLength(0)
}
})
it('reports flush lifecycle metrics', async () => {
const personMessages: FlushResult[] = [
{
messages: [
{ output: PERSONS_OUTPUT, value: Buffer.from('value1') },
{ output: PERSONS_OUTPUT, value: Buffer.from('value2') },
],
teamId: 1,
distinctId: 'user1',
uuid: 'uuid1',
},
{
messages: [{ output: PERSON_DISTINCT_IDS_OUTPUT, value: Buffer.from('value3') }],
teamId: 1,
distinctId: 'user2',
uuid: 'uuid2',
},
]
mockPersonsStore.getFlushStats.mockReturnValue({
dirtyEntryCount: 2,
referencedBatchCount: 2,
cacheEntryCount: 5,
})
mockGroupStore.getFlushStats.mockReturnValue({
dirtyEntryCount: 1,
referencedBatchCount: 1,
cacheEntryCount: 3,
})
mockPersonsStore.flush.mockResolvedValue(personMessages)
mockGroupStore.flush.mockResolvedValue([])
await step(makeInput(7))
expect(batchStoreFlushTriggerBatchSizeHistogram.observe).toHaveBeenCalledWith(7)
expect(batchStoreFlushDirtyEntriesHistogram.observe).toHaveBeenCalledWith({ store: 'person' }, 2)
expect(batchStoreFlushDirtyEntriesHistogram.observe).toHaveBeenCalledWith({ store: 'group' }, 1)
expect(batchStoreFlushReferencedBatchesHistogram.observe).toHaveBeenCalledWith({ store: 'person' }, 2)
expect(batchStoreFlushReferencedBatchesHistogram.observe).toHaveBeenCalledWith({ store: 'group' }, 1)
expect(batchStoreFlushCacheEntriesHistogram.observe).toHaveBeenCalledWith({ store: 'person' }, 5)
expect(batchStoreFlushCacheEntriesHistogram.observe).toHaveBeenCalledWith({ store: 'group' }, 3)
expect(batchStoreFlushOperationsCounter.inc).toHaveBeenCalledWith({ store: 'person', outcome: 'success' })
expect(batchStoreFlushOperationsCounter.inc).toHaveBeenCalledWith({ store: 'group', outcome: 'success' })
expect(batchStoreFlushLatencyHistogram.observe).toHaveBeenCalledWith(
{ store: 'person', outcome: 'success' },
expect.any(Number)
)
expect(batchStoreFlushLatencyHistogram.observe).toHaveBeenCalledWith(
{ store: 'group', outcome: 'success' },
expect.any(Number)
)
expect(batchStoreFlushResultRecordsHistogram.observe).toHaveBeenCalledWith({ store: 'person' }, 2)
expect(batchStoreFlushResultRecordsHistogram.observe).toHaveBeenCalledWith({ store: 'group' }, 0)
expect(batchStoreFlushKafkaMessagesHistogram.observe).toHaveBeenCalledWith({ store: 'person' }, 3)
expect(batchStoreFlushKafkaMessagesHistogram.observe).toHaveBeenCalledWith({ store: 'group' }, 0)
})
it('should handle MessageSizeTooLarge errors gracefully', async () => {
const personMessages: FlushResult[] = [
{
messages: [{ output: PERSONS_OUTPUT, value: Buffer.from('value1') }],
teamId: 1,
distinctId: 'user1',
uuid: 'uuid1',
},
]
mockPersonsStore.flush.mockResolvedValue(personMessages)
mockGroupStore.flush.mockResolvedValue([])
jest.spyOn(mockOutputs, 'produce').mockRejectedValue(
new MessageSizeTooLarge('test', new Error('too large'))
)
const result = await step(makeInput())
expect(isOkResult(result)).toBe(true)
if (isOkResult(result)) {
expect(result.sideEffects).toHaveLength(1)
await result.sideEffects[0]
}
expect(emitIngestionWarning).toHaveBeenCalledWith(mockOutputs, 1, {
type: 'message_size_too_large',
details: {
personId: 'uuid1',
distinctId: 'user1',
step: 'flushBatchStoresStep',
},
pipelineStep: 'flush',
})
})
it('should produce group store flush results as side effects', async () => {
const groupResults: GroupFlushResult[] = [
{
messages: [{ output: GROUPS_OUTPUT, value: Buffer.from('group1') }],
teamId: 1,
groupTypeIndex: 0,
groupKey: 'org-1',
},
]
mockPersonsStore.flush.mockResolvedValue([])
mockGroupStore.flush.mockResolvedValue(groupResults)
const produceSpy = jest.spyOn(mockOutputs, 'produce')
const result = await step(makeInput())
expect(isOkResult(result)).toBe(true)
if (isOkResult(result)) {
expect(result.sideEffects).toHaveLength(1)
}
expect(produceSpy).toHaveBeenCalledWith(GROUPS_OUTPUT, {
key: null,
value: Buffer.from('group1'),
teamId: 1,
})
})
it('should emit a group-specific warning when a group message is too large', async () => {
const groupResults: GroupFlushResult[] = [
{
messages: [{ output: GROUPS_OUTPUT, value: Buffer.from('group1') }],
teamId: 1,
groupTypeIndex: 0,
groupKey: 'org-1',
},
]
mockPersonsStore.flush.mockResolvedValue([])
mockGroupStore.flush.mockResolvedValue(groupResults)
jest.spyOn(mockOutputs, 'produce').mockRejectedValue(
new MessageSizeTooLarge('test', new Error('too large'))
)
const result = await step(makeInput())
expect(isOkResult(result)).toBe(true)
if (isOkResult(result)) {
expect(result.sideEffects).toHaveLength(1)
await result.sideEffects[0]
}
expect(emitIngestionWarning).toHaveBeenCalledWith(mockOutputs, 1, {
type: 'group_upsert_message_size_too_large',
details: {
groupTypeIndex: 0,
groupKey: 'org-1',
},
pipelineStep: 'flush',
})
})
it('should propagate other Kafka produce errors', async () => {
const personMessages: FlushResult[] = [
{
messages: [{ output: PERSONS_OUTPUT, value: Buffer.from('value1') }],
teamId: 1,
distinctId: 'user1',
uuid: 'uuid1',
},
]
mockPersonsStore.flush.mockResolvedValue(personMessages)
mockGroupStore.flush.mockResolvedValue([])
jest.spyOn(mockOutputs, 'produce').mockRejectedValue(new Error('Kafka connection failed'))
const result = await step(makeInput())
expect(isOkResult(result)).toBe(true)
if (isOkResult(result)) {
expect(result.sideEffects).toHaveLength(1)
await expect(result.sideEffects[0]).rejects.toThrow('Kafka connection failed')
}
})
it('should throw if person store flush fails', async () => {
mockPersonsStore.flush.mockRejectedValue(new Error('Database connection failed'))
mockGroupStore.flush.mockResolvedValue([])
await expect(step(makeInput())).rejects.toThrow('Database connection failed')
})
it('should throw if group store flush fails', async () => {
mockPersonsStore.flush.mockResolvedValue([])
mockGroupStore.flush.mockRejectedValue(new Error('Database connection failed'))
await expect(step(makeInput())).rejects.toThrow('Database connection failed')
})
it('calls releaseBatch on both stores with the correct batchId after a successful flush', async () => {
mockPersonsStore.flush.mockResolvedValue([])
mockGroupStore.flush.mockResolvedValue([])
const input = makeInput()
await step(input)
expect(mockPersonsStore.releaseBatch).toHaveBeenCalledWith(input.batchId)
expect(mockGroupStore.releaseBatch).toHaveBeenCalledWith(input.batchId)
})
it('calls releaseBatch on both stores even when flush throws', async () => {
mockPersonsStore.flush.mockRejectedValue(new Error('DB error'))
mockGroupStore.flush.mockResolvedValue([])
const input = makeInput()
await expect(step(input)).rejects.toThrow('DB error')
expect(mockPersonsStore.releaseBatch).toHaveBeenCalledWith(input.batchId)
expect(mockGroupStore.releaseBatch).toHaveBeenCalledWith(input.batchId)
})
it('should handle null values in messages', async () => {
const personMessages: FlushResult[] = [
{
messages: [{ output: PERSONS_OUTPUT, value: null }],
teamId: 1,
},
]
mockPersonsStore.flush.mockResolvedValue(personMessages)
mockGroupStore.flush.mockResolvedValue([])
const produceSpy = jest.spyOn(mockOutputs, 'produce')
await step(makeInput())
expect(produceSpy).toHaveBeenCalledWith(PERSONS_OUTPUT, {
key: null,
value: null,
teamId: 1,
})
})
it('flushes both stores in parallel', async () => {
const callOrder: string[] = []
mockPersonsStore.flush.mockImplementation(() => {
callOrder.push('persons.flush')
return Promise.resolve([])
})
mockGroupStore.flush.mockImplementation(() => {
callOrder.push('group.flush')
return Promise.resolve([])
})
await step(makeInput())
expect(callOrder.sort()).toEqual(['group.flush', 'persons.flush'])
})
it('should produce messages with correct Buffer conversion', async () => {
const personMessages: FlushResult[] = [
{
messages: [
{
output: PERSONS_OUTPUT,
value: Buffer.from('string-value'),
},
],
teamId: 1,
},
]
mockPersonsStore.flush.mockResolvedValue(personMessages)
mockGroupStore.flush.mockResolvedValue([])
const produceSpy = jest.spyOn(mockOutputs, 'produce')
await step(makeInput())
expect(produceSpy).toHaveBeenCalledWith(PERSONS_OUTPUT, {
key: null,
value: Buffer.from('string-value'),
teamId: 1,
})
})
it('should pass through elements and batchContext in the result', async () => {
mockPersonsStore.flush.mockResolvedValue([])
mockGroupStore.flush.mockResolvedValue([])
const input = makeInput(3)
const result = await step(input)
expect(isOkResult(result)).toBe(true)
if (isOkResult(result)) {
expect(result.value.elements).toHaveLength(3)
expect(result.value.elements).toBe(input.elements)
expect(result.value.batchContext).toBe(input.batchContext)
}
})
})
})