:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { mockProducer, mockProducerObserver } from '~/tests/helpers/mocks/producer.mock'
import { DateTime } from 'luxon'
import { CODES, Message } from 'node-rdkafka'
import { KAFKA_APP_METRICS_2, KAFKA_LOGS_CLICKHOUSE, KAFKA_LOGS_INGESTION_DLQ } from '~/common/config/kafka-topics'
import { APP_METRICS_OUTPUT, AppMetricsOutput } from '~/common/outputs'
import { IngestionOutputs } from '~/common/outputs/ingestion-outputs'
import { SingleIngestionOutput } from '~/common/outputs/single-ingestion-output'
import { UsageRecordBatch } from '~/common/usage-ingestion/usage-record-batch'
import { DependencyUnavailableError } from '~/common/utils/db/error'
import { closeHub, createHub } from '~/common/utils/db/hub'
import { createTeam, createTestTeamFixture, getTeam } from '~/tests/helpers/sql'
import { Hub, Team } from '~/types'
import { getDefaultTracesIngestionConsumerConfig } from '../config'
import { INTERNAL_LOGS_HEADER, signInternalLogs } from '../internal-logs'
import { LogRecord, encodeLogRecords } from '../log-record-avro'
import { DEFAULT_LOGS_RETENTION_DAYS, LogsIngestionConsumer, logMessageDlqCounter } from '../logs-ingestion-consumer'
import { LOGS_DLQ_OUTPUT, LOGS_OUTPUT, LogsDlqOutput, LogsOutput } from '../outputs/outputs'
import { DEFAULT_TRACES_RETENTION_DAYS } from '../retention/tracing-config-cache'
import { TracesIngestionConsumer } from '../traces-ingestion-consumer'
import { createLogsIngestionConsumer, createTracesIngestionConsumer } from './create-consumer'
import { LogsIngestionFrameworkConsumer } from './logs-ingestion-framework-consumer'
import { TracesIngestionFrameworkConsumer } from './traces-ingestion-framework-consumer'
const LibrdKafkaError = require('node-rdkafka/lib/error')
const { err2str } = require('node-rdkafka/librdkafka') as { err2str: (code: number) => string }
jest.setTimeout(5000)
jest.mock('~/common/utils/posthog', () => {
const original = jest.requireActual('~/common/utils/posthog')
return {
...original,
captureException: jest.fn(),
}
})
const avro = require('avsc')
const LOG_RECORD_TYPE = avro.Type.forSchema({
type: 'record',
name: 'LogRecord',
fields: [
{ name: 'uuid', type: ['null', 'string'] },
{ name: 'trace_id', type: ['null', 'bytes'] },
{ name: 'span_id', type: ['null', 'bytes'] },
{ name: 'trace_flags', type: ['null', 'int'] },
{ name: 'timestamp', type: ['null', 'long'] },
{ name: 'observed_timestamp', type: ['null', 'long'] },
{ name: 'body', type: ['null', 'string'] },
{ name: 'severity_text', type: ['null', 'string'] },
{ name: 'severity_number', type: ['null', 'int'] },
{ name: 'service_name', type: ['null', 'string'] },
{ name: 'resource_attributes', type: ['null', { type: 'map', values: 'string' }] },
{ name: 'instrumentation_scope', type: ['null', 'string'] },
{ name: 'event_name', type: ['null', 'string'] },
{ name: 'attributes', type: ['null', { type: 'map', values: 'string' }] },
],
})
let offset = 0
async function createKafkaMessages(count: number, headers: Record<string, string>): Promise<Message[]> {
return await Promise.all(
Array.from({ length: count }, async () => {
const record: LogRecord = {
uuid: `test-uuid-${offset}`,
trace_id: null,
span_id: null,
trace_flags: null,
timestamp: Date.now() * 1000,
observed_timestamp: Date.now() * 1000,
body: 'Test log message',
severity_text: 'info',
severity_number: 9,
service_name: 'test-service',
resource_attributes: null,
instrumentation_scope: null,
event_name: null,
attributes: null,
}
const value = await encodeLogRecords(LOG_RECORD_TYPE, 'zstandard', [record])
return {
key: null,
value,
size: value.length,
topic: 'test',
offset: offset++,
timestamp: Date.now(),
partition: 1,
headers: Object.entries(headers).map(([key, header]) => ({ [key]: Buffer.from(header) })),
}
})
)
}
function producedTo(topic: string): { headers?: Record<string, string>; value: unknown }[] {
return mockProducerObserver
.getProducedMessages()
.filter((batch) => batch.topic === topic)
.flatMap((batch) => batch.messages.map((message) => ({ headers: message.headers, value: message.value })))
}
const kafkaDeliveryError = (code: number): Error => LibrdKafkaError.create(new Error(err2str(code)))
function failLogsProduce(makeError: () => unknown, times = Infinity): () => void {
const original = mockProducer.queueMessages
const passthrough = original.bind(mockProducer)
let failures = 0
mockProducer.queueMessages = jest.fn().mockImplementation((topicMessages) => {
const topic = Array.isArray(topicMessages) ? topicMessages[0]?.topic : topicMessages.topic
if (topic === KAFKA_LOGS_CLICKHOUSE && failures++ < times) {
throw makeError()
}
return passthrough(topicMessages)
})
return () => {
mockProducer.queueMessages = original
}
}
describe('LogsIngestionFrameworkConsumer', () => {
let hub: Hub
let team: Team
let team2: Team
let consumer: LogsIngestionFrameworkConsumer
const consumerDeps = (): any => ({
...hub,
outputs: new IngestionOutputs<AppMetricsOutput | LogsOutput | LogsDlqOutput>({
[APP_METRICS_OUTPUT]: new SingleIngestionOutput(
APP_METRICS_OUTPUT,
KAFKA_APP_METRICS_2,
mockProducer,
'test'
),
[LOGS_OUTPUT]: new SingleIngestionOutput(LOGS_OUTPUT, KAFKA_LOGS_CLICKHOUSE, mockProducer, 'test'),
[LOGS_DLQ_OUTPUT]: new SingleIngestionOutput(
LOGS_DLQ_OUTPUT,
KAFKA_LOGS_INGESTION_DLQ,
mockProducer,
'test'
),
}),
usageBatch: new UsageRecordBatch(null, { unit: 'bytes', isTeamEnabled: () => false }),
dependencyRetry: { retryCount: 2, initialRetryDelayMs: 1 },
})
const withoutKafka = <T extends { start(): Promise<void> }>(instance: T): T => {
;(instance as any)['kafkaConsumer'] = {
connect: jest.fn(),
disconnect: jest.fn(),
isHealthy: jest.fn().mockReturnValue({ status: 'healthy' }),
}
return instance
}
const tracesConfig = (overrides: Record<string, unknown> = {}): any => ({
...hub,
...getDefaultTracesIngestionConsumerConfig(),
// A blank traces Redis host reuses the hub's Redis, so no second pool outlives the suite.
TRACES_REDIS_HOST: '',
...overrides,
})
const processAndWait = async (messages: Message[]): Promise<void> => {
await (
await consumer.processKafkaBatch(messages)
).backgroundTask
}
beforeEach(async () => {
jest.spyOn(Date, 'now').mockReturnValue(DateTime.fromISO('2025-01-01T00:00:00Z').toMillis())
offset = 0
hub = await createHub()
team = (await createTestTeamFixture(hub.postgres)).team
team2 = (await getTeam(hub.postgres, await createTeam(hub.postgres, team.organization_id)))!
jest.spyOn(hub.quotaLimiting, 'isTeamTokenQuotaLimited').mockResolvedValue(false)
consumer = withoutKafka(new LogsIngestionFrameworkConsumer(hub, consumerDeps()))
await consumer.start()
})
afterEach(async () => {
await consumer.stop()
await closeHub(hub)
})
it('runs transformations on a batch whose internal logs header fails to verify, flagged as internal', async () => {
await consumer.stop()
const logsTransformer = {
startBatch: jest.fn(),
teamHasTransformations: jest.fn().mockResolvedValue(true),
transformRecords: jest.fn().mockResolvedValue({ recordsDropped: 0, recordsDroppedByFunctionId: new Map() }),
flush: jest.fn().mockResolvedValue(undefined),
}
consumer = withoutKafka(
new LogsIngestionFrameworkConsumer(
{
...hub,
LOGS_TRANSFORMATIONS_ENABLED_TEAMS: '*',
LOGS_INTERNAL_INGESTION_ENABLED_TEAMS: '*',
LOGS_INTERNAL_INGESTION_SIGNING_KEYS: 'current-key',
},
{ ...consumerDeps(), logsTransformer }
)
)
await consumer.start()
await processAndWait(
await createKafkaMessages(1, {
token: team.api_token,
[INTERNAL_LOGS_HEADER]: signInternalLogs('rotated-away-key', 'hog_function', team.id),
})
)
expect(logsTransformer.transformRecords).toHaveBeenCalledWith(team.id, expect.any(Array), undefined, true)
expect(producedTo(KAFKA_LOGS_CLICKHOUSE)).toHaveLength(1)
})
it('produces a valid message to the logs topic with the batch headers', async () => {
const messages = await createKafkaMessages(1, { token: team.api_token })
await processAndWait(messages)
const produced = producedTo(KAFKA_LOGS_CLICKHOUSE)
expect(produced).toHaveLength(1)
expect(produced[0].headers).toEqual({
token: team.api_token,
team_id: team.id.toString(),
'json-parse': 'false',
'retention-days': DEFAULT_LOGS_RETENTION_DAYS.toString(),
source_topic: messages[0].topic,
source_partition: messages[0].partition.toString(),
})
})
it('keeps the messages of overlapping batches apart', async () => {
const first = await createKafkaMessages(2, { token: team.api_token })
const second = await createKafkaMessages(1, { token: team2.api_token })
const [a, b] = await Promise.all([consumer.processKafkaBatch(first), consumer.processKafkaBatch(second)])
await Promise.all([a.backgroundTask, b.backgroundTask])
expect(a.messages.map((m) => m.teamId)).toEqual([team.id, team.id])
expect(b.messages.map((m) => m.teamId)).toEqual([team2.id])
expect(producedTo(KAFKA_LOGS_CLICKHOUSE)).toHaveLength(3)
})
describe('logs produce errors', () => {
it.each([
['a processing error', () => new Error('Producer error')],
['a Kafka error that a replay cannot fix', () => kafkaDeliveryError(CODES.ERRORS.ERR_INVALID_RECORD)],
])('sends the message to the DLQ on %s', async (_, makeError) => {
const messages = await createKafkaMessages(1, { token: team.api_token })
const dlqCounterSpy = jest.spyOn(logMessageDlqCounter, 'inc')
const restore = failLogsProduce(makeError)
try {
await processAndWait(messages)
} finally {
restore()
}
expect(dlqCounterSpy).toHaveBeenCalledWith({ reason: 'Error', team_id: team.id.toString() })
const dlq = producedTo(KAFKA_LOGS_INGESTION_DLQ)
expect(dlq).toHaveLength(1)
expect(dlq[0].headers).toMatchObject({
source_topic: messages[0].topic,
source_partition: messages[0].partition.toString(),
source_offset: messages[0].offset.toString(),
})
expect(dlq[0].headers).toHaveProperty('error_message')
})
it('retries a dependency blip in place and produces the message once', async () => {
const messages = await createKafkaMessages(1, { token: team.api_token })
const restore = failLogsProduce(
() => new DependencyUnavailableError('broker down', 'Kafka', new Error('broker down')),
1
)
try {
await processAndWait(messages)
} finally {
restore()
}
expect(producedTo(KAFKA_LOGS_CLICKHOUSE)).toHaveLength(1)
expect(producedTo(KAFKA_LOGS_INGESTION_DLQ)).toHaveLength(0)
})
it.each([
[
'a dependency is unavailable',
() => new DependencyUnavailableError('broker down', 'Kafka', new Error('broker down')),
],
['a delivery times out', () => kafkaDeliveryError(CODES.ERRORS.ERR__MSG_TIMED_OUT)],
['the producer queue is full', () => LibrdKafkaError.create(CODES.ERRORS.ERR__QUEUE_FULL)],
['the topic rejects authorization', () => kafkaDeliveryError(CODES.ERRORS.ERR_TOPIC_AUTHORIZATION_FAILED)],
])('fails the batch instead of DLQing when %s', async (_, makeError) => {
const messages = await createKafkaMessages(1, { token: team.api_token })
const restore = failLogsProduce(makeError)
try {
// Good data succeeds on redelivery, so quarantining it would lose it to an outage.
await expect(processAndWait(messages)).rejects.toThrow()
} finally {
restore()
}
expect(producedTo(KAFKA_LOGS_INGESTION_DLQ)).toHaveLength(0)
})
it.each([
['the message fails to parse', { bytes_uncompressed: 'not-a-number' }],
['the message fails to process', {}],
])('fails only that batch when %s and the DLQ write fails', async (_, headers) => {
// Two failing messages: the second failure must be awaited too, not left unhandled.
const messages = await createKafkaMessages(2, { token: team.api_token, ...headers })
const original = mockProducer.queueMessages
mockProducer.queueMessages = jest.fn().mockRejectedValue(new Error('DLQ unavailable'))
try {
await expect(processAndWait(messages)).rejects.toThrow('DLQ unavailable')
} finally {
mockProducer.queueMessages = original
}
await processAndWait(await createKafkaMessages(1, { token: team.api_token }))
expect(producedTo(KAFKA_LOGS_CLICKHOUSE)).toHaveLength(1)
})
})
describe('TracesIngestionFrameworkConsumer', () => {
it('reads the TRACES_ consumer and rate limiter config', () => {
const traces = new TracesIngestionFrameworkConsumer(
tracesConfig({
TRACES_INGESTION_CONSUMER_GROUP_ID: 'traces-group',
TRACES_INGESTION_CONSUMER_CONSUME_TOPIC: 'traces-topic',
TRACES_LIMITER_BUCKET_SIZE_KB: 42,
}),
consumerDeps()
)
expect(traces['groupId']).toBe('traces-group')
expect(traces['topic']).toBe('traces-topic')
expect(traces['rateLimiter']['config'].LOGS_LIMITER_BUCKET_SIZE_KB).toBe(42)
expect(traces['rateLimiter']['rateLimiter'].getKeyPrefix()).toBe('@posthog-test/traces-rate-limiter/tokens')
expect(traces.service.id).toBe('TracesIngestionConsumer')
})
it('takes its default retention from the tracing config, not logs_settings', async () => {
const tracingConfigCache = { getRetentionDays: jest.fn().mockResolvedValue(90) }
const traces = new TracesIngestionFrameworkConsumer(tracesConfig(), {
...consumerDeps(),
tracingConfigCache,
})
const fallback = new TracesIngestionFrameworkConsumer(tracesConfig(), consumerDeps())
expect(await traces['defaultRetentionDays'](team.id, { retention_days: 30 })).toBe(90)
expect(tracingConfigCache.getRetentionDays).toHaveBeenCalledWith(team.id)
expect(await fallback['defaultRetentionDays'](team.id, { retention_days: 30 })).toBe(
DEFAULT_TRACES_RETENTION_DAYS
)
})
})
describe('the consumer factories the servers call', () => {
it.each([
[false, LogsIngestionConsumer],
[true, LogsIngestionFrameworkConsumer],
])('builds the logs consumer for LOGS_INGESTION_USE_FRAMEWORK=%s', (useFramework, expected) => {
const built = createLogsIngestionConsumer(
{ ...hub, LOGS_INGESTION_USE_FRAMEWORK: useFramework },
consumerDeps()
)
expect(built.constructor).toBe(expected)
})
it.each([
[false, TracesIngestionConsumer],
[true, TracesIngestionFrameworkConsumer],
])('builds the traces consumer for TRACES_INGESTION_USE_FRAMEWORK=%s', (useFramework, expected) => {
// The logs flag is set the other way, so the traces server must read its own flag.
const built = createTracesIngestionConsumer(
tracesConfig({
TRACES_INGESTION_USE_FRAMEWORK: useFramework,
LOGS_INGESTION_USE_FRAMEWORK: !useFramework,
}),
consumerDeps()
)
expect(built.constructor).toBe(expected)
})
})
})