:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { mockProducer, mockProducerObserver } from '~/tests/helpers/mocks/producer.mock'
import { metrics as metricsApi } from '@opentelemetry/api'
import {
type DataPoint,
InMemoryMetricExporter,
MeterProvider,
PeriodicExportingMetricReader,
} from '@opentelemetry/sdk-metrics'
import { DateTime } from 'luxon'
import { CODES, Message } from 'node-rdkafka'
import { compileHog } from '~/cdp/templates/compiler'
import { HogFunctionType } from '~/cdp/types'
import { KAFKA_APP_METRICS_2, KAFKA_LOGS_CLICKHOUSE, KAFKA_LOGS_INGESTION_DLQ } from '~/common/config/kafka-topics'
import { APP_METRICS_OUTPUT, AppMetricsOutput } from '~/common/outputs'
import { IngestionOutputs } from '~/common/outputs/ingestion-outputs'
import { SingleIngestionOutput } from '~/common/outputs/single-ingestion-output'
import { UsageRecordBatch } from '~/common/usage-ingestion/usage-record-batch'
import { DependencyUnavailableError } from '~/common/utils/db/error'
import { closeHub, createHub } from '~/common/utils/db/hub'
import { PostgresUse } from '~/common/utils/db/postgres'
import { parseJSON } from '~/common/utils/json-parse'
import { forSnapshot } from '~/tests/helpers/snapshots'
import { createTeam, createTestTeamFixture, getTeam } from '~/tests/helpers/sql'
import { Hub, Team } from '~/types'
import { getDefaultTracesIngestionConsumerConfig } from './config'
import * as otelMetrics from './ingestion-otel-metrics'
import { resetLogsIngestionInstrumentsForTests } from './ingestion-otel-metrics'
import { INTERNAL_LOGS_HEADER, INTERNAL_SOURCE_HEADER, signInternalLogs } from './internal-logs'
import { logsPatternBodyKindCounter } from './log-pattern-stage'
import { LogRecord, decodeLogRecords, encodeLogRecords, logsJsonAttributeSniffCounter } from './log-record-avro'
import {
DEFAULT_LOGS_RETENTION_DAYS,
LogsIngestionConsumer,
type LogsIngestionConsumerDeps,
describeBatchPosition,
logMessageDlqCounter,
logMessageDroppedCounter,
logsBillingBytesCreditedCounter,
logsBillingRecordsCreditedCounter,
logsBytesAllowedCounter,
logsBytesDroppedCounter,
logsBytesReceivedCounter,
logsRecordsAllowedCounter,
logsRecordsBytesExceedPayloadCounter,
logsRecordsDroppedCounter,
logsRecordsReceivedCounter,
parseMinTimestampHeader,
parseSizeHeader,
} from './logs-ingestion-consumer'
import { compileMetricRules } from './metrics-rules/compile-metric-rules'
import type { MetricRulesCache } from './metrics-rules/metric-rules-cache'
import type { LogsMetricsEmitter } from './metrics-rules/metrics-emitter'
import { LOGS_DLQ_OUTPUT, LOGS_OUTPUT, LogsDlqOutput, LogsOutput } from './outputs/outputs'
import type { PatternMessageKeysCache } from './pattern-message-keys-cache'
import { MAX_RETENTION_DAYS } from './retention/evaluate-retention'
import { DEFAULT_TRACES_RETENTION_DAYS } from './retention/tracing-config-cache'
import { compileRuleSet } from './sampling/compile-rules'
import type { SamplingRulesCache } from './sampling/sampling-rules-cache'
import { TracesIngestionConsumer } from './traces-ingestion-consumer'
import { LogsTransformerService } from './transformations/logs-transformer.service'
const LibrdKafkaError = require('node-rdkafka/lib/error')
const { err2str } = require('node-rdkafka/librdkafka') as { err2str: (code: number) => string }
// What capture-logs forwards for a producer that signed just before the request arrived. Both times come
// from `Date.now()`, which these tests pin, because a bare `new Date()` still reads the real clock.
function internalLogsHeaders(signature: string): Record<string, Buffer> {
return {
[INTERNAL_LOGS_HEADER]: Buffer.from(signature),
created_at: Buffer.from(new Date(Date.now()).toISOString()),
}
}
const DEFAULT_TEST_TIMEOUT = 5000
jest.setTimeout(DEFAULT_TEST_TIMEOUT)
const SNAPSHOT_OVERRIDES = { overrides: { team_id: 'TEAM_ID' } }
jest.mock('~/common/utils/posthog', () => {
const original = jest.requireActual('~/common/utils/posthog')
return {
...original,
captureException: jest.fn(),
}
})
let offsetIncrementer = 0
const LOG_RECORD_BASE_FIELDS = [
{ name: 'uuid', type: ['null', 'string'] },
{ name: 'trace_id', type: ['null', 'bytes'] },
{ name: 'span_id', type: ['null', 'bytes'] },
{ name: 'trace_flags', type: ['null', 'int'] },
{ name: 'timestamp', type: ['null', 'long'] },
{ name: 'observed_timestamp', type: ['null', 'long'] },
{ name: 'body', type: ['null', 'string'] },
{ name: 'severity_text', type: ['null', 'string'] },
{ name: 'severity_number', type: ['null', 'int'] },
{ name: 'service_name', type: ['null', 'string'] },
{ name: 'resource_attributes', type: ['null', { type: 'map', values: 'string' }] },
{ name: 'instrumentation_scope', type: ['null', 'string'] },
{ name: 'event_name', type: ['null', 'string'] },
{ name: 'attributes', type: ['null', { type: 'map', values: 'string' }] },
]
const createKafkaMessage = async (
logData: any,
headers: Record<string, string> = {},
attributes: Record<string, string> | null = null
): Promise<Message> => {
// Create a LogRecord from the log data
const record: LogRecord = {
uuid: `test-uuid-${offsetIncrementer}`,
trace_id: null,
span_id: null,
trace_flags: null,
timestamp: DateTime.now().toMillis() * 1000, // microseconds
observed_timestamp: DateTime.now().toMillis() * 1000,
body: JSON.stringify(logData),
severity_text: logData.level || 'info',
severity_number: 9,
service_name: logData.service || 'test-service',
resource_attributes: null,
instrumentation_scope: null,
event_name: null,
attributes,
}
// Encode as AVRO
const avro = require('avsc')
const logRecordType = avro.Type.forSchema({
type: 'record',
name: 'LogRecord',
fields: [
...LOG_RECORD_BASE_FIELDS,
{ name: 'pattern', type: ['null', 'string'], default: null },
{ name: 'pattern_version', type: ['null', 'int'], default: null },
],
})
const value = await encodeLogRecords(logRecordType, 'zstandard', [record])
return {
key: null,
value,
size: value.length,
topic: 'test',
offset: offsetIncrementer++,
timestamp: DateTime.now().toMillis(),
partition: 1,
headers: Object.entries(headers).map(([key, value]) => ({
[key]: Buffer.from(value),
})),
}
}
const createKafkaMessages = async (logData: any[], headers: Record<string, string> = {}): Promise<Message[]> => {
return Promise.all(logData.map((data) => createKafkaMessage(data, headers)))
}
// Single Kafka message carrying several log records, so a drop rule can remove some rows while
// others survive (partial drop) — the path that re-encodes and re-forwards to the output topic.
const createMultiRecordKafkaMessage = async (
logDataList: any[],
headers: Record<string, string> = {}
): Promise<Message> => {
const avro = require('avsc')
const logRecordType = avro.Type.forSchema({
type: 'record',
name: 'LogRecord',
fields: [...LOG_RECORD_BASE_FIELDS, { name: 'retention_days', type: ['null', 'int'], default: null }],
})
const records: LogRecord[] = logDataList.map((logData, i) => ({
retention_days: logData.retentionDays ?? null,
uuid: `test-uuid-${offsetIncrementer}-${i}`,
trace_id: null,
span_id: null,
trace_flags: null,
timestamp: logData.timestampMicros ?? DateTime.now().toMillis() * 1000,
observed_timestamp: DateTime.now().toMillis() * 1000,
body: JSON.stringify(logData),
severity_text: logData.level || 'info',
severity_number: 9,
service_name: logData.service || 'test-service',
resource_attributes: null,
instrumentation_scope: null,
event_name: null,
attributes: null,
}))
const value = await encodeLogRecords(logRecordType, 'zstandard', records)
return {
key: null,
value,
size: value.length,
topic: 'test',
offset: offsetIncrementer++,
timestamp: DateTime.now().toMillis(),
partition: 1,
headers: Object.entries(headers).map(([key, value]) => ({
[key]: Buffer.from(value),
})),
}
}
describe('LogsIngestionConsumer', () => {
let consumer: LogsIngestionConsumer
let hub: Hub
let team: Team
let team2: Team
let fixedTime: DateTime
let logMessageDroppedCounterSpy: jest.SpyInstance
let recordLogMessageDroppedSpy: jest.SpyInstance
const createLogsIngestionConsumer = async (
hub: Hub,
overrides: any = {},
depsPartial: Partial<
Pick<
LogsIngestionConsumerDeps,
| 'samplingRulesCache'
| 'metricRulesCache'
| 'metricsEmitter'
| 'logsTransformer'
| 'patternMessageKeysCache'
| 'internalSourcePolicies'
>
> = {}
) => {
const consumer = new LogsIngestionConsumer(
hub,
{
...hub,
outputs: new IngestionOutputs<AppMetricsOutput | LogsOutput | LogsDlqOutput>({
[APP_METRICS_OUTPUT]: new SingleIngestionOutput(
APP_METRICS_OUTPUT,
KAFKA_APP_METRICS_2,
mockProducer,
'test'
),
[LOGS_OUTPUT]: new SingleIngestionOutput(LOGS_OUTPUT, KAFKA_LOGS_CLICKHOUSE, mockProducer, 'test'),
[LOGS_DLQ_OUTPUT]: new SingleIngestionOutput(
LOGS_DLQ_OUTPUT,
KAFKA_LOGS_INGESTION_DLQ,
mockProducer,
'test'
),
}),
usageBatch: new UsageRecordBatch(null, { unit: 'bytes', isTeamEnabled: () => false }),
dependencyRetry: { retryCount: 2, initialRetryDelayMs: 1 },
...depsPartial,
},
overrides
)
// NOTE: We don't actually use kafka so we skip instantiation for faster tests
consumer['kafkaConsumer'] = {
connect: jest.fn(),
disconnect: jest.fn(),
isHealthy: jest.fn().mockReturnValue({ status: 'healthy' }),
} as any
await consumer.start()
return consumer
}
const createLogMessage = (logData?: any): any => ({
level: 'info',
message: 'Test log message',
timestamp: fixedTime.toISO()!,
service: 'test-service',
...logData,
})
beforeEach(async () => {
fixedTime = DateTime.fromObject({ year: 2025, month: 1, day: 1 }, { zone: 'UTC' })
jest.spyOn(Date, 'now').mockReturnValue(fixedTime.toMillis())
jest.spyOn(Date.prototype, 'toISOString').mockReturnValue(fixedTime.toISO()!)
offsetIncrementer = 0
hub = await createHub()
team = (await createTestTeamFixture(hub.postgres)).team
const team2Id = await createTeam(hub.postgres, team.organization_id)
team2 = (await getTeam(hub.postgres, team2Id))!
consumer = await createLogsIngestionConsumer(hub)
logMessageDroppedCounterSpy = jest.spyOn(logMessageDroppedCounter, 'inc')
recordLogMessageDroppedSpy = jest.spyOn(otelMetrics, 'recordLogMessageDropped')
// Default to not quota limited - tests can override this
jest.spyOn(hub.quotaLimiting, 'isTeamTokenQuotaLimited').mockResolvedValue(false)
})
afterEach(async () => {
await consumer.stop()
await closeHub(hub)
})
afterAll(() => {
jest.useRealTimers()
})
const waitForBackgroundTasks = async (
promise: Promise<{
backgroundTask?: Promise<any>
}>
) => {
await (
await promise
).backgroundTask
}
// Simple helper to transform getProducedMessages structure to match getProducedKafkaMessages
const getProducedKafkaMessages = () => {
const producedMessages = mockProducerObserver.getProducedMessages()
const result = []
for (const batch of producedMessages) {
for (const message of batch.messages) {
result.push({
topic: batch.topic,
key: message.key,
value: message.value || {},
headers: message.headers,
})
}
}
return result
}
describe('general', () => {
it('should have the correct config', () => {
expect(consumer['name']).toEqual('LogsIngestionConsumer')
expect(consumer['groupId']).toEqual('ingestion-logs')
expect(consumer['topic']).toEqual('logs_ingestion_test')
})
it('should allow config overrides', async () => {
const overrides = {
LOGS_INGESTION_CONSUMER_GROUP_ID: 'custom-group',
LOGS_INGESTION_CONSUMER_CONSUME_TOPIC: 'custom-topic',
}
const customConsumer = await createLogsIngestionConsumer(hub, overrides)
expect(customConsumer['groupId']).toBe('custom-group')
expect(customConsumer['topic']).toBe('custom-topic')
await customConsumer.stop()
})
it('should process a valid log message', async () => {
const logData = createLogMessage()
const messages = await createKafkaMessages([logData], {
token: team.api_token,
})
await waitForBackgroundTasks(consumer.processKafkaBatch(messages))
expect(
forSnapshot(
getProducedKafkaMessages().map((m) => m.headers),
SNAPSHOT_OVERRIDES
)
).toMatchSnapshot()
})
it('should process multiple log messages', async () => {
const logData = [
createLogMessage({ level: 'info', message: 'First log' }),
createLogMessage({ level: 'error', message: 'Second log' }),
createLogMessage({ level: 'debug', message: 'Third log' }),
]
const messages = await createKafkaMessages(logData, {
token: team.api_token,
})
await waitForBackgroundTasks(consumer.processKafkaBatch(messages))
const producedMessages = getProducedKafkaMessages()
expect(producedMessages).toHaveLength(3)
expect(
forSnapshot(
producedMessages.map((m) => m.headers),
SNAPSHOT_OVERRIDES
)
).toMatchSnapshot()
})
})
describe('message parsing and validation', () => {
it('should drop messages with missing token', async () => {
const logData = createLogMessage()
const messages = await createKafkaMessages([logData], {
// missing token
})
await waitForBackgroundTasks(consumer.processKafkaBatch(messages))
expect(getProducedKafkaMessages()).toHaveLength(0)
expect(logMessageDroppedCounterSpy).toHaveBeenCalledWith({ reason: 'missing_token', team_id: 'unknown' })
})
it('should drop messages with invalid token', async () => {
const logData = createLogMessage()
const messages = await createKafkaMessages([logData], {
token: 'invalid-token',
})
await waitForBackgroundTasks(consumer.processKafkaBatch(messages))
expect(getProducedKafkaMessages()).toHaveLength(0)
})
it('should preserve kafka message headers', async () => {
const logData = createLogMessage()
const messages = await createKafkaMessages([logData], {
myHeader: 'hello',
token: team.api_token,
})
await waitForBackgroundTasks(consumer.processKafkaBatch(messages))
const producedMessages = getProducedKafkaMessages()
expect(
forSnapshot(
producedMessages.map((m) => m.headers),
SNAPSHOT_OVERRIDES
)
).toMatchSnapshot()
})
it('should overwrite existing headers', async () => {
const logData = createLogMessage()
const messages = await createKafkaMessages([logData], {
token: team.api_token,
team_id: '999',
})
await waitForBackgroundTasks(consumer.processKafkaBatch(messages))
expect(
forSnapshot(
getProducedKafkaMessages().map((m) => m.headers),
SNAPSHOT_OVERRIDES
)
).toMatchSnapshot()
})
it('should handle parse errors gracefully', async () => {
const messages = [
{
key: null,
value: Buffer.from('invalid-json-is-fine-we-dont-check-for-errors'),
size: 1,
topic: 'test',
offset: offsetIncrementer++,
timestamp: DateTime.now().toMillis(),
partition: 1,
headers: [{ token: Buffer.from('missing') }],
} as Message,
]
await waitForBackgroundTasks(consumer.processKafkaBatch(messages))
expect(getProducedKafkaMessages()).toHaveLength(0)
expect(logMessageDroppedCounterSpy).toHaveBeenCalledWith({ reason: 'team_not_found', team_id: 'unknown' })
})
it('should drop messages when team lookup fails and record unknown team_id', async () => {
jest.spyOn(hub.teamManager, 'getTeamByToken').mockRejectedValueOnce(new Error('Database error'))
const logData = createLogMessage()
const messages = await createKafkaMessages([logData], {
token: team.api_token,
})
await waitForBackgroundTasks(consumer.processKafkaBatch(messages))
expect(getProducedKafkaMessages()).toHaveLength(0)
expect(logMessageDroppedCounterSpy).toHaveBeenCalledWith({
reason: 'team_lookup_error',
team_id: 'unknown',
})
})
})
describe('batch processing', () => {
it('should handle empty batch', async () => {
const result = await consumer.processBatch([])
expect(result.backgroundTask).toBeUndefined()
expect(result.messages).toEqual([])
await result.backgroundTask
})
it('should process batch with valid messages', async () => {
const logData = createLogMessage()
const messages = await createKafkaMessages([logData], {
token: team.api_token,
})
await waitForBackgroundTasks(consumer.processKafkaBatch(messages))
const producedMessages = getProducedKafkaMessages()
expect(producedMessages).toHaveLength(1)
expect(producedMessages[0].topic).toBe('clickhouse_logs_test')
expect(producedMessages[0].headers).toEqual({
token: team.api_token,
team_id: team.id.toString(),
'json-parse': 'false',
'retention-days': DEFAULT_LOGS_RETENTION_DAYS.toString(),
source_topic: messages[0].topic,
source_partition: messages[0].partition.toString(),
})
})
it('marks a team that receives its first logs as having ingested data', async () => {
const newTeamId = await createTeam(hub.postgres, team.organization_id, undefined, { ingested_event: false })
const newTeam = await getTeam(hub.postgres, newTeamId)
const messages = await createKafkaMessages([createLogMessage()], { token: newTeam!.api_token })
await waitForBackgroundTasks(consumer.processKafkaBatch(messages))
expect((await getTeam(hub.postgres, newTeamId))?.ingested_event).toBe(true)
})
it('should produce messages with correct headers', async () => {
const logData = createLogMessage({ level: 'error', message: 'Critical error' })
const messages = await createKafkaMessages([logData], {
token: team.api_token,
})
await waitForBackgroundTasks(consumer.processKafkaBatch(messages))
const producedMessages = getProducedKafkaMessages()
expect(producedMessages).toHaveLength(1)
const message = producedMessages[0]
expect(message.topic).toBe('clickhouse_logs_test')
expect(message.key).toBeNull()
expect(message.headers).toEqual({
token: team.api_token,
team_id: team.id.toString(),
'json-parse': 'false',
'retention-days': DEFAULT_LOGS_RETENTION_DAYS.toString(),
source_topic: messages[0].topic,
source_partition: messages[0].partition.toString(),
})
})
it('should use custom logs_settings when set', async () => {
// Update team with custom logs settings
await hub.postgres.query(
PostgresUse.COMMON_WRITE,
`UPDATE posthog_team
SET logs_settings = $1
WHERE id = $2`,
[JSON.stringify({ json_parse_logs: false, retention_days: 30 }), team.id],
'updateTeamLogsSettings'
)
const logData = createLogMessage()
const messages = await createKafkaMessages([logData], {
token: team.api_token,
})
await waitForBackgroundTasks(consumer.processKafkaBatch(messages))
const producedMessages = getProducedKafkaMessages()
expect(producedMessages).toHaveLength(1)
const message = producedMessages[0]
expect(message.headers).toEqual({
token: team.api_token,
team_id: team.id.toString(),
'json-parse': 'false',
'retention-days': '30',
source_topic: messages[0].topic,
source_partition: messages[0].partition.toString(),
})
})
})
describe('service interface', () => {
it('should provide correct service interface', () => {
const service = consumer.service
expect(service.id).toBe('LogsIngestionConsumer')
expect(typeof service.onShutdown).toBe('function')
expect(typeof service.healthcheck).toBe('function')
})
it('should return healthy status', () => {
const healthCheck = consumer.isHealthy()
expect(healthCheck).toEqual({ status: 'healthy' })
})
})
describe('error handling', () => {
const kafkaDeliveryError = (code: number): Error => LibrdKafkaError.create(new Error(err2str(code)))
it.each([
['a processing error', () => new Error('Producer error')],
['a Kafka error that a replay cannot fix', () => kafkaDeliveryError(CODES.ERRORS.ERR_INVALID_RECORD)],
])('should send failed messages to DLQ on %s', async (_, makeError) => {
const logData = createLogMessage()
const messages = await createKafkaMessages([logData], {
token: team.api_token,
})
const logMessageDlqCounterSpy = jest.spyOn(logMessageDlqCounter, 'inc')
// Throw for the main logs topic but pass through to the real mock producer for the DLQ.
const originalQueueMessages = mockProducer.queueMessages
const passthrough = originalQueueMessages.bind(mockProducer)
const queueSpy = jest.fn().mockImplementation((topicMessages) => {
const t = Array.isArray(topicMessages) ? topicMessages[0]?.topic : topicMessages.topic
if (t === 'clickhouse_logs_test') {
throw makeError()
}
return passthrough(topicMessages)
})
mockProducer.queueMessages = queueSpy
try {
await waitForBackgroundTasks(consumer.processKafkaBatch(messages))
// Check that DLQ counter was incremented with team_id
expect(logMessageDlqCounterSpy).toHaveBeenCalledWith({
reason: 'Error',
team_id: team.id.toString(),
})
// Check that a message was produced to the DLQ topic
const dlqCalls = queueSpy.mock.calls.filter((call) => {
const t = Array.isArray(call[0]) ? call[0][0]?.topic : call[0].topic
return t === 'logs_ingestion_dlq_test'
})
expect(dlqCalls.length).toBeGreaterThan(0)
// Verify DLQ message has error metadata in headers
const dlqArg = dlqCalls[0][0]
const dlqMessage = (Array.isArray(dlqArg) ? dlqArg[0] : dlqArg).messages[0]
expect(dlqMessage.headers).toHaveProperty('error_message')
expect(dlqMessage.headers).toHaveProperty('error_name')
expect(dlqMessage.headers).toHaveProperty('failed_at')
// Position of the source record, so it can be read back and replayed
expect(dlqMessage.headers.source_topic).toEqual(messages[0].topic)
expect(dlqMessage.headers.source_partition).toEqual(messages[0].partition.toString())
expect(dlqMessage.headers.source_offset).toEqual(messages[0].offset.toString())
} finally {
mockProducer.queueMessages = originalQueueMessages
}
})
it('should retry a dependency blip in place and produce the message', async () => {
const logData = createLogMessage()
const messages = await createKafkaMessages([logData], {
token: team.api_token,
})
const originalQueueMessages = mockProducer.queueMessages
const passthrough = originalQueueMessages.bind(mockProducer)
let logsProduceAttempts = 0
mockProducer.queueMessages = jest.fn().mockImplementation((topicMessages) => {
const t = Array.isArray(topicMessages) ? topicMessages[0]?.topic : topicMessages.topic
if (t === KAFKA_LOGS_CLICKHOUSE && logsProduceAttempts++ === 0) {
throw new DependencyUnavailableError('broker down', 'Kafka', new Error('broker down'))
}
return passthrough(topicMessages)
})
try {
// A blip that clears inside the backoff must not fail the batch: a replay would
// produce every other message of the batch a second time.
await waitForBackgroundTasks(consumer.processKafkaBatch(messages))
const produced = getProducedKafkaMessages()
expect(produced.filter((m) => m.topic === KAFKA_LOGS_CLICKHOUSE)).toHaveLength(1)
expect(produced.filter((m) => m.topic === KAFKA_LOGS_INGESTION_DLQ)).toHaveLength(0)
} finally {
mockProducer.queueMessages = originalQueueMessages
}
})
it.each([
[
'a dependency is unavailable',
() => new DependencyUnavailableError('broker down', 'Kafka', new Error('broker down')),
],
['a delivery times out', () => kafkaDeliveryError(CODES.ERRORS.ERR__MSG_TIMED_OUT)],
['the producer queue is full', () => LibrdKafkaError.create(CODES.ERRORS.ERR__QUEUE_FULL)],
['the topic rejects authorization', () => kafkaDeliveryError(CODES.ERRORS.ERR_TOPIC_AUTHORIZATION_FAILED)],
])('should fail the batch instead of DLQing when %s', async (_, makeError) => {
const logData = createLogMessage()
const messages = await createKafkaMessages([logData], {
token: team.api_token,
})
const originalQueueMessages = mockProducer.queueMessages
const passthrough = originalQueueMessages.bind(mockProducer)
mockProducer.queueMessages = jest.fn().mockImplementation((topicMessages) => {
const t = Array.isArray(topicMessages) ? topicMessages[0]?.topic : topicMessages.topic
if (t === KAFKA_LOGS_CLICKHOUSE) {
throw makeError()
}
return passthrough(topicMessages)
})
try {
// The message is good data that succeeds on redelivery. Quarantining it would
// move it out of the source topic for an outage that was not its fault.
await expect(waitForBackgroundTasks(consumer.processKafkaBatch(messages))).rejects.toThrow()
expect(getProducedKafkaMessages().filter((m) => m.topic === KAFKA_LOGS_INGESTION_DLQ)).toHaveLength(0)
} finally {
mockProducer.queueMessages = originalQueueMessages
}
})
it('should send a message with an unparseable size header to DLQ instead of ingesting it', async () => {
const logData = createLogMessage()
const messages = await createKafkaMessages([logData], {
token: team.api_token,
// NaN here would otherwise flow into the usage counters and per-team billing stats
bytes_uncompressed: 'not-a-number',
})
await waitForBackgroundTasks(consumer.processKafkaBatch(messages))
const produced = getProducedKafkaMessages()
expect(produced.filter((m) => m.topic === KAFKA_LOGS_CLICKHOUSE)).toHaveLength(0)
const dlq = produced.filter((m) => m.topic === KAFKA_LOGS_INGESTION_DLQ)
expect(dlq).toHaveLength(1)
expect(dlq[0]?.headers?.error_message).toEqual('invalid_size_headers')
expect(dlq[0]?.headers?.team_id).toEqual(team.id.toString())
})
it.each([
['the message fails to parse', { bytes_uncompressed: 'not-a-number' }],
['the message fails to process', {}],
])('should fail the batch when %s and cannot be written to the DLQ', async (_, headers) => {
const logData = createLogMessage()
const messages = await createKafkaMessages([logData], {
token: team.api_token,
...headers,
})
// Every produce fails: the logs topic write fails the message, and the DLQ write that
// follows fails too.
const originalQueueMessages = mockProducer.queueMessages
mockProducer.queueMessages = jest.fn().mockRejectedValue(new Error('DLQ unavailable'))
try {
// Resolving here would commit the source offset with no copy anywhere, so the
// only record of the payload would be gone.
await expect(waitForBackgroundTasks(consumer.processKafkaBatch(messages))).rejects.toThrow(
'DLQ unavailable'
)
} finally {
mockProducer.queueMessages = originalQueueMessages
}
})
})
describe('describeBatchPosition', () => {
it('collapses each partition to its offset span', () => {
const at = (partition: number, offset: number): Message => ({ partition, offset }) as Message
expect(describeBatchPosition([at(3, 1000), at(3, 1099), at(7, 42)])).toEqual({
'3': '1000-1099',
'7': '42',
})
})
})
describe('parseSizeHeader', () => {
it.each([
['1234', 1234],
[undefined, 0],
['not-a-number', null],
['-5', null],
])('parses %p as %p', (raw, expected) => {
expect(parseSizeHeader(raw)).toEqual(expected)
})
})
describe('parseMinTimestampHeader', () => {
it.each([
['1790000000000000', 1790000000000000],
[undefined, undefined],
['', undefined],
['not-a-number', undefined],
])('parses %p as %p', (raw, expected) => {
expect(parseMinTimestampHeader(raw)).toEqual(expected)
})
})
describe('message routing', () => {
it('should route messages to correct ClickHouse topic', async () => {
const logData = createLogMessage()
const messages = await createKafkaMessages([logData], {
token: team.api_token,
})
await waitForBackgroundTasks(consumer.processKafkaBatch(messages))
const producedMessages = getProducedKafkaMessages()
expect(producedMessages).toHaveLength(1)
expect(producedMessages[0].topic).toBe('clickhouse_logs_test')
})
it('should handle messages from different teams', async () => {
const logData1 = createLogMessage({ message: 'Team 1 log' })
const logData2 = createLogMessage({ message: 'Team 2 log' })
const messages = [
...(await createKafkaMessages([logData1], {
token: team.api_token,
})),
...(await createKafkaMessages([logData2], {
token: team2.api_token,
})),
]
await waitForBackgroundTasks(consumer.processKafkaBatch(messages))
const producedMessages = getProducedKafkaMessages()
expect(producedMessages).toHaveLength(2)
const team1Message = producedMessages.find((m) => m.headers?.team_id === team.id.toString())
const team2Message = producedMessages.find((m) => m.headers?.team_id === team2.id.toString())
expect(team1Message).toBeDefined()
expect(team2Message).toBeDefined()
expect(team1Message?.headers?.token).toBe(team.api_token)
expect(team2Message?.headers?.token).toBe(team2.api_token)
})
})
describe('message content preservation', () => {
it('should preserve original message content', async () => {
const logData = createLogMessage({
level: 'warn',
message: 'Warning message',
timestamp: '2025-01-01T12:00:00.000Z',
service: 'api-service',
request_id: 'req-123',
user_id: 'user-456',
})
const messages = await createKafkaMessages([logData], {
token: team.api_token,
})
await waitForBackgroundTasks(consumer.processKafkaBatch(messages))
const producedMessages = getProducedKafkaMessages()
expect(producedMessages).toHaveLength(1)
expect(producedMessages[0].value).toEqual(messages[0].value)
})
it('should handle binary log data as we do not parse it', async () => {
const binaryData = Buffer.from('binary log content')
const messages = [
{
key: null,
value: binaryData,
size: 1,
topic: 'test',
offset: offsetIncrementer++,
timestamp: DateTime.now().toMillis(),
partition: 1,
headers: [{ token: Buffer.from(team.api_token) }],
} as Message,
]
await waitForBackgroundTasks(consumer.processKafkaBatch(messages))
const producedMessages = mockProducerObserver.getProducedMessages()
expect(producedMessages).toHaveLength(1)
expect(producedMessages[0].messages[0].value).toEqual(binaryData)
})
})
describe('rate limiting', () => {
let bytesReceivedSpy: jest.SpyInstance
let bytesAllowedSpy: jest.SpyInstance
let bytesDroppedSpy: jest.SpyInstance
let recordsReceivedSpy: jest.SpyInstance
let recordsAllowedSpy: jest.SpyInstance
let recordsDroppedSpy: jest.SpyInstance
beforeEach(() => {
bytesReceivedSpy = jest.spyOn(logsBytesReceivedCounter, 'inc')
bytesAllowedSpy = jest.spyOn(logsBytesAllowedCounter, 'inc')
bytesDroppedSpy = jest.spyOn(logsBytesDroppedCounter, 'inc')
recordsReceivedSpy = jest.spyOn(logsRecordsReceivedCounter, 'inc')
recordsAllowedSpy = jest.spyOn(logsRecordsAllowedCounter, 'inc')
recordsDroppedSpy = jest.spyOn(logsRecordsDroppedCounter, 'inc')
})
it('should track metrics for messages', async () => {
hub.LOGS_LIMITER_BUCKET_SIZE_KB = 2
hub.LOGS_LIMITER_REFILL_RATE_KB_PER_SECOND = 1
hub.LOGS_LIMITER_TTL_SECONDS = 3600
await consumer.stop()
consumer = await createLogsIngestionConsumer(hub)
const logData1 = createLogMessage({ message: 'First' })
const logData2 = createLogMessage({ message: 'Second' })
const messages = [
...(await createKafkaMessages([logData1], {
token: team.api_token,
bytes_uncompressed: '1024',
bytes_compressed: '512',
record_count: '5',
})),
...(await createKafkaMessages([logData2], {
token: team.api_token,
bytes_uncompressed: '2048',
bytes_compressed: '1024',
record_count: '10',
})),
]
await waitForBackgroundTasks(consumer.processKafkaBatch(messages))
// Filter to only the logs topic (excludes app_metrics2 messages)
const logsMessages = getProducedKafkaMessages().filter((m) => m.topic === 'clickhouse_logs_test')
expect(logsMessages).toHaveLength(1)
expect(bytesReceivedSpy).toHaveBeenCalledWith(3072)
expect(bytesAllowedSpy).toHaveBeenCalledWith(1024)
expect(bytesDroppedSpy).toHaveBeenCalledWith({ team_id: team.id.toString() }, 2048)
expect(recordsReceivedSpy).toHaveBeenCalledWith(15)
expect(recordsAllowedSpy).toHaveBeenCalledWith(5)
expect(recordsDroppedSpy).toHaveBeenCalledWith({ team_id: team.id.toString() }, 10)
expect(logMessageDroppedCounterSpy).toHaveBeenCalledWith(
{ reason: 'rate_limited', team_id: team.id.toString() },
1
)
})
it('dual-emits usage counters into the OTel metrics push', async () => {
// Wiring guard: the unit tests on ingestion-otel-metrics prove the record
// functions work; this proves the consumer actually calls them.
const otelExporter = new InMemoryMetricExporter(0)
const otelReader = new PeriodicExportingMetricReader({
exporter: otelExporter,
exportIntervalMillis: 60_000,
})
const otelProvider = new MeterProvider({ readers: [otelReader] })
metricsApi.setGlobalMeterProvider(otelProvider)
resetLogsIngestionInstrumentsForTests()
try {
hub.LOGS_LIMITER_BUCKET_SIZE_KB = 2
hub.LOGS_LIMITER_REFILL_RATE_KB_PER_SECOND = 1
hub.LOGS_LIMITER_TTL_SECONDS = 3600
await consumer.stop()
consumer = await createLogsIngestionConsumer(hub)
const messages = [
...(await createKafkaMessages([createLogMessage({ message: 'First' })], {
token: team.api_token,
bytes_uncompressed: '1024',
bytes_compressed: '512',
record_count: '5',
})),
...(await createKafkaMessages([createLogMessage({ message: 'Second' })], {
token: team.api_token,
bytes_uncompressed: '2048',
bytes_compressed: '1024',
record_count: '10',
})),
]
await waitForBackgroundTasks(consumer.processKafkaBatch(messages))
await otelReader.forceFlush()
const pointsByMetric = new Map(
otelExporter
.getMetrics()
.flatMap((rm) => rm.scopeMetrics)
.flatMap((sm) => sm.metrics)
.map((m) => [m.descriptor.name, m.dataPoints as readonly DataPoint<number>[]])
)
expect(pointsByMetric.get('logs_ingestion_bytes_received_total')?.[0]?.value).toEqual(3072)
expect(pointsByMetric.get('logs_ingestion_bytes_allowed_total')?.[0]?.value).toEqual(1024)
expect(pointsByMetric.get('logs_ingestion_bytes_dropped_total')?.[0]).toMatchObject({
attributes: { team_id: team.id.toString() },
value: 2048,
})
expect(pointsByMetric.get('logs_ingestion_message_dropped_count')?.[0]).toMatchObject({
attributes: { reason: 'rate_limited', team_id: team.id.toString() },
value: 1,
})
} finally {
await otelProvider.shutdown()
metricsApi.disable()
resetLogsIngestionInstrumentsForTests()
}
})
it('should handle missing header values with defaults', async () => {
const logData = createLogMessage()
const messages = await createKafkaMessages([logData], {
token: team.api_token,
})
await waitForBackgroundTasks(consumer.processKafkaBatch(messages))
expect(getProducedKafkaMessages()).toHaveLength(1)
expect(bytesReceivedSpy).toHaveBeenCalledWith(0)
expect(recordsReceivedSpy).toHaveBeenCalledWith(0)
})
})
describe('backfill gate', () => {
it.each([
{
name: 'refuses a backfill request from a team not on the list',
header: true,
listed: false,
produced: 0,
},
{ name: 'accepts a backfill request from a listed team', header: true, listed: true, produced: 1 },
{ name: 'leaves live traffic alone', header: false, listed: false, produced: 1 },
])('$name', async ({ header, listed, produced }) => {
await consumer.stop()
consumer = await createLogsIngestionConsumer(hub, {
LOGS_BACKFILL_ENABLED_TEAMS: listed ? String(team.id) : String(team2.id),
})
// The row is recent on purpose: a backfill request can carry rows from the last day.
const messages = await createKafkaMessages([createLogMessage()], {
token: team.api_token,
record_count: '1',
...(header ? { backfill_days: '540' } : {}),
})
await waitForBackgroundTasks(consumer.processKafkaBatch(messages))
expect(getProducedKafkaMessages().filter((m) => m.topic === KAFKA_LOGS_CLICKHOUSE)).toHaveLength(produced)
if (produced === 0) {
expect(logMessageDroppedCounterSpy).toHaveBeenCalledWith(
{ reason: 'backfill_not_enabled', team_id: team.id.toString() },
1
)
}
})
})
describe('internal logs', () => {
const SIGNING_KEY = 'internal-logs-test-key'
it.each([
{
name: "stores a signed batch outside quota, billing and the team's drop rules",
signed: true,
billed: false,
},
{ name: 'bills a batch whose signature does not verify', signed: false, billed: true },
{
name: "bills a signed batch from a billed source but skips the team's drop rules",
signed: true,
billed: true,
},
])('$name', async ({ signed, billed }) => {
await consumer.stop()
const dropEverything: Pick<SamplingRulesCache, 'getCompiledRuleSet'> = {
getCompiledRuleSet: () =>
Promise.resolve(
compileRuleSet([
{
id: '00000000-0000-0000-0000-0000000000aa',
rule_type: 'severity_sampling',
scope_service: 'test-service',
scope_path_pattern: null,
scope_attribute_filters: [],
config: { actions: { INFO: { type: 'drop' } } },
},
])
),
}
consumer = await createLogsIngestionConsumer(
hub,
{
LOGS_INTERNAL_INGESTION_ENABLED_TEAMS: String(team.id),
LOGS_INTERNAL_INGESTION_SIGNING_KEYS: SIGNING_KEY,
},
{
samplingRulesCache: dropEverything as SamplingRulesCache,
internalSourcePolicies: {
hog_flow: { billed, teamRules: false, serviceName: 'posthog-internal-workflows' },
},
}
)
// A quota limit drops every billed batch, so it applies only to the rows that test the internal bypass.
jest.mocked(hub.quotaLimiting.isTeamTokenQuotaLimited).mockResolvedValue(!(signed && billed))
const [message] = await createKafkaMessages([createLogMessage()], {
token: team.api_token,
bytes_uncompressed: '100',
record_count: '1',
})
const signature = signed
? signInternalLogs(SIGNING_KEY, 'hog_flow', team.id, new Date(Date.now()))
: 'v1.hog_flow.not-a-signature'
message.headers!.push(internalLogsHeaders(signature))
const recordLogsReceivedSpy = jest.spyOn(otelMetrics, 'recordLogsReceived')
const recordInternalLogsSpy = jest.spyOn(otelMetrics, 'recordInternalLogs')
await waitForBackgroundTasks(consumer.processKafkaBatch([message]))
const metricNames = mockProducerObserver
.getProducedKafkaMessagesForTopic(KAFKA_APP_METRICS_2)
.filter((m) => m.value.team_id === team.id)
.map((m) => m.value.metric_name)
const produced = getProducedKafkaMessages().filter((m) => m.topic === KAFKA_LOGS_CLICKHOUSE)
expect({
produced: produced.length,
forwardsSignature: produced.some((m) => INTERNAL_LOGS_HEADER in (m.headers ?? {})),
internalSource: produced[0]?.headers?.[INTERNAL_SOURCE_HEADER],
received: recordLogsReceivedSpy.mock.calls,
internal: recordInternalLogsSpy.mock.calls,
internalMetrics: ['bytes_ingested_internal', 'records_ingested_internal'].every((n) =>
metricNames.includes(n)
),
billedMetrics: ['bytes_ingested', 'records_ingested', 'bytes_received'].some((n) =>
metricNames.includes(n)
),
}).toEqual({
produced: signed ? 1 : 0,
forwardsSignature: false,
internalSource: signed ? 'hog_flow' : undefined,
received: [billed ? [100, 1] : [0, 0]],
internal: billed ? [] : [[team.id, 100, 1]],
internalMetrics: !billed,
billedMetrics: billed,
})
})
it('drops a row past its stamped retention without crediting billing', async () => {
await consumer.stop()
consumer = await createLogsIngestionConsumer(hub, {
LOGS_INTERNAL_INGESTION_ENABLED_TEAMS: String(team.id),
LOGS_INTERNAL_INGESTION_SIGNING_KEYS: SIGNING_KEY,
})
const fiveDaysAgoMicros = DateTime.now().minus({ days: 5 }).toMillis() * 1000
const message = await createMultiRecordKafkaMessage(
[{ level: 'info', timestampMicros: fiveDaysAgoMicros, retentionDays: 1 }, { level: 'info' }],
{
token: team.api_token,
bytes_uncompressed: '200',
record_count: '2',
min_timestamp: fiveDaysAgoMicros.toString(),
}
)
message.headers!.push(
internalLogsHeaders(signInternalLogs(SIGNING_KEY, 'hog_flow', team.id, new Date(Date.now())))
)
const bytesCreditSpy = jest.spyOn(logsBillingBytesCreditedCounter, 'inc')
const recordsCreditSpy = jest.spyOn(logsBillingRecordsCreditedCounter, 'inc')
await waitForBackgroundTasks(consumer.processKafkaBatch([message]))
const expiredMetric = mockProducerObserver
.getProducedKafkaMessagesForTopic(KAFKA_APP_METRICS_2)
.map((m) => m.value)
.find((v) => v.metric_name === 'records_dropped_retention_expired' && v.team_id === team.id)
expect({
expired: expiredMetric?.count,
produced: getProducedKafkaMessages().filter((m) => m.topic === KAFKA_LOGS_CLICKHOUSE).length,
credits: [...bytesCreditSpy.mock.calls, ...recordsCreditSpy.mock.calls],
}).toEqual({ expired: 1, produced: 1, credits: [] })
})
it.each([
{ name: 'drops a signed backfill without writing to the team usage row', backfill: true },
{ name: 'keeps the usage of a failed internal batch out of the next flush', backfill: false },
])('$name', async ({ backfill }) => {
await consumer.stop()
consumer = await createLogsIngestionConsumer(hub, {
LOGS_BACKFILL_ENABLED_TEAMS: String(team2.id),
LOGS_INTERNAL_INGESTION_ENABLED_TEAMS: String(team.id),
LOGS_INTERNAL_INGESTION_SIGNING_KEYS: SIGNING_KEY,
})
const [internal] = await createKafkaMessages([createLogMessage()], {
token: team.api_token,
bytes_uncompressed: '100',
record_count: '1',
...(backfill ? { backfill_days: '540' } : {}),
})
internal.headers!.push(
internalLogsHeaders(signInternalLogs(SIGNING_KEY, 'hog_flow', team.id, new Date(Date.now())))
)
const originalQueueMessages = mockProducer.queueMessages
const passthrough = originalQueueMessages.bind(mockProducer)
mockProducer.queueMessages = jest.fn().mockImplementation((topicMessages) => {
if (topicMessages.topic === KAFKA_LOGS_CLICKHOUSE && !backfill) {
throw new DependencyUnavailableError('broker down', 'Kafka', new Error('broker down'))
}
return passthrough(topicMessages)
})
try {
await consumer.processKafkaBatch([internal]).then((r) => r.backgroundTask?.catch(() => undefined))
mockProducer.queueMessages = originalQueueMessages
// A later batch of another team flushes the shared usage aggregator.
const [next] = await createKafkaMessages([createLogMessage()], { token: team2.api_token })
await waitForBackgroundTasks(consumer.processKafkaBatch([next]))
} finally {
mockProducer.queueMessages = originalQueueMessages
}
expect(
mockProducerObserver
.getProducedKafkaMessagesForTopic(KAFKA_APP_METRICS_2)
.filter((m) => m.value.team_id === team.id)
.map((m) => m.value.metric_name)
).toEqual([])
})
})
describe('filterQuotaLimitedMessages', () => {
it('should allow messages when not quota limited', async () => {
jest.mocked(hub.quotaLimiting.isTeamTokenQuotaLimited).mockResolvedValue(false)
const messages = await createKafkaMessages([createLogMessage()], {
token: team.api_token,
bytes_uncompressed: '1024',
record_count: '5',
})
const parsed = await consumer['_parseKafkaBatch'](messages)
const { quotaAllowedMessages, quotaDroppedMessages } = await consumer['filterQuotaLimitedMessages'](parsed)
expect(quotaAllowedMessages).toHaveLength(1)
expect(quotaDroppedMessages).toHaveLength(0)
expect(hub.quotaLimiting.isTeamTokenQuotaLimited).toHaveBeenCalledWith(team.api_token, 'logs_mb_ingested')
})
it('should drop messages when quota limited', async () => {
jest.mocked(hub.quotaLimiting.isTeamTokenQuotaLimited).mockResolvedValue(true)
const messages = await createKafkaMessages([createLogMessage()], {
token: team.api_token,
bytes_uncompressed: '1024',
record_count: '5',
})
const parsed = await consumer['_parseKafkaBatch'](messages)
const { quotaAllowedMessages, quotaDroppedMessages } = await consumer['filterQuotaLimitedMessages'](parsed)
expect(quotaAllowedMessages).toHaveLength(0)
expect(quotaDroppedMessages).toHaveLength(1)
})
it('should check quota once per unique token', async () => {
jest.mocked(hub.quotaLimiting.isTeamTokenQuotaLimited).mockResolvedValue(false)
const messages = [
...(await createKafkaMessages([createLogMessage()], {
token: team.api_token,
bytes_uncompressed: '100',
record_count: '1',
})),
...(await createKafkaMessages([createLogMessage()], {
token: team.api_token,
bytes_uncompressed: '200',
record_count: '2',
})),
...(await createKafkaMessages([createLogMessage()], {
token: team2.api_token,
bytes_uncompressed: '300',
record_count: '3',
})),
]
const parsed = await consumer['_parseKafkaBatch'](messages)
await consumer['filterQuotaLimitedMessages'](parsed)
// Should only call once per unique token, not once per message
expect(hub.quotaLimiting.isTeamTokenQuotaLimited).toHaveBeenCalledTimes(2)
})
it('should handle mixed quota limited and non-limited teams', async () => {
jest.mocked(hub.quotaLimiting.isTeamTokenQuotaLimited).mockImplementation((token) => {
return Promise.resolve(token === team.api_token) // team1 is limited, team2 is not
})
const messages = [
...(await createKafkaMessages([createLogMessage()], {
token: team.api_token,
bytes_uncompressed: '100',
record_count: '1',
})),
...(await createKafkaMessages([createLogMessage()], {
token: team2.api_token,
bytes_uncompressed: '200',
record_count: '2',
})),
]
const parsed = await consumer['_parseKafkaBatch'](messages)
const { quotaAllowedMessages, quotaDroppedMessages } = await consumer['filterQuotaLimitedMessages'](parsed)
expect(quotaAllowedMessages).toHaveLength(1)
expect(quotaAllowedMessages[0].token).toBe(team2.api_token)
expect(quotaDroppedMessages).toHaveLength(1)
expect(quotaDroppedMessages[0].token).toBe(team.api_token)
})
it('should increment dropped counter when quota limited', async () => {
jest.mocked(hub.quotaLimiting.isTeamTokenQuotaLimited).mockResolvedValue(true)
const messages = await createKafkaMessages([createLogMessage(), createLogMessage()], {
token: team.api_token,
})
const parsed = await consumer['_parseKafkaBatch'](messages)
await consumer['filterQuotaLimitedMessages'](parsed)
expect(logMessageDroppedCounterSpy).toHaveBeenCalledWith(
{ reason: 'quota_limited', team_id: team.id.toString() },
2
)
expect(logMessageDroppedCounterSpy).toHaveBeenCalledTimes(1)
})
})
describe('filterRateLimitedMessages', () => {
it('should return allowed messages', async () => {
const logData = createLogMessage()
const messages = await createKafkaMessages([logData], {
token: team.api_token,
bytes_uncompressed: '1024',
record_count: '5',
})
const parsed = await consumer['_parseKafkaBatch'](messages)
const { rateLimiterAllowedMessages, rateLimiterDroppedMessages } =
await consumer['filterRateLimitedMessages'](parsed)
expect(rateLimiterAllowedMessages).toHaveLength(1)
expect(rateLimiterDroppedMessages).toHaveLength(0)
})
it('should track dropped messages when rate limited', async () => {
hub.LOGS_LIMITER_BUCKET_SIZE_KB = 1 // 1024 bytes - allows first message
hub.LOGS_LIMITER_REFILL_RATE_KB_PER_SECOND = 0.001
hub.LOGS_LIMITER_TTL_SECONDS = 3600
await consumer.stop()
consumer = await createLogsIngestionConsumer(hub)
const messages = [
...(await createKafkaMessages([createLogMessage()], {
token: team.api_token,
bytes_uncompressed: '512', // Fits in bucket
record_count: '1',
})),
...(await createKafkaMessages([createLogMessage()], {
token: team.api_token,
bytes_uncompressed: '2048', // Exceeds remaining bucket
record_count: '5',
})),
]
const parsed = await consumer['_parseKafkaBatch'](messages)
const { rateLimiterAllowedMessages, rateLimiterDroppedMessages } =
await consumer['filterRateLimitedMessages'](parsed)
expect(rateLimiterAllowedMessages).toHaveLength(1)
expect(rateLimiterDroppedMessages).toHaveLength(1)
})
})
describe('quota and rate limiting integration', () => {
it('should apply quota limiting before rate limiting', async () => {
// Team1 is quota limited, team2 is not
jest.mocked(hub.quotaLimiting.isTeamTokenQuotaLimited).mockImplementation((token) => {
return Promise.resolve(token === team.api_token)
})
const messages = [
...(await createKafkaMessages([createLogMessage()], {
token: team.api_token, // quota limited - should be dropped before rate limiter
bytes_uncompressed: '100',
record_count: '1',
})),
...(await createKafkaMessages([createLogMessage()], {
token: team2.api_token, // not quota limited - goes to rate limiter
bytes_uncompressed: '200',
record_count: '2',
})),
]
const result = await consumer.processKafkaBatch(messages)
// Only team2's message should be allowed through
expect(result.messages).toHaveLength(1)
expect(result.messages[0].token).toBe(team2.api_token)
// Verify quota_limited counter was incremented for team1
expect(logMessageDroppedCounterSpy).toHaveBeenCalledWith(
{ reason: 'quota_limited', team_id: team.id.toString() },
1
)
})
it('should drop messages from both quota and rate limiting', async () => {
// Set up rate limit that allows first message but not second
hub.LOGS_LIMITER_BUCKET_SIZE_KB = 1 // 1024 bytes
hub.LOGS_LIMITER_REFILL_RATE_KB_PER_SECOND = 0.001
hub.LOGS_LIMITER_TTL_SECONDS = 3600
await consumer.stop()
consumer = await createLogsIngestionConsumer(hub)
// Team1 is quota limited
jest.mocked(hub.quotaLimiting.isTeamTokenQuotaLimited).mockImplementation((token) => {
return Promise.resolve(token === team.api_token)
})
const messages = [
...(await createKafkaMessages([createLogMessage()], {
token: team.api_token, // quota limited
bytes_uncompressed: '100',
record_count: '1',
})),
...(await createKafkaMessages([createLogMessage()], {
token: team2.api_token, // passes quota, fits in rate limit bucket
bytes_uncompressed: '500',
record_count: '1',
})),
...(await createKafkaMessages([createLogMessage()], {
token: team2.api_token, // passes quota, but exceeds rate limit
bytes_uncompressed: '2000',
record_count: '2',
})),
]
const result = await consumer.processKafkaBatch(messages)
// Only first team2 message should be allowed (quota ok + rate limit ok)
expect(result.messages).toHaveLength(1)
expect(result.messages[0].bytesUncompressed).toBe(500)
// Verify both drop reasons were recorded
expect(logMessageDroppedCounterSpy).toHaveBeenCalledWith(
{ reason: 'quota_limited', team_id: team.id.toString() },
1
)
expect(logMessageDroppedCounterSpy).toHaveBeenCalledWith(
{ reason: 'rate_limited', team_id: team2.id.toString() },
1
)
})
it('should track usage stats correctly for mixed allowed and dropped messages', async () => {
// Team1 is quota limited
jest.mocked(hub.quotaLimiting.isTeamTokenQuotaLimited).mockImplementation((token) => {
return Promise.resolve(token === team.api_token)
})
const messages = [
...(await createKafkaMessages([createLogMessage()], {
token: team.api_token, // quota limited - dropped
bytes_uncompressed: '100',
record_count: '1',
})),
...(await createKafkaMessages([createLogMessage()], {
token: team2.api_token, // allowed
bytes_uncompressed: '200',
record_count: '2',
})),
]
await waitForBackgroundTasks(consumer.processKafkaBatch(messages))
// Check that app_metrics were emitted for both teams
const appMetricsMessages = mockProducerObserver.getProducedKafkaMessagesForTopic(KAFKA_APP_METRICS_2)
// Team1 should have bytes_received and bytes_dropped
const team1Metrics = appMetricsMessages.filter((m) => m.value.team_id === team.id)
const team1BytesReceived = team1Metrics.find((m) => m.value.metric_name === 'bytes_received')
const team1BytesDropped = team1Metrics.find((m) => m.value.metric_name === 'bytes_dropped')
expect(team1BytesReceived?.value.count).toBe(100)
expect(team1BytesDropped?.value.count).toBe(100)
// Team2 should have bytes_received and bytes_ingested
const team2Metrics = appMetricsMessages.filter((m) => m.value.team_id === team2.id)
const team2BytesReceived = team2Metrics.find((m) => m.value.metric_name === 'bytes_received')
const team2BytesIngested = team2Metrics.find((m) => m.value.metric_name === 'bytes_ingested')
expect(team2BytesReceived?.value.count).toBe(200)
expect(team2BytesIngested?.value.count).toBe(200)
})
it('should emit bytes_ingested_records from the bytes_uncompressed_records header', async () => {
const messages = await createKafkaMessages([createLogMessage()], {
token: team.api_token,
bytes_uncompressed: '1024',
bytes_uncompressed_records: '900',
record_count: '5',
})
await waitForBackgroundTasks(consumer.processKafkaBatch(messages))
const appMetricsMessages = mockProducerObserver.getProducedKafkaMessagesForTopic(KAFKA_APP_METRICS_2)
const bytesIngested = appMetricsMessages.find((m) => m.value.metric_name === 'bytes_ingested')
const bytesIngestedRecords = appMetricsMessages.find(
(m) => m.value.metric_name === 'bytes_ingested_records'
)
expect(bytesIngested?.value.count).toBe(1024)
expect(bytesIngestedRecords?.value.count).toBe(900)
})
it('should flag and still emit when bytes_uncompressed_records exceeds bytes_uncompressed', async () => {
const exceedCounterSpy = jest.spyOn(logsRecordsBytesExceedPayloadCounter, 'inc')
const messages = await createKafkaMessages([createLogMessage()], {
token: team.api_token,
bytes_uncompressed: '1024',
bytes_uncompressed_records: '1500',
record_count: '5',
})
await waitForBackgroundTasks(consumer.processKafkaBatch(messages))
expect(exceedCounterSpy).toHaveBeenCalledWith({ team_id: team.id.toString() })
// The comparison metric is emitted as-is, even above the payload size —
// it exists precisely to surface this case.
const appMetricsMessages = mockProducerObserver.getProducedKafkaMessagesForTopic(KAFKA_APP_METRICS_2)
const bytesIngested = appMetricsMessages.find((m) => m.value.metric_name === 'bytes_ingested')
const bytesIngestedRecords = appMetricsMessages.find(
(m) => m.value.metric_name === 'bytes_ingested_records'
)
expect(bytesIngested?.value.count).toBe(1024)
expect(bytesIngestedRecords?.value.count).toBe(1500)
})
})
describe('trackOutgoingTrafficAndBuildUsageStats', () => {
it('should return usageStats with correct structure for allowed messages', async () => {
const logData = createLogMessage()
const messages = await createKafkaMessages([logData], {
token: team.api_token,
bytes_uncompressed: '1024',
record_count: '5',
})
const parsed = await consumer['_parseKafkaBatch'](messages)
const usageStats = consumer['trackOutgoingTrafficAndBuildUsageStats'](parsed, [])
const stats = usageStats.get(team.id)
expect(stats).toBeDefined()
expect(stats!.bytesReceived).toBe(1024)
expect(stats!.recordsReceived).toBe(5)
expect(stats!.bytesAllowed).toBe(1024)
expect(stats!.recordsAllowed).toBe(5)
expect(stats!.bytesDropped).toBe(0)
expect(stats!.recordsDropped).toBe(0)
expect(stats!.piiReplacements).toBe(0)
})
it('should sum bytesAllowedRecords and treat a missing header as zero', async () => {
const messages = [
...(await createKafkaMessages([createLogMessage()], {
token: team.api_token,
bytes_uncompressed: '100',
bytes_uncompressed_records: '80',
record_count: '1',
})),
...(await createKafkaMessages([createLogMessage()], {
token: team.api_token,
bytes_uncompressed: '200',
record_count: '2',
})),
]
const parsed = await consumer['_parseKafkaBatch'](messages)
const usageStats = consumer['trackOutgoingTrafficAndBuildUsageStats'](parsed, [])
const stats = usageStats.get(team.id)
expect(stats!.bytesAllowed).toBe(300)
expect(stats!.bytesAllowedRecords).toBe(80)
})
it('should aggregate stats for multiple messages from same team', async () => {
const messages = [
...(await createKafkaMessages([createLogMessage()], {
token: team.api_token,
bytes_uncompressed: '100',
record_count: '1',
})),
...(await createKafkaMessages([createLogMessage()], {
token: team.api_token,
bytes_uncompressed: '200',
record_count: '2',
})),
]
const parsed = await consumer['_parseKafkaBatch'](messages)
const usageStats = consumer['trackOutgoingTrafficAndBuildUsageStats'](parsed, [])
const stats = usageStats.get(team.id)
expect(stats!.bytesReceived).toBe(300)
expect(stats!.recordsReceived).toBe(3)
expect(stats!.bytesAllowed).toBe(300)
expect(stats!.recordsAllowed).toBe(3)
})
it('should track separate stats for different teams', async () => {
const messages = [
...(await createKafkaMessages([createLogMessage()], {
token: team.api_token,
bytes_uncompressed: '100',
record_count: '1',
})),
...(await createKafkaMessages([createLogMessage()], {
token: team2.api_token,
bytes_uncompressed: '200',
record_count: '2',
})),
]
const parsed = await consumer['_parseKafkaBatch'](messages)
const usageStats = consumer['trackOutgoingTrafficAndBuildUsageStats'](parsed, [])
expect(usageStats.size).toBe(2)
expect(usageStats.get(team.id)!.bytesAllowed).toBe(100)
expect(usageStats.get(team2.id)!.bytesAllowed).toBe(200)
})
it('should track dropped stats correctly', async () => {
// Set up rate limit that allows first message but not second
hub.LOGS_LIMITER_BUCKET_SIZE_KB = 1 // 1024 bytes
hub.LOGS_LIMITER_REFILL_RATE_KB_PER_SECOND = 0.001
hub.LOGS_LIMITER_TTL_SECONDS = 3600
await consumer.stop()
consumer = await createLogsIngestionConsumer(hub)
const messages = [
...(await createKafkaMessages([createLogMessage()], {
token: team.api_token,
bytes_uncompressed: '512', // Fits in bucket
record_count: '1',
})),
...(await createKafkaMessages([createLogMessage()], {
token: team.api_token,
bytes_uncompressed: '2048', // Exceeds remaining bucket
record_count: '5',
})),
]
const parsed = await consumer['_parseKafkaBatch'](messages)
const { rateLimiterAllowedMessages, rateLimiterDroppedMessages } =
await consumer['filterRateLimitedMessages'](parsed)
const usageStats = consumer['trackOutgoingTrafficAndBuildUsageStats'](
rateLimiterAllowedMessages,
rateLimiterDroppedMessages
)
expect(usageStats.get(team.id)!.bytesAllowed).toBe(512)
expect(usageStats.get(team.id)!.bytesDropped).toBe(2048)
})
})
describe('queueUsageMetric', () => {
const parseMetricValue = (value: any): any => {
if (Buffer.isBuffer(value)) {
return parseJSON(value.toString())
}
if (typeof value === 'string') {
return parseJSON(value)
}
return value
}
it('should queue + flush metric with correct structure', async () => {
consumer['queueUsageMetric'](123, 'test_metric', 500)
await consumer['appMetricsAggregator'].flush()
const messages = getProducedKafkaMessages().filter((m) => m.topic === KAFKA_APP_METRICS_2)
expect(messages).toHaveLength(1)
const value = parseMetricValue(messages[0].value)
expect(value?.team_id).toBe(123)
expect(value?.app_source).toBe('logs')
expect(value?.app_source_id).toBe('')
expect(value?.instance_id).toBe('')
expect(value?.metric_kind).toBe('usage')
expect(value?.metric_name).toBe('test_metric')
expect(value?.count).toBe(500)
// timestamp is now set at flush time, not by the caller
expect(typeof value?.timestamp).toBe('string')
})
it('should not queue metric when count is zero', async () => {
consumer['queueUsageMetric'](123, 'test_metric', 0)
await consumer['appMetricsAggregator'].flush()
const messages = getProducedKafkaMessages().filter((m) => m.topic === KAFKA_APP_METRICS_2)
expect(messages).toHaveLength(0)
})
})
describe('emitUsageMetrics', () => {
const parseMetricValue = (value: any): any => {
if (Buffer.isBuffer(value)) {
return parseJSON(value.toString())
}
if (typeof value === 'string') {
return parseJSON(value)
}
return value
}
it('should emit all metric types for each team', async () => {
const usageStats = new Map([
[
team.id,
{
bytesReceived: 1000,
recordsReceived: 10,
bytesAllowed: 800,
recordsAllowed: 8,
bytesAllowedRecords: 700,
bytesDropped: 200,
recordsDropped: 2,
piiReplacements: 0,
retentionDays: 30,
},
],
])
await consumer['emitUsageMetrics'](usageStats)
const messages = getProducedKafkaMessages().filter((m) => m.topic === KAFKA_APP_METRICS_2)
// 7 base metrics + per-tier retention + retention_byte_days.
expect(messages).toHaveLength(9)
const metricNames = messages.map((m) => parseMetricValue(m.value)?.metric_name)
expect(metricNames).toContain('bytes_received')
expect(metricNames).toContain('records_received')
expect(metricNames).toContain('bytes_ingested')
expect(metricNames).toContain('bytes_ingested_records')
expect(metricNames).toContain('records_ingested')
expect(metricNames).toContain('bytes_dropped')
expect(metricNames).toContain('records_dropped')
expect(metricNames).toContain('bytes_ingested_retention_30d')
expect(metricNames).toContain('retention_byte_days')
const retentionByteDays = messages
.map((m) => parseMetricValue(m.value))
.find((m) => m?.metric_name === 'retention_byte_days')
expect(retentionByteDays?.count).toBe(800 * 30)
})
it.each([
{ retentionDays: 14, tierMetric: 'bytes_ingested_retention_14d', byteDayCounts: [] },
{ retentionDays: 30, tierMetric: 'bytes_ingested_retention_30d', byteDayCounts: [500 * 30] },
{
retentionDays: 90,
tierMetric: 'bytes_ingested_retention_90d',
byteDayCounts: [500 * 30, 500 * 30, 500 * 30],
},
{ retentionDays: 45, tierMetric: null, byteDayCounts: [500 * 30, 500 * 15] },
])(
'should emit one retention_byte_days row per month for $retentionDays days alongside the per-tier metric',
async ({ retentionDays, tierMetric, byteDayCounts }) => {
const usageStats = new Map([
[
team.id,
{
bytesReceived: 500,
recordsReceived: 5,
bytesAllowed: 500,
recordsAllowed: 5,
bytesAllowedRecords: 0,
bytesDropped: 0,
recordsDropped: 0,
piiReplacements: 0,
retentionDays,
},
],
])
await consumer['emitUsageMetrics'](usageStats)
const messages = getProducedKafkaMessages().filter((m) => m.topic === KAFKA_APP_METRICS_2)
const parsed = messages.map((m) => parseMetricValue(m.value))
const byteDayRows = parsed
.filter((m) => m?.metric_name === 'retention_byte_days')
.sort((a, b) => a.timestamp.localeCompare(b.timestamp))
expect(byteDayRows.map((m) => m.count)).toEqual(byteDayCounts)
const tierMetrics = parsed.filter((m) => m?.metric_name?.startsWith('bytes_ingested_retention_'))
if (tierMetric) {
expect(tierMetrics).toHaveLength(1)
expect(tierMetrics[0]?.metric_name).toBe(tierMetric)
expect(tierMetrics[0]?.count).toBe(500)
} else {
expect(tierMetrics).toHaveLength(0)
}
}
)
it('should date each retention_byte_days row at the start of its month', async () => {
const usageStats = new Map([
[
team.id,
{
bytesReceived: 500,
recordsReceived: 5,
bytesAllowed: 500,
recordsAllowed: 5,
bytesAllowedRecords: 0,
bytesDropped: 0,
recordsDropped: 0,
piiReplacements: 0,
retentionDays: 90,
},
],
])
// Mid-hour, so the rows show the truncation to the hour.
jest.spyOn(Date, 'now').mockReturnValue(DateTime.fromISO('2025-01-01T05:37:12Z').toMillis())
await consumer['emitUsageMetrics'](usageStats)
const timestamps = getProducedKafkaMessages()
.filter((m) => m.topic === KAFKA_APP_METRICS_2)
.map((m) => parseMetricValue(m.value))
.filter((m) => m?.metric_name === 'retention_byte_days')
.map((m) => m.timestamp)
.sort()
expect(timestamps).toEqual([
'2025-01-01 05:00:00.000',
'2025-01-31 05:00:00.000',
'2025-03-02 05:00:00.000',
])
})
it('should keep the longest retention within ClickHouse partitions-per-insert limit', async () => {
// app_metrics2 is partitioned by month, and ClickHouse fails an insert block that touches
// more than 100 partitions by default. One flush at the maximum retention must stay well
// under that, with room for the current month and a block that crosses a month boundary.
const usageStats = new Map([
[
team.id,
{
bytesReceived: 500,
recordsReceived: 5,
bytesAllowed: 500,
recordsAllowed: 5,
bytesAllowedRecords: 0,
bytesDropped: 0,
recordsDropped: 0,
piiReplacements: 0,
retentionDays: MAX_RETENTION_DAYS,
},
],
])
await consumer['emitUsageMetrics'](usageStats)
const months = new Set(
getProducedKafkaMessages()
.filter((m) => m.topic === KAFKA_APP_METRICS_2)
.map((m) => parseMetricValue(m.value)?.timestamp.slice(0, 7))
)
expect(months.size).toBeLessThanOrEqual(95)
})
it('should skip zero-count metrics', async () => {
const usageStats = new Map([
[
team.id,
{
bytesReceived: 1000,
recordsReceived: 10,
bytesAllowed: 1000,
recordsAllowed: 10,
bytesAllowedRecords: 0,
bytesDropped: 0,
recordsDropped: 0,
piiReplacements: 0,
retentionDays: 14,
},
],
])
await consumer['emitUsageMetrics'](usageStats)
const messages = getProducedKafkaMessages().filter((m) => m.topic === KAFKA_APP_METRICS_2)
// 4 non-zero base metrics (no dropped) + per-tier retention; no retention_byte_days for the default tier.
expect(messages).toHaveLength(5)
const metricNames = messages.map((m) => parseMetricValue(m.value)?.metric_name)
expect(metricNames).not.toContain('bytes_dropped')
expect(metricNames).not.toContain('records_dropped')
expect(metricNames).toContain('bytes_ingested_retention_14d')
expect(metricNames).not.toContain('retention_byte_days')
})
it('should handle empty usageStats', async () => {
const usageStats = new Map()
await consumer['emitUsageMetrics'](usageStats)
const messages = getProducedKafkaMessages().filter((m) => m.topic === KAFKA_APP_METRICS_2)
expect(messages).toHaveLength(0)
})
it('should emit metrics for multiple teams', async () => {
const usageStats = new Map([
[
team.id,
{
bytesReceived: 100,
recordsReceived: 1,
bytesAllowed: 100,
recordsAllowed: 1,
bytesAllowedRecords: 0,
bytesDropped: 0,
recordsDropped: 0,
piiReplacements: 0,
retentionDays: 30,
},
],
[
team2.id,
{
bytesReceived: 200,
recordsReceived: 2,
bytesAllowed: 200,
recordsAllowed: 2,
bytesAllowedRecords: 0,
bytesDropped: 0,
recordsDropped: 0,
piiReplacements: 0,
retentionDays: 90,
},
],
])
await consumer['emitUsageMetrics'](usageStats)
const messages = getProducedKafkaMessages().filter((m) => m.topic === KAFKA_APP_METRICS_2)
// 4 base metrics + per-tier retention + one retention_byte_days row per 30-day month
// (no dropped): 6 for the 30-day team and 8 for the 90-day team.
expect(messages).toHaveLength(14)
const team1Messages = messages.filter((m) => parseMetricValue(m.value)?.team_id === team.id)
const team2Messages = messages.filter((m) => parseMetricValue(m.value)?.team_id === team2.id)
expect(team1Messages).toHaveLength(6)
expect(team2Messages).toHaveLength(8)
})
})
describe('app metrics emission (integration)', () => {
const parseMetricValue = (value: any): any => {
if (Buffer.isBuffer(value)) {
return parseJSON(value.toString())
}
if (typeof value === 'string') {
return parseJSON(value)
}
return value
}
it('should emit usage metrics to app_metrics2 topic', async () => {
const logData = createLogMessage()
const messages = await createKafkaMessages([logData], {
token: team.api_token,
bytes_uncompressed: '1024',
record_count: '5',
})
await waitForBackgroundTasks(consumer.processKafkaBatch(messages))
const appMetricsMessages = getProducedKafkaMessages().filter((m) => m.topic === KAFKA_APP_METRICS_2)
expect(appMetricsMessages.length).toBeGreaterThan(0)
const metricNames = appMetricsMessages.map((m) => {
const value = parseMetricValue(m.value)
return value.metric_name
})
expect(metricNames).toContain('bytes_received')
expect(metricNames).toContain('records_received')
expect(metricNames).toContain('bytes_ingested')
expect(metricNames).toContain('records_ingested')
})
it('should emit pii_replacements when pii scrub is on and the body has pattern matches', async () => {
await hub.postgres.query(
PostgresUse.COMMON_WRITE,
`UPDATE posthog_team
SET logs_settings = $1
WHERE id = $2`,
[JSON.stringify({ pii_scrub_logs: true, json_parse_logs: false }), team.id],
'updateTeamLogsPiiScrub'
)
const logData = createLogMessage({ message: 'email me at [email protected]' })
const messages = await createKafkaMessages([logData], {
token: team.api_token,
bytes_uncompressed: '200',
record_count: '1',
})
await waitForBackgroundTasks(consumer.processKafkaBatch(messages))
const appMetricsMessages = getProducedKafkaMessages().filter((m) => m.topic === KAFKA_APP_METRICS_2)
const pii = appMetricsMessages.find((m) => {
const value = parseMetricValue(m.value)
return value.metric_name === 'pii_replacements' && value.team_id === team.id
})
expect(pii).toBeDefined()
expect(parseMetricValue(pii!.value).count).toBe(1)
})
it('should emit correct metric values per team', async () => {
const logData = createLogMessage()
const messages = await createKafkaMessages([logData], {
token: team.api_token,
bytes_uncompressed: '2048',
record_count: '10',
})
await waitForBackgroundTasks(consumer.processKafkaBatch(messages))
const appMetricsMessages = getProducedKafkaMessages().filter((m) => m.topic === KAFKA_APP_METRICS_2)
const bytesIngestedMetric = appMetricsMessages.find((m) => {
const value = parseMetricValue(m.value)
return value.metric_name === 'bytes_ingested'
})
expect(bytesIngestedMetric).toBeDefined()
const value = parseMetricValue(bytesIngestedMetric!.value)
expect(value.team_id).toBe(team.id)
expect(value.app_source).toBe('logs')
expect(value.metric_kind).toBe('usage')
expect(value.count).toBe(2048)
})
it('should emit dropped metrics when rate limited', async () => {
hub.LOGS_LIMITER_BUCKET_SIZE_KB = 1
hub.LOGS_LIMITER_REFILL_RATE_KB_PER_SECOND = 0.001
hub.LOGS_LIMITER_TTL_SECONDS = 3600
await consumer.stop()
consumer = await createLogsIngestionConsumer(hub)
const logData1 = createLogMessage({ message: 'First' })
const logData2 = createLogMessage({ message: 'Second - will be dropped' })
const messages = [
...(await createKafkaMessages([logData1], {
token: team.api_token,
bytes_uncompressed: '512',
record_count: '2',
})),
...(await createKafkaMessages([logData2], {
token: team.api_token,
bytes_uncompressed: '2048',
record_count: '8',
})),
]
await waitForBackgroundTasks(consumer.processKafkaBatch(messages))
const appMetricsMessages = getProducedKafkaMessages().filter((m) => m.topic === KAFKA_APP_METRICS_2)
const bytesDroppedMetric = appMetricsMessages.find((m) => {
const value = parseMetricValue(m.value)
return value.metric_name === 'bytes_dropped'
})
expect(bytesDroppedMetric).toBeDefined()
const value = parseMetricValue(bytesDroppedMetric!.value)
expect(value.count).toBe(2048)
})
it('should aggregate metrics across multiple messages from same team', async () => {
const logData1 = createLogMessage({ message: 'First' })
const logData2 = createLogMessage({ message: 'Second' })
const messages = [
...(await createKafkaMessages([logData1], {
token: team.api_token,
bytes_uncompressed: '100',
record_count: '1',
})),
...(await createKafkaMessages([logData2], {
token: team.api_token,
bytes_uncompressed: '200',
record_count: '2',
})),
]
await waitForBackgroundTasks(consumer.processKafkaBatch(messages))
const appMetricsMessages = getProducedKafkaMessages().filter((m) => m.topic === KAFKA_APP_METRICS_2)
const bytesIngestedMetric = appMetricsMessages.find((m) => {
const value = parseMetricValue(m.value)
return value.metric_name === 'bytes_ingested' && value.team_id === team.id
})
expect(bytesIngestedMetric).toBeDefined()
const value = parseMetricValue(bytesIngestedMetric!.value)
expect(value.count).toBe(300) // 100 + 200
})
it('should emit separate metrics for different teams', async () => {
const logData1 = createLogMessage({ message: 'Team 1' })
const logData2 = createLogMessage({ message: 'Team 2' })
const messages = [
...(await createKafkaMessages([logData1], {
token: team.api_token,
bytes_uncompressed: '100',
record_count: '1',
})),
...(await createKafkaMessages([logData2], {
token: team2.api_token,
bytes_uncompressed: '200',
record_count: '2',
})),
]
await waitForBackgroundTasks(consumer.processKafkaBatch(messages))
const appMetricsMessages = getProducedKafkaMessages().filter((m) => m.topic === KAFKA_APP_METRICS_2)
const team1Metrics = appMetricsMessages.filter((m) => {
const value = parseMetricValue(m.value)
return value.team_id === team.id && value.metric_name === 'bytes_ingested'
})
const team2Metrics = appMetricsMessages.filter((m) => {
const value = parseMetricValue(m.value)
return value.team_id === team2.id && value.metric_name === 'bytes_ingested'
})
expect(team1Metrics).toHaveLength(1)
expect(team2Metrics).toHaveLength(1)
expect(parseMetricValue(team1Metrics[0].value).count).toBe(100)
expect(parseMetricValue(team2Metrics[0].value).count).toBe(200)
})
it('should not emit metrics with zero count', async () => {
const logData = createLogMessage()
const messages = await createKafkaMessages([logData], {
token: team.api_token,
bytes_uncompressed: '100',
record_count: '1',
})
await waitForBackgroundTasks(consumer.processKafkaBatch(messages))
const appMetricsMessages = getProducedKafkaMessages().filter((m) => m.topic === KAFKA_APP_METRICS_2)
// Should not have drop metrics since nothing was dropped or sampled out
const droppedMetrics = appMetricsMessages.filter((m) => {
const value = parseMetricValue(m.value)
return (
value.metric_name === 'bytes_dropped' ||
value.metric_name === 'records_dropped' ||
value.metric_name === 'sampling_records_dropped_by_rule'
)
})
expect(droppedMetrics).toHaveLength(0)
})
it('drops a row already past retention and reports it, when the header says the message can hold one', async () => {
const fourHundredDaysAgoMicros = DateTime.now().minus({ days: 400 }).toMillis() * 1000
const message = await createMultiRecordKafkaMessage(
[{ level: 'info', timestampMicros: fourHundredDaysAgoMicros }],
{
token: team.api_token,
bytes_uncompressed: '400',
record_count: '1',
min_timestamp: fourHundredDaysAgoMicros.toString(),
}
)
await waitForBackgroundTasks(consumer.processKafkaBatch([message]))
const produced = getProducedKafkaMessages()
expect(produced.some((m) => m.topic === KAFKA_LOGS_CLICKHOUSE)).toBe(false)
const expiredMetric = produced
.filter((m) => m.topic === KAFKA_APP_METRICS_2)
.map((m) => parseMetricValue(m.value))
.find((v) => v.metric_name === 'records_dropped_retention_expired' && v.team_id === team.id)
expect(expiredMetric?.count).toBe(1)
})
describe('sampling usage to app_metrics2', () => {
let mockSamplingCache: Pick<SamplingRulesCache, 'getCompiledRuleSet'>
beforeEach(async () => {
await consumer.stop()
mockSamplingCache = {
getCompiledRuleSet: (teamId: number) =>
Promise.resolve(
teamId === team.id
? compileRuleSet([
{
id: '00000000-0000-0000-0000-0000000000aa',
rule_type: 'severity_sampling',
scope_service: 'test-service',
scope_path_pattern: null,
scope_attribute_filters: [],
config: { actions: { INFO: { type: 'drop' } } },
},
])
: compileRuleSet([])
),
}
consumer = await createLogsIngestionConsumer(
hub,
{},
{ samplingRulesCache: mockSamplingCache as SamplingRulesCache }
)
})
it('should emit sampling_records_dropped_by_rule when head sampling drops log lines', async () => {
const logData = createLogMessage({ level: 'info' })
const messages = await createKafkaMessages([logData], {
token: team.api_token,
bytes_uncompressed: '400',
record_count: '1',
})
await waitForBackgroundTasks(consumer.processKafkaBatch(messages))
const appMetricsMessages = getProducedKafkaMessages().filter((m) => m.topic === KAFKA_APP_METRICS_2)
expect(
appMetricsMessages.some((m) => parseMetricValue(m.value).metric_name === 'sampling_records_dropped')
).toBe(false)
const byRule = appMetricsMessages.find((m) => {
const value = parseMetricValue(m.value)
return (
value.metric_name === 'sampling_records_dropped_by_rule' &&
value.team_id === team.id &&
value.instance_id === '00000000-0000-0000-0000-0000000000aa'
)
})
expect(byRule).toBeDefined()
expect(parseMetricValue(byRule!.value).count).toBe(1)
})
const sendDroppedAndKeptMessages = async () => {
// Two single-record messages: the info one is fully dropped by the rule
// (credit = its whole 400-byte header), the error one is kept (600 bytes).
const messages = [
...(await createKafkaMessages([createLogMessage({ level: 'info' })], {
token: team.api_token,
bytes_uncompressed: '400',
record_count: '1',
})),
...(await createKafkaMessages([createLogMessage({ level: 'error' })], {
token: team.api_token,
bytes_uncompressed: '600',
record_count: '1',
})),
]
await waitForBackgroundTasks(consumer.processKafkaBatch(messages))
}
const teamBytesIngested = (): number | undefined => {
const m = getProducedKafkaMessages()
.filter((m) => m.topic === KAFKA_APP_METRICS_2)
.find((m) => {
const value = parseMetricValue(m.value)
return value.metric_name === 'bytes_ingested' && value.team_id === team.id
})
return m ? parseMetricValue(m.value).count : undefined
}
it('shadow mode (default): computes the drop credit but bills the full gross bytes', async () => {
const creditSpy = jest.spyOn(logsBillingBytesCreditedCounter, 'inc')
await sendDroppedAndKeptMessages()
// The would-be credit is computed and observable...
expect(creditSpy).toHaveBeenCalledWith({ team_id: team.id.toString() }, 400)
// ...but billed usage is untouched: full gross of both messages.
expect(teamBytesIngested()).toBe(1000)
})
it('credits dropped rows off bytes_ingested when LOGS_BILLING_PRORATE_ENABLED', async () => {
await consumer.stop()
consumer = await createLogsIngestionConsumer(
hub,
{ LOGS_BILLING_PRORATE_ENABLED: true },
{ samplingRulesCache: mockSamplingCache as SamplingRulesCache }
)
await sendDroppedAndKeptMessages()
// The fully-dropped message's 400-byte header is credited; only the kept one bills.
expect(teamBytesIngested()).toBe(600)
})
const outputHeaders = (): Record<string, string> | undefined =>
getProducedKafkaMessages().find((m) => m.topic === KAFKA_LOGS_CLICKHOUSE)?.headers as
| Record<string, string>
| undefined
// One message, two equal-content records: the info row is dropped, the error row survives.
// Shadow mode forwards the original gross headers; enabled mode scales both by the same
// dropped content fraction (~0.5), so the 500/1000 ratio is preserved.
it.each([
{ label: 'shadow mode (default) forwards the original gross size headers', enabled: false },
{ label: 'enabled mode pro-rates the forwarded size headers by the dropped fraction', enabled: true },
])('$label', async ({ enabled }) => {
if (enabled) {
await consumer.stop()
consumer = await createLogsIngestionConsumer(
hub,
{ LOGS_BILLING_PRORATE_ENABLED: true },
{ samplingRulesCache: mockSamplingCache as SamplingRulesCache }
)
}
const message = await createMultiRecordKafkaMessage(
[createLogMessage({ level: 'info' }), createLogMessage({ level: 'error' })],
{ token: team.api_token, bytes_uncompressed: '1000', bytes_compressed: '500', record_count: '2' }
)
await waitForBackgroundTasks(consumer.processKafkaBatch([message]))
const uncompressed = parseInt(outputHeaders()!.bytes_uncompressed, 10)
const compressed = parseInt(outputHeaders()!.bytes_compressed, 10)
const recordCount = parseInt(outputHeaders()!.record_count, 10)
if (!enabled) {
expect(uncompressed).toBe(1000)
expect(compressed).toBe(500)
expect(recordCount).toBe(2)
return
}
expect(uncompressed).toBeGreaterThan(0)
expect(uncompressed).toBeLessThan(1000)
expect(compressed).toBeGreaterThan(0)
expect(compressed).toBeLessThan(500)
expect(compressed / uncompressed).toBeCloseTo(0.5, 5)
// The dropped info row is removed from the count exactly; only the error row survives.
expect(recordCount).toBe(1)
})
})
})
describe('JSON attribute sniffing', () => {
it.each([
['disabled', '', 'payload', false],
['different team', 'other_team', 'payload', false],
['allowlisted team', 'this_team', 'payload', true],
['wildcard', '*', 'payload', true],
['missing setting', '*', undefined, false],
['empty setting', '*', '', false],
] as const)('%s preserves output and only counts eligible teams', async (_, allowlist, key, enabled) => {
await consumer.stop()
consumer = await createLogsIngestionConsumer(hub, {
LOGS_JSON_ATTRIBUTE_PARSING_ENABLED_TEAMS:
allowlist === 'this_team'
? ` ${team.id}, ${team2.id} `
: allowlist === 'other_team'
? String(team2.id)
: allowlist,
})
jest.spyOn(hub.teamManager, 'getTeam').mockResolvedValue({
...team,
logs_settings: { json_parse_logs_attribute_key: key },
})
logsJsonAttributeSniffCounter.reset()
const message = await createKafkaMessage(
createLogMessage(),
{ token: team.api_token },
{ payload: JSON.stringify('{"nested":true}') }
)
if (!enabled) {
message.value = Buffer.from('passthrough does not require Avro decoding')
}
await waitForBackgroundTasks(consumer.processKafkaBatch([message]))
const logsOutput = getProducedKafkaMessages().filter((output) => output.topic === KAFKA_LOGS_CLICKHOUSE)
expect(logsOutput).toHaveLength(1)
expect(logsOutput[0].value).toEqual(message.value)
expect((await logsJsonAttributeSniffCounter.get()).values).toEqual(
enabled
? [
expect.objectContaining({
labels: { team_id: String(team.id), outcome: 'looks_like_json' },
value: 1,
}),
]
: []
)
})
})
describe('JSON attribute extraction', () => {
it.each([
['disabled', '', 'payload', false],
['different team', 'other_team', 'payload', false],
['allowlisted team', 'this_team', 'payload', true],
['wildcard', '*', 'payload', true],
['missing setting', '*', undefined, false],
['empty setting', '*', '', false],
] as const)('%s only re-encodes eligible logs', async (_, allowlist, key, enabled) => {
await consumer.stop()
consumer = await createLogsIngestionConsumer(hub, {
LOGS_JSON_ATTRIBUTE_EXTRACTION_ENABLED_TEAMS:
allowlist === 'this_team'
? ` ${team.id}, ${team2.id} `
: allowlist === 'other_team'
? String(team2.id)
: allowlist,
})
jest.spyOn(hub.teamManager, 'getTeam').mockResolvedValue({
...team,
logs_settings: { json_parse_logs_attribute_key: key },
})
const message = await createKafkaMessage(
createLogMessage(),
{ token: team.api_token },
{ payload: JSON.stringify('{"count":7}') }
)
if (!enabled) {
message.value = Buffer.from('disabled extraction must not decode')
}
await waitForBackgroundTasks(consumer.processKafkaBatch([message]))
const output = getProducedKafkaMessages().filter((entry) => entry.topic === KAFKA_LOGS_CLICKHOUSE)
expect(output).toHaveLength(1)
if (enabled) {
const [, , records] = await decodeLogRecords(output[0].value as Buffer)
expect(records[0].attributes).toMatchObject({
payload: JSON.stringify('{"count":7}'),
'payload.count': '7',
})
} else {
expect(output[0].value).toEqual(message.value)
}
})
})
describe('metric rules (generate metrics from logs)', () => {
let mockEmitter: { emit: jest.Mock }
let mockMetricRulesCache: Pick<MetricRulesCache, 'getCompiledRules'>
const countBySeverityRules = compileMetricRules([
{
id: '00000000-0000-0000-0000-0000000000bb',
metric_name: 'log.records',
filter_group: null,
value_attribute: null,
group_by: ['severity_text'],
},
])
const createConsumerWithMetricRules = async (overrides: any = {}, depsPartial: any = {}) => {
await consumer.stop()
consumer = await createLogsIngestionConsumer(hub, overrides, {
metricRulesCache: mockMetricRulesCache as MetricRulesCache,
metricsEmitter: mockEmitter as unknown as LogsMetricsEmitter,
...depsPartial,
})
}
beforeEach(async () => {
mockEmitter = { emit: jest.fn().mockResolvedValue(true) }
mockMetricRulesCache = {
getCompiledRules: (teamId: number) => Promise.resolve(teamId === team.id ? countBySeverityRules : []),
}
await createConsumerWithMetricRules()
})
const emittedDataPoints = (): any[] => {
const payload = mockEmitter.emit.mock.calls[0][2]
return payload.resourceMetrics[0].scopeMetrics[0].metrics[0].sum.dataPoints
}
it('emits one OTLP payload per team with counts grouped by the rule dimensions', async () => {
await createConsumerWithMetricRules({ LOGS_JSON_ATTRIBUTE_PARSING_ENABLED_TEAMS: '*' })
jest.spyOn(hub.teamManager, 'getTeam').mockResolvedValue({
...team,
logs_settings: { json_parse_logs_attribute_key: 'payload' },
})
logsJsonAttributeSniffCounter.reset()
const messages = await createKafkaMessages(
[
createLogMessage({ level: 'info' }),
createLogMessage({ level: 'info' }),
createLogMessage({ level: 'error' }),
],
{ token: team.api_token, bytes_uncompressed: '400', record_count: '1' }
)
await waitForBackgroundTasks(consumer.processKafkaBatch(messages))
expect(mockEmitter.emit).toHaveBeenCalledTimes(1)
const [token, teamId] = mockEmitter.emit.mock.calls[0]
expect(token).toBe(team.api_token)
expect(teamId).toBe(team.id)
const bySeverity = Object.fromEntries(
emittedDataPoints().map((dp: any) => [dp.attributes[0].value.stringValue, dp.asDouble])
)
expect(bySeverity).toEqual({ info: 2, error: 1 })
expect((await logsJsonAttributeSniffCounter.get()).values).toEqual([
expect.objectContaining({
labels: { team_id: String(team.id), outcome: 'missing_key' },
value: 3,
}),
])
})
it('emits nothing when a later internal message fails the batch', async () => {
const key = 'internal-logs-test-key'
await createConsumerWithMetricRules({
LOGS_INTERNAL_INGESTION_ENABLED_TEAMS: String(team.id),
LOGS_INTERNAL_INGESTION_SIGNING_KEYS: key,
})
const [billed, internal] = await createKafkaMessages([createLogMessage(), createLogMessage()], {
token: team.api_token,
})
internal.headers!.push(
internalLogsHeaders(signInternalLogs(key, 'hog_flow', team.id, new Date(Date.now())))
)
const originalQueueMessages = mockProducer.queueMessages
const passthrough = originalQueueMessages.bind(mockProducer)
mockProducer.queueMessages = jest.fn().mockImplementation((topicMessages) => {
const isInternal = topicMessages.messages?.some(
(m: { headers?: Record<string, string> }) => m.headers?.[INTERNAL_SOURCE_HEADER]
)
if (topicMessages.topic === KAFKA_LOGS_CLICKHOUSE && isInternal) {
throw new DependencyUnavailableError('broker down', 'Kafka', new Error('broker down'))
}
return passthrough(topicMessages)
})
try {
await expect(waitForBackgroundTasks(consumer.processKafkaBatch([billed, internal]))).rejects.toThrow(
'broker down'
)
expect(mockEmitter.emit).not.toHaveBeenCalled()
} finally {
mockProducer.queueMessages = originalQueueMessages
}
})
it('produces logs normally when the rules cache fetch throws (fail-open)', async () => {
// Metric rules are a purely additive side feature: a Postgres blip on the
// rules fetch must not DLQ or block the log records themselves.
await createConsumerWithMetricRules(
{},
{
metricRulesCache: {
getCompiledRules: () => Promise.reject(new Error('pg down')),
} as unknown as MetricRulesCache,
}
)
const messages = await createKafkaMessages([createLogMessage({ level: 'info' })], {
token: team.api_token,
bytes_uncompressed: '400',
record_count: '1',
})
await waitForBackgroundTasks(consumer.processKafkaBatch(messages))
const logsOutput = getProducedKafkaMessages().filter((m) => m.topic === KAFKA_LOGS_CLICKHOUSE)
expect(logsOutput).toHaveLength(1)
const dlqOutput = getProducedKafkaMessages().filter((m) => m.topic === KAFKA_LOGS_INGESTION_DLQ)
expect(dlqOutput).toHaveLength(0)
expect(mockEmitter.emit).not.toHaveBeenCalled()
})
it('still counts records that a drop rule removes (pre-drop semantics)', async () => {
const droppingSamplingCache: Pick<SamplingRulesCache, 'getCompiledRuleSet'> = {
getCompiledRuleSet: () =>
Promise.resolve(
compileRuleSet([
{
id: '00000000-0000-0000-0000-0000000000aa',
rule_type: 'severity_sampling',
scope_service: 'test-service',
scope_path_pattern: null,
scope_attribute_filters: [],
config: { actions: { INFO: { type: 'drop' } } },
},
])
),
}
await createConsumerWithMetricRules({}, { samplingRulesCache: droppingSamplingCache })
const message = await createMultiRecordKafkaMessage(
[createLogMessage({ level: 'info' }), createLogMessage({ level: 'error' })],
{ token: team.api_token, bytes_uncompressed: '800', record_count: '2' }
)
await waitForBackgroundTasks(consumer.processKafkaBatch([message]))
// The info row is gone from the ClickHouse-bound output...
const logsOutput = getProducedKafkaMessages().filter((m) => m.topic === KAFKA_LOGS_CLICKHOUSE)
expect(logsOutput).toHaveLength(1)
// ...but both rows fed the generated metric.
const bySeverity = Object.fromEntries(
emittedDataPoints().map((dp: any) => [dp.attributes[0].value.stringValue, dp.asDouble])
)
expect(bySeverity).toEqual({ info: 1, error: 1 })
})
it('does not emit for a team without metric rules', async () => {
const messages = await createKafkaMessages([createLogMessage()], {
token: team2.api_token,
bytes_uncompressed: '400',
record_count: '1',
})
await waitForBackgroundTasks(consumer.processKafkaBatch(messages))
expect(mockEmitter.emit).not.toHaveBeenCalled()
})
it.each([
['emits', {}, ['posthog.log.records']],
['does not emit under the killswitch', { LOGS_METRICS_RULES_KILLSWITCH: true }, null],
])(
'default rules for an allowlisted team without rules of its own: %s',
async (_label, overrides, expectedMetricNames) => {
// Team rules are off, so only the default-rules allowlist can turn emission on.
await createConsumerWithMetricRules({
LOGS_METRICS_RULES_ENABLED_TEAMS: '',
LOGS_DEFAULT_METRIC_RULES_ENABLED_TEAMS: String(team2.id),
...overrides,
})
const messages = await createKafkaMessages([createLogMessage()], {
token: team2.api_token,
bytes_uncompressed: '400',
record_count: '1',
})
await waitForBackgroundTasks(consumer.processKafkaBatch(messages))
if (expectedMetricNames === null) {
expect(mockEmitter.emit).not.toHaveBeenCalled()
return
}
expect(mockEmitter.emit).toHaveBeenCalledTimes(1)
const [, teamId, payload] = mockEmitter.emit.mock.calls[0]
expect(teamId).toBe(team2.id)
const names = payload.resourceMetrics.flatMap((rm: any) =>
rm.scopeMetrics[0].metrics.map((m: any) => m.name)
)
expect(names).toEqual(expectedMetricNames)
}
)
it.each([
['killswitch', { LOGS_METRICS_RULES_KILLSWITCH: true }],
['team gating', { LOGS_METRICS_RULES_ENABLED_TEAMS: '' }],
])('does not evaluate rules when disabled via %s', async (_label, overrides) => {
await createConsumerWithMetricRules(overrides)
const messages = await createKafkaMessages([createLogMessage()], {
token: team.api_token,
bytes_uncompressed: '400',
record_count: '1',
})
await waitForBackgroundTasks(consumer.processKafkaBatch(messages))
expect(mockEmitter.emit).not.toHaveBeenCalled()
})
})
describe('TracesIngestionConsumer billing identity', () => {
const createTracesIngestionConsumer = (configOverrides: Record<string, any> = {}): TracesIngestionConsumer => {
const tracesConsumer = new TracesIngestionConsumer(
// Blank the traces Redis host so it reuses the hub's pool (like the logs-consumer tests),
// avoiding a second eager pool that would keep handles open after the suite.
{ ...hub, ...getDefaultTracesIngestionConsumerConfig(), TRACES_REDIS_HOST: '', ...configOverrides },
{
...hub,
outputs: new IngestionOutputs<AppMetricsOutput | LogsOutput | LogsDlqOutput>({
[APP_METRICS_OUTPUT]: new SingleIngestionOutput(
APP_METRICS_OUTPUT,
KAFKA_APP_METRICS_2,
mockProducer,
'test'
),
[LOGS_OUTPUT]: new SingleIngestionOutput(
LOGS_OUTPUT,
KAFKA_LOGS_CLICKHOUSE,
mockProducer,
'test'
),
[LOGS_DLQ_OUTPUT]: new SingleIngestionOutput(
LOGS_DLQ_OUTPUT,
KAFKA_LOGS_INGESTION_DLQ,
mockProducer,
'test'
),
}),
usageBatch: new UsageRecordBatch(null, { unit: 'bytes', isTeamEnabled: () => false }),
}
)
tracesConsumer['kafkaConsumer'] = {
connect: jest.fn(),
disconnect: jest.fn(),
isHealthy: jest.fn().mockReturnValue({ status: 'healthy' }),
} as any
return tracesConsumer
}
it('bills a signed traces batch, because the internal flag covers logs only', async () => {
const key = 'internal-logs-test-key'
await consumer.stop()
consumer = createTracesIngestionConsumer({
LOGS_INTERNAL_INGESTION_SIGNING_KEYS: key,
LOGS_INTERNAL_INGESTION_ENABLED_TEAMS: String(team.id),
})
await consumer.start()
jest.mocked(hub.quotaLimiting.isTeamTokenQuotaLimited).mockResolvedValue(true)
const message = await createKafkaMessage(createLogMessage(), { token: team.api_token })
message.headers!.push(internalLogsHeaders(signInternalLogs(key, 'hog_flow', team.id, new Date(Date.now()))))
await waitForBackgroundTasks(consumer.processKafkaBatch([message]))
expect(getProducedKafkaMessages().filter((m) => m.topic === KAFKA_LOGS_CLICKHOUSE)).toHaveLength(0)
})
it('leaves pattern masking off even on a wildcard allowlist, because a trace record has no body', () => {
const tracesConsumer = createTracesIngestionConsumer({ LOGS_PATTERN_MASKING_ENABLED_TEAMS: '*' })
expect(tracesConsumer['isPatternMaskingEnabledForTeam'](team.id)).toEqual(false)
})
it('never sniffs or decodes traces for a configured JSON attribute on a wildcard allowlist', async () => {
await consumer.stop()
consumer = createTracesIngestionConsumer({
LOGS_JSON_ATTRIBUTE_PARSING_ENABLED_TEAMS: '*',
LOGS_JSON_ATTRIBUTE_EXTRACTION_ENABLED_TEAMS: '*',
})
await consumer.start()
jest.spyOn(hub.teamManager, 'getTeam').mockResolvedValue({
...team,
logs_settings: { json_parse_logs_attribute_key: 'payload' },
})
logsJsonAttributeSniffCounter.reset()
const message = await createKafkaMessage(createLogMessage(), { token: team.api_token })
message.value = Buffer.from('trace payload must not be decoded as logs')
await waitForBackgroundTasks(consumer.processKafkaBatch([message]))
const tracesOutput = getProducedKafkaMessages().filter((output) => output.topic === KAFKA_LOGS_CLICKHOUSE)
expect(tracesOutput).toHaveLength(1)
expect(tracesOutput[0].value).toEqual(message.value)
expect((await logsJsonAttributeSniffCounter.get()).values).toEqual([])
})
it('meters usage as traces, not logs', async () => {
const tracesConsumer = createTracesIngestionConsumer()
tracesConsumer['queueUsageMetric'](team.id, 'bytes_ingested', 500)
await tracesConsumer['appMetricsAggregator'].flush()
const messages = getProducedKafkaMessages().filter((m) => m.topic === KAFKA_APP_METRICS_2)
expect(messages).toHaveLength(1)
const value = Buffer.isBuffer(messages[0].value)
? parseJSON(messages[0].value.toString())
: parseJSON(messages[0].value as string)
expect(value?.app_source).toBe('traces')
})
it('drops traces when the shared logs and traces bucket is limited', async () => {
// One product, one free tier, one price: a team over the shared limit on logs volume
// alone has to stop ingesting traces too.
jest.mocked(hub.quotaLimiting.isTeamTokenQuotaLimited).mockImplementation((_token, resource) =>
Promise.resolve(resource === 'logs_mb_ingested')
)
const tracesConsumer = createTracesIngestionConsumer()
const messages = await createKafkaMessages([createLogMessage()], {
token: team.api_token,
bytes_uncompressed: '1024',
record_count: '5',
})
const parsed = await tracesConsumer['_parseKafkaBatch'](messages)
const { quotaAllowedMessages, quotaDroppedMessages } =
await tracesConsumer['filterQuotaLimitedMessages'](parsed)
expect(quotaAllowedMessages).toHaveLength(0)
expect(quotaDroppedMessages).toHaveLength(1)
})
it('rate-limits against its own Redis key namespace, not the logs one', () => {
const tracesConsumer = createTracesIngestionConsumer()
const prefix = tracesConsumer['rateLimiter']['rateLimiter'].getKeyPrefix()
expect(prefix).toBe('@posthog-test/traces-rate-limiter/tokens')
expect(prefix).not.toContain('logs-rate-limiter')
})
it('applies TRACES_LIMITER_* tuning rather than the logs limiter config', () => {
const tracesConsumer = createTracesIngestionConsumer({
TRACES_LIMITER_BUCKET_SIZE_KB: 42,
TRACES_LIMITER_REFILL_RATE_KB_PER_SECOND: 7,
})
const limiterConfig = tracesConsumer['rateLimiter']['config']
expect(limiterConfig.LOGS_LIMITER_BUCKET_SIZE_KB).toBe(42)
expect(limiterConfig.LOGS_LIMITER_REFILL_RATE_KB_PER_SECOND).toBe(7)
})
it('evaluates the span retention rules, not the log ones', () => {
const tracesConsumer = createTracesIngestionConsumer()
expect(tracesConsumer['retentionRuleSource']).toBe('spans')
})
it('gates retention on TRACES_RETENTION_* rather than the logs config', () => {
const tracesConsumer = createTracesIngestionConsumer({ TRACES_RETENTION_KILLSWITCH: true })
expect(tracesConsumer['isRetentionEvalEnabledForTeam'](team.id)).toBe(false)
})
it('takes its default retention from the tracing config, not logs_settings', async () => {
const tracingConfigCache = { getRetentionDays: jest.fn().mockResolvedValue(90) }
const tracesConsumer = createTracesIngestionConsumer()
tracesConsumer['tracingConfigCache'] = tracingConfigCache as any
// A different logs period must not reach spans.
const days = await tracesConsumer['defaultRetentionDays'](team.id, { retention_days: 30 })
expect(days).toBe(90)
expect(tracingConfigCache.getRetentionDays).toHaveBeenCalledWith(team.id)
})
it('falls back to the built-in default when no tracing config cache is wired', async () => {
const tracesConsumer = createTracesIngestionConsumer()
expect(await tracesConsumer['defaultRetentionDays'](team.id, { retention_days: 30 })).toBe(
DEFAULT_TRACES_RETENTION_DAYS
)
})
})
describe('hog log transformations', () => {
let transformerConsumer: LogsIngestionConsumer
const buildTransformer = async (
hog: string,
monitoring = { queueAppMetric: jest.fn(), queueLogs: jest.fn(), flush: jest.fn() }
): Promise<LogsTransformerService> => {
const fn = {
id: '00000000-0000-0000-0000-000000000001',
team_id: team.id,
name: 'Test log transformation',
type: 'transformation_log',
enabled: true,
bytecode: await compileHog(hog),
inputs: {},
encrypted_inputs: null,
execution_order: 1,
created_at: new Date().toISOString(),
} as unknown as HogFunctionType
const manager = {
getHogFunctionsForTeams: jest.fn().mockResolvedValue({ [team.id]: [fn] }),
getHogFunctionIdsForTeams: jest.fn().mockResolvedValue({ [team.id]: [fn.id] }),
}
return new LogsTransformerService(manager as any, monitoring as any, {
siteUrl: 'http://localhost:8010',
hogTimeoutMs: 10,
messageBudgetMs: 100,
batchBudgetMs: 2000,
maxErrorLogsPerFunctionPerMessage: 3,
})
}
afterEach(async () => {
await transformerConsumer?.stop()
})
it('applies an enabled transformation to produced records', async () => {
const logsTransformer = await buildTransformer(`
let rec := record
rec.body := replaceAll(rec.body, 'sensitive', '[REDACTED]')
return rec
`)
transformerConsumer = await createLogsIngestionConsumer(
hub,
{ LOGS_TRANSFORMATIONS_ENABLED_TEAMS: '*' },
{ logsTransformer }
)
const messages = await createKafkaMessages([createLogMessage({ message: 'something sensitive here' })], {
token: team.api_token,
})
await waitForBackgroundTasks(transformerConsumer.processKafkaBatch(messages))
const logsMessages = getProducedKafkaMessages().filter((m) => m.topic === 'clickhouse_logs_test')
expect(logsMessages).toHaveLength(1)
const [, , records] = await decodeLogRecords(logsMessages[0].value as Buffer)
expect(records).toHaveLength(1)
expect(records[0].body).toContain('[REDACTED]')
expect(records[0].body).not.toContain('sensitive')
})
it('does not produce a message when transformations drop every record', async () => {
const logsTransformer = await buildTransformer(`return null`)
transformerConsumer = await createLogsIngestionConsumer(
hub,
{ LOGS_TRANSFORMATIONS_ENABLED_TEAMS: '*' },
{ logsTransformer }
)
const messages = await createKafkaMessages([createLogMessage()], { token: team.api_token })
await waitForBackgroundTasks(transformerConsumer.processKafkaBatch(messages))
const logsMessages = getProducedKafkaMessages().filter((m) => m.topic === 'clickhouse_logs_test')
expect(logsMessages).toHaveLength(0)
expect(logMessageDroppedCounterSpy).toHaveBeenCalledWith(
{ reason: 'transformations_all_dropped', team_id: team.id.toString() },
1
)
// The OTel counter has to carry the same reason as the Prometheus one above —
// they were allowed to disagree, so a transformations drop read as sampling.
expect(recordLogMessageDroppedSpy).toHaveBeenCalledWith('transformations_all_dropped', team.id.toString())
})
it('attributes the full-message drop to transformations when sampling kept survivors', async () => {
const logsTransformer = await buildTransformer(`return null`)
// Drop rule that does not match the record, so sampling keeps it and the
// transformation is what removes the last survivor.
const mockSamplingCache: Pick<SamplingRulesCache, 'getCompiledRuleSet'> = {
getCompiledRuleSet: () =>
Promise.resolve(
compileRuleSet([
{
id: '00000000-0000-0000-0000-0000000000ab',
rule_type: 'severity_sampling',
scope_service: 'test-service',
scope_path_pattern: null,
scope_attribute_filters: [],
config: { actions: { INFO: { type: 'drop' } } },
},
])
),
}
transformerConsumer = await createLogsIngestionConsumer(
hub,
{ LOGS_TRANSFORMATIONS_ENABLED_TEAMS: '*' },
{ logsTransformer, samplingRulesCache: mockSamplingCache as SamplingRulesCache }
)
const messages = await createKafkaMessages([createLogMessage({ level: 'error' })], {
token: team.api_token,
})
await waitForBackgroundTasks(transformerConsumer.processKafkaBatch(messages))
const logsMessages = getProducedKafkaMessages().filter((m) => m.topic === 'clickhouse_logs_test')
expect(logsMessages).toHaveLength(0)
expect(logMessageDroppedCounterSpy).toHaveBeenCalledWith(
{ reason: 'transformations_all_dropped', team_id: team.id.toString() },
1
)
})
const internalLogsConfig = {
LOGS_INTERNAL_INGESTION_ENABLED_TEAMS: '*',
LOGS_INTERNAL_INGESTION_SIGNING_KEYS: 'internal-logs-test-key',
}
it('runs transformations on a batch whose internal logs header fails to verify, and keeps its logs out of the producer', async () => {
const monitoring = { queueAppMetric: jest.fn(), queueLogs: jest.fn(), flush: jest.fn() }
const logsTransformer = await buildTransformer(`throw Error('boom')`, monitoring)
transformerConsumer = await createLogsIngestionConsumer(
hub,
{ LOGS_TRANSFORMATIONS_ENABLED_TEAMS: '*', ...internalLogsConfig },
{ logsTransformer }
)
const [message] = await createKafkaMessages([createLogMessage()], { token: team.api_token })
message.headers!.push(
internalLogsHeaders(signInternalLogs('rotated-away-key', 'hog_function', team.id, new Date(Date.now())))
)
await waitForBackgroundTasks(transformerConsumer.processKafkaBatch([message]))
expect(monitoring.queueLogs).toHaveBeenCalledWith(expect.any(Array), 'hog_function', {
aboutInternalLogs: true,
})
})
it.each([
{ name: 'the team is not in the allowlist', config: { LOGS_TRANSFORMATIONS_ENABLED_TEAMS: '' } },
{
name: 'the killswitch is on',
config: { LOGS_TRANSFORMATIONS_ENABLED_TEAMS: '*', LOGS_TRANSFORMATIONS_KILLSWITCH: true },
},
{
name: 'the internal logs header verifies',
config: { LOGS_TRANSFORMATIONS_ENABLED_TEAMS: '*', ...internalLogsConfig },
signingKey: 'internal-logs-test-key',
},
])('does not run transformations when $name', async ({ config, signingKey }) => {
const logsTransformer = await buildTransformer(`return null`)
transformerConsumer = await createLogsIngestionConsumer(hub, config, { logsTransformer })
const [message] = await createKafkaMessages([createLogMessage()], { token: team.api_token })
if (signingKey) {
message.headers!.push(
internalLogsHeaders(signInternalLogs(signingKey, 'hog_function', team.id, new Date(Date.now())))
)
}
await waitForBackgroundTasks(transformerConsumer.processKafkaBatch([message]))
const logsMessages = getProducedKafkaMessages().filter((m) => m.topic === 'clickhouse_logs_test')
expect(logsMessages).toHaveLength(1)
})
})
describe('pattern masking (measure-only)', () => {
let maskingConsumer: LogsIngestionConsumer | undefined
afterEach(async () => {
await maskingConsumer?.stop()
maskingConsumer = undefined
})
it.each([
['a wildcard allowlist runs the masking stage', () => '*', true],
['an empty allowlist does not run the masking stage', () => '', false],
["the team's id in the allowlist runs the masking stage", () => String(team.id), true],
["only another team's id in the allowlist does not run the masking stage", () => String(team2.id), false],
])('%s', async (_name, allowlist, expectMasked) => {
const bodyKindIncSpy = jest.spyOn(logsPatternBodyKindCounter, 'inc')
maskingConsumer = await createLogsIngestionConsumer(hub, {
LOGS_PATTERN_MASKING_ENABLED_TEAMS: allowlist(),
})
const messages = await createKafkaMessages([createLogMessage()], { token: team.api_token })
await waitForBackgroundTasks(maskingConsumer.processKafkaBatch(messages))
const logsMessages = getProducedKafkaMessages().filter((m) => m.topic === 'clickhouse_logs_test')
expect(logsMessages).toHaveLength(1)
if (expectMasked) {
expect(bodyKindIncSpy).toHaveBeenCalled()
} else {
expect(bodyKindIncSpy).not.toHaveBeenCalled()
}
})
it("derives the pattern from the team's message keys, not the defaults", async () => {
const patternMessageKeysCache = {
getMessageKeys: jest.fn().mockResolvedValue(['log']),
} as unknown as PatternMessageKeysCache
maskingConsumer = await createLogsIngestionConsumer(
hub,
{ LOGS_PATTERN_MASKING_ENABLED_TEAMS: '*' },
{ patternMessageKeysCache }
)
const messages = await createKafkaMessages(
[createLogMessage({ message: 'default key wins', log: 'team key wins' })],
{
token: team.api_token,
}
)
await waitForBackgroundTasks(maskingConsumer.processKafkaBatch(messages))
const logsMessages = getProducedKafkaMessages().filter((m) => m.topic === 'clickhouse_logs_test')
const [, , records] = await decodeLogRecords(logsMessages[0].value as Buffer)
expect(records.map((r) => r.pattern)).toEqual(['team key wins'])
})
})
})