Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(logs): replay the batch when the logs topic rejects a produce (#128013)

Frank Hamandec826b84a5d0
diff --git a/nodejs/src/logs/logs-ingestion-consumer.test.ts b/nodejs/src/logs/logs-ingestion-consumer.test.ts
index b3f76d34d4c5..ab611d77da6a 100644
--- a/nodejs/src/logs/logs-ingestion-consumer.test.ts
+++ b/nodejs/src/logs/logs-ingestion-consumer.test.ts
@@ -8,7 +8,7 @@ import {
PeriodicExportingMetricReader,
} from '@opentelemetry/sdk-metrics'
import { DateTime } from 'luxon'
-import { Message } from 'node-rdkafka'
+import { CODES, Message } from 'node-rdkafka'
import { compileHog } from '~/cdp/templates/compiler'
import { HogFunctionType } from '~/cdp/types'
@@ -62,6 +62,9 @@ import type { SamplingRulesCache } from './sampling/sampling-rules-cache'
import { TracesIngestionConsumer } from './traces-ingestion-consumer'
import { LogsTransformerService } from './transformations/logs-transformer.service'
+const LibrdKafkaError = require('node-rdkafka/lib/error')
+const { err2str } = require('node-rdkafka/librdkafka') as { err2str: (code: number) => string }
+
// What capture-logs forwards for a producer that signed just before the request arrived. Both times come
// from `Date.now()`, which these tests pin, because a bare `new Date()` still reads the real clock.
function internalLogsHeaders(signature: string): Record<string, Buffer> {
@@ -603,7 +606,12 @@ describe('LogsIngestionConsumer', () => {
})
describe('error handling', () => {
- it('should send failed messages to DLQ', async () => {
+ const kafkaDeliveryError = (code: number): Error => LibrdKafkaError.create(new Error(err2str(code)))
+
+ it.each([
+ ['a processing error', () => new Error('Producer error')],
+ ['a Kafka error that a replay cannot fix', () => kafkaDeliveryError(CODES.ERRORS.ERR_INVALID_RECORD)],
+ ])('should send failed messages to DLQ on %s', async (_, makeError) => {
const logData = createLogMessage()
const messages = await createKafkaMessages([logData], {
token: team.api_token,
@@ -617,7 +625,7 @@ describe('LogsIngestionConsumer', () => {
const queueSpy = jest.fn().mockImplementation((topicMessages) => {
const t = Array.isArray(topicMessages) ? topicMessages[0]?.topic : topicMessages.topic
if (t === 'clickhouse_logs_test') {
- throw new Error('Producer error')
+ throw makeError()
}
return passthrough(topicMessages)
})
@@ -684,7 +692,15 @@ describe('LogsIngestionConsumer', () => {
}
})
- it('should fail the batch instead of DLQing when a dependency is unavailable', async () => {
+ it.each([
+ [
+ 'a dependency is unavailable',
+ () => new DependencyUnavailableError('broker down', 'Kafka', new Error('broker down')),
+ ],
+ ['a delivery times out', () => kafkaDeliveryError(CODES.ERRORS.ERR__MSG_TIMED_OUT)],
+ ['the producer queue is full', () => LibrdKafkaError.create(CODES.ERRORS.ERR__QUEUE_FULL)],
+ ['the topic rejects authorization', () => kafkaDeliveryError(CODES.ERRORS.ERR_TOPIC_AUTHORIZATION_FAILED)],
+ ])('should fail the batch instead of DLQing when %s', async (_, makeError) => {
const logData = createLogMessage()
const messages = await createKafkaMessages([logData], {
token: team.api_token,
@@ -695,7 +711,7 @@ describe('LogsIngestionConsumer', () => {
mockProducer.queueMessages = jest.fn().mockImplementation((topicMessages) => {
const t = Array.isArray(topicMessages) ? topicMessages[0]?.topic : topicMessages.topic
if (t === KAFKA_LOGS_CLICKHOUSE) {
- throw new DependencyUnavailableError('broker down', 'Kafka', new Error('broker down'))
+ throw makeError()
}
return passthrough(topicMessages)
})
@@ -703,9 +719,7 @@ describe('LogsIngestionConsumer', () => {
try {
// The message is good data that succeeds on redelivery. Quarantining it would
// move it out of the source topic for an outage that was not its fault.
- await expect(waitForBackgroundTasks(consumer.processKafkaBatch(messages))).rejects.toThrow(
- 'broker down'
- )
+ await expect(waitForBackgroundTasks(consumer.processKafkaBatch(messages))).rejects.toThrow()
expect(getProducedKafkaMessages().filter((m) => m.topic === KAFKA_LOGS_INGESTION_DLQ)).toHaveLength(0)
} finally {
mockProducer.queueMessages = originalQueueMessages
diff --git a/nodejs/src/logs/logs-ingestion-consumer.ts b/nodejs/src/logs/logs-ingestion-consumer.ts
index b0152f62c37a..b1717e4504e8 100644
--- a/nodejs/src/logs/logs-ingestion-consumer.ts
+++ b/nodejs/src/logs/logs-ingestion-consumer.ts
@@ -48,6 +48,7 @@ import {
processLogMessageBuffer,
sniffJsonLogAttributes,
} from './log-record-avro'
+import { rethrowKafkaOutageAsDependency } from './logs-produce-errors'
import type { CompiledMetricRule, MetricRuleSource } from './metrics-rules/compile-metric-rules'
import { withDefaultMetricRules } from './metrics-rules/default-metric-rules'
import { MetricRulesCache } from './metrics-rules/metric-rules-cache'
@@ -1249,35 +1250,36 @@ export class LogsIngestionConsumer {
// Await so a rejection here lands in the catch and routes to the DLQ.
await this.retryOnDependencyUnavailable(() =>
- this.deps.outputs.queueMessages(LOGS_OUTPUT, [
- {
- value: processedValue,
- key: null,
- headers: {
- ...withoutInternalLogsHeaders(parseKafkaHeaders(message.message.headers)),
- ...(message.internalLogs !== undefined
- ? { [INTERNAL_SOURCE_HEADER]: message.internalLogs.source }
- : {}),
- token: message.token,
- team_id: message.teamId.toString(),
- 'json-parse': jsonParse.toString(),
- 'retention-days': retentionDays.toString(),
- // The ingestion lag checkpoint needs the source partition, because
- // both topics use random partitioning.
- source_topic: message.message.topic,
- source_partition: message.message.partition.toString(),
- ...(bytesUncompressedHeaderOverride !== undefined
- ? { bytes_uncompressed: bytesUncompressedHeaderOverride.toString() }
- : {}),
- ...(bytesCompressedHeaderOverride !== undefined
- ? { bytes_compressed: bytesCompressedHeaderOverride.toString() }
- : {}),
- ...(recordCountHeaderOverride !== undefined
- ? { record_count: recordCountHeaderOverride.toString() }
- : {}),
+ this.deps.outputs
+ .queueMessages(LOGS_OUTPUT, [
+ {
+ value: processedValue,
+ key: null,
+ headers: {
+ ...withoutInternalLogsHeaders(parseKafkaHeaders(message.message.headers)),
+ ...(message.internalLogs !== undefined
+ ? { [INTERNAL_SOURCE_HEADER]: message.internalLogs.source }
+ : {}),
+ token: message.token,
+ team_id: message.teamId.toString(),
+ 'json-parse': jsonParse.toString(),
+ 'retention-days': retentionDays.toString(),
+ // Both topics use random partitioning, so the ingestion lag checkpoint needs the source partition.
+ source_topic: message.message.topic,
+ source_partition: message.message.partition.toString(),
+ ...(bytesUncompressedHeaderOverride !== undefined
+ ? { bytes_uncompressed: bytesUncompressedHeaderOverride.toString() }
+ : {}),
+ ...(bytesCompressedHeaderOverride !== undefined
+ ? { bytes_compressed: bytesCompressedHeaderOverride.toString() }
+ : {}),
+ ...(recordCountHeaderOverride !== undefined
+ ? { record_count: recordCountHeaderOverride.toString() }
+ : {}),
+ },
},
- },
- ])
+ ])
+ .catch(rethrowKafkaOutageAsDependency)
)
} catch (error) {
// A dependency outage (Postgres, Redis, Kafka) is infrastructure-scoped. The
diff --git a/nodejs/src/logs/logs-produce-errors.ts b/nodejs/src/logs/logs-produce-errors.ts
new file mode 100644
index 000000000000..997c34367d1d
--- /dev/null
+++ b/nodejs/src/logs/logs-produce-errors.ts
@@ -0,0 +1,48 @@
+import { CODES } from 'node-rdkafka'
+
+import { DependencyUnavailableError } from '~/common/utils/db/error'
+
+const { err2str } = require('node-rdkafka/librdkafka') as { err2str: (code: number) => string }
+
+const PER_MESSAGE_KAFKA_ERROR_CODES: readonly number[] = [
+ CODES.ERRORS.ERR_INVALID_MSG,
+ CODES.ERRORS.ERR_INVALID_MSG_SIZE,
+ CODES.ERRORS.ERR_MSG_SIZE_TOO_LARGE,
+ CODES.ERRORS.ERR_RECORD_LIST_TOO_LARGE,
+ CODES.ERRORS.ERR_INVALID_RECORD,
+ CODES.ERRORS.ERR__BAD_MSG,
+ CODES.ERRORS.ERR__INVALID_ARG,
+ CODES.ERRORS.ERR__INVALID_TYPE,
+]
+
+const librdkafkaText = (message: string): string => message.split(': ').pop()!.toLowerCase()
+
+// A delivery report carries only librdkafka's text, which node-rdkafka parses into `code: -1`.
+const PER_MESSAGE_KAFKA_ERROR_TEXTS = new Set(
+ PER_MESSAGE_KAFKA_ERROR_CODES.map((code) => librdkafkaText(err2str(code)))
+)
+
+type LibrdKafkaError = Error & { code: number; origin: string }
+
+function isLibrdKafkaError(error: unknown): error is LibrdKafkaError {
+ if (typeof error !== 'object' || error === null) {
+ return false
+ }
+ const { code, origin } = error as { code?: unknown; origin?: unknown }
+ return typeof code === 'number' && (origin === 'kafka' || origin === 'local' || origin === 'broker')
+}
+
+function isPerMessageKafkaError(error: LibrdKafkaError): boolean {
+ return (
+ PER_MESSAGE_KAFKA_ERROR_CODES.includes(error.code) ||
+ PER_MESSAGE_KAFKA_ERROR_TEXTS.has(librdkafkaText(error.message))
+ )
+}
+
+/** node-rdkafka never marks a delivery error as retriable; raise an outage as one so the batch replays. */
+export function rethrowKafkaOutageAsDependency(error: unknown): never {
+ if (isLibrdKafkaError(error) && !isPerMessageKafkaError(error)) {
+ throw new DependencyUnavailableError(error.message, 'Kafka', error)
+ }
+ throw error
+}