:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(logs): replay the batch when the logs topic rejects a produce (#128013)
Frank Hamandec826b84a5d0
diff --git a/nodejs/src/logs/logs-ingestion-consumer.test.ts b/nodejs/src/logs/logs-ingestion-consumer.test.tsindex b3f76d34d4c5..ab611d77da6a 100644--- a/nodejs/src/logs/logs-ingestion-consumer.test.ts+++ b/nodejs/src/logs/logs-ingestion-consumer.test.ts@@ -8,7 +8,7 @@ import {PeriodicExportingMetricReader,} from '@opentelemetry/sdk-metrics'import { DateTime } from 'luxon'-import { Message } from 'node-rdkafka'+import { CODES, Message } from 'node-rdkafka'import { compileHog } from '~/cdp/templates/compiler'import { HogFunctionType } from '~/cdp/types'@@ -62,6 +62,9 @@ import type { SamplingRulesCache } from './sampling/sampling-rules-cache'import { TracesIngestionConsumer } from './traces-ingestion-consumer'import { LogsTransformerService } from './transformations/logs-transformer.service'+const LibrdKafkaError = require('node-rdkafka/lib/error')+const { err2str } = require('node-rdkafka/librdkafka') as { err2str: (code: number) => string }+// What capture-logs forwards for a producer that signed just before the request arrived. Both times come// from `Date.now()`, which these tests pin, because a bare `new Date()` still reads the real clock.function internalLogsHeaders(signature: string): Record<string, Buffer> {@@ -603,7 +606,12 @@ describe('LogsIngestionConsumer', () => {})describe('error handling', () => {- it('should send failed messages to DLQ', async () => {+ const kafkaDeliveryError = (code: number): Error => LibrdKafkaError.create(new Error(err2str(code)))++ it.each([+ ['a processing error', () => new Error('Producer error')],+ ['a Kafka error that a replay cannot fix', () => kafkaDeliveryError(CODES.ERRORS.ERR_INVALID_RECORD)],+ ])('should send failed messages to DLQ on %s', async (_, makeError) => {const logData = createLogMessage()const messages = await createKafkaMessages([logData], {token: team.api_token,@@ -617,7 +625,7 @@ describe('LogsIngestionConsumer', () => {const queueSpy = jest.fn().mockImplementation((topicMessages) => {const t = Array.isArray(topicMessages) ? topicMessages[0]?.topic : topicMessages.topicif (t === 'clickhouse_logs_test') {- throw new Error('Producer error')+ throw makeError()}return passthrough(topicMessages)})@@ -684,7 +692,15 @@ describe('LogsIngestionConsumer', () => {}})- it('should fail the batch instead of DLQing when a dependency is unavailable', async () => {+ it.each([+ [+ 'a dependency is unavailable',+ () => new DependencyUnavailableError('broker down', 'Kafka', new Error('broker down')),+ ],+ ['a delivery times out', () => kafkaDeliveryError(CODES.ERRORS.ERR__MSG_TIMED_OUT)],+ ['the producer queue is full', () => LibrdKafkaError.create(CODES.ERRORS.ERR__QUEUE_FULL)],+ ['the topic rejects authorization', () => kafkaDeliveryError(CODES.ERRORS.ERR_TOPIC_AUTHORIZATION_FAILED)],+ ])('should fail the batch instead of DLQing when %s', async (_, makeError) => {const logData = createLogMessage()const messages = await createKafkaMessages([logData], {token: team.api_token,@@ -695,7 +711,7 @@ describe('LogsIngestionConsumer', () => {mockProducer.queueMessages = jest.fn().mockImplementation((topicMessages) => {const t = Array.isArray(topicMessages) ? topicMessages[0]?.topic : topicMessages.topicif (t === KAFKA_LOGS_CLICKHOUSE) {- throw new DependencyUnavailableError('broker down', 'Kafka', new Error('broker down'))+ throw makeError()}return passthrough(topicMessages)})@@ -703,9 +719,7 @@ describe('LogsIngestionConsumer', () => {try {// The message is good data that succeeds on redelivery. Quarantining it would// move it out of the source topic for an outage that was not its fault.- await expect(waitForBackgroundTasks(consumer.processKafkaBatch(messages))).rejects.toThrow(- 'broker down'- )+ await expect(waitForBackgroundTasks(consumer.processKafkaBatch(messages))).rejects.toThrow()expect(getProducedKafkaMessages().filter((m) => m.topic === KAFKA_LOGS_INGESTION_DLQ)).toHaveLength(0)} finally {mockProducer.queueMessages = originalQueueMessagesdiff --git a/nodejs/src/logs/logs-ingestion-consumer.ts b/nodejs/src/logs/logs-ingestion-consumer.tsindex b0152f62c37a..b1717e4504e8 100644--- a/nodejs/src/logs/logs-ingestion-consumer.ts+++ b/nodejs/src/logs/logs-ingestion-consumer.ts@@ -48,6 +48,7 @@ import {processLogMessageBuffer,sniffJsonLogAttributes,} from './log-record-avro'+import { rethrowKafkaOutageAsDependency } from './logs-produce-errors'import type { CompiledMetricRule, MetricRuleSource } from './metrics-rules/compile-metric-rules'import { withDefaultMetricRules } from './metrics-rules/default-metric-rules'import { MetricRulesCache } from './metrics-rules/metric-rules-cache'@@ -1249,35 +1250,36 @@ export class LogsIngestionConsumer {// Await so a rejection here lands in the catch and routes to the DLQ.await this.retryOnDependencyUnavailable(() =>- this.deps.outputs.queueMessages(LOGS_OUTPUT, [- {- value: processedValue,- key: null,- headers: {- ...withoutInternalLogsHeaders(parseKafkaHeaders(message.message.headers)),- ...(message.internalLogs !== undefined- ? { [INTERNAL_SOURCE_HEADER]: message.internalLogs.source }- : {}),- token: message.token,- team_id: message.teamId.toString(),- 'json-parse': jsonParse.toString(),- 'retention-days': retentionDays.toString(),- // The ingestion lag checkpoint needs the source partition, because- // both topics use random partitioning.- source_topic: message.message.topic,- source_partition: message.message.partition.toString(),- ...(bytesUncompressedHeaderOverride !== undefined- ? { bytes_uncompressed: bytesUncompressedHeaderOverride.toString() }- : {}),- ...(bytesCompressedHeaderOverride !== undefined- ? { bytes_compressed: bytesCompressedHeaderOverride.toString() }- : {}),- ...(recordCountHeaderOverride !== undefined- ? { record_count: recordCountHeaderOverride.toString() }- : {}),+ this.deps.outputs+ .queueMessages(LOGS_OUTPUT, [+ {+ value: processedValue,+ key: null,+ headers: {+ ...withoutInternalLogsHeaders(parseKafkaHeaders(message.message.headers)),+ ...(message.internalLogs !== undefined+ ? { [INTERNAL_SOURCE_HEADER]: message.internalLogs.source }+ : {}),+ token: message.token,+ team_id: message.teamId.toString(),+ 'json-parse': jsonParse.toString(),+ 'retention-days': retentionDays.toString(),+ // Both topics use random partitioning, so the ingestion lag checkpoint needs the source partition.+ source_topic: message.message.topic,+ source_partition: message.message.partition.toString(),+ ...(bytesUncompressedHeaderOverride !== undefined+ ? { bytes_uncompressed: bytesUncompressedHeaderOverride.toString() }+ : {}),+ ...(bytesCompressedHeaderOverride !== undefined+ ? { bytes_compressed: bytesCompressedHeaderOverride.toString() }+ : {}),+ ...(recordCountHeaderOverride !== undefined+ ? { record_count: recordCountHeaderOverride.toString() }+ : {}),+ },},- },- ])+ ])+ .catch(rethrowKafkaOutageAsDependency))} catch (error) {// A dependency outage (Postgres, Redis, Kafka) is infrastructure-scoped. Thediff --git a/nodejs/src/logs/logs-produce-errors.ts b/nodejs/src/logs/logs-produce-errors.tsnew file mode 100644index 000000000000..997c34367d1d--- /dev/null+++ b/nodejs/src/logs/logs-produce-errors.ts@@ -0,0 +1,48 @@+import { CODES } from 'node-rdkafka'++import { DependencyUnavailableError } from '~/common/utils/db/error'++const { err2str } = require('node-rdkafka/librdkafka') as { err2str: (code: number) => string }++const PER_MESSAGE_KAFKA_ERROR_CODES: readonly number[] = [+ CODES.ERRORS.ERR_INVALID_MSG,+ CODES.ERRORS.ERR_INVALID_MSG_SIZE,+ CODES.ERRORS.ERR_MSG_SIZE_TOO_LARGE,+ CODES.ERRORS.ERR_RECORD_LIST_TOO_LARGE,+ CODES.ERRORS.ERR_INVALID_RECORD,+ CODES.ERRORS.ERR__BAD_MSG,+ CODES.ERRORS.ERR__INVALID_ARG,+ CODES.ERRORS.ERR__INVALID_TYPE,+]++const librdkafkaText = (message: string): string => message.split(': ').pop()!.toLowerCase()++// A delivery report carries only librdkafka's text, which node-rdkafka parses into `code: -1`.+const PER_MESSAGE_KAFKA_ERROR_TEXTS = new Set(+ PER_MESSAGE_KAFKA_ERROR_CODES.map((code) => librdkafkaText(err2str(code)))+)++type LibrdKafkaError = Error & { code: number; origin: string }++function isLibrdKafkaError(error: unknown): error is LibrdKafkaError {+ if (typeof error !== 'object' || error === null) {+ return false+ }+ const { code, origin } = error as { code?: unknown; origin?: unknown }+ return typeof code === 'number' && (origin === 'kafka' || origin === 'local' || origin === 'broker')+}++function isPerMessageKafkaError(error: LibrdKafkaError): boolean {+ return (+ PER_MESSAGE_KAFKA_ERROR_CODES.includes(error.code) ||+ PER_MESSAGE_KAFKA_ERROR_TEXTS.has(librdkafkaText(error.message))+ )+}++/** node-rdkafka never marks a delivery error as retriable; raise an outage as one so the batch replays. */+export function rethrowKafkaOutageAsDependency(error: unknown): never {+ if (isLibrdKafkaError(error) && !isPerMessageKafkaError(error)) {+ throw new DependencyUnavailableError(error.message, 'Kafka', error)+ }+ throw error+}