Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(logs): add a framework logs consumer behind an env flag (#127605)

Frank Hamand836da0a3a26d
diff --git a/nodejs/src/ingestion/framework/builders/index.ts b/nodejs/src/ingestion/framework/builders/index.ts
index 3189676e386a..98a8723d38de 100644
--- a/nodejs/src/ingestion/framework/builders/index.ts
+++ b/nodejs/src/ingestion/framework/builders/index.ts
@@ -8,3 +8,4 @@ export type { RetryOptions } from '~/ingestion/framework/retry'
export type { ChunkProcessingStep } from '~/ingestion/framework/base-chunk-pipeline'
export { BranchingPipelineBuilder, PipelineBuilder, StartPipelineBuilder } from './pipeline-builders'
export { newChunkPipelineBuilder, newBatchingPipeline, newPipelineBuilder } from './helpers'
+export { perElement } from '~/ingestion/framework/per-element-step'
diff --git a/nodejs/src/ingestion/framework/per-element-step.test.ts b/nodejs/src/ingestion/framework/per-element-step.test.ts
new file mode 100644
index 000000000000..f010e187460d
--- /dev/null
+++ b/nodejs/src/ingestion/framework/per-element-step.test.ts
@@ -0,0 +1,30 @@
+import { perElement } from './per-element-step'
+import { ok } from './results'
+
+describe('perElement', () => {
+ it('keeps input order and never runs more than maxConcurrency steps at once', async () => {
+ let running = 0
+ let peak = 0
+ const started: number[] = []
+ const step = perElement(
+ async (value: number) => {
+ started.push(value)
+ running++
+ peak = Math.max(peak, running)
+ // Later values wait fewer ticks and finish first, so output order cannot follow completion order.
+ for (let tick = 0; tick < 8 - value; tick++) {
+ await new Promise((resolve) => setImmediate(resolve))
+ }
+ running--
+ return ok(value * 10)
+ },
+ { maxConcurrency: 3 }
+ )
+
+ const results = await step([1, 2, 3, 4, 5, 6, 7])
+
+ expect(results).toEqual([1, 2, 3, 4, 5, 6, 7].map((value) => ok(value * 10)))
+ expect(peak).toBe(3)
+ expect(started).toEqual([1, 2, 3, 4, 5, 6, 7])
+ })
+})
diff --git a/nodejs/src/ingestion/framework/per-element-step.ts b/nodejs/src/ingestion/framework/per-element-step.ts
new file mode 100644
index 000000000000..086306cc6ff4
--- /dev/null
+++ b/nodejs/src/ingestion/framework/per-element-step.ts
@@ -0,0 +1,16 @@
+import pLimit from 'p-limit'
+
+import { ChunkProcessingStep } from './base-chunk-pipeline'
+import { ProcessingStep } from './steps'
+
+// One chunk step, so instrumentation runs per chunk. A throw rejects the chunk but does not cancel the other values.
+export function perElement<T, U, R extends string = never>(
+ step: ProcessingStep<T, U, R>,
+ options: { maxConcurrency?: number } = {}
+): ChunkProcessingStep<T, U, R> {
+ const limit = options.maxConcurrency !== undefined ? pLimit(options.maxConcurrency) : null
+ const chunkStep: ChunkProcessingStep<T, U, R> = (values) =>
+ Promise.all(values.map((value) => (limit ? limit(() => step(value)) : step(value))))
+ Object.defineProperty(chunkStep, 'name', { value: step.name })
+ return chunkStep
+}
diff --git a/nodejs/src/ingestion/pipelines/metrics/metrics-ingestion-pipeline.ts b/nodejs/src/ingestion/pipelines/metrics/metrics-ingestion-pipeline.ts
index 6d87a0a17333..248dfff3b12d 100644
--- a/nodejs/src/ingestion/pipelines/metrics/metrics-ingestion-pipeline.ts
+++ b/nodejs/src/ingestion/pipelines/metrics/metrics-ingestion-pipeline.ts
@@ -6,14 +6,14 @@ import { QuotaLimiting } from '~/common/services/quota-limiting.service'
import { PromiseScheduler } from '~/common/utils/promise-scheduler'
import { TeamManager } from '~/common/utils/team-manager'
import { BatchingContext, BatchingPipeline } from '~/ingestion/framework/batching-pipeline'
-import { newBatchingPipeline } from '~/ingestion/framework/builders'
+import { newBatchingPipeline, perElement } from '~/ingestion/framework/builders'
import { aggregateKafkaDebugContexts, createBatch } from '~/ingestion/framework/helpers'
import { PipelineConfig } from '~/ingestion/framework/result-handling-pipeline'
import { MetricsUsageAccumulator, MetricsUsageBatchContext } from './metrics-usage'
import { createKeepMetricsUsageStep, createMetricsUsageBeforeBatchStep } from './metrics-usage-steps'
import { MetricsOutput } from './outputs/outputs'
-import { createPrepareMetricsMessageStep, perMessage } from './prepare-metrics-message-step'
+import { createPrepareMetricsMessageStep } from './prepare-metrics-message-step'
import { createProduceMetricsStep } from './produce-metrics-step'
import { createRateLimitMetricsStep } from './rate-limit-metrics-step'
import { MetricsRateLimiterService } from './services/metrics-rate-limiter.service'
@@ -57,9 +57,9 @@ export function createMetricsIngestionPipeline(config: MetricsIngestionPipelineC
batch
.messageAware((b) =>
b
- .pipeChunk(perMessage(createPrepareMetricsMessageStep(teamManager, quotaLimiting)))
+ .pipeChunk(perElement(createPrepareMetricsMessageStep(teamManager, quotaLimiting)))
.pipeChunk(createRateLimitMetricsStep(rateLimiter))
- .pipeChunk(perMessage(createProduceMetricsStep(outputs)))
+ .pipeChunk(perElement(createProduceMetricsStep(outputs)))
)
.handleResults(pipelineConfig)
.handleSideEffects(promiseScheduler, sideEffects),
diff --git a/nodejs/src/ingestion/pipelines/metrics/prepare-metrics-message-step.ts b/nodejs/src/ingestion/pipelines/metrics/prepare-metrics-message-step.ts
index 8a5d6571841a..348e4bc95183 100644
--- a/nodejs/src/ingestion/pipelines/metrics/prepare-metrics-message-step.ts
+++ b/nodejs/src/ingestion/pipelines/metrics/prepare-metrics-message-step.ts
@@ -2,7 +2,6 @@ import { Message } from 'node-rdkafka'
import { QuotaLimiting } from '~/common/services/quota-limiting.service'
import { TeamManager } from '~/common/utils/team-manager'
-import { ChunkProcessingStep } from '~/ingestion/framework/builders'
import { isOkResult } from '~/ingestion/framework/results'
import { ProcessingStep } from '~/ingestion/framework/steps'
@@ -15,15 +14,6 @@ import { createResolveMetricsTeamStep } from './resolve-metrics-team-step'
const RESOLVE_TEAM_RETRY = { tries: 3, sleepMs: 100, name: 'resolve_metrics_team' }
-/** The framework instruments a chunk step once per chunk, and per-message instrumentation costs more than reading the headers. */
-export function perMessage<T, U, R extends string = never>(
- step: ProcessingStep<T, U, R>
-): ChunkProcessingStep<T, U, R> {
- const chunkStep: ChunkProcessingStep<T, U, R> = (values) => Promise.all(values.map((value) => step(value)))
- Object.defineProperty(chunkStep, 'name', { value: step.name })
- return chunkStep
-}
-
export function createPrepareMetricsMessageStep<T extends { message: Message; usage: MetricsUsageAccumulator }>(
teamManager: Pick<TeamManager, 'getTeam' | 'getTeamByToken'>,
quotaLimiting: Pick<QuotaLimiting, 'isTeamTokenQuotaLimited'>
diff --git a/nodejs/src/logs/config.ts b/nodejs/src/logs/config.ts
index a022912ba191..4078d6e974a7 100644
--- a/nodejs/src/logs/config.ts
+++ b/nodejs/src/logs/config.ts
@@ -94,6 +94,7 @@ export type LogsIngestionConsumerConfig = {
* (shadow mode) — billing is unchanged.
*/
LOGS_BILLING_PRORATE_ENABLED: boolean
+ LOGS_INGESTION_USE_FRAMEWORK: boolean
/** Comma-separated team IDs, or `*` for all teams, or empty (default) to disable hog log transformations entirely. */
LOGS_TRANSFORMATIONS_ENABLED_TEAMS: string
/** When `true`, hog log transformations are skipped regardless of the allowlist. */
@@ -152,6 +153,7 @@ export function getDefaultLogsIngestionConsumerConfig(): LogsIngestionConsumerCo
LOGS_JSON_ATTRIBUTE_PARSING_ENABLED_TEAMS: '',
LOGS_JSON_ATTRIBUTE_EXTRACTION_ENABLED_TEAMS: '',
LOGS_BILLING_PRORATE_ENABLED: false,
+ LOGS_INGESTION_USE_FRAMEWORK: false,
LOGS_TRANSFORMATIONS_ENABLED_TEAMS: '',
LOGS_TRANSFORMATIONS_KILLSWITCH: false,
LOGS_TRANSFORMATIONS_HOG_TIMEOUT_MS: 10,
@@ -196,6 +198,7 @@ export type TracesIngestionConsumerConfig = {
TRACES_RETENTION_ENABLED_TEAMS: string
/** When `true`, span retention rules are never evaluated (spans keep the team default via the batch header). */
TRACES_RETENTION_KILLSWITCH: boolean
+ TRACES_INGESTION_USE_FRAMEWORK: boolean
REDIS_URL: string
REDIS_POOL_MIN_SIZE: number
REDIS_POOL_MAX_SIZE: number
@@ -229,6 +232,7 @@ export function getDefaultTracesIngestionConsumerConfig(): TracesIngestionConsum
// Same rollout shape as the logs retention rules: on locally, off in prod until enabled per team.
TRACES_RETENTION_ENABLED_TEAMS: isProdEnv() ? '' : '*',
TRACES_RETENTION_KILLSWITCH: false,
+ TRACES_INGESTION_USE_FRAMEWORK: false,
// Overlapping fields with CommonConfig, included for standalone usage
// ok to connect to localhost over plaintext
// nosemgrep: trailofbits.generic.redis-unencrypted-transport.redis-unencrypted-transport
diff --git a/nodejs/src/logs/framework/admission-steps.ts b/nodejs/src/logs/framework/admission-steps.ts
new file mode 100644
index 000000000000..aa96c2aa6241
--- /dev/null
+++ b/nodejs/src/logs/framework/admission-steps.ts
@@ -0,0 +1,212 @@
+import { ChunkProcessingStep } from '~/ingestion/framework/base-chunk-pipeline'
+import { PipelineResult, drop, ok } from '~/ingestion/framework/results'
+
+import {
+ recordLogMessageDropped,
+ recordLogsAllowed,
+ recordLogsDropped,
+ recordLogsReceived,
+} from '../ingestion-otel-metrics'
+import {
+ DEFAULT_USAGE_STATS,
+ UsageStats,
+ UsageStatsByTeam,
+ logMessageDroppedCounter,
+ logsBytesAllowedCounter,
+ logsBytesAllowedRecordsCounter,
+ logsBytesDroppedCounter,
+ logsBytesReceivedCounter,
+ logsRecordsAllowedCounter,
+ logsRecordsDroppedCounter,
+ logsRecordsReceivedCounter,
+} from '../logs-ingestion-consumer'
+import type { LogsIngestionMessage } from '../types'
+import { LogsPipelineContext, isUnbilledInternal, teamIdMatchesCsv } from './logs-pipeline-context'
+
+export class LogsBatchUsage {
+ readonly byTeam: UsageStatsByTeam = new Map()
+
+ private row(teamId: number): UsageStats {
+ let stats = this.byTeam.get(teamId)
+ if (!stats) {
+ stats = { ...DEFAULT_USAGE_STATS }
+ this.byTeam.set(teamId, stats)
+ }
+ return stats
+ }
+
+ recordAllowed(message: LogsIngestionMessage): void {
+ const stats = this.row(message.teamId)
+ stats.bytesReceived += message.bytesUncompressed
+ stats.recordsReceived += message.recordCount
+ stats.bytesAllowed += message.bytesUncompressed
+ stats.recordsAllowed += message.recordCount
+ stats.bytesAllowedRecords += message.bytesUncompressedRecords
+ }
+
+ recordDropped(message: LogsIngestionMessage): void {
+ const stats = this.row(message.teamId)
+ stats.bytesReceived += message.bytesUncompressed
+ stats.recordsReceived += message.recordCount
+ stats.bytesDropped += message.bytesUncompressed
+ stats.recordsDropped += message.recordCount
+ }
+}
+
+export type LogsAdmissionInput = LogsIngestionMessage & { usage: LogsBatchUsage }
+
+function countMessagesDropped(reason: string, countByTeam: Map<number, number>): void {
+ for (const [teamId, count] of countByTeam) {
+ logMessageDroppedCounter.inc({ reason, team_id: teamId.toString() }, count)
+ recordLogMessageDropped(reason, teamId.toString(), count)
+ }
+}
+
+/** Verified internal logs are never quota or rate limited, so counting them would show as drops. */
+export function createTrackIncomingTrafficStep<T extends LogsAdmissionInput>(): ChunkProcessingStep<T, T> {
+ return function trackIncomingTrafficStep(values) {
+ let bytes = 0
+ let records = 0
+ for (const value of values) {
+ if (isUnbilledInternal(value)) {
+ continue
+ }
+ bytes += value.bytesUncompressed
+ records += value.recordCount
+ }
+ logsBytesReceivedCounter.inc(bytes)
+ logsRecordsReceivedCounter.inc(records)
+ recordLogsReceived(bytes, records)
+ return Promise.resolve(values.map((value) => ok(value)))
+ }
+}
+
+/** Clamping backfilled rows to the ingest time would write the import onto today, so the gate drops the whole message. */
+export function createBackfillGateStep<T extends LogsAdmissionInput>(
+ context: LogsPipelineContext
+): ChunkProcessingStep<T, T> {
+ return function backfillGateStep(values) {
+ const droppedByTeam = new Map<number, { messages: number; records: number; billed: boolean }>()
+ const results = values.map((value): PipelineResult<T> => {
+ if (!value.backfillRequested || teamIdMatchesCsv(context.gates.backfillEnabledTeams, value.teamId)) {
+ return ok(value)
+ }
+ const dropped = droppedByTeam.get(value.teamId) ?? { messages: 0, records: 0, billed: false }
+ dropped.messages++
+ if (!isUnbilledInternal(value)) {
+ dropped.records += value.recordCount
+ dropped.billed = true
+ value.usage.recordDropped(value)
+ }
+ droppedByTeam.set(value.teamId, dropped)
+ return drop('backfill_not_enabled')
+ })
+
+ for (const [teamId, dropped] of droppedByTeam) {
+ logMessageDroppedCounter.inc(
+ { reason: 'backfill_not_enabled', team_id: teamId.toString() },
+ dropped.messages
+ )
+ recordLogMessageDropped('backfill_not_enabled', teamId.toString(), dropped.messages)
+ if (dropped.billed) {
+ context.usageMetrics.queueUsageMetric(teamId, 'records_dropped_backfill_not_enabled', dropped.records)
+ }
+ }
+ return Promise.resolve(results)
+ }
+}
+
+export function createQuotaLimitStep<T extends LogsAdmissionInput>(
+ context: LogsPipelineContext
+): ChunkProcessingStep<T, T> {
+ return async function quotaLimitStep(values) {
+ const billed = values.filter((value) => !isUnbilledInternal(value))
+ const uniqueTokens = [...new Set(billed.map((value) => value.token))]
+ const limited = new Set(
+ (
+ await Promise.all(
+ uniqueTokens.map(async (token) =>
+ (await context.deps.quotaLimiting.isTeamTokenQuotaLimited(
+ token,
+ context.identity.quotaResource
+ ))
+ ? token
+ : null
+ )
+ )
+ ).filter((token): token is string => token !== null)
+ )
+
+ const droppedByTeam = new Map<number, number>()
+ const results = values.map((value): PipelineResult<T> => {
+ if (isUnbilledInternal(value) || !limited.has(value.token)) {
+ return ok(value)
+ }
+ droppedByTeam.set(value.teamId, (droppedByTeam.get(value.teamId) ?? 0) + 1)
+ value.usage.recordDropped(value)
+ return drop('quota_limited')
+ })
+ countMessagesDropped('quota_limited', droppedByTeam)
+ return results
+ }
+}
+
+export function createRateLimitStep<T extends LogsAdmissionInput>(
+ context: LogsPipelineContext
+): ChunkProcessingStep<T, T> {
+ return async function rateLimitStep(values) {
+ const billed = values.filter((value) => !isUnbilledInternal(value))
+ const { dropped } = await context.rateLimiter.filterMessages(billed)
+ const droppedSet = new Set<LogsIngestionMessage>(dropped)
+
+ const droppedByTeam = new Map<number, number>()
+ const results = values.map((value): PipelineResult<T> => {
+ if (!droppedSet.has(value)) {
+ return ok(value)
+ }
+ droppedByTeam.set(value.teamId, (droppedByTeam.get(value.teamId) ?? 0) + 1)
+ value.usage.recordDropped(value)
+ return drop('rate_limited')
+ })
+ countMessagesDropped('rate_limited', droppedByTeam)
+ return results
+ }
+}
+
+/** Allowed is gross here, because processing takes off the drop-rule billing credit. */
+export function createRecordAllowedUsageStep<T extends LogsAdmissionInput>(): ChunkProcessingStep<T, T> {
+ return function recordAllowedUsageStep(values) {
+ let bytesAllowed = 0
+ let recordsAllowed = 0
+ let bytesAllowedRecords = 0
+ for (const value of values) {
+ if (isUnbilledInternal(value)) {
+ continue
+ }
+ value.usage.recordAllowed(value)
+ bytesAllowed += value.bytesUncompressed
+ bytesAllowedRecords += value.bytesUncompressedRecords
+ recordsAllowed += value.recordCount
+ }
+
+ logsBytesAllowedRecordsCounter.inc(bytesAllowedRecords)
+ logsBytesAllowedCounter.inc(bytesAllowed)
+ logsRecordsAllowedCounter.inc(recordsAllowed)
+ recordLogsAllowed(bytesAllowed, recordsAllowed)
+ return Promise.resolve(values.map((value) => ok(value)))
+ }
+}
+
+/** Runs after admission even when every message was dropped, which no chunk step does. */
+export function reportDroppedUsage(usage: LogsBatchUsage): void {
+ for (const [teamId, stats] of usage.byTeam) {
+ const teamIdLabel = teamId.toString()
+ if (stats.bytesDropped > 0) {
+ logsBytesDroppedCounter.inc({ team_id: teamIdLabel }, stats.bytesDropped)
+ }
+ if (stats.recordsDropped > 0) {
+ logsRecordsDroppedCounter.inc({ team_id: teamIdLabel }, stats.recordsDropped)
+ }
+ recordLogsDropped(teamId, stats.bytesDropped, stats.recordsDropped)
+ }
+}
diff --git a/nodejs/src/logs/framework/create-consumer.ts b/nodejs/src/logs/framework/create-consumer.ts
new file mode 100644
index 000000000000..4cd9fd130cdc
--- /dev/null
+++ b/nodejs/src/logs/framework/create-consumer.ts
@@ -0,0 +1,23 @@
+import { LogsIngestionConsumerConfig, TracesIngestionConsumerConfig } from '../config'
+import { LogsIngestionConsumer, LogsIngestionConsumerDeps } from '../logs-ingestion-consumer'
+import { TracesIngestionConsumer, TracesIngestionConsumerDeps } from '../traces-ingestion-consumer'
+import { LogsIngestionFrameworkConsumer } from './logs-ingestion-framework-consumer'
+import { TracesIngestionFrameworkConsumer } from './traces-ingestion-framework-consumer'
+
+export function createLogsIngestionConsumer(
+ config: LogsIngestionConsumerConfig,
+ deps: LogsIngestionConsumerDeps
+): LogsIngestionConsumer | LogsIngestionFrameworkConsumer {
+ return config.LOGS_INGESTION_USE_FRAMEWORK
+ ? new LogsIngestionFrameworkConsumer(config, deps)
+ : new LogsIngestionConsumer(config, deps)
+}
+
+export function createTracesIngestionConsumer(
+ config: LogsIngestionConsumerConfig & TracesIngestionConsumerConfig,
+ deps: TracesIngestionConsumerDeps
+): TracesIngestionConsumer | TracesIngestionFrameworkConsumer {
+ return config.TRACES_INGESTION_USE_FRAMEWORK
+ ? new TracesIngestionFrameworkConsumer(config, deps)
+ : new TracesIngestionConsumer(config, deps)
+}
diff --git a/nodejs/src/logs/framework/finalize-logs-batch.ts b/nodejs/src/logs/framework/finalize-logs-batch.ts
new file mode 100644
index 000000000000..212a1b36300b
--- /dev/null
+++ b/nodejs/src/logs/framework/finalize-logs-batch.ts
@@ -0,0 +1,112 @@
+import { logger } from '~/common/utils/logger'
+import { UUID7 } from '~/common/utils/utils'
+
+import { recordInternalLogs } from '../ingestion-otel-metrics'
+import {
+ DEFAULT_LOGS_RETENTION_DAYS,
+ UsageStatsByTeam,
+ logsInternalBytesCounter,
+ logsInternalRecordsCounter,
+ logsMetricRulesEvalErrorCounter,
+ logsMetricRulesRecordsMatchedCounter,
+ logsMetricRulesSeriesOverflowCounter,
+ logsMetricRulesValueSkippedCounter,
+ retentionBytesMetricName,
+} from '../logs-ingestion-consumer'
+import { buildMetricRulesOtlpPayload } from '../metrics-rules/otlp-payload'
+import type { LogsIngestionMessage } from '../types'
+import { LogsPipelineContext } from './logs-pipeline-context'
+import type { MetricTalliesByTeam } from './process-log-message-step'
+
+/** The usage report matches `bytes_ingested` exactly, so these names never reach billing. */
+export function queueInternalLogsUsage(context: LogsPipelineContext, internalMessages: LogsIngestionMessage[]): void {
+ for (const message of internalMessages) {
+ context.usageMetrics.queueUsageMetric(message.teamId, 'bytes_ingested_internal', message.bytesUncompressed)
+ context.usageMetrics.queueUsageMetric(message.teamId, 'records_ingested_internal', message.recordCount)
+ const teamId = message.teamId.toString()
+ logsInternalBytesCounter.inc({ team_id: teamId }, message.bytesUncompressed)
+ logsInternalRecordsCounter.inc({ team_id: teamId }, message.recordCount)
+ recordInternalLogs(message.teamId, message.bytesUncompressed, message.recordCount)
+ }
+}
+
+export async function emitMetricRuleTallies(context: LogsPipelineContext, byTeam: MetricTalliesByTeam): Promise<void> {
+ for (const [teamId, { token, rules, tallies }] of byTeam) {
+ const teamIdLabel = teamId.toString()
+ if (tallies.valueSkipped > 0) {
+ logsMetricRulesValueSkippedCounter.inc({ team_id: teamIdLabel }, tallies.valueSkipped)
+ }
+ if (tallies.evalErrors > 0) {
+ logsMetricRulesEvalErrorCounter.inc({ team_id: teamIdLabel }, tallies.evalErrors)
+ }
+ let overflow = 0
+ for (const count of tallies.seriesOverflow.values()) {
+ overflow += count
+ }
+ if (overflow > 0) {
+ logsMetricRulesSeriesOverflowCounter.inc({ team_id: teamIdLabel }, overflow)
+ }
+ let matched = 0
+ for (const ruleTallies of tallies.byRule.values()) {
+ for (const entry of ruleTallies.values()) {
+ matched += entry.count
+ }
+ }
+ if (matched > 0) {
+ logsMetricRulesRecordsMatchedCounter.inc({ team_id: teamIdLabel }, matched)
+ }
+
+ const payload = buildMetricRulesOtlpPayload(rules, tallies, Date.now())
+ if (!payload) {
+ continue
+ }
+ try {
+ await context.deps.metricsEmitter!.emit(token, teamId, payload)
+ } catch (error) {
+ logger.warn('[logs-metric-rules] emit threw', { teamId, error: String(error) })
+ }
+ }
+}
+
+export async function emitUsageMetrics(context: LogsPipelineContext, usageStats: UsageStatsByTeam): Promise<void> {
+ const { usageMetrics, identity, deps } = context
+ for (const [teamId, stats] of usageStats) {
+ usageMetrics.queueUsageMetric(teamId, 'bytes_received', stats.bytesReceived)
+ usageMetrics.queueUsageMetric(teamId, 'records_received', stats.recordsReceived)
+ usageMetrics.queueUsageMetric(teamId, 'bytes_ingested', stats.bytesAllowed)
+ // Emitted beside `bytes_ingested` so the two compare before billing switches to records-based bytes.
+ usageMetrics.queueUsageMetric(teamId, 'bytes_ingested_records', stats.bytesAllowedRecords)
+ usageMetrics.queueUsageMetric(teamId, 'records_ingested', stats.recordsAllowed)
+ usageMetrics.queueUsageMetric(teamId, 'bytes_dropped', stats.bytesDropped)
+ usageMetrics.queueUsageMetric(teamId, 'records_dropped', stats.recordsDropped)
+ usageMetrics.queueUsageMetric(teamId, 'pii_replacements', stats.piiReplacements)
+
+ const retentionMetric = retentionBytesMetricName(stats.retentionDays)
+ if (retentionMetric) {
+ usageMetrics.queueUsageMetric(teamId, retentionMetric, stats.bytesAllowed)
+ }
+ if (stats.retentionDays !== DEFAULT_LOGS_RETENTION_DAYS) {
+ usageMetrics.queueRetentionByteDays(teamId, stats.bytesAllowed, stats.retentionDays)
+ }
+ const source = identity.appSource === 'traces' ? 'apm_traces' : 'logs'
+ // A fresh ID per flush stops two pods that flush the same team from collapsing one quantity away.
+ const flushId = new UUID7().toString()
+ deps.usageBatch.add(teamId, `${source}_bytes`, flushId, stats.bytesAllowed, 'bytes')
+ deps.usageBatch.add(
+ teamId,
+ source === 'apm_traces' ? 'apm_traces_spans' : 'logs_records',
+ flushId,
+ stats.recordsAllowed,
+ 'records'
+ )
+ }
+
+ await Promise.all([
+ usageMetrics.flush().catch((error) => {
+ logger.error('🔴', 'Failed to emit usage metrics - billing data may be lost', { error })
+ }),
+ deps.usageBatch.flush().catch((error) => {
+ logger.error('🔴', 'Failed to emit usage records - billing data may be lost', { error })
+ }),
+ ])
+}
diff --git a/nodejs/src/logs/framework/logs-ingestion-framework-consumer.test.ts b/nodejs/src/logs/framework/logs-ingestion-framework-consumer.test.ts
new file mode 100644
index 000000000000..b30566f275f7
--- /dev/null
+++ b/nodejs/src/logs/framework/logs-ingestion-framework-consumer.test.ts
@@ -0,0 +1,365 @@
+import { mockProducer, mockProducerObserver } from '~/tests/helpers/mocks/producer.mock'
+
+import { DateTime } from 'luxon'
+import { CODES, Message } from 'node-rdkafka'
+
+import { KAFKA_APP_METRICS_2, KAFKA_LOGS_CLICKHOUSE, KAFKA_LOGS_INGESTION_DLQ } from '~/common/config/kafka-topics'
+import { APP_METRICS_OUTPUT, AppMetricsOutput } from '~/common/outputs'
+import { IngestionOutputs } from '~/common/outputs/ingestion-outputs'
+import { SingleIngestionOutput } from '~/common/outputs/single-ingestion-output'
+import { UsageRecordBatch } from '~/common/usage-ingestion/usage-record-batch'
+import { DependencyUnavailableError } from '~/common/utils/db/error'
+import { closeHub, createHub } from '~/common/utils/db/hub'
+import { createTeam, createTestTeamFixture, getTeam } from '~/tests/helpers/sql'
+import { Hub, Team } from '~/types'
+
+import { getDefaultTracesIngestionConsumerConfig } from '../config'
+import { LogRecord, encodeLogRecords } from '../log-record-avro'
+import { DEFAULT_LOGS_RETENTION_DAYS, LogsIngestionConsumer, logMessageDlqCounter } from '../logs-ingestion-consumer'
+import { LOGS_DLQ_OUTPUT, LOGS_OUTPUT, LogsDlqOutput, LogsOutput } from '../outputs/outputs'
+import { DEFAULT_TRACES_RETENTION_DAYS } from '../retention/tracing-config-cache'
+import { TracesIngestionConsumer } from '../traces-ingestion-consumer'
+import { createLogsIngestionConsumer, createTracesIngestionConsumer } from './create-consumer'
+import { LogsIngestionFrameworkConsumer } from './logs-ingestion-framework-consumer'
+import { TracesIngestionFrameworkConsumer } from './traces-ingestion-framework-consumer'
+
+const LibrdKafkaError = require('node-rdkafka/lib/error')
+const { err2str } = require('node-rdkafka/librdkafka') as { err2str: (code: number) => string }
+
+jest.setTimeout(5000)
+
+jest.mock('~/common/utils/posthog', () => {
+ const original = jest.requireActual('~/common/utils/posthog')
+ return {
+ ...original,
+ captureException: jest.fn(),
+ }
+})
+
+const avro = require('avsc')
+const LOG_RECORD_TYPE = avro.Type.forSchema({
+ type: 'record',
+ name: 'LogRecord',
+ fields: [
+ { name: 'uuid', type: ['null', 'string'] },
+ { name: 'trace_id', type: ['null', 'bytes'] },
+ { name: 'span_id', type: ['null', 'bytes'] },
+ { name: 'trace_flags', type: ['null', 'int'] },
+ { name: 'timestamp', type: ['null', 'long'] },
+ { name: 'observed_timestamp', type: ['null', 'long'] },
+ { name: 'body', type: ['null', 'string'] },
+ { name: 'severity_text', type: ['null', 'string'] },
+ { name: 'severity_number', type: ['null', 'int'] },
+ { name: 'service_name', type: ['null', 'string'] },
+ { name: 'resource_attributes', type: ['null', { type: 'map', values: 'string' }] },
+ { name: 'instrumentation_scope', type: ['null', 'string'] },
+ { name: 'event_name', type: ['null', 'string'] },
+ { name: 'attributes', type: ['null', { type: 'map', values: 'string' }] },
+ ],
+})
+
+let offset = 0
+
+async function createKafkaMessages(count: number, headers: Record<string, string>): Promise<Message[]> {
+ return await Promise.all(
+ Array.from({ length: count }, async () => {
+ const record: LogRecord = {
+ uuid: `test-uuid-${offset}`,
+ trace_id: null,
+ span_id: null,
+ trace_flags: null,
+ timestamp: Date.now() * 1000,
+ observed_timestamp: Date.now() * 1000,
+ body: 'Test log message',
+ severity_text: 'info',
+ severity_number: 9,
+ service_name: 'test-service',
+ resource_attributes: null,
+ instrumentation_scope: null,
+ event_name: null,
+ attributes: null,
+ }
+ const value = await encodeLogRecords(LOG_RECORD_TYPE, 'zstandard', [record])
+ return {
+ key: null,
+ value,
+ size: value.length,
+ topic: 'test',
+ offset: offset++,
+ timestamp: Date.now(),
+ partition: 1,
+ headers: Object.entries(headers).map(([key, header]) => ({ [key]: Buffer.from(header) })),
+ }
+ })
+ )
+}
+
+function producedTo(topic: string): { headers?: Record<string, string>; value: unknown }[] {
+ return mockProducerObserver
+ .getProducedMessages()
+ .filter((batch) => batch.topic === topic)
+ .flatMap((batch) => batch.messages.map((message) => ({ headers: message.headers, value: message.value })))
+}
+
+const kafkaDeliveryError = (code: number): Error => LibrdKafkaError.create(new Error(err2str(code)))
+
+function failLogsProduce(makeError: () => unknown, times = Infinity): () => void {
+ const original = mockProducer.queueMessages
+ const passthrough = original.bind(mockProducer)
+ let failures = 0
+ mockProducer.queueMessages = jest.fn().mockImplementation((topicMessages) => {
+ const topic = Array.isArray(topicMessages) ? topicMessages[0]?.topic : topicMessages.topic
+ if (topic === KAFKA_LOGS_CLICKHOUSE && failures++ < times) {
+ throw makeError()
+ }
+ return passthrough(topicMessages)
+ })
+ return () => {
+ mockProducer.queueMessages = original
+ }
+}
+
+describe('LogsIngestionFrameworkConsumer', () => {
+ let hub: Hub
+ let team: Team
+ let team2: Team
+ let consumer: LogsIngestionFrameworkConsumer
+
+ const consumerDeps = (): any => ({
+ ...hub,
+ outputs: new IngestionOutputs<AppMetricsOutput | LogsOutput | LogsDlqOutput>({
+ [APP_METRICS_OUTPUT]: new SingleIngestionOutput(
+ APP_METRICS_OUTPUT,
+ KAFKA_APP_METRICS_2,
+ mockProducer,
+ 'test'
+ ),
+ [LOGS_OUTPUT]: new SingleIngestionOutput(LOGS_OUTPUT, KAFKA_LOGS_CLICKHOUSE, mockProducer, 'test'),
+ [LOGS_DLQ_OUTPUT]: new SingleIngestionOutput(
+ LOGS_DLQ_OUTPUT,
+ KAFKA_LOGS_INGESTION_DLQ,
+ mockProducer,
+ 'test'
+ ),
+ }),
+ usageBatch: new UsageRecordBatch(null, { unit: 'bytes', isTeamEnabled: () => false }),
+ dependencyRetry: { retryCount: 2, initialRetryDelayMs: 1 },
+ })
+
+ const withoutKafka = <T extends { start(): Promise<void> }>(instance: T): T => {
+ ;(instance as any)['kafkaConsumer'] = {
+ connect: jest.fn(),
+ disconnect: jest.fn(),
+ isHealthy: jest.fn().mockReturnValue({ status: 'healthy' }),
+ }
+ return instance
+ }
+
+ const tracesConfig = (overrides: Record<string, unknown> = {}): any => ({
+ ...hub,
+ ...getDefaultTracesIngestionConsumerConfig(),
+ // A blank traces Redis host reuses the hub's Redis, so no second pool outlives the suite.
+ TRACES_REDIS_HOST: '',
+ ...overrides,
+ })
+
+ const processAndWait = async (messages: Message[]): Promise<void> => {
+ await (
+ await consumer.processKafkaBatch(messages)
+ ).backgroundTask
+ }
+
+ beforeEach(async () => {
+ jest.spyOn(Date, 'now').mockReturnValue(DateTime.fromISO('2025-01-01T00:00:00Z').toMillis())
+ offset = 0
+ hub = await createHub()
+ team = (await createTestTeamFixture(hub.postgres)).team
+ team2 = (await getTeam(hub.postgres, await createTeam(hub.postgres, team.organization_id)))!
+ jest.spyOn(hub.quotaLimiting, 'isTeamTokenQuotaLimited').mockResolvedValue(false)
+ consumer = withoutKafka(new LogsIngestionFrameworkConsumer(hub, consumerDeps()))
+ await consumer.start()
+ })
+
+ afterEach(async () => {
+ await consumer.stop()
+ await closeHub(hub)
+ })
+
+ it('produces a valid message to the logs topic with the batch headers', async () => {
+ const messages = await createKafkaMessages(1, { token: team.api_token })
+
+ await processAndWait(messages)
+
+ const produced = producedTo(KAFKA_LOGS_CLICKHOUSE)
+ expect(produced).toHaveLength(1)
+ expect(produced[0].headers).toEqual({
+ token: team.api_token,
+ team_id: team.id.toString(),
+ 'json-parse': 'false',
+ 'retention-days': DEFAULT_LOGS_RETENTION_DAYS.toString(),
+ source_topic: messages[0].topic,
+ source_partition: messages[0].partition.toString(),
+ })
+ })
+
+ it('keeps the messages of overlapping batches apart', async () => {
+ const first = await createKafkaMessages(2, { token: team.api_token })
+ const second = await createKafkaMessages(1, { token: team2.api_token })
+
+ const [a, b] = await Promise.all([consumer.processKafkaBatch(first), consumer.processKafkaBatch(second)])
+ await Promise.all([a.backgroundTask, b.backgroundTask])
+
+ expect(a.messages.map((m) => m.teamId)).toEqual([team.id, team.id])
+ expect(b.messages.map((m) => m.teamId)).toEqual([team2.id])
+ expect(producedTo(KAFKA_LOGS_CLICKHOUSE)).toHaveLength(3)
+ })
+
+ describe('logs produce errors', () => {
+ it.each([
+ ['a processing error', () => new Error('Producer error')],
+ ['a Kafka error that a replay cannot fix', () => kafkaDeliveryError(CODES.ERRORS.ERR_INVALID_RECORD)],
+ ])('sends the message to the DLQ on %s', async (_, makeError) => {
+ const messages = await createKafkaMessages(1, { token: team.api_token })
+ const dlqCounterSpy = jest.spyOn(logMessageDlqCounter, 'inc')
+ const restore = failLogsProduce(makeError)
+
+ try {
+ await processAndWait(messages)
+ } finally {
+ restore()
+ }
+
+ expect(dlqCounterSpy).toHaveBeenCalledWith({ reason: 'Error', team_id: team.id.toString() })
+ const dlq = producedTo(KAFKA_LOGS_INGESTION_DLQ)
+ expect(dlq).toHaveLength(1)
+ expect(dlq[0].headers).toMatchObject({
+ source_topic: messages[0].topic,
+ source_partition: messages[0].partition.toString(),
+ source_offset: messages[0].offset.toString(),
+ })
+ expect(dlq[0].headers).toHaveProperty('error_message')
+ })
+
+ it('retries a dependency blip in place and produces the message once', async () => {
+ const messages = await createKafkaMessages(1, { token: team.api_token })
+ const restore = failLogsProduce(
+ () => new DependencyUnavailableError('broker down', 'Kafka', new Error('broker down')),
+ 1
+ )
+
+ try {
+ await processAndWait(messages)
+ } finally {
+ restore()
+ }
+
+ expect(producedTo(KAFKA_LOGS_CLICKHOUSE)).toHaveLength(1)
+ expect(producedTo(KAFKA_LOGS_INGESTION_DLQ)).toHaveLength(0)
+ })
+
+ it.each([
+ [
+ 'a dependency is unavailable',
+ () => new DependencyUnavailableError('broker down', 'Kafka', new Error('broker down')),
+ ],
+ ['a delivery times out', () => kafkaDeliveryError(CODES.ERRORS.ERR__MSG_TIMED_OUT)],
+ ['the producer queue is full', () => LibrdKafkaError.create(CODES.ERRORS.ERR__QUEUE_FULL)],
+ ['the topic rejects authorization', () => kafkaDeliveryError(CODES.ERRORS.ERR_TOPIC_AUTHORIZATION_FAILED)],
+ ])('fails the batch instead of DLQing when %s', async (_, makeError) => {
+ const messages = await createKafkaMessages(1, { token: team.api_token })
+ const restore = failLogsProduce(makeError)
+
+ try {
+ // Good data succeeds on redelivery, so quarantining it would lose it to an outage.
+ await expect(processAndWait(messages)).rejects.toThrow()
+ } finally {
+ restore()
+ }
+
+ expect(producedTo(KAFKA_LOGS_INGESTION_DLQ)).toHaveLength(0)
+ })
+
+ it.each([
+ ['the message fails to parse', { bytes_uncompressed: 'not-a-number' }],
+ ['the message fails to process', {}],
+ ])('fails only that batch when %s and the DLQ write fails', async (_, headers) => {
+ // Two failing messages: the second failure must be awaited too, not left unhandled.
+ const messages = await createKafkaMessages(2, { token: team.api_token, ...headers })
+ const original = mockProducer.queueMessages
+ mockProducer.queueMessages = jest.fn().mockRejectedValue(new Error('DLQ unavailable'))
+
+ try {
+ await expect(processAndWait(messages)).rejects.toThrow('DLQ unavailable')
+ } finally {
+ mockProducer.queueMessages = original
+ }
+
+ await processAndWait(await createKafkaMessages(1, { token: team.api_token }))
+ expect(producedTo(KAFKA_LOGS_CLICKHOUSE)).toHaveLength(1)
+ })
+ })
+
+ describe('TracesIngestionFrameworkConsumer', () => {
+ it('reads the TRACES_ consumer and rate limiter config', () => {
+ const traces = new TracesIngestionFrameworkConsumer(
+ tracesConfig({
+ TRACES_INGESTION_CONSUMER_GROUP_ID: 'traces-group',
+ TRACES_INGESTION_CONSUMER_CONSUME_TOPIC: 'traces-topic',
+ TRACES_LIMITER_BUCKET_SIZE_KB: 42,
+ }),
+ consumerDeps()
+ )
+
+ expect(traces['groupId']).toBe('traces-group')
+ expect(traces['topic']).toBe('traces-topic')
+ expect(traces['rateLimiter']['config'].LOGS_LIMITER_BUCKET_SIZE_KB).toBe(42)
+ expect(traces['rateLimiter']['rateLimiter'].getKeyPrefix()).toBe('@posthog-test/traces-rate-limiter/tokens')
+ expect(traces.service.id).toBe('TracesIngestionConsumer')
+ })
+
+ it('takes its default retention from the tracing config, not logs_settings', async () => {
+ const tracingConfigCache = { getRetentionDays: jest.fn().mockResolvedValue(90) }
+ const traces = new TracesIngestionFrameworkConsumer(tracesConfig(), {
+ ...consumerDeps(),
+ tracingConfigCache,
+ })
+ const fallback = new TracesIngestionFrameworkConsumer(tracesConfig(), consumerDeps())
+
+ expect(await traces['defaultRetentionDays'](team.id, { retention_days: 30 })).toBe(90)
+ expect(tracingConfigCache.getRetentionDays).toHaveBeenCalledWith(team.id)
+ expect(await fallback['defaultRetentionDays'](team.id, { retention_days: 30 })).toBe(
+ DEFAULT_TRACES_RETENTION_DAYS
+ )
+ })
+ })
+
+ describe('the consumer factories the servers call', () => {
+ it.each([
+ [false, LogsIngestionConsumer],
+ [true, LogsIngestionFrameworkConsumer],
+ ])('builds the logs consumer for LOGS_INGESTION_USE_FRAMEWORK=%s', (useFramework, expected) => {
+ const built = createLogsIngestionConsumer(
+ { ...hub, LOGS_INGESTION_USE_FRAMEWORK: useFramework },
+ consumerDeps()
+ )
+
+ expect(built.constructor).toBe(expected)
+ })
+
+ it.each([
+ [false, TracesIngestionConsumer],
+ [true, TracesIngestionFrameworkConsumer],
+ ])('builds the traces consumer for TRACES_INGESTION_USE_FRAMEWORK=%s', (useFramework, expected) => {
+ // The logs flag is set the other way, so the traces server must read its own flag.
+ const built = createTracesIngestionConsumer(
+ tracesConfig({
+ TRACES_INGESTION_USE_FRAMEWORK: useFramework,
+ LOGS_INGESTION_USE_FRAMEWORK: !useFramework,
+ }),
+ consumerDeps()
+ )
+
+ expect(built.constructor).toBe(expected)
+ })
+ })
+})
diff --git a/nodejs/src/logs/framework/logs-ingestion-framework-consumer.ts b/nodejs/src/logs/framework/logs-ingestion-framework-consumer.ts
new file mode 100644
index 000000000000..e8c3813b804c
--- /dev/null
+++ b/nodejs/src/logs/framework/logs-ingestion-framework-consumer.ts
@@ -0,0 +1,270 @@
+import { Message } from 'node-rdkafka'
+
+import { KafkaConsumerInterface, createKafkaConsumer } from '~/common/kafka/consumer'
+import { RedisV2, createRedisV2PoolFromConfig } from '~/common/redis/redis-v2'
+import { AppMetricsAggregator } from '~/common/services/app-metrics-aggregator'
+import { QuotaResource } from '~/common/services/quota-limiting.service'
+import { instrumentFn, instrumented } from '~/common/tracing/tracing-utils'
+import { logger } from '~/common/utils/logger'
+import type { LogsSettings } from '~/types'
+import { HealthCheckResult, PluginServerService } from '~/types'
+
+import { LogsIngestionConsumerConfig } from '../config'
+import { InternalLogsVerifier } from '../internal-logs'
+import {
+ DEFAULT_LOGS_RETENTION_DAYS,
+ LogsIngestionConsumerDeps,
+ UsageStatsByTeam,
+ describeBatchPosition,
+} from '../logs-ingestion-consumer'
+import type { MetricRuleSource } from '../metrics-rules/compile-metric-rules'
+import { MetricSeriesAdmission } from '../metrics-rules/tally'
+import type { RetentionRuleSource } from '../retention/compile-retention-rules'
+import { LogsSamplingService } from '../sampling/logs-sampling.service'
+import { LogsRateLimiterService } from '../services/logs-rate-limiter.service'
+import type { TransformationBatchBudget } from '../transformations/logs-transformer.service'
+import { LogsIngestionMessage } from '../types'
+import { LogsAdmissionInput, LogsBatchUsage, reportDroppedUsage } from './admission-steps'
+import { emitMetricRuleTallies, emitUsageMetrics, queueInternalLogsUsage } from './finalize-logs-batch'
+import {
+ LogsConsumerIdentity,
+ LogsFeatureGates,
+ LogsPipelineContext,
+ LogsUsageMetricsQueue,
+ isUnbilledInternal,
+} from './logs-pipeline-context'
+import {
+ createLogsAdmissionPipeline,
+ createLogsParsePipeline,
+ createLogsProcessingPipeline,
+ runChunkPipeline,
+} from './logs-pipelines'
+import { type LogsProcessingPass, type MetricTalliesByTeam } from './process-log-message-step'
+
+export class LogsIngestionFrameworkConsumer {
+ protected name = 'LogsIngestionConsumer'
+ protected quotaResource: QuotaResource = 'logs_mb_ingested'
+ protected appSource: 'logs' | 'traces' = 'logs'
+ protected metricRuleSource: MetricRuleSource = 'logs'
+ protected retentionRuleSource: RetentionRuleSource = 'logs'
+ protected kafkaConsumer: KafkaConsumerInterface
+ private readonly appMetricsAggregator: AppMetricsAggregator
+ private readonly redis: RedisV2
+ private readonly rateLimiter: LogsRateLimiterService
+ private readonly internalLogsVerifier: InternalLogsVerifier
+ private readonly samplingService: LogsSamplingService
+ private readonly metricSeriesAdmission = new MetricSeriesAdmission()
+ private readonly gates: LogsFeatureGates
+ // Built lazily, because subclass field initializers run after this constructor.
+ private pipelineContext?: LogsPipelineContext
+
+ protected groupId: string
+ protected topic: string
+
+ constructor(
+ config: LogsIngestionConsumerConfig,
+ private deps: LogsIngestionConsumerDeps,
+ overrides: Partial<LogsIngestionConsumerConfig> = {},
+ rateLimiterName: string = 'logs-rate-limiter'
+ ) {
+ const mergedConfig = { ...config, ...overrides }
+
+ this.groupId = mergedConfig.LOGS_INGESTION_CONSUMER_GROUP_ID
+ this.topic = mergedConfig.LOGS_INGESTION_CONSUMER_CONSUME_TOPIC
+
+ this.appMetricsAggregator = new AppMetricsAggregator(deps.outputs)
+
+ this.kafkaConsumer = createKafkaConsumer({ groupId: this.groupId, topic: this.topic })
+ this.redis = createRedisV2PoolFromConfig({
+ connection: mergedConfig.LOGS_REDIS_HOST
+ ? {
+ url: mergedConfig.LOGS_REDIS_HOST,
+ options: {
+ port: mergedConfig.LOGS_REDIS_PORT,
+ tls: mergedConfig.LOGS_REDIS_TLS ? {} : undefined,
+ },
+ name: 'logs-redis',
+ }
+ : { url: mergedConfig.REDIS_URL, name: 'logs-redis-fallback' },
+ poolMinSize: mergedConfig.REDIS_POOL_MIN_SIZE,
+ poolMaxSize: mergedConfig.REDIS_POOL_MAX_SIZE,
+ })
+ this.rateLimiter = new LogsRateLimiterService(mergedConfig, this.redis, rateLimiterName)
+ this.internalLogsVerifier = new InternalLogsVerifier(
+ mergedConfig.LOGS_INTERNAL_INGESTION_SIGNING_KEYS,
+ mergedConfig.LOGS_INTERNAL_INGESTION_ENABLED_TEAMS,
+ deps.internalSourcePolicies
+ )
+ this.samplingService = new LogsSamplingService(this.redis, mergedConfig.LOGS_LIMITER_TTL_SECONDS)
+ this.gates = {
+ backfillEnabledTeams: mergedConfig.LOGS_BACKFILL_ENABLED_TEAMS,
+ samplingEnabledTeams: mergedConfig.LOGS_SAMPLING_ENABLED_TEAMS,
+ samplingKillswitch: mergedConfig.LOGS_SAMPLING_KILLSWITCH,
+ billingProrateEnabled: mergedConfig.LOGS_BILLING_PRORATE_ENABLED,
+ metricRulesEnabledTeams: mergedConfig.LOGS_METRICS_RULES_ENABLED_TEAMS,
+ metricRulesKillswitch: mergedConfig.LOGS_METRICS_RULES_KILLSWITCH,
+ defaultMetricRulesEnabledTeams: mergedConfig.LOGS_DEFAULT_METRIC_RULES_ENABLED_TEAMS,
+ transformationsEnabledTeams: mergedConfig.LOGS_TRANSFORMATIONS_ENABLED_TEAMS,
+ transformationsKillswitch: mergedConfig.LOGS_TRANSFORMATIONS_KILLSWITCH,
+ retentionEnabledTeams: mergedConfig.LOGS_RETENTION_ENABLED_TEAMS,
+ retentionKillswitch: mergedConfig.LOGS_RETENTION_KILLSWITCH,
+ patternMaskingEnabledTeams: mergedConfig.LOGS_PATTERN_MASKING_ENABLED_TEAMS,
+ jsonAttributeParsingEnabledTeams: mergedConfig.LOGS_JSON_ATTRIBUTE_PARSING_ENABLED_TEAMS,
+ jsonAttributeExtractionEnabledTeams: mergedConfig.LOGS_JSON_ATTRIBUTE_EXTRACTION_ENABLED_TEAMS,
+ }
+ }
+
+ protected defaultRetentionDays(_teamId: number, logsSettings: LogsSettings): Promise<number> {
+ return Promise.resolve(logsSettings.retention_days ?? DEFAULT_LOGS_RETENTION_DAYS)
+ }
+
+ // Pipelines are built per batch, because overlapping calls that share a chunk pipeline mix their messages.
+ private getPipelineContext(): LogsPipelineContext {
+ if (!this.pipelineContext) {
+ const identity: LogsConsumerIdentity = {
+ name: this.name,
+ appSource: this.appSource,
+ quotaResource: this.quotaResource,
+ metricRuleSource: this.metricRuleSource,
+ retentionRuleSource: this.retentionRuleSource,
+ }
+ this.pipelineContext = {
+ identity,
+ gates: this.gates,
+ deps: this.deps,
+ usageMetrics: new LogsUsageMetricsQueue(this.appMetricsAggregator, identity.appSource),
+ rateLimiter: this.rateLimiter,
+ samplingService: this.samplingService,
+ internalLogsVerifier: this.internalLogsVerifier,
+ metricSeriesAdmission: this.metricSeriesAdmission,
+ defaultRetentionDays: (teamId, logsSettings) => this.defaultRetentionDays(teamId, logsSettings),
+ }
+ }
+ return this.pipelineContext
+ }
+
+ public get service(): PluginServerService {
+ return {
+ id: this.name,
+ onShutdown: async () => await this.stop(),
+ healthcheck: () => this.isHealthy(),
+ }
+ }
+
+ @instrumented('logsIngestionConsumer.handleEachBatch.parseKafkaMessages')
+ public async _parseKafkaBatch(messages: Message[]): Promise<LogsIngestionMessage[]> {
+ return await runChunkPipeline(
+ createLogsParsePipeline(this.getPipelineContext()),
+ messages.map((message) => ({ message }))
+ )
+ }
+
+ public async processBatch(
+ messages: LogsIngestionMessage[]
+ ): Promise<{ backgroundTask?: Promise<any>; messages: LogsIngestionMessage[] }> {
+ if (!messages.length) {
+ return { messages: [] }
+ }
+ const context = this.getPipelineContext()
+
+ const usage = new LogsBatchUsage()
+ const admitted = await runChunkPipeline<LogsAdmissionInput, LogsAdmissionInput, Record<string, never>>(
+ createLogsAdmissionPipeline(context),
+ messages.map((message) => ({ ...message, usage }))
+ )
+ reportDroppedUsage(usage)
+
+ const billed: LogsIngestionMessage[] = []
+ const internal: LogsIngestionMessage[] = []
+ for (const { usage: _usage, ...message } of admitted) {
+ ;(isUnbilledInternal(message) ? internal : billed).push(message)
+ }
+
+ const transformationBudget = this.deps.logsTransformer?.startBatch()
+
+ return {
+ backgroundTask: this.processAdmitted(context, billed, internal, usage.byTeam, transformationBudget),
+ messages: [...billed, ...internal],
+ }
+ }
+
+ private async processAdmitted(
+ context: LogsPipelineContext,
+ billed: LogsIngestionMessage[],
+ internal: LogsIngestionMessage[],
+ usageStats: UsageStatsByTeam,
+ transformationBudget?: TransformationBatchBudget
+ ): Promise<void> {
+ // Billed messages run first, so PII replacement counts reach the usage rows before they are emitted.
+ const billedPass: LogsProcessingPass = { usage: usageStats, metricTallies: new Map(), transformationBudget }
+ await this.runProcessingPass(context, billed, billedPass)
+ // Internal usage rows are never emitted, so they stay off billed usage. A second pass keeps the concurrency limit.
+ await this.runProcessingPass(context, internal, {
+ usage: new Map(),
+ metricTallies: new Map() as MetricTalliesByTeam,
+ transformationBudget,
+ })
+ // Metric rules and internal usage emit deltas, so they wait until no pass can fail the batch into a replay.
+ queueInternalLogsUsage(context, internal)
+ await emitMetricRuleTallies(context, billedPass.metricTallies)
+ await this.deps.teamManager.setTeamsIngestedData(billed.map((message) => message.teamId))
+ await emitUsageMetrics(context, usageStats)
+ if (this.deps.logsTransformer) {
+ await this.deps.logsTransformer.flush().catch((error) => {
+ logger.error('Failed to flush logs transformer monitoring', { error: String(error) })
+ })
+ }
+ }
+
+ private async runProcessingPass(
+ context: LogsPipelineContext,
+ messages: LogsIngestionMessage[],
+ pass: LogsProcessingPass
+ ): Promise<void> {
+ await runChunkPipeline(
+ createLogsProcessingPipeline(context),
+ messages.map((message) => ({ ...message, pass }))
+ )
+ }
+
+ public async processKafkaBatch(
+ messages: Message[]
+ ): Promise<{ backgroundTask?: Promise<any>; messages: LogsIngestionMessage[] }> {
+ try {
+ const events = await this._parseKafkaBatch(messages)
+ return await this.processBatch(events)
+ } catch (error) {
+ // Rethrow without a DLQ: the failures here are infrastructure-scoped, and the crash keeps at-least-once delivery.
+ logger.error('🔴', `${this.name} - batch failed`, {
+ error: String(error),
+ stack: error instanceof Error ? error.stack : undefined,
+ size: messages.length,
+ offsets: describeBatchPosition(messages),
+ })
+ throw error
+ }
+ }
+
+ public async start(): Promise<void> {
+ await this.kafkaConsumer.connect(async (messages) => {
+ logger.info('🔁', `${this.name} - handling batch`, {
+ size: messages.length,
+ offsets: describeBatchPosition(messages),
+ })
+
+ return await instrumentFn('logsIngestionConsumer.handleEachBatch', async () => {
+ return await this.processKafkaBatch(messages)
+ })
+ })
+ }
+
+ public async stop(): Promise<void> {
+ logger.info('💤', 'Stopping consumer...')
+ await this.kafkaConsumer.disconnect()
+ logger.info('💤', 'Consumer stopped!')
+ }
+
+ public isHealthy(): HealthCheckResult {
+ return this.kafkaConsumer.isHealthy()
+ }
+}
diff --git a/nodejs/src/logs/framework/logs-pipeline-context.ts b/nodejs/src/logs/framework/logs-pipeline-context.ts
new file mode 100644
index 000000000000..74d19801ffc4
--- /dev/null
+++ b/nodejs/src/logs/framework/logs-pipeline-context.ts
@@ -0,0 +1,175 @@
+import { DateTime } from 'luxon'
+import { Message } from 'node-rdkafka'
+
+import { parseKafkaHeaders } from '~/common/kafka/consumer'
+import { retryOnDependencyUnavailableError } from '~/common/kafka/error-handling'
+import { AppMetricsAggregator } from '~/common/services/app-metrics-aggregator'
+import { QuotaResource } from '~/common/services/quota-limiting.service'
+import { logger } from '~/common/utils/logger'
+import type { LogsSettings } from '~/types'
+
+import { recordLogMessageDlq } from '../ingestion-otel-metrics'
+import type { InternalLogsVerifier } from '../internal-logs'
+import { logMessageDlqCounter, teamIdMatchesCsv } from '../logs-ingestion-consumer'
+import type { LogsIngestionConsumerDeps } from '../logs-ingestion-consumer'
+import type { MetricRuleSource } from '../metrics-rules/compile-metric-rules'
+import type { MetricSeriesAdmission } from '../metrics-rules/tally'
+import { LOGS_DLQ_OUTPUT } from '../outputs/outputs'
+import type { RetentionRuleSource } from '../retention/compile-retention-rules'
+import { RETENTION_MONTH_DAYS } from '../retention/evaluate-retention'
+import type { LogsSamplingService } from '../sampling/logs-sampling.service'
+import type { LogsRateLimiterService } from '../services/logs-rate-limiter.service'
+import type { LogsIngestionMessage } from '../types'
+
+export { teamIdMatchesCsv }
+
+/** Traces bill against the logs quota bucket, and both rule sources use the record vocabulary (`spans`), not `traces`. */
+export type LogsConsumerIdentity = {
+ name: string
+ appSource: 'logs' | 'traces'
+ quotaResource: QuotaResource
+ metricRuleSource: MetricRuleSource
+ retentionRuleSource: RetentionRuleSource
+}
+
+export type LogsFeatureGates = {
+ backfillEnabledTeams: string
+ samplingEnabledTeams: string
+ samplingKillswitch: boolean
+ billingProrateEnabled: boolean
+ metricRulesEnabledTeams: string
+ metricRulesKillswitch: boolean
+ defaultMetricRulesEnabledTeams: string
+ transformationsEnabledTeams: string
+ transformationsKillswitch: boolean
+ retentionEnabledTeams: string
+ retentionKillswitch: boolean
+ patternMaskingEnabledTeams: string
+ jsonAttributeParsingEnabledTeams: string
+ jsonAttributeExtractionEnabledTeams: string
+}
+
+/** Lives as long as the consumer, so rows a failed batch queued flush with the next good batch, as billing relies on. */
+export class LogsUsageMetricsQueue {
+ constructor(
+ private readonly aggregator: AppMetricsAggregator,
+ private readonly appSource: string
+ ) {}
+
+ queueUsageMetric(teamId: number, metricName: string, count: number, timestamp?: DateTime): void {
+ if (count === 0) {
+ return
+ }
+ this.aggregator.queue({
+ team_id: teamId,
+ app_source: this.appSource,
+ app_source_id: '',
+ instance_id: '',
+ metric_kind: 'usage',
+ metric_name: metricName,
+ count,
+ timestamp,
+ })
+ }
+
+ /** One row per 30-day month, dated at its start, so each month bills when it begins and the TTL keeps future rows until billed. */
+ queueRetentionByteDays(teamId: number, bytes: number, retentionDays: number): void {
+ const now = DateTime.utc()
+ for (let offsetDays = 0; offsetDays < retentionDays; offsetDays += RETENTION_MONTH_DAYS) {
+ const days = Math.min(RETENTION_MONTH_DAYS, retentionDays - offsetDays)
+ this.queueUsageMetric(teamId, 'retention_byte_days', bytes * days, now.plus({ days: offsetDays }))
+ }
+ }
+
+ queueByRule(teamId: number, metricName: string, byRule: Map<string, number>): void {
+ for (const [ruleId, count] of byRule) {
+ if (count <= 0) {
+ continue
+ }
+ this.aggregator.queue({
+ team_id: teamId,
+ app_source: this.appSource,
+ app_source_id: '',
+ instance_id: ruleId,
+ metric_kind: 'usage',
+ metric_name: metricName,
+ count,
+ })
+ }
+ }
+
+ flush(): Promise<void> {
+ return this.aggregator.flush()
+ }
+}
+
+export type LogsPipelineContext = {
+ identity: LogsConsumerIdentity
+ gates: LogsFeatureGates
+ deps: LogsIngestionConsumerDeps
+ usageMetrics: LogsUsageMetricsQueue
+ rateLimiter: Pick<LogsRateLimiterService, 'filterMessages'>
+ samplingService: LogsSamplingService
+ internalLogsVerifier: InternalLogsVerifier
+ metricSeriesAdmission: MetricSeriesAdmission
+ defaultRetentionDays: (teamId: number, logsSettings: LogsSettings) => Promise<number>
+}
+
+/** Transformations write hog_function logs, which could come back here as internal logs and loop. */
+export function appliesTeamLogRules(message: LogsIngestionMessage): boolean {
+ return message.internalLogs?.teamRules ?? true
+}
+
+export function isUnbilledInternal(message: LogsIngestionMessage): boolean {
+ return message.internalLogs !== undefined && !message.internalLogs.billed
+}
+
+/** A failed batch replays in full and the logs table has no dedupe, so each retry here avoids duplicates. */
+export function retryOnDependencyUnavailable<T>(context: LogsPipelineContext, fn: () => Promise<T>): Promise<T> {
+ return retryOnDependencyUnavailableError(fn, context.deps.dependencyRetry)
+}
+
+/** The DLQ headers are the replay contract, and a failed write rejects so the offset never commits without a copy. */
+export async function quarantineLogMessage(
+ context: LogsPipelineContext,
+ message: Message,
+ error: unknown,
+ owner: { token?: string; teamId?: number }
+): Promise<void> {
+ const errorMessage = error instanceof Error ? error.message : String(error)
+ const errorName = error instanceof Error ? error.name : 'UnknownError'
+ const teamIdLabel = owner.teamId?.toString() ?? 'unknown'
+
+ logMessageDlqCounter.inc({ reason: errorName, team_id: teamIdLabel })
+ recordLogMessageDlq(errorName, teamIdLabel)
+
+ try {
+ await retryOnDependencyUnavailable(context, () =>
+ context.deps.outputs.queueMessages(LOGS_DLQ_OUTPUT, [
+ {
+ value: message.value,
+ key: null,
+ headers: {
+ ...parseKafkaHeaders(message.headers),
+ ...(owner.token !== undefined ? { token: owner.token } : {}),
+ team_id: teamIdLabel,
+ error_message: errorMessage,
+ error_name: errorName,
+ failed_at: new Date().toISOString(),
+ source_topic: message.topic,
+ source_partition: message.partition.toString(),
+ source_offset: message.offset.toString(),
+ },
+ },
+ ])
+ )
+ } catch (dlqError) {
+ logger.error('Failed to produce message to DLQ', {
+ error: dlqError,
+ originalError: errorMessage,
+ partition: message.partition,
+ offset: message.offset,
+ })
+ throw dlqError
+ }
+}
diff --git a/nodejs/src/logs/framework/logs-pipelines.ts b/nodejs/src/logs/framework/logs-pipelines.ts
new file mode 100644
index 000000000000..80c191e25b4c
--- /dev/null
+++ b/nodejs/src/logs/framework/logs-pipelines.ts
@@ -0,0 +1,122 @@
+import { logger } from '~/common/utils/logger'
+import { ChunkProcessingStep } from '~/ingestion/framework/base-chunk-pipeline'
+import { newChunkPipelineBuilder, perElement } from '~/ingestion/framework/builders'
+import { ChunkPipeline, OkResultWithContext } from '~/ingestion/framework/chunk-pipeline.interface'
+import { createOkContext } from '~/ingestion/framework/helpers'
+import { isOkResult, ok } from '~/ingestion/framework/results'
+import { ProcessingStep } from '~/ingestion/framework/steps'
+
+import { MAX_CONCURRENT_MESSAGE_PROCESSES } from '../logs-ingestion-consumer'
+import type { LogsIngestionMessage } from '../types'
+import {
+ LogsAdmissionInput,
+ createBackfillGateStep,
+ createQuotaLimitStep,
+ createRateLimitStep,
+ createRecordAllowedUsageStep,
+ createTrackIncomingTrafficStep,
+} from './admission-steps'
+import { LogsPipelineContext } from './logs-pipeline-context'
+import { LogsPipelineInput, createParseLogsMessageStep } from './parse-logs-message-step'
+import { LogsProcessingInput, LogsProcessingOutcome, createProcessLogMessageStep } from './process-log-message-step'
+
+type LogsItemContext = Record<string, never>
+
+type ParseOutcome = { parsed?: LogsIngestionMessage; failure?: unknown }
+
+/** Carry a failed quarantine as a result, because a thrown step latches the concurrent stage and later rejections go unawaited. */
+function createSettledParseStep(context: LogsPipelineContext): ProcessingStep<LogsPipelineInput, ParseOutcome> {
+ const parse = createParseLogsMessageStep(context)
+ return async function parseLogsMessageStep(input) {
+ try {
+ const result = await parse(input)
+ return isOkResult(result) ? ok({ parsed: result.value }) : result
+ } catch (failure) {
+ return ok({ failure })
+ }
+ }
+}
+
+function createFailParseOnErrorStep(): ChunkProcessingStep<ParseOutcome, LogsIngestionMessage> {
+ return function failParseOnErrorStep(values) {
+ const failed = values.find((value) => value.failure !== undefined)
+ if (failed) {
+ return Promise.reject(failed.failure)
+ }
+ return Promise.resolve(values.map((value) => ok(value.parsed!)))
+ }
+}
+
+/** `perElement`, not `concurrently`, because per-element instrumentation costs more than the parse. */
+export function createLogsParsePipeline(
+ context: LogsPipelineContext
+): ChunkPipeline<LogsPipelineInput, LogsIngestionMessage, LogsItemContext> {
+ return newChunkPipelineBuilder<LogsPipelineInput, LogsItemContext>()
+ .pipeChunk(perElement(createSettledParseStep(context)))
+ .pipeChunk(createFailParseOnErrorStep())
+ .build()
+}
+
+/** The gates run in the order that billing depends on. */
+export function createLogsAdmissionPipeline(
+ context: LogsPipelineContext
+): ChunkPipeline<LogsAdmissionInput, LogsAdmissionInput, LogsItemContext> {
+ return newChunkPipelineBuilder<LogsAdmissionInput, LogsItemContext>()
+ .pipeChunk(createTrackIncomingTrafficStep())
+ .pipeChunk(createBackfillGateStep(context))
+ .pipeChunk(createQuotaLimitStep(context))
+ .pipeChunk(createRateLimitStep(context))
+ .pipeChunk(createRecordAllowedUsageStep())
+ .build()
+}
+
+/** A replay counts the produced messages twice, so a failed pass must never emit its usage or tallies. */
+function createFailPassOnErrorStep<T extends LogsProcessingOutcome>(): ChunkProcessingStep<T, T> {
+ return function failPassOnErrorStep(values) {
+ const failures = values.filter((value) => value.failure !== undefined)
+ const quarantined = values.filter((value) => value.quarantined).length
+ if (failures.length > 0 || quarantined > 0) {
+ logger.error('Failed to process some log messages', {
+ quarantinedCount: quarantined,
+ failureCount: failures.length,
+ totalCount: values.length,
+ })
+ }
+ if (failures.length > 0) {
+ return Promise.reject(failures[0].failure)
+ }
+ return Promise.resolve(values.map((value) => ok(value)))
+ }
+}
+
+export function createLogsProcessingPipeline(
+ context: LogsPipelineContext
+): ChunkPipeline<LogsProcessingInput, LogsProcessingOutcome, LogsItemContext> {
+ return newChunkPipelineBuilder<LogsProcessingInput, LogsItemContext>()
+ .pipeChunk(
+ perElement(createProcessLogMessageStep(context), { maxConcurrency: MAX_CONCURRENT_MESSAGE_PROCESSES })
+ )
+ .pipeChunk(createFailPassOnErrorStep())
+ .build()
+}
+
+export async function runChunkPipeline<TIn, TOut, C extends LogsItemContext>(
+ pipeline: ChunkPipeline<TIn, TOut, C>,
+ values: TIn[]
+): Promise<TOut[]> {
+ if (values.length === 0) {
+ return []
+ }
+ const elements: OkResultWithContext<TIn, C>[] = values.map((value) => createOkContext(value, {} as C))
+ pipeline.feed(elements)
+ const out: TOut[] = []
+ let chunk
+ while ((chunk = await pipeline.next()) !== null) {
+ for (const element of chunk) {
+ if (isOkResult(element.result)) {
+ out.push(element.result.value)
+ }
+ }
+ }
+ return out
+}
diff --git a/nodejs/src/logs/framework/parse-logs-message-step.ts b/nodejs/src/logs/framework/parse-logs-message-step.ts
new file mode 100644
index 000000000000..edaaf048ec74
--- /dev/null
+++ b/nodejs/src/logs/framework/parse-logs-message-step.ts
@@ -0,0 +1,117 @@
+import { Message } from 'node-rdkafka'
+
+import { parseKafkaHeaders } from '~/common/kafka/consumer'
+import { isDevEnv } from '~/common/utils/env-utils'
+import { logger } from '~/common/utils/logger'
+import { drop, ok } from '~/ingestion/framework/results'
+import { ProcessingStep } from '~/ingestion/framework/steps'
+import type { Team } from '~/types'
+
+import { recordLogMessageDropped } from '../ingestion-otel-metrics'
+import { INTERNAL_LOGS_HEADER } from '../internal-logs'
+import {
+ logMessageDroppedCounter,
+ logsRecordsBytesExceedPayloadCounter,
+ parseMinTimestampHeader,
+ parseSizeHeader,
+} from '../logs-ingestion-consumer'
+import type { LogsIngestionMessage } from '../types'
+import { LogsPipelineContext, quarantineLogMessage } from './logs-pipeline-context'
+
+export type LogsPipelineInput = { message: Message }
+
+function countDropped(reason: string): void {
+ logMessageDroppedCounter.inc({ reason, team_id: 'unknown' })
+ recordLogMessageDropped(reason, 'unknown')
+}
+
+/** Unusable size headers quarantine the message, because they fail the same way on every redelivery. */
+export function createParseLogsMessageStep<T extends LogsPipelineInput>(
+ context: LogsPipelineContext
+): ProcessingStep<T, LogsIngestionMessage> {
+ const { deps, internalLogsVerifier } = context
+ return async function parseLogsMessageStep(input) {
+ const message = input.message
+ try {
+ const headers = parseKafkaHeaders(message.headers)
+ const token = headers.token
+
+ if (!token) {
+ logger.warn('missing_token')
+ countDropped('missing_token')
+ return drop('missing_token')
+ }
+
+ let team: Team | null
+ try {
+ if (isDevEnv() && token === 'phc_local') {
+ team = await deps.teamManager.getTeam(1)
+ } else {
+ team = await deps.teamManager.getTeamByToken(token)
+ }
+ } catch (e) {
+ logger.error('team_lookup_error', { error: e })
+ countDropped('team_lookup_error')
+ return drop('team_lookup_error')
+ }
+
+ if (!team) {
+ // capture-logs has no Postgres access, so an unknown token is a client-input drop found only here.
+ logger.warn('team_not_found', { token_with_no_team: token })
+ countDropped('team_not_found')
+ return drop('team_not_found')
+ }
+
+ const bytesUncompressed = parseSizeHeader(headers.bytes_uncompressed)
+ const bytesUncompressedRecords = parseSizeHeader(headers.bytes_uncompressed_records)
+ const bytesCompressed = parseSizeHeader(headers.bytes_compressed)
+ const recordCount = parseSizeHeader(headers.record_count)
+
+ if (
+ bytesUncompressed === null ||
+ bytesUncompressedRecords === null ||
+ bytesCompressed === null ||
+ recordCount === null
+ ) {
+ logger.error('🔴', 'logs_ingestion_invalid_size_headers', {
+ teamId: team.id,
+ partition: message.partition,
+ offset: message.offset,
+ })
+ await quarantineLogMessage(context, message, new Error('invalid_size_headers'), {
+ token,
+ teamId: team.id,
+ })
+ return drop('invalid_size_headers')
+ }
+
+ if (bytesUncompressedRecords > bytesUncompressed) {
+ // Billing can switch to records-based bytes only if the records sum never exceeds the payload size.
+ logsRecordsBytesExceedPayloadCounter.inc({ team_id: team.id.toString() })
+ }
+
+ return ok({
+ token,
+ message,
+ teamId: team.id,
+ bytesUncompressed,
+ bytesUncompressedRecords,
+ bytesCompressed,
+ recordCount,
+ minTimestampMicros: parseMinTimestampHeader(headers.min_timestamp),
+ backfillRequested: headers.backfill_days !== undefined,
+ internalLogs: internalLogsVerifier.verify(headers[INTERNAL_LOGS_HEADER], team.id, headers.created_at),
+ })
+ } catch (e) {
+ // A failed quarantine of bad size headers lands here, and a second DLQ failure fails the batch.
+ logger.error('🔴', 'logs_ingestion_parse_error', {
+ error: String(e),
+ partition: message.partition,
+ offset: message.offset,
+ })
+ countDropped('parse_error')
+ await quarantineLogMessage(context, message, e, {})
+ return drop('parse_error')
+ }
+ }
+}
diff --git a/nodejs/src/logs/framework/process-log-message-step.ts b/nodejs/src/logs/framework/process-log-message-step.ts
new file mode 100644
index 000000000000..840d9b447ba6
--- /dev/null
+++ b/nodejs/src/logs/framework/process-log-message-step.ts
@@ -0,0 +1,303 @@
+import { parseKafkaHeaders } from '~/common/kafka/consumer'
+import { recordPiiReplacements } from '~/common/metrics/otel-metrics'
+import { instrumentFn } from '~/common/tracing/tracing-utils'
+import { DependencyUnavailableError } from '~/common/utils/db/error'
+import { logger } from '~/common/utils/logger'
+import { ok } from '~/ingestion/framework/results'
+import { ProcessingStep } from '~/ingestion/framework/steps'
+
+import { recordLogMessageDropped } from '../ingestion-otel-metrics'
+import { INTERNAL_SOURCE_HEADER, withoutInternalLogsHeaders } from '../internal-logs'
+import { type PiiScrubStats } from '../log-pii-scrub'
+import { type LogRecord, sniffJsonLogAttributes } from '../log-record-avro'
+import {
+ DEFAULT_USAGE_STATS,
+ UsageStatsByTeam,
+ billingByteReductionForDrops,
+ logMessageDroppedCounter,
+ logsBillingBytesCreditedCounter,
+ logsBillingProrateDivergenceHistogram,
+ logsBillingRecordsCreditedCounter,
+ logsDropBatchRowsHistogram,
+ logsDropFractionHistogram,
+} from '../logs-ingestion-consumer'
+import { rethrowKafkaOutageAsDependency } from '../logs-produce-errors'
+import type { CompiledMetricRule } from '../metrics-rules/compile-metric-rules'
+import { withDefaultMetricRules } from '../metrics-rules/default-metric-rules'
+import { type BatchTallies, createBatchTallies, tallyRecords } from '../metrics-rules/tally'
+import { LOGS_OUTPUT } from '../outputs/outputs'
+import { RETENTION_EXPIRED_STAGE_NAME } from '../retention/retention-expired-stage'
+import type { TransformationBatchBudget } from '../transformations/logs-transformer.service'
+import type { LogsIngestionMessage } from '../types'
+import {
+ LogsPipelineContext,
+ appliesTeamLogRules,
+ isUnbilledInternal,
+ quarantineLogMessage,
+ retryOnDependencyUnavailable,
+ teamIdMatchesCsv,
+} from './logs-pipeline-context'
+import { resolveLogBuffer } from './resolve-log-buffer'
+
+export type MetricRuleBatchState = { token: string; rules: CompiledMetricRule[]; tallies: BatchTallies }
+export type MetricTalliesByTeam = Map<number, MetricRuleBatchState>
+
+export type LogsProcessingPass = {
+ usage: UsageStatsByTeam
+ metricTallies: MetricTalliesByTeam
+ transformationBudget?: TransformationBatchBudget
+}
+
+export type LogsProcessingInput = LogsIngestionMessage & { pass: LogsProcessingPass }
+
+/** A message never fails its pass on its own: `failure` fails the pass once every message has settled. */
+export type LogsProcessingOutcome = LogsProcessingInput & { quarantined: boolean; failure?: unknown }
+
+function canEmitMetricRules(context: LogsPipelineContext): boolean {
+ return !context.gates.metricRulesKillswitch && !!context.deps.metricRulesCache && !!context.deps.metricsEmitter
+}
+
+async function getMetricRuleBatchState(
+ context: LogsPipelineContext,
+ byTeam: MetricTalliesByTeam,
+ message: LogsIngestionMessage
+): Promise<MetricRuleBatchState | null> {
+ if (!appliesTeamLogRules(message)) {
+ return null
+ }
+ const teamRulesEnabled =
+ canEmitMetricRules(context) && teamIdMatchesCsv(context.gates.metricRulesEnabledTeams, message.teamId)
+ const defaultRulesEnabled =
+ canEmitMetricRules(context) && teamIdMatchesCsv(context.gates.defaultMetricRulesEnabledTeams, message.teamId)
+ if (!teamRulesEnabled && !defaultRulesEnabled) {
+ return null
+ }
+ let state = byTeam.get(message.teamId)
+ if (!state) {
+ let rules: CompiledMetricRule[] = []
+ try {
+ if (teamRulesEnabled) {
+ const all = await context.deps.metricRulesCache!.getCompiledRules(message.teamId)
+ rules = all.filter((r) => r.source === context.identity.metricRuleSource)
+ }
+ } catch (error) {
+ // Fail open, because metric rules are additive and a rules-fetch failure must never DLQ or block the log records.
+ logger.warn('[logs-metric-rules] rules fetch failed — skipping metric rules for batch', {
+ teamId: message.teamId,
+ error: String(error),
+ })
+ return null
+ }
+ if (defaultRulesEnabled) {
+ rules = withDefaultMetricRules(rules, context.identity.metricRuleSource)
+ }
+ // Re-check after the await, because replacing a concurrent message's state drops its accumulated tallies.
+ state = byTeam.get(message.teamId)
+ if (!state) {
+ state = { token: message.token, rules, tallies: createBatchTallies() }
+ byTeam.set(message.teamId, state)
+ }
+ }
+ return state.rules.length > 0 ? state : null
+}
+
+function addPiiStatsIntoUsage(
+ context: LogsPipelineContext,
+ usage: UsageStatsByTeam,
+ teamId: number,
+ delta: PiiScrubStats
+): void {
+ if (delta.piiReplacements === 0) {
+ return
+ }
+ recordPiiReplacements(context.identity.appSource, delta.piiReplacements)
+ const row = usage.get(teamId) || { ...DEFAULT_USAGE_STATS }
+ row.piiReplacements += delta.piiReplacements
+ usage.set(teamId, row)
+}
+
+type HeaderOverrides = { bytesUncompressed?: number; bytesCompressed?: number; recordCount?: number }
+
+async function processLogMessage(context: LogsPipelineContext, input: LogsProcessingInput): Promise<void> {
+ const { deps, gates, identity, usageMetrics } = context
+ const { pass } = input
+ const team = await retryOnDependencyUnavailable(context, () => deps.teamManager.getTeam(input.teamId))
+ const logsSettings = team?.logs_settings || {}
+ const jsonParse = logsSettings.json_parse_logs ?? false
+ const retentionDays = await retryOnDependencyUnavailable(context, () =>
+ context.defaultRetentionDays(input.teamId, logsSettings)
+ )
+
+ const teamStats = pass.usage.get(input.teamId)
+ if (teamStats) {
+ teamStats.retentionDays = retentionDays
+ }
+
+ if (input.message.value === null) {
+ return
+ }
+
+ const metricRuleState = await getMetricRuleBatchState(context, pass.metricTallies, input)
+ const jsonAttributeKey =
+ identity.appSource === 'logs' && teamIdMatchesCsv(gates.jsonAttributeParsingEnabledTeams, input.teamId)
+ ? logsSettings.json_parse_logs_attribute_key
+ : undefined
+ const onRecordsDecoded =
+ metricRuleState || jsonAttributeKey
+ ? (records: LogRecord[]) => {
+ if (metricRuleState) {
+ const nowMs = Date.now()
+ tallyRecords(metricRuleState.rules, records, metricRuleState.tallies, nowMs, (ruleId, labelKey) =>
+ context.metricSeriesAdmission.admit(input.teamId, ruleId, labelKey, nowMs)
+ )
+ }
+ if (jsonAttributeKey) {
+ sniffJsonLogAttributes(records, jsonAttributeKey, input.teamId)
+ }
+ }
+ : undefined
+
+ const resolved = await instrumentFn(
+ {
+ key: 'logsIngestion.sampling.resolveLogMessageBuffer',
+ measureTime: false,
+ sendException: false,
+ getLoggingContext: () => ({ team_id: input.teamId, inbound_bytes: input.message.value?.length ?? 0 }),
+ },
+ async () =>
+ resolveLogBuffer(
+ context,
+ input,
+ {
+ ...logsSettings,
+ json_parse_logs_attribute_key:
+ identity.appSource === 'logs' &&
+ teamIdMatchesCsv(gates.jsonAttributeExtractionEnabledTeams, input.teamId)
+ ? logsSettings.json_parse_logs_attribute_key
+ : undefined,
+ },
+ retentionDays,
+ onRecordsDecoded,
+ pass.transformationBudget
+ )
+ )
+
+ usageMetrics.queueUsageMetric(
+ input.teamId,
+ 'records_dropped_retention_expired',
+ resolved.recordsDroppedByStage.get(RETENTION_EXPIRED_STAGE_NAME) ?? 0
+ )
+
+ const overrides: HeaderOverrides = {}
+
+ // `bytesReceived` stays gross, while `bytesAllowed` and `recordsAllowed` count only the rows that survive drop rules.
+ if (resolved.recordsDropped > 0) {
+ const header = input.bytesUncompressed
+ const contentCredit = billingByteReductionForDrops(
+ header,
+ resolved.contentBytesDropped,
+ resolved.contentBytesTotal
+ )
+ // A record-weighted credit that diverges from the content-weighted one shows a size-skewed batch.
+ const recordCredit = billingByteReductionForDrops(header, resolved.recordsDropped, input.recordCount)
+
+ if (!isUnbilledInternal(input)) {
+ const teamIdLabel = input.teamId.toString()
+ if (contentCredit > 0) {
+ logsBillingBytesCreditedCounter.inc({ team_id: teamIdLabel }, contentCredit)
+ }
+ logsBillingRecordsCreditedCounter.inc({ team_id: teamIdLabel }, resolved.recordsDropped)
+ }
+ if (input.recordCount > 0) {
+ logsDropBatchRowsHistogram.observe(input.recordCount)
+ }
+ if (resolved.contentBytesTotal > 0) {
+ logsDropFractionHistogram.observe(Math.min(1, resolved.contentBytesDropped / resolved.contentBytesTotal))
+ }
+ if (header > 0) {
+ logsBillingProrateDivergenceHistogram.observe(Math.abs(contentCredit - recordCredit) / header)
+ }
+
+ if (gates.billingProrateEnabled) {
+ const billingRow = pass.usage.get(input.teamId)
+ if (billingRow) {
+ billingRow.bytesAllowed = Math.max(0, billingRow.bytesAllowed - contentCredit)
+ billingRow.recordsAllowed = Math.max(0, billingRow.recordsAllowed - resolved.recordsDropped)
+ }
+
+ const compressedCredit = billingByteReductionForDrops(
+ input.bytesCompressed,
+ resolved.contentBytesDropped,
+ resolved.contentBytesTotal
+ )
+ overrides.bytesUncompressed = Math.max(0, header - contentCredit)
+ overrides.bytesCompressed = Math.max(0, input.bytesCompressed - compressedCredit)
+ overrides.recordCount = Math.max(0, input.recordCount - resolved.recordsDropped)
+ }
+ }
+
+ addPiiStatsIntoUsage(context, pass.usage, input.teamId, resolved.pii)
+ usageMetrics.queueByRule(input.teamId, 'sampling_records_dropped_by_rule', resolved.recordsDroppedByRuleId)
+ usageMetrics.queueByRule(input.teamId, 'bytes_dropped_by_rule', resolved.bytesDroppedByRuleId)
+
+ if (resolved.outcome === 'all_dropped') {
+ logMessageDroppedCounter.inc({ reason: resolved.reason, team_id: input.teamId.toString() }, 1)
+ recordLogMessageDropped(resolved.reason, input.teamId.toString())
+ return
+ }
+
+ await retryOnDependencyUnavailable(context, () =>
+ deps.outputs
+ .queueMessages(LOGS_OUTPUT, [
+ {
+ value: resolved.processedValue,
+ key: null,
+ headers: {
+ ...withoutInternalLogsHeaders(parseKafkaHeaders(input.message.headers)),
+ ...(input.internalLogs !== undefined
+ ? { [INTERNAL_SOURCE_HEADER]: input.internalLogs.source }
+ : {}),
+ token: input.token,
+ team_id: input.teamId.toString(),
+ 'json-parse': jsonParse.toString(),
+ 'retention-days': retentionDays.toString(),
+ // Both topics use random partitioning, so the ingestion lag checkpoint needs the source partition.
+ source_topic: input.message.topic,
+ source_partition: input.message.partition.toString(),
+ ...(overrides.bytesUncompressed !== undefined
+ ? { bytes_uncompressed: overrides.bytesUncompressed.toString() }
+ : {}),
+ ...(overrides.bytesCompressed !== undefined
+ ? { bytes_compressed: overrides.bytesCompressed.toString() }
+ : {}),
+ ...(overrides.recordCount !== undefined
+ ? { record_count: overrides.recordCount.toString() }
+ : {}),
+ },
+ },
+ ])
+ .catch(rethrowKafkaOutageAsDependency)
+ )
+}
+
+export function createProcessLogMessageStep<T extends LogsProcessingInput>(
+ context: LogsPipelineContext
+): ProcessingStep<T, T & { quarantined: boolean; failure?: unknown }> {
+ return async function processLogMessageStep(input) {
+ try {
+ await processLogMessage(context, input)
+ return ok({ ...input, quarantined: false })
+ } catch (error) {
+ if (error instanceof DependencyUnavailableError) {
+ return ok({ ...input, quarantined: false, failure: error })
+ }
+ try {
+ // The DLQ copy records this failure, so the batch can commit past the message.
+ await quarantineLogMessage(context, input.message, error, { token: input.token, teamId: input.teamId })
+ return ok({ ...input, quarantined: true })
+ } catch (dlqError) {
+ return ok({ ...input, quarantined: false, failure: dlqError })
+ }
+ }
+ }
+}
diff --git a/nodejs/src/logs/framework/resolve-log-buffer.ts b/nodejs/src/logs/framework/resolve-log-buffer.ts
new file mode 100644
index 000000000000..f786a8c61938
--- /dev/null
+++ b/nodejs/src/logs/framework/resolve-log-buffer.ts
@@ -0,0 +1,194 @@
+import { trace } from '@opentelemetry/api'
+
+import type { LogsSettings } from '~/types'
+
+import { MESSAGE_KEYS } from '../log-pattern-mask'
+import { logsPatternForcedDecodeCounter, makePatternMaskingStage } from '../log-pattern-stage'
+import { type PiiScrubStats } from '../log-pii-scrub'
+import {
+ type LogRecord,
+ type LogRecordsTransform,
+ bufferProcessingMode,
+ processLogMessageBuffer,
+} from '../log-record-avro'
+import {
+ INTERNAL_LOGS_MIN_RETENTION_DAYS,
+ logsBytesDroppedByRuleCounter,
+ logsSamplingRecordsDroppedCounter,
+ makeTransformStage,
+} from '../logs-ingestion-consumer'
+import { type PipelineStage } from '../pipeline/log-processing-pipeline'
+import { type CompiledRetentionRuleSet } from '../retention/evaluate-retention'
+import { canHoldExpiredRow, makeRetentionExpiredStage } from '../retention/retention-expired-stage'
+import { makeRetentionStage } from '../retention/retention-stage'
+import type { CompiledRuleSet } from '../sampling/evaluate'
+import { SAMPLING_STAGE_NAME } from '../sampling/logs-sampling.service'
+import type { TransformationBatchBudget } from '../transformations/logs-transformer.service'
+import type { LogsIngestionMessage } from '../types'
+import {
+ LogsPipelineContext,
+ appliesTeamLogRules,
+ retryOnDependencyUnavailable,
+ teamIdMatchesCsv,
+} from './logs-pipeline-context'
+
+type ResolvedDrops = {
+ pii: PiiScrubStats
+ recordsDropped: number
+ recordsDroppedByStage: Map<string, number>
+ recordsDroppedByRuleId: Map<string, number>
+ bytesDroppedByRuleId: Map<string, number>
+ contentBytesDropped: number
+ contentBytesTotal: number
+}
+
+export type ResolvedLogBuffer =
+ | ({ outcome: 'produce'; processedValue: Buffer } & ResolvedDrops)
+ | ({ outcome: 'all_dropped'; reason: string } & ResolvedDrops)
+
+function isSamplingEvalEnabledForTeam(context: LogsPipelineContext, teamId: number): boolean {
+ return !context.gates.samplingKillswitch && teamIdMatchesCsv(context.gates.samplingEnabledTeams, teamId)
+}
+
+export function isRetentionEvalEnabledForTeam(context: LogsPipelineContext, teamId: number): boolean {
+ return !context.gates.retentionKillswitch && teamIdMatchesCsv(context.gates.retentionEnabledTeams, teamId)
+}
+
+function isTransformationsEnabledForTeam(context: LogsPipelineContext, teamId: number): boolean {
+ if (context.gates.transformationsKillswitch || !context.deps.logsTransformer) {
+ return false
+ }
+ return teamIdMatchesCsv(context.gates.transformationsEnabledTeams, teamId)
+}
+
+/** Logs only, because a trace record has no `body` field and would skew the log-body metrics. */
+export function isPatternMaskingEnabledForTeam(context: LogsPipelineContext, teamId: number): boolean {
+ return context.identity.appSource === 'logs' && teamIdMatchesCsv(context.gates.patternMaskingEnabledTeams, teamId)
+}
+
+async function buildRecordsTransform(
+ context: LogsPipelineContext,
+ message: LogsIngestionMessage,
+ batchBudget?: TransformationBatchBudget
+): Promise<LogRecordsTransform | undefined> {
+ const transformer = context.deps.logsTransformer
+ if (!transformer || !appliesTeamLogRules(message) || !isTransformationsEnabledForTeam(context, message.teamId)) {
+ return undefined
+ }
+ if (!(await transformer.teamHasTransformations(message.teamId))) {
+ return undefined
+ }
+ return (records) => transformer.transformRecords(message.teamId, records, batchBudget)
+}
+
+export async function resolveLogBuffer(
+ context: LogsPipelineContext,
+ message: LogsIngestionMessage,
+ logsSettings: LogsSettings,
+ defaultRetentionDays: number,
+ onRecordsDecoded?: (records: LogRecord[]) => void,
+ batchBudget?: TransformationBatchBudget
+): Promise<ResolvedLogBuffer> {
+ const { deps, gates } = context
+ const samplingCache = deps.samplingRulesCache
+ const samplingEvalEnabled = isSamplingEvalEnabledForTeam(context, message.teamId)
+ let ruleSet: CompiledRuleSet | null = null
+ if (samplingCache && samplingEvalEnabled && appliesTeamLogRules(message)) {
+ ruleSet = await samplingCache.getCompiledRuleSet(message.teamId)
+ }
+ const useSamplingPipeline = Boolean(ruleSet && ruleSet.rules.length > 0)
+ const recordsTransform = await buildRecordsTransform(context, message, batchBudget)
+
+ // With no rules, the batch `retention-days` header carries the team default and the buffer skips the decode.
+ const retentionCache = deps.retentionRulesCache
+ const retentionEvalEnabled = isRetentionEvalEnabledForTeam(context, message.teamId)
+ let retentionRuleSet: CompiledRetentionRuleSet | null = null
+ if (retentionCache && retentionEvalEnabled && appliesTeamLogRules(message)) {
+ retentionRuleSet = await retentionCache.getCompiledRuleSet(message.teamId, context.identity.retentionRuleSource)
+ }
+ const useRetention = Boolean(retentionRuleSet && retentionRuleSet.rules.length > 0)
+
+ // Retention stamping runs after sampling and transformations, so it only stamps survivors.
+ const stages: PipelineStage[] = []
+ if (useSamplingPipeline && ruleSet) {
+ stages.push(context.samplingService.makeSamplingStage(ruleSet, message.teamId, message.bytesUncompressed))
+ }
+ if (recordsTransform) {
+ stages.push(makeTransformStage(recordsTransform))
+ }
+ let shortestRetentionDays =
+ message.internalLogs !== undefined ? INTERNAL_LOGS_MIN_RETENTION_DAYS : defaultRetentionDays
+ if (useRetention && retentionRuleSet) {
+ stages.push(makeRetentionStage(retentionRuleSet, message.teamId, defaultRetentionDays))
+ shortestRetentionDays = Math.min(
+ defaultRetentionDays,
+ ...retentionRuleSet.rules.map((rule) => rule.retentionDays)
+ )
+ }
+
+ // Only a message that can hold an expired row pays the decode. The stage runs after retention to read the per-row value.
+ const nowMicros = Date.now() * 1000
+ if (canHoldExpiredRow(message.minTimestampMicros, shortestRetentionDays, nowMicros)) {
+ stages.push(makeRetentionExpiredStage(message.teamId, defaultRetentionDays, nowMicros))
+ }
+
+ // Masking runs last and forces a full decode and re-encode, so the counter prices each forced decode.
+ if (isPatternMaskingEnabledForTeam(context, message.teamId)) {
+ const modeWithoutMasking = bufferProcessingMode(logsSettings, stages.length, Boolean(onRecordsDecoded))
+ if (modeWithoutMasking !== 'decode_and_reencode') {
+ logsPatternForcedDecodeCounter.inc({ from: modeWithoutMasking })
+ }
+ const keysCache = deps.patternMessageKeysCache
+ const messageKeys = keysCache
+ ? await retryOnDependencyUnavailable(context, () => keysCache.getMessageKeys(message.teamId))
+ : MESSAGE_KEYS
+ stages.push(makePatternMaskingStage(messageKeys))
+ }
+
+ trace.getActiveSpan()?.setAttributes({
+ 'logs.sampling.killswitch': gates.samplingKillswitch,
+ 'logs.sampling.enabled_teams_configured': Boolean((gates.samplingEnabledTeams || '').trim()),
+ 'logs.sampling.enabled_teams_is_wildcard': (gates.samplingEnabledTeams || '').trim() === '*',
+ 'logs.sampling.cache_present': Boolean(samplingCache),
+ 'logs.sampling.eval_enabled_for_team': samplingEvalEnabled,
+ 'logs.sampling.compiled_rule_count': ruleSet?.rules.length ?? 0,
+ 'logs.transformations.enabled_for_team': Boolean(recordsTransform),
+ 'logs.retention.cache_present': Boolean(retentionCache),
+ 'logs.retention.eval_enabled_for_team': retentionEvalEnabled,
+ 'logs.retention.compiled_rule_count': retentionRuleSet?.rules.length ?? 0,
+ 'logs.retention.use_retention': useRetention,
+ 'logs.sampling.pipeline': useSamplingPipeline
+ ? 'decode_sample_encode'
+ : recordsTransform
+ ? 'decode_transform_encode'
+ : 'passthrough',
+ })
+
+ const { value, pii, drops } = await processLogMessageBuffer(message.message.value!, logsSettings, {
+ onRecordsDecoded,
+ stages,
+ })
+
+ const recordsDroppedBySampling = drops.recordsDroppedByStage.get(SAMPLING_STAGE_NAME) ?? 0
+ if (recordsDroppedBySampling > 0) {
+ logsSamplingRecordsDroppedCounter.inc({ team_id: message.teamId.toString() }, recordsDroppedBySampling)
+ }
+ if (drops.bytesDropped > 0) {
+ logsBytesDroppedByRuleCounter.inc({ team_id: message.teamId.toString() }, drops.bytesDropped)
+ }
+
+ const resolvedDrops: ResolvedDrops = {
+ pii,
+ recordsDropped: drops.recordsDropped,
+ recordsDroppedByStage: drops.recordsDroppedByStage,
+ recordsDroppedByRuleId: drops.recordsDroppedByRuleId,
+ bytesDroppedByRuleId: drops.bytesDroppedByRuleId,
+ contentBytesDropped: drops.contentBytesDropped,
+ contentBytesTotal: drops.contentBytesTotal,
+ }
+ if (value === null) {
+ const reason = drops.droppedBy ? `${drops.droppedBy}_all_dropped` : 'empty_batch'
+ return { outcome: 'all_dropped', reason, ...resolvedDrops }
+ }
+ return { outcome: 'produce', processedValue: value, ...resolvedDrops }
+}
diff --git a/nodejs/src/logs/framework/traces-ingestion-framework-consumer.ts b/nodejs/src/logs/framework/traces-ingestion-framework-consumer.ts
new file mode 100644
index 000000000000..64e68c16e471
--- /dev/null
+++ b/nodejs/src/logs/framework/traces-ingestion-framework-consumer.ts
@@ -0,0 +1,61 @@
+import type { LogsSettings } from '~/types'
+
+import { LogsIngestionConsumerConfig, TracesIngestionConsumerConfig } from '../config'
+import type { MetricRuleSource } from '../metrics-rules/compile-metric-rules'
+import type { RetentionRuleSource } from '../retention/compile-retention-rules'
+import { DEFAULT_TRACES_RETENTION_DAYS, TracingConfigCache } from '../retention/tracing-config-cache'
+import type { TracesIngestionConsumerDeps } from '../traces-ingestion-consumer'
+import { LogsIngestionFrameworkConsumer } from './logs-ingestion-framework-consumer'
+
+/** Keep the config overrides in step with `TracesIngestionConsumer`. */
+export class TracesIngestionFrameworkConsumer extends LogsIngestionFrameworkConsumer {
+ protected override name = 'TracesIngestionConsumer'
+ // Traces bill against the shared logs bucket, so the inherited quota resource stays.
+ protected override appSource: 'logs' | 'traces' = 'traces'
+ protected override metricRuleSource: MetricRuleSource = 'spans'
+ protected override retentionRuleSource: RetentionRuleSource = 'spans'
+ private tracingConfigCache?: TracingConfigCache
+
+ constructor(
+ config: LogsIngestionConsumerConfig & TracesIngestionConsumerConfig,
+ deps: TracesIngestionConsumerDeps
+ ) {
+ super(
+ config,
+ deps,
+ {
+ LOGS_INGESTION_CONSUMER_GROUP_ID: config.TRACES_INGESTION_CONSUMER_GROUP_ID,
+ LOGS_INGESTION_CONSUMER_CONSUME_TOPIC: config.TRACES_INGESTION_CONSUMER_CONSUME_TOPIC,
+ LOGS_REDIS_HOST: config.TRACES_REDIS_HOST,
+ LOGS_REDIS_PORT: config.TRACES_REDIS_PORT,
+ LOGS_REDIS_PASSWORD: config.TRACES_REDIS_PASSWORD,
+ LOGS_REDIS_TLS: config.TRACES_REDIS_TLS,
+ LOGS_LIMITER_ENABLED_TEAMS: config.TRACES_LIMITER_ENABLED_TEAMS,
+ LOGS_LIMITER_DISABLED_FOR_TEAMS: config.TRACES_LIMITER_DISABLED_FOR_TEAMS,
+ LOGS_LIMITER_BUCKET_SIZE_KB: config.TRACES_LIMITER_BUCKET_SIZE_KB,
+ LOGS_LIMITER_REFILL_RATE_KB_PER_SECOND: config.TRACES_LIMITER_REFILL_RATE_KB_PER_SECOND,
+ LOGS_LIMITER_TTL_SECONDS: config.TRACES_LIMITER_TTL_SECONDS,
+ LOGS_LIMITER_TEAM_BUCKET_SIZE_KB: config.TRACES_LIMITER_TEAM_BUCKET_SIZE_KB,
+ LOGS_LIMITER_TEAM_REFILL_RATE_KB_PER_SECOND: config.TRACES_LIMITER_TEAM_REFILL_RATE_KB_PER_SECOND,
+ LOGS_METRICS_RULES_ENABLED_TEAMS: config.TRACES_METRICS_RULES_ENABLED_TEAMS,
+ LOGS_METRICS_RULES_KILLSWITCH: config.TRACES_METRICS_RULES_KILLSWITCH,
+ LOGS_METRICS_RULES_EXPORT_URL: config.TRACES_METRICS_RULES_EXPORT_URL,
+ LOGS_DEFAULT_METRIC_RULES_ENABLED_TEAMS: config.TRACES_DEFAULT_METRIC_RULES_ENABLED_TEAMS,
+ LOGS_RETENTION_ENABLED_TEAMS: config.TRACES_RETENTION_ENABLED_TEAMS,
+ LOGS_RETENTION_KILLSWITCH: config.TRACES_RETENTION_KILLSWITCH,
+ // Internal batches are logs only, so a signed traces batch is still billed.
+ LOGS_INTERNAL_INGESTION_SIGNING_KEYS: '',
+ LOGS_INTERNAL_INGESTION_ENABLED_TEAMS: '',
+ },
+ 'traces-rate-limiter'
+ )
+ this.tracingConfigCache = deps.tracingConfigCache
+ }
+
+ protected override async defaultRetentionDays(teamId: number, _logsSettings: LogsSettings): Promise<number> {
+ if (!this.tracingConfigCache) {
+ return DEFAULT_TRACES_RETENTION_DAYS
+ }
+ return await this.tracingConfigCache.getRetentionDays(teamId)
+ }
+}
diff --git a/nodejs/src/logs/logs-ingestion-consumer.ts b/nodejs/src/logs/logs-ingestion-consumer.ts
index b1717e4504e8..cd409df799c4 100644
--- a/nodejs/src/logs/logs-ingestion-consumer.ts
+++ b/nodejs/src/logs/logs-ingestion-consumer.ts
@@ -111,12 +111,12 @@ export interface LogsIngestionConsumerDeps {
export const DEFAULT_LOGS_RETENTION_DAYS = 14
// A producer can stamp internal rows with less retention than the team default.
-const INTERNAL_LOGS_MIN_RETENTION_DAYS = 1
+export const INTERNAL_LOGS_MIN_RETENTION_DAYS = 1
/** Retention day counts that get their own per-tier usage metric. */
const RETENTION_USAGE_TIERS = new Set([14, 30, 90])
-function retentionBytesMetricName(retentionDays: number): string | null {
+export function retentionBytesMetricName(retentionDays: number): string | null {
return RETENTION_USAGE_TIERS.has(retentionDays) ? `bytes_ingested_retention_${retentionDays}d` : null
}
@@ -134,7 +134,7 @@ export type UsageStats = {
}
/** `raw` is a comma-separated team-ID list, `*` for all teams, or empty for none. */
-function teamIdMatchesCsv(raw: string, teamId: number): boolean {
+export function teamIdMatchesCsv(raw: string, teamId: number): boolean {
const trimmed = (raw || '').trim()
if (!trimmed) {
return false
@@ -192,7 +192,7 @@ export function parseMinTimestampHeader(raw: string | undefined): number | undef
return Number.isSafeInteger(parsed) ? parsed : undefined
}
-const DEFAULT_USAGE_STATS: UsageStats = {
+export const DEFAULT_USAGE_STATS: UsageStats = {
bytesReceived: 0,
recordsReceived: 0,
bytesAllowed: 0,
@@ -207,7 +207,7 @@ const DEFAULT_USAGE_STATS: UsageStats = {
export type UsageStatsByTeam = Map<number, UsageStats>
/** Cap concurrent per-message processing within a single kafka batch. */
-const MAX_CONCURRENT_MESSAGE_PROCESSES = 50
+export const MAX_CONCURRENT_MESSAGE_PROCESSES = 50
export const logMessageDroppedCounter = new Counter({
name: 'logs_ingestion_message_dropped_count',
@@ -371,7 +371,7 @@ export function billingByteReductionForDrops(headerBytes: number, bytesDropped:
* attributed — so it reports no per-rule accounting, just the `transformations` tag when it removed
* any record.
*/
-function makeTransformStage(transform: LogRecordsTransform): PipelineStage {
+export function makeTransformStage(transform: LogRecordsTransform): PipelineStage {
return {
kind: 'filter',
name: 'transformations',
diff --git a/nodejs/src/servers/ingestion-logs-server.ts b/nodejs/src/servers/ingestion-logs-server.ts
index 2f36c279ce3c..1709bdc5c8c2 100644
--- a/nodejs/src/servers/ingestion-logs-server.ts
+++ b/nodejs/src/servers/ingestion-logs-server.ts
@@ -14,7 +14,7 @@ import {
LogsIngestionOutputsConfig,
getDefaultLogsIngestionOutputsConfig,
} from '~/logs/config'
-import { LogsIngestionConsumer } from '~/logs/logs-ingestion-consumer'
+import { createLogsIngestionConsumer } from '~/logs/framework/create-consumer'
import { MetricRulesCache } from '~/logs/metrics-rules/metric-rules-cache'
import { LogsMetricsEmitter } from '~/logs/metrics-rules/metrics-emitter'
import { createProducerRegistry } from '~/logs/outputs/producer-registry'
@@ -163,7 +163,7 @@ export class IngestionLogsServer implements NodeServer {
unit: 'bytes',
isTeamEnabled: usageReportTeamMatcher(this.config),
})
- const consumer = new LogsIngestionConsumer(this.config, {
+ const consumer = createLogsIngestionConsumer(this.config, {
teamManager,
quotaLimiting,
outputs,
diff --git a/nodejs/src/servers/ingestion-traces-server.ts b/nodejs/src/servers/ingestion-traces-server.ts
index 114d03cca304..cd528c9ef678 100644
--- a/nodejs/src/servers/ingestion-traces-server.ts
+++ b/nodejs/src/servers/ingestion-traces-server.ts
@@ -14,6 +14,7 @@ import {
TracesIngestionConsumerConfig,
getDefaultLogsIngestionOutputsConfig,
} from '~/logs/config'
+import { createTracesIngestionConsumer } from '~/logs/framework/create-consumer'
import { MetricRulesCache } from '~/logs/metrics-rules/metric-rules-cache'
import { LogsMetricsEmitter } from '~/logs/metrics-rules/metrics-emitter'
import { createProducerRegistry } from '~/logs/outputs/producer-registry'
@@ -27,7 +28,6 @@ import {
import { createTracesOutputsRegistry } from '~/logs/outputs/registry'
import { RetentionRulesCache } from '~/logs/retention/retention-rules-cache'
import { TracingConfigCache } from '~/logs/retention/tracing-config-cache'
-import { TracesIngestionConsumer } from '~/logs/traces-ingestion-consumer'
import { CommonConfig } from '../common/config'
import { DatabaseConnectionConfig, KafkaBrokerConfig, RedisConnectionsConfig } from '../ingestion/config'
@@ -134,7 +134,7 @@ export class IngestionTracesServer implements NodeServer {
unit: 'bytes',
isTeamEnabled: usageReportTeamMatcher(this.config),
})
- const consumer = new TracesIngestionConsumer(this.config, {
+ const consumer = createTracesIngestionConsumer(this.config, {
teamManager,
quotaLimiting,
outputs,