:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(logs): add a framework logs consumer behind an env flag (#127605)
Frank Hamand836da0a3a26d
diff --git a/nodejs/src/ingestion/framework/builders/index.ts b/nodejs/src/ingestion/framework/builders/index.tsindex 3189676e386a..98a8723d38de 100644--- a/nodejs/src/ingestion/framework/builders/index.ts+++ b/nodejs/src/ingestion/framework/builders/index.ts@@ -8,3 +8,4 @@ export type { RetryOptions } from '~/ingestion/framework/retry'export type { ChunkProcessingStep } from '~/ingestion/framework/base-chunk-pipeline'export { BranchingPipelineBuilder, PipelineBuilder, StartPipelineBuilder } from './pipeline-builders'export { newChunkPipelineBuilder, newBatchingPipeline, newPipelineBuilder } from './helpers'+export { perElement } from '~/ingestion/framework/per-element-step'diff --git a/nodejs/src/ingestion/framework/per-element-step.test.ts b/nodejs/src/ingestion/framework/per-element-step.test.tsnew file mode 100644index 000000000000..f010e187460d--- /dev/null+++ b/nodejs/src/ingestion/framework/per-element-step.test.ts@@ -0,0 +1,30 @@+import { perElement } from './per-element-step'+import { ok } from './results'++describe('perElement', () => {+ it('keeps input order and never runs more than maxConcurrency steps at once', async () => {+ let running = 0+ let peak = 0+ const started: number[] = []+ const step = perElement(+ async (value: number) => {+ started.push(value)+ running+++ peak = Math.max(peak, running)+ // Later values wait fewer ticks and finish first, so output order cannot follow completion order.+ for (let tick = 0; tick < 8 - value; tick++) {+ await new Promise((resolve) => setImmediate(resolve))+ }+ running--+ return ok(value * 10)+ },+ { maxConcurrency: 3 }+ )++ const results = await step([1, 2, 3, 4, 5, 6, 7])++ expect(results).toEqual([1, 2, 3, 4, 5, 6, 7].map((value) => ok(value * 10)))+ expect(peak).toBe(3)+ expect(started).toEqual([1, 2, 3, 4, 5, 6, 7])+ })+})diff --git a/nodejs/src/ingestion/framework/per-element-step.ts b/nodejs/src/ingestion/framework/per-element-step.tsnew file mode 100644index 000000000000..086306cc6ff4--- /dev/null+++ b/nodejs/src/ingestion/framework/per-element-step.ts@@ -0,0 +1,16 @@+import pLimit from 'p-limit'++import { ChunkProcessingStep } from './base-chunk-pipeline'+import { ProcessingStep } from './steps'++// One chunk step, so instrumentation runs per chunk. A throw rejects the chunk but does not cancel the other values.+export function perElement<T, U, R extends string = never>(+ step: ProcessingStep<T, U, R>,+ options: { maxConcurrency?: number } = {}+): ChunkProcessingStep<T, U, R> {+ const limit = options.maxConcurrency !== undefined ? pLimit(options.maxConcurrency) : null+ const chunkStep: ChunkProcessingStep<T, U, R> = (values) =>+ Promise.all(values.map((value) => (limit ? limit(() => step(value)) : step(value))))+ Object.defineProperty(chunkStep, 'name', { value: step.name })+ return chunkStep+}diff --git a/nodejs/src/ingestion/pipelines/metrics/metrics-ingestion-pipeline.ts b/nodejs/src/ingestion/pipelines/metrics/metrics-ingestion-pipeline.tsindex 6d87a0a17333..248dfff3b12d 100644--- a/nodejs/src/ingestion/pipelines/metrics/metrics-ingestion-pipeline.ts+++ b/nodejs/src/ingestion/pipelines/metrics/metrics-ingestion-pipeline.ts@@ -6,14 +6,14 @@ import { QuotaLimiting } from '~/common/services/quota-limiting.service'import { PromiseScheduler } from '~/common/utils/promise-scheduler'import { TeamManager } from '~/common/utils/team-manager'import { BatchingContext, BatchingPipeline } from '~/ingestion/framework/batching-pipeline'-import { newBatchingPipeline } from '~/ingestion/framework/builders'+import { newBatchingPipeline, perElement } from '~/ingestion/framework/builders'import { aggregateKafkaDebugContexts, createBatch } from '~/ingestion/framework/helpers'import { PipelineConfig } from '~/ingestion/framework/result-handling-pipeline'import { MetricsUsageAccumulator, MetricsUsageBatchContext } from './metrics-usage'import { createKeepMetricsUsageStep, createMetricsUsageBeforeBatchStep } from './metrics-usage-steps'import { MetricsOutput } from './outputs/outputs'-import { createPrepareMetricsMessageStep, perMessage } from './prepare-metrics-message-step'+import { createPrepareMetricsMessageStep } from './prepare-metrics-message-step'import { createProduceMetricsStep } from './produce-metrics-step'import { createRateLimitMetricsStep } from './rate-limit-metrics-step'import { MetricsRateLimiterService } from './services/metrics-rate-limiter.service'@@ -57,9 +57,9 @@ export function createMetricsIngestionPipeline(config: MetricsIngestionPipelineCbatch.messageAware((b) =>b- .pipeChunk(perMessage(createPrepareMetricsMessageStep(teamManager, quotaLimiting)))+ .pipeChunk(perElement(createPrepareMetricsMessageStep(teamManager, quotaLimiting))).pipeChunk(createRateLimitMetricsStep(rateLimiter))- .pipeChunk(perMessage(createProduceMetricsStep(outputs)))+ .pipeChunk(perElement(createProduceMetricsStep(outputs)))).handleResults(pipelineConfig).handleSideEffects(promiseScheduler, sideEffects),diff --git a/nodejs/src/ingestion/pipelines/metrics/prepare-metrics-message-step.ts b/nodejs/src/ingestion/pipelines/metrics/prepare-metrics-message-step.tsindex 8a5d6571841a..348e4bc95183 100644--- a/nodejs/src/ingestion/pipelines/metrics/prepare-metrics-message-step.ts+++ b/nodejs/src/ingestion/pipelines/metrics/prepare-metrics-message-step.ts@@ -2,7 +2,6 @@ import { Message } from 'node-rdkafka'import { QuotaLimiting } from '~/common/services/quota-limiting.service'import { TeamManager } from '~/common/utils/team-manager'-import { ChunkProcessingStep } from '~/ingestion/framework/builders'import { isOkResult } from '~/ingestion/framework/results'import { ProcessingStep } from '~/ingestion/framework/steps'@@ -15,15 +14,6 @@ import { createResolveMetricsTeamStep } from './resolve-metrics-team-step'const RESOLVE_TEAM_RETRY = { tries: 3, sleepMs: 100, name: 'resolve_metrics_team' }-/** The framework instruments a chunk step once per chunk, and per-message instrumentation costs more than reading the headers. */-export function perMessage<T, U, R extends string = never>(- step: ProcessingStep<T, U, R>-): ChunkProcessingStep<T, U, R> {- const chunkStep: ChunkProcessingStep<T, U, R> = (values) => Promise.all(values.map((value) => step(value)))- Object.defineProperty(chunkStep, 'name', { value: step.name })- return chunkStep-}-export function createPrepareMetricsMessageStep<T extends { message: Message; usage: MetricsUsageAccumulator }>(teamManager: Pick<TeamManager, 'getTeam' | 'getTeamByToken'>,quotaLimiting: Pick<QuotaLimiting, 'isTeamTokenQuotaLimited'>diff --git a/nodejs/src/logs/config.ts b/nodejs/src/logs/config.tsindex a022912ba191..4078d6e974a7 100644--- a/nodejs/src/logs/config.ts+++ b/nodejs/src/logs/config.ts@@ -94,6 +94,7 @@ export type LogsIngestionConsumerConfig = {* (shadow mode) — billing is unchanged.*/LOGS_BILLING_PRORATE_ENABLED: boolean+ LOGS_INGESTION_USE_FRAMEWORK: boolean/** Comma-separated team IDs, or `*` for all teams, or empty (default) to disable hog log transformations entirely. */LOGS_TRANSFORMATIONS_ENABLED_TEAMS: string/** When `true`, hog log transformations are skipped regardless of the allowlist. */@@ -152,6 +153,7 @@ export function getDefaultLogsIngestionConsumerConfig(): LogsIngestionConsumerCoLOGS_JSON_ATTRIBUTE_PARSING_ENABLED_TEAMS: '',LOGS_JSON_ATTRIBUTE_EXTRACTION_ENABLED_TEAMS: '',LOGS_BILLING_PRORATE_ENABLED: false,+ LOGS_INGESTION_USE_FRAMEWORK: false,LOGS_TRANSFORMATIONS_ENABLED_TEAMS: '',LOGS_TRANSFORMATIONS_KILLSWITCH: false,LOGS_TRANSFORMATIONS_HOG_TIMEOUT_MS: 10,@@ -196,6 +198,7 @@ export type TracesIngestionConsumerConfig = {TRACES_RETENTION_ENABLED_TEAMS: string/** When `true`, span retention rules are never evaluated (spans keep the team default via the batch header). */TRACES_RETENTION_KILLSWITCH: boolean+ TRACES_INGESTION_USE_FRAMEWORK: booleanREDIS_URL: stringREDIS_POOL_MIN_SIZE: numberREDIS_POOL_MAX_SIZE: number@@ -229,6 +232,7 @@ export function getDefaultTracesIngestionConsumerConfig(): TracesIngestionConsum// Same rollout shape as the logs retention rules: on locally, off in prod until enabled per team.TRACES_RETENTION_ENABLED_TEAMS: isProdEnv() ? '' : '*',TRACES_RETENTION_KILLSWITCH: false,+ TRACES_INGESTION_USE_FRAMEWORK: false,// Overlapping fields with CommonConfig, included for standalone usage// ok to connect to localhost over plaintext// nosemgrep: trailofbits.generic.redis-unencrypted-transport.redis-unencrypted-transportdiff --git a/nodejs/src/logs/framework/admission-steps.ts b/nodejs/src/logs/framework/admission-steps.tsnew file mode 100644index 000000000000..aa96c2aa6241--- /dev/null+++ b/nodejs/src/logs/framework/admission-steps.ts@@ -0,0 +1,212 @@+import { ChunkProcessingStep } from '~/ingestion/framework/base-chunk-pipeline'+import { PipelineResult, drop, ok } from '~/ingestion/framework/results'++import {+ recordLogMessageDropped,+ recordLogsAllowed,+ recordLogsDropped,+ recordLogsReceived,+} from '../ingestion-otel-metrics'+import {+ DEFAULT_USAGE_STATS,+ UsageStats,+ UsageStatsByTeam,+ logMessageDroppedCounter,+ logsBytesAllowedCounter,+ logsBytesAllowedRecordsCounter,+ logsBytesDroppedCounter,+ logsBytesReceivedCounter,+ logsRecordsAllowedCounter,+ logsRecordsDroppedCounter,+ logsRecordsReceivedCounter,+} from '../logs-ingestion-consumer'+import type { LogsIngestionMessage } from '../types'+import { LogsPipelineContext, isUnbilledInternal, teamIdMatchesCsv } from './logs-pipeline-context'++export class LogsBatchUsage {+ readonly byTeam: UsageStatsByTeam = new Map()++ private row(teamId: number): UsageStats {+ let stats = this.byTeam.get(teamId)+ if (!stats) {+ stats = { ...DEFAULT_USAGE_STATS }+ this.byTeam.set(teamId, stats)+ }+ return stats+ }++ recordAllowed(message: LogsIngestionMessage): void {+ const stats = this.row(message.teamId)+ stats.bytesReceived += message.bytesUncompressed+ stats.recordsReceived += message.recordCount+ stats.bytesAllowed += message.bytesUncompressed+ stats.recordsAllowed += message.recordCount+ stats.bytesAllowedRecords += message.bytesUncompressedRecords+ }++ recordDropped(message: LogsIngestionMessage): void {+ const stats = this.row(message.teamId)+ stats.bytesReceived += message.bytesUncompressed+ stats.recordsReceived += message.recordCount+ stats.bytesDropped += message.bytesUncompressed+ stats.recordsDropped += message.recordCount+ }+}++export type LogsAdmissionInput = LogsIngestionMessage & { usage: LogsBatchUsage }++function countMessagesDropped(reason: string, countByTeam: Map<number, number>): void {+ for (const [teamId, count] of countByTeam) {+ logMessageDroppedCounter.inc({ reason, team_id: teamId.toString() }, count)+ recordLogMessageDropped(reason, teamId.toString(), count)+ }+}++/** Verified internal logs are never quota or rate limited, so counting them would show as drops. */+export function createTrackIncomingTrafficStep<T extends LogsAdmissionInput>(): ChunkProcessingStep<T, T> {+ return function trackIncomingTrafficStep(values) {+ let bytes = 0+ let records = 0+ for (const value of values) {+ if (isUnbilledInternal(value)) {+ continue+ }+ bytes += value.bytesUncompressed+ records += value.recordCount+ }+ logsBytesReceivedCounter.inc(bytes)+ logsRecordsReceivedCounter.inc(records)+ recordLogsReceived(bytes, records)+ return Promise.resolve(values.map((value) => ok(value)))+ }+}++/** Clamping backfilled rows to the ingest time would write the import onto today, so the gate drops the whole message. */+export function createBackfillGateStep<T extends LogsAdmissionInput>(+ context: LogsPipelineContext+): ChunkProcessingStep<T, T> {+ return function backfillGateStep(values) {+ const droppedByTeam = new Map<number, { messages: number; records: number; billed: boolean }>()+ const results = values.map((value): PipelineResult<T> => {+ if (!value.backfillRequested || teamIdMatchesCsv(context.gates.backfillEnabledTeams, value.teamId)) {+ return ok(value)+ }+ const dropped = droppedByTeam.get(value.teamId) ?? { messages: 0, records: 0, billed: false }+ dropped.messages+++ if (!isUnbilledInternal(value)) {+ dropped.records += value.recordCount+ dropped.billed = true+ value.usage.recordDropped(value)+ }+ droppedByTeam.set(value.teamId, dropped)+ return drop('backfill_not_enabled')+ })++ for (const [teamId, dropped] of droppedByTeam) {+ logMessageDroppedCounter.inc(+ { reason: 'backfill_not_enabled', team_id: teamId.toString() },+ dropped.messages+ )+ recordLogMessageDropped('backfill_not_enabled', teamId.toString(), dropped.messages)+ if (dropped.billed) {+ context.usageMetrics.queueUsageMetric(teamId, 'records_dropped_backfill_not_enabled', dropped.records)+ }+ }+ return Promise.resolve(results)+ }+}++export function createQuotaLimitStep<T extends LogsAdmissionInput>(+ context: LogsPipelineContext+): ChunkProcessingStep<T, T> {+ return async function quotaLimitStep(values) {+ const billed = values.filter((value) => !isUnbilledInternal(value))+ const uniqueTokens = [...new Set(billed.map((value) => value.token))]+ const limited = new Set(+ (+ await Promise.all(+ uniqueTokens.map(async (token) =>+ (await context.deps.quotaLimiting.isTeamTokenQuotaLimited(+ token,+ context.identity.quotaResource+ ))+ ? token+ : null+ )+ )+ ).filter((token): token is string => token !== null)+ )++ const droppedByTeam = new Map<number, number>()+ const results = values.map((value): PipelineResult<T> => {+ if (isUnbilledInternal(value) || !limited.has(value.token)) {+ return ok(value)+ }+ droppedByTeam.set(value.teamId, (droppedByTeam.get(value.teamId) ?? 0) + 1)+ value.usage.recordDropped(value)+ return drop('quota_limited')+ })+ countMessagesDropped('quota_limited', droppedByTeam)+ return results+ }+}++export function createRateLimitStep<T extends LogsAdmissionInput>(+ context: LogsPipelineContext+): ChunkProcessingStep<T, T> {+ return async function rateLimitStep(values) {+ const billed = values.filter((value) => !isUnbilledInternal(value))+ const { dropped } = await context.rateLimiter.filterMessages(billed)+ const droppedSet = new Set<LogsIngestionMessage>(dropped)++ const droppedByTeam = new Map<number, number>()+ const results = values.map((value): PipelineResult<T> => {+ if (!droppedSet.has(value)) {+ return ok(value)+ }+ droppedByTeam.set(value.teamId, (droppedByTeam.get(value.teamId) ?? 0) + 1)+ value.usage.recordDropped(value)+ return drop('rate_limited')+ })+ countMessagesDropped('rate_limited', droppedByTeam)+ return results+ }+}++/** Allowed is gross here, because processing takes off the drop-rule billing credit. */+export function createRecordAllowedUsageStep<T extends LogsAdmissionInput>(): ChunkProcessingStep<T, T> {+ return function recordAllowedUsageStep(values) {+ let bytesAllowed = 0+ let recordsAllowed = 0+ let bytesAllowedRecords = 0+ for (const value of values) {+ if (isUnbilledInternal(value)) {+ continue+ }+ value.usage.recordAllowed(value)+ bytesAllowed += value.bytesUncompressed+ bytesAllowedRecords += value.bytesUncompressedRecords+ recordsAllowed += value.recordCount+ }++ logsBytesAllowedRecordsCounter.inc(bytesAllowedRecords)+ logsBytesAllowedCounter.inc(bytesAllowed)+ logsRecordsAllowedCounter.inc(recordsAllowed)+ recordLogsAllowed(bytesAllowed, recordsAllowed)+ return Promise.resolve(values.map((value) => ok(value)))+ }+}++/** Runs after admission even when every message was dropped, which no chunk step does. */+export function reportDroppedUsage(usage: LogsBatchUsage): void {+ for (const [teamId, stats] of usage.byTeam) {+ const teamIdLabel = teamId.toString()+ if (stats.bytesDropped > 0) {+ logsBytesDroppedCounter.inc({ team_id: teamIdLabel }, stats.bytesDropped)+ }+ if (stats.recordsDropped > 0) {+ logsRecordsDroppedCounter.inc({ team_id: teamIdLabel }, stats.recordsDropped)+ }+ recordLogsDropped(teamId, stats.bytesDropped, stats.recordsDropped)+ }+}diff --git a/nodejs/src/logs/framework/create-consumer.ts b/nodejs/src/logs/framework/create-consumer.tsnew file mode 100644index 000000000000..4cd9fd130cdc--- /dev/null+++ b/nodejs/src/logs/framework/create-consumer.ts@@ -0,0 +1,23 @@+import { LogsIngestionConsumerConfig, TracesIngestionConsumerConfig } from '../config'+import { LogsIngestionConsumer, LogsIngestionConsumerDeps } from '../logs-ingestion-consumer'+import { TracesIngestionConsumer, TracesIngestionConsumerDeps } from '../traces-ingestion-consumer'+import { LogsIngestionFrameworkConsumer } from './logs-ingestion-framework-consumer'+import { TracesIngestionFrameworkConsumer } from './traces-ingestion-framework-consumer'++export function createLogsIngestionConsumer(+ config: LogsIngestionConsumerConfig,+ deps: LogsIngestionConsumerDeps+): LogsIngestionConsumer | LogsIngestionFrameworkConsumer {+ return config.LOGS_INGESTION_USE_FRAMEWORK+ ? new LogsIngestionFrameworkConsumer(config, deps)+ : new LogsIngestionConsumer(config, deps)+}++export function createTracesIngestionConsumer(+ config: LogsIngestionConsumerConfig & TracesIngestionConsumerConfig,+ deps: TracesIngestionConsumerDeps+): TracesIngestionConsumer | TracesIngestionFrameworkConsumer {+ return config.TRACES_INGESTION_USE_FRAMEWORK+ ? new TracesIngestionFrameworkConsumer(config, deps)+ : new TracesIngestionConsumer(config, deps)+}diff --git a/nodejs/src/logs/framework/finalize-logs-batch.ts b/nodejs/src/logs/framework/finalize-logs-batch.tsnew file mode 100644index 000000000000..212a1b36300b--- /dev/null+++ b/nodejs/src/logs/framework/finalize-logs-batch.ts@@ -0,0 +1,112 @@+import { logger } from '~/common/utils/logger'+import { UUID7 } from '~/common/utils/utils'++import { recordInternalLogs } from '../ingestion-otel-metrics'+import {+ DEFAULT_LOGS_RETENTION_DAYS,+ UsageStatsByTeam,+ logsInternalBytesCounter,+ logsInternalRecordsCounter,+ logsMetricRulesEvalErrorCounter,+ logsMetricRulesRecordsMatchedCounter,+ logsMetricRulesSeriesOverflowCounter,+ logsMetricRulesValueSkippedCounter,+ retentionBytesMetricName,+} from '../logs-ingestion-consumer'+import { buildMetricRulesOtlpPayload } from '../metrics-rules/otlp-payload'+import type { LogsIngestionMessage } from '../types'+import { LogsPipelineContext } from './logs-pipeline-context'+import type { MetricTalliesByTeam } from './process-log-message-step'++/** The usage report matches `bytes_ingested` exactly, so these names never reach billing. */+export function queueInternalLogsUsage(context: LogsPipelineContext, internalMessages: LogsIngestionMessage[]): void {+ for (const message of internalMessages) {+ context.usageMetrics.queueUsageMetric(message.teamId, 'bytes_ingested_internal', message.bytesUncompressed)+ context.usageMetrics.queueUsageMetric(message.teamId, 'records_ingested_internal', message.recordCount)+ const teamId = message.teamId.toString()+ logsInternalBytesCounter.inc({ team_id: teamId }, message.bytesUncompressed)+ logsInternalRecordsCounter.inc({ team_id: teamId }, message.recordCount)+ recordInternalLogs(message.teamId, message.bytesUncompressed, message.recordCount)+ }+}++export async function emitMetricRuleTallies(context: LogsPipelineContext, byTeam: MetricTalliesByTeam): Promise<void> {+ for (const [teamId, { token, rules, tallies }] of byTeam) {+ const teamIdLabel = teamId.toString()+ if (tallies.valueSkipped > 0) {+ logsMetricRulesValueSkippedCounter.inc({ team_id: teamIdLabel }, tallies.valueSkipped)+ }+ if (tallies.evalErrors > 0) {+ logsMetricRulesEvalErrorCounter.inc({ team_id: teamIdLabel }, tallies.evalErrors)+ }+ let overflow = 0+ for (const count of tallies.seriesOverflow.values()) {+ overflow += count+ }+ if (overflow > 0) {+ logsMetricRulesSeriesOverflowCounter.inc({ team_id: teamIdLabel }, overflow)+ }+ let matched = 0+ for (const ruleTallies of tallies.byRule.values()) {+ for (const entry of ruleTallies.values()) {+ matched += entry.count+ }+ }+ if (matched > 0) {+ logsMetricRulesRecordsMatchedCounter.inc({ team_id: teamIdLabel }, matched)+ }++ const payload = buildMetricRulesOtlpPayload(rules, tallies, Date.now())+ if (!payload) {+ continue+ }+ try {+ await context.deps.metricsEmitter!.emit(token, teamId, payload)+ } catch (error) {+ logger.warn('[logs-metric-rules] emit threw', { teamId, error: String(error) })+ }+ }+}++export async function emitUsageMetrics(context: LogsPipelineContext, usageStats: UsageStatsByTeam): Promise<void> {+ const { usageMetrics, identity, deps } = context+ for (const [teamId, stats] of usageStats) {+ usageMetrics.queueUsageMetric(teamId, 'bytes_received', stats.bytesReceived)+ usageMetrics.queueUsageMetric(teamId, 'records_received', stats.recordsReceived)+ usageMetrics.queueUsageMetric(teamId, 'bytes_ingested', stats.bytesAllowed)+ // Emitted beside `bytes_ingested` so the two compare before billing switches to records-based bytes.+ usageMetrics.queueUsageMetric(teamId, 'bytes_ingested_records', stats.bytesAllowedRecords)+ usageMetrics.queueUsageMetric(teamId, 'records_ingested', stats.recordsAllowed)+ usageMetrics.queueUsageMetric(teamId, 'bytes_dropped', stats.bytesDropped)+ usageMetrics.queueUsageMetric(teamId, 'records_dropped', stats.recordsDropped)+ usageMetrics.queueUsageMetric(teamId, 'pii_replacements', stats.piiReplacements)++ const retentionMetric = retentionBytesMetricName(stats.retentionDays)+ if (retentionMetric) {+ usageMetrics.queueUsageMetric(teamId, retentionMetric, stats.bytesAllowed)+ }+ if (stats.retentionDays !== DEFAULT_LOGS_RETENTION_DAYS) {+ usageMetrics.queueRetentionByteDays(teamId, stats.bytesAllowed, stats.retentionDays)+ }+ const source = identity.appSource === 'traces' ? 'apm_traces' : 'logs'+ // A fresh ID per flush stops two pods that flush the same team from collapsing one quantity away.+ const flushId = new UUID7().toString()+ deps.usageBatch.add(teamId, `${source}_bytes`, flushId, stats.bytesAllowed, 'bytes')+ deps.usageBatch.add(+ teamId,+ source === 'apm_traces' ? 'apm_traces_spans' : 'logs_records',+ flushId,+ stats.recordsAllowed,+ 'records'+ )+ }++ await Promise.all([+ usageMetrics.flush().catch((error) => {+ logger.error('🔴', 'Failed to emit usage metrics - billing data may be lost', { error })+ }),+ deps.usageBatch.flush().catch((error) => {+ logger.error('🔴', 'Failed to emit usage records - billing data may be lost', { error })+ }),+ ])+}diff --git a/nodejs/src/logs/framework/logs-ingestion-framework-consumer.test.ts b/nodejs/src/logs/framework/logs-ingestion-framework-consumer.test.tsnew file mode 100644index 000000000000..b30566f275f7--- /dev/null+++ b/nodejs/src/logs/framework/logs-ingestion-framework-consumer.test.ts@@ -0,0 +1,365 @@+import { mockProducer, mockProducerObserver } from '~/tests/helpers/mocks/producer.mock'++import { DateTime } from 'luxon'+import { CODES, Message } from 'node-rdkafka'++import { KAFKA_APP_METRICS_2, KAFKA_LOGS_CLICKHOUSE, KAFKA_LOGS_INGESTION_DLQ } from '~/common/config/kafka-topics'+import { APP_METRICS_OUTPUT, AppMetricsOutput } from '~/common/outputs'+import { IngestionOutputs } from '~/common/outputs/ingestion-outputs'+import { SingleIngestionOutput } from '~/common/outputs/single-ingestion-output'+import { UsageRecordBatch } from '~/common/usage-ingestion/usage-record-batch'+import { DependencyUnavailableError } from '~/common/utils/db/error'+import { closeHub, createHub } from '~/common/utils/db/hub'+import { createTeam, createTestTeamFixture, getTeam } from '~/tests/helpers/sql'+import { Hub, Team } from '~/types'++import { getDefaultTracesIngestionConsumerConfig } from '../config'+import { LogRecord, encodeLogRecords } from '../log-record-avro'+import { DEFAULT_LOGS_RETENTION_DAYS, LogsIngestionConsumer, logMessageDlqCounter } from '../logs-ingestion-consumer'+import { LOGS_DLQ_OUTPUT, LOGS_OUTPUT, LogsDlqOutput, LogsOutput } from '../outputs/outputs'+import { DEFAULT_TRACES_RETENTION_DAYS } from '../retention/tracing-config-cache'+import { TracesIngestionConsumer } from '../traces-ingestion-consumer'+import { createLogsIngestionConsumer, createTracesIngestionConsumer } from './create-consumer'+import { LogsIngestionFrameworkConsumer } from './logs-ingestion-framework-consumer'+import { TracesIngestionFrameworkConsumer } from './traces-ingestion-framework-consumer'++const LibrdKafkaError = require('node-rdkafka/lib/error')+const { err2str } = require('node-rdkafka/librdkafka') as { err2str: (code: number) => string }++jest.setTimeout(5000)++jest.mock('~/common/utils/posthog', () => {+ const original = jest.requireActual('~/common/utils/posthog')+ return {+ ...original,+ captureException: jest.fn(),+ }+})++const avro = require('avsc')+const LOG_RECORD_TYPE = avro.Type.forSchema({+ type: 'record',+ name: 'LogRecord',+ fields: [+ { name: 'uuid', type: ['null', 'string'] },+ { name: 'trace_id', type: ['null', 'bytes'] },+ { name: 'span_id', type: ['null', 'bytes'] },+ { name: 'trace_flags', type: ['null', 'int'] },+ { name: 'timestamp', type: ['null', 'long'] },+ { name: 'observed_timestamp', type: ['null', 'long'] },+ { name: 'body', type: ['null', 'string'] },+ { name: 'severity_text', type: ['null', 'string'] },+ { name: 'severity_number', type: ['null', 'int'] },+ { name: 'service_name', type: ['null', 'string'] },+ { name: 'resource_attributes', type: ['null', { type: 'map', values: 'string' }] },+ { name: 'instrumentation_scope', type: ['null', 'string'] },+ { name: 'event_name', type: ['null', 'string'] },+ { name: 'attributes', type: ['null', { type: 'map', values: 'string' }] },+ ],+})++let offset = 0++async function createKafkaMessages(count: number, headers: Record<string, string>): Promise<Message[]> {+ return await Promise.all(+ Array.from({ length: count }, async () => {+ const record: LogRecord = {+ uuid: `test-uuid-${offset}`,+ trace_id: null,+ span_id: null,+ trace_flags: null,+ timestamp: Date.now() * 1000,+ observed_timestamp: Date.now() * 1000,+ body: 'Test log message',+ severity_text: 'info',+ severity_number: 9,+ service_name: 'test-service',+ resource_attributes: null,+ instrumentation_scope: null,+ event_name: null,+ attributes: null,+ }+ const value = await encodeLogRecords(LOG_RECORD_TYPE, 'zstandard', [record])+ return {+ key: null,+ value,+ size: value.length,+ topic: 'test',+ offset: offset++,+ timestamp: Date.now(),+ partition: 1,+ headers: Object.entries(headers).map(([key, header]) => ({ [key]: Buffer.from(header) })),+ }+ })+ )+}++function producedTo(topic: string): { headers?: Record<string, string>; value: unknown }[] {+ return mockProducerObserver+ .getProducedMessages()+ .filter((batch) => batch.topic === topic)+ .flatMap((batch) => batch.messages.map((message) => ({ headers: message.headers, value: message.value })))+}++const kafkaDeliveryError = (code: number): Error => LibrdKafkaError.create(new Error(err2str(code)))++function failLogsProduce(makeError: () => unknown, times = Infinity): () => void {+ const original = mockProducer.queueMessages+ const passthrough = original.bind(mockProducer)+ let failures = 0+ mockProducer.queueMessages = jest.fn().mockImplementation((topicMessages) => {+ const topic = Array.isArray(topicMessages) ? topicMessages[0]?.topic : topicMessages.topic+ if (topic === KAFKA_LOGS_CLICKHOUSE && failures++ < times) {+ throw makeError()+ }+ return passthrough(topicMessages)+ })+ return () => {+ mockProducer.queueMessages = original+ }+}++describe('LogsIngestionFrameworkConsumer', () => {+ let hub: Hub+ let team: Team+ let team2: Team+ let consumer: LogsIngestionFrameworkConsumer++ const consumerDeps = (): any => ({+ ...hub,+ outputs: new IngestionOutputs<AppMetricsOutput | LogsOutput | LogsDlqOutput>({+ [APP_METRICS_OUTPUT]: new SingleIngestionOutput(+ APP_METRICS_OUTPUT,+ KAFKA_APP_METRICS_2,+ mockProducer,+ 'test'+ ),+ [LOGS_OUTPUT]: new SingleIngestionOutput(LOGS_OUTPUT, KAFKA_LOGS_CLICKHOUSE, mockProducer, 'test'),+ [LOGS_DLQ_OUTPUT]: new SingleIngestionOutput(+ LOGS_DLQ_OUTPUT,+ KAFKA_LOGS_INGESTION_DLQ,+ mockProducer,+ 'test'+ ),+ }),+ usageBatch: new UsageRecordBatch(null, { unit: 'bytes', isTeamEnabled: () => false }),+ dependencyRetry: { retryCount: 2, initialRetryDelayMs: 1 },+ })++ const withoutKafka = <T extends { start(): Promise<void> }>(instance: T): T => {+ ;(instance as any)['kafkaConsumer'] = {+ connect: jest.fn(),+ disconnect: jest.fn(),+ isHealthy: jest.fn().mockReturnValue({ status: 'healthy' }),+ }+ return instance+ }++ const tracesConfig = (overrides: Record<string, unknown> = {}): any => ({+ ...hub,+ ...getDefaultTracesIngestionConsumerConfig(),+ // A blank traces Redis host reuses the hub's Redis, so no second pool outlives the suite.+ TRACES_REDIS_HOST: '',+ ...overrides,+ })++ const processAndWait = async (messages: Message[]): Promise<void> => {+ await (+ await consumer.processKafkaBatch(messages)+ ).backgroundTask+ }++ beforeEach(async () => {+ jest.spyOn(Date, 'now').mockReturnValue(DateTime.fromISO('2025-01-01T00:00:00Z').toMillis())+ offset = 0+ hub = await createHub()+ team = (await createTestTeamFixture(hub.postgres)).team+ team2 = (await getTeam(hub.postgres, await createTeam(hub.postgres, team.organization_id)))!+ jest.spyOn(hub.quotaLimiting, 'isTeamTokenQuotaLimited').mockResolvedValue(false)+ consumer = withoutKafka(new LogsIngestionFrameworkConsumer(hub, consumerDeps()))+ await consumer.start()+ })++ afterEach(async () => {+ await consumer.stop()+ await closeHub(hub)+ })++ it('produces a valid message to the logs topic with the batch headers', async () => {+ const messages = await createKafkaMessages(1, { token: team.api_token })++ await processAndWait(messages)++ const produced = producedTo(KAFKA_LOGS_CLICKHOUSE)+ expect(produced).toHaveLength(1)+ expect(produced[0].headers).toEqual({+ token: team.api_token,+ team_id: team.id.toString(),+ 'json-parse': 'false',+ 'retention-days': DEFAULT_LOGS_RETENTION_DAYS.toString(),+ source_topic: messages[0].topic,+ source_partition: messages[0].partition.toString(),+ })+ })++ it('keeps the messages of overlapping batches apart', async () => {+ const first = await createKafkaMessages(2, { token: team.api_token })+ const second = await createKafkaMessages(1, { token: team2.api_token })++ const [a, b] = await Promise.all([consumer.processKafkaBatch(first), consumer.processKafkaBatch(second)])+ await Promise.all([a.backgroundTask, b.backgroundTask])++ expect(a.messages.map((m) => m.teamId)).toEqual([team.id, team.id])+ expect(b.messages.map((m) => m.teamId)).toEqual([team2.id])+ expect(producedTo(KAFKA_LOGS_CLICKHOUSE)).toHaveLength(3)+ })++ describe('logs produce errors', () => {+ it.each([+ ['a processing error', () => new Error('Producer error')],+ ['a Kafka error that a replay cannot fix', () => kafkaDeliveryError(CODES.ERRORS.ERR_INVALID_RECORD)],+ ])('sends the message to the DLQ on %s', async (_, makeError) => {+ const messages = await createKafkaMessages(1, { token: team.api_token })+ const dlqCounterSpy = jest.spyOn(logMessageDlqCounter, 'inc')+ const restore = failLogsProduce(makeError)++ try {+ await processAndWait(messages)+ } finally {+ restore()+ }++ expect(dlqCounterSpy).toHaveBeenCalledWith({ reason: 'Error', team_id: team.id.toString() })+ const dlq = producedTo(KAFKA_LOGS_INGESTION_DLQ)+ expect(dlq).toHaveLength(1)+ expect(dlq[0].headers).toMatchObject({+ source_topic: messages[0].topic,+ source_partition: messages[0].partition.toString(),+ source_offset: messages[0].offset.toString(),+ })+ expect(dlq[0].headers).toHaveProperty('error_message')+ })++ it('retries a dependency blip in place and produces the message once', async () => {+ const messages = await createKafkaMessages(1, { token: team.api_token })+ const restore = failLogsProduce(+ () => new DependencyUnavailableError('broker down', 'Kafka', new Error('broker down')),+ 1+ )++ try {+ await processAndWait(messages)+ } finally {+ restore()+ }++ expect(producedTo(KAFKA_LOGS_CLICKHOUSE)).toHaveLength(1)+ expect(producedTo(KAFKA_LOGS_INGESTION_DLQ)).toHaveLength(0)+ })++ it.each([+ [+ 'a dependency is unavailable',+ () => new DependencyUnavailableError('broker down', 'Kafka', new Error('broker down')),+ ],+ ['a delivery times out', () => kafkaDeliveryError(CODES.ERRORS.ERR__MSG_TIMED_OUT)],+ ['the producer queue is full', () => LibrdKafkaError.create(CODES.ERRORS.ERR__QUEUE_FULL)],+ ['the topic rejects authorization', () => kafkaDeliveryError(CODES.ERRORS.ERR_TOPIC_AUTHORIZATION_FAILED)],+ ])('fails the batch instead of DLQing when %s', async (_, makeError) => {+ const messages = await createKafkaMessages(1, { token: team.api_token })+ const restore = failLogsProduce(makeError)++ try {+ // Good data succeeds on redelivery, so quarantining it would lose it to an outage.+ await expect(processAndWait(messages)).rejects.toThrow()+ } finally {+ restore()+ }++ expect(producedTo(KAFKA_LOGS_INGESTION_DLQ)).toHaveLength(0)+ })++ it.each([+ ['the message fails to parse', { bytes_uncompressed: 'not-a-number' }],+ ['the message fails to process', {}],+ ])('fails only that batch when %s and the DLQ write fails', async (_, headers) => {+ // Two failing messages: the second failure must be awaited too, not left unhandled.+ const messages = await createKafkaMessages(2, { token: team.api_token, ...headers })+ const original = mockProducer.queueMessages+ mockProducer.queueMessages = jest.fn().mockRejectedValue(new Error('DLQ unavailable'))++ try {+ await expect(processAndWait(messages)).rejects.toThrow('DLQ unavailable')+ } finally {+ mockProducer.queueMessages = original+ }++ await processAndWait(await createKafkaMessages(1, { token: team.api_token }))+ expect(producedTo(KAFKA_LOGS_CLICKHOUSE)).toHaveLength(1)+ })+ })++ describe('TracesIngestionFrameworkConsumer', () => {+ it('reads the TRACES_ consumer and rate limiter config', () => {+ const traces = new TracesIngestionFrameworkConsumer(+ tracesConfig({+ TRACES_INGESTION_CONSUMER_GROUP_ID: 'traces-group',+ TRACES_INGESTION_CONSUMER_CONSUME_TOPIC: 'traces-topic',+ TRACES_LIMITER_BUCKET_SIZE_KB: 42,+ }),+ consumerDeps()+ )++ expect(traces['groupId']).toBe('traces-group')+ expect(traces['topic']).toBe('traces-topic')+ expect(traces['rateLimiter']['config'].LOGS_LIMITER_BUCKET_SIZE_KB).toBe(42)+ expect(traces['rateLimiter']['rateLimiter'].getKeyPrefix()).toBe('@posthog-test/traces-rate-limiter/tokens')+ expect(traces.service.id).toBe('TracesIngestionConsumer')+ })++ it('takes its default retention from the tracing config, not logs_settings', async () => {+ const tracingConfigCache = { getRetentionDays: jest.fn().mockResolvedValue(90) }+ const traces = new TracesIngestionFrameworkConsumer(tracesConfig(), {+ ...consumerDeps(),+ tracingConfigCache,+ })+ const fallback = new TracesIngestionFrameworkConsumer(tracesConfig(), consumerDeps())++ expect(await traces['defaultRetentionDays'](team.id, { retention_days: 30 })).toBe(90)+ expect(tracingConfigCache.getRetentionDays).toHaveBeenCalledWith(team.id)+ expect(await fallback['defaultRetentionDays'](team.id, { retention_days: 30 })).toBe(+ DEFAULT_TRACES_RETENTION_DAYS+ )+ })+ })++ describe('the consumer factories the servers call', () => {+ it.each([+ [false, LogsIngestionConsumer],+ [true, LogsIngestionFrameworkConsumer],+ ])('builds the logs consumer for LOGS_INGESTION_USE_FRAMEWORK=%s', (useFramework, expected) => {+ const built = createLogsIngestionConsumer(+ { ...hub, LOGS_INGESTION_USE_FRAMEWORK: useFramework },+ consumerDeps()+ )++ expect(built.constructor).toBe(expected)+ })++ it.each([+ [false, TracesIngestionConsumer],+ [true, TracesIngestionFrameworkConsumer],+ ])('builds the traces consumer for TRACES_INGESTION_USE_FRAMEWORK=%s', (useFramework, expected) => {+ // The logs flag is set the other way, so the traces server must read its own flag.+ const built = createTracesIngestionConsumer(+ tracesConfig({+ TRACES_INGESTION_USE_FRAMEWORK: useFramework,+ LOGS_INGESTION_USE_FRAMEWORK: !useFramework,+ }),+ consumerDeps()+ )++ expect(built.constructor).toBe(expected)+ })+ })+})diff --git a/nodejs/src/logs/framework/logs-ingestion-framework-consumer.ts b/nodejs/src/logs/framework/logs-ingestion-framework-consumer.tsnew file mode 100644index 000000000000..e8c3813b804c--- /dev/null+++ b/nodejs/src/logs/framework/logs-ingestion-framework-consumer.ts@@ -0,0 +1,270 @@+import { Message } from 'node-rdkafka'++import { KafkaConsumerInterface, createKafkaConsumer } from '~/common/kafka/consumer'+import { RedisV2, createRedisV2PoolFromConfig } from '~/common/redis/redis-v2'+import { AppMetricsAggregator } from '~/common/services/app-metrics-aggregator'+import { QuotaResource } from '~/common/services/quota-limiting.service'+import { instrumentFn, instrumented } from '~/common/tracing/tracing-utils'+import { logger } from '~/common/utils/logger'+import type { LogsSettings } from '~/types'+import { HealthCheckResult, PluginServerService } from '~/types'++import { LogsIngestionConsumerConfig } from '../config'+import { InternalLogsVerifier } from '../internal-logs'+import {+ DEFAULT_LOGS_RETENTION_DAYS,+ LogsIngestionConsumerDeps,+ UsageStatsByTeam,+ describeBatchPosition,+} from '../logs-ingestion-consumer'+import type { MetricRuleSource } from '../metrics-rules/compile-metric-rules'+import { MetricSeriesAdmission } from '../metrics-rules/tally'+import type { RetentionRuleSource } from '../retention/compile-retention-rules'+import { LogsSamplingService } from '../sampling/logs-sampling.service'+import { LogsRateLimiterService } from '../services/logs-rate-limiter.service'+import type { TransformationBatchBudget } from '../transformations/logs-transformer.service'+import { LogsIngestionMessage } from '../types'+import { LogsAdmissionInput, LogsBatchUsage, reportDroppedUsage } from './admission-steps'+import { emitMetricRuleTallies, emitUsageMetrics, queueInternalLogsUsage } from './finalize-logs-batch'+import {+ LogsConsumerIdentity,+ LogsFeatureGates,+ LogsPipelineContext,+ LogsUsageMetricsQueue,+ isUnbilledInternal,+} from './logs-pipeline-context'+import {+ createLogsAdmissionPipeline,+ createLogsParsePipeline,+ createLogsProcessingPipeline,+ runChunkPipeline,+} from './logs-pipelines'+import { type LogsProcessingPass, type MetricTalliesByTeam } from './process-log-message-step'++export class LogsIngestionFrameworkConsumer {+ protected name = 'LogsIngestionConsumer'+ protected quotaResource: QuotaResource = 'logs_mb_ingested'+ protected appSource: 'logs' | 'traces' = 'logs'+ protected metricRuleSource: MetricRuleSource = 'logs'+ protected retentionRuleSource: RetentionRuleSource = 'logs'+ protected kafkaConsumer: KafkaConsumerInterface+ private readonly appMetricsAggregator: AppMetricsAggregator+ private readonly redis: RedisV2+ private readonly rateLimiter: LogsRateLimiterService+ private readonly internalLogsVerifier: InternalLogsVerifier+ private readonly samplingService: LogsSamplingService+ private readonly metricSeriesAdmission = new MetricSeriesAdmission()+ private readonly gates: LogsFeatureGates+ // Built lazily, because subclass field initializers run after this constructor.+ private pipelineContext?: LogsPipelineContext++ protected groupId: string+ protected topic: string++ constructor(+ config: LogsIngestionConsumerConfig,+ private deps: LogsIngestionConsumerDeps,+ overrides: Partial<LogsIngestionConsumerConfig> = {},+ rateLimiterName: string = 'logs-rate-limiter'+ ) {+ const mergedConfig = { ...config, ...overrides }++ this.groupId = mergedConfig.LOGS_INGESTION_CONSUMER_GROUP_ID+ this.topic = mergedConfig.LOGS_INGESTION_CONSUMER_CONSUME_TOPIC++ this.appMetricsAggregator = new AppMetricsAggregator(deps.outputs)++ this.kafkaConsumer = createKafkaConsumer({ groupId: this.groupId, topic: this.topic })+ this.redis = createRedisV2PoolFromConfig({+ connection: mergedConfig.LOGS_REDIS_HOST+ ? {+ url: mergedConfig.LOGS_REDIS_HOST,+ options: {+ port: mergedConfig.LOGS_REDIS_PORT,+ tls: mergedConfig.LOGS_REDIS_TLS ? {} : undefined,+ },+ name: 'logs-redis',+ }+ : { url: mergedConfig.REDIS_URL, name: 'logs-redis-fallback' },+ poolMinSize: mergedConfig.REDIS_POOL_MIN_SIZE,+ poolMaxSize: mergedConfig.REDIS_POOL_MAX_SIZE,+ })+ this.rateLimiter = new LogsRateLimiterService(mergedConfig, this.redis, rateLimiterName)+ this.internalLogsVerifier = new InternalLogsVerifier(+ mergedConfig.LOGS_INTERNAL_INGESTION_SIGNING_KEYS,+ mergedConfig.LOGS_INTERNAL_INGESTION_ENABLED_TEAMS,+ deps.internalSourcePolicies+ )+ this.samplingService = new LogsSamplingService(this.redis, mergedConfig.LOGS_LIMITER_TTL_SECONDS)+ this.gates = {+ backfillEnabledTeams: mergedConfig.LOGS_BACKFILL_ENABLED_TEAMS,+ samplingEnabledTeams: mergedConfig.LOGS_SAMPLING_ENABLED_TEAMS,+ samplingKillswitch: mergedConfig.LOGS_SAMPLING_KILLSWITCH,+ billingProrateEnabled: mergedConfig.LOGS_BILLING_PRORATE_ENABLED,+ metricRulesEnabledTeams: mergedConfig.LOGS_METRICS_RULES_ENABLED_TEAMS,+ metricRulesKillswitch: mergedConfig.LOGS_METRICS_RULES_KILLSWITCH,+ defaultMetricRulesEnabledTeams: mergedConfig.LOGS_DEFAULT_METRIC_RULES_ENABLED_TEAMS,+ transformationsEnabledTeams: mergedConfig.LOGS_TRANSFORMATIONS_ENABLED_TEAMS,+ transformationsKillswitch: mergedConfig.LOGS_TRANSFORMATIONS_KILLSWITCH,+ retentionEnabledTeams: mergedConfig.LOGS_RETENTION_ENABLED_TEAMS,+ retentionKillswitch: mergedConfig.LOGS_RETENTION_KILLSWITCH,+ patternMaskingEnabledTeams: mergedConfig.LOGS_PATTERN_MASKING_ENABLED_TEAMS,+ jsonAttributeParsingEnabledTeams: mergedConfig.LOGS_JSON_ATTRIBUTE_PARSING_ENABLED_TEAMS,+ jsonAttributeExtractionEnabledTeams: mergedConfig.LOGS_JSON_ATTRIBUTE_EXTRACTION_ENABLED_TEAMS,+ }+ }++ protected defaultRetentionDays(_teamId: number, logsSettings: LogsSettings): Promise<number> {+ return Promise.resolve(logsSettings.retention_days ?? DEFAULT_LOGS_RETENTION_DAYS)+ }++ // Pipelines are built per batch, because overlapping calls that share a chunk pipeline mix their messages.+ private getPipelineContext(): LogsPipelineContext {+ if (!this.pipelineContext) {+ const identity: LogsConsumerIdentity = {+ name: this.name,+ appSource: this.appSource,+ quotaResource: this.quotaResource,+ metricRuleSource: this.metricRuleSource,+ retentionRuleSource: this.retentionRuleSource,+ }+ this.pipelineContext = {+ identity,+ gates: this.gates,+ deps: this.deps,+ usageMetrics: new LogsUsageMetricsQueue(this.appMetricsAggregator, identity.appSource),+ rateLimiter: this.rateLimiter,+ samplingService: this.samplingService,+ internalLogsVerifier: this.internalLogsVerifier,+ metricSeriesAdmission: this.metricSeriesAdmission,+ defaultRetentionDays: (teamId, logsSettings) => this.defaultRetentionDays(teamId, logsSettings),+ }+ }+ return this.pipelineContext+ }++ public get service(): PluginServerService {+ return {+ id: this.name,+ onShutdown: async () => await this.stop(),+ healthcheck: () => this.isHealthy(),+ }+ }++ @instrumented('logsIngestionConsumer.handleEachBatch.parseKafkaMessages')+ public async _parseKafkaBatch(messages: Message[]): Promise<LogsIngestionMessage[]> {+ return await runChunkPipeline(+ createLogsParsePipeline(this.getPipelineContext()),+ messages.map((message) => ({ message }))+ )+ }++ public async processBatch(+ messages: LogsIngestionMessage[]+ ): Promise<{ backgroundTask?: Promise<any>; messages: LogsIngestionMessage[] }> {+ if (!messages.length) {+ return { messages: [] }+ }+ const context = this.getPipelineContext()++ const usage = new LogsBatchUsage()+ const admitted = await runChunkPipeline<LogsAdmissionInput, LogsAdmissionInput, Record<string, never>>(+ createLogsAdmissionPipeline(context),+ messages.map((message) => ({ ...message, usage }))+ )+ reportDroppedUsage(usage)++ const billed: LogsIngestionMessage[] = []+ const internal: LogsIngestionMessage[] = []+ for (const { usage: _usage, ...message } of admitted) {+ ;(isUnbilledInternal(message) ? internal : billed).push(message)+ }++ const transformationBudget = this.deps.logsTransformer?.startBatch()++ return {+ backgroundTask: this.processAdmitted(context, billed, internal, usage.byTeam, transformationBudget),+ messages: [...billed, ...internal],+ }+ }++ private async processAdmitted(+ context: LogsPipelineContext,+ billed: LogsIngestionMessage[],+ internal: LogsIngestionMessage[],+ usageStats: UsageStatsByTeam,+ transformationBudget?: TransformationBatchBudget+ ): Promise<void> {+ // Billed messages run first, so PII replacement counts reach the usage rows before they are emitted.+ const billedPass: LogsProcessingPass = { usage: usageStats, metricTallies: new Map(), transformationBudget }+ await this.runProcessingPass(context, billed, billedPass)+ // Internal usage rows are never emitted, so they stay off billed usage. A second pass keeps the concurrency limit.+ await this.runProcessingPass(context, internal, {+ usage: new Map(),+ metricTallies: new Map() as MetricTalliesByTeam,+ transformationBudget,+ })+ // Metric rules and internal usage emit deltas, so they wait until no pass can fail the batch into a replay.+ queueInternalLogsUsage(context, internal)+ await emitMetricRuleTallies(context, billedPass.metricTallies)+ await this.deps.teamManager.setTeamsIngestedData(billed.map((message) => message.teamId))+ await emitUsageMetrics(context, usageStats)+ if (this.deps.logsTransformer) {+ await this.deps.logsTransformer.flush().catch((error) => {+ logger.error('Failed to flush logs transformer monitoring', { error: String(error) })+ })+ }+ }++ private async runProcessingPass(+ context: LogsPipelineContext,+ messages: LogsIngestionMessage[],+ pass: LogsProcessingPass+ ): Promise<void> {+ await runChunkPipeline(+ createLogsProcessingPipeline(context),+ messages.map((message) => ({ ...message, pass }))+ )+ }++ public async processKafkaBatch(+ messages: Message[]+ ): Promise<{ backgroundTask?: Promise<any>; messages: LogsIngestionMessage[] }> {+ try {+ const events = await this._parseKafkaBatch(messages)+ return await this.processBatch(events)+ } catch (error) {+ // Rethrow without a DLQ: the failures here are infrastructure-scoped, and the crash keeps at-least-once delivery.+ logger.error('🔴', `${this.name} - batch failed`, {+ error: String(error),+ stack: error instanceof Error ? error.stack : undefined,+ size: messages.length,+ offsets: describeBatchPosition(messages),+ })+ throw error+ }+ }++ public async start(): Promise<void> {+ await this.kafkaConsumer.connect(async (messages) => {+ logger.info('🔁', `${this.name} - handling batch`, {+ size: messages.length,+ offsets: describeBatchPosition(messages),+ })++ return await instrumentFn('logsIngestionConsumer.handleEachBatch', async () => {+ return await this.processKafkaBatch(messages)+ })+ })+ }++ public async stop(): Promise<void> {+ logger.info('💤', 'Stopping consumer...')+ await this.kafkaConsumer.disconnect()+ logger.info('💤', 'Consumer stopped!')+ }++ public isHealthy(): HealthCheckResult {+ return this.kafkaConsumer.isHealthy()+ }+}diff --git a/nodejs/src/logs/framework/logs-pipeline-context.ts b/nodejs/src/logs/framework/logs-pipeline-context.tsnew file mode 100644index 000000000000..74d19801ffc4--- /dev/null+++ b/nodejs/src/logs/framework/logs-pipeline-context.ts@@ -0,0 +1,175 @@+import { DateTime } from 'luxon'+import { Message } from 'node-rdkafka'++import { parseKafkaHeaders } from '~/common/kafka/consumer'+import { retryOnDependencyUnavailableError } from '~/common/kafka/error-handling'+import { AppMetricsAggregator } from '~/common/services/app-metrics-aggregator'+import { QuotaResource } from '~/common/services/quota-limiting.service'+import { logger } from '~/common/utils/logger'+import type { LogsSettings } from '~/types'++import { recordLogMessageDlq } from '../ingestion-otel-metrics'+import type { InternalLogsVerifier } from '../internal-logs'+import { logMessageDlqCounter, teamIdMatchesCsv } from '../logs-ingestion-consumer'+import type { LogsIngestionConsumerDeps } from '../logs-ingestion-consumer'+import type { MetricRuleSource } from '../metrics-rules/compile-metric-rules'+import type { MetricSeriesAdmission } from '../metrics-rules/tally'+import { LOGS_DLQ_OUTPUT } from '../outputs/outputs'+import type { RetentionRuleSource } from '../retention/compile-retention-rules'+import { RETENTION_MONTH_DAYS } from '../retention/evaluate-retention'+import type { LogsSamplingService } from '../sampling/logs-sampling.service'+import type { LogsRateLimiterService } from '../services/logs-rate-limiter.service'+import type { LogsIngestionMessage } from '../types'++export { teamIdMatchesCsv }++/** Traces bill against the logs quota bucket, and both rule sources use the record vocabulary (`spans`), not `traces`. */+export type LogsConsumerIdentity = {+ name: string+ appSource: 'logs' | 'traces'+ quotaResource: QuotaResource+ metricRuleSource: MetricRuleSource+ retentionRuleSource: RetentionRuleSource+}++export type LogsFeatureGates = {+ backfillEnabledTeams: string+ samplingEnabledTeams: string+ samplingKillswitch: boolean+ billingProrateEnabled: boolean+ metricRulesEnabledTeams: string+ metricRulesKillswitch: boolean+ defaultMetricRulesEnabledTeams: string+ transformationsEnabledTeams: string+ transformationsKillswitch: boolean+ retentionEnabledTeams: string+ retentionKillswitch: boolean+ patternMaskingEnabledTeams: string+ jsonAttributeParsingEnabledTeams: string+ jsonAttributeExtractionEnabledTeams: string+}++/** Lives as long as the consumer, so rows a failed batch queued flush with the next good batch, as billing relies on. */+export class LogsUsageMetricsQueue {+ constructor(+ private readonly aggregator: AppMetricsAggregator,+ private readonly appSource: string+ ) {}++ queueUsageMetric(teamId: number, metricName: string, count: number, timestamp?: DateTime): void {+ if (count === 0) {+ return+ }+ this.aggregator.queue({+ team_id: teamId,+ app_source: this.appSource,+ app_source_id: '',+ instance_id: '',+ metric_kind: 'usage',+ metric_name: metricName,+ count,+ timestamp,+ })+ }++ /** One row per 30-day month, dated at its start, so each month bills when it begins and the TTL keeps future rows until billed. */+ queueRetentionByteDays(teamId: number, bytes: number, retentionDays: number): void {+ const now = DateTime.utc()+ for (let offsetDays = 0; offsetDays < retentionDays; offsetDays += RETENTION_MONTH_DAYS) {+ const days = Math.min(RETENTION_MONTH_DAYS, retentionDays - offsetDays)+ this.queueUsageMetric(teamId, 'retention_byte_days', bytes * days, now.plus({ days: offsetDays }))+ }+ }++ queueByRule(teamId: number, metricName: string, byRule: Map<string, number>): void {+ for (const [ruleId, count] of byRule) {+ if (count <= 0) {+ continue+ }+ this.aggregator.queue({+ team_id: teamId,+ app_source: this.appSource,+ app_source_id: '',+ instance_id: ruleId,+ metric_kind: 'usage',+ metric_name: metricName,+ count,+ })+ }+ }++ flush(): Promise<void> {+ return this.aggregator.flush()+ }+}++export type LogsPipelineContext = {+ identity: LogsConsumerIdentity+ gates: LogsFeatureGates+ deps: LogsIngestionConsumerDeps+ usageMetrics: LogsUsageMetricsQueue+ rateLimiter: Pick<LogsRateLimiterService, 'filterMessages'>+ samplingService: LogsSamplingService+ internalLogsVerifier: InternalLogsVerifier+ metricSeriesAdmission: MetricSeriesAdmission+ defaultRetentionDays: (teamId: number, logsSettings: LogsSettings) => Promise<number>+}++/** Transformations write hog_function logs, which could come back here as internal logs and loop. */+export function appliesTeamLogRules(message: LogsIngestionMessage): boolean {+ return message.internalLogs?.teamRules ?? true+}++export function isUnbilledInternal(message: LogsIngestionMessage): boolean {+ return message.internalLogs !== undefined && !message.internalLogs.billed+}++/** A failed batch replays in full and the logs table has no dedupe, so each retry here avoids duplicates. */+export function retryOnDependencyUnavailable<T>(context: LogsPipelineContext, fn: () => Promise<T>): Promise<T> {+ return retryOnDependencyUnavailableError(fn, context.deps.dependencyRetry)+}++/** The DLQ headers are the replay contract, and a failed write rejects so the offset never commits without a copy. */+export async function quarantineLogMessage(+ context: LogsPipelineContext,+ message: Message,+ error: unknown,+ owner: { token?: string; teamId?: number }+): Promise<void> {+ const errorMessage = error instanceof Error ? error.message : String(error)+ const errorName = error instanceof Error ? error.name : 'UnknownError'+ const teamIdLabel = owner.teamId?.toString() ?? 'unknown'++ logMessageDlqCounter.inc({ reason: errorName, team_id: teamIdLabel })+ recordLogMessageDlq(errorName, teamIdLabel)++ try {+ await retryOnDependencyUnavailable(context, () =>+ context.deps.outputs.queueMessages(LOGS_DLQ_OUTPUT, [+ {+ value: message.value,+ key: null,+ headers: {+ ...parseKafkaHeaders(message.headers),+ ...(owner.token !== undefined ? { token: owner.token } : {}),+ team_id: teamIdLabel,+ error_message: errorMessage,+ error_name: errorName,+ failed_at: new Date().toISOString(),+ source_topic: message.topic,+ source_partition: message.partition.toString(),+ source_offset: message.offset.toString(),+ },+ },+ ])+ )+ } catch (dlqError) {+ logger.error('Failed to produce message to DLQ', {+ error: dlqError,+ originalError: errorMessage,+ partition: message.partition,+ offset: message.offset,+ })+ throw dlqError+ }+}diff --git a/nodejs/src/logs/framework/logs-pipelines.ts b/nodejs/src/logs/framework/logs-pipelines.tsnew file mode 100644index 000000000000..80c191e25b4c--- /dev/null+++ b/nodejs/src/logs/framework/logs-pipelines.ts@@ -0,0 +1,122 @@+import { logger } from '~/common/utils/logger'+import { ChunkProcessingStep } from '~/ingestion/framework/base-chunk-pipeline'+import { newChunkPipelineBuilder, perElement } from '~/ingestion/framework/builders'+import { ChunkPipeline, OkResultWithContext } from '~/ingestion/framework/chunk-pipeline.interface'+import { createOkContext } from '~/ingestion/framework/helpers'+import { isOkResult, ok } from '~/ingestion/framework/results'+import { ProcessingStep } from '~/ingestion/framework/steps'++import { MAX_CONCURRENT_MESSAGE_PROCESSES } from '../logs-ingestion-consumer'+import type { LogsIngestionMessage } from '../types'+import {+ LogsAdmissionInput,+ createBackfillGateStep,+ createQuotaLimitStep,+ createRateLimitStep,+ createRecordAllowedUsageStep,+ createTrackIncomingTrafficStep,+} from './admission-steps'+import { LogsPipelineContext } from './logs-pipeline-context'+import { LogsPipelineInput, createParseLogsMessageStep } from './parse-logs-message-step'+import { LogsProcessingInput, LogsProcessingOutcome, createProcessLogMessageStep } from './process-log-message-step'++type LogsItemContext = Record<string, never>++type ParseOutcome = { parsed?: LogsIngestionMessage; failure?: unknown }++/** Carry a failed quarantine as a result, because a thrown step latches the concurrent stage and later rejections go unawaited. */+function createSettledParseStep(context: LogsPipelineContext): ProcessingStep<LogsPipelineInput, ParseOutcome> {+ const parse = createParseLogsMessageStep(context)+ return async function parseLogsMessageStep(input) {+ try {+ const result = await parse(input)+ return isOkResult(result) ? ok({ parsed: result.value }) : result+ } catch (failure) {+ return ok({ failure })+ }+ }+}++function createFailParseOnErrorStep(): ChunkProcessingStep<ParseOutcome, LogsIngestionMessage> {+ return function failParseOnErrorStep(values) {+ const failed = values.find((value) => value.failure !== undefined)+ if (failed) {+ return Promise.reject(failed.failure)+ }+ return Promise.resolve(values.map((value) => ok(value.parsed!)))+ }+}++/** `perElement`, not `concurrently`, because per-element instrumentation costs more than the parse. */+export function createLogsParsePipeline(+ context: LogsPipelineContext+): ChunkPipeline<LogsPipelineInput, LogsIngestionMessage, LogsItemContext> {+ return newChunkPipelineBuilder<LogsPipelineInput, LogsItemContext>()+ .pipeChunk(perElement(createSettledParseStep(context)))+ .pipeChunk(createFailParseOnErrorStep())+ .build()+}++/** The gates run in the order that billing depends on. */+export function createLogsAdmissionPipeline(+ context: LogsPipelineContext+): ChunkPipeline<LogsAdmissionInput, LogsAdmissionInput, LogsItemContext> {+ return newChunkPipelineBuilder<LogsAdmissionInput, LogsItemContext>()+ .pipeChunk(createTrackIncomingTrafficStep())+ .pipeChunk(createBackfillGateStep(context))+ .pipeChunk(createQuotaLimitStep(context))+ .pipeChunk(createRateLimitStep(context))+ .pipeChunk(createRecordAllowedUsageStep())+ .build()+}++/** A replay counts the produced messages twice, so a failed pass must never emit its usage or tallies. */+function createFailPassOnErrorStep<T extends LogsProcessingOutcome>(): ChunkProcessingStep<T, T> {+ return function failPassOnErrorStep(values) {+ const failures = values.filter((value) => value.failure !== undefined)+ const quarantined = values.filter((value) => value.quarantined).length+ if (failures.length > 0 || quarantined > 0) {+ logger.error('Failed to process some log messages', {+ quarantinedCount: quarantined,+ failureCount: failures.length,+ totalCount: values.length,+ })+ }+ if (failures.length > 0) {+ return Promise.reject(failures[0].failure)+ }+ return Promise.resolve(values.map((value) => ok(value)))+ }+}++export function createLogsProcessingPipeline(+ context: LogsPipelineContext+): ChunkPipeline<LogsProcessingInput, LogsProcessingOutcome, LogsItemContext> {+ return newChunkPipelineBuilder<LogsProcessingInput, LogsItemContext>()+ .pipeChunk(+ perElement(createProcessLogMessageStep(context), { maxConcurrency: MAX_CONCURRENT_MESSAGE_PROCESSES })+ )+ .pipeChunk(createFailPassOnErrorStep())+ .build()+}++export async function runChunkPipeline<TIn, TOut, C extends LogsItemContext>(+ pipeline: ChunkPipeline<TIn, TOut, C>,+ values: TIn[]+): Promise<TOut[]> {+ if (values.length === 0) {+ return []+ }+ const elements: OkResultWithContext<TIn, C>[] = values.map((value) => createOkContext(value, {} as C))+ pipeline.feed(elements)+ const out: TOut[] = []+ let chunk+ while ((chunk = await pipeline.next()) !== null) {+ for (const element of chunk) {+ if (isOkResult(element.result)) {+ out.push(element.result.value)+ }+ }+ }+ return out+}diff --git a/nodejs/src/logs/framework/parse-logs-message-step.ts b/nodejs/src/logs/framework/parse-logs-message-step.tsnew file mode 100644index 000000000000..edaaf048ec74--- /dev/null+++ b/nodejs/src/logs/framework/parse-logs-message-step.ts@@ -0,0 +1,117 @@+import { Message } from 'node-rdkafka'++import { parseKafkaHeaders } from '~/common/kafka/consumer'+import { isDevEnv } from '~/common/utils/env-utils'+import { logger } from '~/common/utils/logger'+import { drop, ok } from '~/ingestion/framework/results'+import { ProcessingStep } from '~/ingestion/framework/steps'+import type { Team } from '~/types'++import { recordLogMessageDropped } from '../ingestion-otel-metrics'+import { INTERNAL_LOGS_HEADER } from '../internal-logs'+import {+ logMessageDroppedCounter,+ logsRecordsBytesExceedPayloadCounter,+ parseMinTimestampHeader,+ parseSizeHeader,+} from '../logs-ingestion-consumer'+import type { LogsIngestionMessage } from '../types'+import { LogsPipelineContext, quarantineLogMessage } from './logs-pipeline-context'++export type LogsPipelineInput = { message: Message }++function countDropped(reason: string): void {+ logMessageDroppedCounter.inc({ reason, team_id: 'unknown' })+ recordLogMessageDropped(reason, 'unknown')+}++/** Unusable size headers quarantine the message, because they fail the same way on every redelivery. */+export function createParseLogsMessageStep<T extends LogsPipelineInput>(+ context: LogsPipelineContext+): ProcessingStep<T, LogsIngestionMessage> {+ const { deps, internalLogsVerifier } = context+ return async function parseLogsMessageStep(input) {+ const message = input.message+ try {+ const headers = parseKafkaHeaders(message.headers)+ const token = headers.token++ if (!token) {+ logger.warn('missing_token')+ countDropped('missing_token')+ return drop('missing_token')+ }++ let team: Team | null+ try {+ if (isDevEnv() && token === 'phc_local') {+ team = await deps.teamManager.getTeam(1)+ } else {+ team = await deps.teamManager.getTeamByToken(token)+ }+ } catch (e) {+ logger.error('team_lookup_error', { error: e })+ countDropped('team_lookup_error')+ return drop('team_lookup_error')+ }++ if (!team) {+ // capture-logs has no Postgres access, so an unknown token is a client-input drop found only here.+ logger.warn('team_not_found', { token_with_no_team: token })+ countDropped('team_not_found')+ return drop('team_not_found')+ }++ const bytesUncompressed = parseSizeHeader(headers.bytes_uncompressed)+ const bytesUncompressedRecords = parseSizeHeader(headers.bytes_uncompressed_records)+ const bytesCompressed = parseSizeHeader(headers.bytes_compressed)+ const recordCount = parseSizeHeader(headers.record_count)++ if (+ bytesUncompressed === null ||+ bytesUncompressedRecords === null ||+ bytesCompressed === null ||+ recordCount === null+ ) {+ logger.error('🔴', 'logs_ingestion_invalid_size_headers', {+ teamId: team.id,+ partition: message.partition,+ offset: message.offset,+ })+ await quarantineLogMessage(context, message, new Error('invalid_size_headers'), {+ token,+ teamId: team.id,+ })+ return drop('invalid_size_headers')+ }++ if (bytesUncompressedRecords > bytesUncompressed) {+ // Billing can switch to records-based bytes only if the records sum never exceeds the payload size.+ logsRecordsBytesExceedPayloadCounter.inc({ team_id: team.id.toString() })+ }++ return ok({+ token,+ message,+ teamId: team.id,+ bytesUncompressed,+ bytesUncompressedRecords,+ bytesCompressed,+ recordCount,+ minTimestampMicros: parseMinTimestampHeader(headers.min_timestamp),+ backfillRequested: headers.backfill_days !== undefined,+ internalLogs: internalLogsVerifier.verify(headers[INTERNAL_LOGS_HEADER], team.id, headers.created_at),+ })+ } catch (e) {+ // A failed quarantine of bad size headers lands here, and a second DLQ failure fails the batch.+ logger.error('🔴', 'logs_ingestion_parse_error', {+ error: String(e),+ partition: message.partition,+ offset: message.offset,+ })+ countDropped('parse_error')+ await quarantineLogMessage(context, message, e, {})+ return drop('parse_error')+ }+ }+}diff --git a/nodejs/src/logs/framework/process-log-message-step.ts b/nodejs/src/logs/framework/process-log-message-step.tsnew file mode 100644index 000000000000..840d9b447ba6--- /dev/null+++ b/nodejs/src/logs/framework/process-log-message-step.ts@@ -0,0 +1,303 @@+import { parseKafkaHeaders } from '~/common/kafka/consumer'+import { recordPiiReplacements } from '~/common/metrics/otel-metrics'+import { instrumentFn } from '~/common/tracing/tracing-utils'+import { DependencyUnavailableError } from '~/common/utils/db/error'+import { logger } from '~/common/utils/logger'+import { ok } from '~/ingestion/framework/results'+import { ProcessingStep } from '~/ingestion/framework/steps'++import { recordLogMessageDropped } from '../ingestion-otel-metrics'+import { INTERNAL_SOURCE_HEADER, withoutInternalLogsHeaders } from '../internal-logs'+import { type PiiScrubStats } from '../log-pii-scrub'+import { type LogRecord, sniffJsonLogAttributes } from '../log-record-avro'+import {+ DEFAULT_USAGE_STATS,+ UsageStatsByTeam,+ billingByteReductionForDrops,+ logMessageDroppedCounter,+ logsBillingBytesCreditedCounter,+ logsBillingProrateDivergenceHistogram,+ logsBillingRecordsCreditedCounter,+ logsDropBatchRowsHistogram,+ logsDropFractionHistogram,+} from '../logs-ingestion-consumer'+import { rethrowKafkaOutageAsDependency } from '../logs-produce-errors'+import type { CompiledMetricRule } from '../metrics-rules/compile-metric-rules'+import { withDefaultMetricRules } from '../metrics-rules/default-metric-rules'+import { type BatchTallies, createBatchTallies, tallyRecords } from '../metrics-rules/tally'+import { LOGS_OUTPUT } from '../outputs/outputs'+import { RETENTION_EXPIRED_STAGE_NAME } from '../retention/retention-expired-stage'+import type { TransformationBatchBudget } from '../transformations/logs-transformer.service'+import type { LogsIngestionMessage } from '../types'+import {+ LogsPipelineContext,+ appliesTeamLogRules,+ isUnbilledInternal,+ quarantineLogMessage,+ retryOnDependencyUnavailable,+ teamIdMatchesCsv,+} from './logs-pipeline-context'+import { resolveLogBuffer } from './resolve-log-buffer'++export type MetricRuleBatchState = { token: string; rules: CompiledMetricRule[]; tallies: BatchTallies }+export type MetricTalliesByTeam = Map<number, MetricRuleBatchState>++export type LogsProcessingPass = {+ usage: UsageStatsByTeam+ metricTallies: MetricTalliesByTeam+ transformationBudget?: TransformationBatchBudget+}++export type LogsProcessingInput = LogsIngestionMessage & { pass: LogsProcessingPass }++/** A message never fails its pass on its own: `failure` fails the pass once every message has settled. */+export type LogsProcessingOutcome = LogsProcessingInput & { quarantined: boolean; failure?: unknown }++function canEmitMetricRules(context: LogsPipelineContext): boolean {+ return !context.gates.metricRulesKillswitch && !!context.deps.metricRulesCache && !!context.deps.metricsEmitter+}++async function getMetricRuleBatchState(+ context: LogsPipelineContext,+ byTeam: MetricTalliesByTeam,+ message: LogsIngestionMessage+): Promise<MetricRuleBatchState | null> {+ if (!appliesTeamLogRules(message)) {+ return null+ }+ const teamRulesEnabled =+ canEmitMetricRules(context) && teamIdMatchesCsv(context.gates.metricRulesEnabledTeams, message.teamId)+ const defaultRulesEnabled =+ canEmitMetricRules(context) && teamIdMatchesCsv(context.gates.defaultMetricRulesEnabledTeams, message.teamId)+ if (!teamRulesEnabled && !defaultRulesEnabled) {+ return null+ }+ let state = byTeam.get(message.teamId)+ if (!state) {+ let rules: CompiledMetricRule[] = []+ try {+ if (teamRulesEnabled) {+ const all = await context.deps.metricRulesCache!.getCompiledRules(message.teamId)+ rules = all.filter((r) => r.source === context.identity.metricRuleSource)+ }+ } catch (error) {+ // Fail open, because metric rules are additive and a rules-fetch failure must never DLQ or block the log records.+ logger.warn('[logs-metric-rules] rules fetch failed — skipping metric rules for batch', {+ teamId: message.teamId,+ error: String(error),+ })+ return null+ }+ if (defaultRulesEnabled) {+ rules = withDefaultMetricRules(rules, context.identity.metricRuleSource)+ }+ // Re-check after the await, because replacing a concurrent message's state drops its accumulated tallies.+ state = byTeam.get(message.teamId)+ if (!state) {+ state = { token: message.token, rules, tallies: createBatchTallies() }+ byTeam.set(message.teamId, state)+ }+ }+ return state.rules.length > 0 ? state : null+}++function addPiiStatsIntoUsage(+ context: LogsPipelineContext,+ usage: UsageStatsByTeam,+ teamId: number,+ delta: PiiScrubStats+): void {+ if (delta.piiReplacements === 0) {+ return+ }+ recordPiiReplacements(context.identity.appSource, delta.piiReplacements)+ const row = usage.get(teamId) || { ...DEFAULT_USAGE_STATS }+ row.piiReplacements += delta.piiReplacements+ usage.set(teamId, row)+}++type HeaderOverrides = { bytesUncompressed?: number; bytesCompressed?: number; recordCount?: number }++async function processLogMessage(context: LogsPipelineContext, input: LogsProcessingInput): Promise<void> {+ const { deps, gates, identity, usageMetrics } = context+ const { pass } = input+ const team = await retryOnDependencyUnavailable(context, () => deps.teamManager.getTeam(input.teamId))+ const logsSettings = team?.logs_settings || {}+ const jsonParse = logsSettings.json_parse_logs ?? false+ const retentionDays = await retryOnDependencyUnavailable(context, () =>+ context.defaultRetentionDays(input.teamId, logsSettings)+ )++ const teamStats = pass.usage.get(input.teamId)+ if (teamStats) {+ teamStats.retentionDays = retentionDays+ }++ if (input.message.value === null) {+ return+ }++ const metricRuleState = await getMetricRuleBatchState(context, pass.metricTallies, input)+ const jsonAttributeKey =+ identity.appSource === 'logs' && teamIdMatchesCsv(gates.jsonAttributeParsingEnabledTeams, input.teamId)+ ? logsSettings.json_parse_logs_attribute_key+ : undefined+ const onRecordsDecoded =+ metricRuleState || jsonAttributeKey+ ? (records: LogRecord[]) => {+ if (metricRuleState) {+ const nowMs = Date.now()+ tallyRecords(metricRuleState.rules, records, metricRuleState.tallies, nowMs, (ruleId, labelKey) =>+ context.metricSeriesAdmission.admit(input.teamId, ruleId, labelKey, nowMs)+ )+ }+ if (jsonAttributeKey) {+ sniffJsonLogAttributes(records, jsonAttributeKey, input.teamId)+ }+ }+ : undefined++ const resolved = await instrumentFn(+ {+ key: 'logsIngestion.sampling.resolveLogMessageBuffer',+ measureTime: false,+ sendException: false,+ getLoggingContext: () => ({ team_id: input.teamId, inbound_bytes: input.message.value?.length ?? 0 }),+ },+ async () =>+ resolveLogBuffer(+ context,+ input,+ {+ ...logsSettings,+ json_parse_logs_attribute_key:+ identity.appSource === 'logs' &&+ teamIdMatchesCsv(gates.jsonAttributeExtractionEnabledTeams, input.teamId)+ ? logsSettings.json_parse_logs_attribute_key+ : undefined,+ },+ retentionDays,+ onRecordsDecoded,+ pass.transformationBudget+ )+ )++ usageMetrics.queueUsageMetric(+ input.teamId,+ 'records_dropped_retention_expired',+ resolved.recordsDroppedByStage.get(RETENTION_EXPIRED_STAGE_NAME) ?? 0+ )++ const overrides: HeaderOverrides = {}++ // `bytesReceived` stays gross, while `bytesAllowed` and `recordsAllowed` count only the rows that survive drop rules.+ if (resolved.recordsDropped > 0) {+ const header = input.bytesUncompressed+ const contentCredit = billingByteReductionForDrops(+ header,+ resolved.contentBytesDropped,+ resolved.contentBytesTotal+ )+ // A record-weighted credit that diverges from the content-weighted one shows a size-skewed batch.+ const recordCredit = billingByteReductionForDrops(header, resolved.recordsDropped, input.recordCount)++ if (!isUnbilledInternal(input)) {+ const teamIdLabel = input.teamId.toString()+ if (contentCredit > 0) {+ logsBillingBytesCreditedCounter.inc({ team_id: teamIdLabel }, contentCredit)+ }+ logsBillingRecordsCreditedCounter.inc({ team_id: teamIdLabel }, resolved.recordsDropped)+ }+ if (input.recordCount > 0) {+ logsDropBatchRowsHistogram.observe(input.recordCount)+ }+ if (resolved.contentBytesTotal > 0) {+ logsDropFractionHistogram.observe(Math.min(1, resolved.contentBytesDropped / resolved.contentBytesTotal))+ }+ if (header > 0) {+ logsBillingProrateDivergenceHistogram.observe(Math.abs(contentCredit - recordCredit) / header)+ }++ if (gates.billingProrateEnabled) {+ const billingRow = pass.usage.get(input.teamId)+ if (billingRow) {+ billingRow.bytesAllowed = Math.max(0, billingRow.bytesAllowed - contentCredit)+ billingRow.recordsAllowed = Math.max(0, billingRow.recordsAllowed - resolved.recordsDropped)+ }++ const compressedCredit = billingByteReductionForDrops(+ input.bytesCompressed,+ resolved.contentBytesDropped,+ resolved.contentBytesTotal+ )+ overrides.bytesUncompressed = Math.max(0, header - contentCredit)+ overrides.bytesCompressed = Math.max(0, input.bytesCompressed - compressedCredit)+ overrides.recordCount = Math.max(0, input.recordCount - resolved.recordsDropped)+ }+ }++ addPiiStatsIntoUsage(context, pass.usage, input.teamId, resolved.pii)+ usageMetrics.queueByRule(input.teamId, 'sampling_records_dropped_by_rule', resolved.recordsDroppedByRuleId)+ usageMetrics.queueByRule(input.teamId, 'bytes_dropped_by_rule', resolved.bytesDroppedByRuleId)++ if (resolved.outcome === 'all_dropped') {+ logMessageDroppedCounter.inc({ reason: resolved.reason, team_id: input.teamId.toString() }, 1)+ recordLogMessageDropped(resolved.reason, input.teamId.toString())+ return+ }++ await retryOnDependencyUnavailable(context, () =>+ deps.outputs+ .queueMessages(LOGS_OUTPUT, [+ {+ value: resolved.processedValue,+ key: null,+ headers: {+ ...withoutInternalLogsHeaders(parseKafkaHeaders(input.message.headers)),+ ...(input.internalLogs !== undefined+ ? { [INTERNAL_SOURCE_HEADER]: input.internalLogs.source }+ : {}),+ token: input.token,+ team_id: input.teamId.toString(),+ 'json-parse': jsonParse.toString(),+ 'retention-days': retentionDays.toString(),+ // Both topics use random partitioning, so the ingestion lag checkpoint needs the source partition.+ source_topic: input.message.topic,+ source_partition: input.message.partition.toString(),+ ...(overrides.bytesUncompressed !== undefined+ ? { bytes_uncompressed: overrides.bytesUncompressed.toString() }+ : {}),+ ...(overrides.bytesCompressed !== undefined+ ? { bytes_compressed: overrides.bytesCompressed.toString() }+ : {}),+ ...(overrides.recordCount !== undefined+ ? { record_count: overrides.recordCount.toString() }+ : {}),+ },+ },+ ])+ .catch(rethrowKafkaOutageAsDependency)+ )+}++export function createProcessLogMessageStep<T extends LogsProcessingInput>(+ context: LogsPipelineContext+): ProcessingStep<T, T & { quarantined: boolean; failure?: unknown }> {+ return async function processLogMessageStep(input) {+ try {+ await processLogMessage(context, input)+ return ok({ ...input, quarantined: false })+ } catch (error) {+ if (error instanceof DependencyUnavailableError) {+ return ok({ ...input, quarantined: false, failure: error })+ }+ try {+ // The DLQ copy records this failure, so the batch can commit past the message.+ await quarantineLogMessage(context, input.message, error, { token: input.token, teamId: input.teamId })+ return ok({ ...input, quarantined: true })+ } catch (dlqError) {+ return ok({ ...input, quarantined: false, failure: dlqError })+ }+ }+ }+}diff --git a/nodejs/src/logs/framework/resolve-log-buffer.ts b/nodejs/src/logs/framework/resolve-log-buffer.tsnew file mode 100644index 000000000000..f786a8c61938--- /dev/null+++ b/nodejs/src/logs/framework/resolve-log-buffer.ts@@ -0,0 +1,194 @@+import { trace } from '@opentelemetry/api'++import type { LogsSettings } from '~/types'++import { MESSAGE_KEYS } from '../log-pattern-mask'+import { logsPatternForcedDecodeCounter, makePatternMaskingStage } from '../log-pattern-stage'+import { type PiiScrubStats } from '../log-pii-scrub'+import {+ type LogRecord,+ type LogRecordsTransform,+ bufferProcessingMode,+ processLogMessageBuffer,+} from '../log-record-avro'+import {+ INTERNAL_LOGS_MIN_RETENTION_DAYS,+ logsBytesDroppedByRuleCounter,+ logsSamplingRecordsDroppedCounter,+ makeTransformStage,+} from '../logs-ingestion-consumer'+import { type PipelineStage } from '../pipeline/log-processing-pipeline'+import { type CompiledRetentionRuleSet } from '../retention/evaluate-retention'+import { canHoldExpiredRow, makeRetentionExpiredStage } from '../retention/retention-expired-stage'+import { makeRetentionStage } from '../retention/retention-stage'+import type { CompiledRuleSet } from '../sampling/evaluate'+import { SAMPLING_STAGE_NAME } from '../sampling/logs-sampling.service'+import type { TransformationBatchBudget } from '../transformations/logs-transformer.service'+import type { LogsIngestionMessage } from '../types'+import {+ LogsPipelineContext,+ appliesTeamLogRules,+ retryOnDependencyUnavailable,+ teamIdMatchesCsv,+} from './logs-pipeline-context'++type ResolvedDrops = {+ pii: PiiScrubStats+ recordsDropped: number+ recordsDroppedByStage: Map<string, number>+ recordsDroppedByRuleId: Map<string, number>+ bytesDroppedByRuleId: Map<string, number>+ contentBytesDropped: number+ contentBytesTotal: number+}++export type ResolvedLogBuffer =+ | ({ outcome: 'produce'; processedValue: Buffer } & ResolvedDrops)+ | ({ outcome: 'all_dropped'; reason: string } & ResolvedDrops)++function isSamplingEvalEnabledForTeam(context: LogsPipelineContext, teamId: number): boolean {+ return !context.gates.samplingKillswitch && teamIdMatchesCsv(context.gates.samplingEnabledTeams, teamId)+}++export function isRetentionEvalEnabledForTeam(context: LogsPipelineContext, teamId: number): boolean {+ return !context.gates.retentionKillswitch && teamIdMatchesCsv(context.gates.retentionEnabledTeams, teamId)+}++function isTransformationsEnabledForTeam(context: LogsPipelineContext, teamId: number): boolean {+ if (context.gates.transformationsKillswitch || !context.deps.logsTransformer) {+ return false+ }+ return teamIdMatchesCsv(context.gates.transformationsEnabledTeams, teamId)+}++/** Logs only, because a trace record has no `body` field and would skew the log-body metrics. */+export function isPatternMaskingEnabledForTeam(context: LogsPipelineContext, teamId: number): boolean {+ return context.identity.appSource === 'logs' && teamIdMatchesCsv(context.gates.patternMaskingEnabledTeams, teamId)+}++async function buildRecordsTransform(+ context: LogsPipelineContext,+ message: LogsIngestionMessage,+ batchBudget?: TransformationBatchBudget+): Promise<LogRecordsTransform | undefined> {+ const transformer = context.deps.logsTransformer+ if (!transformer || !appliesTeamLogRules(message) || !isTransformationsEnabledForTeam(context, message.teamId)) {+ return undefined+ }+ if (!(await transformer.teamHasTransformations(message.teamId))) {+ return undefined+ }+ return (records) => transformer.transformRecords(message.teamId, records, batchBudget)+}++export async function resolveLogBuffer(+ context: LogsPipelineContext,+ message: LogsIngestionMessage,+ logsSettings: LogsSettings,+ defaultRetentionDays: number,+ onRecordsDecoded?: (records: LogRecord[]) => void,+ batchBudget?: TransformationBatchBudget+): Promise<ResolvedLogBuffer> {+ const { deps, gates } = context+ const samplingCache = deps.samplingRulesCache+ const samplingEvalEnabled = isSamplingEvalEnabledForTeam(context, message.teamId)+ let ruleSet: CompiledRuleSet | null = null+ if (samplingCache && samplingEvalEnabled && appliesTeamLogRules(message)) {+ ruleSet = await samplingCache.getCompiledRuleSet(message.teamId)+ }+ const useSamplingPipeline = Boolean(ruleSet && ruleSet.rules.length > 0)+ const recordsTransform = await buildRecordsTransform(context, message, batchBudget)++ // With no rules, the batch `retention-days` header carries the team default and the buffer skips the decode.+ const retentionCache = deps.retentionRulesCache+ const retentionEvalEnabled = isRetentionEvalEnabledForTeam(context, message.teamId)+ let retentionRuleSet: CompiledRetentionRuleSet | null = null+ if (retentionCache && retentionEvalEnabled && appliesTeamLogRules(message)) {+ retentionRuleSet = await retentionCache.getCompiledRuleSet(message.teamId, context.identity.retentionRuleSource)+ }+ const useRetention = Boolean(retentionRuleSet && retentionRuleSet.rules.length > 0)++ // Retention stamping runs after sampling and transformations, so it only stamps survivors.+ const stages: PipelineStage[] = []+ if (useSamplingPipeline && ruleSet) {+ stages.push(context.samplingService.makeSamplingStage(ruleSet, message.teamId, message.bytesUncompressed))+ }+ if (recordsTransform) {+ stages.push(makeTransformStage(recordsTransform))+ }+ let shortestRetentionDays =+ message.internalLogs !== undefined ? INTERNAL_LOGS_MIN_RETENTION_DAYS : defaultRetentionDays+ if (useRetention && retentionRuleSet) {+ stages.push(makeRetentionStage(retentionRuleSet, message.teamId, defaultRetentionDays))+ shortestRetentionDays = Math.min(+ defaultRetentionDays,+ ...retentionRuleSet.rules.map((rule) => rule.retentionDays)+ )+ }++ // Only a message that can hold an expired row pays the decode. The stage runs after retention to read the per-row value.+ const nowMicros = Date.now() * 1000+ if (canHoldExpiredRow(message.minTimestampMicros, shortestRetentionDays, nowMicros)) {+ stages.push(makeRetentionExpiredStage(message.teamId, defaultRetentionDays, nowMicros))+ }++ // Masking runs last and forces a full decode and re-encode, so the counter prices each forced decode.+ if (isPatternMaskingEnabledForTeam(context, message.teamId)) {+ const modeWithoutMasking = bufferProcessingMode(logsSettings, stages.length, Boolean(onRecordsDecoded))+ if (modeWithoutMasking !== 'decode_and_reencode') {+ logsPatternForcedDecodeCounter.inc({ from: modeWithoutMasking })+ }+ const keysCache = deps.patternMessageKeysCache+ const messageKeys = keysCache+ ? await retryOnDependencyUnavailable(context, () => keysCache.getMessageKeys(message.teamId))+ : MESSAGE_KEYS+ stages.push(makePatternMaskingStage(messageKeys))+ }++ trace.getActiveSpan()?.setAttributes({+ 'logs.sampling.killswitch': gates.samplingKillswitch,+ 'logs.sampling.enabled_teams_configured': Boolean((gates.samplingEnabledTeams || '').trim()),+ 'logs.sampling.enabled_teams_is_wildcard': (gates.samplingEnabledTeams || '').trim() === '*',+ 'logs.sampling.cache_present': Boolean(samplingCache),+ 'logs.sampling.eval_enabled_for_team': samplingEvalEnabled,+ 'logs.sampling.compiled_rule_count': ruleSet?.rules.length ?? 0,+ 'logs.transformations.enabled_for_team': Boolean(recordsTransform),+ 'logs.retention.cache_present': Boolean(retentionCache),+ 'logs.retention.eval_enabled_for_team': retentionEvalEnabled,+ 'logs.retention.compiled_rule_count': retentionRuleSet?.rules.length ?? 0,+ 'logs.retention.use_retention': useRetention,+ 'logs.sampling.pipeline': useSamplingPipeline+ ? 'decode_sample_encode'+ : recordsTransform+ ? 'decode_transform_encode'+ : 'passthrough',+ })++ const { value, pii, drops } = await processLogMessageBuffer(message.message.value!, logsSettings, {+ onRecordsDecoded,+ stages,+ })++ const recordsDroppedBySampling = drops.recordsDroppedByStage.get(SAMPLING_STAGE_NAME) ?? 0+ if (recordsDroppedBySampling > 0) {+ logsSamplingRecordsDroppedCounter.inc({ team_id: message.teamId.toString() }, recordsDroppedBySampling)+ }+ if (drops.bytesDropped > 0) {+ logsBytesDroppedByRuleCounter.inc({ team_id: message.teamId.toString() }, drops.bytesDropped)+ }++ const resolvedDrops: ResolvedDrops = {+ pii,+ recordsDropped: drops.recordsDropped,+ recordsDroppedByStage: drops.recordsDroppedByStage,+ recordsDroppedByRuleId: drops.recordsDroppedByRuleId,+ bytesDroppedByRuleId: drops.bytesDroppedByRuleId,+ contentBytesDropped: drops.contentBytesDropped,+ contentBytesTotal: drops.contentBytesTotal,+ }+ if (value === null) {+ const reason = drops.droppedBy ? `${drops.droppedBy}_all_dropped` : 'empty_batch'+ return { outcome: 'all_dropped', reason, ...resolvedDrops }+ }+ return { outcome: 'produce', processedValue: value, ...resolvedDrops }+}diff --git a/nodejs/src/logs/framework/traces-ingestion-framework-consumer.ts b/nodejs/src/logs/framework/traces-ingestion-framework-consumer.tsnew file mode 100644index 000000000000..64e68c16e471--- /dev/null+++ b/nodejs/src/logs/framework/traces-ingestion-framework-consumer.ts@@ -0,0 +1,61 @@+import type { LogsSettings } from '~/types'++import { LogsIngestionConsumerConfig, TracesIngestionConsumerConfig } from '../config'+import type { MetricRuleSource } from '../metrics-rules/compile-metric-rules'+import type { RetentionRuleSource } from '../retention/compile-retention-rules'+import { DEFAULT_TRACES_RETENTION_DAYS, TracingConfigCache } from '../retention/tracing-config-cache'+import type { TracesIngestionConsumerDeps } from '../traces-ingestion-consumer'+import { LogsIngestionFrameworkConsumer } from './logs-ingestion-framework-consumer'++/** Keep the config overrides in step with `TracesIngestionConsumer`. */+export class TracesIngestionFrameworkConsumer extends LogsIngestionFrameworkConsumer {+ protected override name = 'TracesIngestionConsumer'+ // Traces bill against the shared logs bucket, so the inherited quota resource stays.+ protected override appSource: 'logs' | 'traces' = 'traces'+ protected override metricRuleSource: MetricRuleSource = 'spans'+ protected override retentionRuleSource: RetentionRuleSource = 'spans'+ private tracingConfigCache?: TracingConfigCache++ constructor(+ config: LogsIngestionConsumerConfig & TracesIngestionConsumerConfig,+ deps: TracesIngestionConsumerDeps+ ) {+ super(+ config,+ deps,+ {+ LOGS_INGESTION_CONSUMER_GROUP_ID: config.TRACES_INGESTION_CONSUMER_GROUP_ID,+ LOGS_INGESTION_CONSUMER_CONSUME_TOPIC: config.TRACES_INGESTION_CONSUMER_CONSUME_TOPIC,+ LOGS_REDIS_HOST: config.TRACES_REDIS_HOST,+ LOGS_REDIS_PORT: config.TRACES_REDIS_PORT,+ LOGS_REDIS_PASSWORD: config.TRACES_REDIS_PASSWORD,+ LOGS_REDIS_TLS: config.TRACES_REDIS_TLS,+ LOGS_LIMITER_ENABLED_TEAMS: config.TRACES_LIMITER_ENABLED_TEAMS,+ LOGS_LIMITER_DISABLED_FOR_TEAMS: config.TRACES_LIMITER_DISABLED_FOR_TEAMS,+ LOGS_LIMITER_BUCKET_SIZE_KB: config.TRACES_LIMITER_BUCKET_SIZE_KB,+ LOGS_LIMITER_REFILL_RATE_KB_PER_SECOND: config.TRACES_LIMITER_REFILL_RATE_KB_PER_SECOND,+ LOGS_LIMITER_TTL_SECONDS: config.TRACES_LIMITER_TTL_SECONDS,+ LOGS_LIMITER_TEAM_BUCKET_SIZE_KB: config.TRACES_LIMITER_TEAM_BUCKET_SIZE_KB,+ LOGS_LIMITER_TEAM_REFILL_RATE_KB_PER_SECOND: config.TRACES_LIMITER_TEAM_REFILL_RATE_KB_PER_SECOND,+ LOGS_METRICS_RULES_ENABLED_TEAMS: config.TRACES_METRICS_RULES_ENABLED_TEAMS,+ LOGS_METRICS_RULES_KILLSWITCH: config.TRACES_METRICS_RULES_KILLSWITCH,+ LOGS_METRICS_RULES_EXPORT_URL: config.TRACES_METRICS_RULES_EXPORT_URL,+ LOGS_DEFAULT_METRIC_RULES_ENABLED_TEAMS: config.TRACES_DEFAULT_METRIC_RULES_ENABLED_TEAMS,+ LOGS_RETENTION_ENABLED_TEAMS: config.TRACES_RETENTION_ENABLED_TEAMS,+ LOGS_RETENTION_KILLSWITCH: config.TRACES_RETENTION_KILLSWITCH,+ // Internal batches are logs only, so a signed traces batch is still billed.+ LOGS_INTERNAL_INGESTION_SIGNING_KEYS: '',+ LOGS_INTERNAL_INGESTION_ENABLED_TEAMS: '',+ },+ 'traces-rate-limiter'+ )+ this.tracingConfigCache = deps.tracingConfigCache+ }++ protected override async defaultRetentionDays(teamId: number, _logsSettings: LogsSettings): Promise<number> {+ if (!this.tracingConfigCache) {+ return DEFAULT_TRACES_RETENTION_DAYS+ }+ return await this.tracingConfigCache.getRetentionDays(teamId)+ }+}diff --git a/nodejs/src/logs/logs-ingestion-consumer.ts b/nodejs/src/logs/logs-ingestion-consumer.tsindex b1717e4504e8..cd409df799c4 100644--- a/nodejs/src/logs/logs-ingestion-consumer.ts+++ b/nodejs/src/logs/logs-ingestion-consumer.ts@@ -111,12 +111,12 @@ export interface LogsIngestionConsumerDeps {export const DEFAULT_LOGS_RETENTION_DAYS = 14// A producer can stamp internal rows with less retention than the team default.-const INTERNAL_LOGS_MIN_RETENTION_DAYS = 1+export const INTERNAL_LOGS_MIN_RETENTION_DAYS = 1/** Retention day counts that get their own per-tier usage metric. */const RETENTION_USAGE_TIERS = new Set([14, 30, 90])-function retentionBytesMetricName(retentionDays: number): string | null {+export function retentionBytesMetricName(retentionDays: number): string | null {return RETENTION_USAGE_TIERS.has(retentionDays) ? `bytes_ingested_retention_${retentionDays}d` : null}@@ -134,7 +134,7 @@ export type UsageStats = {}/** `raw` is a comma-separated team-ID list, `*` for all teams, or empty for none. */-function teamIdMatchesCsv(raw: string, teamId: number): boolean {+export function teamIdMatchesCsv(raw: string, teamId: number): boolean {const trimmed = (raw || '').trim()if (!trimmed) {return false@@ -192,7 +192,7 @@ export function parseMinTimestampHeader(raw: string | undefined): number | undefreturn Number.isSafeInteger(parsed) ? parsed : undefined}-const DEFAULT_USAGE_STATS: UsageStats = {+export const DEFAULT_USAGE_STATS: UsageStats = {bytesReceived: 0,recordsReceived: 0,bytesAllowed: 0,@@ -207,7 +207,7 @@ const DEFAULT_USAGE_STATS: UsageStats = {export type UsageStatsByTeam = Map<number, UsageStats>/** Cap concurrent per-message processing within a single kafka batch. */-const MAX_CONCURRENT_MESSAGE_PROCESSES = 50+export const MAX_CONCURRENT_MESSAGE_PROCESSES = 50export const logMessageDroppedCounter = new Counter({name: 'logs_ingestion_message_dropped_count',@@ -371,7 +371,7 @@ export function billingByteReductionForDrops(headerBytes: number, bytesDropped:* attributed — so it reports no per-rule accounting, just the `transformations` tag when it removed* any record.*/-function makeTransformStage(transform: LogRecordsTransform): PipelineStage {+export function makeTransformStage(transform: LogRecordsTransform): PipelineStage {return {kind: 'filter',name: 'transformations',diff --git a/nodejs/src/servers/ingestion-logs-server.ts b/nodejs/src/servers/ingestion-logs-server.tsindex 2f36c279ce3c..1709bdc5c8c2 100644--- a/nodejs/src/servers/ingestion-logs-server.ts+++ b/nodejs/src/servers/ingestion-logs-server.ts@@ -14,7 +14,7 @@ import {LogsIngestionOutputsConfig,getDefaultLogsIngestionOutputsConfig,} from '~/logs/config'-import { LogsIngestionConsumer } from '~/logs/logs-ingestion-consumer'+import { createLogsIngestionConsumer } from '~/logs/framework/create-consumer'import { MetricRulesCache } from '~/logs/metrics-rules/metric-rules-cache'import { LogsMetricsEmitter } from '~/logs/metrics-rules/metrics-emitter'import { createProducerRegistry } from '~/logs/outputs/producer-registry'@@ -163,7 +163,7 @@ export class IngestionLogsServer implements NodeServer {unit: 'bytes',isTeamEnabled: usageReportTeamMatcher(this.config),})- const consumer = new LogsIngestionConsumer(this.config, {+ const consumer = createLogsIngestionConsumer(this.config, {teamManager,quotaLimiting,outputs,diff --git a/nodejs/src/servers/ingestion-traces-server.ts b/nodejs/src/servers/ingestion-traces-server.tsindex 114d03cca304..cd528c9ef678 100644--- a/nodejs/src/servers/ingestion-traces-server.ts+++ b/nodejs/src/servers/ingestion-traces-server.ts@@ -14,6 +14,7 @@ import {TracesIngestionConsumerConfig,getDefaultLogsIngestionOutputsConfig,} from '~/logs/config'+import { createTracesIngestionConsumer } from '~/logs/framework/create-consumer'import { MetricRulesCache } from '~/logs/metrics-rules/metric-rules-cache'import { LogsMetricsEmitter } from '~/logs/metrics-rules/metrics-emitter'import { createProducerRegistry } from '~/logs/outputs/producer-registry'@@ -27,7 +28,6 @@ import {import { createTracesOutputsRegistry } from '~/logs/outputs/registry'import { RetentionRulesCache } from '~/logs/retention/retention-rules-cache'import { TracingConfigCache } from '~/logs/retention/tracing-config-cache'-import { TracesIngestionConsumer } from '~/logs/traces-ingestion-consumer'import { CommonConfig } from '../common/config'import { DatabaseConnectionConfig, KafkaBrokerConfig, RedisConnectionsConfig } from '../ingestion/config'@@ -134,7 +134,7 @@ export class IngestionTracesServer implements NodeServer {unit: 'bytes',isTeamEnabled: usageReportTeamMatcher(this.config),})- const consumer = new TracesIngestionConsumer(this.config, {+ const consumer = createTracesIngestionConsumer(this.config, {teamManager,quotaLimiting,outputs,