:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { trace } from '@opentelemetry/api'
import { DateTime } from 'luxon'
import { Message } from 'node-rdkafka'
import pLimit from 'p-limit'
import { Counter, Histogram } from 'prom-client'
import { KafkaConsumerInterface, createKafkaConsumer, parseKafkaHeaders } from '~/common/kafka/consumer'
import { retryOnDependencyUnavailableError } from '~/common/kafka/error-handling'
import { recordPiiReplacements } from '~/common/metrics/otel-metrics'
import { AppMetricsOutput } from '~/common/outputs'
import { IngestionOutputs } from '~/common/outputs/ingestion-outputs'
import { RedisV2, createRedisV2PoolFromConfig } from '~/common/redis/redis-v2'
import { AppMetricsAggregator } from '~/common/services/app-metrics-aggregator'
import { QuotaLimiting, QuotaResource } from '~/common/services/quota-limiting.service'
import { instrumentFn, instrumented } from '~/common/tracing/tracing-utils'
import { UsageRecordBatch } from '~/common/usage-ingestion/usage-record-batch'
import { DependencyUnavailableError } from '~/common/utils/db/error'
import { isDevEnv } from '~/common/utils/env-utils'
import { logger } from '~/common/utils/logger'
import { TeamManager } from '~/common/utils/team-manager'
import { UUID7 } from '~/common/utils/utils'
import type { LogsSettings } from '~/types'
import { HealthCheckResult, PluginServerService } from '~/types'
import { LogsIngestionConsumerConfig } from './config'
import {
recordInternalLogs,
recordLogMessageDlq,
recordLogMessageDropped,
recordLogsAllowed,
recordLogsDropped,
recordLogsReceived,
} from './ingestion-otel-metrics'
import {
INTERNAL_LOGS_HEADER,
INTERNAL_SOURCE_HEADER,
InternalLogsVerifier,
type InternalSourcePolicy,
withoutInternalLogsHeaders,
} from './internal-logs'
import { MESSAGE_KEYS } from './log-pattern-mask'
import { logsPatternForcedDecodeCounter, makePatternMaskingStage } from './log-pattern-stage'
import { type PiiScrubStats } from './log-pii-scrub'
import {
type LogRecord,
type LogRecordsTransform,
bufferProcessingMode,
processLogMessageBuffer,
sniffJsonLogAttributes,
} from './log-record-avro'
import { rethrowKafkaOutageAsDependency } from './logs-produce-errors'
import type { CompiledMetricRule, MetricRuleSource } from './metrics-rules/compile-metric-rules'
import { withDefaultMetricRules } from './metrics-rules/default-metric-rules'
import { MetricRulesCache } from './metrics-rules/metric-rules-cache'
import { LogsMetricsEmitter } from './metrics-rules/metrics-emitter'
import { buildMetricRulesOtlpPayload } from './metrics-rules/otlp-payload'
import { type BatchTallies, MetricSeriesAdmission, createBatchTallies, tallyRecords } from './metrics-rules/tally'
import { LOGS_DLQ_OUTPUT, LOGS_OUTPUT, LogsDlqOutput, LogsOutput } from './outputs/outputs'
import { PatternMessageKeysCache } from './pattern-message-keys-cache'
import { EMPTY_STAGE_DROP_STATS, type PipelineStage } from './pipeline/log-processing-pipeline'
import type { RetentionRuleSource } from './retention/compile-retention-rules'
import { type CompiledRetentionRuleSet, RETENTION_MONTH_DAYS } from './retention/evaluate-retention'
import {
RETENTION_EXPIRED_STAGE_NAME,
canHoldExpiredRow,
makeRetentionExpiredStage,
} from './retention/retention-expired-stage'
import { RetentionRulesCache } from './retention/retention-rules-cache'
import { makeRetentionStage } from './retention/retention-stage'
import type { CompiledRuleSet } from './sampling/evaluate'
import { LogsSamplingService, SAMPLING_STAGE_NAME } from './sampling/logs-sampling.service'
import { SamplingRulesCache } from './sampling/sampling-rules-cache'
import { LogsRateLimiterService } from './services/logs-rate-limiter.service'
import { LogsTransformerService, TransformationBatchBudget } from './transformations/logs-transformer.service'
import { LogsIngestionMessage } from './types'
export interface LogsIngestionConsumerDeps {
teamManager: TeamManager
quotaLimiting: QuotaLimiting
/** When set, enabled teams may run head sampling before ClickHouse Kafka produce. */
samplingRulesCache?: SamplingRulesCache
/** When set (with `metricsEmitter`), enabled teams generate metrics from matching log records. */
metricRulesCache?: MetricRulesCache
/** OTLP sender for log-generated metrics; required alongside `metricRulesCache` to activate the feature. */
metricsEmitter?: LogsMetricsEmitter
/** When set, enabled teams run hog log transformations after the built-in processing. */
logsTransformer?: LogsTransformerService
/** When set, enabled teams stamp per-row retention from retention rules before produce. */
retentionRulesCache?: RetentionRulesCache
/** When set, pattern masking reads each team's message keys instead of the defaults. */
patternMessageKeysCache?: PatternMessageKeysCache
/**
* Resolved outputs registry — must include `LOGS_OUTPUT`, `LOGS_DLQ_OUTPUT`,
* and `APP_METRICS_OUTPUT`. The producer + topic for each is wired by the
* server via env vars — this consumer never touches a `KafkaProducerWrapper`
* directly.
*/
outputs: IngestionOutputs<LogsOutput | LogsDlqOutput | AppMetricsOutput>
usageBatch: UsageRecordBatch
/**
* Backoff for a `DependencyUnavailableError` from the team lookup or a Kafka produce. Defaults
* to the `retryOnDependencyUnavailableError` policy (5 tries, 1 s doubling to 16 s). Tests
* shorten it.
*/
dependencyRetry?: { retryCount: number; initialRetryDelayMs: number }
/** Defaults to `INTERNAL_SOURCE_POLICIES`. Tests pass a billed source, which production does not have yet. */
internalSourcePolicies?: Readonly<Record<string, InternalSourcePolicy>>
}
/** Ingestion default when `logs_settings.retention_days` is unset; must match `DEFAULT_LOGS_RETENTION_DAYS` in `posthog/models/team/logs_retention.py`. */
export const DEFAULT_LOGS_RETENTION_DAYS = 14
// A producer can stamp internal rows with less retention than the team default.
export const INTERNAL_LOGS_MIN_RETENTION_DAYS = 1
/** Retention day counts that get their own per-tier usage metric. */
const RETENTION_USAGE_TIERS = new Set([14, 30, 90])
export function retentionBytesMetricName(retentionDays: number): string | null {
return RETENTION_USAGE_TIERS.has(retentionDays) ? `bytes_ingested_retention_${retentionDays}d` : null
}
export type UsageStats = {
bytesReceived: number
recordsReceived: number
bytesAllowed: number
recordsAllowed: number
/** Sum of per-record content sizes for allowed batches — billing comparison candidate for bytesAllowed. */
bytesAllowedRecords: number
bytesDropped: number
recordsDropped: number
piiReplacements: number
retentionDays: number
}
/** `raw` is a comma-separated team-ID list, `*` for all teams, or empty for none. */
export function teamIdMatchesCsv(raw: string, teamId: number): boolean {
const trimmed = (raw || '').trim()
if (!trimmed) {
return false
}
if (trimmed === '*') {
return true
}
return trimmed
.split(',')
.map((s) => parseInt(s.trim(), 10))
.filter((n) => !Number.isNaN(n))
.includes(teamId)
}
/**
* Per-partition offset span of a batch, for the batch log line. A consumer that dies mid-batch
* writes nothing after it, so this is what names the records it was holding.
*/
export function describeBatchPosition(messages: Message[]): Record<string, string> {
const spans = new Map<number, { min: number; max: number }>()
for (const message of messages) {
const span = spans.get(message.partition)
if (span) {
span.min = Math.min(span.min, message.offset)
span.max = Math.max(span.max, message.offset)
} else {
spans.set(message.partition, { min: message.offset, max: message.offset })
}
}
const spanByPartition: Record<string, string> = {}
for (const [partition, { min, max }] of spans) {
spanByPartition[partition.toString()] = min === max ? min.toString() : `${min}-${max}`
}
return spanByPartition
}
/**
* Parse a batch size header, or `null` when the value is unusable. These feed usage counters and
* billing stats, so a malformed one must not reach them: `parseInt` returns NaN for junk, and
* prom-client's own guard treats NaN as falsy and lets it through, which leaves the counter NaN
* for the life of the process. Parsing stays as lenient as before — only NaN and negatives, which
* no caller can use, are rejected.
*/
export function parseSizeHeader(raw: string | undefined): number | null {
const parsed = parseInt(raw ?? '0', 10)
return Number.isFinite(parsed) && parsed >= 0 ? parsed : null
}
export function parseMinTimestampHeader(raw: string | undefined): number | undefined {
if (raw === undefined || raw.trim() === '') {
return undefined
}
const parsed = Number(raw)
return Number.isSafeInteger(parsed) ? parsed : undefined
}
export const DEFAULT_USAGE_STATS: UsageStats = {
bytesReceived: 0,
recordsReceived: 0,
bytesAllowed: 0,
recordsAllowed: 0,
bytesAllowedRecords: 0,
bytesDropped: 0,
recordsDropped: 0,
piiReplacements: 0,
retentionDays: DEFAULT_LOGS_RETENTION_DAYS,
}
export type UsageStatsByTeam = Map<number, UsageStats>
/** Cap concurrent per-message processing within a single kafka batch. */
export const MAX_CONCURRENT_MESSAGE_PROCESSES = 50
export const logMessageDroppedCounter = new Counter({
name: 'logs_ingestion_message_dropped_count',
help: 'The number of logs ingestion messages dropped',
labelNames: ['reason', 'team_id'],
})
export const logMessageDlqCounter = new Counter({
name: 'logs_ingestion_message_dlq_count',
help: 'The number of logs ingestion messages sent to DLQ',
labelNames: ['reason', 'team_id'],
})
export const logsBytesReceivedCounter = new Counter({
name: 'logs_ingestion_bytes_received_total',
help: 'Total uncompressed bytes received for logs ingestion',
})
export const logsBytesAllowedCounter = new Counter({
name: 'logs_ingestion_bytes_allowed_total',
help: 'Total uncompressed bytes allowed through quota and rate limiting. Gross of the drop-rule billing credit: billed bytes_ingested = this − logs_ingestion_billing_bytes_credited_total.',
})
export const logsBytesAllowedRecordsCounter = new Counter({
name: 'logs_ingestion_bytes_allowed_records_total',
help: 'Records-based uncompressed bytes (sum of per-record sizes) allowed through quota and rate limiting',
})
export const logsRecordsBytesExceedPayloadCounter = new Counter({
name: 'logs_ingestion_records_bytes_exceed_payload_total',
help: 'Batches where the records-based bytes sum exceeded the payload-based bytes_uncompressed header',
labelNames: ['team_id'],
})
export const logsBytesDroppedCounter = new Counter({
name: 'logs_ingestion_bytes_dropped_total',
help: 'Total uncompressed bytes dropped due to quota or rate limiting',
labelNames: ['team_id'],
})
export const logsRecordsReceivedCounter = new Counter({
name: 'logs_ingestion_records_received_total',
help: 'Total log records received',
})
export const logsInternalBytesCounter = new Counter({
name: 'logs_ingestion_internal_bytes_total',
help: 'Uncompressed bytes of verified internal logs, which stay out of the received and allowed counters.',
labelNames: ['team_id'],
})
export const logsInternalRecordsCounter = new Counter({
name: 'logs_ingestion_internal_records_total',
help: 'Records of verified internal logs, which stay out of the received and allowed counters.',
labelNames: ['team_id'],
})
export const logsRecordsAllowedCounter = new Counter({
name: 'logs_ingestion_records_allowed_total',
help: 'Total log records allowed through quota and rate limiting. Gross of the drop-rule billing credit: billed records_ingested = this − logs_ingestion_billing_records_credited_total.',
})
export const logsRecordsDroppedCounter = new Counter({
name: 'logs_ingestion_records_dropped_total',
help: 'Total log records dropped due to quota or rate limiting',
labelNames: ['team_id'],
})
export const logsSamplingRecordsDroppedCounter = new Counter({
name: 'logs_ingestion_sampling_records_dropped_total',
help: 'Log records dropped by head sampling rules',
labelNames: ['team_id'],
})
export const logsBytesDroppedByRuleCounter = new Counter({
name: 'logs_ingestion_bytes_dropped_by_rule_total',
help: 'Bytes dropped by drop rules, summed from per-row bytes_uncompressed.',
labelNames: ['team_id'],
})
// --- Billing impact of drop rules (Tier 1) ---
export const logsBillingBytesCreditedCounter = new Counter({
name: 'logs_ingestion_billing_bytes_credited_total',
help: 'Uncompressed header bytes credited back to billing for rows removed by drop rules (pro-rated).',
labelNames: ['team_id'],
})
export const logsBillingRecordsCreditedCounter = new Counter({
name: 'logs_ingestion_billing_records_credited_total',
help: 'Records removed by drop rules and credited back to billing.',
labelNames: ['team_id'],
})
// --- Metric rules (generate metrics from logs) ---
export const logsMetricRulesRecordsMatchedCounter = new Counter({
name: 'logs_metrics_rules_records_matched_total',
help: 'Log records that matched a metric rule and fed a generated metric data point.',
labelNames: ['team_id'],
})
export const logsMetricRulesValueSkippedCounter = new Counter({
name: 'logs_metrics_rules_value_skipped_total',
help: 'Records matching a value-attribute metric rule whose value was missing or non-numeric.',
labelNames: ['team_id'],
})
export const logsMetricRulesSeriesOverflowCounter = new Counter({
name: 'logs_metrics_rules_series_overflow_total',
help: 'Matching records dropped because a metric rule exceeded its per-batch label-set cap.',
labelNames: ['team_id'],
})
export const logsMetricRulesEvalErrorCounter = new Counter({
name: 'logs_metrics_rules_eval_error_total',
help: 'Per-record metric-rule filter evaluations that threw; the record was skipped for that rule.',
labelNames: ['team_id'],
})
// --- Pro-rate accuracy-confidence signals (Tier 2). No team_id label — kept low-cardinality. ---
export const logsBillingProrateDivergenceHistogram = new Histogram({
name: 'logs_ingestion_billing_prorate_divergence',
help: 'Per-message |content-weighted − record-weighted credit| / header. High = size-skewed batch = pro-rate less trustworthy.',
buckets: [0.001, 0.005, 0.01, 0.05, 0.1, 0.25, 0.5, 1],
})
export const logsDropBatchRowsHistogram = new Histogram({
name: 'logs_ingestion_drop_batch_rows',
help: 'Rows per batch for messages that had drop-rule drops; small batches carry larger pro-rate error.',
buckets: [1, 2, 5, 10, 50, 100, 500, 1000, 5000],
})
export const logsDropFractionHistogram = new Histogram({
name: 'logs_ingestion_drop_fraction',
help: 'Fraction of a message content dropped by drop rules; extreme fractions carry larger pro-rate error.',
buckets: [0.1, 0.25, 0.5, 0.75, 0.9, 0.99, 1],
})
/**
* Pro-rate a message's billable uncompressed bytes by the fraction of its content that drop
* rules removed. Billing is on the per-message header (whole-batch wire size), but drops are
* decided per row (content bytes), so we scale the header down by the dropped content fraction
* to bill only what survived.
*
* Weights come from `measureContentWeights`: every sender-controlled field of a row, with resource
* and scope bytes split across the rows that share them. The per-row `bytes_uncompressed` field is
* not used, because it counts shared resource attributes once per row. The result is always ≤ the gross
* header (droppedFraction ≤ 1), so a message is never billed above today's gross. Returns 0 when
* we can't measure (no header or no content bytes), i.e. no credit rather than a wrong one.
*/
export function billingByteReductionForDrops(headerBytes: number, bytesDropped: number, bytesTotal: number): number {
if (headerBytes <= 0 || bytesDropped <= 0 || bytesTotal <= 0) {
return 0
}
const droppedFraction = Math.min(1, bytesDropped / bytesTotal)
return Math.round(headerBytes * droppedFraction)
}
/**
* Wraps a hog log transform (mutates the surviving records in place, may drop) as the last pipeline
* `filter` stage. Partial transform drops are not billed-credited — only the all-dropped case is
* attributed — so it reports no per-rule accounting, just the `transformations` tag when it removed
* any record.
*/
export function makeTransformStage(transform: LogRecordsTransform): PipelineStage {
return {
kind: 'filter',
name: 'transformations',
run: async (records) => {
await transform(records)
return { kept: records, stats: EMPTY_STAGE_DROP_STATS() }
},
}
}
type MetricTalliesByTeam = Map<number, { token: string; rules: CompiledMetricRule[]; tallies: BatchTallies }>
export class LogsIngestionConsumer {
protected name = 'LogsIngestionConsumer'
// Billing identity for quota enforcement and usage metering; overridden by subclasses (e.g. traces).
protected quotaResource: QuotaResource = 'logs_mb_ingested'
protected appSource = 'logs'
// Record source this consumer tallies metric rules for. `appSource` is the
// billing/telemetry identity ('logs' | 'traces'); metric rules are tagged with the
// record source ('logs' | 'spans') instead, so map between the two explicitly
// rather than comparing across vocabularies. TracesIngestionConsumer overrides to 'spans'.
protected metricRuleSource: MetricRuleSource = 'logs'
// Record source this consumer evaluates retention rules for. Same vocabulary as
// `metricRuleSource` ('logs' | 'spans'), not the billing `appSource`.
protected retentionRuleSource: RetentionRuleSource = 'logs'
protected kafkaConsumer: KafkaConsumerInterface
private appMetricsAggregator: AppMetricsAggregator
private redis: RedisV2
private rateLimiter: LogsRateLimiterService
private readonly backfillEnabledTeamsRaw: string
private readonly internalLogsVerifier: InternalLogsVerifier
private samplingService: LogsSamplingService
private readonly samplingEnabledTeamsRaw: string
private readonly samplingKillswitch: boolean
private readonly billingProrateEnabled: boolean
private readonly metricRulesEnabledTeamsRaw: string
private readonly metricRulesKillswitch: boolean
private readonly defaultMetricRulesEnabledTeamsRaw: string
private readonly metricSeriesAdmission = new MetricSeriesAdmission()
private readonly transformationsEnabledTeamsRaw: string
private readonly transformationsKillswitch: boolean
private readonly retentionEnabledTeamsRaw: string
private readonly retentionKillswitch: boolean
private readonly patternMaskingEnabledTeamsRaw: string
private readonly jsonAttributeParsingEnabledTeamsRaw: string
private readonly jsonAttributeExtractionEnabledTeamsRaw: string
protected groupId: string
protected topic: string
constructor(
config: LogsIngestionConsumerConfig,
private deps: LogsIngestionConsumerDeps,
overrides: Partial<LogsIngestionConsumerConfig> = {},
// Redis key namespace for the token-bucket rate limiter. Subclasses (e.g. traces) pass
// their own so their per-team buckets don't share state with logs. Defaults to logs.
rateLimiterName: string = 'logs-rate-limiter'
) {
// Merge overrides once so every downstream consumer reads the same effective config —
// a per-use-site `overrides.X ?? config.X` pattern is easy to forget (the rate limiter did).
const mergedConfig = { ...config, ...overrides }
// The group and topic are configurable allowing for multiple ingestion consumers to be run in parallel
this.groupId = mergedConfig.LOGS_INGESTION_CONSUMER_GROUP_ID
this.topic = mergedConfig.LOGS_INGESTION_CONSUMER_CONSUME_TOPIC
this.appMetricsAggregator = new AppMetricsAggregator(deps.outputs)
this.kafkaConsumer = createKafkaConsumer({ groupId: this.groupId, topic: this.topic })
// Logs ingestion uses its own Redis instance with TLS support
this.redis = createRedisV2PoolFromConfig({
connection: mergedConfig.LOGS_REDIS_HOST
? {
url: mergedConfig.LOGS_REDIS_HOST,
options: {
port: mergedConfig.LOGS_REDIS_PORT,
tls: mergedConfig.LOGS_REDIS_TLS ? {} : undefined,
},
name: 'logs-redis',
}
: { url: mergedConfig.REDIS_URL, name: 'logs-redis-fallback' },
poolMinSize: mergedConfig.REDIS_POOL_MIN_SIZE,
poolMaxSize: mergedConfig.REDIS_POOL_MAX_SIZE,
})
this.rateLimiter = new LogsRateLimiterService(mergedConfig, this.redis, rateLimiterName)
this.backfillEnabledTeamsRaw = mergedConfig.LOGS_BACKFILL_ENABLED_TEAMS
this.internalLogsVerifier = new InternalLogsVerifier(
mergedConfig.LOGS_INTERNAL_INGESTION_SIGNING_KEYS,
mergedConfig.LOGS_INTERNAL_INGESTION_ENABLED_TEAMS,
deps.internalSourcePolicies
)
this.samplingService = new LogsSamplingService(this.redis, mergedConfig.LOGS_LIMITER_TTL_SECONDS)
this.samplingEnabledTeamsRaw = mergedConfig.LOGS_SAMPLING_ENABLED_TEAMS
this.samplingKillswitch = mergedConfig.LOGS_SAMPLING_KILLSWITCH
this.billingProrateEnabled = mergedConfig.LOGS_BILLING_PRORATE_ENABLED
this.metricRulesEnabledTeamsRaw = mergedConfig.LOGS_METRICS_RULES_ENABLED_TEAMS
this.metricRulesKillswitch = mergedConfig.LOGS_METRICS_RULES_KILLSWITCH
this.defaultMetricRulesEnabledTeamsRaw = mergedConfig.LOGS_DEFAULT_METRIC_RULES_ENABLED_TEAMS
this.transformationsEnabledTeamsRaw = mergedConfig.LOGS_TRANSFORMATIONS_ENABLED_TEAMS
this.transformationsKillswitch = mergedConfig.LOGS_TRANSFORMATIONS_KILLSWITCH
this.retentionEnabledTeamsRaw = mergedConfig.LOGS_RETENTION_ENABLED_TEAMS
this.retentionKillswitch = mergedConfig.LOGS_RETENTION_KILLSWITCH
this.patternMaskingEnabledTeamsRaw = mergedConfig.LOGS_PATTERN_MASKING_ENABLED_TEAMS
this.jsonAttributeParsingEnabledTeamsRaw = mergedConfig.LOGS_JSON_ATTRIBUTE_PARSING_ENABLED_TEAMS
this.jsonAttributeExtractionEnabledTeamsRaw = mergedConfig.LOGS_JSON_ATTRIBUTE_EXTRACTION_ENABLED_TEAMS
}
private isSamplingEvalEnabledForTeam(teamId: number): boolean {
if (this.samplingKillswitch) {
return false
}
return teamIdMatchesCsv(this.samplingEnabledTeamsRaw, teamId)
}
private isRetentionEvalEnabledForTeam(teamId: number): boolean {
if (this.retentionKillswitch) {
return false
}
return teamIdMatchesCsv(this.retentionEnabledTeamsRaw, teamId)
}
/**
* The team's default retention period, applied to records no rule matches and sent as the
* batch `retention-days` Kafka header. Traces override this to read their own setting.
*/
protected defaultRetentionDays(_teamId: number, logsSettings: LogsSettings): Promise<number> {
return Promise.resolve(logsSettings.retention_days ?? DEFAULT_LOGS_RETENTION_DAYS)
}
private canEmitMetricRules(): boolean {
return !this.metricRulesKillswitch && !!this.deps.metricRulesCache && !!this.deps.metricsEmitter
}
private isMetricRulesEnabledForTeam(teamId: number): boolean {
return this.canEmitMetricRules() && teamIdMatchesCsv(this.metricRulesEnabledTeamsRaw, teamId)
}
private isDefaultMetricRulesEnabledForTeam(teamId: number): boolean {
return this.canEmitMetricRules() && teamIdMatchesCsv(this.defaultMetricRulesEnabledTeamsRaw, teamId)
}
// Transformations write hog_function logs, which could come back here as internal logs and loop.
private appliesTeamLogRules(message: LogsIngestionMessage): boolean {
return message.verifiedAsInternalLogs?.teamRules ?? true
}
private isUnbilledInternal(message: LogsIngestionMessage): boolean {
return message.verifiedAsInternalLogs !== undefined && !message.verifiedAsInternalLogs.billed
}
private isTransformationsEnabledForTeam(teamId: number): boolean {
if (this.transformationsKillswitch || !this.deps.logsTransformer) {
return false
}
return teamIdMatchesCsv(this.transformationsEnabledTeamsRaw, teamId)
}
/**
* Logs only. `TracesIngestionConsumer` subclasses this one and reads the same config key, but a
* trace record has no `body` field, so masking one measures nothing and would mix trace shapes
* into the log-body split these metrics exist to produce.
*/
private isPatternMaskingEnabledForTeam(teamId: number): boolean {
return this.appSource === 'logs' && teamIdMatchesCsv(this.patternMaskingEnabledTeamsRaw, teamId)
}
/**
* Builds the hog log transformation hook for a message, or undefined when the team
* is not gated in or has no enabled transformation_log functions (the existence
* check is an in-process cache hit, preserving the no-decode passthrough).
*/
private async buildRecordsTransform(
message: LogsIngestionMessage,
batchBudget?: TransformationBatchBudget
): Promise<LogRecordsTransform | undefined> {
const transformer = this.deps.logsTransformer
if (
!transformer ||
!this.appliesTeamLogRules(message) ||
!this.isTransformationsEnabledForTeam(message.teamId)
) {
return undefined
}
if (!(await transformer.teamHasTransformations(message.teamId))) {
return undefined
}
return (records) =>
transformer.transformRecords(message.teamId, records, batchBudget, message.sentAsInternalLogs)
}
/**
* Decode + optional head sampling, or passthrough `processLogMessageBuffer`.
* `all_dropped` means do not enqueue to logs output (every record was sampled out
* or dropped by transformations); `reason` distinguishes the source.
*/
private async resolveLogMessageBufferWithOptionalSampling(
message: LogsIngestionMessage,
logsSettings: LogsSettings,
defaultRetentionDays: number,
onRecordsDecoded?: (records: LogRecord[]) => void,
batchBudget?: TransformationBatchBudget
): Promise<
| {
outcome: 'produce'
processedValue: Buffer
pii: PiiScrubStats
recordsDropped: number
recordsDroppedByStage: Map<string, number>
recordsDroppedByRuleId: Map<string, number>
bytesDroppedByRuleId: Map<string, number>
contentBytesDropped: number
contentBytesTotal: number
}
| {
outcome: 'all_dropped'
reason: string
pii: PiiScrubStats
recordsDropped: number
recordsDroppedByStage: Map<string, number>
recordsDroppedByRuleId: Map<string, number>
bytesDroppedByRuleId: Map<string, number>
contentBytesDropped: number
contentBytesTotal: number
}
> {
const samplingCache = this.deps.samplingRulesCache
const samplingEvalEnabled = this.isSamplingEvalEnabledForTeam(message.teamId)
let ruleSet: CompiledRuleSet | null = null
if (samplingCache && samplingEvalEnabled && this.appliesTeamLogRules(message)) {
ruleSet = await samplingCache.getCompiledRuleSet(message.teamId)
}
const useSamplingPipeline = Boolean(ruleSet && ruleSet.rules.length > 0)
const recordsTransform = await this.buildRecordsTransform(message, batchBudget)
// Per-row retention: resolve the team's retention rules when enabled. Any matching rule stamps
// `retention_days` per surviving record (falling back to the team default), which ClickHouse
// reads in place of the batch `retention-days` header. With no rules the header — still emitted
// below — carries the team default, keeping this a no-decode passthrough.
const retentionCache = this.deps.retentionRulesCache
const retentionEvalEnabled = this.isRetentionEvalEnabledForTeam(message.teamId)
let retentionRuleSet: CompiledRetentionRuleSet | null = null
if (retentionCache && retentionEvalEnabled && this.appliesTeamLogRules(message)) {
retentionRuleSet = await retentionCache.getCompiledRuleSet(message.teamId, this.retentionRuleSource)
}
const useRetention = Boolean(retentionRuleSet && retentionRuleSet.rules.length > 0)
// Assemble the ordered decode → transform → encode pipeline: sampling drop rules + rate limits
// first, hog transformations next, per-row retention stamping last (so it only stamps
// survivors). An empty list (with no metric visitor) leaves the buffer as a no-decode passthrough
// inside `processLogMessageBuffer`.
const stages: PipelineStage[] = []
if (useSamplingPipeline && ruleSet) {
stages.push(this.samplingService.makeSamplingStage(ruleSet, message.teamId, message.bytesUncompressed))
}
if (recordsTransform) {
stages.push(makeTransformStage(recordsTransform))
}
let shortestRetentionDays =
message.verifiedAsInternalLogs !== undefined ? INTERNAL_LOGS_MIN_RETENTION_DAYS : defaultRetentionDays
if (useRetention && retentionRuleSet) {
stages.push(makeRetentionStage(retentionRuleSet, message.teamId, defaultRetentionDays))
shortestRetentionDays = Math.min(
defaultRetentionDays,
...retentionRuleSet.rules.map((rule) => rule.retentionDays)
)
}
// Checking rows for expiry needs a decode, which every message would otherwise pay to find rows
// that only backdated imports produce. The stage runs after the retention stage, so it reads the
// per-row retention that ClickHouse will use.
const nowMicros = Date.now() * 1000
if (canHoldExpiredRow(message.minTimestampMicros, shortestRetentionDays, nowMicros)) {
stages.push(makeRetentionExpiredStage(message.teamId, defaultRetentionDays, nowMicros))
}
// Runs last so it only sees survivors. Adding any stage forces the full decode and re-encode,
// so a batch that would have passed through pays both, and one already decoded for a visitor
// pays the encode. The counter prices each.
if (this.isPatternMaskingEnabledForTeam(message.teamId)) {
const modeWithoutMasking = bufferProcessingMode(logsSettings, stages.length, Boolean(onRecordsDecoded))
if (modeWithoutMasking !== 'decode_and_reencode') {
logsPatternForcedDecodeCounter.inc({ from: modeWithoutMasking })
}
const keysCache = this.deps.patternMessageKeysCache
const messageKeys = keysCache
? await this.retryOnDependencyUnavailable(() => keysCache.getMessageKeys(message.teamId))
: MESSAGE_KEYS
stages.push(makePatternMaskingStage(messageKeys))
}
trace.getActiveSpan()?.setAttributes({
'logs.sampling.killswitch': this.samplingKillswitch,
'logs.sampling.enabled_teams_configured': Boolean((this.samplingEnabledTeamsRaw || '').trim()),
'logs.sampling.enabled_teams_is_wildcard': (this.samplingEnabledTeamsRaw || '').trim() === '*',
'logs.sampling.cache_present': Boolean(samplingCache),
'logs.sampling.eval_enabled_for_team': samplingEvalEnabled,
'logs.sampling.compiled_rule_count': ruleSet?.rules.length ?? 0,
'logs.transformations.enabled_for_team': Boolean(recordsTransform),
'logs.retention.cache_present': Boolean(retentionCache),
'logs.retention.eval_enabled_for_team': retentionEvalEnabled,
'logs.retention.compiled_rule_count': retentionRuleSet?.rules.length ?? 0,
'logs.retention.use_retention': useRetention,
'logs.sampling.pipeline': useSamplingPipeline
? 'decode_sample_encode'
: recordsTransform
? 'decode_transform_encode'
: 'passthrough',
})
const { value, pii, drops } = await processLogMessageBuffer(message.message.value!, logsSettings, {
onRecordsDecoded,
stages,
})
const recordsDroppedBySampling = drops.recordsDroppedByStage.get(SAMPLING_STAGE_NAME) ?? 0
if (recordsDroppedBySampling > 0) {
logsSamplingRecordsDroppedCounter.inc({ team_id: message.teamId.toString() }, recordsDroppedBySampling)
}
if (drops.bytesDropped > 0) {
logsBytesDroppedByRuleCounter.inc({ team_id: message.teamId.toString() }, drops.bytesDropped)
}
if (value === null) {
// `droppedBy` tells us which filter emptied the batch; its absence means the batch decoded
// to zero records to begin with, so attribute it to an empty batch rather than sampling.
const reason = drops.droppedBy ? `${drops.droppedBy}_all_dropped` : 'empty_batch'
return {
outcome: 'all_dropped',
reason,
pii,
recordsDropped: drops.recordsDropped,
recordsDroppedByStage: drops.recordsDroppedByStage,
recordsDroppedByRuleId: drops.recordsDroppedByRuleId,
bytesDroppedByRuleId: drops.bytesDroppedByRuleId,
contentBytesDropped: drops.contentBytesDropped,
contentBytesTotal: drops.contentBytesTotal,
}
}
return {
outcome: 'produce',
processedValue: value,
pii,
recordsDropped: drops.recordsDropped,
recordsDroppedByStage: drops.recordsDroppedByStage,
recordsDroppedByRuleId: drops.recordsDroppedByRuleId,
bytesDroppedByRuleId: drops.bytesDroppedByRuleId,
contentBytesDropped: drops.contentBytesDropped,
contentBytesTotal: drops.contentBytesTotal,
}
}
public get service(): PluginServerService {
return {
id: this.name,
onShutdown: async () => await this.stop(),
healthcheck: () => this.isHealthy(),
}
}
public async processBatch(
messages: LogsIngestionMessage[]
): Promise<{ backgroundTask?: Promise<any>; messages: LogsIngestionMessage[] }> {
if (!messages.length) {
return { messages: [] }
}
this.trackIncomingTraffic(messages)
const { backfillAllowedMessages, backfillDroppedMessages } = this.filterBackfillNotEnabledMessages(messages)
const internalMessages: LogsIngestionMessage[] = []
const billedMessages: LogsIngestionMessage[] = []
for (const message of backfillAllowedMessages) {
;(this.isUnbilledInternal(message) ? internalMessages : billedMessages).push(message)
}
const { quotaAllowedMessages, quotaDroppedMessages } = await this.filterQuotaLimitedMessages(billedMessages)
const { rateLimiterAllowedMessages, rateLimiterDroppedMessages } =
await this.filterRateLimitedMessages(quotaAllowedMessages)
const usageStats = this.trackOutgoingTrafficAndBuildUsageStats(rateLimiterAllowedMessages, [
...backfillDroppedMessages.filter((message) => !this.isUnbilledInternal(message)),
...quotaDroppedMessages,
...rateLimiterDroppedMessages,
])
// One transformation time budget shared by all messages of this batch
const transformationBatchBudget = this.deps.logsTransformer?.startBatch()
return {
// Produce first so PII replacement counts are folded into `usageStats` before MSK usage emit
backgroundTask: (async () => {
const metricTalliesByTeam = await this.processAndProduceLogMessages(
rateLimiterAllowedMessages,
usageStats,
transformationBatchBudget
)
// A usage map that is never emitted keeps their stats off the team's billed row.
// They run after the billed batches, so the per-batch concurrency limit still holds.
await this.processAndProduceLogMessages(internalMessages, new Map(), transformationBatchBudget)
// Metric rules and internal usage emit deltas, so they wait until no group can fail the batch into a replay.
this.queueInternalLogsMetrics(internalMessages)
await this.emitMetricRuleTallies(metricTalliesByTeam)
await this.deps.teamManager.setTeamsIngestedData(
rateLimiterAllowedMessages.map((message) => message.teamId)
)
await this.emitUsageMetrics(usageStats)
// Best-effort flush of transformation app metrics + function logs; never block the data path
if (this.deps.logsTransformer) {
await this.deps.logsTransformer.flush().catch((error) => {
logger.error('Failed to flush logs transformer monitoring', { error: String(error) })
})
}
})(),
messages: [...rateLimiterAllowedMessages, ...internalMessages],
}
}
// The usage report matches `bytes_ingested` exactly, so these names never reach billing.
private queueInternalLogsMetrics(internalMessages: LogsIngestionMessage[]): void {
for (const message of internalMessages) {
this.queueUsageMetric(message.teamId, 'bytes_ingested_internal', message.bytesUncompressed)
this.queueUsageMetric(message.teamId, 'records_ingested_internal', message.recordCount)
const teamId = message.teamId.toString()
logsInternalBytesCounter.inc({ team_id: teamId }, message.bytesUncompressed)
logsInternalRecordsCounter.inc({ team_id: teamId }, message.recordCount)
recordInternalLogs(message.teamId, message.bytesUncompressed, message.recordCount)
}
}
private trackIncomingTraffic(messages: LogsIngestionMessage[]): void {
let totalBytesReceived = 0
let totalRecordsReceived = 0
for (const message of messages) {
// Never quota or rate limited, so counting them would show as drops in received minus allowed.
if (this.isUnbilledInternal(message)) {
continue
}
totalBytesReceived += message.bytesUncompressed
totalRecordsReceived += message.recordCount
}
logsBytesReceivedCounter.inc(totalBytesReceived)
logsRecordsReceivedCounter.inc(totalRecordsReceived)
recordLogsReceived(totalBytesReceived, totalRecordsReceived)
}
private trackOutgoingTrafficAndBuildUsageStats(
allowedMessages: LogsIngestionMessage[],
droppedMessages: LogsIngestionMessage[]
): UsageStatsByTeam {
let totalBytesAllowed = 0
let totalRecordsAllowed = 0
const usageStats: UsageStatsByTeam = new Map()
let totalBytesAllowedRecords = 0
for (const message of allowedMessages) {
const stats = usageStats.get(message.teamId) || { ...DEFAULT_USAGE_STATS }
stats.bytesReceived += message.bytesUncompressed
stats.recordsReceived += message.recordCount
stats.bytesAllowed += message.bytesUncompressed
stats.recordsAllowed += message.recordCount
stats.bytesAllowedRecords += message.bytesUncompressedRecords
usageStats.set(message.teamId, stats)
totalBytesAllowed += message.bytesUncompressed
totalBytesAllowedRecords += message.bytesUncompressedRecords
totalRecordsAllowed += message.recordCount
}
logsBytesAllowedRecordsCounter.inc(totalBytesAllowedRecords)
// Gross, before the drop-rule billing credit applied in processAndProduceLogMessages (see counter help).
logsBytesAllowedCounter.inc(totalBytesAllowed)
logsRecordsAllowedCounter.inc(totalRecordsAllowed)
recordLogsAllowed(totalBytesAllowed, totalRecordsAllowed)
for (const message of droppedMessages) {
const stats = usageStats.get(message.teamId) || { ...DEFAULT_USAGE_STATS }
stats.bytesReceived += message.bytesUncompressed
stats.recordsReceived += message.recordCount
stats.bytesDropped += message.bytesUncompressed
stats.recordsDropped += message.recordCount
usageStats.set(message.teamId, stats)
}
for (const [teamId, stats] of usageStats) {
const teamIdLabel = teamId.toString()
if (stats.bytesDropped > 0) {
logsBytesDroppedCounter.inc({ team_id: teamIdLabel }, stats.bytesDropped)
}
if (stats.recordsDropped > 0) {
logsRecordsDroppedCounter.inc({ team_id: teamIdLabel }, stats.recordsDropped)
}
recordLogsDropped(teamId, stats.bytesDropped, stats.recordsDropped)
}
return usageStats
}
private addPiiStatsIntoUsage(usage: UsageStatsByTeam, teamId: number, delta: PiiScrubStats): void {
if (delta.piiReplacements === 0) {
return
}
recordPiiReplacements(this.appSource, delta.piiReplacements)
const row = usage.get(teamId) || { ...DEFAULT_USAGE_STATS }
row.piiReplacements += delta.piiReplacements
usage.set(teamId, row)
}
// Clamping these rows to the ingest time would write the import onto today, so the whole message is dropped.
private filterBackfillNotEnabledMessages(messages: LogsIngestionMessage[]): {
backfillAllowedMessages: LogsIngestionMessage[]
backfillDroppedMessages: LogsIngestionMessage[]
} {
const backfillAllowedMessages: LogsIngestionMessage[] = []
const backfillDroppedMessages: LogsIngestionMessage[] = []
const droppedRecordsByTeam = new Map<number, { messages: number; records: number; billed: boolean }>()
for (const message of messages) {
if (message.backfillRequested && !teamIdMatchesCsv(this.backfillEnabledTeamsRaw, message.teamId)) {
backfillDroppedMessages.push(message)
const dropped = droppedRecordsByTeam.get(message.teamId) ?? { messages: 0, records: 0, billed: false }
dropped.messages++
if (!this.isUnbilledInternal(message)) {
dropped.records += message.recordCount
dropped.billed = true
}
droppedRecordsByTeam.set(message.teamId, dropped)
} else {
backfillAllowedMessages.push(message)
}
}
for (const [teamId, dropped] of droppedRecordsByTeam) {
logMessageDroppedCounter.inc(
{ reason: 'backfill_not_enabled', team_id: teamId.toString() },
dropped.messages
)
recordLogMessageDropped('backfill_not_enabled', teamId.toString(), dropped.messages)
if (dropped.billed) {
this.queueUsageMetric(teamId, 'records_dropped_backfill_not_enabled', dropped.records)
}
}
return { backfillAllowedMessages, backfillDroppedMessages }
}
private async filterQuotaLimitedMessages(
messages: LogsIngestionMessage[]
): Promise<{ quotaAllowedMessages: LogsIngestionMessage[]; quotaDroppedMessages: LogsIngestionMessage[] }> {
const quotaDroppedMessages: LogsIngestionMessage[] = []
const quotaAllowedMessages: LogsIngestionMessage[] = []
const uniqueTokens = [...new Set(messages.map((m) => m.token))]
const quotaLimitedTokens = new Set(
(
await Promise.all(
uniqueTokens.map(async (token) =>
(await this.deps.quotaLimiting.isTeamTokenQuotaLimited(token, this.quotaResource))
? token
: null
)
)
).filter((token): token is string => token !== null)
)
const droppedCountByTeam = new Map<number, number>()
for (const message of messages) {
if (quotaLimitedTokens.has(message.token)) {
quotaDroppedMessages.push(message)
droppedCountByTeam.set(message.teamId, (droppedCountByTeam.get(message.teamId) || 0) + 1)
} else {
quotaAllowedMessages.push(message)
}
}
for (const [teamId, count] of droppedCountByTeam) {
logMessageDroppedCounter.inc({ reason: 'quota_limited', team_id: teamId.toString() }, count)
recordLogMessageDropped('quota_limited', teamId.toString(), count)
}
return { quotaAllowedMessages, quotaDroppedMessages }
}
private async filterRateLimitedMessages(messages: LogsIngestionMessage[]): Promise<{
rateLimiterAllowedMessages: LogsIngestionMessage[]
rateLimiterDroppedMessages: LogsIngestionMessage[]
}> {
const { allowed, dropped } = await this.rateLimiter.filterMessages(messages)
const droppedCountByTeam = new Map<number, number>()
for (const message of dropped) {
droppedCountByTeam.set(message.teamId, (droppedCountByTeam.get(message.teamId) || 0) + 1)
}
for (const [teamId, count] of droppedCountByTeam) {
logMessageDroppedCounter.inc({ reason: 'rate_limited', team_id: teamId.toString() }, count)
recordLogMessageDropped('rate_limited', teamId.toString(), count)
}
return { rateLimiterAllowedMessages: allowed, rateLimiterDroppedMessages: dropped }
}
/**
* Per-team metric-rule tally state for one batch. Rules are fetched once per team per
* batch (30s-cached); tallies accumulate across the batch's messages and are emitted
* as one OTLP payload per team after produce.
*/
private async getMetricRuleBatchState(
byTeam: MetricTalliesByTeam,
message: LogsIngestionMessage
): Promise<{ token: string; rules: CompiledMetricRule[]; tallies: BatchTallies } | null> {
if (!this.appliesTeamLogRules(message)) {
return null
}
const teamRulesEnabled = this.isMetricRulesEnabledForTeam(message.teamId)
const defaultRulesEnabled = this.isDefaultMetricRulesEnabledForTeam(message.teamId)
if (!teamRulesEnabled && !defaultRulesEnabled) {
return null
}
let state = byTeam.get(message.teamId)
if (!state) {
let rules: CompiledMetricRule[] = []
try {
if (teamRulesEnabled) {
const all = await this.deps.metricRulesCache!.getCompiledRules(message.teamId)
// Each consumer tallies only its own record source: the logs consumer runs
// `logs` rules, the traces consumer runs `spans` rules.
rules = all.filter((r) => r.source === this.metricRuleSource)
}
} catch (error) {
// Fail open: metric rules are a purely additive side feature, so a rules-fetch
// failure (e.g. a Postgres blip) must never DLQ or block the log records —
// skip metric generation for this team this batch instead.
logger.warn('[logs-metric-rules] rules fetch failed — skipping metric rules for batch', {
teamId: message.teamId,
error: String(error),
})
return null
}
if (defaultRulesEnabled) {
rules = withDefaultMetricRules(rules, this.metricRuleSource)
}
// Re-check after the await: concurrent messages for the same team race here,
// and replacing an existing state would silently drop its accumulated tallies.
state = byTeam.get(message.teamId)
if (!state) {
state = { token: message.token, rules, tallies: createBatchTallies() }
byTeam.set(message.teamId, state)
}
}
return state.rules.length > 0 ? state : null
}
private async emitMetricRuleTallies(byTeam: MetricTalliesByTeam): Promise<void> {
for (const [teamId, { token, rules, tallies }] of byTeam) {
const teamIdLabel = teamId.toString()
if (tallies.valueSkipped > 0) {
logsMetricRulesValueSkippedCounter.inc({ team_id: teamIdLabel }, tallies.valueSkipped)
}
if (tallies.evalErrors > 0) {
logsMetricRulesEvalErrorCounter.inc({ team_id: teamIdLabel }, tallies.evalErrors)
}
let overflow = 0
for (const count of tallies.seriesOverflow.values()) {
overflow += count
}
if (overflow > 0) {
logsMetricRulesSeriesOverflowCounter.inc({ team_id: teamIdLabel }, overflow)
}
let matched = 0
for (const ruleTallies of tallies.byRule.values()) {
for (const entry of ruleTallies.values()) {
matched += entry.count
}
}
if (matched > 0) {
logsMetricRulesRecordsMatchedCounter.inc({ team_id: teamIdLabel }, matched)
}
const payload = buildMetricRulesOtlpPayload(rules, tallies, Date.now())
if (!payload) {
continue
}
try {
await this.deps.metricsEmitter!.emit(token, teamId, payload)
} catch (error) {
// Emission is best-effort: a failing metrics endpoint must never fail log ingestion.
logger.warn('[logs-metric-rules] emit threw', { teamId, error: String(error) })
}
}
}
private async processAndProduceLogMessages(
messages: LogsIngestionMessage[],
usageStats: UsageStatsByTeam,
transformationBatchBudget?: TransformationBatchBudget
): Promise<MetricTalliesByTeam> {
const metricTalliesByTeam: MetricTalliesByTeam = new Map()
const limit = pLimit(MAX_CONCURRENT_MESSAGE_PROCESSES)
let quarantined = 0
const results = await Promise.allSettled(
messages.map((message) =>
limit(async () => {
try {
// Fetch team to get logs_settings
const team = await this.retryOnDependencyUnavailable(() =>
this.deps.teamManager.getTeam(message.teamId)
)
const logsSettings = team?.logs_settings || {}
// Extract settings with defaults
const jsonParse = logsSettings.json_parse_logs ?? false
const retentionDays = await this.retryOnDependencyUnavailable(() =>
this.defaultRetentionDays(message.teamId, logsSettings)
)
// Retention is uniform per team; stash it for the retention usage metrics.
const teamStats = usageStats.get(message.teamId)
if (teamStats) {
teamStats.retentionDays = retentionDays
}
// ignore empty messages
if (message.message.value === null) {
return Promise.resolve()
}
const metricRuleState = await this.getMetricRuleBatchState(metricTalliesByTeam, message)
const jsonAttributeKey =
this.appSource === 'logs' &&
teamIdMatchesCsv(this.jsonAttributeParsingEnabledTeamsRaw, message.teamId)
? logsSettings.json_parse_logs_attribute_key
: undefined
const onRecordsDecoded =
metricRuleState || jsonAttributeKey
? (records: LogRecord[]) => {
if (metricRuleState) {
const nowMs = Date.now()
tallyRecords(
metricRuleState.rules,
records,
metricRuleState.tallies,
nowMs,
(ruleId, labelKey) =>
this.metricSeriesAdmission.admit(
message.teamId,
ruleId,
labelKey,
nowMs
)
)
}
if (jsonAttributeKey) {
sniffJsonLogAttributes(records, jsonAttributeKey, message.teamId)
}
}
: undefined
const resolved = await instrumentFn(
{
key: 'logsIngestion.sampling.resolveLogMessageBuffer',
measureTime: false,
sendException: false,
getLoggingContext: () => ({
team_id: message.teamId,
inbound_bytes: message.message.value?.length ?? 0,
}),
},
async () =>
this.resolveLogMessageBufferWithOptionalSampling(
message,
{
...logsSettings,
json_parse_logs_attribute_key:
this.appSource === 'logs' &&
teamIdMatchesCsv(
this.jsonAttributeExtractionEnabledTeamsRaw,
message.teamId
)
? logsSettings.json_parse_logs_attribute_key
: undefined,
},
retentionDays,
onRecordsDecoded,
transformationBatchBudget
)
)
this.queueUsageMetric(
message.teamId,
'records_dropped_retention_expired',
resolved.recordsDroppedByStage.get(RETENTION_EXPIRED_STAGE_NAME) ?? 0
)
let bytesUncompressedHeaderOverride: number | undefined
let bytesCompressedHeaderOverride: number | undefined
let recordCountHeaderOverride: number | undefined
// Drop rules removed rows from this message — credit billing by the dropped
// content fraction. `bytesReceived` stays gross (what was sent); `bytesAllowed`
// (→ bytes_ingested) and `recordsAllowed` reflect only what survived the drops.
if (resolved.recordsDropped > 0) {
const header = message.bytesUncompressed
const contentCredit = billingByteReductionForDrops(
header,
resolved.contentBytesDropped,
resolved.contentBytesTotal
)
// Second estimator (record-weighted) for the accuracy-confidence signal:
// when content- and record-weighted credits diverge, the batch is size-skewed
// and the content-weighted pro-rate we bill on is less trustworthy.
const recordCredit = billingByteReductionForDrops(
header,
resolved.recordsDropped,
message.recordCount
)
if (!this.isUnbilledInternal(message)) {
const teamIdLabel = message.teamId.toString()
if (contentCredit > 0) {
logsBillingBytesCreditedCounter.inc({ team_id: teamIdLabel }, contentCredit)
}
logsBillingRecordsCreditedCounter.inc({ team_id: teamIdLabel }, resolved.recordsDropped)
}
if (message.recordCount > 0) {
logsDropBatchRowsHistogram.observe(message.recordCount)
}
if (resolved.contentBytesTotal > 0) {
logsDropFractionHistogram.observe(
Math.min(1, resolved.contentBytesDropped / resolved.contentBytesTotal)
)
}
if (header > 0) {
logsBillingProrateDivergenceHistogram.observe(
Math.abs(contentCredit - recordCredit) / header
)
}
// Shadow mode unless LOGS_BILLING_PRORATE_ENABLED: the credit above is
// computed and counted (observability) but billed usage is untouched.
if (this.billingProrateEnabled) {
const billingRow = usageStats.get(message.teamId)
if (billingRow) {
billingRow.bytesAllowed = Math.max(0, billingRow.bytesAllowed - contentCredit)
billingRow.recordsAllowed = Math.max(
0,
billingRow.recordsAllowed - resolved.recordsDropped
)
}
// Scale both size headers down by the same dropped content fraction, and
// reduce the record count by the exact number of rows dropped, so
// downstream accounting sees only the surviving rows and bytes.
const compressedCredit = billingByteReductionForDrops(
message.bytesCompressed,
resolved.contentBytesDropped,
resolved.contentBytesTotal
)
bytesUncompressedHeaderOverride = Math.max(0, header - contentCredit)
bytesCompressedHeaderOverride = Math.max(0, message.bytesCompressed - compressedCredit)
recordCountHeaderOverride = Math.max(0, message.recordCount - resolved.recordsDropped)
}
}
if (resolved.outcome === 'all_dropped') {
logMessageDroppedCounter.inc(
{ reason: resolved.reason, team_id: message.teamId.toString() },
1
)
recordLogMessageDropped(resolved.reason, message.teamId.toString())
this.addPiiStatsIntoUsage(usageStats, message.teamId, resolved.pii)
this.queueSamplingRecordsDroppedByRule(message.teamId, resolved.recordsDroppedByRuleId)
this.queueBytesDroppedByRule(message.teamId, resolved.bytesDroppedByRuleId)
return Promise.resolve()
}
const { processedValue, pii, recordsDroppedByRuleId, bytesDroppedByRuleId } = resolved
this.addPiiStatsIntoUsage(usageStats, message.teamId, pii)
this.queueSamplingRecordsDroppedByRule(message.teamId, recordsDroppedByRuleId)
this.queueBytesDroppedByRule(message.teamId, bytesDroppedByRuleId)
// Await so a rejection here lands in the catch and routes to the DLQ.
await this.retryOnDependencyUnavailable(() =>
this.deps.outputs
.queueMessages(LOGS_OUTPUT, [
{
value: processedValue,
key: null,
headers: {
...withoutInternalLogsHeaders(parseKafkaHeaders(message.message.headers)),
...(message.verifiedAsInternalLogs !== undefined
? { [INTERNAL_SOURCE_HEADER]: message.verifiedAsInternalLogs.source }
: {}),
token: message.token,
team_id: message.teamId.toString(),
'json-parse': jsonParse.toString(),
'retention-days': retentionDays.toString(),
// Both topics use random partitioning, so the ingestion lag checkpoint needs the source partition.
source_topic: message.message.topic,
source_partition: message.message.partition.toString(),
...(bytesUncompressedHeaderOverride !== undefined
? { bytes_uncompressed: bytesUncompressedHeaderOverride.toString() }
: {}),
...(bytesCompressedHeaderOverride !== undefined
? { bytes_compressed: bytesCompressedHeaderOverride.toString() }
: {}),
...(recordCountHeaderOverride !== undefined
? { record_count: recordCountHeaderOverride.toString() }
: {}),
},
},
])
.catch(rethrowKafkaOutageAsDependency)
)
} catch (error) {
// A dependency outage (Postgres, Redis, Kafka) is infrastructure-scoped. The
// message itself is fine and processes once the dependency is back, so
// quarantining it would move good data into the DLQ. The retries above
// already absorbed a blip; a longer outage leaves the message on the source
// topic and fails the batch below instead.
if (error instanceof DependencyUnavailableError) {
throw error
}
// Quarantine succeeded: the DLQ copy is the record of this failure, so the
// message resolves and the batch can commit past it. A failed DLQ write
// rejects here instead, so the batch keeps the only copy on the source topic.
await this.produceToDlq(message, error)
quarantined += 1
}
})
)
)
// Every rejection means this batch must not commit: a dependency outage, or a DLQ write
// that failed. `Promise.allSettled` swallows rejections, so rethrow before the tallies and
// usage metrics are emitted. The rejected background task stops the offset commit and the
// consumer replays the batch from the last good commit, which would count them twice.
const failures = results.filter((r): r is PromiseRejectedResult => r.status === 'rejected')
if (failures.length > 0 || quarantined > 0) {
logger.error('Failed to process some log messages', {
quarantinedCount: quarantined,
failureCount: failures.length,
totalCount: messages.length,
})
}
if (failures.length > 0) {
throw failures[0].reason
}
return metricTalliesByTeam
}
private async produceToDlq(message: LogsIngestionMessage, error: unknown): Promise<void> {
await this.produceRawToDlq(message.message, error, { token: message.token, teamId: message.teamId })
}
/**
* Absorb a dependency blip in place. A batch that fails after some of its messages were
* produced is replayed in full, and the logs table has no dedupe, so every retry here is a
* duplicate avoided. Only an outage that outlasts the backoff reaches the caller.
*/
private retryOnDependencyUnavailable<T>(fn: () => Promise<T>): Promise<T> {
return retryOnDependencyUnavailableError(fn, this.deps.dependencyRetry)
}
/**
* Quarantine a raw Kafka message. Takes the message rather than a `LogsIngestionMessage` so
* failures from before the team is resolved can reach the DLQ too — those used to be counted
* and discarded, which lost the only copy of the payload.
*/
private async produceRawToDlq(
message: Message,
error: unknown,
context: { token?: string; teamId?: number }
): Promise<void> {
const errorMessage = error instanceof Error ? error.message : String(error)
const errorName = error instanceof Error ? error.name : 'UnknownError'
const teamIdLabel = context.teamId?.toString() ?? 'unknown'
logMessageDlqCounter.inc({ reason: errorName, team_id: teamIdLabel })
recordLogMessageDlq(errorName, teamIdLabel)
try {
await this.retryOnDependencyUnavailable(() =>
this.deps.outputs.queueMessages(LOGS_DLQ_OUTPUT, [
{
value: message.value,
key: null,
headers: {
...parseKafkaHeaders(message.headers),
...(context.token !== undefined ? { token: context.token } : {}),
team_id: teamIdLabel,
error_message: errorMessage,
error_name: errorName,
failed_at: new Date().toISOString(),
// Where the message came from, so a quarantined record can be read back
// from the source partition and replayed against the consumer.
source_topic: message.topic,
source_partition: message.partition.toString(),
source_offset: message.offset.toString(),
},
},
])
)
} catch (dlqError) {
logger.error('Failed to produce message to DLQ', {
error: dlqError,
originalError: errorMessage,
partition: message.partition,
offset: message.offset,
})
// Rethrow: quarantine is what lets the caller move past the record. Swallowing this
// would commit the source offset with no copy anywhere, which loses the payload for
// good. Failing the batch keeps the record on the source topic until the DLQ is back.
throw dlqError
}
}
private async emitUsageMetrics(usageStats: UsageStatsByTeam): Promise<void> {
for (const [teamId, stats] of usageStats) {
this.queueUsageMetric(teamId, 'bytes_received', stats.bytesReceived)
this.queueUsageMetric(teamId, 'records_received', stats.recordsReceived)
this.queueUsageMetric(teamId, 'bytes_ingested', stats.bytesAllowed)
// Records-based counterpart to `bytes_ingested` (sum of per-record sizes instead of
// payload size). Emitted in parallel so the two can be compared before billing
// switches to the records-based value; not yet read by usage reports.
this.queueUsageMetric(teamId, 'bytes_ingested_records', stats.bytesAllowedRecords)
this.queueUsageMetric(teamId, 'records_ingested', stats.recordsAllowed)
this.queueUsageMetric(teamId, 'bytes_dropped', stats.bytesDropped)
this.queueUsageMetric(teamId, 'records_dropped', stats.recordsDropped)
this.queueUsageMetric(teamId, 'pii_replacements', stats.piiReplacements)
const retentionMetric = retentionBytesMetricName(stats.retentionDays)
if (retentionMetric) {
this.queueUsageMetric(teamId, retentionMetric, stats.bytesAllowed)
}
// Byte-days: ingested bytes weighted by the retention day count, only for teams that
// chose a retention other than the default. The default tier is covered by `bytes_ingested`,
// so it must not be billed a second time here. Uses credit-adjusted `bytesAllowed` to
// reconcile with `bytes_ingested`. Runs beside the per-tier metric above until billing
// leaves fixed tiers.
if (stats.retentionDays !== DEFAULT_LOGS_RETENTION_DAYS) {
this.queueRetentionByteDays(teamId, stats.bytesAllowed, stats.retentionDays)
}
const source = this.appSource === 'traces' ? 'apm_traces' : 'logs'
// These records are per-flush aggregates, not one per billed thing, so there is no
// stable identity to reproduce. A fresh ID per flush is what keeps two pods flushing
// the same team from colliding and collapsing one flush's quantity away.
const flushId = new UUID7().toString()
this.deps.usageBatch.add(teamId, `${source}_bytes`, flushId, stats.bytesAllowed, 'bytes')
this.deps.usageBatch.add(
teamId,
source === 'apm_traces' ? 'apm_traces_spans' : 'logs_records',
flushId,
stats.recordsAllowed,
'records'
)
}
// Best-effort, and independent of each other: neither failing may block ingestion or skip
// the other, and nothing downstream reads either result, so they go out together.
await Promise.all([
this.appMetricsAggregator.flush().catch((error) => {
logger.error('🔴', 'Failed to emit usage metrics - billing data may be lost', { error })
}),
this.deps.usageBatch.flush().catch((error) => {
logger.error('🔴', 'Failed to emit usage records - billing data may be lost', { error })
}),
])
}
private queueUsageMetric(teamId: number, metricName: string, count: number, timestamp?: DateTime): void {
if (count === 0) {
return
}
this.appMetricsAggregator.queue({
team_id: teamId,
app_source: this.appSource,
app_source_id: '',
instance_id: '',
metric_kind: 'usage',
metric_name: metricName,
count,
timestamp,
})
}
/**
* Writes one `retention_byte_days` row per 30-day month of retention, each dated at the start of
* its month, so each month of storage bills when it starts instead of the whole period billing
* on the ingest day. The usage report sums these rows by timestamp, and the `app_metrics2` TTL
* counts from each row's own timestamp, so future rows stay until they are billed.
*/
private queueRetentionByteDays(teamId: number, bytes: number, retentionDays: number): void {
const now = DateTime.utc()
for (let offsetDays = 0; offsetDays < retentionDays; offsetDays += RETENTION_MONTH_DAYS) {
const days = Math.min(RETENTION_MONTH_DAYS, retentionDays - offsetDays)
this.queueUsageMetric(teamId, 'retention_byte_days', bytes * days, now.plus({ days: offsetDays }))
}
}
/**
* Per-rule head sampling drops in app_metrics2 (`sampling_records_dropped_by_rule`).
* Team-level sampling volume in CH is `sum(count)` over those rows (no separate aggregate metric).
*/
private queueSamplingRecordsDroppedByRule(teamId: number, byRule: Map<string, number>): void {
for (const [ruleId, count] of byRule) {
if (count <= 0) {
continue
}
this.appMetricsAggregator.queue({
team_id: teamId,
app_source: this.appSource,
app_source_id: '',
instance_id: ruleId,
metric_kind: 'usage',
metric_name: 'sampling_records_dropped_by_rule',
count,
})
}
}
/**
* Per-rule dropped bytes in app_metrics2 (`bytes_dropped_by_rule`). Summed from per-row
* `bytes_uncompressed`; bridges drop-rule accounting toward billing's `bytes_ingested`.
*/
private queueBytesDroppedByRule(teamId: number, byRule: Map<string, number>): void {
for (const [ruleId, count] of byRule) {
if (count <= 0) {
continue
}
this.appMetricsAggregator.queue({
team_id: teamId,
app_source: this.appSource,
app_source_id: '',
instance_id: ruleId,
metric_kind: 'usage',
metric_name: 'bytes_dropped_by_rule',
count,
})
}
}
@instrumented('logsIngestionConsumer.handleEachBatch.parseKafkaMessages')
public async _parseKafkaBatch(messages: Message[]): Promise<LogsIngestionMessage[]> {
const events: LogsIngestionMessage[] = []
await Promise.all(
messages.map(async (message) => {
try {
const headers = parseKafkaHeaders(message.headers)
const token = headers.token
if (!token) {
logger.warn('missing_token')
logMessageDroppedCounter.inc({ reason: 'missing_token', team_id: 'unknown' })
recordLogMessageDropped('missing_token', 'unknown')
return
}
let team
try {
if (isDevEnv() && token === 'phc_local') {
// phc_local is a special token used in dev to refer to team 1
team = await this.deps.teamManager.getTeam(1)
} else {
team = await this.deps.teamManager.getTeamByToken(token)
}
} catch (e) {
logger.error('team_lookup_error', { error: e })
logMessageDroppedCounter.inc({ reason: 'team_lookup_error', team_id: 'unknown' })
recordLogMessageDropped('team_lookup_error', 'unknown')
return
}
if (!team) {
// A well-formed but unknown or rotated token is a client-input problem, not a
// service fault. capture-logs accepts any well-formed token shape and defers team
// resolution to here because it has no Postgres access, so this fires on every
// dropped message for a bad token. Already tracked via the metrics below.
logger.warn('team_not_found', { token_with_no_team: token })
logMessageDroppedCounter.inc({ reason: 'team_not_found', team_id: 'unknown' })
recordLogMessageDropped('team_not_found', 'unknown')
return
}
const bytesUncompressed = parseSizeHeader(headers.bytes_uncompressed)
const bytesUncompressedRecords = parseSizeHeader(headers.bytes_uncompressed_records)
const bytesCompressed = parseSizeHeader(headers.bytes_compressed)
const recordCount = parseSizeHeader(headers.record_count)
if (
bytesUncompressed === null ||
bytesUncompressedRecords === null ||
bytesCompressed === null ||
recordCount === null
) {
logger.error('🔴', 'logs_ingestion_invalid_size_headers', {
teamId: team.id,
partition: message.partition,
offset: message.offset,
})
await this.produceRawToDlq(message, new Error('invalid_size_headers'), {
token,
teamId: team.id,
})
return
}
if (bytesUncompressedRecords > bytesUncompressed) {
// Billing can only switch from payload-based to records-based bytes if the
// records sum never exceeds the payload size — flag any violation.
logsRecordsBytesExceedPayloadCounter.inc({ team_id: team.id.toString() })
}
events.push({
token,
message,
teamId: team.id,
bytesUncompressed,
bytesUncompressedRecords,
bytesCompressed,
recordCount,
minTimestampMicros: parseMinTimestampHeader(headers.min_timestamp),
backfillRequested: headers.backfill_days !== undefined,
sentAsInternalLogs: headers[INTERNAL_LOGS_HEADER] !== undefined,
verifiedAsInternalLogs: this.internalLogsVerifier.verify(
headers[INTERNAL_LOGS_HEADER],
team.id,
headers.created_at
),
})
} catch (e) {
// A message we cannot parse is message-scoped and will fail the same way on
// every redelivery, so quarantine it instead of discarding the payload.
logger.error('🔴', 'logs_ingestion_parse_error', {
error: String(e),
partition: message.partition,
offset: message.offset,
})
logMessageDroppedCounter.inc({ reason: 'parse_error', team_id: 'unknown' })
recordLogMessageDropped('parse_error', 'unknown')
await this.produceRawToDlq(message, e, {})
return
}
})
)
return events
}
public async processKafkaBatch(
messages: Message[]
): Promise<{ backgroundTask?: Promise<any>; messages: LogsIngestionMessage[] }> {
try {
const events = await this._parseKafkaBatch(messages)
return await this.processBatch(events)
} catch (error) {
// Rethrow: the consumer deliberately lets a batch failure crash the process, which is
// what preserves at-least-once. We do not DLQ here — everything on this path (quota
// lookups, the rate limiter) is infrastructure-scoped, and quarantining a whole batch
// over a Redis blip would drop good data. Name the records first so the restart loop
// that follows points somewhere.
logger.error('🔴', `${this.name} - batch failed`, {
error: String(error),
stack: error instanceof Error ? error.stack : undefined,
size: messages.length,
offsets: describeBatchPosition(messages),
})
throw error
}
}
public async start(): Promise<void> {
// Start consuming messages
await this.kafkaConsumer.connect(async (messages) => {
logger.info('🔁', `${this.name} - handling batch`, {
size: messages.length,
offsets: describeBatchPosition(messages),
})
return await instrumentFn('logsIngestionConsumer.handleEachBatch', async () => {
return await this.processKafkaBatch(messages)
})
})
}
public async stop(): Promise<void> {
logger.info('💤', 'Stopping consumer...')
await this.kafkaConsumer.disconnect()
logger.info('💤', 'Consumer stopped!')
}
public isHealthy(): HealthCheckResult {
return this.kafkaConsumer.isHealthy()
}
}