Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

81.3 KB
Raw Blame
import { trace } from '@opentelemetry/api'
import { DateTime } from 'luxon'
import { Message } from 'node-rdkafka'
import pLimit from 'p-limit'
import { Counter, Histogram } from 'prom-client'

import { KafkaConsumerInterface, createKafkaConsumer, parseKafkaHeaders } from '~/common/kafka/consumer'
import { retryOnDependencyUnavailableError } from '~/common/kafka/error-handling'
import { recordPiiReplacements } from '~/common/metrics/otel-metrics'
import { AppMetricsOutput } from '~/common/outputs'
import { IngestionOutputs } from '~/common/outputs/ingestion-outputs'
import { RedisV2, createRedisV2PoolFromConfig } from '~/common/redis/redis-v2'
import { AppMetricsAggregator } from '~/common/services/app-metrics-aggregator'
import { QuotaLimiting, QuotaResource } from '~/common/services/quota-limiting.service'
import { instrumentFn, instrumented } from '~/common/tracing/tracing-utils'
import { UsageRecordBatch } from '~/common/usage-ingestion/usage-record-batch'
import { DependencyUnavailableError } from '~/common/utils/db/error'
import { isDevEnv } from '~/common/utils/env-utils'
import { logger } from '~/common/utils/logger'
import { TeamManager } from '~/common/utils/team-manager'
import { UUID7 } from '~/common/utils/utils'
import type { LogsSettings } from '~/types'
import { HealthCheckResult, PluginServerService } from '~/types'

import { LogsIngestionConsumerConfig } from './config'
import {
    recordInternalLogs,
    recordLogMessageDlq,
    recordLogMessageDropped,
    recordLogsAllowed,
    recordLogsDropped,
    recordLogsReceived,
} from './ingestion-otel-metrics'
import {
    INTERNAL_LOGS_HEADER,
    INTERNAL_SOURCE_HEADER,
    InternalLogsVerifier,
    type InternalSourcePolicy,
    withoutInternalLogsHeaders,
} from './internal-logs'
import { MESSAGE_KEYS } from './log-pattern-mask'
import { logsPatternForcedDecodeCounter, makePatternMaskingStage } from './log-pattern-stage'
import { type PiiScrubStats } from './log-pii-scrub'
import {
    type LogRecord,
    type LogRecordsTransform,
    bufferProcessingMode,
    processLogMessageBuffer,
    sniffJsonLogAttributes,
} from './log-record-avro'
import { rethrowKafkaOutageAsDependency } from './logs-produce-errors'
import type { CompiledMetricRule, MetricRuleSource } from './metrics-rules/compile-metric-rules'
import { withDefaultMetricRules } from './metrics-rules/default-metric-rules'
import { MetricRulesCache } from './metrics-rules/metric-rules-cache'
import { LogsMetricsEmitter } from './metrics-rules/metrics-emitter'
import { buildMetricRulesOtlpPayload } from './metrics-rules/otlp-payload'
import { type BatchTallies, MetricSeriesAdmission, createBatchTallies, tallyRecords } from './metrics-rules/tally'
import { LOGS_DLQ_OUTPUT, LOGS_OUTPUT, LogsDlqOutput, LogsOutput } from './outputs/outputs'
import { PatternMessageKeysCache } from './pattern-message-keys-cache'
import { EMPTY_STAGE_DROP_STATS, type PipelineStage } from './pipeline/log-processing-pipeline'
import type { RetentionRuleSource } from './retention/compile-retention-rules'
import { type CompiledRetentionRuleSet, RETENTION_MONTH_DAYS } from './retention/evaluate-retention'
import {
    RETENTION_EXPIRED_STAGE_NAME,
    canHoldExpiredRow,
    makeRetentionExpiredStage,
} from './retention/retention-expired-stage'
import { RetentionRulesCache } from './retention/retention-rules-cache'
import { makeRetentionStage } from './retention/retention-stage'
import type { CompiledRuleSet } from './sampling/evaluate'
import { LogsSamplingService, SAMPLING_STAGE_NAME } from './sampling/logs-sampling.service'
import { SamplingRulesCache } from './sampling/sampling-rules-cache'
import { LogsRateLimiterService } from './services/logs-rate-limiter.service'
import { LogsTransformerService, TransformationBatchBudget } from './transformations/logs-transformer.service'
import { LogsIngestionMessage } from './types'

export interface LogsIngestionConsumerDeps {
    teamManager: TeamManager
    quotaLimiting: QuotaLimiting
    /** When set, enabled teams may run head sampling before ClickHouse Kafka produce. */
    samplingRulesCache?: SamplingRulesCache
    /** When set (with `metricsEmitter`), enabled teams generate metrics from matching log records. */
    metricRulesCache?: MetricRulesCache
    /** OTLP sender for log-generated metrics; required alongside `metricRulesCache` to activate the feature. */
    metricsEmitter?: LogsMetricsEmitter
    /** When set, enabled teams run hog log transformations after the built-in processing. */
    logsTransformer?: LogsTransformerService
    /** When set, enabled teams stamp per-row retention from retention rules before produce. */
    retentionRulesCache?: RetentionRulesCache
    /** When set, pattern masking reads each team's message keys instead of the defaults. */
    patternMessageKeysCache?: PatternMessageKeysCache
    /**
     * Resolved outputs registry — must include `LOGS_OUTPUT`, `LOGS_DLQ_OUTPUT`,
     * and `APP_METRICS_OUTPUT`. The producer + topic for each is wired by the
     * server via env vars — this consumer never touches a `KafkaProducerWrapper`
     * directly.
     */
    outputs: IngestionOutputs<LogsOutput | LogsDlqOutput | AppMetricsOutput>
    usageBatch: UsageRecordBatch
    /**
     * Backoff for a `DependencyUnavailableError` from the team lookup or a Kafka produce. Defaults
     * to the `retryOnDependencyUnavailableError` policy (5 tries, 1 s doubling to 16 s). Tests
     * shorten it.
     */
    dependencyRetry?: { retryCount: number; initialRetryDelayMs: number }
    /** Defaults to `INTERNAL_SOURCE_POLICIES`. Tests pass a billed source, which production does not have yet. */
    internalSourcePolicies?: Readonly<Record<string, InternalSourcePolicy>>
}

/** Ingestion default when `logs_settings.retention_days` is unset; must match `DEFAULT_LOGS_RETENTION_DAYS` in `posthog/models/team/logs_retention.py`. */
export const DEFAULT_LOGS_RETENTION_DAYS = 14

// A producer can stamp internal rows with less retention than the team default.
export const INTERNAL_LOGS_MIN_RETENTION_DAYS = 1

/** Retention day counts that get their own per-tier usage metric. */
const RETENTION_USAGE_TIERS = new Set([14, 30, 90])

export function retentionBytesMetricName(retentionDays: number): string | null {
    return RETENTION_USAGE_TIERS.has(retentionDays) ? `bytes_ingested_retention_${retentionDays}d` : null
}

export type UsageStats = {
    bytesReceived: number
    recordsReceived: number
    bytesAllowed: number
    recordsAllowed: number
    /** Sum of per-record content sizes for allowed batches — billing comparison candidate for bytesAllowed. */
    bytesAllowedRecords: number
    bytesDropped: number
    recordsDropped: number
    piiReplacements: number
    retentionDays: number
}

/** `raw` is a comma-separated team-ID list, `*` for all teams, or empty for none. */
export function teamIdMatchesCsv(raw: string, teamId: number): boolean {
    const trimmed = (raw || '').trim()
    if (!trimmed) {
        return false
    }
    if (trimmed === '*') {
        return true
    }
    return trimmed
        .split(',')
        .map((s) => parseInt(s.trim(), 10))
        .filter((n) => !Number.isNaN(n))
        .includes(teamId)
}

/**
 * Per-partition offset span of a batch, for the batch log line. A consumer that dies mid-batch
 * writes nothing after it, so this is what names the records it was holding.
 */
export function describeBatchPosition(messages: Message[]): Record<string, string> {
    const spans = new Map<number, { min: number; max: number }>()
    for (const message of messages) {
        const span = spans.get(message.partition)
        if (span) {
            span.min = Math.min(span.min, message.offset)
            span.max = Math.max(span.max, message.offset)
        } else {
            spans.set(message.partition, { min: message.offset, max: message.offset })
        }
    }

    const spanByPartition: Record<string, string> = {}
    for (const [partition, { min, max }] of spans) {
        spanByPartition[partition.toString()] = min === max ? min.toString() : `${min}-${max}`
    }
    return spanByPartition
}

/**
 * Parse a batch size header, or `null` when the value is unusable. These feed usage counters and
 * billing stats, so a malformed one must not reach them: `parseInt` returns NaN for junk, and
 * prom-client's own guard treats NaN as falsy and lets it through, which leaves the counter NaN
 * for the life of the process. Parsing stays as lenient as before — only NaN and negatives, which
 * no caller can use, are rejected.
 */
export function parseSizeHeader(raw: string | undefined): number | null {
    const parsed = parseInt(raw ?? '0', 10)
    return Number.isFinite(parsed) && parsed >= 0 ? parsed : null
}

export function parseMinTimestampHeader(raw: string | undefined): number | undefined {
    if (raw === undefined || raw.trim() === '') {
        return undefined
    }
    const parsed = Number(raw)
    return Number.isSafeInteger(parsed) ? parsed : undefined
}

export const DEFAULT_USAGE_STATS: UsageStats = {
    bytesReceived: 0,
    recordsReceived: 0,
    bytesAllowed: 0,
    recordsAllowed: 0,
    bytesAllowedRecords: 0,
    bytesDropped: 0,
    recordsDropped: 0,
    piiReplacements: 0,
    retentionDays: DEFAULT_LOGS_RETENTION_DAYS,
}

export type UsageStatsByTeam = Map<number, UsageStats>

/** Cap concurrent per-message processing within a single kafka batch. */
export const MAX_CONCURRENT_MESSAGE_PROCESSES = 50

export const logMessageDroppedCounter = new Counter({
    name: 'logs_ingestion_message_dropped_count',
    help: 'The number of logs ingestion messages dropped',
    labelNames: ['reason', 'team_id'],
})

export const logMessageDlqCounter = new Counter({
    name: 'logs_ingestion_message_dlq_count',
    help: 'The number of logs ingestion messages sent to DLQ',
    labelNames: ['reason', 'team_id'],
})

export const logsBytesReceivedCounter = new Counter({
    name: 'logs_ingestion_bytes_received_total',
    help: 'Total uncompressed bytes received for logs ingestion',
})

export const logsBytesAllowedCounter = new Counter({
    name: 'logs_ingestion_bytes_allowed_total',
    help: 'Total uncompressed bytes allowed through quota and rate limiting. Gross of the drop-rule billing credit: billed bytes_ingested = this − logs_ingestion_billing_bytes_credited_total.',
})

export const logsBytesAllowedRecordsCounter = new Counter({
    name: 'logs_ingestion_bytes_allowed_records_total',
    help: 'Records-based uncompressed bytes (sum of per-record sizes) allowed through quota and rate limiting',
})

export const logsRecordsBytesExceedPayloadCounter = new Counter({
    name: 'logs_ingestion_records_bytes_exceed_payload_total',
    help: 'Batches where the records-based bytes sum exceeded the payload-based bytes_uncompressed header',
    labelNames: ['team_id'],
})

export const logsBytesDroppedCounter = new Counter({
    name: 'logs_ingestion_bytes_dropped_total',
    help: 'Total uncompressed bytes dropped due to quota or rate limiting',
    labelNames: ['team_id'],
})

export const logsRecordsReceivedCounter = new Counter({
    name: 'logs_ingestion_records_received_total',
    help: 'Total log records received',
})

export const logsInternalBytesCounter = new Counter({
    name: 'logs_ingestion_internal_bytes_total',
    help: 'Uncompressed bytes of verified internal logs, which stay out of the received and allowed counters.',
    labelNames: ['team_id'],
})

export const logsInternalRecordsCounter = new Counter({
    name: 'logs_ingestion_internal_records_total',
    help: 'Records of verified internal logs, which stay out of the received and allowed counters.',
    labelNames: ['team_id'],
})

export const logsRecordsAllowedCounter = new Counter({
    name: 'logs_ingestion_records_allowed_total',
    help: 'Total log records allowed through quota and rate limiting. Gross of the drop-rule billing credit: billed records_ingested = this − logs_ingestion_billing_records_credited_total.',
})

export const logsRecordsDroppedCounter = new Counter({
    name: 'logs_ingestion_records_dropped_total',
    help: 'Total log records dropped due to quota or rate limiting',
    labelNames: ['team_id'],
})

export const logsSamplingRecordsDroppedCounter = new Counter({
    name: 'logs_ingestion_sampling_records_dropped_total',
    help: 'Log records dropped by head sampling rules',
    labelNames: ['team_id'],
})

export const logsBytesDroppedByRuleCounter = new Counter({
    name: 'logs_ingestion_bytes_dropped_by_rule_total',
    help: 'Bytes dropped by drop rules, summed from per-row bytes_uncompressed.',
    labelNames: ['team_id'],
})

// --- Billing impact of drop rules (Tier 1) ---
export const logsBillingBytesCreditedCounter = new Counter({
    name: 'logs_ingestion_billing_bytes_credited_total',
    help: 'Uncompressed header bytes credited back to billing for rows removed by drop rules (pro-rated).',
    labelNames: ['team_id'],
})

export const logsBillingRecordsCreditedCounter = new Counter({
    name: 'logs_ingestion_billing_records_credited_total',
    help: 'Records removed by drop rules and credited back to billing.',
    labelNames: ['team_id'],
})

// --- Metric rules (generate metrics from logs) ---
export const logsMetricRulesRecordsMatchedCounter = new Counter({
    name: 'logs_metrics_rules_records_matched_total',
    help: 'Log records that matched a metric rule and fed a generated metric data point.',
    labelNames: ['team_id'],
})

export const logsMetricRulesValueSkippedCounter = new Counter({
    name: 'logs_metrics_rules_value_skipped_total',
    help: 'Records matching a value-attribute metric rule whose value was missing or non-numeric.',
    labelNames: ['team_id'],
})

export const logsMetricRulesSeriesOverflowCounter = new Counter({
    name: 'logs_metrics_rules_series_overflow_total',
    help: 'Matching records dropped because a metric rule exceeded its per-batch label-set cap.',
    labelNames: ['team_id'],
})

export const logsMetricRulesEvalErrorCounter = new Counter({
    name: 'logs_metrics_rules_eval_error_total',
    help: 'Per-record metric-rule filter evaluations that threw; the record was skipped for that rule.',
    labelNames: ['team_id'],
})

// --- Pro-rate accuracy-confidence signals (Tier 2). No team_id label — kept low-cardinality. ---
export const logsBillingProrateDivergenceHistogram = new Histogram({
    name: 'logs_ingestion_billing_prorate_divergence',
    help: 'Per-message |content-weighted − record-weighted credit| / header. High = size-skewed batch = pro-rate less trustworthy.',
    buckets: [0.001, 0.005, 0.01, 0.05, 0.1, 0.25, 0.5, 1],
})

export const logsDropBatchRowsHistogram = new Histogram({
    name: 'logs_ingestion_drop_batch_rows',
    help: 'Rows per batch for messages that had drop-rule drops; small batches carry larger pro-rate error.',
    buckets: [1, 2, 5, 10, 50, 100, 500, 1000, 5000],
})

export const logsDropFractionHistogram = new Histogram({
    name: 'logs_ingestion_drop_fraction',
    help: 'Fraction of a message content dropped by drop rules; extreme fractions carry larger pro-rate error.',
    buckets: [0.1, 0.25, 0.5, 0.75, 0.9, 0.99, 1],
})

/**
 * Pro-rate a message's billable uncompressed bytes by the fraction of its content that drop
 * rules removed. Billing is on the per-message header (whole-batch wire size), but drops are
 * decided per row (content bytes), so we scale the header down by the dropped content fraction
 * to bill only what survived.
 *
 * Weights come from `measureContentWeights`: every sender-controlled field of a row, with resource
 * and scope bytes split across the rows that share them. The per-row `bytes_uncompressed` field is
 * not used, because it counts shared resource attributes once per row. The result is always ≤ the gross
 * header (droppedFraction ≤ 1), so a message is never billed above today's gross. Returns 0 when
 * we can't measure (no header or no content bytes), i.e. no credit rather than a wrong one.
 */
export function billingByteReductionForDrops(headerBytes: number, bytesDropped: number, bytesTotal: number): number {
    if (headerBytes <= 0 || bytesDropped <= 0 || bytesTotal <= 0) {
        return 0
    }
    const droppedFraction = Math.min(1, bytesDropped / bytesTotal)
    return Math.round(headerBytes * droppedFraction)
}

/**
 * Wraps a hog log transform (mutates the surviving records in place, may drop) as the last pipeline
 * `filter` stage. Partial transform drops are not billed-credited — only the all-dropped case is
 * attributed — so it reports no per-rule accounting, just the `transformations` tag when it removed
 * any record.
 */
export function makeTransformStage(transform: LogRecordsTransform): PipelineStage {
    return {
        kind: 'filter',
        name: 'transformations',
        run: async (records) => {
            await transform(records)
            return { kept: records, stats: EMPTY_STAGE_DROP_STATS() }
        },
    }
}

type MetricTalliesByTeam = Map<number, { token: string; rules: CompiledMetricRule[]; tallies: BatchTallies }>

export class LogsIngestionConsumer {
    protected name = 'LogsIngestionConsumer'
    // Billing identity for quota enforcement and usage metering; overridden by subclasses (e.g. traces).
    protected quotaResource: QuotaResource = 'logs_mb_ingested'
    protected appSource = 'logs'
    // Record source this consumer tallies metric rules for. `appSource` is the
    // billing/telemetry identity ('logs' | 'traces'); metric rules are tagged with the
    // record source ('logs' | 'spans') instead, so map between the two explicitly
    // rather than comparing across vocabularies. TracesIngestionConsumer overrides to 'spans'.
    protected metricRuleSource: MetricRuleSource = 'logs'
    // Record source this consumer evaluates retention rules for. Same vocabulary as
    // `metricRuleSource` ('logs' | 'spans'), not the billing `appSource`.
    protected retentionRuleSource: RetentionRuleSource = 'logs'
    protected kafkaConsumer: KafkaConsumerInterface
    private appMetricsAggregator: AppMetricsAggregator
    private redis: RedisV2
    private rateLimiter: LogsRateLimiterService
    private readonly backfillEnabledTeamsRaw: string
    private readonly internalLogsVerifier: InternalLogsVerifier
    private samplingService: LogsSamplingService
    private readonly samplingEnabledTeamsRaw: string
    private readonly samplingKillswitch: boolean
    private readonly billingProrateEnabled: boolean
    private readonly metricRulesEnabledTeamsRaw: string
    private readonly metricRulesKillswitch: boolean
    private readonly defaultMetricRulesEnabledTeamsRaw: string
    private readonly metricSeriesAdmission = new MetricSeriesAdmission()
    private readonly transformationsEnabledTeamsRaw: string
    private readonly transformationsKillswitch: boolean
    private readonly retentionEnabledTeamsRaw: string
    private readonly retentionKillswitch: boolean
    private readonly patternMaskingEnabledTeamsRaw: string
    private readonly jsonAttributeParsingEnabledTeamsRaw: string
    private readonly jsonAttributeExtractionEnabledTeamsRaw: string

    protected groupId: string
    protected topic: string

    constructor(
        config: LogsIngestionConsumerConfig,
        private deps: LogsIngestionConsumerDeps,
        overrides: Partial<LogsIngestionConsumerConfig> = {},
        // Redis key namespace for the token-bucket rate limiter. Subclasses (e.g. traces) pass
        // their own so their per-team buckets don't share state with logs. Defaults to logs.
        rateLimiterName: string = 'logs-rate-limiter'
    ) {
        // Merge overrides once so every downstream consumer reads the same effective config —
        // a per-use-site `overrides.X ?? config.X` pattern is easy to forget (the rate limiter did).
        const mergedConfig = { ...config, ...overrides }

        // The group and topic are configurable allowing for multiple ingestion consumers to be run in parallel
        this.groupId = mergedConfig.LOGS_INGESTION_CONSUMER_GROUP_ID
        this.topic = mergedConfig.LOGS_INGESTION_CONSUMER_CONSUME_TOPIC

        this.appMetricsAggregator = new AppMetricsAggregator(deps.outputs)

        this.kafkaConsumer = createKafkaConsumer({ groupId: this.groupId, topic: this.topic })
        // Logs ingestion uses its own Redis instance with TLS support
        this.redis = createRedisV2PoolFromConfig({
            connection: mergedConfig.LOGS_REDIS_HOST
                ? {
                      url: mergedConfig.LOGS_REDIS_HOST,
                      options: {
                          port: mergedConfig.LOGS_REDIS_PORT,
                          tls: mergedConfig.LOGS_REDIS_TLS ? {} : undefined,
                      },
                      name: 'logs-redis',
                  }
                : { url: mergedConfig.REDIS_URL, name: 'logs-redis-fallback' },
            poolMinSize: mergedConfig.REDIS_POOL_MIN_SIZE,
            poolMaxSize: mergedConfig.REDIS_POOL_MAX_SIZE,
        })
        this.rateLimiter = new LogsRateLimiterService(mergedConfig, this.redis, rateLimiterName)
        this.backfillEnabledTeamsRaw = mergedConfig.LOGS_BACKFILL_ENABLED_TEAMS
        this.internalLogsVerifier = new InternalLogsVerifier(
            mergedConfig.LOGS_INTERNAL_INGESTION_SIGNING_KEYS,
            mergedConfig.LOGS_INTERNAL_INGESTION_ENABLED_TEAMS,
            deps.internalSourcePolicies
        )
        this.samplingService = new LogsSamplingService(this.redis, mergedConfig.LOGS_LIMITER_TTL_SECONDS)
        this.samplingEnabledTeamsRaw = mergedConfig.LOGS_SAMPLING_ENABLED_TEAMS
        this.samplingKillswitch = mergedConfig.LOGS_SAMPLING_KILLSWITCH
        this.billingProrateEnabled = mergedConfig.LOGS_BILLING_PRORATE_ENABLED
        this.metricRulesEnabledTeamsRaw = mergedConfig.LOGS_METRICS_RULES_ENABLED_TEAMS
        this.metricRulesKillswitch = mergedConfig.LOGS_METRICS_RULES_KILLSWITCH
        this.defaultMetricRulesEnabledTeamsRaw = mergedConfig.LOGS_DEFAULT_METRIC_RULES_ENABLED_TEAMS
        this.transformationsEnabledTeamsRaw = mergedConfig.LOGS_TRANSFORMATIONS_ENABLED_TEAMS
        this.transformationsKillswitch = mergedConfig.LOGS_TRANSFORMATIONS_KILLSWITCH
        this.retentionEnabledTeamsRaw = mergedConfig.LOGS_RETENTION_ENABLED_TEAMS
        this.retentionKillswitch = mergedConfig.LOGS_RETENTION_KILLSWITCH
        this.patternMaskingEnabledTeamsRaw = mergedConfig.LOGS_PATTERN_MASKING_ENABLED_TEAMS
        this.jsonAttributeParsingEnabledTeamsRaw = mergedConfig.LOGS_JSON_ATTRIBUTE_PARSING_ENABLED_TEAMS
        this.jsonAttributeExtractionEnabledTeamsRaw = mergedConfig.LOGS_JSON_ATTRIBUTE_EXTRACTION_ENABLED_TEAMS
    }

    private isSamplingEvalEnabledForTeam(teamId: number): boolean {
        if (this.samplingKillswitch) {
            return false
        }
        return teamIdMatchesCsv(this.samplingEnabledTeamsRaw, teamId)
    }

    private isRetentionEvalEnabledForTeam(teamId: number): boolean {
        if (this.retentionKillswitch) {
            return false
        }
        return teamIdMatchesCsv(this.retentionEnabledTeamsRaw, teamId)
    }

    /**
     * The team's default retention period, applied to records no rule matches and sent as the
     * batch `retention-days` Kafka header. Traces override this to read their own setting.
     */
    protected defaultRetentionDays(_teamId: number, logsSettings: LogsSettings): Promise<number> {
        return Promise.resolve(logsSettings.retention_days ?? DEFAULT_LOGS_RETENTION_DAYS)
    }

    private canEmitMetricRules(): boolean {
        return !this.metricRulesKillswitch && !!this.deps.metricRulesCache && !!this.deps.metricsEmitter
    }

    private isMetricRulesEnabledForTeam(teamId: number): boolean {
        return this.canEmitMetricRules() && teamIdMatchesCsv(this.metricRulesEnabledTeamsRaw, teamId)
    }

    private isDefaultMetricRulesEnabledForTeam(teamId: number): boolean {
        return this.canEmitMetricRules() && teamIdMatchesCsv(this.defaultMetricRulesEnabledTeamsRaw, teamId)
    }

    // Transformations write hog_function logs, which could come back here as internal logs and loop.
    private appliesTeamLogRules(message: LogsIngestionMessage): boolean {
        return message.verifiedAsInternalLogs?.teamRules ?? true
    }

    private isUnbilledInternal(message: LogsIngestionMessage): boolean {
        return message.verifiedAsInternalLogs !== undefined && !message.verifiedAsInternalLogs.billed
    }

    private isTransformationsEnabledForTeam(teamId: number): boolean {
        if (this.transformationsKillswitch || !this.deps.logsTransformer) {
            return false
        }
        return teamIdMatchesCsv(this.transformationsEnabledTeamsRaw, teamId)
    }

    /**
     * Logs only. `TracesIngestionConsumer` subclasses this one and reads the same config key, but a
     * trace record has no `body` field, so masking one measures nothing and would mix trace shapes
     * into the log-body split these metrics exist to produce.
     */
    private isPatternMaskingEnabledForTeam(teamId: number): boolean {
        return this.appSource === 'logs' && teamIdMatchesCsv(this.patternMaskingEnabledTeamsRaw, teamId)
    }

    /**
     * Builds the hog log transformation hook for a message, or undefined when the team
     * is not gated in or has no enabled transformation_log functions (the existence
     * check is an in-process cache hit, preserving the no-decode passthrough).
     */
    private async buildRecordsTransform(
        message: LogsIngestionMessage,
        batchBudget?: TransformationBatchBudget
    ): Promise<LogRecordsTransform | undefined> {
        const transformer = this.deps.logsTransformer
        if (
            !transformer ||
            !this.appliesTeamLogRules(message) ||
            !this.isTransformationsEnabledForTeam(message.teamId)
        ) {
            return undefined
        }
        if (!(await transformer.teamHasTransformations(message.teamId))) {
            return undefined
        }
        return (records) =>
            transformer.transformRecords(message.teamId, records, batchBudget, message.sentAsInternalLogs)
    }

    /**
     * Decode + optional head sampling, or passthrough `processLogMessageBuffer`.
     * `all_dropped` means do not enqueue to logs output (every record was sampled out
     * or dropped by transformations); `reason` distinguishes the source.
     */
    private async resolveLogMessageBufferWithOptionalSampling(
        message: LogsIngestionMessage,
        logsSettings: LogsSettings,
        defaultRetentionDays: number,
        onRecordsDecoded?: (records: LogRecord[]) => void,
        batchBudget?: TransformationBatchBudget
    ): Promise<
        | {
              outcome: 'produce'
              processedValue: Buffer
              pii: PiiScrubStats
              recordsDropped: number
              recordsDroppedByStage: Map<string, number>
              recordsDroppedByRuleId: Map<string, number>
              bytesDroppedByRuleId: Map<string, number>
              contentBytesDropped: number
              contentBytesTotal: number
          }
        | {
              outcome: 'all_dropped'
              reason: string
              pii: PiiScrubStats
              recordsDropped: number
              recordsDroppedByStage: Map<string, number>
              recordsDroppedByRuleId: Map<string, number>
              bytesDroppedByRuleId: Map<string, number>
              contentBytesDropped: number
              contentBytesTotal: number
          }
    > {
        const samplingCache = this.deps.samplingRulesCache
        const samplingEvalEnabled = this.isSamplingEvalEnabledForTeam(message.teamId)
        let ruleSet: CompiledRuleSet | null = null
        if (samplingCache && samplingEvalEnabled && this.appliesTeamLogRules(message)) {
            ruleSet = await samplingCache.getCompiledRuleSet(message.teamId)
        }
        const useSamplingPipeline = Boolean(ruleSet && ruleSet.rules.length > 0)
        const recordsTransform = await this.buildRecordsTransform(message, batchBudget)

        // Per-row retention: resolve the team's retention rules when enabled. Any matching rule stamps
        // `retention_days` per surviving record (falling back to the team default), which ClickHouse
        // reads in place of the batch `retention-days` header. With no rules the header — still emitted
        // below — carries the team default, keeping this a no-decode passthrough.
        const retentionCache = this.deps.retentionRulesCache
        const retentionEvalEnabled = this.isRetentionEvalEnabledForTeam(message.teamId)
        let retentionRuleSet: CompiledRetentionRuleSet | null = null
        if (retentionCache && retentionEvalEnabled && this.appliesTeamLogRules(message)) {
            retentionRuleSet = await retentionCache.getCompiledRuleSet(message.teamId, this.retentionRuleSource)
        }
        const useRetention = Boolean(retentionRuleSet && retentionRuleSet.rules.length > 0)

        // Assemble the ordered decode → transform → encode pipeline: sampling drop rules + rate limits
        // first, hog transformations next, per-row retention stamping last (so it only stamps
        // survivors). An empty list (with no metric visitor) leaves the buffer as a no-decode passthrough
        // inside `processLogMessageBuffer`.
        const stages: PipelineStage[] = []
        if (useSamplingPipeline && ruleSet) {
            stages.push(this.samplingService.makeSamplingStage(ruleSet, message.teamId, message.bytesUncompressed))
        }
        if (recordsTransform) {
            stages.push(makeTransformStage(recordsTransform))
        }
        let shortestRetentionDays =
            message.verifiedAsInternalLogs !== undefined ? INTERNAL_LOGS_MIN_RETENTION_DAYS : defaultRetentionDays
        if (useRetention && retentionRuleSet) {
            stages.push(makeRetentionStage(retentionRuleSet, message.teamId, defaultRetentionDays))
            shortestRetentionDays = Math.min(
                defaultRetentionDays,
                ...retentionRuleSet.rules.map((rule) => rule.retentionDays)
            )
        }

        // Checking rows for expiry needs a decode, which every message would otherwise pay to find rows
        // that only backdated imports produce. The stage runs after the retention stage, so it reads the
        // per-row retention that ClickHouse will use.
        const nowMicros = Date.now() * 1000
        if (canHoldExpiredRow(message.minTimestampMicros, shortestRetentionDays, nowMicros)) {
            stages.push(makeRetentionExpiredStage(message.teamId, defaultRetentionDays, nowMicros))
        }

        // Runs last so it only sees survivors. Adding any stage forces the full decode and re-encode,
        // so a batch that would have passed through pays both, and one already decoded for a visitor
        // pays the encode. The counter prices each.
        if (this.isPatternMaskingEnabledForTeam(message.teamId)) {
            const modeWithoutMasking = bufferProcessingMode(logsSettings, stages.length, Boolean(onRecordsDecoded))
            if (modeWithoutMasking !== 'decode_and_reencode') {
                logsPatternForcedDecodeCounter.inc({ from: modeWithoutMasking })
            }
            const keysCache = this.deps.patternMessageKeysCache
            const messageKeys = keysCache
                ? await this.retryOnDependencyUnavailable(() => keysCache.getMessageKeys(message.teamId))
                : MESSAGE_KEYS
            stages.push(makePatternMaskingStage(messageKeys))
        }

        trace.getActiveSpan()?.setAttributes({
            'logs.sampling.killswitch': this.samplingKillswitch,
            'logs.sampling.enabled_teams_configured': Boolean((this.samplingEnabledTeamsRaw || '').trim()),
            'logs.sampling.enabled_teams_is_wildcard': (this.samplingEnabledTeamsRaw || '').trim() === '*',
            'logs.sampling.cache_present': Boolean(samplingCache),
            'logs.sampling.eval_enabled_for_team': samplingEvalEnabled,
            'logs.sampling.compiled_rule_count': ruleSet?.rules.length ?? 0,
            'logs.transformations.enabled_for_team': Boolean(recordsTransform),
            'logs.retention.cache_present': Boolean(retentionCache),
            'logs.retention.eval_enabled_for_team': retentionEvalEnabled,
            'logs.retention.compiled_rule_count': retentionRuleSet?.rules.length ?? 0,
            'logs.retention.use_retention': useRetention,
            'logs.sampling.pipeline': useSamplingPipeline
                ? 'decode_sample_encode'
                : recordsTransform
                  ? 'decode_transform_encode'
                  : 'passthrough',
        })

        const { value, pii, drops } = await processLogMessageBuffer(message.message.value!, logsSettings, {
            onRecordsDecoded,
            stages,
        })

        const recordsDroppedBySampling = drops.recordsDroppedByStage.get(SAMPLING_STAGE_NAME) ?? 0
        if (recordsDroppedBySampling > 0) {
            logsSamplingRecordsDroppedCounter.inc({ team_id: message.teamId.toString() }, recordsDroppedBySampling)
        }
        if (drops.bytesDropped > 0) {
            logsBytesDroppedByRuleCounter.inc({ team_id: message.teamId.toString() }, drops.bytesDropped)
        }

        if (value === null) {
            // `droppedBy` tells us which filter emptied the batch; its absence means the batch decoded
            // to zero records to begin with, so attribute it to an empty batch rather than sampling.
            const reason = drops.droppedBy ? `${drops.droppedBy}_all_dropped` : 'empty_batch'
            return {
                outcome: 'all_dropped',
                reason,
                pii,
                recordsDropped: drops.recordsDropped,
                recordsDroppedByStage: drops.recordsDroppedByStage,
                recordsDroppedByRuleId: drops.recordsDroppedByRuleId,
                bytesDroppedByRuleId: drops.bytesDroppedByRuleId,
                contentBytesDropped: drops.contentBytesDropped,
                contentBytesTotal: drops.contentBytesTotal,
            }
        }
        return {
            outcome: 'produce',
            processedValue: value,
            pii,
            recordsDropped: drops.recordsDropped,
            recordsDroppedByStage: drops.recordsDroppedByStage,
            recordsDroppedByRuleId: drops.recordsDroppedByRuleId,
            bytesDroppedByRuleId: drops.bytesDroppedByRuleId,
            contentBytesDropped: drops.contentBytesDropped,
            contentBytesTotal: drops.contentBytesTotal,
        }
    }

    public get service(): PluginServerService {
        return {
            id: this.name,
            onShutdown: async () => await this.stop(),
            healthcheck: () => this.isHealthy(),
        }
    }

    public async processBatch(
        messages: LogsIngestionMessage[]
    ): Promise<{ backgroundTask?: Promise<any>; messages: LogsIngestionMessage[] }> {
        if (!messages.length) {
            return { messages: [] }
        }

        this.trackIncomingTraffic(messages)

        const { backfillAllowedMessages, backfillDroppedMessages } = this.filterBackfillNotEnabledMessages(messages)
        const internalMessages: LogsIngestionMessage[] = []
        const billedMessages: LogsIngestionMessage[] = []
        for (const message of backfillAllowedMessages) {
            ;(this.isUnbilledInternal(message) ? internalMessages : billedMessages).push(message)
        }
        const { quotaAllowedMessages, quotaDroppedMessages } = await this.filterQuotaLimitedMessages(billedMessages)
        const { rateLimiterAllowedMessages, rateLimiterDroppedMessages } =
            await this.filterRateLimitedMessages(quotaAllowedMessages)

        const usageStats = this.trackOutgoingTrafficAndBuildUsageStats(rateLimiterAllowedMessages, [
            ...backfillDroppedMessages.filter((message) => !this.isUnbilledInternal(message)),
            ...quotaDroppedMessages,
            ...rateLimiterDroppedMessages,
        ])

        // One transformation time budget shared by all messages of this batch
        const transformationBatchBudget = this.deps.logsTransformer?.startBatch()

        return {
            // Produce first so PII replacement counts are folded into `usageStats` before MSK usage emit
            backgroundTask: (async () => {
                const metricTalliesByTeam = await this.processAndProduceLogMessages(
                    rateLimiterAllowedMessages,
                    usageStats,
                    transformationBatchBudget
                )
                // A usage map that is never emitted keeps their stats off the team's billed row.
                // They run after the billed batches, so the per-batch concurrency limit still holds.
                await this.processAndProduceLogMessages(internalMessages, new Map(), transformationBatchBudget)
                // Metric rules and internal usage emit deltas, so they wait until no group can fail the batch into a replay.
                this.queueInternalLogsMetrics(internalMessages)
                await this.emitMetricRuleTallies(metricTalliesByTeam)
                await this.deps.teamManager.setTeamsIngestedData(
                    rateLimiterAllowedMessages.map((message) => message.teamId)
                )
                await this.emitUsageMetrics(usageStats)
                // Best-effort flush of transformation app metrics + function logs; never block the data path
                if (this.deps.logsTransformer) {
                    await this.deps.logsTransformer.flush().catch((error) => {
                        logger.error('Failed to flush logs transformer monitoring', { error: String(error) })
                    })
                }
            })(),
            messages: [...rateLimiterAllowedMessages, ...internalMessages],
        }
    }

    // The usage report matches `bytes_ingested` exactly, so these names never reach billing.
    private queueInternalLogsMetrics(internalMessages: LogsIngestionMessage[]): void {
        for (const message of internalMessages) {
            this.queueUsageMetric(message.teamId, 'bytes_ingested_internal', message.bytesUncompressed)
            this.queueUsageMetric(message.teamId, 'records_ingested_internal', message.recordCount)
            const teamId = message.teamId.toString()
            logsInternalBytesCounter.inc({ team_id: teamId }, message.bytesUncompressed)
            logsInternalRecordsCounter.inc({ team_id: teamId }, message.recordCount)
            recordInternalLogs(message.teamId, message.bytesUncompressed, message.recordCount)
        }
    }

    private trackIncomingTraffic(messages: LogsIngestionMessage[]): void {
        let totalBytesReceived = 0
        let totalRecordsReceived = 0
        for (const message of messages) {
            // Never quota or rate limited, so counting them would show as drops in received minus allowed.
            if (this.isUnbilledInternal(message)) {
                continue
            }
            totalBytesReceived += message.bytesUncompressed
            totalRecordsReceived += message.recordCount
        }
        logsBytesReceivedCounter.inc(totalBytesReceived)
        logsRecordsReceivedCounter.inc(totalRecordsReceived)
        recordLogsReceived(totalBytesReceived, totalRecordsReceived)
    }

    private trackOutgoingTrafficAndBuildUsageStats(
        allowedMessages: LogsIngestionMessage[],
        droppedMessages: LogsIngestionMessage[]
    ): UsageStatsByTeam {
        let totalBytesAllowed = 0
        let totalRecordsAllowed = 0
        const usageStats: UsageStatsByTeam = new Map()

        let totalBytesAllowedRecords = 0
        for (const message of allowedMessages) {
            const stats = usageStats.get(message.teamId) || { ...DEFAULT_USAGE_STATS }
            stats.bytesReceived += message.bytesUncompressed
            stats.recordsReceived += message.recordCount
            stats.bytesAllowed += message.bytesUncompressed
            stats.recordsAllowed += message.recordCount
            stats.bytesAllowedRecords += message.bytesUncompressedRecords
            usageStats.set(message.teamId, stats)

            totalBytesAllowed += message.bytesUncompressed
            totalBytesAllowedRecords += message.bytesUncompressedRecords
            totalRecordsAllowed += message.recordCount
        }

        logsBytesAllowedRecordsCounter.inc(totalBytesAllowedRecords)
        // Gross, before the drop-rule billing credit applied in processAndProduceLogMessages (see counter help).
        logsBytesAllowedCounter.inc(totalBytesAllowed)
        logsRecordsAllowedCounter.inc(totalRecordsAllowed)
        recordLogsAllowed(totalBytesAllowed, totalRecordsAllowed)

        for (const message of droppedMessages) {
            const stats = usageStats.get(message.teamId) || { ...DEFAULT_USAGE_STATS }
            stats.bytesReceived += message.bytesUncompressed
            stats.recordsReceived += message.recordCount
            stats.bytesDropped += message.bytesUncompressed
            stats.recordsDropped += message.recordCount
            usageStats.set(message.teamId, stats)
        }

        for (const [teamId, stats] of usageStats) {
            const teamIdLabel = teamId.toString()
            if (stats.bytesDropped > 0) {
                logsBytesDroppedCounter.inc({ team_id: teamIdLabel }, stats.bytesDropped)
            }
            if (stats.recordsDropped > 0) {
                logsRecordsDroppedCounter.inc({ team_id: teamIdLabel }, stats.recordsDropped)
            }
            recordLogsDropped(teamId, stats.bytesDropped, stats.recordsDropped)
        }

        return usageStats
    }

    private addPiiStatsIntoUsage(usage: UsageStatsByTeam, teamId: number, delta: PiiScrubStats): void {
        if (delta.piiReplacements === 0) {
            return
        }
        recordPiiReplacements(this.appSource, delta.piiReplacements)
        const row = usage.get(teamId) || { ...DEFAULT_USAGE_STATS }
        row.piiReplacements += delta.piiReplacements
        usage.set(teamId, row)
    }

    // Clamping these rows to the ingest time would write the import onto today, so the whole message is dropped.
    private filterBackfillNotEnabledMessages(messages: LogsIngestionMessage[]): {
        backfillAllowedMessages: LogsIngestionMessage[]
        backfillDroppedMessages: LogsIngestionMessage[]
    } {
        const backfillAllowedMessages: LogsIngestionMessage[] = []
        const backfillDroppedMessages: LogsIngestionMessage[] = []
        const droppedRecordsByTeam = new Map<number, { messages: number; records: number; billed: boolean }>()
        for (const message of messages) {
            if (message.backfillRequested && !teamIdMatchesCsv(this.backfillEnabledTeamsRaw, message.teamId)) {
                backfillDroppedMessages.push(message)
                const dropped = droppedRecordsByTeam.get(message.teamId) ?? { messages: 0, records: 0, billed: false }
                dropped.messages++
                if (!this.isUnbilledInternal(message)) {
                    dropped.records += message.recordCount
                    dropped.billed = true
                }
                droppedRecordsByTeam.set(message.teamId, dropped)
            } else {
                backfillAllowedMessages.push(message)
            }
        }

        for (const [teamId, dropped] of droppedRecordsByTeam) {
            logMessageDroppedCounter.inc(
                { reason: 'backfill_not_enabled', team_id: teamId.toString() },
                dropped.messages
            )
            recordLogMessageDropped('backfill_not_enabled', teamId.toString(), dropped.messages)
            if (dropped.billed) {
                this.queueUsageMetric(teamId, 'records_dropped_backfill_not_enabled', dropped.records)
            }
        }

        return { backfillAllowedMessages, backfillDroppedMessages }
    }

    private async filterQuotaLimitedMessages(
        messages: LogsIngestionMessage[]
    ): Promise<{ quotaAllowedMessages: LogsIngestionMessage[]; quotaDroppedMessages: LogsIngestionMessage[] }> {
        const quotaDroppedMessages: LogsIngestionMessage[] = []
        const quotaAllowedMessages: LogsIngestionMessage[] = []

        const uniqueTokens = [...new Set(messages.map((m) => m.token))]

        const quotaLimitedTokens = new Set(
            (
                await Promise.all(
                    uniqueTokens.map(async (token) =>
                        (await this.deps.quotaLimiting.isTeamTokenQuotaLimited(token, this.quotaResource))
                            ? token
                            : null
                    )
                )
            ).filter((token): token is string => token !== null)
        )

        const droppedCountByTeam = new Map<number, number>()
        for (const message of messages) {
            if (quotaLimitedTokens.has(message.token)) {
                quotaDroppedMessages.push(message)
                droppedCountByTeam.set(message.teamId, (droppedCountByTeam.get(message.teamId) || 0) + 1)
            } else {
                quotaAllowedMessages.push(message)
            }
        }

        for (const [teamId, count] of droppedCountByTeam) {
            logMessageDroppedCounter.inc({ reason: 'quota_limited', team_id: teamId.toString() }, count)
            recordLogMessageDropped('quota_limited', teamId.toString(), count)
        }

        return { quotaAllowedMessages, quotaDroppedMessages }
    }

    private async filterRateLimitedMessages(messages: LogsIngestionMessage[]): Promise<{
        rateLimiterAllowedMessages: LogsIngestionMessage[]
        rateLimiterDroppedMessages: LogsIngestionMessage[]
    }> {
        const { allowed, dropped } = await this.rateLimiter.filterMessages(messages)

        const droppedCountByTeam = new Map<number, number>()
        for (const message of dropped) {
            droppedCountByTeam.set(message.teamId, (droppedCountByTeam.get(message.teamId) || 0) + 1)
        }
        for (const [teamId, count] of droppedCountByTeam) {
            logMessageDroppedCounter.inc({ reason: 'rate_limited', team_id: teamId.toString() }, count)
            recordLogMessageDropped('rate_limited', teamId.toString(), count)
        }

        return { rateLimiterAllowedMessages: allowed, rateLimiterDroppedMessages: dropped }
    }

    /**
     * Per-team metric-rule tally state for one batch. Rules are fetched once per team per
     * batch (30s-cached); tallies accumulate across the batch's messages and are emitted
     * as one OTLP payload per team after produce.
     */
    private async getMetricRuleBatchState(
        byTeam: MetricTalliesByTeam,
        message: LogsIngestionMessage
    ): Promise<{ token: string; rules: CompiledMetricRule[]; tallies: BatchTallies } | null> {
        if (!this.appliesTeamLogRules(message)) {
            return null
        }
        const teamRulesEnabled = this.isMetricRulesEnabledForTeam(message.teamId)
        const defaultRulesEnabled = this.isDefaultMetricRulesEnabledForTeam(message.teamId)
        if (!teamRulesEnabled && !defaultRulesEnabled) {
            return null
        }
        let state = byTeam.get(message.teamId)
        if (!state) {
            let rules: CompiledMetricRule[] = []
            try {
                if (teamRulesEnabled) {
                    const all = await this.deps.metricRulesCache!.getCompiledRules(message.teamId)
                    // Each consumer tallies only its own record source: the logs consumer runs
                    // `logs` rules, the traces consumer runs `spans` rules.
                    rules = all.filter((r) => r.source === this.metricRuleSource)
                }
            } catch (error) {
                // Fail open: metric rules are a purely additive side feature, so a rules-fetch
                // failure (e.g. a Postgres blip) must never DLQ or block the log records —
                // skip metric generation for this team this batch instead.
                logger.warn('[logs-metric-rules] rules fetch failed — skipping metric rules for batch', {
                    teamId: message.teamId,
                    error: String(error),
                })
                return null
            }
            if (defaultRulesEnabled) {
                rules = withDefaultMetricRules(rules, this.metricRuleSource)
            }
            // Re-check after the await: concurrent messages for the same team race here,
            // and replacing an existing state would silently drop its accumulated tallies.
            state = byTeam.get(message.teamId)
            if (!state) {
                state = { token: message.token, rules, tallies: createBatchTallies() }
                byTeam.set(message.teamId, state)
            }
        }
        return state.rules.length > 0 ? state : null
    }

    private async emitMetricRuleTallies(byTeam: MetricTalliesByTeam): Promise<void> {
        for (const [teamId, { token, rules, tallies }] of byTeam) {
            const teamIdLabel = teamId.toString()
            if (tallies.valueSkipped > 0) {
                logsMetricRulesValueSkippedCounter.inc({ team_id: teamIdLabel }, tallies.valueSkipped)
            }
            if (tallies.evalErrors > 0) {
                logsMetricRulesEvalErrorCounter.inc({ team_id: teamIdLabel }, tallies.evalErrors)
            }
            let overflow = 0
            for (const count of tallies.seriesOverflow.values()) {
                overflow += count
            }
            if (overflow > 0) {
                logsMetricRulesSeriesOverflowCounter.inc({ team_id: teamIdLabel }, overflow)
            }
            let matched = 0
            for (const ruleTallies of tallies.byRule.values()) {
                for (const entry of ruleTallies.values()) {
                    matched += entry.count
                }
            }
            if (matched > 0) {
                logsMetricRulesRecordsMatchedCounter.inc({ team_id: teamIdLabel }, matched)
            }

            const payload = buildMetricRulesOtlpPayload(rules, tallies, Date.now())
            if (!payload) {
                continue
            }
            try {
                await this.deps.metricsEmitter!.emit(token, teamId, payload)
            } catch (error) {
                // Emission is best-effort: a failing metrics endpoint must never fail log ingestion.
                logger.warn('[logs-metric-rules] emit threw', { teamId, error: String(error) })
            }
        }
    }

    private async processAndProduceLogMessages(
        messages: LogsIngestionMessage[],
        usageStats: UsageStatsByTeam,
        transformationBatchBudget?: TransformationBatchBudget
    ): Promise<MetricTalliesByTeam> {
        const metricTalliesByTeam: MetricTalliesByTeam = new Map()
        const limit = pLimit(MAX_CONCURRENT_MESSAGE_PROCESSES)
        let quarantined = 0
        const results = await Promise.allSettled(
            messages.map((message) =>
                limit(async () => {
                    try {
                        // Fetch team to get logs_settings
                        const team = await this.retryOnDependencyUnavailable(() =>
                            this.deps.teamManager.getTeam(message.teamId)
                        )
                        const logsSettings = team?.logs_settings || {}

                        // Extract settings with defaults
                        const jsonParse = logsSettings.json_parse_logs ?? false
                        const retentionDays = await this.retryOnDependencyUnavailable(() =>
                            this.defaultRetentionDays(message.teamId, logsSettings)
                        )

                        // Retention is uniform per team; stash it for the retention usage metrics.
                        const teamStats = usageStats.get(message.teamId)
                        if (teamStats) {
                            teamStats.retentionDays = retentionDays
                        }

                        // ignore empty messages
                        if (message.message.value === null) {
                            return Promise.resolve()
                        }

                        const metricRuleState = await this.getMetricRuleBatchState(metricTalliesByTeam, message)
                        const jsonAttributeKey =
                            this.appSource === 'logs' &&
                            teamIdMatchesCsv(this.jsonAttributeParsingEnabledTeamsRaw, message.teamId)
                                ? logsSettings.json_parse_logs_attribute_key
                                : undefined
                        const onRecordsDecoded =
                            metricRuleState || jsonAttributeKey
                                ? (records: LogRecord[]) => {
                                      if (metricRuleState) {
                                          const nowMs = Date.now()
                                          tallyRecords(
                                              metricRuleState.rules,
                                              records,
                                              metricRuleState.tallies,
                                              nowMs,
                                              (ruleId, labelKey) =>
                                                  this.metricSeriesAdmission.admit(
                                                      message.teamId,
                                                      ruleId,
                                                      labelKey,
                                                      nowMs
                                                  )
                                          )
                                      }
                                      if (jsonAttributeKey) {
                                          sniffJsonLogAttributes(records, jsonAttributeKey, message.teamId)
                                      }
                                  }
                                : undefined

                        const resolved = await instrumentFn(
                            {
                                key: 'logsIngestion.sampling.resolveLogMessageBuffer',
                                measureTime: false,
                                sendException: false,
                                getLoggingContext: () => ({
                                    team_id: message.teamId,
                                    inbound_bytes: message.message.value?.length ?? 0,
                                }),
                            },
                            async () =>
                                this.resolveLogMessageBufferWithOptionalSampling(
                                    message,
                                    {
                                        ...logsSettings,
                                        json_parse_logs_attribute_key:
                                            this.appSource === 'logs' &&
                                            teamIdMatchesCsv(
                                                this.jsonAttributeExtractionEnabledTeamsRaw,
                                                message.teamId
                                            )
                                                ? logsSettings.json_parse_logs_attribute_key
                                                : undefined,
                                    },
                                    retentionDays,
                                    onRecordsDecoded,
                                    transformationBatchBudget
                                )
                        )

                        this.queueUsageMetric(
                            message.teamId,
                            'records_dropped_retention_expired',
                            resolved.recordsDroppedByStage.get(RETENTION_EXPIRED_STAGE_NAME) ?? 0
                        )

                        let bytesUncompressedHeaderOverride: number | undefined
                        let bytesCompressedHeaderOverride: number | undefined
                        let recordCountHeaderOverride: number | undefined

                        // Drop rules removed rows from this message — credit billing by the dropped
                        // content fraction. `bytesReceived` stays gross (what was sent); `bytesAllowed`
                        // (→ bytes_ingested) and `recordsAllowed` reflect only what survived the drops.
                        if (resolved.recordsDropped > 0) {
                            const header = message.bytesUncompressed
                            const contentCredit = billingByteReductionForDrops(
                                header,
                                resolved.contentBytesDropped,
                                resolved.contentBytesTotal
                            )
                            // Second estimator (record-weighted) for the accuracy-confidence signal:
                            // when content- and record-weighted credits diverge, the batch is size-skewed
                            // and the content-weighted pro-rate we bill on is less trustworthy.
                            const recordCredit = billingByteReductionForDrops(
                                header,
                                resolved.recordsDropped,
                                message.recordCount
                            )

                            if (!this.isUnbilledInternal(message)) {
                                const teamIdLabel = message.teamId.toString()
                                if (contentCredit > 0) {
                                    logsBillingBytesCreditedCounter.inc({ team_id: teamIdLabel }, contentCredit)
                                }
                                logsBillingRecordsCreditedCounter.inc({ team_id: teamIdLabel }, resolved.recordsDropped)
                            }
                            if (message.recordCount > 0) {
                                logsDropBatchRowsHistogram.observe(message.recordCount)
                            }
                            if (resolved.contentBytesTotal > 0) {
                                logsDropFractionHistogram.observe(
                                    Math.min(1, resolved.contentBytesDropped / resolved.contentBytesTotal)
                                )
                            }
                            if (header > 0) {
                                logsBillingProrateDivergenceHistogram.observe(
                                    Math.abs(contentCredit - recordCredit) / header
                                )
                            }

                            // Shadow mode unless LOGS_BILLING_PRORATE_ENABLED: the credit above is
                            // computed and counted (observability) but billed usage is untouched.
                            if (this.billingProrateEnabled) {
                                const billingRow = usageStats.get(message.teamId)
                                if (billingRow) {
                                    billingRow.bytesAllowed = Math.max(0, billingRow.bytesAllowed - contentCredit)
                                    billingRow.recordsAllowed = Math.max(
                                        0,
                                        billingRow.recordsAllowed - resolved.recordsDropped
                                    )
                                }

                                // Scale both size headers down by the same dropped content fraction, and
                                // reduce the record count by the exact number of rows dropped, so
                                // downstream accounting sees only the surviving rows and bytes.
                                const compressedCredit = billingByteReductionForDrops(
                                    message.bytesCompressed,
                                    resolved.contentBytesDropped,
                                    resolved.contentBytesTotal
                                )
                                bytesUncompressedHeaderOverride = Math.max(0, header - contentCredit)
                                bytesCompressedHeaderOverride = Math.max(0, message.bytesCompressed - compressedCredit)
                                recordCountHeaderOverride = Math.max(0, message.recordCount - resolved.recordsDropped)
                            }
                        }

                        if (resolved.outcome === 'all_dropped') {
                            logMessageDroppedCounter.inc(
                                { reason: resolved.reason, team_id: message.teamId.toString() },
                                1
                            )
                            recordLogMessageDropped(resolved.reason, message.teamId.toString())
                            this.addPiiStatsIntoUsage(usageStats, message.teamId, resolved.pii)
                            this.queueSamplingRecordsDroppedByRule(message.teamId, resolved.recordsDroppedByRuleId)
                            this.queueBytesDroppedByRule(message.teamId, resolved.bytesDroppedByRuleId)
                            return Promise.resolve()
                        }
                        const { processedValue, pii, recordsDroppedByRuleId, bytesDroppedByRuleId } = resolved
                        this.addPiiStatsIntoUsage(usageStats, message.teamId, pii)
                        this.queueSamplingRecordsDroppedByRule(message.teamId, recordsDroppedByRuleId)
                        this.queueBytesDroppedByRule(message.teamId, bytesDroppedByRuleId)

                        // Await so a rejection here lands in the catch and routes to the DLQ.
                        await this.retryOnDependencyUnavailable(() =>
                            this.deps.outputs
                                .queueMessages(LOGS_OUTPUT, [
                                    {
                                        value: processedValue,
                                        key: null,
                                        headers: {
                                            ...withoutInternalLogsHeaders(parseKafkaHeaders(message.message.headers)),
                                            ...(message.verifiedAsInternalLogs !== undefined
                                                ? { [INTERNAL_SOURCE_HEADER]: message.verifiedAsInternalLogs.source }
                                                : {}),
                                            token: message.token,
                                            team_id: message.teamId.toString(),
                                            'json-parse': jsonParse.toString(),
                                            'retention-days': retentionDays.toString(),
                                            // Both topics use random partitioning, so the ingestion lag checkpoint needs the source partition.
                                            source_topic: message.message.topic,
                                            source_partition: message.message.partition.toString(),
                                            ...(bytesUncompressedHeaderOverride !== undefined
                                                ? { bytes_uncompressed: bytesUncompressedHeaderOverride.toString() }
                                                : {}),
                                            ...(bytesCompressedHeaderOverride !== undefined
                                                ? { bytes_compressed: bytesCompressedHeaderOverride.toString() }
                                                : {}),
                                            ...(recordCountHeaderOverride !== undefined
                                                ? { record_count: recordCountHeaderOverride.toString() }
                                                : {}),
                                        },
                                    },
                                ])
                                .catch(rethrowKafkaOutageAsDependency)
                        )
                    } catch (error) {
                        // A dependency outage (Postgres, Redis, Kafka) is infrastructure-scoped. The
                        // message itself is fine and processes once the dependency is back, so
                        // quarantining it would move good data into the DLQ. The retries above
                        // already absorbed a blip; a longer outage leaves the message on the source
                        // topic and fails the batch below instead.
                        if (error instanceof DependencyUnavailableError) {
                            throw error
                        }
                        // Quarantine succeeded: the DLQ copy is the record of this failure, so the
                        // message resolves and the batch can commit past it. A failed DLQ write
                        // rejects here instead, so the batch keeps the only copy on the source topic.
                        await this.produceToDlq(message, error)
                        quarantined += 1
                    }
                })
            )
        )

        // Every rejection means this batch must not commit: a dependency outage, or a DLQ write
        // that failed. `Promise.allSettled` swallows rejections, so rethrow before the tallies and
        // usage metrics are emitted. The rejected background task stops the offset commit and the
        // consumer replays the batch from the last good commit, which would count them twice.
        const failures = results.filter((r): r is PromiseRejectedResult => r.status === 'rejected')
        if (failures.length > 0 || quarantined > 0) {
            logger.error('Failed to process some log messages', {
                quarantinedCount: quarantined,
                failureCount: failures.length,
                totalCount: messages.length,
            })
        }
        if (failures.length > 0) {
            throw failures[0].reason
        }

        return metricTalliesByTeam
    }

    private async produceToDlq(message: LogsIngestionMessage, error: unknown): Promise<void> {
        await this.produceRawToDlq(message.message, error, { token: message.token, teamId: message.teamId })
    }

    /**
     * Absorb a dependency blip in place. A batch that fails after some of its messages were
     * produced is replayed in full, and the logs table has no dedupe, so every retry here is a
     * duplicate avoided. Only an outage that outlasts the backoff reaches the caller.
     */
    private retryOnDependencyUnavailable<T>(fn: () => Promise<T>): Promise<T> {
        return retryOnDependencyUnavailableError(fn, this.deps.dependencyRetry)
    }

    /**
     * Quarantine a raw Kafka message. Takes the message rather than a `LogsIngestionMessage` so
     * failures from before the team is resolved can reach the DLQ too — those used to be counted
     * and discarded, which lost the only copy of the payload.
     */
    private async produceRawToDlq(
        message: Message,
        error: unknown,
        context: { token?: string; teamId?: number }
    ): Promise<void> {
        const errorMessage = error instanceof Error ? error.message : String(error)
        const errorName = error instanceof Error ? error.name : 'UnknownError'
        const teamIdLabel = context.teamId?.toString() ?? 'unknown'

        logMessageDlqCounter.inc({ reason: errorName, team_id: teamIdLabel })
        recordLogMessageDlq(errorName, teamIdLabel)

        try {
            await this.retryOnDependencyUnavailable(() =>
                this.deps.outputs.queueMessages(LOGS_DLQ_OUTPUT, [
                    {
                        value: message.value,
                        key: null,
                        headers: {
                            ...parseKafkaHeaders(message.headers),
                            ...(context.token !== undefined ? { token: context.token } : {}),
                            team_id: teamIdLabel,
                            error_message: errorMessage,
                            error_name: errorName,
                            failed_at: new Date().toISOString(),
                            // Where the message came from, so a quarantined record can be read back
                            // from the source partition and replayed against the consumer.
                            source_topic: message.topic,
                            source_partition: message.partition.toString(),
                            source_offset: message.offset.toString(),
                        },
                    },
                ])
            )
        } catch (dlqError) {
            logger.error('Failed to produce message to DLQ', {
                error: dlqError,
                originalError: errorMessage,
                partition: message.partition,
                offset: message.offset,
            })
            // Rethrow: quarantine is what lets the caller move past the record. Swallowing this
            // would commit the source offset with no copy anywhere, which loses the payload for
            // good. Failing the batch keeps the record on the source topic until the DLQ is back.
            throw dlqError
        }
    }

    private async emitUsageMetrics(usageStats: UsageStatsByTeam): Promise<void> {
        for (const [teamId, stats] of usageStats) {
            this.queueUsageMetric(teamId, 'bytes_received', stats.bytesReceived)
            this.queueUsageMetric(teamId, 'records_received', stats.recordsReceived)
            this.queueUsageMetric(teamId, 'bytes_ingested', stats.bytesAllowed)
            // Records-based counterpart to `bytes_ingested` (sum of per-record sizes instead of
            // payload size). Emitted in parallel so the two can be compared before billing
            // switches to the records-based value; not yet read by usage reports.
            this.queueUsageMetric(teamId, 'bytes_ingested_records', stats.bytesAllowedRecords)
            this.queueUsageMetric(teamId, 'records_ingested', stats.recordsAllowed)
            this.queueUsageMetric(teamId, 'bytes_dropped', stats.bytesDropped)
            this.queueUsageMetric(teamId, 'records_dropped', stats.recordsDropped)
            this.queueUsageMetric(teamId, 'pii_replacements', stats.piiReplacements)

            const retentionMetric = retentionBytesMetricName(stats.retentionDays)
            if (retentionMetric) {
                this.queueUsageMetric(teamId, retentionMetric, stats.bytesAllowed)
            }
            // Byte-days: ingested bytes weighted by the retention day count, only for teams that
            // chose a retention other than the default. The default tier is covered by `bytes_ingested`,
            // so it must not be billed a second time here. Uses credit-adjusted `bytesAllowed` to
            // reconcile with `bytes_ingested`. Runs beside the per-tier metric above until billing
            // leaves fixed tiers.
            if (stats.retentionDays !== DEFAULT_LOGS_RETENTION_DAYS) {
                this.queueRetentionByteDays(teamId, stats.bytesAllowed, stats.retentionDays)
            }
            const source = this.appSource === 'traces' ? 'apm_traces' : 'logs'
            // These records are per-flush aggregates, not one per billed thing, so there is no
            // stable identity to reproduce. A fresh ID per flush is what keeps two pods flushing
            // the same team from colliding and collapsing one flush's quantity away.
            const flushId = new UUID7().toString()
            this.deps.usageBatch.add(teamId, `${source}_bytes`, flushId, stats.bytesAllowed, 'bytes')
            this.deps.usageBatch.add(
                teamId,
                source === 'apm_traces' ? 'apm_traces_spans' : 'logs_records',
                flushId,
                stats.recordsAllowed,
                'records'
            )
        }

        // Best-effort, and independent of each other: neither failing may block ingestion or skip
        // the other, and nothing downstream reads either result, so they go out together.
        await Promise.all([
            this.appMetricsAggregator.flush().catch((error) => {
                logger.error('🔴', 'Failed to emit usage metrics - billing data may be lost', { error })
            }),
            this.deps.usageBatch.flush().catch((error) => {
                logger.error('🔴', 'Failed to emit usage records - billing data may be lost', { error })
            }),
        ])
    }

    private queueUsageMetric(teamId: number, metricName: string, count: number, timestamp?: DateTime): void {
        if (count === 0) {
            return
        }
        this.appMetricsAggregator.queue({
            team_id: teamId,
            app_source: this.appSource,
            app_source_id: '',
            instance_id: '',
            metric_kind: 'usage',
            metric_name: metricName,
            count,
            timestamp,
        })
    }

    /**
     * Writes one `retention_byte_days` row per 30-day month of retention, each dated at the start of
     * its month, so each month of storage bills when it starts instead of the whole period billing
     * on the ingest day. The usage report sums these rows by timestamp, and the `app_metrics2` TTL
     * counts from each row's own timestamp, so future rows stay until they are billed.
     */
    private queueRetentionByteDays(teamId: number, bytes: number, retentionDays: number): void {
        const now = DateTime.utc()
        for (let offsetDays = 0; offsetDays < retentionDays; offsetDays += RETENTION_MONTH_DAYS) {
            const days = Math.min(RETENTION_MONTH_DAYS, retentionDays - offsetDays)
            this.queueUsageMetric(teamId, 'retention_byte_days', bytes * days, now.plus({ days: offsetDays }))
        }
    }

    /**
     * Per-rule head sampling drops in app_metrics2 (`sampling_records_dropped_by_rule`).
     * Team-level sampling volume in CH is `sum(count)` over those rows (no separate aggregate metric).
     */
    private queueSamplingRecordsDroppedByRule(teamId: number, byRule: Map<string, number>): void {
        for (const [ruleId, count] of byRule) {
            if (count <= 0) {
                continue
            }
            this.appMetricsAggregator.queue({
                team_id: teamId,
                app_source: this.appSource,
                app_source_id: '',
                instance_id: ruleId,
                metric_kind: 'usage',
                metric_name: 'sampling_records_dropped_by_rule',
                count,
            })
        }
    }

    /**
     * Per-rule dropped bytes in app_metrics2 (`bytes_dropped_by_rule`). Summed from per-row
     * `bytes_uncompressed`; bridges drop-rule accounting toward billing's `bytes_ingested`.
     */
    private queueBytesDroppedByRule(teamId: number, byRule: Map<string, number>): void {
        for (const [ruleId, count] of byRule) {
            if (count <= 0) {
                continue
            }
            this.appMetricsAggregator.queue({
                team_id: teamId,
                app_source: this.appSource,
                app_source_id: '',
                instance_id: ruleId,
                metric_kind: 'usage',
                metric_name: 'bytes_dropped_by_rule',
                count,
            })
        }
    }

    @instrumented('logsIngestionConsumer.handleEachBatch.parseKafkaMessages')
    public async _parseKafkaBatch(messages: Message[]): Promise<LogsIngestionMessage[]> {
        const events: LogsIngestionMessage[] = []

        await Promise.all(
            messages.map(async (message) => {
                try {
                    const headers = parseKafkaHeaders(message.headers)
                    const token = headers.token

                    if (!token) {
                        logger.warn('missing_token')
                        logMessageDroppedCounter.inc({ reason: 'missing_token', team_id: 'unknown' })
                        recordLogMessageDropped('missing_token', 'unknown')
                        return
                    }

                    let team
                    try {
                        if (isDevEnv() && token === 'phc_local') {
                            // phc_local is a special token used in dev to refer to team 1
                            team = await this.deps.teamManager.getTeam(1)
                        } else {
                            team = await this.deps.teamManager.getTeamByToken(token)
                        }
                    } catch (e) {
                        logger.error('team_lookup_error', { error: e })
                        logMessageDroppedCounter.inc({ reason: 'team_lookup_error', team_id: 'unknown' })
                        recordLogMessageDropped('team_lookup_error', 'unknown')
                        return
                    }

                    if (!team) {
                        // A well-formed but unknown or rotated token is a client-input problem, not a
                        // service fault. capture-logs accepts any well-formed token shape and defers team
                        // resolution to here because it has no Postgres access, so this fires on every
                        // dropped message for a bad token. Already tracked via the metrics below.
                        logger.warn('team_not_found', { token_with_no_team: token })
                        logMessageDroppedCounter.inc({ reason: 'team_not_found', team_id: 'unknown' })
                        recordLogMessageDropped('team_not_found', 'unknown')
                        return
                    }

                    const bytesUncompressed = parseSizeHeader(headers.bytes_uncompressed)
                    const bytesUncompressedRecords = parseSizeHeader(headers.bytes_uncompressed_records)
                    const bytesCompressed = parseSizeHeader(headers.bytes_compressed)
                    const recordCount = parseSizeHeader(headers.record_count)

                    if (
                        bytesUncompressed === null ||
                        bytesUncompressedRecords === null ||
                        bytesCompressed === null ||
                        recordCount === null
                    ) {
                        logger.error('🔴', 'logs_ingestion_invalid_size_headers', {
                            teamId: team.id,
                            partition: message.partition,
                            offset: message.offset,
                        })
                        await this.produceRawToDlq(message, new Error('invalid_size_headers'), {
                            token,
                            teamId: team.id,
                        })
                        return
                    }

                    if (bytesUncompressedRecords > bytesUncompressed) {
                        // Billing can only switch from payload-based to records-based bytes if the
                        // records sum never exceeds the payload size — flag any violation.
                        logsRecordsBytesExceedPayloadCounter.inc({ team_id: team.id.toString() })
                    }

                    events.push({
                        token,
                        message,
                        teamId: team.id,
                        bytesUncompressed,
                        bytesUncompressedRecords,
                        bytesCompressed,
                        recordCount,
                        minTimestampMicros: parseMinTimestampHeader(headers.min_timestamp),
                        backfillRequested: headers.backfill_days !== undefined,
                        sentAsInternalLogs: headers[INTERNAL_LOGS_HEADER] !== undefined,
                        verifiedAsInternalLogs: this.internalLogsVerifier.verify(
                            headers[INTERNAL_LOGS_HEADER],
                            team.id,
                            headers.created_at
                        ),
                    })
                } catch (e) {
                    // A message we cannot parse is message-scoped and will fail the same way on
                    // every redelivery, so quarantine it instead of discarding the payload.
                    logger.error('🔴', 'logs_ingestion_parse_error', {
                        error: String(e),
                        partition: message.partition,
                        offset: message.offset,
                    })
                    logMessageDroppedCounter.inc({ reason: 'parse_error', team_id: 'unknown' })
                    recordLogMessageDropped('parse_error', 'unknown')
                    await this.produceRawToDlq(message, e, {})
                    return
                }
            })
        )

        return events
    }

    public async processKafkaBatch(
        messages: Message[]
    ): Promise<{ backgroundTask?: Promise<any>; messages: LogsIngestionMessage[] }> {
        try {
            const events = await this._parseKafkaBatch(messages)
            return await this.processBatch(events)
        } catch (error) {
            // Rethrow: the consumer deliberately lets a batch failure crash the process, which is
            // what preserves at-least-once. We do not DLQ here — everything on this path (quota
            // lookups, the rate limiter) is infrastructure-scoped, and quarantining a whole batch
            // over a Redis blip would drop good data. Name the records first so the restart loop
            // that follows points somewhere.
            logger.error('🔴', `${this.name} - batch failed`, {
                error: String(error),
                stack: error instanceof Error ? error.stack : undefined,
                size: messages.length,
                offsets: describeBatchPosition(messages),
            })
            throw error
        }
    }

    public async start(): Promise<void> {
        // Start consuming messages
        await this.kafkaConsumer.connect(async (messages) => {
            logger.info('🔁', `${this.name} - handling batch`, {
                size: messages.length,
                offsets: describeBatchPosition(messages),
            })

            return await instrumentFn('logsIngestionConsumer.handleEachBatch', async () => {
                return await this.processKafkaBatch(messages)
            })
        })
    }

    public async stop(): Promise<void> {
        logger.info('💤', 'Stopping consumer...')
        await this.kafkaConsumer.disconnect()
        logger.info('💤', 'Consumer stopped!')
    }

    public isHealthy(): HealthCheckResult {
        return this.kafkaConsumer.isHealthy()
    }
}