Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

135 KB
Raw Blame
import { mockProducer, mockProducerObserver } from '~/tests/helpers/mocks/producer.mock'

import { metrics as metricsApi } from '@opentelemetry/api'
import {
    type DataPoint,
    InMemoryMetricExporter,
    MeterProvider,
    PeriodicExportingMetricReader,
} from '@opentelemetry/sdk-metrics'
import { DateTime } from 'luxon'
import { CODES, Message } from 'node-rdkafka'

import { compileHog } from '~/cdp/templates/compiler'
import { HogFunctionType } from '~/cdp/types'
import { KAFKA_APP_METRICS_2, KAFKA_LOGS_CLICKHOUSE, KAFKA_LOGS_INGESTION_DLQ } from '~/common/config/kafka-topics'
import { APP_METRICS_OUTPUT, AppMetricsOutput } from '~/common/outputs'
import { IngestionOutputs } from '~/common/outputs/ingestion-outputs'
import { SingleIngestionOutput } from '~/common/outputs/single-ingestion-output'
import { UsageRecordBatch } from '~/common/usage-ingestion/usage-record-batch'
import { DependencyUnavailableError } from '~/common/utils/db/error'
import { closeHub, createHub } from '~/common/utils/db/hub'
import { PostgresUse } from '~/common/utils/db/postgres'
import { parseJSON } from '~/common/utils/json-parse'
import { forSnapshot } from '~/tests/helpers/snapshots'
import { createTeam, createTestTeamFixture, getTeam } from '~/tests/helpers/sql'
import { Hub, Team } from '~/types'

import { getDefaultTracesIngestionConsumerConfig } from './config'
import * as otelMetrics from './ingestion-otel-metrics'
import { resetLogsIngestionInstrumentsForTests } from './ingestion-otel-metrics'
import { INTERNAL_LOGS_HEADER, INTERNAL_SOURCE_HEADER, signInternalLogs } from './internal-logs'
import { logsPatternBodyKindCounter } from './log-pattern-stage'
import { LogRecord, decodeLogRecords, encodeLogRecords, logsJsonAttributeSniffCounter } from './log-record-avro'
import {
    DEFAULT_LOGS_RETENTION_DAYS,
    LogsIngestionConsumer,
    type LogsIngestionConsumerDeps,
    describeBatchPosition,
    logMessageDlqCounter,
    logMessageDroppedCounter,
    logsBillingBytesCreditedCounter,
    logsBillingRecordsCreditedCounter,
    logsBytesAllowedCounter,
    logsBytesDroppedCounter,
    logsBytesReceivedCounter,
    logsRecordsAllowedCounter,
    logsRecordsBytesExceedPayloadCounter,
    logsRecordsDroppedCounter,
    logsRecordsReceivedCounter,
    parseMinTimestampHeader,
    parseSizeHeader,
} from './logs-ingestion-consumer'
import { compileMetricRules } from './metrics-rules/compile-metric-rules'
import type { MetricRulesCache } from './metrics-rules/metric-rules-cache'
import type { LogsMetricsEmitter } from './metrics-rules/metrics-emitter'
import { LOGS_DLQ_OUTPUT, LOGS_OUTPUT, LogsDlqOutput, LogsOutput } from './outputs/outputs'
import type { PatternMessageKeysCache } from './pattern-message-keys-cache'
import { MAX_RETENTION_DAYS } from './retention/evaluate-retention'
import { DEFAULT_TRACES_RETENTION_DAYS } from './retention/tracing-config-cache'
import { compileRuleSet } from './sampling/compile-rules'
import type { SamplingRulesCache } from './sampling/sampling-rules-cache'
import { TracesIngestionConsumer } from './traces-ingestion-consumer'
import { LogsTransformerService } from './transformations/logs-transformer.service'

const LibrdKafkaError = require('node-rdkafka/lib/error')
const { err2str } = require('node-rdkafka/librdkafka') as { err2str: (code: number) => string }

// What capture-logs forwards for a producer that signed just before the request arrived. Both times come
// from `Date.now()`, which these tests pin, because a bare `new Date()` still reads the real clock.
function internalLogsHeaders(signature: string): Record<string, Buffer> {
    return {
        [INTERNAL_LOGS_HEADER]: Buffer.from(signature),
        created_at: Buffer.from(new Date(Date.now()).toISOString()),
    }
}

const DEFAULT_TEST_TIMEOUT = 5000
jest.setTimeout(DEFAULT_TEST_TIMEOUT)

const SNAPSHOT_OVERRIDES = { overrides: { team_id: 'TEAM_ID' } }

jest.mock('~/common/utils/posthog', () => {
    const original = jest.requireActual('~/common/utils/posthog')
    return {
        ...original,
        captureException: jest.fn(),
    }
})

let offsetIncrementer = 0

const LOG_RECORD_BASE_FIELDS = [
    { name: 'uuid', type: ['null', 'string'] },
    { name: 'trace_id', type: ['null', 'bytes'] },
    { name: 'span_id', type: ['null', 'bytes'] },
    { name: 'trace_flags', type: ['null', 'int'] },
    { name: 'timestamp', type: ['null', 'long'] },
    { name: 'observed_timestamp', type: ['null', 'long'] },
    { name: 'body', type: ['null', 'string'] },
    { name: 'severity_text', type: ['null', 'string'] },
    { name: 'severity_number', type: ['null', 'int'] },
    { name: 'service_name', type: ['null', 'string'] },
    { name: 'resource_attributes', type: ['null', { type: 'map', values: 'string' }] },
    { name: 'instrumentation_scope', type: ['null', 'string'] },
    { name: 'event_name', type: ['null', 'string'] },
    { name: 'attributes', type: ['null', { type: 'map', values: 'string' }] },
]

const createKafkaMessage = async (
    logData: any,
    headers: Record<string, string> = {},
    attributes: Record<string, string> | null = null
): Promise<Message> => {
    // Create a LogRecord from the log data
    const record: LogRecord = {
        uuid: `test-uuid-${offsetIncrementer}`,
        trace_id: null,
        span_id: null,
        trace_flags: null,
        timestamp: DateTime.now().toMillis() * 1000, // microseconds
        observed_timestamp: DateTime.now().toMillis() * 1000,
        body: JSON.stringify(logData),
        severity_text: logData.level || 'info',
        severity_number: 9,
        service_name: logData.service || 'test-service',
        resource_attributes: null,
        instrumentation_scope: null,
        event_name: null,
        attributes,
    }

    // Encode as AVRO
    const avro = require('avsc')
    const logRecordType = avro.Type.forSchema({
        type: 'record',
        name: 'LogRecord',
        fields: [
            ...LOG_RECORD_BASE_FIELDS,
            { name: 'pattern', type: ['null', 'string'], default: null },
            { name: 'pattern_version', type: ['null', 'int'], default: null },
        ],
    })

    const value = await encodeLogRecords(logRecordType, 'zstandard', [record])

    return {
        key: null,
        value,
        size: value.length,
        topic: 'test',
        offset: offsetIncrementer++,
        timestamp: DateTime.now().toMillis(),
        partition: 1,
        headers: Object.entries(headers).map(([key, value]) => ({
            [key]: Buffer.from(value),
        })),
    }
}

const createKafkaMessages = async (logData: any[], headers: Record<string, string> = {}): Promise<Message[]> => {
    return Promise.all(logData.map((data) => createKafkaMessage(data, headers)))
}

// Single Kafka message carrying several log records, so a drop rule can remove some rows while
// others survive (partial drop) — the path that re-encodes and re-forwards to the output topic.
const createMultiRecordKafkaMessage = async (
    logDataList: any[],
    headers: Record<string, string> = {}
): Promise<Message> => {
    const avro = require('avsc')
    const logRecordType = avro.Type.forSchema({
        type: 'record',
        name: 'LogRecord',
        fields: [...LOG_RECORD_BASE_FIELDS, { name: 'retention_days', type: ['null', 'int'], default: null }],
    })

    const records: LogRecord[] = logDataList.map((logData, i) => ({
        retention_days: logData.retentionDays ?? null,
        uuid: `test-uuid-${offsetIncrementer}-${i}`,
        trace_id: null,
        span_id: null,
        trace_flags: null,
        timestamp: logData.timestampMicros ?? DateTime.now().toMillis() * 1000,
        observed_timestamp: DateTime.now().toMillis() * 1000,
        body: JSON.stringify(logData),
        severity_text: logData.level || 'info',
        severity_number: 9,
        service_name: logData.service || 'test-service',
        resource_attributes: null,
        instrumentation_scope: null,
        event_name: null,
        attributes: null,
    }))

    const value = await encodeLogRecords(logRecordType, 'zstandard', records)

    return {
        key: null,
        value,
        size: value.length,
        topic: 'test',
        offset: offsetIncrementer++,
        timestamp: DateTime.now().toMillis(),
        partition: 1,
        headers: Object.entries(headers).map(([key, value]) => ({
            [key]: Buffer.from(value),
        })),
    }
}

describe('LogsIngestionConsumer', () => {
    let consumer: LogsIngestionConsumer
    let hub: Hub
    let team: Team
    let team2: Team
    let fixedTime: DateTime
    let logMessageDroppedCounterSpy: jest.SpyInstance
    let recordLogMessageDroppedSpy: jest.SpyInstance

    const createLogsIngestionConsumer = async (
        hub: Hub,
        overrides: any = {},
        depsPartial: Partial<
            Pick<
                LogsIngestionConsumerDeps,
                | 'samplingRulesCache'
                | 'metricRulesCache'
                | 'metricsEmitter'
                | 'logsTransformer'
                | 'patternMessageKeysCache'
                | 'internalSourcePolicies'
            >
        > = {}
    ) => {
        const consumer = new LogsIngestionConsumer(
            hub,
            {
                ...hub,
                outputs: new IngestionOutputs<AppMetricsOutput | LogsOutput | LogsDlqOutput>({
                    [APP_METRICS_OUTPUT]: new SingleIngestionOutput(
                        APP_METRICS_OUTPUT,
                        KAFKA_APP_METRICS_2,
                        mockProducer,
                        'test'
                    ),
                    [LOGS_OUTPUT]: new SingleIngestionOutput(LOGS_OUTPUT, KAFKA_LOGS_CLICKHOUSE, mockProducer, 'test'),
                    [LOGS_DLQ_OUTPUT]: new SingleIngestionOutput(
                        LOGS_DLQ_OUTPUT,
                        KAFKA_LOGS_INGESTION_DLQ,
                        mockProducer,
                        'test'
                    ),
                }),
                usageBatch: new UsageRecordBatch(null, { unit: 'bytes', isTeamEnabled: () => false }),
                dependencyRetry: { retryCount: 2, initialRetryDelayMs: 1 },
                ...depsPartial,
            },
            overrides
        )
        // NOTE: We don't actually use kafka so we skip instantiation for faster tests
        consumer['kafkaConsumer'] = {
            connect: jest.fn(),
            disconnect: jest.fn(),
            isHealthy: jest.fn().mockReturnValue({ status: 'healthy' }),
        } as any
        await consumer.start()
        return consumer
    }

    const createLogMessage = (logData?: any): any => ({
        level: 'info',
        message: 'Test log message',
        timestamp: fixedTime.toISO()!,
        service: 'test-service',
        ...logData,
    })

    beforeEach(async () => {
        fixedTime = DateTime.fromObject({ year: 2025, month: 1, day: 1 }, { zone: 'UTC' })
        jest.spyOn(Date, 'now').mockReturnValue(fixedTime.toMillis())
        jest.spyOn(Date.prototype, 'toISOString').mockReturnValue(fixedTime.toISO()!)

        offsetIncrementer = 0
        hub = await createHub()

        team = (await createTestTeamFixture(hub.postgres)).team
        const team2Id = await createTeam(hub.postgres, team.organization_id)
        team2 = (await getTeam(hub.postgres, team2Id))!

        consumer = await createLogsIngestionConsumer(hub)

        logMessageDroppedCounterSpy = jest.spyOn(logMessageDroppedCounter, 'inc')
        recordLogMessageDroppedSpy = jest.spyOn(otelMetrics, 'recordLogMessageDropped')

        // Default to not quota limited - tests can override this
        jest.spyOn(hub.quotaLimiting, 'isTeamTokenQuotaLimited').mockResolvedValue(false)
    })

    afterEach(async () => {
        await consumer.stop()
        await closeHub(hub)
    })

    afterAll(() => {
        jest.useRealTimers()
    })

    const waitForBackgroundTasks = async (
        promise: Promise<{
            backgroundTask?: Promise<any>
        }>
    ) => {
        await (
            await promise
        ).backgroundTask
    }

    // Simple helper to transform getProducedMessages structure to match getProducedKafkaMessages
    const getProducedKafkaMessages = () => {
        const producedMessages = mockProducerObserver.getProducedMessages()
        const result = []

        for (const batch of producedMessages) {
            for (const message of batch.messages) {
                result.push({
                    topic: batch.topic,
                    key: message.key,
                    value: message.value || {},
                    headers: message.headers,
                })
            }
        }

        return result
    }

    describe('general', () => {
        it('should have the correct config', () => {
            expect(consumer['name']).toEqual('LogsIngestionConsumer')
            expect(consumer['groupId']).toEqual('ingestion-logs')
            expect(consumer['topic']).toEqual('logs_ingestion_test')
        })

        it('should allow config overrides', async () => {
            const overrides = {
                LOGS_INGESTION_CONSUMER_GROUP_ID: 'custom-group',
                LOGS_INGESTION_CONSUMER_CONSUME_TOPIC: 'custom-topic',
            }
            const customConsumer = await createLogsIngestionConsumer(hub, overrides)

            expect(customConsumer['groupId']).toBe('custom-group')
            expect(customConsumer['topic']).toBe('custom-topic')

            await customConsumer.stop()
        })

        it('should process a valid log message', async () => {
            const logData = createLogMessage()
            const messages = await createKafkaMessages([logData], {
                token: team.api_token,
            })

            await waitForBackgroundTasks(consumer.processKafkaBatch(messages))

            expect(
                forSnapshot(
                    getProducedKafkaMessages().map((m) => m.headers),
                    SNAPSHOT_OVERRIDES
                )
            ).toMatchSnapshot()
        })

        it('should process multiple log messages', async () => {
            const logData = [
                createLogMessage({ level: 'info', message: 'First log' }),
                createLogMessage({ level: 'error', message: 'Second log' }),
                createLogMessage({ level: 'debug', message: 'Third log' }),
            ]
            const messages = await createKafkaMessages(logData, {
                token: team.api_token,
            })

            await waitForBackgroundTasks(consumer.processKafkaBatch(messages))

            const producedMessages = getProducedKafkaMessages()
            expect(producedMessages).toHaveLength(3)
            expect(
                forSnapshot(
                    producedMessages.map((m) => m.headers),
                    SNAPSHOT_OVERRIDES
                )
            ).toMatchSnapshot()
        })
    })

    describe('message parsing and validation', () => {
        it('should drop messages with missing token', async () => {
            const logData = createLogMessage()
            const messages = await createKafkaMessages([logData], {
                // missing token
            })

            await waitForBackgroundTasks(consumer.processKafkaBatch(messages))

            expect(getProducedKafkaMessages()).toHaveLength(0)
            expect(logMessageDroppedCounterSpy).toHaveBeenCalledWith({ reason: 'missing_token', team_id: 'unknown' })
        })

        it('should drop messages with invalid token', async () => {
            const logData = createLogMessage()
            const messages = await createKafkaMessages([logData], {
                token: 'invalid-token',
            })

            await waitForBackgroundTasks(consumer.processKafkaBatch(messages))

            expect(getProducedKafkaMessages()).toHaveLength(0)
        })

        it('should preserve kafka message headers', async () => {
            const logData = createLogMessage()
            const messages = await createKafkaMessages([logData], {
                myHeader: 'hello',
                token: team.api_token,
            })

            await waitForBackgroundTasks(consumer.processKafkaBatch(messages))

            const producedMessages = getProducedKafkaMessages()
            expect(
                forSnapshot(
                    producedMessages.map((m) => m.headers),
                    SNAPSHOT_OVERRIDES
                )
            ).toMatchSnapshot()
        })

        it('should overwrite existing headers', async () => {
            const logData = createLogMessage()
            const messages = await createKafkaMessages([logData], {
                token: team.api_token,
                team_id: '999',
            })

            await waitForBackgroundTasks(consumer.processKafkaBatch(messages))
            expect(
                forSnapshot(
                    getProducedKafkaMessages().map((m) => m.headers),
                    SNAPSHOT_OVERRIDES
                )
            ).toMatchSnapshot()
        })

        it('should handle parse errors gracefully', async () => {
            const messages = [
                {
                    key: null,
                    value: Buffer.from('invalid-json-is-fine-we-dont-check-for-errors'),
                    size: 1,
                    topic: 'test',
                    offset: offsetIncrementer++,
                    timestamp: DateTime.now().toMillis(),
                    partition: 1,
                    headers: [{ token: Buffer.from('missing') }],
                } as Message,
            ]

            await waitForBackgroundTasks(consumer.processKafkaBatch(messages))

            expect(getProducedKafkaMessages()).toHaveLength(0)
            expect(logMessageDroppedCounterSpy).toHaveBeenCalledWith({ reason: 'team_not_found', team_id: 'unknown' })
        })

        it('should drop messages when team lookup fails and record unknown team_id', async () => {
            jest.spyOn(hub.teamManager, 'getTeamByToken').mockRejectedValueOnce(new Error('Database error'))

            const logData = createLogMessage()
            const messages = await createKafkaMessages([logData], {
                token: team.api_token,
            })

            await waitForBackgroundTasks(consumer.processKafkaBatch(messages))

            expect(getProducedKafkaMessages()).toHaveLength(0)
            expect(logMessageDroppedCounterSpy).toHaveBeenCalledWith({
                reason: 'team_lookup_error',
                team_id: 'unknown',
            })
        })
    })

    describe('batch processing', () => {
        it('should handle empty batch', async () => {
            const result = await consumer.processBatch([])

            expect(result.backgroundTask).toBeUndefined()
            expect(result.messages).toEqual([])
            await result.backgroundTask
        })

        it('should process batch with valid messages', async () => {
            const logData = createLogMessage()
            const messages = await createKafkaMessages([logData], {
                token: team.api_token,
            })

            await waitForBackgroundTasks(consumer.processKafkaBatch(messages))

            const producedMessages = getProducedKafkaMessages()
            expect(producedMessages).toHaveLength(1)
            expect(producedMessages[0].topic).toBe('clickhouse_logs_test')
            expect(producedMessages[0].headers).toEqual({
                token: team.api_token,
                team_id: team.id.toString(),
                'json-parse': 'false',
                'retention-days': DEFAULT_LOGS_RETENTION_DAYS.toString(),
                source_topic: messages[0].topic,
                source_partition: messages[0].partition.toString(),
            })
        })

        it('marks a team that receives its first logs as having ingested data', async () => {
            const newTeamId = await createTeam(hub.postgres, team.organization_id, undefined, { ingested_event: false })
            const newTeam = await getTeam(hub.postgres, newTeamId)
            const messages = await createKafkaMessages([createLogMessage()], { token: newTeam!.api_token })

            await waitForBackgroundTasks(consumer.processKafkaBatch(messages))

            expect((await getTeam(hub.postgres, newTeamId))?.ingested_event).toBe(true)
        })

        it('should produce messages with correct headers', async () => {
            const logData = createLogMessage({ level: 'error', message: 'Critical error' })
            const messages = await createKafkaMessages([logData], {
                token: team.api_token,
            })

            await waitForBackgroundTasks(consumer.processKafkaBatch(messages))

            const producedMessages = getProducedKafkaMessages()
            expect(producedMessages).toHaveLength(1)

            const message = producedMessages[0]
            expect(message.topic).toBe('clickhouse_logs_test')
            expect(message.key).toBeNull()
            expect(message.headers).toEqual({
                token: team.api_token,
                team_id: team.id.toString(),
                'json-parse': 'false',
                'retention-days': DEFAULT_LOGS_RETENTION_DAYS.toString(),
                source_topic: messages[0].topic,
                source_partition: messages[0].partition.toString(),
            })
        })

        it('should use custom logs_settings when set', async () => {
            // Update team with custom logs settings
            await hub.postgres.query(
                PostgresUse.COMMON_WRITE,
                `UPDATE posthog_team
                 SET logs_settings = $1
                 WHERE id = $2`,
                [JSON.stringify({ json_parse_logs: false, retention_days: 30 }), team.id],
                'updateTeamLogsSettings'
            )

            const logData = createLogMessage()
            const messages = await createKafkaMessages([logData], {
                token: team.api_token,
            })

            await waitForBackgroundTasks(consumer.processKafkaBatch(messages))

            const producedMessages = getProducedKafkaMessages()
            expect(producedMessages).toHaveLength(1)

            const message = producedMessages[0]
            expect(message.headers).toEqual({
                token: team.api_token,
                team_id: team.id.toString(),
                'json-parse': 'false',
                'retention-days': '30',
                source_topic: messages[0].topic,
                source_partition: messages[0].partition.toString(),
            })
        })
    })

    describe('service interface', () => {
        it('should provide correct service interface', () => {
            const service = consumer.service

            expect(service.id).toBe('LogsIngestionConsumer')
            expect(typeof service.onShutdown).toBe('function')
            expect(typeof service.healthcheck).toBe('function')
        })

        it('should return healthy status', () => {
            const healthCheck = consumer.isHealthy()

            expect(healthCheck).toEqual({ status: 'healthy' })
        })
    })

    describe('error handling', () => {
        const kafkaDeliveryError = (code: number): Error => LibrdKafkaError.create(new Error(err2str(code)))

        it.each([
            ['a processing error', () => new Error('Producer error')],
            ['a Kafka error that a replay cannot fix', () => kafkaDeliveryError(CODES.ERRORS.ERR_INVALID_RECORD)],
        ])('should send failed messages to DLQ on %s', async (_, makeError) => {
            const logData = createLogMessage()
            const messages = await createKafkaMessages([logData], {
                token: team.api_token,
            })

            const logMessageDlqCounterSpy = jest.spyOn(logMessageDlqCounter, 'inc')

            // Throw for the main logs topic but pass through to the real mock producer for the DLQ.
            const originalQueueMessages = mockProducer.queueMessages
            const passthrough = originalQueueMessages.bind(mockProducer)
            const queueSpy = jest.fn().mockImplementation((topicMessages) => {
                const t = Array.isArray(topicMessages) ? topicMessages[0]?.topic : topicMessages.topic
                if (t === 'clickhouse_logs_test') {
                    throw makeError()
                }
                return passthrough(topicMessages)
            })
            mockProducer.queueMessages = queueSpy

            try {
                await waitForBackgroundTasks(consumer.processKafkaBatch(messages))

                // Check that DLQ counter was incremented with team_id
                expect(logMessageDlqCounterSpy).toHaveBeenCalledWith({
                    reason: 'Error',
                    team_id: team.id.toString(),
                })

                // Check that a message was produced to the DLQ topic
                const dlqCalls = queueSpy.mock.calls.filter((call) => {
                    const t = Array.isArray(call[0]) ? call[0][0]?.topic : call[0].topic
                    return t === 'logs_ingestion_dlq_test'
                })
                expect(dlqCalls.length).toBeGreaterThan(0)

                // Verify DLQ message has error metadata in headers
                const dlqArg = dlqCalls[0][0]
                const dlqMessage = (Array.isArray(dlqArg) ? dlqArg[0] : dlqArg).messages[0]
                expect(dlqMessage.headers).toHaveProperty('error_message')
                expect(dlqMessage.headers).toHaveProperty('error_name')
                expect(dlqMessage.headers).toHaveProperty('failed_at')
                // Position of the source record, so it can be read back and replayed
                expect(dlqMessage.headers.source_topic).toEqual(messages[0].topic)
                expect(dlqMessage.headers.source_partition).toEqual(messages[0].partition.toString())
                expect(dlqMessage.headers.source_offset).toEqual(messages[0].offset.toString())
            } finally {
                mockProducer.queueMessages = originalQueueMessages
            }
        })

        it('should retry a dependency blip in place and produce the message', async () => {
            const logData = createLogMessage()
            const messages = await createKafkaMessages([logData], {
                token: team.api_token,
            })

            const originalQueueMessages = mockProducer.queueMessages
            const passthrough = originalQueueMessages.bind(mockProducer)
            let logsProduceAttempts = 0
            mockProducer.queueMessages = jest.fn().mockImplementation((topicMessages) => {
                const t = Array.isArray(topicMessages) ? topicMessages[0]?.topic : topicMessages.topic
                if (t === KAFKA_LOGS_CLICKHOUSE && logsProduceAttempts++ === 0) {
                    throw new DependencyUnavailableError('broker down', 'Kafka', new Error('broker down'))
                }
                return passthrough(topicMessages)
            })

            try {
                // A blip that clears inside the backoff must not fail the batch: a replay would
                // produce every other message of the batch a second time.
                await waitForBackgroundTasks(consumer.processKafkaBatch(messages))

                const produced = getProducedKafkaMessages()
                expect(produced.filter((m) => m.topic === KAFKA_LOGS_CLICKHOUSE)).toHaveLength(1)
                expect(produced.filter((m) => m.topic === KAFKA_LOGS_INGESTION_DLQ)).toHaveLength(0)
            } finally {
                mockProducer.queueMessages = originalQueueMessages
            }
        })

        it.each([
            [
                'a dependency is unavailable',
                () => new DependencyUnavailableError('broker down', 'Kafka', new Error('broker down')),
            ],
            ['a delivery times out', () => kafkaDeliveryError(CODES.ERRORS.ERR__MSG_TIMED_OUT)],
            ['the producer queue is full', () => LibrdKafkaError.create(CODES.ERRORS.ERR__QUEUE_FULL)],
            ['the topic rejects authorization', () => kafkaDeliveryError(CODES.ERRORS.ERR_TOPIC_AUTHORIZATION_FAILED)],
        ])('should fail the batch instead of DLQing when %s', async (_, makeError) => {
            const logData = createLogMessage()
            const messages = await createKafkaMessages([logData], {
                token: team.api_token,
            })

            const originalQueueMessages = mockProducer.queueMessages
            const passthrough = originalQueueMessages.bind(mockProducer)
            mockProducer.queueMessages = jest.fn().mockImplementation((topicMessages) => {
                const t = Array.isArray(topicMessages) ? topicMessages[0]?.topic : topicMessages.topic
                if (t === KAFKA_LOGS_CLICKHOUSE) {
                    throw makeError()
                }
                return passthrough(topicMessages)
            })

            try {
                // The message is good data that succeeds on redelivery. Quarantining it would
                // move it out of the source topic for an outage that was not its fault.
                await expect(waitForBackgroundTasks(consumer.processKafkaBatch(messages))).rejects.toThrow()
                expect(getProducedKafkaMessages().filter((m) => m.topic === KAFKA_LOGS_INGESTION_DLQ)).toHaveLength(0)
            } finally {
                mockProducer.queueMessages = originalQueueMessages
            }
        })

        it('should send a message with an unparseable size header to DLQ instead of ingesting it', async () => {
            const logData = createLogMessage()
            const messages = await createKafkaMessages([logData], {
                token: team.api_token,
                // NaN here would otherwise flow into the usage counters and per-team billing stats
                bytes_uncompressed: 'not-a-number',
            })

            await waitForBackgroundTasks(consumer.processKafkaBatch(messages))

            const produced = getProducedKafkaMessages()
            expect(produced.filter((m) => m.topic === KAFKA_LOGS_CLICKHOUSE)).toHaveLength(0)

            const dlq = produced.filter((m) => m.topic === KAFKA_LOGS_INGESTION_DLQ)
            expect(dlq).toHaveLength(1)
            expect(dlq[0]?.headers?.error_message).toEqual('invalid_size_headers')
            expect(dlq[0]?.headers?.team_id).toEqual(team.id.toString())
        })

        it.each([
            ['the message fails to parse', { bytes_uncompressed: 'not-a-number' }],
            ['the message fails to process', {}],
        ])('should fail the batch when %s and cannot be written to the DLQ', async (_, headers) => {
            const logData = createLogMessage()
            const messages = await createKafkaMessages([logData], {
                token: team.api_token,
                ...headers,
            })

            // Every produce fails: the logs topic write fails the message, and the DLQ write that
            // follows fails too.
            const originalQueueMessages = mockProducer.queueMessages
            mockProducer.queueMessages = jest.fn().mockRejectedValue(new Error('DLQ unavailable'))

            try {
                // Resolving here would commit the source offset with no copy anywhere, so the
                // only record of the payload would be gone.
                await expect(waitForBackgroundTasks(consumer.processKafkaBatch(messages))).rejects.toThrow(
                    'DLQ unavailable'
                )
            } finally {
                mockProducer.queueMessages = originalQueueMessages
            }
        })
    })

    describe('describeBatchPosition', () => {
        it('collapses each partition to its offset span', () => {
            const at = (partition: number, offset: number): Message => ({ partition, offset }) as Message

            expect(describeBatchPosition([at(3, 1000), at(3, 1099), at(7, 42)])).toEqual({
                '3': '1000-1099',
                '7': '42',
            })
        })
    })

    describe('parseSizeHeader', () => {
        it.each([
            ['1234', 1234],
            [undefined, 0],
            ['not-a-number', null],
            ['-5', null],
        ])('parses %p as %p', (raw, expected) => {
            expect(parseSizeHeader(raw)).toEqual(expected)
        })
    })

    describe('parseMinTimestampHeader', () => {
        it.each([
            ['1790000000000000', 1790000000000000],
            [undefined, undefined],
            ['', undefined],
            ['not-a-number', undefined],
        ])('parses %p as %p', (raw, expected) => {
            expect(parseMinTimestampHeader(raw)).toEqual(expected)
        })
    })

    describe('message routing', () => {
        it('should route messages to correct ClickHouse topic', async () => {
            const logData = createLogMessage()
            const messages = await createKafkaMessages([logData], {
                token: team.api_token,
            })

            await waitForBackgroundTasks(consumer.processKafkaBatch(messages))

            const producedMessages = getProducedKafkaMessages()
            expect(producedMessages).toHaveLength(1)
            expect(producedMessages[0].topic).toBe('clickhouse_logs_test')
        })

        it('should handle messages from different teams', async () => {
            const logData1 = createLogMessage({ message: 'Team 1 log' })
            const logData2 = createLogMessage({ message: 'Team 2 log' })

            const messages = [
                ...(await createKafkaMessages([logData1], {
                    token: team.api_token,
                })),
                ...(await createKafkaMessages([logData2], {
                    token: team2.api_token,
                })),
            ]

            await waitForBackgroundTasks(consumer.processKafkaBatch(messages))

            const producedMessages = getProducedKafkaMessages()
            expect(producedMessages).toHaveLength(2)

            const team1Message = producedMessages.find((m) => m.headers?.team_id === team.id.toString())
            const team2Message = producedMessages.find((m) => m.headers?.team_id === team2.id.toString())

            expect(team1Message).toBeDefined()
            expect(team2Message).toBeDefined()
            expect(team1Message?.headers?.token).toBe(team.api_token)
            expect(team2Message?.headers?.token).toBe(team2.api_token)
        })
    })

    describe('message content preservation', () => {
        it('should preserve original message content', async () => {
            const logData = createLogMessage({
                level: 'warn',
                message: 'Warning message',
                timestamp: '2025-01-01T12:00:00.000Z',
                service: 'api-service',
                request_id: 'req-123',
                user_id: 'user-456',
            })

            const messages = await createKafkaMessages([logData], {
                token: team.api_token,
            })

            await waitForBackgroundTasks(consumer.processKafkaBatch(messages))

            const producedMessages = getProducedKafkaMessages()
            expect(producedMessages).toHaveLength(1)

            expect(producedMessages[0].value).toEqual(messages[0].value)
        })

        it('should handle binary log data as we do not parse it', async () => {
            const binaryData = Buffer.from('binary log content')
            const messages = [
                {
                    key: null,
                    value: binaryData,
                    size: 1,
                    topic: 'test',
                    offset: offsetIncrementer++,
                    timestamp: DateTime.now().toMillis(),
                    partition: 1,
                    headers: [{ token: Buffer.from(team.api_token) }],
                } as Message,
            ]

            await waitForBackgroundTasks(consumer.processKafkaBatch(messages))

            const producedMessages = mockProducerObserver.getProducedMessages()
            expect(producedMessages).toHaveLength(1)
            expect(producedMessages[0].messages[0].value).toEqual(binaryData)
        })
    })

    describe('rate limiting', () => {
        let bytesReceivedSpy: jest.SpyInstance
        let bytesAllowedSpy: jest.SpyInstance
        let bytesDroppedSpy: jest.SpyInstance
        let recordsReceivedSpy: jest.SpyInstance
        let recordsAllowedSpy: jest.SpyInstance
        let recordsDroppedSpy: jest.SpyInstance

        beforeEach(() => {
            bytesReceivedSpy = jest.spyOn(logsBytesReceivedCounter, 'inc')
            bytesAllowedSpy = jest.spyOn(logsBytesAllowedCounter, 'inc')
            bytesDroppedSpy = jest.spyOn(logsBytesDroppedCounter, 'inc')
            recordsReceivedSpy = jest.spyOn(logsRecordsReceivedCounter, 'inc')
            recordsAllowedSpy = jest.spyOn(logsRecordsAllowedCounter, 'inc')
            recordsDroppedSpy = jest.spyOn(logsRecordsDroppedCounter, 'inc')
        })

        it('should track metrics for messages', async () => {
            hub.LOGS_LIMITER_BUCKET_SIZE_KB = 2
            hub.LOGS_LIMITER_REFILL_RATE_KB_PER_SECOND = 1
            hub.LOGS_LIMITER_TTL_SECONDS = 3600

            await consumer.stop()
            consumer = await createLogsIngestionConsumer(hub)

            const logData1 = createLogMessage({ message: 'First' })
            const logData2 = createLogMessage({ message: 'Second' })

            const messages = [
                ...(await createKafkaMessages([logData1], {
                    token: team.api_token,
                    bytes_uncompressed: '1024',
                    bytes_compressed: '512',
                    record_count: '5',
                })),
                ...(await createKafkaMessages([logData2], {
                    token: team.api_token,
                    bytes_uncompressed: '2048',
                    bytes_compressed: '1024',
                    record_count: '10',
                })),
            ]

            await waitForBackgroundTasks(consumer.processKafkaBatch(messages))

            // Filter to only the logs topic (excludes app_metrics2 messages)
            const logsMessages = getProducedKafkaMessages().filter((m) => m.topic === 'clickhouse_logs_test')
            expect(logsMessages).toHaveLength(1)
            expect(bytesReceivedSpy).toHaveBeenCalledWith(3072)
            expect(bytesAllowedSpy).toHaveBeenCalledWith(1024)
            expect(bytesDroppedSpy).toHaveBeenCalledWith({ team_id: team.id.toString() }, 2048)
            expect(recordsReceivedSpy).toHaveBeenCalledWith(15)
            expect(recordsAllowedSpy).toHaveBeenCalledWith(5)
            expect(recordsDroppedSpy).toHaveBeenCalledWith({ team_id: team.id.toString() }, 10)
            expect(logMessageDroppedCounterSpy).toHaveBeenCalledWith(
                { reason: 'rate_limited', team_id: team.id.toString() },
                1
            )
        })

        it('dual-emits usage counters into the OTel metrics push', async () => {
            // Wiring guard: the unit tests on ingestion-otel-metrics prove the record
            // functions work; this proves the consumer actually calls them.
            const otelExporter = new InMemoryMetricExporter(0)
            const otelReader = new PeriodicExportingMetricReader({
                exporter: otelExporter,
                exportIntervalMillis: 60_000,
            })
            const otelProvider = new MeterProvider({ readers: [otelReader] })
            metricsApi.setGlobalMeterProvider(otelProvider)
            resetLogsIngestionInstrumentsForTests()

            try {
                hub.LOGS_LIMITER_BUCKET_SIZE_KB = 2
                hub.LOGS_LIMITER_REFILL_RATE_KB_PER_SECOND = 1
                hub.LOGS_LIMITER_TTL_SECONDS = 3600

                await consumer.stop()
                consumer = await createLogsIngestionConsumer(hub)

                const messages = [
                    ...(await createKafkaMessages([createLogMessage({ message: 'First' })], {
                        token: team.api_token,
                        bytes_uncompressed: '1024',
                        bytes_compressed: '512',
                        record_count: '5',
                    })),
                    ...(await createKafkaMessages([createLogMessage({ message: 'Second' })], {
                        token: team.api_token,
                        bytes_uncompressed: '2048',
                        bytes_compressed: '1024',
                        record_count: '10',
                    })),
                ]

                await waitForBackgroundTasks(consumer.processKafkaBatch(messages))
                await otelReader.forceFlush()

                const pointsByMetric = new Map(
                    otelExporter
                        .getMetrics()
                        .flatMap((rm) => rm.scopeMetrics)
                        .flatMap((sm) => sm.metrics)
                        .map((m) => [m.descriptor.name, m.dataPoints as readonly DataPoint<number>[]])
                )
                expect(pointsByMetric.get('logs_ingestion_bytes_received_total')?.[0]?.value).toEqual(3072)
                expect(pointsByMetric.get('logs_ingestion_bytes_allowed_total')?.[0]?.value).toEqual(1024)
                expect(pointsByMetric.get('logs_ingestion_bytes_dropped_total')?.[0]).toMatchObject({
                    attributes: { team_id: team.id.toString() },
                    value: 2048,
                })
                expect(pointsByMetric.get('logs_ingestion_message_dropped_count')?.[0]).toMatchObject({
                    attributes: { reason: 'rate_limited', team_id: team.id.toString() },
                    value: 1,
                })
            } finally {
                await otelProvider.shutdown()
                metricsApi.disable()
                resetLogsIngestionInstrumentsForTests()
            }
        })

        it('should handle missing header values with defaults', async () => {
            const logData = createLogMessage()
            const messages = await createKafkaMessages([logData], {
                token: team.api_token,
            })

            await waitForBackgroundTasks(consumer.processKafkaBatch(messages))

            expect(getProducedKafkaMessages()).toHaveLength(1)
            expect(bytesReceivedSpy).toHaveBeenCalledWith(0)
            expect(recordsReceivedSpy).toHaveBeenCalledWith(0)
        })
    })

    describe('backfill gate', () => {
        it.each([
            {
                name: 'refuses a backfill request from a team not on the list',
                header: true,
                listed: false,
                produced: 0,
            },
            { name: 'accepts a backfill request from a listed team', header: true, listed: true, produced: 1 },
            { name: 'leaves live traffic alone', header: false, listed: false, produced: 1 },
        ])('$name', async ({ header, listed, produced }) => {
            await consumer.stop()
            consumer = await createLogsIngestionConsumer(hub, {
                LOGS_BACKFILL_ENABLED_TEAMS: listed ? String(team.id) : String(team2.id),
            })

            // The row is recent on purpose: a backfill request can carry rows from the last day.
            const messages = await createKafkaMessages([createLogMessage()], {
                token: team.api_token,
                record_count: '1',
                ...(header ? { backfill_days: '540' } : {}),
            })

            await waitForBackgroundTasks(consumer.processKafkaBatch(messages))

            expect(getProducedKafkaMessages().filter((m) => m.topic === KAFKA_LOGS_CLICKHOUSE)).toHaveLength(produced)
            if (produced === 0) {
                expect(logMessageDroppedCounterSpy).toHaveBeenCalledWith(
                    { reason: 'backfill_not_enabled', team_id: team.id.toString() },
                    1
                )
            }
        })
    })

    describe('internal logs', () => {
        const SIGNING_KEY = 'internal-logs-test-key'

        it.each([
            {
                name: "stores a signed batch outside quota, billing and the team's drop rules",
                signed: true,
                billed: false,
            },
            { name: 'bills a batch whose signature does not verify', signed: false, billed: true },
            {
                name: "bills a signed batch from a billed source but skips the team's drop rules",
                signed: true,
                billed: true,
            },
        ])('$name', async ({ signed, billed }) => {
            await consumer.stop()
            const dropEverything: Pick<SamplingRulesCache, 'getCompiledRuleSet'> = {
                getCompiledRuleSet: () =>
                    Promise.resolve(
                        compileRuleSet([
                            {
                                id: '00000000-0000-0000-0000-0000000000aa',
                                rule_type: 'severity_sampling',
                                scope_service: 'test-service',
                                scope_path_pattern: null,
                                scope_attribute_filters: [],
                                config: { actions: { INFO: { type: 'drop' } } },
                            },
                        ])
                    ),
            }
            consumer = await createLogsIngestionConsumer(
                hub,
                {
                    LOGS_INTERNAL_INGESTION_ENABLED_TEAMS: String(team.id),
                    LOGS_INTERNAL_INGESTION_SIGNING_KEYS: SIGNING_KEY,
                },
                {
                    samplingRulesCache: dropEverything as SamplingRulesCache,
                    internalSourcePolicies: {
                        hog_flow: { billed, teamRules: false, serviceName: 'posthog-internal-workflows' },
                    },
                }
            )
            // A quota limit drops every billed batch, so it applies only to the rows that test the internal bypass.
            jest.mocked(hub.quotaLimiting.isTeamTokenQuotaLimited).mockResolvedValue(!(signed && billed))

            const [message] = await createKafkaMessages([createLogMessage()], {
                token: team.api_token,
                bytes_uncompressed: '100',
                record_count: '1',
            })
            const signature = signed
                ? signInternalLogs(SIGNING_KEY, 'hog_flow', team.id, new Date(Date.now()))
                : 'v1.hog_flow.not-a-signature'
            message.headers!.push(internalLogsHeaders(signature))
            const recordLogsReceivedSpy = jest.spyOn(otelMetrics, 'recordLogsReceived')
            const recordInternalLogsSpy = jest.spyOn(otelMetrics, 'recordInternalLogs')

            await waitForBackgroundTasks(consumer.processKafkaBatch([message]))

            const metricNames = mockProducerObserver
                .getProducedKafkaMessagesForTopic(KAFKA_APP_METRICS_2)
                .filter((m) => m.value.team_id === team.id)
                .map((m) => m.value.metric_name)
            const produced = getProducedKafkaMessages().filter((m) => m.topic === KAFKA_LOGS_CLICKHOUSE)
            expect({
                produced: produced.length,
                forwardsSignature: produced.some((m) => INTERNAL_LOGS_HEADER in (m.headers ?? {})),
                internalSource: produced[0]?.headers?.[INTERNAL_SOURCE_HEADER],
                received: recordLogsReceivedSpy.mock.calls,
                internal: recordInternalLogsSpy.mock.calls,
                internalMetrics: ['bytes_ingested_internal', 'records_ingested_internal'].every((n) =>
                    metricNames.includes(n)
                ),
                billedMetrics: ['bytes_ingested', 'records_ingested', 'bytes_received'].some((n) =>
                    metricNames.includes(n)
                ),
            }).toEqual({
                produced: signed ? 1 : 0,
                forwardsSignature: false,
                internalSource: signed ? 'hog_flow' : undefined,
                received: [billed ? [100, 1] : [0, 0]],
                internal: billed ? [] : [[team.id, 100, 1]],
                internalMetrics: !billed,
                billedMetrics: billed,
            })
        })

        it('drops a row past its stamped retention without crediting billing', async () => {
            await consumer.stop()
            consumer = await createLogsIngestionConsumer(hub, {
                LOGS_INTERNAL_INGESTION_ENABLED_TEAMS: String(team.id),
                LOGS_INTERNAL_INGESTION_SIGNING_KEYS: SIGNING_KEY,
            })
            const fiveDaysAgoMicros = DateTime.now().minus({ days: 5 }).toMillis() * 1000
            const message = await createMultiRecordKafkaMessage(
                [{ level: 'info', timestampMicros: fiveDaysAgoMicros, retentionDays: 1 }, { level: 'info' }],
                {
                    token: team.api_token,
                    bytes_uncompressed: '200',
                    record_count: '2',
                    min_timestamp: fiveDaysAgoMicros.toString(),
                }
            )
            message.headers!.push(
                internalLogsHeaders(signInternalLogs(SIGNING_KEY, 'hog_flow', team.id, new Date(Date.now())))
            )
            const bytesCreditSpy = jest.spyOn(logsBillingBytesCreditedCounter, 'inc')
            const recordsCreditSpy = jest.spyOn(logsBillingRecordsCreditedCounter, 'inc')

            await waitForBackgroundTasks(consumer.processKafkaBatch([message]))

            const expiredMetric = mockProducerObserver
                .getProducedKafkaMessagesForTopic(KAFKA_APP_METRICS_2)
                .map((m) => m.value)
                .find((v) => v.metric_name === 'records_dropped_retention_expired' && v.team_id === team.id)
            expect({
                expired: expiredMetric?.count,
                produced: getProducedKafkaMessages().filter((m) => m.topic === KAFKA_LOGS_CLICKHOUSE).length,
                credits: [...bytesCreditSpy.mock.calls, ...recordsCreditSpy.mock.calls],
            }).toEqual({ expired: 1, produced: 1, credits: [] })
        })

        it.each([
            { name: 'drops a signed backfill without writing to the team usage row', backfill: true },
            { name: 'keeps the usage of a failed internal batch out of the next flush', backfill: false },
        ])('$name', async ({ backfill }) => {
            await consumer.stop()
            consumer = await createLogsIngestionConsumer(hub, {
                LOGS_BACKFILL_ENABLED_TEAMS: String(team2.id),
                LOGS_INTERNAL_INGESTION_ENABLED_TEAMS: String(team.id),
                LOGS_INTERNAL_INGESTION_SIGNING_KEYS: SIGNING_KEY,
            })
            const [internal] = await createKafkaMessages([createLogMessage()], {
                token: team.api_token,
                bytes_uncompressed: '100',
                record_count: '1',
                ...(backfill ? { backfill_days: '540' } : {}),
            })
            internal.headers!.push(
                internalLogsHeaders(signInternalLogs(SIGNING_KEY, 'hog_flow', team.id, new Date(Date.now())))
            )
            const originalQueueMessages = mockProducer.queueMessages
            const passthrough = originalQueueMessages.bind(mockProducer)
            mockProducer.queueMessages = jest.fn().mockImplementation((topicMessages) => {
                if (topicMessages.topic === KAFKA_LOGS_CLICKHOUSE && !backfill) {
                    throw new DependencyUnavailableError('broker down', 'Kafka', new Error('broker down'))
                }
                return passthrough(topicMessages)
            })

            try {
                await consumer.processKafkaBatch([internal]).then((r) => r.backgroundTask?.catch(() => undefined))
                mockProducer.queueMessages = originalQueueMessages
                // A later batch of another team flushes the shared usage aggregator.
                const [next] = await createKafkaMessages([createLogMessage()], { token: team2.api_token })
                await waitForBackgroundTasks(consumer.processKafkaBatch([next]))
            } finally {
                mockProducer.queueMessages = originalQueueMessages
            }

            expect(
                mockProducerObserver
                    .getProducedKafkaMessagesForTopic(KAFKA_APP_METRICS_2)
                    .filter((m) => m.value.team_id === team.id)
                    .map((m) => m.value.metric_name)
            ).toEqual([])
        })
    })

    describe('filterQuotaLimitedMessages', () => {
        it('should allow messages when not quota limited', async () => {
            jest.mocked(hub.quotaLimiting.isTeamTokenQuotaLimited).mockResolvedValue(false)

            const messages = await createKafkaMessages([createLogMessage()], {
                token: team.api_token,
                bytes_uncompressed: '1024',
                record_count: '5',
            })

            const parsed = await consumer['_parseKafkaBatch'](messages)
            const { quotaAllowedMessages, quotaDroppedMessages } = await consumer['filterQuotaLimitedMessages'](parsed)

            expect(quotaAllowedMessages).toHaveLength(1)
            expect(quotaDroppedMessages).toHaveLength(0)
            expect(hub.quotaLimiting.isTeamTokenQuotaLimited).toHaveBeenCalledWith(team.api_token, 'logs_mb_ingested')
        })

        it('should drop messages when quota limited', async () => {
            jest.mocked(hub.quotaLimiting.isTeamTokenQuotaLimited).mockResolvedValue(true)

            const messages = await createKafkaMessages([createLogMessage()], {
                token: team.api_token,
                bytes_uncompressed: '1024',
                record_count: '5',
            })

            const parsed = await consumer['_parseKafkaBatch'](messages)
            const { quotaAllowedMessages, quotaDroppedMessages } = await consumer['filterQuotaLimitedMessages'](parsed)

            expect(quotaAllowedMessages).toHaveLength(0)
            expect(quotaDroppedMessages).toHaveLength(1)
        })

        it('should check quota once per unique token', async () => {
            jest.mocked(hub.quotaLimiting.isTeamTokenQuotaLimited).mockResolvedValue(false)

            const messages = [
                ...(await createKafkaMessages([createLogMessage()], {
                    token: team.api_token,
                    bytes_uncompressed: '100',
                    record_count: '1',
                })),
                ...(await createKafkaMessages([createLogMessage()], {
                    token: team.api_token,
                    bytes_uncompressed: '200',
                    record_count: '2',
                })),
                ...(await createKafkaMessages([createLogMessage()], {
                    token: team2.api_token,
                    bytes_uncompressed: '300',
                    record_count: '3',
                })),
            ]

            const parsed = await consumer['_parseKafkaBatch'](messages)
            await consumer['filterQuotaLimitedMessages'](parsed)

            // Should only call once per unique token, not once per message
            expect(hub.quotaLimiting.isTeamTokenQuotaLimited).toHaveBeenCalledTimes(2)
        })

        it('should handle mixed quota limited and non-limited teams', async () => {
            jest.mocked(hub.quotaLimiting.isTeamTokenQuotaLimited).mockImplementation((token) => {
                return Promise.resolve(token === team.api_token) // team1 is limited, team2 is not
            })

            const messages = [
                ...(await createKafkaMessages([createLogMessage()], {
                    token: team.api_token,
                    bytes_uncompressed: '100',
                    record_count: '1',
                })),
                ...(await createKafkaMessages([createLogMessage()], {
                    token: team2.api_token,
                    bytes_uncompressed: '200',
                    record_count: '2',
                })),
            ]

            const parsed = await consumer['_parseKafkaBatch'](messages)
            const { quotaAllowedMessages, quotaDroppedMessages } = await consumer['filterQuotaLimitedMessages'](parsed)

            expect(quotaAllowedMessages).toHaveLength(1)
            expect(quotaAllowedMessages[0].token).toBe(team2.api_token)
            expect(quotaDroppedMessages).toHaveLength(1)
            expect(quotaDroppedMessages[0].token).toBe(team.api_token)
        })

        it('should increment dropped counter when quota limited', async () => {
            jest.mocked(hub.quotaLimiting.isTeamTokenQuotaLimited).mockResolvedValue(true)

            const messages = await createKafkaMessages([createLogMessage(), createLogMessage()], {
                token: team.api_token,
            })

            const parsed = await consumer['_parseKafkaBatch'](messages)
            await consumer['filterQuotaLimitedMessages'](parsed)

            expect(logMessageDroppedCounterSpy).toHaveBeenCalledWith(
                { reason: 'quota_limited', team_id: team.id.toString() },
                2
            )
            expect(logMessageDroppedCounterSpy).toHaveBeenCalledTimes(1)
        })
    })

    describe('filterRateLimitedMessages', () => {
        it('should return allowed messages', async () => {
            const logData = createLogMessage()
            const messages = await createKafkaMessages([logData], {
                token: team.api_token,
                bytes_uncompressed: '1024',
                record_count: '5',
            })

            const parsed = await consumer['_parseKafkaBatch'](messages)
            const { rateLimiterAllowedMessages, rateLimiterDroppedMessages } =
                await consumer['filterRateLimitedMessages'](parsed)

            expect(rateLimiterAllowedMessages).toHaveLength(1)
            expect(rateLimiterDroppedMessages).toHaveLength(0)
        })

        it('should track dropped messages when rate limited', async () => {
            hub.LOGS_LIMITER_BUCKET_SIZE_KB = 1 // 1024 bytes - allows first message
            hub.LOGS_LIMITER_REFILL_RATE_KB_PER_SECOND = 0.001
            hub.LOGS_LIMITER_TTL_SECONDS = 3600

            await consumer.stop()
            consumer = await createLogsIngestionConsumer(hub)

            const messages = [
                ...(await createKafkaMessages([createLogMessage()], {
                    token: team.api_token,
                    bytes_uncompressed: '512', // Fits in bucket
                    record_count: '1',
                })),
                ...(await createKafkaMessages([createLogMessage()], {
                    token: team.api_token,
                    bytes_uncompressed: '2048', // Exceeds remaining bucket
                    record_count: '5',
                })),
            ]

            const parsed = await consumer['_parseKafkaBatch'](messages)
            const { rateLimiterAllowedMessages, rateLimiterDroppedMessages } =
                await consumer['filterRateLimitedMessages'](parsed)

            expect(rateLimiterAllowedMessages).toHaveLength(1)
            expect(rateLimiterDroppedMessages).toHaveLength(1)
        })
    })

    describe('quota and rate limiting integration', () => {
        it('should apply quota limiting before rate limiting', async () => {
            // Team1 is quota limited, team2 is not
            jest.mocked(hub.quotaLimiting.isTeamTokenQuotaLimited).mockImplementation((token) => {
                return Promise.resolve(token === team.api_token)
            })

            const messages = [
                ...(await createKafkaMessages([createLogMessage()], {
                    token: team.api_token, // quota limited - should be dropped before rate limiter
                    bytes_uncompressed: '100',
                    record_count: '1',
                })),
                ...(await createKafkaMessages([createLogMessage()], {
                    token: team2.api_token, // not quota limited - goes to rate limiter
                    bytes_uncompressed: '200',
                    record_count: '2',
                })),
            ]

            const result = await consumer.processKafkaBatch(messages)

            // Only team2's message should be allowed through
            expect(result.messages).toHaveLength(1)
            expect(result.messages[0].token).toBe(team2.api_token)

            // Verify quota_limited counter was incremented for team1
            expect(logMessageDroppedCounterSpy).toHaveBeenCalledWith(
                { reason: 'quota_limited', team_id: team.id.toString() },
                1
            )
        })

        it('should drop messages from both quota and rate limiting', async () => {
            // Set up rate limit that allows first message but not second
            hub.LOGS_LIMITER_BUCKET_SIZE_KB = 1 // 1024 bytes
            hub.LOGS_LIMITER_REFILL_RATE_KB_PER_SECOND = 0.001
            hub.LOGS_LIMITER_TTL_SECONDS = 3600

            await consumer.stop()
            consumer = await createLogsIngestionConsumer(hub)

            // Team1 is quota limited
            jest.mocked(hub.quotaLimiting.isTeamTokenQuotaLimited).mockImplementation((token) => {
                return Promise.resolve(token === team.api_token)
            })

            const messages = [
                ...(await createKafkaMessages([createLogMessage()], {
                    token: team.api_token, // quota limited
                    bytes_uncompressed: '100',
                    record_count: '1',
                })),
                ...(await createKafkaMessages([createLogMessage()], {
                    token: team2.api_token, // passes quota, fits in rate limit bucket
                    bytes_uncompressed: '500',
                    record_count: '1',
                })),
                ...(await createKafkaMessages([createLogMessage()], {
                    token: team2.api_token, // passes quota, but exceeds rate limit
                    bytes_uncompressed: '2000',
                    record_count: '2',
                })),
            ]

            const result = await consumer.processKafkaBatch(messages)

            // Only first team2 message should be allowed (quota ok + rate limit ok)
            expect(result.messages).toHaveLength(1)
            expect(result.messages[0].bytesUncompressed).toBe(500)

            // Verify both drop reasons were recorded
            expect(logMessageDroppedCounterSpy).toHaveBeenCalledWith(
                { reason: 'quota_limited', team_id: team.id.toString() },
                1
            )
            expect(logMessageDroppedCounterSpy).toHaveBeenCalledWith(
                { reason: 'rate_limited', team_id: team2.id.toString() },
                1
            )
        })

        it('should track usage stats correctly for mixed allowed and dropped messages', async () => {
            // Team1 is quota limited
            jest.mocked(hub.quotaLimiting.isTeamTokenQuotaLimited).mockImplementation((token) => {
                return Promise.resolve(token === team.api_token)
            })

            const messages = [
                ...(await createKafkaMessages([createLogMessage()], {
                    token: team.api_token, // quota limited - dropped
                    bytes_uncompressed: '100',
                    record_count: '1',
                })),
                ...(await createKafkaMessages([createLogMessage()], {
                    token: team2.api_token, // allowed
                    bytes_uncompressed: '200',
                    record_count: '2',
                })),
            ]

            await waitForBackgroundTasks(consumer.processKafkaBatch(messages))

            // Check that app_metrics were emitted for both teams
            const appMetricsMessages = mockProducerObserver.getProducedKafkaMessagesForTopic(KAFKA_APP_METRICS_2)

            // Team1 should have bytes_received and bytes_dropped
            const team1Metrics = appMetricsMessages.filter((m) => m.value.team_id === team.id)
            const team1BytesReceived = team1Metrics.find((m) => m.value.metric_name === 'bytes_received')
            const team1BytesDropped = team1Metrics.find((m) => m.value.metric_name === 'bytes_dropped')
            expect(team1BytesReceived?.value.count).toBe(100)
            expect(team1BytesDropped?.value.count).toBe(100)

            // Team2 should have bytes_received and bytes_ingested
            const team2Metrics = appMetricsMessages.filter((m) => m.value.team_id === team2.id)
            const team2BytesReceived = team2Metrics.find((m) => m.value.metric_name === 'bytes_received')
            const team2BytesIngested = team2Metrics.find((m) => m.value.metric_name === 'bytes_ingested')
            expect(team2BytesReceived?.value.count).toBe(200)
            expect(team2BytesIngested?.value.count).toBe(200)
        })

        it('should emit bytes_ingested_records from the bytes_uncompressed_records header', async () => {
            const messages = await createKafkaMessages([createLogMessage()], {
                token: team.api_token,
                bytes_uncompressed: '1024',
                bytes_uncompressed_records: '900',
                record_count: '5',
            })

            await waitForBackgroundTasks(consumer.processKafkaBatch(messages))

            const appMetricsMessages = mockProducerObserver.getProducedKafkaMessagesForTopic(KAFKA_APP_METRICS_2)
            const bytesIngested = appMetricsMessages.find((m) => m.value.metric_name === 'bytes_ingested')
            const bytesIngestedRecords = appMetricsMessages.find(
                (m) => m.value.metric_name === 'bytes_ingested_records'
            )
            expect(bytesIngested?.value.count).toBe(1024)
            expect(bytesIngestedRecords?.value.count).toBe(900)
        })

        it('should flag and still emit when bytes_uncompressed_records exceeds bytes_uncompressed', async () => {
            const exceedCounterSpy = jest.spyOn(logsRecordsBytesExceedPayloadCounter, 'inc')
            const messages = await createKafkaMessages([createLogMessage()], {
                token: team.api_token,
                bytes_uncompressed: '1024',
                bytes_uncompressed_records: '1500',
                record_count: '5',
            })

            await waitForBackgroundTasks(consumer.processKafkaBatch(messages))

            expect(exceedCounterSpy).toHaveBeenCalledWith({ team_id: team.id.toString() })

            // The comparison metric is emitted as-is, even above the payload size —
            // it exists precisely to surface this case.
            const appMetricsMessages = mockProducerObserver.getProducedKafkaMessagesForTopic(KAFKA_APP_METRICS_2)
            const bytesIngested = appMetricsMessages.find((m) => m.value.metric_name === 'bytes_ingested')
            const bytesIngestedRecords = appMetricsMessages.find(
                (m) => m.value.metric_name === 'bytes_ingested_records'
            )
            expect(bytesIngested?.value.count).toBe(1024)
            expect(bytesIngestedRecords?.value.count).toBe(1500)
        })
    })

    describe('trackOutgoingTrafficAndBuildUsageStats', () => {
        it('should return usageStats with correct structure for allowed messages', async () => {
            const logData = createLogMessage()
            const messages = await createKafkaMessages([logData], {
                token: team.api_token,
                bytes_uncompressed: '1024',
                record_count: '5',
            })

            const parsed = await consumer['_parseKafkaBatch'](messages)
            const usageStats = consumer['trackOutgoingTrafficAndBuildUsageStats'](parsed, [])

            const stats = usageStats.get(team.id)
            expect(stats).toBeDefined()
            expect(stats!.bytesReceived).toBe(1024)
            expect(stats!.recordsReceived).toBe(5)
            expect(stats!.bytesAllowed).toBe(1024)
            expect(stats!.recordsAllowed).toBe(5)
            expect(stats!.bytesDropped).toBe(0)
            expect(stats!.recordsDropped).toBe(0)
            expect(stats!.piiReplacements).toBe(0)
        })

        it('should sum bytesAllowedRecords and treat a missing header as zero', async () => {
            const messages = [
                ...(await createKafkaMessages([createLogMessage()], {
                    token: team.api_token,
                    bytes_uncompressed: '100',
                    bytes_uncompressed_records: '80',
                    record_count: '1',
                })),
                ...(await createKafkaMessages([createLogMessage()], {
                    token: team.api_token,
                    bytes_uncompressed: '200',
                    record_count: '2',
                })),
            ]

            const parsed = await consumer['_parseKafkaBatch'](messages)
            const usageStats = consumer['trackOutgoingTrafficAndBuildUsageStats'](parsed, [])

            const stats = usageStats.get(team.id)
            expect(stats!.bytesAllowed).toBe(300)
            expect(stats!.bytesAllowedRecords).toBe(80)
        })

        it('should aggregate stats for multiple messages from same team', async () => {
            const messages = [
                ...(await createKafkaMessages([createLogMessage()], {
                    token: team.api_token,
                    bytes_uncompressed: '100',
                    record_count: '1',
                })),
                ...(await createKafkaMessages([createLogMessage()], {
                    token: team.api_token,
                    bytes_uncompressed: '200',
                    record_count: '2',
                })),
            ]

            const parsed = await consumer['_parseKafkaBatch'](messages)
            const usageStats = consumer['trackOutgoingTrafficAndBuildUsageStats'](parsed, [])

            const stats = usageStats.get(team.id)
            expect(stats!.bytesReceived).toBe(300)
            expect(stats!.recordsReceived).toBe(3)
            expect(stats!.bytesAllowed).toBe(300)
            expect(stats!.recordsAllowed).toBe(3)
        })

        it('should track separate stats for different teams', async () => {
            const messages = [
                ...(await createKafkaMessages([createLogMessage()], {
                    token: team.api_token,
                    bytes_uncompressed: '100',
                    record_count: '1',
                })),
                ...(await createKafkaMessages([createLogMessage()], {
                    token: team2.api_token,
                    bytes_uncompressed: '200',
                    record_count: '2',
                })),
            ]

            const parsed = await consumer['_parseKafkaBatch'](messages)
            const usageStats = consumer['trackOutgoingTrafficAndBuildUsageStats'](parsed, [])

            expect(usageStats.size).toBe(2)
            expect(usageStats.get(team.id)!.bytesAllowed).toBe(100)
            expect(usageStats.get(team2.id)!.bytesAllowed).toBe(200)
        })

        it('should track dropped stats correctly', async () => {
            // Set up rate limit that allows first message but not second
            hub.LOGS_LIMITER_BUCKET_SIZE_KB = 1 // 1024 bytes
            hub.LOGS_LIMITER_REFILL_RATE_KB_PER_SECOND = 0.001
            hub.LOGS_LIMITER_TTL_SECONDS = 3600

            await consumer.stop()
            consumer = await createLogsIngestionConsumer(hub)

            const messages = [
                ...(await createKafkaMessages([createLogMessage()], {
                    token: team.api_token,
                    bytes_uncompressed: '512', // Fits in bucket
                    record_count: '1',
                })),
                ...(await createKafkaMessages([createLogMessage()], {
                    token: team.api_token,
                    bytes_uncompressed: '2048', // Exceeds remaining bucket
                    record_count: '5',
                })),
            ]

            const parsed = await consumer['_parseKafkaBatch'](messages)
            const { rateLimiterAllowedMessages, rateLimiterDroppedMessages } =
                await consumer['filterRateLimitedMessages'](parsed)
            const usageStats = consumer['trackOutgoingTrafficAndBuildUsageStats'](
                rateLimiterAllowedMessages,
                rateLimiterDroppedMessages
            )

            expect(usageStats.get(team.id)!.bytesAllowed).toBe(512)
            expect(usageStats.get(team.id)!.bytesDropped).toBe(2048)
        })
    })

    describe('queueUsageMetric', () => {
        const parseMetricValue = (value: any): any => {
            if (Buffer.isBuffer(value)) {
                return parseJSON(value.toString())
            }
            if (typeof value === 'string') {
                return parseJSON(value)
            }
            return value
        }

        it('should queue + flush metric with correct structure', async () => {
            consumer['queueUsageMetric'](123, 'test_metric', 500)
            await consumer['appMetricsAggregator'].flush()

            const messages = getProducedKafkaMessages().filter((m) => m.topic === KAFKA_APP_METRICS_2)

            expect(messages).toHaveLength(1)
            const value = parseMetricValue(messages[0].value)
            expect(value?.team_id).toBe(123)
            expect(value?.app_source).toBe('logs')
            expect(value?.app_source_id).toBe('')
            expect(value?.instance_id).toBe('')
            expect(value?.metric_kind).toBe('usage')
            expect(value?.metric_name).toBe('test_metric')
            expect(value?.count).toBe(500)
            // timestamp is now set at flush time, not by the caller
            expect(typeof value?.timestamp).toBe('string')
        })

        it('should not queue metric when count is zero', async () => {
            consumer['queueUsageMetric'](123, 'test_metric', 0)
            await consumer['appMetricsAggregator'].flush()

            const messages = getProducedKafkaMessages().filter((m) => m.topic === KAFKA_APP_METRICS_2)

            expect(messages).toHaveLength(0)
        })
    })

    describe('emitUsageMetrics', () => {
        const parseMetricValue = (value: any): any => {
            if (Buffer.isBuffer(value)) {
                return parseJSON(value.toString())
            }
            if (typeof value === 'string') {
                return parseJSON(value)
            }
            return value
        }
        it('should emit all metric types for each team', async () => {
            const usageStats = new Map([
                [
                    team.id,
                    {
                        bytesReceived: 1000,
                        recordsReceived: 10,
                        bytesAllowed: 800,
                        recordsAllowed: 8,
                        bytesAllowedRecords: 700,
                        bytesDropped: 200,
                        recordsDropped: 2,
                        piiReplacements: 0,
                        retentionDays: 30,
                    },
                ],
            ])

            await consumer['emitUsageMetrics'](usageStats)

            const messages = getProducedKafkaMessages().filter((m) => m.topic === KAFKA_APP_METRICS_2)

            // 7 base metrics + per-tier retention + retention_byte_days.
            expect(messages).toHaveLength(9)

            const metricNames = messages.map((m) => parseMetricValue(m.value)?.metric_name)
            expect(metricNames).toContain('bytes_received')
            expect(metricNames).toContain('records_received')
            expect(metricNames).toContain('bytes_ingested')
            expect(metricNames).toContain('bytes_ingested_records')
            expect(metricNames).toContain('records_ingested')
            expect(metricNames).toContain('bytes_dropped')
            expect(metricNames).toContain('records_dropped')
            expect(metricNames).toContain('bytes_ingested_retention_30d')
            expect(metricNames).toContain('retention_byte_days')

            const retentionByteDays = messages
                .map((m) => parseMetricValue(m.value))
                .find((m) => m?.metric_name === 'retention_byte_days')
            expect(retentionByteDays?.count).toBe(800 * 30)
        })

        it.each([
            { retentionDays: 14, tierMetric: 'bytes_ingested_retention_14d', byteDayCounts: [] },
            { retentionDays: 30, tierMetric: 'bytes_ingested_retention_30d', byteDayCounts: [500 * 30] },
            {
                retentionDays: 90,
                tierMetric: 'bytes_ingested_retention_90d',
                byteDayCounts: [500 * 30, 500 * 30, 500 * 30],
            },
            { retentionDays: 45, tierMetric: null, byteDayCounts: [500 * 30, 500 * 15] },
        ])(
            'should emit one retention_byte_days row per month for $retentionDays days alongside the per-tier metric',
            async ({ retentionDays, tierMetric, byteDayCounts }) => {
                const usageStats = new Map([
                    [
                        team.id,
                        {
                            bytesReceived: 500,
                            recordsReceived: 5,
                            bytesAllowed: 500,
                            recordsAllowed: 5,
                            bytesAllowedRecords: 0,
                            bytesDropped: 0,
                            recordsDropped: 0,
                            piiReplacements: 0,
                            retentionDays,
                        },
                    ],
                ])

                await consumer['emitUsageMetrics'](usageStats)

                const messages = getProducedKafkaMessages().filter((m) => m.topic === KAFKA_APP_METRICS_2)
                const parsed = messages.map((m) => parseMetricValue(m.value))

                const byteDayRows = parsed
                    .filter((m) => m?.metric_name === 'retention_byte_days')
                    .sort((a, b) => a.timestamp.localeCompare(b.timestamp))
                expect(byteDayRows.map((m) => m.count)).toEqual(byteDayCounts)

                const tierMetrics = parsed.filter((m) => m?.metric_name?.startsWith('bytes_ingested_retention_'))
                if (tierMetric) {
                    expect(tierMetrics).toHaveLength(1)
                    expect(tierMetrics[0]?.metric_name).toBe(tierMetric)
                    expect(tierMetrics[0]?.count).toBe(500)
                } else {
                    expect(tierMetrics).toHaveLength(0)
                }
            }
        )

        it('should date each retention_byte_days row at the start of its month', async () => {
            const usageStats = new Map([
                [
                    team.id,
                    {
                        bytesReceived: 500,
                        recordsReceived: 5,
                        bytesAllowed: 500,
                        recordsAllowed: 5,
                        bytesAllowedRecords: 0,
                        bytesDropped: 0,
                        recordsDropped: 0,
                        piiReplacements: 0,
                        retentionDays: 90,
                    },
                ],
            ])
            // Mid-hour, so the rows show the truncation to the hour.
            jest.spyOn(Date, 'now').mockReturnValue(DateTime.fromISO('2025-01-01T05:37:12Z').toMillis())

            await consumer['emitUsageMetrics'](usageStats)

            const timestamps = getProducedKafkaMessages()
                .filter((m) => m.topic === KAFKA_APP_METRICS_2)
                .map((m) => parseMetricValue(m.value))
                .filter((m) => m?.metric_name === 'retention_byte_days')
                .map((m) => m.timestamp)
                .sort()

            expect(timestamps).toEqual([
                '2025-01-01 05:00:00.000',
                '2025-01-31 05:00:00.000',
                '2025-03-02 05:00:00.000',
            ])
        })

        it('should keep the longest retention within ClickHouse partitions-per-insert limit', async () => {
            // app_metrics2 is partitioned by month, and ClickHouse fails an insert block that touches
            // more than 100 partitions by default. One flush at the maximum retention must stay well
            // under that, with room for the current month and a block that crosses a month boundary.
            const usageStats = new Map([
                [
                    team.id,
                    {
                        bytesReceived: 500,
                        recordsReceived: 5,
                        bytesAllowed: 500,
                        recordsAllowed: 5,
                        bytesAllowedRecords: 0,
                        bytesDropped: 0,
                        recordsDropped: 0,
                        piiReplacements: 0,
                        retentionDays: MAX_RETENTION_DAYS,
                    },
                ],
            ])

            await consumer['emitUsageMetrics'](usageStats)

            const months = new Set(
                getProducedKafkaMessages()
                    .filter((m) => m.topic === KAFKA_APP_METRICS_2)
                    .map((m) => parseMetricValue(m.value)?.timestamp.slice(0, 7))
            )
            expect(months.size).toBeLessThanOrEqual(95)
        })

        it('should skip zero-count metrics', async () => {
            const usageStats = new Map([
                [
                    team.id,
                    {
                        bytesReceived: 1000,
                        recordsReceived: 10,
                        bytesAllowed: 1000,
                        recordsAllowed: 10,
                        bytesAllowedRecords: 0,
                        bytesDropped: 0,
                        recordsDropped: 0,
                        piiReplacements: 0,
                        retentionDays: 14,
                    },
                ],
            ])

            await consumer['emitUsageMetrics'](usageStats)

            const messages = getProducedKafkaMessages().filter((m) => m.topic === KAFKA_APP_METRICS_2)

            // 4 non-zero base metrics (no dropped) + per-tier retention; no retention_byte_days for the default tier.
            expect(messages).toHaveLength(5)
            const metricNames = messages.map((m) => parseMetricValue(m.value)?.metric_name)
            expect(metricNames).not.toContain('bytes_dropped')
            expect(metricNames).not.toContain('records_dropped')
            expect(metricNames).toContain('bytes_ingested_retention_14d')
            expect(metricNames).not.toContain('retention_byte_days')
        })

        it('should handle empty usageStats', async () => {
            const usageStats = new Map()

            await consumer['emitUsageMetrics'](usageStats)

            const messages = getProducedKafkaMessages().filter((m) => m.topic === KAFKA_APP_METRICS_2)

            expect(messages).toHaveLength(0)
        })

        it('should emit metrics for multiple teams', async () => {
            const usageStats = new Map([
                [
                    team.id,
                    {
                        bytesReceived: 100,
                        recordsReceived: 1,
                        bytesAllowed: 100,
                        recordsAllowed: 1,
                        bytesAllowedRecords: 0,
                        bytesDropped: 0,
                        recordsDropped: 0,
                        piiReplacements: 0,
                        retentionDays: 30,
                    },
                ],
                [
                    team2.id,
                    {
                        bytesReceived: 200,
                        recordsReceived: 2,
                        bytesAllowed: 200,
                        recordsAllowed: 2,
                        bytesAllowedRecords: 0,
                        bytesDropped: 0,
                        recordsDropped: 0,
                        piiReplacements: 0,
                        retentionDays: 90,
                    },
                ],
            ])

            await consumer['emitUsageMetrics'](usageStats)

            const messages = getProducedKafkaMessages().filter((m) => m.topic === KAFKA_APP_METRICS_2)

            // 4 base metrics + per-tier retention + one retention_byte_days row per 30-day month
            // (no dropped): 6 for the 30-day team and 8 for the 90-day team.
            expect(messages).toHaveLength(14)

            const team1Messages = messages.filter((m) => parseMetricValue(m.value)?.team_id === team.id)
            const team2Messages = messages.filter((m) => parseMetricValue(m.value)?.team_id === team2.id)
            expect(team1Messages).toHaveLength(6)
            expect(team2Messages).toHaveLength(8)
        })
    })

    describe('app metrics emission (integration)', () => {
        const parseMetricValue = (value: any): any => {
            if (Buffer.isBuffer(value)) {
                return parseJSON(value.toString())
            }
            if (typeof value === 'string') {
                return parseJSON(value)
            }
            return value
        }

        it('should emit usage metrics to app_metrics2 topic', async () => {
            const logData = createLogMessage()
            const messages = await createKafkaMessages([logData], {
                token: team.api_token,
                bytes_uncompressed: '1024',
                record_count: '5',
            })

            await waitForBackgroundTasks(consumer.processKafkaBatch(messages))

            const appMetricsMessages = getProducedKafkaMessages().filter((m) => m.topic === KAFKA_APP_METRICS_2)

            expect(appMetricsMessages.length).toBeGreaterThan(0)

            const metricNames = appMetricsMessages.map((m) => {
                const value = parseMetricValue(m.value)
                return value.metric_name
            })

            expect(metricNames).toContain('bytes_received')
            expect(metricNames).toContain('records_received')
            expect(metricNames).toContain('bytes_ingested')
            expect(metricNames).toContain('records_ingested')
        })

        it('should emit pii_replacements when pii scrub is on and the body has pattern matches', async () => {
            await hub.postgres.query(
                PostgresUse.COMMON_WRITE,
                `UPDATE posthog_team
                 SET logs_settings = $1
                 WHERE id = $2`,
                [JSON.stringify({ pii_scrub_logs: true, json_parse_logs: false }), team.id],
                'updateTeamLogsPiiScrub'
            )

            const logData = createLogMessage({ message: 'email me at [email protected]' })
            const messages = await createKafkaMessages([logData], {
                token: team.api_token,
                bytes_uncompressed: '200',
                record_count: '1',
            })

            await waitForBackgroundTasks(consumer.processKafkaBatch(messages))

            const appMetricsMessages = getProducedKafkaMessages().filter((m) => m.topic === KAFKA_APP_METRICS_2)
            const pii = appMetricsMessages.find((m) => {
                const value = parseMetricValue(m.value)
                return value.metric_name === 'pii_replacements' && value.team_id === team.id
            })
            expect(pii).toBeDefined()
            expect(parseMetricValue(pii!.value).count).toBe(1)
        })

        it('should emit correct metric values per team', async () => {
            const logData = createLogMessage()
            const messages = await createKafkaMessages([logData], {
                token: team.api_token,
                bytes_uncompressed: '2048',
                record_count: '10',
            })

            await waitForBackgroundTasks(consumer.processKafkaBatch(messages))

            const appMetricsMessages = getProducedKafkaMessages().filter((m) => m.topic === KAFKA_APP_METRICS_2)

            const bytesIngestedMetric = appMetricsMessages.find((m) => {
                const value = parseMetricValue(m.value)
                return value.metric_name === 'bytes_ingested'
            })

            expect(bytesIngestedMetric).toBeDefined()
            const value = parseMetricValue(bytesIngestedMetric!.value)
            expect(value.team_id).toBe(team.id)
            expect(value.app_source).toBe('logs')
            expect(value.metric_kind).toBe('usage')
            expect(value.count).toBe(2048)
        })

        it('should emit dropped metrics when rate limited', async () => {
            hub.LOGS_LIMITER_BUCKET_SIZE_KB = 1
            hub.LOGS_LIMITER_REFILL_RATE_KB_PER_SECOND = 0.001
            hub.LOGS_LIMITER_TTL_SECONDS = 3600

            await consumer.stop()
            consumer = await createLogsIngestionConsumer(hub)

            const logData1 = createLogMessage({ message: 'First' })
            const logData2 = createLogMessage({ message: 'Second - will be dropped' })

            const messages = [
                ...(await createKafkaMessages([logData1], {
                    token: team.api_token,
                    bytes_uncompressed: '512',
                    record_count: '2',
                })),
                ...(await createKafkaMessages([logData2], {
                    token: team.api_token,
                    bytes_uncompressed: '2048',
                    record_count: '8',
                })),
            ]

            await waitForBackgroundTasks(consumer.processKafkaBatch(messages))

            const appMetricsMessages = getProducedKafkaMessages().filter((m) => m.topic === KAFKA_APP_METRICS_2)

            const bytesDroppedMetric = appMetricsMessages.find((m) => {
                const value = parseMetricValue(m.value)
                return value.metric_name === 'bytes_dropped'
            })

            expect(bytesDroppedMetric).toBeDefined()
            const value = parseMetricValue(bytesDroppedMetric!.value)
            expect(value.count).toBe(2048)
        })

        it('should aggregate metrics across multiple messages from same team', async () => {
            const logData1 = createLogMessage({ message: 'First' })
            const logData2 = createLogMessage({ message: 'Second' })

            const messages = [
                ...(await createKafkaMessages([logData1], {
                    token: team.api_token,
                    bytes_uncompressed: '100',
                    record_count: '1',
                })),
                ...(await createKafkaMessages([logData2], {
                    token: team.api_token,
                    bytes_uncompressed: '200',
                    record_count: '2',
                })),
            ]

            await waitForBackgroundTasks(consumer.processKafkaBatch(messages))

            const appMetricsMessages = getProducedKafkaMessages().filter((m) => m.topic === KAFKA_APP_METRICS_2)

            const bytesIngestedMetric = appMetricsMessages.find((m) => {
                const value = parseMetricValue(m.value)
                return value.metric_name === 'bytes_ingested' && value.team_id === team.id
            })

            expect(bytesIngestedMetric).toBeDefined()
            const value = parseMetricValue(bytesIngestedMetric!.value)
            expect(value.count).toBe(300) // 100 + 200
        })

        it('should emit separate metrics for different teams', async () => {
            const logData1 = createLogMessage({ message: 'Team 1' })
            const logData2 = createLogMessage({ message: 'Team 2' })

            const messages = [
                ...(await createKafkaMessages([logData1], {
                    token: team.api_token,
                    bytes_uncompressed: '100',
                    record_count: '1',
                })),
                ...(await createKafkaMessages([logData2], {
                    token: team2.api_token,
                    bytes_uncompressed: '200',
                    record_count: '2',
                })),
            ]

            await waitForBackgroundTasks(consumer.processKafkaBatch(messages))

            const appMetricsMessages = getProducedKafkaMessages().filter((m) => m.topic === KAFKA_APP_METRICS_2)

            const team1Metrics = appMetricsMessages.filter((m) => {
                const value = parseMetricValue(m.value)
                return value.team_id === team.id && value.metric_name === 'bytes_ingested'
            })

            const team2Metrics = appMetricsMessages.filter((m) => {
                const value = parseMetricValue(m.value)
                return value.team_id === team2.id && value.metric_name === 'bytes_ingested'
            })

            expect(team1Metrics).toHaveLength(1)
            expect(team2Metrics).toHaveLength(1)
            expect(parseMetricValue(team1Metrics[0].value).count).toBe(100)
            expect(parseMetricValue(team2Metrics[0].value).count).toBe(200)
        })

        it('should not emit metrics with zero count', async () => {
            const logData = createLogMessage()
            const messages = await createKafkaMessages([logData], {
                token: team.api_token,
                bytes_uncompressed: '100',
                record_count: '1',
            })

            await waitForBackgroundTasks(consumer.processKafkaBatch(messages))

            const appMetricsMessages = getProducedKafkaMessages().filter((m) => m.topic === KAFKA_APP_METRICS_2)

            // Should not have drop metrics since nothing was dropped or sampled out
            const droppedMetrics = appMetricsMessages.filter((m) => {
                const value = parseMetricValue(m.value)
                return (
                    value.metric_name === 'bytes_dropped' ||
                    value.metric_name === 'records_dropped' ||
                    value.metric_name === 'sampling_records_dropped_by_rule'
                )
            })

            expect(droppedMetrics).toHaveLength(0)
        })

        it('drops a row already past retention and reports it, when the header says the message can hold one', async () => {
            const fourHundredDaysAgoMicros = DateTime.now().minus({ days: 400 }).toMillis() * 1000
            const message = await createMultiRecordKafkaMessage(
                [{ level: 'info', timestampMicros: fourHundredDaysAgoMicros }],
                {
                    token: team.api_token,
                    bytes_uncompressed: '400',
                    record_count: '1',
                    min_timestamp: fourHundredDaysAgoMicros.toString(),
                }
            )

            await waitForBackgroundTasks(consumer.processKafkaBatch([message]))

            const produced = getProducedKafkaMessages()
            expect(produced.some((m) => m.topic === KAFKA_LOGS_CLICKHOUSE)).toBe(false)
            const expiredMetric = produced
                .filter((m) => m.topic === KAFKA_APP_METRICS_2)
                .map((m) => parseMetricValue(m.value))
                .find((v) => v.metric_name === 'records_dropped_retention_expired' && v.team_id === team.id)
            expect(expiredMetric?.count).toBe(1)
        })

        describe('sampling usage to app_metrics2', () => {
            let mockSamplingCache: Pick<SamplingRulesCache, 'getCompiledRuleSet'>

            beforeEach(async () => {
                await consumer.stop()
                mockSamplingCache = {
                    getCompiledRuleSet: (teamId: number) =>
                        Promise.resolve(
                            teamId === team.id
                                ? compileRuleSet([
                                      {
                                          id: '00000000-0000-0000-0000-0000000000aa',
                                          rule_type: 'severity_sampling',
                                          scope_service: 'test-service',
                                          scope_path_pattern: null,
                                          scope_attribute_filters: [],
                                          config: { actions: { INFO: { type: 'drop' } } },
                                      },
                                  ])
                                : compileRuleSet([])
                        ),
                }
                consumer = await createLogsIngestionConsumer(
                    hub,
                    {},
                    { samplingRulesCache: mockSamplingCache as SamplingRulesCache }
                )
            })

            it('should emit sampling_records_dropped_by_rule when head sampling drops log lines', async () => {
                const logData = createLogMessage({ level: 'info' })
                const messages = await createKafkaMessages([logData], {
                    token: team.api_token,
                    bytes_uncompressed: '400',
                    record_count: '1',
                })

                await waitForBackgroundTasks(consumer.processKafkaBatch(messages))

                const appMetricsMessages = getProducedKafkaMessages().filter((m) => m.topic === KAFKA_APP_METRICS_2)
                expect(
                    appMetricsMessages.some((m) => parseMetricValue(m.value).metric_name === 'sampling_records_dropped')
                ).toBe(false)

                const byRule = appMetricsMessages.find((m) => {
                    const value = parseMetricValue(m.value)
                    return (
                        value.metric_name === 'sampling_records_dropped_by_rule' &&
                        value.team_id === team.id &&
                        value.instance_id === '00000000-0000-0000-0000-0000000000aa'
                    )
                })
                expect(byRule).toBeDefined()
                expect(parseMetricValue(byRule!.value).count).toBe(1)
            })

            const sendDroppedAndKeptMessages = async () => {
                // Two single-record messages: the info one is fully dropped by the rule
                // (credit = its whole 400-byte header), the error one is kept (600 bytes).
                const messages = [
                    ...(await createKafkaMessages([createLogMessage({ level: 'info' })], {
                        token: team.api_token,
                        bytes_uncompressed: '400',
                        record_count: '1',
                    })),
                    ...(await createKafkaMessages([createLogMessage({ level: 'error' })], {
                        token: team.api_token,
                        bytes_uncompressed: '600',
                        record_count: '1',
                    })),
                ]
                await waitForBackgroundTasks(consumer.processKafkaBatch(messages))
            }

            const teamBytesIngested = (): number | undefined => {
                const m = getProducedKafkaMessages()
                    .filter((m) => m.topic === KAFKA_APP_METRICS_2)
                    .find((m) => {
                        const value = parseMetricValue(m.value)
                        return value.metric_name === 'bytes_ingested' && value.team_id === team.id
                    })
                return m ? parseMetricValue(m.value).count : undefined
            }

            it('shadow mode (default): computes the drop credit but bills the full gross bytes', async () => {
                const creditSpy = jest.spyOn(logsBillingBytesCreditedCounter, 'inc')

                await sendDroppedAndKeptMessages()

                // The would-be credit is computed and observable...
                expect(creditSpy).toHaveBeenCalledWith({ team_id: team.id.toString() }, 400)
                // ...but billed usage is untouched: full gross of both messages.
                expect(teamBytesIngested()).toBe(1000)
            })

            it('credits dropped rows off bytes_ingested when LOGS_BILLING_PRORATE_ENABLED', async () => {
                await consumer.stop()
                consumer = await createLogsIngestionConsumer(
                    hub,
                    { LOGS_BILLING_PRORATE_ENABLED: true },
                    { samplingRulesCache: mockSamplingCache as SamplingRulesCache }
                )

                await sendDroppedAndKeptMessages()

                // The fully-dropped message's 400-byte header is credited; only the kept one bills.
                expect(teamBytesIngested()).toBe(600)
            })

            const outputHeaders = (): Record<string, string> | undefined =>
                getProducedKafkaMessages().find((m) => m.topic === KAFKA_LOGS_CLICKHOUSE)?.headers as
                    | Record<string, string>
                    | undefined

            // One message, two equal-content records: the info row is dropped, the error row survives.
            // Shadow mode forwards the original gross headers; enabled mode scales both by the same
            // dropped content fraction (~0.5), so the 500/1000 ratio is preserved.
            it.each([
                { label: 'shadow mode (default) forwards the original gross size headers', enabled: false },
                { label: 'enabled mode pro-rates the forwarded size headers by the dropped fraction', enabled: true },
            ])('$label', async ({ enabled }) => {
                if (enabled) {
                    await consumer.stop()
                    consumer = await createLogsIngestionConsumer(
                        hub,
                        { LOGS_BILLING_PRORATE_ENABLED: true },
                        { samplingRulesCache: mockSamplingCache as SamplingRulesCache }
                    )
                }

                const message = await createMultiRecordKafkaMessage(
                    [createLogMessage({ level: 'info' }), createLogMessage({ level: 'error' })],
                    { token: team.api_token, bytes_uncompressed: '1000', bytes_compressed: '500', record_count: '2' }
                )
                await waitForBackgroundTasks(consumer.processKafkaBatch([message]))

                const uncompressed = parseInt(outputHeaders()!.bytes_uncompressed, 10)
                const compressed = parseInt(outputHeaders()!.bytes_compressed, 10)
                const recordCount = parseInt(outputHeaders()!.record_count, 10)
                if (!enabled) {
                    expect(uncompressed).toBe(1000)
                    expect(compressed).toBe(500)
                    expect(recordCount).toBe(2)
                    return
                }
                expect(uncompressed).toBeGreaterThan(0)
                expect(uncompressed).toBeLessThan(1000)
                expect(compressed).toBeGreaterThan(0)
                expect(compressed).toBeLessThan(500)
                expect(compressed / uncompressed).toBeCloseTo(0.5, 5)
                // The dropped info row is removed from the count exactly; only the error row survives.
                expect(recordCount).toBe(1)
            })
        })
    })

    describe('JSON attribute sniffing', () => {
        it.each([
            ['disabled', '', 'payload', false],
            ['different team', 'other_team', 'payload', false],
            ['allowlisted team', 'this_team', 'payload', true],
            ['wildcard', '*', 'payload', true],
            ['missing setting', '*', undefined, false],
            ['empty setting', '*', '', false],
        ] as const)('%s preserves output and only counts eligible teams', async (_, allowlist, key, enabled) => {
            await consumer.stop()
            consumer = await createLogsIngestionConsumer(hub, {
                LOGS_JSON_ATTRIBUTE_PARSING_ENABLED_TEAMS:
                    allowlist === 'this_team'
                        ? ` ${team.id}, ${team2.id} `
                        : allowlist === 'other_team'
                          ? String(team2.id)
                          : allowlist,
            })
            jest.spyOn(hub.teamManager, 'getTeam').mockResolvedValue({
                ...team,
                logs_settings: { json_parse_logs_attribute_key: key },
            })
            logsJsonAttributeSniffCounter.reset()
            const message = await createKafkaMessage(
                createLogMessage(),
                { token: team.api_token },
                { payload: JSON.stringify('{"nested":true}') }
            )
            if (!enabled) {
                message.value = Buffer.from('passthrough does not require Avro decoding')
            }

            await waitForBackgroundTasks(consumer.processKafkaBatch([message]))

            const logsOutput = getProducedKafkaMessages().filter((output) => output.topic === KAFKA_LOGS_CLICKHOUSE)
            expect(logsOutput).toHaveLength(1)
            expect(logsOutput[0].value).toEqual(message.value)
            expect((await logsJsonAttributeSniffCounter.get()).values).toEqual(
                enabled
                    ? [
                          expect.objectContaining({
                              labels: { team_id: String(team.id), outcome: 'looks_like_json' },
                              value: 1,
                          }),
                      ]
                    : []
            )
        })
    })

    describe('JSON attribute extraction', () => {
        it.each([
            ['disabled', '', 'payload', false],
            ['different team', 'other_team', 'payload', false],
            ['allowlisted team', 'this_team', 'payload', true],
            ['wildcard', '*', 'payload', true],
            ['missing setting', '*', undefined, false],
            ['empty setting', '*', '', false],
        ] as const)('%s only re-encodes eligible logs', async (_, allowlist, key, enabled) => {
            await consumer.stop()
            consumer = await createLogsIngestionConsumer(hub, {
                LOGS_JSON_ATTRIBUTE_EXTRACTION_ENABLED_TEAMS:
                    allowlist === 'this_team'
                        ? ` ${team.id}, ${team2.id} `
                        : allowlist === 'other_team'
                          ? String(team2.id)
                          : allowlist,
            })
            jest.spyOn(hub.teamManager, 'getTeam').mockResolvedValue({
                ...team,
                logs_settings: { json_parse_logs_attribute_key: key },
            })
            const message = await createKafkaMessage(
                createLogMessage(),
                { token: team.api_token },
                { payload: JSON.stringify('{"count":7}') }
            )
            if (!enabled) {
                message.value = Buffer.from('disabled extraction must not decode')
            }
            await waitForBackgroundTasks(consumer.processKafkaBatch([message]))
            const output = getProducedKafkaMessages().filter((entry) => entry.topic === KAFKA_LOGS_CLICKHOUSE)
            expect(output).toHaveLength(1)
            if (enabled) {
                const [, , records] = await decodeLogRecords(output[0].value as Buffer)
                expect(records[0].attributes).toMatchObject({
                    payload: JSON.stringify('{"count":7}'),
                    'payload.count': '7',
                })
            } else {
                expect(output[0].value).toEqual(message.value)
            }
        })
    })

    describe('metric rules (generate metrics from logs)', () => {
        let mockEmitter: { emit: jest.Mock }
        let mockMetricRulesCache: Pick<MetricRulesCache, 'getCompiledRules'>

        const countBySeverityRules = compileMetricRules([
            {
                id: '00000000-0000-0000-0000-0000000000bb',
                metric_name: 'log.records',
                filter_group: null,
                value_attribute: null,
                group_by: ['severity_text'],
            },
        ])

        const createConsumerWithMetricRules = async (overrides: any = {}, depsPartial: any = {}) => {
            await consumer.stop()
            consumer = await createLogsIngestionConsumer(hub, overrides, {
                metricRulesCache: mockMetricRulesCache as MetricRulesCache,
                metricsEmitter: mockEmitter as unknown as LogsMetricsEmitter,
                ...depsPartial,
            })
        }

        beforeEach(async () => {
            mockEmitter = { emit: jest.fn().mockResolvedValue(true) }
            mockMetricRulesCache = {
                getCompiledRules: (teamId: number) => Promise.resolve(teamId === team.id ? countBySeverityRules : []),
            }
            await createConsumerWithMetricRules()
        })

        const emittedDataPoints = (): any[] => {
            const payload = mockEmitter.emit.mock.calls[0][2]
            return payload.resourceMetrics[0].scopeMetrics[0].metrics[0].sum.dataPoints
        }

        it('emits one OTLP payload per team with counts grouped by the rule dimensions', async () => {
            await createConsumerWithMetricRules({ LOGS_JSON_ATTRIBUTE_PARSING_ENABLED_TEAMS: '*' })
            jest.spyOn(hub.teamManager, 'getTeam').mockResolvedValue({
                ...team,
                logs_settings: { json_parse_logs_attribute_key: 'payload' },
            })
            logsJsonAttributeSniffCounter.reset()
            const messages = await createKafkaMessages(
                [
                    createLogMessage({ level: 'info' }),
                    createLogMessage({ level: 'info' }),
                    createLogMessage({ level: 'error' }),
                ],
                { token: team.api_token, bytes_uncompressed: '400', record_count: '1' }
            )
            await waitForBackgroundTasks(consumer.processKafkaBatch(messages))

            expect(mockEmitter.emit).toHaveBeenCalledTimes(1)
            const [token, teamId] = mockEmitter.emit.mock.calls[0]
            expect(token).toBe(team.api_token)
            expect(teamId).toBe(team.id)

            const bySeverity = Object.fromEntries(
                emittedDataPoints().map((dp: any) => [dp.attributes[0].value.stringValue, dp.asDouble])
            )
            expect(bySeverity).toEqual({ info: 2, error: 1 })
            expect((await logsJsonAttributeSniffCounter.get()).values).toEqual([
                expect.objectContaining({
                    labels: { team_id: String(team.id), outcome: 'missing_key' },
                    value: 3,
                }),
            ])
        })

        it('emits nothing when a later internal message fails the batch', async () => {
            const key = 'internal-logs-test-key'
            await createConsumerWithMetricRules({
                LOGS_INTERNAL_INGESTION_ENABLED_TEAMS: String(team.id),
                LOGS_INTERNAL_INGESTION_SIGNING_KEYS: key,
            })
            const [billed, internal] = await createKafkaMessages([createLogMessage(), createLogMessage()], {
                token: team.api_token,
            })
            internal.headers!.push(
                internalLogsHeaders(signInternalLogs(key, 'hog_flow', team.id, new Date(Date.now())))
            )

            const originalQueueMessages = mockProducer.queueMessages
            const passthrough = originalQueueMessages.bind(mockProducer)
            mockProducer.queueMessages = jest.fn().mockImplementation((topicMessages) => {
                const isInternal = topicMessages.messages?.some(
                    (m: { headers?: Record<string, string> }) => m.headers?.[INTERNAL_SOURCE_HEADER]
                )
                if (topicMessages.topic === KAFKA_LOGS_CLICKHOUSE && isInternal) {
                    throw new DependencyUnavailableError('broker down', 'Kafka', new Error('broker down'))
                }
                return passthrough(topicMessages)
            })

            try {
                await expect(waitForBackgroundTasks(consumer.processKafkaBatch([billed, internal]))).rejects.toThrow(
                    'broker down'
                )
                expect(mockEmitter.emit).not.toHaveBeenCalled()
            } finally {
                mockProducer.queueMessages = originalQueueMessages
            }
        })

        it('produces logs normally when the rules cache fetch throws (fail-open)', async () => {
            // Metric rules are a purely additive side feature: a Postgres blip on the
            // rules fetch must not DLQ or block the log records themselves.
            await createConsumerWithMetricRules(
                {},
                {
                    metricRulesCache: {
                        getCompiledRules: () => Promise.reject(new Error('pg down')),
                    } as unknown as MetricRulesCache,
                }
            )

            const messages = await createKafkaMessages([createLogMessage({ level: 'info' })], {
                token: team.api_token,
                bytes_uncompressed: '400',
                record_count: '1',
            })
            await waitForBackgroundTasks(consumer.processKafkaBatch(messages))

            const logsOutput = getProducedKafkaMessages().filter((m) => m.topic === KAFKA_LOGS_CLICKHOUSE)
            expect(logsOutput).toHaveLength(1)
            const dlqOutput = getProducedKafkaMessages().filter((m) => m.topic === KAFKA_LOGS_INGESTION_DLQ)
            expect(dlqOutput).toHaveLength(0)
            expect(mockEmitter.emit).not.toHaveBeenCalled()
        })

        it('still counts records that a drop rule removes (pre-drop semantics)', async () => {
            const droppingSamplingCache: Pick<SamplingRulesCache, 'getCompiledRuleSet'> = {
                getCompiledRuleSet: () =>
                    Promise.resolve(
                        compileRuleSet([
                            {
                                id: '00000000-0000-0000-0000-0000000000aa',
                                rule_type: 'severity_sampling',
                                scope_service: 'test-service',
                                scope_path_pattern: null,
                                scope_attribute_filters: [],
                                config: { actions: { INFO: { type: 'drop' } } },
                            },
                        ])
                    ),
            }
            await createConsumerWithMetricRules({}, { samplingRulesCache: droppingSamplingCache })

            const message = await createMultiRecordKafkaMessage(
                [createLogMessage({ level: 'info' }), createLogMessage({ level: 'error' })],
                { token: team.api_token, bytes_uncompressed: '800', record_count: '2' }
            )
            await waitForBackgroundTasks(consumer.processKafkaBatch([message]))

            // The info row is gone from the ClickHouse-bound output...
            const logsOutput = getProducedKafkaMessages().filter((m) => m.topic === KAFKA_LOGS_CLICKHOUSE)
            expect(logsOutput).toHaveLength(1)
            // ...but both rows fed the generated metric.
            const bySeverity = Object.fromEntries(
                emittedDataPoints().map((dp: any) => [dp.attributes[0].value.stringValue, dp.asDouble])
            )
            expect(bySeverity).toEqual({ info: 1, error: 1 })
        })

        it('does not emit for a team without metric rules', async () => {
            const messages = await createKafkaMessages([createLogMessage()], {
                token: team2.api_token,
                bytes_uncompressed: '400',
                record_count: '1',
            })
            await waitForBackgroundTasks(consumer.processKafkaBatch(messages))
            expect(mockEmitter.emit).not.toHaveBeenCalled()
        })

        it.each([
            ['emits', {}, ['posthog.log.records']],
            ['does not emit under the killswitch', { LOGS_METRICS_RULES_KILLSWITCH: true }, null],
        ])(
            'default rules for an allowlisted team without rules of its own: %s',
            async (_label, overrides, expectedMetricNames) => {
                // Team rules are off, so only the default-rules allowlist can turn emission on.
                await createConsumerWithMetricRules({
                    LOGS_METRICS_RULES_ENABLED_TEAMS: '',
                    LOGS_DEFAULT_METRIC_RULES_ENABLED_TEAMS: String(team2.id),
                    ...overrides,
                })
                const messages = await createKafkaMessages([createLogMessage()], {
                    token: team2.api_token,
                    bytes_uncompressed: '400',
                    record_count: '1',
                })
                await waitForBackgroundTasks(consumer.processKafkaBatch(messages))
                if (expectedMetricNames === null) {
                    expect(mockEmitter.emit).not.toHaveBeenCalled()
                    return
                }
                expect(mockEmitter.emit).toHaveBeenCalledTimes(1)
                const [, teamId, payload] = mockEmitter.emit.mock.calls[0]
                expect(teamId).toBe(team2.id)
                const names = payload.resourceMetrics.flatMap((rm: any) =>
                    rm.scopeMetrics[0].metrics.map((m: any) => m.name)
                )
                expect(names).toEqual(expectedMetricNames)
            }
        )

        it.each([
            ['killswitch', { LOGS_METRICS_RULES_KILLSWITCH: true }],
            ['team gating', { LOGS_METRICS_RULES_ENABLED_TEAMS: '' }],
        ])('does not evaluate rules when disabled via %s', async (_label, overrides) => {
            await createConsumerWithMetricRules(overrides)
            const messages = await createKafkaMessages([createLogMessage()], {
                token: team.api_token,
                bytes_uncompressed: '400',
                record_count: '1',
            })
            await waitForBackgroundTasks(consumer.processKafkaBatch(messages))
            expect(mockEmitter.emit).not.toHaveBeenCalled()
        })
    })

    describe('TracesIngestionConsumer billing identity', () => {
        const createTracesIngestionConsumer = (configOverrides: Record<string, any> = {}): TracesIngestionConsumer => {
            const tracesConsumer = new TracesIngestionConsumer(
                // Blank the traces Redis host so it reuses the hub's pool (like the logs-consumer tests),
                // avoiding a second eager pool that would keep handles open after the suite.
                { ...hub, ...getDefaultTracesIngestionConsumerConfig(), TRACES_REDIS_HOST: '', ...configOverrides },
                {
                    ...hub,
                    outputs: new IngestionOutputs<AppMetricsOutput | LogsOutput | LogsDlqOutput>({
                        [APP_METRICS_OUTPUT]: new SingleIngestionOutput(
                            APP_METRICS_OUTPUT,
                            KAFKA_APP_METRICS_2,
                            mockProducer,
                            'test'
                        ),
                        [LOGS_OUTPUT]: new SingleIngestionOutput(
                            LOGS_OUTPUT,
                            KAFKA_LOGS_CLICKHOUSE,
                            mockProducer,
                            'test'
                        ),
                        [LOGS_DLQ_OUTPUT]: new SingleIngestionOutput(
                            LOGS_DLQ_OUTPUT,
                            KAFKA_LOGS_INGESTION_DLQ,
                            mockProducer,
                            'test'
                        ),
                    }),
                    usageBatch: new UsageRecordBatch(null, { unit: 'bytes', isTeamEnabled: () => false }),
                }
            )
            tracesConsumer['kafkaConsumer'] = {
                connect: jest.fn(),
                disconnect: jest.fn(),
                isHealthy: jest.fn().mockReturnValue({ status: 'healthy' }),
            } as any
            return tracesConsumer
        }

        it('bills a signed traces batch, because the internal flag covers logs only', async () => {
            const key = 'internal-logs-test-key'
            await consumer.stop()
            consumer = createTracesIngestionConsumer({
                LOGS_INTERNAL_INGESTION_SIGNING_KEYS: key,
                LOGS_INTERNAL_INGESTION_ENABLED_TEAMS: String(team.id),
            })
            await consumer.start()
            jest.mocked(hub.quotaLimiting.isTeamTokenQuotaLimited).mockResolvedValue(true)
            const message = await createKafkaMessage(createLogMessage(), { token: team.api_token })
            message.headers!.push(internalLogsHeaders(signInternalLogs(key, 'hog_flow', team.id, new Date(Date.now()))))

            await waitForBackgroundTasks(consumer.processKafkaBatch([message]))

            expect(getProducedKafkaMessages().filter((m) => m.topic === KAFKA_LOGS_CLICKHOUSE)).toHaveLength(0)
        })

        it('leaves pattern masking off even on a wildcard allowlist, because a trace record has no body', () => {
            const tracesConsumer = createTracesIngestionConsumer({ LOGS_PATTERN_MASKING_ENABLED_TEAMS: '*' })

            expect(tracesConsumer['isPatternMaskingEnabledForTeam'](team.id)).toEqual(false)
        })

        it('never sniffs or decodes traces for a configured JSON attribute on a wildcard allowlist', async () => {
            await consumer.stop()
            consumer = createTracesIngestionConsumer({
                LOGS_JSON_ATTRIBUTE_PARSING_ENABLED_TEAMS: '*',
                LOGS_JSON_ATTRIBUTE_EXTRACTION_ENABLED_TEAMS: '*',
            })
            await consumer.start()
            jest.spyOn(hub.teamManager, 'getTeam').mockResolvedValue({
                ...team,
                logs_settings: { json_parse_logs_attribute_key: 'payload' },
            })
            logsJsonAttributeSniffCounter.reset()
            const message = await createKafkaMessage(createLogMessage(), { token: team.api_token })
            message.value = Buffer.from('trace payload must not be decoded as logs')

            await waitForBackgroundTasks(consumer.processKafkaBatch([message]))

            const tracesOutput = getProducedKafkaMessages().filter((output) => output.topic === KAFKA_LOGS_CLICKHOUSE)
            expect(tracesOutput).toHaveLength(1)
            expect(tracesOutput[0].value).toEqual(message.value)
            expect((await logsJsonAttributeSniffCounter.get()).values).toEqual([])
        })

        it('meters usage as traces, not logs', async () => {
            const tracesConsumer = createTracesIngestionConsumer()
            tracesConsumer['queueUsageMetric'](team.id, 'bytes_ingested', 500)
            await tracesConsumer['appMetricsAggregator'].flush()

            const messages = getProducedKafkaMessages().filter((m) => m.topic === KAFKA_APP_METRICS_2)
            expect(messages).toHaveLength(1)
            const value = Buffer.isBuffer(messages[0].value)
                ? parseJSON(messages[0].value.toString())
                : parseJSON(messages[0].value as string)
            expect(value?.app_source).toBe('traces')
        })

        it('drops traces when the shared logs and traces bucket is limited', async () => {
            // One product, one free tier, one price: a team over the shared limit on logs volume
            // alone has to stop ingesting traces too.
            jest.mocked(hub.quotaLimiting.isTeamTokenQuotaLimited).mockImplementation((_token, resource) =>
                Promise.resolve(resource === 'logs_mb_ingested')
            )
            const tracesConsumer = createTracesIngestionConsumer()

            const messages = await createKafkaMessages([createLogMessage()], {
                token: team.api_token,
                bytes_uncompressed: '1024',
                record_count: '5',
            })
            const parsed = await tracesConsumer['_parseKafkaBatch'](messages)
            const { quotaAllowedMessages, quotaDroppedMessages } =
                await tracesConsumer['filterQuotaLimitedMessages'](parsed)

            expect(quotaAllowedMessages).toHaveLength(0)
            expect(quotaDroppedMessages).toHaveLength(1)
        })

        it('rate-limits against its own Redis key namespace, not the logs one', () => {
            const tracesConsumer = createTracesIngestionConsumer()
            const prefix = tracesConsumer['rateLimiter']['rateLimiter'].getKeyPrefix()

            expect(prefix).toBe('@posthog-test/traces-rate-limiter/tokens')
            expect(prefix).not.toContain('logs-rate-limiter')
        })

        it('applies TRACES_LIMITER_* tuning rather than the logs limiter config', () => {
            const tracesConsumer = createTracesIngestionConsumer({
                TRACES_LIMITER_BUCKET_SIZE_KB: 42,
                TRACES_LIMITER_REFILL_RATE_KB_PER_SECOND: 7,
            })
            const limiterConfig = tracesConsumer['rateLimiter']['config']

            expect(limiterConfig.LOGS_LIMITER_BUCKET_SIZE_KB).toBe(42)
            expect(limiterConfig.LOGS_LIMITER_REFILL_RATE_KB_PER_SECOND).toBe(7)
        })

        it('evaluates the span retention rules, not the log ones', () => {
            const tracesConsumer = createTracesIngestionConsumer()

            expect(tracesConsumer['retentionRuleSource']).toBe('spans')
        })

        it('gates retention on TRACES_RETENTION_* rather than the logs config', () => {
            const tracesConsumer = createTracesIngestionConsumer({ TRACES_RETENTION_KILLSWITCH: true })

            expect(tracesConsumer['isRetentionEvalEnabledForTeam'](team.id)).toBe(false)
        })

        it('takes its default retention from the tracing config, not logs_settings', async () => {
            const tracingConfigCache = { getRetentionDays: jest.fn().mockResolvedValue(90) }
            const tracesConsumer = createTracesIngestionConsumer()
            tracesConsumer['tracingConfigCache'] = tracingConfigCache as any

            // A different logs period must not reach spans.
            const days = await tracesConsumer['defaultRetentionDays'](team.id, { retention_days: 30 })

            expect(days).toBe(90)
            expect(tracingConfigCache.getRetentionDays).toHaveBeenCalledWith(team.id)
        })

        it('falls back to the built-in default when no tracing config cache is wired', async () => {
            const tracesConsumer = createTracesIngestionConsumer()

            expect(await tracesConsumer['defaultRetentionDays'](team.id, { retention_days: 30 })).toBe(
                DEFAULT_TRACES_RETENTION_DAYS
            )
        })
    })

    describe('hog log transformations', () => {
        let transformerConsumer: LogsIngestionConsumer

        const buildTransformer = async (
            hog: string,
            monitoring = { queueAppMetric: jest.fn(), queueLogs: jest.fn(), flush: jest.fn() }
        ): Promise<LogsTransformerService> => {
            const fn = {
                id: '00000000-0000-0000-0000-000000000001',
                team_id: team.id,
                name: 'Test log transformation',
                type: 'transformation_log',
                enabled: true,
                bytecode: await compileHog(hog),
                inputs: {},
                encrypted_inputs: null,
                execution_order: 1,
                created_at: new Date().toISOString(),
            } as unknown as HogFunctionType
            const manager = {
                getHogFunctionsForTeams: jest.fn().mockResolvedValue({ [team.id]: [fn] }),
                getHogFunctionIdsForTeams: jest.fn().mockResolvedValue({ [team.id]: [fn.id] }),
            }
            return new LogsTransformerService(manager as any, monitoring as any, {
                siteUrl: 'http://localhost:8010',
                hogTimeoutMs: 10,
                messageBudgetMs: 100,
                batchBudgetMs: 2000,
                maxErrorLogsPerFunctionPerMessage: 3,
            })
        }

        afterEach(async () => {
            await transformerConsumer?.stop()
        })

        it('applies an enabled transformation to produced records', async () => {
            const logsTransformer = await buildTransformer(`
                let rec := record
                rec.body := replaceAll(rec.body, 'sensitive', '[REDACTED]')
                return rec
            `)
            transformerConsumer = await createLogsIngestionConsumer(
                hub,
                { LOGS_TRANSFORMATIONS_ENABLED_TEAMS: '*' },
                { logsTransformer }
            )

            const messages = await createKafkaMessages([createLogMessage({ message: 'something sensitive here' })], {
                token: team.api_token,
            })
            await waitForBackgroundTasks(transformerConsumer.processKafkaBatch(messages))

            const logsMessages = getProducedKafkaMessages().filter((m) => m.topic === 'clickhouse_logs_test')
            expect(logsMessages).toHaveLength(1)
            const [, , records] = await decodeLogRecords(logsMessages[0].value as Buffer)
            expect(records).toHaveLength(1)
            expect(records[0].body).toContain('[REDACTED]')
            expect(records[0].body).not.toContain('sensitive')
        })

        it('does not produce a message when transformations drop every record', async () => {
            const logsTransformer = await buildTransformer(`return null`)
            transformerConsumer = await createLogsIngestionConsumer(
                hub,
                { LOGS_TRANSFORMATIONS_ENABLED_TEAMS: '*' },
                { logsTransformer }
            )

            const messages = await createKafkaMessages([createLogMessage()], { token: team.api_token })
            await waitForBackgroundTasks(transformerConsumer.processKafkaBatch(messages))

            const logsMessages = getProducedKafkaMessages().filter((m) => m.topic === 'clickhouse_logs_test')
            expect(logsMessages).toHaveLength(0)
            expect(logMessageDroppedCounterSpy).toHaveBeenCalledWith(
                { reason: 'transformations_all_dropped', team_id: team.id.toString() },
                1
            )
            // The OTel counter has to carry the same reason as the Prometheus one above —
            // they were allowed to disagree, so a transformations drop read as sampling.
            expect(recordLogMessageDroppedSpy).toHaveBeenCalledWith('transformations_all_dropped', team.id.toString())
        })

        it('attributes the full-message drop to transformations when sampling kept survivors', async () => {
            const logsTransformer = await buildTransformer(`return null`)
            // Drop rule that does not match the record, so sampling keeps it and the
            // transformation is what removes the last survivor.
            const mockSamplingCache: Pick<SamplingRulesCache, 'getCompiledRuleSet'> = {
                getCompiledRuleSet: () =>
                    Promise.resolve(
                        compileRuleSet([
                            {
                                id: '00000000-0000-0000-0000-0000000000ab',
                                rule_type: 'severity_sampling',
                                scope_service: 'test-service',
                                scope_path_pattern: null,
                                scope_attribute_filters: [],
                                config: { actions: { INFO: { type: 'drop' } } },
                            },
                        ])
                    ),
            }
            transformerConsumer = await createLogsIngestionConsumer(
                hub,
                { LOGS_TRANSFORMATIONS_ENABLED_TEAMS: '*' },
                { logsTransformer, samplingRulesCache: mockSamplingCache as SamplingRulesCache }
            )

            const messages = await createKafkaMessages([createLogMessage({ level: 'error' })], {
                token: team.api_token,
            })
            await waitForBackgroundTasks(transformerConsumer.processKafkaBatch(messages))

            const logsMessages = getProducedKafkaMessages().filter((m) => m.topic === 'clickhouse_logs_test')
            expect(logsMessages).toHaveLength(0)
            expect(logMessageDroppedCounterSpy).toHaveBeenCalledWith(
                { reason: 'transformations_all_dropped', team_id: team.id.toString() },
                1
            )
        })

        const internalLogsConfig = {
            LOGS_INTERNAL_INGESTION_ENABLED_TEAMS: '*',
            LOGS_INTERNAL_INGESTION_SIGNING_KEYS: 'internal-logs-test-key',
        }

        it('runs transformations on a batch whose internal logs header fails to verify, and keeps its logs out of the producer', async () => {
            const monitoring = { queueAppMetric: jest.fn(), queueLogs: jest.fn(), flush: jest.fn() }
            const logsTransformer = await buildTransformer(`throw Error('boom')`, monitoring)
            transformerConsumer = await createLogsIngestionConsumer(
                hub,
                { LOGS_TRANSFORMATIONS_ENABLED_TEAMS: '*', ...internalLogsConfig },
                { logsTransformer }
            )

            const [message] = await createKafkaMessages([createLogMessage()], { token: team.api_token })
            message.headers!.push(
                internalLogsHeaders(signInternalLogs('rotated-away-key', 'hog_function', team.id, new Date(Date.now())))
            )
            await waitForBackgroundTasks(transformerConsumer.processKafkaBatch([message]))

            expect(monitoring.queueLogs).toHaveBeenCalledWith(expect.any(Array), 'hog_function', {
                aboutInternalLogs: true,
            })
        })

        it.each([
            { name: 'the team is not in the allowlist', config: { LOGS_TRANSFORMATIONS_ENABLED_TEAMS: '' } },
            {
                name: 'the killswitch is on',
                config: { LOGS_TRANSFORMATIONS_ENABLED_TEAMS: '*', LOGS_TRANSFORMATIONS_KILLSWITCH: true },
            },
            {
                name: 'the internal logs header verifies',
                config: { LOGS_TRANSFORMATIONS_ENABLED_TEAMS: '*', ...internalLogsConfig },
                signingKey: 'internal-logs-test-key',
            },
        ])('does not run transformations when $name', async ({ config, signingKey }) => {
            const logsTransformer = await buildTransformer(`return null`)
            transformerConsumer = await createLogsIngestionConsumer(hub, config, { logsTransformer })

            const [message] = await createKafkaMessages([createLogMessage()], { token: team.api_token })
            if (signingKey) {
                message.headers!.push(
                    internalLogsHeaders(signInternalLogs(signingKey, 'hog_function', team.id, new Date(Date.now())))
                )
            }
            await waitForBackgroundTasks(transformerConsumer.processKafkaBatch([message]))

            const logsMessages = getProducedKafkaMessages().filter((m) => m.topic === 'clickhouse_logs_test')
            expect(logsMessages).toHaveLength(1)
        })
    })

    describe('pattern masking (measure-only)', () => {
        let maskingConsumer: LogsIngestionConsumer | undefined

        afterEach(async () => {
            await maskingConsumer?.stop()
            maskingConsumer = undefined
        })

        it.each([
            ['a wildcard allowlist runs the masking stage', () => '*', true],
            ['an empty allowlist does not run the masking stage', () => '', false],
            ["the team's id in the allowlist runs the masking stage", () => String(team.id), true],
            ["only another team's id in the allowlist does not run the masking stage", () => String(team2.id), false],
        ])('%s', async (_name, allowlist, expectMasked) => {
            const bodyKindIncSpy = jest.spyOn(logsPatternBodyKindCounter, 'inc')
            maskingConsumer = await createLogsIngestionConsumer(hub, {
                LOGS_PATTERN_MASKING_ENABLED_TEAMS: allowlist(),
            })

            const messages = await createKafkaMessages([createLogMessage()], { token: team.api_token })
            await waitForBackgroundTasks(maskingConsumer.processKafkaBatch(messages))

            const logsMessages = getProducedKafkaMessages().filter((m) => m.topic === 'clickhouse_logs_test')
            expect(logsMessages).toHaveLength(1)
            if (expectMasked) {
                expect(bodyKindIncSpy).toHaveBeenCalled()
            } else {
                expect(bodyKindIncSpy).not.toHaveBeenCalled()
            }
        })

        it("derives the pattern from the team's message keys, not the defaults", async () => {
            const patternMessageKeysCache = {
                getMessageKeys: jest.fn().mockResolvedValue(['log']),
            } as unknown as PatternMessageKeysCache
            maskingConsumer = await createLogsIngestionConsumer(
                hub,
                { LOGS_PATTERN_MASKING_ENABLED_TEAMS: '*' },
                { patternMessageKeysCache }
            )
            const messages = await createKafkaMessages(
                [createLogMessage({ message: 'default key wins', log: 'team key wins' })],
                {
                    token: team.api_token,
                }
            )
            await waitForBackgroundTasks(maskingConsumer.processKafkaBatch(messages))

            const logsMessages = getProducedKafkaMessages().filter((m) => m.topic === 'clickhouse_logs_test')
            const [, , records] = await decodeLogRecords(logsMessages[0].value as Buffer)
            expect(records.map((r) => r.pattern)).toEqual(['team key wins'])
        })
    })
})