Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

50.8 KB
Raw Blame
import { compress } from '@mongodb-js/zstd'
import avro from 'avsc'

import { parseJSON } from '~/common/utils/json-parse'

import * as logBodyParse from './log-body-parse'
import { PII_REDACTED, encodeAttributeCell } from './log-pii-scrub'
import {
    LogRecord,
    bufferProcessingMode,
    decodeLogRecords,
    encodeLogRecords,
    enrichLogRecordFromJsonAttribute,
    enrichLogRecordWithJsonAttributes,
    extractJsonAttributesFromBody,
    flattenJson,
    logProcessingDurationHistogram,
    logsJsonAttributeSniffCounter,
    logsJsonEnrichmentSkippedCounter,
    processLogMessageBuffer,
    sniffJsonLogAttributes,
    transformDecodedLogRecordsInPlace,
} from './log-record-avro'
import { MAX_LOG_RECORD_BYTES, logRecordSizeBytes } from './log-record-size'

const LOG_RECORD_SCHEMA = avro.parse(`{
"type": "record",
"name": "LogRecord",
"doc": "Schema for a structured log or trace event.",
"fields": [
    {
    "name": "uuid",
    "type": ["null", "string"],
    "doc": "Unique identifier for the log record."
    },
    {
    "name": "trace_id",
    "type": ["null", "bytes"],
    "doc": "Identifier for the trace this log is a part of."
    },
    {
    "name": "span_id",
    "type": ["null", "bytes"],
    "doc": "Identifier for the span within the trace."
    },
    {
    "name": "trace_flags",
    "type": ["null", "int"],
    "doc": "Flags associated with the trace."
    },
    {
    "name": "timestamp",
    "type": ["null", {
        "type": "long",
        "logicalType": "timestamp-micros"
    }],
    "doc": "The primary timestamp of the event, in microseconds since epoch."
    },
    {
    "name": "observed_timestamp",
    "type": ["null", {
        "type": "long",
        "logicalType": "timestamp-micros"
    }],
    "doc": "The timestamp when the event was observed or ingested, in microseconds since epoch."
    },
    {
    "name": "body",
    "type": ["null", "string"],
    "doc": "The main content or message of the log."
    },
    {
    "name": "severity_text",
    "type": ["null", "string"],
    "doc": "Human-readable severity level (e.g., 'INFO', 'ERROR')."
    },
    {
    "name": "severity_number",
    "type": ["null", "int"],
    "doc": "Numeric representation of the severity level."
    },
    {
    "name": "service_name",
    "type": ["null", "string"],
    "doc": "The name of the service that generated the event."
    },
    {
    "name": "resource_attributes",
    "type": ["null", {
        "type": "map",
        "values": "string"
    }],
    "doc": "Attributes describing the resource that produced the log (e.g., host, region)."
    },
    {
    "name": "instrumentation_scope",
    "type": ["null", "string"],
    "doc": "The name of the library or framework that captured the log."
    },
    {
    "name": "event_name",
    "type": ["null", "string"],
    "doc": "The name of a specific event that occurred."
    },
    {
    "name": "attributes",
    "type": ["null", {
        "type": "map",
        "values": "string"
    }],
    "doc": "A map of custom string-valued attributes associated with the log."
    },
    {
    "name": "bytes_uncompressed",
    "type": ["null", "long"],
    "doc": "Logical content size of the row (sum of byte lengths of string/map fields). Used by drop-rule accounting; does not include fixed-width numeric or timestamp fields."
    }
]
}`)

const createRecord = (overrides: Partial<LogRecord> = {}): LogRecord => ({
    uuid: null,
    trace_id: null,
    span_id: null,
    trace_flags: null,
    timestamp: null,
    observed_timestamp: null,
    body: null,
    severity_text: null,
    severity_number: null,
    service_name: null,
    resource_attributes: null,
    instrumentation_scope: null,
    event_name: null,
    attributes: null,
    bytes_uncompressed: null,
    ...overrides,
})

describe('log-record-avro', () => {
    describe('sniffJsonLogAttributes', () => {
        it.each([
            [null, 'missing_key'],
            [{}, 'missing_key'],
            [{ payload: '{}' }, 'missing_key'],
            [{ 'payload.json': '{}' }, 'looks_like_json'],
            [{ 'payload.json': ' \n\t[1]' }, 'looks_like_json'],
            [{ 'payload.json': JSON.stringify('{"nested":true}') }, 'looks_like_json'],
            [{ 'payload.json': JSON.stringify('\n\t [1]') }, 'looks_like_json'],
            [{ 'payload.json': '{not valid JSON' }, 'looks_like_json'],
            [{ 'payload.json': '"ordinary text"' }, 'other'],
            [{ 'payload.json': 'null' }, 'other'],
            [{ 'payload.json': '' }, 'other'],
            [{ 'payload.json': ' '.repeat(64) + '{}' }, 'other'],
        ] as const)('classifies %j as %s without mutating the record', async (attributes, outcome) => {
            logsJsonAttributeSniffCounter.reset()
            const record = Object.freeze({ attributes: attributes && Object.freeze(attributes) })

            sniffJsonLogAttributes([record], 'payload.json', 123)
            sniffJsonLogAttributes([record, record], 'payload.json', 456)

            expect((await logsJsonAttributeSniffCounter.get()).values).toEqual([
                expect.objectContaining({ labels: { team_id: '123', outcome }, value: 1 }),
                expect.objectContaining({ labels: { team_id: '456', outcome }, value: 2 }),
            ])
        })
    })

    describe('flattenJson', () => {
        it.each([
            ['flattens simple object', { a: 'b', c: 'd' }, { a: 'b', c: 'd' }],
            ['flattens nested object', { a: { b: 'c' } }, { 'a.b': 'c' }],
            ['flattens deeply nested object', { a: { b: { c: 'd' } } }, { 'a.b.c': 'd' }],
            ['flattens array', { items: ['a', 'b'] }, { 'items.0': 'a', 'items.1': 'b' }],
            [
                'flattens nested array of objects',
                { items: [{ name: 'a' }, { name: 'b' }] },
                { 'items.0.name': 'a', 'items.1.name': 'b' },
            ],
            ['handles null values', { a: null }, { a: 'null' }],
            ['handles undefined values', { a: undefined }, { a: 'undefined' }],
            ['handles number values', { count: 42 }, { count: 42 }],
            ['handles boolean values', { active: true }, { active: true }],
            [
                'handles mixed types',
                { str: 'hello', num: 123, bool: false, nil: null },
                { str: 'hello', num: 123, bool: false, nil: 'null' },
            ],
            ['handles empty object', {}, {}],
            ['handles empty array', { items: [] }, {}],
            [
                'ignores inherited and non-enumerable properties',
                Object.create({ inherited: true }, { kept: { value: 1, enumerable: true }, hidden: { value: 2 } }),
                { kept: 1 },
            ],
            ['preserves later dotted-key overwrites', { path: { value: 1 }, 'path.value': 2 }, { 'path.value': 2 }],
            ['preserves later nested overwrites', { 'path.value': 1, path: { value: 2 } }, { 'path.value': 2 }],
            ['preserves top-level array indexing', ['a', null, { b: true }], { '0': 'a', '1': 'null', '2.b': true }],
        ])('%s', (_, input, expected) => {
            expect(flattenJson(input)).toEqual(expected)
        })

        it.each([
            [{ 'path.value': 1, filler: 2, path: { value: 3 } }, { 'path.value': 3 }],
            [{ text: 1, '': { '2': 2 } }, { '2': 2 }],
            [{ '-1': 1, '': { '0': 2 } }, { '0': 2 }],
            [{ '2': 1, '': { '1': 2 } }, { '1': 2 }],
            [{ '1': 1, '': { '2': 2 } }, { '1': 1 }],
        ])('preserves ordering and collisions beyond the retained field limit for %j', (input, expected) => {
            expect(flattenJson(input, '', 1)).toEqual(expected)
        })

        it.each([
            ['unicode value at budget', { a: '😀' }, 7, { a: '😀' }],
            ['unicode value over budget', { a: '😀' }, 6, null],
            ['escaped value over budget', { a: '\n' }, 4, null],
            ['cumulative paths over budget', { abc: { def: 1 } }, 9, null],
            ['overwritten value releases budget', { 'a.b': 'long', a: { b: 1 } }, 20, { 'a.b': 1 }],
        ] as const)('%s', (_name, input, maxBytes, expected) => {
            expect(flattenJson(input, '', 50, maxBytes)).toEqual(expected)
        })

        it.each([128, 129, 10_000])('bounds deeply nested input at depth %i', (depth) => {
            const input = parseJSON('{"nested":'.repeat(depth) + '1' + '}'.repeat(depth))
            expect(flattenJson(input)).toEqual(depth <= 128 ? { [Array(depth).fill('nested').join('.')]: 1 } : null)
        })

        it.each([
            Array.from({ length: 10_000 }, () => ({})),
            Object.fromEntries(Array.from({ length: 10_000 }, (_, index) => [`key${index}`, {}])),
        ])('bounds wide containers even when they contain no leaves', (input) => {
            expect(flattenJson(input)).toBeNull()
        })

        it.each([
            ['root', 9_999, false, false],
            ['nested', 9_997, true, false],
            ['nested', 9_998, true, true],
        ] as const)(
            'honors the remaining node budget for %s objects with %i children',
            (_name, count, nested, skipped) => {
                const children = Object.fromEntries(Array.from({ length: count }, (_, index) => [`key${index}`, {}]))
                const input = nested ? { first: true, children } : children

                expect(flattenJson(input)).toEqual(skipped ? null : nested ? { first: true } : {})
            }
        )

        it('stops key inspection when a sub-1 MiB object exceeds the node budget', () => {
            const input = Object.fromEntries(Array.from({ length: 100_000 }, (_, index) => [String(index), 0]))
            expect(Buffer.byteLength(JSON.stringify(input))).toBeLessThan(MAX_LOG_RECORD_BYTES)
            const guardedInput = new Proxy(input, {
                getOwnPropertyDescriptor(target, key) {
                    if (key === '10000') {
                        throw new Error('Inspected a key beyond the node budget')
                    }
                    return Reflect.getOwnPropertyDescriptor(target, key)
                },
            })

            expect(flattenJson(guardedInput)).toBeNull()
        })
    })

    describe('extractJsonAttributesFromBody', () => {
        it('extracts attributes from valid JSON body', () => {
            const body = JSON.stringify({ level: 'info', message: 'test' })
            const result = extractJsonAttributesFromBody(body)

            expect(result).toEqual({ level: '\"info\"', message: '\"test\"' })
        })

        it('correctly types number strings', () => {
            const body = JSON.stringify({ numberString: '2', number: 2 })
            const result = extractJsonAttributesFromBody(body)

            expect(result).toEqual({ numberString: '\"2\"', number: '2' })
        })

        it('returns empty object for invalid JSON', () => {
            const result = extractJsonAttributesFromBody('not json')

            expect(result).toEqual({})
        })

        it('returns empty object for null body', () => {
            const result = extractJsonAttributesFromBody(null)

            expect(result).toEqual({})
        })

        it('limits to 50 attributes', () => {
            const largeObject: Record<string, string> = {}
            for (let i = 0; i < 100; i++) {
                largeObject[`key${i}`] = `value${i}`
            }
            const body = JSON.stringify(largeObject)
            const result = extractJsonAttributesFromBody(body)

            expect(Object.keys(result)).toEqual(Array.from({ length: 50 }, (_, index) => `key${index}`))
        })

        it('flattens nested JSON', () => {
            const body = JSON.stringify({
                user: { id: 123, name: 'test' },
                request: { path: '/api' },
            })
            const result = extractJsonAttributesFromBody(body)

            expect(result).toEqual({
                'user.id': '123',
                'user.name': '\"test\"',
                'request.path': '\"/api\"',
            })
        })

        it('returns empty object for primitive JSON values', () => {
            expect(extractJsonAttributesFromBody('"string"')).toEqual({})
            expect(extractJsonAttributesFromBody('123')).toEqual({})
            expect(extractJsonAttributesFromBody('true')).toEqual({})
            expect(extractJsonAttributesFromBody('null')).toEqual({})
        })
    })

    describe('encodeLogRecords and decodeLogRecords', () => {
        it('round-trips multiple LogRecords', async () => {
            const records: LogRecord[] = [
                {
                    uuid: 'test-uuid-1',
                    trace_id: null,
                    span_id: null,
                    trace_flags: null,
                    timestamp: 1704067200000000,
                    observed_timestamp: 1704067200000000,
                    body: 'Test log message 1',
                    severity_text: 'info',
                    severity_number: 9,
                    service_name: 'test-service',
                    resource_attributes: { 'host.name': 'localhost' },
                    instrumentation_scope: '[email protected]',
                    event_name: null,
                    attributes: { key: 'value1' },
                    bytes_uncompressed: 123,
                },
                {
                    uuid: 'test-uuid-2',
                    trace_id: null,
                    span_id: null,
                    trace_flags: null,
                    timestamp: 1704067200000000,
                    observed_timestamp: 1704067200000000,
                    body: 'Test log message 2',
                    severity_text: 'error',
                    severity_number: 17,
                    service_name: 'test-service',
                    resource_attributes: { 'host.name': 'localhost' },
                    instrumentation_scope: '[email protected]',
                    event_name: null,
                    attributes: { key: 'value2' },
                    bytes_uncompressed: 456,
                },
            ]

            const encoded = await encodeLogRecords(LOG_RECORD_SCHEMA, 'zstandard', records)
            const [_, __, decoded] = await decodeLogRecords(encoded)

            expect(decoded).toEqual(records)
        })

        it('handles single record', async () => {
            const records: LogRecord[] = [
                {
                    uuid: 'test-uuid',
                    trace_id: null,
                    span_id: null,
                    trace_flags: null,
                    timestamp: null,
                    observed_timestamp: null,
                    body: null,
                    severity_text: null,
                    severity_number: null,
                    service_name: null,
                    resource_attributes: null,
                    instrumentation_scope: null,
                    event_name: null,
                    attributes: null,
                    bytes_uncompressed: null,
                },
            ]

            const encoded = await encodeLogRecords(LOG_RECORD_SCHEMA, 'zstandard', records)
            const [_, __, decoded] = await decodeLogRecords(encoded)

            expect(decoded).toEqual(records)
        })

        it('preserves bytes_uncompressed across encode/decode', async () => {
            const records: LogRecord[] = [
                {
                    uuid: 'with-bytes',
                    trace_id: null,
                    span_id: null,
                    trace_flags: null,
                    timestamp: null,
                    observed_timestamp: null,
                    body: 'hello world',
                    severity_text: null,
                    severity_number: null,
                    service_name: null,
                    resource_attributes: null,
                    instrumentation_scope: null,
                    event_name: null,
                    attributes: null,
                    bytes_uncompressed: 789,
                },
            ]

            const encoded = await encodeLogRecords(LOG_RECORD_SCHEMA, 'zstandard', records)
            const [_, __, decoded] = await decodeLogRecords(encoded)

            expect(decoded[0].bytes_uncompressed).toBe(789)
        })

        it.each([
            [
                'bytes that are not a container',
                () => Promise.resolve(Buffer.from('not avro data')),
                'invalid magic bytes',
            ],
            [
                'a trailing block count past 2^53',
                async () =>
                    Buffer.concat([
                        await encodeLogRecords(LOG_RECORD_SCHEMA, 'zstandard', [createRecord()]),
                        Buffer.from([0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x20]),
                    ]),
                'potential precision loss',
            ],
        ])('rejects promise for %s', async (_name, build, message) => {
            await expect(decodeLogRecords(await build())).rejects.toThrow(message)
        })

        it('keeps decoding bytes fields after a container whose bytes length is NaN', async () => {
            // uuid null, then trace_id as bytes with a length varint whose float read gives NaN.
            const craftedBlock = Buffer.concat([
                Buffer.from([0x00, 0x02]),
                Buffer.alloc(200, 0x80),
                Buffer.from([0x00]),
            ])
            const crafted = await new Promise<Buffer>((resolve, reject) => {
                const buffers: Buffer[] = []
                const encoder = new avro.streams.BlockEncoder(LOG_RECORD_SCHEMA, {
                    codec: 'zstandard',
                    codecs: {
                        zstandard: (_buf: Buffer, cb: (err: Error | null, compressed?: Buffer) => void) => {
                            compress(craftedBlock, 1).then((compressed) => cb(null, compressed), cb)
                        },
                    },
                })
                encoder.on('data', (buf: Buffer) => buffers.push(buf))
                encoder.on('end', () => resolve(Buffer.concat(buffers)))
                encoder.on('error', reject)
                encoder.end(createRecord())
            })
            const traceId = Buffer.from('0102030405060708090a0b0c0d0e0f10', 'hex')
            const valid = await encodeLogRecords(LOG_RECORD_SCHEMA, 'zstandard', [createRecord({ trace_id: traceId })])

            await expect(decodeLogRecords(crafted)).rejects.toThrow()
            const [, , decoded] = await decodeLogRecords(valid)

            expect(decoded[0].trace_id).toEqual(traceId)
        })
    })

    describe('enrichLogRecordWithJsonAttributes', () => {
        beforeEach(() => {
            logsJsonEnrichmentSkippedCounter.reset()
        })

        it.each([-1, 0, 1])('checks merged UTF-8 content at the size boundary (%i bytes)', async (offset) => {
            const record = createRecord({
                body: '{"a":"é"}',
                severity_text: 'info',
                attributes: { existing: 'true' },
                resource_attributes: { padding: '' },
            })
            const extractedBytes = 5
            record.resource_attributes!.padding = 'x'.repeat(
                MAX_LOG_RECORD_BYTES - logRecordSizeBytes(record) - extractedBytes + offset
            )
            const original = structuredClone(record)
            const attributes = record.attributes

            expect(enrichLogRecordWithJsonAttributes(record)).toBe(record)

            if (offset > 0) {
                expect(record).toEqual(original)
                expect(record.attributes).toBe(attributes)
                expect((await logsJsonEnrichmentSkippedCounter.get()).values).toEqual([
                    expect.objectContaining({ labels: { reason: 'output_size', source: 'body' }, value: 1 }),
                ])
            } else {
                expect(record).toEqual({ ...original, attributes: { existing: 'true', a: '"é"' } })
                expect(logRecordSizeBytes(record)).toBe(MAX_LOG_RECORD_BYTES + offset)
                expect((await logsJsonEnrichmentSkippedCounter.get()).values).toEqual([])
            }
        })

        it('counts only the winning sender value toward the merged budget', () => {
            const record = createRecord({
                body: JSON.stringify({ payload: 'x'.repeat(600 * 1024), added: true }),
                attributes: { payload: '"sender"' },
            })
            enrichLogRecordWithJsonAttributes(record)
            expect(record.attributes).toEqual({ payload: '"sender"', added: 'true' })
        })

        it.each([
            ['input_size', JSON.stringify({ value: '😀'.repeat(MAX_LOG_RECORD_BYTES / 4) })],
            ['output_size', JSON.stringify({ value: 'x'.repeat(600 * 1024), other: true })],
            ['flatten_budget', '{"nested":'.repeat(129) + '1' + '}'.repeat(129)],
        ])('preserves records and counts %s skips through the buffer processor', async (reason, body) => {
            const record = createRecord({ body, attributes: { original: 'true' } })
            const encoded = await encodeLogRecords(LOG_RECORD_SCHEMA, 'zstandard', [record])
            const result = await processLogMessageBuffer(encoded, { json_parse_logs: true })
            const [, , decoded] = await decodeLogRecords(result.value!)

            expect(decoded).toEqual([{ ...record, bytes_uncompressed: null }])
            expect((await logsJsonEnrichmentSkippedCounter.get()).values).toEqual([
                expect.objectContaining({ labels: { reason, source: 'body' }, value: 1 }),
            ])
        })

        it('does not parse oversized bodies on direct or batch enrichment paths', async () => {
            const body = JSON.stringify({ value: '😀'.repeat(MAX_LOG_RECORD_BYTES / 4) })
            const parse = jest.spyOn(logBodyParse, 'parseLogBodyForIngestion')
            try {
                expect(extractJsonAttributesFromBody(body)).toEqual({})
                enrichLogRecordWithJsonAttributes(createRecord({ body }))
                await transformDecodedLogRecordsInPlace([createRecord({ body })], { json_parse_logs: true })
                expect(parse.mock.calls.every(([input]) => input === null)).toBe(true)
            } finally {
                parse.mockRestore()
            }
        })

        it('adds JSON attributes from body', () => {
            const record: LogRecord = {
                uuid: 'test-uuid',
                trace_id: null,
                span_id: null,
                trace_flags: null,
                timestamp: null,
                observed_timestamp: null,
                body: JSON.stringify({ level: 'info', context: { user_id: 123 } }),
                severity_text: null,
                severity_number: null,
                service_name: null,
                resource_attributes: null,
                instrumentation_scope: null,
                event_name: null,
                attributes: null,
            }

            enrichLogRecordWithJsonAttributes(record)

            expect(record.attributes).toEqual({
                level: '\"info\"',
                'context.user_id': '123',
            })
        })

        it('preserves existing attributes', () => {
            const record: LogRecord = {
                uuid: 'test-uuid',
                trace_id: null,
                span_id: null,
                trace_flags: null,
                timestamp: null,
                observed_timestamp: null,
                body: JSON.stringify({ level: 'info', message: 'test' }),
                severity_text: null,
                severity_number: null,
                service_name: null,
                resource_attributes: null,
                instrumentation_scope: null,
                event_name: null,
                attributes: { level: '\"error\"', existing: '\"attribute\"' },
            }

            enrichLogRecordWithJsonAttributes(record)

            expect(record.attributes).toEqual({
                level: '\"error\"',
                existing: '\"attribute\"',
                message: '\"test\"',
            })
        })

        it('does nothing for null body', () => {
            const record: LogRecord = {
                uuid: 'test-uuid',
                trace_id: null,
                span_id: null,
                trace_flags: null,
                timestamp: null,
                observed_timestamp: null,
                body: null,
                severity_text: null,
                severity_number: null,
                service_name: null,
                resource_attributes: null,
                instrumentation_scope: null,
                event_name: null,
                attributes: { existing: '\"attribute\"' },
            }

            enrichLogRecordWithJsonAttributes(record)

            expect(record.attributes).toEqual({ existing: '\"attribute\"' })
        })

        it('does nothing for non-JSON body', () => {
            const record: LogRecord = {
                uuid: 'test-uuid',
                trace_id: null,
                span_id: null,
                trace_flags: null,
                timestamp: null,
                observed_timestamp: null,
                body: 'plain text log message',
                severity_text: null,
                severity_number: null,
                service_name: null,
                resource_attributes: null,
                instrumentation_scope: null,
                event_name: null,
                attributes: null,
            }

            enrichLogRecordWithJsonAttributes(record)

            expect(record.attributes).toBeNull()
        })
    })

    describe('selected JSON attribute extraction', () => {
        it.each([false, true])('extracts objects with string wrapping=%s and a literal dotted key', (wrapped) => {
            const json = JSON.stringify({ user: { id: 'synthetic-user' }, count: 3, enabled: true })
            const source = wrapped ? JSON.stringify(json) : json
            const record = createRecord({ attributes: { 'payload.json': source, 'payload.json.count': '9' } })

            enrichLogRecordFromJsonAttribute(record, 'payload.json')

            expect(record.attributes).toEqual({
                'payload.json': source,
                'payload.json.user.id': JSON.stringify('synthetic-user'),
                'payload.json.count': '9',
                'payload.json.enabled': 'true',
            })
        })

        it.each(
            [[], ['alpha', 'beta'], ['beta', 'alpha'], [{ id: 1 }], [null, 3, true, ['nested'], { id: 2 }]].map(
                (items) => [items]
            )
        )('retains nested array %j as a single JSON-string attribute', (items) => {
            const source = JSON.stringify({ values: items })
            const record = createRecord({ attributes: { payload: source } })

            enrichLogRecordFromJsonAttribute(record, 'payload')

            expect(record.attributes).toEqual({
                payload: source,
                'payload.values': JSON.stringify(JSON.stringify(items)),
            })
        })

        it.each(['[]', '[1,2]', JSON.stringify('[1,2]'), 'null', 'true', '4', '"text"', '{broken', ''])(
            'preserves non-object input %s without extracting fields',
            (source) => {
                const record = createRecord({ attributes: { payload: source } })
                const original = structuredClone(record)
                enrichLogRecordFromJsonAttribute(record, 'payload')
                expect(record).toEqual(original)
            }
        )

        it('counts each array as one retained field', () => {
            const source = JSON.stringify(
                Object.fromEntries(Array.from({ length: 60 }, (_, index) => [`key${index}`, []]))
            )
            const record = createRecord({ attributes: { payload: source } })
            enrichLogRecordFromJsonAttribute(record, 'payload')
            expect(Object.keys(record.attributes!)).toHaveLength(51)
            expect(record.attributes!['payload.key49']).toBe(JSON.stringify('[]'))
            expect(record.attributes!['payload.key50']).toBeUndefined()
        })

        it.each([
            ['input_size', JSON.stringify({ value: 'x'.repeat(MAX_LOG_RECORD_BYTES) })],
            ['output_size', JSON.stringify({ value: '😀'.repeat(150_000) })],
            ['flatten_budget', '{"child":'.repeat(130) + '1' + '}'.repeat(130)],
            ['flatten_budget', '{"values":' + '['.repeat(130) + '1' + ']'.repeat(130) + '}'],
            ['flatten_budget', JSON.stringify({ first: Array(6000).fill(0), second: Array(6000).fill(0) })],
            [
                'flatten_budget',
                JSON.stringify(Object.fromEntries(Array.from({ length: 10_000 }, (_, index) => [index, 0]))),
            ],
        ])('skips %s atomically for oversized or complex input', async (reason, source) => {
            const record = createRecord({ attributes: { payload: source, original: 'true' } })
            const original = structuredClone(record)
            logsJsonEnrichmentSkippedCounter.reset()
            enrichLogRecordFromJsonAttribute(record, 'payload')
            expect(record).toEqual(original)
            expect((await logsJsonEnrichmentSkippedCounter.get()).values).toEqual([
                expect.objectContaining({ labels: { reason, source: 'selected_attribute' }, value: 1 }),
            ])
        })

        it('re-encodes attribute-only enrichment before decoded-record visitors', async () => {
            const record = createRecord({ attributes: { payload: JSON.stringify({ count: 7 }) } })
            const buffer = await encodeLogRecords(LOG_RECORD_SCHEMA, 'zstandard', [record])
            const visitor = jest.fn()
            logProcessingDurationHistogram.reset()
            const result = await processLogMessageBuffer(
                buffer,
                { json_parse_logs_attribute_key: 'payload' },
                { onRecordsDecoded: visitor }
            )
            const [, , records] = await decodeLogRecords(result.value!)
            expect(records[0].attributes).toEqual({ ...record.attributes, 'payload.count': '7' })
            expect(visitor.mock.calls[0][0][0].attributes).toEqual(records[0].attributes)
            expect(result.pii).toEqual({ piiReplacements: 0 })
            // Extraction-only traffic needs its own duration series. Without the label, its cost
            // merges into the bucket that body parsing, scrubbing and untransformed traffic share.
            expect((await logProcessingDurationHistogram.get()).values).toContainEqual(
                expect.objectContaining({
                    metricName: 'logs_ingestion_processing_duration_seconds_count',
                    labels: expect.objectContaining({
                        json_parse_enabled: 'false',
                        pii_scrub_enabled: 'false',
                        attribute_extraction_enabled: 'true',
                    }),
                    value: 1,
                })
            )
        })

        it('gives sender attributes priority over selected JSON, then body JSON', async () => {
            const source = JSON.stringify({ sender: 'selected', selected: 'selected' })
            const record = createRecord({
                attributes: { payload: source, 'payload.sender': JSON.stringify('sender') },
                body: JSON.stringify({ payload: { sender: 'body', selected: 'body', body: 'body', items: [1] } }),
            })
            await transformDecodedLogRecordsInPlace([record], {
                json_parse_logs: true,
                json_parse_logs_attribute_key: 'payload',
            })
            expect(record.attributes).toEqual({
                payload: source,
                'payload.sender': JSON.stringify('sender'),
                'payload.selected': JSON.stringify('selected'),
                'payload.body': JSON.stringify('body'),
                'payload.items.0': '1',
            })
        })

        it('does not extract from a selected key introduced by body parsing', async () => {
            const record = createRecord({ attributes: null, body: JSON.stringify({ payload: '{"nested":true}' }) })
            await transformDecodedLogRecordsInPlace([record], {
                json_parse_logs: true,
                json_parse_logs_attribute_key: 'payload',
            })
            expect(record.attributes).toEqual({ payload: JSON.stringify('{"nested":true}') })
        })

        it('extracts from the scrubbed source rather than copying sensitive values', async () => {
            const record = createRecord({
                attributes: { payload: JSON.stringify({ authorization: 'Bearer synthetic_test_credential' }) },
            })
            const pii = await transformDecodedLogRecordsInPlace([record], {
                pii_scrub_logs: true,
                json_parse_logs_attribute_key: 'payload',
            })
            expect(pii.piiReplacements).toBeGreaterThan(0)
            expect(record.attributes).toEqual({
                payload: JSON.stringify({ authorization: `Bearer ${PII_REDACTED}` }),
                'payload.authorization': JSON.stringify(`Bearer ${PII_REDACTED}`),
            })
        })
    })

    describe('processLogMessageBuffer', () => {
        it('processes buffer with JSON parsing enabled', async () => {
            const records: LogRecord[] = [
                {
                    uuid: 'test-uuid',
                    trace_id: null,
                    span_id: null,
                    trace_flags: null,
                    timestamp: null,
                    observed_timestamp: null,
                    body: JSON.stringify({ level: 'info', message: 'test' }),
                    severity_text: null,
                    severity_number: null,
                    service_name: null,
                    resource_attributes: null,
                    instrumentation_scope: null,
                    event_name: null,
                    attributes: null,
                    bytes_uncompressed: null,
                },
            ]

            const inputBuffer = await encodeLogRecords(LOG_RECORD_SCHEMA, 'zstandard', records)
            const { value: outputBuffer, pii } = await processLogMessageBuffer(inputBuffer, { json_parse_logs: true })
            expect(pii).toEqual({ piiReplacements: 0 })
            const [_, __, decoded] = await decodeLogRecords(outputBuffer!)

            expect(decoded[0]?.attributes).toEqual({
                level: '\"info\"',
                message: '\"test\"',
            })
        })

        it('processes multiple records in buffer', async () => {
            const records: LogRecord[] = [
                {
                    uuid: 'test-uuid-1',
                    trace_id: null,
                    span_id: null,
                    trace_flags: null,
                    timestamp: null,
                    observed_timestamp: null,
                    body: JSON.stringify({ level: 'info', message: 'test1' }),
                    severity_text: null,
                    severity_number: null,
                    service_name: null,
                    resource_attributes: null,
                    instrumentation_scope: null,
                    event_name: null,
                    attributes: null,
                    bytes_uncompressed: null,
                },
                {
                    uuid: 'test-uuid-2',
                    trace_id: null,
                    span_id: null,
                    trace_flags: null,
                    timestamp: null,
                    observed_timestamp: null,
                    body: JSON.stringify({ level: 'error', message: 'test2' }),
                    severity_text: null,
                    severity_number: null,
                    service_name: null,
                    resource_attributes: null,
                    instrumentation_scope: null,
                    event_name: null,
                    attributes: null,
                    bytes_uncompressed: null,
                },
            ]

            const inputBuffer = await encodeLogRecords(LOG_RECORD_SCHEMA, 'zstandard', records)
            const { value: outputBuffer, pii } = await processLogMessageBuffer(inputBuffer, { json_parse_logs: true })
            expect(pii).toEqual({ piiReplacements: 0 })
            const [_, __, decoded] = await decodeLogRecords(outputBuffer!)

            expect(decoded).toHaveLength(2)
            expect(decoded[0]?.attributes).toEqual({
                level: '\"info\"',
                message: '\"test1\"',
            })
            expect(decoded[1]?.attributes).toEqual({
                level: '\"error\"',
                message: '\"test2\"',
            })
        })

        it('returns original buffer when JSON parse and PII scrub are disabled', async () => {
            const records: LogRecord[] = [
                {
                    uuid: 'test-uuid',
                    trace_id: null,
                    span_id: null,
                    trace_flags: null,
                    timestamp: null,
                    observed_timestamp: null,
                    body: JSON.stringify({ level: 'info', message: 'test' }),
                    severity_text: null,
                    severity_number: null,
                    service_name: null,
                    resource_attributes: null,
                    instrumentation_scope: null,
                    event_name: null,
                    attributes: null,
                    bytes_uncompressed: null,
                },
            ]

            const inputBuffer = await encodeLogRecords(LOG_RECORD_SCHEMA, 'zstandard', records)
            const { value: out, pii } = await processLogMessageBuffer(inputBuffer, {
                json_parse_logs: false,
                pii_scrub_logs: false,
            })

            expect(out).toBe(inputBuffer)
            expect(pii).toEqual({ piiReplacements: 0 })
        })

        it('forwards the original buffer when only a visitor ran', async () => {
            const records: LogRecord[] = [
                {
                    uuid: 'test-uuid',
                    trace_id: null,
                    span_id: null,
                    trace_flags: null,
                    timestamp: null,
                    observed_timestamp: null,
                    body: JSON.stringify({ level: 'info', message: 'test' }),
                    severity_text: null,
                    severity_number: null,
                    service_name: null,
                    resource_attributes: null,
                    instrumentation_scope: null,
                    event_name: null,
                    attributes: null,
                    bytes_uncompressed: null,
                },
            ]

            const inputBuffer = await encodeLogRecords(LOG_RECORD_SCHEMA, 'zstandard', records)
            const onRecordsDecoded = jest.fn()
            const { value: out } = await processLogMessageBuffer(
                inputBuffer,
                { json_parse_logs: false, pii_scrub_logs: false },
                { onRecordsDecoded }
            )

            // The visitor forces a decode, but nothing mutated the records, so the buffer is returned by
            // identity rather than re-encoded.
            expect(onRecordsDecoded).toHaveBeenCalledTimes(1)
            expect(onRecordsDecoded.mock.calls[0][0]).toHaveLength(1)
            expect(out).toBe(inputBuffer)
        })

        it.each([
            ['everything off', {}, 0, false, 'passthrough'],
            ['json parse on', { json_parse_logs: true }, 0, false, 'decode_and_reencode'],
            ['pii scrub on', { pii_scrub_logs: true }, 0, false, 'decode_and_reencode'],
            ['attribute extraction on', { json_parse_logs_attribute_key: 'payload' }, 0, false, 'decode_and_reencode'],
            ['empty attribute key', { json_parse_logs_attribute_key: '' }, 0, false, 'passthrough'],
            ['a stage present', {}, 1, false, 'decode_and_reencode'],
            ['a decoded-records visitor present', {}, 0, true, 'decode_only'],
            ['a visitor and a stage', {}, 1, true, 'decode_and_reencode'],
        ])('bufferProcessingMode: %s', (_name, settings, stageCount, hasVisitor, expected) => {
            expect(bufferProcessingMode(settings, stageCount, hasVisitor)).toEqual(expected)
        })

        it('decodes and scrubs only when PII scrub is on without JSON parse', async () => {
            const records: LogRecord[] = [
                {
                    uuid: 'test-uuid',
                    trace_id: null,
                    span_id: null,
                    trace_flags: null,
                    timestamp: null,
                    observed_timestamp: null,
                    body: JSON.stringify({ level: 'info', message: '[email protected]' }),
                    severity_text: null,
                    severity_number: null,
                    service_name: null,
                    resource_attributes: null,
                    instrumentation_scope: null,
                    event_name: null,
                    attributes: null,
                    bytes_uncompressed: null,
                },
            ]

            const inputBuffer = await encodeLogRecords(LOG_RECORD_SCHEMA, 'zstandard', records)
            const { value: outputBuffer, pii } = await processLogMessageBuffer(inputBuffer, {
                json_parse_logs: false,
                pii_scrub_logs: true,
            })
            expect(outputBuffer).not.toBe(inputBuffer)
            expect(pii.piiReplacements).toBeGreaterThanOrEqual(1)

            const [_, __, decoded] = await decodeLogRecords(outputBuffer!)
            expect(decoded[0]?.attributes).toBeNull()
            const body = parseJSON(decoded[0]?.body || '{}') as { message?: string }
            expect(body.message).not.toContain('example.com')
            expect(body.message).toContain('{{REDACTED}}')
        })

        it('scrubs log attributes when json parse is off and pii scrub is on', async () => {
            const records: LogRecord[] = [
                {
                    uuid: 'test-uuid',
                    trace_id: null,
                    span_id: null,
                    trace_flags: null,
                    timestamp: null,
                    observed_timestamp: null,
                    body: 'plain',
                    severity_text: null,
                    severity_number: null,
                    service_name: null,
                    resource_attributes: null,
                    instrumentation_scope: null,
                    event_name: null,
                    attributes: { note: '[email protected]' },
                    bytes_uncompressed: null,
                },
            ]

            const inputBuffer = await encodeLogRecords(LOG_RECORD_SCHEMA, 'zstandard', records)
            const { value: outputBuffer, pii } = await processLogMessageBuffer(inputBuffer, {
                json_parse_logs: false,
                pii_scrub_logs: true,
            })
            expect(pii.piiReplacements).toBe(1)
            const [_, __, decoded] = await decodeLogRecords(outputBuffer!)
            expect(decoded[0]?.body).toBe('plain')
            expect(decoded[0]?.attributes).toEqual({ note: PII_REDACTED })
        })

        it('scrubs body then enriches when both JSON parse and PII scrub are on', async () => {
            const records: LogRecord[] = [
                {
                    uuid: 'test-uuid',
                    trace_id: null,
                    span_id: null,
                    trace_flags: null,
                    timestamp: null,
                    observed_timestamp: null,
                    body: JSON.stringify({ level: 'info', message: '[email protected]' }),
                    severity_text: null,
                    severity_number: null,
                    service_name: null,
                    resource_attributes: null,
                    instrumentation_scope: null,
                    event_name: null,
                    attributes: { note: '[email protected]' },
                    bytes_uncompressed: null,
                },
            ]

            const inputBuffer = await encodeLogRecords(LOG_RECORD_SCHEMA, 'zstandard', records)
            const { value: outputBuffer, pii } = await processLogMessageBuffer(inputBuffer, {
                json_parse_logs: true,
                pii_scrub_logs: true,
            })
            expect(pii.piiReplacements).toBe(2)
            const [_, __, decoded] = await decodeLogRecords(outputBuffer!)
            expect(decoded[0]?.attributes).toEqual({
                level: encodeAttributeCell('info'),
                message: encodeAttributeCell(PII_REDACTED),
                note: PII_REDACTED,
            })
        })

        it('does not call parseLogBodyForIngestion when only PII scrub is on', async () => {
            const spy = jest.spyOn(logBodyParse, 'parseLogBodyForIngestion')
            const records: LogRecord[] = [
                {
                    uuid: 'test-uuid',
                    trace_id: null,
                    span_id: null,
                    trace_flags: null,
                    timestamp: null,
                    observed_timestamp: null,
                    body: JSON.stringify({ level: 'info', message: '[email protected]' }),
                    severity_text: null,
                    severity_number: null,
                    service_name: null,
                    resource_attributes: null,
                    instrumentation_scope: null,
                    event_name: null,
                    attributes: null,
                    bytes_uncompressed: null,
                },
            ]

            const inputBuffer = await encodeLogRecords(LOG_RECORD_SCHEMA, 'zstandard', records)
            await processLogMessageBuffer(inputBuffer, {
                json_parse_logs: false,
                pii_scrub_logs: true,
            })

            expect(spy).not.toHaveBeenCalled()
            spy.mockRestore()
        })

        it('calls parseLogBodyForIngestion once per record when both JSON parse and PII scrub are on', async () => {
            const spy = jest.spyOn(logBodyParse, 'parseLogBodyForIngestion')
            const records: LogRecord[] = [
                {
                    uuid: 'test-uuid',
                    trace_id: null,
                    span_id: null,
                    trace_flags: null,
                    timestamp: null,
                    observed_timestamp: null,
                    body: JSON.stringify({ level: 'info', message: '[email protected]' }),
                    severity_text: null,
                    severity_number: null,
                    service_name: null,
                    resource_attributes: null,
                    instrumentation_scope: null,
                    event_name: null,
                    attributes: null,
                    bytes_uncompressed: null,
                },
                {
                    uuid: 'test-uuid-2',
                    trace_id: null,
                    span_id: null,
                    trace_flags: null,
                    timestamp: null,
                    observed_timestamp: null,
                    body: JSON.stringify({ level: 'warn', message: '[email protected]' }),
                    severity_text: null,
                    severity_number: null,
                    service_name: null,
                    resource_attributes: null,
                    instrumentation_scope: null,
                    event_name: null,
                    attributes: null,
                    bytes_uncompressed: null,
                },
            ]

            const inputBuffer = await encodeLogRecords(LOG_RECORD_SCHEMA, 'zstandard', records)
            await processLogMessageBuffer(inputBuffer, {
                json_parse_logs: true,
                pii_scrub_logs: true,
            })

            expect(spy).toHaveBeenCalledTimes(2)
            spy.mockRestore()
        })

        it('flattens nested JSON keys when both flags are on; body is pattern-scrubbed only', async () => {
            const records: LogRecord[] = [
                {
                    uuid: 'test-uuid',
                    trace_id: null,
                    span_id: null,
                    trace_flags: null,
                    timestamp: null,
                    observed_timestamp: null,
                    body: JSON.stringify({ meta: { api_key: 'leak-value' }, ok: 'keep' }),
                    severity_text: null,
                    severity_number: null,
                    service_name: null,
                    resource_attributes: null,
                    instrumentation_scope: null,
                    event_name: null,
                    attributes: null,
                    bytes_uncompressed: null,
                },
            ]

            const inputBuffer = await encodeLogRecords(LOG_RECORD_SCHEMA, 'zstandard', records)
            const { value: outputBuffer, pii } = await processLogMessageBuffer(inputBuffer, {
                json_parse_logs: true,
                pii_scrub_logs: true,
            })
            expect(pii).toEqual({ piiReplacements: 0 })
            const [_, __, decoded] = await decodeLogRecords(outputBuffer!)
            const body = parseJSON(decoded[0]?.body || '{}') as { meta: { api_key: string }; ok: string }
            // Body is pattern-scrubbed only; nested JSON keys are not redacted by key name.
            expect(body.meta.api_key).toBe('leak-value')
            expect(body.ok).toBe('keep')
            expect(decoded[0]?.attributes).toEqual({
                'meta.api_key': encodeAttributeCell('leak-value'),
                ok: encodeAttributeCell('keep'),
            })
        })

        it('rejects promise for invalid AVRO data', async () => {
            const invalidBuffer = Buffer.from('not avro data')

            await expect(processLogMessageBuffer(invalidBuffer, { json_parse_logs: true })).rejects.toThrow()
            await expect(
                processLogMessageBuffer(invalidBuffer, { json_parse_logs: false, pii_scrub_logs: true })
            ).rejects.toThrow()
        })

        it('limits attributes to 50 when parsing JSON body', async () => {
            const largeObject: Record<string, string> = {}
            for (let i = 0; i < 100; i++) {
                largeObject[`key${i}`] = `value${i}`
            }

            const records: LogRecord[] = [
                {
                    uuid: 'test-uuid',
                    trace_id: null,
                    span_id: null,
                    trace_flags: null,
                    timestamp: null,
                    observed_timestamp: null,
                    body: JSON.stringify(largeObject),
                    severity_text: null,
                    severity_number: null,
                    service_name: null,
                    resource_attributes: null,
                    instrumentation_scope: null,
                    event_name: null,
                    attributes: null,
                    bytes_uncompressed: null,
                },
            ]

            const inputBuffer = await encodeLogRecords(LOG_RECORD_SCHEMA, 'zstandard', records)
            const { value: outputBuffer, pii } = await processLogMessageBuffer(inputBuffer, { json_parse_logs: true })
            expect(pii).toEqual({ piiReplacements: 0 })
            const [_, __, decoded] = await decodeLogRecords(outputBuffer!)

            expect(Object.keys(decoded[0]?.attributes || {}).length).toBe(50)
        })
    })
})