Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(logs): add a signed internal logs producer for capture-logs (#124634)

Jon McCallumc311c03b65d5
diff --git a/nodejs/src/logs/config.ts b/nodejs/src/logs/config.ts
index 2ccd0bc0815b..47ffa1a313f0 100644
--- a/nodejs/src/logs/config.ts
+++ b/nodejs/src/logs/config.ts
@@ -69,7 +69,18 @@ export type LogsIngestionConsumerConfig = {
LOGS_RETENTION_KILLSWITCH: boolean
/** Comma-separated team IDs, or `*` for all teams. A `posthog-internal` batch from any other team is billed as normal. */
LOGS_INTERNAL_INGESTION_ENABLED_TEAMS: string
+ /** Secret. Comma-separated HMAC keys, newest first. Producers sign with the first key, and the consumer accepts any. */
LOGS_INTERNAL_INGESTION_SIGNING_KEYS: string
+ /** MUST be the in-cluster capture-logs service, never a public endpoint: the signature makes logs unbilled. Empty disables sending. */
+ LOGS_INTERNAL_PRODUCER_URL: string
+ /** Comma-separated team IDs, or `*` for all teams, whose internal logs a producer sends to the logs product. */
+ LOGS_INTERNAL_PRODUCER_TEAMS: string
+ /** Requests a producer keeps open at once. A batch over this cap waits in the buffer. */
+ LOGS_INTERNAL_PRODUCER_MAX_IN_FLIGHT: number
+ /** Bytes of records a producer holds while capture-logs is slow or down. A record that does not fit is dropped. */
+ LOGS_INTERNAL_PRODUCER_MAX_BUFFERED_BYTES: number
+ /** How long a producer retries one batch before it gives up on it. */
+ LOGS_INTERNAL_PRODUCER_RETRY_TIMEOUT_MS: number
/** Comma-separated team IDs, or `*` for all teams, or empty (default) to disable measure-only pattern masking. */
LOGS_PATTERN_MASKING_ENABLED_TEAMS: string
LOGS_JSON_ATTRIBUTE_PARSING_ENABLED_TEAMS: string
@@ -128,6 +139,11 @@ export function getDefaultLogsIngestionConsumerConfig(): LogsIngestionConsumerCo
LOGS_RETENTION_KILLSWITCH: false,
LOGS_INTERNAL_INGESTION_ENABLED_TEAMS: '',
LOGS_INTERNAL_INGESTION_SIGNING_KEYS: '',
+ LOGS_INTERNAL_PRODUCER_URL: '',
+ LOGS_INTERNAL_PRODUCER_TEAMS: '',
+ LOGS_INTERNAL_PRODUCER_MAX_IN_FLIGHT: 20,
+ LOGS_INTERNAL_PRODUCER_MAX_BUFFERED_BYTES: 64 * 1024 * 1024,
+ LOGS_INTERNAL_PRODUCER_RETRY_TIMEOUT_MS: 60_000,
// Off by default: enabling forces decode+re-encode for allowlisted teams.
LOGS_PATTERN_MASKING_ENABLED_TEAMS: '',
LOGS_JSON_ATTRIBUTE_PARSING_ENABLED_TEAMS: '',
diff --git a/nodejs/src/logs/internal-logs-producer.test.ts b/nodejs/src/logs/internal-logs-producer.test.ts
new file mode 100644
index 000000000000..3aa98995ffc3
--- /dev/null
+++ b/nodejs/src/logs/internal-logs-producer.test.ts
@@ -0,0 +1,427 @@
+import { DateTime } from 'luxon'
+import { register } from 'prom-client'
+
+import { parseJSON } from '~/common/utils/json-parse'
+import { internalFetch } from '~/common/utils/request'
+
+import { InternalLogsVerifier } from './internal-logs'
+import { InternalLogEntry, InternalLogsProducer, InternalLogsProducerConfig } from './internal-logs-producer'
+
+jest.mock('~/common/utils/request', () => ({ internalFetch: jest.fn() }))
+
+const KEY = 'internal-logs-test-key'
+const URL = 'http://capture-logs.test/i/v1/logs'
+const SIGNED_AT = new Date('2026-10-27T12:00:00Z')
+
+const fetchMock = jest.mocked(internalFetch)
+
+function config(overrides: Partial<InternalLogsProducerConfig> = {}): InternalLogsProducerConfig {
+ return {
+ LOGS_INTERNAL_INGESTION_SIGNING_KEYS: KEY,
+ LOGS_INTERNAL_PRODUCER_URL: URL,
+ LOGS_INTERNAL_PRODUCER_TEAMS: '2',
+ LOGS_INTERNAL_PRODUCER_MAX_IN_FLIGHT: 20,
+ LOGS_INTERNAL_PRODUCER_MAX_BUFFERED_BYTES: 64 * 1024 * 1024,
+ LOGS_INTERNAL_PRODUCER_RETRY_TIMEOUT_MS: 60_000,
+ ...overrides,
+ }
+}
+
+const teamManager = {
+ getTeam: jest.fn((teamId: number) => Promise.resolve({ id: teamId, api_token: `phc_team_${teamId}` } as any)),
+}
+
+function entry(overrides: Partial<InternalLogEntry> = {}): InternalLogEntry {
+ return {
+ team_id: 2,
+ log_source_id: 'flow-1',
+ instance_id: 'run-1',
+ timestamp: DateTime.fromISO('2026-10-27T12:00:00.123Z'),
+ level: 'warn',
+ message: 'Email step skipped',
+ ...overrides,
+ }
+}
+
+// Fake timers leave setImmediate real, and it runs after every resolved mock promise in the delivery chain.
+const settle = (): Promise<void> => new Promise((resolve) => setImmediate(resolve))
+
+function send(producer: InternalLogsProducer, entries: InternalLogEntry[], source = 'hog_flow'): Promise<void> {
+ producer.enqueue(source, entries)
+ producer.flush()
+ return settle()
+}
+
+const ok = { status: 200, dump: () => Promise.resolve() } as any
+
+function heldResponse(): { response: Promise<any>; release: () => void } {
+ let release: () => void = () => {}
+ const response = new Promise((resolve) => {
+ release = () => resolve(ok)
+ })
+ return { response, release }
+}
+
+describe('InternalLogsProducer', () => {
+ beforeEach(() => {
+ jest.useFakeTimers({ now: SIGNED_AT, doNotFake: ['nextTick', 'setImmediate'] })
+ fetchMock.mockReset()
+ fetchMock.mockResolvedValue({ status: 200, dump: () => Promise.resolve() } as any)
+ })
+
+ afterEach(() => {
+ jest.useRealTimers()
+ })
+
+ it('sends a batch the logs consumer verifies as internal', async () => {
+ await send(new InternalLogsProducer(config(), teamManager), [entry()])
+
+ expect(fetchMock).toHaveBeenCalledTimes(1)
+ const [url, options] = fetchMock.mock.calls[0]
+ const headers = options!.headers as Record<string, string>
+ expect(url).toEqual(URL)
+ expect(headers.Authorization).toEqual('Bearer phc_team_2')
+ const verifier = new InternalLogsVerifier(KEY, '2')
+ expect(verifier.verify(headers['x-posthog-internal'], 2, SIGNED_AT.toISOString())).toEqual({
+ source: 'hog_flow',
+ billed: false,
+ teamRules: false,
+ })
+
+ const body = parseJSON(options!.body as string) as any
+ expect(body.resourceLogs[0].resource.attributes).toEqual([
+ { key: 'service.name', value: { stringValue: 'posthog-internal-workflows' } },
+ ])
+ expect(body.resourceLogs[0].scopeLogs[0].logRecords).toEqual([
+ {
+ timeUnixNano: '1793102400123000000',
+ severityText: 'WARN',
+ severityNumber: 13,
+ body: { stringValue: 'Email step skipped' },
+ attributes: [
+ { key: 'log_source_id', value: { stringValue: 'flow-1' } },
+ { key: 'instance_id', value: { stringValue: 'run-1' } },
+ ],
+ },
+ ])
+ })
+
+ it('sends one request per team, each signed for its own team', async () => {
+ await send(new InternalLogsProducer(config({ LOGS_INTERNAL_PRODUCER_TEAMS: '2,3' }), teamManager), [
+ entry({ team_id: 2 }),
+ entry({ team_id: 3 }),
+ entry({ team_id: 2, message: 'second' }),
+ ])
+
+ const sent = fetchMock.mock.calls.map(([, options]) => {
+ const headers = options!.headers as Record<string, string>
+ const body = parseJSON(options!.body as string) as any
+ return { token: headers.Authorization, records: body.resourceLogs[0].scopeLogs[0].logRecords.length }
+ })
+ expect(sent).toEqual([
+ { token: 'Bearer phc_team_2', records: 2 },
+ { token: 'Bearer phc_team_3', records: 1 },
+ ])
+ const verifier = new InternalLogsVerifier(KEY, '*')
+ const team3Signature = (fetchMock.mock.calls[1][1]!.headers as Record<string, string>)['x-posthog-internal']
+ expect(verifier.verify(team3Signature, 3, SIGNED_AT.toISOString())).toBeDefined()
+ expect(verifier.verify(team3Signature, 2, SIGNED_AT.toISOString())).toBeUndefined()
+ })
+
+ it.each([
+ { name: 'a team not on the allowlist', entries: [entry({ team_id: 3 })] },
+ { name: 'a source with no policy', source: 'hog_function', entries: [entry()] },
+ { name: 'no endpoint', overrides: { LOGS_INTERNAL_PRODUCER_URL: '' }, entries: [entry()] },
+ { name: 'no signing key', overrides: { LOGS_INTERNAL_INGESTION_SIGNING_KEYS: ' ' }, entries: [entry()] },
+ { name: 'an empty allowlist', overrides: { LOGS_INTERNAL_PRODUCER_TEAMS: '' }, entries: [entry()] },
+ { name: 'an invalid timestamp, without throwing', entries: [entry({ timestamp: DateTime.invalid('bad') })] },
+ ])('sends nothing for $name', async ({ overrides = {}, source, entries }) => {
+ await send(new InternalLogsProducer(config(overrides), teamManager), entries, source)
+
+ expect(fetchMock).not.toHaveBeenCalled()
+ })
+
+ it.each([
+ {
+ name: 'waits, then leaves with the other waiting batches in one request',
+ maxBufferedBytes: 64 * 1024 * 1024,
+ expectedCalls: 2,
+ },
+ { name: 'is dropped when the buffer is full', maxBufferedBytes: 15_000, expectedCalls: 1 },
+ ])('a batch over the in-flight cap $name', async ({ maxBufferedBytes, expectedCalls }) => {
+ const held = heldResponse()
+ fetchMock.mockReturnValueOnce(held.response)
+ const producer = new InternalLogsProducer(
+ config({
+ LOGS_INTERNAL_PRODUCER_MAX_IN_FLIGHT: 1,
+ LOGS_INTERNAL_PRODUCER_MAX_BUFFERED_BYTES: maxBufferedBytes,
+ }),
+ teamManager
+ )
+ const message = 'x'.repeat(10_000)
+
+ const first = send(producer, [entry({ message })])
+ const waiting = [send(producer, [entry({ message })]), send(producer, [entry({ message })])]
+ await jest.advanceTimersByTimeAsync(0)
+ expect(fetchMock).toHaveBeenCalledTimes(1)
+
+ held.release()
+ await Promise.all([first, ...waiting, settle()])
+ expect(fetchMock).toHaveBeenCalledTimes(expectedCalls)
+ })
+
+ it.each([
+ {
+ name: 'a 503',
+ first: () => fetchMock.mockResolvedValueOnce({ status: 503, dump: () => Promise.resolve() } as any),
+ expectedCalls: 2,
+ },
+ {
+ name: 'a 429',
+ first: () => fetchMock.mockResolvedValueOnce({ status: 429, dump: () => Promise.resolve() } as any),
+ expectedCalls: 2,
+ },
+ {
+ name: 'a network error',
+ first: () => fetchMock.mockRejectedValueOnce(new Error('connect ECONNREFUSED')),
+ expectedCalls: 2,
+ },
+ {
+ name: 'a team lookup error',
+ first: () => teamManager.getTeam.mockRejectedValueOnce(new Error('connection reset')),
+ expectedCalls: 1,
+ },
+ {
+ name: 'a 400',
+ first: () => fetchMock.mockResolvedValueOnce({ status: 400, dump: () => Promise.resolve() } as any),
+ expectedCalls: 1,
+ },
+ ])('after $name, sends $expectedCalls request(s)', async ({ first, expectedCalls }) => {
+ first()
+
+ const sent = send(new InternalLogsProducer(config(), teamManager), [entry()])
+ await jest.advanceTimersByTimeAsync(10_000)
+ await sent
+
+ expect(fetchMock).toHaveBeenCalledTimes(expectedCalls)
+ })
+
+ it('signs each retry again, so a request that hung past the signature window does not make the retry billed', async () => {
+ const retriedAt = new Date(SIGNED_AT.getTime() + 10 * 60 * 1000)
+ fetchMock.mockImplementationOnce(() => {
+ jest.setSystemTime(retriedAt)
+ return Promise.reject(new Error('socket hang up'))
+ })
+
+ const sent = send(
+ new InternalLogsProducer(config({ LOGS_INTERNAL_PRODUCER_RETRY_TIMEOUT_MS: 30 * 60 * 1000 }), teamManager),
+ [entry()]
+ )
+ await jest.advanceTimersByTimeAsync(10_000)
+ await sent
+
+ expect(fetchMock).toHaveBeenCalledTimes(2)
+ const retrySignature = (fetchMock.mock.calls[1][1]!.headers as Record<string, string>)['x-posthog-internal']
+ expect(new InternalLogsVerifier(KEY, '2').verify(retrySignature, 2, retriedAt.toISOString())).toBeDefined()
+ })
+
+ it.each([
+ { name: 'every request fails', stall: () => fetchMock.mockRejectedValue(new Error('connect ECONNREFUSED')) },
+ {
+ name: 'the team lookup never answers',
+ stall: () => teamManager.getTeam.mockReturnValueOnce(new Promise(() => {})),
+ },
+ ])('gives up on a batch at the retry timeout when $name', async ({ stall }) => {
+ register.resetMetrics()
+ stall()
+ const producer = new InternalLogsProducer(
+ config({ LOGS_INTERNAL_PRODUCER_RETRY_TIMEOUT_MS: 2_000 }),
+ teamManager
+ )
+
+ await send(producer, [entry()])
+ await jest.advanceTimersByTimeAsync(5_000)
+
+ const attempts = fetchMock.mock.calls.length
+ await jest.advanceTimersByTimeAsync(60_000)
+ expect(fetchMock).toHaveBeenCalledTimes(attempts)
+ const metric = await register.getSingleMetric('logs_internal_producer_records_total')!.get()
+ expect(Object.fromEntries(metric.values.map((v) => [v.labels.outcome, v.value]))).toEqual({ failed: 1 })
+ expect((producer as any).inFlight.size).toBe(0)
+ })
+
+ it('drain sends entries that were never flushed and waits for them', async () => {
+ const held = heldResponse()
+ fetchMock.mockReturnValueOnce(held.response)
+ const producer = new InternalLogsProducer(config(), teamManager)
+ producer.enqueue('hog_flow', [entry()])
+
+ let drained = false
+ void producer.drain(10_000).then(() => {
+ drained = true
+ })
+ await jest.advanceTimersByTimeAsync(0)
+ expect(fetchMock).toHaveBeenCalledTimes(1)
+ expect(drained).toBe(false)
+
+ held.release()
+ await jest.advanceTimersByTimeAsync(100)
+ expect(drained).toBe(true)
+ })
+
+ it.each([
+ { name: 'a waiting batch', stalledLookup: false },
+ { name: 'a team lookup that outlives it', stalledLookup: true },
+ ])(
+ 'drain returns at its timeout, counts every unsent record once as abandoned, and sends nothing after, with $name',
+ async ({ stalledLookup }) => {
+ register.resetMetrics()
+ const held = heldResponse()
+ fetchMock.mockReturnValueOnce(held.response)
+ let releaseLookup: () => void = () => {}
+ if (stalledLookup) {
+ const getTeam = teamManager.getTeam.getMockImplementation()!
+ teamManager.getTeam
+ .mockImplementationOnce(getTeam)
+ .mockImplementationOnce(
+ (teamId) => new Promise((resolve) => (releaseLookup = () => resolve(getTeam(teamId))))
+ )
+ }
+ const producer = new InternalLogsProducer(
+ config({
+ LOGS_INTERNAL_PRODUCER_TEAMS: '2,3',
+ LOGS_INTERNAL_PRODUCER_MAX_IN_FLIGHT: stalledLookup ? 2 : 1,
+ }),
+ teamManager
+ )
+ void send(producer, [entry()])
+ producer.enqueue('hog_flow', [entry({ team_id: 3 })])
+
+ const drain = producer.drain(1_000)
+ await jest.advanceTimersByTimeAsync(1_000)
+
+ await expect(drain).resolves.toBeUndefined()
+ producer.enqueue('hog_flow', [entry()])
+ await producer.drain(1_000)
+ held.release()
+ releaseLookup()
+ await jest.advanceTimersByTimeAsync(10_000)
+
+ expect(fetchMock).toHaveBeenCalledTimes(1)
+ const metric = await register.getSingleMetric('logs_internal_producer_records_total')!.get()
+ expect(Object.fromEntries(metric.values.map((v) => [v.labels.outcome, v.value]))).toEqual({ abandoned: 3 })
+ expect(jest.getTimerCount()).toBe(0)
+ }
+ )
+
+ it('splits a large team batch into requests under the capture-logs body limit', async () => {
+ const message = 'x'.repeat(10_000)
+ await send(
+ new InternalLogsProducer(config(), teamManager),
+ Array.from({ length: 250 }, () => entry({ message }))
+ )
+
+ const bodies = fetchMock.mock.calls.map(([, options]) => options!.body as string)
+ expect(bodies.length).toBeGreaterThan(1)
+ for (const body of bodies) {
+ expect(Buffer.byteLength(body)).toBeLessThan(2 * 1024 * 1024)
+ }
+ const sent = bodies.map((body) => (parseJSON(body) as any).resourceLogs[0].scopeLogs[0].logRecords.length)
+ expect(sent.reduce((a, b) => a + b, 0)).toEqual(250)
+ })
+
+ it('cuts a message too large for one request, so capture-logs still accepts the record', async () => {
+ await send(new InternalLogsProducer(config(), teamManager), [entry({ message: '"'.repeat(3 * 1024 * 1024) })])
+
+ expect(fetchMock).toHaveBeenCalledTimes(1)
+ const body = fetchMock.mock.calls[0][1]!.body as string
+ expect(Buffer.byteLength(body)).toBeLessThan(2 * 1024 * 1024)
+ expect((parseJSON(body) as any).resourceLogs[0].scopeLogs[0].logRecords[0].body.stringValue).toEqual(
+ '"'.repeat(128 * 1024)
+ )
+ })
+
+ it('replaces an unpaired surrogate so capture-logs can decode the rest of the batch', async () => {
+ await send(new InternalLogsProducer(config(), teamManager), [
+ entry({ message: '😀'.substring(0, 1) }),
+ entry({ message: 'valid' }),
+ ])
+
+ const body = fetchMock.mock.calls[0][1]!.body as string
+ const messages = (parseJSON(body) as any).resourceLogs[0].scopeLogs[0].logRecords.map(
+ (r: any) => r.body.stringValue
+ )
+ expect(messages).toEqual(['\uFFFD', 'valid'])
+ })
+
+ it.each(Array.from({ length: 20 }, (_, i) => i + 1))(
+ 'keeps its accounting under random interleavings, seed %i',
+ async (seed) => {
+ register.resetMetrics()
+ let state = seed
+ const rand = (): number => {
+ state = (state * 1_103_515_245 + 12_345) % 2 ** 31
+ return state / 2 ** 31
+ }
+ const randomSpy = jest.spyOn(Math, 'random').mockImplementation(rand)
+ const open: { resolve: (response: any) => void; reject: (error: Error) => void }[] = []
+ fetchMock.mockImplementation(() => new Promise((resolve, reject) => open.push({ resolve, reject })))
+ const cfg = config({
+ LOGS_INTERNAL_PRODUCER_TEAMS: '*',
+ LOGS_INTERNAL_PRODUCER_MAX_IN_FLIGHT: 1 + Math.floor(rand() * 3),
+ LOGS_INTERNAL_PRODUCER_MAX_BUFFERED_BYTES: 2_000 + Math.floor(rand() * 20_000),
+ LOGS_INTERNAL_PRODUCER_RETRY_TIMEOUT_MS: 5_000,
+ })
+ const producer = new InternalLogsProducer(cfg, teamManager)
+ const internals = producer as any
+ const counted = async (): Promise<number> => {
+ const metric = await register.getSingleMetric('logs_internal_producer_records_total')!.get()
+ return metric.values.reduce((sum, v) => sum + v.value, 0)
+ }
+ let enqueued = 0
+
+ for (let step = 0; step < 150; step++) {
+ const op = rand()
+ if (op < 0.35) {
+ const entries = Array.from({ length: 1 + Math.floor(rand() * 5) }, () =>
+ entry({ team_id: 1 + Math.floor(rand() * 3), message: 'x'.repeat(Math.floor(rand() * 2_000)) })
+ )
+ producer.enqueue('hog_flow', entries)
+ enqueued += entries.length
+ } else if (op < 0.5) {
+ producer.flush()
+ } else if (op < 0.75 && open.length > 0) {
+ const { resolve, reject } = open.splice(Math.floor(rand() * open.length), 1)[0]
+ const status = [200, 200, 400, 503, 0][Math.floor(rand() * 5)]
+ if (status === 0) {
+ reject(new Error('reset'))
+ } else {
+ resolve({ status, dump: () => Promise.resolve() })
+ }
+ } else if (op < 0.97) {
+ await jest.advanceTimersByTimeAsync(Math.floor(rand() * 3_000))
+ } else {
+ void producer.drain(Math.floor(rand() * 2_000))
+ }
+ await jest.advanceTimersByTimeAsync(0)
+
+ const live = [...internals.waiting.values(), ...internals.inFlight]
+ const recordBytes = live.flatMap((b) => b.records).reduce((sum, r) => sum + Buffer.byteLength(r), 0)
+ expect(internals.bufferedBytes).toBe(recordBytes)
+ expect(internals.bufferedBytes).toBeLessThanOrEqual(cfg.LOGS_INTERNAL_PRODUCER_MAX_BUFFERED_BYTES)
+ expect(internals.inFlight.size).toBeLessThanOrEqual(cfg.LOGS_INTERNAL_PRODUCER_MAX_IN_FLIGHT)
+ // A stopped producer already counted its in-flight records as abandoned.
+ const held = internals.stopped ? 0 : live.reduce((sum, b) => sum + b.records.length, 0)
+ expect((await counted()) + held).toBe(enqueued)
+ }
+
+ const drained = producer.drain(0)
+ await jest.advanceTimersByTimeAsync(0)
+ await drained
+ open.splice(0).forEach(({ resolve }) => resolve(ok))
+ await jest.advanceTimersByTimeAsync(60_000)
+ expect(await counted()).toBe(enqueued)
+ randomSpy.mockRestore()
+ }
+ )
+})
diff --git a/nodejs/src/logs/internal-logs-producer.ts b/nodejs/src/logs/internal-logs-producer.ts
new file mode 100644
index 000000000000..a133cf2660d8
--- /dev/null
+++ b/nodejs/src/logs/internal-logs-producer.ts
@@ -0,0 +1,322 @@
+import { DateTime } from 'luxon'
+import { Counter, Gauge } from 'prom-client'
+
+import { buildIntegerMatcher } from '~/common/config/matchers'
+import { logger } from '~/common/utils/logger'
+import { internalFetch } from '~/common/utils/request'
+import { sanitizeForUTF8 } from '~/common/utils/strings'
+import type { TeamManager } from '~/common/utils/team-manager'
+import { delay } from '~/common/utils/utils'
+import type { ValueMatcher } from '~/types'
+
+import type { LogsIngestionConsumerConfig } from './config'
+import { INTERNAL_LOGS_HTTP_HEADER, INTERNAL_SOURCE_POLICIES, signInternalLogs } from './internal-logs'
+
+const SERVICE_NAMES: Readonly<Record<string, string>> = {
+ hog_flow: 'posthog-internal-workflows',
+}
+
+const SEVERITY_NUMBERS: Readonly<Record<string, number>> = { debug: 5, info: 9, warn: 13, error: 17 }
+
+// Half of capture-logs' default 2 MiB body limit, which leaves room for the envelope.
+const MAX_REQUEST_RECORD_BYTES = 1024 * 1024
+
+// A JSON-escaped character takes at most 6 bytes, so a message cut to this length always fits in one request.
+const MAX_MESSAGE_LENGTH = 128 * 1024
+
+export type InternalLogEntry = {
+ team_id: number
+ log_source_id: string
+ instance_id: string
+ timestamp: DateTime
+ level: string
+ message: string
+}
+
+export type InternalLogsProducerConfig = Pick<
+ LogsIngestionConsumerConfig,
+ | 'LOGS_INTERNAL_INGESTION_SIGNING_KEYS'
+ | 'LOGS_INTERNAL_PRODUCER_URL'
+ | 'LOGS_INTERNAL_PRODUCER_TEAMS'
+ | 'LOGS_INTERNAL_PRODUCER_MAX_IN_FLIGHT'
+ | 'LOGS_INTERNAL_PRODUCER_MAX_BUFFERED_BYTES'
+ | 'LOGS_INTERNAL_PRODUCER_RETRY_TIMEOUT_MS'
+>
+
+type Batch = { source: string; teamId: number; records: string[] }
+
+type Request = Batch & { bytes: number }
+
+type AttemptResult = { outcome: 'sent' | 'rejected' | 'unknown_team' | 'retry'; detail: string }
+
+const RETRY_BASE_DELAY_MS = 100
+const RETRY_MAX_DELAY_MS = 5_000
+const DRAIN_POLL_MS = 50
+
+const internalLogsProducerRecordsCounter = new Counter({
+ name: 'logs_internal_producer_records_total',
+ help: 'Internal log records sent to capture-logs, by source and outcome. Only a sent record reaches the logs product.',
+ labelNames: ['source', 'outcome'],
+})
+
+const internalLogsProducerBufferedBytesGauge = new Gauge({
+ name: 'logs_internal_producer_buffered_bytes',
+ help: 'Bytes of internal log records a producer holds in memory, queued, waiting or in flight.',
+})
+
+const internalLogsProducerInFlightGauge = new Gauge({
+ name: 'logs_internal_producer_in_flight_requests',
+ help: 'Requests a producer has open to capture-logs.',
+})
+
+function countRecords(source: string, outcome: string, records: number): void {
+ if (records > 0) {
+ internalLogsProducerRecordsCounter.inc({ source, outcome }, records)
+ }
+}
+
+/**
+ * Delivery runs in the background, so a slow or down capture-logs never slows the caller. A record is lost only when
+ * it is invalid, the buffer is full, its retries time out, capture-logs rejects it, or `drain` times out.
+ */
+export class InternalLogsProducer {
+ private readonly enabled: boolean
+ private readonly isTeamEnabled: ValueMatcher<number>
+ // One batch per source and team, so records that wait for a free request slot leave together in large requests.
+ private waiting = new Map<string, Batch>()
+ // Counts waiting and in-flight records, because each one holds its bytes until its request settles.
+ private bufferedBytes = 0
+ private inFlight = new Set<Request>()
+ // Set when `drain` ends. After it, nothing is sent or counted twice, and a retrying request stops.
+ private stopped = false
+
+ constructor(
+ private readonly config: InternalLogsProducerConfig,
+ private readonly teamManager: Pick<TeamManager, 'getTeam'>
+ ) {
+ this.enabled =
+ config.LOGS_INTERNAL_PRODUCER_URL.length > 0 &&
+ config.LOGS_INTERNAL_INGESTION_SIGNING_KEYS.trim().length > 0
+ this.isTeamEnabled = buildIntegerMatcher(config.LOGS_INTERNAL_PRODUCER_TEAMS, true)
+ }
+
+ enqueue(source: string, entries: InternalLogEntry[]): void {
+ if (
+ !this.enabled ||
+ !Object.hasOwn(INTERNAL_SOURCE_POLICIES, source) ||
+ !Object.hasOwn(SERVICE_NAMES, source)
+ ) {
+ return
+ }
+ let invalid = 0
+ let dropped = 0
+ let abandoned = 0
+ for (const entry of entries) {
+ if (!this.isTeamEnabled(entry.team_id)) {
+ continue
+ }
+ if (this.stopped) {
+ abandoned++
+ continue
+ }
+ let record: string
+ try {
+ record = JSON.stringify(toOtlpLogRecord(entry))
+ } catch {
+ invalid++
+ continue
+ }
+ const bytes = Buffer.byteLength(record)
+ if (this.bufferedBytes + bytes > this.config.LOGS_INTERNAL_PRODUCER_MAX_BUFFERED_BYTES) {
+ dropped++
+ continue
+ }
+ this.bufferedBytes += bytes
+ const key = `${source}:${entry.team_id}`
+ const batch = this.waiting.get(key) ?? { source, teamId: entry.team_id, records: [] }
+ batch.records.push(record)
+ this.waiting.set(key, batch)
+ }
+ countRecords(source, 'invalid', invalid)
+ countRecords(source, 'dropped', dropped)
+ countRecords(source, 'abandoned', abandoned)
+ this.report()
+ }
+
+ /** Starts sending the waiting records. Never throws. */
+ flush(): void {
+ this.pump()
+ }
+
+ /** Sends what is left until the timeout, then stops and counts the rest as abandoned. Call on shutdown. */
+ async drain(timeoutMs: number): Promise<void> {
+ const deadline = Date.now() + timeoutMs
+ this.pump()
+ while (!this.stopped && !this.isIdle() && Date.now() < deadline) {
+ await delay(Math.min(DRAIN_POLL_MS, deadline - Date.now()))
+ }
+ if (!this.stopped && !this.isIdle()) {
+ this.abandon()
+ }
+ this.stopped = true
+ }
+
+ private abandon(): void {
+ this.stopped = true
+ const unsent = [...this.waiting.values(), ...this.inFlight]
+ for (const { source, records } of unsent) {
+ countRecords(source, 'abandoned', records.length)
+ }
+ logger.warn('Internal logs producer stopped before all records were sent', {
+ abandonedRecords: unsent.reduce((sum, { records }) => sum + records.length, 0),
+ inFlightRequests: this.inFlight.size,
+ })
+ this.waiting.clear()
+ this.bufferedBytes = [...this.inFlight].reduce((sum, request) => sum + request.bytes, 0)
+ this.report()
+ }
+
+ private pump(): void {
+ while (this.inFlight.size < this.config.LOGS_INTERNAL_PRODUCER_MAX_IN_FLIGHT && this.waiting.size > 0) {
+ const [key, batch] = this.waiting.entries().next().value!
+ // Moves the rest of the batch to the back, so each team gets its turn.
+ this.waiting.delete(key)
+ const request = takeRequest(batch)
+ if (batch.records.length > 0) {
+ this.waiting.set(key, batch)
+ }
+ this.inFlight.add(request)
+ void this.deliver(request).finally(() => {
+ this.inFlight.delete(request)
+ this.bufferedBytes -= request.bytes
+ this.pump()
+ })
+ }
+ this.report()
+ }
+
+ private isIdle(): boolean {
+ return this.inFlight.size === 0 && this.waiting.size === 0
+ }
+
+ private report(): void {
+ internalLogsProducerBufferedBytesGauge.set(this.bufferedBytes)
+ internalLogsProducerInFlightGauge.set(this.inFlight.size)
+ }
+
+ private async deliver({ source, teamId, records }: Batch): Promise<void> {
+ const body = toOtlpBody(source, records)
+ const deadline = Date.now() + this.config.LOGS_INTERNAL_PRODUCER_RETRY_TIMEOUT_MS
+ for (let attempt = 0; !this.stopped; attempt++) {
+ const result = await this.attempt(source, teamId, body, deadline)
+ if (this.stopped) {
+ return
+ }
+ if (result.outcome !== 'retry') {
+ countRecords(source, result.outcome, records.length)
+ if (result.outcome === 'rejected') {
+ logger.warn('Internal logs batch rejected by capture-logs', {
+ source,
+ teamId,
+ status: result.detail,
+ })
+ }
+ return
+ }
+ const wait = Math.random() * Math.min(RETRY_MAX_DELAY_MS, RETRY_BASE_DELAY_MS * 2 ** attempt)
+ if (Date.now() + wait > deadline) {
+ countRecords(source, 'failed', records.length)
+ logger.warn('Internal logs batch failed after retries', {
+ source,
+ teamId,
+ attempts: attempt + 1,
+ last: result.detail,
+ })
+ return
+ }
+ await delay(wait)
+ }
+ }
+
+ private async attempt(source: string, teamId: number, body: string, deadline: number): Promise<AttemptResult> {
+ let timer: NodeJS.Timeout | undefined
+ try {
+ // A team lookup has no timeout of its own, so a stalled Postgres would hold this request slot past the deadline.
+ const team = await Promise.race([
+ this.teamManager.getTeam(teamId),
+ new Promise<never>((_, reject) => {
+ timer = setTimeout(() => reject(new Error('team lookup timed out')), deadline - Date.now())
+ }),
+ ])
+ if (this.stopped) {
+ return { outcome: 'retry', detail: 'stopped' }
+ }
+ if (!team) {
+ return { outcome: 'unknown_team', detail: String(teamId) }
+ }
+ const response = await internalFetch(this.config.LOGS_INTERNAL_PRODUCER_URL, {
+ method: 'POST',
+ headers: {
+ 'Content-Type': 'application/json',
+ Authorization: `Bearer ${team.api_token}`,
+ [INTERNAL_LOGS_HTTP_HEADER]: signInternalLogs(
+ this.config.LOGS_INTERNAL_INGESTION_SIGNING_KEYS,
+ source,
+ teamId
+ ),
+ },
+ body,
+ })
+ await response.dump()
+ const { status } = response
+ if (status >= 200 && status < 300) {
+ return { outcome: 'sent', detail: String(status) }
+ }
+ // Any other 4xx is a bad request, which fails the same way on every attempt.
+ const retriable = status >= 500 || status === 408 || status === 429
+ return { outcome: retriable ? 'retry' : 'rejected', detail: String(status) }
+ } catch (error) {
+ return { outcome: 'retry', detail: String(error) }
+ } finally {
+ clearTimeout(timer)
+ }
+ }
+}
+
+type OtlpLogRecord = Record<string, unknown>
+
+function toOtlpLogRecord(entry: InternalLogEntry): OtlpLogRecord {
+ return {
+ timeUnixNano: (BigInt(entry.timestamp.toMillis()) * 1_000_000n).toString(),
+ severityText: entry.level.toUpperCase(),
+ severityNumber: SEVERITY_NUMBERS[entry.level] ?? 0,
+ // capture-logs rejects a whole request that holds one unpaired surrogate. The cut can split a pair, so sanitize after it.
+ body: { stringValue: sanitizeForUTF8(entry.message.slice(0, MAX_MESSAGE_LENGTH)) },
+ attributes: [
+ { key: 'log_source_id', value: { stringValue: entry.log_source_id } },
+ { key: 'instance_id', value: { stringValue: entry.instance_id } },
+ ],
+ }
+}
+
+/** Removes up to MAX_REQUEST_RECORD_BYTES of records from the front of the batch, and at least one. */
+function takeRequest(batch: Batch): Request {
+ let count = 0
+ let bytes = 0
+ while (count < batch.records.length) {
+ const recordBytes = Buffer.byteLength(batch.records[count])
+ if (count > 0 && bytes + recordBytes > MAX_REQUEST_RECORD_BYTES) {
+ break
+ }
+ bytes += recordBytes
+ count++
+ }
+ return { source: batch.source, teamId: batch.teamId, records: batch.records.splice(0, count), bytes }
+}
+
+function toOtlpBody(source: string, records: string[]): string {
+ const resource = JSON.stringify({
+ attributes: [{ key: 'service.name', value: { stringValue: SERVICE_NAMES[source] } }],
+ })
+ return `{"resourceLogs":[{"resource":${resource},"scopeLogs":[{"scope":${JSON.stringify({ name: source })},"logRecords":[${records.join(',')}]}]}]}`
+}
diff --git a/nodejs/src/logs/internal-logs.ts b/nodejs/src/logs/internal-logs.ts
index 33f9c492000e..86108a119fe8 100644
--- a/nodejs/src/logs/internal-logs.ts
+++ b/nodejs/src/logs/internal-logs.ts
@@ -7,6 +7,9 @@ import type { ValueMatcher } from '~/types'
export const INTERNAL_LOGS_HEADER = 'posthog-internal'
+// The HTTP request header that capture-logs forwards as the `posthog-internal` Kafka header.
+export const INTERNAL_LOGS_HTTP_HEADER = 'x-posthog-internal'
+
export const INTERNAL_SOURCE_HEADER = 'internal-source'
const SIGNATURE_VERSION = 'v1'