:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(logs): add a signed internal logs producer for capture-logs (#124634)
Jon McCallumc311c03b65d5
diff --git a/nodejs/src/logs/config.ts b/nodejs/src/logs/config.tsindex 2ccd0bc0815b..47ffa1a313f0 100644--- a/nodejs/src/logs/config.ts+++ b/nodejs/src/logs/config.ts@@ -69,7 +69,18 @@ export type LogsIngestionConsumerConfig = {LOGS_RETENTION_KILLSWITCH: boolean/** Comma-separated team IDs, or `*` for all teams. A `posthog-internal` batch from any other team is billed as normal. */LOGS_INTERNAL_INGESTION_ENABLED_TEAMS: string+ /** Secret. Comma-separated HMAC keys, newest first. Producers sign with the first key, and the consumer accepts any. */LOGS_INTERNAL_INGESTION_SIGNING_KEYS: string+ /** MUST be the in-cluster capture-logs service, never a public endpoint: the signature makes logs unbilled. Empty disables sending. */+ LOGS_INTERNAL_PRODUCER_URL: string+ /** Comma-separated team IDs, or `*` for all teams, whose internal logs a producer sends to the logs product. */+ LOGS_INTERNAL_PRODUCER_TEAMS: string+ /** Requests a producer keeps open at once. A batch over this cap waits in the buffer. */+ LOGS_INTERNAL_PRODUCER_MAX_IN_FLIGHT: number+ /** Bytes of records a producer holds while capture-logs is slow or down. A record that does not fit is dropped. */+ LOGS_INTERNAL_PRODUCER_MAX_BUFFERED_BYTES: number+ /** How long a producer retries one batch before it gives up on it. */+ LOGS_INTERNAL_PRODUCER_RETRY_TIMEOUT_MS: number/** Comma-separated team IDs, or `*` for all teams, or empty (default) to disable measure-only pattern masking. */LOGS_PATTERN_MASKING_ENABLED_TEAMS: stringLOGS_JSON_ATTRIBUTE_PARSING_ENABLED_TEAMS: string@@ -128,6 +139,11 @@ export function getDefaultLogsIngestionConsumerConfig(): LogsIngestionConsumerCoLOGS_RETENTION_KILLSWITCH: false,LOGS_INTERNAL_INGESTION_ENABLED_TEAMS: '',LOGS_INTERNAL_INGESTION_SIGNING_KEYS: '',+ LOGS_INTERNAL_PRODUCER_URL: '',+ LOGS_INTERNAL_PRODUCER_TEAMS: '',+ LOGS_INTERNAL_PRODUCER_MAX_IN_FLIGHT: 20,+ LOGS_INTERNAL_PRODUCER_MAX_BUFFERED_BYTES: 64 * 1024 * 1024,+ LOGS_INTERNAL_PRODUCER_RETRY_TIMEOUT_MS: 60_000,// Off by default: enabling forces decode+re-encode for allowlisted teams.LOGS_PATTERN_MASKING_ENABLED_TEAMS: '',LOGS_JSON_ATTRIBUTE_PARSING_ENABLED_TEAMS: '',diff --git a/nodejs/src/logs/internal-logs-producer.test.ts b/nodejs/src/logs/internal-logs-producer.test.tsnew file mode 100644index 000000000000..3aa98995ffc3--- /dev/null+++ b/nodejs/src/logs/internal-logs-producer.test.ts@@ -0,0 +1,427 @@+import { DateTime } from 'luxon'+import { register } from 'prom-client'++import { parseJSON } from '~/common/utils/json-parse'+import { internalFetch } from '~/common/utils/request'++import { InternalLogsVerifier } from './internal-logs'+import { InternalLogEntry, InternalLogsProducer, InternalLogsProducerConfig } from './internal-logs-producer'++jest.mock('~/common/utils/request', () => ({ internalFetch: jest.fn() }))++const KEY = 'internal-logs-test-key'+const URL = 'http://capture-logs.test/i/v1/logs'+const SIGNED_AT = new Date('2026-10-27T12:00:00Z')++const fetchMock = jest.mocked(internalFetch)++function config(overrides: Partial<InternalLogsProducerConfig> = {}): InternalLogsProducerConfig {+ return {+ LOGS_INTERNAL_INGESTION_SIGNING_KEYS: KEY,+ LOGS_INTERNAL_PRODUCER_URL: URL,+ LOGS_INTERNAL_PRODUCER_TEAMS: '2',+ LOGS_INTERNAL_PRODUCER_MAX_IN_FLIGHT: 20,+ LOGS_INTERNAL_PRODUCER_MAX_BUFFERED_BYTES: 64 * 1024 * 1024,+ LOGS_INTERNAL_PRODUCER_RETRY_TIMEOUT_MS: 60_000,+ ...overrides,+ }+}++const teamManager = {+ getTeam: jest.fn((teamId: number) => Promise.resolve({ id: teamId, api_token: `phc_team_${teamId}` } as any)),+}++function entry(overrides: Partial<InternalLogEntry> = {}): InternalLogEntry {+ return {+ team_id: 2,+ log_source_id: 'flow-1',+ instance_id: 'run-1',+ timestamp: DateTime.fromISO('2026-10-27T12:00:00.123Z'),+ level: 'warn',+ message: 'Email step skipped',+ ...overrides,+ }+}++// Fake timers leave setImmediate real, and it runs after every resolved mock promise in the delivery chain.+const settle = (): Promise<void> => new Promise((resolve) => setImmediate(resolve))++function send(producer: InternalLogsProducer, entries: InternalLogEntry[], source = 'hog_flow'): Promise<void> {+ producer.enqueue(source, entries)+ producer.flush()+ return settle()+}++const ok = { status: 200, dump: () => Promise.resolve() } as any++function heldResponse(): { response: Promise<any>; release: () => void } {+ let release: () => void = () => {}+ const response = new Promise((resolve) => {+ release = () => resolve(ok)+ })+ return { response, release }+}++describe('InternalLogsProducer', () => {+ beforeEach(() => {+ jest.useFakeTimers({ now: SIGNED_AT, doNotFake: ['nextTick', 'setImmediate'] })+ fetchMock.mockReset()+ fetchMock.mockResolvedValue({ status: 200, dump: () => Promise.resolve() } as any)+ })++ afterEach(() => {+ jest.useRealTimers()+ })++ it('sends a batch the logs consumer verifies as internal', async () => {+ await send(new InternalLogsProducer(config(), teamManager), [entry()])++ expect(fetchMock).toHaveBeenCalledTimes(1)+ const [url, options] = fetchMock.mock.calls[0]+ const headers = options!.headers as Record<string, string>+ expect(url).toEqual(URL)+ expect(headers.Authorization).toEqual('Bearer phc_team_2')+ const verifier = new InternalLogsVerifier(KEY, '2')+ expect(verifier.verify(headers['x-posthog-internal'], 2, SIGNED_AT.toISOString())).toEqual({+ source: 'hog_flow',+ billed: false,+ teamRules: false,+ })++ const body = parseJSON(options!.body as string) as any+ expect(body.resourceLogs[0].resource.attributes).toEqual([+ { key: 'service.name', value: { stringValue: 'posthog-internal-workflows' } },+ ])+ expect(body.resourceLogs[0].scopeLogs[0].logRecords).toEqual([+ {+ timeUnixNano: '1793102400123000000',+ severityText: 'WARN',+ severityNumber: 13,+ body: { stringValue: 'Email step skipped' },+ attributes: [+ { key: 'log_source_id', value: { stringValue: 'flow-1' } },+ { key: 'instance_id', value: { stringValue: 'run-1' } },+ ],+ },+ ])+ })++ it('sends one request per team, each signed for its own team', async () => {+ await send(new InternalLogsProducer(config({ LOGS_INTERNAL_PRODUCER_TEAMS: '2,3' }), teamManager), [+ entry({ team_id: 2 }),+ entry({ team_id: 3 }),+ entry({ team_id: 2, message: 'second' }),+ ])++ const sent = fetchMock.mock.calls.map(([, options]) => {+ const headers = options!.headers as Record<string, string>+ const body = parseJSON(options!.body as string) as any+ return { token: headers.Authorization, records: body.resourceLogs[0].scopeLogs[0].logRecords.length }+ })+ expect(sent).toEqual([+ { token: 'Bearer phc_team_2', records: 2 },+ { token: 'Bearer phc_team_3', records: 1 },+ ])+ const verifier = new InternalLogsVerifier(KEY, '*')+ const team3Signature = (fetchMock.mock.calls[1][1]!.headers as Record<string, string>)['x-posthog-internal']+ expect(verifier.verify(team3Signature, 3, SIGNED_AT.toISOString())).toBeDefined()+ expect(verifier.verify(team3Signature, 2, SIGNED_AT.toISOString())).toBeUndefined()+ })++ it.each([+ { name: 'a team not on the allowlist', entries: [entry({ team_id: 3 })] },+ { name: 'a source with no policy', source: 'hog_function', entries: [entry()] },+ { name: 'no endpoint', overrides: { LOGS_INTERNAL_PRODUCER_URL: '' }, entries: [entry()] },+ { name: 'no signing key', overrides: { LOGS_INTERNAL_INGESTION_SIGNING_KEYS: ' ' }, entries: [entry()] },+ { name: 'an empty allowlist', overrides: { LOGS_INTERNAL_PRODUCER_TEAMS: '' }, entries: [entry()] },+ { name: 'an invalid timestamp, without throwing', entries: [entry({ timestamp: DateTime.invalid('bad') })] },+ ])('sends nothing for $name', async ({ overrides = {}, source, entries }) => {+ await send(new InternalLogsProducer(config(overrides), teamManager), entries, source)++ expect(fetchMock).not.toHaveBeenCalled()+ })++ it.each([+ {+ name: 'waits, then leaves with the other waiting batches in one request',+ maxBufferedBytes: 64 * 1024 * 1024,+ expectedCalls: 2,+ },+ { name: 'is dropped when the buffer is full', maxBufferedBytes: 15_000, expectedCalls: 1 },+ ])('a batch over the in-flight cap $name', async ({ maxBufferedBytes, expectedCalls }) => {+ const held = heldResponse()+ fetchMock.mockReturnValueOnce(held.response)+ const producer = new InternalLogsProducer(+ config({+ LOGS_INTERNAL_PRODUCER_MAX_IN_FLIGHT: 1,+ LOGS_INTERNAL_PRODUCER_MAX_BUFFERED_BYTES: maxBufferedBytes,+ }),+ teamManager+ )+ const message = 'x'.repeat(10_000)++ const first = send(producer, [entry({ message })])+ const waiting = [send(producer, [entry({ message })]), send(producer, [entry({ message })])]+ await jest.advanceTimersByTimeAsync(0)+ expect(fetchMock).toHaveBeenCalledTimes(1)++ held.release()+ await Promise.all([first, ...waiting, settle()])+ expect(fetchMock).toHaveBeenCalledTimes(expectedCalls)+ })++ it.each([+ {+ name: 'a 503',+ first: () => fetchMock.mockResolvedValueOnce({ status: 503, dump: () => Promise.resolve() } as any),+ expectedCalls: 2,+ },+ {+ name: 'a 429',+ first: () => fetchMock.mockResolvedValueOnce({ status: 429, dump: () => Promise.resolve() } as any),+ expectedCalls: 2,+ },+ {+ name: 'a network error',+ first: () => fetchMock.mockRejectedValueOnce(new Error('connect ECONNREFUSED')),+ expectedCalls: 2,+ },+ {+ name: 'a team lookup error',+ first: () => teamManager.getTeam.mockRejectedValueOnce(new Error('connection reset')),+ expectedCalls: 1,+ },+ {+ name: 'a 400',+ first: () => fetchMock.mockResolvedValueOnce({ status: 400, dump: () => Promise.resolve() } as any),+ expectedCalls: 1,+ },+ ])('after $name, sends $expectedCalls request(s)', async ({ first, expectedCalls }) => {+ first()++ const sent = send(new InternalLogsProducer(config(), teamManager), [entry()])+ await jest.advanceTimersByTimeAsync(10_000)+ await sent++ expect(fetchMock).toHaveBeenCalledTimes(expectedCalls)+ })++ it('signs each retry again, so a request that hung past the signature window does not make the retry billed', async () => {+ const retriedAt = new Date(SIGNED_AT.getTime() + 10 * 60 * 1000)+ fetchMock.mockImplementationOnce(() => {+ jest.setSystemTime(retriedAt)+ return Promise.reject(new Error('socket hang up'))+ })++ const sent = send(+ new InternalLogsProducer(config({ LOGS_INTERNAL_PRODUCER_RETRY_TIMEOUT_MS: 30 * 60 * 1000 }), teamManager),+ [entry()]+ )+ await jest.advanceTimersByTimeAsync(10_000)+ await sent++ expect(fetchMock).toHaveBeenCalledTimes(2)+ const retrySignature = (fetchMock.mock.calls[1][1]!.headers as Record<string, string>)['x-posthog-internal']+ expect(new InternalLogsVerifier(KEY, '2').verify(retrySignature, 2, retriedAt.toISOString())).toBeDefined()+ })++ it.each([+ { name: 'every request fails', stall: () => fetchMock.mockRejectedValue(new Error('connect ECONNREFUSED')) },+ {+ name: 'the team lookup never answers',+ stall: () => teamManager.getTeam.mockReturnValueOnce(new Promise(() => {})),+ },+ ])('gives up on a batch at the retry timeout when $name', async ({ stall }) => {+ register.resetMetrics()+ stall()+ const producer = new InternalLogsProducer(+ config({ LOGS_INTERNAL_PRODUCER_RETRY_TIMEOUT_MS: 2_000 }),+ teamManager+ )++ await send(producer, [entry()])+ await jest.advanceTimersByTimeAsync(5_000)++ const attempts = fetchMock.mock.calls.length+ await jest.advanceTimersByTimeAsync(60_000)+ expect(fetchMock).toHaveBeenCalledTimes(attempts)+ const metric = await register.getSingleMetric('logs_internal_producer_records_total')!.get()+ expect(Object.fromEntries(metric.values.map((v) => [v.labels.outcome, v.value]))).toEqual({ failed: 1 })+ expect((producer as any).inFlight.size).toBe(0)+ })++ it('drain sends entries that were never flushed and waits for them', async () => {+ const held = heldResponse()+ fetchMock.mockReturnValueOnce(held.response)+ const producer = new InternalLogsProducer(config(), teamManager)+ producer.enqueue('hog_flow', [entry()])++ let drained = false+ void producer.drain(10_000).then(() => {+ drained = true+ })+ await jest.advanceTimersByTimeAsync(0)+ expect(fetchMock).toHaveBeenCalledTimes(1)+ expect(drained).toBe(false)++ held.release()+ await jest.advanceTimersByTimeAsync(100)+ expect(drained).toBe(true)+ })++ it.each([+ { name: 'a waiting batch', stalledLookup: false },+ { name: 'a team lookup that outlives it', stalledLookup: true },+ ])(+ 'drain returns at its timeout, counts every unsent record once as abandoned, and sends nothing after, with $name',+ async ({ stalledLookup }) => {+ register.resetMetrics()+ const held = heldResponse()+ fetchMock.mockReturnValueOnce(held.response)+ let releaseLookup: () => void = () => {}+ if (stalledLookup) {+ const getTeam = teamManager.getTeam.getMockImplementation()!+ teamManager.getTeam+ .mockImplementationOnce(getTeam)+ .mockImplementationOnce(+ (teamId) => new Promise((resolve) => (releaseLookup = () => resolve(getTeam(teamId))))+ )+ }+ const producer = new InternalLogsProducer(+ config({+ LOGS_INTERNAL_PRODUCER_TEAMS: '2,3',+ LOGS_INTERNAL_PRODUCER_MAX_IN_FLIGHT: stalledLookup ? 2 : 1,+ }),+ teamManager+ )+ void send(producer, [entry()])+ producer.enqueue('hog_flow', [entry({ team_id: 3 })])++ const drain = producer.drain(1_000)+ await jest.advanceTimersByTimeAsync(1_000)++ await expect(drain).resolves.toBeUndefined()+ producer.enqueue('hog_flow', [entry()])+ await producer.drain(1_000)+ held.release()+ releaseLookup()+ await jest.advanceTimersByTimeAsync(10_000)++ expect(fetchMock).toHaveBeenCalledTimes(1)+ const metric = await register.getSingleMetric('logs_internal_producer_records_total')!.get()+ expect(Object.fromEntries(metric.values.map((v) => [v.labels.outcome, v.value]))).toEqual({ abandoned: 3 })+ expect(jest.getTimerCount()).toBe(0)+ }+ )++ it('splits a large team batch into requests under the capture-logs body limit', async () => {+ const message = 'x'.repeat(10_000)+ await send(+ new InternalLogsProducer(config(), teamManager),+ Array.from({ length: 250 }, () => entry({ message }))+ )++ const bodies = fetchMock.mock.calls.map(([, options]) => options!.body as string)+ expect(bodies.length).toBeGreaterThan(1)+ for (const body of bodies) {+ expect(Buffer.byteLength(body)).toBeLessThan(2 * 1024 * 1024)+ }+ const sent = bodies.map((body) => (parseJSON(body) as any).resourceLogs[0].scopeLogs[0].logRecords.length)+ expect(sent.reduce((a, b) => a + b, 0)).toEqual(250)+ })++ it('cuts a message too large for one request, so capture-logs still accepts the record', async () => {+ await send(new InternalLogsProducer(config(), teamManager), [entry({ message: '"'.repeat(3 * 1024 * 1024) })])++ expect(fetchMock).toHaveBeenCalledTimes(1)+ const body = fetchMock.mock.calls[0][1]!.body as string+ expect(Buffer.byteLength(body)).toBeLessThan(2 * 1024 * 1024)+ expect((parseJSON(body) as any).resourceLogs[0].scopeLogs[0].logRecords[0].body.stringValue).toEqual(+ '"'.repeat(128 * 1024)+ )+ })++ it('replaces an unpaired surrogate so capture-logs can decode the rest of the batch', async () => {+ await send(new InternalLogsProducer(config(), teamManager), [+ entry({ message: '😀'.substring(0, 1) }),+ entry({ message: 'valid' }),+ ])++ const body = fetchMock.mock.calls[0][1]!.body as string+ const messages = (parseJSON(body) as any).resourceLogs[0].scopeLogs[0].logRecords.map(+ (r: any) => r.body.stringValue+ )+ expect(messages).toEqual(['\uFFFD', 'valid'])+ })++ it.each(Array.from({ length: 20 }, (_, i) => i + 1))(+ 'keeps its accounting under random interleavings, seed %i',+ async (seed) => {+ register.resetMetrics()+ let state = seed+ const rand = (): number => {+ state = (state * 1_103_515_245 + 12_345) % 2 ** 31+ return state / 2 ** 31+ }+ const randomSpy = jest.spyOn(Math, 'random').mockImplementation(rand)+ const open: { resolve: (response: any) => void; reject: (error: Error) => void }[] = []+ fetchMock.mockImplementation(() => new Promise((resolve, reject) => open.push({ resolve, reject })))+ const cfg = config({+ LOGS_INTERNAL_PRODUCER_TEAMS: '*',+ LOGS_INTERNAL_PRODUCER_MAX_IN_FLIGHT: 1 + Math.floor(rand() * 3),+ LOGS_INTERNAL_PRODUCER_MAX_BUFFERED_BYTES: 2_000 + Math.floor(rand() * 20_000),+ LOGS_INTERNAL_PRODUCER_RETRY_TIMEOUT_MS: 5_000,+ })+ const producer = new InternalLogsProducer(cfg, teamManager)+ const internals = producer as any+ const counted = async (): Promise<number> => {+ const metric = await register.getSingleMetric('logs_internal_producer_records_total')!.get()+ return metric.values.reduce((sum, v) => sum + v.value, 0)+ }+ let enqueued = 0++ for (let step = 0; step < 150; step++) {+ const op = rand()+ if (op < 0.35) {+ const entries = Array.from({ length: 1 + Math.floor(rand() * 5) }, () =>+ entry({ team_id: 1 + Math.floor(rand() * 3), message: 'x'.repeat(Math.floor(rand() * 2_000)) })+ )+ producer.enqueue('hog_flow', entries)+ enqueued += entries.length+ } else if (op < 0.5) {+ producer.flush()+ } else if (op < 0.75 && open.length > 0) {+ const { resolve, reject } = open.splice(Math.floor(rand() * open.length), 1)[0]+ const status = [200, 200, 400, 503, 0][Math.floor(rand() * 5)]+ if (status === 0) {+ reject(new Error('reset'))+ } else {+ resolve({ status, dump: () => Promise.resolve() })+ }+ } else if (op < 0.97) {+ await jest.advanceTimersByTimeAsync(Math.floor(rand() * 3_000))+ } else {+ void producer.drain(Math.floor(rand() * 2_000))+ }+ await jest.advanceTimersByTimeAsync(0)++ const live = [...internals.waiting.values(), ...internals.inFlight]+ const recordBytes = live.flatMap((b) => b.records).reduce((sum, r) => sum + Buffer.byteLength(r), 0)+ expect(internals.bufferedBytes).toBe(recordBytes)+ expect(internals.bufferedBytes).toBeLessThanOrEqual(cfg.LOGS_INTERNAL_PRODUCER_MAX_BUFFERED_BYTES)+ expect(internals.inFlight.size).toBeLessThanOrEqual(cfg.LOGS_INTERNAL_PRODUCER_MAX_IN_FLIGHT)+ // A stopped producer already counted its in-flight records as abandoned.+ const held = internals.stopped ? 0 : live.reduce((sum, b) => sum + b.records.length, 0)+ expect((await counted()) + held).toBe(enqueued)+ }++ const drained = producer.drain(0)+ await jest.advanceTimersByTimeAsync(0)+ await drained+ open.splice(0).forEach(({ resolve }) => resolve(ok))+ await jest.advanceTimersByTimeAsync(60_000)+ expect(await counted()).toBe(enqueued)+ randomSpy.mockRestore()+ }+ )+})diff --git a/nodejs/src/logs/internal-logs-producer.ts b/nodejs/src/logs/internal-logs-producer.tsnew file mode 100644index 000000000000..a133cf2660d8--- /dev/null+++ b/nodejs/src/logs/internal-logs-producer.ts@@ -0,0 +1,322 @@+import { DateTime } from 'luxon'+import { Counter, Gauge } from 'prom-client'++import { buildIntegerMatcher } from '~/common/config/matchers'+import { logger } from '~/common/utils/logger'+import { internalFetch } from '~/common/utils/request'+import { sanitizeForUTF8 } from '~/common/utils/strings'+import type { TeamManager } from '~/common/utils/team-manager'+import { delay } from '~/common/utils/utils'+import type { ValueMatcher } from '~/types'++import type { LogsIngestionConsumerConfig } from './config'+import { INTERNAL_LOGS_HTTP_HEADER, INTERNAL_SOURCE_POLICIES, signInternalLogs } from './internal-logs'++const SERVICE_NAMES: Readonly<Record<string, string>> = {+ hog_flow: 'posthog-internal-workflows',+}++const SEVERITY_NUMBERS: Readonly<Record<string, number>> = { debug: 5, info: 9, warn: 13, error: 17 }++// Half of capture-logs' default 2 MiB body limit, which leaves room for the envelope.+const MAX_REQUEST_RECORD_BYTES = 1024 * 1024++// A JSON-escaped character takes at most 6 bytes, so a message cut to this length always fits in one request.+const MAX_MESSAGE_LENGTH = 128 * 1024++export type InternalLogEntry = {+ team_id: number+ log_source_id: string+ instance_id: string+ timestamp: DateTime+ level: string+ message: string+}++export type InternalLogsProducerConfig = Pick<+ LogsIngestionConsumerConfig,+ | 'LOGS_INTERNAL_INGESTION_SIGNING_KEYS'+ | 'LOGS_INTERNAL_PRODUCER_URL'+ | 'LOGS_INTERNAL_PRODUCER_TEAMS'+ | 'LOGS_INTERNAL_PRODUCER_MAX_IN_FLIGHT'+ | 'LOGS_INTERNAL_PRODUCER_MAX_BUFFERED_BYTES'+ | 'LOGS_INTERNAL_PRODUCER_RETRY_TIMEOUT_MS'+>++type Batch = { source: string; teamId: number; records: string[] }++type Request = Batch & { bytes: number }++type AttemptResult = { outcome: 'sent' | 'rejected' | 'unknown_team' | 'retry'; detail: string }++const RETRY_BASE_DELAY_MS = 100+const RETRY_MAX_DELAY_MS = 5_000+const DRAIN_POLL_MS = 50++const internalLogsProducerRecordsCounter = new Counter({+ name: 'logs_internal_producer_records_total',+ help: 'Internal log records sent to capture-logs, by source and outcome. Only a sent record reaches the logs product.',+ labelNames: ['source', 'outcome'],+})++const internalLogsProducerBufferedBytesGauge = new Gauge({+ name: 'logs_internal_producer_buffered_bytes',+ help: 'Bytes of internal log records a producer holds in memory, queued, waiting or in flight.',+})++const internalLogsProducerInFlightGauge = new Gauge({+ name: 'logs_internal_producer_in_flight_requests',+ help: 'Requests a producer has open to capture-logs.',+})++function countRecords(source: string, outcome: string, records: number): void {+ if (records > 0) {+ internalLogsProducerRecordsCounter.inc({ source, outcome }, records)+ }+}++/**+ * Delivery runs in the background, so a slow or down capture-logs never slows the caller. A record is lost only when+ * it is invalid, the buffer is full, its retries time out, capture-logs rejects it, or `drain` times out.+ */+export class InternalLogsProducer {+ private readonly enabled: boolean+ private readonly isTeamEnabled: ValueMatcher<number>+ // One batch per source and team, so records that wait for a free request slot leave together in large requests.+ private waiting = new Map<string, Batch>()+ // Counts waiting and in-flight records, because each one holds its bytes until its request settles.+ private bufferedBytes = 0+ private inFlight = new Set<Request>()+ // Set when `drain` ends. After it, nothing is sent or counted twice, and a retrying request stops.+ private stopped = false++ constructor(+ private readonly config: InternalLogsProducerConfig,+ private readonly teamManager: Pick<TeamManager, 'getTeam'>+ ) {+ this.enabled =+ config.LOGS_INTERNAL_PRODUCER_URL.length > 0 &&+ config.LOGS_INTERNAL_INGESTION_SIGNING_KEYS.trim().length > 0+ this.isTeamEnabled = buildIntegerMatcher(config.LOGS_INTERNAL_PRODUCER_TEAMS, true)+ }++ enqueue(source: string, entries: InternalLogEntry[]): void {+ if (+ !this.enabled ||+ !Object.hasOwn(INTERNAL_SOURCE_POLICIES, source) ||+ !Object.hasOwn(SERVICE_NAMES, source)+ ) {+ return+ }+ let invalid = 0+ let dropped = 0+ let abandoned = 0+ for (const entry of entries) {+ if (!this.isTeamEnabled(entry.team_id)) {+ continue+ }+ if (this.stopped) {+ abandoned+++ continue+ }+ let record: string+ try {+ record = JSON.stringify(toOtlpLogRecord(entry))+ } catch {+ invalid+++ continue+ }+ const bytes = Buffer.byteLength(record)+ if (this.bufferedBytes + bytes > this.config.LOGS_INTERNAL_PRODUCER_MAX_BUFFERED_BYTES) {+ dropped+++ continue+ }+ this.bufferedBytes += bytes+ const key = `${source}:${entry.team_id}`+ const batch = this.waiting.get(key) ?? { source, teamId: entry.team_id, records: [] }+ batch.records.push(record)+ this.waiting.set(key, batch)+ }+ countRecords(source, 'invalid', invalid)+ countRecords(source, 'dropped', dropped)+ countRecords(source, 'abandoned', abandoned)+ this.report()+ }++ /** Starts sending the waiting records. Never throws. */+ flush(): void {+ this.pump()+ }++ /** Sends what is left until the timeout, then stops and counts the rest as abandoned. Call on shutdown. */+ async drain(timeoutMs: number): Promise<void> {+ const deadline = Date.now() + timeoutMs+ this.pump()+ while (!this.stopped && !this.isIdle() && Date.now() < deadline) {+ await delay(Math.min(DRAIN_POLL_MS, deadline - Date.now()))+ }+ if (!this.stopped && !this.isIdle()) {+ this.abandon()+ }+ this.stopped = true+ }++ private abandon(): void {+ this.stopped = true+ const unsent = [...this.waiting.values(), ...this.inFlight]+ for (const { source, records } of unsent) {+ countRecords(source, 'abandoned', records.length)+ }+ logger.warn('Internal logs producer stopped before all records were sent', {+ abandonedRecords: unsent.reduce((sum, { records }) => sum + records.length, 0),+ inFlightRequests: this.inFlight.size,+ })+ this.waiting.clear()+ this.bufferedBytes = [...this.inFlight].reduce((sum, request) => sum + request.bytes, 0)+ this.report()+ }++ private pump(): void {+ while (this.inFlight.size < this.config.LOGS_INTERNAL_PRODUCER_MAX_IN_FLIGHT && this.waiting.size > 0) {+ const [key, batch] = this.waiting.entries().next().value!+ // Moves the rest of the batch to the back, so each team gets its turn.+ this.waiting.delete(key)+ const request = takeRequest(batch)+ if (batch.records.length > 0) {+ this.waiting.set(key, batch)+ }+ this.inFlight.add(request)+ void this.deliver(request).finally(() => {+ this.inFlight.delete(request)+ this.bufferedBytes -= request.bytes+ this.pump()+ })+ }+ this.report()+ }++ private isIdle(): boolean {+ return this.inFlight.size === 0 && this.waiting.size === 0+ }++ private report(): void {+ internalLogsProducerBufferedBytesGauge.set(this.bufferedBytes)+ internalLogsProducerInFlightGauge.set(this.inFlight.size)+ }++ private async deliver({ source, teamId, records }: Batch): Promise<void> {+ const body = toOtlpBody(source, records)+ const deadline = Date.now() + this.config.LOGS_INTERNAL_PRODUCER_RETRY_TIMEOUT_MS+ for (let attempt = 0; !this.stopped; attempt++) {+ const result = await this.attempt(source, teamId, body, deadline)+ if (this.stopped) {+ return+ }+ if (result.outcome !== 'retry') {+ countRecords(source, result.outcome, records.length)+ if (result.outcome === 'rejected') {+ logger.warn('Internal logs batch rejected by capture-logs', {+ source,+ teamId,+ status: result.detail,+ })+ }+ return+ }+ const wait = Math.random() * Math.min(RETRY_MAX_DELAY_MS, RETRY_BASE_DELAY_MS * 2 ** attempt)+ if (Date.now() + wait > deadline) {+ countRecords(source, 'failed', records.length)+ logger.warn('Internal logs batch failed after retries', {+ source,+ teamId,+ attempts: attempt + 1,+ last: result.detail,+ })+ return+ }+ await delay(wait)+ }+ }++ private async attempt(source: string, teamId: number, body: string, deadline: number): Promise<AttemptResult> {+ let timer: NodeJS.Timeout | undefined+ try {+ // A team lookup has no timeout of its own, so a stalled Postgres would hold this request slot past the deadline.+ const team = await Promise.race([+ this.teamManager.getTeam(teamId),+ new Promise<never>((_, reject) => {+ timer = setTimeout(() => reject(new Error('team lookup timed out')), deadline - Date.now())+ }),+ ])+ if (this.stopped) {+ return { outcome: 'retry', detail: 'stopped' }+ }+ if (!team) {+ return { outcome: 'unknown_team', detail: String(teamId) }+ }+ const response = await internalFetch(this.config.LOGS_INTERNAL_PRODUCER_URL, {+ method: 'POST',+ headers: {+ 'Content-Type': 'application/json',+ Authorization: `Bearer ${team.api_token}`,+ [INTERNAL_LOGS_HTTP_HEADER]: signInternalLogs(+ this.config.LOGS_INTERNAL_INGESTION_SIGNING_KEYS,+ source,+ teamId+ ),+ },+ body,+ })+ await response.dump()+ const { status } = response+ if (status >= 200 && status < 300) {+ return { outcome: 'sent', detail: String(status) }+ }+ // Any other 4xx is a bad request, which fails the same way on every attempt.+ const retriable = status >= 500 || status === 408 || status === 429+ return { outcome: retriable ? 'retry' : 'rejected', detail: String(status) }+ } catch (error) {+ return { outcome: 'retry', detail: String(error) }+ } finally {+ clearTimeout(timer)+ }+ }+}++type OtlpLogRecord = Record<string, unknown>++function toOtlpLogRecord(entry: InternalLogEntry): OtlpLogRecord {+ return {+ timeUnixNano: (BigInt(entry.timestamp.toMillis()) * 1_000_000n).toString(),+ severityText: entry.level.toUpperCase(),+ severityNumber: SEVERITY_NUMBERS[entry.level] ?? 0,+ // capture-logs rejects a whole request that holds one unpaired surrogate. The cut can split a pair, so sanitize after it.+ body: { stringValue: sanitizeForUTF8(entry.message.slice(0, MAX_MESSAGE_LENGTH)) },+ attributes: [+ { key: 'log_source_id', value: { stringValue: entry.log_source_id } },+ { key: 'instance_id', value: { stringValue: entry.instance_id } },+ ],+ }+}++/** Removes up to MAX_REQUEST_RECORD_BYTES of records from the front of the batch, and at least one. */+function takeRequest(batch: Batch): Request {+ let count = 0+ let bytes = 0+ while (count < batch.records.length) {+ const recordBytes = Buffer.byteLength(batch.records[count])+ if (count > 0 && bytes + recordBytes > MAX_REQUEST_RECORD_BYTES) {+ break+ }+ bytes += recordBytes+ count+++ }+ return { source: batch.source, teamId: batch.teamId, records: batch.records.splice(0, count), bytes }+}++function toOtlpBody(source: string, records: string[]): string {+ const resource = JSON.stringify({+ attributes: [{ key: 'service.name', value: { stringValue: SERVICE_NAMES[source] } }],+ })+ return `{"resourceLogs":[{"resource":${resource},"scopeLogs":[{"scope":${JSON.stringify({ name: source })},"logRecords":[${records.join(',')}]}]}]}`+}diff --git a/nodejs/src/logs/internal-logs.ts b/nodejs/src/logs/internal-logs.tsindex 33f9c492000e..86108a119fe8 100644--- a/nodejs/src/logs/internal-logs.ts+++ b/nodejs/src/logs/internal-logs.ts@@ -7,6 +7,9 @@ import type { ValueMatcher } from '~/types'export const INTERNAL_LOGS_HEADER = 'posthog-internal'+// The HTTP request header that capture-logs forwards as the `posthog-internal` Kafka header.+export const INTERNAL_LOGS_HTTP_HEADER = 'x-posthog-internal'+export const INTERNAL_SOURCE_HEADER = 'internal-source'const SIGNATURE_VERSION = 'v1'