:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(logs): store signed internal log batches outside billing (#121927)
Jon McCallum65b0905eae86
diff --git a/nodejs/src/logs/config.ts b/nodejs/src/logs/config.tsindex f84a4f05e705..2ccd0bc0815b 100644--- a/nodejs/src/logs/config.ts+++ b/nodejs/src/logs/config.ts@@ -67,6 +67,9 @@ export type LogsIngestionConsumerConfig = {LOGS_BACKFILL_ENABLED_TEAMS: string/** When `true`, retention rules are never evaluated (rows keep the team default via the batch header). */LOGS_RETENTION_KILLSWITCH: boolean+ /** Comma-separated team IDs, or `*` for all teams. A `posthog-internal` batch from any other team is billed as normal. */+ LOGS_INTERNAL_INGESTION_ENABLED_TEAMS: string+ LOGS_INTERNAL_INGESTION_SIGNING_KEYS: string/** Comma-separated team IDs, or `*` for all teams, or empty (default) to disable measure-only pattern masking. */LOGS_PATTERN_MASKING_ENABLED_TEAMS: stringLOGS_JSON_ATTRIBUTE_PARSING_ENABLED_TEAMS: string@@ -123,6 +126,8 @@ export function getDefaultLogsIngestionConsumerConfig(): LogsIngestionConsumerCoLOGS_RETENTION_ENABLED_TEAMS: isProdEnv() ? '' : '*',LOGS_BACKFILL_ENABLED_TEAMS: isProdEnv() ? '' : '*',LOGS_RETENTION_KILLSWITCH: false,+ LOGS_INTERNAL_INGESTION_ENABLED_TEAMS: '',+ LOGS_INTERNAL_INGESTION_SIGNING_KEYS: '',// Off by default: enabling forces decode+re-encode for allowlisted teams.LOGS_PATTERN_MASKING_ENABLED_TEAMS: '',LOGS_JSON_ATTRIBUTE_PARSING_ENABLED_TEAMS: '',diff --git a/nodejs/src/logs/ingestion-otel-metrics.test.ts b/nodejs/src/logs/ingestion-otel-metrics.test.tsindex 306a3ca222bf..f6b81c727219 100644--- a/nodejs/src/logs/ingestion-otel-metrics.test.ts+++ b/nodejs/src/logs/ingestion-otel-metrics.test.ts@@ -8,6 +8,7 @@ import {} from '@opentelemetry/sdk-metrics'import {+ recordInternalLogs,recordJsonEnrichmentSkipped,recordLogMessageDlq,recordLogMessageDropped,@@ -89,6 +90,20 @@ describe('ingestion-otel-metrics', () => {{ team_id: '42' },10,],+ [+ 'recordInternalLogs bytes',+ 'logs_ingestion_internal_bytes_total',+ () => recordInternalLogs(2, 4096, 20),+ { team_id: '2' },+ 4096,+ ],+ [+ 'recordInternalLogs records',+ 'logs_ingestion_internal_records_total',+ () => recordInternalLogs(2, 4096, 20),+ { team_id: '2' },+ 20,+ ],['recordLogMessageDropped','logs_ingestion_message_dropped_count',diff --git a/nodejs/src/logs/ingestion-otel-metrics.ts b/nodejs/src/logs/ingestion-otel-metrics.tsindex 0ba34752f381..a9d65eb25049 100644--- a/nodejs/src/logs/ingestion-otel-metrics.ts+++ b/nodejs/src/logs/ingestion-otel-metrics.ts@@ -22,6 +22,8 @@ interface LogsIngestionInstruments {recordsAllowed: CounterbytesDropped: CounterrecordsDropped: Counter+ internalBytes: Counter+ internalRecords: CountermessagesDropped: CountermessagesDlq: CounterprocessingDuration: Histogram@@ -61,6 +63,14 @@ function getInstruments(): LogsIngestionInstruments {recordsDropped: createCounterWithExemplars(meter, 'logs_ingestion_records_dropped_total', {description: 'Total log records dropped due to quota or rate limiting',}),+ internalBytes: createCounterWithExemplars(meter, 'logs_ingestion_internal_bytes_total', {+ description:+ 'Uncompressed bytes of verified internal logs, which stay out of the received and allowed counters',+ unit: 'By',+ }),+ internalRecords: createCounterWithExemplars(meter, 'logs_ingestion_internal_records_total', {+ description: 'Records of verified internal logs, which stay out of the received and allowed counters',+ }),messagesDropped: createCounterWithExemplars(meter, 'logs_ingestion_message_dropped_count', {description: 'The number of logs ingestion messages dropped',}),@@ -106,6 +116,13 @@ export const recordLogsDropped = swallowing((teamId: number, bytes: number, recoaddPositive(recordsDropped, records, attributes)})+export const recordInternalLogs = swallowing((teamId: number, bytes: number, records: number): void => {+ const { internalBytes, internalRecords } = getInstruments()+ const attributes = { team_id: teamId.toString() }+ addPositive(internalBytes, bytes, attributes)+ addPositive(internalRecords, records, attributes)+})+export const recordLogMessageDropped = swallowing((reason: string, teamId: string, count: number = 1): void => {addPositive(getInstruments().messagesDropped, count, { reason, team_id: teamId })})diff --git a/nodejs/src/logs/internal-logs.test.ts b/nodejs/src/logs/internal-logs.test.tsnew file mode 100644index 000000000000..11e8a00507f3--- /dev/null+++ b/nodejs/src/logs/internal-logs.test.ts@@ -0,0 +1,74 @@+import { InternalLogsVerifier, signInternalLogs } from './internal-logs'++const KEY = 'internal-logs-test-key'+const OTHER_KEY = 'internal-logs-other-key'+const SOURCE = 'hog_flow'+const TEAM_ID = 2+const OTHER_TEAM_ID = 3+const PAYLOAD = Buffer.from('avro container bytes')++describe('InternalLogsVerifier', () => {+ it.each([+ {+ name: 'accepts a batch signed with the newest key',+ signature: signInternalLogs(KEY, SOURCE, TEAM_ID, PAYLOAD),+ },+ {+ name: 'accepts a batch signed with an older key during rotation',+ keys: `${OTHER_KEY},${KEY}`,+ signature: signInternalLogs(KEY, SOURCE, TEAM_ID, PAYLOAD),+ },+ {+ name: 'accepts a batch on a wildcard allowlist',+ teams: '*',+ signature: signInternalLogs(KEY, SOURCE, TEAM_ID, PAYLOAD),+ },+ ])('$name', ({ signature, keys = KEY, teams = String(TEAM_ID) }) => {+ expect(new InternalLogsVerifier(keys, teams).verify(signature, TEAM_ID, PAYLOAD)).toEqual({+ source: SOURCE,+ billed: false,+ teamRules: false,+ })+ })++ it.each([+ { name: 'no header', signature: undefined },+ { name: 'a malformed signature', signature: 'v1.not-a-signature' },+ { name: 'an unknown key', signature: signInternalLogs(OTHER_KEY, SOURCE, TEAM_ID, PAYLOAD) },+ { name: "another team's signature", signature: signInternalLogs(KEY, SOURCE, OTHER_TEAM_ID, PAYLOAD) },+ {+ name: 'a signature moved to another source',+ signature: signInternalLogs(KEY, SOURCE, TEAM_ID, PAYLOAD).replace(`.${SOURCE}.`, '.hog_function.'),+ },+ {+ name: 'a signature over different data',+ signature: signInternalLogs(KEY, SOURCE, TEAM_ID, Buffer.from('other')),+ },+ {+ name: 'a team not on the allowlist',+ teams: String(OTHER_TEAM_ID),+ signature: signInternalLogs(KEY, SOURCE, TEAM_ID, PAYLOAD),+ },+ { name: 'no configured key', keys: '', signature: signInternalLogs(KEY, SOURCE, TEAM_ID, PAYLOAD) },+ { name: 'a source with no policy', signature: signInternalLogs(KEY, 'not_a_source', TEAM_ID, PAYLOAD) },+ { name: 'an inherited property as source', signature: signInternalLogs(KEY, 'constructor', TEAM_ID, PAYLOAD) },+ { name: 'the prototype as source', signature: signInternalLogs(KEY, '__proto__', TEAM_ID, PAYLOAD) },+ { name: 'an empty payload', payload: null, signature: signInternalLogs(KEY, SOURCE, TEAM_ID, PAYLOAD) },+ ])(+ 'bills a batch with $name',+ ({ signature, keys = KEY, teams = String(TEAM_ID), payload = PAYLOAD as Buffer | null }) => {+ expect(new InternalLogsVerifier(keys, teams).verify(signature, TEAM_ID, payload)).toBeUndefined()+ }+ )++ it('refuses to sign a source that cannot go into the header', () => {+ expect(() => signInternalLogs(KEY, 'hog.flow', TEAM_ID, PAYLOAD)).toThrow('invalid source')+ })++ // Producers and consumers deploy separately, so a format change must be a new version prefix.+ it('keeps the v1 signature format stable', () => {+ expect(signInternalLogs(KEY, SOURCE, TEAM_ID, PAYLOAD)).toEqual(+ 'v1.hog_flow.b97466e15a9bf1a2658e8d92e5f63ed60df91bd4e838e4ab4426cbb5280d2055'+ )+ })+})diff --git a/nodejs/src/logs/internal-logs.ts b/nodejs/src/logs/internal-logs.tsnew file mode 100644index 000000000000..911cbddf9f19--- /dev/null+++ b/nodejs/src/logs/internal-logs.ts@@ -0,0 +1,130 @@+import { createHash, createHmac, timingSafeEqual } from 'crypto'+import { Counter } from 'prom-client'++import { buildIntegerMatcher } from '~/common/config/matchers'+import type { ValueMatcher } from '~/types'++export const INTERNAL_LOGS_HEADER = 'posthog-internal'++export const INTERNAL_SOURCE_HEADER = 'internal-source'++const SIGNATURE_VERSION = 'v1'++// No dot, because a dot divides the parts of the header value.+const SOURCE_PATTERN = /^[a-z0-9_]{1,64}$/++export type InternalSourcePolicy = {+ /** Counts toward the team's logs usage, quota and rate limits. */+ billed: boolean+ /** Runs the team's sampling, drop, transformation, metric and retention rules. */+ teamRules: boolean+}++export type InternalLogs = InternalSourcePolicy & { source: string }++// A signer names the source and this map decides how it is handled, so a leaked key cannot invent+// a free source. Moving a source to billing changes its entry here, not the producer or the header.+export const INTERNAL_SOURCE_POLICIES: Readonly<Record<string, InternalSourcePolicy>> = {+ hog_flow: { billed: false, teamRules: false },+ // Each log_entries source moves unbilled, one at a time. Add a source when its producer starts signing.+ // hog_function: { billed: false, teamRules: false },+}++type RejectReason = 'not_configured' | 'team_not_allowed' | 'invalid_signature' | 'unknown_source'++const logsInternalLogsCounter = new Counter({+ name: 'logs_ingestion_internal_logs_total',+ help: 'Batches that carried the posthog-internal header, by outcome. A rejected batch is billed as normal.',+ labelNames: ['outcome', 'reason'],+})++function parseKeys(commaSeparatedKeys: string): string[] {+ return commaSeparatedKeys+ .split(',')+ .map((key) => key.trim())+ .filter((key) => key.length > 0)+}++function sha256Hex(payload: Buffer): string {+ return createHash('sha256').update(payload).digest('hex')+}++// No expiry, because a batch that waits in Kafka through consumer lag or a DLQ replay must stay internal.+function computeSignature(key: string, source: string, teamId: number, payloadSha256: string): string {+ const mac = createHmac('sha256', key)+ .update(`${SIGNATURE_VERSION}|${source}|${teamId}|${payloadSha256}`)+ .digest('hex')+ return `${SIGNATURE_VERSION}.${source}.${mac}`+}++export function signInternalLogs(commaSeparatedKeys: string, source: string, teamId: number, payload: Buffer): string {+ const [newestKey] = parseKeys(commaSeparatedKeys)+ if (!newestKey) {+ throw new Error('Cannot sign posthog-internal batch: no signing key configured')+ }+ if (!SOURCE_PATTERN.test(source)) {+ throw new Error(`Cannot sign posthog-internal batch: invalid source ${JSON.stringify(source)}`)+ }+ return computeSignature(newestKey, source, teamId, sha256Hex(payload))+}++// Only a verified batch may set the `internal_source` column, so an incoming copy of the header is removed.+export function withoutInternalLogsHeaders(headers: Record<string, string>): Record<string, string> {+ const { [INTERNAL_LOGS_HEADER]: _signature, [INTERNAL_SOURCE_HEADER]: _source, ...rest } = headers+ return rest+}++export class InternalLogsVerifier {+ private readonly keys: string[]+ private readonly isTeamAllowed: ValueMatcher<number>++ constructor(+ commaSeparatedKeys: string,+ enabledTeams: string,+ private readonly policies: Readonly<Record<string, InternalSourcePolicy>> = INTERNAL_SOURCE_POLICIES+ ) {+ this.keys = parseKeys(commaSeparatedKeys)+ this.isTeamAllowed = buildIntegerMatcher(enabledTeams, true)+ }++ verify(signature: string | undefined, teamId: number, payload: Buffer | null): InternalLogs | undefined {+ if (signature === undefined) {+ return undefined+ }+ const source = signature.split('.')[1] ?? ''+ const reason = this.rejectReason(signature, source, teamId, payload)+ // Own keys only, because a source like `constructor` resolves to an inherited policy with no `billed`.+ const policy = reason || !Object.hasOwn(this.policies, source) ? undefined : this.policies[source]+ if (!policy) {+ logsInternalLogsCounter.inc({ outcome: 'rejected', reason: reason ?? 'unknown_source' })+ return undefined+ }+ logsInternalLogsCounter.inc({ outcome: 'accepted', reason: '' })+ return { source, ...policy }+ }++ private rejectReason(+ signature: string,+ source: string,+ teamId: number,+ payload: Buffer | null+ ): RejectReason | null {+ if (this.keys.length === 0) {+ return 'not_configured'+ }+ if (!this.isTeamAllowed(teamId)) {+ return 'team_not_allowed'+ }+ if (payload === null || !SOURCE_PATTERN.test(source)) {+ return 'invalid_signature'+ }+ const received = Buffer.from(signature)+ const payloadSha256 = sha256Hex(payload)+ // Every key is tried, so a rotation can add a key before producers switch to it.+ const matches = this.keys.some((key) => {+ const expected = Buffer.from(computeSignature(key, source, teamId, payloadSha256))+ return expected.length === received.length && timingSafeEqual(expected, received)+ })+ return matches ? null : 'invalid_signature'+ }+}diff --git a/nodejs/src/logs/logs-ingestion-consumer.test.ts b/nodejs/src/logs/logs-ingestion-consumer.test.tsindex b777f96f44b1..50254b69275d 100644--- a/nodejs/src/logs/logs-ingestion-consumer.test.ts+++ b/nodejs/src/logs/logs-ingestion-consumer.test.ts@@ -28,6 +28,7 @@ import { Hub, Team } from '~/types'import { getDefaultTracesIngestionConsumerConfig } from './config'import * as otelMetrics from './ingestion-otel-metrics'import { resetLogsIngestionInstrumentsForTests } from './ingestion-otel-metrics'+import { INTERNAL_LOGS_HEADER, INTERNAL_SOURCE_HEADER, signInternalLogs } from './internal-logs'import { logsPatternBodyKindCounter } from './log-pattern-stage'import { LogRecord, decodeLogRecords, encodeLogRecords, logsJsonAttributeSniffCounter } from './log-record-avro'import {@@ -38,6 +39,7 @@ import {logMessageDlqCounter,logMessageDroppedCounter,logsBillingBytesCreditedCounter,+ logsBillingRecordsCreditedCounter,logsBytesAllowedCounter,logsBytesDroppedCounter,logsBytesReceivedCounter,@@ -157,10 +159,11 @@ const createMultiRecordKafkaMessage = async (const logRecordType = avro.Type.forSchema({type: 'record',name: 'LogRecord',- fields: LOG_RECORD_BASE_FIELDS,+ fields: [...LOG_RECORD_BASE_FIELDS, { name: 'retention_days', type: ['null', 'int'], default: null }],})const records: LogRecord[] = logDataList.map((logData, i) => ({+ retention_days: logData.retentionDays ?? null,uuid: `test-uuid-${offsetIncrementer}-${i}`,trace_id: null,span_id: null,@@ -213,6 +216,7 @@ describe('LogsIngestionConsumer', () => {| 'metricsEmitter'| 'logsTransformer'| 'patternMessageKeysCache'+ | 'internalSourcePolicies'>> = {}) => {@@ -1037,6 +1041,178 @@ describe('LogsIngestionConsumer', () => {})})+ describe('internal logs', () => {+ const SIGNING_KEY = 'internal-logs-test-key'++ it.each([+ {+ name: "stores a signed batch outside quota, billing and the team's drop rules",+ signed: true,+ billed: false,+ },+ { name: 'bills a batch whose signature does not verify', signed: false, billed: true },+ {+ name: "bills a signed batch from a billed source but skips the team's drop rules",+ signed: true,+ billed: true,+ },+ ])('$name', async ({ signed, billed }) => {+ await consumer.stop()+ const dropEverything: Pick<SamplingRulesCache, 'getCompiledRuleSet'> = {+ getCompiledRuleSet: () =>+ Promise.resolve(+ compileRuleSet([+ {+ id: '00000000-0000-0000-0000-0000000000aa',+ rule_type: 'severity_sampling',+ scope_service: 'test-service',+ scope_path_pattern: null,+ scope_attribute_filters: [],+ config: { actions: { INFO: { type: 'drop' } } },+ },+ ])+ ),+ }+ consumer = await createLogsIngestionConsumer(+ hub,+ {+ LOGS_INTERNAL_INGESTION_ENABLED_TEAMS: String(team.id),+ LOGS_INTERNAL_INGESTION_SIGNING_KEYS: SIGNING_KEY,+ },+ {+ samplingRulesCache: dropEverything as SamplingRulesCache,+ internalSourcePolicies: { hog_flow: { billed, teamRules: false } },+ }+ )+ // A quota limit drops every billed batch, so it applies only to the rows that test the internal bypass.+ jest.mocked(hub.quotaLimiting.isTeamTokenQuotaLimited).mockResolvedValue(!(signed && billed))++ const [message] = await createKafkaMessages([createLogMessage()], {+ token: team.api_token,+ bytes_uncompressed: '100',+ record_count: '1',+ })+ const signature = signed+ ? signInternalLogs(SIGNING_KEY, 'hog_flow', team.id, message.value!)+ : 'v1.hog_flow.not-a-signature'+ message.headers!.push({ [INTERNAL_LOGS_HEADER]: Buffer.from(signature) })+ const recordLogsReceivedSpy = jest.spyOn(otelMetrics, 'recordLogsReceived')+ const recordInternalLogsSpy = jest.spyOn(otelMetrics, 'recordInternalLogs')++ await waitForBackgroundTasks(consumer.processKafkaBatch([message]))++ const metricNames = mockProducerObserver+ .getProducedKafkaMessagesForTopic(KAFKA_APP_METRICS_2)+ .filter((m) => m.value.team_id === team.id)+ .map((m) => m.value.metric_name)+ const produced = getProducedKafkaMessages().filter((m) => m.topic === KAFKA_LOGS_CLICKHOUSE)+ expect({+ produced: produced.length,+ forwardsSignature: produced.some((m) => INTERNAL_LOGS_HEADER in (m.headers ?? {})),+ internalSource: produced[0]?.headers?.[INTERNAL_SOURCE_HEADER],+ received: recordLogsReceivedSpy.mock.calls,+ internal: recordInternalLogsSpy.mock.calls,+ internalMetrics: ['bytes_ingested_internal', 'records_ingested_internal'].every((n) =>+ metricNames.includes(n)+ ),+ billedMetrics: ['bytes_ingested', 'records_ingested', 'bytes_received'].some((n) =>+ metricNames.includes(n)+ ),+ }).toEqual({+ produced: signed ? 1 : 0,+ forwardsSignature: false,+ internalSource: signed ? 'hog_flow' : undefined,+ received: [billed ? [100, 1] : [0, 0]],+ internal: billed ? [] : [[team.id, 100, 1]],+ internalMetrics: !billed,+ billedMetrics: billed,+ })+ })++ it('drops a row past its stamped retention without crediting billing', async () => {+ await consumer.stop()+ consumer = await createLogsIngestionConsumer(hub, {+ LOGS_INTERNAL_INGESTION_ENABLED_TEAMS: String(team.id),+ LOGS_INTERNAL_INGESTION_SIGNING_KEYS: SIGNING_KEY,+ })+ const fiveDaysAgoMicros = DateTime.now().minus({ days: 5 }).toMillis() * 1000+ const message = await createMultiRecordKafkaMessage(+ [{ level: 'info', timestampMicros: fiveDaysAgoMicros, retentionDays: 1 }, { level: 'info' }],+ {+ token: team.api_token,+ bytes_uncompressed: '200',+ record_count: '2',+ min_timestamp: fiveDaysAgoMicros.toString(),+ }+ )+ message.headers!.push({+ [INTERNAL_LOGS_HEADER]: Buffer.from(signInternalLogs(SIGNING_KEY, 'hog_flow', team.id, message.value!)),+ })+ const bytesCreditSpy = jest.spyOn(logsBillingBytesCreditedCounter, 'inc')+ const recordsCreditSpy = jest.spyOn(logsBillingRecordsCreditedCounter, 'inc')++ await waitForBackgroundTasks(consumer.processKafkaBatch([message]))++ const expiredMetric = mockProducerObserver+ .getProducedKafkaMessagesForTopic(KAFKA_APP_METRICS_2)+ .map((m) => m.value)+ .find((v) => v.metric_name === 'records_dropped_retention_expired' && v.team_id === team.id)+ expect({+ expired: expiredMetric?.count,+ produced: getProducedKafkaMessages().filter((m) => m.topic === KAFKA_LOGS_CLICKHOUSE).length,+ credits: [...bytesCreditSpy.mock.calls, ...recordsCreditSpy.mock.calls],+ }).toEqual({ expired: 1, produced: 1, credits: [] })+ })++ it.each([+ { name: 'drops a signed backfill without writing to the team usage row', backfill: true },+ { name: 'keeps the usage of a failed internal batch out of the next flush', backfill: false },+ ])('$name', async ({ backfill }) => {+ await consumer.stop()+ consumer = await createLogsIngestionConsumer(hub, {+ LOGS_BACKFILL_ENABLED_TEAMS: String(team2.id),+ LOGS_INTERNAL_INGESTION_ENABLED_TEAMS: String(team.id),+ LOGS_INTERNAL_INGESTION_SIGNING_KEYS: SIGNING_KEY,+ })+ const [internal] = await createKafkaMessages([createLogMessage()], {+ token: team.api_token,+ bytes_uncompressed: '100',+ record_count: '1',+ ...(backfill ? { backfill_days: '540' } : {}),+ })+ internal.headers!.push({+ [INTERNAL_LOGS_HEADER]: Buffer.from(+ signInternalLogs(SIGNING_KEY, 'hog_flow', team.id, internal.value!)+ ),+ })+ const originalQueueMessages = mockProducer.queueMessages+ const passthrough = originalQueueMessages.bind(mockProducer)+ mockProducer.queueMessages = jest.fn().mockImplementation((topicMessages) => {+ if (topicMessages.topic === KAFKA_LOGS_CLICKHOUSE && !backfill) {+ throw new DependencyUnavailableError('broker down', 'Kafka', new Error('broker down'))+ }+ return passthrough(topicMessages)+ })++ try {+ await consumer.processKafkaBatch([internal]).then((r) => r.backgroundTask?.catch(() => undefined))+ mockProducer.queueMessages = originalQueueMessages+ // A later batch of another team flushes the shared usage aggregator.+ const [next] = await createKafkaMessages([createLogMessage()], { token: team2.api_token })+ await waitForBackgroundTasks(consumer.processKafkaBatch([next]))+ } finally {+ mockProducer.queueMessages = originalQueueMessages+ }++ expect(+ mockProducerObserver+ .getProducedKafkaMessagesForTopic(KAFKA_APP_METRICS_2)+ .filter((m) => m.value.team_id === team.id)+ .map((m) => m.value.metric_name)+ ).toEqual([])+ })+ })+describe('filterQuotaLimitedMessages', () => {it('should allow messages when not quota limited', async () => {jest.mocked(hub.quotaLimiting.isTeamTokenQuotaLimited).mockResolvedValue(false)@@ -2345,6 +2521,41 @@ describe('LogsIngestionConsumer', () => {])})+ it('emits nothing when a later internal message fails the batch', async () => {+ const key = 'internal-logs-test-key'+ await createConsumerWithMetricRules({+ LOGS_INTERNAL_INGESTION_ENABLED_TEAMS: String(team.id),+ LOGS_INTERNAL_INGESTION_SIGNING_KEYS: key,+ })+ const [billed, internal] = await createKafkaMessages([createLogMessage(), createLogMessage()], {+ token: team.api_token,+ })+ internal.headers!.push({+ [INTERNAL_LOGS_HEADER]: Buffer.from(signInternalLogs(key, 'hog_flow', team.id, internal.value!)),+ })++ const originalQueueMessages = mockProducer.queueMessages+ const passthrough = originalQueueMessages.bind(mockProducer)+ mockProducer.queueMessages = jest.fn().mockImplementation((topicMessages) => {+ const isInternal = topicMessages.messages?.some(+ (m: { headers?: Record<string, string> }) => m.headers?.[INTERNAL_SOURCE_HEADER]+ )+ if (topicMessages.topic === KAFKA_LOGS_CLICKHOUSE && isInternal) {+ throw new DependencyUnavailableError('broker down', 'Kafka', new Error('broker down'))+ }+ return passthrough(topicMessages)+ })++ try {+ await expect(waitForBackgroundTasks(consumer.processKafkaBatch([billed, internal]))).rejects.toThrow(+ 'broker down'+ )+ expect(mockEmitter.emit).not.toHaveBeenCalled()+ } finally {+ mockProducer.queueMessages = originalQueueMessages+ }+ })+it('produces logs normally when the rules cache fetch throws (fail-open)', async () => {// Metric rules are a purely additive side feature: a Postgres blip on the// rules fetch must not DLQ or block the log records themselves.@@ -2469,6 +2680,25 @@ describe('LogsIngestionConsumer', () => {return tracesConsumer}+ it('bills a signed traces batch, because the internal flag covers logs only', async () => {+ const key = 'internal-logs-test-key'+ await consumer.stop()+ consumer = createTracesIngestionConsumer({+ LOGS_INTERNAL_INGESTION_SIGNING_KEYS: key,+ LOGS_INTERNAL_INGESTION_ENABLED_TEAMS: String(team.id),+ })+ await consumer.start()+ jest.mocked(hub.quotaLimiting.isTeamTokenQuotaLimited).mockResolvedValue(true)+ const message = await createKafkaMessage(createLogMessage(), { token: team.api_token })+ message.headers!.push({+ [INTERNAL_LOGS_HEADER]: Buffer.from(signInternalLogs(key, 'hog_flow', team.id, message.value!)),+ })++ await waitForBackgroundTasks(consumer.processKafkaBatch([message]))++ expect(getProducedKafkaMessages().filter((m) => m.topic === KAFKA_LOGS_CLICKHOUSE)).toHaveLength(0)+ })+it('leaves pattern masking off even on a wildcard allowlist, because a trace record has no body', () => {const tracesConsumer = createTracesIngestionConsumer({ LOGS_PATTERN_MASKING_ENABLED_TEAMS: '*' })diff --git a/nodejs/src/logs/logs-ingestion-consumer.ts b/nodejs/src/logs/logs-ingestion-consumer.tsindex 1747b1d74aae..c4d7129856c6 100644--- a/nodejs/src/logs/logs-ingestion-consumer.ts+++ b/nodejs/src/logs/logs-ingestion-consumer.ts@@ -24,12 +24,20 @@ import { HealthCheckResult, PluginServerService } from '~/types'import { LogsIngestionConsumerConfig } from './config'import {+ recordInternalLogs,recordLogMessageDlq,recordLogMessageDropped,recordLogsAllowed,recordLogsDropped,recordLogsReceived,} from './ingestion-otel-metrics'+import {+ INTERNAL_LOGS_HEADER,+ INTERNAL_SOURCE_HEADER,+ InternalLogsVerifier,+ type InternalSourcePolicy,+ withoutInternalLogsHeaders,+} from './internal-logs'import { MESSAGE_KEYS } from './log-pattern-mask'import { logsPatternForcedDecodeCounter, makePatternMaskingStage } from './log-pattern-stage'import { type PiiScrubStats } from './log-pii-scrub'@@ -93,11 +101,16 @@ export interface LogsIngestionConsumerDeps {* shorten it.*/dependencyRetry?: { retryCount: number; initialRetryDelayMs: number }+ /** Defaults to `INTERNAL_SOURCE_POLICIES`. Tests pass a billed source, which production does not have yet. */+ internalSourcePolicies?: Readonly<Record<string, InternalSourcePolicy>>}/** Ingestion default when `logs_settings.retention_days` is unset; must match `DEFAULT_LOGS_RETENTION_DAYS` in `posthog/models/team/logs_retention.py`. */export const DEFAULT_LOGS_RETENTION_DAYS = 14+// A producer can stamp internal rows with less retention than the team default.+const INTERNAL_LOGS_MIN_RETENTION_DAYS = 1+/** Retention day counts that get their own per-tier usage metric. */const RETENTION_USAGE_TIERS = new Set([14, 30, 90])@@ -238,6 +251,18 @@ export const logsRecordsReceivedCounter = new Counter({help: 'Total log records received',})+export const logsInternalBytesCounter = new Counter({+ name: 'logs_ingestion_internal_bytes_total',+ help: 'Uncompressed bytes of verified internal logs, which stay out of the received and allowed counters.',+ labelNames: ['team_id'],+})++export const logsInternalRecordsCounter = new Counter({+ name: 'logs_ingestion_internal_records_total',+ help: 'Records of verified internal logs, which stay out of the received and allowed counters.',+ labelNames: ['team_id'],+})+export const logsRecordsAllowedCounter = new Counter({name: 'logs_ingestion_records_allowed_total',help: 'Total log records allowed through quota and rate limiting. Gross of the drop-rule billing credit: billed records_ingested = this − logs_ingestion_billing_records_credited_total.',@@ -375,6 +400,7 @@ export class LogsIngestionConsumer {private redis: RedisV2private rateLimiter: LogsRateLimiterServiceprivate readonly backfillEnabledTeamsRaw: string+ private readonly internalLogsVerifier: InternalLogsVerifierprivate samplingService: LogsSamplingServiceprivate readonly samplingEnabledTeamsRaw: stringprivate readonly samplingKillswitch: boolean@@ -428,6 +454,11 @@ export class LogsIngestionConsumer {})this.rateLimiter = new LogsRateLimiterService(mergedConfig, this.redis, rateLimiterName)this.backfillEnabledTeamsRaw = mergedConfig.LOGS_BACKFILL_ENABLED_TEAMS+ this.internalLogsVerifier = new InternalLogsVerifier(+ mergedConfig.LOGS_INTERNAL_INGESTION_SIGNING_KEYS,+ mergedConfig.LOGS_INTERNAL_INGESTION_ENABLED_TEAMS,+ deps.internalSourcePolicies+ )this.samplingService = new LogsSamplingService(this.redis, mergedConfig.LOGS_LIMITER_TTL_SECONDS)this.samplingEnabledTeamsRaw = mergedConfig.LOGS_SAMPLING_ENABLED_TEAMSthis.samplingKillswitch = mergedConfig.LOGS_SAMPLING_KILLSWITCH@@ -472,6 +503,15 @@ export class LogsIngestionConsumer {return teamIdMatchesCsv(this.metricRulesEnabledTeamsRaw, teamId)}+ // Transformations write hog_function logs, which could come back here as internal logs and loop.+ private appliesTeamLogRules(message: LogsIngestionMessage): boolean {+ return message.internalLogs?.teamRules ?? true+ }++ private isUnbilledInternal(message: LogsIngestionMessage): boolean {+ return message.internalLogs !== undefined && !message.internalLogs.billed+ }+private isTransformationsEnabledForTeam(teamId: number): boolean {if (this.transformationsKillswitch || !this.deps.logsTransformer) {return false@@ -498,7 +538,11 @@ export class LogsIngestionConsumer {batchBudget?: TransformationBatchBudget): Promise<LogRecordsTransform | undefined> {const transformer = this.deps.logsTransformer- if (!transformer || !this.isTransformationsEnabledForTeam(message.teamId)) {+ if (+ !transformer ||+ !this.appliesTeamLogRules(message) ||+ !this.isTransformationsEnabledForTeam(message.teamId)+ ) {return undefined}if (!(await transformer.teamHasTransformations(message.teamId))) {@@ -545,7 +589,7 @@ export class LogsIngestionConsumer {const samplingCache = this.deps.samplingRulesCacheconst samplingEvalEnabled = this.isSamplingEvalEnabledForTeam(message.teamId)let ruleSet: CompiledRuleSet | null = null- if (samplingCache && samplingEvalEnabled) {+ if (samplingCache && samplingEvalEnabled && this.appliesTeamLogRules(message)) {ruleSet = await samplingCache.getCompiledRuleSet(message.teamId)}const useSamplingPipeline = Boolean(ruleSet && ruleSet.rules.length > 0)@@ -558,7 +602,7 @@ export class LogsIngestionConsumer {const retentionCache = this.deps.retentionRulesCacheconst retentionEvalEnabled = this.isRetentionEvalEnabledForTeam(message.teamId)let retentionRuleSet: CompiledRetentionRuleSet | null = null- if (retentionCache && retentionEvalEnabled) {+ if (retentionCache && retentionEvalEnabled && this.appliesTeamLogRules(message)) {retentionRuleSet = await retentionCache.getCompiledRuleSet(message.teamId, this.retentionRuleSource)}const useRetention = Boolean(retentionRuleSet && retentionRuleSet.rules.length > 0)@@ -574,7 +618,8 @@ export class LogsIngestionConsumer {if (recordsTransform) {stages.push(makeTransformStage(recordsTransform))}- let shortestRetentionDays = defaultRetentionDays+ let shortestRetentionDays =+ message.internalLogs !== undefined ? INTERNAL_LOGS_MIN_RETENTION_DAYS : defaultRetentionDaysif (useRetention && retentionRuleSet) {stages.push(makeRetentionStage(retentionRuleSet, message.teamId, defaultRetentionDays))shortestRetentionDays = Math.min(@@ -685,13 +730,17 @@ export class LogsIngestionConsumer {this.trackIncomingTraffic(messages)const { backfillAllowedMessages, backfillDroppedMessages } = this.filterBackfillNotEnabledMessages(messages)- const { quotaAllowedMessages, quotaDroppedMessages } =- await this.filterQuotaLimitedMessages(backfillAllowedMessages)+ const internalMessages: LogsIngestionMessage[] = []+ const billedMessages: LogsIngestionMessage[] = []+ for (const message of backfillAllowedMessages) {+ ;(this.isUnbilledInternal(message) ? internalMessages : billedMessages).push(message)+ }+ const { quotaAllowedMessages, quotaDroppedMessages } = await this.filterQuotaLimitedMessages(billedMessages)const { rateLimiterAllowedMessages, rateLimiterDroppedMessages } =await this.filterRateLimitedMessages(quotaAllowedMessages)const usageStats = this.trackOutgoingTrafficAndBuildUsageStats(rateLimiterAllowedMessages, [- ...backfillDroppedMessages,+ ...backfillDroppedMessages.filter((message) => !this.isUnbilledInternal(message)),...quotaDroppedMessages,...rateLimiterDroppedMessages,])@@ -707,6 +756,11 @@ export class LogsIngestionConsumer {usageStats,transformationBatchBudget)+ // A usage map that is never emitted keeps their stats off the team's billed row.+ // They run after the billed batches, so the per-batch concurrency limit still holds.+ await this.processAndProduceLogMessages(internalMessages, new Map(), transformationBatchBudget)+ // Metric rules and internal usage emit deltas, so they wait until no group can fail the batch into a replay.+ this.queueInternalLogsMetrics(internalMessages)await this.emitMetricRuleTallies(metricTalliesByTeam)await this.deps.teamManager.setTeamsIngestedData(rateLimiterAllowedMessages.map((message) => message.teamId)@@ -719,7 +773,19 @@ export class LogsIngestionConsumer {})}})(),- messages: rateLimiterAllowedMessages,+ messages: [...rateLimiterAllowedMessages, ...internalMessages],+ }+ }++ // The usage report matches `bytes_ingested` exactly, so these names never reach billing.+ private queueInternalLogsMetrics(internalMessages: LogsIngestionMessage[]): void {+ for (const message of internalMessages) {+ this.queueUsageMetric(message.teamId, 'bytes_ingested_internal', message.bytesUncompressed)+ this.queueUsageMetric(message.teamId, 'records_ingested_internal', message.recordCount)+ const teamId = message.teamId.toString()+ logsInternalBytesCounter.inc({ team_id: teamId }, message.bytesUncompressed)+ logsInternalRecordsCounter.inc({ team_id: teamId }, message.recordCount)+ recordInternalLogs(message.teamId, message.bytesUncompressed, message.recordCount)}}@@ -727,6 +793,10 @@ export class LogsIngestionConsumer {let totalBytesReceived = 0let totalRecordsReceived = 0for (const message of messages) {+ // Never quota or rate limited, so counting them would show as drops in received minus allowed.+ if (this.isUnbilledInternal(message)) {+ continue+ }totalBytesReceived += message.bytesUncompressedtotalRecordsReceived += message.recordCount}@@ -804,13 +874,16 @@ export class LogsIngestionConsumer {} {const backfillAllowedMessages: LogsIngestionMessage[] = []const backfillDroppedMessages: LogsIngestionMessage[] = []- const droppedRecordsByTeam = new Map<number, { messages: number; records: number }>()+ const droppedRecordsByTeam = new Map<number, { messages: number; records: number; billed: boolean }>()for (const message of messages) {if (message.backfillRequested && !teamIdMatchesCsv(this.backfillEnabledTeamsRaw, message.teamId)) {backfillDroppedMessages.push(message)- const dropped = droppedRecordsByTeam.get(message.teamId) ?? { messages: 0, records: 0 }+ const dropped = droppedRecordsByTeam.get(message.teamId) ?? { messages: 0, records: 0, billed: false }dropped.messages++- dropped.records += message.recordCount+ if (!this.isUnbilledInternal(message)) {+ dropped.records += message.recordCount+ dropped.billed = true+ }droppedRecordsByTeam.set(message.teamId, dropped)} else {backfillAllowedMessages.push(message)@@ -823,7 +896,9 @@ export class LogsIngestionConsumer {dropped.messages)recordLogMessageDropped('backfill_not_enabled', teamId.toString(), dropped.messages)- this.queueUsageMetric(teamId, 'records_dropped_backfill_not_enabled', dropped.records)+ if (dropped.billed) {+ this.queueUsageMetric(teamId, 'records_dropped_backfill_not_enabled', dropped.records)+ }}return { backfillAllowedMessages, backfillDroppedMessages }@@ -894,7 +969,7 @@ export class LogsIngestionConsumer {byTeam: MetricTalliesByTeam,message: LogsIngestionMessage): Promise<{ token: string; rules: CompiledMetricRule[]; tallies: BatchTallies } | null> {- if (!this.isMetricRulesEnabledForTeam(message.teamId)) {+ if (!this.appliesTeamLogRules(message) || !this.isMetricRulesEnabledForTeam(message.teamId)) {return null}let state = byTeam.get(message.teamId)@@ -1082,11 +1157,13 @@ export class LogsIngestionConsumer {message.recordCount)- const teamIdLabel = message.teamId.toString()- if (contentCredit > 0) {- logsBillingBytesCreditedCounter.inc({ team_id: teamIdLabel }, contentCredit)+ if (!this.isUnbilledInternal(message)) {+ const teamIdLabel = message.teamId.toString()+ if (contentCredit > 0) {+ logsBillingBytesCreditedCounter.inc({ team_id: teamIdLabel }, contentCredit)+ }+ logsBillingRecordsCreditedCounter.inc({ team_id: teamIdLabel }, resolved.recordsDropped)}- logsBillingRecordsCreditedCounter.inc({ team_id: teamIdLabel }, resolved.recordsDropped)if (message.recordCount > 0) {logsDropBatchRowsHistogram.observe(message.recordCount)}@@ -1150,7 +1227,10 @@ export class LogsIngestionConsumer {value: processedValue,key: null,headers: {- ...parseKafkaHeaders(message.message.headers),+ ...withoutInternalLogsHeaders(parseKafkaHeaders(message.message.headers)),+ ...(message.internalLogs !== undefined+ ? { [INTERNAL_SOURCE_HEADER]: message.internalLogs.source }+ : {}),token: message.token,team_id: message.teamId.toString(),'json-parse': jsonParse.toString(),@@ -1483,6 +1563,11 @@ export class LogsIngestionConsumer {recordCount,minTimestampMicros: parseMinTimestampHeader(headers.min_timestamp),backfillRequested: headers.backfill_days !== undefined,+ internalLogs: this.internalLogsVerifier.verify(+ headers[INTERNAL_LOGS_HEADER],+ team.id,+ message.value+ ),})} catch (e) {// A message we cannot parse is message-scoped and will fail the same way ondiff --git a/nodejs/src/logs/traces-ingestion-consumer.ts b/nodejs/src/logs/traces-ingestion-consumer.tsindex b336b919bc9f..cae16c23e4de 100644--- a/nodejs/src/logs/traces-ingestion-consumer.ts+++ b/nodejs/src/logs/traces-ingestion-consumer.ts@@ -54,6 +54,9 @@ export class TracesIngestionConsumer extends LogsIngestionConsumer {// independently of log retention.LOGS_RETENTION_ENABLED_TEAMS: config.TRACES_RETENTION_ENABLED_TEAMS,LOGS_RETENTION_KILLSWITCH: config.TRACES_RETENTION_KILLSWITCH,+ // Internal batches are logs only, so a signed traces batch is still billed.+ LOGS_INTERNAL_INGESTION_SIGNING_KEYS: '',+ LOGS_INTERNAL_INGESTION_ENABLED_TEAMS: '',},// Own Redis key namespace so traces token buckets don't share per-team state with logs.'traces-rate-limiter'diff --git a/nodejs/src/logs/types.ts b/nodejs/src/logs/types.tsindex 51bf8a1f3f21..7af2c810b8bb 100644--- a/nodejs/src/logs/types.ts+++ b/nodejs/src/logs/types.ts@@ -1,5 +1,7 @@import { Message } from 'node-rdkafka'+import type { InternalLogs } from './internal-logs'+export type LogsIngestionMessage = {token: stringteamId: number@@ -12,4 +14,5 @@ export type LogsIngestionMessage = {/** Earliest row timestamp in epoch microseconds, from the `min_timestamp` header; absent from older producers. */minTimestampMicros?: numberbackfillRequested?: boolean+ internalLogs?: InternalLogs}