Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(logs): store signed internal log batches outside billing (#121927)

Jon McCallum65b0905eae86
diff --git a/nodejs/src/logs/config.ts b/nodejs/src/logs/config.ts
index f84a4f05e705..2ccd0bc0815b 100644
--- a/nodejs/src/logs/config.ts
+++ b/nodejs/src/logs/config.ts
@@ -67,6 +67,9 @@ export type LogsIngestionConsumerConfig = {
LOGS_BACKFILL_ENABLED_TEAMS: string
/** When `true`, retention rules are never evaluated (rows keep the team default via the batch header). */
LOGS_RETENTION_KILLSWITCH: boolean
+ /** Comma-separated team IDs, or `*` for all teams. A `posthog-internal` batch from any other team is billed as normal. */
+ LOGS_INTERNAL_INGESTION_ENABLED_TEAMS: string
+ LOGS_INTERNAL_INGESTION_SIGNING_KEYS: string
/** Comma-separated team IDs, or `*` for all teams, or empty (default) to disable measure-only pattern masking. */
LOGS_PATTERN_MASKING_ENABLED_TEAMS: string
LOGS_JSON_ATTRIBUTE_PARSING_ENABLED_TEAMS: string
@@ -123,6 +126,8 @@ export function getDefaultLogsIngestionConsumerConfig(): LogsIngestionConsumerCo
LOGS_RETENTION_ENABLED_TEAMS: isProdEnv() ? '' : '*',
LOGS_BACKFILL_ENABLED_TEAMS: isProdEnv() ? '' : '*',
LOGS_RETENTION_KILLSWITCH: false,
+ LOGS_INTERNAL_INGESTION_ENABLED_TEAMS: '',
+ LOGS_INTERNAL_INGESTION_SIGNING_KEYS: '',
// Off by default: enabling forces decode+re-encode for allowlisted teams.
LOGS_PATTERN_MASKING_ENABLED_TEAMS: '',
LOGS_JSON_ATTRIBUTE_PARSING_ENABLED_TEAMS: '',
diff --git a/nodejs/src/logs/ingestion-otel-metrics.test.ts b/nodejs/src/logs/ingestion-otel-metrics.test.ts
index 306a3ca222bf..f6b81c727219 100644
--- a/nodejs/src/logs/ingestion-otel-metrics.test.ts
+++ b/nodejs/src/logs/ingestion-otel-metrics.test.ts
@@ -8,6 +8,7 @@ import {
} from '@opentelemetry/sdk-metrics'
import {
+ recordInternalLogs,
recordJsonEnrichmentSkipped,
recordLogMessageDlq,
recordLogMessageDropped,
@@ -89,6 +90,20 @@ describe('ingestion-otel-metrics', () => {
{ team_id: '42' },
10,
],
+ [
+ 'recordInternalLogs bytes',
+ 'logs_ingestion_internal_bytes_total',
+ () => recordInternalLogs(2, 4096, 20),
+ { team_id: '2' },
+ 4096,
+ ],
+ [
+ 'recordInternalLogs records',
+ 'logs_ingestion_internal_records_total',
+ () => recordInternalLogs(2, 4096, 20),
+ { team_id: '2' },
+ 20,
+ ],
[
'recordLogMessageDropped',
'logs_ingestion_message_dropped_count',
diff --git a/nodejs/src/logs/ingestion-otel-metrics.ts b/nodejs/src/logs/ingestion-otel-metrics.ts
index 0ba34752f381..a9d65eb25049 100644
--- a/nodejs/src/logs/ingestion-otel-metrics.ts
+++ b/nodejs/src/logs/ingestion-otel-metrics.ts
@@ -22,6 +22,8 @@ interface LogsIngestionInstruments {
recordsAllowed: Counter
bytesDropped: Counter
recordsDropped: Counter
+ internalBytes: Counter
+ internalRecords: Counter
messagesDropped: Counter
messagesDlq: Counter
processingDuration: Histogram
@@ -61,6 +63,14 @@ function getInstruments(): LogsIngestionInstruments {
recordsDropped: createCounterWithExemplars(meter, 'logs_ingestion_records_dropped_total', {
description: 'Total log records dropped due to quota or rate limiting',
}),
+ internalBytes: createCounterWithExemplars(meter, 'logs_ingestion_internal_bytes_total', {
+ description:
+ 'Uncompressed bytes of verified internal logs, which stay out of the received and allowed counters',
+ unit: 'By',
+ }),
+ internalRecords: createCounterWithExemplars(meter, 'logs_ingestion_internal_records_total', {
+ description: 'Records of verified internal logs, which stay out of the received and allowed counters',
+ }),
messagesDropped: createCounterWithExemplars(meter, 'logs_ingestion_message_dropped_count', {
description: 'The number of logs ingestion messages dropped',
}),
@@ -106,6 +116,13 @@ export const recordLogsDropped = swallowing((teamId: number, bytes: number, reco
addPositive(recordsDropped, records, attributes)
})
+export const recordInternalLogs = swallowing((teamId: number, bytes: number, records: number): void => {
+ const { internalBytes, internalRecords } = getInstruments()
+ const attributes = { team_id: teamId.toString() }
+ addPositive(internalBytes, bytes, attributes)
+ addPositive(internalRecords, records, attributes)
+})
+
export const recordLogMessageDropped = swallowing((reason: string, teamId: string, count: number = 1): void => {
addPositive(getInstruments().messagesDropped, count, { reason, team_id: teamId })
})
diff --git a/nodejs/src/logs/internal-logs.test.ts b/nodejs/src/logs/internal-logs.test.ts
new file mode 100644
index 000000000000..11e8a00507f3
--- /dev/null
+++ b/nodejs/src/logs/internal-logs.test.ts
@@ -0,0 +1,74 @@
+import { InternalLogsVerifier, signInternalLogs } from './internal-logs'
+
+const KEY = 'internal-logs-test-key'
+const OTHER_KEY = 'internal-logs-other-key'
+const SOURCE = 'hog_flow'
+const TEAM_ID = 2
+const OTHER_TEAM_ID = 3
+const PAYLOAD = Buffer.from('avro container bytes')
+
+describe('InternalLogsVerifier', () => {
+ it.each([
+ {
+ name: 'accepts a batch signed with the newest key',
+ signature: signInternalLogs(KEY, SOURCE, TEAM_ID, PAYLOAD),
+ },
+ {
+ name: 'accepts a batch signed with an older key during rotation',
+ keys: `${OTHER_KEY},${KEY}`,
+ signature: signInternalLogs(KEY, SOURCE, TEAM_ID, PAYLOAD),
+ },
+ {
+ name: 'accepts a batch on a wildcard allowlist',
+ teams: '*',
+ signature: signInternalLogs(KEY, SOURCE, TEAM_ID, PAYLOAD),
+ },
+ ])('$name', ({ signature, keys = KEY, teams = String(TEAM_ID) }) => {
+ expect(new InternalLogsVerifier(keys, teams).verify(signature, TEAM_ID, PAYLOAD)).toEqual({
+ source: SOURCE,
+ billed: false,
+ teamRules: false,
+ })
+ })
+
+ it.each([
+ { name: 'no header', signature: undefined },
+ { name: 'a malformed signature', signature: 'v1.not-a-signature' },
+ { name: 'an unknown key', signature: signInternalLogs(OTHER_KEY, SOURCE, TEAM_ID, PAYLOAD) },
+ { name: "another team's signature", signature: signInternalLogs(KEY, SOURCE, OTHER_TEAM_ID, PAYLOAD) },
+ {
+ name: 'a signature moved to another source',
+ signature: signInternalLogs(KEY, SOURCE, TEAM_ID, PAYLOAD).replace(`.${SOURCE}.`, '.hog_function.'),
+ },
+ {
+ name: 'a signature over different data',
+ signature: signInternalLogs(KEY, SOURCE, TEAM_ID, Buffer.from('other')),
+ },
+ {
+ name: 'a team not on the allowlist',
+ teams: String(OTHER_TEAM_ID),
+ signature: signInternalLogs(KEY, SOURCE, TEAM_ID, PAYLOAD),
+ },
+ { name: 'no configured key', keys: '', signature: signInternalLogs(KEY, SOURCE, TEAM_ID, PAYLOAD) },
+ { name: 'a source with no policy', signature: signInternalLogs(KEY, 'not_a_source', TEAM_ID, PAYLOAD) },
+ { name: 'an inherited property as source', signature: signInternalLogs(KEY, 'constructor', TEAM_ID, PAYLOAD) },
+ { name: 'the prototype as source', signature: signInternalLogs(KEY, '__proto__', TEAM_ID, PAYLOAD) },
+ { name: 'an empty payload', payload: null, signature: signInternalLogs(KEY, SOURCE, TEAM_ID, PAYLOAD) },
+ ])(
+ 'bills a batch with $name',
+ ({ signature, keys = KEY, teams = String(TEAM_ID), payload = PAYLOAD as Buffer | null }) => {
+ expect(new InternalLogsVerifier(keys, teams).verify(signature, TEAM_ID, payload)).toBeUndefined()
+ }
+ )
+
+ it('refuses to sign a source that cannot go into the header', () => {
+ expect(() => signInternalLogs(KEY, 'hog.flow', TEAM_ID, PAYLOAD)).toThrow('invalid source')
+ })
+
+ // Producers and consumers deploy separately, so a format change must be a new version prefix.
+ it('keeps the v1 signature format stable', () => {
+ expect(signInternalLogs(KEY, SOURCE, TEAM_ID, PAYLOAD)).toEqual(
+ 'v1.hog_flow.b97466e15a9bf1a2658e8d92e5f63ed60df91bd4e838e4ab4426cbb5280d2055'
+ )
+ })
+})
diff --git a/nodejs/src/logs/internal-logs.ts b/nodejs/src/logs/internal-logs.ts
new file mode 100644
index 000000000000..911cbddf9f19
--- /dev/null
+++ b/nodejs/src/logs/internal-logs.ts
@@ -0,0 +1,130 @@
+import { createHash, createHmac, timingSafeEqual } from 'crypto'
+import { Counter } from 'prom-client'
+
+import { buildIntegerMatcher } from '~/common/config/matchers'
+import type { ValueMatcher } from '~/types'
+
+export const INTERNAL_LOGS_HEADER = 'posthog-internal'
+
+export const INTERNAL_SOURCE_HEADER = 'internal-source'
+
+const SIGNATURE_VERSION = 'v1'
+
+// No dot, because a dot divides the parts of the header value.
+const SOURCE_PATTERN = /^[a-z0-9_]{1,64}$/
+
+export type InternalSourcePolicy = {
+ /** Counts toward the team's logs usage, quota and rate limits. */
+ billed: boolean
+ /** Runs the team's sampling, drop, transformation, metric and retention rules. */
+ teamRules: boolean
+}
+
+export type InternalLogs = InternalSourcePolicy & { source: string }
+
+// A signer names the source and this map decides how it is handled, so a leaked key cannot invent
+// a free source. Moving a source to billing changes its entry here, not the producer or the header.
+export const INTERNAL_SOURCE_POLICIES: Readonly<Record<string, InternalSourcePolicy>> = {
+ hog_flow: { billed: false, teamRules: false },
+ // Each log_entries source moves unbilled, one at a time. Add a source when its producer starts signing.
+ // hog_function: { billed: false, teamRules: false },
+}
+
+type RejectReason = 'not_configured' | 'team_not_allowed' | 'invalid_signature' | 'unknown_source'
+
+const logsInternalLogsCounter = new Counter({
+ name: 'logs_ingestion_internal_logs_total',
+ help: 'Batches that carried the posthog-internal header, by outcome. A rejected batch is billed as normal.',
+ labelNames: ['outcome', 'reason'],
+})
+
+function parseKeys(commaSeparatedKeys: string): string[] {
+ return commaSeparatedKeys
+ .split(',')
+ .map((key) => key.trim())
+ .filter((key) => key.length > 0)
+}
+
+function sha256Hex(payload: Buffer): string {
+ return createHash('sha256').update(payload).digest('hex')
+}
+
+// No expiry, because a batch that waits in Kafka through consumer lag or a DLQ replay must stay internal.
+function computeSignature(key: string, source: string, teamId: number, payloadSha256: string): string {
+ const mac = createHmac('sha256', key)
+ .update(`${SIGNATURE_VERSION}|${source}|${teamId}|${payloadSha256}`)
+ .digest('hex')
+ return `${SIGNATURE_VERSION}.${source}.${mac}`
+}
+
+export function signInternalLogs(commaSeparatedKeys: string, source: string, teamId: number, payload: Buffer): string {
+ const [newestKey] = parseKeys(commaSeparatedKeys)
+ if (!newestKey) {
+ throw new Error('Cannot sign posthog-internal batch: no signing key configured')
+ }
+ if (!SOURCE_PATTERN.test(source)) {
+ throw new Error(`Cannot sign posthog-internal batch: invalid source ${JSON.stringify(source)}`)
+ }
+ return computeSignature(newestKey, source, teamId, sha256Hex(payload))
+}
+
+// Only a verified batch may set the `internal_source` column, so an incoming copy of the header is removed.
+export function withoutInternalLogsHeaders(headers: Record<string, string>): Record<string, string> {
+ const { [INTERNAL_LOGS_HEADER]: _signature, [INTERNAL_SOURCE_HEADER]: _source, ...rest } = headers
+ return rest
+}
+
+export class InternalLogsVerifier {
+ private readonly keys: string[]
+ private readonly isTeamAllowed: ValueMatcher<number>
+
+ constructor(
+ commaSeparatedKeys: string,
+ enabledTeams: string,
+ private readonly policies: Readonly<Record<string, InternalSourcePolicy>> = INTERNAL_SOURCE_POLICIES
+ ) {
+ this.keys = parseKeys(commaSeparatedKeys)
+ this.isTeamAllowed = buildIntegerMatcher(enabledTeams, true)
+ }
+
+ verify(signature: string | undefined, teamId: number, payload: Buffer | null): InternalLogs | undefined {
+ if (signature === undefined) {
+ return undefined
+ }
+ const source = signature.split('.')[1] ?? ''
+ const reason = this.rejectReason(signature, source, teamId, payload)
+ // Own keys only, because a source like `constructor` resolves to an inherited policy with no `billed`.
+ const policy = reason || !Object.hasOwn(this.policies, source) ? undefined : this.policies[source]
+ if (!policy) {
+ logsInternalLogsCounter.inc({ outcome: 'rejected', reason: reason ?? 'unknown_source' })
+ return undefined
+ }
+ logsInternalLogsCounter.inc({ outcome: 'accepted', reason: '' })
+ return { source, ...policy }
+ }
+
+ private rejectReason(
+ signature: string,
+ source: string,
+ teamId: number,
+ payload: Buffer | null
+ ): RejectReason | null {
+ if (this.keys.length === 0) {
+ return 'not_configured'
+ }
+ if (!this.isTeamAllowed(teamId)) {
+ return 'team_not_allowed'
+ }
+ if (payload === null || !SOURCE_PATTERN.test(source)) {
+ return 'invalid_signature'
+ }
+ const received = Buffer.from(signature)
+ const payloadSha256 = sha256Hex(payload)
+ // Every key is tried, so a rotation can add a key before producers switch to it.
+ const matches = this.keys.some((key) => {
+ const expected = Buffer.from(computeSignature(key, source, teamId, payloadSha256))
+ return expected.length === received.length && timingSafeEqual(expected, received)
+ })
+ return matches ? null : 'invalid_signature'
+ }
+}
diff --git a/nodejs/src/logs/logs-ingestion-consumer.test.ts b/nodejs/src/logs/logs-ingestion-consumer.test.ts
index b777f96f44b1..50254b69275d 100644
--- a/nodejs/src/logs/logs-ingestion-consumer.test.ts
+++ b/nodejs/src/logs/logs-ingestion-consumer.test.ts
@@ -28,6 +28,7 @@ import { Hub, Team } from '~/types'
import { getDefaultTracesIngestionConsumerConfig } from './config'
import * as otelMetrics from './ingestion-otel-metrics'
import { resetLogsIngestionInstrumentsForTests } from './ingestion-otel-metrics'
+import { INTERNAL_LOGS_HEADER, INTERNAL_SOURCE_HEADER, signInternalLogs } from './internal-logs'
import { logsPatternBodyKindCounter } from './log-pattern-stage'
import { LogRecord, decodeLogRecords, encodeLogRecords, logsJsonAttributeSniffCounter } from './log-record-avro'
import {
@@ -38,6 +39,7 @@ import {
logMessageDlqCounter,
logMessageDroppedCounter,
logsBillingBytesCreditedCounter,
+ logsBillingRecordsCreditedCounter,
logsBytesAllowedCounter,
logsBytesDroppedCounter,
logsBytesReceivedCounter,
@@ -157,10 +159,11 @@ const createMultiRecordKafkaMessage = async (
const logRecordType = avro.Type.forSchema({
type: 'record',
name: 'LogRecord',
- fields: LOG_RECORD_BASE_FIELDS,
+ fields: [...LOG_RECORD_BASE_FIELDS, { name: 'retention_days', type: ['null', 'int'], default: null }],
})
const records: LogRecord[] = logDataList.map((logData, i) => ({
+ retention_days: logData.retentionDays ?? null,
uuid: `test-uuid-${offsetIncrementer}-${i}`,
trace_id: null,
span_id: null,
@@ -213,6 +216,7 @@ describe('LogsIngestionConsumer', () => {
| 'metricsEmitter'
| 'logsTransformer'
| 'patternMessageKeysCache'
+ | 'internalSourcePolicies'
>
> = {}
) => {
@@ -1037,6 +1041,178 @@ describe('LogsIngestionConsumer', () => {
})
})
+ describe('internal logs', () => {
+ const SIGNING_KEY = 'internal-logs-test-key'
+
+ it.each([
+ {
+ name: "stores a signed batch outside quota, billing and the team's drop rules",
+ signed: true,
+ billed: false,
+ },
+ { name: 'bills a batch whose signature does not verify', signed: false, billed: true },
+ {
+ name: "bills a signed batch from a billed source but skips the team's drop rules",
+ signed: true,
+ billed: true,
+ },
+ ])('$name', async ({ signed, billed }) => {
+ await consumer.stop()
+ const dropEverything: Pick<SamplingRulesCache, 'getCompiledRuleSet'> = {
+ getCompiledRuleSet: () =>
+ Promise.resolve(
+ compileRuleSet([
+ {
+ id: '00000000-0000-0000-0000-0000000000aa',
+ rule_type: 'severity_sampling',
+ scope_service: 'test-service',
+ scope_path_pattern: null,
+ scope_attribute_filters: [],
+ config: { actions: { INFO: { type: 'drop' } } },
+ },
+ ])
+ ),
+ }
+ consumer = await createLogsIngestionConsumer(
+ hub,
+ {
+ LOGS_INTERNAL_INGESTION_ENABLED_TEAMS: String(team.id),
+ LOGS_INTERNAL_INGESTION_SIGNING_KEYS: SIGNING_KEY,
+ },
+ {
+ samplingRulesCache: dropEverything as SamplingRulesCache,
+ internalSourcePolicies: { hog_flow: { billed, teamRules: false } },
+ }
+ )
+ // A quota limit drops every billed batch, so it applies only to the rows that test the internal bypass.
+ jest.mocked(hub.quotaLimiting.isTeamTokenQuotaLimited).mockResolvedValue(!(signed && billed))
+
+ const [message] = await createKafkaMessages([createLogMessage()], {
+ token: team.api_token,
+ bytes_uncompressed: '100',
+ record_count: '1',
+ })
+ const signature = signed
+ ? signInternalLogs(SIGNING_KEY, 'hog_flow', team.id, message.value!)
+ : 'v1.hog_flow.not-a-signature'
+ message.headers!.push({ [INTERNAL_LOGS_HEADER]: Buffer.from(signature) })
+ const recordLogsReceivedSpy = jest.spyOn(otelMetrics, 'recordLogsReceived')
+ const recordInternalLogsSpy = jest.spyOn(otelMetrics, 'recordInternalLogs')
+
+ await waitForBackgroundTasks(consumer.processKafkaBatch([message]))
+
+ const metricNames = mockProducerObserver
+ .getProducedKafkaMessagesForTopic(KAFKA_APP_METRICS_2)
+ .filter((m) => m.value.team_id === team.id)
+ .map((m) => m.value.metric_name)
+ const produced = getProducedKafkaMessages().filter((m) => m.topic === KAFKA_LOGS_CLICKHOUSE)
+ expect({
+ produced: produced.length,
+ forwardsSignature: produced.some((m) => INTERNAL_LOGS_HEADER in (m.headers ?? {})),
+ internalSource: produced[0]?.headers?.[INTERNAL_SOURCE_HEADER],
+ received: recordLogsReceivedSpy.mock.calls,
+ internal: recordInternalLogsSpy.mock.calls,
+ internalMetrics: ['bytes_ingested_internal', 'records_ingested_internal'].every((n) =>
+ metricNames.includes(n)
+ ),
+ billedMetrics: ['bytes_ingested', 'records_ingested', 'bytes_received'].some((n) =>
+ metricNames.includes(n)
+ ),
+ }).toEqual({
+ produced: signed ? 1 : 0,
+ forwardsSignature: false,
+ internalSource: signed ? 'hog_flow' : undefined,
+ received: [billed ? [100, 1] : [0, 0]],
+ internal: billed ? [] : [[team.id, 100, 1]],
+ internalMetrics: !billed,
+ billedMetrics: billed,
+ })
+ })
+
+ it('drops a row past its stamped retention without crediting billing', async () => {
+ await consumer.stop()
+ consumer = await createLogsIngestionConsumer(hub, {
+ LOGS_INTERNAL_INGESTION_ENABLED_TEAMS: String(team.id),
+ LOGS_INTERNAL_INGESTION_SIGNING_KEYS: SIGNING_KEY,
+ })
+ const fiveDaysAgoMicros = DateTime.now().minus({ days: 5 }).toMillis() * 1000
+ const message = await createMultiRecordKafkaMessage(
+ [{ level: 'info', timestampMicros: fiveDaysAgoMicros, retentionDays: 1 }, { level: 'info' }],
+ {
+ token: team.api_token,
+ bytes_uncompressed: '200',
+ record_count: '2',
+ min_timestamp: fiveDaysAgoMicros.toString(),
+ }
+ )
+ message.headers!.push({
+ [INTERNAL_LOGS_HEADER]: Buffer.from(signInternalLogs(SIGNING_KEY, 'hog_flow', team.id, message.value!)),
+ })
+ const bytesCreditSpy = jest.spyOn(logsBillingBytesCreditedCounter, 'inc')
+ const recordsCreditSpy = jest.spyOn(logsBillingRecordsCreditedCounter, 'inc')
+
+ await waitForBackgroundTasks(consumer.processKafkaBatch([message]))
+
+ const expiredMetric = mockProducerObserver
+ .getProducedKafkaMessagesForTopic(KAFKA_APP_METRICS_2)
+ .map((m) => m.value)
+ .find((v) => v.metric_name === 'records_dropped_retention_expired' && v.team_id === team.id)
+ expect({
+ expired: expiredMetric?.count,
+ produced: getProducedKafkaMessages().filter((m) => m.topic === KAFKA_LOGS_CLICKHOUSE).length,
+ credits: [...bytesCreditSpy.mock.calls, ...recordsCreditSpy.mock.calls],
+ }).toEqual({ expired: 1, produced: 1, credits: [] })
+ })
+
+ it.each([
+ { name: 'drops a signed backfill without writing to the team usage row', backfill: true },
+ { name: 'keeps the usage of a failed internal batch out of the next flush', backfill: false },
+ ])('$name', async ({ backfill }) => {
+ await consumer.stop()
+ consumer = await createLogsIngestionConsumer(hub, {
+ LOGS_BACKFILL_ENABLED_TEAMS: String(team2.id),
+ LOGS_INTERNAL_INGESTION_ENABLED_TEAMS: String(team.id),
+ LOGS_INTERNAL_INGESTION_SIGNING_KEYS: SIGNING_KEY,
+ })
+ const [internal] = await createKafkaMessages([createLogMessage()], {
+ token: team.api_token,
+ bytes_uncompressed: '100',
+ record_count: '1',
+ ...(backfill ? { backfill_days: '540' } : {}),
+ })
+ internal.headers!.push({
+ [INTERNAL_LOGS_HEADER]: Buffer.from(
+ signInternalLogs(SIGNING_KEY, 'hog_flow', team.id, internal.value!)
+ ),
+ })
+ const originalQueueMessages = mockProducer.queueMessages
+ const passthrough = originalQueueMessages.bind(mockProducer)
+ mockProducer.queueMessages = jest.fn().mockImplementation((topicMessages) => {
+ if (topicMessages.topic === KAFKA_LOGS_CLICKHOUSE && !backfill) {
+ throw new DependencyUnavailableError('broker down', 'Kafka', new Error('broker down'))
+ }
+ return passthrough(topicMessages)
+ })
+
+ try {
+ await consumer.processKafkaBatch([internal]).then((r) => r.backgroundTask?.catch(() => undefined))
+ mockProducer.queueMessages = originalQueueMessages
+ // A later batch of another team flushes the shared usage aggregator.
+ const [next] = await createKafkaMessages([createLogMessage()], { token: team2.api_token })
+ await waitForBackgroundTasks(consumer.processKafkaBatch([next]))
+ } finally {
+ mockProducer.queueMessages = originalQueueMessages
+ }
+
+ expect(
+ mockProducerObserver
+ .getProducedKafkaMessagesForTopic(KAFKA_APP_METRICS_2)
+ .filter((m) => m.value.team_id === team.id)
+ .map((m) => m.value.metric_name)
+ ).toEqual([])
+ })
+ })
+
describe('filterQuotaLimitedMessages', () => {
it('should allow messages when not quota limited', async () => {
jest.mocked(hub.quotaLimiting.isTeamTokenQuotaLimited).mockResolvedValue(false)
@@ -2345,6 +2521,41 @@ describe('LogsIngestionConsumer', () => {
])
})
+ it('emits nothing when a later internal message fails the batch', async () => {
+ const key = 'internal-logs-test-key'
+ await createConsumerWithMetricRules({
+ LOGS_INTERNAL_INGESTION_ENABLED_TEAMS: String(team.id),
+ LOGS_INTERNAL_INGESTION_SIGNING_KEYS: key,
+ })
+ const [billed, internal] = await createKafkaMessages([createLogMessage(), createLogMessage()], {
+ token: team.api_token,
+ })
+ internal.headers!.push({
+ [INTERNAL_LOGS_HEADER]: Buffer.from(signInternalLogs(key, 'hog_flow', team.id, internal.value!)),
+ })
+
+ const originalQueueMessages = mockProducer.queueMessages
+ const passthrough = originalQueueMessages.bind(mockProducer)
+ mockProducer.queueMessages = jest.fn().mockImplementation((topicMessages) => {
+ const isInternal = topicMessages.messages?.some(
+ (m: { headers?: Record<string, string> }) => m.headers?.[INTERNAL_SOURCE_HEADER]
+ )
+ if (topicMessages.topic === KAFKA_LOGS_CLICKHOUSE && isInternal) {
+ throw new DependencyUnavailableError('broker down', 'Kafka', new Error('broker down'))
+ }
+ return passthrough(topicMessages)
+ })
+
+ try {
+ await expect(waitForBackgroundTasks(consumer.processKafkaBatch([billed, internal]))).rejects.toThrow(
+ 'broker down'
+ )
+ expect(mockEmitter.emit).not.toHaveBeenCalled()
+ } finally {
+ mockProducer.queueMessages = originalQueueMessages
+ }
+ })
+
it('produces logs normally when the rules cache fetch throws (fail-open)', async () => {
// Metric rules are a purely additive side feature: a Postgres blip on the
// rules fetch must not DLQ or block the log records themselves.
@@ -2469,6 +2680,25 @@ describe('LogsIngestionConsumer', () => {
return tracesConsumer
}
+ it('bills a signed traces batch, because the internal flag covers logs only', async () => {
+ const key = 'internal-logs-test-key'
+ await consumer.stop()
+ consumer = createTracesIngestionConsumer({
+ LOGS_INTERNAL_INGESTION_SIGNING_KEYS: key,
+ LOGS_INTERNAL_INGESTION_ENABLED_TEAMS: String(team.id),
+ })
+ await consumer.start()
+ jest.mocked(hub.quotaLimiting.isTeamTokenQuotaLimited).mockResolvedValue(true)
+ const message = await createKafkaMessage(createLogMessage(), { token: team.api_token })
+ message.headers!.push({
+ [INTERNAL_LOGS_HEADER]: Buffer.from(signInternalLogs(key, 'hog_flow', team.id, message.value!)),
+ })
+
+ await waitForBackgroundTasks(consumer.processKafkaBatch([message]))
+
+ expect(getProducedKafkaMessages().filter((m) => m.topic === KAFKA_LOGS_CLICKHOUSE)).toHaveLength(0)
+ })
+
it('leaves pattern masking off even on a wildcard allowlist, because a trace record has no body', () => {
const tracesConsumer = createTracesIngestionConsumer({ LOGS_PATTERN_MASKING_ENABLED_TEAMS: '*' })
diff --git a/nodejs/src/logs/logs-ingestion-consumer.ts b/nodejs/src/logs/logs-ingestion-consumer.ts
index 1747b1d74aae..c4d7129856c6 100644
--- a/nodejs/src/logs/logs-ingestion-consumer.ts
+++ b/nodejs/src/logs/logs-ingestion-consumer.ts
@@ -24,12 +24,20 @@ import { HealthCheckResult, PluginServerService } from '~/types'
import { LogsIngestionConsumerConfig } from './config'
import {
+ recordInternalLogs,
recordLogMessageDlq,
recordLogMessageDropped,
recordLogsAllowed,
recordLogsDropped,
recordLogsReceived,
} from './ingestion-otel-metrics'
+import {
+ INTERNAL_LOGS_HEADER,
+ INTERNAL_SOURCE_HEADER,
+ InternalLogsVerifier,
+ type InternalSourcePolicy,
+ withoutInternalLogsHeaders,
+} from './internal-logs'
import { MESSAGE_KEYS } from './log-pattern-mask'
import { logsPatternForcedDecodeCounter, makePatternMaskingStage } from './log-pattern-stage'
import { type PiiScrubStats } from './log-pii-scrub'
@@ -93,11 +101,16 @@ export interface LogsIngestionConsumerDeps {
* shorten it.
*/
dependencyRetry?: { retryCount: number; initialRetryDelayMs: number }
+ /** Defaults to `INTERNAL_SOURCE_POLICIES`. Tests pass a billed source, which production does not have yet. */
+ internalSourcePolicies?: Readonly<Record<string, InternalSourcePolicy>>
}
/** Ingestion default when `logs_settings.retention_days` is unset; must match `DEFAULT_LOGS_RETENTION_DAYS` in `posthog/models/team/logs_retention.py`. */
export const DEFAULT_LOGS_RETENTION_DAYS = 14
+// A producer can stamp internal rows with less retention than the team default.
+const INTERNAL_LOGS_MIN_RETENTION_DAYS = 1
+
/** Retention day counts that get their own per-tier usage metric. */
const RETENTION_USAGE_TIERS = new Set([14, 30, 90])
@@ -238,6 +251,18 @@ export const logsRecordsReceivedCounter = new Counter({
help: 'Total log records received',
})
+export const logsInternalBytesCounter = new Counter({
+ name: 'logs_ingestion_internal_bytes_total',
+ help: 'Uncompressed bytes of verified internal logs, which stay out of the received and allowed counters.',
+ labelNames: ['team_id'],
+})
+
+export const logsInternalRecordsCounter = new Counter({
+ name: 'logs_ingestion_internal_records_total',
+ help: 'Records of verified internal logs, which stay out of the received and allowed counters.',
+ labelNames: ['team_id'],
+})
+
export const logsRecordsAllowedCounter = new Counter({
name: 'logs_ingestion_records_allowed_total',
help: 'Total log records allowed through quota and rate limiting. Gross of the drop-rule billing credit: billed records_ingested = this − logs_ingestion_billing_records_credited_total.',
@@ -375,6 +400,7 @@ export class LogsIngestionConsumer {
private redis: RedisV2
private rateLimiter: LogsRateLimiterService
private readonly backfillEnabledTeamsRaw: string
+ private readonly internalLogsVerifier: InternalLogsVerifier
private samplingService: LogsSamplingService
private readonly samplingEnabledTeamsRaw: string
private readonly samplingKillswitch: boolean
@@ -428,6 +454,11 @@ export class LogsIngestionConsumer {
})
this.rateLimiter = new LogsRateLimiterService(mergedConfig, this.redis, rateLimiterName)
this.backfillEnabledTeamsRaw = mergedConfig.LOGS_BACKFILL_ENABLED_TEAMS
+ this.internalLogsVerifier = new InternalLogsVerifier(
+ mergedConfig.LOGS_INTERNAL_INGESTION_SIGNING_KEYS,
+ mergedConfig.LOGS_INTERNAL_INGESTION_ENABLED_TEAMS,
+ deps.internalSourcePolicies
+ )
this.samplingService = new LogsSamplingService(this.redis, mergedConfig.LOGS_LIMITER_TTL_SECONDS)
this.samplingEnabledTeamsRaw = mergedConfig.LOGS_SAMPLING_ENABLED_TEAMS
this.samplingKillswitch = mergedConfig.LOGS_SAMPLING_KILLSWITCH
@@ -472,6 +503,15 @@ export class LogsIngestionConsumer {
return teamIdMatchesCsv(this.metricRulesEnabledTeamsRaw, teamId)
}
+ // Transformations write hog_function logs, which could come back here as internal logs and loop.
+ private appliesTeamLogRules(message: LogsIngestionMessage): boolean {
+ return message.internalLogs?.teamRules ?? true
+ }
+
+ private isUnbilledInternal(message: LogsIngestionMessage): boolean {
+ return message.internalLogs !== undefined && !message.internalLogs.billed
+ }
+
private isTransformationsEnabledForTeam(teamId: number): boolean {
if (this.transformationsKillswitch || !this.deps.logsTransformer) {
return false
@@ -498,7 +538,11 @@ export class LogsIngestionConsumer {
batchBudget?: TransformationBatchBudget
): Promise<LogRecordsTransform | undefined> {
const transformer = this.deps.logsTransformer
- if (!transformer || !this.isTransformationsEnabledForTeam(message.teamId)) {
+ if (
+ !transformer ||
+ !this.appliesTeamLogRules(message) ||
+ !this.isTransformationsEnabledForTeam(message.teamId)
+ ) {
return undefined
}
if (!(await transformer.teamHasTransformations(message.teamId))) {
@@ -545,7 +589,7 @@ export class LogsIngestionConsumer {
const samplingCache = this.deps.samplingRulesCache
const samplingEvalEnabled = this.isSamplingEvalEnabledForTeam(message.teamId)
let ruleSet: CompiledRuleSet | null = null
- if (samplingCache && samplingEvalEnabled) {
+ if (samplingCache && samplingEvalEnabled && this.appliesTeamLogRules(message)) {
ruleSet = await samplingCache.getCompiledRuleSet(message.teamId)
}
const useSamplingPipeline = Boolean(ruleSet && ruleSet.rules.length > 0)
@@ -558,7 +602,7 @@ export class LogsIngestionConsumer {
const retentionCache = this.deps.retentionRulesCache
const retentionEvalEnabled = this.isRetentionEvalEnabledForTeam(message.teamId)
let retentionRuleSet: CompiledRetentionRuleSet | null = null
- if (retentionCache && retentionEvalEnabled) {
+ if (retentionCache && retentionEvalEnabled && this.appliesTeamLogRules(message)) {
retentionRuleSet = await retentionCache.getCompiledRuleSet(message.teamId, this.retentionRuleSource)
}
const useRetention = Boolean(retentionRuleSet && retentionRuleSet.rules.length > 0)
@@ -574,7 +618,8 @@ export class LogsIngestionConsumer {
if (recordsTransform) {
stages.push(makeTransformStage(recordsTransform))
}
- let shortestRetentionDays = defaultRetentionDays
+ let shortestRetentionDays =
+ message.internalLogs !== undefined ? INTERNAL_LOGS_MIN_RETENTION_DAYS : defaultRetentionDays
if (useRetention && retentionRuleSet) {
stages.push(makeRetentionStage(retentionRuleSet, message.teamId, defaultRetentionDays))
shortestRetentionDays = Math.min(
@@ -685,13 +730,17 @@ export class LogsIngestionConsumer {
this.trackIncomingTraffic(messages)
const { backfillAllowedMessages, backfillDroppedMessages } = this.filterBackfillNotEnabledMessages(messages)
- const { quotaAllowedMessages, quotaDroppedMessages } =
- await this.filterQuotaLimitedMessages(backfillAllowedMessages)
+ const internalMessages: LogsIngestionMessage[] = []
+ const billedMessages: LogsIngestionMessage[] = []
+ for (const message of backfillAllowedMessages) {
+ ;(this.isUnbilledInternal(message) ? internalMessages : billedMessages).push(message)
+ }
+ const { quotaAllowedMessages, quotaDroppedMessages } = await this.filterQuotaLimitedMessages(billedMessages)
const { rateLimiterAllowedMessages, rateLimiterDroppedMessages } =
await this.filterRateLimitedMessages(quotaAllowedMessages)
const usageStats = this.trackOutgoingTrafficAndBuildUsageStats(rateLimiterAllowedMessages, [
- ...backfillDroppedMessages,
+ ...backfillDroppedMessages.filter((message) => !this.isUnbilledInternal(message)),
...quotaDroppedMessages,
...rateLimiterDroppedMessages,
])
@@ -707,6 +756,11 @@ export class LogsIngestionConsumer {
usageStats,
transformationBatchBudget
)
+ // A usage map that is never emitted keeps their stats off the team's billed row.
+ // They run after the billed batches, so the per-batch concurrency limit still holds.
+ await this.processAndProduceLogMessages(internalMessages, new Map(), transformationBatchBudget)
+ // Metric rules and internal usage emit deltas, so they wait until no group can fail the batch into a replay.
+ this.queueInternalLogsMetrics(internalMessages)
await this.emitMetricRuleTallies(metricTalliesByTeam)
await this.deps.teamManager.setTeamsIngestedData(
rateLimiterAllowedMessages.map((message) => message.teamId)
@@ -719,7 +773,19 @@ export class LogsIngestionConsumer {
})
}
})(),
- messages: rateLimiterAllowedMessages,
+ messages: [...rateLimiterAllowedMessages, ...internalMessages],
+ }
+ }
+
+ // The usage report matches `bytes_ingested` exactly, so these names never reach billing.
+ private queueInternalLogsMetrics(internalMessages: LogsIngestionMessage[]): void {
+ for (const message of internalMessages) {
+ this.queueUsageMetric(message.teamId, 'bytes_ingested_internal', message.bytesUncompressed)
+ this.queueUsageMetric(message.teamId, 'records_ingested_internal', message.recordCount)
+ const teamId = message.teamId.toString()
+ logsInternalBytesCounter.inc({ team_id: teamId }, message.bytesUncompressed)
+ logsInternalRecordsCounter.inc({ team_id: teamId }, message.recordCount)
+ recordInternalLogs(message.teamId, message.bytesUncompressed, message.recordCount)
}
}
@@ -727,6 +793,10 @@ export class LogsIngestionConsumer {
let totalBytesReceived = 0
let totalRecordsReceived = 0
for (const message of messages) {
+ // Never quota or rate limited, so counting them would show as drops in received minus allowed.
+ if (this.isUnbilledInternal(message)) {
+ continue
+ }
totalBytesReceived += message.bytesUncompressed
totalRecordsReceived += message.recordCount
}
@@ -804,13 +874,16 @@ export class LogsIngestionConsumer {
} {
const backfillAllowedMessages: LogsIngestionMessage[] = []
const backfillDroppedMessages: LogsIngestionMessage[] = []
- const droppedRecordsByTeam = new Map<number, { messages: number; records: number }>()
+ const droppedRecordsByTeam = new Map<number, { messages: number; records: number; billed: boolean }>()
for (const message of messages) {
if (message.backfillRequested && !teamIdMatchesCsv(this.backfillEnabledTeamsRaw, message.teamId)) {
backfillDroppedMessages.push(message)
- const dropped = droppedRecordsByTeam.get(message.teamId) ?? { messages: 0, records: 0 }
+ const dropped = droppedRecordsByTeam.get(message.teamId) ?? { messages: 0, records: 0, billed: false }
dropped.messages++
- dropped.records += message.recordCount
+ if (!this.isUnbilledInternal(message)) {
+ dropped.records += message.recordCount
+ dropped.billed = true
+ }
droppedRecordsByTeam.set(message.teamId, dropped)
} else {
backfillAllowedMessages.push(message)
@@ -823,7 +896,9 @@ export class LogsIngestionConsumer {
dropped.messages
)
recordLogMessageDropped('backfill_not_enabled', teamId.toString(), dropped.messages)
- this.queueUsageMetric(teamId, 'records_dropped_backfill_not_enabled', dropped.records)
+ if (dropped.billed) {
+ this.queueUsageMetric(teamId, 'records_dropped_backfill_not_enabled', dropped.records)
+ }
}
return { backfillAllowedMessages, backfillDroppedMessages }
@@ -894,7 +969,7 @@ export class LogsIngestionConsumer {
byTeam: MetricTalliesByTeam,
message: LogsIngestionMessage
): Promise<{ token: string; rules: CompiledMetricRule[]; tallies: BatchTallies } | null> {
- if (!this.isMetricRulesEnabledForTeam(message.teamId)) {
+ if (!this.appliesTeamLogRules(message) || !this.isMetricRulesEnabledForTeam(message.teamId)) {
return null
}
let state = byTeam.get(message.teamId)
@@ -1082,11 +1157,13 @@ export class LogsIngestionConsumer {
message.recordCount
)
- const teamIdLabel = message.teamId.toString()
- if (contentCredit > 0) {
- logsBillingBytesCreditedCounter.inc({ team_id: teamIdLabel }, contentCredit)
+ if (!this.isUnbilledInternal(message)) {
+ const teamIdLabel = message.teamId.toString()
+ if (contentCredit > 0) {
+ logsBillingBytesCreditedCounter.inc({ team_id: teamIdLabel }, contentCredit)
+ }
+ logsBillingRecordsCreditedCounter.inc({ team_id: teamIdLabel }, resolved.recordsDropped)
}
- logsBillingRecordsCreditedCounter.inc({ team_id: teamIdLabel }, resolved.recordsDropped)
if (message.recordCount > 0) {
logsDropBatchRowsHistogram.observe(message.recordCount)
}
@@ -1150,7 +1227,10 @@ export class LogsIngestionConsumer {
value: processedValue,
key: null,
headers: {
- ...parseKafkaHeaders(message.message.headers),
+ ...withoutInternalLogsHeaders(parseKafkaHeaders(message.message.headers)),
+ ...(message.internalLogs !== undefined
+ ? { [INTERNAL_SOURCE_HEADER]: message.internalLogs.source }
+ : {}),
token: message.token,
team_id: message.teamId.toString(),
'json-parse': jsonParse.toString(),
@@ -1483,6 +1563,11 @@ export class LogsIngestionConsumer {
recordCount,
minTimestampMicros: parseMinTimestampHeader(headers.min_timestamp),
backfillRequested: headers.backfill_days !== undefined,
+ internalLogs: this.internalLogsVerifier.verify(
+ headers[INTERNAL_LOGS_HEADER],
+ team.id,
+ message.value
+ ),
})
} catch (e) {
// A message we cannot parse is message-scoped and will fail the same way on
diff --git a/nodejs/src/logs/traces-ingestion-consumer.ts b/nodejs/src/logs/traces-ingestion-consumer.ts
index b336b919bc9f..cae16c23e4de 100644
--- a/nodejs/src/logs/traces-ingestion-consumer.ts
+++ b/nodejs/src/logs/traces-ingestion-consumer.ts
@@ -54,6 +54,9 @@ export class TracesIngestionConsumer extends LogsIngestionConsumer {
// independently of log retention.
LOGS_RETENTION_ENABLED_TEAMS: config.TRACES_RETENTION_ENABLED_TEAMS,
LOGS_RETENTION_KILLSWITCH: config.TRACES_RETENTION_KILLSWITCH,
+ // Internal batches are logs only, so a signed traces batch is still billed.
+ LOGS_INTERNAL_INGESTION_SIGNING_KEYS: '',
+ LOGS_INTERNAL_INGESTION_ENABLED_TEAMS: '',
},
// Own Redis key namespace so traces token buckets don't share per-team state with logs.
'traces-rate-limiter'
diff --git a/nodejs/src/logs/types.ts b/nodejs/src/logs/types.ts
index 51bf8a1f3f21..7af2c810b8bb 100644
--- a/nodejs/src/logs/types.ts
+++ b/nodejs/src/logs/types.ts
@@ -1,5 +1,7 @@
import { Message } from 'node-rdkafka'
+import type { InternalLogs } from './internal-logs'
+
export type LogsIngestionMessage = {
token: string
teamId: number
@@ -12,4 +14,5 @@ export type LogsIngestionMessage = {
/** Earliest row timestamp in epoch microseconds, from the `min_timestamp` header; absent from older producers. */
minTimestampMicros?: number
backfillRequested?: boolean
+ internalLogs?: InternalLogs
}