:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(logs): sign internal logs over source, team and time (#123929)
Jon McCallum7550aea8605e
diff --git a/nodejs/src/logs/internal-logs.test.ts b/nodejs/src/logs/internal-logs.test.tsindex 11e8a00507f3..e2c03ee3f9d4 100644--- a/nodejs/src/logs/internal-logs.test.ts+++ b/nodejs/src/logs/internal-logs.test.ts@@ -5,26 +5,32 @@ const OTHER_KEY = 'internal-logs-other-key'const SOURCE = 'hog_flow'const TEAM_ID = 2const OTHER_TEAM_ID = 3-const PAYLOAD = Buffer.from('avro container bytes')+const SIGNED_AT = new Date('2026-10-27T12:00:00Z')+const CREATED_AT = '2026-10-27T12:01:00.123456789+00:00'describe('InternalLogsVerifier', () => {it.each([{name: 'accepts a batch signed with the newest key',- signature: signInternalLogs(KEY, SOURCE, TEAM_ID, PAYLOAD),+ signature: signInternalLogs(KEY, SOURCE, TEAM_ID, SIGNED_AT),},{name: 'accepts a batch signed with an older key during rotation',keys: `${OTHER_KEY},${KEY}`,- signature: signInternalLogs(KEY, SOURCE, TEAM_ID, PAYLOAD),+ signature: signInternalLogs(KEY, SOURCE, TEAM_ID, SIGNED_AT),},{name: 'accepts a batch on a wildcard allowlist',teams: '*',- signature: signInternalLogs(KEY, SOURCE, TEAM_ID, PAYLOAD),+ signature: signInternalLogs(KEY, SOURCE, TEAM_ID, SIGNED_AT),},- ])('$name', ({ signature, keys = KEY, teams = String(TEAM_ID) }) => {- expect(new InternalLogsVerifier(keys, teams).verify(signature, TEAM_ID, PAYLOAD)).toEqual({+ {+ name: 'accepts a producer clock that runs ahead of capture',+ createdAt: '2026-10-27T11:56:00Z',+ signature: signInternalLogs(KEY, SOURCE, TEAM_ID, SIGNED_AT),+ },+ ])('$name', ({ signature, keys = KEY, teams = String(TEAM_ID), createdAt = CREATED_AT }) => {+ expect(new InternalLogsVerifier(keys, teams).verify(signature, TEAM_ID, createdAt)).toEqual({source: SOURCE,billed: false,teamRules: false,@@ -34,41 +40,51 @@ describe('InternalLogsVerifier', () => {it.each([{ name: 'no header', signature: undefined },{ name: 'a malformed signature', signature: 'v1.not-a-signature' },- { name: 'an unknown key', signature: signInternalLogs(OTHER_KEY, SOURCE, TEAM_ID, PAYLOAD) },- { name: "another team's signature", signature: signInternalLogs(KEY, SOURCE, OTHER_TEAM_ID, PAYLOAD) },+ { name: 'an unknown key', signature: signInternalLogs(OTHER_KEY, SOURCE, TEAM_ID, SIGNED_AT) },+ { name: "another team's signature", signature: signInternalLogs(KEY, SOURCE, OTHER_TEAM_ID, SIGNED_AT) },{name: 'a signature moved to another source',- signature: signInternalLogs(KEY, SOURCE, TEAM_ID, PAYLOAD).replace(`.${SOURCE}.`, '.hog_function.'),- },- {- name: 'a signature over different data',- signature: signInternalLogs(KEY, SOURCE, TEAM_ID, Buffer.from('other')),+ signature: signInternalLogs(KEY, SOURCE, TEAM_ID, SIGNED_AT).replace(`.${SOURCE}.`, '.hog_function.'),},{name: 'a team not on the allowlist',teams: String(OTHER_TEAM_ID),- signature: signInternalLogs(KEY, SOURCE, TEAM_ID, PAYLOAD),+ signature: signInternalLogs(KEY, SOURCE, TEAM_ID, SIGNED_AT),+ },+ { name: 'no configured key', keys: '', signature: signInternalLogs(KEY, SOURCE, TEAM_ID, SIGNED_AT) },+ { name: 'a source with no policy', signature: signInternalLogs(KEY, 'not_a_source', TEAM_ID, SIGNED_AT) },+ {+ name: 'an inherited property as source',+ signature: signInternalLogs(KEY, 'constructor', TEAM_ID, SIGNED_AT),+ },+ { name: 'the prototype as source', signature: signInternalLogs(KEY, '__proto__', TEAM_ID, SIGNED_AT) },+ {+ name: 'a signature replayed after the window',+ createdAt: '2026-10-27T12:05:01Z',+ signature: signInternalLogs(KEY, SOURCE, TEAM_ID, SIGNED_AT),+ },+ { name: 'no created_at header', createdAt: null, signature: signInternalLogs(KEY, SOURCE, TEAM_ID, SIGNED_AT) },+ {+ name: 'an unparseable created_at header',+ createdAt: 'not-a-date',+ signature: signInternalLogs(KEY, SOURCE, TEAM_ID, SIGNED_AT),},- { name: 'no configured key', keys: '', signature: signInternalLogs(KEY, SOURCE, TEAM_ID, PAYLOAD) },- { name: 'a source with no policy', signature: signInternalLogs(KEY, 'not_a_source', TEAM_ID, PAYLOAD) },- { name: 'an inherited property as source', signature: signInternalLogs(KEY, 'constructor', TEAM_ID, PAYLOAD) },- { name: 'the prototype as source', signature: signInternalLogs(KEY, '__proto__', TEAM_ID, PAYLOAD) },- { name: 'an empty payload', payload: null, signature: signInternalLogs(KEY, SOURCE, TEAM_ID, PAYLOAD) },])('bills a batch with $name',- ({ signature, keys = KEY, teams = String(TEAM_ID), payload = PAYLOAD as Buffer | null }) => {- expect(new InternalLogsVerifier(keys, teams).verify(signature, TEAM_ID, payload)).toBeUndefined()+ ({ signature, keys = KEY, teams = String(TEAM_ID), createdAt = CREATED_AT as string | null }) => {+ const verifier = new InternalLogsVerifier(keys, teams)+ expect(verifier.verify(signature, TEAM_ID, createdAt ?? undefined)).toBeUndefined()})it('refuses to sign a source that cannot go into the header', () => {- expect(() => signInternalLogs(KEY, 'hog.flow', TEAM_ID, PAYLOAD)).toThrow('invalid source')+ expect(() => signInternalLogs(KEY, 'hog.flow', TEAM_ID, SIGNED_AT)).toThrow('invalid source')})// Producers and consumers deploy separately, so a format change must be a new version prefix.it('keeps the v1 signature format stable', () => {- expect(signInternalLogs(KEY, SOURCE, TEAM_ID, PAYLOAD)).toEqual(- 'v1.hog_flow.b97466e15a9bf1a2658e8d92e5f63ed60df91bd4e838e4ab4426cbb5280d2055'+ expect(signInternalLogs(KEY, SOURCE, TEAM_ID, SIGNED_AT)).toEqual(+ 'v1.hog_flow.1793102400.af443fe982b21dc42052d7d373d7b135c1d709955e8d5309a1bf84a2e8583c3d')})})diff --git a/nodejs/src/logs/internal-logs.ts b/nodejs/src/logs/internal-logs.tsindex 911cbddf9f19..f06263a7f080 100644--- a/nodejs/src/logs/internal-logs.ts+++ b/nodejs/src/logs/internal-logs.ts@@ -1,4 +1,4 @@-import { createHash, createHmac, timingSafeEqual } from 'crypto'+import { createHmac, timingSafeEqual } from 'crypto'import { Counter } from 'prom-client'import { buildIntegerMatcher } from '~/common/config/matchers'@@ -12,6 +12,12 @@ const SIGNATURE_VERSION = 'v1'// No dot, because a dot divides the parts of the header value.const SOURCE_PATTERN = /^[a-z0-9_]{1,64}$/+// Epoch seconds. A strict shape keeps milliseconds, signs and padding out of the freshness check.+const SIGNED_AT_PATTERN = /^[1-9][0-9]{9}$/++// Measured against capture's `created_at`, not this consumer's clock, so consumer lag and DLQ replays+// do not expire a signature. Covers producer and capture clock skew and a short producer retry.+const SIGNATURE_WINDOW_SECONDS = 5 * 60export type InternalSourcePolicy = {/** Counts toward the team's logs usage, quota and rate limits. */@@ -30,11 +36,16 @@ export const INTERNAL_SOURCE_POLICIES: Readonly<Record<string, InternalSourcePol// hog_function: { billed: false, teamRules: false },}-type RejectReason = 'not_configured' | 'team_not_allowed' | 'invalid_signature' | 'unknown_source'+type NotInternalReason =+ | 'not_configured'+ | 'team_not_allowed'+ | 'invalid_signature'+ | 'stale_signature'+ | 'unknown_source'const logsInternalLogsCounter = new Counter({name: 'logs_ingestion_internal_logs_total',- help: 'Batches that carried the posthog-internal header, by outcome. A rejected batch is billed as normal.',+ help: 'Batches that carried the posthog-internal header, by outcome. A not_internal batch is billed as normal.',labelNames: ['outcome', 'reason'],})@@ -45,19 +56,19 @@ function parseKeys(commaSeparatedKeys: string): string[] {.filter((key) => key.length > 0)}-function sha256Hex(payload: Buffer): string {- return createHash('sha256').update(payload).digest('hex')-}--// No expiry, because a batch that waits in Kafka through consumer lag or a DLQ replay must stay internal.-function computeSignature(key: string, source: string, teamId: number, payloadSha256: string): string {- const mac = createHmac('sha256', key)- .update(`${SIGNATURE_VERSION}|${source}|${teamId}|${payloadSha256}`)- .digest('hex')- return `${SIGNATURE_VERSION}.${source}.${mac}`+// No payload hash, because capture-logs re-encodes the request and the producer never sees the Kafka bytes.+function computeSignature(key: string, source: string, teamId: number, signedAt: string): string {+ const mac = createHmac('sha256', key).update(`${SIGNATURE_VERSION}|${source}|${teamId}|${signedAt}`).digest('hex')+ return `${SIGNATURE_VERSION}.${source}.${signedAt}.${mac}`}-export function signInternalLogs(commaSeparatedKeys: string, source: string, teamId: number, payload: Buffer): string {+/** Sign once per HTTP attempt, because a retried signature can fall outside the window and be billed. */+export function signInternalLogs(+ commaSeparatedKeys: string,+ source: string,+ teamId: number,+ signedAt: Date = new Date()+): string {const [newestKey] = parseKeys(commaSeparatedKeys)if (!newestKey) {throw new Error('Cannot sign posthog-internal batch: no signing key configured')@@ -65,7 +76,7 @@ export function signInternalLogs(commaSeparatedKeys: string, source: string, teaif (!SOURCE_PATTERN.test(source)) {throw new Error(`Cannot sign posthog-internal batch: invalid source ${JSON.stringify(source)}`)}- return computeSignature(newestKey, source, teamId, sha256Hex(payload))+ return computeSignature(newestKey, source, teamId, String(Math.floor(signedAt.getTime() / 1000)))}// Only a verified batch may set the `internal_source` column, so an incoming copy of the header is removed.@@ -87,44 +98,50 @@ export class InternalLogsVerifier {this.isTeamAllowed = buildIntegerMatcher(enabledTeams, true)}- verify(signature: string | undefined, teamId: number, payload: Buffer | null): InternalLogs | undefined {+ /** `createdAt` is the `created_at` Kafka header, which capture-logs sets when it receives the request. */+ verify(signature: string | undefined, teamId: number, createdAt: string | undefined): InternalLogs | undefined {if (signature === undefined) {return undefined}- const source = signature.split('.')[1] ?? ''- const reason = this.rejectReason(signature, source, teamId, payload)+ const [, source = '', signedAt = ''] = signature.split('.')+ const reason = this.notInternalReason(signature, source, signedAt, teamId, createdAt)// Own keys only, because a source like `constructor` resolves to an inherited policy with no `billed`.const policy = reason || !Object.hasOwn(this.policies, source) ? undefined : this.policies[source]if (!policy) {- logsInternalLogsCounter.inc({ outcome: 'rejected', reason: reason ?? 'unknown_source' })+ logsInternalLogsCounter.inc({ outcome: 'not_internal', reason: reason ?? 'unknown_source' })return undefined}- logsInternalLogsCounter.inc({ outcome: 'accepted', reason: '' })+ logsInternalLogsCounter.inc({ outcome: 'internal', reason: '' })return { source, ...policy }}- private rejectReason(+ private notInternalReason(signature: string,source: string,+ signedAt: string,teamId: number,- payload: Buffer | null- ): RejectReason | null {+ createdAt: string | undefined+ ): NotInternalReason | null {if (this.keys.length === 0) {return 'not_configured'}if (!this.isTeamAllowed(teamId)) {return 'team_not_allowed'}- if (payload === null || !SOURCE_PATTERN.test(source)) {+ if (!SOURCE_PATTERN.test(source) || !SIGNED_AT_PATTERN.test(signedAt)) {return 'invalid_signature'}const received = Buffer.from(signature)- const payloadSha256 = sha256Hex(payload)// Every key is tried, so a rotation can add a key before producers switch to it.const matches = this.keys.some((key) => {- const expected = Buffer.from(computeSignature(key, source, teamId, payloadSha256))+ const expected = Buffer.from(computeSignature(key, source, teamId, signedAt))return expected.length === received.length && timingSafeEqual(expected, received)})- return matches ? null : 'invalid_signature'+ if (!matches) {+ return 'invalid_signature'+ }+ // A missing or unparseable `created_at` gives NaN, which fails this comparison and bills the batch.+ const skewSeconds = Math.abs(Date.parse(createdAt ?? '') / 1000 - Number(signedAt))+ return skewSeconds <= SIGNATURE_WINDOW_SECONDS ? null : 'stale_signature'}}diff --git a/nodejs/src/logs/logs-ingestion-consumer.test.ts b/nodejs/src/logs/logs-ingestion-consumer.test.tsindex 50254b69275d..970a87891be8 100644--- a/nodejs/src/logs/logs-ingestion-consumer.test.ts+++ b/nodejs/src/logs/logs-ingestion-consumer.test.ts@@ -62,6 +62,15 @@ import type { SamplingRulesCache } from './sampling/sampling-rules-cache'import { TracesIngestionConsumer } from './traces-ingestion-consumer'import { LogsTransformerService } from './transformations/logs-transformer.service'+// What capture-logs forwards for a producer that signed just before the request arrived. Both times come+// from `Date.now()`, which these tests pin, because a bare `new Date()` still reads the real clock.+function internalLogsHeaders(signature: string): Record<string, Buffer> {+ return {+ [INTERNAL_LOGS_HEADER]: Buffer.from(signature),+ created_at: Buffer.from(new Date(Date.now()).toISOString()),+ }+}+const DEFAULT_TEST_TIMEOUT = 5000jest.setTimeout(DEFAULT_TEST_TIMEOUT)@@ -1093,9 +1102,9 @@ describe('LogsIngestionConsumer', () => {record_count: '1',})const signature = signed- ? signInternalLogs(SIGNING_KEY, 'hog_flow', team.id, message.value!)+ ? signInternalLogs(SIGNING_KEY, 'hog_flow', team.id, new Date(Date.now())): 'v1.hog_flow.not-a-signature'- message.headers!.push({ [INTERNAL_LOGS_HEADER]: Buffer.from(signature) })+ message.headers!.push(internalLogsHeaders(signature))const recordLogsReceivedSpy = jest.spyOn(otelMetrics, 'recordLogsReceived')const recordInternalLogsSpy = jest.spyOn(otelMetrics, 'recordInternalLogs')@@ -1145,9 +1154,9 @@ describe('LogsIngestionConsumer', () => {min_timestamp: fiveDaysAgoMicros.toString(),})- message.headers!.push({- [INTERNAL_LOGS_HEADER]: Buffer.from(signInternalLogs(SIGNING_KEY, 'hog_flow', team.id, message.value!)),- })+ message.headers!.push(+ internalLogsHeaders(signInternalLogs(SIGNING_KEY, 'hog_flow', team.id, new Date(Date.now())))+ )const bytesCreditSpy = jest.spyOn(logsBillingBytesCreditedCounter, 'inc')const recordsCreditSpy = jest.spyOn(logsBillingRecordsCreditedCounter, 'inc')@@ -1180,11 +1189,9 @@ describe('LogsIngestionConsumer', () => {record_count: '1',...(backfill ? { backfill_days: '540' } : {}),})- internal.headers!.push({- [INTERNAL_LOGS_HEADER]: Buffer.from(- signInternalLogs(SIGNING_KEY, 'hog_flow', team.id, internal.value!)- ),- })+ internal.headers!.push(+ internalLogsHeaders(signInternalLogs(SIGNING_KEY, 'hog_flow', team.id, new Date(Date.now())))+ )const originalQueueMessages = mockProducer.queueMessagesconst passthrough = originalQueueMessages.bind(mockProducer)mockProducer.queueMessages = jest.fn().mockImplementation((topicMessages) => {@@ -2530,9 +2537,9 @@ describe('LogsIngestionConsumer', () => {const [billed, internal] = await createKafkaMessages([createLogMessage(), createLogMessage()], {token: team.api_token,})- internal.headers!.push({- [INTERNAL_LOGS_HEADER]: Buffer.from(signInternalLogs(key, 'hog_flow', team.id, internal.value!)),- })+ internal.headers!.push(+ internalLogsHeaders(signInternalLogs(key, 'hog_flow', team.id, new Date(Date.now())))+ )const originalQueueMessages = mockProducer.queueMessagesconst passthrough = originalQueueMessages.bind(mockProducer)@@ -2690,9 +2697,7 @@ describe('LogsIngestionConsumer', () => {await consumer.start()jest.mocked(hub.quotaLimiting.isTeamTokenQuotaLimited).mockResolvedValue(true)const message = await createKafkaMessage(createLogMessage(), { token: team.api_token })- message.headers!.push({- [INTERNAL_LOGS_HEADER]: Buffer.from(signInternalLogs(key, 'hog_flow', team.id, message.value!)),- })+ message.headers!.push(internalLogsHeaders(signInternalLogs(key, 'hog_flow', team.id, new Date(Date.now()))))await waitForBackgroundTasks(consumer.processKafkaBatch([message]))diff --git a/nodejs/src/logs/logs-ingestion-consumer.ts b/nodejs/src/logs/logs-ingestion-consumer.tsindex c4d7129856c6..e3efe0513368 100644--- a/nodejs/src/logs/logs-ingestion-consumer.ts+++ b/nodejs/src/logs/logs-ingestion-consumer.ts@@ -1566,7 +1566,7 @@ export class LogsIngestionConsumer {internalLogs: this.internalLogsVerifier.verify(headers[INTERNAL_LOGS_HEADER],team.id,- message.value+ headers.created_at),})} catch (e) {