Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(logs): sign internal logs over source, team and time (#123929)

Jon McCallum7550aea8605e
diff --git a/nodejs/src/logs/internal-logs.test.ts b/nodejs/src/logs/internal-logs.test.ts
index 11e8a00507f3..e2c03ee3f9d4 100644
--- a/nodejs/src/logs/internal-logs.test.ts
+++ b/nodejs/src/logs/internal-logs.test.ts
@@ -5,26 +5,32 @@ const OTHER_KEY = 'internal-logs-other-key'
const SOURCE = 'hog_flow'
const TEAM_ID = 2
const OTHER_TEAM_ID = 3
-const PAYLOAD = Buffer.from('avro container bytes')
+const SIGNED_AT = new Date('2026-10-27T12:00:00Z')
+const CREATED_AT = '2026-10-27T12:01:00.123456789+00:00'
describe('InternalLogsVerifier', () => {
it.each([
{
name: 'accepts a batch signed with the newest key',
- signature: signInternalLogs(KEY, SOURCE, TEAM_ID, PAYLOAD),
+ signature: signInternalLogs(KEY, SOURCE, TEAM_ID, SIGNED_AT),
},
{
name: 'accepts a batch signed with an older key during rotation',
keys: `${OTHER_KEY},${KEY}`,
- signature: signInternalLogs(KEY, SOURCE, TEAM_ID, PAYLOAD),
+ signature: signInternalLogs(KEY, SOURCE, TEAM_ID, SIGNED_AT),
},
{
name: 'accepts a batch on a wildcard allowlist',
teams: '*',
- signature: signInternalLogs(KEY, SOURCE, TEAM_ID, PAYLOAD),
+ signature: signInternalLogs(KEY, SOURCE, TEAM_ID, SIGNED_AT),
},
- ])('$name', ({ signature, keys = KEY, teams = String(TEAM_ID) }) => {
- expect(new InternalLogsVerifier(keys, teams).verify(signature, TEAM_ID, PAYLOAD)).toEqual({
+ {
+ name: 'accepts a producer clock that runs ahead of capture',
+ createdAt: '2026-10-27T11:56:00Z',
+ signature: signInternalLogs(KEY, SOURCE, TEAM_ID, SIGNED_AT),
+ },
+ ])('$name', ({ signature, keys = KEY, teams = String(TEAM_ID), createdAt = CREATED_AT }) => {
+ expect(new InternalLogsVerifier(keys, teams).verify(signature, TEAM_ID, createdAt)).toEqual({
source: SOURCE,
billed: false,
teamRules: false,
@@ -34,41 +40,51 @@ describe('InternalLogsVerifier', () => {
it.each([
{ name: 'no header', signature: undefined },
{ name: 'a malformed signature', signature: 'v1.not-a-signature' },
- { name: 'an unknown key', signature: signInternalLogs(OTHER_KEY, SOURCE, TEAM_ID, PAYLOAD) },
- { name: "another team's signature", signature: signInternalLogs(KEY, SOURCE, OTHER_TEAM_ID, PAYLOAD) },
+ { name: 'an unknown key', signature: signInternalLogs(OTHER_KEY, SOURCE, TEAM_ID, SIGNED_AT) },
+ { name: "another team's signature", signature: signInternalLogs(KEY, SOURCE, OTHER_TEAM_ID, SIGNED_AT) },
{
name: 'a signature moved to another source',
- signature: signInternalLogs(KEY, SOURCE, TEAM_ID, PAYLOAD).replace(`.${SOURCE}.`, '.hog_function.'),
- },
- {
- name: 'a signature over different data',
- signature: signInternalLogs(KEY, SOURCE, TEAM_ID, Buffer.from('other')),
+ signature: signInternalLogs(KEY, SOURCE, TEAM_ID, SIGNED_AT).replace(`.${SOURCE}.`, '.hog_function.'),
},
{
name: 'a team not on the allowlist',
teams: String(OTHER_TEAM_ID),
- signature: signInternalLogs(KEY, SOURCE, TEAM_ID, PAYLOAD),
+ signature: signInternalLogs(KEY, SOURCE, TEAM_ID, SIGNED_AT),
+ },
+ { name: 'no configured key', keys: '', signature: signInternalLogs(KEY, SOURCE, TEAM_ID, SIGNED_AT) },
+ { name: 'a source with no policy', signature: signInternalLogs(KEY, 'not_a_source', TEAM_ID, SIGNED_AT) },
+ {
+ name: 'an inherited property as source',
+ signature: signInternalLogs(KEY, 'constructor', TEAM_ID, SIGNED_AT),
+ },
+ { name: 'the prototype as source', signature: signInternalLogs(KEY, '__proto__', TEAM_ID, SIGNED_AT) },
+ {
+ name: 'a signature replayed after the window',
+ createdAt: '2026-10-27T12:05:01Z',
+ signature: signInternalLogs(KEY, SOURCE, TEAM_ID, SIGNED_AT),
+ },
+ { name: 'no created_at header', createdAt: null, signature: signInternalLogs(KEY, SOURCE, TEAM_ID, SIGNED_AT) },
+ {
+ name: 'an unparseable created_at header',
+ createdAt: 'not-a-date',
+ signature: signInternalLogs(KEY, SOURCE, TEAM_ID, SIGNED_AT),
},
- { name: 'no configured key', keys: '', signature: signInternalLogs(KEY, SOURCE, TEAM_ID, PAYLOAD) },
- { name: 'a source with no policy', signature: signInternalLogs(KEY, 'not_a_source', TEAM_ID, PAYLOAD) },
- { name: 'an inherited property as source', signature: signInternalLogs(KEY, 'constructor', TEAM_ID, PAYLOAD) },
- { name: 'the prototype as source', signature: signInternalLogs(KEY, '__proto__', TEAM_ID, PAYLOAD) },
- { name: 'an empty payload', payload: null, signature: signInternalLogs(KEY, SOURCE, TEAM_ID, PAYLOAD) },
])(
'bills a batch with $name',
- ({ signature, keys = KEY, teams = String(TEAM_ID), payload = PAYLOAD as Buffer | null }) => {
- expect(new InternalLogsVerifier(keys, teams).verify(signature, TEAM_ID, payload)).toBeUndefined()
+ ({ signature, keys = KEY, teams = String(TEAM_ID), createdAt = CREATED_AT as string | null }) => {
+ const verifier = new InternalLogsVerifier(keys, teams)
+ expect(verifier.verify(signature, TEAM_ID, createdAt ?? undefined)).toBeUndefined()
}
)
it('refuses to sign a source that cannot go into the header', () => {
- expect(() => signInternalLogs(KEY, 'hog.flow', TEAM_ID, PAYLOAD)).toThrow('invalid source')
+ expect(() => signInternalLogs(KEY, 'hog.flow', TEAM_ID, SIGNED_AT)).toThrow('invalid source')
})
// Producers and consumers deploy separately, so a format change must be a new version prefix.
it('keeps the v1 signature format stable', () => {
- expect(signInternalLogs(KEY, SOURCE, TEAM_ID, PAYLOAD)).toEqual(
- 'v1.hog_flow.b97466e15a9bf1a2658e8d92e5f63ed60df91bd4e838e4ab4426cbb5280d2055'
+ expect(signInternalLogs(KEY, SOURCE, TEAM_ID, SIGNED_AT)).toEqual(
+ 'v1.hog_flow.1793102400.af443fe982b21dc42052d7d373d7b135c1d709955e8d5309a1bf84a2e8583c3d'
)
})
})
diff --git a/nodejs/src/logs/internal-logs.ts b/nodejs/src/logs/internal-logs.ts
index 911cbddf9f19..f06263a7f080 100644
--- a/nodejs/src/logs/internal-logs.ts
+++ b/nodejs/src/logs/internal-logs.ts
@@ -1,4 +1,4 @@
-import { createHash, createHmac, timingSafeEqual } from 'crypto'
+import { createHmac, timingSafeEqual } from 'crypto'
import { Counter } from 'prom-client'
import { buildIntegerMatcher } from '~/common/config/matchers'
@@ -12,6 +12,12 @@ const SIGNATURE_VERSION = 'v1'
// No dot, because a dot divides the parts of the header value.
const SOURCE_PATTERN = /^[a-z0-9_]{1,64}$/
+// Epoch seconds. A strict shape keeps milliseconds, signs and padding out of the freshness check.
+const SIGNED_AT_PATTERN = /^[1-9][0-9]{9}$/
+
+// Measured against capture's `created_at`, not this consumer's clock, so consumer lag and DLQ replays
+// do not expire a signature. Covers producer and capture clock skew and a short producer retry.
+const SIGNATURE_WINDOW_SECONDS = 5 * 60
export type InternalSourcePolicy = {
/** Counts toward the team's logs usage, quota and rate limits. */
@@ -30,11 +36,16 @@ export const INTERNAL_SOURCE_POLICIES: Readonly<Record<string, InternalSourcePol
// hog_function: { billed: false, teamRules: false },
}
-type RejectReason = 'not_configured' | 'team_not_allowed' | 'invalid_signature' | 'unknown_source'
+type NotInternalReason =
+ | 'not_configured'
+ | 'team_not_allowed'
+ | 'invalid_signature'
+ | 'stale_signature'
+ | 'unknown_source'
const logsInternalLogsCounter = new Counter({
name: 'logs_ingestion_internal_logs_total',
- help: 'Batches that carried the posthog-internal header, by outcome. A rejected batch is billed as normal.',
+ help: 'Batches that carried the posthog-internal header, by outcome. A not_internal batch is billed as normal.',
labelNames: ['outcome', 'reason'],
})
@@ -45,19 +56,19 @@ function parseKeys(commaSeparatedKeys: string): string[] {
.filter((key) => key.length > 0)
}
-function sha256Hex(payload: Buffer): string {
- return createHash('sha256').update(payload).digest('hex')
-}
-
-// No expiry, because a batch that waits in Kafka through consumer lag or a DLQ replay must stay internal.
-function computeSignature(key: string, source: string, teamId: number, payloadSha256: string): string {
- const mac = createHmac('sha256', key)
- .update(`${SIGNATURE_VERSION}|${source}|${teamId}|${payloadSha256}`)
- .digest('hex')
- return `${SIGNATURE_VERSION}.${source}.${mac}`
+// No payload hash, because capture-logs re-encodes the request and the producer never sees the Kafka bytes.
+function computeSignature(key: string, source: string, teamId: number, signedAt: string): string {
+ const mac = createHmac('sha256', key).update(`${SIGNATURE_VERSION}|${source}|${teamId}|${signedAt}`).digest('hex')
+ return `${SIGNATURE_VERSION}.${source}.${signedAt}.${mac}`
}
-export function signInternalLogs(commaSeparatedKeys: string, source: string, teamId: number, payload: Buffer): string {
+/** Sign once per HTTP attempt, because a retried signature can fall outside the window and be billed. */
+export function signInternalLogs(
+ commaSeparatedKeys: string,
+ source: string,
+ teamId: number,
+ signedAt: Date = new Date()
+): string {
const [newestKey] = parseKeys(commaSeparatedKeys)
if (!newestKey) {
throw new Error('Cannot sign posthog-internal batch: no signing key configured')
@@ -65,7 +76,7 @@ export function signInternalLogs(commaSeparatedKeys: string, source: string, tea
if (!SOURCE_PATTERN.test(source)) {
throw new Error(`Cannot sign posthog-internal batch: invalid source ${JSON.stringify(source)}`)
}
- return computeSignature(newestKey, source, teamId, sha256Hex(payload))
+ return computeSignature(newestKey, source, teamId, String(Math.floor(signedAt.getTime() / 1000)))
}
// Only a verified batch may set the `internal_source` column, so an incoming copy of the header is removed.
@@ -87,44 +98,50 @@ export class InternalLogsVerifier {
this.isTeamAllowed = buildIntegerMatcher(enabledTeams, true)
}
- verify(signature: string | undefined, teamId: number, payload: Buffer | null): InternalLogs | undefined {
+ /** `createdAt` is the `created_at` Kafka header, which capture-logs sets when it receives the request. */
+ verify(signature: string | undefined, teamId: number, createdAt: string | undefined): InternalLogs | undefined {
if (signature === undefined) {
return undefined
}
- const source = signature.split('.')[1] ?? ''
- const reason = this.rejectReason(signature, source, teamId, payload)
+ const [, source = '', signedAt = ''] = signature.split('.')
+ const reason = this.notInternalReason(signature, source, signedAt, teamId, createdAt)
// Own keys only, because a source like `constructor` resolves to an inherited policy with no `billed`.
const policy = reason || !Object.hasOwn(this.policies, source) ? undefined : this.policies[source]
if (!policy) {
- logsInternalLogsCounter.inc({ outcome: 'rejected', reason: reason ?? 'unknown_source' })
+ logsInternalLogsCounter.inc({ outcome: 'not_internal', reason: reason ?? 'unknown_source' })
return undefined
}
- logsInternalLogsCounter.inc({ outcome: 'accepted', reason: '' })
+ logsInternalLogsCounter.inc({ outcome: 'internal', reason: '' })
return { source, ...policy }
}
- private rejectReason(
+ private notInternalReason(
signature: string,
source: string,
+ signedAt: string,
teamId: number,
- payload: Buffer | null
- ): RejectReason | null {
+ createdAt: string | undefined
+ ): NotInternalReason | null {
if (this.keys.length === 0) {
return 'not_configured'
}
if (!this.isTeamAllowed(teamId)) {
return 'team_not_allowed'
}
- if (payload === null || !SOURCE_PATTERN.test(source)) {
+ if (!SOURCE_PATTERN.test(source) || !SIGNED_AT_PATTERN.test(signedAt)) {
return 'invalid_signature'
}
const received = Buffer.from(signature)
- const payloadSha256 = sha256Hex(payload)
// Every key is tried, so a rotation can add a key before producers switch to it.
const matches = this.keys.some((key) => {
- const expected = Buffer.from(computeSignature(key, source, teamId, payloadSha256))
+ const expected = Buffer.from(computeSignature(key, source, teamId, signedAt))
return expected.length === received.length && timingSafeEqual(expected, received)
})
- return matches ? null : 'invalid_signature'
+ if (!matches) {
+ return 'invalid_signature'
+ }
+ // A missing or unparseable `created_at` gives NaN, which fails this comparison and bills the batch.
+ const skewSeconds = Math.abs(Date.parse(createdAt ?? '') / 1000 - Number(signedAt))
+ return skewSeconds <= SIGNATURE_WINDOW_SECONDS ? null : 'stale_signature'
}
}
diff --git a/nodejs/src/logs/logs-ingestion-consumer.test.ts b/nodejs/src/logs/logs-ingestion-consumer.test.ts
index 50254b69275d..970a87891be8 100644
--- a/nodejs/src/logs/logs-ingestion-consumer.test.ts
+++ b/nodejs/src/logs/logs-ingestion-consumer.test.ts
@@ -62,6 +62,15 @@ import type { SamplingRulesCache } from './sampling/sampling-rules-cache'
import { TracesIngestionConsumer } from './traces-ingestion-consumer'
import { LogsTransformerService } from './transformations/logs-transformer.service'
+// What capture-logs forwards for a producer that signed just before the request arrived. Both times come
+// from `Date.now()`, which these tests pin, because a bare `new Date()` still reads the real clock.
+function internalLogsHeaders(signature: string): Record<string, Buffer> {
+ return {
+ [INTERNAL_LOGS_HEADER]: Buffer.from(signature),
+ created_at: Buffer.from(new Date(Date.now()).toISOString()),
+ }
+}
+
const DEFAULT_TEST_TIMEOUT = 5000
jest.setTimeout(DEFAULT_TEST_TIMEOUT)
@@ -1093,9 +1102,9 @@ describe('LogsIngestionConsumer', () => {
record_count: '1',
})
const signature = signed
- ? signInternalLogs(SIGNING_KEY, 'hog_flow', team.id, message.value!)
+ ? signInternalLogs(SIGNING_KEY, 'hog_flow', team.id, new Date(Date.now()))
: 'v1.hog_flow.not-a-signature'
- message.headers!.push({ [INTERNAL_LOGS_HEADER]: Buffer.from(signature) })
+ message.headers!.push(internalLogsHeaders(signature))
const recordLogsReceivedSpy = jest.spyOn(otelMetrics, 'recordLogsReceived')
const recordInternalLogsSpy = jest.spyOn(otelMetrics, 'recordInternalLogs')
@@ -1145,9 +1154,9 @@ describe('LogsIngestionConsumer', () => {
min_timestamp: fiveDaysAgoMicros.toString(),
}
)
- message.headers!.push({
- [INTERNAL_LOGS_HEADER]: Buffer.from(signInternalLogs(SIGNING_KEY, 'hog_flow', team.id, message.value!)),
- })
+ message.headers!.push(
+ internalLogsHeaders(signInternalLogs(SIGNING_KEY, 'hog_flow', team.id, new Date(Date.now())))
+ )
const bytesCreditSpy = jest.spyOn(logsBillingBytesCreditedCounter, 'inc')
const recordsCreditSpy = jest.spyOn(logsBillingRecordsCreditedCounter, 'inc')
@@ -1180,11 +1189,9 @@ describe('LogsIngestionConsumer', () => {
record_count: '1',
...(backfill ? { backfill_days: '540' } : {}),
})
- internal.headers!.push({
- [INTERNAL_LOGS_HEADER]: Buffer.from(
- signInternalLogs(SIGNING_KEY, 'hog_flow', team.id, internal.value!)
- ),
- })
+ internal.headers!.push(
+ internalLogsHeaders(signInternalLogs(SIGNING_KEY, 'hog_flow', team.id, new Date(Date.now())))
+ )
const originalQueueMessages = mockProducer.queueMessages
const passthrough = originalQueueMessages.bind(mockProducer)
mockProducer.queueMessages = jest.fn().mockImplementation((topicMessages) => {
@@ -2530,9 +2537,9 @@ describe('LogsIngestionConsumer', () => {
const [billed, internal] = await createKafkaMessages([createLogMessage(), createLogMessage()], {
token: team.api_token,
})
- internal.headers!.push({
- [INTERNAL_LOGS_HEADER]: Buffer.from(signInternalLogs(key, 'hog_flow', team.id, internal.value!)),
- })
+ internal.headers!.push(
+ internalLogsHeaders(signInternalLogs(key, 'hog_flow', team.id, new Date(Date.now())))
+ )
const originalQueueMessages = mockProducer.queueMessages
const passthrough = originalQueueMessages.bind(mockProducer)
@@ -2690,9 +2697,7 @@ describe('LogsIngestionConsumer', () => {
await consumer.start()
jest.mocked(hub.quotaLimiting.isTeamTokenQuotaLimited).mockResolvedValue(true)
const message = await createKafkaMessage(createLogMessage(), { token: team.api_token })
- message.headers!.push({
- [INTERNAL_LOGS_HEADER]: Buffer.from(signInternalLogs(key, 'hog_flow', team.id, message.value!)),
- })
+ message.headers!.push(internalLogsHeaders(signInternalLogs(key, 'hog_flow', team.id, new Date(Date.now()))))
await waitForBackgroundTasks(consumer.processKafkaBatch([message]))
diff --git a/nodejs/src/logs/logs-ingestion-consumer.ts b/nodejs/src/logs/logs-ingestion-consumer.ts
index c4d7129856c6..e3efe0513368 100644
--- a/nodejs/src/logs/logs-ingestion-consumer.ts
+++ b/nodejs/src/logs/logs-ingestion-consumer.ts
@@ -1566,7 +1566,7 @@ export class LogsIngestionConsumer {
internalLogs: this.internalLogsVerifier.verify(
headers[INTERNAL_LOGS_HEADER],
team.id,
- message.value
+ headers.created_at
),
})
} catch (e) {