Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(ingestion): count events whose token resolves as a team id (#123956)

Paweł Ledwońd141be2ccf9b
diff --git a/nodejs/src/ingestion/common/common-ingestion-pipeline.test.ts b/nodejs/src/ingestion/common/common-ingestion-pipeline.test.ts
index b78ef801be0d..cf7bc37dfe9b 100644
--- a/nodejs/src/ingestion/common/common-ingestion-pipeline.test.ts
+++ b/nodejs/src/ingestion/common/common-ingestion-pipeline.test.ts
@@ -14,6 +14,7 @@ import { createOkContext } from '~/ingestion/framework/helpers'
import { PipelineResult, dlq, drop, isDropResult, isOkResult, ok, redirect } from '~/ingestion/framework/results'
import { ProcessingStep } from '~/ingestion/framework/steps'
import { PluginEvent } from '~/plugin-scaffold'
+import { getMetricValues, resetMetrics } from '~/tests/helpers/metrics'
import { createTestTeam } from '~/tests/helpers/team'
import { RecordedTopHogMetric, createRecordingTopHog } from '~/tests/helpers/tophog'
import { EventHeaders, IncomingEvent, Team } from '~/types'
@@ -775,6 +776,25 @@ describe('CommonIngestionPipelineBuilder', () => {
expect(isDropResult(elements[1].result)).toBe(true)
})
+ it('records teams resolved through a team id sent as the token', async () => {
+ resetMetrics()
+ mockTeamManager.getTeamByToken.mockImplementation((token: string) =>
+ Promise.resolve(token === team.api_token || token === String(team.id) ? team : null)
+ )
+ const pipeline = newCommonIngestionPipeline<MessageOnly, MessageOnly>(config)
+ .parseHeaders()
+ .parseMessage()
+ .resolveTeam()
+ .build()
+
+ await runPipeline(pipeline, [createMessage('user-0'), createMessage('user-1', 'test_event', String(team.id))])
+
+ expect((topHogRecords.get('token_resolved_as_team_id') ?? []).map((r) => r.key)).toEqual([
+ { team_id: String(team.id) },
+ ])
+ expect(await getMetricValues('ingestion_token_resolved_as_team_id_total')).toEqual([{ labels: {}, value: 1 }])
+ })
+
it('records parse timing and message sizes for every parsed message', async () => {
const pipeline = newCommonIngestionPipeline<MessageOnly, MessageOnly>(config)
.parseHeaders()
diff --git a/nodejs/src/ingestion/common/common-ingestion-pipeline.ts b/nodejs/src/ingestion/common/common-ingestion-pipeline.ts
index c6ac86f286dd..cc0f612a76ab 100644
--- a/nodejs/src/ingestion/common/common-ingestion-pipeline.ts
+++ b/nodejs/src/ingestion/common/common-ingestion-pipeline.ts
@@ -32,6 +32,7 @@ import {
createParseKafkaMessageStep,
createResolveTeamStep,
parseMessageTopHogMetrics,
+ resolveTeamTopHogMetrics,
} from './steps/event-preprocessing'
import { addTeamToContext } from './subpipelines/helpers'
@@ -334,8 +335,8 @@ export class CommonPreTeamStage<
* open the team-aware scope: every step piped after this runs inside
* `handleIngestionWarnings`, with warnings routed to the resolved team.
*
- * `options.topHog` records metric factories around the team-resolution
- * step (e.g. a per-team resolved counter).
+ * Always records `resolveTeamTopHogMetrics`. `options.topHog` adds more
+ * metric factories around the team-resolution step.
*/
resolveTeam(
this: CommonPreTeamStage<TInput, TContext, ROut, CBatch, TCurrent & { event: IncomingEvent }>,
@@ -345,8 +346,12 @@ export class CommonPreTeamStage<
): CommonTeamStage<TInput, TContext, ROut, CBatch, TeamResolved<TCurrent>, TeamResolved<TCurrent>> {
const step = createResolveTeamStep<TCurrent & { event: IncomingEvent }>(this.config.teamManager)
const topHog = createTopHogWrapper(this.config.topHog)
- const metrics = options?.topHog
- const resolved = this.chain.extend(metrics?.length ? topHog(step, metrics) : step)
+ const resolved = this.chain.extend(
+ topHog(step, [
+ ...resolveTeamTopHogMetrics<TCurrent & { event: IncomingEvent }, TeamResolved<TCurrent>>(),
+ ...(options?.topHog ?? []),
+ ])
+ )
return new CommonTeamStage(
this.config,
this.beforeBatchCallback,
diff --git a/nodejs/src/ingestion/common/metrics.ts b/nodejs/src/ingestion/common/metrics.ts
index 0d799924ccdf..63a6a82f551a 100644
--- a/nodejs/src/ingestion/common/metrics.ts
+++ b/nodejs/src/ingestion/common/metrics.ts
@@ -44,6 +44,11 @@ export const tokenOrTeamPresentCounter = new Counter({
labelNames: ['team_id_present', 'token_present'],
})
+export const tokenResolvedAsTeamIdCounter = new Counter({
+ name: 'ingestion_token_resolved_as_team_id_total',
+ help: 'Count of events whose token is not an API token but matches a team id, so the team was resolved by id.',
+})
+
export const droppedBloatPropertyCounter = new Counter({
name: 'ingestion_dropped_bloat_property_total',
help: 'Count of deprecated posthog-js persistence-cache properties stripped from event payloads before ClickHouse write (see strip-bloat-properties.ts BLOAT_PROPERTIES).',
diff --git a/nodejs/src/ingestion/common/steps/event-preprocessing/index.ts b/nodejs/src/ingestion/common/steps/event-preprocessing/index.ts
index 9b4b348487fb..e99c3388bb60 100644
--- a/nodejs/src/ingestion/common/steps/event-preprocessing/index.ts
+++ b/nodejs/src/ingestion/common/steps/event-preprocessing/index.ts
@@ -7,7 +7,7 @@ export { createParseHeadersStep } from './parse-headers'
export { createParseKafkaMessageStep, parseMessageTopHogMetrics } from './parse-kafka-message'
export { createOverflowLaneTTLRefreshStep } from './overflow-lane-ttl-refresh-step'
export { createRateLimitToOverflowStep } from './rate-limit-to-overflow-step'
-export { createResolveTeamStep } from './resolve-team'
+export { createResolveTeamStep, resolveTeamTopHogMetrics } from './resolve-team'
export { createValidateEventMetadataStep } from './validate-event-metadata'
export { createValidateEventPropertiesStep } from './validate-event-properties'
export { createValidateEventSchemaStep } from './validate-event-schema'
diff --git a/nodejs/src/ingestion/common/steps/event-preprocessing/resolve-team.ts b/nodejs/src/ingestion/common/steps/event-preprocessing/resolve-team.ts
index f2db7be27ade..a4033fc7619a 100644
--- a/nodejs/src/ingestion/common/steps/event-preprocessing/resolve-team.ts
+++ b/nodejs/src/ingestion/common/steps/event-preprocessing/resolve-team.ts
@@ -2,12 +2,32 @@ import { Message } from 'node-rdkafka'
import { eventDroppedCounter } from '~/common/metrics'
import { TeamManager } from '~/common/utils/team-manager'
-import { tokenOrTeamPresentCounter } from '~/ingestion/common/metrics'
+import { tokenOrTeamPresentCounter, tokenResolvedAsTeamIdCounter } from '~/ingestion/common/metrics'
+import { TopHogMetricFactory, countOkIf } from '~/ingestion/framework/extensions/tophog'
import { drop, ok } from '~/ingestion/framework/results'
import { ProcessingStep } from '~/ingestion/framework/steps'
import { PluginEvent } from '~/plugin-scaffold'
import { EventHeaders, IncomingEvent, Team } from '~/types'
+// TeamManager caches teams by id and by token in one key space, so a numeric token loads the team with that id.
+function isResolvedAsTeamId(token: string | undefined, team: Team): boolean {
+ return team.api_token !== token
+}
+
+export function resolveTeamTopHogMetrics<
+ TInput extends { headers: EventHeaders },
+ TOutput extends { team: Team },
+>(): TopHogMetricFactory<TInput, TOutput>[] {
+ return [
+ countOkIf(
+ 'token_resolved_as_team_id',
+ (output, input) => isResolvedAsTeamId(input.headers.token, output.team),
+ (output) => ({ team_id: String(output.team.id) }),
+ { topN: 100 }
+ ),
+ ]
+}
+
export interface ResolveTeamStepInput {
message: Message
headers: EventHeaders
@@ -52,6 +72,10 @@ async function resolveTeam(
return { error: true, cause: 'invalid_token' }
}
+ if (isResolvedAsTeamId(token, team)) {
+ tokenResolvedAsTeamIdCounter.inc()
+ }
+
return { error: false, team }
}
diff --git a/nodejs/src/ingestion/framework/extensions/tophog.test.ts b/nodejs/src/ingestion/framework/extensions/tophog.test.ts
index 420a2946269a..007a57f682cb 100644
--- a/nodejs/src/ingestion/framework/extensions/tophog.test.ts
+++ b/nodejs/src/ingestion/framework/extensions/tophog.test.ts
@@ -9,6 +9,7 @@ import {
averageResult,
count,
countOk,
+ countOkIf,
countResult,
createTopHogWrapper,
max,
@@ -369,6 +370,34 @@ describe('topHog wrapper', () => {
expect(mockTracker.record).not.toHaveBeenCalled()
})
+ it.each([
+ { name: 'records when the predicate holds', teamId: 42, expectedRecords: [[{ team_id: '42' }, 1]] },
+ { name: 'skips when the predicate fails', teamId: 7, expectedRecords: [] },
+ ])('countOkIf $name', async ({ teamId, expectedRecords }) => {
+ const mockTracker = createMockTopHog()
+ const topHog = createTopHogWrapper(mockTracker)
+
+ function myStep(input: { teamId: number }) {
+ return Promise.resolve(ok({ flagged: input.teamId === 42 }))
+ }
+
+ const pipeline = newPipelineBuilder<{ teamId: number }>()
+ .pipe(
+ topHog(myStep, [
+ countOkIf(
+ 'flagged',
+ (output) => output.flagged,
+ (_output, input) => ({ team_id: String(input.teamId) })
+ ),
+ ])
+ )
+ .build()
+
+ await pipeline.process(createOkContext({ teamId }, {}))
+
+ expect(mockTracker.record.mock.calls).toEqual(expectedRecords)
+ })
+
it('should record average metric before step returns', async () => {
const mockTracker = createMockTopHog()
const topHog = createTopHogWrapper(mockTracker)
diff --git a/nodejs/src/ingestion/framework/extensions/tophog.ts b/nodejs/src/ingestion/framework/extensions/tophog.ts
index 373b8c8b119b..15192389e153 100644
--- a/nodejs/src/ingestion/framework/extensions/tophog.ts
+++ b/nodejs/src/ingestion/framework/extensions/tophog.ts
@@ -42,6 +42,15 @@ export function countOk<TInput, TOutput>(
return (registry) => new OutputMetric(registry.registerSum(name, opts), key, () => 1)
}
+export function countOkIf<TInput, TOutput>(
+ name: string,
+ predicate: (output: TOutput, input: TInput) => boolean,
+ key: (output: TOutput, input: TInput) => Record<string, string>,
+ opts?: MetricConfig
+): TopHogMetricFactory<TInput, TOutput> {
+ return (registry) => new OutputMetric(registry.registerSum(name, opts), key, () => 1, predicate)
+}
+
export function sum<TInput, TOutput>(
name: string,
key: (input: TInput) => Record<string, string>,
@@ -185,12 +194,13 @@ class OutputMetric<TInput, TOutput> implements TopHogMetric<TInput, TOutput> {
constructor(
private readonly tracker: Recorder,
private readonly key: (output: TOutput, input: TInput) => Record<string, string>,
- private readonly value: (output: TOutput, input: TInput) => number
+ private readonly value: (output: TOutput, input: TInput) => number,
+ private readonly predicate: (output: TOutput, input: TInput) => boolean = always
) {}
start(input: TInput): (result: PipelineResult<TOutput, string>) => void {
return (result) => {
- if (isOkResult(result)) {
+ if (isOkResult(result) && this.predicate(result.value, input)) {
this.tracker.record(this.key(result.value, input), this.value(result.value, input))
}
}
@@ -213,3 +223,7 @@ class TimingMetric<TInput, TOutput> implements TopHogMetric<TInput, TOutput> {
}
function noop(): void {}
+
+function always(): boolean {
+ return true
+}