:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(ingestion): count events whose token resolves as a team id (#123956)
Paweł Ledwońd141be2ccf9b
diff --git a/nodejs/src/ingestion/common/common-ingestion-pipeline.test.ts b/nodejs/src/ingestion/common/common-ingestion-pipeline.test.tsindex b78ef801be0d..cf7bc37dfe9b 100644--- a/nodejs/src/ingestion/common/common-ingestion-pipeline.test.ts+++ b/nodejs/src/ingestion/common/common-ingestion-pipeline.test.ts@@ -14,6 +14,7 @@ import { createOkContext } from '~/ingestion/framework/helpers'import { PipelineResult, dlq, drop, isDropResult, isOkResult, ok, redirect } from '~/ingestion/framework/results'import { ProcessingStep } from '~/ingestion/framework/steps'import { PluginEvent } from '~/plugin-scaffold'+import { getMetricValues, resetMetrics } from '~/tests/helpers/metrics'import { createTestTeam } from '~/tests/helpers/team'import { RecordedTopHogMetric, createRecordingTopHog } from '~/tests/helpers/tophog'import { EventHeaders, IncomingEvent, Team } from '~/types'@@ -775,6 +776,25 @@ describe('CommonIngestionPipelineBuilder', () => {expect(isDropResult(elements[1].result)).toBe(true)})+ it('records teams resolved through a team id sent as the token', async () => {+ resetMetrics()+ mockTeamManager.getTeamByToken.mockImplementation((token: string) =>+ Promise.resolve(token === team.api_token || token === String(team.id) ? team : null)+ )+ const pipeline = newCommonIngestionPipeline<MessageOnly, MessageOnly>(config)+ .parseHeaders()+ .parseMessage()+ .resolveTeam()+ .build()++ await runPipeline(pipeline, [createMessage('user-0'), createMessage('user-1', 'test_event', String(team.id))])++ expect((topHogRecords.get('token_resolved_as_team_id') ?? []).map((r) => r.key)).toEqual([+ { team_id: String(team.id) },+ ])+ expect(await getMetricValues('ingestion_token_resolved_as_team_id_total')).toEqual([{ labels: {}, value: 1 }])+ })+it('records parse timing and message sizes for every parsed message', async () => {const pipeline = newCommonIngestionPipeline<MessageOnly, MessageOnly>(config).parseHeaders()diff --git a/nodejs/src/ingestion/common/common-ingestion-pipeline.ts b/nodejs/src/ingestion/common/common-ingestion-pipeline.tsindex c6ac86f286dd..cc0f612a76ab 100644--- a/nodejs/src/ingestion/common/common-ingestion-pipeline.ts+++ b/nodejs/src/ingestion/common/common-ingestion-pipeline.ts@@ -32,6 +32,7 @@ import {createParseKafkaMessageStep,createResolveTeamStep,parseMessageTopHogMetrics,+ resolveTeamTopHogMetrics,} from './steps/event-preprocessing'import { addTeamToContext } from './subpipelines/helpers'@@ -334,8 +335,8 @@ export class CommonPreTeamStage<* open the team-aware scope: every step piped after this runs inside* `handleIngestionWarnings`, with warnings routed to the resolved team.*- * `options.topHog` records metric factories around the team-resolution- * step (e.g. a per-team resolved counter).+ * Always records `resolveTeamTopHogMetrics`. `options.topHog` adds more+ * metric factories around the team-resolution step.*/resolveTeam(this: CommonPreTeamStage<TInput, TContext, ROut, CBatch, TCurrent & { event: IncomingEvent }>,@@ -345,8 +346,12 @@ export class CommonPreTeamStage<): CommonTeamStage<TInput, TContext, ROut, CBatch, TeamResolved<TCurrent>, TeamResolved<TCurrent>> {const step = createResolveTeamStep<TCurrent & { event: IncomingEvent }>(this.config.teamManager)const topHog = createTopHogWrapper(this.config.topHog)- const metrics = options?.topHog- const resolved = this.chain.extend(metrics?.length ? topHog(step, metrics) : step)+ const resolved = this.chain.extend(+ topHog(step, [+ ...resolveTeamTopHogMetrics<TCurrent & { event: IncomingEvent }, TeamResolved<TCurrent>>(),+ ...(options?.topHog ?? []),+ ])+ )return new CommonTeamStage(this.config,this.beforeBatchCallback,diff --git a/nodejs/src/ingestion/common/metrics.ts b/nodejs/src/ingestion/common/metrics.tsindex 0d799924ccdf..63a6a82f551a 100644--- a/nodejs/src/ingestion/common/metrics.ts+++ b/nodejs/src/ingestion/common/metrics.ts@@ -44,6 +44,11 @@ export const tokenOrTeamPresentCounter = new Counter({labelNames: ['team_id_present', 'token_present'],})+export const tokenResolvedAsTeamIdCounter = new Counter({+ name: 'ingestion_token_resolved_as_team_id_total',+ help: 'Count of events whose token is not an API token but matches a team id, so the team was resolved by id.',+})+export const droppedBloatPropertyCounter = new Counter({name: 'ingestion_dropped_bloat_property_total',help: 'Count of deprecated posthog-js persistence-cache properties stripped from event payloads before ClickHouse write (see strip-bloat-properties.ts BLOAT_PROPERTIES).',diff --git a/nodejs/src/ingestion/common/steps/event-preprocessing/index.ts b/nodejs/src/ingestion/common/steps/event-preprocessing/index.tsindex 9b4b348487fb..e99c3388bb60 100644--- a/nodejs/src/ingestion/common/steps/event-preprocessing/index.ts+++ b/nodejs/src/ingestion/common/steps/event-preprocessing/index.ts@@ -7,7 +7,7 @@ export { createParseHeadersStep } from './parse-headers'export { createParseKafkaMessageStep, parseMessageTopHogMetrics } from './parse-kafka-message'export { createOverflowLaneTTLRefreshStep } from './overflow-lane-ttl-refresh-step'export { createRateLimitToOverflowStep } from './rate-limit-to-overflow-step'-export { createResolveTeamStep } from './resolve-team'+export { createResolveTeamStep, resolveTeamTopHogMetrics } from './resolve-team'export { createValidateEventMetadataStep } from './validate-event-metadata'export { createValidateEventPropertiesStep } from './validate-event-properties'export { createValidateEventSchemaStep } from './validate-event-schema'diff --git a/nodejs/src/ingestion/common/steps/event-preprocessing/resolve-team.ts b/nodejs/src/ingestion/common/steps/event-preprocessing/resolve-team.tsindex f2db7be27ade..a4033fc7619a 100644--- a/nodejs/src/ingestion/common/steps/event-preprocessing/resolve-team.ts+++ b/nodejs/src/ingestion/common/steps/event-preprocessing/resolve-team.ts@@ -2,12 +2,32 @@ import { Message } from 'node-rdkafka'import { eventDroppedCounter } from '~/common/metrics'import { TeamManager } from '~/common/utils/team-manager'-import { tokenOrTeamPresentCounter } from '~/ingestion/common/metrics'+import { tokenOrTeamPresentCounter, tokenResolvedAsTeamIdCounter } from '~/ingestion/common/metrics'+import { TopHogMetricFactory, countOkIf } from '~/ingestion/framework/extensions/tophog'import { drop, ok } from '~/ingestion/framework/results'import { ProcessingStep } from '~/ingestion/framework/steps'import { PluginEvent } from '~/plugin-scaffold'import { EventHeaders, IncomingEvent, Team } from '~/types'+// TeamManager caches teams by id and by token in one key space, so a numeric token loads the team with that id.+function isResolvedAsTeamId(token: string | undefined, team: Team): boolean {+ return team.api_token !== token+}++export function resolveTeamTopHogMetrics<+ TInput extends { headers: EventHeaders },+ TOutput extends { team: Team },+>(): TopHogMetricFactory<TInput, TOutput>[] {+ return [+ countOkIf(+ 'token_resolved_as_team_id',+ (output, input) => isResolvedAsTeamId(input.headers.token, output.team),+ (output) => ({ team_id: String(output.team.id) }),+ { topN: 100 }+ ),+ ]+}+export interface ResolveTeamStepInput {message: Messageheaders: EventHeaders@@ -52,6 +72,10 @@ async function resolveTeam(return { error: true, cause: 'invalid_token' }}+ if (isResolvedAsTeamId(token, team)) {+ tokenResolvedAsTeamIdCounter.inc()+ }+return { error: false, team }}diff --git a/nodejs/src/ingestion/framework/extensions/tophog.test.ts b/nodejs/src/ingestion/framework/extensions/tophog.test.tsindex 420a2946269a..007a57f682cb 100644--- a/nodejs/src/ingestion/framework/extensions/tophog.test.ts+++ b/nodejs/src/ingestion/framework/extensions/tophog.test.ts@@ -9,6 +9,7 @@ import {averageResult,count,countOk,+ countOkIf,countResult,createTopHogWrapper,max,@@ -369,6 +370,34 @@ describe('topHog wrapper', () => {expect(mockTracker.record).not.toHaveBeenCalled()})+ it.each([+ { name: 'records when the predicate holds', teamId: 42, expectedRecords: [[{ team_id: '42' }, 1]] },+ { name: 'skips when the predicate fails', teamId: 7, expectedRecords: [] },+ ])('countOkIf $name', async ({ teamId, expectedRecords }) => {+ const mockTracker = createMockTopHog()+ const topHog = createTopHogWrapper(mockTracker)++ function myStep(input: { teamId: number }) {+ return Promise.resolve(ok({ flagged: input.teamId === 42 }))+ }++ const pipeline = newPipelineBuilder<{ teamId: number }>()+ .pipe(+ topHog(myStep, [+ countOkIf(+ 'flagged',+ (output) => output.flagged,+ (_output, input) => ({ team_id: String(input.teamId) })+ ),+ ])+ )+ .build()++ await pipeline.process(createOkContext({ teamId }, {}))++ expect(mockTracker.record.mock.calls).toEqual(expectedRecords)+ })+it('should record average metric before step returns', async () => {const mockTracker = createMockTopHog()const topHog = createTopHogWrapper(mockTracker)diff --git a/nodejs/src/ingestion/framework/extensions/tophog.ts b/nodejs/src/ingestion/framework/extensions/tophog.tsindex 373b8c8b119b..15192389e153 100644--- a/nodejs/src/ingestion/framework/extensions/tophog.ts+++ b/nodejs/src/ingestion/framework/extensions/tophog.ts@@ -42,6 +42,15 @@ export function countOk<TInput, TOutput>(return (registry) => new OutputMetric(registry.registerSum(name, opts), key, () => 1)}+export function countOkIf<TInput, TOutput>(+ name: string,+ predicate: (output: TOutput, input: TInput) => boolean,+ key: (output: TOutput, input: TInput) => Record<string, string>,+ opts?: MetricConfig+): TopHogMetricFactory<TInput, TOutput> {+ return (registry) => new OutputMetric(registry.registerSum(name, opts), key, () => 1, predicate)+}+export function sum<TInput, TOutput>(name: string,key: (input: TInput) => Record<string, string>,@@ -185,12 +194,13 @@ class OutputMetric<TInput, TOutput> implements TopHogMetric<TInput, TOutput> {constructor(private readonly tracker: Recorder,private readonly key: (output: TOutput, input: TInput) => Record<string, string>,- private readonly value: (output: TOutput, input: TInput) => number+ private readonly value: (output: TOutput, input: TInput) => number,+ private readonly predicate: (output: TOutput, input: TInput) => boolean = always) {}start(input: TInput): (result: PipelineResult<TOutput, string>) => void {return (result) => {- if (isOkResult(result)) {+ if (isOkResult(result) && this.predicate(result.value, input)) {this.tracker.record(this.key(result.value, input), this.value(result.value, input))}}@@ -213,3 +223,7 @@ class TimingMetric<TInput, TOutput> implements TopHogMetric<TInput, TOutput> {}function noop(): void {}++function always(): boolean {+ return true+}