Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(cdp): stop dropping hogflow matcher events when a team lookup fails (#121802)

Phil Haacke7aed67a37f0
diff --git a/nodejs/src/cdp/consumers/cdp-hogflow-subscription-matcher.consumer.test.ts b/nodejs/src/cdp/consumers/cdp-hogflow-subscription-matcher.consumer.test.ts
index 24e7561ca5e3..c0b27123f31b 100644
--- a/nodejs/src/cdp/consumers/cdp-hogflow-subscription-matcher.consumer.test.ts
+++ b/nodejs/src/cdp/consumers/cdp-hogflow-subscription-matcher.consumer.test.ts
@@ -3,6 +3,7 @@ import '../../../tests/helpers/mocks/consumer.mock'
import { HogFlow } from '~/cdp/schema/hogflow'
import { KAFKA_CDP_INTERNAL_EVENTS, KAFKA_EVENTS_JSON } from '~/common/config/kafka-topics'
import { createKafkaConsumer } from '~/common/kafka/consumer'
+import { DependencyUnavailableError } from '~/common/utils/db/error'
import { parseJSON } from '~/common/utils/json-parse'
import { logger } from '~/common/utils/logger'
import * as posthogUtils from '~/common/utils/posthog'
@@ -176,6 +177,12 @@ class MatcherUnderTest extends CdpHogflowSubscriptionMatcherConsumer {
})
),
}
+ ;(this as any).deps = {
+ teamManager: {
+ getTeam: jest.fn().mockResolvedValue({ id: 1, name: 'Test', person_display_name_properties: null }),
+ },
+ }
+ ;(this as any).config = { SITE_URL: 'http://localhost:8000' }
// Stub hogFlowManager
;(this as any).hogFlowManager = {
getHogFlowsForTeams: jest.fn().mockResolvedValue({}),
@@ -231,6 +238,54 @@ class MatcherUnderTest extends CdpHogflowSubscriptionMatcherConsumer {
const stateBuffer = (state: any): Buffer => Buffer.from(JSON.stringify({ state }))
+const rawEvent = (overrides: Record<string, any>): any => ({
+ value: Buffer.from(
+ JSON.stringify({
+ uuid: 'e-uuid',
+ event: 'wuc_subscribed',
+ team_id: 1,
+ distinct_id: 'user-1',
+ person_id: 'person-uuid-1',
+ timestamp: '2024-01-01 00:00:00.000',
+ properties: '{}',
+ elements_chain: '',
+ ...overrides,
+ })
+ ),
+})
+
+const rawPerson = (overrides: Record<string, any>): any => ({
+ value: Buffer.from(
+ JSON.stringify({
+ id: 'person-uuid-1',
+ team_id: 1,
+ properties: JSON.stringify({ plan: 'enterprise' }),
+ is_deleted: 0,
+ timestamp: '2024-01-01 00:00:00.000',
+ ...overrides,
+ })
+ ),
+})
+
+const rawInternalEvent = (overrides: { team_id?: number; event?: Record<string, any>; person?: any } = {}): any => ({
+ value: Buffer.from(
+ JSON.stringify({
+ team_id: overrides.team_id ?? 1,
+ event: {
+ uuid: 'evt-uuid-1',
+ event: '$insight_alert_firing',
+ distinct_id: 'distinct-1',
+ properties: {},
+ timestamp: '2024-01-01T00:00:00Z',
+ ...overrides.event,
+ },
+ ...(overrides.person !== undefined ? { person: overrides.person } : {}),
+ })
+ ),
+})
+
+const invalidTeamIds = [undefined, null, 'abc', 1.5, 3_000_000_000]
+
describe('CdpHogflowSubscriptionMatcherConsumer', () => {
let matcher: MatcherUnderTest
@@ -1106,29 +1161,7 @@ describe('CdpHogflowSubscriptionMatcherConsumer', () => {
})
describe('_parseKafkaBatch', () => {
- const rawMessage = (overrides: Record<string, any>): any => ({
- value: Buffer.from(
- JSON.stringify({
- uuid: 'e-uuid',
- event: 'wuc_subscribed',
- team_id: 1,
- distinct_id: 'user-1',
- person_id: 'person-uuid-1',
- timestamp: '2024-01-01 00:00:00.000',
- properties: '{}',
- elements_chain: '',
- ...overrides,
- })
- ),
- })
-
beforeEach(() => {
- ;(matcher as any).deps = {
- teamManager: {
- getTeam: jest.fn().mockResolvedValue({ id: 1, name: 'Test', person_display_name_properties: null }),
- },
- }
- ;(matcher as any).config = { SITE_URL: 'http://localhost:8000' }
// Team 1 has an actionable (wait_until_condition) flow so events reach conversion.
matcher.setHogFlows({ 'flow-1': makeHogFlow({ id: 'flow-1', team_id: 1 }) })
})
@@ -1137,9 +1170,9 @@ describe('CdpHogflowSubscriptionMatcherConsumer', () => {
// A job can be parked by distinct_id alone (the lookup has a (team_id, distinct_id)
// branch), so an event carrying only a distinct_id must still flow into matching.
const result = await (matcher as any)._parseKafkaBatch([
- rawMessage({ person_id: '', distinct_id: 'only-distinct' }),
- rawMessage({ person_id: '', distinct_id: '' }),
- rawMessage({ person_id: 'person-uuid-1', distinct_id: 'user-1' }),
+ rawEvent({ person_id: '', distinct_id: 'only-distinct' }),
+ rawEvent({ person_id: '', distinct_id: '' }),
+ rawEvent({ person_id: 'person-uuid-1', distinct_id: 'user-1' }),
])
const distinctIds = result.map((g: HogFunctionInvocationGlobals) => g.event.distinct_id).sort()
@@ -1158,8 +1191,8 @@ describe('CdpHogflowSubscriptionMatcherConsumer', () => {
const getTeam = (matcher as any).deps.teamManager.getTeam
const result = await (matcher as any)._parseKafkaBatch([
- rawMessage({ team_id: 2, distinct_id: 'user-2' }),
- rawMessage({ team_id: 1, distinct_id: 'user-1' }),
+ rawEvent({ team_id: 2, distinct_id: 'user-2' }),
+ rawEvent({ team_id: 1, distinct_id: 'user-1' }),
])
expect(result.map((g: HogFunctionInvocationGlobals) => g.project.id)).toEqual([1])
@@ -1170,26 +1203,7 @@ describe('CdpHogflowSubscriptionMatcherConsumer', () => {
})
describe('_parsePersonBatch', () => {
- const rawPerson = (overrides: Record<string, any>): any => ({
- value: Buffer.from(
- JSON.stringify({
- id: 'person-uuid-1',
- team_id: 1,
- properties: JSON.stringify({ plan: 'enterprise' }),
- is_deleted: 0,
- timestamp: '2024-01-01 00:00:00.000',
- ...overrides,
- })
- ),
- })
-
beforeEach(() => {
- ;(matcher as any).deps = {
- teamManager: {
- getTeam: jest.fn().mockResolvedValue({ id: 1, name: 'Test', person_display_name_properties: null }),
- },
- }
- ;(matcher as any).config = { SITE_URL: 'http://localhost:8000' }
matcher.setHogFlows({ 'flow-1': makeHogFlow({ id: 'flow-1', team_id: 1 }) })
})
@@ -1233,32 +1247,7 @@ describe('CdpHogflowSubscriptionMatcherConsumer', () => {
})
describe('_parseInternalEventsBatch', () => {
- const rawInternalEvent = (
- overrides: { team_id?: number; event?: Record<string, any>; person?: any } = {}
- ): any => ({
- value: Buffer.from(
- JSON.stringify({
- team_id: overrides.team_id ?? 1,
- event: {
- uuid: 'evt-uuid-1',
- event: '$insight_alert_firing',
- distinct_id: 'distinct-1',
- properties: {},
- timestamp: '2024-01-01T00:00:00Z',
- ...overrides.event,
- },
- ...(overrides.person !== undefined ? { person: overrides.person } : {}),
- })
- ),
- })
-
beforeEach(() => {
- ;(matcher as any).deps = {
- teamManager: {
- getTeam: jest.fn().mockResolvedValue({ id: 1, name: 'Test', person_display_name_properties: null }),
- },
- }
- ;(matcher as any).config = { SITE_URL: 'http://localhost:8000' }
matcher.setHogFlows({ 'flow-1': makeHogFlow({ id: 'flow-1', team_id: 1 }) })
})
@@ -1290,17 +1279,127 @@ describe('CdpHogflowSubscriptionMatcherConsumer', () => {
expect(result).toEqual([])
})
+ })
- it('drops a malformed message (schema parse failure) without throwing', async () => {
- const result = await (matcher as any)._parseInternalEventsBatch([
- { value: Buffer.from(JSON.stringify({ team_id: 1 })) }, // missing required `event`
- rawInternalEvent(),
- ])
+ describe('message and lookup failures', () => {
+ beforeEach(() => {
+ matcher.setHogFlows({ 'flow-1': makeHogFlow({ id: 'flow-1', team_id: 1 }) })
+ // Postgres rejects a team id outside int4 on every read. The valid messages here all use team 1.
+ // The mock rejects every other team id.
+ const getHogFlowsForTeam = (matcher as any).hogFlowManager.getHogFlowsForTeam
+ const flowsForTeam = getHogFlowsForTeam.getMockImplementation()
+ getHogFlowsForTeam.mockImplementation((teamId: unknown) =>
+ teamId === 1 ? flowsForTeam(teamId) : Promise.reject(new Error('invalid input syntax for type integer'))
+ )
+ })
+
+ it.each([
+ ...invalidTeamIds.flatMap((teamId) => [
+ {
+ name: `an event with team_id ${teamId}`,
+ parse: '_parseKafkaBatch',
+ bad: rawEvent({ team_id: teamId }),
+ good: rawEvent({}),
+ },
+ {
+ name: `a person with team_id ${teamId}`,
+ parse: '_parsePersonBatch',
+ bad: rawPerson({ team_id: teamId }),
+ good: rawPerson({}),
+ },
+ ]),
+ // The schema already rejects a missing, null or string team_id.
+ ...invalidTeamIds
+ .filter((teamId) => typeof teamId === 'number')
+ .map((teamId) => ({
+ name: `an internal event with team_id ${teamId}`,
+ parse: '_parseInternalEventsBatch',
+ bad: rawInternalEvent({ team_id: teamId }),
+ good: rawInternalEvent(),
+ })),
+ {
+ name: 'an event that is not JSON',
+ parse: '_parseKafkaBatch',
+ bad: { value: Buffer.from('not json') },
+ good: rawEvent({}),
+ },
+ {
+ name: 'an event whose properties are not JSON',
+ parse: '_parseKafkaBatch',
+ bad: rawEvent({ properties: 'not json' }),
+ good: rawEvent({}),
+ },
+ {
+ name: 'a person whose properties are not JSON',
+ parse: '_parsePersonBatch',
+ bad: rawPerson({ properties: 'not json' }),
+ good: rawPerson({}),
+ },
+ {
+ name: 'an internal event without its required event',
+ parse: '_parseInternalEventsBatch',
+ bad: { value: Buffer.from(JSON.stringify({ team_id: 1 })) },
+ good: rawInternalEvent(),
+ },
+ ])('skips $name and keeps the rest of the batch', async ({ parse, bad, good }) => {
+ const result = await (matcher as any)[parse]([bad, good])
- // The bad message is dropped; the valid one still parses.
expect(result).toHaveLength(1)
- expect((result[0] as HogFunctionInvocationGlobals).event.event).toBe('$insight_alert_firing')
})
+
+ const streams = [
+ { stream: 'events', consumer: 'kafkaConsumer', message: rawEvent({}) },
+ { stream: 'realtime-only events', consumer: 'realtimeOnlyEventsKafkaConsumer', message: rawEvent({}) },
+ { stream: 'person', consumer: 'personKafkaConsumer', message: rawPerson({}) },
+ { stream: 'internal events', consumer: 'internalEventsKafkaConsumer', message: rawInternalEvent() },
+ ]
+ const lookupFailures = [
+ {
+ lookup: 'the team lookup hits a retriable Postgres error',
+ fail: (m: MatcherUnderTest): Error => {
+ const error = new DependencyUnavailableError(
+ 'connection reset',
+ 'Postgres',
+ new Error('ECONNRESET')
+ )
+ ;(m as any).deps.teamManager.getTeam.mockRejectedValueOnce(error)
+ return error
+ },
+ },
+ {
+ // handlePostgresError does not mark this transient replica error as retriable.
+ lookup: 'the hog flow lookup is cancelled on the read replica',
+ fail: (m: MatcherUnderTest): Error => {
+ const error = new Error('canceling statement due to conflict with recovery')
+ ;(m as any).hogFlowManager.getHogFlowsForTeam.mockRejectedValueOnce(error)
+ return error
+ },
+ },
+ ]
+
+ it.each(streams.flatMap((s) => lookupFailures.map((f) => ({ ...s, ...f }))))(
+ 'fails the $stream batch when $lookup, and processes the event when the batch is read again',
+ async ({ consumer, message, fail }) => {
+ const connect = jest.fn().mockResolvedValue(undefined)
+ ;(matcher as any)[consumer] = { connect }
+ await matcher.start()
+ const handleBatch = connect.mock.calls[0][0]
+ const processBatch = jest.spyOn(matcher, 'processBatch')
+ const error = fail(matcher)
+
+ // The consumer stores a batch's offsets only after its handler resolves. A rejection
+ // leaves them uncommitted. The pod then reads the batch again after it restarts.
+ await expect(handleBatch([message])).rejects.toBe(error)
+ expect(processBatch).not.toHaveBeenCalled()
+
+ const { backgroundTask } = await handleBatch([message])
+ await backgroundTask
+ expect(processBatch).toHaveBeenCalledWith(
+ [expect.objectContaining({ project: expect.objectContaining({ id: 1 }) })],
+ expect.any(String)
+ )
+ }
+ )
})
describe('_parsePersonDistinctIdBatch', () => {
@@ -1327,11 +1426,13 @@ describe('CdpHogflowSubscriptionMatcherConsumer', () => {
expect(result).toEqual([{ teamId: 1, distinctId: 'anon-did', newPersonId: 'survivor-uuid', version: 0 }])
})
- it('drops deletions, missing-id, versionless and malformed messages', () => {
+ it('drops deletions, missing-id, versionless, invalid-team and malformed messages', () => {
const result = (matcher as any)._parsePersonDistinctIdBatch([
rawMove({ is_deleted: 1 }), // distinct_id being deleted
rawMove({ person_id: '' }), // no person to point at
rawMove({ version: null }), // no version to order against
+ // The move lookups send team_id to int4 columns, which reject these on every read.
+ ...invalidTeamIds.map((teamId) => rawMove({ team_id: teamId })),
{ value: Buffer.from('not json') }, // malformed
rawMove(),
])
diff --git a/nodejs/src/cdp/consumers/cdp-hogflow-subscription-matcher.consumer.ts b/nodejs/src/cdp/consumers/cdp-hogflow-subscription-matcher.consumer.ts
index 756d8b88d0e0..16ccd267494e 100644
--- a/nodejs/src/cdp/consumers/cdp-hogflow-subscription-matcher.consumer.ts
+++ b/nodejs/src/cdp/consumers/cdp-hogflow-subscription-matcher.consumer.ts
@@ -21,6 +21,7 @@ import { instrumentFn, instrumented } from '~/common/tracing/tracing-utils'
import { parseJSON } from '~/common/utils/json-parse'
import { logger } from '~/common/utils/logger'
import { captureException } from '~/common/utils/posthog'
+import { isValidTeamId } from '~/common/utils/team-manager'
import { UUIDT } from '~/common/utils/utils'
import {
@@ -139,7 +140,7 @@ type WakeSource = 'events' | 'person' | 'internal_events'
const counterHogflowMatcherEventSkipped = new Counter({
name: 'cdp_hogflow_matcher_event_skipped',
- help: 'An incoming event was dropped before matching: no identifiers (distinct_id or person_id), or unknown team.',
+ help: 'An incoming event was dropped before matching: no identifiers (distinct_id or person_id), an invalid team id, or unknown team.',
labelNames: ['reason'],
})
@@ -186,6 +187,13 @@ type MatchedJob = {
type FilterGlobals = ReturnType<typeof convertToHogFunctionFilterGlobal>
+// What a stream parser reads from one message: the team to look up, and how to build the globals
+// once the team has loaded.
+type ParsedMessage = {
+ teamId: unknown
+ toGlobals: (team: Team) => HogFunctionInvocationGlobals
+}
+
// Emitted by Django when a run a workflow step dispatched ends; `origin_key` names the parked job.
export const WORKFLOW_STEP_RESUME_EVENT = '$workflow_step_resume'
@@ -768,79 +776,83 @@ export class CdpHogflowSubscriptionMatcherConsumer<
@instrumented('cdpHogflowSubscriptionMatcher.parseKafkaMessages')
public async _parseKafkaBatch(messages: Message[]): Promise<HogFunctionInvocationGlobals[]> {
- const events: HogFunctionInvocationGlobals[] = []
-
- await Promise.all(
- messages.map(async (message) => {
- try {
- const clickHouseEvent = parseJSON(message.value!.toString()) as RawClickHouseEvent
- // A job can be parked by distinct_id or person_id, so an event needs at least
- // one of them to match anything. Drop only events that carry neither.
- if (!clickHouseEvent.person_id && !clickHouseEvent.distinct_id) {
- counterHogflowMatcherEventSkipped.labels({ reason: 'no_identifiers' }).inc()
- return
- }
- // The vast majority of events belong to teams with no wait_until_condition
- // step and no event conversion goal. Bail on those via the in-memory hogflow
- // cache before paying for getTeam + full globals conversion.
- if (!(await this.teamIsActionable(clickHouseEvent.team_id))) {
- counterHogflowMatcherEventSkipped.labels({ reason: 'no_actionable_flow' }).inc()
- return
- }
- const team = await this.deps.teamManager.getTeam(clickHouseEvent.team_id)
- if (!team) {
- counterHogflowMatcherEventSkipped.labels({ reason: 'no_team' }).inc()
- return
- }
- events.push(convertToHogFunctionInvocationGlobals(clickHouseEvent, team, this.config.SITE_URL))
- } catch (e) {
- logger.error('Error parsing message', e)
- counterParseError.labels({ error: e.message }).inc()
- }
- })
- )
-
- return events
+ return await this.parseBatch(messages, 'Error parsing message', (message) => {
+ const clickHouseEvent = parseJSON(message.value!.toString()) as RawClickHouseEvent
+ // A job can be parked by distinct_id or person_id, so an event needs at least
+ // one of them to match anything. Drop only events that carry neither.
+ if (!clickHouseEvent.person_id && !clickHouseEvent.distinct_id) {
+ counterHogflowMatcherEventSkipped.labels({ reason: 'no_identifiers' }).inc()
+ return null
+ }
+ return {
+ teamId: clickHouseEvent.team_id,
+ toGlobals: (team) => convertToHogFunctionInvocationGlobals(clickHouseEvent, team, this.config.SITE_URL),
+ }
+ })
}
@instrumented('cdpHogflowSubscriptionMatcher.parsePersonMessages')
public async _parsePersonBatch(messages: Message[]): Promise<HogFunctionInvocationGlobals[]> {
- const events: HogFunctionInvocationGlobals[] = []
+ return await this.parseBatch(messages, 'Error parsing person message', (message) => {
+ const data = parseJSON(message.value!.toString()) as ClickHousePerson
+ // A deleted person can never satisfy a wait, so skip tombstones.
+ if (data.is_deleted) {
+ return null
+ }
+ // Parked waits are matched by person_id (the job's person.id), so a person
+ // mutation needs an id to wake anything.
+ if (!data.id) {
+ counterHogflowMatcherEventSkipped.labels({ reason: 'no_identifiers' }).inc()
+ return null
+ }
+ return {
+ teamId: data.team_id,
+ toGlobals: (team) => convertClickhousePersonToWakeGlobals(data, team, this.config.SITE_URL),
+ }
+ })
+ }
- await Promise.all(
+ /**
+ * Parsing and building the globals do no I/O. They read only the message and the loaded team, so an
+ * error there means the message is malformed. It fails the same way every time it is read, so this
+ * skips it. This also skips a team id that the lookups cannot query, for the same reason.
+ *
+ * The team gate and the team lookup read Postgres. Their errors fail the batch, so the consumer
+ * does not store the batch's offsets and reads the batch again after the pod restarts. Skipping
+ * the message instead would lose the event, and the wait or conversion goal it should satisfy
+ * would time out.
+ */
+ private async parseBatch(
+ messages: Message[],
+ errorLog: string,
+ parse: (message: Message) => ParsedMessage | null
+ ): Promise<HogFunctionInvocationGlobals[]> {
+ const events = await Promise.all(
messages.map(async (message) => {
- try {
- const data = parseJSON(message.value!.toString()) as ClickHousePerson
- // A deleted person can never satisfy a wait — skip tombstones.
- if (data.is_deleted) {
- return
- }
- // Parked waits are matched by person_id (the job's person.id), so a person
- // mutation needs an id to wake anything.
- if (!data.id) {
- counterHogflowMatcherEventSkipped.labels({ reason: 'no_identifiers' }).inc()
- return
- }
- // clickhouse_person is a firehose; bail before getTeam + JSON parse for teams
- // with no flow the matcher can act on.
- if (!(await this.teamIsActionable(data.team_id))) {
- counterHogflowMatcherEventSkipped.labels({ reason: 'no_actionable_flow' }).inc()
- return
- }
- const team = await this.deps.teamManager.getTeam(data.team_id)
- if (!team) {
- counterHogflowMatcherEventSkipped.labels({ reason: 'no_team' }).inc()
- return
- }
- events.push(convertClickhousePersonToWakeGlobals(data, team, this.config.SITE_URL))
- } catch (e) {
- logger.error('Error parsing person message', e)
- counterParseError.labels({ error: e.message }).inc()
+ const parsed = readMessage(errorLog, () => parse(message))
+ if (!parsed) {
+ return null
+ }
+ if (!isValidTeamId(parsed.teamId)) {
+ counterHogflowMatcherEventSkipped.labels({ reason: 'invalid_team_id' }).inc()
+ return null
}
+ // The vast majority of messages belong to teams with no wait_until_condition step and
+ // no conversion goal. Bail on those via the in-memory hogflow cache before paying for
+ // getTeam + full globals conversion.
+ if (!(await this.teamIsActionable(parsed.teamId))) {
+ counterHogflowMatcherEventSkipped.labels({ reason: 'no_actionable_flow' }).inc()
+ return null
+ }
+ const team = await this.deps.teamManager.getTeam(parsed.teamId)
+ if (!team) {
+ counterHogflowMatcherEventSkipped.labels({ reason: 'no_team' }).inc()
+ return null
+ }
+ return readMessage(errorLog, () => parsed.toGlobals(team))
})
)
-
- return events
+ return events.filter((globals): globals is HogFunctionInvocationGlobals => globals !== null)
}
// Step resumes name their job, so they skip the person and team gates the event parser applies.
@@ -877,34 +889,18 @@ export class CdpHogflowSubscriptionMatcherConsumer<
@instrumented('cdpHogflowSubscriptionMatcher.parseInternalEventMessages')
public async _parseInternalEventsBatch(messages: Message[]): Promise<HogFunctionInvocationGlobals[]> {
- const events: HogFunctionInvocationGlobals[] = []
-
- await Promise.all(
- messages.map(async (message) => {
- try {
- const parsed = CdpInternalEventSchema.parse(parseJSON(message.value!.toString()))
- if (!parsed.event.distinct_id && !parsed.person?.id) {
- counterHogflowMatcherEventSkipped.labels({ reason: 'no_identifiers' }).inc()
- return
- }
- if (!(await this.teamIsActionable(parsed.team_id))) {
- counterHogflowMatcherEventSkipped.labels({ reason: 'no_actionable_flow' }).inc()
- return
- }
- const team = await this.deps.teamManager.getTeam(parsed.team_id)
- if (!team) {
- counterHogflowMatcherEventSkipped.labels({ reason: 'no_team' }).inc()
- return
- }
- events.push(convertInternalEventToHogFunctionInvocationGlobals(parsed, team, this.config.SITE_URL))
- } catch (e) {
- logger.error('Error parsing internal event message', e)
- counterParseError.labels({ error: e.message }).inc()
- }
- })
- )
-
- return events
+ return await this.parseBatch(messages, 'Error parsing internal event message', (message) => {
+ const parsed = CdpInternalEventSchema.parse(parseJSON(message.value!.toString()))
+ if (!parsed.event.distinct_id && !parsed.person?.id) {
+ counterHogflowMatcherEventSkipped.labels({ reason: 'no_identifiers' }).inc()
+ return null
+ }
+ return {
+ teamId: parsed.team_id,
+ toGlobals: (team) =>
+ convertInternalEventToHogFunctionInvocationGlobals(parsed, team, this.config.SITE_URL),
+ }
+ })
}
// Synchronous (no getTeam/globals conversion — a repoint only carries ids), so unlike the other
@@ -912,7 +908,7 @@ export class CdpHogflowSubscriptionMatcherConsumer<
public _parsePersonDistinctIdBatch(messages: Message[]): PersonDistinctIdMove[] {
const moves: PersonDistinctIdMove[] = []
for (const message of messages) {
- try {
+ const move = readMessage('Error parsing person distinct id message', (): PersonDistinctIdMove | null => {
const data = parseJSON(message.value!.toString()) as ClickHousePersonDistinctId2
// version > 0 is a repoint (identify/merge). version 0 is the distinct_id's first mapping
// to a person, which a wait parked before the person resolved needs to acquire its anchor:
@@ -920,26 +916,30 @@ export class CdpHogflowSubscriptionMatcherConsumer<
// unwakeable by any person-property change. processMoveBatch scopes version 0 to exactly
// those jobs, which is what keeps this off the insert firehose.
if (data.version === undefined || data.version === null || data.version < 0) {
- continue
+ return null
}
// A distinct_id being deleted can't wake anything, and must never re-point a wait at a
// person that is going away.
if (data.is_deleted) {
- continue
+ return null
}
if (!data.distinct_id || !data.person_id) {
counterHogflowMatcherEventSkipped.labels({ reason: 'no_identifiers' }).inc()
- continue
+ return null
+ }
+ if (!isValidTeamId(data.team_id)) {
+ counterHogflowMatcherEventSkipped.labels({ reason: 'invalid_team_id' }).inc()
+ return null
}
- moves.push({
+ return {
teamId: data.team_id,
distinctId: data.distinct_id,
newPersonId: data.person_id,
version: data.version,
- })
- } catch (e) {
- logger.error('Error parsing person distinct id message', e)
- counterParseError.labels({ error: e.message }).inc()
+ }
+ })
+ if (move) {
+ moves.push(move)
}
}
return moves
@@ -1276,6 +1276,18 @@ function convertClickhousePersonToWakeGlobals(
}
}
+// Runs a step that reads only the message. A throw counts as a parse error and makes this return null.
+// The caller then skips the message.
+function readMessage<T>(errorLog: string, read: () => T): T | null {
+ try {
+ return read()
+ } catch (e) {
+ logger.error(errorLog, e)
+ counterParseError.labels({ error: e.message }).inc()
+ return null
+ }
+}
+
type IndexedBatch = {
teamIds: number[]
// Parallel (teamId, id) pair arrays for the correlated lookup (see findParkedJobs).
diff --git a/nodejs/src/common/utils/team-manager.ts b/nodejs/src/common/utils/team-manager.ts
index ecafb0cfa953..83005264a89d 100644
--- a/nodejs/src/common/utils/team-manager.ts
+++ b/nodejs/src/common/utils/team-manager.ts
@@ -10,6 +10,13 @@ type RawTeam = Omit<Team, 'available_features'> & {
available_product_features: { key: string; name: string }[]
}
+// Postgres stores team ids in int4 columns. A query that compares one of those columns to a
+// non-integer, or to a number above 2_147_483_647, fails. Team ids start at 1. The check therefore
+// also rejects 0 and negative numbers.
+export function isValidTeamId(value: unknown): value is number {
+ return Number.isInteger(value) && (value as number) > 0 && (value as number) <= 2_147_483_647
+}
+
export interface TeamManagerOptions {
/** Retry transient team-load failures (e.g. a Postgres pooler blip) instead of letting them propagate. */
loaderRetry?: LoaderRetryOptions
@@ -135,7 +142,7 @@ export class TeamManager {
if (/^\d{1,10}$/.test(idOrToken)) {
const parsed = parseInt(idOrToken)
// TODO: stat if this happens
- if (!isNaN(parsed) && parsed > 0 && parsed <= 2147483647) {
+ if (isValidTeamId(parsed)) {
teamIds.push(parsed)
}
} else {