Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(ingestion): sanitize nul bytes in a group type before resolving it (#119288)

Eli Reisman700bbaf8c0d0
diff --git a/nodejs/src/common/groups/group-type-manager.test.ts b/nodejs/src/common/groups/group-type-manager.test.ts
index d4c2727aaace..0a300a100861 100644
--- a/nodejs/src/common/groups/group-type-manager.test.ts
+++ b/nodejs/src/common/groups/group-type-manager.test.ts
@@ -139,6 +139,13 @@ describe('GroupTypeManager()', () => {
})
describe('lookupGroupTypeIndex()', () => {
+ it('returns null for a type object whose toString is not callable', async () => {
+ await hub.groupRepository.insertGroupType(teamId, projectId, 'g0', 0, TEST_TIMESTAMP)
+
+ const groupType = { toString: null } as unknown as string
+ expect(await groupTypeManager.lookupGroupTypeIndex(projectId, groupType)).toBeNull()
+ })
+
it.each([
['an existing group type', 'g0', 0],
['an unknown group type', 'brand-new', null],
@@ -156,6 +163,26 @@ describe('GroupTypeManager()', () => {
})
describe('fetchGroupTypeIndex()', () => {
+ it('stores and resolves a type with a NUL byte as its sanitized form', async () => {
+ jest.mocked(hub.groupRepository.insertGroupType).mockClear()
+
+ expect(
+ await groupTypeManager.fetchGroupTypeIndex(teamId, projectId, 'comp\u0000any', TEST_TIMESTAMP)
+ ).toEqual(0)
+
+ expect(hub.groupRepository.insertGroupType).toHaveBeenCalledWith(
+ teamId,
+ projectId,
+ 'comp\uFFFDany',
+ 0,
+ TEST_TIMESTAMP
+ )
+ expect(
+ await groupTypeManager.fetchGroupTypeIndex(teamId, projectId, 'comp\u0000any', TEST_TIMESTAMP)
+ ).toEqual(0)
+ expect(hub.groupRepository.insertGroupType).toHaveBeenCalledTimes(1)
+ })
+
it('returns null without inserting when the type exceeds the column length', async () => {
jest.mocked(hub.groupRepository.insertGroupType).mockClear()
diff --git a/nodejs/src/common/groups/group-type-manager.ts b/nodejs/src/common/groups/group-type-manager.ts
index f935c6ea250d..9c089d6c029c 100644
--- a/nodejs/src/common/groups/group-type-manager.ts
+++ b/nodejs/src/common/groups/group-type-manager.ts
@@ -2,7 +2,7 @@ import { DateTime } from 'luxon'
import { Counter } from 'prom-client'
import { GroupRepository } from '~/common/groups/repositories/group-repository.interface'
-import { asPostgresParameterText, timeoutGuard } from '~/common/utils/db/utils'
+import { asPostgresParameterText, sanitizeString, timeoutGuard } from '~/common/utils/db/utils'
import { LazyLoader, LoaderRetryOptions } from '~/common/utils/lazy-loader'
import { captureTeamEvent } from '~/common/utils/posthog'
import { TeamManager } from '~/common/utils/team-manager'
@@ -14,6 +14,11 @@ export const MAX_GROUP_TYPES_PER_TEAM = 5
/** posthog_grouptypemapping.group_type is varchar(400); a longer value makes the insert fail. */
export const MAX_GROUP_TYPE_LENGTH = 400
+// Postgres text cannot hold a NUL byte, so every lookup and insert uses the sanitized text.
+export function groupTypeAsStoredText(groupType: unknown): string {
+ return sanitizeString(asPostgresParameterText(groupType))
+}
+
const groupTypeTooLongCounter = new Counter({
name: 'group_type_too_long_total',
help: 'Group type resolutions skipped because the type exceeds the column length.',
@@ -79,7 +84,8 @@ export class GroupTypeManager {
*/
public async lookupGroupTypeIndex(projectId: ProjectId, groupType: string): Promise<GroupTypeIndex | null> {
const groupTypes = await this.fetchGroupTypes(projectId)
- const groupTypeIndex = Object.hasOwn(groupTypes, groupType) ? groupTypes[groupType] : undefined
+ const groupTypeText = groupTypeAsStoredText(groupType)
+ const groupTypeIndex = Object.hasOwn(groupTypes, groupTypeText) ? groupTypes[groupTypeText] : undefined
return typeof groupTypeIndex === 'number' ? groupTypeIndex : null
}
@@ -89,12 +95,13 @@ export class GroupTypeManager {
groupType: string,
eventTimestamp: DateTime
): Promise<GroupTypeIndex | null> {
- if (asPostgresParameterText(groupType).length > MAX_GROUP_TYPE_LENGTH) {
+ const groupTypeText = groupTypeAsStoredText(groupType)
+ if (groupTypeText.length > MAX_GROUP_TYPE_LENGTH) {
groupTypeTooLongCounter.inc()
return null
}
- const existingIndex = await this.lookupGroupTypeIndex(projectId, groupType)
+ const existingIndex = await this.lookupGroupTypeIndex(projectId, groupTypeText)
if (existingIndex !== null) {
return existingIndex
}
@@ -116,7 +123,7 @@ export class GroupTypeManager {
const [groupTypeIndex, isInsert] = await this.groupRepository.insertGroupType(
teamId,
projectId,
- groupType,
+ groupTypeText,
nextAvailableIndex,
eventTimestamp
)
@@ -127,7 +134,7 @@ export class GroupTypeManager {
if (isInsert && groupTypeIndex !== null) {
// TODO: Is the `group type ingested` event being valuable? If not, we can remove
// `captureGroupTypeInsert()`. If yes, we should move this capture to use the project instead of team
- await this.captureGroupTypeInsert(teamId, groupType, groupTypeIndex)
+ await this.captureGroupTypeInsert(teamId, groupTypeText, groupTypeIndex)
}
return groupTypeIndex
}
diff --git a/nodejs/src/ingestion/common/steps/event-preprocessing/validate-event-properties.test.ts b/nodejs/src/ingestion/common/steps/event-preprocessing/validate-event-properties.test.ts
index 61d6c23d0da3..dd6481d4bbfd 100644
--- a/nodejs/src/ingestion/common/steps/event-preprocessing/validate-event-properties.test.ts
+++ b/nodejs/src/ingestion/common/steps/event-preprocessing/validate-event-properties.test.ts
@@ -56,6 +56,40 @@ describe('createValidateEventPropertiesStep', () => {
)
})
+ it('drops $groupidentify with a group_key object whose toString is not callable', async () => {
+ const groupKey = { toString: null, padding: 'a'.repeat(401) }
+ const input = {
+ event: createTestPipelineEvent({
+ event: '$groupidentify',
+ distinct_id: 'user123',
+ team_id: 1,
+ properties: { $group_key: groupKey },
+ }),
+ }
+
+ const result = await step(input)
+
+ expect(result).toEqual(
+ drop(
+ 'group_key_too_long',
+ [],
+ [
+ {
+ type: 'group_key_too_long',
+ details: {
+ eventUuid: '123e4567-e89b-12d3-a456-426614174000',
+ event: '$groupidentify',
+ distinctId: 'user123',
+ groupKey,
+ groupKeyLength: 431,
+ maxLength: 400,
+ },
+ },
+ ]
+ )
+ )
+ })
+
it('should allow $groupidentify events with group_key shorter than 400 characters', async () => {
const shortGroupKey = 'a'.repeat(399)
const input = {
diff --git a/nodejs/src/ingestion/common/steps/event-preprocessing/validate-event-properties.ts b/nodejs/src/ingestion/common/steps/event-preprocessing/validate-event-properties.ts
index 7f522a070714..a19bb756937b 100644
--- a/nodejs/src/ingestion/common/steps/event-preprocessing/validate-event-properties.ts
+++ b/nodejs/src/ingestion/common/steps/event-preprocessing/validate-event-properties.ts
@@ -13,7 +13,8 @@ export function createValidateEventPropertiesStep<T extends { event: PipelineEve
if (event.event === '$groupidentify') {
const groupKey = event.properties?.$group_key
- if (groupKey && groupKey.toString().length > MAX_GROUP_KEY_LENGTH) {
+ const groupKeyText = groupKey ? asPostgresParameterText(groupKey) : ''
+ if (groupKeyText.length > MAX_GROUP_KEY_LENGTH) {
return drop(
'group_key_too_long',
[],
@@ -25,7 +26,7 @@ export function createValidateEventPropertiesStep<T extends { event: PipelineEve
event: event.event,
distinctId: event.distinct_id,
groupKey,
- groupKeyLength: groupKey.toString().length,
+ groupKeyLength: groupKeyText.length,
maxLength: MAX_GROUP_KEY_LENGTH,
},
},
diff --git a/nodejs/src/ingestion/common/steps/event-processing/process-groups-step.ts b/nodejs/src/ingestion/common/steps/event-processing/process-groups-step.ts
index dcb823f63243..beb91ecc8a18 100644
--- a/nodejs/src/ingestion/common/steps/event-processing/process-groups-step.ts
+++ b/nodejs/src/ingestion/common/steps/event-processing/process-groups-step.ts
@@ -124,7 +124,7 @@ function validateGroupSet(preparedEvent: PreIngestionEvent): PipelineWarning | n
eventUuid: preparedEvent.eventUuid,
distinctId: preparedEvent.distinctId,
groupType: asPostgresParameterText(groupType),
- groupKey: sanitizeString(String(groupKey)),
+ groupKey: sanitizeString(asPostgresParameterText(groupKey)),
receivedType: Array.isArray(groupPropertiesToSet) ? 'array' : typeof groupPropertiesToSet,
},
// No `key`: the limiter's bucket map never evicts, so a client-supplied
@@ -152,7 +152,7 @@ async function upsertGroup(
teamId,
projectId,
groupTypeIndex,
- sanitizeString(groupKey.toString()),
+ sanitizeString(asPostgresParameterText(groupKey)),
groupPropertiesToSet || {},
timestamp
)
diff --git a/nodejs/src/ingestion/pipelines/analytics/steps/prefetchGroupsStep.ts b/nodejs/src/ingestion/pipelines/analytics/steps/prefetchGroupsStep.ts
index 7f9c5101356e..526454bbefca 100644
--- a/nodejs/src/ingestion/pipelines/analytics/steps/prefetchGroupsStep.ts
+++ b/nodejs/src/ingestion/pipelines/analytics/steps/prefetchGroupsStep.ts
@@ -1,5 +1,5 @@
import { GroupTypeManager } from '~/common/groups/group-type-manager'
-import { sanitizeString } from '~/common/utils/db/utils'
+import { asPostgresParameterText, sanitizeString } from '~/common/utils/db/utils'
import { GroupStoreForBatch } from '~/ingestion/common/groups/group-store-for-batch'
import { PipelineResult, ok } from '~/ingestion/framework/results'
import { GroupTypeIndex, PipelineEvent, Team, TeamId } from '~/types'
@@ -51,7 +51,7 @@ export function prefetchGroupsStep<T extends PrefetchGroupsStepInput>(
entries.push({
teamId: event.team.id,
groupTypeIndex,
- groupKey: sanitizeString(String(groupKey)),
+ groupKey: sanitizeString(asPostgresParameterText(groupKey)),
batchId: event.groupStoreForBatch.batchId,
})
}