:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(ingestion): sanitize nul bytes in a group type before resolving it (#119288)
Eli Reisman700bbaf8c0d0
diff --git a/nodejs/src/common/groups/group-type-manager.test.ts b/nodejs/src/common/groups/group-type-manager.test.tsindex d4c2727aaace..0a300a100861 100644--- a/nodejs/src/common/groups/group-type-manager.test.ts+++ b/nodejs/src/common/groups/group-type-manager.test.ts@@ -139,6 +139,13 @@ describe('GroupTypeManager()', () => {})describe('lookupGroupTypeIndex()', () => {+ it('returns null for a type object whose toString is not callable', async () => {+ await hub.groupRepository.insertGroupType(teamId, projectId, 'g0', 0, TEST_TIMESTAMP)++ const groupType = { toString: null } as unknown as string+ expect(await groupTypeManager.lookupGroupTypeIndex(projectId, groupType)).toBeNull()+ })+it.each([['an existing group type', 'g0', 0],['an unknown group type', 'brand-new', null],@@ -156,6 +163,26 @@ describe('GroupTypeManager()', () => {})describe('fetchGroupTypeIndex()', () => {+ it('stores and resolves a type with a NUL byte as its sanitized form', async () => {+ jest.mocked(hub.groupRepository.insertGroupType).mockClear()++ expect(+ await groupTypeManager.fetchGroupTypeIndex(teamId, projectId, 'comp\u0000any', TEST_TIMESTAMP)+ ).toEqual(0)++ expect(hub.groupRepository.insertGroupType).toHaveBeenCalledWith(+ teamId,+ projectId,+ 'comp\uFFFDany',+ 0,+ TEST_TIMESTAMP+ )+ expect(+ await groupTypeManager.fetchGroupTypeIndex(teamId, projectId, 'comp\u0000any', TEST_TIMESTAMP)+ ).toEqual(0)+ expect(hub.groupRepository.insertGroupType).toHaveBeenCalledTimes(1)+ })+it('returns null without inserting when the type exceeds the column length', async () => {jest.mocked(hub.groupRepository.insertGroupType).mockClear()diff --git a/nodejs/src/common/groups/group-type-manager.ts b/nodejs/src/common/groups/group-type-manager.tsindex f935c6ea250d..9c089d6c029c 100644--- a/nodejs/src/common/groups/group-type-manager.ts+++ b/nodejs/src/common/groups/group-type-manager.ts@@ -2,7 +2,7 @@ import { DateTime } from 'luxon'import { Counter } from 'prom-client'import { GroupRepository } from '~/common/groups/repositories/group-repository.interface'-import { asPostgresParameterText, timeoutGuard } from '~/common/utils/db/utils'+import { asPostgresParameterText, sanitizeString, timeoutGuard } from '~/common/utils/db/utils'import { LazyLoader, LoaderRetryOptions } from '~/common/utils/lazy-loader'import { captureTeamEvent } from '~/common/utils/posthog'import { TeamManager } from '~/common/utils/team-manager'@@ -14,6 +14,11 @@ export const MAX_GROUP_TYPES_PER_TEAM = 5/** posthog_grouptypemapping.group_type is varchar(400); a longer value makes the insert fail. */export const MAX_GROUP_TYPE_LENGTH = 400+// Postgres text cannot hold a NUL byte, so every lookup and insert uses the sanitized text.+export function groupTypeAsStoredText(groupType: unknown): string {+ return sanitizeString(asPostgresParameterText(groupType))+}+const groupTypeTooLongCounter = new Counter({name: 'group_type_too_long_total',help: 'Group type resolutions skipped because the type exceeds the column length.',@@ -79,7 +84,8 @@ export class GroupTypeManager {*/public async lookupGroupTypeIndex(projectId: ProjectId, groupType: string): Promise<GroupTypeIndex | null> {const groupTypes = await this.fetchGroupTypes(projectId)- const groupTypeIndex = Object.hasOwn(groupTypes, groupType) ? groupTypes[groupType] : undefined+ const groupTypeText = groupTypeAsStoredText(groupType)+ const groupTypeIndex = Object.hasOwn(groupTypes, groupTypeText) ? groupTypes[groupTypeText] : undefinedreturn typeof groupTypeIndex === 'number' ? groupTypeIndex : null}@@ -89,12 +95,13 @@ export class GroupTypeManager {groupType: string,eventTimestamp: DateTime): Promise<GroupTypeIndex | null> {- if (asPostgresParameterText(groupType).length > MAX_GROUP_TYPE_LENGTH) {+ const groupTypeText = groupTypeAsStoredText(groupType)+ if (groupTypeText.length > MAX_GROUP_TYPE_LENGTH) {groupTypeTooLongCounter.inc()return null}- const existingIndex = await this.lookupGroupTypeIndex(projectId, groupType)+ const existingIndex = await this.lookupGroupTypeIndex(projectId, groupTypeText)if (existingIndex !== null) {return existingIndex}@@ -116,7 +123,7 @@ export class GroupTypeManager {const [groupTypeIndex, isInsert] = await this.groupRepository.insertGroupType(teamId,projectId,- groupType,+ groupTypeText,nextAvailableIndex,eventTimestamp)@@ -127,7 +134,7 @@ export class GroupTypeManager {if (isInsert && groupTypeIndex !== null) {// TODO: Is the `group type ingested` event being valuable? If not, we can remove// `captureGroupTypeInsert()`. If yes, we should move this capture to use the project instead of team- await this.captureGroupTypeInsert(teamId, groupType, groupTypeIndex)+ await this.captureGroupTypeInsert(teamId, groupTypeText, groupTypeIndex)}return groupTypeIndex}diff --git a/nodejs/src/ingestion/common/steps/event-preprocessing/validate-event-properties.test.ts b/nodejs/src/ingestion/common/steps/event-preprocessing/validate-event-properties.test.tsindex 61d6c23d0da3..dd6481d4bbfd 100644--- a/nodejs/src/ingestion/common/steps/event-preprocessing/validate-event-properties.test.ts+++ b/nodejs/src/ingestion/common/steps/event-preprocessing/validate-event-properties.test.ts@@ -56,6 +56,40 @@ describe('createValidateEventPropertiesStep', () => {)})+ it('drops $groupidentify with a group_key object whose toString is not callable', async () => {+ const groupKey = { toString: null, padding: 'a'.repeat(401) }+ const input = {+ event: createTestPipelineEvent({+ event: '$groupidentify',+ distinct_id: 'user123',+ team_id: 1,+ properties: { $group_key: groupKey },+ }),+ }++ const result = await step(input)++ expect(result).toEqual(+ drop(+ 'group_key_too_long',+ [],+ [+ {+ type: 'group_key_too_long',+ details: {+ eventUuid: '123e4567-e89b-12d3-a456-426614174000',+ event: '$groupidentify',+ distinctId: 'user123',+ groupKey,+ groupKeyLength: 431,+ maxLength: 400,+ },+ },+ ]+ )+ )+ })+it('should allow $groupidentify events with group_key shorter than 400 characters', async () => {const shortGroupKey = 'a'.repeat(399)const input = {diff --git a/nodejs/src/ingestion/common/steps/event-preprocessing/validate-event-properties.ts b/nodejs/src/ingestion/common/steps/event-preprocessing/validate-event-properties.tsindex 7f522a070714..a19bb756937b 100644--- a/nodejs/src/ingestion/common/steps/event-preprocessing/validate-event-properties.ts+++ b/nodejs/src/ingestion/common/steps/event-preprocessing/validate-event-properties.ts@@ -13,7 +13,8 @@ export function createValidateEventPropertiesStep<T extends { event: PipelineEveif (event.event === '$groupidentify') {const groupKey = event.properties?.$group_key- if (groupKey && groupKey.toString().length > MAX_GROUP_KEY_LENGTH) {+ const groupKeyText = groupKey ? asPostgresParameterText(groupKey) : ''+ if (groupKeyText.length > MAX_GROUP_KEY_LENGTH) {return drop('group_key_too_long',[],@@ -25,7 +26,7 @@ export function createValidateEventPropertiesStep<T extends { event: PipelineEveevent: event.event,distinctId: event.distinct_id,groupKey,- groupKeyLength: groupKey.toString().length,+ groupKeyLength: groupKeyText.length,maxLength: MAX_GROUP_KEY_LENGTH,},},diff --git a/nodejs/src/ingestion/common/steps/event-processing/process-groups-step.ts b/nodejs/src/ingestion/common/steps/event-processing/process-groups-step.tsindex dcb823f63243..beb91ecc8a18 100644--- a/nodejs/src/ingestion/common/steps/event-processing/process-groups-step.ts+++ b/nodejs/src/ingestion/common/steps/event-processing/process-groups-step.ts@@ -124,7 +124,7 @@ function validateGroupSet(preparedEvent: PreIngestionEvent): PipelineWarning | neventUuid: preparedEvent.eventUuid,distinctId: preparedEvent.distinctId,groupType: asPostgresParameterText(groupType),- groupKey: sanitizeString(String(groupKey)),+ groupKey: sanitizeString(asPostgresParameterText(groupKey)),receivedType: Array.isArray(groupPropertiesToSet) ? 'array' : typeof groupPropertiesToSet,},// No `key`: the limiter's bucket map never evicts, so a client-supplied@@ -152,7 +152,7 @@ async function upsertGroup(teamId,projectId,groupTypeIndex,- sanitizeString(groupKey.toString()),+ sanitizeString(asPostgresParameterText(groupKey)),groupPropertiesToSet || {},timestamp)diff --git a/nodejs/src/ingestion/pipelines/analytics/steps/prefetchGroupsStep.ts b/nodejs/src/ingestion/pipelines/analytics/steps/prefetchGroupsStep.tsindex 7f9c5101356e..526454bbefca 100644--- a/nodejs/src/ingestion/pipelines/analytics/steps/prefetchGroupsStep.ts+++ b/nodejs/src/ingestion/pipelines/analytics/steps/prefetchGroupsStep.ts@@ -1,5 +1,5 @@import { GroupTypeManager } from '~/common/groups/group-type-manager'-import { sanitizeString } from '~/common/utils/db/utils'+import { asPostgresParameterText, sanitizeString } from '~/common/utils/db/utils'import { GroupStoreForBatch } from '~/ingestion/common/groups/group-store-for-batch'import { PipelineResult, ok } from '~/ingestion/framework/results'import { GroupTypeIndex, PipelineEvent, Team, TeamId } from '~/types'@@ -51,7 +51,7 @@ export function prefetchGroupsStep<T extends PrefetchGroupsStepInput>(entries.push({teamId: event.team.id,groupTypeIndex,- groupKey: sanitizeString(String(groupKey)),+ groupKey: sanitizeString(asPostgresParameterText(groupKey)),batchId: event.groupStoreForBatch.batchId,})}