:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
// Serial: changes shared ClickHouse merge state and writes shared Kafka topics.
import { KafkaProducerObserver } from '~/tests/helpers/mocks/producer.spy'
import { DateTime } from 'luxon'
import { defaultConfig } from '~/common/config/config'
import {
KAFKA_INGESTION_WARNINGS,
KAFKA_PERSON,
KAFKA_PERSON_DISTINCT_ID,
KAFKA_PERSON_MERGE_EVENTS,
} from '~/common/config/kafka-topics'
import { murmur2Partition } from '~/common/kafka/murmur2'
import { KafkaProducerWrapper } from '~/common/kafka/producer'
import { INGESTION_WARNINGS_OUTPUT } from '~/common/outputs'
import { ASYNC_OUTPUT, PERSONS_OUTPUT, PERSON_DISTINCT_IDS_OUTPUT, PERSON_MERGE_EVENTS_OUTPUT } from '~/common/outputs'
import { IngestionOutputs } from '~/common/outputs/ingestion-outputs'
import { SingleIngestionOutput } from '~/common/outputs/single-ingestion-output'
import { PersonMessage } from '~/common/persons/person-message'
import { fromInternalPerson } from '~/common/persons/person-update-batch'
import { PostgresPersonRepository } from '~/common/persons/repositories/postgres-person-repository'
import { fetchDistinctIdValues } from '~/common/persons/repositories/test-helpers'
import { DependencyUnavailableError } from '~/common/utils/db/error'
import { closeHub, createHub } from '~/common/utils/db/hub'
import { PostgresRouter, PostgresUse } from '~/common/utils/db/postgres'
import { parseJSON } from '~/common/utils/json-parse'
import { defaultRetryConfig } from '~/common/utils/retries'
import { UUIDT } from '~/common/utils/utils'
import { BatchWritingPersonsStore } from '~/ingestion/common/persons/batch-writing-person-store'
import { PersonOutputs } from '~/ingestion/common/persons/person-context'
import { PersonContext } from '~/ingestion/common/persons/person-context'
import { PersonEventProcessor } from '~/ingestion/common/persons/person-event-processor'
import { personMergeEventProducedCounter } from '~/ingestion/common/persons/person-merge-postgres'
import { PersonMergeService } from '~/ingestion/common/persons/person-merge-service'
import {
SourcePersonNotFoundError,
TargetPersonNotFoundError,
createDefaultSyncMergeMode,
} from '~/ingestion/common/persons/person-merge-types'
import { PersonPropertyService } from '~/ingestion/common/persons/person-property-service'
import { uuidFromDistinctId } from '~/ingestion/common/persons/person-uuid'
import { BatchBoundPersonsStore, PersonsStoreForBatch } from '~/ingestion/common/persons/persons-store-for-batch'
import { PipelineResultType, isDlqResult, isOkResult, isRedirectResult } from '~/ingestion/framework/results'
import { PluginEvent, Properties } from '~/plugin-scaffold'
import { Clickhouse } from '~/tests/helpers/clickhouse'
import { ensureKafkaTopics } from '~/tests/helpers/kafka'
import {
ClickHousePerson,
ClickHousePersonDistinctId2,
Hub,
InternalPerson,
PropertiesLastOperation,
PropertiesLastUpdatedAt,
Team,
} from '~/types'
import {
createOrganization,
createTeam,
fetchPostgresDistinctIdsForPerson,
fetchPostgresPersons,
getTeam,
insertRow,
} from '../helpers/sql'
jest.setTimeout(30000)
// Shared real Kafka producer used by every helper below — created in `beforeAll`
// once the hub config is loaded, disconnected in `afterAll`.
let kafkaProducer: KafkaProducerWrapper = undefined as unknown as KafkaProducerWrapper
function createPersonOutputs(kafkaProducer: KafkaProducerWrapper): PersonOutputs {
return new IngestionOutputs({
[PERSONS_OUTPUT]: new SingleIngestionOutput(PERSONS_OUTPUT, KAFKA_PERSON, kafkaProducer, 'test'),
[PERSON_DISTINCT_IDS_OUTPUT]: new SingleIngestionOutput(
INGESTION_WARNINGS_OUTPUT,
KAFKA_PERSON_DISTINCT_ID,
kafkaProducer,
'test'
),
[INGESTION_WARNINGS_OUTPUT]: new SingleIngestionOutput(
INGESTION_WARNINGS_OUTPUT,
KAFKA_INGESTION_WARNINGS,
kafkaProducer,
'test'
),
[PERSON_MERGE_EVENTS_OUTPUT]: new SingleIngestionOutput(
PERSON_MERGE_EVENTS_OUTPUT,
KAFKA_PERSON_MERGE_EVENTS,
kafkaProducer,
'test'
),
})
}
async function createPerson(
hub: Hub,
createdAt: DateTime,
properties: Properties,
propertiesLastUpdatedAt: PropertiesLastUpdatedAt,
propertiesLastOperation: PropertiesLastOperation,
teamId: number,
isUserId: number | null,
isIdentified: boolean,
uuid: string,
primaryDistinctId: { distinctId: string; version?: number },
extraDistinctIds?: { distinctId: string; version?: number }[]
): Promise<InternalPerson> {
const personRepository = new PostgresPersonRepository(hub.postgres)
const result = await personRepository.createPerson(
createdAt,
properties,
propertiesLastUpdatedAt,
propertiesLastOperation,
teamId,
isUserId,
isIdentified,
uuid,
primaryDistinctId,
extraDistinctIds
)
if (!result.success) {
throw new Error('Failed to create person')
}
const personOutputs = createPersonOutputs(kafkaProducer)
await Promise.all(result.messages.map((msg) => personOutputs.produce(msg.output, { value: msg.value, key: null })))
return result.person
}
/** Whether another connection could lock the person row right now. */
function rowLockState(hub: Hub, id: string): Promise<string> {
return hub.postgres
.query(
PostgresUse.PERSONS_WRITE,
'SELECT id FROM posthog_person WHERE id = $1 FOR UPDATE NOWAIT',
[id],
'lockProbe'
)
.then(
() => 'free',
(error: Error) => (error.message.includes('could not obtain lock') ? 'locked' : error.message)
)
}
async function flushPersonStoreToKafka(
kafkaProducer: KafkaProducerWrapper,
personStore: PersonsStoreForBatch,
kafkaAcks: Promise<void>
) {
const kafkaMessages = await personStore.flush()
const personOutputs = createPersonOutputs(kafkaProducer)
await Promise.all(
kafkaMessages
.flatMap((message) => message.messages)
.map((msg) => personOutputs.produce(msg.output, { value: msg.value, key: null }))
)
await kafkaProducer.flush()
await kafkaAcks
return kafkaMessages
}
describe('PersonState.processEvent()', () => {
let hub: Hub
let clickhouse: Clickhouse
let personRepository: PostgresPersonRepository
let mockProducerObserver: KafkaProducerObserver
let teamId: number
let mainTeam: Team
let organizationId: string
// Common Distinct IDs (and their deterministic UUIDs) used in tests below.
const newUserDistinctId = 'new-user'
let newUserUuid: string
const oldUserDistinctId = 'old-user'
let oldUserUuid: string
const firstUserDistinctId = 'first'
let firstUserUuid: string
const secondUserDistinctId = 'second'
let secondUserUuid: string
let timestamp: DateTime
let timestamp2: DateTime
let timestampch: string
beforeAll(async () => {
hub = await createHub({})
kafkaProducer = await KafkaProducerWrapper.create(hub.KAFKA_CLIENT_RACK)
mockProducerObserver = new KafkaProducerObserver(kafkaProducer)
mockProducerObserver.resetKafkaProducer()
clickhouse = Clickhouse.create()
// Pre-create every topic ClickHouse's Kafka engine tables subscribe to. Otherwise any
// table whose topic is missing keeps retrying "Can't get assignment", which saturates
// ClickHouse's background scheduler and intermittently starves the consumers this suite
// waits on via delayUntilEventIngested — flaky timeouts in CI.
await ensureKafkaTopics(await clickhouse.getKafkaEngineTopics())
await clickhouse.exec('SYSTEM STOP MERGES')
organizationId = await createOrganization(hub.postgres)
})
beforeEach(async () => {
teamId = await createTeam(hub.postgres, organizationId)
mainTeam = (await getTeam(hub.postgres, teamId))!
timestamp = DateTime.fromISO('2020-01-01T12:00:05.200Z').toUTC()
timestamp2 = DateTime.fromISO('2020-02-02T12:00:05.200Z').toUTC()
timestampch = '2020-01-01 12:00:05.000'
newUserUuid = uuidFromDistinctId(teamId, newUserDistinctId)
oldUserUuid = uuidFromDistinctId(teamId, oldUserDistinctId)
firstUserUuid = uuidFromDistinctId(teamId, firstUserDistinctId)
secondUserUuid = uuidFromDistinctId(teamId, secondUserDistinctId)
personRepository = new PostgresPersonRepository(hub.postgres)
jest.spyOn(personRepository, 'fetchPerson')
jest.spyOn(personRepository, 'createPerson')
jest.spyOn(personRepository, 'updatePerson')
jest.spyOn(personRepository, 'updatePersonsBatch')
defaultRetryConfig.RETRY_INTERVAL_DEFAULT = 0
})
afterEach(() => {
jest.clearAllTimers()
jest.useRealTimers()
jest.restoreAllMocks()
})
afterAll(async () => {
await kafkaProducer.disconnect()
await closeHub(hub)
await clickhouse.exec('SYSTEM START MERGES')
clickhouse.close()
})
function personProcessor(
event: Partial<PluginEvent>,
propertyService?: PersonPropertyService,
mergeService?: PersonMergeService,
_customHub?: Hub,
processPerson = true,
timestampParam = timestamp,
team = mainTeam
) {
const fullEvent = {
team_id: teamId,
properties: {},
...event,
}
const personsStore = new BatchWritingPersonsStore(personRepository, createPersonOutputs(kafkaProducer))
const context = new PersonContext(
fullEvent as any,
team,
event.distinct_id!,
timestampParam,
processPerson,
createPersonOutputs(kafkaProducer),
new BatchBoundPersonsStore(personsStore, 0),
0,
createDefaultSyncMergeMode(),
false,
false
)
const processor = new PersonEventProcessor(
context,
propertyService ?? new PersonPropertyService(context),
mergeService ?? new PersonMergeService(context)
)
return processor
}
function personPropertyService(
event: Partial<PluginEvent>,
_customHub?: Hub,
processPerson = true,
timestampParam = timestamp,
team = mainTeam,
updateIsIdentified: boolean = false
) {
const fullEvent = {
team_id: teamId,
properties: {},
...event,
}
const personsStore = new BatchWritingPersonsStore(personRepository, createPersonOutputs(kafkaProducer))
const context = new PersonContext(
fullEvent as PluginEvent,
team,
event.distinct_id!,
timestampParam,
processPerson,
createPersonOutputs(kafkaProducer),
new BatchBoundPersonsStore(personsStore, 0),
0,
createDefaultSyncMergeMode(),
false,
false
)
context.updateIsIdentified = updateIsIdentified
return new PersonPropertyService(context)
}
function personMergeService(
event: Partial<PluginEvent>,
customHub?: Hub,
customPersonRepository?: PostgresPersonRepository,
processPerson = true,
timestampParam = timestamp,
team = mainTeam,
mergeMode = createDefaultSyncMergeMode(),
mergeEventsConfig?: { enabled: boolean; partitionCount: number; teamAllowlist?: string },
customPersonsStore?: BatchWritingPersonsStore
) {
const fullEvent = {
team_id: teamId,
properties: {},
...event,
}
// A custom store carries its own merge options; the merge policy
// params here configure only the store this helper builds.
const personsStore =
customPersonsStore ??
new BatchWritingPersonsStore(
customPersonRepository ??
(customHub ? new PostgresPersonRepository(customHub.postgres) : personRepository),
createPersonOutputs(kafkaProducer),
{
mergeEventsEnabled: mergeEventsConfig?.enabled ?? false,
mergeEventsPartitionCount: mergeEventsConfig?.partitionCount ?? 64,
// Allow-all default so the enabled/produce tests are unaffected by the
// team gate; pass an allowlist explicitly to exercise it.
mergeEventsTeamAllowlist: mergeEventsConfig?.teamAllowlist ?? '*',
}
)
const context = new PersonContext(
fullEvent as any,
team,
event.distinct_id!,
timestampParam,
processPerson,
createPersonOutputs(kafkaProducer),
new BatchBoundPersonsStore(personsStore, 0),
0,
mergeMode,
false,
false
)
return new PersonMergeService(context)
}
const sortPersons = (persons: InternalPerson[]) => persons.sort((a, b) => Number(a.id) - Number(b.id))
async function fetchPostgresPersonsH() {
return await fetchPostgresPersons(hub.postgres, teamId)
}
async function fetchPersonsRows() {
const query = `SELECT * FROM person FINAL WHERE team_id = ${teamId} ORDER BY _offset`
return await clickhouse.query<ClickHousePerson>(query)
}
async function fetchPersonsRowsWithVersionHigerEqualThan(version = 1) {
const query = `SELECT * FROM person FINAL WHERE team_id = ${teamId} AND version >= ${version}`
return await clickhouse.query(query)
}
async function fetchDistinctIdsClickhouse(person: InternalPerson) {
return clickhouse.fetchDistinctIdValues(person)
}
async function fetchDistinctIdsClickhouseVersion1() {
const query = `SELECT distinct_id FROM person_distinct_id2 FINAL WHERE team_id = ${teamId} AND version = 1`
return await clickhouse.query(query)
}
const getPersonEventsFromKafka = (): ClickHousePerson[] => {
return mockProducerObserver
.getProducedKafkaMessagesForTopic(KAFKA_PERSON)
.map((x) => x.value as unknown as ClickHousePerson)
}
const getDistinctIdEventsFromKafka = (): ClickHousePersonDistinctId2[] => {
return mockProducerObserver
.getProducedKafkaMessagesForTopic(KAFKA_PERSON_DISTINCT_ID)
.map((x) => x.value as unknown as ClickHousePersonDistinctId2)
}
describe('on person creation', () => {
it('creates deterministic person uuids that are different between teams', async () => {
const event_uuid = new UUIDT().toString()
const primaryTeamId = teamId
const [personPrimaryTeam, kafkaAcks] = await personPropertyService({
event: '$pageview',
distinct_id: newUserDistinctId,
uuid: event_uuid,
}).updateProperties()
const otherTeamId = await createTeam(hub.postgres, organizationId)
const otherTeam = (await getTeam(hub.postgres, otherTeamId))!
teamId = otherTeamId
const [personOtherTeam, kafkaAcksOther] = await personPropertyService(
{
event: '$pageview',
distinct_id: newUserDistinctId,
uuid: event_uuid,
},
undefined,
true,
timestamp,
otherTeam
).updateProperties()
await kafkaProducer.flush()
await kafkaAcks
await kafkaAcksOther
expect(personPrimaryTeam.uuid).toEqual(uuidFromDistinctId(primaryTeamId, newUserDistinctId))
expect(personOtherTeam.uuid).toEqual(uuidFromDistinctId(otherTeamId, newUserDistinctId))
expect(personPrimaryTeam.uuid).not.toEqual(personOtherTeam.uuid)
})
it('creates person if they are new', async () => {
const event_uuid = new UUIDT().toString()
const [person, kafkaAcks] = await personPropertyService({
event: '$pageview',
distinct_id: newUserDistinctId,
uuid: event_uuid,
// `null_byte` validates that `sanitizeJsonbValue` is working as expected
properties: { $set: { null_byte: '\u0000' } },
}).updateProperties()
await kafkaProducer.flush()
await kafkaAcks
expect(person).toEqual(
expect.objectContaining({
id: expect.any(String),
uuid: newUserUuid,
properties: { $creator_event_uuid: event_uuid, null_byte: '\uFFFD' },
created_at: timestamp,
version: 0,
is_identified: false,
})
)
expect(personRepository.fetchPerson).toHaveBeenCalledTimes(1)
expect(personRepository.updatePerson).not.toHaveBeenCalled()
// verify Postgres persons
const persons = sortPersons(await fetchPostgresPersonsH())
expect(persons.length).toEqual(1)
expect(persons[0]).toEqual(person)
// verify Postgres distinct_ids
const distinctIds = await fetchDistinctIdValues(hub.postgres, persons[0])
expect(distinctIds).toEqual(expect.arrayContaining([newUserDistinctId]))
})
it('handles person being created in a race condition', async () => {
const _newPerson = await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, newUserUuid, {
distinctId: newUserDistinctId,
})
jest.spyOn(personRepository, 'fetchPerson').mockImplementationOnce(() => {
return Promise.resolve(undefined)
})
const [person, kafkaAcks] = await personPropertyService({
event: '$pageview',
distinct_id: newUserDistinctId,
}).handleUpdate()
await kafkaProducer.flush()
await kafkaAcks
// if creation fails we should return the person that another thread already created
expect(person).toEqual(
expect.objectContaining({
id: expect.any(String),
uuid: newUserUuid,
properties: {},
created_at: timestamp,
version: 0,
is_identified: false,
})
)
expect(personRepository.updatePerson).not.toHaveBeenCalled()
// verify Postgres persons
const persons = sortPersons(await fetchPostgresPersonsH())
expect(persons.length).toEqual(1)
expect(persons[0]).toEqual(person)
// verify Postgres distinct_ids
const distinctIds = await fetchDistinctIdValues(hub.postgres, person)
expect(distinctIds).toEqual(expect.arrayContaining([newUserDistinctId]))
})
it('handles person being created in a race condition updates properties if needed', async () => {
const createdPerson = await createPerson(
hub,
timestamp,
{ b: 3, c: 4 },
{},
{},
teamId,
null,
false,
newUserUuid,
{ distinctId: newUserDistinctId }
)
let callCounter = 0
const fetchPersonSpy = jest.spyOn(personRepository, 'fetchPerson').mockImplementationOnce(() => {
callCounter++
if (callCounter < 2) {
return Promise.resolve(undefined)
}
return Promise.resolve(createdPerson)
})
const propertyService = personPropertyService({
event: '$pageview',
distinct_id: newUserDistinctId,
properties: {
$set_once: { c: 3, e: 4 },
$set: { b: 4 },
},
})
const [person, kafkaAcks] = await propertyService.handleUpdate()
const context = propertyService.getContext()
await flushPersonStoreToKafka(kafkaProducer, context.personStore, kafkaAcks)
// Explicitly restore the spy to prevent Jest state corruption
fetchPersonSpy.mockRestore()
// if creation fails we should return the person that another thread already created
expect(person).toMatchObject({
id: expect.any(String),
uuid: newUserUuid,
properties: { b: 4, c: 4, e: 4 },
created_at: timestamp,
version: 0,
is_identified: false,
})
// Batch mode uses updatePersonsBatch instead of updatePerson
expect(personRepository.updatePersonsBatch).toHaveBeenCalledTimes(1)
// verify Postgres persons
const persons = sortPersons(await fetchPostgresPersonsH())
expect(persons.length).toEqual(1)
expect(persons[0]).toMatchObject({
id: person.id,
uuid: newUserUuid,
created_at: timestamp,
is_identified: false,
properties: { b: 4, c: 4, e: 4 },
version: 1,
})
// verify Postgres distinct_ids
const distinctIds = await fetchDistinctIdValues(hub.postgres, person)
expect(distinctIds).toEqual(expect.arrayContaining([newUserDistinctId]))
})
it('creates person with properties', async () => {
const [person, kafkaAcks] = await personPropertyService({
event: '$pageview',
distinct_id: newUserDistinctId,
properties: {
$set_once: { a: 1, b: 2 },
$set: { b: 3, c: 4 },
},
}).updateProperties()
await kafkaProducer.flush()
await kafkaAcks
expect(person).toMatchObject({
id: expect.any(String),
uuid: newUserUuid,
properties: { a: 1, b: 3, c: 4 },
created_at: timestamp,
version: 0,
is_identified: false,
})
expect(personRepository.fetchPerson).toHaveBeenCalledTimes(1)
expect(personRepository.updatePerson).not.toHaveBeenCalled()
// verify Postgres persons
const persons = sortPersons(await fetchPostgresPersonsH())
expect(persons.length).toEqual(1)
expect(persons[0]).toEqual(person)
// verify Postgres distinct_ids
const distinctIds = await fetchDistinctIdValues(hub.postgres, persons[0])
expect(distinctIds).toEqual(expect.arrayContaining([newUserDistinctId]))
})
})
describe('on person update', () => {
it('updates person properties', async () => {
await createPerson(hub, timestamp, { b: 3, c: 4, toString: {} }, {}, {}, teamId, null, false, newUserUuid, {
distinctId: newUserDistinctId,
})
const propertyService = personPropertyService({
event: '$pageview',
distinct_id: newUserDistinctId,
properties: {
$set_once: { c: 3, e: 4 },
$set: { b: 4, toString: 1, null_byte: '\u0000' },
},
})
const [person, kafkaAcks] = await propertyService.updateProperties()
const context = propertyService.getContext()
await flushPersonStoreToKafka(kafkaProducer, context.personStore, kafkaAcks)
expect(person).toMatchObject({
id: expect.any(String),
uuid: newUserUuid,
properties: { b: 4, c: 4, e: 4, toString: 1, null_byte: '\u0000' },
is_identified: false,
})
expect(personRepository.fetchPerson).toHaveBeenCalledTimes(1)
// verify Postgres persons
const postgresPersons = sortPersons(await fetchPostgresPersonsH())
expect(postgresPersons.length).toEqual(1)
expect(postgresPersons[0]).toMatchObject({
id: person.id,
uuid: newUserUuid,
created_at: timestamp,
is_identified: false,
// `null_byte` validates that `sanitizeJsonbValue` is working as expected
properties: { b: 4, c: 4, e: 4, toString: 1, null_byte: '\uFFFD' },
version: 1,
})
})
it.each(['$$heatmap', '$exception'])('does not update person properties for %s', async (event: string) => {
const originalPersonProperties = { b: 3, c: 4, toString: {} }
await createPerson(hub, timestamp, originalPersonProperties, {}, {}, teamId, null, false, newUserUuid, {
distinctId: newUserDistinctId,
})
const [person, kafkaAcks] = await personPropertyService({
event: event,
distinct_id: newUserDistinctId,
properties: {
$set_once: { c: 3, e: 4 },
$set: { b: 4, toString: 1, null_byte: '\u0000' },
},
}).updateProperties()
await kafkaProducer.flush()
await kafkaAcks
expect(person).toMatchObject({
id: expect.any(String),
uuid: newUserUuid,
properties: originalPersonProperties,
created_at: timestamp,
version: 0,
is_identified: false,
})
expect(personRepository.fetchPerson).toHaveBeenCalledTimes(1)
// verify Postgres persons
const persons = sortPersons(await fetchPostgresPersonsH())
expect(persons.length).toEqual(1)
expect(persons[0]).toEqual(person)
})
it('updates person properties - no update if not needed', async () => {
await createPerson(hub, timestamp, { $current_url: 123 }, {}, {}, teamId, null, false, newUserUuid, {
distinctId: newUserDistinctId,
})
const [person, kafkaAcks] = await personPropertyService({
event: '$pageview',
distinct_id: newUserDistinctId,
properties: {
$set: { $current_url: 4 },
},
}).updateProperties()
await kafkaProducer.flush()
await kafkaAcks
expect(person).toEqual(
expect.objectContaining({
id: expect.any(String),
uuid: newUserUuid,
properties: { $current_url: 4 }, // Here we keep 4 for passing forward to PoE
created_at: timestamp,
version: 0,
is_identified: false,
})
)
expect(personRepository.fetchPerson).toHaveBeenCalledTimes(1)
// verify Postgres persons
const persons = sortPersons(await fetchPostgresPersonsH())
expect(persons).toEqual([
expect.objectContaining({
id: expect.any(String),
uuid: newUserUuid,
properties: { $current_url: 123 }, // We didn 't update this as it's auto added and it's not a person event
created_at: timestamp,
version: 0,
is_identified: false,
}),
])
})
it('skips database write when only filtered properties change (batch-level filtering)', async () => {
await createPerson(hub, timestamp, { $current_url: 123 }, {}, {}, teamId, null, false, newUserUuid, {
distinctId: newUserDistinctId,
})
const propertyService = personPropertyService({
event: '$pageview',
distinct_id: newUserDistinctId,
properties: {
$set: { $current_url: 4 },
},
})
const [person, kafkaAcks] = await propertyService.updateProperties()
const context = propertyService.getContext()
await flushPersonStoreToKafka(kafkaProducer, context.personStore, kafkaAcks)
expect(person).toEqual(
expect.objectContaining({
id: expect.any(String),
uuid: newUserUuid,
properties: { $current_url: 4 },
created_at: timestamp,
is_identified: false,
})
)
expect(personRepository.fetchPerson).toHaveBeenCalledTimes(1)
const persons = sortPersons(await fetchPostgresPersonsH())
expect(persons.length).toEqual(1)
expect(persons[0]).toEqual(
expect.objectContaining({
id: person.id,
uuid: newUserUuid,
created_at: timestamp,
is_identified: false,
properties: { $current_url: 123 },
version: 0,
})
)
})
it('writes to database when non-filtered properties change', async () => {
await createPerson(hub, timestamp, { name: 'John' }, {}, {}, teamId, null, false, newUserUuid, {
distinctId: newUserDistinctId,
})
const propertyService = personPropertyService({
event: '$pageview',
distinct_id: newUserDistinctId,
properties: {
$set: { name: 'Jane' },
},
})
const [person, kafkaAcks] = await propertyService.updateProperties()
const context = propertyService.getContext()
await flushPersonStoreToKafka(kafkaProducer, context.personStore, kafkaAcks)
expect(person).toEqual(
expect.objectContaining({
id: expect.any(String),
uuid: newUserUuid,
properties: { name: 'Jane' },
created_at: timestamp,
is_identified: false,
})
)
expect(personRepository.fetchPerson).toHaveBeenCalledTimes(1)
const persons = sortPersons(await fetchPostgresPersonsH())
expect(persons.length).toEqual(1)
expect(persons[0]).toEqual(
expect.objectContaining({
id: person.id,
uuid: newUserUuid,
created_at: timestamp,
is_identified: false,
properties: { name: 'Jane' },
version: 1,
})
)
})
it('updates person properties - always update if undefined before', async () => {
await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, newUserUuid, {
distinctId: newUserDistinctId,
})
const propertyService = personPropertyService({
event: '$pageview',
distinct_id: newUserDistinctId,
properties: {
$set: { $initial_current_url: 4 },
},
})
const [person, kafkaAcks] = await propertyService.updateProperties()
const context = propertyService.getContext()
await flushPersonStoreToKafka(kafkaProducer, context.personStore, kafkaAcks)
expect(person).toEqual(
expect.objectContaining({
id: expect.any(String),
uuid: newUserUuid,
properties: { $initial_current_url: 4 }, // Here we keep 4 for passing forward to PoE
created_at: timestamp,
is_identified: false,
})
)
expect(personRepository.fetchPerson).toHaveBeenCalledTimes(1)
// verify Postgres persons
const persons = sortPersons(await fetchPostgresPersonsH())
expect(persons.length).toEqual(1)
expect(persons[0]).toEqual(
expect.objectContaining({
id: person.id,
uuid: newUserUuid,
created_at: timestamp,
is_identified: false,
properties: { $initial_current_url: 4 },
version: 1,
})
) // We updated PG as it was undefined before
})
it('updates person properties - always update for initial properties', async () => {
await createPerson(
hub,
timestamp,
{ $initial_current_url: 123 },
{},
{},
teamId,
null,
false,
newUserUuid,
{ distinctId: newUserDistinctId }
)
const propertyService = personPropertyService({
event: '$pageview',
distinct_id: newUserDistinctId,
properties: {
$set: { $initial_current_url: 4 },
},
})
const [person, kafkaAcks] = await propertyService.updateProperties()
const context = propertyService.getContext()
await flushPersonStoreToKafka(kafkaProducer, context.personStore, kafkaAcks)
expect(person).toEqual(
expect.objectContaining({
id: expect.any(String),
uuid: newUserUuid,
properties: { $initial_current_url: 4 }, // Here we keep 4 for passing forward to PoE
created_at: timestamp,
is_identified: false,
})
)
expect(personRepository.fetchPerson).toHaveBeenCalledTimes(1)
// verify Postgres persons
const persons = sortPersons(await fetchPostgresPersonsH())
expect(persons.length).toEqual(1)
expect(persons[0]).toEqual(
expect.objectContaining({
id: person.id,
uuid: newUserUuid,
created_at: timestamp,
is_identified: false,
properties: { $initial_current_url: 4 },
version: 1,
})
) // We updated PG as it's an initial property
})
it('updating with cached person data shortcuts to update directly', async () => {
const personInitial = await createPerson(
hub,
timestamp,
{ b: 3, c: 4 },
{},
{},
teamId,
null,
false,
newUserUuid,
{ distinctId: newUserDistinctId }
)
const event = {
event: '$pageview',
distinct_id: newUserDistinctId,
properties: {
$set_once: { c: 3, e: 4 },
$set: { b: 4 },
},
}
// create mock merge service,
const mergeService = personMergeService(event)
jest.spyOn(mergeService, 'handleIdentifyOrAlias').mockReturnValue(
Promise.resolve({
success: true,
person: personInitial,
kafkaAck: Promise.resolve(),
needsPersonUpdate: true,
})
)
const personS = personProcessor(event, undefined, mergeService)
const result = await personS.processEvent()
const context = personS.getContext()
await flushPersonStoreToKafka(
kafkaProducer,
context.personStore,
Promise.all(result.sideEffects).then(() => undefined)
)
expect(result.type).toBe(PipelineResultType.OK)
if (isOkResult(result)) {
const person = result.value
expect(person).toEqual(
expect.objectContaining({
id: expect.any(String),
uuid: newUserUuid,
properties: { b: 4, c: 4, e: 4 },
created_at: timestamp,
is_identified: false,
})
)
}
expect(personRepository.fetchPerson).toHaveBeenCalledTimes(0)
// verify Postgres persons
const persons = sortPersons(await fetchPostgresPersonsH())
expect(persons.length).toEqual(1)
expect(persons[0]).toMatchObject({
id: expect.any(String),
uuid: newUserUuid,
created_at: timestamp,
is_identified: false,
properties: { b: 4, c: 4, e: 4 },
version: 1,
})
})
it('does not update person if not needed', async () => {
await createPerson(hub, timestamp, { b: 3, c: 4 }, {}, {}, teamId, null, false, newUserUuid, {
distinctId: newUserDistinctId,
})
const propertyService = personPropertyService({
event: '$pageview',
distinct_id: newUserDistinctId,
properties: {
$set_once: { c: 3 },
$set: { b: 3 },
},
})
const [person, kafkaAcks] = await propertyService.updateProperties()
const context = propertyService.getContext()
await flushPersonStoreToKafka(kafkaProducer, context.personStore, kafkaAcks)
expect(person).toEqual(
expect.objectContaining({
id: expect.any(String),
uuid: newUserUuid,
properties: { b: 3, c: 4 },
created_at: timestamp,
version: 0,
is_identified: false,
})
)
expect(personRepository.fetchPerson).toHaveBeenCalledTimes(1)
expect(personRepository.updatePerson).not.toHaveBeenCalled()
// verify Postgres persons
const persons = sortPersons(await fetchPostgresPersonsH())
expect(persons.length).toEqual(1)
expect(persons[0]).toEqual(person)
})
it('marks user as is_identified', async () => {
await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, newUserUuid, {
distinctId: newUserDistinctId,
})
const personS = personPropertyService(
{
event: '$pageview',
distinct_id: newUserDistinctId,
properties: {},
},
undefined,
true,
timestamp,
mainTeam,
true
)
const [person, kafkaAcks] = await personS.updateProperties()
const context = personS.getContext()
await flushPersonStoreToKafka(kafkaProducer, context.personStore, kafkaAcks)
expect(person).toEqual(
expect.objectContaining({
id: expect.any(String),
uuid: newUserUuid,
properties: {},
created_at: timestamp,
is_identified: true,
})
)
expect(personRepository.fetchPerson).toHaveBeenCalledTimes(1)
// Batch mode uses updatePersonsBatch instead of updatePerson
expect(personRepository.updatePersonsBatch).toHaveBeenCalledTimes(1)
// verify Postgres persons
const persons = sortPersons(await fetchPostgresPersonsH())
expect(persons.length).toEqual(1)
expect(persons[0]).toEqual(
expect.objectContaining({
id: person.id,
uuid: newUserUuid,
created_at: timestamp,
is_identified: true,
properties: {},
version: 1,
})
)
await personS.updateProperties()
// No additional batch call since no changes
expect(personRepository.updatePersonsBatch).toHaveBeenCalledTimes(1)
})
it('handles race condition when person provided has been merged', async () => {
// TODO: we don't handle this currently person having been changed / updated properties can get overridden
// Pass in a person, but another thread merges it - we shouldn't error in this case, but instead if we couldn't update we should retry?
const mergeDeletedPerson: InternalPerson = {
created_at: timestamp,
version: 0,
id: '0',
team_id: teamId,
properties: { a: 5, b: 7 },
is_user_id: 0,
is_identified: false,
uuid: uuidFromDistinctId(teamId, 'deleted-user'),
properties_last_updated_at: {},
properties_last_operation: null,
last_seen_at: null,
}
await createPerson(
hub,
timestamp,
{ a: 6, c: 8 },
{},
{},
teamId,
null,
true,
newUserUuid,
{ distinctId: newUserDistinctId },
[{ distinctId: oldUserDistinctId }]
) // the merged Person
const event = {
event: '$pageview',
distinct_id: newUserDistinctId,
properties: { $set: { a: 7, d: 9 } },
}
// create mock merge service
const mergeService = personMergeService(event)
jest.spyOn(mergeService, 'handleIdentifyOrAlias').mockReturnValue(
Promise.resolve({
success: true,
person: mergeDeletedPerson,
kafkaAck: Promise.resolve(),
needsPersonUpdate: true,
})
)
const personS = personProcessor(event, undefined, mergeService)
const context = personS.getContext()
const result = await personS.processEvent()
await flushPersonStoreToKafka(
kafkaProducer,
context.personStore,
Promise.all(result.sideEffects).then(() => undefined)
)
// Return logic is still unaware that merge happened
expect(result.type).toBe(PipelineResultType.OK)
if (isOkResult(result)) {
const person = result.value
expect(person).toMatchObject({
id: expect.any(String),
uuid: mergeDeletedPerson.uuid,
properties: { a: 7, b: 7, d: 9 },
created_at: timestamp,
version: 0,
is_identified: false,
})
}
expect(personRepository.fetchPerson).toHaveBeenCalledTimes(1)
// The batch and the single-person fallback both miss the merged-away row; the refreshed update lands.
expect(personRepository.updatePersonsBatch).toHaveBeenCalledTimes(3)
expect(personRepository.updatePerson).not.toHaveBeenCalled()
// verify Postgres persons
const persons = sortPersons(await fetchPostgresPersonsH())
expect(persons.length).toEqual(1)
expect(persons[0]).toMatchObject({
id: expect.any(String),
uuid: newUserUuid,
properties: { a: 7, c: 8, d: 9 },
created_at: timestamp,
version: 1,
is_identified: true,
})
})
})
describe('on $identify event', () => {
it(`no-op when $anon_distinct_id not passed`, async () => {
const mergeService: PersonMergeService = personMergeService({
event: '$identify',
distinct_id: newUserDistinctId,
properties: {
$set: { foo: 'bar' },
},
})
const result = await mergeService.handleIdentifyOrAlias()
expect(result.success).toBe(true)
if (!result.success) {
throw new Error('Expected successful merge result')
}
const person = result.person
const kafkaAcks = result.kafkaAck
const context = mergeService.getContext()
await flushPersonStoreToKafka(kafkaProducer, context.personStore, kafkaAcks)
expect(person).toEqual(undefined)
const persons = await fetchPostgresPersonsH()
expect(persons.length).toEqual(0)
})
it(`creates person with both distinct_ids and marks user as is_identified when $anon_distinct_id passed`, async () => {
const mergeService: PersonMergeService = personMergeService({
event: '$identify',
distinct_id: newUserDistinctId,
properties: {
$set: { foo: 'bar' },
$anon_distinct_id: oldUserDistinctId,
},
})
const result = await mergeService.handleIdentifyOrAlias()
expect(result.success).toBe(true)
if (!result.success) {
throw new Error('Expected successful merge result')
}
const person = result.person
const kafkaAcks = result.kafkaAck
const context = mergeService.getContext()
await flushPersonStoreToKafka(kafkaProducer, context.personStore, kafkaAcks)
expect(person).toEqual(
expect.objectContaining({
id: expect.any(String),
uuid: newUserUuid,
properties: { foo: 'bar' },
created_at: timestamp,
version: 0,
is_identified: true,
})
)
expect(personRepository.createPerson).toHaveBeenCalledTimes(1)
expect(personRepository.updatePerson).not.toHaveBeenCalled()
const persons = await fetchPostgresPersonsH()
expect(persons.length).toEqual(1)
expect(persons[0]).toEqual(
expect.objectContaining({
id: expect.any(String),
uuid: newUserUuid,
properties: { foo: 'bar' },
created_at: timestamp,
version: 0,
is_identified: true,
})
)
const distinctIds = await fetchDistinctIdValues(hub.postgres, persons[0])
expect(distinctIds).toEqual(expect.arrayContaining([oldUserDistinctId, newUserDistinctId]))
})
it(`writes the identified flag at the call when both ids already share an unidentified person`, async () => {
await createPerson(
hub,
timestamp,
{},
{},
{},
teamId,
null,
false,
newUserUuid,
{ distinctId: newUserDistinctId },
[{ distinctId: oldUserDistinctId }]
)
const mergeService: PersonMergeService = personMergeService({
event: '$identify',
distinct_id: newUserDistinctId,
properties: {
$anon_distinct_id: oldUserDistinctId,
},
})
const result = await mergeService.handleIdentifyOrAlias()
expect(result.success).toBe(true)
if (!result.success) {
throw new Error('Expected successful merge result')
}
const person = result.person
const kafkaAcks = result.kafkaAck
const context = mergeService.getContext()
await flushPersonStoreToKafka(kafkaProducer, context.personStore, kafkaAcks)
// The same-person outcome commits the identified flag at the call, ahead of the batch flush.
expect(person).toMatchObject({
id: expect.any(String),
uuid: newUserUuid,
properties: {},
created_at: timestamp,
version: 1,
is_identified: true,
})
expect(mergeService.getUpdateIsIdentified()).toBeTruthy()
const persons = await fetchPostgresPersonsH()
expect(persons.length).toEqual(1)
expect(persons[0]).toMatchObject({
id: person!.id,
uuid: newUserUuid,
created_at: timestamp,
is_identified: true,
properties: {},
version: 1,
})
})
it(`add distinct id and marks user is_identified when passed $anon_distinct_id person does not exists and distinct_id does`, async () => {
await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, newUserUuid, {
distinctId: newUserDistinctId,
})
const mergeService: PersonMergeService = personMergeService({
event: '$identify',
distinct_id: newUserDistinctId,
properties: {
$anon_distinct_id: oldUserDistinctId,
},
})
const result = await mergeService.handleIdentifyOrAlias()
expect(result.success).toBe(true)
if (!result.success) {
throw new Error('Expected successful merge result')
}
const person = result.person
const kafkaAcks = result.kafkaAck
const context = mergeService.getContext()
await flushPersonStoreToKafka(kafkaProducer, context.personStore, kafkaAcks)
const persons = await fetchPostgresPersonsH()
// The attach commits the identified flag with the mapping, ahead of the batch flush.
expect(person).toMatchObject({
id: expect.any(String),
uuid: newUserUuid,
properties: {},
created_at: timestamp,
version: 1,
is_identified: true,
})
expect(mergeService.getUpdateIsIdentified()).toBeTruthy()
// verify Postgres persons
expect(persons.length).toEqual(1)
expect(persons[0]).toMatchObject({
id: person!.id,
uuid: newUserUuid,
created_at: timestamp,
is_identified: true,
properties: {},
version: 1,
})
// verify Postgres distinct_ids
const distinctIds = await fetchDistinctIdValues(hub.postgres, persons[0])
expect(distinctIds).toEqual(expect.arrayContaining([oldUserDistinctId, newUserDistinctId]))
})
it(`merges two persons and applies $set properties from multiple $identify events in same batch`, async () => {
// When two persons exist (one for anon_distinct_id, one for distinct_id) and multiple
// $identify events with different $set properties are processed in the same batch,
// all $set properties from all events should be applied to the merged person.
const sharedPersonsStore = new BatchWritingPersonsStore(
personRepository,
createPersonOutputs(kafkaProducer)
)
const createProcessorWithSharedStore = (event: Partial<PluginEvent>, distinctId: string) => {
const fullEvent = {
team_id: teamId,
properties: {},
...event,
}
const context = new PersonContext(
fullEvent as any,
mainTeam,
distinctId,
timestamp,
true,
createPersonOutputs(kafkaProducer),
new BatchBoundPersonsStore(sharedPersonsStore, 0),
0,
createDefaultSyncMergeMode(),
false,
false
)
return new PersonEventProcessor(
context,
new PersonPropertyService(context),
new PersonMergeService(context)
)
}
// Events for anon distinct_id (oldUserDistinctId) - these create a person
const anonEvent1 = createProcessorWithSharedStore(
{
event: 'Application Installed',
distinct_id: oldUserDistinctId,
properties: {},
},
oldUserDistinctId
)
await anonEvent1.processEvent()
const anonEvent2 = createProcessorWithSharedStore(
{
event: '$screen',
distinct_id: oldUserDistinctId,
properties: { $screen_name: '/welcome' },
},
oldUserDistinctId
)
await anonEvent2.processEvent()
// Event for main distinct_id - $feature_flag_called creates the person
const featureFlagProcessor = createProcessorWithSharedStore(
{
event: '$feature_flag_called',
distinct_id: newUserDistinctId,
properties: {
$feature_flag: 'test-flag',
$feature_flag_response: true,
},
},
newUserDistinctId
)
await featureFlagProcessor.processEvent()
// First $identify - merges anon into main
const identify1Processor = createProcessorWithSharedStore(
{
event: '$identify',
distinct_id: newUserDistinctId,
properties: {
$anon_distinct_id: oldUserDistinctId,
$set: {
hasAddedInternship: false,
hasAddedLanguages: false,
isTempUser: true,
wasOpenedOnMobileDevice: true,
},
},
},
newUserDistinctId
)
await identify1Processor.processEvent()
// Second $identify - same merge but with different $set properties
const identify2Processor = createProcessorWithSharedStore(
{
event: '$identify',
distinct_id: newUserDistinctId,
properties: {
$anon_distinct_id: oldUserDistinctId,
$set: {
email: null,
firstName: null,
onboardingStatus: { completed: false },
isTempUser: true,
},
},
},
newUserDistinctId
)
const identify2Result = await identify2Processor.processEvent()
expect(identify2Result.type).toBe(PipelineResultType.OK)
// More events after identify
const postIdentifyEvent = createProcessorWithSharedStore(
{
event: 'answer_onboarding_question',
distinct_id: newUserDistinctId,
properties: {
$set: {
selectedAge: null,
lastSearchResultCount: 37,
selfAttributedReferrer: 'C2',
primarySelectedJobGroupType: 'TRAINING',
},
},
},
newUserDistinctId
)
await postIdentifyEvent.processEvent()
// Flush all at once
const kafkaMessages = await sharedPersonsStore.flush()
const personOutputs = createPersonOutputs(kafkaProducer)
await Promise.all(
kafkaMessages
.flatMap((m) => m.messages)
.map((msg) => personOutputs.produce(msg.output, { value: msg.value, key: null }))
)
await kafkaProducer.flush()
// Verify Postgres person
const persons = await fetchPostgresPersonsH()
expect(persons.length).toEqual(1)
// The person should have properties from BOTH $identify events AND the post-identify event
expect(persons[0]).toMatchObject({
uuid: expect.any(String),
is_identified: true,
})
// Check specific properties from second $identify
expect(persons[0].properties).toMatchObject({
onboardingStatus: { completed: false },
isTempUser: true,
})
// Check properties from first $identify
expect(persons[0].properties).toMatchObject({
hasAddedInternship: false,
wasOpenedOnMobileDevice: true,
})
// Check properties from post-identify event
expect(persons[0].properties).toMatchObject({
selectedAge: null,
lastSearchResultCount: 37,
selfAttributedReferrer: 'C2',
primarySelectedJobGroupType: 'TRAINING',
})
// Verify both distinct_ids are linked
const distinctIds = await fetchDistinctIdValues(hub.postgres, persons[0])
expect(distinctIds).toEqual(expect.arrayContaining([oldUserDistinctId, newUserDistinctId]))
})
it(`setCachedPersonForUpdate with null overwrites existing cached person updates`, async () => {
// Calling setCachedPersonForUpdate(distinctId, null) destroys any existing cached
// person updates for that distinctId, causing pending property updates to be lost
// when the batch is flushed.
const sharedPersonsStore = new BatchWritingPersonsStore(
personRepository,
createPersonOutputs(kafkaProducer)
)
// Helper to create processor with shared store
const createProcessorWithSharedStore = (event: Partial<PluginEvent>, distinctId: string) => {
const fullEvent = {
team_id: teamId,
properties: {},
...event,
}
const context = new PersonContext(
fullEvent as any,
mainTeam,
distinctId,
timestamp,
true,
createPersonOutputs(kafkaProducer),
new BatchBoundPersonsStore(sharedPersonsStore, 0),
0,
createDefaultSyncMergeMode(),
false,
false
)
return new PersonEventProcessor(
context,
new PersonPropertyService(context),
new PersonMergeService(context)
)
}
// Simulate Thread 2: Process events for anon-id first, creating person B
// First event creates the person
const anonEvent1 = createProcessorWithSharedStore(
{
event: '$set',
distinct_id: oldUserDistinctId,
properties: {
$set: {
initialProperty: 'initial-value',
},
},
},
oldUserDistinctId
)
await anonEvent1.processEvent()
// Second event adds pending updates to properties_to_set
const anonEvent2 = createProcessorWithSharedStore(
{
event: '$set',
distinct_id: oldUserDistinctId,
properties: {
$set: {
anonProperty: 'from-anon-events',
},
},
},
oldUserDistinctId
)
await anonEvent2.processEvent()
// Verify person B was created and has pending updates cached
const cachedPersonB = sharedPersonsStore.getCachedPersonForUpdateByDistinctId(teamId, oldUserDistinctId)
expect(cachedPersonB).not.toBeNull()
// The initial property is in properties (written to DB during createPerson)
expect(cachedPersonB?.properties).toMatchObject({
initialProperty: 'initial-value',
})
// The second property is in properties_to_set (pending flush)
expect(cachedPersonB?.properties_to_set).toMatchObject({
anonProperty: 'from-anon-events',
})
// Call setCachedPersonForUpdate with null - this overwrites the cached person
sharedPersonsStore.setCachedPersonForUpdate(teamId, oldUserDistinctId, null)
// Verify the cache was destroyed
const cachedAfterNull = sharedPersonsStore.getCachedPersonForUpdateByDistinctId(teamId, oldUserDistinctId)
expect(cachedAfterNull).toBeUndefined()
// Create another person for a different distinct_id
const mainEvent = createProcessorWithSharedStore(
{
event: '$feature_flag_called',
distinct_id: newUserDistinctId,
properties: {
$set: {
mainProperty: 'from-main-events',
},
},
},
newUserDistinctId
)
await mainEvent.processEvent()
// Flush the batch
await sharedPersonsStore.flush()
const persons = await fetchPostgresPersonsH()
expect(persons.length).toEqual(2)
// The pending property update (anonProperty) was lost because the cache was cleared
const personWithAnonProperty = persons.find((p) => 'anonProperty' in p.properties)
expect(personWithAnonProperty).toBeUndefined()
})
it(`fetchForUpdate preserves cached person when DB query returns null but cache was populated concurrently`, async () => {
// When fetchForUpdate's DB query returns null, it checks if the cache was populated
// by another concurrent operation before caching null. If the cache now has data,
// it returns the cached person instead of overwriting with null.
const sharedPersonsStore = new BatchWritingPersonsStore(
personRepository,
createPersonOutputs(kafkaProducer)
)
// Helper to create processor with shared store
const createProcessorWithSharedStore = (event: Partial<PluginEvent>, distinctId: string) => {
const fullEvent = {
team_id: teamId,
properties: {},
...event,
}
const context = new PersonContext(
fullEvent as any,
mainTeam,
distinctId,
timestamp,
true,
createPersonOutputs(kafkaProducer),
new BatchBoundPersonsStore(sharedPersonsStore, 0),
0,
createDefaultSyncMergeMode(),
false,
false
)
return new PersonEventProcessor(
context,
new PersonPropertyService(context),
new PersonMergeService(context)
)
}
// Thread 2: Process events for anon-id, creating person B with pending updates
const anonEvent1 = createProcessorWithSharedStore(
{
event: '$set',
distinct_id: oldUserDistinctId,
properties: {
$set: {
initialProperty: 'initial-value',
},
},
},
oldUserDistinctId
)
await anonEvent1.processEvent()
const anonEvent2 = createProcessorWithSharedStore(
{
event: '$set',
distinct_id: oldUserDistinctId,
properties: {
$set: {
anonProperty: 'from-anon-events',
},
},
},
oldUserDistinctId
)
await anonEvent2.processEvent()
// Verify person B was cached with pending updates
const cachedPersonB = sharedPersonsStore.getCachedPersonForUpdateByDistinctId(teamId, oldUserDistinctId)
expect(cachedPersonB).not.toBeNull()
expect(cachedPersonB?.properties_to_set).toMatchObject({
anonProperty: 'from-anon-events',
})
// Simulate the check-then-set pattern used in fetchForUpdate:
// Only cache null if the cache is still empty
const currentCache = sharedPersonsStore.getCachedPersonForUpdateByDistinctId(teamId, oldUserDistinctId)
if (currentCache === undefined) {
sharedPersonsStore.setCachedPersonForUpdate(teamId, oldUserDistinctId, null)
}
// The cache should be preserved because currentCache was defined
const cachedAfterCheck = sharedPersonsStore.getCachedPersonForUpdateByDistinctId(teamId, oldUserDistinctId)
expect(cachedAfterCheck).not.toBeNull()
expect(cachedAfterCheck).not.toBeUndefined()
expect(cachedAfterCheck?.properties_to_set).toMatchObject({
anonProperty: 'from-anon-events',
})
// Create another person
const mainEvent = createProcessorWithSharedStore(
{
event: '$set',
distinct_id: newUserDistinctId,
properties: {
$set: {
mainProperty: 'from-main-events',
},
},
},
newUserDistinctId
)
await mainEvent.processEvent()
await sharedPersonsStore.flush()
const persons = await fetchPostgresPersonsH()
expect(persons.length).toEqual(2)
// Both persons should have their properties preserved
const personWithAnonProperty = persons.find((p) => 'anonProperty' in p.properties)
expect(personWithAnonProperty).toBeDefined()
expect(personWithAnonProperty?.properties).toMatchObject({
anonProperty: 'from-anon-events',
})
const personWithMainProperty = persons.find((p) => 'mainProperty' in p.properties)
expect(personWithMainProperty).toBeDefined()
expect(personWithMainProperty?.properties).toMatchObject({
mainProperty: 'from-main-events',
})
})
it(`add distinct id and marks user as is_identified when passed $anon_distinct_id person exists and distinct_id does not`, async () => {
await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, oldUserUuid, {
distinctId: oldUserDistinctId,
})
const mergeService: PersonMergeService = personMergeService({
event: '$identify',
distinct_id: newUserDistinctId,
properties: {
$anon_distinct_id: oldUserDistinctId,
},
})
const result = await mergeService.handleIdentifyOrAlias()
expect(result.success).toBe(true)
if (!result.success) {
throw new Error('Expected successful merge result')
}
const person = result.person
const kafkaAcks = result.kafkaAck
const context = mergeService.getContext()
await flushPersonStoreToKafka(kafkaProducer, context.personStore, kafkaAcks)
const persons = await fetchPostgresPersonsH()
// The attach commits the identified flag with the mapping, ahead of the batch flush.
expect(person).toMatchObject({
id: expect.any(String),
uuid: oldUserUuid,
properties: {},
created_at: timestamp,
version: 1,
is_identified: true,
})
expect(mergeService.getUpdateIsIdentified()).toBeTruthy()
expect(personRepository.updatePerson).not.toHaveBeenCalled()
// verify Postgres persons
expect(persons.length).toEqual(1)
expect(persons[0]).toMatchObject({
id: person!.id,
uuid: oldUserUuid,
created_at: timestamp,
is_identified: true,
properties: {},
version: 1,
})
// verify Postgres distinct_ids
const distinctIds = await fetchDistinctIdValues(hub.postgres, persons[0])
expect(distinctIds).toEqual(expect.arrayContaining([oldUserDistinctId, newUserDistinctId]))
})
it.skip(`merge into distinct_id person and marks user as is_identified when both persons have is_identified false`, async () => {
await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, oldUserUuid, {
distinctId: oldUserDistinctId,
})
await createPerson(hub, timestamp2, {}, {}, {}, teamId, null, false, newUserUuid, {
distinctId: newUserDistinctId,
})
const mergeService: PersonMergeService = personMergeService({
event: '$identify',
distinct_id: newUserDistinctId,
properties: {
$anon_distinct_id: oldUserDistinctId,
},
})
const result = await mergeService.handleIdentifyOrAlias()
expect(result.success).toBe(true)
if (!result.success) {
throw new Error('Expected successful merge result')
}
const person = result.person
const kafkaAcks = result.kafkaAck
const context = mergeService.getContext()
await flushPersonStoreToKafka(kafkaProducer, context.personStore, kafkaAcks)
expect(person).toMatchObject({
id: expect.any(String),
uuid: expect.any(String),
properties: {},
created_at: timestamp,
version: 1,
is_identified: true,
})
// verify Postgres persons
const persons = sortPersons(await fetchPostgresPersonsH())
expect(persons.length).toEqual(1)
expect(persons[0]).toMatchObject({
id: person!.id,
uuid: newUserUuid,
created_at: timestamp,
is_identified: true,
properties: {},
version: 1,
})
expect([newUserUuid, oldUserUuid]).toContain(persons[0].uuid)
// verify Postgres distinct_ids
const distinctIds = await fetchDistinctIdValues(hub.postgres, persons[0])
expect(distinctIds).toEqual(expect.arrayContaining([oldUserDistinctId, newUserDistinctId]))
expect(getPersonEventsFromKafka().filter((x) => x.version >= 1).length).toEqual(2)
// verify ClickHouse persons
const clickhousePersons = getPersonEventsFromKafka()
.filter((x) => x.version >= 1)
.sort((a, b) => a.version - b.version) // but verify full state
expect(clickhousePersons.length).toEqual(2)
expect(clickhousePersons).toEqual(
expect.arrayContaining([
expect.objectContaining({
id: expect.any(String),
version: 1,
is_identified: 1,
}),
expect.objectContaining({
id: expect.any(String),
is_deleted: 1,
version: 1,
}),
])
)
expect(new Set(clickhousePersons.map((p) => p.id))).toEqual(new Set([newUserUuid, oldUserUuid]))
// verify ClickHouse distinct_ids
const clickhouseDistinctIds = getDistinctIdEventsFromKafka()
.filter((x) => x.person_id === persons[0].uuid)
.map((x) => x.distinct_id)
expect(clickhouseDistinctIds).toEqual(expect.arrayContaining([oldUserDistinctId, newUserDistinctId]))
})
it(`merge into distinct_id person and marks user as is_identified when distinct_id user is identified and $anon_distinct_id user is not`, async () => {
await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, oldUserUuid, {
distinctId: oldUserDistinctId,
})
await createPerson(hub, timestamp2, {}, {}, {}, teamId, null, true, newUserUuid, {
distinctId: newUserDistinctId,
})
const mergeService = personMergeService({
event: '$identify',
distinct_id: newUserDistinctId,
properties: {
$anon_distinct_id: oldUserDistinctId,
},
})
const result = await mergeService.handleIdentifyOrAlias()
expect(result.success).toBe(true)
if (!result.success) {
throw new Error('Expected successful merge result')
}
const person = result.person
const kafkaAcks = result.kafkaAck
const context = mergeService.getContext()
await flushPersonStoreToKafka(kafkaProducer, context.personStore, kafkaAcks)
expect(person).toEqual(
expect.objectContaining({
id: expect.any(String),
uuid: expect.any(String),
properties: {},
created_at: timestamp,
version: 1,
is_identified: true,
})
)
// verify Postgres persons
const persons = sortPersons(await fetchPostgresPersonsH())
expect(persons.length).toEqual(1)
expect(persons[0]).toMatchObject({
id: person!.id,
uuid: newUserUuid,
created_at: timestamp,
is_identified: true,
properties: {},
version: 1,
})
expect([newUserUuid, oldUserUuid]).toContain(persons[0].uuid)
// verify Postgres distinct_ids
const distinctIds = await fetchDistinctIdValues(hub.postgres, persons[0])
expect(distinctIds).toEqual(expect.arrayContaining([oldUserDistinctId, newUserDistinctId]))
// verify ClickHouse persons
await clickhouse.delayUntilEventIngested(() => fetchPersonsRowsWithVersionHigerEqualThan(), 2) // wait until merge and delete processed
const clickhousePersons = await fetchPersonsRows() // but verify full state
expect(clickhousePersons.length).toEqual(2)
expect(clickhousePersons).toEqual(
expect.arrayContaining([
expect.objectContaining({
id: expect.any(String),
properties: '{}',
created_at: timestampch,
version: 1,
is_identified: 1,
}),
expect.objectContaining({
id: expect.any(String),
is_deleted: 1,
version: 1,
}),
])
)
expect(new Set(clickhousePersons.map((p) => p.id))).toEqual(new Set([newUserUuid, oldUserUuid]))
// verify ClickHouse distinct_ids
await clickhouse.delayUntilEventIngested(() => fetchDistinctIdsClickhouseVersion1())
const clickHouseDistinctIds = await fetchDistinctIdsClickhouse(persons[0])
expect(clickHouseDistinctIds).toEqual(expect.arrayContaining([oldUserDistinctId, newUserDistinctId]))
})
it(`does not merge people when distinct_id user is not identified and $anon_distinct_id user is`, async () => {
await createPerson(hub, timestamp, {}, {}, {}, teamId, null, true, oldUserUuid, {
distinctId: oldUserDistinctId,
})
await createPerson(hub, timestamp2, {}, {}, {}, teamId, null, false, newUserUuid, {
distinctId: newUserDistinctId,
})
const personS = personMergeService({
event: '$identify',
distinct_id: newUserDistinctId,
properties: {
$anon_distinct_id: oldUserDistinctId,
},
})
const result = await personS.handleIdentifyOrAlias()
expect(result.success).toBe(true)
if (!result.success) {
throw new Error('Expected successful merge result')
}
const person = result.person
const kafkaAcks = result.kafkaAck
await kafkaProducer.flush()
await kafkaAcks
expect(personS.getUpdateIsIdentified()).toBeTruthy()
// The refused merge commits the target's identified flag at the call, ahead of the batch flush.
expect(person).toEqual(
expect.objectContaining({
id: expect.any(String),
uuid: newUserUuid,
properties: {},
created_at: timestamp2,
version: 1,
is_identified: true,
})
)
// verify Postgres persons
const persons = sortPersons(await fetchPostgresPersonsH())
expect(persons.length).toEqual(2)
expect(persons[0]).toEqual(
expect.objectContaining({
id: expect.any(String),
uuid: oldUserUuid,
properties: {},
created_at: timestamp,
version: 0,
is_identified: true,
})
)
expect(persons[1]).toEqual(person)
// verify Postgres distinct_ids
const distinctIds = await fetchDistinctIdValues(hub.postgres, persons[0])
expect(distinctIds).toEqual(expect.arrayContaining([oldUserDistinctId]))
const distinctIds2 = await fetchDistinctIdValues(hub.postgres, persons[1])
expect(distinctIds2).toEqual(expect.arrayContaining([newUserDistinctId]))
})
it(`does not merge people when both users are identified`, async () => {
await createPerson(hub, timestamp, {}, {}, {}, teamId, null, true, oldUserUuid, {
distinctId: oldUserDistinctId,
})
await createPerson(hub, timestamp2, {}, {}, {}, teamId, null, true, newUserUuid, {
distinctId: newUserDistinctId,
})
const result = await personMergeService({
event: '$identify',
distinct_id: newUserDistinctId,
properties: {
$anon_distinct_id: oldUserDistinctId,
},
}).handleIdentifyOrAlias()
expect(result.success).toBe(true)
if (!result.success) {
throw new Error('Expected successful merge result')
}
const person = result.person
const kafkaAcks = result.kafkaAck
await kafkaProducer.flush()
await kafkaAcks
expect(person).toEqual(
expect.objectContaining({
id: expect.any(String),
uuid: newUserUuid,
properties: {},
created_at: timestamp2,
version: 0,
is_identified: true,
})
)
// verify Postgres persons
const persons = sortPersons(await fetchPostgresPersonsH())
expect(persons.length).toEqual(2)
expect(persons[0]).toMatchObject({
id: expect.any(String),
uuid: oldUserUuid,
properties: {},
created_at: timestamp,
version: 0,
is_identified: true,
})
expect(persons[1]).toMatchObject({
id: person!.id,
uuid: newUserUuid,
created_at: timestamp2,
is_identified: true,
properties: {},
version: 0,
})
// verify Postgres distinct_ids
const distinctIds = await fetchDistinctIdValues(hub.postgres, persons[0])
expect(distinctIds).toEqual(expect.arrayContaining([oldUserDistinctId]))
const distinctIds2 = await fetchDistinctIdValues(hub.postgres, persons[1])
expect(distinctIds2).toEqual(expect.arrayContaining([newUserDistinctId]))
})
it(`merge into distinct_id person and updates properties with $set/$set_once`, async () => {
await createPerson(hub, timestamp, { a: 1, b: 2 }, {}, {}, teamId, null, false, oldUserUuid, {
distinctId: oldUserDistinctId,
})
await createPerson(hub, timestamp2, { b: 3, c: 4, d: 5 }, {}, {}, teamId, null, false, newUserUuid, {
distinctId: newUserDistinctId,
})
const mergeService = personMergeService({
event: '$identify',
distinct_id: newUserDistinctId,
properties: {
$set: { d: 6, e: 7 },
$set_once: { a: 8, f: 9 },
$anon_distinct_id: oldUserDistinctId,
},
})
const result = await mergeService.handleIdentifyOrAlias()
expect(result.success).toBe(true)
if (!result.success) {
throw new Error('Expected successful merge result')
}
const person = result.person
const kafkaAcks = result.kafkaAck
const context = mergeService.getContext()
await flushPersonStoreToKafka(kafkaProducer, context.personStore, kafkaAcks)
expect(person).toMatchObject({
id: expect.any(String),
uuid: expect.any(String),
properties: { a: 1, b: 3, c: 4, d: 6, e: 7, f: 9 },
created_at: timestamp,
version: 1,
is_identified: true,
})
// verify Postgres persons
const persons = sortPersons(await fetchPostgresPersonsH())
expect(persons.length).toEqual(1)
expect(persons[0]).toMatchObject({
id: person!.id,
uuid: newUserUuid,
properties: { a: 1, b: 3, c: 4, d: 6, e: 7, f: 9 },
created_at: timestamp,
version: 1,
is_identified: true,
})
expect([newUserUuid, oldUserUuid]).toContain(persons[0].uuid)
// verify Postgres distinct_ids
const distinctIds = await fetchDistinctIdValues(hub.postgres, persons[0])
expect(distinctIds).toEqual(expect.arrayContaining([oldUserDistinctId, newUserDistinctId]))
// verify ClickHouse persons
await clickhouse.delayUntilEventIngested(() => fetchPersonsRowsWithVersionHigerEqualThan(), 2) // wait until merge and delete processed
const clickhousePersons = await fetchPersonsRows() // but verify full state
expect(clickhousePersons.length).toEqual(2)
expect(clickhousePersons).toEqual(
expect.arrayContaining([
expect.objectContaining({
id: expect.any(String),
properties: JSON.stringify({ a: 1, b: 3, c: 4, d: 6, e: 7, f: 9 }),
created_at: timestampch,
version: 1,
is_identified: 1,
}),
expect.objectContaining({
id: expect.any(String),
is_deleted: 1,
version: 1,
}),
])
)
expect(new Set(clickhousePersons.map((p) => p.id))).toEqual(new Set([newUserUuid, oldUserUuid]))
// verify ClickHouse distinct_ids
await clickhouse.delayUntilEventIngested(() => fetchDistinctIdsClickhouseVersion1())
const clickHouseDistinctIds = await fetchDistinctIdsClickhouse(persons[0])
expect(clickHouseDistinctIds).toEqual(expect.arrayContaining([oldUserDistinctId, newUserDistinctId]))
})
it(`merge writes only the keys it changes on the survivor, including the event's $unset`, async () => {
await createPerson(hub, timestamp, { a: 1, b: 2 }, {}, {}, teamId, null, false, oldUserUuid, {
distinctId: oldUserDistinctId,
})
// `nested` is object-valued and unchanged, so it must not travel as a set.
await createPerson(
hub,
timestamp2,
{ b: 3, c: 4, d: 5, constructor: 'x', nested: { k: [1, 2] } },
{},
{},
teamId,
null,
false,
newUserUuid,
{ distinctId: newUserDistinctId }
)
const mergeService = personMergeService({
event: '$identify',
distinct_id: newUserDistinctId,
properties: { $set: { d: 6 }, $unset: ['c', 'constructor'], $anon_distinct_id: oldUserDistinctId },
})
const result = await mergeService.handleIdentifyOrAlias()
expect(result.success).toBe(true)
if (!result.success) {
throw new Error('Expected successful merge result')
}
await flushPersonStoreToKafka(kafkaProducer, mergeService.getContext().personStore, result.kafkaAck)
// The survivor's own key (b) stays out; the source's key (a) only fills a gap; a prototype-named key unsets like any other.
expect(personRepository.updatePersonsBatch).toHaveBeenCalledWith(
[
expect.objectContaining({
uuid: newUserUuid,
properties_to_set: { d: 6 },
properties_to_set_once: { a: 1 },
properties_to_unset: ['c', 'constructor'],
}),
],
expect.anything()
)
const persons = await fetchPostgresPersonsH()
expect(persons.length).toEqual(1)
expect(persons[0]).toMatchObject({
uuid: newUserUuid,
properties: { a: 1, b: 3, d: 6, nested: { k: [1, 2] } },
})
})
it(`a merge does not replay a set this pod already flushed over another pod's newer value`, async () => {
await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, oldUserUuid, {
distinctId: oldUserDistinctId,
})
await createPerson(hub, timestamp2, { own: 'x' }, {}, {}, teamId, null, false, newUserUuid, {
distinctId: newUserDistinctId,
})
const podA = new BatchWritingPersonsStore(personRepository, createPersonOutputs(kafkaProducer))
const source = await podA.fetchForUpdate(teamId, oldUserDistinctId, 0)
await podA.updatePersonWithPropertiesDiffForUpdate(source!, { k: 'v1' }, [], {}, oldUserDistinctId, 0)
await podA.flush()
const podB = new BatchWritingPersonsStore(personRepository, createPersonOutputs(kafkaProducer))
const sourceB = await podB.fetchForUpdate(teamId, oldUserDistinctId, 0)
await podB.updatePersonWithPropertiesDiffForUpdate(sourceB!, { k: 'v2' }, [], {}, oldUserDistinctId, 0)
await podB.flush()
const mergeService = personMergeService(
{
event: '$identify',
distinct_id: newUserDistinctId,
properties: { $anon_distinct_id: oldUserDistinctId },
},
hub,
undefined,
undefined,
undefined,
undefined,
undefined,
undefined,
podA
)
const result = await mergeService.handleIdentifyOrAlias()
expect(result.success).toBe(true)
if (!result.success) {
throw new Error('Expected successful merge result')
}
await flushPersonStoreToKafka(kafkaProducer, mergeService.getContext().personStore, result.kafkaAck)
const persons = await fetchPostgresPersonsH()
expect(persons[0]).toMatchObject({ uuid: newUserUuid, properties: { own: 'x', k: 'v2' } })
})
it(`a merge moves cohorts and flags before it locks the persons, and holds both row locks before it moves distinct ids`, async () => {
await createPerson(hub, timestamp, { a: 1 }, {}, {}, teamId, null, false, oldUserUuid, {
distinctId: oldUserDistinctId,
})
await createPerson(hub, timestamp2, { own: 'x' }, {}, {}, teamId, null, false, newUserUuid, {
distinctId: newUserDistinctId,
})
const batchStore = new BatchWritingPersonsStore(personRepository, createPersonOutputs(kafkaProducer))
const lockState = (id: string): Promise<string> => rowLockState(hub, id)
const moveDistinctIds = personRepository.moveDistinctIds.bind(personRepository)
const updateCohortsAndFeatureFlags =
personRepository.updateCohortsAndFeatureFlagsForMerge.bind(personRepository)
const seenAtCohorts: string[] = []
jest.spyOn(personRepository, 'updateCohortsAndFeatureFlagsForMerge').mockImplementationOnce(
async (team, sourceId, targetId, tx) => {
seenAtCohorts.push(await lockState(sourceId), await lockState(targetId))
return updateCohortsAndFeatureFlags(team, sourceId, targetId, tx)
}
)
const seen: string[] = []
jest.spyOn(personRepository, 'moveDistinctIds').mockImplementationOnce(
async (source, target, limit, tx) => {
seen.push(await lockState(source.id), await lockState(target.id))
return moveDistinctIds(source, target, limit, tx)
}
)
const mergeService = personMergeService(
{
event: '$identify',
distinct_id: newUserDistinctId,
properties: { $anon_distinct_id: oldUserDistinctId },
},
hub,
undefined,
undefined,
undefined,
undefined,
undefined,
undefined,
batchStore
)
const result = await mergeService.handleIdentifyOrAlias()
expect(result.success).toBe(true)
expect(seenAtCohorts).toEqual(['free', 'free'])
expect(seen).toEqual(['locked', 'locked'])
})
it(`a folded merge moves cohorts and flags before it locks the persons`, async () => {
const secondSourceDistinctId = 'fold-second-source'
await createPerson(hub, timestamp, { a: 1 }, {}, {}, teamId, null, false, oldUserUuid, {
distinctId: oldUserDistinctId,
})
await createPerson(
hub,
timestamp,
{ b: 2 },
{},
{},
teamId,
null,
false,
uuidFromDistinctId(teamId, secondSourceDistinctId),
{ distinctId: secondSourceDistinctId }
)
await createPerson(hub, timestamp2, { own: 'x' }, {}, {}, teamId, null, false, newUserUuid, {
distinctId: newUserDistinctId,
})
const batchStore = new BatchWritingPersonsStore(personRepository, createPersonOutputs(kafkaProducer))
const updateBatch = personRepository.updateCohortsAndFeatureFlagsForMergeBatch.bind(personRepository)
const seenAtCohorts: string[] = []
jest.spyOn(personRepository, 'updateCohortsAndFeatureFlagsForMergeBatch').mockImplementationOnce(
async (team, sourceIds, targetId, tx) => {
for (const id of [...sourceIds, targetId]) {
seenAtCohorts.push(await rowLockState(hub, id))
}
return updateBatch(team, sourceIds, targetId, tx)
}
)
const eventUuid = new UUIDT().toString()
const result = await batchStore.mergePersons(
{
teamId,
targetDistinctId: newUserDistinctId,
sources: [
{ distinctId: oldUserDistinctId, eventUuid },
{ distinctId: secondSourceDistinctId, eventUuid: new UUIDT().toString() },
],
eventOps: {
set: {},
setOnce: {},
unset: [],
denied: false,
shouldForceUpdate: true,
eventName: '$identify',
},
eventUuid,
allowIdentifiedSources: true,
mergeMode: createDefaultSyncMergeMode(),
createdAtMs: timestamp.toMillis(),
},
0
)
expect(result.foldAborted).toBeUndefined()
expect(seenAtCohorts).toEqual(['free', 'free', 'free'])
})
it.each([
["survives the merge's own write", {}],
[
"lands over the merge event's $set of the value the merge read, which does not replay",
{ $set: { k: 'v1' } },
],
])(`a source's update flushed from another pod after a merge %s`, async (_case, eventProperties) => {
await createPerson(hub, timestamp, { k: 'v1' }, {}, {}, teamId, null, false, oldUserUuid, {
distinctId: oldUserDistinctId,
})
await createPerson(hub, timestamp2, { own: 'x' }, {}, {}, teamId, null, false, newUserUuid, {
distinctId: newUserDistinctId,
})
// Pod B holds the source's newer value, unflushed.
const podB = new BatchWritingPersonsStore(personRepository, createPersonOutputs(kafkaProducer))
const source = await podB.fetchForUpdate(teamId, oldUserDistinctId, 0)
await podB.updatePersonWithPropertiesDiffForUpdate(source!, { k: 'v2' }, [], {}, oldUserDistinctId, 0)
const podA = new BatchWritingPersonsStore(personRepository, createPersonOutputs(kafkaProducer))
const mergeService = personMergeService(
{
event: '$identify',
distinct_id: newUserDistinctId,
properties: { ...eventProperties, $anon_distinct_id: oldUserDistinctId },
},
hub,
undefined,
undefined,
undefined,
undefined,
undefined,
undefined,
podA
)
const result = await mergeService.handleIdentifyOrAlias()
expect(result.success).toBe(true)
if (!result.success) {
throw new Error('Expected successful merge result')
}
// Pod B's flush finds the source gone and re-targets; pod A's flush comes after.
await podB.flush()
await flushPersonStoreToKafka(kafkaProducer, mergeService.getContext().personStore, result.kafkaAck)
const persons = await fetchPostgresPersonsH()
expect(persons.length).toEqual(1)
expect(persons[0]).toMatchObject({ uuid: newUserUuid, properties: { own: 'x', k: 'v2' } })
// Pod A's merged-in key landed, so it no longer waits to be written.
expect(podA.getCachedPersonForUpdateByPersonId(teamId, persons[0].id)?.properties_to_set_once).toEqual({})
})
it(`a merge that rolls back after its survivor write leaves the survivor's row untouched`, async () => {
await createPerson(hub, timestamp, { a: 1 }, {}, {}, teamId, null, false, oldUserUuid, {
distinctId: oldUserDistinctId,
})
await createPerson(hub, timestamp2, { own: 'x' }, {}, {}, teamId, null, false, newUserUuid, {
distinctId: newUserDistinctId,
})
const batchStore = new BatchWritingPersonsStore(personRepository, createPersonOutputs(kafkaProducer))
// The source's delete, after the survivor's write, fails the transaction on every attempt.
jest.spyOn(personRepository, 'deletePerson').mockRejectedValue(new Error('connection reset'))
const mergeService = personMergeService(
{
event: '$identify',
distinct_id: newUserDistinctId,
properties: { $set: { k: 'a' }, $anon_distinct_id: oldUserDistinctId },
},
hub,
undefined,
undefined,
undefined,
undefined,
undefined,
undefined,
batchStore
)
await mergeService.handleIdentifyOrAlias().catch(() => undefined)
const persons = await fetchPostgresPersonsH()
expect(persons.map(({ uuid, properties, is_identified }) => ({ uuid, properties, is_identified }))).toEqual(
expect.arrayContaining([
{ uuid: oldUserUuid, properties: { a: 1 }, is_identified: false },
{ uuid: newUserUuid, properties: { own: 'x' }, is_identified: false },
])
)
})
it.each([
['holds the key', { k: 'target' }, {}],
['lacks the key', {}, { k: 'pending' }],
])(
`merge lands a source's key over the survivor's unflushed unset when the row %s`,
async (_case, row, set) => {
await createPerson(hub, timestamp, { k: 'source' }, {}, {}, teamId, null, false, oldUserUuid, {
distinctId: oldUserDistinctId,
})
await createPerson(hub, timestamp2, row, {}, {}, teamId, null, false, newUserUuid, {
distinctId: newUserDistinctId,
})
const batchStore = new BatchWritingPersonsStore(personRepository, createPersonOutputs(kafkaProducer))
const target = await batchStore.fetchForUpdate(teamId, newUserDistinctId, 0)
await batchStore.updatePersonWithPropertiesDiffForUpdate(target!, set, [], {}, newUserDistinctId, 0)
await batchStore.updatePersonWithPropertiesDiffForUpdate(target!, {}, ['k'], {}, newUserDistinctId, 0)
const mergeService = personMergeService(
{
event: '$identify',
distinct_id: newUserDistinctId,
properties: { $anon_distinct_id: oldUserDistinctId },
},
hub,
undefined,
undefined,
undefined,
undefined,
undefined,
undefined,
batchStore
)
const result = await mergeService.handleIdentifyOrAlias()
expect(result.success).toBe(true)
if (!result.success) {
throw new Error('Expected successful merge result')
}
await flushPersonStoreToKafka(kafkaProducer, mergeService.getContext().personStore, result.kafkaAck)
const persons = await fetchPostgresPersonsH()
expect(persons[0]).toMatchObject({ uuid: newUserUuid, properties: { k: 'source' } })
}
)
it(`merge carries a property this batch set on the source but has not flushed yet`, async () => {
await createPerson(hub, timestamp, { a: 1 }, {}, {}, teamId, null, false, oldUserUuid, {
distinctId: oldUserDistinctId,
})
await createPerson(hub, timestamp2, { b: 2 }, {}, {}, teamId, null, false, newUserUuid, {
distinctId: newUserDistinctId,
})
const batchStore = new BatchWritingPersonsStore(personRepository, createPersonOutputs(kafkaProducer))
// A $set on the source, buffered in the store; the row does not have it yet.
const source = await batchStore.fetchForUpdate(teamId, oldUserDistinctId, 0)
await batchStore.updatePersonWithPropertiesDiffForUpdate(
source!,
{ pending: 'yes' },
[],
{},
oldUserDistinctId,
0
)
const mergeService = personMergeService(
{
event: '$identify',
distinct_id: newUserDistinctId,
properties: { $anon_distinct_id: oldUserDistinctId },
},
hub,
undefined,
undefined,
undefined,
undefined,
undefined,
undefined,
batchStore
)
const result = await mergeService.handleIdentifyOrAlias()
expect(result.success).toBe(true)
if (!result.success) {
throw new Error('Expected successful merge result')
}
// The committed merge cleared the source's entry before any flush could.
expect(batchStore.getCachedPersonForUpdateByPersonId(teamId, source!.id)).toBeUndefined()
await flushPersonStoreToKafka(kafkaProducer, mergeService.getContext().personStore, result.kafkaAck)
const persons = await fetchPostgresPersonsH()
expect(persons.length).toEqual(1)
expect(persons[0]).toMatchObject({ uuid: newUserUuid, properties: { a: 1, b: 2, pending: 'yes' } })
})
it(`a merge that rolls back after its move leaves the source's pending set for the survivor`, async () => {
await createPerson(hub, timestamp, { a: 1 }, {}, {}, teamId, null, false, oldUserUuid, {
distinctId: oldUserDistinctId,
})
await createPerson(hub, timestamp2, { b: 2 }, {}, {}, teamId, null, false, newUserUuid, {
distinctId: newUserDistinctId,
})
const batchStore = new BatchWritingPersonsStore(personRepository, createPersonOutputs(kafkaProducer))
const source = await batchStore.fetchForUpdate(teamId, oldUserDistinctId, 0)
await batchStore.updatePersonWithPropertiesDiffForUpdate(
source!,
{ pending: 'yes' },
[],
{},
oldUserDistinctId,
0
)
// The first attempt fails after its move; the retry merges afresh.
jest.spyOn(personRepository, 'updateCohortsAndFeatureFlagsForMerge').mockImplementationOnce(() =>
Promise.reject(Object.assign(new Error('deadlock detected'), { code: '40P01' }))
)
const mergeService = personMergeService(
{
event: '$identify',
distinct_id: newUserDistinctId,
properties: { $anon_distinct_id: oldUserDistinctId },
},
hub,
undefined,
undefined,
undefined,
undefined,
undefined,
undefined,
batchStore
)
const result = await mergeService.handleIdentifyOrAlias()
expect(result.success).toBe(true)
if (!result.success) {
throw new Error('Expected successful merge result')
}
await flushPersonStoreToKafka(kafkaProducer, mergeService.getContext().personStore, result.kafkaAck)
const persons = await fetchPostgresPersonsH()
expect(persons.length).toEqual(1)
expect(persons[0]).toMatchObject({ uuid: newUserUuid, properties: { a: 1, b: 2, pending: 'yes' } })
})
it(`a source merged away under a merge still gets its pending set onto the survivor`, async () => {
await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, oldUserUuid, {
distinctId: oldUserDistinctId,
})
await createPerson(hub, timestamp2, { b: 2 }, {}, {}, teamId, null, false, newUserUuid, {
distinctId: newUserDistinctId,
})
const elsewhere = await createPerson(
hub,
timestamp2,
{ c: 3 },
{},
{},
teamId,
null,
false,
new UUIDT().toString(),
{
distinctId: 'elsewhere',
}
)
const batchStore = new BatchWritingPersonsStore(personRepository, createPersonOutputs(kafkaProducer))
const source = await batchStore.fetchForUpdate(teamId, oldUserDistinctId, 0)
await batchStore.updatePersonWithPropertiesDiffForUpdate(
source!,
{ pending: 'yes' },
[],
{},
oldUserDistinctId,
0
)
// Another pod merges the source elsewhere between this merge's fetch and its row locks; the locks find
// it gone, and the retry merges the person its distinct id names now.
jest.spyOn(personRepository, 'readMergeRows').mockImplementationOnce(async (...args) => {
await personRepository.inRawTransaction('elsewhere', async (tx) => {
await personRepository.moveDistinctIds(source!, elsewhere, undefined, tx)
await personRepository.deletePerson(source!, tx)
})
return await PostgresPersonRepository.prototype.readMergeRows.apply(personRepository, args)
})
const mergeService = personMergeService(
{
event: '$identify',
distinct_id: newUserDistinctId,
properties: { $anon_distinct_id: oldUserDistinctId },
},
hub,
undefined,
undefined,
undefined,
undefined,
undefined,
undefined,
batchStore
)
const result = await mergeService.handleIdentifyOrAlias()
expect(result.success).toBe(true)
if (!result.success) {
throw new Error('Expected successful merge result')
}
await flushPersonStoreToKafka(kafkaProducer, mergeService.getContext().personStore, result.kafkaAck)
const persons = await fetchPostgresPersonsH()
expect(persons.length).toEqual(1)
expect(persons[0]).toMatchObject({ uuid: newUserUuid, properties: { b: 2, c: 3, pending: 'yes' } })
})
it(`a merge chain in one batch carries the oldest birth and properties through the middle person`, async () => {
const oldest = DateTime.fromISO('2019-01-01T00:00:00.000Z').toUTC()
await createPerson(hub, oldest, { first: 1 }, {}, {}, teamId, null, false, new UUIDT().toString(), {
distinctId: 'oldest-anon',
})
await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, oldUserUuid, {
distinctId: oldUserDistinctId,
})
await createPerson(hub, timestamp2, {}, {}, {}, teamId, null, false, newUserUuid, {
distinctId: newUserDistinctId,
})
const batchStore = new BatchWritingPersonsStore(personRepository, createPersonOutputs(kafkaProducer))
const merge = (event: Partial<PluginEvent>) =>
personMergeService(
event,
hub,
undefined,
undefined,
undefined,
undefined,
undefined,
undefined,
batchStore
)
// oldest-anon into old-user, buffered; then old-user into new-user before that flushes.
const first = await merge({
event: '$identify',
distinct_id: oldUserDistinctId,
properties: { $anon_distinct_id: 'oldest-anon' },
}).handleIdentifyOrAlias()
expect(first.success).toBe(true)
const secondService = merge({
event: '$merge_dangerously',
distinct_id: newUserDistinctId,
properties: { alias: oldUserDistinctId },
})
const second = await secondService.handleIdentifyOrAlias()
expect(second.success).toBe(true)
if (!second.success) {
throw new Error('Expected successful merge result')
}
await flushPersonStoreToKafka(kafkaProducer, secondService.getContext().personStore, second.kafkaAck)
const persons = await fetchPostgresPersonsH()
expect(persons.length).toEqual(1)
expect(persons[0]).toMatchObject({ uuid: newUserUuid, created_at: oldest, properties: { first: 1 } })
})
it.each([
['before its transaction', 'inTransaction'],
['right before its row locks', 'readMergeRows'],
])(`merge carries a property another writer lands on the source %s`, async (_when, boundary) => {
await createPerson(hub, timestamp, { a: 1 }, {}, {}, teamId, null, false, oldUserUuid, {
distinctId: oldUserDistinctId,
})
const target = await createPerson(hub, timestamp2, { b: 2 }, {}, {}, teamId, null, false, newUserUuid, {
distinctId: newUserDistinctId,
})
// Another pod's flush lands on the source after the merge's reads.
const lateWrite = () =>
hub.postgres.query(
PostgresUse.PERSONS_WRITE,
`UPDATE posthog_person SET properties = properties || '{"late": "yes"}'::jsonb, version = version + 1
WHERE team_id = $1 AND uuid = $2`,
[teamId, oldUserUuid],
'lateSourceWrite'
)
if (boundary === 'inTransaction') {
jest.spyOn(personRepository, 'inTransaction').mockImplementationOnce(async (description, body) => {
await lateWrite()
return await PostgresPersonRepository.prototype.inTransaction.call(
personRepository,
description,
body
)
})
} else {
jest.spyOn(personRepository, 'readMergeRows').mockImplementationOnce(async (...args) => {
await lateWrite()
return await PostgresPersonRepository.prototype.readMergeRows.apply(personRepository, args)
})
}
const mergeService = personMergeService({
event: '$identify',
distinct_id: newUserDistinctId,
properties: { $anon_distinct_id: oldUserDistinctId },
})
const result = await mergeService.handleIdentifyOrAlias()
expect(result.success).toBe(true)
if (!result.success) {
throw new Error('Expected successful merge result')
}
await flushPersonStoreToKafka(kafkaProducer, mergeService.getContext().personStore, result.kafkaAck)
const persons = await fetchPostgresPersonsH()
expect(persons.length).toEqual(1)
expect(persons[0]).toMatchObject({ id: target.id, properties: { a: 1, b: 2, late: 'yes' } })
})
it.each([
{ pool: 'the shared persons pool', poolSize: undefined },
// The merge's transaction holds the only connection, so a conflict re-read on the pool never starts.
{ pool: 'a one-connection persons pool', poolSize: 1 },
])(
`joins both ids when another writer creates the anon person during a neither-exists merge on $pool`,
async ({ poolSize }) => {
const mergePostgres = poolSize
? new PostgresRouter({ ...defaultConfig, POSTGRES_CONNECTION_POOL_SIZE: poolSize })
: hub.postgres
const mergeRepository = poolSize ? new PostgresPersonRepository(mergePostgres) : personRepository
try {
// The anon person lands on another pod before this merge's two-id create.
let raced = false
jest.spyOn(mergeRepository, 'createPerson').mockImplementation(async (...args) => {
if (!raced) {
raced = true
await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, newUserUuid, {
distinctId: newUserDistinctId,
})
}
return await PostgresPersonRepository.prototype.createPerson.apply(mergeRepository, args)
})
const mergeService = personMergeService(
{
event: '$identify',
distinct_id: oldUserDistinctId,
properties: { $anon_distinct_id: newUserDistinctId },
},
undefined,
mergeRepository
)
const result = await mergeService.handleIdentifyOrAlias()
expect(result.success).toBe(true)
if (!result.success) {
throw new Error('Expected successful merge result')
}
await flushPersonStoreToKafka(kafkaProducer, mergeService.getContext().personStore, result.kafkaAck)
const persons = sortPersons(await fetchPostgresPersonsH())
expect(persons.length).toEqual(1)
expect(persons[0].uuid).toEqual(newUserUuid)
const distinctIds = await fetchDistinctIdValues(hub.postgres, persons[0])
expect(distinctIds).toEqual(expect.arrayContaining([oldUserDistinctId, newUserDistinctId]))
} finally {
if (poolSize) {
await mergePostgres.end()
}
}
}
)
it(`does not retry a neither-exists merge when the uuid's holder owns neither id`, async () => {
// A live person holds the target's uuid under a third id, so no retry can attach.
await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, newUserUuid, {
distinctId: 'stranded-holder',
})
const created = jest.spyOn(personRepository, 'createPerson')
const mergeService = personMergeService({
event: '$identify',
distinct_id: newUserDistinctId,
properties: { $anon_distinct_id: oldUserDistinctId },
})
const result = await mergeService.handleIdentifyOrAlias()
expect(result.success).toBe(true)
expect(created).toHaveBeenCalledTimes(1)
})
it(`handles race condition when other thread creates the user`, async () => {
await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, oldUserUuid, {
distinctId: oldUserDistinctId,
})
// Fake the race by assuming createPerson was called before the addDistinctId creation above
jest.spyOn(personRepository, 'addDistinctId').mockImplementation(
async (person, distinctId): Promise<PersonMessage[]> => {
await createPerson(
hub,
timestamp,
{},
{},
{},
teamId,
null,
false,
uuidFromDistinctId(teamId, distinctId),
{ distinctId }
)
return await personRepository.addDistinctId(person, distinctId, 0) // this throws
}
)
const mergeService = personMergeService({
event: '$identify',
distinct_id: oldUserDistinctId,
properties: {
$anon_distinct_id: newUserDistinctId,
},
})
const result = await mergeService.handleIdentifyOrAlias()
expect(result.success).toBe(true)
if (!result.success) {
throw new Error('Expected successful merge result')
}
const person = result.person
const kafkaAcks = result.kafkaAck
const context = mergeService.getContext()
await flushPersonStoreToKafka(kafkaProducer, context.personStore, kafkaAcks)
jest.spyOn(personRepository, 'addDistinctId').mockRestore()
// if creation fails we should return the person that another thread already created
expect(person).toMatchObject({
id: expect.any(String),
uuid: oldUserUuid,
properties: {},
created_at: timestamp,
version: 1,
is_identified: true,
})
// expect(personRepository.updatePerson).not.toHaveBeenCalled()
// verify Postgres persons
const persons = sortPersons(await fetchPostgresPersonsH())
expect(persons.length).toEqual(1)
expect(persons[0]).toMatchObject({
id: expect.any(String),
uuid: oldUserUuid,
properties: {},
created_at: timestamp,
version: 1,
is_identified: true,
})
// verify Postgres distinct_ids
const distinctIds = await fetchDistinctIdValues(hub.postgres, persons[0])
expect(distinctIds).toEqual(expect.arrayContaining([newUserDistinctId]))
})
})
describe('on $create_alias events', () => {
// All the functionality tests are provided above in $identify tests, here we just make sure the calls are equivalent
it('calls merge on $identify as expected', async () => {
const state: PersonMergeService = personMergeService(
{
event: '$identify',
distinct_id: newUserDistinctId,
properties: { $anon_distinct_id: oldUserDistinctId },
},
hub
)
jest.spyOn(state, 'merge').mockImplementation(() => {
return Promise.resolve({
success: true,
person: undefined,
kafkaAck: Promise.resolve(),
needsPersonUpdate: true,
})
})
await state.handleIdentifyOrAlias()
expect(state.merge).toHaveBeenCalledWith(oldUserDistinctId, newUserDistinctId, teamId, timestamp)
jest.spyOn(state, 'merge').mockRestore()
})
it('calls merge on $create_alias as expected', async () => {
const state: PersonMergeService = personMergeService(
{
event: '$create_alias',
distinct_id: newUserDistinctId,
properties: { alias: oldUserDistinctId },
},
hub
)
jest.spyOn(state, 'merge').mockImplementation(() => {
return Promise.resolve({
success: true,
person: undefined,
kafkaAck: Promise.resolve(),
needsPersonUpdate: true,
})
})
await state.handleIdentifyOrAlias()
expect(state.merge).toHaveBeenCalledWith(oldUserDistinctId, newUserDistinctId, teamId, timestamp)
jest.spyOn(state, 'merge').mockRestore()
})
it('calls merge on $merge_dangerously as expected', async () => {
const state: PersonMergeService = personMergeService(
{
event: '$merge_dangerously',
distinct_id: newUserDistinctId,
properties: { alias: oldUserDistinctId },
},
hub
)
jest.spyOn(state, 'merge').mockImplementation(() => {
return Promise.resolve({
success: true,
person: undefined,
kafkaAck: Promise.resolve(),
needsPersonUpdate: true,
})
})
await state.handleIdentifyOrAlias()
expect(state.merge).toHaveBeenCalledWith(oldUserDistinctId, newUserDistinctId, teamId, timestamp)
jest.spyOn(state, 'merge').mockRestore()
})
})
describe('on $merge_dangerously events', () => {
// only difference between $merge_dangerously and $identify
it(`merge_dangerously can merge people when alias id user is identified`, async () => {
await createPerson(hub, timestamp, {}, {}, {}, teamId, null, true, oldUserUuid, {
distinctId: oldUserDistinctId,
})
await createPerson(hub, timestamp2, {}, {}, {}, teamId, null, true, newUserUuid, {
distinctId: newUserDistinctId,
})
const mergeService = personMergeService({
event: '$merge_dangerously',
distinct_id: newUserDistinctId,
properties: {
alias: oldUserDistinctId,
},
})
const result = await mergeService.handleIdentifyOrAlias()
expect(result.success).toBe(true)
if (!result.success) {
throw new Error('Expected successful merge result')
}
const person = result.person
const kafkaAcks = result.kafkaAck
const context = mergeService.getContext()
await flushPersonStoreToKafka(kafkaProducer, context.personStore, kafkaAcks)
expect(person).toEqual(
expect.objectContaining({
id: expect.any(String),
uuid: expect.any(String),
properties: {},
created_at: timestamp,
version: 1,
is_identified: true,
})
)
// verify Postgres persons
const persons = sortPersons(await fetchPostgresPersonsH())
expect(persons.length).toEqual(1)
expect(persons[0]).toMatchObject({
id: person!.id,
uuid: newUserUuid,
created_at: timestamp,
is_identified: true,
properties: {},
version: 1,
})
expect([newUserUuid, oldUserUuid]).toContain(persons[0].uuid)
// verify Postgres distinct_ids
const distinctIds = await fetchDistinctIdValues(hub.postgres, persons[0])
expect(distinctIds).toEqual(expect.arrayContaining([oldUserDistinctId, newUserDistinctId]))
// verify ClickHouse persons
await clickhouse.delayUntilEventIngested(() => fetchPersonsRowsWithVersionHigerEqualThan(), 2) // wait until merge and delete processed
const clickhousePersons = await fetchPersonsRows() // but verify full state
expect(clickhousePersons.length).toEqual(2)
expect(clickhousePersons).toEqual(
expect.arrayContaining([
expect.objectContaining({
id: expect.any(String),
properties: '{}',
created_at: timestampch,
version: 1,
is_identified: 1,
}),
expect.objectContaining({
id: expect.any(String),
is_deleted: 1,
version: 1,
}),
])
)
expect(new Set(clickhousePersons.map((p) => p.id))).toEqual(new Set([newUserUuid, oldUserUuid]))
// verify ClickHouse distinct_ids
await clickhouse.delayUntilEventIngested(() => fetchDistinctIdsClickhouseVersion1())
const clickHouseDistinctIds = await fetchDistinctIdsClickhouse(persons[0])
expect(clickHouseDistinctIds).toEqual(expect.arrayContaining([oldUserDistinctId, newUserDistinctId]))
})
})
describe('illegal aliasing', () => {
const illegalIds = [
'',
' ',
'null',
'undefined',
'"undefined"',
'[object Object]',
'"[object Object]"',
'$posthog_cookieless',
]
it.each(illegalIds)('stops $identify if current distinct_id is illegal: `%s`', async (illegalId: string) => {
const mergeService = personMergeService({
event: '$identify',
distinct_id: illegalId,
properties: {
$anon_distinct_id: 'anonymous_id',
},
})
const result = await mergeService.handleIdentifyOrAlias()
expect(result.success).toBe(true)
if (!result.success) {
throw new Error('Expected successful merge result')
}
const person = result.person
const kafkaAcks = result.kafkaAck
const context = mergeService.getContext()
await flushPersonStoreToKafka(kafkaProducer, context.personStore, kafkaAcks)
expect(person).toEqual(undefined)
const persons = await fetchPostgresPersonsH()
expect(persons.length).toEqual(0)
})
it.each(illegalIds)('stops $identify if $anon_distinct_id is illegal: `%s`', async (illegalId: string) => {
const mergeService = personMergeService({
event: '$identify',
distinct_id: 'some_distinct_id',
properties: {
$anon_distinct_id: illegalId,
},
})
const result = await mergeService.handleIdentifyOrAlias()
expect(result.success).toBe(true)
if (!result.success) {
throw new Error('Expected successful merge result')
}
const person = result.person
const kafkaAcks = result.kafkaAck
const context = mergeService.getContext()
await flushPersonStoreToKafka(kafkaProducer, context.personStore, kafkaAcks)
expect(person).toEqual(undefined)
const persons = await fetchPostgresPersonsH()
expect(persons.length).toEqual(0)
})
it('stops $create_alias if current distinct_id is illegal', async () => {
const mergeService = personMergeService({
event: '$create_alias',
distinct_id: 'false',
properties: {
alias: 'some_distinct_id',
},
})
const result = await mergeService.handleIdentifyOrAlias()
expect(result.success).toBe(true)
if (!result.success) {
throw new Error('Expected successful merge result')
}
const person = result.person
const kafkaAcks = result.kafkaAck
const context = mergeService.getContext()
await flushPersonStoreToKafka(kafkaProducer, context.personStore, kafkaAcks)
expect(person).toEqual(undefined)
const persons = await fetchPostgresPersonsH()
expect(persons.length).toEqual(0)
})
it('stops $create_alias if alias is illegal', async () => {
const mergeService = personMergeService({
event: '$create_alias',
distinct_id: 'some_distinct_id',
properties: {
alias: 'null',
},
})
const result = await mergeService.handleIdentifyOrAlias()
expect(result.success).toBe(true)
if (!result.success) {
throw new Error('Expected successful merge result')
}
const person = result.person
const kafkaAcks = result.kafkaAck
const context = mergeService.getContext()
await flushPersonStoreToKafka(kafkaProducer, context.personStore, kafkaAcks)
expect(person).toEqual(undefined)
const persons = await fetchPostgresPersonsH()
expect(persons.length).toEqual(0)
})
})
describe('foreign key updates in other tables', () => {
it('handles feature flag hash key overrides with no conflicts', async () => {
const anonPerson = await createPerson(
hub,
timestamp.minus({ hours: 1 }),
{},
{},
{},
teamId,
null,
false,
uuidFromDistinctId(teamId, 'anonymous_id'),
{ distinctId: 'anonymous_id' }
)
const identifiedPerson = await createPerson(
hub,
timestamp,
{},
{},
{},
teamId,
null,
false,
uuidFromDistinctId(teamId, 'new_distinct_id'),
{ distinctId: 'new_distinct_id' }
)
// existing overrides
await insertRow(hub.postgres, 'posthog_featureflaghashkeyoverride', {
team_id: teamId,
person_id: anonPerson.id,
feature_flag_key: 'beta-feature',
hash_key: 'example_id',
})
await insertRow(hub.postgres, 'posthog_featureflaghashkeyoverride', {
team_id: teamId,
person_id: identifiedPerson.id,
feature_flag_key: 'multivariate-flag',
hash_key: 'example_id',
})
// this event means the person will be merged
// so hashkeyoverride should be updated to the new person id whichever way we merged
const mergeService = personMergeService({
event: '$identify',
distinct_id: 'new_distinct_id',
properties: {
$anon_distinct_id: 'anonymous_id',
distinct_id: 'new_distinct_id',
},
})
const mergeResult = await mergeService.handleIdentifyOrAlias()
expect(mergeResult.success).toBe(true)
if (!mergeResult.success) {
throw new Error('Expected successful merge result')
}
const kafkaAcks = mergeResult.kafkaAck
const context = mergeService.getContext()
await flushPersonStoreToKafka(kafkaProducer, context.personStore, kafkaAcks)
const [person] = await fetchPostgresPersonsH()
expect([identifiedPerson.id, anonPerson.id]).toContain(person.id)
expect(await fetchDistinctIdValues(hub.postgres, person)).toEqual(['anonymous_id', 'new_distinct_id'])
expect(person.is_identified).toEqual(true)
const result = await hub.postgres.query(
PostgresUse.PERSONS_WRITE,
`SELECT "feature_flag_key", "person_id", "hash_key" FROM "posthog_featureflaghashkeyoverride" WHERE "team_id" = $1`,
[teamId],
'testQueryHashKeyOverride'
)
expect(result.rows).toEqual(
expect.arrayContaining([
{
feature_flag_key: 'beta-feature',
person_id: person.id,
hash_key: 'example_id',
},
{
feature_flag_key: 'multivariate-flag',
person_id: person.id,
hash_key: 'example_id',
},
])
)
})
it('handles feature flag hash key overrides with some conflicts handled gracefully', async () => {
const anonPerson = await createPerson(
hub,
timestamp.minus({ hours: 1 }),
{},
{},
{},
teamId,
null,
false,
uuidFromDistinctId(teamId, 'anonymous_id'),
{ distinctId: 'anonymous_id' }
)
const identifiedPerson = await createPerson(
hub,
timestamp,
{},
{},
{},
teamId,
null,
false,
uuidFromDistinctId(teamId, 'new_distinct_id'),
{ distinctId: 'new_distinct_id' }
)
// existing overrides for both anonPerson and identifiedPerson
// which implies a clash when they are merged
await insertRow(hub.postgres, 'posthog_featureflaghashkeyoverride', {
team_id: teamId,
person_id: anonPerson.id,
feature_flag_key: 'beta-feature',
hash_key: 'anon_id',
})
await insertRow(hub.postgres, 'posthog_featureflaghashkeyoverride', {
team_id: teamId,
person_id: identifiedPerson.id,
feature_flag_key: 'beta-feature',
hash_key: 'identified_id',
})
await insertRow(hub.postgres, 'posthog_featureflaghashkeyoverride', {
team_id: teamId,
person_id: anonPerson.id,
feature_flag_key: 'multivariate-flag',
hash_key: 'other_different_id',
})
// this event means the person will be merged
// so hashkeyoverride should be updated to be either
// we're optimizing on updates to not write on conflict and ordering is not guaranteed
const mergeService = personMergeService({
event: '$identify',
distinct_id: 'new_distinct_id',
properties: {
$anon_distinct_id: 'anonymous_id',
distinct_id: 'new_distinct_id',
},
})
const mergeResult = await mergeService.handleIdentifyOrAlias()
expect(mergeResult.success).toBe(true)
if (!mergeResult.success) {
throw new Error('Expected successful merge result')
}
const kafkaAcks = mergeResult.kafkaAck
const context = mergeService.getContext()
await flushPersonStoreToKafka(kafkaProducer, context.personStore, kafkaAcks)
const [person] = await fetchPostgresPersonsH()
expect([identifiedPerson.id, anonPerson.id]).toContain(person.id)
expect(await fetchDistinctIdValues(hub.postgres, person)).toEqual(['anonymous_id', 'new_distinct_id'])
expect(person.is_identified).toEqual(true)
const result = await hub.postgres.query(
PostgresUse.PERSONS_WRITE,
`SELECT "feature_flag_key", "person_id", "hash_key" FROM "posthog_featureflaghashkeyoverride" WHERE "team_id" = $1`,
[teamId],
'testQueryHashKeyOverride'
)
expect(result.rows).toEqual(
expect.arrayContaining([
expect.objectContaining({
feature_flag_key: 'beta-feature',
person_id: person.id,
hash_key: expect.any(String), // either anon_id or identified_id
}),
{
feature_flag_key: 'multivariate-flag',
person_id: person.id,
hash_key: 'other_different_id',
},
])
)
})
it('handles feature flag hash key overrides with no old overrides but existing new person overrides', async () => {
const anonPerson = await createPerson(
hub,
timestamp.minus({ hours: 1 }),
{},
{},
{},
teamId,
null,
false,
uuidFromDistinctId(teamId, 'anonymous_id'),
{ distinctId: 'anonymous_id' }
)
const identifiedPerson = await createPerson(
hub,
timestamp,
{},
{},
{},
teamId,
null,
false,
uuidFromDistinctId(teamId, 'new_distinct_id'),
{ distinctId: 'new_distinct_id' }
)
await insertRow(hub.postgres, 'posthog_featureflaghashkeyoverride', {
team_id: teamId,
person_id: identifiedPerson.id,
feature_flag_key: 'beta-feature',
hash_key: 'example_id',
})
await insertRow(hub.postgres, 'posthog_featureflaghashkeyoverride', {
team_id: teamId,
person_id: identifiedPerson.id,
feature_flag_key: 'multivariate-flag',
hash_key: 'different_id',
})
const mergeService = personMergeService({
event: '$identify',
distinct_id: 'new_distinct_id',
properties: {
$anon_distinct_id: 'anonymous_id',
},
})
const mergeResult = await mergeService.handleIdentifyOrAlias()
expect(mergeResult.success).toBe(true)
if (!mergeResult.success) {
throw new Error('Expected successful merge result')
}
const kafkaAcks = mergeResult.kafkaAck
const context = mergeService.getContext()
await flushPersonStoreToKafka(kafkaProducer, context.personStore, kafkaAcks)
const [person] = await fetchPostgresPersonsH()
expect([identifiedPerson.id, anonPerson.id]).toContain(person.id)
expect(await fetchDistinctIdValues(hub.postgres, person)).toEqual(['anonymous_id', 'new_distinct_id'])
expect(person.is_identified).toEqual(true)
const result = await hub.postgres.query(
PostgresUse.PERSONS_WRITE,
`SELECT "feature_flag_key", "person_id", "hash_key" FROM "posthog_featureflaghashkeyoverride" WHERE "team_id" = $1`,
[teamId],
'testQueryHashKeyOverride'
)
expect(result.rows).toEqual(
expect.arrayContaining([
{
feature_flag_key: 'beta-feature',
person_id: person.id,
hash_key: 'example_id',
},
{
feature_flag_key: 'multivariate-flag',
person_id: person.id,
hash_key: 'different_id',
},
])
)
})
})
describe('on persons merges', () => {
// For some reason these tests failed if I ran them with a hub shared
// with other tests, so I'm creating a new hub for each test.
let hub: Hub
beforeEach(async () => {
hub = await createHub({})
personRepository = new PostgresPersonRepository(hub.postgres)
jest.spyOn(personRepository, 'fetchPerson')
jest.spyOn(personRepository, 'updatePerson')
jest.spyOn(personRepository, 'updatePersonsBatch')
})
afterEach(async () => {
jest.clearAllTimers()
jest.useRealTimers()
jest.restoreAllMocks()
await closeHub(hub)
})
it(`no-op if persons already merged`, async () => {
await createPerson(
hub,
timestamp,
{},
{},
{},
teamId,
null,
true,
firstUserUuid,
{ distinctId: firstUserDistinctId },
[{ distinctId: secondUserDistinctId }]
)
const state: PersonMergeService = personMergeService({}, hub)
jest.spyOn(kafkaProducer, 'produce')
const result = await state.merge(secondUserDistinctId, firstUserDistinctId, teamId, timestamp)
expect(result.success).toBe(true)
if (!result.success) {
throw new Error('Merge should have succeeded')
}
const person = result.person
const kafkaAcks = result.kafkaAck
await kafkaProducer.flush()
await kafkaAcks
expect(person).toMatchObject({
id: expect.any(String),
uuid: firstUserUuid,
properties: {},
created_at: timestamp,
version: 0,
is_identified: true,
})
expect(personRepository.updatePerson).not.toHaveBeenCalled()
expect(kafkaProducer.produce).not.toHaveBeenCalled()
})
it(`postgres and clickhouse get updated`, async () => {
const first = await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, firstUserUuid, {
distinctId: firstUserDistinctId,
})
await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, secondUserUuid, {
distinctId: secondUserDistinctId,
})
const mergeService: PersonMergeService = personMergeService({}, hub, personRepository)
jest.spyOn(kafkaProducer, 'produce')
const result = await mergeService.merge(secondUserDistinctId, firstUserDistinctId, teamId, timestamp)
expect(result.success).toBe(true)
if (!result.success) {
throw new Error('Merge should have succeeded')
}
const person = result.person
const kafkaAcks = result.kafkaAck
const context = mergeService.getContext()
await flushPersonStoreToKafka(kafkaProducer, context.personStore, kafkaAcks)
expect(person).toMatchObject({
id: first.id,
uuid: firstUserUuid,
properties: {},
created_at: timestamp,
version: 1,
is_identified: true,
})
// Batch mode uses updatePersonsBatch instead of updatePerson
expect(personRepository.updatePersonsBatch).toHaveBeenCalledTimes(1)
// 3 produce calls: move distinct ID + delete source person (from merge) + update target person (from flush)
expect(kafkaProducer.produce).toHaveBeenCalledTimes(3)
// verify Postgres persons
const persons = sortPersons(await fetchPostgresPersonsH())
expect(persons.length).toEqual(1)
expect(persons[0]).toMatchObject({
id: first.id,
uuid: firstUserUuid,
created_at: timestamp,
is_identified: true,
properties: {},
version: 1,
})
// verify Postgres distinct_ids
expect(person).toBeDefined()
const distinctIds = await fetchDistinctIdValues(hub.postgres, person!)
expect(distinctIds).toEqual(expect.arrayContaining([firstUserDistinctId, secondUserDistinctId]))
// verify ClickHouse persons
await clickhouse.delayUntilEventIngested(() => fetchPersonsRowsWithVersionHigerEqualThan(), 2) // wait until merge and delete processed
const clickhousePersons = await fetchPersonsRows() // but verify full state
expect(clickhousePersons).toEqual(
expect.arrayContaining([
expect.objectContaining({
id: firstUserUuid,
properties: '{}',
created_at: timestampch,
version: 1,
is_identified: 1,
}),
expect.objectContaining({
id: secondUserUuid,
is_deleted: 1,
version: 1,
}),
])
)
// verify ClickHouse distinct_ids
await clickhouse.delayUntilEventIngested(() => fetchDistinctIdsClickhouseVersion1())
const clickHouseDistinctIds = await fetchDistinctIdsClickhouse(person!)
expect(clickHouseDistinctIds).toEqual(expect.arrayContaining([firstUserDistinctId, secondUserDistinctId]))
})
const getMergeEventMessages = () =>
mockProducerObserver.getProducedKafkaMessagesForTopic(KAFKA_PERSON_MERGE_EVENTS)
const producedMergeEventCount = async () => (await personMergeEventProducedCounter.get()).values[0]?.value ?? 0
it(`does not emit a person_merge_events message when the gate is off (default)`, async () => {
mockProducerObserver.resetKafkaProducer()
await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, firstUserUuid, {
distinctId: firstUserDistinctId,
})
await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, secondUserUuid, {
distinctId: secondUserDistinctId,
})
// No mergeEventsConfig → gate defaults off.
const mergeService = personMergeService({}, hub, personRepository)
const result = await mergeService.merge(secondUserDistinctId, firstUserDistinctId, teamId, timestamp)
expect(result.success).toBe(true)
if (!result.success) {
throw new Error('Merge should have succeeded')
}
await flushPersonStoreToKafka(kafkaProducer, mergeService.getContext().personStore, result.kafkaAck)
expect(getMergeEventMessages()).toHaveLength(0)
})
it(`emits exactly one person_merge_events message with correct key, partition, and payload when the gate is on`, async () => {
await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, firstUserUuid, {
distinctId: firstUserDistinctId,
})
await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, secondUserUuid, {
distinctId: secondUserDistinctId,
})
// The local test broker has no 64-partition person_merge_events topic, so resolve that
// produce in-process while routing every other message to the real broker via the
// prototype method (the instance method would recurse through this spy). We inspect the
// spy's call args directly rather than the shared observer.
const realProduce = KafkaProducerWrapper.prototype.produce
const produceSpy = jest
.spyOn(kafkaProducer, 'produce')
.mockImplementation((message) =>
message.topic === KAFKA_PERSON_MERGE_EVENTS
? Promise.resolve()
: realProduce.call(kafkaProducer, message)
)
const producedBefore = await producedMergeEventCount()
const mergeService = personMergeService(
{},
hub,
personRepository,
true,
timestamp,
mainTeam,
createDefaultSyncMergeMode(),
{ enabled: true, partitionCount: 64 }
)
const result = await mergeService.merge(secondUserDistinctId, firstUserDistinctId, teamId, timestamp)
expect(result.success).toBe(true)
if (!result.success) {
throw new Error('Merge should have succeeded')
}
// kafkaAck wraps only the person messages; the merge event is produced fire-and-forget.
await flushPersonStoreToKafka(kafkaProducer, mergeService.getContext().personStore, result.kafkaAck)
const mergeProduceCalls = produceSpy.mock.calls.filter(
(call) => call[0].topic === KAFKA_PERSON_MERGE_EVENTS
)
expect(mergeProduceCalls).toHaveLength(1)
// The counter must track actual production, so it shares the allowlist gate with the produce.
expect(await producedMergeEventCount()).toBe(producedBefore + 1)
// second (P_old) is the deleted source; first (P_new) is the target.
const expectedKey = `${teamId}:${secondUserUuid}`
const mergeMessage = mergeProduceCalls[0][0]
expect(mergeMessage.key).toBe(expectedKey)
expect(mergeMessage.partition).toBe(murmur2Partition(expectedKey, 64))
const payload = parseJSON((mergeMessage.value as Buffer).toString())
expect(payload).toMatchObject({
team_id: teamId,
old_person_uuid: secondUserUuid,
new_person_uuid: firstUserUuid,
schema_version: 1,
})
expect(typeof payload.merged_at_ms).toBe('number')
})
it(`detaches the person_merge_events produce from the ack chain: a stuck produce cannot stall kafkaAck`, async () => {
mockProducerObserver.resetKafkaProducer()
await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, firstUserUuid, {
distinctId: firstUserDistinctId,
})
await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, secondUserUuid, {
distinctId: secondUserDistinctId,
})
// Hang the shadow-topic produce; person/distinct-id messages still hit the real broker. A
// detached kafkaAck resolves regardless, so if this test ever times out, someone re-bundled
// the merge event back into the kafkaAck Promise.all and re-coupled it to offset commits.
const realProduce = KafkaProducerWrapper.prototype.produce
let releaseShadowProduce: () => void = () => {}
const shadowProduce = new Promise<void>((resolve) => {
releaseShadowProduce = () => resolve()
})
const produceSpy = jest
.spyOn(kafkaProducer, 'produce')
.mockImplementation((message) =>
message.topic === KAFKA_PERSON_MERGE_EVENTS
? shadowProduce
: realProduce.call(kafkaProducer, message)
)
try {
const mergeService = personMergeService(
{},
hub,
personRepository,
true,
timestamp,
mainTeam,
createDefaultSyncMergeMode(),
{ enabled: true, partitionCount: 64 }
)
const result = await mergeService.merge(secondUserDistinctId, firstUserDistinctId, teamId, timestamp)
expect(result.success).toBe(true)
if (!result.success) {
throw new Error('Merge should have succeeded')
}
await expect(
flushPersonStoreToKafka(kafkaProducer, mergeService.getContext().personStore, result.kafkaAck)
).resolves.toBeDefined()
// Person messages were still delivered to the real broker.
expect(produceSpy.mock.calls.filter((call) => call[0].topic === KAFKA_PERSON).length).toBeGreaterThan(0)
} finally {
// Let the fire-and-forget produce settle so it doesn't leak past the test.
releaseShadowProduce()
await shadowProduce
}
})
it(`does not emit or count a person_merge_events message for a team outside the allowlist`, async () => {
await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, firstUserUuid, {
distinctId: firstUserDistinctId,
})
await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, secondUserUuid, {
distinctId: secondUserDistinctId,
})
// Same in-process resolution as the positive case, so a regression that produced anyway
// surfaces as a failed assertion rather than a missing-topic broker error.
const realProduce = KafkaProducerWrapper.prototype.produce
const produceSpy = jest
.spyOn(kafkaProducer, 'produce')
.mockImplementation((message) =>
message.topic === KAFKA_PERSON_MERGE_EVENTS
? Promise.resolve()
: realProduce.call(kafkaProducer, message)
)
const producedBefore = await producedMergeEventCount()
// Gate is on, but the team is outside the allowlist: neither the produce nor the counter fires.
const mergeService = personMergeService(
{},
hub,
personRepository,
true,
timestamp,
mainTeam,
createDefaultSyncMergeMode(),
{ enabled: true, partitionCount: 64, teamAllowlist: '' }
)
const result = await mergeService.merge(secondUserDistinctId, firstUserDistinctId, teamId, timestamp)
expect(result.success).toBe(true)
if (!result.success) {
throw new Error('Merge should have succeeded')
}
await flushPersonStoreToKafka(kafkaProducer, mergeService.getContext().personStore, result.kafkaAck)
const mergeProduceCalls = produceSpy.mock.calls.filter(
(call) => call[0].topic === KAFKA_PERSON_MERGE_EVENTS
)
expect(mergeProduceCalls).toHaveLength(0)
expect(await producedMergeEventCount()).toBe(producedBefore)
})
it(`does not emit a person_merge_events message when the merge rolls back (moveDistinctIds throws)`, async () => {
mockProducerObserver.resetKafkaProducer()
await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, firstUserUuid, {
distinctId: firstUserDistinctId,
})
await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, secondUserUuid, {
distinctId: secondUserDistinctId,
})
const mergeService = personMergeService(
{},
hub,
personRepository,
true,
timestamp,
mainTeam,
createDefaultSyncMergeMode(),
{ enabled: true, partitionCount: 64 }
)
// Force every transaction attempt to fail before the post-commit
// emission point; merge() retries, so a single rejection would let
// the retry succeed.
jest.spyOn(BatchWritingPersonsStore.prototype, 'inTransaction').mockRejectedValue(
new Error('moveDistinctIds blew up')
)
await expect(
mergeService.merge(secondUserDistinctId, firstUserDistinctId, teamId, timestamp)
).rejects.toThrow()
expect(getMergeEventMessages()).toHaveLength(0)
})
it(`does not emit a person_merge_events message when the merge is not allowed (already identified)`, async () => {
mockProducerObserver.resetKafkaProducer()
// Both persons already identified → $identify refuses the merge (isMergeAllowed=false).
await createPerson(hub, timestamp, {}, {}, {}, teamId, null, true, oldUserUuid, {
distinctId: oldUserDistinctId,
})
await createPerson(hub, timestamp2, {}, {}, {}, teamId, null, true, newUserUuid, {
distinctId: newUserDistinctId,
})
const result = await personMergeService(
{
event: '$identify',
distinct_id: newUserDistinctId,
properties: { $anon_distinct_id: oldUserDistinctId },
},
hub,
personRepository,
true,
timestamp,
mainTeam,
createDefaultSyncMergeMode(),
{ enabled: true, partitionCount: 64 }
).handleIdentifyOrAlias()
expect(result.success).toBe(true)
if (!result.success) {
throw new Error('Refused merges still return a success result')
}
await kafkaProducer.flush()
await result.kafkaAck
// Both persons survive; no merge happened, so no merge event.
const persons = sortPersons(await fetchPostgresPersonsH())
expect(persons.length).toEqual(2)
expect(getMergeEventMessages()).toHaveLength(0)
})
it(`hitting the move limit drops the event: no merge, no IDs moved, goes to DLQ`, async () => {
await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, firstUserUuid, {
distinctId: firstUserDistinctId,
})
const second = await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, secondUserUuid, {
distinctId: secondUserDistinctId,
})
// Add more distinct IDs to source so that limit < total
const repo = new PostgresPersonRepository(hub.postgres)
await repo.addDistinctId(second, 'second-2', 1)
await repo.addDistinctId(second, 'second-3', 1)
// Set a per-call limit of 2
const mergeService: PersonMergeService = personMergeService(
{},
undefined,
undefined,
true,
timestamp,
mainTeam,
{ type: 'LIMIT' as const, limit: 2 }
)
const result = await mergeService.merge(secondUserDistinctId, firstUserDistinctId, teamId, timestamp)
expect(result.success).toBe(false)
if (result.success) {
throw new Error('Expected merge to fail due to limit exceeded')
}
expect(result.error.message).toContain('person_merge_move_limit_hit')
// Persons should be unchanged (no delete, no merge)
const persons = sortPersons(await fetchPostgresPersonsH())
expect(persons.find((p) => p.uuid === firstUserUuid)).toBeTruthy()
expect(persons.find((p) => p.uuid === secondUserUuid)).toBeTruthy()
// Distinct IDs should remain on source (we added 2 extra)
const sourceDistinctIds = await fetchDistinctIdValues(hub.postgres, second)
expect(sourceDistinctIds).toEqual(expect.arrayContaining([secondUserDistinctId, 'second-2', 'second-3']))
})
it(`exact limit hit: delete source and do not emit warning`, async () => {
mockProducerObserver.resetKafkaProducer()
await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, firstUserUuid, {
distinctId: firstUserDistinctId,
})
const second = await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, secondUserUuid, {
distinctId: secondUserDistinctId,
})
// Add one more distinct ID on source so total equals limit (2)
const repo = new PostgresPersonRepository(hub.postgres)
await repo.addDistinctId(second, 'second-2', 1)
const mergeService: PersonMergeService = personMergeService(
{},
undefined,
undefined,
true,
timestamp,
mainTeam,
{ type: 'LIMIT' as const, limit: 2 }
)
const result = await mergeService.merge(secondUserDistinctId, firstUserDistinctId, teamId, timestamp)
expect(result.success).toBe(true)
if (!result.success) {
throw new Error('Merge should have succeeded')
}
const kafkaAcks = result.kafkaAck
const context = mergeService.getContext()
await flushPersonStoreToKafka(kafkaProducer, context.personStore, kafkaAcks)
// Source should be deleted since we moved exactly the limit and none remain
const persons = sortPersons(await fetchPostgresPersonsH())
expect(persons.find((p) => p.uuid === secondUserUuid)).toBeFalsy()
// No warning should be emitted
const warnings = mockProducerObserver.getProducedKafkaMessagesForTopic(KAFKA_INGESTION_WARNINGS)
expect(
warnings.some(
(w) =>
w.value?.type === 'merge_distinct_ids_over_limit' ||
(typeof w.value === 'object' && (w.value as any).type === 'merge_distinct_ids_over_limit')
)
).toBe(false)
})
it(`throws if postgres unavailable`, async () => {
await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, firstUserUuid, {
distinctId: firstUserDistinctId,
})
await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, secondUserUuid, {
distinctId: secondUserDistinctId,
})
const state: PersonMergeService = personMergeService({}, hub)
jest.spyOn(kafkaProducer, 'produce')
// break postgres
const error = new DependencyUnavailableError('testing', 'Postgres', new Error('test'))
jest.spyOn(hub.postgres, 'transaction').mockImplementation(() => {
throw error
})
await expect(state.merge(secondUserDistinctId, firstUserDistinctId, teamId, timestamp)).rejects.toThrow(
error
)
await kafkaProducer.flush()
// merge() retries transient failures, so every attempt opens a transaction.
expect(hub.postgres.transaction).toHaveBeenCalledTimes(3)
jest.spyOn(hub.postgres, 'transaction').mockRestore()
expect(kafkaProducer.produce).not.toHaveBeenCalled()
// verify Postgres persons
const persons = sortPersons(await fetchPostgresPersonsH())
expect(persons).toEqual(
expect.arrayContaining([
expect.objectContaining({
id: expect.any(String),
uuid: firstUserUuid,
properties: {},
created_at: timestamp,
version: 0,
is_identified: false,
}),
expect.objectContaining({
id: expect.any(String),
uuid: secondUserUuid,
properties: {},
created_at: timestamp,
version: 0,
is_identified: false,
}),
])
)
})
it(`retries merges up to retry limit if postgres down`, async () => {
await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, firstUserUuid, {
distinctId: firstUserDistinctId,
})
await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, secondUserUuid, {
distinctId: secondUserDistinctId,
})
const state: PersonMergeService = personMergeService({}, hub)
jest.spyOn(kafkaProducer, 'produce')
// break postgres
const error = new DependencyUnavailableError('testing', 'Postgres', new Error('test'))
// A rejection, not a synchronous throw: the retry wrapper only
// catches rejections, matching how the async store fails for real.
jest.spyOn(state.getContext().personStore, 'mergePersons').mockRejectedValue(error)
await expect(state.merge(secondUserDistinctId, firstUserDistinctId, teamId, timestamp)).rejects.toThrow(
error
)
await kafkaProducer.flush()
expect(state.getContext().personStore.mergePersons).toHaveBeenCalledTimes(3)
jest.spyOn(state.getContext().personStore, 'mergePersons').mockRestore()
expect(kafkaProducer.produce).not.toHaveBeenCalled()
// verify Postgres persons
const persons = sortPersons(await fetchPostgresPersonsH())
expect(persons).toEqual(
expect.arrayContaining([
expect.objectContaining({
id: expect.any(String),
uuid: firstUserUuid,
properties: {},
created_at: timestamp,
version: 0,
is_identified: false,
}),
expect.objectContaining({
id: expect.any(String),
uuid: secondUserUuid,
properties: {},
created_at: timestamp,
version: 0,
is_identified: false,
}),
])
)
})
it(`handleIdentifyOrAlias does not throw on merge failure`, async () => {
// TODO: This the current state, we should probably change it
await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, firstUserUuid, {
distinctId: firstUserDistinctId,
})
await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, secondUserUuid, {
distinctId: secondUserDistinctId,
})
const state: PersonMergeService = personMergeService(
{
event: '$merge_dangerously',
distinct_id: firstUserDistinctId,
properties: { alias: secondUserDistinctId },
},
hub
)
jest.spyOn(kafkaProducer, 'produce')
// break postgres
const error = new DependencyUnavailableError('testing', 'Postgres', new Error('test'))
// A rejection, not a synchronous throw: the retry wrapper only
// catches rejections, matching how the async store fails for real.
jest.spyOn(state.getContext().personStore, 'mergePersons').mockRejectedValue(error)
await state.handleIdentifyOrAlias()
await kafkaProducer.flush()
expect(state.getContext().personStore.mergePersons).toHaveBeenCalledTimes(3)
jest.spyOn(state.getContext().personStore, 'mergePersons').mockRestore()
expect(kafkaProducer.produce).not.toHaveBeenCalled()
// verify Postgres persons
const persons = sortPersons(await fetchPostgresPersonsH())
expect(persons).toEqual(
expect.arrayContaining([
expect.objectContaining({
id: expect.stringMatching(/^[0-9]+$/),
uuid: firstUserUuid,
properties: {},
created_at: timestamp,
version: 0,
is_identified: false,
}),
expect.objectContaining({
id: expect.stringMatching(/^[0-9]+$/),
uuid: secondUserUuid,
properties: {},
created_at: timestamp,
version: 0,
is_identified: false,
}),
])
)
})
it(`retries merge when source person is deleted during merge transaction`, async () => {
await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, firstUserUuid, {
distinctId: firstUserDistinctId,
})
await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, secondUserUuid, {
distinctId: secondUserDistinctId,
})
const state: PersonMergeService = personMergeService({})
let attemptCount = 0
// Mock moveDistinctIds to throw SourcePersonNotFoundError on first attempt, succeed on retry
const originalMoveDistinctIds = personRepository.moveDistinctIds
jest.spyOn(personRepository, 'moveDistinctIds').mockImplementation(async (...args) => {
attemptCount++
if (attemptCount === 1) {
throw new SourcePersonNotFoundError('Source person no longer exists')
}
return originalMoveDistinctIds.call(personRepository, ...args)
})
jest.spyOn(personRepository, 'fetchPerson')
// Should succeed after retry
const result = await state.merge(firstUserDistinctId, secondUserDistinctId, teamId, timestamp)
expect(result.success).toBe(true)
if (!result.success) {
throw new Error('Merge should have succeeded')
}
const person = result.person
const kafkaAcks = result.kafkaAck
await kafkaProducer.flush()
await kafkaAcks
// Verify that moveDistinctIds was called twice (initial + retry)
expect(personRepository.moveDistinctIds).toHaveBeenCalledTimes(2)
// Verify merge succeeded
expect(person).not.toBeUndefined()
expect(person?.uuid).toBe(secondUserUuid) // merged into second user
// Verify that fetchPerson was called for both users
expect(personRepository.fetchPerson).toHaveBeenCalledWith(teamId, firstUserDistinctId, {
callerTag: 'ingestion/person-update-conflict',
useReadReplica: false,
})
expect(personRepository.fetchPerson).toHaveBeenCalledWith(teamId, secondUserDistinctId, {
callerTag: 'ingestion/person-update-conflict',
useReadReplica: false,
})
// Verify correct number of persons remain
const persons = sortPersons(await fetchPostgresPersonsH())
expect(persons.length).toEqual(1)
expect(persons[0].uuid).toBe(secondUserUuid)
})
it(`retries merge when target person is deleted during merge transaction`, async () => {
await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, firstUserUuid, {
distinctId: firstUserDistinctId,
})
await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, secondUserUuid, {
distinctId: secondUserDistinctId,
})
const state: PersonMergeService = personMergeService({})
let attemptCount = 0
// Mock moveDistinctIds to throw TargetPersonNotFoundError on first attempt, succeed on retry
const originalMoveDistinctIds = personRepository.moveDistinctIds
jest.spyOn(personRepository, 'moveDistinctIds').mockImplementation(async (...args) => {
attemptCount++
if (attemptCount === 1) {
throw new TargetPersonNotFoundError('Target person no longer exists')
}
return originalMoveDistinctIds.call(personRepository, ...args)
})
jest.spyOn(personRepository, 'fetchPerson')
// Should succeed after retry
const result = await state.merge(firstUserDistinctId, secondUserDistinctId, teamId, timestamp)
expect(result.success).toBe(true)
if (!result.success) {
throw new Error('Merge should have succeeded')
}
const person = result.person
const kafkaAcks = result.kafkaAck
await kafkaProducer.flush()
await kafkaAcks
// Verify that moveDistinctIds was called twice (initial + retry)
expect(personRepository.moveDistinctIds).toHaveBeenCalledTimes(2)
// Verify merge succeeded
expect(person).not.toBeUndefined()
expect(person?.uuid).toBe(secondUserUuid) // merged into second user
// Verify that fetchPerson was called for both users
expect(personRepository.fetchPerson).toHaveBeenCalledWith(teamId, firstUserDistinctId, {
callerTag: 'ingestion/person-update-conflict',
useReadReplica: false,
})
expect(personRepository.fetchPerson).toHaveBeenCalledWith(teamId, secondUserDistinctId, {
callerTag: 'ingestion/person-update-conflict',
useReadReplica: false,
})
// Verify correct number of persons remain
const persons = sortPersons(await fetchPostgresPersonsH())
expect(persons.length).toEqual(1)
expect(persons[0].uuid).toBe(secondUserUuid)
})
it(`skips merge when source person no longer exists after retry`, async () => {
const firstPerson = await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, firstUserUuid, {
distinctId: firstUserDistinctId,
})
const secondPerson = await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, secondUserUuid, {
distinctId: secondUserDistinctId,
})
const state: PersonMergeService = personMergeService({})
let attemptCount = 0
let fetchPersonCalls = 0
// Mock fetchPerson to return null on retry (person no longer exists)
jest.spyOn(personRepository, 'fetchPerson').mockImplementation(() => {
fetchPersonCalls++
// Fetch source person
if (fetchPersonCalls === 1) {
return Promise.resolve(firstPerson)
}
// Fetch target person
if (fetchPersonCalls === 2) {
return Promise.resolve(secondPerson)
}
// Fetch for retry
return Promise.resolve(undefined)
})
// Mock moveDistinctIds to throw SourcePersonNotFoundError on first attempt
jest.spyOn(personRepository, 'moveDistinctIds').mockImplementation(() => {
attemptCount++
if (attemptCount === 1) {
throw new SourcePersonNotFoundError('Source person no longer exists')
}
// Should not reach here since person lookup returns null
throw new Error('Should not retry when person no longer exists')
})
// Should return target person without error
const result = await state.merge(firstUserDistinctId, secondUserDistinctId, teamId, timestamp)
expect(result.success).toBe(true)
if (!result.success) {
throw new Error('Merge should have succeeded')
}
const person = result.person
const kafkaAcks = result.kafkaAck
await kafkaProducer.flush()
await kafkaAcks
// Verify that moveDistinctIds was only called once (no retry since person doesn't exist)
expect(personRepository.moveDistinctIds).toHaveBeenCalledTimes(1)
// Verify we got the target person back
expect(person).not.toBeUndefined()
expect(person?.uuid).toBe(secondUserUuid)
// Verify both persons still exist since merge was skipped
const persons = sortPersons(await fetchPostgresPersonsH())
expect(persons.length).toEqual(2)
})
it(`skips merge when target person no longer exists after retry`, async () => {
const firstPerson = await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, firstUserUuid, {
distinctId: firstUserDistinctId,
})
const secondPerson = await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, secondUserUuid, {
distinctId: secondUserDistinctId,
})
const state: PersonMergeService = personMergeService({})
let attemptCount = 0
let fetchPersonCalls = 0
jest.spyOn(personRepository, 'fetchPerson').mockImplementation(() => {
fetchPersonCalls++
// Fetch source person
if (fetchPersonCalls === 1) {
return Promise.resolve(firstPerson)
}
// Fetch target person
if (fetchPersonCalls === 2) {
return Promise.resolve(secondPerson)
}
// Fetch for retry
return Promise.resolve(undefined)
})
// Mock moveDistinctIds to throw TargetPersonNotFoundError on first attempt
jest.spyOn(personRepository, 'moveDistinctIds').mockImplementation(() => {
attemptCount++
if (attemptCount === 1) {
throw new TargetPersonNotFoundError('Target person no longer exists')
}
// Should not reach here since person lookup returns null
throw new Error('Should not retry when person no longer exists')
})
// Should return target person without error
const result = await state.merge(firstUserDistinctId, secondUserDistinctId, teamId, timestamp)
expect(result.success).toBe(true)
if (!result.success) {
throw new Error('Merge should have succeeded')
}
const person = result.person
const kafkaAcks = result.kafkaAck
await kafkaProducer.flush()
await kafkaAcks
// Verify that moveDistinctIds was only called once (no retry since person doesn't exist)
expect(personRepository.moveDistinctIds).toHaveBeenCalledTimes(1)
// Verify we got the original target person back (which was the second user)
expect(person).not.toBeUndefined()
expect(person?.uuid).toBe(secondUserUuid)
// Verify both persons still exist since merge was skipped
const persons = sortPersons(await fetchPostgresPersonsH())
expect(persons.length).toEqual(2)
})
it('clears cache and refreshes to new person when merge retry finds distinctId points to different person', async () => {
// Create initial persons
const person1 = await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, firstUserUuid, {
distinctId: firstUserDistinctId,
})
const person2 = await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, secondUserUuid, {
distinctId: secondUserDistinctId,
})
// Create person 3 who will be the "new" person that the distinctId points to after refresh
const person3Uuid = new UUIDT().toString()
const person3 = await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, person3Uuid, {
distinctId: 'person3-distinct-id',
})
// Create a merge service with batch writing store for a $merge_dangerously event
const batchStore = new BatchWritingPersonsStore(personRepository, createPersonOutputs(kafkaProducer))
const mergeService: PersonMergeService = personMergeService(
{
event: '$merge_dangerously',
distinct_id: secondUserDistinctId,
},
hub,
undefined,
undefined,
undefined,
undefined,
undefined,
undefined,
batchStore
)
batchStore.setCachedPersonForUpdate(
teamId,
firstUserDistinctId,
fromInternalPerson(person1, firstUserDistinctId)
)
// Verify the cache is populated
expect(batchStore.getCachedPersonForUpdateByDistinctId(teamId, firstUserDistinctId)).toBeTruthy()
// Mock removeDistinctIdFromCache to verify it's called
const removeDistinctIdFromCacheSpy = jest.spyOn(batchStore, 'removeDistinctIdFromCache')
// Another process moves firstUserDistinctId to person3 and deletes person1 between this merge's fetch
// and its row locks; the locks find person1 gone, and the retry merges person3 instead.
let lockCalls = 0
const readMergeRowsSpy = jest
.spyOn(personRepository, 'readMergeRows')
.mockImplementation(async (...args) => {
lockCalls++
if (lockCalls === 1) {
await personRepository.inRawTransaction('test', async (tx) => {
await personRepository.moveDistinctIds(person1, person3, undefined, tx)
await personRepository.deletePerson(person1, tx)
})
}
return await PostgresPersonRepository.prototype.readMergeRows.apply(personRepository, args)
})
// Attempt to merge persons - this should trigger the retry logic
const result = await mergeService.merge(firstUserDistinctId, secondUserDistinctId, teamId, timestamp)
expect(result.success).toBe(true)
if (!result.success) {
throw new Error('Merge should have succeeded')
}
const person = result.person
// Verify the cache was cleared during retry
expect(removeDistinctIdFromCacheSpy).toHaveBeenCalledWith(teamId, firstUserDistinctId)
// The first attempt's locks found person1 gone; the retry locked person3.
expect(readMergeRowsSpy).toHaveBeenCalledTimes(2)
// Verify we got the target person back
expect(person).toMatchObject({
id: person2.id,
uuid: secondUserUuid,
})
// Verify only 1 person exists
const persons = sortPersons(await fetchPostgresPersonsH())
expect(persons.length).toEqual(1)
expect(persons[0].uuid).toBe(secondUserUuid)
// Verify the distinct ID is now pointing to the target person
const distinctIds = await fetchPostgresDistinctIdsForPerson(hub.postgres, person2.id)
expect(distinctIds.length).toEqual(3)
expect(distinctIds).toMatchObject(
expect.arrayContaining([firstUserDistinctId, secondUserDistinctId, 'person3-distinct-id'])
)
})
it('refreshes to new person when merging dangerously two people', async () => {
// Create initial persons
const person1 = await createPerson(hub, timestamp, {}, {}, {}, teamId, null, true, firstUserUuid, {
distinctId: firstUserDistinctId,
})
const person2 = await createPerson(hub, timestamp, {}, {}, {}, teamId, null, true, secondUserUuid, {
distinctId: secondUserDistinctId,
})
const person3Uuid = new UUIDT().toString()
const person3 = await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, person3Uuid, {
distinctId: 'person3-distinct-id',
})
// Add one distinct id entry to each person, simulating a previous merge
await personRepository.addDistinctId(person1, 'person1-merged-distinct-id', 1)
await personRepository.addDistinctId(person2, 'person2-merged-distinct-id', 1)
const mergeService: PersonMergeService = personMergeService({
event: '$merge_dangerously',
})
// Another process moves firstUserDistinctId to person3 and deletes person1 between this merge's fetch
// and its row locks; the locks find person1 gone, and the retry merges person3 instead.
let lockCalls = 0
const readMergeRowsSpy = jest
.spyOn(personRepository, 'readMergeRows')
.mockImplementation(async (...args) => {
lockCalls++
if (lockCalls === 1) {
await personRepository.moveDistinctIds(person1, person3)
await personRepository.deletePerson(person1)
}
return await PostgresPersonRepository.prototype.readMergeRows.apply(personRepository, args)
})
// Attempt to merge persons - this should trigger the retry logic
const result = await mergeService.merge(firstUserDistinctId, secondUserDistinctId, teamId, timestamp)
expect(result.success).toBe(true)
if (!result.success) {
throw new Error('Merge should have succeeded')
}
const person = result.person
// The first attempt's locks found person1 gone; the retry locked person3.
expect(readMergeRowsSpy).toHaveBeenCalledTimes(2)
// Verify we got the target person back
expect(person).toMatchObject({
id: person2.id,
uuid: secondUserUuid,
})
// Verify only 1 person exists
const persons = sortPersons(await fetchPostgresPersonsH())
expect(persons.length).toEqual(1)
expect(persons[0].uuid).toBe(secondUserUuid)
// Verify the distinct ID is now pointing to the target person
const distinctIds = await fetchPostgresDistinctIdsForPerson(hub.postgres, person2.id)
expect(distinctIds.length).toEqual(5)
expect(distinctIds).toMatchObject(
expect.arrayContaining([
firstUserDistinctId,
secondUserDistinctId,
'person3-distinct-id',
'person1-merged-distinct-id',
'person2-merged-distinct-id',
])
)
})
it('merge with a concurrently tombstoned source converges to a no-op instead of half-applying', async () => {
const target = await createPerson(hub, timestamp, {}, {}, {}, teamId, null, true, firstUserUuid, {
distinctId: firstUserDistinctId,
})
const source = await createPerson(
hub,
timestamp,
{ plan: 'pro' },
{},
{},
teamId,
null,
true,
secondUserUuid,
{ distinctId: secondUserDistinctId }
)
// A concurrent operation tombstones the source after our stale fetch, while an
// identity write that raced the tombstone left a live mapping pointing at it.
// Without the post-claim source liveness check the merge would move that mapping
// and merge the stale source properties into the target. The stale fetch is
// seeded into the store's cache: a fresh fetch filters tombstoned persons, and
// the race this pins is precisely a fetch that predates the tombstone.
const staleCacheStore = new BatchWritingPersonsStore(personRepository, createPersonOutputs(kafkaProducer))
staleCacheStore.setCachedPersonForUpdate(
teamId,
secondUserDistinctId,
fromInternalPerson(source, secondUserDistinctId)
)
await hub.postgres.query(
PostgresUse.PERSONS_WRITE,
'UPDATE posthog_person SET is_deleted = true, version = version + 1 WHERE team_id = $1 AND id = $2',
[teamId, source.id],
'testTombstoneSource'
)
const mergeService: PersonMergeService = personMergeService(
{
event: '$merge_dangerously',
distinct_id: firstUserDistinctId,
uuid: new UUIDT().toString(),
},
hub,
personRepository,
true,
timestamp,
mainTeam,
createDefaultSyncMergeMode(),
undefined,
staleCacheStore
)
const result = await mergeService.merge(secondUserDistinctId, firstUserDistinctId, teamId, timestamp)
// The retry refreshes by the source distinct id, which no longer resolves to a
// live person, so the merge converges to a no-op on the untouched target.
expect(result.success).toBe(true)
if (!result.success) {
throw new Error('Merge should have succeeded')
}
expect(result.person).toMatchObject({ id: target.id, uuid: firstUserUuid })
const persons = await fetchPostgresPersonsH()
expect(persons.length).toEqual(1)
expect(persons[0]).toMatchObject({
id: target.id,
properties: {},
version: 0,
})
const targetDistinctIds = await fetchPostgresDistinctIdsForPerson(hub.postgres, target.id)
expect(targetDistinctIds).toEqual([firstUserDistinctId])
})
it('merge retries past a cached tombstoned person instead of failing', async () => {
const person = await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, firstUserUuid, {
distinctId: firstUserDistinctId,
})
const mergeService: PersonMergeService = personMergeService(
{
event: '$merge_dangerously',
distinct_id: secondUserDistinctId,
properties: { alias: firstUserDistinctId },
uuid: new UUIDT().toString(),
},
hub,
personRepository,
true,
timestamp,
mainTeam,
createDefaultSyncMergeMode()
)
// Warm the batch cache with the live person, then tombstone it and its
// mapping out-of-band: the cache is now stale about liveness, and only a
// purge-on-throw lets the retry see the fresh state.
await mergeService.getContext().personStore.fetchForUpdate(teamId, firstUserDistinctId)
await hub.postgres.query(
PostgresUse.PERSONS_WRITE,
'UPDATE posthog_person SET is_deleted = true, version = version + 1 WHERE team_id = $1 AND id = $2',
[teamId, person.id],
'testTombstonePerson'
)
await hub.postgres.query(
PostgresUse.PERSONS_WRITE,
'UPDATE posthog_persondistinctid SET is_deleted = true, version = COALESCE(version, 0) + 1 WHERE team_id = $1 AND distinct_id = $2',
[teamId, firstUserDistinctId],
'testTombstoneDistinctId'
)
const result = await mergeService.merge(firstUserDistinctId, secondUserDistinctId, teamId, timestamp)
// The purged retry re-fetches, finds neither distinct id live, and
// recreates the person instead of exhausting retries on the cached row.
expect(result.success).toBe(true)
if (!result.success) {
throw new Error('Merge should have succeeded')
}
expect(result.person).toBeDefined()
})
it('merge held off by a live lifecycle claim drops with a warning', async () => {
const target = await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, firstUserUuid, {
distinctId: firstUserDistinctId,
})
await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, secondUserUuid, {
distinctId: secondUserDistinctId,
})
// The file-level observer's spy dies with the first test's restoreAllMocks;
// observe this test's produces through a fresh spy.
const producerObserver = new KafkaProducerObserver(kafkaProducer)
// A concurrent lifecycle operation (e.g. a delete saga) holds the target
// through every merge retry.
const holderOpId = new UUIDT().toString()
await personRepository.claimLifecycleMarks(holderOpId, teamId, [
{ personId: target.id, personUuid: target.uuid, role: 'source' },
])
try {
const mergeService: PersonMergeService = personMergeService(
{
event: '$merge_dangerously',
distinct_id: firstUserDistinctId,
properties: { alias: secondUserDistinctId },
uuid: new UUIDT().toString(),
},
hub,
personRepository,
true,
timestamp,
mainTeam,
createDefaultSyncMergeMode()
)
const result = await mergeService.handleIdentifyOrAlias()
// The merge is dropped, not failed: success with no person, plus the warning.
expect(result.success).toBe(true)
if (!result.success) {
throw new Error('Expected a dropped merge to resolve successfully')
}
expect(result.person).toBeUndefined()
const warnings = producerObserver
.getProducedKafkaMessagesForTopic(KAFKA_INGESTION_WARNINGS)
.filter((w) => (w.value as any)?.type === 'merge_race_condition')
expect(warnings).toHaveLength(1)
} finally {
await personRepository.releaseLifecycleMarks(holderOpId, teamId)
}
})
it('merge stamps the exact death version on the source row and its death message', async () => {
await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, firstUserUuid, {
distinctId: firstUserDistinctId,
})
await createPerson(hub, timestamp, { plan: 'pro' }, {}, {}, teamId, null, false, secondUserUuid, {
distinctId: secondUserDistinctId,
})
const producerObserver = new KafkaProducerObserver(kafkaProducer)
const mergeService: PersonMergeService = personMergeService(
{
event: '$merge_dangerously',
distinct_id: firstUserDistinctId,
properties: { alias: secondUserDistinctId },
uuid: new UUIDT().toString(),
},
hub,
personRepository,
true,
timestamp,
mainTeam,
createDefaultSyncMergeMode()
)
const result = await mergeService.merge(secondUserDistinctId, firstUserDistinctId, teamId, timestamp)
expect(result.success).toBe(true)
if (!result.success) {
throw new Error('Merge should have succeeded')
}
await flushPersonStoreToKafka(kafkaProducer, mergeService.getContext().personStore, result.kafkaAck)
// The source row survives as a tombstone at exactly death version
// (source.version + 1) with scrubbed properties — not a hard delete.
const sourceRows = await hub.postgres.query(
PostgresUse.PERSONS_WRITE,
'SELECT is_deleted, version, properties FROM posthog_person WHERE team_id = $1 AND uuid = $2',
[teamId, secondUserUuid],
'fetchTombstonedSource'
)
expect(sourceRows.rows).toEqual([{ is_deleted: true, version: '1', properties: {} }])
// Filtered reads hide the tombstone: only the target remains visible.
const persons = await fetchPostgresPersonsH()
expect(persons.map((p) => p.uuid)).toEqual([firstUserUuid])
// The death message carries the exact death version, not the +100 fudge.
const deathMessages = producerObserver
.getProducedKafkaMessagesForTopic(KAFKA_PERSON)
.filter((m) => (m.value as any)?.id === secondUserUuid && (m.value as any)?.is_deleted === 1)
expect(deathMessages).toHaveLength(1)
expect(deathMessages[0].value).toMatchObject({ is_deleted: 1, version: 1, properties: '{}' })
})
describe('SYNC mode with batch processing', () => {
it('merges all distinct IDs when batch size is larger than total distinct IDs', async () => {
await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, firstUserUuid, {
distinctId: firstUserDistinctId,
})
const second = await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, secondUserUuid, {
distinctId: secondUserDistinctId,
})
// Add a few more distinct IDs to the source person
const repo = new PostgresPersonRepository(hub.postgres)
await repo.addDistinctId(second, 'second-2', 0)
await repo.addDistinctId(second, 'second-3', 0)
// Use SYNC mode with batch size larger than total distinct IDs (5 > 3)
const mergeService: PersonMergeService = personMergeService({}, hub, repo, true, timestamp, mainTeam, {
type: 'SYNC' as const,
batchSize: 5,
})
jest.spyOn(repo, 'moveDistinctIds')
const result = await mergeService.merge(secondUserDistinctId, firstUserDistinctId, teamId, timestamp)
expect(result.success).toBe(true)
if (!result.success) {
throw new Error('Merge should have succeeded')
}
const person = result.person
const kafkaAcks = result.kafkaAck
const context = mergeService.getContext()
await flushPersonStoreToKafka(kafkaProducer, context.personStore, kafkaAcks)
// Should have called moveDistinctIds only once since batch size > total distinct IDs
expect(repo.moveDistinctIds).toHaveBeenCalledTimes(1)
// Verify all distinct IDs were moved
const distinctIds = await fetchDistinctIdValues(hub.postgres, person!)
expect(distinctIds).toEqual(
expect.arrayContaining([firstUserDistinctId, secondUserDistinctId, 'second-2', 'second-3'])
)
expect(distinctIds.length).toBe(4)
})
it('merges all distinct IDs in multiple batches when batch size is smaller than total', async () => {
await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, firstUserUuid, {
distinctId: firstUserDistinctId,
})
const second = await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, secondUserUuid, {
distinctId: secondUserDistinctId,
})
// Add several distinct IDs to the source person
const repo = new PostgresPersonRepository(hub.postgres)
await repo.addDistinctId(second, 'second-2', 0)
await repo.addDistinctId(second, 'second-3', 0)
await repo.addDistinctId(second, 'second-4', 0)
await repo.addDistinctId(second, 'second-5', 0)
// Use SYNC mode with small batch size (2 < 5 total distinct IDs)
const mergeService: PersonMergeService = personMergeService({}, hub, repo, true, timestamp, mainTeam, {
type: 'SYNC' as const,
batchSize: 2,
})
jest.spyOn(repo, 'moveDistinctIds')
const result = await mergeService.merge(secondUserDistinctId, firstUserDistinctId, teamId, timestamp)
expect(result.success).toBe(true)
if (!result.success) {
throw new Error('Merge should have succeeded')
}
const person = result.person
const kafkaAcks = result.kafkaAck
const context = mergeService.getContext()
await flushPersonStoreToKafka(kafkaProducer, context.personStore, kafkaAcks)
// Should have called moveDistinctIds multiple times due to batching
expect(repo.moveDistinctIds).toHaveBeenCalledTimes(3) // 5 distinct IDs / 2 batch size = 3 calls (2+2+1)
// Verify all distinct IDs were moved despite batching
const distinctIds = await fetchDistinctIdValues(hub.postgres, person!)
expect(distinctIds).toEqual(
expect.arrayContaining([
firstUserDistinctId,
secondUserDistinctId,
'second-2',
'second-3',
'second-4',
'second-5',
])
)
expect(distinctIds.length).toBe(6)
})
it('handles edge case with batch size of 1', async () => {
await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, firstUserUuid, {
distinctId: firstUserDistinctId,
})
const second = await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, secondUserUuid, {
distinctId: secondUserDistinctId,
})
// Add one more distinct ID to the source person
const repo = new PostgresPersonRepository(hub.postgres)
await repo.addDistinctId(second, 'second-2', 0)
// Use SYNC mode with batch size of 1
const mergeService: PersonMergeService = personMergeService({}, hub, repo, true, timestamp, mainTeam, {
type: 'SYNC' as const,
batchSize: 1,
})
jest.spyOn(repo, 'moveDistinctIds')
const result = await mergeService.merge(secondUserDistinctId, firstUserDistinctId, teamId, timestamp)
expect(result.success).toBe(true)
if (!result.success) {
throw new Error('Merge should have succeeded')
}
const person = result.person
const kafkaAcks = result.kafkaAck
const context = mergeService.getContext()
await flushPersonStoreToKafka(kafkaProducer, context.personStore, kafkaAcks)
// Once all for each distinct ID moved, then one more to make sure all distinct IDs were moved
expect(repo.moveDistinctIds).toHaveBeenCalledTimes(3)
// Verify all distinct IDs were moved
const distinctIds = await fetchDistinctIdValues(hub.postgres, person!)
expect(distinctIds).toEqual(
expect.arrayContaining([firstUserDistinctId, secondUserDistinctId, 'second-2'])
)
expect(distinctIds.length).toBe(3)
})
it('handles SYNC mode with undefined batch size (unlimited)', async () => {
await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, firstUserUuid, {
distinctId: firstUserDistinctId,
})
const second = await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, secondUserUuid, {
distinctId: secondUserDistinctId,
})
// Add several distinct IDs to the source person
const repo = new PostgresPersonRepository(hub.postgres)
await repo.addDistinctId(second, 'second-2', 0)
await repo.addDistinctId(second, 'second-3', 0)
await repo.addDistinctId(second, 'second-4', 0)
// Use default SYNC mode (undefined batch size = unlimited)
const mergeService: PersonMergeService = personMergeService({}, hub, repo, true, timestamp, mainTeam, {
type: 'SYNC' as const,
batchSize: undefined,
})
jest.spyOn(repo, 'moveDistinctIds')
const result = await mergeService.merge(secondUserDistinctId, firstUserDistinctId, teamId, timestamp)
expect(result.success).toBe(true)
if (!result.success) {
throw new Error('Merge should have succeeded')
}
const person = result.person
const kafkaAcks = result.kafkaAck
const context = mergeService.getContext()
await flushPersonStoreToKafka(kafkaProducer, context.personStore, kafkaAcks)
// Should have called moveDistinctIds only once (unlimited batch size)
expect(repo.moveDistinctIds).toHaveBeenCalledTimes(1)
// Verify all distinct IDs were moved
const distinctIds = await fetchDistinctIdValues(hub.postgres, person!)
expect(distinctIds).toEqual(
expect.arrayContaining([
firstUserDistinctId,
secondUserDistinctId,
'second-2',
'second-3',
'second-4',
])
)
expect(distinctIds.length).toBe(5)
})
})
describe('Merge Mode Testing', () => {
let hub: Hub
beforeEach(async () => {
hub = await createHub({})
personRepository = new PostgresPersonRepository(hub.postgres)
jest.spyOn(personRepository, 'fetchPerson')
jest.spyOn(personRepository, 'updatePerson')
})
afterEach(async () => {
jest.clearAllTimers()
jest.useRealTimers()
jest.restoreAllMocks()
await closeHub(hub)
})
describe('SYNC mode', () => {
it('merges all distinct IDs in unlimited batches when batchSize is undefined', async () => {
await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, firstUserUuid, {
distinctId: firstUserDistinctId,
})
const second = await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, secondUserUuid, {
distinctId: secondUserDistinctId,
})
// Add many distinct IDs to the source person
const repo = new PostgresPersonRepository(hub.postgres)
for (let i = 2; i <= 10; i++) {
await repo.addDistinctId(second, `second-${i}`, 0)
}
// Use SYNC mode with unlimited batch size (default)
const syncMode = { type: 'SYNC' as const, batchSize: undefined }
const mergeService: PersonMergeService = personMergeService(
{},
hub,
repo,
true,
timestamp,
mainTeam,
syncMode
)
jest.spyOn(repo, 'moveDistinctIds')
const result = await mergeService.merge(
secondUserDistinctId,
firstUserDistinctId,
teamId,
timestamp
)
expect(result.success).toBe(true)
if (!result.success) {
throw new Error('Merge should have succeeded')
}
const person = result.person
const kafkaAcks = result.kafkaAck
const context = mergeService.getContext()
await flushPersonStoreToKafka(kafkaProducer, context.personStore, kafkaAcks)
// Should have called moveDistinctIds only once (unlimited batch size)
expect(repo.moveDistinctIds).toHaveBeenCalledTimes(1)
// Verify all distinct IDs were moved
const distinctIds = await fetchDistinctIdValues(hub.postgres, person!)
expect(distinctIds.length).toBe(11) // 1 original + 10 added
expect(distinctIds).toContain(firstUserDistinctId)
expect(distinctIds).toContain(secondUserDistinctId)
for (let i = 2; i <= 10; i++) {
expect(distinctIds).toContain(`second-${i}`)
}
})
it('merges distinct IDs in specified batches when batchSize is set', async () => {
await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, firstUserUuid, {
distinctId: firstUserDistinctId,
})
const second = await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, secondUserUuid, {
distinctId: secondUserDistinctId,
})
// Add 7 more distinct IDs to the source person (total 8 to move)
const repo = new PostgresPersonRepository(hub.postgres)
for (let i = 2; i <= 8; i++) {
await repo.addDistinctId(second, `second-${i}`, 0)
}
// Use SYNC mode with batch size of 3
const syncMode = { type: 'SYNC' as const, batchSize: 3 }
const mergeService: PersonMergeService = personMergeService(
{},
hub,
repo,
true,
timestamp,
mainTeam,
syncMode
)
jest.spyOn(repo, 'moveDistinctIds')
const result = await mergeService.merge(
secondUserDistinctId,
firstUserDistinctId,
teamId,
timestamp
)
expect(result.success).toBe(true)
if (!result.success) {
throw new Error('Merge should have succeeded')
}
const person = result.person
const kafkaAcks = result.kafkaAck
const context = mergeService.getContext()
await flushPersonStoreToKafka(kafkaProducer, context.personStore, kafkaAcks)
// Should have called moveDistinctIds 3 times: 3+3+2 = 8 distinct IDs
expect(repo.moveDistinctIds).toHaveBeenCalledTimes(3)
// Verify all distinct IDs were moved
const distinctIds = await fetchDistinctIdValues(hub.postgres, person!)
expect(distinctIds.length).toBe(9) // 1 original + 8 moved
expect(distinctIds).toContain(firstUserDistinctId)
expect(distinctIds).toContain(secondUserDistinctId)
for (let i = 2; i <= 8; i++) {
expect(distinctIds).toContain(`second-${i}`)
}
})
it('merges distinct IDs when count exactly equals batch size', async () => {
await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, firstUserUuid, {
distinctId: firstUserDistinctId,
})
const second = await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, secondUserUuid, {
distinctId: secondUserDistinctId,
})
// Add exactly 2 more distinct IDs to match batch size of 3
const repo = new PostgresPersonRepository(hub.postgres)
await repo.addDistinctId(second, 'second-2', 0)
await repo.addDistinctId(second, 'second-3', 0)
// Use SYNC mode with batch size exactly equal to distinct ID count (3)
const syncMode = { type: 'SYNC' as const, batchSize: 3 }
const mergeService: PersonMergeService = personMergeService(
{},
hub,
repo,
true,
timestamp,
mainTeam,
syncMode
)
jest.spyOn(repo, 'moveDistinctIds')
const result = await mergeService.merge(
secondUserDistinctId,
firstUserDistinctId,
teamId,
timestamp
)
expect(result.success).toBe(true)
if (!result.success) {
throw new Error('Merge should have succeeded')
}
const person = result.person
const kafkaAcks = result.kafkaAck
const context = mergeService.getContext()
await flushPersonStoreToKafka(kafkaProducer, context.personStore, kafkaAcks)
// Should have called moveDistinctIds first for the 3 distinct IDs, then to check if anything remains
expect(repo.moveDistinctIds).toHaveBeenCalledTimes(2)
// Verify all distinct IDs were moved
const distinctIds = await fetchDistinctIdValues(hub.postgres, person!)
expect(distinctIds.length).toBe(4) // 1 original + 3 moved
expect(distinctIds).toContain(firstUserDistinctId)
expect(distinctIds).toContain(secondUserDistinctId)
expect(distinctIds).toContain('second-2')
expect(distinctIds).toContain('second-3')
})
})
describe('LIMIT mode', () => {
it('successfully merges when distinct ID count is within limit', async () => {
await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, firstUserUuid, {
distinctId: firstUserDistinctId,
})
const second = await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, secondUserUuid, {
distinctId: secondUserDistinctId,
})
// Add 2 more distinct IDs to the source person (total 3 to move)
const repo = new PostgresPersonRepository(hub.postgres)
await repo.addDistinctId(second, 'second-2', 0)
await repo.addDistinctId(second, 'second-3', 0)
// Use LIMIT mode with limit of 5 (more than the 3 we have)
const limitMode = { type: 'LIMIT' as const, limit: 5 }
const mergeService: PersonMergeService = personMergeService(
{},
hub,
repo,
true,
timestamp,
mainTeam,
limitMode
)
const result = await mergeService.merge(
secondUserDistinctId,
firstUserDistinctId,
teamId,
timestamp
)
expect(result.success).toBe(true)
if (!result.success) {
throw new Error('Merge should have succeeded')
}
const person = result.person
const kafkaAcks = result.kafkaAck
const context = mergeService.getContext()
await flushPersonStoreToKafka(kafkaProducer, context.personStore, kafkaAcks)
// Verify all distinct IDs were moved
const distinctIds = await fetchDistinctIdValues(hub.postgres, person!)
expect(distinctIds.length).toBe(4) // 1 original + 3 moved
expect(distinctIds).toContain(firstUserDistinctId)
expect(distinctIds).toContain(secondUserDistinctId)
expect(distinctIds).toContain('second-2')
expect(distinctIds).toContain('second-3')
})
it('fails with PersonMergeLimitExceededError when distinct ID count exceeds limit', async () => {
await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, firstUserUuid, {
distinctId: firstUserDistinctId,
})
const second = await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, secondUserUuid, {
distinctId: secondUserDistinctId,
})
// Add 4 more distinct IDs to the source person (total 5 to move)
const repo = new PostgresPersonRepository(hub.postgres)
for (let i = 2; i <= 5; i++) {
await repo.addDistinctId(second, `second-${i}`, 0)
}
// Use LIMIT mode with limit of 3 (less than the 5 we have)
const limitMode = { type: 'LIMIT' as const, limit: 3 }
const mergeService: PersonMergeService = personMergeService(
{},
hub,
repo,
true,
timestamp,
mainTeam,
limitMode
)
const result = await mergeService.merge(
secondUserDistinctId,
firstUserDistinctId,
teamId,
timestamp
)
expect(result.success).toBe(false)
if (result.success) {
throw new Error('Expected merge to fail due to limit exceeded')
}
expect(result.error.message).toContain('person_merge_move_limit_hit')
// Verify no merge occurred - both persons should still exist separately
const persons = sortPersons(await fetchPostgresPersonsH())
expect(persons.find((p) => p.uuid === firstUserUuid)).toBeTruthy()
expect(persons.find((p) => p.uuid === secondUserUuid)).toBeTruthy()
// Verify distinct IDs remain with source person
const sourceDistinctIds = await fetchDistinctIdValues(hub.postgres, second)
expect(sourceDistinctIds.length).toBe(5)
expect(sourceDistinctIds).toContain(secondUserDistinctId)
for (let i = 2; i <= 5; i++) {
expect(sourceDistinctIds).toContain(`second-${i}`)
}
})
it('successfully merges when distinct ID count exactly equals limit', async () => {
await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, firstUserUuid, {
distinctId: firstUserDistinctId,
})
const second = await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, secondUserUuid, {
distinctId: secondUserDistinctId,
})
// Add exactly 2 more distinct IDs to match limit of 3
const repo = new PostgresPersonRepository(hub.postgres)
await repo.addDistinctId(second, 'second-2', 0)
await repo.addDistinctId(second, 'second-3', 0)
// Use LIMIT mode with limit exactly equal to distinct ID count (3)
const limitMode = { type: 'LIMIT' as const, limit: 3 }
const mergeService: PersonMergeService = personMergeService(
{},
hub,
repo,
true,
timestamp,
mainTeam,
limitMode
)
const result = await mergeService.merge(
secondUserDistinctId,
firstUserDistinctId,
teamId,
timestamp
)
expect(result.success).toBe(true)
if (!result.success) {
throw new Error('Merge should have succeeded')
}
const person = result.person
const kafkaAcks = result.kafkaAck
const context = mergeService.getContext()
await flushPersonStoreToKafka(kafkaProducer, context.personStore, kafkaAcks)
// Verify all distinct IDs were moved (exactly at limit)
const distinctIds = await fetchDistinctIdValues(hub.postgres, person!)
expect(distinctIds.length).toBe(4) // 1 original + 3 moved
expect(distinctIds).toContain(firstUserDistinctId)
expect(distinctIds).toContain(secondUserDistinctId)
expect(distinctIds).toContain('second-2')
expect(distinctIds).toContain('second-3')
})
})
describe('ASYNC mode', () => {
it('successfully merges when distinct ID count is within limit', async () => {
await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, firstUserUuid, {
distinctId: firstUserDistinctId,
})
const second = await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, secondUserUuid, {
distinctId: secondUserDistinctId,
})
// Add 1 more distinct ID to the source person (total 2 to move)
const repo = new PostgresPersonRepository(hub.postgres)
await repo.addDistinctId(second, 'second-2', 0)
// Use ASYNC mode with limit of 5 (more than the 2 we have)
const asyncMode = { type: 'ASYNC' as const, topic: 'async-merge-topic', limit: 5 }
const mergeService: PersonMergeService = personMergeService(
{},
hub,
repo,
true,
timestamp,
mainTeam,
asyncMode
)
const result = await mergeService.merge(
secondUserDistinctId,
firstUserDistinctId,
teamId,
timestamp
)
expect(result.success).toBe(true)
if (!result.success) {
throw new Error('Merge should have succeeded')
}
const person = result.person
const kafkaAcks = result.kafkaAck
const context = mergeService.getContext()
await flushPersonStoreToKafka(kafkaProducer, context.personStore, kafkaAcks)
// Verify all distinct IDs were moved (same as SYNC when under limit)
const distinctIds = await fetchDistinctIdValues(hub.postgres, person!)
expect(distinctIds.length).toBe(3) // 1 original + 2 moved
expect(distinctIds).toContain(firstUserDistinctId)
expect(distinctIds).toContain(secondUserDistinctId)
expect(distinctIds).toContain('second-2')
})
it('fails with PersonMergeLimitExceededError when distinct ID count exceeds limit', async () => {
await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, firstUserUuid, {
distinctId: firstUserDistinctId,
})
const second = await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, secondUserUuid, {
distinctId: secondUserDistinctId,
})
// Add 4 more distinct IDs to the source person (total 5 to move)
const repo = new PostgresPersonRepository(hub.postgres)
for (let i = 2; i <= 5; i++) {
await repo.addDistinctId(second, `second-${i}`, 0)
}
// Use ASYNC mode with limit of 3 (less than the 5 we have)
const asyncMode = { type: 'ASYNC' as const, topic: 'async-merge-topic', limit: 3 }
const mergeService: PersonMergeService = personMergeService(
{},
hub,
repo,
true,
timestamp,
mainTeam,
asyncMode
)
const result = await mergeService.merge(
secondUserDistinctId,
firstUserDistinctId,
teamId,
timestamp
)
expect(result.success).toBe(false)
if (result.success) {
throw new Error('Expected merge to fail due to limit exceeded')
}
expect(result.error.message).toContain('person_merge_move_limit_hit')
// Verify no merge occurred - both persons should still exist separately
const persons = sortPersons(await fetchPostgresPersonsH())
expect(persons.find((p) => p.uuid === firstUserUuid)).toBeTruthy()
expect(persons.find((p) => p.uuid === secondUserUuid)).toBeTruthy()
// Verify distinct IDs remain with source person
const sourceDistinctIds = await fetchDistinctIdValues(hub.postgres, second)
expect(sourceDistinctIds.length).toBe(5)
expect(sourceDistinctIds).toContain(secondUserDistinctId)
for (let i = 2; i <= 5; i++) {
expect(sourceDistinctIds).toContain(`second-${i}`)
}
})
it('successfully merges when distinct ID count exactly equals limit', async () => {
await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, firstUserUuid, {
distinctId: firstUserDistinctId,
})
const second = await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, secondUserUuid, {
distinctId: secondUserDistinctId,
})
// Add exactly 2 more distinct IDs to match limit of 3
const repo = new PostgresPersonRepository(hub.postgres)
await repo.addDistinctId(second, 'second-2', 0)
await repo.addDistinctId(second, 'second-3', 0)
// Use ASYNC mode with limit exactly equal to distinct ID count (3)
const asyncMode = { type: 'ASYNC' as const, topic: 'async-merge-topic', limit: 3 }
const mergeService: PersonMergeService = personMergeService(
{},
hub,
repo,
true,
timestamp,
mainTeam,
asyncMode
)
const result = await mergeService.merge(
secondUserDistinctId,
firstUserDistinctId,
teamId,
timestamp
)
expect(result.success).toBe(true)
if (!result.success) {
throw new Error('Merge should have succeeded')
}
const person = result.person
const kafkaAcks = result.kafkaAck
const context = mergeService.getContext()
await flushPersonStoreToKafka(kafkaProducer, context.personStore, kafkaAcks)
// Verify all distinct IDs were moved (exactly at limit, same as SYNC when under limit)
const distinctIds = await fetchDistinctIdValues(hub.postgres, person!)
expect(distinctIds.length).toBe(4) // 1 original + 3 moved
expect(distinctIds).toContain(firstUserDistinctId)
expect(distinctIds).toContain(secondUserDistinctId)
expect(distinctIds).toContain('second-2')
expect(distinctIds).toContain('second-3')
})
})
describe('PersonEventProcessor integration with merge modes', () => {
function createPersonEventProcessor(mergeMode: any, event: Partial<PluginEvent>) {
const fullEvent = {
team_id: teamId,
properties: {},
uuid: 'event-uuid',
...event,
}
const personsStore = new BatchWritingPersonsStore(
personRepository,
createPersonOutputs(kafkaProducer)
)
const context = new PersonContext(
fullEvent as any,
mainTeam,
event.distinct_id!,
timestamp,
true, // processPerson
createPersonOutputs(kafkaProducer),
new BatchBoundPersonsStore(personsStore, 0),
0,
mergeMode
)
const processor = new PersonEventProcessor(
context,
new PersonPropertyService(context),
new PersonMergeService(context)
)
return processor
}
it('SYNC mode routes the event to the DLQ when the saga-side limit is exceeded', async () => {
const syncMode = { type: 'SYNC' as const, batchSize: undefined }
const processor = createPersonEventProcessor(syncMode, {
event: '$identify',
distinct_id: firstUserDistinctId,
properties: {
$anon_distinct_id: secondUserDistinctId,
},
})
// Mock the merge service to return a limit exceeded error
const mergeService = (processor as any).mergeService as PersonMergeService
const { PersonMergeLimitExceededError } = await import(
'~/ingestion/common/persons/person-merge-types.js'
)
jest.spyOn(mergeService, 'handleIdentifyOrAlias').mockResolvedValue({
success: false,
error: new PersonMergeLimitExceededError('person_merge_move_limit_hit'),
})
// The Postgres world cannot produce this in SYNC mode, but the
// personhog saga's move limit can; the event goes to the DLQ —
// payload preserved for replay once the limit is raised or
// chunked moves land — rather than being dropped or failing
// the batch into a redelivery loop.
const result = await processor.processEvent()
expect(result.type).toBe(PipelineResultType.DLQ)
expect(result.warnings).toEqual([
expect.objectContaining({
type: 'merge_move_limit_exceeded',
details: expect.objectContaining({ eventUuid: expect.any(String) }),
}),
])
})
it('LIMIT mode returns DLQ result when limit exceeded', async () => {
const limitMode = { type: 'LIMIT' as const, limit: 2 }
const processor = createPersonEventProcessor(limitMode, {
event: '$identify',
distinct_id: firstUserDistinctId,
properties: {
$anon_distinct_id: secondUserDistinctId,
},
})
// Mock the merge service to return a limit exceeded error
const mergeService = (processor as any).mergeService as PersonMergeService
const { PersonMergeLimitExceededError } = await import(
'~/ingestion/common/persons/person-merge-types.js'
)
jest.spyOn(mergeService, 'handleIdentifyOrAlias').mockResolvedValue({
success: false,
error: new PersonMergeLimitExceededError('person_merge_move_limit_hit'),
})
const result = await processor.processEvent()
expect(result.type).toBe(PipelineResultType.DLQ)
if (isDlqResult(result)) {
expect(result.reason).toBe('Merge limit exceeded')
expect((result.error as any).message).toContain('person_merge_move_limit_hit')
}
})
it('ASYNC mode returns redirect result when limit exceeded', async () => {
const asyncMode = { type: 'ASYNC' as const, topic: 'async-merge-topic', limit: 2 }
const processor = createPersonEventProcessor(asyncMode, {
event: '$identify',
distinct_id: firstUserDistinctId,
properties: {
$anon_distinct_id: secondUserDistinctId,
},
})
// Mock the merge service to return a limit exceeded error
const mergeService = (processor as any).mergeService as PersonMergeService
const { PersonMergeLimitExceededError } = await import(
'~/ingestion/common/persons/person-merge-types.js'
)
jest.spyOn(mergeService, 'handleIdentifyOrAlias').mockResolvedValue({
success: false,
error: new PersonMergeLimitExceededError('person_merge_move_limit_hit'),
})
const result = await processor.processEvent()
expect(result.type).toBe(PipelineResultType.REDIRECT)
if (isRedirectResult(result)) {
expect(result.reason).toBe('Event redirected to async merge topic')
expect(result.output).toBe(ASYNC_OUTPUT)
}
})
it('all modes continue normally when merge succeeds', async () => {
const modes = [
{ type: 'SYNC' as const, batchSize: undefined },
{ type: 'LIMIT' as const, limit: 10 },
{ type: 'ASYNC' as const, topic: 'async-merge-topic', limit: 10 },
]
for (const mode of modes) {
const processor = createPersonEventProcessor(mode, {
event: '$identify',
distinct_id: `test-${mode.type.toLowerCase()}`,
properties: {
$anon_distinct_id: `anon-${mode.type.toLowerCase()}`,
},
})
// Mock successful merge
const mergeService = (processor as any).mergeService as PersonMergeService
const mockPerson = {
id: '123',
uuid: 'test-uuid',
team_id: teamId,
properties: {},
created_at: timestamp,
version: 0,
is_identified: true,
is_user_id: null,
properties_last_updated_at: {},
properties_last_operation: {},
last_seen_at: timestamp.startOf('hour'),
}
jest.spyOn(mergeService, 'handleIdentifyOrAlias').mockResolvedValue({
success: true,
person: mockPerson,
kafkaAck: Promise.resolve(),
needsPersonUpdate: true,
})
const result = await processor.processEvent()
expect(result.type).toBe(PipelineResultType.OK)
if (isOkResult(result)) {
expect(result.value).toEqual(mockPerson)
}
}
})
})
})
})
})