Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(persons): write the identified flag at the call when an identify needs no merge (#117882)

Zach Browne19bf29f6edd
diff --git a/nodejs/src/ingestion/common/persons/person-merge-postgres.test.ts b/nodejs/src/ingestion/common/persons/person-merge-postgres.test.ts
index 0a3355698ddf..7d1374ad2b9b 100644
--- a/nodejs/src/ingestion/common/persons/person-merge-postgres.test.ts
+++ b/nodejs/src/ingestion/common/persons/person-merge-postgres.test.ts
@@ -1,3 +1,5 @@
+import { DateTime } from 'luxon'
+
import { buildIntegerMatcher } from '~/common/config/config'
import { PERSON_DISTINCT_IDS_OUTPUT, PERSON_MERGE_EVENTS_OUTPUT } from '~/common/outputs'
import { UUIDT } from '~/common/utils/utils'
@@ -19,6 +21,23 @@ describe('PostgresPersonMerge merge events', () => {
const sourcePerson = { uuid: '01928aaa-bbbb-cccc-dddd-eeeeeeeeeeee' } as InternalPerson
const targetPerson = { uuid: '01928bbb-cccc-dddd-eeee-ffffffffffff' } as InternalPerson
+ function person(overrides: Partial<InternalPerson> = {}): InternalPerson {
+ return {
+ id: 'p1',
+ uuid: targetPerson.uuid,
+ team_id: 2,
+ properties: {},
+ created_at: DateTime.fromISO('2020-01-01T00:00:00.000Z', { zone: 'utc' }),
+ version: 0,
+ is_identified: false,
+ is_user_id: null,
+ properties_last_updated_at: {},
+ properties_last_operation: {},
+ last_seen_at: null,
+ ...overrides,
+ }
+ }
+
function buildMerge(teamId: number, mergeEvents: MergeEventsConfig): PostgresPersonMerge {
mockOutputs = { produce: jest.fn().mockResolvedValue(undefined) }
const eventUuid = new UUIDT().toString()
@@ -305,6 +324,110 @@ describe('PostgresPersonMerge merge events', () => {
expect(result.survivor).toBe(existingPerson)
})
+ // The outcomes that leave the target row untouched follow the one-exists rule: the committed row decides,
+ // and the flag commits at the call rather than riding the flush where another pod would read it late.
+ it.each([
+ { outcome: 'noop_same_person', locked: true, row: false, writes: true },
+ { outcome: 'noop_same_person', locked: true, row: true, writes: false },
+ { outcome: 'noop_same_person', locked: false, row: false, writes: false },
+ { outcome: 'skipped_already_identified', locked: true, row: false, writes: true },
+ { outcome: 'skipped_already_identified', locked: true, row: true, writes: false },
+ { outcome: 'skipped_already_identified', locked: false, row: false, writes: false },
+ ])(
+ 'a $outcome identify (locked $locked, row identified $row) writes the flag at the call: $writes',
+ async ({ outcome, locked, row, writes }) => {
+ const order: string[] = []
+ mockOutputs = { produce: jest.fn().mockResolvedValue(undefined) }
+ const target = person()
+ const source =
+ outcome === 'noop_same_person'
+ ? target
+ : person({ id: 'p0', uuid: sourcePerson.uuid, is_identified: true })
+ const committedRow = person({ is_identified: row, version: 1 })
+ const writtenRow = person({ is_identified: true, version: 2 })
+ const cachedRow = person({ is_identified: true, version: 2, properties: { pending: true } })
+ const flagMessage = { output: PERSON_DISTINCT_IDS_OUTPUT, value: Buffer.from('{"flag":true}') }
+ const tx = {
+ readMergeRows: jest.fn().mockImplementation(() => {
+ order.push('readMergeRows')
+ return Promise.resolve([committedRow])
+ }),
+ updatePersonForMerge: jest.fn().mockImplementation(() => {
+ order.push('updatePersonForMerge')
+ return Promise.resolve([writtenRow, [flagMessage], true])
+ }),
+ }
+ const store = {
+ fetchForUpdate: jest
+ .fn()
+ .mockImplementation((_teamId: number, distinctId: string) =>
+ Promise.resolve(distinctId === 'd' ? target : source)
+ ),
+ inTransaction: jest
+ .fn()
+ .mockImplementation(async (_description: string, body: (tx: unknown) => Promise<unknown>) => {
+ const result = await body(tx)
+ order.push('commit')
+ return result
+ }),
+ takeMergedRow: jest.fn().mockImplementation(() => {
+ order.push('takeMergedRow')
+ return cachedRow
+ }),
+ }
+ const merge = buildSingleSourceMerge(store, new UUIDT().toString(), {
+ isLockedOutcomeTeam: () => locked,
+ })
+
+ const result = await merge.execute()
+
+ expect(result.results).toEqual([{ sourceDistinctId: 'anon', outcome, sourcePersonUuid: source.uuid }])
+ if (writes) {
+ expect(order).toEqual(['readMergeRows', 'updatePersonForMerge', 'commit', 'takeMergedRow'])
+ expect(tx.updatePersonForMerge).toHaveBeenCalledWith(committedRow, { is_identified: true }, 'd')
+ expect(store.takeMergedRow).toHaveBeenCalledWith(writtenRow, 'd', { is_identified: true }, null, 0)
+ expect(mockOutputs.produce).toHaveBeenCalledWith(
+ PERSON_DISTINCT_IDS_OUTPUT,
+ expect.objectContaining({ value: flagMessage.value })
+ )
+ expect(result.survivor).toBe(cachedRow)
+ } else {
+ expect(tx.readMergeRows).toHaveBeenCalledTimes(locked ? 1 : 0)
+ expect(tx.updatePersonForMerge).not.toHaveBeenCalled()
+ expect(store.takeMergedRow).not.toHaveBeenCalled()
+ expect(result.survivor).toBe(target)
+ }
+ await expect(result.kafkaAck).resolves.toBeUndefined()
+ }
+ )
+
+ it('a same-person identify on a row over the size limit leaves the flag to the flush', async () => {
+ mockOutputs = { produce: jest.fn().mockResolvedValue(undefined) }
+ const target = person()
+ const tx = {
+ readMergeRows: jest.fn().mockResolvedValue([target]),
+ updatePersonForMerge: jest.fn().mockRejectedValue(new MergeOutcomeOversizedError('over the limit')),
+ }
+ const store = {
+ fetchForUpdate: jest.fn().mockResolvedValue(target),
+ inTransaction: jest
+ .fn()
+ .mockImplementation((_description: string, body: (tx: unknown) => Promise<unknown>) => body(tx)),
+ takeMergedRow: jest.fn(),
+ }
+ const merge = buildSingleSourceMerge(store, new UUIDT().toString())
+
+ const result = await merge.execute()
+
+ expect(tx.updatePersonForMerge).toHaveBeenCalledTimes(1)
+ expect(store.takeMergedRow).not.toHaveBeenCalled()
+ expect(mockOutputs.produce).not.toHaveBeenCalled()
+ expect(result.results).toEqual([
+ { sourceDistinctId: 'anon', outcome: 'noop_same_person', sourcePersonUuid: target.uuid },
+ ])
+ expect(result.survivor).toBe(target)
+ })
+
// Same contract as the one-exists case: createPerson returns its messages and the
// branch produces after commit, so the creation transaction never spans Kafka.
it('a neither-exists merge produces the creation messages after the transaction', async () => {
@@ -360,9 +483,13 @@ describe('PostgresPersonMerge merge events', () => {
distinctId: 'anon',
message: { output: PERSON_DISTINCT_IDS_OUTPUT, value: Buffer.from('{"version":1}') },
}
+ const tx = { readMergeRows: jest.fn().mockResolvedValue([{ ...person, is_identified: true }]) }
const store = {
fetchForUpdate: jest.fn().mockResolvedValue(person),
fetchPersonDistinctIdMappings: jest.fn().mockResolvedValue([mapping]),
+ inTransaction: jest
+ .fn()
+ .mockImplementation((_description: string, body: (tx: unknown) => Promise<unknown>) => body(tx)),
}
const debounce = new MergeMappingDebounce(100, 60_000)
@@ -396,6 +523,7 @@ describe('PostgresPersonMerge merge events', () => {
distinctId: 'd',
message: { output: PERSON_DISTINCT_IDS_OUTPUT, value: Buffer.from('{"version":0}') },
}
+ const tx = { readMergeRows: jest.fn().mockResolvedValue([{ ...target, is_identified: true }]) }
const store = {
fetchForUpdate: jest
.fn()
@@ -403,6 +531,9 @@ describe('PostgresPersonMerge merge events', () => {
Promise.resolve(distinctId === 'd' ? target : source)
),
fetchPersonDistinctIdMappings: jest.fn().mockResolvedValue([mapping]),
+ inTransaction: jest
+ .fn()
+ .mockImplementation((_description: string, body: (tx: unknown) => Promise<unknown>) => body(tx)),
}
const replay = await buildSingleSourceMerge(store, new UUIDT().toString(), {
diff --git a/nodejs/src/ingestion/common/persons/person-merge-postgres.ts b/nodejs/src/ingestion/common/persons/person-merge-postgres.ts
index 7d52887f647b..d70dd950e74e 100644
--- a/nodejs/src/ingestion/common/persons/person-merge-postgres.ts
+++ b/nodejs/src/ingestion/common/persons/person-merge-postgres.ts
@@ -366,6 +366,51 @@ export class PostgresPersonMerge {
})
}
+ /**
+ * Writes the identified flag at the call for an outcome that leaves the target row untouched, when the
+ * committed row lacks it: the flush would carry it seconds later, and a merge on another pod that reads the
+ * row before then attaches a second user to it. Returns the survivor as the batch sees it and the produce ack.
+ */
+ private async identifyTarget(target: InternalPerson): Promise<[InternalPerson, Promise<void>]> {
+ if (!this.lockedOutcomeEnabled()) {
+ return [target, Promise.resolve()]
+ }
+ const identified: MergePersonUpdate = { is_identified: true }
+ let committed: AttachOutcome
+ try {
+ committed = await this.inTransaction('mergeDistinctIds-Identify', async (tx) => {
+ const row = (
+ await this.lockMergeRows(
+ tx,
+ target,
+ [],
+ () => new TargetPersonNotFoundError('Person was deleted before the merge could identify it')
+ )
+ ).get(target.id)
+ if (row === undefined || row.is_identified) {
+ return [target, null, []] as const
+ }
+ const [survivor, messages, written] = await tx.updatePersonForMerge(
+ row,
+ identified,
+ this.targetDistinctId
+ )
+ return [survivor, written ? identified : null, messages] as const
+ })
+ } catch (error) {
+ // A row over the properties size limit rejects the write; the flag rides the flush, as for an attach.
+ if (!(error instanceof MergeOutcomeOversizedError)) {
+ throw error
+ }
+ return [target, Promise.resolve()]
+ }
+ const [row, update, messages] = committed
+ const survivor = update
+ ? this.store.takeMergedRow(row, this.targetDistinctId, update, null, this.batchId)
+ : target
+ return [survivor, this.produceMessages(messages)]
+ }
+
/**
* Target wins over sources, earlier sources over later; the event's ops apply on top. Returns the pending
* changes each row's view took, so the entries can be settled against them after the commit.
@@ -587,8 +632,9 @@ export class PostgresPersonMerge {
// Same person already; re-emit in case a crashed prior merge never
// produced the pair's mapping.
const { kafkaAck } = await this.reemitSatisfiedMappings([otherPersonDistinctId, mergeIntoDistinctId])
+ const [survivor, flagAck] = await this.identifyTarget(mergeIntoPerson)
return {
- survivor: mergeIntoPerson,
+ survivor,
results: [
{
sourceDistinctId: otherPersonDistinctId,
@@ -596,7 +642,7 @@ export class PostgresPersonMerge {
sourcePersonUuid: otherPerson.uuid,
},
],
- kafkaAck,
+ kafkaAck: joinAcks(kafkaAck, flagAck),
}
}
@@ -849,8 +895,14 @@ export class PostgresPersonMerge {
if (mergeSources.length === 0 && missingSources.length === 0) {
const { kafkaAck: reemitAck } = await this.reemitSatisfiedMappings(noopSourceDistinctIds)
- const kafkaAck = this.bootstrapAck ? joinAcks(this.bootstrapAck, reemitAck) : reemitAck
- return { survivor: target, results: outcomes, kafkaAck }
+ // An illegal pair identifies nobody; any other outcome names the target as a user.
+ const [survivor, flagAck] = outcomes.some((outcome) => outcome.outcome !== 'skipped_illegal')
+ ? await this.identifyTarget(target)
+ : [target, Promise.resolve()]
+ const kafkaAck = this.bootstrapAck
+ ? joinAcks(this.bootstrapAck, reemitAck, flagAck)
+ : joinAcks(reemitAck, flagAck)
+ return { survivor, results: outcomes, kafkaAck }
}
const currentTarget = target
@@ -1000,8 +1052,9 @@ export class PostgresPersonMerge {
// A replay of a committed createTargetBesideIdentifiedSource lands here, so re-emit
// the target's mapping in case the crash came before its produce.
const { kafkaAck } = await this.reemitSatisfiedMappings([mergeIntoDistinctId])
+ const [survivor, flagAck] = await this.identifyTarget(mergeInto)
return {
- survivor: mergeInto,
+ survivor,
results: [
{
sourceDistinctId: otherPersonDistinctId,
@@ -1009,7 +1062,7 @@ export class PostgresPersonMerge {
sourcePersonUuid: otherPerson.uuid,
},
],
- kafkaAck,
+ kafkaAck: joinAcks(kafkaAck, flagAck),
}
}
diff --git a/nodejs/tests/ingestion/person-state-batch.serial.test.ts b/nodejs/tests/ingestion/person-state-batch.serial.test.ts
index 20481f929c51..e5b1d9418613 100644
--- a/nodejs/tests/ingestion/person-state-batch.serial.test.ts
+++ b/nodejs/tests/ingestion/person-state-batch.serial.test.ts
@@ -1220,7 +1220,7 @@ describe('PersonState.processEvent()', () => {
expect(distinctIds).toEqual(expect.arrayContaining([oldUserDistinctId, newUserDistinctId]))
})
- it(`skips database write when is_identified doesn't change (batch-level filtering)`, async () => {
+ it(`writes the identified flag at the call when both ids already share an unidentified person`, async () => {
await createPerson(
hub,
timestamp,
@@ -1251,13 +1251,14 @@ describe('PersonState.processEvent()', () => {
const context = mergeService.getContext()
await flushPersonStoreToKafka(kafkaProducer, context.personStore, kafkaAcks)
+ // The same-person outcome commits the identified flag at the call, ahead of the batch flush.
expect(person).toMatchObject({
id: expect.any(String),
uuid: newUserUuid,
properties: {},
created_at: timestamp,
- version: 0,
- is_identified: false,
+ version: 1,
+ is_identified: true,
})
expect(mergeService.getUpdateIsIdentified()).toBeTruthy()
@@ -1267,9 +1268,9 @@ describe('PersonState.processEvent()', () => {
id: person!.id,
uuid: newUserUuid,
created_at: timestamp,
- is_identified: false,
+ is_identified: true,
properties: {},
- version: 0,
+ version: 1,
})
})
@@ -1970,14 +1971,15 @@ describe('PersonState.processEvent()', () => {
await kafkaAcks
expect(personS.getUpdateIsIdentified()).toBeTruthy()
+ // The refused merge commits the target's identified flag at the call, ahead of the batch flush.
expect(person).toEqual(
expect.objectContaining({
id: expect.any(String),
uuid: newUserUuid,
properties: {},
created_at: timestamp2,
- version: 0,
- is_identified: false,
+ version: 1,
+ is_identified: true,
})
)