:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { isEqual } from 'lodash'
import { DateTime } from 'luxon'
import { Counter, Histogram } from 'prom-client'
import { PERSON_MERGE_EVENTS_OUTPUT } from '~/common/outputs'
import { personMergeFailureCounter } from '~/common/persons/metrics'
import { PersonMessage } from '~/common/persons/person-message'
import { MergePersonUpdate, PendingPersonChanges } from '~/common/persons/person-update-batch'
import { isDistinctIdUnmergeable } from '~/common/persons/person-utils'
import {
PersonClaimedByLifecycleOpError,
PersonTombstoneBlockedError,
} from '~/common/persons/repositories/person-repository'
import { logger } from '~/common/utils/logger'
import { promiseRetry } from '~/common/utils/retries'
import { Properties } from '~/plugin-scaffold'
import { InternalPerson, ValueMatcher } from '~/types'
import type { BatchWritingPersonsStore } from './batch-writing-person-store'
import { MergeMappingDebounce } from './merge-mapping-debounce'
import { PersonOutputs } from './person-context'
import { PersonCreateService } from './person-create-service'
import { buildPersonMergeEventMessage } from './person-merge-event'
import {
MergeOutcomeOversizedError,
PersonMergeLimitExceededError,
PersonMergeRaceConditionError,
PersonMergeResult,
SourcePersonHasDistinctIdsError,
SourcePersonNotFoundError,
TargetPersonNotFoundError,
mergeError,
mergeSuccess,
} from './person-merge-types'
import { applyEventPropertyUpdates, refineEventOps } from './person-update'
import { lifecycleOpIdFromEvent } from './person-uuid'
import {
MergeFoldAbortReason,
MergePersonsRequest,
MergePersonsResult,
MergePersonsSource,
MergePersonsSourceResult,
} from './persons-store'
import {
BatchBoundPersonsStore,
BatchBoundPersonsStoreTransaction,
PersonsStoreForBatch,
PersonsStoreTransactionForBatch,
} from './persons-store-for-batch'
export const mergeTxnAttemptCounter = new Counter({
name: 'person_merge_txn_attempt_total',
help: 'Number of person merge attempts.',
labelNames: ['call', 'oldPersonIdentified', 'newPersonIdentified'],
})
export const mergeTxnSuccessCounter = new Counter({
name: 'person_merge_txn_success_total',
help: 'Number of person merges that succeeded.',
labelNames: ['call', 'oldPersonIdentified', 'newPersonIdentified'],
})
export const mergeFoldExecutedCounter = new Counter({
name: 'person_merge_fold_executed_total',
help: 'Number of folded merge transactions executed.',
})
export const mergeFoldSizeHistogram = new Histogram({
name: 'person_merge_fold_size',
help: 'Number of merge pairs folded into one transaction.',
buckets: [2, 5, 10, 25, 50, 100, 250, 500],
})
export const mergeDistinctIdOverrideCounter = new Counter({
name: 'person_merge_distinct_id_override_total',
help: 'Distinct id mapping rows written during merges, each writing a ClickHouse override row.',
labelNames: ['call'],
})
export const personMergeEventProducedCounter = new Counter({
name: 'person_merge_event_produced_total',
help: 'Number of person_merge_events messages acked by the broker (gate-on merges only).',
})
export const mergeIdentifiedSourceNewTargetCounter = new Counter({
name: 'person_merge_identified_source_new_target_total',
help: 'Identifies refused because the source is identified while the target had no person, so the merge created one for the target.',
})
export const mergeNoopMappingEmissionCounter = new Counter({
name: 'person_merge_noop_mapping_emission_total',
help: 'Distinct ids considered for mapping re-emission on already-satisfied merges.',
labelNames: ['action'],
})
/** Thrown inside the fold transaction to roll it back when merge-mode move bounds would be exceeded. */
class MergeFoldLimitError extends Error {}
/** Thrown inside the fold transaction to roll it back when a concurrent merge invalidated the fetched sources. */
class MergeFoldConflictError extends Error {}
/** Maps a fold failure to the abort reason the caller's fallback counter records. */
function foldAbortReason(error: unknown): MergeFoldAbortReason {
if (error instanceof MergeFoldLimitError) {
return 'limit'
}
if (
error instanceof MergeFoldConflictError ||
error instanceof PersonTombstoneBlockedError ||
error instanceof PersonClaimedByLifecycleOpError
) {
return 'conflict'
}
if ((error as { code?: string })?.code === '40P01') {
return 'deadlock'
}
return 'error'
}
/** One ack the caller can await for a set of produces already in flight. */
async function joinAcks(...acks: Promise<void>[]): Promise<void> {
await Promise.all(acks)
}
/** Gate + partition-count + team allowlist for the cross-partition merge-event producer. */
export interface MergeEventsConfig {
enabled: boolean
partitionCount: number
/** Matches teams allowed to emit merge events; the pg store builds it from PERSON_MERGE_EVENTS_TEAM_ALLOWLIST. */
isTeamEnabled: ValueMatcher<number>
}
/** The Postgres backend's merge policy, derived from config at store construction. */
export interface PostgresMergePolicy {
/** When true, all property changes trigger person updates; mirrors the store option of the same name. */
updateAllProperties: boolean
/** Teams whose merge locks the person rows and writes the survivor in its transaction; others queue it for the flush. */
isLockedOutcomeTeam: ValueMatcher<number>
mergeEvents: MergeEventsConfig
/**
* When set, already-satisfied merges re-emit the committed mappings (debounced),
* healing ClickHouse rows lost between a prior merge's commit and its produce.
*/
noopMappingDebounce?: MergeMappingDebounce
}
/** A committed attach: the survivor, the flag write when there was one, and the messages to produce. */
type AttachOutcome = readonly [InternalPerson, MergePersonUpdate | null, PersonMessage[]]
/**
* The keys a merge changes. The event's $set keys and keys this pod's pending unset hides go as sets, because the view
* can show a set-once that has not landed and the row can still hold a hidden key. Every other key only fills a gap.
*/
function propertyChanges(
before: Properties,
after: Properties,
eventSets: Properties,
hidden: Set<string>
): { toSet: Properties; toSetOnce: Properties; toUnset: string[] } {
const toSet: Properties = {}
const toSetOnce: Properties = {}
for (const [key, value] of Object.entries(after)) {
// Deep equality: the outcome is a copy, so object values never match by reference.
if (Object.hasOwn(eventSets, key) || !isEqual(before[key], value)) {
const fillsGap = !Object.hasOwn(eventSets, key) && !hidden.has(key)
;(fillsGap ? toSetOnce : toSet)[key] = value
}
}
const toUnset = Object.keys(before).filter((key) => !Object.hasOwn(after, key))
return { toSet, toSetOnce, toUnset }
}
/**
* The Postgres implementation of PersonsStore.mergePersons:
* classification, lifecycle-mark claims, distinct-id moves, cohort
* fixups, and source deletion, run against the pg store's own verbs
* and transactions. One instance serves one request. Settled verdicts
* come back as per-source outcomes; retryable conflicts (lifecycle
* claims, tombstone races, creation races) throw for the caller's
* retry loop, which re-enters with fresh fetches.
*/
export class PostgresPersonMerge {
private batchStore: PersonsStoreForBatch
private createService: PersonCreateService
/**
* The bootstrap's produced messages, when one committed before the fold
* ran. Held on the request rather than inside the fold, because an abort
* unwinds past that scope while the commit it produced for stands.
*/
private bootstrapAck?: Promise<void>
/** The distinct ids the store re-pointed at the target during the attempt, purged when it throws. */
private movedDistinctIds: string[] = []
/** Set when an attempt's outcome was oversized; the rerun commits the merge without it. */
private dropOversizedOutcome = false
constructor(
private store: BatchWritingPersonsStore,
private outputs: PersonOutputs,
private policy: PostgresMergePolicy,
private request: MergePersonsRequest,
private batchId: number
) {
this.batchStore = new BatchBoundPersonsStore(store, batchId)
this.createService = new PersonCreateService(this.batchStore, outputs)
}
private get teamId(): number {
return this.request.teamId
}
/** The processing distinct id stamped on verb calls; merges always process under the target's id. */
private get targetDistinctId(): string {
return this.request.targetDistinctId
}
private get eventName(): string {
return this.request.eventOps.eventName
}
private get timestamp(): DateTime {
return DateTime.fromMillis(this.request.createdAtMs, { zone: 'utc' })
}
private lockedOutcomeEnabled(): boolean {
return this.policy.isLockedOutcomeTeam(this.teamId)
}
async execute(): Promise<MergePersonsResult> {
if (this.request.sources.length === 0) {
throw new Error('mergePersons requires at least one source')
}
const result =
this.request.sources.length > 1
? await this.executeFold()
: await this.mergeSingleWithCachePurge(this.request.sources[0])
this.touchWrittenMappings(result)
return result
}
/** Marks written mappings in the debounce so a following duplicate no-op does not re-emit them. */
private touchWrittenMappings(result: MergePersonsResult): void {
const debounce = this.policy.noopMappingDebounce
if (!debounce) {
return
}
const written = result.results
.filter((source) => source.outcome === 'attached' || source.outcome === 'merged')
.map((source) => source.sourceDistinctId)
if (written.length === 0) {
return
}
debounce.touch(this.teamId, [this.targetDistinctId, ...written])
}
/**
* Runs a single-source merge, purging both request ids and every id it
* moved from the batch caches on any throw: merge verbs update the
* caches optimistically inside a transaction that has now rolled back,
* and a retry must re-read committed state rather than a mapping that
* was never committed.
*/
private async mergeSingleWithCachePurge(source: MergePersonsSource): Promise<MergePersonsResult> {
this.movedDistinctIds = []
try {
return await this.mergeSingle(source)
} catch (error) {
this.store.removeDistinctIdFromCache(this.teamId, this.targetDistinctId)
this.store.removeDistinctIdFromCache(this.teamId, source.distinctId)
this.purgeMovedDistinctIds()
throw error
}
}
private purgeMovedDistinctIds(): void {
for (const distinctId of this.movedDistinctIds) {
this.store.removeDistinctIdFromCache(this.teamId, distinctId)
}
this.movedDistinctIds = []
}
/**
* Locks the target and sources, in ascending id order, and returns their rows; a row that is gone throws the
* caller's error for its role.
*/
private async lockMergeRows(
tx: PersonsStoreTransactionForBatch,
target: InternalPerson,
sources: InternalPerson[],
missing: (role: 'target' | 'source') => Error
): Promise<Map<string, InternalPerson>> {
const rows = await tx.readMergeRows(
target.team_id,
target.id,
sources.map((source) => source.id),
this.targetDistinctId
)
const byId = new Map(rows.map((row) => [row.id, row]))
if (!byId.has(target.id)) {
throw missing('target')
}
if (sources.some((source) => !byId.has(source.id))) {
throw missing('source')
}
return byId
}
/**
* Adds the distinct id to the person. With `writeFlag`, the committed row decides whether the identified flag
* is written with the mapping: the batch's view of the person can show the flag pending while Postgres still
* holds false, and a merge on another pod that reads the row before this batch flushes would attach a second
* user to it.
*/
private async attachDistinctId(
existingPerson: InternalPerson,
distinctIdToAdd: string,
distinctIdsToPurge: [string, string],
writeFlag: boolean
): Promise<AttachOutcome> {
const teamId = this.teamId
const lifecycleOpId = lifecycleOpIdFromEvent(teamId, this.request.eventUuid)
const deleted = (): Error =>
new TargetPersonNotFoundError('Person was deleted before the merge could add a distinct id')
this.discardOverrideCounts()
return await this.inTransaction('mergeDistinctIds-OneExists', async (tx) => {
// The person's lifecycle mark keeps a concurrent tombstone from landing
// between this check and the distinct id insert (an orphaned mapping).
await tx.claimLifecycleMarks(
lifecycleOpId,
existingPerson.team_id,
[{ personId: existingPerson.id, personUuid: existingPerson.uuid, role: 'target' }],
this.targetDistinctId
)
if (!(await tx.isPersonLive(existingPerson, this.targetDistinctId))) {
// Purge the stale cache entries so the caller's retry attempt
// re-fetches from Postgres instead of replaying the cached,
// now-tombstoned person into the same failure.
for (const distinctId of distinctIdsToPurge) {
this.store.removeDistinctIdFromCache(teamId, distinctId)
}
throw deleted()
}
// See the note on `distinctIdVersion` in mergeSingle.
const distinctIdVersion = 1
this.recordOverrideCount('oneExists')
// The row first, before the mapping insert takes a key-share lock on it, as the merge paths order theirs.
const row = writeFlag
? (await this.lockMergeRows(tx, existingPerson, [], deleted)).get(existingPerson.id)
: undefined
const messages = await tx.addDistinctId(existingPerson, distinctIdToAdd, distinctIdVersion)
let survivor = existingPerson
let outcome: MergePersonUpdate | null = null
let survivorMessages: PersonMessage[] = []
if (row !== undefined && !row.is_identified) {
const identified: MergePersonUpdate = { is_identified: true }
let written: boolean
;[survivor, survivorMessages, written] = await tx.updatePersonForMerge(
row,
identified,
this.targetDistinctId
)
outcome = written ? identified : null
}
await tx.releaseLifecycleMarks(lifecycleOpId, teamId, this.targetDistinctId)
return [survivor, outcome, [...messages, ...survivorMessages]] as const
})
}
/**
* Writes the identified flag at the call for an outcome that leaves the target row untouched, when the
* committed row lacks it: the flush would carry it seconds later, and a merge on another pod that reads the
* row before then attaches a second user to it. Returns the survivor as the batch sees it and the produce ack.
*/
private async identifyTarget(target: InternalPerson): Promise<[InternalPerson, Promise<void>]> {
if (!this.lockedOutcomeEnabled()) {
return [target, Promise.resolve()]
}
const identified: MergePersonUpdate = { is_identified: true }
let committed: AttachOutcome
try {
committed = await this.inTransaction('mergeDistinctIds-Identify', async (tx) => {
const row = (
await this.lockMergeRows(
tx,
target,
[],
() => new TargetPersonNotFoundError('Person was deleted before the merge could identify it')
)
).get(target.id)
if (row === undefined || row.is_identified) {
return [target, null, []] as const
}
const [survivor, messages, written] = await tx.updatePersonForMerge(
row,
identified,
this.targetDistinctId
)
return [survivor, written ? identified : null, messages] as const
})
} catch (error) {
// A row over the properties size limit rejects the write; the flag rides the flush, as for an attach.
if (!(error instanceof MergeOutcomeOversizedError)) {
throw error
}
return [target, Promise.resolve()]
}
const [row, update, messages] = committed
const survivor = update
? this.store.takeMergedRow(row, this.targetDistinctId, update, null, this.batchId)
: target
return [survivor, this.produceMessages(messages)]
}
/**
* Target wins over sources, earlier sources over later; the event's ops apply on top. Returns the pending
* changes each row's view took, so the entries can be settled against them after the commit.
*/
private mergeOutcome(
byId: Map<string, InternalPerson>,
target: InternalPerson,
sources: InternalPerson[]
): {
changes: { toSet: Properties; toSetOnce: Properties; toUnset: string[] }
createdAt: DateTime
pending: Map<string, PendingPersonChanges | null>
} {
const pendingById = new Map<string, PendingPersonChanges | null>()
const withPending = (row: InternalPerson): InternalPerson => {
const pending = this.store.pendingChanges(row.team_id, row.id)
pendingById.set(row.id, pending)
if (!pending) {
return row
}
const properties = { ...pending.toSetOnce, ...row.properties, ...pending.toSet }
for (const key of pending.toUnset) {
delete properties[key]
}
return { ...row, properties, created_at: DateTime.min(row.created_at, pending.createdAt) }
}
const targetView = withPending(byId.get(target.id)!)
const sourceViews = sources.map((source) => withPending(byId.get(source.id)!))
const after = this.applyMergePrecedence(targetView, sourceViews)
return {
changes: propertyChanges(
targetView.properties,
after.properties,
this.request.eventOps.set,
new Set(this.store.pendingChanges(target.team_id, target.id)?.toUnset)
),
createdAt: DateTime.min(targetView.created_at, ...sourceViews.map((source) => source.created_at)),
pending: pendingById,
}
}
private applyMergePrecedence(target: InternalPerson, sources: InternalPerson[]): InternalPerson {
const merged: Properties = {}
for (let i = sources.length - 1; i >= 0; i--) {
Object.assign(merged, sources[i].properties)
}
Object.assign(merged, target.properties)
const refined = refineEventOps(this.request.eventOps, merged, this.policy.updateAllProperties)
const [after] = applyEventPropertyUpdates(refined, { ...target, properties: merged })
return after
}
private deferredOutcome(target: InternalPerson, sources: InternalPerson[]): MergePersonUpdate {
return {
created_at: DateTime.min(target.created_at, ...sources.map((source) => source.created_at)),
properties: this.applyMergePrecedence(target, sources).properties,
is_identified: true,
}
}
private inTransaction<T>(
description: string,
body: (tx: PersonsStoreTransactionForBatch) => Promise<T>
): Promise<T> {
return this.store.inTransaction(description, (tx) =>
body(new BatchBoundPersonsStoreTransaction(tx, this.batchId))
)
}
/**
* A crash between a prior merge's commit and its produce loses the mapping message
* for good, because the replayed event lands in a satisfied branch and would
* otherwise produce nothing. Re-emit the committed mappings, debounced.
*/
private async reemitSatisfiedMappings(distinctIds: string[]): Promise<{ kafkaAck: Promise<void> }> {
const debounce = this.policy.noopMappingDebounce
if (!debounce) {
return { kafkaAck: Promise.resolve() }
}
const toEmit = debounce.unseen(this.teamId, distinctIds)
mergeNoopMappingEmissionCounter.labels({ action: 'debounced' }).inc(distinctIds.length - toEmit.length)
if (toEmit.length === 0) {
return { kafkaAck: Promise.resolve() }
}
const mappings = await this.store.fetchPersonDistinctIdMappings(this.teamId, toEmit)
mergeNoopMappingEmissionCounter.labels({ action: 'emitted' }).inc(mappings.length)
if (mappings.length === 0) {
debounce.touch(this.teamId, toEmit)
return { kafkaAck: Promise.resolve() }
}
// Mark only on delivery: a failed read or produce replays the event, and the
// replay must find the ids unmarked to heal them.
const kafkaAck = this.produceMessages(mappings.map((mapping) => mapping.message)).then(() =>
debounce.touch(this.teamId, toEmit)
)
return { kafkaAck }
}
private async produceMessages(messages: PersonMessage[]): Promise<void> {
await Promise.all(
messages.map((msg) =>
this.outputs.produce(msg.output, { value: msg.value, key: null, teamId: this.teamId })
)
)
}
/**
* Best-effort emit of a person_merge_events message for the cohort-stream-processor. No-op when
* the gate is off or the team is outside the allowlist. Never throws: a produce failure is
* logged and dropped, so it can never affect ingestion. Delivery is at-most-once; loss is
* accepted until the delivery-guarantees milestone. The message is explicitly partitioned by
* `(team_id, P_old)` so it reaches the worker holding P_old's state — see `buildPersonMergeEventMessage`.
*/
private async producePersonMergeEvent(sourcePerson: InternalPerson, targetPerson: InternalPerson): Promise<void> {
if (!this.policy.mergeEvents.enabled || !this.policy.mergeEvents.isTeamEnabled(this.teamId)) {
return
}
try {
const { key, partition, value } = buildPersonMergeEventMessage(
this.teamId,
sourcePerson.uuid,
targetPerson.uuid,
Date.now(),
this.policy.mergeEvents.partitionCount
)
await this.outputs.produce(PERSON_MERGE_EVENTS_OUTPUT, {
value,
key,
partition,
teamId: this.teamId,
})
personMergeEventProducedCounter.inc()
} catch (error) {
logger.warn('person_merge_events produce failed, dropping', {
team_id: this.teamId,
source_person_uuid: sourcePerson.uuid,
target_person_uuid: targetPerson.uuid,
error,
})
}
}
private async mergeSingle(source: MergePersonsSource): Promise<MergePersonsResult> {
const otherPersonDistinctId = source.distinctId
const mergeIntoDistinctId = this.targetDistinctId
const teamId = this.teamId
const otherPerson = await this.store.fetchForUpdate(teamId, otherPersonDistinctId, this.batchId)
const mergeIntoPerson = await this.store.fetchForUpdate(teamId, mergeIntoDistinctId, this.batchId)
// A note about the `distinctIdVersion` logic you'll find below:
//
// Overrides are only created for `posthog_persondistinctid` rows with version > 0, see:
// https://github.com/PostHog/posthog/blob/92e17ce307a577c4233d4ab252eebc6c2207a5ee/posthog/models/person/sql.py#L269-L287
//
// With $process_person_profile=false, events can exist in ClickHouse stamped with the
// deterministic implied person uuid even though no `posthog_persondistinctid` or
// `posthog_person` rows exist. So every merge-added mapping for a distinct id without a
// person gets version 1: the override either re-points real personless events or is a
// harmless transient row the squash job deletes within days. The exception is the distinct
// id whose uuid a newly created person is born on — its events already point at the right
// person, so it keeps version 0 and stays out of the overrides join.
if (otherPerson && !mergeIntoPerson && otherPerson.is_identified && !this.request.allowIdentifiedSources) {
// Attaching the new target to an identified source would identify one person as two
// users, so the source keeps its person and the target gets its own, as when both exist.
return await this.createTargetBesideIdentifiedSource(otherPerson, otherPersonDistinctId)
} else if ((otherPerson && !mergeIntoPerson) || (!otherPerson && mergeIntoPerson)) {
// Only one of the two Distinct IDs points at an existing Person
const [existingPerson, distinctIdToAdd] = (() => {
if (otherPerson) {
return [otherPerson!, mergeIntoDistinctId]
} else {
return [mergeIntoPerson!, otherPersonDistinctId]
}
})()
let committed: AttachOutcome
try {
committed = await this.attachDistinctId(
existingPerson,
distinctIdToAdd,
[otherPersonDistinctId, mergeIntoDistinctId],
this.lockedOutcomeEnabled()
)
} catch (error) {
// A row already over the properties size limit rejects the flag write. The mapping still commits
// and the flag rides the flush, as the merge path drops its oversized outcome.
if (!(error instanceof MergeOutcomeOversizedError)) {
throw error
}
committed = await this.attachDistinctId(
existingPerson,
distinctIdToAdd,
[otherPersonDistinctId, mergeIntoDistinctId],
false
)
}
const [survivorRow, survivorUpdate, kafkaMessages] = committed
// After the commit, so a rollback leaves nothing of the attach in the cache: the person's entry takes
// the row, and its pending property changes stay pending since the write carried none.
const survivor = survivorUpdate
? this.store.takeMergedRow(survivorRow, this.targetDistinctId, survivorUpdate, null, this.batchId)
: survivorRow
this.flushOverrideCounts()
// Produce only after the transaction commits: a produce awaited inside the
// transaction holds row locks and lifecycle marks across the Kafka roundtrip.
const kafkaAck = this.produceMessages(kafkaMessages)
return {
survivor,
results: [{ sourceDistinctId: otherPersonDistinctId, outcome: 'attached' }],
kafkaAck,
}
} else if (otherPerson && mergeIntoPerson) {
// Both Distinct IDs point at an existing Person
if (otherPerson.id == mergeIntoPerson.id) {
// Same person already; re-emit in case a crashed prior merge never
// produced the pair's mapping.
const { kafkaAck } = await this.reemitSatisfiedMappings([otherPersonDistinctId, mergeIntoDistinctId])
const [survivor, flagAck] = await this.identifyTarget(mergeIntoPerson)
return {
survivor,
results: [
{
sourceDistinctId: otherPersonDistinctId,
outcome: 'noop_same_person',
sourcePersonUuid: otherPerson.uuid,
},
],
kafkaAck: joinAcks(kafkaAck, flagAck),
}
}
return await this.mergePeople({
mergeInto: mergeIntoPerson,
mergeIntoDistinctId: mergeIntoDistinctId,
otherPerson: otherPerson,
otherPersonDistinctId: otherPersonDistinctId,
})
} else {
// Neither Distinct ID points at an existing Person
const distinctId1 = mergeIntoDistinctId
const distinctId2 = otherPersonDistinctId
this.discardOverrideCounts()
const [person, needsPersonUpdate, kafkaMessages] = await this.inTransaction(
'mergeDistinctIds-NeitherExist',
async (tx) => {
// See comment above about `distinctIdVersion`: the first Distinct ID derives the
// new Person's UUID so it never needs an override; the second always gets one.
const distinctId1Version = 0
const distinctId2Version = 1
this.recordOverrideCount('neitherExist')
const [created, wasCreated, messages, idOwned] = await this.createService.createPerson(
this.timestamp,
this.request.eventOps.set,
this.request.eventOps.setOnce,
teamId,
null,
true,
this.request.eventUuid,
{ distinctId: distinctId1, version: distinctId1Version },
[{ distinctId: distinctId2, version: distinctId2Version }],
tx
)
// A retry can attach only when no other writer owns one of the ids by now.
if (idOwned) {
throw new PersonMergeRaceConditionError(
`person for ${distinctId1} or ${distinctId2} was created concurrently during a merge`
)
}
return [created, !wasCreated && !created.is_identified, messages] as const
}
)
this.flushOverrideCounts()
const kafkaAck = this.produceMessages(kafkaMessages)
return {
survivor: person,
results: [{ sourceDistinctId: otherPersonDistinctId, outcome: 'attached' }],
survivorNeedsUpdate: needsPersonUpdate,
kafkaAck,
}
}
}
private async createTargetBesideIdentifiedSource(
source: InternalPerson,
sourceDistinctId: string
): Promise<MergePersonsResult> {
const teamId = this.teamId
const [person, kafkaMessages] = await this.inTransaction('mergeDistinctIds-IdentifiedSource', async (tx) => {
// The target derives the new person's uuid, so its mapping keeps version 0.
const [created, , messages, idOwned] = await this.createService.createPerson(
this.timestamp,
this.request.eventOps.set,
this.request.eventOps.setOnce,
teamId,
null,
true,
this.request.eventUuid,
{ distinctId: this.targetDistinctId, version: 0 },
[],
tx
)
// Another writer now owns the target; the retry re-reads both ids and classifies against it.
// A uuid held by a person that owns neither id resolves to that holder without mapping the
// target, as in the neither-exists branch.
if (idOwned) {
throw new PersonMergeRaceConditionError(
`person for ${this.targetDistinctId} was created concurrently during a merge`
)
}
return [created, messages] as const
})
mergeIdentifiedSourceNewTargetCounter.inc()
const kafkaAck = this.produceMessages(kafkaMessages)
return {
survivor: person,
results: [{ sourceDistinctId, outcome: 'skipped_already_identified', sourcePersonUuid: source.uuid }],
kafkaAck,
}
}
/**
* Folded-merge execution for a multi-source request: every source
* merges into the target inside one transaction. Any failure rolls
* the fold back untouched and answers foldAborted, so the caller can
* fall back to per-event sequential merges.
*/
private async executeFold(): Promise<MergePersonsResult> {
this.movedDistinctIds = []
try {
return await this.executeFoldInner()
} catch (error) {
const reason = foldAbortReason(error)
// The batch store's caches were updated optimistically inside the
// rolled-back transaction (distinct id → target mappings); purge
// them so the sequential fallback re-reads committed state.
this.store.removeDistinctIdFromCache(this.teamId, this.targetDistinctId)
for (const source of this.request.sources) {
this.store.removeDistinctIdFromCache(this.teamId, source.distinctId)
}
this.purgeMovedDistinctIds()
logger.warn('🤔', 'folded merge failed, falling back to sequential merges', {
team_id: this.teamId,
distinct_id: this.targetDistinctId,
pairs: this.request.sources.length,
reason,
error,
})
// A committed bootstrap's messages still have to reach the
// event's ack; the rollback did not unmake that commit.
return { survivor: null, results: [], foldAborted: reason, kafkaAck: this.bootstrapAck }
}
}
/**
* Counts each folded source's distinct ids inside the transaction and
* aborts the fold (rolling it back) when:
* - a source is missing entirely — it was merged away between the locked
* fetch (whose locks were released at statement end) and the transaction,
* so there is nothing left of it to fold;
* - a source's count exceeds the LIMIT/ASYNC move limit — those events
* need their own per-event DLQ/redirect decision;
* - the total exceeds batched SYNC's per-statement batch size.
* Returns the expected total so the caller can verify the move touched
* exactly that many rows. One cheap indexed GROUP BY that rarely trips.
*/
private async assertFoldSourcesWithinMoveBounds(
tx: PersonsStoreTransactionForBatch,
mergeSources: InternalPerson[]
): Promise<number> {
if (mergeSources.length === 0) {
return 0
}
const mergeMode = this.request.mergeMode
const limit = mergeMode.type === 'SYNC' ? undefined : mergeMode.limit
const batchSize = mergeMode.type === 'SYNC' ? mergeMode.batchSize : undefined
const counts = await tx.countDistinctIdsForPersons(
this.teamId,
mergeSources.map((source) => source.id),
this.targetDistinctId
)
let total = 0
for (const source of mergeSources) {
const count = counts.get(source.id)
if (count === undefined) {
throw new MergeFoldConflictError('folded merge source lost its distinct ids concurrently')
}
if (limit !== undefined && count > limit) {
throw new MergeFoldLimitError('folded merge source exceeds distinct id move limit')
}
total += count
}
if (batchSize !== undefined && total > batchSize) {
throw new MergeFoldLimitError('folded merge move exceeds sync batch size')
}
return total
}
private async executeFoldInner(): Promise<MergePersonsResult> {
const teamId = this.teamId
const outcomes: MergePersonsSourceResult[] = []
let target = await this.store.fetchForUpdate(teamId, this.targetDistinctId, this.batchId)
let sourcesToFold = this.request.sources
if (!target) {
// Cold start: no target person yet. Bootstrap it through the
// sequential path for the triggering event's source (which creates
// the person or attaches the distinct_id), then fold the remaining
// sources in their request order.
const bootstrapSource =
this.request.sources.find((source) => source.distinctId === this.request.triggerSourceDistinctId) ??
this.request.sources[0]
const bootstrap = await promiseRetry(
() => this.mergeSingleWithCachePurge(bootstrapSource),
'merge_distinct_ids'
)
if (!bootstrap.survivor) {
throw new Error('merge fold bootstrap did not produce a target person')
}
target = bootstrap.survivor
outcomes.push(...bootstrap.results)
this.bootstrapAck = bootstrap.kafkaAck
sourcesToFold = this.request.sources.filter((source) => source !== bootstrapSource)
}
const sources = await this.store.fetchPersonsForUpdateByDistinctIds(
teamId,
sourcesToFold.map((source) => source.distinctId),
this.batchId
)
const sourceByDistinctId = new Map(sources.map((source) => [source.distinct_id, source]))
// Partition sources, preserving order for property-merge precedence.
const mergeSources: InternalPerson[] = []
const mergedSourceOutcomes: MergePersonsSourceResult[] = []
const seenSourceIds = new Set<string>([target.id])
const missingSources: MergePersonsSource[] = []
const noopSourceDistinctIds: string[] = []
for (const pair of sourcesToFold) {
if (isDistinctIdUnmergeable(pair.distinctId)) {
outcomes.push({ sourceDistinctId: pair.distinctId, outcome: 'skipped_illegal' })
continue
}
const source = sourceByDistinctId.get(pair.distinctId)
if (!source) {
missingSources.push(pair)
continue
}
if (seenSourceIds.has(source.id)) {
noopSourceDistinctIds.push(pair.distinctId)
outcomes.push({
sourceDistinctId: pair.distinctId,
outcome: 'noop_same_person',
sourcePersonUuid: source.uuid,
})
continue
}
if (source.is_identified && !this.request.allowIdentifiedSources) {
outcomes.push({
sourceDistinctId: pair.distinctId,
outcome: 'skipped_already_identified',
sourcePersonUuid: source.uuid,
})
continue
}
seenSourceIds.add(source.id)
mergeSources.push(source)
mergedSourceOutcomes.push({
sourceDistinctId: pair.distinctId,
outcome: 'merged',
sourcePersonUuid: source.uuid,
})
}
if (mergeSources.length === 0 && missingSources.length === 0) {
const { kafkaAck: reemitAck } = await this.reemitSatisfiedMappings(noopSourceDistinctIds)
// An illegal pair identifies nobody; any other outcome names the target as a user.
const [survivor, flagAck] = outcomes.some((outcome) => outcome.outcome !== 'skipped_illegal')
? await this.identifyTarget(target)
: [target, Promise.resolve()]
const kafkaAck = this.bootstrapAck
? joinAcks(this.bootstrapAck, reemitAck, flagAck)
: joinAcks(reemitAck, flagAck)
return { survivor, results: outcomes, kafkaAck }
}
const currentTarget = target
this.discardOverrideCounts()
const lifecycleOpId = lifecycleOpIdFromEvent(teamId, this.request.eventUuid)
let pendingAtRead = new Map<string, PendingPersonChanges | null>()
const [survivorRow, survivorUpdate, kafkaMessages] = await this.inTransaction(
'mergePeopleFold',
async (tx): Promise<[InternalPerson, MergePersonUpdate | null, PersonMessage[]]> => {
// Folds claim every person, keeping concurrent lifecycle operations
// (other merges, the delete saga) off the targets and sources until commit.
await tx.claimLifecycleMarks(
lifecycleOpId,
currentTarget.team_id,
[
{ personId: currentTarget.id, personUuid: currentTarget.uuid, role: 'target' },
...mergeSources.map((source, index) => ({
personId: source.id,
personUuid: source.uuid,
role: 'source' as const,
ordinal: index,
})),
],
this.targetDistinctId
)
if (!(await tx.isPersonLive(currentTarget, this.targetDistinctId))) {
throw new MergeFoldConflictError('Fold target was deleted concurrently')
}
if (!this.lockedOutcomeEnabled()) {
return await this.deferredOutcomeFold(
tx,
currentTarget,
mergeSources,
missingSources,
lifecycleOpId
)
}
// Before the person rows lock, as in mergePeople.
if (mergeSources.length > 0) {
await tx.updateCohortsAndFeatureFlagsForMergeBatch(
teamId,
mergeSources.map((source) => source.id),
currentTarget.id,
this.targetDistinctId
)
}
// The person rows first, before the moves take key-share locks on them, so two merges that share a
// person cannot deadlock; the locks then span the merge.
const rows =
mergeSources.length > 0
? await this.lockMergeRows(
tx,
currentTarget,
mergeSources,
(role) => new MergeFoldConflictError(`Fold ${role} was deleted concurrently`)
)
: undefined
const moveMessages = await this.moveFoldDistinctIds(tx, currentTarget, mergeSources, missingSources)
let survivor = currentTarget
let outcome: MergePersonUpdate | null = null
let survivorMessages: PersonMessage[] = []
let deleteMessages: PersonMessage[] = []
if (rows) {
// Right before the write, so a change this pod made during the moves counts as pending.
const { changes, createdAt, pending } = this.mergeOutcome(rows, currentTarget, mergeSources)
pendingAtRead = pending
const changesWritten: MergePersonUpdate = {
created_at: createdAt,
properties: changes.toSet,
properties_to_set_once: changes.toSetOnce,
properties_to_unset: changes.toUnset,
is_identified: true,
}
// The survivor's write and the sources' delete commit together, every row locked since the read.
let written: boolean
;[survivor, survivorMessages, written] = await tx.updatePersonForMerge(
currentTarget,
changesWritten,
this.targetDistinctId
)
outcome = written ? changesWritten : null
deleteMessages = await tx.deletePersons(mergeSources, this.targetDistinctId)
}
await tx.releaseLifecycleMarks(lifecycleOpId, teamId, this.targetDistinctId)
return [survivor, outcome, [...moveMessages, ...survivorMessages, ...deleteMessages]]
}
)
// After the commit, so a rollback leaves nothing of the merge in the cache: the survivor's entry takes the
// row, and the sources' entries are released.
const mergedPerson = survivorUpdate
? this.store.takeMergedRow(
survivorRow,
this.targetDistinctId,
survivorUpdate,
pendingAtRead.get(currentTarget.id) ?? null,
this.batchId
)
: survivorRow
for (const source of mergeSources) {
this.store.releaseMergedSource(source.team_id, source.id, pendingAtRead.get(source.id) ?? null)
}
this.flushOverrideCounts()
mergeFoldExecutedCounter.inc()
mergeFoldSizeHistogram.observe(this.request.sources.length)
// The bootstrap's produce, when there was one, joins the fold's own
// ack so the caller observes every message this merge produced.
const foldAck = this.produceMessages(kafkaMessages)
const { kafkaAck: reemitAck } = await this.reemitSatisfiedMappings(noopSourceDistinctIds)
const kafkaAck = this.bootstrapAck
? joinAcks(this.bootstrapAck, foldAck, reemitAck)
: joinAcks(foldAck, reemitAck)
for (const source of mergeSources) {
// Same fire-and-forget contract as executeTransaction.
void this.producePersonMergeEvent(source, mergedPerson).catch(() => {})
}
outcomes.push(...mergedSourceOutcomes)
outcomes.push(
...missingSources.map(
(pair): MergePersonsSourceResult => ({
sourceDistinctId: pair.distinctId,
outcome: 'attached',
})
)
)
return { survivor: mergedPerson, results: outcomes, kafkaAck }
}
private async mergePeople({
mergeInto,
mergeIntoDistinctId,
otherPerson,
otherPersonDistinctId,
}: {
mergeInto: InternalPerson
mergeIntoDistinctId: string
otherPerson: InternalPerson
otherPersonDistinctId: string
}): Promise<MergePersonsResult> {
// $merge_dangerously has no restrictions; $create_alias and $identify
// will not merge a user who's already identified into anyone else.
const mergeAllowed = this.request.allowIdentifiedSources || !otherPerson.is_identified
if (!mergeAllowed) {
// A replay of a committed createTargetBesideIdentifiedSource lands here, so re-emit
// the target's mapping in case the crash came before its produce.
const { kafkaAck } = await this.reemitSatisfiedMappings([mergeIntoDistinctId])
const [survivor, flagAck] = await this.identifyTarget(mergeInto)
return {
survivor,
results: [
{
sourceDistinctId: otherPersonDistinctId,
outcome: 'skipped_already_identified',
sourcePersonUuid: otherPerson.uuid,
},
],
kafkaAck: joinAcks(kafkaAck, flagAck),
}
}
// How the merge works:
// Merging properties:
// on key conflict we use the properties from the person provided as the first argument in identify or alias calls (mergeInto person),
// Note it's important for us to compute this before potentially swapping the persons for personID merging purposes in PoEEmbraceJoin mode
// In case of PoE Embrace the join mode:
// we want to keep using the older person to reduce the number of partitions that need to be updated during squash
// to do that we'll swap otherPerson and mergeInto person (after properties merge computation!)
// additionally update person overrides table in postgres and clickhouse
// TODO: ^
// If the merge fails:
// we'll roll back the transaction and then try from scratch in the origial order of persons provided for property merges
// that guarantees consistency of how properties are processed regardless of persons created_at timestamps and rollout state
// we're calling aliasDeprecated as we need to refresh the persons info completely first
const result = await this.handleMergeTransaction(
mergeInto,
mergeIntoDistinctId,
otherPerson,
otherPersonDistinctId
)
if (result.success) {
return {
survivor: result.person ?? null,
results: [
{
sourceDistinctId: otherPersonDistinctId,
outcome: 'merged',
sourcePersonUuid: otherPerson.uuid,
},
],
kafkaAck: result.kafkaAck,
}
}
// A race that persisted through every retry: another merge kept
// winning the persons. The merge drops; the caller emits the
// race-condition warning and keeps the target as the survivor.
if (result.error instanceof PersonMergeRaceConditionError) {
return {
survivor: mergeInto,
results: [
{
sourceDistinctId: otherPersonDistinctId,
outcome: 'skipped_race',
sourcePersonUuid: otherPerson.uuid,
},
],
}
}
let failedOutcome: MergePersonsSourceResult['outcome']
if (result.error instanceof PersonMergeLimitExceededError) {
failedOutcome = 'skipped_move_limit'
} else if (result.error instanceof SourcePersonHasDistinctIdsError) {
failedOutcome = 'failed_source_has_distinct_ids'
} else if (result.error instanceof TargetPersonNotFoundError) {
failedOutcome = 'failed_target_not_found'
} else if (result.error instanceof SourcePersonNotFoundError) {
failedOutcome = 'failed_source_not_found'
} else {
failedOutcome = 'error'
}
return {
survivor: null,
results: [
{
sourceDistinctId: otherPersonDistinctId,
outcome: failedOutcome,
sourcePersonUuid: otherPerson.uuid,
},
],
}
}
private async executeTransaction(
currentTargetPerson: InternalPerson,
currentSourcePerson: InternalPerson
): Promise<PersonMergeResult> {
try {
mergeTxnAttemptCounter
.labels({
call: this.eventName, // $identify, $create_alias or $merge_dangerously
oldPersonIdentified: String(currentSourcePerson.is_identified),
newPersonIdentified: String(currentTargetPerson.is_identified),
})
.inc()
this.discardOverrideCounts()
const lifecycleOpId = lifecycleOpIdFromEvent(this.teamId, this.request.eventUuid)
let sourcePending: PendingPersonChanges | null = null
let targetPending: PendingPersonChanges | null = null
const [survivorRow, survivorUpdate, kafkaMessages] = await this.inTransaction(
'mergePeople',
async (tx): Promise<[InternalPerson, MergePersonUpdate | null, PersonMessage[]]> => {
// Merges claim both persons in the lifecycle mark table: at
// most one live operation (merge or delete saga) may hold a person, so
// neither can be tombstoned under this transaction. The marks say
// nothing about tombstones committed before the claim, so assert both
// persons are still live while holding them. Liveness checks are
// separate statements because a claim that waited on the mark index
// resumes with a stale snapshot.
await tx.claimLifecycleMarks(
lifecycleOpId,
currentTargetPerson.team_id,
[
{
personId: currentTargetPerson.id,
personUuid: currentTargetPerson.uuid,
role: 'target',
},
{
personId: currentSourcePerson.id,
personUuid: currentSourcePerson.uuid,
role: 'source',
ordinal: 0,
},
],
this.targetDistinctId
)
if (!(await tx.isPersonLive(currentTargetPerson, this.targetDistinctId))) {
throw new TargetPersonNotFoundError('Target person was deleted concurrently')
}
if (!(await tx.isPersonLive(currentSourcePerson, this.targetDistinctId))) {
throw new SourcePersonNotFoundError('Source person was deleted concurrently')
}
if (!this.lockedOutcomeEnabled()) {
return await this.deferredOutcomeTransaction(
tx,
currentTargetPerson,
currentSourcePerson,
lifecycleOpId
)
}
// Before the person rows lock, so the locks do not span this step. It locks no person row: the
// override foreign key is checked at commit, and cohort rows have none.
// TODO: Doesn't this table need to add updates to CH too?
await tx.updateCohortsAndFeatureFlagsForMerge(
currentSourcePerson.team_id,
currentSourcePerson.id,
currentTargetPerson.id,
this.targetDistinctId
)
// The person rows first, before the moves take key-share locks on them, so two merges that
// share a person cannot deadlock; the locks then span the merge.
const rows = await this.lockMergeRows(tx, currentTargetPerson, [currentSourcePerson], (role) =>
role === 'target'
? new TargetPersonNotFoundError('Target person was deleted concurrently')
: new SourcePersonNotFoundError('Source person was deleted concurrently')
)
// Move distinct IDs early to establish ownership of the source person quickly.
// This reduces contention when multiple concurrent merges target the same source,
// as subsequent lookups via distinct ID will fail faster.
const allDistinctIdMessages = await this.moveDistinctIdsBasedOnMode(
tx,
currentSourcePerson,
currentTargetPerson
)
// Right before the write, so a change this pod made during the moves counts as pending.
const { changes, createdAt, pending } = this.mergeOutcome(rows, currentTargetPerson, [
currentSourcePerson,
])
sourcePending = pending.get(currentSourcePerson.id) ?? null
targetPending = pending.get(currentTargetPerson.id) ?? null
const outcome: MergePersonUpdate = {
created_at: createdAt,
properties: changes.toSet,
properties_to_set_once: changes.toSetOnce,
properties_to_unset: changes.toUnset,
is_identified: true,
}
// The survivor's write and the source's delete commit together, both rows locked since the read.
const [survivor, survivorMessages, written] = this.dropOversizedOutcome
? this.store.rejectMergeOutcome(currentTargetPerson, outcome, this.targetDistinctId)
: await tx.updatePersonForMerge(currentTargetPerson, outcome, this.targetDistinctId)
const deletePersonMessages = await tx.deletePerson(currentSourcePerson, this.targetDistinctId)
await tx.releaseLifecycleMarks(lifecycleOpId, this.teamId, this.targetDistinctId)
return [
survivor,
written ? outcome : null,
[...allDistinctIdMessages, ...survivorMessages, ...deletePersonMessages],
]
}
)
// After the commit, so a rollback leaves nothing of the merge in the cache: the survivor's entry takes
// the row, and the source's entry is released.
const mergedPerson = survivorUpdate
? this.store.takeMergedRow(
survivorRow,
this.targetDistinctId,
survivorUpdate,
targetPending,
this.batchId
)
: survivorRow
this.store.releaseMergedSource(currentSourcePerson.team_id, currentSourcePerson.id, sourcePending)
this.flushOverrideCounts()
mergeTxnSuccessCounter
.labels({
call: this.eventName, // $identify, $create_alias or $merge_dangerously
oldPersonIdentified: String(currentSourcePerson.is_identified),
newPersonIdentified: String(currentTargetPerson.is_identified),
})
.inc()
// Fire-and-forget after commit: the person_merge_events emission is detached from the ack
// chain, so a produce failure can never block, replay, or crash ingestion.
// producePersonMergeEvent is best-effort and never throws; the call-site catch is a safety
// net that swallows an escaped rejection so a broken never-throws contract can't reach the
// unhandledRejection handler and stop the service.
const kafkaAck = this.produceMessages(kafkaMessages)
void this.producePersonMergeEvent(currentSourcePerson, mergedPerson).catch(() => {})
return mergeSuccess(mergedPerson, kafkaAck, true)
} catch (error) {
// The transaction rolled back, so the ids its move re-pointed at the target are the source's again.
this.purgeMovedDistinctIds()
// Map exceptions to result types - these will cause transaction rollback
if (error instanceof SourcePersonNotFoundError) {
return mergeError(error)
} else if (error instanceof TargetPersonNotFoundError) {
return mergeError(error)
} else if (error instanceof PersonMergeLimitExceededError) {
return mergeError(error)
} else if (error.code === '23503' || error instanceof PersonTombstoneBlockedError) {
// A concurrent merge added a distinct ID to the source person after we've already
// moved the distinct IDs we knew about, but before the delete executed — surfaced
// as PersonTombstoneBlockedError by the tombstone's live-mapping guard, or as a
// foreign key violation. The retry mechanism will:
// 1. Refresh the source person data to see all distinct IDs (including newly added ones)
// 2. Move ALL distinct IDs to the target person
// 3. Successfully delete the now-empty source person
return mergeError(
new SourcePersonHasDistinctIdsError(
'Cannot delete source person due to concurrent distinct ID additions'
)
)
} else {
// Re-throw unexpected errors
throw error
}
}
}
// Override-counter increments are buffered and flushed only after the enclosing
// transaction commits: merges retry and roll back, and rolled-back rows must not
// inflate a metric used for override-inflow capacity planning.
private pendingOverrideCounts: { call: string; count: number }[] = []
private recordOverrideCount(call: string, count: number = 1): void {
if (count > 0) {
this.pendingOverrideCounts.push({ call, count })
}
}
private discardOverrideCounts(): void {
this.pendingOverrideCounts = []
}
private flushOverrideCounts(): void {
for (const { call, count } of this.pendingOverrideCounts) {
mergeDistinctIdOverrideCounter.labels({ call }).inc(count)
}
this.pendingOverrideCounts = []
}
private async moveDistinctIdsBasedOnMode(
tx: PersonsStoreTransactionForBatch,
currentSourcePerson: InternalPerson,
currentTargetPerson: InternalPerson
): Promise<PersonMessage[]> {
if (this.request.mergeMode.type === 'SYNC') {
if (!this.request.mergeMode.batchSize) {
return await this.moveDistinctIdsWithLimit(tx, currentSourcePerson, currentTargetPerson, undefined)
}
return await this.moveDistinctIdsInBatches(
tx,
currentSourcePerson,
currentTargetPerson,
this.request.mergeMode.batchSize
)
} else {
const limit = this.request.mergeMode.limit
return await this.moveDistinctIdsWithLimit(tx, currentSourcePerson, currentTargetPerson, limit)
}
}
private async moveDistinctIdsInBatches(
tx: PersonsStoreTransactionForBatch,
currentSourcePerson: InternalPerson,
currentTargetPerson: InternalPerson,
batchSize: number
): Promise<PersonMessage[]> {
const allDistinctIdMessages: PersonMessage[] = []
let hasMore = true
let hasProcessedAnyDistinctIds = false
while (hasMore) {
const distinctIdResult = await tx.moveDistinctIds(
currentSourcePerson,
currentTargetPerson,
this.targetDistinctId,
batchSize
)
if (!distinctIdResult.success) {
if (distinctIdResult.error === 'SourceNotFound') {
if (hasProcessedAnyDistinctIds) {
// Source person not found after we've already moved some distinct IDs
// This means we've moved all distinct IDs
hasMore = false
break
} else {
// Source person not found on first attempt - this is a real error
throw new SourcePersonNotFoundError('Source person no longer exists')
}
} else if (distinctIdResult.error === 'TargetNotFound') {
throw new TargetPersonNotFoundError('Target person no longer exists')
}
} else {
allDistinctIdMessages.push(...distinctIdResult.messages)
hasProcessedAnyDistinctIds = true
this.recordOverrideCount('bothExistMove', distinctIdResult.distinctIdsMoved.length)
this.movedDistinctIds.push(...distinctIdResult.distinctIdsMoved)
// Check if we moved fewer than the batch size, indicating we're done
hasMore = distinctIdResult.distinctIdsMoved.length >= batchSize
}
}
return allDistinctIdMessages
}
private async moveDistinctIdsWithLimit(
tx: PersonsStoreTransactionForBatch,
currentSourcePerson: InternalPerson,
currentTargetPerson: InternalPerson,
limit: number | undefined
): Promise<PersonMessage[]> {
// Original behavior for LIMIT mode or SYNC without batchSize
const distinctIdResult = await tx.moveDistinctIds(
currentSourcePerson,
currentTargetPerson,
this.targetDistinctId,
limit
)
if (!distinctIdResult.success) {
if (distinctIdResult.error === 'SourceNotFound') {
throw new SourcePersonNotFoundError('Source person no longer exists')
} else if (distinctIdResult.error === 'TargetNotFound') {
throw new TargetPersonNotFoundError('Target person no longer exists')
}
}
const allDistinctIdMessages = distinctIdResult.success ? distinctIdResult.messages : []
if (distinctIdResult.success) {
this.movedDistinctIds.push(...distinctIdResult.distinctIdsMoved)
}
// If moved count equals the per-call limit, verify if it's a partial move by checking remaining IDs
const movedCount = distinctIdResult.success ? distinctIdResult.distinctIdsMoved.length : 0
const hitLimit = limit ? movedCount >= limit : false
if (hitLimit) {
const remaining = await tx.fetchPersonDistinctIds(currentSourcePerson, this.targetDistinctId, 1)
if (remaining.length > 0) {
personMergeFailureCounter.labels({ call: this.eventName }).inc()
// Drop the event by throwing an error that the pipeline will map to DLQ/no-retry
logger.error('🤔', 'person merge move limit hit', {
team_id: this.teamId,
distinct_id: this.targetDistinctId,
})
throw new PersonMergeLimitExceededError('person_merge_move_limit_hit')
}
}
this.recordOverrideCount('bothExistMove', movedCount)
return allDistinctIdMessages
}
private async moveFoldDistinctIds(
tx: PersonsStoreTransactionForBatch,
target: InternalPerson,
mergeSources: InternalPerson[],
missingSources: MergePersonsSource[]
): Promise<PersonMessage[]> {
const expectedMoveCount = await this.assertFoldSourcesWithinMoveBounds(tx, mergeSources)
const moveResult = await tx.moveDistinctIdsFromPersons(mergeSources, target, this.targetDistinctId)
if (!moveResult.success) {
throw new TargetPersonNotFoundError('Target person no longer exists')
}
// A mismatch means a concurrent merge touched the sources
// between the count and the move; abort so the sequential path
// (whose zero-moved handling retries with fresh persons) takes
// over rather than merging stale source properties.
if (moveResult.distinctIdsMoved.length !== expectedMoveCount) {
throw new MergeFoldConflictError('folded merge moved an unexpected number of distinct ids')
}
this.recordOverrideCount('bothExistMove', moveResult.distinctIdsMoved.length)
this.movedDistinctIds.push(...moveResult.distinctIdsMoved)
const addMessages: PersonMessage[] = []
for (const pair of missingSources) {
// See mergeSingle for the distinctIdVersion logic.
const distinctIdVersion = 1
this.recordOverrideCount('fold')
addMessages.push(...(await tx.addDistinctId(target, pair.distinctId, distinctIdVersion)))
}
return [...moveResult.messages, ...addMessages]
}
/** The merge for a team off the locked-outcome allowlist: the survivor is queued for the flush and no person row is locked. */
private async deferredOutcomeTransaction(
tx: PersonsStoreTransactionForBatch,
target: InternalPerson,
source: InternalPerson,
lifecycleOpId: string
): Promise<[InternalPerson, MergePersonUpdate | null, PersonMessage[]]> {
const allDistinctIdMessages = await this.moveDistinctIdsBasedOnMode(tx, source, target)
const survivor = this.store.deferMergeOutcome(
target,
this.deferredOutcome(target, [source]),
this.targetDistinctId,
this.batchId
)
await tx.updateCohortsAndFeatureFlagsForMerge(source.team_id, source.id, target.id, this.targetDistinctId)
const deletePersonMessages = await tx.deletePerson(source, this.targetDistinctId)
await tx.releaseLifecycleMarks(lifecycleOpId, this.teamId, this.targetDistinctId)
return [survivor, null, [...allDistinctIdMessages, ...deletePersonMessages]]
}
private async deferredOutcomeFold(
tx: PersonsStoreTransactionForBatch,
target: InternalPerson,
mergeSources: InternalPerson[],
missingSources: MergePersonsSource[],
lifecycleOpId: string
): Promise<[InternalPerson, MergePersonUpdate | null, PersonMessage[]]> {
const moveMessages = await this.moveFoldDistinctIds(tx, target, mergeSources, missingSources)
let survivor = target
let deleteMessages: PersonMessage[] = []
if (mergeSources.length > 0) {
survivor = this.store.deferMergeOutcome(
target,
this.deferredOutcome(target, mergeSources),
this.targetDistinctId,
this.batchId
)
await tx.updateCohortsAndFeatureFlagsForMergeBatch(
this.teamId,
mergeSources.map((source) => source.id),
target.id,
this.targetDistinctId
)
deleteMessages = await tx.deletePersons(mergeSources, this.targetDistinctId)
}
await tx.releaseLifecycleMarks(lifecycleOpId, this.teamId, this.targetDistinctId)
return [survivor, null, [...moveMessages, ...deleteMessages]]
}
/**
* An oversized outcome aborts the transaction, so it rolls back, and one rerun commits the merge without the
* outcome, as the flush drops an oversized write. The rerun reads the rows afresh.
*/
private async executeTransactionOrDropOversized(
target: InternalPerson,
source: InternalPerson
): Promise<PersonMergeResult> {
try {
return await this.executeTransaction(target, source)
} catch (error) {
if (!(error instanceof MergeOutcomeOversizedError) || this.dropOversizedOutcome) {
throw error
}
this.dropOversizedOutcome = true
return await this.executeTransaction(target, source)
}
}
private async handleMergeTransaction(
targetPerson: InternalPerson,
targetDistinctId: string,
sourcePerson: InternalPerson,
sourceDistinctId: string,
maxRetries: number = 5
): Promise<PersonMergeResult> {
let currentTargetPerson = targetPerson
let currentSourcePerson = sourcePerson
for (let attempt = 0; attempt <= maxRetries; attempt++) {
const result = await this.executeTransactionOrDropOversized(currentTargetPerson, currentSourcePerson)
if (result.success) {
return result
}
// Handle retryable errors
if (attempt < maxRetries) {
if (
result.error instanceof SourcePersonNotFoundError ||
result.error instanceof SourcePersonHasDistinctIdsError
) {
const refreshedPerson = await this.refreshPersonData(
sourceDistinctId,
currentSourcePerson.id,
attempt,
'source'
)
if (!refreshedPerson) {
// A concurrent merge absorbed the source; re-emit in case its
// produce was lost to a crash.
const { kafkaAck } = await this.reemitSatisfiedMappings([sourceDistinctId, targetDistinctId])
return mergeSuccess(currentTargetPerson, kafkaAck, true)
}
currentSourcePerson = refreshedPerson
continue
} else if (result.error instanceof TargetPersonNotFoundError) {
const refreshedPerson = await this.refreshPersonData(
targetDistinctId,
currentTargetPerson.id,
attempt,
'target'
)
if (!refreshedPerson) {
// Same as the source case above.
const { kafkaAck } = await this.reemitSatisfiedMappings([sourceDistinctId, targetDistinctId])
return mergeSuccess(currentTargetPerson, kafkaAck, true)
}
currentTargetPerson = refreshedPerson
continue
} else {
// Non-retryable error, return the failure result
return result
}
} else {
// Max retries reached, return the failure result
return result
}
}
// This should never be reached, but add fallback for race condition
return mergeError(
new PersonMergeRaceConditionError(
`Failed to merge persons due to concurrent merges, ` +
`source person: ${sourcePerson.id}, target person: ${targetPerson.id}, team: ${this.teamId} ` +
`source distinct id: ${sourceDistinctId}, target distinct id: ${targetDistinctId}`
)
)
}
private async refreshPersonData(
distinctId: string,
currentPersonId: string,
attempt: number,
personType: 'source' | 'target'
): Promise<InternalPerson | null> {
logger.info(`${personType} person not found, retrying with fresh data`, {
[`${personType}PersonId`]: currentPersonId,
teamId: this.teamId,
attempt,
distinctId,
})
// Purge the mapping so the refresh reads committed state instead of the cached person.
this.store.removeDistinctIdFromCache(this.teamId, distinctId)
// Fetch the refreshed person data using the new distinct ID
const refreshedPerson = await this.store.fetchForUpdate(this.teamId, distinctId, this.batchId)
if (!refreshedPerson) {
logger.info(`${personType} person no longer exists after refresh, skipping merge`, {
[`${personType}PersonId`]: currentPersonId,
teamId: this.teamId,
attempt,
})
return null
}
return refreshedPerson
}
}