Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(ingestion): read a create conflict's owner on the merge transaction (#127064)

Eli Reismana8eaa92cc5d2
diff --git a/nodejs/src/common/persons/repositories/person-repository-transaction.ts b/nodejs/src/common/persons/repositories/person-repository-transaction.ts
index 1392946b6bdf..2889af534c15 100644
--- a/nodejs/src/common/persons/repositories/person-repository-transaction.ts
+++ b/nodejs/src/common/persons/repositories/person-repository-transaction.ts
@@ -44,6 +44,9 @@ export interface PersonRepositoryTransaction {
/** The target and sources are row-locked, in ascending id order, until the transaction ends. */
readMergeRows(teamId: number, targetId: string, sourceIds: string[]): Promise<InternalPerson[]>
+ /** The live person that owns the distinct id, read on this transaction without a row lock. */
+ readPerson(teamId: Team['id'], distinctId: string): Promise<InternalPerson | undefined>
+
/** See PersonRepository.updatePersonsBatch; the rows are written under this transaction. */
updatePersonsBatch(personUpdates: PersonUpdate[]): Promise<
Map<
diff --git a/nodejs/src/common/persons/repositories/postgres-person-repository-transaction.ts b/nodejs/src/common/persons/repositories/postgres-person-repository-transaction.ts
index 4773861ebf62..06bcc458124d 100644
--- a/nodejs/src/common/persons/repositories/postgres-person-repository-transaction.ts
+++ b/nodejs/src/common/persons/repositories/postgres-person-repository-transaction.ts
@@ -76,6 +76,10 @@ export class PostgresPersonRepositoryTransaction implements PersonRepositoryTran
return await this.repository.readMergeRows(teamId, targetId, sourceIds, this.transaction)
}
+ async readPerson(teamId: Team['id'], distinctId: string): Promise<InternalPerson | undefined> {
+ return await this.repository.readPerson(teamId, distinctId, this.transaction)
+ }
+
async updatePersonsBatch(personUpdates: PersonUpdate[]): Promise<
Map<
string,
diff --git a/nodejs/src/common/persons/repositories/postgres-person-repository.ts b/nodejs/src/common/persons/repositories/postgres-person-repository.ts
index d066ffcf70a7..ae7e02114df4 100644
--- a/nodejs/src/common/persons/repositories/postgres-person-repository.ts
+++ b/nodejs/src/common/persons/repositories/postgres-person-repository.ts
@@ -258,11 +258,15 @@ export class PostgresPersonRepository
async fetchPerson(
teamId: number,
distinctId: string,
- options: { forUpdate?: boolean; useReadReplica?: boolean; callerTag?: string } = {}
+ options: { forUpdate?: boolean; useReadReplica?: boolean; callerTag?: string } = {},
+ tx?: TransactionClient
): Promise<InternalPerson | undefined> {
if (options.forUpdate && options.useReadReplica) {
throw new Error("can't enable both forUpdate and useReadReplica in db::fetchPerson")
}
+ if (tx && options.useReadReplica) {
+ throw new Error("can't read a transaction on the read replica in db::fetchPerson")
+ }
let queryString = `SELECT
posthog_person.id,
@@ -294,7 +298,7 @@ export class PostgresPersonRepository
const values = [teamId, distinctId]
const { rows } = await this.postgres.query<RawPerson>(
- options.useReadReplica ? PostgresUse.PERSONS_READ : PostgresUse.PERSONS_WRITE,
+ tx ?? (options.useReadReplica ? PostgresUse.PERSONS_READ : PostgresUse.PERSONS_WRITE),
queryString,
values,
queryTag('fetchPerson', options.callerTag)
@@ -316,9 +320,10 @@ export class PostgresPersonRepository
* Pass `tx` only from a path whose transaction is still usable. After a unique violation it
* is not: Postgres aborts the transaction and rejects every later statement on it with
* 25P02, and nothing here opens a savepoint (`postgres.ts` runs plain BEGIN/COMMIT/ROLLBACK).
- * A caller recovering from that error must omit `tx` and read on a pool connection, the way
- * the distinct-ID recovery in person-create-service already does. The holder was committed
- * by another transaction, which is why we conflicted with it, so a pool connection sees it.
+ * A caller recovering from that error must omit `tx` and read on a pool connection. The holder
+ * was committed by another transaction, which is why we conflicted with it, so a pool
+ * connection sees it. A CreationConflict result never follows a unique violation, so the
+ * distinct-ID recovery in person-create-service reads on the still-usable transaction.
*/
private async fetchPersonByUuid(
teamId: number,
@@ -1070,6 +1075,10 @@ export class PostgresPersonRepository
return rows.length > 0
}
+ async readPerson(teamId: number, distinctId: string, tx?: TransactionClient): Promise<InternalPerson | undefined> {
+ return await this.fetchPerson(teamId, distinctId, { callerTag: 'ingestion/person-create-conflict' }, tx)
+ }
+
async readMergeRows(
teamId: number,
targetId: string,
diff --git a/nodejs/src/common/persons/repositories/raw-postgres-person-repository.ts b/nodejs/src/common/persons/repositories/raw-postgres-person-repository.ts
index c955565d0794..acedc6b7ea90 100644
--- a/nodejs/src/common/persons/repositories/raw-postgres-person-repository.ts
+++ b/nodejs/src/common/persons/repositories/raw-postgres-person-repository.ts
@@ -86,6 +86,8 @@ export interface RawPostgresPersonRepository {
/** See PersonRepository.isPersonLive. */
isPersonLive(person: InternalPerson, tx?: TransactionClient): Promise<boolean>
+ readPerson(teamId: Team['id'], distinctId: string, tx?: TransactionClient): Promise<InternalPerson | undefined>
+
/** The target and sources are row-locked, in ascending id order, until the transaction ends. */
readMergeRows(
teamId: number,
diff --git a/nodejs/src/ingestion/common/persons/batch-writing-person-store.test.ts b/nodejs/src/ingestion/common/persons/batch-writing-person-store.test.ts
index c79d079169c9..911a57f64abb 100644
--- a/nodejs/src/ingestion/common/persons/batch-writing-person-store.test.ts
+++ b/nodejs/src/ingestion/common/persons/batch-writing-person-store.test.ts
@@ -178,6 +178,7 @@ describe('BatchWritingPersonStore', () => {
releaseLifecycleMarks: jest.fn().mockResolvedValue(undefined),
isPersonLive: jest.fn().mockResolvedValue(true),
readMergeRows: jest.fn().mockResolvedValue([]),
+ readPerson: jest.fn().mockResolvedValue(undefined),
updatePersonsBatch: jest.fn().mockResolvedValue(new Map()),
addDistinctId: jest.fn().mockResolvedValue([]),
moveDistinctIds: jest.fn().mockResolvedValue({ success: true, messages: [], distinctIdsMoved: [] }),
diff --git a/nodejs/src/ingestion/common/persons/batch-writing-person-store.ts b/nodejs/src/ingestion/common/persons/batch-writing-person-store.ts
index 0ada48e1744a..61cf4ac82376 100644
--- a/nodejs/src/ingestion/common/persons/batch-writing-person-store.ts
+++ b/nodejs/src/ingestion/common/persons/batch-writing-person-store.ts
@@ -80,6 +80,7 @@ type MethodName =
| 'releaseLifecycleMarks'
| 'isPersonLive'
| 'readMergeRows'
+ | 'readPerson'
| 'addDistinctId'
| 'moveDistinctIds'
| 'moveDistinctIdsFromPersons'
@@ -1332,6 +1333,19 @@ export class BatchWritingPersonsStore implements PersonsStore, BatchWritingStore
return await tx.readMergeRows(teamId, targetId, sourceIds)
}
+ /**
+ * Reads on the caller's transaction and leaves the batch caches untouched, so a caller that
+ * commits on the result must not assume it carries this pod's pending changes.
+ */
+ async readPerson(
+ teamId: number,
+ distinctId: string,
+ tx: PersonRepositoryTransaction
+ ): Promise<InternalPerson | undefined> {
+ this.incrementDatabaseOperation('readPerson', distinctId)
+ return await tx.readPerson(teamId, distinctId)
+ }
+
pendingChanges(teamId: number, personId: string): PendingPersonChanges | null {
const cached = this.personCache.getCachedPersonForUpdateByPersonId(teamId, personId)
return cached
diff --git a/nodejs/src/ingestion/common/persons/person-create-service.ts b/nodejs/src/ingestion/common/persons/person-create-service.ts
index 4f4a596a5de9..01153774577e 100644
--- a/nodejs/src/ingestion/common/persons/person-create-service.ts
+++ b/nodejs/src/ingestion/common/persons/person-create-service.ts
@@ -74,7 +74,12 @@ export class PersonCreateService {
// Try to fetch the person that was created concurrently
const allDistinctIds = [primaryDistinctId, ...(extraDistinctIds || [])]
for (const distinctIdInfo of allDistinctIds) {
- const existingPerson = await this.store.fetchForUpdate(teamId, distinctIdInfo.distinctId)
+ // Read on the caller's transaction. A pool read needs a second connection while this
+ // transaction holds one, so a pool full of conflicting transactions waits on itself
+ // forever, holding row locks that block every other writer of these persons.
+ const existingPerson = tx
+ ? ((await tx.readPerson(teamId, distinctIdInfo.distinctId)) ?? null)
+ : await this.store.fetchForUpdate(teamId, distinctIdInfo.distinctId)
if (existingPerson) {
return [existingPerson, false, [], true]
}
diff --git a/nodejs/src/ingestion/common/persons/persons-store-for-batch.ts b/nodejs/src/ingestion/common/persons/persons-store-for-batch.ts
index bb12fc927f18..2378c15d137a 100644
--- a/nodejs/src/ingestion/common/persons/persons-store-for-batch.ts
+++ b/nodejs/src/ingestion/common/persons/persons-store-for-batch.ts
@@ -60,6 +60,9 @@ export interface PersonsStoreTransactionForBatch {
/** The target and sources are row-locked, in ascending id order, until the transaction ends. */
readMergeRows(teamId: number, targetId: string, sourceIds: string[], distinctId: string): Promise<InternalPerson[]>
+ /** The live person that owns the distinct id, read on this transaction without a row lock. */
+ readPerson(teamId: number, distinctId: string): Promise<InternalPerson | undefined>
+
addDistinctId(person: InternalPerson, distinctId: string, version: number): Promise<PersonMessage[]>
moveDistinctIds(
@@ -242,6 +245,10 @@ export class BatchBoundPersonsStoreTransaction implements PersonsStoreTransactio
return this.tx.readMergeRows(teamId, targetId, sourceIds, distinctId)
}
+ readPerson(teamId: number, distinctId: string): Promise<InternalPerson | undefined> {
+ return this.tx.readPerson(teamId, distinctId)
+ }
+
addDistinctId(person: InternalPerson, distinctId: string, version: number): Promise<PersonMessage[]> {
return this.tx.addDistinctId(person, distinctId, version, this.batchId)
}
diff --git a/nodejs/src/ingestion/common/persons/persons-store-transaction.ts b/nodejs/src/ingestion/common/persons/persons-store-transaction.ts
index 911b1be637f6..d6b75f3b1de5 100644
--- a/nodejs/src/ingestion/common/persons/persons-store-transaction.ts
+++ b/nodejs/src/ingestion/common/persons/persons-store-transaction.ts
@@ -109,6 +109,10 @@ export class PersonsStoreTransaction {
return await this.store.readMergeRows(teamId, targetId, sourceIds, distinctId, this.tx)
}
+ async readPerson(teamId: number, distinctId: string): Promise<InternalPerson | undefined> {
+ return await this.store.readPerson(teamId, distinctId, this.tx)
+ }
+
async addDistinctId(
person: InternalPerson,
distinctId: string,
diff --git a/nodejs/tests/ingestion/person-state-batch.serial.test.ts b/nodejs/tests/ingestion/person-state-batch.serial.test.ts
index e5b1d9418613..57ad2783f0d4 100644
--- a/nodejs/tests/ingestion/person-state-batch.serial.test.ts
+++ b/nodejs/tests/ingestion/person-state-batch.serial.test.ts
@@ -3,6 +3,7 @@ import { KafkaProducerObserver } from '~/tests/helpers/mocks/producer.spy'
import { DateTime } from 'luxon'
+import { defaultConfig } from '~/common/config/config'
import {
KAFKA_INGESTION_WARNINGS,
KAFKA_PERSON,
@@ -21,7 +22,7 @@ import { PostgresPersonRepository } from '~/common/persons/repositories/postgres
import { fetchDistinctIdValues } from '~/common/persons/repositories/test-helpers'
import { DependencyUnavailableError } from '~/common/utils/db/error'
import { closeHub, createHub } from '~/common/utils/db/hub'
-import { PostgresUse } from '~/common/utils/db/postgres'
+import { PostgresRouter, PostgresUse } from '~/common/utils/db/postgres'
import { parseJSON } from '~/common/utils/json-parse'
import { defaultRetryConfig } from '~/common/utils/retries'
import { UUIDT } from '~/common/utils/utils'
@@ -2750,37 +2751,58 @@ describe('PersonState.processEvent()', () => {
expect(persons[0]).toMatchObject({ id: target.id, properties: { a: 1, b: 2, late: 'yes' } })
})
- it(`joins both ids when another writer creates the anon person during a neither-exists merge`, async () => {
- // The anon person lands on another pod before this merge's two-id create.
- let raced = false
- jest.spyOn(personRepository, 'createPerson').mockImplementation(async (...args) => {
- if (!raced) {
- raced = true
- await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, newUserUuid, {
- distinctId: newUserDistinctId,
+ it.each([
+ { pool: 'the shared persons pool', poolSize: undefined },
+ // The merge's transaction holds the only connection, so a conflict re-read on the pool never starts.
+ { pool: 'a one-connection persons pool', poolSize: 1 },
+ ])(
+ `joins both ids when another writer creates the anon person during a neither-exists merge on $pool`,
+ async ({ poolSize }) => {
+ const mergePostgres = poolSize
+ ? new PostgresRouter({ ...defaultConfig, POSTGRES_CONNECTION_POOL_SIZE: poolSize })
+ : hub.postgres
+ const mergeRepository = poolSize ? new PostgresPersonRepository(mergePostgres) : personRepository
+ try {
+ // The anon person lands on another pod before this merge's two-id create.
+ let raced = false
+ jest.spyOn(mergeRepository, 'createPerson').mockImplementation(async (...args) => {
+ if (!raced) {
+ raced = true
+ await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, newUserUuid, {
+ distinctId: newUserDistinctId,
+ })
+ }
+ return await PostgresPersonRepository.prototype.createPerson.apply(mergeRepository, args)
})
- }
- return await PostgresPersonRepository.prototype.createPerson.apply(personRepository, args)
- })
- const mergeService = personMergeService({
- event: '$identify',
- distinct_id: oldUserDistinctId,
- properties: { $anon_distinct_id: newUserDistinctId },
- })
- const result = await mergeService.handleIdentifyOrAlias()
- expect(result.success).toBe(true)
- if (!result.success) {
- throw new Error('Expected successful merge result')
- }
- await flushPersonStoreToKafka(kafkaProducer, mergeService.getContext().personStore, result.kafkaAck)
+ const mergeService = personMergeService(
+ {
+ event: '$identify',
+ distinct_id: oldUserDistinctId,
+ properties: { $anon_distinct_id: newUserDistinctId },
+ },
+ undefined,
+ mergeRepository
+ )
+ const result = await mergeService.handleIdentifyOrAlias()
+ expect(result.success).toBe(true)
+ if (!result.success) {
+ throw new Error('Expected successful merge result')
+ }
+ await flushPersonStoreToKafka(kafkaProducer, mergeService.getContext().personStore, result.kafkaAck)
- const persons = sortPersons(await fetchPostgresPersonsH())
- expect(persons.length).toEqual(1)
- expect(persons[0].uuid).toEqual(newUserUuid)
- const distinctIds = await fetchDistinctIdValues(hub.postgres, persons[0])
- expect(distinctIds).toEqual(expect.arrayContaining([oldUserDistinctId, newUserDistinctId]))
- })
+ const persons = sortPersons(await fetchPostgresPersonsH())
+ expect(persons.length).toEqual(1)
+ expect(persons[0].uuid).toEqual(newUserUuid)
+ const distinctIds = await fetchDistinctIdValues(hub.postgres, persons[0])
+ expect(distinctIds).toEqual(expect.arrayContaining([oldUserDistinctId, newUserDistinctId]))
+ } finally {
+ if (poolSize) {
+ await mergePostgres.end()
+ }
+ }
+ }
+ )
it(`does not retry a neither-exists merge when the uuid's holder owns neither id`, async () => {
// A live person holds the target's uuid under a third id, so no retry can attach.