:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(ingestion): read a create conflict's owner on the merge transaction (#127064)
Eli Reismana8eaa92cc5d2
diff --git a/nodejs/src/common/persons/repositories/person-repository-transaction.ts b/nodejs/src/common/persons/repositories/person-repository-transaction.tsindex 1392946b6bdf..2889af534c15 100644--- a/nodejs/src/common/persons/repositories/person-repository-transaction.ts+++ b/nodejs/src/common/persons/repositories/person-repository-transaction.ts@@ -44,6 +44,9 @@ export interface PersonRepositoryTransaction {/** The target and sources are row-locked, in ascending id order, until the transaction ends. */readMergeRows(teamId: number, targetId: string, sourceIds: string[]): Promise<InternalPerson[]>+ /** The live person that owns the distinct id, read on this transaction without a row lock. */+ readPerson(teamId: Team['id'], distinctId: string): Promise<InternalPerson | undefined>+/** See PersonRepository.updatePersonsBatch; the rows are written under this transaction. */updatePersonsBatch(personUpdates: PersonUpdate[]): Promise<Map<diff --git a/nodejs/src/common/persons/repositories/postgres-person-repository-transaction.ts b/nodejs/src/common/persons/repositories/postgres-person-repository-transaction.tsindex 4773861ebf62..06bcc458124d 100644--- a/nodejs/src/common/persons/repositories/postgres-person-repository-transaction.ts+++ b/nodejs/src/common/persons/repositories/postgres-person-repository-transaction.ts@@ -76,6 +76,10 @@ export class PostgresPersonRepositoryTransaction implements PersonRepositoryTranreturn await this.repository.readMergeRows(teamId, targetId, sourceIds, this.transaction)}+ async readPerson(teamId: Team['id'], distinctId: string): Promise<InternalPerson | undefined> {+ return await this.repository.readPerson(teamId, distinctId, this.transaction)+ }+async updatePersonsBatch(personUpdates: PersonUpdate[]): Promise<Map<string,diff --git a/nodejs/src/common/persons/repositories/postgres-person-repository.ts b/nodejs/src/common/persons/repositories/postgres-person-repository.tsindex d066ffcf70a7..ae7e02114df4 100644--- a/nodejs/src/common/persons/repositories/postgres-person-repository.ts+++ b/nodejs/src/common/persons/repositories/postgres-person-repository.ts@@ -258,11 +258,15 @@ export class PostgresPersonRepositoryasync fetchPerson(teamId: number,distinctId: string,- options: { forUpdate?: boolean; useReadReplica?: boolean; callerTag?: string } = {}+ options: { forUpdate?: boolean; useReadReplica?: boolean; callerTag?: string } = {},+ tx?: TransactionClient): Promise<InternalPerson | undefined> {if (options.forUpdate && options.useReadReplica) {throw new Error("can't enable both forUpdate and useReadReplica in db::fetchPerson")}+ if (tx && options.useReadReplica) {+ throw new Error("can't read a transaction on the read replica in db::fetchPerson")+ }let queryString = `SELECTposthog_person.id,@@ -294,7 +298,7 @@ export class PostgresPersonRepositoryconst values = [teamId, distinctId]const { rows } = await this.postgres.query<RawPerson>(- options.useReadReplica ? PostgresUse.PERSONS_READ : PostgresUse.PERSONS_WRITE,+ tx ?? (options.useReadReplica ? PostgresUse.PERSONS_READ : PostgresUse.PERSONS_WRITE),queryString,values,queryTag('fetchPerson', options.callerTag)@@ -316,9 +320,10 @@ export class PostgresPersonRepository* Pass `tx` only from a path whose transaction is still usable. After a unique violation it* is not: Postgres aborts the transaction and rejects every later statement on it with* 25P02, and nothing here opens a savepoint (`postgres.ts` runs plain BEGIN/COMMIT/ROLLBACK).- * A caller recovering from that error must omit `tx` and read on a pool connection, the way- * the distinct-ID recovery in person-create-service already does. The holder was committed- * by another transaction, which is why we conflicted with it, so a pool connection sees it.+ * A caller recovering from that error must omit `tx` and read on a pool connection. The holder+ * was committed by another transaction, which is why we conflicted with it, so a pool+ * connection sees it. A CreationConflict result never follows a unique violation, so the+ * distinct-ID recovery in person-create-service reads on the still-usable transaction.*/private async fetchPersonByUuid(teamId: number,@@ -1070,6 +1075,10 @@ export class PostgresPersonRepositoryreturn rows.length > 0}+ async readPerson(teamId: number, distinctId: string, tx?: TransactionClient): Promise<InternalPerson | undefined> {+ return await this.fetchPerson(teamId, distinctId, { callerTag: 'ingestion/person-create-conflict' }, tx)+ }+async readMergeRows(teamId: number,targetId: string,diff --git a/nodejs/src/common/persons/repositories/raw-postgres-person-repository.ts b/nodejs/src/common/persons/repositories/raw-postgres-person-repository.tsindex c955565d0794..acedc6b7ea90 100644--- a/nodejs/src/common/persons/repositories/raw-postgres-person-repository.ts+++ b/nodejs/src/common/persons/repositories/raw-postgres-person-repository.ts@@ -86,6 +86,8 @@ export interface RawPostgresPersonRepository {/** See PersonRepository.isPersonLive. */isPersonLive(person: InternalPerson, tx?: TransactionClient): Promise<boolean>+ readPerson(teamId: Team['id'], distinctId: string, tx?: TransactionClient): Promise<InternalPerson | undefined>+/** The target and sources are row-locked, in ascending id order, until the transaction ends. */readMergeRows(teamId: number,diff --git a/nodejs/src/ingestion/common/persons/batch-writing-person-store.test.ts b/nodejs/src/ingestion/common/persons/batch-writing-person-store.test.tsindex c79d079169c9..911a57f64abb 100644--- a/nodejs/src/ingestion/common/persons/batch-writing-person-store.test.ts+++ b/nodejs/src/ingestion/common/persons/batch-writing-person-store.test.ts@@ -178,6 +178,7 @@ describe('BatchWritingPersonStore', () => {releaseLifecycleMarks: jest.fn().mockResolvedValue(undefined),isPersonLive: jest.fn().mockResolvedValue(true),readMergeRows: jest.fn().mockResolvedValue([]),+ readPerson: jest.fn().mockResolvedValue(undefined),updatePersonsBatch: jest.fn().mockResolvedValue(new Map()),addDistinctId: jest.fn().mockResolvedValue([]),moveDistinctIds: jest.fn().mockResolvedValue({ success: true, messages: [], distinctIdsMoved: [] }),diff --git a/nodejs/src/ingestion/common/persons/batch-writing-person-store.ts b/nodejs/src/ingestion/common/persons/batch-writing-person-store.tsindex 0ada48e1744a..61cf4ac82376 100644--- a/nodejs/src/ingestion/common/persons/batch-writing-person-store.ts+++ b/nodejs/src/ingestion/common/persons/batch-writing-person-store.ts@@ -80,6 +80,7 @@ type MethodName =| 'releaseLifecycleMarks'| 'isPersonLive'| 'readMergeRows'+ | 'readPerson'| 'addDistinctId'| 'moveDistinctIds'| 'moveDistinctIdsFromPersons'@@ -1332,6 +1333,19 @@ export class BatchWritingPersonsStore implements PersonsStore, BatchWritingStorereturn await tx.readMergeRows(teamId, targetId, sourceIds)}+ /**+ * Reads on the caller's transaction and leaves the batch caches untouched, so a caller that+ * commits on the result must not assume it carries this pod's pending changes.+ */+ async readPerson(+ teamId: number,+ distinctId: string,+ tx: PersonRepositoryTransaction+ ): Promise<InternalPerson | undefined> {+ this.incrementDatabaseOperation('readPerson', distinctId)+ return await tx.readPerson(teamId, distinctId)+ }+pendingChanges(teamId: number, personId: string): PendingPersonChanges | null {const cached = this.personCache.getCachedPersonForUpdateByPersonId(teamId, personId)return cacheddiff --git a/nodejs/src/ingestion/common/persons/person-create-service.ts b/nodejs/src/ingestion/common/persons/person-create-service.tsindex 4f4a596a5de9..01153774577e 100644--- a/nodejs/src/ingestion/common/persons/person-create-service.ts+++ b/nodejs/src/ingestion/common/persons/person-create-service.ts@@ -74,7 +74,12 @@ export class PersonCreateService {// Try to fetch the person that was created concurrentlyconst allDistinctIds = [primaryDistinctId, ...(extraDistinctIds || [])]for (const distinctIdInfo of allDistinctIds) {- const existingPerson = await this.store.fetchForUpdate(teamId, distinctIdInfo.distinctId)+ // Read on the caller's transaction. A pool read needs a second connection while this+ // transaction holds one, so a pool full of conflicting transactions waits on itself+ // forever, holding row locks that block every other writer of these persons.+ const existingPerson = tx+ ? ((await tx.readPerson(teamId, distinctIdInfo.distinctId)) ?? null)+ : await this.store.fetchForUpdate(teamId, distinctIdInfo.distinctId)if (existingPerson) {return [existingPerson, false, [], true]}diff --git a/nodejs/src/ingestion/common/persons/persons-store-for-batch.ts b/nodejs/src/ingestion/common/persons/persons-store-for-batch.tsindex bb12fc927f18..2378c15d137a 100644--- a/nodejs/src/ingestion/common/persons/persons-store-for-batch.ts+++ b/nodejs/src/ingestion/common/persons/persons-store-for-batch.ts@@ -60,6 +60,9 @@ export interface PersonsStoreTransactionForBatch {/** The target and sources are row-locked, in ascending id order, until the transaction ends. */readMergeRows(teamId: number, targetId: string, sourceIds: string[], distinctId: string): Promise<InternalPerson[]>+ /** The live person that owns the distinct id, read on this transaction without a row lock. */+ readPerson(teamId: number, distinctId: string): Promise<InternalPerson | undefined>+addDistinctId(person: InternalPerson, distinctId: string, version: number): Promise<PersonMessage[]>moveDistinctIds(@@ -242,6 +245,10 @@ export class BatchBoundPersonsStoreTransaction implements PersonsStoreTransactioreturn this.tx.readMergeRows(teamId, targetId, sourceIds, distinctId)}+ readPerson(teamId: number, distinctId: string): Promise<InternalPerson | undefined> {+ return this.tx.readPerson(teamId, distinctId)+ }+addDistinctId(person: InternalPerson, distinctId: string, version: number): Promise<PersonMessage[]> {return this.tx.addDistinctId(person, distinctId, version, this.batchId)}diff --git a/nodejs/src/ingestion/common/persons/persons-store-transaction.ts b/nodejs/src/ingestion/common/persons/persons-store-transaction.tsindex 911b1be637f6..d6b75f3b1de5 100644--- a/nodejs/src/ingestion/common/persons/persons-store-transaction.ts+++ b/nodejs/src/ingestion/common/persons/persons-store-transaction.ts@@ -109,6 +109,10 @@ export class PersonsStoreTransaction {return await this.store.readMergeRows(teamId, targetId, sourceIds, distinctId, this.tx)}+ async readPerson(teamId: number, distinctId: string): Promise<InternalPerson | undefined> {+ return await this.store.readPerson(teamId, distinctId, this.tx)+ }+async addDistinctId(person: InternalPerson,distinctId: string,diff --git a/nodejs/tests/ingestion/person-state-batch.serial.test.ts b/nodejs/tests/ingestion/person-state-batch.serial.test.tsindex e5b1d9418613..57ad2783f0d4 100644--- a/nodejs/tests/ingestion/person-state-batch.serial.test.ts+++ b/nodejs/tests/ingestion/person-state-batch.serial.test.ts@@ -3,6 +3,7 @@ import { KafkaProducerObserver } from '~/tests/helpers/mocks/producer.spy'import { DateTime } from 'luxon'+import { defaultConfig } from '~/common/config/config'import {KAFKA_INGESTION_WARNINGS,KAFKA_PERSON,@@ -21,7 +22,7 @@ import { PostgresPersonRepository } from '~/common/persons/repositories/postgresimport { fetchDistinctIdValues } from '~/common/persons/repositories/test-helpers'import { DependencyUnavailableError } from '~/common/utils/db/error'import { closeHub, createHub } from '~/common/utils/db/hub'-import { PostgresUse } from '~/common/utils/db/postgres'+import { PostgresRouter, PostgresUse } from '~/common/utils/db/postgres'import { parseJSON } from '~/common/utils/json-parse'import { defaultRetryConfig } from '~/common/utils/retries'import { UUIDT } from '~/common/utils/utils'@@ -2750,37 +2751,58 @@ describe('PersonState.processEvent()', () => {expect(persons[0]).toMatchObject({ id: target.id, properties: { a: 1, b: 2, late: 'yes' } })})- it(`joins both ids when another writer creates the anon person during a neither-exists merge`, async () => {- // The anon person lands on another pod before this merge's two-id create.- let raced = false- jest.spyOn(personRepository, 'createPerson').mockImplementation(async (...args) => {- if (!raced) {- raced = true- await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, newUserUuid, {- distinctId: newUserDistinctId,+ it.each([+ { pool: 'the shared persons pool', poolSize: undefined },+ // The merge's transaction holds the only connection, so a conflict re-read on the pool never starts.+ { pool: 'a one-connection persons pool', poolSize: 1 },+ ])(+ `joins both ids when another writer creates the anon person during a neither-exists merge on $pool`,+ async ({ poolSize }) => {+ const mergePostgres = poolSize+ ? new PostgresRouter({ ...defaultConfig, POSTGRES_CONNECTION_POOL_SIZE: poolSize })+ : hub.postgres+ const mergeRepository = poolSize ? new PostgresPersonRepository(mergePostgres) : personRepository+ try {+ // The anon person lands on another pod before this merge's two-id create.+ let raced = false+ jest.spyOn(mergeRepository, 'createPerson').mockImplementation(async (...args) => {+ if (!raced) {+ raced = true+ await createPerson(hub, timestamp, {}, {}, {}, teamId, null, false, newUserUuid, {+ distinctId: newUserDistinctId,+ })+ }+ return await PostgresPersonRepository.prototype.createPerson.apply(mergeRepository, args)})- }- return await PostgresPersonRepository.prototype.createPerson.apply(personRepository, args)- })- const mergeService = personMergeService({- event: '$identify',- distinct_id: oldUserDistinctId,- properties: { $anon_distinct_id: newUserDistinctId },- })- const result = await mergeService.handleIdentifyOrAlias()- expect(result.success).toBe(true)- if (!result.success) {- throw new Error('Expected successful merge result')- }- await flushPersonStoreToKafka(kafkaProducer, mergeService.getContext().personStore, result.kafkaAck)+ const mergeService = personMergeService(+ {+ event: '$identify',+ distinct_id: oldUserDistinctId,+ properties: { $anon_distinct_id: newUserDistinctId },+ },+ undefined,+ mergeRepository+ )+ const result = await mergeService.handleIdentifyOrAlias()+ expect(result.success).toBe(true)+ if (!result.success) {+ throw new Error('Expected successful merge result')+ }+ await flushPersonStoreToKafka(kafkaProducer, mergeService.getContext().personStore, result.kafkaAck)- const persons = sortPersons(await fetchPostgresPersonsH())- expect(persons.length).toEqual(1)- expect(persons[0].uuid).toEqual(newUserUuid)- const distinctIds = await fetchDistinctIdValues(hub.postgres, persons[0])- expect(distinctIds).toEqual(expect.arrayContaining([oldUserDistinctId, newUserDistinctId]))- })+ const persons = sortPersons(await fetchPostgresPersonsH())+ expect(persons.length).toEqual(1)+ expect(persons[0].uuid).toEqual(newUserUuid)+ const distinctIds = await fetchDistinctIdValues(hub.postgres, persons[0])+ expect(distinctIds).toEqual(expect.arrayContaining([oldUserDistinctId, newUserDistinctId]))+ } finally {+ if (poolSize) {+ await mergePostgres.end()+ }+ }+ }+ )it(`does not retry a neither-exists merge when the uuid's holder owns neither id`, async () => {// A live person holds the target's uuid under a third id, so no retry can attach.