:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(ingestion): drop ops for a person deleted mid-flush (#129607)
Zach Brownea6fde502a29
diff --git a/nodejs/src/ingestion/common/persons/personhog-persons-store.test.ts b/nodejs/src/ingestion/common/persons/personhog-persons-store.test.tsindex 0fa50c89f6fa..1054cb64badf 100644--- a/nodejs/src/ingestion/common/persons/personhog-persons-store.test.ts+++ b/nodejs/src/ingestion/common/persons/personhog-persons-store.test.ts@@ -1651,22 +1651,22 @@ describe('PersonhogPersonsStore', () => {expect(hasProjection('1:7')).toBe(false)})- it('a person gone from identity fails the flush to redeliver, releasing the id', async () => {+ it('a person gone from identity drops its ops without failing the flush, releasing the id', async () => {const bound = store.forBatch(0)await bound.applyEventOps(person, ops({ $set: { k: 'A' } }), 'd1')expect(edgeOf('1:d1')).toBe('1:7')// The person is deleted rather than merged, so the redirect's- // resolve answers nobody. Redelivery recreates through the- // normal pipeline, as Postgres does; dropping here would lose- // the ops.+ // resolve answers nobody. Postgres drops the update here; a+ // failed flush would restart the processor instead.repository.updatePersonProperties.mockRejectedValue(new NoRowsUpdatedError('deleted') as never)repository.resolvePersonsByDistinctIds.mockResolvedValue([] as never)- await expect(bound.flush()).rejects.toThrow(/resolves to nobody/)+ await bound.flush()- expect((store as any).entries.get('1:7').segments).toHaveLength(1)- // Released, so the redelivered events re-resolve rather than- // folding onto the deleted person again.+ expect(repository.updatePersonProperties).toHaveBeenCalledTimes(1)+ expect((store as any).entries.get('1:7')?.segments ?? []).toHaveLength(0)+ // Released, so later events re-resolve rather than folding onto+ // the deleted person again.expect(edgeOf('1:d1')).toBeUndefined()expect(hasProjection('1:7')).toBe(false)})@@ -2848,11 +2848,12 @@ describe('PersonhogPersonsStore', () => {return [] as never})- await expect(bound.flush()).rejects.toThrow(/resolves to nobody/)+ await bound.flush()+ // The deleted person's owed segment drops; the fold this pass+ // never attempted stays for the next one.const sets = (store as any).entries.get('1:7').segments.map((segment: EventOps) => segment.set)- expect(sets).toContainEqual({ later: 'fold' })- expect(sets).toContainEqual({ b: '2' })+ expect(sets).toEqual([{ later: 'fold' }])})it('a replayed merged verdict purges again and keeps the live lane', async () => {diff --git a/nodejs/src/ingestion/common/persons/personhog-persons-store.ts b/nodejs/src/ingestion/common/persons/personhog-persons-store.tsindex b39339bc5bd1..41b9676141b8 100644--- a/nodejs/src/ingestion/common/persons/personhog-persons-store.ts+++ b/nodejs/src/ingestion/common/persons/personhog-persons-store.ts@@ -1390,8 +1390,9 @@ export class PersonhogPersonsStore implements PersonsStore {while (progress.remaining > 0) {try {if (viaRedirect) {- await this.redirectToSurvivor(entry, progress)- personhogStoreFlushCounter.inc({ outcome: 'redirected' })+ if (await this.redirectToSurvivor(entry, progress)) {+ personhogStoreFlushCounter.inc({ outcome: 'redirected' })+ }break}if (entry.heldSince !== undefined) {@@ -1506,23 +1507,41 @@ export class PersonhogPersonsStore implements PersonsStore {* Re-resolves a vanished person's distinct id and writes to whoever* owns it now, matching the Postgres merged-away recovery. One resolve* is enough: the saga repoints mappings before the leader answers a- * tombstone, and no owner means a genuine delete.+ * tombstone, and no owner means a genuine delete. Returns whether the+ * ops were written; a delete drops them.*/- private async redirectToSurvivor(entry: OpsLaneEntry, progress: { remaining: number }): Promise<void> {+ private async redirectToSurvivor(entry: OpsLaneEntry, progress: { remaining: number }): Promise<boolean> {const generation = this.generationOf(entry.teamId)const [resolved] = await this.resolveDistinctIds('redirect_survivor', [{ teamId: entry.teamId, distinctId: entry.distinctId },])const survivorId = resolved?.person?.idif (survivorId === undefined || survivorId === entry.personId) {- // Released, so the redelivered events re-resolve rather than- // folding onto the corpse again.+ // Released, so later events re-resolve rather than folding onto+ // the corpse again.this.removeDistinctIdFromCache(entry.teamId, entry.distinctId, 'redirect_gone')this.clearPersonCacheForPersonId(`${entry.teamId}:${entry.personId}`, 'redirect_gone')+ }+ if (survivorId === undefined) {+ // The person was deleted after these ops folded. Postgres drops+ // them too; failing the flush would restart the processor and+ // replay every in-flight batch for ops no owner can take.+ personhogStoreFlushCounter.inc({ outcome: 'redirect_deleted' })+ logger.warn('🤔', 'ops dropped: the person was deleted and its distinct id resolves to nobody', {+ team_id: entry.teamId,+ person_id: entry.personId,+ distinct_id: entry.distinctId,+ })+ // Folds that arrived during the redirect stay for a later pass.+ entry.segments.splice(0, progress.remaining)+ progress.remaining = 0+ return false+ }+ if (survivorId === entry.personId) {personhogStoreFlushCounter.inc({ outcome: 'redirect_gone' })throw new CountedRedirectError(`person ${entry.personId} in team ${entry.teamId} vanished and ${entry.distinctId} resolves to ` +- `${survivorId === undefined ? 'nobody' : 'the same person'}; failing the flush to redeliver`+ `the same person; failing the flush to redeliver`)}// The resolve proved where the id belongs, so heal the edge, the@@ -1544,6 +1563,7 @@ export class PersonhogPersonsStore implements PersonsStore {// the partial set a failed write may have landed.this.clearPersonCacheForPersonId(`${entry.teamId}:${survivorId}`, 'redirect_survivor')}+ return true}/**