Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(ingestion): drop ops for a person deleted mid-flush (#129607)

Zach Brownea6fde502a29
diff --git a/nodejs/src/ingestion/common/persons/personhog-persons-store.test.ts b/nodejs/src/ingestion/common/persons/personhog-persons-store.test.ts
index 0fa50c89f6fa..1054cb64badf 100644
--- a/nodejs/src/ingestion/common/persons/personhog-persons-store.test.ts
+++ b/nodejs/src/ingestion/common/persons/personhog-persons-store.test.ts
@@ -1651,22 +1651,22 @@ describe('PersonhogPersonsStore', () => {
expect(hasProjection('1:7')).toBe(false)
})
- it('a person gone from identity fails the flush to redeliver, releasing the id', async () => {
+ it('a person gone from identity drops its ops without failing the flush, releasing the id', async () => {
const bound = store.forBatch(0)
await bound.applyEventOps(person, ops({ $set: { k: 'A' } }), 'd1')
expect(edgeOf('1:d1')).toBe('1:7')
// The person is deleted rather than merged, so the redirect's
- // resolve answers nobody. Redelivery recreates through the
- // normal pipeline, as Postgres does; dropping here would lose
- // the ops.
+ // resolve answers nobody. Postgres drops the update here; a
+ // failed flush would restart the processor instead.
repository.updatePersonProperties.mockRejectedValue(new NoRowsUpdatedError('deleted') as never)
repository.resolvePersonsByDistinctIds.mockResolvedValue([] as never)
- await expect(bound.flush()).rejects.toThrow(/resolves to nobody/)
+ await bound.flush()
- expect((store as any).entries.get('1:7').segments).toHaveLength(1)
- // Released, so the redelivered events re-resolve rather than
- // folding onto the deleted person again.
+ expect(repository.updatePersonProperties).toHaveBeenCalledTimes(1)
+ expect((store as any).entries.get('1:7')?.segments ?? []).toHaveLength(0)
+ // Released, so later events re-resolve rather than folding onto
+ // the deleted person again.
expect(edgeOf('1:d1')).toBeUndefined()
expect(hasProjection('1:7')).toBe(false)
})
@@ -2848,11 +2848,12 @@ describe('PersonhogPersonsStore', () => {
return [] as never
})
- await expect(bound.flush()).rejects.toThrow(/resolves to nobody/)
+ await bound.flush()
+ // The deleted person's owed segment drops; the fold this pass
+ // never attempted stays for the next one.
const sets = (store as any).entries.get('1:7').segments.map((segment: EventOps) => segment.set)
- expect(sets).toContainEqual({ later: 'fold' })
- expect(sets).toContainEqual({ b: '2' })
+ expect(sets).toEqual([{ later: 'fold' }])
})
it('a replayed merged verdict purges again and keeps the live lane', async () => {
diff --git a/nodejs/src/ingestion/common/persons/personhog-persons-store.ts b/nodejs/src/ingestion/common/persons/personhog-persons-store.ts
index b39339bc5bd1..41b9676141b8 100644
--- a/nodejs/src/ingestion/common/persons/personhog-persons-store.ts
+++ b/nodejs/src/ingestion/common/persons/personhog-persons-store.ts
@@ -1390,8 +1390,9 @@ export class PersonhogPersonsStore implements PersonsStore {
while (progress.remaining > 0) {
try {
if (viaRedirect) {
- await this.redirectToSurvivor(entry, progress)
- personhogStoreFlushCounter.inc({ outcome: 'redirected' })
+ if (await this.redirectToSurvivor(entry, progress)) {
+ personhogStoreFlushCounter.inc({ outcome: 'redirected' })
+ }
break
}
if (entry.heldSince !== undefined) {
@@ -1506,23 +1507,41 @@ export class PersonhogPersonsStore implements PersonsStore {
* Re-resolves a vanished person's distinct id and writes to whoever
* owns it now, matching the Postgres merged-away recovery. One resolve
* is enough: the saga repoints mappings before the leader answers a
- * tombstone, and no owner means a genuine delete.
+ * tombstone, and no owner means a genuine delete. Returns whether the
+ * ops were written; a delete drops them.
*/
- private async redirectToSurvivor(entry: OpsLaneEntry, progress: { remaining: number }): Promise<void> {
+ private async redirectToSurvivor(entry: OpsLaneEntry, progress: { remaining: number }): Promise<boolean> {
const generation = this.generationOf(entry.teamId)
const [resolved] = await this.resolveDistinctIds('redirect_survivor', [
{ teamId: entry.teamId, distinctId: entry.distinctId },
])
const survivorId = resolved?.person?.id
if (survivorId === undefined || survivorId === entry.personId) {
- // Released, so the redelivered events re-resolve rather than
- // folding onto the corpse again.
+ // Released, so later events re-resolve rather than folding onto
+ // the corpse again.
this.removeDistinctIdFromCache(entry.teamId, entry.distinctId, 'redirect_gone')
this.clearPersonCacheForPersonId(`${entry.teamId}:${entry.personId}`, 'redirect_gone')
+ }
+ if (survivorId === undefined) {
+ // The person was deleted after these ops folded. Postgres drops
+ // them too; failing the flush would restart the processor and
+ // replay every in-flight batch for ops no owner can take.
+ personhogStoreFlushCounter.inc({ outcome: 'redirect_deleted' })
+ logger.warn('🤔', 'ops dropped: the person was deleted and its distinct id resolves to nobody', {
+ team_id: entry.teamId,
+ person_id: entry.personId,
+ distinct_id: entry.distinctId,
+ })
+ // Folds that arrived during the redirect stay for a later pass.
+ entry.segments.splice(0, progress.remaining)
+ progress.remaining = 0
+ return false
+ }
+ if (survivorId === entry.personId) {
personhogStoreFlushCounter.inc({ outcome: 'redirect_gone' })
throw new CountedRedirectError(
`person ${entry.personId} in team ${entry.teamId} vanished and ${entry.distinctId} resolves to ` +
- `${survivorId === undefined ? 'nobody' : 'the same person'}; failing the flush to redeliver`
+ `the same person; failing the flush to redeliver`
)
}
// The resolve proved where the id belongs, so heal the edge, the
@@ -1544,6 +1563,7 @@ export class PersonhogPersonsStore implements PersonsStore {
// the partial set a failed write may have landed.
this.clearPersonCacheForPersonId(`${entry.teamId}:${survivorId}`, 'redirect_survivor')
}
+ return true
}
/**