:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(ingestion): wait out a lifecycle fence in the personhog store flush (#128816)
Zach Brown1edf37ad9feb
diff --git a/nodejs/src/ingestion/common/persons/personhog-persons-store.test.ts b/nodejs/src/ingestion/common/persons/personhog-persons-store.test.tsindex 51a1b39e9978..0fa50c89f6fa 100644--- a/nodejs/src/ingestion/common/persons/personhog-persons-store.test.ts+++ b/nodejs/src/ingestion/common/persons/personhog-persons-store.test.ts@@ -2445,21 +2445,63 @@ describe('PersonhogPersonsStore', () => {})describe('convergence closures', () => {- it('a fence bounce counts as coordination, not on the flush error series', async () => {+ it('a fence bounce waits for the fence to clear and then writes the lane', async () => {personhogStoreFlushCounter.reset()personhogStoreFlushErrorCounter.reset()- const bound = store.forBatch(0)- await bound.applyEventOps(person, ops({ $set: { mine: 'kept' } }), 'd1')- repository.updatePersonProperties.mockRejectedValue(- new PersonhogFencedError('PERSON_MERGING', '7', 'op-1') as never- )+ jest.useFakeTimers()+ try {+ const bound = store.forBatch(0)+ await bound.applyEventOps(person, ops({ $set: { mine: 'kept' } }), 'd1')+ repository.updatePersonProperties.mockRejectedValueOnce(+ new PersonhogFencedError('PERSON_MERGING', '7', 'op-1') as never+ )- await expect(bound.flush()).rejects.toThrow(PersonhogFencedError)+ const flushing = bound.flush()+ await jest.advanceTimersByTimeAsync(1_000)+ await expect(flushing).resolves.toEqual([])++ expect(repository.updatePersonProperties).toHaveBeenCalledTimes(2)+ expect((store as any).entries.get('1:7').segments).toHaveLength(0)+ const outcomes = (await personhogStoreFlushCounter.get()).values+ expect(outcomes.find((entry) => entry.labels.outcome === 'fenced')?.value).toBe(1)+ expect(outcomes.find((entry) => entry.labels.outcome === 'error')?.value ?? 0).toBe(0)+ expect(await counterTotal(personhogStoreFlushErrorCounter)).toBe(0)+ } finally {+ jest.useRealTimers()+ }+ })- const outcomes = (await personhogStoreFlushCounter.get()).values- expect(outcomes.find((entry) => entry.labels.outcome === 'fenced')?.value).toBe(1)- expect(outcomes.find((entry) => entry.labels.outcome === 'error')?.value ?? 0).toBe(0)- expect(await counterTotal(personhogStoreFlushErrorCounter)).toBe(0)+ it('a fence that outlasts the merge timeout fails the flush with the lane kept', async () => {+ personhogStoreFlushCounter.reset()+ jest.useFakeTimers()+ try {+ store = new PersonhogPersonsStore(repository, { requestTimeoutMs: 1_000, mergeTimeoutMs: 20_000 })+ const bound = store.forBatch(0)+ await bound.applyEventOps(person, ops({ $set: { mine: 'kept' } }), 'd1')+ repository.updatePersonProperties.mockRejectedValue(+ new PersonhogFencedError('PERSON_MERGING', '7', 'op-1') as never+ )++ const flushing = bound.flush()+ const settled = flushing.then(+ () => 'acked' as const,+ (error: Error) => error.message+ )+ let done = false+ void settled.then(() => {+ done = true+ })+ await jest.advanceTimersByTimeAsync((store as any).flushWaitBudgetMs + 1_000)+ expect(done).toBe(false)++ await jest.advanceTimersByTimeAsync(20_000)+ expect(await settled).toMatch(/still fenced by a lifecycle op/)+ expect((store as any).entries.get('1:7').segments).toHaveLength(1)+ const outcomes = (await personhogStoreFlushCounter.get()).values+ expect(outcomes.find((entry) => entry.labels.outcome === 'fence_exhausted')?.value).toBe(1)+ } finally {+ jest.useRealTimers()+ }})const mergeReq = (sources = ['anon-1']) => ({diff --git a/nodejs/src/ingestion/common/persons/personhog-persons-store.ts b/nodejs/src/ingestion/common/persons/personhog-persons-store.tsindex 703e1935bcb0..b39339bc5bd1 100644--- a/nodejs/src/ingestion/common/persons/personhog-persons-store.ts+++ b/nodejs/src/ingestion/common/persons/personhog-persons-store.ts@@ -102,6 +102,7 @@ export interface PersonhogPersonsStoreOptions {syncMergeMoveLimit: number/** The injected repository's per-request timeout; the flush's lane wait derives from it. */requestTimeoutMs: number+ mergeTimeoutMs: number}const DEFAULT_OPTIONS: PersonhogPersonsStoreOptions = {@@ -109,6 +110,7 @@ const DEFAULT_OPTIONS: PersonhogPersonsStoreOptions = {updateAllProperties: false,syncMergeMoveLimit: 10_000,requestTimeoutMs: 3_000,+ mergeTimeoutMs: 35_000,}const CALLER_TAG = 'ingestion/personhog-store'@@ -265,6 +267,7 @@ export class PersonhogPersonsStore implements PersonsStore {private flushChain: Promise<void> = Promise.resolve()/** Total lane wait per flush, derived in the constructor from the transport retry budget. */private flushWaitBudgetMs: number+ private fenceWaitBudgetMs: numberconstructor(private repository: PersonHogPersonWriteRepository,@@ -286,6 +289,9 @@ export class PersonhogPersonsStore implements PersonsStore {if (!Number.isInteger(this.options.requestTimeoutMs) || this.options.requestTimeoutMs < 1) {throw new Error(`PERSONHOG_TIMEOUT_MS must be an integer >= 1, got ${this.options.requestTimeoutMs}`)}+ if (!Number.isInteger(this.options.mergeTimeoutMs) || this.options.mergeTimeoutMs < 1) {+ throw new Error(`PERSONHOG_MERGE_TIMEOUT_MS must be an integer >= 1, got ${this.options.mergeTimeoutMs}`)+ }// A flush waits out a claimed lane rather than failing past it. The// wait outlasts one write's full transport retry budget, derived// rather than chosen so the two cannot drift apart; a holder writing@@ -293,6 +299,10 @@ export class PersonhogPersonsStore implements PersonsStore {// redelivery instead of acking over unwritten ops.this.flushWaitBudgetMs =(TRANSPORT_MAX_RETRIES + 1) * this.options.requestTimeoutMs + TRANSPORT_MAX_BACKOFF_MS + FLUSH_WAIT_ROUND_MS+ // A merge saga holds its source's fence until it completes, and the+ // merge call's timeout already outlasts one saga drive, so a fenced+ // lane waits as long as a merge caller does.+ this.fenceWaitBudgetMs = Math.max(this.flushWaitBudgetMs, this.options.mergeTimeoutMs)}forBatch(batchId: number): PersonsStoreForBatch {@@ -1234,9 +1244,10 @@ export class PersonhogPersonsStore implements PersonsStore {* Writes the batch's folded lanes to the leader, one call per segment;* nothing publishes, because the leader's changelog is the person feed.* A missing person redirects to whatever its distinct id resolves to- * now; a person genuinely gone and a size rejection are counted and- * dropped since neither can succeed on retry; anything else fails the- * flush. Passes serialize.+ * now, and a person with no owner fails the flush so its redelivered+ * events re-resolve. A size rejection is counted and dropped because it+ * cannot succeed on retry. A lane fenced by a lifecycle op waits for the+ * fence to clear. Anything else fails the flush. Passes serialize.*/async flush(): Promise<FlushResult[]> {const run = this.flushChain.then(() => this.flushPass())@@ -1251,15 +1262,24 @@ export class PersonhogPersonsStore implements PersonsStore {// Success lets the batch ack, so the pass must not return while any// lane holds unwritten segments: claimed lanes are waited out and// rewritten, or the pass fails and the batch redelivers.+ let fenceSeen = falsefor (let round = 0; ; round++) {- const pass = { deferrals: 0 }+ const pass = { deferrals: 0, fenced: 0 }await this.writeEligibleLanes(pass)if (pass.deferrals === 0) {// No FlushResults: the leader's changelog is the person// feed, so a flush publishes nothing.return []}- if (round * FLUSH_WAIT_ROUND_MS >= this.flushWaitBudgetMs) {+ fenceSeen ||= pass.fenced > 0+ const budgetMs = fenceSeen ? this.fenceWaitBudgetMs : this.flushWaitBudgetMs+ if (round * FLUSH_WAIT_ROUND_MS >= budgetMs) {+ if (pass.fenced > 0) {+ personhogStoreFlushCounter.inc({ outcome: 'fence_exhausted' })+ throw new Error(+ `flush cannot complete: ${pass.fenced} lanes still fenced by a lifecycle op after ${budgetMs}ms`+ )+ }personhogStoreFlushCounter.inc({ outcome: 'parked_exhausted' })throw new Error(`flush cannot complete: ${pass.deferrals} lanes deferred behind writes that did not settle`@@ -1285,7 +1305,7 @@ export class PersonhogPersonsStore implements PersonsStore {}}- private async writeEligibleLanes(pass: { deferrals: number }): Promise<void> {+ private async writeEligibleLanes(pass: { deferrals: number; fenced: number }): Promise<void> {// Before the claim, so a held lane that resolves writes in this pass.await this.resolveHeldLanes()// No await in this block, so the claim snapshot is atomic.@@ -1311,7 +1331,18 @@ export class PersonhogPersonsStore implements PersonsStore {const outcomes = await Promise.allSettled(captured.map(({ personKey, entry, segments }) => limit(() => this.writeEntry(personKey, entry, segments))))- const failed = outcomes.find((outcome): outcome is PromiseRejectedResult => outcome.status === 'rejected')+ let failed: PromiseRejectedResult | undefined+ for (const outcome of outcomes) {+ if (outcome.status !== 'rejected') {+ continue+ }+ if (outcome.reason instanceof PersonhogFencedError) {+ pass.deferrals += 1+ pass.fenced += 1+ continue+ }+ failed ??= outcome+ }if (failed) {throw failed.reason}@@ -1398,9 +1429,9 @@ export class PersonhogPersonsStore implements PersonsStore {if (error instanceof PersonhogFencedError) {// Expected coordination, so it stays off the error// series an alert would page on: the holder settles- // and redelivery flows.+ // and a later round writes the lane.personhogStoreFlushCounter.inc({ outcome: 'fenced' })- logger.warn('flush bounced on a lifecycle fence; the batch redelivers', {+ logger.warn('flush bounced on a lifecycle fence; the lane retries', {teamId: entry.teamId,personId: entry.personId,fencingOpId: error.fencingOpId,diff --git a/nodejs/src/servers/ingestion-api-server.ts b/nodejs/src/servers/ingestion-api-server.tsindex 2520265466f9..ccdbe623f36d 100644--- a/nodejs/src/servers/ingestion-api-server.ts+++ b/nodejs/src/servers/ingestion-api-server.ts@@ -397,6 +397,7 @@ export class IngestionApiServer implements NodeServer {updateAllProperties: this.config.PERSON_PROPERTIES_UPDATE_ALL,syncMergeMoveLimit: this.config.PERSONHOG_SYNC_MERGE_MOVE_LIMIT,requestTimeoutMs: this.config.PERSONHOG_TIMEOUT_MS,+ mergeTimeoutMs: this.config.PERSONHOG_MERGE_TIMEOUT_MS,})personsStore = new RoutingPersonsStore(this.personsStore, personhogStore, personsStoreMode)}