:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(ingestion): retry a load-shed merge refusal instead of settling it as lost (#123738)
Nick Best75b6c96c9d1a
diff --git a/nodejs/src/common/personhog/identity.ts b/nodejs/src/common/personhog/identity.tsindex 474e93f7ecc2..7338fea0579f 100644--- a/nodejs/src/common/personhog/identity.ts+++ b/nodejs/src/common/personhog/identity.ts@@ -84,6 +84,8 @@ export interface MergeSagaRequest {export const SEMANTIC_REFUSAL_METADATA_KEY = 'x-semantic-refusal'/** The op id belongs to a different recorded merge; refused before any durable work. */export const SEMANTIC_REFUSAL_OP_ID_REUSED = 'op_id_reused'+/** Metadata key on a refusal made before the request reached the service; the value is a reason slug. */+export const NOT_APPLIED_METADATA_KEY = 'x-not-applied'/** Only a created person carries a document; a found one is identity only, the leader owns its properties. */export type GetOrCreatePersonOutcome =diff --git a/nodejs/src/ingestion/common/persons/person-context.ts b/nodejs/src/ingestion/common/persons/person-context.tsindex a61612ca781b..b9d976801136 100644--- a/nodejs/src/ingestion/common/persons/person-context.ts+++ b/nodejs/src/ingestion/common/persons/person-context.ts@@ -42,7 +42,9 @@ export class PersonContext {/** Fold plan shared by this event's consecutive $identify run; see person-merge-fold.ts. */public readonly mergeFoldPlan?: MergeFoldPlan,/** The personhog identity service publishes the merge verdict warnings it settles, so this consumer must not. */- public readonly identityEmitsMergeWarnings: boolean = false+ public readonly identityEmitsMergeWarnings: boolean = false,+ /** Per-batch wall-clock budget for re-sending merges the backend refused before they ran; 0 escalates at once. */+ public readonly mergeNotAppliedWaitMs: number = 0) {this.eventProperties = event.properties!}diff --git a/nodejs/src/ingestion/common/persons/person-merge-service.test.ts b/nodejs/src/ingestion/common/persons/person-merge-service.test.tsindex a749b0605f59..2f1ea68e007d 100644--- a/nodejs/src/ingestion/common/persons/person-merge-service.test.ts+++ b/nodejs/src/ingestion/common/persons/person-merge-service.test.ts@@ -14,7 +14,9 @@ import { MergeFoldPlan } from './person-merge-fold'import {PersonMergeService,mergeClaimDroppedCounter,+ mergeFinalFailuresCounter,mergeFoldFallbackCounter,+ mergeNotAppliedRetriesCounter,mergeResponseMismatchCounter,mergeSettledFailureCounter,mergeUnsettledCounter,@@ -22,6 +24,7 @@ import {import {PersonMergeCallFailedError,PersonMergeLimitExceededError,+ PersonMergeNotAppliedError,PersonMergeResponseMismatchError,SourcePersonHasDistinctIdsError,SourcePersonNotFoundError,@@ -61,7 +64,8 @@ describe('PersonMergeService store-owned merges', () => {const makeService = (eventName = '$identify',mergeFoldPlan?: MergeFoldPlan,- identityEmitsMergeWarnings = false+ identityEmitsMergeWarnings = false,+ mergeNotAppliedWaitMs = 0) => {const event = {event: eventName,@@ -88,7 +92,8 @@ describe('PersonMergeService store-owned merges', () => {false,false,mergeFoldPlan,- identityEmitsMergeWarnings+ identityEmitsMergeWarnings,+ mergeNotAppliedWaitMs)return new PersonMergeService(context)}@@ -210,6 +215,112 @@ describe('PersonMergeService store-owned merges', () => {expect(warned).toContain('merge_settled_failure')})+ describe('a merge refused before it ran', () => {+ const refused = (): Error =>+ new PersonMergeNotAppliedError('refused before it ran', 'load_shed', new Error('shed'))+ const warningTypes = (): string[] =>+ outputs.queueMessages.mock.calls+ .flat(3)+ .filter((entry: any) => entry?.value)+ .map((entry: any) => parseJSON(Buffer.from(entry.value).toString()).type)+ const outcomes = async (): Promise<Record<string, number>> =>+ Object.fromEntries(+ (await mergeNotAppliedRetriesCounter.get()).values.map((entry) => [entry.labels.outcome, entry.value])+ )++ beforeEach(() => {+ mergeNotAppliedRetriesCounter.reset()+ jest.useFakeTimers()+ })++ afterEach(() => {+ jest.useRealTimers()+ })++ it('is re-sent past the bounded retries and keeps them once capacity returns', async () => {+ const finalBefore = await counterTotal(mergeFinalFailuresCounter)+ for (let i = 0; i < 5; i++) {+ store.mergePersons.mockRejectedValueOnce(refused())+ }+ // The first accepted call answers unsettled: without its own bounded+ // retries the merge would drop with the race warning instead of settling.+ store.mergePersons.mockResolvedValueOnce({+ survivor,+ results: [{ sourceDistinctId: 'anon-1', outcome: 'skipped_conflict' as const, settled: false }],+ })+ const service = makeService('$identify', undefined, false, 5_000)++ const merging = service.handleIdentifyOrAlias()+ await jest.advanceTimersByTimeAsync(5_000)+ const mergeResult = await merging++ expect(mergeResult).toMatchObject({ success: true, person: survivor })+ expect(store.mergePersons).toHaveBeenCalledTimes(7)+ expect(warningTypes()).toEqual([])+ expect(await counterTotal(mergeFinalFailuresCounter)).toBe(finalBefore)+ expect(await outcomes()).toEqual({ recovered: 1 })+ })++ it('escalates to the pipeline retry when the budget runs out, never settling', async () => {+ const finalBefore = await counterTotal(mergeFinalFailuresCounter)+ store.mergePersons.mockRejectedValue(refused())+ const service = makeService('$identify', undefined, false, 1_000)++ const settled = service.handleIdentifyOrAlias().catch((error) => error)+ await jest.advanceTimersByTimeAsync(10_000)+ const error = await settled++ expect(error).toBeInstanceOf(PersonMergeNotAppliedError)+ expect(error.isRetriable).toBe(true)+ expect(store.mergePersons.mock.calls.length).toBeGreaterThan(4)+ expect(warningTypes()).not.toContain('merge_settled_failure')+ expect(await counterTotal(mergeFinalFailuresCounter)).toBe(finalBefore)+ expect(await outcomes()).toEqual({ escalated: 1 })+ })++ it('shares one budget across the batch, so a later merge escalates at once when it is spent', async () => {+ // Merges for one distinct id run serially within a batch; a fresh+ // budget per event would add up past the consumer stall threshold.+ store.mergePersons.mockRejectedValue(refused())+ const first = makeService('$identify', undefined, false, 1_000)+ .handleIdentifyOrAlias()+ .catch((e) => e)+ await jest.advanceTimersByTimeAsync(10_000)+ await first+ store.mergePersons.mockClear()++ const second = makeService('$identify', undefined, false, 1_000)+ .handleIdentifyOrAlias()+ .catch((e) => e)+ await jest.advanceTimersByTimeAsync(10)+ const error = await second++ expect(error).toBeInstanceOf(PersonMergeNotAppliedError)+ expect(error.isRetriable).toBe(true)+ expect(store.mergePersons).toHaveBeenCalledTimes(3)+ // The second merge never waited, so it is not a second escalation.+ expect(await outcomes()).toEqual({ escalated: 1 })+ })++ it('an accepted re-send that then fails settles as before, counted apart from recovered', async () => {+ for (let i = 0; i < 3; i++) {+ store.mergePersons.mockRejectedValueOnce(refused())+ }+ store.mergePersons.mockRejectedValue(+ new PersonMergeCallFailedError('personhog merge call failed with no verdict', new Error('transport'))+ )+ const service = makeService('$identify', undefined, false, 5_000)++ const merging = service.handleIdentifyOrAlias()+ await jest.advanceTimersByTimeAsync(5_000)+ const result = await merging++ expect(result.success).toBe(true)+ expect(warningTypes()).toContain('merge_settled_failure')+ expect(await outcomes()).toEqual({ accepted_failed: 1 })+ })+ })+it('a deterministic raw refusal acks once with a warning instead of wedging the batch', async () => {// The op_id_reused refusal is deterministic, so rethrowing or// retrying would spin on the same refusal until the recorded opdiff --git a/nodejs/src/ingestion/common/persons/person-merge-service.ts b/nodejs/src/ingestion/common/persons/person-merge-service.tsindex 6eebb9207623..0382c61b364d 100644--- a/nodejs/src/ingestion/common/persons/person-merge-service.ts+++ b/nodejs/src/ingestion/common/persons/person-merge-service.ts@@ -11,12 +11,14 @@ import { timeoutGuard } from '~/common/utils/db/utils'import { logger } from '~/common/utils/logger'import { captureException } from '~/common/utils/posthog'import { promiseRetry } from '~/common/utils/retries'+import { sleep } from '~/common/utils/utils'import { emitIngestionWarning } from '~/ingestion/common/ingestion-warnings'import { PersonContext } from './person-context'import {PersonMergeCallFailedError,PersonMergeLimitExceededError,+ PersonMergeNotAppliedError,PersonMergeRaceConditionError,PersonMergeResponseMismatchError,PersonMergeResult,@@ -34,6 +36,7 @@ import {MergePersonsResult,MergePersonsSourceResult,} from './persons-store'+import type { PersonsStoreForBatch } from './persons-store-for-batch'export const mergeFinalFailuresCounter = new Counter({name: 'person_merge_final_failure_total',@@ -77,6 +80,18 @@ export const mergeClaimDroppedCounter = new Counter({labelNames: ['call'],})+export const mergeNotAppliedRetriesCounter = new Counter({+ name: 'person_merge_not_applied_retries_total',+ help: 'Merges the backend refused before running: recovered once accepted, accepted_failed when the accepted call then failed, escalated when the wait gave up and the pipeline retry took over.',+ labelNames: ['call', 'outcome'],+})++const NOT_APPLIED_FIRST_SLEEP_MS = 250+const NOT_APPLIED_MAX_SLEEP_MS = 2_000++/** One wait budget per batch: merges for one distinct id run serially, so per-event budgets would add up past the stall threshold. */+const batchNotAppliedDeadlines = new WeakMap<PersonsStoreForBatch, number>()+/** Maps a thrown fold failure to the fallback counter's reason label. */function foldAbandonReason(error: unknown): MergeFoldAbortReason {if (error instanceof PersonClaimedByLifecycleOpError || error instanceof PersonTombstoneBlockedError) {@@ -177,6 +192,10 @@ export class PersonMergeService {// A malformed answer is a bug detector; fail the batch.throw e}+ if (e instanceof PersonMergeNotAppliedError) {+ // Nothing ran, so there is nothing to settle; the pipeline retry re-sends it.+ throw e+ }// PersonMergeCallFailedError falls through: bounded retries ran,// so it takes the same give-up the Postgres path has always had.mergeFinalFailuresCounter@@ -299,20 +318,14 @@ export class PersonMergeService {// deliberately unwrapped; retrying re-runs the same comparison.// Mapping inside the retry lets an unsettled verdict re-present the// same op id, which the saga's conflict discard was built for.- try {- return await promiseRetry(- async () =>- this.mapSingleSourceResult(- await this.context.personStore.mergePersons(request),- otherPersonDistinctId,- mergeIntoDistinctId- ),- 'merge_distinct_ids',- undefined,- undefined,- undefined,- [ConnectError, PersonMergeResponseMismatchError]+ const attempt = async (): Promise<PersonMergeResult> =>+ this.mapSingleSourceResult(+ await this.context.personStore.mergePersons(request),+ otherPersonDistinctId,+ mergeIntoDistinctId)+ try {+ return await this.mergeWithRetries(attempt)} catch (error) {if (!(error instanceof PersonMergeUnsettledError || error instanceof PersonMergeRaceConditionError)) {throw error@@ -345,6 +358,75 @@ export class PersonMergeService {}}+ private async mergeWithRetries(attempt: () => Promise<PersonMergeResult>): Promise<PersonMergeResult> {+ const bounded = (nonRetriable: (new (...args: any[]) => Error)[]): Promise<PersonMergeResult> =>+ promiseRetry(attempt, 'merge_distinct_ids', undefined, undefined, undefined, nonRetriable)+ try {+ return await bounded([ConnectError, PersonMergeResponseMismatchError])+ } catch (error) {+ if (!(error instanceof PersonMergeNotAppliedError)) {+ throw error+ }+ // Each re-send is a full bounded run, so a verdict the accepted+ // call answers keeps the retries it would have had without the wait.+ return await this.waitOutNotApplied(+ () => bounded([ConnectError, PersonMergeResponseMismatchError, PersonMergeNotAppliedError]),+ error+ )+ }+ }++ /** The wait is for capacity, not for a saga, so it cannot stack on a saga deadline. */+ private async waitOutNotApplied(+ attempt: () => Promise<PersonMergeResult>,+ refusal: PersonMergeNotAppliedError+ ): Promise<PersonMergeResult> {+ const call = this.context.event.event+ const deadline = this.batchNotAppliedDeadline()+ let sleepMs = NOT_APPLIED_FIRST_SLEEP_MS+ let last = refusal+ let waited = false+ for (let remaining = deadline - Date.now(); remaining > 0; remaining = deadline - Date.now()) {+ waited = true+ // Jittered so a fleet waiting out one burst does not re-herd the backend.+ await sleep(Math.min(remaining, sleepMs * (0.5 + Math.random() * 0.5)))+ sleepMs = Math.min(sleepMs * 2, NOT_APPLIED_MAX_SLEEP_MS)+ try {+ const result = await attempt()+ mergeNotAppliedRetriesCounter.labels({ call, outcome: 'recovered' }).inc()+ return result+ } catch (error) {+ if (!(error instanceof PersonMergeNotAppliedError)) {+ mergeNotAppliedRetriesCounter.labels({ call, outcome: 'accepted_failed' }).inc()+ throw error+ }+ last = error+ }+ }+ // A spent budget is re-entered by every pipeline step retry of the same+ // event, so counting those would report one escalation as five.+ if (waited) {+ mergeNotAppliedRetriesCounter.labels({ call, outcome: 'escalated' }).inc()+ logger.warn('🤔', 'merge refused before it ran through every re-send; escalated to the pipeline retry', {+ team_id: this.context.team.id,+ distinctId: this.context.distinctId,+ event_name: call,+ reason: last.reason,+ })+ }+ throw Object.assign(last, { isRetriable: true as const })+ }++ private batchNotAppliedDeadline(): number {+ const store = this.context.personStore+ let deadline = batchNotAppliedDeadlines.get(store)+ if (deadline === undefined) {+ deadline = Date.now() + this.context.mergeNotAppliedWaitMs+ batchNotAppliedDeadlines.set(store, deadline)+ }+ return deadline+ }+/*** One request for either backend, single-source or folded. The event* uuid doubles as the op id: retries re-enter with the same id anddiff --git a/nodejs/src/ingestion/common/persons/person-merge-types.ts b/nodejs/src/ingestion/common/persons/person-merge-types.tsindex 05a30ae88133..cf7191a2d5d9 100644--- a/nodejs/src/ingestion/common/persons/person-merge-types.ts+++ b/nodejs/src/ingestion/common/persons/person-merge-types.ts@@ -47,6 +47,19 @@ export class PersonMergeCallFailedError extends PersonMergeError {}}+/** The merge backend refused the call before anything ran (`x-not-applied`); re-sent, never settled as lost. */+export class PersonMergeNotAppliedError extends PersonMergeError {+ readonly type = 'NOT_APPLIED' as const++ constructor(+ message: string,+ public readonly reason: string,+ public readonly failure: unknown+ ) {+ super(message)+ }+}+/*** The merge backend answered an unsettled verdict: a retry under the same* op id may change the answer. Retried in-process; exhaustion drops thediff --git a/nodejs/src/ingestion/common/persons/personhog-persons-store.test.ts b/nodejs/src/ingestion/common/persons/personhog-persons-store.test.tsindex 58b998c0000e..51a1b39e9978 100644--- a/nodejs/src/ingestion/common/persons/personhog-persons-store.test.ts+++ b/nodejs/src/ingestion/common/persons/personhog-persons-store.test.ts@@ -7,7 +7,11 @@ import { PersonClaimedByLifecycleOpError } from '~/common/persons/repositories/pimport { NoRowsUpdatedError } from '~/common/utils/utils'import { InternalPerson } from '~/types'-import { PersonMergeCallFailedError, createDefaultSyncMergeMode } from './person-merge-types'+import {+ PersonMergeCallFailedError,+ PersonMergeNotAppliedError,+ createDefaultSyncMergeMode,+} from './person-merge-types'import { EventOps, extractEventOps } from './person-update'import { mergeOpIdFromRequest } from './person-uuid'import {@@ -530,6 +534,22 @@ describe('PersonhogPersonsStore', () => {expect(edgeOf('1:d1')).toBeUndefined()})+ it('a refusal marked not-applied is typed for the re-send, whatever its status code', async () => {+ for (const code of [Code.Unavailable, Code.ResourceExhausted]) {+ repository.mergePersons = jest+ .fn()+ .mockRejectedValue(+ new ConnectError('Server at capacity', code, new Headers({ 'x-not-applied': 'load_shed' }))+ )+ const bound = store.forBatch(0)+ await bound.fetchForUpdate(1, 'd1')+ const error = await bound.mergePersons(mergeRequest()).catch((e) => e)+ expect(error).toBeInstanceOf(PersonMergeNotAppliedError)+ expect(error.reason).toBe('load_shed')+ expect(edgeOf('1:d1')).toBeUndefined()+ }+ })+it('derives one op id per delivery regardless of how the event payload drifts', async () => {// Two deliveries of one event disagree on properties later stages// refresh, but must present the same op id so the second attachesdiff --git a/nodejs/src/ingestion/common/persons/personhog-persons-store.ts b/nodejs/src/ingestion/common/persons/personhog-persons-store.tsindex 607cdd7ec044..703e1935bcb0 100644--- a/nodejs/src/ingestion/common/persons/personhog-persons-store.ts+++ b/nodejs/src/ingestion/common/persons/personhog-persons-store.ts@@ -6,6 +6,7 @@ import { Counter } from 'prom-client'import { TRANSPORT_MAX_BACKOFF_MS, TRANSPORT_MAX_RETRIES } from '~/common/personhog/grpc-retry'import {DistinctIdKey,+ NOT_APPLIED_METADATA_KEY,SEMANTIC_REFUSAL_METADATA_KEY,SEMANTIC_REFUSAL_OP_ID_REUSED,} from '~/common/personhog/identity'@@ -23,7 +24,7 @@ import { BatchWritingStoreFlushStats } from '~/ingestion/common/stores/batch-wriimport { Properties } from '~/plugin-scaffold'import { InternalPerson, PropertiesLastOperation, PropertiesLastUpdatedAt } from '~/types'-import { MergeMode, PersonMergeCallFailedError } from './person-merge-types'+import { MergeMode, PersonMergeCallFailedError, PersonMergeNotAppliedError } from './person-merge-types'import { EventOps, applyEventPropertyUpdates, computeOpsScalarUpdates, foldOps, refineEventOps } from './person-update'import { mergeOpIdFromRequest } from './person-uuid'import { FlushResult, MergePersonsRequest, MergePersonsResult, PersonsStore, isFoldRequest } from './persons-store'@@ -887,6 +888,17 @@ export class PersonhogPersonsStore implements PersonsStore {personhogStoreMergeCallFailedCounter.inc({ error: 'OpIdReusedSettled' })throw error}+ // Keyed on the header: a hop in between may remap the status code.+ const notApplied = error instanceof ConnectError ? error.metadata.get(NOT_APPLIED_METADATA_KEY) : null+ if (notApplied !== null) {+ const reason = /^[a-z0-9_-]{1,64}$/i.test(notApplied) ? notApplied : 'unknown'+ personhogStoreMergeCallFailedCounter.inc({ error: `not_applied_${reason}` })+ throw new PersonMergeNotAppliedError(+ `personhog merge refused before it ran (${reason}): ${error.message}`,+ reason,+ error+ )+ }// No verdict, so an ack could lose the merge. Only the call is// wrapped, so a post-verdict bug surfaces as itself.personhogStoreMergeCallFailedCounter.inc({diff --git a/nodejs/src/ingestion/common/persons/routing-persons-store.test.ts b/nodejs/src/ingestion/common/persons/routing-persons-store.test.tsindex e6216519566b..21da9e801981 100644--- a/nodejs/src/ingestion/common/persons/routing-persons-store.test.ts+++ b/nodejs/src/ingestion/common/persons/routing-persons-store.test.ts@@ -14,7 +14,7 @@ import { logger } from '~/common/utils/logger'import { defaultRetryConfig } from '~/common/utils/retries'import { InternalPerson } from '~/types'-import { PersonMergeCallFailedError } from './person-merge-types'+import { PersonMergeCallFailedError, PersonMergeNotAppliedError } from './person-merge-types'import { EventOps } from './person-update'import { PersonhogPersonsStore } from './personhog-persons-store'import { MergePersonsResult, PersonsBackend, PersonsStore } from './persons-store'@@ -944,6 +944,8 @@ describe('RoutingPersonsStore', () => {})const noVerdict = (): Error => new PersonMergeCallFailedError('no verdict', new Error('shed'))+ const notApplied = (): Error =>+ new PersonMergeNotAppliedError('refused before it ran', 'load_shed', new Error('shed'))it.each([['a no-verdict failure that clears is retried under the same request', [noVerdict()], 2, null, false],@@ -954,6 +956,13 @@ describe('RoutingPersonsStore', () => {'PersonMergeCallFailedError',true,],+ [+ 'a refusal before it ran through every attempt is re-driven at a flush, not waited out',+ [notApplied(), notApplied(), notApplied()],+ 3,+ 'PersonMergeNotAppliedError',+ true,+ ],['a deterministic refusal is counted without a retry or a re-drive',[new ConnectError('refused', Code.InvalidArgument)],diff --git a/nodejs/src/ingestion/common/persons/routing-persons-store.ts b/nodejs/src/ingestion/common/persons/routing-persons-store.tsindex 9b4818cfbe18..31540681fe23 100644--- a/nodejs/src/ingestion/common/persons/routing-persons-store.ts+++ b/nodejs/src/ingestion/common/persons/routing-persons-store.ts@@ -22,7 +22,7 @@ import { BatchWritingStoreFlushStats } from '~/ingestion/common/stores/batch-wriimport { Properties } from '~/plugin-scaffold'import { InternalPerson, PropertiesLastOperation, PropertiesLastUpdatedAt } from '~/types'-import { PersonMergeCallFailedError, PersonMergeUnsettledError } from './person-merge-types'+import { PersonMergeCallFailedError, PersonMergeNotAppliedError, PersonMergeUnsettledError } from './person-merge-types'import { EventOps } from './person-update'import { CREATE_EVENT_NAME, PersonhogPersonsStore } from './personhog-persons-store'import {@@ -648,8 +648,13 @@ export class RoutingPersonsStore implements PersonsStore {}return unsettled}- // Abandoned at the ceiling, the merge is re-driven at a flush under the same op id.- if ((error instanceof PersonMergeCallFailedError || error instanceof ShadowVerbTimeoutError) && defer) {+ // Abandoned at the ceiling or refused before it ran, the merge is re-driven at a flush under the same op id.+ if (+ (error instanceof PersonMergeCallFailedError ||+ error instanceof PersonMergeNotAppliedError ||+ error instanceof ShadowVerbTimeoutError) &&+ defer+ ) {this.deferShadowMerge(request, batchId)}throw errordiff --git a/nodejs/src/ingestion/common/steps/event-processing/event-pipeline-options.ts b/nodejs/src/ingestion/common/steps/event-processing/event-pipeline-options.tsindex 9e8eb719c98b..f24be1fdf6ea 100644--- a/nodejs/src/ingestion/common/steps/event-processing/event-pipeline-options.ts+++ b/nodejs/src/ingestion/common/steps/event-processing/event-pipeline-options.ts@@ -6,6 +6,7 @@ export interface EventPipelineRunnerOptions {PERSON_MERGE_FOLD_ENABLED: booleanPERSON_MERGE_FOLD_TEAM_ALLOWLIST: stringPERSONHOG_IDENTITY_EMITS_MERGE_WARNINGS: boolean+ PERSON_MERGE_NOT_APPLIED_WAIT_MS: numberPERSON_JSONB_SIZE_ESTIMATE_ENABLE: numberPERSON_PROPERTIES_UPDATE_ALL: boolean/** Teams whose $feature_flag_called events default to personless: '*' for all, '' to disable, or comma-separated team IDs */diff --git a/nodejs/src/ingestion/common/steps/event-processing/process-persons-step.ts b/nodejs/src/ingestion/common/steps/event-processing/process-persons-step.tsindex 09e6cee17cc0..c91ea628f0a5 100644--- a/nodejs/src/ingestion/common/steps/event-processing/process-persons-step.ts+++ b/nodejs/src/ingestion/common/steps/event-processing/process-persons-step.ts@@ -63,7 +63,8 @@ export function createProcessPersonsStep<TInput extends ProcessPersonsInput>(options.PERSON_PROPERTIES_UPDATE_ALL,shouldUpdateLastSeenAt,input.mergeFold.type === 'planned' ? input.mergeFold.plan : undefined,- options.PERSONHOG_IDENTITY_EMITS_MERGE_WARNINGS+ options.PERSONHOG_IDENTITY_EMITS_MERGE_WARNINGS,+ options.PERSON_MERGE_NOT_APPLIED_WAIT_MS)const processor = new PersonEventProcessor(diff --git a/nodejs/src/ingestion/config.ts b/nodejs/src/ingestion/config.tsindex 6a532e14c79d..9d88278a0748 100644--- a/nodejs/src/ingestion/config.ts+++ b/nodejs/src/ingestion/config.ts@@ -176,6 +176,10 @@ export type IngestionConsumerConfig = {// comes back skipped_move_limit for the merge-mode policy.PERSONHOG_SYNC_MERGE_MOVE_LIMIT: numberPERSONHOG_IDENTITY_EMITS_MERGE_WARNINGS: boolean+ // Re-send budget per batch for merges refused at admission, measured from the batch's+ // first refusal and shared by every merge and step retry in it. Keep it well under+ // CONSUMER_LOOP_STALL_THRESHOLD_MS, which the whole batch has to fit in.+ PERSON_MERGE_NOT_APPLIED_WAIT_MS: number// Kill switch for emitting person_merge_events to the cohort-stream-processor.// Enable ordering: (1) create the topic, (2) set INGESTION_OUTPUT_PERSON_MERGE_EVENTS_TOPIC// (startup topic verification is then fatal by design), (3) flip this on. Flipping this on before@@ -370,6 +374,7 @@ export function getDefaultIngestionConsumerConfig(): IngestionConsumerConfig {PERSON_MERGE_SYNC_BATCH_SIZE: 0,PERSONHOG_SYNC_MERGE_MOVE_LIMIT: 10_000,PERSONHOG_IDENTITY_EMITS_MERGE_WARNINGS: false,+ PERSON_MERGE_NOT_APPLIED_WAIT_MS: 20_000,PERSON_MERGE_EVENTS_ENABLED: false,PERSON_MERGE_EVENTS_PARTITION_COUNT: 64,PERSON_MERGE_EVENTS_TEAM_ALLOWLIST: '2',diff --git a/nodejs/src/ingestion/ingestion-consumer.ts b/nodejs/src/ingestion/ingestion-consumer.tsindex 6e2f9028369d..633d643d938a 100644--- a/nodejs/src/ingestion/ingestion-consumer.ts+++ b/nodejs/src/ingestion/ingestion-consumer.ts@@ -318,6 +318,7 @@ export class IngestionConsumer {PERSON_MERGE_FOLD_ENABLED: this.config.PERSON_MERGE_FOLD_ENABLED,PERSON_MERGE_FOLD_TEAM_ALLOWLIST: this.config.PERSON_MERGE_FOLD_TEAM_ALLOWLIST,PERSONHOG_IDENTITY_EMITS_MERGE_WARNINGS: this.config.PERSONHOG_IDENTITY_EMITS_MERGE_WARNINGS,+ PERSON_MERGE_NOT_APPLIED_WAIT_MS: this.config.PERSON_MERGE_NOT_APPLIED_WAIT_MS,PERSON_JSONB_SIZE_ESTIMATE_ENABLE: this.config.PERSON_JSONB_SIZE_ESTIMATE_ENABLE,PERSON_PROPERTIES_UPDATE_ALL: this.config.PERSON_PROPERTIES_UPDATE_ALL,FLAG_CALLED_PERSONLESS_DEFAULT_TEAMS: this.config.FLAG_CALLED_PERSONLESS_DEFAULT_TEAMS,diff --git a/nodejs/src/servers/ingestion-api-server.ts b/nodejs/src/servers/ingestion-api-server.tsindex 98b6429f10e3..2520265466f9 100644--- a/nodejs/src/servers/ingestion-api-server.ts+++ b/nodejs/src/servers/ingestion-api-server.ts@@ -437,6 +437,7 @@ export class IngestionApiServer implements NodeServer {PERSON_MERGE_FOLD_ENABLED: this.config.PERSON_MERGE_FOLD_ENABLED,PERSON_MERGE_FOLD_TEAM_ALLOWLIST: this.config.PERSON_MERGE_FOLD_TEAM_ALLOWLIST,PERSONHOG_IDENTITY_EMITS_MERGE_WARNINGS: this.config.PERSONHOG_IDENTITY_EMITS_MERGE_WARNINGS,+ PERSON_MERGE_NOT_APPLIED_WAIT_MS: this.config.PERSON_MERGE_NOT_APPLIED_WAIT_MS,PERSON_JSONB_SIZE_ESTIMATE_ENABLE: this.config.PERSON_JSONB_SIZE_ESTIMATE_ENABLE,PERSON_PROPERTIES_UPDATE_ALL: this.config.PERSON_PROPERTIES_UPDATE_ALL,FLAG_CALLED_PERSONLESS_DEFAULT_TEAMS: this.config.FLAG_CALLED_PERSONLESS_DEFAULT_TEAMS,diff --git a/nodejs/tests/ingestion/common/steps/event-processing/process-persons-step.integration.test.ts b/nodejs/tests/ingestion/common/steps/event-processing/process-persons-step.integration.test.tsindex f372cd6f2ff5..7ea9c8342c98 100644--- a/nodejs/tests/ingestion/common/steps/event-processing/process-persons-step.integration.test.ts+++ b/nodejs/tests/ingestion/common/steps/event-processing/process-persons-step.integration.test.ts@@ -55,6 +55,7 @@ describe('createProcessPersonsStep', () => {PERSON_MERGE_FOLD_ENABLED: false,PERSON_MERGE_FOLD_TEAM_ALLOWLIST: '*',PERSONHOG_IDENTITY_EMITS_MERGE_WARNINGS: false,+ PERSON_MERGE_NOT_APPLIED_WAIT_MS: 0,PERSON_JSONB_SIZE_ESTIMATE_ENABLE: 0,PERSON_PROPERTIES_UPDATE_ALL: false,FLAG_CALLED_PERSONLESS_DEFAULT_TEAMS: '*',