Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(nodejs): retry reads a replica cancels for a recovery conflict (#122274)

Phil Haackbd887a13484a
diff --git a/nodejs/src/cdp/consumers/cdp-hogflow-subscription-matcher.consumer.test.ts b/nodejs/src/cdp/consumers/cdp-hogflow-subscription-matcher.consumer.test.ts
index c0b27123f31b..41a105cfded7 100644
--- a/nodejs/src/cdp/consumers/cdp-hogflow-subscription-matcher.consumer.test.ts
+++ b/nodejs/src/cdp/consumers/cdp-hogflow-subscription-matcher.consumer.test.ts
@@ -1367,10 +1367,10 @@ describe('CdpHogflowSubscriptionMatcherConsumer', () => {
},
},
{
- // handlePostgresError does not mark this transient replica error as retriable.
- lookup: 'the hog flow lookup is cancelled on the read replica',
+ // handlePostgresError does not mark a statement timeout as retriable.
+ lookup: 'the hog flow lookup hits a statement timeout',
fail: (m: MatcherUnderTest): Error => {
- const error = new Error('canceling statement due to conflict with recovery')
+ const error = new Error('canceling statement due to statement timeout')
;(m as any).hogFlowManager.getHogFlowsForTeam.mockRejectedValueOnce(error)
return error
},
diff --git a/nodejs/src/cdp/services/managers/integration-manager.service.test.ts b/nodejs/src/cdp/services/managers/integration-manager.service.test.ts
index e73497d42e10..45f2dd712ecb 100644
--- a/nodejs/src/cdp/services/managers/integration-manager.service.test.ts
+++ b/nodejs/src/cdp/services/managers/integration-manager.service.test.ts
@@ -1,4 +1,5 @@
import { IntegrationType } from '~/cdp/types'
+import { DependencyUnavailableError } from '~/common/utils/db/error'
import { closeHub, createHub } from '~/common/utils/db/hub'
import { PostgresUse } from '~/common/utils/db/postgres'
import { createTestTeamFixture } from '~/tests/helpers/sql'
@@ -58,6 +59,17 @@ describe('IntegrationManager', () => {
})
})
+ it('retries the lookup when the read replica cancels it', async () => {
+ const cancelled = new Error('Canceling statement due to conflict with recovery.')
+ jest.spyOn(hub.postgres, 'query').mockRejectedValueOnce(
+ new DependencyUnavailableError(cancelled.message, 'Postgres', cancelled)
+ )
+
+ const item = await manager.get(integrations[0].id)
+
+ expect(item?.config).toEqual({ team: 'foobar' })
+ })
+
it('updates cached integration data when integration changes', async () => {
// First check - initial state
const item = await manager.get(integrations[0].id)
diff --git a/nodejs/src/cdp/services/managers/integration-manager.service.ts b/nodejs/src/cdp/services/managers/integration-manager.service.ts
index 645907b3ff05..83f3cd76267d 100644
--- a/nodejs/src/cdp/services/managers/integration-manager.service.ts
+++ b/nodejs/src/cdp/services/managers/integration-manager.service.ts
@@ -1,7 +1,7 @@
import { IntegrationType } from '~/cdp/types'
import { EncryptedFields } from '~/cdp/utils/encryption-utils'
import { PostgresRouter, PostgresUse } from '~/common/utils/db/postgres'
-import { LazyLoader } from '~/common/utils/lazy-loader'
+import { DEFAULT_LOADER_RETRY, LazyLoader } from '~/common/utils/lazy-loader'
import { logger } from '~/common/utils/logger'
import { PubSub } from '~/common/utils/pubsub'
@@ -16,6 +16,7 @@ export class IntegrationManagerService {
this.lazyLoader = new LazyLoader({
name: 'integration_manager',
loader: async (ids) => await this.fetchIntegrations(ids),
+ loaderRetry: DEFAULT_LOADER_RETRY,
})
this.pubSub.on<{ integrationIds: IntegrationType['id'][] }>('reload-integrations', (message) => {
logger.debug('⚡', '[PubSub] Reloading integrations!', { integrationIds: message.integrationIds })
diff --git a/nodejs/src/common/utils/db/postgres.ts b/nodejs/src/common/utils/db/postgres.ts
index 030afebffdf8..152be7555eea 100644
--- a/nodejs/src/common/utils/db/postgres.ts
+++ b/nodejs/src/common/utils/db/postgres.ts
@@ -64,6 +64,11 @@ const POSTGRES_UNAVAILABLE_ERROR_MESSAGES = [
'server login has been failing', // PgBouncer cannot authenticate with upstream PG
'pooler is shutting down', // PgBouncer terminating client connections during a restart
'Cannot use a pool after calling end on the pool', // Shutdown ended the pool while work was still in flight
+ // A hot standby cancels a read when WAL replay conflicts with it.
+ // Some servers capitalize the first word of the message, so the entry holds only the words that every variant shares.
+ // Servers report the conflict as SQLSTATE 40001 or 57014.
+ // Both codes also cover other failures, so the list matches this phrase instead of a code.
+ 'conflict with recovery',
]
export function isTransientPgError(err: unknown): boolean {
diff --git a/nodejs/src/ingestion/common/groups/batch-writing-group-store.test.ts b/nodejs/src/ingestion/common/groups/batch-writing-group-store.test.ts
index 4c47f5b75899..43eeace73735 100644
--- a/nodejs/src/ingestion/common/groups/batch-writing-group-store.test.ts
+++ b/nodejs/src/ingestion/common/groups/batch-writing-group-store.test.ts
@@ -9,6 +9,8 @@ import { BatchWritingGroupStore } from './batch-writing-group-store'
import { groupCacheOperationsCounter } from './metrics'
import { ClickhouseGroupRepository } from '~/common/groups/repositories/clickhouse-group-repository'
import { GroupCreateResult, GroupRepository } from '~/common/groups/repositories/group-repository.interface'
+import { PostgresGroupRepository } from '~/common/groups/repositories/postgres-group-repository'
+import { PostgresRouter, PostgresUse } from '~/common/utils/db/postgres'
import { GroupsOutput } from '~/common/outputs'
import { IngestionOutputs } from '~/common/outputs/ingestion-outputs'
@@ -776,6 +778,21 @@ describe('BatchWritingGroupStore', () => {
expect(groupRepository.fetchGroup).not.toHaveBeenCalled()
expect(groupRepository.insertGroup).toHaveBeenCalledTimes(1)
})
+
+ it('resolves when the read replica cancels the prefetch read for a recovery conflict', async () => {
+ // The caller does not await prefetchGroups, so a rejection here would stop the server.
+ const router = new PostgresRouter({ DATABASE_URL: 'postgres://fake', POSTGRES_CONNECTION_POOL_SIZE: 1 })
+ const replica = {
+ query: jest.fn().mockRejectedValue(new Error('Canceling statement due to conflict with recovery.')),
+ }
+ ;(router as any).pools.set(PostgresUse.PERSONS_READ, replica)
+ const store = new BatchWritingGroupStore(new PostgresGroupRepository(router), clickhouseGroupRepository)
+
+ await expect(
+ store.prefetchGroups([{ teamId, groupTypeIndex: 1, groupKey: 'test', batchId: 0 }])
+ ).resolves.toBeUndefined()
+ expect(replica.query).toHaveBeenCalledTimes(1)
+ })
})
describe('persistent cache (concurrentBatches > 1)', () => {
diff --git a/nodejs/tests/utils/db/postgres.test.ts b/nodejs/tests/utils/db/postgres.test.ts
index 589f751a3ed7..663995388513 100644
--- a/nodejs/tests/utils/db/postgres.test.ts
+++ b/nodejs/tests/utils/db/postgres.test.ts
@@ -1,4 +1,3 @@
-import { DependencyUnavailableError } from '~/common/utils/db/error'
import { PostgresUse, handlePostgresError, isTransientPgError } from '~/common/utils/db/postgres'
describe('isTransientPgError', () => {
@@ -16,6 +15,7 @@ describe('isTransientPgError', () => {
['connect ETIMEDOUT 10.0.0.1:5432'],
['query_wait_timeout'],
['server login has been failing, try again later (server_login_retry)'],
+ ['Canceling statement due to conflict with recovery.'],
])('returns true for transient error "%s"', (message) => {
expect(isTransientPgError(new Error(message))).toBe(true)
})
@@ -52,14 +52,12 @@ describe('handlePostgresError', () => {
['connect ETIMEDOUT 10.0.0.1:5432'],
['query_wait_timeout'],
['server login has been failing, try again later (server_login_retry)'],
+ ['canceling statement due to conflict with recovery'],
+ ['Canceling statement due to conflict with recovery.'],
])('throws retriable DependencyUnavailableError for "%s"', (message) => {
- try {
- handlePostgresError(new Error(message), PostgresUse.COMMON_WRITE)
- fail('expected to throw')
- } catch (e) {
- expect(e).toBeInstanceOf(DependencyUnavailableError)
- expect((e as DependencyUnavailableError).isRetriable).toBe(true)
- }
+ expect(() => handlePostgresError(new Error(message), PostgresUse.COMMON_WRITE)).toThrow(
+ expect.objectContaining({ name: 'DependencyUnavailableError', isRetriable: true })
+ )
})
it.each([
@@ -69,4 +67,13 @@ describe('handlePostgresError', () => {
])('does not throw for non-transient error "%s"', (message) => {
expect(() => handlePostgresError(new Error(message), PostgresUse.COMMON_WRITE)).not.toThrow()
})
+
+ it.each([
+ ['could not serialize access due to concurrent update', '40001'],
+ ['canceling statement due to statement timeout', '57014'],
+ ])('does not throw for "%s", which shares SQLSTATE %s with a recovery conflict', (message, code) => {
+ const error = Object.assign(new Error(message), { code })
+
+ expect(() => handlePostgresError(error, PostgresUse.COMMON_WRITE)).not.toThrow()
+ })
})