:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(nodejs): retry reads a replica cancels for a recovery conflict (#122274)
Phil Haackbd887a13484a
diff --git a/nodejs/src/cdp/consumers/cdp-hogflow-subscription-matcher.consumer.test.ts b/nodejs/src/cdp/consumers/cdp-hogflow-subscription-matcher.consumer.test.tsindex c0b27123f31b..41a105cfded7 100644--- a/nodejs/src/cdp/consumers/cdp-hogflow-subscription-matcher.consumer.test.ts+++ b/nodejs/src/cdp/consumers/cdp-hogflow-subscription-matcher.consumer.test.ts@@ -1367,10 +1367,10 @@ describe('CdpHogflowSubscriptionMatcherConsumer', () => {},},{- // handlePostgresError does not mark this transient replica error as retriable.- lookup: 'the hog flow lookup is cancelled on the read replica',+ // handlePostgresError does not mark a statement timeout as retriable.+ lookup: 'the hog flow lookup hits a statement timeout',fail: (m: MatcherUnderTest): Error => {- const error = new Error('canceling statement due to conflict with recovery')+ const error = new Error('canceling statement due to statement timeout');(m as any).hogFlowManager.getHogFlowsForTeam.mockRejectedValueOnce(error)return error},diff --git a/nodejs/src/cdp/services/managers/integration-manager.service.test.ts b/nodejs/src/cdp/services/managers/integration-manager.service.test.tsindex e73497d42e10..45f2dd712ecb 100644--- a/nodejs/src/cdp/services/managers/integration-manager.service.test.ts+++ b/nodejs/src/cdp/services/managers/integration-manager.service.test.ts@@ -1,4 +1,5 @@import { IntegrationType } from '~/cdp/types'+import { DependencyUnavailableError } from '~/common/utils/db/error'import { closeHub, createHub } from '~/common/utils/db/hub'import { PostgresUse } from '~/common/utils/db/postgres'import { createTestTeamFixture } from '~/tests/helpers/sql'@@ -58,6 +59,17 @@ describe('IntegrationManager', () => {})})+ it('retries the lookup when the read replica cancels it', async () => {+ const cancelled = new Error('Canceling statement due to conflict with recovery.')+ jest.spyOn(hub.postgres, 'query').mockRejectedValueOnce(+ new DependencyUnavailableError(cancelled.message, 'Postgres', cancelled)+ )++ const item = await manager.get(integrations[0].id)++ expect(item?.config).toEqual({ team: 'foobar' })+ })+it('updates cached integration data when integration changes', async () => {// First check - initial stateconst item = await manager.get(integrations[0].id)diff --git a/nodejs/src/cdp/services/managers/integration-manager.service.ts b/nodejs/src/cdp/services/managers/integration-manager.service.tsindex 645907b3ff05..83f3cd76267d 100644--- a/nodejs/src/cdp/services/managers/integration-manager.service.ts+++ b/nodejs/src/cdp/services/managers/integration-manager.service.ts@@ -1,7 +1,7 @@import { IntegrationType } from '~/cdp/types'import { EncryptedFields } from '~/cdp/utils/encryption-utils'import { PostgresRouter, PostgresUse } from '~/common/utils/db/postgres'-import { LazyLoader } from '~/common/utils/lazy-loader'+import { DEFAULT_LOADER_RETRY, LazyLoader } from '~/common/utils/lazy-loader'import { logger } from '~/common/utils/logger'import { PubSub } from '~/common/utils/pubsub'@@ -16,6 +16,7 @@ export class IntegrationManagerService {this.lazyLoader = new LazyLoader({name: 'integration_manager',loader: async (ids) => await this.fetchIntegrations(ids),+ loaderRetry: DEFAULT_LOADER_RETRY,})this.pubSub.on<{ integrationIds: IntegrationType['id'][] }>('reload-integrations', (message) => {logger.debug('⚡', '[PubSub] Reloading integrations!', { integrationIds: message.integrationIds })diff --git a/nodejs/src/common/utils/db/postgres.ts b/nodejs/src/common/utils/db/postgres.tsindex 030afebffdf8..152be7555eea 100644--- a/nodejs/src/common/utils/db/postgres.ts+++ b/nodejs/src/common/utils/db/postgres.ts@@ -64,6 +64,11 @@ const POSTGRES_UNAVAILABLE_ERROR_MESSAGES = ['server login has been failing', // PgBouncer cannot authenticate with upstream PG'pooler is shutting down', // PgBouncer terminating client connections during a restart'Cannot use a pool after calling end on the pool', // Shutdown ended the pool while work was still in flight+ // A hot standby cancels a read when WAL replay conflicts with it.+ // Some servers capitalize the first word of the message, so the entry holds only the words that every variant shares.+ // Servers report the conflict as SQLSTATE 40001 or 57014.+ // Both codes also cover other failures, so the list matches this phrase instead of a code.+ 'conflict with recovery',]export function isTransientPgError(err: unknown): boolean {diff --git a/nodejs/src/ingestion/common/groups/batch-writing-group-store.test.ts b/nodejs/src/ingestion/common/groups/batch-writing-group-store.test.tsindex 4c47f5b75899..43eeace73735 100644--- a/nodejs/src/ingestion/common/groups/batch-writing-group-store.test.ts+++ b/nodejs/src/ingestion/common/groups/batch-writing-group-store.test.ts@@ -9,6 +9,8 @@ import { BatchWritingGroupStore } from './batch-writing-group-store'import { groupCacheOperationsCounter } from './metrics'import { ClickhouseGroupRepository } from '~/common/groups/repositories/clickhouse-group-repository'import { GroupCreateResult, GroupRepository } from '~/common/groups/repositories/group-repository.interface'+import { PostgresGroupRepository } from '~/common/groups/repositories/postgres-group-repository'+import { PostgresRouter, PostgresUse } from '~/common/utils/db/postgres'import { GroupsOutput } from '~/common/outputs'import { IngestionOutputs } from '~/common/outputs/ingestion-outputs'@@ -776,6 +778,21 @@ describe('BatchWritingGroupStore', () => {expect(groupRepository.fetchGroup).not.toHaveBeenCalled()expect(groupRepository.insertGroup).toHaveBeenCalledTimes(1)})++ it('resolves when the read replica cancels the prefetch read for a recovery conflict', async () => {+ // The caller does not await prefetchGroups, so a rejection here would stop the server.+ const router = new PostgresRouter({ DATABASE_URL: 'postgres://fake', POSTGRES_CONNECTION_POOL_SIZE: 1 })+ const replica = {+ query: jest.fn().mockRejectedValue(new Error('Canceling statement due to conflict with recovery.')),+ }+ ;(router as any).pools.set(PostgresUse.PERSONS_READ, replica)+ const store = new BatchWritingGroupStore(new PostgresGroupRepository(router), clickhouseGroupRepository)++ await expect(+ store.prefetchGroups([{ teamId, groupTypeIndex: 1, groupKey: 'test', batchId: 0 }])+ ).resolves.toBeUndefined()+ expect(replica.query).toHaveBeenCalledTimes(1)+ })})describe('persistent cache (concurrentBatches > 1)', () => {diff --git a/nodejs/tests/utils/db/postgres.test.ts b/nodejs/tests/utils/db/postgres.test.tsindex 589f751a3ed7..663995388513 100644--- a/nodejs/tests/utils/db/postgres.test.ts+++ b/nodejs/tests/utils/db/postgres.test.ts@@ -1,4 +1,3 @@-import { DependencyUnavailableError } from '~/common/utils/db/error'import { PostgresUse, handlePostgresError, isTransientPgError } from '~/common/utils/db/postgres'describe('isTransientPgError', () => {@@ -16,6 +15,7 @@ describe('isTransientPgError', () => {['connect ETIMEDOUT 10.0.0.1:5432'],['query_wait_timeout'],['server login has been failing, try again later (server_login_retry)'],+ ['Canceling statement due to conflict with recovery.'],])('returns true for transient error "%s"', (message) => {expect(isTransientPgError(new Error(message))).toBe(true)})@@ -52,14 +52,12 @@ describe('handlePostgresError', () => {['connect ETIMEDOUT 10.0.0.1:5432'],['query_wait_timeout'],['server login has been failing, try again later (server_login_retry)'],+ ['canceling statement due to conflict with recovery'],+ ['Canceling statement due to conflict with recovery.'],])('throws retriable DependencyUnavailableError for "%s"', (message) => {- try {- handlePostgresError(new Error(message), PostgresUse.COMMON_WRITE)- fail('expected to throw')- } catch (e) {- expect(e).toBeInstanceOf(DependencyUnavailableError)- expect((e as DependencyUnavailableError).isRetriable).toBe(true)- }+ expect(() => handlePostgresError(new Error(message), PostgresUse.COMMON_WRITE)).toThrow(+ expect.objectContaining({ name: 'DependencyUnavailableError', isRetriable: true })+ )})it.each([@@ -69,4 +67,13 @@ describe('handlePostgresError', () => {])('does not throw for non-transient error "%s"', (message) => {expect(() => handlePostgresError(new Error(message), PostgresUse.COMMON_WRITE)).not.toThrow()})++ it.each([+ ['could not serialize access due to concurrent update', '40001'],+ ['canceling statement due to statement timeout', '57014'],+ ])('does not throw for "%s", which shares SQLSTATE %s with a recovery conflict', (message, code) => {+ const error = Object.assign(new Error(message), { code })++ expect(() => handlePostgresError(error, PostgresUse.COMMON_WRITE)).not.toThrow()+ })})