:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(replay): make keystore redis cache writes best effort (#119442)
posthog[bot]a6552507291d
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/sessions/metrics.ts b/nodejs/src/ingestion/pipelines/sessionreplay/sessions/metrics.tsindex a9516e83e526..b4fd674084ff 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/sessions/metrics.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/sessions/metrics.ts@@ -188,6 +188,11 @@ export class SessionBatchMetrics {buckets: [0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10],})+ private static readonly keystoreRedisWriteErrors = new Counter({+ name: 'recording_blob_ingestion_v2_keystore_redis_write_errors_total',+ help: 'Number of keystore Redis cache writes that failed (the key is still returned)',+ })+private static readonly retentionRedisLatency = new Histogram({name: 'recording_blob_ingestion_v2_retention_redis_latency_seconds',help: 'Time taken for retention service Redis calls (acquire + GET/SET + release)',@@ -316,6 +321,10 @@ export class SessionBatchMetrics {this.keystoreRedisLatency.observe(seconds)}+ public static incrementKeystoreRedisWriteErrors(count: number = 1): void {+ this.keystoreRedisWriteErrors.inc(count)+ }+public static observeRetentionRedisLatency(seconds: number): void {this.retentionRedisLatency.observe(seconds)}diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/shared/keystore/cache/redis-cache.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/shared/keystore/cache/redis-cache.test.tsindex a567a06e80d7..24fc6c7868a9 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/shared/keystore/cache/redis-cache.test.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/shared/keystore/cache/redis-cache.test.ts@@ -1,3 +1,4 @@+import { SessionBatchMetrics } from '~/ingestion/pipelines/sessionreplay/sessions/metrics'import { KeyStore, SessionKey } from '~/ingestion/pipelines/sessionreplay/shared/types'import { RedisCachedKeyStore } from './redis-cache'@@ -5,6 +6,7 @@ import { RedisCachedKeyStore } from './redis-cache'jest.mock('~/ingestion/pipelines/sessionreplay/sessions/metrics', () => ({SessionBatchMetrics: {observeKeystoreRedisLatency: jest.fn(),+ incrementKeystoreRedisWriteErrors: jest.fn(),},}))@@ -83,11 +85,14 @@ describe('RedisCachedKeyStore', () => {await expect(cachedKeyStore.generateKey('session-123', 1, 30)).rejects.toThrow('Generate failed')})- it('should release Redis client even if setex fails', async () => {- mockRedisClient.setex.mockRejectedValue(new Error('Redis setex failed'))+ it('should return the key and release Redis client if setex fails', async () => {+ mockRedisClient.setex.mockRejectedValue(new Error("READONLY You can't write against a read only replica."))- await expect(cachedKeyStore.generateKey('session-123', 1, 30)).rejects.toThrow('Redis setex failed')+ const result = await cachedKeyStore.generateKey('session-123', 1, 30)++ expect(result).toEqual(mockSessionKey)expect(mockRedisPool.release).toHaveBeenCalledWith(mockRedisClient)+ expect(SessionBatchMetrics.incrementKeystoreRedisWriteErrors).toHaveBeenCalled()})})@@ -122,6 +127,25 @@ describe('RedisCachedKeyStore', () => {expect(result).toEqual(mockSessionKey)})+ it.each([+ ['setex rejects', () => mockRedisClient.setex.mockRejectedValue(new Error('READONLY'))],+ [+ 'pool acquire rejects',+ () =>+ mockRedisPool.acquire+ .mockResolvedValueOnce(mockRedisClient)+ .mockRejectedValueOnce(new Error('acquire timeout')),+ ],+ ])('should return the delegate key on Redis miss when %s', async (_, setup) => {+ mockRedisClient.get.mockResolvedValue(null)+ setup()++ const result = await cachedKeyStore.getKey('session-123', 1)++ expect(mockDelegate.getKey).toHaveBeenCalledWith('session-123', 1)+ expect(result).toEqual(mockSessionKey)+ })+it('should propagate delegate getKey error', async () => {mockRedisClient.get.mockResolvedValue(null)mockDelegate.getKey.mockRejectedValue(new Error('Get failed'))@@ -197,6 +221,13 @@ describe('RedisCachedKeyStore', () => {expect(mockRedisClient.del).toHaveBeenCalledWith('@posthog/replay/recording-key:1:session-123')})+ it('should fail and not call delegate if cache clear fails', async () => {+ mockRedisClient.del.mockRejectedValue(new Error('READONLY'))++ await expect(cachedKeyStore.deleteKey('session-123', 1, '[email protected]')).rejects.toThrow('READONLY')+ expect(mockDelegate.deleteKey).not.toHaveBeenCalled()+ })+it('should only clear cache for the specified team', async () => {await cachedKeyStore.deleteKey('session-123', 1, '[email protected]')diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/shared/keystore/cache/redis-cache.ts b/nodejs/src/ingestion/pipelines/sessionreplay/shared/keystore/cache/redis-cache.tsindex b4dc2046ae80..543a30fda4a5 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/shared/keystore/cache/redis-cache.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/shared/keystore/cache/redis-cache.ts@@ -1,3 +1,4 @@+import { logger, serializeError } from '~/common/utils/logger'import { SessionBatchMetrics } from '~/ingestion/pipelines/sessionreplay/sessions/metrics'import {deserializeSessionKey,@@ -36,13 +37,20 @@ export class RedisCachedKeyStore implements KeyStore {}}+ // Best effort: the caller already has the key, so a failed write (e.g. READONLY after a failover) must not fail the request.private async setCached(sessionId: string, teamId: number, key: SessionKey): Promise<void> {const startTime = performance.now()- const client = await this.redisPool.acquire()try {- await client.setex(this.cacheKey(sessionId, teamId), this.ttlSeconds, serializeSessionKey(key))+ const client = await this.redisPool.acquire()+ try {+ await client.setex(this.cacheKey(sessionId, teamId), this.ttlSeconds, serializeSessionKey(key))+ } finally {+ await this.redisPool.release(client)+ }+ } catch (error) {+ SessionBatchMetrics.incrementKeystoreRedisWriteErrors()+ logger.warn('🔑', 'keystore_redis_cache_write_failed', { sessionId, teamId, error: serializeError(error) })} finally {- await this.redisPool.release(client)SessionBatchMetrics.observeKeystoreRedisLatency((performance.now() - startTime) / 1000)}}