Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(replay): make keystore redis cache writes best effort (#119442)

posthog[bot]a6552507291d
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/sessions/metrics.ts b/nodejs/src/ingestion/pipelines/sessionreplay/sessions/metrics.ts
index a9516e83e526..b4fd674084ff 100644
--- a/nodejs/src/ingestion/pipelines/sessionreplay/sessions/metrics.ts
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/sessions/metrics.ts
@@ -188,6 +188,11 @@ export class SessionBatchMetrics {
buckets: [0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10],
})
+ private static readonly keystoreRedisWriteErrors = new Counter({
+ name: 'recording_blob_ingestion_v2_keystore_redis_write_errors_total',
+ help: 'Number of keystore Redis cache writes that failed (the key is still returned)',
+ })
+
private static readonly retentionRedisLatency = new Histogram({
name: 'recording_blob_ingestion_v2_retention_redis_latency_seconds',
help: 'Time taken for retention service Redis calls (acquire + GET/SET + release)',
@@ -316,6 +321,10 @@ export class SessionBatchMetrics {
this.keystoreRedisLatency.observe(seconds)
}
+ public static incrementKeystoreRedisWriteErrors(count: number = 1): void {
+ this.keystoreRedisWriteErrors.inc(count)
+ }
+
public static observeRetentionRedisLatency(seconds: number): void {
this.retentionRedisLatency.observe(seconds)
}
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/shared/keystore/cache/redis-cache.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/shared/keystore/cache/redis-cache.test.ts
index a567a06e80d7..24fc6c7868a9 100644
--- a/nodejs/src/ingestion/pipelines/sessionreplay/shared/keystore/cache/redis-cache.test.ts
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/shared/keystore/cache/redis-cache.test.ts
@@ -1,3 +1,4 @@
+import { SessionBatchMetrics } from '~/ingestion/pipelines/sessionreplay/sessions/metrics'
import { KeyStore, SessionKey } from '~/ingestion/pipelines/sessionreplay/shared/types'
import { RedisCachedKeyStore } from './redis-cache'
@@ -5,6 +6,7 @@ import { RedisCachedKeyStore } from './redis-cache'
jest.mock('~/ingestion/pipelines/sessionreplay/sessions/metrics', () => ({
SessionBatchMetrics: {
observeKeystoreRedisLatency: jest.fn(),
+ incrementKeystoreRedisWriteErrors: jest.fn(),
},
}))
@@ -83,11 +85,14 @@ describe('RedisCachedKeyStore', () => {
await expect(cachedKeyStore.generateKey('session-123', 1, 30)).rejects.toThrow('Generate failed')
})
- it('should release Redis client even if setex fails', async () => {
- mockRedisClient.setex.mockRejectedValue(new Error('Redis setex failed'))
+ it('should return the key and release Redis client if setex fails', async () => {
+ mockRedisClient.setex.mockRejectedValue(new Error("READONLY You can't write against a read only replica."))
- await expect(cachedKeyStore.generateKey('session-123', 1, 30)).rejects.toThrow('Redis setex failed')
+ const result = await cachedKeyStore.generateKey('session-123', 1, 30)
+
+ expect(result).toEqual(mockSessionKey)
expect(mockRedisPool.release).toHaveBeenCalledWith(mockRedisClient)
+ expect(SessionBatchMetrics.incrementKeystoreRedisWriteErrors).toHaveBeenCalled()
})
})
@@ -122,6 +127,25 @@ describe('RedisCachedKeyStore', () => {
expect(result).toEqual(mockSessionKey)
})
+ it.each([
+ ['setex rejects', () => mockRedisClient.setex.mockRejectedValue(new Error('READONLY'))],
+ [
+ 'pool acquire rejects',
+ () =>
+ mockRedisPool.acquire
+ .mockResolvedValueOnce(mockRedisClient)
+ .mockRejectedValueOnce(new Error('acquire timeout')),
+ ],
+ ])('should return the delegate key on Redis miss when %s', async (_, setup) => {
+ mockRedisClient.get.mockResolvedValue(null)
+ setup()
+
+ const result = await cachedKeyStore.getKey('session-123', 1)
+
+ expect(mockDelegate.getKey).toHaveBeenCalledWith('session-123', 1)
+ expect(result).toEqual(mockSessionKey)
+ })
+
it('should propagate delegate getKey error', async () => {
mockRedisClient.get.mockResolvedValue(null)
mockDelegate.getKey.mockRejectedValue(new Error('Get failed'))
@@ -197,6 +221,13 @@ describe('RedisCachedKeyStore', () => {
expect(mockRedisClient.del).toHaveBeenCalledWith('@posthog/replay/recording-key:1:session-123')
})
+ it('should fail and not call delegate if cache clear fails', async () => {
+ mockRedisClient.del.mockRejectedValue(new Error('READONLY'))
+
+ await expect(cachedKeyStore.deleteKey('session-123', 1, '[email protected]')).rejects.toThrow('READONLY')
+ expect(mockDelegate.deleteKey).not.toHaveBeenCalled()
+ })
+
it('should only clear cache for the specified team', async () => {
await cachedKeyStore.deleteKey('session-123', 1, '[email protected]')
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/shared/keystore/cache/redis-cache.ts b/nodejs/src/ingestion/pipelines/sessionreplay/shared/keystore/cache/redis-cache.ts
index b4dc2046ae80..543a30fda4a5 100644
--- a/nodejs/src/ingestion/pipelines/sessionreplay/shared/keystore/cache/redis-cache.ts
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/shared/keystore/cache/redis-cache.ts
@@ -1,3 +1,4 @@
+import { logger, serializeError } from '~/common/utils/logger'
import { SessionBatchMetrics } from '~/ingestion/pipelines/sessionreplay/sessions/metrics'
import {
deserializeSessionKey,
@@ -36,13 +37,20 @@ export class RedisCachedKeyStore implements KeyStore {
}
}
+ // Best effort: the caller already has the key, so a failed write (e.g. READONLY after a failover) must not fail the request.
private async setCached(sessionId: string, teamId: number, key: SessionKey): Promise<void> {
const startTime = performance.now()
- const client = await this.redisPool.acquire()
try {
- await client.setex(this.cacheKey(sessionId, teamId), this.ttlSeconds, serializeSessionKey(key))
+ const client = await this.redisPool.acquire()
+ try {
+ await client.setex(this.cacheKey(sessionId, teamId), this.ttlSeconds, serializeSessionKey(key))
+ } finally {
+ await this.redisPool.release(client)
+ }
+ } catch (error) {
+ SessionBatchMetrics.incrementKeystoreRedisWriteErrors()
+ logger.warn('🔑', 'keystore_redis_cache_write_failed', { sessionId, teamId, error: serializeError(error) })
} finally {
- await this.redisPool.release(client)
SessionBatchMetrics.observeKeystoreRedisLatency((performance.now() - startTime) / 1000)
}
}