:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(replay): report a deleted recording from the block listing at once (#128110)
Tue Haulundea04d4275658
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/shared/keystore/cache/redis-cache.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/shared/keystore/cache/redis-cache.test.tsindex 24fc6c7868a9..55a6ea7e2334 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/shared/keystore/cache/redis-cache.test.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/shared/keystore/cache/redis-cache.test.ts@@ -1,3 +1,4 @@+import { parseJSON } from '~/common/utils/json-parse'import { SessionBatchMetrics } from '~/ingestion/pipelines/sessionreplay/sessions/metrics'import { KeyStore, SessionKey } from '~/ingestion/pipelines/sessionreplay/shared/types'@@ -29,6 +30,8 @@ describe('RedisCachedKeyStore', () => {mockRedisClient = {get: jest.fn().mockResolvedValue(null),setex: jest.fn().mockResolvedValue('OK'),+ set: jest.fn().mockResolvedValue('OK'),+ eval: jest.fn().mockResolvedValue(1),del: jest.fn().mockResolvedValue(1),}@@ -70,10 +73,12 @@ describe('RedisCachedKeyStore', () => {expect(mockDelegate.generateKey).toHaveBeenCalledWith('session-123', 1, 30)expect(mockRedisPool.acquire).toHaveBeenCalled()- expect(mockRedisClient.setex).toHaveBeenCalledWith(+ expect(mockRedisClient.eval).toHaveBeenCalledWith(+ expect.stringContaining("'deleted'"),+ 1,'@posthog/replay/recording-key:1:session-123',- 86400,- expect.any(String)+ expect.any(String),+ 86400)expect(mockRedisPool.release).toHaveBeenCalledWith(mockRedisClient)expect(result).toEqual(mockSessionKey)@@ -85,8 +90,8 @@ describe('RedisCachedKeyStore', () => {await expect(cachedKeyStore.generateKey('session-123', 1, 30)).rejects.toThrow('Generate failed')})- it('should return the key and release Redis client if setex fails', async () => {- mockRedisClient.setex.mockRejectedValue(new Error("READONLY You can't write against a read only replica."))+ it('should return the key and release Redis client if the cache write fails', async () => {+ mockRedisClient.eval.mockRejectedValue(new Error("READONLY You can't write against a read only replica."))const result = await cachedKeyStore.generateKey('session-123', 1, 30)@@ -119,16 +124,18 @@ describe('RedisCachedKeyStore', () => {const result = await cachedKeyStore.getKey('session-123', 1)expect(mockDelegate.getKey).toHaveBeenCalledWith('session-123', 1)- expect(mockRedisClient.setex).toHaveBeenCalledWith(+ expect(mockRedisClient.set).toHaveBeenCalledWith('@posthog/replay/recording-key:1:session-123',+ expect.any(String),+ 'EX',86400,- expect.any(String)+ 'NX')expect(result).toEqual(mockSessionKey)})it.each([- ['setex rejects', () => mockRedisClient.setex.mockRejectedValue(new Error('READONLY'))],+ ['set rejects', () => mockRedisClient.set.mockRejectedValue(new Error('READONLY'))],['pool acquire rejects',() =>@@ -195,6 +202,86 @@ describe('RedisCachedKeyStore', () => {expect(result).toEqual({ status: 'deleted', deletedAt: 1700000000, deletedBy: '[email protected]' })})+ const useInMemoryRedis = (): void => {+ const store = new Map<string, string>()+ mockRedisClient.get.mockImplementation((key: string) => Promise.resolve(store.get(key) ?? null))+ mockRedisClient.del.mockImplementation((key: string) => Promise.resolve(store.delete(key) ? 1 : 0))+ mockRedisClient.setex.mockImplementation((key: string, _ttl: number, value: string) => {+ store.set(key, value)+ return Promise.resolve('OK')+ })+ mockRedisClient.set.mockImplementation((key: string, value: string, ...flags: unknown[]) => {+ if (flags.includes('NX') && store.has(key)) {+ return Promise.resolve(null)+ }+ store.set(key, value)+ return Promise.resolve('OK')+ })+ // The unless-deleted script: write unless the entry holds a deletion marker.+ mockRedisClient.eval.mockImplementation((_script: string, _n: number, key: string, value: string) => {+ const current = store.get(key)+ if (current && parseJSON(current).sessionState === 'deleted') {+ return Promise.resolve(0)+ }+ store.set(key, value)+ return Promise.resolve(1)+ })+ }++ it('keeps the deletion when a lookup that read the live key finishes after the delete', async () => {+ useInMemoryRedis()+ let releaseLookup: (key: SessionKey) => void = () => {}+ mockDelegate.getKey.mockImplementation(() => new Promise((resolve) => (releaseLookup = resolve)))++ const lookup = cachedKeyStore.getKey('session-123', 1)+ await new Promise((resolve) => setImmediate(resolve))+ await cachedKeyStore.deleteKey('session-123', 1, '[email protected]')+ releaseLookup(mockSessionKey)+ await lookup++ mockDelegate.getKey.mockClear()+ const after = await cachedKeyStore.getKey('session-123', 1)+ expect(after.sessionState).toBe('deleted')+ expect(mockDelegate.getKey).not.toHaveBeenCalled()+ })++ it('keeps the deletion when a key generation finishes caching after the delete', async () => {+ useInMemoryRedis()+ let releaseGeneration: (key: SessionKey) => void = () => {}+ mockDelegate.generateKey.mockImplementation(() => new Promise((resolve) => (releaseGeneration = resolve)))++ const generation = cachedKeyStore.generateKey('session-123', 1, 30)+ await new Promise((resolve) => setImmediate(resolve))+ await cachedKeyStore.deleteKey('session-123', 1, '[email protected]')+ releaseGeneration(mockSessionKey)+ await generation++ mockDelegate.getKey.mockClear()+ const after = await cachedKeyStore.getKey('session-123', 1)+ expect(after.sessionState).toBe('deleted')+ expect(mockDelegate.getKey).not.toHaveBeenCalled()+ })++ it('lets a new key replace a cached missing key', async () => {+ useInMemoryRedis()+ mockDelegate.getKey.mockResolvedValue({ ...mockSessionKey, sessionState: 'cleartext' })+ await cachedKeyStore.getKey('session-123', 1)++ await cachedKeyStore.generateKey('session-123', 1, 30)++ mockDelegate.getKey.mockClear()+ expect((await cachedKeyStore.getKey('session-123', 1)).sessionState).toBe('ciphertext')+ })++ it('reports a committed deletion even when the cache cannot record it', async () => {+ mockRedisClient.setex.mockRejectedValue(new Error('READONLY'))++ const result = await cachedKeyStore.deleteKey('session-123', 1, '[email protected]')++ expect(result).toEqual({ status: 'deleted', deletedAt: 1700000000, deletedBy: '[email protected]' })+ expect(SessionBatchMetrics.incrementKeystoreRedisWriteErrors).toHaveBeenCalled()+ })+it('should clear cache even if delegate returns already_deleted', async () => {mockDelegate.deleteKey.mockResolvedValue({status: 'already_deleted',@@ -234,7 +321,6 @@ describe('RedisCachedKeyStore', () => {// Should only delete team 1's cache keyexpect(mockRedisClient.del).toHaveBeenCalledWith('@posthog/replay/recording-key:1:session-123')expect(mockRedisClient.del).not.toHaveBeenCalledWith('@posthog/replay/recording-key:2:session-123')- expect(mockRedisClient.del).toHaveBeenCalledTimes(1)})})@@ -252,10 +338,12 @@ describe('RedisCachedKeyStore', () => {await customTtlCachedKeyStore.generateKey('session-123', 1, 30)- expect(mockRedisClient.setex).toHaveBeenCalledWith(+ expect(mockRedisClient.eval).toHaveBeenCalledWith(+ expect.any(String),+ 1,'@posthog/replay/recording-key:1:session-123',- 3600,- expect.any(String)+ expect.any(String),+ 3600)})})diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/shared/keystore/cache/redis-cache.ts b/nodejs/src/ingestion/pipelines/sessionreplay/shared/keystore/cache/redis-cache.tsindex 543a30fda4a5..ac25726c659f 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/shared/keystore/cache/redis-cache.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/shared/keystore/cache/redis-cache.ts@@ -10,6 +10,13 @@ import { RedisPool } from '~/types'const CACHE_KEY_PREFIX = '@posthog/replay/recording-key'const REDIS_CACHE_TTL_SECONDS = 60 * 60 * 24 // 24 hours+// Sets the key unless the entry already holds a deletion marker, atomically, so no write can bring a deleted key back.+const SET_UNLESS_DELETED_SCRIPT = `local v = redis.call('GET', KEYS[1]) if v then local ok, d = pcall(cjson.decode, v) if ok and d.sessionState == 'deleted' then return 0 end end redis.call('SET', KEYS[1], ARGV[1], 'EX', ARGV[2]) return 1`++// How a write treats the entry already cached. A lookup fills only an empty entry, a new key replaces anything+// but a deletion marker, and a deletion marker replaces everything.+type CacheWriteMode = 'if_absent' | 'unless_deleted' | 'always'+export class RedisCachedKeyStore implements KeyStore {constructor(private readonly delegate: KeyStore,@@ -38,12 +45,21 @@ export class RedisCachedKeyStore implements KeyStore {}// Best effort: the caller already has the key, so a failed write (e.g. READONLY after a failover) must not fail the request.- private async setCached(sessionId: string, teamId: number, key: SessionKey): Promise<void> {+ // A lookup or a key generation that started before a delete can finish after it, so neither may replace the marker.+ private async setCached(sessionId: string, teamId: number, key: SessionKey, mode: CacheWriteMode): Promise<void> {const startTime = performance.now()try {const client = await this.redisPool.acquire()try {- await client.setex(this.cacheKey(sessionId, teamId), this.ttlSeconds, serializeSessionKey(key))+ const cacheKey = this.cacheKey(sessionId, teamId)+ const value = serializeSessionKey(key)+ if (mode === 'if_absent') {+ await client.set(cacheKey, value, 'EX', this.ttlSeconds, 'NX')+ } else if (mode === 'unless_deleted') {+ await client.eval(SET_UNLESS_DELETED_SCRIPT, 1, cacheKey, value, this.ttlSeconds)+ } else {+ await client.setex(cacheKey, this.ttlSeconds, value)+ }} finally {await this.redisPool.release(client)}@@ -68,7 +84,7 @@ export class RedisCachedKeyStore implements KeyStore {async generateKey(sessionId: string, teamId: number, retentionDays: number): Promise<SessionKey> {const key = await this.delegate.generateKey(sessionId, teamId, retentionDays)- await this.setCached(sessionId, teamId, key)+ await this.setCached(sessionId, teamId, key, 'unless_deleted')return key}@@ -79,14 +95,28 @@ export class RedisCachedKeyStore implements KeyStore {}const key = await this.delegate.getKey(sessionId, teamId)- await this.setCached(sessionId, teamId, key)+ await this.setCached(sessionId, teamId, key, 'if_absent')return key}async deleteKey(sessionId: string, teamId: number, deletedBy: string): Promise<DeleteKeyResult> {// Clear cache first to ensure stale data isn't servedawait this.deleteCached(sessionId, teamId)- return this.delegate.deleteKey(sessionId, teamId, deletedBy)+ const result = await this.delegate.deleteKey(sessionId, teamId, deletedBy)+ // The deletion has committed, so a cache error must not report it as failed: setCached only logs and counts it.+ await this.setCached(+ sessionId,+ teamId,+ {+ plaintextKey: Buffer.alloc(0),+ encryptedKey: Buffer.alloc(0),+ sessionState: 'deleted',+ deletedAt: result.deletedAt,+ deletedBy: result.deletedBy,+ },+ 'always'+ )+ return result}stop(): void {diff --git a/nodejs/src/session-replay/recording-api/recording-api.test.ts b/nodejs/src/session-replay/recording-api/recording-api.test.tsindex 7aae40bcd646..11e95381f547 100644--- a/nodejs/src/session-replay/recording-api/recording-api.test.ts+++ b/nodejs/src/session-replay/recording-api/recording-api.test.ts@@ -103,6 +103,7 @@ describe('RecordingApi', () => {formatS3KeyError: jest.fn().mockReturnValue('Invalid key format'),getBlock: jest.fn(),listBlocks: jest.fn(),+ getDeletion: jest.fn().mockResolvedValue(null),deleteRecordings: jest.fn(),encryptBlocks: jest.fn(),} as unknown as jest.Mocked<RecordingService>@@ -428,6 +429,79 @@ describe('RecordingApi', () => {expect(mockService.listBlocks).toHaveBeenCalledWith('session-123', 1)})+ it('returns 410 for a listed session once its key is shredded', async () => {+ mockService.listBlocks.mockResolvedValue([+ {+ key: 'session_recordings/30d/1000-aaa',+ start_byte: 0,+ end_byte: 100,+ start_timestamp: '2024-01-01T00:00:00Z',+ end_timestamp: '2024-01-01T00:01:00Z',+ },+ ])+ mockService.getDeletion.mockResolvedValue({+ ok: false,+ error: 'deleted',+ deletedAt: 1700000000,+ deletedBy: '[email protected]',+ })++ const res = await supertest(app).get('/api/projects/1/recordings/session-123/blocks')++ expect(res.status).toBe(410)+ expect(res.body).toEqual({+ error: 'recording_deleted',+ deleted_at: 1700000000,+ deleted_by: '[email protected]',+ })+ expect(mockService.getDeletion).toHaveBeenCalledWith('session-123', 1)+ })++ it('still lists blocks when the key store does not answer', async () => {+ mockService.listBlocks.mockResolvedValue([+ {+ key: 'session_recordings/30d/1000-aaa',+ start_byte: 0,+ end_byte: 100,+ start_timestamp: '2024-01-01T00:00:00Z',+ end_timestamp: '2024-01-01T00:01:00Z',+ },+ ])+ mockService.getDeletion.mockReturnValue(new Promise(() => {}))++ const res = await supertest(app).get('/api/projects/1/recordings/session-123/blocks')++ expect(res.status).toBe(200)+ expect(res.body.blocks).toHaveLength(1)+ })++ it('does not look up the key of a session with no blocks', async () => {+ mockService.listBlocks.mockResolvedValue([])++ const res = await supertest(app).get('/api/projects/1/recordings/session-123/blocks')++ expect(res.status).toBe(200)+ expect(mockService.getDeletion).not.toHaveBeenCalled()+ })++ it('still lists blocks when the key store fails', async () => {+ mockService.listBlocks.mockResolvedValue([+ {+ key: 'session_recordings/30d/1000-aaa',+ start_byte: 0,+ end_byte: 100,+ start_timestamp: '2024-01-01T00:00:00Z',+ end_timestamp: '2024-01-01T00:01:00Z',+ },+ ])+ mockService.getDeletion.mockRejectedValue(new Error('DynamoDB unavailable'))++ const res = await supertest(app).get('/api/projects/1/recordings/session-123/blocks')++ expect(res.status).toBe(200)+ expect(res.body.blocks).toHaveLength(1)+ })+it('should return empty blocks when session not found', async () => {mockService.listBlocks.mockResolvedValue([])diff --git a/nodejs/src/session-replay/recording-api/recording-api.ts b/nodejs/src/session-replay/recording-api/recording-api.tsindex 688ae4149a04..185f98cdb530 100644--- a/nodejs/src/session-replay/recording-api/recording-api.ts+++ b/nodejs/src/session-replay/recording-api/recording-api.ts@@ -18,7 +18,7 @@ import { HealthCheckResult, HealthCheckResultError, HealthCheckResultOk, PluginSimport { RecordingApiOp, createRecordingApiAuthMiddleware } from './auth'import { ClickHouseCredential } from './clickhouse-credential'-import { RecordingService } from './recording-service'+import { RecordingDeletedResult, RecordingService } from './recording-service'import {DeleteRecordingsBodySchema,EncryptBlocksBodySchema,@@ -28,6 +28,12 @@ import {} from './schemas'import { KeyStore, RecordingApiConfig, RecordingDecryptor, RecordingEncryptor } from './types'+const DELETION_CHECK_TIMEOUT_MS = 1000++function sendRecordingDeleted(res: express.Response, deletion: RecordingDeletedResult): void {+ res.status(410).json({ error: 'recording_deleted', deleted_at: deletion.deletedAt, deleted_by: deletion.deletedBy })+}+export class RecordingApi {private s3Client: S3Client | null = nullprivate s3Bucket: string | null = null@@ -275,11 +281,7 @@ export class RecordingApi {// Serialize responseif (!result.ok) {if (result.error === 'deleted') {- res.status(410).json({- error: 'recording_deleted',- deleted_at: result.deletedAt,- deleted_by: result.deletedBy,- })+ sendRecordingDeleted(res, result)return}res.status(404).json({ error: 'Block not found' })@@ -321,6 +323,12 @@ export class RecordingApi {try {const blocks = await this.recordingService.listBlocks(sessionId, teamId)+ const deletion =+ blocks.length > 0 ? await this.deletionOrNull(this.recordingService, sessionId, teamId) : null+ if (deletion) {+ sendRecordingDeleted(res, deletion)+ return+ }res.json({ blocks })} catch (error) {logger.error('[RecordingApi] Error listing blocks', {@@ -333,6 +341,30 @@ export class RecordingApi {}}+ // The listing only reports a deletion early, so a key store outage or stall must not stop it: getBlock still returns 410.+ private async deletionOrNull(+ service: RecordingService,+ sessionId: string,+ teamId: number+ ): Promise<RecordingDeletedResult | null> {+ let timer: NodeJS.Timeout | undefined+ const timeout = new Promise<never>((_, reject) => {+ timer = setTimeout(() => reject(new Error('Deletion check timed out')), DELETION_CHECK_TIMEOUT_MS)+ })+ try {+ return await Promise.race([service.getDeletion(sessionId, teamId), timeout])+ } catch (error) {+ logger.warn('[RecordingApi] Could not check deletion while listing blocks', {+ error: serializeError(error),+ teamId,+ sessionId,+ })+ return null+ } finally {+ clearTimeout(timer)+ }+ }+private encryptBlocks = async (req: express.Request, res: express.Response): Promise<void> => {const paramsResult = RecordingParamsSchema.safeParse(req.params)if (!paramsResult.success) {@@ -370,11 +402,7 @@ export class RecordingApi {res.status(503).json({ error: 'key_unavailable' })return}- res.status(410).json({- error: 'recording_deleted',- deleted_at: result.deletedAt,- deleted_by: result.deletedBy,- })+ sendRecordingDeleted(res, result)return}res.json({ blocks: result.blocks.map((b) => b.toString('base64')) })diff --git a/nodejs/src/session-replay/recording-api/recording-service.test.ts b/nodejs/src/session-replay/recording-api/recording-service.test.tsindex 6638fb937255..59fdf0255fa4 100644--- a/nodejs/src/session-replay/recording-api/recording-service.test.ts+++ b/nodejs/src/session-replay/recording-api/recording-service.test.ts@@ -11,6 +11,7 @@ import * as envUtils from '~/common/utils/env-utils'import { SodiumRecordingDecryptor, SodiumRecordingEncryptor } from '~/ingestion/pipelines/sessionreplay/shared/crypto'import { SessionFeatureStore } from '~/ingestion/pipelines/sessionreplay/shared/features/session-feature-store'import { SessionMetadataStore } from '~/ingestion/pipelines/sessionreplay/shared/metadata/session-metadata-store'+import { SessionKey } from '~/ingestion/pipelines/sessionreplay/shared/types'import { ClickHouseCredential } from './clickhouse-credential'import { RecordingService } from './recording-service'@@ -370,6 +371,38 @@ describe('RecordingService', () => {})})+ describe('getDeletion', () => {+ const key = (sessionState: 'ciphertext' | 'cleartext' | 'deleted'): SessionKey => ({+ plaintextKey: Buffer.alloc(0),+ encryptedKey: Buffer.alloc(0),+ sessionState,+ })++ it('reports a shredded key with who deleted it and when', async () => {+ mockKeyStore.getKey.mockResolvedValue({+ ...key('deleted'),+ deletedAt: 1700000000,+ deletedBy: '[email protected]',+ })++ expect(await service.getDeletion('sess-1', 1)).toEqual({+ ok: false,+ error: 'deleted',+ deletedAt: 1700000000,+ deletedBy: '[email protected]',+ })+ })++ it.each([+ ['an encrypted', 'ciphertext' as const],+ ['a pre-encryption', 'cleartext' as const],+ ])('reports nothing for %s recording', async (_, sessionState) => {+ mockKeyStore.getKey.mockResolvedValue(key(sessionState))++ expect(await service.getDeletion('sess-1', 1)).toBeNull()+ })+ })+describe('listBlocks', () => {function mockClickhouseResult(rows: any[]): void {mockClickhouse.query.mockResolvedValue({diff --git a/nodejs/src/session-replay/recording-api/recording-service.ts b/nodejs/src/session-replay/recording-api/recording-service.tsindex 761f42ee3e12..e8f4422eff14 100644--- a/nodejs/src/session-replay/recording-api/recording-service.ts+++ b/nodejs/src/session-replay/recording-api/recording-service.ts@@ -12,7 +12,14 @@ import { SessionMetadataStore } from '~/ingestion/pipelines/sessionreplay/sharedimport { ClickHouseCredential } from './clickhouse-credential'import { RecordingApiMetrics } from './metrics'-import { KeyStore, RecordingBlock, RecordingDecryptor, RecordingEncryptor, SessionKeyDeletedError } from './types'+import {+ KeyStore,+ RecordingBlock,+ RecordingDecryptor,+ RecordingEncryptor,+ SessionKey,+ SessionKeyDeletedError,+} from './types'export interface GetBlockParams {sessionId: string@@ -45,6 +52,13 @@ interface BlockListingRow {block_urls: string[]}+function deletionOf(sessionKey: SessionKey): RecordingDeletedResult | null {+ if (sessionKey.sessionState !== 'deleted') {+ return null+ }+ return { ok: false, error: 'deleted', deletedAt: sessionKey.deletedAt, deletedBy: sessionKey.deletedBy ?? '' }+}+export class RecordingService {constructor(private s3Client: S3Client,@@ -83,13 +97,9 @@ export class RecordingService {if (sessionKey.sessionState === 'cleartext' && isCloud()) {return { ok: false, error: 'key_unavailable' }}- if (sessionKey.sessionState === 'deleted') {- return {- ok: false,- error: 'deleted',- deletedAt: sessionKey.deletedAt,- deletedBy: sessionKey.deletedBy ?? '',- }+ const deletion = deletionOf(sessionKey)+ if (deletion) {+ return deletion}return {ok: true,@@ -243,6 +253,13 @@ export class RecordingService {return rows.length > 0}+ // The key store records a deletion at once, while the is_deleted marker reaches ClickHouse later through Kafka.+ // Call it only for a session that has blocks: the key store caches a missing key as cleartext for a day, and a+ // lookup before ingestion creates the key would make getBlock skip decryption for that whole day.+ async getDeletion(sessionId: string, teamId: number): Promise<RecordingDeletedResult | null> {+ return deletionOf(await this.keyStore.getKey(sessionId, teamId))+ }+async listBlocks(sessionId: string, teamId: number): Promise<RecordingBlock[]> {if (!this.clickhouse) {throw new Error('ClickHouse client not initialized')