Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

51.5 KB
Raw Blame
// Serial: deletes and recreates a shared DynamoDB table and uses a shared S3 prefix.
/**
 * Integration tests for the Recording API
 *
 * Tests the full round-trip of:
 * 1. Encrypting block data
 * 2. Decrypting block data
 * 3. Handling deleted sessions (crypto shredding)
 * 4. HTTP endpoints via supertest
 * 5. S3 + decryption pipeline
 * 6. Cache invalidation on delete
 *
 * The encryption contract runs against MemoryKeyStore and always runs. The DynamoDBKeyStore
 * block covers what only a real DynamoDB can show: the composite key, the conditional writes,
 * and the tombstones behind crypto shredding. It needs the dynamodb and objectstorage services
 * from the dev stack, so it skips with:
 *   DYNAMODB_TESTS_DISABLED=1 pnpm jest tests/session-replay/recording-api/recording-api.integration.test.ts
 */
import {
    CreateTableCommand,
    DeleteTableCommand,
    DescribeTableCommand,
    DynamoDBClient,
    ResourceNotFoundException,
} from '@aws-sdk/client-dynamodb'
import { KMSClient } from '@aws-sdk/client-kms'
import { DeleteObjectCommand, ListObjectsV2Command, PutObjectCommand, S3Client } from '@aws-sdk/client-s3'
import { Server } from 'http'
import sodium from 'libsodium-wrappers'
import snappy from 'snappy'
import supertest from 'supertest'
import express from 'ultimate-express'

import { JWT, PosthogJwtAudience } from '~/cdp/utils/jwt-utils'
import { setupExpressApp } from '~/common/api/router'
import { IngestionOutputs } from '~/common/outputs/ingestion-outputs'
import { PostgresRouter } from '~/common/utils/db/postgres'
import { parseJSON } from '~/common/utils/json-parse'
import { SodiumRecordingDecryptor, SodiumRecordingEncryptor } from '~/ingestion/pipelines/sessionreplay/shared/crypto'
import { DynamoDBKeyStore, MemoryKeyStore } from '~/ingestion/pipelines/sessionreplay/shared/keystore'
import { MemoryCachedKeyStore } from '~/ingestion/pipelines/sessionreplay/shared/keystore/cache'
import { ReplayEventsOutput, SessionFeaturesOutput } from '~/ingestion/pipelines/sessionreplay/shared/outputs'
import { RecordingApi } from '~/session-replay/recording-api/recording-api'
import { RecordingService } from '~/session-replay/recording-api/recording-service'
import { KeyStore, RecordingApiConfig, SessionKey, SessionKeyDeletedError } from '~/session-replay/recording-api/types'

import { createStubKmsClient } from './stub-kms-client'

const mockOutputs = {
    queueMessages: jest.fn().mockResolvedValue(undefined),
    produce: jest.fn().mockResolvedValue(undefined),
} as unknown as IngestionOutputs<ReplayEventsOutput | SessionFeaturesOutput>

// Dev-stack services backing the DynamoDBKeyStore block
const DYNAMODB_ENDPOINT = 'http://127.0.0.1:18010'
const S3_ENDPOINT = 'http://127.0.0.1:19000'
const S3_BUCKET = 'posthog'
const S3_PREFIX = 'session_recordings_recording_api_test'
const S3_CREDENTIALS = { accessKeyId: 'object_storage_root_user', secretAccessKey: 'object_storage_root_password' }
const KEYS_TABLE_NAME = 'session-recording-keys'
const shouldRunDynamoDBTests = process.env.DYNAMODB_TESTS_DISABLED !== '1'

const isResourceNotFoundException = (error: unknown): boolean => {
    return (
        error instanceof ResourceNotFoundException ||
        (typeof error === 'object' &&
            error !== null &&
            'name' in error &&
            (error as { name?: unknown }).name === 'ResourceNotFoundException')
    )
}

// Helper functions shared across all tests
const createBlockData = async (events: unknown[]): Promise<Buffer> => {
    const jsonlContent = events.map((event) => JSON.stringify(['window1', event])).join('\n')
    return snappy.compress(jsonlContent)
}

const parseBlockData = async (data: Buffer): Promise<[string, unknown][]> => {
    const decompressed = await snappy.uncompress(data)
    return decompressed
        .toString()
        .trim()
        .split('\n')
        .map((line) => parseJSON(line))
}

describe('Recording API encryption integration', () => {
    beforeAll(async () => {
        await sodium.ready
    })

    /**
     * Shared test suite that runs against any KeyStore implementation
     */
    const runEncryptionTests = (
        getKeyStore: () => KeyStore,
        getEncryptor: () => SodiumRecordingEncryptor,
        getDecryptor: () => SodiumRecordingDecryptor
    ) => {
        describe('encrypt and decrypt round-trip', () => {
            it('should encrypt and decrypt block data correctly', async () => {
                const keyStore = getKeyStore()
                const encryptor = getEncryptor()
                const decryptor = getDecryptor()

                const sessionId = `test-session-${Date.now()}`
                const teamId = 42
                const originalEvents = [
                    { type: 2, data: { source: 1, snapshot: { html: '<div>Hello</div>' } } },
                    { type: 3, data: { source: 2, mutations: [{ id: 1 }] } },
                ]

                // Generate key first (simulates ingestion creating the key)
                const sessionKey = await keyStore.generateKey(sessionId, teamId, 30)

                const blockData = await createBlockData(originalEvents)
                const { data: encrypted } = encryptor.encryptBlockWithKey(sessionId, teamId, blockData, sessionKey)

                // Encrypted data should be different from original
                expect(encrypted.equals(blockData)).toBe(false)

                // Encrypted data should be larger (nonce + ciphertext overhead)
                expect(encrypted.length).toBeGreaterThan(blockData.length)

                // Decrypt
                const { data: decrypted, sessionState } = await decryptor.decryptBlock(sessionId, teamId, encrypted)

                // Decrypted data should match original
                expect(decrypted.equals(blockData)).toBe(true)
                expect(sessionState).toBe('ciphertext')

                // Parse and verify events
                const events = await parseBlockData(decrypted)
                expect(events).toHaveLength(2)
                expect(events[0][1]).toEqual(originalEvents[0])
                expect(events[1][1]).toEqual(originalEvents[1])
            })

            it('should use different nonces for each encryption', async () => {
                const keyStore = getKeyStore()
                const encryptor = getEncryptor()
                const decryptor = getDecryptor()

                const sessionId = `test-nonce-${Date.now()}`
                const teamId = 42
                const blockData = await createBlockData([{ type: 2, data: { content: 'same content' } }])

                // Generate key first
                const sessionKey = await keyStore.generateKey(sessionId, teamId, 30)

                const { data: encrypted1 } = encryptor.encryptBlockWithKey(sessionId, teamId, blockData, sessionKey)
                const { data: encrypted2 } = encryptor.encryptBlockWithKey(sessionId, teamId, blockData, sessionKey)

                // Extract nonces (first NONCEBYTES of each encrypted block)
                const nonce1 = encrypted1.subarray(0, sodium.crypto_secretbox_NONCEBYTES)
                const nonce2 = encrypted2.subarray(0, sodium.crypto_secretbox_NONCEBYTES)

                // Nonces should be different
                expect(Buffer.compare(nonce1, nonce2)).not.toBe(0)

                // Both should decrypt to same content
                const result1 = await decryptor.decryptBlock(sessionId, teamId, encrypted1)
                const result2 = await decryptor.decryptBlock(sessionId, teamId, encrypted2)
                expect(result1.data.equals(result2.data)).toBe(true)
            })

            it('should use different keys for different sessions', async () => {
                const keyStore = getKeyStore()
                const encryptor = getEncryptor()
                const decryptor = getDecryptor()

                const blockData = await createBlockData([{ type: 2, data: { content: 'same content' } }])
                const timestamp = Date.now()

                // Generate keys for both sessions
                const keyA = await keyStore.generateKey(`session-a-${timestamp}`, 42, 30)
                const keyB = await keyStore.generateKey(`session-b-${timestamp}`, 42, 30)

                const { data: encrypted1 } = encryptor.encryptBlockWithKey(
                    `session-a-${timestamp}`,
                    42,
                    blockData,
                    keyA
                )
                const { data: encrypted2 } = encryptor.encryptBlockWithKey(
                    `session-b-${timestamp}`,
                    42,
                    blockData,
                    keyB
                )

                // Decrypt with correct keys
                const result1 = await decryptor.decryptBlock(`session-a-${timestamp}`, 42, encrypted1)
                const result2 = await decryptor.decryptBlock(`session-b-${timestamp}`, 42, encrypted2)

                expect(result1.data.equals(blockData)).toBe(true)
                expect(result2.data.equals(blockData)).toBe(true)

                // Cross-decryption should fail (using wrong key)
                expect(() => {
                    decryptor.decryptBlockWithKey(`session-a-${timestamp}`, 42, encrypted1, keyB)
                }).toThrow()
            })

            it('should use different keys for different teams', async () => {
                const keyStore = getKeyStore()
                const encryptor = getEncryptor()
                const decryptor = getDecryptor()

                const sessionId = `same-session-${Date.now()}`
                const blockData = await createBlockData([{ type: 2, data: { content: 'content' } }])

                // Generate keys for both teams
                const key1 = await keyStore.generateKey(sessionId, 1, 30)
                const key2 = await keyStore.generateKey(sessionId, 2, 30)

                const { data: encryptedTeam1 } = encryptor.encryptBlockWithKey(sessionId, 1, blockData, key1)
                const { data: encryptedTeam2 } = encryptor.encryptBlockWithKey(sessionId, 2, blockData, key2)

                // Decrypt with correct team
                const resultTeam1 = await decryptor.decryptBlock(sessionId, 1, encryptedTeam1)
                const resultTeam2 = await decryptor.decryptBlock(sessionId, 2, encryptedTeam2)

                expect(resultTeam1.data.equals(blockData)).toBe(true)
                expect(resultTeam2.data.equals(blockData)).toBe(true)

                // Cross-team decryption should fail
                expect(() => {
                    decryptor.decryptBlockWithKey(sessionId, 1, encryptedTeam2, key1)
                }).toThrow()
            })
        })

        describe('crypto shredding (key deletion)', () => {
            it('should throw SessionKeyDeletedError after key is deleted', async () => {
                const keyStore = getKeyStore()
                const encryptor = getEncryptor()
                const decryptor = getDecryptor()

                const sessionId = `to-be-deleted-${Date.now()}`
                const teamId = 42
                const blockData = await createBlockData([{ type: 2, data: { secret: 'sensitive data' } }])

                // Generate key first
                const sessionKey = await keyStore.generateKey(sessionId, teamId, 30)

                // Encrypt
                const { data: encrypted } = encryptor.encryptBlockWithKey(sessionId, teamId, blockData, sessionKey)

                // Verify decryption works before deletion
                const resultBefore = await decryptor.decryptBlock(sessionId, teamId, encrypted)
                expect(resultBefore.data.equals(blockData)).toBe(true)

                // Delete the key
                const deleteResult = await keyStore.deleteKey(sessionId, teamId, '[email protected]')
                expect(deleteResult).toEqual({
                    status: 'deleted',
                    deletedAt: expect.any(Number),
                    deletedBy: '[email protected]',
                })

                // Verify decryption fails after deletion
                await expect(decryptor.decryptBlock(sessionId, teamId, encrypted)).rejects.toThrow(
                    SessionKeyDeletedError
                )
            })

            it('should include deletion timestamp in result', async () => {
                const keyStore = getKeyStore()

                const sessionId = `deleted-with-timestamp-${Date.now()}`
                const teamId = 42

                // Generate key first
                await keyStore.generateKey(sessionId, teamId, 30)

                // Delete and capture time (timestamps are in seconds)
                const beforeDelete = Math.floor(Date.now() / 1000)
                await keyStore.deleteKey(sessionId, teamId, '[email protected]')
                const afterDelete = Math.floor(Date.now() / 1000) + 1

                // Get key should return deleted state with timestamp
                const result = await keyStore.getKey(sessionId, teamId)
                expect(result.sessionState).toBe('deleted')
                expect(result.deletedAt).toBeGreaterThanOrEqual(beforeDelete)
                expect(result.deletedAt).toBeLessThanOrEqual(afterDelete)
            })

            it('should throw SessionKeyDeletedError when trying to encrypt deleted session', async () => {
                const keyStore = getKeyStore()
                const encryptor = getEncryptor()

                const sessionId = `deleted-before-encrypt-${Date.now()}`
                const teamId = 42
                const blockData = await createBlockData([{ type: 2, data: { content: 'content' } }])

                // Generate and delete key
                await keyStore.generateKey(sessionId, teamId, 30)
                await keyStore.deleteKey(sessionId, teamId, '[email protected]')

                // Encryption should fail
                await expect(encryptor.encryptBlock(sessionId, teamId, blockData)).rejects.toThrow(
                    SessionKeyDeletedError
                )
            })

            it('should create tombstone when deleting non-existent key', async () => {
                const keyStore = getKeyStore()

                const result = await keyStore.deleteKey(`non-existent-${Date.now()}`, 999, '[email protected]')
                expect(result.status).toBe('deleted')
                expect(result.deletedAt).toBeDefined()
            })

            it('should handle multiple sessions with selective deletion', async () => {
                const keyStore = getKeyStore()
                const encryptor = getEncryptor()
                const decryptor = getDecryptor()

                const timestamp = Date.now()
                const sessions = [
                    `keep-1-${timestamp}`,
                    `delete-1-${timestamp}`,
                    `keep-2-${timestamp}`,
                    `delete-2-${timestamp}`,
                ]
                const teamId = 42
                const encrypted: Record<string, Buffer> = {}

                // Generate keys and encrypt all sessions
                for (const sessionId of sessions) {
                    const sessionKey = await keyStore.generateKey(sessionId, teamId, 30)
                    const blockData = await createBlockData([{ type: 2, data: { session: sessionId } }])
                    encrypted[sessionId] = encryptor.encryptBlockWithKey(sessionId, teamId, blockData, sessionKey).data
                }

                // Delete some sessions
                await keyStore.deleteKey(`delete-1-${timestamp}`, teamId, '[email protected]')
                await keyStore.deleteKey(`delete-2-${timestamp}`, teamId, '[email protected]')

                // Verify kept sessions are still decryptable
                for (const sessionId of [`keep-1-${timestamp}`, `keep-2-${timestamp}`]) {
                    const result = await decryptor.decryptBlock(sessionId, teamId, encrypted[sessionId])
                    const events = await parseBlockData(result.data)
                    expect(events[0][1]).toEqual({ type: 2, data: { session: sessionId } })
                }

                // Verify deleted sessions throw errors
                for (const sessionId of [`delete-1-${timestamp}`, `delete-2-${timestamp}`]) {
                    await expect(decryptor.decryptBlock(sessionId, teamId, encrypted[sessionId])).rejects.toThrow(
                        SessionKeyDeletedError
                    )
                }
            })
        })

        describe('large data handling', () => {
            it('should handle large block data', async () => {
                const keyStore = getKeyStore()
                const encryptor = getEncryptor()
                const decryptor = getDecryptor()

                const sessionId = `large-session-${Date.now()}`
                const teamId = 42

                // Generate key first
                const sessionKey = await keyStore.generateKey(sessionId, teamId, 30)

                // Create large events (100 events with substantial data)
                const largeEvents = Array.from({ length: 100 }, (_, i) => ({
                    type: 3,
                    data: {
                        source: 2,
                        mutations: Array.from({ length: 50 }, (_, j) => ({
                            id: i * 50 + j,
                            text: `This is mutation ${i * 50 + j} with some additional text to make it larger`,
                            attributes: { class: 'test-class', 'data-index': String(i * 50 + j) },
                        })),
                    },
                }))

                const blockData = await createBlockData(largeEvents)
                expect(blockData.length).toBeGreaterThan(50000) // Should be substantial

                const { data: encrypted } = encryptor.encryptBlockWithKey(sessionId, teamId, blockData, sessionKey)
                const result = await decryptor.decryptBlock(sessionId, teamId, encrypted)

                expect(result.data.equals(blockData)).toBe(true)

                const events = await parseBlockData(result.data)
                expect(events).toHaveLength(100)
            })
        })
    }

    // Tests with MemoryKeyStore (always run)
    describe('with MemoryKeyStore', () => {
        let keyStore: MemoryKeyStore
        let encryptor: SodiumRecordingEncryptor
        let decryptor: SodiumRecordingDecryptor

        beforeEach(async () => {
            keyStore = new MemoryKeyStore()
            await keyStore.start()

            encryptor = new SodiumRecordingEncryptor(keyStore)
            await encryptor.start()

            decryptor = new SodiumRecordingDecryptor(keyStore)
            await decryptor.start()
        })

        runEncryptionTests(
            () => keyStore,
            () => encryptor,
            () => decryptor
        )

        describe('cleartext sessions', () => {
            it('should pass through data unchanged for cleartext sessions', async () => {
                const sessionId = 'cleartext-session'
                const teamId = 42
                const blockData = await createBlockData([{ type: 2, data: { content: 'not encrypted' } }])

                // Manually set a cleartext key
                const cleartextKey: SessionKey = {
                    plaintextKey: Buffer.alloc(0),
                    encryptedKey: Buffer.alloc(0),
                    sessionState: 'cleartext',
                }

                const encryptResult = encryptor.encryptBlockWithKey(sessionId, teamId, blockData, cleartextKey)
                expect(encryptResult.data.equals(blockData)).toBe(true)
                expect(encryptResult.sessionState).toBe('cleartext')

                const decryptResult = decryptor.decryptBlockWithKey(sessionId, teamId, encryptResult.data, cleartextKey)
                expect(decryptResult.data.equals(blockData)).toBe(true)
                expect(decryptResult.sessionState).toBe('cleartext')
            })
        })
    })

    const describeDynamoDB = shouldRunDynamoDBTests ? describe : describe.skip

    describeDynamoDB('with DynamoDBKeyStore', () => {
        let dynamoDBClient: DynamoDBClient
        let kmsClient: KMSClient
        let keyStore: DynamoDBKeyStore
        let encryptor: SodiumRecordingEncryptor
        let decryptor: SodiumRecordingDecryptor

        async function setupDynamoDBTable(): Promise<void> {
            try {
                await dynamoDBClient.send(new DeleteTableCommand({ TableName: KEYS_TABLE_NAME }))
                await waitForTableDeletion()
            } catch {
                // Best-effort cleanup: the table is usually absent, and how that surfaces varies.
            }

            await dynamoDBClient.send(
                new CreateTableCommand({
                    TableName: KEYS_TABLE_NAME,
                    KeySchema: [
                        { AttributeName: 'session_id', KeyType: 'HASH' },
                        { AttributeName: 'team_id', KeyType: 'RANGE' },
                    ],
                    AttributeDefinitions: [
                        { AttributeName: 'session_id', AttributeType: 'S' },
                        { AttributeName: 'team_id', AttributeType: 'N' },
                    ],
                    BillingMode: 'PAY_PER_REQUEST',
                })
            )

            await waitForTableActive()
        }

        async function waitForTableActive(): Promise<void> {
            for (let i = 0; i < 30; i++) {
                try {
                    const result = await dynamoDBClient.send(new DescribeTableCommand({ TableName: KEYS_TABLE_NAME }))
                    if (result.Table?.TableStatus === 'ACTIVE') {
                        return
                    }
                } catch {
                    // Table not ready yet
                }
                await new Promise((resolve) => setTimeout(resolve, 1000))
            }
            throw new Error('Timeout waiting for DynamoDB table to become active')
        }

        async function waitForTableDeletion(): Promise<void> {
            for (let i = 0; i < 30; i++) {
                try {
                    await dynamoDBClient.send(new DescribeTableCommand({ TableName: KEYS_TABLE_NAME }))
                } catch (error) {
                    if (isResourceNotFoundException(error)) {
                        return
                    }
                }
                await new Promise((resolve) => setTimeout(resolve, 1000))
            }
            throw new Error('Timeout waiting for DynamoDB table deletion')
        }

        beforeAll(async () => {
            dynamoDBClient = new DynamoDBClient({
                endpoint: DYNAMODB_ENDPOINT,
                region: 'us-east-1',
                credentials: { accessKeyId: 'test', secretAccessKey: 'test' },
            })

            kmsClient = await createStubKmsClient()

            await setupDynamoDBTable()
        }, 30000)

        afterAll(async () => {
            try {
                await dynamoDBClient.send(new DeleteTableCommand({ TableName: KEYS_TABLE_NAME }))
            } catch {
                // Ignore
            }

            dynamoDBClient.destroy()
            kmsClient.destroy()
        })

        beforeEach(async () => {
            keyStore = new DynamoDBKeyStore(dynamoDBClient, kmsClient)
            await keyStore.start()

            encryptor = new SodiumRecordingEncryptor(keyStore)
            await encryptor.start()

            decryptor = new SodiumRecordingDecryptor(keyStore)
            await decryptor.start()
        })

        // The encryption contract itself is a property of the encryptor, not of the keystore, so it
        // runs once against MemoryKeyStore above. What follows is only what DynamoDB decides.
        describe('DynamoDBKeyStore specific', () => {
            it('should round-trip a wrapped key through DynamoDB', async () => {
                const sessionId = `wrapped-key-${Date.now()}`
                const teamId = 1

                const generatedKey = await keyStore.generateKey(sessionId, teamId, 30)

                expect(generatedKey.sessionState).toBe('ciphertext')
                expect(generatedKey.plaintextKey.length).toBe(sodium.crypto_secretbox_KEYBYTES)
                expect(generatedKey.encryptedKey.length).toBeGreaterThan(0)

                const retrievedKey = await keyStore.getKey(sessionId, teamId)

                expect(retrievedKey.sessionState).toBe('ciphertext')
                expect(retrievedKey.plaintextKey.equals(generatedKey.plaintextKey)).toBe(true)
                expect(retrievedKey.encryptedKey.equals(generatedKey.encryptedKey)).toBe(true)
            })

            it('should return the existing key instead of overwriting on a second generate', async () => {
                const sessionId = `regenerate-${Date.now()}`
                const teamId = 50

                const firstKey = await keyStore.generateKey(sessionId, teamId, 30)
                const secondKey = await keyStore.generateKey(sessionId, teamId, 30)

                expect(secondKey.sessionState).toBe('ciphertext')
                expect(secondKey.plaintextKey.equals(firstKey.plaintextKey)).toBe(true)
                expect(secondKey.encryptedKey.equals(firstKey.encryptedKey)).toBe(true)
            })

            it('should not resurrect a shredded session on regenerate', async () => {
                const sessionId = `shredded-regenerate-${Date.now()}`
                const teamId = 60

                await keyStore.generateKey(sessionId, teamId, 30)
                await keyStore.deleteKey(sessionId, teamId, '[email protected]')

                const regenerated = await keyStore.generateKey(sessionId, teamId, 30)
                expect(regenerated.sessionState).toBe('deleted')

                const tombstone = await keyStore.getKey(sessionId, teamId)
                expect(tombstone.sessionState).toBe('deleted')
            })

            it('should return already_deleted when deleting already deleted key', async () => {
                const sessionId = `double-delete-${Date.now()}`
                const teamId = 4

                await keyStore.generateKey(sessionId, teamId, 30)
                const beforeDelete = Math.floor(Date.now() / 1000)
                await keyStore.deleteKey(sessionId, teamId, '[email protected]')
                const afterDelete = Math.floor(Date.now() / 1000) + 1

                const result = await keyStore.deleteKey(sessionId, teamId, '[email protected]')
                expect(result.status).toBe('already_deleted')
                expect(result.deletedAt).toBeGreaterThanOrEqual(beforeDelete)
                expect(result.deletedAt).toBeLessThanOrEqual(afterDelete)
            })

            it('should write a tombstone when deleting a session that has no key', async () => {
                const sessionId = `never-generated-${Date.now()}`

                const result = await keyStore.deleteKey(sessionId, 999, '[email protected]')
                expect(result.status).toBe('deleted')

                // Reading it back is the half that catches a tombstone DynamoDB accepted but
                // getKey cannot parse, which serves a 500 instead of a 410 for a deleted session.
                const tombstone = await keyStore.getKey(sessionId, 999)
                expect(tombstone.sessionState).toBe('deleted')
                expect(tombstone.deletedAt).toBe(result.deletedAt)
            })

            it('should isolate keys between teams', async () => {
                const sessionId = `shared-session-${Date.now()}`
                const team1 = 10
                const team2 = 20

                const key1 = await keyStore.generateKey(sessionId, team1, 30)
                const key2 = await keyStore.generateKey(sessionId, team2, 30)

                expect(key1.plaintextKey.equals(key2.plaintextKey)).toBe(false)

                const retrieved1 = await keyStore.getKey(sessionId, team1)
                const retrieved2 = await keyStore.getKey(sessionId, team2)

                expect(retrieved1.plaintextKey.equals(key1.plaintextKey)).toBe(true)
                expect(retrieved2.plaintextKey.equals(key2.plaintextKey)).toBe(true)
            })

            it('should not affect other teams when deleting a key', async () => {
                const sessionId = `team-isolation-${Date.now()}`
                const team1 = 30
                const team2 = 40

                await keyStore.generateKey(sessionId, team1, 30)
                await keyStore.generateKey(sessionId, team2, 30)

                await keyStore.deleteKey(sessionId, team1, '[email protected]')

                const key1 = await keyStore.getKey(sessionId, team1)
                expect(key1.sessionState).toBe('deleted')

                const key2 = await keyStore.getKey(sessionId, team2)
                expect(key2.sessionState).toBe('ciphertext')
            })
        })

        describe('S3 + decryption pipeline', () => {
            let s3Client: S3Client
            let recordingService: RecordingService

            beforeAll(() => {
                s3Client = new S3Client({
                    endpoint: S3_ENDPOINT,
                    region: 'us-east-1',
                    credentials: S3_CREDENTIALS,
                    forcePathStyle: true,
                })
            })

            afterAll(async () => {
                try {
                    // The bucket is shared with the rest of the dev stack, so clean by prefix.
                    const objects = await s3Client.send(
                        new ListObjectsV2Command({ Bucket: S3_BUCKET, Prefix: S3_PREFIX })
                    )
                    for (const obj of objects.Contents ?? []) {
                        await s3Client.send(new DeleteObjectCommand({ Bucket: S3_BUCKET, Key: obj.Key }))
                    }
                } catch {
                    // Ignore cleanup errors
                }
                s3Client.destroy()
            })

            beforeEach(() => {
                recordingService = new RecordingService(s3Client, S3_BUCKET, S3_PREFIX, keyStore, decryptor)
            })

            it('should upload encrypted block to S3 and fetch it back decrypted', async () => {
                const sessionId = `s3-roundtrip-${Date.now()}`
                const teamId = 1
                const originalEvents = [
                    { type: 2, data: { source: 1, html: '<div>test</div>' } },
                    { type: 3, data: { source: 2, mutations: [{ id: 1 }] } },
                ]

                const sessionKey = await keyStore.generateKey(sessionId, teamId, 30)
                const blockData = await createBlockData(originalEvents)
                const { data: encrypted } = encryptor.encryptBlockWithKey(sessionId, teamId, blockData, sessionKey)

                const s3Key = `${S3_PREFIX}/30d/1700000000-abcdef0123456789`
                await s3Client.send(new PutObjectCommand({ Bucket: S3_BUCKET, Key: s3Key, Body: encrypted }))

                const result = await recordingService.getBlock({
                    sessionId,
                    teamId,
                    key: s3Key,
                    startByte: 0,
                    endByte: encrypted.length - 1,
                })

                expect(result.ok).toBe(true)
                if (result.ok) {
                    expect(result.data.equals(blockData)).toBe(true)
                    const events = await parseBlockData(result.data)
                    expect(events).toHaveLength(2)
                    expect(events[0][1]).toEqual(originalEvents[0])
                }
            })

            it('should return deleted result when key has been deleted', async () => {
                const sessionId = `s3-deleted-${Date.now()}`
                const teamId = 1

                const sessionKey = await keyStore.generateKey(sessionId, teamId, 30)
                const blockData = await createBlockData([{ type: 2, data: { content: 'secret' } }])
                const { data: encrypted } = encryptor.encryptBlockWithKey(sessionId, teamId, blockData, sessionKey)

                const s3Key = `${S3_PREFIX}/30d/1700000001-abcdef0123456789`
                await s3Client.send(new PutObjectCommand({ Bucket: S3_BUCKET, Key: s3Key, Body: encrypted }))

                await keyStore.deleteKey(sessionId, teamId, '[email protected]')

                const result = await recordingService.getBlock({
                    sessionId,
                    teamId,
                    key: s3Key,
                    startByte: 0,
                    endByte: encrypted.length - 1,
                })

                expect(result).toEqual(
                    expect.objectContaining({ ok: false, error: 'deleted', deletedAt: expect.any(Number) })
                )
            })
        })
    })

    // Auth matrix exercised through the real router chain (no Localstack needed — rejections happen
    // in the middleware before the handler touches S3; the few pass-through cases use MemoryKeyStore).
    describe('HTTP auth (team + operation scoped JWT)', () => {
        const JWT_SECRET = 'itest-jwt-secret'
        const INTERNAL_SECRET = 'itest-internal-secret'
        const TEAM = 1
        const SESSION = 'auth-session-1'
        const BLOCK_KEY = 'session_recordings/30d/1764634738680-3cca0f5d3c7cc7ee'

        let keyStore: MemoryKeyStore
        let encryptor: SodiumRecordingEncryptor
        let decryptor: SodiumRecordingDecryptor
        let mockS3Send: jest.Mock
        let recordingService: RecordingService
        let servers: Server[]

        beforeEach(async () => {
            keyStore = new MemoryKeyStore()
            await keyStore.start()
            decryptor = new SodiumRecordingDecryptor(keyStore)
            await decryptor.start()
            encryptor = new SodiumRecordingEncryptor(keyStore)
            await encryptor.start()
            mockS3Send = jest.fn()
            const mockS3Client = { send: mockS3Send, destroy: jest.fn() } as unknown as S3Client
            recordingService = new RecordingService(
                mockS3Client,
                'test-bucket',
                'session_recordings',
                keyStore,
                decryptor
            )
            servers = []
        })

        afterEach(() => {
            servers.forEach((s) => s.close())
        })

        const defaultConfig: Partial<RecordingApiConfig> = {
            RECORDING_API_JWT_SECRET: JWT_SECRET,
        }

        const makeApp = async (config: Partial<RecordingApiConfig> = defaultConfig): Promise<express.Application> => {
            const api = new RecordingApi(config as RecordingApiConfig, {} as PostgresRouter, mockOutputs)
            await api.start(recordingService)
            // Mirror RecordingApiServer's real wiring: the shared-secret middleware fronts the app but
            // exempts the recording-api namespace, which then owns auth per-route. Exercises that
            // interaction (and the JSON body parser) instead of mounting the router on a bare app.
            const app = setupExpressApp({
                internalApiSecret: INTERNAL_SECRET,
                internalApiAuthExcludedPathPrefixes: ['/api/projects/'],
            })
            app.use('/', api.router())
            servers.push(app.listen(0, () => {}))
            return app
        }

        const token = (
            claims: { team_id: number; op: string },
            secret = JWT_SECRET,
            expiresIn: string | number = '5m'
        ) => new JWT(secret).sign(claims, PosthogJwtAudience.RECORDING_API, { expiresIn })

        const bearer = (t: string) => ({ Authorization: `Bearer ${t}` })

        // Set up a real fetchable block so a request that PASSES auth returns 200 (not just "not 401").
        const setupBlock = async (): Promise<{ key: string; startByte: string; endByte: string }> => {
            const sessionKey = await keyStore.generateKey(SESSION, TEAM, 30)
            const blockData = await createBlockData([{ type: 2, data: { content: 'hi' } }])
            const { data: encrypted } = encryptor.encryptBlockWithKey(SESSION, TEAM, blockData, sessionKey)
            mockS3Send.mockResolvedValue({ Body: { transformToByteArray: () => Promise.resolve(encrypted) } })
            return { key: BLOCK_KEY, startByte: '0', endByte: String(encrypted.length - 1) }
        }

        const getBlock = (app: express.Application, headers: Record<string, string>, team: number = TEAM) =>
            supertest(app)
                .get(`/api/projects/${team}/recordings/${SESSION}/block`)
                .query({ key: BLOCK_KEY, start_byte: '0', end_byte: '100' })
                .set(headers)

        const postDelete = (app: express.Application, headers: Record<string, string>, team: number = TEAM) =>
            supertest(app)
                .post(`/api/projects/${team}/recordings/delete`)
                .send({ session_ids: [SESSION], deleted_by: '[email protected]' })
                .set(headers)

        it('valid read token (matching team) reaches the handler and returns the block', async () => {
            const app = await makeApp()
            const block = await setupBlock()
            const res = await supertest(app)
                .get(`/api/projects/${TEAM}/recordings/${SESSION}/block`)
                .query({ key: block.key, start_byte: block.startByte, end_byte: block.endByte })
                .set(bearer(token({ team_id: TEAM, op: 'read' })))
            expect(res.status).toBe(200)
        })

        it('accepts a valid delete token on the delete route', async () => {
            const app = await makeApp()
            const res = await postDelete(app, bearer(token({ team_id: TEAM, op: 'delete' })))
            expect(res.status).not.toBe(401)
            expect(res.status).not.toBe(403)
        })

        it('rejects a token scoped to a different team (403)', async () => {
            const app = await makeApp()
            const res = await getBlock(app, bearer(token({ team_id: 999, op: 'read' })))
            expect(res.status).toBe(403)
        })

        it('rejects a read token on the delete route (403)', async () => {
            const app = await makeApp()
            const res = await postDelete(app, bearer(token({ team_id: TEAM, op: 'read' })))
            expect(res.status).toBe(403)
        })

        it('rejects a delete token on a read route (403)', async () => {
            const app = await makeApp()
            const res = await getBlock(app, bearer(token({ team_id: TEAM, op: 'delete' })))
            expect(res.status).toBe(403)
        })

        it.each([
            ['no token', undefined],
            ['garbage token', 'not-a-jwt'],
            ['expired token', token({ team_id: TEAM, op: 'read' }, JWT_SECRET, -60)],
            ['token signed by an unknown key', token({ team_id: TEAM, op: 'read' }, 'rogue-key')],
        ])('rejects %s with 401', async (_name, t) => {
            const app = await makeApp()
            const res = await getBlock(app, t ? bearer(t) : {})
            expect(res.status).toBe(401)
        })

        // The shared-secret middleware exempts the whole '/api/projects/' prefix, so recording-api's
        // own auth has to cover that prefix entirely. A path this router does not handle must still
        // be rejected with a 401 rather than falling through to an unauthenticated 404, otherwise a
        // route added under the prefix later would ship with no auth at all.
        it.each([
            ['GET', `/api/projects/${TEAM}/not-a-real-route`],
            ['POST', `/api/projects/${TEAM}/recordings/not-a-real-route`],
        ])('requires auth for an unhandled %s path under the exempt prefix', async (method, path) => {
            const app = await makeApp()
            const res = await (method === 'GET' ? supertest(app).get(path) : supertest(app).post(path).send({}))
            expect(res.status).toBe(401)
        })

        it('rejects the shared internal secret that the rest of the app accepts', async () => {
            const app = await makeApp()
            const res = await getBlock(app, { 'X-Internal-Api-Secret': INTERNAL_SECRET })
            expect(res.status).toBe(401)
        })

        describe('key rotation', () => {
            it('accepts a token signed by the retiring key while both keys are configured', async () => {
                const app = await makeApp({ ...defaultConfig, RECORDING_API_JWT_SECRET: 'new-key,old-key' })
                const block = await setupBlock()
                const res = await supertest(app)
                    .get(`/api/projects/${TEAM}/recordings/${SESSION}/block`)
                    .query({ key: block.key, start_byte: block.startByte, end_byte: block.endByte })
                    .set(bearer(token({ team_id: TEAM, op: 'read' }, 'old-key')))
                expect(res.status).toBe(200)
            })

            it('rejects a token signed by a retired key once it is dropped', async () => {
                const app = await makeApp({ ...defaultConfig, RECORDING_API_JWT_SECRET: 'new-key' })
                const res = await getBlock(app, bearer(token({ team_id: TEAM, op: 'read' }, 'old-key')))
                expect(res.status).toBe(401)
            })
        })
    })

    // HTTP integration tests (always run, uses MemoryKeyStore)
    describe('HTTP endpoints', () => {
        let keyStore: MemoryKeyStore
        let encryptor: SodiumRecordingEncryptor
        let decryptor: SodiumRecordingDecryptor
        let app: express.Application
        let server: Server
        let mockS3Send: jest.Mock

        beforeEach(async () => {
            keyStore = new MemoryKeyStore()
            await keyStore.start()

            decryptor = new SodiumRecordingDecryptor(keyStore)
            await decryptor.start()

            encryptor = new SodiumRecordingEncryptor(keyStore)
            await encryptor.start()

            mockS3Send = jest.fn()
            const mockS3Client = { send: mockS3Send, destroy: jest.fn() } as unknown as S3Client

            const recordingService = new RecordingService(
                mockS3Client,
                'test-bucket',
                'session_recordings',
                keyStore,
                decryptor
            )

            const recordingApi = new RecordingApi({} as RecordingApiConfig, {} as PostgresRouter, mockOutputs)
            await recordingApi.start(recordingService)

            app = express()
            app.use('/', recordingApi.router())
            server = app.listen(0, () => {})
        })

        afterEach(() => {
            server.close()
        })

        describe('GET /block', () => {
            it('should return decrypted data with correct headers', async () => {
                const sessionId = 'http-session-1'
                const teamId = 1
                const originalEvents = [{ type: 2, data: { content: 'hello' } }]

                const sessionKey = await keyStore.generateKey(sessionId, teamId, 30)
                const blockData = await createBlockData(originalEvents)
                const { data: encrypted } = encryptor.encryptBlockWithKey(sessionId, teamId, blockData, sessionKey)

                mockS3Send.mockResolvedValue({
                    Body: { transformToByteArray: () => Promise.resolve(encrypted) },
                })

                const res = await supertest(app)
                    .get(`/api/projects/${teamId}/recordings/${sessionId}/block`)
                    .query({
                        key: 'session_recordings/30d/1764634738680-3cca0f5d3c7cc7ee',
                        start_byte: '0',
                        end_byte: String(encrypted.length - 1),
                    })

                expect(res.status).toBe(200)
                expect(res.headers['content-type']).toContain('application/octet-stream')
                expect(res.headers['cache-control']).toBe('public, max-age=2592000, immutable')
                expect(Buffer.from(res.body).equals(blockData)).toBe(true)
            })

            it('should return 410 for deleted session', async () => {
                const sessionId = 'http-deleted-session'
                const teamId = 1

                const sessionKey = await keyStore.generateKey(sessionId, teamId, 30)
                const blockData = await createBlockData([{ type: 2, data: { content: 'secret' } }])
                const { data: encrypted } = encryptor.encryptBlockWithKey(sessionId, teamId, blockData, sessionKey)

                await keyStore.deleteKey(sessionId, teamId, '[email protected]')

                mockS3Send.mockResolvedValue({
                    Body: { transformToByteArray: () => Promise.resolve(encrypted) },
                })

                const res = await supertest(app)
                    .get(`/api/projects/${teamId}/recordings/${sessionId}/block`)
                    .query({
                        key: 'session_recordings/30d/1764634738680-3cca0f5d3c7cc7ee',
                        start_byte: '0',
                        end_byte: String(encrypted.length - 1),
                    })

                expect(res.status).toBe(410)
                expect(res.body.error).toBe('recording_deleted')
                expect(res.body.deleted_at).toBeDefined()
                expect(res.body.deleted_by).toBe('[email protected]')
            })

            it('should return 404 when S3 returns no body', async () => {
                mockS3Send.mockResolvedValue({ Body: null })

                const res = await supertest(app).get('/api/projects/1/recordings/session-1/block').query({
                    key: 'session_recordings/30d/1764634738680-3cca0f5d3c7cc7ee',
                    start_byte: '0',
                    end_byte: '100',
                })

                expect(res.status).toBe(404)
                expect(res.body.error).toBe('Block not found')
            })

            it('should return decompressed JSONL when decompress=true', async () => {
                const sessionId = 'http-decompress-1'
                const teamId = 1
                const originalEvents = [
                    { type: 2, data: { content: 'hello' } },
                    { type: 3, data: { content: 'world' } },
                ]

                const sessionKey = await keyStore.generateKey(sessionId, teamId, 30)
                const blockData = await createBlockData(originalEvents)
                const { data: encrypted } = encryptor.encryptBlockWithKey(sessionId, teamId, blockData, sessionKey)

                mockS3Send.mockResolvedValue({
                    Body: { transformToByteArray: () => Promise.resolve(encrypted) },
                })

                const res = await supertest(app)
                    .get(`/api/projects/${teamId}/recordings/${sessionId}/block`)
                    .query({
                        key: 'session_recordings/30d/1764634738680-3cca0f5d3c7cc7ee',
                        start_byte: '0',
                        end_byte: String(encrypted.length - 1),
                        decompress: 'true',
                    })

                expect(res.status).toBe(200)
                expect(res.headers['content-type']).toContain('application/jsonl')
                expect(res.headers['cache-control']).toBe('public, max-age=2592000, immutable')

                const lines = res.text.trim().split('\n')
                expect(lines).toHaveLength(2)
                expect(parseJSON(lines[0])[1]).toEqual(originalEvents[0])
                expect(parseJSON(lines[1])[1]).toEqual(originalEvents[1])
            })

            it('should return compressed data when decompress is not set', async () => {
                const sessionId = 'http-no-decompress-1'
                const teamId = 1
                const originalEvents = [{ type: 2, data: { content: 'compressed' } }]

                const sessionKey = await keyStore.generateKey(sessionId, teamId, 30)
                const blockData = await createBlockData(originalEvents)
                const { data: encrypted } = encryptor.encryptBlockWithKey(sessionId, teamId, blockData, sessionKey)

                mockS3Send.mockResolvedValue({
                    Body: { transformToByteArray: () => Promise.resolve(encrypted) },
                })

                const res = await supertest(app)
                    .get(`/api/projects/${teamId}/recordings/${sessionId}/block`)
                    .query({
                        key: 'session_recordings/30d/1764634738680-3cca0f5d3c7cc7ee',
                        start_byte: '0',
                        end_byte: String(encrypted.length - 1),
                    })

                expect(res.status).toBe(200)
                expect(res.headers['content-type']).toContain('application/octet-stream')
                expect(Buffer.from(res.body).equals(blockData)).toBe(true)
            })
        })
    })

    // Cache invalidation tests (always run)
    describe('cache invalidation on delete', () => {
        it('should not serve stale cached key after deletion', async () => {
            const memoryKeyStore = new MemoryKeyStore()
            await memoryKeyStore.start()
            const cachedKeyStore = new MemoryCachedKeyStore(memoryKeyStore)

            const sessionId = `cache-invalidation-${Date.now()}`
            const teamId = 1

            await cachedKeyStore.generateKey(sessionId, teamId, 30)

            // Populate cache
            const cachedKey = await cachedKeyStore.getKey(sessionId, teamId)
            expect(cachedKey.sessionState).toBe('ciphertext')

            // Delete through cache layer
            await cachedKeyStore.deleteKey(sessionId, teamId, '[email protected]')

            // Next read must return deleted, not stale cached ciphertext
            const afterDelete = await cachedKeyStore.getKey(sessionId, teamId)
            expect(afterDelete.sessionState).toBe('deleted')
            expect(afterDelete.deletedAt).toBeDefined()
        })

        it('should throw SessionKeyDeletedError when decrypting after cached key deletion', async () => {
            const memoryKeyStore = new MemoryKeyStore()
            await memoryKeyStore.start()
            const cachedKeyStore = new MemoryCachedKeyStore(memoryKeyStore)

            const decryptor = new SodiumRecordingDecryptor(cachedKeyStore)
            await decryptor.start()

            const encryptor = new SodiumRecordingEncryptor(cachedKeyStore)
            await encryptor.start()

            const sessionId = `cache-decrypt-${Date.now()}`
            const teamId = 1

            const sessionKey = await cachedKeyStore.generateKey(sessionId, teamId, 30)
            const blockData = await createBlockData([{ type: 2, data: { content: 'cached' } }])
            const { data: encrypted } = encryptor.encryptBlockWithKey(sessionId, teamId, blockData, sessionKey)

            // Populate cache via decrypt
            const result = await decryptor.decryptBlock(sessionId, teamId, encrypted)
            expect(result.data.equals(blockData)).toBe(true)

            // Delete through cache
            await cachedKeyStore.deleteKey(sessionId, teamId, '[email protected]')

            // Decrypt must fail with SessionKeyDeletedError
            await expect(decryptor.decryptBlock(sessionId, teamId, encrypted)).rejects.toThrow(SessionKeyDeletedError)
        })

        it('should invalidate through nested cache layers', async () => {
            const memoryKeyStore = new MemoryKeyStore()
            await memoryKeyStore.start()

            // Simulate two-layer cache: outer memory cache wrapping inner memory cache
            const innerCache = new MemoryCachedKeyStore(memoryKeyStore)
            const outerCache = new MemoryCachedKeyStore(innerCache)

            const sessionId = `nested-cache-${Date.now()}`
            const teamId = 1

            await outerCache.generateKey(sessionId, teamId, 30)

            // Populate both cache layers
            const key = await outerCache.getKey(sessionId, teamId)
            expect(key.sessionState).toBe('ciphertext')

            // Delete through outer cache
            await outerCache.deleteKey(sessionId, teamId, '[email protected]')

            // All layers must return deleted
            const afterDelete = await outerCache.getKey(sessionId, teamId)
            expect(afterDelete.sessionState).toBe('deleted')
        })
    })
})