:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(replay): serve recording blocks to browsers by opaque ref (#128265)
Tue Haulund99e1a37cd7cb
diff --git a/nodejs/src/cdp/utils/jwt-utils.ts b/nodejs/src/cdp/utils/jwt-utils.tsindex 24193d074546..f38f0d58272b 100644--- a/nodejs/src/cdp/utils/jwt-utils.ts+++ b/nodejs/src/cdp/utils/jwt-utils.ts@@ -2,6 +2,8 @@ import jwt from 'jsonwebtoken'export enum PosthogJwtAudience {RECORDING_API = 'posthog:recording_api',+ // Must match PosthogJwtAudience.REPLAY_BLOCKS in posthog/jwt.py exactly.+ REPLAY_BLOCKS = 'posthog:replay:blocks',SUBSCRIPTION_PREFERENCES = 'posthog:messaging:subscription_preferences',WORKFLOWS_RESCHEDULE_PARKED = 'posthog:workflows:reschedule_parked',WORKFLOWS_CANCEL_INVOCATIONS = 'posthog:workflows:cancel_invocations',diff --git a/nodejs/src/common/api/body-limit.test.ts b/nodejs/src/common/api/body-limit.test.tsindex b57a1925ceb6..94dced979f92 100644--- a/nodejs/src/common/api/body-limit.test.ts+++ b/nodejs/src/common/api/body-limit.test.ts@@ -8,7 +8,7 @@ describe('setupExpressApp JSON body limit', () => {let server: ReturnType<Application['listen']>beforeAll(() => {- app = setupExpressApp({ jsonBodyLimit: '1kb' })+ app = setupExpressApp({ jsonBodyLimit: '1kb', readOnlyPathPrefixes: ['/read-only/'] })app.post('/echo', (req: Request, res: Response) => {res.status(200).json({ ok: true })})@@ -34,6 +34,14 @@ describe('setupExpressApp JSON body limit', () => {expect(res.body).toEqual({ status: 'error', error: 'Request entity too large' })})+ it('refuses a write to a read-only prefix before parsing its body', async () => {+ const res = await supertest(app)+ .post('/read-only/anything')+ .send({ data: 'x'.repeat(2048) })++ expect(res.status).toEqual(405)+ })+it('accepts a JSON body under the limit', async () => {const res = await supertest(app).post('/echo').send({ data: 'x' })diff --git a/nodejs/src/common/api/router.ts b/nodejs/src/common/api/router.tsindex b4786e17ef60..c86aac876f3b 100644--- a/nodejs/src/common/api/router.ts+++ b/nodejs/src/common/api/router.ts@@ -34,8 +34,13 @@ export interface SetupExpressAppOptions {internalApiSecretFallbacks?: string// JSON body size limit, e.g. '20mb'. Overridable in tests.jsonBodyLimit?: string+ // Path prefixes that only serve reads. Other methods get a 405 before the body parser buffers anything, so an+ // unauthenticated caller cannot make the server parse a large body for a route that never reads one.+ readOnlyPathPrefixes?: string[]}+const READ_METHODS = new Set(['GET', 'HEAD', 'OPTIONS'])+export function setupCommonRoutes(app: express.Application,services: Pick<PluginServerService, 'id' | 'healthcheck'>[]@@ -70,6 +75,18 @@ export function setupExpressApp(options: SetupExpressAppOptions = {}): express.A}))+ const readOnlyPathPrefixes = options.readOnlyPathPrefixes ?? []+ if (readOnlyPathPrefixes.length > 0) {+ app.use((req: Request, res: Response, next: NextFunction) => {+ if (!READ_METHODS.has(req.method) && readOnlyPathPrefixes.some((prefix) => req.path.startsWith(prefix))) {+ res.set('Allow', 'GET, HEAD, OPTIONS')+ res.status(405).json({ error: 'Method not allowed' })+ return+ }+ next()+ })+ }+app.use(express.json({limit: options.jsonBodyLimit ?? '20mb',diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/shared/keystore/cache/redis-cache.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/shared/keystore/cache/redis-cache.test.tsindex 55a6ea7e2334..4d2b6987c3bd 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/shared/keystore/cache/redis-cache.test.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/shared/keystore/cache/redis-cache.test.ts@@ -134,6 +134,19 @@ describe('RedisCachedKeyStore', () => {expect(result).toEqual(mockSessionKey)})+ it.each([+ ['caches', 'ciphertext' as const, 1],+ ['skips caching', 'cleartext' as const, 0],+ ])('%s a %s key when built not to cache a missing key', async (_, sessionState, expectedWrites) => {+ mockRedisClient.get.mockResolvedValue(null)+ mockDelegate.getKey.mockResolvedValue({ ...mockSessionKey, sessionState })+ const store = new RedisCachedKeyStore(mockDelegate, mockRedisPool, undefined, false)++ await store.getKey('session-123', 1)++ expect(mockRedisClient.set).toHaveBeenCalledTimes(expectedWrites)+ })+it.each([['set rejects', () => mockRedisClient.set.mockRejectedValue(new Error('READONLY'))],[diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/shared/keystore/cache/redis-cache.ts b/nodejs/src/ingestion/pipelines/sessionreplay/shared/keystore/cache/redis-cache.tsindex ac25726c659f..5492fda06215 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/shared/keystore/cache/redis-cache.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/shared/keystore/cache/redis-cache.ts@@ -21,7 +21,10 @@ export class RedisCachedKeyStore implements KeyStore {constructor(private readonly delegate: KeyStore,private readonly redisPool: RedisPool,- private readonly ttlSeconds: number = REDIS_CACHE_TTL_SECONDS+ private readonly ttlSeconds: number = REDIS_CACHE_TTL_SECONDS,+ // A reader that may ask about a session ingestion has not keyed yet, or before DynamoDB's eventually consistent+ // read sees a new key, passes false: caching that miss would pin the session to cleartext for the cache lifetime.+ private readonly cacheMissingKeys: boolean = true) {}private cacheKey(sessionId: string, teamId: number): string {@@ -95,7 +98,9 @@ export class RedisCachedKeyStore implements KeyStore {}const key = await this.delegate.getKey(sessionId, teamId)- await this.setCached(sessionId, teamId, key, 'if_absent')+ if (this.cacheMissingKeys || key.sessionState !== 'cleartext') {+ await this.setCached(sessionId, teamId, key, 'if_absent')+ }return key}diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/shared/keystore/index.ts b/nodejs/src/ingestion/pipelines/sessionreplay/shared/keystore/index.tsindex fa29a7b2a252..75d5b429b13c 100644--- a/nodejs/src/ingestion/pipelines/sessionreplay/shared/keystore/index.ts+++ b/nodejs/src/ingestion/pipelines/sessionreplay/shared/keystore/index.ts@@ -16,6 +16,8 @@ export { MemoryKeyStore } from './memory-keystore'export interface KeyStoreConfig {kmsEndpoint?: stringdynamoDBEndpoint?: string+ // Per-request deadline for the AWS calls, which then abort and retry once. Unset keeps the SDK default: no deadline.+ requestTimeoutMs?: number}export function getKeyStore(region: string, config?: KeyStoreConfig): KeyStore {@@ -26,14 +28,19 @@ export function getKeyStore(region: string, config?: KeyStoreConfig): KeyStore {dynamoDBEndpoint: config?.dynamoDBEndpoint ?? 'default',})- const kmsClient = new KMSClient({- region,- endpoint: config?.kmsEndpoint,- })- const dynamoDBClient = new DynamoDBClient({- region,- endpoint: config?.dynamoDBEndpoint,- })+ // Without throwOnRequestTimeout the SDK only logs a warning at the deadline and keeps waiting.+ const bounded = config?.requestTimeoutMs+ ? {+ requestHandler: {+ requestTimeout: config.requestTimeoutMs,+ connectionTimeout: config.requestTimeoutMs,+ throwOnRequestTimeout: true,+ },+ maxAttempts: 2,+ }+ : {}+ const kmsClient = new KMSClient({ region, endpoint: config?.kmsEndpoint, ...bounded })+ const dynamoDBClient = new DynamoDBClient({ region, endpoint: config?.dynamoDBEndpoint, ...bounded })return new DynamoDBKeyStore(dynamoDBClient, kmsClient)}diff --git a/nodejs/src/servers/recording-api-server.ts b/nodejs/src/servers/recording-api-server.tsindex 1c5cc4113beb..588e54d4d2e2 100644--- a/nodejs/src/servers/recording-api-server.ts+++ b/nodejs/src/servers/recording-api-server.ts@@ -11,7 +11,10 @@ import {KafkaSessionreplayProducerEnvConfig,getDefaultKafkaSessionreplayProducerEnvConfig,} from '~/ingestion/pipelines/sessionreplay/shared/outputs/producer-config'-import { assertRecordingApiAuthConfigured } from '~/session-replay/recording-api/auth'+import {+ assertRecordingApiRoleAuthConfigured,+ internalApiAuthExcludedPathPrefixes,+} from '~/session-replay/recording-api/auth'import { createProducerRegistry } from '~/session-replay/recording-api/outputs/producer-registry'import { createOutputsRegistry } from '~/session-replay/recording-api/outputs/registry'import { RecordingApi } from '~/session-replay/recording-api/recording-api'@@ -19,8 +22,11 @@ import {RecordingApiConfig,RecordingApiOutputsConfig,type RecordingApiProducerName,+ RecordingApiRole,getDefaultRecordingApiAuthConfig,+ getDefaultRecordingApiBlocksConfig,getDefaultRecordingApiOutputsConfig,+ parseRecordingApiRole,} from '~/session-replay/recording-api/types'import { CommonConfig } from '../common/config'@@ -44,6 +50,7 @@ export class RecordingApiServer implements NodeServer {private postgres?: PostgresRouterprivate producerRegistry?: KafkaProducerRegistry<RecordingApiProducerName>+ private readonly role: RecordingApiRoleconstructor(config: Partial<RecordingApiServerConfig> = {}) {this.config = {@@ -53,12 +60,15 @@ export class RecordingApiServer implements NodeServer {...overrideConfigWithEnv(getDefaultKafkaSessionreplayProducerEnvConfig()),...overrideConfigWithEnv(getDefaultRecordingApiOutputsConfig()),...overrideConfigWithEnv(getDefaultRecordingApiAuthConfig()),+ ...overrideConfigWithEnv(getDefaultRecordingApiBlocksConfig()),...config,}+ this.role = parseRecordingApiRole(this.config.RECORDING_API_ROLE)// recording-api verifies a team-scoped JWT per route, so exempt its routes from the// shared-secret middleware (callers send a Bearer token, not X-Internal-Api-Secret).this.lifecycle = new ServerLifecycle(this.config, {- internalApiAuthExcludedPathPrefixes: ['/api/projects/'],+ internalApiAuthExcludedPathPrefixes: internalApiAuthExcludedPathPrefixes(this.role),+ readOnlyPathPrefixes: ['/replay-blocks/'],})}@@ -74,22 +84,28 @@ export class RecordingApiServer implements NodeServer {}private async startServices(): Promise<void> {- assertRecordingApiAuthConfigured({+ assertRecordingApiRoleAuthConfigured({+ role: this.role,isProd: isProdEnv(),+ blocksJwtSecret: this.config.RECORDING_API_BLOCKS_JWT_SECRET,jwtSecret: this.config.RECORDING_API_JWT_SECRET,allowLegacySecret: this.config.RECORDING_API_ALLOW_LEGACY_SECRET,legacySecret: this.config.INTERNAL_API_SECRET,})- this.postgres = new PostgresRouter(this.config, this.config.PLUGIN_SERVER_MODE ?? undefined)- logger.info('👍', 'Postgres Router ready')+ // A public process serves only browser block reads, so it never connects to Postgres or Kafka.+ let outputs: ReturnType<ReturnType<typeof createOutputsRegistry>['build']> | undefined+ if (this.role !== 'public') {+ this.postgres = new PostgresRouter(this.config, this.config.PLUGIN_SERVER_MODE ?? undefined)+ logger.info('👍', 'Postgres Router ready')- logger.info('🤔', 'Connecting to Kafka...')- this.producerRegistry = await createProducerRegistry(this.config.KAFKA_CLIENT_RACK).build(this.config)- const outputs = createOutputsRegistry().build(this.producerRegistry, this.config)- logger.info('👍', 'Kafka ready')+ logger.info('🤔', 'Connecting to Kafka...')+ this.producerRegistry = await createProducerRegistry(this.config.KAFKA_CLIENT_RACK).build(this.config)+ outputs = createOutputsRegistry().build(this.producerRegistry, this.config)+ logger.info('👍', 'Kafka ready')+ }- const api = new RecordingApi(this.config, this.postgres!, outputs)+ const api = new RecordingApi(this.config, this.postgres, outputs)this.lifecycle.expressApp.use('/', api.router())await api.start()this.lifecycle.services.push(api.service)diff --git a/nodejs/src/session-replay/recording-api/auth.test.ts b/nodejs/src/session-replay/recording-api/auth.test.tsindex db105dca2b83..2ddec0664789 100644--- a/nodejs/src/session-replay/recording-api/auth.test.ts+++ b/nodejs/src/session-replay/recording-api/auth.test.ts@@ -2,7 +2,13 @@ import { NextFunction, Request, Response } from 'ultimate-express'import { JWT, PosthogJwtAudience } from '~/cdp/utils/jwt-utils'-import { RecordingApiAuthOptions, assertRecordingApiAuthConfigured, createRecordingApiAuthMiddleware } from './auth'+import {+ RecordingApiAuthOptions,+ assertRecordingApiAuthConfigured,+ assertRecordingApiRoleAuthConfigured,+ createRecordingApiAuthMiddleware,+ internalApiAuthExcludedPathPrefixes,+} from './auth'describe('createRecordingApiAuthMiddleware', () => {const JWT_SECRET = 'recording-api-secret-key'@@ -283,4 +289,34 @@ describe('assertRecordingApiAuthConfigured', () => {it('passes outside production even with nothing configured (local dev)', () => {expect(() => assertRecordingApiAuthConfigured({ ...base, isProd: false })).not.toThrow()})++ it.each([+ ['public', '', 'k', /RECORDING_API_BLOCKS_JWT_SECRET/],+ ['all', '', 'k', /RECORDING_API_BLOCKS_JWT_SECRET/],+ ['all', 'b', '', /no auth configured in production/],+ ['internal', '', '', /no auth configured in production/],+ ] as const)(+ 'refuses to start a %s process without the secrets it serves',+ (role, blocksJwtSecret, jwtSecret, error) => {+ expect(() => assertRecordingApiRoleAuthConfigured({ ...base, role, blocksJwtSecret, jwtSecret })).toThrow(+ error+ )+ }+ )++ it.each([+ ['public', 'b', ''],+ ['all', 'b', 'k'],+ ['internal', '', 'k'],+ ] as const)('starts a %s process holding only the secrets it serves', (role, blocksJwtSecret, jwtSecret) => {+ expect(() => assertRecordingApiRoleAuthConfigured({ ...base, role, blocksJwtSecret, jwtSecret })).not.toThrow()+ })++ it.each([+ ['internal', false],+ ['public', true],+ ['all', true],+ ] as const)('a %s process exempts the browser route from the shared secret: %s', (role, exempt) => {+ expect(internalApiAuthExcludedPathPrefixes(role).includes('/replay-blocks/')).toBe(exempt)+ })})diff --git a/nodejs/src/session-replay/recording-api/auth.ts b/nodejs/src/session-replay/recording-api/auth.tsindex f80103f88358..f79494f5946c 100644--- a/nodejs/src/session-replay/recording-api/auth.ts+++ b/nodejs/src/session-replay/recording-api/auth.ts@@ -5,6 +5,8 @@ import { JWT, PosthogJwtAudience, hasJwtKeys, makeOptionalJwt } from '~/cdp/utilimport { INTERNAL_SERVICE_CALL_HEADER_NAME } from '~/common/api/middleware/internal-api-auth'import { logger } from '~/common/utils/logger'+import { RecordingApiRole } from './types'+/*** Authorization for recording-api routes.*@@ -49,6 +51,29 @@ function pathTeamId(req: Request): number {return match && /^(0|[1-9]\d*)$/.test(match[1]) ? parseInt(match[1], 10) : NaN}+// Pinned to HS256 to match the Python minter, and logged by failure class only, never the token.+function verifyBearer(+ jwt: JWT,+ token: string,+ audience: PosthogJwtAudience,+ options: { maxAge?: string } = {}+): Record<string, unknown> | undefined {+ try {+ const payload = jwt.verify(token, audience, {+ algorithms: ['HS256'],+ clockTolerance: CLOCK_TOLERANCE_SECONDS,+ ...options,+ })+ return payload && typeof payload === 'object' ? (payload as Record<string, unknown>) : undefined+ } catch (err) {+ logger.warn('Recording API JWT verification failed', {+ audience,+ reason: err instanceof Error ? err.name : 'unknown',+ })+ return undefined+ }+}+function timingSafeEqual(a: string, b: string): boolean {// Trim so a secret mounted from a file with a trailing newline still matches a sender that// trimmed it (mirrors the shared internal-api-auth middleware's newline handling).@@ -79,6 +104,32 @@ export function assertRecordingApiAuthConfigured(opts: {}}+/**+ * Each role must hold exactly the auth it serves. A process serving the browser route needs the blocks secret in+ * production, since no other secret is accepted there. A public process holds no service secret, so it skips the+ * internal check that would otherwise demand one.+ */+export function assertRecordingApiRoleAuthConfigured(opts: {+ role: RecordingApiRole+ isProd: boolean+ blocksJwtSecret: string+ jwtSecret: string+ allowLegacySecret: boolean+ legacySecret: string+}): void {+ if (opts.role !== 'internal' && opts.isProd && !hasJwtKeys(opts.blocksJwtSecret || '')) {+ throw new Error('recording-api serving browser blocks needs RECORDING_API_BLOCKS_JWT_SECRET in production')+ }+ if (opts.role !== 'public') {+ assertRecordingApiAuthConfigured(opts)+ }+}++// The browser block route checks its own browser token, so only a process serving it exempts it from the shared secret.+export function internalApiAuthExcludedPathPrefixes(role: RecordingApiRole): string[] {+ return ['/api/projects/', ...(role === 'internal' ? [] : ['/replay-blocks/'])]+}+export function createRecordingApiAuthMiddleware(options: RecordingApiAuthOptions) {const { jwtSecret, legacySecret, allowLegacySecret, op } = options// Build the verifier once. Comma-separated keys give zero-downtime rotation (verify against all);@@ -98,25 +149,9 @@ export function createRecordingApiAuthMiddleware(options: RecordingApiAuthOptionif (jwt) {const bearer = getBearerToken(req)if (bearer) {- let payload: ReturnType<JWT['verify']>- try {- payload = jwt.verify(bearer, PosthogJwtAudience.RECORDING_API, {- // Pin HS256 to match the Python minter (encode/decode pin JWT_ALGORITHM);- // don't accept other HMAC variants the library would otherwise allow.- algorithms: ['HS256'],- clockTolerance: CLOCK_TOLERANCE_SECONDS,- })- } catch (err) {- // Log the failure class (e.g. TokenExpiredError, JsonWebTokenError) — never the token —- // so rollout/rotation 401s are diagnosable. Then fall through to the legacy secret or 401.- logger.warn('Recording API JWT verification failed', {- reason: err instanceof Error ? err.name : 'unknown',- path: req.path,- })- payload = undefined- }- if (payload && typeof payload === 'object') {- const claims = payload as { team_id?: unknown; op?: unknown }+ // An invalid token falls through to the legacy secret (transition) or a 401.+ const claims = verifyBearer(jwt, bearer, PosthogJwtAudience.RECORDING_API)+ if (claims) {const tokenTeamId = Number(claims.team_id)const requestTeamId = pathTeamId(req)if (!Number.isInteger(tokenTeamId) || tokenTeamId !== requestTeamId) {@@ -147,3 +182,43 @@ export function createRecordingApiAuthMiddleware(options: RecordingApiAuthOptionres.status(401).json({ error: 'Unauthorized' })}}++// A browser block token older than this is refused whatever its exp, so a leaked one stops working soon.+const REPLAY_BLOCKS_TOKEN_MAX_AGE = '2h'++// The browser block route accepts only browser tokens, a separate audience from the service tokens above.+// There is no legacy fallback and no dev bypass: without a secret, every request is refused.+export function createReplayBlocksAuthMiddleware(jwtSecret: string) {+ const jwt = makeOptionalJwt(jwtSecret)++ return (req: Request, res: Response, next: NextFunction): void => {+ // Only a served block may be cached, so every other answer on this route says no-store.+ res.set('Cache-Control', 'no-store')+ const bearer = getBearerToken(req)+ const claims =+ jwt && bearer+ ? verifyBearer(jwt, bearer, PosthogJwtAudience.REPLAY_BLOCKS, { maxAge: REPLAY_BLOCKS_TOKEN_MAX_AGE })+ : undefined+ if (!claims || typeof claims.exp !== 'number') {+ res.status(401).json({ error: 'Unauthorized' })+ return+ }+ // String equality also requires the canonical path spelling, so /01/ never shares a cache entry with /1/.+ if (+ !Number.isInteger(claims.team_id) ||+ String(claims.team_id) !== req.params.team_id ||+ claims.session_id !== req.params.session_id ||+ // The viewer is in the URL so each viewer's blocks are separate browser cache entries: after an account+ // switch the new viewer's URLs never match the previous viewer's cached blocks.+ typeof claims.sub !== 'string' ||+ !claims.sub ||+ claims.sub !== req.params.viewer+ ) {+ res.status(403).json({ error: 'Forbidden: token not scoped to this recording' })+ return+ }+ res.locals.replayBlocksCaller = claims.sub+ res.locals.replayBlocksTier = claims.tier+ next()+ }+}diff --git a/nodejs/src/session-replay/recording-api/block-refs.test.ts b/nodejs/src/session-replay/recording-api/block-refs.test.tsnew file mode 100644index 000000000000..8410372d2093--- /dev/null+++ b/nodejs/src/session-replay/recording-api/block-refs.test.ts@@ -0,0 +1,50 @@+import sodium from 'libsodium-wrappers'++import { BlockLocation, blockRefEncoder, decodeBlockRef } from './block-refs'++describe('block refs', () => {+ const sessionKey = new Uint8Array(32).fill(1)+ const location: BlockLocation = {+ teamId: 1,+ sessionId: 'session-123',+ key: 'session_recordings/30d/1764634738680-3cca0f5d3c7cc7ee',+ startByte: 100,+ endByte: 250,+ }++ beforeAll(async () => {+ await sodium.ready+ })++ it('names a block the same way every time, so its URL stays cacheable', () => {+ const ref = blockRefEncoder(sessionKey)(location)++ expect(blockRefEncoder(sessionKey)(location)).toBe(ref)+ expect(ref).not.toContain('session_recordings')+ expect(decodeBlockRef(sessionKey, ref, 1, 'session-123')).toEqual({+ key: location.key,+ startByte: 100,+ endByte: 250,+ })+ })++ it.each([+ ['another session key', (ref: string) => decodeBlockRef(new Uint8Array(32).fill(2), ref, 1, 'session-123')],+ ['another session', (ref: string) => decodeBlockRef(sessionKey, ref, 1, 'session-999')],+ ['another team', (ref: string) => decodeBlockRef(sessionKey, ref, 2, 'session-123')],+ [+ 'a tampered ref',+ (ref: string) =>+ decodeBlockRef(sessionKey, ref.slice(0, -2) + (ref.endsWith('AA') ? 'BB' : 'AA'), 1, 'session-123'),+ ],+ [+ 'an unknown version',+ (ref: string) => decodeBlockRef(sessionKey, ref.replace(/^v1\./, 'v2.'), 1, 'session-123'),+ ],+ ['an empty body', () => decodeBlockRef(sessionKey, 'v1.', 1, 'session-123')],+ ['a body shorter than its nonce', () => decodeBlockRef(sessionKey, 'v1.AAAA', 1, 'session-123')],+ ['an extra segment', (ref: string) => decodeBlockRef(sessionKey, `${ref}.x`, 1, 'session-123')],+ ])('rejects a ref read with %s', (_, decode) => {+ expect(decode(blockRefEncoder(sessionKey)(location))).toBeNull()+ })+})diff --git a/nodejs/src/session-replay/recording-api/block-refs.ts b/nodejs/src/session-replay/recording-api/block-refs.tsnew file mode 100644index 000000000000..df806c58e194--- /dev/null+++ b/nodejs/src/session-replay/recording-api/block-refs.ts@@ -0,0 +1,83 @@+import sodium from 'libsodium-wrappers'++import { parseJSON } from '~/common/utils/json-parse'++// A ref names a block without revealing its storage key: the location, encrypted with keys derived from the+// session key, under a nonce hashed from the location, so the same block always gets the same cacheable URL.+// Deriving from the session key is deliberate: shredding that key makes every ref for the session unreadable.+export interface BlockLocation {+ teamId: number+ sessionId: string+ key: string+ startByte: number+ endByte: number+}++const REF_VERSION = 'v1'+// crypto_kdf contexts are exactly 8 bytes.+const KDF_CONTEXT = 'blockref'+const ENCRYPTION_SUBKEY_ID = 1+const NONCE_SUBKEY_ID = 2++function encryptionKeyFor(sessionKey: Uint8Array): Uint8Array {+ return sodium.crypto_kdf_derive_from_key(+ sodium.crypto_secretbox_KEYBYTES,+ ENCRYPTION_SUBKEY_ID,+ KDF_CONTEXT,+ sessionKey+ )+}++function serialize(location: BlockLocation): Uint8Array {+ const { teamId, sessionId, key, startByte, endByte } = location+ return sodium.from_string(JSON.stringify([teamId, sessionId, key, startByte, endByte]))+}++// Derives the subkeys once, for naming every block of one listing. Callers must await sodium.ready first.+export function blockRefEncoder(sessionKey: Uint8Array): (location: BlockLocation) => string {+ const encryptionKey = encryptionKeyFor(sessionKey)+ const nonceKey = sodium.crypto_kdf_derive_from_key(+ sodium.crypto_generichash_KEYBYTES,+ NONCE_SUBKEY_ID,+ KDF_CONTEXT,+ sessionKey+ )+ return (location) => {+ const plaintext = serialize(location)+ const nonce = sodium.crypto_generichash(sodium.crypto_secretbox_NONCEBYTES, plaintext, nonceKey)+ const ciphertext = sodium.crypto_secretbox_easy(plaintext, nonce, encryptionKey)+ const payload = new Uint8Array(nonce.length + ciphertext.length)+ payload.set(nonce)+ payload.set(ciphertext, nonce.length)+ return `${REF_VERSION}.${sodium.to_base64(payload, sodium.base64_variants.URLSAFE_NO_PADDING)}`+ }+}++/** Returns null for any ref this session key did not produce for this team and session. */+export function decodeBlockRef(+ sessionKey: Uint8Array,+ ref: string,+ teamId: number,+ sessionId: string+): Omit<BlockLocation, 'teamId' | 'sessionId'> | null {+ const [version, encoded, ...rest] = ref.split('.')+ if (version !== REF_VERSION || !encoded || rest.length > 0) {+ return null+ }+ try {+ const payload = sodium.from_base64(encoded, sodium.base64_variants.URLSAFE_NO_PADDING)+ const nonce = payload.subarray(0, sodium.crypto_secretbox_NONCEBYTES)+ const ciphertext = payload.subarray(sodium.crypto_secretbox_NONCEBYTES)+ const plaintext = sodium.crypto_secretbox_open_easy(ciphertext, nonce, encryptionKeyFor(sessionKey))+ const [refTeamId, refSessionId, key, startByte, endByte] = parseJSON(sodium.to_string(plaintext))+ if (refTeamId !== teamId || refSessionId !== sessionId) {+ return null+ }+ if (typeof key !== 'string' || !Number.isInteger(startByte) || !Number.isInteger(endByte)) {+ return null+ }+ return { key, startByte, endByte }+ } catch {+ return null+ }+}diff --git a/nodejs/src/session-replay/recording-api/metrics.ts b/nodejs/src/session-replay/recording-api/metrics.tsindex 38875d8658ff..ea847550beaa 100644--- a/nodejs/src/session-replay/recording-api/metrics.ts+++ b/nodejs/src/session-replay/recording-api/metrics.ts@@ -10,6 +10,13 @@ export class RecordingApiMetrics {buckets: [0.01, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10],})+ private static readonly keyStoreDuration = new Histogram({+ name: 'recording_api_keystore_duration_seconds',+ help: 'Time taken by one call to the KMS and DynamoDB key store, below the Redis cache',+ labelNames: ['operation', 'result'],+ buckets: [0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2, 3, 5, 10],+ })+private static readonly deleteRecordingsDuration = new Histogram({name: 'recording_api_delete_recordings_duration_seconds',help: 'Time taken to delete recordings',@@ -17,6 +24,14 @@ export class RecordingApiMetrics {buckets: [0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30],})+ public static observeKeyStore(+ operation: 'generate' | 'get' | 'delete',+ result: 'ok' | 'error',+ seconds: number+ ): void {+ this.keyStoreDuration.observe({ operation, result }, seconds)+ }+public static observeGetBlock(result: string,seconds: number,@@ -75,4 +90,23 @@ export class RecordingApiMetrics {public static incrementClickhousePasswordFallback(reason: ClickHousePasswordFallbackReason): void {this.clickhousePasswordFallbacks.labels({ reason }).inc()}++ private static readonly replayBlockRequests = new Counter({+ name: 'recording_api_replay_block_requests_total',+ help: 'Browser block route requests by outcome',+ labelNames: ['result'],+ })++ public static incrementReplayBlockRequest(+ result:+ | 'ok'+ | 'not_found'+ | 'deleted'+ | 'rate_limited'+ | 'rate_limited_log_only'+ | 'rate_limit_unavailable'+ | 'error'+ ): void {+ this.replayBlockRequests.labels({ result }).inc()+ }}diff --git a/nodejs/src/session-replay/recording-api/recording-api.test.ts b/nodejs/src/session-replay/recording-api/recording-api.test.tsindex 11e95381f547..c98e12e8fb91 100644--- a/nodejs/src/session-replay/recording-api/recording-api.test.ts+++ b/nodejs/src/session-replay/recording-api/recording-api.test.ts@@ -83,6 +83,8 @@ describe('RecordingApi', () => {CLICKHOUSE_USER: 'default',CLICKHOUSE_PASSWORD: undefined,CLICKHOUSE_SECURE: false,+ RECORDING_API_AWS_TIMEOUT_MS: 5000,+ RECORDING_API_REDIS_TIMEOUT_MS: 2000,}mockKeyStore = {@@ -103,7 +105,10 @@ describe('RecordingApi', () => {formatS3KeyError: jest.fn().mockReturnValue('Invalid key format'),getBlock: jest.fn(),listBlocks: jest.fn(),- getDeletion: jest.fn().mockResolvedValue(null),+ annotateBlocks: jest+ .fn()+ .mockImplementation((_s: string, _t: number, blocks: unknown) => Promise.resolve(blocks)),+ getBlockByRef: jest.fn(),deleteRecordings: jest.fn(),encryptBlocks: jest.fn(),} as unknown as jest.Mocked<RecordingService>@@ -137,8 +142,9 @@ describe('RecordingApi', () => {expect(getKeyStore).toHaveBeenCalledWith('us-west-2', {kmsEndpoint: undefined,dynamoDBEndpoint: undefined,+ requestTimeoutMs: 5000,})- expect(getBlockDecryptor).toHaveBeenCalledWith(mockKeyStore)+ expect(getBlockDecryptor).toHaveBeenCalled()expect(RecordingService).toHaveBeenCalled()})@@ -439,7 +445,7 @@ describe('RecordingApi', () => {end_timestamp: '2024-01-01T00:01:00Z',},])- mockService.getDeletion.mockResolvedValue({+ mockService.annotateBlocks.mockResolvedValue({ok: false,error: 'deleted',deletedAt: 1700000000,@@ -454,7 +460,7 @@ describe('RecordingApi', () => {deleted_at: 1700000000,deleted_by: '[email protected]',})- expect(mockService.getDeletion).toHaveBeenCalledWith('session-123', 1)+ expect(mockService.annotateBlocks).toHaveBeenCalledWith('session-123', 1, expect.any(Array))})it('still lists blocks when the key store does not answer', async () => {@@ -467,7 +473,7 @@ describe('RecordingApi', () => {end_timestamp: '2024-01-01T00:01:00Z',},])- mockService.getDeletion.mockReturnValue(new Promise(() => {}))+ mockService.annotateBlocks.mockReturnValue(new Promise(() => {}))const res = await supertest(app).get('/api/projects/1/recordings/session-123/blocks')@@ -481,7 +487,7 @@ describe('RecordingApi', () => {const res = await supertest(app).get('/api/projects/1/recordings/session-123/blocks')expect(res.status).toBe(200)- expect(mockService.getDeletion).not.toHaveBeenCalled()+ expect(mockService.annotateBlocks).not.toHaveBeenCalled()})it('still lists blocks when the key store fails', async () => {@@ -494,7 +500,7 @@ describe('RecordingApi', () => {end_timestamp: '2024-01-01T00:01:00Z',},])- mockService.getDeletion.mockRejectedValue(new Error('DynamoDB unavailable'))+ mockService.annotateBlocks.mockRejectedValue(new Error('DynamoDB unavailable'))const res = await supertest(app).get('/api/projects/1/recordings/session-123/blocks')@@ -502,6 +508,22 @@ describe('RecordingApi', () => {expect(res.body.blocks).toHaveLength(1)})+ it('passes block refs through to the listing', async () => {+ const block = {+ key: 'session_recordings/30d/1000-aaa',+ start_byte: 0,+ end_byte: 100,+ start_timestamp: '2024-01-01T00:00:00Z',+ end_timestamp: '2024-01-01T00:01:00Z',+ }+ mockService.listBlocks.mockResolvedValue([block])+ mockService.annotateBlocks.mockResolvedValue([{ ...block, ref: 'v1.abc' }])++ const res = await supertest(app).get('/api/projects/1/recordings/session-123/blocks')++ expect(res.body).toEqual({ blocks: [{ ...block, ref: 'v1.abc' }] })+ })+it('should return empty blocks when session not found', async () => {mockService.listBlocks.mockResolvedValue([])@@ -795,4 +817,190 @@ describe('RecordingApi', () => {expect(mockService.encryptBlocks).toHaveBeenCalledWith('session-1', 1, [Buffer.from('block')], 365)})})++ describe('browser block route', () => {+ const JWT_SECRET = 'recording-api-test-secret'+ const BLOCKS_JWT_SECRET = 'recording-api-blocks-test-secret'+ const BLOCK_URL = '/replay-blocks/1/session-123/u7/v1.abc'+ let server: Server+ let rateLimiter: { rateLimitGrouped: jest.Mock }++ const browserToken = (+ claims: Record<string, unknown> = {},+ expiresIn: string | null = '5m',+ secret = BLOCKS_JWT_SECRET+ ): string =>+ `Bearer ${new JWT(secret).sign(+ { team_id: 1, session_id: 'session-123', sub: 'u7', tier: 'browser', ...claims },+ PosthogJwtAudience.REPLAY_BLOCKS,+ expiresIn ? { expiresIn } : {}+ )}`+ const serviceToken = (): string =>+ `Bearer ${new JWT(JWT_SECRET).sign({ team_id: 1, op: 'read' }, PosthogJwtAudience.RECORDING_API, { expiresIn: '5m' })}`++ const buildApp = async (role: string, enforced = false): Promise<express.Application> => {+ const recordingApi = new RecordingApi(+ {+ RECORDING_API_JWT_SECRET: JWT_SECRET,+ RECORDING_API_BLOCKS_JWT_SECRET: BLOCKS_JWT_SECRET,+ RECORDING_API_ALLOW_LEGACY_SECRET: false,+ RECORDING_API_ROLE: role,+ RECORDING_API_BLOCK_RATE_LIMIT_ENFORCED: enforced,+ } as RecordingApiConfig,+ mockPostgres,+ mockOutputs+ )+ await recordingApi.start(mockService, rateLimiter as any)+ const app = express()+ app.use('/', recordingApi.router())+ server = app.listen(0, () => {})+ return app+ }++ beforeEach(() => {+ rateLimiter = {+ rateLimitGrouped: jest+ .fn()+ .mockImplementation((requests: { id: string }[]) =>+ Promise.resolve(+ requests.map((r) => [r.id, { isRateLimited: false, tokens: 1, tokensBefore: 2 }])+ )+ ),+ }+ mockService.getBlockByRef.mockResolvedValue({ ok: true, data: Buffer.from('block-bytes') })+ })++ afterEach(() => {+ server.close()+ })++ it('serves a block a browser may cache, without the cache varying on cookies', async () => {+ const app = await buildApp('public')++ const res = await supertest(app).get(BLOCK_URL).set('Authorization', browserToken())++ expect(res.status).toBe(200)+ expect(res.headers['cache-control']).toBe('private, max-age=86400, immutable')+ expect(res.headers['vary'] ?? '').not.toMatch(/cookie/i)+ expect(Buffer.from(res.body).toString()).toBe('block-bytes')+ expect(mockService.getBlockByRef).toHaveBeenCalledWith(1, 'session-123', 'v1.abc')+ expect(rateLimiter.rateLimitGrouped).toHaveBeenCalledWith([+ { id: 'viewer:u7', cost: 1, bucketSize: 2000, refillRate: 8 },+ ])+ })++ it.each([+ ['no token', undefined, 401],+ ['a token for another session', () => browserToken({ session_id: 'session-999' }), 403],+ ['a token for another team', () => browserToken({ team_id: 2 }), 403],+ ['a token with no caller', () => browserToken({ sub: undefined }), 403],+ ['a token for another viewer', () => browserToken({ sub: 'u8' }), 403],+ ['a token with a string team', () => browserToken({ team_id: '1' }), 403],+ ['an internal service token', serviceToken, 401],+ ['a browser token signed with the service secret', () => browserToken({}, '5m', JWT_SECRET), 401],+ ['an expired token', () => browserToken({}, '-5m'), 401],+ ['a token that never expires', () => browserToken({}, null), 401],+ [+ 'a token issued over two hours ago',+ () => browserToken({ iat: Math.floor(Date.now() / 1000) - 3 * 3600 }, '1d'),+ 401,+ ],+ ])('refuses %s', async (_, token, expectedStatus) => {+ const app = await buildApp('public')++ const request = supertest(app).get(BLOCK_URL)+ const res = await (token ? request.set('Authorization', token()) : request)++ expect(res.status).toBe(expectedStatus)+ expect(res.headers['cache-control']).toBe('no-store')+ expect(mockService.getBlockByRef).not.toHaveBeenCalled()+ })++ it('refuses a team id spelled with a leading zero', async () => {+ const app = await buildApp('public')++ const res = await supertest(app)+ .get('/replay-blocks/01/session-123/u7/v1.abc')+ .set('Authorization', browserToken())++ expect(res.status).toBe(403)+ })++ it.each([+ ['deleted', { ok: false, error: 'deleted', deletedAt: 1700000000, deletedBy: '[email protected]' }, 410],+ ['an unknown ref', { ok: false, error: 'not_found' }, 404],+ ])('answers a block that is %s without caching it', async (_, result, expectedStatus) => {+ mockService.getBlockByRef.mockResolvedValue(result as any)+ const app = await buildApp('public')++ const res = await supertest(app).get(BLOCK_URL).set('Authorization', browserToken())++ expect(res.status).toBe(expectedStatus)+ expect(res.headers['cache-control']).toBe('no-store')+ })++ it.each([+ ['enforced', true, 429],+ ['log only', false, 200],+ ])('applies the per-caller rate limit when %s', async (_, enforced, expectedStatus) => {+ rateLimiter.rateLimitGrouped.mockResolvedValue([+ ['u7', { isRateLimited: true, tokens: -1, tokensBefore: 0 }],+ ])+ const app = await buildApp('public', enforced)++ const res = await supertest(app).get(BLOCK_URL).set('Authorization', browserToken())++ expect(res.status).toBe(expectedStatus)+ expect(res.headers['cache-control']).toBe(enforced ? 'no-store' : 'private, max-age=86400, immutable')+ if (enforced) {+ expect(res.headers['retry-after']).toBe('1')+ }+ })++ it('serves the block when the rate limiter is unavailable', async () => {+ rateLimiter.rateLimitGrouped.mockRejectedValue(new Error('Redis unavailable'))+ const app = await buildApp('public')++ const res = await supertest(app).get(BLOCK_URL).set('Authorization', browserToken())++ expect(res.status).toBe(200)+ })++ it.each([+ ['a browser viewer, by viewer', { tier: 'browser' }, 'viewer:u7', 2000],+ ['a paid API key, by team', { sub: 'k42', tier: 'paid' }, 'team:1', 1200],+ ['another key of the same team, in the same bucket', { sub: 'k43', tier: 'paid' }, 'team:1', 1200],+ ['an enterprise API key, by team', { sub: 'k42', tier: 'enterprise' }, 'team:1', 2000],+ ['a token with no tier, strictest', { sub: 'k42', tier: undefined }, 'team:1', 240],+ ['an unknown tier, strictest', { sub: 'k42', tier: 'platinum' }, 'team:1', 240],+ ])('rate-limits %s', async (_, claims, expectedId, expectedBucketSize) => {+ const app = await buildApp('public')+ const sub = (claims as { sub?: string }).sub ?? 'u7'++ await supertest(app)+ .get(`/replay-blocks/1/session-123/${sub}/v1.abc`)+ .set('Authorization', browserToken(claims))++ expect(rateLimiter.rateLimitGrouped).toHaveBeenCalledWith([+ expect.objectContaining({ id: expectedId, bucketSize: expectedBucketSize }),+ ])+ })++ it.each([+ ['public', 200, 404],+ ['internal', 404, 200],+ ['all', 200, 200],+ ])('a %s process serves only its own routes', async (role, browserStatus, internalStatus) => {+ mockService.listBlocks.mockResolvedValue([])+ const app = await buildApp(role)++ const browser = await supertest(app).get(BLOCK_URL).set('Authorization', browserToken())+ const internal = await supertest(app)+ .get('/api/projects/1/recordings/session-123/blocks')+ .set('Authorization', serviceToken())++ expect(browser.status).toBe(browserStatus)+ expect(internal.status).toBe(internalStatus)+ })+ })})diff --git a/nodejs/src/session-replay/recording-api/recording-api.ts b/nodejs/src/session-replay/recording-api/recording-api.tsindex 185f98cdb530..fcaa833c0593 100644--- a/nodejs/src/session-replay/recording-api/recording-api.ts+++ b/nodejs/src/session-replay/recording-api/recording-api.ts@@ -4,6 +4,8 @@ import https from 'https'import express from 'ultimate-express'import { IngestionOutputs } from '~/common/outputs/ingestion-outputs'+import { createRedisV2PoolFromConfig } from '~/common/redis/redis-v2'+import { KeyedRateLimiterService } from '~/common/services/keyed-rate-limiter.service'import { PostgresRouter } from '~/common/utils/db/postgres'import { createRedisPoolFromConfig } from '~/common/utils/db/redis'import { logger, serializeError } from '~/common/utils/logger'@@ -16,19 +18,40 @@ import { SessionMetadataStore } from '~/ingestion/pipelines/sessionreplay/sharedimport { ReplayEventsOutput, SessionFeaturesOutput } from '~/ingestion/pipelines/sessionreplay/shared/outputs'import { HealthCheckResult, HealthCheckResultError, HealthCheckResultOk, PluginServerService, RedisPool } from '~/types'-import { RecordingApiOp, createRecordingApiAuthMiddleware } from './auth'+import { RecordingApiOp, createRecordingApiAuthMiddleware, createReplayBlocksAuthMiddleware } from './auth'import { ClickHouseCredential } from './clickhouse-credential'-import { RecordingDeletedResult, RecordingService } from './recording-service'+import { RecordingApiMetrics } from './metrics'+import { ListedBlock, RecordingDeletedResult, RecordingService } from './recording-service'import {DeleteRecordingsBodySchema,EncryptBlocksBodySchema,GetBlockQuerySchema,RecordingParamsSchema,+ ReplayBlockParamsSchema,TeamParamsSchema,} from './schemas'-import { KeyStore, RecordingApiConfig, RecordingDecryptor, RecordingEncryptor } from './types'--const DELETION_CHECK_TIMEOUT_MS = 1000+import { TimedKeyStore } from './timed-keystore'+import {+ KeyStore,+ REPLAY_BLOCK_TIERS,+ RecordingApiConfig,+ RecordingApiRole,+ RecordingBlock,+ RecordingDecryptor,+ RecordingEncryptor,+ ReplayBlockBucket,+ ReplayBlockTier,+ getDefaultRecordingApiBlocksConfig,+ parseRecordingApiRole,+ parseReplayBlockRateLimits,+} from './types'++// The browser keeps each block for a day. Django's sources listing, which hands out block URLs, reads the block+// listing uncached and reports a deletion first, so the player never asks for a cached block of a deleted recording.+const REPLAY_BLOCK_CACHE_CONTROL = 'private, max-age=86400, immutable'++const KEY_LOOKUP_TIMEOUT_MS = 1000+// Each Redis, DynamoDB and KMS call gives up after this, so a lookup the listing abandoned does not pile up.function sendRecordingDeleted(res: express.Response, deletion: RecordingDeletedResult): void {res.status(410).json({ error: 'recording_deleted', deleted_at: deletion.deletedAt, deleted_by: deletion.deletedBy })@@ -44,12 +67,33 @@ export class RecordingApi {private redisPool: RedisPool | null = nullprivate clickhouseClient: ClickHouseClient | null = nullprivate recordingService: RecordingService | null = null+ private blockRateLimiter: KeyedRateLimiterService | null = null+ private readonly role: RecordingApiRole+ private readonly replayBlockBuckets: Record<ReplayBlockTier, ReplayBlockBucket> | null+ // Postgres and the outputs are only used by internal routes, so a public process passes neither.constructor(private config: RecordingApiConfig,- private postgres: PostgresRouter,- private outputs: IngestionOutputs<ReplayEventsOutput | SessionFeaturesOutput>- ) {}+ private postgres?: PostgresRouter,+ private outputs?: IngestionOutputs<ReplayEventsOutput | SessionFeaturesOutput>+ ) {+ this.role = parseRecordingApiRole(config.RECORDING_API_ROLE || 'internal')+ // Only a process serving the browser route reads its limits, so a bad value cannot stop an internal pod.+ this.replayBlockBuckets = this.servesPublicRoutes+ ? parseReplayBlockRateLimits(+ config.RECORDING_API_BLOCK_RATE_LIMITS ||+ getDefaultRecordingApiBlocksConfig().RECORDING_API_BLOCK_RATE_LIMITS+ )+ : null+ }++ private get servesInternalRoutes(): boolean {+ return this.role !== 'public'+ }++ private get servesPublicRoutes(): boolean {+ return this.role !== 'internal'+ }public get service(): PluginServerService {return {@@ -59,9 +103,10 @@ export class RecordingApi {}}- async start(recordingService?: RecordingService): Promise<void> {+ async start(recordingService?: RecordingService, blockRateLimiter?: KeyedRateLimiterService): Promise<void> {if (recordingService) {this.recordingService = recordingService+ this.blockRateLimiter = blockRateLimiter ?? nulllogger.info('[RecordingApi] Started with injected RecordingService')return}@@ -98,28 +143,61 @@ export class RecordingApi {this.s3Client = new S3Client(s3Config)- this.redisPool = createRedisPoolFromConfig({+ // Bounded so a stalled Redis fails fast rather than holding requests the listing has stopped waiting for.+ const redisConfig = (name: string) => ({connection: {url: this.config.SESSION_RECORDING_API_REDIS_HOST,- options: { port: this.config.SESSION_RECORDING_API_REDIS_PORT },- name: 'recording-api',+ options: {+ port: this.config.SESSION_RECORDING_API_REDIS_PORT,+ commandTimeout: this.config.RECORDING_API_REDIS_TIMEOUT_MS,+ },+ name,},poolMinSize: this.config.REDIS_POOL_MIN_SIZE,poolMaxSize: this.config.REDIS_POOL_MAX_SIZE,+ acquireTimeoutMillis: this.config.RECORDING_API_REDIS_TIMEOUT_MS,})+ this.redisPool = createRedisPoolFromConfig(redisConfig('recording-api'))- const keyStore: KeyStore = getKeyStore(s3Region, {- kmsEndpoint: this.config.SESSION_RECORDING_KMS_ENDPOINT,- dynamoDBEndpoint: this.config.SESSION_RECORDING_DYNAMODB_ENDPOINT,- })+ const keyStore: KeyStore = new TimedKeyStore(+ getKeyStore(s3Region, {+ kmsEndpoint: this.config.SESSION_RECORDING_KMS_ENDPOINT,+ dynamoDBEndpoint: this.config.SESSION_RECORDING_DYNAMODB_ENDPOINT,+ requestTimeoutMs: this.config.RECORDING_API_AWS_TIMEOUT_MS,+ })+ )// In-memory caching is intentionally omitted here — a stale in-memory cache on// another instance would allow serving a deleted recording's data.- this.keyStore = new RedisCachedKeyStore(keyStore, this.redisPool)+ // Never cache a missing key: the browser route can name a session ingestion has not keyed yet, and DynamoDB's+ // eventually consistent read can briefly miss a new key. Either cached miss would read as cleartext for a day.+ this.keyStore = new RedisCachedKeyStore(keyStore, this.redisPool, undefined, false)await this.keyStore.start()this.decryptor = getBlockDecryptor(this.keyStore)await this.decryptor.start()+ if (this.servesPublicRoutes) {+ this.blockRateLimiter = new KeyedRateLimiterService(+ { name: 'recording-api-replay-blocks', ttlSeconds: 60 * 60 },+ createRedisV2PoolFromConfig(redisConfig('recording-api-replay-blocks'))+ )+ }++ if (!this.servesInternalRoutes) {+ this.recordingService = new RecordingService(+ this.s3Client,+ this.s3Bucket!,+ this.s3Prefix!,+ this.keyStore,+ this.decryptor+ )+ logger.info('[RecordingApi] Started with public routes only')+ return+ }++ if (!this.outputs) {+ throw new Error('recording-api serving internal routes needs its Kafka outputs')+ }this.encryptor = getBlockEncryptor(this.keyStore)await this.encryptor.start()@@ -183,7 +261,7 @@ export class RecordingApi {if (!this.decryptor) {uninitializedComponents.push('decryptor')}- if (!this.encryptor) {+ if (this.servesInternalRoutes && !this.encryptor) {uninitializedComponents.push('encryptor')}@@ -210,6 +288,17 @@ export class RecordingApi {const encryptPath = '/api/projects/:team_id/recordings/:session_id/encrypt'+ if (this.servesPublicRoutes) {+ router.get(+ '/replay-blocks/:team_id/:session_id/:viewer/:ref',+ createReplayBlocksAuthMiddleware(this.config.RECORDING_API_BLOCKS_JWT_SECRET),+ asyncHandler(this.getBlockByRef)+ )+ }+ if (!this.servesInternalRoutes) {+ return router+ }+const auth = (op: RecordingApiOp) =>createRecordingApiAuthMiddleware({jwtSecret: this.config.RECORDING_API_JWT_SECRET,@@ -323,13 +412,15 @@ export class RecordingApi {try {const blocks = await this.recordingService.listBlocks(sessionId, teamId)- const deletion =- blocks.length > 0 ? await this.deletionOrNull(this.recordingService, sessionId, teamId) : null- if (deletion) {- sendRecordingDeleted(res, deletion)+ const annotated =+ blocks.length > 0+ ? await this.annotateOrPlain(this.recordingService, sessionId, teamId, blocks)+ : blocks+ if (!Array.isArray(annotated)) {+ sendRecordingDeleted(res, annotated)return}- res.json({ blocks })+ res.json({ blocks: annotated })} catch (error) {logger.error('[RecordingApi] Error listing blocks', {error: serializeError(error),@@ -341,30 +432,122 @@ export class RecordingApi {}}- // The listing only reports a deletion early, so a key store outage or stall must not stop it: getBlock still returns 410.- private async deletionOrNull(+ // The listing only reports a deletion and hands out refs as extras, so a key store outage or stall must not stop+ // it: the blocks still list without refs, and getBlock still returns 410.+ private async annotateOrPlain(service: RecordingService,sessionId: string,- teamId: number- ): Promise<RecordingDeletedResult | null> {+ teamId: number,+ blocks: RecordingBlock[]+ ): Promise<RecordingDeletedResult | ListedBlock[]> {let timer: NodeJS.Timeout | undefinedconst timeout = new Promise<never>((_, reject) => {- timer = setTimeout(() => reject(new Error('Deletion check timed out')), DELETION_CHECK_TIMEOUT_MS)+ timer = setTimeout(() => reject(new Error('Key lookup timed out')), KEY_LOOKUP_TIMEOUT_MS)})try {- return await Promise.race([service.getDeletion(sessionId, teamId), timeout])+ return await Promise.race([service.annotateBlocks(sessionId, teamId, blocks), timeout])} catch (error) {- logger.warn('[RecordingApi] Could not check deletion while listing blocks', {+ logger.warn('[RecordingApi] Could not read the session key while listing blocks', {error: serializeError(error),teamId,sessionId,})- return null+ return blocks} finally {clearTimeout(timer)}}+ // A browser viewer has a budget of its own. API callers share their team's budget, as Django's snapshot API limits+ // do, so a team cannot multiply it with more keys. A missing or unknown tier gets the strictest bucket.+ private replayBlockBucket(+ teamId: number,+ caller: string,+ tier: unknown+ ): { id: string; bucket: ReplayBlockBucket } {+ const known = REPLAY_BLOCK_TIERS.includes(tier as ReplayBlockTier) ? (tier as ReplayBlockTier) : 'free'+ return {+ id: known === 'browser' ? `viewer:${caller}` : `team:${teamId}`,+ bucket: this.replayBlockBuckets![known],+ }+ }++ // rateLimitGrouped allows a request that spends the bucket's last token. rateLimitMany refuses it yet still spends+ // the token, so a steady trickle at the refill rate would never get through. A Redis failure lets the request+ // through: the limit protects capacity, and the block fetch needs Redis for the key lookup anyway.+ private async isRateLimited(id: string, bucket: ReplayBlockBucket): Promise<boolean> {+ try {+ const [[, decision]] = await this.blockRateLimiter!.rateLimitGrouped([+ { id, cost: 1, bucketSize: bucket.bucketSize, refillRate: bucket.refillPerSecond },+ ])+ return decision.isRateLimited+ } catch (error) {+ RecordingApiMetrics.incrementReplayBlockRequest('rate_limit_unavailable')+ logger.warn('[RecordingApi] Block rate limit unavailable, allowing the request', {+ error: serializeError(error),+ })+ return false+ }+ }++ private getBlockByRef = async (req: express.Request, res: express.Response): Promise<void> => {+ const paramsResult = ReplayBlockParamsSchema.safeParse(req.params)+ if (!paramsResult.success) {+ res.status(400).json({ error: paramsResult.error.issues[0].message })+ return+ }+ if (!this.recordingService) {+ res.status(503).json({ error: 'Service not initialized' })+ return+ }+ const { team_id: teamId, session_id: sessionId, ref } = paramsResult.data++ try {+ if (this.blockRateLimiter) {+ const { id, bucket } = this.replayBlockBucket(+ teamId,+ res.locals.replayBlocksCaller,+ res.locals.replayBlocksTier+ )+ if (await this.isRateLimited(id, bucket)) {+ const enforced = this.config.RECORDING_API_BLOCK_RATE_LIMIT_ENFORCED+ RecordingApiMetrics.incrementReplayBlockRequest(enforced ? 'rate_limited' : 'rate_limited_log_only')+ if (enforced) {+ // The wait for one token to refill, not a fixed window.+ res.set('Retry-After', String(Math.ceil(1 / bucket.refillPerSecond)))+ res.status(429).json({ error: 'Too many requests' })+ return+ }+ }+ }++ const result = await this.recordingService.getBlockByRef(teamId, sessionId, ref)+ if (!result.ok) {+ RecordingApiMetrics.incrementReplayBlockRequest(result.error === 'deleted' ? 'deleted' : 'not_found')+ if (result.error === 'deleted') {+ sendRecordingDeleted(res, result)+ return+ }+ res.status(404).json({ error: 'Block not found' })+ return+ }+ RecordingApiMetrics.incrementReplayBlockRequest('ok')+ res.set('Content-Type', 'application/octet-stream')+ res.set('Content-Length', String(Buffer.byteLength(result.data)))+ res.set('Cache-Control', REPLAY_BLOCK_CACHE_CONTROL)+ res.send(result.data)+ } catch (error) {+ RecordingApiMetrics.incrementReplayBlockRequest('error')+ logger.error('[RecordingApi] Error serving a block by ref', {+ error: serializeError(error),+ teamId,+ sessionId,+ })+ captureException(error)+ res.status(500).json({ error: 'Failed to fetch block' })+ }+ }+private encryptBlocks = async (req: express.Request, res: express.Response): Promise<void> => {const paramsResult = RecordingParamsSchema.safeParse(req.params)if (!paramsResult.success) {diff --git a/nodejs/src/session-replay/recording-api/recording-service.test.ts b/nodejs/src/session-replay/recording-api/recording-service.test.tsindex 59fdf0255fa4..f5f4399875a5 100644--- a/nodejs/src/session-replay/recording-api/recording-service.test.ts+++ b/nodejs/src/session-replay/recording-api/recording-service.test.ts@@ -371,21 +371,32 @@ describe('RecordingService', () => {})})- describe('getDeletion', () => {- const key = (sessionState: 'ciphertext' | 'cleartext' | 'deleted'): SessionKey => ({- plaintextKey: Buffer.alloc(0),+ describe('block refs', () => {+ const sessionKey = (sessionState: 'ciphertext' | 'cleartext' | 'deleted'): SessionKey => ({+ plaintextKey: Buffer.alloc(32, 7),encryptedKey: Buffer.alloc(0),sessionState,})+ const block = {+ key: 'session_recordings/30d/1764634738680-3cca0f5d3c7cc7ee',+ start_byte: 100,+ end_byte: 250,+ start_timestamp: '2024-01-01T00:00:00Z',+ end_timestamp: '2024-01-01T00:01:00Z',+ }- it('reports a shredded key with who deleted it and when', async () => {+ beforeAll(async () => {+ await sodium.ready+ })++ it('reports a shredded key in place of the blocks', async () => {mockKeyStore.getKey.mockResolvedValue({- ...key('deleted'),+ ...sessionKey('deleted'),deletedAt: 1700000000,deletedBy: '[email protected]',})- expect(await service.getDeletion('sess-1', 1)).toEqual({+ expect(await service.annotateBlocks('sess-1', 1, [block])).toEqual({ok: false,error: 'deleted',deletedAt: 1700000000,@@ -393,13 +404,47 @@ describe('RecordingService', () => {})})+ it('gives a pre-encryption recording no refs', async () => {+ mockKeyStore.getKey.mockResolvedValue(sessionKey('cleartext'))++ expect(await service.annotateBlocks('sess-1', 1, [block])).toEqual([block])+ expect(mockKeyStore.getKey).toHaveBeenCalledWith('sess-1', 1)+ })++ it('fetches the byte range a listed ref names', async () => {+ mockKeyStore.getKey.mockResolvedValue(sessionKey('ciphertext'))+ const [listed] = (await service.annotateBlocks('sess-1', 1, [block])) as { ref: string }[]+ mockS3Send.mockResolvedValue({+ Body: { transformToByteArray: jest.fn().mockResolvedValue(new Uint8Array([1])) },+ })+ mockDecryptor.decryptBlockWithKey = jest+ .fn()+ .mockReturnValue({ data: Buffer.from('data'), sessionState: 'ciphertext' })+ mockKeyStore.getKey.mockClear()++ const result = await service.getBlockByRef(1, 'sess-1', listed.ref)++ expect(result).toEqual({ ok: true, data: Buffer.from('data') })+ expect(mockKeyStore.getKey).toHaveBeenCalledTimes(1)+ expect(mockKeyStore.getKey).toHaveBeenCalledWith('sess-1', 1)+ expect(mockDecryptor.decryptBlock).not.toHaveBeenCalled()+ expect(mockS3Send).toHaveBeenCalledWith(+ expect.objectContaining({ input: { Bucket: 'test-bucket', Key: block.key, Range: 'bytes=100-250' } })+ )+ })+it.each([- ['an encrypted', 'ciphertext' as const],- ['a pre-encryption', 'cleartext' as const],- ])('reports nothing for %s recording', async (_, sessionState) => {- mockKeyStore.getKey.mockResolvedValue(key(sessionState))+ ['a deleted session', 'deleted', 'deleted'],+ ['a pre-encryption session', 'cleartext', 'not_found'],+ ] as const)('serves no block by ref for %s', async (_, sessionState, expectedError) => {+ mockKeyStore.getKey.mockResolvedValue(sessionKey('ciphertext'))+ const [listed] = (await service.annotateBlocks('sess-1', 1, [block])) as { ref: string }[]+ mockKeyStore.getKey.mockResolvedValue(sessionKey(sessionState))++ const result = await service.getBlockByRef(1, 'sess-1', listed.ref)- expect(await service.getDeletion('sess-1', 1)).toBeNull()+ expect(result).toMatchObject({ ok: false, error: expectedError })+ expect(mockS3Send).not.toHaveBeenCalled()})})diff --git a/nodejs/src/session-replay/recording-api/recording-service.ts b/nodejs/src/session-replay/recording-api/recording-service.tsindex e8f4422eff14..40937adc418a 100644--- a/nodejs/src/session-replay/recording-api/recording-service.ts+++ b/nodejs/src/session-replay/recording-api/recording-service.ts@@ -10,6 +10,7 @@ import { SessionFeatureStore } from '~/ingestion/pipelines/sessionreplay/shared/import { createDeletionBlockMetadata } from '~/ingestion/pipelines/sessionreplay/shared/metadata/session-block-metadata'import { SessionMetadataStore } from '~/ingestion/pipelines/sessionreplay/shared/metadata/session-metadata-store'+import { blockRefEncoder, decodeBlockRef } from './block-refs'import { ClickHouseCredential } from './clickhouse-credential'import { RecordingApiMetrics } from './metrics'import {@@ -28,12 +29,16 @@ export interface GetBlockParams {startByte: numberendByte: numberdecompress?: boolean+ // A caller that already read the session key passes it, so the block is decrypted without a second lookup.+ sessionKey?: SessionKey}export type RecordingDeletedResult = { ok: false; error: 'deleted'; deletedAt?: number; deletedBy: string }export type GetBlockResult = { ok: true; data: Buffer } | { ok: false; error: 'not_found' } | RecordingDeletedResult+export type ListedBlock = RecordingBlock & { ref?: string }+export type EncryptBlocksResult =| { ok: true; blocks: Buffer[] }| { ok: false; error: 'recording_exists' }@@ -155,11 +160,9 @@ export class RecordingService {bytesReceived: bodyContents.length,})- const { data, sessionState } = await this.decryptor.decryptBlock(- sessionId,- teamId,- Buffer.from(bodyContents)- )+ const { data, sessionState } = params.sessionKey+ ? this.decryptor.decryptBlockWithKey(sessionId, teamId, Buffer.from(bodyContents), params.sessionKey)+ : await this.decryptor.decryptBlock(sessionId, teamId, Buffer.from(bodyContents))logger.debug('[RecordingService] Decrypted block', {sessionId,@@ -253,11 +256,43 @@ export class RecordingService {return rows.length > 0}- // The key store records a deletion at once, while the is_deleted marker reaches ClickHouse later through Kafka.- // Call it only for a session that has blocks: the key store caches a missing key as cleartext for a day, and a- // lookup before ingestion creates the key would make getBlock skip decryption for that whole day.- async getDeletion(sessionId: string, teamId: number): Promise<RecordingDeletedResult | null> {- return deletionOf(await this.keyStore.getKey(sessionId, teamId))+ // Reports a deletion the key store recorded before ClickHouse learns of it through Kafka, and names each block of an+ // encrypted session with a ref the browser can fetch and cache. Cleartext sessions have no key to derive refs from.+ // Call it only for a session that has blocks; a session without any has no key worth reading yet.+ async annotateBlocks(+ sessionId: string,+ teamId: number,+ blocks: RecordingBlock[]+ ): Promise<RecordingDeletedResult | ListedBlock[]> {+ const sessionKey = await this.keyStore.getKey(sessionId, teamId)+ const deletion = deletionOf(sessionKey)+ if (deletion) {+ return deletion+ }+ if (sessionKey.sessionState !== 'ciphertext') {+ return blocks+ }+ const encodeRef = blockRefEncoder(sessionKey.plaintextKey)+ return blocks.map((block) => ({+ ...block,+ ref: encodeRef({ teamId, sessionId, key: block.key, startByte: block.start_byte, endByte: block.end_byte }),+ }))+ }++ async getBlockByRef(teamId: number, sessionId: string, ref: string): Promise<GetBlockResult> {+ const sessionKey = await this.keyStore.getKey(sessionId, teamId)+ const deletion = deletionOf(sessionKey)+ if (deletion) {+ return deletion+ }+ const location =+ sessionKey.sessionState === 'ciphertext'+ ? decodeBlockRef(sessionKey.plaintextKey, ref, teamId, sessionId)+ : null+ if (!location || !this.validateS3Key(location.key)) {+ return { ok: false, error: 'not_found' }+ }+ return this.getBlock({ sessionId, teamId, ...location, sessionKey })}async listBlocks(sessionId: string, teamId: number): Promise<RecordingBlock[]> {diff --git a/nodejs/src/session-replay/recording-api/schemas.ts b/nodejs/src/session-replay/recording-api/schemas.tsindex c26a959ee458..0f8c8a3e14d4 100644--- a/nodejs/src/session-replay/recording-api/schemas.ts+++ b/nodejs/src/session-replay/recording-api/schemas.ts@@ -24,6 +24,12 @@ export const RecordingParamsSchema = z.object({session_id: z.string({ error: 'Missing session_id parameter' }).min(1, 'Invalid session_id parameter'),})+// Path params for the browser block route. A ref is short, so the length cap only rejects junk early.+export const ReplayBlockParamsSchema = RecordingParamsSchema.extend({+ viewer: z.string().regex(/^[A-Za-z0-9_-]{1,64}$/, 'Invalid viewer'),+ ref: z.string().regex(/^v1\.[A-Za-z0-9_-]{1,1024}$/, 'Invalid block ref'),+})+// Path params with team_id only (no session_id)export const TeamParamsSchema = z.object({team_id: positiveIntString('team_id'),diff --git a/nodejs/src/session-replay/recording-api/timed-keystore.ts b/nodejs/src/session-replay/recording-api/timed-keystore.tsnew file mode 100644index 000000000000..6a306e5ba97c--- /dev/null+++ b/nodejs/src/session-replay/recording-api/timed-keystore.ts@@ -0,0 +1,40 @@+import { DeleteKeyResult, KeyStore, SessionKey } from '~/ingestion/pipelines/sessionreplay/shared/types'++import { RecordingApiMetrics } from './metrics'++// Times each call that misses the Redis cache, so the AWS deadline can follow the measured tail.+export class TimedKeyStore implements KeyStore {+ constructor(private readonly delegate: KeyStore) {}++ start(): Promise<void> {+ return this.delegate.start()+ }++ stop(): void {+ this.delegate.stop()+ }++ generateKey(sessionId: string, teamId: number, retentionDays: number): Promise<SessionKey> {+ return this.timed('generate', () => this.delegate.generateKey(sessionId, teamId, retentionDays))+ }++ getKey(sessionId: string, teamId: number): Promise<SessionKey> {+ return this.timed('get', () => this.delegate.getKey(sessionId, teamId))+ }++ deleteKey(sessionId: string, teamId: number, deletedBy: string): Promise<DeleteKeyResult> {+ return this.timed('delete', () => this.delegate.deleteKey(sessionId, teamId, deletedBy))+ }++ private async timed<T>(operation: 'generate' | 'get' | 'delete', call: () => Promise<T>): Promise<T> {+ const start = performance.now()+ let result: 'ok' | 'error' = 'error'+ try {+ const value = await call()+ result = 'ok'+ return value+ } finally {+ RecordingApiMetrics.observeKeyStore(operation, result, (performance.now() - start) / 1000)+ }+ }+}diff --git a/nodejs/src/session-replay/recording-api/types.test.ts b/nodejs/src/session-replay/recording-api/types.test.tsnew file mode 100644index 000000000000..62a698cfbc72--- /dev/null+++ b/nodejs/src/session-replay/recording-api/types.test.ts@@ -0,0 +1,18 @@+import { getDefaultRecordingApiBlocksConfig, parseReplayBlockRateLimits } from './types'++describe('parseReplayBlockRateLimits', () => {+ it('reads the default limits for every tier', () => {+ const limits = parseReplayBlockRateLimits(getDefaultRecordingApiBlocksConfig().RECORDING_API_BLOCK_RATE_LIMITS)++ expect(limits.free).toEqual({ bucketSize: 240, refillPerSecond: 1200 / 3600 })+ expect(Object.keys(limits).sort()).toEqual(['browser', 'enterprise', 'free', 'paid'])+ })++ it.each([+ ['a missing tier', '{"browser":[1,1],"free":[1,1],"paid":[1,1]}'],+ ['a non-positive size', '{"browser":[0,1],"free":[1,1],"paid":[1,1],"enterprise":[1,1]}'],+ ['a malformed entry', '{"browser":[1],"free":[1,1],"paid":[1,1],"enterprise":[1,1]}'],+ ])('refuses %s', (_, raw) => {+ expect(() => parseReplayBlockRateLimits(raw)).toThrow(/RECORDING_API_BLOCK_RATE_LIMITS/)+ })+})diff --git a/nodejs/src/session-replay/recording-api/types.ts b/nodejs/src/session-replay/recording-api/types.tsindex 9160eff20285..39b91dfa91e8 100644--- a/nodejs/src/session-replay/recording-api/types.ts+++ b/nodejs/src/session-replay/recording-api/types.ts@@ -10,6 +10,7 @@ import {KAFKA_CLICKHOUSE_SESSION_REPLAY_FEATURES,} from '~/common/config/kafka-topics'import { isProdEnv } from '~/common/utils/env-utils'+import { parseJSON } from '~/common/utils/json-parse'import { SessionRecordingApiConfig, SessionRecordingConfig } from '~/ingestion/pipelines/sessionreplay/config'import {INGESTION_SESSIONREPLAY_PRODUCER,@@ -57,6 +58,84 @@ export function getDefaultRecordingApiAuthConfig(): RecordingApiAuthConfig {}}+/**+ * Which routes a recording-api process serves. `internal` serves the service-to-service routes that+ * Django and Temporal call. `public` serves only the browser block route, so pods reachable from the+ * internet run none of the internal routes. `all` serves both, for local development.+ */+export type RecordingApiRole = 'internal' | 'public' | 'all'++/**+ * The browser block route's token buckets, sized in blocks since one request fetches one block. The token's `tier`+ * picks the bucket: `browser` for the player, and `free`/`paid`/`enterprise` for API callers, which mirror Django's+ * per-team snapshot API limits at 20 blocks per request. Enforced by default; false serves over-limit requests and+ * only counts them.+ */+export const REPLAY_BLOCK_TIERS = ['browser', 'free', 'paid', 'enterprise'] as const+export type ReplayBlockTier = (typeof REPLAY_BLOCK_TIERS)[number]+export type ReplayBlockBucket = { bucketSize: number; refillPerSecond: number }++export type RecordingApiBlocksConfig = {+ RECORDING_API_ROLE: string+ // Signs only browser block tokens. Kept apart from RECORDING_API_JWT_SECRET, so the internet-facing pods, which+ // hold only this one, cannot sign a service token for an internal route.+ RECORDING_API_BLOCKS_JWT_SECRET: string+ RECORDING_API_BLOCK_RATE_LIMIT_ENFORCED: boolean+ // JSON object of tier -> [bucket size, refill per second].+ RECORDING_API_BLOCK_RATE_LIMITS: string+ // Deadline for one KMS or DynamoDB attempt. The SDK makes two attempts.+ RECORDING_API_AWS_TIMEOUT_MS: number+ // Deadline for a Redis command and for checking a connection out of the pool.+ RECORDING_API_REDIS_TIMEOUT_MS: number+}++export function getDefaultRecordingApiBlocksConfig(): RecordingApiBlocksConfig {+ return {+ RECORDING_API_ROLE: 'internal',+ // Dev/test default matches the Django minter so local end-to-end calls validate.+ RECORDING_API_BLOCKS_JWT_SECRET: isProdEnv() ? '' : 'dev-recording-api-blocks-jwt-secret',+ RECORDING_API_BLOCK_RATE_LIMIT_ENFORCED: true,+ RECORDING_API_BLOCK_RATE_LIMITS: JSON.stringify({+ browser: [2000, 8],+ free: [240, 1200 / 3600],+ paid: [1200, 6000 / 3600],+ enterprise: [2000, 8000 / 3600],+ }),+ // Deletes, which are mostly KMS and DynamoDB calls, almost never take over 2.5s in total.+ RECORDING_API_AWS_TIMEOUT_MS: 5000,+ RECORDING_API_REDIS_TIMEOUT_MS: 2000,+ }+}++export function parseReplayBlockRateLimits(raw: string): Record<ReplayBlockTier, ReplayBlockBucket> {+ let parsed: Record<string, unknown>+ try {+ parsed = parseJSON(raw) as Record<string, unknown>+ } catch {+ throw new Error('Invalid RECORDING_API_BLOCK_RATE_LIMITS: not valid JSON')+ }+ const buckets = {} as Record<ReplayBlockTier, ReplayBlockBucket>+ for (const tier of REPLAY_BLOCK_TIERS) {+ const value = parsed?.[tier]+ if (+ !Array.isArray(value) ||+ value.length !== 2 ||+ !value.every((n) => typeof n === 'number' && Number.isFinite(n) && n > 0)+ ) {+ throw new Error(`Invalid RECORDING_API_BLOCK_RATE_LIMITS: '${tier}' needs [bucket size, refill per second]`)+ }+ buckets[tier] = { bucketSize: value[0], refillPerSecond: value[1] }+ }+ return buckets+}++export function parseRecordingApiRole(role: string): RecordingApiRole {+ if (role === 'internal' || role === 'public' || role === 'all') {+ return role+ }+ throw new Error(`Invalid RECORDING_API_ROLE '${role}': expected internal, public or all`)+}+/*** Configuration for the Recording API.* Postgres is passed as an explicit constructor param, not included here.@@ -87,7 +166,8 @@ export type RecordingApiConfig = Pick<| 'SESSION_RECORDING_V2_S3_BUCKET'| 'SESSION_RECORDING_V2_S3_PREFIX'> &- RecordingApiAuthConfig+ RecordingApiAuthConfig &+ RecordingApiBlocksConfig/*** Recording API outputs — topic and producer routing per output. All keys