Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(replay): serve recording blocks to browsers by opaque ref (#128265)

Tue Haulund99e1a37cd7cb
diff --git a/nodejs/src/cdp/utils/jwt-utils.ts b/nodejs/src/cdp/utils/jwt-utils.ts
index 24193d074546..f38f0d58272b 100644
--- a/nodejs/src/cdp/utils/jwt-utils.ts
+++ b/nodejs/src/cdp/utils/jwt-utils.ts
@@ -2,6 +2,8 @@ import jwt from 'jsonwebtoken'
export enum PosthogJwtAudience {
RECORDING_API = 'posthog:recording_api',
+ // Must match PosthogJwtAudience.REPLAY_BLOCKS in posthog/jwt.py exactly.
+ REPLAY_BLOCKS = 'posthog:replay:blocks',
SUBSCRIPTION_PREFERENCES = 'posthog:messaging:subscription_preferences',
WORKFLOWS_RESCHEDULE_PARKED = 'posthog:workflows:reschedule_parked',
WORKFLOWS_CANCEL_INVOCATIONS = 'posthog:workflows:cancel_invocations',
diff --git a/nodejs/src/common/api/body-limit.test.ts b/nodejs/src/common/api/body-limit.test.ts
index b57a1925ceb6..94dced979f92 100644
--- a/nodejs/src/common/api/body-limit.test.ts
+++ b/nodejs/src/common/api/body-limit.test.ts
@@ -8,7 +8,7 @@ describe('setupExpressApp JSON body limit', () => {
let server: ReturnType<Application['listen']>
beforeAll(() => {
- app = setupExpressApp({ jsonBodyLimit: '1kb' })
+ app = setupExpressApp({ jsonBodyLimit: '1kb', readOnlyPathPrefixes: ['/read-only/'] })
app.post('/echo', (req: Request, res: Response) => {
res.status(200).json({ ok: true })
})
@@ -34,6 +34,14 @@ describe('setupExpressApp JSON body limit', () => {
expect(res.body).toEqual({ status: 'error', error: 'Request entity too large' })
})
+ it('refuses a write to a read-only prefix before parsing its body', async () => {
+ const res = await supertest(app)
+ .post('/read-only/anything')
+ .send({ data: 'x'.repeat(2048) })
+
+ expect(res.status).toEqual(405)
+ })
+
it('accepts a JSON body under the limit', async () => {
const res = await supertest(app).post('/echo').send({ data: 'x' })
diff --git a/nodejs/src/common/api/router.ts b/nodejs/src/common/api/router.ts
index b4786e17ef60..c86aac876f3b 100644
--- a/nodejs/src/common/api/router.ts
+++ b/nodejs/src/common/api/router.ts
@@ -34,8 +34,13 @@ export interface SetupExpressAppOptions {
internalApiSecretFallbacks?: string
// JSON body size limit, e.g. '20mb'. Overridable in tests.
jsonBodyLimit?: string
+ // Path prefixes that only serve reads. Other methods get a 405 before the body parser buffers anything, so an
+ // unauthenticated caller cannot make the server parse a large body for a route that never reads one.
+ readOnlyPathPrefixes?: string[]
}
+const READ_METHODS = new Set(['GET', 'HEAD', 'OPTIONS'])
+
export function setupCommonRoutes(
app: express.Application,
services: Pick<PluginServerService, 'id' | 'healthcheck'>[]
@@ -70,6 +75,18 @@ export function setupExpressApp(options: SetupExpressAppOptions = {}): express.A
})
)
+ const readOnlyPathPrefixes = options.readOnlyPathPrefixes ?? []
+ if (readOnlyPathPrefixes.length > 0) {
+ app.use((req: Request, res: Response, next: NextFunction) => {
+ if (!READ_METHODS.has(req.method) && readOnlyPathPrefixes.some((prefix) => req.path.startsWith(prefix))) {
+ res.set('Allow', 'GET, HEAD, OPTIONS')
+ res.status(405).json({ error: 'Method not allowed' })
+ return
+ }
+ next()
+ })
+ }
+
app.use(
express.json({
limit: options.jsonBodyLimit ?? '20mb',
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/shared/keystore/cache/redis-cache.test.ts b/nodejs/src/ingestion/pipelines/sessionreplay/shared/keystore/cache/redis-cache.test.ts
index 55a6ea7e2334..4d2b6987c3bd 100644
--- a/nodejs/src/ingestion/pipelines/sessionreplay/shared/keystore/cache/redis-cache.test.ts
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/shared/keystore/cache/redis-cache.test.ts
@@ -134,6 +134,19 @@ describe('RedisCachedKeyStore', () => {
expect(result).toEqual(mockSessionKey)
})
+ it.each([
+ ['caches', 'ciphertext' as const, 1],
+ ['skips caching', 'cleartext' as const, 0],
+ ])('%s a %s key when built not to cache a missing key', async (_, sessionState, expectedWrites) => {
+ mockRedisClient.get.mockResolvedValue(null)
+ mockDelegate.getKey.mockResolvedValue({ ...mockSessionKey, sessionState })
+ const store = new RedisCachedKeyStore(mockDelegate, mockRedisPool, undefined, false)
+
+ await store.getKey('session-123', 1)
+
+ expect(mockRedisClient.set).toHaveBeenCalledTimes(expectedWrites)
+ })
+
it.each([
['set rejects', () => mockRedisClient.set.mockRejectedValue(new Error('READONLY'))],
[
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/shared/keystore/cache/redis-cache.ts b/nodejs/src/ingestion/pipelines/sessionreplay/shared/keystore/cache/redis-cache.ts
index ac25726c659f..5492fda06215 100644
--- a/nodejs/src/ingestion/pipelines/sessionreplay/shared/keystore/cache/redis-cache.ts
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/shared/keystore/cache/redis-cache.ts
@@ -21,7 +21,10 @@ export class RedisCachedKeyStore implements KeyStore {
constructor(
private readonly delegate: KeyStore,
private readonly redisPool: RedisPool,
- private readonly ttlSeconds: number = REDIS_CACHE_TTL_SECONDS
+ private readonly ttlSeconds: number = REDIS_CACHE_TTL_SECONDS,
+ // A reader that may ask about a session ingestion has not keyed yet, or before DynamoDB's eventually consistent
+ // read sees a new key, passes false: caching that miss would pin the session to cleartext for the cache lifetime.
+ private readonly cacheMissingKeys: boolean = true
) {}
private cacheKey(sessionId: string, teamId: number): string {
@@ -95,7 +98,9 @@ export class RedisCachedKeyStore implements KeyStore {
}
const key = await this.delegate.getKey(sessionId, teamId)
- await this.setCached(sessionId, teamId, key, 'if_absent')
+ if (this.cacheMissingKeys || key.sessionState !== 'cleartext') {
+ await this.setCached(sessionId, teamId, key, 'if_absent')
+ }
return key
}
diff --git a/nodejs/src/ingestion/pipelines/sessionreplay/shared/keystore/index.ts b/nodejs/src/ingestion/pipelines/sessionreplay/shared/keystore/index.ts
index fa29a7b2a252..75d5b429b13c 100644
--- a/nodejs/src/ingestion/pipelines/sessionreplay/shared/keystore/index.ts
+++ b/nodejs/src/ingestion/pipelines/sessionreplay/shared/keystore/index.ts
@@ -16,6 +16,8 @@ export { MemoryKeyStore } from './memory-keystore'
export interface KeyStoreConfig {
kmsEndpoint?: string
dynamoDBEndpoint?: string
+ // Per-request deadline for the AWS calls, which then abort and retry once. Unset keeps the SDK default: no deadline.
+ requestTimeoutMs?: number
}
export function getKeyStore(region: string, config?: KeyStoreConfig): KeyStore {
@@ -26,14 +28,19 @@ export function getKeyStore(region: string, config?: KeyStoreConfig): KeyStore {
dynamoDBEndpoint: config?.dynamoDBEndpoint ?? 'default',
})
- const kmsClient = new KMSClient({
- region,
- endpoint: config?.kmsEndpoint,
- })
- const dynamoDBClient = new DynamoDBClient({
- region,
- endpoint: config?.dynamoDBEndpoint,
- })
+ // Without throwOnRequestTimeout the SDK only logs a warning at the deadline and keeps waiting.
+ const bounded = config?.requestTimeoutMs
+ ? {
+ requestHandler: {
+ requestTimeout: config.requestTimeoutMs,
+ connectionTimeout: config.requestTimeoutMs,
+ throwOnRequestTimeout: true,
+ },
+ maxAttempts: 2,
+ }
+ : {}
+ const kmsClient = new KMSClient({ region, endpoint: config?.kmsEndpoint, ...bounded })
+ const dynamoDBClient = new DynamoDBClient({ region, endpoint: config?.dynamoDBEndpoint, ...bounded })
return new DynamoDBKeyStore(dynamoDBClient, kmsClient)
}
diff --git a/nodejs/src/servers/recording-api-server.ts b/nodejs/src/servers/recording-api-server.ts
index 1c5cc4113beb..588e54d4d2e2 100644
--- a/nodejs/src/servers/recording-api-server.ts
+++ b/nodejs/src/servers/recording-api-server.ts
@@ -11,7 +11,10 @@ import {
KafkaSessionreplayProducerEnvConfig,
getDefaultKafkaSessionreplayProducerEnvConfig,
} from '~/ingestion/pipelines/sessionreplay/shared/outputs/producer-config'
-import { assertRecordingApiAuthConfigured } from '~/session-replay/recording-api/auth'
+import {
+ assertRecordingApiRoleAuthConfigured,
+ internalApiAuthExcludedPathPrefixes,
+} from '~/session-replay/recording-api/auth'
import { createProducerRegistry } from '~/session-replay/recording-api/outputs/producer-registry'
import { createOutputsRegistry } from '~/session-replay/recording-api/outputs/registry'
import { RecordingApi } from '~/session-replay/recording-api/recording-api'
@@ -19,8 +22,11 @@ import {
RecordingApiConfig,
RecordingApiOutputsConfig,
type RecordingApiProducerName,
+ RecordingApiRole,
getDefaultRecordingApiAuthConfig,
+ getDefaultRecordingApiBlocksConfig,
getDefaultRecordingApiOutputsConfig,
+ parseRecordingApiRole,
} from '~/session-replay/recording-api/types'
import { CommonConfig } from '../common/config'
@@ -44,6 +50,7 @@ export class RecordingApiServer implements NodeServer {
private postgres?: PostgresRouter
private producerRegistry?: KafkaProducerRegistry<RecordingApiProducerName>
+ private readonly role: RecordingApiRole
constructor(config: Partial<RecordingApiServerConfig> = {}) {
this.config = {
@@ -53,12 +60,15 @@ export class RecordingApiServer implements NodeServer {
...overrideConfigWithEnv(getDefaultKafkaSessionreplayProducerEnvConfig()),
...overrideConfigWithEnv(getDefaultRecordingApiOutputsConfig()),
...overrideConfigWithEnv(getDefaultRecordingApiAuthConfig()),
+ ...overrideConfigWithEnv(getDefaultRecordingApiBlocksConfig()),
...config,
}
+ this.role = parseRecordingApiRole(this.config.RECORDING_API_ROLE)
// recording-api verifies a team-scoped JWT per route, so exempt its routes from the
// shared-secret middleware (callers send a Bearer token, not X-Internal-Api-Secret).
this.lifecycle = new ServerLifecycle(this.config, {
- internalApiAuthExcludedPathPrefixes: ['/api/projects/'],
+ internalApiAuthExcludedPathPrefixes: internalApiAuthExcludedPathPrefixes(this.role),
+ readOnlyPathPrefixes: ['/replay-blocks/'],
})
}
@@ -74,22 +84,28 @@ export class RecordingApiServer implements NodeServer {
}
private async startServices(): Promise<void> {
- assertRecordingApiAuthConfigured({
+ assertRecordingApiRoleAuthConfigured({
+ role: this.role,
isProd: isProdEnv(),
+ blocksJwtSecret: this.config.RECORDING_API_BLOCKS_JWT_SECRET,
jwtSecret: this.config.RECORDING_API_JWT_SECRET,
allowLegacySecret: this.config.RECORDING_API_ALLOW_LEGACY_SECRET,
legacySecret: this.config.INTERNAL_API_SECRET,
})
- this.postgres = new PostgresRouter(this.config, this.config.PLUGIN_SERVER_MODE ?? undefined)
- logger.info('👍', 'Postgres Router ready')
+ // A public process serves only browser block reads, so it never connects to Postgres or Kafka.
+ let outputs: ReturnType<ReturnType<typeof createOutputsRegistry>['build']> | undefined
+ if (this.role !== 'public') {
+ this.postgres = new PostgresRouter(this.config, this.config.PLUGIN_SERVER_MODE ?? undefined)
+ logger.info('👍', 'Postgres Router ready')
- logger.info('🤔', 'Connecting to Kafka...')
- this.producerRegistry = await createProducerRegistry(this.config.KAFKA_CLIENT_RACK).build(this.config)
- const outputs = createOutputsRegistry().build(this.producerRegistry, this.config)
- logger.info('👍', 'Kafka ready')
+ logger.info('🤔', 'Connecting to Kafka...')
+ this.producerRegistry = await createProducerRegistry(this.config.KAFKA_CLIENT_RACK).build(this.config)
+ outputs = createOutputsRegistry().build(this.producerRegistry, this.config)
+ logger.info('👍', 'Kafka ready')
+ }
- const api = new RecordingApi(this.config, this.postgres!, outputs)
+ const api = new RecordingApi(this.config, this.postgres, outputs)
this.lifecycle.expressApp.use('/', api.router())
await api.start()
this.lifecycle.services.push(api.service)
diff --git a/nodejs/src/session-replay/recording-api/auth.test.ts b/nodejs/src/session-replay/recording-api/auth.test.ts
index db105dca2b83..2ddec0664789 100644
--- a/nodejs/src/session-replay/recording-api/auth.test.ts
+++ b/nodejs/src/session-replay/recording-api/auth.test.ts
@@ -2,7 +2,13 @@ import { NextFunction, Request, Response } from 'ultimate-express'
import { JWT, PosthogJwtAudience } from '~/cdp/utils/jwt-utils'
-import { RecordingApiAuthOptions, assertRecordingApiAuthConfigured, createRecordingApiAuthMiddleware } from './auth'
+import {
+ RecordingApiAuthOptions,
+ assertRecordingApiAuthConfigured,
+ assertRecordingApiRoleAuthConfigured,
+ createRecordingApiAuthMiddleware,
+ internalApiAuthExcludedPathPrefixes,
+} from './auth'
describe('createRecordingApiAuthMiddleware', () => {
const JWT_SECRET = 'recording-api-secret-key'
@@ -283,4 +289,34 @@ describe('assertRecordingApiAuthConfigured', () => {
it('passes outside production even with nothing configured (local dev)', () => {
expect(() => assertRecordingApiAuthConfigured({ ...base, isProd: false })).not.toThrow()
})
+
+ it.each([
+ ['public', '', 'k', /RECORDING_API_BLOCKS_JWT_SECRET/],
+ ['all', '', 'k', /RECORDING_API_BLOCKS_JWT_SECRET/],
+ ['all', 'b', '', /no auth configured in production/],
+ ['internal', '', '', /no auth configured in production/],
+ ] as const)(
+ 'refuses to start a %s process without the secrets it serves',
+ (role, blocksJwtSecret, jwtSecret, error) => {
+ expect(() => assertRecordingApiRoleAuthConfigured({ ...base, role, blocksJwtSecret, jwtSecret })).toThrow(
+ error
+ )
+ }
+ )
+
+ it.each([
+ ['public', 'b', ''],
+ ['all', 'b', 'k'],
+ ['internal', '', 'k'],
+ ] as const)('starts a %s process holding only the secrets it serves', (role, blocksJwtSecret, jwtSecret) => {
+ expect(() => assertRecordingApiRoleAuthConfigured({ ...base, role, blocksJwtSecret, jwtSecret })).not.toThrow()
+ })
+
+ it.each([
+ ['internal', false],
+ ['public', true],
+ ['all', true],
+ ] as const)('a %s process exempts the browser route from the shared secret: %s', (role, exempt) => {
+ expect(internalApiAuthExcludedPathPrefixes(role).includes('/replay-blocks/')).toBe(exempt)
+ })
})
diff --git a/nodejs/src/session-replay/recording-api/auth.ts b/nodejs/src/session-replay/recording-api/auth.ts
index f80103f88358..f79494f5946c 100644
--- a/nodejs/src/session-replay/recording-api/auth.ts
+++ b/nodejs/src/session-replay/recording-api/auth.ts
@@ -5,6 +5,8 @@ import { JWT, PosthogJwtAudience, hasJwtKeys, makeOptionalJwt } from '~/cdp/util
import { INTERNAL_SERVICE_CALL_HEADER_NAME } from '~/common/api/middleware/internal-api-auth'
import { logger } from '~/common/utils/logger'
+import { RecordingApiRole } from './types'
+
/**
* Authorization for recording-api routes.
*
@@ -49,6 +51,29 @@ function pathTeamId(req: Request): number {
return match && /^(0|[1-9]\d*)$/.test(match[1]) ? parseInt(match[1], 10) : NaN
}
+// Pinned to HS256 to match the Python minter, and logged by failure class only, never the token.
+function verifyBearer(
+ jwt: JWT,
+ token: string,
+ audience: PosthogJwtAudience,
+ options: { maxAge?: string } = {}
+): Record<string, unknown> | undefined {
+ try {
+ const payload = jwt.verify(token, audience, {
+ algorithms: ['HS256'],
+ clockTolerance: CLOCK_TOLERANCE_SECONDS,
+ ...options,
+ })
+ return payload && typeof payload === 'object' ? (payload as Record<string, unknown>) : undefined
+ } catch (err) {
+ logger.warn('Recording API JWT verification failed', {
+ audience,
+ reason: err instanceof Error ? err.name : 'unknown',
+ })
+ return undefined
+ }
+}
+
function timingSafeEqual(a: string, b: string): boolean {
// Trim so a secret mounted from a file with a trailing newline still matches a sender that
// trimmed it (mirrors the shared internal-api-auth middleware's newline handling).
@@ -79,6 +104,32 @@ export function assertRecordingApiAuthConfigured(opts: {
}
}
+/**
+ * Each role must hold exactly the auth it serves. A process serving the browser route needs the blocks secret in
+ * production, since no other secret is accepted there. A public process holds no service secret, so it skips the
+ * internal check that would otherwise demand one.
+ */
+export function assertRecordingApiRoleAuthConfigured(opts: {
+ role: RecordingApiRole
+ isProd: boolean
+ blocksJwtSecret: string
+ jwtSecret: string
+ allowLegacySecret: boolean
+ legacySecret: string
+}): void {
+ if (opts.role !== 'internal' && opts.isProd && !hasJwtKeys(opts.blocksJwtSecret || '')) {
+ throw new Error('recording-api serving browser blocks needs RECORDING_API_BLOCKS_JWT_SECRET in production')
+ }
+ if (opts.role !== 'public') {
+ assertRecordingApiAuthConfigured(opts)
+ }
+}
+
+// The browser block route checks its own browser token, so only a process serving it exempts it from the shared secret.
+export function internalApiAuthExcludedPathPrefixes(role: RecordingApiRole): string[] {
+ return ['/api/projects/', ...(role === 'internal' ? [] : ['/replay-blocks/'])]
+}
+
export function createRecordingApiAuthMiddleware(options: RecordingApiAuthOptions) {
const { jwtSecret, legacySecret, allowLegacySecret, op } = options
// Build the verifier once. Comma-separated keys give zero-downtime rotation (verify against all);
@@ -98,25 +149,9 @@ export function createRecordingApiAuthMiddleware(options: RecordingApiAuthOption
if (jwt) {
const bearer = getBearerToken(req)
if (bearer) {
- let payload: ReturnType<JWT['verify']>
- try {
- payload = jwt.verify(bearer, PosthogJwtAudience.RECORDING_API, {
- // Pin HS256 to match the Python minter (encode/decode pin JWT_ALGORITHM);
- // don't accept other HMAC variants the library would otherwise allow.
- algorithms: ['HS256'],
- clockTolerance: CLOCK_TOLERANCE_SECONDS,
- })
- } catch (err) {
- // Log the failure class (e.g. TokenExpiredError, JsonWebTokenError) — never the token —
- // so rollout/rotation 401s are diagnosable. Then fall through to the legacy secret or 401.
- logger.warn('Recording API JWT verification failed', {
- reason: err instanceof Error ? err.name : 'unknown',
- path: req.path,
- })
- payload = undefined
- }
- if (payload && typeof payload === 'object') {
- const claims = payload as { team_id?: unknown; op?: unknown }
+ // An invalid token falls through to the legacy secret (transition) or a 401.
+ const claims = verifyBearer(jwt, bearer, PosthogJwtAudience.RECORDING_API)
+ if (claims) {
const tokenTeamId = Number(claims.team_id)
const requestTeamId = pathTeamId(req)
if (!Number.isInteger(tokenTeamId) || tokenTeamId !== requestTeamId) {
@@ -147,3 +182,43 @@ export function createRecordingApiAuthMiddleware(options: RecordingApiAuthOption
res.status(401).json({ error: 'Unauthorized' })
}
}
+
+// A browser block token older than this is refused whatever its exp, so a leaked one stops working soon.
+const REPLAY_BLOCKS_TOKEN_MAX_AGE = '2h'
+
+// The browser block route accepts only browser tokens, a separate audience from the service tokens above.
+// There is no legacy fallback and no dev bypass: without a secret, every request is refused.
+export function createReplayBlocksAuthMiddleware(jwtSecret: string) {
+ const jwt = makeOptionalJwt(jwtSecret)
+
+ return (req: Request, res: Response, next: NextFunction): void => {
+ // Only a served block may be cached, so every other answer on this route says no-store.
+ res.set('Cache-Control', 'no-store')
+ const bearer = getBearerToken(req)
+ const claims =
+ jwt && bearer
+ ? verifyBearer(jwt, bearer, PosthogJwtAudience.REPLAY_BLOCKS, { maxAge: REPLAY_BLOCKS_TOKEN_MAX_AGE })
+ : undefined
+ if (!claims || typeof claims.exp !== 'number') {
+ res.status(401).json({ error: 'Unauthorized' })
+ return
+ }
+ // String equality also requires the canonical path spelling, so /01/ never shares a cache entry with /1/.
+ if (
+ !Number.isInteger(claims.team_id) ||
+ String(claims.team_id) !== req.params.team_id ||
+ claims.session_id !== req.params.session_id ||
+ // The viewer is in the URL so each viewer's blocks are separate browser cache entries: after an account
+ // switch the new viewer's URLs never match the previous viewer's cached blocks.
+ typeof claims.sub !== 'string' ||
+ !claims.sub ||
+ claims.sub !== req.params.viewer
+ ) {
+ res.status(403).json({ error: 'Forbidden: token not scoped to this recording' })
+ return
+ }
+ res.locals.replayBlocksCaller = claims.sub
+ res.locals.replayBlocksTier = claims.tier
+ next()
+ }
+}
diff --git a/nodejs/src/session-replay/recording-api/block-refs.test.ts b/nodejs/src/session-replay/recording-api/block-refs.test.ts
new file mode 100644
index 000000000000..8410372d2093
--- /dev/null
+++ b/nodejs/src/session-replay/recording-api/block-refs.test.ts
@@ -0,0 +1,50 @@
+import sodium from 'libsodium-wrappers'
+
+import { BlockLocation, blockRefEncoder, decodeBlockRef } from './block-refs'
+
+describe('block refs', () => {
+ const sessionKey = new Uint8Array(32).fill(1)
+ const location: BlockLocation = {
+ teamId: 1,
+ sessionId: 'session-123',
+ key: 'session_recordings/30d/1764634738680-3cca0f5d3c7cc7ee',
+ startByte: 100,
+ endByte: 250,
+ }
+
+ beforeAll(async () => {
+ await sodium.ready
+ })
+
+ it('names a block the same way every time, so its URL stays cacheable', () => {
+ const ref = blockRefEncoder(sessionKey)(location)
+
+ expect(blockRefEncoder(sessionKey)(location)).toBe(ref)
+ expect(ref).not.toContain('session_recordings')
+ expect(decodeBlockRef(sessionKey, ref, 1, 'session-123')).toEqual({
+ key: location.key,
+ startByte: 100,
+ endByte: 250,
+ })
+ })
+
+ it.each([
+ ['another session key', (ref: string) => decodeBlockRef(new Uint8Array(32).fill(2), ref, 1, 'session-123')],
+ ['another session', (ref: string) => decodeBlockRef(sessionKey, ref, 1, 'session-999')],
+ ['another team', (ref: string) => decodeBlockRef(sessionKey, ref, 2, 'session-123')],
+ [
+ 'a tampered ref',
+ (ref: string) =>
+ decodeBlockRef(sessionKey, ref.slice(0, -2) + (ref.endsWith('AA') ? 'BB' : 'AA'), 1, 'session-123'),
+ ],
+ [
+ 'an unknown version',
+ (ref: string) => decodeBlockRef(sessionKey, ref.replace(/^v1\./, 'v2.'), 1, 'session-123'),
+ ],
+ ['an empty body', () => decodeBlockRef(sessionKey, 'v1.', 1, 'session-123')],
+ ['a body shorter than its nonce', () => decodeBlockRef(sessionKey, 'v1.AAAA', 1, 'session-123')],
+ ['an extra segment', (ref: string) => decodeBlockRef(sessionKey, `${ref}.x`, 1, 'session-123')],
+ ])('rejects a ref read with %s', (_, decode) => {
+ expect(decode(blockRefEncoder(sessionKey)(location))).toBeNull()
+ })
+})
diff --git a/nodejs/src/session-replay/recording-api/block-refs.ts b/nodejs/src/session-replay/recording-api/block-refs.ts
new file mode 100644
index 000000000000..df806c58e194
--- /dev/null
+++ b/nodejs/src/session-replay/recording-api/block-refs.ts
@@ -0,0 +1,83 @@
+import sodium from 'libsodium-wrappers'
+
+import { parseJSON } from '~/common/utils/json-parse'
+
+// A ref names a block without revealing its storage key: the location, encrypted with keys derived from the
+// session key, under a nonce hashed from the location, so the same block always gets the same cacheable URL.
+// Deriving from the session key is deliberate: shredding that key makes every ref for the session unreadable.
+export interface BlockLocation {
+ teamId: number
+ sessionId: string
+ key: string
+ startByte: number
+ endByte: number
+}
+
+const REF_VERSION = 'v1'
+// crypto_kdf contexts are exactly 8 bytes.
+const KDF_CONTEXT = 'blockref'
+const ENCRYPTION_SUBKEY_ID = 1
+const NONCE_SUBKEY_ID = 2
+
+function encryptionKeyFor(sessionKey: Uint8Array): Uint8Array {
+ return sodium.crypto_kdf_derive_from_key(
+ sodium.crypto_secretbox_KEYBYTES,
+ ENCRYPTION_SUBKEY_ID,
+ KDF_CONTEXT,
+ sessionKey
+ )
+}
+
+function serialize(location: BlockLocation): Uint8Array {
+ const { teamId, sessionId, key, startByte, endByte } = location
+ return sodium.from_string(JSON.stringify([teamId, sessionId, key, startByte, endByte]))
+}
+
+// Derives the subkeys once, for naming every block of one listing. Callers must await sodium.ready first.
+export function blockRefEncoder(sessionKey: Uint8Array): (location: BlockLocation) => string {
+ const encryptionKey = encryptionKeyFor(sessionKey)
+ const nonceKey = sodium.crypto_kdf_derive_from_key(
+ sodium.crypto_generichash_KEYBYTES,
+ NONCE_SUBKEY_ID,
+ KDF_CONTEXT,
+ sessionKey
+ )
+ return (location) => {
+ const plaintext = serialize(location)
+ const nonce = sodium.crypto_generichash(sodium.crypto_secretbox_NONCEBYTES, plaintext, nonceKey)
+ const ciphertext = sodium.crypto_secretbox_easy(plaintext, nonce, encryptionKey)
+ const payload = new Uint8Array(nonce.length + ciphertext.length)
+ payload.set(nonce)
+ payload.set(ciphertext, nonce.length)
+ return `${REF_VERSION}.${sodium.to_base64(payload, sodium.base64_variants.URLSAFE_NO_PADDING)}`
+ }
+}
+
+/** Returns null for any ref this session key did not produce for this team and session. */
+export function decodeBlockRef(
+ sessionKey: Uint8Array,
+ ref: string,
+ teamId: number,
+ sessionId: string
+): Omit<BlockLocation, 'teamId' | 'sessionId'> | null {
+ const [version, encoded, ...rest] = ref.split('.')
+ if (version !== REF_VERSION || !encoded || rest.length > 0) {
+ return null
+ }
+ try {
+ const payload = sodium.from_base64(encoded, sodium.base64_variants.URLSAFE_NO_PADDING)
+ const nonce = payload.subarray(0, sodium.crypto_secretbox_NONCEBYTES)
+ const ciphertext = payload.subarray(sodium.crypto_secretbox_NONCEBYTES)
+ const plaintext = sodium.crypto_secretbox_open_easy(ciphertext, nonce, encryptionKeyFor(sessionKey))
+ const [refTeamId, refSessionId, key, startByte, endByte] = parseJSON(sodium.to_string(plaintext))
+ if (refTeamId !== teamId || refSessionId !== sessionId) {
+ return null
+ }
+ if (typeof key !== 'string' || !Number.isInteger(startByte) || !Number.isInteger(endByte)) {
+ return null
+ }
+ return { key, startByte, endByte }
+ } catch {
+ return null
+ }
+}
diff --git a/nodejs/src/session-replay/recording-api/metrics.ts b/nodejs/src/session-replay/recording-api/metrics.ts
index 38875d8658ff..ea847550beaa 100644
--- a/nodejs/src/session-replay/recording-api/metrics.ts
+++ b/nodejs/src/session-replay/recording-api/metrics.ts
@@ -10,6 +10,13 @@ export class RecordingApiMetrics {
buckets: [0.01, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10],
})
+ private static readonly keyStoreDuration = new Histogram({
+ name: 'recording_api_keystore_duration_seconds',
+ help: 'Time taken by one call to the KMS and DynamoDB key store, below the Redis cache',
+ labelNames: ['operation', 'result'],
+ buckets: [0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2, 3, 5, 10],
+ })
+
private static readonly deleteRecordingsDuration = new Histogram({
name: 'recording_api_delete_recordings_duration_seconds',
help: 'Time taken to delete recordings',
@@ -17,6 +24,14 @@ export class RecordingApiMetrics {
buckets: [0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30],
})
+ public static observeKeyStore(
+ operation: 'generate' | 'get' | 'delete',
+ result: 'ok' | 'error',
+ seconds: number
+ ): void {
+ this.keyStoreDuration.observe({ operation, result }, seconds)
+ }
+
public static observeGetBlock(
result: string,
seconds: number,
@@ -75,4 +90,23 @@ export class RecordingApiMetrics {
public static incrementClickhousePasswordFallback(reason: ClickHousePasswordFallbackReason): void {
this.clickhousePasswordFallbacks.labels({ reason }).inc()
}
+
+ private static readonly replayBlockRequests = new Counter({
+ name: 'recording_api_replay_block_requests_total',
+ help: 'Browser block route requests by outcome',
+ labelNames: ['result'],
+ })
+
+ public static incrementReplayBlockRequest(
+ result:
+ | 'ok'
+ | 'not_found'
+ | 'deleted'
+ | 'rate_limited'
+ | 'rate_limited_log_only'
+ | 'rate_limit_unavailable'
+ | 'error'
+ ): void {
+ this.replayBlockRequests.labels({ result }).inc()
+ }
}
diff --git a/nodejs/src/session-replay/recording-api/recording-api.test.ts b/nodejs/src/session-replay/recording-api/recording-api.test.ts
index 11e95381f547..c98e12e8fb91 100644
--- a/nodejs/src/session-replay/recording-api/recording-api.test.ts
+++ b/nodejs/src/session-replay/recording-api/recording-api.test.ts
@@ -83,6 +83,8 @@ describe('RecordingApi', () => {
CLICKHOUSE_USER: 'default',
CLICKHOUSE_PASSWORD: undefined,
CLICKHOUSE_SECURE: false,
+ RECORDING_API_AWS_TIMEOUT_MS: 5000,
+ RECORDING_API_REDIS_TIMEOUT_MS: 2000,
}
mockKeyStore = {
@@ -103,7 +105,10 @@ describe('RecordingApi', () => {
formatS3KeyError: jest.fn().mockReturnValue('Invalid key format'),
getBlock: jest.fn(),
listBlocks: jest.fn(),
- getDeletion: jest.fn().mockResolvedValue(null),
+ annotateBlocks: jest
+ .fn()
+ .mockImplementation((_s: string, _t: number, blocks: unknown) => Promise.resolve(blocks)),
+ getBlockByRef: jest.fn(),
deleteRecordings: jest.fn(),
encryptBlocks: jest.fn(),
} as unknown as jest.Mocked<RecordingService>
@@ -137,8 +142,9 @@ describe('RecordingApi', () => {
expect(getKeyStore).toHaveBeenCalledWith('us-west-2', {
kmsEndpoint: undefined,
dynamoDBEndpoint: undefined,
+ requestTimeoutMs: 5000,
})
- expect(getBlockDecryptor).toHaveBeenCalledWith(mockKeyStore)
+ expect(getBlockDecryptor).toHaveBeenCalled()
expect(RecordingService).toHaveBeenCalled()
})
@@ -439,7 +445,7 @@ describe('RecordingApi', () => {
end_timestamp: '2024-01-01T00:01:00Z',
},
])
- mockService.getDeletion.mockResolvedValue({
+ mockService.annotateBlocks.mockResolvedValue({
ok: false,
error: 'deleted',
deletedAt: 1700000000,
@@ -454,7 +460,7 @@ describe('RecordingApi', () => {
deleted_at: 1700000000,
deleted_by: '[email protected]',
})
- expect(mockService.getDeletion).toHaveBeenCalledWith('session-123', 1)
+ expect(mockService.annotateBlocks).toHaveBeenCalledWith('session-123', 1, expect.any(Array))
})
it('still lists blocks when the key store does not answer', async () => {
@@ -467,7 +473,7 @@ describe('RecordingApi', () => {
end_timestamp: '2024-01-01T00:01:00Z',
},
])
- mockService.getDeletion.mockReturnValue(new Promise(() => {}))
+ mockService.annotateBlocks.mockReturnValue(new Promise(() => {}))
const res = await supertest(app).get('/api/projects/1/recordings/session-123/blocks')
@@ -481,7 +487,7 @@ describe('RecordingApi', () => {
const res = await supertest(app).get('/api/projects/1/recordings/session-123/blocks')
expect(res.status).toBe(200)
- expect(mockService.getDeletion).not.toHaveBeenCalled()
+ expect(mockService.annotateBlocks).not.toHaveBeenCalled()
})
it('still lists blocks when the key store fails', async () => {
@@ -494,7 +500,7 @@ describe('RecordingApi', () => {
end_timestamp: '2024-01-01T00:01:00Z',
},
])
- mockService.getDeletion.mockRejectedValue(new Error('DynamoDB unavailable'))
+ mockService.annotateBlocks.mockRejectedValue(new Error('DynamoDB unavailable'))
const res = await supertest(app).get('/api/projects/1/recordings/session-123/blocks')
@@ -502,6 +508,22 @@ describe('RecordingApi', () => {
expect(res.body.blocks).toHaveLength(1)
})
+ it('passes block refs through to the listing', async () => {
+ const block = {
+ key: 'session_recordings/30d/1000-aaa',
+ start_byte: 0,
+ end_byte: 100,
+ start_timestamp: '2024-01-01T00:00:00Z',
+ end_timestamp: '2024-01-01T00:01:00Z',
+ }
+ mockService.listBlocks.mockResolvedValue([block])
+ mockService.annotateBlocks.mockResolvedValue([{ ...block, ref: 'v1.abc' }])
+
+ const res = await supertest(app).get('/api/projects/1/recordings/session-123/blocks')
+
+ expect(res.body).toEqual({ blocks: [{ ...block, ref: 'v1.abc' }] })
+ })
+
it('should return empty blocks when session not found', async () => {
mockService.listBlocks.mockResolvedValue([])
@@ -795,4 +817,190 @@ describe('RecordingApi', () => {
expect(mockService.encryptBlocks).toHaveBeenCalledWith('session-1', 1, [Buffer.from('block')], 365)
})
})
+
+ describe('browser block route', () => {
+ const JWT_SECRET = 'recording-api-test-secret'
+ const BLOCKS_JWT_SECRET = 'recording-api-blocks-test-secret'
+ const BLOCK_URL = '/replay-blocks/1/session-123/u7/v1.abc'
+ let server: Server
+ let rateLimiter: { rateLimitGrouped: jest.Mock }
+
+ const browserToken = (
+ claims: Record<string, unknown> = {},
+ expiresIn: string | null = '5m',
+ secret = BLOCKS_JWT_SECRET
+ ): string =>
+ `Bearer ${new JWT(secret).sign(
+ { team_id: 1, session_id: 'session-123', sub: 'u7', tier: 'browser', ...claims },
+ PosthogJwtAudience.REPLAY_BLOCKS,
+ expiresIn ? { expiresIn } : {}
+ )}`
+ const serviceToken = (): string =>
+ `Bearer ${new JWT(JWT_SECRET).sign({ team_id: 1, op: 'read' }, PosthogJwtAudience.RECORDING_API, { expiresIn: '5m' })}`
+
+ const buildApp = async (role: string, enforced = false): Promise<express.Application> => {
+ const recordingApi = new RecordingApi(
+ {
+ RECORDING_API_JWT_SECRET: JWT_SECRET,
+ RECORDING_API_BLOCKS_JWT_SECRET: BLOCKS_JWT_SECRET,
+ RECORDING_API_ALLOW_LEGACY_SECRET: false,
+ RECORDING_API_ROLE: role,
+ RECORDING_API_BLOCK_RATE_LIMIT_ENFORCED: enforced,
+ } as RecordingApiConfig,
+ mockPostgres,
+ mockOutputs
+ )
+ await recordingApi.start(mockService, rateLimiter as any)
+ const app = express()
+ app.use('/', recordingApi.router())
+ server = app.listen(0, () => {})
+ return app
+ }
+
+ beforeEach(() => {
+ rateLimiter = {
+ rateLimitGrouped: jest
+ .fn()
+ .mockImplementation((requests: { id: string }[]) =>
+ Promise.resolve(
+ requests.map((r) => [r.id, { isRateLimited: false, tokens: 1, tokensBefore: 2 }])
+ )
+ ),
+ }
+ mockService.getBlockByRef.mockResolvedValue({ ok: true, data: Buffer.from('block-bytes') })
+ })
+
+ afterEach(() => {
+ server.close()
+ })
+
+ it('serves a block a browser may cache, without the cache varying on cookies', async () => {
+ const app = await buildApp('public')
+
+ const res = await supertest(app).get(BLOCK_URL).set('Authorization', browserToken())
+
+ expect(res.status).toBe(200)
+ expect(res.headers['cache-control']).toBe('private, max-age=86400, immutable')
+ expect(res.headers['vary'] ?? '').not.toMatch(/cookie/i)
+ expect(Buffer.from(res.body).toString()).toBe('block-bytes')
+ expect(mockService.getBlockByRef).toHaveBeenCalledWith(1, 'session-123', 'v1.abc')
+ expect(rateLimiter.rateLimitGrouped).toHaveBeenCalledWith([
+ { id: 'viewer:u7', cost: 1, bucketSize: 2000, refillRate: 8 },
+ ])
+ })
+
+ it.each([
+ ['no token', undefined, 401],
+ ['a token for another session', () => browserToken({ session_id: 'session-999' }), 403],
+ ['a token for another team', () => browserToken({ team_id: 2 }), 403],
+ ['a token with no caller', () => browserToken({ sub: undefined }), 403],
+ ['a token for another viewer', () => browserToken({ sub: 'u8' }), 403],
+ ['a token with a string team', () => browserToken({ team_id: '1' }), 403],
+ ['an internal service token', serviceToken, 401],
+ ['a browser token signed with the service secret', () => browserToken({}, '5m', JWT_SECRET), 401],
+ ['an expired token', () => browserToken({}, '-5m'), 401],
+ ['a token that never expires', () => browserToken({}, null), 401],
+ [
+ 'a token issued over two hours ago',
+ () => browserToken({ iat: Math.floor(Date.now() / 1000) - 3 * 3600 }, '1d'),
+ 401,
+ ],
+ ])('refuses %s', async (_, token, expectedStatus) => {
+ const app = await buildApp('public')
+
+ const request = supertest(app).get(BLOCK_URL)
+ const res = await (token ? request.set('Authorization', token()) : request)
+
+ expect(res.status).toBe(expectedStatus)
+ expect(res.headers['cache-control']).toBe('no-store')
+ expect(mockService.getBlockByRef).not.toHaveBeenCalled()
+ })
+
+ it('refuses a team id spelled with a leading zero', async () => {
+ const app = await buildApp('public')
+
+ const res = await supertest(app)
+ .get('/replay-blocks/01/session-123/u7/v1.abc')
+ .set('Authorization', browserToken())
+
+ expect(res.status).toBe(403)
+ })
+
+ it.each([
+ ['deleted', { ok: false, error: 'deleted', deletedAt: 1700000000, deletedBy: '[email protected]' }, 410],
+ ['an unknown ref', { ok: false, error: 'not_found' }, 404],
+ ])('answers a block that is %s without caching it', async (_, result, expectedStatus) => {
+ mockService.getBlockByRef.mockResolvedValue(result as any)
+ const app = await buildApp('public')
+
+ const res = await supertest(app).get(BLOCK_URL).set('Authorization', browserToken())
+
+ expect(res.status).toBe(expectedStatus)
+ expect(res.headers['cache-control']).toBe('no-store')
+ })
+
+ it.each([
+ ['enforced', true, 429],
+ ['log only', false, 200],
+ ])('applies the per-caller rate limit when %s', async (_, enforced, expectedStatus) => {
+ rateLimiter.rateLimitGrouped.mockResolvedValue([
+ ['u7', { isRateLimited: true, tokens: -1, tokensBefore: 0 }],
+ ])
+ const app = await buildApp('public', enforced)
+
+ const res = await supertest(app).get(BLOCK_URL).set('Authorization', browserToken())
+
+ expect(res.status).toBe(expectedStatus)
+ expect(res.headers['cache-control']).toBe(enforced ? 'no-store' : 'private, max-age=86400, immutable')
+ if (enforced) {
+ expect(res.headers['retry-after']).toBe('1')
+ }
+ })
+
+ it('serves the block when the rate limiter is unavailable', async () => {
+ rateLimiter.rateLimitGrouped.mockRejectedValue(new Error('Redis unavailable'))
+ const app = await buildApp('public')
+
+ const res = await supertest(app).get(BLOCK_URL).set('Authorization', browserToken())
+
+ expect(res.status).toBe(200)
+ })
+
+ it.each([
+ ['a browser viewer, by viewer', { tier: 'browser' }, 'viewer:u7', 2000],
+ ['a paid API key, by team', { sub: 'k42', tier: 'paid' }, 'team:1', 1200],
+ ['another key of the same team, in the same bucket', { sub: 'k43', tier: 'paid' }, 'team:1', 1200],
+ ['an enterprise API key, by team', { sub: 'k42', tier: 'enterprise' }, 'team:1', 2000],
+ ['a token with no tier, strictest', { sub: 'k42', tier: undefined }, 'team:1', 240],
+ ['an unknown tier, strictest', { sub: 'k42', tier: 'platinum' }, 'team:1', 240],
+ ])('rate-limits %s', async (_, claims, expectedId, expectedBucketSize) => {
+ const app = await buildApp('public')
+ const sub = (claims as { sub?: string }).sub ?? 'u7'
+
+ await supertest(app)
+ .get(`/replay-blocks/1/session-123/${sub}/v1.abc`)
+ .set('Authorization', browserToken(claims))
+
+ expect(rateLimiter.rateLimitGrouped).toHaveBeenCalledWith([
+ expect.objectContaining({ id: expectedId, bucketSize: expectedBucketSize }),
+ ])
+ })
+
+ it.each([
+ ['public', 200, 404],
+ ['internal', 404, 200],
+ ['all', 200, 200],
+ ])('a %s process serves only its own routes', async (role, browserStatus, internalStatus) => {
+ mockService.listBlocks.mockResolvedValue([])
+ const app = await buildApp(role)
+
+ const browser = await supertest(app).get(BLOCK_URL).set('Authorization', browserToken())
+ const internal = await supertest(app)
+ .get('/api/projects/1/recordings/session-123/blocks')
+ .set('Authorization', serviceToken())
+
+ expect(browser.status).toBe(browserStatus)
+ expect(internal.status).toBe(internalStatus)
+ })
+ })
})
diff --git a/nodejs/src/session-replay/recording-api/recording-api.ts b/nodejs/src/session-replay/recording-api/recording-api.ts
index 185f98cdb530..fcaa833c0593 100644
--- a/nodejs/src/session-replay/recording-api/recording-api.ts
+++ b/nodejs/src/session-replay/recording-api/recording-api.ts
@@ -4,6 +4,8 @@ import https from 'https'
import express from 'ultimate-express'
import { IngestionOutputs } from '~/common/outputs/ingestion-outputs'
+import { createRedisV2PoolFromConfig } from '~/common/redis/redis-v2'
+import { KeyedRateLimiterService } from '~/common/services/keyed-rate-limiter.service'
import { PostgresRouter } from '~/common/utils/db/postgres'
import { createRedisPoolFromConfig } from '~/common/utils/db/redis'
import { logger, serializeError } from '~/common/utils/logger'
@@ -16,19 +18,40 @@ import { SessionMetadataStore } from '~/ingestion/pipelines/sessionreplay/shared
import { ReplayEventsOutput, SessionFeaturesOutput } from '~/ingestion/pipelines/sessionreplay/shared/outputs'
import { HealthCheckResult, HealthCheckResultError, HealthCheckResultOk, PluginServerService, RedisPool } from '~/types'
-import { RecordingApiOp, createRecordingApiAuthMiddleware } from './auth'
+import { RecordingApiOp, createRecordingApiAuthMiddleware, createReplayBlocksAuthMiddleware } from './auth'
import { ClickHouseCredential } from './clickhouse-credential'
-import { RecordingDeletedResult, RecordingService } from './recording-service'
+import { RecordingApiMetrics } from './metrics'
+import { ListedBlock, RecordingDeletedResult, RecordingService } from './recording-service'
import {
DeleteRecordingsBodySchema,
EncryptBlocksBodySchema,
GetBlockQuerySchema,
RecordingParamsSchema,
+ ReplayBlockParamsSchema,
TeamParamsSchema,
} from './schemas'
-import { KeyStore, RecordingApiConfig, RecordingDecryptor, RecordingEncryptor } from './types'
-
-const DELETION_CHECK_TIMEOUT_MS = 1000
+import { TimedKeyStore } from './timed-keystore'
+import {
+ KeyStore,
+ REPLAY_BLOCK_TIERS,
+ RecordingApiConfig,
+ RecordingApiRole,
+ RecordingBlock,
+ RecordingDecryptor,
+ RecordingEncryptor,
+ ReplayBlockBucket,
+ ReplayBlockTier,
+ getDefaultRecordingApiBlocksConfig,
+ parseRecordingApiRole,
+ parseReplayBlockRateLimits,
+} from './types'
+
+// The browser keeps each block for a day. Django's sources listing, which hands out block URLs, reads the block
+// listing uncached and reports a deletion first, so the player never asks for a cached block of a deleted recording.
+const REPLAY_BLOCK_CACHE_CONTROL = 'private, max-age=86400, immutable'
+
+const KEY_LOOKUP_TIMEOUT_MS = 1000
+// Each Redis, DynamoDB and KMS call gives up after this, so a lookup the listing abandoned does not pile up.
function sendRecordingDeleted(res: express.Response, deletion: RecordingDeletedResult): void {
res.status(410).json({ error: 'recording_deleted', deleted_at: deletion.deletedAt, deleted_by: deletion.deletedBy })
@@ -44,12 +67,33 @@ export class RecordingApi {
private redisPool: RedisPool | null = null
private clickhouseClient: ClickHouseClient | null = null
private recordingService: RecordingService | null = null
+ private blockRateLimiter: KeyedRateLimiterService | null = null
+ private readonly role: RecordingApiRole
+ private readonly replayBlockBuckets: Record<ReplayBlockTier, ReplayBlockBucket> | null
+ // Postgres and the outputs are only used by internal routes, so a public process passes neither.
constructor(
private config: RecordingApiConfig,
- private postgres: PostgresRouter,
- private outputs: IngestionOutputs<ReplayEventsOutput | SessionFeaturesOutput>
- ) {}
+ private postgres?: PostgresRouter,
+ private outputs?: IngestionOutputs<ReplayEventsOutput | SessionFeaturesOutput>
+ ) {
+ this.role = parseRecordingApiRole(config.RECORDING_API_ROLE || 'internal')
+ // Only a process serving the browser route reads its limits, so a bad value cannot stop an internal pod.
+ this.replayBlockBuckets = this.servesPublicRoutes
+ ? parseReplayBlockRateLimits(
+ config.RECORDING_API_BLOCK_RATE_LIMITS ||
+ getDefaultRecordingApiBlocksConfig().RECORDING_API_BLOCK_RATE_LIMITS
+ )
+ : null
+ }
+
+ private get servesInternalRoutes(): boolean {
+ return this.role !== 'public'
+ }
+
+ private get servesPublicRoutes(): boolean {
+ return this.role !== 'internal'
+ }
public get service(): PluginServerService {
return {
@@ -59,9 +103,10 @@ export class RecordingApi {
}
}
- async start(recordingService?: RecordingService): Promise<void> {
+ async start(recordingService?: RecordingService, blockRateLimiter?: KeyedRateLimiterService): Promise<void> {
if (recordingService) {
this.recordingService = recordingService
+ this.blockRateLimiter = blockRateLimiter ?? null
logger.info('[RecordingApi] Started with injected RecordingService')
return
}
@@ -98,28 +143,61 @@ export class RecordingApi {
this.s3Client = new S3Client(s3Config)
- this.redisPool = createRedisPoolFromConfig({
+ // Bounded so a stalled Redis fails fast rather than holding requests the listing has stopped waiting for.
+ const redisConfig = (name: string) => ({
connection: {
url: this.config.SESSION_RECORDING_API_REDIS_HOST,
- options: { port: this.config.SESSION_RECORDING_API_REDIS_PORT },
- name: 'recording-api',
+ options: {
+ port: this.config.SESSION_RECORDING_API_REDIS_PORT,
+ commandTimeout: this.config.RECORDING_API_REDIS_TIMEOUT_MS,
+ },
+ name,
},
poolMinSize: this.config.REDIS_POOL_MIN_SIZE,
poolMaxSize: this.config.REDIS_POOL_MAX_SIZE,
+ acquireTimeoutMillis: this.config.RECORDING_API_REDIS_TIMEOUT_MS,
})
+ this.redisPool = createRedisPoolFromConfig(redisConfig('recording-api'))
- const keyStore: KeyStore = getKeyStore(s3Region, {
- kmsEndpoint: this.config.SESSION_RECORDING_KMS_ENDPOINT,
- dynamoDBEndpoint: this.config.SESSION_RECORDING_DYNAMODB_ENDPOINT,
- })
+ const keyStore: KeyStore = new TimedKeyStore(
+ getKeyStore(s3Region, {
+ kmsEndpoint: this.config.SESSION_RECORDING_KMS_ENDPOINT,
+ dynamoDBEndpoint: this.config.SESSION_RECORDING_DYNAMODB_ENDPOINT,
+ requestTimeoutMs: this.config.RECORDING_API_AWS_TIMEOUT_MS,
+ })
+ )
// In-memory caching is intentionally omitted here — a stale in-memory cache on
// another instance would allow serving a deleted recording's data.
- this.keyStore = new RedisCachedKeyStore(keyStore, this.redisPool)
+ // Never cache a missing key: the browser route can name a session ingestion has not keyed yet, and DynamoDB's
+ // eventually consistent read can briefly miss a new key. Either cached miss would read as cleartext for a day.
+ this.keyStore = new RedisCachedKeyStore(keyStore, this.redisPool, undefined, false)
await this.keyStore.start()
this.decryptor = getBlockDecryptor(this.keyStore)
await this.decryptor.start()
+ if (this.servesPublicRoutes) {
+ this.blockRateLimiter = new KeyedRateLimiterService(
+ { name: 'recording-api-replay-blocks', ttlSeconds: 60 * 60 },
+ createRedisV2PoolFromConfig(redisConfig('recording-api-replay-blocks'))
+ )
+ }
+
+ if (!this.servesInternalRoutes) {
+ this.recordingService = new RecordingService(
+ this.s3Client,
+ this.s3Bucket!,
+ this.s3Prefix!,
+ this.keyStore,
+ this.decryptor
+ )
+ logger.info('[RecordingApi] Started with public routes only')
+ return
+ }
+
+ if (!this.outputs) {
+ throw new Error('recording-api serving internal routes needs its Kafka outputs')
+ }
this.encryptor = getBlockEncryptor(this.keyStore)
await this.encryptor.start()
@@ -183,7 +261,7 @@ export class RecordingApi {
if (!this.decryptor) {
uninitializedComponents.push('decryptor')
}
- if (!this.encryptor) {
+ if (this.servesInternalRoutes && !this.encryptor) {
uninitializedComponents.push('encryptor')
}
@@ -210,6 +288,17 @@ export class RecordingApi {
const encryptPath = '/api/projects/:team_id/recordings/:session_id/encrypt'
+ if (this.servesPublicRoutes) {
+ router.get(
+ '/replay-blocks/:team_id/:session_id/:viewer/:ref',
+ createReplayBlocksAuthMiddleware(this.config.RECORDING_API_BLOCKS_JWT_SECRET),
+ asyncHandler(this.getBlockByRef)
+ )
+ }
+ if (!this.servesInternalRoutes) {
+ return router
+ }
+
const auth = (op: RecordingApiOp) =>
createRecordingApiAuthMiddleware({
jwtSecret: this.config.RECORDING_API_JWT_SECRET,
@@ -323,13 +412,15 @@ export class RecordingApi {
try {
const blocks = await this.recordingService.listBlocks(sessionId, teamId)
- const deletion =
- blocks.length > 0 ? await this.deletionOrNull(this.recordingService, sessionId, teamId) : null
- if (deletion) {
- sendRecordingDeleted(res, deletion)
+ const annotated =
+ blocks.length > 0
+ ? await this.annotateOrPlain(this.recordingService, sessionId, teamId, blocks)
+ : blocks
+ if (!Array.isArray(annotated)) {
+ sendRecordingDeleted(res, annotated)
return
}
- res.json({ blocks })
+ res.json({ blocks: annotated })
} catch (error) {
logger.error('[RecordingApi] Error listing blocks', {
error: serializeError(error),
@@ -341,30 +432,122 @@ export class RecordingApi {
}
}
- // The listing only reports a deletion early, so a key store outage or stall must not stop it: getBlock still returns 410.
- private async deletionOrNull(
+ // The listing only reports a deletion and hands out refs as extras, so a key store outage or stall must not stop
+ // it: the blocks still list without refs, and getBlock still returns 410.
+ private async annotateOrPlain(
service: RecordingService,
sessionId: string,
- teamId: number
- ): Promise<RecordingDeletedResult | null> {
+ teamId: number,
+ blocks: RecordingBlock[]
+ ): Promise<RecordingDeletedResult | ListedBlock[]> {
let timer: NodeJS.Timeout | undefined
const timeout = new Promise<never>((_, reject) => {
- timer = setTimeout(() => reject(new Error('Deletion check timed out')), DELETION_CHECK_TIMEOUT_MS)
+ timer = setTimeout(() => reject(new Error('Key lookup timed out')), KEY_LOOKUP_TIMEOUT_MS)
})
try {
- return await Promise.race([service.getDeletion(sessionId, teamId), timeout])
+ return await Promise.race([service.annotateBlocks(sessionId, teamId, blocks), timeout])
} catch (error) {
- logger.warn('[RecordingApi] Could not check deletion while listing blocks', {
+ logger.warn('[RecordingApi] Could not read the session key while listing blocks', {
error: serializeError(error),
teamId,
sessionId,
})
- return null
+ return blocks
} finally {
clearTimeout(timer)
}
}
+ // A browser viewer has a budget of its own. API callers share their team's budget, as Django's snapshot API limits
+ // do, so a team cannot multiply it with more keys. A missing or unknown tier gets the strictest bucket.
+ private replayBlockBucket(
+ teamId: number,
+ caller: string,
+ tier: unknown
+ ): { id: string; bucket: ReplayBlockBucket } {
+ const known = REPLAY_BLOCK_TIERS.includes(tier as ReplayBlockTier) ? (tier as ReplayBlockTier) : 'free'
+ return {
+ id: known === 'browser' ? `viewer:${caller}` : `team:${teamId}`,
+ bucket: this.replayBlockBuckets![known],
+ }
+ }
+
+ // rateLimitGrouped allows a request that spends the bucket's last token. rateLimitMany refuses it yet still spends
+ // the token, so a steady trickle at the refill rate would never get through. A Redis failure lets the request
+ // through: the limit protects capacity, and the block fetch needs Redis for the key lookup anyway.
+ private async isRateLimited(id: string, bucket: ReplayBlockBucket): Promise<boolean> {
+ try {
+ const [[, decision]] = await this.blockRateLimiter!.rateLimitGrouped([
+ { id, cost: 1, bucketSize: bucket.bucketSize, refillRate: bucket.refillPerSecond },
+ ])
+ return decision.isRateLimited
+ } catch (error) {
+ RecordingApiMetrics.incrementReplayBlockRequest('rate_limit_unavailable')
+ logger.warn('[RecordingApi] Block rate limit unavailable, allowing the request', {
+ error: serializeError(error),
+ })
+ return false
+ }
+ }
+
+ private getBlockByRef = async (req: express.Request, res: express.Response): Promise<void> => {
+ const paramsResult = ReplayBlockParamsSchema.safeParse(req.params)
+ if (!paramsResult.success) {
+ res.status(400).json({ error: paramsResult.error.issues[0].message })
+ return
+ }
+ if (!this.recordingService) {
+ res.status(503).json({ error: 'Service not initialized' })
+ return
+ }
+ const { team_id: teamId, session_id: sessionId, ref } = paramsResult.data
+
+ try {
+ if (this.blockRateLimiter) {
+ const { id, bucket } = this.replayBlockBucket(
+ teamId,
+ res.locals.replayBlocksCaller,
+ res.locals.replayBlocksTier
+ )
+ if (await this.isRateLimited(id, bucket)) {
+ const enforced = this.config.RECORDING_API_BLOCK_RATE_LIMIT_ENFORCED
+ RecordingApiMetrics.incrementReplayBlockRequest(enforced ? 'rate_limited' : 'rate_limited_log_only')
+ if (enforced) {
+ // The wait for one token to refill, not a fixed window.
+ res.set('Retry-After', String(Math.ceil(1 / bucket.refillPerSecond)))
+ res.status(429).json({ error: 'Too many requests' })
+ return
+ }
+ }
+ }
+
+ const result = await this.recordingService.getBlockByRef(teamId, sessionId, ref)
+ if (!result.ok) {
+ RecordingApiMetrics.incrementReplayBlockRequest(result.error === 'deleted' ? 'deleted' : 'not_found')
+ if (result.error === 'deleted') {
+ sendRecordingDeleted(res, result)
+ return
+ }
+ res.status(404).json({ error: 'Block not found' })
+ return
+ }
+ RecordingApiMetrics.incrementReplayBlockRequest('ok')
+ res.set('Content-Type', 'application/octet-stream')
+ res.set('Content-Length', String(Buffer.byteLength(result.data)))
+ res.set('Cache-Control', REPLAY_BLOCK_CACHE_CONTROL)
+ res.send(result.data)
+ } catch (error) {
+ RecordingApiMetrics.incrementReplayBlockRequest('error')
+ logger.error('[RecordingApi] Error serving a block by ref', {
+ error: serializeError(error),
+ teamId,
+ sessionId,
+ })
+ captureException(error)
+ res.status(500).json({ error: 'Failed to fetch block' })
+ }
+ }
+
private encryptBlocks = async (req: express.Request, res: express.Response): Promise<void> => {
const paramsResult = RecordingParamsSchema.safeParse(req.params)
if (!paramsResult.success) {
diff --git a/nodejs/src/session-replay/recording-api/recording-service.test.ts b/nodejs/src/session-replay/recording-api/recording-service.test.ts
index 59fdf0255fa4..f5f4399875a5 100644
--- a/nodejs/src/session-replay/recording-api/recording-service.test.ts
+++ b/nodejs/src/session-replay/recording-api/recording-service.test.ts
@@ -371,21 +371,32 @@ describe('RecordingService', () => {
})
})
- describe('getDeletion', () => {
- const key = (sessionState: 'ciphertext' | 'cleartext' | 'deleted'): SessionKey => ({
- plaintextKey: Buffer.alloc(0),
+ describe('block refs', () => {
+ const sessionKey = (sessionState: 'ciphertext' | 'cleartext' | 'deleted'): SessionKey => ({
+ plaintextKey: Buffer.alloc(32, 7),
encryptedKey: Buffer.alloc(0),
sessionState,
})
+ const block = {
+ key: 'session_recordings/30d/1764634738680-3cca0f5d3c7cc7ee',
+ start_byte: 100,
+ end_byte: 250,
+ start_timestamp: '2024-01-01T00:00:00Z',
+ end_timestamp: '2024-01-01T00:01:00Z',
+ }
- it('reports a shredded key with who deleted it and when', async () => {
+ beforeAll(async () => {
+ await sodium.ready
+ })
+
+ it('reports a shredded key in place of the blocks', async () => {
mockKeyStore.getKey.mockResolvedValue({
- ...key('deleted'),
+ ...sessionKey('deleted'),
deletedAt: 1700000000,
deletedBy: '[email protected]',
})
- expect(await service.getDeletion('sess-1', 1)).toEqual({
+ expect(await service.annotateBlocks('sess-1', 1, [block])).toEqual({
ok: false,
error: 'deleted',
deletedAt: 1700000000,
@@ -393,13 +404,47 @@ describe('RecordingService', () => {
})
})
+ it('gives a pre-encryption recording no refs', async () => {
+ mockKeyStore.getKey.mockResolvedValue(sessionKey('cleartext'))
+
+ expect(await service.annotateBlocks('sess-1', 1, [block])).toEqual([block])
+ expect(mockKeyStore.getKey).toHaveBeenCalledWith('sess-1', 1)
+ })
+
+ it('fetches the byte range a listed ref names', async () => {
+ mockKeyStore.getKey.mockResolvedValue(sessionKey('ciphertext'))
+ const [listed] = (await service.annotateBlocks('sess-1', 1, [block])) as { ref: string }[]
+ mockS3Send.mockResolvedValue({
+ Body: { transformToByteArray: jest.fn().mockResolvedValue(new Uint8Array([1])) },
+ })
+ mockDecryptor.decryptBlockWithKey = jest
+ .fn()
+ .mockReturnValue({ data: Buffer.from('data'), sessionState: 'ciphertext' })
+ mockKeyStore.getKey.mockClear()
+
+ const result = await service.getBlockByRef(1, 'sess-1', listed.ref)
+
+ expect(result).toEqual({ ok: true, data: Buffer.from('data') })
+ expect(mockKeyStore.getKey).toHaveBeenCalledTimes(1)
+ expect(mockKeyStore.getKey).toHaveBeenCalledWith('sess-1', 1)
+ expect(mockDecryptor.decryptBlock).not.toHaveBeenCalled()
+ expect(mockS3Send).toHaveBeenCalledWith(
+ expect.objectContaining({ input: { Bucket: 'test-bucket', Key: block.key, Range: 'bytes=100-250' } })
+ )
+ })
+
it.each([
- ['an encrypted', 'ciphertext' as const],
- ['a pre-encryption', 'cleartext' as const],
- ])('reports nothing for %s recording', async (_, sessionState) => {
- mockKeyStore.getKey.mockResolvedValue(key(sessionState))
+ ['a deleted session', 'deleted', 'deleted'],
+ ['a pre-encryption session', 'cleartext', 'not_found'],
+ ] as const)('serves no block by ref for %s', async (_, sessionState, expectedError) => {
+ mockKeyStore.getKey.mockResolvedValue(sessionKey('ciphertext'))
+ const [listed] = (await service.annotateBlocks('sess-1', 1, [block])) as { ref: string }[]
+ mockKeyStore.getKey.mockResolvedValue(sessionKey(sessionState))
+
+ const result = await service.getBlockByRef(1, 'sess-1', listed.ref)
- expect(await service.getDeletion('sess-1', 1)).toBeNull()
+ expect(result).toMatchObject({ ok: false, error: expectedError })
+ expect(mockS3Send).not.toHaveBeenCalled()
})
})
diff --git a/nodejs/src/session-replay/recording-api/recording-service.ts b/nodejs/src/session-replay/recording-api/recording-service.ts
index e8f4422eff14..40937adc418a 100644
--- a/nodejs/src/session-replay/recording-api/recording-service.ts
+++ b/nodejs/src/session-replay/recording-api/recording-service.ts
@@ -10,6 +10,7 @@ import { SessionFeatureStore } from '~/ingestion/pipelines/sessionreplay/shared/
import { createDeletionBlockMetadata } from '~/ingestion/pipelines/sessionreplay/shared/metadata/session-block-metadata'
import { SessionMetadataStore } from '~/ingestion/pipelines/sessionreplay/shared/metadata/session-metadata-store'
+import { blockRefEncoder, decodeBlockRef } from './block-refs'
import { ClickHouseCredential } from './clickhouse-credential'
import { RecordingApiMetrics } from './metrics'
import {
@@ -28,12 +29,16 @@ export interface GetBlockParams {
startByte: number
endByte: number
decompress?: boolean
+ // A caller that already read the session key passes it, so the block is decrypted without a second lookup.
+ sessionKey?: SessionKey
}
export type RecordingDeletedResult = { ok: false; error: 'deleted'; deletedAt?: number; deletedBy: string }
export type GetBlockResult = { ok: true; data: Buffer } | { ok: false; error: 'not_found' } | RecordingDeletedResult
+export type ListedBlock = RecordingBlock & { ref?: string }
+
export type EncryptBlocksResult =
| { ok: true; blocks: Buffer[] }
| { ok: false; error: 'recording_exists' }
@@ -155,11 +160,9 @@ export class RecordingService {
bytesReceived: bodyContents.length,
})
- const { data, sessionState } = await this.decryptor.decryptBlock(
- sessionId,
- teamId,
- Buffer.from(bodyContents)
- )
+ const { data, sessionState } = params.sessionKey
+ ? this.decryptor.decryptBlockWithKey(sessionId, teamId, Buffer.from(bodyContents), params.sessionKey)
+ : await this.decryptor.decryptBlock(sessionId, teamId, Buffer.from(bodyContents))
logger.debug('[RecordingService] Decrypted block', {
sessionId,
@@ -253,11 +256,43 @@ export class RecordingService {
return rows.length > 0
}
- // The key store records a deletion at once, while the is_deleted marker reaches ClickHouse later through Kafka.
- // Call it only for a session that has blocks: the key store caches a missing key as cleartext for a day, and a
- // lookup before ingestion creates the key would make getBlock skip decryption for that whole day.
- async getDeletion(sessionId: string, teamId: number): Promise<RecordingDeletedResult | null> {
- return deletionOf(await this.keyStore.getKey(sessionId, teamId))
+ // Reports a deletion the key store recorded before ClickHouse learns of it through Kafka, and names each block of an
+ // encrypted session with a ref the browser can fetch and cache. Cleartext sessions have no key to derive refs from.
+ // Call it only for a session that has blocks; a session without any has no key worth reading yet.
+ async annotateBlocks(
+ sessionId: string,
+ teamId: number,
+ blocks: RecordingBlock[]
+ ): Promise<RecordingDeletedResult | ListedBlock[]> {
+ const sessionKey = await this.keyStore.getKey(sessionId, teamId)
+ const deletion = deletionOf(sessionKey)
+ if (deletion) {
+ return deletion
+ }
+ if (sessionKey.sessionState !== 'ciphertext') {
+ return blocks
+ }
+ const encodeRef = blockRefEncoder(sessionKey.plaintextKey)
+ return blocks.map((block) => ({
+ ...block,
+ ref: encodeRef({ teamId, sessionId, key: block.key, startByte: block.start_byte, endByte: block.end_byte }),
+ }))
+ }
+
+ async getBlockByRef(teamId: number, sessionId: string, ref: string): Promise<GetBlockResult> {
+ const sessionKey = await this.keyStore.getKey(sessionId, teamId)
+ const deletion = deletionOf(sessionKey)
+ if (deletion) {
+ return deletion
+ }
+ const location =
+ sessionKey.sessionState === 'ciphertext'
+ ? decodeBlockRef(sessionKey.plaintextKey, ref, teamId, sessionId)
+ : null
+ if (!location || !this.validateS3Key(location.key)) {
+ return { ok: false, error: 'not_found' }
+ }
+ return this.getBlock({ sessionId, teamId, ...location, sessionKey })
}
async listBlocks(sessionId: string, teamId: number): Promise<RecordingBlock[]> {
diff --git a/nodejs/src/session-replay/recording-api/schemas.ts b/nodejs/src/session-replay/recording-api/schemas.ts
index c26a959ee458..0f8c8a3e14d4 100644
--- a/nodejs/src/session-replay/recording-api/schemas.ts
+++ b/nodejs/src/session-replay/recording-api/schemas.ts
@@ -24,6 +24,12 @@ export const RecordingParamsSchema = z.object({
session_id: z.string({ error: 'Missing session_id parameter' }).min(1, 'Invalid session_id parameter'),
})
+// Path params for the browser block route. A ref is short, so the length cap only rejects junk early.
+export const ReplayBlockParamsSchema = RecordingParamsSchema.extend({
+ viewer: z.string().regex(/^[A-Za-z0-9_-]{1,64}$/, 'Invalid viewer'),
+ ref: z.string().regex(/^v1\.[A-Za-z0-9_-]{1,1024}$/, 'Invalid block ref'),
+})
+
// Path params with team_id only (no session_id)
export const TeamParamsSchema = z.object({
team_id: positiveIntString('team_id'),
diff --git a/nodejs/src/session-replay/recording-api/timed-keystore.ts b/nodejs/src/session-replay/recording-api/timed-keystore.ts
new file mode 100644
index 000000000000..6a306e5ba97c
--- /dev/null
+++ b/nodejs/src/session-replay/recording-api/timed-keystore.ts
@@ -0,0 +1,40 @@
+import { DeleteKeyResult, KeyStore, SessionKey } from '~/ingestion/pipelines/sessionreplay/shared/types'
+
+import { RecordingApiMetrics } from './metrics'
+
+// Times each call that misses the Redis cache, so the AWS deadline can follow the measured tail.
+export class TimedKeyStore implements KeyStore {
+ constructor(private readonly delegate: KeyStore) {}
+
+ start(): Promise<void> {
+ return this.delegate.start()
+ }
+
+ stop(): void {
+ this.delegate.stop()
+ }
+
+ generateKey(sessionId: string, teamId: number, retentionDays: number): Promise<SessionKey> {
+ return this.timed('generate', () => this.delegate.generateKey(sessionId, teamId, retentionDays))
+ }
+
+ getKey(sessionId: string, teamId: number): Promise<SessionKey> {
+ return this.timed('get', () => this.delegate.getKey(sessionId, teamId))
+ }
+
+ deleteKey(sessionId: string, teamId: number, deletedBy: string): Promise<DeleteKeyResult> {
+ return this.timed('delete', () => this.delegate.deleteKey(sessionId, teamId, deletedBy))
+ }
+
+ private async timed<T>(operation: 'generate' | 'get' | 'delete', call: () => Promise<T>): Promise<T> {
+ const start = performance.now()
+ let result: 'ok' | 'error' = 'error'
+ try {
+ const value = await call()
+ result = 'ok'
+ return value
+ } finally {
+ RecordingApiMetrics.observeKeyStore(operation, result, (performance.now() - start) / 1000)
+ }
+ }
+}
diff --git a/nodejs/src/session-replay/recording-api/types.test.ts b/nodejs/src/session-replay/recording-api/types.test.ts
new file mode 100644
index 000000000000..62a698cfbc72
--- /dev/null
+++ b/nodejs/src/session-replay/recording-api/types.test.ts
@@ -0,0 +1,18 @@
+import { getDefaultRecordingApiBlocksConfig, parseReplayBlockRateLimits } from './types'
+
+describe('parseReplayBlockRateLimits', () => {
+ it('reads the default limits for every tier', () => {
+ const limits = parseReplayBlockRateLimits(getDefaultRecordingApiBlocksConfig().RECORDING_API_BLOCK_RATE_LIMITS)
+
+ expect(limits.free).toEqual({ bucketSize: 240, refillPerSecond: 1200 / 3600 })
+ expect(Object.keys(limits).sort()).toEqual(['browser', 'enterprise', 'free', 'paid'])
+ })
+
+ it.each([
+ ['a missing tier', '{"browser":[1,1],"free":[1,1],"paid":[1,1]}'],
+ ['a non-positive size', '{"browser":[0,1],"free":[1,1],"paid":[1,1],"enterprise":[1,1]}'],
+ ['a malformed entry', '{"browser":[1],"free":[1,1],"paid":[1,1],"enterprise":[1,1]}'],
+ ])('refuses %s', (_, raw) => {
+ expect(() => parseReplayBlockRateLimits(raw)).toThrow(/RECORDING_API_BLOCK_RATE_LIMITS/)
+ })
+})
diff --git a/nodejs/src/session-replay/recording-api/types.ts b/nodejs/src/session-replay/recording-api/types.ts
index 9160eff20285..39b91dfa91e8 100644
--- a/nodejs/src/session-replay/recording-api/types.ts
+++ b/nodejs/src/session-replay/recording-api/types.ts
@@ -10,6 +10,7 @@ import {
KAFKA_CLICKHOUSE_SESSION_REPLAY_FEATURES,
} from '~/common/config/kafka-topics'
import { isProdEnv } from '~/common/utils/env-utils'
+import { parseJSON } from '~/common/utils/json-parse'
import { SessionRecordingApiConfig, SessionRecordingConfig } from '~/ingestion/pipelines/sessionreplay/config'
import {
INGESTION_SESSIONREPLAY_PRODUCER,
@@ -57,6 +58,84 @@ export function getDefaultRecordingApiAuthConfig(): RecordingApiAuthConfig {
}
}
+/**
+ * Which routes a recording-api process serves. `internal` serves the service-to-service routes that
+ * Django and Temporal call. `public` serves only the browser block route, so pods reachable from the
+ * internet run none of the internal routes. `all` serves both, for local development.
+ */
+export type RecordingApiRole = 'internal' | 'public' | 'all'
+
+/**
+ * The browser block route's token buckets, sized in blocks since one request fetches one block. The token's `tier`
+ * picks the bucket: `browser` for the player, and `free`/`paid`/`enterprise` for API callers, which mirror Django's
+ * per-team snapshot API limits at 20 blocks per request. Enforced by default; false serves over-limit requests and
+ * only counts them.
+ */
+export const REPLAY_BLOCK_TIERS = ['browser', 'free', 'paid', 'enterprise'] as const
+export type ReplayBlockTier = (typeof REPLAY_BLOCK_TIERS)[number]
+export type ReplayBlockBucket = { bucketSize: number; refillPerSecond: number }
+
+export type RecordingApiBlocksConfig = {
+ RECORDING_API_ROLE: string
+ // Signs only browser block tokens. Kept apart from RECORDING_API_JWT_SECRET, so the internet-facing pods, which
+ // hold only this one, cannot sign a service token for an internal route.
+ RECORDING_API_BLOCKS_JWT_SECRET: string
+ RECORDING_API_BLOCK_RATE_LIMIT_ENFORCED: boolean
+ // JSON object of tier -> [bucket size, refill per second].
+ RECORDING_API_BLOCK_RATE_LIMITS: string
+ // Deadline for one KMS or DynamoDB attempt. The SDK makes two attempts.
+ RECORDING_API_AWS_TIMEOUT_MS: number
+ // Deadline for a Redis command and for checking a connection out of the pool.
+ RECORDING_API_REDIS_TIMEOUT_MS: number
+}
+
+export function getDefaultRecordingApiBlocksConfig(): RecordingApiBlocksConfig {
+ return {
+ RECORDING_API_ROLE: 'internal',
+ // Dev/test default matches the Django minter so local end-to-end calls validate.
+ RECORDING_API_BLOCKS_JWT_SECRET: isProdEnv() ? '' : 'dev-recording-api-blocks-jwt-secret',
+ RECORDING_API_BLOCK_RATE_LIMIT_ENFORCED: true,
+ RECORDING_API_BLOCK_RATE_LIMITS: JSON.stringify({
+ browser: [2000, 8],
+ free: [240, 1200 / 3600],
+ paid: [1200, 6000 / 3600],
+ enterprise: [2000, 8000 / 3600],
+ }),
+ // Deletes, which are mostly KMS and DynamoDB calls, almost never take over 2.5s in total.
+ RECORDING_API_AWS_TIMEOUT_MS: 5000,
+ RECORDING_API_REDIS_TIMEOUT_MS: 2000,
+ }
+}
+
+export function parseReplayBlockRateLimits(raw: string): Record<ReplayBlockTier, ReplayBlockBucket> {
+ let parsed: Record<string, unknown>
+ try {
+ parsed = parseJSON(raw) as Record<string, unknown>
+ } catch {
+ throw new Error('Invalid RECORDING_API_BLOCK_RATE_LIMITS: not valid JSON')
+ }
+ const buckets = {} as Record<ReplayBlockTier, ReplayBlockBucket>
+ for (const tier of REPLAY_BLOCK_TIERS) {
+ const value = parsed?.[tier]
+ if (
+ !Array.isArray(value) ||
+ value.length !== 2 ||
+ !value.every((n) => typeof n === 'number' && Number.isFinite(n) && n > 0)
+ ) {
+ throw new Error(`Invalid RECORDING_API_BLOCK_RATE_LIMITS: '${tier}' needs [bucket size, refill per second]`)
+ }
+ buckets[tier] = { bucketSize: value[0], refillPerSecond: value[1] }
+ }
+ return buckets
+}
+
+export function parseRecordingApiRole(role: string): RecordingApiRole {
+ if (role === 'internal' || role === 'public' || role === 'all') {
+ return role
+ }
+ throw new Error(`Invalid RECORDING_API_ROLE '${role}': expected internal, public or all`)
+}
+
/**
* Configuration for the Recording API.
* Postgres is passed as an explicit constructor param, not included here.
@@ -87,7 +166,8 @@ export type RecordingApiConfig = Pick<
| 'SESSION_RECORDING_V2_S3_BUCKET'
| 'SESSION_RECORDING_V2_S3_PREFIX'
> &
- RecordingApiAuthConfig
+ RecordingApiAuthConfig &
+ RecordingApiBlocksConfig
/**
* Recording API outputs — topic and producer routing per output. All keys