Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

27.5 KB
Raw Blame
import { GetObjectCommand, NoSuchKey, S3Client } from '@aws-sdk/client-s3'
import { ClickHouseClient } from '@clickhouse/client'
import snappy from 'snappy'

import { PostgresRouter, PostgresUse } from '~/common/utils/db/postgres'
import { isCloud } from '~/common/utils/env-utils'
import { logger, serializeError } from '~/common/utils/logger'
import { ValidRetentionPeriods } from '~/ingestion/pipelines/sessionreplay/shared/constants'
import { SessionFeatureStore } from '~/ingestion/pipelines/sessionreplay/shared/features/session-feature-store'
import { createDeletionBlockMetadata } from '~/ingestion/pipelines/sessionreplay/shared/metadata/session-block-metadata'
import { SessionMetadataStore } from '~/ingestion/pipelines/sessionreplay/shared/metadata/session-metadata-store'

import { blockRefEncoder, decodeBlockRef } from './block-refs'
import { ClickHouseCredential } from './clickhouse-credential'
import { RecordingApiMetrics } from './metrics'
import {
    KeyStore,
    RecordingBlock,
    RecordingDecryptor,
    RecordingEncryptor,
    SessionKey,
    SessionKeyDeletedError,
} from './types'

export interface GetBlockParams {
    sessionId: string
    teamId: number
    key: string
    startByte: number
    endByte: number
    decompress?: boolean
    // A caller that already read the session key passes it, so the block is decrypted without a second lookup.
    sessionKey?: SessionKey
}

export type RecordingDeletedResult = { ok: false; error: 'deleted'; deletedAt?: number; deletedBy: string }

export type GetBlockResult = { ok: true; data: Buffer } | { ok: false; error: 'not_found' } | RecordingDeletedResult

export type ListedBlock = RecordingBlock & { ref?: string }

export type EncryptBlocksResult =
    | { ok: true; blocks: Buffer[] }
    | { ok: false; error: 'recording_exists' }
    | { ok: false; error: 'key_unavailable' }
    | RecordingDeletedResult

export type DeleteRecordingResult =
    | { sessionId: string; ok: true; status: 'deleted'; deletedAt: number; deletedBy: string }
    | { sessionId: string; ok: true; status: 'already_deleted'; deletedAt: number; deletedBy: string }
    | { sessionId: string; ok: false; status: 'delete_failed' }

interface BlockListingRow {
    start_time: string
    block_first_timestamps: string[]
    block_last_timestamps: string[]
    block_urls: string[]
}

function deletionOf(sessionKey: SessionKey): RecordingDeletedResult | null {
    if (sessionKey.sessionState !== 'deleted') {
        return null
    }
    return { ok: false, error: 'deleted', deletedAt: sessionKey.deletedAt, deletedBy: sessionKey.deletedBy ?? '' }
}

export class RecordingService {
    constructor(
        private s3Client: S3Client,
        private s3Bucket: string,
        private s3Prefix: string,
        private keyStore: KeyStore,
        private decryptor: RecordingDecryptor,
        private metadataStore?: SessionMetadataStore,
        private featureStore?: SessionFeatureStore,
        private postgres?: PostgresRouter,
        private clickhouse?: { client: ClickHouseClient; credential: ClickHouseCredential },
        private encryptor?: RecordingEncryptor
    ) {}

    // Lets a caller write a copied recording without its own access to KMS or the key table.
    async encryptBlocks(
        sessionId: string,
        teamId: number,
        blocks: Buffer[],
        retentionDays: number
    ): Promise<EncryptBlocksResult> {
        const encryptor = this.encryptor
        if (!encryptor) {
            throw new Error('Encryptor not initialized')
        }
        // Encrypting only for sessions with no stored recording keeps this route from touching ingested data,
        // and stops a new key from breaking the plaintext blocks of a pre-encryption recording.
        if (await this.recordingExists(sessionId, teamId)) {
            return { ok: false, error: 'recording_exists' }
        }
        let sessionKey = await this.keyStore.getKey(sessionId, teamId)
        if (sessionKey.sessionState === 'cleartext') {
            sessionKey = await this.keyStore.generateKey(sessionId, teamId, retentionDays)
        }
        // A lost race on the conditional key write can read back no key, and encrypting with it returns plaintext.
        if (sessionKey.sessionState === 'cleartext' && isCloud()) {
            return { ok: false, error: 'key_unavailable' }
        }
        const deletion = deletionOf(sessionKey)
        if (deletion) {
            return deletion
        }
        return {
            ok: true,
            blocks: blocks.map((block) => encryptor.encryptBlockWithKey(sessionId, teamId, block, sessionKey).data),
        }
    }

    validateS3Key(key: string): boolean {
        const pattern = `^${this.s3Prefix}/(${ValidRetentionPeriods.join('|')})/\\d+-[0-9a-f]{16}$`
        return new RegExp(pattern).test(key)
    }

    formatS3KeyError(): string {
        return `Invalid key format: must match ${this.s3Prefix}/{${ValidRetentionPeriods.join(',')}}/{timestamp}-{hex}`
    }

    async getBlock(params: GetBlockParams): Promise<GetBlockResult> {
        const { sessionId, teamId, key, startByte, endByte, decompress } = params
        const startTime = performance.now()

        logger.debug('[RecordingService] getBlock request', {
            teamId,
            sessionId,
            key,
            start: startByte,
            end: endByte,
        })

        try {
            const command = new GetObjectCommand({
                Bucket: this.s3Bucket,
                Key: key,
                Range: `bytes=${startByte}-${endByte}`,
            })

            logger.debug('[RecordingService] Fetching from S3', {
                bucket: this.s3Bucket,
                key,
                range: `bytes=${startByte}-${endByte}`,
            })

            const response = await this.s3Client.send(command)

            if (!response.Body) {
                logger.debug('[RecordingService] S3 returned no body', { key })
                RecordingApiMetrics.observeGetBlock('not_found', (performance.now() - startTime) / 1000, 'unknown')
                return { ok: false, error: 'not_found' }
            }

            const bodyContents = await response.Body.transformToByteArray()
            logger.debug('[RecordingService] S3 returned data', {
                key,
                bytesReceived: bodyContents.length,
            })

            const { data, sessionState } = params.sessionKey
                ? this.decryptor.decryptBlockWithKey(sessionId, teamId, Buffer.from(bodyContents), params.sessionKey)
                : await this.decryptor.decryptBlock(sessionId, teamId, Buffer.from(bodyContents))

            logger.debug('[RecordingService] Decrypted block', {
                sessionId,
                teamId,
                inputSize: bodyContents.length,
                outputSize: data.length,
                sessionState,
            })

            let result = data
            if (decompress) {
                try {
                    result = (await snappy.uncompress(data, { asBuffer: true })) as Buffer
                } catch (decompressError) {
                    logger.error('[RecordingService] Failed to decompress block', {
                        sessionId,
                        teamId,
                        key,
                        dataSize: data.length,
                        error: serializeError(decompressError),
                    })
                    throw decompressError
                }
            }

            RecordingApiMetrics.observeGetBlock(
                'success',
                (performance.now() - startTime) / 1000,
                sessionState,
                !!decompress
            )
            return { ok: true, data: result }
        } catch (error) {
            if (error instanceof NoSuchKey) {
                logger.warn('[RecordingService] S3 object not found (NoSuchKey)', {
                    key,
                    teamId,
                    sessionId,
                })
                RecordingApiMetrics.observeGetBlock('not_found', (performance.now() - startTime) / 1000, 'unknown')
                return { ok: false, error: 'not_found' }
            }

            if (error instanceof SessionKeyDeletedError) {
                logger.info('[RecordingService] Session key has been deleted', {
                    teamId,
                    sessionId,
                    deleted_at: error.deletedAt,
                    deleted_by: error.deletedBy,
                })
                RecordingApiMetrics.observeGetBlock('deleted', (performance.now() - startTime) / 1000, 'unknown')
                return { ok: false, error: 'deleted', deletedAt: error.deletedAt, deletedBy: error.deletedBy }
            }

            RecordingApiMetrics.observeGetBlock('error', (performance.now() - startTime) / 1000, 'unknown')
            throw error
        }
    }

    private async recordingExists(sessionId: string, teamId: number): Promise<boolean> {
        if (!this.clickhouse) {
            throw new Error('ClickHouse client not initialized')
        }
        const { client, credential } = this.clickhouse
        // No retention or is_deleted filter: an expired or deleted recording still has blocks a new key would break.
        const result = await client.query({
            query: `/* team_id:${teamId} query_type:recording_api_recording_exists */ SELECT 1
                FROM session_replay_events
                PREWHERE team_id = {team_id:Int32} AND session_id = {session_id:String}
                LIMIT 1
            `,
            query_params: { team_id: teamId, session_id: sessionId },
            format: 'JSONEachRow',
            auth: await credential.auth(),
            clickhouse_settings: {
                log_comment: JSON.stringify({
                    team_id: teamId,
                    product: 'replay',
                    kind: 'node',
                    query_type: 'recording_api_recording_exists',
                    service_name: 'recording-api',
                    container_hostname: process.env.HOSTNAME || 'unknown',
                }),
                max_execution_time: 30,
                // The lookup has no date bound, so cap it to keep it from starving playback listings on the shared pool.
                max_threads: 4,
                max_bytes_to_read: '10000000000',
            },
        })
        const rows = await result.json()
        return rows.length > 0
    }

    // Reports a deletion the key store recorded before ClickHouse learns of it through Kafka, and names each block of an
    // encrypted session with a ref the browser can fetch and cache. Cleartext sessions have no key to derive refs from.
    // Call it only for a session that has blocks; a session without any has no key worth reading yet.
    async annotateBlocks(
        sessionId: string,
        teamId: number,
        blocks: RecordingBlock[]
    ): Promise<RecordingDeletedResult | ListedBlock[]> {
        const sessionKey = await this.keyStore.getKey(sessionId, teamId)
        const deletion = deletionOf(sessionKey)
        if (deletion) {
            return deletion
        }
        if (sessionKey.sessionState !== 'ciphertext') {
            return blocks
        }
        const encodeRef = blockRefEncoder(sessionKey.plaintextKey)
        return blocks.map((block) => ({
            ...block,
            ref: encodeRef({ teamId, sessionId, key: block.key, startByte: block.start_byte, endByte: block.end_byte }),
        }))
    }

    async getBlockByRef(teamId: number, sessionId: string, ref: string): Promise<GetBlockResult> {
        const sessionKey = await this.keyStore.getKey(sessionId, teamId)
        const deletion = deletionOf(sessionKey)
        if (deletion) {
            return deletion
        }
        const location =
            sessionKey.sessionState === 'ciphertext'
                ? decodeBlockRef(sessionKey.plaintextKey, ref, teamId, sessionId)
                : null
        if (!location || !this.validateS3Key(location.key)) {
            return { ok: false, error: 'not_found' }
        }
        return this.getBlock({ sessionId, teamId, ...location, sessionKey })
    }

    async listBlocks(sessionId: string, teamId: number): Promise<RecordingBlock[]> {
        if (!this.clickhouse) {
            throw new Error('ClickHouse client not initialized')
        }
        const { client, credential } = this.clickhouse

        const startTime = performance.now()

        logger.debug('[RecordingService] listBlocks request', { teamId, sessionId })

        try {
            const result = await client.query({
                query: `/* team_id:${teamId} query_type:recording_api_list_blocks */ SELECT
                        min(min_first_timestamp) as start_time,
                        groupArrayArray(block_first_timestamps) as block_first_timestamps,
                        groupArrayArray(block_last_timestamps) as block_last_timestamps,
                        groupArrayArray(block_urls) as block_urls,
                        max(retention_period_days) as retention_period_days,
                        dateTrunc('DAY', start_time) + toIntervalDay(coalesce(retention_period_days, 30)) as expiry_time
                    FROM session_replay_events
                    PREWHERE
                        team_id = {team_id:Int32}
                        AND session_id = {session_id:String}
                        AND min_first_timestamp <= now()
                    GROUP BY session_id
                    HAVING
                        expiry_time >= now()
                        AND max(is_deleted) = 0
                `,
                query_params: {
                    team_id: teamId,
                    session_id: sessionId,
                },
                format: 'JSONEachRow',
                auth: await credential.auth(),
                clickhouse_settings: {
                    date_time_output_format: 'iso',
                    log_comment: JSON.stringify({
                        team_id: teamId,
                        product: 'replay',
                        kind: 'node',
                        query_type: 'recording_api_list_blocks',
                        service_name: 'recording-api',
                        container_hostname: process.env.HOSTNAME || 'unknown',
                    }),
                    max_execution_time: 30,
                    max_threads: 45,
                    max_bytes_to_read: '10000000000',
                },
            })

            const rows = await result.json<BlockListingRow>()
            if (rows.length === 0) {
                RecordingApiMetrics.observeListBlocks('empty', (performance.now() - startTime) / 1000)
                return []
            }

            const blocks = RecordingService.buildBlockList(sessionId, teamId, rows[0])

            RecordingApiMetrics.observeListBlocks(
                blocks.length > 0 ? 'success' : 'empty',
                (performance.now() - startTime) / 1000
            )
            logger.debug('[RecordingService] listBlocks complete', {
                teamId,
                sessionId,
                blockCount: blocks.length,
            })

            return blocks
        } catch (error) {
            RecordingApiMetrics.observeListBlocks('error', (performance.now() - startTime) / 1000)
            throw error
        }
    }

    static buildBlockList(sessionId: string, teamId: number, row: BlockListingRow): RecordingBlock[] {
        const { start_time, block_first_timestamps, block_last_timestamps, block_urls } = row

        if (
            !block_first_timestamps?.length ||
            !block_last_timestamps?.length ||
            !block_urls?.length ||
            block_first_timestamps.length !== block_last_timestamps.length ||
            block_first_timestamps.length !== block_urls.length
        ) {
            logger.error('[RecordingService] Block listing arrays length mismatch', {
                sessionId,
                teamId,
                firstTimestampsLength: block_first_timestamps?.length ?? 0,
                lastTimestampsLength: block_last_timestamps?.length ?? 0,
                urlsLength: block_urls?.length ?? 0,
            })
            return []
        }

        const blocks: { sortKey: string; block: RecordingBlock }[] = []
        for (let i = 0; i < block_urls.length; i++) {
            const parsed = RecordingService.parseBlockUrl(block_urls[i])
            if (parsed) {
                blocks.push({
                    sortKey: block_first_timestamps[i],
                    block: {
                        ...parsed,
                        start_timestamp: block_first_timestamps[i],
                        end_timestamp: block_last_timestamps[i],
                    },
                })
            }
        }

        blocks.sort((a, b) => a.sortKey.localeCompare(b.sortKey))

        if (blocks.length === 0 || blocks[0].sortKey !== start_time) {
            return []
        }

        return blocks.map((b) => b.block)
    }

    static parseBlockUrl(blockUrl: string): { key: string; start_byte: number; end_byte: number } | null {
        try {
            const url = new URL(blockUrl)
            const key = url.pathname.replace(/^\//, '')
            const match = url.search.match(/range=bytes=(\d+)-(\d+)/)
            if (!match) {
                logger.warn('[RecordingService] Invalid block URL range', { blockUrl })
                return null
            }
            return { key, start_byte: parseInt(match[1]), end_byte: parseInt(match[2]) }
        } catch {
            logger.warn('[RecordingService] Failed to parse block URL', { blockUrl })
            return null
        }
    }

    async deleteRecordings(sessionIds: string[], teamId: number, deletedBy: string): Promise<DeleteRecordingResult[]> {
        const startTime = performance.now()
        logger.debug('[RecordingService] deleteRecordings request', { teamId, count: sessionIds.length, deletedBy })

        const results = await Promise.all(
            sessionIds.map((sessionId) => this.deleteRecording(sessionId, teamId, deletedBy))
        )

        const deleted = results.filter((r) => r.ok && r.status === 'deleted')
        const alreadyDeleted = results.filter((r) => r.ok && r.status === 'already_deleted')
        const failed = results.filter((r) => !r.ok)

        if (deleted.length > 0) {
            RecordingApiMetrics.incrementRecordingsDeleted('deleted', deleted.length)
        }
        if (alreadyDeleted.length > 0) {
            RecordingApiMetrics.incrementRecordingsDeleted('already_deleted', alreadyDeleted.length)
        }
        if (failed.length > 0) {
            RecordingApiMetrics.incrementRecordingsDeleted('failed', failed.length)
        }

        await this.propagateDeletion(
            deleted.map((r) => r.sessionId),
            // A shred that already happened still gets its derived data cleaned up. Cleanup is
            // best-effort, so without this a failed pass is never retried: the next delete of the
            // same recording reports already_deleted and skips it.
            [...deleted, ...alreadyDeleted].map((r) => r.sessionId),
            teamId,
            deletedBy
        )

        const outcome = failed.length === 0 ? 'success' : failed.length === sessionIds.length ? 'failure' : 'partial'
        RecordingApiMetrics.observeDeleteRecordings(outcome, (performance.now() - startTime) / 1000)
        logger.info('[RecordingService] deleteRecordings complete', {
            teamId,
            deleted: deleted.length,
            failed: failed.length,
        })

        return results
    }

    private async deleteRecording(
        sessionId: string,
        teamId: number,
        deletedBy: string
    ): Promise<DeleteRecordingResult> {
        const result = await this.keyStore.deleteKey(sessionId, teamId, deletedBy).catch((): null => null)
        if (result === null) {
            return { sessionId, ok: false, status: 'delete_failed' }
        }
        return { sessionId, ok: true, ...result }
    }

    /**
     * Propagate a successful key shred to Kafka, Postgres, and the activity log.
     * Failures are non-fatal — the key shred already makes the recording unplayable (410).
     * Stale metadata from failed propagation is harmless but won't be automatically cleaned up.
     */
    private async propagateDeletion(
        sessionIds: string[],
        cleanupSessionIds: string[],
        teamId: number,
        deletedBy: string
    ): Promise<void> {
        if (sessionIds.length === 0 && cleanupSessionIds.length === 0) {
            return
        }
        const [kafka, postgres, activity] = await Promise.allSettled([
            this.emitDeletionEvents(sessionIds, teamId),
            this.deletePostgresRecords(cleanupSessionIds, teamId),
            this.logActivity(sessionIds, teamId, deletedBy),
        ])
        if (kafka.status === 'rejected') {
            RecordingApiMetrics.incrementCleanupFailure('kafka')
            logger.error('[RecordingService] Failed to emit deletion events to Kafka', {
                teamId,
                error: serializeError(kafka.reason),
            })
        }
        if (postgres.status === 'rejected') {
            RecordingApiMetrics.incrementCleanupFailure('postgres')
            logger.error('[RecordingService] Failed to delete Postgres records', {
                teamId,
                error: serializeError(postgres.reason),
            })
        }
        if (activity.status === 'rejected') {
            RecordingApiMetrics.incrementCleanupFailure('activity_log')
            logger.error('[RecordingService] Failed to log activity', {
                teamId,
                error: serializeError(activity.reason),
            })
        }
    }

    private async emitDeletionEvents(sessionIds: string[], teamId: number): Promise<void> {
        await Promise.all([
            this.metadataStore?.storeSessionBlocks(
                sessionIds.map((sessionId) => createDeletionBlockMetadata(sessionId, teamId))
            ),
            this.featureStore?.storeDeletionMarkers(sessionIds.map((sessionId) => ({ sessionId, teamId }))),
        ])
    }

    private async deletePostgresRecords(sessionIds: string[], teamId: number): Promise<void> {
        if (sessionIds.length === 0 || !this.postgres) {
            return
        }

        const tables = [
            'ee_single_session_summary',
            'posthog_exportedrecording',
            'posthog_comment',
            'replay_vision_replayobservation',
            'posthog_exportedasset',
        ] as const
        const results = await Promise.allSettled([
            this.postgres.query(
                PostgresUse.COMMON_WRITE,
                `DELETE FROM ee_single_session_summary WHERE team_id = $1 AND session_id = ANY($2)`,
                [teamId, sessionIds],
                'deleteSessionSummaries'
            ),
            this.postgres.query(
                PostgresUse.COMMON_WRITE,
                `DELETE FROM posthog_exportedrecording WHERE team_id = $1 AND session_id = ANY($2)`,
                [teamId, sessionIds],
                'deleteExportedRecordings'
            ),
            this.postgres.query(
                PostgresUse.COMMON_WRITE,
                `DELETE FROM posthog_comment WHERE team_id = $1 AND scope = 'recording' AND item_id = ANY($2)`,
                [teamId, sessionIds],
                'deleteRecordingComments'
            ),
            // One statement: the children's foreign keys are NO ACTION, so separate deletes would have to run child-first.
            // The AsyncDeletion row (deletion_type 5 = Event) queues the observation's $recording_observed event for the ClickHouse sweep, atomically with the delete.
            this.postgres.query(
                PostgresUse.COMMON_WRITE,
                `WITH observations AS (
                     SELECT id FROM replay_vision_replayobservation
                     WHERE team_id = $1 AND session_id = ANY($2)
                 ), queued_events AS (
                     INSERT INTO posthog_asyncdeletion (deletion_type, team_id, key, created_at)
                     SELECT 5, $1, id::text, now() FROM observations
                     ON CONFLICT DO NOTHING
                 ), deleted_labels AS (
                     DELETE FROM replay_vision_replayobservationlabel
                     WHERE observation_id IN (SELECT id FROM observations)
                 ), deleted_views AS (
                     DELETE FROM replay_vision_replayobservationview
                     WHERE observation_id IN (SELECT id FROM observations)
                 ), deleted_matches AS (
                     DELETE FROM replay_vision_visionalertmatch
                     WHERE observation_id IN (SELECT id FROM observations)
                 ), deleted_media AS (
                     DELETE FROM replay_vision_replayobservationmedia
                     WHERE observation_id IN (SELECT id FROM observations)
                 ), deleted_dismissals AS (
                     DELETE FROM replay_vision_replayobservationfeeddismissal
                     WHERE observation_id IN (SELECT id FROM observations)
                 ), deleted_impressions AS (
                     DELETE FROM replay_vision_replayobservationfeedimpression
                     WHERE observation_id IN (SELECT id FROM observations)
                 ), deleted_similarities AS (
                     DELETE FROM replay_vision_replayobservationanchorsimilarity
                     WHERE observation_id IN (SELECT id FROM observations)
                 ), deleted_ranks AS (
                     DELETE FROM replay_vision_replayobservationrank
                     WHERE observation_id IN (SELECT id FROM observations)
                 ), deleted_feed_entries AS (
                     DELETE FROM replay_vision_replayhighlightfeedentry
                     WHERE observation_id IN (SELECT id FROM observations)
                 )
                 DELETE FROM replay_vision_replayobservation
                 WHERE id IN (SELECT id FROM observations)`,
                [teamId, sessionIds],
                'deleteReplayVisionObservations'
            ),
            // Expired, not deleted: the row is the only pointer to the stored object, which the expiry sweep needs.
            this.postgres.query(
                PostgresUse.COMMON_WRITE,
                // `= ANY` over jsonb so the planner can use exportedasset_session; a subquery cannot.
                `UPDATE posthog_exportedasset
                 SET expires_after = now()
                 WHERE team_id = $1
                   AND expires_after > now()
                   AND export_context -> 'session_recording_id' = ANY($2::jsonb[])`,
                [teamId, sessionIds.map((sessionId) => JSON.stringify(sessionId))],
                'expireRecordingExports'
            ),
        ])

        const failures: string[] = []
        for (const [i, result] of results.entries()) {
            if (result.status === 'rejected') {
                failures.push(tables[i])
            }
        }

        if (failures.length > 0) {
            throw new Error(`Failed to delete from: ${failures.join(', ')}`)
        }

        logger.info('[RecordingService] PostgreSQL records deleted', { teamId, sessionCount: sessionIds.length })
    }

    private async logActivity(sessionIds: string[], teamId: number, deletedBy: string): Promise<void> {
        if (sessionIds.length === 0 || !this.postgres) {
            return
        }

        const detail = JSON.stringify({ type: 'recording_shredded', deleted_by: deletedBy })
        await this.postgres.query(
            PostgresUse.COMMON_WRITE,
            `INSERT INTO posthog_activitylog (id, team_id, is_system, activity, item_id, scope, detail, created_at)
             SELECT gen_random_uuid(), $1, true, 'deleted', unnest($2::text[]), 'Replay', $3::jsonb, now()`,
            [teamId, sessionIds, detail],
            'logRecordingDeletion'
        )
    }
}