:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(replay-recorder-v4): store sealed session blocks (#117970)
Nicholas Waltzddc429e81c25
diff --git a/rust/Cargo.lock b/rust/Cargo.lockindex 209ef864d212..14198df61440 100644--- a/rust/Cargo.lock+++ b/rust/Cargo.lock@@ -11052,7 +11052,14 @@ name = "replay-recorder-v4"version = "0.1.0"dependencies = ["arc-swap",+ "async-trait",+ "aws-config",+ "aws-sdk-dynamodb",+ "aws-sdk-kms",+ "aws-sdk-s3","axum 0.7.9",+ "brotlic",+ "chrono","common-alloc","common-database","common-kafka",@@ -11060,10 +11067,12 @@ dependencies = ["envconfig","lifecycle","metrics",+ "rand 0.8.5","rdkafka","replay-core-v4","serde_json","sqlx",+ "thiserror 2.0.18","tokio","tracing","tracing-subscriber",diff --git a/rust/ai-research/replay-recorder-v4/Cargo.toml b/rust/ai-research/replay-recorder-v4/Cargo.tomlindex 0b8bee87bfa4..56d4510be171 100644--- a/rust/ai-research/replay-recorder-v4/Cargo.toml+++ b/rust/ai-research/replay-recorder-v4/Cargo.toml@@ -17,14 +17,24 @@ lifecycle = { path = "../../common/lifecycle" }replay-core-v4 = { path = "../replay-core-v4" }arc-swap = { workspace = true }+async-trait = { workspace = true }+aws-config = { workspace = true }+aws-sdk-dynamodb = { workspace = true }+aws-sdk-kms = { workspace = true }+aws-sdk-s3 = { workspace = true }axum = { workspace = true }+brotlic = "0.8"+chrono = { workspace = true }envconfig = { workspace = true }metrics = { workspace = true }+rand = { workspace = true }rdkafka = { workspace = true }sqlx = { workspace = true }+thiserror = { workspace = true }tokio = { workspace = true }tracing = { workspace = true }tracing-subscriber = { workspace = true }[dev-dependencies]serde_json = { workspace = true }+tokio = { workspace = true, features = ["test-util"] }diff --git a/rust/ai-research/replay-recorder-v4/src/config.rs b/rust/ai-research/replay-recorder-v4/src/config.rsindex 0951a8ed98c2..ade7a7309a23 100644--- a/rust/ai-research/replay-recorder-v4/src/config.rs+++ b/rust/ai-research/replay-recorder-v4/src/config.rs@@ -39,6 +39,20 @@ pub struct Config {#[envconfig(from = "AI_RESEARCH_V4_SESSION_IDLE_SECS", default = "3600")]pub session_idle_secs: u64,+ /// Off, the recorder reads, admits and inspects, and stores nothing: no keys, no objects.+ #[envconfig(from = "AI_RESEARCH_V4_WRITE_ENABLED", default = "false")]+ pub write_enabled: bool,++ /// Resource names come from the deployment, as the v3 mirror's do.+ #[envconfig(from = "AI_RESEARCH_V4_KEY_TABLE", default = "")]+ pub key_table: String,++ #[envconfig(from = "AI_RESEARCH_V4_KMS_KEY_ARN", default = "")]+ pub kms_key_arn: String,++ #[envconfig(from = "AI_RESEARCH_V4_RRWEB_BUCKET", default = "")]+ pub rrweb_bucket: String,+#[envconfig(default = "9104")]pub metrics_port: u16,}diff --git a/rust/ai-research/replay-recorder-v4/src/consumer.rs b/rust/ai-research/replay-recorder-v4/src/consumer.rsindex 9438f0183862..dde0f8fe8935 100644--- a/rust/ai-research/replay-recorder-v4/src/consumer.rs+++ b/rust/ai-research/replay-recorder-v4/src/consumer.rs@@ -12,9 +12,17 @@ use rdkafka::consumer::{use rdkafka::message::{BorrowedMessage, Headers, Message as _};use rdkafka::{ClientConfig, ClientContext, Offset, TopicPartitionList};+use chrono::Utc;+use replay_core_v4::keys::batch::{KeyStore, SessionId};+use replay_core_v4::keys::kms::AwsKms;+use replay_core_v4::keys::rows::write_day;+use replay_core_v4::keys::table::{AwsKeyTable, CachedKeyTable};+use crate::batch::{Batch, Block, Inspectors, SessionKey};use crate::process::{admit, read, sampled};+use crate::store::BlobStore;use crate::teams::TeamCache;+use crate::write::{write_blocks, WriteError};/// Records the partitions each rebalance takes away, so the loop can drop what it buffered for them.pub struct RebalanceContext {@@ -112,6 +120,21 @@ fn header<'m>(message: &'m BorrowedMessage<'_>, name: &str) -> Option<&'m str> {.and_then(|v| std::str::from_utf8(v).ok())}+/// What a flush needs to store blocks. Without it the recorder runs dry.+pub struct Writer {+ pub keys: KeyStore<CachedKeyTable<AwsKeyTable>, AwsKms>,+ pub store: Arc<dyn BlobStore>,+ pub rrweb_bucket: String,+}++#[derive(Debug, thiserror::Error)]+pub enum FlushError {+ #[error(transparent)]+ Keys(#[from] replay_core_v4::keys::batch::KeyError),+ #[error(transparent)]+ Write(#[from] WriteError),+}+pub struct Settings {pub sample_percentage: f64,pub flush_interval: Duration,@@ -123,6 +146,7 @@ pub async fn run(consumer: RecorderConsumer,settings: Settings,teams: Arc<TeamCache>,+ writer: Option<Writer>,handle: Handle,) {let shutdown = handle.shutdown_signal();@@ -155,7 +179,12 @@ pub async fn run(}if batch.should_flush(settings.flush_interval, settings.flush_max_bytes) {let (blocks, offsets) = inspectors.inspect(std::mem::take(&mut batch));- flush(&blocks);+ if let Err(error) = flush(&blocks, writer.as_ref()).await {+ // Nothing of this flush is committed, so a restart reads it again from the last commit.+ tracing::error!(%error, "flush failed");+ handle.signal_failure(format!("flush failed: {error}"));+ break;+ }consumer.commit(&offsets);inspectors.evict_idle(settings.session_idle);}@@ -200,8 +229,8 @@ fn handle_message(}}-/// Reports what a flush inspected. The writers that store blocks come in the next layers.-fn flush(blocks: &[Block]) {+/// Stores a flush's blocks, or only reports them when the recorder runs dry.+async fn flush(blocks: &[Block], writer: Option<&Writer>) -> Result<(), FlushError> {counter!("ai_research_v4_recorder_blocks_total").increment(blocks.len() as u64);for block in blocks {counter!("ai_research_v4_recorder_block_events_total").increment(block.stats.event_count);@@ -212,4 +241,29 @@ fn flush(blocks: &[Block]) {counter!("ai_research_v4_recorder_orphan_mutation_blocks_total").increment(1);}}+ let Some(writer) = writer else { return Ok(()) };+ let now = Utc::now();+ let sessions: Vec<SessionId> = blocks+ .iter()+ .map(|b| SessionId {+ team_id: b.session.team_id,+ session_id: b.session.session_id.clone(),+ })+ .collect();+ // Keys are stored before anything is sealed with them, as in v3.+ let keys = writer+ .keys+ .keys_for_flush(&sessions, &write_day(now))+ .await?;+ counter!("ai_research_v4_recorder_dropped_sessions_total").increment(keys.dropped.len() as u64);+ let written = write_blocks(+ blocks,+ &keys,+ &writer.rrweb_bucket,+ writer.store.as_ref(),+ now,+ )+ .await?;+ counter!("ai_research_v4_recorder_written_blocks_total").increment(written.len() as u64);+ Ok(())}diff --git a/rust/ai-research/replay-recorder-v4/src/lib.rs b/rust/ai-research/replay-recorder-v4/src/lib.rsindex f471229fe802..72357f281585 100644--- a/rust/ai-research/replay-recorder-v4/src/lib.rs+++ b/rust/ai-research/replay-recorder-v4/src/lib.rs@@ -5,4 +5,6 @@ pub mod batch;pub mod config;pub mod consumer;pub mod process;+pub mod store;pub mod teams;+pub mod write;diff --git a/rust/ai-research/replay-recorder-v4/src/main.rs b/rust/ai-research/replay-recorder-v4/src/main.rsindex 0a7a1a40a6eb..6f5e44652486 100644--- a/rust/ai-research/replay-recorder-v4/src/main.rs+++ b/rust/ai-research/replay-recorder-v4/src/main.rs@@ -11,8 +11,12 @@ use tracing_subscriber::layer::SubscriberExt;use tracing_subscriber::util::SubscriberInitExt;use tracing_subscriber::{fmt, EnvFilter};+use replay_core_v4::keys::batch::KeyStore;+use replay_core_v4::keys::kms::{AwsKms, CachedKms, KmsLimits};+use replay_core_v4::keys::table::{AwsKeyTable, CachedKeyTable};use replay_recorder_v4::config::Config;-use replay_recorder_v4::consumer::{run, RecorderConsumer, Settings};+use replay_recorder_v4::consumer::{run, RecorderConsumer, Settings, Writer};+use replay_recorder_v4::store::S3Store;use replay_recorder_v4::teams::TeamCache;common_alloc::used!();@@ -112,7 +116,39 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {flush_max_bytes: config.flush_max_bytes,session_idle: Duration::from_secs(config.session_idle_secs),};- tokio::spawn(run(consumer, settings, teams, consumer_handle));+ let writer = if config.write_enabled {+ let aws = aws_config::defaults(aws_config::BehaviorVersion::latest())+ .load()+ .await;+ // The cache lifetimes are v3's: a session row's lifetime bounds how late a deletion is seen.+ let table = CachedKeyTable::new(+ AwsKeyTable::new(+ aws_sdk_dynamodb::Client::new(&aws),+ config.key_table.clone(),+ ),+ 100_000,+ Duration::from_secs(300),+ Duration::from_secs(3600),+ );+ let kms = CachedKms::new(+ AwsKms::new(+ aws_sdk_kms::Client::new(&aws),+ config.kms_key_arn.clone(),+ KmsLimits::default(),+ ),+ 100_000,+ Duration::from_secs(1800),+ );+ Some(Writer {+ keys: KeyStore::new(table, kms),+ store: Arc::new(S3Store::new(aws_sdk_s3::Client::new(&aws))),+ rrweb_bucket: config.rrweb_bucket.clone(),+ })+ } else {+ tracing::info!("writes are off: the recorder stores nothing");+ None+ };+ tokio::spawn(run(consumer, settings, teams, writer, consumer_handle));monitor.wait().await?;Ok(())diff --git a/rust/ai-research/replay-recorder-v4/src/store.rs b/rust/ai-research/replay-recorder-v4/src/store.rsnew file mode 100644index 000000000000..7c01635373b8--- /dev/null+++ b/rust/ai-research/replay-recorder-v4/src/store.rs@@ -0,0 +1,51 @@+//! Object storage. Tests replace S3 at this boundary.++use async_trait::async_trait;++#[derive(Debug, thiserror::Error)]+#[error("object store write failed: {0}")]+pub struct StoreError(pub String);++#[async_trait]+pub trait BlobStore: Send + Sync {+ /// Writes an object that must not exist yet. The recorder names every object uniquely, so an existing one means+ /// the SDK retried this same write, and the first write stands.+ async fn put_new(&self, bucket: &str, key: &str, body: Vec<u8>) -> Result<(), StoreError>;+}++pub struct S3Store {+ client: aws_sdk_s3::Client,+}++impl S3Store {+ pub fn new(client: aws_sdk_s3::Client) -> Self {+ Self { client }+ }+}++#[async_trait]+impl BlobStore for S3Store {+ async fn put_new(&self, bucket: &str, key: &str, body: Vec<u8>) -> Result<(), StoreError> {+ let result = self+ .client+ .put_object()+ .bucket(bucket)+ .key(key)+ .if_none_match("*")+ .body(body.into())+ .send()+ .await;+ match result {+ Ok(_) => Ok(()),+ // 412: the SDK retried this PUT after a first try that stored the object, so it is there.+ Err(error)+ if error+ .raw_response()+ .is_some_and(|r| r.status().as_u16() == 412) =>+ {+ Ok(())+ }+ Err(error) => Err(StoreError(error.to_string())),+ }+ }+}diff --git a/rust/ai-research/replay-recorder-v4/src/write.rs b/rust/ai-research/replay-recorder-v4/src/write.rsnew file mode 100644index 000000000000..37b97e0b7d76--- /dev/null+++ b/rust/ai-research/replay-recorder-v4/src/write.rs@@ -0,0 +1,257 @@+//! Writes a flush: keys first, then one object of sealed session blocks.++use std::io::Write;++use chrono::{DateTime, Utc};+use rand::Rng;+use replay_core_v4::bytescan::write_json_string;+use replay_core_v4::envelope::{seal_frame, Codec, Context, Version};+use replay_core_v4::keys::batch::{FlushKeys, SessionId};+use replay_core_v4::keys::rows::write_day;++use crate::batch::{Block, SessionKey};+use crate::store::{BlobStore, StoreError};++/// Brotli settings for blocks, the same as the v3 lane, which uses the same C encoder. The lane keeps blocks for months+/// and reads them rarely, so it takes the smaller output for more CPU.+const BLOCK_BROTLI_QUALITY: u8 = 9;+const BLOCK_BROTLI_WINDOW_BITS: u8 = 22;++#[derive(Debug, thiserror::Error)]+pub enum WriteError {+ #[error(transparent)]+ Store(#[from] StoreError),+ #[error("sealing a block failed: {0}")]+ Seal(String),+}++/// Where one session's block landed, for the block index.+#[derive(Debug, Clone, PartialEq, Eq)]+pub struct Written {+ pub session: SessionKey,+ pub key_month: String,+ pub object_key: String,+ /// Inclusive byte range of the sealed frame in the object, as v3 records it.+ pub byte_start: u64,+ pub byte_end: u64,+}++/// The block in the main lane's format: one `["window_id",event]` line per event.+pub fn block_lines(block: &Block) -> Vec<u8> {+ let mut lines =+ Vec::with_capacity(block.stats.byte_size as usize + block.stats.event_count as usize * 16);+ for message in &block.messages {+ let mut window = Vec::new();+ write_json_string(+ message.window_id.as_deref().unwrap_or_default(),+ &mut window,+ );+ for event in &message.events {+ lines.push(b'[');+ lines.extend_from_slice(&window);+ lines.push(b',');+ lines.extend_from_slice(&message.inner[event.span.0..event.span.1]);+ lines.extend_from_slice(b"]\n");+ }+ }+ lines+}++fn block_encoder() -> Result<brotlic::BrotliEncoder, WriteError> {+ let seal_error = |e: brotlic::SetParameterError| WriteError::Seal(e.to_string());+ brotlic::BrotliEncoderOptions::new()+ .quality(brotlic::Quality::new(BLOCK_BROTLI_QUALITY).map_err(seal_error)?)+ .window_size(brotlic::WindowSize::new(BLOCK_BROTLI_WINDOW_BITS).map_err(seal_error)?)+ .build()+ .map_err(seal_error)+}++pub fn object_key(write_day: &str, at: DateTime<Utc>) -> String {+ format!(+ "blocks/dt={write_day}/{}-{:016x}",+ at.timestamp_millis(),+ rand::thread_rng().gen::<u64>()+ )+}++/// Seals each block under its session key into one object. Blocks whose session has no key, because a deletion+/// tombstoned it, are left out.+pub async fn write_blocks(+ blocks: &[Block],+ keys: &FlushKeys,+ bucket: &str,+ store: &dyn BlobStore,+ at: DateTime<Utc>,+) -> Result<Vec<Written>, WriteError> {+ let object_key = object_key(&write_day(at), at);+ let mut object = Vec::new();+ let mut written = Vec::new();+ for block in blocks {+ let id = SessionId {+ team_id: block.session.team_id,+ session_id: block.session.session_id.clone(),+ };+ let Some(session_key) = keys.sessions.get(&id) else {+ continue;+ };+ let mut encoder = brotlic::CompressorWriter::with_encoder(block_encoder()?, Vec::new());+ encoder+ .write_all(&block_lines(block))+ .map_err(|e| WriteError::Seal(e.to_string()))?;+ let compressed = encoder+ .into_inner()+ .map_err(|e| WriteError::Seal(e.to_string()))?;+ let context = Context {+ team_id: block.session.team_id,+ session_id: Some(block.session.session_id.clone()),+ key_month: Some(session_key.key_month.clone()),+ kind: "rrweb".to_owned(),+ codec: Some(Codec::Brotli),+ ..Context::default()+ };+ let frame = seal_frame(&session_key.key, Version::V4, &context, &compressed)+ .map_err(|e| WriteError::Seal(e.to_string()))?;+ let byte_start = object.len() as u64;+ object.extend_from_slice(&frame);+ written.push(Written {+ session: block.session.clone(),+ key_month: session_key.key_month.clone(),+ object_key: object_key.clone(),+ byte_start,+ byte_end: object.len() as u64 - 1,+ });+ }+ if !written.is_empty() {+ store.put_new(bucket, &object_key, object).await?;+ }+ Ok(written)+}++#[cfg(test)]+mod tests {+ use std::collections::HashMap;+ use std::io::{self, Read};+ use std::sync::Mutex;++ use async_trait::async_trait;+ use replay_core_v4::envelope::open_frame;+ use replay_core_v4::inspect::stats::BlockStats;+ use replay_core_v4::keys::batch::SessionKey as Key;+ use replay_core_v4::scan::message::read_message;+ use serde_json::{json, Value};++ use super::*;++ #[derive(Default)]+ struct MemoryStore {+ objects: Mutex<HashMap<(String, String), Vec<u8>>>,+ }++ fn decompress(compressed: &[u8]) -> io::Result<Vec<u8>> {+ let mut plain = Vec::new();+ brotlic::DecompressorReader::new(compressed).read_to_end(&mut plain)?;+ Ok(plain)+ }++ #[async_trait]+ impl BlobStore for MemoryStore {+ async fn put_new(&self, bucket: &str, key: &str, body: Vec<u8>) -> Result<(), StoreError> {+ self.objects+ .lock()+ .unwrap()+ .entry((bucket.to_owned(), key.to_owned()))+ .or_insert(body);+ Ok(())+ }+ }++ fn block(session: &str, events: Value) -> Block {+ let inner = json!({"event": "$snapshot_items", "properties": {"$session_id": session, "$window_id": "w\"1", "$snapshot_items": events}});+ let message = read_message(+ json!({"distinct_id": "d", "data": inner.to_string()})+ .to_string()+ .into_bytes(),+ )+ .unwrap();+ Block {+ session: SessionKey {+ team_id: 7,+ session_id: session.to_owned(),+ },+ messages: vec![message],+ stats: BlockStats::default(),+ moments: Vec::new(),+ }+ }++ #[tokio::test]+ async fn seals_each_kept_session_into_one_object_by_byte_range() {+ let events = json!([{"type": 4, "timestamp": 1, "data": {"href": "https://example.com"}}, {"type": 3, "timestamp": 2, "data": {"source": 1}}]);+ let blocks = [+ block("kept", events.clone()),+ block("tombstoned", events.clone()),+ ];+ let mut keys = FlushKeys::default();+ keys.sessions.insert(+ SessionId {+ team_id: 7,+ session_id: "kept".to_owned(),+ },+ Key {+ key: [5; 32],+ key_month: "2026-09".to_owned(),+ },+ );+ let store = MemoryStore::default();+ let at = DateTime::parse_from_rfc3339("2026-10-15T12:00:00Z")+ .unwrap()+ .with_timezone(&Utc);++ let written = write_blocks(&blocks, &keys, "rrweb-v4", &store, at)+ .await+ .unwrap();+ assert_eq!(+ written+ .iter()+ .map(|w| w.session.session_id.as_str())+ .collect::<Vec<_>>(),+ vec!["kept"]+ );+ let block = &written[0];+ assert!(block.object_key.starts_with("blocks/dt=2026-10-15/"));+ assert_eq!(block.key_month, "2026-09");++ let object = store.objects.lock().unwrap()+ [&("rrweb-v4".to_owned(), block.object_key.clone())]+ .clone();+ let frame = &object[block.byte_start as usize..=block.byte_end as usize];+ let context = Context {+ team_id: 7,+ session_id: Some("kept".to_owned()),+ key_month: Some("2026-09".to_owned()),+ kind: "rrweb".to_owned(),+ codec: Some(Codec::Brotli),+ ..Context::default()+ };+ let lines =+ decompress(&open_frame(&[5; 32], Version::V4, &context, frame).unwrap()).unwrap();+ let lines: Vec<Value> = String::from_utf8(lines)+ .unwrap()+ .lines()+ .map(|l| serde_json::from_str(l).unwrap())+ .collect();+ let expected: Vec<Value> = events+ .as_array()+ .unwrap()+ .iter()+ .map(|e| json!(["w\"1", e]))+ .collect();+ assert_eq!(lines, expected);++ let nothing = write_blocks(&blocks[1..], &keys, "rrweb-v4", &store, at)+ .await+ .unwrap();+ assert!(nothing.is_empty());+ assert_eq!(store.objects.lock().unwrap().len(), 1);+ }+}