Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(replay-recorder-v4): store sealed session blocks (#117970)

Nicholas Waltzddc429e81c25
diff --git a/rust/Cargo.lock b/rust/Cargo.lock
index 209ef864d212..14198df61440 100644
--- a/rust/Cargo.lock
+++ b/rust/Cargo.lock
@@ -11052,7 +11052,14 @@ name = "replay-recorder-v4"
version = "0.1.0"
dependencies = [
"arc-swap",
+ "async-trait",
+ "aws-config",
+ "aws-sdk-dynamodb",
+ "aws-sdk-kms",
+ "aws-sdk-s3",
"axum 0.7.9",
+ "brotlic",
+ "chrono",
"common-alloc",
"common-database",
"common-kafka",
@@ -11060,10 +11067,12 @@ dependencies = [
"envconfig",
"lifecycle",
"metrics",
+ "rand 0.8.5",
"rdkafka",
"replay-core-v4",
"serde_json",
"sqlx",
+ "thiserror 2.0.18",
"tokio",
"tracing",
"tracing-subscriber",
diff --git a/rust/ai-research/replay-recorder-v4/Cargo.toml b/rust/ai-research/replay-recorder-v4/Cargo.toml
index 0b8bee87bfa4..56d4510be171 100644
--- a/rust/ai-research/replay-recorder-v4/Cargo.toml
+++ b/rust/ai-research/replay-recorder-v4/Cargo.toml
@@ -17,14 +17,24 @@ lifecycle = { path = "../../common/lifecycle" }
replay-core-v4 = { path = "../replay-core-v4" }
arc-swap = { workspace = true }
+async-trait = { workspace = true }
+aws-config = { workspace = true }
+aws-sdk-dynamodb = { workspace = true }
+aws-sdk-kms = { workspace = true }
+aws-sdk-s3 = { workspace = true }
axum = { workspace = true }
+brotlic = "0.8"
+chrono = { workspace = true }
envconfig = { workspace = true }
metrics = { workspace = true }
+rand = { workspace = true }
rdkafka = { workspace = true }
sqlx = { workspace = true }
+thiserror = { workspace = true }
tokio = { workspace = true }
tracing = { workspace = true }
tracing-subscriber = { workspace = true }
[dev-dependencies]
serde_json = { workspace = true }
+tokio = { workspace = true, features = ["test-util"] }
diff --git a/rust/ai-research/replay-recorder-v4/src/config.rs b/rust/ai-research/replay-recorder-v4/src/config.rs
index 0951a8ed98c2..ade7a7309a23 100644
--- a/rust/ai-research/replay-recorder-v4/src/config.rs
+++ b/rust/ai-research/replay-recorder-v4/src/config.rs
@@ -39,6 +39,20 @@ pub struct Config {
#[envconfig(from = "AI_RESEARCH_V4_SESSION_IDLE_SECS", default = "3600")]
pub session_idle_secs: u64,
+ /// Off, the recorder reads, admits and inspects, and stores nothing: no keys, no objects.
+ #[envconfig(from = "AI_RESEARCH_V4_WRITE_ENABLED", default = "false")]
+ pub write_enabled: bool,
+
+ /// Resource names come from the deployment, as the v3 mirror's do.
+ #[envconfig(from = "AI_RESEARCH_V4_KEY_TABLE", default = "")]
+ pub key_table: String,
+
+ #[envconfig(from = "AI_RESEARCH_V4_KMS_KEY_ARN", default = "")]
+ pub kms_key_arn: String,
+
+ #[envconfig(from = "AI_RESEARCH_V4_RRWEB_BUCKET", default = "")]
+ pub rrweb_bucket: String,
+
#[envconfig(default = "9104")]
pub metrics_port: u16,
}
diff --git a/rust/ai-research/replay-recorder-v4/src/consumer.rs b/rust/ai-research/replay-recorder-v4/src/consumer.rs
index 9438f0183862..dde0f8fe8935 100644
--- a/rust/ai-research/replay-recorder-v4/src/consumer.rs
+++ b/rust/ai-research/replay-recorder-v4/src/consumer.rs
@@ -12,9 +12,17 @@ use rdkafka::consumer::{
use rdkafka::message::{BorrowedMessage, Headers, Message as _};
use rdkafka::{ClientConfig, ClientContext, Offset, TopicPartitionList};
+use chrono::Utc;
+use replay_core_v4::keys::batch::{KeyStore, SessionId};
+use replay_core_v4::keys::kms::AwsKms;
+use replay_core_v4::keys::rows::write_day;
+use replay_core_v4::keys::table::{AwsKeyTable, CachedKeyTable};
+
use crate::batch::{Batch, Block, Inspectors, SessionKey};
use crate::process::{admit, read, sampled};
+use crate::store::BlobStore;
use crate::teams::TeamCache;
+use crate::write::{write_blocks, WriteError};
/// Records the partitions each rebalance takes away, so the loop can drop what it buffered for them.
pub struct RebalanceContext {
@@ -112,6 +120,21 @@ fn header<'m>(message: &'m BorrowedMessage<'_>, name: &str) -> Option<&'m str> {
.and_then(|v| std::str::from_utf8(v).ok())
}
+/// What a flush needs to store blocks. Without it the recorder runs dry.
+pub struct Writer {
+ pub keys: KeyStore<CachedKeyTable<AwsKeyTable>, AwsKms>,
+ pub store: Arc<dyn BlobStore>,
+ pub rrweb_bucket: String,
+}
+
+#[derive(Debug, thiserror::Error)]
+pub enum FlushError {
+ #[error(transparent)]
+ Keys(#[from] replay_core_v4::keys::batch::KeyError),
+ #[error(transparent)]
+ Write(#[from] WriteError),
+}
+
pub struct Settings {
pub sample_percentage: f64,
pub flush_interval: Duration,
@@ -123,6 +146,7 @@ pub async fn run(
consumer: RecorderConsumer,
settings: Settings,
teams: Arc<TeamCache>,
+ writer: Option<Writer>,
handle: Handle,
) {
let shutdown = handle.shutdown_signal();
@@ -155,7 +179,12 @@ pub async fn run(
}
if batch.should_flush(settings.flush_interval, settings.flush_max_bytes) {
let (blocks, offsets) = inspectors.inspect(std::mem::take(&mut batch));
- flush(&blocks);
+ if let Err(error) = flush(&blocks, writer.as_ref()).await {
+ // Nothing of this flush is committed, so a restart reads it again from the last commit.
+ tracing::error!(%error, "flush failed");
+ handle.signal_failure(format!("flush failed: {error}"));
+ break;
+ }
consumer.commit(&offsets);
inspectors.evict_idle(settings.session_idle);
}
@@ -200,8 +229,8 @@ fn handle_message(
}
}
-/// Reports what a flush inspected. The writers that store blocks come in the next layers.
-fn flush(blocks: &[Block]) {
+/// Stores a flush's blocks, or only reports them when the recorder runs dry.
+async fn flush(blocks: &[Block], writer: Option<&Writer>) -> Result<(), FlushError> {
counter!("ai_research_v4_recorder_blocks_total").increment(blocks.len() as u64);
for block in blocks {
counter!("ai_research_v4_recorder_block_events_total").increment(block.stats.event_count);
@@ -212,4 +241,29 @@ fn flush(blocks: &[Block]) {
counter!("ai_research_v4_recorder_orphan_mutation_blocks_total").increment(1);
}
}
+ let Some(writer) = writer else { return Ok(()) };
+ let now = Utc::now();
+ let sessions: Vec<SessionId> = blocks
+ .iter()
+ .map(|b| SessionId {
+ team_id: b.session.team_id,
+ session_id: b.session.session_id.clone(),
+ })
+ .collect();
+ // Keys are stored before anything is sealed with them, as in v3.
+ let keys = writer
+ .keys
+ .keys_for_flush(&sessions, &write_day(now))
+ .await?;
+ counter!("ai_research_v4_recorder_dropped_sessions_total").increment(keys.dropped.len() as u64);
+ let written = write_blocks(
+ blocks,
+ &keys,
+ &writer.rrweb_bucket,
+ writer.store.as_ref(),
+ now,
+ )
+ .await?;
+ counter!("ai_research_v4_recorder_written_blocks_total").increment(written.len() as u64);
+ Ok(())
}
diff --git a/rust/ai-research/replay-recorder-v4/src/lib.rs b/rust/ai-research/replay-recorder-v4/src/lib.rs
index f471229fe802..72357f281585 100644
--- a/rust/ai-research/replay-recorder-v4/src/lib.rs
+++ b/rust/ai-research/replay-recorder-v4/src/lib.rs
@@ -5,4 +5,6 @@ pub mod batch;
pub mod config;
pub mod consumer;
pub mod process;
+pub mod store;
pub mod teams;
+pub mod write;
diff --git a/rust/ai-research/replay-recorder-v4/src/main.rs b/rust/ai-research/replay-recorder-v4/src/main.rs
index 0a7a1a40a6eb..6f5e44652486 100644
--- a/rust/ai-research/replay-recorder-v4/src/main.rs
+++ b/rust/ai-research/replay-recorder-v4/src/main.rs
@@ -11,8 +11,12 @@ use tracing_subscriber::layer::SubscriberExt;
use tracing_subscriber::util::SubscriberInitExt;
use tracing_subscriber::{fmt, EnvFilter};
+use replay_core_v4::keys::batch::KeyStore;
+use replay_core_v4::keys::kms::{AwsKms, CachedKms, KmsLimits};
+use replay_core_v4::keys::table::{AwsKeyTable, CachedKeyTable};
use replay_recorder_v4::config::Config;
-use replay_recorder_v4::consumer::{run, RecorderConsumer, Settings};
+use replay_recorder_v4::consumer::{run, RecorderConsumer, Settings, Writer};
+use replay_recorder_v4::store::S3Store;
use replay_recorder_v4::teams::TeamCache;
common_alloc::used!();
@@ -112,7 +116,39 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
flush_max_bytes: config.flush_max_bytes,
session_idle: Duration::from_secs(config.session_idle_secs),
};
- tokio::spawn(run(consumer, settings, teams, consumer_handle));
+ let writer = if config.write_enabled {
+ let aws = aws_config::defaults(aws_config::BehaviorVersion::latest())
+ .load()
+ .await;
+ // The cache lifetimes are v3's: a session row's lifetime bounds how late a deletion is seen.
+ let table = CachedKeyTable::new(
+ AwsKeyTable::new(
+ aws_sdk_dynamodb::Client::new(&aws),
+ config.key_table.clone(),
+ ),
+ 100_000,
+ Duration::from_secs(300),
+ Duration::from_secs(3600),
+ );
+ let kms = CachedKms::new(
+ AwsKms::new(
+ aws_sdk_kms::Client::new(&aws),
+ config.kms_key_arn.clone(),
+ KmsLimits::default(),
+ ),
+ 100_000,
+ Duration::from_secs(1800),
+ );
+ Some(Writer {
+ keys: KeyStore::new(table, kms),
+ store: Arc::new(S3Store::new(aws_sdk_s3::Client::new(&aws))),
+ rrweb_bucket: config.rrweb_bucket.clone(),
+ })
+ } else {
+ tracing::info!("writes are off: the recorder stores nothing");
+ None
+ };
+ tokio::spawn(run(consumer, settings, teams, writer, consumer_handle));
monitor.wait().await?;
Ok(())
diff --git a/rust/ai-research/replay-recorder-v4/src/store.rs b/rust/ai-research/replay-recorder-v4/src/store.rs
new file mode 100644
index 000000000000..7c01635373b8
--- /dev/null
+++ b/rust/ai-research/replay-recorder-v4/src/store.rs
@@ -0,0 +1,51 @@
+//! Object storage. Tests replace S3 at this boundary.
+
+use async_trait::async_trait;
+
+#[derive(Debug, thiserror::Error)]
+#[error("object store write failed: {0}")]
+pub struct StoreError(pub String);
+
+#[async_trait]
+pub trait BlobStore: Send + Sync {
+ /// Writes an object that must not exist yet. The recorder names every object uniquely, so an existing one means
+ /// the SDK retried this same write, and the first write stands.
+ async fn put_new(&self, bucket: &str, key: &str, body: Vec<u8>) -> Result<(), StoreError>;
+}
+
+pub struct S3Store {
+ client: aws_sdk_s3::Client,
+}
+
+impl S3Store {
+ pub fn new(client: aws_sdk_s3::Client) -> Self {
+ Self { client }
+ }
+}
+
+#[async_trait]
+impl BlobStore for S3Store {
+ async fn put_new(&self, bucket: &str, key: &str, body: Vec<u8>) -> Result<(), StoreError> {
+ let result = self
+ .client
+ .put_object()
+ .bucket(bucket)
+ .key(key)
+ .if_none_match("*")
+ .body(body.into())
+ .send()
+ .await;
+ match result {
+ Ok(_) => Ok(()),
+ // 412: the SDK retried this PUT after a first try that stored the object, so it is there.
+ Err(error)
+ if error
+ .raw_response()
+ .is_some_and(|r| r.status().as_u16() == 412) =>
+ {
+ Ok(())
+ }
+ Err(error) => Err(StoreError(error.to_string())),
+ }
+ }
+}
diff --git a/rust/ai-research/replay-recorder-v4/src/write.rs b/rust/ai-research/replay-recorder-v4/src/write.rs
new file mode 100644
index 000000000000..37b97e0b7d76
--- /dev/null
+++ b/rust/ai-research/replay-recorder-v4/src/write.rs
@@ -0,0 +1,257 @@
+//! Writes a flush: keys first, then one object of sealed session blocks.
+
+use std::io::Write;
+
+use chrono::{DateTime, Utc};
+use rand::Rng;
+use replay_core_v4::bytescan::write_json_string;
+use replay_core_v4::envelope::{seal_frame, Codec, Context, Version};
+use replay_core_v4::keys::batch::{FlushKeys, SessionId};
+use replay_core_v4::keys::rows::write_day;
+
+use crate::batch::{Block, SessionKey};
+use crate::store::{BlobStore, StoreError};
+
+/// Brotli settings for blocks, the same as the v3 lane, which uses the same C encoder. The lane keeps blocks for months
+/// and reads them rarely, so it takes the smaller output for more CPU.
+const BLOCK_BROTLI_QUALITY: u8 = 9;
+const BLOCK_BROTLI_WINDOW_BITS: u8 = 22;
+
+#[derive(Debug, thiserror::Error)]
+pub enum WriteError {
+ #[error(transparent)]
+ Store(#[from] StoreError),
+ #[error("sealing a block failed: {0}")]
+ Seal(String),
+}
+
+/// Where one session's block landed, for the block index.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct Written {
+ pub session: SessionKey,
+ pub key_month: String,
+ pub object_key: String,
+ /// Inclusive byte range of the sealed frame in the object, as v3 records it.
+ pub byte_start: u64,
+ pub byte_end: u64,
+}
+
+/// The block in the main lane's format: one `["window_id",event]` line per event.
+pub fn block_lines(block: &Block) -> Vec<u8> {
+ let mut lines =
+ Vec::with_capacity(block.stats.byte_size as usize + block.stats.event_count as usize * 16);
+ for message in &block.messages {
+ let mut window = Vec::new();
+ write_json_string(
+ message.window_id.as_deref().unwrap_or_default(),
+ &mut window,
+ );
+ for event in &message.events {
+ lines.push(b'[');
+ lines.extend_from_slice(&window);
+ lines.push(b',');
+ lines.extend_from_slice(&message.inner[event.span.0..event.span.1]);
+ lines.extend_from_slice(b"]\n");
+ }
+ }
+ lines
+}
+
+fn block_encoder() -> Result<brotlic::BrotliEncoder, WriteError> {
+ let seal_error = |e: brotlic::SetParameterError| WriteError::Seal(e.to_string());
+ brotlic::BrotliEncoderOptions::new()
+ .quality(brotlic::Quality::new(BLOCK_BROTLI_QUALITY).map_err(seal_error)?)
+ .window_size(brotlic::WindowSize::new(BLOCK_BROTLI_WINDOW_BITS).map_err(seal_error)?)
+ .build()
+ .map_err(seal_error)
+}
+
+pub fn object_key(write_day: &str, at: DateTime<Utc>) -> String {
+ format!(
+ "blocks/dt={write_day}/{}-{:016x}",
+ at.timestamp_millis(),
+ rand::thread_rng().gen::<u64>()
+ )
+}
+
+/// Seals each block under its session key into one object. Blocks whose session has no key, because a deletion
+/// tombstoned it, are left out.
+pub async fn write_blocks(
+ blocks: &[Block],
+ keys: &FlushKeys,
+ bucket: &str,
+ store: &dyn BlobStore,
+ at: DateTime<Utc>,
+) -> Result<Vec<Written>, WriteError> {
+ let object_key = object_key(&write_day(at), at);
+ let mut object = Vec::new();
+ let mut written = Vec::new();
+ for block in blocks {
+ let id = SessionId {
+ team_id: block.session.team_id,
+ session_id: block.session.session_id.clone(),
+ };
+ let Some(session_key) = keys.sessions.get(&id) else {
+ continue;
+ };
+ let mut encoder = brotlic::CompressorWriter::with_encoder(block_encoder()?, Vec::new());
+ encoder
+ .write_all(&block_lines(block))
+ .map_err(|e| WriteError::Seal(e.to_string()))?;
+ let compressed = encoder
+ .into_inner()
+ .map_err(|e| WriteError::Seal(e.to_string()))?;
+ let context = Context {
+ team_id: block.session.team_id,
+ session_id: Some(block.session.session_id.clone()),
+ key_month: Some(session_key.key_month.clone()),
+ kind: "rrweb".to_owned(),
+ codec: Some(Codec::Brotli),
+ ..Context::default()
+ };
+ let frame = seal_frame(&session_key.key, Version::V4, &context, &compressed)
+ .map_err(|e| WriteError::Seal(e.to_string()))?;
+ let byte_start = object.len() as u64;
+ object.extend_from_slice(&frame);
+ written.push(Written {
+ session: block.session.clone(),
+ key_month: session_key.key_month.clone(),
+ object_key: object_key.clone(),
+ byte_start,
+ byte_end: object.len() as u64 - 1,
+ });
+ }
+ if !written.is_empty() {
+ store.put_new(bucket, &object_key, object).await?;
+ }
+ Ok(written)
+}
+
+#[cfg(test)]
+mod tests {
+ use std::collections::HashMap;
+ use std::io::{self, Read};
+ use std::sync::Mutex;
+
+ use async_trait::async_trait;
+ use replay_core_v4::envelope::open_frame;
+ use replay_core_v4::inspect::stats::BlockStats;
+ use replay_core_v4::keys::batch::SessionKey as Key;
+ use replay_core_v4::scan::message::read_message;
+ use serde_json::{json, Value};
+
+ use super::*;
+
+ #[derive(Default)]
+ struct MemoryStore {
+ objects: Mutex<HashMap<(String, String), Vec<u8>>>,
+ }
+
+ fn decompress(compressed: &[u8]) -> io::Result<Vec<u8>> {
+ let mut plain = Vec::new();
+ brotlic::DecompressorReader::new(compressed).read_to_end(&mut plain)?;
+ Ok(plain)
+ }
+
+ #[async_trait]
+ impl BlobStore for MemoryStore {
+ async fn put_new(&self, bucket: &str, key: &str, body: Vec<u8>) -> Result<(), StoreError> {
+ self.objects
+ .lock()
+ .unwrap()
+ .entry((bucket.to_owned(), key.to_owned()))
+ .or_insert(body);
+ Ok(())
+ }
+ }
+
+ fn block(session: &str, events: Value) -> Block {
+ let inner = json!({"event": "$snapshot_items", "properties": {"$session_id": session, "$window_id": "w\"1", "$snapshot_items": events}});
+ let message = read_message(
+ json!({"distinct_id": "d", "data": inner.to_string()})
+ .to_string()
+ .into_bytes(),
+ )
+ .unwrap();
+ Block {
+ session: SessionKey {
+ team_id: 7,
+ session_id: session.to_owned(),
+ },
+ messages: vec![message],
+ stats: BlockStats::default(),
+ moments: Vec::new(),
+ }
+ }
+
+ #[tokio::test]
+ async fn seals_each_kept_session_into_one_object_by_byte_range() {
+ let events = json!([{"type": 4, "timestamp": 1, "data": {"href": "https://example.com"}}, {"type": 3, "timestamp": 2, "data": {"source": 1}}]);
+ let blocks = [
+ block("kept", events.clone()),
+ block("tombstoned", events.clone()),
+ ];
+ let mut keys = FlushKeys::default();
+ keys.sessions.insert(
+ SessionId {
+ team_id: 7,
+ session_id: "kept".to_owned(),
+ },
+ Key {
+ key: [5; 32],
+ key_month: "2026-09".to_owned(),
+ },
+ );
+ let store = MemoryStore::default();
+ let at = DateTime::parse_from_rfc3339("2026-10-15T12:00:00Z")
+ .unwrap()
+ .with_timezone(&Utc);
+
+ let written = write_blocks(&blocks, &keys, "rrweb-v4", &store, at)
+ .await
+ .unwrap();
+ assert_eq!(
+ written
+ .iter()
+ .map(|w| w.session.session_id.as_str())
+ .collect::<Vec<_>>(),
+ vec!["kept"]
+ );
+ let block = &written[0];
+ assert!(block.object_key.starts_with("blocks/dt=2026-10-15/"));
+ assert_eq!(block.key_month, "2026-09");
+
+ let object = store.objects.lock().unwrap()
+ [&("rrweb-v4".to_owned(), block.object_key.clone())]
+ .clone();
+ let frame = &object[block.byte_start as usize..=block.byte_end as usize];
+ let context = Context {
+ team_id: 7,
+ session_id: Some("kept".to_owned()),
+ key_month: Some("2026-09".to_owned()),
+ kind: "rrweb".to_owned(),
+ codec: Some(Codec::Brotli),
+ ..Context::default()
+ };
+ let lines =
+ decompress(&open_frame(&[5; 32], Version::V4, &context, frame).unwrap()).unwrap();
+ let lines: Vec<Value> = String::from_utf8(lines)
+ .unwrap()
+ .lines()
+ .map(|l| serde_json::from_str(l).unwrap())
+ .collect();
+ let expected: Vec<Value> = events
+ .as_array()
+ .unwrap()
+ .iter()
+ .map(|e| json!(["w\"1", e]))
+ .collect();
+ assert_eq!(lines, expected);
+
+ let nothing = write_blocks(&blocks[1..], &keys, "rrweb-v4", &store, at)
+ .await
+ .unwrap();
+ assert!(nothing.is_empty());
+ assert_eq!(store.objects.lock().unwrap().len(), 1);
+ }
+}