:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(replay-recorder-v4): write the v4 block index (#117976)
Nicholas Waltz2b0eab038d3f
diff --git a/rust/Cargo.lock b/rust/Cargo.lockindex eed5f9ac1588..5c20920b2140 100644--- a/rust/Cargo.lock+++ b/rust/Cargo.lock@@ -11051,6 +11051,7 @@ dependencies = [name = "replay-recorder-v4"version = "0.1.0"dependencies = [+ "aes-gcm","arc-swap","arrow-array","arrow-schema",diff --git a/rust/ai-research/replay-recorder-v4/Cargo.toml b/rust/ai-research/replay-recorder-v4/Cargo.tomlindex 93c953a66601..25184f82cfb3 100644--- a/rust/ai-research/replay-recorder-v4/Cargo.toml+++ b/rust/ai-research/replay-recorder-v4/Cargo.toml@@ -33,6 +33,7 @@ metrics = { workspace = true }parquet = "58"rand = { workspace = true }rdkafka = { workspace = true }+serde_json = { workspace = true }sqlx = { workspace = true }thiserror = { workspace = true }tokio = { workspace = true }@@ -40,6 +41,7 @@ tracing = { workspace = true }tracing-subscriber = { workspace = true }[dev-dependencies]+aes-gcm = "0.10"base64 = { workspace = true }bytes = { workspace = true }serde_json = { workspace = true }diff --git a/rust/ai-research/replay-recorder-v4/src/consumer.rs b/rust/ai-research/replay-recorder-v4/src/consumer.rsindex 999bb1991c83..20a0523f6b7a 100644--- a/rust/ai-research/replay-recorder-v4/src/consumer.rs+++ b/rust/ai-research/replay-recorder-v4/src/consumer.rs@@ -22,7 +22,7 @@ use crate::batch::{Batch, Block, Inspectors, SessionKey};use crate::process::{admit, read, sampled};use crate::store::BlobStore;use crate::teams::TeamCache;-use crate::write::{prepare, write_blocks, write_images, WriteError};+use crate::write::{prepare, write_block_index, write_blocks, write_images, WriteError};/// Records the partitions each rebalance takes away, so the loop can drop what it buffered for them.pub struct RebalanceContext {@@ -280,5 +280,14 @@ async fn flush(blocks: &mut [Block], writer: Option<&Writer>) -> Result<(), Flus).await?;counter!("ai_research_v4_recorder_written_blocks_total").increment(written.len() as u64);+ write_block_index(+ blocks,+ &written,+ &keys,+ &writer.rrweb_bucket,+ writer.store.as_ref(),+ now,+ )+ .await?;Ok(())}diff --git a/rust/ai-research/replay-recorder-v4/src/index.rs b/rust/ai-research/replay-recorder-v4/src/index.rsindex b15228543aff..39045a4c5f69 100644--- a/rust/ai-research/replay-recorder-v4/src/index.rs+++ b/rust/ai-research/replay-recorder-v4/src/index.rs@@ -4,8 +4,13 @@use std::sync::Arc;-use arrow_array::{ArrayRef, Int64Array, RecordBatch, StringArray};-use arrow_schema::{ArrowError, DataType, Field, Schema};+use arrow_array::builder::{Int32Builder, Int64Builder, ListBuilder, StringBuilder, StructBuilder};+use arrow_array::{ArrayRef, BooleanArray, Int32Array, Int64Array, RecordBatch, StringArray};+use arrow_schema::{ArrowError, DataType, Field, Fields, Schema};+use replay_core_v4::inspect::session::Moment;+use replay_core_v4::inspect::stats::BlockStats;++use crate::write::Written;use parquet::arrow::ArrowWriter;use parquet::basic::Compression;use parquet::errors::ParquetError;@@ -36,6 +41,11 @@ fn int64(value: u64) -> i64 {i64::try_from(value).unwrap_or(i64::MAX)}+/// A count as its signed column. Real values fit, and a larger one saturates.+fn int32(value: u32) -> i32 {+ i32::try_from(value).unwrap_or(i32::MAX)+}+/// Where one inline image sits in a pack in `images-v4`.#[derive(Debug, Clone, PartialEq, Eq)]pub struct ImageIndexRow {@@ -86,6 +96,267 @@ pub fn image_index(rows: &[ImageIndexRow]) -> Result<RecordBatch, IndexError> {Ok(RecordBatch::try_new(Arc::new(schema), columns)?)}+/// One block's index row: where it is, what the inspector found, and its sealed URLs.+pub struct BlockIndexRow<'a> {+ pub written: &'a Written,+ pub stats: &'a BlockStats,+ pub moments: &'a [Moment],+ pub snapshot_source: Option<&'a str>,+ pub snapshot_library: Option<&'a str>,+ /// The block's visited URLs as a v4 JSON envelope under the session key.+ pub urls: Option<String>,+}++fn moment_fields() -> Fields {+ Fields::from(vec![+ Field::new("kind", DataType::Utf8, false),+ Field::new("window_id", DataType::Utf8, false),+ Field::new("ts_ms", DataType::Int64, false),+ Field::new("event_index", DataType::Int32, false),+ ])+}++fn moments_column(rows: &[BlockIndexRow<'_>]) -> ArrayRef {+ let fields = moment_fields();+ let mut builder = ListBuilder::new(StructBuilder::from_fields(fields.clone(), rows.len()))+ .with_field(Field::new("item", DataType::Struct(fields), false));+ for row in rows {+ let entries = builder.values();+ for moment in row.moments {+ entries+ .field_builder::<StringBuilder>(0)+ .expect("kind")+ .append_value(moment.kind);+ entries+ .field_builder::<StringBuilder>(1)+ .expect("window_id")+ .append_value(&moment.window_id);+ entries+ .field_builder::<Int64Builder>(2)+ .expect("ts_ms")+ .append_value(moment.ts_ms);+ entries+ .field_builder::<Int32Builder>(3)+ .expect("event_index")+ .append_value(int32(moment.event_index));+ entries.append(true);+ }+ builder.append(true);+ }+ Arc::new(builder.finish())+}++/// A list of strings per row, with no null lists or items.+fn strings_column(rows: &[BlockIndexRow<'_>], values: fn(&BlockStats) -> &[String]) -> ArrayRef {+ let mut builder = ListBuilder::new(StringBuilder::new()).with_field(Field::new(+ "item",+ DataType::Utf8,+ false,+ ));+ for row in rows {+ for value in values(row.stats) {+ builder.values().append_value(value);+ }+ builder.append(true);+ }+ Arc::new(builder.finish())+}++pub fn block_index(rows: &[BlockIndexRow<'_>]) -> Result<RecordBatch, IndexError> {+ let mut fields = Vec::new();+ let mut columns: Vec<ArrayRef> = Vec::new();+ let mut push = |name: &str, data_type: DataType, nullable: bool, column: ArrayRef| {+ fields.push(Field::new(name, data_type, nullable));+ columns.push(column);+ };+ macro_rules! utf8 {+ ($name:literal, $value:expr) => {+ push(+ $name,+ DataType::Utf8,+ false,+ Arc::new(StringArray::from_iter_values(rows.iter().map($value))),+ )+ };+ }+ macro_rules! optional_utf8 {+ ($name:literal, $value:expr) => {+ push(+ $name,+ DataType::Utf8,+ true,+ Arc::new(rows.iter().map($value).collect::<StringArray>()),+ )+ };+ }+ macro_rules! stat {+ ($array:ident, $data_type:expr, $convert:expr, $($field:ident),*) => {+ $( push(stringify!($field), $data_type, false, Arc::new($array::from_iter_values(rows.iter().map(|r| $convert(r.stats.$field))))); )*+ };+ }+ macro_rules! flag {+ ($($field:ident),*) => {+ $( push(stringify!($field), DataType::Boolean, false, Arc::new(rows.iter().map(|r| Some(r.stats.$field)).collect::<BooleanArray>())); )*+ };+ }++ push(+ "team_id",+ DataType::Int64,+ false,+ Arc::new(Int64Array::from_iter_values(+ rows.iter().map(|r| r.written.session.team_id),+ )),+ );+ utf8!("session_id", |r| r.written.session.session_id.clone());+ utf8!("key_month", |r| r.written.key_month.clone());+ utf8!("block_s3_key", |r| r.written.object_key.clone());+ push(+ "block_byte_start",+ DataType::Int64,+ false,+ Arc::new(Int64Array::from_iter_values(+ rows.iter().map(|r| int64(r.written.byte_start)),+ )),+ );+ push(+ "block_byte_end",+ DataType::Int64,+ false,+ Arc::new(Int64Array::from_iter_values(+ rows.iter().map(|r| int64(r.written.byte_end)),+ )),+ );+ push(+ "first_ts_ms",+ DataType::Int64,+ true,+ Arc::new(+ rows.iter()+ .map(|r| r.stats.first_ts_ms)+ .collect::<Int64Array>(),+ ),+ );+ push(+ "last_ts_ms",+ DataType::Int64,+ true,+ Arc::new(+ rows.iter()+ .map(|r| r.stats.last_ts_ms)+ .collect::<Int64Array>(),+ ),+ );+ stat!(Int64Array, DataType::Int64, int64, event_count, byte_size);+ optional_utf8!("snapshot_source", |r| r.snapshot_source);+ optional_utf8!("snapshot_library", |r| r.snapshot_library);+ optional_utf8!("host", |r| r.stats.host.as_deref());+ stat!(+ Int32Array,+ DataType::Int32,+ int32,+ full_snapshots,+ events_before_first_full_snapshot,+ orphan_mutations,+ decode_failures,+ dom_nodes,+ remote_stylesheets,+ inline_stylesheets,+ iframes,+ canvases,+ videos,+ shadow_roots+ );+ stat!(+ Int64Array,+ DataType::Int64,+ i64::from,+ max_gap_ms,+ active_ms+ );+ stat!(Int64Array, DataType::Int64, int64, dom_text_chars);+ flag!(cold_start, node_cap_reached, moments_truncated, touch_input);+ stat!(+ Int32Array,+ DataType::Int32,+ int32,+ clicks,+ mouse_interactions,+ rage_clicks,+ dead_clicks,+ keypresses,+ scrolls,+ mouse_moves,+ touches,+ touch_moves,+ selections,+ media_events,+ viewport_resizes,+ navigations,+ console_errors,+ console_warnings,+ network_requests,+ network_failures,+ images_extracted,+ images_inline,+ image_urls+ );+ stat!(+ Int64Array,+ DataType::Int64,+ int64,+ image_bytes,+ written_jsonl_bytes,+ max_written_event_bytes+ );+ push(+ "viewport_width",+ DataType::Int32,+ true,+ Arc::new(+ rows.iter()+ .map(|r| r.stats.viewport_width.map(int32))+ .collect::<Int32Array>(),+ ),+ );+ push(+ "viewport_height",+ DataType::Int32,+ true,+ Arc::new(+ rows.iter()+ .map(|r| r.stats.viewport_height.map(int32))+ .collect::<Int32Array>(),+ ),+ );+ push(+ "viewport_ts_ms",+ DataType::Int64,+ true,+ Arc::new(+ rows.iter()+ .map(|r| r.stats.viewport_ts_ms)+ .collect::<Int64Array>(),+ ),+ );+ let moments = moments_column(rows);+ push("moments", moments.data_type().clone(), false, moments);+ let fingerprints = strings_column(rows, |s| &s.url_fingerprints);+ push(+ "url_fingerprints",+ fingerprints.data_type().clone(),+ false,+ fingerprints,+ );+ let windows = strings_column(rows, |s| &s.window_ids);+ push("window_ids", windows.data_type().clone(), false, windows);+ optional_utf8!("urls", |r| r.urls.as_deref());+ Ok(RecordBatch::try_new(+ Arc::new(Schema::new(fields)),+ columns,+ )?)+}+#[cfg(test)]mod tests {use bytes::Bytes;diff --git a/rust/ai-research/replay-recorder-v4/src/write.rs b/rust/ai-research/replay-recorder-v4/src/write.rsindex f60ce6a7453b..d4056972ca44 100644--- a/rust/ai-research/replay-recorder-v4/src/write.rs+++ b/rust/ai-research/replay-recorder-v4/src/write.rs@@ -9,15 +9,20 @@ use std::io::Write;use chrono::{DateTime, Utc};use rand::Rng;use replay_core_v4::bytescan::write_json_string;-use replay_core_v4::envelope::{seal_frame, Codec, Context, Version};+use replay_core_v4::envelope::{seal_frame, seal_json_envelope, Codec, Context, Version};+use replay_core_v4::inspect::events::event_page_url;+use replay_core_v4::inspect::stats::MAX_URL_FINGERPRINTS;use replay_core_v4::keys::batch::{FlushKeys, SessionId, SessionKey as Key};use replay_core_v4::keys::rows::write_day;+use replay_core_v4::refs::url_fingerprint;use replay_core_v4::scan::cv::extract_event_images;use replay_core_v4::scan::images::{Image, ImageLimits, ImageSink};-use replay_core_v4::seal::image_content_key;+use replay_core_v4::seal::{image_content_key, url_fingerprint_key};use crate::batch::{Block, SessionKey};-use crate::index::{image_index, write_parquet, ImageIndexRow, IndexError};+use crate::index::{+ block_index, image_index, write_parquet, BlockIndexRow, ImageIndexRow, IndexError,+};use crate::store::{BlobStore, StoreError};/// Brotli settings for blocks, the same as the v3 lane, which uses the same C encoder. The lane keeps blocks for months@@ -108,6 +113,7 @@ pub fn prepare(blocks: &mut [Block], keys: &FlushKeys, key_month: &str) -> Prepacontinue;};let content_key = image_content_key(team_key);+ block.stats.url_fingerprints = url_fingerprints(&key.key, &block_urls(block));let images = prepared.images.entry(block.session.team_id).or_default();let mut lines = Vec::with_capacity(block.stats.byte_size as usize + block.stats.event_count as usize * 16,@@ -271,16 +277,126 @@ pub async fn write_blocks(Ok(written)}+/// The pages a block visited, from its Meta events, in order and without repeats in a row.+/// An SPA navigation, as a `$url_changed` custom event, counts as a visit too.+pub fn block_urls(block: &Block) -> Vec<String> {+ let mut urls: Vec<String> = Vec::new();+ for message in &block.messages {+ for event in &message.events {+ if let Some(href) = event_page_url(&message.inner, event) {+ if urls.last() != Some(&href) {+ urls.push(href);+ }+ }+ }+ }+ urls+}++/// The block's distinct pages as fingerprints under the session's fingerprint key, the first+/// [`MAX_URL_FINGERPRINTS`] in the order the block saw them.+pub fn url_fingerprints(session_key: &[u8; 32], urls: &[String]) -> Vec<String> {+ let key = url_fingerprint_key(session_key);+ let mut fingerprints = Vec::new();+ for url in urls {+ if fingerprints.len() >= MAX_URL_FINGERPRINTS {+ break;+ }+ let fingerprint = url_fingerprint(&key, url);+ if !fingerprints.contains(&fingerprint) {+ fingerprints.push(fingerprint);+ }+ }+ fingerprints+}++/// Writes one block index file for the flush, after its blocks, so no row points at a block that is not stored.+pub async fn write_block_index(+ blocks: &[Block],+ written: &[Written],+ keys: &FlushKeys,+ rrweb_bucket: &str,+ store: &dyn BlobStore,+ at: DateTime<Utc>,+) -> Result<usize, WriteError> {+ let by_session: BTreeMap<&SessionKey, &Block> =+ blocks.iter().map(|b| (&b.session, b)).collect();+ let mut rows = Vec::with_capacity(written.len());+ for written in written {+ let Some(block) = by_session.get(&written.session) else {+ continue;+ };+ let id = SessionId {+ team_id: written.session.team_id,+ session_id: written.session.session_id.clone(),+ };+ let Some(key) = keys.sessions.get(&id) else {+ continue;+ };+ let urls = block_urls(block);+ let urls = if urls.is_empty() {+ None+ } else {+ let context = Context {+ team_id: id.team_id,+ session_id: Some(id.session_id.clone()),+ key_month: Some(key.key_month.clone()),+ kind: "index".to_owned(),+ reference: Some("urls".to_owned()),+ ..Context::default()+ };+ let json = serde_json::to_vec(&urls).map_err(|e| WriteError::Seal(e.to_string()))?;+ Some(+ seal_json_envelope(&key.key, Version::V4, &context, &json)+ .map_err(|e| WriteError::Seal(e.to_string()))?+ .to_string(),+ )+ };+ let first = block.messages.first();+ rows.push(BlockIndexRow {+ written,+ stats: &block.stats,+ moments: &block.moments,+ snapshot_source: first.and_then(|m| m.snapshot_source.as_deref()),+ snapshot_library: first.and_then(|m| m.library.as_deref()),+ urls,+ });+ }+ if rows.is_empty() {+ return Ok(0);+ }+ let index_key = format!(+ "index/blocks/dt={}/{}.parquet",+ write_day(at),+ unique_suffix(at)+ );+ store+ .put_new(+ rrweb_bucket,+ &index_key,+ write_parquet(&block_index(&rows)?)?,+ )+ .await?;+ Ok(rows.len())+}+#[cfg(test)]mod tests {use std::collections::HashMap;use std::io::{self, Read};use std::sync::Mutex;+ use aes_gcm::aead::{Aead, KeyInit, Payload};+ use aes_gcm::{Aes256Gcm, Nonce};+ use arrow_array::{Array, Int64Array, ListArray, StringArray, StructArray};+ use arrow_schema::DataType;use async_trait::async_trait;use base64::engine::general_purpose::STANDARD;use base64::Engine;+ use bytes::Bytes;+ use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;use replay_core_v4::envelope::open_frame;+ use replay_core_v4::inspect::session::Moment;use replay_core_v4::inspect::stats::BlockStats;use replay_core_v4::scan::message::read_message;use serde_json::{json, Value};@@ -349,8 +465,11 @@ mod tests {#[tokio::test]async fn stores_images_once_and_blocks_that_point_at_them() {let events = json!([+ {"type": 4, "timestamp": 0, "data": {"href": "https://app.example.com/a"}},{"type": 2, "timestamp": 1, "data": {"node": {"attributes": {"src": uri(b"new"), "srcset": format!("{} 2x", uri(b"new"))}}}},{"type": 3, "timestamp": 2, "data": {"source": 0, "adds": [{"parentId": 1, "node": {"attributes": {"src": uri(b"old")}}}]}},+ {"type": 5, "timestamp": 3, "data": {"tag": "$url_changed", "payload": {"href": "https://app.example.com/b"}}},+ {"type": 4, "timestamp": 4, "data": {"href": "https://app.example.com/a"}},]);let mut blocks = vec![block("kept", events.clone()), block("tombstoned", events)];let mut keys = FlushKeys::default();@@ -454,7 +573,7 @@ mod tests {decompress(&open_frame(&[5; 32], Version::V4, &context, frame).unwrap()).unwrap(),).unwrap();- let first: Value = serde_json::from_str(lines.lines().next().unwrap()).unwrap();+ let first: Value = serde_json::from_str(lines.lines().nth(1).unwrap()).unwrap();let reference = first[1]["data"]["node"]["attributes"]["src"].as_str().unwrap();@@ -489,5 +608,147 @@ mod tests {.count(),1);++ // The block index row points at the block, keeps the moments, and seals the visited pages.+ blocks[0].stats.window_ids = vec!["w1".to_owned()];+ blocks[0].moments = vec![Moment {+ kind: "full_snapshot",+ window_id: "w1".into(),+ ts_ms: 1,+ event_index: 1,+ }];+ assert_eq!(+ write_block_index(&blocks, &written, &keys, "rrweb-v4", &store, at)+ .await+ .unwrap(),+ 1+ );+ let index_key = store+ .objects+ .lock()+ .unwrap()+ .keys()+ .find(|(_, key)| key.starts_with("index/blocks/"))+ .unwrap()+ .1+ .clone();+ let batch = ParquetRecordBatchReaderBuilder::try_new(Bytes::from(+ store.get("rrweb-v4", &index_key),+ ))+ .unwrap()+ .build()+ .unwrap()+ .next()+ .unwrap()+ .unwrap();+ let column = |name: &str| {+ batch+ .column_by_name(name)+ .unwrap_or_else(|| panic!("no column {name}"))+ .clone()+ };+ let text = |name: &str| {+ column(name)+ .as_any()+ .downcast_ref::<StringArray>()+ .unwrap()+ .value(0)+ .to_owned()+ };+ assert_eq!(text("block_s3_key"), block.object_key);+ assert_eq!(+ column("block_byte_end")+ .as_any()+ .downcast_ref::<Int64Array>()+ .unwrap()+ .value(0),+ i64::try_from(block.byte_end).unwrap()+ );+ for (name, data_type) in [+ ("orphan_mutations", DataType::Int32),+ ("active_ms", DataType::Int64),+ ("rage_clicks", DataType::Int32),+ ("cold_start", DataType::Boolean),+ ("viewport_width", DataType::Int32),+ ("image_urls", DataType::Int32),+ ("decode_failures", DataType::Int32),+ ("mouse_interactions", DataType::Int32),+ ("touch_moves", DataType::Int32),+ ("moments_truncated", DataType::Boolean),+ ("written_jsonl_bytes", DataType::Int64),+ ("max_written_event_bytes", DataType::Int64),+ ] {+ assert_eq!(column(name).data_type(), &data_type, "{name}");+ }+ let strings = |name: &str| -> Vec<String> {+ let list = column(name);+ let values = list.as_any().downcast_ref::<ListArray>().unwrap().value(0);+ let values = values.as_any().downcast_ref::<StringArray>().unwrap();+ values.iter().map(|v| v.unwrap().to_owned()).collect()+ };+ // The page the block sees again keeps its first place, under the session's own fingerprint key.+ let fingerprint_key = replay_core_v4::seal::url_fingerprint_key(&[5; 32]);+ let fingerprint = |path: &str| {+ url_fingerprint(&fingerprint_key, &format!("https://app.example.com{path}"))+ };+ assert_eq!(+ strings("url_fingerprints"),+ [fingerprint("/a"), fingerprint("/b")]+ );+ assert_eq!(strings("window_ids"), ["w1"]);+ let moments = column("moments");+ let moments = moments+ .as_any()+ .downcast_ref::<ListArray>()+ .unwrap()+ .value(0);+ let moments = moments.as_any().downcast_ref::<StructArray>().unwrap();+ assert_eq!(+ moments.column_by_name("event_index").unwrap().data_type(),+ &DataType::Int32+ );+ let kinds = moments.column_by_name("kind").unwrap().clone();+ assert_eq!(+ kinds+ .as_any()+ .downcast_ref::<StringArray>()+ .unwrap()+ .value(0),+ "full_snapshot"+ );+ let envelope: Value = serde_json::from_str(&text("urls")).unwrap();+ let sealed = STANDARD+ .decode(envelope["ciphertext"].as_str().unwrap())+ .unwrap();+ let nonce = STANDARD+ .decode(envelope["nonce"].as_str().unwrap())+ .unwrap();+ let aad = Context {+ team_id: 7,+ session_id: Some("kept".to_owned()),+ key_month: Some("2026-09".to_owned()),+ kind: "index".to_owned(),+ reference: Some("urls".to_owned()),+ ..Context::default()+ }+ .authenticated_data(Version::V4);+ let urls = Aes256Gcm::new_from_slice(&[5; 32])+ .unwrap()+ .decrypt(+ Nonce::from_slice(&nonce),+ Payload {+ msg: &sealed,+ aad: aad.as_bytes(),+ },+ )+ .unwrap();+ assert_eq!(+ serde_json::from_slice::<Vec<String>>(&urls).unwrap(),+ [+ "https://app.example.com/a",+ "https://app.example.com/b",+ "https://app.example.com/a"+ ]+ );}}