:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(replay-recorder-v4): send the v4 index to a kafka topic (#119746)
Nicholas Waltzef8608ab5d16
diff --git a/rust/Cargo.lock b/rust/Cargo.lockindex e871da13c7a6..d7978d6f69a8 100644--- a/rust/Cargo.lock+++ b/rust/Cargo.lock@@ -11054,8 +11054,6 @@ dependencies = ["aes-gcm","aho-corasick","arc-swap",- "arrow-array",- "arrow-schema","async-trait","aws-config","aws-sdk-dynamodb",@@ -11064,7 +11062,6 @@ dependencies = ["axum 0.7.9","base64 0.22.1","brotlic",- "bytes","capture","chrono","common-alloc",@@ -11077,10 +11074,10 @@ dependencies = ["lifecycle","metrics","moka",- "parquet","rand 0.8.5","rdkafka","replay-core-v4",+ "serde","serde_json","sqlx","thiserror 2.0.18",diff --git a/rust/ai-research/replay-recorder-v4/Cargo.toml b/rust/ai-research/replay-recorder-v4/Cargo.tomlindex ed045fbc5545..850ec63e079e 100644--- a/rust/ai-research/replay-recorder-v4/Cargo.toml+++ b/rust/ai-research/replay-recorder-v4/Cargo.toml@@ -21,9 +21,6 @@ replay-core-v4 = { path = "../replay-core-v4" }aho-corasick = { workspace = true }arc-swap = { workspace = true }-# The major version deltalite links, so the workspace builds one arrow.-arrow-array = "58"-arrow-schema = "58"async-trait = { workspace = true }aws-config = { workspace = true }aws-sdk-dynamodb = { workspace = true }@@ -36,9 +33,9 @@ envconfig = { workspace = true }futures = { workspace = true }metrics = { workspace = true }moka = { workspace = true }-parquet = "58"rand = { workspace = true }rdkafka = { workspace = true }+serde = { workspace = true }serde_json = { workspace = true }sqlx = { workspace = true }thiserror = { workspace = true }@@ -50,6 +47,5 @@ url = { workspace = true }[dev-dependencies]aes-gcm = "0.10"base64 = { workspace = true }-bytes = { workspace = true }serde_json = { workspace = true }tokio = { workspace = true, features = ["test-util"] }diff --git a/rust/ai-research/replay-recorder-v4/src/batch.rs b/rust/ai-research/replay-recorder-v4/src/batch.rsindex ec0b29047134..995d6c137a91 100644--- a/rust/ai-research/replay-recorder-v4/src/batch.rs+++ b/rust/ai-research/replay-recorder-v4/src/batch.rs@@ -32,6 +32,7 @@ pub struct Tp {/// An admitted message, not decoded yet.pub struct Raw {pub offset: i64,+ pub captured_at_ms: Option<i64>,pub payload: Vec<u8>,pub content_encoding: Option<String>,}@@ -95,6 +96,8 @@ pub struct Block {pub remote_urls: Vec<String>,/// Set when the inspector stopped short of naming every remote URL, so that the block must not be stored.pub url_limit: Option<UrlLimit>,+ /// The earliest Kafka timestamp of the block's messages, for the capture watermark of the index topic.+ pub captured_at_ms: Option<i64>,}/// The decoded `cv` strings one flush keeps, on top of the decoded payloads that the flush size counts.@@ -421,6 +424,7 @@ fn inspect_session(let mut decoded = Vec::with_capacity(parts.len());let mut session_spent = 0;for (origin, raws) in parts {+ let captured_at_ms = raws.iter().filter_map(|raw| raw.captured_at_ms).min();let mut messages = Vec::with_capacity(raws.len());for raw in raws {// A growing list can hold twice its events, so the reader stops at half of what the budget has room for.@@ -472,7 +476,7 @@ fn inspect_session(}counter!("ai_research_v4_recorder_messages_total", "outcome" => "recorded").increment(messages.len() as u64);- decoded.push((origin, messages));+ decoded.push((origin, messages, captured_at_ms));}// A session can have messages on the main and the overflow topic in one flush. The inspector takes them in replay// order, so that a mutation never comes before the nodes it changes. Each still counts toward its partition's block.@@ -480,7 +484,7 @@ fn inspect_session(decoded.iter().map(|_| BlockInspection::default()).collect();let times: Vec<Vec<f64>> = decoded.iter()- .map(|(_, messages)| messages.iter().map(replay_time).collect())+ .map(|(_, messages, _)| messages.iter().map(replay_time).collect()).collect();let mut next = vec![0; decoded.len()];while let Some(part) = (0..decoded.len())@@ -493,7 +497,7 @@ fn inspect_session(let blocks = decoded.into_iter().zip(inspections)- .map(|((origin, messages), block)| {+ .map(|((origin, messages, captured_at_ms), block)| {let inspected = block.finish();Block {session: session.clone(),@@ -504,6 +508,7 @@ fn inspect_session(image_urls: inspected.image_urls,remote_urls: inspected.remote_urls,url_limit: inspected.url_limit,+ captured_at_ms,}}).collect();@@ -578,6 +583,7 @@ mod tests {let inner = json!({"event": "$snapshot_items", "properties": {"$session_id": session, "$window_id": "w", "$snapshot_source": "web", "$snapshot_items": items}});Raw {offset,+ captured_at_ms: None,payload: json!({"distinct_id": "d", "data": inner.to_string()}).to_string().into_bytes(),@@ -796,6 +802,7 @@ mod tests {async fn a_session_that_decodes_nothing_keeps_its_state_as_it_was() {let unreadable = |offset: i64| Raw {offset,+ captured_at_ms: None,payload: b"not json".to_vec(),content_encoding: None,};diff --git a/rust/ai-research/replay-recorder-v4/src/config.rs b/rust/ai-research/replay-recorder-v4/src/config.rsindex 45c685b89633..e7e90f5926fd 100644--- a/rust/ai-research/replay-recorder-v4/src/config.rs+++ b/rust/ai-research/replay-recorder-v4/src/config.rs@@ -127,6 +127,10 @@ pub struct Config {)]pub image_fetch_topic: String,+ /// Where the block and image index rows go for the Parquet sink.+ #[envconfig(from = "AI_RESEARCH_V4_INDEX_TOPIC", default = "ai_research_v4_index")]+ pub index_topic: String,+/// A fetch job not delivered in this time fails its flush. It is well under the flush timeout.#[envconfig(from = "AI_RESEARCH_V4_IMAGE_FETCH_DELIVERY_TIMEOUT_MS",@@ -134,6 +138,11 @@ pub struct Config {)]pub image_fetch_delivery_timeout_ms: u32,+ /// An index row not delivered in this time is lost and counted. The flush does not wait for it, so it can be long+ /// enough to ride out a broker slowdown.+ #[envconfig(from = "AI_RESEARCH_V4_INDEX_DELIVERY_TIMEOUT_MS", default = "300000")]+ pub index_delivery_timeout_ms: u32,+#[envconfig(default = "9104")]pub metrics_port: u16,}@@ -205,6 +214,7 @@ impl Config {("AI_RESEARCH_V4_RRWEB_BUCKET", &self.rrweb_bucket),("AI_RESEARCH_V4_IMAGES_BUCKET", &self.images_bucket),("AI_RESEARCH_V4_IMAGE_FETCH_TOPIC", &self.image_fetch_topic),+ ("AI_RESEARCH_V4_INDEX_TOPIC", &self.index_topic),]);}let missing: Vec<&str> = requireddiff --git a/rust/ai-research/replay-recorder-v4/src/consumer.rs b/rust/ai-research/replay-recorder-v4/src/consumer.rsindex 43ef92e747cb..e117d139674e 100644--- a/rust/ai-research/replay-recorder-v4/src/consumer.rs+++ b/rust/ai-research/replay-recorder-v4/src/consumer.rs@@ -19,7 +19,7 @@ use rdkafka::{ClientConfig, ClientContext, Offset, TopicPartitionList};use crate::batch::{Batch, Block, Inspectors, Raw, SessionKey, Tp};use crate::config::{retry_backoff, FLUSH_DEADLINE, MAX_POLL_INTERVAL};-use crate::flush::{write_flush, FlushWork, Writer};+use crate::flush::{write_flush, FlushWork, Writer, PRODUCER_FLUSH_ON_SHUTDOWN};use crate::limits::{Restrictions, SessionLimiter};use crate::process::{admit, sampled, Admitted, Headers, Skip};use crate::teams::TeamCache;@@ -419,6 +419,20 @@ pub async fn run(sources: Sources,writer: Option<Writer>,handle: Handle,+) {+ consume(&consumer, &settings, &sources, writer.as_ref(), &handle).await;+ // The flushes do not wait for their index rows, so the producer can still hold some.+ if let Some(writer) = &writer {+ writer.flush_producers(PRODUCER_FLUSH_ON_SHUTDOWN).await;+ }+}++async fn consume(+ consumer: &RecorderConsumer,+ settings: &Settings,+ sources: &Sources,+ writer: Option<&Writer>,+ handle: &Handle,) {let shutdown = handle.shutdown_signal();tokio::pin!(shutdown);@@ -444,16 +458,16 @@ pub async fn run(None => break,}}- handle_messages(&messages, &consumer, &settings, &sources, &mut batch).await;+ handle_messages(&messages, consumer, settings, sources, &mut batch).await;drop(messages);if let Some(error) = failed {- if receive_failed(&consumer, &handle, error) {+ if receive_failed(consumer, handle, error) {return;}}}Err(error) => {- if receive_failed(&consumer, &handle, error) {+ if receive_failed(consumer, handle, error) {return;}}@@ -469,10 +483,10 @@ pub async fn run(let work = flush(std::mem::take(&mut batch),&mut inspectors,- &consumer,- &settings,- writer.as_ref(),- &handle,+ consumer,+ settings,+ writer,+ handle,settings.flush_retries,);tokio::pin!(work);@@ -500,10 +514,10 @@ pub async fn run(flush(batch,&mut inspectors,- &consumer,- &settings,- writer.as_ref(),- &handle,+ consumer,+ settings,+ writer,+ handle,0,),)@@ -588,6 +602,7 @@ async fn handle_messages(.increment(1);let raw = Raw {offset: message.offset(),+ captured_at_ms: message.timestamp().to_millis(),payload: message.payload().unwrap_or_default().to_vec(),content_encoding: header(message, "content-encoding").map(str::to_owned),};@@ -826,6 +841,7 @@ mod tests {image_urls: Vec::new(),remote_urls: Vec::new(),url_limit: None,+ captured_at_ms: None,})};let offsets = HashMap::from([(tp(0), (1, 5)), (tp(1), (1, 5)), (tp(2), (1, 5))]);diff --git a/rust/ai-research/replay-recorder-v4/src/flush.rs b/rust/ai-research/replay-recorder-v4/src/flush.rsindex 1261a3267ad0..f930b118644e 100644--- a/rust/ai-research/replay-recorder-v4/src/flush.rs+++ b/rust/ai-research/replay-recorder-v4/src/flush.rs@@ -1,5 +1,8 @@//! One attempt to store a flush's blocks. It writes nothing that a retry cannot write again: object names are unique,//! and the block index rows name their attempt.+//!+//! The index rows go to the index topic, and the Parquet sink writes them to S3. The flush does not wait for their+//! delivery: the data is stored by then, and a lost row is counted.use std::collections::{BTreeMap, BTreeSet, HashMap};use std::sync::{Arc, Mutex, OnceLock, PoisonError};@@ -10,7 +13,7 @@ use common_kafka::kafka_producer::KafkaContext;use futures::stream::{self, StreamExt};use metrics::counter;use rdkafka::message::{Header, OwnedHeaders};-use rdkafka::producer::{FutureProducer, FutureRecord};+use rdkafka::producer::{FutureProducer, FutureRecord, Producer};use rdkafka::util::Timeout;use replay_core_v4::inspect::stats::BlockStats;use replay_core_v4::keys::batch::{FlushKeys, KeyStore, SessionId};@@ -20,11 +23,12 @@ use replay_core_v4::keys::table::{AwsKeyTable, CachedKeyTable};use tokio::sync::Semaphore;use crate::batch::Block;+use crate::index::{Index, IndexMessage};use crate::store::BlobStore;use crate::urls::{fetch_jobs, records};use crate::write::{- compress_block, new_images, object_key, seal_blocking, seal_compressed, session_keyed_frames,- write_block_index, write_blocks, write_images, CompressedBlock, FlushAttempt, WriteError,+ block_index_rows, compress_block, new_images, object_key, seal_blocking, seal_compressed,+ session_keyed_frames, write_blocks, write_images, CompressedBlock, FlushAttempt, WriteError,};/// How long a fetch job may wait for room in the producer queue.@@ -33,6 +37,8 @@ const ENQUEUE_TIMEOUT: Duration = Duration::from_secs(10);/// Fetch job records sent at once, so a slow broker slows the flush instead of filling the producer queue.const SENDS_IN_FLIGHT: usize = 64;+pub const PRODUCER_FLUSH_ON_SHUTDOWN: Duration = Duration::from_secs(5);+/// What a flush needs to store blocks. Without it the recorder runs dry.pub struct Writer {pub keys: KeyStore<CachedKeyTable<AwsKeyTable>, AwsKms>,@@ -41,6 +47,9 @@ pub struct Writer {pub images_bucket: String,pub producer: FutureProducer<KafkaContext>,pub image_fetch_topic: String,+ /// The flush does not wait for index rows, so they get their own producer with a delivery timeout of minutes.+ pub index_producer: FutureProducer<KafkaContext>,+ pub index_topic: String,/// Remote image references this process sent to the fetcher, by UTC day. A reference goes out again each day, so/// the fetcher sees it within every refresh interval and keeps the image as long as the blocks that reference it.pub sent_urls: moka::sync::Cache<(String, String), ()>,@@ -54,6 +63,33 @@ pub enum FlushError {Keys(#[from] replay_core_v4::keys::batch::KeyError),#[error(transparent)]Write(#[from] WriteError),+ #[error("encoding an index row failed: {0}")]+ Encode(#[from] serde_json::Error),+}++impl Writer {+ /// Waits, up to `timeout` for both at once, for the producers to deliver what they hold, so that shutdown loses+ /// no index rows.+ pub async fn flush_producers(&self, timeout: Duration) {+ futures::future::join(+ flush_producer("image fetch", &self.producer, timeout),+ flush_producer("index", &self.index_producer, timeout),+ )+ .await;+ }+}++async fn flush_producer(name: &str, producer: &FutureProducer<KafkaContext>, timeout: Duration) {+ let flushing = producer.clone();+ let flushed =+ tokio::task::spawn_blocking(move || flushing.flush(Timeout::After(timeout))).await;+ match flushed {+ Ok(Ok(())) => {}+ Ok(Err(error)) => {+ tracing::warn!(producer = name, %error, in_queue = producer.in_flight_count(), "the producer did not deliver everything on shutdown")+ }+ Err(error) => tracing::warn!(producer = name, %error, "flushing the producer failed"),+ }}/// What the attempts of one flush share: the keys, the write day they are for, and each block compressed. A retry then@@ -215,20 +251,19 @@ pub async fn write_flush();let session_frames = session_keyed_frames(&sealed);- let rows = write_images(+ let image_rows = write_images(new_images(&mut sealed),&keys,&day,month,&writer.images_bucket,- &writer.rrweb_bucket,&writer.store,now,concurrency,&writer.seal_permits,).await?;- counter!("ai_research_v4_recorder_stored_images_total").increment(rows.len() as u64);+ counter!("ai_research_v4_recorder_stored_images_total").increment(image_rows.len() as u64);// What sealing images under session keys would store, to compare with the frames above.counter!("ai_research_v4_recorder_session_keyed_image_frames_total").increment(session_frames);@@ -246,16 +281,17 @@ pub async fn write_flush(.zip(&written).map(|(&index, written)| (&*blocks[index], &stats[index], written)).collect();- write_block_index(- &entries,- &keys,- &attempt,- &writer.rrweb_bucket,- writer.store.as_ref(),- now,- )- .await?;+ // An image row covers data from every block that holds the image, so it takes the flush's earliest capture.+ let flush_captured_at_ms = blocks.iter().filter_map(|b| b.captured_at_ms).min();+ let mut index = image_rows+ .iter()+ .map(|row| IndexMessage::image(row, flush_captured_at_ms))+ .collect::<Result<Vec<_>, _>>()?;+ for row in block_index_rows(&entries, &keys, &attempt)? {+ index.push(IndexMessage::block(&row)?);+ }produce_jobs(writer, jobs, &day, now.timestamp_millis()).await;+ send_index(&writer.index_producer, &writer.index_topic, &day, index);Ok(())}@@ -318,10 +354,86 @@ async fn produce_jobs(metrics::gauge!("ai_research_v4_recorder_sent_urls").set(writer.sent_urls.entry_count() as f64);}+/// Hands the index rows to the producer and returns without waiting for their delivery. A task counts the outcomes.+/// `write_day` is the `dt=` partition of the blocks and packs the rows point at, so the sink files them with those.+pub fn send_index(+ producer: &FutureProducer<KafkaContext>,+ topic: &str,+ write_day: &str,+ messages: Vec<IndexMessage>,+) {+ let mut deliveries = Vec::with_capacity(messages.len());+ let mut failed = Failures::default();+ for message in &messages {+ let captured_at = message.captured_at_ms.map(|ms| ms.to_string());+ let mut headers = OwnedHeaders::new()+ .insert(Header {+ key: "ml-index",+ value: Some(message.index.as_str()),+ })+ .insert(Header {+ key: "ml-write-day",+ value: Some(write_day),+ });+ if let Some(captured_at) = &captured_at {+ headers = headers.insert(Header {+ key: "capture-timestamp-ms",+ value: Some(captured_at),+ });+ }+ let record = FutureRecord::to(topic)+ .key(&message.key)+ .payload(&message.payload)+ .headers(headers);+ match producer.send_result(record) {+ Ok(delivery) => deliveries.push((message.index, delivery)),+ Err((error, _)) => failed.add(message.index, error.to_string()),+ }+ }+ tokio::spawn(async move {+ for (index, delivery) in deliveries {+ match delivery.await {+ Ok(Ok(_)) => counter!("ai_research_v4_recorder_index_rows_total", "index" => index.as_str(), "outcome" => "delivered").increment(1),+ Ok(Err((error, _))) => failed.add(index, error.to_string()),+ Err(_) => failed.add(index, "the producer dropped the row".to_owned()),+ }+ }+ failed.report();+ });+}++/// The index rows of one flush that did not reach the topic. They are logged once per flush, not once per row.+#[derive(Default)]+struct Failures {+ count: u64,+ first: Option<String>,+}++impl Failures {+ fn add(&mut self, index: Index, error: String) {+ counter!("ai_research_v4_recorder_index_rows_total", "index" => index.as_str(), "outcome" => "failed").increment(1);+ self.count += 1;+ self.first.get_or_insert(error);+ }++ fn report(self) {+ if let Some(error) = self.first {+ tracing::warn!(rows = self.count, %error, "index rows were not delivered");+ }+ }+}+#[cfg(test)]mod tests {+ use common_kafka::test::create_mock_kafka;+ use rdkafka::consumer::{BaseConsumer, Consumer};+ use rdkafka::message::{Headers, Message};+ use rdkafka::{ClientConfig, Offset, TopicPartitionList};+use super::*;+ const TOPIC: &str = "ai_research_v4_index";+#[test]fn a_retry_reuses_the_compressed_block() {let work = FlushWork::default();@@ -330,4 +442,91 @@ mod tests {let again = work.compressed(1, || panic!("compressed a second time"));assert!(matches!(again, Err(WriteError::Seal(e)) if e == "sealing failed: once"));}++ fn message(index: Index, key: &str, captured_at_ms: Option<i64>) -> IndexMessage {+ IndexMessage {+ index,+ key: key.to_owned(),+ payload: format!(r#"{{"row":"{key}"}}"#).into_bytes(),+ captured_at_ms,+ }+ }++ #[tokio::test]+ async fn sends_one_record_per_row_with_its_index_and_write_day() {+ let (cluster, producer) = create_mock_kafka().await;+ cluster.create_topic(TOPIC, 1, 1).unwrap();+ send_index(+ &producer,+ TOPIC,+ "2026-10-15",+ vec![+ message(Index::Blocks, "session-1", Some(1_000)),+ message(Index::Images, "image:v4:7:2026-10:abc", None),+ ],+ );+ producer+ .flush(Timeout::After(Duration::from_secs(5)))+ .unwrap();++ let consumer: BaseConsumer = ClientConfig::new()+ .set("bootstrap.servers", cluster.bootstrap_servers())+ .set("group.id", "test")+ .create()+ .unwrap();+ let mut assignment = TopicPartitionList::new();+ assignment+ .add_partition_offset(TOPIC, 0, Offset::Beginning)+ .unwrap();+ consumer.assign(&assignment).unwrap();+ let mut received = Vec::new();+ while received.len() < 2 {+ let record = consumer+ .poll(Duration::from_secs(5))+ .expect("a record before the timeout")+ .unwrap();+ let headers: Vec<(String, String)> = record+ .headers()+ .map(|headers| {+ headers+ .iter()+ .map(|h| {+ (+ h.key.to_owned(),+ String::from_utf8(h.value.unwrap().to_vec()).unwrap(),+ )+ })+ .collect()+ })+ .unwrap_or_default();+ received.push((+ String::from_utf8(record.key().unwrap().to_vec()).unwrap(),+ String::from_utf8(record.payload().unwrap().to_vec()).unwrap(),+ headers,+ ));+ }+ let header = |key: &str, value: &str| (key.to_owned(), value.to_owned());+ assert_eq!(+ received,+ [+ (+ "session-1".to_owned(),+ r#"{"row":"session-1"}"#.to_owned(),+ vec![+ header("ml-index", "blocks"),+ header("ml-write-day", "2026-10-15"),+ header("capture-timestamp-ms", "1000"),+ ]+ ),+ (+ "image:v4:7:2026-10:abc".to_owned(),+ r#"{"row":"image:v4:7:2026-10:abc"}"#.to_owned(),+ vec![+ header("ml-index", "images"),+ header("ml-write-day", "2026-10-15"),+ ]+ ),+ ]+ );+ }}diff --git a/rust/ai-research/replay-recorder-v4/src/index.rs b/rust/ai-research/replay-recorder-v4/src/index.rsindex e0bbfd06d231..675742df79a3 100644--- a/rust/ai-research/replay-recorder-v4/src/index.rs+++ b/rust/ai-research/replay-recorder-v4/src/index.rs@@ -1,6 +1,7 @@-//! The v4 index files: Parquet in `rrweb-v4/index/`. Column names are the ones in `replay-data.md`.-//!-//! Integer columns are signed, the types parquetjs writes, so a parquetjs writer can produce the same files.+//! The v4 index rows. The recorder sends each row as one JSON message to the index topic, and the Parquet sink writes+//! them to `rrweb-v4/index/`. The JSON keys are the Parquet column names in `replay-data.md`. Integer values fit the+//! signed column types parquetjs writes, and an INT64 value stays in the range a JavaScript number holds exactly,+//! which the sink checks: a value past either range saturates.//!//! A flush can store its blocks and index rows and then fail before it commits its Kafka offsets. The next attempt//! for the partition, in this process or after a restart, starts again at the same committed offset and writes the@@ -15,324 +16,115 @@//! flush is whole when that many distinct `flush_row_index` values are present. Check this before any filter on//! sessions or teams.-use std::sync::Arc;--use arrow_array::builder::{Int32Builder, Int64Builder, ListBuilder, StringBuilder, StructBuilder};-use arrow_array::{ArrayRef, BooleanArray, Int32Array, Int64Array, RecordBatch, StringArray};-use arrow_schema::{ArrowError, DataType, Field, Fields, Schema};use replay_core_v4::inspect::session::Moment;use replay_core_v4::inspect::stats::BlockStats;+use serde::ser::{SerializeMap, Serializer};+use serde::Serialize;-use crate::batch::Origin;-use crate::write::{FlushAttempt, Written};-use parquet::arrow::ArrowWriter;-use parquet::basic::Compression;-use parquet::errors::ParquetError;-use parquet::file::properties::WriterProperties;+/// `Number.MAX_SAFE_INTEGER`. The sink refuses a row with an integer past it, and that row's flush is then never whole.+const MAX_SAFE_INTEGER: i64 = (1 << 53) - 1;-#[derive(Debug, thiserror::Error)]-pub enum IndexError {- #[error(transparent)]- Arrow(#[from] ArrowError),- #[error(transparent)]- Parquet(#[from] ParquetError),+fn int64(value: u64) -> i64 {+ i64::try_from(value).map_or(MAX_SAFE_INTEGER, safe)}-/// Writes one Parquet file, snappy-compressed as the v3 catalog is.-pub fn write_parquet(batch: &RecordBatch) -> Result<Vec<u8>, IndexError> {- let properties = WriterProperties::builder()- .set_compression(Compression::SNAPPY)- .build();- let mut out = Vec::new();- let mut writer = ArrowWriter::try_new(&mut out, batch.schema(), Some(properties))?;- writer.write(batch)?;- writer.close()?;- Ok(out)+fn safe(value: i64) -> i64 {+ value.clamp(-MAX_SAFE_INTEGER, MAX_SAFE_INTEGER)}-/// A count or size as its signed column. Real values fit, and a larger one saturates.-fn int64(value: u64) -> i64 {- i64::try_from(value).unwrap_or(i64::MAX)+fn serialize_safe<S: Serializer>(value: &i64, serializer: S) -> Result<S::Ok, S::Error> {+ serializer.serialize_i64(safe(*value))}-/// A count as its signed column. Real values fit, and a larger one saturates.fn int32(value: u32) -> i32 {i32::try_from(value).unwrap_or(i32::MAX)}-/// Where one inline image sits in a pack in `images-v4`.-#[derive(Debug, Clone, PartialEq, Eq)]-pub struct ImageIndexRow {- pub team_id: i64,- pub key_month: String,- pub image_ref: String,- pub pack_s3_key: String,- /// Inclusive byte range of the sealed image in the pack.- pub byte_start: u64,- pub byte_end: u64,- pub mime: String,- pub byte_size: u64,-}--pub fn image_index(rows: &[ImageIndexRow]) -> Result<RecordBatch, IndexError> {- let schema = Schema::new(vec![- Field::new("team_id", DataType::Int64, false),- Field::new("key_month", DataType::Utf8, false),- Field::new("image_ref", DataType::Utf8, false),- Field::new("pack_s3_key", DataType::Utf8, false),- Field::new("byte_start", DataType::Int64, false),- Field::new("byte_end", DataType::Int64, false),- Field::new("mime", DataType::Utf8, false),- Field::new("byte_size", DataType::Int64, false),- ]);- let columns: Vec<ArrayRef> = vec![- Arc::new(Int64Array::from_iter_values(rows.iter().map(|r| r.team_id))),- Arc::new(StringArray::from_iter_values(- rows.iter().map(|r| &r.key_month),- )),- Arc::new(StringArray::from_iter_values(- rows.iter().map(|r| &r.image_ref),- )),- Arc::new(StringArray::from_iter_values(- rows.iter().map(|r| &r.pack_s3_key),- )),- Arc::new(Int64Array::from_iter_values(- rows.iter().map(|r| int64(r.byte_start)),- )),- Arc::new(Int64Array::from_iter_values(- rows.iter().map(|r| int64(r.byte_end)),- )),- Arc::new(StringArray::from_iter_values(rows.iter().map(|r| &r.mime))),- Arc::new(Int64Array::from_iter_values(- rows.iter().map(|r| int64(r.byte_size)),- )),- ];- Ok(RecordBatch::try_new(Arc::new(schema), columns)?)-}--/// One block's index row: where it is, what the inspector found, and its sealed URLs.-pub struct BlockIndexRow<'a> {- pub written: &'a Written,- pub origin: &'a Origin,- pub attempt: &'a FlushAttempt,- pub stats: &'a BlockStats,- pub moments: &'a [Moment],- pub snapshot_source: Option<&'a str>,- pub snapshot_library: Option<&'a str>,- /// The block's visited URLs as a v4 JSON envelope under the session key.- pub urls: Option<String>,- /// The block rows this flush writes for the row's partition, and the row's place among them. With `flush_id`,- /// `topic` and `partition`, the place names the row across retries of the same publish.- pub flush_row_count: u32,- pub flush_row_index: u32,-}--fn moment_fields() -> Fields {- Fields::from(vec![- Field::new("kind", DataType::Utf8, false),- Field::new("window_id", DataType::Utf8, false),- Field::new("ts_ms", DataType::Int64, false),- Field::new("event_index", DataType::Int32, false),- ])+fn serialize_int64<S: Serializer>(value: &u64, serializer: S) -> Result<S::Ok, S::Error> {+ serializer.serialize_i64(int64(*value))}-fn moments_column(rows: &[BlockIndexRow<'_>]) -> ArrayRef {- let fields = moment_fields();- let mut builder = ListBuilder::new(StructBuilder::from_fields(fields.clone(), rows.len()))- .with_field(Field::new("item", DataType::Struct(fields), false));- for row in rows {- let entries = builder.values();- for moment in row.moments {- entries- .field_builder::<StringBuilder>(0)- .expect("kind")- .append_value(moment.kind);- entries- .field_builder::<StringBuilder>(1)- .expect("window_id")- .append_value(&moment.window_id);- entries- .field_builder::<Int64Builder>(2)- .expect("ts_ms")- .append_value(moment.ts_ms);- entries- .field_builder::<Int32Builder>(3)- .expect("event_index")- .append_value(int32(moment.event_index));- entries.append(true);- }- builder.append(true);- }- Arc::new(builder.finish())+fn serialize_int32<S: Serializer>(value: &u32, serializer: S) -> Result<S::Ok, S::Error> {+ serializer.serialize_i32(int32(*value))}-/// A list of strings per row, with no null lists or items.-fn strings_column(rows: &[BlockIndexRow<'_>], values: fn(&BlockStats) -> &[String]) -> ArrayRef {- let mut builder = ListBuilder::new(StringBuilder::new()).with_field(Field::new(- "item",- DataType::Utf8,- false,- ));- for row in rows {- for value in values(row.stats) {- builder.values().append_value(value);- }- builder.append(true);- }- Arc::new(builder.finish())-}--pub fn block_index(rows: &[BlockIndexRow<'_>]) -> Result<RecordBatch, IndexError> {- let mut fields = Vec::new();- let mut columns: Vec<ArrayRef> = Vec::new();- let mut push = |name: &str, data_type: DataType, nullable: bool, column: ArrayRef| {- fields.push(Field::new(name, data_type, nullable));- columns.push(column);- };- macro_rules! utf8 {- ($name:literal, $value:expr) => {- push(- $name,- DataType::Utf8,- false,- Arc::new(StringArray::from_iter_values(rows.iter().map($value))),- )- };- }- macro_rules! optional_utf8 {- ($name:literal, $value:expr) => {- push(- $name,- DataType::Utf8,- true,- Arc::new(rows.iter().map($value).collect::<StringArray>()),- )- };- }- macro_rules! stat {- ($array:ident, $data_type:expr, $convert:expr, $($field:ident),*) => {- $( push(stringify!($field), $data_type, false, Arc::new($array::from_iter_values(rows.iter().map(|r| $convert(r.stats.$field))))); )*- };- }- macro_rules! flag {- ($($field:ident),*) => {- $( push(stringify!($field), DataType::Boolean, false, Arc::new(rows.iter().map(|r| Some(r.stats.$field)).collect::<BooleanArray>())); )*+fn serialize_stats<S: Serializer>(stats: &&BlockStats, serializer: S) -> Result<S::Ok, S::Error> {+ // Destructured in full, so a new stats field does not compile until it has a column type here.+ let BlockStats {+ first_ts_ms,+ last_ts_ms,+ event_count,+ byte_size,+ host,+ full_snapshots,+ events_before_first_full_snapshot,+ orphan_mutations,+ max_gap_ms,+ dom_nodes,+ dom_text_chars,+ remote_stylesheets,+ inline_stylesheets,+ iframes,+ canvases,+ videos,+ shadow_roots,+ cold_start,+ node_cap_reached,+ active_ms,+ clicks,+ rage_clicks,+ dead_clicks,+ keypresses,+ scrolls,+ mouse_moves,+ touches,+ selections,+ media_events,+ viewport_resizes,+ navigations,+ console_errors,+ console_warnings,+ network_requests,+ network_failures,+ viewport_width,+ viewport_height,+ viewport_ts_ms,+ touch_input,+ images_extracted,+ image_bytes,+ images_inline,+ image_urls,+ written_jsonl_bytes,+ max_written_event_bytes,+ decode_failures,+ moments_truncated,+ mouse_interactions,+ touch_moves,+ url_fingerprints,+ window_ids,+ } = *stats;+ let mut map = serializer.serialize_map(None)?;+ macro_rules! columns {+ ($convert:expr => $($field:ident),*) => {+ $( map.serialize_entry(stringify!($field), &$convert($field))?; )*};}-- push(- "team_id",- DataType::Int64,- false,- Arc::new(Int64Array::from_iter_values(- rows.iter().map(|r| r.written.session.team_id),- )),- );- utf8!("session_id", |r| r.written.session.session_id.clone());- utf8!("key_month", |r| r.written.key_month.clone());- utf8!("block_s3_key", |r| r.written.object_key.clone());- utf8!("topic", |r| r.origin.tp.topic);- push(- "partition",- DataType::Int32,- false,- Arc::new(Int32Array::from_iter_values(- rows.iter().map(|r| r.origin.tp.partition),- )),- );- push(- "start_offset",- DataType::Int64,- false,- Arc::new(Int64Array::from_iter_values(- rows.iter().map(|r| r.origin.start_offset),- )),- );- push(- "end_offset",- DataType::Int64,- false,- Arc::new(Int64Array::from_iter_values(- rows.iter().map(|r| r.origin.end_offset),- )),- );- utf8!("flush_id", |r| r.attempt.flush_id.clone());- push(- "flushed_at_ms",- DataType::Int64,- false,- Arc::new(Int64Array::from_iter_values(- rows.iter().map(|r| r.attempt.flushed_at_ms),- )),- );- push(- "flush_end_offset",- DataType::Int64,- false,- Arc::new(Int64Array::from_iter_values(- rows.iter().map(|r| r.origin.flush_end_offset),- )),- );- push(- "flush_row_count",- DataType::Int32,- false,- Arc::new(Int32Array::from_iter_values(- rows.iter().map(|r| int32(r.flush_row_count)),- )),- );- push(- "flush_row_index",- DataType::Int32,- false,- Arc::new(Int32Array::from_iter_values(- rows.iter().map(|r| int32(r.flush_row_index)),- )),- );- push(- "block_byte_start",- DataType::Int64,- false,- Arc::new(Int64Array::from_iter_values(- rows.iter().map(|r| int64(r.written.byte_start)),- )),- );- push(- "block_byte_end",- DataType::Int64,- false,- Arc::new(Int64Array::from_iter_values(- rows.iter().map(|r| int64(r.written.byte_end)),- )),- );- push(- "first_ts_ms",- DataType::Int64,- true,- Arc::new(- rows.iter()- .map(|r| r.stats.first_ts_ms)- .collect::<Int64Array>(),- ),- );- push(- "last_ts_ms",- DataType::Int64,- true,- Arc::new(- rows.iter()- .map(|r| r.stats.last_ts_ms)- .collect::<Int64Array>(),- ),+ columns!(|v: &Option<i64>| v.map(safe) => first_ts_ms, last_ts_ms, viewport_ts_ms);+ columns!(|v: &i64| safe(*v) => max_gap_ms, active_ms);+ columns!(+ |v: &u64| int64(*v) =>+ event_count,+ byte_size,+ dom_text_chars,+ image_bytes,+ written_jsonl_bytes,+ max_written_event_bytes);- stat!(Int64Array, DataType::Int64, int64, event_count, byte_size);- optional_utf8!("snapshot_source", |r| r.snapshot_source);- optional_utf8!("snapshot_library", |r| r.snapshot_library);- optional_utf8!("host", |r| r.stats.host.as_deref());- stat!(- Int32Array,- DataType::Int32,- int32,+ columns!(|v: &bool| *v => cold_start, node_cap_reached, moments_truncated, touch_input);+ columns!(|v: &Vec<String>| v.clone() => url_fingerprints, window_ids);+ columns!(|v: &Option<u32>| v.map(int32) => viewport_width, viewport_height);+ columns!(+ |v: &u32| int32(*v) =>full_snapshots,events_before_first_full_snapshot,orphan_mutations,@@ -343,21 +135,7 @@ pub fn block_index(rows: &[BlockIndexRow<'_>]) -> Result<RecordBatch, IndexErroriframes,canvases,videos,- shadow_roots- );- stat!(- Int64Array,- DataType::Int64,- i64::from,- max_gap_ms,- active_ms- );- stat!(Int64Array, DataType::Int64, int64, dom_text_chars);- flag!(cold_start, node_cap_reached, moments_truncated, touch_input);- stat!(- Int32Array,- DataType::Int32,- int32,+ shadow_roots,clicks,mouse_interactions,rage_clicks,@@ -379,72 +157,355 @@ pub fn block_index(rows: &[BlockIndexRow<'_>]) -> Result<RecordBatch, IndexErrorimages_inline,image_urls);- stat!(- Int64Array,- DataType::Int64,- int64,- image_bytes,- written_jsonl_bytes,- max_written_event_bytes- );- push(- "viewport_width",- DataType::Int32,- true,- Arc::new(- rows.iter()- .map(|r| r.stats.viewport_width.map(int32))- .collect::<Int32Array>(),- ),- );- push(- "viewport_height",- DataType::Int32,- true,- Arc::new(- rows.iter()- .map(|r| r.stats.viewport_height.map(int32))- .collect::<Int32Array>(),- ),- );- push(- "viewport_ts_ms",- DataType::Int64,- true,- Arc::new(- rows.iter()- .map(|r| r.stats.viewport_ts_ms)- .collect::<Int64Array>(),- ),- );- let moments = moments_column(rows);- push("moments", moments.data_type().clone(), false, moments);- let fingerprints = strings_column(rows, |s| &s.url_fingerprints);- push(- "url_fingerprints",- fingerprints.data_type().clone(),- false,- fingerprints,- );- let windows = strings_column(rows, |s| &s.window_ids);- push("window_ids", windows.data_type().clone(), false, windows);- optional_utf8!("urls", |r| r.urls.as_deref());- Ok(RecordBatch::try_new(- Arc::new(Schema::new(fields)),- columns,- )?)+ map.serialize_entry("host", host)?;+ map.end()+}++fn serialize_moments<S: Serializer>(moments: &&[Moment], serializer: S) -> Result<S::Ok, S::Error> {+ #[derive(Serialize)]+ struct Entry<'a> {+ kind: &'a str,+ window_id: &'a str,+ ts_ms: i64,+ event_index: i32,+ }+ serializer.collect_seq(moments.iter().map(|moment| Entry {+ kind: moment.kind,+ window_id: &moment.window_id,+ ts_ms: safe(moment.ts_ms),+ event_index: int32(moment.event_index),+ }))+}++/// Where one inline image sits in a pack in `images-v4`.+#[derive(Debug, Clone, PartialEq, Eq, Serialize)]+pub struct ImageIndexRow {+ #[serde(serialize_with = "serialize_safe")]+ pub team_id: i64,+ pub key_month: String,+ pub image_ref: String,+ pub pack_s3_key: String,+ /// Inclusive byte range of the sealed image in the pack.+ #[serde(serialize_with = "serialize_int64")]+ pub byte_start: u64,+ #[serde(serialize_with = "serialize_int64")]+ pub byte_end: u64,+ pub mime: String,+ #[serde(serialize_with = "serialize_int64")]+ pub byte_size: u64,+}++/// One block's index row: where it is, what the inspector found, and its sealed URLs.+#[derive(Debug, Serialize)]+pub struct BlockIndexRow<'a> {+ #[serde(serialize_with = "serialize_safe")]+ pub team_id: i64,+ pub session_id: &'a str,+ pub key_month: &'a str,+ pub block_s3_key: &'a str,+ pub topic: &'static str,+ pub partition: i32,+ #[serde(serialize_with = "serialize_safe")]+ pub start_offset: i64,+ #[serde(serialize_with = "serialize_safe")]+ pub end_offset: i64,+ /// The last offset the flush covers on the partition, a message that gave no row included.+ #[serde(serialize_with = "serialize_safe")]+ pub flush_end_offset: i64,+ /// The block rows this flush writes for the partition, and this row's place among them. With `flush_id`,+ /// `topic` and `partition`, the place names the row across retries of the same publish.+ #[serde(serialize_with = "serialize_int32")]+ pub flush_row_count: u32,+ #[serde(serialize_with = "serialize_int32")]+ pub flush_row_index: u32,+ pub flush_id: &'a str,+ #[serde(serialize_with = "serialize_safe")]+ pub flushed_at_ms: i64,+ /// Inclusive byte range of the sealed frame in the block object.+ #[serde(serialize_with = "serialize_int64")]+ pub block_byte_start: u64,+ #[serde(serialize_with = "serialize_int64")]+ pub block_byte_end: u64,+ #[serde(flatten, serialize_with = "serialize_stats")]+ pub stats: &'a BlockStats,+ pub snapshot_source: Option<&'a str>,+ pub snapshot_library: Option<&'a str>,+ #[serde(serialize_with = "serialize_moments")]+ pub moments: &'a [Moment],+ /// The block's visited URLs as a v4 JSON envelope under the session key.+ pub urls: Option<String>,+ /// Some visited URLs are not in `urls`: the block had more, or longer, than an index row keeps.+ pub urls_truncated: bool,+ /// Goes in the `capture-timestamp-ms` header, not in the row.+ #[serde(skip)]+ pub captured_at_ms: Option<i64>,+}++/// The index a row belongs to, as the `ml-index` header names it.+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum Index {+ Blocks,+ Images,+}++impl Index {+ pub fn as_str(self) -> &'static str {+ match self {+ Index::Blocks => "blocks",+ Index::Images => "images",+ }+ }+}++#[derive(Debug, Clone, PartialEq, Eq)]+pub struct IndexMessage {+ pub index: Index,+ /// The session ID of a block row, the image reference of an image row.+ pub key: String,+ pub payload: Vec<u8>,+ pub captured_at_ms: Option<i64>,+}++impl IndexMessage {+ pub fn block(row: &BlockIndexRow<'_>) -> serde_json::Result<Self> {+ Ok(Self {+ index: Index::Blocks,+ key: row.session_id.to_owned(),+ payload: serde_json::to_vec(row)?,+ captured_at_ms: row.captured_at_ms,+ })+ }++ pub fn image(row: &ImageIndexRow, captured_at_ms: Option<i64>) -> serde_json::Result<Self> {+ Ok(Self {+ index: Index::Images,+ key: row.image_ref.clone(),+ payload: serde_json::to_vec(row)?,+ captured_at_ms,+ })+ }}#[cfg(test)]mod tests {- use bytes::Bytes;- use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;+ use serde_json::{json, Value};use super::*;+ /// The columns of the Parquet files the recorder wrote before the sink took them over, in file order, with their+ /// parquetjs type and whether they are optional. The sink writes the same files, so a renamed field or a changed+ /// type here breaks every reader.+ const BLOCK_COLUMNS: &[(&str, &str, bool)] = &[+ ("team_id", "INT64", false),+ ("session_id", "UTF8", false),+ ("key_month", "UTF8", false),+ ("block_s3_key", "UTF8", false),+ ("topic", "UTF8", false),+ ("partition", "INT32", false),+ ("start_offset", "INT64", false),+ ("end_offset", "INT64", false),+ ("flush_end_offset", "INT64", false),+ ("flush_row_count", "INT32", false),+ ("flush_row_index", "INT32", false),+ ("flush_id", "UTF8", false),+ ("flushed_at_ms", "INT64", false),+ ("block_byte_start", "INT64", false),+ ("block_byte_end", "INT64", false),+ ("first_ts_ms", "INT64", true),+ ("last_ts_ms", "INT64", true),+ ("event_count", "INT64", false),+ ("byte_size", "INT64", false),+ ("snapshot_source", "UTF8", true),+ ("snapshot_library", "UTF8", true),+ ("host", "UTF8", true),+ ("full_snapshots", "INT32", false),+ ("events_before_first_full_snapshot", "INT32", false),+ ("orphan_mutations", "INT32", false),+ ("decode_failures", "INT32", false),+ ("dom_nodes", "INT32", false),+ ("remote_stylesheets", "INT32", false),+ ("inline_stylesheets", "INT32", false),+ ("iframes", "INT32", false),+ ("canvases", "INT32", false),+ ("videos", "INT32", false),+ ("shadow_roots", "INT32", false),+ ("max_gap_ms", "INT64", false),+ ("active_ms", "INT64", false),+ ("dom_text_chars", "INT64", false),+ ("cold_start", "BOOLEAN", false),+ ("node_cap_reached", "BOOLEAN", false),+ ("moments_truncated", "BOOLEAN", false),+ ("touch_input", "BOOLEAN", false),+ ("clicks", "INT32", false),+ ("mouse_interactions", "INT32", false),+ ("rage_clicks", "INT32", false),+ ("dead_clicks", "INT32", false),+ ("keypresses", "INT32", false),+ ("scrolls", "INT32", false),+ ("mouse_moves", "INT32", false),+ ("touches", "INT32", false),+ ("touch_moves", "INT32", false),+ ("selections", "INT32", false),+ ("media_events", "INT32", false),+ ("viewport_resizes", "INT32", false),+ ("navigations", "INT32", false),+ ("console_errors", "INT32", false),+ ("console_warnings", "INT32", false),+ ("network_requests", "INT32", false),+ ("network_failures", "INT32", false),+ ("images_extracted", "INT32", false),+ ("images_inline", "INT32", false),+ ("image_urls", "INT32", false),+ ("image_bytes", "INT64", false),+ ("written_jsonl_bytes", "INT64", false),+ ("max_written_event_bytes", "INT64", false),+ ("viewport_width", "INT32", true),+ ("viewport_height", "INT32", true),+ ("viewport_ts_ms", "INT64", true),+ ("moments", "MOMENTS", false),+ ("url_fingerprints", "LIST<UTF8>", false),+ ("window_ids", "LIST<UTF8>", false),+ ("urls", "UTF8", true),+ ("urls_truncated", "BOOLEAN", false),+ ];++ const IMAGE_COLUMNS: &[(&str, &str, bool)] = &[+ ("team_id", "INT64", false),+ ("key_month", "UTF8", false),+ ("image_ref", "UTF8", false),+ ("pack_s3_key", "UTF8", false),+ ("byte_start", "INT64", false),+ ("byte_end", "INT64", false),+ ("mime", "UTF8", false),+ ("byte_size", "INT64", false),+ ];++ fn fits(value: &Value, column_type: &str) -> bool {+ match column_type {+ "UTF8" => value.is_string(),+ "BOOLEAN" => value.is_boolean(),+ "INT64" => value.as_i64().is_some_and(|n| n.abs() <= MAX_SAFE_INTEGER),+ "INT32" => value.as_i64().is_some_and(|n| i32::try_from(n).is_ok()),+ "LIST<UTF8>" => value+ .as_array()+ .is_some_and(|items| items.iter().all(|item| fits(item, "UTF8"))),+ "MOMENTS" => value.as_array().is_some_and(|moments| {+ moments.iter().all(|moment| {+ let moment = moment.as_object().unwrap();+ moment.len() == 4+ && fits(&moment["kind"], "UTF8")+ && fits(&moment["window_id"], "UTF8")+ && fits(&moment["ts_ms"], "INT64")+ && fits(&moment["event_index"], "INT32")+ })+ }),+ other => panic!("no column type {other}"),+ }+ }++ /// Parses a row and checks it has exactly the columns, each value of its column's type.+ fn assert_columns(payload: &[u8], columns: &[(&str, &str, bool)]) -> Value {+ let value: Value = serde_json::from_slice(payload).unwrap();+ let row = value.as_object().unwrap();+ let mut keys: Vec<&str> = row.keys().map(String::as_str).collect();+ keys.sort_unstable();+ let mut names: Vec<&str> = columns.iter().map(|(name, _, _)| *name).collect();+ names.sort_unstable();+ assert_eq!(keys, names);+ for (name, column_type, optional) in columns {+ let cell = &row[*name];+ assert!(+ (*optional && cell.is_null()) || fits(cell, column_type),+ "{name} = {cell} is not {column_type}"+ );+ }+ value+ }+#[test]- fn image_index_reads_back_with_the_documented_columns() {- let rows = vec![ImageIndexRow {+ fn rows_have_exactly_the_parquet_columns_and_types() {+ let stats = BlockStats {+ // 2^53, and a 1e100 timestamp read as an integer: past what JavaScript holds exactly.+ first_ts_ms: Some(1 << 53),+ max_gap_ms: i64::MIN,+ event_count: u64::MAX,+ clicks: u32::MAX,+ viewport_width: Some(u32::MAX),+ written_jsonl_bytes: u64::MAX,+ url_fingerprints: vec!["b08517cee3ff5ecc".to_owned()],+ window_ids: vec!["w1".to_owned(), "w2".to_owned()],+ ..BlockStats::default()+ };+ let moments = [Moment {+ kind: "full_snapshot",+ window_id: "w1".into(),+ ts_ms: 1e100 as i64,+ event_index: u32::MAX,+ }];+ let block = BlockIndexRow {+ team_id: 7,+ session_id: "s",+ key_month: "2026-09",+ block_s3_key: "blocks/dt=2026-10-15/1-0",+ topic: "main",+ partition: 2,+ start_offset: 100,+ end_offset: 105,+ flush_end_offset: 107,+ flush_row_count: u32::MAX,+ flush_row_index: 0,+ flush_id: "0000000000000001",+ flushed_at_ms: 1_000,+ block_byte_start: 0,+ block_byte_end: u64::MAX,+ stats: &stats,+ snapshot_source: Some("web"),+ snapshot_library: None,+ moments: &moments,+ urls: None,+ urls_truncated: false,+ captured_at_ms: Some(5),+ };+ let message = IndexMessage::block(&block).unwrap();+ let value = assert_columns(&message.payload, BLOCK_COLUMNS);+ assert_eq!(+ [+ &value["first_ts_ms"],+ &value["max_gap_ms"],+ &value["last_ts_ms"],+ &value["cold_start"],+ &value["snapshot_library"],+ &value["event_count"],+ &value["block_byte_end"],+ &value["clicks"],+ &value["viewport_width"],+ &value["written_jsonl_bytes"],+ &value["flush_row_count"],+ &value["url_fingerprints"],+ &value["window_ids"],+ ],+ [+ &json!(MAX_SAFE_INTEGER),+ &json!(-MAX_SAFE_INTEGER),+ &Value::Null,+ &json!(false),+ &Value::Null,+ &json!(MAX_SAFE_INTEGER),+ &json!(MAX_SAFE_INTEGER),+ &json!(i32::MAX),+ &json!(i32::MAX),+ &json!(MAX_SAFE_INTEGER),+ &json!(i32::MAX),+ &json!(["b08517cee3ff5ecc"]),+ &json!(["w1", "w2"]),+ ]+ );+ assert_eq!(+ value["moments"],+ json!([{"kind": "full_snapshot", "window_id": "w1", "ts_ms": MAX_SAFE_INTEGER, "event_index": i32::MAX}])+ );++ let image = ImageIndexRow {team_id: 7,key_month: "2026-10".to_owned(),image_ref: "image:v4:7:2026-10:abc".to_owned(),@@ -453,44 +514,9 @@ mod tests {byte_end: 99,mime: "image/png".to_owned(),byte_size: u64::MAX,- }];- let file = write_parquet(&image_index(&rows).unwrap()).unwrap();- let mut reader = ParquetRecordBatchReaderBuilder::try_new(Bytes::from(file))- .unwrap()- .build()- .unwrap();- let batch = reader.next().unwrap().unwrap();- let columns: Vec<_> = batch- .schema()- .fields()- .iter()- .map(|f| (f.name().clone(), f.data_type().clone(), f.is_nullable()))- .collect();- let column = |name: &str, data_type| (name.to_owned(), data_type, false);- assert_eq!(- columns,- [- column("team_id", DataType::Int64),- column("key_month", DataType::Utf8),- column("image_ref", DataType::Utf8),- column("pack_s3_key", DataType::Utf8),- column("byte_start", DataType::Int64),- column("byte_end", DataType::Int64),- column("mime", DataType::Utf8),- column("byte_size", DataType::Int64),- ]- );- let sizes = batch- .column(7)- .as_any()- .downcast_ref::<Int64Array>()- .unwrap();- assert_eq!(sizes.value(0), i64::MAX);- let refs = batch- .column(2)- .as_any()- .downcast_ref::<StringArray>()- .unwrap();- assert_eq!(refs.value(0), "image:v4:7:2026-10:abc");+ };+ let message = IndexMessage::image(&image, None).unwrap();+ let value = assert_columns(&message.payload, IMAGE_COLUMNS);+ assert_eq!(value["byte_size"], json!(MAX_SAFE_INTEGER));}}diff --git a/rust/ai-research/replay-recorder-v4/src/main.rs b/rust/ai-research/replay-recorder-v4/src/main.rsindex d30f7fb97b72..5110d7a9d44f 100644--- a/rust/ai-research/replay-recorder-v4/src/main.rs+++ b/rust/ai-research/replay-recorder-v4/src/main.rs@@ -18,13 +18,17 @@ use replay_core_v4::keys::kms::{AwsKms, CachedKms, KmsLimits};use replay_core_v4::keys::table::{AwsKeyTable, CachedKeyTable};use replay_recorder_v4::config::Config;use replay_recorder_v4::consumer::{run, RecorderConsumer, Settings, Sources};-use replay_recorder_v4::flush::Writer;+use replay_recorder_v4::flush::{Writer, PRODUCER_FLUSH_ON_SHUTDOWN};use replay_recorder_v4::limits::{Restrictions, SessionLimiter};use replay_recorder_v4::store::S3Store;use replay_recorder_v4::teams::TeamCache;common_alloc::used!();+/// The index producer's local queue. A block row is a few KB, so this holds several large flushes.+const INDEX_QUEUE_MIB: u32 = 512;+const INDEX_QUEUE_MESSAGES: u32 = 500_000;+#[tokio::main]async fn main() -> Result<(), Box<dyn std::error::Error>> {let config = Config::init_from_env().expect("invalid configuration");@@ -39,27 +43,33 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {.init();tracing::info!(topic = %config.kafka_topic, overflow_topic = %config.kafka_overflow_topic, group = %config.kafka_consumer_group, sample = config.sample_percentage, "starting replay-recorder-v4");+ // The consumer flushes the buffer, then the producer, on shutdown.+ let consumer_shutdown =+ Duration::from_secs(config.shutdown_flush_secs + 5) + PRODUCER_FLUSH_ON_SHUTDOWN;let mut manager = Manager::builder("replay-recorder-v4")- .with_global_shutdown_timeout(Duration::from_secs(config.shutdown_flush_secs + 15))+ .with_global_shutdown_timeout(consumer_shutdown + Duration::from_secs(10)).build();let consumer_handle = manager.register("consumer",ComponentOptions::new()- .with_graceful_shutdown(Duration::from_secs(config.shutdown_flush_secs + 5))+ .with_graceful_shutdown(consumer_shutdown).with_liveness_deadline(Duration::from_secs(60)),);let restrictions_handle = manager.register("event-restrictions",ComponentOptions::new().with_graceful_shutdown(Duration::from_secs(5)),);- // The producer reports itself live from librdkafka's statistics callback.- let producer_handle = config.write_enabled.then(|| {- manager.register(- "image-fetch-producer",- ComponentOptions::new()- .with_graceful_shutdown(Duration::from_secs(5))- .with_liveness_deadline(Duration::from_secs(60)),- )+ // A producer reports itself live from librdkafka's statistics callback.+ let producer_handles = config.write_enabled.then(|| {+ let mut register = |name: &str| {+ manager.register(+ name,+ ComponentOptions::new()+ .with_graceful_shutdown(Duration::from_secs(5))+ .with_liveness_deadline(Duration::from_secs(60)),+ )+ };+ (register("image-fetch-producer"), register("index-producer"))});let metrics_handle = manager.register("metrics-server",@@ -164,7 +174,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {redirecting_topic: (config.ingestion_overflow_mode == "redirect").then(|| config.kafka_topic.clone()),};- let writer = if let Some(producer_handle) = producer_handle {+ let writer = if let Some((producer_handle, index_producer_handle)) = producer_handles {let aws = aws_config::defaults(aws_config::BehaviorVersion::latest()).load().await;@@ -202,6 +212,19 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {).await?,image_fetch_topic: config.image_fetch_topic.clone(),+ index_producer: create_kafka_producer(+ &KafkaConfig {+ kafka_compression_codec: "lz4".to_owned(),+ kafka_message_timeout_ms: config.index_delivery_timeout_ms,+ // Rows of a few flushes, so a slow broker holds them here instead of losing them at once.+ kafka_producer_queue_mib: INDEX_QUEUE_MIB,+ kafka_producer_queue_messages: INDEX_QUEUE_MESSAGES,+ ..config.kafka.clone()+ },+ index_producer_handle,+ )+ .await?,+ index_topic: config.index_topic.clone(),sent_urls: replay_recorder_v4::urls::sent_urls(500_000),seal_permits: Arc::new(tokio::sync::Semaphore::new(settings.flush_concurrency)),})diff --git a/rust/ai-research/replay-recorder-v4/src/urls.rs b/rust/ai-research/replay-recorder-v4/src/urls.rsindex af25d0355791..99143d4bd782 100644--- a/rust/ai-research/replay-recorder-v4/src/urls.rs+++ b/rust/ai-research/replay-recorder-v4/src/urls.rs@@ -207,6 +207,7 @@ mod tests {image_urls: urls.iter().map(|u| (*u).to_owned()).collect(),remote_urls: Vec::new(),url_limit: None,+ captured_at_ms: None,};let mut stats = BlockStats::default();let mut keys = FlushKeys::default();diff --git a/rust/ai-research/replay-recorder-v4/src/write.rs b/rust/ai-research/replay-recorder-v4/src/write.rsindex 056f7a419b12..248e3b256b39 100644--- a/rust/ai-research/replay-recorder-v4/src/write.rs+++ b/rust/ai-research/replay-recorder-v4/src/write.rs@@ -1,7 +1,7 @@-//! Writes a flush: image packs and their index first, then one object of sealed session blocks, then their index.+//! Writes a flush: image packs first, then one object of sealed session blocks, and builds the index rows for both.//!//! The order matters for a crash between writes. A pack without its blocks is unreferenced, which is harmless. A block-//! whose images were never stored would hold references that resolve to nothing.+//! whose images were never stored would hold references that resolve to nothing. The rows go out only after both.use std::collections::{BTreeMap, HashMap, HashSet};use std::io::{self, Write};@@ -9,6 +9,7 @@ use std::sync::Arc;use chrono::{DateTime, Utc};use futures::stream::{self, StreamExt, TryStreamExt};+use metrics::counter;use rand::Rng;use replay_core_v4::bytescan::write_json_string;use replay_core_v4::envelope::{seal_frame, seal_json_envelope, Codec, Context, Version};@@ -16,7 +17,6 @@ use replay_core_v4::inspect::events::event_page_url;use replay_core_v4::inspect::stats::BlockStats;use replay_core_v4::inspect::stats::MAX_URL_FINGERPRINTS;use replay_core_v4::keys::batch::{FlushKeys, SessionId, SessionKey as Key};-use replay_core_v4::keys::rows::write_day;use replay_core_v4::refs::url_fingerprint;use replay_core_v4::scan::cv::extract_event_images;use replay_core_v4::scan::images::{Image, ImageLimits, ImageSink};@@ -24,10 +24,8 @@ use replay_core_v4::seal::{image_content_key, url_fingerprint_key};use tokio::sync::Semaphore;-use crate::batch::{Block, SessionKey, Tp};-use crate::index::{- block_index, image_index, write_parquet, BlockIndexRow, ImageIndexRow, IndexError,-};+use crate::batch::{Block, SessionKey};+use crate::index::{BlockIndexRow, ImageIndexRow};use crate::rewrite::{canonical_strings, UrlRewrite};use crate::store::{BlobStore, StoreError};@@ -41,8 +39,6 @@ const BLOCK_WRITE_BUFFER: usize = 256 * 1024;pub enum WriteError {#[error(transparent)]Store(#[from] StoreError),- #[error(transparent)]- Index(#[from] IndexError),#[error("sealing failed: {0}")]Seal(String),#[error("sealing panicked: {0}")]@@ -341,8 +337,8 @@ fn seal_pack(Ok((pack, rows))}-/// Seals each team's images under its team month key into one pack, stores the packs concurrently, then writes the-/// image index for the flush.+/// Seals each team's images under its team month key into one pack and stores the packs concurrently. Returns the+/// image index rows of the stored packs.#[allow(clippy::too_many_arguments)]pub async fn write_images(images: BTreeMap<i64, Vec<Arc<Image>>>,@@ -350,7 +346,6 @@ pub async fn write_images(write_day: &str,key_month: &str,images_bucket: &str,- rrweb_bucket: &str,store: &Arc<dyn BlobStore>,at: DateTime<Utc>,concurrency: usize,@@ -377,20 +372,13 @@ pub async fn write_images(})}).collect();- let rows: Vec<ImageIndexRow> = stream::iter(packs)+ Ok(stream::iter(packs).buffer_unordered(concurrency.max(1)).try_collect::<Vec<_>>().await?.into_iter().flatten()- .collect();- if !rows.is_empty() {- let index_key = format!("index/images/dt={write_day}/{}.parquet", unique_suffix(at));- let batch = image_index(&rows)?;- let file = blocking(move || write_parquet(&batch).map_err(WriteError::from)).await?;- store.put_new(rrweb_bucket, &index_key, file).await?;- }- Ok(rows)+ .collect())}/// Writes the flush's sealed blocks as one object. Returns where each landed, in the same order.@@ -466,6 +454,33 @@ pub fn block_urls(block: &Block) -> Vec<String> {urls}+/// The most visited pages a block index row keeps, the longest page it keeps and their bytes together. The sealed list+/// then stays far below the index producer's message limit of 1,000,000 bytes with the row's other columns, and a+/// Parquet page of these values stays small.+const MAX_INDEX_URLS: usize = 100;+const MAX_INDEX_URL_BYTES: usize = 2048;+const MAX_INDEX_URLS_BYTES: usize = 32 * 1024;++/// The visited pages that fit in a block index row, in order, and whether any page was left out.+pub fn index_urls(urls: Vec<String>) -> (Vec<String>, bool) {+ let mut kept = Vec::new();+ let mut bytes = 0;+ let mut truncated = false;+ for url in urls {+ if kept.len() >= MAX_INDEX_URLS || bytes + url.len() > MAX_INDEX_URLS_BYTES {+ truncated = true;+ break;+ }+ if url.len() > MAX_INDEX_URL_BYTES {+ truncated = true;+ continue;+ }+ bytes += url.len();+ kept.push(url);+ }+ (kept, truncated)+}+/// The block's distinct pages as fingerprints under the session's fingerprint key, the first/// [`MAX_URL_FINGERPRINTS`] in the order the block saw them.pub fn url_fingerprints(session_key: &[u8; 32], urls: &[String]) -> Vec<String> {@@ -485,14 +500,14 @@ pub fn url_fingerprints(session_key: &[u8; 32], urls: &[String]) -> Vec<String>/// Numbers the rows of each partition in order and gives each row the count of its partition.fn number_rows(rows: &mut [BlockIndexRow<'_>]) {- let mut counts: HashMap<Tp, u32> = HashMap::new();+ let mut counts: HashMap<(&str, i32), u32> = HashMap::new();for row in rows.iter_mut() {- let count = counts.entry(row.origin.tp).or_default();+ let count = counts.entry((row.topic, row.partition)).or_default();row.flush_row_index = *count;*count += 1;}for row in rows.iter_mut() {- row.flush_row_count = counts[&row.origin.tp];+ row.flush_row_count = counts[&(row.topic, row.partition)];}}@@ -512,16 +527,13 @@ impl FlushAttempt {}}-/// Writes one block index file for the flush, after its blocks, so no row points at a block that is not stored.+/// The flush's block index rows, built after its blocks are stored, so no row points at a block that is not./// Each entry is a block, its stats as this attempt filled them, and where it landed.-pub async fn write_block_index(- entries: &[(&Block, &BlockStats, &Written)],+pub fn block_index_rows<'a>(+ entries: &[(&'a Block, &'a BlockStats, &'a Written)],keys: &FlushKeys,- attempt: &FlushAttempt,- rrweb_bucket: &str,- store: &dyn BlobStore,- at: DateTime<Utc>,-) -> Result<usize, WriteError> {+ attempt: &'a FlushAttempt,+) -> Result<Vec<BlockIndexRow<'a>>, WriteError> {let mut rows = Vec::with_capacity(entries.len());for &(block, stats, written) in entries {let id = SessionId {@@ -531,7 +543,10 @@ pub async fn write_block_index(let Some(key) = keys.sessions.get(&id) else {continue;};- let urls = block_urls(block);+ let (urls, urls_truncated) = index_urls(block_urls(block));+ if urls_truncated {+ counter!("ai_research_v4_recorder_index_urls_truncated_total").increment(1);+ }let urls = if urls.is_empty() {None} else {@@ -552,32 +567,32 @@ pub async fn write_block_index(};let first = block.messages.first();rows.push(BlockIndexRow {- written,- origin: &block.origin,- attempt,+ team_id: written.session.team_id,+ session_id: &written.session.session_id,+ key_month: &written.key_month,+ block_s3_key: &written.object_key,+ topic: block.origin.tp.topic,+ partition: block.origin.tp.partition,+ start_offset: block.origin.start_offset,+ end_offset: block.origin.end_offset,+ flush_end_offset: block.origin.flush_end_offset,+ flush_row_count: 0,+ flush_row_index: 0,+ flush_id: &attempt.flush_id,+ flushed_at_ms: attempt.flushed_at_ms,+ block_byte_start: written.byte_start,+ block_byte_end: written.byte_end,stats,- moments: &block.moments,snapshot_source: first.and_then(|m| m.snapshot_source.as_deref()),snapshot_library: first.and_then(|m| m.library.as_deref()),+ moments: &block.moments,urls,- flush_row_count: 0,- flush_row_index: 0,+ urls_truncated,+ captured_at_ms: block.captured_at_ms,});}- if rows.is_empty() {- return Ok(0);- }number_rows(&mut rows);- let index_key = format!(- "index/blocks/dt={}/{}.parquet",- write_day(at),- unique_suffix(at)- );- let count = rows.len();- let batch = block_index(&rows)?;- let file = blocking(move || write_parquet(&batch).map_err(WriteError::from)).await?;- store.put_new(rrweb_bucket, &index_key, file).await?;- Ok(count)+ Ok(rows)}#[cfg(test)]@@ -588,13 +603,9 @@ mod tests {use aes_gcm::aead::{Aead, KeyInit, Payload};use aes_gcm::{Aes256Gcm, Nonce};- use arrow_array::{Array, Int64Array, ListArray, RecordBatch, StringArray, StructArray};- use arrow_schema::DataType;use async_trait::async_trait;use base64::engine::general_purpose::STANDARD;use base64::Engine;- use bytes::Bytes;- use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;use replay_core_v4::envelope::open_frame;use replay_core_v4::inspect::dom::MAX_REMOTE_URL_BYTES;use replay_core_v4::inspect::session::Moment;@@ -613,29 +624,6 @@ mod tests {fn get(&self, bucket: &str, key: &str) -> Vec<u8> {self.objects.lock().unwrap()[&(bucket.to_owned(), key.to_owned())].clone()}-- fn keys(&self, prefix: &str) -> Vec<String> {- let mut keys: Vec<String> = self- .objects- .lock()- .unwrap()- .keys()- .filter(|(_, key)| key.starts_with(prefix))- .map(|(_, key)| key.clone())- .collect();- keys.sort();- keys- }-- fn parquet(&self, key: &str) -> RecordBatch {- ParquetRecordBatchReaderBuilder::try_new(Bytes::from(self.get("rrweb-v4", key)))- .unwrap()- .build()- .unwrap()- .next()- .unwrap()- .unwrap()- }}fn decompress(compressed: &[u8]) -> io::Result<Vec<u8>> {@@ -695,6 +683,7 @@ mod tests {image_urls: Vec::new(),remote_urls: Vec::new(),url_limit: None,+ captured_at_ms: None,}}@@ -788,6 +777,26 @@ mod tests {assert!(!stored_lines(&event, &[public]).is_empty());}+ #[test]+ fn keeps_the_sealed_urls_of_a_row_under_the_message_limit() {+ let page = |i: usize| format!("https://app.example.com/{i}/{}", "p".repeat(1000));+ let (kept, truncated) = index_urls((0..1000).map(page).collect());+ assert!(truncated);+ assert!(+ kept.len() <= MAX_INDEX_URLS+ && kept.iter().map(String::len).sum::<usize>() <= MAX_INDEX_URLS_BYTES+ );+ let long = format!(+ "https://app.example.com/{}",+ "l".repeat(MAX_INDEX_URL_BYTES)+ );+ assert_eq!(+ index_urls(vec![page(1), long, page(2)]),+ (vec![page(1), page(2)], true)+ );+ assert_eq!(index_urls(vec![page(1)]), (vec![page(1)], false));+ }+#[test]fn a_block_carries_the_reference_its_fetch_job_names() {let (fetched, declined) = (@@ -961,7 +970,6 @@ mod tests {"2026-10-15","2026-10","images-v4",- "rrweb-v4",&store,at(),4,@@ -1024,7 +1032,6 @@ mod tests {);// The team month key seals images, as the fetcher's, so a session key does not open them.assert!(open_frame(&[5; 32], Version::V4, &image_context, sealed_image).is_err());- assert_eq!(memory.keys("index/images/dt=2026-10-15/").len(), 1);// The block index row points at the block, keeps the moments, and seals the visited pages.blocks[0].moments = vec![Moment {@@ -1033,93 +1040,19 @@ mod tests {ts_ms: 1,event_index: 1,}];- let stats = BlockStats {- url_fingerprints: sealed[0].url_fingerprints.clone(),- window_ids: vec!["w1".to_owned()],- ..BlockStats::default()- };- assert_eq!(- write_block_index(- &[(&blocks[0], &stats, &written[0])],- &keys,- &FlushAttempt::new(at()),- "rrweb-v4",- store.as_ref(),- at()- )- .await- .unwrap(),- 1- );- let batch = memory.parquet(&memory.keys("index/blocks/")[0]);- let column = |name: &str| {- batch- .column_by_name(name)- .unwrap_or_else(|| panic!("no column {name}"))- .clone()- };- let text = |name: &str| {- column(name)- .as_any()- .downcast_ref::<StringArray>()- .unwrap()- .value(0)- .to_owned()- };- assert_eq!(text("block_s3_key"), block.object_key);- assert_eq!(- column("block_byte_end")- .as_any()- .downcast_ref::<Int64Array>()- .unwrap()- .value(0),- i64::try_from(block.byte_end).unwrap()- );- for (name, data_type) in [- ("orphan_mutations", DataType::Int32),- ("active_ms", DataType::Int64),- ("rage_clicks", DataType::Int32),- ("cold_start", DataType::Boolean),- ("viewport_width", DataType::Int32),- ("image_urls", DataType::Int32),- ("decode_failures", DataType::Int32),- ("mouse_interactions", DataType::Int32),- ("touch_moves", DataType::Int32),- ("moments_truncated", DataType::Boolean),- ("written_jsonl_bytes", DataType::Int64),- ("max_written_event_bytes", DataType::Int64),- ] {- assert_eq!(column(name).data_type(), &data_type, "{name}");- }- let strings = |name: &str| -> Vec<String> {- let list = column(name);- let values = list.as_any().downcast_ref::<ListArray>().unwrap().value(0);- let values = values.as_any().downcast_ref::<StringArray>().unwrap();- values.iter().map(|v| v.unwrap().to_owned()).collect()+ let stats = BlockStats::default();+ let attempt = FlushAttempt::new(at());+ let index =+ block_index_rows(&[(&blocks[0], &stats, &written[0])], &keys, &attempt).unwrap();+ let [row] = index.as_slice() else {+ panic!("one row per stored block");};- assert_eq!(strings("url_fingerprints"), stats.url_fingerprints);- assert_eq!(strings("window_ids"), ["w1"]);- let moments = column("moments");- let moments = moments- .as_any()- .downcast_ref::<ListArray>()- .unwrap()- .value(0);- let moments = moments.as_any().downcast_ref::<StructArray>().unwrap();assert_eq!(- moments.column_by_name("event_index").unwrap().data_type(),- &DataType::Int32+ (row.block_s3_key, row.block_byte_end),+ (block.object_key.as_str(), block.byte_end));- let kinds = moments.column_by_name("kind").unwrap().clone();- assert_eq!(- kinds- .as_any()- .downcast_ref::<StringArray>()- .unwrap()- .value(0),- "full_snapshot"- );- let envelope: Value = serde_json::from_str(&text("urls")).unwrap();+ assert_eq!(row.moments, blocks[0].moments);+ let envelope: Value = serde_json::from_str(row.urls.as_deref().unwrap()).unwrap();let sealed_urls = STANDARD.decode(envelope["ciphertext"].as_str().unwrap()).unwrap();@@ -1155,9 +1088,8 @@ mod tests {);}- #[tokio::test]- async fn readers_drop_the_rows_of_earlier_attempts_that_a_retry_covers() {- let store = MemoryStore::default();+ #[test]+ fn readers_drop_the_rows_of_earlier_attempts_that_a_retry_covers() {let mut keys = FlushKeys::default();for session in ["a", "b", "c"] {let (id, key) = session_key(session);@@ -1199,6 +1131,8 @@ mod tests {flush_id: "0000000000000002".to_owned(),flushed_at_ms: 2_000,};+ let mut rows = Vec::new();+ let mut positions = Vec::new();for (attempt, entries) in [(&first, vec![(&a, &stats, &wa)]),(&between, vec![(&early_b, &stats, &wb)]),@@ -1211,61 +1145,19 @@ mod tests {],),] {- write_block_index(&entries, &keys, attempt, "rrweb-v4", &store, at())- .await- .unwrap();- }-- let mut rows = Vec::new();- let mut positions = Vec::new();- for key in store.keys("index/blocks/") {- let batch = store.parquet(&key);- let text = |name: &str| {- batch- .column_by_name(name)- .unwrap()- .as_any()- .downcast_ref::<StringArray>()- .unwrap()- .clone()- };- let int = |name: &str| {- batch- .column_by_name(name)- .unwrap()- .as_any()- .downcast_ref::<Int64Array>()- .unwrap()- .clone()- };- let int32 = |name: &str| {- batch- .column_by_name(name)- .unwrap()- .as_any()- .downcast_ref::<arrow_array::Int32Array>()- .unwrap()- .clone()- };- let partition = int32("partition");- let (topic, session, flush_id) = (text("topic"), text("session_id"), text("flush_id"));- let (start, end, at_ms) =- (int("start_offset"), int("end_offset"), int("flushed_at_ms"));- let flush_end = int("flush_end_offset");- let (count, index) = (int32("flush_row_count"), int32("flush_row_index"));- for i in 0..batch.num_rows() {+ for row in block_index_rows(&entries, &keys, attempt).unwrap() {positions.push((- flush_id.value(i).to_owned(),- partition.value(i),- flush_end.value(i),- count.value(i),- index.value(i),+ row.flush_id.to_owned(),+ row.partition,+ row.flush_end_offset,+ row.flush_row_count,+ row.flush_row_index,));rows.push((- (topic.value(i).to_owned(), partition.value(i)),- (start.value(i), end.value(i)),- (at_ms.value(i), flush_id.value(i).to_owned()),- session.value(i).to_owned(),+ (row.topic.to_owned(), row.partition),+ (row.start_offset, row.end_offset),+ (row.flushed_at_ms, row.flush_id.to_owned()),+ row.session_id.to_owned(),));}}