Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(replay-recorder-v4): send the v4 index to a kafka topic (#119746)

Nicholas Waltzef8608ab5d16
diff --git a/rust/Cargo.lock b/rust/Cargo.lock
index e871da13c7a6..d7978d6f69a8 100644
--- a/rust/Cargo.lock
+++ b/rust/Cargo.lock
@@ -11054,8 +11054,6 @@ dependencies = [
"aes-gcm",
"aho-corasick",
"arc-swap",
- "arrow-array",
- "arrow-schema",
"async-trait",
"aws-config",
"aws-sdk-dynamodb",
@@ -11064,7 +11062,6 @@ dependencies = [
"axum 0.7.9",
"base64 0.22.1",
"brotlic",
- "bytes",
"capture",
"chrono",
"common-alloc",
@@ -11077,10 +11074,10 @@ dependencies = [
"lifecycle",
"metrics",
"moka",
- "parquet",
"rand 0.8.5",
"rdkafka",
"replay-core-v4",
+ "serde",
"serde_json",
"sqlx",
"thiserror 2.0.18",
diff --git a/rust/ai-research/replay-recorder-v4/Cargo.toml b/rust/ai-research/replay-recorder-v4/Cargo.toml
index ed045fbc5545..850ec63e079e 100644
--- a/rust/ai-research/replay-recorder-v4/Cargo.toml
+++ b/rust/ai-research/replay-recorder-v4/Cargo.toml
@@ -21,9 +21,6 @@ replay-core-v4 = { path = "../replay-core-v4" }
aho-corasick = { workspace = true }
arc-swap = { workspace = true }
-# The major version deltalite links, so the workspace builds one arrow.
-arrow-array = "58"
-arrow-schema = "58"
async-trait = { workspace = true }
aws-config = { workspace = true }
aws-sdk-dynamodb = { workspace = true }
@@ -36,9 +33,9 @@ envconfig = { workspace = true }
futures = { workspace = true }
metrics = { workspace = true }
moka = { workspace = true }
-parquet = "58"
rand = { workspace = true }
rdkafka = { workspace = true }
+serde = { workspace = true }
serde_json = { workspace = true }
sqlx = { workspace = true }
thiserror = { workspace = true }
@@ -50,6 +47,5 @@ url = { workspace = true }
[dev-dependencies]
aes-gcm = "0.10"
base64 = { workspace = true }
-bytes = { workspace = true }
serde_json = { workspace = true }
tokio = { workspace = true, features = ["test-util"] }
diff --git a/rust/ai-research/replay-recorder-v4/src/batch.rs b/rust/ai-research/replay-recorder-v4/src/batch.rs
index ec0b29047134..995d6c137a91 100644
--- a/rust/ai-research/replay-recorder-v4/src/batch.rs
+++ b/rust/ai-research/replay-recorder-v4/src/batch.rs
@@ -32,6 +32,7 @@ pub struct Tp {
/// An admitted message, not decoded yet.
pub struct Raw {
pub offset: i64,
+ pub captured_at_ms: Option<i64>,
pub payload: Vec<u8>,
pub content_encoding: Option<String>,
}
@@ -95,6 +96,8 @@ pub struct Block {
pub remote_urls: Vec<String>,
/// Set when the inspector stopped short of naming every remote URL, so that the block must not be stored.
pub url_limit: Option<UrlLimit>,
+ /// The earliest Kafka timestamp of the block's messages, for the capture watermark of the index topic.
+ pub captured_at_ms: Option<i64>,
}
/// The decoded `cv` strings one flush keeps, on top of the decoded payloads that the flush size counts.
@@ -421,6 +424,7 @@ fn inspect_session(
let mut decoded = Vec::with_capacity(parts.len());
let mut session_spent = 0;
for (origin, raws) in parts {
+ let captured_at_ms = raws.iter().filter_map(|raw| raw.captured_at_ms).min();
let mut messages = Vec::with_capacity(raws.len());
for raw in raws {
// A growing list can hold twice its events, so the reader stops at half of what the budget has room for.
@@ -472,7 +476,7 @@ fn inspect_session(
}
counter!("ai_research_v4_recorder_messages_total", "outcome" => "recorded")
.increment(messages.len() as u64);
- decoded.push((origin, messages));
+ decoded.push((origin, messages, captured_at_ms));
}
// A session can have messages on the main and the overflow topic in one flush. The inspector takes them in replay
// order, so that a mutation never comes before the nodes it changes. Each still counts toward its partition's block.
@@ -480,7 +484,7 @@ fn inspect_session(
decoded.iter().map(|_| BlockInspection::default()).collect();
let times: Vec<Vec<f64>> = decoded
.iter()
- .map(|(_, messages)| messages.iter().map(replay_time).collect())
+ .map(|(_, messages, _)| messages.iter().map(replay_time).collect())
.collect();
let mut next = vec![0; decoded.len()];
while let Some(part) = (0..decoded.len())
@@ -493,7 +497,7 @@ fn inspect_session(
let blocks = decoded
.into_iter()
.zip(inspections)
- .map(|((origin, messages), block)| {
+ .map(|((origin, messages, captured_at_ms), block)| {
let inspected = block.finish();
Block {
session: session.clone(),
@@ -504,6 +508,7 @@ fn inspect_session(
image_urls: inspected.image_urls,
remote_urls: inspected.remote_urls,
url_limit: inspected.url_limit,
+ captured_at_ms,
}
})
.collect();
@@ -578,6 +583,7 @@ mod tests {
let inner = json!({"event": "$snapshot_items", "properties": {"$session_id": session, "$window_id": "w", "$snapshot_source": "web", "$snapshot_items": items}});
Raw {
offset,
+ captured_at_ms: None,
payload: json!({"distinct_id": "d", "data": inner.to_string()})
.to_string()
.into_bytes(),
@@ -796,6 +802,7 @@ mod tests {
async fn a_session_that_decodes_nothing_keeps_its_state_as_it_was() {
let unreadable = |offset: i64| Raw {
offset,
+ captured_at_ms: None,
payload: b"not json".to_vec(),
content_encoding: None,
};
diff --git a/rust/ai-research/replay-recorder-v4/src/config.rs b/rust/ai-research/replay-recorder-v4/src/config.rs
index 45c685b89633..e7e90f5926fd 100644
--- a/rust/ai-research/replay-recorder-v4/src/config.rs
+++ b/rust/ai-research/replay-recorder-v4/src/config.rs
@@ -127,6 +127,10 @@ pub struct Config {
)]
pub image_fetch_topic: String,
+ /// Where the block and image index rows go for the Parquet sink.
+ #[envconfig(from = "AI_RESEARCH_V4_INDEX_TOPIC", default = "ai_research_v4_index")]
+ pub index_topic: String,
+
/// A fetch job not delivered in this time fails its flush. It is well under the flush timeout.
#[envconfig(
from = "AI_RESEARCH_V4_IMAGE_FETCH_DELIVERY_TIMEOUT_MS",
@@ -134,6 +138,11 @@ pub struct Config {
)]
pub image_fetch_delivery_timeout_ms: u32,
+ /// An index row not delivered in this time is lost and counted. The flush does not wait for it, so it can be long
+ /// enough to ride out a broker slowdown.
+ #[envconfig(from = "AI_RESEARCH_V4_INDEX_DELIVERY_TIMEOUT_MS", default = "300000")]
+ pub index_delivery_timeout_ms: u32,
+
#[envconfig(default = "9104")]
pub metrics_port: u16,
}
@@ -205,6 +214,7 @@ impl Config {
("AI_RESEARCH_V4_RRWEB_BUCKET", &self.rrweb_bucket),
("AI_RESEARCH_V4_IMAGES_BUCKET", &self.images_bucket),
("AI_RESEARCH_V4_IMAGE_FETCH_TOPIC", &self.image_fetch_topic),
+ ("AI_RESEARCH_V4_INDEX_TOPIC", &self.index_topic),
]);
}
let missing: Vec<&str> = required
diff --git a/rust/ai-research/replay-recorder-v4/src/consumer.rs b/rust/ai-research/replay-recorder-v4/src/consumer.rs
index 43ef92e747cb..e117d139674e 100644
--- a/rust/ai-research/replay-recorder-v4/src/consumer.rs
+++ b/rust/ai-research/replay-recorder-v4/src/consumer.rs
@@ -19,7 +19,7 @@ use rdkafka::{ClientConfig, ClientContext, Offset, TopicPartitionList};
use crate::batch::{Batch, Block, Inspectors, Raw, SessionKey, Tp};
use crate::config::{retry_backoff, FLUSH_DEADLINE, MAX_POLL_INTERVAL};
-use crate::flush::{write_flush, FlushWork, Writer};
+use crate::flush::{write_flush, FlushWork, Writer, PRODUCER_FLUSH_ON_SHUTDOWN};
use crate::limits::{Restrictions, SessionLimiter};
use crate::process::{admit, sampled, Admitted, Headers, Skip};
use crate::teams::TeamCache;
@@ -419,6 +419,20 @@ pub async fn run(
sources: Sources,
writer: Option<Writer>,
handle: Handle,
+) {
+ consume(&consumer, &settings, &sources, writer.as_ref(), &handle).await;
+ // The flushes do not wait for their index rows, so the producer can still hold some.
+ if let Some(writer) = &writer {
+ writer.flush_producers(PRODUCER_FLUSH_ON_SHUTDOWN).await;
+ }
+}
+
+async fn consume(
+ consumer: &RecorderConsumer,
+ settings: &Settings,
+ sources: &Sources,
+ writer: Option<&Writer>,
+ handle: &Handle,
) {
let shutdown = handle.shutdown_signal();
tokio::pin!(shutdown);
@@ -444,16 +458,16 @@ pub async fn run(
None => break,
}
}
- handle_messages(&messages, &consumer, &settings, &sources, &mut batch).await;
+ handle_messages(&messages, consumer, settings, sources, &mut batch).await;
drop(messages);
if let Some(error) = failed {
- if receive_failed(&consumer, &handle, error) {
+ if receive_failed(consumer, handle, error) {
return;
}
}
}
Err(error) => {
- if receive_failed(&consumer, &handle, error) {
+ if receive_failed(consumer, handle, error) {
return;
}
}
@@ -469,10 +483,10 @@ pub async fn run(
let work = flush(
std::mem::take(&mut batch),
&mut inspectors,
- &consumer,
- &settings,
- writer.as_ref(),
- &handle,
+ consumer,
+ settings,
+ writer,
+ handle,
settings.flush_retries,
);
tokio::pin!(work);
@@ -500,10 +514,10 @@ pub async fn run(
flush(
batch,
&mut inspectors,
- &consumer,
- &settings,
- writer.as_ref(),
- &handle,
+ consumer,
+ settings,
+ writer,
+ handle,
0,
),
)
@@ -588,6 +602,7 @@ async fn handle_messages(
.increment(1);
let raw = Raw {
offset: message.offset(),
+ captured_at_ms: message.timestamp().to_millis(),
payload: message.payload().unwrap_or_default().to_vec(),
content_encoding: header(message, "content-encoding").map(str::to_owned),
};
@@ -826,6 +841,7 @@ mod tests {
image_urls: Vec::new(),
remote_urls: Vec::new(),
url_limit: None,
+ captured_at_ms: None,
})
};
let offsets = HashMap::from([(tp(0), (1, 5)), (tp(1), (1, 5)), (tp(2), (1, 5))]);
diff --git a/rust/ai-research/replay-recorder-v4/src/flush.rs b/rust/ai-research/replay-recorder-v4/src/flush.rs
index 1261a3267ad0..f930b118644e 100644
--- a/rust/ai-research/replay-recorder-v4/src/flush.rs
+++ b/rust/ai-research/replay-recorder-v4/src/flush.rs
@@ -1,5 +1,8 @@
//! One attempt to store a flush's blocks. It writes nothing that a retry cannot write again: object names are unique,
//! and the block index rows name their attempt.
+//!
+//! The index rows go to the index topic, and the Parquet sink writes them to S3. The flush does not wait for their
+//! delivery: the data is stored by then, and a lost row is counted.
use std::collections::{BTreeMap, BTreeSet, HashMap};
use std::sync::{Arc, Mutex, OnceLock, PoisonError};
@@ -10,7 +13,7 @@ use common_kafka::kafka_producer::KafkaContext;
use futures::stream::{self, StreamExt};
use metrics::counter;
use rdkafka::message::{Header, OwnedHeaders};
-use rdkafka::producer::{FutureProducer, FutureRecord};
+use rdkafka::producer::{FutureProducer, FutureRecord, Producer};
use rdkafka::util::Timeout;
use replay_core_v4::inspect::stats::BlockStats;
use replay_core_v4::keys::batch::{FlushKeys, KeyStore, SessionId};
@@ -20,11 +23,12 @@ use replay_core_v4::keys::table::{AwsKeyTable, CachedKeyTable};
use tokio::sync::Semaphore;
use crate::batch::Block;
+use crate::index::{Index, IndexMessage};
use crate::store::BlobStore;
use crate::urls::{fetch_jobs, records};
use crate::write::{
- compress_block, new_images, object_key, seal_blocking, seal_compressed, session_keyed_frames,
- write_block_index, write_blocks, write_images, CompressedBlock, FlushAttempt, WriteError,
+ block_index_rows, compress_block, new_images, object_key, seal_blocking, seal_compressed,
+ session_keyed_frames, write_blocks, write_images, CompressedBlock, FlushAttempt, WriteError,
};
/// How long a fetch job may wait for room in the producer queue.
@@ -33,6 +37,8 @@ const ENQUEUE_TIMEOUT: Duration = Duration::from_secs(10);
/// Fetch job records sent at once, so a slow broker slows the flush instead of filling the producer queue.
const SENDS_IN_FLIGHT: usize = 64;
+pub const PRODUCER_FLUSH_ON_SHUTDOWN: Duration = Duration::from_secs(5);
+
/// What a flush needs to store blocks. Without it the recorder runs dry.
pub struct Writer {
pub keys: KeyStore<CachedKeyTable<AwsKeyTable>, AwsKms>,
@@ -41,6 +47,9 @@ pub struct Writer {
pub images_bucket: String,
pub producer: FutureProducer<KafkaContext>,
pub image_fetch_topic: String,
+ /// The flush does not wait for index rows, so they get their own producer with a delivery timeout of minutes.
+ pub index_producer: FutureProducer<KafkaContext>,
+ pub index_topic: String,
/// Remote image references this process sent to the fetcher, by UTC day. A reference goes out again each day, so
/// the fetcher sees it within every refresh interval and keeps the image as long as the blocks that reference it.
pub sent_urls: moka::sync::Cache<(String, String), ()>,
@@ -54,6 +63,33 @@ pub enum FlushError {
Keys(#[from] replay_core_v4::keys::batch::KeyError),
#[error(transparent)]
Write(#[from] WriteError),
+ #[error("encoding an index row failed: {0}")]
+ Encode(#[from] serde_json::Error),
+}
+
+impl Writer {
+ /// Waits, up to `timeout` for both at once, for the producers to deliver what they hold, so that shutdown loses
+ /// no index rows.
+ pub async fn flush_producers(&self, timeout: Duration) {
+ futures::future::join(
+ flush_producer("image fetch", &self.producer, timeout),
+ flush_producer("index", &self.index_producer, timeout),
+ )
+ .await;
+ }
+}
+
+async fn flush_producer(name: &str, producer: &FutureProducer<KafkaContext>, timeout: Duration) {
+ let flushing = producer.clone();
+ let flushed =
+ tokio::task::spawn_blocking(move || flushing.flush(Timeout::After(timeout))).await;
+ match flushed {
+ Ok(Ok(())) => {}
+ Ok(Err(error)) => {
+ tracing::warn!(producer = name, %error, in_queue = producer.in_flight_count(), "the producer did not deliver everything on shutdown")
+ }
+ Err(error) => tracing::warn!(producer = name, %error, "flushing the producer failed"),
+ }
}
/// What the attempts of one flush share: the keys, the write day they are for, and each block compressed. A retry then
@@ -215,20 +251,19 @@ pub async fn write_flush(
);
let session_frames = session_keyed_frames(&sealed);
- let rows = write_images(
+ let image_rows = write_images(
new_images(&mut sealed),
&keys,
&day,
month,
&writer.images_bucket,
- &writer.rrweb_bucket,
&writer.store,
now,
concurrency,
&writer.seal_permits,
)
.await?;
- counter!("ai_research_v4_recorder_stored_images_total").increment(rows.len() as u64);
+ counter!("ai_research_v4_recorder_stored_images_total").increment(image_rows.len() as u64);
// What sealing images under session keys would store, to compare with the frames above.
counter!("ai_research_v4_recorder_session_keyed_image_frames_total").increment(session_frames);
@@ -246,16 +281,17 @@ pub async fn write_flush(
.zip(&written)
.map(|(&index, written)| (&*blocks[index], &stats[index], written))
.collect();
- write_block_index(
- &entries,
- &keys,
- &attempt,
- &writer.rrweb_bucket,
- writer.store.as_ref(),
- now,
- )
- .await?;
+ // An image row covers data from every block that holds the image, so it takes the flush's earliest capture.
+ let flush_captured_at_ms = blocks.iter().filter_map(|b| b.captured_at_ms).min();
+ let mut index = image_rows
+ .iter()
+ .map(|row| IndexMessage::image(row, flush_captured_at_ms))
+ .collect::<Result<Vec<_>, _>>()?;
+ for row in block_index_rows(&entries, &keys, &attempt)? {
+ index.push(IndexMessage::block(&row)?);
+ }
produce_jobs(writer, jobs, &day, now.timestamp_millis()).await;
+ send_index(&writer.index_producer, &writer.index_topic, &day, index);
Ok(())
}
@@ -318,10 +354,86 @@ async fn produce_jobs(
metrics::gauge!("ai_research_v4_recorder_sent_urls").set(writer.sent_urls.entry_count() as f64);
}
+/// Hands the index rows to the producer and returns without waiting for their delivery. A task counts the outcomes.
+/// `write_day` is the `dt=` partition of the blocks and packs the rows point at, so the sink files them with those.
+pub fn send_index(
+ producer: &FutureProducer<KafkaContext>,
+ topic: &str,
+ write_day: &str,
+ messages: Vec<IndexMessage>,
+) {
+ let mut deliveries = Vec::with_capacity(messages.len());
+ let mut failed = Failures::default();
+ for message in &messages {
+ let captured_at = message.captured_at_ms.map(|ms| ms.to_string());
+ let mut headers = OwnedHeaders::new()
+ .insert(Header {
+ key: "ml-index",
+ value: Some(message.index.as_str()),
+ })
+ .insert(Header {
+ key: "ml-write-day",
+ value: Some(write_day),
+ });
+ if let Some(captured_at) = &captured_at {
+ headers = headers.insert(Header {
+ key: "capture-timestamp-ms",
+ value: Some(captured_at),
+ });
+ }
+ let record = FutureRecord::to(topic)
+ .key(&message.key)
+ .payload(&message.payload)
+ .headers(headers);
+ match producer.send_result(record) {
+ Ok(delivery) => deliveries.push((message.index, delivery)),
+ Err((error, _)) => failed.add(message.index, error.to_string()),
+ }
+ }
+ tokio::spawn(async move {
+ for (index, delivery) in deliveries {
+ match delivery.await {
+ Ok(Ok(_)) => counter!("ai_research_v4_recorder_index_rows_total", "index" => index.as_str(), "outcome" => "delivered").increment(1),
+ Ok(Err((error, _))) => failed.add(index, error.to_string()),
+ Err(_) => failed.add(index, "the producer dropped the row".to_owned()),
+ }
+ }
+ failed.report();
+ });
+}
+
+/// The index rows of one flush that did not reach the topic. They are logged once per flush, not once per row.
+#[derive(Default)]
+struct Failures {
+ count: u64,
+ first: Option<String>,
+}
+
+impl Failures {
+ fn add(&mut self, index: Index, error: String) {
+ counter!("ai_research_v4_recorder_index_rows_total", "index" => index.as_str(), "outcome" => "failed").increment(1);
+ self.count += 1;
+ self.first.get_or_insert(error);
+ }
+
+ fn report(self) {
+ if let Some(error) = self.first {
+ tracing::warn!(rows = self.count, %error, "index rows were not delivered");
+ }
+ }
+}
+
#[cfg(test)]
mod tests {
+ use common_kafka::test::create_mock_kafka;
+ use rdkafka::consumer::{BaseConsumer, Consumer};
+ use rdkafka::message::{Headers, Message};
+ use rdkafka::{ClientConfig, Offset, TopicPartitionList};
+
use super::*;
+ const TOPIC: &str = "ai_research_v4_index";
+
#[test]
fn a_retry_reuses_the_compressed_block() {
let work = FlushWork::default();
@@ -330,4 +442,91 @@ mod tests {
let again = work.compressed(1, || panic!("compressed a second time"));
assert!(matches!(again, Err(WriteError::Seal(e)) if e == "sealing failed: once"));
}
+
+ fn message(index: Index, key: &str, captured_at_ms: Option<i64>) -> IndexMessage {
+ IndexMessage {
+ index,
+ key: key.to_owned(),
+ payload: format!(r#"{{"row":"{key}"}}"#).into_bytes(),
+ captured_at_ms,
+ }
+ }
+
+ #[tokio::test]
+ async fn sends_one_record_per_row_with_its_index_and_write_day() {
+ let (cluster, producer) = create_mock_kafka().await;
+ cluster.create_topic(TOPIC, 1, 1).unwrap();
+ send_index(
+ &producer,
+ TOPIC,
+ "2026-10-15",
+ vec![
+ message(Index::Blocks, "session-1", Some(1_000)),
+ message(Index::Images, "image:v4:7:2026-10:abc", None),
+ ],
+ );
+ producer
+ .flush(Timeout::After(Duration::from_secs(5)))
+ .unwrap();
+
+ let consumer: BaseConsumer = ClientConfig::new()
+ .set("bootstrap.servers", cluster.bootstrap_servers())
+ .set("group.id", "test")
+ .create()
+ .unwrap();
+ let mut assignment = TopicPartitionList::new();
+ assignment
+ .add_partition_offset(TOPIC, 0, Offset::Beginning)
+ .unwrap();
+ consumer.assign(&assignment).unwrap();
+ let mut received = Vec::new();
+ while received.len() < 2 {
+ let record = consumer
+ .poll(Duration::from_secs(5))
+ .expect("a record before the timeout")
+ .unwrap();
+ let headers: Vec<(String, String)> = record
+ .headers()
+ .map(|headers| {
+ headers
+ .iter()
+ .map(|h| {
+ (
+ h.key.to_owned(),
+ String::from_utf8(h.value.unwrap().to_vec()).unwrap(),
+ )
+ })
+ .collect()
+ })
+ .unwrap_or_default();
+ received.push((
+ String::from_utf8(record.key().unwrap().to_vec()).unwrap(),
+ String::from_utf8(record.payload().unwrap().to_vec()).unwrap(),
+ headers,
+ ));
+ }
+ let header = |key: &str, value: &str| (key.to_owned(), value.to_owned());
+ assert_eq!(
+ received,
+ [
+ (
+ "session-1".to_owned(),
+ r#"{"row":"session-1"}"#.to_owned(),
+ vec![
+ header("ml-index", "blocks"),
+ header("ml-write-day", "2026-10-15"),
+ header("capture-timestamp-ms", "1000"),
+ ]
+ ),
+ (
+ "image:v4:7:2026-10:abc".to_owned(),
+ r#"{"row":"image:v4:7:2026-10:abc"}"#.to_owned(),
+ vec![
+ header("ml-index", "images"),
+ header("ml-write-day", "2026-10-15"),
+ ]
+ ),
+ ]
+ );
+ }
}
diff --git a/rust/ai-research/replay-recorder-v4/src/index.rs b/rust/ai-research/replay-recorder-v4/src/index.rs
index e0bbfd06d231..675742df79a3 100644
--- a/rust/ai-research/replay-recorder-v4/src/index.rs
+++ b/rust/ai-research/replay-recorder-v4/src/index.rs
@@ -1,6 +1,7 @@
-//! The v4 index files: Parquet in `rrweb-v4/index/`. Column names are the ones in `replay-data.md`.
-//!
-//! Integer columns are signed, the types parquetjs writes, so a parquetjs writer can produce the same files.
+//! The v4 index rows. The recorder sends each row as one JSON message to the index topic, and the Parquet sink writes
+//! them to `rrweb-v4/index/`. The JSON keys are the Parquet column names in `replay-data.md`. Integer values fit the
+//! signed column types parquetjs writes, and an INT64 value stays in the range a JavaScript number holds exactly,
+//! which the sink checks: a value past either range saturates.
//!
//! A flush can store its blocks and index rows and then fail before it commits its Kafka offsets. The next attempt
//! for the partition, in this process or after a restart, starts again at the same committed offset and writes the
@@ -15,324 +16,115 @@
//! flush is whole when that many distinct `flush_row_index` values are present. Check this before any filter on
//! sessions or teams.
-use std::sync::Arc;
-
-use arrow_array::builder::{Int32Builder, Int64Builder, ListBuilder, StringBuilder, StructBuilder};
-use arrow_array::{ArrayRef, BooleanArray, Int32Array, Int64Array, RecordBatch, StringArray};
-use arrow_schema::{ArrowError, DataType, Field, Fields, Schema};
use replay_core_v4::inspect::session::Moment;
use replay_core_v4::inspect::stats::BlockStats;
+use serde::ser::{SerializeMap, Serializer};
+use serde::Serialize;
-use crate::batch::Origin;
-use crate::write::{FlushAttempt, Written};
-use parquet::arrow::ArrowWriter;
-use parquet::basic::Compression;
-use parquet::errors::ParquetError;
-use parquet::file::properties::WriterProperties;
+/// `Number.MAX_SAFE_INTEGER`. The sink refuses a row with an integer past it, and that row's flush is then never whole.
+const MAX_SAFE_INTEGER: i64 = (1 << 53) - 1;
-#[derive(Debug, thiserror::Error)]
-pub enum IndexError {
- #[error(transparent)]
- Arrow(#[from] ArrowError),
- #[error(transparent)]
- Parquet(#[from] ParquetError),
+fn int64(value: u64) -> i64 {
+ i64::try_from(value).map_or(MAX_SAFE_INTEGER, safe)
}
-/// Writes one Parquet file, snappy-compressed as the v3 catalog is.
-pub fn write_parquet(batch: &RecordBatch) -> Result<Vec<u8>, IndexError> {
- let properties = WriterProperties::builder()
- .set_compression(Compression::SNAPPY)
- .build();
- let mut out = Vec::new();
- let mut writer = ArrowWriter::try_new(&mut out, batch.schema(), Some(properties))?;
- writer.write(batch)?;
- writer.close()?;
- Ok(out)
+fn safe(value: i64) -> i64 {
+ value.clamp(-MAX_SAFE_INTEGER, MAX_SAFE_INTEGER)
}
-/// A count or size as its signed column. Real values fit, and a larger one saturates.
-fn int64(value: u64) -> i64 {
- i64::try_from(value).unwrap_or(i64::MAX)
+fn serialize_safe<S: Serializer>(value: &i64, serializer: S) -> Result<S::Ok, S::Error> {
+ serializer.serialize_i64(safe(*value))
}
-/// A count as its signed column. Real values fit, and a larger one saturates.
fn int32(value: u32) -> i32 {
i32::try_from(value).unwrap_or(i32::MAX)
}
-/// Where one inline image sits in a pack in `images-v4`.
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub struct ImageIndexRow {
- pub team_id: i64,
- pub key_month: String,
- pub image_ref: String,
- pub pack_s3_key: String,
- /// Inclusive byte range of the sealed image in the pack.
- pub byte_start: u64,
- pub byte_end: u64,
- pub mime: String,
- pub byte_size: u64,
-}
-
-pub fn image_index(rows: &[ImageIndexRow]) -> Result<RecordBatch, IndexError> {
- let schema = Schema::new(vec![
- Field::new("team_id", DataType::Int64, false),
- Field::new("key_month", DataType::Utf8, false),
- Field::new("image_ref", DataType::Utf8, false),
- Field::new("pack_s3_key", DataType::Utf8, false),
- Field::new("byte_start", DataType::Int64, false),
- Field::new("byte_end", DataType::Int64, false),
- Field::new("mime", DataType::Utf8, false),
- Field::new("byte_size", DataType::Int64, false),
- ]);
- let columns: Vec<ArrayRef> = vec![
- Arc::new(Int64Array::from_iter_values(rows.iter().map(|r| r.team_id))),
- Arc::new(StringArray::from_iter_values(
- rows.iter().map(|r| &r.key_month),
- )),
- Arc::new(StringArray::from_iter_values(
- rows.iter().map(|r| &r.image_ref),
- )),
- Arc::new(StringArray::from_iter_values(
- rows.iter().map(|r| &r.pack_s3_key),
- )),
- Arc::new(Int64Array::from_iter_values(
- rows.iter().map(|r| int64(r.byte_start)),
- )),
- Arc::new(Int64Array::from_iter_values(
- rows.iter().map(|r| int64(r.byte_end)),
- )),
- Arc::new(StringArray::from_iter_values(rows.iter().map(|r| &r.mime))),
- Arc::new(Int64Array::from_iter_values(
- rows.iter().map(|r| int64(r.byte_size)),
- )),
- ];
- Ok(RecordBatch::try_new(Arc::new(schema), columns)?)
-}
-
-/// One block's index row: where it is, what the inspector found, and its sealed URLs.
-pub struct BlockIndexRow<'a> {
- pub written: &'a Written,
- pub origin: &'a Origin,
- pub attempt: &'a FlushAttempt,
- pub stats: &'a BlockStats,
- pub moments: &'a [Moment],
- pub snapshot_source: Option<&'a str>,
- pub snapshot_library: Option<&'a str>,
- /// The block's visited URLs as a v4 JSON envelope under the session key.
- pub urls: Option<String>,
- /// The block rows this flush writes for the row's partition, and the row's place among them. With `flush_id`,
- /// `topic` and `partition`, the place names the row across retries of the same publish.
- pub flush_row_count: u32,
- pub flush_row_index: u32,
-}
-
-fn moment_fields() -> Fields {
- Fields::from(vec![
- Field::new("kind", DataType::Utf8, false),
- Field::new("window_id", DataType::Utf8, false),
- Field::new("ts_ms", DataType::Int64, false),
- Field::new("event_index", DataType::Int32, false),
- ])
+fn serialize_int64<S: Serializer>(value: &u64, serializer: S) -> Result<S::Ok, S::Error> {
+ serializer.serialize_i64(int64(*value))
}
-fn moments_column(rows: &[BlockIndexRow<'_>]) -> ArrayRef {
- let fields = moment_fields();
- let mut builder = ListBuilder::new(StructBuilder::from_fields(fields.clone(), rows.len()))
- .with_field(Field::new("item", DataType::Struct(fields), false));
- for row in rows {
- let entries = builder.values();
- for moment in row.moments {
- entries
- .field_builder::<StringBuilder>(0)
- .expect("kind")
- .append_value(moment.kind);
- entries
- .field_builder::<StringBuilder>(1)
- .expect("window_id")
- .append_value(&moment.window_id);
- entries
- .field_builder::<Int64Builder>(2)
- .expect("ts_ms")
- .append_value(moment.ts_ms);
- entries
- .field_builder::<Int32Builder>(3)
- .expect("event_index")
- .append_value(int32(moment.event_index));
- entries.append(true);
- }
- builder.append(true);
- }
- Arc::new(builder.finish())
+fn serialize_int32<S: Serializer>(value: &u32, serializer: S) -> Result<S::Ok, S::Error> {
+ serializer.serialize_i32(int32(*value))
}
-/// A list of strings per row, with no null lists or items.
-fn strings_column(rows: &[BlockIndexRow<'_>], values: fn(&BlockStats) -> &[String]) -> ArrayRef {
- let mut builder = ListBuilder::new(StringBuilder::new()).with_field(Field::new(
- "item",
- DataType::Utf8,
- false,
- ));
- for row in rows {
- for value in values(row.stats) {
- builder.values().append_value(value);
- }
- builder.append(true);
- }
- Arc::new(builder.finish())
-}
-
-pub fn block_index(rows: &[BlockIndexRow<'_>]) -> Result<RecordBatch, IndexError> {
- let mut fields = Vec::new();
- let mut columns: Vec<ArrayRef> = Vec::new();
- let mut push = |name: &str, data_type: DataType, nullable: bool, column: ArrayRef| {
- fields.push(Field::new(name, data_type, nullable));
- columns.push(column);
- };
- macro_rules! utf8 {
- ($name:literal, $value:expr) => {
- push(
- $name,
- DataType::Utf8,
- false,
- Arc::new(StringArray::from_iter_values(rows.iter().map($value))),
- )
- };
- }
- macro_rules! optional_utf8 {
- ($name:literal, $value:expr) => {
- push(
- $name,
- DataType::Utf8,
- true,
- Arc::new(rows.iter().map($value).collect::<StringArray>()),
- )
- };
- }
- macro_rules! stat {
- ($array:ident, $data_type:expr, $convert:expr, $($field:ident),*) => {
- $( push(stringify!($field), $data_type, false, Arc::new($array::from_iter_values(rows.iter().map(|r| $convert(r.stats.$field))))); )*
- };
- }
- macro_rules! flag {
- ($($field:ident),*) => {
- $( push(stringify!($field), DataType::Boolean, false, Arc::new(rows.iter().map(|r| Some(r.stats.$field)).collect::<BooleanArray>())); )*
+fn serialize_stats<S: Serializer>(stats: &&BlockStats, serializer: S) -> Result<S::Ok, S::Error> {
+ // Destructured in full, so a new stats field does not compile until it has a column type here.
+ let BlockStats {
+ first_ts_ms,
+ last_ts_ms,
+ event_count,
+ byte_size,
+ host,
+ full_snapshots,
+ events_before_first_full_snapshot,
+ orphan_mutations,
+ max_gap_ms,
+ dom_nodes,
+ dom_text_chars,
+ remote_stylesheets,
+ inline_stylesheets,
+ iframes,
+ canvases,
+ videos,
+ shadow_roots,
+ cold_start,
+ node_cap_reached,
+ active_ms,
+ clicks,
+ rage_clicks,
+ dead_clicks,
+ keypresses,
+ scrolls,
+ mouse_moves,
+ touches,
+ selections,
+ media_events,
+ viewport_resizes,
+ navigations,
+ console_errors,
+ console_warnings,
+ network_requests,
+ network_failures,
+ viewport_width,
+ viewport_height,
+ viewport_ts_ms,
+ touch_input,
+ images_extracted,
+ image_bytes,
+ images_inline,
+ image_urls,
+ written_jsonl_bytes,
+ max_written_event_bytes,
+ decode_failures,
+ moments_truncated,
+ mouse_interactions,
+ touch_moves,
+ url_fingerprints,
+ window_ids,
+ } = *stats;
+ let mut map = serializer.serialize_map(None)?;
+ macro_rules! columns {
+ ($convert:expr => $($field:ident),*) => {
+ $( map.serialize_entry(stringify!($field), &$convert($field))?; )*
};
}
-
- push(
- "team_id",
- DataType::Int64,
- false,
- Arc::new(Int64Array::from_iter_values(
- rows.iter().map(|r| r.written.session.team_id),
- )),
- );
- utf8!("session_id", |r| r.written.session.session_id.clone());
- utf8!("key_month", |r| r.written.key_month.clone());
- utf8!("block_s3_key", |r| r.written.object_key.clone());
- utf8!("topic", |r| r.origin.tp.topic);
- push(
- "partition",
- DataType::Int32,
- false,
- Arc::new(Int32Array::from_iter_values(
- rows.iter().map(|r| r.origin.tp.partition),
- )),
- );
- push(
- "start_offset",
- DataType::Int64,
- false,
- Arc::new(Int64Array::from_iter_values(
- rows.iter().map(|r| r.origin.start_offset),
- )),
- );
- push(
- "end_offset",
- DataType::Int64,
- false,
- Arc::new(Int64Array::from_iter_values(
- rows.iter().map(|r| r.origin.end_offset),
- )),
- );
- utf8!("flush_id", |r| r.attempt.flush_id.clone());
- push(
- "flushed_at_ms",
- DataType::Int64,
- false,
- Arc::new(Int64Array::from_iter_values(
- rows.iter().map(|r| r.attempt.flushed_at_ms),
- )),
- );
- push(
- "flush_end_offset",
- DataType::Int64,
- false,
- Arc::new(Int64Array::from_iter_values(
- rows.iter().map(|r| r.origin.flush_end_offset),
- )),
- );
- push(
- "flush_row_count",
- DataType::Int32,
- false,
- Arc::new(Int32Array::from_iter_values(
- rows.iter().map(|r| int32(r.flush_row_count)),
- )),
- );
- push(
- "flush_row_index",
- DataType::Int32,
- false,
- Arc::new(Int32Array::from_iter_values(
- rows.iter().map(|r| int32(r.flush_row_index)),
- )),
- );
- push(
- "block_byte_start",
- DataType::Int64,
- false,
- Arc::new(Int64Array::from_iter_values(
- rows.iter().map(|r| int64(r.written.byte_start)),
- )),
- );
- push(
- "block_byte_end",
- DataType::Int64,
- false,
- Arc::new(Int64Array::from_iter_values(
- rows.iter().map(|r| int64(r.written.byte_end)),
- )),
- );
- push(
- "first_ts_ms",
- DataType::Int64,
- true,
- Arc::new(
- rows.iter()
- .map(|r| r.stats.first_ts_ms)
- .collect::<Int64Array>(),
- ),
- );
- push(
- "last_ts_ms",
- DataType::Int64,
- true,
- Arc::new(
- rows.iter()
- .map(|r| r.stats.last_ts_ms)
- .collect::<Int64Array>(),
- ),
+ columns!(|v: &Option<i64>| v.map(safe) => first_ts_ms, last_ts_ms, viewport_ts_ms);
+ columns!(|v: &i64| safe(*v) => max_gap_ms, active_ms);
+ columns!(
+ |v: &u64| int64(*v) =>
+ event_count,
+ byte_size,
+ dom_text_chars,
+ image_bytes,
+ written_jsonl_bytes,
+ max_written_event_bytes
);
- stat!(Int64Array, DataType::Int64, int64, event_count, byte_size);
- optional_utf8!("snapshot_source", |r| r.snapshot_source);
- optional_utf8!("snapshot_library", |r| r.snapshot_library);
- optional_utf8!("host", |r| r.stats.host.as_deref());
- stat!(
- Int32Array,
- DataType::Int32,
- int32,
+ columns!(|v: &bool| *v => cold_start, node_cap_reached, moments_truncated, touch_input);
+ columns!(|v: &Vec<String>| v.clone() => url_fingerprints, window_ids);
+ columns!(|v: &Option<u32>| v.map(int32) => viewport_width, viewport_height);
+ columns!(
+ |v: &u32| int32(*v) =>
full_snapshots,
events_before_first_full_snapshot,
orphan_mutations,
@@ -343,21 +135,7 @@ pub fn block_index(rows: &[BlockIndexRow<'_>]) -> Result<RecordBatch, IndexError
iframes,
canvases,
videos,
- shadow_roots
- );
- stat!(
- Int64Array,
- DataType::Int64,
- i64::from,
- max_gap_ms,
- active_ms
- );
- stat!(Int64Array, DataType::Int64, int64, dom_text_chars);
- flag!(cold_start, node_cap_reached, moments_truncated, touch_input);
- stat!(
- Int32Array,
- DataType::Int32,
- int32,
+ shadow_roots,
clicks,
mouse_interactions,
rage_clicks,
@@ -379,72 +157,355 @@ pub fn block_index(rows: &[BlockIndexRow<'_>]) -> Result<RecordBatch, IndexError
images_inline,
image_urls
);
- stat!(
- Int64Array,
- DataType::Int64,
- int64,
- image_bytes,
- written_jsonl_bytes,
- max_written_event_bytes
- );
- push(
- "viewport_width",
- DataType::Int32,
- true,
- Arc::new(
- rows.iter()
- .map(|r| r.stats.viewport_width.map(int32))
- .collect::<Int32Array>(),
- ),
- );
- push(
- "viewport_height",
- DataType::Int32,
- true,
- Arc::new(
- rows.iter()
- .map(|r| r.stats.viewport_height.map(int32))
- .collect::<Int32Array>(),
- ),
- );
- push(
- "viewport_ts_ms",
- DataType::Int64,
- true,
- Arc::new(
- rows.iter()
- .map(|r| r.stats.viewport_ts_ms)
- .collect::<Int64Array>(),
- ),
- );
- let moments = moments_column(rows);
- push("moments", moments.data_type().clone(), false, moments);
- let fingerprints = strings_column(rows, |s| &s.url_fingerprints);
- push(
- "url_fingerprints",
- fingerprints.data_type().clone(),
- false,
- fingerprints,
- );
- let windows = strings_column(rows, |s| &s.window_ids);
- push("window_ids", windows.data_type().clone(), false, windows);
- optional_utf8!("urls", |r| r.urls.as_deref());
- Ok(RecordBatch::try_new(
- Arc::new(Schema::new(fields)),
- columns,
- )?)
+ map.serialize_entry("host", host)?;
+ map.end()
+}
+
+fn serialize_moments<S: Serializer>(moments: &&[Moment], serializer: S) -> Result<S::Ok, S::Error> {
+ #[derive(Serialize)]
+ struct Entry<'a> {
+ kind: &'a str,
+ window_id: &'a str,
+ ts_ms: i64,
+ event_index: i32,
+ }
+ serializer.collect_seq(moments.iter().map(|moment| Entry {
+ kind: moment.kind,
+ window_id: &moment.window_id,
+ ts_ms: safe(moment.ts_ms),
+ event_index: int32(moment.event_index),
+ }))
+}
+
+/// Where one inline image sits in a pack in `images-v4`.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
+pub struct ImageIndexRow {
+ #[serde(serialize_with = "serialize_safe")]
+ pub team_id: i64,
+ pub key_month: String,
+ pub image_ref: String,
+ pub pack_s3_key: String,
+ /// Inclusive byte range of the sealed image in the pack.
+ #[serde(serialize_with = "serialize_int64")]
+ pub byte_start: u64,
+ #[serde(serialize_with = "serialize_int64")]
+ pub byte_end: u64,
+ pub mime: String,
+ #[serde(serialize_with = "serialize_int64")]
+ pub byte_size: u64,
+}
+
+/// One block's index row: where it is, what the inspector found, and its sealed URLs.
+#[derive(Debug, Serialize)]
+pub struct BlockIndexRow<'a> {
+ #[serde(serialize_with = "serialize_safe")]
+ pub team_id: i64,
+ pub session_id: &'a str,
+ pub key_month: &'a str,
+ pub block_s3_key: &'a str,
+ pub topic: &'static str,
+ pub partition: i32,
+ #[serde(serialize_with = "serialize_safe")]
+ pub start_offset: i64,
+ #[serde(serialize_with = "serialize_safe")]
+ pub end_offset: i64,
+ /// The last offset the flush covers on the partition, a message that gave no row included.
+ #[serde(serialize_with = "serialize_safe")]
+ pub flush_end_offset: i64,
+ /// The block rows this flush writes for the partition, and this row's place among them. With `flush_id`,
+ /// `topic` and `partition`, the place names the row across retries of the same publish.
+ #[serde(serialize_with = "serialize_int32")]
+ pub flush_row_count: u32,
+ #[serde(serialize_with = "serialize_int32")]
+ pub flush_row_index: u32,
+ pub flush_id: &'a str,
+ #[serde(serialize_with = "serialize_safe")]
+ pub flushed_at_ms: i64,
+ /// Inclusive byte range of the sealed frame in the block object.
+ #[serde(serialize_with = "serialize_int64")]
+ pub block_byte_start: u64,
+ #[serde(serialize_with = "serialize_int64")]
+ pub block_byte_end: u64,
+ #[serde(flatten, serialize_with = "serialize_stats")]
+ pub stats: &'a BlockStats,
+ pub snapshot_source: Option<&'a str>,
+ pub snapshot_library: Option<&'a str>,
+ #[serde(serialize_with = "serialize_moments")]
+ pub moments: &'a [Moment],
+ /// The block's visited URLs as a v4 JSON envelope under the session key.
+ pub urls: Option<String>,
+ /// Some visited URLs are not in `urls`: the block had more, or longer, than an index row keeps.
+ pub urls_truncated: bool,
+ /// Goes in the `capture-timestamp-ms` header, not in the row.
+ #[serde(skip)]
+ pub captured_at_ms: Option<i64>,
+}
+
+/// The index a row belongs to, as the `ml-index` header names it.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum Index {
+ Blocks,
+ Images,
+}
+
+impl Index {
+ pub fn as_str(self) -> &'static str {
+ match self {
+ Index::Blocks => "blocks",
+ Index::Images => "images",
+ }
+ }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct IndexMessage {
+ pub index: Index,
+ /// The session ID of a block row, the image reference of an image row.
+ pub key: String,
+ pub payload: Vec<u8>,
+ pub captured_at_ms: Option<i64>,
+}
+
+impl IndexMessage {
+ pub fn block(row: &BlockIndexRow<'_>) -> serde_json::Result<Self> {
+ Ok(Self {
+ index: Index::Blocks,
+ key: row.session_id.to_owned(),
+ payload: serde_json::to_vec(row)?,
+ captured_at_ms: row.captured_at_ms,
+ })
+ }
+
+ pub fn image(row: &ImageIndexRow, captured_at_ms: Option<i64>) -> serde_json::Result<Self> {
+ Ok(Self {
+ index: Index::Images,
+ key: row.image_ref.clone(),
+ payload: serde_json::to_vec(row)?,
+ captured_at_ms,
+ })
+ }
}
#[cfg(test)]
mod tests {
- use bytes::Bytes;
- use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
+ use serde_json::{json, Value};
use super::*;
+ /// The columns of the Parquet files the recorder wrote before the sink took them over, in file order, with their
+ /// parquetjs type and whether they are optional. The sink writes the same files, so a renamed field or a changed
+ /// type here breaks every reader.
+ const BLOCK_COLUMNS: &[(&str, &str, bool)] = &[
+ ("team_id", "INT64", false),
+ ("session_id", "UTF8", false),
+ ("key_month", "UTF8", false),
+ ("block_s3_key", "UTF8", false),
+ ("topic", "UTF8", false),
+ ("partition", "INT32", false),
+ ("start_offset", "INT64", false),
+ ("end_offset", "INT64", false),
+ ("flush_end_offset", "INT64", false),
+ ("flush_row_count", "INT32", false),
+ ("flush_row_index", "INT32", false),
+ ("flush_id", "UTF8", false),
+ ("flushed_at_ms", "INT64", false),
+ ("block_byte_start", "INT64", false),
+ ("block_byte_end", "INT64", false),
+ ("first_ts_ms", "INT64", true),
+ ("last_ts_ms", "INT64", true),
+ ("event_count", "INT64", false),
+ ("byte_size", "INT64", false),
+ ("snapshot_source", "UTF8", true),
+ ("snapshot_library", "UTF8", true),
+ ("host", "UTF8", true),
+ ("full_snapshots", "INT32", false),
+ ("events_before_first_full_snapshot", "INT32", false),
+ ("orphan_mutations", "INT32", false),
+ ("decode_failures", "INT32", false),
+ ("dom_nodes", "INT32", false),
+ ("remote_stylesheets", "INT32", false),
+ ("inline_stylesheets", "INT32", false),
+ ("iframes", "INT32", false),
+ ("canvases", "INT32", false),
+ ("videos", "INT32", false),
+ ("shadow_roots", "INT32", false),
+ ("max_gap_ms", "INT64", false),
+ ("active_ms", "INT64", false),
+ ("dom_text_chars", "INT64", false),
+ ("cold_start", "BOOLEAN", false),
+ ("node_cap_reached", "BOOLEAN", false),
+ ("moments_truncated", "BOOLEAN", false),
+ ("touch_input", "BOOLEAN", false),
+ ("clicks", "INT32", false),
+ ("mouse_interactions", "INT32", false),
+ ("rage_clicks", "INT32", false),
+ ("dead_clicks", "INT32", false),
+ ("keypresses", "INT32", false),
+ ("scrolls", "INT32", false),
+ ("mouse_moves", "INT32", false),
+ ("touches", "INT32", false),
+ ("touch_moves", "INT32", false),
+ ("selections", "INT32", false),
+ ("media_events", "INT32", false),
+ ("viewport_resizes", "INT32", false),
+ ("navigations", "INT32", false),
+ ("console_errors", "INT32", false),
+ ("console_warnings", "INT32", false),
+ ("network_requests", "INT32", false),
+ ("network_failures", "INT32", false),
+ ("images_extracted", "INT32", false),
+ ("images_inline", "INT32", false),
+ ("image_urls", "INT32", false),
+ ("image_bytes", "INT64", false),
+ ("written_jsonl_bytes", "INT64", false),
+ ("max_written_event_bytes", "INT64", false),
+ ("viewport_width", "INT32", true),
+ ("viewport_height", "INT32", true),
+ ("viewport_ts_ms", "INT64", true),
+ ("moments", "MOMENTS", false),
+ ("url_fingerprints", "LIST<UTF8>", false),
+ ("window_ids", "LIST<UTF8>", false),
+ ("urls", "UTF8", true),
+ ("urls_truncated", "BOOLEAN", false),
+ ];
+
+ const IMAGE_COLUMNS: &[(&str, &str, bool)] = &[
+ ("team_id", "INT64", false),
+ ("key_month", "UTF8", false),
+ ("image_ref", "UTF8", false),
+ ("pack_s3_key", "UTF8", false),
+ ("byte_start", "INT64", false),
+ ("byte_end", "INT64", false),
+ ("mime", "UTF8", false),
+ ("byte_size", "INT64", false),
+ ];
+
+ fn fits(value: &Value, column_type: &str) -> bool {
+ match column_type {
+ "UTF8" => value.is_string(),
+ "BOOLEAN" => value.is_boolean(),
+ "INT64" => value.as_i64().is_some_and(|n| n.abs() <= MAX_SAFE_INTEGER),
+ "INT32" => value.as_i64().is_some_and(|n| i32::try_from(n).is_ok()),
+ "LIST<UTF8>" => value
+ .as_array()
+ .is_some_and(|items| items.iter().all(|item| fits(item, "UTF8"))),
+ "MOMENTS" => value.as_array().is_some_and(|moments| {
+ moments.iter().all(|moment| {
+ let moment = moment.as_object().unwrap();
+ moment.len() == 4
+ && fits(&moment["kind"], "UTF8")
+ && fits(&moment["window_id"], "UTF8")
+ && fits(&moment["ts_ms"], "INT64")
+ && fits(&moment["event_index"], "INT32")
+ })
+ }),
+ other => panic!("no column type {other}"),
+ }
+ }
+
+ /// Parses a row and checks it has exactly the columns, each value of its column's type.
+ fn assert_columns(payload: &[u8], columns: &[(&str, &str, bool)]) -> Value {
+ let value: Value = serde_json::from_slice(payload).unwrap();
+ let row = value.as_object().unwrap();
+ let mut keys: Vec<&str> = row.keys().map(String::as_str).collect();
+ keys.sort_unstable();
+ let mut names: Vec<&str> = columns.iter().map(|(name, _, _)| *name).collect();
+ names.sort_unstable();
+ assert_eq!(keys, names);
+ for (name, column_type, optional) in columns {
+ let cell = &row[*name];
+ assert!(
+ (*optional && cell.is_null()) || fits(cell, column_type),
+ "{name} = {cell} is not {column_type}"
+ );
+ }
+ value
+ }
+
#[test]
- fn image_index_reads_back_with_the_documented_columns() {
- let rows = vec![ImageIndexRow {
+ fn rows_have_exactly_the_parquet_columns_and_types() {
+ let stats = BlockStats {
+ // 2^53, and a 1e100 timestamp read as an integer: past what JavaScript holds exactly.
+ first_ts_ms: Some(1 << 53),
+ max_gap_ms: i64::MIN,
+ event_count: u64::MAX,
+ clicks: u32::MAX,
+ viewport_width: Some(u32::MAX),
+ written_jsonl_bytes: u64::MAX,
+ url_fingerprints: vec!["b08517cee3ff5ecc".to_owned()],
+ window_ids: vec!["w1".to_owned(), "w2".to_owned()],
+ ..BlockStats::default()
+ };
+ let moments = [Moment {
+ kind: "full_snapshot",
+ window_id: "w1".into(),
+ ts_ms: 1e100 as i64,
+ event_index: u32::MAX,
+ }];
+ let block = BlockIndexRow {
+ team_id: 7,
+ session_id: "s",
+ key_month: "2026-09",
+ block_s3_key: "blocks/dt=2026-10-15/1-0",
+ topic: "main",
+ partition: 2,
+ start_offset: 100,
+ end_offset: 105,
+ flush_end_offset: 107,
+ flush_row_count: u32::MAX,
+ flush_row_index: 0,
+ flush_id: "0000000000000001",
+ flushed_at_ms: 1_000,
+ block_byte_start: 0,
+ block_byte_end: u64::MAX,
+ stats: &stats,
+ snapshot_source: Some("web"),
+ snapshot_library: None,
+ moments: &moments,
+ urls: None,
+ urls_truncated: false,
+ captured_at_ms: Some(5),
+ };
+ let message = IndexMessage::block(&block).unwrap();
+ let value = assert_columns(&message.payload, BLOCK_COLUMNS);
+ assert_eq!(
+ [
+ &value["first_ts_ms"],
+ &value["max_gap_ms"],
+ &value["last_ts_ms"],
+ &value["cold_start"],
+ &value["snapshot_library"],
+ &value["event_count"],
+ &value["block_byte_end"],
+ &value["clicks"],
+ &value["viewport_width"],
+ &value["written_jsonl_bytes"],
+ &value["flush_row_count"],
+ &value["url_fingerprints"],
+ &value["window_ids"],
+ ],
+ [
+ &json!(MAX_SAFE_INTEGER),
+ &json!(-MAX_SAFE_INTEGER),
+ &Value::Null,
+ &json!(false),
+ &Value::Null,
+ &json!(MAX_SAFE_INTEGER),
+ &json!(MAX_SAFE_INTEGER),
+ &json!(i32::MAX),
+ &json!(i32::MAX),
+ &json!(MAX_SAFE_INTEGER),
+ &json!(i32::MAX),
+ &json!(["b08517cee3ff5ecc"]),
+ &json!(["w1", "w2"]),
+ ]
+ );
+ assert_eq!(
+ value["moments"],
+ json!([{"kind": "full_snapshot", "window_id": "w1", "ts_ms": MAX_SAFE_INTEGER, "event_index": i32::MAX}])
+ );
+
+ let image = ImageIndexRow {
team_id: 7,
key_month: "2026-10".to_owned(),
image_ref: "image:v4:7:2026-10:abc".to_owned(),
@@ -453,44 +514,9 @@ mod tests {
byte_end: 99,
mime: "image/png".to_owned(),
byte_size: u64::MAX,
- }];
- let file = write_parquet(&image_index(&rows).unwrap()).unwrap();
- let mut reader = ParquetRecordBatchReaderBuilder::try_new(Bytes::from(file))
- .unwrap()
- .build()
- .unwrap();
- let batch = reader.next().unwrap().unwrap();
- let columns: Vec<_> = batch
- .schema()
- .fields()
- .iter()
- .map(|f| (f.name().clone(), f.data_type().clone(), f.is_nullable()))
- .collect();
- let column = |name: &str, data_type| (name.to_owned(), data_type, false);
- assert_eq!(
- columns,
- [
- column("team_id", DataType::Int64),
- column("key_month", DataType::Utf8),
- column("image_ref", DataType::Utf8),
- column("pack_s3_key", DataType::Utf8),
- column("byte_start", DataType::Int64),
- column("byte_end", DataType::Int64),
- column("mime", DataType::Utf8),
- column("byte_size", DataType::Int64),
- ]
- );
- let sizes = batch
- .column(7)
- .as_any()
- .downcast_ref::<Int64Array>()
- .unwrap();
- assert_eq!(sizes.value(0), i64::MAX);
- let refs = batch
- .column(2)
- .as_any()
- .downcast_ref::<StringArray>()
- .unwrap();
- assert_eq!(refs.value(0), "image:v4:7:2026-10:abc");
+ };
+ let message = IndexMessage::image(&image, None).unwrap();
+ let value = assert_columns(&message.payload, IMAGE_COLUMNS);
+ assert_eq!(value["byte_size"], json!(MAX_SAFE_INTEGER));
}
}
diff --git a/rust/ai-research/replay-recorder-v4/src/main.rs b/rust/ai-research/replay-recorder-v4/src/main.rs
index d30f7fb97b72..5110d7a9d44f 100644
--- a/rust/ai-research/replay-recorder-v4/src/main.rs
+++ b/rust/ai-research/replay-recorder-v4/src/main.rs
@@ -18,13 +18,17 @@ use replay_core_v4::keys::kms::{AwsKms, CachedKms, KmsLimits};
use replay_core_v4::keys::table::{AwsKeyTable, CachedKeyTable};
use replay_recorder_v4::config::Config;
use replay_recorder_v4::consumer::{run, RecorderConsumer, Settings, Sources};
-use replay_recorder_v4::flush::Writer;
+use replay_recorder_v4::flush::{Writer, PRODUCER_FLUSH_ON_SHUTDOWN};
use replay_recorder_v4::limits::{Restrictions, SessionLimiter};
use replay_recorder_v4::store::S3Store;
use replay_recorder_v4::teams::TeamCache;
common_alloc::used!();
+/// The index producer's local queue. A block row is a few KB, so this holds several large flushes.
+const INDEX_QUEUE_MIB: u32 = 512;
+const INDEX_QUEUE_MESSAGES: u32 = 500_000;
+
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
let config = Config::init_from_env().expect("invalid configuration");
@@ -39,27 +43,33 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
.init();
tracing::info!(topic = %config.kafka_topic, overflow_topic = %config.kafka_overflow_topic, group = %config.kafka_consumer_group, sample = config.sample_percentage, "starting replay-recorder-v4");
+ // The consumer flushes the buffer, then the producer, on shutdown.
+ let consumer_shutdown =
+ Duration::from_secs(config.shutdown_flush_secs + 5) + PRODUCER_FLUSH_ON_SHUTDOWN;
let mut manager = Manager::builder("replay-recorder-v4")
- .with_global_shutdown_timeout(Duration::from_secs(config.shutdown_flush_secs + 15))
+ .with_global_shutdown_timeout(consumer_shutdown + Duration::from_secs(10))
.build();
let consumer_handle = manager.register(
"consumer",
ComponentOptions::new()
- .with_graceful_shutdown(Duration::from_secs(config.shutdown_flush_secs + 5))
+ .with_graceful_shutdown(consumer_shutdown)
.with_liveness_deadline(Duration::from_secs(60)),
);
let restrictions_handle = manager.register(
"event-restrictions",
ComponentOptions::new().with_graceful_shutdown(Duration::from_secs(5)),
);
- // The producer reports itself live from librdkafka's statistics callback.
- let producer_handle = config.write_enabled.then(|| {
- manager.register(
- "image-fetch-producer",
- ComponentOptions::new()
- .with_graceful_shutdown(Duration::from_secs(5))
- .with_liveness_deadline(Duration::from_secs(60)),
- )
+ // A producer reports itself live from librdkafka's statistics callback.
+ let producer_handles = config.write_enabled.then(|| {
+ let mut register = |name: &str| {
+ manager.register(
+ name,
+ ComponentOptions::new()
+ .with_graceful_shutdown(Duration::from_secs(5))
+ .with_liveness_deadline(Duration::from_secs(60)),
+ )
+ };
+ (register("image-fetch-producer"), register("index-producer"))
});
let metrics_handle = manager.register(
"metrics-server",
@@ -164,7 +174,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
redirecting_topic: (config.ingestion_overflow_mode == "redirect")
.then(|| config.kafka_topic.clone()),
};
- let writer = if let Some(producer_handle) = producer_handle {
+ let writer = if let Some((producer_handle, index_producer_handle)) = producer_handles {
let aws = aws_config::defaults(aws_config::BehaviorVersion::latest())
.load()
.await;
@@ -202,6 +212,19 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
)
.await?,
image_fetch_topic: config.image_fetch_topic.clone(),
+ index_producer: create_kafka_producer(
+ &KafkaConfig {
+ kafka_compression_codec: "lz4".to_owned(),
+ kafka_message_timeout_ms: config.index_delivery_timeout_ms,
+ // Rows of a few flushes, so a slow broker holds them here instead of losing them at once.
+ kafka_producer_queue_mib: INDEX_QUEUE_MIB,
+ kafka_producer_queue_messages: INDEX_QUEUE_MESSAGES,
+ ..config.kafka.clone()
+ },
+ index_producer_handle,
+ )
+ .await?,
+ index_topic: config.index_topic.clone(),
sent_urls: replay_recorder_v4::urls::sent_urls(500_000),
seal_permits: Arc::new(tokio::sync::Semaphore::new(settings.flush_concurrency)),
})
diff --git a/rust/ai-research/replay-recorder-v4/src/urls.rs b/rust/ai-research/replay-recorder-v4/src/urls.rs
index af25d0355791..99143d4bd782 100644
--- a/rust/ai-research/replay-recorder-v4/src/urls.rs
+++ b/rust/ai-research/replay-recorder-v4/src/urls.rs
@@ -207,6 +207,7 @@ mod tests {
image_urls: urls.iter().map(|u| (*u).to_owned()).collect(),
remote_urls: Vec::new(),
url_limit: None,
+ captured_at_ms: None,
};
let mut stats = BlockStats::default();
let mut keys = FlushKeys::default();
diff --git a/rust/ai-research/replay-recorder-v4/src/write.rs b/rust/ai-research/replay-recorder-v4/src/write.rs
index 056f7a419b12..248e3b256b39 100644
--- a/rust/ai-research/replay-recorder-v4/src/write.rs
+++ b/rust/ai-research/replay-recorder-v4/src/write.rs
@@ -1,7 +1,7 @@
-//! Writes a flush: image packs and their index first, then one object of sealed session blocks, then their index.
+//! Writes a flush: image packs first, then one object of sealed session blocks, and builds the index rows for both.
//!
//! The order matters for a crash between writes. A pack without its blocks is unreferenced, which is harmless. A block
-//! whose images were never stored would hold references that resolve to nothing.
+//! whose images were never stored would hold references that resolve to nothing. The rows go out only after both.
use std::collections::{BTreeMap, HashMap, HashSet};
use std::io::{self, Write};
@@ -9,6 +9,7 @@ use std::sync::Arc;
use chrono::{DateTime, Utc};
use futures::stream::{self, StreamExt, TryStreamExt};
+use metrics::counter;
use rand::Rng;
use replay_core_v4::bytescan::write_json_string;
use replay_core_v4::envelope::{seal_frame, seal_json_envelope, Codec, Context, Version};
@@ -16,7 +17,6 @@ use replay_core_v4::inspect::events::event_page_url;
use replay_core_v4::inspect::stats::BlockStats;
use replay_core_v4::inspect::stats::MAX_URL_FINGERPRINTS;
use replay_core_v4::keys::batch::{FlushKeys, SessionId, SessionKey as Key};
-use replay_core_v4::keys::rows::write_day;
use replay_core_v4::refs::url_fingerprint;
use replay_core_v4::scan::cv::extract_event_images;
use replay_core_v4::scan::images::{Image, ImageLimits, ImageSink};
@@ -24,10 +24,8 @@ use replay_core_v4::seal::{image_content_key, url_fingerprint_key};
use tokio::sync::Semaphore;
-use crate::batch::{Block, SessionKey, Tp};
-use crate::index::{
- block_index, image_index, write_parquet, BlockIndexRow, ImageIndexRow, IndexError,
-};
+use crate::batch::{Block, SessionKey};
+use crate::index::{BlockIndexRow, ImageIndexRow};
use crate::rewrite::{canonical_strings, UrlRewrite};
use crate::store::{BlobStore, StoreError};
@@ -41,8 +39,6 @@ const BLOCK_WRITE_BUFFER: usize = 256 * 1024;
pub enum WriteError {
#[error(transparent)]
Store(#[from] StoreError),
- #[error(transparent)]
- Index(#[from] IndexError),
#[error("sealing failed: {0}")]
Seal(String),
#[error("sealing panicked: {0}")]
@@ -341,8 +337,8 @@ fn seal_pack(
Ok((pack, rows))
}
-/// Seals each team's images under its team month key into one pack, stores the packs concurrently, then writes the
-/// image index for the flush.
+/// Seals each team's images under its team month key into one pack and stores the packs concurrently. Returns the
+/// image index rows of the stored packs.
#[allow(clippy::too_many_arguments)]
pub async fn write_images(
images: BTreeMap<i64, Vec<Arc<Image>>>,
@@ -350,7 +346,6 @@ pub async fn write_images(
write_day: &str,
key_month: &str,
images_bucket: &str,
- rrweb_bucket: &str,
store: &Arc<dyn BlobStore>,
at: DateTime<Utc>,
concurrency: usize,
@@ -377,20 +372,13 @@ pub async fn write_images(
})
})
.collect();
- let rows: Vec<ImageIndexRow> = stream::iter(packs)
+ Ok(stream::iter(packs)
.buffer_unordered(concurrency.max(1))
.try_collect::<Vec<_>>()
.await?
.into_iter()
.flatten()
- .collect();
- if !rows.is_empty() {
- let index_key = format!("index/images/dt={write_day}/{}.parquet", unique_suffix(at));
- let batch = image_index(&rows)?;
- let file = blocking(move || write_parquet(&batch).map_err(WriteError::from)).await?;
- store.put_new(rrweb_bucket, &index_key, file).await?;
- }
- Ok(rows)
+ .collect())
}
/// Writes the flush's sealed blocks as one object. Returns where each landed, in the same order.
@@ -466,6 +454,33 @@ pub fn block_urls(block: &Block) -> Vec<String> {
urls
}
+/// The most visited pages a block index row keeps, the longest page it keeps and their bytes together. The sealed list
+/// then stays far below the index producer's message limit of 1,000,000 bytes with the row's other columns, and a
+/// Parquet page of these values stays small.
+const MAX_INDEX_URLS: usize = 100;
+const MAX_INDEX_URL_BYTES: usize = 2048;
+const MAX_INDEX_URLS_BYTES: usize = 32 * 1024;
+
+/// The visited pages that fit in a block index row, in order, and whether any page was left out.
+pub fn index_urls(urls: Vec<String>) -> (Vec<String>, bool) {
+ let mut kept = Vec::new();
+ let mut bytes = 0;
+ let mut truncated = false;
+ for url in urls {
+ if kept.len() >= MAX_INDEX_URLS || bytes + url.len() > MAX_INDEX_URLS_BYTES {
+ truncated = true;
+ break;
+ }
+ if url.len() > MAX_INDEX_URL_BYTES {
+ truncated = true;
+ continue;
+ }
+ bytes += url.len();
+ kept.push(url);
+ }
+ (kept, truncated)
+}
+
/// The block's distinct pages as fingerprints under the session's fingerprint key, the first
/// [`MAX_URL_FINGERPRINTS`] in the order the block saw them.
pub fn url_fingerprints(session_key: &[u8; 32], urls: &[String]) -> Vec<String> {
@@ -485,14 +500,14 @@ pub fn url_fingerprints(session_key: &[u8; 32], urls: &[String]) -> Vec<String>
/// Numbers the rows of each partition in order and gives each row the count of its partition.
fn number_rows(rows: &mut [BlockIndexRow<'_>]) {
- let mut counts: HashMap<Tp, u32> = HashMap::new();
+ let mut counts: HashMap<(&str, i32), u32> = HashMap::new();
for row in rows.iter_mut() {
- let count = counts.entry(row.origin.tp).or_default();
+ let count = counts.entry((row.topic, row.partition)).or_default();
row.flush_row_index = *count;
*count += 1;
}
for row in rows.iter_mut() {
- row.flush_row_count = counts[&row.origin.tp];
+ row.flush_row_count = counts[&(row.topic, row.partition)];
}
}
@@ -512,16 +527,13 @@ impl FlushAttempt {
}
}
-/// Writes one block index file for the flush, after its blocks, so no row points at a block that is not stored.
+/// The flush's block index rows, built after its blocks are stored, so no row points at a block that is not.
/// Each entry is a block, its stats as this attempt filled them, and where it landed.
-pub async fn write_block_index(
- entries: &[(&Block, &BlockStats, &Written)],
+pub fn block_index_rows<'a>(
+ entries: &[(&'a Block, &'a BlockStats, &'a Written)],
keys: &FlushKeys,
- attempt: &FlushAttempt,
- rrweb_bucket: &str,
- store: &dyn BlobStore,
- at: DateTime<Utc>,
-) -> Result<usize, WriteError> {
+ attempt: &'a FlushAttempt,
+) -> Result<Vec<BlockIndexRow<'a>>, WriteError> {
let mut rows = Vec::with_capacity(entries.len());
for &(block, stats, written) in entries {
let id = SessionId {
@@ -531,7 +543,10 @@ pub async fn write_block_index(
let Some(key) = keys.sessions.get(&id) else {
continue;
};
- let urls = block_urls(block);
+ let (urls, urls_truncated) = index_urls(block_urls(block));
+ if urls_truncated {
+ counter!("ai_research_v4_recorder_index_urls_truncated_total").increment(1);
+ }
let urls = if urls.is_empty() {
None
} else {
@@ -552,32 +567,32 @@ pub async fn write_block_index(
};
let first = block.messages.first();
rows.push(BlockIndexRow {
- written,
- origin: &block.origin,
- attempt,
+ team_id: written.session.team_id,
+ session_id: &written.session.session_id,
+ key_month: &written.key_month,
+ block_s3_key: &written.object_key,
+ topic: block.origin.tp.topic,
+ partition: block.origin.tp.partition,
+ start_offset: block.origin.start_offset,
+ end_offset: block.origin.end_offset,
+ flush_end_offset: block.origin.flush_end_offset,
+ flush_row_count: 0,
+ flush_row_index: 0,
+ flush_id: &attempt.flush_id,
+ flushed_at_ms: attempt.flushed_at_ms,
+ block_byte_start: written.byte_start,
+ block_byte_end: written.byte_end,
stats,
- moments: &block.moments,
snapshot_source: first.and_then(|m| m.snapshot_source.as_deref()),
snapshot_library: first.and_then(|m| m.library.as_deref()),
+ moments: &block.moments,
urls,
- flush_row_count: 0,
- flush_row_index: 0,
+ urls_truncated,
+ captured_at_ms: block.captured_at_ms,
});
}
- if rows.is_empty() {
- return Ok(0);
- }
number_rows(&mut rows);
- let index_key = format!(
- "index/blocks/dt={}/{}.parquet",
- write_day(at),
- unique_suffix(at)
- );
- let count = rows.len();
- let batch = block_index(&rows)?;
- let file = blocking(move || write_parquet(&batch).map_err(WriteError::from)).await?;
- store.put_new(rrweb_bucket, &index_key, file).await?;
- Ok(count)
+ Ok(rows)
}
#[cfg(test)]
@@ -588,13 +603,9 @@ mod tests {
use aes_gcm::aead::{Aead, KeyInit, Payload};
use aes_gcm::{Aes256Gcm, Nonce};
- use arrow_array::{Array, Int64Array, ListArray, RecordBatch, StringArray, StructArray};
- use arrow_schema::DataType;
use async_trait::async_trait;
use base64::engine::general_purpose::STANDARD;
use base64::Engine;
- use bytes::Bytes;
- use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
use replay_core_v4::envelope::open_frame;
use replay_core_v4::inspect::dom::MAX_REMOTE_URL_BYTES;
use replay_core_v4::inspect::session::Moment;
@@ -613,29 +624,6 @@ mod tests {
fn get(&self, bucket: &str, key: &str) -> Vec<u8> {
self.objects.lock().unwrap()[&(bucket.to_owned(), key.to_owned())].clone()
}
-
- fn keys(&self, prefix: &str) -> Vec<String> {
- let mut keys: Vec<String> = self
- .objects
- .lock()
- .unwrap()
- .keys()
- .filter(|(_, key)| key.starts_with(prefix))
- .map(|(_, key)| key.clone())
- .collect();
- keys.sort();
- keys
- }
-
- fn parquet(&self, key: &str) -> RecordBatch {
- ParquetRecordBatchReaderBuilder::try_new(Bytes::from(self.get("rrweb-v4", key)))
- .unwrap()
- .build()
- .unwrap()
- .next()
- .unwrap()
- .unwrap()
- }
}
fn decompress(compressed: &[u8]) -> io::Result<Vec<u8>> {
@@ -695,6 +683,7 @@ mod tests {
image_urls: Vec::new(),
remote_urls: Vec::new(),
url_limit: None,
+ captured_at_ms: None,
}
}
@@ -788,6 +777,26 @@ mod tests {
assert!(!stored_lines(&event, &[public]).is_empty());
}
+ #[test]
+ fn keeps_the_sealed_urls_of_a_row_under_the_message_limit() {
+ let page = |i: usize| format!("https://app.example.com/{i}/{}", "p".repeat(1000));
+ let (kept, truncated) = index_urls((0..1000).map(page).collect());
+ assert!(truncated);
+ assert!(
+ kept.len() <= MAX_INDEX_URLS
+ && kept.iter().map(String::len).sum::<usize>() <= MAX_INDEX_URLS_BYTES
+ );
+ let long = format!(
+ "https://app.example.com/{}",
+ "l".repeat(MAX_INDEX_URL_BYTES)
+ );
+ assert_eq!(
+ index_urls(vec![page(1), long, page(2)]),
+ (vec![page(1), page(2)], true)
+ );
+ assert_eq!(index_urls(vec![page(1)]), (vec![page(1)], false));
+ }
+
#[test]
fn a_block_carries_the_reference_its_fetch_job_names() {
let (fetched, declined) = (
@@ -961,7 +970,6 @@ mod tests {
"2026-10-15",
"2026-10",
"images-v4",
- "rrweb-v4",
&store,
at(),
4,
@@ -1024,7 +1032,6 @@ mod tests {
);
// The team month key seals images, as the fetcher's, so a session key does not open them.
assert!(open_frame(&[5; 32], Version::V4, &image_context, sealed_image).is_err());
- assert_eq!(memory.keys("index/images/dt=2026-10-15/").len(), 1);
// The block index row points at the block, keeps the moments, and seals the visited pages.
blocks[0].moments = vec![Moment {
@@ -1033,93 +1040,19 @@ mod tests {
ts_ms: 1,
event_index: 1,
}];
- let stats = BlockStats {
- url_fingerprints: sealed[0].url_fingerprints.clone(),
- window_ids: vec!["w1".to_owned()],
- ..BlockStats::default()
- };
- assert_eq!(
- write_block_index(
- &[(&blocks[0], &stats, &written[0])],
- &keys,
- &FlushAttempt::new(at()),
- "rrweb-v4",
- store.as_ref(),
- at()
- )
- .await
- .unwrap(),
- 1
- );
- let batch = memory.parquet(&memory.keys("index/blocks/")[0]);
- let column = |name: &str| {
- batch
- .column_by_name(name)
- .unwrap_or_else(|| panic!("no column {name}"))
- .clone()
- };
- let text = |name: &str| {
- column(name)
- .as_any()
- .downcast_ref::<StringArray>()
- .unwrap()
- .value(0)
- .to_owned()
- };
- assert_eq!(text("block_s3_key"), block.object_key);
- assert_eq!(
- column("block_byte_end")
- .as_any()
- .downcast_ref::<Int64Array>()
- .unwrap()
- .value(0),
- i64::try_from(block.byte_end).unwrap()
- );
- for (name, data_type) in [
- ("orphan_mutations", DataType::Int32),
- ("active_ms", DataType::Int64),
- ("rage_clicks", DataType::Int32),
- ("cold_start", DataType::Boolean),
- ("viewport_width", DataType::Int32),
- ("image_urls", DataType::Int32),
- ("decode_failures", DataType::Int32),
- ("mouse_interactions", DataType::Int32),
- ("touch_moves", DataType::Int32),
- ("moments_truncated", DataType::Boolean),
- ("written_jsonl_bytes", DataType::Int64),
- ("max_written_event_bytes", DataType::Int64),
- ] {
- assert_eq!(column(name).data_type(), &data_type, "{name}");
- }
- let strings = |name: &str| -> Vec<String> {
- let list = column(name);
- let values = list.as_any().downcast_ref::<ListArray>().unwrap().value(0);
- let values = values.as_any().downcast_ref::<StringArray>().unwrap();
- values.iter().map(|v| v.unwrap().to_owned()).collect()
+ let stats = BlockStats::default();
+ let attempt = FlushAttempt::new(at());
+ let index =
+ block_index_rows(&[(&blocks[0], &stats, &written[0])], &keys, &attempt).unwrap();
+ let [row] = index.as_slice() else {
+ panic!("one row per stored block");
};
- assert_eq!(strings("url_fingerprints"), stats.url_fingerprints);
- assert_eq!(strings("window_ids"), ["w1"]);
- let moments = column("moments");
- let moments = moments
- .as_any()
- .downcast_ref::<ListArray>()
- .unwrap()
- .value(0);
- let moments = moments.as_any().downcast_ref::<StructArray>().unwrap();
assert_eq!(
- moments.column_by_name("event_index").unwrap().data_type(),
- &DataType::Int32
+ (row.block_s3_key, row.block_byte_end),
+ (block.object_key.as_str(), block.byte_end)
);
- let kinds = moments.column_by_name("kind").unwrap().clone();
- assert_eq!(
- kinds
- .as_any()
- .downcast_ref::<StringArray>()
- .unwrap()
- .value(0),
- "full_snapshot"
- );
- let envelope: Value = serde_json::from_str(&text("urls")).unwrap();
+ assert_eq!(row.moments, blocks[0].moments);
+ let envelope: Value = serde_json::from_str(row.urls.as_deref().unwrap()).unwrap();
let sealed_urls = STANDARD
.decode(envelope["ciphertext"].as_str().unwrap())
.unwrap();
@@ -1155,9 +1088,8 @@ mod tests {
);
}
- #[tokio::test]
- async fn readers_drop_the_rows_of_earlier_attempts_that_a_retry_covers() {
- let store = MemoryStore::default();
+ #[test]
+ fn readers_drop_the_rows_of_earlier_attempts_that_a_retry_covers() {
let mut keys = FlushKeys::default();
for session in ["a", "b", "c"] {
let (id, key) = session_key(session);
@@ -1199,6 +1131,8 @@ mod tests {
flush_id: "0000000000000002".to_owned(),
flushed_at_ms: 2_000,
};
+ let mut rows = Vec::new();
+ let mut positions = Vec::new();
for (attempt, entries) in [
(&first, vec![(&a, &stats, &wa)]),
(&between, vec![(&early_b, &stats, &wb)]),
@@ -1211,61 +1145,19 @@ mod tests {
],
),
] {
- write_block_index(&entries, &keys, attempt, "rrweb-v4", &store, at())
- .await
- .unwrap();
- }
-
- let mut rows = Vec::new();
- let mut positions = Vec::new();
- for key in store.keys("index/blocks/") {
- let batch = store.parquet(&key);
- let text = |name: &str| {
- batch
- .column_by_name(name)
- .unwrap()
- .as_any()
- .downcast_ref::<StringArray>()
- .unwrap()
- .clone()
- };
- let int = |name: &str| {
- batch
- .column_by_name(name)
- .unwrap()
- .as_any()
- .downcast_ref::<Int64Array>()
- .unwrap()
- .clone()
- };
- let int32 = |name: &str| {
- batch
- .column_by_name(name)
- .unwrap()
- .as_any()
- .downcast_ref::<arrow_array::Int32Array>()
- .unwrap()
- .clone()
- };
- let partition = int32("partition");
- let (topic, session, flush_id) = (text("topic"), text("session_id"), text("flush_id"));
- let (start, end, at_ms) =
- (int("start_offset"), int("end_offset"), int("flushed_at_ms"));
- let flush_end = int("flush_end_offset");
- let (count, index) = (int32("flush_row_count"), int32("flush_row_index"));
- for i in 0..batch.num_rows() {
+ for row in block_index_rows(&entries, &keys, attempt).unwrap() {
positions.push((
- flush_id.value(i).to_owned(),
- partition.value(i),
- flush_end.value(i),
- count.value(i),
- index.value(i),
+ row.flush_id.to_owned(),
+ row.partition,
+ row.flush_end_offset,
+ row.flush_row_count,
+ row.flush_row_index,
));
rows.push((
- (topic.value(i).to_owned(), partition.value(i)),
- (start.value(i), end.value(i)),
- (at_ms.value(i), flush_id.value(i).to_owned()),
- session.value(i).to_owned(),
+ (row.topic.to_owned(), row.partition),
+ (row.start_offset, row.end_offset),
+ (row.flushed_at_ms, row.flush_id.to_owned()),
+ row.session_id.to_owned(),
));
}
}