Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(replay-recorder-v4): flush in parallel off the async runtime (#118419)

Nicholas Waltz5af65405e57b
diff --git a/rust/ai-research/replay-recorder-v4/src/batch.rs b/rust/ai-research/replay-recorder-v4/src/batch.rs
index 337a77c92348..b572566e72b8 100644
--- a/rust/ai-research/replay-recorder-v4/src/batch.rs
+++ b/rust/ai-research/replay-recorder-v4/src/batch.rs
@@ -3,6 +3,7 @@
use std::collections::{BTreeMap, HashMap};
use std::time::{Duration, Instant};
+use futures::stream::{self, StreamExt};
use metrics::counter;
use replay_core_v4::inspect::dom::UrlLimit;
use replay_core_v4::inspect::session::{BlockInspection, Moment, SessionInspector};
@@ -115,64 +116,74 @@ pub struct Inspectors {
}
impl Inspectors {
- /// Decodes every buffered message, runs it through its session's inspector and returns one block per session.
- pub fn inspect(&mut self, batch: Batch) -> (Vec<Block>, HashMap<Tp, i64>) {
- let now = Instant::now();
+ /// Decodes every buffered message and runs it through its session's inspector, with up to `concurrency` sessions
+ /// at once off the async runtime. Returns one block per session.
+ pub async fn inspect(
+ &mut self,
+ batch: Batch,
+ concurrency: usize,
+ ) -> (Vec<Block>, HashMap<Tp, i64>) {
let mut by_session: BTreeMap<SessionKey, (Vec<Tp>, Vec<Raw>)> = BTreeMap::new();
for ((session, tp), pending) in batch.sessions {
let (tps, raws) = by_session.entry(session).or_default();
tps.push(tp);
raws.extend(pending.messages);
}
- let mut blocks = Vec::with_capacity(by_session.len());
- for (session, (tps, raws)) in by_session {
- let mut messages = Vec::with_capacity(raws.len());
- for raw in raws {
- let message = read(raw.payload, raw.content_encoding.as_deref())
- .and_then(|m| check_read(&m, &session.session_id).map(|()| m));
- match message {
- Ok(mut message) => {
- // Decoded once here, for the inspector and the block writer both.
- message.decode_compressed();
- messages.push(message);
- }
- Err(skip) => {
- counter!("ai_research_v4_recorder_messages_total", "outcome" => skip.as_str())
- .increment(1);
+ let tasks: Vec<_> = by_session
+ .into_iter()
+ .map(|(session, (tps, raws))| {
+ let (inspector, prior) = match self.sessions.remove(&session) {
+ Some((inspector, partitions, last_seen)) => {
+ (inspector, Some((partitions, last_seen)))
}
+ None => (SessionInspector::default(), None),
+ };
+ (session, inspector, prior, tps, raws)
+ })
+ .collect();
+ let done: Vec<_> = stream::iter(tasks)
+ .map(|(session, inspector, prior, tps, raws)| {
+ let (key, messages) = (session.clone(), raws.len());
+ let task = tokio::task::spawn_blocking(move || {
+ let (inspector, block) = inspect_session(&session, inspector, raws);
+ (session, inspector, prior, tps, block)
+ });
+ async move { (key, messages, task.await) }
+ })
+ .buffer_unordered(concurrency.max(1))
+ .collect()
+ .await;
+ let now = Instant::now();
+ let mut blocks = Vec::new();
+ for (key, messages, result) in done {
+ // A panic loses that session's block and its inspector state, and the rest of the flush goes on.
+ let (session, inspector, prior, tps, block) = match result {
+ Ok(inspected) => inspected,
+ Err(error) => {
+ tracing::error!(%error, team_id = key.team_id, "inspecting a session panicked: its block is dropped");
+ counter!("ai_research_v4_recorder_messages_total", "outcome" => "panicked")
+ .increment(messages as u64);
+ continue;
+ }
+ };
+ // A session that decoded nothing left its state as it was, and does not count as seen.
+ let Some(block) = block else {
+ if let Some((partitions, last_seen)) = prior {
+ self.sessions
+ .insert(session, (inspector, partitions, last_seen));
}
- }
- if messages.is_empty() {
continue;
- }
- counter!("ai_research_v4_recorder_messages_total", "outcome" => "recorded")
- .increment(messages.len() as u64);
- // Only a session that decoded something counts as seen.
- let (inspector, partitions, last_seen) = self
- .sessions
- .entry(session.clone())
- .or_insert_with(|| (SessionInspector::default(), Vec::new(), now));
+ };
+ let mut partitions = prior.map(|(partitions, _)| partitions).unwrap_or_default();
for tp in tps {
if !partitions.contains(&tp) {
partitions.push(tp);
}
}
- *last_seen = now;
- let mut block = BlockInspection::default();
- for message in &messages {
- inspector.observe_message(&mut block, message);
- }
- let inspected = block.finish();
- blocks.push(Block {
- session,
- messages,
- stats: inspected.stats,
- moments: inspected.moments,
- image_urls: inspected.image_urls,
- remote_urls: inspected.remote_urls,
- url_limit: inspected.url_limit,
- });
+ self.sessions.insert(session, (inspector, partitions, now));
+ blocks.push(block);
}
+ blocks.sort_by(|a, b| a.session.cmp(&b.session));
(blocks, batch.offsets)
}
@@ -188,6 +199,49 @@ impl Inspectors {
}
}
+fn inspect_session(
+ session: &SessionKey,
+ mut inspector: SessionInspector,
+ raws: Vec<Raw>,
+) -> (SessionInspector, Option<Block>) {
+ let mut messages = Vec::with_capacity(raws.len());
+ for raw in raws {
+ let message = read(raw.payload, raw.content_encoding.as_deref())
+ .and_then(|m| check_read(&m, &session.session_id).map(|()| m));
+ match message {
+ Ok(mut message) => {
+ // Decoded once here, for the inspector and the block writer both.
+ message.decode_compressed();
+ messages.push(message);
+ }
+ Err(skip) => {
+ counter!("ai_research_v4_recorder_messages_total", "outcome" => skip.as_str())
+ .increment(1);
+ }
+ }
+ }
+ if messages.is_empty() {
+ return (inspector, None);
+ }
+ counter!("ai_research_v4_recorder_messages_total", "outcome" => "recorded")
+ .increment(messages.len() as u64);
+ let mut block = BlockInspection::default();
+ for message in &messages {
+ inspector.observe_message(&mut block, message);
+ }
+ let inspected = block.finish();
+ let block = Block {
+ session: session.clone(),
+ messages,
+ stats: inspected.stats,
+ moments: inspected.moments,
+ image_urls: inspected.image_urls,
+ remote_urls: inspected.remote_urls,
+ url_limit: inspected.url_limit,
+ };
+ (inspector, Some(block))
+}
+
#[cfg(test)]
mod tests {
use serde_json::{json, Value};
@@ -227,8 +281,8 @@ mod tests {
json!([{"type": 3, "timestamp": 3, "data": {"source": 1, "positions": []}}])
}
- #[test]
- fn a_revoked_partition_leaves_nothing_behind() {
+ #[tokio::test]
+ async fn a_revoked_partition_leaves_nothing_behind() {
let mut batch = Batch::default();
batch.add(tp(0), key("a"), raw(10, "a", mouse()));
batch.add(tp(1), key("b"), raw(20, "b", mouse()));
@@ -237,7 +291,7 @@ mod tests {
batch.drop_partitions(&[tp(1)]);
inspectors.drop_partitions(&[tp(1)]);
- let (blocks, offsets) = inspectors.inspect(batch);
+ let (blocks, offsets) = inspectors.inspect(batch, 2).await;
assert_eq!(
blocks
.iter()
@@ -248,8 +302,8 @@ mod tests {
assert_eq!(offsets, HashMap::from([(tp(0), 10)]));
}
- #[test]
- fn a_session_that_decodes_nothing_keeps_no_state() {
+ #[tokio::test]
+ async fn a_session_that_decodes_nothing_keeps_no_state() {
let mut batch = Batch::default();
batch.add(
tp(0),
@@ -261,22 +315,22 @@ mod tests {
},
);
let mut inspectors = Inspectors::default();
- assert!(inspectors.inspect(batch).0.is_empty());
+ assert!(inspectors.inspect(batch, 1).await.0.is_empty());
assert!(inspectors.sessions.is_empty());
}
- #[test]
- fn session_state_carries_across_flushes() {
+ #[tokio::test]
+ async fn session_state_carries_across_flushes() {
let mut inspectors = Inspectors::default();
let mut first = Batch::default();
first.add(tp(0), key("a"), raw(1, "a", snapshot()));
first.add(tp(0), key("b"), raw(2, "b", mouse()));
- inspectors.inspect(first);
+ inspectors.inspect(first, 2).await;
let mut second = Batch::default();
second.add(tp(0), key("a"), raw(3, "a", mouse()));
second.add(tp(0), key("b"), raw(4, "b", mouse()));
- let (blocks, _) = inspectors.inspect(second);
+ let (blocks, _) = inspectors.inspect(second, 2).await;
let cold: Vec<_> = blocks
.iter()
.map(|b| (b.session.session_id.as_str(), b.stats.cold_start))
diff --git a/rust/ai-research/replay-recorder-v4/src/config.rs b/rust/ai-research/replay-recorder-v4/src/config.rs
index 6c0892ffc75a..5a5183b36f93 100644
--- a/rust/ai-research/replay-recorder-v4/src/config.rs
+++ b/rust/ai-research/replay-recorder-v4/src/config.rs
@@ -73,6 +73,10 @@ pub struct Config {
#[envconfig(from = "AI_RESEARCH_V4_FLUSH_MAX_BYTES", default = "67108864")]
pub flush_max_bytes: usize,
+ /// How many sessions of a flush are decoded, inspected and sealed at once. 0 is one per CPU.
+ #[envconfig(from = "AI_RESEARCH_V4_FLUSH_CONCURRENCY", default = "0")]
+ pub flush_concurrency: usize,
+
/// A session's inspector is dropped after this long without a message, so memory follows live sessions.
#[envconfig(from = "AI_RESEARCH_V4_SESSION_IDLE_SECS", default = "3600")]
pub session_idle_secs: u64,
diff --git a/rust/ai-research/replay-recorder-v4/src/consumer.rs b/rust/ai-research/replay-recorder-v4/src/consumer.rs
index 961c618fb02c..6d754419009d 100644
--- a/rust/ai-research/replay-recorder-v4/src/consumer.rs
+++ b/rust/ai-research/replay-recorder-v4/src/consumer.rs
@@ -5,7 +5,6 @@ use std::sync::{Arc, Mutex};
use std::time::{Duration, SystemTime, UNIX_EPOCH};
use common_kafka::config::KafkaConfig;
-use common_kafka::kafka_producer::KafkaContext;
use futures::FutureExt;
use lifecycle::Handle;
use metrics::counter;
@@ -16,22 +15,11 @@ use rdkafka::error::KafkaError;
use rdkafka::message::{BorrowedMessage, Headers as _, Message as _};
use rdkafka::{ClientConfig, ClientContext, Offset, TopicPartitionList};
-use chrono::Utc;
-use replay_core_v4::keys::batch::{KeyStore, SessionId};
-use replay_core_v4::keys::kms::AwsKms;
-use replay_core_v4::keys::rows::{key_month, write_day};
-use replay_core_v4::keys::table::{AwsKeyTable, CachedKeyTable};
-
use crate::batch::{Batch, Block, Inspectors, Raw, SessionKey, Tp};
+use crate::flush::{write_flush, FlushError, Writer};
use crate::limits::{Restrictions, SessionLimiter};
use crate::process::{admit, sampled, Admitted, Headers, Skip};
-use crate::store::BlobStore;
use crate::teams::TeamCache;
-use crate::urls::{fetch_jobs, records};
-use crate::write::{prepare, write_block_index, write_blocks, write_images, WriteError};
-use rdkafka::message::{Header, OwnedHeaders};
-use rdkafka::producer::{FutureProducer, FutureRecord};
-use rdkafka::util::Timeout;
/// The most messages read before the limiter decides on their sessions.
const POLL_BATCH: usize = 1000;
@@ -149,31 +137,11 @@ fn header<'m>(message: &'m BorrowedMessage<'_>, name: &str) -> Option<&'m str> {
.and_then(|v| std::str::from_utf8(v).ok())
}
-/// What a flush needs to store blocks. Without it the recorder runs dry.
-pub struct Writer {
- pub keys: KeyStore<CachedKeyTable<AwsKeyTable>, AwsKms>,
- pub store: Arc<dyn BlobStore>,
- pub rrweb_bucket: String,
- pub images_bucket: String,
- pub producer: FutureProducer<KafkaContext>,
- pub image_fetch_topic: String,
- /// Remote image references this process sent to the fetcher, by UTC day. A reference goes out again each day, so
- /// the fetcher sees it within every refresh interval and keeps the image as long as the blocks that reference it.
- pub sent_urls: moka::sync::Cache<(String, String), ()>,
-}
-
-#[derive(Debug, thiserror::Error)]
-pub enum FlushError {
- #[error(transparent)]
- Keys(#[from] replay_core_v4::keys::batch::KeyError),
- #[error(transparent)]
- Write(#[from] WriteError),
-}
-
pub struct Settings {
pub sample_percentage: f64,
pub flush_interval: Duration,
pub flush_max_bytes: usize,
+ pub flush_concurrency: usize,
pub session_idle: Duration,
/// The main topic, when the main lane moves its force-overflow messages to the overflow topic.
pub redirecting_topic: Option<String>,
@@ -239,8 +207,10 @@ pub async fn run(
inspectors.drop_partitions(&revoked);
}
if batch.should_flush(settings.flush_interval, settings.flush_max_bytes) {
- let (mut blocks, offsets) = inspectors.inspect(std::mem::take(&mut batch));
- if let Err(error) = flush(&mut blocks, writer.as_ref()).await {
+ let (blocks, offsets) = inspectors
+ .inspect(std::mem::take(&mut batch), settings.flush_concurrency)
+ .await;
+ if let Err(error) = flush(blocks, writer.as_ref(), settings.flush_concurrency).await {
// Nothing of this flush is committed, so a restart reads it again from the last commit.
tracing::error!(%error, "flush failed");
handle.signal_failure(format!("flush failed: {error}"));
@@ -354,9 +324,13 @@ async fn admit_message(
}
/// Stores a flush's blocks, or only reports them when the recorder runs dry.
-async fn flush(blocks: &mut [Block], writer: Option<&Writer>) -> Result<(), FlushError> {
+async fn flush(
+ blocks: Vec<Block>,
+ writer: Option<&Writer>,
+ concurrency: usize,
+) -> Result<(), FlushError> {
counter!("ai_research_v4_recorder_blocks_total").increment(blocks.len() as u64);
- for block in blocks.iter() {
+ for block in &blocks {
counter!("ai_research_v4_recorder_block_events_total").increment(block.stats.event_count);
if block.stats.cold_start {
counter!("ai_research_v4_recorder_cold_start_blocks_total").increment(1);
@@ -366,108 +340,6 @@ async fn flush(blocks: &mut [Block], writer: Option<&Writer>) -> Result<(), Flus
}
}
let Some(writer) = writer else { return Ok(()) };
- let now = Utc::now();
- let day = write_day(now);
- let month = key_month(&day);
- let sessions: Vec<SessionId> = blocks
- .iter()
- .map(|b| SessionId {
- team_id: b.session.team_id,
- session_id: b.session.session_id.clone(),
- })
- .collect();
- // Keys are stored before anything is sealed with them, as in v3.
- let keys = writer.keys.keys_for_flush(&sessions, &day).await?;
- counter!("ai_research_v4_recorder_dropped_sessions_total").increment(keys.dropped.len() as u64);
- let prepared = prepare(blocks, &keys, month);
- // Before the index is written: building the jobs counts each block's image URLs into its stats.
- // A dropped block's images are not stored, so the fetcher does not fetch them.
- let kept = blocks
- .iter_mut()
- .zip(&prepared.kept)
- .filter_map(|(block, &kept)| kept.then_some(block));
- let jobs = fetch_jobs(kept, &keys, month, now.timestamp_millis(), &|reference| {
- writer
- .sent_urls
- .contains_key(&(day.clone(), reference.to_owned()))
- });
- let rows = write_images(
- &prepared.images,
- &keys,
- &day,
- month,
- &writer.images_bucket,
- &writer.rrweb_bucket,
- writer.store.as_ref(),
- now,
- )
- .await?;
- counter!("ai_research_v4_recorder_stored_images_total").increment(rows.len() as u64);
- // What sealing images under session keys would store, to compare with the frames above.
- counter!("ai_research_v4_recorder_session_keyed_image_frames_total")
- .increment(prepared.session_frames);
- let written = write_blocks(
- &prepared.blocks,
- &writer.rrweb_bucket,
- writer.store.as_ref(),
- now,
- )
- .await?;
- counter!("ai_research_v4_recorder_written_blocks_total").increment(written.len() as u64);
- write_block_index(
- blocks,
- &written,
- &keys,
- &writer.rrweb_bucket,
- writer.store.as_ref(),
- now,
- )
- .await?;
- produce_jobs(writer, jobs, &day, now.timestamp_millis()).await;
- Ok(())
-}
-
-/// Sends the flush's fetch jobs and waits for their delivery before the flush commits, as the v3 mirror does. A send
-/// that fails does not fail the flush, because its blocks are stored by then. The references it did not send stay
-/// unmarked, so a later flush that sees them sends them again.
-async fn produce_jobs(
- writer: &Writer,
- jobs: std::collections::BTreeMap<String, Vec<serde_json::Value>>,
- day: &str,
- now_ms: i64,
-) {
- let captured_at = now_ms.to_string();
- let records = records(jobs);
- let total: usize = records
- .iter()
- .map(|(_, _, references)| references.len())
- .sum();
- let mut sent = Vec::new();
- for (domain, payload, references) in records {
- let headers = OwnedHeaders::new().insert(Header {
- key: "capture-timestamp-ms",
- value: Some(&captured_at),
- });
- let record = FutureRecord::to(&writer.image_fetch_topic)
- .key(&domain)
- .payload(&payload)
- .headers(headers);
- if let Err((error, _)) = writer
- .producer
- .send(record, Timeout::After(Duration::from_secs(30)))
- .await
- {
- // The next record would wait as long on the same broker.
- tracing::warn!(%error, "sending image fetch jobs failed");
- break;
- }
- sent.extend(references);
- }
- counter!("ai_research_v4_recorder_image_fetch_jobs_total").increment(sent.len() as u64);
- counter!("ai_research_v4_recorder_image_fetch_jobs_unsent_total")
- .increment((total - sent.len()) as u64);
- for reference in sent {
- writer.sent_urls.insert((day.to_owned(), reference), ());
- }
- metrics::gauge!("ai_research_v4_recorder_sent_urls").set(writer.sent_urls.entry_count() as f64);
+ let blocks: Vec<Arc<Block>> = blocks.into_iter().map(Arc::new).collect();
+ write_flush(&blocks, writer, concurrency).await
}
diff --git a/rust/ai-research/replay-recorder-v4/src/flush.rs b/rust/ai-research/replay-recorder-v4/src/flush.rs
new file mode 100644
index 000000000000..ee92ed2b76f2
--- /dev/null
+++ b/rust/ai-research/replay-recorder-v4/src/flush.rs
@@ -0,0 +1,259 @@
+//! Stores a flush's blocks: their keys, the sealed blocks, the image packs, both indexes and the fetch jobs.
+
+use std::collections::{BTreeMap, BTreeSet};
+use std::sync::Arc;
+use std::time::Duration;
+
+use chrono::Utc;
+use common_kafka::kafka_producer::KafkaContext;
+use futures::stream::{self, StreamExt};
+use metrics::counter;
+use rdkafka::message::{Header, OwnedHeaders};
+use rdkafka::producer::{FutureProducer, FutureRecord};
+use rdkafka::util::Timeout;
+use replay_core_v4::inspect::stats::BlockStats;
+use replay_core_v4::keys::batch::{KeyStore, SessionId};
+use replay_core_v4::keys::kms::AwsKms;
+use replay_core_v4::keys::rows::{key_month, write_day};
+use replay_core_v4::keys::table::{AwsKeyTable, CachedKeyTable};
+use tokio::sync::Semaphore;
+
+use crate::batch::Block;
+use crate::store::BlobStore;
+use crate::urls::{fetch_jobs, records};
+use crate::write::{
+ new_images, object_key, seal_block, seal_blocking, session_keyed_frames, write_block_index,
+ write_blocks, write_images, WriteError,
+};
+
+/// How long a fetch job may wait for room in the producer queue.
+const ENQUEUE_TIMEOUT: Duration = Duration::from_secs(10);
+
+/// Fetch job records sent at once, so a slow broker slows the flush instead of filling the producer queue.
+const SENDS_IN_FLIGHT: usize = 64;
+
+/// What a flush needs to store blocks. Without it the recorder runs dry.
+pub struct Writer {
+ pub keys: KeyStore<CachedKeyTable<AwsKeyTable>, AwsKms>,
+ pub store: Arc<dyn BlobStore>,
+ pub rrweb_bucket: String,
+ pub images_bucket: String,
+ pub producer: FutureProducer<KafkaContext>,
+ pub image_fetch_topic: String,
+ /// Remote image references this process sent to the fetcher, by UTC day. A reference goes out again each day, so
+ /// the fetcher sees it within every refresh interval and keeps the image as long as the blocks that reference it.
+ pub sent_urls: moka::sync::Cache<(String, String), ()>,
+ /// Seals running at once, over all attempts: the flush concurrency.
+ pub seal_permits: Arc<Semaphore>,
+}
+
+#[derive(Debug, thiserror::Error)]
+pub enum FlushError {
+ #[error(transparent)]
+ Keys(#[from] replay_core_v4::keys::batch::KeyError),
+ #[error(transparent)]
+ Write(#[from] WriteError),
+}
+
+/// Stores the blocks once. Up to `concurrency` blocks are sealed at once off the async runtime, and the packs, and
+/// the fetch jobs, go out concurrently.
+pub async fn write_flush(
+ blocks: &[Arc<Block>],
+ writer: &Writer,
+ concurrency: usize,
+) -> Result<(), FlushError> {
+ let now = Utc::now();
+ let day = write_day(now);
+ let month = key_month(&day);
+ let sessions: Vec<SessionId> = blocks
+ .iter()
+ .map(|b| SessionId {
+ team_id: b.session.team_id,
+ session_id: b.session.session_id.clone(),
+ })
+ .collect::<BTreeSet<_>>()
+ .into_iter()
+ .collect();
+ // Keys are stored before anything is sealed with them, as in v3.
+ let keys = Arc::new(writer.keys.keys_for_flush(&sessions, &day).await?);
+ counter!("ai_research_v4_recorder_dropped_sessions_total").increment(keys.dropped.len() as u64);
+
+ let object_key = object_key(&day, now);
+ let seals: Vec<_> = blocks
+ .iter()
+ .enumerate()
+ .filter_map(|(index, block)| {
+ let id = SessionId {
+ team_id: block.session.team_id,
+ session_id: block.session.session_id.clone(),
+ };
+ let key = keys.sessions.get(&id)?.clone();
+ let team_key = *keys.team_months.get(&block.session.team_id)?;
+ let (block, month) = (block.clone(), month.to_owned());
+ Some(seal_blocking(&writer.seal_permits, move || {
+ seal_block(&block, &key, &team_key, &month).map(|sealed| (index, sealed))
+ }))
+ })
+ .collect();
+ let mut done = Vec::with_capacity(seals.len());
+ let results: Vec<_> = stream::iter(seals)
+ .buffer_unordered(concurrency.max(1))
+ .collect()
+ .await;
+ for result in results {
+ match result {
+ Ok(sealed) => done.push(sealed),
+ // A panic in one block loses that block only. Its messages are committed with the rest of the flush.
+ Err(WriteError::Panicked(error)) => {
+ tracing::error!(%error, "sealing a block panicked: the block is dropped");
+ counter!("ai_research_v4_recorder_dropped_blocks_total", "outcome" => "panicked")
+ .increment(1);
+ }
+ Err(error) => return Err(error.into()),
+ }
+ }
+ // Fail closed: a block that still holds a declined URL, a credential or a private host, is not stored.
+ done.retain(|(_, block)| {
+ let Some(outcome) = block.counts.dropped else {
+ return true;
+ };
+ counter!("ai_research_v4_recorder_dropped_blocks_total", "outcome" => outcome).increment(1);
+ false
+ });
+ let (indices, mut sealed): (Vec<usize>, Vec<_>) = done.into_iter().unzip();
+
+ // The writer's columns go into a copy, so the shared blocks stay as the inspector left them.
+ let mut stats: Vec<BlockStats> = blocks.iter().map(|b| b.stats.clone()).collect();
+ for (&index, block) in indices.iter().zip(&mut sealed) {
+ stats[index].images_extracted += block.counts.extracted;
+ stats[index].image_bytes += block.counts.bytes;
+ stats[index].images_inline += block.counts.inline;
+ stats[index].written_jsonl_bytes += block.counts.jsonl_bytes;
+ stats[index].max_written_event_bytes = block.counts.max_line_bytes;
+ stats[index].url_fingerprints = std::mem::take(&mut block.url_fingerprints);
+ }
+ // Before the index is written: building the jobs counts each block's image URLs into its stats. A dropped block's
+ // images are not stored, so the fetcher does not fetch them.
+ let mut kept = vec![false; blocks.len()];
+ for &index in &indices {
+ kept[index] = true;
+ }
+ let jobs = fetch_jobs(
+ blocks
+ .iter()
+ .map(|b| &**b)
+ .zip(stats.iter_mut())
+ .zip(kept)
+ .filter_map(|(pair, kept)| kept.then_some(pair)),
+ &keys,
+ month,
+ now.timestamp_millis(),
+ &|reference| {
+ writer
+ .sent_urls
+ .contains_key(&(day.clone(), reference.to_owned()))
+ },
+ );
+
+ let session_frames = session_keyed_frames(&sealed);
+ let rows = write_images(
+ new_images(&mut sealed),
+ &keys,
+ &day,
+ month,
+ &writer.images_bucket,
+ &writer.rrweb_bucket,
+ &writer.store,
+ now,
+ concurrency,
+ &writer.seal_permits,
+ )
+ .await?;
+ counter!("ai_research_v4_recorder_stored_images_total").increment(rows.len() as u64);
+ // What sealing images under session keys would store, to compare with the frames above.
+ counter!("ai_research_v4_recorder_session_keyed_image_frames_total").increment(session_frames);
+
+ let written = write_blocks(
+ &sealed,
+ &object_key,
+ &writer.rrweb_bucket,
+ writer.store.as_ref(),
+ )
+ .await?;
+ drop(sealed);
+ counter!("ai_research_v4_recorder_written_blocks_total").increment(written.len() as u64);
+ let entries: Vec<_> = indices
+ .iter()
+ .zip(&written)
+ .map(|(&index, written)| (&*blocks[index], &stats[index], written))
+ .collect();
+ write_block_index(
+ &entries,
+ &keys,
+ &writer.rrweb_bucket,
+ writer.store.as_ref(),
+ now,
+ )
+ .await?;
+ produce_jobs(writer, jobs, &day, now.timestamp_millis()).await;
+ Ok(())
+}
+
+/// Sends every fetch job of the flush, then waits for all deliveries before the flush commits, as the v3 mirror does.
+/// A send that fails does not fail the flush, because its blocks are stored by then. The references it did not send
+/// stay unmarked, so a later flush that sees them sends them again.
+async fn produce_jobs(
+ writer: &Writer,
+ jobs: BTreeMap<String, Vec<serde_json::Value>>,
+ day: &str,
+ now_ms: i64,
+) {
+ let captured_at = now_ms.to_string();
+ let deliveries: Vec<_> = records(jobs)
+ .into_iter()
+ .map(|(domain, payload, references)| {
+ let captured_at = &captured_at;
+ async move {
+ let headers = OwnedHeaders::new().insert(Header {
+ key: "capture-timestamp-ms",
+ value: Some(captured_at),
+ });
+ let record = FutureRecord::to(&writer.image_fetch_topic)
+ .key(&domain)
+ .payload(&payload)
+ .headers(headers);
+ match writer
+ .producer
+ .send(record, Timeout::After(ENQUEUE_TIMEOUT))
+ .await
+ {
+ Ok(_) => Ok(references),
+ Err((error, _)) => Err((references.len(), error)),
+ }
+ }
+ })
+ .collect();
+ let delivered: Vec<_> = stream::iter(deliveries)
+ .buffer_unordered(SENDS_IN_FLIGHT)
+ .collect()
+ .await;
+ let mut sent = Vec::new();
+ let mut unsent = 0;
+ for delivery in delivered {
+ match delivery {
+ Ok(references) => sent.extend(references),
+ Err((references, error)) => {
+ if unsent == 0 {
+ tracing::warn!(%error, "sending image fetch jobs failed");
+ }
+ unsent += references;
+ }
+ }
+ }
+ counter!("ai_research_v4_recorder_image_fetch_jobs_total").increment(sent.len() as u64);
+ counter!("ai_research_v4_recorder_image_fetch_jobs_unsent_total").increment(unsent as u64);
+ for reference in sent {
+ writer.sent_urls.insert((day.to_owned(), reference), ());
+ }
+ metrics::gauge!("ai_research_v4_recorder_sent_urls").set(writer.sent_urls.entry_count() as f64);
+}
diff --git a/rust/ai-research/replay-recorder-v4/src/lib.rs b/rust/ai-research/replay-recorder-v4/src/lib.rs
index 1927a89e129c..ce0d17aa5ad6 100644
--- a/rust/ai-research/replay-recorder-v4/src/lib.rs
+++ b/rust/ai-research/replay-recorder-v4/src/lib.rs
@@ -4,6 +4,7 @@
pub mod batch;
pub mod config;
pub mod consumer;
+pub mod flush;
pub mod index;
pub mod limits;
pub mod process;
diff --git a/rust/ai-research/replay-recorder-v4/src/main.rs b/rust/ai-research/replay-recorder-v4/src/main.rs
index 0f574ba168ef..b7d12d04be6d 100644
--- a/rust/ai-research/replay-recorder-v4/src/main.rs
+++ b/rust/ai-research/replay-recorder-v4/src/main.rs
@@ -17,7 +17,8 @@ use replay_core_v4::keys::batch::KeyStore;
use replay_core_v4::keys::kms::{AwsKms, CachedKms, KmsLimits};
use replay_core_v4::keys::table::{AwsKeyTable, CachedKeyTable};
use replay_recorder_v4::config::Config;
-use replay_recorder_v4::consumer::{run, RecorderConsumer, Settings, Sources, Writer};
+use replay_recorder_v4::consumer::{run, RecorderConsumer, Settings, Sources};
+use replay_recorder_v4::flush::Writer;
use replay_recorder_v4::limits::{Restrictions, SessionLimiter};
use replay_recorder_v4::store::S3Store;
use replay_recorder_v4::teams::TeamCache;
@@ -152,6 +153,11 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
sample_percentage: config.sample_percentage,
flush_interval: Duration::from_millis(config.flush_interval_ms),
flush_max_bytes: config.flush_max_bytes,
+ flush_concurrency: if config.flush_concurrency == 0 {
+ std::thread::available_parallelism().map_or(4, |n| n.get())
+ } else {
+ config.flush_concurrency
+ },
session_idle: Duration::from_secs(config.session_idle_secs),
redirecting_topic: (config.ingestion_overflow_mode == "redirect")
.then(|| config.kafka_topic.clone()),
@@ -195,6 +201,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
.await?,
image_fetch_topic: config.image_fetch_topic.clone(),
sent_urls: replay_recorder_v4::urls::sent_urls(500_000),
+ seal_permits: Arc::new(tokio::sync::Semaphore::new(settings.flush_concurrency)),
})
} else {
tracing::info!("writes are off: the recorder stores nothing");
diff --git a/rust/ai-research/replay-recorder-v4/src/urls.rs b/rust/ai-research/replay-recorder-v4/src/urls.rs
index 7f7c471c6212..2396007f4458 100644
--- a/rust/ai-research/replay-recorder-v4/src/urls.rs
+++ b/rust/ai-research/replay-recorder-v4/src/urls.rs
@@ -12,6 +12,8 @@ use replay_core_v4::url_policy::{
};
use serde_json::{json, Value};
+use replay_core_v4::inspect::stats::BlockStats;
+
use crate::batch::Block;
/// The fetcher's limits on one Kafka record: its job count, and the byte length of its value without key or headers.
@@ -21,9 +23,9 @@ const MAX_RECORD_BYTES: usize = 512 * 1024;
const REMAINING_HOPS: u32 = 10;
/// Builds one job per new remote image URL, grouped by registrable domain, the topic's key. Counts each block's URLs
-/// in its stats. `already_sent` names references this process sent today.
+/// in the stats paired with it. `already_sent` names references this process sent today.
pub fn fetch_jobs<'b>(
- blocks: impl IntoIterator<Item = &'b mut Block>,
+ blocks: impl IntoIterator<Item = (&'b Block, &'b mut BlockStats)>,
keys: &FlushKeys,
key_month: &str,
now_ms: i64,
@@ -31,7 +33,7 @@ pub fn fetch_jobs<'b>(
) -> BTreeMap<String, Vec<Value>> {
let mut jobs: BTreeMap<String, Vec<Value>> = BTreeMap::new();
let mut seen = HashSet::new();
- for block in blocks {
+ for (block, stats) in blocks {
let id = SessionId {
team_id: block.session.team_id,
session_id: block.session.session_id.clone(),
@@ -49,7 +51,7 @@ pub fn fetch_jobs<'b>(
else {
continue;
};
- block.stats.image_urls += 1;
+ stats.image_urls += 1;
if already_sent(&reference) || !seen.insert(reference.clone()) {
continue;
}
@@ -157,7 +159,6 @@ pub fn sent_urls(capacity: u64) -> moka::sync::Cache<(String, String), ()> {
#[cfg(test)]
mod tests {
- use replay_core_v4::inspect::stats::BlockStats;
use replay_core_v4::keys::batch::SessionKey as Key;
use super::*;
@@ -186,7 +187,7 @@ mod tests {
"http://localhost/private.png",
"https://bat.bing.com/action/0",
];
- let mut blocks = vec![Block {
+ let block = Block {
session: SessionKey {
team_id: 7,
session_id: "s".to_owned(),
@@ -197,7 +198,8 @@ mod tests {
image_urls: urls.iter().map(|u| (*u).to_owned()).collect(),
remote_urls: Vec::new(),
url_limit: None,
- }];
+ };
+ let mut stats = BlockStats::default();
let mut keys = FlushKeys::default();
keys.sessions.insert(
SessionId {
@@ -219,9 +221,13 @@ mod tests {
);
let sent = image_url_ref(7, "2026-10", &sent_hash);
- let jobs = fetch_jobs(&mut blocks, &keys, "2026-10", 1_000, &|reference| {
- reference == sent
- });
+ let jobs = fetch_jobs(
+ [(&block, &mut stats)],
+ &keys,
+ "2026-10",
+ 1_000,
+ &|reference| reference == sent,
+ );
let domains: Vec<_> = jobs.keys().cloned().collect();
assert_eq!(domains, ["example.com"]);
let fetched: Vec<_> = jobs["example.com"]
@@ -245,7 +251,7 @@ mod tests {
(Some("s"), Some(10))
);
// Every public URL the block referenced counts, sent before or not.
- assert_eq!(blocks[0].stats.image_urls, 4);
+ assert_eq!(stats.image_urls, 4);
}
#[test]
diff --git a/rust/ai-research/replay-recorder-v4/src/write.rs b/rust/ai-research/replay-recorder-v4/src/write.rs
index 3fe258cd0d0b..b981bddd63a9 100644
--- a/rust/ai-research/replay-recorder-v4/src/write.rs
+++ b/rust/ai-research/replay-recorder-v4/src/write.rs
@@ -1,17 +1,19 @@
-//! Writes a flush: image packs and their index first, then one object of sealed session blocks.
+//! Writes a flush: image packs and their index first, then one object of sealed session blocks, then their index.
//!
//! The order matters for a crash between writes. A pack without its blocks is unreferenced, which is harmless. A block
//! whose images were never stored would hold references that resolve to nothing.
use std::collections::{BTreeMap, HashSet};
-use std::io::Write;
+use std::io::{self, Write};
+use std::sync::Arc;
use chrono::{DateTime, Utc};
-use metrics::counter;
+use futures::stream::{self, StreamExt, TryStreamExt};
use rand::Rng;
use replay_core_v4::bytescan::write_json_string;
use replay_core_v4::envelope::{seal_frame, seal_json_envelope, Codec, Context, Version};
use replay_core_v4::inspect::events::event_page_url;
+use replay_core_v4::inspect::stats::BlockStats;
use replay_core_v4::inspect::stats::MAX_URL_FINGERPRINTS;
use replay_core_v4::keys::batch::{FlushKeys, SessionId, SessionKey as Key};
use replay_core_v4::keys::rows::write_day;
@@ -20,6 +22,8 @@ use replay_core_v4::scan::cv::extract_event_images;
use replay_core_v4::scan::images::{Image, ImageLimits, ImageSink};
use replay_core_v4::seal::{image_content_key, url_fingerprint_key};
+use tokio::sync::Semaphore;
+
use crate::batch::{Block, SessionKey};
use crate::index::{
block_index, image_index, write_parquet, BlockIndexRow, ImageIndexRow, IndexError,
@@ -31,6 +35,7 @@ use crate::store::{BlobStore, StoreError};
/// and reads them rarely, so it takes the smaller output for more CPU.
const BLOCK_BROTLI_QUALITY: u8 = 9;
const BLOCK_BROTLI_WINDOW_BITS: u8 = 22;
+const BLOCK_WRITE_BUFFER: usize = 256 * 1024;
#[derive(Debug, thiserror::Error)]
pub enum WriteError {
@@ -40,9 +45,11 @@ pub enum WriteError {
Index(#[from] IndexError),
#[error("sealing failed: {0}")]
Seal(String),
+ #[error("sealing panicked: {0}")]
+ Panicked(String),
}
-/// Where one session's block landed, for the block index.
+/// Where one block landed, for the block index.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Written {
pub session: SessionKey,
@@ -53,31 +60,28 @@ pub struct Written {
pub byte_end: u64,
}
-/// A session's block, ready to seal: its key, and its lines with the images moved out.
-pub struct PreparedBlock {
- pub session: SessionKey,
- pub key: Key,
- pub lines: Vec<u8>,
-}
-
-fn block_encoder() -> Result<brotlic::BrotliEncoder, WriteError> {
- let seal_error = |e: brotlic::SetParameterError| WriteError::Seal(e.to_string());
- brotlic::BrotliEncoderOptions::new()
- .quality(brotlic::Quality::new(BLOCK_BROTLI_QUALITY).map_err(seal_error)?)
- .window_size(brotlic::WindowSize::new(BLOCK_BROTLI_WINDOW_BITS).map_err(seal_error)?)
- .build()
- .map_err(seal_error)
+/// The columns of a block's stats that the writer fills.
+#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
+pub struct WriterCounts {
+ pub extracted: u32,
+ pub bytes: u64,
+ pub inline: u32,
+ /// The bytes of the lines the writer gives the encoder, and the longest line.
+ pub jsonl_bytes: u64,
+ pub max_line_bytes: u64,
+ /// Why the block must not be stored, as the outcome of its drop: a URL the policy declines stays in it, or the
+ /// inspector did not name all its URLs.
+ pub dropped: Option<&'static str>,
}
-pub struct Prepared {
- pub blocks: Vec<PreparedBlock>,
- /// The flush's new images by team, each stored once.
- pub images: BTreeMap<i64, Vec<Image>>,
- /// The frames the packs would hold if each session sealed its images under its own key: one per distinct
- /// (session, reference) pair, where `images` holds one per reference.
- pub session_frames: u64,
- /// Which of the blocks it was given are stored. The fetch jobs come from these only.
- pub kept: Vec<bool>,
+/// A block sealed for one object, with the images it moved out.
+pub struct SealedBlock {
+ pub session: SessionKey,
+ pub key_month: String,
+ pub frame: Vec<u8>,
+ pub images: Vec<Image>,
+ pub counts: WriterCounts,
+ pub url_fingerprints: Vec<String>,
}
fn unique_suffix(at: DateTime<Utc>) -> String {
@@ -92,218 +96,318 @@ pub fn object_key(write_day: &str, at: DateTime<Utc>) -> String {
format!("blocks/dt={write_day}/{}", unique_suffix(at))
}
-/// Moves each kept session's images out and builds its lines in the main lane's format: one `["window_id",event]`
-/// line per event. Fills the image and written size columns of each block's stats. Every image goes into this
-/// flush's packs, even one an earlier flush stored: a pack expires by its own write time, so it must be no older than
-/// the blocks that reference it.
-pub fn prepare(blocks: &mut [Block], keys: &FlushKeys, key_month: &str) -> Prepared {
- let mut prepared = Prepared {
- blocks: Vec::new(),
- images: BTreeMap::new(),
- session_frames: 0,
- kept: vec![false; blocks.len()],
- };
- // A reference names its team, so it names one frame of the flush.
- let mut seen: HashSet<String> = HashSet::new();
- let mut seen_in_session: HashSet<(String, String)> = HashSet::new();
- for (index, block) in blocks.iter_mut().enumerate() {
- let id = SessionId {
- team_id: block.session.team_id,
- session_id: block.session.session_id.clone(),
- };
- let (Some(key), Some(team_key)) = (
- keys.sessions.get(&id),
- keys.team_months.get(&block.session.team_id),
- ) else {
- continue;
- };
- // Fail closed: the inspector did not name every URL, so one it missed could be declined.
- if let Some(limit) = block.url_limit {
- counter!("ai_research_v4_recorder_dropped_blocks_total", "outcome" => limit.label())
- .increment(1);
- continue;
- }
- let content_key = image_content_key(team_key);
- let mut rewrite = UrlRewrite::new(
- &block.remote_urls,
+/// Moves a block's images out and writes its lines to `out` in the main lane's format: one `["window_id",event]` line
+/// per event. Every image goes into this flush's packs, even one an earlier flush stored: a pack expires by its own
+/// write time, so it must be no older than the blocks that reference it.
+/// Expanded `cv` events can make a block far larger than its messages, so only one event is held at a time.
+pub fn prepare_block(
+ block: &Block,
+ team_key: &[u8; 32],
+ key_month: &str,
+ out: &mut impl Write,
+) -> io::Result<(Vec<Image>, WriterCounts)> {
+ // Fail closed: a URL the inspector did not name could be one the policy declines.
+ if let Some(limit) = block.url_limit {
+ let dropped = Some(limit.label());
+ return Ok((
+ Vec::new(),
+ WriterCounts {
+ dropped,
+ ..WriterCounts::default()
+ },
+ ));
+ }
+ let content_key = image_content_key(team_key);
+ let mut rewrite = UrlRewrite::new(
+ &block.remote_urls,
+ block.session.team_id,
+ key_month,
+ team_key,
+ );
+ let mut images = Vec::new();
+ let mut kept = HashSet::new();
+ let mut counts = WriterCounts::default();
+ for message in &block.messages {
+ let mut window = Vec::new();
+ write_json_string(
+ message.window_id.as_deref().unwrap_or_default(),
+ &mut window,
+ );
+ let mut sink = ImageSink::new(
+ &content_key,
block.session.team_id,
key_month,
- team_key,
- );
- block.stats.url_fingerprints = url_fingerprints(&key.key, &block_urls(block));
- let mut block_images = Vec::new();
- let mut lines = Vec::with_capacity(
- block.stats.byte_size as usize + block.stats.event_count as usize * 16,
+ ImageLimits::default(),
);
- for message in &block.messages {
- let mut window = Vec::new();
- write_json_string(
- message.window_id.as_deref().unwrap_or_default(),
- &mut window,
- );
- let mut sink = ImageSink::new(
- &content_key,
- block.session.team_id,
- key_month,
- ImageLimits::default(),
+ for event in &message.events {
+ let extracted = extract_event_images(&message.inner, event, &mut sink);
+ out.write_all(b"[")?;
+ out.write_all(&window)?;
+ out.write_all(b",")?;
+ let canonical = canonical_strings(
+ extracted
+ .as_deref()
+ .unwrap_or(&message.inner[event.span.0..event.span.1]),
);
- for event in &message.events {
- let extracted = extract_event_images(&message.inner, event, &mut sink);
- let line_start = lines.len();
- lines.push(b'[');
- lines.extend_from_slice(&window);
- lines.push(b',');
- let canonical = canonical_strings(
- extracted
- .as_deref()
- .unwrap_or(&message.inner[event.span.0..event.span.1]),
- );
- lines.extend_from_slice(&rewrite.apply(&canonical));
- lines.extend_from_slice(b"]\n");
- let line = (lines.len() - line_start) as u64;
- block.stats.written_jsonl_bytes += line;
- block.stats.max_written_event_bytes = block.stats.max_written_event_bytes.max(line);
- }
- block.stats.images_extracted += sink.images.len() as u32;
- block.stats.image_bytes += sink
+ let event_bytes = rewrite.apply(&canonical);
+ out.write_all(&event_bytes)?;
+ out.write_all(b"]\n")?;
+ let line = (window.len() + event_bytes.len() + 4) as u64;
+ counts.jsonl_bytes += line;
+ counts.max_line_bytes = counts.max_line_bytes.max(line);
+ }
+ counts.extracted += sink.images.len() as u32;
+ counts.bytes += sink
+ .images
+ .iter()
+ .map(|i| i.bytes.len() as u64)
+ .sum::<u64>();
+ counts.inline += sink.left_inline;
+ images.extend(
+ sink.images
+ .into_iter()
+ .filter(|image| kept.insert(image.reference.clone())),
+ );
+ }
+ // Fail closed: the flush drops a block that still holds a declined URL, a credential or a private host.
+ if !rewrite.finish() {
+ counts.dropped = Some("declined_unrewritten");
+ }
+ Ok((images, counts))
+}
+
+fn block_encoder() -> Result<brotlic::BrotliEncoder, WriteError> {
+ let seal_error = |e: brotlic::SetParameterError| WriteError::Seal(e.to_string());
+ brotlic::BrotliEncoderOptions::new()
+ .quality(brotlic::Quality::new(BLOCK_BROTLI_QUALITY).map_err(seal_error)?)
+ .window_size(brotlic::WindowSize::new(BLOCK_BROTLI_WINDOW_BITS).map_err(seal_error)?)
+ .build()
+ .map_err(seal_error)
+}
+
+/// Builds, compresses and seals one block.
+pub fn seal_block(
+ block: &Block,
+ key: &Key,
+ team_key: &[u8; 32],
+ key_month: &str,
+) -> Result<SealedBlock, WriteError> {
+ let seal_error = |e: io::Error| WriteError::Seal(e.to_string());
+ // Lines go into the encoder as they are built, so memory follows the compressed block and not its plaintext.
+ // A brotli stream does not state its decompressed size, so a reader decompresses it as a stream.
+ let encoder = brotlic::CompressorWriter::with_encoder(block_encoder()?, Vec::new());
+ // The encoder works on each write it gets, so small writes cost far more CPU than the same bytes in one.
+ let mut lines = io::BufWriter::with_capacity(BLOCK_WRITE_BUFFER, encoder);
+ let (images, counts) =
+ prepare_block(block, team_key, key_month, &mut lines).map_err(seal_error)?;
+ let encoder = lines.into_inner().map_err(|e| seal_error(e.into_error()))?;
+ let compressed = encoder
+ .into_inner()
+ .map_err(|e| WriteError::Seal(e.to_string()))?;
+ let frame = seal_frame(
+ &key.key,
+ Version::V4,
+ &block_context(&block.session, &key.key_month),
+ &compressed,
+ )
+ .map_err(|e| WriteError::Seal(e.to_string()))?;
+ Ok(SealedBlock {
+ session: block.session.clone(),
+ key_month: key.key_month.clone(),
+ frame,
+ images,
+ counts,
+ url_fingerprints: url_fingerprints(&key.key, &block_urls(block)),
+ })
+}
+
+/// The authenticated data of a block frame.
+pub fn block_context(session: &SessionKey, key_month: &str) -> Context {
+ Context {
+ team_id: session.team_id,
+ session_id: Some(session.session_id.clone()),
+ key_month: Some(key_month.to_owned()),
+ kind: "rrweb".to_owned(),
+ codec: Some(Codec::Brotli),
+ ..Context::default()
+ }
+}
+
+/// The frames the flush's packs would hold if each session sealed its images under its own key: one per distinct
+/// (session, reference) pair, where [`new_images`] keeps one per reference.
+pub fn session_keyed_frames(sealed: &[SealedBlock]) -> u64 {
+ let pairs: HashSet<(&str, &str)> = sealed
+ .iter()
+ .flat_map(|block| {
+ let session = block.session.session_id.as_str();
+ block
.images
.iter()
- .map(|i| i.bytes.len() as u64)
- .sum::<u64>();
- block.stats.images_inline += sink.left_inline;
- block_images.extend(sink.images);
- }
- // Fail closed: a declined URL, a credential or a private host, must not reach storage.
- if !rewrite.finish() {
- counter!("ai_research_v4_recorder_dropped_blocks_total", "outcome" => "declined_unrewritten")
- .increment(1);
- continue;
- }
- let images = prepared.images.entry(block.session.team_id).or_default();
- for image in block_images {
- if seen_in_session.insert((id.session_id.clone(), image.reference.clone())) {
- prepared.session_frames += 1;
- }
+ .map(move |image| (session, image.reference.as_str()))
+ })
+ .collect();
+ pairs.len() as u64
+}
+
+/// The flush's new images by team, each once.
+pub fn new_images(sealed: &mut [SealedBlock]) -> BTreeMap<i64, Vec<Image>> {
+ let mut images: BTreeMap<i64, Vec<Image>> = BTreeMap::new();
+ let mut seen: HashSet<String> = HashSet::new();
+ for block in sealed {
+ for image in std::mem::take(&mut block.images) {
if seen.insert(image.reference.clone()) {
- images.push(image);
+ images.entry(block.session.team_id).or_default().push(image);
}
}
- prepared.kept[index] = true;
- prepared.blocks.push(PreparedBlock {
- session: block.session.clone(),
- key: key.clone(),
- lines,
+ }
+ images
+}
+
+fn seal_pack(
+ team_id: i64,
+ team_images: &[Image],
+ team_key: &[u8; 32],
+ key_month: &str,
+ pack_key: &str,
+) -> Result<(Vec<u8>, Vec<ImageIndexRow>), WriteError> {
+ let mut pack = Vec::new();
+ let mut rows = Vec::with_capacity(team_images.len());
+ for image in team_images {
+ let context = Context {
+ team_id,
+ key_month: Some(key_month.to_owned()),
+ kind: "image".to_owned(),
+ codec: Some(Codec::None),
+ reference: Some(image.reference.clone()),
+ ..Context::default()
+ };
+ let frame = seal_frame(team_key, Version::V4, &context, &image.bytes)
+ .map_err(|e| WriteError::Seal(e.to_string()))?;
+ let byte_start = pack.len() as u64;
+ pack.extend_from_slice(&frame);
+ rows.push(ImageIndexRow {
+ team_id,
+ key_month: key_month.to_owned(),
+ image_ref: image.reference.clone(),
+ pack_s3_key: pack_key.to_owned(),
+ byte_start,
+ byte_end: pack.len() as u64 - 1,
+ mime: image.mime.to_owned(),
+ byte_size: image.bytes.len() as u64,
});
}
- prepared.images.retain(|_, images| !images.is_empty());
- prepared
+ Ok((pack, rows))
}
-/// Seals each team's images under its team month key into one pack, then writes the image index for the flush.
+/// Seals each team's images under its team month key into one pack, stores the packs concurrently, then writes the
+/// image index for the flush.
#[allow(clippy::too_many_arguments)]
pub async fn write_images(
- images: &BTreeMap<i64, Vec<Image>>,
+ images: BTreeMap<i64, Vec<Image>>,
keys: &FlushKeys,
write_day: &str,
key_month: &str,
images_bucket: &str,
rrweb_bucket: &str,
- store: &dyn BlobStore,
+ store: &Arc<dyn BlobStore>,
at: DateTime<Utc>,
+ concurrency: usize,
+ permits: &Arc<Semaphore>,
) -> Result<Vec<ImageIndexRow>, WriteError> {
- let mut rows = Vec::new();
- for (&team_id, team_images) in images {
- let Some(team_key) = keys.team_months.get(&team_id) else {
- continue;
- };
- let pack_key = format!(
- "images/packs/dt={write_day}/{team_id}/{}",
- unique_suffix(at)
- );
- let mut pack = Vec::new();
- for image in team_images {
- let context = Context {
- team_id,
- key_month: Some(key_month.to_owned()),
- kind: "image".to_owned(),
- codec: Some(Codec::None),
- reference: Some(image.reference.clone()),
- ..Context::default()
- };
- let frame = seal_frame(team_key, Version::V4, &context, &image.bytes)
- .map_err(|e| WriteError::Seal(e.to_string()))?;
- let byte_start = pack.len() as u64;
- pack.extend_from_slice(&frame);
- rows.push(ImageIndexRow {
- team_id,
- key_month: key_month.to_owned(),
- image_ref: image.reference.clone(),
- pack_s3_key: pack_key.clone(),
- byte_start,
- byte_end: pack.len() as u64 - 1,
- mime: image.mime.to_owned(),
- byte_size: image.bytes.len() as u64,
- });
- }
- store.put_new(images_bucket, &pack_key, pack).await?;
- }
+ let packs: Vec<_> = images
+ .into_iter()
+ .filter_map(|(team_id, team_images)| {
+ let team_key = *keys.team_months.get(&team_id)?;
+ let pack_key = format!(
+ "images/packs/dt={write_day}/{team_id}/{}",
+ unique_suffix(at)
+ );
+ let key_month = key_month.to_owned();
+ let store = store.clone();
+ Some(async move {
+ let key = pack_key.clone();
+ let (pack, rows) = seal_blocking(permits, move || {
+ seal_pack(team_id, &team_images, &team_key, &key_month, &key)
+ })
+ .await?;
+ store.put_new(images_bucket, &pack_key, pack).await?;
+ Ok::<_, WriteError>(rows)
+ })
+ })
+ .collect();
+ let rows: Vec<ImageIndexRow> = stream::iter(packs)
+ .buffer_unordered(concurrency.max(1))
+ .try_collect::<Vec<_>>()
+ .await?
+ .into_iter()
+ .flatten()
+ .collect();
if !rows.is_empty() {
let index_key = format!("index/images/dt={write_day}/{}.parquet", unique_suffix(at));
- store
- .put_new(
- rrweb_bucket,
- &index_key,
- write_parquet(&image_index(&rows)?)?,
- )
- .await?;
+ let batch = image_index(&rows)?;
+ let file = blocking(move || write_parquet(&batch).map_err(WriteError::from)).await?;
+ store.put_new(rrweb_bucket, &index_key, file).await?;
}
Ok(rows)
}
-/// Seals each prepared block under its session key into one object.
+/// Writes the flush's sealed blocks as one object. Returns where each landed, in the same order.
pub async fn write_blocks(
- blocks: &[PreparedBlock],
+ blocks: &[SealedBlock],
+ object_key: &str,
bucket: &str,
store: &dyn BlobStore,
- at: DateTime<Utc>,
) -> Result<Vec<Written>, WriteError> {
- let object_key = object_key(&write_day(at), at);
- let mut object = Vec::new();
- let mut written = Vec::new();
+ let mut object = Vec::with_capacity(blocks.iter().map(|b| b.frame.len()).sum());
+ let mut written = Vec::with_capacity(blocks.len());
for block in blocks {
- let mut encoder = brotlic::CompressorWriter::with_encoder(block_encoder()?, Vec::new());
- encoder
- .write_all(&block.lines)
- .map_err(|e| WriteError::Seal(e.to_string()))?;
- let compressed = encoder
- .into_inner()
- .map_err(|e| WriteError::Seal(e.to_string()))?;
- let context = Context {
- team_id: block.session.team_id,
- session_id: Some(block.session.session_id.clone()),
- key_month: Some(block.key.key_month.clone()),
- kind: "rrweb".to_owned(),
- codec: Some(Codec::Brotli),
- ..Context::default()
- };
- let frame = seal_frame(&block.key.key, Version::V4, &context, &compressed)
- .map_err(|e| WriteError::Seal(e.to_string()))?;
let byte_start = object.len() as u64;
- object.extend_from_slice(&frame);
+ object.extend_from_slice(&block.frame);
written.push(Written {
session: block.session.clone(),
- key_month: block.key.key_month.clone(),
- object_key: object_key.clone(),
+ key_month: block.key_month.clone(),
+ object_key: object_key.to_owned(),
byte_start,
byte_end: object.len() as u64 - 1,
});
}
if !written.is_empty() {
- store.put_new(bucket, &object_key, object).await?;
+ store.put_new(bucket, object_key, object).await?;
}
Ok(written)
}
+/// Runs CPU work off the async runtime.
+pub async fn blocking<T: Send + 'static>(
+ work: impl FnOnce() -> Result<T, WriteError> + Send + 'static,
+) -> Result<T, WriteError> {
+ tokio::task::spawn_blocking(work).await.map_err(|e| {
+ if e.is_panic() {
+ WriteError::Panicked(e.to_string())
+ } else {
+ WriteError::Seal(e.to_string())
+ }
+ })?
+}
+
+/// Runs a seal off the async runtime under one of `permits`. The seal holds its permit until it ends, so the seals of
+/// an attempt that timed out still count against the limit.
+pub async fn seal_blocking<T: Send + 'static>(
+ permits: &Arc<Semaphore>,
+ work: impl FnOnce() -> Result<T, WriteError> + Send + 'static,
+) -> Result<T, WriteError> {
+ let permit = permits
+ .clone()
+ .acquire_owned()
+ .await
+ .map_err(|e| WriteError::Seal(e.to_string()))?;
+ blocking(move || {
+ let _permit = permit;
+ work()
+ })
+ .await
+}
+
/// The pages a block visited, from its Meta events, in order and without repeats in a row.
/// An SPA navigation, as a `$url_changed` custom event, counts as a visit too.
pub fn block_urls(block: &Block) -> Vec<String> {
@@ -338,21 +442,16 @@ pub fn url_fingerprints(session_key: &[u8; 32], urls: &[String]) -> Vec<String>
}
/// Writes one block index file for the flush, after its blocks, so no row points at a block that is not stored.
+/// Each entry is a block, its stats as this attempt filled them, and where it landed.
pub async fn write_block_index(
- blocks: &[Block],
- written: &[Written],
+ entries: &[(&Block, &BlockStats, &Written)],
keys: &FlushKeys,
rrweb_bucket: &str,
store: &dyn BlobStore,
at: DateTime<Utc>,
) -> Result<usize, WriteError> {
- let by_session: BTreeMap<&SessionKey, &Block> =
- blocks.iter().map(|b| (&b.session, b)).collect();
- let mut rows = Vec::with_capacity(written.len());
- for written in written {
- let Some(block) = by_session.get(&written.session) else {
- continue;
- };
+ let mut rows = Vec::with_capacity(entries.len());
+ for &(block, stats, written) in entries {
let id = SessionId {
team_id: written.session.team_id,
session_id: written.session.session_id.clone(),
@@ -382,7 +481,7 @@ pub async fn write_block_index(
let first = block.messages.first();
rows.push(BlockIndexRow {
written,
- stats: &block.stats,
+ stats,
moments: &block.moments,
snapshot_source: first.and_then(|m| m.snapshot_source.as_deref()),
snapshot_library: first.and_then(|m| m.library.as_deref()),
@@ -397,25 +496,22 @@ pub async fn write_block_index(
write_day(at),
unique_suffix(at)
);
- store
- .put_new(
- rrweb_bucket,
- &index_key,
- write_parquet(&block_index(&rows)?)?,
- )
- .await?;
- Ok(rows.len())
+ let count = rows.len();
+ let batch = block_index(&rows)?;
+ let file = blocking(move || write_parquet(&batch).map_err(WriteError::from)).await?;
+ store.put_new(rrweb_bucket, &index_key, file).await?;
+ Ok(count)
}
#[cfg(test)]
mod tests {
use std::collections::HashMap;
- use std::io::{self, Read};
+ use std::io::Read;
use std::sync::Mutex;
use aes_gcm::aead::{Aead, KeyInit, Payload};
use aes_gcm::{Aes256Gcm, Nonce};
- use arrow_array::{Array, Int64Array, ListArray, StringArray, StructArray};
+ use arrow_array::{Array, Int64Array, ListArray, RecordBatch, StringArray, StructArray};
use arrow_schema::DataType;
use async_trait::async_trait;
use base64::engine::general_purpose::STANDARD;
@@ -425,7 +521,6 @@ mod tests {
use replay_core_v4::envelope::open_frame;
use replay_core_v4::inspect::dom::MAX_REMOTE_URL_BYTES;
use replay_core_v4::inspect::session::Moment;
- use replay_core_v4::inspect::stats::BlockStats;
use replay_core_v4::scan::message::read_message;
use serde_json::{json, Value};
@@ -440,6 +535,29 @@ mod tests {
fn get(&self, bucket: &str, key: &str) -> Vec<u8> {
self.objects.lock().unwrap()[&(bucket.to_owned(), key.to_owned())].clone()
}
+
+ fn keys(&self, prefix: &str) -> Vec<String> {
+ let mut keys: Vec<String> = self
+ .objects
+ .lock()
+ .unwrap()
+ .keys()
+ .filter(|(_, key)| key.starts_with(prefix))
+ .map(|(_, key)| key.clone())
+ .collect();
+ keys.sort();
+ keys
+ }
+
+ fn parquet(&self, key: &str) -> RecordBatch {
+ ParquetRecordBatchReaderBuilder::try_new(Bytes::from(self.get("rrweb-v4", key)))
+ .unwrap()
+ .build()
+ .unwrap()
+ .next()
+ .unwrap()
+ .unwrap()
+ }
}
fn decompress(compressed: &[u8]) -> io::Result<Vec<u8>> {
@@ -489,13 +607,6 @@ mod tests {
}
}
- /// The length of each line, its line break included.
- fn lines_of(lines: &[u8]) -> impl Iterator<Item = u64> + '_ {
- lines
- .split_inclusive(|&b| b == b'\n')
- .map(|line| line.len() as u64)
- }
-
fn uri(bytes: &[u8]) -> String {
format!("data:image/png;base64,{}", STANDARD.encode(bytes))
}
@@ -514,25 +625,12 @@ mod tests {
inspected
.remote_urls
.extend(extra.iter().map(|url| (*url).to_owned()));
- let mut keys = FlushKeys::default();
- keys.sessions.insert(
- SessionId {
- team_id: 7,
- session_id: "kept".to_owned(),
- },
- Key {
- key: [5; 32],
- key_month: "2026-10".to_owned(),
- },
- );
- keys.team_months.insert(7, [6; 32]);
- let mut blocks = vec![inspected];
- let prepared = prepare(&mut blocks, &keys, "2026-10");
- prepared
- .blocks
- .first()
- .map(|b| String::from_utf8(b.lines.clone()).unwrap())
- .unwrap_or_default()
+ let mut lines = Vec::new();
+ let (_, counts) = prepare_block(&inspected, &[6; 32], "2026-10", &mut lines).unwrap();
+ if counts.dropped.is_some() {
+ return String::new();
+ }
+ String::from_utf8(lines).unwrap()
}
#[test]
@@ -606,31 +704,18 @@ mod tests {
"https://cdn.example.com/a.png?token=secret",
);
let events = json!([{"type": 2, "timestamp": 1, "data": {"node": {"attributes": {"src": fetched, "srcset": format!("{declined} 2x")}}}}]);
- let mut blocks = vec![block("kept", events.clone()), block("dropped", events)];
- for block in &mut blocks {
- block.image_urls = vec![fetched.to_owned()];
- block.remote_urls = vec![fetched.to_owned(), declined.to_owned()];
- }
- blocks[1].url_limit = Some(replay_core_v4::inspect::dom::UrlLimit::UrlLength);
+ let mut block = block("kept", events);
+ block.image_urls = vec![fetched.to_owned()];
+ block.remote_urls = vec![fetched.to_owned(), declined.to_owned()];
let mut keys = FlushKeys::default();
- for session in ["kept", "dropped"] {
- keys.sessions.insert(
- SessionId {
- team_id: 7,
- session_id: session.to_owned(),
- },
- Key {
- key: [5; 32],
- key_month: "2026-10".to_owned(),
- },
- );
- }
+ let (id, key) = session_key("kept");
+ keys.sessions.insert(id, key);
keys.team_months.insert(7, [6; 32]);
- let prepared = prepare(&mut blocks, &keys, "2026-10");
- // Only a stored block sends fetch jobs.
- assert_eq!(prepared.kept, [true, false]);
- let jobs = crate::urls::fetch_jobs(&mut blocks[..1], &keys, "2026-10", 1, &|_| false);
- let lines = String::from_utf8(prepared.blocks[0].lines.clone()).unwrap();
+ let mut stats = BlockStats::default();
+ let jobs = crate::urls::fetch_jobs([(&block, &mut stats)], &keys, "2026-10", 1, &|_| false);
+ let mut lines = Vec::new();
+ prepare_block(&block, &[6; 32], "2026-10", &mut lines).unwrap();
+ let lines = String::from_utf8(lines).unwrap();
let reference = jobs["example.com"][0]["originalRef"].as_str().unwrap();
assert!(
lines.contains(&format!(r#""src":"{reference}""#)),
@@ -643,6 +728,56 @@ mod tests {
assert!(!lines.contains("cdn.example.com"));
}
+ fn session_key(session: &str) -> (SessionId, Key) {
+ (
+ SessionId {
+ team_id: 7,
+ session_id: session.to_owned(),
+ },
+ Key {
+ key: [5; 32],
+ key_month: "2026-09".to_owned(),
+ },
+ )
+ }
+
+ fn at() -> DateTime<Utc> {
+ DateTime::parse_from_rfc3339("2026-10-15T12:00:00Z")
+ .unwrap()
+ .with_timezone(&Utc)
+ }
+
+ #[test]
+ fn keeps_one_copy_of_an_image_that_several_messages_repeat() {
+ let events = json!([{"type": 2, "timestamp": 1, "data": {"node": {"attributes": {"src": uri(b"logo")}}}}]);
+ let mut repeated = block("s", events.clone());
+ repeated.messages.extend(block("s", events).messages);
+ let (images, counts) =
+ prepare_block(&repeated, &[6; 32], "2026-10", &mut io::sink()).unwrap();
+ assert_eq!((images.len(), counts.extracted), (1, 2));
+ }
+
+ #[tokio::test]
+ async fn a_seal_of_an_abandoned_attempt_keeps_its_permit() {
+ let permits = Arc::new(Semaphore::new(1));
+ let (release, wait) = std::sync::mpsc::channel::<()>();
+ let abandoned = seal_blocking(&permits, move || {
+ wait.recv().ok();
+ Ok(())
+ });
+ assert!(
+ tokio::time::timeout(std::time::Duration::from_millis(50), abandoned)
+ .await
+ .is_err()
+ );
+ assert_eq!(permits.available_permits(), 0);
+ release.send(()).unwrap();
+ seal_blocking(&permits, || Ok(())).await.unwrap();
+ // The flush drops a block whose seal panicked and keeps the others, so a panic is told apart.
+ let panicked = seal_blocking(&permits, || -> Result<(), WriteError> { panic!("seal") });
+ assert!(matches!(panicked.await, Err(WriteError::Panicked(_))));
+ }
+
#[tokio::test]
async fn stores_images_once_and_blocks_that_point_at_them() {
let events = json!([
@@ -652,108 +787,94 @@ mod tests {
{"type": 5, "timestamp": 3, "data": {"tag": "$url_changed", "payload": {"href": "https://app.example.com/b"}}},
{"type": 4, "timestamp": 4, "data": {"href": "https://app.example.com/a"}},
]);
- let mut blocks = vec![block("kept", events.clone()), block("tombstoned", events)];
+ let mut blocks = [block("kept", events.clone()), block("again", events)];
let mut keys = FlushKeys::default();
- keys.sessions.insert(
- SessionId {
- team_id: 7,
- session_id: "kept".to_owned(),
- },
- Key {
- key: [5; 32],
- key_month: "2026-09".to_owned(),
- },
- );
+ for session in ["kept", "again"] {
+ let (id, key) = session_key(session);
+ keys.sessions.insert(id, key);
+ }
keys.team_months.insert(7, [6; 32]);
let old_ref = replay_core_v4::refs::image_ref(
7,
"2026-10",
&replay_core_v4::refs::reference_hash(&image_content_key(&[6; 32]), b"old"),
);
- let prepared = prepare(&mut blocks, &keys, "2026-10");
-
- // The same image in two sessions of the team is one frame and one index row.
- let same = json!([{"type": 2, "timestamp": 1, "data": {"node": {"attributes": {"src": uri(b"shared")}}}}]);
- let mut two = vec![block("kept", same.clone()), block("other", same)];
- let mut both = FlushKeys::default();
- both.team_months.insert(7, [6; 32]);
- for (session, key) in [("kept", [5; 32]), ("other", [4; 32])] {
- both.sessions.insert(
- SessionId {
- team_id: 7,
- session_id: session.to_owned(),
- },
- Key {
- key,
- key_month: "2026-10".to_owned(),
- },
- );
- }
- // A key per session would need a frame in each.
- let shared = prepare(&mut two, &both, "2026-10");
- assert_eq!((shared.images[&7].len(), shared.session_frames), (1, 2));
+ let object = object_key("2026-10-15", at());
+ let mut sealed: Vec<SealedBlock> = blocks
+ .iter()
+ .map(|b| {
+ seal_block(
+ b,
+ &keys.sessions[&session_key(&b.session.session_id).0],
+ &[6; 32],
+ "2026-10",
+ )
+ .unwrap()
+ })
+ .collect();
+ assert_eq!(
+ sealed[0].counts,
+ WriterCounts {
+ extracted: 2,
+ bytes: 6,
+ inline: 0,
+ ..sealed[0].counts
+ }
+ );
+ // The page the block sees again keeps its first place, under the session's own fingerprint key.
+ let fingerprint_key = url_fingerprint_key(&[5; 32]);
+ let fingerprint = |path: &str| {
+ url_fingerprint(&fingerprint_key, &format!("https://app.example.com{path}"))
+ };
+ assert_eq!(
+ sealed[0].url_fingerprints,
+ [fingerprint("/a"), fingerprint("/b")]
+ );
- let store = MemoryStore::default();
- let at = DateTime::parse_from_rfc3339("2026-10-15T12:00:00Z")
- .unwrap()
- .with_timezone(&Utc);
+ let session_frames = session_keyed_frames(&sealed);
+ let memory = Arc::new(MemoryStore::default());
+ let store: Arc<dyn BlobStore> = memory.clone();
let rows = write_images(
- &prepared.images,
+ new_images(&mut sealed),
&keys,
"2026-10-15",
"2026-10",
"images-v4",
"rrweb-v4",
&store,
- at,
+ at(),
+ 4,
+ &Arc::new(Semaphore::new(4)),
)
.await
.unwrap();
- let written = write_blocks(&prepared.blocks, "rrweb-v4", &store, at)
+ let written = write_blocks(&sealed, &object, "rrweb-v4", store.as_ref())
.await
.unwrap();
- // The pack holds both images, also the one an earlier flush stored.
+ // Both blocks moved the same two images out, and the pack holds each once.
assert_eq!(rows.len(), 2);
+ // A key per session would need each session's copy.
+ assert_eq!(session_frames, 4);
assert!(rows.iter().any(|row| row.image_ref == old_ref));
- assert_eq!(
- (
- blocks[0].stats.images_extracted,
- blocks[0].stats.images_inline
- ),
- (2, 0)
- );
- let longest = lines_of(&prepared.blocks[0].lines).max().unwrap();
- assert_eq!(
- (
- blocks[0].stats.written_jsonl_bytes,
- blocks[0].stats.max_written_event_bytes
- ),
- (prepared.blocks[0].lines.len() as u64, longest)
- );
- assert_eq!(
- written
- .iter()
- .map(|w| w.session.session_id.as_str())
- .collect::<Vec<_>>(),
- vec!["kept"]
- );
+ assert_eq!(written.len(), 2);
let block = &written[0];
- let frame = &store.get("rrweb-v4", &block.object_key)
+ let frame = &memory.get("rrweb-v4", &block.object_key)
[block.byte_start as usize..=block.byte_end as usize];
- let context = Context {
- team_id: 7,
- session_id: Some("kept".to_owned()),
- key_month: Some("2026-09".to_owned()),
- kind: "rrweb".to_owned(),
- codec: Some(Codec::Brotli),
- ..Context::default()
- };
+ let context = block_context(&blocks[0].session, "2026-09");
let lines = String::from_utf8(
decompress(&open_frame(&[5; 32], Version::V4, &context, frame).unwrap()).unwrap(),
)
.unwrap();
+ let longest = lines.split_inclusive('\n').map(str::len).max().unwrap();
+ assert_eq!(
+ (
+ sealed[0].counts.jsonl_bytes,
+ sealed[0].counts.max_line_bytes
+ ),
+ (lines.len() as u64, longest as u64)
+ );
let first: Value = serde_json::from_str(lines.lines().nth(1).unwrap()).unwrap();
let reference = first[1]["data"]["node"]["attributes"]["src"]
.as_str()
@@ -763,7 +884,7 @@ mod tests {
// The reference resolves through the index to the image's bytes in the pack.
let row = &rows[0];
- let pack = store.get("images-v4", &row.pack_s3_key);
+ let pack = memory.get("images-v4", &row.pack_s3_key);
let image_context = Context {
team_id: 7,
key_month: Some("2026-10".to_owned()),
@@ -772,56 +893,40 @@ mod tests {
reference: Some(row.image_ref.clone()),
..Context::default()
};
- let sealed = &pack[row.byte_start as usize..=row.byte_end as usize];
+ let sealed_image = &pack[row.byte_start as usize..=row.byte_end as usize];
assert_eq!(
- open_frame(&[6; 32], Version::V4, &image_context, sealed).unwrap(),
+ open_frame(&[6; 32], Version::V4, &image_context, sealed_image).unwrap(),
b"new"
);
- // The team month key seals images, as the fetcher's, so the session key does not open them.
- assert!(open_frame(&[5; 32], Version::V4, &image_context, sealed).is_err());
- assert_eq!(
- store
- .objects
- .lock()
- .unwrap()
- .keys()
- .filter(|(_, key)| key.starts_with("index/images/dt=2026-10-15/"))
- .count(),
- 1
- );
+ // The team month key seals images, as the fetcher's, so a session key does not open them.
+ assert!(open_frame(&[5; 32], Version::V4, &image_context, sealed_image).is_err());
+ assert_eq!(memory.keys("index/images/dt=2026-10-15/").len(), 1);
// The block index row points at the block, keeps the moments, and seals the visited pages.
- blocks[0].stats.window_ids = vec!["w1".to_owned()];
blocks[0].moments = vec![Moment {
kind: "full_snapshot",
window_id: "w1".into(),
ts_ms: 1,
event_index: 1,
}];
+ let stats = BlockStats {
+ url_fingerprints: sealed[0].url_fingerprints.clone(),
+ window_ids: vec!["w1".to_owned()],
+ ..BlockStats::default()
+ };
assert_eq!(
- write_block_index(&blocks, &written, &keys, "rrweb-v4", &store, at)
- .await
- .unwrap(),
+ write_block_index(
+ &[(&blocks[0], &stats, &written[0])],
+ &keys,
+ "rrweb-v4",
+ store.as_ref(),
+ at()
+ )
+ .await
+ .unwrap(),
1
);
- let index_key = store
- .objects
- .lock()
- .unwrap()
- .keys()
- .find(|(_, key)| key.starts_with("index/blocks/"))
- .unwrap()
- .1
- .clone();
- let batch = ParquetRecordBatchReaderBuilder::try_new(Bytes::from(
- store.get("rrweb-v4", &index_key),
- ))
- .unwrap()
- .build()
- .unwrap()
- .next()
- .unwrap()
- .unwrap();
+ let batch = memory.parquet(&memory.keys("index/blocks/")[0]);
let column = |name: &str| {
batch
.column_by_name(name)
@@ -867,15 +972,7 @@ mod tests {
let values = values.as_any().downcast_ref::<StringArray>().unwrap();
values.iter().map(|v| v.unwrap().to_owned()).collect()
};
- // The page the block sees again keeps its first place, under the session's own fingerprint key.
- let fingerprint_key = replay_core_v4::seal::url_fingerprint_key(&[5; 32]);
- let fingerprint = |path: &str| {
- url_fingerprint(&fingerprint_key, &format!("https://app.example.com{path}"))
- };
- assert_eq!(
- strings("url_fingerprints"),
- [fingerprint("/a"), fingerprint("/b")]
- );
+ assert_eq!(strings("url_fingerprints"), stats.url_fingerprints);
assert_eq!(strings("window_ids"), ["w1"]);
let moments = column("moments");
let moments = moments
@@ -898,7 +995,7 @@ mod tests {
"full_snapshot"
);
let envelope: Value = serde_json::from_str(&text("urls")).unwrap();
- let sealed = STANDARD
+ let sealed_urls = STANDARD
.decode(envelope["ciphertext"].as_str().unwrap())
.unwrap();
let nonce = STANDARD
@@ -918,7 +1015,7 @@ mod tests {
.decrypt(
Nonce::from_slice(&nonce),
Payload {
- msg: &sealed,
+ msg: &sealed_urls,
aad: aad.as_bytes(),
},
)