:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(replay-recorder-v4): flush in parallel off the async runtime (#118419)
Nicholas Waltz5af65405e57b
diff --git a/rust/ai-research/replay-recorder-v4/src/batch.rs b/rust/ai-research/replay-recorder-v4/src/batch.rsindex 337a77c92348..b572566e72b8 100644--- a/rust/ai-research/replay-recorder-v4/src/batch.rs+++ b/rust/ai-research/replay-recorder-v4/src/batch.rs@@ -3,6 +3,7 @@use std::collections::{BTreeMap, HashMap};use std::time::{Duration, Instant};+use futures::stream::{self, StreamExt};use metrics::counter;use replay_core_v4::inspect::dom::UrlLimit;use replay_core_v4::inspect::session::{BlockInspection, Moment, SessionInspector};@@ -115,64 +116,74 @@ pub struct Inspectors {}impl Inspectors {- /// Decodes every buffered message, runs it through its session's inspector and returns one block per session.- pub fn inspect(&mut self, batch: Batch) -> (Vec<Block>, HashMap<Tp, i64>) {- let now = Instant::now();+ /// Decodes every buffered message and runs it through its session's inspector, with up to `concurrency` sessions+ /// at once off the async runtime. Returns one block per session.+ pub async fn inspect(+ &mut self,+ batch: Batch,+ concurrency: usize,+ ) -> (Vec<Block>, HashMap<Tp, i64>) {let mut by_session: BTreeMap<SessionKey, (Vec<Tp>, Vec<Raw>)> = BTreeMap::new();for ((session, tp), pending) in batch.sessions {let (tps, raws) = by_session.entry(session).or_default();tps.push(tp);raws.extend(pending.messages);}- let mut blocks = Vec::with_capacity(by_session.len());- for (session, (tps, raws)) in by_session {- let mut messages = Vec::with_capacity(raws.len());- for raw in raws {- let message = read(raw.payload, raw.content_encoding.as_deref())- .and_then(|m| check_read(&m, &session.session_id).map(|()| m));- match message {- Ok(mut message) => {- // Decoded once here, for the inspector and the block writer both.- message.decode_compressed();- messages.push(message);- }- Err(skip) => {- counter!("ai_research_v4_recorder_messages_total", "outcome" => skip.as_str())- .increment(1);+ let tasks: Vec<_> = by_session+ .into_iter()+ .map(|(session, (tps, raws))| {+ let (inspector, prior) = match self.sessions.remove(&session) {+ Some((inspector, partitions, last_seen)) => {+ (inspector, Some((partitions, last_seen)))}+ None => (SessionInspector::default(), None),+ };+ (session, inspector, prior, tps, raws)+ })+ .collect();+ let done: Vec<_> = stream::iter(tasks)+ .map(|(session, inspector, prior, tps, raws)| {+ let (key, messages) = (session.clone(), raws.len());+ let task = tokio::task::spawn_blocking(move || {+ let (inspector, block) = inspect_session(&session, inspector, raws);+ (session, inspector, prior, tps, block)+ });+ async move { (key, messages, task.await) }+ })+ .buffer_unordered(concurrency.max(1))+ .collect()+ .await;+ let now = Instant::now();+ let mut blocks = Vec::new();+ for (key, messages, result) in done {+ // A panic loses that session's block and its inspector state, and the rest of the flush goes on.+ let (session, inspector, prior, tps, block) = match result {+ Ok(inspected) => inspected,+ Err(error) => {+ tracing::error!(%error, team_id = key.team_id, "inspecting a session panicked: its block is dropped");+ counter!("ai_research_v4_recorder_messages_total", "outcome" => "panicked")+ .increment(messages as u64);+ continue;+ }+ };+ // A session that decoded nothing left its state as it was, and does not count as seen.+ let Some(block) = block else {+ if let Some((partitions, last_seen)) = prior {+ self.sessions+ .insert(session, (inspector, partitions, last_seen));}- }- if messages.is_empty() {continue;- }- counter!("ai_research_v4_recorder_messages_total", "outcome" => "recorded")- .increment(messages.len() as u64);- // Only a session that decoded something counts as seen.- let (inspector, partitions, last_seen) = self- .sessions- .entry(session.clone())- .or_insert_with(|| (SessionInspector::default(), Vec::new(), now));+ };+ let mut partitions = prior.map(|(partitions, _)| partitions).unwrap_or_default();for tp in tps {if !partitions.contains(&tp) {partitions.push(tp);}}- *last_seen = now;- let mut block = BlockInspection::default();- for message in &messages {- inspector.observe_message(&mut block, message);- }- let inspected = block.finish();- blocks.push(Block {- session,- messages,- stats: inspected.stats,- moments: inspected.moments,- image_urls: inspected.image_urls,- remote_urls: inspected.remote_urls,- url_limit: inspected.url_limit,- });+ self.sessions.insert(session, (inspector, partitions, now));+ blocks.push(block);}+ blocks.sort_by(|a, b| a.session.cmp(&b.session));(blocks, batch.offsets)}@@ -188,6 +199,49 @@ impl Inspectors {}}+fn inspect_session(+ session: &SessionKey,+ mut inspector: SessionInspector,+ raws: Vec<Raw>,+) -> (SessionInspector, Option<Block>) {+ let mut messages = Vec::with_capacity(raws.len());+ for raw in raws {+ let message = read(raw.payload, raw.content_encoding.as_deref())+ .and_then(|m| check_read(&m, &session.session_id).map(|()| m));+ match message {+ Ok(mut message) => {+ // Decoded once here, for the inspector and the block writer both.+ message.decode_compressed();+ messages.push(message);+ }+ Err(skip) => {+ counter!("ai_research_v4_recorder_messages_total", "outcome" => skip.as_str())+ .increment(1);+ }+ }+ }+ if messages.is_empty() {+ return (inspector, None);+ }+ counter!("ai_research_v4_recorder_messages_total", "outcome" => "recorded")+ .increment(messages.len() as u64);+ let mut block = BlockInspection::default();+ for message in &messages {+ inspector.observe_message(&mut block, message);+ }+ let inspected = block.finish();+ let block = Block {+ session: session.clone(),+ messages,+ stats: inspected.stats,+ moments: inspected.moments,+ image_urls: inspected.image_urls,+ remote_urls: inspected.remote_urls,+ url_limit: inspected.url_limit,+ };+ (inspector, Some(block))+}+#[cfg(test)]mod tests {use serde_json::{json, Value};@@ -227,8 +281,8 @@ mod tests {json!([{"type": 3, "timestamp": 3, "data": {"source": 1, "positions": []}}])}- #[test]- fn a_revoked_partition_leaves_nothing_behind() {+ #[tokio::test]+ async fn a_revoked_partition_leaves_nothing_behind() {let mut batch = Batch::default();batch.add(tp(0), key("a"), raw(10, "a", mouse()));batch.add(tp(1), key("b"), raw(20, "b", mouse()));@@ -237,7 +291,7 @@ mod tests {batch.drop_partitions(&[tp(1)]);inspectors.drop_partitions(&[tp(1)]);- let (blocks, offsets) = inspectors.inspect(batch);+ let (blocks, offsets) = inspectors.inspect(batch, 2).await;assert_eq!(blocks.iter()@@ -248,8 +302,8 @@ mod tests {assert_eq!(offsets, HashMap::from([(tp(0), 10)]));}- #[test]- fn a_session_that_decodes_nothing_keeps_no_state() {+ #[tokio::test]+ async fn a_session_that_decodes_nothing_keeps_no_state() {let mut batch = Batch::default();batch.add(tp(0),@@ -261,22 +315,22 @@ mod tests {},);let mut inspectors = Inspectors::default();- assert!(inspectors.inspect(batch).0.is_empty());+ assert!(inspectors.inspect(batch, 1).await.0.is_empty());assert!(inspectors.sessions.is_empty());}- #[test]- fn session_state_carries_across_flushes() {+ #[tokio::test]+ async fn session_state_carries_across_flushes() {let mut inspectors = Inspectors::default();let mut first = Batch::default();first.add(tp(0), key("a"), raw(1, "a", snapshot()));first.add(tp(0), key("b"), raw(2, "b", mouse()));- inspectors.inspect(first);+ inspectors.inspect(first, 2).await;let mut second = Batch::default();second.add(tp(0), key("a"), raw(3, "a", mouse()));second.add(tp(0), key("b"), raw(4, "b", mouse()));- let (blocks, _) = inspectors.inspect(second);+ let (blocks, _) = inspectors.inspect(second, 2).await;let cold: Vec<_> = blocks.iter().map(|b| (b.session.session_id.as_str(), b.stats.cold_start))diff --git a/rust/ai-research/replay-recorder-v4/src/config.rs b/rust/ai-research/replay-recorder-v4/src/config.rsindex 6c0892ffc75a..5a5183b36f93 100644--- a/rust/ai-research/replay-recorder-v4/src/config.rs+++ b/rust/ai-research/replay-recorder-v4/src/config.rs@@ -73,6 +73,10 @@ pub struct Config {#[envconfig(from = "AI_RESEARCH_V4_FLUSH_MAX_BYTES", default = "67108864")]pub flush_max_bytes: usize,+ /// How many sessions of a flush are decoded, inspected and sealed at once. 0 is one per CPU.+ #[envconfig(from = "AI_RESEARCH_V4_FLUSH_CONCURRENCY", default = "0")]+ pub flush_concurrency: usize,+/// A session's inspector is dropped after this long without a message, so memory follows live sessions.#[envconfig(from = "AI_RESEARCH_V4_SESSION_IDLE_SECS", default = "3600")]pub session_idle_secs: u64,diff --git a/rust/ai-research/replay-recorder-v4/src/consumer.rs b/rust/ai-research/replay-recorder-v4/src/consumer.rsindex 961c618fb02c..6d754419009d 100644--- a/rust/ai-research/replay-recorder-v4/src/consumer.rs+++ b/rust/ai-research/replay-recorder-v4/src/consumer.rs@@ -5,7 +5,6 @@ use std::sync::{Arc, Mutex};use std::time::{Duration, SystemTime, UNIX_EPOCH};use common_kafka::config::KafkaConfig;-use common_kafka::kafka_producer::KafkaContext;use futures::FutureExt;use lifecycle::Handle;use metrics::counter;@@ -16,22 +15,11 @@ use rdkafka::error::KafkaError;use rdkafka::message::{BorrowedMessage, Headers as _, Message as _};use rdkafka::{ClientConfig, ClientContext, Offset, TopicPartitionList};-use chrono::Utc;-use replay_core_v4::keys::batch::{KeyStore, SessionId};-use replay_core_v4::keys::kms::AwsKms;-use replay_core_v4::keys::rows::{key_month, write_day};-use replay_core_v4::keys::table::{AwsKeyTable, CachedKeyTable};-use crate::batch::{Batch, Block, Inspectors, Raw, SessionKey, Tp};+use crate::flush::{write_flush, FlushError, Writer};use crate::limits::{Restrictions, SessionLimiter};use crate::process::{admit, sampled, Admitted, Headers, Skip};-use crate::store::BlobStore;use crate::teams::TeamCache;-use crate::urls::{fetch_jobs, records};-use crate::write::{prepare, write_block_index, write_blocks, write_images, WriteError};-use rdkafka::message::{Header, OwnedHeaders};-use rdkafka::producer::{FutureProducer, FutureRecord};-use rdkafka::util::Timeout;/// The most messages read before the limiter decides on their sessions.const POLL_BATCH: usize = 1000;@@ -149,31 +137,11 @@ fn header<'m>(message: &'m BorrowedMessage<'_>, name: &str) -> Option<&'m str> {.and_then(|v| std::str::from_utf8(v).ok())}-/// What a flush needs to store blocks. Without it the recorder runs dry.-pub struct Writer {- pub keys: KeyStore<CachedKeyTable<AwsKeyTable>, AwsKms>,- pub store: Arc<dyn BlobStore>,- pub rrweb_bucket: String,- pub images_bucket: String,- pub producer: FutureProducer<KafkaContext>,- pub image_fetch_topic: String,- /// Remote image references this process sent to the fetcher, by UTC day. A reference goes out again each day, so- /// the fetcher sees it within every refresh interval and keeps the image as long as the blocks that reference it.- pub sent_urls: moka::sync::Cache<(String, String), ()>,-}--#[derive(Debug, thiserror::Error)]-pub enum FlushError {- #[error(transparent)]- Keys(#[from] replay_core_v4::keys::batch::KeyError),- #[error(transparent)]- Write(#[from] WriteError),-}-pub struct Settings {pub sample_percentage: f64,pub flush_interval: Duration,pub flush_max_bytes: usize,+ pub flush_concurrency: usize,pub session_idle: Duration,/// The main topic, when the main lane moves its force-overflow messages to the overflow topic.pub redirecting_topic: Option<String>,@@ -239,8 +207,10 @@ pub async fn run(inspectors.drop_partitions(&revoked);}if batch.should_flush(settings.flush_interval, settings.flush_max_bytes) {- let (mut blocks, offsets) = inspectors.inspect(std::mem::take(&mut batch));- if let Err(error) = flush(&mut blocks, writer.as_ref()).await {+ let (blocks, offsets) = inspectors+ .inspect(std::mem::take(&mut batch), settings.flush_concurrency)+ .await;+ if let Err(error) = flush(blocks, writer.as_ref(), settings.flush_concurrency).await {// Nothing of this flush is committed, so a restart reads it again from the last commit.tracing::error!(%error, "flush failed");handle.signal_failure(format!("flush failed: {error}"));@@ -354,9 +324,13 @@ async fn admit_message(}/// Stores a flush's blocks, or only reports them when the recorder runs dry.-async fn flush(blocks: &mut [Block], writer: Option<&Writer>) -> Result<(), FlushError> {+async fn flush(+ blocks: Vec<Block>,+ writer: Option<&Writer>,+ concurrency: usize,+) -> Result<(), FlushError> {counter!("ai_research_v4_recorder_blocks_total").increment(blocks.len() as u64);- for block in blocks.iter() {+ for block in &blocks {counter!("ai_research_v4_recorder_block_events_total").increment(block.stats.event_count);if block.stats.cold_start {counter!("ai_research_v4_recorder_cold_start_blocks_total").increment(1);@@ -366,108 +340,6 @@ async fn flush(blocks: &mut [Block], writer: Option<&Writer>) -> Result<(), Flus}}let Some(writer) = writer else { return Ok(()) };- let now = Utc::now();- let day = write_day(now);- let month = key_month(&day);- let sessions: Vec<SessionId> = blocks- .iter()- .map(|b| SessionId {- team_id: b.session.team_id,- session_id: b.session.session_id.clone(),- })- .collect();- // Keys are stored before anything is sealed with them, as in v3.- let keys = writer.keys.keys_for_flush(&sessions, &day).await?;- counter!("ai_research_v4_recorder_dropped_sessions_total").increment(keys.dropped.len() as u64);- let prepared = prepare(blocks, &keys, month);- // Before the index is written: building the jobs counts each block's image URLs into its stats.- // A dropped block's images are not stored, so the fetcher does not fetch them.- let kept = blocks- .iter_mut()- .zip(&prepared.kept)- .filter_map(|(block, &kept)| kept.then_some(block));- let jobs = fetch_jobs(kept, &keys, month, now.timestamp_millis(), &|reference| {- writer- .sent_urls- .contains_key(&(day.clone(), reference.to_owned()))- });- let rows = write_images(- &prepared.images,- &keys,- &day,- month,- &writer.images_bucket,- &writer.rrweb_bucket,- writer.store.as_ref(),- now,- )- .await?;- counter!("ai_research_v4_recorder_stored_images_total").increment(rows.len() as u64);- // What sealing images under session keys would store, to compare with the frames above.- counter!("ai_research_v4_recorder_session_keyed_image_frames_total")- .increment(prepared.session_frames);- let written = write_blocks(- &prepared.blocks,- &writer.rrweb_bucket,- writer.store.as_ref(),- now,- )- .await?;- counter!("ai_research_v4_recorder_written_blocks_total").increment(written.len() as u64);- write_block_index(- blocks,- &written,- &keys,- &writer.rrweb_bucket,- writer.store.as_ref(),- now,- )- .await?;- produce_jobs(writer, jobs, &day, now.timestamp_millis()).await;- Ok(())-}--/// Sends the flush's fetch jobs and waits for their delivery before the flush commits, as the v3 mirror does. A send-/// that fails does not fail the flush, because its blocks are stored by then. The references it did not send stay-/// unmarked, so a later flush that sees them sends them again.-async fn produce_jobs(- writer: &Writer,- jobs: std::collections::BTreeMap<String, Vec<serde_json::Value>>,- day: &str,- now_ms: i64,-) {- let captured_at = now_ms.to_string();- let records = records(jobs);- let total: usize = records- .iter()- .map(|(_, _, references)| references.len())- .sum();- let mut sent = Vec::new();- for (domain, payload, references) in records {- let headers = OwnedHeaders::new().insert(Header {- key: "capture-timestamp-ms",- value: Some(&captured_at),- });- let record = FutureRecord::to(&writer.image_fetch_topic)- .key(&domain)- .payload(&payload)- .headers(headers);- if let Err((error, _)) = writer- .producer- .send(record, Timeout::After(Duration::from_secs(30)))- .await- {- // The next record would wait as long on the same broker.- tracing::warn!(%error, "sending image fetch jobs failed");- break;- }- sent.extend(references);- }- counter!("ai_research_v4_recorder_image_fetch_jobs_total").increment(sent.len() as u64);- counter!("ai_research_v4_recorder_image_fetch_jobs_unsent_total")- .increment((total - sent.len()) as u64);- for reference in sent {- writer.sent_urls.insert((day.to_owned(), reference), ());- }- metrics::gauge!("ai_research_v4_recorder_sent_urls").set(writer.sent_urls.entry_count() as f64);+ let blocks: Vec<Arc<Block>> = blocks.into_iter().map(Arc::new).collect();+ write_flush(&blocks, writer, concurrency).await}diff --git a/rust/ai-research/replay-recorder-v4/src/flush.rs b/rust/ai-research/replay-recorder-v4/src/flush.rsnew file mode 100644index 000000000000..ee92ed2b76f2--- /dev/null+++ b/rust/ai-research/replay-recorder-v4/src/flush.rs@@ -0,0 +1,259 @@+//! Stores a flush's blocks: their keys, the sealed blocks, the image packs, both indexes and the fetch jobs.++use std::collections::{BTreeMap, BTreeSet};+use std::sync::Arc;+use std::time::Duration;++use chrono::Utc;+use common_kafka::kafka_producer::KafkaContext;+use futures::stream::{self, StreamExt};+use metrics::counter;+use rdkafka::message::{Header, OwnedHeaders};+use rdkafka::producer::{FutureProducer, FutureRecord};+use rdkafka::util::Timeout;+use replay_core_v4::inspect::stats::BlockStats;+use replay_core_v4::keys::batch::{KeyStore, SessionId};+use replay_core_v4::keys::kms::AwsKms;+use replay_core_v4::keys::rows::{key_month, write_day};+use replay_core_v4::keys::table::{AwsKeyTable, CachedKeyTable};+use tokio::sync::Semaphore;++use crate::batch::Block;+use crate::store::BlobStore;+use crate::urls::{fetch_jobs, records};+use crate::write::{+ new_images, object_key, seal_block, seal_blocking, session_keyed_frames, write_block_index,+ write_blocks, write_images, WriteError,+};++/// How long a fetch job may wait for room in the producer queue.+const ENQUEUE_TIMEOUT: Duration = Duration::from_secs(10);++/// Fetch job records sent at once, so a slow broker slows the flush instead of filling the producer queue.+const SENDS_IN_FLIGHT: usize = 64;++/// What a flush needs to store blocks. Without it the recorder runs dry.+pub struct Writer {+ pub keys: KeyStore<CachedKeyTable<AwsKeyTable>, AwsKms>,+ pub store: Arc<dyn BlobStore>,+ pub rrweb_bucket: String,+ pub images_bucket: String,+ pub producer: FutureProducer<KafkaContext>,+ pub image_fetch_topic: String,+ /// Remote image references this process sent to the fetcher, by UTC day. A reference goes out again each day, so+ /// the fetcher sees it within every refresh interval and keeps the image as long as the blocks that reference it.+ pub sent_urls: moka::sync::Cache<(String, String), ()>,+ /// Seals running at once, over all attempts: the flush concurrency.+ pub seal_permits: Arc<Semaphore>,+}++#[derive(Debug, thiserror::Error)]+pub enum FlushError {+ #[error(transparent)]+ Keys(#[from] replay_core_v4::keys::batch::KeyError),+ #[error(transparent)]+ Write(#[from] WriteError),+}++/// Stores the blocks once. Up to `concurrency` blocks are sealed at once off the async runtime, and the packs, and+/// the fetch jobs, go out concurrently.+pub async fn write_flush(+ blocks: &[Arc<Block>],+ writer: &Writer,+ concurrency: usize,+) -> Result<(), FlushError> {+ let now = Utc::now();+ let day = write_day(now);+ let month = key_month(&day);+ let sessions: Vec<SessionId> = blocks+ .iter()+ .map(|b| SessionId {+ team_id: b.session.team_id,+ session_id: b.session.session_id.clone(),+ })+ .collect::<BTreeSet<_>>()+ .into_iter()+ .collect();+ // Keys are stored before anything is sealed with them, as in v3.+ let keys = Arc::new(writer.keys.keys_for_flush(&sessions, &day).await?);+ counter!("ai_research_v4_recorder_dropped_sessions_total").increment(keys.dropped.len() as u64);++ let object_key = object_key(&day, now);+ let seals: Vec<_> = blocks+ .iter()+ .enumerate()+ .filter_map(|(index, block)| {+ let id = SessionId {+ team_id: block.session.team_id,+ session_id: block.session.session_id.clone(),+ };+ let key = keys.sessions.get(&id)?.clone();+ let team_key = *keys.team_months.get(&block.session.team_id)?;+ let (block, month) = (block.clone(), month.to_owned());+ Some(seal_blocking(&writer.seal_permits, move || {+ seal_block(&block, &key, &team_key, &month).map(|sealed| (index, sealed))+ }))+ })+ .collect();+ let mut done = Vec::with_capacity(seals.len());+ let results: Vec<_> = stream::iter(seals)+ .buffer_unordered(concurrency.max(1))+ .collect()+ .await;+ for result in results {+ match result {+ Ok(sealed) => done.push(sealed),+ // A panic in one block loses that block only. Its messages are committed with the rest of the flush.+ Err(WriteError::Panicked(error)) => {+ tracing::error!(%error, "sealing a block panicked: the block is dropped");+ counter!("ai_research_v4_recorder_dropped_blocks_total", "outcome" => "panicked")+ .increment(1);+ }+ Err(error) => return Err(error.into()),+ }+ }+ // Fail closed: a block that still holds a declined URL, a credential or a private host, is not stored.+ done.retain(|(_, block)| {+ let Some(outcome) = block.counts.dropped else {+ return true;+ };+ counter!("ai_research_v4_recorder_dropped_blocks_total", "outcome" => outcome).increment(1);+ false+ });+ let (indices, mut sealed): (Vec<usize>, Vec<_>) = done.into_iter().unzip();++ // The writer's columns go into a copy, so the shared blocks stay as the inspector left them.+ let mut stats: Vec<BlockStats> = blocks.iter().map(|b| b.stats.clone()).collect();+ for (&index, block) in indices.iter().zip(&mut sealed) {+ stats[index].images_extracted += block.counts.extracted;+ stats[index].image_bytes += block.counts.bytes;+ stats[index].images_inline += block.counts.inline;+ stats[index].written_jsonl_bytes += block.counts.jsonl_bytes;+ stats[index].max_written_event_bytes = block.counts.max_line_bytes;+ stats[index].url_fingerprints = std::mem::take(&mut block.url_fingerprints);+ }+ // Before the index is written: building the jobs counts each block's image URLs into its stats. A dropped block's+ // images are not stored, so the fetcher does not fetch them.+ let mut kept = vec![false; blocks.len()];+ for &index in &indices {+ kept[index] = true;+ }+ let jobs = fetch_jobs(+ blocks+ .iter()+ .map(|b| &**b)+ .zip(stats.iter_mut())+ .zip(kept)+ .filter_map(|(pair, kept)| kept.then_some(pair)),+ &keys,+ month,+ now.timestamp_millis(),+ &|reference| {+ writer+ .sent_urls+ .contains_key(&(day.clone(), reference.to_owned()))+ },+ );++ let session_frames = session_keyed_frames(&sealed);+ let rows = write_images(+ new_images(&mut sealed),+ &keys,+ &day,+ month,+ &writer.images_bucket,+ &writer.rrweb_bucket,+ &writer.store,+ now,+ concurrency,+ &writer.seal_permits,+ )+ .await?;+ counter!("ai_research_v4_recorder_stored_images_total").increment(rows.len() as u64);+ // What sealing images under session keys would store, to compare with the frames above.+ counter!("ai_research_v4_recorder_session_keyed_image_frames_total").increment(session_frames);++ let written = write_blocks(+ &sealed,+ &object_key,+ &writer.rrweb_bucket,+ writer.store.as_ref(),+ )+ .await?;+ drop(sealed);+ counter!("ai_research_v4_recorder_written_blocks_total").increment(written.len() as u64);+ let entries: Vec<_> = indices+ .iter()+ .zip(&written)+ .map(|(&index, written)| (&*blocks[index], &stats[index], written))+ .collect();+ write_block_index(+ &entries,+ &keys,+ &writer.rrweb_bucket,+ writer.store.as_ref(),+ now,+ )+ .await?;+ produce_jobs(writer, jobs, &day, now.timestamp_millis()).await;+ Ok(())+}++/// Sends every fetch job of the flush, then waits for all deliveries before the flush commits, as the v3 mirror does.+/// A send that fails does not fail the flush, because its blocks are stored by then. The references it did not send+/// stay unmarked, so a later flush that sees them sends them again.+async fn produce_jobs(+ writer: &Writer,+ jobs: BTreeMap<String, Vec<serde_json::Value>>,+ day: &str,+ now_ms: i64,+) {+ let captured_at = now_ms.to_string();+ let deliveries: Vec<_> = records(jobs)+ .into_iter()+ .map(|(domain, payload, references)| {+ let captured_at = &captured_at;+ async move {+ let headers = OwnedHeaders::new().insert(Header {+ key: "capture-timestamp-ms",+ value: Some(captured_at),+ });+ let record = FutureRecord::to(&writer.image_fetch_topic)+ .key(&domain)+ .payload(&payload)+ .headers(headers);+ match writer+ .producer+ .send(record, Timeout::After(ENQUEUE_TIMEOUT))+ .await+ {+ Ok(_) => Ok(references),+ Err((error, _)) => Err((references.len(), error)),+ }+ }+ })+ .collect();+ let delivered: Vec<_> = stream::iter(deliveries)+ .buffer_unordered(SENDS_IN_FLIGHT)+ .collect()+ .await;+ let mut sent = Vec::new();+ let mut unsent = 0;+ for delivery in delivered {+ match delivery {+ Ok(references) => sent.extend(references),+ Err((references, error)) => {+ if unsent == 0 {+ tracing::warn!(%error, "sending image fetch jobs failed");+ }+ unsent += references;+ }+ }+ }+ counter!("ai_research_v4_recorder_image_fetch_jobs_total").increment(sent.len() as u64);+ counter!("ai_research_v4_recorder_image_fetch_jobs_unsent_total").increment(unsent as u64);+ for reference in sent {+ writer.sent_urls.insert((day.to_owned(), reference), ());+ }+ metrics::gauge!("ai_research_v4_recorder_sent_urls").set(writer.sent_urls.entry_count() as f64);+}diff --git a/rust/ai-research/replay-recorder-v4/src/lib.rs b/rust/ai-research/replay-recorder-v4/src/lib.rsindex 1927a89e129c..ce0d17aa5ad6 100644--- a/rust/ai-research/replay-recorder-v4/src/lib.rs+++ b/rust/ai-research/replay-recorder-v4/src/lib.rs@@ -4,6 +4,7 @@pub mod batch;pub mod config;pub mod consumer;+pub mod flush;pub mod index;pub mod limits;pub mod process;diff --git a/rust/ai-research/replay-recorder-v4/src/main.rs b/rust/ai-research/replay-recorder-v4/src/main.rsindex 0f574ba168ef..b7d12d04be6d 100644--- a/rust/ai-research/replay-recorder-v4/src/main.rs+++ b/rust/ai-research/replay-recorder-v4/src/main.rs@@ -17,7 +17,8 @@ use replay_core_v4::keys::batch::KeyStore;use replay_core_v4::keys::kms::{AwsKms, CachedKms, KmsLimits};use replay_core_v4::keys::table::{AwsKeyTable, CachedKeyTable};use replay_recorder_v4::config::Config;-use replay_recorder_v4::consumer::{run, RecorderConsumer, Settings, Sources, Writer};+use replay_recorder_v4::consumer::{run, RecorderConsumer, Settings, Sources};+use replay_recorder_v4::flush::Writer;use replay_recorder_v4::limits::{Restrictions, SessionLimiter};use replay_recorder_v4::store::S3Store;use replay_recorder_v4::teams::TeamCache;@@ -152,6 +153,11 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {sample_percentage: config.sample_percentage,flush_interval: Duration::from_millis(config.flush_interval_ms),flush_max_bytes: config.flush_max_bytes,+ flush_concurrency: if config.flush_concurrency == 0 {+ std::thread::available_parallelism().map_or(4, |n| n.get())+ } else {+ config.flush_concurrency+ },session_idle: Duration::from_secs(config.session_idle_secs),redirecting_topic: (config.ingestion_overflow_mode == "redirect").then(|| config.kafka_topic.clone()),@@ -195,6 +201,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {.await?,image_fetch_topic: config.image_fetch_topic.clone(),sent_urls: replay_recorder_v4::urls::sent_urls(500_000),+ seal_permits: Arc::new(tokio::sync::Semaphore::new(settings.flush_concurrency)),})} else {tracing::info!("writes are off: the recorder stores nothing");diff --git a/rust/ai-research/replay-recorder-v4/src/urls.rs b/rust/ai-research/replay-recorder-v4/src/urls.rsindex 7f7c471c6212..2396007f4458 100644--- a/rust/ai-research/replay-recorder-v4/src/urls.rs+++ b/rust/ai-research/replay-recorder-v4/src/urls.rs@@ -12,6 +12,8 @@ use replay_core_v4::url_policy::{};use serde_json::{json, Value};+use replay_core_v4::inspect::stats::BlockStats;+use crate::batch::Block;/// The fetcher's limits on one Kafka record: its job count, and the byte length of its value without key or headers.@@ -21,9 +23,9 @@ const MAX_RECORD_BYTES: usize = 512 * 1024;const REMAINING_HOPS: u32 = 10;/// Builds one job per new remote image URL, grouped by registrable domain, the topic's key. Counts each block's URLs-/// in its stats. `already_sent` names references this process sent today.+/// in the stats paired with it. `already_sent` names references this process sent today.pub fn fetch_jobs<'b>(- blocks: impl IntoIterator<Item = &'b mut Block>,+ blocks: impl IntoIterator<Item = (&'b Block, &'b mut BlockStats)>,keys: &FlushKeys,key_month: &str,now_ms: i64,@@ -31,7 +33,7 @@ pub fn fetch_jobs<'b>() -> BTreeMap<String, Vec<Value>> {let mut jobs: BTreeMap<String, Vec<Value>> = BTreeMap::new();let mut seen = HashSet::new();- for block in blocks {+ for (block, stats) in blocks {let id = SessionId {team_id: block.session.team_id,session_id: block.session.session_id.clone(),@@ -49,7 +51,7 @@ pub fn fetch_jobs<'b>(else {continue;};- block.stats.image_urls += 1;+ stats.image_urls += 1;if already_sent(&reference) || !seen.insert(reference.clone()) {continue;}@@ -157,7 +159,6 @@ pub fn sent_urls(capacity: u64) -> moka::sync::Cache<(String, String), ()> {#[cfg(test)]mod tests {- use replay_core_v4::inspect::stats::BlockStats;use replay_core_v4::keys::batch::SessionKey as Key;use super::*;@@ -186,7 +187,7 @@ mod tests {"http://localhost/private.png","https://bat.bing.com/action/0",];- let mut blocks = vec![Block {+ let block = Block {session: SessionKey {team_id: 7,session_id: "s".to_owned(),@@ -197,7 +198,8 @@ mod tests {image_urls: urls.iter().map(|u| (*u).to_owned()).collect(),remote_urls: Vec::new(),url_limit: None,- }];+ };+ let mut stats = BlockStats::default();let mut keys = FlushKeys::default();keys.sessions.insert(SessionId {@@ -219,9 +221,13 @@ mod tests {);let sent = image_url_ref(7, "2026-10", &sent_hash);- let jobs = fetch_jobs(&mut blocks, &keys, "2026-10", 1_000, &|reference| {- reference == sent- });+ let jobs = fetch_jobs(+ [(&block, &mut stats)],+ &keys,+ "2026-10",+ 1_000,+ &|reference| reference == sent,+ );let domains: Vec<_> = jobs.keys().cloned().collect();assert_eq!(domains, ["example.com"]);let fetched: Vec<_> = jobs["example.com"]@@ -245,7 +251,7 @@ mod tests {(Some("s"), Some(10)));// Every public URL the block referenced counts, sent before or not.- assert_eq!(blocks[0].stats.image_urls, 4);+ assert_eq!(stats.image_urls, 4);}#[test]diff --git a/rust/ai-research/replay-recorder-v4/src/write.rs b/rust/ai-research/replay-recorder-v4/src/write.rsindex 3fe258cd0d0b..b981bddd63a9 100644--- a/rust/ai-research/replay-recorder-v4/src/write.rs+++ b/rust/ai-research/replay-recorder-v4/src/write.rs@@ -1,17 +1,19 @@-//! Writes a flush: image packs and their index first, then one object of sealed session blocks.+//! Writes a flush: image packs and their index first, then one object of sealed session blocks, then their index.//!//! The order matters for a crash between writes. A pack without its blocks is unreferenced, which is harmless. A block//! whose images were never stored would hold references that resolve to nothing.use std::collections::{BTreeMap, HashSet};-use std::io::Write;+use std::io::{self, Write};+use std::sync::Arc;use chrono::{DateTime, Utc};-use metrics::counter;+use futures::stream::{self, StreamExt, TryStreamExt};use rand::Rng;use replay_core_v4::bytescan::write_json_string;use replay_core_v4::envelope::{seal_frame, seal_json_envelope, Codec, Context, Version};use replay_core_v4::inspect::events::event_page_url;+use replay_core_v4::inspect::stats::BlockStats;use replay_core_v4::inspect::stats::MAX_URL_FINGERPRINTS;use replay_core_v4::keys::batch::{FlushKeys, SessionId, SessionKey as Key};use replay_core_v4::keys::rows::write_day;@@ -20,6 +22,8 @@ use replay_core_v4::scan::cv::extract_event_images;use replay_core_v4::scan::images::{Image, ImageLimits, ImageSink};use replay_core_v4::seal::{image_content_key, url_fingerprint_key};+use tokio::sync::Semaphore;+use crate::batch::{Block, SessionKey};use crate::index::{block_index, image_index, write_parquet, BlockIndexRow, ImageIndexRow, IndexError,@@ -31,6 +35,7 @@ use crate::store::{BlobStore, StoreError};/// and reads them rarely, so it takes the smaller output for more CPU.const BLOCK_BROTLI_QUALITY: u8 = 9;const BLOCK_BROTLI_WINDOW_BITS: u8 = 22;+const BLOCK_WRITE_BUFFER: usize = 256 * 1024;#[derive(Debug, thiserror::Error)]pub enum WriteError {@@ -40,9 +45,11 @@ pub enum WriteError {Index(#[from] IndexError),#[error("sealing failed: {0}")]Seal(String),+ #[error("sealing panicked: {0}")]+ Panicked(String),}-/// Where one session's block landed, for the block index.+/// Where one block landed, for the block index.#[derive(Debug, Clone, PartialEq, Eq)]pub struct Written {pub session: SessionKey,@@ -53,31 +60,28 @@ pub struct Written {pub byte_end: u64,}-/// A session's block, ready to seal: its key, and its lines with the images moved out.-pub struct PreparedBlock {- pub session: SessionKey,- pub key: Key,- pub lines: Vec<u8>,-}--fn block_encoder() -> Result<brotlic::BrotliEncoder, WriteError> {- let seal_error = |e: brotlic::SetParameterError| WriteError::Seal(e.to_string());- brotlic::BrotliEncoderOptions::new()- .quality(brotlic::Quality::new(BLOCK_BROTLI_QUALITY).map_err(seal_error)?)- .window_size(brotlic::WindowSize::new(BLOCK_BROTLI_WINDOW_BITS).map_err(seal_error)?)- .build()- .map_err(seal_error)+/// The columns of a block's stats that the writer fills.+#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]+pub struct WriterCounts {+ pub extracted: u32,+ pub bytes: u64,+ pub inline: u32,+ /// The bytes of the lines the writer gives the encoder, and the longest line.+ pub jsonl_bytes: u64,+ pub max_line_bytes: u64,+ /// Why the block must not be stored, as the outcome of its drop: a URL the policy declines stays in it, or the+ /// inspector did not name all its URLs.+ pub dropped: Option<&'static str>,}-pub struct Prepared {- pub blocks: Vec<PreparedBlock>,- /// The flush's new images by team, each stored once.- pub images: BTreeMap<i64, Vec<Image>>,- /// The frames the packs would hold if each session sealed its images under its own key: one per distinct- /// (session, reference) pair, where `images` holds one per reference.- pub session_frames: u64,- /// Which of the blocks it was given are stored. The fetch jobs come from these only.- pub kept: Vec<bool>,+/// A block sealed for one object, with the images it moved out.+pub struct SealedBlock {+ pub session: SessionKey,+ pub key_month: String,+ pub frame: Vec<u8>,+ pub images: Vec<Image>,+ pub counts: WriterCounts,+ pub url_fingerprints: Vec<String>,}fn unique_suffix(at: DateTime<Utc>) -> String {@@ -92,218 +96,318 @@ pub fn object_key(write_day: &str, at: DateTime<Utc>) -> String {format!("blocks/dt={write_day}/{}", unique_suffix(at))}-/// Moves each kept session's images out and builds its lines in the main lane's format: one `["window_id",event]`-/// line per event. Fills the image and written size columns of each block's stats. Every image goes into this-/// flush's packs, even one an earlier flush stored: a pack expires by its own write time, so it must be no older than-/// the blocks that reference it.-pub fn prepare(blocks: &mut [Block], keys: &FlushKeys, key_month: &str) -> Prepared {- let mut prepared = Prepared {- blocks: Vec::new(),- images: BTreeMap::new(),- session_frames: 0,- kept: vec![false; blocks.len()],- };- // A reference names its team, so it names one frame of the flush.- let mut seen: HashSet<String> = HashSet::new();- let mut seen_in_session: HashSet<(String, String)> = HashSet::new();- for (index, block) in blocks.iter_mut().enumerate() {- let id = SessionId {- team_id: block.session.team_id,- session_id: block.session.session_id.clone(),- };- let (Some(key), Some(team_key)) = (- keys.sessions.get(&id),- keys.team_months.get(&block.session.team_id),- ) else {- continue;- };- // Fail closed: the inspector did not name every URL, so one it missed could be declined.- if let Some(limit) = block.url_limit {- counter!("ai_research_v4_recorder_dropped_blocks_total", "outcome" => limit.label())- .increment(1);- continue;- }- let content_key = image_content_key(team_key);- let mut rewrite = UrlRewrite::new(- &block.remote_urls,+/// Moves a block's images out and writes its lines to `out` in the main lane's format: one `["window_id",event]` line+/// per event. Every image goes into this flush's packs, even one an earlier flush stored: a pack expires by its own+/// write time, so it must be no older than the blocks that reference it.+/// Expanded `cv` events can make a block far larger than its messages, so only one event is held at a time.+pub fn prepare_block(+ block: &Block,+ team_key: &[u8; 32],+ key_month: &str,+ out: &mut impl Write,+) -> io::Result<(Vec<Image>, WriterCounts)> {+ // Fail closed: a URL the inspector did not name could be one the policy declines.+ if let Some(limit) = block.url_limit {+ let dropped = Some(limit.label());+ return Ok((+ Vec::new(),+ WriterCounts {+ dropped,+ ..WriterCounts::default()+ },+ ));+ }+ let content_key = image_content_key(team_key);+ let mut rewrite = UrlRewrite::new(+ &block.remote_urls,+ block.session.team_id,+ key_month,+ team_key,+ );+ let mut images = Vec::new();+ let mut kept = HashSet::new();+ let mut counts = WriterCounts::default();+ for message in &block.messages {+ let mut window = Vec::new();+ write_json_string(+ message.window_id.as_deref().unwrap_or_default(),+ &mut window,+ );+ let mut sink = ImageSink::new(+ &content_key,block.session.team_id,key_month,- team_key,- );- block.stats.url_fingerprints = url_fingerprints(&key.key, &block_urls(block));- let mut block_images = Vec::new();- let mut lines = Vec::with_capacity(- block.stats.byte_size as usize + block.stats.event_count as usize * 16,+ ImageLimits::default(),);- for message in &block.messages {- let mut window = Vec::new();- write_json_string(- message.window_id.as_deref().unwrap_or_default(),- &mut window,- );- let mut sink = ImageSink::new(- &content_key,- block.session.team_id,- key_month,- ImageLimits::default(),+ for event in &message.events {+ let extracted = extract_event_images(&message.inner, event, &mut sink);+ out.write_all(b"[")?;+ out.write_all(&window)?;+ out.write_all(b",")?;+ let canonical = canonical_strings(+ extracted+ .as_deref()+ .unwrap_or(&message.inner[event.span.0..event.span.1]),);- for event in &message.events {- let extracted = extract_event_images(&message.inner, event, &mut sink);- let line_start = lines.len();- lines.push(b'[');- lines.extend_from_slice(&window);- lines.push(b',');- let canonical = canonical_strings(- extracted- .as_deref()- .unwrap_or(&message.inner[event.span.0..event.span.1]),- );- lines.extend_from_slice(&rewrite.apply(&canonical));- lines.extend_from_slice(b"]\n");- let line = (lines.len() - line_start) as u64;- block.stats.written_jsonl_bytes += line;- block.stats.max_written_event_bytes = block.stats.max_written_event_bytes.max(line);- }- block.stats.images_extracted += sink.images.len() as u32;- block.stats.image_bytes += sink+ let event_bytes = rewrite.apply(&canonical);+ out.write_all(&event_bytes)?;+ out.write_all(b"]\n")?;+ let line = (window.len() + event_bytes.len() + 4) as u64;+ counts.jsonl_bytes += line;+ counts.max_line_bytes = counts.max_line_bytes.max(line);+ }+ counts.extracted += sink.images.len() as u32;+ counts.bytes += sink+ .images+ .iter()+ .map(|i| i.bytes.len() as u64)+ .sum::<u64>();+ counts.inline += sink.left_inline;+ images.extend(+ sink.images+ .into_iter()+ .filter(|image| kept.insert(image.reference.clone())),+ );+ }+ // Fail closed: the flush drops a block that still holds a declined URL, a credential or a private host.+ if !rewrite.finish() {+ counts.dropped = Some("declined_unrewritten");+ }+ Ok((images, counts))+}++fn block_encoder() -> Result<brotlic::BrotliEncoder, WriteError> {+ let seal_error = |e: brotlic::SetParameterError| WriteError::Seal(e.to_string());+ brotlic::BrotliEncoderOptions::new()+ .quality(brotlic::Quality::new(BLOCK_BROTLI_QUALITY).map_err(seal_error)?)+ .window_size(brotlic::WindowSize::new(BLOCK_BROTLI_WINDOW_BITS).map_err(seal_error)?)+ .build()+ .map_err(seal_error)+}++/// Builds, compresses and seals one block.+pub fn seal_block(+ block: &Block,+ key: &Key,+ team_key: &[u8; 32],+ key_month: &str,+) -> Result<SealedBlock, WriteError> {+ let seal_error = |e: io::Error| WriteError::Seal(e.to_string());+ // Lines go into the encoder as they are built, so memory follows the compressed block and not its plaintext.+ // A brotli stream does not state its decompressed size, so a reader decompresses it as a stream.+ let encoder = brotlic::CompressorWriter::with_encoder(block_encoder()?, Vec::new());+ // The encoder works on each write it gets, so small writes cost far more CPU than the same bytes in one.+ let mut lines = io::BufWriter::with_capacity(BLOCK_WRITE_BUFFER, encoder);+ let (images, counts) =+ prepare_block(block, team_key, key_month, &mut lines).map_err(seal_error)?;+ let encoder = lines.into_inner().map_err(|e| seal_error(e.into_error()))?;+ let compressed = encoder+ .into_inner()+ .map_err(|e| WriteError::Seal(e.to_string()))?;+ let frame = seal_frame(+ &key.key,+ Version::V4,+ &block_context(&block.session, &key.key_month),+ &compressed,+ )+ .map_err(|e| WriteError::Seal(e.to_string()))?;+ Ok(SealedBlock {+ session: block.session.clone(),+ key_month: key.key_month.clone(),+ frame,+ images,+ counts,+ url_fingerprints: url_fingerprints(&key.key, &block_urls(block)),+ })+}++/// The authenticated data of a block frame.+pub fn block_context(session: &SessionKey, key_month: &str) -> Context {+ Context {+ team_id: session.team_id,+ session_id: Some(session.session_id.clone()),+ key_month: Some(key_month.to_owned()),+ kind: "rrweb".to_owned(),+ codec: Some(Codec::Brotli),+ ..Context::default()+ }+}++/// The frames the flush's packs would hold if each session sealed its images under its own key: one per distinct+/// (session, reference) pair, where [`new_images`] keeps one per reference.+pub fn session_keyed_frames(sealed: &[SealedBlock]) -> u64 {+ let pairs: HashSet<(&str, &str)> = sealed+ .iter()+ .flat_map(|block| {+ let session = block.session.session_id.as_str();+ block.images.iter()- .map(|i| i.bytes.len() as u64)- .sum::<u64>();- block.stats.images_inline += sink.left_inline;- block_images.extend(sink.images);- }- // Fail closed: a declined URL, a credential or a private host, must not reach storage.- if !rewrite.finish() {- counter!("ai_research_v4_recorder_dropped_blocks_total", "outcome" => "declined_unrewritten")- .increment(1);- continue;- }- let images = prepared.images.entry(block.session.team_id).or_default();- for image in block_images {- if seen_in_session.insert((id.session_id.clone(), image.reference.clone())) {- prepared.session_frames += 1;- }+ .map(move |image| (session, image.reference.as_str()))+ })+ .collect();+ pairs.len() as u64+}++/// The flush's new images by team, each once.+pub fn new_images(sealed: &mut [SealedBlock]) -> BTreeMap<i64, Vec<Image>> {+ let mut images: BTreeMap<i64, Vec<Image>> = BTreeMap::new();+ let mut seen: HashSet<String> = HashSet::new();+ for block in sealed {+ for image in std::mem::take(&mut block.images) {if seen.insert(image.reference.clone()) {- images.push(image);+ images.entry(block.session.team_id).or_default().push(image);}}- prepared.kept[index] = true;- prepared.blocks.push(PreparedBlock {- session: block.session.clone(),- key: key.clone(),- lines,+ }+ images+}++fn seal_pack(+ team_id: i64,+ team_images: &[Image],+ team_key: &[u8; 32],+ key_month: &str,+ pack_key: &str,+) -> Result<(Vec<u8>, Vec<ImageIndexRow>), WriteError> {+ let mut pack = Vec::new();+ let mut rows = Vec::with_capacity(team_images.len());+ for image in team_images {+ let context = Context {+ team_id,+ key_month: Some(key_month.to_owned()),+ kind: "image".to_owned(),+ codec: Some(Codec::None),+ reference: Some(image.reference.clone()),+ ..Context::default()+ };+ let frame = seal_frame(team_key, Version::V4, &context, &image.bytes)+ .map_err(|e| WriteError::Seal(e.to_string()))?;+ let byte_start = pack.len() as u64;+ pack.extend_from_slice(&frame);+ rows.push(ImageIndexRow {+ team_id,+ key_month: key_month.to_owned(),+ image_ref: image.reference.clone(),+ pack_s3_key: pack_key.to_owned(),+ byte_start,+ byte_end: pack.len() as u64 - 1,+ mime: image.mime.to_owned(),+ byte_size: image.bytes.len() as u64,});}- prepared.images.retain(|_, images| !images.is_empty());- prepared+ Ok((pack, rows))}-/// Seals each team's images under its team month key into one pack, then writes the image index for the flush.+/// Seals each team's images under its team month key into one pack, stores the packs concurrently, then writes the+/// image index for the flush.#[allow(clippy::too_many_arguments)]pub async fn write_images(- images: &BTreeMap<i64, Vec<Image>>,+ images: BTreeMap<i64, Vec<Image>>,keys: &FlushKeys,write_day: &str,key_month: &str,images_bucket: &str,rrweb_bucket: &str,- store: &dyn BlobStore,+ store: &Arc<dyn BlobStore>,at: DateTime<Utc>,+ concurrency: usize,+ permits: &Arc<Semaphore>,) -> Result<Vec<ImageIndexRow>, WriteError> {- let mut rows = Vec::new();- for (&team_id, team_images) in images {- let Some(team_key) = keys.team_months.get(&team_id) else {- continue;- };- let pack_key = format!(- "images/packs/dt={write_day}/{team_id}/{}",- unique_suffix(at)- );- let mut pack = Vec::new();- for image in team_images {- let context = Context {- team_id,- key_month: Some(key_month.to_owned()),- kind: "image".to_owned(),- codec: Some(Codec::None),- reference: Some(image.reference.clone()),- ..Context::default()- };- let frame = seal_frame(team_key, Version::V4, &context, &image.bytes)- .map_err(|e| WriteError::Seal(e.to_string()))?;- let byte_start = pack.len() as u64;- pack.extend_from_slice(&frame);- rows.push(ImageIndexRow {- team_id,- key_month: key_month.to_owned(),- image_ref: image.reference.clone(),- pack_s3_key: pack_key.clone(),- byte_start,- byte_end: pack.len() as u64 - 1,- mime: image.mime.to_owned(),- byte_size: image.bytes.len() as u64,- });- }- store.put_new(images_bucket, &pack_key, pack).await?;- }+ let packs: Vec<_> = images+ .into_iter()+ .filter_map(|(team_id, team_images)| {+ let team_key = *keys.team_months.get(&team_id)?;+ let pack_key = format!(+ "images/packs/dt={write_day}/{team_id}/{}",+ unique_suffix(at)+ );+ let key_month = key_month.to_owned();+ let store = store.clone();+ Some(async move {+ let key = pack_key.clone();+ let (pack, rows) = seal_blocking(permits, move || {+ seal_pack(team_id, &team_images, &team_key, &key_month, &key)+ })+ .await?;+ store.put_new(images_bucket, &pack_key, pack).await?;+ Ok::<_, WriteError>(rows)+ })+ })+ .collect();+ let rows: Vec<ImageIndexRow> = stream::iter(packs)+ .buffer_unordered(concurrency.max(1))+ .try_collect::<Vec<_>>()+ .await?+ .into_iter()+ .flatten()+ .collect();if !rows.is_empty() {let index_key = format!("index/images/dt={write_day}/{}.parquet", unique_suffix(at));- store- .put_new(- rrweb_bucket,- &index_key,- write_parquet(&image_index(&rows)?)?,- )- .await?;+ let batch = image_index(&rows)?;+ let file = blocking(move || write_parquet(&batch).map_err(WriteError::from)).await?;+ store.put_new(rrweb_bucket, &index_key, file).await?;}Ok(rows)}-/// Seals each prepared block under its session key into one object.+/// Writes the flush's sealed blocks as one object. Returns where each landed, in the same order.pub async fn write_blocks(- blocks: &[PreparedBlock],+ blocks: &[SealedBlock],+ object_key: &str,bucket: &str,store: &dyn BlobStore,- at: DateTime<Utc>,) -> Result<Vec<Written>, WriteError> {- let object_key = object_key(&write_day(at), at);- let mut object = Vec::new();- let mut written = Vec::new();+ let mut object = Vec::with_capacity(blocks.iter().map(|b| b.frame.len()).sum());+ let mut written = Vec::with_capacity(blocks.len());for block in blocks {- let mut encoder = brotlic::CompressorWriter::with_encoder(block_encoder()?, Vec::new());- encoder- .write_all(&block.lines)- .map_err(|e| WriteError::Seal(e.to_string()))?;- let compressed = encoder- .into_inner()- .map_err(|e| WriteError::Seal(e.to_string()))?;- let context = Context {- team_id: block.session.team_id,- session_id: Some(block.session.session_id.clone()),- key_month: Some(block.key.key_month.clone()),- kind: "rrweb".to_owned(),- codec: Some(Codec::Brotli),- ..Context::default()- };- let frame = seal_frame(&block.key.key, Version::V4, &context, &compressed)- .map_err(|e| WriteError::Seal(e.to_string()))?;let byte_start = object.len() as u64;- object.extend_from_slice(&frame);+ object.extend_from_slice(&block.frame);written.push(Written {session: block.session.clone(),- key_month: block.key.key_month.clone(),- object_key: object_key.clone(),+ key_month: block.key_month.clone(),+ object_key: object_key.to_owned(),byte_start,byte_end: object.len() as u64 - 1,});}if !written.is_empty() {- store.put_new(bucket, &object_key, object).await?;+ store.put_new(bucket, object_key, object).await?;}Ok(written)}+/// Runs CPU work off the async runtime.+pub async fn blocking<T: Send + 'static>(+ work: impl FnOnce() -> Result<T, WriteError> + Send + 'static,+) -> Result<T, WriteError> {+ tokio::task::spawn_blocking(work).await.map_err(|e| {+ if e.is_panic() {+ WriteError::Panicked(e.to_string())+ } else {+ WriteError::Seal(e.to_string())+ }+ })?+}++/// Runs a seal off the async runtime under one of `permits`. The seal holds its permit until it ends, so the seals of+/// an attempt that timed out still count against the limit.+pub async fn seal_blocking<T: Send + 'static>(+ permits: &Arc<Semaphore>,+ work: impl FnOnce() -> Result<T, WriteError> + Send + 'static,+) -> Result<T, WriteError> {+ let permit = permits+ .clone()+ .acquire_owned()+ .await+ .map_err(|e| WriteError::Seal(e.to_string()))?;+ blocking(move || {+ let _permit = permit;+ work()+ })+ .await+}+/// The pages a block visited, from its Meta events, in order and without repeats in a row./// An SPA navigation, as a `$url_changed` custom event, counts as a visit too.pub fn block_urls(block: &Block) -> Vec<String> {@@ -338,21 +442,16 @@ pub fn url_fingerprints(session_key: &[u8; 32], urls: &[String]) -> Vec<String>}/// Writes one block index file for the flush, after its blocks, so no row points at a block that is not stored.+/// Each entry is a block, its stats as this attempt filled them, and where it landed.pub async fn write_block_index(- blocks: &[Block],- written: &[Written],+ entries: &[(&Block, &BlockStats, &Written)],keys: &FlushKeys,rrweb_bucket: &str,store: &dyn BlobStore,at: DateTime<Utc>,) -> Result<usize, WriteError> {- let by_session: BTreeMap<&SessionKey, &Block> =- blocks.iter().map(|b| (&b.session, b)).collect();- let mut rows = Vec::with_capacity(written.len());- for written in written {- let Some(block) = by_session.get(&written.session) else {- continue;- };+ let mut rows = Vec::with_capacity(entries.len());+ for &(block, stats, written) in entries {let id = SessionId {team_id: written.session.team_id,session_id: written.session.session_id.clone(),@@ -382,7 +481,7 @@ pub async fn write_block_index(let first = block.messages.first();rows.push(BlockIndexRow {written,- stats: &block.stats,+ stats,moments: &block.moments,snapshot_source: first.and_then(|m| m.snapshot_source.as_deref()),snapshot_library: first.and_then(|m| m.library.as_deref()),@@ -397,25 +496,22 @@ pub async fn write_block_index(write_day(at),unique_suffix(at));- store- .put_new(- rrweb_bucket,- &index_key,- write_parquet(&block_index(&rows)?)?,- )- .await?;- Ok(rows.len())+ let count = rows.len();+ let batch = block_index(&rows)?;+ let file = blocking(move || write_parquet(&batch).map_err(WriteError::from)).await?;+ store.put_new(rrweb_bucket, &index_key, file).await?;+ Ok(count)}#[cfg(test)]mod tests {use std::collections::HashMap;- use std::io::{self, Read};+ use std::io::Read;use std::sync::Mutex;use aes_gcm::aead::{Aead, KeyInit, Payload};use aes_gcm::{Aes256Gcm, Nonce};- use arrow_array::{Array, Int64Array, ListArray, StringArray, StructArray};+ use arrow_array::{Array, Int64Array, ListArray, RecordBatch, StringArray, StructArray};use arrow_schema::DataType;use async_trait::async_trait;use base64::engine::general_purpose::STANDARD;@@ -425,7 +521,6 @@ mod tests {use replay_core_v4::envelope::open_frame;use replay_core_v4::inspect::dom::MAX_REMOTE_URL_BYTES;use replay_core_v4::inspect::session::Moment;- use replay_core_v4::inspect::stats::BlockStats;use replay_core_v4::scan::message::read_message;use serde_json::{json, Value};@@ -440,6 +535,29 @@ mod tests {fn get(&self, bucket: &str, key: &str) -> Vec<u8> {self.objects.lock().unwrap()[&(bucket.to_owned(), key.to_owned())].clone()}++ fn keys(&self, prefix: &str) -> Vec<String> {+ let mut keys: Vec<String> = self+ .objects+ .lock()+ .unwrap()+ .keys()+ .filter(|(_, key)| key.starts_with(prefix))+ .map(|(_, key)| key.clone())+ .collect();+ keys.sort();+ keys+ }++ fn parquet(&self, key: &str) -> RecordBatch {+ ParquetRecordBatchReaderBuilder::try_new(Bytes::from(self.get("rrweb-v4", key)))+ .unwrap()+ .build()+ .unwrap()+ .next()+ .unwrap()+ .unwrap()+ }}fn decompress(compressed: &[u8]) -> io::Result<Vec<u8>> {@@ -489,13 +607,6 @@ mod tests {}}- /// The length of each line, its line break included.- fn lines_of(lines: &[u8]) -> impl Iterator<Item = u64> + '_ {- lines- .split_inclusive(|&b| b == b'\n')- .map(|line| line.len() as u64)- }-fn uri(bytes: &[u8]) -> String {format!("data:image/png;base64,{}", STANDARD.encode(bytes))}@@ -514,25 +625,12 @@ mod tests {inspected.remote_urls.extend(extra.iter().map(|url| (*url).to_owned()));- let mut keys = FlushKeys::default();- keys.sessions.insert(- SessionId {- team_id: 7,- session_id: "kept".to_owned(),- },- Key {- key: [5; 32],- key_month: "2026-10".to_owned(),- },- );- keys.team_months.insert(7, [6; 32]);- let mut blocks = vec![inspected];- let prepared = prepare(&mut blocks, &keys, "2026-10");- prepared- .blocks- .first()- .map(|b| String::from_utf8(b.lines.clone()).unwrap())- .unwrap_or_default()+ let mut lines = Vec::new();+ let (_, counts) = prepare_block(&inspected, &[6; 32], "2026-10", &mut lines).unwrap();+ if counts.dropped.is_some() {+ return String::new();+ }+ String::from_utf8(lines).unwrap()}#[test]@@ -606,31 +704,18 @@ mod tests {"https://cdn.example.com/a.png?token=secret",);let events = json!([{"type": 2, "timestamp": 1, "data": {"node": {"attributes": {"src": fetched, "srcset": format!("{declined} 2x")}}}}]);- let mut blocks = vec![block("kept", events.clone()), block("dropped", events)];- for block in &mut blocks {- block.image_urls = vec![fetched.to_owned()];- block.remote_urls = vec![fetched.to_owned(), declined.to_owned()];- }- blocks[1].url_limit = Some(replay_core_v4::inspect::dom::UrlLimit::UrlLength);+ let mut block = block("kept", events);+ block.image_urls = vec![fetched.to_owned()];+ block.remote_urls = vec![fetched.to_owned(), declined.to_owned()];let mut keys = FlushKeys::default();- for session in ["kept", "dropped"] {- keys.sessions.insert(- SessionId {- team_id: 7,- session_id: session.to_owned(),- },- Key {- key: [5; 32],- key_month: "2026-10".to_owned(),- },- );- }+ let (id, key) = session_key("kept");+ keys.sessions.insert(id, key);keys.team_months.insert(7, [6; 32]);- let prepared = prepare(&mut blocks, &keys, "2026-10");- // Only a stored block sends fetch jobs.- assert_eq!(prepared.kept, [true, false]);- let jobs = crate::urls::fetch_jobs(&mut blocks[..1], &keys, "2026-10", 1, &|_| false);- let lines = String::from_utf8(prepared.blocks[0].lines.clone()).unwrap();+ let mut stats = BlockStats::default();+ let jobs = crate::urls::fetch_jobs([(&block, &mut stats)], &keys, "2026-10", 1, &|_| false);+ let mut lines = Vec::new();+ prepare_block(&block, &[6; 32], "2026-10", &mut lines).unwrap();+ let lines = String::from_utf8(lines).unwrap();let reference = jobs["example.com"][0]["originalRef"].as_str().unwrap();assert!(lines.contains(&format!(r#""src":"{reference}""#)),@@ -643,6 +728,56 @@ mod tests {assert!(!lines.contains("cdn.example.com"));}+ fn session_key(session: &str) -> (SessionId, Key) {+ (+ SessionId {+ team_id: 7,+ session_id: session.to_owned(),+ },+ Key {+ key: [5; 32],+ key_month: "2026-09".to_owned(),+ },+ )+ }++ fn at() -> DateTime<Utc> {+ DateTime::parse_from_rfc3339("2026-10-15T12:00:00Z")+ .unwrap()+ .with_timezone(&Utc)+ }++ #[test]+ fn keeps_one_copy_of_an_image_that_several_messages_repeat() {+ let events = json!([{"type": 2, "timestamp": 1, "data": {"node": {"attributes": {"src": uri(b"logo")}}}}]);+ let mut repeated = block("s", events.clone());+ repeated.messages.extend(block("s", events).messages);+ let (images, counts) =+ prepare_block(&repeated, &[6; 32], "2026-10", &mut io::sink()).unwrap();+ assert_eq!((images.len(), counts.extracted), (1, 2));+ }++ #[tokio::test]+ async fn a_seal_of_an_abandoned_attempt_keeps_its_permit() {+ let permits = Arc::new(Semaphore::new(1));+ let (release, wait) = std::sync::mpsc::channel::<()>();+ let abandoned = seal_blocking(&permits, move || {+ wait.recv().ok();+ Ok(())+ });+ assert!(+ tokio::time::timeout(std::time::Duration::from_millis(50), abandoned)+ .await+ .is_err()+ );+ assert_eq!(permits.available_permits(), 0);+ release.send(()).unwrap();+ seal_blocking(&permits, || Ok(())).await.unwrap();+ // The flush drops a block whose seal panicked and keeps the others, so a panic is told apart.+ let panicked = seal_blocking(&permits, || -> Result<(), WriteError> { panic!("seal") });+ assert!(matches!(panicked.await, Err(WriteError::Panicked(_))));+ }+#[tokio::test]async fn stores_images_once_and_blocks_that_point_at_them() {let events = json!([@@ -652,108 +787,94 @@ mod tests {{"type": 5, "timestamp": 3, "data": {"tag": "$url_changed", "payload": {"href": "https://app.example.com/b"}}},{"type": 4, "timestamp": 4, "data": {"href": "https://app.example.com/a"}},]);- let mut blocks = vec![block("kept", events.clone()), block("tombstoned", events)];+ let mut blocks = [block("kept", events.clone()), block("again", events)];let mut keys = FlushKeys::default();- keys.sessions.insert(- SessionId {- team_id: 7,- session_id: "kept".to_owned(),- },- Key {- key: [5; 32],- key_month: "2026-09".to_owned(),- },- );+ for session in ["kept", "again"] {+ let (id, key) = session_key(session);+ keys.sessions.insert(id, key);+ }keys.team_months.insert(7, [6; 32]);let old_ref = replay_core_v4::refs::image_ref(7,"2026-10",&replay_core_v4::refs::reference_hash(&image_content_key(&[6; 32]), b"old"),);- let prepared = prepare(&mut blocks, &keys, "2026-10");-- // The same image in two sessions of the team is one frame and one index row.- let same = json!([{"type": 2, "timestamp": 1, "data": {"node": {"attributes": {"src": uri(b"shared")}}}}]);- let mut two = vec![block("kept", same.clone()), block("other", same)];- let mut both = FlushKeys::default();- both.team_months.insert(7, [6; 32]);- for (session, key) in [("kept", [5; 32]), ("other", [4; 32])] {- both.sessions.insert(- SessionId {- team_id: 7,- session_id: session.to_owned(),- },- Key {- key,- key_month: "2026-10".to_owned(),- },- );- }- // A key per session would need a frame in each.- let shared = prepare(&mut two, &both, "2026-10");- assert_eq!((shared.images[&7].len(), shared.session_frames), (1, 2));+ let object = object_key("2026-10-15", at());+ let mut sealed: Vec<SealedBlock> = blocks+ .iter()+ .map(|b| {+ seal_block(+ b,+ &keys.sessions[&session_key(&b.session.session_id).0],+ &[6; 32],+ "2026-10",+ )+ .unwrap()+ })+ .collect();+ assert_eq!(+ sealed[0].counts,+ WriterCounts {+ extracted: 2,+ bytes: 6,+ inline: 0,+ ..sealed[0].counts+ }+ );+ // The page the block sees again keeps its first place, under the session's own fingerprint key.+ let fingerprint_key = url_fingerprint_key(&[5; 32]);+ let fingerprint = |path: &str| {+ url_fingerprint(&fingerprint_key, &format!("https://app.example.com{path}"))+ };+ assert_eq!(+ sealed[0].url_fingerprints,+ [fingerprint("/a"), fingerprint("/b")]+ );- let store = MemoryStore::default();- let at = DateTime::parse_from_rfc3339("2026-10-15T12:00:00Z")- .unwrap()- .with_timezone(&Utc);+ let session_frames = session_keyed_frames(&sealed);+ let memory = Arc::new(MemoryStore::default());+ let store: Arc<dyn BlobStore> = memory.clone();let rows = write_images(- &prepared.images,+ new_images(&mut sealed),&keys,"2026-10-15","2026-10","images-v4","rrweb-v4",&store,- at,+ at(),+ 4,+ &Arc::new(Semaphore::new(4)),).await.unwrap();- let written = write_blocks(&prepared.blocks, "rrweb-v4", &store, at)+ let written = write_blocks(&sealed, &object, "rrweb-v4", store.as_ref()).await.unwrap();- // The pack holds both images, also the one an earlier flush stored.+ // Both blocks moved the same two images out, and the pack holds each once.assert_eq!(rows.len(), 2);+ // A key per session would need each session's copy.+ assert_eq!(session_frames, 4);assert!(rows.iter().any(|row| row.image_ref == old_ref));- assert_eq!(- (- blocks[0].stats.images_extracted,- blocks[0].stats.images_inline- ),- (2, 0)- );- let longest = lines_of(&prepared.blocks[0].lines).max().unwrap();- assert_eq!(- (- blocks[0].stats.written_jsonl_bytes,- blocks[0].stats.max_written_event_bytes- ),- (prepared.blocks[0].lines.len() as u64, longest)- );- assert_eq!(- written- .iter()- .map(|w| w.session.session_id.as_str())- .collect::<Vec<_>>(),- vec!["kept"]- );+ assert_eq!(written.len(), 2);let block = &written[0];- let frame = &store.get("rrweb-v4", &block.object_key)+ let frame = &memory.get("rrweb-v4", &block.object_key)[block.byte_start as usize..=block.byte_end as usize];- let context = Context {- team_id: 7,- session_id: Some("kept".to_owned()),- key_month: Some("2026-09".to_owned()),- kind: "rrweb".to_owned(),- codec: Some(Codec::Brotli),- ..Context::default()- };+ let context = block_context(&blocks[0].session, "2026-09");let lines = String::from_utf8(decompress(&open_frame(&[5; 32], Version::V4, &context, frame).unwrap()).unwrap(),).unwrap();+ let longest = lines.split_inclusive('\n').map(str::len).max().unwrap();+ assert_eq!(+ (+ sealed[0].counts.jsonl_bytes,+ sealed[0].counts.max_line_bytes+ ),+ (lines.len() as u64, longest as u64)+ );let first: Value = serde_json::from_str(lines.lines().nth(1).unwrap()).unwrap();let reference = first[1]["data"]["node"]["attributes"]["src"].as_str()@@ -763,7 +884,7 @@ mod tests {// The reference resolves through the index to the image's bytes in the pack.let row = &rows[0];- let pack = store.get("images-v4", &row.pack_s3_key);+ let pack = memory.get("images-v4", &row.pack_s3_key);let image_context = Context {team_id: 7,key_month: Some("2026-10".to_owned()),@@ -772,56 +893,40 @@ mod tests {reference: Some(row.image_ref.clone()),..Context::default()};- let sealed = &pack[row.byte_start as usize..=row.byte_end as usize];+ let sealed_image = &pack[row.byte_start as usize..=row.byte_end as usize];assert_eq!(- open_frame(&[6; 32], Version::V4, &image_context, sealed).unwrap(),+ open_frame(&[6; 32], Version::V4, &image_context, sealed_image).unwrap(),b"new");- // The team month key seals images, as the fetcher's, so the session key does not open them.- assert!(open_frame(&[5; 32], Version::V4, &image_context, sealed).is_err());- assert_eq!(- store- .objects- .lock()- .unwrap()- .keys()- .filter(|(_, key)| key.starts_with("index/images/dt=2026-10-15/"))- .count(),- 1- );+ // The team month key seals images, as the fetcher's, so a session key does not open them.+ assert!(open_frame(&[5; 32], Version::V4, &image_context, sealed_image).is_err());+ assert_eq!(memory.keys("index/images/dt=2026-10-15/").len(), 1);// The block index row points at the block, keeps the moments, and seals the visited pages.- blocks[0].stats.window_ids = vec!["w1".to_owned()];blocks[0].moments = vec![Moment {kind: "full_snapshot",window_id: "w1".into(),ts_ms: 1,event_index: 1,}];+ let stats = BlockStats {+ url_fingerprints: sealed[0].url_fingerprints.clone(),+ window_ids: vec!["w1".to_owned()],+ ..BlockStats::default()+ };assert_eq!(- write_block_index(&blocks, &written, &keys, "rrweb-v4", &store, at)- .await- .unwrap(),+ write_block_index(+ &[(&blocks[0], &stats, &written[0])],+ &keys,+ "rrweb-v4",+ store.as_ref(),+ at()+ )+ .await+ .unwrap(),1);- let index_key = store- .objects- .lock()- .unwrap()- .keys()- .find(|(_, key)| key.starts_with("index/blocks/"))- .unwrap()- .1- .clone();- let batch = ParquetRecordBatchReaderBuilder::try_new(Bytes::from(- store.get("rrweb-v4", &index_key),- ))- .unwrap()- .build()- .unwrap()- .next()- .unwrap()- .unwrap();+ let batch = memory.parquet(&memory.keys("index/blocks/")[0]);let column = |name: &str| {batch.column_by_name(name)@@ -867,15 +972,7 @@ mod tests {let values = values.as_any().downcast_ref::<StringArray>().unwrap();values.iter().map(|v| v.unwrap().to_owned()).collect()};- // The page the block sees again keeps its first place, under the session's own fingerprint key.- let fingerprint_key = replay_core_v4::seal::url_fingerprint_key(&[5; 32]);- let fingerprint = |path: &str| {- url_fingerprint(&fingerprint_key, &format!("https://app.example.com{path}"))- };- assert_eq!(- strings("url_fingerprints"),- [fingerprint("/a"), fingerprint("/b")]- );+ assert_eq!(strings("url_fingerprints"), stats.url_fingerprints);assert_eq!(strings("window_ids"), ["w1"]);let moments = column("moments");let moments = moments@@ -898,7 +995,7 @@ mod tests {"full_snapshot");let envelope: Value = serde_json::from_str(&text("urls")).unwrap();- let sealed = STANDARD+ let sealed_urls = STANDARD.decode(envelope["ciphertext"].as_str().unwrap()).unwrap();let nonce = STANDARD@@ -918,7 +1015,7 @@ mod tests {.decrypt(Nonce::from_slice(&nonce),Payload {- msg: &sealed,+ msg: &sealed_urls,aad: aad.as_bytes(),},)