Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(replay-recorder-v4): record kafka positions so readers can drop retried blocks (#118421)

Nicholas Waltz5f80da7fae58
diff --git a/rust/ai-research/replay-recorder-v4/src/batch.rs b/rust/ai-research/replay-recorder-v4/src/batch.rs
index c081e87a1a0d..ec0b29047134 100644
--- a/rust/ai-research/replay-recorder-v4/src/batch.rs
+++ b/rust/ai-research/replay-recorder-v4/src/batch.rs
@@ -1,6 +1,8 @@
//! Messages buffered between flushes, and the per-session inspectors that outlive them.
use std::collections::{BTreeMap, HashMap};
+use std::sync::atomic::{AtomicUsize, Ordering};
+use std::sync::Arc;
use std::time::{Duration, Instant};
use futures::stream::{self, StreamExt};
@@ -9,9 +11,10 @@ use replay_core_v4::compression::{GZIP_MAGIC, MAX_DECOMPRESSED_BYTES};
use replay_core_v4::inspect::dom::UrlLimit;
use replay_core_v4::inspect::session::{BlockInspection, Moment, SessionInspector};
use replay_core_v4::inspect::stats::BlockStats;
-use replay_core_v4::scan::message::Message;
+use replay_core_v4::scan::cv::{Expansion, MESSAGE_DECODE_BUDGET};
+use replay_core_v4::scan::message::{Event, Message};
-use crate::process::{check_read, read};
+use crate::process::{check_read, read_within};
#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub struct SessionKey {
@@ -66,9 +69,23 @@ pub struct Span {
pub last: i64,
}
+/// Where a block's messages came from. A failed flush commits nothing, so every later attempt for the partition
+/// starts at the same `start_offset`. Readers use this to remove the rows of earlier attempts.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct Origin {
+ pub tp: Tp,
+ /// The first offset the flush handled on the partition.
+ pub start_offset: i64,
+ /// The block's last message.
+ pub end_offset: i64,
+ /// The last offset the flush handled on the partition, a skipped message included.
+ pub flush_end_offset: i64,
+}
+
/// What a flush hands to the writers.
pub struct Block {
pub session: SessionKey,
+ pub origin: Origin,
pub messages: Vec<Message>,
pub stats: BlockStats,
pub moments: Vec<Moment>,
@@ -80,6 +97,45 @@ pub struct Block {
pub url_limit: Option<UrlLimit>,
}
+/// The decoded `cv` strings one flush keeps, on top of the decoded payloads that the flush size counts.
+const FLUSH_DECODE_BUDGET: usize = 512 * 1024 * 1024;
+
+/// The decoded `cv` strings one flush expands into its blocks, kept or decoded again. Small gzip payloads can expand
+/// a thousandfold, so this bounds the work, and the images, of such a flush. Later events stay as they arrived.
+const FLUSH_EXPAND_BUDGET: usize = 2 * 1024 * 1024 * 1024;
+
+/// The part of [`FLUSH_EXPAND_BUDGET`] one session may use. A session's block compresses on one thread, so this keeps
+/// one session from taking longer than a flush attempt may.
+const SESSION_EXPAND_BUDGET: usize = 256 * 1024 * 1024;
+
+/// The parsed events one flush holds. The flush size counts the payloads, and an event as small as `{}` takes far more
+/// memory parsed than in its payload, so a message that does not fit is skipped.
+const FLUSH_EVENTS_BUDGET: usize = 1024 * 1024 * 1024;
+
+struct DecodeBudget {
+ kept: AtomicUsize,
+ expanded: AtomicUsize,
+ events: AtomicUsize,
+}
+
+impl DecodeBudget {
+ fn new(kept: usize, expanded: usize) -> Self {
+ Self {
+ kept: AtomicUsize::new(kept),
+ expanded: AtomicUsize::new(expanded),
+ events: AtomicUsize::new(FLUSH_EVENTS_BUDGET),
+ }
+ }
+}
+
+fn take(budget: &AtomicUsize, bytes: usize) -> bool {
+ budget
+ .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |left| {
+ left.checked_sub(bytes)
+ })
+ .is_ok()
+}
+
pub struct Batch {
pub sessions: BTreeMap<(SessionKey, Tp), Pending>,
pub spans: HashMap<Tp, Span>,
@@ -163,40 +219,106 @@ impl Batch {
}
}
+struct Live {
+ inspector: SessionInspector,
+ partitions: Vec<Tp>,
+ last_seen: Instant,
+ bytes: usize,
+}
+
+impl Live {
+ fn new(
+ session: &SessionKey,
+ inspector: SessionInspector,
+ partitions: Vec<Tp>,
+ last_seen: Instant,
+ ) -> Self {
+ let mut live = Self {
+ inspector,
+ partitions,
+ last_seen,
+ bytes: 0,
+ };
+ live.measure(session);
+ live
+ }
+
+ fn measure(&mut self, session: &SessionKey) {
+ self.bytes = std::mem::size_of::<(SessionKey, Live)>()
+ + session.session_id.capacity()
+ + self.partitions.capacity() * std::mem::size_of::<Tp>()
+ + self.inspector.retained_bytes();
+ }
+
+ /// Keeps the session's activity and clicks, so its later blocks still count them.
+ fn evict_dom(&mut self, session: &SessionKey) {
+ self.inspector.evict_dom();
+ self.measure(session);
+ }
+}
+
/// Session inspectors by session. They keep DOM and click state across flushes.
-#[derive(Default)]
pub struct Inspectors {
- sessions: HashMap<SessionKey, (SessionInspector, Vec<Tp>, Instant)>,
+ sessions: HashMap<SessionKey, Live>,
+ max_bytes: usize,
+}
+
+impl Default for Inspectors {
+ fn default() -> Self {
+ Self::new(usize::MAX)
+ }
}
impl Inspectors {
+ pub fn new(max_bytes: usize) -> Self {
+ Self {
+ sessions: HashMap::new(),
+ max_bytes,
+ }
+ }
+
/// Decodes every buffered message and runs it through its session's inspector, with up to `concurrency` sessions
- /// at once off the async runtime. Returns one block per session.
+ /// at once off the async runtime. Returns one block per session and partition.
pub async fn inspect(&mut self, batch: Batch, concurrency: usize) -> Vec<Block> {
- let mut by_session: BTreeMap<SessionKey, (Vec<Tp>, Vec<Raw>)> = BTreeMap::new();
+ let mut work: BTreeMap<SessionKey, Vec<(Origin, Vec<Raw>)>> = BTreeMap::new();
for ((session, tp), pending) in batch.sessions {
- let (tps, raws) = by_session.entry(session).or_default();
- tps.push(tp);
- raws.extend(pending.messages);
+ let Some(span) = batch.spans.get(&tp) else {
+ continue;
+ };
+ let origin = Origin {
+ tp,
+ start_offset: span.first,
+ end_offset: pending
+ .messages
+ .iter()
+ .map(|m| m.offset)
+ .max()
+ .unwrap_or(span.first),
+ flush_end_offset: span.last,
+ };
+ work.entry(session)
+ .or_default()
+ .push((origin, pending.messages));
}
- let tasks: Vec<_> = by_session
+ let budget = Arc::new(DecodeBudget::new(FLUSH_DECODE_BUDGET, FLUSH_EXPAND_BUDGET));
+ let tasks: Vec<_> = work
.into_iter()
- .map(|(session, (tps, raws))| {
- let (inspector, prior) = match self.sessions.remove(&session) {
- Some((inspector, partitions, last_seen)) => {
- (inspector, Some((partitions, last_seen)))
- }
- None => (SessionInspector::default(), None),
+ .map(|(session, parts)| {
+ let (inspector, partitions, last_seen) = match self.sessions.remove(&session) {
+ Some(live) => (live.inspector, live.partitions, Some(live.last_seen)),
+ None => Default::default(),
};
- (session, inspector, prior, tps, raws)
+ (session, inspector, partitions, last_seen, parts)
})
.collect();
let done: Vec<_> = stream::iter(tasks)
- .map(|(session, inspector, prior, tps, raws)| {
- let (key, messages) = (session.clone(), raws.len());
+ .map(|(session, inspector, partitions, last_seen, parts)| {
+ let budget = budget.clone();
+ let key = session.clone();
+ let messages: usize = parts.iter().map(|(_, raws)| raws.len()).sum();
let task = tokio::task::spawn_blocking(move || {
- let (inspector, block) = inspect_session(&session, inspector, raws);
- (session, inspector, prior, tps, block)
+ let inspected = inspect_session(session, inspector, partitions, parts, &budget);
+ (inspected, last_seen)
});
async move { (key, messages, task.await) }
})
@@ -207,7 +329,7 @@ impl Inspectors {
let mut blocks = Vec::new();
for (key, messages, result) in done {
// A panic loses that session's block and its inspector state, and the rest of the flush goes on.
- let (session, inspector, prior, tps, block) = match result {
+ let ((session, inspector, partitions, session_blocks), last_seen) = match result {
Ok(inspected) => inspected,
Err(error) => {
tracing::error!(%error, team_id = key.team_id, "inspecting a session panicked: its block is dropped");
@@ -217,79 +339,225 @@ impl Inspectors {
}
};
// A session that decoded nothing left its state as it was, and does not count as seen.
- let Some(block) = block else {
- if let Some((partitions, last_seen)) = prior {
- self.sessions
- .insert(session, (inspector, partitions, last_seen));
- }
- continue;
+ let last_seen = if session_blocks.is_empty() {
+ last_seen
+ } else {
+ Some(now)
};
- let mut partitions = prior.map(|(partitions, _)| partitions).unwrap_or_default();
- for tp in tps {
- if !partitions.contains(&tp) {
- partitions.push(tp);
- }
+ if let Some(last_seen) = last_seen {
+ let live = Live::new(&session, inspector, partitions, last_seen);
+ self.sessions.insert(session, live);
}
- self.sessions.insert(session, (inspector, partitions, now));
- blocks.push(block);
+ blocks.extend(session_blocks);
}
- blocks.sort_by(|a, b| a.session.cmp(&b.session));
+ self.keep_within_budget();
+ blocks.sort_by(|a, b| (&a.session, a.origin.tp).cmp(&(&b.session, b.origin.tp)));
blocks
}
pub fn drop_partitions(&mut self, revoked: &[Tp]) {
self.sessions
- .retain(|_, (_, partitions, _)| !partitions.iter().any(|tp| revoked.contains(tp)));
+ .retain(|_, live| !live.partitions.iter().any(|tp| revoked.contains(tp)));
}
pub fn evict_idle(&mut self, idle: Duration) {
self.sessions
- .retain(|_, (_, _, last_seen)| last_seen.elapsed() < idle);
+ .retain(|_, live| live.last_seen.elapsed() < idle);
metrics::gauge!("ai_research_v4_recorder_live_sessions").set(self.sessions.len() as f64);
+ metrics::gauge!("ai_research_v4_recorder_retained_bytes").set(self.retained_bytes() as f64);
+ }
+
+ pub fn retained_bytes(&self) -> usize {
+ self.sessions.values().map(|live| live.bytes).sum()
+ }
+
+ /// Evicts the DOM state of the sessions seen least recently until all fit. An evicted session keeps a small entry
+ /// that marks its later blocks unverifiable. Whole sessions go only when those entries alone do not fit.
+ fn keep_within_budget(&mut self) {
+ let mut total = self.retained_bytes();
+ if total > self.max_bytes {
+ let mut oldest: Vec<(Instant, SessionKey)> = self
+ .sessions
+ .iter()
+ .map(|(session, live)| (live.last_seen, session.clone()))
+ .collect();
+ oldest.sort_unstable();
+ for (_, session) in &oldest {
+ if total <= self.max_bytes {
+ break;
+ }
+ let live = self.sessions.get_mut(session).expect("listed above");
+ let bytes = live.bytes;
+ live.evict_dom(session);
+ total = total - bytes + live.bytes;
+ if live.bytes < bytes {
+ counter!("ai_research_v4_recorder_retained_evictions_total", "kind" => "dom")
+ .increment(1);
+ }
+ }
+ for (_, session) in &oldest {
+ if total <= self.max_bytes {
+ break;
+ }
+ if let Some(live) = self.sessions.remove(session) {
+ total -= live.bytes;
+ counter!("ai_research_v4_recorder_retained_evictions_total", "kind" => "session")
+ .increment(1);
+ }
+ }
+ }
+ metrics::gauge!("ai_research_v4_recorder_retained_bytes").set(total as f64);
}
}
+/// `partitions` are those the inspector has seen the session on. Its state goes when any of them is revoked.
fn inspect_session(
- session: &SessionKey,
+ session: SessionKey,
mut inspector: SessionInspector,
- raws: Vec<Raw>,
-) -> (SessionInspector, Option<Block>) {
- let mut messages = Vec::with_capacity(raws.len());
- for raw in raws {
- let message = read(raw.payload, raw.content_encoding.as_deref())
- .and_then(|m| check_read(&m, &session.session_id).map(|()| m));
- match message {
- Ok(mut message) => {
- // Decoded once here, for the inspector and the block writer both.
- message.decode_compressed();
- messages.push(message);
- }
- Err(skip) => {
- counter!("ai_research_v4_recorder_messages_total", "outcome" => skip.as_str())
- .increment(1);
+ mut partitions: Vec<Tp>,
+ parts: Vec<(Origin, Vec<Raw>)>,
+ budget: &DecodeBudget,
+) -> (SessionKey, SessionInspector, Vec<Tp>, Vec<Block>) {
+ let mut decoded = Vec::with_capacity(parts.len());
+ let mut session_spent = 0;
+ for (origin, raws) in parts {
+ let mut messages = Vec::with_capacity(raws.len());
+ for raw in raws {
+ // A growing list can hold twice its events, so the reader stops at half of what the budget has room for.
+ let max_events = budget.events.load(Ordering::Relaxed) / size_of::<Event>() / 2;
+ let message = read_within(raw.payload, raw.content_encoding.as_deref(), max_events)
+ .and_then(|m| check_read(&m, &session.session_id).map(|()| m));
+ match message {
+ Ok(mut message) => {
+ if !take(
+ &budget.events,
+ message.buffered_bytes() - message.inner.len(),
+ ) {
+ counter!("ai_research_v4_recorder_messages_total", "outcome" => "over_budget")
+ .increment(1);
+ continue;
+ }
+ // Decoded once here, for the inspector and the block writer both.
+ let flush_room = budget.expanded.load(Ordering::Relaxed);
+ let session_room = SESSION_EXPAND_BUDGET.saturating_sub(session_spent);
+ let spent = message.decode_compressed_within(flush_room.min(session_room));
+ session_spent += spent;
+ if session_room < flush_room {
+ // The session, not the flush, ran out, so the flush's room stays for the other sessions.
+ if message
+ .events
+ .iter()
+ .any(|e| e.expansion == Some(Expansion::Skipped))
+ {
+ counter!("ai_research_v4_recorder_unexpanded_messages_total")
+ .increment(1);
+ }
+ keep_within(&mut message, spent, usize::MAX, budget);
+ } else {
+ keep_within(&mut message, spent, flush_room, budget);
+ }
+ messages.push(message);
+ }
+ Err(skip) => {
+ counter!("ai_research_v4_recorder_messages_total", "outcome" => skip.as_str())
+ .increment(1);
+ }
}
}
+ if messages.is_empty() {
+ continue;
+ }
+ if !partitions.contains(&origin.tp) {
+ partitions.push(origin.tp);
+ }
+ counter!("ai_research_v4_recorder_messages_total", "outcome" => "recorded")
+ .increment(messages.len() as u64);
+ decoded.push((origin, messages));
+ }
+ // A session can have messages on the main and the overflow topic in one flush. The inspector takes them in replay
+ // order, so that a mutation never comes before the nodes it changes. Each still counts toward its partition's block.
+ let mut inspections: Vec<BlockInspection> =
+ decoded.iter().map(|_| BlockInspection::default()).collect();
+ let times: Vec<Vec<f64>> = decoded
+ .iter()
+ .map(|(_, messages)| messages.iter().map(replay_time).collect())
+ .collect();
+ let mut next = vec![0; decoded.len()];
+ while let Some(part) = (0..decoded.len())
+ .filter(|&part| next[part] < times[part].len())
+ .min_by(|&a, &b| times[a][next[a]].total_cmp(&times[b][next[b]]))
+ {
+ inspector.observe_message(&mut inspections[part], &decoded[part].1[next[part]]);
+ next[part] += 1;
+ }
+ let blocks = decoded
+ .into_iter()
+ .zip(inspections)
+ .map(|((origin, messages), block)| {
+ let inspected = block.finish();
+ Block {
+ session: session.clone(),
+ origin,
+ messages,
+ stats: inspected.stats,
+ moments: inspected.moments,
+ image_urls: inspected.image_urls,
+ remote_urls: inspected.remote_urls,
+ url_limit: inspected.url_limit,
+ }
+ })
+ .collect();
+ (session, inspector, partitions, blocks)
+}
+
+/// When a message's events start. One without a timestamp goes as early as its partition's order allows.
+fn replay_time(message: &Message) -> f64 {
+ message
+ .events
+ .iter()
+ .find_map(|e| e.timestamp_ms)
+ .unwrap_or(f64::NEG_INFINITY)
+}
+
+/// Drops the message's decoded strings when the flush has no room left to keep them. The inspector and the block
+/// writer then each decode them again, which costs CPU but keeps the flush's memory bounded. When the flush has no
+/// room left to expand them at all, they stay as they arrived.
+///
+/// `spent` is what the decode used, failed events included, and `room` what the flush had left before it.
+fn keep_within(message: &mut Message, spent: usize, room: usize, budget: &DecodeBudget) {
+ // An event that did not fit in the flush's room uses the room up, so that each later message does not decode one
+ // event and then stop.
+ let overran = room < MESSAGE_DECODE_BUDGET
+ && message
+ .events
+ .iter()
+ .any(|e| e.expansion == Some(Expansion::Skipped));
+ let charged = budget
+ .expanded
+ .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |left| {
+ Some(if overran {
+ 0
+ } else {
+ left.saturating_sub(spent)
+ })
+ })
+ .is_ok_and(|left| left >= spent);
+ if overran || !charged {
+ counter!("ai_research_v4_recorder_unexpanded_messages_total").increment(1);
}
- if messages.is_empty() {
- return (inspector, None);
- }
- counter!("ai_research_v4_recorder_messages_total", "outcome" => "recorded")
- .increment(messages.len() as u64);
- let mut block = BlockInspection::default();
- for message in &messages {
- inspector.observe_message(&mut block, message);
- }
- let inspected = block.finish();
- let block = Block {
- session: session.clone(),
- messages,
- stats: inspected.stats,
- moments: inspected.moments,
- image_urls: inspected.image_urls,
- remote_urls: inspected.remote_urls,
- url_limit: inspected.url_limit,
+ let decoded = message.decoded_bytes();
+ let replacement = if !charged {
+ Some(Expansion::Skipped)
+ } else if !take(&budget.kept, decoded) {
+ None
+ } else {
+ return;
};
- (inspector, Some(block))
+ for event in &mut message.events {
+ if matches!(event.expansion, Some(Expansion::Decoded(_))) {
+ event.expansion = replacement.clone();
+ }
+ }
}
#[cfg(test)]
@@ -345,7 +613,11 @@ mod tests {
}
fn mouse() -> Value {
- json!([{"type": 3, "timestamp": 3, "data": {"source": 1, "positions": []}}])
+ mouse_at(3)
+ }
+
+ fn mouse_at(ts: i64) -> Value {
+ json!([{"type": 3, "timestamp": ts, "data": {"source": 1, "positions": []}}])
}
#[tokio::test]
@@ -390,22 +662,160 @@ mod tests {
assert_eq!(batch.offsets(), HashMap::from([(tp(0), 11)]));
}
+ #[test]
+ fn a_flush_without_room_keeps_its_expansions_bounded() {
+ let cv = |data: &str| -> String {
+ gzip(data.as_bytes())
+ .unwrap()
+ .iter()
+ .map(|&b| char::from(b))
+ .collect()
+ };
+ let data = json!({"node": {"type": 0, "childNodes": [], "id": 1}}).to_string();
+ let n = data.len();
+ // `room` is what the flush had left to expand when the session decoded the message.
+ let kept = |data: &str, kept: usize, expanded: usize, room: usize| {
+ let items = json!([{"type": 2, "timestamp": 1, "data": cv(data), "cv": "2024-10"}]);
+ let mut message = crate::process::read(raw(1, "a", items).payload, None).unwrap();
+ let budget = DecodeBudget::new(kept, expanded);
+ let spent = message.decode_compressed_within(room);
+ keep_within(&mut message, spent, room, &budget);
+ let expansion = match &message.events[0].expansion {
+ None => "decode again",
+ Some(Expansion::Decoded(_)) => "kept",
+ Some(Expansion::Skipped) => "as it arrived",
+ Some(Expansion::Failed) => "failed",
+ Some(other) => panic!("{other:?}"),
+ };
+ (
+ expansion,
+ budget.kept.into_inner(),
+ budget.expanded.into_inner(),
+ )
+ };
+ assert_eq!(kept(&data, n, n, n), ("kept", 0, 0));
+ assert_eq!(kept(&data, n - 1, n, n), ("decode again", n - 1, 0));
+ // An event that does not fit uses up the room, so that later messages do not each decode one event.
+ assert_eq!(kept(&data, n, n + 1, n - 1), ("as it arrived", n, 0));
+ // Another session took the room while this one decoded.
+ assert_eq!(kept(&data, n, n - 1, n), ("as it arrived", n, 0));
+ // A string that does not decode still used the room it decompressed.
+ assert_eq!(kept("not json", n, n + 8, n + 8), ("failed", n, n));
+ }
+
#[tokio::test]
- async fn a_session_that_decodes_nothing_keeps_no_state() {
+ async fn inspects_a_session_on_both_topics_in_replay_order() {
+ let overflow = Tp {
+ topic: "overflow",
+ partition: 0,
+ };
let mut batch = Batch::default();
- batch.add(
- tp(0),
- 0,
- key("a"),
- Raw {
- offset: 1,
- payload: b"not json".to_vec(),
- content_encoding: None,
- },
+ batch.add(tp(0), 0, key("a"), raw(10, "a", mouse()));
+ batch.add(overflow, 0, key("a"), raw(20, "a", snapshot()));
+ let blocks = Inspectors::default().inspect(batch, 1).await;
+ let cold: Vec<_> = blocks
+ .iter()
+ .map(|b| (b.origin.tp.topic, b.stats.cold_start))
+ .collect();
+ assert_eq!(cold, [("main", false), ("overflow", false)]);
+ }
+
+ #[tokio::test]
+ async fn revoking_any_partition_a_session_was_seen_on_drops_its_state() {
+ let mut inspectors = Inspectors::default();
+ let mut first = Batch::default();
+ first.add(tp(0), 0, key("a"), raw(1, "a", snapshot()));
+ inspectors.inspect(first, 1).await;
+ let mut second = Batch::default();
+ second.add(tp(1), 0, key("a"), raw(1, "a", mouse()));
+ inspectors.inspect(second, 1).await;
+
+ inspectors.drop_partitions(&[tp(0)]);
+ let mut third = Batch::default();
+ third.add(tp(1), 0, key("a"), raw(2, "a", mouse()));
+ let blocks = inspectors.inspect(third, 1).await;
+ assert!(blocks[0].stats.cold_start);
+ }
+
+ #[tokio::test]
+ async fn over_budget_the_session_seen_least_recently_loses_its_dom_state() {
+ let flush = |items: &[(&str, Value)]| {
+ let mut batch = Batch::default();
+ for (offset, (session, value)) in items.iter().enumerate() {
+ batch.add(
+ tp(0),
+ 0,
+ key(session),
+ raw(offset as i64, session, value.clone()),
+ );
+ }
+ batch
+ };
+ let mut measure = Inspectors::default();
+ measure.inspect(flush(&[("a", snapshot())]), 1).await;
+ let one = measure.retained_bytes();
+
+ let mut inspectors = Inspectors::new(one + one / 2);
+ inspectors
+ .inspect(flush(&[("a", snapshot()), ("a", mouse())]), 1)
+ .await;
+ inspectors.inspect(flush(&[("b", snapshot())]), 1).await;
+ assert!(inspectors.retained_bytes() <= one + one / 2);
+
+ let flags = |blocks: Vec<Block>| -> Vec<_> {
+ blocks
+ .iter()
+ .map(|b| {
+ (
+ b.session.session_id.clone(),
+ b.stats.cold_start,
+ b.stats.node_cap_reached,
+ b.stats.active_ms,
+ )
+ })
+ .collect()
+ };
+ let blocks = inspectors
+ .inspect(flush(&[("a", mouse_at(1003)), ("b", mouse_at(1003))]), 1)
+ .await;
+ // `a` lost the DOM its snapshot built, so its blocks do not claim to play until it sends a new one. It keeps
+ // its activity, so its active time goes on from its mouse move at 3.
+ assert_eq!(
+ flags(blocks),
+ [
+ ("a".to_owned(), true, true, 999),
+ ("b".to_owned(), false, false, 1)
+ ]
);
+ inspectors.inspect(flush(&[("a", snapshot())]), 1).await;
+ let blocks = inspectors.inspect(flush(&[("a", mouse())]), 1).await;
+ assert_eq!(flags(blocks), [("a".to_owned(), false, false, 0)]);
+ }
+
+ #[tokio::test]
+ async fn a_session_that_decodes_nothing_keeps_its_state_as_it_was() {
+ let unreadable = |offset: i64| Raw {
+ offset,
+ payload: b"not json".to_vec(),
+ content_encoding: None,
+ };
let mut inspectors = Inspectors::default();
- assert!(inspectors.inspect(batch, 1).await.is_empty());
- assert!(inspectors.sessions.is_empty());
+ let mut first = Batch::default();
+ first.add(tp(0), 0, key("a"), raw(1, "a", snapshot()));
+ inspectors.inspect(first, 1).await;
+ let retained = inspectors.retained_bytes();
+
+ let mut second = Batch::default();
+ second.add(tp(1), 0, key("a"), unreadable(1));
+ second.add(tp(1), 0, key("b"), unreadable(2));
+ assert!(inspectors.inspect(second, 1).await.is_empty());
+ assert_eq!(inspectors.retained_bytes(), retained);
+
+ // `a` was never recorded from partition 1, so its revoke leaves `a`'s DOM state.
+ inspectors.drop_partitions(&[tp(1)]);
+ let mut third = Batch::default();
+ third.add(tp(0), 0, key("a"), raw(2, "a", mouse()));
+ assert!(!inspectors.inspect(third, 1).await[0].stats.cold_start);
}
#[tokio::test]
@@ -419,12 +829,22 @@ mod tests {
let mut second = Batch::default();
second.add(tp(0), 0, key("a"), raw(3, "a", mouse()));
second.add(tp(0), 0, key("b"), raw(4, "b", mouse()));
+ // A message the flush skips still counts towards the flush's end on the partition.
+ second.handled(tp(0), 0, 5);
let blocks = inspectors.inspect(second, 2).await;
- let cold: Vec<_> = blocks
+ let seen: Vec<_> = blocks
.iter()
- .map(|b| (b.session.session_id.as_str(), b.stats.cold_start))
+ .map(|b| {
+ (
+ b.session.session_id.as_str(),
+ b.stats.cold_start,
+ b.origin.start_offset,
+ b.origin.end_offset,
+ b.origin.flush_end_offset,
+ )
+ })
.collect();
// `a` started with its snapshot in this process. `b` was already running when this process first saw it.
- assert_eq!(cold, vec![("a", false), ("b", true)]);
+ assert_eq!(seen, vec![("a", false, 3, 3, 5), ("b", true, 3, 4, 5)]);
}
}
diff --git a/rust/ai-research/replay-recorder-v4/src/config.rs b/rust/ai-research/replay-recorder-v4/src/config.rs
index 7b8aa2ad33b4..b90da5183cf8 100644
--- a/rust/ai-research/replay-recorder-v4/src/config.rs
+++ b/rust/ai-research/replay-recorder-v4/src/config.rs
@@ -96,6 +96,14 @@ pub struct Config {
#[envconfig(from = "AI_RESEARCH_V4_SESSION_IDLE_SECS", default = "3600")]
pub session_idle_secs: u64,
+ /// The inspector state all sessions keep between flushes. Past it the sessions seen least recently lose their DOM
+ /// state. It only stops runaway growth, such as many huge pages at once, so keep it well above normal use.
+ #[envconfig(
+ from = "AI_RESEARCH_V4_SESSION_STATE_MAX_BYTES",
+ default = "4294967296"
+ )]
+ pub session_state_max_bytes: usize,
+
/// Off, the recorder reads, admits and inspects, and stores nothing: no keys, no objects.
#[envconfig(from = "AI_RESEARCH_V4_WRITE_ENABLED", default = "false")]
pub write_enabled: bool,
diff --git a/rust/ai-research/replay-recorder-v4/src/consumer.rs b/rust/ai-research/replay-recorder-v4/src/consumer.rs
index 37c8bc1e1537..43ef92e747cb 100644
--- a/rust/ai-research/replay-recorder-v4/src/consumer.rs
+++ b/rust/ai-research/replay-recorder-v4/src/consumer.rs
@@ -28,6 +28,10 @@ const KAFKA_TIMEOUT: Duration = Duration::from_secs(10);
const COMMIT_ATTEMPTS: u32 = 3;
const HEARTBEAT: Duration = Duration::from_secs(5);
+/// The consumer's `session.timeout.ms`, the replay lanes' value. Past it the group can give the partitions to another
+/// process, so a flush stops storing then.
+pub const SESSION_TIMEOUT: Duration = Duration::from_secs(90);
+
/// The most messages read before the limiter decides on their sessions.
const POLL_BATCH: usize = 1000;
@@ -241,6 +245,10 @@ impl RecorderConsumer {
.set("enable.auto.commit", "false")
.set("enable.auto.offset.store", "false")
.set("partition.assignment.strategy", "cooperative-sticky")
+ .set(
+ "session.timeout.ms",
+ SESSION_TIMEOUT.as_millis().to_string(),
+ )
// `Config::validate` keeps every flush under this, so the group keeps the partitions while one runs.
.set(
"max.poll.interval.ms",
@@ -393,6 +401,7 @@ pub struct Settings {
pub flush_retries: u32,
pub shutdown_flush: Duration,
pub session_idle: Duration,
+ pub session_state_max_bytes: usize,
/// The main topic, when the main lane moves its force-overflow messages to the overflow topic.
pub redirecting_topic: Option<String>,
}
@@ -414,7 +423,7 @@ pub async fn run(
let shutdown = handle.shutdown_signal();
tokio::pin!(shutdown);
let mut batch = Batch::default();
- let mut inspectors = Inspectors::default();
+ let mut inspectors = Inspectors::new(settings.session_state_max_bytes);
let mut tick = tokio::time::interval(Duration::from_secs(1));
loop {
tokio::select! {
@@ -673,12 +682,36 @@ async fn flush_steps(
.await;
report(&blocks);
if let Some(writer) = writer {
- let blocks: Vec<Arc<Block>> = blocks.into_iter().map(Arc::new).collect();
+ let mut blocks: Vec<Arc<Block>> = blocks.into_iter().map(Arc::new).collect();
let work = Arc::new(FlushWork::default());
+ let started = Instant::now();
let mut attempt = 0;
loop {
+ // Readers keep the newest attempt of a flush, so an attempt after this process lost a partition would
+ // win over the new owner's flush. It also loses the partitions when the group hears nothing for the
+ // session timeout, so no attempt runs past it.
+ let owned = still_owned(
+ blocks.clone(),
+ &offsets,
+ consumer.consumer.assignment_lost(),
+ |tp| consumer.current_epoch(tp),
+ );
+ if owned.is_empty() && !blocks.is_empty() {
+ tracing::warn!(
+ "the flush lost its partitions: nothing more is written or committed"
+ );
+ return Ok(());
+ }
+ blocks = owned;
+ let left = SESSION_TIMEOUT.saturating_sub(started.elapsed());
+ if left.is_zero() {
+ return Err(format!(
+ "the flush did not store its blocks within the session timeout of {SESSION_TIMEOUT:?}"
+ ));
+ }
+ let timeout = settings.flush_timeout.min(left);
let stored = tokio::time::timeout(
- settings.flush_timeout,
+ timeout,
with_heartbeat(
handle,
write_flush(&blocks, writer, settings.flush_concurrency, &work),
@@ -688,7 +721,7 @@ async fn flush_steps(
let error = match stored {
Ok(Ok(())) => break,
Ok(Err(error)) => error.to_string(),
- Err(_) => format!("timed out after {:?}", settings.flush_timeout),
+ Err(_) => format!("timed out after {timeout:?}"),
};
counter!("ai_research_v4_recorder_flush_failures_total").increment(1);
attempt += 1;
@@ -704,6 +737,28 @@ async fn flush_steps(
Ok(())
}
+/// The blocks this process may still write: none once the group took its assignment away, and otherwise those of the
+/// partitions it still owns under the assignment the flush read them in.
+fn still_owned(
+ blocks: Vec<Arc<Block>>,
+ offsets: &HashMap<Tp, (u64, i64)>,
+ lost: bool,
+ current_epoch: impl Fn(Tp) -> Option<u64>,
+) -> Vec<Arc<Block>> {
+ if lost {
+ return Vec::new();
+ }
+ blocks
+ .into_iter()
+ .filter(|block| {
+ let tp = block.origin.tp;
+ offsets
+ .get(&tp)
+ .is_some_and(|&(epoch, _)| current_epoch(tp) == Some(epoch))
+ })
+ .collect()
+}
+
fn report(blocks: &[Block]) {
counter!("ai_research_v4_recorder_blocks_total").increment(blocks.len() as u64);
for block in blocks {
@@ -719,7 +774,10 @@ fn report(blocks: &[Block]) {
#[cfg(test)]
mod tests {
+ use replay_core_v4::inspect::stats::BlockStats;
+
use super::*;
+ use crate::batch::Origin;
fn tp(partition: i32) -> Tp {
Tp {
@@ -747,4 +805,41 @@ mod tests {
state.starts_committed(&starts, true, now);
assert!(state.uncommitted.is_empty());
}
+
+ #[test]
+ fn a_retry_writes_only_the_partitions_this_process_still_owns() {
+ let block = |partition: i32| {
+ Arc::new(Block {
+ session: SessionKey {
+ team_id: 7,
+ session_id: "s".to_owned(),
+ },
+ origin: Origin {
+ tp: tp(partition),
+ start_offset: 0,
+ end_offset: 0,
+ flush_end_offset: 0,
+ },
+ messages: Vec::new(),
+ stats: BlockStats::default(),
+ moments: Vec::new(),
+ image_urls: Vec::new(),
+ remote_urls: Vec::new(),
+ url_limit: None,
+ })
+ };
+ let offsets = HashMap::from([(tp(0), (1, 5)), (tp(1), (1, 5)), (tp(2), (1, 5))]);
+ // Partition 1 was assigned again, and partition 2 was revoked.
+ let epochs = HashMap::from([(tp(0), 1), (tp(1), 2)]);
+ let owned = |lost: bool| -> Vec<i32> {
+ still_owned(vec![block(0), block(1), block(2)], &offsets, lost, |tp| {
+ epochs.get(&tp).copied()
+ })
+ .iter()
+ .map(|b| b.origin.tp.partition)
+ .collect()
+ };
+ assert_eq!(owned(false), [0]);
+ assert!(owned(true).is_empty());
+ }
}
diff --git a/rust/ai-research/replay-recorder-v4/src/flush.rs b/rust/ai-research/replay-recorder-v4/src/flush.rs
index 8fc9aa768c48..1261a3267ad0 100644
--- a/rust/ai-research/replay-recorder-v4/src/flush.rs
+++ b/rust/ai-research/replay-recorder-v4/src/flush.rs
@@ -1,5 +1,5 @@
-//! One attempt to store a flush's blocks. It writes nothing that a retry cannot write again: object names are
-//! unique.
+//! One attempt to store a flush's blocks. It writes nothing that a retry cannot write again: object names are unique,
+//! and the block index rows name their attempt.
use std::collections::{BTreeMap, BTreeSet, HashMap};
use std::sync::{Arc, Mutex, OnceLock, PoisonError};
@@ -24,7 +24,7 @@ use crate::store::BlobStore;
use crate::urls::{fetch_jobs, records};
use crate::write::{
compress_block, new_images, object_key, seal_blocking, seal_compressed, session_keyed_frames,
- write_block_index, write_blocks, write_images, CompressedBlock, WriteError,
+ write_block_index, write_blocks, write_images, CompressedBlock, FlushAttempt, WriteError,
};
/// How long a fetch job may wait for room in the producer queue.
@@ -127,6 +127,7 @@ pub async fn write_flush(
};
drop(resolved);
let month = key_month(&day);
+ let attempt = FlushAttempt::new(now);
let object_key = object_key(&day, now);
let seals: Vec<_> = blocks
@@ -139,12 +140,17 @@ pub async fn write_flush(
};
let key = keys.sessions.get(&id)?.clone();
let team_key = *keys.team_months.get(&block.session.team_id)?;
- let (block, month, work) = (block.clone(), month.to_owned(), work.clone());
+ let (block, object_key, month, work) = (
+ block.clone(),
+ object_key.clone(),
+ month.to_owned(),
+ work.clone(),
+ );
Some(seal_blocking(&writer.seal_permits, move || {
let compressed = work.compressed(Arc::as_ptr(&block) as usize, || {
compress_block(&block, &key, &team_key, &month)
})?;
- seal_compressed(&compressed).map(|sealed| (index, sealed))
+ seal_compressed(&compressed, &object_key).map(|sealed| (index, sealed))
}))
})
.collect();
@@ -243,6 +249,7 @@ pub async fn write_flush(
write_block_index(
&entries,
&keys,
+ &attempt,
&writer.rrweb_bucket,
writer.store.as_ref(),
now,
diff --git a/rust/ai-research/replay-recorder-v4/src/index.rs b/rust/ai-research/replay-recorder-v4/src/index.rs
index 39045a4c5f69..e0bbfd06d231 100644
--- a/rust/ai-research/replay-recorder-v4/src/index.rs
+++ b/rust/ai-research/replay-recorder-v4/src/index.rs
@@ -1,6 +1,19 @@
//! The v4 index files: Parquet in `rrweb-v4/index/`. Column names are the ones in `replay-data.md`.
//!
//! Integer columns are signed, the types parquetjs writes, so a parquetjs writer can produce the same files.
+//!
+//! A flush can store its blocks and index rows and then fail before it commits its Kafka offsets. The next attempt
+//! for the partition, in this process or after a restart, starts again at the same committed offset and writes the
+//! same messages again. Every block index row therefore names its flush attempt (`flush_id`, `flushed_at_ms`) and the
+//! Kafka data it came from (`topic`, `partition`, `start_offset` of the flush on that partition, the block's own
+//! `end_offset`). A retry can cover a different range than the attempts it replaces, so the rule for readers works on
+//! ranges: per (`topic`, `partition`), a flush covers its `start_offset` to its highest `end_offset`. Walk the flushes
+//! from the highest (`flushed_at_ms`, `flush_id`) and keep a flush only if its range overlaps no flush already kept.
+//!
+//! The rows of a flush can also arrive in part. Per (`flush_id`, `topic`, `partition`), every row holds the same
+//! `flush_end_offset`, the last offset the flush covers there, and `flush_row_count`, the rows it writes there. The
+//! flush is whole when that many distinct `flush_row_index` values are present. Check this before any filter on
+//! sessions or teams.
use std::sync::Arc;
@@ -10,7 +23,8 @@ use arrow_schema::{ArrowError, DataType, Field, Fields, Schema};
use replay_core_v4::inspect::session::Moment;
use replay_core_v4::inspect::stats::BlockStats;
-use crate::write::Written;
+use crate::batch::Origin;
+use crate::write::{FlushAttempt, Written};
use parquet::arrow::ArrowWriter;
use parquet::basic::Compression;
use parquet::errors::ParquetError;
@@ -99,12 +113,18 @@ pub fn image_index(rows: &[ImageIndexRow]) -> Result<RecordBatch, IndexError> {
/// One block's index row: where it is, what the inspector found, and its sealed URLs.
pub struct BlockIndexRow<'a> {
pub written: &'a Written,
+ pub origin: &'a Origin,
+ pub attempt: &'a FlushAttempt,
pub stats: &'a BlockStats,
pub moments: &'a [Moment],
pub snapshot_source: Option<&'a str>,
pub snapshot_library: Option<&'a str>,
/// The block's visited URLs as a v4 JSON envelope under the session key.
pub urls: Option<String>,
+ /// The block rows this flush writes for the row's partition, and the row's place among them. With `flush_id`,
+ /// `topic` and `partition`, the place names the row across retries of the same publish.
+ pub flush_row_count: u32,
+ pub flush_row_index: u32,
}
fn moment_fields() -> Fields {
@@ -211,6 +231,64 @@ pub fn block_index(rows: &[BlockIndexRow<'_>]) -> Result<RecordBatch, IndexError
utf8!("session_id", |r| r.written.session.session_id.clone());
utf8!("key_month", |r| r.written.key_month.clone());
utf8!("block_s3_key", |r| r.written.object_key.clone());
+ utf8!("topic", |r| r.origin.tp.topic);
+ push(
+ "partition",
+ DataType::Int32,
+ false,
+ Arc::new(Int32Array::from_iter_values(
+ rows.iter().map(|r| r.origin.tp.partition),
+ )),
+ );
+ push(
+ "start_offset",
+ DataType::Int64,
+ false,
+ Arc::new(Int64Array::from_iter_values(
+ rows.iter().map(|r| r.origin.start_offset),
+ )),
+ );
+ push(
+ "end_offset",
+ DataType::Int64,
+ false,
+ Arc::new(Int64Array::from_iter_values(
+ rows.iter().map(|r| r.origin.end_offset),
+ )),
+ );
+ utf8!("flush_id", |r| r.attempt.flush_id.clone());
+ push(
+ "flushed_at_ms",
+ DataType::Int64,
+ false,
+ Arc::new(Int64Array::from_iter_values(
+ rows.iter().map(|r| r.attempt.flushed_at_ms),
+ )),
+ );
+ push(
+ "flush_end_offset",
+ DataType::Int64,
+ false,
+ Arc::new(Int64Array::from_iter_values(
+ rows.iter().map(|r| r.origin.flush_end_offset),
+ )),
+ );
+ push(
+ "flush_row_count",
+ DataType::Int32,
+ false,
+ Arc::new(Int32Array::from_iter_values(
+ rows.iter().map(|r| int32(r.flush_row_count)),
+ )),
+ );
+ push(
+ "flush_row_index",
+ DataType::Int32,
+ false,
+ Arc::new(Int32Array::from_iter_values(
+ rows.iter().map(|r| int32(r.flush_row_index)),
+ )),
+ );
push(
"block_byte_start",
DataType::Int64,
diff --git a/rust/ai-research/replay-recorder-v4/src/main.rs b/rust/ai-research/replay-recorder-v4/src/main.rs
index eafc3fdcf7fe..d30f7fb97b72 100644
--- a/rust/ai-research/replay-recorder-v4/src/main.rs
+++ b/rust/ai-research/replay-recorder-v4/src/main.rs
@@ -160,6 +160,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
flush_retries: config.flush_retries,
shutdown_flush: Duration::from_secs(config.shutdown_flush_secs),
session_idle: Duration::from_secs(config.session_idle_secs),
+ session_state_max_bytes: config.session_state_max_bytes,
redirecting_topic: (config.ingestion_overflow_mode == "redirect")
.then(|| config.kafka_topic.clone()),
};
diff --git a/rust/ai-research/replay-recorder-v4/src/process.rs b/rust/ai-research/replay-recorder-v4/src/process.rs
index 272ef219a227..e8ac225a9475 100644
--- a/rust/ai-research/replay-recorder-v4/src/process.rs
+++ b/rust/ai-research/replay-recorder-v4/src/process.rs
@@ -7,7 +7,7 @@ use std::borrow::Cow;
use replay_core_v4::gate::in_sample;
use replay_core_v4::payload::{decompress_payload, is_web_replay};
-use replay_core_v4::scan::message::{read_message, Message};
+use replay_core_v4::scan::message::{read_message_within, Message, ReadError};
use replay_core_v4::session::{drop_reason, DropReason};
use crate::teams::Team;
@@ -29,6 +29,8 @@ pub enum Skip {
Unreadable,
SessionMismatch,
NotWeb,
+ /// The flush has no room left for the message's events.
+ OverBudget,
}
impl Skip {
@@ -48,6 +50,7 @@ impl Skip {
Skip::Unreadable => "unreadable",
Skip::SessionMismatch => "session_mismatch",
Skip::NotWeb => "not_web",
+ Skip::OverBudget => "over_budget",
}
}
}
@@ -145,8 +148,21 @@ pub fn admit(
/// Decompresses and reads an admitted message.
pub fn read(payload: Vec<u8>, content_encoding: Option<&str>) -> Result<Message, Skip> {
+ read_within(payload, content_encoding, usize::MAX)
+}
+
+/// As [`read`], with at most `max_events` events, so that a message of many tiny events stops before its event list
+/// outgrows the flush's budget.
+pub fn read_within(
+ payload: Vec<u8>,
+ content_encoding: Option<&str>,
+ max_events: usize,
+) -> Result<Message, Skip> {
let raw = decompress_payload(payload, content_encoding).map_err(|_| Skip::Undecompressable)?;
- read_message(raw).map_err(|_| Skip::Unreadable)
+ read_message_within(raw, max_events).map_err(|error| match error {
+ ReadError::TooManyEvents => Skip::OverBudget,
+ _ => Skip::Unreadable,
+ })
}
/// The rules that need the payload: the session the headers named, and a web replay.
@@ -213,6 +229,12 @@ mod tests {
read(b"not json".to_vec(), None).unwrap_err(),
Skip::Unreadable
);
+ // The flush's event budget stops the reader before the event list grows past it.
+ assert_eq!(
+ read_within(plain.clone(), None, 0).unwrap_err(),
+ Skip::OverBudget
+ );
+ assert!(read_within(plain, None, 1).is_ok());
}
#[test]
diff --git a/rust/ai-research/replay-recorder-v4/src/urls.rs b/rust/ai-research/replay-recorder-v4/src/urls.rs
index 2396007f4458..af25d0355791 100644
--- a/rust/ai-research/replay-recorder-v4/src/urls.rs
+++ b/rust/ai-research/replay-recorder-v4/src/urls.rs
@@ -162,7 +162,7 @@ mod tests {
use replay_core_v4::keys::batch::SessionKey as Key;
use super::*;
- use crate::batch::SessionKey;
+ use crate::batch::{Origin, SessionKey, Tp};
#[test]
fn a_full_sent_cache_still_keeps_new_marks() {
@@ -192,6 +192,15 @@ mod tests {
team_id: 7,
session_id: "s".to_owned(),
},
+ origin: Origin {
+ tp: Tp {
+ topic: "main",
+ partition: 0,
+ },
+ start_offset: 0,
+ end_offset: 0,
+ flush_end_offset: 0,
+ },
messages: Vec::new(),
stats: BlockStats::default(),
moments: Vec::new(),
diff --git a/rust/ai-research/replay-recorder-v4/src/write.rs b/rust/ai-research/replay-recorder-v4/src/write.rs
index 4e811f3ce2b3..056f7a419b12 100644
--- a/rust/ai-research/replay-recorder-v4/src/write.rs
+++ b/rust/ai-research/replay-recorder-v4/src/write.rs
@@ -3,7 +3,7 @@
//! The order matters for a crash between writes. A pack without its blocks is unreferenced, which is harmless. A block
//! whose images were never stored would hold references that resolve to nothing.
-use std::collections::{BTreeMap, HashSet};
+use std::collections::{BTreeMap, HashMap, HashSet};
use std::io::{self, Write};
use std::sync::Arc;
@@ -24,7 +24,7 @@ use replay_core_v4::seal::{image_content_key, url_fingerprint_key};
use tokio::sync::Semaphore;
-use crate::batch::{Block, SessionKey};
+use crate::batch::{Block, SessionKey, Tp};
use crate::index::{
block_index, image_index, write_parquet, BlockIndexRow, ImageIndexRow, IndexError,
};
@@ -196,14 +196,19 @@ fn block_encoder() -> Result<brotlic::BrotliEncoder, WriteError> {
.map_err(seal_error)
}
-/// Builds, compresses and seals one block.
+/// Builds, compresses and seals one block for `object_key`. The key is in the frame's authenticated data, so a
+/// frame does not open from another object, even one of the same session.
pub fn seal_block(
block: &Block,
key: &Key,
team_key: &[u8; 32],
key_month: &str,
+ object_key: &str,
) -> Result<SealedBlock, WriteError> {
- seal_compressed(&compress_block(block, key, team_key, key_month)?)
+ seal_compressed(
+ &compress_block(block, key, team_key, key_month)?,
+ object_key,
+ )
}
/// Builds and compresses one block. This is most of the work of a flush, so a retry does not do it again.
@@ -235,12 +240,15 @@ pub fn compress_block(
})
}
-/// Seals a compressed block for one attempt.
-pub fn seal_compressed(block: &CompressedBlock) -> Result<SealedBlock, WriteError> {
+/// Seals a compressed block for one attempt's `object_key`.
+pub fn seal_compressed(
+ block: &CompressedBlock,
+ object_key: &str,
+) -> Result<SealedBlock, WriteError> {
let frame = seal_frame(
&block.key.key,
Version::V4,
- &block_context(&block.session, &block.key.key_month),
+ &block_context(&block.session, &block.key.key_month, object_key),
&block.body,
)
.map_err(|e| WriteError::Seal(e.to_string()))?;
@@ -255,13 +263,14 @@ pub fn seal_compressed(block: &CompressedBlock) -> Result<SealedBlock, WriteErro
}
/// The authenticated data of a block frame.
-pub fn block_context(session: &SessionKey, key_month: &str) -> Context {
+pub fn block_context(session: &SessionKey, key_month: &str, object_key: &str) -> Context {
Context {
team_id: session.team_id,
session_id: Some(session.session_id.clone()),
key_month: Some(key_month.to_owned()),
kind: "rrweb".to_owned(),
codec: Some(Codec::Brotli),
+ reference: Some(object_key.to_owned()),
..Context::default()
}
}
@@ -474,11 +483,41 @@ pub fn url_fingerprints(session_key: &[u8; 32], urls: &[String]) -> Vec<String>
fingerprints
}
+/// Numbers the rows of each partition in order and gives each row the count of its partition.
+fn number_rows(rows: &mut [BlockIndexRow<'_>]) {
+ let mut counts: HashMap<Tp, u32> = HashMap::new();
+ for row in rows.iter_mut() {
+ let count = counts.entry(row.origin.tp).or_default();
+ row.flush_row_index = *count;
+ *count += 1;
+ }
+ for row in rows.iter_mut() {
+ row.flush_row_count = counts[&row.origin.tp];
+ }
+}
+
+/// Which flush attempt wrote a block index row. See `crate::index` for how readers use it.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct FlushAttempt {
+ pub flush_id: String,
+ pub flushed_at_ms: i64,
+}
+
+impl FlushAttempt {
+ pub fn new(at: DateTime<Utc>) -> Self {
+ Self {
+ flush_id: format!("{:016x}", rand::thread_rng().gen::<u64>()),
+ flushed_at_ms: at.timestamp_millis(),
+ }
+ }
+}
+
/// Writes one block index file for the flush, after its blocks, so no row points at a block that is not stored.
/// Each entry is a block, its stats as this attempt filled them, and where it landed.
pub async fn write_block_index(
entries: &[(&Block, &BlockStats, &Written)],
keys: &FlushKeys,
+ attempt: &FlushAttempt,
rrweb_bucket: &str,
store: &dyn BlobStore,
at: DateTime<Utc>,
@@ -514,16 +553,21 @@ pub async fn write_block_index(
let first = block.messages.first();
rows.push(BlockIndexRow {
written,
+ origin: &block.origin,
+ attempt,
stats,
moments: &block.moments,
snapshot_source: first.and_then(|m| m.snapshot_source.as_deref()),
snapshot_library: first.and_then(|m| m.library.as_deref()),
urls,
+ flush_row_count: 0,
+ flush_row_index: 0,
});
}
if rows.is_empty() {
return Ok(0);
}
+ number_rows(&mut rows);
let index_key = format!(
"index/blocks/dt={}/{}.parquet",
write_day(at),
@@ -558,6 +602,7 @@ mod tests {
use serde_json::{json, Value};
use super::*;
+ use crate::batch::{Origin, Tp};
#[derive(Default)]
struct MemoryStore {
@@ -611,6 +656,18 @@ mod tests {
}
}
+ fn origin(partition: i32, start_offset: i64, end_offset: i64) -> Origin {
+ Origin {
+ tp: Tp {
+ topic: "main",
+ partition,
+ },
+ start_offset,
+ end_offset,
+ flush_end_offset: end_offset,
+ }
+ }
+
fn block(session: &str, events: Value) -> Block {
block_of(session, &events.to_string())
}
@@ -631,6 +688,7 @@ mod tests {
team_id: 7,
session_id: session.to_owned(),
},
+ origin: origin(0, 100, 105),
messages: vec![message],
stats: BlockStats::default(),
moments: Vec::new(),
@@ -780,6 +838,35 @@ mod tests {
.with_timezone(&Utc)
}
+ #[test]
+ fn block_context_reproduces_the_shared_vector() {
+ // The Node writers and readers in other languages check themselves against the same file.
+ let vector: Value = serde_json::from_str(include_str!(
+ "../../../../nodejs/src/ingestion/pipelines/sessionreplay/ai-research/v4-keys/encryption-vector-v4.json"
+ ))
+ .unwrap();
+ let block = &vector["block"];
+ let context = &block["context"];
+ let session = SessionKey {
+ team_id: context["teamId"].as_i64().unwrap(),
+ session_id: context["sessionId"].as_str().unwrap().to_owned(),
+ };
+ let built = block_context(
+ &session,
+ context["keyMonth"].as_str().unwrap(),
+ context["ref"].as_str().unwrap(),
+ );
+ assert_eq!(
+ built.authenticated_data(Version::V4),
+ block["aad"].as_str().unwrap()
+ );
+ let body = STANDARD.decode(block["body"].as_str().unwrap()).unwrap();
+ assert_eq!(
+ decompress(&body).unwrap(),
+ STANDARD.decode(block["plain"].as_str().unwrap()).unwrap()
+ );
+ }
+
#[test]
fn keeps_one_copy_of_an_image_that_several_messages_repeat() {
let events = json!([{"type": 2, "timestamp": 1, "data": {"node": {"attributes": {"src": uri(b"logo")}}}}]);
@@ -841,6 +928,7 @@ mod tests {
&keys.sessions[&session_key(&b.session.session_id).0],
&[6; 32],
"2026-10",
+ &object,
)
.unwrap()
})
@@ -895,7 +983,7 @@ mod tests {
let block = &written[0];
let frame = &memory.get("rrweb-v4", &block.object_key)
[block.byte_start as usize..=block.byte_end as usize];
- let context = block_context(&blocks[0].session, "2026-09");
+ let context = block_context(&blocks[0].session, "2026-09", &object);
let lines = String::from_utf8(
decompress(&open_frame(&[5; 32], Version::V4, &context, frame).unwrap()).unwrap(),
)
@@ -908,6 +996,9 @@ mod tests {
),
(lines.len() as u64, longest as u64)
);
+ // The frame is bound to its object: it does not open as a block of another one.
+ let elsewhere = block_context(&blocks[0].session, "2026-09", "blocks/dt=2026-10-15/other");
+ assert!(open_frame(&[5; 32], Version::V4, &elsewhere, frame).is_err());
let first: Value = serde_json::from_str(lines.lines().nth(1).unwrap()).unwrap();
let reference = first[1]["data"]["node"]["attributes"]["src"]
.as_str()
@@ -951,6 +1042,7 @@ mod tests {
write_block_index(
&[(&blocks[0], &stats, &written[0])],
&keys,
+ &FlushAttempt::new(at()),
"rrweb-v4",
store.as_ref(),
at()
@@ -1062,4 +1154,169 @@ mod tests {
]
);
}
+
+ #[tokio::test]
+ async fn readers_drop_the_rows_of_earlier_attempts_that_a_retry_covers() {
+ let store = MemoryStore::default();
+ let mut keys = FlushKeys::default();
+ for session in ["a", "b", "c"] {
+ let (id, key) = session_key(session);
+ keys.sessions.insert(id, key);
+ }
+ let mouse = json!([{"type": 3, "timestamp": 3, "data": {"source": 1, "positions": []}}]);
+ let stats = BlockStats::default();
+ let written = |session: &str| Written {
+ session: SessionKey {
+ team_id: 7,
+ session_id: session.to_owned(),
+ },
+ key_month: "2026-09".to_owned(),
+ object_key: format!("blocks/{session}"),
+ byte_start: 0,
+ byte_end: 9,
+ };
+ // Two attempts wrote their rows and failed before the commit: offsets 100-105 and 106-110 of partition 0.
+ // The retry starts at the committed offset 100, reads both ranges in one flush, and also writes partition 1.
+ let a = block("a", mouse.clone());
+ let mut early_b = block("b", mouse.clone());
+ early_b.origin = origin(0, 106, 110);
+ let mut retried_a = block("a", mouse.clone());
+ retried_a.origin.flush_end_offset = 110;
+ let mut b = block("b", mouse.clone());
+ b.origin = origin(0, 100, 110);
+ let mut c = block("c", mouse);
+ c.origin = origin(1, 40, 41);
+ let (wa, wb, wc) = (written("a"), written("b"), written("c"));
+ let first = FlushAttempt {
+ flush_id: "0000000000000001".to_owned(),
+ flushed_at_ms: 1_000,
+ };
+ let between = FlushAttempt {
+ flush_id: "0000000000000003".to_owned(),
+ flushed_at_ms: 1_500,
+ };
+ let second = FlushAttempt {
+ flush_id: "0000000000000002".to_owned(),
+ flushed_at_ms: 2_000,
+ };
+ for (attempt, entries) in [
+ (&first, vec![(&a, &stats, &wa)]),
+ (&between, vec![(&early_b, &stats, &wb)]),
+ (
+ &second,
+ vec![
+ (&retried_a, &stats, &wa),
+ (&b, &stats, &wb),
+ (&c, &stats, &wc),
+ ],
+ ),
+ ] {
+ write_block_index(&entries, &keys, attempt, "rrweb-v4", &store, at())
+ .await
+ .unwrap();
+ }
+
+ let mut rows = Vec::new();
+ let mut positions = Vec::new();
+ for key in store.keys("index/blocks/") {
+ let batch = store.parquet(&key);
+ let text = |name: &str| {
+ batch
+ .column_by_name(name)
+ .unwrap()
+ .as_any()
+ .downcast_ref::<StringArray>()
+ .unwrap()
+ .clone()
+ };
+ let int = |name: &str| {
+ batch
+ .column_by_name(name)
+ .unwrap()
+ .as_any()
+ .downcast_ref::<Int64Array>()
+ .unwrap()
+ .clone()
+ };
+ let int32 = |name: &str| {
+ batch
+ .column_by_name(name)
+ .unwrap()
+ .as_any()
+ .downcast_ref::<arrow_array::Int32Array>()
+ .unwrap()
+ .clone()
+ };
+ let partition = int32("partition");
+ let (topic, session, flush_id) = (text("topic"), text("session_id"), text("flush_id"));
+ let (start, end, at_ms) =
+ (int("start_offset"), int("end_offset"), int("flushed_at_ms"));
+ let flush_end = int("flush_end_offset");
+ let (count, index) = (int32("flush_row_count"), int32("flush_row_index"));
+ for i in 0..batch.num_rows() {
+ positions.push((
+ flush_id.value(i).to_owned(),
+ partition.value(i),
+ flush_end.value(i),
+ count.value(i),
+ index.value(i),
+ ));
+ rows.push((
+ (topic.value(i).to_owned(), partition.value(i)),
+ (start.value(i), end.value(i)),
+ (at_ms.value(i), flush_id.value(i).to_owned()),
+ session.value(i).to_owned(),
+ ));
+ }
+ }
+ assert_eq!(rows.len(), 5);
+ // Each partition of a flush numbers its rows, so a reader can tell that all of them arrived.
+ positions.sort();
+ let retry: Vec<_> = positions
+ .iter()
+ .filter(|p| p.0 == "0000000000000002")
+ .map(|p| (p.1, p.2, p.3, p.4))
+ .collect();
+ assert_eq!(retry, [(0, 110, 2, 0), (0, 110, 2, 1), (1, 41, 1, 0)]);
+
+ // The documented rule, per (topic, partition): a flush covers its start offset to its highest end offset.
+ // Walk the flushes from the latest and keep one only if it overlaps no flush already kept.
+ type Flush = ((String, i32), (i64, String));
+ let mut ranges: HashMap<Flush, (i64, i64)> = HashMap::new();
+ for (tp, (start, end), flush, _) in &rows {
+ let range = ranges
+ .entry((tp.clone(), flush.clone()))
+ .or_insert((*start, *end));
+ range.1 = range.1.max(*end);
+ }
+ let mut flushes: Vec<_> = ranges.into_iter().collect();
+ flushes.sort_by(|a, b| b.0 .1.cmp(&a.0 .1));
+ let mut kept_flushes: Vec<(Flush, (i64, i64))> = Vec::new();
+ for (flush, range) in flushes {
+ let overlaps = kept_flushes
+ .iter()
+ .any(|(k, r)| k.0 == flush.0 && range.0 <= r.1 && r.0 <= range.1);
+ if !overlaps {
+ kept_flushes.push((flush, range));
+ }
+ }
+ let mut kept: Vec<_> = rows
+ .iter()
+ .filter(|(tp, _, flush, _)| {
+ kept_flushes
+ .iter()
+ .any(|(k, _)| k.0 == *tp && k.1 == *flush)
+ })
+ .map(|(_, _, flush, session)| (session.as_str(), flush.1.as_str()))
+ .collect();
+ kept.sort();
+ assert_eq!(
+ kept,
+ [
+ ("a", "0000000000000002"),
+ ("b", "0000000000000002"),
+ ("c", "0000000000000002")
+ ]
+ );
+ }
}