:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(replay-recorder-v4): keep the pod live and safe through slow flushes (#118420)
Nicholas Waltz4cb1c737a70d
diff --git a/rust/ai-research/replay-recorder-v4/src/batch.rs b/rust/ai-research/replay-recorder-v4/src/batch.rsindex b572566e72b8..c081e87a1a0d 100644--- a/rust/ai-research/replay-recorder-v4/src/batch.rs+++ b/rust/ai-research/replay-recorder-v4/src/batch.rs@@ -5,6 +5,7 @@ use std::time::{Duration, Instant};use futures::stream::{self, StreamExt};use metrics::counter;+use replay_core_v4::compression::{GZIP_MAGIC, MAX_DECOMPRESSED_BYTES};use replay_core_v4::inspect::dom::UrlLimit;use replay_core_v4::inspect::session::{BlockInspection, Moment, SessionInspector};use replay_core_v4::inspect::stats::BlockStats;@@ -32,12 +33,39 @@ pub struct Raw {pub content_encoding: Option<String>,}-/// One session's messages from one partition in the current flush. Capture keys messages by session, so a session-/// has one partition per topic.+impl Raw {+ /// The bytes the payload decompresses to. Decompression allocates the size that the lz4 prefix or the gzip footer+ /// states, or fails, so the flush size counts that instead of the compressed bytes.+ pub fn decoded_len(&self) -> usize {+ let stated = if self.content_encoding.as_deref() == Some("lz4") {+ self.payload.get(..4)+ } else if self.payload.starts_with(&GZIP_MAGIC) {+ self.payload.get(self.payload.len().saturating_sub(4)..)+ } else {+ None+ };+ let stated = stated+ .and_then(|b| <[u8; 4]>::try_from(b).ok())+ .map(|b| u32::from_le_bytes(b) as usize)+ .filter(|&n| n <= MAX_DECOMPRESSED_BYTES);+ self.payload.len().max(stated.unwrap_or(0))+ }+}++/// One session's messages from one partition in the current flush, read under one assignment of that partition.pub struct Pending {+ pub epoch: u64,pub messages: Vec<Raw>,}+/// The offsets a flush handled on one partition, recorded or skipped alike.+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub struct Span {+ pub epoch: u64,+ pub first: i64,+ pub last: i64,+}+/// What a flush hands to the writers.pub struct Block {pub session: SessionKey,@@ -54,8 +82,7 @@ pub struct Block {pub struct Batch {pub sessions: BTreeMap<(SessionKey, Tp), Pending>,- /// The last offset handled on each partition, recorded or skipped alike.- pub offsets: HashMap<Tp, i64>,+ pub spans: HashMap<Tp, Span>,bytes: usize,started: Instant,}@@ -64,7 +91,7 @@ impl Default for Batch {fn default() -> Self {Self {sessions: BTreeMap::new(),- offsets: HashMap::new(),+ spans: HashMap::new(),bytes: 0,started: Instant::now(),}@@ -72,18 +99,34 @@ impl Default for Batch {}impl Batch {- /// Marks an offset handled. A skipped message still moves its partition forward.- pub fn handled(&mut self, tp: Tp, offset: i64) {- let last = self.offsets.entry(tp).or_insert(offset);- *last = (*last).max(offset);+ /// Marks an offset handled. A skipped message still moves its partition forward. A message of a newer assignment+ /// drops what an older one left, which Kafka delivers again from the last commit.+ pub fn handled(&mut self, tp: Tp, epoch: u64, offset: i64) {+ match self.spans.get_mut(&tp) {+ Some(span) if span.epoch == epoch => span.last = span.last.max(offset),+ stale => {+ if stale.is_some() {+ self.retain(|t, e| t != tp || e == epoch);+ }+ self.spans.insert(+ tp,+ Span {+ epoch,+ first: offset,+ last: offset,+ },+ );+ }+ }}- pub fn add(&mut self, tp: Tp, session: SessionKey, raw: Raw) {- self.handled(tp, raw.offset);- self.bytes += raw.payload.len();+ pub fn add(&mut self, tp: Tp, epoch: u64, session: SessionKey, raw: Raw) {+ self.handled(tp, epoch, raw.offset);+ self.bytes += raw.decoded_len();self.sessions.entry((session, tp)).or_insert_with(|| Pending {+ epoch,messages: Vec::new(),}).messages@@ -91,22 +134,33 @@ impl Batch {}pub fn should_flush(&self, interval: Duration, max_bytes: usize) -> bool {- !self.offsets.is_empty() && (self.bytes >= max_bytes || self.started.elapsed() >= interval)+ !self.spans.is_empty() && (self.bytes >= max_bytes || self.started.elapsed() >= interval)}- /// Forgets partitions this process no longer owns. Their messages are not committed, so the new owner reads them.- pub fn drop_partitions(&mut self, revoked: &[Tp]) {- self.sessions.retain(|(_, tp), _| !revoked.contains(tp));- for tp in revoked {- self.offsets.remove(tp);- }+ pub fn is_empty(&self) -> bool {+ self.spans.is_empty()+ }++ /// Keeps what was read under an assignment that `keep` accepts. The rest is not committed, so it is read again.+ pub fn retain(&mut self, keep: impl Fn(Tp, u64) -> bool) {+ self.sessions+ .retain(|(_, tp), pending| keep(*tp, pending.epoch));+ self.spans.retain(|tp, span| keep(*tp, span.epoch));self.bytes = self.sessions.values().flat_map(|p| &p.messages)- .map(|m| m.payload.len())+ .map(Raw::decoded_len).sum();}++ /// The next offset to commit on each partition.+ pub fn offsets(&self) -> HashMap<Tp, i64> {+ self.spans+ .iter()+ .map(|(tp, span)| (*tp, span.last + 1))+ .collect()+ }}/// Session inspectors by session. They keep DOM and click state across flushes.@@ -118,11 +172,7 @@ pub struct Inspectors {impl Inspectors {/// Decodes every buffered message and runs it through its session's inspector, with up to `concurrency` sessions/// at once off the async runtime. Returns one block per session.- pub async fn inspect(- &mut self,- batch: Batch,- concurrency: usize,- ) -> (Vec<Block>, HashMap<Tp, i64>) {+ pub async fn inspect(&mut self, batch: Batch, concurrency: usize) -> Vec<Block> {let mut by_session: BTreeMap<SessionKey, (Vec<Tp>, Vec<Raw>)> = BTreeMap::new();for ((session, tp), pending) in batch.sessions {let (tps, raws) = by_session.entry(session).or_default();@@ -184,7 +234,7 @@ impl Inspectors {blocks.push(block);}blocks.sort_by(|a, b| a.session.cmp(&b.session));- (blocks, batch.offsets)+ blocks}pub fn drop_partitions(&mut self, revoked: &[Tp]) {@@ -244,10 +294,18 @@ fn inspect_session(#[cfg(test)]mod tests {+ use replay_core_v4::compression::gzip;use serde_json::{json, Value};use super::*;+ fn tp(partition: i32) -> Tp {+ Tp {+ topic: "main",+ partition,+ }+ }+fn raw(offset: i64, session: &str, items: Value) -> Raw {let inner = json!({"event": "$snapshot_items", "properties": {"$session_id": session, "$window_id": "w", "$snapshot_source": "web", "$snapshot_items": items}});Raw {@@ -259,13 +317,6 @@ mod tests {}}- fn tp(partition: i32) -> Tp {- Tp {- topic: "main",- partition,- }- }-fn key(session: &str) -> SessionKey {SessionKey {team_id: 7,@@ -273,6 +324,22 @@ mod tests {}}+ #[test]+ fn counts_a_compressed_payload_at_its_decoded_size() {+ let plain = raw(+ 1,+ "a",+ json!([{"type": 5, "timestamp": 1, "data": "a".repeat(100_000)}]),+ );+ let size = plain.payload.len();+ let gzipped = Raw {+ payload: gzip(&plain.payload).unwrap(),+ ..plain+ };+ assert!(gzipped.payload.len() < size / 10);+ assert_eq!(gzipped.decoded_len(), size);+ }+fn snapshot() -> Value {json!([{"type": 4, "timestamp": 1, "data": {"href": "https://example.com"}}, {"type": 2, "timestamp": 2, "data": {"node": {"type": 0, "childNodes": [], "id": 1}}}])}@@ -284,14 +351,15 @@ mod tests {#[tokio::test]async fn a_revoked_partition_leaves_nothing_behind() {let mut batch = Batch::default();- batch.add(tp(0), key("a"), raw(10, "a", mouse()));- batch.add(tp(1), key("b"), raw(20, "b", mouse()));- batch.handled(tp(1), 21);+ batch.add(tp(0), 0, key("a"), raw(10, "a", mouse()));+ batch.add(tp(1), 0, key("b"), raw(20, "b", mouse()));+ batch.handled(tp(1), 0, 21);let mut inspectors = Inspectors::default();- batch.drop_partitions(&[tp(1)]);+ batch.retain(|tp, _| tp.partition != 1);inspectors.drop_partitions(&[tp(1)]);- let (blocks, offsets) = inspectors.inspect(batch, 2).await;+ assert_eq!(batch.offsets(), HashMap::from([(tp(0), 11)]));+ let blocks = inspectors.inspect(batch, 2).await;assert_eq!(blocks.iter()@@ -299,7 +367,27 @@ mod tests {.collect::<Vec<_>>(),vec!["a"]);- assert_eq!(offsets, HashMap::from([(tp(0), 10)]));+ }++ #[test]+ fn a_reassigned_partition_keeps_only_what_its_new_assignment_read() {+ // The partition was revoked and assigned again inside one poll, so Kafka delivers offset 10 again.+ let mut batch = Batch::default();+ batch.add(tp(0), 0, key("a"), raw(10, "a", mouse()));+ batch.add(tp(0), 0, key("a"), raw(11, "a", mouse()));+ batch.add(tp(0), 1, key("a"), raw(10, "a", mouse()));+ batch.retain(|_, epoch| epoch == 1);++ let pending = &batch.sessions[&(key("a"), tp(0))];+ assert_eq!(+ pending+ .messages+ .iter()+ .map(|m| m.offset)+ .collect::<Vec<_>>(),+ [10]+ );+ assert_eq!(batch.offsets(), HashMap::from([(tp(0), 11)]));}#[tokio::test]@@ -307,6 +395,7 @@ mod tests {let mut batch = Batch::default();batch.add(tp(0),+ 0,key("a"),Raw {offset: 1,@@ -315,7 +404,7 @@ mod tests {},);let mut inspectors = Inspectors::default();- assert!(inspectors.inspect(batch, 1).await.0.is_empty());+ assert!(inspectors.inspect(batch, 1).await.is_empty());assert!(inspectors.sessions.is_empty());}@@ -323,14 +412,14 @@ mod tests {async fn session_state_carries_across_flushes() {let mut inspectors = Inspectors::default();let mut first = Batch::default();- first.add(tp(0), key("a"), raw(1, "a", snapshot()));- first.add(tp(0), key("b"), raw(2, "b", mouse()));+ first.add(tp(0), 0, key("a"), raw(1, "a", snapshot()));+ first.add(tp(0), 0, key("b"), raw(2, "b", mouse()));inspectors.inspect(first, 2).await;let mut second = Batch::default();- second.add(tp(0), key("a"), raw(3, "a", mouse()));- second.add(tp(0), key("b"), raw(4, "b", mouse()));- let (blocks, _) = inspectors.inspect(second, 2).await;+ second.add(tp(0), 0, key("a"), raw(3, "a", mouse()));+ second.add(tp(0), 0, key("b"), raw(4, "b", mouse()));+ let blocks = inspectors.inspect(second, 2).await;let cold: Vec<_> = blocks.iter().map(|b| (b.session.session_id.as_str(), b.stats.cold_start))diff --git a/rust/ai-research/replay-recorder-v4/src/config.rs b/rust/ai-research/replay-recorder-v4/src/config.rsindex 5a5183b36f93..7b8aa2ad33b4 100644--- a/rust/ai-research/replay-recorder-v4/src/config.rs+++ b/rust/ai-research/replay-recorder-v4/src/config.rs@@ -1,3 +1,5 @@+use std::time::Duration;+use common_kafka::config::KafkaConfig;use envconfig::Envconfig;@@ -77,6 +79,19 @@ pub struct Config {#[envconfig(from = "AI_RESEARCH_V4_FLUSH_CONCURRENCY", default = "0")]pub flush_concurrency: usize,+ /// A flush attempt that takes longer fails. The consumer does not poll Kafka during a flush, so all attempts+ /// together stay under the poll interval and the group keeps the partitions while they run.+ #[envconfig(from = "AI_RESEARCH_V4_FLUSH_TIMEOUT_SECS", default = "60")]+ pub flush_timeout_secs: u64,++ /// How often a failed flush is tried again in-process before the recorder exits.+ #[envconfig(from = "AI_RESEARCH_V4_FLUSH_RETRIES", default = "3")]+ pub flush_retries: u32,++ /// On shutdown the buffer is flushed if that takes at most this long. Otherwise nothing is committed.+ #[envconfig(from = "AI_RESEARCH_V4_SHUTDOWN_FLUSH_SECS", default = "20")]+ pub shutdown_flush_secs: u64,+/// A session's inspector is dropped after this long without a message, so memory follows live sessions.#[envconfig(from = "AI_RESEARCH_V4_SESSION_IDLE_SECS", default = "3600")]pub session_idle_secs: u64,@@ -105,7 +120,7 @@ pub struct Config {)]pub image_fetch_topic: String,- /// A fetch job not delivered in this time fails its flush.+ /// A fetch job not delivered in this time fails its flush. It is well under the flush timeout.#[envconfig(from = "AI_RESEARCH_V4_IMAGE_FETCH_DELIVERY_TIMEOUT_MS",default = "20000"@@ -116,10 +131,30 @@ pub struct Config {pub metrics_port: u16,}+/// The consumer's `max.poll.interval.ms`.+pub const MAX_POLL_INTERVAL: Duration = Duration::from_secs(300);++/// A whole flush, from the first decode to the commit, ends within this time, so it never outlives the poll interval.+pub const FLUSH_DEADLINE: Duration = Duration::from_secs(MAX_POLL_INTERVAL.as_secs() - 10);+/// The slowest refill of a scaled new-session bucket: one session each 100 seconds.const MIN_NEW_SESSION_REPLENISH_RATE: f64 = 0.01;+/// The pause before retry `attempt` of a failed flush.+pub fn retry_backoff(attempt: u32) -> Duration {+ Duration::from_secs(1 << attempt.min(5))+}+impl Config {+ /// The longest a flush can store for, over all its attempts.+ pub fn flush_budget(&self) -> Duration {+ let attempts = Duration::from_secs(self.flush_timeout_secs) * (self.flush_retries + 1);+ attempts+ + (1..=self.flush_retries)+ .map(retry_backoff)+ .sum::<Duration>()+ }+/// The bucket v4 runs, as `(capacity, replenish rate per second)`. v4 sees only its sample of the sessions, so both/// scale by the sample, and a flood trips v4 at about the rate that trips the main lane.pub fn new_session_bucket(&self) -> (f64, f64) {@@ -139,6 +174,71 @@ impl Config {),)}++ /// Catches settings that would otherwise fail at the first flush and crash loop.+ pub fn validate(&self) -> Result<(), String> {+ if !(0.0..=100.0).contains(&self.sample_percentage) {+ return Err(format!(+ "AI_RESEARCH_V4_SAMPLE_PERCENTAGE must be from 0 to 100, not {}",+ self.sample_percentage+ ));+ }+ let mut required = vec![+ ("KAFKA_TOPIC", &self.kafka_topic),+ (+ "EVENT_RESTRICTIONS_REDIS_URL",+ &self.event_restrictions_redis_url,+ ),+ ("AI_RESEARCH_V4_REDIS_URL", &self.redis_url),+ ];+ if self.write_enabled {+ required.extend([+ ("AI_RESEARCH_V4_KEY_TABLE", &self.key_table),+ ("AI_RESEARCH_V4_KMS_KEY_ARN", &self.kms_key_arn),+ ("AI_RESEARCH_V4_RRWEB_BUCKET", &self.rrweb_bucket),+ ("AI_RESEARCH_V4_IMAGES_BUCKET", &self.images_bucket),+ ("AI_RESEARCH_V4_IMAGE_FETCH_TOPIC", &self.image_fetch_topic),+ ]);+ }+ let missing: Vec<&str> = required+ .into_iter()+ .filter(|(_, value)| value.trim().is_empty())+ .map(|(name, _)| name)+ .collect();+ if !missing.is_empty() {+ return Err(format!("missing settings: {}", missing.join(", ")));+ }+ for (name, value) in [+ (+ "AI_RESEARCH_V4_NEW_SESSION_BUCKET_CAPACITY",+ self.new_session_bucket_capacity,+ ),+ (+ "AI_RESEARCH_V4_NEW_SESSION_BUCKET_REPLENISH_RATE",+ self.new_session_bucket_replenish_rate,+ ),+ ] {+ if !(value.is_finite() && value >= 0.0) {+ return Err(format!("{name} must be 0 or more, not {value}"));+ }+ }+ if self.team_refresh_secs == 0 {+ return Err("AI_RESEARCH_V4_TEAM_REFRESH_SECS must be over 0".to_owned());+ }+ if self.flush_budget() >= FLUSH_DEADLINE {+ return Err(format!(+ "AI_RESEARCH_V4_FLUSH_TIMEOUT_SECS and AI_RESEARCH_V4_FLUSH_RETRIES allow a flush of {:?}, which is not under the Kafka poll interval",+ self.flush_budget()+ ));+ }+ if self.image_fetch_delivery_timeout_ms as u64 >= self.flush_timeout_secs * 1000 {+ return Err(+ "AI_RESEARCH_V4_IMAGE_FETCH_DELIVERY_TIMEOUT_MS must be under the flush timeout"+ .to_owned(),+ );+ }+ Ok(())+ }}#[cfg(test)]@@ -166,6 +266,37 @@ mod tests {Config::init_from_hashmap(&env).unwrap()}+ #[test]+ fn writes_need_every_resource_name_at_startup() {+ assert_eq!(config(&[]).validate(), Ok(()));+ let error = config(&[+ ("AI_RESEARCH_V4_WRITE_ENABLED", "true"),+ ("AI_RESEARCH_V4_KEY_TABLE", "keys"),+ ("AI_RESEARCH_V4_RRWEB_BUCKET", "rrweb"),+ ])+ .validate()+ .unwrap_err();+ assert_eq!(+ error,+ "missing settings: AI_RESEARCH_V4_KMS_KEY_ARN, AI_RESEARCH_V4_IMAGES_BUCKET"+ );+ assert!(config(&[("AI_RESEARCH_V4_SAMPLE_PERCENTAGE", "110")])+ .validate()+ .is_err());+ assert!(config(&[("AI_RESEARCH_V4_TEAM_REFRESH_SECS", "0")])+ .validate()+ .is_err());+ assert!(+ config(&[("AI_RESEARCH_V4_NEW_SESSION_BUCKET_CAPACITY", "-1")])+ .validate()+ .is_err()+ );+ assert_eq!(+ config(&[("AI_RESEARCH_V4_REDIS_URL", " ")]).validate(),+ Err("missing settings: AI_RESEARCH_V4_REDIS_URL".to_owned())+ );+ }+#[test]fn scales_the_new_session_bucket_by_the_sample() {let bucket = |sample: &str, capacity: &str, rate: &str| {diff --git a/rust/ai-research/replay-recorder-v4/src/consumer.rs b/rust/ai-research/replay-recorder-v4/src/consumer.rsindex 6d754419009d..37c8bc1e1537 100644--- a/rust/ai-research/replay-recorder-v4/src/consumer.rs+++ b/rust/ai-research/replay-recorder-v4/src/consumer.rs@@ -1,33 +1,82 @@//! The consume loop: gate and admit each message from its key and headers, buffer it, and flush on time or size.-use std::collections::{BTreeMap, HashSet};+use std::collections::{BTreeMap, HashMap, HashSet};+use std::future::Future;use std::sync::{Arc, Mutex};-use std::time::{Duration, SystemTime, UNIX_EPOCH};+use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};use common_kafka::config::KafkaConfig;use futures::FutureExt;use lifecycle::Handle;use metrics::counter;use rdkafka::consumer::{- BaseConsumer, CommitMode, Consumer, ConsumerContext, Rebalance, StreamConsumer,+ BaseConsumer, CommitMode, Consumer, ConsumerContext, RebalanceProtocol, StreamConsumer,};-use rdkafka::error::KafkaError;+use rdkafka::error::{KafkaError, RDKafkaErrorCode};use rdkafka::message::{BorrowedMessage, Headers as _, Message as _};+use rdkafka::types::RDKafkaRespErr;use rdkafka::{ClientConfig, ClientContext, Offset, TopicPartitionList};use crate::batch::{Batch, Block, Inspectors, Raw, SessionKey, Tp};-use crate::flush::{write_flush, FlushError, Writer};+use crate::config::{retry_backoff, FLUSH_DEADLINE, MAX_POLL_INTERVAL};+use crate::flush::{write_flush, FlushWork, Writer};use crate::limits::{Restrictions, SessionLimiter};use crate::process::{admit, sampled, Admitted, Headers, Skip};use crate::teams::TeamCache;+const KAFKA_TIMEOUT: Duration = Duration::from_secs(10);+const COMMIT_ATTEMPTS: u32 = 3;+const HEARTBEAT: Duration = Duration::from_secs(5);+/// The most messages read before the limiter decides on their sessions.const POLL_BATCH: usize = 1000;-/// Records the partitions each rebalance takes away, so the loop can drop what it buffered for them.+/// The assignment state the rebalance callbacks keep. A partition's epoch moves on every revoke, so a message read+/// before the revoke is never mistaken for one read after a new assignment of the same partition.+#[derive(Default)]+struct Assignments {+ epochs: HashMap<Tp, (u64, bool)>,+ revoked: Vec<Tp>,+ /// Partitions without a known committed offset, with the first offset read once there is one. It stays here until+ /// it commits or the partition is revoked.+ uncommitted: HashMap<Tp, Option<i64>>,+ /// After a failed start commit, the next one waits until then.+ start_retry_at: Option<Instant>,+}++/// The pause after a failed start commit, so a broker that does not answer does not stall every message.+const START_RETRY: Duration = Duration::from_secs(5);++impl Assignments {+ /// The start offsets to commit now: all pending ones when `force`, otherwise none during the pause after a failure.+ fn starts_due(&self, now: Instant, force: bool) -> Vec<(Tp, i64)> {+ if !force && self.start_retry_at.is_some_and(|at| now < at) {+ return Vec::new();+ }+ self.uncommitted+ .iter()+ .filter_map(|(&tp, start)| Some((tp, (*start)?)))+ .collect()+ }++ fn starts_committed(&mut self, starts: &[(Tp, i64)], committed: bool, now: Instant) {+ if committed {+ for (tp, offset) in starts {+ if self.uncommitted.get(tp) == Some(&Some(*offset)) {+ self.uncommitted.remove(tp);+ }+ }+ self.start_retry_at = None;+ } else {+ self.start_retry_at = Some(now + START_RETRY);+ }+ }+}+pub struct RebalanceContext {topics: Vec<&'static str>,- revoked: Arc<Mutex<Vec<Tp>>>,+ commit_latest: bool,+ state: Arc<Mutex<Assignments>>,}impl RebalanceContext {@@ -35,26 +84,139 @@ impl RebalanceContext {let topic = *self.topics.iter().find(|t| **t == topic)?;Some(Tp { topic, partition })}++ /// A new group starts at the head of the topic. Assigning that head and committing it before any fetch means a+ /// partition revoked before its first flush sends its next owner to the same offset, past nothing unread.+ fn start_at_heads(&self, consumer: &BaseConsumer<Self>, assigned: &mut TopicPartitionList) {+ // One deadline for the whole assignment, so many slow partitions cannot hold the rebalance past liveness.+ let deadline = Instant::now() + KAFKA_TIMEOUT;+ let remaining = || deadline.saturating_duration_since(Instant::now());+ let committed = match consumer.committed_offsets(assigned.clone(), remaining()) {+ Ok(committed) => committed,+ Err(error) => {+ // Each partition commits its first offset read instead. Where an offset is committed already, the+ // first message read is at that offset, so the commit does not move it.+ tracing::warn!(%error, "reading committed offsets failed");+ let assigned = assigned+ .elements()+ .iter()+ .filter_map(|e| self.tp(e.topic(), e.partition()))+ .collect::<Vec<_>>();+ self.state+ .lock()+ .unwrap()+ .uncommitted+ .extend(assigned.into_iter().map(|tp| (tp, None)));+ return;+ }+ };+ let mut heads = TopicPartitionList::new();+ let mut uncommitted = Vec::new();+ for element in committed.elements() {+ if element.offset() != Offset::Invalid {+ continue;+ }+ let Some(tp) = self.tp(element.topic(), element.partition()) else {+ continue;+ };+ let head = if self.commit_latest && !remaining().is_zero() {+ consumer+ .fetch_watermarks(tp.topic, tp.partition, remaining())+ .map_err(|error| tracing::warn!(%error, "reading a partition head failed"))+ .ok()+ } else {+ None+ };+ let started = head.is_some_and(|(_, high)| {+ let offset = Offset::Offset(high);+ assigned+ .set_partition_offset(tp.topic, tp.partition, offset)+ .and_then(|()| heads.add_partition_offset(tp.topic, tp.partition, offset))+ .map_err(|error| tracing::warn!(%error, "building the starting offsets failed"))+ .is_ok()+ });+ if !started {+ uncommitted.push(tp);+ }+ }+ if heads.count() > 0 {+ if let Err(error) = consumer.commit(&heads, CommitMode::Sync) {+ // The first message read commits its offset instead.+ tracing::warn!(%error, "committing the starting offsets failed");+ uncommitted.extend(+ heads+ .elements()+ .iter()+ .filter_map(|e| self.tp(e.topic(), e.partition())),+ );+ }+ }+ self.state+ .lock()+ .unwrap()+ .uncommitted+ .extend(uncommitted.into_iter().map(|tp| (tp, None)));+ }++ fn mark(&self, partitions: &TopicPartitionList, assigned: bool) {+ let mut state = self.state.lock().unwrap();+ for element in partitions.elements() {+ if let Some(tp) = self.tp(element.topic(), element.partition()) {+ if assigned {+ state.epochs.entry(tp).or_default().1 = true;+ } else {+ let entry = state.epochs.entry(tp).or_default();+ *entry = (entry.0 + 1, false);+ state.revoked.push(tp);+ state.uncommitted.remove(&tp);+ }+ }+ }+ }}impl ClientContext for RebalanceContext {}impl ConsumerContext for RebalanceContext {- fn pre_rebalance(&self, _: &BaseConsumer<Self>, rebalance: &Rebalance<'_>) {- if let Rebalance::Revoke(partitions) = rebalance {- let revoked = partitions- .elements()- .iter()- .filter_map(|e| self.tp(e.topic(), e.partition()))- .collect::<Vec<_>>();- self.revoked.lock().unwrap().extend(revoked);+ /// rdkafka's own rebalance, with the starting offsets set on the assignment before it is applied.+ fn rebalance(+ &self,+ consumer: &BaseConsumer<Self>,+ err: RDKafkaRespErr,+ partitions: &mut TopicPartitionList,+ ) {+ let cooperative = matches!(+ consumer.rebalance_protocol(),+ RebalanceProtocol::Cooperative+ );+ let applied = if err == RDKafkaRespErr::RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS {+ self.start_at_heads(consumer, partitions);+ self.mark(partitions, true);+ if cooperative {+ consumer.incremental_assign(partitions)+ } else {+ consumer.assign(partitions)+ }+ } else {+ if err != RDKafkaRespErr::RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS {+ tracing::error!(error = %RDKafkaErrorCode::from(err), "rebalancing failed");+ }+ self.mark(partitions, false);+ if cooperative {+ consumer.incremental_unassign(partitions)+ } else {+ consumer.unassign()+ }+ };+ if let Err(error) = applied {+ tracing::error!(%error, "applying a rebalance failed");}}}pub struct RecorderConsumer {- consumer: StreamConsumer<RebalanceContext>,- revoked: Arc<Mutex<Vec<Tp>>>,+ consumer: Arc<StreamConsumer<RebalanceContext>>,+ state: Arc<Mutex<Assignments>>,}impl RecorderConsumer {@@ -78,7 +240,12 @@ impl RecorderConsumer {// Offsets move only after a flush has stored the blocks for them..set("enable.auto.commit", "false").set("enable.auto.offset.store", "false")- .set("partition.assignment.strategy", "cooperative-sticky");+ .set("partition.assignment.strategy", "cooperative-sticky")+ // `Config::validate` keeps every flush under this, so the group keeps the partitions while one runs.+ .set(+ "max.poll.interval.ms",+ MAX_POLL_INTERVAL.as_millis().to_string(),+ );if kafka.kafka_tls {config.set("security.protocol", "ssl")@@ -87,14 +254,18 @@ impl RecorderConsumer {if !kafka.kafka_client_rack.is_empty() {config.set("client.rack", &kafka.kafka_client_rack);}- let revoked = Arc::new(Mutex::new(Vec::new()));+ let state = Arc::new(Mutex::new(Assignments::default()));let consumer: StreamConsumer<RebalanceContext> =config.create_with_context(RebalanceContext {topics: topics.clone(),- revoked: revoked.clone(),+ commit_latest: offset_reset == "latest",+ state: state.clone(),})?;consumer.subscribe(&topics)?;- Ok(Self { consumer, revoked })+ Ok(Self {+ consumer: Arc::new(consumer),+ state,+ })}/// The main topic comes first. The overflow lane's limiter only counts, so v4's does the same there.@@ -103,29 +274,105 @@ impl RecorderConsumer {}fn take_revoked(&self) -> Vec<Tp> {- std::mem::take(&mut *self.revoked.lock().unwrap())+ std::mem::take(&mut self.state.lock().unwrap().revoked)}- /// Commits the next offset of every partition this process still owns.- fn commit(&self, offsets: &std::collections::HashMap<Tp, i64>) {- let assigned = self.consumer.assignment().unwrap_or_default();+ /// The epoch of a partition this process owns now.+ fn current_epoch(&self, tp: Tp) -> Option<u64> {+ match self.state.lock().unwrap().epochs.get(&tp) {+ Some(&(epoch, true)) => Some(epoch),+ _ => None,+ }+ }++ /// Commits the first offset read on a partition that had none, so a restart before the first flush starts where+ /// this process started, and the flush's `start_offset` stays the same.+ async fn commit_start(&self, tp: Tp, offset: i64) {+ match self.state.lock().unwrap().uncommitted.get_mut(&tp) {+ Some(start) => {+ start.get_or_insert(offset);+ }+ None => return,+ }+ self.commit_starts(false).await;+ }++ /// Commits the start offsets that are still pending. A failed one is tried again on a later message, and with+ /// `force` before a flush stores anything.+ async fn commit_starts(&self, force: bool) {+ let starts = self.state.lock().unwrap().starts_due(Instant::now(), force);+ if starts.is_empty() {+ return;+ }let mut list = TopicPartitionList::new();- for (&tp, &offset) in offsets {- if assigned.find_partition(tp.topic, tp.partition).is_some() {- if let Err(error) =- list.add_partition_offset(tp.topic, tp.partition, Offset::Offset(offset + 1))- {- tracing::warn!(%error, partition = tp.partition, "building the offset commit failed");- }+ let committed = match starts.iter().try_for_each(|(tp, offset)| {+ list.add_partition_offset(tp.topic, tp.partition, Offset::Offset(*offset))+ }) {+ Ok(()) => self.commit_list(list).await,+ Err(error) => Err(error.to_string()),+ };+ if let Err(error) = &committed {+ tracing::warn!(%error, "committing the start offsets failed");+ }+ self.state+ .lock()+ .unwrap()+ .starts_committed(&starts, committed.is_ok(), Instant::now());+ }++ /// Commits the next offset of every partition this process still owns under the same assignment.+ async fn commit(&self, offsets: &HashMap<Tp, (u64, i64)>) -> Result<(), String> {+ let mut list = TopicPartitionList::new();+ for (&tp, &(epoch, offset)) in offsets {+ if self.current_epoch(tp) == Some(epoch) {+ list.add_partition_offset(tp.topic, tp.partition, Offset::Offset(offset))+ .map_err(|e| e.to_string())?;}}- if list.count() > 0 {- if let Err(error) = self.consumer.commit(&list, CommitMode::Async) {- tracing::warn!(%error, "committing offsets failed");- counter!("ai_research_v4_recorder_commit_failures_total").increment(1);+ if list.count() == 0 {+ return Ok(());+ }+ let mut attempt = 0;+ loop {+ match self.commit_list(list.clone()).await {+ Ok(()) => {+ // A later offset committed, so a pending start offset is of no use any more.+ let mut state = self.state.lock().unwrap();+ for element in list.elements() {+ if let Some(tp) = self+ .consumer+ .context()+ .tp(element.topic(), element.partition())+ {+ state.uncommitted.remove(&tp);+ }+ }+ return Ok(());+ }+ Err(error) => {+ counter!("ai_research_v4_recorder_commit_failures_total").increment(1);+ attempt += 1;+ if attempt >= COMMIT_ATTEMPTS {+ return Err(format!("committing offsets failed: {error}"));+ }+ tracing::warn!(%error, attempt, "committing offsets failed");+ tokio::time::sleep(Duration::from_millis(500 * u64::from(attempt))).await;+ }}}}++ /// A synchronous commit on a blocking thread, so the caller's deadlines and heartbeats keep running. A commit+ /// that outlives `KAFKA_TIMEOUT` can still land later. Its offsets were already stored or read, so it skips nothing.+ async fn commit_list(&self, list: TopicPartitionList) -> Result<(), String> {+ let consumer = self.consumer.clone();+ let commit = tokio::task::spawn_blocking(move || consumer.commit(&list, CommitMode::Sync));+ match tokio::time::timeout(KAFKA_TIMEOUT, commit).await {+ Ok(Ok(committed)) => committed.map_err(|e| e.to_string()),+ Ok(Err(error)) => Err(error.to_string()),+ Err(_) => Err(format!("timed out after {KAFKA_TIMEOUT:?}")),+ }+ }}fn header<'m>(message: &'m BorrowedMessage<'_>, name: &str) -> Option<&'m str> {@@ -142,6 +389,9 @@ pub struct Settings {pub flush_interval: Duration,pub flush_max_bytes: usize,pub flush_concurrency: usize,+ pub flush_timeout: Duration,+ pub flush_retries: u32,+ pub shutdown_flush: Duration,pub session_idle: Duration,/// The main topic, when the main lane moves its force-overflow messages to the overflow topic.pub redirecting_topic: Option<String>,@@ -203,21 +453,61 @@ pub async fn run(handle.report_healthy();let revoked = consumer.take_revoked();if !revoked.is_empty() {- batch.drop_partitions(&revoked);inspectors.drop_partitions(&revoked);+ batch.retain(|tp, epoch| consumer.current_epoch(tp) == Some(epoch));}if batch.should_flush(settings.flush_interval, settings.flush_max_bytes) {- let (blocks, offsets) = inspectors- .inspect(std::mem::take(&mut batch), settings.flush_concurrency)- .await;- if let Err(error) = flush(blocks, writer.as_ref(), settings.flush_concurrency).await {+ let work = flush(+ std::mem::take(&mut batch),+ &mut inspectors,+ &consumer,+ &settings,+ writer.as_ref(),+ &handle,+ settings.flush_retries,+ );+ tokio::pin!(work);+ let flushed = tokio::select! {+ flushed = &mut work => flushed,+ _ = &mut shutdown => {+ // The running flush holds the whole buffer, so it goes on under the shutdown deadline.+ flush_on_shutdown(settings.shutdown_flush, work).await;+ return;+ }+ };+ if let Err(error) = flushed {// Nothing of this flush is committed, so a restart reads it again from the last commit.tracing::error!(%error, "flush failed");handle.signal_failure(format!("flush failed: {error}"));- break;+ return;}- consumer.commit(&offsets);- inspectors.evict_idle(settings.session_idle);+ }+ }+ if batch.is_empty() {+ return;+ }+ flush_on_shutdown(+ settings.shutdown_flush,+ flush(+ batch,+ &mut inspectors,+ &consumer,+ &settings,+ writer.as_ref(),+ &handle,+ 0,+ ),+ )+ .await;+}++/// A flush cut off by the deadline before its commit leaves the buffer to the next owner, which reads it again.+async fn flush_on_shutdown(deadline: Duration, work: impl Future<Output = Result<(), String>>) {+ match tokio::time::timeout(deadline, work).await {+ Ok(Ok(())) => tracing::info!("flushed the buffer on shutdown"),+ Ok(Err(error)) => tracing::warn!(%error, "the flush on shutdown failed: nothing committed"),+ Err(_) => {+ tracing::warn!("the flush on shutdown did not fit the grace period: nothing committed")}}}@@ -251,17 +541,21 @@ async fn handle_messages(else {continue;};+ let Some(epoch) = consumer.current_epoch(tp) else {+ continue;+ };+ consumer.commit_start(tp, message.offset()).await;let session = admit_message(message, tp, settings, sources).await.map(|admitted| SessionKey {team_id: admitted.team_id,session_id: admitted.session_id,});- admitted.push((message, tp, session));+ admitted.push((message, tp, epoch, session));}// The limiter decides before anything is buffered, so a blocked session's messages never reach a block.let mut sessions: BTreeMap<SessionKey, bool> = BTreeMap::new();- for (_, tp, session) in &admitted {+ for (_, tp, _, session) in &admitted {if let Ok(session) = session {*sessions.entry(session.clone()).or_default() |= consumer.is_main(*tp);}@@ -271,7 +565,7 @@ async fn handle_messages(} else {sources.limiter.blocked(&sessions).await};- for (message, tp, session) in admitted {+ for (message, tp, epoch, session) in admitted {let session = session.and_then(|session| {if blocked.contains(&session) {Err(Skip::Blocked)@@ -288,12 +582,12 @@ async fn handle_messages(payload: message.payload().unwrap_or_default().to_vec(),content_encoding: header(message, "content-encoding").map(str::to_owned),};- batch.add(tp, session, raw);+ batch.add(tp, epoch, session, raw);}Err(skip) => {counter!("ai_research_v4_recorder_messages_total", "outcome" => skip.as_str()).increment(1);- batch.handled(tp, message.offset());+ batch.handled(tp, epoch, message.offset());}}}@@ -323,14 +617,96 @@ async fn admit_message(admit(&headers, restriction, redirect_overflow, team, now_ms)}-/// Stores a flush's blocks, or only reports them when the recorder runs dry.+/// Reports the component healthy while `work` runs. `flush` bounds `work` with a deadline, so a hung flush still ends.+async fn with_heartbeat<T>(handle: &Handle, work: impl Future<Output = T>) -> T {+ tokio::pin!(work);+ let mut beat = tokio::time::interval(HEARTBEAT);+ loop {+ tokio::select! {+ out = &mut work => return out,+ _ = beat.tick() => handle.report_healthy(),+ }+ }+}++/// Decodes and inspects the buffered sessions, stores them with up to `retries` more attempts, and only then commits.+/// All of it ends within `FLUSH_DEADLINE`.+#[allow(clippy::too_many_arguments)]async fn flush(- blocks: Vec<Block>,+ batch: Batch,+ inspectors: &mut Inspectors,+ consumer: &RecorderConsumer,+ settings: &Settings,+ writer: Option<&Writer>,+ handle: &Handle,+ retries: u32,+) -> Result<(), String> {+ let work = flush_steps(+ batch, inspectors, consumer, settings, writer, handle, retries,+ );+ tokio::time::timeout(FLUSH_DEADLINE, work)+ .await+ .unwrap_or_else(|_| Err(format!("the flush did not end in {FLUSH_DEADLINE:?}")))+}++#[allow(clippy::too_many_arguments)]+async fn flush_steps(+ batch: Batch,+ inspectors: &mut Inspectors,+ consumer: &RecorderConsumer,+ settings: &Settings,writer: Option<&Writer>,- concurrency: usize,-) -> Result<(), FlushError> {+ handle: &Handle,+ retries: u32,+) -> Result<(), String> {+ let offsets: HashMap<Tp, (u64, i64)> = batch+ .spans+ .iter()+ .map(|(tp, span)| (*tp, (span.epoch, span.last + 1)))+ .collect();+ // Before anything is stored, so a restart after a failed flush reads it again from where this process started.+ with_heartbeat(handle, consumer.commit_starts(true)).await;+ let blocks = with_heartbeat(+ handle,+ inspectors.inspect(batch, settings.flush_concurrency),+ )+ .await;+ report(&blocks);+ if let Some(writer) = writer {+ let blocks: Vec<Arc<Block>> = blocks.into_iter().map(Arc::new).collect();+ let work = Arc::new(FlushWork::default());+ let mut attempt = 0;+ loop {+ let stored = tokio::time::timeout(+ settings.flush_timeout,+ with_heartbeat(+ handle,+ write_flush(&blocks, writer, settings.flush_concurrency, &work),+ ),+ )+ .await;+ let error = match stored {+ Ok(Ok(())) => break,+ Ok(Err(error)) => error.to_string(),+ Err(_) => format!("timed out after {:?}", settings.flush_timeout),+ };+ counter!("ai_research_v4_recorder_flush_failures_total").increment(1);+ attempt += 1;+ if attempt > retries {+ return Err(error);+ }+ tracing::warn!(%error, attempt, "flush attempt failed, trying again");+ with_heartbeat(handle, tokio::time::sleep(retry_backoff(attempt))).await;+ }+ }+ with_heartbeat(handle, consumer.commit(&offsets)).await?;+ inspectors.evict_idle(settings.session_idle);+ Ok(())+}++fn report(blocks: &[Block]) {counter!("ai_research_v4_recorder_blocks_total").increment(blocks.len() as u64);- for block in &blocks {+ for block in blocks {counter!("ai_research_v4_recorder_block_events_total").increment(block.stats.event_count);if block.stats.cold_start {counter!("ai_research_v4_recorder_cold_start_blocks_total").increment(1);@@ -339,7 +715,36 @@ async fn flush(counter!("ai_research_v4_recorder_orphan_mutation_blocks_total").increment(1);}}- let Some(writer) = writer else { return Ok(()) };- let blocks: Vec<Arc<Block>> = blocks.into_iter().map(Arc::new).collect();- write_flush(&blocks, writer, concurrency).await+}++#[cfg(test)]+mod tests {+ use super::*;++ fn tp(partition: i32) -> Tp {+ Tp {+ topic: "main",+ partition,+ }+ }++ #[test]+ fn a_failed_start_commit_stays_pending_at_the_first_offset() {+ let mut state = Assignments::default();+ state.uncommitted.insert(tp(0), None);+ state.uncommitted.get_mut(&tp(0)).unwrap().get_or_insert(10);+ let now = Instant::now();+ let starts = state.starts_due(now, false);+ assert_eq!(starts, [(tp(0), 10)]);+ state.starts_committed(&starts, false, now);++ // A later message does not move the start, and waits for the pause unless the flush forces it.+ state.uncommitted.get_mut(&tp(0)).unwrap().get_or_insert(11);+ assert!(state.starts_due(now, false).is_empty());+ assert_eq!(state.starts_due(now, true), [(tp(0), 10)]);+ let starts = state.starts_due(now + START_RETRY, false);+ assert_eq!(starts, [(tp(0), 10)]);+ state.starts_committed(&starts, true, now);+ assert!(state.uncommitted.is_empty());+ }}diff --git a/rust/ai-research/replay-recorder-v4/src/flush.rs b/rust/ai-research/replay-recorder-v4/src/flush.rsindex ee92ed2b76f2..8fc9aa768c48 100644--- a/rust/ai-research/replay-recorder-v4/src/flush.rs+++ b/rust/ai-research/replay-recorder-v4/src/flush.rs@@ -1,7 +1,8 @@-//! Stores a flush's blocks: their keys, the sealed blocks, the image packs, both indexes and the fetch jobs.+//! One attempt to store a flush's blocks. It writes nothing that a retry cannot write again: object names are+//! unique.-use std::collections::{BTreeMap, BTreeSet};-use std::sync::Arc;+use std::collections::{BTreeMap, BTreeSet, HashMap};+use std::sync::{Arc, Mutex, OnceLock, PoisonError};use std::time::Duration;use chrono::Utc;@@ -12,7 +13,7 @@ use rdkafka::message::{Header, OwnedHeaders};use rdkafka::producer::{FutureProducer, FutureRecord};use rdkafka::util::Timeout;use replay_core_v4::inspect::stats::BlockStats;-use replay_core_v4::keys::batch::{KeyStore, SessionId};+use replay_core_v4::keys::batch::{FlushKeys, KeyStore, SessionId};use replay_core_v4::keys::kms::AwsKms;use replay_core_v4::keys::rows::{key_month, write_day};use replay_core_v4::keys::table::{AwsKeyTable, CachedKeyTable};@@ -22,8 +23,8 @@ use crate::batch::Block;use crate::store::BlobStore;use crate::urls::{fetch_jobs, records};use crate::write::{- new_images, object_key, seal_block, seal_blocking, session_keyed_frames, write_block_index,- write_blocks, write_images, WriteError,+ compress_block, new_images, object_key, seal_blocking, seal_compressed, session_keyed_frames,+ write_block_index, write_blocks, write_images, CompressedBlock, WriteError,};/// How long a fetch job may wait for room in the producer queue.@@ -55,16 +56,43 @@ pub enum FlushError {Write(#[from] WriteError),}-/// Stores the blocks once. Up to `concurrency` blocks are sealed at once off the async runtime, and the packs, and-/// the fetch jobs, go out concurrently.-pub async fn write_flush(+/// What the attempts of one flush share: the keys, the write day they are for, and each block compressed. A retry then+/// only seals again, and a compression that an attempt timed out on goes on and serves the next attempt.+#[derive(Default)]+pub struct FlushWork {+ keys: tokio::sync::Mutex<Option<(String, Arc<FlushKeys>)>>,+ compressed: Mutex<HashMap<usize, Arc<Compressing>>>,+}++type Compressing = OnceLock<Result<Arc<CompressedBlock>, String>>;++impl FlushWork {+ /// The block compressed once. `block` names it by its address, which its `Arc` keeps. A second caller waits for+ /// the first one's result.+ fn compressed(+ &self,+ block: usize,+ compress: impl FnOnce() -> Result<CompressedBlock, WriteError>,+ ) -> Result<Arc<CompressedBlock>, WriteError> {+ let cell = self+ .compressed+ .lock()+ .unwrap_or_else(PoisonError::into_inner)+ .entry(block)+ .or_default()+ .clone();+ cell.get_or_init(|| compress().map(Arc::new).map_err(|e| e.to_string()))+ .clone()+ .map_err(WriteError::Seal)+ }+}++/// Resolves the keys of the flush's sessions. Keys are stored before anything is sealed with them, as in v3.+async fn resolve_keys(blocks: &[Arc<Block>],writer: &Writer,- concurrency: usize,-) -> Result<(), FlushError> {- let now = Utc::now();- let day = write_day(now);- let month = key_month(&day);+ day: &str,+) -> Result<FlushKeys, FlushError> {let sessions: Vec<SessionId> = blocks.iter().map(|b| SessionId {@@ -74,9 +102,31 @@ pub async fn write_flush(.collect::<BTreeSet<_>>().into_iter().collect();- // Keys are stored before anything is sealed with them, as in v3.- let keys = Arc::new(writer.keys.keys_for_flush(&sessions, &day).await?);+ let keys = writer.keys.keys_for_flush(&sessions, day).await?;counter!("ai_research_v4_recorder_dropped_sessions_total").increment(keys.dropped.len() as u64);+ Ok(keys)+}++/// Stores the blocks once. Up to `concurrency` blocks are sealed at once off the async runtime, and the packs, and+/// the fetch jobs, go out concurrently.+pub async fn write_flush(+ blocks: &[Arc<Block>],+ writer: &Writer,+ concurrency: usize,+ work: &Arc<FlushWork>,+) -> Result<(), FlushError> {+ let now = Utc::now();+ let mut resolved = work.keys.lock().await;+ let (day, keys) = match &*resolved {+ Some(done) => done.clone(),+ None => {+ let day = write_day(now);+ let keys = Arc::new(resolve_keys(blocks, writer, &day).await?);+ resolved.insert((day, keys)).clone()+ }+ };+ drop(resolved);+ let month = key_month(&day);let object_key = object_key(&day, now);let seals: Vec<_> = blocks@@ -89,9 +139,12 @@ pub async fn write_flush(};let key = keys.sessions.get(&id)?.clone();let team_key = *keys.team_months.get(&block.session.team_id)?;- let (block, month) = (block.clone(), month.to_owned());+ let (block, month, work) = (block.clone(), month.to_owned(), work.clone());Some(seal_blocking(&writer.seal_permits, move || {- seal_block(&block, &key, &team_key, &month).map(|sealed| (index, sealed))+ let compressed = work.compressed(Arc::as_ptr(&block) as usize, || {+ compress_block(&block, &key, &team_key, &month)+ })?;+ seal_compressed(&compressed).map(|sealed| (index, sealed))}))}).collect();@@ -122,7 +175,7 @@ pub async fn write_flush(});let (indices, mut sealed): (Vec<usize>, Vec<_>) = done.into_iter().unzip();- // The writer's columns go into a copy, so the shared blocks stay as the inspector left them.+ // This attempt's own copy: a retry fills the writer's columns again from zero.let mut stats: Vec<BlockStats> = blocks.iter().map(|b| b.stats.clone()).collect();for (&index, block) in indices.iter().zip(&mut sealed) {stats[index].images_extracted += block.counts.extracted;@@ -257,3 +310,17 @@ async fn produce_jobs(}metrics::gauge!("ai_research_v4_recorder_sent_urls").set(writer.sent_urls.entry_count() as f64);}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn a_retry_reuses_the_compressed_block() {+ let work = FlushWork::default();+ let compress = || Err(WriteError::Seal("once".to_owned()));+ assert!(work.compressed(1, compress).is_err());+ let again = work.compressed(1, || panic!("compressed a second time"));+ assert!(matches!(again, Err(WriteError::Seal(e)) if e == "sealing failed: once"));+ }+}diff --git a/rust/ai-research/replay-recorder-v4/src/main.rs b/rust/ai-research/replay-recorder-v4/src/main.rsindex b7d12d04be6d..eafc3fdcf7fe 100644--- a/rust/ai-research/replay-recorder-v4/src/main.rs+++ b/rust/ai-research/replay-recorder-v4/src/main.rs@@ -28,6 +28,7 @@ common_alloc::used!();#[tokio::main]async fn main() -> Result<(), Box<dyn std::error::Error>> {let config = Config::init_from_env().expect("invalid configuration");+ config.validate()?;tracing_subscriber::registry().with(fmt::layer().with_target(true).with_level(true)).with(@@ -39,12 +40,12 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {tracing::info!(topic = %config.kafka_topic, overflow_topic = %config.kafka_overflow_topic, group = %config.kafka_consumer_group, sample = config.sample_percentage, "starting replay-recorder-v4");let mut manager = Manager::builder("replay-recorder-v4")- .with_global_shutdown_timeout(Duration::from_secs(30))+ .with_global_shutdown_timeout(Duration::from_secs(config.shutdown_flush_secs + 15)).build();let consumer_handle = manager.register("consumer",ComponentOptions::new()- .with_graceful_shutdown(Duration::from_secs(15))+ .with_graceful_shutdown(Duration::from_secs(config.shutdown_flush_secs + 5)).with_liveness_deadline(Duration::from_secs(60)),);let restrictions_handle = manager.register(@@ -93,9 +94,6 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {.expect("metrics server");});- if config.team_refresh_secs == 0 {- return Err("AI_RESEARCH_V4_TEAM_REFRESH_SECS must be over 0".into());- }// The first team list must load before any message is read: an empty list would skip every message as unknown.let pool = get_pool_with_config(&config.database_url,@@ -158,6 +156,9 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {} else {config.flush_concurrency},+ flush_timeout: Duration::from_secs(config.flush_timeout_secs),+ flush_retries: config.flush_retries,+ shutdown_flush: Duration::from_secs(config.shutdown_flush_secs),session_idle: Duration::from_secs(config.session_idle_secs),redirecting_topic: (config.ingestion_overflow_mode == "redirect").then(|| config.kafka_topic.clone()),diff --git a/rust/ai-research/replay-recorder-v4/src/write.rs b/rust/ai-research/replay-recorder-v4/src/write.rsindex b981bddd63a9..4e811f3ce2b3 100644--- a/rust/ai-research/replay-recorder-v4/src/write.rs+++ b/rust/ai-research/replay-recorder-v4/src/write.rs@@ -74,12 +74,23 @@ pub struct WriterCounts {pub dropped: Option<&'static str>,}+/// A block compressed once for all the attempts of its flush, with the images it moved out.+pub struct CompressedBlock {+ pub session: SessionKey,+ pub key: Key,+ pub body: Vec<u8>,+ /// Shared, so that each attempt of a flush seals the images without a copy.+ pub images: Vec<Arc<Image>>,+ pub counts: WriterCounts,+ pub url_fingerprints: Vec<String>,+}+/// A block sealed for one object, with the images it moved out.pub struct SealedBlock {pub session: SessionKey,pub key_month: String,pub frame: Vec<u8>,- pub images: Vec<Image>,+ pub images: Vec<Arc<Image>>,pub counts: WriterCounts,pub url_fingerprints: Vec<String>,}@@ -192,6 +203,16 @@ pub fn seal_block(team_key: &[u8; 32],key_month: &str,) -> Result<SealedBlock, WriteError> {+ seal_compressed(&compress_block(block, key, team_key, key_month)?)+}++/// Builds and compresses one block. This is most of the work of a flush, so a retry does not do it again.+pub fn compress_block(+ block: &Block,+ key: &Key,+ team_key: &[u8; 32],+ key_month: &str,+) -> Result<CompressedBlock, WriteError> {let seal_error = |e: io::Error| WriteError::Seal(e.to_string());// Lines go into the encoder as they are built, so memory follows the compressed block and not its plaintext.// A brotli stream does not state its decompressed size, so a reader decompresses it as a stream.@@ -201,23 +222,35 @@ pub fn seal_block(let (images, counts) =prepare_block(block, team_key, key_month, &mut lines).map_err(seal_error)?;let encoder = lines.into_inner().map_err(|e| seal_error(e.into_error()))?;- let compressed = encoder+ let body = encoder.into_inner().map_err(|e| WriteError::Seal(e.to_string()))?;+ Ok(CompressedBlock {+ session: block.session.clone(),+ key: key.clone(),+ body,+ images: images.into_iter().map(Arc::new).collect(),+ counts,+ url_fingerprints: url_fingerprints(&key.key, &block_urls(block)),+ })+}++/// Seals a compressed block for one attempt.+pub fn seal_compressed(block: &CompressedBlock) -> Result<SealedBlock, WriteError> {let frame = seal_frame(- &key.key,+ &block.key.key,Version::V4,- &block_context(&block.session, &key.key_month),- &compressed,+ &block_context(&block.session, &block.key.key_month),+ &block.body,).map_err(|e| WriteError::Seal(e.to_string()))?;Ok(SealedBlock {session: block.session.clone(),- key_month: key.key_month.clone(),+ key_month: block.key.key_month.clone(),frame,- images,- counts,- url_fingerprints: url_fingerprints(&key.key, &block_urls(block)),+ images: block.images.clone(),+ counts: block.counts,+ url_fingerprints: block.url_fingerprints.clone(),})}@@ -250,8 +283,8 @@ pub fn session_keyed_frames(sealed: &[SealedBlock]) -> u64 {}/// The flush's new images by team, each once.-pub fn new_images(sealed: &mut [SealedBlock]) -> BTreeMap<i64, Vec<Image>> {- let mut images: BTreeMap<i64, Vec<Image>> = BTreeMap::new();+pub fn new_images(sealed: &mut [SealedBlock]) -> BTreeMap<i64, Vec<Arc<Image>>> {+ let mut images: BTreeMap<i64, Vec<Arc<Image>>> = BTreeMap::new();let mut seen: HashSet<String> = HashSet::new();for block in sealed {for image in std::mem::take(&mut block.images) {@@ -265,7 +298,7 @@ pub fn new_images(sealed: &mut [SealedBlock]) -> BTreeMap<i64, Vec<Image>> {fn seal_pack(team_id: i64,- team_images: &[Image],+ team_images: &[Arc<Image>],team_key: &[u8; 32],key_month: &str,pack_key: &str,@@ -303,7 +336,7 @@ fn seal_pack(/// image index for the flush.#[allow(clippy::too_many_arguments)]pub async fn write_images(- images: BTreeMap<i64, Vec<Image>>,+ images: BTreeMap<i64, Vec<Arc<Image>>>,keys: &FlushKeys,write_day: &str,key_month: &str,