Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(replay-recorder-v4): keep the pod live and safe through slow flushes (#118420)

Nicholas Waltz4cb1c737a70d
diff --git a/rust/ai-research/replay-recorder-v4/src/batch.rs b/rust/ai-research/replay-recorder-v4/src/batch.rs
index b572566e72b8..c081e87a1a0d 100644
--- a/rust/ai-research/replay-recorder-v4/src/batch.rs
+++ b/rust/ai-research/replay-recorder-v4/src/batch.rs
@@ -5,6 +5,7 @@ use std::time::{Duration, Instant};
use futures::stream::{self, StreamExt};
use metrics::counter;
+use replay_core_v4::compression::{GZIP_MAGIC, MAX_DECOMPRESSED_BYTES};
use replay_core_v4::inspect::dom::UrlLimit;
use replay_core_v4::inspect::session::{BlockInspection, Moment, SessionInspector};
use replay_core_v4::inspect::stats::BlockStats;
@@ -32,12 +33,39 @@ pub struct Raw {
pub content_encoding: Option<String>,
}
-/// One session's messages from one partition in the current flush. Capture keys messages by session, so a session
-/// has one partition per topic.
+impl Raw {
+ /// The bytes the payload decompresses to. Decompression allocates the size that the lz4 prefix or the gzip footer
+ /// states, or fails, so the flush size counts that instead of the compressed bytes.
+ pub fn decoded_len(&self) -> usize {
+ let stated = if self.content_encoding.as_deref() == Some("lz4") {
+ self.payload.get(..4)
+ } else if self.payload.starts_with(&GZIP_MAGIC) {
+ self.payload.get(self.payload.len().saturating_sub(4)..)
+ } else {
+ None
+ };
+ let stated = stated
+ .and_then(|b| <[u8; 4]>::try_from(b).ok())
+ .map(|b| u32::from_le_bytes(b) as usize)
+ .filter(|&n| n <= MAX_DECOMPRESSED_BYTES);
+ self.payload.len().max(stated.unwrap_or(0))
+ }
+}
+
+/// One session's messages from one partition in the current flush, read under one assignment of that partition.
pub struct Pending {
+ pub epoch: u64,
pub messages: Vec<Raw>,
}
+/// The offsets a flush handled on one partition, recorded or skipped alike.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct Span {
+ pub epoch: u64,
+ pub first: i64,
+ pub last: i64,
+}
+
/// What a flush hands to the writers.
pub struct Block {
pub session: SessionKey,
@@ -54,8 +82,7 @@ pub struct Block {
pub struct Batch {
pub sessions: BTreeMap<(SessionKey, Tp), Pending>,
- /// The last offset handled on each partition, recorded or skipped alike.
- pub offsets: HashMap<Tp, i64>,
+ pub spans: HashMap<Tp, Span>,
bytes: usize,
started: Instant,
}
@@ -64,7 +91,7 @@ impl Default for Batch {
fn default() -> Self {
Self {
sessions: BTreeMap::new(),
- offsets: HashMap::new(),
+ spans: HashMap::new(),
bytes: 0,
started: Instant::now(),
}
@@ -72,18 +99,34 @@ impl Default for Batch {
}
impl Batch {
- /// Marks an offset handled. A skipped message still moves its partition forward.
- pub fn handled(&mut self, tp: Tp, offset: i64) {
- let last = self.offsets.entry(tp).or_insert(offset);
- *last = (*last).max(offset);
+ /// Marks an offset handled. A skipped message still moves its partition forward. A message of a newer assignment
+ /// drops what an older one left, which Kafka delivers again from the last commit.
+ pub fn handled(&mut self, tp: Tp, epoch: u64, offset: i64) {
+ match self.spans.get_mut(&tp) {
+ Some(span) if span.epoch == epoch => span.last = span.last.max(offset),
+ stale => {
+ if stale.is_some() {
+ self.retain(|t, e| t != tp || e == epoch);
+ }
+ self.spans.insert(
+ tp,
+ Span {
+ epoch,
+ first: offset,
+ last: offset,
+ },
+ );
+ }
+ }
}
- pub fn add(&mut self, tp: Tp, session: SessionKey, raw: Raw) {
- self.handled(tp, raw.offset);
- self.bytes += raw.payload.len();
+ pub fn add(&mut self, tp: Tp, epoch: u64, session: SessionKey, raw: Raw) {
+ self.handled(tp, epoch, raw.offset);
+ self.bytes += raw.decoded_len();
self.sessions
.entry((session, tp))
.or_insert_with(|| Pending {
+ epoch,
messages: Vec::new(),
})
.messages
@@ -91,22 +134,33 @@ impl Batch {
}
pub fn should_flush(&self, interval: Duration, max_bytes: usize) -> bool {
- !self.offsets.is_empty() && (self.bytes >= max_bytes || self.started.elapsed() >= interval)
+ !self.spans.is_empty() && (self.bytes >= max_bytes || self.started.elapsed() >= interval)
}
- /// Forgets partitions this process no longer owns. Their messages are not committed, so the new owner reads them.
- pub fn drop_partitions(&mut self, revoked: &[Tp]) {
- self.sessions.retain(|(_, tp), _| !revoked.contains(tp));
- for tp in revoked {
- self.offsets.remove(tp);
- }
+ pub fn is_empty(&self) -> bool {
+ self.spans.is_empty()
+ }
+
+ /// Keeps what was read under an assignment that `keep` accepts. The rest is not committed, so it is read again.
+ pub fn retain(&mut self, keep: impl Fn(Tp, u64) -> bool) {
+ self.sessions
+ .retain(|(_, tp), pending| keep(*tp, pending.epoch));
+ self.spans.retain(|tp, span| keep(*tp, span.epoch));
self.bytes = self
.sessions
.values()
.flat_map(|p| &p.messages)
- .map(|m| m.payload.len())
+ .map(Raw::decoded_len)
.sum();
}
+
+ /// The next offset to commit on each partition.
+ pub fn offsets(&self) -> HashMap<Tp, i64> {
+ self.spans
+ .iter()
+ .map(|(tp, span)| (*tp, span.last + 1))
+ .collect()
+ }
}
/// Session inspectors by session. They keep DOM and click state across flushes.
@@ -118,11 +172,7 @@ pub struct Inspectors {
impl Inspectors {
/// Decodes every buffered message and runs it through its session's inspector, with up to `concurrency` sessions
/// at once off the async runtime. Returns one block per session.
- pub async fn inspect(
- &mut self,
- batch: Batch,
- concurrency: usize,
- ) -> (Vec<Block>, HashMap<Tp, i64>) {
+ pub async fn inspect(&mut self, batch: Batch, concurrency: usize) -> Vec<Block> {
let mut by_session: BTreeMap<SessionKey, (Vec<Tp>, Vec<Raw>)> = BTreeMap::new();
for ((session, tp), pending) in batch.sessions {
let (tps, raws) = by_session.entry(session).or_default();
@@ -184,7 +234,7 @@ impl Inspectors {
blocks.push(block);
}
blocks.sort_by(|a, b| a.session.cmp(&b.session));
- (blocks, batch.offsets)
+ blocks
}
pub fn drop_partitions(&mut self, revoked: &[Tp]) {
@@ -244,10 +294,18 @@ fn inspect_session(
#[cfg(test)]
mod tests {
+ use replay_core_v4::compression::gzip;
use serde_json::{json, Value};
use super::*;
+ fn tp(partition: i32) -> Tp {
+ Tp {
+ topic: "main",
+ partition,
+ }
+ }
+
fn raw(offset: i64, session: &str, items: Value) -> Raw {
let inner = json!({"event": "$snapshot_items", "properties": {"$session_id": session, "$window_id": "w", "$snapshot_source": "web", "$snapshot_items": items}});
Raw {
@@ -259,13 +317,6 @@ mod tests {
}
}
- fn tp(partition: i32) -> Tp {
- Tp {
- topic: "main",
- partition,
- }
- }
-
fn key(session: &str) -> SessionKey {
SessionKey {
team_id: 7,
@@ -273,6 +324,22 @@ mod tests {
}
}
+ #[test]
+ fn counts_a_compressed_payload_at_its_decoded_size() {
+ let plain = raw(
+ 1,
+ "a",
+ json!([{"type": 5, "timestamp": 1, "data": "a".repeat(100_000)}]),
+ );
+ let size = plain.payload.len();
+ let gzipped = Raw {
+ payload: gzip(&plain.payload).unwrap(),
+ ..plain
+ };
+ assert!(gzipped.payload.len() < size / 10);
+ assert_eq!(gzipped.decoded_len(), size);
+ }
+
fn snapshot() -> Value {
json!([{"type": 4, "timestamp": 1, "data": {"href": "https://example.com"}}, {"type": 2, "timestamp": 2, "data": {"node": {"type": 0, "childNodes": [], "id": 1}}}])
}
@@ -284,14 +351,15 @@ mod tests {
#[tokio::test]
async fn a_revoked_partition_leaves_nothing_behind() {
let mut batch = Batch::default();
- batch.add(tp(0), key("a"), raw(10, "a", mouse()));
- batch.add(tp(1), key("b"), raw(20, "b", mouse()));
- batch.handled(tp(1), 21);
+ batch.add(tp(0), 0, key("a"), raw(10, "a", mouse()));
+ batch.add(tp(1), 0, key("b"), raw(20, "b", mouse()));
+ batch.handled(tp(1), 0, 21);
let mut inspectors = Inspectors::default();
- batch.drop_partitions(&[tp(1)]);
+ batch.retain(|tp, _| tp.partition != 1);
inspectors.drop_partitions(&[tp(1)]);
- let (blocks, offsets) = inspectors.inspect(batch, 2).await;
+ assert_eq!(batch.offsets(), HashMap::from([(tp(0), 11)]));
+ let blocks = inspectors.inspect(batch, 2).await;
assert_eq!(
blocks
.iter()
@@ -299,7 +367,27 @@ mod tests {
.collect::<Vec<_>>(),
vec!["a"]
);
- assert_eq!(offsets, HashMap::from([(tp(0), 10)]));
+ }
+
+ #[test]
+ fn a_reassigned_partition_keeps_only_what_its_new_assignment_read() {
+ // The partition was revoked and assigned again inside one poll, so Kafka delivers offset 10 again.
+ let mut batch = Batch::default();
+ batch.add(tp(0), 0, key("a"), raw(10, "a", mouse()));
+ batch.add(tp(0), 0, key("a"), raw(11, "a", mouse()));
+ batch.add(tp(0), 1, key("a"), raw(10, "a", mouse()));
+ batch.retain(|_, epoch| epoch == 1);
+
+ let pending = &batch.sessions[&(key("a"), tp(0))];
+ assert_eq!(
+ pending
+ .messages
+ .iter()
+ .map(|m| m.offset)
+ .collect::<Vec<_>>(),
+ [10]
+ );
+ assert_eq!(batch.offsets(), HashMap::from([(tp(0), 11)]));
}
#[tokio::test]
@@ -307,6 +395,7 @@ mod tests {
let mut batch = Batch::default();
batch.add(
tp(0),
+ 0,
key("a"),
Raw {
offset: 1,
@@ -315,7 +404,7 @@ mod tests {
},
);
let mut inspectors = Inspectors::default();
- assert!(inspectors.inspect(batch, 1).await.0.is_empty());
+ assert!(inspectors.inspect(batch, 1).await.is_empty());
assert!(inspectors.sessions.is_empty());
}
@@ -323,14 +412,14 @@ mod tests {
async fn session_state_carries_across_flushes() {
let mut inspectors = Inspectors::default();
let mut first = Batch::default();
- first.add(tp(0), key("a"), raw(1, "a", snapshot()));
- first.add(tp(0), key("b"), raw(2, "b", mouse()));
+ first.add(tp(0), 0, key("a"), raw(1, "a", snapshot()));
+ first.add(tp(0), 0, key("b"), raw(2, "b", mouse()));
inspectors.inspect(first, 2).await;
let mut second = Batch::default();
- second.add(tp(0), key("a"), raw(3, "a", mouse()));
- second.add(tp(0), key("b"), raw(4, "b", mouse()));
- let (blocks, _) = inspectors.inspect(second, 2).await;
+ second.add(tp(0), 0, key("a"), raw(3, "a", mouse()));
+ second.add(tp(0), 0, key("b"), raw(4, "b", mouse()));
+ let blocks = inspectors.inspect(second, 2).await;
let cold: Vec<_> = blocks
.iter()
.map(|b| (b.session.session_id.as_str(), b.stats.cold_start))
diff --git a/rust/ai-research/replay-recorder-v4/src/config.rs b/rust/ai-research/replay-recorder-v4/src/config.rs
index 5a5183b36f93..7b8aa2ad33b4 100644
--- a/rust/ai-research/replay-recorder-v4/src/config.rs
+++ b/rust/ai-research/replay-recorder-v4/src/config.rs
@@ -1,3 +1,5 @@
+use std::time::Duration;
+
use common_kafka::config::KafkaConfig;
use envconfig::Envconfig;
@@ -77,6 +79,19 @@ pub struct Config {
#[envconfig(from = "AI_RESEARCH_V4_FLUSH_CONCURRENCY", default = "0")]
pub flush_concurrency: usize,
+ /// A flush attempt that takes longer fails. The consumer does not poll Kafka during a flush, so all attempts
+ /// together stay under the poll interval and the group keeps the partitions while they run.
+ #[envconfig(from = "AI_RESEARCH_V4_FLUSH_TIMEOUT_SECS", default = "60")]
+ pub flush_timeout_secs: u64,
+
+ /// How often a failed flush is tried again in-process before the recorder exits.
+ #[envconfig(from = "AI_RESEARCH_V4_FLUSH_RETRIES", default = "3")]
+ pub flush_retries: u32,
+
+ /// On shutdown the buffer is flushed if that takes at most this long. Otherwise nothing is committed.
+ #[envconfig(from = "AI_RESEARCH_V4_SHUTDOWN_FLUSH_SECS", default = "20")]
+ pub shutdown_flush_secs: u64,
+
/// A session's inspector is dropped after this long without a message, so memory follows live sessions.
#[envconfig(from = "AI_RESEARCH_V4_SESSION_IDLE_SECS", default = "3600")]
pub session_idle_secs: u64,
@@ -105,7 +120,7 @@ pub struct Config {
)]
pub image_fetch_topic: String,
- /// A fetch job not delivered in this time fails its flush.
+ /// A fetch job not delivered in this time fails its flush. It is well under the flush timeout.
#[envconfig(
from = "AI_RESEARCH_V4_IMAGE_FETCH_DELIVERY_TIMEOUT_MS",
default = "20000"
@@ -116,10 +131,30 @@ pub struct Config {
pub metrics_port: u16,
}
+/// The consumer's `max.poll.interval.ms`.
+pub const MAX_POLL_INTERVAL: Duration = Duration::from_secs(300);
+
+/// A whole flush, from the first decode to the commit, ends within this time, so it never outlives the poll interval.
+pub const FLUSH_DEADLINE: Duration = Duration::from_secs(MAX_POLL_INTERVAL.as_secs() - 10);
+
/// The slowest refill of a scaled new-session bucket: one session each 100 seconds.
const MIN_NEW_SESSION_REPLENISH_RATE: f64 = 0.01;
+/// The pause before retry `attempt` of a failed flush.
+pub fn retry_backoff(attempt: u32) -> Duration {
+ Duration::from_secs(1 << attempt.min(5))
+}
+
impl Config {
+ /// The longest a flush can store for, over all its attempts.
+ pub fn flush_budget(&self) -> Duration {
+ let attempts = Duration::from_secs(self.flush_timeout_secs) * (self.flush_retries + 1);
+ attempts
+ + (1..=self.flush_retries)
+ .map(retry_backoff)
+ .sum::<Duration>()
+ }
+
/// The bucket v4 runs, as `(capacity, replenish rate per second)`. v4 sees only its sample of the sessions, so both
/// scale by the sample, and a flood trips v4 at about the rate that trips the main lane.
pub fn new_session_bucket(&self) -> (f64, f64) {
@@ -139,6 +174,71 @@ impl Config {
),
)
}
+
+ /// Catches settings that would otherwise fail at the first flush and crash loop.
+ pub fn validate(&self) -> Result<(), String> {
+ if !(0.0..=100.0).contains(&self.sample_percentage) {
+ return Err(format!(
+ "AI_RESEARCH_V4_SAMPLE_PERCENTAGE must be from 0 to 100, not {}",
+ self.sample_percentage
+ ));
+ }
+ let mut required = vec![
+ ("KAFKA_TOPIC", &self.kafka_topic),
+ (
+ "EVENT_RESTRICTIONS_REDIS_URL",
+ &self.event_restrictions_redis_url,
+ ),
+ ("AI_RESEARCH_V4_REDIS_URL", &self.redis_url),
+ ];
+ if self.write_enabled {
+ required.extend([
+ ("AI_RESEARCH_V4_KEY_TABLE", &self.key_table),
+ ("AI_RESEARCH_V4_KMS_KEY_ARN", &self.kms_key_arn),
+ ("AI_RESEARCH_V4_RRWEB_BUCKET", &self.rrweb_bucket),
+ ("AI_RESEARCH_V4_IMAGES_BUCKET", &self.images_bucket),
+ ("AI_RESEARCH_V4_IMAGE_FETCH_TOPIC", &self.image_fetch_topic),
+ ]);
+ }
+ let missing: Vec<&str> = required
+ .into_iter()
+ .filter(|(_, value)| value.trim().is_empty())
+ .map(|(name, _)| name)
+ .collect();
+ if !missing.is_empty() {
+ return Err(format!("missing settings: {}", missing.join(", ")));
+ }
+ for (name, value) in [
+ (
+ "AI_RESEARCH_V4_NEW_SESSION_BUCKET_CAPACITY",
+ self.new_session_bucket_capacity,
+ ),
+ (
+ "AI_RESEARCH_V4_NEW_SESSION_BUCKET_REPLENISH_RATE",
+ self.new_session_bucket_replenish_rate,
+ ),
+ ] {
+ if !(value.is_finite() && value >= 0.0) {
+ return Err(format!("{name} must be 0 or more, not {value}"));
+ }
+ }
+ if self.team_refresh_secs == 0 {
+ return Err("AI_RESEARCH_V4_TEAM_REFRESH_SECS must be over 0".to_owned());
+ }
+ if self.flush_budget() >= FLUSH_DEADLINE {
+ return Err(format!(
+ "AI_RESEARCH_V4_FLUSH_TIMEOUT_SECS and AI_RESEARCH_V4_FLUSH_RETRIES allow a flush of {:?}, which is not under the Kafka poll interval",
+ self.flush_budget()
+ ));
+ }
+ if self.image_fetch_delivery_timeout_ms as u64 >= self.flush_timeout_secs * 1000 {
+ return Err(
+ "AI_RESEARCH_V4_IMAGE_FETCH_DELIVERY_TIMEOUT_MS must be under the flush timeout"
+ .to_owned(),
+ );
+ }
+ Ok(())
+ }
}
#[cfg(test)]
@@ -166,6 +266,37 @@ mod tests {
Config::init_from_hashmap(&env).unwrap()
}
+ #[test]
+ fn writes_need_every_resource_name_at_startup() {
+ assert_eq!(config(&[]).validate(), Ok(()));
+ let error = config(&[
+ ("AI_RESEARCH_V4_WRITE_ENABLED", "true"),
+ ("AI_RESEARCH_V4_KEY_TABLE", "keys"),
+ ("AI_RESEARCH_V4_RRWEB_BUCKET", "rrweb"),
+ ])
+ .validate()
+ .unwrap_err();
+ assert_eq!(
+ error,
+ "missing settings: AI_RESEARCH_V4_KMS_KEY_ARN, AI_RESEARCH_V4_IMAGES_BUCKET"
+ );
+ assert!(config(&[("AI_RESEARCH_V4_SAMPLE_PERCENTAGE", "110")])
+ .validate()
+ .is_err());
+ assert!(config(&[("AI_RESEARCH_V4_TEAM_REFRESH_SECS", "0")])
+ .validate()
+ .is_err());
+ assert!(
+ config(&[("AI_RESEARCH_V4_NEW_SESSION_BUCKET_CAPACITY", "-1")])
+ .validate()
+ .is_err()
+ );
+ assert_eq!(
+ config(&[("AI_RESEARCH_V4_REDIS_URL", " ")]).validate(),
+ Err("missing settings: AI_RESEARCH_V4_REDIS_URL".to_owned())
+ );
+ }
+
#[test]
fn scales_the_new_session_bucket_by_the_sample() {
let bucket = |sample: &str, capacity: &str, rate: &str| {
diff --git a/rust/ai-research/replay-recorder-v4/src/consumer.rs b/rust/ai-research/replay-recorder-v4/src/consumer.rs
index 6d754419009d..37c8bc1e1537 100644
--- a/rust/ai-research/replay-recorder-v4/src/consumer.rs
+++ b/rust/ai-research/replay-recorder-v4/src/consumer.rs
@@ -1,33 +1,82 @@
//! The consume loop: gate and admit each message from its key and headers, buffer it, and flush on time or size.
-use std::collections::{BTreeMap, HashSet};
+use std::collections::{BTreeMap, HashMap, HashSet};
+use std::future::Future;
use std::sync::{Arc, Mutex};
-use std::time::{Duration, SystemTime, UNIX_EPOCH};
+use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
use common_kafka::config::KafkaConfig;
use futures::FutureExt;
use lifecycle::Handle;
use metrics::counter;
use rdkafka::consumer::{
- BaseConsumer, CommitMode, Consumer, ConsumerContext, Rebalance, StreamConsumer,
+ BaseConsumer, CommitMode, Consumer, ConsumerContext, RebalanceProtocol, StreamConsumer,
};
-use rdkafka::error::KafkaError;
+use rdkafka::error::{KafkaError, RDKafkaErrorCode};
use rdkafka::message::{BorrowedMessage, Headers as _, Message as _};
+use rdkafka::types::RDKafkaRespErr;
use rdkafka::{ClientConfig, ClientContext, Offset, TopicPartitionList};
use crate::batch::{Batch, Block, Inspectors, Raw, SessionKey, Tp};
-use crate::flush::{write_flush, FlushError, Writer};
+use crate::config::{retry_backoff, FLUSH_DEADLINE, MAX_POLL_INTERVAL};
+use crate::flush::{write_flush, FlushWork, Writer};
use crate::limits::{Restrictions, SessionLimiter};
use crate::process::{admit, sampled, Admitted, Headers, Skip};
use crate::teams::TeamCache;
+const KAFKA_TIMEOUT: Duration = Duration::from_secs(10);
+const COMMIT_ATTEMPTS: u32 = 3;
+const HEARTBEAT: Duration = Duration::from_secs(5);
+
/// The most messages read before the limiter decides on their sessions.
const POLL_BATCH: usize = 1000;
-/// Records the partitions each rebalance takes away, so the loop can drop what it buffered for them.
+/// The assignment state the rebalance callbacks keep. A partition's epoch moves on every revoke, so a message read
+/// before the revoke is never mistaken for one read after a new assignment of the same partition.
+#[derive(Default)]
+struct Assignments {
+ epochs: HashMap<Tp, (u64, bool)>,
+ revoked: Vec<Tp>,
+ /// Partitions without a known committed offset, with the first offset read once there is one. It stays here until
+ /// it commits or the partition is revoked.
+ uncommitted: HashMap<Tp, Option<i64>>,
+ /// After a failed start commit, the next one waits until then.
+ start_retry_at: Option<Instant>,
+}
+
+/// The pause after a failed start commit, so a broker that does not answer does not stall every message.
+const START_RETRY: Duration = Duration::from_secs(5);
+
+impl Assignments {
+ /// The start offsets to commit now: all pending ones when `force`, otherwise none during the pause after a failure.
+ fn starts_due(&self, now: Instant, force: bool) -> Vec<(Tp, i64)> {
+ if !force && self.start_retry_at.is_some_and(|at| now < at) {
+ return Vec::new();
+ }
+ self.uncommitted
+ .iter()
+ .filter_map(|(&tp, start)| Some((tp, (*start)?)))
+ .collect()
+ }
+
+ fn starts_committed(&mut self, starts: &[(Tp, i64)], committed: bool, now: Instant) {
+ if committed {
+ for (tp, offset) in starts {
+ if self.uncommitted.get(tp) == Some(&Some(*offset)) {
+ self.uncommitted.remove(tp);
+ }
+ }
+ self.start_retry_at = None;
+ } else {
+ self.start_retry_at = Some(now + START_RETRY);
+ }
+ }
+}
+
pub struct RebalanceContext {
topics: Vec<&'static str>,
- revoked: Arc<Mutex<Vec<Tp>>>,
+ commit_latest: bool,
+ state: Arc<Mutex<Assignments>>,
}
impl RebalanceContext {
@@ -35,26 +84,139 @@ impl RebalanceContext {
let topic = *self.topics.iter().find(|t| **t == topic)?;
Some(Tp { topic, partition })
}
+
+ /// A new group starts at the head of the topic. Assigning that head and committing it before any fetch means a
+ /// partition revoked before its first flush sends its next owner to the same offset, past nothing unread.
+ fn start_at_heads(&self, consumer: &BaseConsumer<Self>, assigned: &mut TopicPartitionList) {
+ // One deadline for the whole assignment, so many slow partitions cannot hold the rebalance past liveness.
+ let deadline = Instant::now() + KAFKA_TIMEOUT;
+ let remaining = || deadline.saturating_duration_since(Instant::now());
+ let committed = match consumer.committed_offsets(assigned.clone(), remaining()) {
+ Ok(committed) => committed,
+ Err(error) => {
+ // Each partition commits its first offset read instead. Where an offset is committed already, the
+ // first message read is at that offset, so the commit does not move it.
+ tracing::warn!(%error, "reading committed offsets failed");
+ let assigned = assigned
+ .elements()
+ .iter()
+ .filter_map(|e| self.tp(e.topic(), e.partition()))
+ .collect::<Vec<_>>();
+ self.state
+ .lock()
+ .unwrap()
+ .uncommitted
+ .extend(assigned.into_iter().map(|tp| (tp, None)));
+ return;
+ }
+ };
+ let mut heads = TopicPartitionList::new();
+ let mut uncommitted = Vec::new();
+ for element in committed.elements() {
+ if element.offset() != Offset::Invalid {
+ continue;
+ }
+ let Some(tp) = self.tp(element.topic(), element.partition()) else {
+ continue;
+ };
+ let head = if self.commit_latest && !remaining().is_zero() {
+ consumer
+ .fetch_watermarks(tp.topic, tp.partition, remaining())
+ .map_err(|error| tracing::warn!(%error, "reading a partition head failed"))
+ .ok()
+ } else {
+ None
+ };
+ let started = head.is_some_and(|(_, high)| {
+ let offset = Offset::Offset(high);
+ assigned
+ .set_partition_offset(tp.topic, tp.partition, offset)
+ .and_then(|()| heads.add_partition_offset(tp.topic, tp.partition, offset))
+ .map_err(|error| tracing::warn!(%error, "building the starting offsets failed"))
+ .is_ok()
+ });
+ if !started {
+ uncommitted.push(tp);
+ }
+ }
+ if heads.count() > 0 {
+ if let Err(error) = consumer.commit(&heads, CommitMode::Sync) {
+ // The first message read commits its offset instead.
+ tracing::warn!(%error, "committing the starting offsets failed");
+ uncommitted.extend(
+ heads
+ .elements()
+ .iter()
+ .filter_map(|e| self.tp(e.topic(), e.partition())),
+ );
+ }
+ }
+ self.state
+ .lock()
+ .unwrap()
+ .uncommitted
+ .extend(uncommitted.into_iter().map(|tp| (tp, None)));
+ }
+
+ fn mark(&self, partitions: &TopicPartitionList, assigned: bool) {
+ let mut state = self.state.lock().unwrap();
+ for element in partitions.elements() {
+ if let Some(tp) = self.tp(element.topic(), element.partition()) {
+ if assigned {
+ state.epochs.entry(tp).or_default().1 = true;
+ } else {
+ let entry = state.epochs.entry(tp).or_default();
+ *entry = (entry.0 + 1, false);
+ state.revoked.push(tp);
+ state.uncommitted.remove(&tp);
+ }
+ }
+ }
+ }
}
impl ClientContext for RebalanceContext {}
impl ConsumerContext for RebalanceContext {
- fn pre_rebalance(&self, _: &BaseConsumer<Self>, rebalance: &Rebalance<'_>) {
- if let Rebalance::Revoke(partitions) = rebalance {
- let revoked = partitions
- .elements()
- .iter()
- .filter_map(|e| self.tp(e.topic(), e.partition()))
- .collect::<Vec<_>>();
- self.revoked.lock().unwrap().extend(revoked);
+ /// rdkafka's own rebalance, with the starting offsets set on the assignment before it is applied.
+ fn rebalance(
+ &self,
+ consumer: &BaseConsumer<Self>,
+ err: RDKafkaRespErr,
+ partitions: &mut TopicPartitionList,
+ ) {
+ let cooperative = matches!(
+ consumer.rebalance_protocol(),
+ RebalanceProtocol::Cooperative
+ );
+ let applied = if err == RDKafkaRespErr::RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS {
+ self.start_at_heads(consumer, partitions);
+ self.mark(partitions, true);
+ if cooperative {
+ consumer.incremental_assign(partitions)
+ } else {
+ consumer.assign(partitions)
+ }
+ } else {
+ if err != RDKafkaRespErr::RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS {
+ tracing::error!(error = %RDKafkaErrorCode::from(err), "rebalancing failed");
+ }
+ self.mark(partitions, false);
+ if cooperative {
+ consumer.incremental_unassign(partitions)
+ } else {
+ consumer.unassign()
+ }
+ };
+ if let Err(error) = applied {
+ tracing::error!(%error, "applying a rebalance failed");
}
}
}
pub struct RecorderConsumer {
- consumer: StreamConsumer<RebalanceContext>,
- revoked: Arc<Mutex<Vec<Tp>>>,
+ consumer: Arc<StreamConsumer<RebalanceContext>>,
+ state: Arc<Mutex<Assignments>>,
}
impl RecorderConsumer {
@@ -78,7 +240,12 @@ impl RecorderConsumer {
// Offsets move only after a flush has stored the blocks for them.
.set("enable.auto.commit", "false")
.set("enable.auto.offset.store", "false")
- .set("partition.assignment.strategy", "cooperative-sticky");
+ .set("partition.assignment.strategy", "cooperative-sticky")
+ // `Config::validate` keeps every flush under this, so the group keeps the partitions while one runs.
+ .set(
+ "max.poll.interval.ms",
+ MAX_POLL_INTERVAL.as_millis().to_string(),
+ );
if kafka.kafka_tls {
config
.set("security.protocol", "ssl")
@@ -87,14 +254,18 @@ impl RecorderConsumer {
if !kafka.kafka_client_rack.is_empty() {
config.set("client.rack", &kafka.kafka_client_rack);
}
- let revoked = Arc::new(Mutex::new(Vec::new()));
+ let state = Arc::new(Mutex::new(Assignments::default()));
let consumer: StreamConsumer<RebalanceContext> =
config.create_with_context(RebalanceContext {
topics: topics.clone(),
- revoked: revoked.clone(),
+ commit_latest: offset_reset == "latest",
+ state: state.clone(),
})?;
consumer.subscribe(&topics)?;
- Ok(Self { consumer, revoked })
+ Ok(Self {
+ consumer: Arc::new(consumer),
+ state,
+ })
}
/// The main topic comes first. The overflow lane's limiter only counts, so v4's does the same there.
@@ -103,29 +274,105 @@ impl RecorderConsumer {
}
fn take_revoked(&self) -> Vec<Tp> {
- std::mem::take(&mut *self.revoked.lock().unwrap())
+ std::mem::take(&mut self.state.lock().unwrap().revoked)
}
- /// Commits the next offset of every partition this process still owns.
- fn commit(&self, offsets: &std::collections::HashMap<Tp, i64>) {
- let assigned = self.consumer.assignment().unwrap_or_default();
+ /// The epoch of a partition this process owns now.
+ fn current_epoch(&self, tp: Tp) -> Option<u64> {
+ match self.state.lock().unwrap().epochs.get(&tp) {
+ Some(&(epoch, true)) => Some(epoch),
+ _ => None,
+ }
+ }
+
+ /// Commits the first offset read on a partition that had none, so a restart before the first flush starts where
+ /// this process started, and the flush's `start_offset` stays the same.
+ async fn commit_start(&self, tp: Tp, offset: i64) {
+ match self.state.lock().unwrap().uncommitted.get_mut(&tp) {
+ Some(start) => {
+ start.get_or_insert(offset);
+ }
+ None => return,
+ }
+ self.commit_starts(false).await;
+ }
+
+ /// Commits the start offsets that are still pending. A failed one is tried again on a later message, and with
+ /// `force` before a flush stores anything.
+ async fn commit_starts(&self, force: bool) {
+ let starts = self.state.lock().unwrap().starts_due(Instant::now(), force);
+ if starts.is_empty() {
+ return;
+ }
let mut list = TopicPartitionList::new();
- for (&tp, &offset) in offsets {
- if assigned.find_partition(tp.topic, tp.partition).is_some() {
- if let Err(error) =
- list.add_partition_offset(tp.topic, tp.partition, Offset::Offset(offset + 1))
- {
- tracing::warn!(%error, partition = tp.partition, "building the offset commit failed");
- }
+ let committed = match starts.iter().try_for_each(|(tp, offset)| {
+ list.add_partition_offset(tp.topic, tp.partition, Offset::Offset(*offset))
+ }) {
+ Ok(()) => self.commit_list(list).await,
+ Err(error) => Err(error.to_string()),
+ };
+ if let Err(error) = &committed {
+ tracing::warn!(%error, "committing the start offsets failed");
+ }
+ self.state
+ .lock()
+ .unwrap()
+ .starts_committed(&starts, committed.is_ok(), Instant::now());
+ }
+
+ /// Commits the next offset of every partition this process still owns under the same assignment.
+ async fn commit(&self, offsets: &HashMap<Tp, (u64, i64)>) -> Result<(), String> {
+ let mut list = TopicPartitionList::new();
+ for (&tp, &(epoch, offset)) in offsets {
+ if self.current_epoch(tp) == Some(epoch) {
+ list.add_partition_offset(tp.topic, tp.partition, Offset::Offset(offset))
+ .map_err(|e| e.to_string())?;
}
}
- if list.count() > 0 {
- if let Err(error) = self.consumer.commit(&list, CommitMode::Async) {
- tracing::warn!(%error, "committing offsets failed");
- counter!("ai_research_v4_recorder_commit_failures_total").increment(1);
+ if list.count() == 0 {
+ return Ok(());
+ }
+ let mut attempt = 0;
+ loop {
+ match self.commit_list(list.clone()).await {
+ Ok(()) => {
+ // A later offset committed, so a pending start offset is of no use any more.
+ let mut state = self.state.lock().unwrap();
+ for element in list.elements() {
+ if let Some(tp) = self
+ .consumer
+ .context()
+ .tp(element.topic(), element.partition())
+ {
+ state.uncommitted.remove(&tp);
+ }
+ }
+ return Ok(());
+ }
+ Err(error) => {
+ counter!("ai_research_v4_recorder_commit_failures_total").increment(1);
+ attempt += 1;
+ if attempt >= COMMIT_ATTEMPTS {
+ return Err(format!("committing offsets failed: {error}"));
+ }
+ tracing::warn!(%error, attempt, "committing offsets failed");
+ tokio::time::sleep(Duration::from_millis(500 * u64::from(attempt))).await;
+ }
}
}
}
+
+ /// A synchronous commit on a blocking thread, so the caller's deadlines and heartbeats keep running. A commit
+ /// that outlives `KAFKA_TIMEOUT` can still land later. Its offsets were already stored or read, so it skips nothing.
+ async fn commit_list(&self, list: TopicPartitionList) -> Result<(), String> {
+ let consumer = self.consumer.clone();
+ let commit = tokio::task::spawn_blocking(move || consumer.commit(&list, CommitMode::Sync));
+ match tokio::time::timeout(KAFKA_TIMEOUT, commit).await {
+ Ok(Ok(committed)) => committed.map_err(|e| e.to_string()),
+ Ok(Err(error)) => Err(error.to_string()),
+ Err(_) => Err(format!("timed out after {KAFKA_TIMEOUT:?}")),
+ }
+ }
}
fn header<'m>(message: &'m BorrowedMessage<'_>, name: &str) -> Option<&'m str> {
@@ -142,6 +389,9 @@ pub struct Settings {
pub flush_interval: Duration,
pub flush_max_bytes: usize,
pub flush_concurrency: usize,
+ pub flush_timeout: Duration,
+ pub flush_retries: u32,
+ pub shutdown_flush: Duration,
pub session_idle: Duration,
/// The main topic, when the main lane moves its force-overflow messages to the overflow topic.
pub redirecting_topic: Option<String>,
@@ -203,21 +453,61 @@ pub async fn run(
handle.report_healthy();
let revoked = consumer.take_revoked();
if !revoked.is_empty() {
- batch.drop_partitions(&revoked);
inspectors.drop_partitions(&revoked);
+ batch.retain(|tp, epoch| consumer.current_epoch(tp) == Some(epoch));
}
if batch.should_flush(settings.flush_interval, settings.flush_max_bytes) {
- let (blocks, offsets) = inspectors
- .inspect(std::mem::take(&mut batch), settings.flush_concurrency)
- .await;
- if let Err(error) = flush(blocks, writer.as_ref(), settings.flush_concurrency).await {
+ let work = flush(
+ std::mem::take(&mut batch),
+ &mut inspectors,
+ &consumer,
+ &settings,
+ writer.as_ref(),
+ &handle,
+ settings.flush_retries,
+ );
+ tokio::pin!(work);
+ let flushed = tokio::select! {
+ flushed = &mut work => flushed,
+ _ = &mut shutdown => {
+ // The running flush holds the whole buffer, so it goes on under the shutdown deadline.
+ flush_on_shutdown(settings.shutdown_flush, work).await;
+ return;
+ }
+ };
+ if let Err(error) = flushed {
// Nothing of this flush is committed, so a restart reads it again from the last commit.
tracing::error!(%error, "flush failed");
handle.signal_failure(format!("flush failed: {error}"));
- break;
+ return;
}
- consumer.commit(&offsets);
- inspectors.evict_idle(settings.session_idle);
+ }
+ }
+ if batch.is_empty() {
+ return;
+ }
+ flush_on_shutdown(
+ settings.shutdown_flush,
+ flush(
+ batch,
+ &mut inspectors,
+ &consumer,
+ &settings,
+ writer.as_ref(),
+ &handle,
+ 0,
+ ),
+ )
+ .await;
+}
+
+/// A flush cut off by the deadline before its commit leaves the buffer to the next owner, which reads it again.
+async fn flush_on_shutdown(deadline: Duration, work: impl Future<Output = Result<(), String>>) {
+ match tokio::time::timeout(deadline, work).await {
+ Ok(Ok(())) => tracing::info!("flushed the buffer on shutdown"),
+ Ok(Err(error)) => tracing::warn!(%error, "the flush on shutdown failed: nothing committed"),
+ Err(_) => {
+ tracing::warn!("the flush on shutdown did not fit the grace period: nothing committed")
}
}
}
@@ -251,17 +541,21 @@ async fn handle_messages(
else {
continue;
};
+ let Some(epoch) = consumer.current_epoch(tp) else {
+ continue;
+ };
+ consumer.commit_start(tp, message.offset()).await;
let session = admit_message(message, tp, settings, sources)
.await
.map(|admitted| SessionKey {
team_id: admitted.team_id,
session_id: admitted.session_id,
});
- admitted.push((message, tp, session));
+ admitted.push((message, tp, epoch, session));
}
// The limiter decides before anything is buffered, so a blocked session's messages never reach a block.
let mut sessions: BTreeMap<SessionKey, bool> = BTreeMap::new();
- for (_, tp, session) in &admitted {
+ for (_, tp, _, session) in &admitted {
if let Ok(session) = session {
*sessions.entry(session.clone()).or_default() |= consumer.is_main(*tp);
}
@@ -271,7 +565,7 @@ async fn handle_messages(
} else {
sources.limiter.blocked(&sessions).await
};
- for (message, tp, session) in admitted {
+ for (message, tp, epoch, session) in admitted {
let session = session.and_then(|session| {
if blocked.contains(&session) {
Err(Skip::Blocked)
@@ -288,12 +582,12 @@ async fn handle_messages(
payload: message.payload().unwrap_or_default().to_vec(),
content_encoding: header(message, "content-encoding").map(str::to_owned),
};
- batch.add(tp, session, raw);
+ batch.add(tp, epoch, session, raw);
}
Err(skip) => {
counter!("ai_research_v4_recorder_messages_total", "outcome" => skip.as_str())
.increment(1);
- batch.handled(tp, message.offset());
+ batch.handled(tp, epoch, message.offset());
}
}
}
@@ -323,14 +617,96 @@ async fn admit_message(
admit(&headers, restriction, redirect_overflow, team, now_ms)
}
-/// Stores a flush's blocks, or only reports them when the recorder runs dry.
+/// Reports the component healthy while `work` runs. `flush` bounds `work` with a deadline, so a hung flush still ends.
+async fn with_heartbeat<T>(handle: &Handle, work: impl Future<Output = T>) -> T {
+ tokio::pin!(work);
+ let mut beat = tokio::time::interval(HEARTBEAT);
+ loop {
+ tokio::select! {
+ out = &mut work => return out,
+ _ = beat.tick() => handle.report_healthy(),
+ }
+ }
+}
+
+/// Decodes and inspects the buffered sessions, stores them with up to `retries` more attempts, and only then commits.
+/// All of it ends within `FLUSH_DEADLINE`.
+#[allow(clippy::too_many_arguments)]
async fn flush(
- blocks: Vec<Block>,
+ batch: Batch,
+ inspectors: &mut Inspectors,
+ consumer: &RecorderConsumer,
+ settings: &Settings,
+ writer: Option<&Writer>,
+ handle: &Handle,
+ retries: u32,
+) -> Result<(), String> {
+ let work = flush_steps(
+ batch, inspectors, consumer, settings, writer, handle, retries,
+ );
+ tokio::time::timeout(FLUSH_DEADLINE, work)
+ .await
+ .unwrap_or_else(|_| Err(format!("the flush did not end in {FLUSH_DEADLINE:?}")))
+}
+
+#[allow(clippy::too_many_arguments)]
+async fn flush_steps(
+ batch: Batch,
+ inspectors: &mut Inspectors,
+ consumer: &RecorderConsumer,
+ settings: &Settings,
writer: Option<&Writer>,
- concurrency: usize,
-) -> Result<(), FlushError> {
+ handle: &Handle,
+ retries: u32,
+) -> Result<(), String> {
+ let offsets: HashMap<Tp, (u64, i64)> = batch
+ .spans
+ .iter()
+ .map(|(tp, span)| (*tp, (span.epoch, span.last + 1)))
+ .collect();
+ // Before anything is stored, so a restart after a failed flush reads it again from where this process started.
+ with_heartbeat(handle, consumer.commit_starts(true)).await;
+ let blocks = with_heartbeat(
+ handle,
+ inspectors.inspect(batch, settings.flush_concurrency),
+ )
+ .await;
+ report(&blocks);
+ if let Some(writer) = writer {
+ let blocks: Vec<Arc<Block>> = blocks.into_iter().map(Arc::new).collect();
+ let work = Arc::new(FlushWork::default());
+ let mut attempt = 0;
+ loop {
+ let stored = tokio::time::timeout(
+ settings.flush_timeout,
+ with_heartbeat(
+ handle,
+ write_flush(&blocks, writer, settings.flush_concurrency, &work),
+ ),
+ )
+ .await;
+ let error = match stored {
+ Ok(Ok(())) => break,
+ Ok(Err(error)) => error.to_string(),
+ Err(_) => format!("timed out after {:?}", settings.flush_timeout),
+ };
+ counter!("ai_research_v4_recorder_flush_failures_total").increment(1);
+ attempt += 1;
+ if attempt > retries {
+ return Err(error);
+ }
+ tracing::warn!(%error, attempt, "flush attempt failed, trying again");
+ with_heartbeat(handle, tokio::time::sleep(retry_backoff(attempt))).await;
+ }
+ }
+ with_heartbeat(handle, consumer.commit(&offsets)).await?;
+ inspectors.evict_idle(settings.session_idle);
+ Ok(())
+}
+
+fn report(blocks: &[Block]) {
counter!("ai_research_v4_recorder_blocks_total").increment(blocks.len() as u64);
- for block in &blocks {
+ for block in blocks {
counter!("ai_research_v4_recorder_block_events_total").increment(block.stats.event_count);
if block.stats.cold_start {
counter!("ai_research_v4_recorder_cold_start_blocks_total").increment(1);
@@ -339,7 +715,36 @@ async fn flush(
counter!("ai_research_v4_recorder_orphan_mutation_blocks_total").increment(1);
}
}
- let Some(writer) = writer else { return Ok(()) };
- let blocks: Vec<Arc<Block>> = blocks.into_iter().map(Arc::new).collect();
- write_flush(&blocks, writer, concurrency).await
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn tp(partition: i32) -> Tp {
+ Tp {
+ topic: "main",
+ partition,
+ }
+ }
+
+ #[test]
+ fn a_failed_start_commit_stays_pending_at_the_first_offset() {
+ let mut state = Assignments::default();
+ state.uncommitted.insert(tp(0), None);
+ state.uncommitted.get_mut(&tp(0)).unwrap().get_or_insert(10);
+ let now = Instant::now();
+ let starts = state.starts_due(now, false);
+ assert_eq!(starts, [(tp(0), 10)]);
+ state.starts_committed(&starts, false, now);
+
+ // A later message does not move the start, and waits for the pause unless the flush forces it.
+ state.uncommitted.get_mut(&tp(0)).unwrap().get_or_insert(11);
+ assert!(state.starts_due(now, false).is_empty());
+ assert_eq!(state.starts_due(now, true), [(tp(0), 10)]);
+ let starts = state.starts_due(now + START_RETRY, false);
+ assert_eq!(starts, [(tp(0), 10)]);
+ state.starts_committed(&starts, true, now);
+ assert!(state.uncommitted.is_empty());
+ }
}
diff --git a/rust/ai-research/replay-recorder-v4/src/flush.rs b/rust/ai-research/replay-recorder-v4/src/flush.rs
index ee92ed2b76f2..8fc9aa768c48 100644
--- a/rust/ai-research/replay-recorder-v4/src/flush.rs
+++ b/rust/ai-research/replay-recorder-v4/src/flush.rs
@@ -1,7 +1,8 @@
-//! Stores a flush's blocks: their keys, the sealed blocks, the image packs, both indexes and the fetch jobs.
+//! One attempt to store a flush's blocks. It writes nothing that a retry cannot write again: object names are
+//! unique.
-use std::collections::{BTreeMap, BTreeSet};
-use std::sync::Arc;
+use std::collections::{BTreeMap, BTreeSet, HashMap};
+use std::sync::{Arc, Mutex, OnceLock, PoisonError};
use std::time::Duration;
use chrono::Utc;
@@ -12,7 +13,7 @@ use rdkafka::message::{Header, OwnedHeaders};
use rdkafka::producer::{FutureProducer, FutureRecord};
use rdkafka::util::Timeout;
use replay_core_v4::inspect::stats::BlockStats;
-use replay_core_v4::keys::batch::{KeyStore, SessionId};
+use replay_core_v4::keys::batch::{FlushKeys, KeyStore, SessionId};
use replay_core_v4::keys::kms::AwsKms;
use replay_core_v4::keys::rows::{key_month, write_day};
use replay_core_v4::keys::table::{AwsKeyTable, CachedKeyTable};
@@ -22,8 +23,8 @@ use crate::batch::Block;
use crate::store::BlobStore;
use crate::urls::{fetch_jobs, records};
use crate::write::{
- new_images, object_key, seal_block, seal_blocking, session_keyed_frames, write_block_index,
- write_blocks, write_images, WriteError,
+ compress_block, new_images, object_key, seal_blocking, seal_compressed, session_keyed_frames,
+ write_block_index, write_blocks, write_images, CompressedBlock, WriteError,
};
/// How long a fetch job may wait for room in the producer queue.
@@ -55,16 +56,43 @@ pub enum FlushError {
Write(#[from] WriteError),
}
-/// Stores the blocks once. Up to `concurrency` blocks are sealed at once off the async runtime, and the packs, and
-/// the fetch jobs, go out concurrently.
-pub async fn write_flush(
+/// What the attempts of one flush share: the keys, the write day they are for, and each block compressed. A retry then
+/// only seals again, and a compression that an attempt timed out on goes on and serves the next attempt.
+#[derive(Default)]
+pub struct FlushWork {
+ keys: tokio::sync::Mutex<Option<(String, Arc<FlushKeys>)>>,
+ compressed: Mutex<HashMap<usize, Arc<Compressing>>>,
+}
+
+type Compressing = OnceLock<Result<Arc<CompressedBlock>, String>>;
+
+impl FlushWork {
+ /// The block compressed once. `block` names it by its address, which its `Arc` keeps. A second caller waits for
+ /// the first one's result.
+ fn compressed(
+ &self,
+ block: usize,
+ compress: impl FnOnce() -> Result<CompressedBlock, WriteError>,
+ ) -> Result<Arc<CompressedBlock>, WriteError> {
+ let cell = self
+ .compressed
+ .lock()
+ .unwrap_or_else(PoisonError::into_inner)
+ .entry(block)
+ .or_default()
+ .clone();
+ cell.get_or_init(|| compress().map(Arc::new).map_err(|e| e.to_string()))
+ .clone()
+ .map_err(WriteError::Seal)
+ }
+}
+
+/// Resolves the keys of the flush's sessions. Keys are stored before anything is sealed with them, as in v3.
+async fn resolve_keys(
blocks: &[Arc<Block>],
writer: &Writer,
- concurrency: usize,
-) -> Result<(), FlushError> {
- let now = Utc::now();
- let day = write_day(now);
- let month = key_month(&day);
+ day: &str,
+) -> Result<FlushKeys, FlushError> {
let sessions: Vec<SessionId> = blocks
.iter()
.map(|b| SessionId {
@@ -74,9 +102,31 @@ pub async fn write_flush(
.collect::<BTreeSet<_>>()
.into_iter()
.collect();
- // Keys are stored before anything is sealed with them, as in v3.
- let keys = Arc::new(writer.keys.keys_for_flush(&sessions, &day).await?);
+ let keys = writer.keys.keys_for_flush(&sessions, day).await?;
counter!("ai_research_v4_recorder_dropped_sessions_total").increment(keys.dropped.len() as u64);
+ Ok(keys)
+}
+
+/// Stores the blocks once. Up to `concurrency` blocks are sealed at once off the async runtime, and the packs, and
+/// the fetch jobs, go out concurrently.
+pub async fn write_flush(
+ blocks: &[Arc<Block>],
+ writer: &Writer,
+ concurrency: usize,
+ work: &Arc<FlushWork>,
+) -> Result<(), FlushError> {
+ let now = Utc::now();
+ let mut resolved = work.keys.lock().await;
+ let (day, keys) = match &*resolved {
+ Some(done) => done.clone(),
+ None => {
+ let day = write_day(now);
+ let keys = Arc::new(resolve_keys(blocks, writer, &day).await?);
+ resolved.insert((day, keys)).clone()
+ }
+ };
+ drop(resolved);
+ let month = key_month(&day);
let object_key = object_key(&day, now);
let seals: Vec<_> = blocks
@@ -89,9 +139,12 @@ pub async fn write_flush(
};
let key = keys.sessions.get(&id)?.clone();
let team_key = *keys.team_months.get(&block.session.team_id)?;
- let (block, month) = (block.clone(), month.to_owned());
+ let (block, month, work) = (block.clone(), month.to_owned(), work.clone());
Some(seal_blocking(&writer.seal_permits, move || {
- seal_block(&block, &key, &team_key, &month).map(|sealed| (index, sealed))
+ let compressed = work.compressed(Arc::as_ptr(&block) as usize, || {
+ compress_block(&block, &key, &team_key, &month)
+ })?;
+ seal_compressed(&compressed).map(|sealed| (index, sealed))
}))
})
.collect();
@@ -122,7 +175,7 @@ pub async fn write_flush(
});
let (indices, mut sealed): (Vec<usize>, Vec<_>) = done.into_iter().unzip();
- // The writer's columns go into a copy, so the shared blocks stay as the inspector left them.
+ // This attempt's own copy: a retry fills the writer's columns again from zero.
let mut stats: Vec<BlockStats> = blocks.iter().map(|b| b.stats.clone()).collect();
for (&index, block) in indices.iter().zip(&mut sealed) {
stats[index].images_extracted += block.counts.extracted;
@@ -257,3 +310,17 @@ async fn produce_jobs(
}
metrics::gauge!("ai_research_v4_recorder_sent_urls").set(writer.sent_urls.entry_count() as f64);
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn a_retry_reuses_the_compressed_block() {
+ let work = FlushWork::default();
+ let compress = || Err(WriteError::Seal("once".to_owned()));
+ assert!(work.compressed(1, compress).is_err());
+ let again = work.compressed(1, || panic!("compressed a second time"));
+ assert!(matches!(again, Err(WriteError::Seal(e)) if e == "sealing failed: once"));
+ }
+}
diff --git a/rust/ai-research/replay-recorder-v4/src/main.rs b/rust/ai-research/replay-recorder-v4/src/main.rs
index b7d12d04be6d..eafc3fdcf7fe 100644
--- a/rust/ai-research/replay-recorder-v4/src/main.rs
+++ b/rust/ai-research/replay-recorder-v4/src/main.rs
@@ -28,6 +28,7 @@ common_alloc::used!();
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
let config = Config::init_from_env().expect("invalid configuration");
+ config.validate()?;
tracing_subscriber::registry()
.with(fmt::layer().with_target(true).with_level(true))
.with(
@@ -39,12 +40,12 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
tracing::info!(topic = %config.kafka_topic, overflow_topic = %config.kafka_overflow_topic, group = %config.kafka_consumer_group, sample = config.sample_percentage, "starting replay-recorder-v4");
let mut manager = Manager::builder("replay-recorder-v4")
- .with_global_shutdown_timeout(Duration::from_secs(30))
+ .with_global_shutdown_timeout(Duration::from_secs(config.shutdown_flush_secs + 15))
.build();
let consumer_handle = manager.register(
"consumer",
ComponentOptions::new()
- .with_graceful_shutdown(Duration::from_secs(15))
+ .with_graceful_shutdown(Duration::from_secs(config.shutdown_flush_secs + 5))
.with_liveness_deadline(Duration::from_secs(60)),
);
let restrictions_handle = manager.register(
@@ -93,9 +94,6 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
.expect("metrics server");
});
- if config.team_refresh_secs == 0 {
- return Err("AI_RESEARCH_V4_TEAM_REFRESH_SECS must be over 0".into());
- }
// The first team list must load before any message is read: an empty list would skip every message as unknown.
let pool = get_pool_with_config(
&config.database_url,
@@ -158,6 +156,9 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
} else {
config.flush_concurrency
},
+ flush_timeout: Duration::from_secs(config.flush_timeout_secs),
+ flush_retries: config.flush_retries,
+ shutdown_flush: Duration::from_secs(config.shutdown_flush_secs),
session_idle: Duration::from_secs(config.session_idle_secs),
redirecting_topic: (config.ingestion_overflow_mode == "redirect")
.then(|| config.kafka_topic.clone()),
diff --git a/rust/ai-research/replay-recorder-v4/src/write.rs b/rust/ai-research/replay-recorder-v4/src/write.rs
index b981bddd63a9..4e811f3ce2b3 100644
--- a/rust/ai-research/replay-recorder-v4/src/write.rs
+++ b/rust/ai-research/replay-recorder-v4/src/write.rs
@@ -74,12 +74,23 @@ pub struct WriterCounts {
pub dropped: Option<&'static str>,
}
+/// A block compressed once for all the attempts of its flush, with the images it moved out.
+pub struct CompressedBlock {
+ pub session: SessionKey,
+ pub key: Key,
+ pub body: Vec<u8>,
+ /// Shared, so that each attempt of a flush seals the images without a copy.
+ pub images: Vec<Arc<Image>>,
+ pub counts: WriterCounts,
+ pub url_fingerprints: Vec<String>,
+}
+
/// A block sealed for one object, with the images it moved out.
pub struct SealedBlock {
pub session: SessionKey,
pub key_month: String,
pub frame: Vec<u8>,
- pub images: Vec<Image>,
+ pub images: Vec<Arc<Image>>,
pub counts: WriterCounts,
pub url_fingerprints: Vec<String>,
}
@@ -192,6 +203,16 @@ pub fn seal_block(
team_key: &[u8; 32],
key_month: &str,
) -> Result<SealedBlock, WriteError> {
+ seal_compressed(&compress_block(block, key, team_key, key_month)?)
+}
+
+/// Builds and compresses one block. This is most of the work of a flush, so a retry does not do it again.
+pub fn compress_block(
+ block: &Block,
+ key: &Key,
+ team_key: &[u8; 32],
+ key_month: &str,
+) -> Result<CompressedBlock, WriteError> {
let seal_error = |e: io::Error| WriteError::Seal(e.to_string());
// Lines go into the encoder as they are built, so memory follows the compressed block and not its plaintext.
// A brotli stream does not state its decompressed size, so a reader decompresses it as a stream.
@@ -201,23 +222,35 @@ pub fn seal_block(
let (images, counts) =
prepare_block(block, team_key, key_month, &mut lines).map_err(seal_error)?;
let encoder = lines.into_inner().map_err(|e| seal_error(e.into_error()))?;
- let compressed = encoder
+ let body = encoder
.into_inner()
.map_err(|e| WriteError::Seal(e.to_string()))?;
+ Ok(CompressedBlock {
+ session: block.session.clone(),
+ key: key.clone(),
+ body,
+ images: images.into_iter().map(Arc::new).collect(),
+ counts,
+ url_fingerprints: url_fingerprints(&key.key, &block_urls(block)),
+ })
+}
+
+/// Seals a compressed block for one attempt.
+pub fn seal_compressed(block: &CompressedBlock) -> Result<SealedBlock, WriteError> {
let frame = seal_frame(
- &key.key,
+ &block.key.key,
Version::V4,
- &block_context(&block.session, &key.key_month),
- &compressed,
+ &block_context(&block.session, &block.key.key_month),
+ &block.body,
)
.map_err(|e| WriteError::Seal(e.to_string()))?;
Ok(SealedBlock {
session: block.session.clone(),
- key_month: key.key_month.clone(),
+ key_month: block.key.key_month.clone(),
frame,
- images,
- counts,
- url_fingerprints: url_fingerprints(&key.key, &block_urls(block)),
+ images: block.images.clone(),
+ counts: block.counts,
+ url_fingerprints: block.url_fingerprints.clone(),
})
}
@@ -250,8 +283,8 @@ pub fn session_keyed_frames(sealed: &[SealedBlock]) -> u64 {
}
/// The flush's new images by team, each once.
-pub fn new_images(sealed: &mut [SealedBlock]) -> BTreeMap<i64, Vec<Image>> {
- let mut images: BTreeMap<i64, Vec<Image>> = BTreeMap::new();
+pub fn new_images(sealed: &mut [SealedBlock]) -> BTreeMap<i64, Vec<Arc<Image>>> {
+ let mut images: BTreeMap<i64, Vec<Arc<Image>>> = BTreeMap::new();
let mut seen: HashSet<String> = HashSet::new();
for block in sealed {
for image in std::mem::take(&mut block.images) {
@@ -265,7 +298,7 @@ pub fn new_images(sealed: &mut [SealedBlock]) -> BTreeMap<i64, Vec<Image>> {
fn seal_pack(
team_id: i64,
- team_images: &[Image],
+ team_images: &[Arc<Image>],
team_key: &[u8; 32],
key_month: &str,
pack_key: &str,
@@ -303,7 +336,7 @@ fn seal_pack(
/// image index for the flush.
#[allow(clippy::too_many_arguments)]
pub async fn write_images(
- images: BTreeMap<i64, Vec<Image>>,
+ images: BTreeMap<i64, Vec<Arc<Image>>>,
keys: &FlushKeys,
write_day: &str,
key_month: &str,