Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(replay-recorder-v4): keep running after a failed offset commit (#127550)

Nicholas Waltzb8095bf610df
diff --git a/products/ai_training/docs/replay-data.md b/products/ai_training/docs/replay-data.md
index 5fe09ecb6cb5..71a1b54746d7 100644
--- a/products/ai_training/docs/replay-data.md
+++ b/products/ai_training/docs/replay-data.md
@@ -522,7 +522,8 @@ The recorder does not write the two index prefixes itself, as in v3:
### Block index
A flush can write its blocks and index rows and then fail before it commits its Kafka offsets.
-The next attempt for that partition starts at the same committed offset and writes the same messages again.
+The recorder keeps running and tries to commit those offsets again later, and it exits only when commits keep failing for 10 minutes.
+When the partition moves first, or the process restarts, the next attempt for that partition starts at the same committed offset and writes the same messages again.
A retry can also cover a different range than the attempt it replaces, for example one flush that replays two earlier flushes.
Every row therefore names its flush and its Kafka position.
The index rows are published without waiting for delivery, so a retry can also reach the index only in part.
diff --git a/rust/ai-research/replay-recorder-v4/src/consumer.rs b/rust/ai-research/replay-recorder-v4/src/consumer.rs
index 761bbad04b94..406220ae3557 100644
--- a/rust/ai-research/replay-recorder-v4/src/consumer.rs
+++ b/rust/ai-research/replay-recorder-v4/src/consumer.rs
@@ -25,9 +25,12 @@ use crate::process::{admit, sampled, Admitted, Headers, Skip};
use crate::teams::TeamCache;
const KAFKA_TIMEOUT: Duration = Duration::from_secs(10);
-const COMMIT_ATTEMPTS: u32 = 3;
const HEARTBEAT: Duration = Duration::from_secs(5);
+/// How long offset commits can keep failing before the process exits. A rebalance can wait for one member for the whole
+/// poll interval, so this limit outlasts a rebalance. It also limits what a restart reads again.
+const COMMIT_GIVE_UP: Duration = Duration::from_secs(2 * MAX_POLL_INTERVAL.as_secs());
+
/// The consumer's `session.timeout.ms`, the replay lanes' value. Past it the group can give the partitions to another
/// process, so a flush stops storing then.
pub const SESSION_TIMEOUT: Duration = Duration::from_secs(90);
@@ -46,6 +49,10 @@ struct Assignments {
uncommitted: HashMap<Tp, Option<i64>>,
/// After a failed start commit, the next one waits until then.
start_retry_at: Option<Instant>,
+ /// The next offset and epoch of each partition with stored blocks that are not committed.
+ unsettled: HashMap<Tp, (u64, i64)>,
+ /// When the offset commits started to fail, and when the last one failed.
+ commit_failures: Option<(Instant, Instant)>,
}
/// The pause after a failed start commit, so a broker that does not answer does not stall every message.
@@ -75,6 +82,39 @@ impl Assignments {
self.start_retry_at = Some(now + START_RETRY);
}
}
+
+ /// A partition that was revoked is the next owner's, even when it came back under a later epoch.
+ fn offsets_due(&mut self, flushed: &HashMap<Tp, (u64, i64)>) -> Vec<(Tp, i64)> {
+ self.unsettled.extend(flushed);
+ let epochs = &self.epochs;
+ self.unsettled.retain(
+ |tp, (epoch, _)| matches!(epochs.get(tp), Some(&(current, true)) if current == *epoch),
+ );
+ self.unsettled
+ .iter()
+ .map(|(&tp, &(_, offset))| (tp, offset))
+ .collect()
+ }
+
+ fn offsets_committed(&mut self) {
+ // A later offset committed, so a pending start offset is of no use any more.
+ for tp in self.unsettled.keys() {
+ self.uncommitted.remove(tp);
+ }
+ self.unsettled.clear();
+ self.commit_failures = None;
+ }
+
+ fn commit_failed(&mut self, now: Instant) -> Duration {
+ let (since, last) = self.commit_failures.get_or_insert((now, now));
+ *last = now;
+ now.saturating_duration_since(*since)
+ }
+
+ fn commit_retry_due(&self, now: Instant, interval: Duration) -> bool {
+ self.commit_failures
+ .is_some_and(|(_, last)| now.saturating_duration_since(last) >= interval)
+ }
}
pub struct RebalanceContext {
@@ -328,48 +368,55 @@ impl RecorderConsumer {
.starts_committed(&starts, committed.is_ok(), Instant::now());
}
- /// Commits the next offset of every partition this process still owns under the same assignment.
- async fn commit(&self, offsets: &HashMap<Tp, (u64, i64)>) -> Result<(), String> {
- let mut list = TopicPartitionList::new();
- for (&tp, &(epoch, offset)) in offsets {
- if self.current_epoch(tp) == Some(epoch) {
- list.add_partition_offset(tp.topic, tp.partition, Offset::Offset(offset))
- .map_err(|e| e.to_string())?;
+ /// Commits the next offset of every partition this process still owns under the same assignment, with the offsets
+ /// of earlier flushes that did not commit. A failed commit is not retried here, because the rebalance callbacks run
+ /// only when the consume loop polls, so a rebalance cannot end while this waits. The blocks are stored, so until a
+ /// later commit a failure means only that a new owner or a restart reads them again.
+ async fn commit(&self, flushed: &HashMap<Tp, (u64, i64)>) -> Result<(), String> {
+ let due = {
+ let mut state = self.state.lock().unwrap();
+ let due = state.offsets_due(flushed);
+ if due.is_empty() {
+ state.commit_failures = None;
+ return Ok(());
}
+ due
+ };
+ let mut list = TopicPartitionList::new();
+ for (tp, offset) in due {
+ list.add_partition_offset(tp.topic, tp.partition, Offset::Offset(offset))
+ .map_err(|e| e.to_string())?;
}
- if list.count() == 0 {
- return Ok(());
- }
- let mut attempt = 0;
- loop {
- match self.commit_list(list.clone()).await {
- Ok(()) => {
- // A later offset committed, so a pending start offset is of no use any more.
- let mut state = self.state.lock().unwrap();
- for element in list.elements() {
- if let Some(tp) = self
- .consumer
- .context()
- .tp(element.topic(), element.partition())
- {
- state.uncommitted.remove(&tp);
- }
- }
- return Ok(());
- }
- Err(error) => {
- counter!("ai_research_v4_recorder_commit_failures_total").increment(1);
- attempt += 1;
- if attempt >= COMMIT_ATTEMPTS {
- return Err(format!("committing offsets failed: {error}"));
- }
- tracing::warn!(%error, attempt, "committing offsets failed");
- tokio::time::sleep(Duration::from_millis(500 * u64::from(attempt))).await;
+ match self.commit_list(list).await {
+ Ok(()) => {
+ self.state.lock().unwrap().offsets_committed();
+ Ok(())
+ }
+ Err(error) => {
+ counter!("ai_research_v4_recorder_commit_failures_total").increment(1);
+ let failing = self.state.lock().unwrap().commit_failed(Instant::now());
+ if failing >= COMMIT_GIVE_UP {
+ return Err(format!(
+ "committing offsets failed for {failing:?}: {error}"
+ ));
}
+ tracing::warn!(%error, ?failing, "committing offsets failed, trying again later");
+ Ok(())
}
}
}
+ async fn retry_commit(&self) -> Result<(), String> {
+ self.commit(&HashMap::new()).await
+ }
+
+ fn commit_retry_due(&self, interval: Duration) -> bool {
+ self.state
+ .lock()
+ .unwrap()
+ .commit_retry_due(Instant::now(), interval)
+ }
+
/// A synchronous commit on a blocking thread, so the caller's deadlines and heartbeats keep running. A commit
/// that outlives `KAFKA_TIMEOUT` can still land later. Its offsets were already stored or read, so it skips nothing.
async fn commit_list(&self, list: TopicPartitionList) -> Result<(), String> {
@@ -526,9 +573,21 @@ async fn consume(
handle.signal_failure(format!("flush failed: {error}"));
return;
}
+ } else if batch.is_empty() && consumer.commit_retry_due(settings.flush_interval) {
+ // Without new messages, no flush comes to commit these offsets.
+ if let Err(error) = with_heartbeat(handle, consumer.retry_commit()).await {
+ tracing::error!(%error, "commit retry failed");
+ handle.signal_failure(error);
+ return;
+ }
}
}
if batch.is_empty() {
+ if let Ok(Err(error)) =
+ tokio::time::timeout(settings.shutdown_flush, consumer.retry_commit()).await
+ {
+ tracing::warn!(%error, "the commit on shutdown failed");
+ }
return;
}
flush_on_shutdown(
@@ -870,6 +929,68 @@ mod tests {
assert!(state.uncommitted.is_empty());
}
+ #[test]
+ fn a_failed_commit_waits_for_the_next_flush_while_the_partition_stays_owned() {
+ let mut state = Assignments::default();
+ for partition in 0..3 {
+ state.epochs.insert(tp(partition), (1, true));
+ }
+ let due = |state: &mut Assignments, flushed: &[(Tp, (u64, i64))]| {
+ let mut due = state.offsets_due(&flushed.iter().copied().collect());
+ due.sort_by_key(|(tp, _)| tp.partition);
+ due
+ };
+
+ let first = [(tp(0), (1, 10)), (tp(1), (1, 10)), (tp(2), (1, 10))];
+ assert_eq!(
+ due(&mut state, &first),
+ [(tp(0), 10), (tp(1), 10), (tp(2), 10)]
+ );
+
+ // Partition 0 reads more, partition 1 has no new messages, and partition 2 comes back under a later epoch.
+ state.epochs.insert(tp(2), (2, true));
+ assert_eq!(
+ due(&mut state, &[(tp(0), (1, 20))]),
+ [(tp(0), 20), (tp(1), 10)]
+ );
+
+ state.epochs.insert(tp(1), (2, false));
+ assert_eq!(due(&mut state, &[]), [(tp(0), 20)]);
+
+ state.uncommitted.insert(tp(0), Some(5));
+ state.offsets_committed();
+ assert!(due(&mut state, &[]).is_empty());
+ assert!(state.uncommitted.is_empty());
+ }
+
+ #[test]
+ fn only_a_success_resets_how_long_the_commits_have_failed() {
+ let mut state = Assignments::default();
+ let start = Instant::now();
+ assert_eq!(state.commit_failed(start), Duration::ZERO);
+ assert_eq!(state.commit_failed(start + COMMIT_GIVE_UP), COMMIT_GIVE_UP);
+ state.offsets_committed();
+ assert_eq!(state.commit_failed(start + COMMIT_GIVE_UP), Duration::ZERO);
+ }
+
+ #[test]
+ fn an_idle_consumer_retries_a_failed_commit_once_per_interval() {
+ let mut state = Assignments::default();
+ let start = Instant::now();
+ let interval = Duration::from_secs(60);
+ assert!(!state.commit_retry_due(start + interval, interval));
+
+ state.commit_failed(start);
+ assert!(!state.commit_retry_due(start + interval - Duration::from_secs(1), interval));
+ assert!(state.commit_retry_due(start + interval, interval));
+
+ state.commit_failed(start + interval);
+ assert!(!state.commit_retry_due(start + interval, interval));
+
+ state.offsets_committed();
+ assert!(!state.commit_retry_due(start + 2 * interval, interval));
+ }
+
#[test]
fn a_retry_writes_only_the_partitions_this_process_still_owns() {
let block = |partition: i32| {