Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(cohorts): reset and report only owned slices in a checkpoint restore (#123703)

Matheus Batista2466d5c00fb4
diff --git a/docs/internal/realtime-cohorts/state-store-and-durability.md b/docs/internal/realtime-cohorts/state-store-and-durability.md
index d0d42a538f86..3341b4970318 100644
--- a/docs/internal/realtime-cohorts/state-store-and-durability.md
+++ b/docs/internal/realtime-cohorts/state-store-and-durability.md
@@ -195,7 +195,7 @@ The boot then creates the store, and every slice begins behind the coverage fenc
### The replay window
A restored slice is only correct if the broker can still replay every input past the checkpoint's position for it.
-The restore checks each partition against the enabled inputs' watermarks:
+The restore checks each partition the pod owns against the enabled inputs' watermarks, because boot recovery deletes the others:
- a partition the manifest does not list is reset,
- a partition where any enabled input's position lies outside the broker's `[low, high]` range is reset,
diff --git a/rust/cohort-stream-processor/src/consumers/events.rs b/rust/cohort-stream-processor/src/consumers/events.rs
index 3324c37d083c..3f789d628de6 100644
--- a/rust/cohort-stream-processor/src/consumers/events.rs
+++ b/rust/cohort-stream-processor/src/consumers/events.rs
@@ -978,7 +978,7 @@ impl CohortStreamEventsConsumer {
self.wait_for_capture(&restore).await;
let mut resume = ResumePoints::default();
let topic = InputTopic::new(self.topic.as_str());
- resume.rewind_to(&restore.plan().positions(&topic, &self.claimed.ownership()));
+ resume.rewind_to(&restore.plan().positions(&topic));
BootPhase::Rewinding {
resume,
restore: Box::new(restore),
diff --git a/rust/cohort-stream-processor/src/store/durability/recovery.rs b/rust/cohort-stream-processor/src/store/durability/recovery.rs
index c527b180455b..2187a833ae28 100644
--- a/rust/cohort-stream-processor/src/store/durability/recovery.rs
+++ b/rust/cohort-stream-processor/src/store/durability/recovery.rs
@@ -327,7 +327,7 @@ async fn prepare_store(
remove_dir_if_present(&stage).map_err(RestoreBlocked::Local)?;
if !config.effective_wipe_on_start() && live_store_is_intact(&live) {
- match PendingRestore::resume(&live, window, config.partition_count()) {
+ match PendingRestore::resume(&live, window, &ownership) {
Ok(Some(pending)) => return Ok((BootStore::Restored(Box::new(pending)), "pending")),
Ok(None) => match source {
StoreSource::Own => return Ok((BootStore::Reopened, "reopen_live")),
@@ -620,9 +620,9 @@ impl<W: ReplayWindow> Restore<'_, W> {
/// The window check.
fn replay_plan(&self, manifest: &OffsetManifest) -> Result<Admission, RestoreBlocked> {
- let plan = RestorePlan::check(manifest, self.window, self.ownership.partition_count())
+ let plan = RestorePlan::check(manifest, self.window, &self.ownership)
.map_err(RestoreBlocked::Positions)?;
- if !plan.resumes_any(&self.ownership) {
+ if !plan.resumes_any() {
// Older candidates hold older positions, which the broker deletes first, so retrying
// cannot help.
return Ok(Admission::Rejected(Unusable::NothingToReplay));
@@ -1266,39 +1266,48 @@ mod tests {
#[tokio::test]
async fn a_local_checkpoint_stays_pending_and_each_boot_checks_its_window_again() {
- let boot = Boot::new();
- let baseline = boot.local_checkpoint();
- let uploaded = boot.lineage_dir().join(UPLOADED_FILENAME);
- UploadBaseline::save(&uploaded, &boot.config.checkpoint_s3_bucket, baseline).unwrap();
-
- let Ok(BootStore::Restored(published)) = boot.prepare(&window()).await else {
- panic!("a fresh local checkpoint restores");
- };
- assert_eq!(published.source().label(), "local");
- assert!(boot.live().join(MARKER_FILENAME).is_file());
- assert!(
- !uploaded.exists(),
- "a local restore leaves no upload baseline, so the next upload is full",
- );
+ // Pod 1 of 2's checkpoint lacks partition 0, so a wrong split resets it.
+ for boot in [Boot::new(), Boot::of(split(2, 1))] {
+ let baseline = boot.local_checkpoint();
+ let uploaded = boot.lineage_dir().join(UPLOADED_FILENAME);
+ UploadBaseline::save(&uploaded, &boot.config.checkpoint_s3_bucket, baseline).unwrap();
+
+ let Ok(BootStore::Restored(published)) = boot.prepare(&window()).await else {
+ panic!("{}: a fresh local checkpoint restores", boot.ownership);
+ };
+ assert_eq!(published.source().label(), "local");
+ assert_eq!(published.plan().resets().count(), 0, "{}", boot.ownership);
+ assert!(boot.live().join(MARKER_FILENAME).is_file());
+ assert!(
+ !uploaded.exists(),
+ "a local restore leaves no upload baseline, so the next upload is full",
+ );
- let Ok(BootStore::Restored(resumed)) = boot.prepare(&window()).await else {
- panic!("a store with a marker resumes its restore instead of reopening");
- };
- assert_eq!(resumed.plan(), published.plan());
+ let Ok(BootStore::Restored(resumed)) = boot.prepare(&window()).await else {
+ panic!(
+ "{}: a store with a marker resumes its restore instead of reopening",
+ boot.ownership
+ );
+ };
+ assert_eq!(resumed.plan(), published.plan(), "{}", boot.ownership);
- // The checkpoint resumes the events at 42.
- let Ok(BootStore::Restored(expired)) = boot.prepare(&window_at(Some(43))).await else {
- panic!("a pending restore resumes whatever the window holds");
- };
- assert_eq!(expired.plan().resets().count(), 1);
- assert!(matches!(
- boot.prepare(&window_at(None)).await,
- Err(RestoreBlocked::Positions(_))
- ));
- assert!(
- boot.live().join(MARKER_FILENAME).is_file(),
- "a window that cannot be read keeps the published store for the next round",
- );
+ // The checkpoint resumes the events at 42.
+ let Ok(BootStore::Restored(expired)) = boot.prepare(&window_at(Some(43))).await else {
+ panic!(
+ "{}: a pending restore resumes whatever the window holds",
+ boot.ownership
+ );
+ };
+ assert_eq!(expired.plan().resets().count(), 1, "{}", boot.ownership);
+ assert!(matches!(
+ boot.prepare(&window_at(None)).await,
+ Err(RestoreBlocked::Positions(_))
+ ));
+ assert!(
+ boot.live().join(MARKER_FILENAME).is_file(),
+ "a window that cannot be read keeps the published store for the next round",
+ );
+ }
}
#[tokio::test]
@@ -1315,7 +1324,7 @@ mod tests {
panic!("the store with an unreadable marker is replaced by a restore");
};
assert_eq!(pending.source().label(), "local");
- PendingRestore::resume(&boot.live(), &window(), 1)
+ PendingRestore::resume(&boot.live(), &window(), &boot.ownership)
.unwrap()
.expect("the new restore carries a readable marker");
}
diff --git a/rust/cohort-stream-processor/src/store/durability/restore_plan.rs b/rust/cohort-stream-processor/src/store/durability/restore_plan.rs
index 73c6f0c206aa..79bf87cb4c50 100644
--- a/rust/cohort-stream-processor/src/store/durability/restore_plan.rs
+++ b/rust/cohort-stream-processor/src/store/durability/restore_plan.rs
@@ -63,28 +63,27 @@ impl ResetReason {
}
}
-/// One [`SliceRestore`] per partition of the topic, built only by [`Self::check`].
+/// One [`SliceRestore`] per owned partition, built only by [`Self::check`].
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct RestorePlan(BTreeMap<u16, SliceRestore>);
impl RestorePlan {
- /// Check every partition's checkpoint positions against the broker. A slice resumes only when
- /// every enabled input it records can still be replayed from its position: within
- /// `[low, high]`. An enabled input the manifest lacks, because its gate turned on after the
- /// capture, gets no position and resumes from its group's commit. A manifest input that is no
- /// longer enabled is ignored.
+ /// Checks only the owned partitions, because boot recovery deletes the rest. A slice resumes
+ /// when every enabled input it records is within the broker's `[low, high]`. An enabled input
+ /// the manifest lacks resumes from its group's commit; a disabled one is ignored.
pub fn check(
manifest: &OffsetManifest,
window: &impl ReplayWindow,
- partition_count: u16,
+ ownership: &PartitionOwnership,
) -> KafkaResult<Self> {
let checked: Vec<&InputTopic> = window.inputs().intersection(manifest.inputs()).collect();
+ let partition_count = ownership.partition_count();
let bounds = checked
.iter()
.map(|&topic| Ok((topic, window.watermarks(topic, partition_count)?)))
.collect::<KafkaResult<BTreeMap<_, _>>>()?;
let mut slices = BTreeMap::new();
- for partition in 0..partition_count {
+ for partition in ownership.partitions() {
let Some(recorded) = manifest.slice(partition) else {
slices.insert(partition, SliceRestore::Reset(ResetReason::NotInCheckpoint));
continue;
@@ -116,11 +115,10 @@ impl RestorePlan {
Ok(Self(slices))
}
- /// True when a slice `ownership` owns resumes. Boot deletes the slices of the other partitions.
- pub fn resumes_any(&self, ownership: &PartitionOwnership) -> bool {
- self.0.iter().any(|(&partition, slice)| {
- ownership.owns(partition) && matches!(slice, SliceRestore::Resume(_))
- })
+ pub fn resumes_any(&self) -> bool {
+ self.0
+ .values()
+ .any(|slice| matches!(slice, SliceRestore::Resume(_)))
}
pub fn resets(&self) -> impl Iterator<Item = (u16, &ResetReason)> {
@@ -144,15 +142,10 @@ impl RestorePlan {
})
}
- /// Where `topic` resumes on every slice the plan keeps that `ownership` owns.
- pub fn positions(
- &self,
- topic: &InputTopic,
- ownership: &PartitionOwnership,
- ) -> BTreeMap<u16, ResumeOffset> {
+ /// Where `topic` resumes on each kept slice.
+ pub fn positions(&self, topic: &InputTopic) -> BTreeMap<u16, ResumeOffset> {
self.0
.iter()
- .filter(|&(&partition, _)| ownership.owns(partition))
.filter_map(|(&partition, slice)| match slice {
SliceRestore::Resume(positions) => Some((partition, *positions.get(topic)?)),
SliceRestore::Reset(_) => None,
@@ -263,7 +256,7 @@ mod tests {
],
);
- let plan = RestorePlan::check(&recorded, &window, 5).unwrap();
+ let plan = RestorePlan::check(&recorded, &window, &PartitionOwnership::single(5)).unwrap();
assert_eq!(
plan.0,
@@ -303,26 +296,24 @@ mod tests {
),
]),
);
- assert!(plan.resumes_any(&PartitionOwnership::single(5)));
+ assert!(plan.resumes_any());
assert_eq!(
- plan.positions(&InputTopic::new(EVENTS), &PartitionOwnership::single(5)),
+ plan.positions(&InputTopic::new(EVENTS)),
BTreeMap::from([(0, offset(10)), (4, offset(10))]),
);
let pod_of_eight = |ordinal| {
PartitionOwnership::new(NonZeroU16::new(8).unwrap(), PodOrdinal::from(ordinal), 8)
.unwrap()
};
- for (ordinal, kept) in [(0, 0), (4, 4)] {
+ for ordinal in 0..5 {
assert_eq!(
- plan.positions(&InputTopic::new(MERGES), &pod_of_eight(ordinal)),
- BTreeMap::from([(kept, offset(5))]),
- "pod {ordinal} positions only its own kept slice",
+ RestorePlan::check(&recorded, &window, &pod_of_eight(ordinal))
+ .unwrap()
+ .0,
+ BTreeMap::from([(ordinal, plan.0[&ordinal].clone())]),
+ "pod {ordinal} of 8",
);
}
- assert!(
- !plan.resumes_any(&pod_of_eight(1)),
- "pod 1's only slice resets, so other pods' kept slices do not make it resume",
- );
}
#[test]
@@ -331,7 +322,7 @@ mod tests {
// The merge input was disabled after the capture, and the seed input enabled after it.
let window = window(&[EVENTS, SEEDS], &[((MERGES, 0), (900, 1_000))]);
- let plan = RestorePlan::check(&recorded, &window, 1).unwrap();
+ let plan = RestorePlan::check(&recorded, &window, &PartitionOwnership::single(1)).unwrap();
assert_eq!(
plan.0,
@@ -340,9 +331,7 @@ mod tests {
SliceRestore::Resume(BTreeMap::from([(InputTopic::new(EVENTS), offset(10))])),
)]),
);
- assert!(plan
- .positions(&InputTopic::new(SEEDS), &PartitionOwnership::single(1))
- .is_empty());
+ assert!(plan.positions(&InputTopic::new(SEEDS)).is_empty());
}
#[test]
@@ -350,9 +339,9 @@ mod tests {
let recorded = manifest(&[(0, &[(EVENTS, 10)])]);
let window = window(&[EVENTS], &[((EVENTS, 0), (11, 20))]);
- let plan = RestorePlan::check(&recorded, &window, 2).unwrap();
+ let plan = RestorePlan::check(&recorded, &window, &PartitionOwnership::single(2)).unwrap();
- assert!(!plan.resumes_any(&PartitionOwnership::single(2)));
+ assert!(!plan.resumes_any());
assert_eq!(
plan.resets()
.map(|(partition, _)| partition)
diff --git a/rust/cohort-stream-processor/src/store/durability/stage.rs b/rust/cohort-stream-processor/src/store/durability/stage.rs
index 9afd9b2b8a1c..a266713fabc1 100644
--- a/rust/cohort-stream-processor/src/store/durability/stage.rs
+++ b/rust/cohort-stream-processor/src/store/durability/stage.rs
@@ -22,7 +22,7 @@ use super::manifest::OffsetManifest;
use super::metadata::CheckpointMetadata;
use super::restore_plan::{ReplayWindow, RestorePlan};
use crate::observability::metrics::CHECKPOINT_RESTORE_SLICES_RESET_TOTAL;
-use crate::partitions::{AdmittedSplit, InputGroups, InputTopic, ResumeOffset};
+use crate::partitions::{AdmittedSplit, InputGroups, InputTopic, PartitionOwnership, ResumeOffset};
use crate::store::{CohortStore, StoreError};
/// The marker a published restore carries inside the store directory. RocksDB ignores files it
@@ -174,7 +174,7 @@ impl PendingRestore {
pub(crate) fn resume(
live: &Path,
window: &impl ReplayWindow,
- partition_count: u16,
+ ownership: &PartitionOwnership,
) -> Result<Option<Self>, ResumeError> {
let marker = live.join(MARKER_FILENAME);
let bytes = match std::fs::read(&marker) {
@@ -189,8 +189,8 @@ impl PendingRestore {
format_args!("restore record version {}", record.version),
)));
}
- let plan = RestorePlan::check(&record.manifest, window, partition_count)
- .map_err(ResumeError::Window)?;
+ let plan =
+ RestorePlan::check(&record.manifest, window, ownership).map_err(ResumeError::Window)?;
Ok(Some(Self {
marker,
record,
@@ -211,19 +211,15 @@ impl PendingRestore {
self.record.manifest.captured_at()
}
- /// Commits each follower's planned positions on the partitions `admitted` owns, with its stamp.
- /// The other partitions belong to other pods, whose followers keep their own commits. Safe to
- /// repeat: nothing consumes a follower before the restore finishes.
+ /// Commits the plan's follower positions with `admitted`'s stamp. The plan was checked under
+ /// `admitted`'s split, so it names only this pod's partitions. Safe to repeat before `finish`.
pub(crate) fn position_followers(
&self,
groups: &InputGroups,
admitted: &AdmittedSplit,
) -> KafkaResult<()> {
for follower in groups.followers() {
- follower.commit(
- &admitted.stamp(),
- &self.plan.positions(follower.topic(), &admitted.ownership()),
- )?;
+ follower.commit(&admitted.stamp(), &self.plan.positions(follower.topic()))?;
}
Ok(())
}
@@ -470,7 +466,8 @@ mod tests {
let root = TempDir::new().unwrap();
let live = root.path().join("store");
let stage = staged(root.path());
- let plan = RestorePlan::check(&manifest(), &window(), 1).unwrap();
+ let plan =
+ RestorePlan::check(&manifest(), &window(), &PartitionOwnership::single(1)).unwrap();
let published = ValidatedStage::validate(stage.clone(), local())
.unwrap()
@@ -488,7 +485,7 @@ mod tests {
);
drop(store);
- let resumed = PendingRestore::resume(&live, &window(), 1)
+ let resumed = PendingRestore::resume(&live, &window(), &PartitionOwnership::single(1))
.unwrap()
.expect("the marker survives the store's open");
assert_eq!(resumed.plan(), &plan);
@@ -543,15 +540,17 @@ mod tests {
Err(FinishError::Commit(_))
));
assert!(
- PendingRestore::resume(&live, &window(), 1)
+ PendingRestore::resume(&live, &window(), &PartitionOwnership::single(1))
.unwrap()
.is_some(),
"a restore that did not finish is still pending",
);
published.finish(&owned, &held, || Ok(())).unwrap();
- assert!(PendingRestore::resume(&live, &window(), 1)
- .unwrap()
- .is_none());
+ assert!(
+ PendingRestore::resume(&live, &window(), &PartitionOwnership::single(1))
+ .unwrap()
+ .is_none()
+ );
}
}
diff --git a/rust/cohort-stream-processor/tests/events_consumer.rs b/rust/cohort-stream-processor/tests/events_consumer.rs
index 9b19ca4b96d9..9766f8f60247 100644
--- a/rust/cohort-stream-processor/tests/events_consumer.rs
+++ b/rust/cohort-stream-processor/tests/events_consumer.rs
@@ -1552,9 +1552,7 @@ async fn s3_restore_replays_past_the_checkpoint_after_the_broker_moved_on() {
let restore = restore
.expect("with the live store and local checkpoint gone, the boot restores a checkpoint");
assert_eq!(restore.source().label(), "s3");
- let resumed = restore
- .plan()
- .positions(&InputTopic::new(topic.as_str()), &SINGLE_POD);
+ let resumed = restore.plan().positions(&InputTopic::new(topic.as_str()));
assert_eq!(
resumed
.get(&(idle_partition as u16))
@@ -1569,7 +1567,7 @@ async fn s3_restore_replays_past_the_checkpoint_after_the_broker_moved_on() {
assert_eq!(
restore
.plan()
- .positions(&InputTopic::new(topics.merges.as_str()), &SINGLE_POD)
+ .positions(&InputTopic::new(topics.merges.as_str()))
.get(&(merges_with_events as u16))
.map(|offset| offset.get()),
Some(0),
diff --git a/rust/cohort-stream-processor/tests/merge_durability.rs b/rust/cohort-stream-processor/tests/merge_durability.rs
index 682e4ff70685..4cbf421a9ae6 100644
--- a/rust/cohort-stream-processor/tests/merge_durability.rs
+++ b/rust/cohort-stream-processor/tests/merge_durability.rs
@@ -1164,10 +1164,10 @@ async fn merge_and_cascade_state_survive_an_s3_disaster_restore() {
// The followers resume past the merge on P_old's partition and the transfer on P_new's.
let merge_resume = restore
.plan()
- .positions(&InputTopic::new(topics.merges.as_str()), &admitted.ownership());
+ .positions(&InputTopic::new(topics.merges.as_str()));
let transfer_resume = restore
.plan()
- .positions(&InputTopic::new(topics.transfers.as_str()), &admitted.ownership());
+ .positions(&InputTopic::new(topics.transfers.as_str()));
assert!(
merge_resume.get(&part(p_old)).map(|offset| offset.get()) >= Some(1),
"the merge follower resumes past the merge on P_old's partition: {merge_resume:?}",