:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(cohorts): reset and report only owned slices in a checkpoint restore (#123703)
Matheus Batista2466d5c00fb4
diff --git a/docs/internal/realtime-cohorts/state-store-and-durability.md b/docs/internal/realtime-cohorts/state-store-and-durability.mdindex d0d42a538f86..3341b4970318 100644--- a/docs/internal/realtime-cohorts/state-store-and-durability.md+++ b/docs/internal/realtime-cohorts/state-store-and-durability.md@@ -195,7 +195,7 @@ The boot then creates the store, and every slice begins behind the coverage fenc### The replay windowA restored slice is only correct if the broker can still replay every input past the checkpoint's position for it.-The restore checks each partition against the enabled inputs' watermarks:+The restore checks each partition the pod owns against the enabled inputs' watermarks, because boot recovery deletes the others:- a partition the manifest does not list is reset,- a partition where any enabled input's position lies outside the broker's `[low, high]` range is reset,diff --git a/rust/cohort-stream-processor/src/consumers/events.rs b/rust/cohort-stream-processor/src/consumers/events.rsindex 3324c37d083c..3f789d628de6 100644--- a/rust/cohort-stream-processor/src/consumers/events.rs+++ b/rust/cohort-stream-processor/src/consumers/events.rs@@ -978,7 +978,7 @@ impl CohortStreamEventsConsumer {self.wait_for_capture(&restore).await;let mut resume = ResumePoints::default();let topic = InputTopic::new(self.topic.as_str());- resume.rewind_to(&restore.plan().positions(&topic, &self.claimed.ownership()));+ resume.rewind_to(&restore.plan().positions(&topic));BootPhase::Rewinding {resume,restore: Box::new(restore),diff --git a/rust/cohort-stream-processor/src/store/durability/recovery.rs b/rust/cohort-stream-processor/src/store/durability/recovery.rsindex c527b180455b..2187a833ae28 100644--- a/rust/cohort-stream-processor/src/store/durability/recovery.rs+++ b/rust/cohort-stream-processor/src/store/durability/recovery.rs@@ -327,7 +327,7 @@ async fn prepare_store(remove_dir_if_present(&stage).map_err(RestoreBlocked::Local)?;if !config.effective_wipe_on_start() && live_store_is_intact(&live) {- match PendingRestore::resume(&live, window, config.partition_count()) {+ match PendingRestore::resume(&live, window, &ownership) {Ok(Some(pending)) => return Ok((BootStore::Restored(Box::new(pending)), "pending")),Ok(None) => match source {StoreSource::Own => return Ok((BootStore::Reopened, "reopen_live")),@@ -620,9 +620,9 @@ impl<W: ReplayWindow> Restore<'_, W> {/// The window check.fn replay_plan(&self, manifest: &OffsetManifest) -> Result<Admission, RestoreBlocked> {- let plan = RestorePlan::check(manifest, self.window, self.ownership.partition_count())+ let plan = RestorePlan::check(manifest, self.window, &self.ownership).map_err(RestoreBlocked::Positions)?;- if !plan.resumes_any(&self.ownership) {+ if !plan.resumes_any() {// Older candidates hold older positions, which the broker deletes first, so retrying// cannot help.return Ok(Admission::Rejected(Unusable::NothingToReplay));@@ -1266,39 +1266,48 @@ mod tests {#[tokio::test]async fn a_local_checkpoint_stays_pending_and_each_boot_checks_its_window_again() {- let boot = Boot::new();- let baseline = boot.local_checkpoint();- let uploaded = boot.lineage_dir().join(UPLOADED_FILENAME);- UploadBaseline::save(&uploaded, &boot.config.checkpoint_s3_bucket, baseline).unwrap();-- let Ok(BootStore::Restored(published)) = boot.prepare(&window()).await else {- panic!("a fresh local checkpoint restores");- };- assert_eq!(published.source().label(), "local");- assert!(boot.live().join(MARKER_FILENAME).is_file());- assert!(- !uploaded.exists(),- "a local restore leaves no upload baseline, so the next upload is full",- );+ // Pod 1 of 2's checkpoint lacks partition 0, so a wrong split resets it.+ for boot in [Boot::new(), Boot::of(split(2, 1))] {+ let baseline = boot.local_checkpoint();+ let uploaded = boot.lineage_dir().join(UPLOADED_FILENAME);+ UploadBaseline::save(&uploaded, &boot.config.checkpoint_s3_bucket, baseline).unwrap();++ let Ok(BootStore::Restored(published)) = boot.prepare(&window()).await else {+ panic!("{}: a fresh local checkpoint restores", boot.ownership);+ };+ assert_eq!(published.source().label(), "local");+ assert_eq!(published.plan().resets().count(), 0, "{}", boot.ownership);+ assert!(boot.live().join(MARKER_FILENAME).is_file());+ assert!(+ !uploaded.exists(),+ "a local restore leaves no upload baseline, so the next upload is full",+ );- let Ok(BootStore::Restored(resumed)) = boot.prepare(&window()).await else {- panic!("a store with a marker resumes its restore instead of reopening");- };- assert_eq!(resumed.plan(), published.plan());+ let Ok(BootStore::Restored(resumed)) = boot.prepare(&window()).await else {+ panic!(+ "{}: a store with a marker resumes its restore instead of reopening",+ boot.ownership+ );+ };+ assert_eq!(resumed.plan(), published.plan(), "{}", boot.ownership);- // The checkpoint resumes the events at 42.- let Ok(BootStore::Restored(expired)) = boot.prepare(&window_at(Some(43))).await else {- panic!("a pending restore resumes whatever the window holds");- };- assert_eq!(expired.plan().resets().count(), 1);- assert!(matches!(- boot.prepare(&window_at(None)).await,- Err(RestoreBlocked::Positions(_))- ));- assert!(- boot.live().join(MARKER_FILENAME).is_file(),- "a window that cannot be read keeps the published store for the next round",- );+ // The checkpoint resumes the events at 42.+ let Ok(BootStore::Restored(expired)) = boot.prepare(&window_at(Some(43))).await else {+ panic!(+ "{}: a pending restore resumes whatever the window holds",+ boot.ownership+ );+ };+ assert_eq!(expired.plan().resets().count(), 1, "{}", boot.ownership);+ assert!(matches!(+ boot.prepare(&window_at(None)).await,+ Err(RestoreBlocked::Positions(_))+ ));+ assert!(+ boot.live().join(MARKER_FILENAME).is_file(),+ "a window that cannot be read keeps the published store for the next round",+ );+ }}#[tokio::test]@@ -1315,7 +1324,7 @@ mod tests {panic!("the store with an unreadable marker is replaced by a restore");};assert_eq!(pending.source().label(), "local");- PendingRestore::resume(&boot.live(), &window(), 1)+ PendingRestore::resume(&boot.live(), &window(), &boot.ownership).unwrap().expect("the new restore carries a readable marker");}diff --git a/rust/cohort-stream-processor/src/store/durability/restore_plan.rs b/rust/cohort-stream-processor/src/store/durability/restore_plan.rsindex 73c6f0c206aa..79bf87cb4c50 100644--- a/rust/cohort-stream-processor/src/store/durability/restore_plan.rs+++ b/rust/cohort-stream-processor/src/store/durability/restore_plan.rs@@ -63,28 +63,27 @@ impl ResetReason {}}-/// One [`SliceRestore`] per partition of the topic, built only by [`Self::check`].+/// One [`SliceRestore`] per owned partition, built only by [`Self::check`].#[derive(Debug, Clone, PartialEq, Eq)]pub struct RestorePlan(BTreeMap<u16, SliceRestore>);impl RestorePlan {- /// Check every partition's checkpoint positions against the broker. A slice resumes only when- /// every enabled input it records can still be replayed from its position: within- /// `[low, high]`. An enabled input the manifest lacks, because its gate turned on after the- /// capture, gets no position and resumes from its group's commit. A manifest input that is no- /// longer enabled is ignored.+ /// Checks only the owned partitions, because boot recovery deletes the rest. A slice resumes+ /// when every enabled input it records is within the broker's `[low, high]`. An enabled input+ /// the manifest lacks resumes from its group's commit; a disabled one is ignored.pub fn check(manifest: &OffsetManifest,window: &impl ReplayWindow,- partition_count: u16,+ ownership: &PartitionOwnership,) -> KafkaResult<Self> {let checked: Vec<&InputTopic> = window.inputs().intersection(manifest.inputs()).collect();+ let partition_count = ownership.partition_count();let bounds = checked.iter().map(|&topic| Ok((topic, window.watermarks(topic, partition_count)?))).collect::<KafkaResult<BTreeMap<_, _>>>()?;let mut slices = BTreeMap::new();- for partition in 0..partition_count {+ for partition in ownership.partitions() {let Some(recorded) = manifest.slice(partition) else {slices.insert(partition, SliceRestore::Reset(ResetReason::NotInCheckpoint));continue;@@ -116,11 +115,10 @@ impl RestorePlan {Ok(Self(slices))}- /// True when a slice `ownership` owns resumes. Boot deletes the slices of the other partitions.- pub fn resumes_any(&self, ownership: &PartitionOwnership) -> bool {- self.0.iter().any(|(&partition, slice)| {- ownership.owns(partition) && matches!(slice, SliceRestore::Resume(_))- })+ pub fn resumes_any(&self) -> bool {+ self.0+ .values()+ .any(|slice| matches!(slice, SliceRestore::Resume(_)))}pub fn resets(&self) -> impl Iterator<Item = (u16, &ResetReason)> {@@ -144,15 +142,10 @@ impl RestorePlan {})}- /// Where `topic` resumes on every slice the plan keeps that `ownership` owns.- pub fn positions(- &self,- topic: &InputTopic,- ownership: &PartitionOwnership,- ) -> BTreeMap<u16, ResumeOffset> {+ /// Where `topic` resumes on each kept slice.+ pub fn positions(&self, topic: &InputTopic) -> BTreeMap<u16, ResumeOffset> {self.0.iter()- .filter(|&(&partition, _)| ownership.owns(partition)).filter_map(|(&partition, slice)| match slice {SliceRestore::Resume(positions) => Some((partition, *positions.get(topic)?)),SliceRestore::Reset(_) => None,@@ -263,7 +256,7 @@ mod tests {],);- let plan = RestorePlan::check(&recorded, &window, 5).unwrap();+ let plan = RestorePlan::check(&recorded, &window, &PartitionOwnership::single(5)).unwrap();assert_eq!(plan.0,@@ -303,26 +296,24 @@ mod tests {),]),);- assert!(plan.resumes_any(&PartitionOwnership::single(5)));+ assert!(plan.resumes_any());assert_eq!(- plan.positions(&InputTopic::new(EVENTS), &PartitionOwnership::single(5)),+ plan.positions(&InputTopic::new(EVENTS)),BTreeMap::from([(0, offset(10)), (4, offset(10))]),);let pod_of_eight = |ordinal| {PartitionOwnership::new(NonZeroU16::new(8).unwrap(), PodOrdinal::from(ordinal), 8).unwrap()};- for (ordinal, kept) in [(0, 0), (4, 4)] {+ for ordinal in 0..5 {assert_eq!(- plan.positions(&InputTopic::new(MERGES), &pod_of_eight(ordinal)),- BTreeMap::from([(kept, offset(5))]),- "pod {ordinal} positions only its own kept slice",+ RestorePlan::check(&recorded, &window, &pod_of_eight(ordinal))+ .unwrap()+ .0,+ BTreeMap::from([(ordinal, plan.0[&ordinal].clone())]),+ "pod {ordinal} of 8",);}- assert!(- !plan.resumes_any(&pod_of_eight(1)),- "pod 1's only slice resets, so other pods' kept slices do not make it resume",- );}#[test]@@ -331,7 +322,7 @@ mod tests {// The merge input was disabled after the capture, and the seed input enabled after it.let window = window(&[EVENTS, SEEDS], &[((MERGES, 0), (900, 1_000))]);- let plan = RestorePlan::check(&recorded, &window, 1).unwrap();+ let plan = RestorePlan::check(&recorded, &window, &PartitionOwnership::single(1)).unwrap();assert_eq!(plan.0,@@ -340,9 +331,7 @@ mod tests {SliceRestore::Resume(BTreeMap::from([(InputTopic::new(EVENTS), offset(10))])),)]),);- assert!(plan- .positions(&InputTopic::new(SEEDS), &PartitionOwnership::single(1))- .is_empty());+ assert!(plan.positions(&InputTopic::new(SEEDS)).is_empty());}#[test]@@ -350,9 +339,9 @@ mod tests {let recorded = manifest(&[(0, &[(EVENTS, 10)])]);let window = window(&[EVENTS], &[((EVENTS, 0), (11, 20))]);- let plan = RestorePlan::check(&recorded, &window, 2).unwrap();+ let plan = RestorePlan::check(&recorded, &window, &PartitionOwnership::single(2)).unwrap();- assert!(!plan.resumes_any(&PartitionOwnership::single(2)));+ assert!(!plan.resumes_any());assert_eq!(plan.resets().map(|(partition, _)| partition)diff --git a/rust/cohort-stream-processor/src/store/durability/stage.rs b/rust/cohort-stream-processor/src/store/durability/stage.rsindex 9afd9b2b8a1c..a266713fabc1 100644--- a/rust/cohort-stream-processor/src/store/durability/stage.rs+++ b/rust/cohort-stream-processor/src/store/durability/stage.rs@@ -22,7 +22,7 @@ use super::manifest::OffsetManifest;use super::metadata::CheckpointMetadata;use super::restore_plan::{ReplayWindow, RestorePlan};use crate::observability::metrics::CHECKPOINT_RESTORE_SLICES_RESET_TOTAL;-use crate::partitions::{AdmittedSplit, InputGroups, InputTopic, ResumeOffset};+use crate::partitions::{AdmittedSplit, InputGroups, InputTopic, PartitionOwnership, ResumeOffset};use crate::store::{CohortStore, StoreError};/// The marker a published restore carries inside the store directory. RocksDB ignores files it@@ -174,7 +174,7 @@ impl PendingRestore {pub(crate) fn resume(live: &Path,window: &impl ReplayWindow,- partition_count: u16,+ ownership: &PartitionOwnership,) -> Result<Option<Self>, ResumeError> {let marker = live.join(MARKER_FILENAME);let bytes = match std::fs::read(&marker) {@@ -189,8 +189,8 @@ impl PendingRestore {format_args!("restore record version {}", record.version),)));}- let plan = RestorePlan::check(&record.manifest, window, partition_count)- .map_err(ResumeError::Window)?;+ let plan =+ RestorePlan::check(&record.manifest, window, ownership).map_err(ResumeError::Window)?;Ok(Some(Self {marker,record,@@ -211,19 +211,15 @@ impl PendingRestore {self.record.manifest.captured_at()}- /// Commits each follower's planned positions on the partitions `admitted` owns, with its stamp.- /// The other partitions belong to other pods, whose followers keep their own commits. Safe to- /// repeat: nothing consumes a follower before the restore finishes.+ /// Commits the plan's follower positions with `admitted`'s stamp. The plan was checked under+ /// `admitted`'s split, so it names only this pod's partitions. Safe to repeat before `finish`.pub(crate) fn position_followers(&self,groups: &InputGroups,admitted: &AdmittedSplit,) -> KafkaResult<()> {for follower in groups.followers() {- follower.commit(- &admitted.stamp(),- &self.plan.positions(follower.topic(), &admitted.ownership()),- )?;+ follower.commit(&admitted.stamp(), &self.plan.positions(follower.topic()))?;}Ok(())}@@ -470,7 +466,8 @@ mod tests {let root = TempDir::new().unwrap();let live = root.path().join("store");let stage = staged(root.path());- let plan = RestorePlan::check(&manifest(), &window(), 1).unwrap();+ let plan =+ RestorePlan::check(&manifest(), &window(), &PartitionOwnership::single(1)).unwrap();let published = ValidatedStage::validate(stage.clone(), local()).unwrap()@@ -488,7 +485,7 @@ mod tests {);drop(store);- let resumed = PendingRestore::resume(&live, &window(), 1)+ let resumed = PendingRestore::resume(&live, &window(), &PartitionOwnership::single(1)).unwrap().expect("the marker survives the store's open");assert_eq!(resumed.plan(), &plan);@@ -543,15 +540,17 @@ mod tests {Err(FinishError::Commit(_))));assert!(- PendingRestore::resume(&live, &window(), 1)+ PendingRestore::resume(&live, &window(), &PartitionOwnership::single(1)).unwrap().is_some(),"a restore that did not finish is still pending",);published.finish(&owned, &held, || Ok(())).unwrap();- assert!(PendingRestore::resume(&live, &window(), 1)- .unwrap()- .is_none());+ assert!(+ PendingRestore::resume(&live, &window(), &PartitionOwnership::single(1))+ .unwrap()+ .is_none()+ );}}diff --git a/rust/cohort-stream-processor/tests/events_consumer.rs b/rust/cohort-stream-processor/tests/events_consumer.rsindex 9b19ca4b96d9..9766f8f60247 100644--- a/rust/cohort-stream-processor/tests/events_consumer.rs+++ b/rust/cohort-stream-processor/tests/events_consumer.rs@@ -1552,9 +1552,7 @@ async fn s3_restore_replays_past_the_checkpoint_after_the_broker_moved_on() {let restore = restore.expect("with the live store and local checkpoint gone, the boot restores a checkpoint");assert_eq!(restore.source().label(), "s3");- let resumed = restore- .plan()- .positions(&InputTopic::new(topic.as_str()), &SINGLE_POD);+ let resumed = restore.plan().positions(&InputTopic::new(topic.as_str()));assert_eq!(resumed.get(&(idle_partition as u16))@@ -1569,7 +1567,7 @@ async fn s3_restore_replays_past_the_checkpoint_after_the_broker_moved_on() {assert_eq!(restore.plan()- .positions(&InputTopic::new(topics.merges.as_str()), &SINGLE_POD)+ .positions(&InputTopic::new(topics.merges.as_str())).get(&(merges_with_events as u16)).map(|offset| offset.get()),Some(0),diff --git a/rust/cohort-stream-processor/tests/merge_durability.rs b/rust/cohort-stream-processor/tests/merge_durability.rsindex 682e4ff70685..4cbf421a9ae6 100644--- a/rust/cohort-stream-processor/tests/merge_durability.rs+++ b/rust/cohort-stream-processor/tests/merge_durability.rs@@ -1164,10 +1164,10 @@ async fn merge_and_cascade_state_survive_an_s3_disaster_restore() {// The followers resume past the merge on P_old's partition and the transfer on P_new's.let merge_resume = restore.plan()- .positions(&InputTopic::new(topics.merges.as_str()), &admitted.ownership());+ .positions(&InputTopic::new(topics.merges.as_str()));let transfer_resume = restore.plan()- .positions(&InputTopic::new(topics.transfers.as_str()), &admitted.ownership());+ .positions(&InputTopic::new(topics.transfers.as_str()));assert!(merge_resume.get(&part(p_old)).map(|offset| offset.get()) >= Some(1),"the merge follower resumes past the merge on P_old's partition: {merge_resume:?}",