:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(personhog): retire drained changelog generations from drain reports (#121318)
Zach Brownce3d49243d79
diff --git a/rust/Cargo.lock b/rust/Cargo.lockindex 3d30f7b574b0..6198b40dedcc 100644--- a/rust/Cargo.lock+++ b/rust/Cargo.lock@@ -9179,7 +9179,6 @@ dependencies = ["axum 0.7.9","chrono","clap",- "common-kafka","common-metrics","futures","hdrhistogram",diff --git a/rust/ingestion-control-plane/src/changelog.rs b/rust/ingestion-control-plane/src/changelog.rsindex 7965cd6134bf..71aa08104bd9 100644--- a/rust/ingestion-control-plane/src/changelog.rs+++ b/rust/ingestion-control-plane/src/changelog.rs@@ -100,6 +100,8 @@ pub struct GroupDrain {pub drained_partitions: usize,pub partitions: usize,pub error: Option<String>,+ /// The group's own drain report, which the coordinator retires on.+ pub reported: bool,}/// A draining generation and how far each gating group has read it.@@ -126,18 +128,21 @@ async fn group_drain(config: &Config, group: &str, topic: &str) -> GroupDrain {drained_partitions: drained.values().filter(|ok| **ok).count(),partitions: drained.len(),error: None,+ reported: false,},Ok(Err(e)) => GroupDrain {group: group.to_string(),drained_partitions: 0,partitions: 0,error: Some(e.to_string()),+ reported: false,},Err(e) => GroupDrain {group: group.to_string(),drained_partitions: 0,partitions: 0,error: Some(format!("probe task panicked: {e}")),+ reported: false,},}}@@ -180,6 +185,13 @@ pub async fn retire(if !live.draining.iter().any(|g| g.topic == topic) {bail!("{topic} is not a draining generation");}+ if let Some(migration) = store.get_migration().await? {+ bail!(+ "migration {} is {:?}; retire once it completes",+ migration.id,+ migration.phase+ );+ }for group in live.gated_groups() {let drain = group_drain(config, group, topic).await;if let Some(error) = drain.error {@@ -197,8 +209,13 @@ pub async fn retire(.retire_changelog_topic(&live, mod_revision, topic).await?{- bail!("the config moved; refresh and retry");+ bail!("the config moved or a migration started; refresh and retry");}+ common_metrics::inc(+ "personhog_changelog_retires_total",+ &[("trigger".to_string(), "manual".to_string())],+ 1,+ );Ok(())}@@ -316,16 +333,27 @@ pub async fn view(config: &Config, client: &Client, prefix: &str) -> anyhow::Reshistory.sort_by_key(|m| std::cmp::Reverse(m.created_at_ms));let mut draining = Vec::new();if let Some(changelog) = &changelog {+ let reports = store.list_drain_reports().await?;+ let drain = |group: &str, topic: &str| {+ let reported = reports.iter().any(|r| r.group == group && r.topic == topic);+ let (group, topic) = (group.to_string(), topic.to_string());+ async move {+ GroupDrain {+ reported,+ ..group_drain(config, &group, &topic).await+ }+ }+ };for generation in &changelog.draining {draining.push(DrainingView {topic: generation.topic.clone(),partitions: generation.partitions,epoch: generation.epoch,- writer: group_drain(config, &changelog.writer_group, &generation.topic).await,+ writer: drain(&changelog.writer_group, &generation.topic).await,retire_groups: {let mut drains = Vec::new();for group in &changelog.retire_groups {- drains.push(group_drain(config, group, &generation.topic).await);+ drains.push(drain(group, &generation.topic).await);}drains},diff --git a/rust/ingestion-control-plane/src/personhog.rs b/rust/ingestion-control-plane/src/personhog.rsindex 5e4e1036a041..68d3b55c85b2 100644--- a/rust/ingestion-control-plane/src/personhog.rs+++ b/rust/ingestion-control-plane/src/personhog.rs@@ -131,7 +131,10 @@ pub fn parse_snapshot(kvs: &[RawKv], prefix: &str, revision: i64) -> RawSnapshotif let Some(changelog) = parse::<ChangelogConfig>(kv, errors) {snapshot.changelog = Some(changelog);}- } else if suffix.starts_with("migrations/") || suffix.starts_with("seal_acks/") {+ } else if suffix.starts_with("migrations/")+ || suffix.starts_with("seal_acks/")+ || suffix.starts_with("changelog/drained/")+ {// The changelog panel reads these.} else {snapshot.unknown_keys.push(kv.key.clone());@@ -743,6 +746,7 @@ mod tests {kv("migrations/current", serde_json::json!({}), 0),kv("migrations/history/m-1", serde_json::json!({}), 0),kv("seal_acks/pod-a", serde_json::json!({}), 0),+ kv("changelog/drained/old/writer", serde_json::json!({}), 0),];assert!(parse_snapshot(&kvs, PREFIX, 42).unknown_keys.is_empty());}diff --git a/rust/ingestion-control-plane/src/ui/index.html b/rust/ingestion-control-plane/src/ui/index.htmlindex 7eb1a381d705..7fded89ad5f5 100644--- a/rust/ingestion-control-plane/src/ui/index.html+++ b/rust/ingestion-control-plane/src/ui/index.html@@ -1594,11 +1594,11 @@method: "POST",headers: { "content-type": "application/json" },body: JSON.stringify({ topic: g.topic }),- }, `Retire ${g.topic}?\n\nEvery gating group has read it to the end. It leaves the config; delete the topic in Terraform afterwards.`);+ }, `Retire ${g.topic} now?\n\nEvery gating group has read it to the end. The coordinator retires it on its own once each group reports the drain. Retiring now skips that wait. Delete the topic in Terraform afterwards.`);body.append(el("div", { class: "row", style: "margin-top: 6px" },el("span", { class: "pill muted mono" }, `${g.topic} (${fmtInt(g.partitions)}, epoch ${fmtInt(g.epoch)}) draining`),groups.map((d) => el("span", { class: `pill ${d.error ? "bad" : isDrained(d) ? "good" : "warn"}`, title: d.error || "" },- `${d.group}: ${fmtInt(d.drained_partitions)}/${fmtInt(d.partitions)}`)),+ `${d.group}: ${fmtInt(d.drained_partitions)}/${fmtInt(d.partitions)}${d.reported ? " · reported" : ""}`)),retire));}if (view.history.length > 0) {diff --git a/rust/personhog-coordination/README.md b/rust/personhog-coordination/README.mdindex 9daadeee7650..f5b2eedfa791 100644--- a/rust/personhog-coordination/README.md+++ b/rust/personhog-coordination/README.md@@ -133,7 +133,7 @@ Freezing holds every partition in place (a held self-handoff that never passes DThe ordinary planner places only when no saga holds placement: no migration, or a Resuming one whose holds are all restarted. Every coordinator plan carries the record it was decided on, and its transactions require that record unchanged, so a stalled pass that wakes after a successor acted lands nothing. The planner's record is its plan basis: the changelog config and the migration record, read in one transaction and compared by the config's mod revision (a flip or retire moves it) and the migration's create revision (a new or archived saga moves it; the driver's own writes do not). A migration pass's record is the saga at the mod revision it read, which any write to it moves: a cancel, a phase change, a blocking-list write, the archive. A saga cancelled before its flip and archived leaves the planner's basis as it was, and a plan landing after it meets the world it was planned for.-A leader serving a partition holds in its cache every acked write for that partition's keys that Postgres does not have yet, so a restart warms from every source that can hold one: for each draining generation, the old partitions congruent to the new one modulo `gcd(old, new)`, from the writer's committed offset to their sealed end and filtered to the keys that hash to the new partition, then the live partition itself. Marks from an old source keep their own stream (`topic`, source partition), so recovery fetches them from the old topic and the prune settles them against the writer's offsets there. A mark also names the cache partition it belongs to, and a release or prune of one partition only removes the marks and lifecycle fences homed there: after a flip, an old partition and a new one on the same pod share keys, and either may be released or pruned first. Nothing in the saga waits for the writer: it drains the oldest undrained generation first, flushes a sealed partition's tail as soon as it reaches the end, and re-subscribes in place once the group has drained the topic; its offsets carry their topic, since a resubscribe reuses the partition numbers. The control plane's retire op removes a generation from `draining` once the writer group and every group in `retire_groups`, such as the relay's, have read each partition to its end; the config watch delivers a retire as it delivers a flip, and a leader settles the marks on a retired stream without the writer's offsets. Deleting the topic stays a Terraform change.+A leader serving a partition holds in its cache every acked write for that partition's keys that Postgres does not have yet, so a restart warms from every source that can hold one: for each draining generation, the old partitions congruent to the new one modulo `gcd(old, new)`, from the writer's committed offset to their sealed end and filtered to the keys that hash to the new partition, then the live partition itself. Marks from an old source keep their own stream (`topic`, source partition), so recovery fetches them from the old topic and the prune settles them against the writer's offsets there. A mark also names the cache partition it belongs to, and a release or prune of one partition only removes the marks and lifecycle fences homed there: after a flip, an old partition and a new one on the same pod share keys, and either may be released or pruned first. Nothing in the saga waits for the writer: it drains the oldest undrained generation first, flushes a sealed partition's tail as soon as it reaches the end, and re-subscribes in place once the group has drained the topic; its offsets carry their topic, since a resubscribe reuses the partition numbers. A generation retires from `draining` once the writer group and every group in `retire_groups`, such as the relay's, have read each partition to its end. Each consumer's follower proves that with the same probe it moves on, then writes a drain report under `changelog/drained/{topic}/{group}`; the coordinator retires the oldest generation every gated group has reported, in a transaction that requires the config unchanged and no saga, and deletes that generation's reports with it. It also deletes reports for any topic no longer draining, since a retired topic never returns. A migration request made while a generation drains waits, naming the groups that have not reported. The control plane's retire op does the same retire after its own probe, as a manual override. The config watch delivers a retire as it delivers a flip, and a leader settles the marks on a retired stream without the writer's offsets. Deleting the topic stays a Terraform change.Routers park requests per key, so a key's requests keep one queue across the flip: a request goes straight to a leader only when its key has nothing parked or in flight and its partition is not stashing, and the live count decides only which partition's drain takes a key. A drain first reads the config from etcd, best effort, then takes the keys that map to its partition under the count the router has adopted; the leader bounces any key it took under an older one, and the reconcile pass drains every pending partition, with no owner when nothing names it; a write for a partition born in the flip parks until that partition's fresh handoff drains it. A router keeps a grow's floor past the retire until its own routing table has a route for every born partition, because its assignment and config watches are independent and the retire can arrive first. A leader answers a stale `x-partition` with `FAILED_PRECONDITION`, which the router treats as a bounce, so the write lands once the router's config watch delivers the flip; a router that has not adopted within its retries answers `UNAVAILABLE`, never an ack; a lifecycle batch with a member off the routed partition is refused with `INVALID_ARGUMENT`, which the identity service answers by retrying each member alone.diff --git a/rust/personhog-coordination/src/coordinator.rs b/rust/personhog-coordination/src/coordinator.rsindex a1469639ccce..766e14fdb678 100644--- a/rust/personhog-coordination/src/coordinator.rs+++ b/rust/personhog-coordination/src/coordinator.rs@@ -658,6 +658,7 @@ impl Coordinator {let migration_stream = self.store.watch_migration_from(anchor).await?;let seal_acks_stream = self.store.watch_seal_acks_from(anchor).await?;let request_stream = self.store.watch_migration_request_from(anchor).await?;+ let drain_reports_stream = self.store.watch_drain_reports_from(anchor).await?;let mut tasks = tokio::task::JoinSet::new();@@ -771,6 +772,7 @@ impl Coordinator {migration_stream,seal_acks_stream,request_stream,+ drain_reports_stream,).await});@@ -879,9 +881,11 @@ impl Coordinator {mut migration_stream: WatchStream,mut seal_acks_stream: WatchStream,mut request_stream: WatchStream,+ mut drain_reports_stream: WatchStream,) -> Result<()> {let _gauge = migration::PhaseGauge;migration::register_transition_metrics();+ migration::register_retire_metrics();loop {tokio::select! {_ = cancel.cancelled() => return Ok(()),@@ -895,6 +899,9 @@ impl Coordinator {msg = request_stream.message() => {util::live_watch_response(msg?, "migration request")?;}+ msg = drain_reports_stream.message() => {+ util::live_watch_response(msg?, "drain report")?;+ }}let deadline = tokio::time::Instant::now() + debounce_interval;loop {@@ -911,6 +918,9 @@ impl Coordinator {msg = request_stream.message() => {util::live_watch_response(msg?, "migration request")?;}+ msg = drain_reports_stream.message() => {+ util::live_watch_response(msg?, "drain report")?;+ }}}Self::drive_migration_static(@@ -933,7 +943,13 @@ impl Coordinator {) -> Result<()> {let Some((migration, mod_revision)) = store.get_migration_with_mod_revision().await? else {migration::record_phase(None);- // No saga: a waiting request may start once the fleet is quiet.+ // No saga: a drained generation retires, then a waiting request may+ // start once nothing drains and the fleet is quiet.+ let draining = match migration::retire_drained(store).await? {+ migration::Retirement::Retired => return Ok(()),+ migration::Retirement::Waiting(waiting) => waiting,+ migration::Retirement::Clear => Vec::new(),+ };let Some((request, request_revision)) =store.get_migration_request_with_mod_revision().await?else {@@ -944,8 +960,15 @@ impl Coordinator {Some(k8s) => members_for_k8s(k8s, &pods, request.to.partitions).await,None => Member::active_all(&active_pod_names(&pods)),};- return migration::consider_request(store, &members, &pods, request, request_revision)- .await;+ return migration::consider_request(+ store,+ &members,+ &pods,+ request,+ request_revision,+ draining,+ )+ .await;};let pods = store.list_pods().await?;let members = match k8s_awareness {diff --git a/rust/personhog-coordination/src/follow.rs b/rust/personhog-coordination/src/follow.rsindex 8823a56cf049..721c8d0b1a41 100644--- a/rust/personhog-coordination/src/follow.rs+++ b/rust/personhog-coordination/src/follow.rs@@ -7,12 +7,13 @@ use std::ops::ControlFlow;use std::sync::Arc;use std::time::Duration;+use metrics::counter;use tokio_util::sync::CancellationToken;use tracing::info;use crate::changelog::{watch_changelog_config_loop, CHANGELOG_RECONCILE_INTERVAL};use crate::store::PersonhogStore;-use crate::types::ChangelogConfig;+use crate::types::{ChangelogConfig, DrainReport};/// How often a consumer on a sealed generation re-checks its group's drain./// Each check reads every partition of the generation under a probe group.@@ -32,6 +33,8 @@ pub trait ChangelogSubscription {#[derive(Default)]pub struct Follower {drained: HashSet<String>,+ /// Proven drained, with no drain report written yet.+ unreported: Vec<String>,}impl Follower {@@ -54,6 +57,7 @@ impl Follower {if !drained(generation.topic.clone()).await? {return Ok(generation.topic);}+ self.unreported.push(generation.topic.clone());self.drained.insert(generation.topic);}Ok(changelog.topic.clone())@@ -97,17 +101,62 @@ impl Follower {info!(from = %subscribed, to = %wanted, "moved the subscription");Ok(Some(subscribed))}++ /// Writes a drain report for every generation proven drained since the+ /// last call; a failed write is kept and retried on the next call.+ pub async fn report(&mut self, store: &PersonhogStore, reporter: &Reporter) {+ for topic in std::mem::take(&mut self.unreported) {+ let report = DrainReport {+ topic: topic.clone(),+ group: reporter.group.clone(),+ reported_by: reporter.pod.clone(),+ reported_at_ms: crate::util::now_millis(),+ };+ match store.put_drain_report(&report).await {+ Ok(()) => {+ counter!("personhog_changelog_drain_reports_total", "outcome" => "written")+ .increment(1);+ info!(topic = %topic, group = %reporter.group, "reported the generation drained");+ }+ Err(e) => {+ counter!("personhog_changelog_drain_reports_total", "outcome" => "failed")+ .increment(1);+ tracing::warn!(topic = %topic, error = %e, "could not report the generation drained; retrying");+ self.unreported.push(topic);+ }+ }+ }+ }+}++/// Who writes a follower's drain reports: the consumer group they speak for, and the pod.+#[derive(Debug, Clone)]+pub struct Reporter {+ pub group: String,+ pub pod: String,+}++impl Reporter {+ /// The pod is named by `HOSTNAME`, which Kubernetes sets to the pod name.+ pub fn for_group(group: &str) -> Self {+ Self {+ group: group.to_string(),+ pod: std::env::var("HOSTNAME").unwrap_or_else(|_| "unknown".to_string()),+ }+ }}/// Follows the config until `cancel`: re-checks on every config change, and on a/// timer that is short while the subscription drains a sealed generation.-/// `on_move` sees each move as (from, to).+/// Reports each generation it proves drained. `on_move` sees each move as (from, to).+#[allow(clippy::too_many_arguments)]pub async fn follow_changelog<S, F, Fut, M>(store: Arc<PersonhogStore>,loaded: ChangelogConfig,mut follower: Follower,subscription: Arc<S>,drained: F,+ reporter: Reporter,on_move: M,cancel: CancellationToken,) where@@ -116,9 +165,11 @@ pub async fn follow_changelog<S, F, Fut, M>(Fut: Future<Output = Result<bool, String>>,M: Fn(&str, &str),{+ counter!("personhog_changelog_drain_reports_total", "outcome" => "written").increment(0);+ counter!("personhog_changelog_drain_reports_total", "outcome" => "failed").increment(0);let (config_tx, mut config_rx) = tokio::sync::watch::channel(loaded.clone());let watch = watch_changelog_config_loop(- store,+ Arc::clone(&store),loaded,CHANGELOG_RECONCILE_INTERVAL,cancel.clone(),@@ -141,6 +192,7 @@ pub async fn follow_changelog<S, F, Fut, M>("could not follow the changelog config; keeping the subscription"),}+ follower.report(&store, &reporter).await;let poll = if subscription.is_draining() {DRAIN_POLL_INTERVAL} else {diff --git a/rust/personhog-coordination/src/migration.rs b/rust/personhog-coordination/src/migration.rsindex cc24a8e5fc14..8c2474c589cc 100644--- a/rust/personhog-coordination/src/migration.rs+++ b/rust/personhog-coordination/src/migration.rs@@ -13,8 +13,8 @@ use crate::protocol::migration::{use crate::store::PersonhogStore;use crate::strategy::{AssignmentStrategy, Member, PlacementPolicy};use crate::types::{- AssignmentPrecondition, HandoffPhase, HandoffReplacement, HandoffState, MigrationPhase,- MigrationRequest, MigrationState, PodStatus, RegisteredPod,+ AssignmentPrecondition, Generation, HandoffPhase, HandoffReplacement, HandoffState,+ MigrationPhase, MigrationRequest, MigrationState, PodStatus, RegisteredPod,};use crate::util;@@ -536,8 +536,6 @@ async fn admission_refusal(let reason = match store.get_changelog_config().await? {None => Some("config_not_seeded"),Some(live) if live.target() != request.from => Some("from_is_not_the_live_config"),- // The protocol is checked with at most one draining generation.- Some(live) if !live.draining.is_empty() => Some("generation_still_draining"),_ if request.to.topic == request.from.topic => Some("same_topic"),_ if members.iter()@@ -557,14 +555,107 @@ async fn admission_refusal(.then_some("topic_already_retired"))}-/// The only way a saga starts: it waits, holding nothing, until the fleet is quiet,-/// then starts in a transaction that proves no handoff appeared meanwhile.+/// What the coordinator's retire pass did.+pub enum Retirement {+ /// A generation left the config; the next pass sees the new config.+ Retired,+ /// Generations still draining, each with the groups that have not reported it.+ Waiting(Vec<String>),+ /// Nothing drains.+ Clear,+}++/// Retires the oldest draining generation once every gated group has reported+/// it drained. Runs only while no saga exists; the retire's transaction requires that too.+pub async fn retire_drained(store: &PersonhogStore) -> Result<Retirement> {+ let Some((config, mod_revision)) = store.get_changelog_config_with_mod_revision().await? else {+ return Ok(Retirement::Clear);+ };+ let reports = store.list_drain_reports().await?;+ // A topic never returns to the config once it leaves `draining`, so a report+ // for any other topic is late and safe to drop.+ let stale: HashSet<&str> = reports+ .iter()+ .map(|r| r.topic.as_str())+ .filter(|topic| config.draining.iter().all(|g| g.topic != *topic))+ .collect();+ for topic in stale {+ store.delete_drain_reports_for(topic).await?;+ counter!("personhog_changelog_drain_reports_swept_total").increment(1);+ }++ let mut generations = config.draining.clone();+ generations.sort_by_key(|g| g.epoch);+ let missing: Vec<(&Generation, Vec<&str>)> = generations+ .iter()+ .map(|generation| {+ let groups = config+ .gated_groups()+ .filter(|group| {+ !reports+ .iter()+ .any(|r| r.topic == generation.topic && r.group == *group)+ })+ .collect();+ (generation, groups)+ })+ .collect();+ // Set on every pass between sagas; during a saga nothing can retire.+ gauge!("personhog_changelog_retire_waiting_groups").set(+ missing+ .iter()+ .map(|(_, groups)| groups.len())+ .sum::<usize>() as f64,+ );++ if let Some((generation, _)) = missing.iter().find(|(_, groups)| groups.is_empty()) {+ if store+ .retire_changelog_topic(&config, mod_revision, &generation.topic)+ .await?+ {+ counter!("personhog_changelog_retires_total", "trigger" => "auto").increment(1);+ tracing::info!(+ topic = %generation.topic,+ groups = ?config.gated_groups().collect::<Vec<_>>(),+ "retired a drained changelog generation"+ );+ }+ return Ok(Retirement::Retired);+ }+ let waiting: Vec<String> = missing+ .iter()+ .map(|(generation, groups)| {+ format!(+ "{}: waiting on drain reports from {}",+ generation.topic,+ groups.join(", ")+ )+ })+ .collect();+ Ok(if waiting.is_empty() {+ Retirement::Clear+ } else {+ Retirement::Waiting(waiting)+ })+}++/// Registers the retire metrics at zero, so the first retire shows as an increase.+pub fn register_retire_metrics() {+ counter!("personhog_changelog_retires_total", "trigger" => "auto").increment(0);+ counter!("personhog_changelog_drain_reports_swept_total").increment(0);+ gauge!("personhog_changelog_retire_waiting_groups").set(0.0);+}++/// The only way a saga starts: it waits, holding nothing, until no generation+/// drains and the fleet is quiet, then starts in a transaction that proves no+/// handoff appeared meanwhile.pub async fn consider_request(store: &PersonhogStore,members: &[Member],pods: &[RegisteredPod],request: MigrationRequest,mod_revision: i64,+ draining: Vec<String>,) -> Result<()> {let now_ms = util::now_millis();// The expiry runs on the coordinator's clock, from its first sighting.@@ -592,6 +683,20 @@ pub async fn consider_request(}return Ok(());}+ // The protocol is checked with at most one draining generation.+ if !draining.is_empty() {+ if draining != request.waiting {+ tracing::info!(migration = %request.id, waiting = ?draining, "migration waiting for a generation to retire");+ let waiting = MigrationRequest {+ waiting: draining,+ ..request+ };+ store+ .update_migration_request(&waiting, mod_revision)+ .await?;+ }+ return Ok(());+ }let (handoffs, quiet_at) = store.list_handoffs_with_revision().await?;let assignments = store.list_assignments().await?;let ready: HashSet<&str> = podsdiff --git a/rust/personhog-coordination/src/store.rs b/rust/personhog-coordination/src/store.rsindex f9df1b18a868..4a2fa6ca65a9 100644--- a/rust/personhog-coordination/src/store.rs+++ b/rust/personhog-coordination/src/store.rs@@ -49,6 +49,9 @@ enum StoreKey<'a> {MigrationHistoryPrefix,SealAck(&'a str),SealAcksPrefix,+ DrainReport { topic: &'a str, group: &'a str },+ DrainReportsForTopic(&'a str),+ DrainReportsPrefix,}impl StoreKey<'_> {@@ -88,6 +91,11 @@ impl StoreKey<'_> {StoreKey::MigrationHistoryPrefix => format!("{prefix}migrations/history/"),StoreKey::SealAck(pod) => format!("{prefix}seal_acks/{pod}"),StoreKey::SealAcksPrefix => format!("{prefix}seal_acks/"),+ StoreKey::DrainReport { topic, group } => {+ format!("{prefix}changelog/drained/{topic}/{group}")+ }+ StoreKey::DrainReportsForTopic(topic) => format!("{prefix}changelog/drained/{topic}/"),+ StoreKey::DrainReportsPrefix => format!("{prefix}changelog/drained/"),}}}diff --git a/rust/personhog-coordination/src/store/migration.rs b/rust/personhog-coordination/src/store/migration.rsindex 3e076c1a4340..849c108e7141 100644--- a/rust/personhog-coordination/src/store/migration.rs+++ b/rust/personhog-coordination/src/store/migration.rs@@ -5,7 +5,8 @@ use etcd_client::{Compare, CompareOp, DeleteOptions, Txn, TxnOp, WatchStream};use super::{count_call, PersonhogStore, StoreKey};use crate::error::Result;use crate::types::{- ChangelogConfig, Generation, MigrationPhase, MigrationRequest, MigrationState, SealAck,+ ChangelogConfig, DrainReport, Generation, MigrationPhase, MigrationRequest, MigrationState,+ SealAck,};impl PersonhogStore {@@ -286,8 +287,10 @@ impl PersonhogStore {Ok(self.inner.txn(txn).await?.succeeded())}- /// Drop a drained generation from the config the caller's drain checks- /// read, only while that config is still current.+ /// Drop a drained generation, and its drain reports, from the config the+ /// caller's drain checks read: only while that config is still current and+ /// no saga exists. A saga completes only once every partition it created+ /// has an owner, which routers rely on once nothing drains.pub async fn retire_changelog_topic(&self,config: &ChangelogConfig,@@ -295,23 +298,55 @@ impl PersonhogStore {topic: &str,) -> Result<bool> {count_call("retire_changelog_topic");- let mut config = config.clone();- let before = config.draining.len();- config.draining.retain(|g| g.topic != topic);- if config.draining.len() == before {+ let mut retired = config.clone();+ retired.draining.retain(|g| g.topic != topic);+ if retired.draining.len() == config.draining.len() {return Ok(false);}let key = self.key(StoreKey::ChangelogConfig);let txn = Txn::new()- .when(vec![Compare::mod_revision(- key.clone(),- CompareOp::Equal,- mod_revision,- )])- .and_then(vec![TxnOp::put(key, serde_json::to_vec(&config)?, None)]);+ .when(vec![+ Compare::mod_revision(key.clone(), CompareOp::Equal, mod_revision),+ Compare::create_revision(self.key(StoreKey::Migration), CompareOp::Equal, 0),+ ])+ .and_then(vec![+ TxnOp::put(key, serde_json::to_vec(&retired)?, None),+ TxnOp::delete(+ self.key(StoreKey::DrainReportsForTopic(topic)),+ Some(DeleteOptions::new().with_prefix()),+ ),+ ]);Ok(self.inner.txn(txn).await?.succeeded())}+ pub async fn put_drain_report(&self, report: &DrainReport) -> Result<()> {+ count_call("put_drain_report");+ let key = self.key(StoreKey::DrainReport {+ topic: &report.topic,+ group: &report.group,+ });+ Ok(self.inner.put(&key, report, None).await?)+ }++ pub async fn list_drain_reports(&self) -> Result<Vec<DrainReport>> {+ count_call("list_drain_reports");+ let key = self.key(StoreKey::DrainReportsPrefix);+ Ok(self.inner.list(&key).await?)+ }++ pub async fn delete_drain_reports_for(&self, topic: &str) -> Result<()> {+ count_call("delete_drain_reports_for");+ let key = self.key(StoreKey::DrainReportsForTopic(topic));+ self.inner.delete_prefix(&key).await?;+ Ok(())+ }++ pub async fn watch_drain_reports_from(&self, start_revision: i64) -> Result<WatchStream> {+ count_call("watch_drain_reports_from");+ let key = self.key(StoreKey::DrainReportsPrefix);+ Ok(self.inner.watch_from(&key, start_revision).await?)+ }+/// Write a request, refused while another request or a saga exists.pub async fn create_migration_request(&self, request: &MigrationRequest) -> Result<bool> {count_call("create_migration_request");diff --git a/rust/personhog-coordination/src/types.rs b/rust/personhog-coordination/src/types.rsindex f4c7d1aad5df..4e1dd76a1ca3 100644--- a/rust/personhog-coordination/src/types.rs+++ b/rust/personhog-coordination/src/types.rs@@ -471,6 +471,16 @@ pub struct MigrationState {pub cancel_reason: Option<String>,}+/// A changelog consumer group's proof that it has read a draining generation+/// to its sealed end; retire waits for one from every group the config names.+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]+pub struct DrainReport {+ pub topic: String,+ pub group: String,+ pub reported_by: String,+ pub reported_at_ms: i64,+}+/// One leader's report of the old partitions it sealed; the flip takes the union.#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]pub struct SealAck {diff --git a/rust/personhog-coordination/tests/migration.rs b/rust/personhog-coordination/tests/migration.rsindex 4a1068df0e6a..9611427e4b82 100644--- a/rust/personhog-coordination/tests/migration.rs+++ b/rust/personhog-coordination/tests/migration.rs@@ -4,12 +4,13 @@ use std::sync::Arc;use std::time::Duration;use personhog_coordination::authority::AuthorityClock;+use personhog_coordination::follow::{Follower, Reporter};use personhog_coordination::pod::{PodConfig, PodHandle};use personhog_coordination::store::PersonhogStore;use personhog_coordination::strategy::StickyBalancedStrategy;use personhog_coordination::types::{- AssignmentPrecondition, AssignmentStatus, ChangelogConfig, ChangelogTarget, Generation,- HandoffPhase, HandoffState, MigrationPhase, MigrationRequest, MigrationState,+ AssignmentPrecondition, AssignmentStatus, ChangelogConfig, ChangelogTarget, DrainReport,+ Generation, HandoffPhase, HandoffState, MigrationPhase, MigrationRequest, MigrationState,PartitionAssignment, SealAck,};use tokio_util::sync::CancellationToken;@@ -667,34 +668,176 @@ async fn a_request_that_fails_admission_is_rejected_with_its_reason() {store.get_changelog_config().await.unwrap().unwrap().epoch,1);+ cancel.cancel();+}- store- .put_changelog_config(&ChangelogConfig {- topic: TOPIC.to_string(),+const TOPIC_V0: &str = "personhog_updates_test_v0";++fn draining_config(retire_groups: &[&str]) -> ChangelogConfig {+ ChangelogConfig {+ topic: TOPIC.to_string(),+ partitions: PARTITIONS,+ epoch: 1,+ writer_group: "writer".to_string(),+ retire_groups: retire_groups.iter().map(|g| g.to_string()).collect(),+ draining: vec![Generation {+ topic: TOPIC_V0.to_string(),partitions: PARTITIONS,- epoch: 1,- writer_group: "writer".to_string(),- retire_groups: vec![],- draining: vec![Generation {- topic: "personhog_updates_test_v0".to_string(),- partitions: PARTITIONS,- epoch: 0,- }],- })+ epoch: 0,+ }],+ }+}++fn drain_report(topic: &str, group: &str) -> DrainReport {+ DrainReport {+ topic: topic.to_string(),+ group: group.to_string(),+ reported_by: format!("{group}-0"),+ reported_at_ms: now_ms(),+ }+}++async fn wait_for_request_waiting(store: &Arc<PersonhogStore>, waiting: &[&str]) {+ wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {+ let store = Arc::clone(store);+ let waiting: Vec<String> = waiting.iter().map(|w| w.to_string()).collect();+ async move {+ store+ .get_migration_request_with_mod_revision()+ .await+ .unwrap()+ .is_some_and(|(r, _)| r.waiting == waiting)+ }+ })+ .await;+}++/// A request made while a generation drains waits, naming the groups that have+/// not reported it; once every gated group has, the coordinator retires the+/// generation, drops its reports and late ones for other topics, and starts the request.+#[tokio::test]+async fn a_drained_generation_retires_once_every_gated_group_reports_it() {+ let Fleet { store, cancel, .. } = fleet("migration-auto-retire").await;+ start_coordinator_with_config(+ Arc::clone(&store),+ coordinator_config("coordinator-0", 10, Duration::from_secs(86_400)),+ Arc::new(StickyBalancedStrategy),+ cancel.clone(),+ );+ wait_for_steady_state(&store, PARTITIONS).await;+ store+ .put_changelog_config(&draining_config(&["relay"])).await.unwrap();+ store+ .put_drain_report(&drain_report("personhog_updates_test_retired", "writer"))+ .await+ .unwrap();+assert!(store- .create_migration_request(&request("m-2", 8))+ .create_migration_request(&request("m-1", 8)).await.unwrap());- let archived = wait_for_rejection(&store, "m-2").await;+ wait_for_request_waiting(+ &store,+ &["personhog_updates_test_v0: waiting on drain reports from writer, relay"],+ )+ .await;++ store+ .put_drain_report(&drain_report(TOPIC_V0, "writer"))+ .await+ .unwrap();+ wait_for_request_waiting(+ &store,+ &["personhog_updates_test_v0: waiting on drain reports from relay"],+ )+ .await;assert_eq!(- archived.blocking,- vec!["generation_still_draining".to_string()]+ store+ .get_changelog_config()+ .await+ .unwrap()+ .unwrap()+ .draining+ .len(),+ 1);++ store+ .put_drain_report(&drain_report(TOPIC_V0, "relay"))+ .await+ .unwrap();+ wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {+ let store = Arc::clone(&store);+ async move { store.get_migration().await.unwrap().is_some() }+ })+ .await;+ let config = store.get_changelog_config().await.unwrap().unwrap();+ assert!(config.draining.is_empty());+ assert_eq!(config.epoch, 1);+ assert!(store.list_drain_reports().await.unwrap().is_empty());cancel.cancel();}+/// The coordinator's retire lands only while no saga exists, and takes the+/// generation's drain reports with it.+#[tokio::test]+async fn a_retire_lands_only_while_no_saga_exists() {+ let store = test_store("migration-retire-guard").await;+ store+ .put_changelog_config(&draining_config(&[]))+ .await+ .unwrap();+ store+ .put_drain_report(&drain_report(TOPIC_V0, "writer"))+ .await+ .unwrap();+ assert!(store.create_migration(&migration("m-1", 8)).await.unwrap());++ let (config, rev) = store+ .get_changelog_config_with_mod_revision()+ .await+ .unwrap()+ .unwrap();+ assert!(!store+ .retire_changelog_topic(&config, rev, TOPIC_V0)+ .await+ .unwrap());+ assert_eq!(+ store+ .get_changelog_config()+ .await+ .unwrap()+ .unwrap()+ .draining+ .len(),+ 1+ );++ let (migration, migration_rev) = store+ .get_migration_with_mod_revision()+ .await+ .unwrap()+ .unwrap();+ assert!(store+ .archive_migration(&migration, migration_rev, MigrationPhase::Cancelled)+ .await+ .unwrap());+ assert!(store+ .retire_changelog_topic(&config, rev, TOPIC_V0)+ .await+ .unwrap());+ assert!(store+ .get_changelog_config()+ .await+ .unwrap()+ .unwrap()+ .draining+ .is_empty());+ assert!(store.list_drain_reports().await.unwrap().is_empty());+}+#[tokio::test]async fn the_flip_waits_for_every_partition_to_be_sealed_and_cannot_be_cancelled() {let Fleet { store, cancel, .. } = fleet("migration-gates").await;@@ -1229,7 +1372,7 @@ async fn a_request_waits_while_an_owner_is_draining() {.await.unwrap().unwrap();- consider_request(&store, &members, &pods, req, rev)+ consider_request(&store, &members, &pods, req, rev, Vec::new()).await.unwrap();}@@ -1260,3 +1403,28 @@ async fn a_request_waits_while_an_owner_is_draining() {pass().await;assert_eq!(store.get_migration().await.unwrap().unwrap().id, "m-1");}++/// A consumer that proves a generation drained, even one already drained when+/// it starts and never subscribed to, reports it for its group.+#[tokio::test]+async fn a_follower_reports_each_generation_it_proves_drained() {+ let store = test_store("migration-drain-report").await;+ let mut follower = Follower::default();+ let topic = follower+ .select_topic(&draining_config(&[]), |_| async { Ok(true) })+ .await+ .unwrap();+ assert_eq!(topic, TOPIC);++ let reporter = Reporter {+ group: "writer".to_string(),+ pod: "writer-0".to_string(),+ };+ follower.report(&store, &reporter).await;+ let reports = store.list_drain_reports().await.unwrap();+ assert_eq!(reports.len(), 1);+ assert_eq!(+ (reports[0].topic.as_str(), reports[0].group.as_str()),+ (TOPIC_V0, "writer")+ );+}diff --git a/rust/personhog-test-harness/Cargo.toml b/rust/personhog-test-harness/Cargo.tomlindex d9ab8e5d50a4..e61abbd1fd78 100644--- a/rust/personhog-test-harness/Cargo.toml+++ b/rust/personhog-test-harness/Cargo.toml@@ -8,7 +8,6 @@ publish = falsepersonhog-common = { path = "../personhog-common" }personhog-proto = { path = "../personhog-proto" }common-metrics = { path = "../common/metrics" }-common-kafka = { path = "../common/kafka" }personhog-coordination = { path = "../personhog-coordination" }assignment-coordination = { path = "../common/assignment-coordination" }diff --git a/rust/personhog-test-harness/src/stack/mod.rs b/rust/personhog-test-harness/src/stack/mod.rsindex b0b27d40b6cb..a2030b7bfa09 100644--- a/rust/personhog-test-harness/src/stack/mod.rs+++ b/rust/personhog-test-harness/src/stack/mod.rs@@ -1314,47 +1314,31 @@ impl Stack {Ok(saga)}- /// Wait for the writer to drain every old generation on its own, then retire each one.+ /// Wait for the coordinator to retire every old generation on its own,+ /// once the writer reports it drained.pub async fn retire_drained(&mut self) -> Result<()> {let pending = std::mem::take(&mut self.pending_retire);for (old_topic, saga_end) in pending {- let drained_at = Instant::now();+ let waited_from = Instant::now();loop {- let base = kafka::base_config(&self.config.kafka_hosts);- let topic = old_topic.clone();- let drained = tokio::task::spawn_blocking(move || {- common_kafka::probe::group_drained(- &base,- WRITER_CONSUMER_GROUP,- &topic,- Duration::from_secs(5),- )- })- .await??;- if drained {+ let draining = self+ .store+ .get_changelog_config()+ .await?+ .is_some_and(|c| c.draining.iter().any(|g| g.topic == old_topic));+ if !draining {break;}ensure!(- drained_at.elapsed() < RETIREMENT_DEADLINE,- "the writer did not drain {old_topic} within {RETIREMENT_DEADLINE:?}"+ waited_from.elapsed() < RETIREMENT_DEADLINE,+ "the coordinator did not retire {old_topic} within {RETIREMENT_DEADLINE:?}");tokio::time::sleep(MIGRATION_POLL).await;}println!(- "Migration: {old_topic} drained by the writer; checked {:.1}s after the saga, waited {:.1}s",+ "Migration: {old_topic} retired by the coordinator; checked {:.1}s after the saga, waited {:.1}s",saga_end.elapsed().as_secs_f64(),- drained_at.elapsed().as_secs_f64()- );- let (live, mod_revision) = self- .store- .get_changelog_config_with_mod_revision()- .await?- .context("config/changelog is not seeded")?;- ensure!(- self.store- .retire_changelog_topic(&live, mod_revision, &old_topic)- .await?,- "{old_topic} was not listed as draining"+ waited_from.elapsed().as_secs_f64());}// The writer re-checks the drain every 2s and flushes every second.diff --git a/rust/personhog-writer/src/main.rs b/rust/personhog-writer/src/main.rsindex 2bbbb3d34a0d..0e10d22c0618 100644--- a/rust/personhog-writer/src/main.rs+++ b/rust/personhog-writer/src/main.rs@@ -9,7 +9,7 @@ use envconfig::Envconfig;use lifecycle::{ComponentOptions, Manager};use metrics::counter;use personhog_coordination::changelog::{load_changelog_config, CHANGELOG_BOOT_POLL_INTERVAL};-use personhog_coordination::follow::{follow_changelog, Follower};+use personhog_coordination::follow::{follow_changelog, Follower, Reporter};use personhog_coordination::store::PersonhogStore;use tokio::sync::mpsc;use tokio_util::sync::CancellationToken;@@ -249,6 +249,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {follower,consumer,|topic: String| drained(&config, topic),+ Reporter::for_group(&config.kafka_consumer_group),|_, _| counter!("personhog_writer_resubscriptions_total").increment(1),cancel.clone(),);