Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(personhog): migrate the changelog to a new topic through a coordinator saga (#119812)

Zach Brown495ee4397a49
diff --git a/rust/Cargo.lock b/rust/Cargo.lock
index 23443538905d..505d632aa9fb 100644
--- a/rust/Cargo.lock
+++ b/rust/Cargo.lock
@@ -6396,6 +6396,7 @@ dependencies = [
"chrono",
"common-alloc",
"common-database",
+ "common-kafka",
"common-metrics",
"common-types",
"dashmap 6.1.0",
@@ -9178,6 +9179,7 @@ dependencies = [
"serde_json",
"sqlx",
"tokio",
+ "tokio-util",
"tracing",
"tracing-subscriber",
"uuid",
diff --git a/rust/common/kafka/src/lib.rs b/rust/common/kafka/src/lib.rs
index 0d21d783e41b..e2849e52fc10 100644
--- a/rust/common/kafka/src/lib.rs
+++ b/rust/common/kafka/src/lib.rs
@@ -3,6 +3,7 @@ pub mod error;
pub mod kafka_consumer;
pub mod kafka_messages;
pub mod kafka_producer;
+pub mod probe;
pub mod test;
pub mod transaction;
diff --git a/rust/common/kafka/src/probe.rs b/rust/common/kafka/src/probe.rs
new file mode 100644
index 000000000000..121121af5650
--- /dev/null
+++ b/rust/common/kafka/src/probe.rs
@@ -0,0 +1,186 @@
+//! Whether a consumer group has consumed a topic's partitions to their end.
+
+use std::collections::HashMap;
+use std::time::{Duration, Instant};
+
+use rdkafka::consumer::{BaseConsumer, Consumer, ConsumerContext};
+use rdkafka::error::KafkaError;
+use rdkafka::message::Message;
+use rdkafka::types::RDKafkaErrorCode;
+use rdkafka::{ClientConfig, ClientContext, Offset, TopicPartitionList};
+
+/// A consumer context that does not log end-of-partition, which the probe polls for.
+pub struct QuietEof;
+
+impl ClientContext for QuietEof {
+ fn error(&self, error: KafkaError, reason: &str) {
+ if !matches!(
+ error,
+ KafkaError::PartitionEOF(_) | KafkaError::Global(RDKafkaErrorCode::PartitionEOF)
+ ) {
+ tracing::error!(%error, reason, "librdkafka");
+ }
+ }
+}
+
+impl ConsumerContext for QuietEof {}
+
+fn client_config(base: &ClientConfig, group_id: &str) -> ClientConfig {
+ let mut cfg = base.clone();
+ cfg.set("group.id", group_id)
+ .set("enable.auto.commit", "false")
+ .set("enable.auto.offset.store", "false")
+ .set("auto.offset.reset", "earliest")
+ .set("isolation.level", "read_committed")
+ .set("enable.partition.eof", "true");
+ cfg
+}
+
+/// Whether `group` has read each partition to its end; transaction markers never count.
+/// A timeout or a missing topic is an error. Blocking.
+pub fn partitions_drained(
+ base: &ClientConfig,
+ group: &str,
+ topic: &str,
+ timeout: Duration,
+) -> Result<HashMap<u32, bool>, KafkaError> {
+ let group_consumer: BaseConsumer = client_config(base, group).create()?;
+ let metadata = group_consumer.fetch_metadata(Some(topic), timeout)?;
+ let partitions: Vec<i32> = metadata
+ .topics()
+ .iter()
+ .find(|t| t.name() == topic)
+ .map(|t| t.partitions().iter().map(|p| p.id()).collect())
+ .unwrap_or_default();
+ if partitions.is_empty() {
+ return Err(KafkaError::MetadataFetch(
+ RDKafkaErrorCode::UnknownTopicOrPartition,
+ ));
+ }
+ let mut tpl = TopicPartitionList::new();
+ for partition in &partitions {
+ tpl.add_partition(topic, *partition);
+ }
+ let committed = group_consumer.committed_offsets(tpl, timeout)?;
+
+ // The probe reads under its own group so it never moves the group's offsets.
+ let probe_group = format!("{group}-drain-probe");
+ let probe: BaseConsumer<QuietEof> =
+ client_config(base, &probe_group).create_with_context(QuietEof)?;
+ let mut assignment = TopicPartitionList::new();
+ for partition in &partitions {
+ let start = match committed
+ .find_partition(topic, *partition)
+ .map(|e| e.offset())
+ {
+ Some(Offset::Offset(offset)) => Offset::Offset(offset),
+ _ => Offset::Beginning,
+ };
+ assignment.add_partition_offset(topic, *partition, start)?;
+ }
+ probe.assign(&assignment)?;
+
+ let mut resolved: HashMap<u32, bool> = HashMap::new();
+ let deadline = Instant::now() + timeout;
+ while resolved.len() < partitions.len() {
+ let remaining = deadline.saturating_duration_since(Instant::now());
+ if remaining.is_zero() {
+ break;
+ }
+ let settled = match probe.poll(remaining) {
+ None => continue,
+ Some(Ok(message)) => {
+ resolved.insert(message.partition() as u32, false);
+ message.partition()
+ }
+ Some(Err(KafkaError::PartitionEOF(partition))) => {
+ resolved.entry(partition as u32).or_insert(true);
+ partition
+ }
+ Some(Err(e)) => return Err(e),
+ };
+ // One answer per partition is all the probe needs; the rest of its records stay unread.
+ let mut done = TopicPartitionList::new();
+ done.add_partition(topic, settled);
+ probe.pause(&done)?;
+ }
+ if resolved.len() < partitions.len() {
+ return Err(KafkaError::MessageConsumption(
+ RDKafkaErrorCode::OperationTimedOut,
+ ));
+ }
+ Ok(resolved)
+}
+
+pub fn group_drained(
+ base: &ClientConfig,
+ group: &str,
+ topic: &str,
+ timeout: Duration,
+) -> Result<bool, KafkaError> {
+ let drained = partitions_drained(base, group, topic, timeout)?;
+ Ok(drained.values().all(|ok| *ok))
+}
+
+#[cfg(test)]
+mod tests {
+ use rdkafka::consumer::CommitMode;
+ use rdkafka::producer::FutureRecord;
+
+ use super::*;
+
+ const TOPIC: &str = "probe-topic";
+ const GROUP: &str = "probe-writer";
+
+ fn commit(bootstrap: &str, partition: i32, offset: i64) {
+ let consumer: BaseConsumer = ClientConfig::new()
+ .set("bootstrap.servers", bootstrap)
+ .set("group.id", GROUP)
+ .set("enable.auto.commit", "false")
+ .create()
+ .expect("group consumer");
+ let mut tpl = TopicPartitionList::new();
+ tpl.add_partition_offset(TOPIC, partition, Offset::Offset(offset))
+ .expect("tpl");
+ consumer.assign(&tpl).expect("assign");
+ consumer.commit(&tpl, CommitMode::Sync).expect("commit");
+ }
+
+ #[tokio::test]
+ async fn a_partition_is_drained_only_once_the_group_committed_its_end() {
+ let (cluster, producer) = crate::test::create_mock_kafka().await;
+ cluster.create_topic(TOPIC, 2, 1).expect("create topic");
+ for _ in 0..3 {
+ producer
+ .send_result(FutureRecord::<(), _>::to(TOPIC).partition(0).payload("x"))
+ .expect("enqueue")
+ .await
+ .expect("produce canceled")
+ .expect("produce failed");
+ }
+ let bootstrap = cluster.bootstrap_servers();
+ let mut base = ClientConfig::new();
+ base.set("bootstrap.servers", &bootstrap);
+ let timeout = Duration::from_secs(10);
+ let drained = || {
+ let base = base.clone();
+ move || partitions_drained(&base, GROUP, TOPIC, timeout).expect("probe")
+ };
+
+ let none = tokio::task::spawn_blocking(drained()).await.unwrap();
+ assert_eq!(none, HashMap::from([(0, false), (1, true)]));
+
+ commit(&bootstrap, 0, 1);
+ let partial = tokio::task::spawn_blocking(drained()).await.unwrap();
+ assert!(!partial[&0]);
+
+ commit(&bootstrap, 0, 3);
+ let base_for_group = base.clone();
+ let all = tokio::task::spawn_blocking(move || {
+ group_drained(&base_for_group, GROUP, TOPIC, timeout).expect("probe")
+ })
+ .await
+ .unwrap();
+ assert!(all);
+ }
+}
diff --git a/rust/ingestion-control-plane/Cargo.toml b/rust/ingestion-control-plane/Cargo.toml
index 98589781e4f4..6c3baa31d6f7 100644
--- a/rust/ingestion-control-plane/Cargo.toml
+++ b/rust/ingestion-control-plane/Cargo.toml
@@ -22,6 +22,7 @@ health = { path = "../common/health" }
k8s-awareness = { path = "../common/k8s-awareness" }
kube = { version = "0.98", features = ["client"] }
personhog-coordination = { path = "../personhog-coordination" }
+common-kafka = { path = "../common/kafka" }
rdkafka = { workspace = true }
reqwest = { workspace = true }
rustls = { version = "0.23", default-features = false, features = ["ring", "std"] }
diff --git a/rust/ingestion-control-plane/README.md b/rust/ingestion-control-plane/README.md
index a45997e395d8..6f075a1bd3fd 100644
--- a/rust/ingestion-control-plane/README.md
+++ b/rust/ingestion-control-plane/README.md
@@ -23,7 +23,7 @@ Lists live ingestion-consumer pods and serves the consumer's routing debug UI pe
### Personhog topology
-Protocol-aware view of personhog's etcd coordination state (`PERSONHOG_ETCD_PREFIX`, default `/personhog/`): the whole prefix is read in one consistent snapshot and interpreted with the `personhog-coordination` types and the coordinator's own pure predicates, so the derived diagnostics cannot drift from the protocol. Shows the elected coordinator, registered pods and routers (with remaining lease TTLs), per-partition ownership, and in-flight handoffs with their phase age and what each is waiting on — distinguishing a stuck participant ("waiting on freeze acks from router-2") from a stuck coordinator ("quorum met; waiting on the coordinator to advance"). Derived issues cover stuck handoffs past their per-phase deadline, assignments owned by unregistered pods with no replanning handoff, lingering `Complete` records the coordinator hasn't cleaned up, dead new owners awaiting cancellation, unassigned partitions, stale acks from previous handoff attempts, and unparseable records. The deadline knobs (`PERSONHOG_HANDOFF_DEADLINE_SECS`, `PERSONHOG_WARMING_DEADLINE_SECS`) must match the coordinator's to agree with its cancellation decisions. The tool writes no coordination state except seeding `config/changelog`, and a force-reassignment must go through a handoff record, never a raw assignment write.
+Protocol-aware view of personhog's etcd coordination state (`PERSONHOG_ETCD_PREFIX`, default `/personhog/`): the whole prefix is read in one consistent snapshot and interpreted with the `personhog-coordination` types and the coordinator's own pure predicates, so the derived diagnostics cannot drift from the protocol. Shows the elected coordinator, registered pods and routers (with remaining lease TTLs), per-partition ownership, and in-flight handoffs with their phase age and what each is waiting on — distinguishing a stuck participant ("waiting on freeze acks from router-2") from a stuck coordinator ("quorum met; waiting on the coordinator to advance"). Derived issues cover stuck handoffs past their per-phase deadline, assignments owned by unregistered pods with no replanning handoff, lingering `Complete` records the coordinator hasn't cleaned up, dead new owners awaiting cancellation, unassigned partitions, stale acks from previous handoff attempts, and unparseable records. The deadline knobs (`PERSONHOG_HANDOFF_DEADLINE_SECS`, `PERSONHOG_WARMING_DEADLINE_SECS`) must match the coordinator's to agree with its cancellation decisions. The tool writes no coordination state except from the changelog panel (seeding `config/changelog`, requesting or cancelling a migration, and retiring a drained generation), and a force-reassignment must go through a handoff record, never a raw assignment write.
### Personhog parity
diff --git a/rust/ingestion-control-plane/src/api.rs b/rust/ingestion-control-plane/src/api.rs
index 341c4f7553cc..d87307b91c15 100644
--- a/rust/ingestion-control-plane/src/api.rs
+++ b/rust/ingestion-control-plane/src/api.rs
@@ -6,9 +6,9 @@ use anyhow::Context;
use axum::extract::{Path, Query, State};
use axum::http::StatusCode;
use axum::response::{IntoResponse, Response};
-use axum::routing::{get, post};
+use axum::routing::{delete, get, post};
use axum::{Json, Router};
-use personhog_coordination::types::ChangelogConfig;
+use personhog_coordination::types::{ChangelogConfig, MigrationRequest, MigrationState};
use serde::{Deserialize, Serialize};
use serde_json::json;
use uuid::Uuid;
@@ -471,6 +471,65 @@ async fn personhog_seed_changelog(
Ok(Json(seeded))
}
+#[derive(Deserialize)]
+struct StartMigrationRequest {
+ topic: String,
+}
+
+async fn personhog_start_migration(
+ State(state): State<AppState>,
+ Json(request): Json<StartMigrationRequest>,
+) -> Result<Json<MigrationRequest>, ApiError> {
+ let prefix = state.config.personhog_etcd_prefix.clone();
+ validated_etcd_key(&state, &prefix)?;
+ let topic = request.topic.trim();
+ if topic.is_empty() {
+ return Err(ApiError::bad_request("topic must not be empty"));
+ }
+ let client = etcd_client(&state).await?;
+ let migration = changelog::start_migration(&state.config, &client, &prefix, topic)
+ .await
+ .map_err(|e| ApiError::bad_request(format!("migration refused: {e:#}")))?;
+ Ok(Json(migration))
+}
+
+#[derive(Deserialize)]
+struct RetireTopicRequest {
+ topic: String,
+}
+
+async fn personhog_retire_topic(
+ State(state): State<AppState>,
+ Json(request): Json<RetireTopicRequest>,
+) -> Result<Json<serde_json::Value>, ApiError> {
+ let prefix = state.config.personhog_etcd_prefix.clone();
+ validated_etcd_key(&state, &prefix)?;
+ let client = etcd_client(&state).await?;
+ changelog::retire(&state.config, &client, &prefix, request.topic.trim())
+ .await
+ .map_err(|e| ApiError::bad_request(format!("retire refused: {e:#}")))?;
+ Ok(Json(json!({ "retired": request.topic.trim() })))
+}
+
+#[derive(Deserialize)]
+struct StopMigrationRequest {
+ /// The request or saga the operator saw; anything else is left alone.
+ id: String,
+}
+
+async fn personhog_cancel_migration(
+ State(state): State<AppState>,
+ Json(request): Json<StopMigrationRequest>,
+) -> Result<Json<MigrationState>, ApiError> {
+ let prefix = state.config.personhog_etcd_prefix.clone();
+ validated_etcd_key(&state, &prefix)?;
+ let client = etcd_client(&state).await?;
+ let migration = changelog::cancel_migration(&client, &prefix, &request.id)
+ .await
+ .map_err(|e| ApiError::bad_request(format!("cancel refused: {e:#}")))?;
+ Ok(Json(migration))
+}
+
#[derive(Serialize)]
struct PodsResponse {
pods: Vec<DiscoveredPod>,
@@ -509,6 +568,15 @@ pub fn router(state: AppState) -> Router {
"/api/personhog/changelog/seed",
post(personhog_seed_changelog),
)
+ .route("/api/personhog/migrations", post(personhog_start_migration))
+ .route(
+ "/api/personhog/migrations/current",
+ delete(personhog_cancel_migration),
+ )
+ .route(
+ "/api/personhog/changelog/retire",
+ post(personhog_retire_topic),
+ )
// The consumer debug UI is served per pod; its relative `debug/...`
// fetches resolve to the proxy route below. Pods are addressed by
// namespace + name so identical names across lanes cannot collide.
diff --git a/rust/ingestion-control-plane/src/changelog.rs b/rust/ingestion-control-plane/src/changelog.rs
index 7ed962059913..994ad7c6dc18 100644
--- a/rust/ingestion-control-plane/src/changelog.rs
+++ b/rust/ingestion-control-plane/src/changelog.rs
@@ -4,12 +4,14 @@ use anyhow::{bail, Context};
use assignment_coordination::store::{EtcdStore, StoreConfig};
use etcd_client::Client;
use personhog_coordination::store::PersonhogStore;
-use personhog_coordination::types::ChangelogConfig;
+use personhog_coordination::types::{
+ ChangelogConfig, ChangelogTarget, MigrationPhase, MigrationRequest, MigrationState,
+};
use rdkafka::consumer::Consumer;
use serde::Serialize;
use crate::config::Config;
-use crate::kafka::client::metadata_consumer;
+use crate::kafka::client::{base_client_config, metadata_consumer};
use crate::kafka::lag::{scan_group_lag, ConsumerTarget};
fn store(client: &Client, prefix: &str) -> PersonhogStore {
@@ -62,7 +64,7 @@ pub async fn seed(
.seed_changelog_config(topic, partitions, writer_group)
.await?;
if !created {
- bail!("config/changelog is already seeded");
+ bail!("config/changelog is already seeded; only a migration may move it");
}
Ok(ChangelogConfig {
writer_group: writer_group.to_string(),
@@ -76,6 +78,7 @@ pub struct GroupLag {
pub partitions: usize,
pub lagging_partitions: usize,
pub total_lag: i64,
+ /// The worst single partition's lag: what one restart replays in a grow.
pub max_partition_lag: i64,
pub error: Option<String>,
}
@@ -84,12 +87,216 @@ pub struct GroupLag {
pub struct ChangelogView {
pub config: Option<ChangelogConfig>,
pub writer: Option<GroupLag>,
+ pub migration: Option<MigrationState>,
+ pub request: Option<MigrationRequest>,
+ pub history: Vec<MigrationState>,
+ pub draining: Vec<DrainingView>,
+}
+
+#[derive(Serialize)]
+pub struct GroupDrain {
+ pub group: String,
+ pub drained_partitions: usize,
+ pub partitions: usize,
+ pub error: Option<String>,
+}
+
+/// A draining generation and how far the writer group has read it.
+#[derive(Serialize)]
+pub struct DrainingView {
+ pub topic: String,
+ pub partitions: u32,
+ pub epoch: u64,
+ pub writer: GroupDrain,
+}
+
+async fn group_drain(config: &Config, group: &str, topic: &str) -> GroupDrain {
+ let base = base_client_config(config);
+ let timeout = Duration::from_millis(config.kafka_metadata_timeout_ms);
+ let (group_owned, topic_owned) = (group.to_string(), topic.to_string());
+ let probed = tokio::task::spawn_blocking(move || {
+ common_kafka::probe::partitions_drained(&base, &group_owned, &topic_owned, timeout)
+ })
+ .await;
+ match probed {
+ Ok(Ok(drained)) => GroupDrain {
+ group: group.to_string(),
+ drained_partitions: drained.values().filter(|ok| **ok).count(),
+ partitions: drained.len(),
+ error: None,
+ },
+ Ok(Err(e)) => GroupDrain {
+ group: group.to_string(),
+ drained_partitions: 0,
+ partitions: 0,
+ error: Some(e.to_string()),
+ },
+ Err(e) => GroupDrain {
+ group: group.to_string(),
+ drained_partitions: 0,
+ partitions: 0,
+ error: Some(format!("probe task panicked: {e}")),
+ },
+ }
+}
+
+/// Drop a drained generation from the config; deleting the topic stays a Terraform change.
+pub async fn retire(
+ config: &Config,
+ client: &Client,
+ prefix: &str,
+ topic: &str,
+) -> anyhow::Result<()> {
+ let config = &config.personhog_kafka();
+ let store = store(client, prefix);
+ let live = store
+ .get_changelog_config()
+ .await?
+ .context("config/changelog is not seeded")?;
+ if !live.draining.iter().any(|g| g.topic == topic) {
+ bail!("{topic} is not a draining generation");
+ }
+ let group = &live.writer_group;
+ let drain = group_drain(config, group, topic).await;
+ if let Some(error) = drain.error {
+ bail!("group {group}: {error}");
+ }
+ if drain.partitions == 0 || drain.drained_partitions < drain.partitions {
+ bail!(
+ "group {group} has read {} of {} partitions of {topic} to the end",
+ drain.drained_partitions,
+ drain.partitions
+ );
+ }
+ if !store.retire_changelog_topic(topic).await? {
+ bail!("the config moved; refresh and retry");
+ }
+ Ok(())
+}
+
+/// Request a migration; the coordinator starts it once the fleet is quiet.
+/// Writer lag does not block it; the panel shows the lag to the operator.
+pub async fn start_migration(
+ config: &Config,
+ client: &Client,
+ prefix: &str,
+ topic: &str,
+) -> anyhow::Result<MigrationRequest> {
+ let config = &config.personhog_kafka();
+ let store = store(client, prefix);
+ let live = store
+ .get_changelog_config()
+ .await?
+ .context("config/changelog is not seeded")?;
+ if live.topic == topic {
+ bail!("{topic} is already the live changelog");
+ }
+ let partitions = topic_partition_count(config, topic).await?;
+ let now_ms = assignment_coordination::util::now_millis();
+ let request = MigrationRequest {
+ id: format!("m-{now_ms}"),
+ from: live.target(),
+ to: ChangelogTarget {
+ topic: topic.to_string(),
+ partitions,
+ },
+ requested_at_ms: now_ms,
+ seen_at_ms: None,
+ waiting: Vec::new(),
+ };
+ if !store.create_migration_request(&request).await? {
+ bail!("a migration is already requested or in flight");
+ }
+ Ok(request)
+}
+
+const WITHDRAW_ATTEMPTS: usize = 5;
+
+/// Withdraw a waiting request, or cancel a running saga before Sealing.
+pub async fn cancel_migration(
+ client: &Client,
+ prefix: &str,
+ id: &str,
+) -> anyhow::Result<MigrationState> {
+ let store = store(client, prefix);
+ // The coordinator rewrites a waiting request as handoffs move, so a withdraw
+ // retries on that race; if this request started instead, it is cancelled below.
+ for _ in 0..WITHDRAW_ATTEMPTS {
+ let Some((request, mod_revision)) = store.get_migration_request_with_mod_revision().await?
+ else {
+ break;
+ };
+ if request.id != id {
+ bail!(
+ "the waiting request is {}, not {id}; refresh and retry",
+ request.id
+ );
+ }
+ if store
+ .reject_migration_request(&request, mod_revision, "withdrawn")
+ .await?
+ {
+ return Ok(request.rejected("withdrawn", assignment_coordination::util::now_millis()));
+ }
+ }
+ let Some((migration, mod_revision)) = store.get_migration_with_mod_revision().await? else {
+ if let Some(ended) = store
+ .list_migration_history()
+ .await?
+ .into_iter()
+ .find(|m| m.id == id)
+ {
+ bail!(
+ "{id} already ended: {:?} {}",
+ ended.phase,
+ ended.blocking.join("; ")
+ );
+ }
+ bail!("no migration is in flight");
+ };
+ if migration.id != id {
+ bail!(
+ "the running migration is {}, not {id}; refresh and retry",
+ migration.id
+ );
+ }
+ if !migration
+ .phase
+ .may_transition_to(MigrationPhase::Cancelling)
+ {
+ bail!("a migration in {:?} cannot be cancelled", migration.phase);
+ }
+ if !store
+ .cancel_migration(&migration.id, mod_revision, "operator")
+ .await?
+ {
+ bail!("the migration moved on; refresh and retry");
+ }
+ Ok(migration)
}
pub async fn view(config: &Config, client: &Client, prefix: &str) -> anyhow::Result<ChangelogView> {
let config = &config.personhog_kafka();
let store = store(client, prefix);
let changelog = store.get_changelog_config().await?;
+ let migration = store.get_migration().await?;
+ let request = store
+ .get_migration_request_with_mod_revision()
+ .await?
+ .map(|(request, _)| request);
+ let mut history = store.list_migration_history().await?;
+ history.sort_by_key(|m| std::cmp::Reverse(m.created_at_ms));
+ let mut draining = Vec::new();
+ if let Some(changelog) = &changelog {
+ for generation in &changelog.draining {
+ draining.push(DrainingView {
+ topic: generation.topic.clone(),
+ partitions: generation.partitions,
+ epoch: generation.epoch,
+ writer: group_drain(config, &changelog.writer_group, &generation.topic).await,
+ });
+ }
+ }
let writer = match &changelog {
Some(changelog) => Some(group_lag(config, &changelog.writer_group, &changelog.topic).await),
None => None,
@@ -97,6 +304,10 @@ pub async fn view(config: &Config, client: &Client, prefix: &str) -> anyhow::Res
Ok(ChangelogView {
config: changelog,
writer,
+ migration,
+ request,
+ history,
+ draining,
})
}
diff --git a/rust/ingestion-control-plane/src/kafka/client.rs b/rust/ingestion-control-plane/src/kafka/client.rs
index 06773a9ff2e5..d3d2ef571379 100644
--- a/rust/ingestion-control-plane/src/kafka/client.rs
+++ b/rust/ingestion-control-plane/src/kafka/client.rs
@@ -8,7 +8,7 @@ use crate::config::Config;
/// ingestion group so inspection can never touch production offsets.
pub const INSPECTOR_GROUP_ID: &str = "ingestion-control-plane-inspector";
-fn base_client_config(config: &Config) -> ClientConfig {
+pub fn base_client_config(config: &Config) -> ClientConfig {
let mut cfg = ClientConfig::new();
cfg.set("bootstrap.servers", &config.kafka_hosts);
// Certificate verification is disabled against MSK across the whole
diff --git a/rust/ingestion-control-plane/src/personhog.rs b/rust/ingestion-control-plane/src/personhog.rs
index 5ffd2b9fe004..5e4e1036a041 100644
--- a/rust/ingestion-control-plane/src/personhog.rs
+++ b/rust/ingestion-control-plane/src/personhog.rs
@@ -131,6 +131,8 @@ pub fn parse_snapshot(kvs: &[RawKv], prefix: &str, revision: i64) -> RawSnapshot
if let Some(changelog) = parse::<ChangelogConfig>(kv, errors) {
snapshot.changelog = Some(changelog);
}
+ } else if suffix.starts_with("migrations/") || suffix.starts_with("seal_acks/") {
+ // The changelog panel reads these.
} else {
snapshot.unknown_keys.push(kv.key.clone());
}
@@ -734,6 +736,17 @@ mod tests {
.collect()
}
+ #[test]
+ fn migration_keys_are_part_of_the_layout() {
+ let kvs = vec![
+ changelog_kv(1),
+ kv("migrations/current", serde_json::json!({}), 0),
+ kv("migrations/history/m-1", serde_json::json!({}), 0),
+ kv("seal_acks/pod-a", serde_json::json!({}), 0),
+ ];
+ assert!(parse_snapshot(&kvs, PREFIX, 42).unknown_keys.is_empty());
+ }
+
#[test]
fn a_stuck_freezing_handoff_names_the_missing_router() {
let kvs = vec![
diff --git a/rust/ingestion-control-plane/src/ui/index.html b/rust/ingestion-control-plane/src/ui/index.html
index 1081a38987d2..d76d6885beee 100644
--- a/rust/ingestion-control-plane/src/ui/index.html
+++ b/rust/ingestion-control-plane/src/ui/index.html
@@ -1341,9 +1341,11 @@
this.ui = {
summary: el("div", { class: "row", style: "padding: 0 0 10px" }),
changelogStatus: el("div", {}, el("div", { class: "empty" }, "Loading…")),
- // Rebuilt only when the config's presence changes, so a refresh keeps what is typed.
+ // Rebuilt only when the action it offers changes, so a refresh keeps what is typed.
changelogForm: el("div"),
changelogFormKey: null,
+ changelogError: el("div", { style: "padding: 0 14px" }),
+ changelogDetail: el("div"),
issuesBody: el("div", {}, el("div", { class: "empty" }, "Loading…")),
partitionsBody: el("div", {}, el("div", { class: "empty" }, "Loading…")),
podsBody: el("div", {}, el("div", { class: "empty" }, "Loading…")),
@@ -1361,7 +1363,8 @@
});
root.append(
this.ui.summary,
- el("div", { class: "panel" }, el("h2", {}, "Changelog"), this.ui.changelogStatus, this.ui.changelogForm),
+ el("div", { class: "panel" }, el("h2", {}, "Changelog"),
+ this.ui.changelogStatus, this.ui.changelogForm, this.ui.changelogError, this.ui.changelogDetail),
el("div", { class: "panel" }, el("h2", {}, "Issues"), this.ui.issuesBody),
el("div", { class: "panel" }, el("h2", {}, "Partitions"), this.ui.partitionsBody),
el("div", { class: "panel" }, el("h2", {}, "Pods"), this.ui.podsBody),
@@ -1390,14 +1393,16 @@
},
renderChangelog(view) {
- const formKey = view.config ? "seeded" : "unseeded";
+ const formKey = this.changelogFormKeyOf(view);
if (this.ui.changelogFormKey !== formKey) {
this.ui.changelogFormKey = formKey;
- this.ui.changelogForm.replaceChildren(...(view.config ? [] : [this.seedForm()]));
+ this.ui.changelogError.replaceChildren();
+ this.ui.changelogForm.replaceChildren(view.config ? this.migrationForm(view) : this.seedForm());
}
if (!view.config) {
this.ui.changelogStatus.replaceChildren(
el("div", { class: "muted", style: "padding: 6px 14px 8px" }, "Not seeded. Every personhog service waits at startup until the changelog is seeded."));
+ this.ui.changelogDetail.replaceChildren();
return;
}
const c = view.config;
@@ -1405,13 +1410,15 @@
el("div", { class: "row", style: "padding: 6px 14px" },
el("span", { class: "pill accent mono" }, c.topic),
el("span", { class: "pill muted" }, `${fmtInt(c.partitions)} partitions`),
- el("span", { class: "pill muted" }, `epoch ${fmtInt(c.epoch)}`)),
+ el("span", { class: "pill muted" }, `epoch ${fmtInt(c.epoch)}`)));
+ this.ui.changelogDetail.replaceChildren(
+ this.migrationDetail(view),
!view.writer
? el("div", { class: "empty" }, "No writer group.")
: el("table", {},
el("thead", {}, el("tr", {},
el("th", {}, "writer group"), el("th", {}, "partitions"), el("th", {}, "lagging"), el("th", {}, "total lag"),
- el("th", {}, "worst partition"))),
+ el("th", { title: "A grow's restart replays about this many records per partition; a shrink by k replays about k partitions' worth. Larger replays lengthen the pause." }, "worst partition"))),
el("tbody", {}, [view.writer].map((g) =>
el("tr", {},
el("td", { class: "mono" }, g.group),
@@ -1421,6 +1428,24 @@
el("td", { class: "mono" }, g.error ? "–" : fmtInt(g.max_partition_lag)))))));
},
+ changelogFormKeyOf(view) {
+ if (!view.config) return "unseeded";
+ if (view.migration) return `migration:${view.migration.id}:${view.migration.phase}`;
+ if (view.request) return `request:${view.request.id}`;
+ return "idle";
+ },
+
+ async postChangelogAction(path, options, confirmText) {
+ if (!window.confirm(confirmText)) return;
+ try {
+ await fetchJSON(path, options);
+ this.ui.changelogError.replaceChildren();
+ this.refresh();
+ } catch (e) {
+ this.ui.changelogError.replaceChildren(el("div", { class: "error-box" }, String(e)));
+ }
+ },
+
seedForm() {
const topicInput = el("input", { type: "text", placeholder: "changelog topic, e.g. personhog_updates_64", style: "width: 360px" });
const groupInput = el("input", { type: "text", placeholder: "writer consumer group, e.g. personhog-writer", style: "width: 280px" });
@@ -1433,7 +1458,7 @@
error.replaceChildren(el("div", { class: "error-box" }, "Enter the topic and the writer's consumer group."));
return;
}
- if (!window.confirm(`Seed config/changelog with topic ${topic}, with writer group ${writer_group}?\n\nThe partition count is read from the topic's metadata. Every personhog participant adopts this object.`)) return;
+ if (!window.confirm(`Seed config/changelog with topic ${topic}, with writer group ${writer_group}?\n\nThe partition count is read from the topic's metadata. Every personhog participant adopts this object; only a migration can move it afterward.`)) return;
button.disabled = true;
try {
await fetchJSON("api/personhog/changelog/seed", {
@@ -1453,6 +1478,97 @@
error);
},
+ migrationForm(view) {
+ const m = view.migration;
+ const r = view.request;
+ const form = el("div", { class: "row", style: "padding: 6px 14px; border-top: 1px solid var(--border)" });
+ if (m) {
+ const cancellable = ["Freezing", "Draining"].includes(m.phase);
+ const cancel = el("button", { disabled: !cancellable }, "Cancel migration");
+ cancel.onclick = () => this.postChangelogAction("api/personhog/migrations/current", {
+ method: "DELETE",
+ headers: { "content-type": "application/json" },
+ body: JSON.stringify({ id: m.id }),
+ },
+ `Cancel the migration to ${m.to.topic}? Held partitions resume on ${m.from.topic}.`);
+ form.append(
+ el("span", { class: "pill warn" }, `migrating: ${m.phase}`),
+ el("span", { class: "mono" }, `${m.from.topic} (${fmtInt(m.from.partitions)}) → ${m.to.topic} (${fmtInt(m.to.partitions)})`),
+ cancel);
+ } else if (r) {
+ const withdraw = el("button", {}, "Stop migration");
+ withdraw.onclick = () => this.postChangelogAction("api/personhog/migrations/current", {
+ method: "DELETE",
+ headers: { "content-type": "application/json" },
+ body: JSON.stringify({ id: r.id }),
+ },
+ `Stop the migration to ${r.to.topic}? It has not started, so nothing is held; if it starts in the meantime, it is cancelled before the flip.`);
+ form.append(
+ el("span", { class: "pill muted" }, "requested: waiting to start"),
+ el("span", { class: "mono" }, `${r.from.topic} (${fmtInt(r.from.partitions)}) → ${r.to.topic} (${fmtInt(r.to.partitions)})`),
+ withdraw);
+ } else {
+ const topicInput = el("input", { type: "text", placeholder: "target topic, e.g. personhog_updates_128", style: "width: 360px" });
+ const start = el("button", {}, "Migrate changelog");
+ start.onclick = () => {
+ const topic = topicInput.value.trim();
+ if (!topic) return;
+ this.postChangelogAction("api/personhog/migrations", {
+ method: "POST",
+ headers: { "content-type": "application/json" },
+ body: JSON.stringify({ topic }),
+ }, `Migrate the changelog to ${topic}?\n\nThe coordinator starts it once no partition is moving and every owner is ready, then every partition pauses briefly while the fleet restarts onto the new topic. The partition count is read from the topic's metadata.`);
+ };
+ form.append(topicInput, start);
+ }
+ return form;
+ },
+
+ migrationDetail(view) {
+ const m = view.migration;
+ const r = view.request;
+ const body = el("div", { style: "padding: 0 14px 6px" });
+ if (m) {
+ body.append(m.blocking.length === 0
+ ? el("div", { class: "muted" }, "Nothing blocking.")
+ : el("ul", { class: "mono", style: "margin: 4px 0" }, m.blocking.slice(0, 12).map((b) => el("li", {}, b))));
+ } else if (r) {
+ body.append(r.waiting.length === 0
+ ? el("div", { class: "muted" }, "Starting once no partition is moving and every owner is ready.")
+ : el("ul", { class: "mono", style: "margin: 4px 0" }, r.waiting.slice(0, 12).map((w) => el("li", {}, w))));
+ }
+ for (const g of view.draining) {
+ const w = g.writer;
+ const drained = !w.error && w.partitions > 0 && w.drained_partitions === w.partitions;
+ const retire = el("button", { disabled: !drained }, `Retire ${g.topic}`);
+ retire.onclick = () => this.postChangelogAction("api/personhog/changelog/retire", {
+ method: "POST",
+ headers: { "content-type": "application/json" },
+ body: JSON.stringify({ topic: g.topic }),
+ }, `Retire ${g.topic}?\n\nThe writer group has read it to the end. It leaves the config; delete the topic in Terraform afterwards.`);
+ body.append(el("div", { class: "row", style: "margin-top: 6px" },
+ el("span", { class: "pill muted mono" }, `${g.topic} (${fmtInt(g.partitions)}, epoch ${fmtInt(g.epoch)}) draining`),
+ el("span", { class: `pill ${w.error ? "bad" : drained ? "good" : "warn"}`, title: w.error || "" },
+ `${w.group}: ${fmtInt(w.drained_partitions)}/${fmtInt(w.partitions)}`),
+ retire));
+ }
+ if (view.history.length > 0) {
+ body.append(el("table", {},
+ el("thead", {}, el("tr", {}, el("th", {}, "migration"), el("th", {}, "from"), el("th", {}, "to"), el("th", {}, "outcome"))),
+ el("tbody", {}, view.history.slice(0, 5).map((h) =>
+ el("tr", {},
+ el("td", { class: "mono" }, h.id),
+ el("td", { class: "mono" }, `${h.from.topic} (${fmtInt(h.from.partitions)})`),
+ el("td", { class: "mono" }, `${h.to.topic} (${fmtInt(h.to.partitions)})`),
+ el("td", {},
+ el("span", { class: `pill ${h.phase === "Complete" ? "good" : "bad"}`, title: h.blocking.join("; ") }, h.phase),
+ h.cancel_reason ? el("span", { class: "muted mono", style: "margin-left: 8px" }, h.cancel_reason)
+ : h.phase !== "Complete" && h.blocking.length > 0 ? el("span", { class: "muted mono", style: "margin-left: 8px" }, h.blocking.join("; ")) : null,
+ h.phase === "Complete" && !view.draining.some((g) => g.topic === h.from.topic) ? el("span", { class: "muted", style: "margin-left: 8px" }, `${h.from.topic} is retired; delete it in Terraform`) : null))))));
+ }
+ return body;
+ },
+
render(data) {
const readyPods = data.pods.filter((p) => p.status === "Ready").length;
const errors = data.issues.filter((issue) => issue.severity === "error").length;
diff --git a/rust/personhog-coordination/README.md b/rust/personhog-coordination/README.md
index 4ab0cbdd1e7e..e8b37a406a57 100644
--- a/rust/personhog-coordination/README.md
+++ b/rust/personhog-coordination/README.md
@@ -90,7 +90,7 @@ Several failure modes are handled explicitly:
- **Chunked plan application**: A plan is applied in transactions sized to etcd's `--max-txn-ops` (`ETCD_MAX_TXN_OPS`, default 128); an unchunked over-budget plan would be rejected outright and retried forever. Atomicity is per partition — each unit keeps its guards and writes in one transaction — so a conflicted unit stands down alone and a partially applied plan is placement drift the next replan heals. Stateright explores partial application (`chunked_plans`).
- **Phase-aware deadline**: Time-in-phase is bounded, not total age — wedged is a property of a phase. Freezing and Draining wait only on acknowledgements and share a tight budget (`COORDINATOR_HANDOFF_DEADLINE_SECS`, default 120 s); Warming replays a changelog whose length scales with the partition and gets its own generous budget (`COORDINATOR_WARMING_DEADLINE_SECS`, default 1800 s, 0 disables). A total-age deadline would cancel a legitimately long warm and restart it from zero, forever. The reconcile tick detects both cancellation triggers and wakes the planning loop; the replacement itself is a planning decision.
- **Participant stall watchdog**: A router's registration is lease-backed, and the lease keepalive is its own task — so a router whose handoff watch loop has stalled would stay registered (counted in freeze quorums) while never acking. The watch loop stamps a progress marker from a ticker arm inside its own select; a watchdog task fails the run when the stamp goes stale (`ROUTER_PARTICIPANT_STALL_SECS`, default 60 s), deregistering the router on the way down so quorums stop counting it and the process supervisor restarts a healthy participant. Pods deliberately get no such watchdog: their converge loop legitimately blocks for warm-duration, and a pod that stops acking is routed *around* by the phase deadline and replacement instead of being restarted mid-warm.
-- **Level-triggered reconciliation everywhere**: Watches are the latency path; every participant also has a truth path that re-derives its state from a fresh snapshot. The coordinator's tick re-evaluates handoffs; the router's reconcile pass (an arm of its watch loop, `ROUTER_RECONCILE_SECS`, default 5 s) re-asserts stashes and freeze acks, converges its table to the assignment authority, and drains any stash whose partition has no handoff to the assignment owner — which heals a missed Complete, a silently dead watch stream, or an out-of-protocol raw deletion; the pod's tick re-converges every involved partition, including locally-held state the durable world no longer mentions. Every reconcile action is idempotent against the event path. Reconcile failures are tolerated, not fatal: a failed pass leaves the participant exactly as stale as the previous tick (the watch-driven steady state), so brief etcd blips never restart the fleet — sustained outage self-fences through the lease keepalive instead. A consecutive-failure budget (`ROUTER_RECONCILE_FAILURE_BUDGET`, default 12 ticks; same default on pods) bounds the partial mode where snapshot reads fail while the lease stays healthy, and `personhog_coordination_reconcile_failures_total{component}` counts every failure.
+- **Level-triggered reconciliation everywhere**: Watches are the latency path; every participant also has a truth path that re-derives its state from a fresh snapshot. The coordinator's tick re-evaluates handoffs; the router's reconcile pass (an arm of its watch loop, `ROUTER_RECONCILE_SECS`, default 5 s) re-asserts stashes and freeze acks, converges its table to the assignment authority (dropping routes at or above the live partition count), and drains any stash whose partition has no handoff: to its assignment owner, or with no owner when no assignment names it — which heals a missed Complete, a silently dead watch stream, or a deleted handoff; the pod's tick re-converges every involved partition, including locally-held state the durable world no longer mentions. Every reconcile action is idempotent against the event path. Reconcile failures are tolerated, not fatal: a failed pass leaves the participant exactly as stale as the previous tick (the watch-driven steady state), so brief etcd blips never restart the fleet — sustained outage self-fences through the lease keepalive instead. A consecutive-failure budget (`ROUTER_RECONCILE_FAILURE_BUDGET`, default 12 ticks; same default on pods) bounds the partial mode where snapshot reads fail while the lease stays healthy, and `personhog_coordination_reconcile_failures_total{component}` counts every failure.
- **Warm provenance**: A pod records what each partition warm was installed for (serving, or a specific handoff id). Warming honors only a warm installed for the *current* handoff — one left over from an earlier ownership era is released and rebuilt, because its cache can hold values that predate writes the changelog accepted since, and a fresh warm replays only from the writer's committed offset. Without this, a returning owner whose `Released` convergence never ran could ack a warm it never performed and lose acknowledged writes — a counterexample the stateright model found and now pins.
- **Coordinator restart**: On startup, the coordinator runs `reconcile_pending_handoffs`. For every existing handoff it applies cleanup if `Complete`, otherwise calls `check_phase_advance` to nudge the state machine. This handles the case where acks were written before the coordinator was up — the ack-watch only fires on Put events, so without reconcile those acks would be invisible.
- **Reconcile tick**: Beyond the startup reconcile, the coordinator re-evaluates every in-flight handoff on a fixed interval (`reconcile_interval`, default 5 s). This is the liveness backstop for state changes that fire no watched event — e.g. a router departing (nothing watches router registrations) newly satisfying a freeze quorum. Everything the tick drives is idempotent: phase transitions use CAS and cleanup tolerates already-deleted records.
@@ -123,13 +123,19 @@ Several failure modes are handled explicitly:
**Mid-stream watch reconnects don't replay missed events — bounded by reconciliation**: The snapshot-to-watch-attach gap is closed (watches are revision-anchored to the snapshot that seeded them), but if an *established* etcd watch stream drops and the underlying client reconnects, events that fired during the disconnect are not replayed. Every participant now bounds the resulting staleness with its level-triggered truth path — the coordinator's tick (which also wakes the planner unconditionally), the router's reconcile pass, and the pod's converge-all tick — so a missed event costs at most one reconcile interval, not correctness.
-## Changelog generations
+## Changelog migrations
-`config/changelog` (`{topic, partitions, epoch, writer_group, draining}`) is the one live object every participant reads. A pod captures it by a strong read when it acquires a partition and serves the partition under that capture until it releases it.
+A changelog resize is a new topic and a cutover, never in-place growth. `config/changelog` (`{topic, partitions, epoch, writer_group, draining}`) is the one live object every participant captures by a strong read at acquisition, and `migrations/current` is the saga only the coordinator creates and advances.
-A leader serving a partition holds in its cache every acked write for that partition's keys that Postgres does not have yet, so a warm reads every source that can hold one: for each draining generation, the old partitions congruent to the new one modulo `gcd(old, new)`, from the writer's committed offset to their end and filtered to the keys that hash to the new partition, then the live partition itself. Marks from an old source keep their own stream (`topic`, source partition), so recovery fetches them from the old topic and the prune settles them against the writer's offsets there. A mark also names the cache partition it belongs to, and a release or prune of one partition only removes the marks and lifecycle fences homed there: an old partition and a new one on the same pod share keys, and either may be released or pruned first.
+The control plane only writes a request, `migrations/requested`. The coordinator starts the saga from it only while the fleet is quiet: no handoff record, and every old partition assigned to a registered, Ready pod. It waits, holding nothing, until then, and creates `migrations/current` in a transaction that requires no handoff record to exist at commit. Creating the saga record moves the planner's plan basis, so no handoff can begin after the start either. The owner half of quiet is checked when the request is evaluated, so an owner that begins shutting down in between is held like any other and reaches the freeze's `holder_died` cancel if it deregisters. The freeze also cancels the saga if it ever finds an ordinary handoff, before anything that handoff touches could be stranded; the quiet start is what keeps that from happening, so a start does not end in a cancel. Admission (the live config, the target topic, no generation still draining, no rollout) runs at the same step, and a request still waiting five minutes after the coordinator first saw it is rejected, with what it was waiting on, so a saga only starts while whoever asked is plausibly watching.
-Routers park requests per key, so a key's requests keep one queue when the partition count changes: a request goes straight to a leader only when its key has nothing parked or in flight and its partition is not stashing, and the live count decides only which partition's drain takes a key. A drain takes the keys that map to its partition under the count the router has adopted, and the leader bounces any key it took under an older one. A leader answers a stale `x-partition` with `FAILED_PRECONDITION`, which the router treats as a bounce, so the write lands once the router's config watch delivers the new count; a router that has not adopted within its retries answers `UNAVAILABLE`, never an ack; a lifecycle batch with a member off the routed partition is refused with `INVALID_ARGUMENT`, which the identity service answers by retrying each member alone.
+Freezing holds every partition in place (a held self-handoff that never passes Draining, so routers stash and the holder keeps its cache). Draining waits on the holders. Sealing has every live leader claim and drop every lane id on the old topic for every partition the assignment table does not give to a registered peer, which covers its own partitions whether or not it has loaded them and every partition without a registered owner, so no stale producer survives; it is the point of no return. Two seconds into Sealing, every leader also seals whatever no ack reports sealed yet, so a registered owner that cannot seal does not hold the fleet; every hold is drained by then, so nothing still produces to those partitions. A leader acks each seal as it lands, so the grace measures progress rather than a whole pass, and its report keeps every partition it has sealed for the life of the process; a restarted leader reseals once.
+
+The ordinary planner places only when no saga holds placement: no migration, or a Resuming one whose holds are all restarted. Every coordinator plan carries the record it was decided on, and its transactions require that record unchanged, so a stalled pass that wakes after a successor acted lands nothing. The planner's record is its plan basis: the changelog config and the migration record, read in one transaction and compared by the config's mod revision (a flip or retire moves it) and the migration's create revision (a new or archived saga moves it; the driver's own writes do not). A migration pass's record is the saga at the mod revision it read, which any write to it moves: a cancel, a phase change, a blocking-list write, the archive. A saga cancelled before its flip and archived leaves the planner's basis as it was, and a plan landing after it meets the world it was planned for.
+
+A leader serving a partition holds in its cache every acked write for that partition's keys that Postgres does not have yet, so a restart warms from every source that can hold one: for each draining generation, the old partitions congruent to the new one modulo `gcd(old, new)`, from the writer's committed offset to their sealed end and filtered to the keys that hash to the new partition, then the live partition itself. Marks from an old source keep their own stream (`topic`, source partition), so recovery fetches them from the old topic and the prune settles them against the writer's offsets there. A mark also names the cache partition it belongs to, and a release or prune of one partition only removes the marks and lifecycle fences homed there: after a flip, an old partition and a new one on the same pod share keys, and either may be released or pruned first. Nothing in the saga waits for the writer: it drains the oldest undrained generation first, flushes a sealed partition's tail as soon as it reaches the end, and re-subscribes in place once the group has drained the topic; its offsets carry their topic, since a resubscribe reuses the partition numbers. The control plane's retire op removes a generation from `draining` once the writer group has read each partition to its end; the config watch delivers a retire as it delivers a flip, and a leader settles the marks on a retired stream without the writer's offsets. Deleting the topic stays a Terraform change.
+
+Routers park requests per key, so a key's requests keep one queue across the flip: a request goes straight to a leader only when its key has nothing parked or in flight and its partition is not stashing, and the live count decides only which partition's drain takes a key. A drain first reads the config from etcd, best effort, then takes the keys that map to its partition under the count the router has adopted; the leader bounces any key it took under an older one, and the reconcile pass drains every pending partition, with no owner when nothing names it; a write for a partition born in the flip parks until that partition's fresh handoff drains it. A leader answers a stale `x-partition` with `FAILED_PRECONDITION`, which the router treats as a bounce, so the write lands once the router's config watch delivers the flip; a router that has not adopted within its retries answers `UNAVAILABLE`, never an ack; a lifecycle batch with a member off the routed partition is refused with `INVALID_ARGUMENT`, which the identity service answers by retrying each member alone.
## Modules
diff --git a/rust/personhog-coordination/src/changelog.rs b/rust/personhog-coordination/src/changelog.rs
index 7d73ae115a0b..30134010cd79 100644
--- a/rust/personhog-coordination/src/changelog.rs
+++ b/rust/personhog-coordination/src/changelog.rs
@@ -183,17 +183,30 @@ pub fn warm_sources(config: &ChangelogConfig, partition: u32) -> Vec<WarmSource>
pub const CHANGELOG_RECONCILE_INTERVAL: Duration = Duration::from_secs(30);
pub const CHANGELOG_BOOT_POLL_INTERVAL: Duration = Duration::from_secs(1);
-/// The partition count has its own atomic: routing reads it per request.
+/// The partition count and the draining floor have their own atomics:
+/// routing reads them per request.
#[derive(Clone)]
pub struct LiveChangelog {
current: Arc<std::sync::RwLock<ChangelogConfig>>,
partitions: Arc<AtomicU32>,
+ draining_floor: Arc<AtomicU32>,
+}
+
+/// The smallest partition count among the draining generations; `u32::MAX` when none drains.
+fn draining_floor_of(config: &ChangelogConfig) -> u32 {
+ config
+ .draining
+ .iter()
+ .map(|g| g.partitions)
+ .min()
+ .unwrap_or(u32::MAX)
}
impl LiveChangelog {
pub fn new(config: ChangelogConfig) -> Self {
Self {
partitions: Arc::new(AtomicU32::new(config.partitions)),
+ draining_floor: Arc::new(AtomicU32::new(draining_floor_of(&config))),
current: Arc::new(std::sync::RwLock::new(config)),
}
}
@@ -202,6 +215,13 @@ impl LiveChangelog {
self.partitions.load(Ordering::Acquire)
}
+ /// Partitions at or above the floor were born in a flip some consumer
+ /// is still catching up on; `None` once nothing drains.
+ pub fn draining_floor(&self) -> Option<u32> {
+ let floor = self.draining_floor.load(Ordering::Acquire);
+ (floor != u32::MAX).then_some(floor)
+ }
+
pub fn current(&self) -> ChangelogConfig {
self.current
.read()
@@ -225,6 +245,9 @@ impl LiveChangelog {
return false;
}
*current = config.clone();
+ // The floor first: a reader that sees the new count sees its floor too.
+ self.draining_floor
+ .store(draining_floor_of(config), Ordering::Release);
self.partitions.store(config.partitions, Ordering::Release);
true
}
diff --git a/rust/personhog-coordination/src/coordinator.rs b/rust/personhog-coordination/src/coordinator.rs
index a2a424f85aa8..87cb4f5ed228 100644
--- a/rust/personhog-coordination/src/coordinator.rs
+++ b/rust/personhog-coordination/src/coordinator.rs
@@ -15,6 +15,8 @@ use k8s_awareness::types::{ControllerKind, ControllerRef};
use k8s_awareness::{classify_departure, ClusterIntent, DepartureReason, K8sAwareness};
use crate::error::{Error, Result};
+use crate::migration;
+use crate::protocol::migration::planner_may_place;
use crate::protocol::{
drain_satisfied, freeze_quorum_met, missing_freeze_ackers, past_phase_deadline,
plan_partial_rebalance, warm_satisfied,
@@ -90,6 +92,9 @@ pub struct CoordinatorConfig {
/// not exceed the server's value — a larger transaction is refused
/// outright, and the planner would retry it forever.
pub max_txn_ops: usize,
+ /// Frozen partitions are a write outage, so the saga cancels past this.
+ pub migration_pre_flip_deadline: Duration,
+ pub migration_debounce_interval: Duration,
}
impl Default for CoordinatorConfig {
@@ -116,6 +121,8 @@ impl Default for CoordinatorConfig {
// etcd's own default; deployments that raise the server
// flag raise this with it from the same chart value.
max_txn_ops: 128,
+ migration_pre_flip_deadline: Duration::from_secs(30),
+ migration_debounce_interval: Duration::from_millis(10),
}
}
}
@@ -648,6 +655,9 @@ impl Coordinator {
let drained_acks_stream = self.store.watch_drained_acks_from(anchor).await?;
let warmed_acks_stream = self.store.watch_warmed_acks_from(anchor).await?;
let routers_stream = self.store.watch_routers_from(anchor).await?;
+ let migration_stream = self.store.watch_migration_from(anchor).await?;
+ let seal_acks_stream = self.store.watch_seal_acks_from(anchor).await?;
+ let request_stream = self.store.watch_migration_request_from(anchor).await?;
let mut tasks = tokio::task::JoinSet::new();
@@ -659,6 +669,8 @@ impl Coordinator {
// single planner task; `Notify` stores a permit, so a wake fired
// mid-plan is picked up on the next iteration rather than lost.
let replan = Arc::new(Notify::new());
+ // The migration driver's own wake; the planner stands down while holds exist.
+ let migrate = Arc::new(Notify::new());
{
let store = Arc::clone(&self.store);
@@ -668,6 +680,7 @@ impl Coordinator {
let deadlines = self.config.phase_deadlines();
let max_txn_ops = self.config.max_txn_ops;
let replan = Arc::clone(&replan);
+ let migrate = Arc::clone(&migrate);
let token = cancel.child_token();
tasks.spawn(async move {
Self::watch_pods_loop(
@@ -678,6 +691,7 @@ impl Coordinator {
deadlines,
max_txn_ops,
replan,
+ migrate,
token,
pods_stream,
)
@@ -691,6 +705,7 @@ impl Coordinator {
let k8s_awareness = self.k8s_awareness.clone();
let deadlines = self.config.phase_deadlines();
let max_txn_ops = self.config.max_txn_ops;
+ let migrate = Arc::clone(&migrate);
let token = cancel.child_token();
tasks.spawn(async move {
Self::watch_handoffs_loop(
@@ -699,6 +714,7 @@ impl Coordinator {
k8s_awareness,
deadlines,
max_txn_ops,
+ migrate,
token,
handoffs_stream,
)
@@ -708,25 +724,55 @@ impl Coordinator {
{
let store = Arc::clone(&self.store);
+ let migrate = Arc::clone(&migrate);
let token = cancel.child_token();
tasks.spawn(async move {
- Self::run_ack_watch("freeze", freeze_acks_stream, &store, token).await
+ Self::run_ack_watch("freeze", freeze_acks_stream, &store, migrate, token).await
});
}
{
let store = Arc::clone(&self.store);
+ let migrate = Arc::clone(&migrate);
let token = cancel.child_token();
tasks.spawn(async move {
- Self::run_ack_watch("drained", drained_acks_stream, &store, token).await
+ Self::run_ack_watch("drained", drained_acks_stream, &store, migrate, token).await
});
}
{
let store = Arc::clone(&self.store);
+ let migrate = Arc::clone(&migrate);
let token = cancel.child_token();
tasks.spawn(async move {
- Self::run_ack_watch("warmed", warmed_acks_stream, &store, token).await
+ Self::run_ack_watch("warmed", warmed_acks_stream, &store, migrate, token).await
+ });
+ }
+
+ {
+ let store = Arc::clone(&self.store);
+ let strategy = Arc::clone(&self.strategy);
+ let k8s_awareness = self.k8s_awareness.clone();
+ let pre_flip_deadline = self.config.migration_pre_flip_deadline;
+ let debounce_interval = self.config.migration_debounce_interval;
+ let max_txn_ops = self.config.max_txn_ops;
+ let migrate = Arc::clone(&migrate);
+ let token = cancel.child_token();
+ tasks.spawn(async move {
+ Self::migration_loop(
+ store,
+ strategy,
+ k8s_awareness,
+ pre_flip_deadline,
+ debounce_interval,
+ max_txn_ops,
+ migrate,
+ token,
+ migration_stream,
+ seal_acks_stream,
+ request_stream,
+ )
+ .await
});
}
@@ -742,10 +788,11 @@ impl Coordinator {
let store = Arc::clone(&self.store);
let interval = self.config.reconcile_interval;
let replan = Arc::clone(&replan);
+ let migrate = Arc::clone(&migrate);
let token = cancel.child_token();
- tasks.spawn(
- async move { Self::reconcile_tick_loop(store, interval, replan, token).await },
- );
+ tasks.spawn(async move {
+ Self::reconcile_tick_loop(store, interval, replan, migrate, token).await
+ });
}
// Reconcile any handoffs that already have full ack quorum.
@@ -777,6 +824,7 @@ impl Coordinator {
deadlines: PhaseDeadlines,
max_txn_ops: usize,
replan: Arc<Notify>,
+ migrate: Arc<Notify>,
cancel: CancellationToken,
mut stream: WatchStream,
) -> Result<()> {
@@ -814,9 +862,108 @@ impl Coordinator {
max_txn_ops,
)
.await?;
+ migrate.notify_one();
+ }
+ }
+
+ #[allow(clippy::too_many_arguments)]
+ async fn migration_loop(
+ store: Arc<PersonhogStore>,
+ strategy: Arc<dyn AssignmentStrategy>,
+ k8s_awareness: Option<Arc<K8sAwareness>>,
+ pre_flip_deadline: Duration,
+ debounce_interval: Duration,
+ max_txn_ops: usize,
+ migrate: Arc<Notify>,
+ cancel: CancellationToken,
+ mut migration_stream: WatchStream,
+ mut seal_acks_stream: WatchStream,
+ mut request_stream: WatchStream,
+ ) -> Result<()> {
+ let _gauge = migration::PhaseGauge;
+ loop {
+ tokio::select! {
+ _ = cancel.cancelled() => return Ok(()),
+ _ = migrate.notified() => {}
+ msg = migration_stream.message() => {
+ util::live_watch_response(msg?, "migration")?;
+ }
+ msg = seal_acks_stream.message() => {
+ util::live_watch_response(msg?, "seal ack")?;
+ }
+ msg = request_stream.message() => {
+ util::live_watch_response(msg?, "migration request")?;
+ }
+ }
+ let deadline = tokio::time::Instant::now() + debounce_interval;
+ loop {
+ tokio::select! {
+ _ = cancel.cancelled() => return Ok(()),
+ _ = tokio::time::sleep_until(deadline) => break,
+ _ = migrate.notified() => {}
+ msg = migration_stream.message() => {
+ util::live_watch_response(msg?, "migration")?;
+ }
+ msg = seal_acks_stream.message() => {
+ util::live_watch_response(msg?, "seal ack")?;
+ }
+ msg = request_stream.message() => {
+ util::live_watch_response(msg?, "migration request")?;
+ }
+ }
+ }
+ Self::drive_migration_static(
+ &store,
+ strategy.as_ref(),
+ k8s_awareness.as_deref(),
+ pre_flip_deadline,
+ max_txn_ops,
+ )
+ .await?;
}
}
+ async fn drive_migration_static(
+ store: &PersonhogStore,
+ strategy: &dyn AssignmentStrategy,
+ k8s_awareness: Option<&K8sAwareness>,
+ pre_flip_deadline: Duration,
+ max_txn_ops: usize,
+ ) -> Result<()> {
+ let Some((migration, mod_revision)) = store.get_migration_with_mod_revision().await? else {
+ migration::record_phase(None);
+ // No saga: a waiting request may start once the fleet is quiet.
+ let Some((request, request_revision)) =
+ store.get_migration_request_with_mod_revision().await?
+ else {
+ return Ok(());
+ };
+ let pods = store.list_pods().await?;
+ let members = match k8s_awareness {
+ Some(k8s) => members_for_k8s(k8s, &pods, request.to.partitions).await,
+ None => Member::active_all(&active_pod_names(&pods)),
+ };
+ return migration::consider_request(store, &members, &pods, request, request_revision)
+ .await;
+ };
+ let pods = store.list_pods().await?;
+ let members = match k8s_awareness {
+ Some(k8s) => members_for_k8s(k8s, &pods, migration.to.partitions).await,
+ None => Member::active_all(&active_pod_names(&pods)),
+ };
+ migration::reconcile(
+ store,
+ strategy,
+ &members,
+ &pods,
+ pre_flip_deadline,
+ max_txn_ops,
+ migration,
+ mod_revision,
+ )
+ .await
+ }
+
fn log_pod_events(resp: &etcd_client::WatchResponse) {
for event in resp.events() {
match event.event_type() {
@@ -826,12 +973,14 @@ impl Coordinator {
}
}
+ #[allow(clippy::too_many_arguments)]
async fn watch_handoffs_loop(
store: Arc<PersonhogStore>,
strategy: Arc<dyn AssignmentStrategy>,
k8s_awareness: Option<Arc<K8sAwareness>>,
deadlines: PhaseDeadlines,
max_txn_ops: usize,
+ migrate: Arc<Notify>,
cancel: CancellationToken,
mut stream: WatchStream,
) -> Result<()> {
@@ -840,6 +989,7 @@ impl Coordinator {
_ = cancel.cancelled() => return Ok(()),
msg = stream.message() => {
let resp = util::live_watch_response(msg?, "handoff")?;
+ migrate.notify_one();
for event in resp.events() {
if event.event_type() == EventType::Put {
match parse_watch_value::<HandoffState>(event) {
@@ -887,6 +1037,7 @@ impl Coordinator {
kind: &str,
mut stream: WatchStream,
store: &PersonhogStore,
+ migrate: Arc<Notify>,
cancel: CancellationToken,
) -> Result<()> {
loop {
@@ -894,6 +1045,7 @@ impl Coordinator {
_ = cancel.cancelled() => return Ok(()),
msg = stream.message() => {
let resp = util::live_watch_response(msg?, kind)?;
+ migrate.notify_one();
for event in resp.events() {
if event.event_type() == EventType::Put {
let partition = event.kv().and_then(|kv| {
@@ -949,6 +1101,7 @@ impl Coordinator {
store: Arc<PersonhogStore>,
interval: Duration,
replan: Arc<Notify>,
+ migrate: Arc<Notify>,
cancel: CancellationToken,
) -> Result<()> {
let mut tick = tokio::time::interval(interval);
@@ -988,6 +1141,7 @@ impl Coordinator {
// planning decision — the planner replaces a doomed
// handoff with whatever resolves its stashes.
replan.notify_one();
+ migrate.notify_one();
// The gauge refresh is best-effort and runs after the
// reconcile pass: its reads exist only for metrics and
// must never delay or interrupt handoff advancement.
@@ -1163,6 +1317,10 @@ impl Coordinator {
}
}
HandoffPhase::Draining => {
+ // Held records stop here; the migration driver replaces them.
+ if handoff.is_held() {
+ return Ok(());
+ }
// Drain semantics live in `protocol::drain_satisfied`
// (shared with the stateright model).
let pods = store.list_pods().await?;
@@ -1281,11 +1439,21 @@ impl Coordinator {
max_txn_ops: usize,
) -> Result<()> {
let pods = store.list_pods().await?;
- let Some(changelog) = store.get_changelog_config().await? else {
+ // Every plan this pass writes requires the basis unchanged; the
+ // fallback deletes below are guarded on their own records instead.
+ let basis = store.read_plan_basis().await?;
+ let Some(changelog) = &basis.config else {
tracing::debug!("changelog config not seeded, skipping assignment");
return Ok(());
};
let (total_partitions, config_epoch) = (changelog.partitions, changelog.epoch);
+ let in_flight = store.list_handoffs_with_mod_revisions().await?;
+ if !planner_may_place(
+ basis.migration.as_ref().map(|m| m.phase),
+ in_flight.iter().any(|(h, _)| h.is_held()),
+ ) {
+ return Ok(());
+ }
// K8s-aware placement policies for smarter rebalancing
let members = match k8s_awareness {
@@ -1306,11 +1474,11 @@ impl Coordinator {
// Everything else pins its partition: the plan excludes it and
// attributes it to its target, so a stuck handoff defers only
// its own partition.
- let in_flight = store.list_handoffs_with_mod_revisions().await?;
let registered: HashSet<&str> = pods.iter().map(|p| p.pod_name.as_str()).collect();
let now_ms = util::now_millis();
let (cancelled, pinned): (Vec<_>, Vec<_>) = in_flight.into_iter().partition(|(h, _)| {
h.phase != HandoffPhase::Complete
+ && !h.is_held()
&& (!registered.contains(h.new_owner.as_str())
|| past_phase_deadline(h, now_ms, deadlines.handoff, deadlines.warming))
});
@@ -1377,6 +1545,7 @@ impl Coordinator {
freeze_quorum_ref: Some(freeze_quorum_id.clone()),
created_at_ms: now_ms,
phase_entered_at_ms: now_ms,
+ held_by: None,
config_epoch,
})
.collect();
@@ -1466,6 +1635,7 @@ impl Coordinator {
freeze_quorum_ref: None,
created_at_ms: now_ms,
phase_entered_at_ms: now_ms,
+ held_by: None,
config_epoch,
},
expected_mod_revision: mod_revision,
@@ -1524,8 +1694,8 @@ impl Coordinator {
crate::store::PlanApplication::default()
} else {
store
- .apply_plan(
- &[],
+ .apply_plan_on(
+ &basis,
&creations,
&replacements,
&preconditions,
@@ -1540,6 +1710,12 @@ impl Coordinator {
)
.await?
};
+ if !application.stood_down.is_empty() {
+ tracing::info!(
+ stood_down = application.stood_down.len(),
+ "the plan's basis moved (a migration began or ended, or the config changed); the rest stood down"
+ );
+ }
if !application.conflicted.is_empty() {
// A concurrent invocation (the empty-set re-trigger racing a
// pod event, or a failing-over coordinator) got to these
diff --git a/rust/personhog-coordination/src/lib.rs b/rust/personhog-coordination/src/lib.rs
index 489e5b289116..9bf795e257e8 100644
--- a/rust/personhog-coordination/src/lib.rs
+++ b/rust/personhog-coordination/src/lib.rs
@@ -3,6 +3,7 @@ pub mod changelog;
pub mod coordinator;
pub mod error;
pub mod hash;
+pub mod migration;
pub mod pod;
pub mod protocol;
pub mod routing_table;
diff --git a/rust/personhog-coordination/src/migration.rs b/rust/personhog-coordination/src/migration.rs
new file mode 100644
index 000000000000..a89a1d94c187
--- /dev/null
+++ b/rust/personhog-coordination/src/migration.rs
@@ -0,0 +1,613 @@
+//! The changelog migration driver: one CAS-guarded reconcile pass over `migrations/current`.
+
+use std::collections::{HashMap, HashSet};
+use std::time::Duration;
+
+use metrics::{counter, gauge};
+
+use crate::error::Result;
+use crate::protocol::drain_satisfied;
+use crate::protocol::migration::{
+ fleet_quiet, migration_unsealed, migration_unsettled, plan_freeze, plan_resume,
+};
+use crate::store::PersonhogStore;
+use crate::strategy::{AssignmentStrategy, Member, PlacementPolicy};
+use crate::types::{
+ AssignmentPrecondition, HandoffPhase, HandoffReplacement, HandoffState, MigrationPhase,
+ MigrationRequest, MigrationState, PodStatus, RegisteredPod,
+};
+use crate::util;
+
+const PHASES: [MigrationPhase; 7] = [
+ MigrationPhase::Freezing,
+ MigrationPhase::Draining,
+ MigrationPhase::Sealing,
+ MigrationPhase::Resuming,
+ MigrationPhase::Complete,
+ MigrationPhase::Cancelling,
+ MigrationPhase::Cancelled,
+];
+
+pub fn phase_label(phase: MigrationPhase) -> &'static str {
+ match phase {
+ MigrationPhase::Freezing => "freezing",
+ MigrationPhase::Draining => "draining",
+ MigrationPhase::Sealing => "sealing",
+ MigrationPhase::Resuming => "resuming",
+ MigrationPhase::Complete => "complete",
+ MigrationPhase::Cancelling => "cancelling",
+ MigrationPhase::Cancelled => "cancelled",
+ }
+}
+
+/// Clears the phase gauge when a coordinator's migration loop ends, so a
+/// coordinator that lost leadership stops reporting the saga.
+pub(crate) struct PhaseGauge;
+
+impl Drop for PhaseGauge {
+ fn drop(&mut self) {
+ record_phase(None);
+ }
+}
+
+pub fn record_phase(current: Option<MigrationPhase>) {
+ for phase in PHASES {
+ gauge!("personhog_migration_phase", "phase" => phase_label(phase))
+ .set(if current == Some(phase) { 1.0 } else { 0.0 });
+ }
+}
+
+struct Pass<'a> {
+ store: &'a PersonhogStore,
+ pods: &'a [RegisteredPod],
+ max_txn_ops: usize,
+ migration: &'a MigrationState,
+ mod_revision: i64,
+ /// The live config's epoch, which every record this pass writes carries.
+ epoch: u64,
+}
+
+impl Pass<'_> {
+ fn registered(&self, pod: &str) -> bool {
+ self.pods.iter().any(|p| p.pod_name == pod)
+ }
+
+ fn address_of(&self, pod: &str) -> Option<String> {
+ self.pods
+ .iter()
+ .find(|p| p.pod_name == pod)
+ .and_then(|p| p.advertise_address.clone())
+ }
+
+ /// Apply this pass's plan on the record it read. False when a successor moved
+ /// the record, in which case nothing landed and the pass must stop.
+ async fn apply(
+ &self,
+ creations: &[HandoffState],
+ replacements: &[HandoffReplacement],
+ preconditions: &[AssignmentPrecondition],
+ freeze_quorum: Option<(&str, &[String])>,
+ ) -> Result<bool> {
+ let application = self
+ .store
+ .apply_migration_plan(
+ self.mod_revision,
+ creations,
+ replacements,
+ preconditions,
+ freeze_quorum,
+ self.max_txn_ops,
+ )
+ .await?;
+ if application.stood_down.is_empty() {
+ return Ok(true);
+ }
+ tracing::warn!(
+ migration = %self.migration.id,
+ phase = ?self.migration.phase,
+ stood_down = application.stood_down.len(),
+ "the saga record moved under this pass; its plan stood down"
+ );
+ Ok(false)
+ }
+
+ async fn held(&self) -> Result<Vec<(HandoffState, i64)>> {
+ let handoffs = self.store.list_handoffs_with_mod_revisions().await?;
+ Ok(handoffs
+ .into_iter()
+ .filter(|(h, _)| h.held_by.as_deref() == Some(&self.migration.id))
+ .collect())
+ }
+
+ async fn advance(&self, next: MigrationPhase) -> Result<bool> {
+ let advanced = self
+ .store
+ .cas_migration_phase(&self.migration.id, self.mod_revision, next)
+ .await?;
+ if advanced {
+ counter!("personhog_migration_transitions_total", "to" => phase_label(next))
+ .increment(1);
+ tracing::info!(migration = %self.migration.id, from = ?self.migration.phase, to = ?next, "migration advanced");
+ }
+ Ok(advanced)
+ }
+
+ async fn cancel(&self, reason: &'static str) -> Result<()> {
+ if self
+ .store
+ .cancel_migration(&self.migration.id, self.mod_revision, reason)
+ .await?
+ {
+ tracing::error!(migration = %self.migration.id, phase = ?self.migration.phase, reason, blocking = ?self.migration.blocking, "cancelled migration");
+ }
+ Ok(())
+ }
+
+ async fn block(&self, blocking: Vec<String>) -> Result<()> {
+ if self.migration.blocking != blocking {
+ tracing::info!(migration = %self.migration.id, phase = ?self.migration.phase, ?blocking, "migration waiting");
+ self.store
+ .set_migration_blocking(self.mod_revision, blocking)
+ .await?;
+ }
+ Ok(())
+ }
+
+ async fn gate(&self, blocking: Vec<String>, next: MigrationPhase) -> Result<()> {
+ if blocking.is_empty() {
+ self.advance(next).await?;
+ Ok(())
+ } else {
+ self.block(blocking).await
+ }
+ }
+
+ async fn archive(&self, final_phase: MigrationPhase) -> Result<()> {
+ if self
+ .store
+ .archive_migration(self.migration, self.mod_revision, final_phase)
+ .await?
+ {
+ counter!("personhog_migration_transitions_total", "to" => phase_label(final_phase))
+ .increment(1);
+ // Counted here, once, whoever cancelled: the driver or an operator.
+ if final_phase == MigrationPhase::Cancelled {
+ let reason = self.migration.cancel_reason.as_deref().unwrap_or("unknown");
+ counter!("personhog_migration_cancelled_total", "reason" => reason.to_string())
+ .increment(1);
+ }
+ record_phase(None);
+ tracing::info!(migration = %self.migration.id, phase = ?final_phase, "migration archived");
+ }
+ Ok(())
+ }
+
+ /// A hold: the partition stays with its owner, frozen and drained, until resume.
+ fn hold(&self, partition: u32, owner: &str, quorum_id: &str) -> HandoffState {
+ HandoffState {
+ held_by: Some(self.migration.id.clone()),
+ ..self.fresh(partition, Some(owner), owner, Some(quorum_id))
+ }
+ }
+
+ /// A restart over the new changelog starts at Warming: it is frozen and drained already.
+ fn restart(&self, partition: u32, owner: &str) -> HandoffState {
+ HandoffState {
+ phase: HandoffPhase::Warming,
+ freeze_quorum: Some(Vec::new()),
+ ..self.fresh(partition, Some(owner), owner, None)
+ }
+ }
+
+ /// A cancelled hold ends as Complete, keeping the partition with its holder.
+ fn reaffirm(&self, partition: u32, owner: &str) -> HandoffState {
+ HandoffState {
+ phase: HandoffPhase::Complete,
+ freeze_quorum: Some(Vec::new()),
+ ..self.fresh(partition, None, owner, None)
+ }
+ }
+
+ /// A partition the grow created, placed with no prior owner.
+ fn place(&self, partition: u32, owner: &str, quorum_id: &str) -> HandoffState {
+ self.fresh(partition, None, owner, Some(quorum_id))
+ }
+
+ /// A Freezing record under the live config's epoch.
+ fn fresh(
+ &self,
+ partition: u32,
+ old_owner: Option<&str>,
+ owner: &str,
+ quorum_id: Option<&str>,
+ ) -> HandoffState {
+ let now_ms = util::now_millis();
+ HandoffState {
+ partition,
+ old_owner: old_owner.map(str::to_string),
+ new_owner: owner.to_string(),
+ new_owner_address: self.address_of(owner),
+ phase: HandoffPhase::Freezing,
+ started_at: now_ms / 1000,
+ handoff_id: util::new_handoff_id(),
+ freeze_quorum: None,
+ freeze_quorum_ref: quorum_id.map(str::to_string),
+ created_at_ms: now_ms,
+ phase_entered_at_ms: now_ms,
+ held_by: None,
+ config_epoch: self.epoch,
+ }
+ }
+
+ async fn router_quorum(&self) -> Result<Vec<String>> {
+ let routers = self.store.list_routers().await?;
+ Ok(routers.into_iter().map(|r| r.router_name).collect())
+ }
+
+ async fn freeze(&self) -> Result<()> {
+ let handoffs = self.store.list_handoffs().await?;
+ let assignments = self.store.list_assignments_with_mod_revisions().await?;
+ let states: Vec<_> = assignments.iter().map(|(a, _)| a.clone()).collect();
+ let registered: HashSet<&str> = self.pods.iter().map(|p| p.pod_name.as_str()).collect();
+ let plan = plan_freeze(
+ self.migration.from.partitions,
+ &states,
+ &handoffs,
+ &registered,
+ );
+ if let Some((partition, handoff_id)) = plan.stray_handoff {
+ tracing::error!(migration = %self.migration.id, partition, handoff_id, "an ordinary handoff exists during the freeze; the quiet start should have ruled it out");
+ return self.cancel("handoff_during_freeze").await;
+ }
+ if let Some((partition, holder)) = plan.holder_died {
+ tracing::error!(migration = %self.migration.id, partition, holder, "the holder is not registered");
+ return self.cancel("holder_died").await;
+ }
+ let revisions: HashMap<u32, i64> = assignments
+ .iter()
+ .map(|(a, rev)| (a.partition, *rev))
+ .collect();
+ let quorum = self.router_quorum().await?;
+ let quorum_id = util::new_handoff_id();
+ let mut creations = Vec::new();
+ let mut preconditions = Vec::new();
+ for (partition, owner) in &plan.holds {
+ creations.push(self.hold(*partition, owner, &quorum_id));
+ preconditions.push(AssignmentPrecondition::UnchangedSince {
+ partition: *partition,
+ mod_revision: revisions[partition],
+ });
+ }
+ if !creations.is_empty() {
+ tracing::info!(migration = %self.migration.id, holds = creations.len(), "holding partitions");
+ if !self
+ .apply(&creations, &[], &preconditions, Some((&quorum_id, &quorum)))
+ .await?
+ {
+ return Ok(());
+ }
+ }
+ self.gate(plan.blocking, MigrationPhase::Draining).await
+ }
+
+ async fn drain(&self) -> Result<()> {
+ let mut blocking = Vec::new();
+ for (held, _) in self.held().await? {
+ if !self.registered(&held.new_owner) {
+ tracing::error!(migration = %self.migration.id, partition = held.partition, holder = %held.new_owner, "the holder is not registered");
+ return self.cancel("holder_died").await;
+ }
+ if held.phase == HandoffPhase::Freezing {
+ blocking.push(format!("partition {}: freeze quorum", held.partition));
+ continue;
+ }
+ let acks = self.store.list_drained_acks(held.partition).await?;
+ if !drain_satisfied(self.pods, &acks, &held) {
+ blocking.push(format!("partition {}: drain", held.partition));
+ }
+ }
+ self.gate(blocking, MigrationPhase::Sealing).await
+ }
+
+ /// Every live leader seals orphaned partitions too, so a dead holder never blocks.
+ async fn seal(&self) -> Result<()> {
+ let acks = self.store.list_seal_acks().await?;
+ let blocking: Vec<String> =
+ migration_unsealed(&acks, &self.migration.id, self.migration.from.partitions)
+ .into_iter()
+ .map(|p| format!("partition {p}: not sealed"))
+ .collect();
+ if !blocking.is_empty() {
+ return self.block(blocking).await;
+ }
+ let Some((config, config_mod_revision)) =
+ self.store.get_changelog_config_with_mod_revision().await?
+ else {
+ return self.block(vec!["config vanished".to_string()]).await;
+ };
+ if self
+ .store
+ .flip_changelog(
+ self.migration,
+ self.mod_revision,
+ &config,
+ config_mod_revision,
+ )
+ .await?
+ {
+ counter!("personhog_migration_transitions_total", "to" => phase_label(MigrationPhase::Resuming))
+ .increment(1);
+ tracing::info!(
+ migration = %self.migration.id,
+ topic = %self.migration.to.topic,
+ partitions = self.migration.to.partitions,
+ epoch = config.epoch + 1,
+ "changelog flipped"
+ );
+ }
+ Ok(())
+ }
+
+ async fn resume(&self, strategy: &dyn AssignmentStrategy, members: &[Member]) -> Result<()> {
+ let held = self.held().await?;
+ let to = self.migration.to.partitions;
+ if !held.is_empty() {
+ let assignments = self.store.list_assignments_with_mod_revisions().await?;
+ let handoffs = self.store.list_handoffs().await?;
+ let states: Vec<_> = assignments.iter().map(|(a, _)| a.clone()).collect();
+ let holds: Vec<(u32, String)> = held
+ .iter()
+ .map(|(h, _)| (h.partition, h.new_owner.clone()))
+ .collect();
+ let registered: HashSet<&str> = self.pods.iter().map(|p| p.pod_name.as_str()).collect();
+ let plan = plan_resume(
+ strategy,
+ members,
+ to,
+ &holds,
+ &states,
+ &handoffs,
+ &registered,
+ );
+
+ let held_revisions: HashMap<u32, i64> =
+ held.iter().map(|(h, rev)| (h.partition, *rev)).collect();
+ let assignment_revisions: HashMap<u32, i64> = assignments
+ .iter()
+ .map(|(a, rev)| (a.partition, *rev))
+ .collect();
+ for partition in &plan.dissolve {
+ self.store
+ .dissolve_partition(*partition, held_revisions[partition])
+ .await?;
+ }
+ for partition in &plan.abandon {
+ self.store
+ .delete_handoff_and_acks_if_unchanged(*partition, held_revisions[partition])
+ .await?;
+ }
+ let mut preconditions = Vec::new();
+ let mut replacements = Vec::new();
+ for (partition, holder) in &plan.restart {
+ replacements.push(HandoffReplacement {
+ handoff: self.restart(*partition, holder),
+ expected_mod_revision: held_revisions[partition],
+ });
+ preconditions.push(match assignment_revisions.get(partition) {
+ Some(&mod_revision) => AssignmentPrecondition::UnchangedSince {
+ partition: *partition,
+ mod_revision,
+ },
+ None => AssignmentPrecondition::Absent {
+ partition: *partition,
+ },
+ });
+ }
+ let quorum = self.router_quorum().await?;
+ let quorum_id = util::new_handoff_id();
+ let mut creations = Vec::new();
+ for (partition, owner) in &plan.place {
+ creations.push(self.place(*partition, owner, &quorum_id));
+ preconditions.push(AssignmentPrecondition::Absent {
+ partition: *partition,
+ });
+ }
+ if !replacements.is_empty() || !creations.is_empty() {
+ tracing::info!(migration = %self.migration.id, restarts = replacements.len(), fresh = creations.len(), "restarting partitions over the new changelog");
+ if !self
+ .apply(
+ &creations,
+ &replacements,
+ &preconditions,
+ Some((&quorum_id, &quorum)),
+ )
+ .await?
+ {
+ return Ok(());
+ }
+ }
+ let blocking = held
+ .iter()
+ .map(|(h, _)| format!("partition {}: restarting", h.partition))
+ .collect();
+ return self.block(blocking).await;
+ }
+ let handoffs = self.store.list_handoffs().await?;
+ let assignments = self.store.list_assignments().await?;
+ let registered: HashSet<&str> = self.pods.iter().map(|p| p.pod_name.as_str()).collect();
+ let unsettled = migration_unsettled(&assignments, &handoffs, &registered, to);
+ if unsettled.is_empty() {
+ return self.archive(MigrationPhase::Complete).await;
+ }
+ self.block(
+ unsettled
+ .into_iter()
+ .map(|p| format!("partition {p}: not settled"))
+ .collect(),
+ )
+ .await
+ }
+
+ async fn unwind(&self) -> Result<()> {
+ let held = self.held().await?;
+ if held.is_empty() {
+ return self.archive(MigrationPhase::Cancelled).await;
+ }
+ let mut replacements = Vec::new();
+ for (h, rev) in held {
+ if !self.registered(&h.new_owner) {
+ self.store
+ .delete_handoff_and_acks_if_unchanged(h.partition, rev)
+ .await?;
+ continue;
+ }
+ replacements.push(HandoffReplacement {
+ handoff: self.reaffirm(h.partition, &h.new_owner),
+ expected_mod_revision: rev,
+ });
+ }
+ if !replacements.is_empty() {
+ self.apply(&[], &replacements, &[], None).await?;
+ }
+ Ok(())
+ }
+}
+
+#[allow(clippy::too_many_arguments)]
+pub(crate) async fn reconcile(
+ store: &PersonhogStore,
+ strategy: &dyn AssignmentStrategy,
+ members: &[Member],
+ pods: &[RegisteredPod],
+ pre_flip_deadline: Duration,
+ max_txn_ops: usize,
+ migration: MigrationState,
+ mod_revision: i64,
+) -> Result<()> {
+ record_phase(Some(migration.phase));
+ let Some(config) = store.get_changelog_config().await? else {
+ tracing::error!(migration = %migration.id, "config/changelog is missing; the saga waits");
+ return Ok(());
+ };
+ let pass = Pass {
+ store,
+ pods,
+ max_txn_ops,
+ migration: &migration,
+ mod_revision,
+ epoch: config.epoch,
+ };
+ let cancellable = migration
+ .phase
+ .may_transition_to(MigrationPhase::Cancelling);
+ let age_ms = util::now_millis().saturating_sub(migration.created_at_ms);
+ if cancellable && age_ms > pre_flip_deadline.as_millis() as i64 {
+ return pass.cancel("pre_flip_deadline").await;
+ }
+ match migration.phase {
+ MigrationPhase::Freezing => pass.freeze().await,
+ MigrationPhase::Draining => pass.drain().await,
+ MigrationPhase::Sealing => pass.seal().await,
+ MigrationPhase::Resuming => pass.resume(strategy, members).await,
+ MigrationPhase::Cancelling => pass.unwind().await,
+ // A live record in a terminal phase is an archive that did not finish.
+ MigrationPhase::Complete | MigrationPhase::Cancelled => pass.archive(migration.phase).await,
+ }
+}
+
+/// A request older than this is rejected rather than started, so a saga only
+/// begins while whoever asked for it is plausibly still watching.
+pub const REQUEST_EXPIRY: Duration = Duration::from_secs(300);
+
+/// Why a request may not begin, checked at the moment it would start.
+async fn admission_refusal(
+ store: &PersonhogStore,
+ request: &MigrationRequest,
+ members: &[Member],
+) -> Result<Option<&'static str>> {
+ let reason = match store.get_changelog_config().await? {
+ None => Some("config_not_seeded"),
+ Some(live) if live.target() != request.from => Some("from_is_not_the_live_config"),
+ // The protocol is checked with at most one draining generation.
+ Some(live) if !live.draining.is_empty() => Some("generation_still_draining"),
+ _ if request.to.topic == request.from.topic => Some("same_topic"),
+ _ if members
+ .iter()
+ .any(|m| m.policy != PlacementPolicy::Active { cap: None }) =>
+ {
+ Some("rollout_in_progress")
+ }
+ _ => None,
+ };
+ if reason.is_some() {
+ return Ok(reason);
+ }
+ let history = store.list_migration_history().await?;
+ Ok(history
+ .iter()
+ .any(|h| h.from.topic == request.to.topic && h.phase == MigrationPhase::Complete)
+ .then_some("topic_already_retired"))
+}
+
+/// The only way a saga starts: it waits, holding nothing, until the fleet is quiet,
+/// then starts in a transaction that proves no handoff appeared meanwhile.
+pub async fn consider_request(
+ store: &PersonhogStore,
+ members: &[Member],
+ pods: &[RegisteredPod],
+ request: MigrationRequest,
+ mod_revision: i64,
+) -> Result<()> {
+ let now_ms = util::now_millis();
+ // The expiry runs on the coordinator's clock, from its first sighting.
+ let Some(seen_at_ms) = request.seen_at_ms else {
+ let seen = MigrationRequest {
+ seen_at_ms: Some(now_ms),
+ ..request
+ };
+ store.update_migration_request(&seen, mod_revision).await?;
+ return Ok(());
+ };
+ let refusal = if now_ms.saturating_sub(seen_at_ms) > REQUEST_EXPIRY.as_millis() as i64 {
+ Some("request_expired")
+ } else {
+ admission_refusal(store, &request, members).await?
+ };
+ if let Some(reason) = refusal {
+ if store
+ .reject_migration_request(&request, mod_revision, reason)
+ .await?
+ {
+ counter!("personhog_migration_requests_rejected_total", "reason" => reason)
+ .increment(1);
+ tracing::warn!(migration = %request.id, reason, "migration request rejected");
+ }
+ return Ok(());
+ }
+ let (handoffs, quiet_at) = store.list_handoffs_with_revision().await?;
+ let assignments = store.list_assignments().await?;
+ let ready: HashSet<&str> = pods
+ .iter()
+ .filter(|p| p.status == PodStatus::Ready)
+ .map(|p| p.pod_name.as_str())
+ .collect();
+ let waiting = fleet_quiet(request.from.partitions, &assignments, &handoffs, &ready);
+ if !waiting.is_empty() {
+ if waiting != request.waiting {
+ tracing::info!(migration = %request.id, ?waiting, "migration waiting for a quiet fleet");
+ let waiting = MigrationRequest { waiting, ..request };
+ store
+ .update_migration_request(&waiting, mod_revision)
+ .await?;
+ }
+ return Ok(());
+ }
+ if store
+ .start_migration(mod_revision, quiet_at, &request.start(now_ms))
+ .await?
+ {
+ counter!("personhog_migration_transitions_total", "to" => "freezing").increment(1);
+ tracing::info!(migration = %request.id, waited_ms = now_ms - seen_at_ms, "migration started");
+ }
+ Ok(())
+}
diff --git a/rust/personhog-coordination/src/pod.rs b/rust/personhog-coordination/src/pod.rs
index b30995516cb7..9b896ce9c57c 100644
--- a/rust/personhog-coordination/src/pod.rs
+++ b/rust/personhog-coordination/src/pod.rs
@@ -65,6 +65,16 @@ pub enum DesiredState {
Released,
}
+/// Whether `pod` is the incoming owner of a handoff still freezing or draining.
+/// A hold's connections are the leader's seal duty to prepare, not this hint's.
+pub fn prepare_hinted(pod: &str, handoff: Option<&HandoffState>) -> bool {
+ handoff.is_some_and(|h| {
+ h.new_owner == pod
+ && !h.is_held()
+ && matches!(h.phase, HandoffPhase::Freezing | HandoffPhase::Draining)
+ })
+}
+
/// Pure derivation of a pod's desired state for a partition. The handoff,
/// when it involves this pod, takes precedence over the assignment: the
/// assignment names the *old* owner (or nobody) for the whole life of a
@@ -75,6 +85,14 @@ pub fn desired_state(
handoff: Option<&HandoffState>,
) -> DesiredState {
if let Some(h) = handoff {
+ // A migration restart: created at Warming, it re-acquires over the new changelog.
+ if h.old_owner.as_deref() == Some(pod) && h.new_owner == pod && !h.is_held() {
+ match h.phase {
+ HandoffPhase::Warming => return DesiredState::Acquiring,
+ HandoffPhase::Complete => return DesiredState::Serving,
+ HandoffPhase::Freezing | HandoffPhase::Draining => {}
+ }
+ }
if h.old_owner.as_deref() == Some(pod) {
return match h.phase {
HandoffPhase::Freezing => DesiredState::Serving,
@@ -1259,12 +1277,8 @@ impl PodHandle {
// and deliberately not `did_work`: preparation is a hint, and
// counting it as progress would let a pod that only ever
// prepares look healthy to the budgets.
- if let Some(h) = handoff {
- if h.new_owner == *pod
- && matches!(h.phase, HandoffPhase::Freezing | HandoffPhase::Draining)
- {
- self.handler.prepare_acquire(partition).await;
- }
+ if prepare_hinted(pod, handoff) {
+ self.handler.prepare_acquire(partition).await;
}
match desired {
@@ -2033,11 +2047,50 @@ mod tests {
freeze_quorum_ref: None,
created_at_ms: 0,
phase_entered_at_ms: 0,
+ held_by: None,
config_epoch: 0,
new_owner_address: None,
}
}
+ fn held(phase: HandoffPhase) -> HandoffState {
+ let mut h = handoff(Some(POD), POD, phase);
+ h.held_by = Some("m-test".to_string());
+ h
+ }
+
+ #[test]
+ fn only_a_pending_incoming_owner_is_hinted() {
+ use HandoffPhase::*;
+ let cases = [
+ (
+ "incoming, freezing",
+ Some(handoff(Some("other"), POD, Freezing)),
+ true,
+ ),
+ (
+ "incoming, draining",
+ Some(handoff(Some("other"), POD, Draining)),
+ true,
+ ),
+ (
+ "incoming, warming",
+ Some(handoff(Some("other"), POD, Warming)),
+ false,
+ ),
+ (
+ "outgoing, draining",
+ Some(handoff(Some(POD), "other", Draining)),
+ false,
+ ),
+ ("held, draining", Some(held(Draining)), false),
+ ("no handoff", None, false),
+ ];
+ for (name, h, expected) in cases {
+ assert_eq!(prepare_hinted(POD, h.as_ref()), expected, "{name}");
+ }
+ }
+
#[test]
fn desired_state_covers_the_full_role_phase_matrix() {
use DesiredState::*;
@@ -2119,6 +2172,30 @@ mod tests {
Some(handoff(None, POD, Warming)),
Acquiring,
),
+ (
+ "held holder keeps serving through Freezing",
+ Some(assignment(POD)),
+ Some(held(Freezing)),
+ Serving,
+ ),
+ (
+ "held holder drains and acks in Draining",
+ Some(assignment(POD)),
+ Some(held(Draining)),
+ Drained { ack: true },
+ ),
+ (
+ "same-owner replacement re-acquires in Warming",
+ Some(assignment(POD)),
+ Some(handoff(Some(POD), POD, Warming)),
+ Acquiring,
+ ),
+ (
+ "same-owner replacement serves at Complete",
+ Some(assignment(POD)),
+ Some(handoff(Some(POD), POD, Complete)),
+ Serving,
+ ),
(
"handoff between two other pods defers to owned assignment",
Some(assignment(POD)),
diff --git a/rust/personhog-coordination/src/protocol.rs b/rust/personhog-coordination/src/protocol.rs
index 6336cee7909e..2765e1d1952f 100644
--- a/rust/personhog-coordination/src/protocol.rs
+++ b/rust/personhog-coordination/src/protocol.rs
@@ -21,6 +21,8 @@ use crate::types::{
RouterFreezeAck,
};
+pub mod migration;
+
/// One handoff a rebalance has decided to create. `old_owner` is `None`
/// for a fresh assignment (no prior owner), which skips the drain.
#[derive(Clone, Debug, PartialEq, Eq)]
@@ -292,8 +294,9 @@ mod tests {
freeze_quorum_ref: None,
created_at_ms: 0,
phase_entered_at_ms: 0,
- new_owner_address: None,
+ held_by: None,
config_epoch: 0,
+ new_owner_address: None,
}
}
diff --git a/rust/personhog-coordination/src/protocol/migration.rs b/rust/personhog-coordination/src/protocol/migration.rs
new file mode 100644
index 000000000000..7dd362e8a5ae
--- /dev/null
+++ b/rust/personhog-coordination/src/protocol/migration.rs
@@ -0,0 +1,308 @@
+//! The migration saga's decisions, shared by the coordinator, the leaders and the model.
+
+use std::collections::{HashMap, HashSet};
+
+use super::plan_rebalance;
+use crate::strategy::{AssignmentStrategy, Member};
+use crate::types::{
+ HandoffPhase, HandoffState, MigrationPhase, MigrationState, PartitionAssignment, SealAck,
+};
+
+/// What one freeze pass does: the holds to create and what still blocks Draining.
+#[derive(Debug, Default, PartialEq, Eq)]
+pub struct FreezePlan {
+ pub holds: Vec<(u32, String)>,
+ pub blocking: Vec<String>,
+ /// A partition whose owner or holder is not registered, which cancels the saga.
+ pub holder_died: Option<(u32, String)>,
+ /// An ordinary handoff on an old partition, which the quiet start rules out;
+ /// it cancels the saga before anything it touches can be stranded.
+ pub stray_handoff: Option<(u32, String)>,
+}
+
+/// Why the fleet is not quiet enough to start a saga, empty when it is: no
+/// handoff record exists, and every old partition is assigned to a ready pod.
+pub fn fleet_quiet(
+ from_partitions: u32,
+ assignments: &[PartitionAssignment],
+ handoffs: &[HandoffState],
+ ready: &HashSet<&str>,
+) -> Vec<String> {
+ let mut waiting: Vec<String> = handoffs
+ .iter()
+ .map(|h| format!("partition {}: handoff in {:?}", h.partition, h.phase))
+ .collect();
+ let owners: HashMap<u32, &str> = assignments
+ .iter()
+ .map(|a| (a.partition, a.owner.as_str()))
+ .collect();
+ for partition in 0..from_partitions {
+ match owners.get(&partition) {
+ None => waiting.push(format!("partition {partition}: unassigned")),
+ Some(owner) if !ready.contains(owner) => {
+ waiting.push(format!("partition {partition}: owner {owner} is not ready"))
+ }
+ Some(_) => {}
+ }
+ }
+ waiting
+}
+
+/// The freeze decision: every assigned old partition gets a hold.
+pub fn plan_freeze(
+ from_partitions: u32,
+ assignments: &[PartitionAssignment],
+ handoffs: &[HandoffState],
+ registered: &HashSet<&str>,
+) -> FreezePlan {
+ let by_partition: HashMap<u32, &HandoffState> =
+ handoffs.iter().map(|h| (h.partition, h)).collect();
+ let mut plan = FreezePlan::default();
+ for assignment in assignments.iter().filter(|a| a.partition < from_partitions) {
+ let partition = assignment.partition;
+ match by_partition.get(&partition) {
+ Some(h) if h.is_held() => {
+ if !registered.contains(h.new_owner.as_str()) {
+ plan.holder_died = Some((partition, h.new_owner.clone()));
+ }
+ if h.phase == HandoffPhase::Freezing {
+ plan.blocking
+ .push(format!("partition {partition}: freeze quorum"));
+ }
+ }
+ Some(_) => {}
+ // A dead owner's partition is the planner's; holding it would only cancel next pass.
+ None if !registered.contains(assignment.owner.as_str()) => {
+ plan.holder_died = Some((partition, assignment.owner.clone()));
+ }
+ None => {
+ plan.holds.push((partition, assignment.owner.clone()));
+ plan.blocking
+ .push(format!("partition {partition}: hold created"));
+ }
+ }
+ }
+ plan.stray_handoff = handoffs
+ .iter()
+ .find(|h| h.partition < from_partitions && !h.is_held())
+ .map(|h| (h.partition, h.handoff_id.clone()));
+ plan
+}
+
+/// The old-topic partitions one leader seals: those no registered peer owns in the
+/// assignment table, plus any no ack reports sealed once the seal is overdue.
+pub fn seal_targets(
+ pod_name: &str,
+ migration_id: &str,
+ from_partitions: u32,
+ assignments: &[PartitionAssignment],
+ registered: &HashSet<&str>,
+ acks: &[SealAck],
+ overdue: bool,
+) -> Vec<u32> {
+ let owned_by_a_peer: HashSet<u32> = assignments
+ .iter()
+ .filter(|a| a.owner != pod_name && registered.contains(a.owner.as_str()))
+ .map(|a| a.partition)
+ .collect();
+ let unsealed: HashSet<u32> = if overdue {
+ migration_unsealed(acks, migration_id, from_partitions)
+ .into_iter()
+ .collect()
+ } else {
+ HashSet::new()
+ };
+ (0..from_partitions)
+ .filter(|p| !owned_by_a_peer.contains(p) || unsealed.contains(p))
+ .collect()
+}
+
+/// Whether the ordinary planner may place: no saga, or a Resuming one whose
+/// holds are all restarted. Until then the migration driver owns placement.
+pub fn planner_may_place(migration_phase: Option<MigrationPhase>, holds_remain: bool) -> bool {
+ match migration_phase {
+ None => true,
+ Some(MigrationPhase::Resuming) => !holds_remain,
+ Some(_) => false,
+ }
+}
+
+/// How long after Sealing begins every leader also seals whatever no ack reports sealed.
+/// Counted from the phase, not per owner, so a slow pass is resealed by peers; seals are idempotent.
+pub const SEAL_GRACE_MS: i64 = 2_000;
+
+/// Past the grace, peers also seal an owner's unsealed partitions. Safe because
+/// every hold is drained before Sealing.
+pub fn seal_overdue(migration: &MigrationState, now_ms: i64) -> bool {
+ migration.phase == MigrationPhase::Sealing
+ && now_ms.saturating_sub(migration.phase_entered_at_ms) >= SEAL_GRACE_MS
+}
+
+/// Old-topic partitions no seal ack of this migration reports sealed.
+pub fn migration_unsealed(acks: &[SealAck], migration_id: &str, partitions: u32) -> Vec<u32> {
+ let sealed: HashSet<u32> = acks
+ .iter()
+ .filter(|a| a.migration_id == migration_id)
+ .flat_map(|a| a.sealed.iter().copied())
+ .collect();
+ (0..partitions).filter(|p| !sealed.contains(p)).collect()
+}
+
+/// What one resume pass does with the saga's holds.
+#[derive(Debug, Default, PartialEq, Eq)]
+pub struct ResumePlan {
+ /// Holds the shrink removed: the hold, its acks and the assignment go together.
+ pub dissolve: Vec<u32>,
+ /// Holds whose holder is gone; the ordinary planner places them once the holds end.
+ pub abandon: Vec<u32>,
+ /// Holds restarted over the new changelog by their holder.
+ pub restart: Vec<(u32, String)>,
+ /// Partitions the grow created, placed fresh in the same plan.
+ pub place: Vec<(u32, String)>,
+}
+
+/// The resume decision over the holds, each `(partition, holder)`: restart every
+/// live hold in place, and place the partitions a grow created around them.
+pub fn plan_resume<S: AssignmentStrategy + ?Sized>(
+ strategy: &S,
+ members: &[Member],
+ to_partitions: u32,
+ held: &[(u32, String)],
+ assignments: &[PartitionAssignment],
+ handoffs: &[HandoffState],
+ registered: &HashSet<&str>,
+) -> ResumePlan {
+ let mut plan = ResumePlan::default();
+ let mut current: HashMap<u32, String> = HashMap::new();
+ for (partition, holder) in held {
+ if *partition >= to_partitions {
+ plan.dissolve.push(*partition);
+ } else if !registered.contains(holder.as_str()) {
+ plan.abandon.push(*partition);
+ } else {
+ current.insert(*partition, holder.clone());
+ plan.restart.push((*partition, holder.clone()));
+ }
+ }
+ let in_flight: HashSet<u32> = handoffs.iter().map(|h| h.partition).collect();
+ let assigned: HashSet<u32> = assignments.iter().map(|a| a.partition).collect();
+ for a in assignments {
+ if a.partition < to_partitions
+ && registered.contains(a.owner.as_str())
+ && !in_flight.contains(&a.partition)
+ {
+ current.insert(a.partition, a.owner.clone());
+ }
+ }
+ let mut placed = plan_rebalance(strategy, &current, members, to_partitions).handoffs;
+ placed.sort_by_key(|h| h.partition);
+ for planned in placed {
+ let p = planned.partition;
+ // Only partitions with no assignment and no handoff are new.
+ if !current.contains_key(&p) && !in_flight.contains(&p) && !assigned.contains(&p) {
+ plan.place.push((p, planned.new_owner));
+ }
+ }
+ plan
+}
+
+/// Partitions of the new space without a registered owner or with a handoff in flight.
+pub fn migration_unsettled(
+ assignments: &[PartitionAssignment],
+ handoffs: &[HandoffState],
+ registered: &HashSet<&str>,
+ to_partitions: u32,
+) -> Vec<u32> {
+ let owned: HashSet<u32> = assignments
+ .iter()
+ .filter(|a| registered.contains(a.owner.as_str()))
+ .map(|a| a.partition)
+ .collect();
+ let in_flight: HashSet<u32> = handoffs.iter().map(|h| h.partition).collect();
+ (0..to_partitions)
+ .filter(|p| !owned.contains(p) || in_flight.contains(p))
+ .collect()
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn handoff(partition: u32, old_owner: Option<&str>, new_owner: &str) -> HandoffState {
+ HandoffState {
+ partition,
+ old_owner: old_owner.map(str::to_string),
+ new_owner: new_owner.to_string(),
+ phase: HandoffPhase::Warming,
+ started_at: 0,
+ handoff_id: String::new(),
+ freeze_quorum: None,
+ freeze_quorum_ref: None,
+ created_at_ms: 0,
+ phase_entered_at_ms: 0,
+ held_by: None,
+ config_epoch: 0,
+ new_owner_address: None,
+ }
+ }
+
+ fn owned(partition: u32, owner: &str) -> PartitionAssignment {
+ PartitionAssignment {
+ partition,
+ owner: owner.to_string(),
+ advertise_address: None,
+ status: crate::types::AssignmentStatus::Active,
+ }
+ }
+
+ #[test]
+ fn the_fleet_is_quiet_only_with_no_handoff_and_every_owner_ready() {
+ let ready: HashSet<&str> = ["p0", "p1"].into_iter().collect();
+ let settled = [owned(0, "p0"), owned(1, "p1")];
+ assert!(fleet_quiet(2, &settled, &[], &ready).is_empty());
+
+ let cases: [(&str, Vec<PartitionAssignment>, Vec<HandoffState>); 3] = [
+ (
+ "a handoff in flight",
+ settled.to_vec(),
+ vec![handoff(1, Some("p1"), "p0")],
+ ),
+ (
+ "an owner that is gone",
+ vec![owned(0, "p0"), owned(1, "gone")],
+ vec![],
+ ),
+ ("an unassigned partition", vec![owned(0, "p0")], vec![]),
+ ];
+ for (name, assignments, handoffs) in cases {
+ assert!(
+ !fleet_quiet(2, &assignments, &handoffs, &ready).is_empty(),
+ "{name}"
+ );
+ }
+ }
+
+ #[test]
+ fn an_ordinary_handoff_during_the_freeze_is_a_stray() {
+ let registered: HashSet<&str> = ["p0", "p1"].into_iter().collect();
+ // Partition 1 has no assignment yet: a fresh placement.
+ let placing = handoff(1, None, "p1");
+ let plan = plan_freeze(2, &[owned(0, "p0")], &[placing], &registered);
+ assert_eq!(plan.stray_handoff.map(|(p, _)| p), Some(1));
+ }
+
+ #[test]
+ fn the_freeze_names_a_holder_that_is_gone() {
+ let registered: HashSet<&str> = ["p0"].into_iter().collect();
+ let unheld = plan_freeze(2, &[owned(0, "p0"), owned(1, "gone")], &[], &registered);
+ assert_eq!(unheld.holder_died, Some((1, "gone".to_string())));
+ assert_eq!(unheld.holds, vec![(0, "p0".to_string())]);
+
+ let hold = HandoffState {
+ held_by: Some("m-1".to_string()),
+ ..handoff(1, Some("gone"), "gone")
+ };
+ let held = plan_freeze(2, &[owned(0, "p0"), owned(1, "gone")], &[hold], &registered);
+ assert_eq!(held.holder_died, Some((1, "gone".to_string())));
+ }
+}
diff --git a/rust/personhog-coordination/src/routing_table.rs b/rust/personhog-coordination/src/routing_table.rs
index ae5e1eec84ae..5a016b4ae902 100644
--- a/rust/personhog-coordination/src/routing_table.rs
+++ b/rust/personhog-coordination/src/routing_table.rs
@@ -50,6 +50,12 @@ pub trait StashHandler: Send + Sync {
fn stash_pending(&self, _partition: u32) -> bool {
true
}
+
+ /// Every partition for which `stash_pending` holds, so the reconcile
+ /// pass can drain those no handoff or assignment names.
+ fn pending_partitions(&self) -> Vec<u32> {
+ Vec::new()
+ }
}
/// Runs drains off the handoff watch loop, one lane per partition.
@@ -933,17 +939,8 @@ impl RoutingTable {
}
}
EventType::Delete => {
- // A handoff record is never deleted while
- // non-terminal — cancellation replaces it
- // with the record that resolves its stashes
- // — so a deletion has exactly one protocol
- // meaning: cleanup after Complete, which
- // requires nothing from the router. If a
- // stash is somehow still open here (an
- // out-of-protocol raw deletion), it stays
- // parked; the reconcile tick derives
- // disposal from durable state and drains it
- // to the assignment owner.
+ // A deletion needs nothing from the router. A stash
+ // still open stays parked; the reconcile tick drains it.
let Some(kv) = event.kv() else { continue };
let key = from_utf8(kv.key()).unwrap_or("");
let Some(partition) = store::extract_partition_from_key(key) else {
@@ -1060,7 +1057,9 @@ impl RoutingTable {
store.put_freeze_acks(&acks, max_txn_ops).await?;
let assignments = store.list_assignments().await?;
+ let mut owned = constrained.clone();
for assignment in assignments {
+ owned.insert(assignment.partition);
if constrained.contains(&assignment.partition) {
continue;
}
@@ -1087,6 +1086,26 @@ impl RoutingTable {
);
}
}
+ // A pending partition nothing names drains with no owner: it answers what outlived
+ // the stash wait and closes the freeze.
+ for partition in handler.pending_partitions() {
+ if !owned.contains(&partition) {
+ lanes.request(
+ Arc::clone(handler),
+ router_name.to_string(),
+ partition,
+ String::new(),
+ );
+ }
+ }
+ // A route at or above the live count names a partition a shrink dissolved. Read after
+ // the listings, so a flip they reflect is in the count and their new routes survive.
+ if let Some(config) = store.get_changelog_config().await? {
+ table
+ .write()
+ .await
+ .retain(|partition, _| *partition < config.partitions);
+ }
Ok(())
}
diff --git a/rust/personhog-coordination/src/store.rs b/rust/personhog-coordination/src/store.rs
index 2032e333d980..f9df1b18a868 100644
--- a/rust/personhog-coordination/src/store.rs
+++ b/rust/personhog-coordination/src/store.rs
@@ -2,15 +2,19 @@ use std::collections::HashMap;
use std::sync::{Arc, Mutex as StdMutex};
use assignment_coordination::store::EtcdStore;
-use etcd_client::{Compare, CompareOp, DeleteOptions, PutOptions, Txn, TxnOp, WatchStream};
+use etcd_client::{
+ Compare, CompareOp, DeleteOptions, PutOptions, Txn, TxnOp, TxnOpResponse, WatchStream,
+};
use crate::error::{Error, Result};
use crate::types::{
AssignmentPrecondition, AssignmentStatus, ChangelogConfig, HandoffReplacement, HandoffState,
- LeaderInfo, PartitionAssignment, PodDrainedAck, PodStatus, PodWarmedAck, RegisteredPod,
- RegisteredRouter, RouterFreezeAck,
+ LeaderInfo, MigrationState, PartitionAssignment, PodDrainedAck, PodStatus, PodWarmedAck,
+ RegisteredPod, RegisteredRouter, RouterFreezeAck,
};
+mod migration;
+
/// All etcd key patterns used by the PersonHog store.
enum StoreKey<'a> {
Pod(&'a str),
@@ -39,6 +43,12 @@ enum StoreKey<'a> {
Leader,
Generation,
ChangelogConfig,
+ Migration,
+ MigrationRequest,
+ MigrationHistory(&'a str),
+ MigrationHistoryPrefix,
+ SealAck(&'a str),
+ SealAcksPrefix,
}
impl StoreKey<'_> {
@@ -72,6 +82,12 @@ impl StoreKey<'_> {
StoreKey::Leader => format!("{prefix}coordinator/leader"),
StoreKey::Generation => format!("{prefix}generation"),
StoreKey::ChangelogConfig => format!("{prefix}config/changelog"),
+ StoreKey::Migration => format!("{prefix}migrations/current"),
+ StoreKey::MigrationRequest => format!("{prefix}migrations/requested"),
+ StoreKey::MigrationHistory(id) => format!("{prefix}migrations/history/{id}"),
+ StoreKey::MigrationHistoryPrefix => format!("{prefix}migrations/history/"),
+ StoreKey::SealAck(pod) => format!("{prefix}seal_acks/{pod}"),
+ StoreKey::SealAcksPrefix => format!("{prefix}seal_acks/"),
}
}
}
@@ -112,6 +128,44 @@ pub struct PlanApplication {
/// Chunks the server refused as too large and that applied per
/// unit instead: the configured budget exceeds the server's.
pub over_budget_chunks: usize,
+ /// Partitions dropped because the record the plan was decided on moved
+ /// after it was read (see `Authority`).
+ pub stood_down: Vec<u32>,
+}
+
+impl PlanApplication {
+ fn stand_down(&mut self, units: &[PlanUnit], authority: Option<Authority>) {
+ self.stood_down.extend(units.iter().map(|u| u.partition));
+ let by = match authority {
+ Some(Authority::Migration { .. }) => "driver",
+ _ => "planner",
+ };
+ metrics::counter!("personhog_coordination_plan_stood_down_total", "by" => by).increment(1);
+ }
+}
+
+/// The changelog config and migration record a planner plan rests on; its transactions
+/// require both unchanged. The config compares by mod revision (a flip or retire moves it),
+/// the migration by create revision (a new or archived saga moves it).
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct PlanBasis {
+ pub config: Option<ChangelogConfig>,
+ pub migration: Option<MigrationState>,
+ config_revision: i64,
+ migration_revision: i64,
+}
+
+/// The record a coordinator plan was decided on. Its transactions require it
+/// unchanged, so a stalled pass that wakes after a successor acted lands nothing.
+#[derive(Debug, Clone, Copy)]
+enum Authority {
+ /// The planner: its plan basis.
+ Planner {
+ config_revision: i64,
+ migration_revision: i64,
+ },
+ /// The migration driver: the saga record at the revision its pass read.
+ Migration { mod_revision: i64 },
}
/// One partition's slice of a plan: the guards that make its
@@ -162,11 +216,13 @@ fn is_txn_too_large(e: &Error) -> bool {
/// Greedy chunk boundaries over per-unit `(guards, ops)` costs: a chunk
/// takes units while its compare list and its op list both stay within
/// `budget` — etcd checks the two lists separately against
-/// `--max-txn-ops`. `reserve_ops` holds back room in every chunk for
-/// ops added on top of its units (the freeze-quorum write).
+/// `--max-txn-ops`. `reserve_guards` and `reserve_ops` hold back room in
+/// every chunk for what is added on top of its units: the plan basis
+/// compares and the freeze-quorum write.
fn plan_chunk_ranges(
costs: &[(usize, usize)],
budget: usize,
+ reserve_guards: usize,
reserve_ops: usize,
) -> Vec<std::ops::Range<usize>> {
let budget = budget.max(1);
@@ -174,7 +230,7 @@ fn plan_chunk_ranges(
let mut start = 0;
let (mut guards, mut ops) = (0, 0);
for (i, &(g, o)) in costs.iter().enumerate() {
- let fits = guards + g <= budget && ops + o + reserve_ops <= budget;
+ let fits = guards + g + reserve_guards <= budget && ops + o + reserve_ops <= budget;
// A chunk always takes at least one unit, so a budget below a
// single unit's cost still makes progress and lets the server
// be the judge.
@@ -699,6 +755,7 @@ impl PersonhogStore {
/// the predecessor's acks in the same transaction. A non-terminal
/// handoff record is never deleted; it is only ever replaced by the
/// thing that resolves its stashes.
+ #[cfg(any(test, feature = "test-support"))]
pub async fn create_assignments_and_handoffs(
&self,
assignments: &[PartitionAssignment],
@@ -724,7 +781,9 @@ impl PersonhogStore {
/// unchunked plan past `--max-txn-ops` is rejected as a hard error,
/// and a coordinator retrying it re-earns the same rejection
/// forever. A partially applied plan is placement drift the next
- /// reconcile-tick replan heals.
+ /// reconcile-tick replan heals. Production plans carry the record they
+ /// were decided on (`apply_plan_on`, `apply_migration_plan`).
+ #[cfg(any(test, feature = "test-support"))]
pub async fn apply_plan(
&self,
assignments: &[PartitionAssignment],
@@ -736,6 +795,156 @@ impl PersonhogStore {
// handoff is ever durable with a dangling reference.
freeze_quorum: Option<(&str, &[String])>,
max_txn_ops: usize,
+ ) -> Result<PlanApplication> {
+ self.apply_plan_inner(
+ assignments,
+ handoffs,
+ replacements,
+ preconditions,
+ freeze_quorum,
+ max_txn_ops,
+ None,
+ )
+ .await
+ }
+
+ /// The planner's plan, landing only while `basis` still holds.
+ pub async fn apply_plan_on(
+ &self,
+ basis: &PlanBasis,
+ handoffs: &[HandoffState],
+ replacements: &[HandoffReplacement],
+ preconditions: &[AssignmentPrecondition],
+ freeze_quorum: Option<(&str, &[String])>,
+ max_txn_ops: usize,
+ ) -> Result<PlanApplication> {
+ let authority = Authority::Planner {
+ config_revision: basis.config_revision,
+ migration_revision: basis.migration_revision,
+ };
+ self.apply_plan_inner(
+ &[],
+ handoffs,
+ replacements,
+ preconditions,
+ freeze_quorum,
+ max_txn_ops,
+ Some(authority),
+ )
+ .await
+ }
+
+ /// A migration pass's plan; it lands only while the saga record is unchanged.
+ pub async fn apply_migration_plan(
+ &self,
+ migration_mod_revision: i64,
+ handoffs: &[HandoffState],
+ replacements: &[HandoffReplacement],
+ preconditions: &[AssignmentPrecondition],
+ freeze_quorum: Option<(&str, &[String])>,
+ max_txn_ops: usize,
+ ) -> Result<PlanApplication> {
+ self.apply_plan_inner(
+ &[],
+ handoffs,
+ replacements,
+ preconditions,
+ freeze_quorum,
+ max_txn_ops,
+ Some(Authority::Migration {
+ mod_revision: migration_mod_revision,
+ }),
+ )
+ .await
+ }
+
+ /// The changelog config and the migration record, read in one transaction.
+ pub async fn read_plan_basis(&self) -> Result<PlanBasis> {
+ count_call("read_plan_basis");
+ let txn = Txn::new().and_then(vec![
+ TxnOp::get(self.key(StoreKey::ChangelogConfig), None),
+ TxnOp::get(self.key(StoreKey::Migration), None),
+ ]);
+ let resp = self.inner.txn(txn).await?;
+ let mut kvs = Vec::new();
+ for op in resp.op_responses() {
+ match op {
+ TxnOpResponse::Get(get) => kvs.push(get.kvs().first().cloned()),
+ _ => {
+ return Err(crate::error::Error::invalid_state(
+ "plan basis read returned a non-get response",
+ ))
+ }
+ }
+ }
+ let [config, migration]: [Option<etcd_client::KeyValue>; 2] = kvs
+ .try_into()
+ .map_err(|_| crate::error::Error::invalid_state("plan basis read lost a get"))?;
+ Ok(PlanBasis {
+ config_revision: config.as_ref().map_or(0, |kv| kv.mod_revision()),
+ migration_revision: migration.as_ref().map_or(0, |kv| kv.create_revision()),
+ config: config
+ .map(|kv| serde_json::from_slice(kv.value()))
+ .transpose()?,
+ migration: migration
+ .map(|kv| serde_json::from_slice(kv.value()))
+ .transpose()?,
+ })
+ }
+
+ fn authority_guards(&self, authority: Authority) -> Vec<Compare> {
+ let migration = self.key(StoreKey::Migration);
+ match authority {
+ Authority::Planner {
+ config_revision,
+ migration_revision,
+ } => vec![
+ Compare::mod_revision(
+ self.key(StoreKey::ChangelogConfig),
+ CompareOp::Equal,
+ config_revision,
+ ),
+ Compare::create_revision(migration, CompareOp::Equal, migration_revision),
+ ],
+ Authority::Migration { mod_revision } => {
+ vec![Compare::mod_revision(
+ migration,
+ CompareOp::Equal,
+ mod_revision,
+ )]
+ }
+ }
+ }
+
+ /// Whether a plan's authority has moved; a moved authority fails every unit alike.
+ async fn authority_moved(&self, authority: Option<Authority>) -> Result<bool> {
+ Ok(match authority {
+ None => false,
+ Some(Authority::Planner {
+ config_revision,
+ migration_revision,
+ }) => {
+ let now = self.read_plan_basis().await?;
+ (now.config_revision, now.migration_revision)
+ != (config_revision, migration_revision)
+ }
+ Some(Authority::Migration { mod_revision }) => self
+ .get_migration_with_mod_revision()
+ .await?
+ .is_none_or(|(_, now)| now != mod_revision),
+ })
+ }
+
+ #[allow(clippy::too_many_arguments)]
+ async fn apply_plan_inner(
+ &self,
+ assignments: &[PartitionAssignment],
+ handoffs: &[HandoffState],
+ replacements: &[HandoffReplacement],
+ preconditions: &[AssignmentPrecondition],
+ freeze_quorum: Option<(&str, &[String])>,
+ max_txn_ops: usize,
+ authority: Option<Authority>,
) -> Result<PlanApplication> {
count_call("apply_plan");
let mut units: HashMap<u32, PlanUnit> = HashMap::new();
@@ -842,19 +1051,27 @@ impl PersonhogStore {
.map(|u| (u.guards.len(), u.ops.len()))
.collect();
let reserve_ops = usize::from(quorum.is_some());
- let ranges = plan_chunk_ranges(&costs, max_txn_ops, reserve_ops);
+ let authority_guards = authority
+ .map(|a| self.authority_guards(a))
+ .unwrap_or_default();
+ let reserve_guards = authority_guards.len();
+ let ranges = plan_chunk_ranges(&costs, max_txn_ops, reserve_guards, reserve_ops);
metrics::histogram!("personhog_coordination_plan_chunks").record(ranges.len() as f64);
let mut application = PlanApplication::default();
for range in ranges {
- let chunk = &units[range];
- match self.apply_units(chunk, &quorum).await {
+ let chunk = &units[range.clone()];
+ match self.apply_units(chunk, &quorum, &authority_guards).await {
Ok(true) => {
application
.applied
.extend(chunk.iter().map(|u| u.partition));
continue;
}
+ Ok(false) if self.authority_moved(authority).await? => {
+ application.stand_down(&units[range.start..], authority);
+ return Ok(application);
+ }
Ok(false) => {}
// The server's effective --max-txn-ops can sit below the
// configured budget (values drift, a member not yet
@@ -877,14 +1094,26 @@ impl PersonhogStore {
// A failed guard names no key, so retry the chunk one
// partition at a time: the concurrent write that broke the
// chunk stands down only the units it actually touched.
- for u in chunk {
- if self.apply_units(std::slice::from_ref(u), &quorum).await? {
+ // One authority check per fallback: past a move every unit fails alike.
+ let mut checked = false;
+ for (i, u) in chunk.iter().enumerate() {
+ if self
+ .apply_units(std::slice::from_ref(u), &quorum, &authority_guards)
+ .await?
+ {
application.applied.push(u.partition);
- } else {
- metrics::counter!("personhog_coordination_plan_units_conflicted_total")
- .increment(1);
- application.conflicted.push(u.partition);
+ continue;
+ }
+ if !checked {
+ checked = true;
+ if self.authority_moved(authority).await? {
+ application.stand_down(&units[range.start + i..], authority);
+ return Ok(application);
+ }
}
+ metrics::counter!("personhog_coordination_plan_units_conflicted_total")
+ .increment(1);
+ application.conflicted.push(u.partition);
}
}
Ok(application)
@@ -897,8 +1126,9 @@ impl PersonhogStore {
&self,
chunk: &[PlanUnit],
quorum: &Option<(String, Vec<u8>)>,
+ authority_guards: &[Compare],
) -> Result<bool> {
- let mut guards: Vec<Compare> = Vec::new();
+ let mut guards: Vec<Compare> = authority_guards.to_vec();
let mut ops: Vec<TxnOp> = Vec::new();
if let Some((key, value)) = quorum {
if chunk.iter().any(|u| u.references_quorum) {
@@ -1199,6 +1429,14 @@ impl PersonhogStore {
Ok(self.inner.get_with_revision(&key).await?)
}
+ pub async fn get_changelog_config_with_mod_revision(
+ &self,
+ ) -> Result<Option<(ChangelogConfig, i64)>> {
+ count_call("get_changelog_config_with_mod_revision");
+ let key = self.key(StoreKey::ChangelogConfig);
+ Ok(self.inner.get_with_mod_revision(&key).await?)
+ }
+
/// Create-if-absent at epoch 1; only a flip may move a live config.
pub async fn seed_changelog_config(
&self,
@@ -1298,7 +1536,7 @@ mod tests {
#[test]
fn creations_chunk_on_the_compare_list() {
let costs = vec![(2, 1); 256];
- let ranges = plan_chunk_ranges(&costs, 128, 1);
+ let ranges = plan_chunk_ranges(&costs, 128, 0, 1);
assert_contiguous_cover(&ranges, costs.len());
for r in &ranges {
let guards: usize = costs[r.clone()].iter().map(|c| c.0).sum();
@@ -1319,7 +1557,7 @@ mod tests {
#[test]
fn replacements_chunk_on_the_op_list_with_the_reserve() {
let costs = vec![(2, 4); 256];
- let ranges = plan_chunk_ranges(&costs, 128, 1);
+ let ranges = plan_chunk_ranges(&costs, 128, 0, 1);
assert_contiguous_cover(&ranges, costs.len());
for r in &ranges {
let ops: usize = costs[r.clone()].iter().map(|c| c.1).sum();
@@ -1332,7 +1570,7 @@ mod tests {
#[test]
fn a_fitting_plan_is_one_chunk() {
let costs = vec![(2, 1); 60];
- assert_eq!(plan_chunk_ranges(&costs, 128, 1), vec![0..60]);
+ assert_eq!(plan_chunk_ranges(&costs, 128, 0, 1), vec![0..60]);
}
/// A budget below a single unit's cost still emits that unit alone:
@@ -1340,6 +1578,14 @@ mod tests {
#[test]
fn an_oversized_unit_is_its_own_chunk() {
let costs = vec![(2, 4), (2, 4)];
- assert_eq!(plan_chunk_ranges(&costs, 1, 0), vec![0..1, 1..2]);
+ assert_eq!(plan_chunk_ranges(&costs, 1, 0, 0), vec![0..1, 1..2]);
+ }
+
+ /// A plan's authority compares are reserved in every chunk.
+ #[test]
+ fn a_reserved_guard_takes_a_compare_slot() {
+ let costs = vec![(2, 1), (2, 1)];
+ assert_eq!(plan_chunk_ranges(&costs, 4, 0, 0), vec![0..2]);
+ assert_eq!(plan_chunk_ranges(&costs, 4, 1, 0), vec![0..1, 1..2]);
}
}
diff --git a/rust/personhog-coordination/src/store/migration.rs b/rust/personhog-coordination/src/store/migration.rs
new file mode 100644
index 000000000000..3d68d210ec5f
--- /dev/null
+++ b/rust/personhog-coordination/src/store/migration.rs
@@ -0,0 +1,407 @@
+//! The migration saga's store operations: requests, the saga record, seal acks and the flip.
+
+use etcd_client::{Compare, CompareOp, DeleteOptions, Txn, TxnOp, WatchStream};
+
+use super::{count_call, PersonhogStore, StoreKey};
+use crate::error::Result;
+use crate::types::{
+ ChangelogConfig, Generation, MigrationPhase, MigrationRequest, MigrationState, SealAck,
+};
+
+impl PersonhogStore {
+ pub async fn get_migration(&self) -> Result<Option<MigrationState>> {
+ count_call("get_migration");
+ let key = self.key(StoreKey::Migration);
+ Ok(self.inner.get(&key).await?)
+ }
+
+ pub async fn get_migration_with_mod_revision(&self) -> Result<Option<(MigrationState, i64)>> {
+ count_call("get_migration_with_mod_revision");
+ let key = self.key(StoreKey::Migration);
+ Ok(self.inner.get_with_mod_revision(&key).await?)
+ }
+
+ /// Writes the saga record directly, skipping the request and its quiet-fleet start.
+ #[cfg(any(test, feature = "test-support"))]
+ pub async fn create_migration(&self, migration: &MigrationState) -> Result<bool> {
+ count_call("create_migration");
+ let key = self.key(StoreKey::Migration);
+ let txn = Txn::new()
+ .when(vec![Compare::create_revision(
+ key.clone(),
+ CompareOp::Equal,
+ 0,
+ )])
+ .and_then(vec![TxnOp::put(key, serde_json::to_vec(migration)?, None)]);
+ let resp = self.inner.txn(txn).await?;
+ Ok(resp.succeeded())
+ }
+
+ pub async fn cas_migration_phase(
+ &self,
+ expected_id: &str,
+ expected_mod_revision: i64,
+ next: MigrationPhase,
+ ) -> Result<bool> {
+ count_call("cas_migration_phase");
+ self.cas_phase(expected_id, expected_mod_revision, next, None)
+ .await
+ }
+
+ /// Move the saga to Cancelling, recording why.
+ pub async fn cancel_migration(
+ &self,
+ expected_id: &str,
+ expected_mod_revision: i64,
+ reason: &str,
+ ) -> Result<bool> {
+ count_call("cancel_migration");
+ self.cas_phase(
+ expected_id,
+ expected_mod_revision,
+ MigrationPhase::Cancelling,
+ Some(reason),
+ )
+ .await
+ }
+
+ async fn cas_phase(
+ &self,
+ expected_id: &str,
+ expected_mod_revision: i64,
+ next: MigrationPhase,
+ cancel_reason: Option<&str>,
+ ) -> Result<bool> {
+ let key = self.key(StoreKey::Migration);
+ let Some((mut migration, mod_revision)) = self
+ .inner
+ .get_with_mod_revision::<MigrationState>(&key)
+ .await?
+ else {
+ return Ok(false);
+ };
+ if migration.id != expected_id
+ || mod_revision != expected_mod_revision
+ || !migration.phase.may_transition_to(next)
+ {
+ return Ok(false);
+ }
+ migration.phase = next;
+ migration.phase_entered_at_ms = assignment_coordination::util::now_millis();
+ migration.blocking.clear();
+ if let Some(reason) = cancel_reason {
+ migration.cancel_reason = Some(reason.to_string());
+ }
+ let txn = Txn::new()
+ .when(vec![Compare::mod_revision(
+ key.clone(),
+ CompareOp::Equal,
+ mod_revision,
+ )])
+ .and_then(vec![TxnOp::put(key, serde_json::to_vec(&migration)?, None)]);
+ Ok(self.inner.txn(txn).await?.succeeded())
+ }
+
+ pub async fn set_migration_blocking(
+ &self,
+ expected_mod_revision: i64,
+ blocking: Vec<String>,
+ ) -> Result<bool> {
+ count_call("set_migration_blocking");
+ let key = self.key(StoreKey::Migration);
+ let Some((mut migration, mod_revision)) = self
+ .inner
+ .get_with_mod_revision::<MigrationState>(&key)
+ .await?
+ else {
+ return Ok(false);
+ };
+ if mod_revision != expected_mod_revision {
+ return Ok(false);
+ }
+ migration.blocking = blocking;
+ let txn = Txn::new()
+ .when(vec![Compare::mod_revision(
+ key.clone(),
+ CompareOp::Equal,
+ mod_revision,
+ )])
+ .and_then(vec![TxnOp::put(key, serde_json::to_vec(&migration)?, None)]);
+ Ok(self.inner.txn(txn).await?.succeeded())
+ }
+
+ pub async fn flip_changelog(
+ &self,
+ migration: &MigrationState,
+ migration_mod_revision: i64,
+ config: &ChangelogConfig,
+ config_mod_revision: i64,
+ ) -> Result<bool> {
+ count_call("flip_changelog");
+ let migration_key = self.key(StoreKey::Migration);
+ let config_key = self.key(StoreKey::ChangelogConfig);
+ let mut draining = config.draining.clone();
+ draining.push(Generation {
+ topic: migration.from.topic.clone(),
+ partitions: migration.from.partitions,
+ epoch: config.epoch,
+ });
+ let flipped = ChangelogConfig {
+ topic: migration.to.topic.clone(),
+ partitions: migration.to.partitions,
+ epoch: config.epoch + 1,
+ writer_group: config.writer_group.clone(),
+ draining,
+ };
+ let mut record = migration.clone();
+ record.phase = MigrationPhase::Resuming;
+ record.phase_entered_at_ms = assignment_coordination::util::now_millis();
+ record.blocking.clear();
+ let txn = Txn::new()
+ .when(vec![
+ Compare::mod_revision(
+ migration_key.clone(),
+ CompareOp::Equal,
+ migration_mod_revision,
+ ),
+ Compare::mod_revision(config_key.clone(), CompareOp::Equal, config_mod_revision),
+ ])
+ .and_then(vec![
+ TxnOp::put(config_key, serde_json::to_vec(&flipped)?, None),
+ TxnOp::put(migration_key, serde_json::to_vec(&record)?, None),
+ ]);
+ Ok(self.inner.txn(txn).await?.succeeded())
+ }
+
+ pub async fn archive_migration(
+ &self,
+ migration: &MigrationState,
+ mod_revision: i64,
+ final_phase: MigrationPhase,
+ ) -> Result<bool> {
+ count_call("archive_migration");
+ let key = self.key(StoreKey::Migration);
+ let mut record = migration.clone();
+ record.phase = final_phase;
+ record.phase_entered_at_ms = assignment_coordination::util::now_millis();
+ let txn = Txn::new()
+ .when(vec![Compare::mod_revision(
+ key.clone(),
+ CompareOp::Equal,
+ mod_revision,
+ )])
+ .and_then(vec![
+ TxnOp::put(
+ self.key(StoreKey::MigrationHistory(&migration.id)),
+ serde_json::to_vec(&record)?,
+ None,
+ ),
+ TxnOp::delete(
+ self.key(StoreKey::SealAcksPrefix),
+ Some(DeleteOptions::new().with_prefix()),
+ ),
+ TxnOp::delete(key, None),
+ ]);
+ Ok(self.inner.txn(txn).await?.succeeded())
+ }
+
+ pub async fn list_migration_history(&self) -> Result<Vec<MigrationState>> {
+ count_call("list_migration_history");
+ let key = self.key(StoreKey::MigrationHistoryPrefix);
+ Ok(self.inner.list(&key).await?)
+ }
+
+ pub async fn put_seal_ack(&self, ack: &SealAck) -> Result<()> {
+ count_call("put_seal_ack");
+ let key = self.key(StoreKey::SealAck(&ack.pod_name));
+ Ok(self.inner.put(&key, ack, None).await?)
+ }
+
+ /// The mod revision of one pod's seal ack, to tell a rewrite from no write.
+ #[cfg(any(test, feature = "test-support"))]
+ pub async fn seal_ack_mod_revision(&self, pod_name: &str) -> Result<Option<i64>> {
+ let key = self.key(StoreKey::SealAck(pod_name));
+ Ok(self
+ .inner
+ .get_with_mod_revision::<SealAck>(&key)
+ .await?
+ .map(|(_, rev)| rev))
+ }
+
+ pub async fn list_seal_acks(&self) -> Result<Vec<SealAck>> {
+ count_call("list_seal_acks");
+ let key = self.key(StoreKey::SealAcksPrefix);
+ Ok(self.inner.list(&key).await?)
+ }
+
+ /// Drop a drained generation from the config, guarded on its revision.
+ pub async fn retire_changelog_topic(&self, topic: &str) -> Result<bool> {
+ count_call("retire_changelog_topic");
+ let Some((mut config, mod_revision)) =
+ self.get_changelog_config_with_mod_revision().await?
+ else {
+ return Ok(false);
+ };
+ let before = config.draining.len();
+ config.draining.retain(|g| g.topic != topic);
+ if config.draining.len() == before {
+ return Ok(false);
+ }
+ let key = self.key(StoreKey::ChangelogConfig);
+ let txn = Txn::new()
+ .when(vec![Compare::mod_revision(
+ key.clone(),
+ CompareOp::Equal,
+ mod_revision,
+ )])
+ .and_then(vec![TxnOp::put(key, serde_json::to_vec(&config)?, None)]);
+ Ok(self.inner.txn(txn).await?.succeeded())
+ }
+
+ /// Write a request, refused while another request or a saga exists.
+ pub async fn create_migration_request(&self, request: &MigrationRequest) -> Result<bool> {
+ count_call("create_migration_request");
+ let key = self.key(StoreKey::MigrationRequest);
+ let txn = Txn::new()
+ .when(vec![
+ Compare::create_revision(key.clone(), CompareOp::Equal, 0),
+ Compare::create_revision(self.key(StoreKey::Migration), CompareOp::Equal, 0),
+ ])
+ .and_then(vec![TxnOp::put(key, serde_json::to_vec(request)?, None)]);
+ Ok(self.inner.txn(txn).await?.succeeded())
+ }
+
+ pub async fn get_migration_request_with_mod_revision(
+ &self,
+ ) -> Result<Option<(MigrationRequest, i64)>> {
+ count_call("get_migration_request_with_mod_revision");
+ let key = self.key(StoreKey::MigrationRequest);
+ Ok(self.inner.get_with_mod_revision(&key).await?)
+ }
+
+ /// Rewrite a waiting request (its first sighting or what it waits on), guarded on its revision.
+ pub async fn update_migration_request(
+ &self,
+ record: &MigrationRequest,
+ mod_revision: i64,
+ ) -> Result<bool> {
+ count_call("update_migration_request");
+ let key = self.key(StoreKey::MigrationRequest);
+ let txn = Txn::new()
+ .when(vec![Compare::mod_revision(
+ key.clone(),
+ CompareOp::Equal,
+ mod_revision,
+ )])
+ .and_then(vec![TxnOp::put(key, serde_json::to_vec(record)?, None)]);
+ Ok(self.inner.txn(txn).await?.succeeded())
+ }
+
+ /// Start the saga from a request, only if no handoff record exists that was
+ /// written after `handoffs_revision`; from an empty read, that is none at all.
+ pub async fn start_migration(
+ &self,
+ request_mod_revision: i64,
+ handoffs_revision: i64,
+ migration: &MigrationState,
+ ) -> Result<bool> {
+ count_call("start_migration");
+ let request_key = self.key(StoreKey::MigrationRequest);
+ let migration_key = self.key(StoreKey::Migration);
+ let txn = Txn::new()
+ .when(vec![
+ Compare::mod_revision(request_key.clone(), CompareOp::Equal, request_mod_revision),
+ Compare::create_revision(migration_key.clone(), CompareOp::Equal, 0),
+ Compare::mod_revision(
+ self.key(StoreKey::HandoffsPrefix),
+ CompareOp::Less,
+ handoffs_revision + 1,
+ )
+ .with_prefix(),
+ ])
+ .and_then(vec![
+ TxnOp::put(migration_key, serde_json::to_vec(migration)?, None),
+ TxnOp::delete(request_key, None),
+ ]);
+ Ok(self.inner.txn(txn).await?.succeeded())
+ }
+
+ /// Archive a request that will not start, with its reason, guarded on its revision.
+ pub async fn reject_migration_request(
+ &self,
+ request: &MigrationRequest,
+ mod_revision: i64,
+ reason: &str,
+ ) -> Result<bool> {
+ count_call("reject_migration_request");
+ let key = self.key(StoreKey::MigrationRequest);
+ let record = request.rejected(reason, assignment_coordination::util::now_millis());
+ let txn = Txn::new()
+ .when(vec![Compare::mod_revision(
+ key.clone(),
+ CompareOp::Equal,
+ mod_revision,
+ )])
+ .and_then(vec![
+ TxnOp::put(
+ self.key(StoreKey::MigrationHistory(&request.id)),
+ serde_json::to_vec(&record)?,
+ None,
+ ),
+ TxnOp::delete(key, None),
+ ]);
+ Ok(self.inner.txn(txn).await?.succeeded())
+ }
+
+ pub async fn watch_migration_request_from(&self, start_revision: i64) -> Result<WatchStream> {
+ count_call("watch_migration_request_from");
+ let key = self.key(StoreKey::MigrationRequest);
+ Ok(self.inner.watch_key_from(&key, start_revision).await?)
+ }
+
+ pub async fn watch_migration_from(&self, start_revision: i64) -> Result<WatchStream> {
+ count_call("watch_migration_from");
+ let key = self.key(StoreKey::Migration);
+ Ok(self.inner.watch_key_from(&key, start_revision).await?)
+ }
+
+ pub async fn watch_seal_acks_from(&self, start_revision: i64) -> Result<WatchStream> {
+ count_call("watch_seal_acks_from");
+ let key = self.key(StoreKey::SealAcksPrefix);
+ Ok(self.inner.watch_from(&key, start_revision).await?)
+ }
+
+ /// A shrunk partition's hold, acks and assignment go in one guarded transaction.
+ pub async fn dissolve_partition(
+ &self,
+ partition: u32,
+ handoff_mod_revision: i64,
+ ) -> Result<bool> {
+ count_call("dissolve_partition");
+ let handoff_key = self.key(StoreKey::Handoff(partition));
+ let prefix_delete = || Some(DeleteOptions::new().with_prefix());
+ let txn = Txn::new()
+ .when(vec![Compare::mod_revision(
+ handoff_key.clone(),
+ CompareOp::Equal,
+ handoff_mod_revision,
+ )])
+ .and_then(vec![
+ TxnOp::delete(
+ self.key(StoreKey::FreezeAcksForPartition(partition)),
+ prefix_delete(),
+ ),
+ TxnOp::delete(
+ self.key(StoreKey::DrainedAcksForPartition(partition)),
+ prefix_delete(),
+ ),
+ TxnOp::delete(
+ self.key(StoreKey::WarmedAcksForPartition(partition)),
+ prefix_delete(),
+ ),
+ TxnOp::delete(self.key(StoreKey::Assignment(partition)), None),
+ TxnOp::delete(handoff_key, None),
+ ]);
+ Ok(self.inner.txn(txn).await?.succeeded())
+ }
+}
diff --git a/rust/personhog-coordination/src/types.rs b/rust/personhog-coordination/src/types.rs
index 25f7a3d1d6dd..25e2e54be77f 100644
--- a/rust/personhog-coordination/src/types.rs
+++ b/rust/personhog-coordination/src/types.rs
@@ -1,3 +1,5 @@
+use std::collections::BTreeSet;
+
use k8s_awareness::types::ControllerRef;
use serde::{Deserialize, Serialize};
@@ -188,11 +190,20 @@ pub struct HandoffState {
/// Zero means the record predates the field.
#[serde(default)]
pub phase_entered_at_ms: i64,
+ /// A migration's hold: driver-owned, never advanced past Draining.
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub held_by: Option<String>,
/// Changelog epoch the record was planned under.
#[serde(default)]
pub config_epoch: u64,
}
+impl HandoffState {
+ pub fn is_held(&self) -> bool {
+ self.held_by.is_some()
+ }
+}
+
/// State machine for partition handoffs:
///
/// ```text
@@ -361,6 +372,100 @@ pub struct ChangelogTarget {
pub partitions: u32,
}
+/// Sealing is the point of no return: the old topic's ids are re-initialized there.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)]
+pub enum MigrationPhase {
+ Freezing,
+ Draining,
+ Sealing,
+ Resuming,
+ Complete,
+ Cancelling,
+ Cancelled,
+}
+
+impl MigrationPhase {
+ pub fn may_transition_to(self, next: MigrationPhase) -> bool {
+ matches!(
+ (self, next),
+ (Self::Freezing, Self::Draining)
+ | (Self::Draining, Self::Sealing)
+ | (Self::Sealing, Self::Resuming)
+ | (Self::Resuming, Self::Complete)
+ | (Self::Freezing | Self::Draining, Self::Cancelling)
+ | (Self::Cancelling, Self::Cancelled)
+ )
+ }
+}
+
+/// A request to migrate: the control plane writes it, and the coordinator
+/// starts the saga from it only once the fleet is quiet (`protocol::migration::fleet_quiet`).
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct MigrationRequest {
+ pub id: String,
+ pub from: ChangelogTarget,
+ pub to: ChangelogTarget,
+ pub requested_at_ms: i64,
+ /// When the coordinator first saw it, on the coordinator's clock; the expiry counts from here.
+ #[serde(default)]
+ pub seen_at_ms: Option<i64>,
+ /// Why the coordinator has not started it yet.
+ #[serde(default)]
+ pub waiting: Vec<String>,
+}
+
+impl MigrationRequest {
+ /// The saga record the start creates; its pre-flip deadline counts from `now_ms`.
+ pub fn start(&self, now_ms: i64) -> MigrationState {
+ MigrationState {
+ id: self.id.clone(),
+ from: self.from.clone(),
+ to: self.to.clone(),
+ phase: MigrationPhase::Freezing,
+ created_at_ms: now_ms,
+ phase_entered_at_ms: now_ms,
+ blocking: Vec::new(),
+ cancel_reason: None,
+ }
+ }
+
+ /// The history record of a request that never started.
+ pub fn rejected(&self, reason: &str, now_ms: i64) -> MigrationState {
+ MigrationState {
+ phase: MigrationPhase::Cancelled,
+ created_at_ms: self.requested_at_ms,
+ phase_entered_at_ms: now_ms,
+ blocking: std::iter::once(reason.to_string())
+ .chain(self.waiting.iter().cloned())
+ .collect(),
+ ..self.start(self.requested_at_ms)
+ }
+ }
+}
+
+/// The saga record: only the coordinator creates and advances it.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct MigrationState {
+ pub id: String,
+ pub from: ChangelogTarget,
+ pub to: ChangelogTarget,
+ pub phase: MigrationPhase,
+ pub created_at_ms: i64,
+ pub phase_entered_at_ms: i64,
+ #[serde(default)]
+ pub blocking: Vec<String>,
+ /// Why the saga was cancelled, recorded by the cancelling write.
+ pub cancel_reason: Option<String>,
+}
+
+/// One leader's report of the old partitions it sealed; the flip takes the union.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct SealAck {
+ pub migration_id: String,
+ pub pod_name: String,
+ pub sealed: BTreeSet<u32>,
+}
+
#[cfg(test)]
mod tests {
use super::*;
@@ -384,6 +489,7 @@ mod tests {
assert!(h.freeze_quorum.is_none());
assert_eq!(h.created_at_ms, 0);
assert_eq!(h.phase_entered_at_ms, 0);
+ assert!(!h.is_held());
assert_eq!(h.config_epoch, 0);
}
@@ -486,6 +592,7 @@ mod tests {
freeze_quorum_ref: None,
created_at_ms: 0,
phase_entered_at_ms: 0,
+ held_by: None,
config_epoch: 0,
new_owner_address: None,
};
@@ -507,6 +614,7 @@ mod tests {
freeze_quorum_ref: None,
created_at_ms: 0,
phase_entered_at_ms: 0,
+ held_by: None,
config_epoch: 0,
new_owner_address: None,
};
@@ -588,4 +696,18 @@ mod tests {
assert_eq!(serde_json::to_string(&phase).unwrap(), expected);
}
}
+
+ #[test]
+ fn a_saga_cancels_only_before_the_seal() {
+ use MigrationPhase::*;
+ for phase in [
+ Freezing, Draining, Sealing, Resuming, Complete, Cancelling, Cancelled,
+ ] {
+ assert_eq!(
+ phase.may_transition_to(Cancelling),
+ matches!(phase, Freezing | Draining),
+ "{phase:?}"
+ );
+ }
+ }
}
diff --git a/rust/personhog-coordination/tests/common/mod.rs b/rust/personhog-coordination/tests/common/mod.rs
index 334d3a25c9eb..5d37cb6776cc 100644
--- a/rust/personhog-coordination/tests/common/mod.rs
+++ b/rust/personhog-coordination/tests/common/mod.rs
@@ -138,32 +138,47 @@ pub fn start_coordinator_with_deadline(
strategy: Arc<dyn AssignmentStrategy>,
cancel: CancellationToken,
) -> JoinHandle<Result<()>> {
+ let config = coordinator_config(name, leader_lease_ttl, handoff_deadline);
+ start_coordinator_with_config(store, config, strategy, cancel)
+}
+
+pub fn coordinator_config(
+ name: &str,
+ leader_lease_ttl: i64,
+ handoff_deadline: Duration,
+) -> CoordinatorConfig {
let keepalive_secs = (leader_lease_ttl as u64 / 3).max(1);
- let coordinator = Coordinator::new(
- store,
- CoordinatorConfig {
- name: name.to_string(),
- leader_lease_ttl,
- keepalive_interval: Duration::from_secs(keepalive_secs),
- // Short enough that a failover never waits on the leader-key
- // watch alone.
- standby_poll_interval: Duration::from_millis(500),
- run_retry_backoff: Duration::from_millis(10),
- backoff_decay_window: Duration::from_secs(300),
- rebalance_debounce_interval: Duration::from_millis(100),
- reconcile_interval: Duration::from_millis(500),
- // Callers default this to a day: these tests deliberately
- // park handoffs mid-phase to assert what the protocol does
- // with them, and a live deadline would replace the state
- // under test. The cancellation test passes a short one
- // explicitly.
- handoff_deadline,
- warming_deadline: Duration::from_secs(86_400),
- max_txn_ops: 128,
- },
- strategy,
- None,
- );
+ CoordinatorConfig {
+ name: name.to_string(),
+ leader_lease_ttl,
+ keepalive_interval: Duration::from_secs(keepalive_secs),
+ // Short enough that a failover never waits on the leader-key
+ // watch alone.
+ standby_poll_interval: Duration::from_millis(500),
+ run_retry_backoff: Duration::from_millis(10),
+ backoff_decay_window: Duration::from_secs(300),
+ rebalance_debounce_interval: Duration::from_millis(100),
+ reconcile_interval: Duration::from_millis(500),
+ // Callers default this to a day: these tests deliberately
+ // park handoffs mid-phase to assert what the protocol does
+ // with them, and a live deadline would replace the state
+ // under test. The cancellation test passes a short one
+ // explicitly.
+ handoff_deadline,
+ warming_deadline: Duration::from_secs(86_400),
+ max_txn_ops: 128,
+ migration_pre_flip_deadline: Duration::from_secs(86_400),
+ migration_debounce_interval: Duration::from_millis(10),
+ }
+}
+
+pub fn start_coordinator_with_config(
+ store: Arc<PersonhogStore>,
+ config: CoordinatorConfig,
+ strategy: Arc<dyn AssignmentStrategy>,
+ cancel: CancellationToken,
+) -> JoinHandle<Result<()>> {
+ let coordinator = Coordinator::new(store, config, strategy, None);
let token = cancel.child_token();
tokio::spawn(async move {
coordinator.run(token).await;
diff --git a/rust/personhog-coordination/tests/integration.rs b/rust/personhog-coordination/tests/integration.rs
index d615e95426d2..cfd82853647c 100644
--- a/rust/personhog-coordination/tests/integration.rs
+++ b/rust/personhog-coordination/tests/integration.rs
@@ -1471,6 +1471,7 @@ async fn a_dead_new_owner_handoff_is_reaffirmed_to_the_current_owner() {
freeze_quorum_ref: None,
created_at_ms: 0,
phase_entered_at_ms: 0,
+ held_by: None,
config_epoch: 0,
new_owner_address: None,
};
@@ -1606,6 +1607,7 @@ async fn stale_handoff_address_never_overwrites_a_fresher_registration() {
freeze_quorum_ref: None,
created_at_ms: 0,
phase_entered_at_ms: 0,
+ held_by: None,
config_epoch: 0,
})
.await
@@ -1838,6 +1840,7 @@ async fn late_joining_router_during_warming_begins_stash() {
freeze_quorum_ref: None,
created_at_ms: 0,
phase_entered_at_ms: 0,
+ held_by: None,
config_epoch: 0,
new_owner_address: None,
};
@@ -1910,6 +1913,7 @@ async fn dead_old_owner_in_freezing_advances_to_completion() {
freeze_quorum_ref: None,
created_at_ms: 0,
phase_entered_at_ms: 0,
+ held_by: None,
config_epoch: 0,
new_owner_address: None,
};
@@ -1967,6 +1971,7 @@ async fn late_joining_router_during_freezing_acks_and_stashes() {
freeze_quorum_ref: None,
created_at_ms: 0,
phase_entered_at_ms: 0,
+ held_by: None,
config_epoch: 0,
new_owner_address: None,
};
@@ -2054,6 +2059,7 @@ async fn a_dead_new_owner_handoff_in_warming_is_reaffirmed() {
freeze_quorum_ref: None,
created_at_ms: 0,
phase_entered_at_ms: 0,
+ held_by: None,
config_epoch: 0,
new_owner_address: None,
};
@@ -2160,6 +2166,7 @@ async fn reconcile_advances_warming_with_pre_staged_warmed_ack() {
freeze_quorum_ref: None,
created_at_ms: 0,
phase_entered_at_ms: 0,
+ held_by: None,
config_epoch: 0,
new_owner_address: None,
};
@@ -2245,6 +2252,7 @@ async fn draining_old_owner_blocks_phase_advance() {
freeze_quorum_ref: None,
created_at_ms: 0,
phase_entered_at_ms: 0,
+ held_by: None,
config_epoch: 0,
new_owner_address: None,
};
@@ -2355,6 +2363,7 @@ async fn draining_old_owner_does_not_trigger_cleanup() {
freeze_quorum_ref: None,
created_at_ms: 0,
phase_entered_at_ms: 0,
+ held_by: None,
config_epoch: 0,
new_owner_address: None,
};
@@ -2470,6 +2479,7 @@ async fn a_handoff_that_can_never_reach_quorum_is_cancelled() {
freeze_quorum_ref: None,
created_at_ms: 0,
phase_entered_at_ms: 0,
+ held_by: None,
config_epoch: 0,
})
.await
@@ -2560,6 +2570,7 @@ async fn freezing_blocks_until_routers_ack_before_draining() {
freeze_quorum_ref: None,
created_at_ms: 0,
phase_entered_at_ms: 0,
+ held_by: None,
config_epoch: 0,
new_owner_address: None,
};
@@ -2665,6 +2676,7 @@ async fn initial_assignment_skips_draining_phase() {
freeze_quorum_ref: None,
created_at_ms: 0,
phase_entered_at_ms: 0,
+ held_by: None,
config_epoch: 0,
new_owner_address: None,
};
@@ -2732,6 +2744,7 @@ async fn dead_old_owner_in_draining_recovers() {
freeze_quorum_ref: None,
created_at_ms: 0,
phase_entered_at_ms: 0,
+ held_by: None,
config_epoch: 0,
new_owner_address: None,
};
@@ -2839,6 +2852,7 @@ async fn reconcile_advances_draining_with_pre_staged_drained_ack() {
freeze_quorum_ref: None,
created_at_ms: 0,
phase_entered_at_ms: 0,
+ held_by: None,
config_epoch: 0,
new_owner_address: None,
};
@@ -2904,6 +2918,7 @@ async fn late_joining_router_during_draining_begins_stash_no_ack() {
freeze_quorum_ref: None,
created_at_ms: 0,
phase_entered_at_ms: 0,
+ held_by: None,
config_epoch: 0,
new_owner_address: None,
};
@@ -2996,6 +3011,7 @@ async fn a_dead_new_owner_handoff_in_draining_is_reaffirmed() {
freeze_quorum_ref: None,
created_at_ms: 0,
phase_entered_at_ms: 0,
+ held_by: None,
config_epoch: 0,
new_owner_address: None,
};
@@ -3250,6 +3266,7 @@ async fn conflicting_plan_cannot_replace_an_in_flight_handoff() {
freeze_quorum_ref: None,
created_at_ms: 0,
phase_entered_at_ms: 0,
+ held_by: None,
config_epoch: 0,
new_owner_address: None,
};
@@ -3271,6 +3288,7 @@ async fn conflicting_plan_cannot_replace_an_in_flight_handoff() {
freeze_quorum_ref: None,
created_at_ms: 0,
phase_entered_at_ms: 0,
+ held_by: None,
config_epoch: 0,
new_owner_address: None,
};
@@ -3326,6 +3344,7 @@ async fn stale_plan_is_rejected_when_the_assignment_moved() {
freeze_quorum_ref: None,
created_at_ms: 0,
phase_entered_at_ms: 0,
+ held_by: None,
config_epoch: 0,
new_owner_address: None,
};
@@ -3410,6 +3429,7 @@ async fn fresh_plan_is_rejected_when_an_assignment_appeared() {
freeze_quorum_ref: None,
created_at_ms: 0,
phase_entered_at_ms: 0,
+ held_by: None,
config_epoch: 0,
new_owner_address: None,
};
diff --git a/rust/personhog-coordination/tests/migration.rs b/rust/personhog-coordination/tests/migration.rs
new file mode 100644
index 000000000000..1754ba2e35b2
--- /dev/null
+++ b/rust/personhog-coordination/tests/migration.rs
@@ -0,0 +1,1233 @@
+mod common;
+
+use std::sync::Arc;
+use std::time::Duration;
+
+use personhog_coordination::authority::AuthorityClock;
+use personhog_coordination::pod::{PodConfig, PodHandle};
+use personhog_coordination::store::PersonhogStore;
+use personhog_coordination::strategy::StickyBalancedStrategy;
+use personhog_coordination::types::{
+ AssignmentPrecondition, AssignmentStatus, ChangelogConfig, ChangelogTarget, Generation,
+ HandoffPhase, HandoffState, MigrationPhase, MigrationRequest, MigrationState,
+ PartitionAssignment, SealAck,
+};
+use tokio_util::sync::CancellationToken;
+
+use common::{
+ coordinator_config, start_coordinator_with_config, start_coordinator_with_deadline, start_pod,
+ start_pod_with_hanging_drain, start_router, test_store, wait_for_condition, HandoffEvent,
+ MockHandoffHandler, PodHandles, POLL_INTERVAL, WAIT_TIMEOUT,
+};
+
+const PARTITIONS: u32 = 4;
+const TOPIC: &str = "personhog_updates_test";
+
+fn now_ms() -> i64 {
+ std::time::SystemTime::now()
+ .duration_since(std::time::UNIX_EPOCH)
+ .expect("clock")
+ .as_millis() as i64
+}
+
+fn handoff(partition: u32, pod: &str, phase: HandoffPhase, config_epoch: u64) -> HandoffState {
+ let now_ms = now_ms();
+ HandoffState {
+ partition,
+ old_owner: Some(pod.to_string()),
+ new_owner: pod.to_string(),
+ new_owner_address: None,
+ phase,
+ started_at: now_ms / 1000,
+ handoff_id: format!("h-{partition}"),
+ freeze_quorum: Some(Vec::new()),
+ freeze_quorum_ref: None,
+ created_at_ms: now_ms,
+ phase_entered_at_ms: now_ms,
+ held_by: None,
+ config_epoch,
+ }
+}
+
+fn assignment(partition: u32, owner: &str) -> PartitionAssignment {
+ PartitionAssignment {
+ partition,
+ owner: owner.to_string(),
+ advertise_address: None,
+ status: AssignmentStatus::Active,
+ }
+}
+
+fn start_ticking_pod(
+ store: Arc<PersonhogStore>,
+ name: &str,
+ cancel: CancellationToken,
+) -> PodHandles {
+ let (handler, events) = MockHandoffHandler::new();
+ let pod = PodHandle::new(
+ store,
+ PodConfig {
+ pod_name: name.to_string(),
+ lease_ttl: 10,
+ heartbeat_interval: Duration::from_secs(3),
+ reconcile_interval: Duration::from_millis(200),
+ ..Default::default()
+ },
+ Arc::new(handler),
+ None,
+ Arc::new(AuthorityClock::unclaimed()),
+ );
+ let token = cancel.child_token();
+ let join_handle = tokio::spawn(async move { pod.run(token).await });
+ PodHandles {
+ events,
+ join_handle: Some(join_handle),
+ }
+}
+
+async fn events_of(pod: &PodHandles) -> Vec<HandoffEvent> {
+ pod.events.lock().await.clone()
+}
+
+#[tokio::test]
+async fn a_held_handoff_stops_in_draining_and_outlives_the_deadline() {
+ let store = test_store("migration-held").await;
+ let cancel = CancellationToken::new();
+ store
+ .seed_changelog_config(TOPIC, PARTITIONS, "personhog-writer")
+ .await
+ .unwrap();
+
+ // Set up before the coordinator looks: a balanced world plans nothing.
+ store
+ .put_assignments(&[
+ assignment(0, "writer-0"),
+ assignment(1, "writer-ghost"),
+ assignment(2, "writer-0"),
+ assignment(3, "writer-0"),
+ ])
+ .await
+ .unwrap();
+ for (partition, holder) in [(0, "writer-0"), (1, "writer-ghost")] {
+ let mut held = handoff(partition, holder, HandoffPhase::Freezing, 1);
+ held.held_by = Some("m-1".to_string());
+ store.put_handoff(&held).await.unwrap();
+ }
+
+ let deadline = Duration::from_millis(500);
+ let _coord = start_coordinator_with_deadline(
+ Arc::clone(&store),
+ "coordinator-0",
+ 10,
+ deadline,
+ Arc::new(StickyBalancedStrategy),
+ cancel.clone(),
+ );
+ let pod = start_pod(Arc::clone(&store), "writer-0", cancel.clone());
+
+ wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
+ let pod_events = Arc::clone(&pod.events);
+ async move { pod_events.lock().await.contains(&HandoffEvent::Drained(0)) }
+ })
+ .await;
+
+ tokio::time::sleep(deadline * 4).await;
+ let handoffs = store.list_handoffs().await.unwrap();
+ assert_eq!(handoffs.len(), 2, "{handoffs:?}");
+ for h in &handoffs {
+ assert_eq!(h.phase, HandoffPhase::Draining, "partition {}", h.partition);
+ assert_eq!(h.held_by.as_deref(), Some("m-1"));
+ assert_eq!(h.handoff_id, format!("h-{}", h.partition));
+ }
+ let events = events_of(&pod).await;
+ let drained_at = events
+ .iter()
+ .position(|e| *e == HandoffEvent::Drained(0))
+ .unwrap();
+ assert!(
+ !events[drained_at..].contains(&HandoffEvent::Warmed(0)),
+ "a held partition re-warmed: {events:?}"
+ );
+ cancel.cancel();
+}
+
+#[tokio::test]
+async fn an_acquisition_parks_until_the_config_reaches_the_handoffs_epoch() {
+ let store = test_store("migration-park").await;
+ let cancel = CancellationToken::new();
+ store
+ .seed_changelog_config(TOPIC, PARTITIONS, "personhog-writer")
+ .await
+ .unwrap();
+ let pod = start_ticking_pod(Arc::clone(&store), "writer-0", cancel.clone());
+
+ let mut fresh = handoff(0, "writer-0", HandoffPhase::Warming, 2);
+ fresh.old_owner = None;
+ store.put_handoff(&fresh).await.unwrap();
+
+ tokio::time::sleep(Duration::from_secs(1)).await;
+ assert!(
+ !events_of(&pod).await.contains(&HandoffEvent::Warmed(0)),
+ "warmed under a config older than the handoff's epoch"
+ );
+ assert!(store.list_warmed_acks(0).await.unwrap().is_empty());
+
+ store
+ .put_changelog_config(&ChangelogConfig {
+ topic: TOPIC.to_string(),
+ partitions: PARTITIONS,
+ epoch: 2,
+ writer_group: String::new(),
+ draining: Vec::new(),
+ })
+ .await
+ .unwrap();
+ wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
+ let store = Arc::clone(&store);
+ async move { !store.list_warmed_acks(0).await.unwrap().is_empty() }
+ })
+ .await;
+ assert!(events_of(&pod).await.contains(&HandoffEvent::Warmed(0)));
+ cancel.cancel();
+}
+
+// ── The saga ────────────────────────────────────────────────────
+
+const TOPIC_V2: &str = "personhog_updates_test_v2";
+
+fn migration(id: &str, to_partitions: u32) -> MigrationState {
+ let now_ms = now_ms();
+ MigrationState {
+ id: id.to_string(),
+ from: ChangelogTarget {
+ topic: TOPIC.to_string(),
+ partitions: PARTITIONS,
+ },
+ to: ChangelogTarget {
+ topic: TOPIC_V2.to_string(),
+ partitions: to_partitions,
+ },
+ phase: MigrationPhase::Freezing,
+ created_at_ms: now_ms,
+ phase_entered_at_ms: now_ms,
+ blocking: vec![],
+ cancel_reason: None,
+ }
+}
+
+async fn wait_for_phase(store: &Arc<PersonhogStore>, phase: MigrationPhase) {
+ wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
+ let store = Arc::clone(store);
+ async move {
+ store
+ .get_migration()
+ .await
+ .unwrap()
+ .is_some_and(|m| m.phase == phase)
+ }
+ })
+ .await;
+}
+
+/// A request that never starts leaves no saga record; wait for its history entry.
+async fn wait_for_rejection(store: &Arc<PersonhogStore>, id: &str) -> MigrationState {
+ wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
+ let store = Arc::clone(store);
+ let id = id.to_string();
+ async move {
+ store
+ .list_migration_history()
+ .await
+ .unwrap()
+ .iter()
+ .any(|m| m.id == id)
+ }
+ })
+ .await;
+ store
+ .list_migration_history()
+ .await
+ .unwrap()
+ .into_iter()
+ .find(|m| m.id == id)
+ .expect("rejected record")
+}
+
+async fn wait_for_archive(store: &Arc<PersonhogStore>, id: &str) -> MigrationState {
+ wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
+ let store = Arc::clone(store);
+ async move { store.get_migration().await.unwrap().is_none() }
+ })
+ .await;
+ store
+ .list_migration_history()
+ .await
+ .unwrap()
+ .into_iter()
+ .find(|m| m.id == id)
+ .expect("archived record")
+}
+
+async fn wait_for_steady_state(store: &Arc<PersonhogStore>, partitions: u32) {
+ wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
+ let store = Arc::clone(store);
+ async move {
+ store.list_handoffs().await.unwrap().is_empty()
+ && store.list_assignments().await.unwrap().len() == partitions as usize
+ }
+ })
+ .await;
+}
+
+async fn ack_seal(store: &PersonhogStore, id: &str, pod: &str, partitions: std::ops::Range<u32>) {
+ store
+ .put_seal_ack(&SealAck {
+ migration_id: id.to_string(),
+ pod_name: pod.to_string(),
+ sealed: partitions.collect(),
+ })
+ .await
+ .unwrap();
+}
+
+/// Each pod acks every partition, as the leader duty does for its own and the orphaned ones.
+async fn ack_seal_all(store: &PersonhogStore, id: &str, pods: &[&str]) {
+ for pod in pods {
+ ack_seal(store, id, pod, 0..PARTITIONS).await;
+ }
+}
+
+async fn current_phase(store: &PersonhogStore) -> (MigrationPhase, Vec<String>) {
+ let m = store
+ .get_migration()
+ .await
+ .unwrap()
+ .expect("migration in flight");
+ (m.phase, m.blocking)
+}
+
+fn restarted(events: &[HandoffEvent], partition: u32) -> bool {
+ let released = events
+ .iter()
+ .position(|e| *e == HandoffEvent::Released(partition));
+ released.is_some_and(|at| events[at..].contains(&HandoffEvent::Warmed(partition)))
+}
+
+struct Fleet {
+ store: Arc<PersonhogStore>,
+ cancel: CancellationToken,
+ pods: Vec<PodHandles>,
+}
+
+async fn fleet(test_name: &str) -> Fleet {
+ let store = test_store(test_name).await;
+ let cancel = CancellationToken::new();
+ store
+ .seed_changelog_config(TOPIC, PARTITIONS, "writer")
+ .await
+ .unwrap();
+ let _router = start_router(Arc::clone(&store), "router-0", cancel.clone());
+ let pods = vec![
+ start_ticking_pod(Arc::clone(&store), "writer-0", cancel.clone()),
+ start_ticking_pod(Arc::clone(&store), "writer-1", cancel.clone()),
+ ];
+ Fleet {
+ store,
+ cancel,
+ pods,
+ }
+}
+
+#[tokio::test]
+async fn a_grow_migration_restarts_every_partition_over_the_new_changelog() {
+ let Fleet {
+ store,
+ cancel,
+ pods,
+ } = fleet("migration-grow").await;
+ let first = CancellationToken::new();
+ start_coordinator_with_config(
+ Arc::clone(&store),
+ coordinator_config("coordinator-0", 10, Duration::from_secs(86_400)),
+ Arc::new(StickyBalancedStrategy),
+ first.clone(),
+ );
+ wait_for_steady_state(&store, PARTITIONS).await;
+
+ assert!(store.create_migration(&migration("m-1", 8)).await.unwrap());
+ wait_for_phase(&store, MigrationPhase::Sealing).await;
+ for h in store.list_handoffs().await.unwrap() {
+ assert_eq!(h.held_by.as_deref(), Some("m-1"), "{h:?}");
+ assert_eq!(h.phase, HandoffPhase::Draining, "{h:?}");
+ }
+
+ // A successor coordinator picks the saga up where it stands.
+ first.cancel();
+ start_coordinator_with_config(
+ Arc::clone(&store),
+ coordinator_config("coordinator-1", 10, Duration::from_secs(86_400)),
+ Arc::new(StickyBalancedStrategy),
+ cancel.clone(),
+ );
+ assert_eq!(
+ store.get_changelog_config().await.unwrap().unwrap().epoch,
+ 1,
+ "flipped before the seal"
+ );
+ ack_seal_all(&store, "m-1", &["writer-0", "writer-1"]).await;
+
+ let archived = wait_for_archive(&store, "m-1").await;
+ assert_eq!(archived.phase, MigrationPhase::Complete);
+ let config = store.get_changelog_config().await.unwrap().unwrap();
+ assert_eq!(config.topic, TOPIC_V2);
+ assert_eq!(config.partitions, 8);
+ assert_eq!(config.epoch, 2);
+ assert_eq!(config.writer_group, "writer");
+ assert_eq!(
+ config.draining,
+ vec![Generation {
+ topic: TOPIC.to_string(),
+ partitions: PARTITIONS,
+ epoch: 1,
+ }]
+ );
+ assert!(store.list_seal_acks().await.unwrap().is_empty());
+ wait_for_steady_state(&store, 8).await;
+ let mut events = Vec::new();
+ for pod in &pods {
+ events.extend(events_of(pod).await);
+ }
+ for partition in 0..PARTITIONS {
+ assert!(
+ restarted(&events, partition),
+ "partition {partition} never restarted: {events:?}"
+ );
+ // The restart starts at Warming, so nothing drains after the release.
+ let released = events
+ .iter()
+ .position(|e| *e == HandoffEvent::Released(partition))
+ .unwrap();
+ assert!(
+ !events[released..].contains(&HandoffEvent::Drained(partition)),
+ "partition {partition} drained again for its restart: {events:?}"
+ );
+ }
+ assert!(store.retire_changelog_topic(TOPIC).await.unwrap());
+ assert!(store
+ .get_changelog_config()
+ .await
+ .unwrap()
+ .unwrap()
+ .draining
+ .is_empty());
+ cancel.cancel();
+}
+
+#[tokio::test]
+async fn a_shrink_migration_dissolves_the_partitions_past_the_new_space() {
+ let Fleet {
+ store,
+ cancel,
+ pods,
+ } = fleet("migration-shrink").await;
+ start_coordinator_with_config(
+ Arc::clone(&store),
+ coordinator_config("coordinator-0", 10, Duration::from_secs(86_400)),
+ Arc::new(StickyBalancedStrategy),
+ cancel.clone(),
+ );
+ wait_for_steady_state(&store, PARTITIONS).await;
+
+ assert!(store.create_migration(&migration("m-1", 2)).await.unwrap());
+ wait_for_phase(&store, MigrationPhase::Sealing).await;
+ ack_seal_all(&store, "m-1", &["writer-0", "writer-1"]).await;
+
+ let archived = wait_for_archive(&store, "m-1").await;
+ assert_eq!(archived.phase, MigrationPhase::Complete);
+ wait_for_steady_state(&store, 2).await;
+ let owned: Vec<u32> = store
+ .list_assignments()
+ .await
+ .unwrap()
+ .iter()
+ .map(|a| a.partition)
+ .collect();
+ assert_eq!(owned, vec![0, 1]);
+ let mut events = Vec::new();
+ for pod in &pods {
+ events.extend(events_of(pod).await);
+ }
+ for partition in [2, 3] {
+ assert!(
+ events.contains(&HandoffEvent::Released(partition)),
+ "{events:?}"
+ );
+ assert!(
+ !restarted(&events, partition),
+ "a dissolved partition re-warmed: {events:?}"
+ );
+ }
+ cancel.cancel();
+}
+
+#[tokio::test]
+async fn a_migration_past_its_pre_flip_deadline_cancels_and_resumes_in_place() {
+ let store = test_store("migration-deadline").await;
+ let cancel = CancellationToken::new();
+ store
+ .seed_changelog_config(TOPIC, PARTITIONS, "writer")
+ .await
+ .unwrap();
+ let _router = start_router(Arc::clone(&store), "router-0", cancel.clone());
+ let pod = start_pod_with_hanging_drain(Arc::clone(&store), "writer-0", 0, 10, cancel.clone());
+ let mut config = coordinator_config("coordinator-0", 10, Duration::from_secs(86_400));
+ config.migration_pre_flip_deadline = Duration::from_secs(2);
+ start_coordinator_with_config(
+ Arc::clone(&store),
+ config,
+ Arc::new(StickyBalancedStrategy),
+ cancel.clone(),
+ );
+ wait_for_steady_state(&store, PARTITIONS).await;
+
+ assert!(store.create_migration(&migration("m-1", 8)).await.unwrap());
+ wait_for_phase(&store, MigrationPhase::Draining).await;
+ let archived = wait_for_archive(&store, "m-1").await;
+ assert_eq!(archived.phase, MigrationPhase::Cancelled);
+ assert_eq!(archived.cancel_reason.as_deref(), Some("pre_flip_deadline"));
+ assert_eq!(
+ store.get_changelog_config().await.unwrap().unwrap().epoch,
+ 1
+ );
+ wait_for_steady_state(&store, PARTITIONS).await;
+ let events = events_of(&pod).await;
+ assert!(
+ !events
+ .iter()
+ .any(|e| matches!(e, HandoffEvent::Released(_))),
+ "a cancelled migration released a partition: {events:?}"
+ );
+ assert!(store.list_seal_acks().await.unwrap().is_empty());
+ cancel.cancel();
+}
+
+#[tokio::test]
+async fn a_holder_dying_during_sealing_is_sealed_by_its_peers_and_replaced() {
+ let Fleet {
+ store,
+ cancel,
+ pods,
+ } = fleet("migration-holder-died").await;
+ start_coordinator_with_config(
+ Arc::clone(&store),
+ coordinator_config("coordinator-0", 10, Duration::from_secs(86_400)),
+ Arc::new(StickyBalancedStrategy),
+ cancel.clone(),
+ );
+ wait_for_steady_state(&store, PARTITIONS).await;
+
+ assert!(store.create_migration(&migration("m-1", 8)).await.unwrap());
+ wait_for_phase(&store, MigrationPhase::Sealing).await;
+ let mut victim = pods.into_iter().nth(1).unwrap();
+ victim.join_handle.take().unwrap().abort();
+ store.delete_pod("writer-1").await.unwrap();
+ ack_seal_all(&store, "m-1", &["writer-0"]).await;
+
+ let archived = wait_for_archive(&store, "m-1").await;
+ assert_eq!(archived.phase, MigrationPhase::Complete);
+ assert_eq!(
+ store.get_changelog_config().await.unwrap().unwrap().epoch,
+ 2
+ );
+ wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
+ let store = Arc::clone(&store);
+ async move {
+ let assignments = store.list_assignments().await.unwrap();
+ assignments.len() == 8
+ && assignments.iter().all(|a| a.owner == "writer-0")
+ && store.list_handoffs().await.unwrap().is_empty()
+ }
+ })
+ .await;
+ cancel.cancel();
+}
+
+/// A holder gone before the seal cancels the saga: its hold is deleted, the
+/// live holds resume in place, and the planner reassigns its partition.
+#[tokio::test]
+async fn a_holder_dying_before_the_seal_cancels_the_saga() {
+ let Fleet {
+ store,
+ cancel,
+ pods,
+ } = fleet("migration-holder-died-draining").await;
+ let holders = [
+ (0, "writer-0"),
+ (1, "writer-1"),
+ (2, "writer-0"),
+ (3, "writer-gone"),
+ ];
+ let assignments: Vec<_> = holders
+ .iter()
+ .map(|(partition, holder)| assignment(*partition, holder))
+ .collect();
+ store.put_assignments(&assignments).await.unwrap();
+ for (partition, holder) in holders {
+ let mut held = handoff(partition, holder, HandoffPhase::Draining, 1);
+ held.held_by = Some("m-1".to_string());
+ store.put_handoff(&held).await.unwrap();
+ }
+ let draining = MigrationState {
+ phase: MigrationPhase::Draining,
+ ..migration("m-1", 8)
+ };
+ assert!(store.create_migration(&draining).await.unwrap());
+ start_coordinator_with_config(
+ Arc::clone(&store),
+ coordinator_config("coordinator-0", 10, Duration::from_secs(86_400)),
+ Arc::new(StickyBalancedStrategy),
+ cancel.clone(),
+ );
+
+ let archived = wait_for_archive(&store, "m-1").await;
+ assert_eq!(archived.phase, MigrationPhase::Cancelled);
+ assert_eq!(archived.cancel_reason.as_deref(), Some("holder_died"));
+ wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
+ let store = Arc::clone(&store);
+ async move {
+ let assignments = store.list_assignments().await.unwrap();
+ assignments.len() == PARTITIONS as usize
+ && assignments.iter().all(|a| a.owner != "writer-gone")
+ && store.list_handoffs().await.unwrap().is_empty()
+ }
+ })
+ .await;
+ for pod in &pods {
+ let events = events_of(pod).await;
+ assert!(
+ !events
+ .iter()
+ .any(|e| matches!(e, HandoffEvent::Released(_))),
+ "a cancelled migration released a partition: {events:?}"
+ );
+ }
+ assert_eq!(
+ store.get_changelog_config().await.unwrap().unwrap().epoch,
+ 1
+ );
+ cancel.cancel();
+}
+
+#[tokio::test]
+async fn a_request_that_fails_admission_is_rejected_with_its_reason() {
+ let Fleet { store, cancel, .. } = fleet("migration-admission").await;
+ start_coordinator_with_config(
+ Arc::clone(&store),
+ coordinator_config("coordinator-0", 10, Duration::from_secs(86_400)),
+ Arc::new(StickyBalancedStrategy),
+ cancel.clone(),
+ );
+ wait_for_steady_state(&store, PARTITIONS).await;
+
+ let mut same_topic = request("m-1", 8);
+ same_topic.to.topic = TOPIC.to_string();
+ assert!(store.create_migration_request(&same_topic).await.unwrap());
+ let archived = wait_for_rejection(&store, "m-1").await;
+ assert_eq!(archived.phase, MigrationPhase::Cancelled);
+ assert_eq!(archived.blocking, vec!["same_topic".to_string()]);
+ assert!(store.list_handoffs().await.unwrap().is_empty());
+ assert_eq!(
+ store.get_changelog_config().await.unwrap().unwrap().epoch,
+ 1
+ );
+
+ store
+ .put_changelog_config(&ChangelogConfig {
+ topic: TOPIC.to_string(),
+ partitions: PARTITIONS,
+ epoch: 1,
+ writer_group: "writer".to_string(),
+ draining: vec![Generation {
+ topic: "personhog_updates_test_v0".to_string(),
+ partitions: PARTITIONS,
+ epoch: 0,
+ }],
+ })
+ .await
+ .unwrap();
+ assert!(store
+ .create_migration_request(&request("m-2", 8))
+ .await
+ .unwrap());
+ let archived = wait_for_rejection(&store, "m-2").await;
+ assert_eq!(
+ archived.blocking,
+ vec!["generation_still_draining".to_string()]
+ );
+ cancel.cancel();
+}
+
+#[tokio::test]
+async fn the_flip_waits_for_every_partition_to_be_sealed_and_cannot_be_cancelled() {
+ let Fleet { store, cancel, .. } = fleet("migration-gates").await;
+ start_coordinator_with_config(
+ Arc::clone(&store),
+ coordinator_config("coordinator-0", 10, Duration::from_secs(86_400)),
+ Arc::new(StickyBalancedStrategy),
+ cancel.clone(),
+ );
+ wait_for_steady_state(&store, PARTITIONS).await;
+ assert!(store.create_migration(&migration("m-1", 8)).await.unwrap());
+ wait_for_phase(&store, MigrationPhase::Sealing).await;
+
+ tokio::time::sleep(Duration::from_millis(500)).await;
+ let (phase, blocking) = current_phase(&store).await;
+ assert_eq!(phase, MigrationPhase::Sealing);
+ assert_eq!(blocking.len(), PARTITIONS as usize, "{blocking:?}");
+ assert_eq!(
+ store.get_changelog_config().await.unwrap().unwrap().epoch,
+ 1
+ );
+ let (_, rev) = store
+ .get_migration_with_mod_revision()
+ .await
+ .unwrap()
+ .unwrap();
+ assert!(!store
+ .cancel_migration("m-1", rev, "operator")
+ .await
+ .unwrap());
+ assert_eq!(current_phase(&store).await.0, MigrationPhase::Sealing);
+
+ // The gate is the union across leaders: half from one pod leaves the rest blocking.
+ ack_seal(&store, "m-1", "writer-0", 0..2).await;
+ tokio::time::sleep(Duration::from_millis(500)).await;
+ let (phase, blocking) = current_phase(&store).await;
+ assert_eq!(phase, MigrationPhase::Sealing);
+ assert_eq!(blocking.len(), 2, "{blocking:?}");
+ assert_eq!(
+ store.get_changelog_config().await.unwrap().unwrap().epoch,
+ 1
+ );
+ ack_seal(&store, "m-1", "writer-1", 2..PARTITIONS).await;
+ let archived = wait_for_archive(&store, "m-1").await;
+ assert_eq!(archived.phase, MigrationPhase::Complete);
+ cancel.cancel();
+}
+
+/// A plan lands only on the basis it was read on, across a migration's start and a full migration.
+#[tokio::test]
+async fn a_plan_lands_only_on_the_basis_it_was_read_on() {
+ let store = test_store("migration-plan-basis").await;
+ assert!(store
+ .seed_changelog_config(TOPIC, PARTITIONS, "personhog-writer")
+ .await
+ .unwrap());
+ let absent = |partition| vec![AssignmentPrecondition::Absent { partition }];
+ let fresh = |partition| HandoffState {
+ old_owner: None,
+ ..handoff(partition, "leader-a", HandoffPhase::Freezing, 1)
+ };
+ let apply = |basis, partition| {
+ let store = Arc::clone(&store);
+ async move {
+ store
+ .apply_plan_on(
+ &basis,
+ &[fresh(partition)],
+ &[],
+ &absent(partition),
+ None,
+ 128,
+ )
+ .await
+ .unwrap()
+ }
+ };
+
+ // A saga starts after the read.
+ let before = store.read_plan_basis().await.unwrap();
+ assert!(store.create_migration(&migration("m-1", 8)).await.unwrap());
+ let stale = apply(before.clone(), 3).await;
+ assert_eq!(stale.stood_down, vec![3], "{stale:?}");
+
+ // The driver's own writes to a live saga do not move the planner's basis.
+ let during = store.read_plan_basis().await.unwrap();
+ let (_, rev) = store
+ .get_migration_with_mod_revision()
+ .await
+ .unwrap()
+ .unwrap();
+ assert!(store
+ .set_migration_blocking(rev, vec!["partition 0: restarting".to_string()])
+ .await
+ .unwrap());
+ assert_eq!(apply(during, 6).await.applied, vec![6]);
+ store.delete_handoff(6).await.unwrap();
+
+ // The saga flips and is archived: the migration key is absent again.
+ let flipped = ChangelogConfig {
+ topic: TOPIC_V2.to_string(),
+ partitions: 8,
+ epoch: 2,
+ writer_group: String::new(),
+ draining: vec![],
+ };
+ store.put_changelog_config(&flipped).await.unwrap();
+ let (m, rev) = store
+ .get_migration_with_mod_revision()
+ .await
+ .unwrap()
+ .unwrap();
+ assert!(store
+ .archive_migration(&m, rev, MigrationPhase::Complete)
+ .await
+ .unwrap());
+ let aba = apply(before, 4).await;
+ assert_eq!(aba.stood_down, vec![4], "{aba:?}");
+ assert!(store.list_handoffs().await.unwrap().is_empty());
+
+ let current = store.read_plan_basis().await.unwrap();
+ assert_eq!(apply(current, 5).await.applied, vec![5]);
+}
+
+/// A migration pass's writes land only on the record it read: a stalled pass
+/// that wakes after a successor moved or archived the saga lands nothing.
+#[tokio::test]
+async fn a_migration_pass_lands_nothing_once_the_record_moved() {
+ let store = test_store("migration-pass-authority").await;
+ assert!(store.create_migration(&migration("m-1", 8)).await.unwrap());
+ let absent = |partition| vec![AssignmentPrecondition::Absent { partition }];
+ let hold = |partition| HandoffState {
+ held_by: Some("m-1".to_string()),
+ ..handoff(partition, "leader-a", HandoffPhase::Freezing, 1)
+ };
+ let apply = |mod_revision, partition| {
+ let store = Arc::clone(&store);
+ async move {
+ store
+ .apply_migration_plan(
+ mod_revision,
+ &[hold(partition)],
+ &[],
+ &absent(partition),
+ None,
+ 128,
+ )
+ .await
+ .unwrap()
+ }
+ };
+
+ // A successor's write moves the record.
+ let (_, read) = store
+ .get_migration_with_mod_revision()
+ .await
+ .unwrap()
+ .unwrap();
+ assert!(store
+ .set_migration_blocking(read, vec!["partition 0: drain".to_string()])
+ .await
+ .unwrap());
+ assert_eq!(apply(read, 3).await.stood_down, vec![3]);
+
+ // A plan split across chunks stands down whole.
+ let holds: Vec<HandoffState> = (10..14).map(hold).collect();
+ let preconditions: Vec<AssignmentPrecondition> = (10..14).flat_map(absent).collect();
+ let split = store
+ .apply_migration_plan(read, &holds, &[], &preconditions, None, 3)
+ .await
+ .unwrap();
+ assert_eq!(split.stood_down, vec![10, 11, 12, 13], "{split:?}");
+ assert!(split.applied.is_empty() && split.conflicted.is_empty());
+
+ // A successor archives the saga; the stale hold must not outlive it.
+ let (m, current) = store
+ .get_migration_with_mod_revision()
+ .await
+ .unwrap()
+ .unwrap();
+ assert_eq!(apply(current, 4).await.applied, vec![4]);
+ store.delete_handoff(4).await.unwrap();
+ let (_, current) = store
+ .get_migration_with_mod_revision()
+ .await
+ .unwrap()
+ .unwrap();
+ assert!(store
+ .archive_migration(&m, current, MigrationPhase::Cancelled)
+ .await
+ .unwrap());
+ assert_eq!(apply(current, 5).await.stood_down, vec![5]);
+ assert!(store.list_handoffs().await.unwrap().is_empty());
+}
+
+fn request(id: &str, to_partitions: u32) -> MigrationRequest {
+ MigrationRequest {
+ id: id.to_string(),
+ from: ChangelogTarget {
+ topic: TOPIC.to_string(),
+ partitions: PARTITIONS,
+ },
+ to: ChangelogTarget {
+ topic: TOPIC_V2.to_string(),
+ partitions: to_partitions,
+ },
+ requested_at_ms: now_ms(),
+ seen_at_ms: None,
+ waiting: vec![],
+ }
+}
+
+/// The saga starts only in a transaction that proves no handoff was created
+/// or changed since the coordinator saw the fleet quiet.
+#[tokio::test]
+async fn a_migration_starts_only_from_a_fleet_seen_quiet() {
+ let store = test_store("migration-quiet-start").await;
+ assert!(store
+ .create_migration_request(&request("m-1", 8))
+ .await
+ .unwrap());
+ let (req, req_rev) = store
+ .get_migration_request_with_mod_revision()
+ .await
+ .unwrap()
+ .unwrap();
+
+ // A handoff starts between the quiet read and the start.
+ let (handoffs, quiet_at) = store.list_handoffs_with_revision().await.unwrap();
+ assert!(handoffs.is_empty());
+ store
+ .apply_plan(
+ &[],
+ &[handoff(3, "leader-a", HandoffPhase::Freezing, 1)],
+ &[],
+ &[],
+ None,
+ 128,
+ )
+ .await
+ .unwrap();
+ assert!(!store
+ .start_migration(req_rev, quiet_at, &req.start(now_ms()))
+ .await
+ .unwrap());
+ assert!(store.get_migration().await.unwrap().is_none());
+
+ // Once it is gone the fleet is quiet again, and the start lands.
+ store.delete_handoff(3).await.unwrap();
+ let (_, quiet_at) = store.list_handoffs_with_revision().await.unwrap();
+ assert!(store
+ .start_migration(req_rev, quiet_at, &req.start(now_ms()))
+ .await
+ .unwrap());
+ assert_eq!(store.get_migration().await.unwrap().unwrap().id, "m-1");
+ assert!(store
+ .get_migration_request_with_mod_revision()
+ .await
+ .unwrap()
+ .is_none());
+
+ // No second request while the saga runs.
+ assert!(!store
+ .create_migration_request(&request("m-2", 8))
+ .await
+ .unwrap());
+}
+
+/// Handoffs stuck on a pod that then dies: the request waits holding nothing,
+/// the planner clears the dead pod's handoffs, and only then does the saga start.
+#[tokio::test]
+async fn a_request_waits_holding_nothing_until_the_fleet_is_quiet() {
+ let Fleet { store, cancel, .. } = fleet("migration-request-waits").await;
+ start_coordinator_with_config(
+ Arc::clone(&store),
+ coordinator_config("coordinator-0", 10, Duration::from_secs(86_400)),
+ Arc::new(StickyBalancedStrategy),
+ cancel.clone(),
+ );
+ wait_for_steady_state(&store, PARTITIONS).await;
+
+ // A registered pod that never warms: the planner moves partitions to it and they stick.
+ let lease = store.grant_lease(60).await.unwrap();
+ store
+ .register_pod(
+ &personhog_coordination::types::RegisteredPod {
+ pod_name: "stand-in".to_string(),
+ generation: String::new(),
+ status: personhog_coordination::types::PodStatus::Ready,
+ registered_at: 0,
+ last_heartbeat: 0,
+ controller: None,
+ advertise_address: None,
+ },
+ lease,
+ )
+ .await
+ .unwrap();
+ wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
+ let store = Arc::clone(&store);
+ async move {
+ store
+ .list_handoffs()
+ .await
+ .unwrap()
+ .iter()
+ .any(|h| h.new_owner == "stand-in")
+ }
+ })
+ .await;
+ assert!(store
+ .create_migration_request(&request("m-1", 8))
+ .await
+ .unwrap());
+
+ // It reports what it waits on, and holds nothing meanwhile.
+ wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
+ let store = Arc::clone(&store);
+ async move {
+ store
+ .get_migration_request_with_mod_revision()
+ .await
+ .unwrap()
+ .is_some_and(|(r, _)| !r.waiting.is_empty())
+ }
+ })
+ .await;
+ assert!(store.get_migration().await.unwrap().is_none());
+ assert!(store
+ .list_handoffs()
+ .await
+ .unwrap()
+ .iter()
+ .all(|h| !h.is_held()));
+
+ // The stand-in dies; the planner clears its handoffs, then the saga starts quiet.
+ store.revoke_lease(lease).await.unwrap();
+ let started = tokio::time::timeout(Duration::from_secs(30), async {
+ loop {
+ if store.get_migration().await.unwrap().is_some() {
+ assert!(
+ store
+ .list_handoffs()
+ .await
+ .unwrap()
+ .iter()
+ .all(|h| h.is_held()),
+ "the saga started with an ordinary handoff in flight"
+ );
+ return;
+ }
+ tokio::time::sleep(Duration::from_millis(20)).await;
+ }
+ })
+ .await;
+ assert!(started.is_ok(), "the saga never started");
+ cancel.cancel();
+}
+
+/// A request left waiting past the expiry is rejected, never started late.
+#[tokio::test]
+async fn a_stale_request_is_rejected_with_its_reason() {
+ let Fleet { store, cancel, .. } = fleet("migration-request-expires").await;
+ start_coordinator_with_config(
+ Arc::clone(&store),
+ coordinator_config("coordinator-0", 10, Duration::from_secs(86_400)),
+ Arc::new(StickyBalancedStrategy),
+ cancel.clone(),
+ );
+ wait_for_steady_state(&store, PARTITIONS).await;
+ // The coordinator first saw it ten minutes ago.
+ let stale = MigrationRequest {
+ seen_at_ms: Some(now_ms() - 600_000),
+ ..request("m-1", 8)
+ };
+ assert!(store.create_migration_request(&stale).await.unwrap());
+ let archived = wait_for_rejection(&store, "m-1").await;
+ assert_eq!(archived.blocking[0], "request_expired");
+ assert!(store.get_migration().await.unwrap().is_none());
+ cancel.cancel();
+}
+
+/// Expiry runs on the coordinator's clock: a control plane whose clock is far
+/// behind still gets its request started, not rejected on arrival.
+#[tokio::test]
+async fn a_request_from_a_skewed_clock_is_not_expired_on_arrival() {
+ let Fleet { store, cancel, .. } = fleet("migration-request-skew").await;
+ start_coordinator_with_config(
+ Arc::clone(&store),
+ coordinator_config("coordinator-0", 10, Duration::from_secs(86_400)),
+ Arc::new(StickyBalancedStrategy),
+ cancel.clone(),
+ );
+ wait_for_steady_state(&store, PARTITIONS).await;
+ let skewed = MigrationRequest {
+ requested_at_ms: now_ms() - 3_600_000,
+ ..request("m-1", 8)
+ };
+ assert!(store.create_migration_request(&skewed).await.unwrap());
+ wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
+ let store = Arc::clone(&store);
+ async move { store.get_migration().await.unwrap().is_some() }
+ })
+ .await;
+ cancel.cancel();
+}
+
+/// Only a completed migration retires its source topic: a request back to it
+/// is refused, while a source named only by a rejected request stays usable.
+#[tokio::test]
+async fn only_a_completed_migration_retires_its_source_topic() {
+ let Fleet { store, cancel, .. } = fleet("migration-retired-topic").await;
+ let target = |topic: &str| ChangelogTarget {
+ topic: topic.to_string(),
+ partitions: 8,
+ };
+ // History, written before any coordinator runs: one completed saga away
+ // from "retired", one rejected request away from TOPIC_V2.
+ let mut done = migration("m-done", 8);
+ done.from = target("personhog_updates_retired");
+ assert!(store.create_migration(&done).await.unwrap());
+ let (m, rev) = store
+ .get_migration_with_mod_revision()
+ .await
+ .unwrap()
+ .unwrap();
+ assert!(store
+ .archive_migration(&m, rev, MigrationPhase::Complete)
+ .await
+ .unwrap());
+ let rejected = MigrationRequest {
+ from: target(TOPIC_V2),
+ ..request("m-rejected", 8)
+ };
+ assert!(store.create_migration_request(&rejected).await.unwrap());
+ let (r, rev) = store
+ .get_migration_request_with_mod_revision()
+ .await
+ .unwrap()
+ .unwrap();
+ assert!(store
+ .reject_migration_request(&r, rev, "test")
+ .await
+ .unwrap());
+
+ start_coordinator_with_config(
+ Arc::clone(&store),
+ coordinator_config("coordinator-0", 10, Duration::from_secs(86_400)),
+ Arc::new(StickyBalancedStrategy),
+ cancel.clone(),
+ );
+ wait_for_steady_state(&store, PARTITIONS).await;
+
+ let back = MigrationRequest {
+ to: target("personhog_updates_retired"),
+ ..request("m-back", 8)
+ };
+ assert!(store.create_migration_request(&back).await.unwrap());
+ let refused = wait_for_rejection(&store, "m-back").await;
+ assert_eq!(refused.blocking, vec!["topic_already_retired".to_string()]);
+
+ assert!(store
+ .create_migration_request(&request("m-ok", 8))
+ .await
+ .unwrap());
+ wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
+ let store = Arc::clone(&store);
+ async move {
+ store
+ .get_migration()
+ .await
+ .unwrap()
+ .is_some_and(|m| m.id == "m-ok")
+ }
+ })
+ .await;
+ cancel.cancel();
+}
+
+/// An owner that is shutting down keeps a request waiting: the fleet is quiet
+/// only when every old partition's owner is registered and Ready.
+#[tokio::test]
+async fn a_request_waits_while_an_owner_is_draining() {
+ use personhog_coordination::migration::consider_request;
+ use personhog_coordination::strategy::Member;
+ use personhog_coordination::types::{PodStatus, RegisteredPod};
+
+ let store = test_store("migration-request-draining-owner").await;
+ assert!(store
+ .seed_changelog_config(TOPIC, PARTITIONS, "personhog-writer")
+ .await
+ .unwrap());
+ let pod = |name: &str, status: PodStatus| RegisteredPod {
+ pod_name: name.to_string(),
+ generation: String::new(),
+ status,
+ registered_at: 0,
+ last_heartbeat: 0,
+ controller: None,
+ advertise_address: None,
+ };
+ let lease = store.grant_lease(60).await.unwrap();
+ store
+ .register_pod(&pod("p0", PodStatus::Ready), lease)
+ .await
+ .unwrap();
+ store
+ .register_pod(&pod("p1", PodStatus::Draining), lease)
+ .await
+ .unwrap();
+ store
+ .put_assignments(
+ &(0..PARTITIONS)
+ .map(|p| assignment(p, if p % 2 == 0 { "p0" } else { "p1" }))
+ .collect::<Vec<_>>(),
+ )
+ .await
+ .unwrap();
+ assert!(store
+ .create_migration_request(&request("m-1", 8))
+ .await
+ .unwrap());
+ let members = Member::active_all(&["p0", "p1"]);
+ let pass = || {
+ let store = Arc::clone(&store);
+ let members = members.clone();
+ async move {
+ let pods = store.list_pods().await.unwrap();
+ let (req, rev) = store
+ .get_migration_request_with_mod_revision()
+ .await
+ .unwrap()
+ .unwrap();
+ consider_request(&store, &members, &pods, req, rev)
+ .await
+ .unwrap();
+ }
+ };
+
+ // First the sighting, then a pass that must wait on the draining owner.
+ pass().await;
+ pass().await;
+ assert!(store.get_migration().await.unwrap().is_none());
+ let (waiting, _) = store
+ .get_migration_request_with_mod_revision()
+ .await
+ .unwrap()
+ .unwrap();
+ assert!(
+ waiting
+ .waiting
+ .iter()
+ .any(|w| w.contains("p1 is not ready")),
+ "{:?}",
+ waiting.waiting
+ );
+
+ store
+ .register_pod(&pod("p1", PodStatus::Ready), lease)
+ .await
+ .unwrap();
+ pass().await;
+ assert_eq!(store.get_migration().await.unwrap().unwrap().id, "m-1");
+}
diff --git a/rust/personhog-coordination/tests/protocol_hardening.rs b/rust/personhog-coordination/tests/protocol_hardening.rs
index c99eeb2c28bb..657635c17e60 100644
--- a/rust/personhog-coordination/tests/protocol_hardening.rs
+++ b/rust/personhog-coordination/tests/protocol_hardening.rs
@@ -62,6 +62,7 @@ async fn put_handoff(
freeze_quorum_ref: None,
created_at_ms: 0,
phase_entered_at_ms: 0,
+ held_by: None,
config_epoch: 0,
new_owner_address: None,
};
@@ -2063,6 +2064,55 @@ async fn the_reconcile_pass_heals_an_out_of_protocol_deletion() {
cancel.cancel();
}
+/// Routes for partitions a shrink dissolved must not outlive it, or a later
+/// grow forwards writes for the reborn partitions to their old owner.
+#[tokio::test]
+async fn the_reconcile_pass_drops_routes_above_the_live_count() {
+ let store = test_store("reconcile-drops-dissolved").await;
+ assert!(store
+ .seed_changelog_config("personhog_updates", 2, "personhog-writer")
+ .await
+ .unwrap());
+ let assignments: Vec<PartitionAssignment> = (0..4)
+ .map(|partition| PartitionAssignment {
+ partition,
+ owner: "pod-a".to_string(),
+ status: AssignmentStatus::Active,
+ advertise_address: None,
+ })
+ .collect();
+ assert!(store
+ .create_assignments_and_handoffs(&assignments, &[], &[], 128)
+ .await
+ .expect("write assignments"));
+
+ let router = RoutingTable::new(
+ Arc::clone(&store),
+ RoutingTableConfig {
+ router_name: "rdd-router".to_string(),
+ reconcile_interval: Duration::from_millis(200),
+ ..RoutingTableConfig::default()
+ },
+ );
+ let table = router.table_handle();
+ let (handler, _events) = MockCutoverHandler::new();
+ let cancel = CancellationToken::new();
+ let token = cancel.child_token();
+ tokio::spawn(async move { router.run(token, Arc::new(handler)).await });
+
+ wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
+ let table = Arc::clone(&table);
+ async move {
+ let mut routed: Vec<u32> = table.read().await.keys().copied().collect();
+ routed.sort_unstable();
+ routed == vec![0, 1]
+ }
+ })
+ .await;
+
+ cancel.cancel();
+}
+
/// The reconcile pass re-derives freeze acks from the snapshot, so an
/// ack lost out-of-protocol — or a Freezing event lost to a dead watch
/// stream — is repaired on the next pass instead of wedging the quorum
@@ -2617,6 +2667,126 @@ async fn the_reconcile_pass_skips_drains_for_settled_partitions() {
cancel.cancel();
}
+/// A cutover handler whose stash always has something parked, on `pending`
+/// among others.
+struct PendingCutoverHandler {
+ events: Arc<Mutex<Vec<CutoverEvent>>>,
+ pending: Vec<u32>,
+}
+
+#[async_trait]
+impl StashHandler for PendingCutoverHandler {
+ async fn begin_stash(&self, _partition: u32, _new_owner: &str) -> Result<()> {
+ Ok(())
+ }
+
+ async fn drain_stash(
+ &self,
+ partition: u32,
+ target: &str,
+ _cancel: CancellationToken,
+ ) -> Result<()> {
+ self.events.lock().await.push(CutoverEvent::StashDrained {
+ partition,
+ target: target.to_string(),
+ });
+ Ok(())
+ }
+
+ fn stash_pending(&self, _partition: u32) -> bool {
+ true
+ }
+
+ fn pending_partitions(&self) -> Vec<u32> {
+ self.pending.clone()
+ }
+}
+
+/// A stash whose owner is gone still drains, so its parked requests get answered.
+#[tokio::test]
+async fn the_reconcile_pass_drains_a_gone_owners_stash() {
+ let store = test_store("reconcile-gone-owner").await;
+ assert!(store
+ .create_assignments_and_handoffs(
+ &[PartitionAssignment {
+ partition: 0,
+ owner: "pod-gone".to_string(),
+ status: AssignmentStatus::Active,
+ advertise_address: None,
+ }],
+ &[],
+ &[],
+ 128
+ )
+ .await
+ .expect("write assignment"));
+
+ let router = RoutingTable::new(
+ Arc::clone(&store),
+ RoutingTableConfig {
+ router_name: "gone-owner-router".to_string(),
+ reconcile_interval: Duration::from_millis(200),
+ ..RoutingTableConfig::default()
+ },
+ );
+ let events = Arc::new(Mutex::new(Vec::new()));
+ let handler = PendingCutoverHandler {
+ events: Arc::clone(&events),
+ pending: vec![0],
+ };
+ let cancel = CancellationToken::new();
+ let token = cancel.child_token();
+ tokio::spawn(async move { router.run(token, Arc::new(handler)).await });
+
+ wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
+ let events = Arc::clone(&events);
+ async move {
+ events.lock().await.iter().any(|e| {
+ matches!(e, CutoverEvent::StashDrained { partition: 0, target } if target == "pod-gone")
+ })
+ }
+ })
+ .await;
+
+ cancel.cancel();
+}
+
+/// A pending partition nothing names still drains, so its parked requests get answered.
+#[tokio::test]
+async fn the_reconcile_pass_drains_a_pending_partition_nothing_names() {
+ let store = test_store("reconcile-unnamed-pending").await;
+ let router = RoutingTable::new(
+ Arc::clone(&store),
+ RoutingTableConfig {
+ router_name: "unnamed-pending-router".to_string(),
+ reconcile_interval: Duration::from_millis(200),
+ ..RoutingTableConfig::default()
+ },
+ );
+ let events = Arc::new(Mutex::new(Vec::new()));
+ let handler = PendingCutoverHandler {
+ events: Arc::clone(&events),
+ pending: vec![6],
+ };
+ let cancel = CancellationToken::new();
+ let token = cancel.child_token();
+ tokio::spawn(async move { router.run(token, Arc::new(handler)).await });
+
+ wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
+ let events = Arc::clone(&events);
+ async move {
+ events
+ .lock()
+ .await
+ .iter()
+ .any(|e| matches!(e, CutoverEvent::StashDrained { partition: 6, .. }))
+ }
+ })
+ .await;
+
+ cancel.cancel();
+}
+
// ============================================================
// Convergence lanes: partitions converge concurrently,
// single-flight per partition
@@ -2651,6 +2821,7 @@ async fn put_handoff_with_id(
freeze_quorum_ref: None,
created_at_ms: 0,
phase_entered_at_ms: 0,
+ held_by: None,
config_epoch: 0,
new_owner_address: None,
};
@@ -4816,6 +4987,7 @@ async fn a_freeze_quorum_reference_that_is_gone_requires_every_live_router() {
freeze_quorum_ref: Some("never-written".to_string()),
created_at_ms: 0,
phase_entered_at_ms: 0,
+ held_by: None,
config_epoch: 0,
};
@@ -5490,6 +5662,7 @@ async fn a_recorded_membership_advances_past_a_router_outside_it() {
freeze_quorum_ref: Some("narrow-membership".to_string()),
created_at_ms: 0,
phase_entered_at_ms: 0,
+ held_by: None,
config_epoch: 0,
};
store.put_handoff(&handoff).await.expect("write handoff");
@@ -5638,6 +5811,7 @@ async fn a_cached_absent_membership_still_requires_every_live_router() {
freeze_quorum_ref: Some("never-written".to_string()),
created_at_ms: 0,
phase_entered_at_ms: 0,
+ held_by: None,
config_epoch: 0,
};
@@ -5759,6 +5933,7 @@ async fn a_membership_already_read_is_answered_without_reading_again() {
freeze_quorum_ref: Some(id.to_string()),
created_at_ms: 0,
phase_entered_at_ms: 0,
+ held_by: None,
config_epoch: 0,
};
@@ -5832,6 +6007,7 @@ async fn a_plan_past_the_server_txn_budget_still_applies() {
freeze_quorum_ref: Some(quorum_id.to_string()),
created_at_ms: now * 1000,
phase_entered_at_ms: now * 1000,
+ held_by: None,
config_epoch: 0,
})
.collect();
@@ -5888,6 +6064,7 @@ async fn a_conflicted_partition_stands_down_alone() {
freeze_quorum_ref: None,
created_at_ms: 1_000_000,
phase_entered_at_ms: 1_000_000,
+ held_by: None,
config_epoch: 0,
};
@@ -5953,6 +6130,7 @@ async fn a_swept_quorum_delete_loses_to_a_concurrent_re_put() {
freeze_quorum_ref: Some(quorum_id.to_string()),
created_at_ms: 0,
phase_entered_at_ms: 0,
+ held_by: None,
config_epoch: 0,
};
let listed_revision = |store: Arc<personhog_coordination::store::PersonhogStore>| async move {
@@ -6037,6 +6215,7 @@ async fn a_missing_membership_is_resolved_again_once_written() {
freeze_quorum_ref: Some(quorum_id.to_string()),
created_at_ms: 0,
phase_entered_at_ms: 0,
+ held_by: None,
config_epoch: 0,
};
@@ -6088,6 +6267,7 @@ async fn an_over_server_budget_chunk_degrades_to_per_unit_transactions() {
freeze_quorum_ref: None,
created_at_ms: 0,
phase_entered_at_ms: 0,
+ held_by: None,
config_epoch: 0,
})
.collect();
diff --git a/rust/personhog-leader/src/coordination/mod.rs b/rust/personhog-leader/src/coordination/mod.rs
index 4b76808cf369..3a38f9db4eeb 100644
--- a/rust/personhog-leader/src/coordination/mod.rs
+++ b/rust/personhog-leader/src/coordination/mod.rs
@@ -251,6 +251,9 @@ impl HandoffHandler for LeaderHandoffHandler {
async fn capture_changelog(&self, partition: u32, config: &ChangelogConfig) {
preregister_partition_fencing_metrics(partition);
+ // Adopted before the capture lands, so the live config is never older than a
+ // topic this pod marks; the prune's retirement check relies on that.
+ self.changelog.adopt(config);
self.captures.insert(
partition,
Capture {
@@ -601,6 +604,28 @@ mod tests {
assert!(!handler.captures.contains_key(&0));
}
+ /// A capture newer than the watch's view is adopted first, so the prune never
+ /// judges a topic this pod marks as retired.
+ #[tokio::test]
+ async fn a_capture_ahead_of_the_watch_keeps_its_topic_live() {
+ let mut handler = handler();
+ handler.changelog = LiveChangelog::new(ChangelogConfig {
+ writer_group: "personhog-writer".to_string(),
+ ..ChangelogConfig::new("personhog_updates", 4)
+ });
+ let flipped = ChangelogConfig {
+ epoch: 2,
+ writer_group: "personhog-writer".to_string(),
+ ..ChangelogConfig::new("personhog_updates_v2", 8)
+ };
+ handler.capture_changelog(0, &flipped).await;
+ assert!(!crate::settle::retired_topic(
+ &handler.changelog.current(),
+ "personhog-writer",
+ "personhog_updates_v2"
+ ));
+ }
+
/// After a shrink, old-5 and new-1 share keys on this pod, and the
/// release of old-5 may run after new-1 warmed.
#[tokio::test]
diff --git a/rust/personhog-leader/src/fencing.rs b/rust/personhog-leader/src/fencing.rs
index 83a0f394cab7..10df22fcf02b 100644
--- a/rust/personhog-leader/src/fencing.rs
+++ b/rust/personhog-leader/src/fencing.rs
@@ -967,6 +967,65 @@ impl FencedChangelogProducers {
self.acquire_installed(partition, target).await.map(|_| ())
}
+ /// Claim and drop each lane's id; a dial in flight is waited for, not duplicated.
+ pub async fn seal(
+ &self,
+ partition: u32,
+ target: &ChangelogTarget,
+ ) -> Result<&'static str, String> {
+ let topic = target.topic.as_str();
+ let start = Instant::now();
+ while start.elapsed() < self.init_timeout
+ && self
+ .prepared
+ .get(&partition)
+ .is_some_and(|slot| matches!(slot.value(), Prepared::Connecting { .. }))
+ {
+ sleep(Duration::from_millis(10)).await;
+ }
+ let mut parked = self.take_prepared(partition, topic);
+ let path = if parked.is_empty() {
+ "cold"
+ } else {
+ "prepared"
+ };
+ parked.resize_with(self.lanes, || None);
+ let dial = self.dial(partition, target);
+ let claimed = spawn_blocking(move || {
+ let fell_back = AtomicBool::new(false);
+ let results = per_lane(parked, |lane, parked| {
+ claim_lane(&dial, lane, parked, &fell_back)
+ });
+ let mut producers = Vec::with_capacity(results.len());
+ let mut failure = None;
+ for result in results {
+ match result {
+ Ok(producer) => producers.push(producer),
+ Err(e) => {
+ failure.get_or_insert(e);
+ }
+ }
+ }
+ // The ids are claimed once init returns; the destroy that
+ // follows takes hundreds of milliseconds and nobody waits for it.
+ drop_off_worker(producers);
+ match failure {
+ Some(e) => Err(e),
+ None => Ok(()),
+ }
+ })
+ .await
+ .map_err(|join| format!("seal join: {join}"))?;
+ histogram!("personhog_leader_fence_seal_ms", "path" => path)
+ .record(start.elapsed().as_secs_f64() * 1000.0);
+ counter!(
+ "personhog_leader_fence_seal_total",
+ "outcome" => if claimed.is_ok() { "ok" } else { "error" }
+ )
+ .increment(1);
+ claimed.map(|()| path)
+ }
+
/// Mark the installed lanes as about to be fenced by this pod's own
/// re-acquisition, so the fences their writers then see count as a
/// heal rather than a takeover.
diff --git a/rust/personhog-leader/src/lib.rs b/rust/personhog-leader/src/lib.rs
index f2de61a2ae30..8f70981693ee 100644
--- a/rust/personhog-leader/src/lib.rs
+++ b/rust/personhog-leader/src/lib.rs
@@ -6,6 +6,7 @@ pub mod fence;
pub mod fencing;
pub mod inflight;
pub mod kafka;
+pub mod migration;
pub mod person_update;
pub mod pg;
pub mod producer_stats;
diff --git a/rust/personhog-leader/src/main.rs b/rust/personhog-leader/src/main.rs
index 2a36dbfac708..9e16b4a34082 100644
--- a/rust/personhog-leader/src/main.rs
+++ b/rust/personhog-leader/src/main.rs
@@ -40,10 +40,11 @@ use personhog_leader::fencing::{
preregister_fencing_metrics, FencedChangelogProducers, FencedProducerConfig,
};
use personhog_leader::inflight::InflightTracker;
+use personhog_leader::migration::SealDuty;
use personhog_leader::pg::{validate_table_name, LifecycleTables, PgFallback};
use personhog_leader::recovery::{ChangelogRecovery, RecoveryConfig};
use personhog_leader::service::{sweep_idle_locks, PersonHogLeaderService, PropertySizeLimits};
-use personhog_leader::settle::prune_and_settle_tick;
+use personhog_leader::settle::{prune_and_settle_tick, retired_topic};
use personhog_leader::warming::{
changelog_partition_count, WarmClientPools, WarmingConfig, WarmingRetryPolicy,
};
@@ -147,6 +148,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
ComponentOptions::new().is_observability(true),
);
let changelog_handle = manager.register("changelog-config", ComponentOptions::new());
+ let migration_handle = manager.register("migration-duty", ComponentOptions::new());
let readiness = manager.readiness_handler();
let liveness = manager.liveness_handler();
@@ -456,6 +458,22 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
.with_repair_nudge(Arc::clone(&fence_repair_nudge)),
);
+ {
+ let duty = SealDuty {
+ store: Arc::clone(&store),
+ pod_name: config.pod_name.clone(),
+ fenced: Arc::clone(&fenced),
+ };
+ tokio::spawn(async move {
+ let _guard = migration_handle.process_scope();
+ let cancel = CancellationToken::new();
+ tokio::select! {
+ _ = migration_handle.shutdown_signal() => cancel.cancel(),
+ _ = duty.run(cancel.clone()) => {}
+ }
+ });
+ }
+
// One clock for the process: the coordination session claims and
// surrenders it, the data plane reads it per request.
let authority = Arc::new(AuthorityClock::unclaimed());
@@ -680,6 +698,8 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
Arc::clone(&cache),
Arc::clone(&locks),
Arc::clone(&warm_pools),
+ changelog.clone(),
+ config.writer_consumer_group.clone(),
Duration::from_secs(config.warm_committed_offsets_timeout_secs),
Duration::from_secs(config.dirty_index_prune_interval_secs.max(1)),
));
@@ -772,11 +792,14 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
/// export the dirty-count and writer-lag gauges as a side effect. One
/// batched OffsetFetch covers every partition with marks, on a pooled
/// client — each tick reuses the connection instead of rebuilding one.
+#[allow(clippy::too_many_arguments)]
async fn run_dirty_index_prune_loop(
dirty_index: Arc<DirtyIndex>,
cache: Arc<PartitionedCache>,
locks: Arc<DashMap<PersonCacheKey, Arc<tokio::sync::Mutex<()>>>>,
pools: Arc<WarmClientPools>,
+ changelog: LiveChangelog,
+ writer_group: String,
offsets_timeout: Duration,
prune_interval: Duration,
) {
@@ -786,12 +809,15 @@ async fn run_dirty_index_prune_loop(
gauge!("personhog_leader_dirty_index_size").set(dirty_index.len() as f64);
gauge!("personhog_leader_dirty_index_max_entries").set(dirty_index.max_entries() as f64);
gauge!("personhog_leader_cache_weight_bytes").set(cache.usage_bytes() as f64);
+ // Read when judged, not when the tick starts: a capture adopts before its marks
+ // exist, so a stream seen with marks is never judged on an older config.
let lags = prune_and_settle_tick(
&dirty_index,
&cache,
&locks,
&pools.offsets,
offsets_timeout,
+ |topic| retired_topic(&changelog.current(), &writer_group, topic),
)
.await;
// A partition's lag sums the streams feeding it; a stream the writer never
diff --git a/rust/personhog-leader/src/migration.rs b/rust/personhog-leader/src/migration.rs
new file mode 100644
index 000000000000..07f59f6a0354
--- /dev/null
+++ b/rust/personhog-leader/src/migration.rs
@@ -0,0 +1,255 @@
+//! The leader's share of a changelog migration: seal the old generation and ack it.
+
+use std::collections::{hash_map, BTreeSet, HashMap, HashSet};
+use std::sync::Arc;
+use std::time::Duration;
+
+use assignment_coordination::store::parse_watch_value;
+
+use metrics::counter;
+use personhog_coordination::protocol::migration::{seal_overdue, seal_targets};
+use personhog_coordination::store::PersonhogStore;
+use personhog_coordination::types::{MigrationPhase, MigrationState, SealAck};
+use tokio::sync::Semaphore;
+use tokio::task::{JoinHandle, JoinSet};
+use tokio_util::sync::CancellationToken;
+use tracing::{info, warn};
+
+use crate::fencing::FencedChangelogProducers;
+
+pub const ACTIVE_INTERVAL: Duration = Duration::from_millis(250);
+pub const IDLE_INTERVAL: Duration = Duration::from_secs(2);
+/// Each seal spends one blocking thread per lane.
+const SEAL_CONCURRENCY: usize = 8;
+
+pub struct SealDuty {
+ pub store: Arc<PersonhogStore>,
+ pub pod_name: String,
+ pub fenced: Arc<FencedChangelogProducers>,
+}
+
+impl SealDuty {
+ /// Wakes when the saga or its phase changes, and on an interval for retries.
+ pub async fn run(self, cancel: CancellationToken) {
+ let mut sealed: HashSet<(String, u32)> = HashSet::new();
+ let mut preconnecting: HashMap<(String, u32), JoinHandle<()>> = HashMap::new();
+ let mut acked: Option<(String, BTreeSet<u32>)> = None;
+ let mut seen: Option<(String, MigrationPhase)> = None;
+ let preconnects = Arc::new(Semaphore::new(SEAL_CONCURRENCY));
+ let mut watch = None;
+ loop {
+ if watch.is_none() {
+ watch = match self.store.current_revision().await {
+ Ok(revision) => self.store.watch_migration_from(revision + 1).await.ok(),
+ Err(_) => None,
+ };
+ }
+ let interval = match self.store.get_migration().await {
+ Ok(Some(migration)) => {
+ seen = Some((migration.id.clone(), migration.phase));
+ let pass = match migration.phase {
+ MigrationPhase::Freezing | MigrationPhase::Draining => {
+ self.prepare(&migration, &mut preconnecting, &preconnects)
+ .await
+ }
+ MigrationPhase::Sealing => {
+ self.tick(&migration, &mut sealed, &mut acked).await
+ }
+ _ => Ok(()),
+ };
+ if let Err(e) = pass {
+ warn!(migration = %migration.id, error = %e, "seal duty pass failed");
+ counter!("personhog_leader_seal_duty_failures_total").increment(1);
+ }
+ // Only Sealing retries on a clock; the watch wakes every other phase.
+ if migration.phase == MigrationPhase::Sealing {
+ ACTIVE_INTERVAL
+ } else {
+ IDLE_INTERVAL
+ }
+ }
+ Ok(None) => {
+ seen = None;
+ sealed.clear();
+ preconnecting.clear();
+ acked = None;
+ IDLE_INTERVAL
+ }
+ Err(e) => {
+ warn!(error = %e, "seal duty could not read the migration");
+ IDLE_INTERVAL
+ }
+ };
+ let woke = async {
+ let Some(stream) = watch.as_mut() else {
+ return std::future::pending().await;
+ };
+ loop {
+ match stream.message().await {
+ // The coordinator rewrites the record on every seal ack; only a
+ // new saga or phase is worth a pass.
+ Ok(Some(resp)) if !resp.canceled() => {
+ // A deletion has no value to parse, so it reads as no saga.
+ let changed = resp.events().iter().any(|e| {
+ parse_watch_value::<MigrationState>(e)
+ .ok()
+ .map(|m| (m.id, m.phase))
+ != seen
+ });
+ if changed {
+ return true;
+ }
+ }
+ // A cancelled response ends the stream; rebuild it like a dead one.
+ _ => return false,
+ }
+ }
+ };
+ tokio::select! {
+ _ = cancel.cancelled() => return,
+ _ = tokio::time::sleep(interval) => {}
+ live = woke => {
+ if !live {
+ watch = None;
+ }
+ }
+ }
+ }
+ }
+
+ async fn targets(&self, migration: &MigrationState, overdue: bool) -> Result<Vec<u32>, String> {
+ let pods = self
+ .store
+ .list_pods()
+ .await
+ .map_err(|e| format!("list pods: {e}"))?;
+ let registered: HashSet<&str> = pods.iter().map(|p| p.pod_name.as_str()).collect();
+ let assignments = self
+ .store
+ .list_assignments()
+ .await
+ .map_err(|e| format!("list assignments: {e}"))?;
+ // Past the grace the acks matter: what no ack reports sealed is a target too.
+ let acks = if overdue {
+ self.store
+ .list_seal_acks()
+ .await
+ .map_err(|e| format!("list seal acks: {e}"))?
+ } else {
+ Vec::new()
+ };
+ Ok(seal_targets(
+ &self.pod_name,
+ &migration.id,
+ migration.from.partitions,
+ &assignments,
+ &registered,
+ &acks,
+ overdue,
+ ))
+ }
+
+ /// Asks for each target's old-topic connection unless a request for it is still
+ /// running: one the sweep discarded is dialed again on the next pass.
+ async fn prepare(
+ &self,
+ migration: &MigrationState,
+ preconnecting: &mut HashMap<(String, u32), JoinHandle<()>>,
+ preconnects: &Arc<Semaphore>,
+ ) -> Result<(), String> {
+ preconnecting.retain(|_, task| !task.is_finished());
+ for partition in self.targets(migration, false).await? {
+ let hash_map::Entry::Vacant(slot) =
+ preconnecting.entry((migration.id.clone(), partition))
+ else {
+ continue;
+ };
+ let fenced = Arc::clone(&self.fenced);
+ let from = migration.from.clone();
+ let permits = Arc::clone(preconnects);
+ slot.insert(tokio::spawn(async move {
+ let _permit = permits.acquire().await;
+ fenced.preconnect(partition, &from).await
+ }));
+ }
+ Ok(())
+ }
+
+ /// One Sealing pass: seal every target not yet sealed, then ack the state of all of them.
+ pub async fn tick(
+ &self,
+ migration: &MigrationState,
+ sealed: &mut HashSet<(String, u32)>,
+ acked: &mut Option<(String, BTreeSet<u32>)>,
+ ) -> Result<(), String> {
+ let overdue = seal_overdue(migration, assignment_coordination::util::now_millis());
+ let targets = self.targets(migration, overdue).await?;
+ let topic = migration.from.topic.as_str();
+ let pending: Vec<u32> = targets
+ .iter()
+ .copied()
+ .filter(|p| !sealed.contains(&(migration.id.clone(), *p)))
+ .collect();
+ if overdue && !pending.is_empty() {
+ counter!("personhog_leader_seal_overdue_passes_total").increment(1);
+ warn!(migration = %migration.id, partitions = ?pending, "partitions still unsealed past the seal grace; sealing every one this pod can reach");
+ }
+ if !pending.is_empty() {
+ let permits = Arc::new(Semaphore::new(SEAL_CONCURRENCY));
+ let mut seals = JoinSet::new();
+ for partition in pending {
+ let fenced = Arc::clone(&self.fenced);
+ let from = migration.from.clone();
+ let permits = Arc::clone(&permits);
+ seals.spawn(async move {
+ let _permit = permits.acquire().await;
+ (partition, fenced.seal(partition, &from).await)
+ });
+ }
+ while let Some(joined) = seals.join_next().await {
+ let (partition, result) = joined.map_err(|e| format!("seal join: {e}"))?;
+ match result {
+ Ok(path) => {
+ info!(partition, %topic, path, "sealed the old changelog partition");
+ sealed.insert((migration.id.clone(), partition));
+ // Acked as it lands; a failed write is retried by the next ack.
+ if let Err(e) = self.ack(migration, sealed, acked).await {
+ warn!(partition, error = %e, "seal ack failed; the next ack carries it");
+ }
+ }
+ Err(e) => warn!(partition, %topic, error = %e, "seal failed; retrying"),
+ }
+ }
+ }
+ self.ack(migration, sealed, acked).await
+ }
+
+ /// Reports every partition this process has sealed, so an overdue target
+ /// drops out of the unsealed set once acked; a restarted leader reseals.
+ async fn ack(
+ &self,
+ migration: &MigrationState,
+ sealed: &HashSet<(String, u32)>,
+ acked: &mut Option<(String, BTreeSet<u32>)>,
+ ) -> Result<(), String> {
+ let report: BTreeSet<u32> = sealed
+ .iter()
+ .filter(|(id, _)| *id == migration.id)
+ .map(|(_, p)| *p)
+ .collect();
+ // An unchanged ack wakes the coordinator for nothing.
+ if acked.as_ref() == Some(&(migration.id.clone(), report.clone())) {
+ return Ok(());
+ }
+ self.store
+ .put_seal_ack(&SealAck {
+ migration_id: migration.id.clone(),
+ pod_name: self.pod_name.clone(),
+ sealed: report.clone(),
+ })
+ .await
+ .map_err(|e| format!("write seal ack: {e}"))?;
+ *acked = Some((migration.id.clone(), report));
+ Ok(())
+ }
+}
diff --git a/rust/personhog-leader/src/settle.rs b/rust/personhog-leader/src/settle.rs
index 4c32df6c7e84..e2887235c611 100644
--- a/rust/personhog-leader/src/settle.rs
+++ b/rust/personhog-leader/src/settle.rs
@@ -7,6 +7,7 @@ use std::time::{Duration, Instant};
use dashmap::DashMap;
use metrics::{counter, histogram};
+use personhog_coordination::types::ChangelogConfig;
use tokio::sync::Mutex;
use crate::cache::{
@@ -14,23 +15,42 @@ use crate::cache::{
};
use crate::warming::{fetch_writer_committed_offsets, ConsumerPool};
+/// A topic listed neither live nor draining was retired, which proves the config's
+/// writer group applied it. The proof covers this leader only when that group is its own.
+pub fn retired_topic(live: &ChangelogConfig, writer_group: &str, topic: &str) -> bool {
+ live.writer_group == writer_group
+ && live.topic != topic
+ && live.draining.iter().all(|g| g.topic != topic)
+}
+
/// One prune pass over the published partitions: fetch committed offsets per
-/// topic, prune the applied marks and settle their death documents. A failed
-/// fetch leaves its topic's marks waiting.
+/// topic, prune the applied marks and settle their death documents. A retired
+/// stream settles without a fetch; a failed fetch leaves its topic's marks waiting.
pub async fn prune_and_settle_tick(
dirty_index: &DirtyIndex,
cache: &PartitionedCache,
locks: &DashMap<PersonCacheKey, Arc<Mutex<()>>>,
offsets_pool: &ConsumerPool,
offsets_timeout: Duration,
+ retired: impl Fn(&str) -> bool,
) -> Vec<StreamLag> {
let targets = dirty_index.streams_with_marks();
if targets.is_empty() {
return Vec::new();
}
+ // Judged after the listing: a stream seen with marks was captured under a config this read covers.
+ let mut retired_topics: HashMap<Arc<str>, bool> = HashMap::new();
+ for (_, stream) in &targets {
+ retired_topics
+ .entry(Arc::clone(&stream.topic))
+ .or_insert_with(|| retired(&stream.topic));
+ }
// One OffsetFetch per topic a live mark lives on.
let mut by_topic: HashMap<Arc<str>, Vec<u32>> = HashMap::new();
for (_, stream) in &targets {
+ if retired_topics[&stream.topic] {
+ continue;
+ }
let partitions = by_topic.entry(Arc::clone(&stream.topic)).or_default();
if !partitions.contains(&stream.partition) {
partitions.push(stream.partition);
@@ -55,17 +75,26 @@ pub async fn prune_and_settle_tick(
// next, so a catch-up never materializes more than one chunk.
let started = Instant::now();
let mut removed_total = 0u64;
+ let mut retired_total = 0u64;
let mut lags = Vec::with_capacity(targets.len());
for (partition, stream) in targets {
- let offset = committed
- .get(&stream.topic)
- .and_then(|offsets| offsets.get(&stream.partition))
- .copied();
+ let on_retired = retired_topics[&stream.topic];
+ let offset = if on_retired {
+ Some(i64::MAX)
+ } else {
+ committed
+ .get(&stream.topic)
+ .and_then(|offsets| offsets.get(&stream.partition))
+ .copied()
+ };
if let Some(offset) = offset {
if cache.is_published(partition) {
loop {
let (pruned, exhausted) = dirty_index.prune_chunk(partition, &stream, offset);
removed_total += pruned.removed as u64;
+ if on_retired {
+ retired_total += pruned.removed as u64;
+ }
if !pruned.death_marks.is_empty() {
drop_settled_death_documents(cache, locks, &pruned.death_marks).await;
}
@@ -75,15 +104,21 @@ pub async fn prune_and_settle_tick(
}
}
}
- lags.push(StreamLag {
- partition,
- stream,
- committed: offset,
- });
+ if !on_retired {
+ lags.push(StreamLag {
+ partition,
+ stream,
+ committed: offset,
+ });
+ }
}
if removed_total > 0 {
counter!("personhog_leader_dirty_index_pruned_total").increment(removed_total);
}
+ if retired_total > 0 {
+ counter!("personhog_leader_dirty_index_retired_stream_marks_total")
+ .increment(retired_total);
+ }
// A long tick means the settle parked on per-key locks, not an idle
// pass.
histogram!("personhog_leader_prune_tick_duration_ms")
@@ -138,6 +173,55 @@ pub async fn drop_settled_death_documents(
mod tests {
use super::*;
use crate::cache::CachedPerson;
+ use personhog_coordination::types::Generation;
+
+ #[test]
+ fn a_topic_is_retired_only_when_the_config_vouches_for_the_writer() {
+ let config = |topic: &str, writer_group: &str, draining: &[&str]| ChangelogConfig {
+ topic: topic.to_string(),
+ partitions: 8,
+ epoch: 2,
+ writer_group: writer_group.to_string(),
+ draining: draining
+ .iter()
+ .map(|t| Generation {
+ topic: t.to_string(),
+ partitions: 4,
+ epoch: 1,
+ })
+ .collect(),
+ };
+ let cases = [
+ (
+ "the live topic",
+ config("live", "writer", &[]),
+ "live",
+ false,
+ ),
+ (
+ "a draining topic",
+ config("live", "writer", &["old"]),
+ "old",
+ false,
+ ),
+ (
+ "a topic listed nowhere",
+ config("live", "writer", &["old"]),
+ "gone",
+ true,
+ ),
+ (
+ "another writer group",
+ config("live", "other", &[]),
+ "gone",
+ false,
+ ),
+ ("no writer group", config("", "", &[]), "gone", false),
+ ];
+ for (name, live, topic, expected) in cases {
+ assert_eq!(retired_topic(&live, "writer", topic), expected, "{name}");
+ }
+ }
fn death_person(version: i64) -> CachedPerson {
CachedPerson {
diff --git a/rust/personhog-leader/tests/common/mod.rs b/rust/personhog-leader/tests/common/mod.rs
index 5c8e2d6fbe8e..8cbff5d90a08 100644
--- a/rust/personhog-leader/tests/common/mod.rs
+++ b/rust/personhog-leader/tests/common/mod.rs
@@ -118,6 +118,8 @@ pub fn start_coordinator(
handoff_deadline: Duration::from_secs(86_400),
warming_deadline: Duration::from_secs(86_400),
max_txn_ops: 128,
+ migration_pre_flip_deadline: Duration::from_secs(86_400),
+ migration_debounce_interval: Duration::from_millis(10),
},
strategy,
None,
diff --git a/rust/personhog-leader/tests/fencing_integration.rs b/rust/personhog-leader/tests/fencing_integration.rs
index 1c16932126eb..5f011bc29680 100644
--- a/rust/personhog-leader/tests/fencing_integration.rs
+++ b/rust/personhog-leader/tests/fencing_integration.rs
@@ -140,6 +140,44 @@ async fn acquiring_on_another_topic_produces_there_and_discards_a_stale_preconne
assert_eq!(read_committed_count(&new).await, 1);
}
+/// Sealing claims every lane's id on the old topic, so a holder still producing
+/// there is fenced and nothing more lands, on the cold and the prepared path.
+#[tokio::test]
+async fn a_seal_fences_the_holder_on_the_old_topic() {
+ for prepared in [false, true] {
+ let topic = format!("fence_seal_{}", uuid::Uuid::new_v4().simple());
+ let holder = fenced_producers(2);
+ holder
+ .acquire(0, &target(&topic))
+ .await
+ .expect("holder acquires");
+ holder
+ .produce(0, &test_person(1))
+ .await
+ .expect("holder produces before the seal");
+
+ let sealer = fenced_producers(2);
+ if prepared {
+ sealer.preconnect(0, &target(&topic)).await;
+ }
+ let path = sealer
+ .seal(0, &target(&topic))
+ .await
+ .expect("the seal claims every lane");
+ assert_eq!(path, if prepared { "prepared" } else { "cold" });
+
+ match holder.produce(0, &test_person(2)).await {
+ Err(FencedProduceError::Fenced) | Err(FencedProduceError::NotAcquired) => {}
+ other => panic!("prepared={prepared}: the holder must be fenced, got {other:?}"),
+ }
+ assert_eq!(
+ read_committed_count(&topic).await,
+ 1,
+ "prepared={prepared}: nothing lands after the seal"
+ );
+ }
+}
+
#[tokio::test]
async fn a_prepared_connection_still_fences_the_previous_owner() {
let topic = format!("fence_prepared_{}", uuid::Uuid::new_v4().simple());
diff --git a/rust/personhog-leader/tests/seal_duty_integration.rs b/rust/personhog-leader/tests/seal_duty_integration.rs
new file mode 100644
index 000000000000..5fad725d6e34
--- /dev/null
+++ b/rust/personhog-leader/tests/seal_duty_integration.rs
@@ -0,0 +1,287 @@
+//! The seal duty against real etcd and Kafka: which partitions a leader
+//! seals during a changelog migration, and what it acks.
+
+mod common;
+
+use std::collections::HashSet;
+use std::sync::Arc;
+use std::time::Duration;
+
+use personhog_coordination::types::{
+ AssignmentStatus, ChangelogTarget, MigrationPhase, MigrationState, PartitionAssignment,
+ PodStatus, RegisteredPod,
+};
+use personhog_leader::migration::SealDuty;
+
+use common::{owned_broker_topic, test_fenced_producers, test_store};
+
+fn registered(pod_name: &str) -> RegisteredPod {
+ RegisteredPod {
+ pod_name: pod_name.to_string(),
+ generation: String::new(),
+ status: PodStatus::Ready,
+ registered_at: 0,
+ last_heartbeat: 0,
+ controller: None,
+ advertise_address: None,
+ }
+}
+
+fn now_millis() -> i64 {
+ std::time::SystemTime::now()
+ .duration_since(std::time::UNIX_EPOCH)
+ .unwrap()
+ .as_millis() as i64
+}
+
+fn assignment(partition: u32, owner: &str) -> PartitionAssignment {
+ PartitionAssignment {
+ partition,
+ owner: owner.to_string(),
+ advertise_address: None,
+ status: AssignmentStatus::Active,
+ }
+}
+
+/// A holder restarted mid-Sealing owns partitions it never loaded. It still
+/// seals them: its peers see a registered owner and leave them to it.
+#[tokio::test]
+async fn a_pod_seals_the_partitions_it_owns_by_assignment_without_a_capture() {
+ let store = test_store("seal-duty-assignment").await;
+ let old = owned_broker_topic(2).await;
+ let lease = store.grant_lease(60).await.unwrap();
+ for pod in ["leader-a", "leader-b"] {
+ store.register_pod(&registered(pod), lease).await.unwrap();
+ }
+ store
+ .put_assignments(&[assignment(0, "leader-a"), assignment(1, "leader-b")])
+ .await
+ .unwrap();
+ let migration = MigrationState {
+ id: "m-1".to_string(),
+ from: ChangelogTarget {
+ topic: old.topic.clone(),
+ partitions: 2,
+ },
+ to: ChangelogTarget {
+ topic: format!("{}_g2", old.topic),
+ partitions: 4,
+ },
+ phase: MigrationPhase::Sealing,
+ // Inside the seal grace, so peers still leave each other's partitions alone.
+ created_at_ms: now_millis(),
+ phase_entered_at_ms: now_millis(),
+ blocking: Vec::new(),
+ cancel_reason: None,
+ };
+ assert!(store.create_migration(&migration).await.unwrap());
+
+ let duty = SealDuty {
+ store: Arc::clone(&store),
+ pod_name: "leader-a".to_string(),
+ fenced: Arc::new(test_fenced_producers()),
+ };
+ let mut sealed = HashSet::new();
+ duty.tick(&migration, &mut sealed, &mut None).await.unwrap();
+
+ let acks = store.list_seal_acks().await.unwrap();
+ let ack = acks
+ .iter()
+ .find(|a| a.pod_name == "leader-a")
+ .expect("the pod acks its pass");
+ assert!(
+ ack.sealed.contains(&0),
+ "its own partition is sealed although nothing is loaded: {ack:?}"
+ );
+ assert!(
+ !ack.sealed.contains(&1),
+ "a registered peer's partition is left to the peer: {ack:?}"
+ );
+}
+
+/// A registered holder that cannot seal must not hold the fleet: once the
+/// seal is overdue, its peers seal its partitions too.
+#[tokio::test]
+async fn a_registered_peers_partition_is_sealed_once_the_seal_is_overdue() {
+ let store = test_store("seal-duty-overdue").await;
+ let old = owned_broker_topic(2).await;
+ let lease = store.grant_lease(60).await.unwrap();
+ for pod in ["leader-a", "leader-b"] {
+ store.register_pod(&registered(pod), lease).await.unwrap();
+ }
+ store
+ .put_assignments(&[assignment(0, "leader-a"), assignment(1, "leader-b")])
+ .await
+ .unwrap();
+ let now_ms = now_millis();
+ let migration = MigrationState {
+ id: "m-2".to_string(),
+ from: ChangelogTarget {
+ topic: old.topic.clone(),
+ partitions: 2,
+ },
+ to: ChangelogTarget {
+ topic: format!("{}_g2", old.topic),
+ partitions: 4,
+ },
+ phase: MigrationPhase::Sealing,
+ created_at_ms: now_ms - 20_000,
+ phase_entered_at_ms: now_ms - 10_000,
+ blocking: Vec::new(),
+ cancel_reason: None,
+ };
+ assert!(store.create_migration(&migration).await.unwrap());
+ let duty = SealDuty {
+ store: Arc::clone(&store),
+ pod_name: "leader-a".to_string(),
+ fenced: Arc::new(test_fenced_producers()),
+ };
+ let mut sealed = HashSet::new();
+ let mut acked = None;
+ // Once its own ack reports the partition sealed, it is no longer overdue;
+ // the report must keep it rather than flap between passes.
+ let mut revision_after_first = None;
+ for pass in 0..2 {
+ duty.tick(&migration, &mut sealed, &mut acked)
+ .await
+ .unwrap();
+ // An unchanged report is not rewritten, so it wakes no one.
+ let revision = store.seal_ack_mod_revision("leader-a").await.unwrap();
+ if pass == 0 {
+ revision_after_first = revision;
+ } else {
+ assert_eq!(
+ revision, revision_after_first,
+ "the second pass wrote to etcd"
+ );
+ }
+ let acks = store.list_seal_acks().await.unwrap();
+ let ack = acks
+ .iter()
+ .find(|a| a.pod_name == "leader-a")
+ .expect("the pod acks its pass");
+ assert!(
+ ack.sealed.contains(&1),
+ "pass {pass}: the overdue peer's partition stays reported sealed: {ack:?}"
+ );
+ }
+}
+
+/// A seal is acked as it lands, so a pod still working through its partitions
+/// does not look overdue to its peers.
+#[tokio::test]
+async fn each_seal_is_acked_before_the_pass_ends() {
+ let store = test_store("seal-duty-early-ack").await;
+ let old = owned_broker_topic(2).await;
+ let lease = store.grant_lease(60).await.unwrap();
+ store
+ .register_pod(&registered("leader-a"), lease)
+ .await
+ .unwrap();
+ store
+ .put_assignments(&[assignment(0, "leader-a"), assignment(1, "leader-a")])
+ .await
+ .unwrap();
+ let migration = MigrationState {
+ id: "m-1".to_string(),
+ from: ChangelogTarget {
+ topic: old.topic.clone(),
+ partitions: 2,
+ },
+ to: ChangelogTarget {
+ topic: format!("{}_g2", old.topic),
+ partitions: 4,
+ },
+ phase: MigrationPhase::Sealing,
+ created_at_ms: now_millis(),
+ phase_entered_at_ms: now_millis(),
+ blocking: Vec::new(),
+ cancel_reason: None,
+ };
+ assert!(store.create_migration(&migration).await.unwrap());
+
+ let fenced = Arc::new(test_fenced_producers());
+ // A dial in flight holds partition 1's seal until the init timeout.
+ fenced.stage_connecting_for_test(1, Duration::ZERO);
+ let duty = SealDuty {
+ store: Arc::clone(&store),
+ pod_name: "leader-a".to_string(),
+ fenced,
+ };
+ let pass = tokio::spawn(async move {
+ let mut sealed = HashSet::new();
+ duty.tick(&migration, &mut sealed, &mut None).await.unwrap();
+ });
+
+ tokio::time::timeout(Duration::from_secs(5), async {
+ loop {
+ let acks = store.list_seal_acks().await.unwrap();
+ if acks.iter().any(|a| a.sealed.contains(&0)) {
+ break;
+ }
+ tokio::time::sleep(Duration::from_millis(20)).await;
+ }
+ })
+ .await
+ .expect("partition 0 is acked while partition 1 is still sealing");
+ assert!(!pass.is_finished(), "the pass is still sealing partition 1");
+ pass.abort();
+}
+
+/// A preconnect the fencing sweep discards is dialed again on the next pass.
+#[tokio::test]
+async fn a_swept_preconnect_is_dialed_again_on_the_next_pass() {
+ let store = test_store("seal-duty-preconnect").await;
+ let old = owned_broker_topic(1).await;
+ let lease = store.grant_lease(60).await.unwrap();
+ store
+ .register_pod(&registered("leader-a"), lease)
+ .await
+ .unwrap();
+ store
+ .put_assignments(&[assignment(0, "leader-a")])
+ .await
+ .unwrap();
+ let migration = MigrationState {
+ id: "m-1".to_string(),
+ from: ChangelogTarget {
+ topic: old.topic.clone(),
+ partitions: 1,
+ },
+ to: ChangelogTarget {
+ topic: format!("{}_g2", old.topic),
+ partitions: 2,
+ },
+ phase: MigrationPhase::Draining,
+ created_at_ms: now_millis(),
+ phase_entered_at_ms: now_millis(),
+ blocking: Vec::new(),
+ cancel_reason: None,
+ };
+ assert!(store.create_migration(&migration).await.unwrap());
+ let fenced = Arc::new(test_fenced_producers());
+ let duty = SealDuty {
+ store: Arc::clone(&store),
+ pod_name: "leader-a".to_string(),
+ fenced: Arc::clone(&fenced),
+ };
+ let cancel = tokio_util::sync::CancellationToken::new();
+ tokio::spawn(duty.run(cancel.clone()));
+ let prepared = || async {
+ tokio::time::timeout(Duration::from_secs(10), async {
+ while !fenced.has_prepared(0) {
+ tokio::time::sleep(Duration::from_millis(20)).await;
+ }
+ })
+ .await
+ };
+
+ assert!(prepared().await.is_ok(), "the duty preconnects its target");
+ fenced.sweep_prepared();
+ assert!(!fenced.has_prepared(0));
+ assert!(
+ prepared().await.is_ok(),
+ "the next pass dials the swept connection again"
+ );
+ cancel.cancel();
+}
diff --git a/rust/personhog-leader/tests/warming_integration.rs b/rust/personhog-leader/tests/warming_integration.rs
index 55bfb5358d6e..28f19fd15174 100644
--- a/rust/personhog-leader/tests/warming_integration.rs
+++ b/rust/personhog-leader/tests/warming_integration.rs
@@ -846,6 +846,7 @@ async fn the_prune_tick_settles_published_partitions_only() {
&locks,
&pools.offsets,
Duration::from_secs(5),
+ |_| false,
)
.await;
@@ -867,12 +868,95 @@ async fn the_prune_tick_settles_published_partitions_only() {
&locks,
&pools.offsets,
Duration::from_secs(5),
+ |_| false,
)
.await;
assert!(dirty_index.get(&warming_key).is_none());
assert!(cache.peek(1, &warming_key).is_none());
}
+/// A retired stream's marks settle without the writer's offsets, death
+/// documents included; a live stream's marks keep waiting for the writer.
+#[tokio::test]
+async fn the_prune_tick_settles_a_retired_streams_marks_without_offsets() {
+ let (cluster, _producer) = create_mock_kafka().await;
+ let (_cfg, pools) = warming_config_for("prune-tick-retired", &cluster);
+ const RETIRED_TOPIC: &str = "personhog_updates_retired";
+
+ let cache = PartitionedCache::new(1 << 20);
+ let dirty_index = DirtyIndex::new(1_000_000);
+ let locks = dashmap::DashMap::new();
+ cache.create_partition(0);
+
+ let death_doc = |person_id: i64| {
+ let mut person = make_person(1, person_id);
+ person.is_deleted = true;
+ person.version = 2;
+ personhog_leader::cache::CachedPerson::try_from(person).unwrap()
+ };
+ let mark_on = |topic: &str, partition: u32| personhog_leader::cache::DirtyMark {
+ version: 2,
+ offset: 0,
+ partition: 0,
+ stream: personhog_leader::cache::Stream {
+ topic: std::sync::Arc::from(topic),
+ partition,
+ },
+ is_deleted: true,
+ };
+ let retired_key = PersonCacheKey {
+ team_id: 1,
+ person_id: 1,
+ };
+ let live_key = PersonCacheKey {
+ team_id: 1,
+ person_id: 2,
+ };
+ let moved_key = PersonCacheKey {
+ team_id: 1,
+ person_id: 3,
+ };
+ cache.put(0, retired_key.clone(), death_doc(1));
+ dirty_index.mark(retired_key.clone(), mark_on(RETIRED_TOPIC, 3));
+ cache.put(0, live_key.clone(), death_doc(2));
+ dirty_index.mark(live_key.clone(), mark_on(CHANGELOG_TOPIC, 0));
+ // Queued under the retired stream, but its live mark moved on.
+ dirty_index.mark(moved_key.clone(), mark_on(RETIRED_TOPIC, 3));
+ dirty_index.mark(moved_key.clone(), mark_on(CHANGELOG_TOPIC, 0));
+
+ // The writer committed on neither topic.
+ tokio::time::timeout(
+ Duration::from_secs(30),
+ personhog_leader::settle::prune_and_settle_tick(
+ &dirty_index,
+ &cache,
+ &locks,
+ &pools.offsets,
+ Duration::from_secs(5),
+ |topic| topic == RETIRED_TOPIC,
+ ),
+ )
+ .await
+ .expect("the tick must return once the retired stream is exhausted");
+
+ assert!(dirty_index.get(&retired_key).is_none());
+ assert_eq!(
+ dirty_index
+ .get(&moved_key)
+ .map(|m| m.stream.topic.to_string()),
+ Some(CHANGELOG_TOPIC.to_string()),
+ "a key that moved to the live stream keeps waiting there"
+ );
+ assert!(
+ cache.peek(0, &retired_key).is_none(),
+ "the retired stream's death document settles"
+ );
+ assert!(
+ dirty_index.get(&live_key).is_some(),
+ "a live stream waits for the writer's offset"
+ );
+}
+
/// One tick drains a death backlog larger than a prune chunk: the chunk
/// loop must run to exhaustion, not stop after the first slice.
#[tokio::test]
@@ -921,6 +1005,7 @@ async fn the_prune_tick_drains_a_multi_chunk_death_backlog() {
&locks,
&pools.offsets,
Duration::from_secs(5),
+ |_| false,
)
.await;
@@ -1086,6 +1171,7 @@ async fn a_restart_warms_the_old_generations_congruent_tail_for_its_keys() {
&locks,
&pools.offsets,
Duration::from_secs(5),
+ |_| false,
)
.await;
assert!(dirty_index.is_empty(), "every old-stream mark settles");
diff --git a/rust/personhog-router/src/backend/leader.rs b/rust/personhog-router/src/backend/leader.rs
index eff032c33abe..4b23e4fc8537 100644
--- a/rust/personhog-router/src/backend/leader.rs
+++ b/rust/personhog-router/src/backend/leader.rs
@@ -435,6 +435,19 @@ impl LeaderBackend {
loop {
// Per attempt: a bounce may be the leader refusing a stale modulus.
let partition = self.partition_for_person(key.0, key.1);
+ // A partition born in the flip parks its writes until its first handoff drains
+ // them. The table's read lock ensures a route added later meets the stash.
+ if self
+ .config
+ .changelog
+ .draining_floor()
+ .is_some_and(|floor| partition >= floor)
+ {
+ let table = self.routing_table.read().await;
+ if !table.contains_key(&partition) {
+ self.stash.begin_stash(partition);
+ }
+ }
// The stash module emits its own enqueued/rejected counters
// at the source; we don't double-count here. It borrows the
// frame and headers, cloning only when the request actually
diff --git a/rust/personhog-router/src/backend/stash.rs b/rust/personhog-router/src/backend/stash.rs
index 0490785bf199..e9d104a0ffd0 100644
--- a/rust/personhog-router/src/backend/stash.rs
+++ b/rust/personhog-router/src/backend/stash.rs
@@ -20,6 +20,7 @@
//! An entry leaves the stash only with a definitive outcome. An attempt that
//! cannot finish puts its entries back in order, so a retry is never overtaken.
+use std::collections::BTreeSet;
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
use std::sync::{Arc, Mutex, MutexGuard};
use std::time::Instant;
@@ -252,6 +253,23 @@ impl StashTable {
self.inner.stashing.contains_key(&partition) || self.inner.holds_keys_of(partition)
}
+ /// Every partition `is_pending` holds for, scanning one shard at a time.
+ pub fn pending_partitions(&self) -> Vec<u32> {
+ let inner = &self.inner;
+ let mut pending: BTreeSet<u32> = inner.stashing.iter().map(|e| *e.key()).collect();
+ for shard in &inner.shards {
+ if shard.held.load(Ordering::Acquire) > 0 {
+ pending.extend(
+ shard
+ .lock()
+ .deferring_keys()
+ .map(|key| inner.partition_of(key)),
+ );
+ }
+ }
+ pending.into_iter().collect()
+ }
+
/// Begin buffering requests for `partition`. Each call opens a new
/// generation, so a drain that started before it cannot close it.
pub fn begin_stash(&self, partition: u32) {
@@ -862,6 +880,30 @@ mod tests {
assert_eq!(taker.take_for_attempt(usize::MAX).len(), 1);
}
+ /// The reconcile pass drains what this lists, so it must name a freeze
+ /// with nothing parked and a parked key's partition under the live count.
+ #[test]
+ fn pending_partitions_names_freezes_and_parked_keys() {
+ let changelog = LiveChangelog::new(ChangelogConfig::new("personhog_updates", 8));
+ let table = StashTable::with_bounds(usize::MAX, usize::MAX, changelog.clone());
+ let person_id = moving_person(8, 4);
+ let old = partition_for_person(1, person_id, 8);
+ table.begin_stash(old);
+ assert!(matches!(
+ enqueue(&table, person_id),
+ StashDecision::Stashed(_)
+ ));
+ let empty_freeze = (0..8).find(|p| *p != old).unwrap();
+ table.begin_stash(empty_freeze);
+
+ adopt(&changelog, 4);
+ let new = partition_for_person(1, person_id, 4);
+ let pending = table.pending_partitions();
+ for partition in [old, empty_freeze, new] {
+ assert!(pending.contains(&partition), "{partition} in {pending:?}");
+ }
+ }
+
/// A freeze opened while a drain runs belongs to the next handoff;
/// the running drain's settle must not end it.
#[test]
diff --git a/rust/personhog-router/src/config.rs b/rust/personhog-router/src/config.rs
index 5dbd5bdcb654..bc13138a868d 100644
--- a/rust/personhog-router/src/config.rs
+++ b/rust/personhog-router/src/config.rs
@@ -375,6 +375,14 @@ pub struct Config {
#[envconfig(default = "120")]
pub coordinator_handoff_deadline_secs: u64,
+ /// Frozen partitions are a write outage, so a migration cancels past this.
+ #[envconfig(default = "30")]
+ pub coordinator_migration_pre_flip_deadline_secs: u64,
+
+ /// Coalesces a burst of watch events into one migration pass.
+ #[envconfig(default = "10")]
+ pub coordinator_migration_debounce_ms: u64,
+
// ── K8s awareness (leader mode only) ────────────────────────
/// Enable K8s-aware departure classification for smarter rebalancing.
/// When disabled, falls back to lease-based behavior.
@@ -1029,6 +1037,14 @@ impl Config {
Duration::from_secs(self.coordinator_handoff_deadline_secs)
}
+ pub fn coordinator_migration_pre_flip_deadline(&self) -> Duration {
+ Duration::from_secs(self.coordinator_migration_pre_flip_deadline_secs)
+ }
+
+ pub fn coordinator_migration_debounce_interval(&self) -> Duration {
+ Duration::from_millis(self.coordinator_migration_debounce_ms)
+ }
+
pub fn router_reconcile_interval(&self) -> Duration {
Duration::from_secs(self.router_reconcile_secs)
}
diff --git a/rust/personhog-router/src/main.rs b/rust/personhog-router/src/main.rs
index 418dc38fc082..f8fe9c171072 100644
--- a/rust/personhog-router/src/main.rs
+++ b/rust/personhog-router/src/main.rs
@@ -335,6 +335,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
Arc::clone(&leader_backend),
config.stash_max_wait(),
config.stash_drain_concurrency,
+ Arc::clone(&store),
));
// Start routing table (etcd registration + assignment/handoff watches)
@@ -391,6 +392,8 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
// The same server limit the routing table's ack
// batches mirror: one env, one `--max-txn-ops`.
max_txn_ops: config.etcd_max_txn_ops,
+ migration_pre_flip_deadline: config.coordinator_migration_pre_flip_deadline(),
+ migration_debounce_interval: config.coordinator_migration_debounce_interval(),
},
Arc::new(StickyBalancedStrategy),
k8s_awareness,
diff --git a/rust/personhog-router/src/stash_handler.rs b/rust/personhog-router/src/stash_handler.rs
index 9d334059e638..c44d92dbca5b 100644
--- a/rust/personhog-router/src/stash_handler.rs
+++ b/rust/personhog-router/src/stash_handler.rs
@@ -7,6 +7,7 @@ use futures::future::join_all;
use keyed_stash::Entry;
use personhog_coordination::error::Result as CoordResult;
use personhog_coordination::routing_table::StashHandler;
+use personhog_coordination::store::PersonhogStore;
use tokio_util::sync::CancellationToken;
use tonic::Code;
@@ -19,6 +20,9 @@ use crate::grpc_http::{grpc_error_response, is_grpc_error_response};
/// How long a drain waits before looking again for a key it could not take.
const SETTLE_RETRY: Duration = Duration::from_millis(10);
+/// How long a drain waits for its config read before draining under the adopted count.
+const CONFIG_READ_TIMEOUT: Duration = Duration::from_secs(1);
+
/// Stash handler for the router. Reacts to handoff phase transitions:
///
/// * `Freezing` / `Draining` / `Warming` → `begin_stash`: start (or
@@ -97,6 +101,8 @@ pub struct RouterStashHandler {
/// Per-key ordering is preserved (sequential within a key); across
/// keys we fan out up to this many at once.
drain_concurrency: usize,
+ /// A drain reads the config here first, in case the config watch lags its handoff.
+ store: Arc<PersonhogStore>,
}
impl RouterStashHandler {
@@ -104,6 +110,7 @@ impl RouterStashHandler {
leader_backend: Arc<LeaderBackend>,
max_stash_wait: Duration,
drain_concurrency: usize,
+ store: Arc<PersonhogStore>,
) -> Self {
// A drain_concurrency of 0 would mean "never forward" — treat
// as 1 (fully sequential) to keep the contract sensible if
@@ -113,6 +120,30 @@ impl RouterStashHandler {
leader_backend,
max_stash_wait,
drain_concurrency,
+ store,
+ }
+ }
+
+ /// Read the config from etcd first, so a drain that beat the config watch takes the
+ /// right keys. Best effort: on failure it drains under the adopted count.
+ async fn adopt_current_config(&self, cancel: &CancellationToken) {
+ let read = tokio::select! {
+ _ = cancel.cancelled() => return,
+ read = tokio::time::timeout(CONFIG_READ_TIMEOUT, self.store.get_changelog_config()) => read,
+ };
+ match read {
+ Ok(Ok(Some(config))) => {
+ self.leader_backend.changelog().adopt(&config);
+ }
+ Ok(Ok(None)) => {}
+ Ok(Err(e)) => {
+ metrics::counter!("personhog_router_drain_config_read_failures_total").increment(1);
+ tracing::warn!(error = %e, "drain could not read config/changelog; draining under the adopted count");
+ }
+ Err(_) => {
+ metrics::counter!("personhog_router_drain_config_read_failures_total").increment(1);
+ tracing::warn!("drain config read timed out; draining under the adopted count");
+ }
}
}
}
@@ -303,6 +334,10 @@ impl StashHandler for RouterStashHandler {
self.leader_backend.stash_table().is_pending(partition)
}
+ fn pending_partitions(&self) -> Vec<u32> {
+ self.leader_backend.stash_table().pending_partitions()
+ }
+
async fn begin_stash(&self, partition: u32, new_owner: &str) -> CoordResult<()> {
tracing::info!(
partition,
@@ -319,6 +354,7 @@ impl StashHandler for RouterStashHandler {
new_owner: &str,
cancel: CancellationToken,
) -> CoordResult<()> {
+ self.adopt_current_config(&cancel).await;
let Some(session) = self.leader_backend.stash_table().drain_session(partition) else {
// Nothing pending — the partition never froze, or a prior
// drain already settled it.
@@ -332,7 +368,10 @@ impl StashHandler for RouterStashHandler {
// owner's entry stays in the cache unused — the routing table
// no longer points at it — and is reclaimed only if that
// address is ever resolved again; the cache has no eviction.
- self.leader_backend.clear_client_cache(new_owner);
+ // A drain no handoff or assignment names has no new owner.
+ if !new_owner.is_empty() {
+ self.leader_backend.clear_client_cache(new_owner);
+ }
let mut completed_total: u64 = 0;
let mut bounced_waves: u32 = 0;
diff --git a/rust/personhog-router/tests/stash_integration.rs b/rust/personhog-router/tests/stash_integration.rs
index c20316bf084f..12a0dd8aad8a 100644
--- a/rust/personhog-router/tests/stash_integration.rs
+++ b/rust/personhog-router/tests/stash_integration.rs
@@ -20,7 +20,7 @@ use personhog_common::partitioning::partition_for_person;
use personhog_coordination::changelog::LiveChangelog;
use personhog_coordination::routing_table::StashHandler;
use personhog_coordination::store::PersonhogStore;
-use personhog_coordination::types::ChangelogConfig;
+use personhog_coordination::types::{ChangelogConfig, Generation};
use personhog_proto::personhog::types::v1::{
FoldPersonDocumentRequest, FoldPersonDocumentResponse, GetPersonRequest, GetPersonResponse,
Person, SealedSourceSnapshot, UpdatePersonPropertiesRequest, UpdatePersonPropertiesResponse,
@@ -45,11 +45,16 @@ const TEST_MAX_STASH_WAIT: Duration = Duration::from_secs(30);
/// exercised but doesn't dwarf the test's worker count.
const TEST_DRAIN_CONCURRENCY: usize = 4;
-fn new_test_handler(backend: Arc<LeaderBackend>) -> RouterStashHandler {
- RouterStashHandler::new(backend, TEST_MAX_STASH_WAIT, TEST_DRAIN_CONCURRENCY)
+async fn new_test_handler(backend: Arc<LeaderBackend>) -> RouterStashHandler {
+ RouterStashHandler::new(
+ backend,
+ TEST_MAX_STASH_WAIT,
+ TEST_DRAIN_CONCURRENCY,
+ test_store().await,
+ )
}
-/// A store under a fresh prefix, so no other test's config is visible.
+/// A store under a fresh prefix: a drain's config read finds nothing and keeps the backend's.
async fn test_store() -> Arc<PersonhogStore> {
let inner = EtcdStore::connect(StoreConfig {
endpoints: vec!["http://localhost:2379".to_string()],
@@ -203,6 +208,89 @@ async fn forward_read(
response
}
+/// A partition born in the flip has no owner until its fresh handoff lands;
+/// a write for it parks and drains with that handoff instead of bouncing.
+#[tokio::test]
+async fn a_write_for_a_partition_born_in_the_flip_parks_until_its_handoff_drains() {
+ let old_count = NUM_PARTITIONS / 2;
+ let team_id = 1;
+ let person_id = (1..)
+ .find(|id| partition_for_person(team_id, *id, NUM_PARTITIONS) >= old_count)
+ .expect("some key lands in the new space");
+ let partition = partition_for_person(team_id, person_id, NUM_PARTITIONS);
+ let person = Person {
+ id: person_id,
+ team_id,
+ ..create_test_person()
+ };
+ let leader_addr = start_test_leader(TestLeaderService::new().with_person(person)).await;
+
+ let routing_table = Arc::new(RwLock::new(HashMap::new()));
+ for owned in 0..old_count {
+ routing_table
+ .write()
+ .await
+ .insert(owned, "leader-0".to_string());
+ }
+ let changelog = LiveChangelog::new(ChangelogConfig {
+ topic: "personhog_updates_v2".to_string(),
+ partitions: NUM_PARTITIONS,
+ epoch: 2,
+ writer_group: String::new(),
+ draining: vec![Generation {
+ topic: "personhog_updates_v1".to_string(),
+ partitions: old_count,
+ epoch: 1,
+ }],
+ });
+ let leader_url = format!("http://{leader_addr}");
+ let resolver: personhog_router::backend::AddressResolver =
+ Arc::new(move |_pod: &str| Some(leader_url.clone()));
+ let backend = Arc::new(LeaderBackend::new(
+ Arc::clone(&routing_table),
+ resolver,
+ LeaderBackendConfig {
+ changelog: changelog.clone(),
+ timeout: Duration::from_secs(5),
+ num_channels: 1,
+ http2_windows: Http2Windows::default(),
+ },
+ StashTable::with_bounds(usize::MAX, usize::MAX, changelog),
+ ));
+ let handler = new_test_handler(Arc::clone(&backend)).await;
+
+ let req = mk_request(team_id, person_id, "[email protected]");
+ let backend_for_call = Arc::clone(&backend);
+ let in_flight = tokio::spawn(async move { forward(&backend_for_call, req).await });
+ let parked = tokio::time::timeout(Duration::from_secs(2), async {
+ while !handler.stash_pending(partition) {
+ tokio::time::sleep(Duration::from_millis(10)).await;
+ }
+ })
+ .await;
+ assert!(parked.is_ok(), "the write parks instead of bouncing");
+ assert!(!in_flight.is_finished());
+
+ // The fresh handoff lands and completes.
+ routing_table
+ .write()
+ .await
+ .insert(partition, "leader-0".to_string());
+ handler
+ .drain_stash(partition, "leader-0", CancellationToken::new())
+ .await
+ .expect("drain_stash should succeed");
+
+ let raw = tokio::time::timeout(Duration::from_secs(2), in_flight)
+ .await
+ .expect("the drain releases the parked write")
+ .expect("task should not panic");
+ let response = decode_response::<UpdatePersonPropertiesResponse>(raw)
+ .await
+ .expect("the write succeeds once its partition has an owner");
+ assert_eq!(response.person.expect("a person").id, person_id);
+}
+
/// A request that arrives while the stash is open must park on a oneshot,
/// and a subsequent `drain_stash` must forward it to the leader and deliver
/// the leader's reply back to the original caller. This is the core contract
@@ -214,7 +302,7 @@ async fn request_during_stash_completes_after_drain() {
let stash = StashTable::with_bounds(usize::MAX, usize::MAX, changelog());
let backend = make_backend(leader_addr, stash.clone()).await;
- let handler = new_test_handler(Arc::clone(&backend));
+ let handler = new_test_handler(Arc::clone(&backend)).await;
// Open the stash for the partition this request will land on.
let partition = backend.partition_for_person(person.team_id, person.id);
@@ -264,7 +352,7 @@ async fn multiple_stashed_requests_drain_in_fifo() {
let stash = StashTable::with_bounds(usize::MAX, usize::MAX, changelog());
let backend = make_backend(leader_addr, stash.clone()).await;
- let handler = new_test_handler(Arc::clone(&backend));
+ let handler = new_test_handler(Arc::clone(&backend)).await;
let partition = backend.partition_for_person(person.team_id, person.id);
handler.begin_stash(partition, "leader-new").await.unwrap();
@@ -322,7 +410,7 @@ async fn requests_for_unstashed_partition_forward_immediately() {
let stash = StashTable::with_bounds(usize::MAX, usize::MAX, changelog());
let backend = make_backend(leader_addr, stash.clone()).await;
- let handler = new_test_handler(Arc::clone(&backend));
+ let handler = new_test_handler(Arc::clone(&backend)).await;
// Stash a partition that isn't ours, then send a request for our
// partition. It should not park.
@@ -354,7 +442,7 @@ async fn stash_full_returns_unavailable_via_backend() {
// Tight cap: only one stashed message per partition.
let stash = StashTable::with_bounds(1, usize::MAX, changelog());
let backend = make_backend(leader_addr, stash.clone()).await;
- let handler = new_test_handler(Arc::clone(&backend));
+ let handler = new_test_handler(Arc::clone(&backend)).await;
let partition = backend.partition_for_person(person.team_id, person.id);
handler.begin_stash(partition, "leader-new").await.unwrap();
@@ -389,7 +477,7 @@ async fn back_to_back_handoffs_use_fresh_queue() {
let stash = StashTable::with_bounds(usize::MAX, usize::MAX, changelog());
let backend = make_backend(leader_addr, stash.clone()).await;
- let handler = new_test_handler(Arc::clone(&backend));
+ let handler = new_test_handler(Arc::clone(&backend)).await;
let partition = backend.partition_for_person(person.team_id, person.id);
@@ -451,7 +539,7 @@ async fn ordering_preserved_when_request_arrives_during_drain() {
let stash = StashTable::with_bounds(usize::MAX, usize::MAX, changelog());
let backend = make_backend(leader_addr, stash.clone()).await;
- let handler = new_test_handler(Arc::clone(&backend));
+ let handler = new_test_handler(Arc::clone(&backend)).await;
let partition = backend.partition_for_person(person.team_id, person.id);
handler.begin_stash(partition, "leader-new").await.unwrap();
@@ -522,7 +610,12 @@ async fn stash_wait_exceeded_returns_unavailable() {
let backend = make_backend(leader_addr, stash.clone()).await;
// Tight deadline — any request that ages past ~50ms in the stash
// gets failed fast on drain.
- let handler = RouterStashHandler::new(Arc::clone(&backend), Duration::from_millis(50), 4);
+ let handler = RouterStashHandler::new(
+ Arc::clone(&backend),
+ Duration::from_millis(50),
+ 4,
+ test_store().await,
+ );
let partition = backend.partition_for_person(person.team_id, person.id);
handler.begin_stash(partition, "leader-new").await.unwrap();
@@ -611,7 +704,12 @@ async fn fence_bounce_parks_requests_until_a_later_drain_delivers() {
let backend = make_backend(leader_addr, stash.clone()).await;
// Generous deadline so the deadline path can never produce a client
// error and mask a bounce that wrongly surfaced one.
- let handler = RouterStashHandler::new(Arc::clone(&backend), Duration::from_secs(60), 4);
+ let handler = RouterStashHandler::new(
+ Arc::clone(&backend),
+ Duration::from_secs(60),
+ 4,
+ test_store().await,
+ );
let partition = backend.partition_for_person(person.team_id, person.id);
handler.begin_stash(partition, "leader-new").await.unwrap();
@@ -668,7 +766,12 @@ async fn transport_failure_parks_requests_until_a_later_drain_delivers() {
let backend = make_backend(leader_addr, stash.clone()).await;
// Generous deadline so the deadline path can never produce a client
// error and mask a bounce that wrongly surfaced one.
- let handler = RouterStashHandler::new(Arc::clone(&backend), Duration::from_secs(60), 4);
+ let handler = RouterStashHandler::new(
+ Arc::clone(&backend),
+ Duration::from_secs(60),
+ 4,
+ test_store().await,
+ );
let partition = backend.partition_for_person(person.team_id, person.id);
handler.begin_stash(partition, "leader-new").await.unwrap();
@@ -789,7 +892,12 @@ async fn a_bouncing_live_write_parks_once_the_stash_opens() {
let stash = StashTable::with_bounds(usize::MAX, usize::MAX, changelog());
let backend = make_backend(leader_addr, stash.clone()).await;
- let handler = RouterStashHandler::new(Arc::clone(&backend), Duration::from_secs(60), 4);
+ let handler = RouterStashHandler::new(
+ Arc::clone(&backend),
+ Duration::from_secs(60),
+ 4,
+ test_store().await,
+ );
let partition = backend.partition_for_person(person.team_id, person.id);
let req = mk_request(person.team_id, person.id, "[email protected]");
@@ -832,7 +940,7 @@ async fn stashed_strong_read_observes_stashed_write() {
let stash = StashTable::with_bounds(usize::MAX, usize::MAX, changelog());
let backend = make_backend(leader_addr, stash.clone()).await;
- let handler = new_test_handler(Arc::clone(&backend));
+ let handler = new_test_handler(Arc::clone(&backend)).await;
let partition = backend.partition_for_person(person.team_id, person.id);
handler.begin_stash(partition, "leader-new").await.unwrap();
@@ -882,7 +990,12 @@ async fn stashed_read_past_deadline_returns_unavailable() {
let stash = StashTable::with_bounds(usize::MAX, usize::MAX, changelog());
let backend = make_backend(leader_addr, stash.clone()).await;
- let handler = RouterStashHandler::new(Arc::clone(&backend), Duration::from_millis(50), 4);
+ let handler = RouterStashHandler::new(
+ Arc::clone(&backend),
+ Duration::from_millis(50),
+ 4,
+ test_store().await,
+ );
let partition = backend.partition_for_person(person.team_id, person.id);
handler.begin_stash(partition, "leader-new").await.unwrap();
@@ -924,7 +1037,7 @@ async fn drain_converges_with_concurrent_arrivals() {
let stash = StashTable::with_bounds(usize::MAX, usize::MAX, changelog());
let backend = make_backend(leader_addr, stash.clone()).await;
- let handler = Arc::new(new_test_handler(Arc::clone(&backend)));
+ let handler = Arc::new(new_test_handler(Arc::clone(&backend)).await);
let partition = backend.partition_for_person(person.team_id, person.id);
handler.begin_stash(partition, "leader-new").await.unwrap();
@@ -1004,7 +1117,7 @@ async fn drain_cancellation_returns_promptly_and_puts_the_entry_back() {
let stash = StashTable::with_bounds(usize::MAX, usize::MAX, changelog());
let backend = make_backend(leader_addr, stash).await;
- let handler = new_test_handler(Arc::clone(&backend));
+ let handler = new_test_handler(Arc::clone(&backend)).await;
let (team_id, person_id) = (1, 42);
let partition = backend.partition_for_person(team_id, person_id);
@@ -1035,13 +1148,176 @@ async fn drain_cancellation_returns_promptly_and_puts_the_entry_back() {
.expect("drain task must not panic")
.expect("a cancelled drain is a pause, not an error");
- let handler = new_test_handler(Arc::clone(&backend));
+ let handler = new_test_handler(Arc::clone(&backend)).await;
assert!(
handler.stash_pending(partition),
"the abandoned entry must be put back for the next drain"
);
}
+/// A drain toward nobody (a partition no handoff or assignment names) still
+/// answers what outlived the stash wait and closes the freeze.
+#[tokio::test]
+async fn a_drain_toward_nobody_answers_expired_requests_and_ends_the_freeze() {
+ let changelog = changelog();
+ let backend = Arc::new(LeaderBackend::new(
+ Arc::new(RwLock::new(HashMap::new())),
+ Arc::new(|_: &str| None),
+ LeaderBackendConfig {
+ changelog: changelog.clone(),
+ timeout: Duration::from_secs(5),
+ num_channels: 1,
+ http2_windows: Http2Windows::default(),
+ },
+ StashTable::with_bounds(usize::MAX, usize::MAX, changelog),
+ ));
+ let handler = RouterStashHandler::new(
+ Arc::clone(&backend),
+ Duration::from_millis(50),
+ 4,
+ test_store().await,
+ );
+ let partition = backend.partition_for_person(1, 1);
+ handler.begin_stash(partition, "leader-gone").await.unwrap();
+ let backend_for_call = Arc::clone(&backend);
+ let in_flight =
+ tokio::spawn(
+ async move { forward(&backend_for_call, mk_request(1, 1, "[email protected]")).await },
+ );
+ tokio::time::sleep(Duration::from_millis(100)).await;
+
+ handler
+ .drain_stash(partition, "", CancellationToken::new())
+ .await
+ .unwrap();
+
+ let raw = tokio::time::timeout(Duration::from_secs(2), in_flight)
+ .await
+ .expect("the drain answers the expired request")
+ .unwrap();
+ assert_eq!(
+ decode_response::<UpdatePersonPropertiesResponse>(raw)
+ .await
+ .err(),
+ Some(Code::Unavailable)
+ );
+ assert!(!handler.stash_pending(partition));
+}
+
+/// A backend whose table sends every partition below `partitions` to
+/// `leader-0` at `leader_addr`.
+fn backend_routing_all(
+ leader_addr: std::net::SocketAddr,
+ partitions: u32,
+ changelog: LiveChangelog,
+) -> Arc<LeaderBackend> {
+ let leader_url = format!("http://{leader_addr}");
+ Arc::new(LeaderBackend::new(
+ Arc::new(RwLock::new(
+ (0..partitions)
+ .map(|partition| (partition, "leader-0".to_string()))
+ .collect::<HashMap<_, _>>(),
+ )),
+ Arc::new(move |_: &str| Some(leader_url.clone())),
+ LeaderBackendConfig {
+ changelog: changelog.clone(),
+ timeout: Duration::from_secs(5),
+ num_channels: 1,
+ http2_windows: Http2Windows::default(),
+ },
+ StashTable::with_bounds(usize::MAX, usize::MAX, changelog),
+ ))
+}
+
+/// A drain that beats the config watch reads the config first and takes the right keys.
+#[tokio::test]
+async fn a_drain_adopts_the_config_from_etcd_first() {
+ let person_id = moving_person();
+ let person = Person {
+ id: person_id,
+ team_id: 1,
+ ..create_test_person()
+ };
+ let leader_addr = start_test_leader(TestLeaderService::new().with_person(person)).await;
+ let backend = backend_routing_all(leader_addr, NUM_PARTITIONS * 2, changelog());
+ let store = test_store().await;
+ store.put_changelog_config(&grown_config()).await.unwrap();
+ let handler = RouterStashHandler::new(Arc::clone(&backend), TEST_MAX_STASH_WAIT, 1, store);
+ handler
+ .begin_stash(
+ partition_for_person(1, person_id, NUM_PARTITIONS),
+ "leader-0",
+ )
+ .await
+ .unwrap();
+ let backend_for_call = Arc::clone(&backend);
+ let in_flight = tokio::spawn(async move {
+ forward(&backend_for_call, mk_request(1, person_id, "[email protected]")).await
+ });
+ tokio::time::sleep(Duration::from_millis(50)).await;
+ assert!(!in_flight.is_finished(), "the write parks");
+
+ let new_partition = partition_for_person(1, person_id, NUM_PARTITIONS * 2);
+ handler
+ .drain_stash(new_partition, "leader-0", CancellationToken::new())
+ .await
+ .unwrap();
+
+ let raw = tokio::time::timeout(Duration::from_secs(2), in_flight)
+ .await
+ .expect("the drain delivers the parked write")
+ .unwrap();
+ assert!(decode_response::<UpdatePersonPropertiesResponse>(raw)
+ .await
+ .is_ok());
+}
+
+/// The config read is best effort: when it fails, the drain still runs under
+/// the count the router holds.
+#[tokio::test]
+async fn a_drain_runs_when_its_config_read_fails() {
+ let person = create_test_person();
+ let leader_addr = start_test_leader(TestLeaderService::new().with_person(person.clone())).await;
+ let backend = backend_routing_all(leader_addr, NUM_PARTITIONS, changelog());
+ // Nothing listens on port 1, so the read fails.
+ let unreachable = EtcdStore::connect(StoreConfig {
+ endpoints: vec!["http://127.0.0.1:1".to_string()],
+ prefix: "/stash-test-unreachable/".to_string(),
+ })
+ .await
+ .expect("the etcd client connects lazily");
+ let handler = RouterStashHandler::new(
+ Arc::clone(&backend),
+ TEST_MAX_STASH_WAIT,
+ 1,
+ Arc::new(PersonhogStore::new(unreachable)),
+ );
+ let partition = backend.partition_for_person(person.team_id, person.id);
+ handler.begin_stash(partition, "leader-0").await.unwrap();
+ let backend_for_call = Arc::clone(&backend);
+ let in_flight = tokio::spawn(async move {
+ forward(
+ &backend_for_call,
+ mk_request(person.team_id, person.id, "[email protected]"),
+ )
+ .await
+ });
+ tokio::time::sleep(Duration::from_millis(50)).await;
+
+ handler
+ .drain_stash(partition, "leader-0", CancellationToken::new())
+ .await
+ .unwrap();
+
+ let raw = tokio::time::timeout(Duration::from_secs(2), in_flight)
+ .await
+ .expect("the drain delivers the parked write")
+ .unwrap();
+ assert!(decode_response::<UpdatePersonPropertiesResponse>(raw)
+ .await
+ .is_ok());
+}
+
/// A person whose partition differs between the current and the grown count.
fn moving_person() -> i64 {
(1..)
@@ -1082,7 +1358,12 @@ async fn a_flip_in_etcd_moves_a_parked_key_to_its_new_partition() {
NUM_PARTITIONS * 2,
)
.await;
- let handler = RouterStashHandler::new(Arc::clone(&backend), Duration::from_secs(60), 4);
+ let handler = RouterStashHandler::new(
+ Arc::clone(&backend),
+ Duration::from_secs(60),
+ 4,
+ Arc::clone(&store),
+ );
let old_partition = partition_for_person(1, person.id, NUM_PARTITIONS);
let new_partition = partition_for_person(1, person.id, NUM_PARTITIONS * 2);
diff --git a/rust/personhog-stateright/src/model.rs b/rust/personhog-stateright/src/model.rs
index 9e3a857ed3e7..21a8b4714ba2 100644
--- a/rust/personhog-stateright/src/model.rs
+++ b/rust/personhog-stateright/src/model.rs
@@ -55,6 +55,7 @@ fn production_handoff(p: Partition, h: &Handoff) -> HandoffState {
freeze_quorum_ref: None,
created_at_ms: 0,
phase_entered_at_ms: 0,
+ held_by: None,
config_epoch: 0,
}
}
diff --git a/rust/personhog-writer/Cargo.toml b/rust/personhog-writer/Cargo.toml
index b5bb4f39afe5..9b5525c16e98 100644
--- a/rust/personhog-writer/Cargo.toml
+++ b/rust/personhog-writer/Cargo.toml
@@ -26,6 +26,7 @@ rdkafka = { workspace = true }
serde_json = { workspace = true }
sqlx = { workspace = true }
tokio = { workspace = true }
+tokio-util = { workspace = true }
tracing = { workspace = true }
tracing-subscriber = { workspace = true }
uuid = { workspace = true }
diff --git a/rust/personhog-writer/README.md b/rust/personhog-writer/README.md
index 3d19597b83da..d97b0843c8ed 100644
--- a/rust/personhog-writer/README.md
+++ b/rust/personhog-writer/README.md
@@ -137,7 +137,6 @@ Consumer lag per partition is monitored externally via KMinion (`kminion_kafka_c
| `KAFKA_CLIENT_ID` | (empty) | Pod name for static group membership |
| `KAFKA_CLIENT_RACK` | (empty) | Rack ID for rack-aware consumption |
| `KAFKA_CONSUMER_GROUP` | `personhog-writer` | Consumer group ID |
-| `KAFKA_CONSUMER_OFFSET_RESET` | `earliest` | Offset reset policy |
| `ETCD_ENDPOINTS` | `http://localhost:2379` | etcd endpoints; the topic comes from `config/changelog` |
| `ETCD_PREFIX` | `/personhog/` | etcd key prefix |
| `DATABASE_URL` | (required) | Postgres connection string |
diff --git a/rust/personhog-writer/src/buffer.rs b/rust/personhog-writer/src/buffer.rs
index 07dd59d55d09..02fce21ec0ba 100644
--- a/rust/personhog-writer/src/buffer.rs
+++ b/rust/personhog-writer/src/buffer.rs
@@ -1,14 +1,17 @@
use std::collections::hash_map::Entry;
use std::collections::{HashMap, HashSet};
+use std::sync::Arc;
use metrics::counter;
use personhog_proto::personhog::types::v1::Person;
+use crate::kafka::Source;
+
/// A buffered person plus the partition its latest message arrived on, so
/// drains can select whole partitions.
pub struct BufferedPerson {
pub person: Person,
- pub partition: i32,
+ pub source: Source,
}
/// A partition-complete slice of the buffer, ready to flush. Offsets cover
@@ -17,7 +20,7 @@ pub struct BufferedPerson {
pub struct DrainedBatch {
pub persons: Vec<BufferedPerson>,
/// Max offset seen per drained partition.
- pub offsets: HashMap<i32, i64>,
+ pub offsets: HashMap<Source, i64>,
/// Oldest message timestamp across the drained partitions (millis
/// since epoch), for end-to-end latency measurement.
pub oldest_message_ts_ms: Option<i64>,
@@ -28,9 +31,9 @@ pub struct DrainedBatch {
pub struct PersonBuffer {
entries: HashMap<(i64, i64), BufferedPerson>,
/// Max offset seen per partition for offset commits.
- offsets: HashMap<i32, i64>,
+ offsets: HashMap<Source, i64>,
/// Oldest message timestamp per partition with buffered entries.
- oldest_ts_ms: HashMap<i32, i64>,
+ oldest_ts_ms: HashMap<Source, i64>,
capacity: usize,
}
@@ -47,9 +50,9 @@ impl PersonBuffer {
/// Insert a person into the buffer. Later messages for the same person
/// overwrite earlier ones -- Kafka partition ordering guarantees the
/// latest version always arrives last.
- pub fn insert(&mut self, person: Person, partition: i32, offset: i64, ts_ms: Option<i64>) {
+ pub fn insert(&mut self, person: Person, source: Source, offset: i64, ts_ms: Option<i64>) {
self.offsets
- .entry(partition)
+ .entry(source.clone())
.and_modify(|o| {
if offset > *o {
*o = offset;
@@ -59,7 +62,7 @@ impl PersonBuffer {
if let Some(ts_ms) = ts_ms {
self.oldest_ts_ms
- .entry(partition)
+ .entry(source.clone())
.and_modify(|t| {
if ts_ms < *t {
*t = ts_ms;
@@ -71,11 +74,11 @@ impl PersonBuffer {
let key = (person.team_id, person.id);
match self.entries.entry(key) {
Entry::Occupied(mut e) => {
- e.insert(BufferedPerson { person, partition });
+ e.insert(BufferedPerson { person, source });
counter!("personhog_writer_messages_deduped_total").increment(1);
}
Entry::Vacant(e) => {
- e.insert(BufferedPerson { person, partition });
+ e.insert(BufferedPerson { person, source });
}
}
}
@@ -93,18 +96,18 @@ impl PersonBuffer {
}
/// Get the current offset for a partition, if tracked.
- pub fn partition_offset(&self, partition: i32) -> Option<i64> {
- self.offsets.get(&partition).copied()
+ pub fn source_offset(&self, source: &Source) -> Option<i64> {
+ self.offsets.get(source).copied()
}
- pub fn remove_partitions(&mut self, partitions: &[i32]) -> usize {
- let revoked: HashSet<i32> = partitions.iter().copied().collect();
+ pub fn remove_sources(&mut self, sources: &[Source]) -> usize {
+ let revoked: HashSet<&Source> = sources.iter().collect();
let before = self.entries.len();
self.entries
- .retain(|_, buffered| !revoked.contains(&buffered.partition));
- for partition in &revoked {
- self.offsets.remove(partition);
- self.oldest_ts_ms.remove(partition);
+ .retain(|_, buffered| !revoked.contains(&buffered.source));
+ for source in &revoked {
+ self.offsets.remove(*source);
+ self.oldest_ts_ms.remove(*source);
}
before - self.entries.len()
}
@@ -119,36 +122,41 @@ impl PersonBuffer {
return None;
}
- let mut keys_by_partition: HashMap<i32, Vec<(i64, i64)>> = HashMap::new();
+ let mut keys_by_source: HashMap<Source, Vec<(i64, i64)>> = HashMap::new();
for (key, buffered) in &self.entries {
- keys_by_partition
- .entry(buffered.partition)
+ keys_by_source
+ .entry(buffered.source.clone())
.or_default()
.push(*key);
}
// Oldest-first keeps the e2e latency tail short; the partition id
// tiebreak makes drain order deterministic.
- let mut partitions: Vec<i32> = keys_by_partition.keys().copied().collect();
- partitions
- .sort_unstable_by_key(|p| (self.oldest_ts_ms.get(p).copied().unwrap_or(i64::MAX), *p));
+ let mut sources: Vec<Source> = keys_by_source.keys().cloned().collect();
+ sources.sort_unstable_by_key(|s| {
+ (
+ self.oldest_ts_ms.get(s).copied().unwrap_or(i64::MAX),
+ s.partition,
+ Arc::clone(&s.topic),
+ )
+ });
let mut persons = Vec::new();
let mut offsets = HashMap::new();
let mut oldest_message_ts_ms: Option<i64> = None;
- for partition in partitions {
+ for source in sources {
if persons.len() >= max_rows {
break;
}
- for key in &keys_by_partition[&partition] {
+ for key in &keys_by_source[&source] {
let buffered = self.entries.remove(key).expect("key collected above");
persons.push(buffered);
}
- if let Some(offset) = self.offsets.remove(&partition) {
- offsets.insert(partition, offset);
+ if let Some(offset) = self.offsets.remove(&source) {
+ offsets.insert(source.clone(), offset);
}
- if let Some(ts) = self.oldest_ts_ms.remove(&partition) {
+ if let Some(ts) = self.oldest_ts_ms.remove(&source) {
oldest_message_ts_ms = Some(oldest_message_ts_ms.map_or(ts, |cur| cur.min(ts)));
}
}
@@ -165,6 +173,13 @@ impl PersonBuffer {
mod tests {
use super::*;
+ fn src(partition: i32) -> Source {
+ Source {
+ topic: Arc::from("personhog_updates"),
+ partition,
+ }
+ }
+
fn make_person(team_id: i64, person_id: i64, version: i64) -> Person {
Person {
id: person_id,
@@ -185,14 +200,14 @@ mod tests {
#[test]
fn insert_and_drain() {
let mut buf = PersonBuffer::new(100);
- buf.insert(make_person(1, 42, 1), 0, 0, None);
- buf.insert(make_person(1, 43, 1), 0, 1, None);
+ buf.insert(make_person(1, 42, 1), src(0), 0, None);
+ buf.insert(make_person(1, 43, 1), src(0), 1, None);
assert_eq!(buf.len(), 2);
let batch = buf.drain_up_to(usize::MAX).unwrap();
assert_eq!(batch.persons.len(), 2);
- assert_eq!(batch.offsets[&0], 1);
+ assert_eq!(batch.offsets[&src(0)], 1);
assert_eq!(buf.len(), 0);
assert!(buf.drain_up_to(usize::MAX).is_none());
}
@@ -200,9 +215,9 @@ mod tests {
#[test]
fn dedup_keeps_latest_message() {
let mut buf = PersonBuffer::new(100);
- buf.insert(make_person(1, 42, 1), 0, 0, None);
- buf.insert(make_person(1, 42, 2), 0, 1, None);
- buf.insert(make_person(1, 42, 3), 0, 2, None);
+ buf.insert(make_person(1, 42, 1), src(0), 0, None);
+ buf.insert(make_person(1, 42, 2), src(0), 1, None);
+ buf.insert(make_person(1, 42, 3), src(0), 2, None);
assert_eq!(buf.len(), 1);
@@ -215,37 +230,37 @@ mod tests {
let mut buf = PersonBuffer::new(2);
assert!(!buf.is_full());
- buf.insert(make_person(1, 1, 1), 0, 0, None);
+ buf.insert(make_person(1, 1, 1), src(0), 0, None);
assert!(!buf.is_full());
- buf.insert(make_person(1, 2, 1), 0, 1, None);
+ buf.insert(make_person(1, 2, 1), src(0), 1, None);
assert!(buf.is_full());
}
#[test]
fn tracks_max_offset_per_partition() {
let mut buf = PersonBuffer::new(100);
- buf.insert(make_person(1, 1, 1), 0, 10, None);
- buf.insert(make_person(1, 2, 1), 0, 5, None);
- buf.insert(make_person(1, 3, 1), 1, 3, None);
+ buf.insert(make_person(1, 1, 1), src(0), 10, None);
+ buf.insert(make_person(1, 2, 1), src(0), 5, None);
+ buf.insert(make_person(1, 3, 1), src(1), 3, None);
- assert_eq!(buf.partition_offset(0), Some(10));
- assert_eq!(buf.partition_offset(1), Some(3));
- assert_eq!(buf.partition_offset(2), None);
+ assert_eq!(buf.source_offset(&src(0)), Some(10));
+ assert_eq!(buf.source_offset(&src(1)), Some(3));
+ assert_eq!(buf.source_offset(&src(2)), None);
let batch = buf.drain_up_to(usize::MAX).unwrap();
- assert_eq!(batch.offsets[&0], 10);
- assert_eq!(batch.offsets[&1], 3);
+ assert_eq!(batch.offsets[&src(0)], 10);
+ assert_eq!(batch.offsets[&src(1)], 3);
}
#[test]
fn capped_drain_takes_whole_partitions_and_only_their_offsets() {
let mut buf = PersonBuffer::new(100);
// Partition 0 is older (ts 100), partition 1 newer (ts 200).
- buf.insert(make_person(1, 1, 1), 0, 10, Some(100));
- buf.insert(make_person(1, 2, 1), 0, 11, Some(150));
- buf.insert(make_person(1, 3, 1), 1, 20, Some(200));
- buf.insert(make_person(1, 4, 1), 1, 21, Some(250));
+ buf.insert(make_person(1, 1, 1), src(0), 10, Some(100));
+ buf.insert(make_person(1, 2, 1), src(0), 11, Some(150));
+ buf.insert(make_person(1, 3, 1), src(1), 20, Some(200));
+ buf.insert(make_person(1, 4, 1), src(1), 21, Some(250));
let batch = buf.drain_up_to(2).unwrap();
assert_eq!(batch.persons.len(), 2);
@@ -254,17 +269,17 @@ mod tests {
.iter()
.all(|p| p.person.id == 1 || p.person.id == 2));
assert_eq!(batch.offsets.len(), 1);
- assert_eq!(batch.offsets[&0], 11);
+ assert_eq!(batch.offsets[&src(0)], 11);
assert_eq!(batch.oldest_message_ts_ms, Some(100));
// Partition 1 stays fully buffered, offset uncommitted.
assert_eq!(buf.len(), 2);
- assert_eq!(buf.partition_offset(1), Some(21));
- assert_eq!(buf.partition_offset(0), None);
+ assert_eq!(buf.source_offset(&src(1)), Some(21));
+ assert_eq!(buf.source_offset(&src(0)), None);
let rest = buf.drain_up_to(2).unwrap();
assert_eq!(rest.persons.len(), 2);
- assert_eq!(rest.offsets[&1], 21);
+ assert_eq!(rest.offsets[&src(1)], 21);
assert_eq!(rest.oldest_message_ts_ms, Some(200));
assert!(buf.is_empty());
}
@@ -273,37 +288,37 @@ mod tests {
fn capped_drain_never_splits_a_partition() {
let mut buf = PersonBuffer::new(100);
for id in 1..=5 {
- buf.insert(make_person(1, id, 1), 0, id, None);
+ buf.insert(make_person(1, id, 1), src(0), id, None);
}
let batch = buf.drain_up_to(2).unwrap();
assert_eq!(batch.persons.len(), 5);
- assert_eq!(batch.offsets[&0], 5);
+ assert_eq!(batch.offsets[&src(0)], 5);
assert!(buf.is_empty());
}
#[test]
fn capped_drain_prefers_oldest_partition() {
let mut buf = PersonBuffer::new(100);
- buf.insert(make_person(1, 1, 1), 3, 0, Some(500));
- buf.insert(make_person(1, 2, 1), 7, 0, Some(100));
+ buf.insert(make_person(1, 1, 1), src(3), 0, Some(500));
+ buf.insert(make_person(1, 2, 1), src(7), 0, Some(100));
let batch = buf.drain_up_to(1).unwrap();
assert_eq!(batch.persons[0].person.id, 2);
assert_eq!(batch.offsets.len(), 1);
- assert_eq!(batch.offsets[&7], 0);
+ assert_eq!(batch.offsets[&src(7)], 0);
}
#[test]
- fn remove_partitions_drops_rows_offsets_and_timestamps() {
+ fn remove_sources_drops_rows_offsets_and_timestamps() {
let mut buf = PersonBuffer::new(100);
- buf.insert(make_person(1, 1, 1), 0, 10, Some(100));
- buf.insert(make_person(1, 2, 1), 1, 20, Some(50));
- buf.insert(make_person(1, 3, 1), 1, 21, Some(60));
+ buf.insert(make_person(1, 1, 1), src(0), 10, Some(100));
+ buf.insert(make_person(1, 2, 1), src(1), 20, Some(50));
+ buf.insert(make_person(1, 3, 1), src(1), 21, Some(60));
- assert_eq!(buf.remove_partitions(&[1]), 2);
+ assert_eq!(buf.remove_sources(&[src(1)]), 2);
assert_eq!(buf.len(), 1);
- assert_eq!(buf.partition_offset(1), None);
+ assert_eq!(buf.source_offset(&src(1)), None);
let batch = buf.drain_up_to(usize::MAX).unwrap();
assert_eq!(
@@ -314,7 +329,7 @@ mod tests {
.collect::<Vec<_>>(),
vec![1]
);
- assert_eq!(batch.offsets, HashMap::from([(0, 10)]));
+ assert_eq!(batch.offsets, HashMap::from([(src(0), 10)]));
assert_eq!(batch.oldest_message_ts_ms, Some(100));
}
}
diff --git a/rust/personhog-writer/src/config.rs b/rust/personhog-writer/src/config.rs
index d8852efdc20f..0cee824d0f70 100644
--- a/rust/personhog-writer/src/config.rs
+++ b/rust/personhog-writer/src/config.rs
@@ -12,9 +12,6 @@ pub struct Config {
#[envconfig(default = "personhog-writer")]
pub kafka_consumer_group: String,
- #[envconfig(default = "earliest")]
- pub kafka_consumer_offset_reset: String,
-
// ── Postgres ─────────────────────────────────────────────────
pub database_url: String,
diff --git a/rust/personhog-writer/src/consumer.rs b/rust/personhog-writer/src/consumer.rs
index c6b765c755db..d79f4a4bd156 100644
--- a/rust/personhog-writer/src/consumer.rs
+++ b/rust/personhog-writer/src/consumer.rs
@@ -6,17 +6,18 @@ use lifecycle::Handle;
use metrics::{counter, gauge, histogram};
use personhog_proto::personhog::types::v1::Person;
use prost::Message;
+use rdkafka::error::KafkaError;
use rdkafka::message::Message as KafkaMessage;
use tokio::sync::mpsc;
use tracing::{error, info, warn};
use crate::buffer::{BufferedPerson, PersonBuffer};
-use crate::kafka::PersonConsumer;
+use crate::kafka::{PersonConsumer, Source};
/// Batch of persons and their Kafka offsets, sent from consumer to writer.
pub struct FlushBatch {
pub persons: Vec<BufferedPerson>,
- pub offsets: HashMap<i32, i64>,
+ pub offsets: HashMap<Source, i64>,
/// Timestamp of the oldest Kafka message in this batch (millis since epoch).
/// Used to compute end-to-end latency from ingestion to PG commit.
pub oldest_message_ts_ms: Option<i64>,
@@ -31,7 +32,7 @@ fn drop_revoked_partitions(consumer: &PersonConsumer, lanes: &mut [Lane]) {
}
let dropped: usize = lanes
.iter_mut()
- .map(|lane| lane.buffer.remove_partitions(&revoked))
+ .map(|lane| lane.buffer.remove_sources(&revoked))
.sum();
counter!("personhog_writer_partitions_revoked_total").increment(revoked.len() as u64);
counter!("personhog_writer_revoked_rows_dropped_total", "stage" => "buffer")
@@ -61,6 +62,8 @@ pub struct ConsumerTask {
flush_interval: Duration,
flush_buffer_size: usize,
handle: Handle,
+ /// The last message's topic, shared by every row from it.
+ topic: Arc<str>,
}
impl ConsumerTask {
@@ -99,15 +102,30 @@ impl ConsumerTask {
flush_tx,
})
.collect();
+ let topic = Arc::from(consumer.topic().as_str());
Self {
consumer,
lanes,
flush_interval,
flush_buffer_size,
handle,
+ topic,
}
}
+}
+
+/// Rows share one allocation per topic; the cache is the last topic seen.
+fn source_from(cache: &mut Arc<str>, topic: &str, partition: i32) -> Source {
+ if &**cache != topic {
+ *cache = Arc::from(topic);
+ }
+ Source {
+ topic: Arc::clone(cache),
+ partition,
+ }
+}
+impl ConsumerTask {
pub async fn run(mut self) {
let _guard = self.handle.process_scope();
let mut flush_timer = tokio::time::interval(self.flush_interval);
@@ -160,6 +178,8 @@ impl ConsumerTask {
let partition = borrowed_msg.partition();
let offset = borrowed_msg.offset();
let ts_ms = borrowed_msg.timestamp().to_millis();
+ let source =
+ source_from(&mut self.topic, borrowed_msg.topic(), partition);
drop_revoked_partitions(&self.consumer, &mut self.lanes);
@@ -170,7 +190,7 @@ impl ConsumerTask {
.increment(1);
let idx = (partition as usize) % self.lanes.len();
let lane = &mut self.lanes[idx];
- lane.buffer.insert(person, partition, offset, ts_ms);
+ lane.buffer.insert(person, source, offset, ts_ms);
// Size-triggered flushes never block the
// consume loop: if this lane's writer is
@@ -198,6 +218,11 @@ impl ConsumerTask {
}
}
}
+ Err(KafkaError::PartitionEOF(partition)) => {
+ if self.consumer.is_draining() {
+ self.flush_partition_end(partition).await;
+ }
+ }
Err(e) => {
counter!("personhog_writer_kafka_errors_total").increment(1);
warn!(error = %e, "Kafka recv error");
@@ -222,6 +247,18 @@ impl ConsumerTask {
info!("Consumer task stopped");
}
+ /// A sealed partition's tail flushes at once instead of waiting out the timer.
+ async fn flush_partition_end(&mut self, partition: i32) {
+ let idx = (partition as usize) % self.lanes.len();
+ while !self.lanes[idx].buffer.is_empty() {
+ counter!("personhog_writer_flushes_by_trigger_total", "trigger" => "partition_end")
+ .increment(1);
+ self.send_flush(idx).await;
+ }
+ counter!("personhog_writer_partition_ends_total").increment(1);
+ info!(partition, "reached the end of a sealed partition");
+ }
+
/// Drain up to `flush_buffer_size` rows (whole partitions only) from a
/// lane's buffer into a batch. Offset and buffer-size gauges are
/// recorded here, per flush — updating them per message is too
@@ -231,10 +268,10 @@ impl ConsumerTask {
let lane = &mut self.lanes[idx];
let batch = lane.buffer.drain_up_to(self.flush_buffer_size)?;
- for (partition, offset) in &batch.offsets {
+ for (source, offset) in &batch.offsets {
gauge!(
"personhog_writer_partition_offset",
- "partition" => partition.to_string()
+ "partition" => source.partition.to_string()
)
.set(*offset as f64);
}
diff --git a/rust/personhog-writer/src/follow.rs b/rust/personhog-writer/src/follow.rs
new file mode 100644
index 000000000000..8e87a2557d5e
--- /dev/null
+++ b/rust/personhog-writer/src/follow.rs
@@ -0,0 +1,69 @@
+//! The subscription follows `config/changelog`: the oldest generation the
+//! group has not drained first, then the live topic.
+
+use std::collections::HashSet;
+use std::future::Future;
+
+use metrics::counter;
+use personhog_coordination::types::ChangelogConfig;
+use tracing::info;
+
+use crate::kafka::PersonConsumer;
+
+/// Chooses the subscription. A generation once proven drained is never chosen
+/// again, so a slow or failed probe cannot send the writer back to it.
+#[derive(Default)]
+pub struct Follower {
+ drained: HashSet<String>,
+}
+
+impl Follower {
+ /// The oldest draining generation not proven drained, else the live topic.
+ pub async fn select_topic<F, Fut>(
+ &mut self,
+ changelog: &ChangelogConfig,
+ drained: F,
+ ) -> Result<String, String>
+ where
+ F: Fn(String) -> Fut,
+ Fut: Future<Output = Result<bool, String>>,
+ {
+ let mut generations = changelog.draining.clone();
+ generations.sort_by_key(|g| g.epoch);
+ for generation in generations {
+ if self.drained.contains(&generation.topic) {
+ continue;
+ }
+ if !drained(generation.topic.clone()).await? {
+ return Ok(generation.topic);
+ }
+ self.drained.insert(generation.topic);
+ }
+ Ok(changelog.topic.clone())
+ }
+
+ /// One pass of the follow loop. The draining flag follows the config before the
+ /// subscription is touched, so it holds whether or not the subscription moved.
+ pub async fn follow_once<F, Fut>(
+ &mut self,
+ consumer: &PersonConsumer,
+ changelog: &ChangelogConfig,
+ drained: F,
+ ) -> Result<(), String>
+ where
+ F: Fn(String) -> Fut,
+ Fut: Future<Output = Result<bool, String>>,
+ {
+ let wanted = self.select_topic(changelog, drained).await?;
+ consumer.set_draining(wanted != changelog.topic);
+ let subscribed = consumer.topic();
+ if wanted != subscribed {
+ consumer
+ .resubscribe(wanted.clone())
+ .map_err(|e| format!("re-subscribe to {wanted}: {e}"))?;
+ info!(from = %subscribed, to = %wanted, "moved the subscription");
+ counter!("personhog_writer_resubscriptions_total").increment(1);
+ }
+ Ok(())
+ }
+}
diff --git a/rust/personhog-writer/src/kafka.rs b/rust/personhog-writer/src/kafka.rs
index e6a43f908264..ae07952d9180 100644
--- a/rust/personhog-writer/src/kafka.rs
+++ b/rust/personhog-writer/src/kafka.rs
@@ -1,25 +1,69 @@
use std::collections::{HashMap, HashSet};
use std::sync::atomic::{AtomicBool, Ordering};
-use std::sync::{Arc, Mutex};
+use std::sync::{Arc, Mutex, RwLock};
use common_kafka::config::KafkaConfig;
use metrics::counter;
use rdkafka::consumer::{
BaseConsumer, CommitMode, Consumer, ConsumerContext, Rebalance, StreamConsumer,
};
+use rdkafka::error::KafkaError;
use rdkafka::message::BorrowedMessage;
+use rdkafka::types::RDKafkaErrorCode;
use rdkafka::{ClientConfig, ClientContext, TopicPartitionList};
+/// The changelog partition a row arrived on; a resubscribe changes the topic and
+/// keeps the numbers, so offsets, assignments and revocations carry both.
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct Source {
+ pub topic: Arc<str>,
+ pub partition: i32,
+}
+
// ── Consumer ────────────────────────────────────────────────────
+/// Broker address and transport settings alone.
+pub fn base_client_config(kafka: &KafkaConfig) -> ClientConfig {
+ let mut client_config = ClientConfig::new();
+ client_config.set("bootstrap.servers", &kafka.kafka_hosts);
+ if kafka.kafka_tls {
+ client_config
+ .set("security.protocol", "ssl")
+ .set("enable.ssl.certificate.verification", "false");
+ }
+ if !kafka.kafka_client_rack.is_empty() {
+ client_config.set("client.rack", &kafka.kafka_client_rack);
+ }
+ client_config
+}
+
+fn group_client_config(kafka: &KafkaConfig, consumer_group: &str) -> ClientConfig {
+ let mut client_config = base_client_config(kafka);
+ client_config
+ .set("group.id", consumer_group)
+ .set("enable.auto.commit", "false")
+ .set("enable.auto.offset.store", "false");
+ client_config
+}
+
/// Records the partitions each rebalance takes away, so the consume loop can
/// drop their buffered rows before the new owner starts reading them.
struct RebalanceContext {
- revoked: Arc<Mutex<Vec<i32>>>,
+ revoked: Arc<Mutex<Vec<Source>>>,
joined: Arc<AtomicBool>,
}
-impl ClientContext for RebalanceContext {}
+impl ClientContext for RebalanceContext {
+ /// End-of-partition is the drain's signal, not an error; the callback reports it as a global code.
+ fn error(&self, error: KafkaError, reason: &str) {
+ if !matches!(
+ error,
+ KafkaError::PartitionEOF(_) | KafkaError::Global(RDKafkaErrorCode::PartitionEOF)
+ ) {
+ tracing::error!(%error, reason, "librdkafka");
+ }
+ }
+}
impl ConsumerContext for RebalanceContext {
fn pre_rebalance(&self, _base_consumer: &BaseConsumer<Self>, rebalance: &Rebalance<'_>) {
@@ -27,7 +71,10 @@ impl ConsumerContext for RebalanceContext {
Rebalance::Assign(_) => self.joined.store(true, Ordering::SeqCst),
Rebalance::Revoke(partitions) => {
let mut revoked = self.revoked.lock().unwrap();
- revoked.extend(partitions.elements().iter().map(|e| e.partition()));
+ revoked.extend(partitions.elements().iter().map(|e| Source {
+ topic: Arc::from(e.topic()),
+ partition: e.partition(),
+ }));
}
Rebalance::Error(_) => {}
}
@@ -38,8 +85,10 @@ impl ConsumerContext for RebalanceContext {
/// interface for receiving messages and committing offsets.
pub struct PersonConsumer {
consumer: StreamConsumer<RebalanceContext>,
- topic: String,
- revoked: Arc<Mutex<Vec<i32>>>,
+ topic: RwLock<String>,
+ /// Set while the subscribed topic is a sealed generation.
+ draining: AtomicBool,
+ revoked: Arc<Mutex<Vec<Source>>>,
joined: Arc<AtomicBool>,
}
@@ -54,16 +103,12 @@ impl PersonConsumer {
pub fn from_config(
kafka: &KafkaConfig,
consumer_group: &str,
- offset_reset: &str,
topic: String,
) -> Result<Self, rdkafka::error::KafkaError> {
- let mut client_config = ClientConfig::new();
+ let mut client_config = group_client_config(kafka, consumer_group);
client_config
- .set("bootstrap.servers", &kafka.kafka_hosts)
- .set("group.id", consumer_group)
- .set("auto.offset.reset", offset_reset)
- .set("enable.auto.commit", "false")
- .set("enable.auto.offset.store", "false")
+ // A generation the group never committed on must be read from its first record.
+ .set("auto.offset.reset", "earliest")
// Only committed transactions: with the leader's epoch fencing
// on, aborted windows and zombie leftovers must never reach
// Postgres. Identical behavior on a non-transactional topic.
@@ -74,6 +119,8 @@ impl PersonConsumer {
// leader's derived broker transaction timeout (about 6s at
// the production lease).
.set("isolation.level", "read_committed")
+ // Lets the consume loop flush a sealed partition's tail as soon as it reaches the end.
+ .set("enable.partition.eof", "true")
// Cooperative-sticky: during scale events, only partitions that need
// to move are revoked. Non-moving partitions keep being consumed.
.set("partition.assignment.strategy", "cooperative-sticky");
@@ -88,16 +135,6 @@ impl PersonConsumer {
.set("group.instance.id", &kafka.kafka_client_id);
}
- if kafka.kafka_tls {
- client_config
- .set("security.protocol", "ssl")
- .set("enable.ssl.certificate.verification", "false");
- }
-
- if !kafka.kafka_client_rack.is_empty() {
- client_config.set("client.rack", &kafka.kafka_client_rack);
- }
-
Self::new(&client_config, topic)
}
@@ -114,24 +151,42 @@ impl PersonConsumer {
consumer.subscribe(&[&topic])?;
Ok(Self {
consumer,
- topic,
+ topic: RwLock::new(topic),
+ draining: AtomicBool::new(false),
revoked,
joined,
})
}
+ /// Only called once the group has drained the current topic.
+ pub fn resubscribe(&self, topic: String) -> Result<(), rdkafka::error::KafkaError> {
+ self.consumer.subscribe(&[&topic])?;
+ *self.topic.write().expect("topic lock poisoned") = topic;
+ Ok(())
+ }
+
+ pub fn set_draining(&self, draining: bool) {
+ self.draining.store(draining, Ordering::SeqCst);
+ }
+
+ pub fn is_draining(&self) -> bool {
+ self.draining.load(Ordering::SeqCst)
+ }
+
pub async fn recv(&self) -> Result<BorrowedMessage<'_>, rdkafka::error::KafkaError> {
self.consumer.recv().await
}
- pub fn take_revoked(&self) -> Vec<i32> {
+ pub fn take_revoked(&self) -> Vec<Source> {
std::mem::take(&mut *self.revoked.lock().unwrap())
}
/// `None` until the first assignment arrives: before that, an empty
/// assignment means the group is still forming, not that this pod owns
/// nothing.
- pub fn assigned_partitions(&self) -> Result<Option<HashSet<i32>>, rdkafka::error::KafkaError> {
+ pub fn assigned_partitions(
+ &self,
+ ) -> Result<Option<HashSet<Source>>, rdkafka::error::KafkaError> {
if !self.joined.load(Ordering::SeqCst) {
return Ok(None);
}
@@ -140,19 +195,28 @@ impl PersonConsumer {
.assignment()?
.elements()
.iter()
- .map(|e| e.partition())
+ .map(|e| Source {
+ topic: Arc::from(e.topic()),
+ partition: e.partition(),
+ })
.collect(),
))
}
- pub fn positions(&self) -> Result<HashMap<i32, i64>, rdkafka::error::KafkaError> {
+ pub fn positions(&self) -> Result<HashMap<Source, i64>, rdkafka::error::KafkaError> {
Ok(self
.consumer
.position()?
.elements()
.iter()
.filter_map(|e| match e.offset() {
- rdkafka::Offset::Offset(next) => Some((e.partition(), next)),
+ rdkafka::Offset::Offset(next) => Some((
+ Source {
+ topic: Arc::from(e.topic()),
+ partition: e.partition(),
+ },
+ next,
+ )),
_ => None,
})
.collect())
@@ -160,23 +224,27 @@ impl PersonConsumer {
pub fn commit_offsets(
&self,
- offsets: &HashMap<i32, i64>,
+ offsets: &HashMap<Source, i64>,
) -> Result<(), rdkafka::error::KafkaError> {
if offsets.is_empty() {
return Ok(());
}
- // A partition revoked while its batch was in flight belongs to
- // another pod now; committing it here would move that pod's offset.
+ // A partition revoked while its batch was in flight belongs to another pod
+ // now. The topic comes from the rows: a resubscribe reuses the numbers.
let assigned = self.assigned_partitions()?.unwrap_or_default();
let mut tpl = TopicPartitionList::new();
let mut skipped: u64 = 0;
- for (partition, offset) in offsets {
- if !assigned.contains(partition) {
+ for (source, offset) in offsets {
+ if !assigned.contains(source) {
skipped += 1;
continue;
}
- tpl.add_partition_offset(&self.topic, *partition, rdkafka::Offset::Offset(offset + 1))?;
+ tpl.add_partition_offset(
+ &source.topic,
+ source.partition,
+ rdkafka::Offset::Offset(offset + 1),
+ )?;
}
if skipped > 0 {
counter!("personhog_writer_offset_commits_skipped_total", "reason" => "unassigned")
@@ -190,7 +258,7 @@ impl PersonConsumer {
Ok(())
}
- pub fn topic(&self) -> &str {
- &self.topic
+ pub fn topic(&self) -> String {
+ self.topic.read().expect("topic lock poisoned").clone()
}
}
diff --git a/rust/personhog-writer/src/lib.rs b/rust/personhog-writer/src/lib.rs
index 3997678bd150..26a50e12fb60 100644
--- a/rust/personhog-writer/src/lib.rs
+++ b/rust/personhog-writer/src/lib.rs
@@ -2,6 +2,7 @@ pub mod buffer;
pub mod config;
pub mod consumer;
pub mod error;
+pub mod follow;
pub mod kafka;
pub mod pg;
pub mod store;
diff --git a/rust/personhog-writer/src/main.rs b/rust/personhog-writer/src/main.rs
index 6f2bcb812978..1db2b08c858d 100644
--- a/rust/personhog-writer/src/main.rs
+++ b/rust/personhog-writer/src/main.rs
@@ -1,3 +1,4 @@
+use std::ops::ControlFlow;
use std::sync::Arc;
use std::time::Duration;
@@ -7,9 +8,13 @@ use common_database::{get_pool_with_config, PoolConfig};
use common_metrics::{setup_metrics_routes_with_overrides, Matcher};
use envconfig::Envconfig;
use lifecycle::{ComponentOptions, Manager};
-use personhog_coordination::changelog::{load_changelog_config, CHANGELOG_BOOT_POLL_INTERVAL};
+use personhog_coordination::changelog::{
+ load_changelog_config, watch_changelog_config_loop, CHANGELOG_BOOT_POLL_INTERVAL,
+ CHANGELOG_RECONCILE_INTERVAL,
+};
use personhog_coordination::store::PersonhogStore;
use tokio::sync::mpsc;
+use tokio_util::sync::CancellationToken;
use tracing::level_filters::LevelFilter;
use tracing_subscriber::fmt;
use tracing_subscriber::layer::SubscriberExt;
@@ -18,7 +23,8 @@ use tracing_subscriber::EnvFilter;
use personhog_writer::config::Config;
use personhog_writer::consumer::ConsumerTask;
-use personhog_writer::kafka::PersonConsumer;
+use personhog_writer::follow::Follower;
+use personhog_writer::kafka::{base_client_config, PersonConsumer};
use personhog_writer::pg::PgStore;
use personhog_writer::store::PersonWriteStore;
use personhog_writer::writer::WriterTask;
@@ -57,8 +63,8 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
prefix: config.etcd_prefix.clone(),
})
.await?;
- let loaded =
- load_changelog_config(&PersonhogStore::new(store), CHANGELOG_BOOT_POLL_INTERVAL).await?;
+ let store = Arc::new(PersonhogStore::new(store));
+ let loaded = load_changelog_config(&store, CHANGELOG_BOOT_POLL_INTERVAL).await?;
if loaded.writer_group != config.kafka_consumer_group {
return Err(format!(
"config/changelog names writer group {} but KAFKA_CONSUMER_GROUP is {}",
@@ -71,6 +77,11 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
epoch = loaded.epoch,
"loaded changelog config"
);
+ // Oldest undrained generation first, so a restart mid-drain resumes it.
+ let mut follower = Follower::default();
+ let topic = follower
+ .select_topic(&loaded, |topic| drained(&config, topic))
+ .await?;
let mut manager = Manager::builder("personhog-writer")
.with_global_shutdown_timeout(Duration::from_secs(30))
@@ -100,6 +111,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
"metrics-server",
ComponentOptions::new().is_observability(true),
);
+ let changelog_handle = manager.register("changelog-config", ComponentOptions::new());
let readiness = manager.readiness_handler();
let liveness = manager.liveness_handler();
@@ -169,14 +181,13 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
});
}
- // Kafka consumer
let kafka_consumer = Arc::new(PersonConsumer::from_config(
&config.kafka,
&config.kafka_consumer_group,
- &config.kafka_consumer_offset_reset,
- loaded.topic.clone(),
+ topic.clone(),
)?);
- tracing::info!("Subscribed to Kafka topic: {}", loaded.topic);
+ kafka_consumer.set_draining(topic != loaded.topic);
+ tracing::info!(topic = %topic, draining = kafka_consumer.is_draining(), "Subscribed to Kafka topic");
// One statement-concurrency budget for the whole pod: every lane's
// chunk and per-row statements draw from it, so lanes can overlap
@@ -215,7 +226,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
// Consumer task
let consumer_task = ConsumerTask::new(
- kafka_consumer,
+ Arc::clone(&kafka_consumer),
lane_txs,
(config.buffer_capacity / lanes).max(1),
config.flush_interval(),
@@ -227,6 +238,71 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
consumer_task.run().await;
});
+ // The subscription follows the config; the drain is read from the group's commits.
+ {
+ let consumer = Arc::clone(&kafka_consumer);
+ let config = config.clone();
+ let handle = changelog_handle;
+ tokio::spawn(async move {
+ let _guard = handle.process_scope();
+ let cancel = CancellationToken::new();
+ let (config_tx, mut config_rx) = tokio::sync::watch::channel(loaded.clone());
+ let watch = watch_changelog_config_loop(
+ store,
+ loaded.clone(),
+ CHANGELOG_RECONCILE_INTERVAL,
+ cancel.clone(),
+ move |next| {
+ drop(config_tx.send(next.clone()));
+ ControlFlow::Continue(())
+ },
+ );
+ let probe = |topic: String| drained(&config, topic);
+ let follow = async {
+ loop {
+ let current = config_rx.borrow().clone();
+ if let Err(e) = follower.follow_once(&consumer, &current, &probe).await {
+ tracing::warn!(error = %e, "could not follow the changelog config; keeping the subscription");
+ }
+ let poll = if consumer.is_draining() {
+ DRAIN_POLL_INTERVAL
+ } else {
+ CHANGELOG_RECONCILE_INTERVAL
+ };
+ tokio::select! {
+ changed = config_rx.changed() => {
+ if changed.is_err() {
+ return;
+ }
+ }
+ _ = tokio::time::sleep(poll) => {}
+ }
+ }
+ };
+ tokio::select! {
+ _ = handle.shutdown_signal() => cancel.cancel(),
+ _ = watch => {}
+ _ = follow => {}
+ }
+ });
+ }
+
monitor_guard.wait().await?;
Ok(())
}
+
+/// How often a writer on a sealed generation re-checks the group's drain.
+/// Each check reads every partition of the generation under a probe group.
+const DRAIN_POLL_INTERVAL: Duration = Duration::from_secs(2);
+
+/// Whether the writer group has read every partition of `topic` to its end.
+async fn drained(config: &Config, topic: String) -> Result<bool, String> {
+ let base = base_client_config(&config.kafka);
+ let group = config.kafka_consumer_group.clone();
+ tokio::task::spawn_blocking(move || {
+ common_kafka::probe::group_drained(&base, &group, &topic, Duration::from_secs(10))
+ })
+ .await
+ .map_err(|e| format!("drain probe join: {e}"))?
+ .map_err(|e| format!("drain probe: {e}"))
+}
diff --git a/rust/personhog-writer/src/writer.rs b/rust/personhog-writer/src/writer.rs
index e479be276157..6440deed726c 100644
--- a/rust/personhog-writer/src/writer.rs
+++ b/rust/personhog-writer/src/writer.rs
@@ -12,28 +12,30 @@ use tokio::time::Instant;
use tracing::{debug, error, info, warn};
use crate::consumer::FlushBatch;
-use crate::kafka::PersonConsumer;
+use crate::kafka::{PersonConsumer, Source};
use crate::store::{BatchOutcome, PersonDb, PersonWriteStore};
/// The offset-commit and assignment surface the writer needs from Kafka.
/// Abstracted so the retry loop is testable without a broker.
pub trait OffsetCommitter: Send + Sync {
- fn commit_offsets(&self, offsets: &HashMap<i32, i64>)
- -> Result<(), rdkafka::error::KafkaError>;
+ fn commit_offsets(
+ &self,
+ offsets: &HashMap<Source, i64>,
+ ) -> Result<(), rdkafka::error::KafkaError>;
/// `None` while the consumer has not joined its group yet.
- fn assigned_partitions(&self) -> Result<Option<HashSet<i32>>, rdkafka::error::KafkaError>;
+ fn assigned_partitions(&self) -> Result<Option<HashSet<Source>>, rdkafka::error::KafkaError>;
}
impl OffsetCommitter for PersonConsumer {
fn commit_offsets(
&self,
- offsets: &HashMap<i32, i64>,
+ offsets: &HashMap<Source, i64>,
) -> Result<(), rdkafka::error::KafkaError> {
PersonConsumer::commit_offsets(self, offsets)
}
- fn assigned_partitions(&self) -> Result<Option<HashSet<i32>>, rdkafka::error::KafkaError> {
+ fn assigned_partitions(&self) -> Result<Option<HashSet<Source>>, rdkafka::error::KafkaError> {
PersonConsumer::assigned_partitions(self)
}
}
@@ -147,8 +149,8 @@ impl<D: PersonDb + 'static, C: OffsetCommitter + 'static> WriterTask<D, C> {
Ok(None) => {}
Ok(Some(assigned)) => {
let before = persons.len();
- persons.retain(|row| assigned.contains(&row.partition));
- offsets.retain(|partition, _| assigned.contains(partition));
+ persons.retain(|row| assigned.contains(&row.source));
+ offsets.retain(|source, _| assigned.contains(source));
let dropped = before - persons.len();
if dropped > 0 {
counter!("personhog_writer_revoked_rows_dropped_total", "stage" => "queued")
@@ -269,7 +271,7 @@ impl<D: PersonDb + 'static, C: OffsetCommitter + 'static> WriterTask<D, C> {
}
}
- fn finish(&mut self, rows: usize, offsets: &HashMap<i32, i64>, oldest_ts_ms: Option<i64>) {
+ fn finish(&mut self, rows: usize, offsets: &HashMap<Source, i64>, oldest_ts_ms: Option<i64>) {
self.consecutive_failures = 0;
self.liveness.report_healthy();
self.commit_and_record(offsets, oldest_ts_ms, rows);
@@ -277,7 +279,7 @@ impl<D: PersonDb + 'static, C: OffsetCommitter + 'static> WriterTask<D, C> {
fn commit_and_record(
&self,
- offsets: &HashMap<i32, i64>,
+ offsets: &HashMap<Source, i64>,
oldest_ts_ms: Option<i64>,
rows_written: usize,
) {
@@ -345,16 +347,23 @@ mod tests {
}
}
+ fn source(partition: i32) -> Source {
+ Source {
+ topic: Arc::from("personhog_updates"),
+ partition,
+ }
+ }
+
struct RecordingCommitter {
- commits: Mutex<Vec<HashMap<i32, i64>>>,
- assigned: Option<HashSet<i32>>,
+ commits: Mutex<Vec<HashMap<Source, i64>>>,
+ assigned: Option<HashSet<Source>>,
}
impl RecordingCommitter {
fn owning(partitions: &[i32]) -> Self {
Self {
commits: Mutex::default(),
- assigned: Some(partitions.iter().copied().collect()),
+ assigned: Some(partitions.iter().map(|p| source(*p)).collect()),
}
}
@@ -375,13 +384,15 @@ mod tests {
impl OffsetCommitter for RecordingCommitter {
fn commit_offsets(
&self,
- offsets: &HashMap<i32, i64>,
+ offsets: &HashMap<Source, i64>,
) -> Result<(), rdkafka::error::KafkaError> {
self.commits.lock().unwrap().push(offsets.clone());
Ok(())
}
- fn assigned_partitions(&self) -> Result<Option<HashSet<i32>>, rdkafka::error::KafkaError> {
+ fn assigned_partitions(
+ &self,
+ ) -> Result<Option<HashSet<Source>>, rdkafka::error::KafkaError> {
Ok(self.assigned.clone())
}
}
@@ -424,7 +435,7 @@ mod tests {
fn row(partition: i32, id: i64) -> BufferedPerson {
BufferedPerson {
- partition,
+ source: source(partition),
person: Person {
id,
team_id: 1,
@@ -437,7 +448,7 @@ mod tests {
fn batch(rows: i64) -> FlushBatch {
FlushBatch {
persons: (0..rows).map(|id| row(0, id)).collect(),
- offsets: HashMap::from([(0, rows - 1)]),
+ offsets: HashMap::from([(source(0), rows - 1)]),
oldest_message_ts_ms: None,
}
}
@@ -470,13 +481,13 @@ mod tests {
#[tokio::test(start_paused = true)]
async fn queued_batch_is_filtered_by_the_current_assignment() {
- let both: HashMap<i32, i64> = HashMap::from([(0, 1), (1, 4)]);
+ let both: HashMap<Source, i64> = HashMap::from([(source(0), 1), (source(1), 4)]);
let cases = vec![
(
"owns 0",
RecordingCommitter::owning(&[0]),
2,
- vec![HashMap::from([(0, 1)])],
+ vec![HashMap::from([(source(0), 1)])],
),
(
"owns both",
@@ -492,7 +503,7 @@ mod tests {
scripted_writer_with(committer, vec![]);
let mut queued = batch(2);
queued.persons.push(row(1, 7));
- queued.offsets.insert(1, 4);
+ queued.offsets.insert(source(1), 4);
let flow = writer.process_batch(queued).await;
diff --git a/rust/personhog-writer/tests/integration.rs b/rust/personhog-writer/tests/integration.rs
index 01808f0e04f7..c4030f812f62 100644
--- a/rust/personhog-writer/tests/integration.rs
+++ b/rust/personhog-writer/tests/integration.rs
@@ -9,10 +9,12 @@ use common::{
cleanup_team, create_local_kafka_producer, create_mock_kafka, create_test_pool, make_person,
unique_team_id, KAFKA_BOOTSTRAP, TARGET_TABLE, TOPIC,
};
+use personhog_coordination::types::{ChangelogConfig, Generation};
use personhog_proto::personhog::types::v1::Person;
use personhog_writer::buffer::BufferedPerson;
use personhog_writer::consumer::{ConsumerTask, FlushBatch};
-use personhog_writer::kafka::PersonConsumer;
+use personhog_writer::follow::Follower;
+use personhog_writer::kafka::{PersonConsumer, Source};
use personhog_writer::pg::PgStore;
use personhog_writer::store::{
BatchOutcome, PersonDb, PersonWriteStore, WriteError, WriteErrorKind,
@@ -24,11 +26,22 @@ use rdkafka::producer::FutureRecord;
use rdkafka::{ClientConfig, Offset, TopicPartitionList};
use tokio::sync::mpsc;
+fn source_on(topic: &str, partition: i32) -> Source {
+ Source {
+ topic: Arc::from(topic),
+ partition,
+ }
+}
+
+fn source(partition: i32) -> Source {
+ source_on(TOPIC, partition)
+}
+
fn rows(persons: Vec<Person>) -> Vec<BufferedPerson> {
persons
.into_iter()
.map(|person| BufferedPerson {
- partition: 0,
+ source: source(0),
person,
})
.collect()
@@ -723,8 +736,11 @@ async fn multi_lane_consumer_routes_partitions_and_flushes_independently() {
.await
.expect("lane 0 should flush on size")
.expect("lane 0 channel should be open");
- assert_eq!(batch0.offsets.keys().copied().collect::<Vec<_>>(), vec![0]);
- assert_eq!(batch0.offsets[&0], 3); // four messages, offsets 0..=3
+ assert_eq!(
+ batch0.offsets.keys().cloned().collect::<Vec<_>>(),
+ vec![source(0)]
+ );
+ assert_eq!(batch0.offsets[&source(0)], 3); // four messages, offsets 0..=3
let mut persons0 = batch0.persons;
persons0.sort_by_key(|p| p.person.id);
assert_eq!(
@@ -767,8 +783,11 @@ async fn multi_lane_consumer_routes_partitions_and_flushes_independently() {
.await
.expect("lane 1 should flush on size")
.expect("lane 1 channel should be open");
- assert_eq!(batch1.offsets.keys().copied().collect::<Vec<_>>(), vec![1]);
- assert_eq!(batch1.offsets[&1], 2); // three messages, offsets 0..=2
+ assert_eq!(
+ batch1.offsets.keys().cloned().collect::<Vec<_>>(),
+ vec![source(1)]
+ );
+ assert_eq!(batch1.offsets[&source(1)], 2); // three messages, offsets 0..=2
let mut ids1: Vec<i64> = batch1.persons.iter().map(|p| p.person.id).collect();
ids1.sort_unstable();
assert_eq!(ids1, vec![4, 5, 6]);
@@ -848,14 +867,15 @@ async fn capped_drains_commit_only_the_partitions_they_carry() {
.iter()
.map(|p| if p.person.id <= 3 { 0 } else { 1 })
.collect();
- let committed: std::collections::HashSet<i32> = batch.offsets.keys().copied().collect();
+ let committed: std::collections::HashSet<i32> =
+ batch.offsets.keys().map(|s| s.partition).collect();
assert_eq!(
committed, partitions_in_batch,
"a batch's offsets must cover exactly the partitions it drained"
);
- for (partition, offset) in &batch.offsets {
- let entry = max_offset_seen.entry(*partition).or_insert(-1);
+ for (source, offset) in &batch.offsets {
+ let entry = max_offset_seen.entry(source.partition).or_insert(-1);
*entry = (*entry).max(*offset);
}
seen_persons += batch.persons.len();
@@ -955,7 +975,7 @@ async fn revoked_partitions_are_dropped_and_their_offsets_stay_uncommitted() {
wait_until(Duration::from_secs(20), || {
first
.positions()
- .map(|p| p.get(&0) == Some(&2) && p.get(&1) == Some(&2))
+ .map(|p| p.get(&source(0)) == Some(&2) && p.get(&source(1)) == Some(&2))
.unwrap_or(false)
})
.await
@@ -981,7 +1001,7 @@ async fn revoked_partitions_are_dropped_and_their_offsets_stay_uncommitted() {
let mut flushed_partitions = HashSet::new();
while let Ok(Some(batch)) = tokio::time::timeout(Duration::from_secs(10), rx.recv()).await {
flushed_ids.extend(batch.persons.iter().map(|p| p.person.id));
- flushed_partitions.extend(batch.offsets.keys().copied());
+ flushed_partitions.extend(batch.offsets.keys().map(|s| s.partition));
}
flushed_ids.sort_unstable();
let kept_ids = if kept == 0 { vec![1, 2] } else { vec![3, 4] };
@@ -992,7 +1012,7 @@ async fn revoked_partitions_are_dropped_and_their_offsets_stay_uncommitted() {
assert_eq!(flushed_partitions, HashSet::from([kept]));
first
- .commit_offsets(&HashMap::from([(kept, 1), (revoked, 1)]))
+ .commit_offsets(&HashMap::from([(source(kept), 1), (source(revoked), 1)]))
.unwrap();
let mut probe = TopicPartitionList::new();
probe.add_partition(TOPIC, kept);
@@ -1015,6 +1035,165 @@ async fn revoked_partitions_are_dropped_and_their_offsets_stay_uncommitted() {
);
}
+const NEXT_TOPIC: &str = "personhog_updates_g2";
+
+fn plain_member_config(bootstrap: &str, group: &str) -> ClientConfig {
+ ClientConfig::new()
+ .set("bootstrap.servers", bootstrap)
+ .set("group.id", group)
+ .set("auto.offset.reset", "earliest")
+ .set("enable.auto.commit", "false")
+ .set("enable.auto.offset.store", "false")
+ .set("session.timeout.ms", "6000")
+ .set("heartbeat.interval.ms", "1000")
+ .clone()
+}
+
+/// After a resubscribe the assignment reuses the partition numbers, so an
+/// old-topic batch still in flight must never commit onto the next topic.
+#[tokio::test]
+async fn a_late_batch_never_commits_its_offsets_onto_the_next_topic() {
+ let (mock_cluster, _producer) = common::create_mock_kafka_with_partitions(2).await;
+ mock_cluster
+ .create_topic(NEXT_TOPIC, 2, 1)
+ .expect("create the next generation");
+ let bootstrap = mock_cluster.bootstrap_servers();
+ let member = plain_member_config(&bootstrap, "test-late-batch");
+
+ let consumer = Arc::new(PersonConsumer::new(&member, TOPIC.to_string()).unwrap());
+ // Keeps the client polled, so rebalances land.
+ {
+ let consumer = Arc::clone(&consumer);
+ tokio::spawn(async move {
+ loop {
+ drop(consumer.recv().await);
+ }
+ });
+ }
+ let assigned_to = |consumer: &PersonConsumer, topic: &str| {
+ consumer
+ .assigned_partitions()
+ .ok()
+ .flatten()
+ .is_some_and(|a| a == HashSet::from([source_on(topic, 0), source_on(topic, 1)]))
+ };
+ wait_until(Duration::from_secs(20), || assigned_to(&consumer, TOPIC))
+ .await
+ .expect("the member owns the first topic");
+
+ let probe: StreamConsumer = plain_member_config(&bootstrap, "test-late-batch")
+ .create()
+ .unwrap();
+ let mut both = TopicPartitionList::new();
+ for topic in [TOPIC, NEXT_TOPIC] {
+ both.add_partition(topic, 0);
+ both.add_partition(topic, 1);
+ }
+ let committed_on = |topic: &str, partition: i32| {
+ probe
+ .committed_offsets(both.clone(), Duration::from_secs(2))
+ .ok()
+ .and_then(|c| {
+ c.elements()
+ .iter()
+ .find(|e| e.topic() == topic && e.partition() == partition)
+ .and_then(|e| match e.offset() {
+ Offset::Offset(offset) => Some(offset),
+ _ => None,
+ })
+ })
+ };
+
+ consumer
+ .commit_offsets(&HashMap::from([(source(0), 1)]))
+ .unwrap();
+ wait_until(Duration::from_secs(10), || {
+ committed_on(TOPIC, 0) == Some(2)
+ })
+ .await
+ .expect("the first topic's commit lands");
+
+ consumer.resubscribe(NEXT_TOPIC.to_string()).unwrap();
+ wait_until(Duration::from_secs(20), || {
+ assigned_to(&consumer, NEXT_TOPIC)
+ })
+ .await
+ .expect("the member moves to the next topic");
+
+ // Commits are sequenced on the client: the live one landing proves the late one was handled.
+ consumer
+ .commit_offsets(&HashMap::from([(source(1), 5)]))
+ .unwrap();
+ consumer
+ .commit_offsets(&HashMap::from([(source_on(NEXT_TOPIC, 0), 0)]))
+ .unwrap();
+ wait_until(Duration::from_secs(10), || {
+ committed_on(NEXT_TOPIC, 0) == Some(1)
+ })
+ .await
+ .expect("the next topic's commit lands");
+
+ assert_eq!(
+ committed_on(NEXT_TOPIC, 1),
+ None,
+ "an old-topic batch must not commit onto the next topic"
+ );
+ assert_eq!(
+ committed_on(TOPIC, 1),
+ None,
+ "a batch for a partition no longer assigned is not committed"
+ );
+ assert_eq!(committed_on(TOPIC, 0), Some(2));
+}
+
+/// The draining flag follows the subscribed topic, not the move: a writer
+/// already on the topic when it was sealed raises it too.
+#[tokio::test]
+async fn the_draining_flag_follows_the_subscribed_topic_not_the_move() {
+ let (mock_cluster, _producer) = common::create_mock_kafka_with_partitions(1).await;
+ mock_cluster
+ .create_topic(NEXT_TOPIC, 1, 1)
+ .expect("create the next generation");
+ let member = plain_member_config(&mock_cluster.bootstrap_servers(), "test-follow");
+ let consumer = PersonConsumer::new(&member, TOPIC.to_string()).unwrap();
+ let flipped = ChangelogConfig {
+ topic: NEXT_TOPIC.to_string(),
+ partitions: 1,
+ epoch: 2,
+ writer_group: "test-follow".to_string(),
+ draining: vec![Generation {
+ topic: TOPIC.to_string(),
+ partitions: 1,
+ epoch: 1,
+ }],
+ };
+
+ let mut follower = Follower::default();
+ follower
+ .follow_once(&consumer, &flipped, |_| async { Ok(false) })
+ .await
+ .unwrap();
+ assert_eq!(consumer.topic(), TOPIC);
+ assert!(
+ consumer.is_draining(),
+ "the subscribed topic is a sealed generation"
+ );
+
+ follower
+ .follow_once(&consumer, &flipped, |_| async { Ok(true) })
+ .await
+ .unwrap();
+ assert_eq!(consumer.topic(), NEXT_TOPIC);
+ assert!(!consumer.is_draining());
+
+ // A later probe that cannot vouch for the old generation never sends it back.
+ follower
+ .follow_once(&consumer, &flipped, |_| async { Ok(false) })
+ .await
+ .unwrap();
+ assert_eq!(consumer.topic(), NEXT_TOPIC);
+}
+
// ============================================================
// Writer task: signals completion through channel
// ============================================================