:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(personhog): read the changelog topic and count from config/changelog (#119809)
Zach Brown010d107868e3
diff --git a/bin/mprocs.yaml b/bin/mprocs.yamlindex 0b8aab85a996..a9a50d949d8d 100755--- a/bin/mprocs.yaml+++ b/bin/mprocs.yaml@@ -494,7 +494,7 @@ procs:personhog-etcd-init:shell: |-bin/wait-for-docker --only etcd && \- docker exec posthog-etcd-1 etcdctl put /personhog/config/total_partitions 4 && \+ docker exec posthog-etcd-1 etcdctl put /personhog/config/changelog '{"topic":"personhog_updates","partitions":4,"epoch":1,"writer_group":"personhog-writer","draining":[]}' && \echo 'etcd init complete'capability: personhogready_pattern: 'etcd init complete'diff --git a/bin/start-rust-service b/bin/start-rust-serviceindex ab1b5a357d64..558f9d97098b 100755--- a/bin/start-rust-service+++ b/bin/start-rust-service@@ -376,7 +376,6 @@ case "$RS_SVC" in# In production, pod names resolve via K8s DNS.export POD_NAME=127.0.0.1export KAFKA_HOSTS="${KAFKA_HOSTS:-localhost:9092}"- export KAFKA_PERSON_STATE_TOPIC=personhog_updatesexport FALLBACK_DATABASE_URL="postgres://posthog:posthog@${PGHOST:-localhost}:${PGPORT:-5432}/posthog_persons"# Pair with the identity service's and the writer's default table, or# leader cache misses read a table the identity never wrote.@@ -389,7 +388,8 @@ case "$RS_SVC" inexport RUST_BACKTRACE=1export DATABASE_URL="postgres://posthog:posthog@${PGHOST:-localhost}:${PGPORT:-5432}/posthog_persons"export KAFKA_HOSTS="${KAFKA_HOSTS:-localhost:9092}"- export KAFKA_TOPIC=personhog_updates+ export ETCD_ENDPOINTS=http://localhost:2379+ export ETCD_PREFIX=/personhog/export KAFKA_CONSUMER_GROUP=personhog-writerexport FLUSH_INTERVAL_MS=5000export FLUSH_BUFFER_SIZE=1000diff --git a/docker-compose.base.yml b/docker-compose.base.ymlindex 59060a994113..db7619db5e5b 100644--- a/docker-compose.base.yml+++ b/docker-compose.base.yml@@ -583,7 +583,6 @@ services:ETCD_PREFIX: '/personhog/'POD_NAME: 'personhog-leader'KAFKA_HOSTS: 'kafka:9092'- KAFKA_PERSON_STATE_TOPIC: 'personhog_updates'FALLBACK_DATABASE_URL: 'postgres://posthog:posthog@db:5432/posthog_persons'# Must name the same table the identity service and the writer# target (their defaults), or leader misses read the wrong table.@@ -601,7 +600,8 @@ services:environment:DATABASE_URL: 'postgres://posthog:posthog@db:5432/posthog_persons'KAFKA_HOSTS: 'kafka:9092'- KAFKA_TOPIC: 'personhog_updates'+ ETCD_ENDPOINTS: 'http://etcd:2379'+ ETCD_PREFIX: '/personhog/'KAFKA_CONSUMER_GROUP: 'personhog-writer'FLUSH_INTERVAL_MS: '5000'FLUSH_BUFFER_SIZE: '1000'diff --git a/docker-compose.dev.yml b/docker-compose.dev.ymlindex 3232173c76f1..ea6b820bd501 100644--- a/docker-compose.dev.yml+++ b/docker-compose.dev.yml@@ -729,10 +729,14 @@ services:environment:DATABASE_URL: postgres://posthog:posthog@db:5432/${POSTHOG_PERSONS_DB_NAME:-posthog}depends_on:+ etcd:+ condition: service_healthydb:condition: service_healthykafka:condition: service_started+ personhog-etcd-init:+ condition: service_completed_successfullyprofiles:- ingestion@@ -752,12 +756,19 @@ services:profiles:- ingestion- # Seeds the partition count the leaders and routers coordinate under;+ # Seeds the changelog config the personhog services read at boot;# idempotent, and the same key bin/mprocs.yaml writes for hogli runs.personhog-etcd-init:image: quay.io/coreos/etcd:v3.6.8restart: 'no'- entrypoint: ['etcdctl', '--endpoints=http://etcd:2379', 'put', '/personhog/config/total_partitions', '4']+ entrypoint:+ [+ 'etcdctl',+ '--endpoints=http://etcd:2379',+ 'put',+ '/personhog/config/changelog',+ '{"topic":"personhog_updates","partitions":4,"epoch":1,"writer_group":"personhog-writer","draining":[]}',+ ]depends_on:etcd:condition: service_healthydiff --git a/rust/Cargo.lock b/rust/Cargo.lockindex e96ef7757838..23443538905d 100644--- a/rust/Cargo.lock+++ b/rust/Cargo.lock@@ -9156,6 +9156,7 @@ dependencies = [name = "personhog-writer"version = "0.1.0"dependencies = [+ "assignment-coordination","async-trait","axum 0.7.9","chrono",@@ -9170,6 +9171,7 @@ dependencies = ["lifecycle","metrics","personhog-common",+ "personhog-coordination","personhog-proto","prost 0.13.5","rdkafka",diff --git a/rust/ingestion-control-plane/src/personhog.rs b/rust/ingestion-control-plane/src/personhog.rsindex 9c4a30de0cdb..5ffd2b9fe004 100644--- a/rust/ingestion-control-plane/src/personhog.rs+++ b/rust/ingestion-control-plane/src/personhog.rs@@ -63,7 +63,6 @@ pub struct RawSnapshot {pub drained_acks: Vec<PodDrainedAck>,pub warmed_acks: Vec<PodWarmedAck>,pub leader: Option<(LeaderInfo, i64)>,- pub total_partitions: Option<u32>,pub changelog: Option<ChangelogConfig>,pub parse_errors: Vec<String>,pub unknown_keys: Vec<String>,@@ -128,14 +127,6 @@ pub fn parse_snapshot(kvs: &[RawKv], prefix: &str, revision: i64) -> RawSnapshotif let Some(leader) = parse::<LeaderInfo>(kv, errors) {snapshot.leader = Some((leader, kv.lease));}- } else if suffix == "config/total_partitions" {- match std::str::from_utf8(&kv.value)- .ok()- .and_then(|s| s.parse::<u32>().ok())- {- Some(count) => snapshot.total_partitions = Some(count),- None => errors.push(format!("invalid total_partitions at {}", kv.key)),- }} else if suffix == "config/changelog" {if let Some(changelog) = parse::<ChangelogConfig>(kv, errors) {snapshot.changelog = Some(changelog);@@ -314,12 +305,13 @@ pub fn derive_view(});}- if raw.total_partitions.is_none() {+ let total_partitions = raw.changelog.as_ref().map(|c| c.partitions);+ if raw.changelog.is_none() {issues.push(Issue {severity: Severity::Error,partition: None,- summary: "config/total_partitions is missing: the coordinator cannot plan, and \- leaders and routers fail at startup"+ summary: "config/changelog is not seeded: the coordinator cannot plan, \+ and every service waits at startup".to_string(),});}@@ -352,7 +344,7 @@ pub fn derive_view(// Partitions worth a row: everything under the configured count, plus// anything that has state anyway (misconfiguration shows itself).- let mut partition_ids: Vec<u32> = (0..raw.total_partitions.unwrap_or(0)).collect();+ let mut partition_ids: Vec<u32> = (0..total_partitions.unwrap_or(0)).collect();for partition in assignments_by_partition.keys().chain(handoffs_by_partition.keys())@@ -368,13 +360,13 @@ pub fn derive_view(let assignment = assignments_by_partition.get(&partition);let handoff = handoffs_by_partition.get(&partition);- if let Some(total) = raw.total_partitions {+ if let Some(total) = total_partitions {if partition >= total && (assignment.is_some() || handoff.is_some()) {issues.push(Issue {severity: Severity::Warning,partition: Some(partition),summary: format!(- "partition {partition} has state but total_partitions is {total}"+ "partition {partition} has state but the changelog has {total} partitions"),});}@@ -541,7 +533,7 @@ pub fn derive_view(fetched_at: Utc::now(),revision: raw.revision,prefix: prefix.to_string(),- total_partitions: raw.total_partitions,+ total_partitions,changelog: raw.changelog.clone(),leader: raw.leader.as_ref().map(|(leader, _)| LeaderView {holder: leader.holder.clone(),@@ -721,10 +713,10 @@ mod tests {)}- fn total_partitions_kv(count: u32) -> RawKv {+ fn changelog_kv(count: u32) -> RawKv {RawKv {- key: format!("{PREFIX}config/total_partitions"),- value: count.to_string().into_bytes(),+ key: format!("{PREFIX}config/changelog"),+ value: serde_json::to_vec(&ChangelogConfig::new("personhog_updates", count)).unwrap(),lease: 0,}}@@ -745,7 +737,7 @@ mod tests {#[test]fn a_stuck_freezing_handoff_names_the_missing_router() {let kvs = vec![- total_partitions_kv(1),+ changelog_kv(1),pod_kv("pod-a"),pod_kv("pod-b"),router_kv("router-0"),@@ -789,7 +781,7 @@ mod tests {#[test]fn a_satisfied_phase_points_at_the_coordinator() {let kvs = vec![- total_partitions_kv(1),+ changelog_kv(1),pod_kv("pod-b"),assignment_kv(0, "pod-b"),// Draining with a dead old owner: drain is vacuously satisfied,@@ -808,7 +800,7 @@ mod tests {#[test]fn a_dead_owner_without_a_handoff_is_an_error_and_with_one_is_not() {let kvs = vec![- total_partitions_kv(2),+ changelog_kv(2),pod_kv("pod-b"),assignment_kv(0, "pod-gone"),assignment_kv(1, "pod-gone"),@@ -826,7 +818,7 @@ mod tests {#[test]fn a_lingering_complete_handoff_flags_the_coordinator() {let kvs = vec![- total_partitions_kv(1),+ changelog_kv(1),pod_kv("pod-b"),assignment_kv(0, "pod-b"),handoff_kv(0, "Complete", None, "pod-b", &[], 1_000),@@ -849,7 +841,7 @@ mod tests {#[test]fn unassigned_and_out_of_range_partitions_are_flagged() {let kvs = vec![- total_partitions_kv(2),+ changelog_kv(2),pod_kv("pod-a"),assignment_kv(0, "pod-a"),assignment_kv(5, "pod-a"),@@ -862,7 +854,7 @@ mod tests {.any(|s| s.contains("partition 1 has no assignment and no handoff")));assert!(warnings.iter()- .any(|s| s.contains("partition 5 has state but total_partitions is 2")));+ .any(|s| s.contains("partition 5 has state but the changelog has 2 partitions")));// The out-of-range partition still renders a row.assert!(topology.partitions.iter().any(|p| p.partition == 5));}@@ -870,7 +862,7 @@ mod tests {#[test]fn stale_acks_are_inert_info_and_matching_acks_are_not() {let kvs = vec![- total_partitions_kv(1),+ changelog_kv(1),pod_kv("pod-a"),pod_kv("pod-b"),router_kv("router-0"),@@ -898,13 +890,13 @@ mod tests {}#[test]- fn missing_total_partitions_is_an_error_without_unassigned_noise() {+ fn an_unseeded_changelog_is_an_error_without_unassigned_noise() {let kvs = vec![pod_kv("pod-a"), assignment_kv(0, "pod-a")];let topology = view(&kvs, 10_000);assert!(issue_summaries(&topology, Severity::Error).iter()- .any(|s| s.contains("config/total_partitions is missing")));+ .any(|s| s.contains("config/changelog is not seeded")));assert!(!issue_summaries(&topology, Severity::Warning).iter().any(|s| s.contains("no assignment")));@@ -913,7 +905,7 @@ mod tests {#[test]fn a_dead_new_owner_warns_and_unparseable_records_surface() {let mut kvs = vec![- total_partitions_kv(1),+ changelog_kv(1),pod_kv("pod-a"),assignment_kv(0, "pod-a"),handoff_kv(0, "Warming", Some("pod-a"), "pod-gone", &[], 9_000),@@ -936,7 +928,7 @@ mod tests {#[test]fn pods_carry_their_assigned_partitions_and_lease_ttls() {let kvs = vec![- total_partitions_kv(2),+ changelog_kv(2),pod_kv("pod-a"),assignment_kv(1, "pod-a"),assignment_kv(0, "pod-a"),diff --git a/rust/ingestion-control-plane/src/ui/index.html b/rust/ingestion-control-plane/src/ui/index.htmlindex 2affd1bcc63c..1081a38987d2 100644--- a/rust/ingestion-control-plane/src/ui/index.html+++ b/rust/ingestion-control-plane/src/ui/index.html@@ -1397,7 +1397,7 @@}if (!view.config) {this.ui.changelogStatus.replaceChildren(- el("div", { class: "muted", style: "padding: 6px 14px 8px" }, "Not seeded."));+ el("div", { class: "muted", style: "padding: 6px 14px 8px" }, "Not seeded. Every personhog service waits at startup until the changelog is seeded."));return;}const c = view.config;@@ -1433,7 +1433,7 @@error.replaceChildren(el("div", { class: "error-box" }, "Enter the topic and the writer's consumer group."));return;}- if (!window.confirm(`Seed config/changelog with topic ${topic}, with writer group ${writer_group}?\n\nThe partition count is read from the topic's metadata.`)) return;+ if (!window.confirm(`Seed config/changelog with topic ${topic}, with writer group ${writer_group}?\n\nThe partition count is read from the topic's metadata. Every personhog participant adopts this object.`)) return;button.disabled = true;try {await fetchJSON("api/personhog/changelog/seed", {@@ -1464,7 +1464,7 @@? el("span", { class: "pill good", title: `lease ${data.leader.lease_id}${data.leader.lease_ttl_secs !== null ? ` · ttl ${data.leader.lease_ttl_secs}s` : ""}` }, `coordinator ${data.leader.holder}`): el("span", { class: "pill bad" }, "no coordinator"),el("span", { class: "pill muted" },- data.total_partitions === null ? "total_partitions missing" : `${fmtInt(data.total_partitions)} partitions`),+ data.total_partitions === null ? "changelog not seeded" : `${fmtInt(data.total_partitions)} partitions`),el("span", { class: `pill ${readyPods === data.pods.length && readyPods > 0 ? "good" : "warn"}` },`${fmtInt(readyPods)}/${fmtInt(data.pods.length)} pods ready`),el("span", { class: "pill muted" }, `${fmtInt(data.routers.length)} router${data.routers.length === 1 ? "" : "s"}`),diff --git a/rust/personhog-coordination/README.md b/rust/personhog-coordination/README.mdindex b4b011127a83..0adb6875a8da 100644--- a/rust/personhog-coordination/README.md+++ b/rust/personhog-coordination/README.md@@ -123,6 +123,10 @@ Several failure modes are handled explicitly:**Mid-stream watch reconnects don't replay missed events — bounded by reconciliation**: The snapshot-to-watch-attach gap is closed (watches are revision-anchored to the snapshot that seeded them), but if an *established* etcd watch stream drops and the underlying client reconnects, events that fired during the disconnect are not replayed. Every participant now bounds the resulting staleness with its level-triggered truth path — the coordinator's tick (which also wakes the planner unconditionally), the router's reconcile pass, and the pod's converge-all tick — so a missed event costs at most one reconcile interval, not correctness.+## Changelog config++`config/changelog` (`{topic, partitions, epoch, writer_group, draining}`) is the one live object every participant reads. Every service loads it at boot; the router and the identity service also follow it through a watch, so the partition count they hash under is the live one.+## Modules| Module | Responsibility |diff --git a/rust/personhog-coordination/src/changelog.rs b/rust/personhog-coordination/src/changelog.rsnew file mode 100644index 000000000000..da0a1cdf0dad--- /dev/null+++ b/rust/personhog-coordination/src/changelog.rs@@ -0,0 +1,221 @@+//! Boot-time load and the shared watch of `config/changelog`.++use std::ops::ControlFlow;+use std::sync::atomic::{AtomicU32, Ordering};+use std::sync::Arc;+use std::time::Duration;++use etcd_client::EventType;+use tokio_util::sync::CancellationToken;++use crate::error::{Error, Result};+use crate::store::PersonhogStore;+use crate::types::ChangelogConfig;+use crate::util;++/// Polls until `config/changelog` is seeded; it is the only source of the topic and count.+pub async fn load_changelog_config(+ store: &PersonhogStore,+ poll_interval: Duration,+) -> Result<ChangelogConfig> {+ let mut polls: u64 = 0;+ loop {+ if let Some(config) = store.get_changelog_config().await? {+ return Ok(config);+ }+ if polls.is_multiple_of(10) {+ tracing::warn!("config/changelog is not seeded; waiting for the control plane");+ }+ polls += 1;+ tokio::time::sleep(poll_interval).await;+ }+}++/// Delivers each object that supersedes the last one adopted; `initial` is what boot adopted.+pub async fn watch_changelog_config_loop<F>(+ store: Arc<PersonhogStore>,+ initial: ChangelogConfig,+ reconcile_interval: Duration,+ cancel: CancellationToken,+ mut on_config: F,+) where+ F: FnMut(&ChangelogConfig) -> ControlFlow<()>,+{+ let mut adopted = initial;+ let mut backoff = Duration::from_millis(100);+ 'establish: loop {+ if cancel.is_cancelled() {+ return;+ }+ let established = async {+ let (current, revision) = store.get_changelog_config_with_revision().await?;+ let stream = store.watch_changelog_config_from(revision + 1).await?;+ Ok::<_, Error>((current, stream))+ };+ let (current, mut stream) = match established.await {+ Ok(established) => established,+ Err(e) => {+ tracing::warn!(error = %e, "changelog config watch could not be established");+ metrics::counter!("personhog_changelog_config_watch_reconnects_total").increment(1);+ tokio::select! {+ _ = cancel.cancelled() => return,+ _ = tokio::time::sleep(backoff) => {}+ }+ backoff = (backoff * 2).min(Duration::from_secs(5));+ continue 'establish;+ }+ };+ backoff = Duration::from_millis(100);+ if let Some(config) = ¤t {+ if deliver(&mut adopted, config, "establish", &mut on_config).is_break() {+ return;+ }+ }++ let mut reconcile = tokio::time::interval(reconcile_interval);+ reconcile.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);+ reconcile.tick().await;+ loop {+ tokio::select! {+ _ = cancel.cancelled() => return,+ _ = reconcile.tick() => {+ match store.get_changelog_config().await {+ Ok(Some(config)) => {+ if deliver(&mut adopted, &config, "reconcile", &mut on_config).is_break() {+ return;+ }+ }+ Ok(None) => {}+ Err(e) => {+ tracing::warn!(error = %e, "changelog config reconcile read failed");+ }+ }+ }+ msg = stream.message() => {+ let resp = match msg.map_err(Error::from).and_then(|resp| util::live_watch_response(resp, "changelog config")) {+ Ok(resp) => resp,+ Err(e) => {+ tracing::warn!(error = %e, "changelog config watch broke; re-establishing");+ metrics::counter!("personhog_changelog_config_watch_reconnects_total").increment(1);+ continue 'establish;+ }+ };+ for event in resp.events() {+ if event.event_type() != EventType::Put {+ continue;+ }+ let Some(kv) = event.kv() else { continue };+ match serde_json::from_slice::<ChangelogConfig>(kv.value()) {+ Ok(config) => {+ if deliver(&mut adopted, &config, "watch", &mut on_config).is_break() {+ return;+ }+ }+ Err(e) => {+ tracing::error!(error = %e, "unreadable changelog config event; the reconcile read will retry");+ }+ }+ }+ }+ }+ }+ }+}++/// Bounds the staleness a dropped watch stream can hide.+pub const CHANGELOG_RECONCILE_INTERVAL: Duration = Duration::from_secs(30);+pub const CHANGELOG_BOOT_POLL_INTERVAL: Duration = Duration::from_secs(1);++/// The partition count has its own atomic: routing reads it per request.+#[derive(Clone)]+pub struct LiveChangelog {+ current: Arc<std::sync::RwLock<ChangelogConfig>>,+ partitions: Arc<AtomicU32>,+}++impl LiveChangelog {+ pub fn new(config: ChangelogConfig) -> Self {+ Self {+ partitions: Arc::new(AtomicU32::new(config.partitions)),+ current: Arc::new(std::sync::RwLock::new(config)),+ }+ }++ pub fn partitions(&self) -> u32 {+ self.partitions.load(Ordering::Acquire)+ }++ pub fn current(&self) -> ChangelogConfig {+ self.current+ .read()+ .expect("changelog config lock poisoned")+ .clone()+ }++ pub fn epoch(&self) -> u64 {+ self.current+ .read()+ .expect("changelog config lock poisoned")+ .epoch+ }++ pub fn adopt(&self, config: &ChangelogConfig) -> bool {+ let mut current = self+ .current+ .write()+ .expect("changelog config lock poisoned");+ if !config.supersedes(¤t) {+ return false;+ }+ *current = config.clone();+ self.partitions.store(config.partitions, Ordering::Release);+ true+ }+}++pub async fn keep_changelog_live(+ store: Arc<PersonhogStore>,+ live: LiveChangelog,+ cancel: CancellationToken,+) {+ let initial = live.current();+ watch_changelog_config_loop(+ store,+ initial,+ CHANGELOG_RECONCILE_INTERVAL,+ cancel,+ |config| {+ live.adopt(config);+ ControlFlow::Continue(())+ },+ )+ .await+}++fn deliver<F>(+ adopted: &mut ChangelogConfig,+ config: &ChangelogConfig,+ source: &'static str,+ on_config: &mut F,+) -> ControlFlow<()>+where+ F: FnMut(&ChangelogConfig) -> ControlFlow<()>,+{+ if !config.supersedes(adopted) {+ metrics::counter!("personhog_changelog_config_stale_deliveries_total", "source" => source)+ .increment(1);+ return ControlFlow::Continue(());+ }+ *adopted = config.clone();+ metrics::counter!("personhog_changelog_config_adoptions_total", "source" => source)+ .increment(1);+ metrics::gauge!("personhog_changelog_config_epoch").set(config.epoch as f64);+ tracing::info!(+ topic = %config.topic,+ partitions = config.partitions,+ epoch = config.epoch,+ source,+ "adopted changelog config"+ );+ on_config(config)+}diff --git a/rust/personhog-coordination/src/coordinator.rs b/rust/personhog-coordination/src/coordinator.rsindex 3a463d2acad8..3669cfeb24cc 100644--- a/rust/personhog-coordination/src/coordinator.rs+++ b/rust/personhog-coordination/src/coordinator.rs@@ -1281,14 +1281,11 @@ impl Coordinator {max_txn_ops: usize,) -> Result<()> {let pods = store.list_pods().await?;- let total_partitions = match store.get_total_partitions().await {- Ok(n) => n,- Err(Error::NotFound(_)) => {- tracing::debug!("total_partitions not set, skipping assignment");- return Ok(());- }- Err(e) => return Err(e),+ let Some(changelog) = store.get_changelog_config().await? else {+ tracing::debug!("changelog config not seeded, skipping assignment");+ return Ok(());};+ let total_partitions = changelog.partitions;// K8s-aware placement policies for smarter rebalancinglet members = match k8s_awareness {diff --git a/rust/personhog-coordination/src/lib.rs b/rust/personhog-coordination/src/lib.rsindex d0f696f6182e..489e5b289116 100644--- a/rust/personhog-coordination/src/lib.rs+++ b/rust/personhog-coordination/src/lib.rs@@ -1,4 +1,5 @@pub mod authority;+pub mod changelog;pub mod coordinator;pub mod error;pub mod hash;diff --git a/rust/personhog-coordination/src/store.rs b/rust/personhog-coordination/src/store.rsindex ba240882b7f9..2032e333d980 100644--- a/rust/personhog-coordination/src/store.rs+++ b/rust/personhog-coordination/src/store.rs@@ -1,5 +1,4 @@use std::collections::HashMap;-use std::str::from_utf8;use std::sync::{Arc, Mutex as StdMutex};use assignment_coordination::store::EtcdStore;@@ -39,7 +38,6 @@ enum StoreKey<'a> {FreezeQuorumsPrefix,Leader,Generation,- TotalPartitions,ChangelogConfig,}@@ -73,7 +71,6 @@ impl StoreKey<'_> {StoreKey::FreezeQuorumsPrefix => format!("{prefix}freeze_quorums/"),StoreKey::Leader => format!("{prefix}coordinator/leader"),StoreKey::Generation => format!("{prefix}generation"),- StoreKey::TotalPartitions => format!("{prefix}config/total_partitions"),StoreKey::ChangelogConfig => format!("{prefix}config/changelog"),}}@@ -1188,31 +1185,20 @@ impl PersonhogStore {// ── Config operations ───────────────────────────────────────- pub async fn get_total_partitions(&self) -> Result<u32> {- count_call("get_total_partitions");- let key = self.key(StoreKey::TotalPartitions);- let bytes = self- .inner- .get_raw(&key)- .await?- .ok_or_else(|| Error::NotFound(key))?;- let s = from_utf8(&bytes)- .map_err(|e| Error::invalid_state(format!("non-utf8 total_partitions: {e}")))?;- s.parse::<u32>()- .map_err(|e| Error::invalid_state(format!("invalid total_partitions: {e}")))- }-- pub async fn set_total_partitions(&self, count: u32) -> Result<()> {- let key = self.key(StoreKey::TotalPartitions);- Ok(self.inner.put_raw(&key, count.to_string()).await?)- }-pub async fn get_changelog_config(&self) -> Result<Option<ChangelogConfig>> {count_call("get_changelog_config");let key = self.key(StoreKey::ChangelogConfig);Ok(self.inner.get(&key).await?)}+ pub async fn get_changelog_config_with_revision(+ &self,+ ) -> Result<(Option<ChangelogConfig>, i64)> {+ count_call("get_changelog_config_with_revision");+ let key = self.key(StoreKey::ChangelogConfig);+ Ok(self.inner.get_with_revision(&key).await?)+ }+/// Create-if-absent at epoch 1; only a flip may move a live config.pub async fn seed_changelog_config(&self,@@ -1237,6 +1223,19 @@ impl PersonhogStore {Ok(resp.succeeded())}+ /// Bypasses the protocol: see the crate's `test-support` feature.+ #[cfg(any(test, feature = "test-support"))]+ pub async fn put_changelog_config(&self, config: &ChangelogConfig) -> Result<()> {+ let key = self.key(StoreKey::ChangelogConfig);+ Ok(self.inner.put(&key, config, None).await?)+ }++ pub async fn watch_changelog_config_from(&self, start_revision: i64) -> Result<WatchStream> {+ count_call("watch_changelog_config_from");+ let key = self.key(StoreKey::ChangelogConfig);+ Ok(self.inner.watch_key_from(&key, start_revision).await?)+ }+pub async fn get_generation(&self) -> Result<String> {let key = self.key(StoreKey::Generation);let bytes = selfdiff --git a/rust/personhog-coordination/src/types.rs b/rust/personhog-coordination/src/types.rsindex c7e9738ccd48..6e4aa94bde8a 100644--- a/rust/personhog-coordination/src/types.rs+++ b/rust/personhog-coordination/src/types.rs@@ -335,6 +335,14 @@ impl ChangelogConfig {draining: Vec::new(),}}++ /// A flip raises the epoch. Within an epoch only a retire changes the+ /// object, and a retire only shrinks `draining`, so a same-epoch object+ /// with more generations is an older one delivered late.+ pub fn supersedes(&self, adopted: &ChangelogConfig) -> bool {+ self.epoch > adopted.epoch+ || (self.epoch == adopted.epoch && self.draining.len() < adopted.draining.len())+ }}#[cfg(test)]@@ -362,6 +370,35 @@ mod tests {assert_eq!(h.phase_entered_at_ms, 0);}+ #[test]+ fn a_config_supersedes_by_epoch_then_by_retired_generations() {+ let generation = |epoch| Generation {+ topic: format!("g{epoch}"),+ partitions: 64,+ epoch,+ };+ let config = |epoch: u64, generations: &[u64]| ChangelogConfig {+ epoch,+ draining: generations.iter().map(|e| generation(*e)).collect(),+ ..ChangelogConfig::new("g", 128)+ };+ let adopted = config(3, &[2]);+ let cases = [+ ("a newer epoch", config(4, &[2, 3]), true),+ ("a retire at the same epoch", config(3, &[]), true),+ ("the same object", config(3, &[2]), false),+ (+ "the pre-retire object delivered late",+ config(3, &[1, 2]),+ false,+ ),+ ("an older epoch", config(2, &[]), false),+ ];+ for (name, candidate, expected) in cases {+ assert_eq!(candidate.supersedes(&adopted), expected, "{name}");+ }+ }+#[test]fn registered_pod_roundtrip() {let pod = RegisteredPod {diff --git a/rust/personhog-coordination/tests/changelog.rs b/rust/personhog-coordination/tests/changelog.rsindex 48290c4f25c9..bc304f617a93 100644--- a/rust/personhog-coordination/tests/changelog.rs+++ b/rust/personhog-coordination/tests/changelog.rs@@ -1,6 +1,15 @@mod common;-use common::test_store;+use std::ops::ControlFlow;+use std::sync::Arc;+use std::time::Duration;++use personhog_coordination::changelog::{load_changelog_config, watch_changelog_config_loop};+use personhog_coordination::types::{ChangelogConfig, Generation};+use tokio::sync::mpsc;+use tokio_util::sync::CancellationToken;++use common::{store_at, test_store, FlakyProxy, ETCD_ENDPOINT};#[tokio::test]async fn a_second_seed_is_refused() {@@ -24,3 +33,175 @@ async fn a_second_seed_is_refused() {("personhog_updates_64", 64, 1, "personhog-writer"));}++#[tokio::test]+async fn boot_polls_until_the_config_is_seeded() {+ let store = test_store("changelog-boot").await;+ let seeder = Arc::clone(&store);+ tokio::spawn(async move {+ tokio::time::sleep(Duration::from_millis(200)).await;+ seeder+ .seed_changelog_config("personhog_updates_64", 64, "personhog-writer")+ .await+ .unwrap();+ });+ let loaded = tokio::time::timeout(+ Duration::from_secs(5),+ load_changelog_config(&store, Duration::from_millis(20)),+ )+ .await+ .expect("boot polls until the config is seeded")+ .unwrap();+ assert_eq!((loaded.partitions, loaded.epoch), (64, 1));+}++/// What a loop starts from before it has adopted anything.+fn nothing_adopted() -> ChangelogConfig {+ ChangelogConfig {+ epoch: 0,+ ..ChangelogConfig::new("unadopted", 0)+ }+}++async fn next_adopted(rx: &mut mpsc::UnboundedReceiver<ChangelogConfig>) -> ChangelogConfig {+ tokio::time::timeout(Duration::from_secs(5), rx.recv())+ .await+ .expect("a config is adopted")+ .expect("the loop is alive")+}++#[tokio::test]+async fn the_watch_delivers_newer_epochs_and_retires_but_nothing_stale() {+ let store = test_store("changelog-watch").await;+ store+ .seed_changelog_config("personhog_updates_64", 64, "personhog-writer")+ .await+ .unwrap();++ let (tx, mut rx) = mpsc::unbounded_channel();+ let cancel = CancellationToken::new();+ let loop_store = Arc::clone(&store);+ let loop_cancel = cancel.clone();+ tokio::spawn(async move {+ watch_changelog_config_loop(+ loop_store,+ nothing_adopted(),+ Duration::from_secs(30),+ loop_cancel,+ |c| {+ tx.send(c.clone()).unwrap();+ ControlFlow::Continue(())+ },+ )+ .await;+ });++ assert_eq!(next_adopted(&mut rx).await.epoch, 1);++ let flipped = ChangelogConfig {+ topic: "personhog_updates_128".into(),+ partitions: 128,+ epoch: 2,+ writer_group: String::new(),+ draining: Vec::new(),+ };+ store.put_changelog_config(&flipped).await.unwrap();+ assert_eq!(next_adopted(&mut rx).await, flipped);++ store.put_changelog_config(&flipped).await.unwrap();+ store+ .put_changelog_config(&ChangelogConfig {+ epoch: 1,+ ..flipped.clone()+ })+ .await+ .unwrap();+ let newer = ChangelogConfig {+ epoch: 3,+ draining: vec![Generation {+ topic: "personhog_updates_128".into(),+ partitions: 128,+ epoch: 2,+ }],+ ..flipped.clone()+ };+ store.put_changelog_config(&newer).await.unwrap();+ assert_eq!(next_adopted(&mut rx).await, newer);++ // A retire keeps the epoch and only shrinks `draining`.+ let retired = ChangelogConfig {+ draining: Vec::new(),+ ..newer.clone()+ };+ store.put_changelog_config(&retired).await.unwrap();+ assert_eq!(next_adopted(&mut rx).await, retired);+ // The pre-retire object arriving late is older, not a new generation.+ store.put_changelog_config(&newer).await.unwrap();+ store.put_changelog_config(&retired).await.unwrap();+ // Deliveries keep revision order, so the next one proves the stale writes were dropped.+ let sentinel = ChangelogConfig {+ epoch: 4,+ ..retired.clone()+ };+ store.put_changelog_config(&sentinel).await.unwrap();+ assert_eq!(next_adopted(&mut rx).await, sentinel);++ cancel.cancel();+}++#[tokio::test]+async fn a_severed_watch_still_adopts_the_flip_through_reconcile() {+ let proxy = FlakyProxy::start("127.0.0.1:2379").await;+ let prefix = format!("/test-changelog-sever-{}/", uuid::Uuid::new_v4());+ let watched = store_at(&proxy.endpoint, &prefix).await;+ let direct = store_at(ETCD_ENDPOINT, &prefix).await;+ direct+ .seed_changelog_config("personhog_updates_64", 64, "personhog-writer")+ .await+ .unwrap();++ let (tx, mut rx) = mpsc::unbounded_channel();+ let cancel = CancellationToken::new();+ let loop_cancel = cancel.clone();+ tokio::spawn(async move {+ watch_changelog_config_loop(+ watched,+ nothing_adopted(),+ Duration::from_millis(300),+ loop_cancel,+ |c| {+ tx.send(c.epoch).unwrap();+ ControlFlow::Continue(())+ },+ )+ .await;+ });+ assert_eq!(+ tokio::time::timeout(Duration::from_secs(5), rx.recv())+ .await+ .unwrap(),+ Some(1)+ );++ proxy.set_blackholed(true);+ proxy.sever();+ direct+ .put_changelog_config(&ChangelogConfig {+ topic: "personhog_updates_128".into(),+ partitions: 128,+ epoch: 2,+ writer_group: String::new(),+ draining: Vec::new(),+ })+ .await+ .unwrap();+ proxy.set_blackholed(false);++ assert_eq!(+ tokio::time::timeout(Duration::from_secs(10), rx.recv())+ .await+ .expect("the flip is adopted after the stream heals"),+ Some(2)+ );+ cancel.cancel();+}diff --git a/rust/personhog-coordination/tests/integration.rs b/rust/personhog-coordination/tests/integration.rsindex ff2916d8b4de..f01555bb4f1f 100644--- a/rust/personhog-coordination/tests/integration.rs+++ b/rust/personhog-coordination/tests/integration.rs@@ -83,7 +83,10 @@ async fn single_pod_gets_all_partitions(let store = test_store(test_name).await;let cancel = CancellationToken::new();- store.set_total_partitions(NUM_PARTITIONS).await.unwrap();+ assert!(store+ .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")+ .await+ .unwrap());let _coord = start_coordinator(Arc::clone(&store), strategy, cancel.clone());let _pod = start_pod(Arc::clone(&store), "writer-0", cancel.clone());@@ -125,7 +128,10 @@ async fn scale_up_triggers_handoff(let store = test_store(test_name).await;let cancel = CancellationToken::new();- store.set_total_partitions(NUM_PARTITIONS).await.unwrap();+ assert!(store+ .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")+ .await+ .unwrap());// Start coordinator, one pod, and one routerlet _coord = start_coordinator(Arc::clone(&store), strategy, cancel.clone());@@ -256,7 +262,10 @@ async fn pod_crash_reassigns_partitions(let store = test_store(test_name).await;let cancel = CancellationToken::new();- store.set_total_partitions(NUM_PARTITIONS).await.unwrap();+ assert!(store+ .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")+ .await+ .unwrap());// Start coordinator and two pods (short lease so crash is detected fast)let _coord = start_coordinator(Arc::clone(&store), strategy, cancel.clone());@@ -325,7 +334,10 @@ async fn leader_failover(let store = test_store(test_name).await;let cancel = CancellationToken::new();- store.set_total_partitions(NUM_PARTITIONS).await.unwrap();+ assert!(store+ .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")+ .await+ .unwrap());// Start coordinator-0 with short lease so failover is fastlet leader_cancel = CancellationToken::new();@@ -413,7 +425,10 @@ async fn multi_router_ack_quorum(let store = test_store(test_name).await;let cancel = CancellationToken::new();- store.set_total_partitions(NUM_PARTITIONS).await.unwrap();+ assert!(store+ .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")+ .await+ .unwrap());let _coord = start_coordinator(Arc::clone(&store), strategy, cancel.clone());let _pod0 = start_pod(Arc::clone(&store), "writer-0", cancel.clone());@@ -513,7 +528,10 @@ async fn pod_crash_during_warming_restores_assignments(let store = test_store(test_name).await;let cancel = CancellationToken::new();- store.set_total_partitions(MANY_PARTITIONS).await.unwrap();+ assert!(store+ .seed_changelog_config("personhog_updates", MANY_PARTITIONS, "personhog-writer")+ .await+ .unwrap());let _coord = start_coordinator(Arc::clone(&store), strategy, cancel.clone());let _router = start_router(Arc::clone(&store), "router-0", cancel.clone());@@ -640,7 +658,10 @@ async fn scale_down_to_one_pod(let store = test_store(test_name).await;let cancel = CancellationToken::new();- store.set_total_partitions(MANY_PARTITIONS).await.unwrap();+ assert!(store+ .seed_changelog_config("personhog_updates", MANY_PARTITIONS, "personhog-writer")+ .await+ .unwrap());let _coord = start_coordinator(Arc::clone(&store), strategy, cancel.clone());let _router = start_router(Arc::clone(&store), "router-0", cancel.clone());@@ -723,7 +744,10 @@ async fn rapid_pod_joins(let store = test_store(test_name).await;let cancel = CancellationToken::new();- store.set_total_partitions(MANY_PARTITIONS).await.unwrap();+ assert!(store+ .seed_changelog_config("personhog_updates", MANY_PARTITIONS, "personhog-writer")+ .await+ .unwrap());let _coord = start_coordinator(Arc::clone(&store), Arc::clone(&strategy), cancel.clone());let _router = start_router(Arc::clone(&store), "router-0", cancel.clone());@@ -813,7 +837,10 @@ async fn rolling_update(let store = test_store(test_name).await;let cancel = CancellationToken::new();- store.set_total_partitions(MANY_PARTITIONS).await.unwrap();+ assert!(store+ .seed_changelog_config("personhog_updates", MANY_PARTITIONS, "personhog-writer")+ .await+ .unwrap());let _coord = start_coordinator(Arc::clone(&store), strategy, cancel.clone());let _router = start_router(Arc::clone(&store), "router-0", cancel.clone());@@ -940,7 +967,10 @@ async fn coordinator_starts_after_pods(let store = test_store(test_name).await;let cancel = CancellationToken::new();- store.set_total_partitions(NUM_PARTITIONS).await.unwrap();+ assert!(store+ .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")+ .await+ .unwrap());// Start pods and router BEFORE the coordinatorlet _pod0 = start_pod(Arc::clone(&store), "writer-0", cancel.clone());@@ -1005,7 +1035,10 @@ async fn debounce_batches_rapid_pod_changes(let store = test_store(test_name).await;let cancel = CancellationToken::new();- store.set_total_partitions(MANY_PARTITIONS).await.unwrap();+ assert!(store+ .seed_changelog_config("personhog_updates", MANY_PARTITIONS, "personhog-writer")+ .await+ .unwrap());let _coord = start_coordinator_with_debounce(Arc::clone(&store),@@ -1104,7 +1137,10 @@ async fn graceful_drain_transfers_partitions(_evenly_distributed: bool,) {let store = test_store(test_name).await;- store.set_total_partitions(NUM_PARTITIONS).await.unwrap();+ assert!(store+ .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")+ .await+ .unwrap());let coord_cancel = CancellationToken::new();let _coord = start_coordinator(@@ -1199,7 +1235,10 @@ async fn graceful_drain_transfers_partitions(#[tokio::test]async fn drain_status_write_failure_exits_cleanly() {let store = test_store("drain-write-fail").await;- store.set_total_partitions(NUM_PARTITIONS).await.unwrap();+ assert!(store+ .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")+ .await+ .unwrap());let coord_cancel = CancellationToken::new();let strategy: Arc<dyn AssignmentStrategy> = Arc::new(StickyBalancedStrategy);@@ -1259,7 +1298,10 @@ async fn initial_assignment_creates_handoff_with_no_old_owner() {let store = test_store("initial-no-old-owner").await;let cancel = CancellationToken::new();- store.set_total_partitions(NUM_PARTITIONS).await.unwrap();+ assert!(store+ .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")+ .await+ .unwrap());let strategy: Arc<dyn AssignmentStrategy> = Arc::new(StickyBalancedStrategy);let _coord = start_coordinator(Arc::clone(&store), strategy, cancel.clone());@@ -1319,7 +1361,10 @@ async fn old_owner_retains_partition_through_warming() {let store = test_store("retain-through-warming").await;let cancel = CancellationToken::new();- store.set_total_partitions(NUM_PARTITIONS).await.unwrap();+ assert!(store+ .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")+ .await+ .unwrap());let strategy: Arc<dyn AssignmentStrategy> = Arc::new(StickyBalancedStrategy);let _coord = start_coordinator(Arc::clone(&store), strategy, cancel.clone());@@ -1390,7 +1435,10 @@ async fn a_dead_new_owner_handoff_is_reaffirmed_to_the_current_owner() {let store = test_store("delete-drains-stash").await;let cancel = CancellationToken::new();- store.set_total_partitions(NUM_PARTITIONS).await.unwrap();+ assert!(store+ .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")+ .await+ .unwrap());let strategy: Arc<dyn AssignmentStrategy> = Arc::new(StickyBalancedStrategy);let _coord = start_coordinator(Arc::clone(&store), strategy, cancel.clone());@@ -1511,7 +1559,10 @@ async fn stale_handoff_address_never_overwrites_a_fresher_registration() {let store = test_store("addr-authority").await;let cancel = CancellationToken::new();- store.set_total_partitions(NUM_PARTITIONS).await.unwrap();+ assert!(store+ .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")+ .await+ .unwrap());let strategy: Arc<dyn AssignmentStrategy> = Arc::new(StickyBalancedStrategy);let _coord = start_coordinator(Arc::clone(&store), strategy, cancel.clone());@@ -1588,7 +1639,10 @@ async fn pod_re_registration_refreshes_router_addresses_without_a_handoff() {let store = test_store("addr-refresh").await;let cancel = CancellationToken::new();- store.set_total_partitions(NUM_PARTITIONS).await.unwrap();+ assert!(store+ .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")+ .await+ .unwrap());let strategy: Arc<dyn AssignmentStrategy> = Arc::new(StickyBalancedStrategy);let _coord = start_coordinator(Arc::clone(&store), strategy, cancel.clone());@@ -1686,7 +1740,10 @@ async fn routing_table_updated_at_handoff_complete() {let store = test_store("routing-table-inline").await;let cancel = CancellationToken::new();- store.set_total_partitions(NUM_PARTITIONS).await.unwrap();+ assert!(store+ .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")+ .await+ .unwrap());let strategy: Arc<dyn AssignmentStrategy> = Arc::new(StickyBalancedStrategy);let _coord = start_coordinator(Arc::clone(&store), strategy, cancel.clone());@@ -1832,7 +1889,10 @@ async fn dead_old_owner_in_freezing_advances_to_completion() {let store = test_store("dead-old-owner-freezing").await;let cancel = CancellationToken::new();- store.set_total_partitions(NUM_PARTITIONS).await.unwrap();+ assert!(store+ .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")+ .await+ .unwrap());// Inject a Freezing handoff: old_owner=phantom-old (never registered// as a pod), new_owner=writer-0 (will be registered next).@@ -1952,7 +2012,10 @@ async fn a_dead_new_owner_handoff_in_warming_is_reaffirmed() {let store = test_store("delete-during-warming").await;let cancel = CancellationToken::new();- store.set_total_partitions(NUM_PARTITIONS).await.unwrap();+ assert!(store+ .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")+ .await+ .unwrap());let strategy: Arc<dyn AssignmentStrategy> = Arc::new(StickyBalancedStrategy);let _coord = start_coordinator(Arc::clone(&store), strategy, cancel.clone());@@ -2070,7 +2133,10 @@ async fn reconcile_advances_warming_with_pre_staged_warmed_ack() {let store = test_store("reconcile-pre-staged-ack").await;let cancel = CancellationToken::new();- store.set_total_partitions(NUM_PARTITIONS).await.unwrap();+ assert!(store+ .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")+ .await+ .unwrap());// Pre-stage a Warming handoff for partition 6 with no coordinator// running. New owner already "warmed" (we inject the ack directly).@@ -2251,7 +2317,10 @@ async fn draining_old_owner_does_not_trigger_cleanup() {let store = test_store("draining-no-cleanup").await;let cancel = CancellationToken::new();- store.set_total_partitions(NUM_PARTITIONS).await.unwrap();+ assert!(store+ .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")+ .await+ .unwrap());// Register a Draining pod with no DrainedAck written yet.let lease = store.grant_lease(30).await.unwrap();@@ -2324,7 +2393,10 @@ async fn a_handoff_that_can_never_reach_quorum_is_cancelled() {let store = test_store("handoff-deadline-cancels").await;let cancel = CancellationToken::new();- store.set_total_partitions(NUM_PARTITIONS).await.unwrap();+ assert!(store+ .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")+ .await+ .unwrap());// A live new owner, so the dead-new-owner cleanup path stays out of// this: the handoff must be removed by the deadline alone.@@ -2424,7 +2496,10 @@ async fn freezing_blocks_until_routers_ack_before_draining() {let store = test_store("freezing-gates-draining").await;let cancel = CancellationToken::new();- store.set_total_partitions(NUM_PARTITIONS).await.unwrap();+ assert!(store+ .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")+ .await+ .unwrap());// Register both owner pods: the new owner so the dead-new-owner cancellation// doesn't delete the injected handoff, the old owner so the scenario@@ -2542,7 +2617,10 @@ async fn initial_assignment_skips_draining_phase() {let store = test_store("initial-skips-draining").await;let cancel = CancellationToken::new();- store.set_total_partitions(NUM_PARTITIONS).await.unwrap();+ assert!(store+ .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")+ .await+ .unwrap());// Register the new_owner pod so the dead-new-owner cancellation doesn't// delete the injected handoff for missing-target reasons.@@ -2608,7 +2686,10 @@ async fn dead_old_owner_in_draining_recovers() {let store = test_store("dead-old-owner-draining").await;let cancel = CancellationToken::new();- store.set_total_partitions(NUM_PARTITIONS).await.unwrap();+ assert!(store+ .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")+ .await+ .unwrap());// Register both owners so the injected handoff starts as a live// in-flight Draining handoff waiting on a real DrainedAck.@@ -2711,7 +2792,10 @@ async fn reconcile_advances_draining_with_pre_staged_drained_ack() {let store = test_store("reconcile-draining-ack").await;let cancel = CancellationToken::new();- store.set_total_partitions(NUM_PARTITIONS).await.unwrap();+ assert!(store+ .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")+ .await+ .unwrap());// Pre-stage both pods (so cleanup doesn't fire on either) plus a// Draining handoff plus a matching DrainedAck — all before the@@ -2860,7 +2944,10 @@ async fn a_dead_new_owner_handoff_in_draining_is_reaffirmed() {let store = test_store("delete-during-draining").await;let cancel = CancellationToken::new();- store.set_total_partitions(NUM_PARTITIONS).await.unwrap();+ assert!(store+ .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")+ .await+ .unwrap());let strategy: Arc<dyn AssignmentStrategy> = Arc::new(StickyBalancedStrategy);let _coord = start_coordinator(Arc::clone(&store), strategy, cancel.clone());@@ -3045,7 +3132,10 @@ async fn routing_table_deregisters_router_on_graceful_exit() {async fn stuck_handoff_defers_only_its_own_partition() {let store = test_store("stuck_handoff_defers_only_its_own_partition").await;let cancel = CancellationToken::new();- store.set_total_partitions(NUM_PARTITIONS).await.unwrap();+ assert!(store+ .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")+ .await+ .unwrap());let _coord = start_coordinator(Arc::clone(&store),@@ -3350,7 +3440,10 @@ async fn fresh_plan_is_rejected_when_an_assignment_appeared() {async fn rebalance_never_writes_assignment_records() {let store = test_store("rebalance_never_writes_assignment_records").await;let cancel = CancellationToken::new();- store.set_total_partitions(NUM_PARTITIONS).await.unwrap();+ assert!(store+ .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")+ .await+ .unwrap());let _coord = start_coordinator(Arc::clone(&store),diff --git a/rust/personhog-coordination/tests/k3s_integration.rs b/rust/personhog-coordination/tests/k3s_integration.rsindex cc41250ef4fb..07f0d6a0d379 100644--- a/rust/personhog-coordination/tests/k3s_integration.rs+++ b/rust/personhog-coordination/tests/k3s_integration.rs@@ -553,7 +553,10 @@ async fn deployment_rollout_reassigns_partitions() {// answering from the scaffolding's watch, which is exactly how this// gap previously escaped the suite).let store = test_store("deploy-rollout-k3s").await;- store.set_total_partitions(NUM_PARTITIONS).await.unwrap();+ assert!(store+ .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")+ .await+ .unwrap());let coordinator_awareness = Arc::new(K8sAwareness::new(k8s_client.clone(),@@ -746,7 +749,10 @@ async fn deployment_rollout_caps_first_new_pod_at_quota() {tokio::time::sleep(Duration::from_secs(3)).await;let store = test_store("deploy-quota-k3s").await;- store.set_total_partitions(NUM_PARTITIONS).await.unwrap();+ assert!(store+ .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")+ .await+ .unwrap());// Fresh awareness for the coordinator so the test exercises the lazy// watch bootstrap from pod registrations (see@@ -963,7 +969,10 @@ async fn statefulset_rollout_pod_skips_drain() {tokio::time::sleep(Duration::from_secs(3)).await;let store = test_store("ss-drain-skip-k3s").await;- store.set_total_partitions(NUM_PARTITIONS).await.unwrap();+ assert!(store+ .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")+ .await+ .unwrap());let coord_cancel = CancellationToken::new();let _coord = start_coordinator_k8s(Arc::clone(&store), None, coord_cancel.clone());@@ -1090,7 +1099,10 @@ async fn scale_down_pod_drains_normally() {tokio::time::sleep(Duration::from_secs(3)).await;let store = test_store("scale-down-k3s").await;- store.set_total_partitions(NUM_PARTITIONS).await.unwrap();+ assert!(store+ .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")+ .await+ .unwrap());let cancel = CancellationToken::new();let _coord = start_coordinator_k8s(diff --git a/rust/personhog-coordination/tests/protocol_hardening.rs b/rust/personhog-coordination/tests/protocol_hardening.rsindex 557744d7544a..b1ce656f5d8a 100644--- a/rust/personhog-coordination/tests/protocol_hardening.rs+++ b/rust/personhog-coordination/tests/protocol_hardening.rs@@ -979,7 +979,10 @@ async fn dead_old_owner_handoff_advances_in_place_not_cleaned_up() {let store = test_store("advance-not-cleanup").await;let cancel = CancellationToken::new();- store.set_total_partitions(1).await.expect("partitions");+ assert!(store+ .seed_changelog_config("personhog_updates", 1, "personhog-writer")+ .await+ .expect("partitions"));// The new owner is registered (directly — it never warms, so the// handoff parks in Warming where we can observe it). The old owner@@ -4550,7 +4553,10 @@ async fn a_replaced_handoff_reaches_the_old_owner_it_no_longer_names() {#[tokio::test]async fn a_plan_records_its_freeze_quorum_once_and_collects_it_after() {let store = test_store("freeze-quorum-by-reference").await;- store.set_total_partitions(2).await.expect("set partitions");+ assert!(store+ .seed_changelog_config("personhog_updates", 2, "personhog-writer")+ .await+ .expect("set partitions"));let cancel = CancellationToken::new();// A registered router that never acks parks every handoff in@@ -4857,7 +4863,10 @@ async fn a_leader_that_goes_between_the_read_and_the_watch_is_still_delivered()#[tokio::test]async fn the_sweep_spares_a_membership_a_live_handoff_refers_to() {let store = test_store("freeze-quorum-sweep-spares").await;- store.set_total_partitions(2).await.expect("set partitions");+ assert!(store+ .seed_changelog_config("personhog_updates", 2, "personhog-writer")+ .await+ .expect("set partitions"));let cancel = CancellationToken::new();// A registered router that never acks parks the handoffs indiff --git a/rust/personhog-identity/src/lifecycle/delete.rs b/rust/personhog-identity/src/lifecycle/delete.rsindex 8b023b4b332d..7afb4d66d79a 100644--- a/rust/personhog-identity/src/lifecycle/delete.rs+++ b/rust/personhog-identity/src/lifecycle/delete.rs@@ -28,6 +28,7 @@ use serde::{Deserialize, Serialize};use serde_json::Value;use uuid::Uuid;+use personhog_coordination::changelog::LiveChangelog;use personhog_proto::personhog::types::v1::LifecycleOpType;use crate::config::{IdentityTables, OutboxWrites};@@ -150,9 +151,8 @@ pub struct DeleteDriver {leader: Arc<dyn LifecycleLeader>,tables: IdentityTables,leader_call_concurrency: usize,- /// The leaders' partition count, which decides which victims share a- /// fence call.- num_partitions: u32,+ /// Read per batch so a flip takes effect on the next fence call.+ changelog: LiveChangelog,outbox_writes: OutboxWrites,}@@ -161,17 +161,17 @@ impl DeleteDriver {leader: Arc<dyn LifecycleLeader>,tables: IdentityTables,leader_call_concurrency: usize,- num_partitions: u32,+ changelog: LiveChangelog,outbox_writes: OutboxWrites,) -> Self {tables.validate().expect("invalid identity table set");- assert!(num_partitions > 0, "num_partitions must be > 0");+ assert!(changelog.partitions() > 0, "num_partitions must be > 0");Self {leader,tables,// Clamped to 1: a zero-width buffered stream never polls.leader_call_concurrency: leader_call_concurrency.max(1),- num_partitions,+ changelog,outbox_writes,}}@@ -180,7 +180,7 @@ impl DeleteDriver {LeaderCalls::new(self.leader.as_ref(),self.leader_call_concurrency,- self.num_partitions,+ self.changelog.partitions(),)}}diff --git a/rust/personhog-identity/src/lifecycle/mod.rs b/rust/personhog-identity/src/lifecycle/mod.rsindex aa446402e541..93158fb5ecc0 100644--- a/rust/personhog-identity/src/lifecycle/mod.rs+++ b/rust/personhog-identity/src/lifecycle/mod.rs@@ -18,6 +18,7 @@ use std::sync::Arc;use tonic::{Request, Response, Status};+use personhog_coordination::changelog::LiveChangelog;use personhog_proto::personhog::lifecycle::v1::person_hog_lifecycle_server::PersonHogLifecycle;use personhog_proto::personhog::lifecycle::v1::{DeletePersonOutcome, DeletePersonResult, DeletePersonsRequest, DeletePersonsResponse,@@ -42,7 +43,7 @@ impl PersonHogLifecycleService {leader: Arc<dyn LifecycleLeader>,tables: crate::config::IdentityTables,leader_call_concurrency: usize,- num_partitions: u32,+ changelog: LiveChangelog,outbox_writes: OutboxWrites,) -> Self {Self {@@ -51,7 +52,7 @@ impl PersonHogLifecycleService {leader,tables,leader_call_concurrency,- num_partitions,+ changelog,outbox_writes,),}diff --git a/rust/personhog-identity/src/main.rs b/rust/personhog-identity/src/main.rsindex fd50cd93635c..58bb6a14211d 100644--- a/rust/personhog-identity/src/main.rs+++ b/rust/personhog-identity/src/main.rs@@ -22,6 +22,9 @@ use tracing_subscriber::EnvFilter;use personhog_common::client::RouterClient;use personhog_common::query_tag;+use personhog_coordination::changelog::{+ keep_changelog_live, load_changelog_config, LiveChangelog, CHANGELOG_BOOT_POLL_INTERVAL,+};use personhog_coordination::store::PersonhogStore;use personhog_identity::config::Config;use personhog_identity::leader::LifecycleLeader;@@ -35,6 +38,7 @@ use personhog_identity::service::merge::MergeEntrance;use personhog_identity::service::PersonHogIdentityService;use personhog_identity::storage::postgres::PostgresIdentityStorage;use personhog_identity::warnings;+use tokio_util::sync::CancellationToken;common_alloc::used!();@@ -144,19 +148,22 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {// The delete saga groups its fence calls by the leaders' partition// count, which lives in etcd; a service that cannot read it would// batch wrong, so refuse to start instead.- let num_partitions = {- let etcd_store = EtcdStore::connect(StoreConfig {- endpoints: config.etcd_endpoint_list(),- prefix: config.etcd_prefix.clone(),- })+ let etcd_store = EtcdStore::connect(StoreConfig {+ endpoints: config.etcd_endpoint_list(),+ prefix: config.etcd_prefix.clone(),+ })+ .await+ .expect("Failed to connect to etcd");+ let store = Arc::new(PersonhogStore::new(etcd_store));+ let loaded = load_changelog_config(&store, CHANGELOG_BOOT_POLL_INTERVAL).await- .expect("Failed to connect to etcd");- PersonhogStore::new(etcd_store)- .get_total_partitions()- .await- .expect("Failed to read total_partitions from etcd")- };- tracing::info!(num_partitions, "loaded partition count from etcd");+ .expect("Failed to load the changelog config from etcd");+ tracing::info!(+ num_partitions = loaded.partitions,+ epoch = loaded.epoch,+ "loaded changelog config"+ );+ let changelog = LiveChangelog::new(loaded);// Build lifecycle manager and register componentslet mut manager = Manager::builder("personhog-identity").build();@@ -184,6 +191,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {.is_advisory(true),)});+ let changelog_handle = manager.register("changelog_config", ComponentOptions::new());let readiness = manager.readiness_handler();let liveness = manager.liveness_handler();@@ -218,6 +226,19 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {.unwrap().install_recorder().expect("Failed to install metrics recorder");+ {+ let store = Arc::clone(&store);+ let changelog = changelog.clone();+ tokio::spawn(async move {+ let _guard = changelog_handle.process_scope();+ let cancel = CancellationToken::new();+ tokio::select! {+ _ = changelog_handle.shutdown_signal() => cancel.cancel(),+ _ = keep_changelog_live(store, changelog, cancel.clone()) => {}+ }+ });+ }+// Metrics/health HTTP server (observability handle — stays alive during standard drain)let metrics_port = config.metrics_port;tokio::spawn(async move {@@ -348,7 +369,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {lifecycle_leader.clone(),config.tables(),config.lifecycle_leader_call_concurrency,- num_partitions,+ changelog.clone(),outbox_writes,);let sweeper_engine = engine.clone();@@ -413,7 +434,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {lifecycle_leader,config.tables(),config.lifecycle_leader_call_concurrency,- num_partitions,+ changelog.clone(),outbox_writes,);let service = PersonHogIdentityService::new(diff --git a/rust/personhog-identity/tests/common/mod.rs b/rust/personhog-identity/tests/common/mod.rsindex e8aec3faca99..defbfc2ad68f 100644--- a/rust/personhog-identity/tests/common/mod.rs+++ b/rust/personhog-identity/tests/common/mod.rs@@ -11,6 +11,8 @@ use rand::Rng;use sqlx::postgres::PgPool;use uuid::Uuid;+use personhog_coordination::changelog::LiveChangelog;+use personhog_coordination::types::ChangelogConfig;use personhog_identity::config::{IdentityTables, OutboxWrites};use personhog_identity::pools::IdentityPools;use personhog_identity::publish::config::MergeEventsAllowlist;@@ -32,6 +34,10 @@ pub const FAN_OUT_CONCURRENCY: usize = 8;/// Partition count the delete driver groups its fence batches by.pub const NUM_PARTITIONS: u32 = 4;+pub fn changelog() -> LiveChangelog {+ LiveChangelog::new(ChangelogConfig::new("personhog_updates", NUM_PARTITIONS))+}+pub fn default_tables() -> IdentityTables {IdentityTables::real()}diff --git a/rust/personhog-identity/tests/lifecycle_delete_tests.rs b/rust/personhog-identity/tests/lifecycle_delete_tests.rsindex 9afac8986b07..1e1bf79b9b4c 100644--- a/rust/personhog-identity/tests/lifecycle_delete_tests.rs+++ b/rust/personhog-identity/tests/lifecycle_delete_tests.rs@@ -36,7 +36,7 @@ impl TestContext {leader.clone(),self.tables.clone(),common::FAN_OUT_CONCURRENCY,- common::NUM_PARTITIONS,+ common::changelog(),outbox_writes,)}@@ -597,7 +597,7 @@ impl FencedHarness {leader.clone(),ctx.tables.clone(),common::FAN_OUT_CONCURRENCY,- common::NUM_PARTITIONS,+ common::changelog(),OutboxWrites::Disabled,);Self {diff --git a/rust/personhog-identity/tests/lifecycle_service_tests.rs b/rust/personhog-identity/tests/lifecycle_service_tests.rsindex 861042d428ce..c8dcb939a420 100644--- a/rust/personhog-identity/tests/lifecycle_service_tests.rs+++ b/rust/personhog-identity/tests/lifecycle_service_tests.rs@@ -46,7 +46,7 @@ async fn delete_status(request: DeletePersonsRequest) -> Code {Arc::new(SimLeader::new(pool, tables.clone())),tables,common::FAN_OUT_CONCURRENCY,- common::NUM_PARTITIONS,+ common::changelog(),personhog_identity::config::OutboxWrites::Disabled,);servicediff --git a/rust/personhog-leader/src/config.rs b/rust/personhog-leader/src/config.rsindex 8265a03f3655..6a778dba4d2e 100644--- a/rust/personhog-leader/src/config.rs+++ b/rust/personhog-leader/src/config.rs@@ -140,9 +140,6 @@ pub struct Config {#[envconfig(nested = true)]pub kafka: KafkaConfig,- #[envconfig(default = "personhog_updates")]- pub kafka_person_state_topic: String,-// ── Warming ──────────────────────────────────────────────────/// Consumer group name used by the writer pod. The leader queries this/// group's committed offsets during warming: any Kafka message at ordiff --git a/rust/personhog-leader/src/main.rs b/rust/personhog-leader/src/main.rsindex 1b266f1b428c..34a7d66222e9 100644--- a/rust/personhog-leader/src/main.rs+++ b/rust/personhog-leader/src/main.rs@@ -14,6 +14,7 @@ use personhog_common::async_gzip::{AsyncGzipConfig, AsyncGzipLayer};use personhog_common::grpc::{tracked_tcp_incoming, GrpcLoadShedLayer, GrpcMetricsLayer};use personhog_common::metrics::{WARM_LATENCY_BUCKETS_MS, WRITE_PATH_LATENCY_BUCKETS_MS};use personhog_coordination::authority::AuthorityClock;+use personhog_coordination::changelog::{load_changelog_config, CHANGELOG_BOOT_POLL_INTERVAL};use personhog_coordination::pod::{PodConfig, PodHandle};use personhog_coordination::store::PersonhogStore;use personhog_proto::personhog::leader::v1::person_hog_leader_server::PersonHogLeaderServer;@@ -40,7 +41,9 @@ use personhog_leader::pg::{validate_table_name, LifecycleTables, PgFallback};use personhog_leader::recovery::{ChangelogRecovery, RecoveryConfig};use personhog_leader::service::{sweep_idle_locks, PersonHogLeaderService, PropertySizeLimits};use personhog_leader::settle::prune_and_settle_tick;-use personhog_leader::warming::{WarmClientPools, WarmingConfig, WarmingRetryPolicy};+use personhog_leader::warming::{+ changelog_partition_count, WarmClientPools, WarmingConfig, WarmingRetryPolicy,+};use personhog_leader::warnings::WarningsProducer;common_alloc::used!();@@ -96,7 +99,6 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {tracing::info!("etcd endpoints: {}", config.etcd_endpoints);tracing::info!("etcd prefix: {}", config.etcd_prefix);tracing::info!("Pod name: {}", config.pod_name);- tracing::info!("Kafka changelog topic: {}", config.kafka_person_state_topic);// Shutdown order matters: the coordination drain (phase 0) hands this// pod's partitions off, which requires the gRPC server to keep serving@@ -349,14 +351,32 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {.expect("Failed to connect to etcd");let store = Arc::new(PersonhogStore::new(etcd_store));- // Read total_partitions from etcd (set by kafka-assigner) — the same- // source the router hashes against, so partition validation can never- // drift between the two.- let num_partitions = store- .get_total_partitions()+ // The same source the router hashes against.+ let loaded = load_changelog_config(&store, CHANGELOG_BOOT_POLL_INTERVAL).await- .expect("Failed to read total_partitions from etcd");- tracing::info!(num_partitions, "loaded partition count from etcd");+ .expect("Failed to load the changelog config from etcd");+ let changelog_topic = loaded.topic.clone();+ let num_partitions = loaded.partitions;+ let topic_partitions =+ changelog_partition_count(&config.kafka, &changelog_topic, Duration::from_secs(10))+ .await+ .expect("Failed to read the changelog topic's metadata");+ assert_eq!(+ topic_partitions, num_partitions,+ "changelog topic {changelog_topic} has {topic_partitions} partitions but the config says {num_partitions}"+ );+ // Retirement settles marks on the writer group the config names, so it must be ours.+ assert_eq!(+ loaded.writer_group, config.writer_consumer_group,+ "config/changelog names writer group {} but WRITER_CONSUMER_GROUP is {}",+ loaded.writer_group, config.writer_consumer_group+ );+ tracing::info!(+ num_partitions,+ topic = %changelog_topic,+ epoch = loaded.epoch,+ "loaded changelog config"+ );let locks = Arc::new(DashMap::new());let fences: personhog_leader::fence::FenceMap = Arc::new(DashMap::new());@@ -372,7 +392,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {let recovery = Arc::new(ChangelogRecovery::new(RecoveryConfig {kafka: config.kafka.clone(),- topic: config.kafka_person_state_topic.clone(),+ topic: changelog_topic.clone(),pod_name: config.pod_name.clone(),recv_timeout: Duration::from_secs(config.recovery_recv_timeout_secs),pool_size: config.recovery_pool_size,@@ -413,7 +433,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {let fenced = Arc::new(FencedChangelogProducers::new(FencedProducerConfig {kafka: fencing_kafka,- topic: config.kafka_person_state_topic.clone(),+ topic: changelog_topic.clone(),init_timeout: config.fencing_init_timeout(),commit_timeout: config.fencing_txn_timeout(),broker_txn_timeout: config.fencing_broker_txn_timeout(),@@ -503,7 +523,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {Arc::clone(&dirty_index),WarmingConfig {kafka: config.kafka.clone(),- topic: config.kafka_person_state_topic.clone(),+ topic: changelog_topic.clone(),pod_name: config.pod_name.clone(),writer_consumer_group: config.writer_consumer_group.clone(),lookback_offsets: config.warm_lookback_offsets,@@ -650,7 +670,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {Arc::clone(&cache),Arc::clone(&locks),Arc::clone(&warm_pools),- config.kafka_person_state_topic.clone(),+ changelog_topic.clone(),Duration::from_secs(config.warm_committed_offsets_timeout_secs),Duration::from_secs(config.dirty_index_prune_interval_secs.max(1)),));diff --git a/rust/personhog-leader/src/warming.rs b/rust/personhog-leader/src/warming.rsindex 86da0e1adb36..55c73a14c5a9 100644--- a/rust/personhog-leader/src/warming.rs+++ b/rust/personhog-leader/src/warming.rs@@ -112,6 +112,43 @@ pub struct WarmingConfig {pub retry: WarmingRetryPolicy,}+/// The topic's partition count from broker metadata, for the boot check.+pub async fn changelog_partition_count(+ kafka: &KafkaConfig,+ topic: &str,+ timeout: Duration,+) -> CoordResult<u32> {+ let consumer = make_consumer(kafka, "personhog-leader-boot")+ .map_err(|e| CoordError::invalid_state(format!("create metadata client: {e}")))?;+ let topic_name = topic.to_string();+ tokio::task::spawn_blocking(move || {+ let metadata = consumer+ .fetch_metadata(Some(&topic_name), timeout)+ .map_err(|e| {+ CoordError::invalid_state(format!("fetch metadata for {topic_name}: {e}"))+ })?;+ let topic_metadata = metadata+ .topics()+ .iter()+ .find(|t| t.name() == topic_name)+ .ok_or_else(|| {+ CoordError::invalid_state(format!(+ "topic {topic_name} is absent from broker metadata"+ ))+ })?;+ if let Some(error) = topic_metadata.error() {+ return Err(CoordError::invalid_state(format!(+ "topic {topic_name}: {error:?}"+ )));+ }+ u32::try_from(topic_metadata.partitions().len()).map_err(|_| {+ CoordError::invalid_state(format!("topic {topic_name} partition count overflow"))+ })+ })+ .await+ .map_err(|e| CoordError::invalid_state(format!("fetch metadata join: {e}")))?+}+/// Build a consumer used by warming. Auto-commit and auto-offset-store are/// disabled so this is safe to instantiate with any group id — including/// the writer's, where mutating stored offsets would corrupt the writer'sdiff --git a/rust/personhog-leader/tests/integration.rs b/rust/personhog-leader/tests/integration.rsindex f9f5b6ae9547..33ebef66a429 100644--- a/rust/personhog-leader/tests/integration.rs+++ b/rust/personhog-leader/tests/integration.rs@@ -71,7 +71,10 @@ async fn service_accepts_requests_after_coordination_warmup() {let store = test_store("leader-warmup").await;let cancel = CancellationToken::new();- store.set_total_partitions(NUM_PARTITIONS).await.unwrap();+ assert!(store+ .seed_changelog_config(common::CHANGELOG_TOPIC, NUM_PARTITIONS, "personhog-writer")+ .await+ .unwrap());let _coord = start_coordinator(Arc::clone(&store),@@ -700,7 +703,10 @@ async fn release_partition_stops_serving() {let store = test_store("leader-release").await;let cancel = CancellationToken::new();- store.set_total_partitions(NUM_PARTITIONS).await.unwrap();+ assert!(store+ .seed_changelog_config(common::CHANGELOG_TOPIC, NUM_PARTITIONS, "personhog-writer")+ .await+ .unwrap());let _coord = start_coordinator(Arc::clone(&store),@@ -818,7 +824,10 @@ async fn rewarm_after_pod_crash() {let store = test_store("leader-rewarm").await;let cancel = CancellationToken::new();- store.set_total_partitions(NUM_PARTITIONS).await.unwrap();+ assert!(store+ .seed_changelog_config(common::CHANGELOG_TOPIC, NUM_PARTITIONS, "personhog-writer")+ .await+ .unwrap());let _coord = start_coordinator(Arc::clone(&store),diff --git a/rust/personhog-router/src/backend/leader.rs b/rust/personhog-router/src/backend/leader.rsindex c9d60e6015c2..2b408cbadd73 100644--- a/rust/personhog-router/src/backend/leader.rs+++ b/rust/personhog-router/src/backend/leader.rs@@ -21,6 +21,7 @@ use personhog_common::grpc::{current_client_name, SEMANTIC_REFUSAL_METADATA_KEY}const FENCED_METADATA_KEY: &str = "x-person-fenced";const FENCED_OP_ID_METADATA_KEY: &str = "x-person-fenced-op-id";use personhog_common::partitioning::partition_for_person;+use personhog_coordination::changelog::LiveChangelog;use super::stash::{StashDecision, StashTable};use crate::config::Http2Windows;@@ -117,7 +118,7 @@ pub enum ForwardDecision {/// Static configuration for `LeaderBackend`. Bundles the knobs that come/// from `Config` so the constructor stays narrow as we add fields.pub struct LeaderBackendConfig {- pub num_partitions: u32,+ pub changelog: LiveChangelog,pub timeout: Duration,pub num_channels: usize,pub http2_windows: Http2Windows,@@ -177,8 +178,8 @@ impl LeaderBackend {stash: StashTable,) -> Self {assert!(- config.num_partitions > 0,- "num_partitions must be > 0 to avoid division by zero in partition_for_person"+ config.changelog.partitions() > 0,+ "the changelog must have partitions to avoid division by zero in partition_for_person");Self {routing_table,@@ -195,6 +196,11 @@ impl LeaderBackend {self.stash.clone()}+ /// The live config this backend routes on.+ pub fn changelog(&self) -> &LiveChangelog {+ &self.config.changelog+ }+/// Remove the cached gRPC channel for a pod so the next request/// reconnects. Called during partition handoff cutover to drop the/// connection to the old leader pod.@@ -207,7 +213,7 @@ impl LeaderBackend {/// Compute the Kafka partition for a person using murmur2./// The key is `team_id:person_id`, matching the Kafka topic key.pub fn partition_for_person(&self, team_id: i64, person_id: i64) -> u32 {- partition_for_person(team_id, person_id, self.config.num_partitions)+ partition_for_person(team_id, person_id, self.config.changelog.partitions())}/// Resolve the leader gRPC channel for a given partition, building and@@ -543,10 +549,15 @@ mod tests {}use super::*;+ use personhog_coordination::types::ChangelogConfig;++ fn changelog(num_partitions: u32) -> LiveChangelog {+ LiveChangelog::new(ChangelogConfig::new("personhog_updates", num_partitions))+ }fn test_config(num_partitions: u32) -> LeaderBackendConfig {LeaderBackendConfig {- num_partitions,+ changelog: changelog(num_partitions),timeout: Duration::from_secs(5),num_channels: 4,http2_windows: Http2Windows::default(),diff --git a/rust/personhog-router/src/main.rs b/rust/personhog-router/src/main.rsindex 6aefb619af9a..3035bdee6ad4 100644--- a/rust/personhog-router/src/main.rs+++ b/rust/personhog-router/src/main.rs@@ -9,6 +9,9 @@ use lifecycle::{ComponentOptions, Handle, Manager};use metrics_exporter_prometheus::{Matcher, PrometheusBuilder, PrometheusHandle};use personhog_common::grpc::{tracked_tcp_incoming, GrpcMetricsLayer};use personhog_common::metrics::WRITE_PATH_LATENCY_BUCKETS_MS;+use personhog_coordination::changelog::{+ keep_changelog_live, load_changelog_config, LiveChangelog, CHANGELOG_BOOT_POLL_INTERVAL,+};use personhog_coordination::coordinator::{Coordinator, CoordinatorConfig};use personhog_coordination::routing_table::{RoutingTable, RoutingTableConfig, StashHandler};use personhog_coordination::store::PersonhogStore;@@ -151,6 +154,8 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {let identity_discovery_handle = (config.identity_enabled&& config.identity_discovery_mode == DiscoveryMode::K8s).then(|| manager.register("identity-discovery", discovery_options()));+ let changelog_handle = (config.router_mode == RouterMode::Leader)+ .then(|| manager.register("changelog-config", ComponentOptions::new()));let readiness = manager.readiness_handler();let liveness = manager.liveness_handler();@@ -253,12 +258,16 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {.expect("Failed to connect to etcd");let store = Arc::new(PersonhogStore::new(etcd_store));- // Read total_partitions from etcd (set by kafka-assigner)- let num_partitions = store- .get_total_partitions()+ let loaded = load_changelog_config(&store, CHANGELOG_BOOT_POLL_INTERVAL).await- .expect("Failed to read total_partitions from etcd");- tracing::info!(num_partitions, "loaded partition count from etcd");+ .expect("Failed to load the changelog config from etcd");+ let num_partitions = loaded.partitions;+ tracing::info!(+ num_partitions,+ epoch = loaded.epoch,+ "loaded changelog config"+ );+ let changelog = LiveChangelog::new(loaded);// Build the routing table and leader backend, sharing the same// partition-to-pod mapping so both see consistent state.@@ -294,7 +303,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {.map(|address| format!("http://{address}"))}),LeaderBackendConfig {- num_partitions,+ changelog,timeout: config.backend_timeout(),num_channels: config.leader_channels,http2_windows: config.backend_http2_windows(),@@ -305,6 +314,22 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {),));+ // New requests hash under the live partition count.+ {+ let store = Arc::clone(&store);+ let changelog = leader_backend.changelog().clone();+ let handle =+ changelog_handle.expect("changelog handle must be registered in leader mode");+ tokio::spawn(async move {+ let _guard = handle.process_scope();+ let cancel = CancellationToken::new();+ tokio::select! {+ _ = handle.shutdown_signal() => cancel.cancel(),+ _ = keep_changelog_live(store, changelog, cancel.clone()) => {}+ }+ });+ }+let stash_handler: Arc<dyn StashHandler> = Arc::new(RouterStashHandler::new(Arc::clone(&leader_backend),config.stash_max_wait(),diff --git a/rust/personhog-router/tests/common/mod.rs b/rust/personhog-router/tests/common/mod.rsindex de96f4fffeda..c22cd5e4dd44 100644--- a/rust/personhog-router/tests/common/mod.rs+++ b/rust/personhog-router/tests/common/mod.rs@@ -9,6 +9,8 @@ use std::time::Duration;use dashmap::DashMap;use personhog_common::async_gzip::{AsyncGzipConfig, AsyncGzipLayer};use personhog_common::grpc::NOT_APPLIED_HEADER;+use personhog_coordination::changelog::LiveChangelog;+use personhog_coordination::types::ChangelogConfig;use personhog_proto::personhog::identity::v1 as identity;use personhog_proto::personhog::identity::v1::person_hog_identity_server::{PersonHogIdentity, PersonHogIdentityServer,@@ -987,11 +989,12 @@ fn make_leader_backend(leader_addr: SocketAddr, num_partitions: u32) -> Arc<Leadlet leader_url = format!("http://{}", leader_addr);let address_resolver: Arc<dyn Fn(&str) -> Option<String> + Send + Sync> =Arc::new(move |_pod_name| Some(leader_url.clone()));+ let changelog = LiveChangelog::new(ChangelogConfig::new("personhog_updates", num_partitions));Arc::new(LeaderBackend::new(routing_table,address_resolver,LeaderBackendConfig {- num_partitions,+ changelog,timeout: Duration::from_secs(5),num_channels: 1,http2_windows: Http2Windows::default(),@@ -1021,11 +1024,12 @@ fn make_dying_leader_backend(leader_addr: SocketAddr, num_partitions: u32) -> ArSome("http://192.0.2.1:1".to_string())}});+ let changelog = LiveChangelog::new(ChangelogConfig::new("personhog_updates", num_partitions));Arc::new(LeaderBackend::new(routing_table,address_resolver,LeaderBackendConfig {- num_partitions,+ changelog,timeout: Duration::from_millis(200),num_channels: 1,http2_windows: Http2Windows::default(),diff --git a/rust/personhog-router/tests/h2_window_wiring.rs b/rust/personhog-router/tests/h2_window_wiring.rsindex 5f2d696d4c58..a6dc9634c6bd 100644--- a/rust/personhog-router/tests/h2_window_wiring.rs+++ b/rust/personhog-router/tests/h2_window_wiring.rs@@ -9,6 +9,8 @@ use common::{h2_frame, read_h2_frame, H2Frame, TestLeaderService, H2_FRAME_SETTINGS, H2_FRAME_WINDOW_UPDATE,H2_PREFACE,};+use personhog_coordination::changelog::LiveChangelog;+use personhog_coordination::types::ChangelogConfig;use personhog_proto::personhog::leader::v1::person_hog_leader_server::PersonHogLeaderServer;use personhog_router::backend::discovery::{build_discovered_endpoint, EndpointConfig};use personhog_router::backend::{@@ -136,11 +138,12 @@ fn discovered_channel(addr: SocketAddr, http2_windows: Http2Windows) -> Channelasync fn leader_channel(addr: SocketAddr, http2_windows: Http2Windows) -> Channel {let mut routing = HashMap::new();routing.insert(0, "leader-0".to_string());+ let changelog = LiveChangelog::new(ChangelogConfig::new("personhog_updates", 1));let backend = LeaderBackend::new(Arc::new(RwLock::new(routing)),Arc::new(move |_| Some(format!("http://{addr}"))),LeaderBackendConfig {- num_partitions: 1,+ changelog,timeout: Duration::from_secs(5),num_channels: 1,http2_windows,diff --git a/rust/personhog-router/tests/stash_integration.rs b/rust/personhog-router/tests/stash_integration.rsindex 058e4bfaa5d4..bb3a8af5fc6a 100644--- a/rust/personhog-router/tests/stash_integration.rs+++ b/rust/personhog-router/tests/stash_integration.rs@@ -15,7 +15,9 @@ use bytes::Bytes;use common::{create_test_person, start_test_leader, start_test_leader_at, TestLeaderService};use http::HeaderMap;use http_body_util::BodyExt;+use personhog_coordination::changelog::LiveChangelog;use personhog_coordination::routing_table::StashHandler;+use personhog_coordination::types::ChangelogConfig;use personhog_proto::personhog::types::v1::{FoldPersonDocumentRequest, FoldPersonDocumentResponse, GetPersonRequest, GetPersonResponse,Person, SealedSourceSnapshot, UpdatePersonPropertiesRequest, UpdatePersonPropertiesResponse,@@ -64,7 +66,10 @@ async fn make_backend(leader_addr: std::net::SocketAddr, stash: StashTable) -> Arouting_table,resolver,LeaderBackendConfig {- num_partitions: NUM_PARTITIONS,+ changelog: LiveChangelog::new(ChangelogConfig::new(+ "personhog_updates",+ NUM_PARTITIONS,+ )),timeout: Duration::from_secs(5),num_channels: 1,http2_windows: Http2Windows::default(),diff --git a/rust/personhog-test-harness/src/cli.rs b/rust/personhog-test-harness/src/cli.rsindex 96b7e3bd0985..595c2f5df2ee 100644--- a/rust/personhog-test-harness/src/cli.rs+++ b/rust/personhog-test-harness/src/cli.rs@@ -660,14 +660,13 @@ pub struct TrafficArgs {/// Environment the stack assigns per leader; overriding any of these/// would break pod identity or point a spawned leader at the wrong-/// topic or table, and the resulting run would look like a real result.+/// table, and the resulting run would look like a real result.const RESERVED_LEADER_ENV: &[&str] = &["POD_NAME","GRPC_ADDRESS","METRICS_PORT","ETCD_ENDPOINTS","ETCD_PREFIX",- "KAFKA_PERSON_STATE_TOPIC","FALLBACK_TABLE","LIFECYCLE_OP_TABLE","LIFECYCLE_OP_PERSON_TABLE",diff --git a/rust/personhog-test-harness/src/stack/etcd.rs b/rust/personhog-test-harness/src/stack/etcd.rsindex bb501a077f4f..7eeea79eb836 100644--- a/rust/personhog-test-harness/src/stack/etcd.rs+++ b/rust/personhog-test-harness/src/stack/etcd.rs@@ -14,14 +14,18 @@ pub async fn connect(endpoints: &str, prefix: &str) -> Result<PersonhogStore> {Ok(PersonhogStore::new(store))}-/// Wipe all coordination state under the harness prefix and set the-/// partition count, giving each run a clean slate.-pub async fn reset(store: &PersonhogStore, partitions: u32) -> Result<()> {+/// Wipe the harness prefix and seed the config as the control plane would.+pub async fn reset(+ store: &PersonhogStore,+ topic: &str,+ partitions: u32,+ writer_group: &str,+) -> Result<()> {store.delete_all().await.context("clearing etcd prefix")?;store- .set_total_partitions(partitions)+ .seed_changelog_config(topic, partitions, writer_group).await- .context("setting total_partitions")?;+ .context("seeding config/changelog")?;Ok(())}diff --git a/rust/personhog-test-harness/src/stack/kafka.rs b/rust/personhog-test-harness/src/stack/kafka.rsindex 360056a46f41..580db3b417b4 100644--- a/rust/personhog-test-harness/src/stack/kafka.rs+++ b/rust/personhog-test-harness/src/stack/kafka.rs@@ -19,7 +19,7 @@ fn admin_client(kafka_hosts: &str) -> Result<AdminClient<DefaultClientContext>>}/// Create a per-run topic with exactly `partitions` partitions. For the-/// changelog the count must match `total_partitions` in etcd — a mismatch+/// changelog the count must match `config/changelog` in etcd — a mismatch/// is a silent mis-shard — so an existing topic with a different count is/// an error rather than something to adopt.pub async fn create_topic(kafka_hosts: &str, topic: &str, partitions: u32) -> Result<()> {diff --git a/rust/personhog-test-harness/src/stack/mod.rs b/rust/personhog-test-harness/src/stack/mod.rsindex bc8518a81722..531bbfd6ecd1 100644--- a/rust/personhog-test-harness/src/stack/mod.rs+++ b/rust/personhog-test-harness/src/stack/mod.rs@@ -32,6 +32,7 @@ const REPLICA_METRICS_PORT: u16 = 24151;const IDENTITY_METRICS_PORT: u16 = 24152;const WRITER_METRICS_PORT: u16 = 24153;const ROUTER_METRICS_BASE_PORT: u16 = 24154;+const WRITER_CONSUMER_GROUP: &str = "personhog-test-harness-writer";const LEADER_METRICS_BASE_PORT: u16 = 24160;/// Above the leader range, which grows with every scale-up.const PUBLISHER_METRICS_BASE_PORT: u16 = 24200;@@ -228,7 +229,7 @@ impl Stack {);let store = etcd::connect(&config.etcd_endpoints, ETCD_PREFIX).await?;- etcd::reset(&store, config.partitions).await?;+ etcd::reset(&store, &topic, config.partitions, WRITER_CONSUMER_GROUP).await?;let publish_topics = config.publish.as_ref().map(|_| PublishTopics {distinct_ids: format!("{topic}_distinct_ids"),merge_events: format!("{topic}_merge_events"),@@ -259,11 +260,9 @@ impl Stack {&[("DATABASE_URL", config.persons_db_url.clone()),("KAFKA_HOSTS", config.kafka_hosts.clone()),- ("KAFKA_TOPIC", topic.clone()),- (- "KAFKA_CONSUMER_GROUP",- "personhog-test-harness-writer".to_string(),- ),+ ("ETCD_ENDPOINTS", config.etcd_endpoints.clone()),+ ("ETCD_PREFIX", ETCD_PREFIX.to_string()),+ ("KAFKA_CONSUMER_GROUP", WRITER_CONSUMER_GROUP.to_string()),("PG_TARGET_TABLE", config.pg_target_table.clone()),("FLUSH_INTERVAL_MS",@@ -527,14 +526,10 @@ impl Stack {("ETCD_ENDPOINTS", self.config.etcd_endpoints.clone()),("ETCD_PREFIX", ETCD_PREFIX.to_string()),("KAFKA_HOSTS", self.config.kafka_hosts.clone()),- ("KAFKA_PERSON_STATE_TOPIC", self.topic.clone()),// Must match the spawned writer's consumer group or the// leader's committed-offset queries (warming ranges, dirty// index pruning) silently see no progress at all.- (- "WRITER_CONSUMER_GROUP",- "personhog-test-harness-writer".to_string(),- ),+ ("WRITER_CONSUMER_GROUP", WRITER_CONSUMER_GROUP.to_string()),("FALLBACK_DATABASE_URL", self.config.persons_db_url.clone()),// Pair the fallback with the spawned writer's target: the// dirty index treats an unmarked person's PG row asdiff --git a/rust/personhog-writer/Cargo.toml b/rust/personhog-writer/Cargo.tomlindex 027cfe866cdf..b5bb4f39afe5 100644--- a/rust/personhog-writer/Cargo.toml+++ b/rust/personhog-writer/Cargo.toml@@ -12,6 +12,8 @@ common-alloc = { path = "../common/alloc" }common-liveness = { path = "../common/liveness" }common-metrics = { path = "../common/metrics" }lifecycle = { path = "../common/lifecycle" }+assignment-coordination = { path = "../common/assignment-coordination" }+personhog-coordination = { path = "../personhog-coordination" }async-trait = { workspace = true }axum = { workspace = true }diff --git a/rust/personhog-writer/README.md b/rust/personhog-writer/README.mdindex e5c61cfa589f..3d19597b83da 100644--- a/rust/personhog-writer/README.md+++ b/rust/personhog-writer/README.md@@ -136,9 +136,10 @@ Consumer lag per partition is monitored externally via KMinion (`kminion_kafka_c| `KAFKA_TLS` | `false` | Enable TLS for Kafka || `KAFKA_CLIENT_ID` | (empty) | Pod name for static group membership || `KAFKA_CLIENT_RACK` | (empty) | Rack ID for rack-aware consumption |-| `KAFKA_TOPIC` | `personhog_updates` | Topic to consume || `KAFKA_CONSUMER_GROUP` | `personhog-writer` | Consumer group ID || `KAFKA_CONSUMER_OFFSET_RESET` | `earliest` | Offset reset policy |+| `ETCD_ENDPOINTS` | `http://localhost:2379` | etcd endpoints; the topic comes from `config/changelog` |+| `ETCD_PREFIX` | `/personhog/` | etcd key prefix || `DATABASE_URL` | (required) | Postgres connection string || `PG_MAX_CONNECTIONS` | `20` | Connection pool size || `PG_TARGET_TABLE` | `personhog_person_tmp` | Target table (`posthog_person` for production cutover) |diff --git a/rust/personhog-writer/src/config.rs b/rust/personhog-writer/src/config.rsindex 773e802e4360..d8852efdc20f 100644--- a/rust/personhog-writer/src/config.rs+++ b/rust/personhog-writer/src/config.rs@@ -9,9 +9,6 @@ pub struct Config {#[envconfig(nested = true)]pub kafka: KafkaConfig,- #[envconfig(default = "personhog_updates")]- pub kafka_topic: String,-#[envconfig(default = "personhog-writer")]pub kafka_consumer_group: String,@@ -80,6 +77,13 @@ pub struct Config {#[envconfig(default = "1")]pub writer_lanes: usize,+ // ── etcd ─────────────────────────────────────────────────────+ #[envconfig(default = "http://localhost:2379")]+ pub etcd_endpoints: String,++ #[envconfig(default = "/personhog/")]+ pub etcd_prefix: String,+// ── Service ──────────────────────────────────────────────────#[envconfig(default = "9103")]pub metrics_port: u16,@@ -89,4 +93,13 @@ impl Config {pub fn flush_interval(&self) -> Duration {Duration::from_millis(self.flush_interval_ms)}++ pub fn etcd_endpoint_list(&self) -> Vec<String> {+ self.etcd_endpoints+ .split(',')+ .map(str::trim)+ .filter(|s| !s.is_empty())+ .map(String::from)+ .collect()+ }}diff --git a/rust/personhog-writer/src/main.rs b/rust/personhog-writer/src/main.rsindex ff7aceec0b47..6f2bcb812978 100644--- a/rust/personhog-writer/src/main.rs+++ b/rust/personhog-writer/src/main.rs@@ -1,11 +1,14 @@use std::sync::Arc;use std::time::Duration;+use assignment_coordination::store::{EtcdStore, StoreConfig};use axum::{routing::get, Router};use common_database::{get_pool_with_config, PoolConfig};use common_metrics::{setup_metrics_routes_with_overrides, Matcher};use envconfig::Envconfig;use lifecycle::{ComponentOptions, Manager};+use personhog_coordination::changelog::{load_changelog_config, CHANGELOG_BOOT_POLL_INTERVAL};+use personhog_coordination::store::PersonhogStore;use tokio::sync::mpsc;use tracing::level_filters::LevelFilter;use tracing_subscriber::fmt;@@ -41,7 +44,6 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {.init();tracing::info!("Starting personhog-writer");- tracing::info!("Kafka topic: {}", config.kafka_topic);tracing::info!("Consumer group: {}", config.kafka_consumer_group);tracing::info!("Flush interval: {}ms", config.flush_interval_ms);tracing::info!("Flush buffer size: {}", config.flush_buffer_size);@@ -50,6 +52,26 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {tracing::info!("Upsert concurrency: {}", config.upsert_concurrency);tracing::info!("Metrics port: {}", config.metrics_port);+ let store = EtcdStore::connect(StoreConfig {+ endpoints: config.etcd_endpoint_list(),+ prefix: config.etcd_prefix.clone(),+ })+ .await?;+ let loaded =+ load_changelog_config(&PersonhogStore::new(store), CHANGELOG_BOOT_POLL_INTERVAL).await?;+ if loaded.writer_group != config.kafka_consumer_group {+ return Err(format!(+ "config/changelog names writer group {} but KAFKA_CONSUMER_GROUP is {}",+ loaded.writer_group, config.kafka_consumer_group+ )+ .into());+ }+ tracing::info!(+ topic = %loaded.topic,+ epoch = loaded.epoch,+ "loaded changelog config"+ );+let mut manager = Manager::builder("personhog-writer").with_global_shutdown_timeout(Duration::from_secs(30)).build();@@ -152,9 +174,9 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {&config.kafka,&config.kafka_consumer_group,&config.kafka_consumer_offset_reset,- config.kafka_topic.clone(),+ loaded.topic.clone(),)?);- tracing::info!("Subscribed to Kafka topic: {}", config.kafka_topic);+ tracing::info!("Subscribed to Kafka topic: {}", loaded.topic);// One statement-concurrency budget for the whole pod: every lane's// chunk and per-row statements draw from it, so lanes can overlap