Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(personhog): read the changelog topic and count from config/changelog (#119809)

Zach Brown010d107868e3
diff --git a/bin/mprocs.yaml b/bin/mprocs.yaml
index 0b8aab85a996..a9a50d949d8d 100755
--- a/bin/mprocs.yaml
+++ b/bin/mprocs.yaml
@@ -494,7 +494,7 @@ procs:
personhog-etcd-init:
shell: |-
bin/wait-for-docker --only etcd && \
- docker exec posthog-etcd-1 etcdctl put /personhog/config/total_partitions 4 && \
+ docker exec posthog-etcd-1 etcdctl put /personhog/config/changelog '{"topic":"personhog_updates","partitions":4,"epoch":1,"writer_group":"personhog-writer","draining":[]}' && \
echo 'etcd init complete'
capability: personhog
ready_pattern: 'etcd init complete'
diff --git a/bin/start-rust-service b/bin/start-rust-service
index ab1b5a357d64..558f9d97098b 100755
--- a/bin/start-rust-service
+++ b/bin/start-rust-service
@@ -376,7 +376,6 @@ case "$RS_SVC" in
# In production, pod names resolve via K8s DNS.
export POD_NAME=127.0.0.1
export KAFKA_HOSTS="${KAFKA_HOSTS:-localhost:9092}"
- export KAFKA_PERSON_STATE_TOPIC=personhog_updates
export FALLBACK_DATABASE_URL="postgres://posthog:posthog@${PGHOST:-localhost}:${PGPORT:-5432}/posthog_persons"
# Pair with the identity service's and the writer's default table, or
# leader cache misses read a table the identity never wrote.
@@ -389,7 +388,8 @@ case "$RS_SVC" in
export RUST_BACKTRACE=1
export DATABASE_URL="postgres://posthog:posthog@${PGHOST:-localhost}:${PGPORT:-5432}/posthog_persons"
export KAFKA_HOSTS="${KAFKA_HOSTS:-localhost:9092}"
- export KAFKA_TOPIC=personhog_updates
+ export ETCD_ENDPOINTS=http://localhost:2379
+ export ETCD_PREFIX=/personhog/
export KAFKA_CONSUMER_GROUP=personhog-writer
export FLUSH_INTERVAL_MS=5000
export FLUSH_BUFFER_SIZE=1000
diff --git a/docker-compose.base.yml b/docker-compose.base.yml
index 59060a994113..db7619db5e5b 100644
--- a/docker-compose.base.yml
+++ b/docker-compose.base.yml
@@ -583,7 +583,6 @@ services:
ETCD_PREFIX: '/personhog/'
POD_NAME: 'personhog-leader'
KAFKA_HOSTS: 'kafka:9092'
- KAFKA_PERSON_STATE_TOPIC: 'personhog_updates'
FALLBACK_DATABASE_URL: 'postgres://posthog:posthog@db:5432/posthog_persons'
# Must name the same table the identity service and the writer
# target (their defaults), or leader misses read the wrong table.
@@ -601,7 +600,8 @@ services:
environment:
DATABASE_URL: 'postgres://posthog:posthog@db:5432/posthog_persons'
KAFKA_HOSTS: 'kafka:9092'
- KAFKA_TOPIC: 'personhog_updates'
+ ETCD_ENDPOINTS: 'http://etcd:2379'
+ ETCD_PREFIX: '/personhog/'
KAFKA_CONSUMER_GROUP: 'personhog-writer'
FLUSH_INTERVAL_MS: '5000'
FLUSH_BUFFER_SIZE: '1000'
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 3232173c76f1..ea6b820bd501 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -729,10 +729,14 @@ services:
environment:
DATABASE_URL: postgres://posthog:posthog@db:5432/${POSTHOG_PERSONS_DB_NAME:-posthog}
depends_on:
+ etcd:
+ condition: service_healthy
db:
condition: service_healthy
kafka:
condition: service_started
+ personhog-etcd-init:
+ condition: service_completed_successfully
profiles:
- ingestion
@@ -752,12 +756,19 @@ services:
profiles:
- ingestion
- # Seeds the partition count the leaders and routers coordinate under;
+ # Seeds the changelog config the personhog services read at boot;
# idempotent, and the same key bin/mprocs.yaml writes for hogli runs.
personhog-etcd-init:
image: quay.io/coreos/etcd:v3.6.8
restart: 'no'
- entrypoint: ['etcdctl', '--endpoints=http://etcd:2379', 'put', '/personhog/config/total_partitions', '4']
+ entrypoint:
+ [
+ 'etcdctl',
+ '--endpoints=http://etcd:2379',
+ 'put',
+ '/personhog/config/changelog',
+ '{"topic":"personhog_updates","partitions":4,"epoch":1,"writer_group":"personhog-writer","draining":[]}',
+ ]
depends_on:
etcd:
condition: service_healthy
diff --git a/rust/Cargo.lock b/rust/Cargo.lock
index e96ef7757838..23443538905d 100644
--- a/rust/Cargo.lock
+++ b/rust/Cargo.lock
@@ -9156,6 +9156,7 @@ dependencies = [
name = "personhog-writer"
version = "0.1.0"
dependencies = [
+ "assignment-coordination",
"async-trait",
"axum 0.7.9",
"chrono",
@@ -9170,6 +9171,7 @@ dependencies = [
"lifecycle",
"metrics",
"personhog-common",
+ "personhog-coordination",
"personhog-proto",
"prost 0.13.5",
"rdkafka",
diff --git a/rust/ingestion-control-plane/src/personhog.rs b/rust/ingestion-control-plane/src/personhog.rs
index 9c4a30de0cdb..5ffd2b9fe004 100644
--- a/rust/ingestion-control-plane/src/personhog.rs
+++ b/rust/ingestion-control-plane/src/personhog.rs
@@ -63,7 +63,6 @@ pub struct RawSnapshot {
pub drained_acks: Vec<PodDrainedAck>,
pub warmed_acks: Vec<PodWarmedAck>,
pub leader: Option<(LeaderInfo, i64)>,
- pub total_partitions: Option<u32>,
pub changelog: Option<ChangelogConfig>,
pub parse_errors: Vec<String>,
pub unknown_keys: Vec<String>,
@@ -128,14 +127,6 @@ pub fn parse_snapshot(kvs: &[RawKv], prefix: &str, revision: i64) -> RawSnapshot
if let Some(leader) = parse::<LeaderInfo>(kv, errors) {
snapshot.leader = Some((leader, kv.lease));
}
- } else if suffix == "config/total_partitions" {
- match std::str::from_utf8(&kv.value)
- .ok()
- .and_then(|s| s.parse::<u32>().ok())
- {
- Some(count) => snapshot.total_partitions = Some(count),
- None => errors.push(format!("invalid total_partitions at {}", kv.key)),
- }
} else if suffix == "config/changelog" {
if let Some(changelog) = parse::<ChangelogConfig>(kv, errors) {
snapshot.changelog = Some(changelog);
@@ -314,12 +305,13 @@ pub fn derive_view(
});
}
- if raw.total_partitions.is_none() {
+ let total_partitions = raw.changelog.as_ref().map(|c| c.partitions);
+ if raw.changelog.is_none() {
issues.push(Issue {
severity: Severity::Error,
partition: None,
- summary: "config/total_partitions is missing: the coordinator cannot plan, and \
- leaders and routers fail at startup"
+ summary: "config/changelog is not seeded: the coordinator cannot plan, \
+ and every service waits at startup"
.to_string(),
});
}
@@ -352,7 +344,7 @@ pub fn derive_view(
// Partitions worth a row: everything under the configured count, plus
// anything that has state anyway (misconfiguration shows itself).
- let mut partition_ids: Vec<u32> = (0..raw.total_partitions.unwrap_or(0)).collect();
+ let mut partition_ids: Vec<u32> = (0..total_partitions.unwrap_or(0)).collect();
for partition in assignments_by_partition
.keys()
.chain(handoffs_by_partition.keys())
@@ -368,13 +360,13 @@ pub fn derive_view(
let assignment = assignments_by_partition.get(&partition);
let handoff = handoffs_by_partition.get(&partition);
- if let Some(total) = raw.total_partitions {
+ if let Some(total) = total_partitions {
if partition >= total && (assignment.is_some() || handoff.is_some()) {
issues.push(Issue {
severity: Severity::Warning,
partition: Some(partition),
summary: format!(
- "partition {partition} has state but total_partitions is {total}"
+ "partition {partition} has state but the changelog has {total} partitions"
),
});
}
@@ -541,7 +533,7 @@ pub fn derive_view(
fetched_at: Utc::now(),
revision: raw.revision,
prefix: prefix.to_string(),
- total_partitions: raw.total_partitions,
+ total_partitions,
changelog: raw.changelog.clone(),
leader: raw.leader.as_ref().map(|(leader, _)| LeaderView {
holder: leader.holder.clone(),
@@ -721,10 +713,10 @@ mod tests {
)
}
- fn total_partitions_kv(count: u32) -> RawKv {
+ fn changelog_kv(count: u32) -> RawKv {
RawKv {
- key: format!("{PREFIX}config/total_partitions"),
- value: count.to_string().into_bytes(),
+ key: format!("{PREFIX}config/changelog"),
+ value: serde_json::to_vec(&ChangelogConfig::new("personhog_updates", count)).unwrap(),
lease: 0,
}
}
@@ -745,7 +737,7 @@ mod tests {
#[test]
fn a_stuck_freezing_handoff_names_the_missing_router() {
let kvs = vec![
- total_partitions_kv(1),
+ changelog_kv(1),
pod_kv("pod-a"),
pod_kv("pod-b"),
router_kv("router-0"),
@@ -789,7 +781,7 @@ mod tests {
#[test]
fn a_satisfied_phase_points_at_the_coordinator() {
let kvs = vec![
- total_partitions_kv(1),
+ changelog_kv(1),
pod_kv("pod-b"),
assignment_kv(0, "pod-b"),
// Draining with a dead old owner: drain is vacuously satisfied,
@@ -808,7 +800,7 @@ mod tests {
#[test]
fn a_dead_owner_without_a_handoff_is_an_error_and_with_one_is_not() {
let kvs = vec![
- total_partitions_kv(2),
+ changelog_kv(2),
pod_kv("pod-b"),
assignment_kv(0, "pod-gone"),
assignment_kv(1, "pod-gone"),
@@ -826,7 +818,7 @@ mod tests {
#[test]
fn a_lingering_complete_handoff_flags_the_coordinator() {
let kvs = vec![
- total_partitions_kv(1),
+ changelog_kv(1),
pod_kv("pod-b"),
assignment_kv(0, "pod-b"),
handoff_kv(0, "Complete", None, "pod-b", &[], 1_000),
@@ -849,7 +841,7 @@ mod tests {
#[test]
fn unassigned_and_out_of_range_partitions_are_flagged() {
let kvs = vec![
- total_partitions_kv(2),
+ changelog_kv(2),
pod_kv("pod-a"),
assignment_kv(0, "pod-a"),
assignment_kv(5, "pod-a"),
@@ -862,7 +854,7 @@ mod tests {
.any(|s| s.contains("partition 1 has no assignment and no handoff")));
assert!(warnings
.iter()
- .any(|s| s.contains("partition 5 has state but total_partitions is 2")));
+ .any(|s| s.contains("partition 5 has state but the changelog has 2 partitions")));
// The out-of-range partition still renders a row.
assert!(topology.partitions.iter().any(|p| p.partition == 5));
}
@@ -870,7 +862,7 @@ mod tests {
#[test]
fn stale_acks_are_inert_info_and_matching_acks_are_not() {
let kvs = vec![
- total_partitions_kv(1),
+ changelog_kv(1),
pod_kv("pod-a"),
pod_kv("pod-b"),
router_kv("router-0"),
@@ -898,13 +890,13 @@ mod tests {
}
#[test]
- fn missing_total_partitions_is_an_error_without_unassigned_noise() {
+ fn an_unseeded_changelog_is_an_error_without_unassigned_noise() {
let kvs = vec![pod_kv("pod-a"), assignment_kv(0, "pod-a")];
let topology = view(&kvs, 10_000);
assert!(issue_summaries(&topology, Severity::Error)
.iter()
- .any(|s| s.contains("config/total_partitions is missing")));
+ .any(|s| s.contains("config/changelog is not seeded")));
assert!(!issue_summaries(&topology, Severity::Warning)
.iter()
.any(|s| s.contains("no assignment")));
@@ -913,7 +905,7 @@ mod tests {
#[test]
fn a_dead_new_owner_warns_and_unparseable_records_surface() {
let mut kvs = vec![
- total_partitions_kv(1),
+ changelog_kv(1),
pod_kv("pod-a"),
assignment_kv(0, "pod-a"),
handoff_kv(0, "Warming", Some("pod-a"), "pod-gone", &[], 9_000),
@@ -936,7 +928,7 @@ mod tests {
#[test]
fn pods_carry_their_assigned_partitions_and_lease_ttls() {
let kvs = vec![
- total_partitions_kv(2),
+ changelog_kv(2),
pod_kv("pod-a"),
assignment_kv(1, "pod-a"),
assignment_kv(0, "pod-a"),
diff --git a/rust/ingestion-control-plane/src/ui/index.html b/rust/ingestion-control-plane/src/ui/index.html
index 2affd1bcc63c..1081a38987d2 100644
--- a/rust/ingestion-control-plane/src/ui/index.html
+++ b/rust/ingestion-control-plane/src/ui/index.html
@@ -1397,7 +1397,7 @@
}
if (!view.config) {
this.ui.changelogStatus.replaceChildren(
- el("div", { class: "muted", style: "padding: 6px 14px 8px" }, "Not seeded."));
+ el("div", { class: "muted", style: "padding: 6px 14px 8px" }, "Not seeded. Every personhog service waits at startup until the changelog is seeded."));
return;
}
const c = view.config;
@@ -1433,7 +1433,7 @@
error.replaceChildren(el("div", { class: "error-box" }, "Enter the topic and the writer's consumer group."));
return;
}
- if (!window.confirm(`Seed config/changelog with topic ${topic}, with writer group ${writer_group}?\n\nThe partition count is read from the topic's metadata.`)) return;
+ if (!window.confirm(`Seed config/changelog with topic ${topic}, with writer group ${writer_group}?\n\nThe partition count is read from the topic's metadata. Every personhog participant adopts this object.`)) return;
button.disabled = true;
try {
await fetchJSON("api/personhog/changelog/seed", {
@@ -1464,7 +1464,7 @@
? el("span", { class: "pill good", title: `lease ${data.leader.lease_id}${data.leader.lease_ttl_secs !== null ? ` · ttl ${data.leader.lease_ttl_secs}s` : ""}` }, `coordinator ${data.leader.holder}`)
: el("span", { class: "pill bad" }, "no coordinator"),
el("span", { class: "pill muted" },
- data.total_partitions === null ? "total_partitions missing" : `${fmtInt(data.total_partitions)} partitions`),
+ data.total_partitions === null ? "changelog not seeded" : `${fmtInt(data.total_partitions)} partitions`),
el("span", { class: `pill ${readyPods === data.pods.length && readyPods > 0 ? "good" : "warn"}` },
`${fmtInt(readyPods)}/${fmtInt(data.pods.length)} pods ready`),
el("span", { class: "pill muted" }, `${fmtInt(data.routers.length)} router${data.routers.length === 1 ? "" : "s"}`),
diff --git a/rust/personhog-coordination/README.md b/rust/personhog-coordination/README.md
index b4b011127a83..0adb6875a8da 100644
--- a/rust/personhog-coordination/README.md
+++ b/rust/personhog-coordination/README.md
@@ -123,6 +123,10 @@ Several failure modes are handled explicitly:
**Mid-stream watch reconnects don't replay missed events — bounded by reconciliation**: The snapshot-to-watch-attach gap is closed (watches are revision-anchored to the snapshot that seeded them), but if an *established* etcd watch stream drops and the underlying client reconnects, events that fired during the disconnect are not replayed. Every participant now bounds the resulting staleness with its level-triggered truth path — the coordinator's tick (which also wakes the planner unconditionally), the router's reconcile pass, and the pod's converge-all tick — so a missed event costs at most one reconcile interval, not correctness.
+## Changelog config
+
+`config/changelog` (`{topic, partitions, epoch, writer_group, draining}`) is the one live object every participant reads. Every service loads it at boot; the router and the identity service also follow it through a watch, so the partition count they hash under is the live one.
+
## Modules
| Module | Responsibility |
diff --git a/rust/personhog-coordination/src/changelog.rs b/rust/personhog-coordination/src/changelog.rs
new file mode 100644
index 000000000000..da0a1cdf0dad
--- /dev/null
+++ b/rust/personhog-coordination/src/changelog.rs
@@ -0,0 +1,221 @@
+//! Boot-time load and the shared watch of `config/changelog`.
+
+use std::ops::ControlFlow;
+use std::sync::atomic::{AtomicU32, Ordering};
+use std::sync::Arc;
+use std::time::Duration;
+
+use etcd_client::EventType;
+use tokio_util::sync::CancellationToken;
+
+use crate::error::{Error, Result};
+use crate::store::PersonhogStore;
+use crate::types::ChangelogConfig;
+use crate::util;
+
+/// Polls until `config/changelog` is seeded; it is the only source of the topic and count.
+pub async fn load_changelog_config(
+ store: &PersonhogStore,
+ poll_interval: Duration,
+) -> Result<ChangelogConfig> {
+ let mut polls: u64 = 0;
+ loop {
+ if let Some(config) = store.get_changelog_config().await? {
+ return Ok(config);
+ }
+ if polls.is_multiple_of(10) {
+ tracing::warn!("config/changelog is not seeded; waiting for the control plane");
+ }
+ polls += 1;
+ tokio::time::sleep(poll_interval).await;
+ }
+}
+
+/// Delivers each object that supersedes the last one adopted; `initial` is what boot adopted.
+pub async fn watch_changelog_config_loop<F>(
+ store: Arc<PersonhogStore>,
+ initial: ChangelogConfig,
+ reconcile_interval: Duration,
+ cancel: CancellationToken,
+ mut on_config: F,
+) where
+ F: FnMut(&ChangelogConfig) -> ControlFlow<()>,
+{
+ let mut adopted = initial;
+ let mut backoff = Duration::from_millis(100);
+ 'establish: loop {
+ if cancel.is_cancelled() {
+ return;
+ }
+ let established = async {
+ let (current, revision) = store.get_changelog_config_with_revision().await?;
+ let stream = store.watch_changelog_config_from(revision + 1).await?;
+ Ok::<_, Error>((current, stream))
+ };
+ let (current, mut stream) = match established.await {
+ Ok(established) => established,
+ Err(e) => {
+ tracing::warn!(error = %e, "changelog config watch could not be established");
+ metrics::counter!("personhog_changelog_config_watch_reconnects_total").increment(1);
+ tokio::select! {
+ _ = cancel.cancelled() => return,
+ _ = tokio::time::sleep(backoff) => {}
+ }
+ backoff = (backoff * 2).min(Duration::from_secs(5));
+ continue 'establish;
+ }
+ };
+ backoff = Duration::from_millis(100);
+ if let Some(config) = &current {
+ if deliver(&mut adopted, config, "establish", &mut on_config).is_break() {
+ return;
+ }
+ }
+
+ let mut reconcile = tokio::time::interval(reconcile_interval);
+ reconcile.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
+ reconcile.tick().await;
+ loop {
+ tokio::select! {
+ _ = cancel.cancelled() => return,
+ _ = reconcile.tick() => {
+ match store.get_changelog_config().await {
+ Ok(Some(config)) => {
+ if deliver(&mut adopted, &config, "reconcile", &mut on_config).is_break() {
+ return;
+ }
+ }
+ Ok(None) => {}
+ Err(e) => {
+ tracing::warn!(error = %e, "changelog config reconcile read failed");
+ }
+ }
+ }
+ msg = stream.message() => {
+ let resp = match msg.map_err(Error::from).and_then(|resp| util::live_watch_response(resp, "changelog config")) {
+ Ok(resp) => resp,
+ Err(e) => {
+ tracing::warn!(error = %e, "changelog config watch broke; re-establishing");
+ metrics::counter!("personhog_changelog_config_watch_reconnects_total").increment(1);
+ continue 'establish;
+ }
+ };
+ for event in resp.events() {
+ if event.event_type() != EventType::Put {
+ continue;
+ }
+ let Some(kv) = event.kv() else { continue };
+ match serde_json::from_slice::<ChangelogConfig>(kv.value()) {
+ Ok(config) => {
+ if deliver(&mut adopted, &config, "watch", &mut on_config).is_break() {
+ return;
+ }
+ }
+ Err(e) => {
+ tracing::error!(error = %e, "unreadable changelog config event; the reconcile read will retry");
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+}
+
+/// Bounds the staleness a dropped watch stream can hide.
+pub const CHANGELOG_RECONCILE_INTERVAL: Duration = Duration::from_secs(30);
+pub const CHANGELOG_BOOT_POLL_INTERVAL: Duration = Duration::from_secs(1);
+
+/// The partition count has its own atomic: routing reads it per request.
+#[derive(Clone)]
+pub struct LiveChangelog {
+ current: Arc<std::sync::RwLock<ChangelogConfig>>,
+ partitions: Arc<AtomicU32>,
+}
+
+impl LiveChangelog {
+ pub fn new(config: ChangelogConfig) -> Self {
+ Self {
+ partitions: Arc::new(AtomicU32::new(config.partitions)),
+ current: Arc::new(std::sync::RwLock::new(config)),
+ }
+ }
+
+ pub fn partitions(&self) -> u32 {
+ self.partitions.load(Ordering::Acquire)
+ }
+
+ pub fn current(&self) -> ChangelogConfig {
+ self.current
+ .read()
+ .expect("changelog config lock poisoned")
+ .clone()
+ }
+
+ pub fn epoch(&self) -> u64 {
+ self.current
+ .read()
+ .expect("changelog config lock poisoned")
+ .epoch
+ }
+
+ pub fn adopt(&self, config: &ChangelogConfig) -> bool {
+ let mut current = self
+ .current
+ .write()
+ .expect("changelog config lock poisoned");
+ if !config.supersedes(&current) {
+ return false;
+ }
+ *current = config.clone();
+ self.partitions.store(config.partitions, Ordering::Release);
+ true
+ }
+}
+
+pub async fn keep_changelog_live(
+ store: Arc<PersonhogStore>,
+ live: LiveChangelog,
+ cancel: CancellationToken,
+) {
+ let initial = live.current();
+ watch_changelog_config_loop(
+ store,
+ initial,
+ CHANGELOG_RECONCILE_INTERVAL,
+ cancel,
+ |config| {
+ live.adopt(config);
+ ControlFlow::Continue(())
+ },
+ )
+ .await
+}
+
+fn deliver<F>(
+ adopted: &mut ChangelogConfig,
+ config: &ChangelogConfig,
+ source: &'static str,
+ on_config: &mut F,
+) -> ControlFlow<()>
+where
+ F: FnMut(&ChangelogConfig) -> ControlFlow<()>,
+{
+ if !config.supersedes(adopted) {
+ metrics::counter!("personhog_changelog_config_stale_deliveries_total", "source" => source)
+ .increment(1);
+ return ControlFlow::Continue(());
+ }
+ *adopted = config.clone();
+ metrics::counter!("personhog_changelog_config_adoptions_total", "source" => source)
+ .increment(1);
+ metrics::gauge!("personhog_changelog_config_epoch").set(config.epoch as f64);
+ tracing::info!(
+ topic = %config.topic,
+ partitions = config.partitions,
+ epoch = config.epoch,
+ source,
+ "adopted changelog config"
+ );
+ on_config(config)
+}
diff --git a/rust/personhog-coordination/src/coordinator.rs b/rust/personhog-coordination/src/coordinator.rs
index 3a463d2acad8..3669cfeb24cc 100644
--- a/rust/personhog-coordination/src/coordinator.rs
+++ b/rust/personhog-coordination/src/coordinator.rs
@@ -1281,14 +1281,11 @@ impl Coordinator {
max_txn_ops: usize,
) -> Result<()> {
let pods = store.list_pods().await?;
- let total_partitions = match store.get_total_partitions().await {
- Ok(n) => n,
- Err(Error::NotFound(_)) => {
- tracing::debug!("total_partitions not set, skipping assignment");
- return Ok(());
- }
- Err(e) => return Err(e),
+ let Some(changelog) = store.get_changelog_config().await? else {
+ tracing::debug!("changelog config not seeded, skipping assignment");
+ return Ok(());
};
+ let total_partitions = changelog.partitions;
// K8s-aware placement policies for smarter rebalancing
let members = match k8s_awareness {
diff --git a/rust/personhog-coordination/src/lib.rs b/rust/personhog-coordination/src/lib.rs
index d0f696f6182e..489e5b289116 100644
--- a/rust/personhog-coordination/src/lib.rs
+++ b/rust/personhog-coordination/src/lib.rs
@@ -1,4 +1,5 @@
pub mod authority;
+pub mod changelog;
pub mod coordinator;
pub mod error;
pub mod hash;
diff --git a/rust/personhog-coordination/src/store.rs b/rust/personhog-coordination/src/store.rs
index ba240882b7f9..2032e333d980 100644
--- a/rust/personhog-coordination/src/store.rs
+++ b/rust/personhog-coordination/src/store.rs
@@ -1,5 +1,4 @@
use std::collections::HashMap;
-use std::str::from_utf8;
use std::sync::{Arc, Mutex as StdMutex};
use assignment_coordination::store::EtcdStore;
@@ -39,7 +38,6 @@ enum StoreKey<'a> {
FreezeQuorumsPrefix,
Leader,
Generation,
- TotalPartitions,
ChangelogConfig,
}
@@ -73,7 +71,6 @@ impl StoreKey<'_> {
StoreKey::FreezeQuorumsPrefix => format!("{prefix}freeze_quorums/"),
StoreKey::Leader => format!("{prefix}coordinator/leader"),
StoreKey::Generation => format!("{prefix}generation"),
- StoreKey::TotalPartitions => format!("{prefix}config/total_partitions"),
StoreKey::ChangelogConfig => format!("{prefix}config/changelog"),
}
}
@@ -1188,31 +1185,20 @@ impl PersonhogStore {
// ── Config operations ───────────────────────────────────────
- pub async fn get_total_partitions(&self) -> Result<u32> {
- count_call("get_total_partitions");
- let key = self.key(StoreKey::TotalPartitions);
- let bytes = self
- .inner
- .get_raw(&key)
- .await?
- .ok_or_else(|| Error::NotFound(key))?;
- let s = from_utf8(&bytes)
- .map_err(|e| Error::invalid_state(format!("non-utf8 total_partitions: {e}")))?;
- s.parse::<u32>()
- .map_err(|e| Error::invalid_state(format!("invalid total_partitions: {e}")))
- }
-
- pub async fn set_total_partitions(&self, count: u32) -> Result<()> {
- let key = self.key(StoreKey::TotalPartitions);
- Ok(self.inner.put_raw(&key, count.to_string()).await?)
- }
-
pub async fn get_changelog_config(&self) -> Result<Option<ChangelogConfig>> {
count_call("get_changelog_config");
let key = self.key(StoreKey::ChangelogConfig);
Ok(self.inner.get(&key).await?)
}
+ pub async fn get_changelog_config_with_revision(
+ &self,
+ ) -> Result<(Option<ChangelogConfig>, i64)> {
+ count_call("get_changelog_config_with_revision");
+ let key = self.key(StoreKey::ChangelogConfig);
+ Ok(self.inner.get_with_revision(&key).await?)
+ }
+
/// Create-if-absent at epoch 1; only a flip may move a live config.
pub async fn seed_changelog_config(
&self,
@@ -1237,6 +1223,19 @@ impl PersonhogStore {
Ok(resp.succeeded())
}
+ /// Bypasses the protocol: see the crate's `test-support` feature.
+ #[cfg(any(test, feature = "test-support"))]
+ pub async fn put_changelog_config(&self, config: &ChangelogConfig) -> Result<()> {
+ let key = self.key(StoreKey::ChangelogConfig);
+ Ok(self.inner.put(&key, config, None).await?)
+ }
+
+ pub async fn watch_changelog_config_from(&self, start_revision: i64) -> Result<WatchStream> {
+ count_call("watch_changelog_config_from");
+ let key = self.key(StoreKey::ChangelogConfig);
+ Ok(self.inner.watch_key_from(&key, start_revision).await?)
+ }
+
pub async fn get_generation(&self) -> Result<String> {
let key = self.key(StoreKey::Generation);
let bytes = self
diff --git a/rust/personhog-coordination/src/types.rs b/rust/personhog-coordination/src/types.rs
index c7e9738ccd48..6e4aa94bde8a 100644
--- a/rust/personhog-coordination/src/types.rs
+++ b/rust/personhog-coordination/src/types.rs
@@ -335,6 +335,14 @@ impl ChangelogConfig {
draining: Vec::new(),
}
}
+
+ /// A flip raises the epoch. Within an epoch only a retire changes the
+ /// object, and a retire only shrinks `draining`, so a same-epoch object
+ /// with more generations is an older one delivered late.
+ pub fn supersedes(&self, adopted: &ChangelogConfig) -> bool {
+ self.epoch > adopted.epoch
+ || (self.epoch == adopted.epoch && self.draining.len() < adopted.draining.len())
+ }
}
#[cfg(test)]
@@ -362,6 +370,35 @@ mod tests {
assert_eq!(h.phase_entered_at_ms, 0);
}
+ #[test]
+ fn a_config_supersedes_by_epoch_then_by_retired_generations() {
+ let generation = |epoch| Generation {
+ topic: format!("g{epoch}"),
+ partitions: 64,
+ epoch,
+ };
+ let config = |epoch: u64, generations: &[u64]| ChangelogConfig {
+ epoch,
+ draining: generations.iter().map(|e| generation(*e)).collect(),
+ ..ChangelogConfig::new("g", 128)
+ };
+ let adopted = config(3, &[2]);
+ let cases = [
+ ("a newer epoch", config(4, &[2, 3]), true),
+ ("a retire at the same epoch", config(3, &[]), true),
+ ("the same object", config(3, &[2]), false),
+ (
+ "the pre-retire object delivered late",
+ config(3, &[1, 2]),
+ false,
+ ),
+ ("an older epoch", config(2, &[]), false),
+ ];
+ for (name, candidate, expected) in cases {
+ assert_eq!(candidate.supersedes(&adopted), expected, "{name}");
+ }
+ }
+
#[test]
fn registered_pod_roundtrip() {
let pod = RegisteredPod {
diff --git a/rust/personhog-coordination/tests/changelog.rs b/rust/personhog-coordination/tests/changelog.rs
index 48290c4f25c9..bc304f617a93 100644
--- a/rust/personhog-coordination/tests/changelog.rs
+++ b/rust/personhog-coordination/tests/changelog.rs
@@ -1,6 +1,15 @@
mod common;
-use common::test_store;
+use std::ops::ControlFlow;
+use std::sync::Arc;
+use std::time::Duration;
+
+use personhog_coordination::changelog::{load_changelog_config, watch_changelog_config_loop};
+use personhog_coordination::types::{ChangelogConfig, Generation};
+use tokio::sync::mpsc;
+use tokio_util::sync::CancellationToken;
+
+use common::{store_at, test_store, FlakyProxy, ETCD_ENDPOINT};
#[tokio::test]
async fn a_second_seed_is_refused() {
@@ -24,3 +33,175 @@ async fn a_second_seed_is_refused() {
("personhog_updates_64", 64, 1, "personhog-writer")
);
}
+
+#[tokio::test]
+async fn boot_polls_until_the_config_is_seeded() {
+ let store = test_store("changelog-boot").await;
+ let seeder = Arc::clone(&store);
+ tokio::spawn(async move {
+ tokio::time::sleep(Duration::from_millis(200)).await;
+ seeder
+ .seed_changelog_config("personhog_updates_64", 64, "personhog-writer")
+ .await
+ .unwrap();
+ });
+ let loaded = tokio::time::timeout(
+ Duration::from_secs(5),
+ load_changelog_config(&store, Duration::from_millis(20)),
+ )
+ .await
+ .expect("boot polls until the config is seeded")
+ .unwrap();
+ assert_eq!((loaded.partitions, loaded.epoch), (64, 1));
+}
+
+/// What a loop starts from before it has adopted anything.
+fn nothing_adopted() -> ChangelogConfig {
+ ChangelogConfig {
+ epoch: 0,
+ ..ChangelogConfig::new("unadopted", 0)
+ }
+}
+
+async fn next_adopted(rx: &mut mpsc::UnboundedReceiver<ChangelogConfig>) -> ChangelogConfig {
+ tokio::time::timeout(Duration::from_secs(5), rx.recv())
+ .await
+ .expect("a config is adopted")
+ .expect("the loop is alive")
+}
+
+#[tokio::test]
+async fn the_watch_delivers_newer_epochs_and_retires_but_nothing_stale() {
+ let store = test_store("changelog-watch").await;
+ store
+ .seed_changelog_config("personhog_updates_64", 64, "personhog-writer")
+ .await
+ .unwrap();
+
+ let (tx, mut rx) = mpsc::unbounded_channel();
+ let cancel = CancellationToken::new();
+ let loop_store = Arc::clone(&store);
+ let loop_cancel = cancel.clone();
+ tokio::spawn(async move {
+ watch_changelog_config_loop(
+ loop_store,
+ nothing_adopted(),
+ Duration::from_secs(30),
+ loop_cancel,
+ |c| {
+ tx.send(c.clone()).unwrap();
+ ControlFlow::Continue(())
+ },
+ )
+ .await;
+ });
+
+ assert_eq!(next_adopted(&mut rx).await.epoch, 1);
+
+ let flipped = ChangelogConfig {
+ topic: "personhog_updates_128".into(),
+ partitions: 128,
+ epoch: 2,
+ writer_group: String::new(),
+ draining: Vec::new(),
+ };
+ store.put_changelog_config(&flipped).await.unwrap();
+ assert_eq!(next_adopted(&mut rx).await, flipped);
+
+ store.put_changelog_config(&flipped).await.unwrap();
+ store
+ .put_changelog_config(&ChangelogConfig {
+ epoch: 1,
+ ..flipped.clone()
+ })
+ .await
+ .unwrap();
+ let newer = ChangelogConfig {
+ epoch: 3,
+ draining: vec![Generation {
+ topic: "personhog_updates_128".into(),
+ partitions: 128,
+ epoch: 2,
+ }],
+ ..flipped.clone()
+ };
+ store.put_changelog_config(&newer).await.unwrap();
+ assert_eq!(next_adopted(&mut rx).await, newer);
+
+ // A retire keeps the epoch and only shrinks `draining`.
+ let retired = ChangelogConfig {
+ draining: Vec::new(),
+ ..newer.clone()
+ };
+ store.put_changelog_config(&retired).await.unwrap();
+ assert_eq!(next_adopted(&mut rx).await, retired);
+ // The pre-retire object arriving late is older, not a new generation.
+ store.put_changelog_config(&newer).await.unwrap();
+ store.put_changelog_config(&retired).await.unwrap();
+ // Deliveries keep revision order, so the next one proves the stale writes were dropped.
+ let sentinel = ChangelogConfig {
+ epoch: 4,
+ ..retired.clone()
+ };
+ store.put_changelog_config(&sentinel).await.unwrap();
+ assert_eq!(next_adopted(&mut rx).await, sentinel);
+
+ cancel.cancel();
+}
+
+#[tokio::test]
+async fn a_severed_watch_still_adopts_the_flip_through_reconcile() {
+ let proxy = FlakyProxy::start("127.0.0.1:2379").await;
+ let prefix = format!("/test-changelog-sever-{}/", uuid::Uuid::new_v4());
+ let watched = store_at(&proxy.endpoint, &prefix).await;
+ let direct = store_at(ETCD_ENDPOINT, &prefix).await;
+ direct
+ .seed_changelog_config("personhog_updates_64", 64, "personhog-writer")
+ .await
+ .unwrap();
+
+ let (tx, mut rx) = mpsc::unbounded_channel();
+ let cancel = CancellationToken::new();
+ let loop_cancel = cancel.clone();
+ tokio::spawn(async move {
+ watch_changelog_config_loop(
+ watched,
+ nothing_adopted(),
+ Duration::from_millis(300),
+ loop_cancel,
+ |c| {
+ tx.send(c.epoch).unwrap();
+ ControlFlow::Continue(())
+ },
+ )
+ .await;
+ });
+ assert_eq!(
+ tokio::time::timeout(Duration::from_secs(5), rx.recv())
+ .await
+ .unwrap(),
+ Some(1)
+ );
+
+ proxy.set_blackholed(true);
+ proxy.sever();
+ direct
+ .put_changelog_config(&ChangelogConfig {
+ topic: "personhog_updates_128".into(),
+ partitions: 128,
+ epoch: 2,
+ writer_group: String::new(),
+ draining: Vec::new(),
+ })
+ .await
+ .unwrap();
+ proxy.set_blackholed(false);
+
+ assert_eq!(
+ tokio::time::timeout(Duration::from_secs(10), rx.recv())
+ .await
+ .expect("the flip is adopted after the stream heals"),
+ Some(2)
+ );
+ cancel.cancel();
+}
diff --git a/rust/personhog-coordination/tests/integration.rs b/rust/personhog-coordination/tests/integration.rs
index ff2916d8b4de..f01555bb4f1f 100644
--- a/rust/personhog-coordination/tests/integration.rs
+++ b/rust/personhog-coordination/tests/integration.rs
@@ -83,7 +83,10 @@ async fn single_pod_gets_all_partitions(
let store = test_store(test_name).await;
let cancel = CancellationToken::new();
- store.set_total_partitions(NUM_PARTITIONS).await.unwrap();
+ assert!(store
+ .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")
+ .await
+ .unwrap());
let _coord = start_coordinator(Arc::clone(&store), strategy, cancel.clone());
let _pod = start_pod(Arc::clone(&store), "writer-0", cancel.clone());
@@ -125,7 +128,10 @@ async fn scale_up_triggers_handoff(
let store = test_store(test_name).await;
let cancel = CancellationToken::new();
- store.set_total_partitions(NUM_PARTITIONS).await.unwrap();
+ assert!(store
+ .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")
+ .await
+ .unwrap());
// Start coordinator, one pod, and one router
let _coord = start_coordinator(Arc::clone(&store), strategy, cancel.clone());
@@ -256,7 +262,10 @@ async fn pod_crash_reassigns_partitions(
let store = test_store(test_name).await;
let cancel = CancellationToken::new();
- store.set_total_partitions(NUM_PARTITIONS).await.unwrap();
+ assert!(store
+ .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")
+ .await
+ .unwrap());
// Start coordinator and two pods (short lease so crash is detected fast)
let _coord = start_coordinator(Arc::clone(&store), strategy, cancel.clone());
@@ -325,7 +334,10 @@ async fn leader_failover(
let store = test_store(test_name).await;
let cancel = CancellationToken::new();
- store.set_total_partitions(NUM_PARTITIONS).await.unwrap();
+ assert!(store
+ .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")
+ .await
+ .unwrap());
// Start coordinator-0 with short lease so failover is fast
let leader_cancel = CancellationToken::new();
@@ -413,7 +425,10 @@ async fn multi_router_ack_quorum(
let store = test_store(test_name).await;
let cancel = CancellationToken::new();
- store.set_total_partitions(NUM_PARTITIONS).await.unwrap();
+ assert!(store
+ .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")
+ .await
+ .unwrap());
let _coord = start_coordinator(Arc::clone(&store), strategy, cancel.clone());
let _pod0 = start_pod(Arc::clone(&store), "writer-0", cancel.clone());
@@ -513,7 +528,10 @@ async fn pod_crash_during_warming_restores_assignments(
let store = test_store(test_name).await;
let cancel = CancellationToken::new();
- store.set_total_partitions(MANY_PARTITIONS).await.unwrap();
+ assert!(store
+ .seed_changelog_config("personhog_updates", MANY_PARTITIONS, "personhog-writer")
+ .await
+ .unwrap());
let _coord = start_coordinator(Arc::clone(&store), strategy, cancel.clone());
let _router = start_router(Arc::clone(&store), "router-0", cancel.clone());
@@ -640,7 +658,10 @@ async fn scale_down_to_one_pod(
let store = test_store(test_name).await;
let cancel = CancellationToken::new();
- store.set_total_partitions(MANY_PARTITIONS).await.unwrap();
+ assert!(store
+ .seed_changelog_config("personhog_updates", MANY_PARTITIONS, "personhog-writer")
+ .await
+ .unwrap());
let _coord = start_coordinator(Arc::clone(&store), strategy, cancel.clone());
let _router = start_router(Arc::clone(&store), "router-0", cancel.clone());
@@ -723,7 +744,10 @@ async fn rapid_pod_joins(
let store = test_store(test_name).await;
let cancel = CancellationToken::new();
- store.set_total_partitions(MANY_PARTITIONS).await.unwrap();
+ assert!(store
+ .seed_changelog_config("personhog_updates", MANY_PARTITIONS, "personhog-writer")
+ .await
+ .unwrap());
let _coord = start_coordinator(Arc::clone(&store), Arc::clone(&strategy), cancel.clone());
let _router = start_router(Arc::clone(&store), "router-0", cancel.clone());
@@ -813,7 +837,10 @@ async fn rolling_update(
let store = test_store(test_name).await;
let cancel = CancellationToken::new();
- store.set_total_partitions(MANY_PARTITIONS).await.unwrap();
+ assert!(store
+ .seed_changelog_config("personhog_updates", MANY_PARTITIONS, "personhog-writer")
+ .await
+ .unwrap());
let _coord = start_coordinator(Arc::clone(&store), strategy, cancel.clone());
let _router = start_router(Arc::clone(&store), "router-0", cancel.clone());
@@ -940,7 +967,10 @@ async fn coordinator_starts_after_pods(
let store = test_store(test_name).await;
let cancel = CancellationToken::new();
- store.set_total_partitions(NUM_PARTITIONS).await.unwrap();
+ assert!(store
+ .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")
+ .await
+ .unwrap());
// Start pods and router BEFORE the coordinator
let _pod0 = start_pod(Arc::clone(&store), "writer-0", cancel.clone());
@@ -1005,7 +1035,10 @@ async fn debounce_batches_rapid_pod_changes(
let store = test_store(test_name).await;
let cancel = CancellationToken::new();
- store.set_total_partitions(MANY_PARTITIONS).await.unwrap();
+ assert!(store
+ .seed_changelog_config("personhog_updates", MANY_PARTITIONS, "personhog-writer")
+ .await
+ .unwrap());
let _coord = start_coordinator_with_debounce(
Arc::clone(&store),
@@ -1104,7 +1137,10 @@ async fn graceful_drain_transfers_partitions(
_evenly_distributed: bool,
) {
let store = test_store(test_name).await;
- store.set_total_partitions(NUM_PARTITIONS).await.unwrap();
+ assert!(store
+ .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")
+ .await
+ .unwrap());
let coord_cancel = CancellationToken::new();
let _coord = start_coordinator(
@@ -1199,7 +1235,10 @@ async fn graceful_drain_transfers_partitions(
#[tokio::test]
async fn drain_status_write_failure_exits_cleanly() {
let store = test_store("drain-write-fail").await;
- store.set_total_partitions(NUM_PARTITIONS).await.unwrap();
+ assert!(store
+ .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")
+ .await
+ .unwrap());
let coord_cancel = CancellationToken::new();
let strategy: Arc<dyn AssignmentStrategy> = Arc::new(StickyBalancedStrategy);
@@ -1259,7 +1298,10 @@ async fn initial_assignment_creates_handoff_with_no_old_owner() {
let store = test_store("initial-no-old-owner").await;
let cancel = CancellationToken::new();
- store.set_total_partitions(NUM_PARTITIONS).await.unwrap();
+ assert!(store
+ .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")
+ .await
+ .unwrap());
let strategy: Arc<dyn AssignmentStrategy> = Arc::new(StickyBalancedStrategy);
let _coord = start_coordinator(Arc::clone(&store), strategy, cancel.clone());
@@ -1319,7 +1361,10 @@ async fn old_owner_retains_partition_through_warming() {
let store = test_store("retain-through-warming").await;
let cancel = CancellationToken::new();
- store.set_total_partitions(NUM_PARTITIONS).await.unwrap();
+ assert!(store
+ .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")
+ .await
+ .unwrap());
let strategy: Arc<dyn AssignmentStrategy> = Arc::new(StickyBalancedStrategy);
let _coord = start_coordinator(Arc::clone(&store), strategy, cancel.clone());
@@ -1390,7 +1435,10 @@ async fn a_dead_new_owner_handoff_is_reaffirmed_to_the_current_owner() {
let store = test_store("delete-drains-stash").await;
let cancel = CancellationToken::new();
- store.set_total_partitions(NUM_PARTITIONS).await.unwrap();
+ assert!(store
+ .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")
+ .await
+ .unwrap());
let strategy: Arc<dyn AssignmentStrategy> = Arc::new(StickyBalancedStrategy);
let _coord = start_coordinator(Arc::clone(&store), strategy, cancel.clone());
@@ -1511,7 +1559,10 @@ async fn stale_handoff_address_never_overwrites_a_fresher_registration() {
let store = test_store("addr-authority").await;
let cancel = CancellationToken::new();
- store.set_total_partitions(NUM_PARTITIONS).await.unwrap();
+ assert!(store
+ .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")
+ .await
+ .unwrap());
let strategy: Arc<dyn AssignmentStrategy> = Arc::new(StickyBalancedStrategy);
let _coord = start_coordinator(Arc::clone(&store), strategy, cancel.clone());
@@ -1588,7 +1639,10 @@ async fn pod_re_registration_refreshes_router_addresses_without_a_handoff() {
let store = test_store("addr-refresh").await;
let cancel = CancellationToken::new();
- store.set_total_partitions(NUM_PARTITIONS).await.unwrap();
+ assert!(store
+ .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")
+ .await
+ .unwrap());
let strategy: Arc<dyn AssignmentStrategy> = Arc::new(StickyBalancedStrategy);
let _coord = start_coordinator(Arc::clone(&store), strategy, cancel.clone());
@@ -1686,7 +1740,10 @@ async fn routing_table_updated_at_handoff_complete() {
let store = test_store("routing-table-inline").await;
let cancel = CancellationToken::new();
- store.set_total_partitions(NUM_PARTITIONS).await.unwrap();
+ assert!(store
+ .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")
+ .await
+ .unwrap());
let strategy: Arc<dyn AssignmentStrategy> = Arc::new(StickyBalancedStrategy);
let _coord = start_coordinator(Arc::clone(&store), strategy, cancel.clone());
@@ -1832,7 +1889,10 @@ async fn dead_old_owner_in_freezing_advances_to_completion() {
let store = test_store("dead-old-owner-freezing").await;
let cancel = CancellationToken::new();
- store.set_total_partitions(NUM_PARTITIONS).await.unwrap();
+ assert!(store
+ .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")
+ .await
+ .unwrap());
// Inject a Freezing handoff: old_owner=phantom-old (never registered
// as a pod), new_owner=writer-0 (will be registered next).
@@ -1952,7 +2012,10 @@ async fn a_dead_new_owner_handoff_in_warming_is_reaffirmed() {
let store = test_store("delete-during-warming").await;
let cancel = CancellationToken::new();
- store.set_total_partitions(NUM_PARTITIONS).await.unwrap();
+ assert!(store
+ .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")
+ .await
+ .unwrap());
let strategy: Arc<dyn AssignmentStrategy> = Arc::new(StickyBalancedStrategy);
let _coord = start_coordinator(Arc::clone(&store), strategy, cancel.clone());
@@ -2070,7 +2133,10 @@ async fn reconcile_advances_warming_with_pre_staged_warmed_ack() {
let store = test_store("reconcile-pre-staged-ack").await;
let cancel = CancellationToken::new();
- store.set_total_partitions(NUM_PARTITIONS).await.unwrap();
+ assert!(store
+ .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")
+ .await
+ .unwrap());
// Pre-stage a Warming handoff for partition 6 with no coordinator
// running. New owner already "warmed" (we inject the ack directly).
@@ -2251,7 +2317,10 @@ async fn draining_old_owner_does_not_trigger_cleanup() {
let store = test_store("draining-no-cleanup").await;
let cancel = CancellationToken::new();
- store.set_total_partitions(NUM_PARTITIONS).await.unwrap();
+ assert!(store
+ .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")
+ .await
+ .unwrap());
// Register a Draining pod with no DrainedAck written yet.
let lease = store.grant_lease(30).await.unwrap();
@@ -2324,7 +2393,10 @@ async fn a_handoff_that_can_never_reach_quorum_is_cancelled() {
let store = test_store("handoff-deadline-cancels").await;
let cancel = CancellationToken::new();
- store.set_total_partitions(NUM_PARTITIONS).await.unwrap();
+ assert!(store
+ .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")
+ .await
+ .unwrap());
// A live new owner, so the dead-new-owner cleanup path stays out of
// this: the handoff must be removed by the deadline alone.
@@ -2424,7 +2496,10 @@ async fn freezing_blocks_until_routers_ack_before_draining() {
let store = test_store("freezing-gates-draining").await;
let cancel = CancellationToken::new();
- store.set_total_partitions(NUM_PARTITIONS).await.unwrap();
+ assert!(store
+ .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")
+ .await
+ .unwrap());
// Register both owner pods: the new owner so the dead-new-owner cancellation
// doesn't delete the injected handoff, the old owner so the scenario
@@ -2542,7 +2617,10 @@ async fn initial_assignment_skips_draining_phase() {
let store = test_store("initial-skips-draining").await;
let cancel = CancellationToken::new();
- store.set_total_partitions(NUM_PARTITIONS).await.unwrap();
+ assert!(store
+ .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")
+ .await
+ .unwrap());
// Register the new_owner pod so the dead-new-owner cancellation doesn't
// delete the injected handoff for missing-target reasons.
@@ -2608,7 +2686,10 @@ async fn dead_old_owner_in_draining_recovers() {
let store = test_store("dead-old-owner-draining").await;
let cancel = CancellationToken::new();
- store.set_total_partitions(NUM_PARTITIONS).await.unwrap();
+ assert!(store
+ .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")
+ .await
+ .unwrap());
// Register both owners so the injected handoff starts as a live
// in-flight Draining handoff waiting on a real DrainedAck.
@@ -2711,7 +2792,10 @@ async fn reconcile_advances_draining_with_pre_staged_drained_ack() {
let store = test_store("reconcile-draining-ack").await;
let cancel = CancellationToken::new();
- store.set_total_partitions(NUM_PARTITIONS).await.unwrap();
+ assert!(store
+ .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")
+ .await
+ .unwrap());
// Pre-stage both pods (so cleanup doesn't fire on either) plus a
// Draining handoff plus a matching DrainedAck — all before the
@@ -2860,7 +2944,10 @@ async fn a_dead_new_owner_handoff_in_draining_is_reaffirmed() {
let store = test_store("delete-during-draining").await;
let cancel = CancellationToken::new();
- store.set_total_partitions(NUM_PARTITIONS).await.unwrap();
+ assert!(store
+ .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")
+ .await
+ .unwrap());
let strategy: Arc<dyn AssignmentStrategy> = Arc::new(StickyBalancedStrategy);
let _coord = start_coordinator(Arc::clone(&store), strategy, cancel.clone());
@@ -3045,7 +3132,10 @@ async fn routing_table_deregisters_router_on_graceful_exit() {
async fn stuck_handoff_defers_only_its_own_partition() {
let store = test_store("stuck_handoff_defers_only_its_own_partition").await;
let cancel = CancellationToken::new();
- store.set_total_partitions(NUM_PARTITIONS).await.unwrap();
+ assert!(store
+ .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")
+ .await
+ .unwrap());
let _coord = start_coordinator(
Arc::clone(&store),
@@ -3350,7 +3440,10 @@ async fn fresh_plan_is_rejected_when_an_assignment_appeared() {
async fn rebalance_never_writes_assignment_records() {
let store = test_store("rebalance_never_writes_assignment_records").await;
let cancel = CancellationToken::new();
- store.set_total_partitions(NUM_PARTITIONS).await.unwrap();
+ assert!(store
+ .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")
+ .await
+ .unwrap());
let _coord = start_coordinator(
Arc::clone(&store),
diff --git a/rust/personhog-coordination/tests/k3s_integration.rs b/rust/personhog-coordination/tests/k3s_integration.rs
index cc41250ef4fb..07f0d6a0d379 100644
--- a/rust/personhog-coordination/tests/k3s_integration.rs
+++ b/rust/personhog-coordination/tests/k3s_integration.rs
@@ -553,7 +553,10 @@ async fn deployment_rollout_reassigns_partitions() {
// answering from the scaffolding's watch, which is exactly how this
// gap previously escaped the suite).
let store = test_store("deploy-rollout-k3s").await;
- store.set_total_partitions(NUM_PARTITIONS).await.unwrap();
+ assert!(store
+ .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")
+ .await
+ .unwrap());
let coordinator_awareness = Arc::new(K8sAwareness::new(
k8s_client.clone(),
@@ -746,7 +749,10 @@ async fn deployment_rollout_caps_first_new_pod_at_quota() {
tokio::time::sleep(Duration::from_secs(3)).await;
let store = test_store("deploy-quota-k3s").await;
- store.set_total_partitions(NUM_PARTITIONS).await.unwrap();
+ assert!(store
+ .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")
+ .await
+ .unwrap());
// Fresh awareness for the coordinator so the test exercises the lazy
// watch bootstrap from pod registrations (see
@@ -963,7 +969,10 @@ async fn statefulset_rollout_pod_skips_drain() {
tokio::time::sleep(Duration::from_secs(3)).await;
let store = test_store("ss-drain-skip-k3s").await;
- store.set_total_partitions(NUM_PARTITIONS).await.unwrap();
+ assert!(store
+ .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")
+ .await
+ .unwrap());
let coord_cancel = CancellationToken::new();
let _coord = start_coordinator_k8s(Arc::clone(&store), None, coord_cancel.clone());
@@ -1090,7 +1099,10 @@ async fn scale_down_pod_drains_normally() {
tokio::time::sleep(Duration::from_secs(3)).await;
let store = test_store("scale-down-k3s").await;
- store.set_total_partitions(NUM_PARTITIONS).await.unwrap();
+ assert!(store
+ .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")
+ .await
+ .unwrap());
let cancel = CancellationToken::new();
let _coord = start_coordinator_k8s(
diff --git a/rust/personhog-coordination/tests/protocol_hardening.rs b/rust/personhog-coordination/tests/protocol_hardening.rs
index 557744d7544a..b1ce656f5d8a 100644
--- a/rust/personhog-coordination/tests/protocol_hardening.rs
+++ b/rust/personhog-coordination/tests/protocol_hardening.rs
@@ -979,7 +979,10 @@ async fn dead_old_owner_handoff_advances_in_place_not_cleaned_up() {
let store = test_store("advance-not-cleanup").await;
let cancel = CancellationToken::new();
- store.set_total_partitions(1).await.expect("partitions");
+ assert!(store
+ .seed_changelog_config("personhog_updates", 1, "personhog-writer")
+ .await
+ .expect("partitions"));
// The new owner is registered (directly — it never warms, so the
// handoff parks in Warming where we can observe it). The old owner
@@ -4550,7 +4553,10 @@ async fn a_replaced_handoff_reaches_the_old_owner_it_no_longer_names() {
#[tokio::test]
async fn a_plan_records_its_freeze_quorum_once_and_collects_it_after() {
let store = test_store("freeze-quorum-by-reference").await;
- store.set_total_partitions(2).await.expect("set partitions");
+ assert!(store
+ .seed_changelog_config("personhog_updates", 2, "personhog-writer")
+ .await
+ .expect("set partitions"));
let cancel = CancellationToken::new();
// A registered router that never acks parks every handoff in
@@ -4857,7 +4863,10 @@ async fn a_leader_that_goes_between_the_read_and_the_watch_is_still_delivered()
#[tokio::test]
async fn the_sweep_spares_a_membership_a_live_handoff_refers_to() {
let store = test_store("freeze-quorum-sweep-spares").await;
- store.set_total_partitions(2).await.expect("set partitions");
+ assert!(store
+ .seed_changelog_config("personhog_updates", 2, "personhog-writer")
+ .await
+ .expect("set partitions"));
let cancel = CancellationToken::new();
// A registered router that never acks parks the handoffs in
diff --git a/rust/personhog-identity/src/lifecycle/delete.rs b/rust/personhog-identity/src/lifecycle/delete.rs
index 8b023b4b332d..7afb4d66d79a 100644
--- a/rust/personhog-identity/src/lifecycle/delete.rs
+++ b/rust/personhog-identity/src/lifecycle/delete.rs
@@ -28,6 +28,7 @@ use serde::{Deserialize, Serialize};
use serde_json::Value;
use uuid::Uuid;
+use personhog_coordination::changelog::LiveChangelog;
use personhog_proto::personhog::types::v1::LifecycleOpType;
use crate::config::{IdentityTables, OutboxWrites};
@@ -150,9 +151,8 @@ pub struct DeleteDriver {
leader: Arc<dyn LifecycleLeader>,
tables: IdentityTables,
leader_call_concurrency: usize,
- /// The leaders' partition count, which decides which victims share a
- /// fence call.
- num_partitions: u32,
+ /// Read per batch so a flip takes effect on the next fence call.
+ changelog: LiveChangelog,
outbox_writes: OutboxWrites,
}
@@ -161,17 +161,17 @@ impl DeleteDriver {
leader: Arc<dyn LifecycleLeader>,
tables: IdentityTables,
leader_call_concurrency: usize,
- num_partitions: u32,
+ changelog: LiveChangelog,
outbox_writes: OutboxWrites,
) -> Self {
tables.validate().expect("invalid identity table set");
- assert!(num_partitions > 0, "num_partitions must be > 0");
+ assert!(changelog.partitions() > 0, "num_partitions must be > 0");
Self {
leader,
tables,
// Clamped to 1: a zero-width buffered stream never polls.
leader_call_concurrency: leader_call_concurrency.max(1),
- num_partitions,
+ changelog,
outbox_writes,
}
}
@@ -180,7 +180,7 @@ impl DeleteDriver {
LeaderCalls::new(
self.leader.as_ref(),
self.leader_call_concurrency,
- self.num_partitions,
+ self.changelog.partitions(),
)
}
}
diff --git a/rust/personhog-identity/src/lifecycle/mod.rs b/rust/personhog-identity/src/lifecycle/mod.rs
index aa446402e541..93158fb5ecc0 100644
--- a/rust/personhog-identity/src/lifecycle/mod.rs
+++ b/rust/personhog-identity/src/lifecycle/mod.rs
@@ -18,6 +18,7 @@ use std::sync::Arc;
use tonic::{Request, Response, Status};
+use personhog_coordination::changelog::LiveChangelog;
use personhog_proto::personhog::lifecycle::v1::person_hog_lifecycle_server::PersonHogLifecycle;
use personhog_proto::personhog::lifecycle::v1::{
DeletePersonOutcome, DeletePersonResult, DeletePersonsRequest, DeletePersonsResponse,
@@ -42,7 +43,7 @@ impl PersonHogLifecycleService {
leader: Arc<dyn LifecycleLeader>,
tables: crate::config::IdentityTables,
leader_call_concurrency: usize,
- num_partitions: u32,
+ changelog: LiveChangelog,
outbox_writes: OutboxWrites,
) -> Self {
Self {
@@ -51,7 +52,7 @@ impl PersonHogLifecycleService {
leader,
tables,
leader_call_concurrency,
- num_partitions,
+ changelog,
outbox_writes,
),
}
diff --git a/rust/personhog-identity/src/main.rs b/rust/personhog-identity/src/main.rs
index fd50cd93635c..58bb6a14211d 100644
--- a/rust/personhog-identity/src/main.rs
+++ b/rust/personhog-identity/src/main.rs
@@ -22,6 +22,9 @@ use tracing_subscriber::EnvFilter;
use personhog_common::client::RouterClient;
use personhog_common::query_tag;
+use personhog_coordination::changelog::{
+ keep_changelog_live, load_changelog_config, LiveChangelog, CHANGELOG_BOOT_POLL_INTERVAL,
+};
use personhog_coordination::store::PersonhogStore;
use personhog_identity::config::Config;
use personhog_identity::leader::LifecycleLeader;
@@ -35,6 +38,7 @@ use personhog_identity::service::merge::MergeEntrance;
use personhog_identity::service::PersonHogIdentityService;
use personhog_identity::storage::postgres::PostgresIdentityStorage;
use personhog_identity::warnings;
+use tokio_util::sync::CancellationToken;
common_alloc::used!();
@@ -144,19 +148,22 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
// The delete saga groups its fence calls by the leaders' partition
// count, which lives in etcd; a service that cannot read it would
// batch wrong, so refuse to start instead.
- let num_partitions = {
- let etcd_store = EtcdStore::connect(StoreConfig {
- endpoints: config.etcd_endpoint_list(),
- prefix: config.etcd_prefix.clone(),
- })
+ let etcd_store = EtcdStore::connect(StoreConfig {
+ endpoints: config.etcd_endpoint_list(),
+ prefix: config.etcd_prefix.clone(),
+ })
+ .await
+ .expect("Failed to connect to etcd");
+ let store = Arc::new(PersonhogStore::new(etcd_store));
+ let loaded = load_changelog_config(&store, CHANGELOG_BOOT_POLL_INTERVAL)
.await
- .expect("Failed to connect to etcd");
- PersonhogStore::new(etcd_store)
- .get_total_partitions()
- .await
- .expect("Failed to read total_partitions from etcd")
- };
- tracing::info!(num_partitions, "loaded partition count from etcd");
+ .expect("Failed to load the changelog config from etcd");
+ tracing::info!(
+ num_partitions = loaded.partitions,
+ epoch = loaded.epoch,
+ "loaded changelog config"
+ );
+ let changelog = LiveChangelog::new(loaded);
// Build lifecycle manager and register components
let mut manager = Manager::builder("personhog-identity").build();
@@ -184,6 +191,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
.is_advisory(true),
)
});
+ let changelog_handle = manager.register("changelog_config", ComponentOptions::new());
let readiness = manager.readiness_handler();
let liveness = manager.liveness_handler();
@@ -218,6 +226,19 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
.unwrap()
.install_recorder()
.expect("Failed to install metrics recorder");
+ {
+ let store = Arc::clone(&store);
+ let changelog = changelog.clone();
+ tokio::spawn(async move {
+ let _guard = changelog_handle.process_scope();
+ let cancel = CancellationToken::new();
+ tokio::select! {
+ _ = changelog_handle.shutdown_signal() => cancel.cancel(),
+ _ = keep_changelog_live(store, changelog, cancel.clone()) => {}
+ }
+ });
+ }
+
// Metrics/health HTTP server (observability handle — stays alive during standard drain)
let metrics_port = config.metrics_port;
tokio::spawn(async move {
@@ -348,7 +369,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
lifecycle_leader.clone(),
config.tables(),
config.lifecycle_leader_call_concurrency,
- num_partitions,
+ changelog.clone(),
outbox_writes,
);
let sweeper_engine = engine.clone();
@@ -413,7 +434,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
lifecycle_leader,
config.tables(),
config.lifecycle_leader_call_concurrency,
- num_partitions,
+ changelog.clone(),
outbox_writes,
);
let service = PersonHogIdentityService::new(
diff --git a/rust/personhog-identity/tests/common/mod.rs b/rust/personhog-identity/tests/common/mod.rs
index e8aec3faca99..defbfc2ad68f 100644
--- a/rust/personhog-identity/tests/common/mod.rs
+++ b/rust/personhog-identity/tests/common/mod.rs
@@ -11,6 +11,8 @@ use rand::Rng;
use sqlx::postgres::PgPool;
use uuid::Uuid;
+use personhog_coordination::changelog::LiveChangelog;
+use personhog_coordination::types::ChangelogConfig;
use personhog_identity::config::{IdentityTables, OutboxWrites};
use personhog_identity::pools::IdentityPools;
use personhog_identity::publish::config::MergeEventsAllowlist;
@@ -32,6 +34,10 @@ pub const FAN_OUT_CONCURRENCY: usize = 8;
/// Partition count the delete driver groups its fence batches by.
pub const NUM_PARTITIONS: u32 = 4;
+pub fn changelog() -> LiveChangelog {
+ LiveChangelog::new(ChangelogConfig::new("personhog_updates", NUM_PARTITIONS))
+}
+
pub fn default_tables() -> IdentityTables {
IdentityTables::real()
}
diff --git a/rust/personhog-identity/tests/lifecycle_delete_tests.rs b/rust/personhog-identity/tests/lifecycle_delete_tests.rs
index 9afac8986b07..1e1bf79b9b4c 100644
--- a/rust/personhog-identity/tests/lifecycle_delete_tests.rs
+++ b/rust/personhog-identity/tests/lifecycle_delete_tests.rs
@@ -36,7 +36,7 @@ impl TestContext {
leader.clone(),
self.tables.clone(),
common::FAN_OUT_CONCURRENCY,
- common::NUM_PARTITIONS,
+ common::changelog(),
outbox_writes,
)
}
@@ -597,7 +597,7 @@ impl FencedHarness {
leader.clone(),
ctx.tables.clone(),
common::FAN_OUT_CONCURRENCY,
- common::NUM_PARTITIONS,
+ common::changelog(),
OutboxWrites::Disabled,
);
Self {
diff --git a/rust/personhog-identity/tests/lifecycle_service_tests.rs b/rust/personhog-identity/tests/lifecycle_service_tests.rs
index 861042d428ce..c8dcb939a420 100644
--- a/rust/personhog-identity/tests/lifecycle_service_tests.rs
+++ b/rust/personhog-identity/tests/lifecycle_service_tests.rs
@@ -46,7 +46,7 @@ async fn delete_status(request: DeletePersonsRequest) -> Code {
Arc::new(SimLeader::new(pool, tables.clone())),
tables,
common::FAN_OUT_CONCURRENCY,
- common::NUM_PARTITIONS,
+ common::changelog(),
personhog_identity::config::OutboxWrites::Disabled,
);
service
diff --git a/rust/personhog-leader/src/config.rs b/rust/personhog-leader/src/config.rs
index 8265a03f3655..6a778dba4d2e 100644
--- a/rust/personhog-leader/src/config.rs
+++ b/rust/personhog-leader/src/config.rs
@@ -140,9 +140,6 @@ pub struct Config {
#[envconfig(nested = true)]
pub kafka: KafkaConfig,
- #[envconfig(default = "personhog_updates")]
- pub kafka_person_state_topic: String,
-
// ── Warming ──────────────────────────────────────────────────
/// Consumer group name used by the writer pod. The leader queries this
/// group's committed offsets during warming: any Kafka message at or
diff --git a/rust/personhog-leader/src/main.rs b/rust/personhog-leader/src/main.rs
index 1b266f1b428c..34a7d66222e9 100644
--- a/rust/personhog-leader/src/main.rs
+++ b/rust/personhog-leader/src/main.rs
@@ -14,6 +14,7 @@ use personhog_common::async_gzip::{AsyncGzipConfig, AsyncGzipLayer};
use personhog_common::grpc::{tracked_tcp_incoming, GrpcLoadShedLayer, GrpcMetricsLayer};
use personhog_common::metrics::{WARM_LATENCY_BUCKETS_MS, WRITE_PATH_LATENCY_BUCKETS_MS};
use personhog_coordination::authority::AuthorityClock;
+use personhog_coordination::changelog::{load_changelog_config, CHANGELOG_BOOT_POLL_INTERVAL};
use personhog_coordination::pod::{PodConfig, PodHandle};
use personhog_coordination::store::PersonhogStore;
use personhog_proto::personhog::leader::v1::person_hog_leader_server::PersonHogLeaderServer;
@@ -40,7 +41,9 @@ use personhog_leader::pg::{validate_table_name, LifecycleTables, PgFallback};
use personhog_leader::recovery::{ChangelogRecovery, RecoveryConfig};
use personhog_leader::service::{sweep_idle_locks, PersonHogLeaderService, PropertySizeLimits};
use personhog_leader::settle::prune_and_settle_tick;
-use personhog_leader::warming::{WarmClientPools, WarmingConfig, WarmingRetryPolicy};
+use personhog_leader::warming::{
+ changelog_partition_count, WarmClientPools, WarmingConfig, WarmingRetryPolicy,
+};
use personhog_leader::warnings::WarningsProducer;
common_alloc::used!();
@@ -96,7 +99,6 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
tracing::info!("etcd endpoints: {}", config.etcd_endpoints);
tracing::info!("etcd prefix: {}", config.etcd_prefix);
tracing::info!("Pod name: {}", config.pod_name);
- tracing::info!("Kafka changelog topic: {}", config.kafka_person_state_topic);
// Shutdown order matters: the coordination drain (phase 0) hands this
// pod's partitions off, which requires the gRPC server to keep serving
@@ -349,14 +351,32 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
.expect("Failed to connect to etcd");
let store = Arc::new(PersonhogStore::new(etcd_store));
- // Read total_partitions from etcd (set by kafka-assigner) — the same
- // source the router hashes against, so partition validation can never
- // drift between the two.
- let num_partitions = store
- .get_total_partitions()
+ // The same source the router hashes against.
+ let loaded = load_changelog_config(&store, CHANGELOG_BOOT_POLL_INTERVAL)
.await
- .expect("Failed to read total_partitions from etcd");
- tracing::info!(num_partitions, "loaded partition count from etcd");
+ .expect("Failed to load the changelog config from etcd");
+ let changelog_topic = loaded.topic.clone();
+ let num_partitions = loaded.partitions;
+ let topic_partitions =
+ changelog_partition_count(&config.kafka, &changelog_topic, Duration::from_secs(10))
+ .await
+ .expect("Failed to read the changelog topic's metadata");
+ assert_eq!(
+ topic_partitions, num_partitions,
+ "changelog topic {changelog_topic} has {topic_partitions} partitions but the config says {num_partitions}"
+ );
+ // Retirement settles marks on the writer group the config names, so it must be ours.
+ assert_eq!(
+ loaded.writer_group, config.writer_consumer_group,
+ "config/changelog names writer group {} but WRITER_CONSUMER_GROUP is {}",
+ loaded.writer_group, config.writer_consumer_group
+ );
+ tracing::info!(
+ num_partitions,
+ topic = %changelog_topic,
+ epoch = loaded.epoch,
+ "loaded changelog config"
+ );
let locks = Arc::new(DashMap::new());
let fences: personhog_leader::fence::FenceMap = Arc::new(DashMap::new());
@@ -372,7 +392,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
let recovery = Arc::new(
ChangelogRecovery::new(RecoveryConfig {
kafka: config.kafka.clone(),
- topic: config.kafka_person_state_topic.clone(),
+ topic: changelog_topic.clone(),
pod_name: config.pod_name.clone(),
recv_timeout: Duration::from_secs(config.recovery_recv_timeout_secs),
pool_size: config.recovery_pool_size,
@@ -413,7 +433,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
let fenced = Arc::new(
FencedChangelogProducers::new(FencedProducerConfig {
kafka: fencing_kafka,
- topic: config.kafka_person_state_topic.clone(),
+ topic: changelog_topic.clone(),
init_timeout: config.fencing_init_timeout(),
commit_timeout: config.fencing_txn_timeout(),
broker_txn_timeout: config.fencing_broker_txn_timeout(),
@@ -503,7 +523,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
Arc::clone(&dirty_index),
WarmingConfig {
kafka: config.kafka.clone(),
- topic: config.kafka_person_state_topic.clone(),
+ topic: changelog_topic.clone(),
pod_name: config.pod_name.clone(),
writer_consumer_group: config.writer_consumer_group.clone(),
lookback_offsets: config.warm_lookback_offsets,
@@ -650,7 +670,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
Arc::clone(&cache),
Arc::clone(&locks),
Arc::clone(&warm_pools),
- config.kafka_person_state_topic.clone(),
+ changelog_topic.clone(),
Duration::from_secs(config.warm_committed_offsets_timeout_secs),
Duration::from_secs(config.dirty_index_prune_interval_secs.max(1)),
));
diff --git a/rust/personhog-leader/src/warming.rs b/rust/personhog-leader/src/warming.rs
index 86da0e1adb36..55c73a14c5a9 100644
--- a/rust/personhog-leader/src/warming.rs
+++ b/rust/personhog-leader/src/warming.rs
@@ -112,6 +112,43 @@ pub struct WarmingConfig {
pub retry: WarmingRetryPolicy,
}
+/// The topic's partition count from broker metadata, for the boot check.
+pub async fn changelog_partition_count(
+ kafka: &KafkaConfig,
+ topic: &str,
+ timeout: Duration,
+) -> CoordResult<u32> {
+ let consumer = make_consumer(kafka, "personhog-leader-boot")
+ .map_err(|e| CoordError::invalid_state(format!("create metadata client: {e}")))?;
+ let topic_name = topic.to_string();
+ tokio::task::spawn_blocking(move || {
+ let metadata = consumer
+ .fetch_metadata(Some(&topic_name), timeout)
+ .map_err(|e| {
+ CoordError::invalid_state(format!("fetch metadata for {topic_name}: {e}"))
+ })?;
+ let topic_metadata = metadata
+ .topics()
+ .iter()
+ .find(|t| t.name() == topic_name)
+ .ok_or_else(|| {
+ CoordError::invalid_state(format!(
+ "topic {topic_name} is absent from broker metadata"
+ ))
+ })?;
+ if let Some(error) = topic_metadata.error() {
+ return Err(CoordError::invalid_state(format!(
+ "topic {topic_name}: {error:?}"
+ )));
+ }
+ u32::try_from(topic_metadata.partitions().len()).map_err(|_| {
+ CoordError::invalid_state(format!("topic {topic_name} partition count overflow"))
+ })
+ })
+ .await
+ .map_err(|e| CoordError::invalid_state(format!("fetch metadata join: {e}")))?
+}
+
/// Build a consumer used by warming. Auto-commit and auto-offset-store are
/// disabled so this is safe to instantiate with any group id — including
/// the writer's, where mutating stored offsets would corrupt the writer's
diff --git a/rust/personhog-leader/tests/integration.rs b/rust/personhog-leader/tests/integration.rs
index f9f5b6ae9547..33ebef66a429 100644
--- a/rust/personhog-leader/tests/integration.rs
+++ b/rust/personhog-leader/tests/integration.rs
@@ -71,7 +71,10 @@ async fn service_accepts_requests_after_coordination_warmup() {
let store = test_store("leader-warmup").await;
let cancel = CancellationToken::new();
- store.set_total_partitions(NUM_PARTITIONS).await.unwrap();
+ assert!(store
+ .seed_changelog_config(common::CHANGELOG_TOPIC, NUM_PARTITIONS, "personhog-writer")
+ .await
+ .unwrap());
let _coord = start_coordinator(
Arc::clone(&store),
@@ -700,7 +703,10 @@ async fn release_partition_stops_serving() {
let store = test_store("leader-release").await;
let cancel = CancellationToken::new();
- store.set_total_partitions(NUM_PARTITIONS).await.unwrap();
+ assert!(store
+ .seed_changelog_config(common::CHANGELOG_TOPIC, NUM_PARTITIONS, "personhog-writer")
+ .await
+ .unwrap());
let _coord = start_coordinator(
Arc::clone(&store),
@@ -818,7 +824,10 @@ async fn rewarm_after_pod_crash() {
let store = test_store("leader-rewarm").await;
let cancel = CancellationToken::new();
- store.set_total_partitions(NUM_PARTITIONS).await.unwrap();
+ assert!(store
+ .seed_changelog_config(common::CHANGELOG_TOPIC, NUM_PARTITIONS, "personhog-writer")
+ .await
+ .unwrap());
let _coord = start_coordinator(
Arc::clone(&store),
diff --git a/rust/personhog-router/src/backend/leader.rs b/rust/personhog-router/src/backend/leader.rs
index c9d60e6015c2..2b408cbadd73 100644
--- a/rust/personhog-router/src/backend/leader.rs
+++ b/rust/personhog-router/src/backend/leader.rs
@@ -21,6 +21,7 @@ use personhog_common::grpc::{current_client_name, SEMANTIC_REFUSAL_METADATA_KEY}
const FENCED_METADATA_KEY: &str = "x-person-fenced";
const FENCED_OP_ID_METADATA_KEY: &str = "x-person-fenced-op-id";
use personhog_common::partitioning::partition_for_person;
+use personhog_coordination::changelog::LiveChangelog;
use super::stash::{StashDecision, StashTable};
use crate::config::Http2Windows;
@@ -117,7 +118,7 @@ pub enum ForwardDecision {
/// Static configuration for `LeaderBackend`. Bundles the knobs that come
/// from `Config` so the constructor stays narrow as we add fields.
pub struct LeaderBackendConfig {
- pub num_partitions: u32,
+ pub changelog: LiveChangelog,
pub timeout: Duration,
pub num_channels: usize,
pub http2_windows: Http2Windows,
@@ -177,8 +178,8 @@ impl LeaderBackend {
stash: StashTable,
) -> Self {
assert!(
- config.num_partitions > 0,
- "num_partitions must be > 0 to avoid division by zero in partition_for_person"
+ config.changelog.partitions() > 0,
+ "the changelog must have partitions to avoid division by zero in partition_for_person"
);
Self {
routing_table,
@@ -195,6 +196,11 @@ impl LeaderBackend {
self.stash.clone()
}
+ /// The live config this backend routes on.
+ pub fn changelog(&self) -> &LiveChangelog {
+ &self.config.changelog
+ }
+
/// Remove the cached gRPC channel for a pod so the next request
/// reconnects. Called during partition handoff cutover to drop the
/// connection to the old leader pod.
@@ -207,7 +213,7 @@ impl LeaderBackend {
/// Compute the Kafka partition for a person using murmur2.
/// The key is `team_id:person_id`, matching the Kafka topic key.
pub fn partition_for_person(&self, team_id: i64, person_id: i64) -> u32 {
- partition_for_person(team_id, person_id, self.config.num_partitions)
+ partition_for_person(team_id, person_id, self.config.changelog.partitions())
}
/// Resolve the leader gRPC channel for a given partition, building and
@@ -543,10 +549,15 @@ mod tests {
}
use super::*;
+ use personhog_coordination::types::ChangelogConfig;
+
+ fn changelog(num_partitions: u32) -> LiveChangelog {
+ LiveChangelog::new(ChangelogConfig::new("personhog_updates", num_partitions))
+ }
fn test_config(num_partitions: u32) -> LeaderBackendConfig {
LeaderBackendConfig {
- num_partitions,
+ changelog: changelog(num_partitions),
timeout: Duration::from_secs(5),
num_channels: 4,
http2_windows: Http2Windows::default(),
diff --git a/rust/personhog-router/src/main.rs b/rust/personhog-router/src/main.rs
index 6aefb619af9a..3035bdee6ad4 100644
--- a/rust/personhog-router/src/main.rs
+++ b/rust/personhog-router/src/main.rs
@@ -9,6 +9,9 @@ use lifecycle::{ComponentOptions, Handle, Manager};
use metrics_exporter_prometheus::{Matcher, PrometheusBuilder, PrometheusHandle};
use personhog_common::grpc::{tracked_tcp_incoming, GrpcMetricsLayer};
use personhog_common::metrics::WRITE_PATH_LATENCY_BUCKETS_MS;
+use personhog_coordination::changelog::{
+ keep_changelog_live, load_changelog_config, LiveChangelog, CHANGELOG_BOOT_POLL_INTERVAL,
+};
use personhog_coordination::coordinator::{Coordinator, CoordinatorConfig};
use personhog_coordination::routing_table::{RoutingTable, RoutingTableConfig, StashHandler};
use personhog_coordination::store::PersonhogStore;
@@ -151,6 +154,8 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
let identity_discovery_handle = (config.identity_enabled
&& config.identity_discovery_mode == DiscoveryMode::K8s)
.then(|| manager.register("identity-discovery", discovery_options()));
+ let changelog_handle = (config.router_mode == RouterMode::Leader)
+ .then(|| manager.register("changelog-config", ComponentOptions::new()));
let readiness = manager.readiness_handler();
let liveness = manager.liveness_handler();
@@ -253,12 +258,16 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
.expect("Failed to connect to etcd");
let store = Arc::new(PersonhogStore::new(etcd_store));
- // Read total_partitions from etcd (set by kafka-assigner)
- let num_partitions = store
- .get_total_partitions()
+ let loaded = load_changelog_config(&store, CHANGELOG_BOOT_POLL_INTERVAL)
.await
- .expect("Failed to read total_partitions from etcd");
- tracing::info!(num_partitions, "loaded partition count from etcd");
+ .expect("Failed to load the changelog config from etcd");
+ let num_partitions = loaded.partitions;
+ tracing::info!(
+ num_partitions,
+ epoch = loaded.epoch,
+ "loaded changelog config"
+ );
+ let changelog = LiveChangelog::new(loaded);
// Build the routing table and leader backend, sharing the same
// partition-to-pod mapping so both see consistent state.
@@ -294,7 +303,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
.map(|address| format!("http://{address}"))
}),
LeaderBackendConfig {
- num_partitions,
+ changelog,
timeout: config.backend_timeout(),
num_channels: config.leader_channels,
http2_windows: config.backend_http2_windows(),
@@ -305,6 +314,22 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
),
));
+ // New requests hash under the live partition count.
+ {
+ let store = Arc::clone(&store);
+ let changelog = leader_backend.changelog().clone();
+ let handle =
+ changelog_handle.expect("changelog handle must be registered in leader mode");
+ tokio::spawn(async move {
+ let _guard = handle.process_scope();
+ let cancel = CancellationToken::new();
+ tokio::select! {
+ _ = handle.shutdown_signal() => cancel.cancel(),
+ _ = keep_changelog_live(store, changelog, cancel.clone()) => {}
+ }
+ });
+ }
+
let stash_handler: Arc<dyn StashHandler> = Arc::new(RouterStashHandler::new(
Arc::clone(&leader_backend),
config.stash_max_wait(),
diff --git a/rust/personhog-router/tests/common/mod.rs b/rust/personhog-router/tests/common/mod.rs
index de96f4fffeda..c22cd5e4dd44 100644
--- a/rust/personhog-router/tests/common/mod.rs
+++ b/rust/personhog-router/tests/common/mod.rs
@@ -9,6 +9,8 @@ use std::time::Duration;
use dashmap::DashMap;
use personhog_common::async_gzip::{AsyncGzipConfig, AsyncGzipLayer};
use personhog_common::grpc::NOT_APPLIED_HEADER;
+use personhog_coordination::changelog::LiveChangelog;
+use personhog_coordination::types::ChangelogConfig;
use personhog_proto::personhog::identity::v1 as identity;
use personhog_proto::personhog::identity::v1::person_hog_identity_server::{
PersonHogIdentity, PersonHogIdentityServer,
@@ -987,11 +989,12 @@ fn make_leader_backend(leader_addr: SocketAddr, num_partitions: u32) -> Arc<Lead
let leader_url = format!("http://{}", leader_addr);
let address_resolver: Arc<dyn Fn(&str) -> Option<String> + Send + Sync> =
Arc::new(move |_pod_name| Some(leader_url.clone()));
+ let changelog = LiveChangelog::new(ChangelogConfig::new("personhog_updates", num_partitions));
Arc::new(LeaderBackend::new(
routing_table,
address_resolver,
LeaderBackendConfig {
- num_partitions,
+ changelog,
timeout: Duration::from_secs(5),
num_channels: 1,
http2_windows: Http2Windows::default(),
@@ -1021,11 +1024,12 @@ fn make_dying_leader_backend(leader_addr: SocketAddr, num_partitions: u32) -> Ar
Some("http://192.0.2.1:1".to_string())
}
});
+ let changelog = LiveChangelog::new(ChangelogConfig::new("personhog_updates", num_partitions));
Arc::new(LeaderBackend::new(
routing_table,
address_resolver,
LeaderBackendConfig {
- num_partitions,
+ changelog,
timeout: Duration::from_millis(200),
num_channels: 1,
http2_windows: Http2Windows::default(),
diff --git a/rust/personhog-router/tests/h2_window_wiring.rs b/rust/personhog-router/tests/h2_window_wiring.rs
index 5f2d696d4c58..a6dc9634c6bd 100644
--- a/rust/personhog-router/tests/h2_window_wiring.rs
+++ b/rust/personhog-router/tests/h2_window_wiring.rs
@@ -9,6 +9,8 @@ use common::{
h2_frame, read_h2_frame, H2Frame, TestLeaderService, H2_FRAME_SETTINGS, H2_FRAME_WINDOW_UPDATE,
H2_PREFACE,
};
+use personhog_coordination::changelog::LiveChangelog;
+use personhog_coordination::types::ChangelogConfig;
use personhog_proto::personhog::leader::v1::person_hog_leader_server::PersonHogLeaderServer;
use personhog_router::backend::discovery::{build_discovered_endpoint, EndpointConfig};
use personhog_router::backend::{
@@ -136,11 +138,12 @@ fn discovered_channel(addr: SocketAddr, http2_windows: Http2Windows) -> Channel
async fn leader_channel(addr: SocketAddr, http2_windows: Http2Windows) -> Channel {
let mut routing = HashMap::new();
routing.insert(0, "leader-0".to_string());
+ let changelog = LiveChangelog::new(ChangelogConfig::new("personhog_updates", 1));
let backend = LeaderBackend::new(
Arc::new(RwLock::new(routing)),
Arc::new(move |_| Some(format!("http://{addr}"))),
LeaderBackendConfig {
- num_partitions: 1,
+ changelog,
timeout: Duration::from_secs(5),
num_channels: 1,
http2_windows,
diff --git a/rust/personhog-router/tests/stash_integration.rs b/rust/personhog-router/tests/stash_integration.rs
index 058e4bfaa5d4..bb3a8af5fc6a 100644
--- a/rust/personhog-router/tests/stash_integration.rs
+++ b/rust/personhog-router/tests/stash_integration.rs
@@ -15,7 +15,9 @@ use bytes::Bytes;
use common::{create_test_person, start_test_leader, start_test_leader_at, TestLeaderService};
use http::HeaderMap;
use http_body_util::BodyExt;
+use personhog_coordination::changelog::LiveChangelog;
use personhog_coordination::routing_table::StashHandler;
+use personhog_coordination::types::ChangelogConfig;
use personhog_proto::personhog::types::v1::{
FoldPersonDocumentRequest, FoldPersonDocumentResponse, GetPersonRequest, GetPersonResponse,
Person, SealedSourceSnapshot, UpdatePersonPropertiesRequest, UpdatePersonPropertiesResponse,
@@ -64,7 +66,10 @@ async fn make_backend(leader_addr: std::net::SocketAddr, stash: StashTable) -> A
routing_table,
resolver,
LeaderBackendConfig {
- num_partitions: NUM_PARTITIONS,
+ changelog: LiveChangelog::new(ChangelogConfig::new(
+ "personhog_updates",
+ NUM_PARTITIONS,
+ )),
timeout: Duration::from_secs(5),
num_channels: 1,
http2_windows: Http2Windows::default(),
diff --git a/rust/personhog-test-harness/src/cli.rs b/rust/personhog-test-harness/src/cli.rs
index 96b7e3bd0985..595c2f5df2ee 100644
--- a/rust/personhog-test-harness/src/cli.rs
+++ b/rust/personhog-test-harness/src/cli.rs
@@ -660,14 +660,13 @@ pub struct TrafficArgs {
/// Environment the stack assigns per leader; overriding any of these
/// would break pod identity or point a spawned leader at the wrong
-/// topic or table, and the resulting run would look like a real result.
+/// table, and the resulting run would look like a real result.
const RESERVED_LEADER_ENV: &[&str] = &[
"POD_NAME",
"GRPC_ADDRESS",
"METRICS_PORT",
"ETCD_ENDPOINTS",
"ETCD_PREFIX",
- "KAFKA_PERSON_STATE_TOPIC",
"FALLBACK_TABLE",
"LIFECYCLE_OP_TABLE",
"LIFECYCLE_OP_PERSON_TABLE",
diff --git a/rust/personhog-test-harness/src/stack/etcd.rs b/rust/personhog-test-harness/src/stack/etcd.rs
index bb501a077f4f..7eeea79eb836 100644
--- a/rust/personhog-test-harness/src/stack/etcd.rs
+++ b/rust/personhog-test-harness/src/stack/etcd.rs
@@ -14,14 +14,18 @@ pub async fn connect(endpoints: &str, prefix: &str) -> Result<PersonhogStore> {
Ok(PersonhogStore::new(store))
}
-/// Wipe all coordination state under the harness prefix and set the
-/// partition count, giving each run a clean slate.
-pub async fn reset(store: &PersonhogStore, partitions: u32) -> Result<()> {
+/// Wipe the harness prefix and seed the config as the control plane would.
+pub async fn reset(
+ store: &PersonhogStore,
+ topic: &str,
+ partitions: u32,
+ writer_group: &str,
+) -> Result<()> {
store.delete_all().await.context("clearing etcd prefix")?;
store
- .set_total_partitions(partitions)
+ .seed_changelog_config(topic, partitions, writer_group)
.await
- .context("setting total_partitions")?;
+ .context("seeding config/changelog")?;
Ok(())
}
diff --git a/rust/personhog-test-harness/src/stack/kafka.rs b/rust/personhog-test-harness/src/stack/kafka.rs
index 360056a46f41..580db3b417b4 100644
--- a/rust/personhog-test-harness/src/stack/kafka.rs
+++ b/rust/personhog-test-harness/src/stack/kafka.rs
@@ -19,7 +19,7 @@ fn admin_client(kafka_hosts: &str) -> Result<AdminClient<DefaultClientContext>>
}
/// Create a per-run topic with exactly `partitions` partitions. For the
-/// changelog the count must match `total_partitions` in etcd — a mismatch
+/// changelog the count must match `config/changelog` in etcd — a mismatch
/// is a silent mis-shard — so an existing topic with a different count is
/// an error rather than something to adopt.
pub async fn create_topic(kafka_hosts: &str, topic: &str, partitions: u32) -> Result<()> {
diff --git a/rust/personhog-test-harness/src/stack/mod.rs b/rust/personhog-test-harness/src/stack/mod.rs
index bc8518a81722..531bbfd6ecd1 100644
--- a/rust/personhog-test-harness/src/stack/mod.rs
+++ b/rust/personhog-test-harness/src/stack/mod.rs
@@ -32,6 +32,7 @@ const REPLICA_METRICS_PORT: u16 = 24151;
const IDENTITY_METRICS_PORT: u16 = 24152;
const WRITER_METRICS_PORT: u16 = 24153;
const ROUTER_METRICS_BASE_PORT: u16 = 24154;
+const WRITER_CONSUMER_GROUP: &str = "personhog-test-harness-writer";
const LEADER_METRICS_BASE_PORT: u16 = 24160;
/// Above the leader range, which grows with every scale-up.
const PUBLISHER_METRICS_BASE_PORT: u16 = 24200;
@@ -228,7 +229,7 @@ impl Stack {
);
let store = etcd::connect(&config.etcd_endpoints, ETCD_PREFIX).await?;
- etcd::reset(&store, config.partitions).await?;
+ etcd::reset(&store, &topic, config.partitions, WRITER_CONSUMER_GROUP).await?;
let publish_topics = config.publish.as_ref().map(|_| PublishTopics {
distinct_ids: format!("{topic}_distinct_ids"),
merge_events: format!("{topic}_merge_events"),
@@ -259,11 +260,9 @@ impl Stack {
&[
("DATABASE_URL", config.persons_db_url.clone()),
("KAFKA_HOSTS", config.kafka_hosts.clone()),
- ("KAFKA_TOPIC", topic.clone()),
- (
- "KAFKA_CONSUMER_GROUP",
- "personhog-test-harness-writer".to_string(),
- ),
+ ("ETCD_ENDPOINTS", config.etcd_endpoints.clone()),
+ ("ETCD_PREFIX", ETCD_PREFIX.to_string()),
+ ("KAFKA_CONSUMER_GROUP", WRITER_CONSUMER_GROUP.to_string()),
("PG_TARGET_TABLE", config.pg_target_table.clone()),
(
"FLUSH_INTERVAL_MS",
@@ -527,14 +526,10 @@ impl Stack {
("ETCD_ENDPOINTS", self.config.etcd_endpoints.clone()),
("ETCD_PREFIX", ETCD_PREFIX.to_string()),
("KAFKA_HOSTS", self.config.kafka_hosts.clone()),
- ("KAFKA_PERSON_STATE_TOPIC", self.topic.clone()),
// Must match the spawned writer's consumer group or the
// leader's committed-offset queries (warming ranges, dirty
// index pruning) silently see no progress at all.
- (
- "WRITER_CONSUMER_GROUP",
- "personhog-test-harness-writer".to_string(),
- ),
+ ("WRITER_CONSUMER_GROUP", WRITER_CONSUMER_GROUP.to_string()),
("FALLBACK_DATABASE_URL", self.config.persons_db_url.clone()),
// Pair the fallback with the spawned writer's target: the
// dirty index treats an unmarked person's PG row as
diff --git a/rust/personhog-writer/Cargo.toml b/rust/personhog-writer/Cargo.toml
index 027cfe866cdf..b5bb4f39afe5 100644
--- a/rust/personhog-writer/Cargo.toml
+++ b/rust/personhog-writer/Cargo.toml
@@ -12,6 +12,8 @@ common-alloc = { path = "../common/alloc" }
common-liveness = { path = "../common/liveness" }
common-metrics = { path = "../common/metrics" }
lifecycle = { path = "../common/lifecycle" }
+assignment-coordination = { path = "../common/assignment-coordination" }
+personhog-coordination = { path = "../personhog-coordination" }
async-trait = { workspace = true }
axum = { workspace = true }
diff --git a/rust/personhog-writer/README.md b/rust/personhog-writer/README.md
index e5c61cfa589f..3d19597b83da 100644
--- a/rust/personhog-writer/README.md
+++ b/rust/personhog-writer/README.md
@@ -136,9 +136,10 @@ Consumer lag per partition is monitored externally via KMinion (`kminion_kafka_c
| `KAFKA_TLS` | `false` | Enable TLS for Kafka |
| `KAFKA_CLIENT_ID` | (empty) | Pod name for static group membership |
| `KAFKA_CLIENT_RACK` | (empty) | Rack ID for rack-aware consumption |
-| `KAFKA_TOPIC` | `personhog_updates` | Topic to consume |
| `KAFKA_CONSUMER_GROUP` | `personhog-writer` | Consumer group ID |
| `KAFKA_CONSUMER_OFFSET_RESET` | `earliest` | Offset reset policy |
+| `ETCD_ENDPOINTS` | `http://localhost:2379` | etcd endpoints; the topic comes from `config/changelog` |
+| `ETCD_PREFIX` | `/personhog/` | etcd key prefix |
| `DATABASE_URL` | (required) | Postgres connection string |
| `PG_MAX_CONNECTIONS` | `20` | Connection pool size |
| `PG_TARGET_TABLE` | `personhog_person_tmp` | Target table (`posthog_person` for production cutover) |
diff --git a/rust/personhog-writer/src/config.rs b/rust/personhog-writer/src/config.rs
index 773e802e4360..d8852efdc20f 100644
--- a/rust/personhog-writer/src/config.rs
+++ b/rust/personhog-writer/src/config.rs
@@ -9,9 +9,6 @@ pub struct Config {
#[envconfig(nested = true)]
pub kafka: KafkaConfig,
- #[envconfig(default = "personhog_updates")]
- pub kafka_topic: String,
-
#[envconfig(default = "personhog-writer")]
pub kafka_consumer_group: String,
@@ -80,6 +77,13 @@ pub struct Config {
#[envconfig(default = "1")]
pub writer_lanes: usize,
+ // ── etcd ─────────────────────────────────────────────────────
+ #[envconfig(default = "http://localhost:2379")]
+ pub etcd_endpoints: String,
+
+ #[envconfig(default = "/personhog/")]
+ pub etcd_prefix: String,
+
// ── Service ──────────────────────────────────────────────────
#[envconfig(default = "9103")]
pub metrics_port: u16,
@@ -89,4 +93,13 @@ impl Config {
pub fn flush_interval(&self) -> Duration {
Duration::from_millis(self.flush_interval_ms)
}
+
+ pub fn etcd_endpoint_list(&self) -> Vec<String> {
+ self.etcd_endpoints
+ .split(',')
+ .map(str::trim)
+ .filter(|s| !s.is_empty())
+ .map(String::from)
+ .collect()
+ }
}
diff --git a/rust/personhog-writer/src/main.rs b/rust/personhog-writer/src/main.rs
index ff7aceec0b47..6f2bcb812978 100644
--- a/rust/personhog-writer/src/main.rs
+++ b/rust/personhog-writer/src/main.rs
@@ -1,11 +1,14 @@
use std::sync::Arc;
use std::time::Duration;
+use assignment_coordination::store::{EtcdStore, StoreConfig};
use axum::{routing::get, Router};
use common_database::{get_pool_with_config, PoolConfig};
use common_metrics::{setup_metrics_routes_with_overrides, Matcher};
use envconfig::Envconfig;
use lifecycle::{ComponentOptions, Manager};
+use personhog_coordination::changelog::{load_changelog_config, CHANGELOG_BOOT_POLL_INTERVAL};
+use personhog_coordination::store::PersonhogStore;
use tokio::sync::mpsc;
use tracing::level_filters::LevelFilter;
use tracing_subscriber::fmt;
@@ -41,7 +44,6 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
.init();
tracing::info!("Starting personhog-writer");
- tracing::info!("Kafka topic: {}", config.kafka_topic);
tracing::info!("Consumer group: {}", config.kafka_consumer_group);
tracing::info!("Flush interval: {}ms", config.flush_interval_ms);
tracing::info!("Flush buffer size: {}", config.flush_buffer_size);
@@ -50,6 +52,26 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
tracing::info!("Upsert concurrency: {}", config.upsert_concurrency);
tracing::info!("Metrics port: {}", config.metrics_port);
+ let store = EtcdStore::connect(StoreConfig {
+ endpoints: config.etcd_endpoint_list(),
+ prefix: config.etcd_prefix.clone(),
+ })
+ .await?;
+ let loaded =
+ load_changelog_config(&PersonhogStore::new(store), CHANGELOG_BOOT_POLL_INTERVAL).await?;
+ if loaded.writer_group != config.kafka_consumer_group {
+ return Err(format!(
+ "config/changelog names writer group {} but KAFKA_CONSUMER_GROUP is {}",
+ loaded.writer_group, config.kafka_consumer_group
+ )
+ .into());
+ }
+ tracing::info!(
+ topic = %loaded.topic,
+ epoch = loaded.epoch,
+ "loaded changelog config"
+ );
+
let mut manager = Manager::builder("personhog-writer")
.with_global_shutdown_timeout(Duration::from_secs(30))
.build();
@@ -152,9 +174,9 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
&config.kafka,
&config.kafka_consumer_group,
&config.kafka_consumer_offset_reset,
- config.kafka_topic.clone(),
+ loaded.topic.clone(),
)?);
- tracing::info!("Subscribed to Kafka topic: {}", config.kafka_topic);
+ tracing::info!("Subscribed to Kafka topic: {}", loaded.topic);
// One statement-concurrency budget for the whole pod: every lane's
// chunk and per-row statements draw from it, so lanes can overlap