:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
use std::collections::HashMap;
use std::sync::{Arc, Mutex as StdMutex};
use assignment_coordination::store::EtcdStore;
use etcd_client::{
Compare, CompareOp, DeleteOptions, PutOptions, Txn, TxnOp, TxnOpResponse, WatchStream,
};
use crate::error::{Error, Result};
use crate::types::{
AssignmentPrecondition, AssignmentStatus, ChangelogConfig, ClusterEndpoint, HandoffReplacement,
HandoffState, LeaderInfo, MigrationState, PartitionAssignment, PodDrainedAck, PodStatus,
PodWarmedAck, RegisteredPod, RegisteredRouter, RouterFreezeAck,
};
mod migration;
/// All etcd key patterns used by the PersonHog store.
enum StoreKey<'a> {
Pod(&'a str),
PodsPrefix,
Router(&'a str),
RoutersPrefix,
Assignment(u32),
AssignmentsPrefix,
Handoff(u32),
HandoffsPrefix,
// Freeze acks: each router writes one when it has begun stashing for a partition.
FreezeAck { partition: u32, router: &'a str },
FreezeAcksForPartition(u32),
FreezeAcksPrefix,
// Drained acks: the old owner writes one when all inflight handlers have completed.
DrainedAck { partition: u32, pod: &'a str },
DrainedAcksForPartition(u32),
DrainedAcksPrefix,
// Warmed acks: the new owner writes one after consuming Kafka up to the stable HWM.
WarmedAck { partition: u32, pod: &'a str },
WarmedAcksForPartition(u32),
WarmedAcksPrefix,
// Freeze quorum membership, shared by every handoff one plan creates.
FreezeQuorum(&'a str),
FreezeQuorumsPrefix,
Leader,
Generation,
ChangelogConfig,
Migration,
MigrationRequest,
MigrationHistory(&'a str),
MigrationHistoryPrefix,
SealAck(&'a str),
SealAcksPrefix,
DrainReport { topic: &'a str, group: &'a str },
DrainReportsForTopic(&'a str),
DrainReportsPrefix,
}
impl StoreKey<'_> {
fn resolve(&self, prefix: &str) -> String {
match self {
StoreKey::Pod(name) => format!("{prefix}pods/{name}"),
StoreKey::PodsPrefix => format!("{prefix}pods/"),
StoreKey::Router(name) => format!("{prefix}routers/{name}"),
StoreKey::RoutersPrefix => format!("{prefix}routers/"),
StoreKey::Assignment(p) => format!("{prefix}assignments/{p}"),
StoreKey::AssignmentsPrefix => format!("{prefix}assignments/"),
StoreKey::Handoff(p) => format!("{prefix}handoffs/{p}"),
StoreKey::HandoffsPrefix => format!("{prefix}handoffs/"),
StoreKey::FreezeAck { partition, router } => {
format!("{prefix}freeze_acks/{partition}/{router}")
}
StoreKey::FreezeAcksForPartition(p) => format!("{prefix}freeze_acks/{p}/"),
StoreKey::FreezeAcksPrefix => format!("{prefix}freeze_acks/"),
StoreKey::DrainedAck { partition, pod } => {
format!("{prefix}drained_acks/{partition}/{pod}")
}
StoreKey::DrainedAcksForPartition(p) => format!("{prefix}drained_acks/{p}/"),
StoreKey::DrainedAcksPrefix => format!("{prefix}drained_acks/"),
StoreKey::WarmedAck { partition, pod } => {
format!("{prefix}warmed_acks/{partition}/{pod}")
}
StoreKey::WarmedAcksForPartition(p) => format!("{prefix}warmed_acks/{p}/"),
StoreKey::WarmedAcksPrefix => format!("{prefix}warmed_acks/"),
StoreKey::FreezeQuorum(id) => format!("{prefix}freeze_quorums/{id}"),
StoreKey::FreezeQuorumsPrefix => format!("{prefix}freeze_quorums/"),
StoreKey::Leader => format!("{prefix}coordinator/leader"),
StoreKey::Generation => format!("{prefix}generation"),
StoreKey::ChangelogConfig => format!("{prefix}config/changelog"),
StoreKey::Migration => format!("{prefix}migrations/current"),
StoreKey::MigrationRequest => format!("{prefix}migrations/requested"),
StoreKey::MigrationHistory(id) => format!("{prefix}migrations/history/{id}"),
StoreKey::MigrationHistoryPrefix => format!("{prefix}migrations/history/"),
StoreKey::SealAck(pod) => format!("{prefix}seal_acks/{pod}"),
StoreKey::SealAcksPrefix => format!("{prefix}seal_acks/"),
StoreKey::DrainReport { topic, group } => {
format!("{prefix}changelog/drained/{topic}/{group}")
}
StoreKey::DrainReportsForTopic(topic) => format!("{prefix}changelog/drained/{topic}/"),
StoreKey::DrainReportsPrefix => format!("{prefix}changelog/drained/"),
}
}
}
/// Domain-specific store for PersonHog coordination state.
///
/// Wraps the shared `EtcdStore` (generic JSON helpers, lease ops) and adds
/// PersonHog-specific key resolution and domain operations.
#[derive(Clone)]
pub struct PersonhogStore {
inner: EtcdStore,
/// Freeze-quorum memberships already read, by record id. An id
/// names one immutable value (a chunked plan re-puts it only
/// byte-identically), so a recorded membership cannot go stale; a
/// plan's handoffs share one id, so this turns a few hundred reads
/// per reconcile pass into one. Absence is never cached — the
/// sweep can delete a record a later chunk re-creates.
freeze_quorums: Arc<StdMutex<HashMap<String, Vec<String>>>>,
}
/// Counts store calls by the method that made them. The shared etcd
/// layer labels by primitive — `get`, `list_with_revision` — which is
/// too coarse to attribute load: a single `get` label covers the
/// coordinator's handoff read, a pod's convergence read, and every
/// other single-key lookup in the system.
pub(crate) fn count_call(site: &'static str) {
metrics::counter!("personhog_coordination_store_calls_total", "site" => site).increment(1);
}
/// How a plan application ended, per partition.
#[derive(Debug, Default)]
pub struct PlanApplication {
/// Partitions whose disposition landed.
pub applied: Vec<u32>,
/// Partitions whose guards failed — a record changed between the
/// planner's read and the write; the next plan re-reads them.
pub conflicted: Vec<u32>,
/// Chunks the server refused as too large and that applied per
/// unit instead: the configured budget exceeds the server's.
pub over_budget_chunks: usize,
/// Partitions dropped because the record the plan was decided on moved
/// after it was read (see `Authority`).
pub stood_down: Vec<u32>,
}
impl PlanApplication {
fn stand_down(&mut self, units: &[PlanUnit], authority: Option<Authority>) {
self.stood_down.extend(units.iter().map(|u| u.partition));
let by = match authority {
Some(Authority::Migration { .. }) => "driver",
_ => "planner",
};
metrics::counter!("personhog_coordination_plan_stood_down_total", "by" => by).increment(1);
}
}
/// The changelog config and migration record a planner plan rests on; its transactions
/// require both unchanged. The config compares by mod revision (a flip or retire moves it),
/// the migration by create revision (a new or archived saga moves it).
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct PlanBasis {
pub config: Option<ChangelogConfig>,
pub migration: Option<MigrationState>,
config_revision: i64,
migration_revision: i64,
}
/// The record a coordinator plan was decided on. Its transactions require it
/// unchanged, so a stalled pass that wakes after a successor acted lands nothing.
#[derive(Debug, Clone, Copy)]
enum Authority {
/// The planner: its plan basis.
Planner {
config_revision: i64,
migration_revision: i64,
},
/// The migration driver: the saga record at the revision its pass read.
Migration { mod_revision: i64 },
}
/// One partition's slice of a plan: the guards that make its
/// disposition safe and the ops that apply it. Chunking never splits a
/// unit, so a partition's disposition stays atomic across any split.
struct PlanUnit {
partition: u32,
guards: Vec<UnitGuard>,
ops: Vec<UnitOp>,
bytes: usize,
references_quorum: bool,
}
enum UnitGuard {
/// The key was never created (create_revision 0).
Absent(String),
/// The key's mod_revision is exactly this.
ModRevision(String, i64),
}
impl UnitGuard {
fn key(&self) -> &str {
match self {
UnitGuard::Absent(key) | UnitGuard::ModRevision(key, _) => key,
}
}
}
enum UnitOp {
Put(String, Vec<u8>),
DeletePrefix(String),
}
/// Whether etcd refused a transaction for its size — past
/// `--max-txn-ops` (`ErrGRPCTooManyOps`) or `--max-request-bytes`
/// (`ErrGRPCRequestTooLarge`). Matched on the message: the gRPC code
/// is a generic InvalidArgument.
fn is_txn_too_large(e: &Error) -> bool {
matches!(
e,
Error::Store(assignment_coordination::error::Error::Etcd(
etcd_client::Error::GRpcStatus(status)
)) if status.message().contains("too many operations")
|| status.message().contains("request is too large")
)
}
/// Greedy chunk boundaries over per-unit `(guards, ops)` costs: a chunk
/// takes units while its compare list and its op list both stay within
/// `budget` — etcd checks the two lists separately against
/// `--max-txn-ops`. `reserve_guards` and `reserve_ops` hold back room in
/// every chunk for what is added on top of its units: the plan basis
/// compares and the freeze-quorum write.
fn plan_chunk_ranges(
costs: &[(usize, usize)],
budget: usize,
reserve_guards: usize,
reserve_ops: usize,
) -> Vec<std::ops::Range<usize>> {
let budget = budget.max(1);
let mut ranges = Vec::new();
let mut start = 0;
let (mut guards, mut ops) = (0, 0);
for (i, &(g, o)) in costs.iter().enumerate() {
let fits = guards + g + reserve_guards <= budget && ops + o + reserve_ops <= budget;
// A chunk always takes at least one unit, so a budget below a
// single unit's cost still makes progress and lets the server
// be the judge.
if i > start && !fits {
ranges.push(start..i);
start = i;
guards = 0;
ops = 0;
}
guards += g;
ops += o;
}
if start < costs.len() {
ranges.push(start..costs.len());
}
ranges
}
impl PersonhogStore {
pub fn new(inner: EtcdStore) -> Self {
Self {
inner,
freeze_quorums: Arc::new(StdMutex::new(HashMap::new())),
}
}
pub fn inner(&self) -> &EtcdStore {
&self.inner
}
fn key(&self, k: StoreKey<'_>) -> String {
k.resolve(self.inner.prefix())
}
// ── Pod operations ──────────────────────────────────────────
pub async fn register_pod(&self, pod: &RegisteredPod, lease_id: i64) -> Result<()> {
count_call("register_pod");
let key = self.key(StoreKey::Pod(&pod.pod_name));
Ok(self.inner.put(&key, pod, Some(lease_id)).await?)
}
/// The exact key `register_pod` writes for a pod, for watchers that
/// must match their own registration and nothing else under the
/// prefix.
pub fn pod_registration_key(&self, pod_name: &str) -> String {
self.key(StoreKey::Pod(pod_name))
}
pub async fn get_pod(&self, pod_name: &str) -> Result<Option<RegisteredPod>> {
count_call("get_pod");
let key = self.key(StoreKey::Pod(pod_name));
Ok(self.inner.get(&key).await?)
}
/// Bypasses the protocol: see the crate's `test-support` feature.
#[cfg(any(test, feature = "test-support"))]
pub async fn delete_pod(&self, pod_name: &str) -> Result<()> {
let key = self.key(StoreKey::Pod(pod_name));
Ok(self.inner.delete(&key).await?)
}
pub async fn list_pods(&self) -> Result<Vec<RegisteredPod>> {
count_call("list_pods");
let key = self.key(StoreKey::PodsPrefix);
Ok(self.inner.list(&key).await?)
}
/// List pods along with the etcd revision of the snapshot, so a watch
/// can be anchored strictly after it.
pub async fn list_pods_with_revision(&self) -> Result<(Vec<RegisteredPod>, i64)> {
count_call("list_pods_with_revision");
let key = self.key(StoreKey::PodsPrefix);
Ok(self.inner.list_with_revision(&key).await?)
}
pub async fn update_pod_status(
&self,
pod_name: &str,
status: PodStatus,
lease_id: i64,
) -> Result<()> {
count_call("update_pod_status");
let key = self.key(StoreKey::Pod(pod_name));
let mut pod: RegisteredPod = self
.inner
.get(&key)
.await?
.ok_or_else(|| Error::NotFound(format!("pod {pod_name}")))?;
pod.status = status;
Ok(self.inner.put(&key, &pod, Some(lease_id)).await?)
}
pub async fn watch_pods(&self) -> Result<WatchStream> {
let key = self.key(StoreKey::PodsPrefix);
Ok(self.inner.watch(&key).await?)
}
/// Watch pod registrations from an explicit revision (inclusive),
/// replaying events since that revision even if they predate the
/// watch's creation.
pub async fn watch_pods_from(&self, start_revision: i64) -> Result<WatchStream> {
count_call("watch_pods_from");
let key = self.key(StoreKey::PodsPrefix);
Ok(self.inner.watch_from(&key, start_revision).await?)
}
/// The current etcd store revision, for anchoring watches.
pub async fn current_revision(&self) -> Result<i64> {
count_call("current_revision");
Ok(self.inner.current_revision().await?)
}
// ── Router operations ────────────────────────────────────────
pub async fn register_router(&self, router: &RegisteredRouter, lease_id: i64) -> Result<()> {
count_call("register_router");
let key = self.key(StoreKey::Router(&router.router_name));
Ok(self.inner.put(&key, router, Some(lease_id)).await?)
}
pub async fn get_router_with_revision(
&self,
router_name: &str,
) -> Result<(Option<RegisteredRouter>, i64)> {
count_call("get_router_with_revision");
let key = self.key(StoreKey::Router(router_name));
Ok(self.inner.get_with_revision(&key).await?)
}
pub async fn watch_router_from(
&self,
router_name: &str,
start_revision: i64,
) -> Result<WatchStream> {
count_call("watch_router_from");
let key = self.key(StoreKey::Router(router_name));
Ok(self.inner.watch_key_from(&key, start_revision).await?)
}
pub async fn list_routers(&self) -> Result<Vec<RegisteredRouter>> {
count_call("list_routers");
let key = self.key(StoreKey::RoutersPrefix);
Ok(self.inner.list(&key).await?)
}
pub async fn watch_routers(&self) -> Result<WatchStream> {
let key = self.key(StoreKey::RoutersPrefix);
Ok(self.inner.watch(&key).await?)
}
pub async fn watch_routers_from(&self, start_revision: i64) -> Result<WatchStream> {
count_call("watch_routers_from");
let key = self.key(StoreKey::RoutersPrefix);
Ok(self.inner.watch_from(&key, start_revision).await?)
}
// ── Assignment operations ───────────────────────────────────
pub async fn get_assignment(&self, partition: u32) -> Result<Option<PartitionAssignment>> {
count_call("get_assignment");
let key = self.key(StoreKey::Assignment(partition));
Ok(self.inner.get(&key).await?)
}
pub async fn list_assignments(&self) -> Result<Vec<PartitionAssignment>> {
count_call("list_assignments");
let key = self.key(StoreKey::AssignmentsPrefix);
Ok(self.inner.list(&key).await?)
}
/// Like `list_assignments`, but also returns the etcd revision of the
/// snapshot, for gap-free snapshot-then-watch handshakes.
pub async fn list_assignments_with_revision(&self) -> Result<(Vec<PartitionAssignment>, i64)> {
count_call("list_assignments_with_revision");
let key = self.key(StoreKey::AssignmentsPrefix);
Ok(self.inner.list_with_revision(&key).await?)
}
/// Like `list_assignments`, but pairs each record with its key's
/// `mod_revision` so a plan can assert, at apply time, that the
/// assignments it read are unchanged (`AssignmentPrecondition`).
pub async fn list_assignments_with_mod_revisions(
&self,
) -> Result<Vec<(PartitionAssignment, i64)>> {
count_call("list_assignments_with_mod_revisions");
let key = self.key(StoreKey::AssignmentsPrefix);
Ok(self.inner.list_with_mod_revisions(&key).await?)
}
/// Bypasses the protocol: see the crate's `test-support` feature.
#[cfg(any(test, feature = "test-support"))]
pub async fn put_assignments(&self, assignments: &[PartitionAssignment]) -> Result<()> {
if assignments.is_empty() {
return Ok(());
}
let ops: Vec<TxnOp> = assignments
.iter()
.map(|a| {
let key = self.key(StoreKey::Assignment(a.partition));
let value = serde_json::to_vec(a)?;
Ok(TxnOp::put(key, value, None))
})
.collect::<Result<Vec<_>>>()?;
let txn = Txn::new().and_then(ops);
self.inner.txn(txn).await?;
Ok(())
}
// ── Handoff operations ──────────────────────────────────────
pub async fn get_handoff(&self, partition: u32) -> Result<Option<HandoffState>> {
count_call("get_handoff");
let key = self.key(StoreKey::Handoff(partition));
Ok(self.inner.get(&key).await?)
}
pub async fn list_handoffs(&self) -> Result<Vec<HandoffState>> {
count_call("list_handoffs");
let key = self.key(StoreKey::HandoffsPrefix);
Ok(self.inner.list(&key).await?)
}
/// Like `list_handoffs`, but also returns the etcd revision of the
/// snapshot. Pair with `watch_handoffs_from(revision + 1)` for a
/// gap-free snapshot-then-watch handshake.
pub async fn list_handoffs_with_revision(&self) -> Result<(Vec<HandoffState>, i64)> {
count_call("list_handoffs_with_revision");
let key = self.key(StoreKey::HandoffsPrefix);
Ok(self.inner.list_with_revision(&key).await?)
}
/// Like `list_handoffs`, but pairs each record with its key's
/// `mod_revision`, so a later replacement can be guarded on the
/// record being exactly the one this snapshot read.
pub async fn list_handoffs_with_mod_revisions(&self) -> Result<Vec<(HandoffState, i64)>> {
count_call("list_handoffs_with_mod_revisions");
let key = self.key(StoreKey::HandoffsPrefix);
Ok(self.inner.list_with_mod_revisions(&key).await?)
}
/// Bypasses the protocol: see the crate's `test-support` feature.
#[cfg(any(test, feature = "test-support"))]
pub async fn put_handoff(&self, handoff: &HandoffState) -> Result<()> {
count_call("put_handoff");
let key = self.key(StoreKey::Handoff(handoff.partition));
Ok(self.inner.put(&key, handoff, None).await?)
}
/// Compare-and-swap a handoff's phase. Reads the current handoff,
/// verifies its phase matches `expected`, and writes a copy with
/// `new_phase` only if the etcd key's mod_revision hasn't changed —
/// mod_revision rather than version, so the guard can never match a
/// different incarnation of the key after a delete-and-recreate.
///
/// Returns `Ok(true)` if the swap succeeded, `Ok(false)` if the handoff
/// was concurrently modified or its phase no longer matches `expected`.
/// Used by `check_phase_advance` to avoid duplicate phase writes when
/// multiple watch loops fire `check_phase_advance` concurrently for the
/// same partition.
/// `expected_id` names the handoff attempt the caller validated. The
/// key can be replaced between that validation and this write —
/// cancellation swaps in a successor and deletes the old acks in one
/// transaction — and a `mod_revision` guard taken on a re-read here
/// would not notice, because it only proves nothing changed since
/// *this* function looked.
pub async fn cas_handoff_phase(
&self,
partition: u32,
expected_id: &str,
expected: crate::types::HandoffPhase,
new_phase: crate::types::HandoffPhase,
) -> Result<bool> {
count_call("cas_handoff_phase");
let handoff_key = self.key(StoreKey::Handoff(partition));
let Some((mut handoff, mod_revision)) = self
.inner
.get_with_mod_revision::<HandoffState>(&handoff_key)
.await?
else {
return Ok(false);
};
if handoff.phase != expected || handoff.handoff_id != expected_id {
return Ok(false);
}
handoff.phase = new_phase;
// The phase clock restarts with the phase: duration metrics and
// the per-phase age gauge read this stamp.
handoff.phase_entered_at_ms = assignment_coordination::util::now_millis();
let txn = Txn::new()
.when(vec![Compare::mod_revision(
handoff_key.clone(),
CompareOp::Equal,
mod_revision,
)])
.and_then(vec![TxnOp::put(
handoff_key,
serde_json::to_vec(&handoff)?,
None,
)]);
let resp = self.inner.txn(txn).await?;
Ok(resp.succeeded())
}
/// Read a handoff along with its etcd mod_revision, for use with
/// `delete_handoff_and_acks_if_unchanged`.
pub async fn get_handoff_with_mod_revision(
&self,
partition: u32,
) -> Result<Option<(HandoffState, i64)>> {
count_call("get_handoff_with_mod_revision");
let key = self.key(StoreKey::Handoff(partition));
Ok(self.inner.get_with_mod_revision(&key).await?)
}
/// Atomically delete a handoff record and every ack written for its
/// partition — but only if the record's mod_revision still matches
/// `expected_mod_revision`. Cleanup decisions are made against snapshot
/// reads that can race a concurrent delete-and-recreate at the same
/// key (cancellation followed by an immediate rebalance); an unguarded
/// delete pair would destroy the healthy successor handoff and its
/// acks. Returns whether the delete happened.
pub async fn delete_handoff_and_acks_if_unchanged(
&self,
partition: u32,
expected_mod_revision: i64,
) -> Result<bool> {
count_call("delete_handoff_and_acks_if_unchanged");
let handoff_key = self.key(StoreKey::Handoff(partition));
let prefix_delete = || Some(DeleteOptions::new().with_prefix());
let txn = Txn::new()
.when(vec![Compare::mod_revision(
handoff_key.clone(),
CompareOp::Equal,
expected_mod_revision,
)])
.and_then(vec![
TxnOp::delete(
self.key(StoreKey::FreezeAcksForPartition(partition)),
prefix_delete(),
),
TxnOp::delete(
self.key(StoreKey::DrainedAcksForPartition(partition)),
prefix_delete(),
),
TxnOp::delete(
self.key(StoreKey::WarmedAcksForPartition(partition)),
prefix_delete(),
),
TxnOp::delete(handoff_key, None),
]);
let resp = self.inner.txn(txn).await?;
Ok(resp.succeeded())
}
/// Bypasses the protocol: see the crate's `test-support` feature.
#[cfg(any(test, feature = "test-support"))]
pub async fn delete_handoff(&self, partition: u32) -> Result<()> {
let key = self.key(StoreKey::Handoff(partition));
Ok(self.inner.delete(&key).await?)
}
pub async fn watch_handoffs(&self) -> Result<WatchStream> {
let key = self.key(StoreKey::HandoffsPrefix);
Ok(self.inner.watch(&key).await?)
}
/// Watch handoffs from an explicit revision (inclusive), replaying
/// events since that revision even if they predate the watch's
/// creation.
pub async fn watch_handoffs_from(&self, start_revision: i64) -> Result<WatchStream> {
count_call("watch_handoffs_from");
let key = self.key(StoreKey::HandoffsPrefix);
Ok(self.inner.watch_from(&key, start_revision).await?)
}
// ── Freeze ack operations (router -> coordinator) ────────────
pub async fn put_freeze_ack(&self, ack: &RouterFreezeAck) -> Result<()> {
count_call("put_freeze_ack");
let key = self.key(StoreKey::FreezeAck {
partition: ack.partition,
router: &ack.router_name,
});
// The store stamps the millisecond clock so span metrics never
// depend on each writer remembering to.
let mut stamped = ack.clone();
stamped.acked_at_ms = assignment_coordination::util::now_millis();
Ok(self.inner.put(&key, &stamped, None).await?)
}
/// One transaction per chunk of acks in place of one round trip per
/// ack: a plan freezing hundreds of partitions has every router
/// acking each of them, and serial puts hold the freeze quorum on
/// the slowest router's whole sequence. `max_txn_ops` must not
/// exceed the server's own limit, which refuses a larger batch
/// outright. An empty batch costs nothing.
pub async fn put_freeze_acks(
&self,
acks: &[RouterFreezeAck],
max_txn_ops: usize,
) -> Result<()> {
for chunk in acks.chunks(max_txn_ops.max(1)) {
count_call("put_freeze_acks");
let mut ops = Vec::with_capacity(chunk.len());
for ack in chunk {
let key = self.key(StoreKey::FreezeAck {
partition: ack.partition,
router: &ack.router_name,
});
// The store stamps the millisecond clock so span metrics
// never depend on each writer remembering to.
let mut stamped = ack.clone();
stamped.acked_at_ms = assignment_coordination::util::now_millis();
ops.push(TxnOp::put(key, serde_json::to_vec(&stamped)?, None));
}
metrics::counter!("personhog_coordination_freeze_acks_written_total")
.increment(chunk.len() as u64);
self.inner.txn(Txn::new().and_then(ops)).await?;
}
Ok(())
}
pub async fn list_freeze_acks(&self, partition: u32) -> Result<Vec<RouterFreezeAck>> {
count_call("list_freeze_acks");
let key = self.key(StoreKey::FreezeAcksForPartition(partition));
Ok(self.inner.list(&key).await?)
}
pub async fn delete_freeze_acks(&self, partition: u32) -> Result<()> {
let key = self.key(StoreKey::FreezeAcksForPartition(partition));
Ok(self.inner.delete_prefix(&key).await?)
}
pub async fn watch_freeze_acks(&self) -> Result<WatchStream> {
let key = self.key(StoreKey::FreezeAcksPrefix);
Ok(self.inner.watch(&key).await?)
}
pub async fn watch_freeze_acks_from(&self, start_revision: i64) -> Result<WatchStream> {
count_call("watch_freeze_acks_from");
let key = self.key(StoreKey::FreezeAcksPrefix);
Ok(self.inner.watch_from(&key, start_revision).await?)
}
// ── Drained ack operations (old owner -> coordinator) ────────
pub async fn put_drained_ack(&self, ack: &PodDrainedAck) -> Result<()> {
count_call("put_drained_ack");
let key = self.key(StoreKey::DrainedAck {
partition: ack.partition,
pod: &ack.pod_name,
});
// The store stamps the millisecond clock so span metrics never
// depend on each writer remembering to.
let mut stamped = ack.clone();
stamped.acked_at_ms = assignment_coordination::util::now_millis();
Ok(self.inner.put(&key, &stamped, None).await?)
}
pub async fn list_drained_acks(&self, partition: u32) -> Result<Vec<PodDrainedAck>> {
count_call("list_drained_acks");
let key = self.key(StoreKey::DrainedAcksForPartition(partition));
Ok(self.inner.list(&key).await?)
}
pub async fn delete_drained_acks(&self, partition: u32) -> Result<()> {
let key = self.key(StoreKey::DrainedAcksForPartition(partition));
Ok(self.inner.delete_prefix(&key).await?)
}
pub async fn watch_drained_acks(&self) -> Result<WatchStream> {
let key = self.key(StoreKey::DrainedAcksPrefix);
Ok(self.inner.watch(&key).await?)
}
pub async fn watch_drained_acks_from(&self, start_revision: i64) -> Result<WatchStream> {
count_call("watch_drained_acks_from");
let key = self.key(StoreKey::DrainedAcksPrefix);
Ok(self.inner.watch_from(&key, start_revision).await?)
}
// ── Warmed ack operations (new owner -> coordinator) ─────────
pub async fn put_warmed_ack(&self, ack: &PodWarmedAck) -> Result<()> {
count_call("put_warmed_ack");
let key = self.key(StoreKey::WarmedAck {
partition: ack.partition,
pod: &ack.pod_name,
});
// The store stamps the millisecond clock so span metrics never
// depend on each writer remembering to.
let mut stamped = ack.clone();
stamped.acked_at_ms = assignment_coordination::util::now_millis();
Ok(self.inner.put(&key, &stamped, None).await?)
}
pub async fn list_warmed_acks(&self, partition: u32) -> Result<Vec<PodWarmedAck>> {
count_call("list_warmed_acks");
let key = self.key(StoreKey::WarmedAcksForPartition(partition));
Ok(self.inner.list(&key).await?)
}
pub async fn delete_warmed_acks(&self, partition: u32) -> Result<()> {
let key = self.key(StoreKey::WarmedAcksForPartition(partition));
Ok(self.inner.delete_prefix(&key).await?)
}
pub async fn watch_warmed_acks(&self) -> Result<WatchStream> {
let key = self.key(StoreKey::WarmedAcksPrefix);
Ok(self.inner.watch(&key).await?)
}
pub async fn watch_warmed_acks_from(&self, start_revision: i64) -> Result<WatchStream> {
count_call("watch_warmed_acks_from");
let key = self.key(StoreKey::WarmedAcksPrefix);
Ok(self.inner.watch_from(&key, start_revision).await?)
}
// ── Transactional operations ────────────────────────────────
/// Atomically write assignments and create handoff states.
/// Returns whether every disposition applied. Each partition's unit
/// is guarded so it only lands if the world it was computed from is
/// still the world (an assignment-only unit carries no guard):
///
/// * every handoff key must be absent — concurrent planners (the pod
/// watch racing the handoff watch's re-trigger, or a failing-over
/// coordinator) can both plan the same partition, and an unguarded
/// put would replace the first handoff and orphan its acks;
/// * every `AssignmentPrecondition` must hold — a handoff's
/// `old_owner` is only meaningful if the assignment it was read
/// from is unchanged. Without this, a plan whose snapshot predates
/// a full create→complete→cleanup cycle of the same partition
/// passes the absence guard and drains the wrong pod, leaving the
/// real owner unfenced beside the new owner's warm cutoff.
///
/// `replacements` carry cancellations-by-replacement: each swaps the
/// record at its partition's key — guarded on the `mod_revision` the
/// planner read — for the successor (or reaffirm) record, deleting
/// the predecessor's acks in the same transaction. A non-terminal
/// handoff record is never deleted; it is only ever replaced by the
/// thing that resolves its stashes.
#[cfg(any(test, feature = "test-support"))]
pub async fn create_assignments_and_handoffs(
&self,
assignments: &[PartitionAssignment],
handoffs: &[HandoffState],
preconditions: &[AssignmentPrecondition],
max_txn_ops: usize,
) -> Result<bool> {
// The handoffs come ready-made, so there is no shared
// membership to write alongside them.
let application = self
.apply_plan(assignments, handoffs, &[], preconditions, None, max_txn_ops)
.await?;
Ok(application.conflicted.is_empty())
}
/// Apply a plan — creations (absent-guarded) plus
/// cancellations-by-replacement (mod_revision-guarded) — in
/// transactions sized to `max_txn_ops`, the server's txn budget.
///
/// Atomicity is per partition, not per plan: every guard protects
/// one partition's disposition, so a partition's guards and writes
/// share one transaction, but the plan itself may split. An
/// unchunked plan past `--max-txn-ops` is rejected as a hard error,
/// and a coordinator retrying it re-earns the same rejection
/// forever. A partially applied plan is placement drift the next
/// reconcile-tick replan heals. Production plans carry the record they
/// were decided on (`apply_plan_on`, `apply_migration_plan`).
#[cfg(any(test, feature = "test-support"))]
pub async fn apply_plan(
&self,
assignments: &[PartitionAssignment],
handoffs: &[HandoffState],
replacements: &[HandoffReplacement],
preconditions: &[AssignmentPrecondition],
// The membership the plan's handoffs refer to, re-put
// (idempotently) in every chunk that references it so no
// handoff is ever durable with a dangling reference.
freeze_quorum: Option<(&str, &[String])>,
max_txn_ops: usize,
) -> Result<PlanApplication> {
self.apply_plan_inner(
assignments,
handoffs,
replacements,
preconditions,
freeze_quorum,
max_txn_ops,
None,
)
.await
}
/// The planner's plan, landing only while `basis` still holds.
pub async fn apply_plan_on(
&self,
basis: &PlanBasis,
handoffs: &[HandoffState],
replacements: &[HandoffReplacement],
preconditions: &[AssignmentPrecondition],
freeze_quorum: Option<(&str, &[String])>,
max_txn_ops: usize,
) -> Result<PlanApplication> {
let authority = Authority::Planner {
config_revision: basis.config_revision,
migration_revision: basis.migration_revision,
};
self.apply_plan_inner(
&[],
handoffs,
replacements,
preconditions,
freeze_quorum,
max_txn_ops,
Some(authority),
)
.await
}
/// A migration pass's plan; it lands only while the saga record is unchanged.
pub async fn apply_migration_plan(
&self,
migration_mod_revision: i64,
handoffs: &[HandoffState],
replacements: &[HandoffReplacement],
preconditions: &[AssignmentPrecondition],
freeze_quorum: Option<(&str, &[String])>,
max_txn_ops: usize,
) -> Result<PlanApplication> {
self.apply_plan_inner(
&[],
handoffs,
replacements,
preconditions,
freeze_quorum,
max_txn_ops,
Some(Authority::Migration {
mod_revision: migration_mod_revision,
}),
)
.await
}
/// The changelog config and the migration record, read in one transaction.
pub async fn read_plan_basis(&self) -> Result<PlanBasis> {
count_call("read_plan_basis");
let txn = Txn::new().and_then(vec![
TxnOp::get(self.key(StoreKey::ChangelogConfig), None),
TxnOp::get(self.key(StoreKey::Migration), None),
]);
let resp = self.inner.txn(txn).await?;
let mut kvs = Vec::new();
for op in resp.op_responses() {
match op {
TxnOpResponse::Get(get) => kvs.push(get.kvs().first().cloned()),
_ => {
return Err(crate::error::Error::invalid_state(
"plan basis read returned a non-get response",
))
}
}
}
let [config, migration]: [Option<etcd_client::KeyValue>; 2] = kvs
.try_into()
.map_err(|_| crate::error::Error::invalid_state("plan basis read lost a get"))?;
Ok(PlanBasis {
config_revision: config.as_ref().map_or(0, |kv| kv.mod_revision()),
migration_revision: migration.as_ref().map_or(0, |kv| kv.create_revision()),
config: config
.map(|kv| serde_json::from_slice(kv.value()))
.transpose()?,
migration: migration
.map(|kv| serde_json::from_slice(kv.value()))
.transpose()?,
})
}
fn authority_guards(&self, authority: Authority) -> Vec<Compare> {
let migration = self.key(StoreKey::Migration);
match authority {
Authority::Planner {
config_revision,
migration_revision,
} => vec![
Compare::mod_revision(
self.key(StoreKey::ChangelogConfig),
CompareOp::Equal,
config_revision,
),
Compare::create_revision(migration, CompareOp::Equal, migration_revision),
],
Authority::Migration { mod_revision } => {
vec![Compare::mod_revision(
migration,
CompareOp::Equal,
mod_revision,
)]
}
}
}
/// Whether a plan's authority has moved; a moved authority fails every unit alike.
async fn authority_moved(&self, authority: Option<Authority>) -> Result<bool> {
Ok(match authority {
None => false,
Some(Authority::Planner {
config_revision,
migration_revision,
}) => {
let now = self.read_plan_basis().await?;
(now.config_revision, now.migration_revision)
!= (config_revision, migration_revision)
}
Some(Authority::Migration { mod_revision }) => self
.get_migration_with_mod_revision()
.await?
.is_none_or(|(_, now)| now != mod_revision),
})
}
#[allow(clippy::too_many_arguments)]
async fn apply_plan_inner(
&self,
assignments: &[PartitionAssignment],
handoffs: &[HandoffState],
replacements: &[HandoffReplacement],
preconditions: &[AssignmentPrecondition],
freeze_quorum: Option<(&str, &[String])>,
max_txn_ops: usize,
authority: Option<Authority>,
) -> Result<PlanApplication> {
count_call("apply_plan");
let mut units: HashMap<u32, PlanUnit> = HashMap::new();
fn unit(units: &mut HashMap<u32, PlanUnit>, partition: u32) -> &mut PlanUnit {
units.entry(partition).or_insert_with(|| PlanUnit {
partition,
guards: Vec::new(),
ops: Vec::new(),
bytes: 0,
references_quorum: false,
})
}
for a in assignments {
let key = self.key(StoreKey::Assignment(a.partition));
let value = serde_json::to_vec(a)?;
let u = unit(&mut units, a.partition);
u.bytes += key.len() + value.len();
u.ops.push(UnitOp::Put(key, value));
}
for h in handoffs {
let key = self.key(StoreKey::Handoff(h.partition));
let value = serde_json::to_vec(h)?;
let u = unit(&mut units, h.partition);
u.bytes += key.len() + value.len();
// A key that was never created has create_revision 0 — the
// canonical etcd existence guard.
u.bytes += key.len();
u.guards.push(UnitGuard::Absent(key.clone()));
u.ops.push(UnitOp::Put(key, value));
u.references_quorum |= h.freeze_quorum_ref.is_some();
}
for r in replacements {
let partition = r.handoff.partition;
let key = self.key(StoreKey::Handoff(partition));
let value = serde_json::to_vec(&r.handoff)?;
let u = unit(&mut units, partition);
u.bytes += key.len() + value.len();
u.bytes += key.len();
u.guards
.push(UnitGuard::ModRevision(key.clone(), r.expected_mod_revision));
for acks in [
self.key(StoreKey::FreezeAcksForPartition(partition)),
self.key(StoreKey::DrainedAcksForPartition(partition)),
self.key(StoreKey::WarmedAcksForPartition(partition)),
] {
// A prefix delete carries `range_end` as well as the
// key, and they are the same length.
u.bytes += acks.len() * 2;
u.ops.push(UnitOp::DeletePrefix(acks));
}
u.ops.push(UnitOp::Put(key, value));
u.references_quorum |= r.handoff.freeze_quorum_ref.is_some();
}
for precondition in preconditions {
let (partition, guard) = match precondition {
AssignmentPrecondition::UnchangedSince {
partition,
mod_revision,
} => (
*partition,
UnitGuard::ModRevision(
self.key(StoreKey::Assignment(*partition)),
*mod_revision,
),
),
AssignmentPrecondition::Absent { partition } => (
*partition,
UnitGuard::Absent(self.key(StoreKey::Assignment(*partition))),
),
};
let u = unit(&mut units, partition);
u.bytes += guard.key().len();
u.guards.push(guard);
}
// Partition order, so chunk composition is a function of the
// plan alone rather than of map iteration.
let mut units: Vec<PlanUnit> = units.into_values().collect();
units.sort_by_key(|u| u.partition);
let quorum = match freeze_quorum {
Some((id, members)) => Some((
self.key(StoreKey::FreezeQuorum(id)),
serde_json::to_vec(members)?,
)),
None => None,
};
// Totals across chunks, keys and deletes included, so the
// series keeps its meaning from the one-transaction era.
let plan_bytes: usize = units.iter().map(|u| u.bytes).sum::<usize>()
+ quorum.as_ref().map_or(0, |(k, v)| k.len() + v.len());
let total_guards: usize = units.iter().map(|u| u.guards.len()).sum();
let total_ops: usize = units.iter().map(|u| u.ops.len()).sum();
metrics::histogram!("personhog_coordination_plan_bytes").record(plan_bytes as f64);
metrics::histogram!("personhog_coordination_plan_ops", "list" => "guards")
.record(total_guards as f64);
metrics::histogram!("personhog_coordination_plan_ops", "list" => "ops")
.record(total_ops as f64);
let costs: Vec<(usize, usize)> = units
.iter()
.map(|u| (u.guards.len(), u.ops.len()))
.collect();
let reserve_ops = usize::from(quorum.is_some());
let authority_guards = authority
.map(|a| self.authority_guards(a))
.unwrap_or_default();
let reserve_guards = authority_guards.len();
let ranges = plan_chunk_ranges(&costs, max_txn_ops, reserve_guards, reserve_ops);
metrics::histogram!("personhog_coordination_plan_chunks").record(ranges.len() as f64);
let mut application = PlanApplication::default();
for range in ranges {
let chunk = &units[range.clone()];
match self.apply_units(chunk, &quorum, &authority_guards).await {
Ok(true) => {
application
.applied
.extend(chunk.iter().map(|u| u.partition));
continue;
}
Ok(false) if self.authority_moved(authority).await? => {
application.stand_down(&units[range.start..], authority);
return Ok(application);
}
Ok(false) => {}
// The server's effective --max-txn-ops can sit below the
// configured budget (values drift, a member not yet
// restarted with a raised flag). Degrade to the per-unit
// path — under any workable server limit — instead of
// handing back a rejection the planner retries forever.
Err(e) if chunk.len() > 1 && is_txn_too_large(&e) => {
application.over_budget_chunks += 1;
metrics::counter!("personhog_coordination_plan_chunk_over_server_budget_total")
.increment(1);
tracing::warn!(
units = chunk.len(),
max_txn_ops,
error = %e,
"plan chunk exceeds the server txn budget; applying per unit"
);
}
Err(e) => return Err(e),
}
// A failed guard names no key, so retry the chunk one
// partition at a time: the concurrent write that broke the
// chunk stands down only the units it actually touched.
// One authority check per fallback: past a move every unit fails alike.
let mut checked = false;
for (i, u) in chunk.iter().enumerate() {
if self
.apply_units(std::slice::from_ref(u), &quorum, &authority_guards)
.await?
{
application.applied.push(u.partition);
continue;
}
if !checked {
checked = true;
if self.authority_moved(authority).await? {
application.stand_down(&units[range.start + i..], authority);
return Ok(application);
}
}
metrics::counter!("personhog_coordination_plan_units_conflicted_total")
.increment(1);
application.conflicted.push(u.partition);
}
}
Ok(application)
}
/// One plan chunk as one transaction: the units' guards and ops,
/// plus the freeze-quorum write when any unit references it.
/// Returns whether the transaction applied.
async fn apply_units(
&self,
chunk: &[PlanUnit],
quorum: &Option<(String, Vec<u8>)>,
authority_guards: &[Compare],
) -> Result<bool> {
let mut guards: Vec<Compare> = authority_guards.to_vec();
let mut ops: Vec<TxnOp> = Vec::new();
if let Some((key, value)) = quorum {
if chunk.iter().any(|u| u.references_quorum) {
ops.push(TxnOp::put(key.clone(), value.clone(), None));
}
}
for u in chunk {
for g in &u.guards {
guards.push(match g {
UnitGuard::Absent(key) => {
Compare::create_revision(key.clone(), CompareOp::Equal, 0)
}
UnitGuard::ModRevision(key, rev) => {
Compare::mod_revision(key.clone(), CompareOp::Equal, *rev)
}
});
}
for op in &u.ops {
ops.push(match op {
UnitOp::Put(key, value) => TxnOp::put(key.clone(), value.clone(), None),
UnitOp::DeletePrefix(key) => {
TxnOp::delete(key.clone(), Some(DeleteOptions::new().with_prefix()))
}
});
}
}
let txn = Txn::new().when(guards).and_then(ops);
let resp = self.inner.txn(txn).await?;
Ok(resp.succeeded())
}
/// Atomically: set handoff phase to Complete and update the assignment owner.
///
/// Uses compare-and-swap on the handoff key's version to prevent stale
/// writes (e.g. if another actor already completed or deleted the handoff
/// between our read and write).
///
/// Returns `Ok(false)` if the handoff was modified concurrently (CAS failed),
/// or if the record at the key is no longer the attempt the caller
/// validated.
///
/// The `expected_*` arguments are what make the second case
/// detectable, and they are not optional rigour. Completion is the
/// step that writes the assignment, so completing the wrong record
/// hands the partition to a pod that never froze, drained, or warmed
/// — while the old owner is still admitting writes — and routers cut
/// over to it. A `mod_revision` guard alone cannot catch that: it
/// proves nothing changed since this function's own re-read, not that
/// the record is the one whose warm was verified.
///
/// **Invariant:** this is the only code path that ever *changes* an
/// assignment's `owner`. Routers rely on observing handoff Complete
/// events to update their in-memory routing tables — they do not watch
/// assignments. Anything that mutates `assignments/{partition}` outside
/// of this method will be invisible to routers. If we ever need a
/// force-reassignment ops tool, it should create a handoff record and
/// let the protocol advance it, not write to the assignment key.
pub async fn complete_handoff(
&self,
partition: u32,
expected_id: &str,
expected_phase: crate::types::HandoffPhase,
) -> Result<bool> {
count_call("complete_handoff");
let handoff_key = self.key(StoreKey::Handoff(partition));
let (mut handoff, mod_revision) = self
.inner
.get_with_mod_revision::<HandoffState>(&handoff_key)
.await?
.ok_or_else(|| Error::NotFound(format!("handoff for partition {partition}")))?;
if handoff.handoff_id != expected_id || handoff.phase != expected_phase {
return Ok(false);
}
handoff.phase = crate::types::HandoffPhase::Complete;
handoff.phase_entered_at_ms = assignment_coordination::util::now_millis();
let assignment = PartitionAssignment {
partition,
owner: handoff.new_owner.clone(),
advertise_address: handoff.new_owner_address.clone(),
status: AssignmentStatus::Active,
};
let assignment_key = self.key(StoreKey::Assignment(partition));
let txn = Txn::new()
.when(vec![Compare::mod_revision(
handoff_key.clone(),
CompareOp::Equal,
mod_revision,
)])
.and_then(vec![
TxnOp::put(handoff_key, serde_json::to_vec(&handoff)?, None),
TxnOp::put(assignment_key, serde_json::to_vec(&assignment)?, None),
]);
let resp = self.inner.txn(txn).await?;
Ok(resp.succeeded())
}
// ── Freeze quorum membership ────────────────────────────────
/// The membership record `id` names, or `None` if no record exists.
///
/// Callers treat a missing record as "no membership recorded" and
/// fall back to requiring every live router, which is the stricter
/// rule — so a record lost to garbage collection or an incomplete
/// write delays a handoff rather than advancing it early.
pub async fn get_freeze_quorum(&self, id: &str) -> Result<Option<Vec<String>>> {
count_call("get_freeze_quorum");
let key = self.key(StoreKey::FreezeQuorum(id));
Ok(self.inner.get(&key).await?)
}
/// The membership a handoff requires, from wherever it is recorded.
///
/// Records written before the membership moved into its own key
/// carry it inline. `None` — no reference and nothing inline, or a
/// reference whose record has gone — means the caller falls back to
/// requiring every live router.
pub async fn resolve_freeze_quorum(
&self,
handoff: &HandoffState,
) -> Result<Option<Vec<String>>> {
match &handoff.freeze_quorum_ref {
Some(id) => {
// A hit answers from memory: confirming against etcd
// would cost the read this cache removes.
let cached = self
.freeze_quorums
.lock()
.expect("freeze quorum cache lock poisoned")
.get(id)
.cloned();
if let Some(members) = cached {
return Ok(Some(members));
}
let members = self.get_freeze_quorum(id).await?;
match &members {
Some(recorded) => {
let mut cache = self
.freeze_quorums
.lock()
.expect("freeze quorum cache lock poisoned");
// Sized for a rolling deploy (plans in flight,
// not one). Evicting the oldest — ids lead with
// milliseconds, so smallest is oldest — keeps
// the working set.
if cache.len() >= 32 {
if let Some(oldest) = cache.keys().min().cloned() {
cache.remove(&oldest);
}
}
cache.insert(id.clone(), recorded.clone());
}
// Never cached: a chunked plan's re-puts can
// re-create a swept record, so absence is not
// permanent — and pinning it would hold every later
// resolution on the every-live-router fallback.
None => {
crate::util::record_unresolved_freeze_quorum();
tracing::warn!(
partition = handoff.partition,
quorum_id = %id,
"freeze quorum record is missing; requiring every live router"
);
}
}
Ok(members)
}
None => Ok(handoff.freeze_quorum.clone()),
}
}
/// The ids of every membership record currently stored, each with
/// its mod_revision for the sweep's guarded delete.
pub async fn list_freeze_quorum_ids(&self) -> Result<Vec<(String, i64)>> {
count_call("list_freeze_quorum_ids");
let prefix = self.key(StoreKey::FreezeQuorumsPrefix);
let keys = self.inner.list_keys_with_mod_revisions(&prefix).await?;
Ok(keys
.iter()
.filter_map(|(key, rev)| {
key.strip_prefix(prefix.as_str())
.map(|id| (id.to_string(), *rev))
})
.collect())
}
/// Delete a membership record only if unchanged since `mod_revision`.
/// A chunked plan re-puts its record with every chunk, so a re-put
/// after the sweep's read bumps the revision and the delete loses —
/// otherwise the sweep could collect a record a chunk it never saw
/// still references.
pub async fn delete_freeze_quorum_if_unchanged(
&self,
id: &str,
mod_revision: i64,
) -> Result<bool> {
count_call("delete_freeze_quorum");
let key = self.key(StoreKey::FreezeQuorum(id));
let txn = Txn::new()
.when(vec![Compare::mod_revision(
key.clone(),
CompareOp::Equal,
mod_revision,
)])
.and_then(vec![TxnOp::delete(key, None)]);
let resp = self.inner.txn(txn).await?;
Ok(resp.succeeded())
}
// ── Leader election ─────────────────────────────────────────
/// Try to acquire coordinator leadership using compare-and-swap.
///
/// Returns `true` if this instance became the leader.
pub async fn try_acquire_leadership(&self, holder: &str, lease_id: i64) -> Result<bool> {
count_call("try_acquire_leadership");
let key = self.key(StoreKey::Leader);
let leader = LeaderInfo {
holder: holder.to_string(),
lease_id,
};
let value = serde_json::to_vec(&leader)?;
// CAS: only succeed if the key does not exist (version == 0)
let txn = Txn::new()
.when(vec![Compare::version(key.clone(), CompareOp::Equal, 0)])
.and_then(vec![TxnOp::put(
key.clone(),
value,
Some(PutOptions::new().with_lease(lease_id)),
)])
.or_else(vec![TxnOp::get(key, None)]);
let resp = self.inner.txn(txn).await?;
Ok(resp.succeeded())
}
pub async fn get_leader(&self) -> Result<Option<LeaderInfo>> {
count_call("get_leader");
let key = self.key(StoreKey::Leader);
Ok(self.inner.get(&key).await?)
}
/// The current leader and the revision that answer was read at. A
/// standby anchors its watch on that revision so the leader cannot
/// disappear unobserved in the gap before the watch attaches.
pub async fn get_leader_with_revision(&self) -> Result<(Option<LeaderInfo>, i64)> {
count_call("get_leader_with_revision");
let key = self.key(StoreKey::Leader);
Ok(self.inner.get_with_revision(&key).await?)
}
/// Watch the leader key alone, from `start_revision` inclusive.
pub async fn watch_leader_from(&self, start_revision: i64) -> Result<WatchStream> {
count_call("watch_leader_from");
let key = self.key(StoreKey::Leader);
Ok(self.inner.watch_key_from(&key, start_revision).await?)
}
// ── Lease operations ────────────────────────────────────────
pub async fn grant_lease(&self, ttl: i64) -> Result<i64> {
count_call("grant_lease");
Ok(self.inner.grant_lease(ttl).await?)
}
pub async fn keep_alive(
&self,
lease_id: i64,
) -> Result<(etcd_client::LeaseKeeper, etcd_client::LeaseKeepAliveStream)> {
count_call("keep_alive");
Ok(self.inner.keep_alive(lease_id).await?)
}
pub async fn revoke_lease(&self, lease_id: i64) -> Result<()> {
count_call("revoke_lease");
Ok(self.inner.revoke_lease(lease_id).await?)
}
// ── Config operations ───────────────────────────────────────
pub async fn get_changelog_config(&self) -> Result<Option<ChangelogConfig>> {
count_call("get_changelog_config");
let key = self.key(StoreKey::ChangelogConfig);
Ok(self.inner.get(&key).await?)
}
pub async fn get_changelog_config_with_revision(
&self,
) -> Result<(Option<ChangelogConfig>, i64)> {
count_call("get_changelog_config_with_revision");
let key = self.key(StoreKey::ChangelogConfig);
Ok(self.inner.get_with_revision(&key).await?)
}
pub async fn get_changelog_config_with_mod_revision(
&self,
) -> Result<Option<(ChangelogConfig, i64)>> {
count_call("get_changelog_config_with_mod_revision");
let key = self.key(StoreKey::ChangelogConfig);
Ok(self.inner.get_with_mod_revision(&key).await?)
}
/// Create-if-absent at epoch 1; only a flip may move a live config.
pub async fn seed_changelog_config(
&self,
topic: &str,
partitions: u32,
writer_group: &str,
) -> Result<bool> {
self.seed_changelog_config_on(topic, partitions, writer_group, None)
.await
}
pub async fn seed_changelog_config_on(
&self,
topic: &str,
partitions: u32,
writer_group: &str,
cluster: Option<ClusterEndpoint>,
) -> Result<bool> {
count_call("seed_changelog_config");
let config = ChangelogConfig {
writer_group: writer_group.to_string(),
cluster,
..ChangelogConfig::new(topic, partitions)
};
self.create_changelog_config(serde_json::to_vec(&config)?)
.await
}
/// Create-if-absent with bytes a backup recorded, written back as they
/// were stored so fields this build does not know survive the round trip.
pub async fn restore_changelog_config_raw(&self, value: &[u8]) -> Result<bool> {
count_call("restore_changelog_config_raw");
self.create_changelog_config(value.to_vec()).await
}
async fn create_changelog_config(&self, value: Vec<u8>) -> Result<bool> {
let key = self.key(StoreKey::ChangelogConfig);
let txn = Txn::new()
.when(vec![Compare::create_revision(
key.clone(),
CompareOp::Equal,
0,
)])
.and_then(vec![TxnOp::put(key, value, None)]);
let resp = self.inner.txn(txn).await?;
Ok(resp.succeeded())
}
/// The stored config bytes, unparsed, and the revision of the read.
pub async fn get_changelog_config_raw_with_revision(&self) -> Result<(Option<Vec<u8>>, i64)> {
count_call("get_changelog_config_raw_with_revision");
let key = self.key(StoreKey::ChangelogConfig);
let resp = self
.inner
.client()
.clone()
.get(key, None)
.await
.map_err(assignment_coordination::error::Error::from)?;
let revision = resp.header().map(|h| h.revision()).unwrap_or(0);
Ok((resp.kvs().first().map(|kv| kv.value().to_vec()), revision))
}
/// Bypasses the protocol: see the crate's `test-support` feature.
#[cfg(any(test, feature = "test-support"))]
pub async fn put_changelog_config(&self, config: &ChangelogConfig) -> Result<()> {
let key = self.key(StoreKey::ChangelogConfig);
Ok(self.inner.put(&key, config, None).await?)
}
pub async fn watch_changelog_config_from(&self, start_revision: i64) -> Result<WatchStream> {
count_call("watch_changelog_config_from");
let key = self.key(StoreKey::ChangelogConfig);
Ok(self.inner.watch_key_from(&key, start_revision).await?)
}
pub async fn get_generation(&self) -> Result<String> {
let key = self.key(StoreKey::Generation);
let bytes = self
.inner
.get_raw(&key)
.await?
.ok_or_else(|| Error::NotFound(key))?;
String::from_utf8(bytes)
.map_err(|e| Error::invalid_state(format!("non-utf8 generation: {e}")))
}
pub async fn set_generation(&self, generation: &str) -> Result<()> {
let key = self.key(StoreKey::Generation);
Ok(self.inner.put_raw(&key, generation).await?)
}
// ── Cleanup ─────────────────────────────────────────────────
/// Delete all keys under the store's prefix.
pub async fn delete_all(&self) -> Result<()> {
Ok(self.inner.delete_all().await?)
}
}
/// Extract a partition number from an etcd key like `{prefix}assignments/42`.
pub fn extract_partition_from_key(key: &str) -> Option<u32> {
key.rsplit('/').next()?.parse().ok()
}
/// Extract the second-to-last segment as a partition number from keys like
/// `{prefix}handoff_acks/42/router-0`.
pub fn extract_partition_from_ack_key(key: &str) -> Option<u32> {
let parts: Vec<&str> = key.rsplitn(3, '/').collect();
if parts.len() >= 2 {
parts[1].parse().ok()
} else {
None
}
}
#[cfg(test)]
mod tests {
use super::plan_chunk_ranges;
/// Every unit lands in exactly one chunk, in order — chunking must
/// never drop or reorder a partition's disposition.
fn assert_contiguous_cover(ranges: &[std::ops::Range<usize>], len: usize) {
let mut next = 0;
for r in ranges {
assert_eq!(r.start, next, "chunks must be contiguous");
assert!(r.end > r.start, "chunks must be non-empty");
next = r.end;
}
assert_eq!(next, len, "chunks must cover every unit");
}
/// The 256→1 collapse shape: 256 creations of (2 guards, 1 op)
/// against etcd's default budget. The compare list is what binds —
/// an unchunked plan carries 512 compares and is rejected outright.
#[test]
fn creations_chunk_on_the_compare_list() {
let costs = vec![(2, 1); 256];
let ranges = plan_chunk_ranges(&costs, 128, 0, 1);
assert_contiguous_cover(&ranges, costs.len());
for r in &ranges {
let guards: usize = costs[r.clone()].iter().map(|c| c.0).sum();
let ops: usize = costs[r.clone()].iter().map(|c| c.1).sum();
assert!(guards <= 128, "chunk carries {guards} compares");
assert!(ops < 128, "chunk carries {ops} ops plus the quorum write");
}
assert_eq!(
ranges.len(),
4,
"512 compares over a 128 budget is 4 chunks"
);
}
/// The mass-cancellation shape: replacements cost (2 guards, 4 ops),
/// so the op list binds, and the reserved quorum write counts
/// against every chunk.
#[test]
fn replacements_chunk_on_the_op_list_with_the_reserve() {
let costs = vec![(2, 4); 256];
let ranges = plan_chunk_ranges(&costs, 128, 0, 1);
assert_contiguous_cover(&ranges, costs.len());
for r in &ranges {
let ops: usize = costs[r.clone()].iter().map(|c| c.1).sum();
assert!(ops < 128, "chunk carries {ops} ops plus the quorum write");
}
}
/// A plan that fits stays one transaction — chunking must not tax
/// the healthy case with extra round trips.
#[test]
fn a_fitting_plan_is_one_chunk() {
let costs = vec![(2, 1); 60];
assert_eq!(plan_chunk_ranges(&costs, 128, 0, 1), vec![0..60]);
}
/// A budget below a single unit's cost still emits that unit alone:
/// the packer must make progress and let the server be the judge.
#[test]
fn an_oversized_unit_is_its_own_chunk() {
let costs = vec![(2, 4), (2, 4)];
assert_eq!(plan_chunk_ranges(&costs, 1, 0, 0), vec![0..1, 1..2]);
}
/// A plan's authority compares are reserved in every chunk.
#[test]
fn a_reserved_guard_takes_a_compare_slot() {
let costs = vec![(2, 1), (2, 1)];
assert_eq!(plan_chunk_ranges(&costs, 4, 0, 0), vec![0..2]);
assert_eq!(plan_chunk_ranges(&costs, 4, 1, 0), vec![0..1, 1..2]);
}
}