:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(personhog): park router requests per key during handoffs (#119810)
Zach Brown49669cadf9fb
diff --git a/rust/common/keyed-stash/src/lib.rs b/rust/common/keyed-stash/src/lib.rsindex ff3b12f9b70f..7c296f9ec09d 100644--- a/rust/common/keyed-stash/src/lib.rs+++ b/rust/common/keyed-stash/src/lib.rs@@ -81,8 +81,12 @@ impl<K: Eq + Hash + Clone, T> KeyedStash<K, T> {/// known. Sending new work directly while this holds would let it/// race ahead of the key's backlog.pub fn is_deferring(&self, key: &K) -> bool {+ self.has_in_attempt(key) || self.queues.get(key).is_some_and(|q| !q.is_empty())+ }++ /// Whether `key` has entries taken for an attempt and not yet completed or put back.+ pub fn has_in_attempt(&self, key: &K) -> bool {self.in_attempt.get(key).is_some_and(|n| *n > 0)- || self.queues.get(key).is_some_and(|q| !q.is_empty())}/// Take up to `max` entries from the front of `key`'s queue for a@@ -161,6 +165,12 @@ impl<K: Eq + Hash + Clone, T> KeyedStash<K, T> {self.queues.keys()}+ /// Keys for which [`is_deferring`](Self::is_deferring) holds; a key+ /// with both queued and in-attempt entries may appear twice.+ pub fn deferring_keys(&self) -> impl Iterator<Item = &K> {+ self.queues.keys().chain(self.in_attempt.keys())+ }+/// Total queued entries across all keys (in-attempt entries are not/// queued).pub fn len(&self) -> usize {@@ -174,6 +184,8 @@ impl<K: Eq + Hash + Clone, T> KeyedStash<K, T> {#[cfg(test)]mod tests {+ use std::collections::HashSet;+use super::*;use proptest::prelude::*;@@ -270,6 +282,10 @@ mod tests {q.put_back(&key, taken);}}+ let deferring: HashSet<u8> = q.deferring_keys().copied().collect();+ for k in 0u8..4 {+ prop_assert_eq!(deferring.contains(&k), q.is_deferring(&k));+ }}// Per-key delivery order == admission order.diff --git a/rust/personhog-common/src/partitioning.rs b/rust/personhog-common/src/partitioning.rsindex f116990c56fd..8e3460dfcb93 100644--- a/rust/personhog-common/src/partitioning.rs+++ b/rust/personhog-common/src/partitioning.rs@@ -8,6 +8,12 @@/// matches messages produced with the same key by any standard Kafka client.pub fn partition_for_person(team_id: i64, person_id: i64, num_partitions: u32) -> u32 {assert!(num_partitions > 0, "num_partitions must be > 0");+ person_hash(team_id, person_id) % num_partitions+}++/// The non-negative hash `partition_for_person` reduces modulo the partition+/// count, for callers that ask the same key's partition under several counts.+pub fn person_hash(team_id: i64, person_id: i64) -> u32 {// i64 max string length is 20 chars. Two i64s + ':' = 41 bytes max.let mut buf = [0u8; 41];let len = {@@ -16,16 +22,19 @@ pub fn partition_for_person(team_id: i64, person_id: i64, num_partitions: u32) -write!(cursor, "{team_id}:{person_id}").unwrap();cursor.position() as usize};- partition_for_key(&buf[..len], num_partitions)+ key_hash(&buf[..len])}/// Kafka's default partitioner for a keyed record, which Node's/// `murmur2Partition` and librdkafka's `murmur2_random` both reproduce.pub fn partition_for_key(key: &[u8], num_partitions: u32) -> u32 {assert!(num_partitions > 0, "num_partitions must be > 0");+ key_hash(key) % num_partitions+}++fn key_hash(key: &[u8]) -> u32 {// Kafka's toPositive: hash & 0x7fffffff- let positive = (kafka_murmur2(key) & 0x7fffffff) as u32;- positive % num_partitions+ (kafka_murmur2(key) & 0x7fffffff) as u32}/// Kafka-compatible murmur2 hash.diff --git a/rust/personhog-coordination/README.md b/rust/personhog-coordination/README.mdindex 0adb6875a8da..20dd2ff94351 100644--- a/rust/personhog-coordination/README.md+++ b/rust/personhog-coordination/README.md@@ -127,6 +127,8 @@ Several failure modes are handled explicitly:`config/changelog` (`{topic, partitions, epoch, writer_group, draining}`) is the one live object every participant reads. Every service loads it at boot; the router and the identity service also follow it through a watch, so the partition count they hash under is the live one.+Routers park requests per key, so a key's requests keep one queue when the partition count changes: a request goes straight to a leader only when its key has nothing parked or in flight and its partition is not stashing, and the live count decides only which partition's drain takes a key. A drain takes the keys that map to its partition under the count the router has adopted.+## Modules| Module | Responsibility |diff --git a/rust/personhog-coordination/src/routing_table.rs b/rust/personhog-coordination/src/routing_table.rsindex 0ef373ede5e3..ae5e1eec84ae 100644--- a/rust/personhog-coordination/src/routing_table.rs+++ b/rust/personhog-coordination/src/routing_table.rs@@ -42,17 +42,8 @@ pub trait StashHandler: Send + Sync {cancel: CancellationToken,) -> Result<()>;- /// Whether the partition's stash lifecycle is still open — an entry- /// exists, holding parked requests or an unflushed (possibly empty)- /// stash window. This is the authority the reconcile pass consults- /// before re-requesting a drain: a settled partition has no entry- /// (the drain's final act evicts it), while a drain that yielded- /// early leaves the entry and its backlog behind, so re-requesting- /// is exactly the retry mechanism. Entry existence — not backlog- /// depth — is the correct predicate, because a completed handoff- /// with an empty stash window still needs one drain to settle and- /// evict the entry; gating on parked requests would leave that- /// window open forever.+ /// Whether the partition still needs a drain: it is stashing, or parked+ /// requests map to it. The reconcile pass re-requests a drain while this holds.////// Defaults to `true`, which preserves the always-request behavior/// for implementations that don't track stash state.@@ -68,8 +59,8 @@ pub trait StashHandler: Send + Sync {/// on the watch loop, where it would gate freeze acks and routing updates/// for every partition. Lanes serialize drains per partition: requesting a/// drain toward a different target supersedes the one in flight (its token-/// is cancelled and it yields at a batch boundary, leaving the stash entry-/// live), and the new drain starts only after the old one has fully+/// is cancelled and it yields at a batch boundary, leaving its backlog+/// parked), and the new drain starts only after the old one has fully/// stopped, so two drains can never interleave batches toward different/// targets. A request toward the same target is absorbed by the running/// drain, whose loop already covers later arrivals. Observing a@@ -115,8 +106,8 @@ impl DrainLanes {let task_token = token.clone();let mut lanes = self.lanes.lock().expect("drain lanes lock poisoned");// A drain already running toward the same target covers everything- // this request would: its loop keeps taking runs until it observes- // the queue fully settled, including arrivals after this point.+ // this request would: its loop keeps taking runs until no key of+ // the partition remains, including arrivals after this point.// Restarting it would only churn — cancel, put back, re-take the// same backlog — for no coverage gain.if let Some(prev) = lanes.get(&partition) {@@ -979,7 +970,7 @@ impl RoutingTable {/// One level-triggered convergence pass over a fresh snapshot: the/// router's equivalent of the coordinator's reconcile tick and the/// pod's fresh-read convergence. Every action here is idempotent- /// against the event path — `begin_stash` no-ops on a live entry,+ /// against the event path — `begin_stash` renews a freeze,/// freeze acks are id-correlated, table writes are last-writer-wins/// from the authority, and drain requests are absorbed by a running/// same-target lane.@@ -1046,8 +1037,8 @@ impl RoutingTable {.write().await.insert(handoff.partition, handoff.new_owner.clone());- // Only request a drain while the stash lifecycle is- // open. A settled partition has no entry, and its+ // Only request a drain while the partition is+ // pending. A settled partition is not, and its// finished lane never absorbs (a finished drain may// have yielded with backlog), so an unconditional// request here would respawn a no-op drain everydiff --git a/rust/personhog-router/Cargo.toml b/rust/personhog-router/Cargo.tomlindex e34fa205f36b..85dced4338cf 100644--- a/rust/personhog-router/Cargo.toml+++ b/rust/personhog-router/Cargo.toml@@ -38,6 +38,7 @@ tracing = { workspace = true }tracing-subscriber = { workspace = true }[dev-dependencies]+personhog-coordination = { path = "../personhog-coordination", features = ["test-support"] }# Test-only: the store layer emits the etcd payload histogram from both# this binary and every binary on the shared recorder, so the two bucket# ladders have to match or the series cannot be aggregated across jobs.diff --git a/rust/personhog-router/src/backend/leader.rs b/rust/personhog-router/src/backend/leader.rsindex 2b408cbadd73..eff032c33abe 100644--- a/rust/personhog-router/src/backend/leader.rs+++ b/rust/personhog-router/src/backend/leader.rs@@ -15,11 +15,6 @@ use tonic::{Code, Status};use tower::{Service, ServiceExt};use personhog_common::grpc::{current_client_name, SEMANTIC_REFUSAL_METADATA_KEY};--/// Set by the leader on writes refused for a lifecycle hold. Mirrored so-/// the router does not depend on the leader crate.-const FENCED_METADATA_KEY: &str = "x-person-fenced";-const FENCED_OP_ID_METADATA_KEY: &str = "x-person-fenced-op-id";use personhog_common::partitioning::partition_for_person;use personhog_coordination::changelog::LiveChangelog;@@ -27,6 +22,11 @@ use super::stash::{StashDecision, StashTable};use crate::config::Http2Windows;use crate::grpc_http::{grpc_error_response, grpc_status_code};+/// Set by the leader on writes refused for a lifecycle hold. Mirrored so+/// the router does not depend on the leader crate.+const FENCED_METADATA_KEY: &str = "x-person-fenced";+const FENCED_OP_ID_METADATA_KEY: &str = "x-person-fenced-op-id";+pub type AddressResolver = Arc<dyn Fn(&str) -> Option<String> + Send + Sync>;/// gRPC path prefix for the leader service; raw forwards target@@ -148,9 +148,7 @@ pub struct LeaderBackend {/// Resolves pod_name → gRPC address.address_resolver: AddressResolver,config: LeaderBackendConfig,- /// Per-partition stash queue used to buffer writes during partition- /// handoffs. Consulted before every write; normal operation has no stash- /// entries and hits the dashmap miss-path once per request.+ /// Parks leader-path requests during partition handoffs; lock-free when nothing is parked.stash: StashTable,}@@ -426,7 +424,6 @@ impl LeaderBackend {pub async fn forward_or_stash(&self,method: &'static str,- partition: u32,key: (i64, i64),headers: HeaderMap,frame: Bytes,@@ -436,15 +433,20 @@ impl LeaderBackend {// needs to treat a replay as at-least-once.let mut possibly_applied = false;loop {+ // Per attempt: a bounce may be the leader refusing a stale modulus.+ let partition = self.partition_for_person(key.0, key.1);// The stash module emits its own enqueued/rejected counters// at the source; we don't double-count here. It borrows the// frame and headers, cloning only when the request actually// parks, so the steady-state forward path copies nothing.- match self- .stash- .enqueue_or_forward(partition, method, &frame, &headers, key, possibly_applied)- .await- {+ match self.stash.enqueue_or_forward(+ method,+ &frame,+ &headers,+ key,+ partition,+ possibly_applied,+ ) {StashDecision::Stashed(rx) => {let response = rx.await.unwrap_or_else(|_| {grpc_error_response(@@ -575,7 +577,7 @@ mod tests {num_channels,..test_config(8)},- StashTable::with_bounds(usize::MAX, usize::MAX),+ StashTable::with_bounds(usize::MAX, usize::MAX, changelog(8)),);let partition = backend.partition_for_person(1, 42);routing_table@@ -601,7 +603,7 @@ mod tests {routing_table,resolver,test_config(8),- StashTable::with_bounds(usize::MAX, usize::MAX),+ StashTable::with_bounds(usize::MAX, usize::MAX, changelog(8)),);let partition = backend.partition_for_person(1, 42);@@ -623,14 +625,12 @@ mod tests {routing_table,resolver,test_config(8),- StashTable::with_bounds(usize::MAX, usize::MAX),+ StashTable::with_bounds(usize::MAX, usize::MAX, changelog(8)),);- let partition = backend.partition_for_person(1, 42);let (response, call_ms) = backend.forward_or_stash("UpdatePersonProperties",- partition,(1, 42),HeaderMap::new(),Bytes::from_static(b"x"),@@ -650,7 +650,7 @@ mod tests {Arc::clone(&routing_table),Arc::new(|_| None), // resolver returns Nonetest_config(8),- StashTable::with_bounds(usize::MAX, usize::MAX),+ StashTable::with_bounds(usize::MAX, usize::MAX, changelog(8)),);let partition = backend.partition_for_person(1, 42);@@ -673,7 +673,7 @@ mod tests {Arc::clone(&routing_table),resolver,test_config(8),- StashTable::with_bounds(usize::MAX, usize::MAX),+ StashTable::with_bounds(usize::MAX, usize::MAX, changelog(8)),);let partition = backend.partition_for_person(1, 42);@@ -762,7 +762,7 @@ mod tests {let routing_table = Arc::new(RwLock::new(HashMap::new()));let resolver: AddressResolver = Arc::new(|_| Some("http://localhost:50053".to_string()));// `max_messages = 0` rejects any enqueue once the stash is open.- let stash = StashTable::with_bounds(0, usize::MAX);+ let stash = StashTable::with_bounds(0, usize::MAX, changelog(8));let backend = LeaderBackend::new(Arc::clone(&routing_table),resolver,@@ -773,12 +773,11 @@ mod tests {// Determine which partition this request lands on, then open the// stash for that exact partition.let partition = backend.partition_for_person(1, 42);- stash.begin_stash(partition).await;+ stash.begin_stash(partition);let (response, call_ms) = backend.forward_or_stash("UpdatePersonProperties",- partition,(1, 42),HeaderMap::new(),Bytes::new(),diff --git a/rust/personhog-router/src/backend/stash.rs b/rust/personhog-router/src/backend/stash.rsindex dd7843318ca0..0490785bf199 100644--- a/rust/personhog-router/src/backend/stash.rs+++ b/rust/personhog-router/src/backend/stash.rs@@ -1,4 +1,4 @@-//! Per-partition stash for the leader path.+//! Per-key stash for the leader path.//!//! During a partition handoff, the coordinator advances the handoff state//! through `Freezing → Draining → Warming → Complete`. From the moment@@ -14,25 +14,23 @@//! (per-key FIFO) drains after it, instead of racing ahead to the old//! owner's frozen cache.//!-//! Internally each partition holds a [`KeyedStash`] keyed by-//! `(team_id, person_id)` — the leader's per-person serialization-//! boundary. Entries are seq-stamped at admission and leave the stash-//! only through a definitive outcome (delivered, or failed in a way the-//! client was told about); an attempt that cannot finish — the target-//! bounced it, the drain was paused or superseded — puts its entries-//! back at their sequence positions, so a retry can never be overtaken-//! by newer same-key work. Draining runs through a [`DrainSession`],-//! which owns the take → complete/put-back lifecycle and the-//! empty-queue eviction handshake.--use std::sync::Arc;+//! Requests park per person `(team_id, person_id)`, so a person keeps one queue+//! whatever the partition count does; the count only decides which drain takes it.+//!+//! An entry leaves the stash only with a definitive outcome. An attempt that+//! cannot finish puts its entries back in order, so a retry is never overtaken.++use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};+use std::sync::{Arc, Mutex, MutexGuard};use std::time::Instant;use bytes::Bytes;use dashmap::DashMap;use http::HeaderMap;use keyed_stash::{Entry, KeyedStash};-use tokio::sync::{oneshot, Mutex};+use personhog_common::partitioning::person_hash;+use personhog_coordination::changelog::LiveChangelog;+use tokio::sync::oneshot;use tonic::body::BoxBody;/// The leader's per-person serialization key: `(team_id, person_id)`.@@ -48,6 +46,9 @@ pub type StashKey = (i64, i64);/// approximation, not a user-tunable knob.const PER_REQUEST_OVERHEAD: usize = 64;+/// Shards of the key space, fixed so a key's shard never depends on the partition count.+const SHARDS: u32 = 64;+/// Why the stash rejected a request. Used for metric labelling.#[derive(Debug, Clone, Copy, PartialEq, Eq)]enum RejectCause {@@ -83,9 +84,6 @@ pub struct StashedRequest {/// The client's request headers, forwarded verbatim (the router stamps/// `x-partition` at forward time).pub headers: HeaderMap,- /// The leader's per-person serialization key, used to preserve- /// per-key ordering during drain.- pub key: StashKey,pub reply: oneshot::Sender<http::Response<BoxBody>>,/// Wall-clock time the request was enqueued. Used to enforce the/// per-request drain deadline and to record stash-wait histograms,@@ -99,70 +97,28 @@ pub struct StashedRequest {/// contract in `personhog-leader`'s README — and is counted so/// operators can see how often it happens.pub possibly_applied: bool,+ /// The partition whose bounds admission charged.+ charged_to: u32,}-impl StashedRequest {- /// The size this request counts against the byte bound. Callers- /// recording a definitive outcome pass this back to- /// [`DrainSession::complete`] so the bound releases exactly what- /// admission charged.- pub fn approximate_size(&self) -> usize {- approximate_size(&self.frame)- }-}--/// Inner keyed queue plus running admission totals. `messages` and-/// `bytes` count entries from admission until a definitive outcome —-/// including entries taken for an in-flight attempt, which still hold-/// their memory — so the bounds reflect what the router is actually-/// holding, not just what is queued.-struct StashQueue {- queue: KeyedStash<StashKey, StashedRequest>,- messages: usize,+/// What admission charged a request, released by its definitive outcome.+#[derive(Debug, Clone, Copy)]+pub struct Charge {+ partition: u32,bytes: usize,}-impl StashQueue {- fn new() -> Self {- Self {- queue: KeyedStash::new(),- messages: 0,- bytes: 0,- }- }-}--struct PartitionStash {- max_messages: usize,- max_bytes: usize,- /// `Some(queue)` while alive; `None` once a drain session has both- /// observed the queue fully settled and evicted this entry from the- /// dashmap — both inside the same critical section. The `None` state- /// therefore doubles as a tombstone: a concurrent `begin_stash` that- /// races the eviction and observes `None` knows the dashmap entry is- /// already gone and that the next `get_or_create` will produce a- /// fresh entry rather than re-grab this doomed `Arc`.- queue: Mutex<Option<StashQueue>>,-}--impl PartitionStash {- fn new(max_messages: usize, max_bytes: usize) -> Self {- Self {- max_messages,- max_bytes,- // Initialize to `Some` so a freshly-created entry is- // immediately ready for enqueues. A `None` queue- // unambiguously signals "drained" — no other origin- // produces it.- queue: Mutex::new(Some(StashQueue::new())),+impl StashedRequest {+ /// Passed back to [`DrainSession::complete`] so the bounds release what admission charged.+ pub fn charge(&self) -> Charge {+ Charge {+ partition: self.charged_to,+ bytes: approximate_size(&self.frame),}}}-/// Outcome of an `enqueue_or_forward` call. `Forward` means "not frozen,-/// route normally"; `Stashed` means the request has been enqueued and the-/// caller should await the receiver for its reply; `Rejected` means the-/// stash is full (either too many messages or too many bytes).+/// `Forward`: route normally. `Stashed`: await the receiver. `Rejected`: the bounds are full.pub enum StashDecision {Forward,Stashed(oneshot::Receiver<http::Response<BoxBody>>),@@ -177,154 +133,218 @@ pub struct TakenKeyRun {pub entries: Vec<Entry<StashedRequest>>,}+/// A parked key with its hash, so its partition under any count is one modulo.+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]+struct ParkedKey {+ key: StashKey,+ hash: u32,+}++impl ParkedKey {+ fn of(key: StashKey) -> Self {+ Self {+ key,+ hash: person_hash(key.0, key.1),+ }+ }+}++struct Shard {+ queue: Mutex<KeyedStash<ParkedKey, StashedRequest>>,+ /// Entries from admission until a definitive outcome; zero lets a request skip the lock.+ held: AtomicUsize,+}++impl Shard {+ fn lock(&self) -> MutexGuard<'_, KeyedStash<ParkedKey, StashedRequest>> {+ self.queue.lock().expect("stash shard lock poisoned")+ }+}++/// Admission totals for one partition, counting every entry until its outcome.+#[derive(Default)]+struct Usage {+ messages: usize,+ bytes: usize,+}++struct Inner {+ shards: Vec<Shard>,+ /// Stashing partitions, each with the generation of the freeze that opened it.+ stashing: DashMap<u32, u64>,+ next_generation: AtomicU64,+ usage: DashMap<u32, Usage>,+ max_messages: usize,+ max_bytes: usize,+ changelog: LiveChangelog,+}++impl Inner {+ fn shard_of(&self, key: &ParkedKey) -> &Shard {+ &self.shards[(key.hash % SHARDS) as usize]+ }++ fn partition_of(&self, key: &ParkedKey) -> u32 {+ key.hash % self.changelog.partitions()+ }++ /// Whether any key that maps to `partition` has parked or in-flight+ /// requests, locking one shard at a time.+ fn holds_keys_of(&self, partition: u32) -> bool {+ self.shards.iter().any(|shard| {+ shard.held.load(Ordering::Acquire) > 0+ && shard+ .lock()+ .deferring_keys()+ .any(|key| self.partition_of(key) == partition)+ })+ }++ fn release(&self, shard: &Shard, charge: Charge) {+ shard.held.fetch_sub(1, Ordering::AcqRel);+ match self.usage.entry(charge.partition) {+ dashmap::Entry::Occupied(mut usage) => {+ let usage_ref = usage.get_mut();+ usage_ref.messages -= 1;+ usage_ref.bytes = usage_ref.bytes.saturating_sub(charge.bytes);+ if usage_ref.messages == 0 {+ usage.remove();+ }+ metrics::gauge!("personhog_router_stash_queued_messages").decrement(1.0);+ metrics::gauge!("personhog_router_stash_queued_bytes")+ .decrement(charge.bytes as f64);+ }+ dashmap::Entry::Vacant(_) => debug_assert!(false, "released a charge never admitted"),+ }+ }+}+/// Shared stash table. Cheap to clone (holds an `Arc`).#[derive(Clone)]pub struct StashTable {- inner: Arc<DashMap<u32, Arc<PartitionStash>>>,- max_messages: usize,- max_bytes: usize,+ inner: Arc<Inner>,}impl StashTable {- pub fn with_bounds(max_messages: usize, max_bytes: usize) -> Self {+ pub fn with_bounds(max_messages: usize, max_bytes: usize, changelog: LiveChangelog) -> Self {+ let shards = (0..SHARDS)+ .map(|_| Shard {+ queue: Mutex::new(KeyedStash::new()),+ held: AtomicUsize::new(0),+ })+ .collect();Self {- inner: Arc::new(DashMap::new()),- max_messages,- max_bytes,+ inner: Arc::new(Inner {+ shards,+ stashing: DashMap::new(),+ next_generation: AtomicU64::new(0),+ usage: DashMap::new(),+ max_messages,+ max_bytes,+ changelog,+ }),}}- /// Whether the partition currently has a stash entry — a live- /// window, or backlog a yielded drain left parked. The entry only- /// disappears through drain's settle-and-evict, so its existence is- /// exactly "the stash lifecycle for this partition is not closed".- pub fn has_entry(&self, partition: u32) -> bool {- self.inner.contains_key(&partition)+ /// Whether `partition` still needs a drain: it is stashing, or a key+ /// that maps to it under the live count has parked or in-flight requests.+ pub fn is_pending(&self, partition: u32) -> bool {+ self.inner.stashing.contains_key(&partition) || self.inner.holds_keys_of(partition)}- fn get_or_create(&self, partition: u32) -> Arc<PartitionStash> {- self.inner- .entry(partition)- .or_insert_with(|| Arc::new(PartitionStash::new(self.max_messages, self.max_bytes)))- .clone()+ /// Begin buffering requests for `partition`. Each call opens a new+ /// generation, so a drain that started before it cannot close it.+ pub fn begin_stash(&self, partition: u32) {+ let generation = self.inner.next_generation.fetch_add(1, Ordering::Relaxed);+ self.inner.stashing.insert(partition, generation);}- /// Begin buffering requests for `partition`. Idempotent: safe to call- /// multiple times (watch reconnects etc.).- ///- /// Each iteration is a single attempt to bind to a live dashmap- /// entry. The loop terminates because the only way to observe- /// `None` is to race a drain session's eviction, which removes the- /// entry inside its queue lock — meaning the next `get_or_create`- /// will create a fresh entry rather than re-grab the doomed `Arc`.- /// In production the routing-table layer awaits the drain before- /// issuing the next `begin_stash` for the same partition, so this- /// loop runs at most twice.- pub async fn begin_stash(&self, partition: u32) {- while !self.try_acquire_alive_entry(partition).await {}- }-- /// One attempt to bind to the live dashmap entry for `partition`.- /// Returns `true` if we successfully observed a `Some` queue (the- /// entry is alive and ready for enqueues); `false` if we raced an- /// eviction that left a tombstoned `None`, in which case the caller- /// should retry to pick up the fresh entry.- async fn try_acquire_alive_entry(&self, partition: u32) -> bool {- let stash = self.get_or_create(partition);- let guard = stash.queue.lock().await;- guard.is_some()- }-- /// Enqueue a request if the partition is frozen; otherwise return- /// `Forward` to signal the caller should route normally.- /// Takes the frame and headers by reference and clones them only when- /// the request actually parks — the steady state (no handoff in- /// progress) returns `Forward` from the dashmap miss without copying- /// anything.+ /// Park the request if its key has parked or in-flight requests or `partition`+ /// is stashing; otherwise `Forward`. Clones the frame only when it parks.////// `possibly_applied` carries the direct path's transport-failure mark/// into the parked entry: a request that bounced at the transport/// layer before parking may already have reached the leader, and the/// drain's eventual re-forward of it must count as a replay.- pub async fn enqueue_or_forward(+ pub fn enqueue_or_forward(&self,- partition: u32,method: &'static str,frame: &Bytes,headers: &HeaderMap,key: StashKey,+ partition: u32,possibly_applied: bool,) -> StashDecision {- let stash = match self.inner.get(&partition) {- Some(entry) => Arc::clone(entry.value()),- None => return StashDecision::Forward,- };- let mut guard = stash.queue.lock().await;- // `None` here means a drain session already settled this stash- // and removed the dashmap entry. We arrived holding an Arc to- // the orphaned PartitionStash. Returning Forward routes via the- // normal path; the partition is no longer stashing.- let Some(queue) = guard.as_mut() else {+ let inner = &self.inner;+ let parked = ParkedKey::of(key);+ let shard = inner.shard_of(&parked);+ if shard.held.load(Ordering::Acquire) == 0 && !inner.stashing.contains_key(&partition) {return StashDecision::Forward;- };+ }+ // Under the shard lock, so a drain's settle check sees this request or it sees the settle.+ let mut queue = shard.lock();+ if !queue.is_deferring(&parked) && !inner.stashing.contains_key(&partition) {+ return StashDecision::Forward;+ }let request_size = approximate_size(frame);-- if queue.messages >= stash.max_messages {- metrics::counter!(- "personhog_router_stash_rejected_total",- "cause" => RejectCause::MaxMessages.label()- )- .increment(1);- return StashDecision::Rejected;- }- if queue.bytes.saturating_add(request_size) > stash.max_bytes {- metrics::counter!(- "personhog_router_stash_rejected_total",- "cause" => RejectCause::MaxBytes.label()- )- .increment(1);+ // The check and the charge share one hold of the partition's entry, since+ // the shard lock only serializes parks of keys in the same shard.+ let usage = inner.usage.entry(partition);+ let (messages, bytes) = match &usage {+ dashmap::Entry::Occupied(entry) => (entry.get().messages, entry.get().bytes),+ dashmap::Entry::Vacant(_) => (0, 0),+ };+ let rejected = if messages >= inner.max_messages {+ Some(RejectCause::MaxMessages)+ } else if bytes.saturating_add(request_size) > inner.max_bytes {+ Some(RejectCause::MaxBytes)+ } else {+ None+ };+ if let Some(cause) = rejected {+ drop(usage);+ metrics::counter!("personhog_router_stash_rejected_total", "cause" => cause.label())+ .increment(1);return StashDecision::Rejected;}+ let mut charged = usage.or_default();+ charged.messages += 1;+ charged.bytes += request_size;+ drop(charged);let (tx, rx) = oneshot::channel();- queue.queue.push(- key,+ queue.push(+ parked,StashedRequest {method,frame: frame.clone(),headers: headers.clone(),- key,reply: tx,enqueued_at: Instant::now(),possibly_applied,+ charged_to: partition,},);- queue.messages += 1;- queue.bytes += request_size;+ shard.held.fetch_add(1, Ordering::AcqRel);metrics::counter!("personhog_router_stash_enqueued_total").increment(1);- // Occupancy gauges count a parked entry until its outcome- // resolves, so in-attempt entries still occupy capacity.+ // Occupancy counts a parked entry until its outcome resolves.metrics::gauge!("personhog_router_stash_queued_messages").increment(1.0);metrics::gauge!("personhog_router_stash_queued_bytes").increment(request_size as f64);StashDecision::Stashed(rx)}- /// Open a drain session for `partition`, or `None` when nothing is- /// stashed. The session pins the partition's stash so every take,- /// complete, and put-back lands on the same queue the entries came- /// from, even across the eviction handshake.- ///- /// The drain-lane layer serializes drains per partition, so at most- /// one session is active for a partition at a time; the session API- /// relies on that for its eviction reasoning but tolerates a racing- /// `begin_stash` (see [`DrainSession::finish_if_settled`]).+ /// Open a drain session for `partition`, or `None` when nothing is pending.+ /// After a count change two sessions can meet on one key; each skips keys the other holds.pub fn drain_session(&self, partition: u32) -> Option<DrainSession> {- let stash = Arc::clone(self.inner.get(&partition)?.value());+ if !self.is_pending(partition) {+ return None;+ }Some(DrainSession {inner: Arc::clone(&self.inner),partition,- stash,+ generation: self.inner.stashing.get(&partition).map(|g| *g),+ candidates: Mutex::new(Vec::new()),})}}@@ -342,110 +362,108 @@ impl StashTable {/// superseded). The entries re-enter at their sequence positions,/// ahead of everything admitted after them.///-/// When a take finds no backlog, [`finish_if_settled`](Self::finish_if_settled)-/// closes the drain: if the queue is fully settled it tombstones the-/// queue and evicts the dashmap entry in one critical section, so any-/// racing `enqueue_or_forward` either pushes before the settle check-/// (forcing the caller through another take) or observes the entry-/// already gone and forwards live.-///-/// Requests that arrive *during* a session land on the same queue (the-/// dashmap entry stays present until the settle eviction) and are picked-/// up by a later take, preserving per-key admission order across the-/// cutover.+/// [`finish_if_settled`](Self::finish_if_settled) closes the drain once the partition has no keys left.pub struct DrainSession {- inner: Arc<DashMap<u32, Arc<PartitionStash>>>,+ inner: Arc<Inner>,partition: u32,- stash: Arc<PartitionStash>,+ /// The freeze this session may close; `None` when it drains keys alone.+ generation: Option<u64>,+ /// Keys of the partition found by the last scan, not yet taken.+ candidates: Mutex<Vec<ParkedKey>>,}impl DrainSession {- /// Take the front run of up to `max_keys` keys for a delivery- /// attempt. Returns an empty vec when no backlog is queued. Every- /// returned entry is in-attempt until completed or put back;- /// dropping entries instead would lose their clients' replies and- /// permanently block the queue from settling.- pub async fn take_for_attempt(&self, max_keys: usize) -> Vec<TakenKeyRun> {- let mut guard = self.stash.queue.lock().await;- let Some(q) = guard.as_mut() else {- return Vec::new();- };- let keys: Vec<StashKey> = q- .queue- .keys_with_backlog()- .take(max_keys)- .copied()- .collect();- keys.into_iter()- .map(|key| TakenKeyRun {- entries: q.queue.take_front(&key, usize::MAX),- key,- })- .collect()+ fn candidates(&self) -> MutexGuard<'_, Vec<ParkedKey>> {+ self.candidates+ .lock()+ .expect("drain candidates lock poisoned")+ }++ /// Take the front runs of up to `max_keys` keys for one attempt. Each entry+ /// stays in-attempt until completed or put back.+ pub fn take_for_attempt(&self, max_keys: usize) -> Vec<TakenKeyRun> {+ let inner = &self.inner;+ if self.candidates().is_empty() {+ let found: Vec<ParkedKey> = inner+ .shards+ .iter()+ .filter(|shard| shard.held.load(Ordering::Acquire) > 0)+ .flat_map(|shard| {+ shard+ .lock()+ .keys_with_backlog()+ .filter(|key| inner.partition_of(key) == self.partition)+ .copied()+ .collect::<Vec<_>>()+ })+ .collect();+ *self.candidates() = found;+ }+ let mut runs = Vec::new();+ while runs.len() < max_keys {+ let Some(key) = self.candidates().pop() else {+ break;+ };+ let mut queue = inner.shard_of(&key).lock();+ // Another session holds this key, or a count change moved it elsewhere.+ if queue.has_in_attempt(&key) || inner.partition_of(&key) != self.partition {+ continue;+ }+ let entries = queue.take_front(&key, usize::MAX);+ if !entries.is_empty() {+ runs.push(TakenKeyRun {+ key: key.key,+ entries,+ });+ }+ }+ runs}/// Record a definitive outcome for one in-attempt entry of `key`,- /// releasing `size` bytes (the entry's `approximate_size`) from the- /// admission bounds.- pub async fn complete(&self, key: StashKey, size: usize) {- let mut guard = self.stash.queue.lock().await;- // In-attempt entries block the settle eviction and this session- // is the only writer of outcomes, so the queue cannot have been- // tombstoned while an attempt was outstanding.- let Some(q) = guard.as_mut() else {- debug_assert!(false, "complete() on a tombstoned stash");- return;- };- q.queue.complete(&key);- q.messages -= 1;- q.bytes = q.bytes.saturating_sub(size);- metrics::gauge!("personhog_router_stash_queued_messages").decrement(1.0);- metrics::gauge!("personhog_router_stash_queued_bytes").decrement(size as f64);+ /// releasing what admission charged it.+ pub fn complete(&self, key: StashKey, charge: Charge) {+ let key = ParkedKey::of(key);+ let shard = self.inner.shard_of(&key);+ shard.lock().complete(&key);+ self.inner.release(shard, charge);}/// Return in-attempt entries whose delivery was aborted before any/// outcome existed. They re-enter at their sequence positions and/// keep their admission charges; the next attempt sees them first,/// in order.- pub async fn put_back(&self, key: StashKey, entries: Vec<Entry<StashedRequest>>) {- if entries.is_empty() {- return;+ pub fn put_back(&self, key: StashKey, entries: Vec<Entry<StashedRequest>>) {+ let key = ParkedKey::of(key);+ self.inner.shard_of(&key).lock().put_back(&key, entries);+ }++ /// Close the session and end its freeze once the partition has no parked or+ /// in-flight keys; all shards stay locked across the check and the close.+ /// A request hashed under an older count can still park; the reconcile pass drains it.+ pub fn finish_if_settled(&self) -> bool {+ let inner = &self.inner;+ // A key still held needs no global lock to see.+ if inner.holds_keys_of(self.partition) {+ return false;}- let mut guard = self.stash.queue.lock().await;- // Same reasoning as `complete`: outstanding in-attempt entries- // make the tombstone unreachable.- let Some(q) = guard.as_mut() else {- debug_assert!(false, "put_back() on a tombstoned stash");- return;+ // With no freeze to close, a request arriving after the check forwards either way.+ let Some(generation) = self.generation else {+ return true;};- q.queue.put_back(&key, entries);- }-- /// Close the session if the queue is fully settled — no queued- /// entries and no in-attempt entries. On success the queue is- /// tombstoned and the dashmap entry evicted inside the same critical- /// section, and `true` is returned: the partition has left the stash- /// path and new requests forward live. Returns `false` when an- /// arrival raced in since the last take; the caller should take- /// again.- ///- /// The eviction removes the entry only if it still holds this- /// session's stash, so a `begin_stash` that already replaced a- /// tombstoned entry with a fresh one can never have its new queue- /// evicted from under it.- pub async fn finish_if_settled(&self) -> bool {- let mut guard = self.stash.queue.lock().await;- match guard.as_ref() {- // Already tombstoned — a prior settle got there first.- None => true,- Some(q) if q.queue.is_empty() => {- *guard = None;- self.inner- .remove_if(&self.partition, |_, v| Arc::ptr_eq(v, &self.stash));- true- }- Some(_) => false,+ let shards: Vec<_> = inner.shards.iter().map(Shard::lock).collect();+ let pending = shards.iter().any(|queue| {+ queue+ .deferring_keys()+ .any(|key| inner.partition_of(key) == self.partition)+ });+ if pending {+ return false;}+ inner+ .stashing+ .remove_if(&self.partition, |_, opened| *opened == generation);+ true}}@@ -454,66 +472,66 @@ mod tests {use super::*;use http::HeaderValue;use http_body_util::{BodyExt, Empty};+ use personhog_common::partitioning::partition_for_person;+ use personhog_coordination::types::ChangelogConfig;++ /// A table over `partitions` partitions; with one, every key maps to partition 0.+ fn table_over(partitions: u32, max_messages: usize, max_bytes: usize) -> StashTable {+ StashTable::with_bounds(+ max_messages,+ max_bytes,+ LiveChangelog::new(ChangelogConfig::new("personhog_updates", partitions)),+ )+ }/// Drain everything with a trivial always-succeeds forwarder,/// collecting the requests in delivery order. Mirrors the handler's/// session loop: take, complete each entry, settle when no backlog/// remains.- async fn drain_to_vec(table: &StashTable, partition: u32) -> Vec<StashedRequest> {+ fn drain_to_vec(table: &StashTable, partition: u32) -> Vec<StashedRequest> {let Some(session) = table.drain_session(partition) else {return Vec::new();};let mut out = Vec::new();loop {- let taken = session.take_for_attempt(usize::MAX).await;+ let taken = session.take_for_attempt(usize::MAX);if taken.is_empty() {- if session.finish_if_settled().await {+ if session.finish_if_settled() {return out;}continue;}for run in taken {for entry in run.entries {- let size = entry.item.approximate_size();+ let charge = entry.item.charge();out.push(entry.item);- session.complete(run.key, size).await;+ session.complete(run.key, charge);}}}}+ fn enqueue_frame(table: &StashTable, person_id: i64, frame: Bytes) -> StashDecision {+ let partition = partition_for_person(1, person_id, table.inner.changelog.partitions());+ table.enqueue_or_forward(+ "UpdatePersonProperties",+ &frame,+ &HeaderMap::new(),+ (1, person_id),+ partition,+ false,+ )+ }+/// Enqueue a minimal stashed write for `(team_id = 1, person_id)`.- async fn enqueue(table: &StashTable, partition: u32, person_id: i64) -> StashDecision {- table- .enqueue_or_forward(- partition,- "UpdatePersonProperties",- &Bytes::from_static(b"x"),- &HeaderMap::new(),- (1, person_id),- false,- )- .await+ fn enqueue(table: &StashTable, person_id: i64) -> StashDecision {+ enqueue_frame(table, person_id, Bytes::from_static(b"x"))}/// Enqueue a stashed write whose frame is `payload` bytes, exercising/// the byte-based stash bound.- async fn enqueue_sized(- table: &StashTable,- partition: u32,- person_id: i64,- payload: usize,- ) -> StashDecision {- table- .enqueue_or_forward(- partition,- "UpdatePersonProperties",- &Bytes::from(vec![0u8; payload]),- &HeaderMap::new(),- (1, person_id),- false,- )- .await+ fn enqueue_sized(table: &StashTable, person_id: i64, payload: usize) -> StashDecision {+ enqueue_frame(table, person_id, Bytes::from(vec![0u8; payload]))}/// A trivial gRPC response for exercising the reply channel.@@ -523,96 +541,98 @@ mod tests {))}- #[tokio::test]- async fn forward_when_not_frozen() {- let table = StashTable::with_bounds(usize::MAX, usize::MAX);- match enqueue(&table, 0, 1).await {+ /// A person whose partition differs between `from` and `to` partitions.+ fn moving_person(from: u32, to: u32) -> i64 {+ (1..)+ .find(|id| partition_for_person(1, *id, from) != partition_for_person(1, *id, to))+ .unwrap()+ }++ fn adopt(changelog: &LiveChangelog, partitions: u32) {+ changelog.adopt(&ChangelogConfig {+ epoch: changelog.epoch() + 1,+ ..ChangelogConfig::new("personhog_updates", partitions)+ });+ }++ #[test]+ fn forward_when_not_frozen() {+ let table = table_over(1, usize::MAX, usize::MAX);+ match enqueue(&table, 1) {StashDecision::Forward => {}_ => panic!("expected Forward"),}}- #[tokio::test]- async fn begin_then_enqueue_then_drain_preserves_per_key_order() {- let table = StashTable::with_bounds(usize::MAX, usize::MAX);- table.begin_stash(0).await;+ #[test]+ fn begin_then_enqueue_then_drain_preserves_per_key_order() {+ let table = table_over(1, usize::MAX, usize::MAX);+ table.begin_stash(0);// Same key, distinguishable frames — ordering is a per-key// contract, so the assertion must not depend on cross-key// iteration order.- let _rx1 = match enqueue_sized(&table, 0, 1, 1).await {+ let _rx1 = match enqueue_sized(&table, 1, 1) {StashDecision::Stashed(rx) => rx,_ => panic!("expected Stashed"),};- let _rx2 = match enqueue_sized(&table, 0, 1, 2).await {+ let _rx2 = match enqueue_sized(&table, 1, 2) {StashDecision::Stashed(rx) => rx,_ => panic!("expected Stashed"),};- let drained = drain_to_vec(&table, 0).await;+ let drained = drain_to_vec(&table, 0);assert_eq!(drained.len(), 2);let frame_lens: Vec<usize> = drained.iter().map(|s| s.frame.len()).collect();assert_eq!(frame_lens, vec![1, 2]);// After drain, new requests forward.- match enqueue(&table, 0, 3).await {+ match enqueue(&table, 3) {StashDecision::Forward => {}_ => panic!("expected Forward after drain"),}}- /// A settled drain must remove the dashmap entry so subsequent- /// steady-state requests for that partition can short-circuit on- /// `dashmap.get` returning `None`, avoiding the per-request Mutex- /// lock that `enqueue_or_forward` would otherwise take.- #[tokio::test]- async fn drain_removes_dashmap_entry() {- let table = StashTable::with_bounds(usize::MAX, usize::MAX);- table.begin_stash(0).await;- assert!(- table.inner.contains_key(&0),- "begin_stash must populate the entry"- );- drain_to_vec(&table, 0).await;- assert!(- !table.inner.contains_key(&0),- "settle must remove the entry so future requests skip the lock path"- );+ /// A settled drain must end the freeze, or the partition never+ /// leaves the stash path.+ #[test]+ fn a_settled_drain_ends_the_freeze() {+ let table = table_over(1, usize::MAX, usize::MAX);+ table.begin_stash(0);+ assert!(table.is_pending(0), "begin_stash must open the freeze");+ drain_to_vec(&table, 0);+ assert!(!table.is_pending(0), "settle must end the freeze");}- /// An aborted attempt puts its entries back and leaves the entry- /// live: whatever is parked belongs to the successor drain, and new- /// requests must keep stashing in the meantime. Evicting instead- /// would let requests bypass the stash and race ahead of the parked- /// backlog.- #[tokio::test]- async fn put_back_leaves_queue_live_for_successor() {- let table = StashTable::with_bounds(usize::MAX, usize::MAX);- table.begin_stash(0).await;- let _rx = match enqueue(&table, 0, 1).await {+ /// An aborted attempt leaves the partition pending, so new requests keep parking behind the backlog.+ #[test]+ fn put_back_leaves_queue_live_for_successor() {+ let table = table_over(1, usize::MAX, usize::MAX);+ table.begin_stash(0);+ let _rx = match enqueue(&table, 1) {StashDecision::Stashed(rx) => rx,_ => panic!("expected Stashed"),};// An attempt starts, then aborts (pause/supersession/bounce) and// puts everything back without settling.- let session = table.drain_session(0).expect("entry must be live");- let mut taken = session.take_for_attempt(usize::MAX).await;+ let session = table.drain_session(0).expect("partition must be pending");+ let mut taken = session.take_for_attempt(usize::MAX);assert_eq!(taken.len(), 1);let run = taken.remove(0);- session.put_back(run.key, run.entries).await;+ session.put_back(run.key, run.entries);drop(session);assert!(- table.inner.contains_key(&0),- "aborted drain must leave the entry live"+ table.is_pending(0),+ "aborted drain must leave the partition pending");// Requests keep parking, and a successor drain collects everything.- let _rx2 = match enqueue(&table, 0, 2).await {+ let _rx2 = match enqueue(&table, 2) {StashDecision::Stashed(rx) => rx,- _ => panic!("expected Stashed while entry is live"),+ _ => panic!("expected Stashed while the partition is pending"),};- let drained = drain_to_vec(&table, 0).await;+ let drained = drain_to_vec(&table, 0);assert_eq!(drained.len(),2,@@ -620,190 +640,257 @@ mod tests {);}- /// In-attempt entries must block the settle eviction: the queue can- /// look empty (backlog fully taken) while outcomes are still- /// outstanding, and settling then would orphan the eventual put-back- /// into a tombstoned queue, dropping the clients' replies.- #[tokio::test]- async fn in_attempt_entries_block_settle() {- let table = StashTable::with_bounds(usize::MAX, usize::MAX);- table.begin_stash(0).await;- let _rx = match enqueue(&table, 0, 1).await {+ /// In-attempt entries block the settle, so new requests cannot overtake a pending attempt.+ #[test]+ fn in_attempt_entries_block_settle() {+ let table = table_over(1, usize::MAX, usize::MAX);+ table.begin_stash(0);+ let _rx = match enqueue(&table, 1) {StashDecision::Stashed(rx) => rx,_ => panic!("expected Stashed"),};- let session = table.drain_session(0).expect("entry must be live");- let mut taken = session.take_for_attempt(usize::MAX).await;+ let session = table.drain_session(0).expect("partition must be pending");+ let mut taken = session.take_for_attempt(usize::MAX);let run = taken.remove(0);assert!(- !session.finish_if_settled().await,+ !session.finish_if_settled(),"settle must refuse while an attempt is outstanding");let entry = run.entries.into_iter().next().unwrap();- let size = entry.item.approximate_size();- session.complete(run.key, size).await;+ session.complete(run.key, entry.item.charge());assert!(- session.finish_if_settled().await,+ session.finish_if_settled(),"settle must succeed once every outcome is recorded");- assert!(!table.inner.contains_key(&0));+ assert!(!table.is_pending(0));}/// Back-to-back handoffs: drain → begin_stash for the same partition- /// must produce a fresh empty queue, not preserve stale state.- #[tokio::test]- async fn drain_then_begin_stash_starts_fresh() {- let table = StashTable::with_bounds(usize::MAX, usize::MAX);- table.begin_stash(0).await;- let _rx = match enqueue(&table, 0, 1).await {+ /// must stash again.+ #[test]+ fn drain_then_begin_stash_starts_fresh() {+ let table = table_over(1, usize::MAX, usize::MAX);+ table.begin_stash(0);+ let _rx = match enqueue(&table, 1) {StashDecision::Stashed(rx) => rx,_ => panic!("expected Stashed"),};- let drained = drain_to_vec(&table, 0).await;+ let drained = drain_to_vec(&table, 0);assert_eq!(drained.len(), 1);- // New handoff begins- table.begin_stash(0).await;- // Fresh queue; brand-new requests stash, not forward- match enqueue(&table, 0, 2).await {+ table.begin_stash(0);+ match enqueue(&table, 2) {StashDecision::Stashed(_) => {}_ => panic!("expected Stashed for fresh handoff"),}}- #[tokio::test]- async fn reject_when_message_count_exceeds_max() {- let table = StashTable::with_bounds(2, usize::MAX);- table.begin_stash(0).await;+ #[test]+ fn reject_when_message_count_exceeds_max() {+ let table = table_over(1, 2, usize::MAX);+ table.begin_stash(0);- assert!(matches!(- enqueue(&table, 0, 1).await,- StashDecision::Stashed(_)- ));- assert!(matches!(- enqueue(&table, 0, 2).await,- StashDecision::Stashed(_)- ));- assert!(matches!(- enqueue(&table, 0, 3).await,- StashDecision::Rejected- ));+ assert!(matches!(enqueue(&table, 1), StashDecision::Stashed(_)));+ assert!(matches!(enqueue(&table, 2), StashDecision::Stashed(_)));+ assert!(matches!(enqueue(&table, 3), StashDecision::Rejected));}- #[tokio::test]- async fn reject_when_byte_total_exceeds_max() {+ #[test]+ fn reject_when_byte_total_exceeds_max() {// Cap at ~5KB; each request below has ~2KB payload. After two// requests we're near 4KB; a third request would push over the cap.- let table = StashTable::with_bounds(usize::MAX, 5 * 1024);- table.begin_stash(0).await;+ let table = table_over(1, usize::MAX, 5 * 1024);+ table.begin_stash(0);assert!(matches!(- enqueue_sized(&table, 0, 1, 2 * 1024).await,+ enqueue_sized(&table, 1, 2 * 1024),StashDecision::Stashed(_)));assert!(matches!(- enqueue_sized(&table, 0, 2, 2 * 1024).await,+ enqueue_sized(&table, 2, 2 * 1024),StashDecision::Stashed(_)));assert!(matches!(- enqueue_sized(&table, 0, 3, 2 * 1024).await,+ enqueue_sized(&table, 3, 2 * 1024),StashDecision::Rejected));}/// Either bound triggers rejection — whichever is hit first.- #[tokio::test]- async fn message_count_takes_precedence_when_hit_first() {+ #[test]+ fn message_count_takes_precedence_when_hit_first() {// Generous byte budget but tight message budget.- let table = StashTable::with_bounds(1, 100 * 1024 * 1024);- table.begin_stash(0).await;+ let table = table_over(1, 1, 100 * 1024 * 1024);+ table.begin_stash(0);- assert!(matches!(- enqueue(&table, 0, 1).await,- StashDecision::Stashed(_)- ));+ assert!(matches!(enqueue(&table, 1), StashDecision::Stashed(_)));// Second message rejected on count even though bytes are nowhere near.- assert!(matches!(- enqueue(&table, 0, 2).await,- StashDecision::Rejected- ));+ assert!(matches!(enqueue(&table, 2), StashDecision::Rejected));}/// The admission bounds must release on definitive outcomes, not on/// takes: an entry taken for an attempt still holds its memory, and/// releasing at take time would let a slow drain admit unbounded/// backlog behind it.- #[tokio::test]- async fn bounds_release_on_complete_not_on_take() {- let table = StashTable::with_bounds(1, usize::MAX);- table.begin_stash(0).await;- assert!(matches!(- enqueue(&table, 0, 1).await,- StashDecision::Stashed(_)- ));-- let session = table.drain_session(0).expect("entry must be live");- let mut taken = session.take_for_attempt(usize::MAX).await;+ #[test]+ fn bounds_release_on_complete_not_on_take() {+ let table = table_over(1, 1, usize::MAX);+ table.begin_stash(0);+ assert!(matches!(enqueue(&table, 1), StashDecision::Stashed(_)));++ let session = table.drain_session(0).expect("partition must be pending");+ let mut taken = session.take_for_attempt(usize::MAX);let run = taken.remove(0);// Taken but not completed — still counted against the bound.- assert!(matches!(- enqueue(&table, 0, 2).await,- StashDecision::Rejected- ));+ assert!(matches!(enqueue(&table, 2), StashDecision::Rejected));let entry = run.entries.into_iter().next().unwrap();- let size = entry.item.approximate_size();- session.complete(run.key, size).await;+ session.complete(run.key, entry.item.charge());// Outcome recorded — capacity is released.+ assert!(matches!(enqueue(&table, 3), StashDecision::Stashed(_)));+ }++ #[test]+ fn partitions_are_independent() {+ let table = table_over(2, usize::MAX, usize::MAX);+ table.begin_stash(0);+ let on = |partition: u32| {+ (1..)+ .find(|id| partition_for_person(1, *id, 2) == partition)+ .unwrap()+ };++ // p0 stashes, p1 forwards+ assert!(matches!(enqueue(&table, on(0)), StashDecision::Stashed(_)));+ assert!(matches!(enqueue(&table, on(1)), StashDecision::Forward));+ }++ /// A key moved by a count change stays ordered behind its writes parked under the old partition.+ #[test]+ fn a_key_keeps_its_order_across_a_count_change() {+ let changelog = LiveChangelog::new(ChangelogConfig::new("personhog_updates", 8));+ let table = StashTable::with_bounds(usize::MAX, usize::MAX, changelog.clone());+ let person_id = moving_person(8, 4);+ let old = partition_for_person(1, person_id, 8);+ let new = partition_for_person(1, person_id, 4);+ table.begin_stash(old);+ table.begin_stash(new);assert!(matches!(- enqueue(&table, 0, 3).await,+ enqueue_frame(&table, person_id, Bytes::from_static(b"older")),StashDecision::Stashed(_)));- }- #[tokio::test]- async fn partitions_are_independent() {- let table = StashTable::with_bounds(usize::MAX, usize::MAX);- table.begin_stash(0).await;+ drain_to_vec(&table, new);+ adopt(&changelog, 4);- // p0 stashes, p1 forwardsassert!(matches!(- enqueue(&table, 0, 1).await,+ enqueue_frame(&table, person_id, Bytes::from_static(b"newer")),StashDecision::Stashed(_)));+ let frames: Vec<Bytes> = drain_to_vec(&table, new)+ .into_iter()+ .map(|r| r.frame)+ .collect();+ assert_eq!(+ frames,+ vec![Bytes::from_static(b"older"), Bytes::from_static(b"newer")]+ );+ }++ /// After a count change two partitions' drains can both reach one key.+ /// While one holds the key's head, the other must not take its tail.+ #[test]+ fn a_key_out_with_another_drain_is_skipped() {+ let changelog = LiveChangelog::new(ChangelogConfig::new("personhog_updates", 8));+ let table = StashTable::with_bounds(usize::MAX, usize::MAX, changelog.clone());+ let person_id = moving_person(8, 4);+ let old = partition_for_person(1, person_id, 8);+ let new = partition_for_person(1, person_id, 4);+ table.begin_stash(old);assert!(matches!(- enqueue(&table, 1, 1).await,- StashDecision::Forward+ enqueue_frame(&table, person_id, Bytes::from_static(b"head")),+ StashDecision::Stashed(_)+ ));+ let holder = table.drain_session(old).unwrap();+ let mut head = holder.take_for_attempt(usize::MAX);+ assert!(matches!(+ enqueue_frame(&table, person_id, Bytes::from_static(b"tail")),+ StashDecision::Stashed(_)));++ adopt(&changelog, 4);+ let other = table.drain_session(new).unwrap();+ assert!(other.take_for_attempt(usize::MAX).is_empty());+ assert!(!other.finish_if_settled());++ let run = head.remove(0);+ let entry = run.entries.into_iter().next().unwrap();+ holder.complete(run.key, entry.item.charge());+ let frames: Vec<Bytes> = drain_to_vec(&table, new)+ .into_iter()+ .map(|r| r.frame)+ .collect();+ assert_eq!(frames, vec![Bytes::from_static(b"tail")]);+ }++ /// A key out with another drain still needs its new partition's drain:+ /// if that attempt bounces and puts the key back, only that drain takes it.+ #[test]+ fn a_key_out_for_delivery_keeps_its_new_partition_pending() {+ let changelog = LiveChangelog::new(ChangelogConfig::new("personhog_updates", 8));+ let table = StashTable::with_bounds(usize::MAX, usize::MAX, changelog.clone());+ let person_id = moving_person(8, 4);+ let old = partition_for_person(1, person_id, 8);+ let new = partition_for_person(1, person_id, 4);+ table.begin_stash(old);+ assert!(matches!(+ enqueue(&table, person_id),+ StashDecision::Stashed(_)+ ));+ let holder = table.drain_session(old).unwrap();+ let mut head = holder.take_for_attempt(usize::MAX);++ adopt(&changelog, 4);+ assert!(table.is_pending(new));++ let run = head.remove(0);+ holder.put_back(run.key, run.entries);+ assert!(holder.take_for_attempt(usize::MAX).is_empty());+ let taker = table.drain_session(new).unwrap();+ assert_eq!(taker.take_for_attempt(usize::MAX).len(), 1);+ }++ /// A freeze opened while a drain runs belongs to the next handoff;+ /// the running drain's settle must not end it.+ #[test]+ fn a_freeze_opened_during_a_drain_survives_its_settle() {+ let table = table_over(1, usize::MAX, usize::MAX);+ table.begin_stash(0);+ let session = table.drain_session(0).unwrap();+ table.begin_stash(0);+ assert!(session.finish_if_settled());+ assert!(matches!(enqueue(&table, 1), StashDecision::Stashed(_)));}- /// Race between concurrent enqueue and drain. The `Arc<PartitionStash>`- /// + `Option<StashQueue>` design exists so that an enqueue that has- /// already cloned the Arc when the settle eviction runs sees `None`,- /// and returns `Forward` rather than pushing into a dead queue. With- /// many iterations this drives the race window repeatedly; the- /// invariant is that every request must end up either in the drained- /// batch or forwarded — never lost, never duplicated.+ /// Concurrent enqueues and a drain: every request is drained or forwarded exactly once.#[tokio::test(flavor = "multi_thread", worker_threads = 4)]async fn enqueue_and_drain_race() {for iteration in 0..200 {- let table = StashTable::with_bounds(usize::MAX, usize::MAX);- table.begin_stash(0).await;+ let table = table_over(1, usize::MAX, usize::MAX);+ table.begin_stash(0);const ENQUEUERS: usize = 8;let mut handles = Vec::with_capacity(ENQUEUERS);for i in 0..ENQUEUERS {let table = table.clone();- handles.push(tokio::spawn(- async move { enqueue(&table, 0, i as i64).await },- ));+ handles.push(tokio::spawn(async move { enqueue(&table, i as i64) }));}// Race the drain against the in-flight enqueues.let drain_table = table.clone();- let drain_handle = tokio::spawn(async move { drain_to_vec(&drain_table, 0).await });+ let drain_handle = tokio::spawn(async move { drain_to_vec(&drain_table, 0) });let drained = drain_handle.await.unwrap();let mut stashed_count = 0usize;@@ -840,15 +927,15 @@ mod tests {/// path relies on.#[tokio::test]async fn drained_request_reply_round_trips() {- let table = StashTable::with_bounds(usize::MAX, usize::MAX);- table.begin_stash(0).await;+ let table = table_over(1, usize::MAX, usize::MAX);+ table.begin_stash(0);- let rx = match enqueue(&table, 0, 1).await {+ let rx = match enqueue(&table, 1) {StashDecision::Stashed(rx) => rx,_ => panic!("expected Stashed"),};- let mut drained = drain_to_vec(&table, 0).await;+ let mut drained = drain_to_vec(&table, 0);let req = drained.remove(0);let mut response = test_response();response@@ -866,83 +953,22 @@ mod tests {/// disconnected) before drain delivered the reply, the `send` must/// return `Err`. The drain handler relies on this signal to bump the/// `personhog_router_stash_dropped_total` counter.- #[tokio::test]- async fn dropped_receiver_makes_send_fail() {- let table = StashTable::with_bounds(usize::MAX, usize::MAX);- table.begin_stash(0).await;+ #[test]+ fn dropped_receiver_makes_send_fail() {+ let table = table_over(1, usize::MAX, usize::MAX);+ table.begin_stash(0);- let rx = match enqueue(&table, 0, 1).await {+ let rx = match enqueue(&table, 1) {StashDecision::Stashed(rx) => rx,_ => panic!("expected Stashed"),};drop(rx);- let mut drained = drain_to_vec(&table, 0).await;+ let mut drained = drain_to_vec(&table, 0);let req = drained.remove(0);assert!(req.reply.send(test_response()).is_err(),"send must return Err after the receiver is dropped");}-- /// Regression for the structural race the `drained` tombstone closes:- /// a `begin_stash` that observes the post-settle `None` queue between- /// the settle tombstoning the queue and evicting the dashmap entry- /// must not initialize a new queue on the doomed `Arc`. With the- /// tombstone, `begin_stash` sees `None`, drops the lock, and retries- /// via `get_or_create` — which produces a fresh dashmap entry once- /// the eviction finishes.- ///- /// The race window is microseconds; we run many iterations to- /// exercise it. The protocol awaits the drain before issuing the next- /// `begin_stash` for a partition today, so this race cannot trigger- /// in production — the test guards against future callers that- /// might break that ordering.- #[tokio::test(flavor = "multi_thread", worker_threads = 4)]- async fn drain_does_not_orphan_concurrent_begin_stash() {- for iteration in 0..500 {- let table = StashTable::with_bounds(usize::MAX, usize::MAX);- table.begin_stash(0).await;-- // Drain in one task; concurrent begin_stash in another.- // The key invariant: after both complete, the dashmap must- // contain a live entry for the partition (because- // begin_stash was the most recent successful call), so a- // subsequent enqueue sees it and parks.- let drain_table = table.clone();- let drain_handle = tokio::spawn(async move {- drain_to_vec(&drain_table, 0).await;- });- let begin_table = table.clone();- let begin_handle = tokio::spawn(async move { begin_table.begin_stash(0).await });-- drain_handle.await.unwrap();- begin_handle.await.unwrap();-- // Only meaningful to assert when begin_stash logically ran- // *after* the settle (eviction removed the entry, begin_stash- // created a fresh one). If begin_stash logically ran first- // (observed the prior `Some` queue, was idempotent), then- // the drain legitimately drained that prior queue and a- // subsequent enqueue forwards via the direct path — that's a- // protocol-violation scenario, not a stash-module bug, and- // the routing-table layer prevents it. We accept either- // outcome here; what we *don't* accept is a non-empty- // drained queue that was just initialized by begin_stash on- // an orphaned Arc, which the tombstone prevents.- let outcome = enqueue(&table, 0, 1).await;- match outcome {- StashDecision::Stashed(_) => {- // Fresh dashmap entry exists — begin_stash set it up correctly.- }- StashDecision::Forward => {- // Begin_stash logically ran first and was idempotent;- // the drain emptied the prior queue. No orphaned Arc.- }- StashDecision::Rejected => {- panic!("unexpected Rejected on iteration {iteration}");- }- }- }- }}diff --git a/rust/personhog-router/src/main.rs b/rust/personhog-router/src/main.rsindex 3035bdee6ad4..418dc38fc082 100644--- a/rust/personhog-router/src/main.rs+++ b/rust/personhog-router/src/main.rs@@ -303,7 +303,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {.map(|address| format!("http://{address}"))}),LeaderBackendConfig {- changelog,+ changelog: changelog.clone(),timeout: config.backend_timeout(),num_channels: config.leader_channels,http2_windows: config.backend_http2_windows(),@@ -311,6 +311,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {StashTable::with_bounds(config.stash_max_messages_per_partition,config.stash_max_bytes_per_partition,+ changelog.clone(),),));diff --git a/rust/personhog-router/src/proxy.rs b/rust/personhog-router/src/proxy.rsindex c56a4928bfde..fb19255decb6 100644--- a/rust/personhog-router/src/proxy.rs+++ b/rust/personhog-router/src/proxy.rs@@ -620,17 +620,9 @@ impl RawProxyInner {Err(resp) => return (resp, None),};- let partition = leader.partition_for_person(team_id, person_id);-let _in_flight = ClientInFlightGuard::new("leader");leader- .forward_or_stash(- method,- partition,- (team_id, person_id),- parts.headers,- body_bytes,- )+ .forward_or_stash(method, (team_id, person_id), parts.headers, body_bytes).await}}diff --git a/rust/personhog-router/src/stash_handler.rs b/rust/personhog-router/src/stash_handler.rsindex fd2888b6e9ca..9d334059e638 100644--- a/rust/personhog-router/src/stash_handler.rs+++ b/rust/personhog-router/src/stash_handler.rs@@ -16,21 +16,23 @@ use crate::backend::{};use crate::grpc_http::{grpc_error_response, is_grpc_error_response};+/// How long a drain waits before looking again for a key it could not take.+const SETTLE_RETRY: Duration = Duration::from_millis(10);+/// Stash handler for the router. Reacts to handoff phase transitions:////// * `Freezing` / `Draining` / `Warming` → `begin_stash`: start (or/// re-confirm) buffering leader-path requests — writes and strong/// reads — for the partition in the shared `StashTable`. The/// routing-table layer calls `begin_stash` on every non-terminal phase-/// the router observes, so the call must be idempotent —-/// `StashTable::begin_stash` no-ops if the entry is already live. New-/// leader-path requests park in per-key queues while the handoff-/// progresses through `Freezing → Draining → Warming`.+/// the router observes, so the call must be idempotent; a repeat only+/// renews the freeze. New leader-path requests park in per-key queues+/// while the handoff progresses through `Freezing → Draining → Warming`./// * `Complete` → `drain_stash`: forward the buffered requests to the/// new owner, each to the method it arrived on, through a-/// `DrainSession`. The drain loops taking per-key front runs until it-/// observes the queue fully settled; any request that arrives during-/// the drain lands on the same queue and is picked up by a later take.+/// `DrainSession`. The drain loops taking per-key front runs until no+/// key of the partition remains; any request that arrives during the+/// drain parks behind its key and is picked up by a later take.////// Every stashed request ends in exactly one of two ways: a definitive/// outcome delivered to its client (the forwarded response, an expiry@@ -197,7 +199,7 @@ struct KeyRunOutcome {/// yet attempted. Retry accounting stays with the callers — only an/// actually-attempted head counts as a stash-path retry, so the/// never-attempted tail goes back uncounted.-async fn put_back_rest(+fn put_back_rest(session: &DrainSession,key: StashKey,head: Entry<StashedRequest>,@@ -205,7 +207,7 @@ async fn put_back_rest() {let mut entries = vec![head];entries.extend(rest);- session.put_back(key, entries).await;+ session.put_back(key, entries);}/// Forward one key's front run sequentially, recording a definitive@@ -226,7 +228,7 @@ async fn forward_key_run(let mut entries = run.entries.into_iter();while let Some(mut entry) = entries.next() {if cancel.is_cancelled() {- put_back_rest(session, run.key, entry, entries).await;+ put_back_rest(session, run.key, entry, entries);return outcome;}// Race the forward against cancellation: an in-flight call can@@ -250,7 +252,7 @@ async fn forward_key_run("reason" => "cancelled").increment(1);- put_back_rest(session, run.key, entry, entries).await;+ put_back_rest(session, run.key, entry, entries);return outcome;};match disposition {@@ -258,7 +260,7 @@ async fn forward_key_run(response,outcome: label,} => {- let size = entry.item.approximate_size();+ let charge = entry.item.charge();metrics::histogram!("personhog_router_stash_wait_duration_ms").record(entry.item.enqueued_at.elapsed().as_secs_f64() * 1000.0);if entry.item.reply.send(response).is_err() {@@ -273,7 +275,7 @@ async fn forward_key_run("outcome" => label).increment(1);- session.complete(run.key, size).await;+ session.complete(run.key, charge);outcome.completed += 1;}Disposition::Bounce(reason) => {@@ -286,7 +288,7 @@ async fn forward_key_run("reason" => reason.label()).increment(1);- put_back_rest(session, run.key, entry, entries).await;+ put_back_rest(session, run.key, entry, entries);outcome.bounced = true;return outcome;}@@ -298,7 +300,7 @@ async fn forward_key_run(#[async_trait]impl StashHandler for RouterStashHandler {fn stash_pending(&self, partition: u32) -> bool {- self.leader_backend.stash_table().has_entry(partition)+ self.leader_backend.stash_table().is_pending(partition)}async fn begin_stash(&self, partition: u32, new_owner: &str) -> CoordResult<()> {@@ -307,10 +309,7 @@ impl StashHandler for RouterStashHandler {new_owner,"beginning stash for partition handoff");- self.leader_backend- .stash_table()- .begin_stash(partition)- .await;+ self.leader_backend.stash_table().begin_stash(partition);Ok(())}@@ -320,9 +319,8 @@ impl StashHandler for RouterStashHandler {new_owner: &str,cancel: CancellationToken,) -> CoordResult<()> {- let stash_table = self.leader_backend.stash_table();- let Some(session) = stash_table.drain_session(partition) else {- // Nothing stashed — the partition never froze, or a prior+ let Some(session) = self.leader_backend.stash_table().drain_session(partition) else {+ // Nothing pending — the partition never froze, or a prior// drain already settled it.return Ok(());};@@ -344,12 +342,17 @@ impl StashHandler for RouterStashHandler {// flight, the backlog stays parked for the successor.break;}- let taken = session.take_for_attempt(self.drain_concurrency).await;+ let taken = session.take_for_attempt(self.drain_concurrency);if taken.is_empty() {- if session.finish_if_settled().await {+ if session.finish_if_settled() {break;}- // An arrival raced the settle check; take again.+ // An arrival raced the settle check, or another drain holds one of our keys.+ metrics::counter!("personhog_router_stash_settle_retries_total").increment(1);+ tokio::select! {+ _ = cancel.cancelled() => break,+ _ = tokio::time::sleep(SETTLE_RETRY) => {}+ }continue;}diff --git a/rust/personhog-router/tests/common/mod.rs b/rust/personhog-router/tests/common/mod.rsindex c22cd5e4dd44..ad50e2555c47 100644--- a/rust/personhog-router/tests/common/mod.rs+++ b/rust/personhog-router/tests/common/mod.rs@@ -740,6 +740,9 @@ pub struct TestLeaderService {/// partition fence above this one names its holder, which is the fact/// the refusal has to carry all the way back to the caller.person_fence_op: Arc<Mutex<Option<String>>>,+ /// Runs on every write before it is answered, so a test can change the+ /// router's view at an exact point between its attempts.+ on_write: Option<Arc<dyn Fn() + Send + Sync>>,}impl TestLeaderService {@@ -748,9 +751,15 @@ impl TestLeaderService {persons: DashMap::new(),fenced: Arc::new(AtomicBool::new(false)),person_fence_op: Arc::new(Mutex::new(None)),+ on_write: None,}}+ pub fn on_write(mut self, hook: impl Fn() + Send + Sync + 'static) -> Self {+ self.on_write = Some(Arc::new(hook));+ self+ }+/// Refuse every write for this person the way the leader refuses one/// held by a lifecycle operation: FAILED_PRECONDITION carrying the/// fence keys and the holding op's id.@@ -868,6 +877,9 @@ impl PersonHogLeader for TestLeaderService {request: Request<UpdatePersonPropertiesRequest>,) -> Result<Response<UpdatePersonPropertiesResponse>, Status> {require_partition_metadata(&request)?;+ if let Some(hook) = &self.on_write {+ hook();+ }let req = request.into_inner();if self.fenced.load(Ordering::SeqCst) {return Err(Status::failed_precondition(@@ -994,12 +1006,12 @@ fn make_leader_backend(leader_addr: SocketAddr, num_partitions: u32) -> Arc<Leadrouting_table,address_resolver,LeaderBackendConfig {- changelog,+ changelog: changelog.clone(),timeout: Duration::from_secs(5),num_channels: 1,http2_windows: Http2Windows::default(),},- StashTable::with_bounds(usize::MAX, usize::MAX),+ StashTable::with_bounds(usize::MAX, usize::MAX, changelog),))}@@ -1029,12 +1041,12 @@ fn make_dying_leader_backend(leader_addr: SocketAddr, num_partitions: u32) -> Arrouting_table,address_resolver,LeaderBackendConfig {- changelog,+ changelog: changelog.clone(),timeout: Duration::from_millis(200),num_channels: 1,http2_windows: Http2Windows::default(),},- StashTable::with_bounds(usize::MAX, usize::MAX),+ StashTable::with_bounds(usize::MAX, usize::MAX, changelog),))}diff --git a/rust/personhog-router/tests/h2_window_wiring.rs b/rust/personhog-router/tests/h2_window_wiring.rsindex a6dc9634c6bd..aa58c743308c 100644--- a/rust/personhog-router/tests/h2_window_wiring.rs+++ b/rust/personhog-router/tests/h2_window_wiring.rs@@ -143,12 +143,12 @@ async fn leader_channel(addr: SocketAddr, http2_windows: Http2Windows) -> ChanneArc::new(RwLock::new(routing)),Arc::new(move |_| Some(format!("http://{addr}"))),LeaderBackendConfig {- changelog,+ changelog: changelog.clone(),timeout: Duration::from_secs(5),num_channels: 1,http2_windows,},- StashTable::with_bounds(usize::MAX, usize::MAX),+ StashTable::with_bounds(usize::MAX, usize::MAX, changelog),);backend.resolve_leader_channel(0).await.unwrap()}diff --git a/rust/personhog-router/tests/stash_integration.rs b/rust/personhog-router/tests/stash_integration.rsindex bb3a8af5fc6a..c20316bf084f 100644--- a/rust/personhog-router/tests/stash_integration.rs+++ b/rust/personhog-router/tests/stash_integration.rs@@ -11,12 +11,15 @@ use std::sync::atomic::Ordering;use std::sync::Arc;use std::time::Duration;+use assignment_coordination::store::{EtcdStore, StoreConfig};use bytes::Bytes;use common::{create_test_person, start_test_leader, start_test_leader_at, TestLeaderService};use http::HeaderMap;use http_body_util::BodyExt;+use personhog_common::partitioning::partition_for_person;use personhog_coordination::changelog::LiveChangelog;use personhog_coordination::routing_table::StashHandler;+use personhog_coordination::store::PersonhogStore;use personhog_coordination::types::ChangelogConfig;use personhog_proto::personhog::types::v1::{FoldPersonDocumentRequest, FoldPersonDocumentResponse, GetPersonRequest, GetPersonResponse,@@ -46,13 +49,38 @@ fn new_test_handler(backend: Arc<LeaderBackend>) -> RouterStashHandler {RouterStashHandler::new(backend, TEST_MAX_STASH_WAIT, TEST_DRAIN_CONCURRENCY)}+/// A store under a fresh prefix, so no other test's config is visible.+async fn test_store() -> Arc<PersonhogStore> {+ let inner = EtcdStore::connect(StoreConfig {+ endpoints: vec!["http://localhost:2379".to_string()],+ prefix: format!("/stash-test-{}/", rand::random::<u64>()),+ })+ .await+ .expect("failed to connect to etcd");+ Arc::new(PersonhogStore::new(inner))+}++fn changelog() -> LiveChangelog {+ LiveChangelog::new(ChangelogConfig::new("personhog_updates", NUM_PARTITIONS))+}+/// Build a `LeaderBackend` whose routing table maps every partition at the/// given `leader_addr` and whose stash uses the supplied bounds.async fn make_backend(leader_addr: std::net::SocketAddr, stash: StashTable) -> Arc<LeaderBackend> {+ backend_with(leader_addr, changelog(), stash, NUM_PARTITIONS).await+}++/// A backend routing partitions `0..routed` to `leader_addr`, reading the count from `changelog`.+async fn backend_with(+ leader_addr: std::net::SocketAddr,+ changelog: LiveChangelog,+ stash: StashTable,+ routed: u32,+) -> Arc<LeaderBackend> {let routing_table = Arc::new(RwLock::new(HashMap::new()));{let mut table = routing_table.write().await;- for partition in 0..NUM_PARTITIONS {+ for partition in 0..routed {table.insert(partition, "leader-0".to_string());}}@@ -66,10 +94,7 @@ async fn make_backend(leader_addr: std::net::SocketAddr, stash: StashTable) -> Arouting_table,resolver,LeaderBackendConfig {- changelog: LiveChangelog::new(ChangelogConfig::new(- "personhog_updates",- NUM_PARTITIONS,- )),+ changelog,timeout: Duration::from_secs(5),num_channels: 1,http2_windows: Http2Windows::default(),@@ -109,12 +134,10 @@ async fn forward(backend: &LeaderBackend,req: UpdatePersonPropertiesRequest,) -> http::Response<BoxBody> {- let partition = backend.partition_for_person(req.team_id, req.person_id);let key = (req.team_id, req.person_id);let (response, _call_ms) = backend.forward_or_stash("UpdatePersonProperties",- partition,key,HeaderMap::new(),encode_frame(&req),@@ -169,11 +192,9 @@ async fn forward_read(buf.extend_from_slice(&(encoded.len() as u32).to_be_bytes());buf.extend(encoded);- let partition = backend.partition_for_person(team_id, person_id);let (response, _call_ms) = backend.forward_or_stash("GetPerson",- partition,(team_id, person_id),HeaderMap::new(),Bytes::from(buf),@@ -191,7 +212,7 @@ async fn request_during_stash_completes_after_drain() {let person = create_test_person();let leader_addr = start_test_leader(TestLeaderService::new().with_person(person.clone())).await;- let stash = StashTable::with_bounds(usize::MAX, usize::MAX);+ let stash = StashTable::with_bounds(usize::MAX, usize::MAX, changelog());let backend = make_backend(leader_addr, stash.clone()).await;let handler = new_test_handler(Arc::clone(&backend));@@ -241,7 +262,7 @@ async fn multiple_stashed_requests_drain_in_fifo() {let person = create_test_person();let leader_addr = start_test_leader(TestLeaderService::new().with_person(person.clone())).await;- let stash = StashTable::with_bounds(usize::MAX, usize::MAX);+ let stash = StashTable::with_bounds(usize::MAX, usize::MAX, changelog());let backend = make_backend(leader_addr, stash.clone()).await;let handler = new_test_handler(Arc::clone(&backend));@@ -299,7 +320,7 @@ async fn requests_for_unstashed_partition_forward_immediately() {let person = create_test_person();let leader_addr = start_test_leader(TestLeaderService::new().with_person(person.clone())).await;- let stash = StashTable::with_bounds(usize::MAX, usize::MAX);+ let stash = StashTable::with_bounds(usize::MAX, usize::MAX, changelog());let backend = make_backend(leader_addr, stash.clone()).await;let handler = new_test_handler(Arc::clone(&backend));@@ -331,7 +352,7 @@ async fn stash_full_returns_unavailable_via_backend() {let leader_addr = start_test_leader(TestLeaderService::new().with_person(person.clone())).await;// Tight cap: only one stashed message per partition.- let stash = StashTable::with_bounds(1, usize::MAX);+ let stash = StashTable::with_bounds(1, usize::MAX, changelog());let backend = make_backend(leader_addr, stash.clone()).await;let handler = new_test_handler(Arc::clone(&backend));@@ -366,7 +387,7 @@ async fn back_to_back_handoffs_use_fresh_queue() {let person = create_test_person();let leader_addr = start_test_leader(TestLeaderService::new().with_person(person.clone())).await;- let stash = StashTable::with_bounds(usize::MAX, usize::MAX);+ let stash = StashTable::with_bounds(usize::MAX, usize::MAX, changelog());let backend = make_backend(leader_addr, stash.clone()).await;let handler = new_test_handler(Arc::clone(&backend));@@ -428,7 +449,7 @@ async fn ordering_preserved_when_request_arrives_during_drain() {let person = create_test_person();let leader_addr = start_test_leader(TestLeaderService::new().with_person(person.clone())).await;- let stash = StashTable::with_bounds(usize::MAX, usize::MAX);+ let stash = StashTable::with_bounds(usize::MAX, usize::MAX, changelog());let backend = make_backend(leader_addr, stash.clone()).await;let handler = new_test_handler(Arc::clone(&backend));@@ -497,7 +518,7 @@ async fn stash_wait_exceeded_returns_unavailable() {let person = create_test_person();let leader_addr = start_test_leader(TestLeaderService::new().with_person(person.clone())).await;- let stash = StashTable::with_bounds(usize::MAX, usize::MAX);+ let stash = StashTable::with_bounds(usize::MAX, usize::MAX, changelog());let backend = make_backend(leader_addr, stash.clone()).await;// Tight deadline — any request that ages past ~50ms in the stash// gets failed fast on drain.@@ -537,10 +558,9 @@ async fn stash_wait_exceeded_returns_unavailable() {async fn a_semantic_refusal_passes_through_instead_of_bouncing() {let person = create_test_person();let leader_addr = start_test_leader(TestLeaderService::new().with_person(person.clone())).await;- let stash = StashTable::with_bounds(usize::MAX, usize::MAX);+ let stash = StashTable::with_bounds(usize::MAX, usize::MAX, changelog());let backend = make_backend(leader_addr, stash).await;- let partition = backend.partition_for_person(person.team_id, person.id);let req = FoldPersonDocumentRequest {team_id: person.team_id,person_id: person.id,@@ -555,7 +575,6 @@ async fn a_semantic_refusal_passes_through_instead_of_bouncing() {let (response, _call_ms) = backend.forward_or_stash("FoldPersonDocument",- partition,(person.team_id, person.id),HeaderMap::new(),encode_frame(&req),@@ -588,7 +607,7 @@ async fn fence_bounce_parks_requests_until_a_later_drain_delivers() {let fence = service.fence_flag();let leader_addr = start_test_leader(service).await;- let stash = StashTable::with_bounds(usize::MAX, usize::MAX);+ let stash = StashTable::with_bounds(usize::MAX, usize::MAX, changelog());let backend = make_backend(leader_addr, stash.clone()).await;// Generous deadline so the deadline path can never produce a client// error and mask a bounce that wrongly surfaced one.@@ -645,7 +664,7 @@ async fn transport_failure_parks_requests_until_a_later_drain_delivers() {let leader_addr = reserved.local_addr().unwrap();drop(reserved);- let stash = StashTable::with_bounds(usize::MAX, usize::MAX);+ let stash = StashTable::with_bounds(usize::MAX, usize::MAX, changelog());let backend = make_backend(leader_addr, stash.clone()).await;// Generous deadline so the deadline path can never produce a client// error and mask a bounce that wrongly surfaced one.@@ -704,7 +723,7 @@ async fn a_live_write_rides_out_a_fence_via_router_retry() {let fence = service.fence_flag();let leader_addr = start_test_leader(service).await;- let stash = StashTable::with_bounds(usize::MAX, usize::MAX);+ let stash = StashTable::with_bounds(usize::MAX, usize::MAX, changelog());let backend = make_backend(leader_addr, stash.clone()).await;// The fence clears mid-retry, as it does in production once the@@ -739,7 +758,7 @@ async fn a_live_write_past_the_bounce_budget_fails_unavailable() {).await;- let stash = StashTable::with_bounds(usize::MAX, usize::MAX);+ let stash = StashTable::with_bounds(usize::MAX, usize::MAX, changelog());let backend = make_backend(leader_addr, stash.clone()).await;let req = mk_request(person.team_id, person.id, "[email protected]");@@ -768,7 +787,7 @@ async fn a_bouncing_live_write_parks_once_the_stash_opens() {let leader_addr = reserved.local_addr().unwrap();drop(reserved);- let stash = StashTable::with_bounds(usize::MAX, usize::MAX);+ let stash = StashTable::with_bounds(usize::MAX, usize::MAX, changelog());let backend = make_backend(leader_addr, stash.clone()).await;let handler = RouterStashHandler::new(Arc::clone(&backend), Duration::from_secs(60), 4);@@ -811,7 +830,7 @@ async fn stashed_strong_read_observes_stashed_write() {let person = create_test_person();let leader_addr = start_test_leader(TestLeaderService::new().with_person(person.clone())).await;- let stash = StashTable::with_bounds(usize::MAX, usize::MAX);+ let stash = StashTable::with_bounds(usize::MAX, usize::MAX, changelog());let backend = make_backend(leader_addr, stash.clone()).await;let handler = new_test_handler(Arc::clone(&backend));@@ -861,7 +880,7 @@ async fn stashed_read_past_deadline_returns_unavailable() {let person = create_test_person();let leader_addr = start_test_leader(TestLeaderService::new().with_person(person.clone())).await;- let stash = StashTable::with_bounds(usize::MAX, usize::MAX);+ let stash = StashTable::with_bounds(usize::MAX, usize::MAX, changelog());let backend = make_backend(leader_addr, stash.clone()).await;let handler = RouterStashHandler::new(Arc::clone(&backend), Duration::from_millis(50), 4);@@ -903,7 +922,7 @@ async fn drain_converges_with_concurrent_arrivals() {let person = create_test_person();let leader_addr = start_test_leader(TestLeaderService::new().with_person(person.clone())).await;- let stash = StashTable::with_bounds(usize::MAX, usize::MAX);+ let stash = StashTable::with_bounds(usize::MAX, usize::MAX, changelog());let backend = make_backend(leader_addr, stash.clone()).await;let handler = Arc::new(new_test_handler(Arc::clone(&backend)));@@ -983,7 +1002,7 @@ async fn drain_cancellation_returns_promptly_and_puts_the_entry_back() {}});- let stash = StashTable::with_bounds(usize::MAX, usize::MAX);+ let stash = StashTable::with_bounds(usize::MAX, usize::MAX, changelog());let backend = make_backend(leader_addr, stash).await;let handler = new_test_handler(Arc::clone(&backend));@@ -1022,3 +1041,152 @@ async fn drain_cancellation_returns_promptly_and_puts_the_entry_back() {"the abandoned entry must be put back for the next drain");}++/// A person whose partition differs between the current and the grown count.+fn moving_person() -> i64 {+ (1..)+ .find(|id| {+ partition_for_person(1, *id, NUM_PARTITIONS)+ != partition_for_person(1, *id, NUM_PARTITIONS * 2)+ })+ .unwrap()+}++fn grown_config() -> ChangelogConfig {+ ChangelogConfig {+ epoch: 2,+ ..ChangelogConfig::new("personhog_updates_v2", NUM_PARTITIONS * 2)+ }+}++/// The config watch adopts a flip from etcd, and the new partition's drain+/// delivers a parked key once the old partition's drain gives it back.+#[tokio::test]+async fn a_flip_in_etcd_moves_a_parked_key_to_its_new_partition() {+ let store = test_store().await;+ assert!(store+ .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")+ .await+ .unwrap());+ let person = Person {+ id: moving_person(),+ ..create_test_person()+ };+ let leader_addr = start_test_leader(TestLeaderService::new().with_person(person.clone())).await;+ let changelog = changelog();+ let stash = StashTable::with_bounds(usize::MAX, usize::MAX, changelog.clone());+ let backend = backend_with(+ leader_addr,+ changelog.clone(),+ stash.clone(),+ NUM_PARTITIONS * 2,+ )+ .await;+ let handler = RouterStashHandler::new(Arc::clone(&backend), Duration::from_secs(60), 4);+ let old_partition = partition_for_person(1, person.id, NUM_PARTITIONS);+ let new_partition = partition_for_person(1, person.id, NUM_PARTITIONS * 2);++ stash.begin_stash(old_partition);+ let backend_for_call = Arc::clone(&backend);+ let req = mk_request(1, person.id, "[email protected]");+ let pending = tokio::spawn(async move { forward(&backend_for_call, req).await });+ let holder = stash.drain_session(old_partition).unwrap();+ let mut held = tokio::time::timeout(Duration::from_secs(5), async {+ loop {+ let taken = holder.take_for_attempt(usize::MAX);+ if !taken.is_empty() {+ return taken;+ }+ tokio::time::sleep(Duration::from_millis(10)).await;+ }+ })+ .await+ .expect("the old partition's drain takes the parked request");++ store.put_changelog_config(&grown_config()).await.unwrap();+ let cancel = CancellationToken::new();+ tokio::spawn(personhog_coordination::changelog::keep_changelog_live(+ Arc::clone(&store),+ changelog.clone(),+ cancel.clone(),+ ));+ tokio::time::timeout(Duration::from_secs(5), async {+ while !stash.is_pending(new_partition) {+ tokio::time::sleep(Duration::from_millis(10)).await;+ }+ })+ .await+ .expect("the parked key belongs to its partition under the adopted count");++ let mut drain = tokio::spawn(async move {+ handler+ .drain_stash(new_partition, "leader-0", CancellationToken::new())+ .await+ });+ assert!(+ tokio::time::timeout(Duration::from_millis(200), &mut drain)+ .await+ .is_err(),+ "the new drain waits while the old one holds the key"+ );+ let run = held.remove(0);+ holder.put_back(run.key, run.entries);++ tokio::time::timeout(Duration::from_secs(5), drain)+ .await+ .expect("the drain settles once it delivers the key")+ .expect("drain task must not panic")+ .expect("drain succeeds");+ decode_response::<UpdatePersonPropertiesResponse>(pending.await.unwrap())+ .await+ .expect("the parked write is delivered through the new partition");+ cancel.cancel();+}++/// A retry after a bounce recomputes the partition, so it sees the freeze+/// on the key's new partition instead of forwarding under the old count.+#[tokio::test]+async fn a_retry_after_a_bounce_routes_by_the_adopted_count() {+ let person_id = moving_person();+ let new_partition = partition_for_person(1, person_id, NUM_PARTITIONS * 2);+ let changelog = changelog();+ let stash = StashTable::with_bounds(usize::MAX, usize::MAX, changelog.clone());+ let flipped = Arc::new(std::sync::atomic::AtomicBool::new(false));+ let service = TestLeaderService::new().fenced().on_write({+ let (changelog, stash) = (changelog.clone(), stash.clone());+ move || {+ if !flipped.swap(true, Ordering::SeqCst) {+ changelog.adopt(&grown_config());+ stash.begin_stash(new_partition);+ }+ }+ });+ let leader_addr = start_test_leader(service).await;+ let backend = backend_with(leader_addr, changelog, stash.clone(), NUM_PARTITIONS * 2).await;++ let req = mk_request(1, person_id, "[email protected]");+ let pending = tokio::spawn(async move { forward(&backend, req).await });+ let session = tokio::time::timeout(Duration::from_secs(5), async {+ loop {+ if let Some(session) = stash.drain_session(new_partition) {+ return session;+ }+ tokio::time::sleep(Duration::from_millis(10)).await;+ }+ })+ .await+ .expect("the first attempt freezes the new partition");+ let taken = tokio::time::timeout(Duration::from_secs(5), async {+ loop {+ let taken = session.take_for_attempt(usize::MAX);+ if !taken.is_empty() {+ return taken;+ }+ tokio::time::sleep(Duration::from_millis(10)).await;+ }+ })+ .await+ .expect("the retry parks behind the new partition's freeze");+ assert_eq!(taken[0].key, (1, person_id));+ assert!(!pending.is_finished());+}