Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(personhog): park router requests per key during handoffs (#119810)

Zach Brown49669cadf9fb
diff --git a/rust/common/keyed-stash/src/lib.rs b/rust/common/keyed-stash/src/lib.rs
index ff3b12f9b70f..7c296f9ec09d 100644
--- a/rust/common/keyed-stash/src/lib.rs
+++ b/rust/common/keyed-stash/src/lib.rs
@@ -81,8 +81,12 @@ impl<K: Eq + Hash + Clone, T> KeyedStash<K, T> {
/// known. Sending new work directly while this holds would let it
/// race ahead of the key's backlog.
pub fn is_deferring(&self, key: &K) -> bool {
+ self.has_in_attempt(key) || self.queues.get(key).is_some_and(|q| !q.is_empty())
+ }
+
+ /// Whether `key` has entries taken for an attempt and not yet completed or put back.
+ pub fn has_in_attempt(&self, key: &K) -> bool {
self.in_attempt.get(key).is_some_and(|n| *n > 0)
- || self.queues.get(key).is_some_and(|q| !q.is_empty())
}
/// Take up to `max` entries from the front of `key`'s queue for a
@@ -161,6 +165,12 @@ impl<K: Eq + Hash + Clone, T> KeyedStash<K, T> {
self.queues.keys()
}
+ /// Keys for which [`is_deferring`](Self::is_deferring) holds; a key
+ /// with both queued and in-attempt entries may appear twice.
+ pub fn deferring_keys(&self) -> impl Iterator<Item = &K> {
+ self.queues.keys().chain(self.in_attempt.keys())
+ }
+
/// Total queued entries across all keys (in-attempt entries are not
/// queued).
pub fn len(&self) -> usize {
@@ -174,6 +184,8 @@ impl<K: Eq + Hash + Clone, T> KeyedStash<K, T> {
#[cfg(test)]
mod tests {
+ use std::collections::HashSet;
+
use super::*;
use proptest::prelude::*;
@@ -270,6 +282,10 @@ mod tests {
q.put_back(&key, taken);
}
}
+ let deferring: HashSet<u8> = q.deferring_keys().copied().collect();
+ for k in 0u8..4 {
+ prop_assert_eq!(deferring.contains(&k), q.is_deferring(&k));
+ }
}
// Per-key delivery order == admission order.
diff --git a/rust/personhog-common/src/partitioning.rs b/rust/personhog-common/src/partitioning.rs
index f116990c56fd..8e3460dfcb93 100644
--- a/rust/personhog-common/src/partitioning.rs
+++ b/rust/personhog-common/src/partitioning.rs
@@ -8,6 +8,12 @@
/// matches messages produced with the same key by any standard Kafka client.
pub fn partition_for_person(team_id: i64, person_id: i64, num_partitions: u32) -> u32 {
assert!(num_partitions > 0, "num_partitions must be > 0");
+ person_hash(team_id, person_id) % num_partitions
+}
+
+/// The non-negative hash `partition_for_person` reduces modulo the partition
+/// count, for callers that ask the same key's partition under several counts.
+pub fn person_hash(team_id: i64, person_id: i64) -> u32 {
// i64 max string length is 20 chars. Two i64s + ':' = 41 bytes max.
let mut buf = [0u8; 41];
let len = {
@@ -16,16 +22,19 @@ pub fn partition_for_person(team_id: i64, person_id: i64, num_partitions: u32) -
write!(cursor, "{team_id}:{person_id}").unwrap();
cursor.position() as usize
};
- partition_for_key(&buf[..len], num_partitions)
+ key_hash(&buf[..len])
}
/// Kafka's default partitioner for a keyed record, which Node's
/// `murmur2Partition` and librdkafka's `murmur2_random` both reproduce.
pub fn partition_for_key(key: &[u8], num_partitions: u32) -> u32 {
assert!(num_partitions > 0, "num_partitions must be > 0");
+ key_hash(key) % num_partitions
+}
+
+fn key_hash(key: &[u8]) -> u32 {
// Kafka's toPositive: hash & 0x7fffffff
- let positive = (kafka_murmur2(key) & 0x7fffffff) as u32;
- positive % num_partitions
+ (kafka_murmur2(key) & 0x7fffffff) as u32
}
/// Kafka-compatible murmur2 hash.
diff --git a/rust/personhog-coordination/README.md b/rust/personhog-coordination/README.md
index 0adb6875a8da..20dd2ff94351 100644
--- a/rust/personhog-coordination/README.md
+++ b/rust/personhog-coordination/README.md
@@ -127,6 +127,8 @@ Several failure modes are handled explicitly:
`config/changelog` (`{topic, partitions, epoch, writer_group, draining}`) is the one live object every participant reads. Every service loads it at boot; the router and the identity service also follow it through a watch, so the partition count they hash under is the live one.
+Routers park requests per key, so a key's requests keep one queue when the partition count changes: a request goes straight to a leader only when its key has nothing parked or in flight and its partition is not stashing, and the live count decides only which partition's drain takes a key. A drain takes the keys that map to its partition under the count the router has adopted.
+
## Modules
| Module | Responsibility |
diff --git a/rust/personhog-coordination/src/routing_table.rs b/rust/personhog-coordination/src/routing_table.rs
index 0ef373ede5e3..ae5e1eec84ae 100644
--- a/rust/personhog-coordination/src/routing_table.rs
+++ b/rust/personhog-coordination/src/routing_table.rs
@@ -42,17 +42,8 @@ pub trait StashHandler: Send + Sync {
cancel: CancellationToken,
) -> Result<()>;
- /// Whether the partition's stash lifecycle is still open — an entry
- /// exists, holding parked requests or an unflushed (possibly empty)
- /// stash window. This is the authority the reconcile pass consults
- /// before re-requesting a drain: a settled partition has no entry
- /// (the drain's final act evicts it), while a drain that yielded
- /// early leaves the entry and its backlog behind, so re-requesting
- /// is exactly the retry mechanism. Entry existence — not backlog
- /// depth — is the correct predicate, because a completed handoff
- /// with an empty stash window still needs one drain to settle and
- /// evict the entry; gating on parked requests would leave that
- /// window open forever.
+ /// Whether the partition still needs a drain: it is stashing, or parked
+ /// requests map to it. The reconcile pass re-requests a drain while this holds.
///
/// Defaults to `true`, which preserves the always-request behavior
/// for implementations that don't track stash state.
@@ -68,8 +59,8 @@ pub trait StashHandler: Send + Sync {
/// on the watch loop, where it would gate freeze acks and routing updates
/// for every partition. Lanes serialize drains per partition: requesting a
/// drain toward a different target supersedes the one in flight (its token
-/// is cancelled and it yields at a batch boundary, leaving the stash entry
-/// live), and the new drain starts only after the old one has fully
+/// is cancelled and it yields at a batch boundary, leaving its backlog
+/// parked), and the new drain starts only after the old one has fully
/// stopped, so two drains can never interleave batches toward different
/// targets. A request toward the same target is absorbed by the running
/// drain, whose loop already covers later arrivals. Observing a
@@ -115,8 +106,8 @@ impl DrainLanes {
let task_token = token.clone();
let mut lanes = self.lanes.lock().expect("drain lanes lock poisoned");
// A drain already running toward the same target covers everything
- // this request would: its loop keeps taking runs until it observes
- // the queue fully settled, including arrivals after this point.
+ // this request would: its loop keeps taking runs until no key of
+ // the partition remains, including arrivals after this point.
// Restarting it would only churn — cancel, put back, re-take the
// same backlog — for no coverage gain.
if let Some(prev) = lanes.get(&partition) {
@@ -979,7 +970,7 @@ impl RoutingTable {
/// One level-triggered convergence pass over a fresh snapshot: the
/// router's equivalent of the coordinator's reconcile tick and the
/// pod's fresh-read convergence. Every action here is idempotent
- /// against the event path — `begin_stash` no-ops on a live entry,
+ /// against the event path — `begin_stash` renews a freeze,
/// freeze acks are id-correlated, table writes are last-writer-wins
/// from the authority, and drain requests are absorbed by a running
/// same-target lane.
@@ -1046,8 +1037,8 @@ impl RoutingTable {
.write()
.await
.insert(handoff.partition, handoff.new_owner.clone());
- // Only request a drain while the stash lifecycle is
- // open. A settled partition has no entry, and its
+ // Only request a drain while the partition is
+ // pending. A settled partition is not, and its
// finished lane never absorbs (a finished drain may
// have yielded with backlog), so an unconditional
// request here would respawn a no-op drain every
diff --git a/rust/personhog-router/Cargo.toml b/rust/personhog-router/Cargo.toml
index e34fa205f36b..85dced4338cf 100644
--- a/rust/personhog-router/Cargo.toml
+++ b/rust/personhog-router/Cargo.toml
@@ -38,6 +38,7 @@ tracing = { workspace = true }
tracing-subscriber = { workspace = true }
[dev-dependencies]
+personhog-coordination = { path = "../personhog-coordination", features = ["test-support"] }
# Test-only: the store layer emits the etcd payload histogram from both
# this binary and every binary on the shared recorder, so the two bucket
# ladders have to match or the series cannot be aggregated across jobs.
diff --git a/rust/personhog-router/src/backend/leader.rs b/rust/personhog-router/src/backend/leader.rs
index 2b408cbadd73..eff032c33abe 100644
--- a/rust/personhog-router/src/backend/leader.rs
+++ b/rust/personhog-router/src/backend/leader.rs
@@ -15,11 +15,6 @@ use tonic::{Code, Status};
use tower::{Service, ServiceExt};
use personhog_common::grpc::{current_client_name, SEMANTIC_REFUSAL_METADATA_KEY};
-
-/// Set by the leader on writes refused for a lifecycle hold. Mirrored so
-/// the router does not depend on the leader crate.
-const FENCED_METADATA_KEY: &str = "x-person-fenced";
-const FENCED_OP_ID_METADATA_KEY: &str = "x-person-fenced-op-id";
use personhog_common::partitioning::partition_for_person;
use personhog_coordination::changelog::LiveChangelog;
@@ -27,6 +22,11 @@ use super::stash::{StashDecision, StashTable};
use crate::config::Http2Windows;
use crate::grpc_http::{grpc_error_response, grpc_status_code};
+/// Set by the leader on writes refused for a lifecycle hold. Mirrored so
+/// the router does not depend on the leader crate.
+const FENCED_METADATA_KEY: &str = "x-person-fenced";
+const FENCED_OP_ID_METADATA_KEY: &str = "x-person-fenced-op-id";
+
pub type AddressResolver = Arc<dyn Fn(&str) -> Option<String> + Send + Sync>;
/// gRPC path prefix for the leader service; raw forwards target
@@ -148,9 +148,7 @@ pub struct LeaderBackend {
/// Resolves pod_name → gRPC address.
address_resolver: AddressResolver,
config: LeaderBackendConfig,
- /// Per-partition stash queue used to buffer writes during partition
- /// handoffs. Consulted before every write; normal operation has no stash
- /// entries and hits the dashmap miss-path once per request.
+ /// Parks leader-path requests during partition handoffs; lock-free when nothing is parked.
stash: StashTable,
}
@@ -426,7 +424,6 @@ impl LeaderBackend {
pub async fn forward_or_stash(
&self,
method: &'static str,
- partition: u32,
key: (i64, i64),
headers: HeaderMap,
frame: Bytes,
@@ -436,15 +433,20 @@ impl LeaderBackend {
// needs to treat a replay as at-least-once.
let mut possibly_applied = false;
loop {
+ // Per attempt: a bounce may be the leader refusing a stale modulus.
+ let partition = self.partition_for_person(key.0, key.1);
// The stash module emits its own enqueued/rejected counters
// at the source; we don't double-count here. It borrows the
// frame and headers, cloning only when the request actually
// parks, so the steady-state forward path copies nothing.
- match self
- .stash
- .enqueue_or_forward(partition, method, &frame, &headers, key, possibly_applied)
- .await
- {
+ match self.stash.enqueue_or_forward(
+ method,
+ &frame,
+ &headers,
+ key,
+ partition,
+ possibly_applied,
+ ) {
StashDecision::Stashed(rx) => {
let response = rx.await.unwrap_or_else(|_| {
grpc_error_response(
@@ -575,7 +577,7 @@ mod tests {
num_channels,
..test_config(8)
},
- StashTable::with_bounds(usize::MAX, usize::MAX),
+ StashTable::with_bounds(usize::MAX, usize::MAX, changelog(8)),
);
let partition = backend.partition_for_person(1, 42);
routing_table
@@ -601,7 +603,7 @@ mod tests {
routing_table,
resolver,
test_config(8),
- StashTable::with_bounds(usize::MAX, usize::MAX),
+ StashTable::with_bounds(usize::MAX, usize::MAX, changelog(8)),
);
let partition = backend.partition_for_person(1, 42);
@@ -623,14 +625,12 @@ mod tests {
routing_table,
resolver,
test_config(8),
- StashTable::with_bounds(usize::MAX, usize::MAX),
+ StashTable::with_bounds(usize::MAX, usize::MAX, changelog(8)),
);
- let partition = backend.partition_for_person(1, 42);
let (response, call_ms) = backend
.forward_or_stash(
"UpdatePersonProperties",
- partition,
(1, 42),
HeaderMap::new(),
Bytes::from_static(b"x"),
@@ -650,7 +650,7 @@ mod tests {
Arc::clone(&routing_table),
Arc::new(|_| None), // resolver returns None
test_config(8),
- StashTable::with_bounds(usize::MAX, usize::MAX),
+ StashTable::with_bounds(usize::MAX, usize::MAX, changelog(8)),
);
let partition = backend.partition_for_person(1, 42);
@@ -673,7 +673,7 @@ mod tests {
Arc::clone(&routing_table),
resolver,
test_config(8),
- StashTable::with_bounds(usize::MAX, usize::MAX),
+ StashTable::with_bounds(usize::MAX, usize::MAX, changelog(8)),
);
let partition = backend.partition_for_person(1, 42);
@@ -762,7 +762,7 @@ mod tests {
let routing_table = Arc::new(RwLock::new(HashMap::new()));
let resolver: AddressResolver = Arc::new(|_| Some("http://localhost:50053".to_string()));
// `max_messages = 0` rejects any enqueue once the stash is open.
- let stash = StashTable::with_bounds(0, usize::MAX);
+ let stash = StashTable::with_bounds(0, usize::MAX, changelog(8));
let backend = LeaderBackend::new(
Arc::clone(&routing_table),
resolver,
@@ -773,12 +773,11 @@ mod tests {
// Determine which partition this request lands on, then open the
// stash for that exact partition.
let partition = backend.partition_for_person(1, 42);
- stash.begin_stash(partition).await;
+ stash.begin_stash(partition);
let (response, call_ms) = backend
.forward_or_stash(
"UpdatePersonProperties",
- partition,
(1, 42),
HeaderMap::new(),
Bytes::new(),
diff --git a/rust/personhog-router/src/backend/stash.rs b/rust/personhog-router/src/backend/stash.rs
index dd7843318ca0..0490785bf199 100644
--- a/rust/personhog-router/src/backend/stash.rs
+++ b/rust/personhog-router/src/backend/stash.rs
@@ -1,4 +1,4 @@
-//! Per-partition stash for the leader path.
+//! Per-key stash for the leader path.
//!
//! During a partition handoff, the coordinator advances the handoff state
//! through `Freezing → Draining → Warming → Complete`. From the moment
@@ -14,25 +14,23 @@
//! (per-key FIFO) drains after it, instead of racing ahead to the old
//! owner's frozen cache.
//!
-//! Internally each partition holds a [`KeyedStash`] keyed by
-//! `(team_id, person_id)` — the leader's per-person serialization
-//! boundary. Entries are seq-stamped at admission and leave the stash
-//! only through a definitive outcome (delivered, or failed in a way the
-//! client was told about); an attempt that cannot finish — the target
-//! bounced it, the drain was paused or superseded — puts its entries
-//! back at their sequence positions, so a retry can never be overtaken
-//! by newer same-key work. Draining runs through a [`DrainSession`],
-//! which owns the take → complete/put-back lifecycle and the
-//! empty-queue eviction handshake.
-
-use std::sync::Arc;
+//! Requests park per person `(team_id, person_id)`, so a person keeps one queue
+//! whatever the partition count does; the count only decides which drain takes it.
+//!
+//! An entry leaves the stash only with a definitive outcome. An attempt that
+//! cannot finish puts its entries back in order, so a retry is never overtaken.
+
+use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
+use std::sync::{Arc, Mutex, MutexGuard};
use std::time::Instant;
use bytes::Bytes;
use dashmap::DashMap;
use http::HeaderMap;
use keyed_stash::{Entry, KeyedStash};
-use tokio::sync::{oneshot, Mutex};
+use personhog_common::partitioning::person_hash;
+use personhog_coordination::changelog::LiveChangelog;
+use tokio::sync::oneshot;
use tonic::body::BoxBody;
/// The leader's per-person serialization key: `(team_id, person_id)`.
@@ -48,6 +46,9 @@ pub type StashKey = (i64, i64);
/// approximation, not a user-tunable knob.
const PER_REQUEST_OVERHEAD: usize = 64;
+/// Shards of the key space, fixed so a key's shard never depends on the partition count.
+const SHARDS: u32 = 64;
+
/// Why the stash rejected a request. Used for metric labelling.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum RejectCause {
@@ -83,9 +84,6 @@ pub struct StashedRequest {
/// The client's request headers, forwarded verbatim (the router stamps
/// `x-partition` at forward time).
pub headers: HeaderMap,
- /// The leader's per-person serialization key, used to preserve
- /// per-key ordering during drain.
- pub key: StashKey,
pub reply: oneshot::Sender<http::Response<BoxBody>>,
/// Wall-clock time the request was enqueued. Used to enforce the
/// per-request drain deadline and to record stash-wait histograms,
@@ -99,70 +97,28 @@ pub struct StashedRequest {
/// contract in `personhog-leader`'s README — and is counted so
/// operators can see how often it happens.
pub possibly_applied: bool,
+ /// The partition whose bounds admission charged.
+ charged_to: u32,
}
-impl StashedRequest {
- /// The size this request counts against the byte bound. Callers
- /// recording a definitive outcome pass this back to
- /// [`DrainSession::complete`] so the bound releases exactly what
- /// admission charged.
- pub fn approximate_size(&self) -> usize {
- approximate_size(&self.frame)
- }
-}
-
-/// Inner keyed queue plus running admission totals. `messages` and
-/// `bytes` count entries from admission until a definitive outcome —
-/// including entries taken for an in-flight attempt, which still hold
-/// their memory — so the bounds reflect what the router is actually
-/// holding, not just what is queued.
-struct StashQueue {
- queue: KeyedStash<StashKey, StashedRequest>,
- messages: usize,
+/// What admission charged a request, released by its definitive outcome.
+#[derive(Debug, Clone, Copy)]
+pub struct Charge {
+ partition: u32,
bytes: usize,
}
-impl StashQueue {
- fn new() -> Self {
- Self {
- queue: KeyedStash::new(),
- messages: 0,
- bytes: 0,
- }
- }
-}
-
-struct PartitionStash {
- max_messages: usize,
- max_bytes: usize,
- /// `Some(queue)` while alive; `None` once a drain session has both
- /// observed the queue fully settled and evicted this entry from the
- /// dashmap — both inside the same critical section. The `None` state
- /// therefore doubles as a tombstone: a concurrent `begin_stash` that
- /// races the eviction and observes `None` knows the dashmap entry is
- /// already gone and that the next `get_or_create` will produce a
- /// fresh entry rather than re-grab this doomed `Arc`.
- queue: Mutex<Option<StashQueue>>,
-}
-
-impl PartitionStash {
- fn new(max_messages: usize, max_bytes: usize) -> Self {
- Self {
- max_messages,
- max_bytes,
- // Initialize to `Some` so a freshly-created entry is
- // immediately ready for enqueues. A `None` queue
- // unambiguously signals "drained" — no other origin
- // produces it.
- queue: Mutex::new(Some(StashQueue::new())),
+impl StashedRequest {
+ /// Passed back to [`DrainSession::complete`] so the bounds release what admission charged.
+ pub fn charge(&self) -> Charge {
+ Charge {
+ partition: self.charged_to,
+ bytes: approximate_size(&self.frame),
}
}
}
-/// Outcome of an `enqueue_or_forward` call. `Forward` means "not frozen,
-/// route normally"; `Stashed` means the request has been enqueued and the
-/// caller should await the receiver for its reply; `Rejected` means the
-/// stash is full (either too many messages or too many bytes).
+/// `Forward`: route normally. `Stashed`: await the receiver. `Rejected`: the bounds are full.
pub enum StashDecision {
Forward,
Stashed(oneshot::Receiver<http::Response<BoxBody>>),
@@ -177,154 +133,218 @@ pub struct TakenKeyRun {
pub entries: Vec<Entry<StashedRequest>>,
}
+/// A parked key with its hash, so its partition under any count is one modulo.
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
+struct ParkedKey {
+ key: StashKey,
+ hash: u32,
+}
+
+impl ParkedKey {
+ fn of(key: StashKey) -> Self {
+ Self {
+ key,
+ hash: person_hash(key.0, key.1),
+ }
+ }
+}
+
+struct Shard {
+ queue: Mutex<KeyedStash<ParkedKey, StashedRequest>>,
+ /// Entries from admission until a definitive outcome; zero lets a request skip the lock.
+ held: AtomicUsize,
+}
+
+impl Shard {
+ fn lock(&self) -> MutexGuard<'_, KeyedStash<ParkedKey, StashedRequest>> {
+ self.queue.lock().expect("stash shard lock poisoned")
+ }
+}
+
+/// Admission totals for one partition, counting every entry until its outcome.
+#[derive(Default)]
+struct Usage {
+ messages: usize,
+ bytes: usize,
+}
+
+struct Inner {
+ shards: Vec<Shard>,
+ /// Stashing partitions, each with the generation of the freeze that opened it.
+ stashing: DashMap<u32, u64>,
+ next_generation: AtomicU64,
+ usage: DashMap<u32, Usage>,
+ max_messages: usize,
+ max_bytes: usize,
+ changelog: LiveChangelog,
+}
+
+impl Inner {
+ fn shard_of(&self, key: &ParkedKey) -> &Shard {
+ &self.shards[(key.hash % SHARDS) as usize]
+ }
+
+ fn partition_of(&self, key: &ParkedKey) -> u32 {
+ key.hash % self.changelog.partitions()
+ }
+
+ /// Whether any key that maps to `partition` has parked or in-flight
+ /// requests, locking one shard at a time.
+ fn holds_keys_of(&self, partition: u32) -> bool {
+ self.shards.iter().any(|shard| {
+ shard.held.load(Ordering::Acquire) > 0
+ && shard
+ .lock()
+ .deferring_keys()
+ .any(|key| self.partition_of(key) == partition)
+ })
+ }
+
+ fn release(&self, shard: &Shard, charge: Charge) {
+ shard.held.fetch_sub(1, Ordering::AcqRel);
+ match self.usage.entry(charge.partition) {
+ dashmap::Entry::Occupied(mut usage) => {
+ let usage_ref = usage.get_mut();
+ usage_ref.messages -= 1;
+ usage_ref.bytes = usage_ref.bytes.saturating_sub(charge.bytes);
+ if usage_ref.messages == 0 {
+ usage.remove();
+ }
+ metrics::gauge!("personhog_router_stash_queued_messages").decrement(1.0);
+ metrics::gauge!("personhog_router_stash_queued_bytes")
+ .decrement(charge.bytes as f64);
+ }
+ dashmap::Entry::Vacant(_) => debug_assert!(false, "released a charge never admitted"),
+ }
+ }
+}
+
/// Shared stash table. Cheap to clone (holds an `Arc`).
#[derive(Clone)]
pub struct StashTable {
- inner: Arc<DashMap<u32, Arc<PartitionStash>>>,
- max_messages: usize,
- max_bytes: usize,
+ inner: Arc<Inner>,
}
impl StashTable {
- pub fn with_bounds(max_messages: usize, max_bytes: usize) -> Self {
+ pub fn with_bounds(max_messages: usize, max_bytes: usize, changelog: LiveChangelog) -> Self {
+ let shards = (0..SHARDS)
+ .map(|_| Shard {
+ queue: Mutex::new(KeyedStash::new()),
+ held: AtomicUsize::new(0),
+ })
+ .collect();
Self {
- inner: Arc::new(DashMap::new()),
- max_messages,
- max_bytes,
+ inner: Arc::new(Inner {
+ shards,
+ stashing: DashMap::new(),
+ next_generation: AtomicU64::new(0),
+ usage: DashMap::new(),
+ max_messages,
+ max_bytes,
+ changelog,
+ }),
}
}
- /// Whether the partition currently has a stash entry — a live
- /// window, or backlog a yielded drain left parked. The entry only
- /// disappears through drain's settle-and-evict, so its existence is
- /// exactly "the stash lifecycle for this partition is not closed".
- pub fn has_entry(&self, partition: u32) -> bool {
- self.inner.contains_key(&partition)
+ /// Whether `partition` still needs a drain: it is stashing, or a key
+ /// that maps to it under the live count has parked or in-flight requests.
+ pub fn is_pending(&self, partition: u32) -> bool {
+ self.inner.stashing.contains_key(&partition) || self.inner.holds_keys_of(partition)
}
- fn get_or_create(&self, partition: u32) -> Arc<PartitionStash> {
- self.inner
- .entry(partition)
- .or_insert_with(|| Arc::new(PartitionStash::new(self.max_messages, self.max_bytes)))
- .clone()
+ /// Begin buffering requests for `partition`. Each call opens a new
+ /// generation, so a drain that started before it cannot close it.
+ pub fn begin_stash(&self, partition: u32) {
+ let generation = self.inner.next_generation.fetch_add(1, Ordering::Relaxed);
+ self.inner.stashing.insert(partition, generation);
}
- /// Begin buffering requests for `partition`. Idempotent: safe to call
- /// multiple times (watch reconnects etc.).
- ///
- /// Each iteration is a single attempt to bind to a live dashmap
- /// entry. The loop terminates because the only way to observe
- /// `None` is to race a drain session's eviction, which removes the
- /// entry inside its queue lock — meaning the next `get_or_create`
- /// will create a fresh entry rather than re-grab the doomed `Arc`.
- /// In production the routing-table layer awaits the drain before
- /// issuing the next `begin_stash` for the same partition, so this
- /// loop runs at most twice.
- pub async fn begin_stash(&self, partition: u32) {
- while !self.try_acquire_alive_entry(partition).await {}
- }
-
- /// One attempt to bind to the live dashmap entry for `partition`.
- /// Returns `true` if we successfully observed a `Some` queue (the
- /// entry is alive and ready for enqueues); `false` if we raced an
- /// eviction that left a tombstoned `None`, in which case the caller
- /// should retry to pick up the fresh entry.
- async fn try_acquire_alive_entry(&self, partition: u32) -> bool {
- let stash = self.get_or_create(partition);
- let guard = stash.queue.lock().await;
- guard.is_some()
- }
-
- /// Enqueue a request if the partition is frozen; otherwise return
- /// `Forward` to signal the caller should route normally.
- /// Takes the frame and headers by reference and clones them only when
- /// the request actually parks — the steady state (no handoff in
- /// progress) returns `Forward` from the dashmap miss without copying
- /// anything.
+ /// Park the request if its key has parked or in-flight requests or `partition`
+ /// is stashing; otherwise `Forward`. Clones the frame only when it parks.
///
/// `possibly_applied` carries the direct path's transport-failure mark
/// into the parked entry: a request that bounced at the transport
/// layer before parking may already have reached the leader, and the
/// drain's eventual re-forward of it must count as a replay.
- pub async fn enqueue_or_forward(
+ pub fn enqueue_or_forward(
&self,
- partition: u32,
method: &'static str,
frame: &Bytes,
headers: &HeaderMap,
key: StashKey,
+ partition: u32,
possibly_applied: bool,
) -> StashDecision {
- let stash = match self.inner.get(&partition) {
- Some(entry) => Arc::clone(entry.value()),
- None => return StashDecision::Forward,
- };
- let mut guard = stash.queue.lock().await;
- // `None` here means a drain session already settled this stash
- // and removed the dashmap entry. We arrived holding an Arc to
- // the orphaned PartitionStash. Returning Forward routes via the
- // normal path; the partition is no longer stashing.
- let Some(queue) = guard.as_mut() else {
+ let inner = &self.inner;
+ let parked = ParkedKey::of(key);
+ let shard = inner.shard_of(&parked);
+ if shard.held.load(Ordering::Acquire) == 0 && !inner.stashing.contains_key(&partition) {
return StashDecision::Forward;
- };
+ }
+ // Under the shard lock, so a drain's settle check sees this request or it sees the settle.
+ let mut queue = shard.lock();
+ if !queue.is_deferring(&parked) && !inner.stashing.contains_key(&partition) {
+ return StashDecision::Forward;
+ }
let request_size = approximate_size(frame);
-
- if queue.messages >= stash.max_messages {
- metrics::counter!(
- "personhog_router_stash_rejected_total",
- "cause" => RejectCause::MaxMessages.label()
- )
- .increment(1);
- return StashDecision::Rejected;
- }
- if queue.bytes.saturating_add(request_size) > stash.max_bytes {
- metrics::counter!(
- "personhog_router_stash_rejected_total",
- "cause" => RejectCause::MaxBytes.label()
- )
- .increment(1);
+ // The check and the charge share one hold of the partition's entry, since
+ // the shard lock only serializes parks of keys in the same shard.
+ let usage = inner.usage.entry(partition);
+ let (messages, bytes) = match &usage {
+ dashmap::Entry::Occupied(entry) => (entry.get().messages, entry.get().bytes),
+ dashmap::Entry::Vacant(_) => (0, 0),
+ };
+ let rejected = if messages >= inner.max_messages {
+ Some(RejectCause::MaxMessages)
+ } else if bytes.saturating_add(request_size) > inner.max_bytes {
+ Some(RejectCause::MaxBytes)
+ } else {
+ None
+ };
+ if let Some(cause) = rejected {
+ drop(usage);
+ metrics::counter!("personhog_router_stash_rejected_total", "cause" => cause.label())
+ .increment(1);
return StashDecision::Rejected;
}
+ let mut charged = usage.or_default();
+ charged.messages += 1;
+ charged.bytes += request_size;
+ drop(charged);
let (tx, rx) = oneshot::channel();
- queue.queue.push(
- key,
+ queue.push(
+ parked,
StashedRequest {
method,
frame: frame.clone(),
headers: headers.clone(),
- key,
reply: tx,
enqueued_at: Instant::now(),
possibly_applied,
+ charged_to: partition,
},
);
- queue.messages += 1;
- queue.bytes += request_size;
+ shard.held.fetch_add(1, Ordering::AcqRel);
metrics::counter!("personhog_router_stash_enqueued_total").increment(1);
- // Occupancy gauges count a parked entry until its outcome
- // resolves, so in-attempt entries still occupy capacity.
+ // Occupancy counts a parked entry until its outcome resolves.
metrics::gauge!("personhog_router_stash_queued_messages").increment(1.0);
metrics::gauge!("personhog_router_stash_queued_bytes").increment(request_size as f64);
StashDecision::Stashed(rx)
}
- /// Open a drain session for `partition`, or `None` when nothing is
- /// stashed. The session pins the partition's stash so every take,
- /// complete, and put-back lands on the same queue the entries came
- /// from, even across the eviction handshake.
- ///
- /// The drain-lane layer serializes drains per partition, so at most
- /// one session is active for a partition at a time; the session API
- /// relies on that for its eviction reasoning but tolerates a racing
- /// `begin_stash` (see [`DrainSession::finish_if_settled`]).
+ /// Open a drain session for `partition`, or `None` when nothing is pending.
+ /// After a count change two sessions can meet on one key; each skips keys the other holds.
pub fn drain_session(&self, partition: u32) -> Option<DrainSession> {
- let stash = Arc::clone(self.inner.get(&partition)?.value());
+ if !self.is_pending(partition) {
+ return None;
+ }
Some(DrainSession {
inner: Arc::clone(&self.inner),
partition,
- stash,
+ generation: self.inner.stashing.get(&partition).map(|g| *g),
+ candidates: Mutex::new(Vec::new()),
})
}
}
@@ -342,110 +362,108 @@ impl StashTable {
/// superseded). The entries re-enter at their sequence positions,
/// ahead of everything admitted after them.
///
-/// When a take finds no backlog, [`finish_if_settled`](Self::finish_if_settled)
-/// closes the drain: if the queue is fully settled it tombstones the
-/// queue and evicts the dashmap entry in one critical section, so any
-/// racing `enqueue_or_forward` either pushes before the settle check
-/// (forcing the caller through another take) or observes the entry
-/// already gone and forwards live.
-///
-/// Requests that arrive *during* a session land on the same queue (the
-/// dashmap entry stays present until the settle eviction) and are picked
-/// up by a later take, preserving per-key admission order across the
-/// cutover.
+/// [`finish_if_settled`](Self::finish_if_settled) closes the drain once the partition has no keys left.
pub struct DrainSession {
- inner: Arc<DashMap<u32, Arc<PartitionStash>>>,
+ inner: Arc<Inner>,
partition: u32,
- stash: Arc<PartitionStash>,
+ /// The freeze this session may close; `None` when it drains keys alone.
+ generation: Option<u64>,
+ /// Keys of the partition found by the last scan, not yet taken.
+ candidates: Mutex<Vec<ParkedKey>>,
}
impl DrainSession {
- /// Take the front run of up to `max_keys` keys for a delivery
- /// attempt. Returns an empty vec when no backlog is queued. Every
- /// returned entry is in-attempt until completed or put back;
- /// dropping entries instead would lose their clients' replies and
- /// permanently block the queue from settling.
- pub async fn take_for_attempt(&self, max_keys: usize) -> Vec<TakenKeyRun> {
- let mut guard = self.stash.queue.lock().await;
- let Some(q) = guard.as_mut() else {
- return Vec::new();
- };
- let keys: Vec<StashKey> = q
- .queue
- .keys_with_backlog()
- .take(max_keys)
- .copied()
- .collect();
- keys.into_iter()
- .map(|key| TakenKeyRun {
- entries: q.queue.take_front(&key, usize::MAX),
- key,
- })
- .collect()
+ fn candidates(&self) -> MutexGuard<'_, Vec<ParkedKey>> {
+ self.candidates
+ .lock()
+ .expect("drain candidates lock poisoned")
+ }
+
+ /// Take the front runs of up to `max_keys` keys for one attempt. Each entry
+ /// stays in-attempt until completed or put back.
+ pub fn take_for_attempt(&self, max_keys: usize) -> Vec<TakenKeyRun> {
+ let inner = &self.inner;
+ if self.candidates().is_empty() {
+ let found: Vec<ParkedKey> = inner
+ .shards
+ .iter()
+ .filter(|shard| shard.held.load(Ordering::Acquire) > 0)
+ .flat_map(|shard| {
+ shard
+ .lock()
+ .keys_with_backlog()
+ .filter(|key| inner.partition_of(key) == self.partition)
+ .copied()
+ .collect::<Vec<_>>()
+ })
+ .collect();
+ *self.candidates() = found;
+ }
+ let mut runs = Vec::new();
+ while runs.len() < max_keys {
+ let Some(key) = self.candidates().pop() else {
+ break;
+ };
+ let mut queue = inner.shard_of(&key).lock();
+ // Another session holds this key, or a count change moved it elsewhere.
+ if queue.has_in_attempt(&key) || inner.partition_of(&key) != self.partition {
+ continue;
+ }
+ let entries = queue.take_front(&key, usize::MAX);
+ if !entries.is_empty() {
+ runs.push(TakenKeyRun {
+ key: key.key,
+ entries,
+ });
+ }
+ }
+ runs
}
/// Record a definitive outcome for one in-attempt entry of `key`,
- /// releasing `size` bytes (the entry's `approximate_size`) from the
- /// admission bounds.
- pub async fn complete(&self, key: StashKey, size: usize) {
- let mut guard = self.stash.queue.lock().await;
- // In-attempt entries block the settle eviction and this session
- // is the only writer of outcomes, so the queue cannot have been
- // tombstoned while an attempt was outstanding.
- let Some(q) = guard.as_mut() else {
- debug_assert!(false, "complete() on a tombstoned stash");
- return;
- };
- q.queue.complete(&key);
- q.messages -= 1;
- q.bytes = q.bytes.saturating_sub(size);
- metrics::gauge!("personhog_router_stash_queued_messages").decrement(1.0);
- metrics::gauge!("personhog_router_stash_queued_bytes").decrement(size as f64);
+ /// releasing what admission charged it.
+ pub fn complete(&self, key: StashKey, charge: Charge) {
+ let key = ParkedKey::of(key);
+ let shard = self.inner.shard_of(&key);
+ shard.lock().complete(&key);
+ self.inner.release(shard, charge);
}
/// Return in-attempt entries whose delivery was aborted before any
/// outcome existed. They re-enter at their sequence positions and
/// keep their admission charges; the next attempt sees them first,
/// in order.
- pub async fn put_back(&self, key: StashKey, entries: Vec<Entry<StashedRequest>>) {
- if entries.is_empty() {
- return;
+ pub fn put_back(&self, key: StashKey, entries: Vec<Entry<StashedRequest>>) {
+ let key = ParkedKey::of(key);
+ self.inner.shard_of(&key).lock().put_back(&key, entries);
+ }
+
+ /// Close the session and end its freeze once the partition has no parked or
+ /// in-flight keys; all shards stay locked across the check and the close.
+ /// A request hashed under an older count can still park; the reconcile pass drains it.
+ pub fn finish_if_settled(&self) -> bool {
+ let inner = &self.inner;
+ // A key still held needs no global lock to see.
+ if inner.holds_keys_of(self.partition) {
+ return false;
}
- let mut guard = self.stash.queue.lock().await;
- // Same reasoning as `complete`: outstanding in-attempt entries
- // make the tombstone unreachable.
- let Some(q) = guard.as_mut() else {
- debug_assert!(false, "put_back() on a tombstoned stash");
- return;
+ // With no freeze to close, a request arriving after the check forwards either way.
+ let Some(generation) = self.generation else {
+ return true;
};
- q.queue.put_back(&key, entries);
- }
-
- /// Close the session if the queue is fully settled — no queued
- /// entries and no in-attempt entries. On success the queue is
- /// tombstoned and the dashmap entry evicted inside the same critical
- /// section, and `true` is returned: the partition has left the stash
- /// path and new requests forward live. Returns `false` when an
- /// arrival raced in since the last take; the caller should take
- /// again.
- ///
- /// The eviction removes the entry only if it still holds this
- /// session's stash, so a `begin_stash` that already replaced a
- /// tombstoned entry with a fresh one can never have its new queue
- /// evicted from under it.
- pub async fn finish_if_settled(&self) -> bool {
- let mut guard = self.stash.queue.lock().await;
- match guard.as_ref() {
- // Already tombstoned — a prior settle got there first.
- None => true,
- Some(q) if q.queue.is_empty() => {
- *guard = None;
- self.inner
- .remove_if(&self.partition, |_, v| Arc::ptr_eq(v, &self.stash));
- true
- }
- Some(_) => false,
+ let shards: Vec<_> = inner.shards.iter().map(Shard::lock).collect();
+ let pending = shards.iter().any(|queue| {
+ queue
+ .deferring_keys()
+ .any(|key| inner.partition_of(key) == self.partition)
+ });
+ if pending {
+ return false;
}
+ inner
+ .stashing
+ .remove_if(&self.partition, |_, opened| *opened == generation);
+ true
}
}
@@ -454,66 +472,66 @@ mod tests {
use super::*;
use http::HeaderValue;
use http_body_util::{BodyExt, Empty};
+ use personhog_common::partitioning::partition_for_person;
+ use personhog_coordination::types::ChangelogConfig;
+
+ /// A table over `partitions` partitions; with one, every key maps to partition 0.
+ fn table_over(partitions: u32, max_messages: usize, max_bytes: usize) -> StashTable {
+ StashTable::with_bounds(
+ max_messages,
+ max_bytes,
+ LiveChangelog::new(ChangelogConfig::new("personhog_updates", partitions)),
+ )
+ }
/// Drain everything with a trivial always-succeeds forwarder,
/// collecting the requests in delivery order. Mirrors the handler's
/// session loop: take, complete each entry, settle when no backlog
/// remains.
- async fn drain_to_vec(table: &StashTable, partition: u32) -> Vec<StashedRequest> {
+ fn drain_to_vec(table: &StashTable, partition: u32) -> Vec<StashedRequest> {
let Some(session) = table.drain_session(partition) else {
return Vec::new();
};
let mut out = Vec::new();
loop {
- let taken = session.take_for_attempt(usize::MAX).await;
+ let taken = session.take_for_attempt(usize::MAX);
if taken.is_empty() {
- if session.finish_if_settled().await {
+ if session.finish_if_settled() {
return out;
}
continue;
}
for run in taken {
for entry in run.entries {
- let size = entry.item.approximate_size();
+ let charge = entry.item.charge();
out.push(entry.item);
- session.complete(run.key, size).await;
+ session.complete(run.key, charge);
}
}
}
}
+ fn enqueue_frame(table: &StashTable, person_id: i64, frame: Bytes) -> StashDecision {
+ let partition = partition_for_person(1, person_id, table.inner.changelog.partitions());
+ table.enqueue_or_forward(
+ "UpdatePersonProperties",
+ &frame,
+ &HeaderMap::new(),
+ (1, person_id),
+ partition,
+ false,
+ )
+ }
+
/// Enqueue a minimal stashed write for `(team_id = 1, person_id)`.
- async fn enqueue(table: &StashTable, partition: u32, person_id: i64) -> StashDecision {
- table
- .enqueue_or_forward(
- partition,
- "UpdatePersonProperties",
- &Bytes::from_static(b"x"),
- &HeaderMap::new(),
- (1, person_id),
- false,
- )
- .await
+ fn enqueue(table: &StashTable, person_id: i64) -> StashDecision {
+ enqueue_frame(table, person_id, Bytes::from_static(b"x"))
}
/// Enqueue a stashed write whose frame is `payload` bytes, exercising
/// the byte-based stash bound.
- async fn enqueue_sized(
- table: &StashTable,
- partition: u32,
- person_id: i64,
- payload: usize,
- ) -> StashDecision {
- table
- .enqueue_or_forward(
- partition,
- "UpdatePersonProperties",
- &Bytes::from(vec![0u8; payload]),
- &HeaderMap::new(),
- (1, person_id),
- false,
- )
- .await
+ fn enqueue_sized(table: &StashTable, person_id: i64, payload: usize) -> StashDecision {
+ enqueue_frame(table, person_id, Bytes::from(vec![0u8; payload]))
}
/// A trivial gRPC response for exercising the reply channel.
@@ -523,96 +541,98 @@ mod tests {
))
}
- #[tokio::test]
- async fn forward_when_not_frozen() {
- let table = StashTable::with_bounds(usize::MAX, usize::MAX);
- match enqueue(&table, 0, 1).await {
+ /// A person whose partition differs between `from` and `to` partitions.
+ fn moving_person(from: u32, to: u32) -> i64 {
+ (1..)
+ .find(|id| partition_for_person(1, *id, from) != partition_for_person(1, *id, to))
+ .unwrap()
+ }
+
+ fn adopt(changelog: &LiveChangelog, partitions: u32) {
+ changelog.adopt(&ChangelogConfig {
+ epoch: changelog.epoch() + 1,
+ ..ChangelogConfig::new("personhog_updates", partitions)
+ });
+ }
+
+ #[test]
+ fn forward_when_not_frozen() {
+ let table = table_over(1, usize::MAX, usize::MAX);
+ match enqueue(&table, 1) {
StashDecision::Forward => {}
_ => panic!("expected Forward"),
}
}
- #[tokio::test]
- async fn begin_then_enqueue_then_drain_preserves_per_key_order() {
- let table = StashTable::with_bounds(usize::MAX, usize::MAX);
- table.begin_stash(0).await;
+ #[test]
+ fn begin_then_enqueue_then_drain_preserves_per_key_order() {
+ let table = table_over(1, usize::MAX, usize::MAX);
+ table.begin_stash(0);
// Same key, distinguishable frames — ordering is a per-key
// contract, so the assertion must not depend on cross-key
// iteration order.
- let _rx1 = match enqueue_sized(&table, 0, 1, 1).await {
+ let _rx1 = match enqueue_sized(&table, 1, 1) {
StashDecision::Stashed(rx) => rx,
_ => panic!("expected Stashed"),
};
- let _rx2 = match enqueue_sized(&table, 0, 1, 2).await {
+ let _rx2 = match enqueue_sized(&table, 1, 2) {
StashDecision::Stashed(rx) => rx,
_ => panic!("expected Stashed"),
};
- let drained = drain_to_vec(&table, 0).await;
+ let drained = drain_to_vec(&table, 0);
assert_eq!(drained.len(), 2);
let frame_lens: Vec<usize> = drained.iter().map(|s| s.frame.len()).collect();
assert_eq!(frame_lens, vec![1, 2]);
// After drain, new requests forward.
- match enqueue(&table, 0, 3).await {
+ match enqueue(&table, 3) {
StashDecision::Forward => {}
_ => panic!("expected Forward after drain"),
}
}
- /// A settled drain must remove the dashmap entry so subsequent
- /// steady-state requests for that partition can short-circuit on
- /// `dashmap.get` returning `None`, avoiding the per-request Mutex
- /// lock that `enqueue_or_forward` would otherwise take.
- #[tokio::test]
- async fn drain_removes_dashmap_entry() {
- let table = StashTable::with_bounds(usize::MAX, usize::MAX);
- table.begin_stash(0).await;
- assert!(
- table.inner.contains_key(&0),
- "begin_stash must populate the entry"
- );
- drain_to_vec(&table, 0).await;
- assert!(
- !table.inner.contains_key(&0),
- "settle must remove the entry so future requests skip the lock path"
- );
+ /// A settled drain must end the freeze, or the partition never
+ /// leaves the stash path.
+ #[test]
+ fn a_settled_drain_ends_the_freeze() {
+ let table = table_over(1, usize::MAX, usize::MAX);
+ table.begin_stash(0);
+ assert!(table.is_pending(0), "begin_stash must open the freeze");
+ drain_to_vec(&table, 0);
+ assert!(!table.is_pending(0), "settle must end the freeze");
}
- /// An aborted attempt puts its entries back and leaves the entry
- /// live: whatever is parked belongs to the successor drain, and new
- /// requests must keep stashing in the meantime. Evicting instead
- /// would let requests bypass the stash and race ahead of the parked
- /// backlog.
- #[tokio::test]
- async fn put_back_leaves_queue_live_for_successor() {
- let table = StashTable::with_bounds(usize::MAX, usize::MAX);
- table.begin_stash(0).await;
- let _rx = match enqueue(&table, 0, 1).await {
+ /// An aborted attempt leaves the partition pending, so new requests keep parking behind the backlog.
+ #[test]
+ fn put_back_leaves_queue_live_for_successor() {
+ let table = table_over(1, usize::MAX, usize::MAX);
+ table.begin_stash(0);
+ let _rx = match enqueue(&table, 1) {
StashDecision::Stashed(rx) => rx,
_ => panic!("expected Stashed"),
};
// An attempt starts, then aborts (pause/supersession/bounce) and
// puts everything back without settling.
- let session = table.drain_session(0).expect("entry must be live");
- let mut taken = session.take_for_attempt(usize::MAX).await;
+ let session = table.drain_session(0).expect("partition must be pending");
+ let mut taken = session.take_for_attempt(usize::MAX);
assert_eq!(taken.len(), 1);
let run = taken.remove(0);
- session.put_back(run.key, run.entries).await;
+ session.put_back(run.key, run.entries);
drop(session);
assert!(
- table.inner.contains_key(&0),
- "aborted drain must leave the entry live"
+ table.is_pending(0),
+ "aborted drain must leave the partition pending"
);
// Requests keep parking, and a successor drain collects everything.
- let _rx2 = match enqueue(&table, 0, 2).await {
+ let _rx2 = match enqueue(&table, 2) {
StashDecision::Stashed(rx) => rx,
- _ => panic!("expected Stashed while entry is live"),
+ _ => panic!("expected Stashed while the partition is pending"),
};
- let drained = drain_to_vec(&table, 0).await;
+ let drained = drain_to_vec(&table, 0);
assert_eq!(
drained.len(),
2,
@@ -620,190 +640,257 @@ mod tests {
);
}
- /// In-attempt entries must block the settle eviction: the queue can
- /// look empty (backlog fully taken) while outcomes are still
- /// outstanding, and settling then would orphan the eventual put-back
- /// into a tombstoned queue, dropping the clients' replies.
- #[tokio::test]
- async fn in_attempt_entries_block_settle() {
- let table = StashTable::with_bounds(usize::MAX, usize::MAX);
- table.begin_stash(0).await;
- let _rx = match enqueue(&table, 0, 1).await {
+ /// In-attempt entries block the settle, so new requests cannot overtake a pending attempt.
+ #[test]
+ fn in_attempt_entries_block_settle() {
+ let table = table_over(1, usize::MAX, usize::MAX);
+ table.begin_stash(0);
+ let _rx = match enqueue(&table, 1) {
StashDecision::Stashed(rx) => rx,
_ => panic!("expected Stashed"),
};
- let session = table.drain_session(0).expect("entry must be live");
- let mut taken = session.take_for_attempt(usize::MAX).await;
+ let session = table.drain_session(0).expect("partition must be pending");
+ let mut taken = session.take_for_attempt(usize::MAX);
let run = taken.remove(0);
assert!(
- !session.finish_if_settled().await,
+ !session.finish_if_settled(),
"settle must refuse while an attempt is outstanding"
);
let entry = run.entries.into_iter().next().unwrap();
- let size = entry.item.approximate_size();
- session.complete(run.key, size).await;
+ session.complete(run.key, entry.item.charge());
assert!(
- session.finish_if_settled().await,
+ session.finish_if_settled(),
"settle must succeed once every outcome is recorded"
);
- assert!(!table.inner.contains_key(&0));
+ assert!(!table.is_pending(0));
}
/// Back-to-back handoffs: drain → begin_stash for the same partition
- /// must produce a fresh empty queue, not preserve stale state.
- #[tokio::test]
- async fn drain_then_begin_stash_starts_fresh() {
- let table = StashTable::with_bounds(usize::MAX, usize::MAX);
- table.begin_stash(0).await;
- let _rx = match enqueue(&table, 0, 1).await {
+ /// must stash again.
+ #[test]
+ fn drain_then_begin_stash_starts_fresh() {
+ let table = table_over(1, usize::MAX, usize::MAX);
+ table.begin_stash(0);
+ let _rx = match enqueue(&table, 1) {
StashDecision::Stashed(rx) => rx,
_ => panic!("expected Stashed"),
};
- let drained = drain_to_vec(&table, 0).await;
+ let drained = drain_to_vec(&table, 0);
assert_eq!(drained.len(), 1);
- // New handoff begins
- table.begin_stash(0).await;
- // Fresh queue; brand-new requests stash, not forward
- match enqueue(&table, 0, 2).await {
+ table.begin_stash(0);
+ match enqueue(&table, 2) {
StashDecision::Stashed(_) => {}
_ => panic!("expected Stashed for fresh handoff"),
}
}
- #[tokio::test]
- async fn reject_when_message_count_exceeds_max() {
- let table = StashTable::with_bounds(2, usize::MAX);
- table.begin_stash(0).await;
+ #[test]
+ fn reject_when_message_count_exceeds_max() {
+ let table = table_over(1, 2, usize::MAX);
+ table.begin_stash(0);
- assert!(matches!(
- enqueue(&table, 0, 1).await,
- StashDecision::Stashed(_)
- ));
- assert!(matches!(
- enqueue(&table, 0, 2).await,
- StashDecision::Stashed(_)
- ));
- assert!(matches!(
- enqueue(&table, 0, 3).await,
- StashDecision::Rejected
- ));
+ assert!(matches!(enqueue(&table, 1), StashDecision::Stashed(_)));
+ assert!(matches!(enqueue(&table, 2), StashDecision::Stashed(_)));
+ assert!(matches!(enqueue(&table, 3), StashDecision::Rejected));
}
- #[tokio::test]
- async fn reject_when_byte_total_exceeds_max() {
+ #[test]
+ fn reject_when_byte_total_exceeds_max() {
// Cap at ~5KB; each request below has ~2KB payload. After two
// requests we're near 4KB; a third request would push over the cap.
- let table = StashTable::with_bounds(usize::MAX, 5 * 1024);
- table.begin_stash(0).await;
+ let table = table_over(1, usize::MAX, 5 * 1024);
+ table.begin_stash(0);
assert!(matches!(
- enqueue_sized(&table, 0, 1, 2 * 1024).await,
+ enqueue_sized(&table, 1, 2 * 1024),
StashDecision::Stashed(_)
));
assert!(matches!(
- enqueue_sized(&table, 0, 2, 2 * 1024).await,
+ enqueue_sized(&table, 2, 2 * 1024),
StashDecision::Stashed(_)
));
assert!(matches!(
- enqueue_sized(&table, 0, 3, 2 * 1024).await,
+ enqueue_sized(&table, 3, 2 * 1024),
StashDecision::Rejected
));
}
/// Either bound triggers rejection — whichever is hit first.
- #[tokio::test]
- async fn message_count_takes_precedence_when_hit_first() {
+ #[test]
+ fn message_count_takes_precedence_when_hit_first() {
// Generous byte budget but tight message budget.
- let table = StashTable::with_bounds(1, 100 * 1024 * 1024);
- table.begin_stash(0).await;
+ let table = table_over(1, 1, 100 * 1024 * 1024);
+ table.begin_stash(0);
- assert!(matches!(
- enqueue(&table, 0, 1).await,
- StashDecision::Stashed(_)
- ));
+ assert!(matches!(enqueue(&table, 1), StashDecision::Stashed(_)));
// Second message rejected on count even though bytes are nowhere near.
- assert!(matches!(
- enqueue(&table, 0, 2).await,
- StashDecision::Rejected
- ));
+ assert!(matches!(enqueue(&table, 2), StashDecision::Rejected));
}
/// The admission bounds must release on definitive outcomes, not on
/// takes: an entry taken for an attempt still holds its memory, and
/// releasing at take time would let a slow drain admit unbounded
/// backlog behind it.
- #[tokio::test]
- async fn bounds_release_on_complete_not_on_take() {
- let table = StashTable::with_bounds(1, usize::MAX);
- table.begin_stash(0).await;
- assert!(matches!(
- enqueue(&table, 0, 1).await,
- StashDecision::Stashed(_)
- ));
-
- let session = table.drain_session(0).expect("entry must be live");
- let mut taken = session.take_for_attempt(usize::MAX).await;
+ #[test]
+ fn bounds_release_on_complete_not_on_take() {
+ let table = table_over(1, 1, usize::MAX);
+ table.begin_stash(0);
+ assert!(matches!(enqueue(&table, 1), StashDecision::Stashed(_)));
+
+ let session = table.drain_session(0).expect("partition must be pending");
+ let mut taken = session.take_for_attempt(usize::MAX);
let run = taken.remove(0);
// Taken but not completed — still counted against the bound.
- assert!(matches!(
- enqueue(&table, 0, 2).await,
- StashDecision::Rejected
- ));
+ assert!(matches!(enqueue(&table, 2), StashDecision::Rejected));
let entry = run.entries.into_iter().next().unwrap();
- let size = entry.item.approximate_size();
- session.complete(run.key, size).await;
+ session.complete(run.key, entry.item.charge());
// Outcome recorded — capacity is released.
+ assert!(matches!(enqueue(&table, 3), StashDecision::Stashed(_)));
+ }
+
+ #[test]
+ fn partitions_are_independent() {
+ let table = table_over(2, usize::MAX, usize::MAX);
+ table.begin_stash(0);
+ let on = |partition: u32| {
+ (1..)
+ .find(|id| partition_for_person(1, *id, 2) == partition)
+ .unwrap()
+ };
+
+ // p0 stashes, p1 forwards
+ assert!(matches!(enqueue(&table, on(0)), StashDecision::Stashed(_)));
+ assert!(matches!(enqueue(&table, on(1)), StashDecision::Forward));
+ }
+
+ /// A key moved by a count change stays ordered behind its writes parked under the old partition.
+ #[test]
+ fn a_key_keeps_its_order_across_a_count_change() {
+ let changelog = LiveChangelog::new(ChangelogConfig::new("personhog_updates", 8));
+ let table = StashTable::with_bounds(usize::MAX, usize::MAX, changelog.clone());
+ let person_id = moving_person(8, 4);
+ let old = partition_for_person(1, person_id, 8);
+ let new = partition_for_person(1, person_id, 4);
+ table.begin_stash(old);
+ table.begin_stash(new);
assert!(matches!(
- enqueue(&table, 0, 3).await,
+ enqueue_frame(&table, person_id, Bytes::from_static(b"older")),
StashDecision::Stashed(_)
));
- }
- #[tokio::test]
- async fn partitions_are_independent() {
- let table = StashTable::with_bounds(usize::MAX, usize::MAX);
- table.begin_stash(0).await;
+ drain_to_vec(&table, new);
+ adopt(&changelog, 4);
- // p0 stashes, p1 forwards
assert!(matches!(
- enqueue(&table, 0, 1).await,
+ enqueue_frame(&table, person_id, Bytes::from_static(b"newer")),
StashDecision::Stashed(_)
));
+ let frames: Vec<Bytes> = drain_to_vec(&table, new)
+ .into_iter()
+ .map(|r| r.frame)
+ .collect();
+ assert_eq!(
+ frames,
+ vec![Bytes::from_static(b"older"), Bytes::from_static(b"newer")]
+ );
+ }
+
+ /// After a count change two partitions' drains can both reach one key.
+ /// While one holds the key's head, the other must not take its tail.
+ #[test]
+ fn a_key_out_with_another_drain_is_skipped() {
+ let changelog = LiveChangelog::new(ChangelogConfig::new("personhog_updates", 8));
+ let table = StashTable::with_bounds(usize::MAX, usize::MAX, changelog.clone());
+ let person_id = moving_person(8, 4);
+ let old = partition_for_person(1, person_id, 8);
+ let new = partition_for_person(1, person_id, 4);
+ table.begin_stash(old);
assert!(matches!(
- enqueue(&table, 1, 1).await,
- StashDecision::Forward
+ enqueue_frame(&table, person_id, Bytes::from_static(b"head")),
+ StashDecision::Stashed(_)
+ ));
+ let holder = table.drain_session(old).unwrap();
+ let mut head = holder.take_for_attempt(usize::MAX);
+ assert!(matches!(
+ enqueue_frame(&table, person_id, Bytes::from_static(b"tail")),
+ StashDecision::Stashed(_)
));
+
+ adopt(&changelog, 4);
+ let other = table.drain_session(new).unwrap();
+ assert!(other.take_for_attempt(usize::MAX).is_empty());
+ assert!(!other.finish_if_settled());
+
+ let run = head.remove(0);
+ let entry = run.entries.into_iter().next().unwrap();
+ holder.complete(run.key, entry.item.charge());
+ let frames: Vec<Bytes> = drain_to_vec(&table, new)
+ .into_iter()
+ .map(|r| r.frame)
+ .collect();
+ assert_eq!(frames, vec![Bytes::from_static(b"tail")]);
+ }
+
+ /// A key out with another drain still needs its new partition's drain:
+ /// if that attempt bounces and puts the key back, only that drain takes it.
+ #[test]
+ fn a_key_out_for_delivery_keeps_its_new_partition_pending() {
+ let changelog = LiveChangelog::new(ChangelogConfig::new("personhog_updates", 8));
+ let table = StashTable::with_bounds(usize::MAX, usize::MAX, changelog.clone());
+ let person_id = moving_person(8, 4);
+ let old = partition_for_person(1, person_id, 8);
+ let new = partition_for_person(1, person_id, 4);
+ table.begin_stash(old);
+ assert!(matches!(
+ enqueue(&table, person_id),
+ StashDecision::Stashed(_)
+ ));
+ let holder = table.drain_session(old).unwrap();
+ let mut head = holder.take_for_attempt(usize::MAX);
+
+ adopt(&changelog, 4);
+ assert!(table.is_pending(new));
+
+ let run = head.remove(0);
+ holder.put_back(run.key, run.entries);
+ assert!(holder.take_for_attempt(usize::MAX).is_empty());
+ let taker = table.drain_session(new).unwrap();
+ assert_eq!(taker.take_for_attempt(usize::MAX).len(), 1);
+ }
+
+ /// A freeze opened while a drain runs belongs to the next handoff;
+ /// the running drain's settle must not end it.
+ #[test]
+ fn a_freeze_opened_during_a_drain_survives_its_settle() {
+ let table = table_over(1, usize::MAX, usize::MAX);
+ table.begin_stash(0);
+ let session = table.drain_session(0).unwrap();
+ table.begin_stash(0);
+ assert!(session.finish_if_settled());
+ assert!(matches!(enqueue(&table, 1), StashDecision::Stashed(_)));
}
- /// Race between concurrent enqueue and drain. The `Arc<PartitionStash>`
- /// + `Option<StashQueue>` design exists so that an enqueue that has
- /// already cloned the Arc when the settle eviction runs sees `None`,
- /// and returns `Forward` rather than pushing into a dead queue. With
- /// many iterations this drives the race window repeatedly; the
- /// invariant is that every request must end up either in the drained
- /// batch or forwarded — never lost, never duplicated.
+ /// Concurrent enqueues and a drain: every request is drained or forwarded exactly once.
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn enqueue_and_drain_race() {
for iteration in 0..200 {
- let table = StashTable::with_bounds(usize::MAX, usize::MAX);
- table.begin_stash(0).await;
+ let table = table_over(1, usize::MAX, usize::MAX);
+ table.begin_stash(0);
const ENQUEUERS: usize = 8;
let mut handles = Vec::with_capacity(ENQUEUERS);
for i in 0..ENQUEUERS {
let table = table.clone();
- handles.push(tokio::spawn(
- async move { enqueue(&table, 0, i as i64).await },
- ));
+ handles.push(tokio::spawn(async move { enqueue(&table, i as i64) }));
}
// Race the drain against the in-flight enqueues.
let drain_table = table.clone();
- let drain_handle = tokio::spawn(async move { drain_to_vec(&drain_table, 0).await });
+ let drain_handle = tokio::spawn(async move { drain_to_vec(&drain_table, 0) });
let drained = drain_handle.await.unwrap();
let mut stashed_count = 0usize;
@@ -840,15 +927,15 @@ mod tests {
/// path relies on.
#[tokio::test]
async fn drained_request_reply_round_trips() {
- let table = StashTable::with_bounds(usize::MAX, usize::MAX);
- table.begin_stash(0).await;
+ let table = table_over(1, usize::MAX, usize::MAX);
+ table.begin_stash(0);
- let rx = match enqueue(&table, 0, 1).await {
+ let rx = match enqueue(&table, 1) {
StashDecision::Stashed(rx) => rx,
_ => panic!("expected Stashed"),
};
- let mut drained = drain_to_vec(&table, 0).await;
+ let mut drained = drain_to_vec(&table, 0);
let req = drained.remove(0);
let mut response = test_response();
response
@@ -866,83 +953,22 @@ mod tests {
/// disconnected) before drain delivered the reply, the `send` must
/// return `Err`. The drain handler relies on this signal to bump the
/// `personhog_router_stash_dropped_total` counter.
- #[tokio::test]
- async fn dropped_receiver_makes_send_fail() {
- let table = StashTable::with_bounds(usize::MAX, usize::MAX);
- table.begin_stash(0).await;
+ #[test]
+ fn dropped_receiver_makes_send_fail() {
+ let table = table_over(1, usize::MAX, usize::MAX);
+ table.begin_stash(0);
- let rx = match enqueue(&table, 0, 1).await {
+ let rx = match enqueue(&table, 1) {
StashDecision::Stashed(rx) => rx,
_ => panic!("expected Stashed"),
};
drop(rx);
- let mut drained = drain_to_vec(&table, 0).await;
+ let mut drained = drain_to_vec(&table, 0);
let req = drained.remove(0);
assert!(
req.reply.send(test_response()).is_err(),
"send must return Err after the receiver is dropped"
);
}
-
- /// Regression for the structural race the `drained` tombstone closes:
- /// a `begin_stash` that observes the post-settle `None` queue between
- /// the settle tombstoning the queue and evicting the dashmap entry
- /// must not initialize a new queue on the doomed `Arc`. With the
- /// tombstone, `begin_stash` sees `None`, drops the lock, and retries
- /// via `get_or_create` — which produces a fresh dashmap entry once
- /// the eviction finishes.
- ///
- /// The race window is microseconds; we run many iterations to
- /// exercise it. The protocol awaits the drain before issuing the next
- /// `begin_stash` for a partition today, so this race cannot trigger
- /// in production — the test guards against future callers that
- /// might break that ordering.
- #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
- async fn drain_does_not_orphan_concurrent_begin_stash() {
- for iteration in 0..500 {
- let table = StashTable::with_bounds(usize::MAX, usize::MAX);
- table.begin_stash(0).await;
-
- // Drain in one task; concurrent begin_stash in another.
- // The key invariant: after both complete, the dashmap must
- // contain a live entry for the partition (because
- // begin_stash was the most recent successful call), so a
- // subsequent enqueue sees it and parks.
- let drain_table = table.clone();
- let drain_handle = tokio::spawn(async move {
- drain_to_vec(&drain_table, 0).await;
- });
- let begin_table = table.clone();
- let begin_handle = tokio::spawn(async move { begin_table.begin_stash(0).await });
-
- drain_handle.await.unwrap();
- begin_handle.await.unwrap();
-
- // Only meaningful to assert when begin_stash logically ran
- // *after* the settle (eviction removed the entry, begin_stash
- // created a fresh one). If begin_stash logically ran first
- // (observed the prior `Some` queue, was idempotent), then
- // the drain legitimately drained that prior queue and a
- // subsequent enqueue forwards via the direct path — that's a
- // protocol-violation scenario, not a stash-module bug, and
- // the routing-table layer prevents it. We accept either
- // outcome here; what we *don't* accept is a non-empty
- // drained queue that was just initialized by begin_stash on
- // an orphaned Arc, which the tombstone prevents.
- let outcome = enqueue(&table, 0, 1).await;
- match outcome {
- StashDecision::Stashed(_) => {
- // Fresh dashmap entry exists — begin_stash set it up correctly.
- }
- StashDecision::Forward => {
- // Begin_stash logically ran first and was idempotent;
- // the drain emptied the prior queue. No orphaned Arc.
- }
- StashDecision::Rejected => {
- panic!("unexpected Rejected on iteration {iteration}");
- }
- }
- }
- }
}
diff --git a/rust/personhog-router/src/main.rs b/rust/personhog-router/src/main.rs
index 3035bdee6ad4..418dc38fc082 100644
--- a/rust/personhog-router/src/main.rs
+++ b/rust/personhog-router/src/main.rs
@@ -303,7 +303,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
.map(|address| format!("http://{address}"))
}),
LeaderBackendConfig {
- changelog,
+ changelog: changelog.clone(),
timeout: config.backend_timeout(),
num_channels: config.leader_channels,
http2_windows: config.backend_http2_windows(),
@@ -311,6 +311,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
StashTable::with_bounds(
config.stash_max_messages_per_partition,
config.stash_max_bytes_per_partition,
+ changelog.clone(),
),
));
diff --git a/rust/personhog-router/src/proxy.rs b/rust/personhog-router/src/proxy.rs
index c56a4928bfde..fb19255decb6 100644
--- a/rust/personhog-router/src/proxy.rs
+++ b/rust/personhog-router/src/proxy.rs
@@ -620,17 +620,9 @@ impl RawProxyInner {
Err(resp) => return (resp, None),
};
- let partition = leader.partition_for_person(team_id, person_id);
-
let _in_flight = ClientInFlightGuard::new("leader");
leader
- .forward_or_stash(
- method,
- partition,
- (team_id, person_id),
- parts.headers,
- body_bytes,
- )
+ .forward_or_stash(method, (team_id, person_id), parts.headers, body_bytes)
.await
}
}
diff --git a/rust/personhog-router/src/stash_handler.rs b/rust/personhog-router/src/stash_handler.rs
index fd2888b6e9ca..9d334059e638 100644
--- a/rust/personhog-router/src/stash_handler.rs
+++ b/rust/personhog-router/src/stash_handler.rs
@@ -16,21 +16,23 @@ use crate::backend::{
};
use crate::grpc_http::{grpc_error_response, is_grpc_error_response};
+/// How long a drain waits before looking again for a key it could not take.
+const SETTLE_RETRY: Duration = Duration::from_millis(10);
+
/// Stash handler for the router. Reacts to handoff phase transitions:
///
/// * `Freezing` / `Draining` / `Warming` → `begin_stash`: start (or
/// re-confirm) buffering leader-path requests — writes and strong
/// reads — for the partition in the shared `StashTable`. The
/// routing-table layer calls `begin_stash` on every non-terminal phase
-/// the router observes, so the call must be idempotent —
-/// `StashTable::begin_stash` no-ops if the entry is already live. New
-/// leader-path requests park in per-key queues while the handoff
-/// progresses through `Freezing → Draining → Warming`.
+/// the router observes, so the call must be idempotent; a repeat only
+/// renews the freeze. New leader-path requests park in per-key queues
+/// while the handoff progresses through `Freezing → Draining → Warming`.
/// * `Complete` → `drain_stash`: forward the buffered requests to the
/// new owner, each to the method it arrived on, through a
-/// `DrainSession`. The drain loops taking per-key front runs until it
-/// observes the queue fully settled; any request that arrives during
-/// the drain lands on the same queue and is picked up by a later take.
+/// `DrainSession`. The drain loops taking per-key front runs until no
+/// key of the partition remains; any request that arrives during the
+/// drain parks behind its key and is picked up by a later take.
///
/// Every stashed request ends in exactly one of two ways: a definitive
/// outcome delivered to its client (the forwarded response, an expiry
@@ -197,7 +199,7 @@ struct KeyRunOutcome {
/// yet attempted. Retry accounting stays with the callers — only an
/// actually-attempted head counts as a stash-path retry, so the
/// never-attempted tail goes back uncounted.
-async fn put_back_rest(
+fn put_back_rest(
session: &DrainSession,
key: StashKey,
head: Entry<StashedRequest>,
@@ -205,7 +207,7 @@ async fn put_back_rest(
) {
let mut entries = vec![head];
entries.extend(rest);
- session.put_back(key, entries).await;
+ session.put_back(key, entries);
}
/// Forward one key's front run sequentially, recording a definitive
@@ -226,7 +228,7 @@ async fn forward_key_run(
let mut entries = run.entries.into_iter();
while let Some(mut entry) = entries.next() {
if cancel.is_cancelled() {
- put_back_rest(session, run.key, entry, entries).await;
+ put_back_rest(session, run.key, entry, entries);
return outcome;
}
// Race the forward against cancellation: an in-flight call can
@@ -250,7 +252,7 @@ async fn forward_key_run(
"reason" => "cancelled"
)
.increment(1);
- put_back_rest(session, run.key, entry, entries).await;
+ put_back_rest(session, run.key, entry, entries);
return outcome;
};
match disposition {
@@ -258,7 +260,7 @@ async fn forward_key_run(
response,
outcome: label,
} => {
- let size = entry.item.approximate_size();
+ let charge = entry.item.charge();
metrics::histogram!("personhog_router_stash_wait_duration_ms")
.record(entry.item.enqueued_at.elapsed().as_secs_f64() * 1000.0);
if entry.item.reply.send(response).is_err() {
@@ -273,7 +275,7 @@ async fn forward_key_run(
"outcome" => label
)
.increment(1);
- session.complete(run.key, size).await;
+ session.complete(run.key, charge);
outcome.completed += 1;
}
Disposition::Bounce(reason) => {
@@ -286,7 +288,7 @@ async fn forward_key_run(
"reason" => reason.label()
)
.increment(1);
- put_back_rest(session, run.key, entry, entries).await;
+ put_back_rest(session, run.key, entry, entries);
outcome.bounced = true;
return outcome;
}
@@ -298,7 +300,7 @@ async fn forward_key_run(
#[async_trait]
impl StashHandler for RouterStashHandler {
fn stash_pending(&self, partition: u32) -> bool {
- self.leader_backend.stash_table().has_entry(partition)
+ self.leader_backend.stash_table().is_pending(partition)
}
async fn begin_stash(&self, partition: u32, new_owner: &str) -> CoordResult<()> {
@@ -307,10 +309,7 @@ impl StashHandler for RouterStashHandler {
new_owner,
"beginning stash for partition handoff"
);
- self.leader_backend
- .stash_table()
- .begin_stash(partition)
- .await;
+ self.leader_backend.stash_table().begin_stash(partition);
Ok(())
}
@@ -320,9 +319,8 @@ impl StashHandler for RouterStashHandler {
new_owner: &str,
cancel: CancellationToken,
) -> CoordResult<()> {
- let stash_table = self.leader_backend.stash_table();
- let Some(session) = stash_table.drain_session(partition) else {
- // Nothing stashed — the partition never froze, or a prior
+ let Some(session) = self.leader_backend.stash_table().drain_session(partition) else {
+ // Nothing pending — the partition never froze, or a prior
// drain already settled it.
return Ok(());
};
@@ -344,12 +342,17 @@ impl StashHandler for RouterStashHandler {
// flight, the backlog stays parked for the successor.
break;
}
- let taken = session.take_for_attempt(self.drain_concurrency).await;
+ let taken = session.take_for_attempt(self.drain_concurrency);
if taken.is_empty() {
- if session.finish_if_settled().await {
+ if session.finish_if_settled() {
break;
}
- // An arrival raced the settle check; take again.
+ // An arrival raced the settle check, or another drain holds one of our keys.
+ metrics::counter!("personhog_router_stash_settle_retries_total").increment(1);
+ tokio::select! {
+ _ = cancel.cancelled() => break,
+ _ = tokio::time::sleep(SETTLE_RETRY) => {}
+ }
continue;
}
diff --git a/rust/personhog-router/tests/common/mod.rs b/rust/personhog-router/tests/common/mod.rs
index c22cd5e4dd44..ad50e2555c47 100644
--- a/rust/personhog-router/tests/common/mod.rs
+++ b/rust/personhog-router/tests/common/mod.rs
@@ -740,6 +740,9 @@ pub struct TestLeaderService {
/// partition fence above this one names its holder, which is the fact
/// the refusal has to carry all the way back to the caller.
person_fence_op: Arc<Mutex<Option<String>>>,
+ /// Runs on every write before it is answered, so a test can change the
+ /// router's view at an exact point between its attempts.
+ on_write: Option<Arc<dyn Fn() + Send + Sync>>,
}
impl TestLeaderService {
@@ -748,9 +751,15 @@ impl TestLeaderService {
persons: DashMap::new(),
fenced: Arc::new(AtomicBool::new(false)),
person_fence_op: Arc::new(Mutex::new(None)),
+ on_write: None,
}
}
+ pub fn on_write(mut self, hook: impl Fn() + Send + Sync + 'static) -> Self {
+ self.on_write = Some(Arc::new(hook));
+ self
+ }
+
/// Refuse every write for this person the way the leader refuses one
/// held by a lifecycle operation: FAILED_PRECONDITION carrying the
/// fence keys and the holding op's id.
@@ -868,6 +877,9 @@ impl PersonHogLeader for TestLeaderService {
request: Request<UpdatePersonPropertiesRequest>,
) -> Result<Response<UpdatePersonPropertiesResponse>, Status> {
require_partition_metadata(&request)?;
+ if let Some(hook) = &self.on_write {
+ hook();
+ }
let req = request.into_inner();
if self.fenced.load(Ordering::SeqCst) {
return Err(Status::failed_precondition(
@@ -994,12 +1006,12 @@ fn make_leader_backend(leader_addr: SocketAddr, num_partitions: u32) -> Arc<Lead
routing_table,
address_resolver,
LeaderBackendConfig {
- changelog,
+ changelog: changelog.clone(),
timeout: Duration::from_secs(5),
num_channels: 1,
http2_windows: Http2Windows::default(),
},
- StashTable::with_bounds(usize::MAX, usize::MAX),
+ StashTable::with_bounds(usize::MAX, usize::MAX, changelog),
))
}
@@ -1029,12 +1041,12 @@ fn make_dying_leader_backend(leader_addr: SocketAddr, num_partitions: u32) -> Ar
routing_table,
address_resolver,
LeaderBackendConfig {
- changelog,
+ changelog: changelog.clone(),
timeout: Duration::from_millis(200),
num_channels: 1,
http2_windows: Http2Windows::default(),
},
- StashTable::with_bounds(usize::MAX, usize::MAX),
+ StashTable::with_bounds(usize::MAX, usize::MAX, changelog),
))
}
diff --git a/rust/personhog-router/tests/h2_window_wiring.rs b/rust/personhog-router/tests/h2_window_wiring.rs
index a6dc9634c6bd..aa58c743308c 100644
--- a/rust/personhog-router/tests/h2_window_wiring.rs
+++ b/rust/personhog-router/tests/h2_window_wiring.rs
@@ -143,12 +143,12 @@ async fn leader_channel(addr: SocketAddr, http2_windows: Http2Windows) -> Channe
Arc::new(RwLock::new(routing)),
Arc::new(move |_| Some(format!("http://{addr}"))),
LeaderBackendConfig {
- changelog,
+ changelog: changelog.clone(),
timeout: Duration::from_secs(5),
num_channels: 1,
http2_windows,
},
- StashTable::with_bounds(usize::MAX, usize::MAX),
+ StashTable::with_bounds(usize::MAX, usize::MAX, changelog),
);
backend.resolve_leader_channel(0).await.unwrap()
}
diff --git a/rust/personhog-router/tests/stash_integration.rs b/rust/personhog-router/tests/stash_integration.rs
index bb3a8af5fc6a..c20316bf084f 100644
--- a/rust/personhog-router/tests/stash_integration.rs
+++ b/rust/personhog-router/tests/stash_integration.rs
@@ -11,12 +11,15 @@ use std::sync::atomic::Ordering;
use std::sync::Arc;
use std::time::Duration;
+use assignment_coordination::store::{EtcdStore, StoreConfig};
use bytes::Bytes;
use common::{create_test_person, start_test_leader, start_test_leader_at, TestLeaderService};
use http::HeaderMap;
use http_body_util::BodyExt;
+use personhog_common::partitioning::partition_for_person;
use personhog_coordination::changelog::LiveChangelog;
use personhog_coordination::routing_table::StashHandler;
+use personhog_coordination::store::PersonhogStore;
use personhog_coordination::types::ChangelogConfig;
use personhog_proto::personhog::types::v1::{
FoldPersonDocumentRequest, FoldPersonDocumentResponse, GetPersonRequest, GetPersonResponse,
@@ -46,13 +49,38 @@ fn new_test_handler(backend: Arc<LeaderBackend>) -> RouterStashHandler {
RouterStashHandler::new(backend, TEST_MAX_STASH_WAIT, TEST_DRAIN_CONCURRENCY)
}
+/// A store under a fresh prefix, so no other test's config is visible.
+async fn test_store() -> Arc<PersonhogStore> {
+ let inner = EtcdStore::connect(StoreConfig {
+ endpoints: vec!["http://localhost:2379".to_string()],
+ prefix: format!("/stash-test-{}/", rand::random::<u64>()),
+ })
+ .await
+ .expect("failed to connect to etcd");
+ Arc::new(PersonhogStore::new(inner))
+}
+
+fn changelog() -> LiveChangelog {
+ LiveChangelog::new(ChangelogConfig::new("personhog_updates", NUM_PARTITIONS))
+}
+
/// Build a `LeaderBackend` whose routing table maps every partition at the
/// given `leader_addr` and whose stash uses the supplied bounds.
async fn make_backend(leader_addr: std::net::SocketAddr, stash: StashTable) -> Arc<LeaderBackend> {
+ backend_with(leader_addr, changelog(), stash, NUM_PARTITIONS).await
+}
+
+/// A backend routing partitions `0..routed` to `leader_addr`, reading the count from `changelog`.
+async fn backend_with(
+ leader_addr: std::net::SocketAddr,
+ changelog: LiveChangelog,
+ stash: StashTable,
+ routed: u32,
+) -> Arc<LeaderBackend> {
let routing_table = Arc::new(RwLock::new(HashMap::new()));
{
let mut table = routing_table.write().await;
- for partition in 0..NUM_PARTITIONS {
+ for partition in 0..routed {
table.insert(partition, "leader-0".to_string());
}
}
@@ -66,10 +94,7 @@ async fn make_backend(leader_addr: std::net::SocketAddr, stash: StashTable) -> A
routing_table,
resolver,
LeaderBackendConfig {
- changelog: LiveChangelog::new(ChangelogConfig::new(
- "personhog_updates",
- NUM_PARTITIONS,
- )),
+ changelog,
timeout: Duration::from_secs(5),
num_channels: 1,
http2_windows: Http2Windows::default(),
@@ -109,12 +134,10 @@ async fn forward(
backend: &LeaderBackend,
req: UpdatePersonPropertiesRequest,
) -> http::Response<BoxBody> {
- let partition = backend.partition_for_person(req.team_id, req.person_id);
let key = (req.team_id, req.person_id);
let (response, _call_ms) = backend
.forward_or_stash(
"UpdatePersonProperties",
- partition,
key,
HeaderMap::new(),
encode_frame(&req),
@@ -169,11 +192,9 @@ async fn forward_read(
buf.extend_from_slice(&(encoded.len() as u32).to_be_bytes());
buf.extend(encoded);
- let partition = backend.partition_for_person(team_id, person_id);
let (response, _call_ms) = backend
.forward_or_stash(
"GetPerson",
- partition,
(team_id, person_id),
HeaderMap::new(),
Bytes::from(buf),
@@ -191,7 +212,7 @@ async fn request_during_stash_completes_after_drain() {
let person = create_test_person();
let leader_addr = start_test_leader(TestLeaderService::new().with_person(person.clone())).await;
- let stash = StashTable::with_bounds(usize::MAX, usize::MAX);
+ let stash = StashTable::with_bounds(usize::MAX, usize::MAX, changelog());
let backend = make_backend(leader_addr, stash.clone()).await;
let handler = new_test_handler(Arc::clone(&backend));
@@ -241,7 +262,7 @@ async fn multiple_stashed_requests_drain_in_fifo() {
let person = create_test_person();
let leader_addr = start_test_leader(TestLeaderService::new().with_person(person.clone())).await;
- let stash = StashTable::with_bounds(usize::MAX, usize::MAX);
+ let stash = StashTable::with_bounds(usize::MAX, usize::MAX, changelog());
let backend = make_backend(leader_addr, stash.clone()).await;
let handler = new_test_handler(Arc::clone(&backend));
@@ -299,7 +320,7 @@ async fn requests_for_unstashed_partition_forward_immediately() {
let person = create_test_person();
let leader_addr = start_test_leader(TestLeaderService::new().with_person(person.clone())).await;
- let stash = StashTable::with_bounds(usize::MAX, usize::MAX);
+ let stash = StashTable::with_bounds(usize::MAX, usize::MAX, changelog());
let backend = make_backend(leader_addr, stash.clone()).await;
let handler = new_test_handler(Arc::clone(&backend));
@@ -331,7 +352,7 @@ async fn stash_full_returns_unavailable_via_backend() {
let leader_addr = start_test_leader(TestLeaderService::new().with_person(person.clone())).await;
// Tight cap: only one stashed message per partition.
- let stash = StashTable::with_bounds(1, usize::MAX);
+ let stash = StashTable::with_bounds(1, usize::MAX, changelog());
let backend = make_backend(leader_addr, stash.clone()).await;
let handler = new_test_handler(Arc::clone(&backend));
@@ -366,7 +387,7 @@ async fn back_to_back_handoffs_use_fresh_queue() {
let person = create_test_person();
let leader_addr = start_test_leader(TestLeaderService::new().with_person(person.clone())).await;
- let stash = StashTable::with_bounds(usize::MAX, usize::MAX);
+ let stash = StashTable::with_bounds(usize::MAX, usize::MAX, changelog());
let backend = make_backend(leader_addr, stash.clone()).await;
let handler = new_test_handler(Arc::clone(&backend));
@@ -428,7 +449,7 @@ async fn ordering_preserved_when_request_arrives_during_drain() {
let person = create_test_person();
let leader_addr = start_test_leader(TestLeaderService::new().with_person(person.clone())).await;
- let stash = StashTable::with_bounds(usize::MAX, usize::MAX);
+ let stash = StashTable::with_bounds(usize::MAX, usize::MAX, changelog());
let backend = make_backend(leader_addr, stash.clone()).await;
let handler = new_test_handler(Arc::clone(&backend));
@@ -497,7 +518,7 @@ async fn stash_wait_exceeded_returns_unavailable() {
let person = create_test_person();
let leader_addr = start_test_leader(TestLeaderService::new().with_person(person.clone())).await;
- let stash = StashTable::with_bounds(usize::MAX, usize::MAX);
+ let stash = StashTable::with_bounds(usize::MAX, usize::MAX, changelog());
let backend = make_backend(leader_addr, stash.clone()).await;
// Tight deadline — any request that ages past ~50ms in the stash
// gets failed fast on drain.
@@ -537,10 +558,9 @@ async fn stash_wait_exceeded_returns_unavailable() {
async fn a_semantic_refusal_passes_through_instead_of_bouncing() {
let person = create_test_person();
let leader_addr = start_test_leader(TestLeaderService::new().with_person(person.clone())).await;
- let stash = StashTable::with_bounds(usize::MAX, usize::MAX);
+ let stash = StashTable::with_bounds(usize::MAX, usize::MAX, changelog());
let backend = make_backend(leader_addr, stash).await;
- let partition = backend.partition_for_person(person.team_id, person.id);
let req = FoldPersonDocumentRequest {
team_id: person.team_id,
person_id: person.id,
@@ -555,7 +575,6 @@ async fn a_semantic_refusal_passes_through_instead_of_bouncing() {
let (response, _call_ms) = backend
.forward_or_stash(
"FoldPersonDocument",
- partition,
(person.team_id, person.id),
HeaderMap::new(),
encode_frame(&req),
@@ -588,7 +607,7 @@ async fn fence_bounce_parks_requests_until_a_later_drain_delivers() {
let fence = service.fence_flag();
let leader_addr = start_test_leader(service).await;
- let stash = StashTable::with_bounds(usize::MAX, usize::MAX);
+ let stash = StashTable::with_bounds(usize::MAX, usize::MAX, changelog());
let backend = make_backend(leader_addr, stash.clone()).await;
// Generous deadline so the deadline path can never produce a client
// error and mask a bounce that wrongly surfaced one.
@@ -645,7 +664,7 @@ async fn transport_failure_parks_requests_until_a_later_drain_delivers() {
let leader_addr = reserved.local_addr().unwrap();
drop(reserved);
- let stash = StashTable::with_bounds(usize::MAX, usize::MAX);
+ let stash = StashTable::with_bounds(usize::MAX, usize::MAX, changelog());
let backend = make_backend(leader_addr, stash.clone()).await;
// Generous deadline so the deadline path can never produce a client
// error and mask a bounce that wrongly surfaced one.
@@ -704,7 +723,7 @@ async fn a_live_write_rides_out_a_fence_via_router_retry() {
let fence = service.fence_flag();
let leader_addr = start_test_leader(service).await;
- let stash = StashTable::with_bounds(usize::MAX, usize::MAX);
+ let stash = StashTable::with_bounds(usize::MAX, usize::MAX, changelog());
let backend = make_backend(leader_addr, stash.clone()).await;
// The fence clears mid-retry, as it does in production once the
@@ -739,7 +758,7 @@ async fn a_live_write_past_the_bounce_budget_fails_unavailable() {
)
.await;
- let stash = StashTable::with_bounds(usize::MAX, usize::MAX);
+ let stash = StashTable::with_bounds(usize::MAX, usize::MAX, changelog());
let backend = make_backend(leader_addr, stash.clone()).await;
let req = mk_request(person.team_id, person.id, "[email protected]");
@@ -768,7 +787,7 @@ async fn a_bouncing_live_write_parks_once_the_stash_opens() {
let leader_addr = reserved.local_addr().unwrap();
drop(reserved);
- let stash = StashTable::with_bounds(usize::MAX, usize::MAX);
+ let stash = StashTable::with_bounds(usize::MAX, usize::MAX, changelog());
let backend = make_backend(leader_addr, stash.clone()).await;
let handler = RouterStashHandler::new(Arc::clone(&backend), Duration::from_secs(60), 4);
@@ -811,7 +830,7 @@ async fn stashed_strong_read_observes_stashed_write() {
let person = create_test_person();
let leader_addr = start_test_leader(TestLeaderService::new().with_person(person.clone())).await;
- let stash = StashTable::with_bounds(usize::MAX, usize::MAX);
+ let stash = StashTable::with_bounds(usize::MAX, usize::MAX, changelog());
let backend = make_backend(leader_addr, stash.clone()).await;
let handler = new_test_handler(Arc::clone(&backend));
@@ -861,7 +880,7 @@ async fn stashed_read_past_deadline_returns_unavailable() {
let person = create_test_person();
let leader_addr = start_test_leader(TestLeaderService::new().with_person(person.clone())).await;
- let stash = StashTable::with_bounds(usize::MAX, usize::MAX);
+ let stash = StashTable::with_bounds(usize::MAX, usize::MAX, changelog());
let backend = make_backend(leader_addr, stash.clone()).await;
let handler = RouterStashHandler::new(Arc::clone(&backend), Duration::from_millis(50), 4);
@@ -903,7 +922,7 @@ async fn drain_converges_with_concurrent_arrivals() {
let person = create_test_person();
let leader_addr = start_test_leader(TestLeaderService::new().with_person(person.clone())).await;
- let stash = StashTable::with_bounds(usize::MAX, usize::MAX);
+ let stash = StashTable::with_bounds(usize::MAX, usize::MAX, changelog());
let backend = make_backend(leader_addr, stash.clone()).await;
let handler = Arc::new(new_test_handler(Arc::clone(&backend)));
@@ -983,7 +1002,7 @@ async fn drain_cancellation_returns_promptly_and_puts_the_entry_back() {
}
});
- let stash = StashTable::with_bounds(usize::MAX, usize::MAX);
+ let stash = StashTable::with_bounds(usize::MAX, usize::MAX, changelog());
let backend = make_backend(leader_addr, stash).await;
let handler = new_test_handler(Arc::clone(&backend));
@@ -1022,3 +1041,152 @@ async fn drain_cancellation_returns_promptly_and_puts_the_entry_back() {
"the abandoned entry must be put back for the next drain"
);
}
+
+/// A person whose partition differs between the current and the grown count.
+fn moving_person() -> i64 {
+ (1..)
+ .find(|id| {
+ partition_for_person(1, *id, NUM_PARTITIONS)
+ != partition_for_person(1, *id, NUM_PARTITIONS * 2)
+ })
+ .unwrap()
+}
+
+fn grown_config() -> ChangelogConfig {
+ ChangelogConfig {
+ epoch: 2,
+ ..ChangelogConfig::new("personhog_updates_v2", NUM_PARTITIONS * 2)
+ }
+}
+
+/// The config watch adopts a flip from etcd, and the new partition's drain
+/// delivers a parked key once the old partition's drain gives it back.
+#[tokio::test]
+async fn a_flip_in_etcd_moves_a_parked_key_to_its_new_partition() {
+ let store = test_store().await;
+ assert!(store
+ .seed_changelog_config("personhog_updates", NUM_PARTITIONS, "personhog-writer")
+ .await
+ .unwrap());
+ let person = Person {
+ id: moving_person(),
+ ..create_test_person()
+ };
+ let leader_addr = start_test_leader(TestLeaderService::new().with_person(person.clone())).await;
+ let changelog = changelog();
+ let stash = StashTable::with_bounds(usize::MAX, usize::MAX, changelog.clone());
+ let backend = backend_with(
+ leader_addr,
+ changelog.clone(),
+ stash.clone(),
+ NUM_PARTITIONS * 2,
+ )
+ .await;
+ let handler = RouterStashHandler::new(Arc::clone(&backend), Duration::from_secs(60), 4);
+ let old_partition = partition_for_person(1, person.id, NUM_PARTITIONS);
+ let new_partition = partition_for_person(1, person.id, NUM_PARTITIONS * 2);
+
+ stash.begin_stash(old_partition);
+ let backend_for_call = Arc::clone(&backend);
+ let req = mk_request(1, person.id, "[email protected]");
+ let pending = tokio::spawn(async move { forward(&backend_for_call, req).await });
+ let holder = stash.drain_session(old_partition).unwrap();
+ let mut held = tokio::time::timeout(Duration::from_secs(5), async {
+ loop {
+ let taken = holder.take_for_attempt(usize::MAX);
+ if !taken.is_empty() {
+ return taken;
+ }
+ tokio::time::sleep(Duration::from_millis(10)).await;
+ }
+ })
+ .await
+ .expect("the old partition's drain takes the parked request");
+
+ store.put_changelog_config(&grown_config()).await.unwrap();
+ let cancel = CancellationToken::new();
+ tokio::spawn(personhog_coordination::changelog::keep_changelog_live(
+ Arc::clone(&store),
+ changelog.clone(),
+ cancel.clone(),
+ ));
+ tokio::time::timeout(Duration::from_secs(5), async {
+ while !stash.is_pending(new_partition) {
+ tokio::time::sleep(Duration::from_millis(10)).await;
+ }
+ })
+ .await
+ .expect("the parked key belongs to its partition under the adopted count");
+
+ let mut drain = tokio::spawn(async move {
+ handler
+ .drain_stash(new_partition, "leader-0", CancellationToken::new())
+ .await
+ });
+ assert!(
+ tokio::time::timeout(Duration::from_millis(200), &mut drain)
+ .await
+ .is_err(),
+ "the new drain waits while the old one holds the key"
+ );
+ let run = held.remove(0);
+ holder.put_back(run.key, run.entries);
+
+ tokio::time::timeout(Duration::from_secs(5), drain)
+ .await
+ .expect("the drain settles once it delivers the key")
+ .expect("drain task must not panic")
+ .expect("drain succeeds");
+ decode_response::<UpdatePersonPropertiesResponse>(pending.await.unwrap())
+ .await
+ .expect("the parked write is delivered through the new partition");
+ cancel.cancel();
+}
+
+/// A retry after a bounce recomputes the partition, so it sees the freeze
+/// on the key's new partition instead of forwarding under the old count.
+#[tokio::test]
+async fn a_retry_after_a_bounce_routes_by_the_adopted_count() {
+ let person_id = moving_person();
+ let new_partition = partition_for_person(1, person_id, NUM_PARTITIONS * 2);
+ let changelog = changelog();
+ let stash = StashTable::with_bounds(usize::MAX, usize::MAX, changelog.clone());
+ let flipped = Arc::new(std::sync::atomic::AtomicBool::new(false));
+ let service = TestLeaderService::new().fenced().on_write({
+ let (changelog, stash) = (changelog.clone(), stash.clone());
+ move || {
+ if !flipped.swap(true, Ordering::SeqCst) {
+ changelog.adopt(&grown_config());
+ stash.begin_stash(new_partition);
+ }
+ }
+ });
+ let leader_addr = start_test_leader(service).await;
+ let backend = backend_with(leader_addr, changelog, stash.clone(), NUM_PARTITIONS * 2).await;
+
+ let req = mk_request(1, person_id, "[email protected]");
+ let pending = tokio::spawn(async move { forward(&backend, req).await });
+ let session = tokio::time::timeout(Duration::from_secs(5), async {
+ loop {
+ if let Some(session) = stash.drain_session(new_partition) {
+ return session;
+ }
+ tokio::time::sleep(Duration::from_millis(10)).await;
+ }
+ })
+ .await
+ .expect("the first attempt freezes the new partition");
+ let taken = tokio::time::timeout(Duration::from_secs(5), async {
+ loop {
+ let taken = session.take_for_attempt(usize::MAX);
+ if !taken.is_empty() {
+ return taken;
+ }
+ tokio::time::sleep(Duration::from_millis(10)).await;
+ }
+ })
+ .await
+ .expect("the retry parks behind the new partition's freeze");
+ assert_eq!(taken[0].key, (1, person_id));
+ assert!(!pending.is_finished());
+}