:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
//! Per-partition Stage 1 worker.
//!
//! [`Stage1Worker::spawn`] drains one partition's two lanes on a dedicated tokio task. Per live
//! sub-batch it hands its membership changes to a bounded `ProduceWindow`, produces straggler
//! re-keys and awaits their acks, then marks the offset processed. Live output is at most once, so
//! the mark does not wait for the window's acks. A failed re-key skips the mark; a store error is
//! logged and the event is skipped. Between live sub-batches it applies at most one seed run: a
//! quantum of seeds leaves the lane only once the previous quantum has fully applied, and its runs
//! go one per round, so live latency is at most one run and an admitted seed backlog never queues
//! in front of live traffic.
//!
//! A `Sweep` message records an eviction request rather than running one. The pass then advances one
//! bounded batch per round, alternating with live batches and running ahead of seeds
//! ([`crate::workers::sweep_path`]), so a wave of deadlines coming due together costs live traffic
//! one batch at a time instead of the whole wave at once.
use std::borrow::Cow;
use std::collections::VecDeque;
use std::sync::Arc;
use std::time::{Duration, Instant};
use cohort_core::seed::CoverageStartMs;
use futures::FutureExt;
use metrics::{counter, gauge, histogram, Histogram};
use tokio::task::JoinHandle;
use tracing::{error, info, warn};
use uuid::Uuid;
use crate::cascade::{first_cascade, CascadeMessage};
use crate::consumers::events::CohortStreamEvent;
use crate::consumers::readiness::RebuildGuard;
use crate::consumers::seeds::{ConsumedSeed, SeedWork};
use crate::filters::manager::CatalogHandle;
use crate::filters::reverse_index::TeamFilters;
use crate::filters::TeamId;
use crate::merge::tombstone_redirect::{self, Resolution};
use crate::observability::metrics::{
CASCADE_PRODUCE_ERRORS_TOTAL, COHORT_STREAM_OFFSET_AHEAD_OF_DISPATCH,
EVICTION_QUEUE_REBUILT_KEYS_TOTAL, MERGE_REDIRECT_HOP_CAPPED_TOTAL,
MERGE_REKEY_PRODUCE_FAILURE_TOTAL, OUTPUT_MEMBERSHIP_CHANGES_EMITTED, OUTPUT_PRODUCE_ERRORS,
OUTPUT_WINDOW_WAITS_TOTAL, SLICES_BEGUN_TOTAL, SLICE_COVERED_SINCE_SECONDS,
STAGE1_EVENTS_PROCESSED, STAGE1_EVENTS_SKIPPED, STAGE1_EVENT_PROCESS_DURATION,
STAGE1_REDIRECT_LOOKUP_DURATION, STAGE1_STATE_DECODE_ERROR, STAGE1_TRANSITIONS,
};
use crate::partitions::offset_tracker::{MarkOutcome, OffsetTracker};
use crate::partitions::router::WorkerInbox;
use crate::partitions::shuffle_message::ShuffleMessage;
use crate::producer::{
map_transition, CohortMembershipChange, LastUpdatedClock, MembershipSink, MembershipStatus,
OutputBuffer,
};
use crate::stage1::key::LeafStateKey;
use crate::stage1::state::{StateVariant, StatefulRecord};
use crate::stage1::transition::{LeafTransition, TransitionKind};
use crate::store::{BehavioralKey, ReadLane, SliceCoverage, SliceTenure, StoreHandle};
use crate::sweep::EvictionQueue;
use crate::workers::cascade_path::handle_cascades;
use crate::workers::event_path::{
process_event_offloaded, schedule_deadline, EventNameGating, SkipReason,
};
use crate::workers::merge_gc::{handle_merge_gc, MergeGcCursor};
use crate::workers::merge_path::{handle_apply, handle_merge, handle_redrive, MergeWorkerDeps};
use crate::workers::reconcile::{handle_reconcile_drain, ReconcileQueue};
use crate::workers::seed_apply::{apply_seed_group, ApplyDeps, BatchMarks};
use crate::workers::seed_run::{group_seeds, row_weight, Admitted, SeedGroup};
use crate::workers::stage2_gc::{handle_stage2_orphan_gc, Stage2GcCursor};
use crate::workers::stage2_path::compose_stage2;
use crate::workers::sweep_path::{run_sweep_turn, SweepSchedule};
const REBUILD_SCAN_PAGE: usize = 10_000;
/// Cooperative-yield cadence inside the worker fold. `handle_event` is synchronous, so a backlog of
/// CPU-bound events would hold the runtime thread, starving the commit task and consume loop. A
/// wall-clock interval adapts to per-event cost across catalogs of different sizes.
const WORKER_YIELD_INTERVAL: Duration = Duration::from_millis(5);
/// Unacked live output batches a worker allows before it waits. Sized for a few broker round trips
/// of a partition's peak live output; `output_window_waits_total` says whether it binds.
const PRODUCE_WINDOW: usize = 4;
pub struct Stage1Worker {
partition_id: u16,
handle: JoinHandle<()>,
}
/// Whether a spawning worker rebuilds its `EvictionQueue` from `cf_behavioral`, so a dormant
/// person's `Left` still fires after a restart. A rebuild needs a [`RebuildGuard`], so the readiness
/// gate counts every rebuild that runs.
pub enum EvictionRestore {
Skip,
Rebuild(RebuildGuard),
}
impl Stage1Worker {
/// Spawn with event-name gating disabled. The gating-aware variant is [`Self::spawn_with_gating`].
#[allow(clippy::too_many_arguments)]
pub fn spawn(
partition_id: u16,
inbox: WorkerInbox,
store: StoreHandle,
catalog: Arc<CatalogHandle>,
sink: Arc<dyn MembershipSink>,
tracker: Arc<OffsetTracker>,
merge: Arc<MergeWorkerDeps>,
restore: EvictionRestore,
) -> Self {
Self::spawn_with_gating(
partition_id,
inbox,
store,
catalog,
sink,
tracker,
merge,
restore,
EventNameGating::Disabled,
CoverageStartMs::now(),
)
}
#[allow(clippy::too_many_arguments)]
pub fn spawn_with_gating(
partition_id: u16,
inbox: WorkerInbox,
store: StoreHandle,
catalog: Arc<CatalogHandle>,
sink: Arc<dyn MembershipSink>,
tracker: Arc<OffsetTracker>,
merge: Arc<MergeWorkerDeps>,
restore: EvictionRestore,
event_name_gating: EventNameGating,
assigned_at: CoverageStartMs,
) -> Self {
let handle = tokio::spawn(run_worker(
partition_id,
inbox,
store,
catalog,
sink,
tracker,
merge,
restore,
event_name_gating,
assigned_at,
));
Self {
partition_id,
handle,
}
}
pub fn partition_id(&self) -> u16 {
self.partition_id
}
pub async fn join(self) -> Result<(), tokio::task::JoinError> {
self.handle.await
}
}
/// What one [`run_worker`] select round yielded, so the loop body is a plain `match`.
enum Turn {
/// A live sub-batch, or `None` for a closed live lane.
Live(Option<Vec<ShuffleMessage>>),
/// One bounded eviction batch of the pending sweep work.
Sweep,
/// The pending quantum has a run to apply.
Run,
/// Seeds taken from the lane this round; `0` means the lane closed.
Seeds(usize),
}
/// One quantum's runs, applied one per round so live batches interleave with them. The marks are
/// published only after the last run (see [`BatchMarks`]); a hold publishes as it happens.
#[derive(Default)]
struct PendingRuns {
groups: VecDeque<SeedGroup>,
marks: BatchMarks,
}
/// Live output batches handed to the producer whose acks the worker has not collected yet.
///
/// Live output is at most once and its state is already committed, so the offset mark does not
/// wait for the ack. The bound is for a broker stall: with [`PRODUCE_WINDOW`] batches unacked the
/// worker waits, so the stall shows as lag on this partition instead of as output held in memory
/// until the message timeout drops it.
///
/// [`submit`](Self::submit) enqueues a batch's records before it returns, and only the acks are
/// collected on a spawned task. So batches are enqueued in submit order, ahead of anything the
/// worker produces next. A producer retry can still reorder batches that are in flight together.
/// With version writes on, the membership consumer's version check puts them back in order. With
/// them off, an older change can win until the next change for that cohort and person.
struct ProduceWindow {
partition_id: u16,
sink: Arc<dyn MembershipSink>,
merge: Arc<MergeWorkerDeps>,
in_flight: VecDeque<JoinHandle<()>>,
}
impl ProduceWindow {
fn new(partition_id: u16, sink: Arc<dyn MembershipSink>, merge: Arc<MergeWorkerDeps>) -> Self {
Self {
partition_id,
sink,
merge,
in_flight: VecDeque::new(),
}
}
/// Hand the buffered changes, if any, to the producer. Waits only while the window is full.
///
/// The worker submits before each merge, transfer, cascade, sweep or reconcile message and at
/// the end of each live sub-batch, so batches enter the window in state-commit order.
async fn submit(&mut self, buffer: &mut OutputBuffer, max_offset: Option<i64>) {
let Some(changes) = buffer.take() else {
return;
};
while self.in_flight.front().is_some_and(JoinHandle::is_finished) {
self.settle_oldest().await;
}
if self.in_flight.len() >= PRODUCE_WINDOW {
counter!(OUTPUT_WINDOW_WAITS_TOTAL).increment(1);
self.settle_oldest().await;
}
let mut produce = Box::pin(produce_live(
self.partition_id,
Arc::clone(&self.sink),
Arc::clone(&self.merge),
changes,
max_offset.unwrap_or(0),
));
// The first poll enqueues every record, because the producer's enqueue is synchronous and
// only the acks are awaited. A spawned task first runs when the worker yields, which can be
// after the worker's next inline produce. Enqueueing here keeps Kafka order equal to
// state-commit order except across a producer retry, and Kafka order is what the membership
// consumer applies when its version writes are off.
if produce.as_mut().now_or_never().is_none() {
self.in_flight.push_back(tokio::spawn(produce));
}
}
/// Collect every outstanding produce, so a stopped worker leaves no output of its tenure in
/// flight.
async fn drain(mut self) {
while !self.in_flight.is_empty() {
self.settle_oldest().await;
}
}
async fn settle_oldest(&mut self) {
let Some(produce) = self.in_flight.pop_front() else {
return;
};
if let Err(error) = produce.await {
error!(
partition_id = self.partition_id,
error = %error,
"live produce task failed; its output is lost",
);
}
}
}
/// Produce one live batch's membership changes and their first-hop cascades. A failed leg is
/// counted and logged but never retried, because the batch's state is already committed and a
/// replay finds nothing left to emit.
async fn produce_live(
partition_id: u16,
sink: Arc<dyn MembershipSink>,
merge: Arc<MergeWorkerDeps>,
changes: Vec<CohortMembershipChange>,
source_offset: i64,
) {
let produced = changes.len();
// Built from a borrow before `changes` moves: the cascade payload embeds the change.
let cascades = first_cascades(&merge, &changes, source_offset);
let (membership_errors, cascade_errors) = tokio::join!(
produce_membership(&sink, changes),
produce_cascades(&merge, cascades),
);
if membership_errors + cascade_errors > 0 {
warn!(
partition_id,
changes = produced,
membership_errors,
cascade_errors,
"live produce failed; the changes are lost, and only a reconcile re-emits them",
);
}
}
#[allow(clippy::too_many_arguments)]
async fn run_worker(
partition_id: u16,
inbox: WorkerInbox,
handle: StoreHandle,
catalog: Arc<CatalogHandle>,
sink: Arc<dyn MembershipSink>,
tracker: Arc<OffsetTracker>,
merge: Arc<MergeWorkerDeps>,
restore: EvictionRestore,
event_name_gating: EventNameGating,
assigned_at: CoverageStartMs,
) {
info!(partition_id, "stage 1 worker started");
let mut queue = EvictionQueue::<BehavioralKey>::new();
let mut sweep = SweepSchedule::new(partition_id);
// Before any message, so the slice holds every event from the assignment on.
let coverage = resolve_slice_coverage(partition_id, assigned_at, &handle).await;
let mut reconcile_queue = ReconcileQueue::new(
partition_id,
merge.reconcile.backlog.clone(),
handle.clone(),
catalog.clone(),
coverage,
);
// No-op for a cold partition (bloom-filtered scan finds nothing to schedule). A failed scan logs
// and stops early, so the guard drops and the readiness gate cannot stick on a store error.
match restore {
EvictionRestore::Skip => {}
EvictionRestore::Rebuild(rebuilding) => {
rebuild_eviction_queue(partition_id, &handle, &mut queue).await;
drop(rebuilding);
}
}
// In-memory resume cursors; loss on rebalance is benign (GC re-scans from the start).
let mut gc_cursor = MergeGcCursor::default();
let mut stage2_gc_cursor = Stage2GcCursor::default();
// Persists across batches so a stream of buffered batches still yields on the wall-clock interval.
let mut last_yield = Instant::now();
let mut last_updated_clock = LastUpdatedClock::default();
let mut window = ProduceWindow::new(partition_id, Arc::clone(&sink), Arc::clone(&merge));
let event_partition = EventPartition::new(partition_id);
// Two locals, not `inbox.live`/`inbox.seeds`: both branch futures live at once inside the
// macro, so they must borrow disjoint places.
let WorkerInbox {
mut live,
mut seeds,
} = inbox;
// The seed quantum, from the run ceiling: one turn takes at most one run's worth of seeds.
// The row budget, a kind change or a control seed can still cut a quantum into several runs,
// which then apply one per round. `NonZeroUsize` on purpose: `recv_many` with a limit of 0
// returns 0 at once, which reads here as a closed lane.
let quantum = merge.seed_budget.seeds.get();
// Grown by `recv_many` as seeds arrive, not sized up front: the run ceiling is a config knob,
// and an oversized one must not reserve that many seeds per worker at spawn.
let mut seed_buf: Vec<ConsumedSeed> = Vec::new();
let mut pending = PendingRuns::default();
let (mut live_open, mut seed_open) = (true, true);
loop {
// Live before seeds: a seed run is bounded work, a live backlog is not. A seed branch is
// reached only while the live lane is empty, so live latency is at most one seed run. The
// eviction sweep sits between the two: it is maintenance the queue already owes, not
// admitted backlog, so it runs ahead of seeds, and it alternates with live rather than
// queueing behind it.
//
// A partition whose live lane is never empty makes no seed progress, by design: live
// arrivals there already outpace the worker, a seed run would only deepen that backlog,
// and the seed consumer's live-lag gate stops admitting to it. Its lane then fills and
// `try_route_seeds` refuses, which is the alarm: `cohort_seed_paused_partitions` under
// cause `channel_full` and `cohort_seed_oldest_held_age_ms` rise while
// `cohort_seed_apply_run_size` goes flat.
//
// A draining sweep pass reads the same way for as long as it lasts, since it too sits ahead
// of both seed branches. Check `sweep_queue_lag_seconds` before concluding the live lane is
// the cause: a pass is bounded and clears, a live backlog does not.
let turn = tokio::select! {
biased;
// Ahead of live, but only for the one turn a live batch just earned it: eviction and live
// then alternate, so a lane that is never empty cannot hold a due wave indefinitely.
() = std::future::ready(()), if sweep.owes_turn() => Turn::Sweep,
batch = live.recv(), if live_open => Turn::Live(batch),
// Live is idle, so the pass keeps going rather than waiting for the next tick.
() = std::future::ready(()), if sweep.has_work() => Turn::Sweep,
// Ready at once: the quantum's remaining runs go one per round, behind any live batch.
() = std::future::ready(()), if !pending.groups.is_empty() => Turn::Run,
// The lane is polled again only once the previous quantum has fully applied.
taken = seeds.recv_many(&mut seed_buf, quantum),
if seed_open && pending.groups.is_empty() => Turn::Seeds(taken),
// Both lanes are closed and drained and no run is pending, so nothing more can arrive.
else => break,
};
let batch = match turn {
Turn::Live(Some(batch)) => batch,
Turn::Live(None) => {
live_open = false;
continue;
}
Turn::Seeds(0) => {
seed_open = false;
continue;
}
// `recv_many` appends, so drain it: otherwise turn n re-groups turns 1..n-1.
Turn::Seeds(_) => {
debug_assert!(
pending.groups.is_empty(),
"a quantum is taken only once the last one applied"
);
pending = group_seed_turn(
partition_id,
&catalog,
&merge,
seed_buf.drain(..).map(Admitted::from).collect(),
);
continue;
}
Turn::Sweep => {
run_sweep_turn(
partition_id,
&handle,
&catalog,
&sink,
&merge,
&mut queue,
&mut last_updated_clock,
&mut sweep,
)
.await;
continue;
}
Turn::Run => {
apply_next_run(
partition_id,
&handle,
&catalog,
&sink,
&merge,
&mut queue,
&mut reconcile_queue,
&mut last_updated_clock,
&mut pending,
)
.await;
if pending.groups.is_empty() {
// The whole quantum applied: its marks can publish, and the lane's meter stops
// counting the seeds it took.
std::mem::take(&mut pending.marks).publish(&merge.seed_tracker, partition_id);
seeds.finish_turn();
}
continue;
}
};
// Claimed before the batch runs, not after: every exit path below (re-key failure, clean
// mark) must still yield the next turn, or a partition that keeps failing to re-key would
// starve eviction.
sweep.live_batch_done();
let mut buffer = OutputBuffer::new();
let mut re_keys: Vec<CohortStreamEvent> = Vec::new();
let mut max_offset: Option<i64> = None;
// Observed into the live watermarks only post-mark, so a held batch never advances the
// seed fence.
let mut max_broker_ts: Option<i64> = None;
let mut messages = batch.into_iter().peekable();
while let Some(message) = messages.next() {
let last_updated = last_updated_clock.next();
match message {
ShuffleMessage::Event {
event,
cse_offset,
broker_ts_ms,
} => {
max_offset =
Some(max_offset.map_or(cse_offset, |current| current.max(cse_offset)));
max_broker_ts = max_broker_ts.max(broker_ts_ms);
let effects = handle_event(
&event_partition,
&handle,
&catalog,
&event,
&last_updated,
merge.partition_count,
event_name_gating,
)
.await;
buffer.extend(effects.changes);
for (key, deadline) in effects.schedules {
queue.schedule(key, deadline);
}
re_keys.extend(effects.re_keys);
}
ShuffleMessage::Sweep { due_before_ms } => {
// The eviction runs on a later turn, but the buffer is still submitted here,
// ahead of the request that may retract this batch's own `Entered` output.
window.submit(&mut buffer, max_offset).await;
sweep.request(due_before_ms, &queue);
}
ShuffleMessage::Merge { event, offset } => {
window.submit(&mut buffer, max_offset).await;
handle_merge(
partition_id,
&handle,
&catalog,
&sink,
&merge,
&mut queue,
&last_updated,
&event,
offset,
)
.await;
}
ShuffleMessage::Transfer { transfer, offset } => {
window.submit(&mut buffer, max_offset).await;
handle_apply(
partition_id,
&handle,
&catalog,
&sink,
&merge,
&mut queue,
&last_updated,
&transfer,
offset,
)
.await;
}
ShuffleMessage::Cascade { message, offset } => {
// A sub-batch carries one topic, so the run is normally the whole sub-batch.
let mut run = vec![(message, offset)];
while let Some(ShuffleMessage::Cascade { message, offset }) =
messages.next_if(|next| matches!(next, ShuffleMessage::Cascade { .. }))
{
run.push((message, offset));
}
window.submit(&mut buffer, max_offset).await;
handle_cascades(
partition_id,
&handle,
&catalog,
&sink,
&merge,
&mut last_updated_clock,
run,
)
.await;
}
ShuffleMessage::RedrivePendingTransfers => {
handle_redrive(partition_id, &handle, &merge).await;
}
ShuffleMessage::MergeCfGc {
marker_cutoff_ms,
tombstone_cutoff_ms,
} => {
// The cursor moves into the section and back out by value; a teardown
// cancellation resets it to `Default`, which is benign — the GC re-scans from the
// prefix start next tenure.
let scan_limit = merge.gc_scan_limit;
let mut cursor = std::mem::take(&mut gc_cursor);
gc_cursor = handle
.run_section("merge_gc", move |store| {
handle_merge_gc(
partition_id,
store,
&mut cursor,
marker_cutoff_ms,
tombstone_cutoff_ms,
scan_limit,
);
cursor
})
.await
.unwrap_or_default();
if merge.stage2_orphan_gc_enabled {
let catalog = catalog.clone();
let mut cursor = std::mem::take(&mut stage2_gc_cursor);
stage2_gc_cursor = handle
.run_section("stage2_orphan_gc", move |store| {
handle_stage2_orphan_gc(
partition_id,
store,
&catalog,
&mut cursor,
scan_limit,
);
cursor
})
.await
.unwrap_or_default();
}
}
ShuffleMessage::ReconcileDrain => {
window.submit(&mut buffer, max_offset).await;
handle_reconcile_drain(
partition_id,
&handle,
&sink,
&merge,
&mut reconcile_queue,
&last_updated,
)
.await;
}
}
if last_yield.elapsed() >= WORKER_YIELD_INTERVAL {
tokio::task::yield_now().await;
last_yield = Instant::now();
}
}
window.submit(&mut buffer, max_offset).await;
if !re_keys.is_empty() {
let produced = re_keys.len() as u64;
let acks = merge.stream_event_sink.produce(re_keys).await;
let errors = acks.iter().filter(|result| result.is_err()).count();
if errors > 0 {
counter!(MERGE_REKEY_PRODUCE_FAILURE_TOTAL).increment(errors as u64);
warn!(
partition_id,
errors,
"straggler re-key produce to cohort_stream_events failed; holding offset for replay",
);
continue;
}
tombstone_redirect::record_re_keyed(produced);
}
if let Some(max_offset) = max_offset {
if let MarkOutcome::CappedAheadOfDispatch =
tracker.mark_processed(partition_id as i32, max_offset + 1)
{
counter!(COHORT_STREAM_OFFSET_AHEAD_OF_DISPATCH).increment(1);
warn!(
partition_id,
next_offset = max_offset + 1,
"offset mark exceeded the dispatch ceiling and was capped (F1 invariant violation)",
);
}
}
// Strictly post-mark: a consume-time watermark would reopen the double-count branch the
// fence deletes.
if let Some(broker_ts_ms) = max_broker_ts {
merge
.live_watermarks
.observe(partition_id as i32, broker_ts_ms);
}
}
window.drain().await;
info!(partition_id, "stage 1 worker stopped");
}
/// Split one quantum into the runs it will apply, one per worker round.
///
/// No live-buffer submit is needed first: the live `OutputBuffer` is submitted at the end of every
/// live sub-batch, so it is always empty when a seed run starts.
fn group_seed_turn(
partition_id: u16,
catalog: &CatalogHandle,
merge: &MergeWorkerDeps,
seeds: Vec<Admitted<SeedWork>>,
) -> PendingRuns {
// Worker receipt is the first point that both proves these exact seeds landed and orders their
// ceiling before even a zero-work run can mark them processed. The dispatcher also records the
// delivered batch maximum, but that post-send accounting can race a fast worker on a
// multithreaded runtime.
if let Some(max) = seeds.iter().map(|seed| seed.offset.0).max() {
merge
.seed_tracker
.mark_dispatched(partition_id as i32, max + 1);
}
let snapshot = catalog.load();
let groups = group_seeds(seeds, merge.seed_budget, |work| row_weight(&snapshot, work));
PendingRuns {
groups: groups.into(),
marks: BatchMarks::default(),
}
}
/// Apply the pending quantum's next run. Each run folds under its own catalog snapshot, so a
/// catalog refresh between two runs of one quantum is the same edit race a live batch runs under.
#[allow(clippy::too_many_arguments)]
async fn apply_next_run(
partition_id: u16,
handle: &StoreHandle,
catalog: &CatalogHandle,
sink: &Arc<dyn MembershipSink>,
merge: &MergeWorkerDeps,
queue: &mut EvictionQueue<BehavioralKey>,
reconcile_queue: &mut ReconcileQueue,
clock: &mut LastUpdatedClock,
pending: &mut PendingRuns,
) {
let Some(group) = pending.groups.pop_front() else {
return;
};
let snapshot = catalog.load();
let deps = ApplyDeps {
partition_id,
handle,
catalog: &snapshot,
sink,
merge,
};
apply_seed_group(
deps,
queue,
reconcile_queue,
clock,
&mut pending.marks,
group,
)
.await;
}
pub(crate) fn count_by_status(changes: &[CohortMembershipChange]) -> (u64, u64) {
changes
.iter()
.fold((0, 0), |(entered, left), change| match change.status {
MembershipStatus::Entered => (entered + 1, left),
MembershipStatus::Left => (entered, left + 1),
})
}
/// Produce membership `changes`, await one ack per change, and record metrics. Returns the failed
/// or missing/extra-ack count (`0` = fully acked). The caller owns the per-site warn and recovery
/// action.
#[hotpath::measure]
pub(crate) async fn produce_membership(
sink: &Arc<dyn MembershipSink>,
changes: Vec<CohortMembershipChange>,
) -> usize {
let (entered, left) = count_by_status(&changes);
let expected_acks = changes.len();
let acks = sink.produce(changes).await;
let errors =
acks.iter().filter(|result| result.is_err()).count() + acks.len().abs_diff(expected_acks);
if errors > 0 {
counter!(OUTPUT_PRODUCE_ERRORS).increment(errors as u64);
return errors;
}
if entered > 0 {
counter!(OUTPUT_MEMBERSHIP_CHANGES_EMITTED, "status" => MembershipStatus::Entered.as_str())
.increment(entered);
}
if left > 0 {
counter!(OUTPUT_MEMBERSHIP_CHANGES_EMITTED, "status" => MembershipStatus::Left.as_str())
.increment(left);
}
0
}
/// Build the first-hop cascade messages for each membership change. Empty when the gate is off or
/// the slice is empty, so the off path allocates nothing. Must be called before `changes` is moved
/// into `produce_membership`. `source_offset` is informational only.
pub(crate) fn first_cascades(
merge: &MergeWorkerDeps,
changes: &[CohortMembershipChange],
source_offset: i64,
) -> Vec<CascadeMessage> {
if !merge.cascade.enabled || changes.is_empty() {
return Vec::new();
}
changes
.iter()
.map(|change| first_cascade(change.clone(), source_offset))
.collect()
}
/// Produce cascade messages and await one ack per message. Returns the failed or
/// missing/extra-ack count (`0` when empty or fully acked). The caller owns the recovery posture:
/// the live, sweep and merge paths drop (at-most-once); the cascade consumer and seed apply hold
/// the offset. Shared by the first-hop leg and onward-hop consumer legs.
#[hotpath::measure]
pub(crate) async fn produce_cascades(
merge: &MergeWorkerDeps,
cascades: Vec<CascadeMessage>,
) -> usize {
if cascades.is_empty() {
return 0;
}
let expected_acks = cascades.len();
let acks = merge.cascade_sink.produce(cascades).await;
let errors =
acks.iter().filter(|result| result.is_err()).count() + acks.len().abs_diff(expected_acks);
if errors > 0 {
counter!(CASCADE_PRODUCE_ERRORS_TOTAL).increment(errors as u64);
}
errors
}
#[derive(Default)]
struct EventEffects {
changes: Vec<CohortMembershipChange>,
schedules: Vec<(BehavioralKey, i64)>,
re_keys: Vec<CohortStreamEvent>,
}
/// The partition a worker folds events on, with the meters labelled by it. The worker builds it
/// once, so each event records into an already-registered handle instead of a per-event registry
/// lookup, and [`Self::new`] derives the `partition` label from the same id the events fold under.
struct EventPartition {
id: u16,
redirect_lookup: Histogram,
}
impl EventPartition {
fn new(id: u16) -> Self {
let label: Arc<str> = Arc::from(id.to_string());
Self {
id,
redirect_lookup: histogram!(STAGE1_REDIRECT_LOOKUP_DURATION, "partition" => label),
}
}
}
#[allow(clippy::too_many_arguments)]
#[hotpath::measure]
async fn handle_event(
partition: &EventPartition,
handle: &StoreHandle,
catalog: &CatalogHandle,
event: &CohortStreamEvent,
last_updated: &str,
partition_count: u32,
event_name_gating: EventNameGating,
) -> EventEffects {
let partition_id = partition.id;
let snapshot = catalog.load();
let Some(team_filters) = snapshot.team(TeamId(event.team_id)) else {
counter!(STAGE1_EVENTS_SKIPPED, "reason" => SkipReason::NoTeamFilters.as_str())
.increment(1);
return EventEffects::default();
};
let filters: &TeamFilters = team_filters;
let lookup_started = Instant::now();
let redirected = redirect_for_tombstone(partition_id, handle, event, partition_count).await;
partition
.redirect_lookup
.record(lookup_started.elapsed().as_secs_f64());
let resolved: Cow<'_, CohortStreamEvent> = match redirected {
Redirected::Process(event) => event,
Redirected::ReKey(re_keyed) => {
return EventEffects {
re_keys: vec![re_keyed],
..EventEffects::default()
}
}
};
let started = Instant::now();
let result =
process_event_offloaded(partition_id, handle, filters, &resolved, event_name_gating).await;
histogram!(STAGE1_EVENT_PROCESS_DURATION).record(started.elapsed().as_secs_f64());
match result {
Ok(outcome) => {
if let Some(reason) = outcome.skipped {
counter!(STAGE1_EVENTS_SKIPPED, "reason" => reason.as_str()).increment(1);
return EventEffects::default();
}
counter!(STAGE1_EVENTS_PROCESSED).increment(1);
let mut changes = Vec::new();
for transition in &outcome.transitions {
if let Some(kind) = transition_metric_label(filters, transition) {
counter!(STAGE1_TRANSITIONS, "kind" => kind).increment(1);
}
changes.extend(map_transition(filters, transition, last_updated));
}
match compose_stage2(
partition_id,
handle,
filters,
&affected_leaves(&outcome.transitions),
outcome.event_ms,
last_updated,
ReadLane::Event,
)
.await
{
Ok(stage2_changes) => changes.extend(stage2_changes),
Err(error) => warn!(
partition_id,
team_id = event.team_id,
error = %error,
"stage 2 composition failed; skipping (self-heals on the person's next event)",
),
}
EventEffects {
changes,
schedules: outcome.schedules,
re_keys: Vec::new(),
}
}
Err(error) => {
counter!(STAGE1_EVENTS_SKIPPED, "reason" => "store_error").increment(1);
warn!(
partition_id,
team_id = event.team_id,
error = %error,
"stage 1 store error; skipping event without holding the offset",
);
EventEffects::default()
}
}
}
enum Redirected<'a> {
Process(Cow<'a, CohortStreamEvent>),
ReKey(CohortStreamEvent),
}
async fn redirect_for_tombstone<'a>(
partition_id: u16,
handle: &StoreHandle,
event: &'a CohortStreamEvent,
partition_count: u32,
) -> Redirected<'a> {
let Ok(person_id) = Uuid::parse_str(&event.person_id) else {
return Redirected::Process(Cow::Borrowed(event));
};
let resolution = match tombstone_redirect::resolve_offloaded(
handle,
partition_id,
TeamId(event.team_id),
person_id,
partition_count,
ReadLane::Event,
)
.await
{
Ok(resolution) => resolution,
Err(error) => {
warn!(
partition_id,
team_id = event.team_id,
error = %error,
"tombstone preflight read failed; processing without redirect",
);
return Redirected::Process(Cow::Borrowed(event));
}
};
tombstone_redirect::record_redirect(&resolution);
match resolution {
Resolution::NotMerged => Redirected::Process(Cow::Borrowed(event)),
Resolution::Inline {
final_person,
origin,
} => Redirected::Process(Cow::Owned(rewrite_to(event, final_person, origin))),
Resolution::CrossPartition {
target_person,
origin,
} => {
if event.redirect_hops >= tombstone_redirect::MAX_CROSS_PARTITION_REDIRECT_HOPS {
counter!(MERGE_REDIRECT_HOP_CAPPED_TOTAL).increment(1);
warn!(
partition_id,
team_id = event.team_id,
%target_person,
%origin,
hops = event.redirect_hops,
"cross-partition redirect hop cap hit (corrupt tombstone cycle?); processing inline at the best-known target",
);
return Redirected::Process(Cow::Owned(rewrite_to(event, target_person, origin)));
}
let mut re_keyed = rewrite_to(event, target_person, origin);
re_keyed.redirect_hops += 1;
Redirected::ReKey(re_keyed)
}
}
}
fn rewrite_to(event: &CohortStreamEvent, final_person: Uuid, origin: Uuid) -> CohortStreamEvent {
CohortStreamEvent {
person_id: final_person.to_string(),
redirected_from: event
.redirected_from
.clone()
.or_else(|| Some(origin.to_string())),
..event.clone()
}
}
/// The worker's slice coverage, beginning the slice at its assignment when it has no record.
async fn resolve_slice_coverage(
partition_id: u16,
assigned_at: CoverageStartMs,
handle: &StoreHandle,
) -> SliceCoverage {
let coverage = match handle
.resume_or_begin_slice(partition_id, assigned_at)
.await
{
Ok(SliceTenure::Resumed(coverage)) => coverage,
Ok(SliceTenure::Begun(since)) => {
counter!(SLICES_BEGUN_TOTAL).increment(1);
info!(
partition_id,
covered_since_ms = since.0,
"slice begins without earlier history; runs with an earlier boundary will be withheld",
);
SliceCoverage::Since(since)
}
Err(err) => {
// Fails closed: this worker certifies less, never more.
warn!(
partition_id,
error = %err,
"slice coverage unreadable; withholding runs with an earlier boundary",
);
SliceCoverage::Since(CoverageStartMs::now())
}
};
let covered_since_seconds = match coverage {
SliceCoverage::Complete => 0.0,
SliceCoverage::Since(since) => since.0 as f64 / 1000.0,
};
let partition: Arc<str> = Arc::from(partition_id.to_string());
gauge!(SLICE_COVERED_SINCE_SECONDS, "partition" => partition).set(covered_since_seconds);
coverage
}
/// Re-seed the per-worker [`EvictionQueue`] from `cf_behavioral`, scheduling every behavioral key on
/// its stored deadline. Skips `PersonProperty` variants (no time-based eviction) and `i64::MAX`
/// deadlines (permanent). Corrupt records are counted and skipped — the event path re-derives them. A
/// scan error stops early; new events reschedule any missing keys.
async fn rebuild_eviction_queue(
partition_id: u16,
handle: &StoreHandle,
queue: &mut EvictionQueue<BehavioralKey>,
) {
let mut cursor: Option<Vec<u8>> = None;
let mut rebuilt: u64 = 0;
loop {
let page = match handle
.scan_behavioral(partition_id, cursor.clone(), REBUILD_SCAN_PAGE)
.await
{
Ok(page) => page,
Err(err) => {
warn!(
partition_id,
error = %err,
"durable restore: cf_behavioral scan failed; eviction queue may be incomplete",
);
break;
}
};
let page_len = page.len();
let next_cursor = page.last().map(|(k, _)| k.encode().to_vec());
for (key, value) in page {
match StatefulRecord::decode(&value) {
// Use the same scheduling policy as the event path to prevent drift.
Ok(record) => {
if let Some(deadline) = schedule_deadline(&record.state) {
queue.schedule(key, deadline);
rebuilt += 1;
}
}
Err(_) => counter!(STAGE1_STATE_DECODE_ERROR).increment(1),
}
}
cursor = next_cursor;
if page_len < REBUILD_SCAN_PAGE {
break;
}
// Workers re-seed concurrently on restart; yield between pages so the boot scan doesn't
// saturate the runtime before the consume loop starts.
tokio::task::yield_now().await;
}
if rebuilt > 0 {
counter!(EVICTION_QUEUE_REBUILT_KEYS_TOTAL, "partition" => partition_id.to_string())
.increment(rebuilt);
info!(
partition_id,
rebuilt, "durable restore: re-seeded eviction queue from cf_behavioral",
);
}
}
/// The `(leaf, person)` pairs [`compose_stage2`] recomputes for.
pub fn affected_leaves(transitions: &[LeafTransition]) -> Vec<(LeafStateKey, Uuid)> {
transitions
.iter()
.map(|transition| (transition.leaf_state_key, transition.person_id))
.collect()
}
pub(crate) fn transition_metric_label(
filters: &TeamFilters,
transition: &LeafTransition,
) -> Option<&'static str> {
let variant = filters.by_lsk.get(&transition.leaf_state_key)?.variant;
match (variant, transition.kind) {
(StateVariant::BehavioralSingle, TransitionKind::Entered) => Some("behavioral_entered"),
(StateVariant::BehavioralSingle, TransitionKind::Left) => Some("behavioral_left"),
(StateVariant::BehavioralDailyBuckets, TransitionKind::Entered) => {
Some("behavioral_daily_entered")
}
(StateVariant::BehavioralDailyBuckets, TransitionKind::Left) => {
Some("behavioral_daily_left")
}
(StateVariant::BehavioralCompressedHistory, TransitionKind::Entered) => {
Some("behavioral_compressed_entered")
}
(StateVariant::BehavioralCompressedHistory, TransitionKind::Left) => {
Some("behavioral_compressed_left")
}
(StateVariant::PersonProperty, TransitionKind::Entered) => Some("person_entered"),
(StateVariant::PersonProperty, TransitionKind::Left) => Some("person_left"),
}
}
#[cfg(test)]
// Tests seed and assert against `CohortStore` directly, the sanctioned direct-store surface for tests.
#[allow(clippy::disallowed_methods)]
mod tombstone_redirect_tests {
use super::*;
use chrono_tz::UTC;
use serde_json::json;
use tempfile::TempDir;
use tokio::sync::mpsc;
use cohort_core::seed::{
BehavioralShapeHash, ReconcileScope, ReconcileTile, RunBoundaryMs, RunId,
};
use crate::consumers::seeds::SeedWork;
use crate::filters::{CohortId, FilterCatalog, TeamFiltersBuilder};
use crate::merge::transfer::Tombstone;
use crate::partitions::partitioner::{partition_of, COHORT_PARTITION_COUNT};
use crate::producer::{
CaptureCascadeSink, CaptureReconcileMarkerSink, CaptureSink, CaptureStreamEventSink,
CaptureTransferSink, ReconcileMarker,
};
use crate::stage1::person_record::PersonRecord;
use crate::stage1::state::AppliedOffsets;
use crate::stage2::state::Stage2State;
use crate::store::{
CohortStore, OffloadConfig, OffloadMode, PersonRecordKey, PersonRecords, Stage2Key,
StoreConfig, TombstoneKey,
};
use crate::workers::merge_path::TransferRetryPolicy;
use crate::workers::CascadeConfig;
const TEAM: i32 = 7;
const PERSON_HASH: [u8; 16] = *b"fedcba9876543210";
fn temp_store() -> (TempDir, CohortStore) {
let dir = TempDir::new().unwrap();
let store = CohortStore::open(&StoreConfig {
path: dir.path().join("db"),
..StoreConfig::default()
})
.unwrap();
(dir, store)
}
/// Wraps a test store so tests exercise the same blocking-pool transport as production.
fn test_handle(store: &CohortStore) -> StoreHandle {
StoreHandle::new(
store.clone(),
OffloadConfig {
mode: OffloadMode::All,
event_read_permits: 16,
maintenance_permits: 6,
},
)
}
fn person_catalog() -> Arc<CatalogHandle> {
let leaf = json!({
"type": "person", "key": "email", "value": "[email protected]", "operator": "exact",
"conditionHash": "fedcba9876543210",
"bytecode": ["_H", 1, 32, "[email protected]", 32, "email", 32, "properties", 32, "person", 1, 3, 11],
});
let cohort = json!({ "properties": { "type": "AND", "values": [leaf] } });
let mut builder = TeamFiltersBuilder::default();
builder
.add_cohort(CohortId(1), TeamId(TEAM), &cohort)
.unwrap();
Arc::new(CatalogHandle::from_catalog(FilterCatalog::from_teams([(
TeamId(TEAM),
builder.freeze(UTC),
)])))
}
fn reconcile_catalog(filters_hash: &str) -> Arc<CatalogHandle> {
let leaf = json!({
"type": "person", "key": "email", "value": "[email protected]", "operator": "exact",
"conditionHash": "fedcba9876543210",
"bytecode": ["_H", 1, 32, "[email protected]", 32, "email", 32, "properties", 32, "person", 1, 3, 11],
});
let cohort = json!({ "properties": { "type": "AND", "values": [leaf] } });
let mut builder = TeamFiltersBuilder::default();
builder
.add_cohort(CohortId(1), TeamId(TEAM), &cohort)
.unwrap();
builder.set_behavioral_shape_hash(
CohortId(1),
BehavioralShapeHash::parse(filters_hash).unwrap(),
);
Arc::new(CatalogHandle::from_catalog(FilterCatalog::from_teams([(
TeamId(TEAM),
builder.freeze(UTC),
)])))
}
fn applied(entries: &[(i32, i64)]) -> AppliedOffsets {
let mut applied = AppliedOffsets::default();
for &(partition, offset) in entries {
applied.record(partition, offset);
}
applied
}
fn person_event(
person: Uuid,
email: &str,
source_partition: i32,
source_offset: i64,
) -> CohortStreamEvent {
CohortStreamEvent {
team_id: TEAM,
person_id: person.to_string(),
distinct_id: "d".to_string(),
uuid: "u".to_string(),
event: "$pageview".to_string(),
timestamp: "2026-05-26 12:34:56.789000".to_string(),
properties: Some("{}".to_string()),
person_properties: Some(format!(r#"{{"email":"{email}"}}"#)),
elements_chain: None,
source_offset,
source_partition,
redirected_from: None,
redirect_hops: 0,
}
}
fn cross_partition_pair() -> (Uuid, u16, Uuid) {
let p_old = Uuid::from_u128(1);
let partition_id = partition_of(TeamId(TEAM), &p_old, COHORT_PARTITION_COUNT) as u16;
let p_new = (10u128..)
.map(Uuid::from_u128)
.find(|p| partition_of(TeamId(TEAM), p, COHORT_PARTITION_COUNT) as u16 != partition_id)
.expect("some uuid hashes off p_old's partition");
(p_old, partition_id, p_new)
}
fn merge_deps_with(stream_sink: CaptureStreamEventSink) -> Arc<MergeWorkerDeps> {
Arc::new(MergeWorkerDeps {
transfer_sink: Arc::new(CaptureTransferSink::new()),
stream_event_sink: Arc::new(stream_sink),
merge_tracker: Arc::new(OffsetTracker::new()),
transfer_tracker: Arc::new(OffsetTracker::new()),
retry: TransferRetryPolicy::default(),
gc_scan_limit: crate::workers::DEFAULT_MERGE_GC_SCAN_LIMIT,
stage2_orphan_gc_enabled: true,
cascade_sink: Arc::new(crate::producer::CaptureCascadeSink::new()),
cascade_tracker: Arc::new(OffsetTracker::new()),
cascade: crate::workers::CascadeConfig::default(),
partition_count: COHORT_PARTITION_COUNT,
seed_tile_sink: Arc::new(crate::producer::CaptureSeedTileSink::new()),
seed_tracker: Arc::new(crate::partitions::offset_tracker::OffsetTracker::new()),
live_watermarks: Arc::new(crate::partitions::watermarks::LiveWatermarks::new()),
register_transfer_enabled: false,
reconcile: crate::workers::ReconcileDeps::default(),
person_seed: crate::workers::PersonSeedDeps::default(),
seed_budget: crate::workers::seed_run::RunBudget::default(),
})
}
fn consumed_reconcile(tile: ReconcileTile, offset: i64) -> ConsumedSeed {
ConsumedSeed {
work: SeedWork::Reconcile(tile),
partition: 0,
offset,
broker_ts_ms: None,
}
}
fn reconcile_deps() -> (Arc<MergeWorkerDeps>, CaptureReconcileMarkerSink) {
let mut deps = Arc::try_unwrap(merge_deps_with(CaptureStreamEventSink::new()))
.unwrap_or_else(|_| panic!("test owns the only dependency Arc"));
deps.reconcile.enabled = true;
deps.reconcile.scan_page = 2;
let markers = CaptureReconcileMarkerSink::new();
deps.reconcile.marker_sink = Arc::new(markers.clone());
(Arc::new(deps), markers)
}
/// Deps with the `stage2_orphan_gc_enabled` kill-switch set to `stage2_orphan_gc_enabled`.
fn merge_deps_stage2_gc(stage2_orphan_gc_enabled: bool) -> Arc<MergeWorkerDeps> {
Arc::new(MergeWorkerDeps {
transfer_sink: Arc::new(CaptureTransferSink::new()),
stream_event_sink: Arc::new(CaptureStreamEventSink::new()),
merge_tracker: Arc::new(OffsetTracker::new()),
transfer_tracker: Arc::new(OffsetTracker::new()),
retry: TransferRetryPolicy::default(),
gc_scan_limit: crate::workers::DEFAULT_MERGE_GC_SCAN_LIMIT,
stage2_orphan_gc_enabled,
cascade_sink: Arc::new(crate::producer::CaptureCascadeSink::new()),
cascade_tracker: Arc::new(OffsetTracker::new()),
cascade: crate::workers::CascadeConfig::default(),
partition_count: COHORT_PARTITION_COUNT,
seed_tile_sink: Arc::new(crate::producer::CaptureSeedTileSink::new()),
seed_tracker: Arc::new(crate::partitions::offset_tracker::OffsetTracker::new()),
live_watermarks: Arc::new(crate::partitions::watermarks::LiveWatermarks::new()),
register_transfer_enabled: false,
reconcile: crate::workers::ReconcileDeps::default(),
person_seed: crate::workers::PersonSeedDeps::default(),
seed_budget: crate::workers::seed_run::RunBudget::default(),
})
}
/// Deps with a captured cascade sink and the cascade gate set to `enabled`.
fn merge_deps_cascade(cascade_sink: CaptureCascadeSink, enabled: bool) -> Arc<MergeWorkerDeps> {
Arc::new(MergeWorkerDeps {
transfer_sink: Arc::new(CaptureTransferSink::new()),
stream_event_sink: Arc::new(CaptureStreamEventSink::new()),
merge_tracker: Arc::new(OffsetTracker::new()),
transfer_tracker: Arc::new(OffsetTracker::new()),
retry: TransferRetryPolicy::default(),
gc_scan_limit: crate::workers::DEFAULT_MERGE_GC_SCAN_LIMIT,
stage2_orphan_gc_enabled: true,
cascade_sink: Arc::new(cascade_sink),
cascade_tracker: Arc::new(OffsetTracker::new()),
cascade: CascadeConfig {
enabled,
depth_cap: 8,
fanout_cap: 1000,
},
partition_count: COHORT_PARTITION_COUNT,
seed_tile_sink: Arc::new(crate::producer::CaptureSeedTileSink::new()),
seed_tracker: Arc::new(crate::partitions::offset_tracker::OffsetTracker::new()),
live_watermarks: Arc::new(crate::partitions::watermarks::LiveWatermarks::new()),
register_transfer_enabled: false,
reconcile: crate::workers::ReconcileDeps::default(),
person_seed: crate::workers::PersonSeedDeps::default(),
seed_budget: crate::workers::seed_run::RunBudget::default(),
})
}
/// Run one event that flips cohort 1 through a worker, returning the partition and its tracker.
async fn run_flip(
store: &CohortStore,
merge: Arc<MergeWorkerDeps>,
membership: &CaptureSink,
person: Uuid,
) -> (u16, Arc<OffsetTracker>) {
let partition_id = partition_of(TeamId(TEAM), &person, COHORT_PARTITION_COUNT) as u16;
let tracker = Arc::new(OffsetTracker::new());
run_batch(
partition_id,
store,
person_catalog(),
membership,
&tracker,
merge,
1,
vec![ShuffleMessage::Event {
event: Box::new(person_event(person, "[email protected]", 5, 0)),
cse_offset: 0,
broker_ts_ms: None,
}],
vec![],
)
.await;
(partition_id, tracker)
}
#[tokio::test]
async fn seed_receipt_raises_the_ceiling_before_a_disabled_reconcile_skip() {
let (_dir, store) = temp_store();
let membership = CaptureSink::new();
let merge = merge_deps_with(CaptureStreamEventSink::new());
let seed_tracker = merge.seed_tracker.clone();
let events_tracker = Arc::new(OffsetTracker::new());
let tile = ReconcileTile::new(
TeamId(TEAM),
CohortId(1),
ReconcileScope::Behavioral(BehavioralShapeHash::parse("0123456789abcdef").unwrap()),
RunId(Uuid::from_u128(1)),
RunBoundaryMs(0),
);
// Bypass EventDispatcher's post-send seed ceiling accounting. The worker must establish the
// ceiling from receipt before the disabled fast path can mark the offset processed.
run_batch(
0,
&store,
person_catalog(),
&membership,
&events_tracker,
merge,
0,
vec![],
vec![consumed_reconcile(tile, 5)],
)
.await;
assert_eq!(seed_tracker.committable_offsets().get(&0), Some(&6));
}
#[tokio::test]
async fn worker_admits_and_drains_a_zero_row_reconcile_before_releasing_its_offset() {
const FILTERS_HASH: &str = "shape-v1";
let (_dir, store) = temp_store();
let membership = CaptureSink::new();
let (merge, marker_sink) = reconcile_deps();
let seed_tracker = merge.seed_tracker.clone();
let backlog = merge.reconcile.backlog.clone();
let events_tracker = Arc::new(OffsetTracker::new());
let tile = ReconcileTile::new(
TeamId(TEAM),
CohortId(1),
ReconcileScope::Behavioral(BehavioralShapeHash::parse(FILTERS_HASH).unwrap()),
RunId(Uuid::from_u128(7)),
// After the slice begins, so the run certifies.
RunBoundaryMs(chrono::Utc::now().timestamp_millis() + 86_400_000),
);
// The drain tick must arrive after the tile is admitted, so the two lanes are driven by
// hand: a tick that beats the tile finds an empty queue and drains nothing. In production
// that is only latency — the next 2 s tick drains it.
let (live_tx, live_rx) = mpsc::channel(4);
let (seed_tx, seed_rx) = mpsc::channel(4);
let worker = Stage1Worker::spawn(
0,
WorkerInbox::unmetered(live_rx, seed_rx),
test_handle(&store),
reconcile_catalog(FILTERS_HASH),
Arc::new(membership.clone()),
events_tracker.clone(),
merge,
EvictionRestore::Skip,
);
seed_tx.send(consumed_reconcile(tile, 5)).await.unwrap();
drop(seed_tx);
// Bounded by the clock, not by a poll count: the admission's store section runs on the
// blocking pool, so a busy CI box needs wall time, not yields.
let deadline = Instant::now() + Duration::from_secs(10);
while backlog.is_empty() {
assert!(
Instant::now() < deadline,
"the reconcile tile was never admitted"
);
tokio::time::sleep(Duration::from_millis(5)).await;
}
live_tx
.send(vec![ShuffleMessage::ReconcileDrain])
.await
.unwrap();
drop(live_tx);
worker.join().await.unwrap();
assert!(membership.changes().is_empty());
let markers = marker_sink.markers();
assert_eq!(markers.len(), 1);
assert!(
matches!(markers[0], ReconcileMarker::Complete(_)),
"{markers:?}"
);
assert_eq!(markers[0].partition(), 0);
assert_eq!(markers[0].run_id(), RunId(Uuid::from_u128(7)));
assert_eq!(seed_tracker.committable_offsets().get(&0), Some(&6));
assert!(backlog.is_empty());
}
#[tokio::test]
async fn maintenance_boundary_keeps_later_live_output_strictly_newer_in_one_batch() {
let (_dir, store) = temp_store();
let membership = CaptureSink::new();
let tracker = Arc::new(OffsetTracker::new());
let person = Uuid::from_u128(0x71AE);
let mut leaves = person_event(person, "[email protected]", 5, 1);
leaves.timestamp = "2026-05-26 12:34:56.790000".to_string();
run_batch(
0,
&store,
person_catalog(),
&membership,
&tracker,
merge_deps_with(CaptureStreamEventSink::new()),
2,
vec![
ShuffleMessage::Event {
event: Box::new(person_event(person, "[email protected]", 5, 0)),
cse_offset: 0,
broker_ts_ms: None,
},
ShuffleMessage::ReconcileDrain,
ShuffleMessage::Event {
event: Box::new(leaves),
cse_offset: 1,
broker_ts_ms: None,
},
],
vec![],
)
.await;
let changes = membership.changes();
assert_eq!(changes.len(), 2);
assert_eq!(changes[0].status, MembershipStatus::Entered);
assert_eq!(changes[1].status, MembershipStatus::Left);
assert!(changes[0].last_updated < changes[1].last_updated);
}
#[tokio::test]
async fn event_flip_with_cascade_on_produces_a_depth_one_first_cascade() {
let (_dir, store) = temp_store();
let membership = CaptureSink::new();
let cascade = CaptureCascadeSink::new();
let (partition_id, tracker) = run_flip(
&store,
merge_deps_cascade(cascade.clone(), true),
&membership,
Uuid::from_u128(0x5A1CE),
)
.await;
assert_eq!(membership.changes().len(), 1, "alice entered cohort 1");
let cascades = cascade.messages();
assert_eq!(cascades.len(), 1, "one first-hop cascade for the flip");
assert_eq!(cascades[0].change.cohort_id, 1);
assert_eq!(cascades[0].depth, 1);
assert_eq!(cascades[0].cascade_chain, vec![1]);
assert_eq!(cascades[0].originating_cohort_id, 1);
assert_eq!(
tracker.committable_offsets().get(&(partition_id as i32)),
Some(&1),
"the acked two-topic produce releases the events offset",
);
}
#[tokio::test]
async fn event_flip_with_cascade_off_produces_no_cascade() {
let (_dir, store) = temp_store();
let membership = CaptureSink::new();
let cascade = CaptureCascadeSink::new();
let (partition_id, tracker) = run_flip(
&store,
merge_deps_cascade(cascade.clone(), false),
&membership,
Uuid::from_u128(0x5A1CF),
)
.await;
assert_eq!(
membership.changes().len(),
1,
"the flip still emits membership"
);
assert!(
cascade.messages().is_empty(),
"gate off: no cascade produced"
);
assert_eq!(
tracker.committable_offsets().get(&(partition_id as i32)),
Some(&1),
);
}
#[tokio::test]
async fn cascade_produce_failure_does_not_hold_the_events_batch() {
let (_dir, store) = temp_store();
let membership = CaptureSink::new();
let cascade = CaptureCascadeSink::failing_always();
let (partition_id, tracker) = run_flip(
&store,
merge_deps_cascade(cascade.clone(), true),
&membership,
Uuid::from_u128(0x5A1D0),
)
.await;
assert_eq!(
membership.changes().len(),
1,
"the membership leg acks on its own"
);
assert!(cascade.messages().is_empty(), "the cascade produce failed");
assert_eq!(
tracker.committable_offsets().get(&(partition_id as i32)),
Some(&1),
"live output is at most once, so a failed leg does not hold the events offset",
);
}
/// A membership sink that records changes when it is called, as a broker enqueue does, and
/// holds each ack until a [`GatedSink::release`], oldest first.
#[derive(Clone)]
struct GatedSink {
enqueued: CaptureSink,
acked: Arc<std::sync::atomic::AtomicUsize>,
gate: Arc<tokio::sync::Semaphore>,
}
impl GatedSink {
fn new() -> Self {
Self {
enqueued: CaptureSink::new(),
acked: Arc::default(),
gate: Arc::new(tokio::sync::Semaphore::new(0)),
}
}
fn release(&self, produces: usize) {
self.gate.add_permits(produces);
}
fn enqueued_cohorts(&self) -> Vec<i32> {
self.enqueued
.changes()
.iter()
.map(|change| change.cohort_id)
.collect()
}
fn acked(&self) -> usize {
self.acked.load(std::sync::atomic::Ordering::SeqCst)
}
}
#[async_trait::async_trait]
impl MembershipSink for GatedSink {
async fn produce(
&self,
changes: Vec<CohortMembershipChange>,
) -> Vec<Result<(), common_kafka::kafka_producer::KafkaProduceError>> {
let acks = self.enqueued.produce(changes).await;
self.gate
.acquire()
.await
.expect("the gate is never closed")
.forget();
self.acked.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
acks
}
}
fn entered(cohort_id: i32) -> CohortMembershipChange {
CohortMembershipChange {
team_id: TEAM,
cohort_id,
person_id: Uuid::from_u128(1).to_string(),
last_updated: "2026-05-26 12:34:56.789000".to_string(),
status: MembershipStatus::Entered,
origin: None,
run_id: None,
}
}
#[tokio::test]
async fn produce_window_enqueues_in_submit_order_and_waits_at_capacity() {
let sink = GatedSink::new();
let mut window = ProduceWindow::new(
0,
Arc::new(sink.clone()),
merge_deps_with(CaptureStreamEventSink::new()),
);
let mut buffer = OutputBuffer::new();
for cohort_id in 0..PRODUCE_WINDOW as i32 {
buffer.push(entered(cohort_id));
assert!(
window.submit(&mut buffer, None).now_or_never().is_some(),
"a window with room hands batch {cohort_id} over without waiting",
);
}
assert_eq!(
sink.enqueued_cohorts(),
(0..PRODUCE_WINDOW as i32).collect::<Vec<_>>(),
"each submit enqueues its batch before it returns",
);
buffer.push(entered(PRODUCE_WINDOW as i32));
{
let mut full = std::pin::pin!(window.submit(&mut buffer, None));
assert!(
futures::poll!(&mut full).is_pending(),
"a full window waits for its oldest ack",
);
assert_eq!(
sink.enqueued_cohorts().len(),
PRODUCE_WINDOW,
"the waiting submit enqueues its batch only after the oldest acks",
);
sink.release(1);
full.await;
}
sink.release(PRODUCE_WINDOW);
window.drain().await;
assert_eq!(
sink.enqueued_cohorts(),
(0..=PRODUCE_WINDOW as i32).collect::<Vec<_>>()
);
assert_eq!(sink.acked(), PRODUCE_WINDOW + 1, "drain collects every ack");
}
#[tokio::test]
async fn the_offset_marks_while_the_membership_produce_is_unacked() {
let (_dir, store) = temp_store();
let membership = GatedSink::new();
let tracker = Arc::new(OffsetTracker::new());
let person = Uuid::from_u128(0x0AC4);
let partition_id = partition_of(TeamId(TEAM), &person, COHORT_PARTITION_COUNT) as u16;
let (live_tx, live_rx) = mpsc::channel(4);
let (seed_tx, seed_rx) = mpsc::channel(4);
drop(seed_tx);
let worker = Stage1Worker::spawn(
partition_id,
WorkerInbox::unmetered(live_rx, seed_rx),
test_handle(&store),
person_catalog(),
Arc::new(membership.clone()),
tracker.clone(),
merge_deps_with(CaptureStreamEventSink::new()),
EvictionRestore::Skip,
);
tracker.mark_dispatched(partition_id as i32, 1);
live_tx
.send(vec![ShuffleMessage::Event {
event: Box::new(person_event(person, "[email protected]", 5, 0)),
cse_offset: 0,
broker_ts_ms: None,
}])
.await
.unwrap();
let deadline = Instant::now() + Duration::from_secs(10);
while tracker.committable_offsets().get(&(partition_id as i32)) != Some(&1) {
assert!(
Instant::now() < deadline,
"the offset never marked while the produce waited for its ack",
);
tokio::time::sleep(Duration::from_millis(5)).await;
}
assert_eq!(
membership.enqueued.changes().len(),
1,
"the flip was enqueued"
);
assert_eq!(membership.acked(), 0, "and is still unacked");
membership.release(1);
drop(live_tx);
worker.join().await.unwrap();
}
#[tokio::test]
async fn a_run_of_cascade_messages_marks_every_offset() {
let (_dir, store) = temp_store();
let merge = merge_deps_cascade(CaptureCascadeSink::new(), true);
merge.cascade_tracker.mark_dispatched(0, 3);
let cascade = |person: u128, offset: i64| ShuffleMessage::Cascade {
message: Box::new(first_cascade(
CohortMembershipChange {
person_id: Uuid::from_u128(person).to_string(),
..entered(1)
},
0,
)),
offset,
};
run_batch(
0,
&store,
person_catalog(),
&CaptureSink::new(),
&Arc::new(OffsetTracker::new()),
merge.clone(),
0,
vec![cascade(1, 0), cascade(2, 1), cascade(3, 2)],
vec![],
)
.await;
assert_eq!(
merge.cascade_tracker.committable_offsets().get(&0),
Some(&3),
"every message the arm gathers into the run is settled",
);
}
/// Drive one worker over both lanes: `live` is sent as a single sub-batch, `seeds` one per
/// lane slot, then both senders drop so the worker drains and exits.
#[allow(clippy::too_many_arguments)]
async fn run_batch(
partition_id: u16,
store: &CohortStore,
catalog: Arc<CatalogHandle>,
membership: &CaptureSink,
tracker: &Arc<OffsetTracker>,
merge: Arc<MergeWorkerDeps>,
dispatched: i64,
live: Vec<ShuffleMessage>,
seeds: Vec<ConsumedSeed>,
) {
let (live_tx, live_rx) = mpsc::channel(4);
let (seed_tx, seed_rx) = mpsc::channel(16);
let worker = Stage1Worker::spawn(
partition_id,
WorkerInbox::unmetered(live_rx, seed_rx),
test_handle(store),
catalog,
Arc::new(membership.clone()),
tracker.clone(),
merge,
EvictionRestore::Skip,
);
tracker.mark_dispatched(partition_id as i32, dispatched);
for seed in seeds {
seed_tx.send(seed).await.unwrap();
}
drop(seed_tx);
if !live.is_empty() {
live_tx.send(live).await.unwrap();
}
drop(live_tx);
worker.join().await.unwrap();
}
async fn run_one_straggler(
partition_id: u16,
store: &CohortStore,
catalog: Arc<CatalogHandle>,
membership: &CaptureSink,
tracker: &Arc<OffsetTracker>,
merge: Arc<MergeWorkerDeps>,
event: CohortStreamEvent,
) {
run_batch(
partition_id,
store,
catalog,
membership,
tracker,
merge,
1,
vec![ShuffleMessage::Event {
event: Box::new(event),
cse_offset: 0,
broker_ts_ms: Some(STRAGGLER_BROKER_TS_MS),
}],
vec![],
)
.await;
}
const STRAGGLER_BROKER_TS_MS: i64 = 1_750_000_000_000;
fn record_key(partition_id: u16, person: Uuid) -> PersonRecordKey {
PersonRecordKey::new(partition_id, TEAM as u64, person)
}
fn read_person_record(
store: &CohortStore,
partition_id: u16,
person: Uuid,
) -> Option<PersonRecord> {
store
.get_person_record(&record_key(partition_id, person))
.unwrap()
.map(|bytes| PersonRecord::decode(&bytes).unwrap())
}
fn write_tombstone(store: &CohortStore, partition_id: u16, old: Uuid, new: Uuid) {
let value = Tombstone {
new_person: new,
merged_at_ms: 1,
};
store
.write_batch(|b| {
b.put_tombstone(
&TombstoneKey {
partition_id,
team_id: TEAM as u64,
person: old,
},
&value.encode(),
)
})
.unwrap();
}
#[tokio::test]
async fn inline_redirect_folds_into_redirect_dedup_origin_not_the_main_map() {
let (_dir, store) = temp_store();
let catalog = person_catalog();
let p_new = Uuid::from_u128(2);
let partition_id = partition_of(TeamId(TEAM), &p_new, COHORT_PARTITION_COUNT) as u16;
let p_old = Uuid::from_u128(1);
write_tombstone(&store, partition_id, p_old, p_new);
// Seed P_new's record: not yet a member, main-map offset {5:50}, and an ancestor entry for
// p_old at {5:100}. The absent-baseline fingerprints (0,0) never match the event, so the
// straggler re-evaluates and the email leaf enters.
let mut seed = PersonRecord::absent();
seed.applied_offsets = applied(&[(5, 50)]);
seed.redirect_dedup.insert(p_old, applied(&[(5, 100)]));
store
.write_batch(|b| {
b.put::<PersonRecords>(&record_key(partition_id, p_new), &seed.encode())
})
.unwrap();
let straggler = person_event(p_old, "[email protected]", 5, 101);
let effects = handle_event(
&EventPartition::new(partition_id),
&test_handle(&store),
&catalog,
&straggler,
"ts",
COHORT_PARTITION_COUNT,
EventNameGating::Disabled,
)
.await;
assert_eq!(effects.changes.len(), 1, "the straggler entered P_new");
assert_eq!(effects.changes[0].person_id, p_new.to_string());
assert_eq!(effects.changes[0].status, MembershipStatus::Entered);
let after = read_person_record(&store, partition_id, p_new).unwrap();
assert!(
after.matched.contains(&PERSON_HASH),
"the email leaf entered the record's matched set",
);
assert!(
after.redirect_dedup[&p_old].is_replay(5, 101),
"the fold advanced redirect_dedup[origin]",
);
assert!(
after.applied_offsets.is_replay(5, 50) && !after.applied_offsets.is_replay(5, 51),
"the main map is untouched by a redirected straggler",
);
assert!(
read_person_record(&store, partition_id, p_old).is_none(),
"no record written for P_old",
);
}
#[tokio::test]
async fn cross_partition_redirect_re_keys_the_straggler_to_the_target() {
let (_dir, store) = temp_store();
let catalog = person_catalog();
let (p_old, partition_id, p_new) = cross_partition_pair();
write_tombstone(&store, partition_id, p_old, p_new);
let stream_sink = CaptureStreamEventSink::new();
let membership = CaptureSink::new();
let tracker = Arc::new(OffsetTracker::new());
run_one_straggler(
partition_id,
&store,
catalog,
&membership,
&tracker,
merge_deps_with(stream_sink.clone()),
person_event(p_old, "[email protected]", 5, 9),
)
.await;
let produced = stream_sink.events();
assert_eq!(produced.len(), 1, "one re-keyed event produced");
let re_keyed = &produced[0];
assert_eq!(
re_keyed.person_id,
p_new.to_string(),
"person rewritten to the target",
);
assert_eq!(
re_keyed.redirected_from,
Some(p_old.to_string()),
"first origin stamped",
);
assert_eq!(
(re_keyed.source_partition, re_keyed.source_offset),
(5, 9),
"original source coords preserved for the target's redirect_dedup",
);
assert_eq!(re_keyed.redirect_hops, 1, "one produced hop");
assert!(membership.changes().is_empty(), "no local processing");
assert!(
read_person_record(&store, partition_id, p_old).is_none(),
"no record written for P_old in the source slice",
);
assert!(
read_person_record(&store, partition_id, p_new).is_none(),
"no record written for P_new in the source slice",
);
assert_eq!(
tracker.committable_offsets().get(&(partition_id as i32)),
Some(&1),
"the acked re-key produce releases the events offset",
);
}
#[tokio::test]
async fn re_key_produce_failure_holds_the_events_offset_until_redelivery_succeeds() {
let (_dir, store) = temp_store();
let catalog = person_catalog();
let (p_old, partition_id, p_new) = cross_partition_pair();
write_tombstone(&store, partition_id, p_old, p_new);
let membership = CaptureSink::new();
let tracker = Arc::new(OffsetTracker::new());
let failing = CaptureStreamEventSink::failing_always();
let failing_deps = merge_deps_with(failing.clone());
run_one_straggler(
partition_id,
&store,
catalog.clone(),
&membership,
&tracker,
failing_deps.clone(),
person_event(p_old, "[email protected]", 5, 9),
)
.await;
assert!(
failing.events().is_empty(),
"the failed flush recorded nothing"
);
assert_eq!(
tracker.committable_offsets().get(&(partition_id as i32)),
None,
"the failed produce holds the events offset",
);
assert_eq!(
failing_deps.live_watermarks.get(partition_id as i32),
None,
"an unmarked batch never advances the seed fence",
);
let succeeding = CaptureStreamEventSink::new();
let succeeding_deps = merge_deps_with(succeeding.clone());
run_one_straggler(
partition_id,
&store,
catalog,
&membership,
&tracker,
succeeding_deps.clone(),
person_event(p_old, "[email protected]", 5, 9),
)
.await;
assert_eq!(
succeeding_deps.live_watermarks.get(partition_id as i32),
Some(crate::partitions::WatermarkMs(STRAGGLER_BROKER_TS_MS)),
"the marked redelivery advances it",
);
let produced = succeeding.events();
assert_eq!(
produced.len(),
1,
"the redelivery re-derives and produces the re-key"
);
assert_eq!(produced[0].person_id, p_new.to_string());
assert_eq!(
(produced[0].source_partition, produced[0].source_offset),
(5, 9),
"the re-derived copy carries the original source coords (dedupable at the target)",
);
assert_eq!(
tracker.committable_offsets().get(&(partition_id as i32)),
Some(&1),
"the later success releases the held offset",
);
}
#[tokio::test]
async fn membership_produce_failure_in_a_mixed_batch_drops_the_change_and_still_re_keys_and_marks(
) {
let (_dir, store) = temp_store();
let catalog = person_catalog();
let (p_old, partition_id, p_new) = cross_partition_pair();
write_tombstone(&store, partition_id, p_old, p_new);
let alice = (100u128..)
.map(Uuid::from_u128)
.find(|p| partition_of(TeamId(TEAM), p, COHORT_PARTITION_COUNT) as u16 == partition_id)
.expect("some uuid hashes onto p_old's partition");
let membership = CaptureSink::failing_first(1);
let stream_sink = CaptureStreamEventSink::new();
let tracker = Arc::new(OffsetTracker::new());
run_batch(
partition_id,
&store,
catalog,
&membership,
&tracker,
merge_deps_with(stream_sink.clone()),
2,
vec![
ShuffleMessage::Event {
event: Box::new(person_event(alice, "[email protected]", 5, 0)),
cse_offset: 0,
broker_ts_ms: None,
},
ShuffleMessage::Event {
event: Box::new(person_event(p_old, "[email protected]", 5, 9)),
cse_offset: 1,
broker_ts_ms: None,
},
],
vec![],
)
.await;
assert!(
membership.changes().is_empty(),
"the failed produce dropped alice's flip (at-most-once)",
);
let produced = stream_sink.events();
assert_eq!(produced.len(), 1, "the straggler's re-key still produces");
assert_eq!(produced[0].person_id, p_new.to_string());
assert_eq!(
(produced[0].source_partition, produced[0].source_offset),
(5, 9),
"the re-key carries the straggler's original source coords",
);
assert_eq!(
tracker.committable_offsets().get(&(partition_id as i32)),
Some(&2),
"the mark passes both events",
);
}
#[tokio::test]
async fn re_keyed_event_folds_into_p_new_exactly_once_via_redirect_dedup() {
let (_dir, store) = temp_store();
let catalog = person_catalog();
let (p_old, source_partition, p_new) = cross_partition_pair();
let target_partition = partition_of(TeamId(TEAM), &p_new, COHORT_PARTITION_COUNT) as u16;
assert_ne!(source_partition, target_partition);
write_tombstone(&store, source_partition, p_old, p_new);
let handle = test_handle(&store);
let recorder = metrics_exporter_prometheus::PrometheusBuilder::new().build_recorder();
let prometheus = recorder.handle();
let _local = metrics::set_default_local_recorder(&recorder);
let straggler = person_event(p_old, "[email protected]", 5, 9);
let mut effects = handle_event(
&EventPartition::new(source_partition),
&handle,
&catalog,
&straggler,
"ts",
COHORT_PARTITION_COUNT,
EventNameGating::Disabled,
)
.await;
assert!(effects.changes.is_empty());
assert!(effects.schedules.is_empty());
assert_eq!(effects.re_keys.len(), 1);
let re_keyed = effects.re_keys.pop().unwrap();
let effects = handle_event(
&EventPartition::new(target_partition),
&handle,
&catalog,
&re_keyed,
"ts",
COHORT_PARTITION_COUNT,
EventNameGating::Disabled,
)
.await;
assert_eq!(effects.changes.len(), 1, "folds into P_new exactly once");
assert_eq!(effects.changes[0].person_id, p_new.to_string());
assert_eq!(effects.changes[0].status, MembershipStatus::Entered);
assert!(effects.re_keys.is_empty(), "no further hop: P_new is live");
let folded = read_person_record(&store, target_partition, p_new)
.expect("P_new record written in the target slice");
assert!(
folded.matched.contains(&PERSON_HASH),
"the email leaf entered P_new's record",
);
assert!(
folded.redirect_dedup[&p_old].is_replay(5, 9),
"the fold recorded the original source coords under redirect_dedup[origin]",
);
assert!(
!folded.applied_offsets.is_replay(5, 9),
"the main map stays untouched by a redirected straggler",
);
let dup = handle_event(
&EventPartition::new(target_partition),
&handle,
&catalog,
&re_keyed,
"ts",
COHORT_PARTITION_COUNT,
EventNameGating::Disabled,
)
.await;
assert!(dup.changes.is_empty(), "the duplicate folds zero times");
assert!(dup.re_keys.is_empty());
let rendered = prometheus.render();
for (partition, lookups) in [(source_partition, 1), (target_partition, 2)] {
let series = format!(
"{STAGE1_REDIRECT_LOOKUP_DURATION}_count{{partition=\"{partition}\"}} {lookups}"
);
assert!(
rendered.lines().any(|line| line == series),
"the re-keyed lookup counts too, under its own partition: {series} not in\n{rendered}",
);
}
}
#[tokio::test]
async fn hop_capped_redirect_processes_inline_at_the_best_known_target() {
let (_dir, store) = temp_store();
let catalog = person_catalog();
let (p_old, partition_id, p_new) = cross_partition_pair();
write_tombstone(&store, partition_id, p_old, p_new);
let straggler = CohortStreamEvent {
redirect_hops: tombstone_redirect::MAX_CROSS_PARTITION_REDIRECT_HOPS,
..person_event(p_old, "[email protected]", 5, 9)
};
let effects = handle_event(
&EventPartition::new(partition_id),
&test_handle(&store),
&catalog,
&straggler,
"ts",
COHORT_PARTITION_COUNT,
EventNameGating::Disabled,
)
.await;
assert!(effects.re_keys.is_empty(), "no re-produce at the cap");
assert_eq!(effects.changes.len(), 1, "processed inline instead");
assert_eq!(effects.changes[0].person_id, p_new.to_string());
let folded = read_person_record(&store, partition_id, p_new)
.expect("the degraded fold writes P_new's record in the local slice");
assert!(
folded.redirect_dedup[&p_old].is_replay(5, 9),
"the inline degrade still dedups by the chain origin",
);
assert!(
read_person_record(&store, partition_id, p_old).is_none(),
"no P_old record rebuilt",
);
}
#[tokio::test]
async fn no_tombstone_processes_the_event_normally() {
let (_dir, store) = temp_store();
let catalog = person_catalog();
let alice = Uuid::from_u128(3);
let partition_id = partition_of(TeamId(TEAM), &alice, COHORT_PARTITION_COUNT) as u16;
let event = person_event(alice, "[email protected]", 5, 0);
let effects = handle_event(
&EventPartition::new(partition_id),
&test_handle(&store),
&catalog,
&event,
"ts",
COHORT_PARTITION_COUNT,
EventNameGating::Disabled,
)
.await;
assert_eq!(effects.changes.len(), 1);
assert_eq!(effects.changes[0].person_id, alice.to_string());
assert!(
read_person_record(&store, partition_id, alice).is_some(),
"alice's own record was written",
);
}
/// The `MergeCfGc` arm runs the `cf_stage2` orphan GC only when `stage2_orphan_gc_enabled`: a
/// catalog-absent cohort's row survives with the kill-switch off and is reclaimed with it on.
#[tokio::test]
async fn merge_cf_gc_arm_runs_stage2_orphan_gc_only_when_enabled() {
for (enabled, expect_present) in [(false, true), (true, false)] {
let (_dir, store) = temp_store();
let person = Uuid::from_u128(0xC0FFEE);
let partition_id = partition_of(TeamId(TEAM), &person, COHORT_PARTITION_COUNT) as u16;
let orphan = Stage2Key {
partition_id,
team_id: TEAM as u64,
cohort_id: 99,
person_id: person,
};
store
.write_batch(|b| {
b.put_stage2(
&orphan,
&Stage2State {
in_cohort: true,
last_evaluated_at_ms: 1,
}
.encode(),
)
})
.unwrap();
let membership = CaptureSink::new();
let tracker = Arc::new(OffsetTracker::new());
run_batch(
partition_id,
&store,
person_catalog(),
&membership,
&tracker,
merge_deps_stage2_gc(enabled),
1,
vec![ShuffleMessage::MergeCfGc {
marker_cutoff_ms: 0,
tombstone_cutoff_ms: 0,
}],
vec![],
)
.await;
assert_eq!(
store.get_stage2(&orphan).unwrap().is_some(),
expect_present,
"MergeCfGc arm runs the orphan GC iff enabled (enabled={enabled})",
);
}
}
}