:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
//! The `cohort_stream_seed_events` follower consumer — the backfill day-tile input.
//!
//! It is assigned the partitions this pod owns, like the events consumer. Four admission gates
//! share one holdover + pause mechanism: the apply fence (a tile dispatches only once its
//! partition's live watermark clears `s_chunk + margin`), a full seed lane, live-priority (the
//! partition's live watermark age crossed the pause threshold — live traffic always wins), and
//! pod-wide disk pressure. An un-dispatched tile was never `mark_dispatched`ed, so its offset cannot commit.
//! The live-lag gate remains an admission backstop because an admitted seed run still uses the
//! worker and its CPU and I/O even though separate lanes keep queued seeds behind live traffic.
//! Consume-side skips ride the seed lane so their offsets mark in order; they never close the
//! fence, but a live-lag/disk gate holds them too (nothing leapfrogs a gated partition).
//! The [`PauseLedger`] records why each partition is held and since when; the pause target and
//! the age/count gauges all derive from it.
use std::collections::{HashMap, HashSet};
use std::sync::Arc;
use std::time::{Duration, Instant};
use cohort_core::seed::{decode_seed, DecodedSeed, PersonSeed, ReconcileTile, SChunkMs, SeedTile};
use lifecycle::Handle;
use metrics::{counter, gauge, histogram};
use rdkafka::consumer::{CommitMode, Consumer, StreamConsumer};
use rdkafka::message::Message;
use rdkafka::{Offset, TopicPartitionList};
use tracing::{debug, info, warn};
use crate::consumers::events::{fsync_then_commit, run_pauser_loop, EventDispatcher};
use crate::consumers::merges::{committable_after_drain, owned_committable_offsets};
use crate::observability::disk::{DiskUtilization, SharedDiskUtilization};
use crate::observability::metrics::{
COHORT_STREAM_KAFKA_RECV_ERRORS, COHORT_STREAM_SEEDS_CONSUMED,
COHORT_STREAM_SEEDS_CONSUME_BATCH_SIZE, COHORT_STREAM_SEED_DESERIALIZE_ERRORS,
LIVE_WATERMARK_AGE_MS, SEED_FENCED_PARTITIONS, SEED_FENCE_DEFICIT_MS,
SEED_IDLE_PROBE_DURATION_SECONDS, SEED_IDLE_PROBE_LAST_PASS_TIMESTAMP_SECONDS,
SEED_NO_WATERMARK_PARTITIONS, SEED_OLDEST_HELD_AGE_MS, SEED_PAUSED_PARTITIONS,
SEED_PAUSE_AGE_MS,
};
use crate::partitions::backpressure::PartitionHoldover;
use crate::partitions::pacing::{
AgeMs, CauseSet, Hysteresis, PauseCause, PauseLedger, SeedPacing, SeedPacingConfig, UsedPct,
};
use crate::partitions::pause::PartitionPauser;
use crate::partitions::rebalance::CohortConsumerContext;
use crate::partitions::stamp::CommitStamp;
use crate::partitions::watermarks::WatermarkMs;
/// Back-off after a Kafka transport error, mirroring the sibling consume loops.
const RECV_ERROR_BACKOFF: Duration = Duration::from_millis(500);
/// Timeout for the idle probe's blocking watermark/committed fetches.
const PROBE_FETCH_TIMEOUT: Duration = Duration::from_secs(5);
/// Why a consumed seed payload is skipped rather than applied.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum SeedSkipReason {
/// A kind the current worker path does not handle.
UnknownKind,
/// A newer schema version. The skip commits and never replays, so this consumer must be
/// upgraded before any seeder emits a new schema.
UnsupportedSchema,
/// Empty or undecodable payload: deterministic bytes, so halting would wedge the partition.
DecodeError,
}
impl SeedSkipReason {
pub fn as_str(self) -> &'static str {
match self {
Self::UnknownKind => "unknown_kind",
Self::UnsupportedSchema => "unsupported_schema",
Self::DecodeError => "decode_error",
}
}
}
/// A decoded unit of seed work, routed to the owning partition worker.
#[derive(Debug)]
pub enum SeedWork {
Tile(SeedTile),
Person(PersonSeed),
Reconcile(ReconcileTile),
Skip(SeedSkipReason),
}
/// One message consumed from `cohort_stream_seed_events`, paired with its commit coordinates.
#[derive(Debug)]
pub struct ConsumedSeed {
pub work: SeedWork,
pub partition: i32,
pub offset: i64,
/// Broker timestamp of the consumed message — the oldest-held age gauge's input. `None`
/// (`Timestamp::NotAvailable`) reads as age 0.
pub broker_ts_ms: Option<i64>,
}
impl ConsumedSeed {
/// The fence input. Control messages, person seeds, and skips are fence-open: none of them
/// bounds the arrival of events over the live stream.
fn s_chunk_ms(&self) -> Option<SChunkMs> {
match &self.work {
SeedWork::Tile(tile) => Some(tile.s_chunk_ms()),
SeedWork::Person(_) | SeedWork::Reconcile(_) | SeedWork::Skip(_) => None,
}
}
}
/// The apply-fence verdict for one tile.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum FenceDecision {
Open,
/// How far the watermark trails `s_chunk + margin`; `None` = no watermark at all.
Closed {
deficit_ms: Option<i64>,
},
}
/// Open only when the watermark exceeds `s_chunk + margin`; an absent watermark is fail-closed.
pub(crate) fn fence_decision(
watermark: Option<WatermarkMs>,
s_chunk: SChunkMs,
margin_ms: i64,
) -> FenceDecision {
let Some(WatermarkMs(watermark_ms)) = watermark else {
return FenceDecision::Closed { deficit_ms: None };
};
let bound = s_chunk.0.saturating_add(margin_ms);
if watermark_ms > bound {
FenceDecision::Open
} else {
FenceDecision::Closed {
deficit_ms: Some(bound - watermark_ms),
}
}
}
/// The gates evaluated at admission, beyond the per-tile fence.
pub(crate) struct AdmissionPolicy<'a, F: Fn(i32) -> Option<WatermarkMs>> {
pub fence_margin_ms: i64,
pub watermark_of: F,
/// Partitions gated by live-priority: hold everything, including leading skips.
pub live_lagging: &'a HashSet<i32>,
/// Pod-wide disk gate: hold everything for every partition.
pub disk_pressure: bool,
}
/// A consumed batch split at the admission gates, preserving per-partition FIFO.
#[derive(Debug, Default)]
pub(crate) struct AdmissionSplit {
pub admitted: Vec<ConsumedSeed>,
pub held: HashMap<i32, Vec<ConsumedSeed>>,
/// Why each partition that closed in this split is held.
pub causes: HashMap<i32, CauseSet>,
/// Deficit per partition whose fence closed in this split.
pub deficits: HashMap<i32, Option<i64>>,
}
/// Admit each partition's open prefix; from the first fence-closed tile hold everything for that
/// partition (skips included) so nothing leapfrogs a held offset. A gated partition
/// (live-lag/disk) holds everything from its first message, leading skips included.
/// `already_held` partitions queue entirely behind their holdover (their causes were attributed
/// when the holdover re-split this cycle).
pub(crate) fn split_for_admission<F: Fn(i32) -> Option<WatermarkMs>>(
seeds: Vec<ConsumedSeed>,
already_held: &HashSet<i32>,
policy: &AdmissionPolicy<'_, F>,
) -> AdmissionSplit {
let mut split = AdmissionSplit::default();
let mut closed: HashSet<i32> = HashSet::new();
// Gated partitions whose first tile has not been fence-probed yet: even when gated, the first
// tile still evaluates the fence so the fence cause and deficit stay continuous through a
// live-lag/disk episode.
let mut fence_probe_pending: HashSet<i32> = HashSet::new();
for seed in seeds {
let partition = seed.partition;
if already_held.contains(&partition) {
split.held.entry(partition).or_default().push(seed);
continue;
}
if !closed.contains(&partition) {
let gate = gate_causes(policy, partition);
if !gate.is_empty() {
closed.insert(partition);
fence_probe_pending.insert(partition);
split.causes.insert(partition, gate);
}
}
if closed.contains(&partition) {
if fence_probe_pending.contains(&partition) {
if let Some(s_chunk) = seed.s_chunk_ms() {
fence_probe_pending.remove(&partition);
if let FenceDecision::Closed { deficit_ms } = fence_decision(
(policy.watermark_of)(partition),
s_chunk,
policy.fence_margin_ms,
) {
split
.causes
.entry(partition)
.or_default()
.insert(PauseCause::Fence);
split.deficits.insert(partition, deficit_ms);
}
}
}
split.held.entry(partition).or_default().push(seed);
continue;
}
let decision = match seed.s_chunk_ms() {
None => FenceDecision::Open,
Some(s_chunk) => fence_decision(
(policy.watermark_of)(partition),
s_chunk,
policy.fence_margin_ms,
),
};
match decision {
FenceDecision::Open => split.admitted.push(seed),
FenceDecision::Closed { deficit_ms } => {
closed.insert(partition);
split
.causes
.entry(partition)
.or_default()
.insert(PauseCause::Fence);
split.deficits.insert(partition, deficit_ms);
split.held.entry(partition).or_default().push(seed);
}
}
}
split
}
/// The partition-level gate verdict under `policy`, independent of any tile.
fn gate_causes<F: Fn(i32) -> Option<WatermarkMs>>(
policy: &AdmissionPolicy<'_, F>,
partition: i32,
) -> CauseSet {
let mut causes = CauseSet::default();
if policy.live_lagging.contains(&partition) {
causes.insert(PauseCause::LiveLag);
}
if policy.disk_pressure {
causes.insert(PauseCause::DiskPressure);
}
causes
}
/// Holdover of fence-closed, backpressured, or pacing-gated seeds.
type SeedHoldover = PartitionHoldover<ConsumedSeed>;
/// The holdover flattened in per-partition FIFO order.
fn drain_held(holdover: &mut SeedHoldover) -> Vec<ConsumedSeed> {
holdover
.take_held()
.into_iter()
.flat_map(|(_, seeds)| seeds)
.collect()
}
/// One pass over the owned partitions: watermark ages (for the age gauge), the live-lag verdicts,
/// and the no-watermark count.
#[derive(Debug, Default)]
struct LiveLagAssessment {
/// Partitions whose seed applies must yield to live consumption this cycle.
lagging: HashSet<i32>,
/// Watermark age per partition that has one.
ages_ms: HashMap<i32, i64>,
/// Owned partitions with no watermark at all: fence-fail-closed with an unknown deficit, so
/// otherwise invisible on the deficit gauge.
no_watermark: usize,
}
/// An absent watermark is never a live-lag hold — tiles are already fence-fail-closed there, and
/// a redundant hold would poison the hysteresis memory on a new tenure — it is only counted.
fn assess_live_lag(
owned: &HashSet<i32>,
watermark_of: impl Fn(i32) -> Option<WatermarkMs>,
ledger: &PauseLedger,
hysteresis: Option<Hysteresis<AgeMs>>,
now_ms: i64,
) -> LiveLagAssessment {
let mut assessment = LiveLagAssessment::default();
for &partition in owned {
let Some(WatermarkMs(watermark_ms)) = watermark_of(partition) else {
assessment.no_watermark += 1;
continue;
};
let age_ms = now_ms.saturating_sub(watermark_ms);
assessment.ages_ms.insert(partition, age_ms);
let Some(hysteresis) = hysteresis else {
continue;
};
let engaged = ledger.has_cause(partition, PauseCause::LiveLag);
if hysteresis.decide(engaged, AgeMs(age_ms)) {
assessment.lagging.insert(partition);
}
}
assessment
}
/// This cycle's per-partition causes: the two splits' verdicts unioned, plus `ChannelFull` for
/// every partition whose dispatch bounced. Feeds [`PauseLedger::reconcile`], so a partition absent
/// here has drained and its episode ends.
fn merge_cycle_causes(
refence: HashMap<i32, CauseSet>,
fresh: HashMap<i32, CauseSet>,
bounced: impl IntoIterator<Item = i32>,
) -> HashMap<i32, CauseSet> {
let mut merged = refence;
for (partition, causes) in fresh {
let entry = merged.entry(partition).or_default();
*entry = entry.union(causes);
}
for partition in bounced {
merged
.entry(partition)
.or_default()
.insert(PauseCause::ChannelFull);
}
merged
}
/// The admission core of one consume cycle: prune revoked state, assess the gates, split and
/// dispatch held seeds before fresh ones, reconcile the ledger, and emit the pacing gauges.
/// Both clock readings are inputs, so tests can drive it without a consumer.
#[allow(clippy::too_many_arguments)]
fn run_admission_cycle(
dispatcher: &EventDispatcher,
pacing_config: &SeedPacingConfig,
disk_sample: Option<DiskUtilization>,
fence_margin_ms: i64,
seeds: Vec<ConsumedSeed>,
holdover: &mut SeedHoldover,
pacing: &mut SeedPacing,
prev_paused_target: &mut HashSet<i32>,
pause_tx: &tokio::sync::mpsc::UnboundedSender<HashSet<i32>>,
now_ms: i64,
now: Instant,
) {
let owned = dispatcher.owned_set();
holdover.prune_revoked(&owned);
pacing.ledger.drop_unowned(&owned);
let watermarks = dispatcher.merge_deps().live_watermarks.clone();
let watermark_of = |partition: i32| watermarks.get(partition);
let live_lag = assess_live_lag(
&owned,
watermark_of,
&pacing.ledger,
pacing_config.live_lag,
now_ms,
);
for (&partition, &age_ms) in &live_lag.ages_ms {
gauge!(LIVE_WATERMARK_AGE_MS, "partition" => partition.to_string()).set(age_ms as f64);
}
gauge!(SEED_NO_WATERMARK_PARTITIONS).set(live_lag.no_watermark as f64);
// Pod-wide disk verdict; an absent sample can never pause (fail-open).
pacing.disk_engaged = match (pacing_config.disk, disk_sample) {
(Some(hysteresis), Some(sample)) => {
hysteresis.decide(pacing.disk_engaged, UsedPct(sample.used_pct()))
}
_ => false,
};
let policy = AdmissionPolicy {
fence_margin_ms,
watermark_of,
live_lagging: &live_lag.lagging,
disk_pressure: pacing.disk_engaged,
};
// Held before fresh; the channel-full remainder re-absorbs before the still-fenced
// suffix so per-partition FIFO holds.
let refence = split_for_admission(drain_held(holdover), &HashSet::new(), &policy);
let still_full = dispatcher.dispatch_seeds(refence.admitted);
let mut bounced: Vec<i32> = still_full.keys().copied().collect();
holdover.absorb(still_full);
holdover.absorb(refence.held);
// Fresh seeds queue behind held partitions rather than leapfrogging older offsets.
let fresh = split_for_admission(seeds, &holdover.held_partitions(), &policy);
let fresh_full = dispatcher.dispatch_seeds(fresh.admitted);
bounced.extend(fresh_full.keys().copied());
holdover.absorb(fresh_full);
holdover.absorb(fresh.held);
for (partition, deficit) in refence.deficits.into_iter().chain(fresh.deficits) {
if let Some(deficit_ms) = deficit {
gauge!(SEED_FENCE_DEFICIT_MS, "partition" => partition.to_string())
.set(deficit_ms as f64);
}
}
let causes = merge_cycle_causes(refence.causes, fresh.causes, bounced);
// Reconcile owned ∪ caused: owned partitions with no causes drain (episode ends), and a
// just-revoked partition whose polled seeds were held this cycle still enters the ledger so
// the target matches the holdover until both prune next cycle.
let mut to_reconcile = owned;
to_reconcile.extend(causes.keys().copied());
for partition in to_reconcile {
pacing.ledger.reconcile(
partition,
causes.get(&partition).copied().unwrap_or_default(),
now,
);
}
// A partition is paused iff its holdover is non-empty; the ledger is the causes-and-age view
// of that same set.
debug_assert_eq!(pacing.ledger.pause_target(), holdover.held_partitions());
let target = pacing.ledger.pause_target();
for partition in prev_paused_target.difference(&target) {
// Zero the per-partition gauges for drained partitions so they clear.
let label: Arc<str> = Arc::from(partition.to_string());
gauge!(SEED_FENCE_DEFICIT_MS, "partition" => label.clone()).set(0.0);
gauge!(SEED_PAUSE_AGE_MS, "partition" => label.clone()).set(0.0);
gauge!(SEED_OLDEST_HELD_AGE_MS, "partition" => label).set(0.0);
}
// A partition can stay held (live-lag/disk/channel-full) after its fence opens; nothing
// rewrites the deficit then, so clear it rather than pin the last fence-closed reading.
for &partition in &target {
if !pacing.ledger.has_cause(partition, PauseCause::Fence) {
gauge!(SEED_FENCE_DEFICIT_MS, "partition" => partition.to_string()).set(0.0);
}
}
if (!target.is_empty() || !prev_paused_target.is_empty())
&& pause_tx.send(target.clone()).is_err()
{
debug!("seed pauser task has exited; skipping a pause/resume update");
}
for &partition in &target {
if let Some(age) = pacing.ledger.age(partition, now) {
gauge!(SEED_PAUSE_AGE_MS, "partition" => partition.to_string())
.set(age.as_millis() as f64);
}
}
for (partition, head) in holdover.held_heads() {
let age_ms = head.broker_ts_ms.map_or(0, |ts| (now_ms - ts).max(0));
gauge!(SEED_OLDEST_HELD_AGE_MS, "partition" => partition.to_string()).set(age_ms as f64);
}
for cause in PauseCause::ALL {
gauge!(SEED_PAUSED_PARTITIONS, "cause" => cause.as_str())
.set(pacing.ledger.count_with(cause) as f64);
}
gauge!(SEED_FENCED_PARTITIONS).set(holdover.held_partition_count() as f64);
*prev_paused_target = target;
}
/// The seed-topic follower consume loop. Commits go through the seed tracker +
/// `fsync_then_commit`, so a committed offset is a durably-applied tile.
pub struct SeedFollowerConsumer {
consumer: Arc<StreamConsumer>,
topic: String,
stamp: CommitStamp,
/// The events group's consumer, shared for the idle probe's watermark + boot-committed reads.
events_consumer: Arc<StreamConsumer<CohortConsumerContext>>,
events_topic: String,
dispatcher: Arc<EventDispatcher>,
handle: Handle,
pauser: Arc<dyn PartitionPauser>,
recv_batch_size: usize,
recv_batch_timeout: Duration,
offset_commit_interval: Duration,
fence_margin_ms: i64,
idle_probe_interval: Duration,
pacing_config: SeedPacingConfig,
disk_state: Arc<SharedDiskUtilization>,
}
impl SeedFollowerConsumer {
#[allow(clippy::too_many_arguments)]
pub fn new(
consumer: Arc<StreamConsumer>,
topic: String,
stamp: CommitStamp,
events_consumer: Arc<StreamConsumer<CohortConsumerContext>>,
events_topic: String,
dispatcher: Arc<EventDispatcher>,
handle: Handle,
pauser: Arc<dyn PartitionPauser>,
recv_batch_size: usize,
recv_batch_timeout: Duration,
offset_commit_interval: Duration,
fence_margin_ms: i64,
idle_probe_interval: Duration,
pacing_config: SeedPacingConfig,
disk_state: Arc<SharedDiskUtilization>,
) -> Self {
Self {
consumer,
topic,
stamp,
events_consumer,
events_topic,
dispatcher,
handle,
pauser,
recv_batch_size,
recv_batch_timeout,
offset_commit_interval,
fence_margin_ms,
idle_probe_interval,
pacing_config,
disk_state,
}
}
pub async fn process(self) {
let _guard = self.handle.process_scope();
info!(topic = %self.topic, "seed follower consume loop starting");
// Pause/resume off-loop: librdkafka's calls are synchronous FFI and must never delay
// the heartbeat.
let (pause_tx, pause_rx) = tokio::sync::mpsc::unbounded_channel::<HashSet<i32>>();
let pauser_task = tokio::spawn(run_pauser_loop(self.pauser.clone(), pause_rx));
let probe_task = tokio::spawn(run_idle_probe_loop(
self.events_consumer.clone(),
self.events_topic.clone(),
self.dispatcher.clone(),
self.idle_probe_interval,
self.handle.clone(),
));
let mut holdover = SeedHoldover::default();
let mut pacing = SeedPacing::default();
let mut prev_paused_target: HashSet<i32> = HashSet::new();
let mut commit_deadline = tokio::time::Instant::now() + self.offset_commit_interval;
let owned_at_shutdown = loop {
tokio::select! {
biased;
_ = self.handle.shutdown_recv() => {
info!(topic = %self.topic, "shutdown signal received, stopping seed consume loop");
break self.dispatcher.owned_set();
}
outcome = self.consume_batch() => {
self.cycle(outcome, &mut holdover, &mut pacing, &mut prev_paused_target, &pause_tx).await;
let now = tokio::time::Instant::now();
if now >= commit_deadline {
fsync_then_commit(
self.dispatcher.handle(),
&self.consumer,
&self.stamp,
self.owned_committable_offsets(),
&self.topic,
CommitMode::Async,
)
.await;
commit_deadline = now + self.offset_commit_interval;
}
}
}
};
drop(pause_tx);
if let Err(err) = pauser_task.await {
warn!(error = %err, "seed pauser task did not exit cleanly");
}
if let Err(err) = probe_task.await {
warn!(error = %err, "seed idle-probe task did not exit cleanly");
}
let offsets = committable_after_drain(
&self.dispatcher.merge_deps().seed_tracker,
&self.dispatcher,
owned_at_shutdown,
&self.topic,
)
.await;
fsync_then_commit(
self.dispatcher.handle(),
&self.consumer,
&self.stamp,
offsets,
&self.topic,
CommitMode::Sync,
)
.await;
info!(topic = %self.topic, "seed follower consume loop stopped");
}
/// One non-blocking cycle: prune revoked holdover, assess the pacing gates, re-admit and
/// redispatch held seeds before fresh ones, dispatch the polled batch, reconcile the paused
/// target and gauges.
async fn cycle(
&self,
outcome: SeedConsumeOutcome,
holdover: &mut SeedHoldover,
pacing: &mut SeedPacing,
prev_paused_target: &mut HashSet<i32>,
pause_tx: &tokio::sync::mpsc::UnboundedSender<HashSet<i32>>,
) {
histogram!(COHORT_STREAM_SEEDS_CONSUME_BATCH_SIZE).record(outcome.seeds.len() as f64);
if !outcome.seeds.is_empty() {
counter!(COHORT_STREAM_SEEDS_CONSUMED).increment(outcome.seeds.len() as u64);
}
if outcome.deserialize_errors > 0 {
counter!(COHORT_STREAM_SEED_DESERIALIZE_ERRORS).increment(outcome.deserialize_errors);
}
run_admission_cycle(
&self.dispatcher,
&self.pacing_config,
self.disk_state.latest(),
self.fence_margin_ms,
outcome.seeds,
holdover,
pacing,
prev_paused_target,
pause_tx,
chrono::Utc::now().timestamp_millis(),
Instant::now(),
);
if outcome.transport_error {
tokio::time::sleep(RECV_ERROR_BACKOFF).await;
} else {
self.handle.report_healthy();
}
}
async fn consume_batch(&self) -> SeedConsumeOutcome {
let mut outcome = SeedConsumeOutcome {
seeds: Vec::with_capacity(self.recv_batch_size),
deserialize_errors: 0,
transport_error: false,
};
tokio::select! {
_ = tokio::time::sleep(self.recv_batch_timeout) => {}
_ = async {
while outcome.seeds.len() < self.recv_batch_size {
match self.consumer.recv().await {
Ok(message) => {
let partition = message.partition();
let offset = message.offset();
let work = decode_payload(message.payload(), partition, offset);
if matches!(work, SeedWork::Skip(SeedSkipReason::DecodeError)) {
outcome.deserialize_errors += 1;
}
outcome.seeds.push(ConsumedSeed {
work,
partition,
offset,
broker_ts_ms: message.timestamp().to_millis(),
});
}
Err(err) => {
outcome.transport_error = true;
counter!(COHORT_STREAM_KAFKA_RECV_ERRORS).increment(1);
warn!(topic = %self.topic, error = %err, "kafka recv error while consuming seed topic");
break;
}
}
}
} => {}
}
outcome
}
fn owned_committable_offsets(&self) -> HashMap<i32, i64> {
owned_committable_offsets(&self.dispatcher.merge_deps().seed_tracker, &self.dispatcher)
}
}
/// Decode one payload into channel-riding work so every outcome retains its in-order offset.
fn decode_payload(payload: Option<&[u8]>, partition: i32, offset: i64) -> SeedWork {
let Some(payload) = payload else {
debug!(
partition,
offset, "skipping seed message with empty payload"
);
return SeedWork::Skip(SeedSkipReason::DecodeError);
};
match decode_seed(payload) {
Ok(DecodedSeed::Tile(tile)) => SeedWork::Tile(tile),
Ok(DecodedSeed::Person(seed)) => SeedWork::Person(seed),
Ok(DecodedSeed::Reconcile(tile)) => SeedWork::Reconcile(tile),
Ok(DecodedSeed::UnknownKind { kind, .. }) => {
debug!(partition, offset, kind, "skipping seed of unknown kind");
SeedWork::Skip(SeedSkipReason::UnknownKind)
}
Ok(DecodedSeed::UnsupportedSchema {
kind,
schema_version,
}) => {
debug!(
partition,
offset, kind, schema_version, "skipping seed at unsupported schema version",
);
SeedWork::Skip(SeedSkipReason::UnsupportedSchema)
}
Err(err) => {
debug!(partition, offset, error = %err, "skipping undeserializable seed message");
SeedWork::Skip(SeedSkipReason::DecodeError)
}
}
}
struct SeedConsumeOutcome {
seeds: Vec<ConsumedSeed>,
deserialize_errors: u64,
transport_error: bool,
}
/// Advance idle partitions' watermarks: once everything retained is folded, "now" is a valid
/// arrival bound. Accepted residual: a silent shuffler stall longer than the margin opens the
/// fence early, surfaced via the watermark-age gauge + shuffler-lag alerting.
async fn run_idle_probe_loop(
events_consumer: Arc<StreamConsumer<CohortConsumerContext>>,
events_topic: String,
dispatcher: Arc<EventDispatcher>,
interval: Duration,
handle: Handle,
) {
let mut ticker = tokio::time::interval(interval);
ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
// Skip the immediate first tick so the probe never races boot recovery.
ticker.tick().await;
loop {
tokio::select! {
biased;
_ = handle.shutdown_recv() => break,
_ = ticker.tick() => {
let owned = dispatcher.owned_partitions();
if owned.is_empty() {
continue;
}
let folded = dispatcher.events_tracker().committable_offsets();
let consumer = events_consumer.clone();
let topic = events_topic.clone();
let pass_started = Instant::now();
let probe = tokio::task::spawn_blocking(move || {
probe_idle_partitions(consumer.as_ref(), &topic, &owned, &folded)
});
// Race against shutdown: the blocking fetch can spend ~5 s per partition against
// an unresponsive broker.
let probed = tokio::select! {
biased;
_ = handle.shutdown_recv() => break,
probed = probe => probed,
};
match probed {
Ok(idle_partitions) => {
histogram!(SEED_IDLE_PROBE_DURATION_SECONDS)
.record(pass_started.elapsed().as_secs_f64());
let now_ms = chrono::Utc::now().timestamp_millis();
gauge!(SEED_IDLE_PROBE_LAST_PASS_TIMESTAMP_SECONDS)
.set(now_ms as f64 / 1_000.0);
advance_probed_idle(
&dispatcher.merge_deps().live_watermarks,
&dispatcher.owned_set(),
idle_partitions,
now_ms,
);
}
Err(err) => warn!(error = %err, "seed idle probe task failed"),
}
}
}
}
}
/// Advance probed-idle watermarks, re-checking ownership at advance time: the probe's snapshot is
/// stale by up to the blocking fetch, and re-creating an entry a mid-probe revoke forgot would
/// open the fence for a later tenure before its replayed events fold. The assign path forgets
/// independently, covering the residual race.
fn advance_probed_idle(
watermarks: &crate::partitions::watermarks::LiveWatermarks,
owned_now: &HashSet<i32>,
idle_partitions: Vec<i32>,
now_ms: i64,
) {
for partition in idle_partitions {
if owned_now.contains(&partition) {
watermarks.advance_idle(partition, now_ms);
}
}
}
/// Blocking half of the idle probe: compare each owned partition's high watermark to the folded
/// frontier, falling back to the events group's committed offset before any fold this tenure.
/// Errors skip the partition (fence stays closed).
fn probe_idle_partitions<C: rdkafka::consumer::ConsumerContext + 'static>(
consumer: &StreamConsumer<C>,
topic: &str,
owned: &[i32],
folded: &HashMap<i32, i64>,
) -> Vec<i32> {
let unfolded: Vec<i32> = owned
.iter()
.copied()
.filter(|partition| !folded.contains_key(partition))
.collect();
let committed: HashMap<i32, i64> = if unfolded.is_empty() {
HashMap::new()
} else {
let mut tpl = TopicPartitionList::new();
for &partition in &unfolded {
tpl.add_partition(topic, partition);
}
match consumer.committed_offsets(tpl, PROBE_FETCH_TIMEOUT) {
Ok(tpl) => tpl
.elements_for_topic(topic)
.iter()
.filter_map(|elem| match elem.offset() {
Offset::Offset(next) => Some((elem.partition(), next)),
_ => None,
})
.collect(),
Err(err) => {
warn!(topic, error = %err, "idle probe: committed-offset fetch failed; boot-edge partitions stay fenced");
HashMap::new()
}
}
};
let mut idle = Vec::new();
for &partition in owned {
let (low, high) = match consumer.fetch_watermarks(topic, partition, PROBE_FETCH_TIMEOUT) {
Ok(bounds) => bounds,
Err(err) => {
debug!(topic, partition, error = %err, "idle probe: watermark fetch failed; partition stays fenced");
continue;
}
};
let frontier = folded
.get(&partition)
.copied()
.or_else(|| committed.get(&partition).copied());
if frontier_is_caught_up(frontier, low, high) {
idle.push(partition);
}
}
idle
}
/// Whether every retained live message is behind the folded frontier (a next-to-consume offset).
/// No frontier is idle only with nothing retained. A wrong `true` opens the fence over unfolded
/// live events.
fn frontier_is_caught_up(frontier: Option<i64>, low: i64, high: i64) -> bool {
match frontier {
Some(next) => next >= high,
None => high == low,
}
}
#[cfg(test)]
mod tests {
use std::num::NonZeroU32;
use cohort_core::seed::{
BehavioralShapeHash, ClaimEpoch, ConditionHash, PersonSeed, ReconcileScope, ReconcileTile,
RunBoundaryMs, RunId, SChunkMs, ScannedAtMs,
};
use uuid::Uuid;
use crate::filters::{CohortId, TeamId};
use crate::partitions::watermarks::LiveWatermarks;
use super::*;
const MARGIN_MS: i64 = 600_000;
fn tile_at(s_chunk_ms: i64) -> SeedTile {
SeedTile::new(
TeamId(2),
Uuid::from_u128(7),
ConditionHash::parse("0123456789abcdef").unwrap(),
NonZeroU32::new(1).unwrap(),
20_614,
SChunkMs(s_chunk_ms),
RunId(Uuid::nil()),
ClaimEpoch(1),
)
}
fn seed(partition: i32, offset: i64, s_chunk_ms: i64) -> ConsumedSeed {
ConsumedSeed {
work: SeedWork::Tile(tile_at(s_chunk_ms)),
partition,
offset,
broker_ts_ms: None,
}
}
fn skip(partition: i32, offset: i64) -> ConsumedSeed {
ConsumedSeed {
work: SeedWork::Skip(SeedSkipReason::UnknownKind),
partition,
offset,
broker_ts_ms: None,
}
}
fn person_seed() -> PersonSeed {
PersonSeed::new(
TeamId(2),
Uuid::from_u128(7),
vec![ConditionHash::parse("0123456789abcdef").unwrap()],
vec![ConditionHash::parse("0123456789abcdef").unwrap()],
ScannedAtMs(1_700_000_000_000),
RunId(Uuid::nil()),
ClaimEpoch(1),
)
.unwrap()
}
fn person(partition: i32, offset: i64) -> ConsumedSeed {
ConsumedSeed {
work: SeedWork::Person(person_seed()),
partition,
offset,
broker_ts_ms: None,
}
}
fn offsets(seeds: &[ConsumedSeed]) -> Vec<i64> {
seeds.iter().map(|seed| seed.offset).collect()
}
/// A policy with only the fence gate active: no live-lag set, no disk pressure.
fn fence_only<F: Fn(i32) -> Option<WatermarkMs>>(
live_lagging: &HashSet<i32>,
watermark_of: F,
) -> AdmissionPolicy<'_, F> {
AdmissionPolicy {
fence_margin_ms: MARGIN_MS,
watermark_of,
live_lagging,
disk_pressure: false,
}
}
fn causes_of(list: &[PauseCause]) -> CauseSet {
let mut set = CauseSet::default();
for &cause in list {
set.insert(cause);
}
set
}
#[test]
fn fence_decision_table_pins_the_boundary_arithmetic() {
let s_chunk = SChunkMs(1_000_000);
// No watermark at all: fail-closed with an unknown deficit.
assert_eq!(
fence_decision(None, s_chunk, MARGIN_MS),
FenceDecision::Closed { deficit_ms: None },
);
// Exactly at the bound is still closed — the watermark must *exceed* s_chunk + margin.
assert_eq!(
fence_decision(Some(WatermarkMs(1_000_000 + MARGIN_MS)), s_chunk, MARGIN_MS),
FenceDecision::Closed {
deficit_ms: Some(0)
},
);
assert_eq!(
fence_decision(
Some(WatermarkMs(1_000_000 + MARGIN_MS + 1)),
s_chunk,
MARGIN_MS
),
FenceDecision::Open,
);
assert_eq!(
fence_decision(Some(WatermarkMs(900_000)), s_chunk, MARGIN_MS),
FenceDecision::Closed {
deficit_ms: Some(100_000 + MARGIN_MS)
},
);
// A far-future s_chunk must not overflow the bound arithmetic.
assert_eq!(
fence_decision(Some(WatermarkMs(i64::MAX)), SChunkMs(i64::MAX), MARGIN_MS),
FenceDecision::Closed {
deficit_ms: Some(0)
},
);
}
#[test]
fn split_admits_the_open_prefix_and_holds_everything_from_the_first_closed_tile() {
let watermarks = LiveWatermarks::new();
// Partition 1's watermark clears s_chunk 100 but not 10_000_000.
watermarks.observe(1, 100 + MARGIN_MS + 1);
let batch = vec![
seed(1, 10, 100), // open
skip(1, 11), // skip in the open prefix: admitted
seed(1, 12, 10_000_000), // closed → holds from here
skip(1, 13), // skip AFTER the closed tile: held, FIFO preserved
seed(1, 14, 100), // would be open, but queues behind the held offset
];
let none = HashSet::new();
let split = split_for_admission(
batch,
&HashSet::new(),
&fence_only(&none, |p| watermarks.get(p)),
);
assert_eq!(offsets(&split.admitted), vec![10, 11]);
assert_eq!(offsets(&split.held[&1]), vec![12, 13, 14]);
assert_eq!(
split.deficits[&1],
Some(10_000_000 + MARGIN_MS - (100 + MARGIN_MS + 1))
);
assert_eq!(split.causes[&1], causes_of(&[PauseCause::Fence]));
}
#[test]
fn split_holds_everything_for_an_absent_watermark_except_leading_skips() {
let batch = vec![skip(3, 1), seed(3, 2, 0), skip(3, 3)];
let none = HashSet::new();
let split = split_for_admission(batch, &HashSet::new(), &fence_only(&none, |_| None));
assert_eq!(
offsets(&split.admitted),
vec![1],
"a leading skip is always admitted; it cannot double-count anything",
);
assert_eq!(offsets(&split.held[&3]), vec![2, 3]);
assert_eq!(split.deficits[&3], None, "no watermark → unknown deficit");
}
#[test]
fn split_queues_an_already_held_partition_entirely_behind_its_holdover() {
let watermarks = LiveWatermarks::new();
watermarks.observe(5, i64::MAX); // fence wide open
let batch = vec![seed(5, 20, 0), skip(5, 21)];
let none = HashSet::new();
let split = split_for_admission(
batch,
&HashSet::from([5]),
&fence_only(&none, |p| watermarks.get(p)),
);
assert!(split.admitted.is_empty(), "nothing leapfrogs a held offset");
assert_eq!(offsets(&split.held[&5]), vec![20, 21]);
}
#[test]
fn split_isolates_partitions_from_each_other() {
let watermarks = LiveWatermarks::new();
watermarks.observe(1, i64::MAX);
// Partition 2 has no watermark: closed.
let batch = vec![seed(1, 1, 0), seed(2, 1, 0), seed(1, 2, 0)];
let none = HashSet::new();
let split = split_for_admission(
batch,
&HashSet::new(),
&fence_only(&none, |p| watermarks.get(p)),
);
assert_eq!(offsets(&split.admitted), vec![1, 2]);
assert!(split.admitted.iter().all(|seed| seed.partition == 1));
assert_eq!(offsets(&split.held[&2]), vec![1]);
}
/// Live-priority violated by leaking fence-open tiles: a lagging partition must hold its
/// whole batch — leading skips included — while other partitions stay unaffected.
#[test]
fn live_lag_gate_holds_everything_including_leading_skips() {
let watermarks = LiveWatermarks::new();
watermarks.observe(3, i64::MAX); // fence wide open — the gate must hold regardless
watermarks.observe(4, i64::MAX);
let batch = vec![skip(3, 1), seed(3, 2, 0), skip(3, 3), seed(4, 7, 0)];
let lagging = HashSet::from([3]);
let split = split_for_admission(
batch,
&HashSet::new(),
&fence_only(&lagging, |p| watermarks.get(p)),
);
assert_eq!(
offsets(&split.admitted),
vec![7],
"only the un-gated partition flows"
);
assert_eq!(offsets(&split.held[&3]), vec![1, 2, 3]);
assert_eq!(split.causes[&3], causes_of(&[PauseCause::LiveLag]));
assert!(
!split.deficits.contains_key(&3),
"an open fence adds neither a fence cause nor a deficit while gated",
);
}
/// The pod-wide disk gate must hold every partition, fence state notwithstanding.
#[test]
fn disk_gate_holds_all_partitions() {
let watermarks = LiveWatermarks::new();
watermarks.observe(1, i64::MAX);
watermarks.observe(2, i64::MAX);
let batch = vec![seed(1, 10, 0), skip(1, 11), seed(2, 20, 0)];
let none = HashSet::new();
let policy = AdmissionPolicy {
disk_pressure: true,
..fence_only(&none, |p| watermarks.get(p))
};
let split = split_for_admission(batch, &HashSet::new(), &policy);
assert!(split.admitted.is_empty());
assert_eq!(offsets(&split.held[&1]), vec![10, 11]);
assert_eq!(offsets(&split.held[&2]), vec![20]);
assert_eq!(split.causes[&1], causes_of(&[PauseCause::DiskPressure]));
assert_eq!(split.causes[&2], causes_of(&[PauseCause::DiskPressure]));
}
/// The fence gauges must stay continuous through a live-lag/disk episode: a gated partition
/// still probes its first tile's fence and attributes the cause + deficit.
#[test]
fn gated_partition_still_attributes_fence_and_deficit() {
let watermarks = LiveWatermarks::new();
watermarks.observe(1, 100 + MARGIN_MS + 1); // clears s_chunk 100, not 10_000_000
let batch = vec![skip(1, 9), seed(1, 10, 10_000_000), seed(1, 11, 100)];
let lagging = HashSet::from([1]);
let split = split_for_admission(
batch,
&HashSet::new(),
&fence_only(&lagging, |p| watermarks.get(p)),
);
assert!(split.admitted.is_empty());
assert_eq!(offsets(&split.held[&1]), vec![9, 10, 11]);
assert_eq!(
split.causes[&1],
causes_of(&[PauseCause::LiveLag, PauseCause::Fence]),
);
assert_eq!(
split.deficits[&1],
Some(10_000_000 + MARGIN_MS - (100 + MARGIN_MS + 1)),
"the first tile's deficit is attributed even while gated",
);
}
/// An absent watermark is fence-fail-closed already; a redundant live-lag hold would poison
/// the hysteresis memory on a new tenure. It must be counted, never gated.
#[test]
fn absent_watermark_counts_no_watermark_but_never_live_lags() {
let watermarks = LiveWatermarks::new();
let now_ms = 1_000_000;
watermarks.observe(1, now_ms - 500_000); // deep lag: gated
// Partition 2 has no watermark at all.
let owned = HashSet::from([1, 2]);
let hysteresis = Hysteresis::new(AgeMs(120_000), AgeMs(60_000)).unwrap();
let assessment = assess_live_lag(
&owned,
|p| watermarks.get(p),
&PauseLedger::default(),
Some(hysteresis),
now_ms,
);
assert_eq!(assessment.lagging, HashSet::from([1]));
assert_eq!(assessment.no_watermark, 1);
assert_eq!(assessment.ages_ms, HashMap::from([(1, 500_000)]));
}
/// The release threshold must apply only to partitions the ledger remembers as live-lagging;
/// an age between the thresholds engages nothing new.
#[test]
fn live_lag_hysteresis_uses_ledger_memory() {
let watermarks = LiveWatermarks::new();
let now_ms = 1_000_000;
// Both partitions sit between release (60s) and engage (120s).
watermarks.observe(1, now_ms - 90_000);
watermarks.observe(2, now_ms - 90_000);
let owned = HashSet::from([1, 2]);
let hysteresis = Hysteresis::new(AgeMs(120_000), AgeMs(60_000)).unwrap();
let mut ledger = PauseLedger::default();
ledger.reconcile(1, causes_of(&[PauseCause::LiveLag]), Instant::now());
let assessment = assess_live_lag(
&owned,
|p| watermarks.get(p),
&ledger,
Some(hysteresis),
now_ms,
);
assert_eq!(
assessment.lagging,
HashSet::from([1]),
"only the remembered partition stays engaged between the thresholds",
);
}
/// A partition whose ledger causes cleared but whose dispatch bounced must re-enter the
/// causes map as channel-full, or the ledger would resume a partition that still holds work.
#[test]
fn merge_cycle_causes_adds_channel_full_for_bounced_partitions() {
let refence = HashMap::from([(1, causes_of(&[PauseCause::Fence]))]);
let fresh = HashMap::from([
(1, causes_of(&[PauseCause::LiveLag])),
(2, causes_of(&[PauseCause::DiskPressure])),
]);
let merged = merge_cycle_causes(refence, fresh, [1, 3]);
assert_eq!(
merged[&1],
causes_of(&[
PauseCause::Fence,
PauseCause::LiveLag,
PauseCause::ChannelFull
]),
);
assert_eq!(merged[&2], causes_of(&[PauseCause::DiskPressure]));
assert_eq!(merged[&3], causes_of(&[PauseCause::ChannelFull]));
}
#[test]
fn holdover_absorb_take_preserves_per_partition_fifo_and_prunes_revoked() {
let mut holdover = SeedHoldover::default();
holdover.absorb(HashMap::from([(1, vec![seed(1, 10, 0), seed(1, 11, 0)])]));
holdover.absorb(HashMap::from([
(1, vec![seed(1, 12, 0)]),
(2, vec![seed(2, 5, 0)]),
]));
assert_eq!(holdover.held_partitions(), HashSet::from([1, 2]));
assert_eq!(holdover.held_partition_count(), 2);
holdover.prune_revoked(&HashSet::from([1]));
assert_eq!(holdover.held_partitions(), HashSet::from([1]));
let taken = drain_held(&mut holdover);
assert_eq!(
offsets(&taken),
vec![10, 11, 12],
"FIFO within the partition"
);
assert!(holdover.held_partitions().is_empty());
}
#[test]
fn decode_payload_classifies_tiles_skips_and_garbage() {
let tile = tile_at(1_700_000_000_000);
let bytes = serde_json::to_vec(&tile).unwrap();
assert!(matches!(
decode_payload(Some(&bytes), 0, 0),
SeedWork::Tile(decoded) if decoded == tile,
));
let reconcile = ReconcileTile::new(
TeamId(2),
CohortId(42),
ReconcileScope::Behavioral(
BehavioralShapeHash::parse(
"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
)
.unwrap(),
),
RunId(Uuid::nil()),
RunBoundaryMs(0),
);
let bytes = serde_json::to_vec(&reconcile).unwrap();
assert!(matches!(
decode_payload(Some(&bytes), 0, 0),
SeedWork::Reconcile(decoded) if decoded == reconcile,
));
// An undecodable person seed would skip-and-commit and never replay, so the decode arm has
// to land before any seeder emits the kind.
let person = person_seed();
let bytes = serde_json::to_vec(&person).unwrap();
assert!(matches!(
decode_payload(Some(&bytes), 0, 0),
SeedWork::Person(decoded) if decoded == person,
));
let mut newer = serde_json::to_value(&tile).unwrap();
newer["schema_version"] = serde_json::json!(2);
let bytes = serde_json::to_vec(&newer).unwrap();
assert!(matches!(
decode_payload(Some(&bytes), 0, 0),
SeedWork::Skip(SeedSkipReason::UnsupportedSchema),
));
assert!(matches!(
decode_payload(Some(b"not json"), 0, 0),
SeedWork::Skip(SeedSkipReason::DecodeError),
));
assert!(matches!(
decode_payload(None, 0, 0),
SeedWork::Skip(SeedSkipReason::DecodeError),
));
}
#[test]
fn reconcile_is_fence_open_but_never_leapfrogs_a_closed_tile() {
let reconcile = ReconcileTile::new(
TeamId(2),
CohortId(42),
ReconcileScope::Behavioral(BehavioralShapeHash::parse("0123456789abcdef").unwrap()),
RunId(Uuid::nil()),
RunBoundaryMs(0),
);
let reconcile_seed = |offset| ConsumedSeed {
work: SeedWork::Reconcile(reconcile.clone()),
partition: 3,
offset,
broker_ts_ms: None,
};
let none = HashSet::new();
let open_prefix = split_for_admission(
vec![reconcile_seed(1), seed(3, 2, 0)],
&HashSet::new(),
&fence_only(&none, |_| None),
);
assert_eq!(offsets(&open_prefix.admitted), vec![1]);
assert_eq!(offsets(&open_prefix.held[&3]), vec![2]);
let closed_prefix = split_for_admission(
vec![seed(3, 3, 0), reconcile_seed(4)],
&HashSet::new(),
&fence_only(&none, |_| None),
);
assert!(closed_prefix.admitted.is_empty());
assert_eq!(offsets(&closed_prefix.held[&3]), vec![3, 4]);
}
#[test]
fn person_seeds_are_fence_open_but_never_leapfrog_a_held_partition() {
let none = HashSet::new();
let open_prefix = split_for_admission(
vec![person(3, 1), seed(3, 2, 0)],
&HashSet::new(),
&fence_only(&none, |_| None),
);
assert_eq!(offsets(&open_prefix.admitted), vec![1]);
assert_eq!(offsets(&open_prefix.held[&3]), vec![2]);
let closed_prefix = split_for_admission(
vec![seed(3, 3, 0), person(3, 4)],
&HashSet::new(),
&fence_only(&none, |_| None),
);
assert!(closed_prefix.admitted.is_empty());
assert_eq!(offsets(&closed_prefix.held[&3]), vec![3, 4]);
let watermarks = LiveWatermarks::new();
watermarks.observe(3, i64::MAX); // fence wide open — live-priority must hold regardless
let lagging = HashSet::from([3]);
let gated = split_for_admission(
vec![person(3, 5), person(3, 6)],
&HashSet::new(),
&fence_only(&lagging, |p| watermarks.get(p)),
);
assert!(gated.admitted.is_empty());
assert_eq!(offsets(&gated.held[&3]), vec![5, 6]);
assert_eq!(gated.causes[&3], causes_of(&[PauseCause::LiveLag]));
}
/// A wrong `true` declares a partition with unfolded live events idle — the fence's
/// fail-open direction.
#[test]
fn frontier_is_caught_up_table() {
let cases = [
(
Some(10),
0,
10,
true,
"folded frontier at the high watermark",
),
(Some(9), 0, 10, false, "one retained message unfolded"),
(
Some(11),
0,
10,
true,
"frontier past the watermark (post-truncation)",
),
(Some(0), 0, 0, true, "empty partition with a zero frontier"),
(
None,
5,
5,
true,
"boot edge: nothing retained, nothing committed",
),
(None, 0, 0, true, "boot edge: never-produced partition"),
(
None,
4,
10,
false,
"no frontier with retained messages stays fenced",
),
];
for (frontier, low, high, expected, why) in cases {
assert_eq!(
frontier_is_caught_up(frontier, low, high),
expected,
"{why}"
);
}
}
/// A revoke mid-probe forgot the watermark; the advance-time re-check must not re-create it.
#[test]
fn probe_advance_never_recreates_a_watermark_revoked_mid_probe() {
let watermarks = LiveWatermarks::new();
watermarks.observe(5, 1_000);
watermarks.observe(6, 1_000);
// Mid-probe: partition 5 is revoked and forgotten; the probe's stale snapshot still
// reports both partitions idle.
watermarks.forget_partition(5);
let owned_now = HashSet::from([6]);
advance_probed_idle(&watermarks, &owned_now, vec![5, 6], 2_000);
assert_eq!(
watermarks.get(5),
None,
"the revoked partition stays fail-closed",
);
assert_eq!(
watermarks.get(6),
Some(WatermarkMs(2_000)),
"the still-owned partition advances",
);
}
mod admission_cycle {
//! Behavioral checks of [`run_admission_cycle`] against a real dispatcher and store;
//! the pause channel and the seed tracker's ceiling are the observables.
use tempfile::TempDir;
use crate::observability::disk::DiskUtilization;
use crate::partitions::{OffsetTracker, PartitionRouter};
use crate::producer::CaptureSink;
use crate::store::{CohortStore, OffloadConfig, OffloadMode, StoreConfig, StoreHandle};
use crate::workers::MergeWorkerDeps;
use super::*;
fn pacing_dispatcher(dir: &TempDir) -> Arc<EventDispatcher> {
let store = CohortStore::open(&StoreConfig {
path: dir.path().join("db"),
..StoreConfig::default()
})
.expect("open store");
let handle = StoreHandle::new(
store,
OffloadConfig {
mode: OffloadMode::All,
event_read_permits: 16,
maintenance_permits: 6,
},
);
Arc::new(EventDispatcher::new(
PartitionRouter::new(64),
Arc::new(OffsetTracker::new()),
handle,
Arc::new(crate::filters::CatalogHandle::new()),
Arc::new(CaptureSink::new()),
MergeWorkerDeps::capture(),
))
}
/// The spawned worker marks a dispatched skip processed; a held skip never advances.
async fn wait_for_committable(dispatcher: &EventDispatcher, partition: i32, expected: i64) {
let deadline = Instant::now() + Duration::from_secs(10);
loop {
let committable = dispatcher
.merge_deps()
.seed_tracker
.committable_offsets()
.get(&partition)
.copied();
if committable == Some(expected) {
return;
}
assert!(
Instant::now() < deadline,
"timed out waiting for partition {partition} committable {expected}, at {committable:?}",
);
tokio::time::sleep(Duration::from_millis(20)).await;
}
}
/// A lagging live watermark holds the polled seeds and pauses the partition within one
/// cycle; recovery past the release threshold redispatches the held offset before the
/// fresh one.
#[tokio::test]
async fn live_lag_pauses_within_a_cycle_and_resumes_held_before_fresh() {
let dir = TempDir::new().unwrap();
let dispatcher = pacing_dispatcher(&dir);
dispatcher.assign_partition(0);
let watermarks = dispatcher.merge_deps().live_watermarks.clone();
let pacing_config = SeedPacingConfig {
live_lag: Some(Hysteresis::new(AgeMs(120_000), AgeMs(60_000)).unwrap()),
disk: None,
};
let mut holdover = SeedHoldover::default();
let mut pacing = SeedPacing::default();
let mut prev = HashSet::new();
let (pause_tx, mut pause_rx) = tokio::sync::mpsc::unbounded_channel();
let now_ms = chrono::Utc::now().timestamp_millis();
watermarks.observe(0, now_ms - 300_000);
run_admission_cycle(
&dispatcher,
&pacing_config,
None,
MARGIN_MS,
vec![skip(0, 5)],
&mut holdover,
&mut pacing,
&mut prev,
&pause_tx,
now_ms,
Instant::now(),
);
assert_eq!(holdover.held_partitions(), HashSet::from([0]));
assert_eq!(pause_rx.try_recv().unwrap(), HashSet::from([0]));
assert!(
dispatcher
.merge_deps()
.seed_tracker
.committable_offsets()
.is_empty(),
"a gated skip never dispatches, so its offset cannot commit",
);
let now_ms = chrono::Utc::now().timestamp_millis();
watermarks.observe(0, now_ms);
run_admission_cycle(
&dispatcher,
&pacing_config,
None,
MARGIN_MS,
vec![skip(0, 6)],
&mut holdover,
&mut pacing,
&mut prev,
&pause_tx,
now_ms,
Instant::now(),
);
assert!(holdover.held_partitions().is_empty());
assert_eq!(pause_rx.try_recv().unwrap(), HashSet::new());
// Committable 7 requires both offsets marked: held 5 redispatched before fresh 6.
wait_for_committable(&dispatcher, 0, 7).await;
}
/// A sample at or above the pause threshold holds every partition, a between-thresholds
/// sample stays engaged (hysteresis), and an absent sample releases — a broken probe
/// must never wedge seeding.
#[tokio::test]
async fn disk_pressure_pauses_all_partitions_and_an_absent_sample_never_pauses() {
let dir = TempDir::new().unwrap();
let dispatcher = pacing_dispatcher(&dir);
dispatcher.assign_partition(1);
dispatcher.assign_partition(2);
let watermarks = dispatcher.merge_deps().live_watermarks.clone();
let now_ms = chrono::Utc::now().timestamp_millis();
watermarks.observe(1, now_ms);
watermarks.observe(2, now_ms);
let pacing_config = SeedPacingConfig {
live_lag: None,
disk: Some(Hysteresis::new(UsedPct(60.0), UsedPct(55.0)).unwrap()),
};
let mut holdover = SeedHoldover::default();
let mut pacing = SeedPacing::default();
let mut prev = HashSet::new();
let (pause_tx, mut pause_rx) = tokio::sync::mpsc::unbounded_channel();
let over = DiskUtilization {
total_bytes: 100,
available_bytes: 30, // 70% used
};
run_admission_cycle(
&dispatcher,
&pacing_config,
Some(over),
MARGIN_MS,
vec![skip(1, 0), skip(2, 0)],
&mut holdover,
&mut pacing,
&mut prev,
&pause_tx,
now_ms,
Instant::now(),
);
assert!(pacing.disk_engaged);
assert_eq!(holdover.held_partitions(), HashSet::from([1, 2]));
assert_eq!(pause_rx.try_recv().unwrap(), HashSet::from([1, 2]));
// Between the thresholds: sticky, still held.
let between = DiskUtilization {
total_bytes: 100,
available_bytes: 43, // 57% used
};
run_admission_cycle(
&dispatcher,
&pacing_config,
Some(between),
MARGIN_MS,
vec![],
&mut holdover,
&mut pacing,
&mut prev,
&pause_tx,
now_ms,
Instant::now(),
);
assert!(
pacing.disk_engaged,
"57% is above the 55% release threshold"
);
assert_eq!(holdover.held_partitions(), HashSet::from([1, 2]));
assert_eq!(pause_rx.try_recv().unwrap(), HashSet::from([1, 2]));
// The probe breaks: fail-open, everything drains.
run_admission_cycle(
&dispatcher,
&pacing_config,
None,
MARGIN_MS,
vec![],
&mut holdover,
&mut pacing,
&mut prev,
&pause_tx,
now_ms,
Instant::now(),
);
assert!(!pacing.disk_engaged);
assert!(holdover.held_partitions().is_empty());
assert_eq!(pause_rx.try_recv().unwrap(), HashSet::new());
wait_for_committable(&dispatcher, 1, 1).await;
wait_for_committable(&dispatcher, 2, 1).await;
}
}
}