:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
//! In-memory reconcile work admitted by the seed consumer.
//!
//! The queue belongs to one partition worker. Its deferred offsets deliberately die with the
//! worker without completing; partition teardown then forgets the tracker tenure so Kafka replays
//! every unfinished reconcile from the beginning.
use std::collections::VecDeque;
use std::sync::atomic::{AtomicI64, Ordering};
use std::sync::Arc;
use std::time::Instant;
use metrics::{counter, gauge, histogram};
use tracing::{debug, info, warn};
use cohort_core::clickhouse_timestamp_to_millis;
use cohort_core::filters::{CohortId, TeamId};
#[cfg(test)]
use cohort_core::seed::RunId;
use cohort_core::seed::{CatalogMiss, ReconcileScope, ReconcileTile, ScopeKind, WithheldReason};
use cohort_core::CohortEligibility;
use crate::filters::manager::{CatalogHandle, CatalogView, RefreshSeq};
use crate::filters::reverse_index::TeamFilters;
use crate::observability::metrics::{
COHORT_STREAM_OFFSET_AHEAD_OF_DISPATCH, RECONCILE_BITS_FIXED_TOTAL,
RECONCILE_JOBS_COMPLETED_TOTAL, RECONCILE_JOBS_DEFERRED_TOTAL, RECONCILE_JOBS_WITHHELD_TOTAL,
RECONCILE_MARKERS_EMITTED_TOTAL, RECONCILE_MARKER_PRODUCE_ERRORS,
RECONCILE_PAGE_DURATION_SECONDS, RECONCILE_QUEUE_DEPTH, RECONCILE_ROWS_EMITTED_TOTAL,
RECONCILE_ROWS_SCANNED_TOTAL,
};
use crate::partitions::offset_tracker::{DeferredOffset, MarkOutcome, OffsetTracker};
use crate::producer::{
ChangeOrigin, CohortMembershipChange, MembershipSink, NoopReconcileMarkerSink,
ReconcileCompleteMarker, ReconcileMarker, ReconcileMarkerSink, ReconcileWithheldMarker,
};
use crate::stage2::Stage2State;
use crate::store::{
ReadLane, SliceCoverage, Stage2CohortPrefix, Stage2DirtyKey, Stage2Key, StagedBatch,
StoreHandle,
};
use crate::workers::merge_path::MergeWorkerDeps;
use crate::workers::reconcile_page::read_page;
#[cfg(test)]
use crate::workers::reconcile_page::MAX_ROWS;
use crate::workers::worker::{
count_by_status, first_cascades, produce_cascades, produce_membership,
};
/// Emit a `warn!` carrying the reconcile job identity (`team_id`, `cohort_id`, `run_id`) read from
/// `$tile`, so the drain warnings share one definition of those fields. Extra fields and the message
/// follow exactly as in a bare `warn!`: `warn_job!(tile, partition_id, error = %error, "…")`.
macro_rules! warn_job {
($tile:expr, $($rest:tt)*) => {
warn!(
team_id = $tile.team_id().0,
cohort_id = $tile.cohort_id().0,
run_id = %$tile.run_id().0,
$($rest)*
)
};
}
/// Default number of Stage 2 rows one reconcile drain tick may scan.
pub const DEFAULT_RECONCILE_SCAN_PAGE: usize = 256;
/// Pod-wide count of admitted jobs still owned by partition queues.
#[derive(Debug, Default)]
pub struct ReconcileBacklog(AtomicI64);
impl ReconcileBacklog {
pub fn len(&self) -> i64 {
self.0.load(Ordering::Relaxed)
}
pub fn is_empty(&self) -> bool {
self.len() == 0
}
fn add(&self) {
self.0.fetch_add(1, Ordering::Relaxed);
}
fn done(&self, count: usize) {
let count = i64::try_from(count).expect("an in-memory queue cannot exceed i64::MAX jobs");
let updated = self
.0
.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| {
current.checked_sub(count).filter(|next| *next >= 0)
});
debug_assert!(updated.is_ok(), "reconcile backlog ownership underflow");
}
}
/// Runtime dependencies shared by every partition's reconcile queue.
#[derive(Clone)]
pub struct ReconcileDeps {
pub enabled: bool,
pub scan_page: usize,
pub backlog: Arc<ReconcileBacklog>,
/// Sink for `cohort_reconcile_markers`. With the reconcile gate off this is
/// [`NoopReconcileMarkerSink`], which fails every produce rather than acking a marker the seeder
/// would then wait for.
pub marker_sink: Arc<dyn ReconcileMarkerSink>,
}
impl Default for ReconcileDeps {
fn default() -> Self {
Self {
enabled: false,
scan_page: DEFAULT_RECONCILE_SCAN_PAGE,
backlog: Arc::new(ReconcileBacklog::default()),
marker_sink: Arc::new(NoopReconcileMarkerSink),
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
enum ScanPhase {
Scanning { cursor: Option<Vec<u8>> },
DrainingDirty { cursor: Option<Vec<u8>> },
MarkerReady,
}
#[derive(Debug)]
struct ReconcileJob {
tile: ReconcileTile,
offset: DeferredOffset,
/// The newest catalog refresh that had begun when the job was admitted. Only a snapshot from a
/// later refresh can discard it.
admitted_after: RefreshSeq,
/// Set the first time the job waits for that refresh, so the wait counts and logs once.
deferral_counted: bool,
dirty_tracking: Option<crate::store::Stage2DirtyTrackingGuard>,
phase: ScanPhase,
rows_scanned: u64,
bits_fixed: u64,
}
pub(crate) enum SupersedeOutcome {
NoQueuedJob,
Replaced(DeferredOffset),
RetainedNewerOrEqual,
}
/// FIFO reconcile work for one partition worker.
pub(crate) struct ReconcileQueue {
jobs: VecDeque<ReconcileJob>,
backlog: Arc<ReconcileBacklog>,
partition_label: Arc<str>,
handle: StoreHandle,
/// `enqueue` records the newest refresh this handle has begun on each job, and the drain compares
/// it with the refresh this handle last published.
catalog: Arc<CatalogHandle>,
/// Fixed for the worker's life: a record changes only when its slice is deleted.
coverage: SliceCoverage,
}
impl ReconcileQueue {
pub(crate) fn new(
partition_id: u16,
backlog: Arc<ReconcileBacklog>,
handle: StoreHandle,
catalog: Arc<CatalogHandle>,
coverage: SliceCoverage,
) -> Self {
let queue = Self {
jobs: VecDeque::new(),
backlog,
partition_label: Arc::from(partition_id.to_string()),
handle,
catalog,
coverage,
};
queue.record_depth();
queue
}
pub(crate) fn enqueue(&mut self, tile: ReconcileTile, offset: DeferredOffset) {
self.jobs.push_back(ReconcileJob {
tile,
offset,
admitted_after: self.catalog.last_begun(),
deferral_counted: false,
dirty_tracking: None,
phase: ScanPhase::Scanning { cursor: None },
rows_scanned: 0,
bits_fixed: 0,
});
self.backlog.add();
self.record_depth();
}
/// Start dirty capture only for the queue head, immediately before its first scan. Mutations
/// while a job waits behind another snapshot are covered by its future full scan and need no
/// per-person metadata.
fn activate_front_tracking(&mut self, prefix: Stage2CohortPrefix) {
if self
.jobs
.front()
.is_some_and(|job| job.dirty_tracking.is_none())
{
let tracking = self.handle.track_stage2_dirty(prefix);
self.jobs
.front_mut()
.expect("the queue head was present before acquiring its tracking lease")
.dirty_tracking = Some(tracking);
}
}
/// Replace a queued job only when the incoming Kafka offset is newer. A follower rewind may
/// replay an older job while a newer run is still queued; that replay must never evict the newer
/// snapshot.
///
/// Scoped by kind as well as cohort: a mixed cohort can be reconciled by a behavioral and a
/// person-property run at once, and those two snapshots answer different fences — evicting one
/// for the other would leave its run permanently short a marker.
pub(crate) fn supersede_if_newer(
&mut self,
team_id: TeamId,
cohort_id: CohortId,
scope: ScopeKind,
incoming_offset: i64,
) -> SupersedeOutcome {
let Some(position) = self.jobs.iter().position(|job| {
job.tile.team_id() == team_id
&& job.tile.cohort_id() == cohort_id
&& job.tile.scope().kind() == scope
}) else {
return SupersedeOutcome::NoQueuedJob;
};
if incoming_offset <= self.jobs[position].offset.offset() {
return SupersedeOutcome::RetainedNewerOrEqual;
}
let job = self
.jobs
.remove(position)
.expect("position came from the same queue");
self.backlog.done(1);
self.record_depth();
SupersedeOutcome::Replaced(job.offset)
}
fn front(&self) -> Option<&ReconcileJob> {
self.jobs.front()
}
fn front_mut(&mut self) -> Option<&mut ReconcileJob> {
self.jobs.front_mut()
}
fn finish_front(&mut self) -> Option<ReconcileJob> {
let job = self.jobs.pop_front()?;
self.backlog.done(1);
self.record_depth();
Some(job)
}
fn record_depth(&self) {
gauge!(RECONCILE_QUEUE_DEPTH, "partition" => self.partition_label.clone())
.set(self.jobs.len() as f64);
}
#[cfg(test)]
pub(crate) fn len(&self) -> usize {
self.jobs.len()
}
#[cfg(test)]
pub(crate) fn front_run_id(&self) -> Option<RunId> {
self.jobs.front().map(|job| job.tile.run_id())
}
}
impl Drop for ReconcileQueue {
fn drop(&mut self) {
self.backlog.done(self.jobs.len());
gauge!(RECONCILE_QUEUE_DEPTH, "partition" => self.partition_label.clone()).set(0.0);
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum ReconcileRetryReason {
CatalogNotLoaded,
/// The snapshot predates the job, so the miss may be a write it has not read yet.
AwaitingRefresh(CatalogMiss),
}
#[derive(Debug)]
enum ReconcileGuard {
/// Carries the team filters and eligibility the guard found, so the drain does not look them up
/// again.
Proceed {
filters: Arc<TeamFilters>,
eligibility: CohortEligibility,
},
Retry(ReconcileRetryReason),
Withhold(WithheldReason),
}
/// The catalog's answer alone, with no view of how fresh the catalog is.
fn catalog_verdict(
filters: Option<&Arc<TeamFilters>>,
tile: &ReconcileTile,
) -> Result<(Arc<TeamFilters>, CohortEligibility), CatalogMiss> {
let Some(filters) = filters else {
return Err(CatalogMiss::TeamAbsent);
};
if !filters.cohorts.contains_key(&tile.cohort_id()) {
return Err(CatalogMiss::CohortAbsent);
}
let Some(&eligibility) = filters.eligibility.get(&tile.cohort_id()) else {
return Err(CatalogMiss::CohortAbsent);
};
if !eligibility.registers_membership() {
return Err(CatalogMiss::NotEmitting);
}
// The tile's scope picks the guard map, so a person hash is never checked against a behavioral
// one. An absent entry means the cohort has no leaves of that kind (the loader omits the column
// when it is empty) or the run was superseded; either way this is the intended fail-closed skip,
// observable via the `hash_unknown` withheld marker.
let matches = match tile.scope() {
ReconcileScope::Behavioral(pinned) => filters
.behavioral_shape_hashes
.get(&tile.cohort_id())
.map(|current| current == pinned),
ReconcileScope::PersonProperty(pinned) => filters
.person_shape_hashes
.get(&tile.cohort_id())
.map(|current| current == pinned),
};
match matches {
None => Err(CatalogMiss::HashUnknown),
Some(false) => Err(CatalogMiss::HashMismatch),
Some(true) => Ok((filters.clone(), eligibility)),
}
}
/// The one place that decides a job's fate. Every catalog miss reads the catalog, so a miss
/// against a snapshot whose refresh began before the job was admitted may only mean that refresh
/// had not read the run's writes yet. Such a miss waits. After one later refresh it stands, so a
/// superseded run still fails closed.
///
/// Coverage is checked after the catalog and before any scan, so a partial slice emits no rows.
fn evaluate_guard(
view: &CatalogView,
admitted_after: RefreshSeq,
tile: &ReconcileTile,
coverage: SliceCoverage,
) -> ReconcileGuard {
let (published, catalog) = match view {
CatalogView::NotLoaded => {
return ReconcileGuard::Retry(ReconcileRetryReason::CatalogNotLoaded)
}
CatalogView::Loaded { published, catalog } => (*published, catalog),
};
match catalog_verdict(catalog.team(tile.team_id()), tile) {
Err(miss) if published > admitted_after => {
ReconcileGuard::Withhold(WithheldReason::Catalog(miss))
}
Err(miss) => ReconcileGuard::Retry(ReconcileRetryReason::AwaitingRefresh(miss)),
Ok((filters, eligibility)) => match coverage.admits(tile.boundary()) {
Ok(()) => ReconcileGuard::Proceed {
filters,
eligibility,
},
Err(covered_since) => {
ReconcileGuard::Withhold(WithheldReason::PartialCoverage { covered_since })
}
},
}
}
/// Advance the queue head by at most one scan page or one marker produce per tick.
#[allow(clippy::too_many_arguments)]
#[hotpath::measure]
pub(crate) async fn handle_reconcile_drain(
partition_id: u16,
handle: &StoreHandle,
sink: &Arc<dyn MembershipSink>,
merge: &MergeWorkerDeps,
queue: &mut ReconcileQueue,
last_updated: &str,
) {
loop {
let Some(job) = queue.front() else {
return;
};
let tile = job.tile.clone();
let phase = job.phase.clone();
let source_offset = job.offset.offset();
let (filters, eligibility) = match evaluate_guard(
&queue.catalog.view(),
job.admitted_after,
&tile,
queue.coverage,
) {
ReconcileGuard::Retry(ReconcileRetryReason::CatalogNotLoaded) => {
debug!(
partition_id,
team_id = tile.team_id().0,
cohort_id = tile.cohort_id().0,
run_id = %tile.run_id().0,
"reconcile drain waiting for the first filter catalog load",
);
return;
}
ReconcileGuard::Retry(ReconcileRetryReason::AwaitingRefresh(reason)) => {
let job = queue
.front_mut()
.expect("the guarded queue head is still present");
if !job.deferral_counted {
job.deferral_counted = true;
counter!(
RECONCILE_JOBS_DEFERRED_TOTAL,
"reason" => reason.as_str(),
"kind" => tile.scope().kind().as_str(),
)
.increment(1);
info!(
partition_id,
team_id = tile.team_id().0,
cohort_id = tile.cohort_id().0,
run_id = %tile.run_id().0,
reason = reason.as_str(),
kind = tile.scope().kind().as_str(),
"reconcile job waits for a catalog refresh that began after it arrived",
);
}
return;
}
ReconcileGuard::Withhold(reason) => {
drain_withheld(partition_id, merge, queue, reason).await;
return;
}
ReconcileGuard::Proceed {
filters,
eligibility,
} => (filters, eligibility),
};
// Only flipped bits of a full-tree cohort cascade to referrers; single-leaf fixes never do.
let cascades_flips = eligibility.writes_cf_stage2();
let prefix = Stage2CohortPrefix {
partition_id,
team_id: tile.team_id().0 as u64,
cohort_id: tile.cohort_id().0 as u64,
};
queue.activate_front_tracking(prefix);
let step = match phase {
ScanPhase::Scanning { cursor } => {
drain_scanning(
partition_id,
handle,
sink,
merge,
queue,
&tile,
&filters,
cascades_flips,
prefix,
source_offset,
cursor,
last_updated,
)
.await
}
ScanPhase::DrainingDirty { cursor } => {
drain_dirty(
partition_id,
handle,
sink,
merge,
queue,
&tile,
&filters,
cascades_flips,
prefix,
source_offset,
cursor,
last_updated,
)
.await
}
ScanPhase::MarkerReady => {
drain_marker(
partition_id,
handle,
merge,
queue,
&tile,
prefix,
last_updated,
)
.await
}
};
match step {
DrainStep::Reenter => continue,
DrainStep::Yield => return,
}
}
}
/// Whether the drain loop should re-enter against the head's newly transitioned phase
/// ([`DrainStep::Reenter`]) or yield the tick because its work budget is spent ([`DrainStep::Yield`]).
enum DrainStep {
Reenter,
Yield,
}
/// Scan one Stage 2 page for the cohort and emit its current membership. An empty page hands off to
/// dirty verification; a settled non-empty page is the tick's whole budget.
#[allow(clippy::too_many_arguments)]
async fn drain_scanning(
partition_id: u16,
handle: &StoreHandle,
sink: &Arc<dyn MembershipSink>,
merge: &MergeWorkerDeps,
queue: &mut ReconcileQueue,
tile: &ReconcileTile,
filters: &Arc<TeamFilters>,
cascades_flips: bool,
prefix: Stage2CohortPrefix,
source_offset: i64,
cursor: Option<Vec<u8>>,
last_updated: &str,
) -> DrainStep {
let page = match handle
.scan_stage2_cohort(prefix, cursor, merge.reconcile.scan_page)
.await
{
Ok(page) => page,
Err(error) => {
warn_job!(
tile,
partition_id,
error = %error,
"reconcile Stage 2 scan failed; retrying this page on the next tick",
);
return DrainStep::Yield;
}
};
if page.is_empty() {
queue
.front_mut()
.expect("the scanned queue head is still present")
.phase = ScanPhase::DrainingDirty { cursor: None };
return DrainStep::Reenter;
}
let dirty_to_clear: Vec<Stage2DirtyKey> =
page.iter().copied().map(Stage2DirtyKey::new).collect();
let Some(progress) = settle_reconcile_page(
partition_id,
handle,
sink,
merge,
tile,
filters,
cascades_flips,
&page,
&dirty_to_clear,
source_offset,
last_updated,
)
.await
else {
return DrainStep::Yield;
};
let short_page = page.len() < merge.reconcile.scan_page;
let next_cursor = page
.last()
.expect("a non-empty page has a final key")
.encode()
.to_vec();
let job = queue
.front_mut()
.expect("the advanced queue head is still present");
job.rows_scanned = job.rows_scanned.saturating_add(progress.rows_scanned);
job.bits_fixed = job.bits_fixed.saturating_add(progress.bits_fixed);
job.phase = if short_page {
ScanPhase::DrainingDirty { cursor: None }
} else {
ScanPhase::Scanning {
cursor: Some(next_cursor),
}
};
// One non-empty settlement page is the tick's work budget. Even a short final main page leaves
// dirty verification for the next tick, so mutations behind the cursor cannot silently double the
// configured per-tick ceiling.
DrainStep::Yield
}
/// Re-settle rows mutated behind the main scan, verifying from the cohort prefix until the dirty set
/// is empty before advancing to the marker.
#[allow(clippy::too_many_arguments)]
async fn drain_dirty(
partition_id: u16,
handle: &StoreHandle,
sink: &Arc<dyn MembershipSink>,
merge: &MergeWorkerDeps,
queue: &mut ReconcileQueue,
tile: &ReconcileTile,
filters: &Arc<TeamFilters>,
cascades_flips: bool,
prefix: Stage2CohortPrefix,
source_offset: i64,
cursor: Option<Vec<u8>>,
last_updated: &str,
) -> DrainStep {
let page = match handle
.scan_stage2_dirty(prefix, cursor.clone(), merge.reconcile.scan_page)
.await
{
Ok(page) => page,
Err(error) => {
warn_job!(
tile,
partition_id,
error = %error,
"reconcile dirty scan failed; retrying this page on the next tick",
);
return DrainStep::Yield;
}
};
if page.is_empty() {
let job = queue
.front_mut()
.expect("the dirty-scanning queue head is still present");
if cursor.is_some() {
// Verify once more from the prefix so a marker inserted behind the cursor
// between ticks cannot be missed.
job.phase = ScanPhase::DrainingDirty { cursor: None };
} else {
job.phase = ScanPhase::MarkerReady;
}
return DrainStep::Reenter;
}
let stage2_keys: Vec<Stage2Key> = page.iter().map(|dirty| dirty.stage2_key()).collect();
let current_rows = match handle
.multi_get_stage2(stage2_keys.clone(), ReadLane::Maintenance)
.await
{
Ok(rows) => rows,
Err(error) => {
warn_job!(
tile,
partition_id,
error = %error,
"reconcile dirty-row read failed; retrying this page on the next tick",
);
return DrainStep::Yield;
}
};
let existing_keys: Vec<Stage2Key> = stage2_keys
.into_iter()
.zip(current_rows)
.filter_map(|(key, value)| value.is_some().then_some(key))
.collect();
let Some(progress) = settle_reconcile_page(
partition_id,
handle,
sink,
merge,
tile,
filters,
cascades_flips,
&existing_keys,
&page,
source_offset,
last_updated,
)
.await
else {
return DrainStep::Yield;
};
let short_page = page.len() < merge.reconcile.scan_page;
let next_cursor = page
.last()
.expect("a non-empty dirty page has a final key")
.encode()
.to_vec();
let job = queue
.front_mut()
.expect("the advanced dirty queue head is still present");
job.rows_scanned = job.rows_scanned.saturating_add(progress.rows_scanned);
job.bits_fixed = job.bits_fixed.saturating_add(progress.bits_fixed);
job.phase = ScanPhase::DrainingDirty {
cursor: Some(next_cursor.clone()),
};
// A full page does not prove exhaustion, so first look strictly after it. This is
// verification only: if another page exists, the next tick settles it.
if !short_page {
match handle.scan_stage2_dirty(prefix, Some(next_cursor), 1).await {
Ok(remaining) if !remaining.is_empty() => return DrainStep::Yield,
Ok(_) => {}
Err(error) => {
warn_job!(
tile,
partition_id,
error = %error,
"reconcile dirty tail verification failed; retrying next tick",
);
return DrainStep::Yield;
}
}
}
// Verify once from the prefix to catch work inserted behind an earlier cursor. The
// worker owns this partition serially, so an empty result immediately after the
// settlement closes the hot-single-key case even when page size is one. Never
// settle a second nonempty page in this tick.
queue
.front_mut()
.expect("the verified dirty queue head is still present")
.phase = ScanPhase::DrainingDirty { cursor: None };
match handle.scan_stage2_dirty(prefix, None, 1).await {
Ok(remaining) if !remaining.is_empty() => DrainStep::Yield,
Ok(_) => {
queue
.front_mut()
.expect("the verified dirty queue head is still present")
.phase = ScanPhase::MarkerReady;
DrainStep::Reenter
}
Err(error) => {
warn_job!(
tile,
partition_id,
error = %error,
"reconcile dirty prefix verification failed; retrying next tick",
);
DrainStep::Yield
}
}
}
/// Withhold the consecutive withheld jobs at the head in one produce, releasing the acked prefix.
async fn drain_withheld(
partition_id: u16,
merge: &MergeWorkerDeps,
queue: &mut ReconcileQueue,
head: WithheldReason,
) {
let view = queue.catalog.view();
let coverage = queue.coverage;
let reasons: Vec<WithheldReason> = std::iter::once(head)
.chain(queue.jobs.iter().skip(1).map_while(|job| {
match evaluate_guard(&view, job.admitted_after, &job.tile, coverage) {
ReconcileGuard::Withhold(reason) => Some(reason),
ReconcileGuard::Proceed { .. } | ReconcileGuard::Retry(_) => None,
}
}))
// A tick produces at most one scan page of messages, as a scan tick does, so a burst spreads
// over ticks instead of holding the live lane behind one unbounded produce.
.take(merge.reconcile.scan_page)
.collect();
let markers = queue
.jobs
.iter()
.zip(&reasons)
.map(|(job, &reason)| {
ReconcileMarker::Withheld(ReconcileWithheldMarker::new(
job.tile.team_id(),
job.tile.cohort_id(),
partition_id,
job.tile.run_id(),
reason,
))
})
.collect();
let acks = merge.reconcile.marker_sink.produce(markers).await;
let acked = acks
.iter()
.take_while(|ack| ack.is_ok())
.count()
.min(reasons.len());
for reason in &reasons[..acked] {
let job = queue
.finish_front()
.expect("an acked withheld job is still queued");
let kind = job.tile.scope().kind().as_str();
counter!(RECONCILE_MARKERS_EMITTED_TOTAL, "kind" => kind, "marker" => "withheld")
.increment(1);
complete_offset(
&merge.seed_tracker,
partition_id,
job.offset,
"withheld reconcile",
);
counter!(
RECONCILE_JOBS_WITHHELD_TOTAL,
"reason" => reason.as_str(),
"kind" => kind,
)
.increment(1);
warn_job!(
job.tile,
partition_id,
reason = reason.as_str(),
covered_since_ms = reason.covered_since().map(|since| since.0),
kind,
"withholding the reconcile certificate",
);
}
if acked < reasons.len() {
let tile = &queue
.front()
.expect("the unacked withheld job is still queued")
.tile;
counter!(
RECONCILE_MARKER_PRODUCE_ERRORS,
"kind" => tile.scope().kind().as_str(),
"marker" => "withheld",
)
.increment(1);
warn_job!(
tile,
partition_id,
unacked = reasons.len() - acked,
"reconcile withheld-marker produce failed; deciding the jobs again on the next tick",
);
}
}
/// Emit the per-partition completion marker once the dirty set is drained. A failed marker produce
/// retries the marker only; work dirtied while the marker was pending is settled before the retry.
async fn drain_marker(
partition_id: u16,
handle: &StoreHandle,
merge: &MergeWorkerDeps,
queue: &mut ReconcileQueue,
tile: &ReconcileTile,
prefix: Stage2CohortPrefix,
last_updated: &str,
) -> DrainStep {
// A marker produce may have failed on a prior tick. Recheck the dirty prefix before
// every retry so newly arrived work is settled first without rescanning the cohort.
let pending_dirty = match handle.scan_stage2_dirty(prefix, None, 1).await {
Ok(page) => page,
Err(error) => {
warn_job!(
tile,
partition_id,
error = %error,
"reconcile dirty verification failed before marker; retrying on the next tick",
);
return DrainStep::Yield;
}
};
if !pending_dirty.is_empty() {
queue
.front_mut()
.expect("the marker-ready queue head is still present")
.phase = ScanPhase::DrainingDirty { cursor: None };
return DrainStep::Reenter;
}
let marker = ReconcileMarker::Complete(ReconcileCompleteMarker::new(
tile.team_id(),
tile.cohort_id(),
partition_id,
tile.run_id(),
last_updated.to_string(),
));
let acks = merge.reconcile.marker_sink.produce(vec![marker]).await;
let kind = tile.scope().kind().as_str();
// Exactly one marker went in, so anything but a single successful ack is a produce failure.
let failed_acks = acks.iter().filter(|ack| ack.is_err()).count();
if acks.len() != 1 || failed_acks > 0 {
counter!(RECONCILE_MARKER_PRODUCE_ERRORS, "kind" => kind, "marker" => "complete")
.increment(1);
warn_job!(
tile,
partition_id,
ack_count = acks.len(),
failed_acks,
"reconcile completion-marker produce failed; rechecking dirty work before retry",
);
return DrainStep::Yield;
}
counter!(RECONCILE_MARKERS_EMITTED_TOTAL, "kind" => kind, "marker" => "complete").increment(1);
let completed = queue
.finish_front()
.expect("the marker-producing queue head is still present");
complete_offset(
&merge.seed_tracker,
partition_id,
completed.offset,
"completed reconcile",
);
counter!(RECONCILE_JOBS_COMPLETED_TOTAL, "kind" => kind).increment(1);
info!(
partition_id,
team_id = tile.team_id().0,
cohort_id = tile.cohort_id().0,
run_id = %tile.run_id().0,
rows_scanned = completed.rows_scanned,
bits_fixed = completed.bits_fixed,
"reconcile job completed",
);
DrainStep::Yield
}
struct PageProgress {
rows_scanned: u64,
bits_fixed: u64,
}
/// The ordered steps one settlement page takes. Doubles as the `stage` metric label, so the
/// duration histogram can never name a step the page does not have.
#[derive(Debug, Clone, Copy)]
enum PageStage {
/// Every section of the page's read and evaluation, permit waits included.
Recompute,
MembershipProduce,
CascadeProduce,
Commit,
}
impl PageStage {
fn as_str(self) -> &'static str {
match self {
Self::Recompute => "recompute",
Self::MembershipProduce => "membership_produce",
Self::CascadeProduce => "cascade_produce",
Self::Commit => "commit",
}
}
}
/// Splits one page's wall clock into per-stage samples: each [`mark`](Self::mark) records the time
/// since the previous one. A page that fails records no sample for the step that failed, so the
/// histogram stays a picture of settled work.
struct PageClock {
last: Instant,
}
impl PageClock {
fn start() -> Self {
Self {
last: Instant::now(),
}
}
fn mark(&mut self, stage: PageStage) {
let now = Instant::now();
histogram!(RECONCILE_PAGE_DURATION_SECONDS, "stage" => stage.as_str())
.record(now.duration_since(self.last).as_secs_f64());
self.last = now;
}
}
/// Emit one current reconcile page, then atomically commit any corrected bits and clear the exact
/// dirty markers covered by that evaluation. Returning `None` leaves the queue phase unchanged so
/// the same page is retried on the next tick.
#[allow(clippy::too_many_arguments)]
async fn settle_reconcile_page(
partition_id: u16,
handle: &StoreHandle,
sink: &Arc<dyn MembershipSink>,
merge: &MergeWorkerDeps,
tile: &ReconcileTile,
filters: &Arc<TeamFilters>,
cascades_flips: bool,
page: &[Stage2Key],
dirty_to_clear: &[Stage2DirtyKey],
source_offset: i64,
last_updated: &str,
) -> Option<PageProgress> {
let evaluated_at_ms = clickhouse_timestamp_to_millis(last_updated)
.expect("worker-generated last_updated timestamps always parse");
let mut clock = PageClock::start();
let diffs = match read_page(handle, filters, tile.cohort_id(), page).await {
Ok(diffs) => diffs,
Err(error) => {
warn_job!(
tile,
partition_id,
rows = page.len(),
error = %error,
"reconcile page read failed; retrying this page on the next tick",
);
return None;
}
};
clock.mark(PageStage::Recompute);
debug_assert_eq!(
diffs.len(),
page.len(),
"the page reader returns one diff per requested row",
);
let mut changes = Vec::with_capacity(diffs.len());
let mut cascade_changes = Vec::new();
let mut writes: Vec<(Stage2Key, Stage2State)> = Vec::new();
let mut fixed_entered = 0u64;
let mut fixed_left = 0u64;
for diff in &diffs {
let change = CohortMembershipChange {
team_id: tile.team_id().0,
cohort_id: tile.cohort_id().0,
person_id: diff.stage2_key.person_id.to_string(),
last_updated: last_updated.to_string(),
status: diff.status(),
origin: Some(ChangeOrigin::Reconcile),
run_id: Some(tile.run_id()),
};
if diff.requires_write() {
writes.push((
diff.stage2_key,
Stage2State {
in_cohort: diff.new_bit,
last_evaluated_at_ms: evaluated_at_ms,
},
));
}
if diff.flipped() {
if diff.new_bit {
fixed_entered += 1;
} else {
fixed_left += 1;
}
if cascades_flips {
cascade_changes.push(change.clone());
}
}
changes.push(change);
}
let cascades = first_cascades(merge, &cascade_changes, source_offset);
let (entered, left) = count_by_status(&changes);
let membership_errors = if changes.is_empty() {
0
} else {
produce_membership(sink, changes).await
};
if membership_errors > 0 {
warn_job!(
tile,
partition_id,
errors = membership_errors,
"reconcile membership produce failed; retrying this page on the next tick",
);
return None;
}
clock.mark(PageStage::MembershipProduce);
let cascade_errors = produce_cascades(merge, cascades).await;
if cascade_errors > 0 {
warn_job!(
tile,
partition_id,
errors = cascade_errors,
"reconcile cascade produce failed; retrying this page on the next tick",
);
return None;
}
clock.mark(PageStage::CascadeProduce);
let mut staged = StagedBatch::default();
for (key, state) in &writes {
staged.put_stage2(key, &state.encode());
}
// These deletes are deliberately staged after the puts. A fix marks its row dirty through the
// normal typed write path, then this page-clear consumes that marker in the same atomic batch.
for dirty in dirty_to_clear {
staged.delete_stage2_dirty(dirty);
}
if !staged.is_empty() {
if let Err(error) = handle.commit(staged).await {
warn_job!(
tile,
partition_id,
error = %error,
"reconcile Stage 2 settlement failed; retrying this page on the next tick",
);
return None;
}
}
clock.mark(PageStage::Commit);
// Count only durably-settled pages: membership produce, cascade produce, and commit have all
// succeeded here. Any earlier failure returns `None` above and retries the whole page, so
// incrementing here keeps a retry from double-counting.
counter!(RECONCILE_ROWS_SCANNED_TOTAL).increment(diffs.len() as u64);
if entered > 0 {
counter!(RECONCILE_ROWS_EMITTED_TOTAL, "status" => "entered").increment(entered);
}
if left > 0 {
counter!(RECONCILE_ROWS_EMITTED_TOTAL, "status" => "left").increment(left);
}
if fixed_entered > 0 {
counter!(RECONCILE_BITS_FIXED_TOTAL, "direction" => "entered").increment(fixed_entered);
}
if fixed_left > 0 {
counter!(RECONCILE_BITS_FIXED_TOTAL, "direction" => "left").increment(fixed_left);
}
Some(PageProgress {
rows_scanned: diffs.len() as u64,
bits_fixed: fixed_entered + fixed_left,
})
}
fn complete_offset(
tracker: &OffsetTracker,
partition_id: u16,
offset: DeferredOffset,
operation: &'static str,
) {
match tracker.complete_deferred(offset) {
Some(MarkOutcome::WithinDispatch) => {}
Some(MarkOutcome::CappedAheadOfDispatch) => {
counter!(COHORT_STREAM_OFFSET_AHEAD_OF_DISPATCH).increment(1);
warn!(
partition_id,
operation, "reconcile completion exceeded the seed dispatch ceiling",
);
}
None => warn!(
partition_id,
operation, "reconcile completion belonged to an expired seed-tracker tenure",
),
}
}
#[cfg(test)]
#[allow(clippy::disallowed_methods)]
mod tests {
use std::sync::atomic::AtomicBool;
use async_trait::async_trait;
use chrono_tz::UTC;
use common_kafka::kafka_producer::KafkaProduceError;
use serde_json::{json, Value};
use tempfile::TempDir;
use uuid::Uuid;
use cohort_core::seed::{BehavioralShapeHash, CoverageStartMs, PersonShapeHash, RunBoundaryMs};
use cohort_core::{CohortEligibility, ExcludedReason, FilterCatalog, LeafStateKey};
use crate::filters::tree::{CohortTree, FilterNode};
use crate::filters::{BoolOp, TeamFiltersBuilder};
use crate::partitions::offset_tracker::OffsetTracker;
use crate::producer::{
CaptureCascadeSink, CaptureReconcileMarkerSink, CaptureSink, MembershipStatus,
};
use crate::stage1::person_record::{MatchedSet, PersonRecord};
use crate::stage1::state::{AppliedOffsets, Stage1State, StatefulRecord};
use crate::stage2::state::Stage2Ownership;
use crate::store::{
Behavioral, BehavioralKey, CohortStore, OffloadConfig, OffloadMode, PersonRecordKey,
PersonRecords, StoreConfig,
};
use super::*;
const TEAM: i32 = 7;
const COHORT: i32 = 1;
const PARTITION: u16 = 0;
const BEHAVIORAL_HASH: [u8; 16] = *b"0123456789abcdef";
const PERSON_HASH: [u8; 16] = *b"fedcba9876543210";
const SHAPE_HASH: &str = "shape-v1";
const TS: &str = "2026-05-26 12:34:56.789123";
/// Start of a partial slice, and the default tile boundary.
const SLICE_START: i64 = 1_791_547_200_000;
fn temp_store() -> (TempDir, CohortStore) {
let dir = TempDir::new().unwrap();
let store = CohortStore::open(&StoreConfig {
path: dir.path().join("db"),
..StoreConfig::default()
})
.unwrap();
(dir, store)
}
fn store_handle(store: &CohortStore) -> StoreHandle {
StoreHandle::new(
store.clone(),
OffloadConfig {
mode: OffloadMode::All,
event_read_permits: 16,
maintenance_permits: 6,
},
)
}
fn behavioral_leaf() -> Value {
json!({
"type": "behavioral", "value": "performed_event", "key": "$pageview",
"time_value": 7, "time_interval": "day",
"conditionHash": "0123456789abcdef",
"bytecode": ["_H", 1, 32, "$pageview", 32, "event", 1, 1, 11],
})
}
fn person_leaf() -> Value {
json!({
"type": "person", "key": "email", "value": "[email protected]", "operator": "exact",
"conditionHash": "fedcba9876543210",
"bytecode": ["_H", 1, 32, "[email protected]", 32, "email", 32, "properties", 32, "person", 1, 3, 11],
})
}
/// `behavioral_hash` is the catalog's behavioral shape hash for the cohort, `None` for one it
/// has not read yet.
fn cohort_filters(single_leaf: bool, behavioral_hash: Option<&str>) -> TeamFilters {
let values = if single_leaf {
vec![behavioral_leaf()]
} else {
vec![behavioral_leaf(), person_leaf()]
};
let cohort = json!({ "properties": { "type": "AND", "values": values } });
let mut builder = TeamFiltersBuilder::default();
builder
.add_cohort(CohortId(COHORT), TeamId(TEAM), &cohort)
.unwrap();
if let Some(hash) = behavioral_hash {
builder.set_behavioral_shape_hash(
CohortId(COHORT),
BehavioralShapeHash::parse(hash).unwrap(),
);
}
if !single_leaf {
// A mixed cohort carries both guards, so either kind of run can reconcile it.
builder.set_person_shape_hash(
CohortId(COHORT),
PersonShapeHash::parse(SHAPE_HASH).unwrap(),
);
}
builder.freeze(UTC)
}
fn catalog(single_leaf: bool) -> (Arc<CatalogHandle>, LeafStateKey) {
catalog_over(cohort_filters(single_leaf, Some(SHAPE_HASH)))
}
fn catalog_over(filters: TeamFilters) -> (Arc<CatalogHandle>, LeafStateKey) {
let lsk = filters.by_condition_to_lsk[&BEHAVIORAL_HASH][0];
(handle_for(filters), lsk)
}
/// A person-only cohort: no behavioral leaf, so only the person guard fences its reconcile.
fn person_catalog() -> (Arc<CatalogHandle>, LeafStateKey) {
let cohort = json!({ "properties": { "type": "AND", "values": [person_leaf()] } });
let mut builder = TeamFiltersBuilder::default();
builder
.add_cohort(CohortId(COHORT), TeamId(TEAM), &cohort)
.unwrap();
builder.set_person_shape_hash(
CohortId(COHORT),
PersonShapeHash::parse(SHAPE_HASH).unwrap(),
);
(
handle_for(builder.freeze(UTC)),
LeafStateKey::for_person_property(&PERSON_HASH),
)
}
fn unloaded() -> Arc<CatalogHandle> {
Arc::new(CatalogHandle::new())
}
fn team_catalog(filters: TeamFilters) -> FilterCatalog {
FilterCatalog::from_teams([(TeamId(TEAM), filters)])
}
fn handle_for(filters: TeamFilters) -> Arc<CatalogHandle> {
Arc::new(CatalogHandle::from_catalog(team_catalog(filters)))
}
struct DrainShell {
_dir: TempDir,
store: CohortStore,
handle: StoreHandle,
catalog: Arc<CatalogHandle>,
sink: Arc<dyn MembershipSink>,
markers: CaptureReconcileMarkerSink,
deps: MergeWorkerDeps,
queue: ReconcileQueue,
lsk: LeafStateKey,
}
impl DrainShell {
fn new(
single_leaf: bool,
sink: Arc<dyn MembershipSink>,
cascade_sink: CaptureCascadeSink,
scan_page: usize,
) -> Self {
Self::over(catalog(single_leaf), sink, cascade_sink, scan_page)
}
fn person_only(
sink: Arc<dyn MembershipSink>,
cascade_sink: CaptureCascadeSink,
scan_page: usize,
) -> Self {
Self::over(person_catalog(), sink, cascade_sink, scan_page)
}
fn over(
(catalog, lsk): (Arc<CatalogHandle>, LeafStateKey),
sink: Arc<dyn MembershipSink>,
cascade_sink: CaptureCascadeSink,
scan_page: usize,
) -> Self {
let (_dir, store) = temp_store();
let handle = store_handle(&store);
let mut deps = Arc::try_unwrap(MergeWorkerDeps::capture())
.unwrap_or_else(|_| panic!("test owns the only dependency Arc"));
deps.reconcile.enabled = true;
deps.reconcile.scan_page = scan_page;
deps.cascade.enabled = true;
deps.cascade_sink = Arc::new(cascade_sink);
let markers = CaptureReconcileMarkerSink::new();
deps.reconcile.marker_sink = Arc::new(markers.clone());
let queue = ReconcileQueue::new(
PARTITION,
deps.reconcile.backlog.clone(),
handle.clone(),
catalog.clone(),
SliceCoverage::Complete,
);
Self {
_dir,
store,
handle,
catalog,
sink,
markers,
deps,
queue,
lsk,
}
}
/// Replaces the queue, so call it before enqueueing.
fn cover(&mut self, coverage: SliceCoverage) {
assert_eq!(self.queue.len(), 0, "covering replaces the queue");
self.queue = ReconcileQueue::new(
PARTITION,
self.deps.reconcile.backlog.clone(),
self.handle.clone(),
self.catalog.clone(),
coverage,
);
}
/// Re-arm the marker sink to fail its first produce, isolating the marker phase's retry from
/// the membership leg's.
fn fail_first_marker(&mut self) {
self.markers = CaptureReconcileMarkerSink::failing_first(1);
self.deps.reconcile.marker_sink = Arc::new(self.markers.clone());
}
fn write_current(&self, person: Uuid, behavioral: bool, person_match: bool, prior: bool) {
if behavioral {
let key = BehavioralKey::new(PARTITION, TEAM as u64, person, self.lsk);
let state = Stage1State::BehavioralSingle {
has_match: true,
last_event_at_ms: 1_700_000_000_000,
earliest_eviction_at_ms: i64::MAX,
};
let record = StatefulRecord::new(state, AppliedOffsets::default());
self.store
.write_batch(|batch| batch.put::<Behavioral>(&key, &record.encode()))
.unwrap();
}
if person_match {
let key = PersonRecordKey::new(PARTITION, TEAM as u64, person);
let mut record = PersonRecord::absent();
record.matched = MatchedSet::from_iter([PERSON_HASH]);
self.store
.write_batch(|batch| batch.put::<PersonRecords>(&key, &record.encode()))
.unwrap();
}
let key = Stage2Key {
partition_id: PARTITION,
team_id: TEAM as u64,
cohort_id: COHORT as u64,
person_id: person,
};
let state = Stage2State {
in_cohort: prior,
last_evaluated_at_ms: 1,
};
self.store
.write_batch(|batch| batch.put_stage2(&key, &state.encode()))
.unwrap();
}
fn stage2_bit(&self, person: Uuid) -> bool {
let key = Stage2Key {
partition_id: PARTITION,
team_id: TEAM as u64,
cohort_id: COHORT as u64,
person_id: person,
};
Stage2State::decode(&self.store.get_stage2(&key).unwrap().unwrap())
.unwrap()
.in_cohort
}
fn enqueue(&mut self, tile: ReconcileTile, offset: i64) {
self.deps
.seed_tracker
.mark_dispatched(PARTITION as i32, offset + 1);
if self.committable().is_none() {
assert_eq!(
self.deps
.seed_tracker
.mark_processed(PARTITION as i32, offset),
MarkOutcome::WithinDispatch,
);
}
let deferred = self.deps.seed_tracker.defer(PARTITION as i32, offset);
self.queue.enqueue(tile, deferred);
}
/// Admit a newer run over a queued job exactly as `admit_reconcile` does: supersede, hand the
/// superseded floor to `replace_deferred`, then enqueue the replacement at the new offset.
fn supersede(&mut self, tile: ReconcileTile, offset: i64) {
self.deps
.seed_tracker
.mark_dispatched(PARTITION as i32, offset + 1);
let SupersedeOutcome::Replaced(superseded) = self.queue.supersede_if_newer(
tile.team_id(),
tile.cohort_id(),
tile.scope().kind(),
offset,
) else {
panic!("a newer run must supersede the queued job");
};
let (replacement, _) = self
.deps
.seed_tracker
.replace_deferred(superseded, offset)
.expect("the superseded reconcile retains its deferred offset");
self.queue.enqueue(tile, replacement);
}
async fn tick(&mut self) {
handle_reconcile_drain(
PARTITION,
&self.handle,
&self.sink,
&self.deps,
&mut self.queue,
TS,
)
.await;
}
fn committable(&self) -> Option<i64> {
self.deps
.seed_tracker
.committable_offsets()
.get(&(PARTITION as i32))
.copied()
}
}
fn tile(team_id: i32, cohort_id: i32, run_id: u128) -> ReconcileTile {
scoped_tile(team_id, cohort_id, run_id, ScopeKind::Behavioral)
}
fn scoped_tile(team_id: i32, cohort_id: i32, run_id: u128, guard: ScopeKind) -> ReconcileTile {
bounded_tile(team_id, cohort_id, run_id, guard, SLICE_START)
}
fn bounded_tile(
team_id: i32,
cohort_id: i32,
run_id: u128,
guard: ScopeKind,
boundary: i64,
) -> ReconcileTile {
ReconcileTile::new(
TeamId(team_id),
CohortId(cohort_id),
ReconcileScope::parse(guard, SHAPE_HASH).unwrap(),
RunId(Uuid::from_u128(run_id)),
RunBoundaryMs(boundary),
)
}
#[test]
fn queue_tracks_backlog_and_drop_releases_only_the_backlog_count() {
let (_dir, store) = temp_store();
let tracker = OffsetTracker::new();
let backlog = Arc::new(ReconcileBacklog::default());
tracker.mark_dispatched(3, 11);
{
let mut queue = ReconcileQueue::new(
3,
backlog.clone(),
store_handle(&store),
unloaded(),
SliceCoverage::Complete,
);
queue.enqueue(tile(1, 7, 1), tracker.defer(3, 10));
assert_eq!(queue.len(), 1);
assert_eq!(backlog.len(), 1);
}
assert!(backlog.is_empty());
assert_eq!(
tracker.mark_processed(3, 11),
crate::partitions::offset_tracker::MarkOutcome::WithinDispatch,
);
assert_eq!(
tracker.committable_offsets().get(&3),
Some(&10),
"dropping a queued job does not complete its deferred offset",
);
}
#[test]
fn only_the_active_queue_head_tracks_dirty_people() {
let (_dir, store) = temp_store();
let tracker = OffsetTracker::new();
let backlog = Arc::new(ReconcileBacklog::default());
tracker.mark_dispatched(PARTITION as i32, 12);
let mut queue = ReconcileQueue::new(
PARTITION,
backlog,
store_handle(&store),
unloaded(),
SliceCoverage::Complete,
);
queue.enqueue(tile(TEAM, COHORT, 1), tracker.defer(PARTITION as i32, 10));
queue.enqueue(
tile(TEAM, COHORT + 1, 2),
tracker.defer(PARTITION as i32, 11),
);
let head_prefix = Stage2CohortPrefix {
partition_id: PARTITION,
team_id: TEAM as u64,
cohort_id: COHORT as u64,
};
let waiting_prefix = Stage2CohortPrefix {
cohort_id: (COHORT + 1) as u64,
..head_prefix
};
// Enqueue alone captures nothing: a future full scan covers these writes.
for (prefix, person) in [(head_prefix, 1), (waiting_prefix, 2)] {
let key = Stage2Key {
partition_id: prefix.partition_id,
team_id: prefix.team_id,
cohort_id: prefix.cohort_id,
person_id: Uuid::from_u128(person),
};
store
.write_batch(|batch| batch.put_stage2(&key, b"before"))
.unwrap();
assert!(store
.scan_stage2_dirty(prefix, None, 10)
.unwrap()
.is_empty());
}
queue.activate_front_tracking(head_prefix);
for (prefix, person) in [(head_prefix, 3), (waiting_prefix, 4)] {
let key = Stage2Key {
partition_id: prefix.partition_id,
team_id: prefix.team_id,
cohort_id: prefix.cohort_id,
person_id: Uuid::from_u128(person),
};
store
.write_batch(|batch| batch.put_stage2(&key, b"during"))
.unwrap();
}
assert_eq!(
store
.scan_stage2_dirty(head_prefix, None, 10)
.unwrap()
.len(),
1
);
assert!(store
.scan_stage2_dirty(waiting_prefix, None, 10)
.unwrap()
.is_empty());
}
#[test]
fn supersede_matches_team_and_cohort_and_preserves_other_jobs() {
let (_dir, store) = temp_store();
let tracker = OffsetTracker::new();
let backlog = Arc::new(ReconcileBacklog::default());
tracker.mark_dispatched(3, 14);
let mut queue = ReconcileQueue::new(
3,
backlog.clone(),
store_handle(&store),
unloaded(),
SliceCoverage::Complete,
);
queue.enqueue(tile(1, 7, 1), tracker.defer(3, 10));
queue.enqueue(tile(1, 8, 2), tracker.defer(3, 11));
queue.enqueue(tile(2, 7, 3), tracker.defer(3, 12));
let SupersedeOutcome::Replaced(superseded) =
queue.supersede_if_newer(TeamId(1), CohortId(7), ScopeKind::Behavioral, 13)
else {
panic!("the newer run should supersede team 1 cohort 7");
};
assert_eq!(queue.len(), 2);
assert_eq!(queue.front_run_id(), Some(RunId(Uuid::from_u128(2))));
assert_eq!(backlog.len(), 2);
assert_eq!(
tracker.complete_deferred(superseded),
Some(crate::partitions::offset_tracker::MarkOutcome::WithinDispatch),
);
assert!(matches!(
queue.supersede_if_newer(TeamId(2), CohortId(7), ScopeKind::Behavioral, 12),
SupersedeOutcome::RetainedNewerOrEqual,
));
}
#[test]
fn a_mixed_cohorts_two_guards_queue_side_by_side_and_supersede_only_their_own() {
let (_dir, store) = temp_store();
let tracker = OffsetTracker::new();
let backlog = Arc::new(ReconcileBacklog::default());
tracker.mark_dispatched(3, 20);
let mut queue = ReconcileQueue::new(
3,
backlog.clone(),
store_handle(&store),
unloaded(),
SliceCoverage::Complete,
);
queue.enqueue(
scoped_tile(TEAM, COHORT, 1, ScopeKind::Behavioral),
tracker.defer(3, 10),
);
queue.enqueue(
scoped_tile(TEAM, COHORT, 2, ScopeKind::PersonProperty),
tracker.defer(3, 11),
);
// A newer person run evicts only the queued person job; the behavioral snapshot is a
// different fence and still owes its own marker.
let SupersedeOutcome::Replaced(superseded) = queue.supersede_if_newer(
TeamId(TEAM),
CohortId(COHORT),
ScopeKind::PersonProperty,
12,
) else {
panic!("the newer person run should supersede the queued person job");
};
assert_eq!(queue.len(), 1);
assert_eq!(queue.front_run_id(), Some(RunId(Uuid::from_u128(1))));
tracker.complete_deferred(superseded);
assert_eq!(
tracker.committable_offsets().get(&3),
Some(&10),
"the surviving behavioral job still pins the partition's floor at its own offset",
);
assert!(
matches!(
queue.supersede_if_newer(
TeamId(TEAM),
CohortId(COHORT),
ScopeKind::PersonProperty,
13,
),
SupersedeOutcome::NoQueuedJob,
),
"the surviving behavioral job is invisible to a person-kind supersession",
);
}
#[test]
fn older_or_duplicate_replay_never_evicts_the_queued_run() {
let (_dir, store) = temp_store();
let tracker = OffsetTracker::new();
let backlog = Arc::new(ReconcileBacklog::default());
tracker.mark_dispatched(3, 20);
let mut queue = ReconcileQueue::new(
3,
backlog.clone(),
store_handle(&store),
unloaded(),
SliceCoverage::Complete,
);
queue.enqueue(tile(1, 7, 2), tracker.defer(3, 15));
for replayed_offset in [10, 15] {
assert!(matches!(
queue.supersede_if_newer(
TeamId(1),
CohortId(7),
ScopeKind::Behavioral,
replayed_offset,
),
SupersedeOutcome::RetainedNewerOrEqual,
));
assert_eq!(queue.front_run_id(), Some(RunId(Uuid::from_u128(2))));
assert_eq!(backlog.len(), 1);
assert_eq!(tracker.committable_offsets().get(&3), None);
}
}
#[tokio::test]
async fn superseding_a_mid_drain_run_reemits_under_the_new_run_and_releases_its_offset() {
let sink = CaptureSink::new();
// A one-row page over three people keeps run A mid-scan (active dirty lease, cursor set) when
// the newer run arrives.
let mut shell = DrainShell::new(true, Arc::new(sink.clone()), CaptureCascadeSink::new(), 1);
let people = [Uuid::from_u128(1), Uuid::from_u128(2), Uuid::from_u128(3)];
for person in people {
shell.write_current(person, true, false, true);
}
shell.enqueue(tile(TEAM, COHORT, 100), 5);
shell.tick().await;
assert_eq!(
sink.changes().len(),
1,
"run A emitted one page before being superseded",
);
assert!(matches!(
shell.queue.front().map(|job| &job.phase),
Some(ScanPhase::Scanning { cursor: Some(_) }),
));
shell.supersede(tile(TEAM, COHORT, 200), 7);
assert_eq!(shell.queue.len(), 1);
assert_eq!(
shell.queue.front_run_id(),
Some(RunId(Uuid::from_u128(200)))
);
for _ in 0..16 {
if shell.queue.front().is_none() {
break;
}
shell.tick().await;
}
assert!(shell.queue.front().is_none());
assert!(shell.deps.reconcile.backlog.is_empty());
let markers = shell.markers.markers();
assert_eq!(
markers.len(),
1,
"the superseded run never emits its own marker"
);
assert_eq!(markers[0].run_id(), RunId(Uuid::from_u128(200)));
let reemitted: std::collections::HashSet<String> = sink
.changes()
.iter()
.filter(|change| change.run_id == Some(RunId(Uuid::from_u128(200))))
.map(|change| change.person_id.clone())
.collect();
assert_eq!(
reemitted,
people
.iter()
.map(Uuid::to_string)
.collect::<std::collections::HashSet<_>>(),
"run B re-emits the full current membership",
);
assert_eq!(
shell.committable(),
Some(8),
"the deferred floor releases only after run B completes",
);
}
fn guard_filters(
eligibility: Option<CohortEligibility>,
guards: &[(ScopeKind, &str)],
) -> TeamFilters {
let mut filters = TeamFilters::default();
filters.cohorts.insert(
CohortId(COHORT),
CohortTree {
cohort_id: CohortId(COHORT),
team_id: TeamId(TEAM),
root: FilterNode::Group {
op: BoolOp::And,
children: Vec::new(),
},
},
);
if let Some(eligibility) = eligibility {
filters.eligibility.insert(CohortId(COHORT), eligibility);
}
for &(kind, hash) in guards {
match kind {
ScopeKind::Behavioral => {
filters
.behavioral_shape_hashes
.insert(CohortId(COHORT), BehavioralShapeHash::parse(hash).unwrap());
}
ScopeKind::PersonProperty => {
filters
.person_shape_hashes
.insert(CohortId(COHORT), PersonShapeHash::parse(hash).unwrap());
}
}
}
filters
}
fn verdict(
filters: TeamFilters,
tile: &ReconcileTile,
) -> Result<CohortEligibility, CatalogMiss> {
catalog_verdict(Some(&Arc::new(filters)), tile).map(|(_, eligibility)| eligibility)
}
#[test]
fn catalog_verdict_names_every_discard_reason_and_proves_what_proceeds() {
let reconcile = tile(TEAM, COHORT, 1);
assert_eq!(
catalog_verdict(None, &reconcile).map(|(_, eligibility)| eligibility),
Err(CatalogMiss::TeamAbsent),
);
assert_eq!(
verdict(TeamFilters::default(), &reconcile),
Err(CatalogMiss::CohortAbsent),
);
let behavioral_guard = &[(ScopeKind::Behavioral, SHAPE_HASH)];
assert_eq!(
verdict(guard_filters(None, behavioral_guard), &reconcile),
Err(CatalogMiss::CohortAbsent),
);
let excluded = guard_filters(
Some(CohortEligibility::Excluded(ExcludedReason::HasDroppedLeaf)),
behavioral_guard,
);
assert_eq!(verdict(excluded, &reconcile), Err(CatalogMiss::NotEmitting),);
for eligibility in [
CohortEligibility::SingleLeaf(LeafStateKey(BEHAVIORAL_HASH)),
CohortEligibility::Stage2Composable,
CohortEligibility::Stage2ComposableRef,
] {
assert_eq!(
verdict(
guard_filters(Some(eligibility), behavioral_guard),
&reconcile
),
Ok(eligibility),
"{eligibility:?} registers membership",
);
}
}
#[test]
fn each_guard_kind_consults_only_its_own_map() {
let emitting = Some(CohortEligibility::Stage2Composable);
for (guard, other) in [
(ScopeKind::Behavioral, ScopeKind::PersonProperty),
(ScopeKind::PersonProperty, ScopeKind::Behavioral),
] {
let reconcile = scoped_tile(TEAM, COHORT, 1, guard);
assert_eq!(
verdict(guard_filters(emitting, &[(guard, SHAPE_HASH)]), &reconcile),
Ok(CohortEligibility::Stage2Composable),
"{guard} matches its own map",
);
assert_eq!(
verdict(guard_filters(emitting, &[(guard, "shape-v2")]), &reconcile),
Err(CatalogMiss::HashMismatch),
"{guard} diverged",
);
assert_eq!(
verdict(guard_filters(emitting, &[]), &reconcile),
Err(CatalogMiss::HashUnknown),
"{guard} has no persisted hash",
);
// The catalog carries only the *other* kind's hash, at the very value the tile pins.
assert_eq!(
verdict(guard_filters(emitting, &[(other, SHAPE_HASH)]), &reconcile),
Err(CatalogMiss::HashUnknown),
"{guard} must not be satisfied by the {other} map",
);
}
}
/// The guard's decision without the filters it carries, so cases compare by value.
#[derive(Debug, PartialEq, Eq)]
enum Fate {
Proceed(CohortEligibility),
Retry(ReconcileRetryReason),
Withhold(WithheldReason),
}
impl From<ReconcileGuard> for Fate {
fn from(guard: ReconcileGuard) -> Self {
match guard {
ReconcileGuard::Proceed { eligibility, .. } => Self::Proceed(eligibility),
ReconcileGuard::Retry(reason) => Self::Retry(reason),
ReconcileGuard::Withhold(reason) => Self::Withhold(reason),
}
}
}
#[derive(Debug, Clone, Copy)]
enum Freshness {
NotLoaded,
RefreshedBeforeAdmission,
RefreshedAfterAdmission,
}
/// What the refresh the case publishes holds for the job's cohort.
#[derive(Debug, Clone, Copy)]
enum Snapshot {
Emitting,
MissingTeam,
}
impl Snapshot {
fn catalog(self) -> FilterCatalog {
match self {
Self::Emitting => team_catalog(guard_filters(
Some(CohortEligibility::Stage2Composable),
&[(ScopeKind::Behavioral, SHAPE_HASH)],
)),
Self::MissingTeam => FilterCatalog::from_teams([]),
}
}
}
#[derive(Debug, Clone, Copy)]
enum Slice {
Complete,
BeganAtBoundary,
BeganAfterBoundary,
}
impl Slice {
fn coverage(self) -> SliceCoverage {
match self {
Self::Complete => SliceCoverage::Complete,
Self::BeganAtBoundary | Self::BeganAfterBoundary => {
SliceCoverage::Since(CoverageStartMs(SLICE_START))
}
}
}
fn run_boundary(self) -> i64 {
match self {
Self::BeganAtBoundary => SLICE_START,
Self::Complete | Self::BeganAfterBoundary => SLICE_START - 1,
}
}
}
#[test]
fn a_catalog_miss_withholds_only_after_a_later_refresh_and_coverage_only_after_the_catalog() {
let team_absent = CatalogMiss::TeamAbsent;
let partial = WithheldReason::PartialCoverage {
covered_since: CoverageStartMs(SLICE_START),
};
let cases = [
// Nothing is published before the first load, so the snapshot kind does not matter.
(
Freshness::NotLoaded,
Snapshot::MissingTeam,
Slice::BeganAfterBoundary,
Fate::Retry(ReconcileRetryReason::CatalogNotLoaded),
),
(
Freshness::RefreshedBeforeAdmission,
Snapshot::Emitting,
Slice::Complete,
Fate::Proceed(CohortEligibility::Stage2Composable),
),
// The catalog wait comes before coverage.
(
Freshness::RefreshedBeforeAdmission,
Snapshot::MissingTeam,
Slice::BeganAfterBoundary,
Fate::Retry(ReconcileRetryReason::AwaitingRefresh(team_absent)),
),
(
Freshness::RefreshedAfterAdmission,
Snapshot::Emitting,
Slice::Complete,
Fate::Proceed(CohortEligibility::Stage2Composable),
),
(
Freshness::RefreshedAfterAdmission,
Snapshot::Emitting,
Slice::BeganAtBoundary,
Fate::Proceed(CohortEligibility::Stage2Composable),
),
(
Freshness::RefreshedAfterAdmission,
Snapshot::Emitting,
Slice::BeganAfterBoundary,
Fate::Withhold(partial),
),
// The catalog's reason wins over coverage.
(
Freshness::RefreshedAfterAdmission,
Snapshot::MissingTeam,
Slice::BeganAfterBoundary,
Fate::Withhold(WithheldReason::Catalog(team_absent)),
),
];
for (freshness, snapshot, slice, expected) in cases {
let reconcile =
bounded_tile(TEAM, COHORT, 1, ScopeKind::Behavioral, slice.run_boundary());
let catalog = CatalogHandle::new();
let admitted_after = match freshness {
Freshness::NotLoaded => catalog.last_begun(),
Freshness::RefreshedBeforeAdmission => {
catalog.publish(snapshot.catalog());
catalog.last_begun()
}
Freshness::RefreshedAfterAdmission => {
let admitted_after = catalog.last_begun();
catalog.publish(snapshot.catalog());
admitted_after
}
};
assert_eq!(
Fate::from(evaluate_guard(
&catalog.view(),
admitted_after,
&reconcile,
slice.coverage(),
)),
expected,
"{freshness:?} with {snapshot:?} on a {slice:?} slice",
);
}
}
fn counted(metrics: &metrics_exporter_prometheus::PrometheusHandle, series: &str) -> u64 {
metrics
.render()
.lines()
.filter(|line| !line.starts_with('#'))
.find(|line| series.split(',').all(|part| line.contains(part)))
.and_then(|line| line.rsplit(' ').next())
.map_or(0, |count| count.parse().unwrap())
}
/// A run's reconcile can reach the processor before the catalog refresh that reads the run's
/// pinned hash. The job waits with its seed offset held, and the next refresh decides it.
#[tokio::test]
async fn a_reconcile_that_beats_its_catalog_refresh_waits_for_the_next_one() {
for (refreshed_hash, completes) in [(SHAPE_HASH, true), ("shape-v2", false)] {
let recorder = metrics_exporter_prometheus::PrometheusBuilder::new().build_recorder();
let metrics = recorder.handle();
let _local = metrics::set_default_local_recorder(&recorder);
let mut shell = DrainShell::over(
catalog_over(cohort_filters(true, None)),
Arc::new(CaptureSink::new()),
CaptureCascadeSink::new(),
8,
);
// A refresh in flight at admission may have read Postgres before the run's write, so
// its snapshot cannot discard the job either.
let catalog = shell.catalog.clone();
let in_flight = catalog.begin_refresh();
shell.enqueue(tile(TEAM, COHORT, 1), 5);
in_flight.publish(team_catalog(cohort_filters(true, None)));
shell.tick().await;
shell.tick().await;
assert!(shell.markers.markers().is_empty(), "{refreshed_hash}");
assert_eq!(shell.queue.len(), 1, "the job waits for the next refresh");
assert_eq!(
shell.committable(),
Some(5),
"its seed offset stays deferred"
);
assert_eq!(
counted(
&metrics,
"cohort_reconcile_jobs_deferred_total,reason=\"hash_unknown\""
),
1,
"a wait over several ticks counts once",
);
shell
.catalog
.publish(team_catalog(cohort_filters(true, Some(refreshed_hash))));
shell.tick().await;
let markers = shell.markers.markers();
assert_eq!(markers.len(), 1, "{refreshed_hash}");
assert_eq!(markers[0].run_id(), RunId(Uuid::from_u128(1)));
if completes {
assert!(matches!(markers[0], ReconcileMarker::Complete(_)));
} else {
// A superseded run fails closed with a withheld marker.
assert!(
matches!(
&markers[0],
ReconcileMarker::Withheld(withheld)
if withheld.reason()
== WithheldReason::Catalog(CatalogMiss::HashMismatch)
),
"{markers:?}",
);
assert_eq!(
counted(
&metrics,
"cohort_reconcile_jobs_discarded_total,reason=\"hash_mismatch\""
),
1,
);
}
assert!(shell.queue.front().is_none(), "{refreshed_hash}");
assert_eq!(shell.committable(), Some(6), "{refreshed_hash}");
}
}
#[tokio::test]
async fn drain_emits_the_full_snapshot_fixes_only_flips_and_paginates() {
let sink = CaptureSink::new();
let cascades = CaptureCascadeSink::new();
let mut shell = DrainShell::new(false, Arc::new(sink.clone()), cascades.clone(), 2);
let alice = Uuid::from_u128(1);
let bob = Uuid::from_u128(2);
let charlie = Uuid::from_u128(3);
shell.write_current(alice, true, true, false);
shell.write_current(bob, false, false, true);
shell.write_current(charlie, true, true, true);
shell.enqueue(tile(TEAM, COHORT, 11), 5);
shell.tick().await;
assert_eq!(sink.changes().len(), 2, "one bounded page was emitted");
assert!(shell.markers.markers().is_empty());
assert_eq!(shell.committable(), Some(5));
assert!(matches!(
shell.queue.front().map(|job| &job.phase),
Some(ScanPhase::Scanning {
cursor: Some(_),
..
}),
));
shell.tick().await;
let changes = sink.changes();
assert_eq!(changes.len(), 3, "unchanged rows are emitted too");
assert_eq!(
changes
.iter()
.map(|change| change.status)
.collect::<Vec<_>>(),
vec![
MembershipStatus::Entered,
MembershipStatus::Left,
MembershipStatus::Entered,
],
);
assert!(changes.iter().all(|change| {
change.origin == Some(ChangeOrigin::Reconcile)
&& change.run_id == Some(RunId(Uuid::from_u128(11)))
}));
assert!(shell.stage2_bit(alice));
assert!(!shell.stage2_bit(bob));
assert!(shell.stage2_bit(charlie));
let cascade_messages = cascades.messages();
assert_eq!(cascade_messages.len(), 2, "only flipped bits cascade");
assert_eq!(cascade_messages[0].change.person_id, alice.to_string());
assert_eq!(cascade_messages[1].change.person_id, bob.to_string());
shell.tick().await;
assert_eq!(shell.markers.markers().len(), 1);
assert!(shell.queue.front().is_none());
assert!(shell.deps.reconcile.backlog.is_empty());
assert_eq!(shell.committable(), Some(6));
}
#[tokio::test]
async fn one_cohorts_two_guarded_jobs_each_drain_to_their_own_marker() {
let sink = CaptureSink::new();
let mut shell =
DrainShell::new(false, Arc::new(sink.clone()), CaptureCascadeSink::new(), 8);
shell.write_current(Uuid::from_u128(1), true, true, false);
shell.enqueue(scoped_tile(TEAM, COHORT, 31, ScopeKind::Behavioral), 5);
shell.enqueue(scoped_tile(TEAM, COHORT, 32, ScopeKind::PersonProperty), 6);
for _ in 0..8 {
if shell.queue.front().is_none() {
break;
}
shell.tick().await;
}
// Both jobs lease the same Stage 2 prefix in turn; neither may release the other's capture.
assert_eq!(
shell
.markers
.markers()
.iter()
.map(ReconcileMarker::run_id)
.collect::<Vec<_>>(),
vec![RunId(Uuid::from_u128(31)), RunId(Uuid::from_u128(32))],
);
assert!(shell.deps.reconcile.backlog.is_empty());
assert_eq!(shell.committable(), Some(7));
}
#[tokio::test]
async fn a_person_only_cohort_drains_under_its_person_guard_and_marks_complete() {
let sink = CaptureSink::new();
let mut shell =
DrainShell::person_only(Arc::new(sink.clone()), CaptureCascadeSink::new(), 8);
let matching = Uuid::from_u128(1);
let left = Uuid::from_u128(2);
shell.write_current(matching, false, true, false);
shell.write_current(left, false, false, true);
shell.enqueue(scoped_tile(TEAM, COHORT, 21, ScopeKind::PersonProperty), 5);
shell.tick().await;
shell.tick().await;
let statuses: Vec<_> = sink
.changes()
.iter()
.map(|change| (change.person_id.clone(), change.status))
.collect();
assert_eq!(
statuses,
vec![
(matching.to_string(), MembershipStatus::Entered),
(left.to_string(), MembershipStatus::Left),
],
"the snapshot is computed from person state, not a behavioral leaf",
);
assert_eq!(shell.markers.markers().len(), 1);
assert_eq!(
shell.markers.markers()[0].run_id(),
RunId(Uuid::from_u128(21))
);
assert!(shell.queue.front().is_none());
}
#[tokio::test]
async fn noop_reconcile_claims_a_transferred_fallback() {
let sink = CaptureSink::new();
let cascades = CaptureCascadeSink::new();
let mut shell = DrainShell::new(false, Arc::new(sink.clone()), cascades.clone(), 2);
let person = Uuid::from_u128(1);
let key = Stage2Key {
partition_id: PARTITION,
team_id: TEAM as u64,
cohort_id: COHORT as u64,
person_id: person,
};
let fallback = Stage2State {
in_cohort: false,
last_evaluated_at_ms: 1,
};
shell
.store
.write_batch(|batch| {
batch.put_stage2(&key, &fallback.encode_transferred_fallback());
})
.unwrap();
shell.enqueue(tile(TEAM, COHORT, 12), 5);
shell.tick().await;
let bytes = shell.store.get_stage2(&key).unwrap().unwrap();
let (state, ownership) = Stage2State::decode_with_ownership(&bytes).unwrap();
assert!(!state.in_cohort);
assert_eq!(ownership, Stage2Ownership::Local);
assert_eq!(
sink.changes().len(),
1,
"the full snapshot still emits the row"
);
assert_eq!(sink.changes()[0].status, MembershipStatus::Left);
assert!(
cascades.messages().is_empty(),
"ownership settlement is not a flip"
);
}
#[tokio::test]
async fn paged_scan_drains_a_row_inserted_behind_its_cursor_without_restarting() {
let sink = CaptureSink::new();
let mut shell = DrainShell::new(true, Arc::new(sink.clone()), CaptureCascadeSink::new(), 1);
let later_key = Uuid::from_u128(2);
let inserted_behind_cursor = Uuid::from_u128(1);
shell.write_current(later_key, true, false, true);
shell.enqueue(tile(TEAM, COHORT, 111), 5);
shell.tick().await;
assert_eq!(sink.changes().len(), 1);
assert_eq!(sink.changes()[0].person_id, later_key.to_string());
assert!(shell.markers.markers().is_empty());
shell.write_current(inserted_behind_cursor, true, false, true);
shell.tick().await;
assert_eq!(
sink.changes()[1].person_id,
inserted_behind_cursor.to_string(),
"the dirty-person pass catches a row inserted behind the main cursor",
);
assert_eq!(
sink.changes()
.iter()
.map(|change| change.person_id.clone())
.collect::<Vec<_>>(),
vec![later_key.to_string(), inserted_behind_cursor.to_string(),],
"the main cohort page is not rescanned because another person changed",
);
assert_eq!(shell.markers.markers().len(), 1);
assert_eq!(shell.committable(), Some(6));
}
#[tokio::test]
async fn repeatedly_mutated_hot_person_does_not_restart_main_scan() {
let sink = CaptureSink::new();
let mut shell = DrainShell::new(true, Arc::new(sink.clone()), CaptureCascadeSink::new(), 1);
let hot = Uuid::from_u128(10);
let second = Uuid::from_u128(20);
let third = Uuid::from_u128(30);
for person in [hot, second, third] {
shell.write_current(person, true, false, true);
}
shell.enqueue(tile(TEAM, COHORT, 112), 5);
for expected in [hot, second, third] {
shell.tick().await;
assert_eq!(
sink.changes().last().unwrap().person_id,
expected.to_string(),
"the main cursor advances despite a mutation behind it",
);
shell.write_current(hot, true, false, true);
}
shell.tick().await;
assert_eq!(sink.changes().last().unwrap().person_id, hot.to_string());
assert_eq!(
shell.markers.markers().len(),
1,
"the job completes in the same tick that settles the full hot-person page",
);
assert_eq!(shell.committable(), Some(6));
}
#[tokio::test]
async fn deleted_dirty_row_is_cleared_without_being_resurrected_or_emitted_again() {
let sink = CaptureSink::new();
let mut shell = DrainShell::new(true, Arc::new(sink.clone()), CaptureCascadeSink::new(), 1);
let alice = Uuid::from_u128(1);
let bob = Uuid::from_u128(2);
shell.write_current(alice, true, false, true);
shell.write_current(bob, true, false, true);
shell.enqueue(tile(TEAM, COHORT, 113), 5);
shell.tick().await;
let deleted_key = Stage2Key {
partition_id: PARTITION,
team_id: TEAM as u64,
cohort_id: COHORT as u64,
person_id: alice,
};
shell
.store
.write_batch(|batch| batch.delete_stage2(&deleted_key))
.unwrap();
shell.tick().await;
shell.tick().await;
shell.tick().await;
assert_eq!(
sink.changes()
.iter()
.map(|change| change.person_id.clone())
.collect::<Vec<_>>(),
vec![alice.to_string(), bob.to_string()],
"the tombstoned scan row is not evaluated or emitted from its dirty marker",
);
assert!(shell.store.get_stage2(&deleted_key).unwrap().is_none());
assert_eq!(shell.markers.markers().len(), 1);
}
#[tokio::test]
async fn single_leaf_fix_never_cascades() {
let sink = CaptureSink::new();
let cascades = CaptureCascadeSink::new();
let mut shell = DrainShell::new(true, Arc::new(sink.clone()), cascades.clone(), 8);
let alice = Uuid::from_u128(1);
shell.write_current(alice, true, false, false);
shell.enqueue(tile(TEAM, COHORT, 12), 5);
shell.tick().await;
assert_eq!(sink.changes().len(), 1);
assert_eq!(sink.changes()[0].status, MembershipStatus::Entered);
assert!(shell.stage2_bit(alice));
assert!(cascades.messages().is_empty());
shell.tick().await;
assert_eq!(shell.markers.markers().len(), 1);
assert_eq!(shell.committable(), Some(6));
}
#[tokio::test]
async fn a_partial_slice_withholds_without_rows_and_the_next_run_still_completes() {
let sink = CaptureSink::new();
let mut shell =
DrainShell::new(false, Arc::new(sink.clone()), CaptureCascadeSink::new(), 8);
shell.cover(SliceCoverage::Since(CoverageStartMs(SLICE_START)));
shell.fail_first_marker();
shell.write_current(Uuid::from_u128(1), true, true, true);
let before_slice =
|run_id| bounded_tile(TEAM, COHORT, run_id, ScopeKind::Behavioral, SLICE_START - 1);
shell.enqueue(before_slice(1), 5);
shell.enqueue(before_slice(3), 6);
shell.enqueue(tile(TEAM, COHORT, 2), 7);
// Not part of the head's withheld burst.
shell.enqueue(before_slice(4), 8);
// The produce fails, so nothing is released or scanned.
shell.tick().await;
assert!(shell.markers.markers().is_empty());
assert!(sink.changes().is_empty());
assert_eq!(shell.queue.front_run_id(), Some(RunId(Uuid::from_u128(1))));
assert_eq!(shell.committable(), Some(5));
// One produce for both withheld jobs.
shell.tick().await;
assert_eq!(shell.markers.produce_calls(), 2, "one produce per tick");
assert_eq!(shell.queue.front_run_id(), Some(RunId(Uuid::from_u128(2))));
assert_eq!(shell.committable(), Some(7));
for _ in 0..8 {
if shell.queue.front().is_none() {
break;
}
shell.tick().await;
}
let partial = WithheldReason::PartialCoverage {
covered_since: CoverageStartMs(SLICE_START),
};
let markers = shell.markers.markers();
assert!(
matches!(
markers.as_slice(),
[
ReconcileMarker::Withheld(first),
ReconcileMarker::Withheld(second),
ReconcileMarker::Complete(complete),
ReconcileMarker::Withheld(last),
] if first.run_id() == RunId(Uuid::from_u128(1))
&& second.run_id() == RunId(Uuid::from_u128(3))
&& complete.run_id() == RunId(Uuid::from_u128(2))
&& last.run_id() == RunId(Uuid::from_u128(4))
&& [first, second, last].iter().all(|marker| marker.reason() == partial)
),
"{markers:?}",
);
assert!(
sink.changes()
.iter()
.all(|change| change.run_id == Some(RunId(Uuid::from_u128(2)))),
"a slice missing the run's history emits no rows for it",
);
assert!(!sink.changes().is_empty());
assert!(shell.deps.reconcile.backlog.is_empty());
assert_eq!(shell.committable(), Some(9));
}
#[tokio::test]
async fn a_withheld_burst_produces_at_most_one_scan_page_per_tick() {
let mut shell = DrainShell::new(
false,
Arc::new(CaptureSink::new()),
CaptureCascadeSink::new(),
2,
);
shell.cover(SliceCoverage::Since(CoverageStartMs(SLICE_START)));
let before_slice =
|run_id| bounded_tile(TEAM, COHORT, run_id, ScopeKind::Behavioral, SLICE_START - 1);
for run_id in 1..=3 {
shell.enqueue(before_slice(run_id), 4 + run_id as i64);
}
shell.tick().await;
assert_eq!(shell.markers.markers().len(), 2);
assert_eq!(shell.queue.front_run_id(), Some(RunId(Uuid::from_u128(3))));
}
/// Fails the first marker of its first produce and acks the rest.
#[derive(Default)]
struct FailHeadOnce(CaptureReconcileMarkerSink, AtomicBool);
#[async_trait]
impl ReconcileMarkerSink for FailHeadOnce {
async fn produce(
&self,
markers: Vec<ReconcileMarker>,
) -> Vec<Result<(), KafkaProduceError>> {
if self.1.swap(true, Ordering::SeqCst) {
return self.0.produce(markers).await;
}
let mut acks = vec![Err(KafkaProduceError::KafkaProduceCanceled)];
acks.extend(self.0.produce(markers[1..].to_vec()).await);
acks
}
}
#[tokio::test]
async fn a_later_ack_never_releases_an_unacked_withheld_head() {
let mut shell = DrainShell::new(
false,
Arc::new(CaptureSink::new()),
CaptureCascadeSink::new(),
8,
);
shell.cover(SliceCoverage::Since(CoverageStartMs(SLICE_START)));
shell.deps.reconcile.marker_sink = Arc::new(FailHeadOnce::default());
let before_slice =
|run_id| bounded_tile(TEAM, COHORT, run_id, ScopeKind::Behavioral, SLICE_START - 1);
shell.enqueue(before_slice(1), 5);
shell.enqueue(before_slice(2), 6);
shell.tick().await;
assert_eq!(shell.queue.front_run_id(), Some(RunId(Uuid::from_u128(1))));
assert_eq!(
shell.committable(),
Some(5),
"a later ack must not release an unacked head"
);
}
#[tokio::test]
async fn membership_failure_retries_the_same_page_without_advancing() {
let sink = CaptureSink::failing_first(1);
let mut shell = DrainShell::new(true, Arc::new(sink.clone()), CaptureCascadeSink::new(), 8);
let alice = Uuid::from_u128(1);
shell.write_current(alice, true, false, false);
shell.enqueue(tile(TEAM, COHORT, 13), 5);
shell.tick().await;
assert!(sink.changes().is_empty());
assert!(shell.markers.markers().is_empty());
assert!(!shell.stage2_bit(alice));
assert!(matches!(
shell.queue.front().map(|job| &job.phase),
Some(ScanPhase::Scanning { cursor: None }),
));
shell.tick().await;
assert_eq!(sink.changes().len(), 1);
assert!(shell.stage2_bit(alice));
shell.tick().await;
assert_eq!(shell.markers.markers().len(), 1);
assert_eq!(shell.committable(), Some(6));
}
/// A page wider than one read section still settles as one unit: the produce sees the whole
/// page, and a failure re-reads every row rather than resuming from the sections already read.
#[tokio::test]
async fn a_page_spanning_read_sections_retries_as_one_unit_after_a_membership_failure() {
let sink = CaptureSink::failing_first(1);
let rows = MAX_ROWS + 1;
let mut shell =
DrainShell::new(true, Arc::new(sink.clone()), CaptureCascadeSink::new(), 64);
let people: Vec<Uuid> = (1..=rows as u128).map(Uuid::from_u128).collect();
for &who in &people {
shell.write_current(who, true, false, false);
}
shell.enqueue(tile(TEAM, COHORT, 16), 5);
shell.tick().await;
assert_eq!(sink.produce_calls(), 1, "one produce for the whole page");
assert!(sink.changes().is_empty());
assert!(shell.markers.markers().is_empty());
assert!(
people.iter().all(|&who| !shell.stage2_bit(who)),
"no row of a failed page commits",
);
assert!(matches!(
shell.queue.front().map(|job| &job.phase),
Some(ScanPhase::Scanning { cursor: None }),
));
assert_eq!(shell.committable(), Some(5));
shell.tick().await;
assert_eq!(
sink.changes().len(),
rows,
"the whole page was emitted again"
);
assert!(people.iter().all(|&who| shell.stage2_bit(who)));
shell.tick().await;
assert_eq!(shell.markers.markers().len(), 1);
assert_eq!(shell.committable(), Some(6));
}
#[tokio::test]
async fn cascade_failure_retries_before_committing_or_advancing() {
let sink = CaptureSink::new();
let cascades = CaptureCascadeSink::failing_first(1);
let mut shell = DrainShell::new(false, Arc::new(sink.clone()), cascades.clone(), 8);
let alice = Uuid::from_u128(1);
shell.write_current(alice, true, true, false);
shell.enqueue(tile(TEAM, COHORT, 14), 5);
shell.tick().await;
assert_eq!(sink.changes().len(), 1, "the membership leg acked");
assert!(shell.markers.markers().is_empty());
assert!(!shell.stage2_bit(alice));
assert!(matches!(
shell.queue.front().map(|job| &job.phase),
Some(ScanPhase::Scanning { cursor: None }),
));
shell.tick().await;
assert_eq!(sink.changes().len(), 2, "the page was emitted again");
assert_eq!(cascades.messages().len(), 1);
assert!(shell.stage2_bit(alice));
shell.tick().await;
assert_eq!(shell.markers.markers().len(), 1);
assert_eq!(shell.committable(), Some(6));
}
#[tokio::test]
async fn marker_failure_retries_only_the_marker_phase() {
let sink = CaptureSink::new();
let mut shell = DrainShell::new(true, Arc::new(sink.clone()), CaptureCascadeSink::new(), 8);
shell.fail_first_marker();
let alice = Uuid::from_u128(1);
shell.write_current(alice, true, false, false);
shell.enqueue(tile(TEAM, COHORT, 15), 5);
shell.tick().await;
shell.tick().await;
assert_eq!(sink.changes().len(), 1);
assert!(shell.markers.markers().is_empty());
assert!(shell.stage2_bit(alice), "the page committed before marker");
assert!(matches!(
shell.queue.front().map(|job| &job.phase),
Some(ScanPhase::MarkerReady),
));
assert_eq!(shell.committable(), Some(5));
shell.tick().await;
assert_eq!(sink.changes().len(), 1, "the page was not rescanned");
assert_eq!(shell.markers.markers().len(), 1);
assert!(shell.queue.front().is_none());
assert_eq!(shell.committable(), Some(6));
}
#[tokio::test]
async fn marker_retry_drains_new_dirty_work_without_rescanning_clean_rows() {
let sink = CaptureSink::new();
let mut shell = DrainShell::new(true, Arc::new(sink.clone()), CaptureCascadeSink::new(), 8);
shell.fail_first_marker();
let alice = Uuid::from_u128(2);
let bob = Uuid::from_u128(1);
shell.write_current(alice, true, false, true);
shell.enqueue(tile(TEAM, COHORT, 16), 5);
shell.tick().await;
shell.tick().await;
assert_eq!(sink.changes().len(), 1);
assert!(shell.markers.markers().is_empty());
assert!(matches!(
shell.queue.front().map(|job| &job.phase),
Some(ScanPhase::MarkerReady),
));
shell.write_current(bob, true, false, true);
shell.tick().await;
assert_eq!(
sink.changes()
.iter()
.map(|change| change.person_id.clone())
.collect::<Vec<_>>(),
vec![alice.to_string(), bob.to_string()],
"a failed marker drains only the person that changed while the marker was pending",
);
assert_eq!(shell.markers.markers().len(), 1);
assert_eq!(shell.committable(), Some(6));
}
}