Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

54.2 KB
Raw Blame
//! Merge-protocol worker paths: the produce/track I/O around the drain/apply handlers in
//! [`crate::merge`].
//!
//! [`handle_merge`] drains a `PersonMergeEvent` on P_old's worker and owns the transfer produce;
//! [`handle_apply`] applies a `MergeStateTransfer` on P_new's worker. Both produce their own
//! membership output and mark their own [`OffsetTracker`].

use std::sync::Arc;
use std::time::{Duration, Instant};

use chrono_tz::UTC;
use metrics::{counter, gauge, histogram};
use tracing::{debug, warn};

use crate::filters::manager::CatalogHandle;
use crate::filters::reverse_index::TeamFilters;
use crate::filters::{FilterCatalog, TeamFiltersBuilder, TeamId};
use crate::merge::apply_handler::{handle_transfer, ApplyOutcome};
use crate::merge::drain_handler::{
    handle_merge_event_with_transfer_mode, DrainOutcome, MembershipRegisterTransferMode,
};
use crate::merge::transfer::{MergeStateTransfer, PendingTransfer, PersonMergeEvent};
use crate::observability::metrics::{
    COHORT_STREAM_OFFSET_AHEAD_OF_DISPATCH, MERGE_APPLY_DURATION_SECONDS,
    MERGE_DRAIN_DURATION_SECONDS, MERGE_DROPPED_TOTAL, MERGE_HELD_OFFSET_GAUGE,
    MERGE_OUTBOX_CLEAR_FAILURE_TOTAL, MERGE_PENDING_TRANSFERS_GAUGE,
    MERGE_TRANSFERS_SKIPPED_EMPTY_TOTAL, MERGE_TRANSFER_FORWARDS_TOTAL,
    MERGE_TRANSFER_PRODUCE_FAILURE_TOTAL, STAGE1_TRANSITIONS,
};
use crate::partitions::offset_tracker::{MarkOutcome, OffsetTracker};
use crate::partitions::partitioner::COHORT_PARTITION_COUNT;
use crate::partitions::watermarks::LiveWatermarks;
use crate::producer::{
    map_transition, CaptureCascadeSink, CaptureSeedTileSink, CaptureStreamEventSink,
    CaptureTransferSink, CascadeSink, CohortMembershipChange, MembershipSink, SeedTileSink,
    StreamEventSink, TransferSink,
};
use crate::stage1::transition::LeafTransition;
use crate::store::{BehavioralKey, PendingTransferKey, ReadLane, StoreHandle};
use crate::sweep::EvictionQueue;
use crate::workers::person_seed_path::PersonSeedDeps;
use crate::workers::seed_run::RunBudget;
use crate::workers::stage2_path::compose_stage2;
use crate::workers::worker::{
    affected_leaves, first_cascades, produce_cascades, produce_membership, transition_metric_label,
};
use crate::workers::ReconcileDeps;

/// Inline bounded backoff for the transfer produce.
///
/// Worst-case worker hold = `(max_retries + 1)` produce attempts, each fast-failing at the transfer
/// sink's `message.timeout.ms`, plus the sum of the backoff sleeps. With the defaults — 6 attempts ×
/// the 2 s transfer timeout ([`crate::config::Config::merge_transfer_message_timeout_ms`]) + ≈15.5 s
/// of backoff ≈ **27.5 s** — that fits the 30 s graceful-shutdown window (the binding constraint
/// here, tighter than the 60 s liveness deadline), and only because the transfer sink uses its own
/// short timeout.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct TransferRetryPolicy {
    /// Retries after the initial attempt (total attempts = `max_retries + 1`).
    pub max_retries: u32,
    pub base: Duration,
    pub cap: Duration,
}

impl Default for TransferRetryPolicy {
    fn default() -> Self {
        Self {
            max_retries: 5,
            base: Duration::from_millis(500),
            cap: Duration::from_secs(8),
        }
    }
}

impl TransferRetryPolicy {
    /// The pause before 1-based retry `attempt`: `base * 2^(attempt-1)`, capped at `cap`.
    fn backoff(&self, attempt: u32) -> Duration {
        self.base
            .saturating_mul(2u32.saturating_pow(attempt.saturating_sub(1)))
            .min(self.cap)
    }
}

/// Default merge-CF GC per-tick, per-CF scan cap when the deps are built without explicit config
/// (tests). Mirrors `Config::merge_gc_scan_limit`'s default.
pub const DEFAULT_MERGE_GC_SCAN_LIMIT: usize = 10_000;

/// Cascade depth/fan-out caps and the master gate. With `enabled` false the cascade transport is
/// inert (the producer builds nothing, the consumer drains without re-evaluating).
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct CascadeConfig {
    pub enabled: bool,
    /// An incoming cascade at `depth >= this` drops its outgoing hop.
    pub depth_cap: u8,
    /// Max referrer re-evaluations per upstream flip; the remainder self-heals.
    pub fanout_cap: usize,
}

impl Default for CascadeConfig {
    fn default() -> Self {
        Self {
            enabled: false,
            depth_cap: 8,
            fanout_cap: 1000,
        }
    }
}

pub struct MergeWorkerDeps {
    pub transfer_sink: Arc<dyn TransferSink>,
    pub stream_event_sink: Arc<dyn StreamEventSink>,
    pub merge_tracker: Arc<OffsetTracker>,
    pub transfer_tracker: Arc<OffsetTracker>,
    pub retry: TransferRetryPolicy,
    /// Max keys scanned (and at most deleted) per merge CF, per partition, per GC tick. Reused as the
    /// `cf_stage2` orphan-GC page cap.
    pub gc_scan_limit: usize,
    /// Whether the `cf_stage2` orphan GC pass runs on each merge-GC tick (kill-switch, default on).
    pub stage2_orphan_gc_enabled: bool,
    /// Sink for the internal `cohort_cascade_events` topic (a no-op when the gate is off).
    pub cascade_sink: Arc<dyn CascadeSink>,
    /// Isolated from the merge/transfer trackers so cascade-path lag is observable separately.
    pub cascade_tracker: Arc<OffsetTracker>,
    pub cascade: CascadeConfig,
    /// Partition count the merge math routes against. Production 64; test lanes lower it. Threaded
    /// from [`crate::config::Config::cohort_partition_count`] so a re-partitioned lane cannot
    /// misroute against a hardcoded literal.
    pub partition_count: u32,
    /// Sink for cross-partition tile re-keys back into `cohort_stream_seed_events` (a no-op when
    /// the seed gate is off).
    pub seed_tile_sink: Arc<dyn SeedTileSink>,
    /// Isolated from every other tracker so seed offsets can never contaminate the events ceiling.
    pub seed_tracker: Arc<OffsetTracker>,
    /// Fold-frontier watermarks the seed fence reads; the worker advances them post-mark.
    pub live_watermarks: Arc<LiveWatermarks>,
    /// Whether cross-partition merge drains ship the additive membership-register payload. Keep
    /// `false` until every pod in the fleet can apply the transfer; while off the drain ships leaves
    /// only and never holds. Wired from `COHORT_REGISTER_TRANSFER_ENABLED`.
    pub register_transfer_enabled: bool,
    /// Reconcile admission and the pod-wide scheduler wake-up count.
    pub reconcile: ReconcileDeps,
    /// Person-property seed admission and its live-priority margin.
    pub person_seed: PersonSeedDeps,
    /// Ceilings the partition workers group a channel batch's seeds into runs under.
    pub seed_budget: RunBudget,
}

impl MergeWorkerDeps {
    pub fn capture() -> Arc<Self> {
        Arc::new(Self {
            transfer_sink: Arc::new(CaptureTransferSink::new()),
            stream_event_sink: Arc::new(CaptureStreamEventSink::new()),
            merge_tracker: Arc::new(OffsetTracker::new()),
            transfer_tracker: Arc::new(OffsetTracker::new()),
            retry: TransferRetryPolicy::default(),
            gc_scan_limit: DEFAULT_MERGE_GC_SCAN_LIMIT,
            stage2_orphan_gc_enabled: true,
            cascade_sink: Arc::new(CaptureCascadeSink::new()),
            cascade_tracker: Arc::new(OffsetTracker::new()),
            cascade: CascadeConfig::default(),
            partition_count: COHORT_PARTITION_COUNT,
            seed_tile_sink: Arc::new(CaptureSeedTileSink::new()),
            seed_tracker: Arc::new(OffsetTracker::new()),
            live_watermarks: Arc::new(LiveWatermarks::new()),
            register_transfer_enabled: false,
            reconcile: ReconcileDeps::default(),
            person_seed: PersonSeedDeps::default(),
            seed_budget: RunBudget::default(),
        })
    }
}

#[allow(clippy::too_many_arguments)]
#[hotpath::measure]
pub(crate) async fn handle_merge(
    partition_id: u16,
    handle: &StoreHandle,
    catalog: &CatalogHandle,
    sink: &Arc<dyn MembershipSink>,
    merge: &MergeWorkerDeps,
    queue: &mut EvictionQueue<BehavioralKey>,
    last_updated: &str,
    event: &PersonMergeEvent,
    offset: i64,
) {
    let msg_coords = (partition_id as i32, offset);

    let Some(filters) = merge_filters(&catalog.load_full(), TeamId(event.team_id)) else {
        counter!(MERGE_DROPPED_TOTAL, "path" => "drain").increment(1);
        mark_processed(&merge.merge_tracker, partition_id, offset);
        return;
    };

    // The whole drain state machine runs as one sync section on the blocking pool with owned inputs.
    let section = {
        let filters = filters.clone();
        let event_owned = event.clone();
        let partition_count = merge.partition_count;
        let register_transfer_mode =
            MembershipRegisterTransferMode::from_enabled(merge.register_transfer_enabled);
        handle
            .run_section("merge_drain", move |store| {
                // Timed inside the section so the histogram keeps measuring drain execution only;
                // permit and pool-queue waits are on `store_offload_*{op="merge_drain"}`.
                let started = Instant::now();
                let outcome = handle_merge_event_with_transfer_mode(
                    partition_id,
                    store,
                    &filters,
                    &event_owned,
                    msg_coords,
                    partition_count,
                    register_transfer_mode,
                );
                histogram!(MERGE_DRAIN_DURATION_SECONDS).record(started.elapsed().as_secs_f64());
                outcome
            })
            .await
    };

    // The section is only `Err` on teardown cancellation; hold the offset so the next tenure's
    // redelivery resolves the merge.
    let outcome = match section {
        Ok(outcome) => outcome,
        Err(_cancelled) => {
            warn!(
                partition_id,
                team_id = event.team_id,
                "merge drain offloaded section cancelled by shutdown; holding the merge offset for redelivery",
            );
            hold(&merge.merge_tracker, partition_id, offset);
            return;
        }
    };

    let pending_key = PendingTransferKey {
        partition_id,
        team_id: event.team_id as u64,
        old_person: event.old_person_uuid,
    };
    match outcome {
        Ok(DrainOutcome::FastPath {
            transitions,
            effects,
        }) => {
            // The drain's atomic write has committed; sync the queue before any produce.
            effects.apply_to(queue);
            produce_merge_transitions(
                partition_id,
                handle,
                sink,
                merge,
                &filters,
                &transitions,
                event.merged_at_ms,
                last_updated,
                offset,
            )
            .await;
            mark_processed(&merge.merge_tracker, partition_id, offset);
        }
        Ok(DrainOutcome::Drained { transfer, effects }) => {
            effects.apply_to(queue);
            // Nothing to apply — skip the produce. The transfer type owns this predicate so drain
            // staging and worker settlement cannot disagree about new payload classes.
            if !transfer.has_payload() {
                counter!(MERGE_TRANSFERS_SKIPPED_EMPTY_TOTAL).increment(1);
                mark_processed(&merge.merge_tracker, partition_id, offset);
                return;
            }
            produce_and_settle(partition_id, handle, merge, &transfer, &pending_key, offset).await;
        }
        Ok(DrainOutcome::AlreadyDrained) => {
            match handle.get_pending_transfer(&pending_key).await {
                Ok(None) => mark_processed(&merge.merge_tracker, partition_id, offset),
                Ok(Some(bytes)) => match PendingTransfer::decode(&bytes) {
                    Ok(pending) => {
                        produce_and_settle(
                            partition_id,
                            handle,
                            merge,
                            &pending.transfer,
                            &pending_key,
                            offset,
                        )
                        .await;
                    }
                    Err(error) => {
                        warn!(
                            partition_id,
                            team_id = event.team_id,
                            old_person = %event.old_person_uuid,
                            error = %error,
                            "pending transfer failed to decode; committing past it (entry left for the redrive to surface)",
                        );
                        mark_processed(&merge.merge_tracker, partition_id, offset);
                    }
                },
                Err(error) => {
                    // Category A: the outbox read itself failed, so we cannot re-produce the staged
                    // transfer this pass and have no other handle on it — a sticky hold redelivers the
                    // merge so a later pass (or the next tenure) can resolve it.
                    warn!(
                        partition_id,
                        team_id = event.team_id,
                        error = %error,
                        "pending transfer read failed; holding the merge offset for redelivery",
                    );
                    hold(&merge.merge_tracker, partition_id, offset);
                }
            }
        }
        Ok(DrainOutcome::Skipped(_)) => {
            mark_processed(&merge.merge_tracker, partition_id, offset);
        }
        Err(error) => {
            // Category A: the drain's atomic `write_batch` failed, so it left no
            // `cf_merge_drains_applied` marker and produced no transfer — nothing recovers this but a
            // replay of the merge message. A sticky hold redelivers it; advancing past it would drop
            // the merge silently.
            warn!(
                partition_id,
                team_id = event.team_id,
                old_person = %event.old_person_uuid,
                error = %error,
                "merge drain store error; holding the merge offset for redelivery",
            );
            hold(&merge.merge_tracker, partition_id, offset);
        }
    }
}

#[allow(clippy::too_many_arguments)]
#[hotpath::measure]
pub(crate) async fn handle_apply(
    partition_id: u16,
    handle: &StoreHandle,
    catalog: &CatalogHandle,
    sink: &Arc<dyn MembershipSink>,
    merge: &MergeWorkerDeps,
    queue: &mut EvictionQueue<BehavioralKey>,
    last_updated: &str,
    transfer: &MergeStateTransfer,
    offset: i64,
) {
    let transfer_coords = (partition_id as i32, offset);

    let Some(filters) = merge_filters(&catalog.load_full(), TeamId(transfer.team_id)) else {
        counter!(MERGE_DROPPED_TOTAL, "path" => "apply").increment(1);
        mark_processed(&merge.transfer_tracker, partition_id, offset);
        return;
    };

    // The whole apply state machine runs as one sync section on the blocking pool with owned inputs.
    let section = {
        let filters = filters.clone();
        let transfer_owned = transfer.clone();
        let partition_count = merge.partition_count;
        handle
            .run_section("merge_apply", move |store| {
                // Timed inside the section so the histogram keeps measuring apply execution only;
                // permit and pool-queue waits are on `store_offload_*{op="merge_apply"}`.
                let started = Instant::now();
                let outcome = handle_transfer(
                    partition_id,
                    store,
                    &filters,
                    &transfer_owned,
                    transfer_coords,
                    partition_count,
                );
                histogram!(MERGE_APPLY_DURATION_SECONDS).record(started.elapsed().as_secs_f64());
                outcome
            })
            .await
    };

    let outcome = match section {
        Ok(outcome) => outcome,
        Err(_cancelled) => {
            warn!(
                partition_id,
                team_id = transfer.team_id,
                "transfer apply offloaded section cancelled by shutdown; holding the transfer offset for redelivery",
            );
            hold(&merge.transfer_tracker, partition_id, offset);
            return;
        }
    };

    match outcome {
        Ok(ApplyOutcome::Applied {
            transitions,
            effects,
        }) => {
            // The apply's atomic write has committed; sync the queue before any produce.
            effects.apply_to(queue);
            produce_merge_transitions(
                partition_id,
                handle,
                sink,
                merge,
                &filters,
                &transitions,
                transfer.merged_at_ms,
                last_updated,
                offset,
            )
            .await;
            mark_processed(&merge.transfer_tracker, partition_id, offset);
        }
        Ok(ApplyOutcome::AlreadyApplied) => {
            mark_processed(&merge.transfer_tracker, partition_id, offset);
        }
        Ok(ApplyOutcome::Forward {
            transfer: forwarded,
        }) => {
            forward_transfer(partition_id, merge, &forwarded, offset).await;
        }
        Ok(ApplyOutcome::HopCapped) => {
            // No state and no marker were written; mark so a corrupt tombstone cycle does not wedge
            // the partition (the counter already fired in the handler).
            mark_processed(&merge.transfer_tracker, partition_id, offset);
        }
        Err(error) => {
            // Category A: the atomic apply `write_batch` failed, so no `cf_merge_applied` marker was
            // written and the Kafka transfer is the last copy of P_old's state (drain already deleted
            // its leaves, the outbox slot was cleared on the produce ack). A sticky hold replays the
            // transfer cleanly; advancing past it would lose the state for good.
            warn!(
                partition_id,
                team_id = transfer.team_id,
                new_person = %transfer.new_person_uuid,
                error = %error,
                "transfer apply store error; holding the transfer offset for redelivery",
            );
            hold(&merge.transfer_tracker, partition_id, offset);
        }
    }
}

/// Forward a re-targeted transfer to the survivor's partition (the raced chained-merge case).
///
/// A **single** produce attempt on the transfer sink (not the inline retry budget — the apply path
/// must not hold the worker): a failed produce holds the transfer offset (no mark) so redelivery
/// re-resolves and re-produces. The forward writes no state, so the hold is self-healing. The
/// transfer offset is marked only after the ack, and the forward is counted post-ack (`re_keyed`).
async fn forward_transfer(
    partition_id: u16,
    merge: &MergeWorkerDeps,
    forwarded: &MergeStateTransfer,
    offset: i64,
) {
    let acks = merge.transfer_sink.produce(vec![forwarded.clone()]).await;
    if !acks.iter().all(Result::is_ok) {
        // Category B: the forward wrote no state and no outbox entry, so the failed produce leaves
        // nothing to redrive — a sticky hold redelivers the transfer, which re-resolves and re-forwards
        // (the survivor's marker dedups any duplicate).
        warn!(
            partition_id,
            team_id = forwarded.team_id,
            old_person = %forwarded.old_person_uuid,
            target = %forwarded.new_person_uuid,
            forward_hops = forwarded.forward_hops,
            "transfer forward produce failed; holding the transfer offset for redelivery",
        );
        hold(&merge.transfer_tracker, partition_id, offset);
        return;
    }
    counter!(MERGE_TRANSFER_FORWARDS_TOTAL, "path" => "re_keyed").increment(1);
    mark_processed(&merge.transfer_tracker, partition_id, offset);
}

const REDRIVE_MAX_ATTEMPTS_PER_TICK: usize = 8;

/// Cap on the pending-transfer scan: a small multiple of the per-tick attempt cap, so the redrive has
/// the next few ticks' worth of work staged without copying the whole outbox each tick.
const SCAN_PENDING_TRANSFERS_LIMIT: usize = REDRIVE_MAX_ATTEMPTS_PER_TICK * 8;

#[hotpath::measure]
pub(crate) async fn handle_redrive(
    partition_id: u16,
    handle: &StoreHandle,
    merge: &MergeWorkerDeps,
) {
    // Per-op through the facade, not one section: each produce interleaves with a store clear, and a
    // section would serialize the whole tick's Kafka round-trips on one blocking thread.
    let entries = match handle
        .scan_pending_transfers(partition_id, None, SCAN_PENDING_TRANSFERS_LIMIT)
        .await
    {
        Ok(entries) => entries,
        Err(error) => {
            warn!(
                partition_id,
                error = %error,
                "pending-transfer redrive scan failed; retrying next tick",
            );
            return;
        }
    };
    // Saturates at the scan limit, so this reflects `min(backlog, SCAN_PENDING_TRANSFERS_LIMIT)` — it
    // still distinguishes an empty outbox from a backed-up one, just not the exact backlog depth.
    gauge!(MERGE_PENDING_TRANSFERS_GAUGE, "partition" => partition_id.to_string())
        .set(entries.len() as f64);

    let mut attempted = 0usize;
    for (key, bytes) in entries {
        if attempted == REDRIVE_MAX_ATTEMPTS_PER_TICK {
            debug!(
                partition_id,
                "redrive tick reached its produce-attempt cap; remaining entries wait for the next tick",
            );
            break;
        }
        let pending = match PendingTransfer::decode(&bytes) {
            Ok(pending) => pending,
            Err(error) => {
                warn!(
                    partition_id,
                    team_id = key.team_id,
                    old_person = %key.old_person,
                    error = %error,
                    "pending transfer failed to decode during redrive; leaving the entry in place",
                );
                continue;
            }
        };
        let team_id = pending.transfer.team_id;
        let old_person = pending.transfer.old_person_uuid;
        attempted += 1;
        let acks = merge.transfer_sink.produce(vec![pending.transfer]).await;
        if !acks.iter().all(Result::is_ok) {
            warn!(
                partition_id,
                team_id,
                old_person = %old_person,
                "redrive transfer produce failed; leaving the entry for the next tick",
            );
            continue;
        }
        if let Err(error) = handle.clear_pending_transfer(&key).await {
            counter!(MERGE_OUTBOX_CLEAR_FAILURE_TOTAL).increment(1);
            warn!(
                partition_id,
                team_id,
                old_person = %old_person,
                error = %error,
                "outbox clear failed after an acked redrive produce; marking anyway",
            );
        }
        debug_assert_eq!(pending.merge_msg_partition, partition_id as i32);
        mark_processed(&merge.merge_tracker, partition_id, pending.merge_msg_offset);
    }
}

/// Produce `transfer` with inline retry; on ack clear its outbox slot and mark the merge offset.
async fn produce_and_settle(
    partition_id: u16,
    handle: &StoreHandle,
    merge: &MergeWorkerDeps,
    transfer: &MergeStateTransfer,
    pending_key: &PendingTransferKey,
    offset: i64,
) {
    if !produce_transfer_with_retry(&merge.transfer_sink, transfer, &merge.retry, partition_id)
        .await
    {
        // Category C: deliberately **no** `hold` here. The transfer stays staged in
        // `cf_pending_transfers` and the periodic redrive (`handle_redrive`) owns recovery — it
        // re-produces and then `mark_processed`es `K + 1`. A sticky hold would pin `K` even after the
        // redrive advances past it, wedging the partition on a message the outbox already handles.
        counter!(MERGE_TRANSFER_PRODUCE_FAILURE_TOTAL).increment(1);
        warn!(
            partition_id,
            team_id = transfer.team_id,
            old_person = %transfer.old_person_uuid,
            new_person = %transfer.new_person_uuid,
            "transfer produce exhausted its retry budget; leaving the entry pending and skipping the offset mark",
        );
        return;
    }
    if let Err(error) = handle.clear_pending_transfer(pending_key).await {
        counter!(MERGE_OUTBOX_CLEAR_FAILURE_TOTAL).increment(1);
        warn!(
            partition_id,
            team_id = transfer.team_id,
            old_person = %transfer.old_person_uuid,
            error = %error,
            "outbox clear failed after an acked transfer produce; committing anyway",
        );
    }
    mark_processed(&merge.merge_tracker, partition_id, offset);
}

/// Produce one transfer, retrying inline with bounded backoff. Returns whether the produce was acked.
async fn produce_transfer_with_retry(
    sink: &Arc<dyn TransferSink>,
    transfer: &MergeStateTransfer,
    retry: &TransferRetryPolicy,
    partition_id: u16,
) -> bool {
    for attempt in 0..=retry.max_retries {
        if attempt > 0 {
            tokio::time::sleep(retry.backoff(attempt)).await;
        }
        let acks = sink.produce(vec![transfer.clone()]).await;
        if acks.iter().all(Result::is_ok) {
            return true;
        }
        debug!(
            partition_id,
            attempt,
            team_id = transfer.team_id,
            old_person = %transfer.old_person_uuid,
            "transfer produce attempt failed",
        );
    }
    false
}

/// Fan transitions into membership output (single-leaf + Stage 2), then produce (at-most-once).
/// `source_offset` seeds each flip's first cascade; both produces drop on failure.
#[allow(clippy::too_many_arguments)]
async fn produce_merge_transitions(
    partition_id: u16,
    handle: &StoreHandle,
    sink: &Arc<dyn MembershipSink>,
    merge: &MergeWorkerDeps,
    filters: &TeamFilters,
    transitions: &[LeafTransition],
    merged_at_ms: i64,
    last_updated: &str,
    source_offset: i64,
) {
    if transitions.is_empty() {
        return;
    }
    let mut changes: Vec<CohortMembershipChange> = Vec::new();
    for transition in transitions {
        if let Some(kind) = transition_metric_label(filters, transition) {
            counter!(STAGE1_TRANSITIONS, "kind" => kind).increment(1);
        }
        changes.extend(map_transition(filters, transition, last_updated));
    }
    match compose_stage2(
        partition_id,
        handle,
        filters,
        &affected_leaves(transitions),
        merged_at_ms,
        last_updated,
        ReadLane::Event,
    )
    .await
    {
        Ok(stage2_changes) => changes.extend(stage2_changes),
        Err(error) => warn!(
            partition_id,
            error = %error,
            "merge stage 2 composition failed; skipping (self-heals on the person's next event)",
        ),
    }
    if changes.is_empty() {
        return;
    }

    let cascades = first_cascades(merge, &changes, source_offset);
    let errors = produce_membership(sink, changes).await;
    if errors > 0 {
        warn!(
            partition_id,
            errors,
            "merge membership produce failed; dropping (state already committed, at-most-once)",
        );
        return;
    }
    let cascade_errors = produce_cascades(merge, cascades).await;
    if cascade_errors > 0 {
        warn!(
            partition_id,
            errors = cascade_errors,
            "merge cascade produce failed; dropping (at-most-once). Recovery depends on each referrer being re-evaluated on its next event; the sweep does not re-evaluate cohort-ref shapes with no behavioral leaf",
        );
    }
}

fn mark_processed(tracker: &OffsetTracker, partition_id: u16, offset: i64) {
    if let MarkOutcome::CappedAheadOfDispatch =
        tracker.mark_processed(partition_id as i32, offset + 1)
    {
        counter!(COHORT_STREAM_OFFSET_AHEAD_OF_DISPATCH).increment(1);
        warn!(
            partition_id,
            next_offset = offset + 1,
            "offset mark exceeded the dispatch ceiling and was capped (F1 invariant violation)",
        );
    }
}

/// Pin the partition's commit floor at the failed message's **own** offset (no `+ 1`, unlike
/// [`mark_processed`]) so Kafka redelivers it instead of a later success leapfrogging it. The hold is
/// sticky for the worker's tenure; emit [`MERGE_HELD_OFFSET_GAUGE`] so the resulting commit-stall is
/// never silent (alert on a sustained non-zero level).
fn hold(tracker: &OffsetTracker, partition_id: u16, offset: i64) {
    // Report the resulting floor (an earlier hold may pin a lower offset), not the raw `offset`, so
    // the gauge matches the position Kafka will actually redeliver.
    let floor = tracker.hold(partition_id as i32, offset);
    gauge!(MERGE_HELD_OFFSET_GAUGE, "partition" => partition_id.to_string()).set(floor as f64);
}

/// The filters a merge or a transfer of `team_id` runs with, or `None` when the catalog does not
/// process the team. Even with empty filters, a drain writes a tombstone and a drain marker and an
/// apply writes an apply marker, and the processor must hold no state for a team it does not
/// process. A processed team with no realtime cohort gets empty filters, so its tombstones still
/// redirect stragglers to the survivor.
///
/// The empty boot catalog processes no team, but it drops nothing, because
/// `spawn_follower_after_boot` starts the merge and transfer followers only after the first catalog
/// load.
fn merge_filters(catalog: &FilterCatalog, team_id: TeamId) -> Option<Arc<TeamFilters>> {
    catalog.processes(team_id).then(|| {
        catalog
            .team(team_id)
            .cloned()
            .unwrap_or_else(|| Arc::new(TeamFiltersBuilder::default().freeze(UTC)))
    })
}

#[cfg(test)]
// Tests seed and assert against `CohortStore` directly while driving `handle_redrive` through the
// `StoreHandle` facade.
#[allow(clippy::disallowed_methods)]
mod tests {
    use std::collections::HashSet;

    use super::*;
    use common_types::cohort::TeamAllowlist;
    use envconfig::Envconfig;
    use serde_json::json;
    use tempfile::TempDir;
    use uuid::Uuid;

    use crate::filters::{CohortId, FilterCatalog};
    use crate::merge::transfer::{TransferLeaf, MERGE_EVENT_SCHEMA_VERSION};
    use crate::partitions::partitioner::partition_of;
    use crate::producer::CaptureSink;
    use crate::stage1::key::LeafStateKey;
    use crate::stage1::state::{AppliedOffsets, Stage1State, StatefulRecord};
    use crate::stage2::Stage2State;
    use crate::store::{
        CohortStore, MergeAppliedKey, OffloadConfig, OffloadMode, Stage2Key, StoreConfig,
        TombstoneKey,
    };

    /// Wrap a test store in the `All` operating point so `handle_redrive` runs on the blocking pool.
    fn handle(store: &CohortStore) -> StoreHandle {
        StoreHandle::new(
            store.clone(),
            OffloadConfig {
                mode: OffloadMode::All,
                event_read_permits: 16,
                maintenance_permits: 6,
            },
        )
    }

    #[test]
    fn default_retry_policy_budget() {
        let policy = TransferRetryPolicy::default();
        assert_eq!(policy.max_retries, 5);
        let backoffs: Vec<Duration> = (1..=policy.max_retries)
            .map(|a| policy.backoff(a))
            .collect();
        assert_eq!(
            backoffs,
            vec![
                Duration::from_millis(500),
                Duration::from_secs(1),
                Duration::from_secs(2),
                Duration::from_secs(4),
                Duration::from_secs(8),
            ],
        );
        let total: Duration = backoffs.iter().sum();
        assert_eq!(total, Duration::from_millis(15_500), "≈15.5 s end to end");
    }

    /// The inline transfer-retry worst case — every produce attempt fast-failing at the dedicated
    /// transfer timeout, plus all backoff sleeps — must stay inside the 30 s graceful-shutdown window
    /// (the binding constraint, tighter than the 60 s liveness deadline). Sourced from the live config
    /// default and the retry-policy default so a drift in *either* trips this test.
    #[test]
    fn inline_transfer_retry_worst_case_fits_the_graceful_shutdown_window() {
        let config = crate::config::Config::init_from_hashmap(&std::collections::HashMap::new())
            .expect("envconfig defaults load");
        let policy = config.transfer_retry_policy();

        // total attempts = initial + retries; each can block for the full transfer message timeout.
        let attempts = policy.max_retries + 1;
        let per_attempt =
            Duration::from_millis(u64::from(config.merge_transfer_message_timeout_ms));
        let produce_block = per_attempt * attempts;

        let backoff: Duration = (1..=policy.max_retries).map(|a| policy.backoff(a)).sum();
        let worst_case = produce_block + backoff;

        // 6 × 2 s + 15.5 s = 27.5 s with the current defaults.
        assert_eq!(worst_case, Duration::from_millis(27_500));

        const GRACEFUL_SHUTDOWN_WINDOW: Duration = Duration::from_secs(30);
        assert!(
            worst_case <= GRACEFUL_SHUTDOWN_WINDOW,
            "inline transfer retry worst case {worst_case:?} exceeds the {GRACEFUL_SHUTDOWN_WINDOW:?} graceful-shutdown window",
        );
    }

    #[test]
    fn backoff_is_capped_and_saturating() {
        let policy = TransferRetryPolicy {
            max_retries: 64,
            base: Duration::from_secs(1),
            cap: Duration::from_secs(8),
        };
        assert_eq!(policy.backoff(4), Duration::from_secs(8));
        assert_eq!(policy.backoff(64), Duration::from_secs(8));
        assert_eq!(policy.backoff(0), Duration::from_secs(1));
    }

    #[tokio::test(start_paused = true)]
    async fn produce_with_retry_succeeds_after_transient_failures_without_wall_clock_sleep() {
        let wall = std::time::Instant::now();
        let capture = CaptureTransferSink::failing_first(2);
        let sink: Arc<dyn TransferSink> = Arc::new(capture.clone());
        let transfer = MergeStateTransfer {
            team_id: 7,
            old_person_uuid: uuid::Uuid::from_u128(1),
            new_person_uuid: uuid::Uuid::from_u128(2),
            merged_at_ms: 1,
            source_partition: 0,
            source_offset: 0,
            leaves: vec![],
            membership_registers: vec![],
            forward_hops: 0,

            person_dedup: None,
        };

        let acked =
            produce_transfer_with_retry(&sink, &transfer, &TransferRetryPolicy::default(), 0).await;

        assert!(acked, "third attempt succeeds within the budget");
        assert_eq!(capture.transfers(), vec![transfer]);
        assert!(
            wall.elapsed() < Duration::from_secs(2),
            "backoff ran on the paused clock, not the wall clock",
        );
    }

    #[tokio::test(start_paused = true)]
    async fn produce_with_retry_exhausts_after_max_retries() {
        let capture = CaptureTransferSink::failing_always();
        let sink: Arc<dyn TransferSink> = Arc::new(capture.clone());
        let transfer = MergeStateTransfer {
            team_id: 7,
            old_person_uuid: uuid::Uuid::from_u128(1),
            new_person_uuid: uuid::Uuid::from_u128(2),
            merged_at_ms: 1,
            source_partition: 0,
            source_offset: 0,
            leaves: vec![],
            membership_registers: vec![],
            forward_hops: 0,

            person_dedup: None,
        };

        let acked =
            produce_transfer_with_retry(&sink, &transfer, &TransferRetryPolicy::default(), 0).await;

        assert!(!acked);
        assert!(capture.transfers().is_empty());
    }

    const REDRIVE_PARTITION: u16 = 3;

    fn temp_store() -> (TempDir, CohortStore) {
        let dir = TempDir::new().unwrap();
        let store = CohortStore::open(&StoreConfig {
            path: dir.path().join("db"),
            ..StoreConfig::default()
        })
        .unwrap();
        (dir, store)
    }

    fn capture_deps(transfer_sink: CaptureTransferSink) -> MergeWorkerDeps {
        MergeWorkerDeps {
            transfer_sink: Arc::new(transfer_sink),
            stream_event_sink: Arc::new(CaptureStreamEventSink::new()),
            merge_tracker: Arc::new(OffsetTracker::new()),
            transfer_tracker: Arc::new(OffsetTracker::new()),
            retry: TransferRetryPolicy::default(),
            gc_scan_limit: DEFAULT_MERGE_GC_SCAN_LIMIT,
            stage2_orphan_gc_enabled: true,
            cascade_sink: Arc::new(CaptureCascadeSink::new()),
            cascade_tracker: Arc::new(OffsetTracker::new()),
            cascade: CascadeConfig::default(),
            partition_count: COHORT_PARTITION_COUNT,
            seed_tile_sink: Arc::new(crate::producer::CaptureSeedTileSink::new()),
            seed_tracker: Arc::new(crate::partitions::offset_tracker::OffsetTracker::new()),
            live_watermarks: Arc::new(crate::partitions::watermarks::LiveWatermarks::new()),
            register_transfer_enabled: false,
            reconcile: ReconcileDeps::default(),
            person_seed: PersonSeedDeps::default(),
            seed_budget: RunBudget::default(),
        }
    }

    fn staged_pending(p_old: Uuid, merge_offset: i64) -> (PendingTransferKey, PendingTransfer) {
        let transfer = MergeStateTransfer {
            team_id: 7,
            old_person_uuid: p_old,
            new_person_uuid: Uuid::from_u128(0xBEEF),
            merged_at_ms: 1,
            source_partition: REDRIVE_PARTITION as i32,
            source_offset: merge_offset,
            leaves: vec![TransferLeaf::new(
                LeafStateKey([0xAB; 16]),
                StatefulRecord::new(
                    Stage1State::BehavioralSingle {
                        has_match: true,
                        last_event_at_ms: 1,
                        earliest_eviction_at_ms: i64::MAX,
                    },
                    AppliedOffsets::default(),
                ),
            )],
            membership_registers: vec![],
            forward_hops: 0,

            person_dedup: None,
        };
        let key = PendingTransferKey {
            partition_id: REDRIVE_PARTITION,
            team_id: 7,
            old_person: p_old,
        };
        let pending = PendingTransfer {
            transfer,
            merge_msg_partition: REDRIVE_PARTITION as i32,
            merge_msg_offset: merge_offset,
        };
        (key, pending)
    }

    fn stage(store: &CohortStore, key: &PendingTransferKey, bytes: &[u8]) {
        store
            .write_batch(|batch| batch.put_pending_transfer(key, bytes))
            .unwrap();
    }

    #[tokio::test]
    async fn redrive_re_produces_a_staged_entry_clears_it_and_marks_the_stored_coords() {
        let (_dir, store) = temp_store();
        let (key, pending) = staged_pending(Uuid::from_u128(1), 41);
        stage(&store, &key, &pending.encode());

        let sink = CaptureTransferSink::new();
        let deps = capture_deps(sink.clone());
        // The exhausted merge was dispatched this tenure, so its ceiling is already raised.
        deps.merge_tracker
            .mark_dispatched(REDRIVE_PARTITION as i32, 42);

        handle_redrive(REDRIVE_PARTITION, &handle(&store), &deps).await;

        assert_eq!(sink.transfers(), vec![pending.transfer]);
        assert!(
            store.get_pending_transfer(&key).unwrap().is_none(),
            "the entry was cleared after the ack",
        );
        assert_eq!(
            deps.merge_tracker
                .committable_offsets()
                .get(&(REDRIVE_PARTITION as i32)),
            Some(&42),
            "the stored merge-message coordinates were marked",
        );
    }

    #[tokio::test]
    async fn redrive_produce_failure_leaves_the_entry_and_the_offset_for_the_next_tick() {
        let (_dir, store) = temp_store();
        let (key, pending) = staged_pending(Uuid::from_u128(1), 41);
        stage(&store, &key, &pending.encode());

        let sink = CaptureTransferSink::failing_first(1);
        let deps = capture_deps(sink.clone());
        deps.merge_tracker
            .mark_dispatched(REDRIVE_PARTITION as i32, 42);

        handle_redrive(REDRIVE_PARTITION, &handle(&store), &deps).await;
        assert!(sink.transfers().is_empty());
        assert!(store.get_pending_transfer(&key).unwrap().is_some());
        assert!(
            deps.merge_tracker.committable_offsets().is_empty(),
            "a failed redrive marks nothing",
        );

        handle_redrive(REDRIVE_PARTITION, &handle(&store), &deps).await;
        assert_eq!(sink.transfers(), vec![pending.transfer]);
        assert!(store.get_pending_transfer(&key).unwrap().is_none());
        assert_eq!(
            deps.merge_tracker
                .committable_offsets()
                .get(&(REDRIVE_PARTITION as i32)),
            Some(&42),
        );
    }

    #[tokio::test(start_paused = true)]
    async fn redrive_makes_a_single_produce_attempt_per_tick_with_no_backoff() {
        let (_dir, store) = temp_store();
        let (key, pending) = staged_pending(Uuid::from_u128(1), 41);
        stage(&store, &key, &pending.encode());
        let deps = capture_deps(CaptureTransferSink::failing_always());
        deps.merge_tracker
            .mark_dispatched(REDRIVE_PARTITION as i32, 42);

        let before = tokio::time::Instant::now();
        handle_redrive(REDRIVE_PARTITION, &handle(&store), &deps).await;

        assert_eq!(
            tokio::time::Instant::now() - before,
            Duration::ZERO,
            "no backoff sleep on the paused clock — exactly one attempt per tick",
        );
        assert!(store.get_pending_transfer(&key).unwrap().is_some());
    }

    #[tokio::test]
    async fn redrive_leaves_an_undecodable_entry_in_place_and_still_recovers_the_rest() {
        let (_dir, store) = temp_store();
        let corrupt_key = PendingTransferKey {
            partition_id: REDRIVE_PARTITION,
            team_id: 7,
            old_person: Uuid::from_u128(1),
        };
        stage(&store, &corrupt_key, b"not-a-pending-transfer");
        let (good_key, good) = staged_pending(Uuid::from_u128(2), 41);
        stage(&store, &good_key, &good.encode());

        let sink = CaptureTransferSink::new();
        let deps = capture_deps(sink.clone());
        deps.merge_tracker
            .mark_dispatched(REDRIVE_PARTITION as i32, 42);

        handle_redrive(REDRIVE_PARTITION, &handle(&store), &deps).await;

        assert_eq!(
            sink.transfers(),
            vec![good.transfer],
            "the good entry was recovered",
        );
        assert!(store.get_pending_transfer(&good_key).unwrap().is_none());
        assert!(
            store.get_pending_transfer(&corrupt_key).unwrap().is_some(),
            "the corrupt entry stays for every later tick to surface",
        );
    }

    #[tokio::test]
    async fn redrive_attempts_at_most_the_cap_per_tick_and_the_remainder_survives() {
        let (_dir, store) = temp_store();
        let backlog = REDRIVE_MAX_ATTEMPTS_PER_TICK + 3;
        let staged: Vec<_> = (0..backlog)
            .map(|i| {
                let (key, pending) = staged_pending(Uuid::from_u128(i as u128 + 1), 41 + i as i64);
                stage(&store, &key, &pending.encode());
                key
            })
            .collect();

        let sink = CaptureTransferSink::new();
        let deps = capture_deps(sink.clone());
        deps.merge_tracker
            .mark_dispatched(REDRIVE_PARTITION as i32, 41 + backlog as i64);

        handle_redrive(REDRIVE_PARTITION, &handle(&store), &deps).await;
        assert_eq!(
            sink.transfers().len(),
            REDRIVE_MAX_ATTEMPTS_PER_TICK,
            "tick 1 attempted exactly the cap",
        );
        for key in &staged[..REDRIVE_MAX_ATTEMPTS_PER_TICK] {
            assert!(store.get_pending_transfer(key).unwrap().is_none());
        }
        for key in &staged[REDRIVE_MAX_ATTEMPTS_PER_TICK..] {
            assert!(
                store.get_pending_transfer(key).unwrap().is_some(),
                "entries past the cap survive to the next tick",
            );
        }

        handle_redrive(REDRIVE_PARTITION, &handle(&store), &deps).await;
        assert_eq!(
            sink.transfers().len(),
            backlog,
            "tick 2 drained the remainder",
        );
        for key in &staged {
            assert!(store.get_pending_transfer(key).unwrap().is_none());
        }
    }

    const FORWARD_PARTITION: u16 = 5;

    fn forwarded_transfer() -> MergeStateTransfer {
        MergeStateTransfer {
            team_id: 7,
            old_person_uuid: Uuid::from_u128(0xA),
            new_person_uuid: Uuid::from_u128(0xC),
            merged_at_ms: 1,
            source_partition: 9,
            source_offset: 100,
            leaves: vec![],
            membership_registers: vec![],
            forward_hops: 1,

            person_dedup: None,
        }
    }

    #[tokio::test]
    async fn forward_produce_failure_holds_the_transfer_offset_until_redelivery_succeeds() {
        let forwarded = forwarded_transfer();

        // First attempt: the sink fails → nothing produced, offset not marked. A separate fresh sink
        // models the recovered attempt below.
        let failing_sink = CaptureTransferSink::failing_always();
        let failing = capture_deps(failing_sink.clone());
        forward_transfer(FORWARD_PARTITION, &failing, &forwarded, 80).await;
        assert!(
            failing_sink.transfers().is_empty(),
            "a failed forward produced nothing",
        );
        assert!(
            failing.transfer_tracker.committable_offsets().is_empty(),
            "a failed forward marks no offset",
        );

        // Redelivery against a working sink: produced once, offset marked once.
        let working_sink = CaptureTransferSink::new();
        let working = capture_deps(working_sink.clone());
        working
            .transfer_tracker
            .mark_dispatched(FORWARD_PARTITION as i32, 81);
        forward_transfer(FORWARD_PARTITION, &working, &forwarded, 80).await;
        assert_eq!(
            working_sink.transfers(),
            vec![forwarded],
            "the redelivery forwarded exactly once",
        );
        assert_eq!(
            working
                .transfer_tracker
                .committable_offsets()
                .get(&(FORWARD_PARTITION as i32)),
            Some(&81),
            "the transfer offset is marked only after the ack",
        );
    }

    /// A store-error hold must be sticky: a later success on the same partition must not leapfrog
    /// the held offset. `hold` pins `K` (the failed offset) and a subsequent `mark_processed` at
    /// `K' > K` must leave committable at `K`, not advance it.
    #[test]
    fn an_apply_store_error_hold_is_not_leapfrogged_by_a_later_successful_transfer() {
        const P: u16 = 6;
        let deps = capture_deps(CaptureTransferSink::new());
        // The partition processed transfers up to offset 40 this tenure, then saw 41 (fails) and 43.
        deps.transfer_tracker.mark_dispatched(P as i32, 44);
        mark_processed(&deps.transfer_tracker, P, 40);
        assert_eq!(
            deps.transfer_tracker.committable_offsets().get(&(P as i32)),
            Some(&41),
            "before the failure, committable is the next-offset 41",
        );

        // The apply store-error arm holds at the failed message's own offset K=41 (no +1).
        hold(&deps.transfer_tracker, P, 41);

        // A later transfer for the same partition applies cleanly and marks next-offset 44.
        mark_processed(&deps.transfer_tracker, P, 43);
        assert_eq!(
            deps.transfer_tracker.committable_offsets().get(&(P as i32)),
            Some(&41),
            "the later success must NOT advance committable past the held offset 41",
        );
    }

    /// With `register_transfer_enabled` false, a cross-partition merge of a person that owns a
    /// single-leaf register row must drain and advance its offset (never hold and wedge the partition)
    /// and reclaim P_old's register row. Guards against reintroducing a gate-off hold.
    #[tokio::test]
    async fn disabled_register_transfer_drains_the_cross_partition_merge_without_holding() {
        const K: i64 = 50;

        let cohort = json!({
            "properties": {
                "type": "AND",
                "values": [{
                    "type": "behavioral", "key": "$pageview", "value": "performed_event",
                    "time_value": 7, "time_interval": "day",
                    "conditionHash": "0123456789abcdef",
                    "bytecode": ["_H", 1, 32, "$pageview", 32, "event", 1, 1, 11],
                }]
            }
        });
        let mut builder = TeamFiltersBuilder::default();
        builder.add_cohort(CohortId(1), TeamId(7), &cohort).unwrap();
        let filters = builder.freeze(UTC);
        let catalog =
            CatalogHandle::from_catalog(FilterCatalog::from_teams([(TeamId(7), filters)]));

        let old_person = Uuid::from_u128(1);
        let old_partition = partition_of(TeamId(7), &old_person, COHORT_PARTITION_COUNT) as u16;
        let new_person = (2u128..)
            .map(Uuid::from_u128)
            .find(|person| {
                partition_of(TeamId(7), person, COHORT_PARTITION_COUNT) as u16 != old_partition
            })
            .unwrap();
        let register_key = Stage2Key {
            partition_id: old_partition,
            team_id: 7,
            cohort_id: 1,
            person_id: old_person,
        };

        let (_dir, store) = temp_store();
        store
            .write_batch(|batch| {
                batch.put_stage2(
                    &register_key,
                    &Stage2State {
                        in_cohort: true,
                        last_evaluated_at_ms: 1,
                    }
                    .encode(),
                );
            })
            .unwrap();

        // `capture_deps` builds with `register_transfer_enabled: false`.
        let deps = capture_deps(CaptureTransferSink::new());
        deps.merge_tracker
            .mark_dispatched(old_partition as i32, K + 1);

        let sink: Arc<dyn MembershipSink> = Arc::new(CaptureSink::new());
        let mut queue: EvictionQueue<BehavioralKey> = EvictionQueue::new();
        handle_merge(
            old_partition,
            &handle(&store),
            &catalog,
            &sink,
            &deps,
            &mut queue,
            "1",
            &PersonMergeEvent {
                team_id: 7,
                old_person_uuid: old_person,
                new_person_uuid: new_person,
                merged_at_ms: 2,
                schema_version: MERGE_EVENT_SCHEMA_VERSION,
            },
            K,
        )
        .await;

        assert_eq!(
            deps.merge_tracker
                .committable_offsets()
                .get(&(old_partition as i32)),
            Some(&(K + 1)),
            "the disabled-transfer merge drains and advances past its offset rather than holding",
        );
        assert!(
            store.get_stage2(&register_key).unwrap().is_none(),
            "P_old's register row is reclaimed by the drain, like the pre-register pipeline",
        );
    }

    /// The team-without-cohorts tests in `consumers::events` pin that a processed team still drains.
    #[tokio::test]
    async fn a_merge_and_its_transfer_of_a_team_the_catalog_does_not_process_write_nothing() {
        const TEAM: i32 = 8;
        const MERGE_OFFSET: i64 = 50;
        const TRANSFER_OFFSET: i64 = 80;

        let catalog = CatalogHandle::from_catalog(FilterCatalog::for_processing(
            TeamAllowlist::Only(HashSet::from([7])),
            [],
        ));
        let old_person = Uuid::from_u128(1);
        let new_person = Uuid::from_u128(2);
        let old_partition = partition_of(TeamId(TEAM), &old_person, COHORT_PARTITION_COUNT) as u16;
        let new_partition = partition_of(TeamId(TEAM), &new_person, COHORT_PARTITION_COUNT) as u16;

        let (_dir, store) = temp_store();
        let deps = capture_deps(CaptureTransferSink::new());
        deps.merge_tracker
            .mark_dispatched(old_partition as i32, MERGE_OFFSET + 1);
        deps.transfer_tracker
            .mark_dispatched(new_partition as i32, TRANSFER_OFFSET + 1);
        let sink: Arc<dyn MembershipSink> = Arc::new(CaptureSink::new());
        let mut queue = EvictionQueue::new();

        handle_merge(
            old_partition,
            &handle(&store),
            &catalog,
            &sink,
            &deps,
            &mut queue,
            "1",
            &PersonMergeEvent {
                team_id: TEAM,
                old_person_uuid: old_person,
                new_person_uuid: new_person,
                merged_at_ms: 2,
                schema_version: MERGE_EVENT_SCHEMA_VERSION,
            },
            MERGE_OFFSET,
        )
        .await;
        handle_apply(
            new_partition,
            &handle(&store),
            &catalog,
            &sink,
            &deps,
            &mut queue,
            "1",
            &MergeStateTransfer {
                team_id: TEAM,
                old_person_uuid: old_person,
                new_person_uuid: new_person,
                merged_at_ms: 2,
                source_partition: old_partition as i32,
                source_offset: MERGE_OFFSET,
                leaves: vec![],
                membership_registers: vec![],
                forward_hops: 0,
                person_dedup: None,
            },
            TRANSFER_OFFSET,
        )
        .await;

        assert!(
            store
                .get_tombstone(&TombstoneKey {
                    partition_id: old_partition,
                    team_id: TEAM as u64,
                    person: old_person,
                })
                .unwrap()
                .is_none(),
            "the dropped merge wrote no tombstone",
        );
        assert!(
            store
                .get_merge_applied(&MergeAppliedKey {
                    partition_id: new_partition,
                    team_id: TEAM as u64,
                    new_person,
                    source_partition: old_partition as i32,
                    source_offset: MERGE_OFFSET,
                })
                .unwrap()
                .is_none(),
            "the dropped transfer wrote no apply marker",
        );
        assert_eq!(
            deps.merge_tracker
                .committable_offsets()
                .get(&(old_partition as i32)),
            Some(&(MERGE_OFFSET + 1)),
            "the dropped merge commits past its offset",
        );
        assert_eq!(
            deps.transfer_tracker
                .committable_offsets()
                .get(&(new_partition as i32)),
            Some(&(TRANSFER_OFFSET + 1)),
            "the dropped transfer commits past its offset",
        );
    }
}