Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

114 KB
Raw Blame
mod common;

use std::num::NonZeroU32;
use std::sync::Arc;
use std::time::{Duration, Instant};

use common_ingestion_warnings::throttle::{WarningThrottle, DEFAULT_THROTTLE_PERIOD};

use dashmap::DashMap;

use common::{
    create_leader_client, create_local_kafka_producer, create_test_kafka,
    create_test_kafka_with_partitions, person_id_for_partition, seed_person, seed_test_changelog,
    served_captures, start_coordinator, start_leader_pod, start_leader_pod_with_lease_ttl,
    start_leader_with_pg_fallback, start_router, target, test_cached_person, test_recovery,
    test_store, test_warming_config, wait_for_condition, KAFKA_BOOTSTRAP, NUM_PARTITIONS,
    POLL_INTERVAL, WAIT_TIMEOUT,
};
use personhog_common::partitioning::partition_for_person;
use personhog_coordination::pod::HandoffHandler;
use personhog_coordination::strategy::StickyBalancedStrategy;
use personhog_coordination::types::ChangelogConfig;
use personhog_leader::cache::{
    CacheLookup, CachedPerson, DirtyIndex, DirtyMark, PartitionedCache, PersonCacheKey,
};
use personhog_leader::coordination::LeaderHandoffHandler;
use personhog_leader::inflight::InflightTracker;
use personhog_leader::service::{PersonHogLeaderService, PropertySizeLimits};
use personhog_leader::warming::WarmClientPools;
use personhog_leader::warnings::WarningsProducer;
use personhog_proto::personhog::leader::v1::person_hog_leader_server::PersonHogLeader;
use personhog_proto::personhog::leader::v1::person_hog_leader_server::PersonHogLeaderServer;
use personhog_proto::personhog::types::v1::{
    GetPersonRequest, Person, UpdatePersonPropertiesRequest,
};
use prost::Message;
use rdkafka::consumer::{BaseConsumer, Consumer};
use rdkafka::{ClientConfig, Message as KafkaMessage, TopicPartitionList};
use tokio::net::TcpListener;
use tokio_util::sync::CancellationToken;
use tonic::transport::Server;
use tonic::Request;

/// Wrap a request with the `x-partition` metadata the leader reads in place
/// of a body field, matching how the router forwards to the leader.
fn with_partition<T>(req: T, partition: u32) -> Request<T> {
    let mut request = Request::new(req);
    request
        .metadata_mut()
        .insert("x-partition", partition.to_string().parse().unwrap());
    request
}

/// Build a `GetPerson` request carrying the partition in `x-partition`
/// metadata, matching how the router forwards strong reads to the leader.
fn leader_get_request(team_id: i64, person_id: i64, partition: u32) -> Request<GetPersonRequest> {
    with_partition(
        GetPersonRequest {
            team_id,
            person_id,
            read_options: None,
        },
        partition,
    )
}

// ============================================================
// Test 1: Full lifecycle via etcd coordination
// ============================================================

#[tokio::test]
async fn service_accepts_requests_after_coordination_warmup() {
    let store = test_store("leader-warmup").await;
    let cancel = CancellationToken::new();

    let _topic = seed_test_changelog(&store).await;

    let _coord = start_coordinator(
        Arc::clone(&store),
        Arc::new(StickyBalancedStrategy),
        cancel.clone(),
    );
    let _router = start_router(Arc::clone(&store), "router-0", cancel.clone());

    // Single pod gets all partitions
    let pod = start_leader_pod(Arc::clone(&store), "leader-0", 1 << 20, cancel.clone()).await;

    // Wait for all partitions to be assigned
    let check_store = Arc::clone(&store);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let store = Arc::clone(&check_store);
        async move {
            let assignments = store.list_assignments().await.unwrap_or_default();
            assignments.len() == NUM_PARTITIONS as usize
                && assignments.iter().all(|a| a.owner == "leader-0")
        }
    })
    .await;

    // Wait for handoffs to complete (warmup finished)
    let check_store = Arc::clone(&store);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let store = Arc::clone(&check_store);
        async move { store.list_handoffs().await.unwrap_or_default().is_empty() }
    })
    .await;

    // Wait for all partitions to be warmed in the cache (may lag behind etcd state)
    let check_cache = Arc::clone(&pod.cache);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let cache = Arc::clone(&check_cache);
        async move { (0..NUM_PARTITIONS).all(|p| cache.has_partition(p)) }
    })
    .await;

    // Seed a person into partition 0
    let person = test_cached_person();
    seed_person(&pod.cache, 0, person.clone());

    // gRPC get_person should succeed
    let mut client = create_leader_client(pod.leader_addr).await;
    let response = client
        .get_person(leader_get_request(1, 42, 0))
        .await
        .unwrap();

    let proto_person = response.into_inner().person.unwrap();
    assert_eq!(proto_person.id, 42);
    assert_eq!(proto_person.team_id, 1);

    // Update should succeed
    let response = client
        .update_person_properties(with_partition(
            UpdatePersonPropertiesRequest {
                force_update: false,
                team_id: 1,
                person_id: 42,
                event_name: "$set".to_string(),
                set_properties: serde_json::to_vec(&serde_json::json!({"name": "Updated"}))
                    .unwrap(),
                set_once_properties: vec![],
                unset_properties: vec![],
                is_identified: None,
                last_seen_at: None,
            },
            0,
        ))
        .await
        .unwrap();

    let result = response.into_inner();
    assert!(result.updated);
    assert_eq!(result.person.unwrap().version, 2);

    // Read back should reflect the update
    let response = client
        .get_person(leader_get_request(1, 42, 0))
        .await
        .unwrap();

    let proto_person = response.into_inner().person.unwrap();
    assert_eq!(proto_person.version, 2);

    let props: serde_json::Value = serde_json::from_slice(&proto_person.properties).unwrap();
    assert_eq!(props["name"], "Updated");
    assert_eq!(props["email"], "[email protected]");

    cancel.cancel();
}

// ============================================================
// Test 2: Unowned partition returns FailedPrecondition
// (no etcd needed)
// ============================================================

#[tokio::test]
async fn unowned_partition_returns_failed_precondition() {
    // Create service + cache directly, no coordination
    let cache = Arc::new(PartitionedCache::new(1 << 20));
    let _mock_cluster = create_test_kafka().await;
    // One set for both: a second would fence the first at the broker.
    let fenced =
        common::fenced_producers_acquired(&_mock_cluster.topic, NUM_PARTITIONS as i32).await;
    let service = PersonHogLeaderService::new(
        Arc::clone(&cache),
        None,
        Arc::new(DashMap::new()),
        Arc::new(InflightTracker::new()),
        served_captures(&_mock_cluster.topic, NUM_PARTITIONS),
        Arc::new(DirtyIndex::new(1_000_000)),
        test_recovery(KAFKA_BOOTSTRAP),
        PropertySizeLimits::new(655360, 524288),
        WarningsProducer::disabled(),
        Arc::new(DashMap::new()),
        Arc::clone(&fenced),
        common::live_authority(),
        std::sync::Arc::new(personhog_leader::emitted::EmittedVersions::new(1_000_000)),
    );

    let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
    let addr = listener.local_addr().unwrap();
    let cancel = CancellationToken::new();
    let token = cancel.child_token();

    tokio::spawn(async move {
        Server::builder()
            .add_service(PersonHogLeaderServer::new(service))
            .serve_with_incoming_shutdown(
                tokio_stream::wrappers::TcpListenerStream::new(listener),
                token.cancelled(),
            )
            .await
            .unwrap();
    });
    tokio::time::sleep(std::time::Duration::from_millis(10)).await;

    let mut client = create_leader_client(addr).await;

    // Request for unowned partition → FailedPrecondition
    let result = client.get_person(leader_get_request(1, 42, 0)).await;
    assert!(result.is_err());
    assert_eq!(result.unwrap_err().code(), tonic::Code::FailedPrecondition);

    // Write to unowned partition → FailedPrecondition
    let result = client
        .update_person_properties(with_partition(
            UpdatePersonPropertiesRequest {
                force_update: false,
                team_id: 1,
                person_id: 42,
                event_name: "$set".to_string(),
                set_properties: vec![],
                set_once_properties: vec![],
                unset_properties: vec![],
                is_identified: None,
                last_seen_at: None,
            },
            0,
        ))
        .await;
    assert!(result.is_err());
    assert_eq!(result.unwrap_err().code(), tonic::Code::FailedPrecondition);

    // Manually warm partition → now NotFound (partition exists, no data)
    cache.create_partition(0);

    let result = client.get_person(leader_get_request(1, 42, 0)).await;
    assert!(result.is_err());
    assert_eq!(result.unwrap_err().code(), tonic::Code::NotFound);

    cancel.cancel();
}

// ============================================================
// Test 2b: Requests without x-partition metadata fail closed
// (no etcd needed)
// ============================================================

/// The partition arrives only via `x-partition` metadata, stamped by the
/// router. A request without it is misrouted or malformed and must be
/// rejected with INVALID_ARGUMENT rather than served against a guessed
/// partition — even when the person exists and its partition is warm.
#[tokio::test]
async fn missing_partition_metadata_returns_invalid_argument() {
    let cache = Arc::new(PartitionedCache::new(1 << 20));
    let _mock_cluster = create_test_kafka().await;
    // One set for both: a second would fence the first at the broker.
    let fenced =
        common::fenced_producers_acquired(&_mock_cluster.topic, NUM_PARTITIONS as i32).await;
    let service = PersonHogLeaderService::new(
        Arc::clone(&cache),
        None,
        Arc::new(DashMap::new()),
        Arc::new(InflightTracker::new()),
        served_captures(&_mock_cluster.topic, NUM_PARTITIONS),
        Arc::new(DirtyIndex::new(1_000_000)),
        test_recovery(KAFKA_BOOTSTRAP),
        PropertySizeLimits::new(655360, 524288),
        WarningsProducer::disabled(),
        Arc::new(DashMap::new()),
        Arc::clone(&fenced),
        common::live_authority(),
        std::sync::Arc::new(personhog_leader::emitted::EmittedVersions::new(1_000_000)),
    );

    // Warm the partition and seed the person so the only failure mode
    // left is the missing metadata itself.
    cache.create_partition(0);
    seed_person(&cache, 0, test_cached_person());

    let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
    let addr = listener.local_addr().unwrap();
    let cancel = CancellationToken::new();
    let token = cancel.child_token();

    tokio::spawn(async move {
        Server::builder()
            .add_service(PersonHogLeaderServer::new(service))
            .serve_with_incoming_shutdown(
                tokio_stream::wrappers::TcpListenerStream::new(listener),
                token.cancelled(),
            )
            .await
            .unwrap();
    });
    tokio::time::sleep(Duration::from_millis(10)).await;

    let mut client = create_leader_client(addr).await;

    let result = client
        .get_person(GetPersonRequest {
            team_id: 1,
            person_id: 42,
            read_options: None,
        })
        .await;
    let status = result.expect_err("read without x-partition must fail closed");
    assert_eq!(status.code(), tonic::Code::InvalidArgument);
    assert!(status.message().contains("x-partition"));

    let result = client
        .update_person_properties(UpdatePersonPropertiesRequest {
            force_update: false,
            team_id: 1,
            person_id: 42,
            event_name: "$set".to_string(),
            set_properties: vec![],
            set_once_properties: vec![],
            unset_properties: vec![],
            is_identified: None,
            last_seen_at: None,
        })
        .await;
    let status = result.expect_err("write without x-partition must fail closed");
    assert_eq!(status.code(), tonic::Code::InvalidArgument);
    assert!(status.message().contains("x-partition"));

    cancel.cancel();
}

// ============================================================
// Test 2c: x-partition must match the partition derived from the
// request's own key (no etcd needed)
// ============================================================

/// The leader validates the router's routing decision against the decoded
/// body: `x-partition` must equal `partition_for_person(team_id,
/// person_id)`. A mismatch means a client stamped wrong routing-key
/// headers or the hash implementations diverged — serving it would read or
/// write through the wrong partition's cache, so it must be rejected even
/// when the named partition is warm and the person exists there.
#[tokio::test]
async fn mismatched_partition_metadata_returns_failed_precondition() {
    let cache = Arc::new(PartitionedCache::new(1 << 20));
    let _mock_cluster = create_test_kafka().await;
    // One set for both: a second would fence the first at the broker.
    let fenced =
        common::fenced_producers_acquired(&_mock_cluster.topic, NUM_PARTITIONS as i32).await;
    let service = PersonHogLeaderService::new(
        Arc::clone(&cache),
        None,
        Arc::new(DashMap::new()),
        Arc::new(InflightTracker::new()),
        served_captures(&_mock_cluster.topic, NUM_PARTITIONS),
        Arc::new(DirtyIndex::new(1_000_000)),
        test_recovery(KAFKA_BOOTSTRAP),
        PropertySizeLimits::new(655360, 524288),
        WarningsProducer::disabled(),
        Arc::new(DashMap::new()),
        Arc::clone(&fenced),
        common::live_authority(),
        std::sync::Arc::new(personhog_leader::emitted::EmittedVersions::new(1_000_000)),
    );

    // Key (1, 42) hashes to some true partition; pick a different one and
    // warm + seed it so the only failure mode left is the mismatch itself.
    let true_partition = partition_for_person(1, 42, NUM_PARTITIONS);
    let wrong_partition = (true_partition + 1) % NUM_PARTITIONS;
    cache.create_partition(wrong_partition);
    seed_person(&cache, wrong_partition, test_cached_person());

    let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
    let addr = listener.local_addr().unwrap();
    let cancel = CancellationToken::new();
    let token = cancel.child_token();

    tokio::spawn(async move {
        Server::builder()
            .add_service(PersonHogLeaderServer::new(service))
            .serve_with_incoming_shutdown(
                tokio_stream::wrappers::TcpListenerStream::new(listener),
                token.cancelled(),
            )
            .await
            .unwrap();
    });
    tokio::time::sleep(Duration::from_millis(10)).await;

    let mut client = create_leader_client(addr).await;

    let result = client
        .get_person(leader_get_request(1, 42, wrong_partition))
        .await;
    let status = result.expect_err("read with mismatched x-partition must fail closed");
    assert_eq!(status.code(), tonic::Code::FailedPrecondition);
    assert!(status.message().contains("does not match"));

    let result = client
        .update_person_properties(with_partition(
            UpdatePersonPropertiesRequest {
                force_update: false,
                team_id: 1,
                person_id: 42,
                event_name: "$set".to_string(),
                set_properties: vec![],
                set_once_properties: vec![],
                unset_properties: vec![],
                is_identified: None,
                last_seen_at: None,
            },
            wrong_partition,
        ))
        .await;
    let status = result.expect_err("write with mismatched x-partition must fail closed");
    assert_eq!(status.code(), tonic::Code::FailedPrecondition);
    assert!(status.message().contains("does not match"));

    cancel.cancel();
}

// ============================================================
// Test 2b: Post-drain write fencing
// (no etcd needed)
// ============================================================

/// Once the old owner has drained a partition, every router has acked the
/// freeze — so a later write can only come from a router serving with a
/// stale table (an expired lease it hasn't noticed, a missed freeze).
/// Accepting it would produce to Kafka past the HWM that warming
/// snapshots, silently losing the write from the new owner's cache. After
/// draining, the old owner must reject writes for the partition while
/// continuing to serve reads: the frozen state remains the latest until
/// cutover. Releasing the partition clears the fence with it.
#[tokio::test]
async fn writes_fenced_after_drain_reads_still_served() {
    let cache = Arc::new(PartitionedCache::new(1 << 20));
    let _mock_cluster = create_test_kafka().await;
    let inflight = Arc::new(InflightTracker::new());
    let dirty_index = Arc::new(DirtyIndex::new(1_000_000));
    let recovery = test_recovery(KAFKA_BOOTSTRAP);
    // One set for both: a second would fence the first at the broker.
    let fenced =
        common::fenced_producers_acquired(&_mock_cluster.topic, NUM_PARTITIONS as i32).await;
    let captures = served_captures(&_mock_cluster.topic, NUM_PARTITIONS);
    let service = PersonHogLeaderService::new(
        Arc::clone(&cache),
        None,
        Arc::new(DashMap::new()),
        Arc::clone(&inflight),
        Arc::clone(&captures),
        Arc::clone(&dirty_index),
        Arc::clone(&recovery),
        PropertySizeLimits::new(655360, 524288),
        WarningsProducer::disabled(),
        Arc::new(DashMap::new()),
        Arc::clone(&fenced),
        common::live_authority(),
        std::sync::Arc::new(personhog_leader::emitted::EmittedVersions::new(1_000_000)),
    );
    // The handler shares the cache, inflight tracker, dirty index, and
    // recovery pool with the service, exactly as main.rs wires them.
    let warming = test_warming_config("fence-pod", KAFKA_BOOTSTRAP);
    let pools = Arc::new(WarmClientPools::new(
        &warming.kafka,
        "fence-pod",
        &warming.writer_consumer_group,
    ));
    let changelog = common::live_changelog(&_mock_cluster.topic);
    let handler = LeaderHandoffHandler::new(
        Arc::clone(&cache),
        Arc::clone(&inflight),
        Arc::clone(&dirty_index),
        warming,
        Arc::new(DashMap::new()),
        None,
        changelog,
        Arc::clone(&captures),
        pools,
        Arc::clone(&fenced),
        common::live_authority(),
        std::sync::Arc::new(personhog_leader::emitted::EmittedVersions::new(1_000_000)),
    );

    cache.create_partition(0);
    seed_person(&cache, 0, test_cached_person());

    let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
    let addr = listener.local_addr().unwrap();
    let cancel = CancellationToken::new();
    let token = cancel.child_token();

    tokio::spawn(async move {
        Server::builder()
            .add_service(PersonHogLeaderServer::new(service))
            .serve_with_incoming_shutdown(
                tokio_stream::wrappers::TcpListenerStream::new(listener),
                token.cancelled(),
            )
            .await
            .unwrap();
    });
    tokio::time::sleep(Duration::from_millis(10)).await;

    let mut client = create_leader_client(addr).await;

    let update = |email: &str| {
        with_partition(
            UpdatePersonPropertiesRequest {
                force_update: false,
                team_id: 1,
                person_id: 42,
                event_name: "$set".to_string(),
                set_properties: serde_json::to_vec(&serde_json::json!({ "email": email })).unwrap(),
                set_once_properties: vec![],
                unset_properties: vec![],
                is_identified: None,
                last_seen_at: None,
            },
            0,
        )
    };

    // Pre-drain: writes flow normally.
    let response = client
        .update_person_properties(update("[email protected]"))
        .await
        .unwrap();
    assert!(response.into_inner().updated);

    // Drain the partition, as the pod does on observing Draining.
    handler.drain_partition_inflight(0).await.unwrap();

    // Post-drain: writes must be fenced…
    let status = client
        .update_person_properties(update("[email protected]"))
        .await
        .expect_err("write after drain must be rejected");
    assert_eq!(status.code(), tonic::Code::FailedPrecondition);

    // …while reads keep serving the frozen, still-latest state.
    let response = client
        .get_person(leader_get_request(1, 42, 0))
        .await
        .expect("reads must keep serving after drain");
    let props: serde_json::Value =
        serde_json::from_slice(&response.into_inner().person.unwrap().properties).unwrap();
    assert_eq!(props["email"], "[email protected]");

    // Releasing clears the fence along with the partition. The warm is
    // stood in for, epoch included, since warming needs a changelog this
    // test never writes.
    handler.release_partition(0).await.unwrap();
    handler
        .capture_changelog(
            0,
            &ChangelogConfig::new(&_mock_cluster.topic, NUM_PARTITIONS),
        )
        .await;
    cache.create_partition(0);
    seed_person(&cache, 0, test_cached_person());
    fenced
        .acquire(0, &target(&_mock_cluster.topic))
        .await
        .expect("the re-warm takes the epoch");
    let response = client
        .update_person_properties(update("[email protected]"))
        .await
        .expect("writes must flow again after release + re-warm");
    assert!(response.into_inner().updated);

    cancel.cancel();
}

/// The fence must go up before the drain starts waiting on inflight
/// handlers — fencing only after `wait_until_empty` returns would leave a
/// window where a write lands between the inflight count hitting zero and
/// the DrainedAck being written, advancing the Kafka HWM past what
/// warming will read. With an inflight write held open, new writes must
/// already be rejected while the drain is still waiting.
#[tokio::test]
async fn drain_fences_before_waiting_on_inflight() {
    let cache = Arc::new(PartitionedCache::new(1 << 20));
    let _mock_cluster = create_test_kafka().await;
    let inflight = Arc::new(InflightTracker::new());
    let dirty_index = Arc::new(DirtyIndex::new(1_000_000));
    let recovery = test_recovery(KAFKA_BOOTSTRAP);
    // One set for both: a second would fence the first at the broker.
    let fenced =
        common::fenced_producers_acquired(&_mock_cluster.topic, NUM_PARTITIONS as i32).await;
    let captures = served_captures(&_mock_cluster.topic, NUM_PARTITIONS);
    let service = PersonHogLeaderService::new(
        Arc::clone(&cache),
        None,
        Arc::new(DashMap::new()),
        Arc::clone(&inflight),
        Arc::clone(&captures),
        Arc::clone(&dirty_index),
        Arc::clone(&recovery),
        PropertySizeLimits::new(655360, 524288),
        WarningsProducer::disabled(),
        Arc::new(DashMap::new()),
        Arc::clone(&fenced),
        common::live_authority(),
        std::sync::Arc::new(personhog_leader::emitted::EmittedVersions::new(1_000_000)),
    );
    let warming = test_warming_config("fence-race-pod", KAFKA_BOOTSTRAP);
    let pools = Arc::new(WarmClientPools::new(
        &warming.kafka,
        "fence-race-pod",
        &warming.writer_consumer_group,
    ));
    let changelog = common::live_changelog(&_mock_cluster.topic);
    let handler = Arc::new(LeaderHandoffHandler::new(
        Arc::clone(&cache),
        Arc::clone(&inflight),
        Arc::clone(&dirty_index),
        warming,
        Arc::new(DashMap::new()),
        None,
        changelog,
        Arc::clone(&captures),
        pools,
        Arc::clone(&fenced),
        common::live_authority(),
        std::sync::Arc::new(personhog_leader::emitted::EmittedVersions::new(1_000_000)),
    ));

    cache.create_partition(0);
    seed_person(&cache, 0, test_cached_person());

    let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
    let addr = listener.local_addr().unwrap();
    let cancel = CancellationToken::new();
    let token = cancel.child_token();

    tokio::spawn(async move {
        Server::builder()
            .add_service(PersonHogLeaderServer::new(service))
            .serve_with_incoming_shutdown(
                tokio_stream::wrappers::TcpListenerStream::new(listener),
                token.cancelled(),
            )
            .await
            .unwrap();
    });
    tokio::time::sleep(Duration::from_millis(10)).await;

    let mut client = create_leader_client(addr).await;

    // Hold a simulated in-flight write so the drain parks in
    // wait_until_empty rather than completing immediately.
    let guard = inflight.begin(0);

    let drain_handler = Arc::clone(&handler);
    let drain = tokio::spawn(async move { drain_handler.drain_partition_inflight(0).await });

    // Give the drain a moment to start waiting.
    tokio::time::sleep(Duration::from_millis(100)).await;
    assert!(
        !drain.is_finished(),
        "drain must wait for the held inflight"
    );

    // A write arriving mid-drain must already be fenced.
    let status = client
        .update_person_properties(with_partition(
            UpdatePersonPropertiesRequest {
                force_update: false,
                team_id: 1,
                person_id: 42,
                event_name: "$set".to_string(),
                set_properties: serde_json::to_vec(
                    &serde_json::json!({"email": "[email protected]"}),
                )
                .unwrap(),
                set_once_properties: vec![],
                unset_properties: vec![],
                is_identified: None,
                last_seen_at: None,
            },
            0,
        ))
        .await
        .expect_err("write during drain must be rejected");
    assert_eq!(status.code(), tonic::Code::FailedPrecondition);

    // Releasing the held write lets the drain complete.
    drop(guard);
    tokio::time::timeout(Duration::from_secs(5), drain)
        .await
        .expect("drain must complete once inflight reaches zero")
        .expect("drain task must not panic")
        .expect("drain must succeed");

    cancel.cancel();
}

// ============================================================
// Test 3: Release partition stops serving
// ============================================================

#[tokio::test]
async fn release_partition_stops_serving() {
    let store = test_store("leader-release").await;
    let cancel = CancellationToken::new();

    let _topic = seed_test_changelog(&store).await;

    let _coord = start_coordinator(
        Arc::clone(&store),
        Arc::new(StickyBalancedStrategy),
        cancel.clone(),
    );
    let _router = start_router(Arc::clone(&store), "router-0", cancel.clone());

    // Start pod 1 — gets all partitions
    let pod1 = start_leader_pod(Arc::clone(&store), "leader-0", 1 << 20, cancel.clone()).await;

    let check_store = Arc::clone(&store);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let store = Arc::clone(&check_store);
        async move {
            let assignments = store.list_assignments().await.unwrap_or_default();
            let handoffs = store.list_handoffs().await.unwrap_or_default();
            assignments.len() == NUM_PARTITIONS as usize
                && assignments.iter().all(|a| a.owner == "leader-0")
                && handoffs.is_empty()
        }
    })
    .await;

    // Wait for cache to reflect assignments (may lag behind etcd state)
    let check_cache = Arc::clone(&pod1.cache);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let cache = Arc::clone(&check_cache);
        async move { (0..NUM_PARTITIONS).all(|p| cache.has_partition(p)) }
    })
    .await;

    // Seed a person into partition 0
    seed_person(&pod1.cache, 0, test_cached_person());

    // Verify get_person works on pod 1
    let mut client1 = create_leader_client(pod1.leader_addr).await;
    let response = client1
        .get_person(leader_get_request(1, 42, 0))
        .await
        .unwrap();
    assert_eq!(response.into_inner().person.unwrap().id, 42);

    // Start pod 2 — triggers rebalance
    let pod2 = start_leader_pod(Arc::clone(&store), "leader-1", 1 << 20, cancel.clone()).await;

    // Wait for balanced assignment and handoffs to settle
    let check_store = Arc::clone(&store);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let store = Arc::clone(&check_store);
        async move {
            let assignments = store.list_assignments().await.unwrap_or_default();
            let handoffs = store.list_handoffs().await.unwrap_or_default();
            let has_both = assignments.iter().any(|a| a.owner == "leader-0")
                && assignments.iter().any(|a| a.owner == "leader-1");
            assignments.len() == NUM_PARTITIONS as usize && has_both && handoffs.is_empty()
        }
    })
    .await;

    // Find a partition that moved from pod 1 to pod 2
    let assignments = store.list_assignments().await.unwrap();
    let moved_partition = assignments
        .iter()
        .find(|a| a.owner == "leader-1")
        .expect("pod 2 should own at least one partition")
        .partition;

    // Wait for caches to reflect the rebalance (may lag behind etcd state)
    let check_cache1 = Arc::clone(&pod1.cache);
    let check_cache2 = Arc::clone(&pod2.cache);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let c1 = Arc::clone(&check_cache1);
        let c2 = Arc::clone(&check_cache2);
        async move { !c1.has_partition(moved_partition) && c2.has_partition(moved_partition) }
    })
    .await;

    // A key that actually hashes to the moved partition, so the request
    // passes the leader's partition validation.
    let moved_person_id = person_id_for_partition(1, moved_partition);

    // Pod 1: released partition → FailedPrecondition
    let result = client1
        .get_person(leader_get_request(1, moved_person_id, moved_partition))
        .await;
    assert!(result.is_err());
    assert_eq!(
        result.unwrap_err().code(),
        tonic::Code::FailedPrecondition,
        "pod 1 should reject requests for released partition {moved_partition}"
    );

    // Pod 2: warmed partition but empty cache → NotFound
    let mut client2 = create_leader_client(pod2.leader_addr).await;
    let result = client2
        .get_person(leader_get_request(1, moved_person_id, moved_partition))
        .await;
    assert!(result.is_err());
    assert_eq!(
        result.unwrap_err().code(),
        tonic::Code::NotFound,
        "pod 2 should have partition {moved_partition} warmed but no data"
    );

    cancel.cancel();
}

// ============================================================
// Test 4: Rewarm after pod crash
// ============================================================

#[tokio::test]
async fn rewarm_after_pod_crash() {
    let store = test_store("leader-rewarm").await;
    let cancel = CancellationToken::new();

    let _topic = seed_test_changelog(&store).await;

    let _coord = start_coordinator(
        Arc::clone(&store),
        Arc::new(StickyBalancedStrategy),
        cancel.clone(),
    );
    let _router = start_router(Arc::clone(&store), "router-0", cancel.clone());

    // Pod 1 with short lease (crash detectable quickly)
    let pod1_cancel = CancellationToken::new();
    let _pod1 = start_leader_pod_with_lease_ttl(
        Arc::clone(&store),
        "leader-0",
        1 << 20,
        2,
        pod1_cancel.clone(),
    )
    .await;

    // Pod 2 (long-lived)
    let pod2 = start_leader_pod(Arc::clone(&store), "leader-1", 1 << 20, cancel.clone()).await;

    // Wait for balanced assignment
    let check_store = Arc::clone(&store);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let store = Arc::clone(&check_store);
        async move {
            let assignments = store.list_assignments().await.unwrap_or_default();
            let handoffs = store.list_handoffs().await.unwrap_or_default();
            let has_both = assignments.iter().any(|a| a.owner == "leader-0")
                && assignments.iter().any(|a| a.owner == "leader-1");
            assignments.len() == NUM_PARTITIONS as usize && has_both && handoffs.is_empty()
        }
    })
    .await;

    // Kill pod 1
    pod1_cancel.cancel();

    // Wait for pod 1 to disappear and all partitions to move to pod 2
    let check_store = Arc::clone(&store);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let store = Arc::clone(&check_store);
        async move {
            let pods = store.list_pods().await.unwrap_or_default();
            let assignments = store.list_assignments().await.unwrap_or_default();
            let handoffs = store.list_handoffs().await.unwrap_or_default();
            pods.len() == 1
                && assignments.len() == NUM_PARTITIONS as usize
                && assignments.iter().all(|a| a.owner == "leader-1")
                && handoffs.is_empty()
        }
    })
    .await;

    // Wait for pod 2 to re-warm all partitions (may lag behind etcd state)
    let check_cache = Arc::clone(&pod2.cache);
    wait_for_condition(WAIT_TIMEOUT, POLL_INTERVAL, || {
        let cache = Arc::clone(&check_cache);
        async move { (0..NUM_PARTITIONS).all(|p| cache.has_partition(p)) }
    })
    .await;

    // Partitions are warm but empty → NotFound
    let mut client2 = create_leader_client(pod2.leader_addr).await;
    let result = client2.get_person(leader_get_request(1, 42, 0)).await;
    assert!(result.is_err());
    assert_eq!(result.unwrap_err().code(), tonic::Code::NotFound);

    cancel.cancel();
}

// ============================================================
// Test 5: Successful update produces person state to Kafka
// (no etcd needed)
// ============================================================

#[tokio::test]
async fn update_produces_person_state_to_kafka() {
    // The changelog must land on the exact Kafka partition the request's
    // `x-partition` named — warming rebuilds a routing partition's cache by
    // consuming the same-numbered Kafka partition, so key-hash placement
    // (whose partitioner config could diverge from the router's murmur2)
    // is not acceptable. The key (team 1, person 2) murmur2-hashes to
    // partition 2 (so it passes the leader's partition validation) but
    // librdkafka's default crc32 partitioner would place it on partition 0,
    // keeping misplacement observable: the consumer below reads only
    // partition 2 and must find the message there.
    const PERSON_ID: i64 = 2;
    let routing_partition: u32 = partition_for_person(1, PERSON_ID, NUM_PARTITIONS);
    assert_eq!(routing_partition, 2, "test key must map to partition 2");
    let mock_cluster = create_test_kafka_with_partitions(4).await;

    let cache = Arc::new(PartitionedCache::new(1 << 20));
    let service = PersonHogLeaderService::new(
        Arc::clone(&cache),
        None,
        Arc::new(DashMap::new()),
        Arc::new(InflightTracker::new()),
        served_captures(&mock_cluster.topic, NUM_PARTITIONS),
        Arc::new(DirtyIndex::new(1_000_000)),
        test_recovery(KAFKA_BOOTSTRAP),
        PropertySizeLimits::new(655360, 524288),
        WarningsProducer::disabled(),
        Arc::new(DashMap::new()),
        common::fenced_producers_acquired(&mock_cluster.topic, 4).await,
        common::live_authority(),
        std::sync::Arc::new(personhog_leader::emitted::EmittedVersions::new(1_000_000)),
    );

    cache.create_partition(routing_partition);
    let person = CachedPerson {
        id: PERSON_ID,
        ..test_cached_person()
    };
    seed_person(&cache, routing_partition, person);

    let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
    let addr = listener.local_addr().unwrap();
    let cancel = CancellationToken::new();
    let token = cancel.child_token();

    tokio::spawn(async move {
        Server::builder()
            .add_service(PersonHogLeaderServer::new(service))
            .serve_with_incoming_shutdown(
                tokio_stream::wrappers::TcpListenerStream::new(listener),
                token.cancelled(),
            )
            .await
            .unwrap();
    });
    tokio::time::sleep(Duration::from_millis(10)).await;

    let mut client = create_leader_client(addr).await;

    // Perform an update
    let response = client
        .update_person_properties(with_partition(
            UpdatePersonPropertiesRequest {
                force_update: false,
                team_id: 1,
                person_id: PERSON_ID,
                event_name: "$set".to_string(),
                set_properties: serde_json::to_vec(&serde_json::json!({"name": "Kafka Test"}))
                    .unwrap(),
                set_once_properties: vec![],
                unset_properties: vec![],
                is_identified: None,
                last_seen_at: None,
            },
            routing_partition,
        ))
        .await
        .unwrap();

    let result = response.into_inner();
    assert!(result.updated);
    assert_eq!(result.person.unwrap().version, 2);

    // Consume only the routing partition — finding the message here proves
    // the explicit-partition produce, not just delivery.
    let consumer: BaseConsumer = ClientConfig::new()
        .set("bootstrap.servers", mock_cluster.bootstrap_servers())
        .set("group.id", "test-consumer")
        .create()
        .expect("failed to create consumer");

    let mut tpl = TopicPartitionList::new();
    tpl.add_partition_offset(
        &mock_cluster.topic,
        routing_partition as i32,
        rdkafka::Offset::Beginning,
    )
    .unwrap();
    consumer.assign(&tpl).unwrap();

    let msg = common::poll_record(&consumer, Duration::from_secs(5), "the routing partition");

    // Verify message key
    let key = std::str::from_utf8(msg.key().unwrap()).unwrap();
    assert_eq!(key, "1:2");

    // Verify payload is a valid Person proto with updated state
    let person = Person::decode(msg.payload().unwrap()).unwrap();
    assert_eq!(person.id, PERSON_ID);
    assert_eq!(person.team_id, 1);
    assert_eq!(person.version, 2);

    let props: serde_json::Value = serde_json::from_slice(&person.properties).unwrap();
    assert_eq!(props["name"], "Kafka Test");
    assert_eq!(props["email"], "[email protected]");

    cancel.cancel();
}

// ============================================================
// Test 6: a changelog write this pod cannot land leaves the cache
// unchanged and returns an error (no etcd needed)
// ============================================================

#[tokio::test]
async fn kafka_produce_failure_leaves_cache_unchanged() {
    let cache = Arc::new(PartitionedCache::new(1 << 20));
    let mock_cluster = create_test_kafka().await;
    let fenced =
        common::fenced_producers_acquired(&mock_cluster.topic, NUM_PARTITIONS as i32).await;

    let service = PersonHogLeaderService::new(
        Arc::clone(&cache),
        None,
        Arc::new(DashMap::new()),
        Arc::new(InflightTracker::new()),
        served_captures(&mock_cluster.topic, NUM_PARTITIONS),
        Arc::new(DirtyIndex::new(1_000_000)),
        test_recovery(KAFKA_BOOTSTRAP),
        PropertySizeLimits::new(655360, 524288),
        WarningsProducer::disabled(),
        Arc::new(DashMap::new()),
        Arc::clone(&fenced),
        common::live_authority(),
        std::sync::Arc::new(personhog_leader::emitted::EmittedVersions::new(1_000_000)),
    );

    cache.create_partition(0);
    let person = test_cached_person();
    seed_person(&cache, 0, person);

    let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
    let addr = listener.local_addr().unwrap();
    let cancel = CancellationToken::new();
    let token = cancel.child_token();

    tokio::spawn(async move {
        Server::builder()
            .add_service(PersonHogLeaderServer::new(service))
            .serve_with_incoming_shutdown(
                tokio_stream::wrappers::TcpListenerStream::new(listener),
                token.cancelled(),
            )
            .await
            .unwrap();
    });
    tokio::time::sleep(Duration::from_millis(10)).await;

    // A successor takes the partition's transactional id, which is what
    // a changelog write this pod cannot land looks like.
    let successor =
        common::fenced_producers_acquired(&mock_cluster.topic, NUM_PARTITIONS as i32).await;

    let mut client = create_leader_client(addr).await;

    // Update should fail because Kafka produce fails
    let result = client
        .update_person_properties(with_partition(
            UpdatePersonPropertiesRequest {
                force_update: false,
                team_id: 1,
                person_id: 42,
                event_name: "$set".to_string(),
                set_properties: serde_json::to_vec(&serde_json::json!({"name": "Should Rollback"}))
                    .unwrap(),
                set_once_properties: vec![],
                unset_properties: vec![],
                is_identified: None,
                last_seen_at: None,
            },
            0,
        ))
        .await;

    assert!(result.is_err());
    assert_eq!(
        result.unwrap_err().code(),
        tonic::Code::FailedPrecondition,
        "a fenced pod answers in the ownership vocabulary, so the router re-resolves"
    );

    // Cache was never updated since the produce failed before the cache write
    let cache_key = PersonCacheKey {
        team_id: 1,
        person_id: 42,
    };
    let CacheLookup::Found(cached) = cache.get(0, &cache_key) else {
        panic!("expected original person in cache");
    };
    assert_eq!(cached.version, 1);
    assert_eq!(
        cached.parse_properties().unwrap()["email"],
        "[email protected]"
    );
    assert!(cached.parse_properties().unwrap().get("name").is_none());

    // This pod re-takes the epoch, as the convergence heal does.
    drop(successor);
    fenced.expect_self_fence(0);
    fenced
        .acquire(0, &target(&mock_cluster.topic))
        .await
        .expect("this pod re-takes the fence");

    let response = client
        .update_person_properties(with_partition(
            UpdatePersonPropertiesRequest {
                force_update: false,
                team_id: 1,
                person_id: 42,
                event_name: "$set".to_string(),
                set_properties: serde_json::to_vec(&serde_json::json!({"name": "After Recovery"}))
                    .unwrap(),
                set_once_properties: vec![],
                unset_properties: vec![],
                is_identified: None,
                last_seen_at: None,
            },
            0,
        ))
        .await
        .unwrap();

    let result = response.into_inner();
    assert!(result.updated);
    // Two, not three: a record the broker refused never existed, so its
    // version is free again.
    assert_eq!(
        result.person.unwrap().version,
        2,
        "a version the broker refused is not spent"
    );

    cancel.cancel();
}

// ============================================================
// Test 7: E2E - update produces to local Kafka and is consumable
// Requires local Kafka (127.0.0.1:9092) and etcd running.
// ============================================================

#[tokio::test]
async fn e2e_update_produces_to_local_kafka() {
    let owned = common::owned_broker_topic(NUM_PARTITIONS as i32).await;
    let topic = owned.topic.clone();
    let cache = Arc::new(PartitionedCache::new(1 << 20));

    let service = PersonHogLeaderService::new(
        Arc::clone(&cache),
        None,
        Arc::new(DashMap::new()),
        Arc::new(InflightTracker::new()),
        served_captures(&topic, NUM_PARTITIONS),
        Arc::new(DirtyIndex::new(1_000_000)),
        test_recovery(KAFKA_BOOTSTRAP),
        PropertySizeLimits::new(655360, 524288),
        WarningsProducer::disabled(),
        Arc::new(DashMap::new()),
        common::fenced_producers_acquired(&topic, NUM_PARTITIONS as i32).await,
        common::live_authority(),
        std::sync::Arc::new(personhog_leader::emitted::EmittedVersions::new(1_000_000)),
    );

    cache.create_partition(0);
    let person = test_cached_person();
    seed_person(&cache, 0, person);

    let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
    let addr = listener.local_addr().unwrap();
    let cancel = CancellationToken::new();
    let token = cancel.child_token();

    tokio::spawn(async move {
        Server::builder()
            .add_service(PersonHogLeaderServer::new(service))
            .serve_with_incoming_shutdown(
                tokio_stream::wrappers::TcpListenerStream::new(listener),
                token.cancelled(),
            )
            .await
            .unwrap();
    });
    tokio::time::sleep(Duration::from_millis(10)).await;

    let mut client = create_leader_client(addr).await;

    // Perform an update via gRPC
    let response = client
        .update_person_properties(with_partition(
            UpdatePersonPropertiesRequest {
                force_update: false,
                team_id: 1,
                person_id: 42,
                event_name: "$set".to_string(),
                set_properties: serde_json::to_vec(&serde_json::json!({"name": "E2E Test"}))
                    .unwrap(),
                set_once_properties: vec![],
                unset_properties: vec![],
                is_identified: None,
                last_seen_at: None,
            },
            0,
        ))
        .await
        .unwrap();

    let result = response.into_inner();
    assert!(result.updated);
    assert_eq!(result.person.unwrap().version, 2);

    // Consume from local Kafka and verify the message
    let consumer: BaseConsumer = ClientConfig::new()
        .set("bootstrap.servers", KAFKA_BOOTSTRAP)
        .set("group.id", format!("e2e-test-{}", uuid::Uuid::new_v4()))
        .create()
        .expect("failed to create local Kafka consumer");

    // From the beginning: counting back from the end lands on the
    // transaction's commit marker, which no consumer is handed.
    let mut tpl = TopicPartitionList::new();
    tpl.add_partition_offset(&topic, 0, rdkafka::Offset::Beginning)
        .unwrap();
    consumer.assign(&tpl).unwrap();

    let msg = common::poll_record(&consumer, Duration::from_secs(10), "local Kafka");

    let key = std::str::from_utf8(msg.key().unwrap()).unwrap();
    assert_eq!(key, "1:42");

    let person = Person::decode(msg.payload().unwrap()).unwrap();
    assert_eq!(person.id, 42);
    assert_eq!(person.team_id, 1);
    assert_eq!(person.version, 2);

    let props: serde_json::Value = serde_json::from_slice(&person.properties).unwrap();
    assert_eq!(props["name"], "E2E Test");
    assert_eq!(props["email"], "[email protected]");

    cancel.cancel();
}

// ============================================================
// Test 8: PG fallback on cache miss
// Requires local Postgres.
// ============================================================

#[tokio::test]
async fn pg_fallback_loads_person_on_cache_miss() {
    let cancel = CancellationToken::new();
    let (addr, cache, _mock_cluster) = start_leader_with_pg_fallback(cancel.clone()).await;

    // Seed a person under this test's own team id. Sampling an arbitrary
    // existing row instead would race concurrently-running tests that
    // delete their transient rows between our sample and the fallback read.
    let pool = common::create_persons_pool().await;
    let team_id: i32 = 99_061;
    sqlx::query("DELETE FROM posthog_person WHERE team_id = $1")
        .bind(team_id)
        .execute(&pool)
        .await
        .unwrap();
    let (person_id,): (i64,) = sqlx::query_as(
        "INSERT INTO posthog_person (created_at, properties, is_identified, uuid, version, team_id)
         VALUES (now(), '{\"plan\": \"pro\"}'::jsonb, false, gen_random_uuid(), 1, $1)
         RETURNING id",
    )
    .bind(team_id)
    .fetch_one(&pool)
    .await
    .unwrap();

    // Warm the key's own partition (the cache is empty — no persons seeded)
    let partition = partition_for_person(team_id as i64, person_id, NUM_PARTITIONS);
    cache.create_partition(partition);

    let mut client = create_leader_client(addr).await;

    // First call: cache miss → PG fallback → loads and caches the person
    let response = client
        .get_person(leader_get_request(team_id as i64, person_id, partition))
        .await
        .unwrap();

    let person = response.into_inner().person.unwrap();
    assert_eq!(person.id, person_id);
    assert_eq!(person.team_id, team_id as i64);

    // Verify person is now cached
    let key = PersonCacheKey {
        team_id: team_id as i64,
        person_id,
    };
    assert!(
        matches!(cache.get(partition, &key), CacheLookup::Found(_)),
        "person should be cached after PG fallback"
    );

    sqlx::query("DELETE FROM posthog_person WHERE team_id = $1")
        .bind(team_id)
        .execute(&pool)
        .await
        .unwrap();
    cancel.cancel();
}

/// Rows written by other services can hold numerics whose PG-expanded
/// rendering serde_json rejects even though JSON.parse reads them fine
/// (JS `Number.MAX_VALUE` is the canonical case — a common "unlimited"
/// sentinel). The fallback must load such rows the way JS would — rounding
/// representable values, clamping beyond-f64 garbage — never panic or
/// leave the person permanently unloadable.
#[tokio::test]
async fn pg_fallback_reads_numerics_the_leaders_parser_rejects() {
    let pool = common::create_persons_pool().await;
    let team_id: i32 = 99_060;
    sqlx::query("DELETE FROM posthog_person WHERE team_id = $1")
        .bind(team_id)
        .execute(&pool)
        .await
        .unwrap();

    // Written exactly as Node's pg driver would: e-notation literals that
    // PG stores as full-precision numerics and renders back expanded.
    let row: (i64,) = sqlx::query_as(
        "INSERT INTO posthog_person (created_at, properties, is_identified, uuid, version, team_id)
         VALUES (now(),
                 '{\"credits\": 1.7976931348623157e+308, \"overflow\": 2e308, \"plan\": \"pro\"}'::jsonb,
                 false, gen_random_uuid(), 1, $1)
         RETURNING id",
    )
    .bind(team_id)
    .fetch_one(&pool)
    .await
    .unwrap();

    let key = PersonCacheKey {
        team_id: team_id as i64,
        person_id: row.0,
    };
    let person = personhog_leader::pg::load_person_from_pg(&pool, "posthog_person", &key)
        .await
        .expect("load must not fail")
        .expect("person exists");

    // The representable sentinel reads as the exact double JS reads;
    // beyond-f64 garbage clamps instead of poisoning the row; neighbors
    // are untouched.
    assert_eq!(
        person.parse_properties().unwrap()["credits"]
            .as_f64()
            .unwrap(),
        f64::MAX
    );
    assert_eq!(
        person.parse_properties().unwrap()["overflow"]
            .as_f64()
            .unwrap(),
        1e307
    );
    assert_eq!(person.parse_properties().unwrap()["plan"], "pro");

    sqlx::query("DELETE FROM posthog_person WHERE team_id = $1")
        .bind(team_id)
        .execute(&pool)
        .await
        .unwrap();
}

// ============================================================
// Test 9: PG fallback returns NotFound for non-existent person
// ============================================================

#[tokio::test]
async fn pg_fallback_returns_not_found_for_missing_person() {
    let cancel = CancellationToken::new();
    let (addr, cache, _mock_cluster) = start_leader_with_pg_fallback(cancel.clone()).await;

    let partition = partition_for_person(99999, 99999999, NUM_PARTITIONS);
    cache.create_partition(partition);

    let mut client = create_leader_client(addr).await;

    // Query a person that doesn't exist in PG
    let result = client
        .get_person(leader_get_request(99999, 99999999, partition))
        .await;

    assert!(result.is_err());
    assert_eq!(result.unwrap_err().code(), tonic::Code::NotFound);

    cancel.cancel();
}

// ============================================================
// Test 10: Update triggers PG fallback on cache miss then applies changes
// ============================================================

#[tokio::test]
async fn update_triggers_pg_fallback_then_applies_changes() {
    let cancel = CancellationToken::new();
    let (addr, cache, _mock_cluster) = start_leader_with_pg_fallback(cancel.clone()).await;

    // Seed a person under this test's own team id (see the cache-miss
    // test above for why sampling an arbitrary row is not safe).
    let pool = common::create_persons_pool().await;
    let team_id: i32 = 99_062;
    sqlx::query("DELETE FROM posthog_person WHERE team_id = $1")
        .bind(team_id)
        .execute(&pool)
        .await
        .unwrap();
    let (person_id,): (i64,) = sqlx::query_as(
        "INSERT INTO posthog_person (created_at, properties, is_identified, uuid, version, team_id)
         VALUES (now(), '{\"plan\": \"pro\"}'::jsonb, false, gen_random_uuid(), 1, $1)
         RETURNING id",
    )
    .bind(team_id)
    .fetch_one(&pool)
    .await
    .unwrap();

    let partition = partition_for_person(team_id as i64, person_id, NUM_PARTITIONS);
    cache.create_partition(partition);

    let mut client = create_leader_client(addr).await;

    // Update a person not in cache — should load from PG then apply
    let response = client
        .update_person_properties(with_partition(
            UpdatePersonPropertiesRequest {
                force_update: false,
                team_id: team_id as i64,
                person_id,
                event_name: "$set".to_string(),
                set_properties: serde_json::to_vec(&serde_json::json!({
                    "pg_fallback_test": "it_works"
                }))
                .unwrap(),
                set_once_properties: vec![],
                unset_properties: vec![],
                is_identified: None,
                last_seen_at: None,
            },
            partition,
        ))
        .await
        .unwrap();

    let result = response.into_inner();
    assert!(result.updated);

    let updated_person = result.person.unwrap();
    assert_eq!(updated_person.id, person_id);
    let props: serde_json::Value = serde_json::from_slice(&updated_person.properties).unwrap();
    assert_eq!(props["pg_fallback_test"], "it_works");

    sqlx::query("DELETE FROM posthog_person WHERE team_id = $1")
        .bind(team_id)
        .execute(&pool)
        .await
        .unwrap();
    cancel.cancel();
}

// ============================================================
// Dirty-index recovery: eviction of a person whose latest state
// exists only in the changelog must recover from Kafka, never PG
// ============================================================

#[tokio::test]
async fn evicted_dirty_person_recovers_from_changelog() {
    const PERSON_ID: i64 = 2;
    let routing_partition: u32 = partition_for_person(1, PERSON_ID, NUM_PARTITIONS);
    let mock_cluster = create_test_kafka_with_partitions(4).await;

    let cache = Arc::new(PartitionedCache::new(1 << 20));
    let dirty_index = Arc::new(DirtyIndex::new(1_000_000));
    // Recovery reads from the same mock broker the update produces to.
    // Deliberately no PG pool: a recovery path that (wrongly) fell back to
    // PG would return NotFound and fail the final assertion.
    let recovery = test_recovery(&mock_cluster.bootstrap_servers());
    let service = PersonHogLeaderService::new(
        Arc::clone(&cache),
        None,
        Arc::new(DashMap::new()),
        Arc::new(InflightTracker::new()),
        served_captures(&mock_cluster.topic, NUM_PARTITIONS),
        Arc::clone(&dirty_index),
        Arc::clone(&recovery),
        PropertySizeLimits::new(655360, 524288),
        WarningsProducer::disabled(),
        Arc::new(DashMap::new()),
        common::fenced_producers_acquired(&mock_cluster.topic, 4).await,
        common::live_authority(),
        std::sync::Arc::new(personhog_leader::emitted::EmittedVersions::new(1_000_000)),
    );

    cache.create_partition(routing_partition);
    let person = CachedPerson {
        id: PERSON_ID,
        ..test_cached_person()
    };
    seed_person(&cache, routing_partition, person);

    let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
    let addr = listener.local_addr().unwrap();
    let cancel = CancellationToken::new();
    let token = cancel.child_token();
    tokio::spawn(async move {
        Server::builder()
            .add_service(PersonHogLeaderServer::new(service))
            .serve_with_incoming_shutdown(
                tokio_stream::wrappers::TcpListenerStream::new(listener),
                token.cancelled(),
            )
            .await
            .unwrap();
    });
    tokio::time::sleep(Duration::from_millis(10)).await;
    let mut client = create_leader_client(addr).await;

    // The update produces the v2 record and marks the dirty index.
    client
        .update_person_properties(with_partition(
            UpdatePersonPropertiesRequest {
                force_update: false,
                team_id: 1,
                person_id: PERSON_ID,
                event_name: "$set".to_string(),
                set_properties: serde_json::to_vec(&serde_json::json!({"name": "Recovered"}))
                    .unwrap(),
                set_once_properties: vec![],
                unset_properties: vec![],
                is_identified: None,
                last_seen_at: None,
            },
            routing_partition,
        ))
        .await
        .unwrap();
    assert_eq!(dirty_index.len(), 1, "the acked write must be marked dirty");

    // Simulate eviction of the freshly written entry.
    let key = PersonCacheKey {
        team_id: 1,
        person_id: PERSON_ID,
    };
    cache.remove(routing_partition, &key);

    // The read must return the updated state, recovered from the changelog
    // record at the marked offset.
    let response = client
        .get_person(with_partition(
            GetPersonRequest {
                team_id: 1,
                person_id: PERSON_ID,
                read_options: None,
            },
            routing_partition,
        ))
        .await
        .unwrap();
    let person = response.into_inner().person.unwrap();
    assert_eq!(person.version, 2);
    let props: serde_json::Value = serde_json::from_slice(&person.properties).unwrap();
    assert_eq!(props["name"], "Recovered");

    cancel.cancel();
}

// ============================================================
// Dirty-index recovery failure: when the changelog fetch cannot
// complete, the person is unavailable — not silently stale
// ============================================================

#[tokio::test]
async fn dirty_person_with_failed_recovery_is_unavailable_not_stale() {
    const PERSON_ID: i64 = 2;
    let routing_partition: u32 = partition_for_person(1, PERSON_ID, NUM_PARTITIONS);
    let mock_cluster = create_test_kafka_with_partitions(4).await;

    let cache = Arc::new(PartitionedCache::new(1 << 20));
    let dirty_index = Arc::new(DirtyIndex::new(1_000_000));
    let recovery = test_recovery(&mock_cluster.bootstrap_servers());
    let service = PersonHogLeaderService::new(
        Arc::clone(&cache),
        None,
        Arc::new(DashMap::new()),
        Arc::new(InflightTracker::new()),
        served_captures(&mock_cluster.topic, NUM_PARTITIONS),
        Arc::clone(&dirty_index),
        Arc::clone(&recovery),
        PropertySizeLimits::new(655360, 524288),
        WarningsProducer::disabled(),
        Arc::new(DashMap::new()),
        common::fenced_producers_acquired(&mock_cluster.topic, 4).await,
        common::live_authority(),
        std::sync::Arc::new(personhog_leader::emitted::EmittedVersions::new(1_000_000)),
    );

    cache.create_partition(routing_partition);
    let person = CachedPerson {
        id: PERSON_ID,
        ..test_cached_person()
    };
    seed_person(&cache, routing_partition, person);

    let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
    let addr = listener.local_addr().unwrap();
    let cancel = CancellationToken::new();
    let token = cancel.child_token();
    tokio::spawn(async move {
        Server::builder()
            .add_service(PersonHogLeaderServer::new(service))
            .serve_with_incoming_shutdown(
                tokio_stream::wrappers::TcpListenerStream::new(listener),
                token.cancelled(),
            )
            .await
            .unwrap();
    });
    tokio::time::sleep(Duration::from_millis(10)).await;
    let mut client = create_leader_client(addr).await;

    client
        .update_person_properties(with_partition(
            UpdatePersonPropertiesRequest {
                force_update: false,
                team_id: 1,
                person_id: PERSON_ID,
                event_name: "$set".to_string(),
                set_properties: serde_json::to_vec(&serde_json::json!({"name": "Pending"}))
                    .unwrap(),
                set_once_properties: vec![],
                unset_properties: vec![],
                is_identified: None,
                last_seen_at: None,
            },
            routing_partition,
        ))
        .await
        .unwrap();

    let key = PersonCacheKey {
        team_id: 1,
        person_id: PERSON_ID,
    };
    // Point the mark past the HWM so the recovery fetch cannot complete
    // (mark only accepts newer offsets, so bump well beyond the record's).
    let mark = dirty_index.get(&key).unwrap();
    dirty_index.mark(
        key.clone(),
        DirtyMark {
            version: mark.version,
            offset: mark.offset + 1_000,
            partition: mark.partition,
            stream: mark.stream.clone(),
            is_deleted: mark.is_deleted,
        },
    );
    cache.remove(routing_partition, &key);

    let status = client
        .get_person(with_partition(
            GetPersonRequest {
                team_id: 1,
                person_id: PERSON_ID,
                read_options: None,
            },
            routing_partition,
        ))
        .await
        .unwrap_err();
    assert_eq!(status.code(), tonic::Code::Unavailable);

    cancel.cancel();
}

// ============================================================
// Dirty-index bound: a full index sheds writes for new persons
// rather than acking writes it cannot track
// ============================================================

#[tokio::test]
async fn writes_shed_when_dirty_index_is_full() {
    let partition: u32 = 2;
    let person_a = person_id_for_partition(1, partition);
    let person_b = (person_a + 1..)
        .find(|pid| partition_for_person(1, *pid, NUM_PARTITIONS) == partition)
        .expect("a second person maps to the same partition");

    let mock_cluster = create_test_kafka_with_partitions(4).await;
    let cache = Arc::new(PartitionedCache::new(1 << 20));
    // Capacity 1: the first person's mark fills the index.
    let dirty_index = Arc::new(DirtyIndex::new(1));
    let service = PersonHogLeaderService::new(
        Arc::clone(&cache),
        None,
        Arc::new(DashMap::new()),
        Arc::new(InflightTracker::new()),
        served_captures(&mock_cluster.topic, NUM_PARTITIONS),
        Arc::clone(&dirty_index),
        test_recovery(&mock_cluster.bootstrap_servers()),
        PropertySizeLimits::new(655360, 524288),
        WarningsProducer::disabled(),
        Arc::new(DashMap::new()),
        common::fenced_producers_acquired(&mock_cluster.topic, 4).await,
        common::live_authority(),
        std::sync::Arc::new(personhog_leader::emitted::EmittedVersions::new(1_000_000)),
    );

    cache.create_partition(partition);
    seed_person(
        &cache,
        partition,
        CachedPerson {
            id: person_a,
            ..test_cached_person()
        },
    );
    seed_person(
        &cache,
        partition,
        CachedPerson {
            id: person_b,
            ..test_cached_person()
        },
    );

    let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
    let addr = listener.local_addr().unwrap();
    let cancel = CancellationToken::new();
    let token = cancel.child_token();
    tokio::spawn(async move {
        Server::builder()
            .add_service(PersonHogLeaderServer::new(service))
            .serve_with_incoming_shutdown(
                tokio_stream::wrappers::TcpListenerStream::new(listener),
                token.cancelled(),
            )
            .await
            .unwrap();
    });
    tokio::time::sleep(Duration::from_millis(10)).await;
    let mut client = create_leader_client(addr).await;

    let update_for = |person_id: i64, value: &str| UpdatePersonPropertiesRequest {
        force_update: false,
        team_id: 1,
        person_id,
        event_name: "$set".to_string(),
        set_properties: serde_json::to_vec(&serde_json::json!({ "k": value })).unwrap(),
        set_once_properties: vec![],
        unset_properties: vec![],
        is_identified: None,
        last_seen_at: None,
    };

    // First person fills the index.
    client
        .update_person_properties(with_partition(update_for(person_a, "v1"), partition))
        .await
        .expect("first person's write is admitted");

    // The same person stays admitted at capacity — updating its mark does
    // not grow the index.
    client
        .update_person_properties(with_partition(update_for(person_a, "v2"), partition))
        .await
        .expect("marked person's writes stay admitted at capacity");

    // A new person cannot be tracked: the write sheds instead of acking.
    let status = client
        .update_person_properties(with_partition(update_for(person_b, "v1"), partition))
        .await
        .unwrap_err();
    assert_eq!(status.code(), tonic::Code::ResourceExhausted);

    cancel.cancel();
}

// ============================================================
// Recovery version tripwire: a record whose version disagrees
// with the acked mark must fail the read, never be served
// ============================================================

#[tokio::test]
async fn recovery_fails_when_record_version_disagrees_with_the_mark() {
    const PERSON_ID: i64 = 2;
    let routing_partition: u32 = partition_for_person(1, PERSON_ID, NUM_PARTITIONS);
    let mock_cluster = create_test_kafka_with_partitions(4).await;

    let cache = Arc::new(PartitionedCache::new(1 << 20));
    let dirty_index = Arc::new(DirtyIndex::new(1_000_000));
    let recovery = test_recovery(&mock_cluster.bootstrap_servers());
    let service = PersonHogLeaderService::new(
        Arc::clone(&cache),
        None,
        Arc::new(DashMap::new()),
        Arc::new(InflightTracker::new()),
        served_captures(&mock_cluster.topic, NUM_PARTITIONS),
        Arc::clone(&dirty_index),
        Arc::clone(&recovery),
        PropertySizeLimits::new(655360, 524288),
        WarningsProducer::disabled(),
        Arc::new(DashMap::new()),
        common::fenced_producers_acquired(&mock_cluster.topic, 4).await,
        common::live_authority(),
        std::sync::Arc::new(personhog_leader::emitted::EmittedVersions::new(1_000_000)),
    );

    cache.create_partition(routing_partition);
    let person = CachedPerson {
        id: PERSON_ID,
        ..test_cached_person()
    };
    seed_person(&cache, routing_partition, person);

    let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
    let addr = listener.local_addr().unwrap();
    let cancel = CancellationToken::new();
    let token = cancel.child_token();
    tokio::spawn(async move {
        Server::builder()
            .add_service(PersonHogLeaderServer::new(service))
            .serve_with_incoming_shutdown(
                tokio_stream::wrappers::TcpListenerStream::new(listener),
                token.cancelled(),
            )
            .await
            .unwrap();
    });
    tokio::time::sleep(Duration::from_millis(10)).await;
    let mut client = create_leader_client(addr).await;

    client
        .update_person_properties(with_partition(
            UpdatePersonPropertiesRequest {
                force_update: false,
                team_id: 1,
                person_id: PERSON_ID,
                event_name: "$set".to_string(),
                set_properties: serde_json::to_vec(&serde_json::json!({"name": "Tripwire"}))
                    .unwrap(),
                set_once_properties: vec![],
                unset_properties: vec![],
                is_identified: None,
                last_seen_at: None,
            },
            routing_partition,
        ))
        .await
        .unwrap();

    let key = PersonCacheKey {
        team_id: 1,
        person_id: PERSON_ID,
    };
    // Corrupt the mark's version while keeping the true offset — `mark`
    // only replaces newer offsets, so clear the partition and re-insert.
    // The seek now lands on the right record, but its version disagrees.
    let mark = dirty_index.get(&key).unwrap();
    dirty_index.clear_partition(routing_partition);
    dirty_index.mark(
        key.clone(),
        DirtyMark {
            version: mark.version + 1,
            ..mark
        },
    );
    cache.remove(routing_partition, &key);

    let status = client
        .get_person(with_partition(
            GetPersonRequest {
                team_id: 1,
                person_id: PERSON_ID,
                read_options: None,
            },
            routing_partition,
        ))
        .await
        .unwrap_err();
    assert_eq!(status.code(), tonic::Code::Unavailable);

    cancel.cancel();
}

// ============================================================
// Recovery consumer pooling: repeated recoveries on a partition
// reuse one consumer, repositioning forward and backward
// ============================================================

#[tokio::test]
async fn recovery_reuses_the_partition_consumer_across_fetches() {
    let partition: u32 = 2;
    let person_a = person_id_for_partition(1, partition);
    let person_b = (person_a + 1..)
        .find(|pid| partition_for_person(1, *pid, NUM_PARTITIONS) == partition)
        .expect("a second person maps to the same partition");

    let mock_cluster = create_test_kafka_with_partitions(4).await;
    let cache = Arc::new(PartitionedCache::new(1 << 20));
    let dirty_index = Arc::new(DirtyIndex::new(1_000_000));
    let recovery = test_recovery(&mock_cluster.bootstrap_servers());
    let service = PersonHogLeaderService::new(
        Arc::clone(&cache),
        None,
        Arc::new(DashMap::new()),
        Arc::new(InflightTracker::new()),
        served_captures(&mock_cluster.topic, NUM_PARTITIONS),
        Arc::clone(&dirty_index),
        Arc::clone(&recovery),
        PropertySizeLimits::new(655360, 524288),
        WarningsProducer::disabled(),
        Arc::new(DashMap::new()),
        common::fenced_producers_acquired(&mock_cluster.topic, 4).await,
        common::live_authority(),
        std::sync::Arc::new(personhog_leader::emitted::EmittedVersions::new(1_000_000)),
    );

    cache.create_partition(partition);
    for person_id in [person_a, person_b] {
        seed_person(
            &cache,
            partition,
            CachedPerson {
                id: person_id,
                ..test_cached_person()
            },
        );
    }

    let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
    let addr = listener.local_addr().unwrap();
    let cancel = CancellationToken::new();
    let token = cancel.child_token();
    tokio::spawn(async move {
        Server::builder()
            .add_service(PersonHogLeaderServer::new(service))
            .serve_with_incoming_shutdown(
                tokio_stream::wrappers::TcpListenerStream::new(listener),
                token.cancelled(),
            )
            .await
            .unwrap();
    });
    tokio::time::sleep(Duration::from_millis(10)).await;
    let client = create_leader_client(addr).await;

    let update_for = |person_id: i64, value: &str| UpdatePersonPropertiesRequest {
        force_update: false,
        team_id: 1,
        person_id,
        event_name: "$set".to_string(),
        set_properties: serde_json::to_vec(&serde_json::json!({ "who": value })).unwrap(),
        set_once_properties: vec![],
        unset_properties: vec![],
        is_identified: None,
        last_seen_at: None,
    };
    // A's record lands at offset 0, B's at offset 1.
    for (person_id, value) in [(person_a, "a"), (person_b, "b")] {
        client
            .clone()
            .update_person_properties(with_partition(update_for(person_id, value), partition))
            .await
            .unwrap();
    }

    let recover = |person_id: i64| {
        let mut client = client.clone();
        async move {
            let response = client
                .get_person(with_partition(
                    GetPersonRequest {
                        team_id: 1,
                        person_id,
                        read_options: None,
                    },
                    partition,
                ))
                .await
                .unwrap();
            let person = response.into_inner().person.unwrap();
            let props: serde_json::Value = serde_json::from_slice(&person.properties).unwrap();
            props["who"].as_str().unwrap().to_string()
        }
    };
    let evict = |person_id: i64| {
        cache.remove(
            partition,
            &PersonCacheKey {
                team_id: 1,
                person_id,
            },
        )
    };

    // All three recoveries share the partition's pooled consumer: the
    // first fetches at offset 0, the second repositions forward, and the
    // third repositions backward past a record the consumer already read.
    // A consumer that fails to reposition serves the wrong record (key
    // mismatch) or times out — either fails the unwraps above.
    evict(person_a);
    assert_eq!(recover(person_a).await, "a");
    evict(person_b);
    assert_eq!(recover(person_b).await, "b");
    evict(person_a);
    assert_eq!(recover(person_a).await, "a");

    cancel.cancel();
}

/// A client disconnect mid-recovery drops the request future at an await
/// point. Checkout is an RAII guard, so the cancelled fetch's consumer
/// must come home: without Drop-based return, every cancellation leaked
/// the consumer while freeing its permit, and pool_size cancellations
/// left permits pointing at an empty pool — a panic under the pool mutex
/// that poisoned it and disabled recovery until restart.
#[tokio::test]
async fn cancelled_recovery_returns_its_consumer_to_the_pool() {
    use common::test_kafka_config;
    use personhog_leader::recovery::{ChangelogRecovery, RecoveryConfig};
    use rdkafka::producer::FutureRecord;

    let mock_cluster = create_test_kafka().await;
    let producer = create_local_kafka_producer().await;
    let mut kafka = test_kafka_config();
    kafka.kafka_hosts = mock_cluster.bootstrap_servers();
    // pool_size 1: a single leaked consumer exhausts the pool.
    let recovery = ChangelogRecovery::new(RecoveryConfig {
        kafka,
        pod_name: "cancel-pod".to_string(),
        recv_timeout: Duration::from_secs(2),
        pool_size: 1,
    })
    .unwrap();

    let key = PersonCacheKey {
        team_id: 1,
        person_id: 7,
    };
    let mark = DirtyMark {
        version: 1,
        offset: 0,
        partition: 0,
        stream: personhog_leader::cache::Stream {
            cluster: None,
            topic: std::sync::Arc::from(mock_cluster.topic.as_str()),
            partition: 0,
        },
        is_deleted: false,
    };

    // Nothing is produced yet, so each fetch parks awaiting the record;
    // cancel it mid-park, more times than the pool holds consumers.
    for _ in 0..3 {
        let cancelled = tokio::time::timeout(
            Duration::from_millis(50),
            recovery.fetch_person_at(&mark, &key),
        )
        .await;
        assert!(
            cancelled.is_err(),
            "the fetch must still be parked when cancelled"
        );
    }

    // The pool must still function end to end: produce the sought record
    // and the next fetch — checkout included — succeeds.
    let person = Person {
        id: 7,
        uuid: "00000000-0000-0000-0000-000000000007".to_string(),
        team_id: 1,
        properties: serde_json::to_vec(&serde_json::json!({"k": "v"})).unwrap(),
        properties_last_updated_at: Vec::new(),
        properties_last_operation: Vec::new(),
        created_at: 1_700_000_000,
        version: 1,
        is_identified: false,
        is_user_id: None,
        last_seen_at: None,
        is_deleted: false,
    };
    let payload = person.encode_to_vec();
    producer
        .send(
            FutureRecord::to(&mock_cluster.topic)
                .key("1:7")
                .partition(0)
                .payload(&payload),
            Duration::from_secs(5),
        )
        .await
        .expect("produce");

    let recovered = recovery
        .fetch_person_at(&mark, &key)
        .await
        .expect("the pool must survive cancellations");
    assert_eq!(recovered.id, 7);
}

#[tokio::test]
async fn a_mark_on_another_cluster_recovers_from_its_broker() {
    use personhog_coordination::types::ClusterEndpoint;
    use rdkafka::producer::FutureRecord;

    let Some(secondary) = common::secondary_kafka_bootstrap() else {
        return;
    };
    let old_topic = common::owned_broker_topic_on(&secondary, 1).await;
    let producer = common::create_kafka_producer_on(&secondary).await;
    let person = Person {
        id: 7,
        uuid: "00000000-0000-0000-0000-000000000007".to_string(),
        team_id: 1,
        properties: b"{}".to_vec(),
        properties_last_updated_at: Vec::new(),
        properties_last_operation: Vec::new(),
        created_at: 1_700_000_000,
        version: 1,
        is_identified: false,
        is_user_id: None,
        last_seen_at: None,
        is_deleted: false,
    };
    producer
        .send(
            FutureRecord::to(&old_topic.topic)
                .key("1:7")
                .partition(0)
                .payload(&person.encode_to_vec()),
            Duration::from_secs(5),
        )
        .await
        .expect("produce");

    let recovery = test_recovery(KAFKA_BOOTSTRAP);
    let mark = DirtyMark {
        version: person.version,
        offset: 0,
        partition: 0,
        stream: personhog_leader::cache::Stream {
            cluster: Some(std::sync::Arc::new(ClusterEndpoint {
                name: "old-cluster".to_string(),
                hosts: secondary,
                tls: false,
            })),
            topic: std::sync::Arc::from(old_topic.topic.as_str()),
            partition: 0,
        },
        is_deleted: false,
    };
    let key = PersonCacheKey {
        team_id: 1,
        person_id: 7,
    };
    let recovered = recovery
        .fetch_person_at(&mark, &key)
        .await
        .expect("recovers from the mark's cluster");
    assert_eq!(recovered.id, 7);
}

// ============================================================
// Admission-time property size enforcement, mirroring the Node
// pipeline's policy: an update that would newly push a within-limit
// row over the ceiling is rejected; a row already over it is
// remediated (existing properties trimmed, the update discarded) —
// so every acked record is applyable by the writer verbatim.
// ============================================================

#[tokio::test]
async fn oversize_updates_are_rejected_and_oversized_rows_remediated() {
    const PERSON_ID: i64 = 9;
    let routing_partition: u32 = partition_for_person(1, PERSON_ID, NUM_PARTITIONS);
    let mock_cluster = create_test_kafka_with_partitions(4).await;
    // This test reads its warnings back, so they must be its own: on a
    // shared topic it would collect whichever two arrived first.
    let warnings_topic = common::owned_broker_topic(1).await;

    let cache = Arc::new(PartitionedCache::new(1 << 20));
    let dirty_index = Arc::new(DirtyIndex::new(1_000_000));
    let recovery = test_recovery(&mock_cluster.bootstrap_servers());
    let service = PersonHogLeaderService::new(
        Arc::clone(&cache),
        None,
        Arc::new(DashMap::new()),
        Arc::new(InflightTracker::new()),
        served_captures(&mock_cluster.topic, NUM_PARTITIONS),
        Arc::clone(&dirty_index),
        recovery,
        PropertySizeLimits::new(655360, 524288),
        // Burst 2 so the trim and reject below — same team, same type —
        // both clear the throttle; the third enforcement action then
        // exercises suppression.
        common::local_warnings_producer(
            &warnings_topic.topic,
            WarningThrottle::new(DEFAULT_THROTTLE_PERIOD, NonZeroU32::new(2).unwrap()),
        )
        .await,
        Arc::new(DashMap::new()),
        common::fenced_producers_acquired(&mock_cluster.topic, 4).await,
        common::live_authority(),
        std::sync::Arc::new(personhog_leader::emitted::EmittedVersions::new(1_000_000)),
    );

    cache.create_partition(routing_partition);
    let person = CachedPerson {
        id: PERSON_ID,
        ..test_cached_person()
    };
    seed_person(&cache, routing_partition, person);

    let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
    let addr = listener.local_addr().unwrap();
    let cancel = CancellationToken::new();
    let token = cancel.child_token();
    tokio::spawn(async move {
        Server::builder()
            .add_service(PersonHogLeaderServer::new(service))
            .serve_with_incoming_shutdown(
                tokio_stream::wrappers::TcpListenerStream::new(listener),
                token.cancelled(),
            )
            .await
            .unwrap();
    });
    tokio::time::sleep(Duration::from_millis(10)).await;
    let mut client = create_leader_client(addr).await;

    // New violation: the seeded person is within limits and this update
    // would push the merged state over the ceiling — rejected outright,
    // nothing stored, matching the Node pipeline's
    // attempt_to_violate_limit path.
    let big = "x".repeat(400_000);
    let err = client
        .update_person_properties(with_partition(
            UpdatePersonPropertiesRequest {
                force_update: false,
                team_id: 1,
                person_id: PERSON_ID,
                event_name: "$set".to_string(),
                set_properties: serde_json::to_vec(&serde_json::json!({
                    "email": "[email protected]", "custom_a": big, "custom_b": big,
                }))
                .unwrap(),
                set_once_properties: vec![],
                unset_properties: vec![],
                is_identified: None,
                last_seen_at: None,
            },
            routing_partition,
        ))
        .await
        .expect_err("a newly violating update must be rejected");
    assert_eq!(err.code(), tonic::Code::InvalidArgument);
    assert!(err.message().contains("size limit"));
    assert!(
        !err.message().contains("xxx"),
        "the error must carry sizes, never property values"
    );

    // Rejection leaves no residue: the rejected update's keys are absent
    // from a strong read, and a normal update still works.
    let read = client
        .get_person(with_partition(
            GetPersonRequest {
                team_id: 1,
                person_id: PERSON_ID,
                read_options: None,
            },
            routing_partition,
        ))
        .await
        .unwrap()
        .into_inner();
    let read_props: serde_json::Value =
        serde_json::from_slice(&read.person.unwrap().properties).unwrap();
    assert!(!read_props.as_object().unwrap().contains_key("custom_a"));
    let response = client
        .update_person_properties(with_partition(
            UpdatePersonPropertiesRequest {
                force_update: false,
                team_id: 1,
                person_id: PERSON_ID,
                event_name: "$set".to_string(),
                set_properties: serde_json::to_vec(&serde_json::json!({"name": "after"})).unwrap(),
                set_once_properties: vec![],
                unset_properties: vec![],
                is_identified: None,
                last_seen_at: None,
            },
            routing_partition,
        ))
        .await
        .unwrap()
        .into_inner();
    assert!(response.updated);

    // Remediation: a row already over the ceiling (legacy rows, and rows
    // from the Node writer during dual-write) is healed on its next
    // update — existing properties trimmed alphabetically to the target
    // and the triggering update's changes discarded, matching the Node
    // pipeline's existing_record_violates_limit path. custom_a goes
    // (alphabetically first) and custom_b, which alone fits the target,
    // survives.
    const OVERSIZED_PERSON_ID: i64 = 10;
    let oversized_partition = partition_for_person(1, OVERSIZED_PERSON_ID, NUM_PARTITIONS);
    if oversized_partition != routing_partition {
        cache.create_partition(oversized_partition);
    }
    seed_person(
        &cache,
        oversized_partition,
        CachedPerson {
            id: OVERSIZED_PERSON_ID,
            properties: serde_json::to_vec(&serde_json::json!({
                "email": "[email protected]", "custom_a": big, "custom_b": big,
            }))
            .unwrap(),
            ..test_cached_person()
        },
    );
    let response = client
        .update_person_properties(with_partition(
            UpdatePersonPropertiesRequest {
                force_update: false,
                team_id: 1,
                person_id: OVERSIZED_PERSON_ID,
                event_name: "$set".to_string(),
                set_properties: serde_json::to_vec(&serde_json::json!({"name": "discarded"}))
                    .unwrap(),
                set_once_properties: vec![],
                unset_properties: vec![],
                is_identified: None,
                last_seen_at: None,
            },
            oversized_partition,
        ))
        .await
        .unwrap()
        .into_inner();
    assert!(response.updated);
    let stored: serde_json::Value =
        serde_json::from_slice(&response.person.unwrap().properties).unwrap();
    let stored = stored.as_object().unwrap();
    assert_eq!(stored["email"], "[email protected]", "protected properties survive");
    assert!(
        !stored.contains_key("custom_a"),
        "custom_a must be trimmed (alphabetically first)"
    );
    assert!(stored.contains_key("custom_b"));
    assert!(
        !stored.contains_key("name"),
        "the triggering update's changes are discarded, as in Node"
    );

    // The stored state IS the remediated state: a strong read agrees
    // with the update response.
    let read = client
        .get_person(with_partition(
            GetPersonRequest {
                team_id: 1,
                person_id: OVERSIZED_PERSON_ID,
                read_options: None,
            },
            oversized_partition,
        ))
        .await
        .unwrap()
        .into_inner();
    let read_props: serde_json::Value =
        serde_json::from_slice(&read.person.unwrap().properties).unwrap();
    assert!(!read_props.as_object().unwrap().contains_key("custom_a"));

    // Unremediable: the stored row is over the ceiling and its protected
    // property alone exceeds the trim target — nothing can be trimmed,
    // so the update is rejected. (Its warning lands after the throttle
    // burst of 2 and is suppressed; the error is asserted directly.)
    const UNREMEDIABLE_PERSON_ID: i64 = 12;
    let unremediable_partition = partition_for_person(1, UNREMEDIABLE_PERSON_ID, NUM_PARTITIONS);
    if unremediable_partition != routing_partition && unremediable_partition != oversized_partition
    {
        cache.create_partition(unremediable_partition);
    }
    let huge_email = "e".repeat(700_000);
    seed_person(
        &cache,
        unremediable_partition,
        CachedPerson {
            id: UNREMEDIABLE_PERSON_ID,
            properties: serde_json::to_vec(&serde_json::json!({ "email": huge_email })).unwrap(),
            ..test_cached_person()
        },
    );
    let err = client
        .update_person_properties(with_partition(
            UpdatePersonPropertiesRequest {
                force_update: false,
                team_id: 1,
                person_id: UNREMEDIABLE_PERSON_ID,
                event_name: "$set".to_string(),
                set_properties: serde_json::to_vec(&serde_json::json!({"name": "x"})).unwrap(),
                set_once_properties: vec![],
                unset_properties: vec![],
                is_identified: None,
                last_seen_at: None,
            },
            unremediable_partition,
        ))
        .await
        .expect_err("an unremediable row's update must be rejected");
    assert_eq!(err.code(), tonic::Code::InvalidArgument);
    assert!(err.message().contains("trim target"));
    assert!(
        !err.message().contains("eee"),
        "the error must carry sizes, never property values"
    );

    // The first two enforcement actions emitted in-product warnings from
    // the leader: one rejection, one remediation trim.
    let consumer: BaseConsumer = ClientConfig::new()
        .set("bootstrap.servers", mock_cluster.bootstrap_servers())
        .set("group.id", "warnings-check")
        .create()
        .unwrap();
    let mut tpl = TopicPartitionList::new();
    tpl.add_partition_offset(&warnings_topic.topic, 0, rdkafka::Offset::Beginning)
        .unwrap();
    consumer.assign(&tpl).unwrap();
    let mut warnings = Vec::new();
    let deadline = Instant::now() + Duration::from_secs(10);
    while warnings.len() < 2 && Instant::now() < deadline {
        if let Some(Ok(message)) = consumer.poll(Duration::from_millis(200)) {
            let payload: serde_json::Value =
                serde_json::from_slice(message.payload().unwrap()).unwrap();
            assert_eq!(payload["type"], "person_properties_size_violation");
            assert_eq!(payload["source"], "personhog-leader");
            // The pipeline's canonical shape: a ClickHouse-format
            // timestamp, and category/severity in details (the v2 table
            // materializes columns from those exact keys).
            let timestamp = payload["timestamp"].as_str().unwrap();
            assert!(
                timestamp.len() == 23 && timestamp.as_bytes()[10] == b' ',
                "timestamp must be ClickHouse-format, got {timestamp}"
            );
            let details: serde_json::Value =
                serde_json::from_str(payload["details"].as_str().unwrap()).unwrap();
            assert_eq!(details["category"], "size");
            assert_eq!(details["severity"], "error");
            assert_eq!(details["pipelineStep"], "personhog_admission");
            assert!(
                details["personId"].as_str().unwrap().contains('-'),
                "personId is the uuid"
            );
            warnings.push(payload["details"].as_str().unwrap().to_string());
        }
    }
    assert_eq!(
        warnings.len(),
        2,
        "expected a trim warning and a reject warning"
    );
    assert!(warnings.iter().any(|w| w.contains("trimmed")));
    assert!(warnings.iter().any(|w| w.contains("rejected")));

    // Team 1's (team, type) budget is exhausted: another enforcement
    // action still rejects, but its warning is suppressed. A fresh
    // team's warning still emits, and — the warnings topic being a
    // single partition — arriving as the very next message proves the
    // suppressed one was never produced.
    const TEAM_2_PERSON_ID: i64 = 11;
    let team2_partition = partition_for_person(2, TEAM_2_PERSON_ID, NUM_PARTITIONS);
    if team2_partition != routing_partition {
        cache.create_partition(team2_partition);
    }
    seed_person(
        &cache,
        team2_partition,
        CachedPerson {
            id: TEAM_2_PERSON_ID,
            team_id: 2,
            ..test_cached_person()
        },
    );

    let oversized_update = serde_json::to_vec(&serde_json::json!({
        "email": "[email protected]", "custom_y": big, "custom_z": big,
    }))
    .unwrap();
    client
        .update_person_properties(with_partition(
            UpdatePersonPropertiesRequest {
                force_update: false,
                team_id: 1,
                person_id: PERSON_ID,
                event_name: "$set".to_string(),
                set_properties: oversized_update.clone(),
                set_once_properties: vec![],
                unset_properties: vec![],
                is_identified: None,
                last_seen_at: None,
            },
            routing_partition,
        ))
        .await
        .expect_err("throttling gates the warning, not the enforcement");

    client
        .update_person_properties(with_partition(
            UpdatePersonPropertiesRequest {
                force_update: false,
                team_id: 2,
                person_id: TEAM_2_PERSON_ID,
                event_name: "$set".to_string(),
                set_properties: oversized_update,
                set_once_properties: vec![],
                unset_properties: vec![],
                is_identified: None,
                last_seen_at: None,
            },
            team2_partition,
        ))
        .await
        .expect_err("team 2's newly violating update is rejected too");

    let deadline = Instant::now() + Duration::from_secs(10);
    let next = loop {
        match consumer.poll(Duration::from_millis(200)) {
            Some(Ok(message)) => break message,
            _ => assert!(Instant::now() < deadline, "team 2's warning must arrive"),
        }
    };
    let payload: serde_json::Value = serde_json::from_slice(next.payload().unwrap()).unwrap();
    assert_eq!(
        payload["team_id"], 2,
        "team 1's third warning must be suppressed by the throttle"
    );

    // NUL bytes are sanitized at admission (`\u{0000}` → `\u{FFFD}`,
    // matching the Node pipeline): Postgres jsonb refuses NUL, so an
    // unsanitized record would be unapplyable by the writer. The stored
    // state — response and strong read alike — carries the sanitized form.
    let response = client
        .update_person_properties(with_partition(
            UpdatePersonPropertiesRequest {
                force_update: false,
                team_id: 1,
                person_id: PERSON_ID,
                event_name: "$set".to_string(),
                set_properties: serde_json::to_vec(&serde_json::json!({
                    "nul\u{0000}key": "nul\u{0000}value",
                }))
                .unwrap(),
                set_once_properties: vec![],
                unset_properties: vec![],
                is_identified: None,
                last_seen_at: None,
            },
            routing_partition,
        ))
        .await
        .unwrap()
        .into_inner();
    assert!(response.updated);
    let stored: serde_json::Value =
        serde_json::from_slice(&response.person.unwrap().properties).unwrap();
    let stored = stored.as_object().unwrap();
    assert_eq!(stored["nul\u{FFFD}key"], "nul\u{FFFD}value");
    assert!(!stored.contains_key("nul\u{0000}key"));

    let read = client
        .get_person(with_partition(
            GetPersonRequest {
                team_id: 1,
                person_id: PERSON_ID,
                read_options: None,
            },
            routing_partition,
        ))
        .await
        .unwrap()
        .into_inner();
    let read_person = read.person.unwrap();
    let read_props: serde_json::Value = serde_json::from_slice(&read_person.properties).unwrap();
    assert_eq!(read_props["nul\u{FFFD}key"], "nul\u{FFFD}value");
    let version_after_nul = read_person.version;

    // Resending the identical NUL-bearing update must hit the no-change
    // fast path: inputs are sanitized before diffing, so the raw NUL form
    // compares equal to its stored U+FFFD form instead of producing a
    // fresh record and version bump on every repeat.
    let response = client
        .update_person_properties(with_partition(
            UpdatePersonPropertiesRequest {
                force_update: false,
                team_id: 1,
                person_id: PERSON_ID,
                event_name: "$set".to_string(),
                set_properties: serde_json::to_vec(&serde_json::json!({
                    "nul\u{0000}key": "nul\u{0000}value",
                }))
                .unwrap(),
                set_once_properties: vec![],
                unset_properties: vec![],
                is_identified: None,
                last_seen_at: None,
            },
            routing_partition,
        ))
        .await
        .unwrap()
        .into_inner();
    assert!(
        !response.updated,
        "a sanitized repeat must be detected as a no-op"
    );
    assert_eq!(
        response.person.unwrap().version,
        version_after_nul,
        "a no-op repeat must not bump the version"
    );

    cancel.cancel();
}

// ============================================================
// A version this pod emitted without hearing the outcome is spent
// ============================================================

/// A cancelled request, or a commit whose fate stayed unknown, can leave a
/// record on the changelog that this pod never learned about. The cache
/// still holds the version from before that write, so the next update for
/// the same person would derive the same number again — and the writer's
/// strict version guard keeps whichever of the two records arrived first,
/// discarding the other. When the discarded one is the acked write, the
/// acknowledgement was a lie.
///
/// Driving the RPC rather than the derivation: the floor existing is not
/// the property that matters, the write path consulting it is.
#[tokio::test]
async fn an_unresolved_version_is_never_reused() {
    let cache = Arc::new(PartitionedCache::new(1 << 20));
    let _mock_cluster = create_test_kafka().await;
    // One set for both: a second would fence the first at the broker.
    let fenced =
        common::fenced_producers_acquired(&_mock_cluster.topic, NUM_PARTITIONS as i32).await;
    let emitted_versions = Arc::new(personhog_leader::emitted::EmittedVersions::new(1_000_000));
    let service = PersonHogLeaderService::new(
        Arc::clone(&cache),
        None,
        Arc::new(DashMap::new()),
        Arc::new(InflightTracker::new()),
        served_captures(&_mock_cluster.topic, NUM_PARTITIONS),
        Arc::new(DirtyIndex::new(1_000_000)),
        test_recovery(KAFKA_BOOTSTRAP),
        PropertySizeLimits::new(655360, 524288),
        WarningsProducer::disabled(),
        Arc::new(DashMap::new()),
        Arc::clone(&fenced),
        common::live_authority(),
        Arc::clone(&emitted_versions),
    );

    cache.create_partition(0);
    let person = test_cached_person();
    let cached_version = person.version;
    seed_person(&cache, 0, person);

    // The state a write that was never answered for leaves behind: the
    // cache is untouched, but versions up to this one may already exist.
    let spent = cached_version + 4;
    emitted_versions.raise_for_test(
        0,
        PersonCacheKey {
            team_id: 1,
            person_id: 42,
        },
        spent,
    );

    let response = service
        .update_person_properties(with_partition(
            UpdatePersonPropertiesRequest {
                force_update: false,
                team_id: 1,
                person_id: 42,
                event_name: "$set".to_string(),
                set_properties: serde_json::to_vec(&serde_json::json!({"name": "Updated"}))
                    .unwrap(),
                set_once_properties: vec![],
                unset_properties: vec![],
                is_identified: None,
                last_seen_at: None,
            },
            0,
        ))
        .await
        .expect("the update itself must still succeed");

    assert_eq!(
        response.into_inner().person.unwrap().version,
        spent + 1,
        "the write must derive past every version this pod already emitted, \
         not merely past the one its cache happens to hold"
    );
}

// ============================================================
// The fenced write path, driven through the RPC
// ============================================================

/// Nothing constructed the service with fencing on, so its entire fenced
/// arm was unreachable: a `panic!` at the top of it left the whole suite
/// green. That arm decides, for every failure mode, whether the write's
/// version is handed back for reuse — and handing back a version whose
/// record may be on the changelog is the acked-write loss the floors
/// exist to prevent.
///
/// This drives the real RPC against a real broker with fencing enabled,
/// and asserts the two answers that matter: a fenced partition bounces,
/// and the version it consumed is *not* reused afterwards.
#[tokio::test]
async fn a_fenced_write_that_bounces_does_not_hand_its_version_back() {
    let topic = format!("fenced_svc_{}", uuid::Uuid::new_v4().simple());
    let cache = Arc::new(PartitionedCache::new(1 << 20));
    let _mock_cluster = create_test_kafka().await;
    let fenced = Arc::new(common::test_fenced_producers());

    let service = PersonHogLeaderService::new(
        Arc::clone(&cache),
        None,
        Arc::new(DashMap::new()),
        Arc::new(InflightTracker::new()),
        served_captures(&topic, NUM_PARTITIONS),
        Arc::new(DirtyIndex::new(1_000_000)),
        test_recovery(KAFKA_BOOTSTRAP),
        PropertySizeLimits::new(655360, 524288),
        WarningsProducer::disabled(),
        Arc::new(DashMap::new()),
        Arc::clone(&fenced),
        common::live_authority(),
        std::sync::Arc::new(personhog_leader::emitted::EmittedVersions::new(1_000_000)),
    );

    cache.create_partition(0);
    let person = test_cached_person();
    let base_version = person.version;
    seed_person(&cache, 0, person);
    fenced
        .acquire(0, &target(&topic))
        .await
        .expect("take the partition's fence");

    // Each call must change something, or the handler takes its
    // no-change fast path and never reaches the produce at all.
    let update = |n: i64| {
        let mut request = Request::new(UpdatePersonPropertiesRequest {
            force_update: false,
            team_id: 1,
            person_id: 42,
            event_name: "$set".to_string(),
            set_properties: serde_json::to_vec(&serde_json::json!({"n": n})).unwrap(),
            set_once_properties: vec![],
            unset_properties: vec![],
            is_identified: None,
            last_seen_at: None,
        });
        request
            .metadata_mut()
            .insert("x-partition", "0".parse().unwrap());
        request
    };

    // A healthy fenced write goes through the transaction window.
    let first = service
        .update_person_properties(update(1))
        .await
        .expect("a fenced write with a held fence must succeed");
    assert_eq!(first.into_inner().person.unwrap().version, base_version + 1);

    // Give the fence up beneath the service: the next write has no
    // producer and must answer as unowned so the router re-resolves.
    fenced.release(0);
    let bounced = service
        .update_person_properties(update(2))
        .await
        .expect_err("a write with no fence must not be acked");
    assert_eq!(
        bounced.code(),
        tonic::Code::FailedPrecondition,
        "an unowned partition is the router's bounce class, not a retryable failure"
    );

    // The bounced write never reached the broker, so its version is free
    // to derive again — the next successful write must land exactly one
    // past the first, not two.
    fenced
        .acquire(0, &target(&topic))
        .await
        .expect("re-take the fence");
    let third = service
        .update_person_properties(update(3))
        .await
        .expect("a re-acquired fence writes again");
    assert_eq!(
        third.into_inner().person.unwrap().version,
        base_version + 2,
        "a write that provably never left the pod must not burn its version"
    );
}

/// A window fenced with its own outcome unknown answers two questions at
/// once, and the handler has to honour both: the router gets the
/// ownership bounce, and the version stays spent because the commit may
/// have landed on an attempt librdkafka re-issued internally. Handing the
/// version back would put a second record at the same number behind one
/// that may be committed, and the writer's first-wins guard then discards
/// whichever arrived second — which is the acked one.
///
/// The cached entry stays. Evicting it resolves nothing, since recovery
/// reads the last *marked* offset — the previous write that did succeed —
/// and it would answer NOT_FOUND outright once that mark is pruned on a
/// pod with no fallback pool configured.
#[tokio::test]
async fn a_fence_with_an_unknown_outcome_keeps_both_its_version_and_its_cache_entry() {
    for (staged, expected) in [
        (
            personhog_leader::fencing::FencedProduceError::FencedUncertain("staged".to_string()),
            tonic::Code::FailedPrecondition,
        ),
        (
            personhog_leader::fencing::FencedProduceError::Indeterminate("staged".to_string()),
            tonic::Code::Unknown,
        ),
    ] {
        an_unknown_outcome_keeps_its_version(staged, expected).await;
    }
}

/// Both doubt arms carry the same obligation: the record may be in the
/// log, so the version stays spent and the cached entry stays put. They
/// differ only in what the caller is told — an ownership bounce when the
/// partition also moved, plain doubt when it did not.
async fn an_unknown_outcome_keeps_its_version(
    staged: personhog_leader::fencing::FencedProduceError,
    expected: tonic::Code,
) {
    let topic = format!("fenced_uncertain_{}", uuid::Uuid::new_v4().simple());
    let cache = Arc::new(PartitionedCache::new(1 << 20));
    let _mock_cluster = create_test_kafka().await;
    let fenced = Arc::new(common::test_fenced_producers());
    let emitted_versions = Arc::new(personhog_leader::emitted::EmittedVersions::new(1_000_000));

    let service = PersonHogLeaderService::new(
        Arc::clone(&cache),
        None,
        Arc::new(DashMap::new()),
        Arc::new(InflightTracker::new()),
        served_captures(&topic, NUM_PARTITIONS),
        Arc::new(DirtyIndex::new(1_000_000)),
        test_recovery(KAFKA_BOOTSTRAP),
        PropertySizeLimits::new(655360, 524288),
        WarningsProducer::disabled(),
        Arc::new(DashMap::new()),
        Arc::clone(&fenced),
        common::live_authority(),
        Arc::clone(&emitted_versions),
    );

    cache.create_partition(0);
    let person = test_cached_person();
    let base_version = person.version;
    seed_person(&cache, 0, person);
    fenced
        .acquire(0, &target(&topic))
        .await
        .expect("take the partition's fence");

    let cache_key = PersonCacheKey {
        team_id: 1,
        person_id: 42,
    };
    let update = |n: i64| {
        let mut request = Request::new(UpdatePersonPropertiesRequest {
            force_update: false,
            team_id: 1,
            person_id: 42,
            event_name: "$set".to_string(),
            set_properties: serde_json::to_vec(&serde_json::json!({"n": n})).unwrap(),
            set_once_properties: vec![],
            unset_properties: vec![],
            is_identified: None,
            last_seen_at: None,
        });
        request
            .metadata_mut()
            .insert("x-partition", "0".parse().unwrap());
        request
    };

    service
        .update_person_properties(update(1))
        .await
        .expect("a fenced write with a held fence must succeed");

    // The outcome a broker fault inside a transaction would produce.
    fenced.stage_failure_for_test(0, staged);
    let bounced = service
        .update_person_properties(update(2))
        .await
        .expect_err("an uncertain fence must not be acked");
    assert_eq!(
        bounced.code(),
        expected,
        "the caller's answer: {}",
        bounced.message()
    );

    assert_eq!(
        emitted_versions.floor_for(0, &cache_key, 0),
        base_version + 2,
        "a version whose record may be committed must stay spent"
    );
    assert!(
        matches!(
            cache.get(0, &cache_key),
            personhog_leader::cache::CacheLookup::Found(_)
        ),
        "the entry must survive, or a pruned mark leaves the person unreadable"
    );
}

/// The scalar fields carry merge semantics, not assignment: identification
/// only ever ORs true (and counts as a change on its own — an $identify
/// with no property diffs must still produce a record), while last-seen
/// only ever advances and never earns a record by itself.
#[tokio::test]
async fn scalar_fields_merge_rather_than_assign() {
    const HOUR_MS: i64 = 3_600_000;
    let topic = format!("scalar_merge_{}", uuid::Uuid::new_v4().simple());
    let cache = Arc::new(PartitionedCache::new(1 << 20));
    let _mock_cluster = create_test_kafka().await;
    let emitted_versions = Arc::new(personhog_leader::emitted::EmittedVersions::new(1_000_000));

    let service = PersonHogLeaderService::new(
        Arc::clone(&cache),
        None,
        Arc::new(DashMap::new()),
        Arc::new(InflightTracker::new()),
        served_captures(&topic, NUM_PARTITIONS),
        Arc::new(DirtyIndex::new(1_000_000)),
        test_recovery(KAFKA_BOOTSTRAP),
        PropertySizeLimits::new(655360, 524288),
        WarningsProducer::disabled(),
        Arc::new(DashMap::new()),
        common::fenced_producers_acquired(&topic, NUM_PARTITIONS as i32).await,
        common::live_authority(),
        Arc::clone(&emitted_versions),
    );

    cache.create_partition(0);
    seed_person(&cache, 0, test_cached_person());

    let update = |set: serde_json::Value, identified: Option<bool>, seen: Option<i64>| {
        let mut request = Request::new(UpdatePersonPropertiesRequest {
            force_update: false,
            team_id: 1,
            person_id: 42,
            event_name: "$identify".to_string(),
            set_properties: if set.is_null() {
                vec![]
            } else {
                serde_json::to_vec(&set).unwrap()
            },
            set_once_properties: vec![],
            unset_properties: vec![],
            is_identified: identified,
            last_seen_at: seen,
        });
        request
            .metadata_mut()
            .insert("x-partition", "0".parse().unwrap());
        request
    };

    // A last-seen-only advance is a change of its own: ingestion's
    // direct write path persists it, so this path must too. The leader
    // floors to the hour, so a raw mid-hour timestamp stores the hour.
    let result = service
        .update_person_properties(update(serde_json::Value::Null, None, Some(HOUR_MS + 1_234)))
        .await
        .expect("last-seen-only must be accepted")
        .into_inner();
    assert!(result.updated, "a last-seen-only advance earns a record");
    let person = result.person.unwrap();
    assert_eq!(person.version, 2);
    assert_eq!(
        person.last_seen_at,
        Some(HOUR_MS),
        "stored at hour granularity"
    );

    // A second event inside the stored hour is a no-op: the flooring
    // bounds record volume at one per person-hour regardless of the
    // caller's precision.
    let result = service
        .update_person_properties(update(
            serde_json::Value::Null,
            None,
            Some(HOUR_MS + 900_000),
        ))
        .await
        .expect("a same-hour timestamp must be accepted")
        .into_inner();
    assert!(
        !result.updated,
        "a raw timestamp inside the stored hour must not produce a record"
    );

    // Identification is a change, and the newer last-seen merges in.
    let result = service
        .update_person_properties(update(
            serde_json::Value::Null,
            Some(true),
            Some(5 * HOUR_MS),
        ))
        .await
        .expect("identification must apply")
        .into_inner();
    assert!(result.updated, "the false→true transition earns a record");
    let person = result.person.unwrap();
    assert_eq!(person.version, 3);
    assert!(person.is_identified);
    assert_eq!(person.last_seen_at, Some(5 * HOUR_MS));

    // OR semantics: false does not revert, and re-asserting true is not a
    // change.
    let result = service
        .update_person_properties(update(serde_json::Value::Null, Some(false), None))
        .await
        .expect("a false is a no-op, not an error")
        .into_inner();
    assert!(
        !result.updated,
        "identification never reverts through this RPC"
    );
    assert!(result.person.unwrap().is_identified);
    let result = service
        .update_person_properties(update(serde_json::Value::Null, Some(true), None))
        .await
        .expect("re-asserting true is accepted")
        .into_inner();
    assert!(
        !result.updated,
        "an already-identified person has nothing to change"
    );

    // A non-advancing last-seen is not a change: only forward movement
    // of the stored value earns a record.
    let result = service
        .update_person_properties(update(serde_json::Value::Null, None, Some(3 * HOUR_MS)))
        .await
        .expect("a stale last-seen is a no-op, not an error")
        .into_inner();
    assert!(
        !result.updated,
        "a last-seen at or below the stored value must not produce a record"
    );

    // Max-merge: an older last-seen cannot lower the stored one, and a
    // property change carries the newest value forward.
    let result = service
        .update_person_properties(update(
            serde_json::json!({"plan": "pro"}),
            None,
            Some(3 * HOUR_MS),
        ))
        .await
        .expect("a property change with a stale last-seen must apply")
        .into_inner();
    assert!(result.updated);
    let person = result.person.unwrap();
    assert_eq!(person.version, 4);
    assert_eq!(
        person.last_seen_at,
        Some(5 * HOUR_MS),
        "a stale last-seen must not lower the stored value"
    );

    // A last-seen outside the writer's bindable range is discarded, not
    // fatal: the update it rides on still lands.
    let result = service
        .update_person_properties(update(
            serde_json::json!({"plan": "max"}),
            None,
            Some(i64::MAX),
        ))
        .await
        .expect("an absurd last-seen must not fail the update it rides on")
        .into_inner();
    assert!(result.updated);
    let person = result.person.unwrap();
    assert_eq!(person.version, 5);
    assert_eq!(
        person.last_seen_at,
        Some(5 * HOUR_MS),
        "an out-of-range last-seen must be discarded, not stored"
    );
}