Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

60.5 KB
Raw Blame
mod common;

use std::collections::{HashMap, HashSet};
use std::sync::Arc;
use std::time::Duration;

use chrono::{DateTime, TimeZone, Utc};
use common::{
    cleanup_team, create_local_kafka_producer, create_mock_kafka, create_test_pool, make_person,
    unique_team_id, KAFKA_BOOTSTRAP, TARGET_TABLE, TOPIC,
};
use personhog_coordination::follow::Follower;
use personhog_coordination::types::{ChangelogConfig, Generation};
use personhog_proto::personhog::types::v1::Person;
use personhog_writer::buffer::BufferedPerson;
use personhog_writer::consumer::{ConsumerTask, FlushBatch};
use personhog_writer::kafka::{PersonConsumer, Source};
use personhog_writer::pg::PgStore;
use personhog_writer::store::{
    BatchOutcome, PersonDb, PersonWriteStore, WriteError, WriteErrorKind,
};
use personhog_writer::writer::WriterTask;
use prost::Message;
use rdkafka::consumer::{Consumer, StreamConsumer};
use rdkafka::producer::FutureRecord;
use rdkafka::{ClientConfig, Offset, TopicPartitionList};
use tokio::sync::mpsc;

fn source_on(topic: &str, partition: i32) -> Source {
    Source {
        topic: Arc::from(topic),
        partition,
    }
}

fn source(partition: i32) -> Source {
    source_on(TOPIC, partition)
}

fn rows(persons: Vec<Person>) -> Vec<BufferedPerson> {
    persons
        .into_iter()
        .map(|person| BufferedPerson {
            source: source(0),
            person,
        })
        .collect()
}

// ============================================================
// PG Writer: upsert correctness
// ============================================================

#[tokio::test]
async fn writer_upserts_person_to_pg() {
    let pool = create_test_pool().await;
    let team_id: i32 = unique_team_id();

    let writer = PersonWriteStore::new(
        PgStore::new(pool.clone(), TARGET_TABLE.to_string()),
        common::test_store_config(),
        common::test_permits(),
    );
    let person = make_person(team_id as i64, 1, 1);

    assert!(matches!(
        writer.upsert_batch(vec![person]).await,
        BatchOutcome::Success
    ));

    let row: (i64, i64, bool) = sqlx::query_as(
        "SELECT id, version, is_identified FROM personhog_person_tmp WHERE team_id = $1 AND id = $2",
    )
    .bind(team_id)
    .bind(1_i64)
    .fetch_one(&pool)
    .await
    .unwrap();

    assert_eq!(row.0, 1); // id
    assert_eq!(row.1, 1); // version
    assert!(!row.2); // is_identified

    cleanup_team(&pool, team_id).await;
}

/// Persons created before personhog from a client's zero `DateTime` carry
/// year 1 as `created_at`; the leader admits what this upsert binds, so
/// the bind has to hold at the far end of the range.
#[tokio::test]
async fn writer_upserts_a_year_one_created_at() {
    const YEAR_ONE_MS: i64 = -62_135_596_800_000;
    let pool = create_test_pool().await;
    let team_id: i32 = unique_team_id();

    let writer = PersonWriteStore::new(
        PgStore::new(pool.clone(), TARGET_TABLE.to_string()),
        common::test_store_config(),
        common::test_permits(),
    );
    let person = Person {
        created_at: YEAR_ONE_MS,
        last_seen_at: Some(YEAR_ONE_MS),
        ..make_person(team_id as i64, 1, 1)
    };

    assert!(matches!(
        writer.upsert_batch(vec![person]).await,
        BatchOutcome::Success
    ));

    let row: (DateTime<Utc>, Option<DateTime<Utc>>) = sqlx::query_as(
        "SELECT created_at, last_seen_at FROM personhog_person_tmp WHERE team_id = $1 AND id = $2",
    )
    .bind(team_id)
    .bind(1_i64)
    .fetch_one(&pool)
    .await
    .unwrap();
    let year_one = Utc.timestamp_millis_opt(YEAR_ONE_MS).unwrap();
    assert_eq!(row.0, year_one);
    assert_eq!(row.1, Some(year_one));

    // The leader admits down to Postgres's own floor, Julian day 0, so
    // that exact instant has to bind as well.
    const PG_TIMESTAMPTZ_MIN_MS: i64 = -210_866_803_200_000;
    let floor_person = Person {
        created_at: PG_TIMESTAMPTZ_MIN_MS,
        ..make_person(team_id as i64, 2, 1)
    };
    assert!(matches!(
        writer.upsert_batch(vec![floor_person]).await,
        BatchOutcome::Success
    ));
    let floor: (DateTime<Utc>,) = sqlx::query_as(
        "SELECT created_at FROM personhog_person_tmp WHERE team_id = $1 AND id = $2",
    )
    .bind(team_id)
    .bind(2_i64)
    .fetch_one(&pool)
    .await
    .unwrap();
    assert_eq!(
        floor.0,
        Utc.timestamp_millis_opt(PG_TIMESTAMPTZ_MIN_MS).unwrap()
    );

    cleanup_team(&pool, team_id).await;
}

#[tokio::test]
async fn writer_projects_death_documents_as_tombstones() {
    let pool = create_test_pool().await;
    let team_id: i32 = unique_team_id();

    let writer = PersonWriteStore::new(
        PgStore::new(pool.clone(), TARGET_TABLE.to_string()),
        common::test_store_config(),
        common::test_permits(),
    );

    let fetch_row = |pool: sqlx::PgPool| async move {
        let row: (i64, bool, String) = sqlx::query_as(
            "SELECT version, is_deleted, properties::text \
             FROM personhog_person_tmp WHERE team_id = $1 AND id = $2",
        )
        .bind(team_id)
        .bind(1_i64)
        .fetch_one(&pool)
        .await
        .unwrap();
        row
    };

    // Birth: live.
    assert!(matches!(
        writer
            .upsert_batch(vec![make_person(team_id as i64, 1, 1)])
            .await,
        BatchOutcome::Success
    ));
    let row = fetch_row(pool.clone()).await;
    assert!(!row.1);

    // The death document projects as an ordinary update whose tombstone
    // lands on the row: is_deleted set, properties scrubbed — never a row
    // delete (burial is the GC job's, after ClickHouse erasure).
    let mut death = make_person(team_id as i64, 1, 3);
    death.is_deleted = true;
    death.properties = b"{}".to_vec();
    assert!(matches!(
        writer.upsert_batch(vec![death]).await,
        BatchOutcome::Success
    ));
    let row = fetch_row(pool.clone()).await;
    assert_eq!(row.0, 3);
    assert!(row.1, "the tombstone is projected, not a row delete");
    assert_eq!(row.2, "{}");

    // A lagging pre-death record must not resurrect the tombstone.
    let stale = make_person(team_id as i64, 1, 2);
    assert!(matches!(
        writer.upsert_batch(vec![stale]).await,
        BatchOutcome::Success
    ));
    let row = fetch_row(pool.clone()).await;
    assert_eq!(row.0, 3, "version guard holds");
    assert!(row.1, "the tombstone survives lagging writes");

    cleanup_team(&pool, team_id).await;
}

#[tokio::test]
async fn writer_version_guard_skips_stale_updates() {
    let pool = create_test_pool().await;
    let team_id: i32 = unique_team_id();

    let writer = PersonWriteStore::new(
        PgStore::new(pool.clone(), TARGET_TABLE.to_string()),
        common::test_store_config(),
        common::test_permits(),
    );

    // Write version 5
    let person_v5 = make_person(team_id as i64, 1, 5);
    assert!(matches!(
        writer.upsert_batch(vec![person_v5]).await,
        BatchOutcome::Success
    ));

    // Attempt to write version 3 (stale) -- should be a no-op
    let mut person_v3 = make_person(team_id as i64, 1, 3);
    person_v3.properties =
        serde_json::to_vec(&serde_json::json!({"email": "[email protected]"})).unwrap();
    assert!(matches!(
        writer.upsert_batch(vec![person_v3]).await,
        BatchOutcome::Success
    ));

    let row: (i64,) =
        sqlx::query_as("SELECT version FROM personhog_person_tmp WHERE team_id = $1 AND id = $2")
            .bind(team_id)
            .bind(1_i64)
            .fetch_one(&pool)
            .await
            .unwrap();

    assert_eq!(row.0, 5); // version unchanged

    cleanup_team(&pool, team_id).await;
}

#[tokio::test]
async fn writer_merges_meta_and_last_seen_instead_of_assigning() {
    let pool = create_test_pool().await;
    let team_id: i32 = 99_006;
    cleanup_team(&pool, team_id).await;

    let writer = PersonWriteStore::new(
        PgStore::new(pool.clone(), TARGET_TABLE.to_string()),
        common::test_store_config(),
        common::test_permits(),
    );

    let fetch = |pool: sqlx::PgPool| async move {
        let row: (i64, Option<String>, Option<DateTime<Utc>>) = sqlx::query_as(
            "SELECT version, properties_last_updated_at::text, last_seen_at
             FROM personhog_person_tmp WHERE team_id = $1 AND id = $2",
        )
        .bind(team_id)
        .bind(1_i64)
        .fetch_one(&pool)
        .await
        .unwrap();
        (row.0, row.1, row.2.map(|t| t.timestamp_millis()))
    };

    // Seed a row carrying meta and a last-seen value, as ingestion's direct
    // writer produces today.
    let mut person = make_person(team_id as i64, 1, 1);
    person.properties_last_updated_at =
        serde_json::to_vec(&serde_json::json!({"email": "2026-01-01T00:00:00Z"})).unwrap();
    person.last_seen_at = Some(5000);
    assert!(matches!(
        writer.upsert_batch(vec![person]).await,
        BatchOutcome::Success
    ));
    let (version, meta, last_seen) = fetch(pool.clone()).await;
    assert_eq!((version, last_seen), (1, Some(5000)));
    let seeded_meta = meta.expect("insert must land the seeded meta");

    // A leader-shaped record: newer version, no meta, older last-seen. The
    // version guard admits it, but it must not erase the meta or lower the
    // last-seen value.
    let mut leader_record = make_person(team_id as i64, 1, 2);
    leader_record.last_seen_at = Some(3000);
    assert!(matches!(
        writer.upsert_batch(vec![leader_record]).await,
        BatchOutcome::Success
    ));
    let (version, meta, last_seen) = fetch(pool.clone()).await;
    assert_eq!(version, 2, "the version guard must admit the newer record");
    assert_eq!(
        meta.as_deref(),
        Some(seeded_meta.as_str()),
        "a record without meta must leave the stored meta in place"
    );
    assert_eq!(
        last_seen,
        Some(5000),
        "a record with an older last-seen must not lower the stored value"
    );

    // A record with no last-seen at all (predating the field) must also
    // leave the stored value in place.
    let no_last_seen = make_person(team_id as i64, 1, 3);
    assert!(matches!(
        writer.upsert_batch(vec![no_last_seen]).await,
        BatchOutcome::Success
    ));
    let (version, _, last_seen) = fetch(pool.clone()).await;
    assert_eq!((version, last_seen), (3, Some(5000)));

    // A record that does carry meta and a newer last-seen replaces both —
    // merging must not freeze the columns at their first values.
    let mut advancing = make_person(team_id as i64, 1, 4);
    advancing.properties_last_updated_at =
        serde_json::to_vec(&serde_json::json!({"email": "2026-02-01T00:00:00Z"})).unwrap();
    advancing.last_seen_at = Some(6000);
    assert!(matches!(
        writer.upsert_batch(vec![advancing]).await,
        BatchOutcome::Success
    ));
    let (version, meta, last_seen) = fetch(pool.clone()).await;
    assert_eq!((version, last_seen), (4, Some(6000)));
    assert_ne!(
        meta.as_deref(),
        Some(seeded_meta.as_str()),
        "a record carrying meta must replace the stored meta"
    );

    cleanup_team(&pool, team_id).await;
}

#[tokio::test]
async fn writer_upsert_batch_multiple_persons() {
    let pool = create_test_pool().await;
    let team_id: i32 = unique_team_id();

    let writer = PersonWriteStore::new(
        PgStore::new(pool.clone(), TARGET_TABLE.to_string()),
        common::test_store_config(),
        common::test_permits(),
    );
    let persons: Vec<Person> = (1..=10)
        .map(|i| make_person(team_id as i64, i, 1))
        .collect();

    assert!(matches!(
        writer.upsert_batch(persons).await,
        BatchOutcome::Success
    ));

    let count: (i64,) =
        sqlx::query_as("SELECT COUNT(*) FROM personhog_person_tmp WHERE team_id = $1")
            .bind(team_id)
            .fetch_one(&pool)
            .await
            .unwrap();

    assert_eq!(count.0, 10);

    cleanup_team(&pool, team_id).await;
}

#[tokio::test]
async fn writer_surfaces_invalid_uuids_as_violations_never_silent_skips() {
    let pool = create_test_pool().await;
    let team_id: i32 = unique_team_id();

    let writer = PersonWriteStore::new(
        PgStore::new(pool.clone(), TARGET_TABLE.to_string()),
        common::test_store_config(),
        common::test_permits(),
    );

    let valid_person = make_person(team_id as i64, 1, 1);
    let mut bad_person = make_person(team_id as i64, 2, 1);
    bad_person.uuid = "not-a-valid-uuid".to_string();
    let another_valid = make_person(team_id as i64, 3, 1);

    // An unbindable row fails its chunk: silently dropping it would
    // permanently diverge PG from the cache and changelog.
    let outcome = writer
        .upsert_batch(vec![valid_person, bad_person, another_valid])
        .await;
    let BatchOutcome::Partial {
        transient,
        saturated,
        data_failed,
    } = outcome
    else {
        panic!("a chunk with an unbindable row must be data-failed");
    };
    assert!(transient.is_empty());
    assert!(saturated.is_empty());
    assert_eq!(data_failed.len(), 3);

    // The per-row pass isolates the poison row as a violation; the valid
    // rows apply.
    let fallback = writer.upsert_rows_parallel(data_failed).await;
    assert!(fallback.transient.is_empty());
    assert_eq!(fallback.violations.len(), 1);
    assert_eq!(fallback.violations[0].person_id, 2);

    let count: (i64,) =
        sqlx::query_as("SELECT COUNT(*) FROM personhog_person_tmp WHERE team_id = $1")
            .bind(team_id)
            .fetch_one(&pool)
            .await
            .unwrap();
    assert_eq!(
        count.0, 2,
        "valid rows apply; the poison row writes nothing"
    );

    cleanup_team(&pool, team_id).await;
}

// ============================================================
// Consumer + Writer: channel flow with mock Kafka
// ============================================================

#[tokio::test]
async fn consumer_flushes_on_buffer_size_threshold() {
    let (mock_cluster, _producer) = create_mock_kafka().await;
    let pool = create_test_pool().await;
    let team_id: i32 = unique_team_id();

    let (flush_tx, flush_rx) = mpsc::channel::<FlushBatch>(2);

    let client_config = ClientConfig::new()
        .set("bootstrap.servers", mock_cluster.bootstrap_servers())
        .set("group.id", "test-flush-threshold")
        .set("auto.offset.reset", "earliest")
        .set("enable.auto.commit", "false")
        .set("enable.auto.offset.store", "false")
        .clone();
    let kafka_consumer = Arc::new(PersonConsumer::new(&client_config, TOPIC.to_string()).unwrap());

    let mut manager = lifecycle::Manager::builder("test")
        .with_trap_signals(false)
        .build();
    let consumer_handle = manager.register(
        "consumer",
        lifecycle::ComponentOptions::new().with_graceful_shutdown(Duration::from_secs(5)),
    );
    let writer_handle = manager.register(
        "writer",
        lifecycle::ComponentOptions::new().with_graceful_shutdown(Duration::from_secs(5)),
    );
    let _monitor = manager.monitor_background();

    // Start writer task
    let writer = PersonWriteStore::new(
        PgStore::new(pool.clone(), TARGET_TABLE.to_string()),
        common::test_store_config(),
        common::test_permits(),
    );
    let writer_task = WriterTask::new(Arc::clone(&kafka_consumer), writer, flush_rx, writer_handle);
    tokio::spawn(async move { writer_task.run().await });

    // Produce 6 messages to mock Kafka (two full batches of 3)
    let producer = &_producer;
    for i in 1..=6 {
        let person = make_person(team_id as i64, i, 1);
        let payload = person.encode_to_vec();
        let key = format!("{}:{}", team_id, i);
        let record = FutureRecord::to(TOPIC).key(&key).payload(&payload);
        producer
            .send_result(record)
            .unwrap()
            .await
            .unwrap()
            .unwrap();
    }

    // Start consumer with flush_buffer_size=3 (flushes at exactly 3 and 6)
    let consumer_task = ConsumerTask::new(
        kafka_consumer,
        vec![flush_tx],
        100,
        Duration::from_secs(60), // long timer so only size triggers flush
        3,                       // flush at 3 messages
        consumer_handle,
    );
    tokio::spawn(async move { consumer_task.run().await });

    // Wait for all 6 rows to appear in PG (two flush cycles)
    let mut retries = 0;
    loop {
        let count: (i64,) =
            sqlx::query_as("SELECT COUNT(*) FROM personhog_person_tmp WHERE team_id = $1")
                .bind(team_id)
                .fetch_one(&pool)
                .await
                .unwrap();

        if count.0 >= 6 {
            break;
        }

        retries += 1;
        if retries > 50 {
            panic!(
                "Expected 6 rows in personhog_person_tmp for team {}, got {}",
                team_id, count.0
            );
        }
        tokio::time::sleep(Duration::from_millis(100)).await;
    }

    cleanup_team(&pool, team_id).await;
}

// ============================================================
// Consumer + Writer: timer-based flush
// ============================================================

#[tokio::test]
async fn consumer_flushes_on_timer() {
    let (mock_cluster, _producer) = create_mock_kafka().await;
    let pool = create_test_pool().await;
    let team_id: i32 = unique_team_id();

    let (flush_tx, flush_rx) = mpsc::channel::<FlushBatch>(2);

    let client_config = ClientConfig::new()
        .set("bootstrap.servers", mock_cluster.bootstrap_servers())
        .set("group.id", "test-flush-timer")
        .set("auto.offset.reset", "earliest")
        .set("enable.auto.commit", "false")
        .set("enable.auto.offset.store", "false")
        .clone();
    let kafka_consumer = Arc::new(PersonConsumer::new(&client_config, TOPIC.to_string()).unwrap());

    let mut manager = lifecycle::Manager::builder("test")
        .with_trap_signals(false)
        .build();
    let consumer_handle = manager.register(
        "consumer",
        lifecycle::ComponentOptions::new().with_graceful_shutdown(Duration::from_secs(5)),
    );
    let writer_handle = manager.register(
        "writer",
        lifecycle::ComponentOptions::new().with_graceful_shutdown(Duration::from_secs(5)),
    );
    let _monitor = manager.monitor_background();

    let writer = PersonWriteStore::new(
        PgStore::new(pool.clone(), TARGET_TABLE.to_string()),
        common::test_store_config(),
        common::test_permits(),
    );
    let writer_task = WriterTask::new(Arc::clone(&kafka_consumer), writer, flush_rx, writer_handle);
    tokio::spawn(async move { writer_task.run().await });

    // Produce 2 messages -- below the size threshold of 1000
    let producer = &_producer;
    for i in 1..=2 {
        let person = make_person(team_id as i64, i, 1);
        let payload = person.encode_to_vec();
        let key = format!("{}:{}", team_id, i);
        let record = FutureRecord::to(TOPIC).key(&key).payload(&payload);
        producer
            .send_result(record)
            .unwrap()
            .await
            .unwrap()
            .unwrap();
    }

    // Start consumer with high size threshold but short timer (500ms)
    let consumer_task = ConsumerTask::new(
        kafka_consumer,
        vec![flush_tx],
        100,
        Duration::from_millis(500), // short timer triggers flush
        1000,                       // high threshold so only timer triggers
        consumer_handle,
    );
    tokio::spawn(async move { consumer_task.run().await });

    // Wait for rows -- should appear within ~1s (timer flush)
    let mut retries = 0;
    loop {
        let count: (i64,) =
            sqlx::query_as("SELECT COUNT(*) FROM personhog_person_tmp WHERE team_id = $1")
                .bind(team_id)
                .fetch_one(&pool)
                .await
                .unwrap();

        if count.0 >= 2 {
            break;
        }

        retries += 1;
        if retries > 30 {
            panic!(
                "Expected 2 rows from timer flush for team {}, got {}",
                team_id, count.0
            );
        }
        tokio::time::sleep(Duration::from_millis(200)).await;
    }

    cleanup_team(&pool, team_id).await;
}

// ============================================================
// Consumer + Writer: dedup under load
// ============================================================

#[tokio::test]
async fn consumer_deduplicates_multiple_updates_for_same_person() {
    let (mock_cluster, _producer) = create_mock_kafka().await;
    let pool = create_test_pool().await;
    let team_id: i32 = unique_team_id();

    let (flush_tx, flush_rx) = mpsc::channel::<FlushBatch>(2);

    let client_config = ClientConfig::new()
        .set("bootstrap.servers", mock_cluster.bootstrap_servers())
        .set("group.id", "test-dedup")
        .set("auto.offset.reset", "earliest")
        .set("enable.auto.commit", "false")
        .set("enable.auto.offset.store", "false")
        .clone();
    let kafka_consumer = Arc::new(PersonConsumer::new(&client_config, TOPIC.to_string()).unwrap());

    let mut manager = lifecycle::Manager::builder("test")
        .with_trap_signals(false)
        .build();
    let consumer_handle = manager.register(
        "consumer",
        lifecycle::ComponentOptions::new().with_graceful_shutdown(Duration::from_secs(5)),
    );
    let writer_handle = manager.register(
        "writer",
        lifecycle::ComponentOptions::new().with_graceful_shutdown(Duration::from_secs(5)),
    );
    let _monitor = manager.monitor_background();

    let writer = PersonWriteStore::new(
        PgStore::new(pool.clone(), TARGET_TABLE.to_string()),
        common::test_store_config(),
        common::test_permits(),
    );
    let writer_task = WriterTask::new(Arc::clone(&kafka_consumer), writer, flush_rx, writer_handle);
    tokio::spawn(async move { writer_task.run().await });

    // Produce 5 updates for the same person (id=1) with increasing versions
    let producer = &_producer;
    for v in 1..=5 {
        let mut person = make_person(team_id as i64, 1, v);
        person.properties = serde_json::to_vec(&serde_json::json!({"version_seen": v})).unwrap();
        let payload = person.encode_to_vec();
        let key = format!("{}:1", team_id);
        let record = FutureRecord::to(TOPIC).key(&key).payload(&payload);
        producer
            .send_result(record)
            .unwrap()
            .await
            .unwrap()
            .unwrap();
    }

    // Start consumer with high size threshold, short timer
    let consumer_task = ConsumerTask::new(
        kafka_consumer,
        vec![flush_tx],
        100,
        Duration::from_millis(500),
        1000,
        consumer_handle,
    );
    tokio::spawn(async move { consumer_task.run().await });

    // Wait for the row to appear
    let mut retries = 0;
    loop {
        let result: Option<(i64, i64)> = sqlx::query_as(
            "SELECT version, id FROM personhog_person_tmp WHERE team_id = $1 AND id = $2",
        )
        .bind(team_id)
        .bind(1_i64)
        .fetch_optional(&pool)
        .await
        .unwrap();

        if let Some((version, _)) = result {
            // Should have the latest version, not an intermediate one
            assert_eq!(version, 5, "expected latest version 5 after dedup");
            break;
        }

        retries += 1;
        if retries > 30 {
            panic!("Row never appeared for deduped person");
        }
        tokio::time::sleep(Duration::from_millis(200)).await;
    }

    // Should be exactly 1 row, not 5
    let count: (i64,) =
        sqlx::query_as("SELECT COUNT(*) FROM personhog_person_tmp WHERE team_id = $1")
            .bind(team_id)
            .fetch_one(&pool)
            .await
            .unwrap();
    assert_eq!(count.0, 1, "expected 1 row after dedup, not 5");

    cleanup_team(&pool, team_id).await;
}

// ============================================================
// Consumer: multi-lane routing, lane-local dedup, independent flush
// ============================================================

/// With two lanes over a two-partition topic, each lane must receive only
/// its own partition's persons and offsets (routing + disjoint commits),
/// dedup within its own buffer, and flush on size without draining the
/// other lane. Asserted on the lane channels directly — the consumer's
/// public output — so no writer or PG is involved.
#[tokio::test]
async fn multi_lane_consumer_routes_partitions_and_flushes_independently() {
    let (mock_cluster, producer) = common::create_mock_kafka_with_partitions(2).await;
    let team_id: i64 = 99_060;

    let (tx0, mut rx0) = mpsc::channel::<FlushBatch>(2);
    let (tx1, mut rx1) = mpsc::channel::<FlushBatch>(2);

    let client_config = ClientConfig::new()
        .set("bootstrap.servers", mock_cluster.bootstrap_servers())
        .set("group.id", "test-multi-lane")
        .set("auto.offset.reset", "earliest")
        .set("enable.auto.commit", "false")
        .set("enable.auto.offset.store", "false")
        .clone();
    let kafka_consumer = Arc::new(PersonConsumer::new(&client_config, TOPIC.to_string()).unwrap());

    let mut manager = lifecycle::Manager::builder("test")
        .with_trap_signals(false)
        .build();
    let consumer_handle = manager.register(
        "consumer",
        lifecycle::ComponentOptions::new().with_graceful_shutdown(Duration::from_secs(5)),
    );
    let _monitor = manager.monitor_background();

    // Partition 0: person 1 twice (dedup) plus persons 2 and 3 — four
    // messages but three unique persons, hitting the size threshold of 3.
    // Partition 1: person 4 only, staying below the threshold.
    let mut to_produce = vec![
        (0, make_person(team_id, 1, 1)),
        (0, make_person(team_id, 1, 2)),
        (0, make_person(team_id, 2, 1)),
        (0, make_person(team_id, 3, 1)),
        (1, make_person(team_id, 4, 1)),
    ];
    for (partition, person) in to_produce.drain(..) {
        let payload = person.encode_to_vec();
        let key = format!("{}:{}", team_id, person.id);
        let record = FutureRecord::to(TOPIC)
            .partition(partition)
            .key(&key)
            .payload(&payload);
        producer
            .send_result(record)
            .unwrap()
            .await
            .unwrap()
            .unwrap();
    }

    let consumer_task = ConsumerTask::new(
        kafka_consumer,
        vec![tx0, tx1],
        100,
        Duration::from_secs(60), // long timer so only size triggers flush
        3,
        consumer_handle,
    );
    tokio::spawn(async move { consumer_task.run().await });

    // Lane 0 flushes on size with only partition 0's persons and offsets,
    // person 1 deduped to its latest version.
    let batch0 = tokio::time::timeout(Duration::from_secs(15), rx0.recv())
        .await
        .expect("lane 0 should flush on size")
        .expect("lane 0 channel should be open");
    assert_eq!(
        batch0.offsets.keys().cloned().collect::<Vec<_>>(),
        vec![source(0)]
    );
    assert_eq!(batch0.offsets[&source(0)], 3); // four messages, offsets 0..=3
    let mut persons0 = batch0.persons;
    persons0.sort_by_key(|p| p.person.id);
    assert_eq!(
        persons0.iter().map(|p| p.person.id).collect::<Vec<_>>(),
        vec![1, 2, 3]
    );
    assert_eq!(
        persons0[0].person.version, 2,
        "dedup must keep the latest version"
    );

    // Lane 1 is below its size threshold and the timer is far away, so a
    // lane-0 flush must not have drained it.
    assert!(
        tokio::time::timeout(Duration::from_millis(500), rx1.recv())
            .await
            .is_err(),
        "lane 1 must not flush before reaching its own threshold"
    );

    // Two more persons on partition 1 push lane 1 to its threshold; its
    // batch carries only partition 1's persons and offsets.
    for id in [5, 6] {
        let person = make_person(team_id, id, 1);
        let payload = person.encode_to_vec();
        let key = format!("{team_id}:{id}");
        let record = FutureRecord::to(TOPIC)
            .partition(1)
            .key(&key)
            .payload(&payload);
        producer
            .send_result(record)
            .unwrap()
            .await
            .unwrap()
            .unwrap();
    }

    let batch1 = tokio::time::timeout(Duration::from_secs(15), rx1.recv())
        .await
        .expect("lane 1 should flush on size")
        .expect("lane 1 channel should be open");
    assert_eq!(
        batch1.offsets.keys().cloned().collect::<Vec<_>>(),
        vec![source(1)]
    );
    assert_eq!(batch1.offsets[&source(1)], 2); // three messages, offsets 0..=2
    let mut ids1: Vec<i64> = batch1.persons.iter().map(|p| p.person.id).collect();
    ids1.sort_unstable();
    assert_eq!(ids1, vec![4, 5, 6]);
}

/// Flushes drain whole partitions up to the flush size, so a batch's
/// committed offsets must cover exactly the partitions of the persons it
/// carries — never a partition whose persons are still buffered. Asserted
/// on the lane channel directly across however many capped batches the
/// consumer emits; arrival interleaving decides the exact split.
#[tokio::test]
async fn capped_drains_commit_only_the_partitions_they_carry() {
    let (mock_cluster, producer) = common::create_mock_kafka_with_partitions(2).await;
    let team_id: i64 = 99_061;

    let (tx, mut rx) = mpsc::channel::<FlushBatch>(2);

    let client_config = ClientConfig::new()
        .set("bootstrap.servers", mock_cluster.bootstrap_servers())
        .set("group.id", "test-capped-drain")
        .set("auto.offset.reset", "earliest")
        .set("enable.auto.commit", "false")
        .set("enable.auto.offset.store", "false")
        .clone();
    let kafka_consumer = Arc::new(PersonConsumer::new(&client_config, TOPIC.to_string()).unwrap());

    let mut manager = lifecycle::Manager::builder("test")
        .with_trap_signals(false)
        .build();
    let consumer_handle = manager.register(
        "consumer",
        lifecycle::ComponentOptions::new().with_graceful_shutdown(Duration::from_secs(5)),
    );
    let _monitor = manager.monitor_background();

    // Person ids encode their partition: 1..=3 on partition 0, 4..=6 on
    // partition 1, so each flushed person maps back to a partition.
    for (partition, id) in [(0, 1), (0, 2), (0, 3), (1, 4), (1, 5), (1, 6)] {
        let person = make_person(team_id, id, 1);
        let payload = person.encode_to_vec();
        let key = format!("{team_id}:{id}");
        let record = FutureRecord::to(TOPIC)
            .partition(partition)
            .key(&key)
            .payload(&payload);
        producer
            .send_result(record)
            .unwrap()
            .await
            .unwrap()
            .unwrap();
    }

    // Flush size 3 caps each drain below the 6 buffered persons; the short
    // timer sweeps up any tail the size trigger leaves behind.
    let consumer_task = ConsumerTask::new(
        kafka_consumer,
        vec![tx],
        100,
        Duration::from_secs(2),
        3,
        consumer_handle,
    );
    tokio::spawn(async move { consumer_task.run().await });

    let mut seen_persons = 0;
    let mut batches = 0;
    let mut max_offset_seen: HashMap<i32, i64> = HashMap::new();
    while seen_persons < 6 {
        let batch = tokio::time::timeout(Duration::from_secs(15), rx.recv())
            .await
            .expect("expected another capped batch")
            .expect("consumer channel should be open");

        let partitions_in_batch: std::collections::HashSet<i32> = batch
            .persons
            .iter()
            .map(|p| if p.person.id <= 3 { 0 } else { 1 })
            .collect();
        let committed: std::collections::HashSet<i32> =
            batch.offsets.keys().map(|s| s.partition).collect();
        assert_eq!(
            committed, partitions_in_batch,
            "a batch's offsets must cover exactly the partitions it drained"
        );

        for (source, offset) in &batch.offsets {
            let entry = max_offset_seen.entry(source.partition).or_insert(-1);
            *entry = (*entry).max(*offset);
        }
        seen_persons += batch.persons.len();
        batches += 1;
    }

    assert!(batches >= 2, "a capped drain must split 6 rows at cap 3");
    // Three messages per partition, offsets 0..=2, all eventually committed.
    assert_eq!(max_offset_seen[&0], 2);
    assert_eq!(max_offset_seen[&1], 2);
}

async fn wait_until(limit: Duration, mut ready: impl FnMut() -> bool) -> Result<(), ()> {
    let deadline = tokio::time::Instant::now() + limit;
    while !ready() {
        if tokio::time::Instant::now() >= deadline {
            return Err(());
        }
        tokio::time::sleep(Duration::from_millis(100)).await;
    }
    Ok(())
}

fn committed_offset(tpl: &TopicPartitionList, partition: i32) -> Option<i64> {
    tpl.elements()
        .iter()
        .find(|e| e.partition() == partition)
        .and_then(|e| match e.offset() {
            Offset::Offset(offset) => Some(offset),
            _ => None,
        })
}

/// A second member joining revokes one partition from the first, which must
/// flush only the partition it kept and leave the revoked one uncommitted.
#[tokio::test]
async fn revoked_partitions_are_dropped_and_their_offsets_stay_uncommitted() {
    let (mock_cluster, producer) = common::create_mock_kafka_with_partitions(2).await;
    let team_id: i64 = 99_062;

    let member_config = |client_id: &str| {
        ClientConfig::new()
            .set("bootstrap.servers", mock_cluster.bootstrap_servers())
            .set("group.id", "test-revoke")
            .set("client.id", client_id)
            .set("auto.offset.reset", "earliest")
            .set("enable.auto.commit", "false")
            .set("enable.auto.offset.store", "false")
            .set("partition.assignment.strategy", "cooperative-sticky")
            .set("session.timeout.ms", "6000")
            .set("heartbeat.interval.ms", "1000")
            .clone()
    };

    // Two persons per partition, produced before the first member joins so
    // it reads them as soon as it is assigned. Ids encode the partition.
    for (partition, id) in [(0, 1), (0, 2), (1, 3), (1, 4)] {
        let person = make_person(team_id, id, 1);
        let payload = person.encode_to_vec();
        let key = format!("{team_id}:{id}");
        let record = FutureRecord::to(TOPIC)
            .partition(partition)
            .key(&key)
            .payload(&payload);
        producer
            .send_result(record)
            .unwrap()
            .await
            .unwrap()
            .unwrap();
    }

    let first = Arc::new(PersonConsumer::new(&member_config("first"), TOPIC.to_string()).unwrap());
    let (tx, mut rx) = mpsc::channel::<FlushBatch>(4);
    let mut manager = lifecycle::Manager::builder("test")
        .with_trap_signals(false)
        .build();
    let consumer_handle = manager.register(
        "consumer",
        lifecycle::ComponentOptions::new().with_graceful_shutdown(Duration::from_secs(5)),
    );
    let shutdown = consumer_handle.clone();
    let _monitor = manager.monitor_background();

    // A timer far beyond the test and a size threshold above the row count:
    // only the shutdown flush drains the buffer.
    let consumer_task = ConsumerTask::new(
        Arc::clone(&first),
        vec![tx],
        100,
        Duration::from_secs(600),
        100,
        consumer_handle,
    );
    tokio::spawn(async move { consumer_task.run().await });

    wait_until(Duration::from_secs(20), || {
        first
            .positions()
            .map(|p| p.get(&source(0)) == Some(&2) && p.get(&source(1)) == Some(&2))
            .unwrap_or(false)
    })
    .await
    .expect("the first member should consume both partitions");

    // The second member joins and keeps polling so the rebalance completes.
    let second: Arc<StreamConsumer> = Arc::new(member_config("second").create().unwrap());
    second.subscribe(&[TOPIC]).unwrap();
    {
        let second = Arc::clone(&second);
        tokio::spawn(async move { while second.recv().await.is_ok() {} });
    }
    wait_until(Duration::from_secs(20), || {
        second.assignment().map(|a| a.count() == 1).unwrap_or(false)
    })
    .await
    .expect("the second member should take one partition");
    let revoked = second.assignment().unwrap().elements()[0].partition();
    let kept = 1 - revoked;

    shutdown.request_shutdown();
    let mut flushed_ids = Vec::new();
    let mut flushed_partitions = HashSet::new();
    while let Ok(Some(batch)) = tokio::time::timeout(Duration::from_secs(10), rx.recv()).await {
        flushed_ids.extend(batch.persons.iter().map(|p| p.person.id));
        flushed_partitions.extend(batch.offsets.keys().map(|s| s.partition));
    }
    flushed_ids.sort_unstable();
    let kept_ids = if kept == 0 { vec![1, 2] } else { vec![3, 4] };
    assert_eq!(
        flushed_ids, kept_ids,
        "only the kept partition's rows may flush"
    );
    assert_eq!(flushed_partitions, HashSet::from([kept]));

    first
        .commit_offsets(&HashMap::from([(source(kept), 1), (source(revoked), 1)]))
        .unwrap();
    let mut probe = TopicPartitionList::new();
    probe.add_partition(TOPIC, kept);
    probe.add_partition(TOPIC, revoked);
    wait_until(Duration::from_secs(10), || {
        second
            .committed_offsets(probe.clone(), Duration::from_secs(2))
            .map(|c| committed_offset(&c, kept) == Some(2))
            .unwrap_or(false)
    })
    .await
    .expect("the kept partition's commit should land");
    let committed = second
        .committed_offsets(probe.clone(), Duration::from_secs(2))
        .unwrap();
    assert_eq!(
        committed_offset(&committed, revoked),
        None,
        "the revoked partition must stay uncommitted"
    );
}

const NEXT_TOPIC: &str = "personhog_updates_g2";

fn plain_member_config(bootstrap: &str, group: &str) -> ClientConfig {
    ClientConfig::new()
        .set("bootstrap.servers", bootstrap)
        .set("group.id", group)
        .set("auto.offset.reset", "earliest")
        .set("enable.auto.commit", "false")
        .set("enable.auto.offset.store", "false")
        .set("session.timeout.ms", "6000")
        .set("heartbeat.interval.ms", "1000")
        .clone()
}

/// After a resubscribe the assignment reuses the partition numbers, so an
/// old-topic batch still in flight must never commit onto the next topic.
#[tokio::test]
async fn a_late_batch_never_commits_its_offsets_onto_the_next_topic() {
    let (mock_cluster, _producer) = common::create_mock_kafka_with_partitions(2).await;
    mock_cluster
        .create_topic(NEXT_TOPIC, 2, 1)
        .expect("create the next generation");
    let bootstrap = mock_cluster.bootstrap_servers();
    let member = plain_member_config(&bootstrap, "test-late-batch");

    let consumer = Arc::new(PersonConsumer::new(&member, TOPIC.to_string()).unwrap());
    // Keeps the client polled, so rebalances land.
    {
        let consumer = Arc::clone(&consumer);
        tokio::spawn(async move {
            loop {
                drop(consumer.recv().await);
            }
        });
    }
    let assigned_to = |consumer: &PersonConsumer, topic: &str| {
        consumer
            .assigned_partitions()
            .ok()
            .flatten()
            .is_some_and(|a| a == HashSet::from([source_on(topic, 0), source_on(topic, 1)]))
    };
    wait_until(Duration::from_secs(20), || assigned_to(&consumer, TOPIC))
        .await
        .expect("the member owns the first topic");

    let probe: StreamConsumer = plain_member_config(&bootstrap, "test-late-batch")
        .create()
        .unwrap();
    let mut both = TopicPartitionList::new();
    for topic in [TOPIC, NEXT_TOPIC] {
        both.add_partition(topic, 0);
        both.add_partition(topic, 1);
    }
    let committed_on = |topic: &str, partition: i32| {
        probe
            .committed_offsets(both.clone(), Duration::from_secs(2))
            .ok()
            .and_then(|c| {
                c.elements()
                    .iter()
                    .find(|e| e.topic() == topic && e.partition() == partition)
                    .and_then(|e| match e.offset() {
                        Offset::Offset(offset) => Some(offset),
                        _ => None,
                    })
            })
    };

    consumer
        .commit_offsets(&HashMap::from([(source(0), 1)]))
        .unwrap();
    wait_until(Duration::from_secs(10), || {
        committed_on(TOPIC, 0) == Some(2)
    })
    .await
    .expect("the first topic's commit lands");

    consumer.resubscribe(NEXT_TOPIC.to_string()).unwrap();
    wait_until(Duration::from_secs(20), || {
        assigned_to(&consumer, NEXT_TOPIC)
    })
    .await
    .expect("the member moves to the next topic");

    // Commits are sequenced on the client: the live one landing proves the late one was handled.
    consumer
        .commit_offsets(&HashMap::from([(source(1), 5)]))
        .unwrap();
    consumer
        .commit_offsets(&HashMap::from([(source_on(NEXT_TOPIC, 0), 0)]))
        .unwrap();
    wait_until(Duration::from_secs(10), || {
        committed_on(NEXT_TOPIC, 0) == Some(1)
    })
    .await
    .expect("the next topic's commit lands");

    assert_eq!(
        committed_on(NEXT_TOPIC, 1),
        None,
        "an old-topic batch must not commit onto the next topic"
    );
    assert_eq!(
        committed_on(TOPIC, 1),
        None,
        "a batch for a partition no longer assigned is not committed"
    );
    assert_eq!(committed_on(TOPIC, 0), Some(2));
}

/// The draining flag follows the subscribed topic, not the move: a writer
/// already on the topic when it was sealed raises it too.
#[tokio::test]
async fn the_draining_flag_follows_the_subscribed_topic_not_the_move() {
    let (mock_cluster, _producer) = common::create_mock_kafka_with_partitions(1).await;
    mock_cluster
        .create_topic(NEXT_TOPIC, 1, 1)
        .expect("create the next generation");
    let member = plain_member_config(&mock_cluster.bootstrap_servers(), "test-follow");
    let consumer = PersonConsumer::new(&member, TOPIC.to_string()).unwrap();
    let flipped = ChangelogConfig {
        cluster: None,
        topic: NEXT_TOPIC.to_string(),
        partitions: 1,
        epoch: 2,
        writer_group: "test-follow".to_string(),
        retire_groups: vec![],
        draining: vec![Generation {
            cluster: None,
            topic: TOPIC.to_string(),
            partitions: 1,
            epoch: 1,
        }],
    };

    let mut follower = Follower::default();
    follower
        .follow_once(&consumer, &flipped, |_| async { Ok(false) })
        .await
        .unwrap();
    assert_eq!(consumer.topic(), TOPIC);
    assert!(
        consumer.is_draining(),
        "the subscribed topic is a sealed generation"
    );

    follower
        .follow_once(&consumer, &flipped, |_| async { Ok(true) })
        .await
        .unwrap();
    assert_eq!(consumer.topic(), NEXT_TOPIC);
    assert!(!consumer.is_draining());

    // A later probe that cannot vouch for the old generation never sends it back.
    follower
        .follow_once(&consumer, &flipped, |_| async { Ok(false) })
        .await
        .unwrap();
    assert_eq!(consumer.topic(), NEXT_TOPIC);
}

// ============================================================
// Writer task: signals completion through channel
// ============================================================

#[tokio::test]
async fn writer_processes_batch_from_channel() {
    let pool = create_test_pool().await;
    let team_id: i32 = unique_team_id();

    let (flush_tx, flush_rx) = mpsc::channel::<FlushBatch>(2);

    let (mock_cluster, _) = create_mock_kafka().await;
    let mut client_config = ClientConfig::new();
    client_config
        .set("bootstrap.servers", mock_cluster.bootstrap_servers())
        .set("group.id", "test-writer-channel")
        .set("enable.auto.commit", "false")
        .set("enable.auto.offset.store", "false");
    let kafka_consumer = Arc::new(PersonConsumer::new(&client_config, TOPIC.to_string()).unwrap());

    let mut manager = lifecycle::Manager::builder("test")
        .with_trap_signals(false)
        .build();
    let writer_handle = manager.register("writer", lifecycle::ComponentOptions::new());
    let _monitor = manager.monitor_background();

    let writer = PersonWriteStore::new(
        PgStore::new(pool.clone(), TARGET_TABLE.to_string()),
        common::test_store_config(),
        common::test_permits(),
    );
    let writer_task = WriterTask::new(Arc::clone(&kafka_consumer), writer, flush_rx, writer_handle);
    tokio::spawn(async move { writer_task.run().await });

    // Send a batch directly through the channel
    let person = make_person(team_id as i64, 1, 1);
    let batch = FlushBatch {
        persons: rows(vec![person]),
        offsets: HashMap::new(),
        oldest_message_ts_ms: None,
    };
    flush_tx.send(batch).await.unwrap();

    // Wait for the row to appear
    let mut retries = 0;
    loop {
        let count: (i64,) = sqlx::query_as(
            "SELECT COUNT(*) FROM personhog_person_tmp WHERE team_id = $1 AND id = $2",
        )
        .bind(team_id)
        .bind(1_i64)
        .fetch_one(&pool)
        .await
        .unwrap();

        if count.0 == 1 {
            break;
        }

        retries += 1;
        if retries > 50 {
            panic!("Row never appeared in personhog_person_tmp");
        }
        tokio::time::sleep(Duration::from_millis(100)).await;
    }

    cleanup_team(&pool, team_id).await;
}

// ============================================================
// Writer task: failure and recovery (mock-based)
// ============================================================

/// Mock DB that fails chunks a configurable number of times, then succeeds.
/// Rows succeed unless `row_error_kind` is set. Exercises the full
/// orchestration layer above it.
struct MockDb {
    chunk_remaining_failures: std::sync::atomic::AtomicU32,
    chunk_error_kind: WriteErrorKind,
    row_error_kind: Option<WriteErrorKind>,
}

#[async_trait::async_trait]
impl PersonDb for MockDb {
    async fn execute_chunk(&self, _chunk: &[Person]) -> Result<(), WriteError> {
        let remaining = self.chunk_remaining_failures.fetch_update(
            std::sync::atomic::Ordering::SeqCst,
            std::sync::atomic::Ordering::SeqCst,
            |v| if v > 0 { Some(v - 1) } else { None },
        );
        if remaining.is_ok() {
            Err(WriteError {
                message: "mock failure".to_string(),
                kind: self.chunk_error_kind,
            })
        } else {
            Ok(())
        }
    }

    async fn execute_row(&self, _person: &Person) -> Result<(), WriteError> {
        match self.row_error_kind {
            Some(kind) => Err(WriteError {
                message: "mock row failure".to_string(),
                kind,
            }),
            None => Ok(()),
        }
    }
}

#[tokio::test]
async fn writer_crashes_after_exhausting_transient_retries() {
    let (flush_tx, flush_rx) = mpsc::channel::<FlushBatch>(4);

    let (mock_cluster, _) = create_mock_kafka().await;
    let mut client_config = ClientConfig::new();
    client_config
        .set("bootstrap.servers", mock_cluster.bootstrap_servers())
        .set("group.id", "test-writer-crash")
        .set("enable.auto.commit", "false")
        .set("enable.auto.offset.store", "false");
    let kafka_consumer = Arc::new(PersonConsumer::new(&client_config, TOPIC.to_string()).unwrap());

    let mut manager = lifecycle::Manager::builder("test")
        .with_trap_signals(false)
        .build();
    let writer_handle = manager.register("writer", lifecycle::ComponentOptions::new());
    let monitor = manager.monitor_background();

    // Always fail with transient errors
    let mock_db = MockDb {
        chunk_remaining_failures: std::sync::atomic::AtomicU32::new(100),
        chunk_error_kind: WriteErrorKind::Transient,
        row_error_kind: None,
    };
    let store = PersonWriteStore::new(
        mock_db,
        personhog_writer::store::StoreConfig {
            chunk_size: 10,
            row_fallback_concurrency: 4,
        },
        common::test_permits(),
    );

    let writer_task = WriterTask::new(Arc::clone(&kafka_consumer), store, flush_rx, writer_handle);
    tokio::spawn(async move { writer_task.run().await });

    let batch = FlushBatch {
        persons: rows(vec![make_person(99_040, 1, 1)]),
        offsets: HashMap::new(),
        oldest_message_ts_ms: None,
    };
    flush_tx.send(batch).await.unwrap();

    // Should crash within backoff time (1s + 2s + 4s = 7s + overhead)
    let result = tokio::time::timeout(Duration::from_secs(15), monitor.wait()).await;
    assert!(result.is_ok(), "lifecycle should shut down within timeout");
    assert!(
        result.unwrap().is_err(),
        "lifecycle should report component failure"
    );
}

#[tokio::test]
async fn writer_recovers_on_transient_retry() {
    let (flush_tx, flush_rx) = mpsc::channel::<FlushBatch>(4);

    let (mock_cluster, _) = create_mock_kafka().await;
    let mut client_config = ClientConfig::new();
    client_config
        .set("bootstrap.servers", mock_cluster.bootstrap_servers())
        .set("group.id", "test-writer-recover")
        .set("enable.auto.commit", "false")
        .set("enable.auto.offset.store", "false");
    let kafka_consumer = Arc::new(PersonConsumer::new(&client_config, TOPIC.to_string()).unwrap());

    let mut manager = lifecycle::Manager::builder("test")
        .with_trap_signals(false)
        .build();
    let writer_handle = manager.register("writer", lifecycle::ComponentOptions::new());
    let _monitor = manager.monitor_background();

    // Fail once, then succeed
    let mock_db = MockDb {
        chunk_remaining_failures: std::sync::atomic::AtomicU32::new(1),
        chunk_error_kind: WriteErrorKind::Transient,
        row_error_kind: None,
    };
    let store = PersonWriteStore::new(
        mock_db,
        personhog_writer::store::StoreConfig {
            chunk_size: 10,
            row_fallback_concurrency: 4,
        },
        common::test_permits(),
    );

    let writer_task = WriterTask::new(Arc::clone(&kafka_consumer), store, flush_rx, writer_handle);
    tokio::spawn(async move { writer_task.run().await });

    let batch = FlushBatch {
        persons: rows(vec![make_person(99_041, 1, 1)]),
        offsets: HashMap::new(),
        oldest_message_ts_ms: None,
    };
    flush_tx.send(batch).await.unwrap();

    // Send a second batch to verify the writer is still alive after recovery
    tokio::time::sleep(Duration::from_secs(3)).await;
    let batch2 = FlushBatch {
        persons: rows(vec![make_person(99_041, 2, 1)]),
        offsets: HashMap::new(),
        oldest_message_ts_ms: None,
    };
    assert!(
        flush_tx.send(batch2).await.is_ok(),
        "writer should still be alive after transient recovery"
    );
}

#[tokio::test]
async fn writer_falls_back_to_per_row_on_data_error() {
    let (flush_tx, flush_rx) = mpsc::channel::<FlushBatch>(4);

    let (mock_cluster, _) = create_mock_kafka().await;
    let mut client_config = ClientConfig::new();
    client_config
        .set("bootstrap.servers", mock_cluster.bootstrap_servers())
        .set("group.id", "test-writer-fallback")
        .set("enable.auto.commit", "false")
        .set("enable.auto.offset.store", "false");
    let kafka_consumer = Arc::new(PersonConsumer::new(&client_config, TOPIC.to_string()).unwrap());

    let mut manager = lifecycle::Manager::builder("test")
        .with_trap_signals(false)
        .build();
    let writer_handle = manager.register("writer", lifecycle::ComponentOptions::new());
    let _monitor = manager.monitor_background();

    // Batch always fails with data error, per-row always succeeds
    let mock_db = MockDb {
        chunk_remaining_failures: std::sync::atomic::AtomicU32::new(100),
        chunk_error_kind: WriteErrorKind::Data,
        row_error_kind: None,
    };
    let store = PersonWriteStore::new(
        mock_db,
        personhog_writer::store::StoreConfig {
            chunk_size: 10,
            row_fallback_concurrency: 4,
        },
        common::test_permits(),
    );

    let writer_task = WriterTask::new(Arc::clone(&kafka_consumer), store, flush_rx, writer_handle);
    tokio::spawn(async move { writer_task.run().await });

    let batch = FlushBatch {
        persons: rows(vec![make_person(99_042, 1, 1), make_person(99_042, 2, 1)]),
        offsets: HashMap::new(),
        oldest_message_ts_ms: None,
    };
    flush_tx.send(batch).await.unwrap();

    // Writer should handle the data error via fallback and stay alive
    tokio::time::sleep(Duration::from_secs(1)).await;
    let batch2 = FlushBatch {
        persons: rows(vec![make_person(99_042, 3, 1)]),
        offsets: HashMap::new(),
        oldest_message_ts_ms: None,
    };
    assert!(
        flush_tx.send(batch2).await.is_ok(),
        "writer should still be alive after data error fallback"
    );
}

/// After a batch with unapplyable rows signals failure, the writer must
/// stop receiving entirely: Kafka offset commits are cumulative per
/// partition, so processing a later batch and committing its offsets
/// would silently skip the failed batch's uncommitted rows after restart.
/// The observable halt is the flush channel closing.
#[tokio::test]
async fn unapplyable_batch_halts_the_writer_before_any_later_batch() {
    let (flush_tx, flush_rx) = mpsc::channel::<FlushBatch>(4);

    let (mock_cluster, _) = create_mock_kafka().await;
    let mut client_config = ClientConfig::new();
    client_config
        .set("bootstrap.servers", mock_cluster.bootstrap_servers())
        .set("group.id", "test-writer-halt")
        .set("enable.auto.commit", "false")
        .set("enable.auto.offset.store", "false");
    let kafka_consumer = Arc::new(PersonConsumer::new(&client_config, TOPIC.to_string()).unwrap());

    let mut manager = lifecycle::Manager::builder("test")
        .with_trap_signals(false)
        .build();
    let writer_handle = manager.register("writer", lifecycle::ComponentOptions::new());
    let monitor = manager.monitor_background();

    // The first chunk fails with a data error and its per-row fallback
    // fails too — an unapplyable batch despite leader admission. Later
    // chunks would succeed, so a writer that kept going would commit the
    // buffered batch's offsets past the failed one.
    let mock_db = MockDb {
        chunk_remaining_failures: std::sync::atomic::AtomicU32::new(1),
        chunk_error_kind: WriteErrorKind::Data,
        row_error_kind: Some(WriteErrorKind::PropertiesSizeViolation),
    };
    let store = PersonWriteStore::new(
        mock_db,
        personhog_writer::store::StoreConfig {
            chunk_size: 10,
            row_fallback_concurrency: 4,
        },
        common::test_permits(),
    );

    let writer_task = WriterTask::new(Arc::clone(&kafka_consumer), store, flush_rx, writer_handle);
    tokio::spawn(async move { writer_task.run().await });

    // A second batch is already buffered behind the poison one — the
    // regression this guards is the writer pulling and committing it.
    let poison = FlushBatch {
        persons: rows(vec![make_person(99_043, 1, 1)]),
        offsets: HashMap::new(),
        oldest_message_ts_ms: None,
    };
    let buffered = FlushBatch {
        persons: rows(vec![make_person(99_043, 2, 1)]),
        offsets: HashMap::new(),
        oldest_message_ts_ms: None,
    };
    flush_tx.send(poison).await.unwrap();
    flush_tx.send(buffered).await.unwrap();

    // The writer must drop its receiver without consuming the buffered
    // batch, and the lifecycle must report the failure.
    tokio::time::timeout(Duration::from_secs(5), flush_tx.closed())
        .await
        .expect("writer should stop receiving after an unapplyable batch");
    let result = tokio::time::timeout(Duration::from_secs(5), monitor.wait())
        .await
        .expect("lifecycle should shut down");
    assert!(result.is_err(), "lifecycle should report component failure");
}

// ============================================================
// Properties size violation: invariant violation, never corrected
// ============================================================

/// A row violating check_properties_size cannot reach the writer through
/// a correctly-admitting leader, so the writer treats one as an admission
/// bug: a non-transient error that halts the flush without committing and
/// writes nothing. A writer-side trim or skip would make Postgres diverge
/// from the cache and changelog — the exact drift leader admission exists
/// to prevent.
#[tokio::test]
async fn properties_size_violation_errors_and_writes_nothing() {
    let pool = create_test_pool().await;
    let team_id: i32 = unique_team_id();

    let store = PersonWriteStore::new(
        PgStore::new(pool.clone(), TARGET_TABLE.to_string()),
        common::test_store_config(),
        common::test_permits(),
    );

    // Would have been trimmable under the old semantics: protected email
    // is small, two large custom properties push past the constraint.
    let mut props = serde_json::Map::new();
    props.insert(
        "email".to_string(),
        serde_json::json!("[email protected]"),
    );
    let big_value = "x".repeat(400_000);
    props.insert("custom_a".to_string(), serde_json::json!(big_value));
    props.insert("custom_b".to_string(), serde_json::json!(big_value));

    let mut person = make_person(team_id as i64, 1, 1);
    person.properties = serde_json::to_vec(&serde_json::Value::Object(props)).unwrap();

    let err = store
        .upsert_row(&person)
        .await
        .expect_err("an unapplyable row must error, never be skipped");
    assert!(matches!(err.kind, WriteErrorKind::PropertiesSizeViolation));

    let count: (i64,) =
        sqlx::query_as("SELECT COUNT(*) FROM personhog_person_tmp WHERE team_id = $1 AND id = $2")
            .bind(team_id)
            .bind(1_i64)
            .fetch_one(&pool)
            .await
            .unwrap();
    assert_eq!(count.0, 0, "an unapplyable person must write nothing");

    cleanup_team(&pool, team_id).await;
}

// ============================================================
// E2E: real Kafka + real PG
// ============================================================

#[tokio::test]
async fn e2e_produce_to_kafka_and_verify_pg_write() {
    let pool = create_test_pool().await;
    let team_id: i32 = unique_team_id();

    // Produce a Person proto to local Kafka
    let producer = create_local_kafka_producer().await;
    let person = make_person(team_id as i64, 42, 1);
    let payload = person.encode_to_vec();
    let key = format!("{team_id}:42");
    let record = FutureRecord::to(TOPIC).key(&key).payload(&payload);
    producer
        .send_result(record)
        .unwrap()
        .await
        .unwrap()
        .unwrap();

    // Set up the full consumer + writer pipeline against local Kafka
    let (flush_tx, flush_rx) = mpsc::channel::<FlushBatch>(2);

    let client_config = ClientConfig::new()
        .set("bootstrap.servers", KAFKA_BOOTSTRAP)
        .set(
            "group.id",
            format!("e2e-pg-writer-{}", uuid::Uuid::new_v4()),
        )
        .set("auto.offset.reset", "earliest")
        .set("enable.auto.commit", "false")
        .set("enable.auto.offset.store", "false")
        .clone();
    let kafka_consumer = Arc::new(PersonConsumer::new(&client_config, TOPIC.to_string()).unwrap());

    let mut manager = lifecycle::Manager::builder("test")
        .with_trap_signals(false)
        .build();
    let consumer_handle = manager.register(
        "consumer",
        lifecycle::ComponentOptions::new().with_graceful_shutdown(Duration::from_secs(5)),
    );
    let writer_handle = manager.register(
        "writer",
        lifecycle::ComponentOptions::new().with_graceful_shutdown(Duration::from_secs(5)),
    );
    let _monitor = manager.monitor_background();

    let writer = PersonWriteStore::new(
        PgStore::new(pool.clone(), TARGET_TABLE.to_string()),
        common::test_store_config(),
        common::test_permits(),
    );
    let writer_task = WriterTask::new(Arc::clone(&kafka_consumer), writer, flush_rx, writer_handle);
    tokio::spawn(async move { writer_task.run().await });

    let consumer_task = ConsumerTask::new(
        kafka_consumer,
        vec![flush_tx],
        50000,
        Duration::from_millis(500), // fast flush for test
        1,                          // flush after every message
        consumer_handle,
    );
    tokio::spawn(async move { consumer_task.run().await });

    // Wait for the row to appear in PG
    let mut retries = 0;
    loop {
        let result: Option<(i64, i64)> = sqlx::query_as(
            "SELECT id, version FROM personhog_person_tmp WHERE team_id = $1 AND id = $2",
        )
        .bind(team_id)
        .bind(42_i64)
        .fetch_optional(&pool)
        .await
        .unwrap();

        if let Some((id, version)) = result {
            assert_eq!(id, 42);
            assert_eq!(version, 1);
            break;
        }

        retries += 1;
        if retries > 100 {
            panic!("Row never appeared in personhog_person_tmp for team {team_id}, person 42");
        }
        tokio::time::sleep(Duration::from_millis(100)).await;
    }

    cleanup_team(&pool, team_id).await;
}