Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

11.2 KB
Raw Blame
//! Integration test for person processing filter with Kafka emission
//!
//! Tests that when FORCE_DISABLE_PERSON_PROCESSING env var is set,
//! events matching token:distinct_id pairs are sent to Kafka without a key
//! and with force_disable_person_processing header set to true.
//!
//! Requires Kafka running at localhost:9092. Skips if Kafka is unreachable.

use anyhow::Result;
use batch_import_worker::{
    config::Config,
    context::AppContext,
    emit::{kafka::KafkaEmitter, Emitter},
    job::config::KafkaEmitterConfig,
};
use common_types::{CapturedEvent, CapturedEventHeaders, InternallyCapturedEvent};
use envconfig::Envconfig;
use rdkafka::{
    admin::{AdminClient, AdminOptions, NewTopic, TopicReplication},
    config::ClientConfig,
    consumer::{Consumer, StreamConsumer},
    message::Message,
};
use std::sync::{Arc, OnceLock};
use std::time::Duration;
use tokio::sync::Mutex as TokioMutex;
use uuid::Uuid;

const KAFKA_BROKERS: &str = "localhost:9092";

// Global mutex to serialize Kafka integration tests
static KAFKA_TEST_MUTEX: OnceLock<TokioMutex<()>> = OnceLock::new();

/// Helper to create Kafka topics before tests
async fn create_kafka_topic(topic: &str) -> Result<()> {
    let admin_client: AdminClient<_> = ClientConfig::new()
        .set("bootstrap.servers", KAFKA_BROKERS)
        .create()?;

    let new_topic = NewTopic::new(topic, 1, TopicReplication::Fixed(1));
    let opts = AdminOptions::new().operation_timeout(Some(Duration::from_secs(10)));

    match admin_client.create_topics(&[new_topic], &opts).await {
        Ok(results) => {
            for result in results {
                match result {
                    Ok(topic) => println!("Created topic: {topic}"),
                    Err((topic, error)) => {
                        println!("Topic {topic} result: {error:?}");
                    }
                }
            }
        }
        Err(e) => {
            println!("Failed to create topic: {e:?}");
        }
    }

    Ok(())
}

/// Create test events
fn create_test_events() -> Vec<InternallyCapturedEvent> {
    vec![
        InternallyCapturedEvent {
            inner: CapturedEvent {
                uuid: Uuid::new_v4(),
                distinct_id: "user1".to_string(),
                session_id: None,
                ip: "127.0.0.1".to_string(),
                data: r#"{"event":"test_event","properties":{}}"#.to_string(),
                now: "2024-01-01T00:00:00Z".to_string(),
                sent_at: None,
                token: "token1".to_string(),
                event: "test_event".to_string(),
                timestamp: chrono::Utc::now(),
                is_cookieless_mode: false,
                historical_migration: false,
            },
            team_id: 1,
        },
        InternallyCapturedEvent {
            inner: CapturedEvent {
                uuid: Uuid::new_v4(),
                distinct_id: "user2".to_string(),
                session_id: None,
                ip: "127.0.0.1".to_string(),
                data: r#"{"event":"test_event","properties":{}}"#.to_string(),
                now: "2024-01-01T00:00:00Z".to_string(),
                sent_at: None,
                token: "token1".to_string(),
                event: "test_event".to_string(),
                timestamp: chrono::Utc::now(),
                is_cookieless_mode: false,
                historical_migration: false,
            },
            team_id: 1,
        },
        InternallyCapturedEvent {
            inner: CapturedEvent {
                uuid: Uuid::new_v4(),
                distinct_id: "user3".to_string(),
                session_id: None,
                ip: "127.0.0.1".to_string(),
                data: r#"{"event":"test_event","properties":{}}"#.to_string(),
                now: "2024-01-01T00:00:00Z".to_string(),
                sent_at: None,
                token: "token2".to_string(),
                event: "test_event".to_string(),
                timestamp: chrono::Utc::now(),
                is_cookieless_mode: false,
                historical_migration: false,
            },
            team_id: 1,
        },
    ]
}

/// Consume messages from Kafka topic and return them with their keys, headers, and payload
async fn consume_messages_with_metadata(
    topic: &str,
    expected_count: usize,
    timeout: Duration,
) -> Result<Vec<(Option<String>, CapturedEventHeaders, String)>> {
    let consumer: StreamConsumer = ClientConfig::new()
        .set("bootstrap.servers", KAFKA_BROKERS)
        .set("group.id", format!("test-{}", Uuid::new_v4()))
        .set("auto.offset.reset", "earliest")
        .set("enable.auto.commit", "false")
        .create()?;

    consumer.subscribe(&[topic])?;

    let mut messages = Vec::new();
    let start = std::time::Instant::now();

    while start.elapsed() < timeout && messages.len() < expected_count {
        match tokio::time::timeout(Duration::from_millis(100), consumer.recv()).await {
            Ok(Ok(msg)) => {
                let key = msg.key().map(|k| String::from_utf8_lossy(k).to_string());
                let headers = msg
                    .headers()
                    .map(|h| CapturedEventHeaders::from(h.detach()));
                let payload = msg
                    .payload()
                    .map(|p| String::from_utf8_lossy(p).to_string())
                    .unwrap_or_default();

                if let Some(headers) = headers {
                    messages.push((key, headers, payload));
                }
            }
            Ok(Err(e)) => {
                if !matches!(e, rdkafka::error::KafkaError::NoMessageReceived) {
                    println!("Error consuming message: {e:?}");
                }
            }
            Err(_) => {}
        }
    }

    Ok(messages)
}

#[tokio::test]
async fn test_person_processing_filter_with_kafka() -> Result<()> {
    let _guard = KAFKA_TEST_MUTEX
        .get_or_init(|| TokioMutex::new(()))
        .lock()
        .await;

    let topic = format!("test-person-processing-{}", Uuid::new_v4());
    create_kafka_topic(&topic).await?;

    // Create a mock config with the person processing filter
    std::env::set_var(
        "FORCE_DISABLE_PERSON_PROCESSING",
        "token1:user1,token2:user3",
    );
    std::env::set_var("KAFKA_HOSTS", KAFKA_BROKERS);
    std::env::set_var(
        "DATABASE_URL",
        "postgres://posthog:posthog@localhost:5432/posthog",
    );

    let config = Config::init_from_env()?;
    let context = Arc::new(AppContext::new(&config).await?);

    // Create KafkaEmitter
    let emitter_config = KafkaEmitterConfig {
        topic: topic.clone(),
        send_rate: 1000,
        transaction_timeout_seconds: 60,
    };

    let mut emitter = KafkaEmitter::new(emitter_config, "test-txn-id", context).await?;

    // Emit test events
    let events = create_test_events();
    let txn = emitter.begin_write().await?;
    txn.emit(&events).await?;
    txn.commit_write().await?;

    // Consume messages and verify
    let messages = consume_messages_with_metadata(&topic, 3, Duration::from_secs(10)).await?;

    assert_eq!(messages.len(), 3, "Should have received 3 messages");

    // Verify all payloads deserialize as InternallyCapturedEvent (not as a tuple/array)
    for (i, (_, _, payload)) in messages.iter().enumerate() {
        let parsed: serde_json::Value =
            serde_json::from_str(payload).expect("payload should be valid JSON");
        assert!(
            parsed.is_object(),
            "Message {i} payload should be a JSON object, got: {payload}"
        );
        let deserialized: InternallyCapturedEvent =
            serde_json::from_str(payload).unwrap_or_else(|e| {
                panic!("Message {i} payload should deserialize as InternallyCapturedEvent: {e}")
            });
        assert_eq!(deserialized.inner.token, events[i].inner.token);
        assert_eq!(deserialized.inner.distinct_id, events[i].inner.distinct_id);
    }

    // Event 1: token1:user1 should have no key and force_disable_person_processing=true
    let (key1, headers1, _) = &messages[0];
    assert!(key1.is_none(), "First event should have no key");
    assert_eq!(
        headers1.force_disable_person_processing,
        Some(true),
        "First event should have force_disable_person_processing=true"
    );
    assert_eq!(headers1.token, Some("token1".to_string()));
    assert_eq!(headers1.distinct_id, Some("user1".to_string()));

    // Event 2: token1:user2 should have a key and no force_disable_person_processing
    let (key2, headers2, _) = &messages[1];
    assert!(
        key2.is_some(),
        "Second event should have a key (not in filter)"
    );
    assert_eq!(
        headers2.force_disable_person_processing, None,
        "Second event should not have force_disable_person_processing header"
    );
    assert_eq!(headers2.token, Some("token1".to_string()));
    assert_eq!(headers2.distinct_id, Some("user2".to_string()));

    // Event 3: token2:user3 should have no key and force_disable_person_processing=true
    let (key3, headers3, _) = &messages[2];
    assert!(key3.is_none(), "Third event should have no key");
    assert_eq!(
        headers3.force_disable_person_processing,
        Some(true),
        "Third event should have force_disable_person_processing=true"
    );
    assert_eq!(headers3.token, Some("token2".to_string()));
    assert_eq!(headers3.distinct_id, Some("user3".to_string()));

    // Clean up env vars
    std::env::remove_var("FORCE_DISABLE_PERSON_PROCESSING");

    Ok(())
}

#[tokio::test]
async fn test_empty_person_processing_filter() -> Result<()> {
    let _guard = KAFKA_TEST_MUTEX
        .get_or_init(|| TokioMutex::new(()))
        .lock()
        .await;

    let topic = format!("test-person-processing-empty-{}", Uuid::new_v4());
    create_kafka_topic(&topic).await?;

    // Empty filter - all events should have keys
    std::env::set_var("FORCE_DISABLE_PERSON_PROCESSING", "");
    std::env::set_var("KAFKA_HOSTS", KAFKA_BROKERS);
    std::env::set_var(
        "DATABASE_URL",
        "postgres://posthog:posthog@localhost:5432/posthog",
    );

    let config = Config::init_from_env()?;
    let context = Arc::new(AppContext::new(&config).await?);

    let emitter_config = KafkaEmitterConfig {
        topic: topic.clone(),
        send_rate: 1000,
        transaction_timeout_seconds: 60,
    };

    let mut emitter = KafkaEmitter::new(emitter_config, "test-txn-id-2", context).await?;

    let events = create_test_events();
    let txn = emitter.begin_write().await?;
    txn.emit(&events).await?;
    txn.commit_write().await?;

    let messages = consume_messages_with_metadata(&topic, 3, Duration::from_secs(10)).await?;

    assert_eq!(messages.len(), 3, "Should have received 3 messages");

    // All events should have keys and no force_disable_person_processing header
    for (key, headers, payload) in &messages {
        assert!(key.is_some(), "All events should have keys");
        assert_eq!(
            headers.force_disable_person_processing, None,
            "No events should have force_disable_person_processing header"
        );
        let parsed: serde_json::Value =
            serde_json::from_str(payload).expect("payload should be valid JSON");
        assert!(
            parsed.is_object(),
            "Payload should be a JSON object, got: {payload}"
        );
    }

    std::env::remove_var("FORCE_DISABLE_PERSON_PROCESSING");

    Ok(())
}