Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

11.4 KB
Raw Blame
//! End-to-end trial run test: the real `TrialJob` against local Postgres and
//! SeaweedFS (docker-compose.dev.yml). A `FolderSource` chunked small enough to
//! split lines at chunk boundaries feeds the production trial parser, pages land
//! in SeaweedFS via the real trial store client, and progress/status flushes go
//! through the lease-guarded Postgres path.
//!
//! Skips when the dev stack isn't running (SeaweedFS unreachable is a hard
//! failure in CI, matching the temp-bucket integration test).

use std::sync::Arc;

use batch_import_worker::config::Config;
use batch_import_worker::context::AppContext;
use batch_import_worker::job::config::{
    FolderSourceConfig, JobConfig, JobSecrets, SinkConfig, SourceConfig,
};
use batch_import_worker::job::model::{JobModel, JobStatus};
use batch_import_worker::parse::content::ContentType;
use batch_import_worker::parse::format::FormatConfig;
use batch_import_worker::trial::TrialJob;
use chrono::Utc;
use envconfig::Envconfig;
use lifecycle::{ComponentOptions, Manager};
use object_store::aws::AmazonS3Builder;
use object_store::path::Path;
use object_store::{ObjectStore, ObjectStoreExt};
use serde_json::Value;
use tempfile::TempDir;
use uuid::Uuid;

const SEAWEEDFS_ENDPOINT: &str = "http://localhost:8333";
const TEST_BUCKET: &str = "posthog";
const TEAM_ID: i32 = 1;
// Small enough to split the ~100-byte test lines across chunk boundaries.
const CHUNK_SIZE: usize = 96;
const RECORDS_PER_PAGE: usize = 3;

fn valid_line(n: usize) -> String {
    format!(
        r#"{{"event":"event_{n}","distinct_id":"user_{n}","timestamp":"2024-01-01T00:00:00Z","properties":{{}}}}"#
    )
}

/// 12 lines: 10 valid, one malformed (index 3), one missing distinct_id (index 7).
fn source_lines() -> Vec<String> {
    (0..12)
        .map(|n| match n {
            3 => "{broken json".to_string(),
            7 => r#"{"event":"orphan","properties":{}}"#.to_string(),
            _ => valid_line(n),
        })
        .collect()
}

fn seaweedfs_store() -> Arc<dyn ObjectStore> {
    let store = AmazonS3Builder::new()
        .with_bucket_name(TEST_BUCKET)
        .with_endpoint(SEAWEEDFS_ENDPOINT)
        .with_region("us-east-1")
        .with_allow_http(true)
        .with_virtual_hosted_style_request(false)
        // SeaweedFS dev runs in open-access mode; any credentials are accepted.
        .with_access_key_id("any")
        .with_secret_access_key("any")
        .build()
        .expect("failed to build SeaweedFS object_store");
    Arc::new(store)
}

/// Probe SeaweedFS: unreachable is a silent skip locally but a hard failure in
/// CI, where the dev stack is always booted.
async fn seaweedfs_reachable(store: &Arc<dyn ObjectStore>) -> bool {
    let probe = Path::from("__reachability_probe__");
    let result = tokio::time::timeout(std::time::Duration::from_secs(3), store.head(&probe)).await;
    let reachable = matches!(
        result,
        Ok(Ok(_)) | Ok(Err(object_store::Error::NotFound { .. }))
    );
    if !reachable && std::env::var("CI").is_ok() {
        panic!("SeaweedFS unreachable at {SEAWEEDFS_ENDPOINT} in CI — the dev stack must be up");
    }
    reachable
}

struct TrialRun {
    // Owns the source directory for the duration of the run.
    _dir: TempDir,
    context: Arc<AppContext>,
    store: Arc<dyn ObjectStore>,
    prefix: String,
    job_id: Uuid,
    model: JobModel,
}

impl TrialRun {
    /// Set up config, context, source dir, and a leased `posthog_batchimport`
    /// row, exactly as a claimed trial job would look. Returns `None` (skip)
    /// when the dev stack or the seed team isn't available.
    async fn create(record_limit: u64) -> Option<Self> {
        let store = seaweedfs_store();
        if !seaweedfs_reachable(&store).await {
            eprintln!("SeaweedFS unreachable at {SEAWEEDFS_ENDPOINT}, skipping test");
            return None;
        }

        let mut config = Config::init_from_env().unwrap();
        config.trial_bucket_name = TEST_BUCKET.to_string();
        config.trial_bucket_endpoint = SEAWEEDFS_ENDPOINT.to_string();
        config.trial_bucket_region = "us-east-1".to_string();
        config.trial_bucket_access_key_id = "any".to_string();
        config.trial_bucket_secret_access_key = "any".to_string();
        config.trial_bucket_force_path_style = true;
        config.trial_bucket_prefix = "trial-e2e-test/".to_string();
        config.trial_records_per_page = RECORDS_PER_PAGE;
        config.trial_chunk_size = CHUNK_SIZE;

        let Ok(context) = AppContext::new(&config).await else {
            eprintln!(
                "Postgres unreachable at {}, skipping test",
                config.database_url
            );
            return None;
        };
        let context = Arc::new(context);

        // The trial parser resolves the team's token; the dev/CI database seeds
        // team 1. Skip (rather than fabricate a team) when it's absent.
        if context.get_token_for_team_id(TEAM_ID).await.is_err() {
            eprintln!("team {TEAM_ID} not found in database, skipping test");
            return None;
        }

        let dir = TempDir::new().unwrap();
        std::fs::write(
            dir.path().join("events.jsonl"),
            source_lines().join("\n") + "\n",
        )
        .unwrap();

        let job_id = Uuid::now_v7();
        let lease_id = format!("trial-e2e-{job_id}");
        let import_config = JobConfig {
            source: SourceConfig::Folder(FolderSourceConfig {
                path: dir.path().to_string_lossy().to_string(),
            }),
            data_format: FormatConfig::JsonLines {
                skip_blanks: true,
                content: ContentType::Captured,
            },
            sink: SinkConfig::TrialS3 { record_limit },
            import_events: true,
            generate_identify_events: false,
            generate_group_identify_events: false,
        };

        sqlx::query(
            r#"
            INSERT INTO posthog_batchimport
                (id, team_id, created_at, updated_at, status, import_config, secrets, lease_id, leased_until, backoff_attempt)
            VALUES ($1, $2, now(), now(), 'running', $3, '', $4, now() + interval '30 minutes', 0)
            "#,
        )
        .bind(job_id)
        .bind(TEAM_ID)
        .bind(serde_json::to_value(&import_config).unwrap())
        .bind(&lease_id)
        .execute(&context.db)
        .await
        .unwrap();

        let model = JobModel {
            id: job_id,
            team_id: TEAM_ID,
            created_at: Utc::now(),
            updated_at: Utc::now(),
            lease_id: Some(lease_id.clone()),
            leased_until: None,
            status: JobStatus::Running,
            status_message: None,
            display_status_message: None,
            state: None,
            import_config,
            secrets: JobSecrets::empty(),
            backoff_attempt: 0,
            backoff_until: None,
            pending_commit: None,
            was_leased: false,
            recovered_commit: None,
        };

        let prefix = format!("trial-e2e-test/team_{TEAM_ID}/job_{job_id}");
        Some(Self {
            _dir: dir,
            context,
            store,
            prefix,
            job_id,
            model,
        })
    }

    async fn run_job(&self) {
        let mut manager = Manager::builder("trial-e2e-test")
            .with_trap_signals(false)
            .with_prestop_check(false)
            .build();
        let handle = manager.register("job", ComponentOptions::new());

        let job = TrialJob::new(self.model.clone(), self.context.clone(), handle)
            .await
            .unwrap();
        job.run().await.unwrap();
    }

    async fn db_status_and_state(&self) -> (String, Value) {
        let row: (String, Value) =
            sqlx::query_as("SELECT status, state FROM posthog_batchimport WHERE id = $1")
                .bind(self.job_id)
                .fetch_one(&self.context.db)
                .await
                .unwrap();
        row
    }

    async fn read_object(&self, rel: &str) -> Vec<u8> {
        self.store
            .get(&Path::from(format!("{}/{rel}", self.prefix)))
            .await
            .unwrap()
            .bytes()
            .await
            .unwrap()
            .to_vec()
    }

    async fn read_all_page_lines(&self, pages: u64) -> Vec<Value> {
        let mut lines = vec![];
        for page in 0..pages {
            let bytes = self.read_object(&format!("pages/{page:05}.jsonl")).await;
            for line in String::from_utf8(bytes).unwrap().lines() {
                lines.push(serde_json::from_str(line).unwrap());
            }
        }
        lines
    }

    /// Best-effort teardown: the row is deleted; objects are left under the
    /// uniquely-named job prefix.
    async fn cleanup(&self) {
        sqlx::query("DELETE FROM posthog_batchimport WHERE id = $1")
            .bind(self.job_id)
            .execute(&self.context.db)
            .await
            .ok();
    }
}

#[tokio::test]
async fn trial_job_pairs_every_source_line_and_completes() {
    let Some(run) = TrialRun::create(100).await else {
        return;
    };
    run.run_job().await;

    let (status, state) = run.db_status_and_state().await;
    assert_eq!(status, "completed");

    // Progress persisted through the lease-guarded flush path
    let trial = &state["trial"];
    assert_eq!(trial["records_emitted"], 12);
    assert_eq!(trial["summary"]["dropped_records"], 2);
    assert_eq!(trial["summary"]["output_events"], 10);
    let pages = trial["pages_written"].as_u64().unwrap();

    // Every source line becomes exactly one record despite lines being split
    // across ~96-byte chunks, and bad lines don't abort the run.
    let lines = run.read_all_page_lines(pages).await;
    assert_eq!(lines.len(), 12);
    for (i, line) in lines.iter().enumerate() {
        assert_eq!(
            line["seq"].as_u64(),
            Some(i as u64),
            "seq must be continuous"
        );
    }

    // The two planted failures carry their reasons and original source
    assert!(lines[3]["error"].as_str().is_some());
    assert_eq!(
        lines[3]["source"],
        Value::String("{broken json".to_string())
    );
    assert_eq!(lines[7]["error"], "No distinct_id found");
    assert_eq!(lines[7]["source"]["event"], "orphan");

    // A valid record pairs its source with the transformed output
    assert_eq!(lines[0]["source"]["event"], "event_0");
    assert_eq!(lines[0]["outputs"][0]["event"], "event_0");
    assert_eq!(lines[0]["outputs"][0]["distinct_id"], "user_0");
    assert_eq!(
        lines[0]["outputs"][0]["payload"]["properties"]["$import_job_id"],
        run.job_id.to_string()
    );

    // Summary is readable and consistent with the pages
    let summary: Value = serde_json::from_slice(&run.read_object("summary.json").await).unwrap();
    assert_eq!(summary["records"], 12);
    assert_eq!(summary["pages"], pages);
    assert_eq!(summary["error_counts"]["No distinct_id found"], 1);

    run.cleanup().await;
}

#[tokio::test]
async fn trial_job_stops_at_the_record_limit() {
    let Some(run) = TrialRun::create(4).await else {
        return;
    };
    run.run_job().await;

    let (status, state) = run.db_status_and_state().await;
    assert_eq!(status, "completed");
    assert_eq!(state["trial"]["records_emitted"], 4);

    let pages = state["trial"]["pages_written"].as_u64().unwrap();
    let lines = run.read_all_page_lines(pages).await;
    assert_eq!(lines.len(), 4);
    assert_eq!(lines.last().unwrap()["seq"], 3);

    run.cleanup().await;
}