Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

9.9 KB
Raw Blame
//! In-process harness driving the real batch-import pipeline: a genuine
//! `DateRangeExportSource` (with the production extractor for the provider)
//! pointed at the [`mock_export`](super::mock_export) server, the production
//! JSONL parser and per-provider transform, and the DB-free job fetch loop
//! (`select_and_fetch_next_chunk`).
//!
//! Commits are modeled by the shared `JobState`: the loop advances part offsets
//! in place, exactly as a successfully committed chunk would persist them.
//! [`Harness::restart`] simulates a pod replacement: the source (and with it
//! every prepared key, streaming reader, and staged `.raw` file) is dropped and
//! rebuilt, while the persisted `JobState` survives - the same shape as a
//! deploy landing mid-part in production.

use std::collections::HashMap;
use std::sync::Arc;

use anyhow::Error;
use batch_import_worker::cache::{MockGroupCache, MockIdentifyCache};
use batch_import_worker::extractor::ExtractorType;
use batch_import_worker::job::config::{JobConfig, JobSecrets};
use batch_import_worker::job::model::{JobModel, JobState, JobStatus, PartState};
use batch_import_worker::job::select_and_fetch_next_chunk;
use batch_import_worker::parse::content::amplitude::AmplitudeEvent;
use batch_import_worker::parse::content::mixpanel::MixpanelEvent;
use batch_import_worker::parse::content::TransformContext;
use batch_import_worker::parse::format::{json_nd, skip_geoip, ParserFn};
use batch_import_worker::parse::Parsed;
use batch_import_worker::source::date_range_export::{AuthConfig, DateRangeExportSource};
use batch_import_worker::source::DataSource;
use chrono::{DateTime, TimeZone, Utc};
use common_types::InternallyCapturedEvent;
use rayon::iter::{IntoParallelIterator, ParallelIterator};
use tempfile::TempDir;
use tokio::sync::Mutex;
use uuid::Uuid;

use super::mock_export::{MockExport, Provider};

pub const TEAM_ID: i32 = 42;
pub const TOKEN: &str = "e2e-test-token";

pub struct Harness {
    pub mock: MockExport,
    provider: Provider,
    chunk_size: usize,
    job_id: Uuid,
    staging: TempDir,
    start: DateTime<Utc>,
    end: DateTime<Utc>,
    state: Mutex<JobState>,
    model: Mutex<JobModel>,
    parser: Arc<ParserFn>,
    source: DateRangeExportSource,
    pub emitted: Vec<InternallyCapturedEvent>,
}

/// Outcome of driving the fetch loop.
#[derive(Debug)]
pub enum RunOutcome {
    /// Every part is done.
    Complete,
    /// The requested number of chunks was fetched with work still remaining.
    MoreRemaining,
}

impl Harness {
    /// A harness over `days` consecutive UTC days starting at `start_day`
    /// (`%Y-%m-%d`). One part per day, matching production Mixpanel imports.
    pub async fn new(mock: MockExport, start_day: &str, days: u32, chunk_size: usize) -> Self {
        let provider = mock.provider();
        let date = chrono::NaiveDate::parse_from_str(start_day, "%Y-%m-%d").unwrap();
        let start = Utc
            .from_utc_datetime(&date.and_hms_opt(0, 0, 0).unwrap())
            .to_utc();
        let end = start + chrono::Duration::days(days as i64);

        let staging = TempDir::new().unwrap();
        let job_id = Uuid::now_v7();
        let source = build_source(&mock, provider, staging.path(), start, end);
        source.prepare_for_job().await.unwrap();

        let parts: Vec<PartState> = source
            .keys()
            .await
            .unwrap()
            .into_iter()
            .map(|key| PartState {
                key,
                current_offset: 0,
                total_size: None,
            })
            .collect();
        let job_state = JobState {
            parts,
            trial: None,
            events_emitted: 0,
        };

        let model = JobModel {
            id: job_id,
            team_id: TEAM_ID,
            created_at: Utc::now(),
            updated_at: Utc::now(),
            lease_id: None,
            leased_until: None,
            status: JobStatus::Running,
            status_message: None,
            display_status_message: None,
            state: Some(job_state.clone()),
            import_config: minimal_job_config(provider),
            secrets: JobSecrets::empty(),
            backoff_attempt: 0,
            backoff_until: None,
            pending_commit: None,
            was_leased: false,
            recovered_commit: None,
        };

        Self {
            provider,
            chunk_size,
            job_id,
            staging,
            start,
            end,
            state: Mutex::new(job_state),
            model: Mutex::new(model),
            parser: Arc::new(build_parser(provider, job_id)),
            source,
            mock,
            emitted: Vec::new(),
        }
    }

    /// Drop and rebuild the source: prepared keys, streaming readers, and staged
    /// `.raw` files are all lost, while the committed `JobState` survives. This
    /// is a pod replacement mid-job.
    pub async fn restart(&mut self) {
        self.source = build_source(
            &self.mock,
            self.provider,
            self.staging.path(),
            self.start,
            self.end,
        );
        self.source.prepare_for_job().await.unwrap();
    }

    /// Fetch and parse up to `max_chunks` chunks, appending emitted events.
    pub async fn run_chunks(&mut self, max_chunks: usize) -> Result<RunOutcome, Error> {
        for _ in 0..max_chunks {
            let next = select_and_fetch_next_chunk(
                &self.state,
                &self.model,
                &self.source,
                &self.parser,
                self.chunk_size,
                self.job_id,
            )
            .await?;
            match next {
                None => return Ok(RunOutcome::Complete),
                Some((_key, parsed, _reset_backoff)) => self.emitted.extend(parsed.data),
            }
        }
        Ok(RunOutcome::MoreRemaining)
    }

    /// Drive the loop until every part is done.
    pub async fn run_to_end(&mut self) -> Result<(), Error> {
        loop {
            if matches!(self.run_chunks(usize::MAX).await?, RunOutcome::Complete) {
                return Ok(());
            }
        }
    }

    /// The committed part states (the harness's stand-in for the DB row).
    pub async fn parts(&self) -> Vec<PartState> {
        self.state.lock().await.parts.clone()
    }
}

fn build_source(
    mock: &MockExport,
    provider: Provider,
    staging: &std::path::Path,
    start: DateTime<Utc>,
    end: DateTime<Utc>,
) -> DateRangeExportSource {
    let extractor = match provider {
        Provider::Mixpanel => ExtractorType::PlainGzip,
        Provider::Amplitude => ExtractorType::ZipGzipJson,
    };
    let auth = match provider {
        Provider::Mixpanel => AuthConfig::MixpanelAuth {
            secret_key: "mock-secret".to_string(),
        },
        Provider::Amplitude => AuthConfig::BasicAuth {
            username: "mock-key".to_string(),
            password: "mock-secret".to_string(),
        },
    };

    DateRangeExportSource::builder(
        mock.export_url(),
        start,
        end,
        86_400,
        extractor.create_extractor(),
        staging.to_path_buf(),
    )
    .with_query_params("from_date".to_string(), "to_date".to_string())
    .with_date_format("%Y-%m-%d".to_string())
    .with_auth(auth)
    .with_retries(0)
    .with_headers(HashMap::new())
    .build()
    .unwrap()
}

/// Assemble the production parser for `provider` around a hand-built
/// `TransformContext`, mirroring `FormatConfig::get_parser` without its
/// `AppContext` (team-token DB lookup) dependency.
fn build_parser(provider: Provider, job_id: Uuid) -> ParserFn {
    let context = TransformContext {
        team_id: TEAM_ID,
        token: TOKEN.to_string(),
        job_id,
        identify_cache: Arc::new(MockIdentifyCache::new()),
        group_cache: Arc::new(MockGroupCache::new()),
        import_events: true,
        generate_identify_events: true,
        generate_group_identify_events: true,
    };

    match provider {
        Provider::Mixpanel => {
            let format_parse = json_nd::<MixpanelEvent>(true);
            let transform =
                MixpanelEvent::parse_fn(context, false, chrono::Duration::zero(), skip_geoip());
            Box::new(move |data| {
                let parsed = format_parse(data)?;
                let consumed = parsed.consumed;
                let result: Result<Vec<_>, Error> = parsed
                    .data
                    .into_par_iter()
                    .map(&transform)
                    .filter_map(|x| x.transpose())
                    .collect();
                Ok(Parsed {
                    data: result?,
                    consumed,
                })
            })
        }
        Provider::Amplitude => {
            let format_parse = json_nd::<AmplitudeEvent>(true);
            let transform = AmplitudeEvent::parse_fn(context, skip_geoip());
            Box::new(move |data| {
                let parsed = format_parse(data)?;
                let consumed = parsed.consumed;
                let result: Result<Vec<Vec<_>>, Error> =
                    parsed.data.into_par_iter().map(&transform).collect();
                Ok(Parsed {
                    data: result?.into_iter().flatten().collect(),
                    consumed,
                })
            })
        }
    }
}

/// The smallest valid `JobConfig`; `select_and_fetch_next_chunk` never reads it,
/// but `JobModel` requires one.
fn minimal_job_config(provider: Provider) -> JobConfig {
    let content = match provider {
        Provider::Mixpanel => serde_json::json!({
            "type": "mixpanel",
            "skip_no_distinct_id": false,
            "timestamp_offset_seconds": null,
        }),
        Provider::Amplitude => serde_json::json!({ "type": "amplitude" }),
    };
    serde_json::from_value(serde_json::json!({
        "source": { "type": "folder", "path": "/unused" },
        "data_format": { "type": "json_lines", "skip_blanks": true, "content": content },
        "sink": { "type": "noop" },
    }))
    .expect("minimal job config must deserialize")
}