Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

9.9 KB
Raw Blame
use std::sync::Arc;

use anyhow::{Context, Error};
use lifecycle::Handle;
use tokio::sync::Mutex;
use tracing::{info, warn};
use uuid::Uuid;

use crate::{
    context::AppContext,
    error::ensure_user_message,
    job::{
        cleanup_committed_part_if_done,
        config::SinkConfig,
        handle_fetch_error, load_or_init_state,
        model::{JobModel, JobState, JobStatus},
        select_and_fetch_next_chunk,
    },
    parse::{format::ParserFnFor, Parsed},
    source::DataSource,
    staging::s3_client::create_trial_output_store,
    trial::{parser::build_trial_parser, sink::TrialSink, TrialRecord},
};

/// User-facing message for infrastructure failures on the trial output path
/// (bucket config, page/summary writes, progress flushes) — never something
/// that suggests a problem with the user's data.
const TRIAL_UNAVAILABLE_MSG: &str =
    "Trial runs are temporarily unavailable. Please try again later.";

/// A bounded trial run of an import: same source, format, and transforms as the
/// real thing, but each source record is paired with its would-be output (or the
/// reason it failed) and written to the trial output bucket instead of being
/// captured.
///
/// Unlike [`crate::job::Job`], processing is sequential — no pipelined
/// fetch/commit, no two-stage offset commit. Durability comes from ordering:
/// pages are written to object storage before the progress that counts them is
/// flushed, so a worker death re-fetches at most one chunk and overwrites the
/// same page indices. Re-transformed records can differ in generated values
/// (fresh UUIDs for events without one, identify-cache dedup state), which is
/// acceptable for a preview.
pub struct TrialJob {
    context: Arc<AppContext>,
    handle: Handle,
    job_id: Uuid,
    chunk_size: usize,
    state: Mutex<JobState>,
    model: Mutex<JobModel>,
    source: Box<dyn DataSource>,
    transform: Arc<ParserFnFor<TrialRecord>>,
    sink: TrialSink,
    record_limit: u64,
}

impl TrialJob {
    pub async fn new(
        mut model: JobModel,
        context: Arc<AppContext>,
        handle: Handle,
    ) -> Result<Self, Error> {
        let SinkConfig::TrialS3 { record_limit } = &model.import_config.sink else {
            anyhow::bail!(
                "TrialJob constructed for job {} without a trial_s3 sink",
                model.id
            );
        };
        let record_limit = (*record_limit).clamp(1, context.config.trial_max_record_limit);

        // Fail fast (pausing the job with a user-facing message) when this
        // worker has no trial bucket configured, before any source work starts.
        let store = create_trial_output_store(&context.config)
            .await
            .map_err(|e| ensure_user_message(e, TRIAL_UNAVAILABLE_MSG))?;
        let prefix = format!(
            "{}/team_{}/job_{}",
            context.config.trial_bucket_prefix.trim_matches('/'),
            model.team_id,
            model.id
        );
        let sink = TrialSink::new(store, prefix, context.config.trial_records_per_page);

        let is_restarting = model.state.is_some();
        let source = model
            .import_config
            .source
            .construct(
                &model.secrets,
                context.clone(),
                is_restarting,
                model.id,
                model.team_id,
            )
            .await
            .with_context(|| "Failed to construct data source for trial job".to_string())?;
        source.prepare_for_job().await?;

        let transform = Arc::new(build_trial_parser(&model, context.clone()).await?);
        let state = load_or_init_state(source.as_ref(), &mut model).await?;

        let job_id = model.id;
        let chunk_size = context.config.trial_chunk_size;

        Ok(Self {
            context,
            handle,
            job_id,
            chunk_size,
            state: Mutex::new(state),
            model: Mutex::new(model),
            source,
            transform,
            sink,
            record_limit,
        })
    }

    pub async fn run(self) -> Result<(), Error> {
        loop {
            if self.handle.is_shutting_down() {
                // Keep remote staging on shutdown so the pod that re-claims the
                // trial attaches to staged parts instead of re-downloading.
                info!("Shutting down, releasing in-flight trial job resources");
                if let Err(e) = self.source.release_job_resources().await {
                    warn!("Failed to release trial job source resources: {:?}", e);
                }
                return Ok(());
            }

            let records_emitted = {
                let state = self.state.lock().await;
                state.trial.as_ref().map_or(0, |t| t.records_emitted)
            };
            if records_emitted >= self.record_limit {
                break;
            }

            let fetched = select_and_fetch_next_chunk(
                &self.state,
                &self.model,
                self.source.as_ref(),
                &self.transform,
                self.chunk_size,
                self.job_id,
            )
            .await;

            let outcome = match fetched {
                Ok(Some((key, parsed, reset_backoff))) => self
                    .commit_chunk(&key, parsed, reset_backoff)
                    .await
                    // Commit failures (page writes, progress flushes) are on
                    // our side, never the user's data: transient ones still
                    // back off below, but a pause must surface the infra
                    // message rather than implying a data problem.
                    .map_err(|e| ensure_user_message(e, TRIAL_UNAVAILABLE_MSG)),
                Ok(None) => break, // Source exhausted before the record limit
                Err(e) => Err(e),
            };

            match outcome {
                Ok(true) => {}
                Ok(false) => return self.stop().await,
                Err(e) => {
                    // Same classification as the import path: transient errors
                    // (source rate limits, object-store weather) back off and the
                    // trial resumes from its persisted progress; anything else
                    // pauses with a user-facing message.
                    handle_fetch_error(
                        &e,
                        &self.context,
                        &self.model,
                        &self.state,
                        self.source.as_ref(),
                    )
                    .await?;
                    return Ok(());
                }
            }
        }

        self.complete().await
    }

    /// Stop a trial whose row is no longer running because an operator or a customer
    /// changed its status. Progress is already saved, so a resume continues from it.
    async fn stop(self) -> Result<(), Error> {
        info!(job_id = %self.job_id, "Trial job is no longer running, stopping");
        if let Err(e) = self.source.release_job_resources().await {
            warn!("Failed to release trial job source resources: {:?}", e);
        }
        self.model
            .lock()
            .await
            .release_lease(&self.context.db)
            .await
    }

    /// Persist one fetched chunk: write its records as pages, fold them into the
    /// running summary, then flush the advanced offsets and trial progress.
    /// Ordering matters — pages first, progress second — so a crash in between
    /// re-fetches the chunk and overwrites the same pages. Returns false when the
    /// job is no longer running.
    async fn commit_chunk(
        &self,
        key: &str,
        parsed: Parsed<Vec<TrialRecord>>,
        reset_backoff: bool,
    ) -> Result<bool, Error> {
        let mut progress = {
            let state = self.state.lock().await;
            state.trial.clone().unwrap_or_default()
        };

        let mut records = parsed.data;
        progress.truncate_to_budget(&mut records, self.record_limit);

        if !records.is_empty() {
            let pages = self
                .sink
                .write_pages(progress.pages_written, progress.records_emitted, &records)
                .await?;
            progress.absorb(&records, pages);
        }

        let state_snapshot = {
            let mut state = self.state.lock().await;
            state.trial = Some(progress);
            state.clone()
        };

        let written = {
            let mut model = self.model.lock().await;
            if reset_backoff && model.backoff_attempt > 0 {
                model.reset_backoff_in_db(&self.context.db).await?;
            }
            model.state = Some(state_snapshot);
            model.save_progress(&self.context.db, true, false).await?
        };

        cleanup_committed_part_if_done(self.source.as_ref(), &self.model, key).await;
        Ok(written.status == JobStatus::Running)
    }

    /// Write the summary and mark the job completed. Idempotent: a resume after
    /// a failure here re-enters via the record-limit (or exhausted-source) check
    /// and rewrites the same summary from the persisted progress.
    async fn complete(self) -> Result<(), Error> {
        if let Err(e) = self.source.cleanup_after_job().await {
            warn!("Failed to cleanup after trial job: {:?}", e);
        }

        let progress = {
            let state = self.state.lock().await;
            state.trial.clone().unwrap_or_default()
        };
        self.sink.write_summary(&progress).await?;

        let mut model = self.model.lock().await;
        let status = model.complete(&self.context.db).await?;
        if status == JobStatus::Completed {
            info!(
                job_id = %self.job_id,
                records = progress.records_emitted,
                "Trial job completed"
            );
        } else {
            info!(job_id = %self.job_id, %status, "Trial job reached its end, but an operator status was kept");
        }
        Ok(())
    }
}