Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

10.5 KB
Raw Blame
use std::{sync::Arc, time::Duration};

use anyhow::Error;
use axum::{routing::get, Router};
use batch_import_worker::{
    config::Config,
    context::AppContext,
    error::get_user_message,
    job::{config::SinkConfig, model::JobModel, Job},
    metrics, staging,
    trial::TrialJob,
};
use common_metrics::setup_metrics_routes;
use envconfig::Envconfig;
use lifecycle::{ComponentOptions, Manager};

use tracing::level_filters::LevelFilter;
use tracing::{error, info, warn};
use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter, Layer};

common_alloc::used!();

const MAX_CONSECUTIVE_CLAIM_FAILURES: u32 = 10;

fn setup_tracing() {
    let log_layer = tracing_subscriber::fmt::layer().with_filter(
        EnvFilter::builder()
            .with_default_directive(LevelFilter::INFO.into())
            .from_env_lossy()
            .add_directive("pyroscope=warn".parse().unwrap())
            .add_directive("rdkafka=warn".parse().unwrap()),
    );
    tracing_subscriber::registry().with(log_layer).init();
}

pub async fn index() -> &'static str {
    "batch import worker"
}

/// Pause a job whose initialization failed, so it isn't endlessly re-claimed;
/// the developer message keeps the full error, the user sees the extracted one.
async fn pause_job_on_init_error(model: &mut JobModel, context: Arc<AppContext>, e: &Error) {
    let error_msg = format!("Job initialization failed for job {}: {:?}", model.id, e);
    error!("{}", error_msg);
    let user_facing_error_message = get_user_message(e);
    if let Err(pause_err) = model
        .pause(context, error_msg, Some(user_facing_error_message))
        .await
    {
        error!(
            "Failed to pause job after initialization error: {:?}",
            pause_err
        );
    }
}

#[tokio::main]
pub async fn main() -> Result<(), Error> {
    setup_tracing();
    info!("Starting up...");

    let config = Config::init_from_env().unwrap();

    // Start continuous profiling if enabled (keep _agent alive for the duration of the program)
    let _profiling_agent = match config.continuous_profiling.start_agent() {
        Ok(agent) => agent,
        Err(e) => {
            error!("Failed to start continuous profiling agent: {e}");
            None
        }
    };

    let staging_dir = config.staging_dir();
    match staging::sweep_staging_dir(&staging_dir).await {
        Ok(removed) => {
            if removed > 0 {
                metrics::staging_sweep_removed(removed);
            }
        }
        Err(e) => {
            warn!("Failed to sweep staging dir on startup: {e:#}");
        }
    }
    if let Err(e) = staging::ensure_staging_dir(&staging_dir).await {
        error!(
            "Failed to create staging dir {}: {e:#}",
            staging_dir.display()
        );
        return Err(e);
    }

    // Periodically report staging directory disk usage
    {
        let staging_dir = staging_dir.clone();
        tokio::spawn(async move {
            let start = tokio::time::Instant::now() + Duration::from_secs(60);
            let mut interval = tokio::time::interval_at(start, Duration::from_secs(60));
            loop {
                interval.tick().await;
                let bytes = staging::staging_dir_bytes(&staging_dir).await;
                metrics::staging_dir_bytes(bytes as f64);
            }
        });
    }

    let context = Arc::new(AppContext::new(&config).await.unwrap());

    // Periodically report the number of active jobs in the queue so the autoscaler
    // can scale replicas to match pending + in-flight work. Every replica reports the
    // same global count; the KEDA trigger collapses them with `max()`.
    {
        let context = context.clone();
        tokio::spawn(async move {
            let start = tokio::time::Instant::now() + Duration::from_secs(60);
            let mut interval = tokio::time::interval_at(start, Duration::from_secs(60));
            loop {
                interval.tick().await;
                match JobModel::count_active_jobs(&context.db).await {
                    Ok(count) => metrics::active_jobs(count as f64),
                    Err(e) => warn!("Failed to count active jobs for autoscaling metric: {e:#}"),
                }
            }
        });
    }

    let mut manager = Manager::builder("batch-import-worker").build();

    let job_handle = manager.register(
        "job-loop",
        ComponentOptions::new().with_graceful_shutdown(Duration::from_secs(30)),
    );
    let metrics_handle = manager.register(
        "metrics-server",
        ComponentOptions::new().is_observability(true),
    );

    let readiness = manager.readiness_handler();
    let liveness = manager.liveness_handler();
    let monitor = manager.monitor_background();

    // Metrics/health HTTP server (observability handle -- stays alive during standard drain)
    let bind = format!("{}:{}", config.host, config.port);
    tokio::spawn(async move {
        let _guard = metrics_handle.process_scope();

        let health_router = Router::new()
            .route("/", get(index))
            .route(
                "/_readiness",
                get(move || {
                    let r = readiness.clone();
                    async move { r.check().await }
                }),
            )
            .route("/_liveness", get(move || async move { liveness.check() }));
        let router = setup_metrics_routes(health_router);

        let listener = tokio::net::TcpListener::bind(&bind)
            .await
            .expect("Failed to bind metrics port");
        info!("Metrics server listening on {}", bind);
        axum::serve(listener, router)
            .with_graceful_shutdown(metrics_handle.shutdown_signal())
            .await
            .expect("Metrics server error");
    });

    // Job processing loop runs inline (not spawned) because Job::process() holds
    // tracing format args across .await points, making its future !Send.
    // The lifecycle monitor runs on a background OS thread and handles signals.
    {
        let _guard = job_handle.process_scope();
        let mut consecutive_claim_failures: u32 = 0;

        while !job_handle.is_shutting_down() {
            let claim_result = JobModel::claim_next_job(context.clone()).await;

            let claimed = match claim_result {
                Ok(model) => {
                    consecutive_claim_failures = 0;
                    model
                }
                Err(e) => {
                    consecutive_claim_failures += 1;
                    if consecutive_claim_failures >= MAX_CONSECUTIVE_CLAIM_FAILURES {
                        error!(
                            "Failed to claim next job ({consecutive_claim_failures} consecutive failures), triggering shutdown: {e:?}"
                        );
                        job_handle.signal_failure(format!("Failed to claim next job: {e}"));
                    } else {
                        error!(
                            "Failed to claim next job (attempt {consecutive_claim_failures}/{MAX_CONSECUTIVE_CLAIM_FAILURES}): {e:?}"
                        );
                    }
                    None
                }
            };

            let Some(mut model) = claimed else {
                if job_handle.is_shutting_down() {
                    break;
                }
                tokio::select! {
                    _ = tokio::time::sleep(Duration::from_secs(5)) => {},
                    _ = job_handle.shutdown_recv() => break,
                }
                continue;
            };

            info!("Claimed job: {:?}", model.id);

            match model.recover_interrupted_commit(context.clone()).await {
                Ok(true) => continue,
                Ok(false) => {}
                Err(e) => {
                    error!("Failed to recover interrupted job {}: {:?}", model.id, e);
                    continue;
                }
            }

            // Trial jobs (trial_s3 sink) take their own bounded, sequential path
            // instead of the import pipeline.
            if matches!(model.import_config.sink, SinkConfig::TrialS3 { .. }) {
                match TrialJob::new(model.clone(), context.clone(), job_handle.clone()).await {
                    Ok(trial_job) => {
                        if let Err(e) = trial_job.run().await {
                            // Like the import path, the job transitions itself
                            // (pause/backoff) where it can; anything escaping here
                            // is left for the lease to expire and a re-claim.
                            error!("Error processing trial job: {:?}, dropping", e);
                        }
                    }
                    Err(e) => {
                        pause_job_on_init_error(&mut model, context.clone(), &e).await;
                    }
                }
                continue;
            }

            let mut next_step =
                match Job::new(model.clone(), context.clone(), job_handle.clone()).await {
                    Ok(job) => Some(job),
                    Err(e) => {
                        pause_job_on_init_error(&mut model, context.clone(), &e).await;
                        continue;
                    }
                };

            while let Some(job) = next_step {
                if job_handle.is_shutting_down() {
                    // Keep remote staging on shutdown (deploys included): the pod
                    // that re-claims the job attaches to staged parts instead of
                    // re-downloading. Local temp files are still freed.
                    info!("Shutting down, releasing in-flight job resources before dropping");
                    if let Err(e) = job.source.release_job_resources().await {
                        warn!("Failed to release job source resources on shutdown: {e:?}");
                    }
                    break;
                }
                next_step = match job.process().await {
                    Ok(next) => next,
                    Err(e) => {
                        // If an error occurs that should prevent the job from being picked up by
                        // a subsequent worker, the job will already be in a paused state. We don't
                        // try to set the job model state in PG here -- the job handles that itself.
                        error!("Error processing job: {:?}, dropping", e);
                        None
                    }
                };
            }
        }

        info!("Shutting down");
    }
    // _guard dropped here during shutdown → signals completion to monitor

    monitor.wait().await?;

    Ok(())
}