:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
use std::{sync::Arc, time::Duration};
use anyhow::Error;
use axum::{routing::get, Router};
use batch_import_worker::{
config::Config,
context::AppContext,
error::get_user_message,
job::{config::SinkConfig, model::JobModel, Job},
metrics, staging,
trial::TrialJob,
};
use common_metrics::setup_metrics_routes;
use envconfig::Envconfig;
use lifecycle::{ComponentOptions, Manager};
use tracing::level_filters::LevelFilter;
use tracing::{error, info, warn};
use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter, Layer};
common_alloc::used!();
const MAX_CONSECUTIVE_CLAIM_FAILURES: u32 = 10;
fn setup_tracing() {
let log_layer = tracing_subscriber::fmt::layer().with_filter(
EnvFilter::builder()
.with_default_directive(LevelFilter::INFO.into())
.from_env_lossy()
.add_directive("pyroscope=warn".parse().unwrap())
.add_directive("rdkafka=warn".parse().unwrap()),
);
tracing_subscriber::registry().with(log_layer).init();
}
pub async fn index() -> &'static str {
"batch import worker"
}
/// Pause a job whose initialization failed, so it isn't endlessly re-claimed;
/// the developer message keeps the full error, the user sees the extracted one.
async fn pause_job_on_init_error(model: &mut JobModel, context: Arc<AppContext>, e: &Error) {
let error_msg = format!("Job initialization failed for job {}: {:?}", model.id, e);
error!("{}", error_msg);
let user_facing_error_message = get_user_message(e);
if let Err(pause_err) = model
.pause(context, error_msg, Some(user_facing_error_message))
.await
{
error!(
"Failed to pause job after initialization error: {:?}",
pause_err
);
}
}
#[tokio::main]
pub async fn main() -> Result<(), Error> {
setup_tracing();
info!("Starting up...");
let config = Config::init_from_env().unwrap();
// Start continuous profiling if enabled (keep _agent alive for the duration of the program)
let _profiling_agent = match config.continuous_profiling.start_agent() {
Ok(agent) => agent,
Err(e) => {
error!("Failed to start continuous profiling agent: {e}");
None
}
};
let staging_dir = config.staging_dir();
match staging::sweep_staging_dir(&staging_dir).await {
Ok(removed) => {
if removed > 0 {
metrics::staging_sweep_removed(removed);
}
}
Err(e) => {
warn!("Failed to sweep staging dir on startup: {e:#}");
}
}
if let Err(e) = staging::ensure_staging_dir(&staging_dir).await {
error!(
"Failed to create staging dir {}: {e:#}",
staging_dir.display()
);
return Err(e);
}
// Periodically report staging directory disk usage
{
let staging_dir = staging_dir.clone();
tokio::spawn(async move {
let start = tokio::time::Instant::now() + Duration::from_secs(60);
let mut interval = tokio::time::interval_at(start, Duration::from_secs(60));
loop {
interval.tick().await;
let bytes = staging::staging_dir_bytes(&staging_dir).await;
metrics::staging_dir_bytes(bytes as f64);
}
});
}
let context = Arc::new(AppContext::new(&config).await.unwrap());
// Periodically report the number of active jobs in the queue so the autoscaler
// can scale replicas to match pending + in-flight work. Every replica reports the
// same global count; the KEDA trigger collapses them with `max()`.
{
let context = context.clone();
tokio::spawn(async move {
let start = tokio::time::Instant::now() + Duration::from_secs(60);
let mut interval = tokio::time::interval_at(start, Duration::from_secs(60));
loop {
interval.tick().await;
match JobModel::count_active_jobs(&context.db).await {
Ok(count) => metrics::active_jobs(count as f64),
Err(e) => warn!("Failed to count active jobs for autoscaling metric: {e:#}"),
}
}
});
}
let mut manager = Manager::builder("batch-import-worker").build();
let job_handle = manager.register(
"job-loop",
ComponentOptions::new().with_graceful_shutdown(Duration::from_secs(30)),
);
let metrics_handle = manager.register(
"metrics-server",
ComponentOptions::new().is_observability(true),
);
let readiness = manager.readiness_handler();
let liveness = manager.liveness_handler();
let monitor = manager.monitor_background();
// Metrics/health HTTP server (observability handle -- stays alive during standard drain)
let bind = format!("{}:{}", config.host, config.port);
tokio::spawn(async move {
let _guard = metrics_handle.process_scope();
let health_router = Router::new()
.route("/", get(index))
.route(
"/_readiness",
get(move || {
let r = readiness.clone();
async move { r.check().await }
}),
)
.route("/_liveness", get(move || async move { liveness.check() }));
let router = setup_metrics_routes(health_router);
let listener = tokio::net::TcpListener::bind(&bind)
.await
.expect("Failed to bind metrics port");
info!("Metrics server listening on {}", bind);
axum::serve(listener, router)
.with_graceful_shutdown(metrics_handle.shutdown_signal())
.await
.expect("Metrics server error");
});
// Job processing loop runs inline (not spawned) because Job::process() holds
// tracing format args across .await points, making its future !Send.
// The lifecycle monitor runs on a background OS thread and handles signals.
{
let _guard = job_handle.process_scope();
let mut consecutive_claim_failures: u32 = 0;
while !job_handle.is_shutting_down() {
let claim_result = JobModel::claim_next_job(context.clone()).await;
let claimed = match claim_result {
Ok(model) => {
consecutive_claim_failures = 0;
model
}
Err(e) => {
consecutive_claim_failures += 1;
if consecutive_claim_failures >= MAX_CONSECUTIVE_CLAIM_FAILURES {
error!(
"Failed to claim next job ({consecutive_claim_failures} consecutive failures), triggering shutdown: {e:?}"
);
job_handle.signal_failure(format!("Failed to claim next job: {e}"));
} else {
error!(
"Failed to claim next job (attempt {consecutive_claim_failures}/{MAX_CONSECUTIVE_CLAIM_FAILURES}): {e:?}"
);
}
None
}
};
let Some(mut model) = claimed else {
if job_handle.is_shutting_down() {
break;
}
tokio::select! {
_ = tokio::time::sleep(Duration::from_secs(5)) => {},
_ = job_handle.shutdown_recv() => break,
}
continue;
};
info!("Claimed job: {:?}", model.id);
match model.recover_interrupted_commit(context.clone()).await {
Ok(true) => continue,
Ok(false) => {}
Err(e) => {
error!("Failed to recover interrupted job {}: {:?}", model.id, e);
continue;
}
}
// Trial jobs (trial_s3 sink) take their own bounded, sequential path
// instead of the import pipeline.
if matches!(model.import_config.sink, SinkConfig::TrialS3 { .. }) {
match TrialJob::new(model.clone(), context.clone(), job_handle.clone()).await {
Ok(trial_job) => {
if let Err(e) = trial_job.run().await {
// Like the import path, the job transitions itself
// (pause/backoff) where it can; anything escaping here
// is left for the lease to expire and a re-claim.
error!("Error processing trial job: {:?}, dropping", e);
}
}
Err(e) => {
pause_job_on_init_error(&mut model, context.clone(), &e).await;
}
}
continue;
}
let mut next_step =
match Job::new(model.clone(), context.clone(), job_handle.clone()).await {
Ok(job) => Some(job),
Err(e) => {
pause_job_on_init_error(&mut model, context.clone(), &e).await;
continue;
}
};
while let Some(job) = next_step {
if job_handle.is_shutting_down() {
// Keep remote staging on shutdown (deploys included): the pod
// that re-claims the job attaches to staged parts instead of
// re-downloading. Local temp files are still freed.
info!("Shutting down, releasing in-flight job resources before dropping");
if let Err(e) = job.source.release_job_resources().await {
warn!("Failed to release job source resources on shutdown: {e:?}");
}
break;
}
next_step = match job.process().await {
Ok(next) => next,
Err(e) => {
// If an error occurs that should prevent the job from being picked up by
// a subsequent worker, the job will already be in a paused state. We don't
// try to set the job model state in PG here -- the job handles that itself.
error!("Error processing job: {:?}, dropping", e);
None
}
};
}
}
info!("Shutting down");
}
// _guard dropped here during shutdown → signals completion to monitor
monitor.wait().await?;
Ok(())
}