:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
use std::sync::Arc;
use anyhow::{Context, Error};
use lifecycle::Handle;
use tokio::sync::Mutex;
use tracing::{info, warn};
use uuid::Uuid;
use crate::{
context::AppContext,
error::ensure_user_message,
job::{
cleanup_committed_part_if_done,
config::SinkConfig,
handle_fetch_error, load_or_init_state,
model::{JobModel, JobState, JobStatus},
select_and_fetch_next_chunk,
},
parse::{format::ParserFnFor, Parsed},
source::DataSource,
staging::s3_client::create_trial_output_store,
trial::{parser::build_trial_parser, sink::TrialSink, TrialRecord},
};
/// User-facing message for infrastructure failures on the trial output path
/// (bucket config, page/summary writes, progress flushes) — never something
/// that suggests a problem with the user's data.
const TRIAL_UNAVAILABLE_MSG: &str =
"Trial runs are temporarily unavailable. Please try again later.";
/// A bounded trial run of an import: same source, format, and transforms as the
/// real thing, but each source record is paired with its would-be output (or the
/// reason it failed) and written to the trial output bucket instead of being
/// captured.
///
/// Unlike [`crate::job::Job`], processing is sequential — no pipelined
/// fetch/commit, no two-stage offset commit. Durability comes from ordering:
/// pages are written to object storage before the progress that counts them is
/// flushed, so a worker death re-fetches at most one chunk and overwrites the
/// same page indices. Re-transformed records can differ in generated values
/// (fresh UUIDs for events without one, identify-cache dedup state), which is
/// acceptable for a preview.
pub struct TrialJob {
context: Arc<AppContext>,
handle: Handle,
job_id: Uuid,
chunk_size: usize,
state: Mutex<JobState>,
model: Mutex<JobModel>,
source: Box<dyn DataSource>,
transform: Arc<ParserFnFor<TrialRecord>>,
sink: TrialSink,
record_limit: u64,
}
impl TrialJob {
pub async fn new(
mut model: JobModel,
context: Arc<AppContext>,
handle: Handle,
) -> Result<Self, Error> {
let SinkConfig::TrialS3 { record_limit } = &model.import_config.sink else {
anyhow::bail!(
"TrialJob constructed for job {} without a trial_s3 sink",
model.id
);
};
let record_limit = (*record_limit).clamp(1, context.config.trial_max_record_limit);
// Fail fast (pausing the job with a user-facing message) when this
// worker has no trial bucket configured, before any source work starts.
let store = create_trial_output_store(&context.config)
.await
.map_err(|e| ensure_user_message(e, TRIAL_UNAVAILABLE_MSG))?;
let prefix = format!(
"{}/team_{}/job_{}",
context.config.trial_bucket_prefix.trim_matches('/'),
model.team_id,
model.id
);
let sink = TrialSink::new(store, prefix, context.config.trial_records_per_page);
let is_restarting = model.state.is_some();
let source = model
.import_config
.source
.construct(
&model.secrets,
context.clone(),
is_restarting,
model.id,
model.team_id,
)
.await
.with_context(|| "Failed to construct data source for trial job".to_string())?;
source.prepare_for_job().await?;
let transform = Arc::new(build_trial_parser(&model, context.clone()).await?);
let state = load_or_init_state(source.as_ref(), &mut model).await?;
let job_id = model.id;
let chunk_size = context.config.trial_chunk_size;
Ok(Self {
context,
handle,
job_id,
chunk_size,
state: Mutex::new(state),
model: Mutex::new(model),
source,
transform,
sink,
record_limit,
})
}
pub async fn run(self) -> Result<(), Error> {
loop {
if self.handle.is_shutting_down() {
// Keep remote staging on shutdown so the pod that re-claims the
// trial attaches to staged parts instead of re-downloading.
info!("Shutting down, releasing in-flight trial job resources");
if let Err(e) = self.source.release_job_resources().await {
warn!("Failed to release trial job source resources: {:?}", e);
}
return Ok(());
}
let records_emitted = {
let state = self.state.lock().await;
state.trial.as_ref().map_or(0, |t| t.records_emitted)
};
if records_emitted >= self.record_limit {
break;
}
let fetched = select_and_fetch_next_chunk(
&self.state,
&self.model,
self.source.as_ref(),
&self.transform,
self.chunk_size,
self.job_id,
)
.await;
let outcome = match fetched {
Ok(Some((key, parsed, reset_backoff))) => self
.commit_chunk(&key, parsed, reset_backoff)
.await
// Commit failures (page writes, progress flushes) are on
// our side, never the user's data: transient ones still
// back off below, but a pause must surface the infra
// message rather than implying a data problem.
.map_err(|e| ensure_user_message(e, TRIAL_UNAVAILABLE_MSG)),
Ok(None) => break, // Source exhausted before the record limit
Err(e) => Err(e),
};
match outcome {
Ok(true) => {}
Ok(false) => return self.stop().await,
Err(e) => {
// Same classification as the import path: transient errors
// (source rate limits, object-store weather) back off and the
// trial resumes from its persisted progress; anything else
// pauses with a user-facing message.
handle_fetch_error(
&e,
&self.context,
&self.model,
&self.state,
self.source.as_ref(),
)
.await?;
return Ok(());
}
}
}
self.complete().await
}
/// Stop a trial whose row is no longer running because an operator or a customer
/// changed its status. Progress is already saved, so a resume continues from it.
async fn stop(self) -> Result<(), Error> {
info!(job_id = %self.job_id, "Trial job is no longer running, stopping");
if let Err(e) = self.source.release_job_resources().await {
warn!("Failed to release trial job source resources: {:?}", e);
}
self.model
.lock()
.await
.release_lease(&self.context.db)
.await
}
/// Persist one fetched chunk: write its records as pages, fold them into the
/// running summary, then flush the advanced offsets and trial progress.
/// Ordering matters — pages first, progress second — so a crash in between
/// re-fetches the chunk and overwrites the same pages. Returns false when the
/// job is no longer running.
async fn commit_chunk(
&self,
key: &str,
parsed: Parsed<Vec<TrialRecord>>,
reset_backoff: bool,
) -> Result<bool, Error> {
let mut progress = {
let state = self.state.lock().await;
state.trial.clone().unwrap_or_default()
};
let mut records = parsed.data;
progress.truncate_to_budget(&mut records, self.record_limit);
if !records.is_empty() {
let pages = self
.sink
.write_pages(progress.pages_written, progress.records_emitted, &records)
.await?;
progress.absorb(&records, pages);
}
let state_snapshot = {
let mut state = self.state.lock().await;
state.trial = Some(progress);
state.clone()
};
let written = {
let mut model = self.model.lock().await;
if reset_backoff && model.backoff_attempt > 0 {
model.reset_backoff_in_db(&self.context.db).await?;
}
model.state = Some(state_snapshot);
model.save_progress(&self.context.db, true, false).await?
};
cleanup_committed_part_if_done(self.source.as_ref(), &self.model, key).await;
Ok(written.status == JobStatus::Running)
}
/// Write the summary and mark the job completed. Idempotent: a resume after
/// a failure here re-enters via the record-limit (or exhausted-source) check
/// and rewrites the same summary from the persisted progress.
async fn complete(self) -> Result<(), Error> {
if let Err(e) = self.source.cleanup_after_job().await {
warn!("Failed to cleanup after trial job: {:?}", e);
}
let progress = {
let state = self.state.lock().await;
state.trial.clone().unwrap_or_default()
};
self.sink.write_summary(&progress).await?;
let mut model = self.model.lock().await;
let status = model.complete(&self.context.db).await?;
if status == JobStatus::Completed {
info!(
job_id = %self.job_id,
records = progress.records_emitted,
"Trial job completed"
);
} else {
info!(job_id = %self.job_id, %status, "Trial job reached its end, but an operator status was kept");
}
Ok(())
}
}