Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

138 KB
Raw Blame
use std::sync::Arc;

use anyhow::{Context, Error};
use uuid::Uuid;

use crate::metrics as metric_emit;
use common_types::InternallyCapturedEvent;
use lifecycle::Handle;
use model::{JobModel, JobState, PartState};
use tokio::sync::Mutex;
use tracing::{debug, error, info, warn};

use crate::{
    context::AppContext,
    emit::Emitter,
    error::{
        extract_retry_after_from_error, get_user_message, is_rate_limited_error, is_timeout_error,
        is_transient_network_error, is_transient_object_store_error, is_transient_server_error,
        UserError,
    },
    extractor::detect_compression_magic,
    job::{backoff::format_backoff_messages, config::SinkConfig},
    parse::{
        format::{ParserFn, ParserFnFor},
        Parsed,
    },
    source::DataSource,
};

pub mod backoff;
pub mod config;
pub mod model;
#[cfg(test)]
pub(crate) mod test_db;

#[derive(Debug, PartialEq)]
enum ErrorHandlingDecision {
    Backoff {
        delay: std::time::Duration,
        status_msg: String,
        display_msg: String,
    },
    Pause {
        error_msg: String,
        display_msg: String,
    },
}

fn decide_on_error(
    err: &anyhow::Error,
    current_date_range: Option<&str>,
    policy: crate::job::backoff::BackoffPolicy,
    current_attempt: u32,
    user_message: &str,
) -> ErrorHandlingDecision {
    if is_rate_limited_error(err) {
        let mut delay = policy.next_delay(current_attempt);
        if let Some(ra) = extract_retry_after_from_error(err) {
            delay = std::cmp::min(ra, policy.max_delay);
        }
        let (status_msg, display_msg) =
            format_backoff_messages(current_date_range, delay, "Rate limited (429)");
        ErrorHandlingDecision::Backoff {
            delay,
            status_msg,
            display_msg,
        }
    } else if is_timeout_error(err) {
        let delay = policy.next_delay(current_attempt);
        let (status_msg, display_msg) =
            format_backoff_messages(current_date_range, delay, "Request timed out");
        ErrorHandlingDecision::Backoff {
            delay,
            status_msg,
            display_msg,
        }
    } else if is_transient_network_error(err) {
        let delay = policy.next_delay(current_attempt);
        let (status_msg, display_msg) =
            format_backoff_messages(current_date_range, delay, "Transient network error");
        ErrorHandlingDecision::Backoff {
            delay,
            status_msg,
            display_msg,
        }
    } else if is_transient_server_error(err) {
        let delay = policy.next_delay(current_attempt);
        let (status_msg, display_msg) =
            format_backoff_messages(current_date_range, delay, "Upstream server error (5xx)");
        ErrorHandlingDecision::Backoff {
            delay,
            status_msg,
            display_msg,
        }
    } else if is_transient_object_store_error(err) {
        // Temp-bucket staging I/O that outlived the S3 client's internal retries
        // (throttling, 5xx, timeouts, transport). Backoff and retry rather than
        // pausing a customer job over infrastructure weather.
        let delay = policy.next_delay(current_attempt);
        let (status_msg, display_msg) =
            format_backoff_messages(current_date_range, delay, "Temporary storage error");
        ErrorHandlingDecision::Backoff {
            delay,
            status_msg,
            display_msg,
        }
    } else {
        // Use the full error chain for developer-facing message
        let error_msg = match current_date_range {
            Some(dr) => format!("{err:#} (Date range: {dr})"),
            None => format!("{err:#}"),
        };
        let display_msg = match current_date_range {
            Some(dr) => format!("{user_message} (Date range: {dr})"),
            None => user_message.to_string(),
        };
        ErrorHandlingDecision::Pause {
            error_msg,
            display_msg,
        }
    }
}

fn should_pause_due_to_max_attempts(next_attempt: u32, max_attempts: u32) -> bool {
    max_attempts > 0 && next_attempt >= max_attempts
}

/// Classify a fetch/parse error and transition the job accordingly: transient
/// errors schedule a backoff (or pause once max attempts is exhausted), anything
/// else pauses the job with a user-facing message. Shared by the import path
/// ([`Job::process`]) and the trial path.
///
/// Cleanup is deferred until after classification: transient interruptions keep
/// remote staging for the resume to attach to, while source-side pauses sweep it
/// for a clean re-download.
pub(crate) async fn handle_fetch_error(
    e: &Error,
    context: &Arc<AppContext>,
    model: &Mutex<JobModel>,
    state: &Mutex<JobState>,
    source: &dyn DataSource,
) -> Result<(), Error> {
    let user_facing_error_message = get_user_message(e);
    let current_date_range = {
        let state = state.lock().await;
        state
            .parts
            .iter()
            .find(|p| !p.is_done())
            .and_then(|p| source.get_date_range_for_key(&p.key))
    };

    let policy = context.config.backoff_policy();
    let current_attempt = {
        let model = model.lock().await;
        model.backoff_attempt.max(0) as u32
    };
    let next_attempt = current_attempt.saturating_add(1);
    match decide_on_error(
        e,
        current_date_range.as_deref(),
        policy,
        current_attempt,
        &user_facing_error_message,
    ) {
        ErrorHandlingDecision::Backoff {
            delay,
            status_msg,
            display_msg,
        } => {
            // Transient (or transient-persisted, below): keep staged
            // remote parts so the resume attaches without re-hitting an
            // origin that is likely still rate-limited or flaky.
            if let Err(cleanup_err) = source.release_job_resources().await {
                warn!("Failed to release job resources: {:?}", cleanup_err);
            }
            if should_pause_due_to_max_attempts(next_attempt, context.config.backoff_max_attempts) {
                let mut model = model.lock().await;
                let msg = match current_date_range.as_deref() {
                    Some(dr) => format!(
                        "Max backoff attempts reached for date range {dr} (attempt {next_attempt}). Pausing."
                    ),
                    None => {
                        format!("Max backoff attempts reached (attempt {next_attempt}). Pausing.")
                    }
                };
                model
                    .pause(
                        context.clone(),
                        msg,
                        Some(
                            "Transient error persisted. Job paused after maximum retries."
                                .to_string(),
                        ),
                    )
                    .await?;
                return Ok(());
            }

            error!(
                job_id = %model.lock().await.id,
                attempt = next_attempt,
                delay_secs = delay.as_secs(),
                "transient error, scheduling retry: {:#}",
                e
            );
            metric_emit::backoff_event(delay.as_secs_f64());

            let mut model = model.lock().await;
            model
                .schedule_backoff(
                    &context.db,
                    delay,
                    status_msg,
                    Some(display_msg),
                    next_attempt as i32,
                )
                .await?;
            Ok(())
        }
        ErrorHandlingDecision::Pause {
            error_msg,
            display_msg,
        } => {
            // Source-side pause: a human intervenes and may fix the
            // source data in place, so the resume must re-download a
            // clean copy, never attach to a stale staged one. Staged
            // sources quarantine (rather than delete) the staged bytes,
            // since they are the exact stream the failing offset points
            // into — the evidence for debugging the parse failure.
            if let Err(cleanup_err) = source.cleanup_after_data_error().await {
                warn!("Failed to cleanup after data error: {:?}", cleanup_err);
            }
            let mut model = model.lock().await;
            error!(
                job_id = %model.id,
                user_msg = %error_msg,
                "Pausing job due to error: {:#}",
                e
            );
            model
                .pause(context.clone(), error_msg, Some(display_msg))
                .await?;
            Ok(())
        }
    }
}

async fn reset_backoff_after_success(
    context: Arc<AppContext>,
    model: &mut JobModel,
) -> Result<(), Error> {
    model.reset_backoff_in_db(&context.db).await
}

/// Mirror a part's discovered total size into the model state, which is what
/// gets persisted to the DB on the next part commit.
async fn mirror_part_total(model: &Mutex<JobModel>, key: &str, total: u64) {
    let mut model = model.lock().await;
    if let Some(model_state) = &mut model.state {
        if let Some(model_part) = model_state.parts.iter_mut().find(|p| p.key == key) {
            model_part.total_size = Some(total);
        }
    }
}

/// DB-free core of [`Job::get_next_chunk`]: pick the next unfinished part, lazily
/// discover its (decompressed) size, fetch+parse one chunk, and advance the
/// in-memory part offset. A newly discovered `total_size` is mirrored into the
/// model in memory only — no DB write happens here, which keeps the fetch/select
/// loop testable without a database. Per-part staging cleanup is not driven from
/// here: the source frees each `.raw` on its own EOF read (see
/// [`crate::source::read_prepared_chunk`]).
///
/// Returns the fetched `(key, parsed, reset_backoff)`. `reset_backoff` is `false`
/// only on the empty-key short-circuit (so the caller skips the success backoff
/// reset, matching the original control flow) and `true` otherwise. `None` means
/// every part is done.
pub async fn select_and_fetch_next_chunk<T: Send + 'static>(
    state: &Mutex<JobState>,
    model: &Mutex<JobModel>,
    source: &dyn DataSource,
    transform: &Arc<ParserFnFor<T>>,
    chunk_size: usize,
    job_id: Uuid,
) -> Result<Option<(String, Parsed<Vec<T>>, bool)>, Error> {
    let mut state = state.lock().await;

    let Some(next_part) = state.parts.iter_mut().find(|p| !p.is_done()) else {
        info!(job_id = %job_id, "Found no next part, returning");
        return Ok(None); // We're done fetching
    };

    let key = next_part.key.clone();
    source.prepare_key(&key).await?;

    if next_part.total_size.is_none() {
        if let Some(actual_size) = source.size(&key).await? {
            next_part.total_size = Some(actual_size);
            info!(job_id = %job_id, "Updated total size for key {}: {}", key, actual_size);
            mirror_part_total(model, &key, actual_size).await;

            // Streaming sources only learn their size on the read that observes
            // end-of-stream, which is usually also the read that consumed the last
            // bytes - so by the time the size is known here, the offset may already
            // be at (or past) the end. Re-check completion now: fetching again would
            // return an empty chunk, and advancing on one would push the offset past
            // the total, making the part permanently un-finishable.
            if next_part.is_done() {
                info!(
                    job_id = %job_id,
                    "Part {} (size {}) was fully consumed when its size was discovered, moving on",
                    key, actual_size
                );
                return Ok(Some((
                    key,
                    Parsed {
                        consumed: 0,
                        data: vec![],
                    },
                    false,
                )));
            }
        }
    }

    info!(job_id = %job_id, "Fetching part chunk {:?}", next_part);

    let next_chunk = source
        .get_chunk(&next_part.key, next_part.current_offset, chunk_size as u64)
        .await
        .context(format!("Fetching part chunk {next_part:?}"))?;

    // `>=` because sources clamp reads to the end of the part, so the final chunk
    // ends exactly at total_size - a strict `>` can never be true and would leave
    // the unconsumed-bytes check below unreachable.
    let is_last_chunk = match next_part.total_size {
        Some(total_size) => next_part.current_offset + next_chunk.len() as u64 >= total_size,
        None => false,
    };

    let chunk_bytes = next_chunk.len();

    // An empty chunk before the part looks finished means the source hit
    // end-of-stream at or before this offset. For streaming sources the size is
    // only discoverable on the read that observes EOF - which for an empty part
    // (a day with no events: an empty 200 body, a gzip of nothing, a zip with no
    // data members) or a resume at a not-yet-persisted EOF is this very read. So
    // re-consult the size now: if the source's own view confirms everything was
    // consumed, the part is complete. Anything else (a ranged source serving
    // empty bodies mid-part, a stream that shrank below the recorded total) can
    // never make progress, so fail rather than loop.
    if chunk_bytes == 0 && !is_last_chunk {
        let refreshed_size = source.size(&key).await?;
        let confirmed_complete = refreshed_size.is_some_and(|actual| {
            let total_consistent = next_part.total_size.is_none_or(|t| t == actual);
            total_consistent && next_part.current_offset >= actual
        });

        if !confirmed_complete {
            return Err(Error::msg(format!(
                "Source returned no data for part {} at offset {} before the end of the part",
                next_part.key, next_part.current_offset
            )));
        }

        let actual_size = refreshed_size.expect("confirmed_complete implies the size is known");
        if next_part.current_offset > actual_size {
            // A resumed offset past the stream's end: the export shrank between
            // downloads. Everything the source can serve was already consumed,
            // so completing is the only non-stranding option - but it deserves
            // a loud trace, since the tail of the previous download's data may
            // not exist in this download at all.
            warn!(
                job_id = %job_id,
                "Part {} resumed at offset {} past the stream end {}; marking complete",
                key, next_part.current_offset, actual_size
            );
        }
        next_part.total_size = Some(actual_size);
        mirror_part_total(model, &key, actual_size).await;
        info!(
            job_id = %job_id,
            "Part {} (size {}) has no data left to read at offset {}, marking complete",
            key, actual_size, next_part.current_offset
        );
        return Ok(Some((
            key,
            Parsed {
                consumed: 0,
                data: vec![],
            },
            false,
        )));
    }

    // A part that still *begins* with a known compression magic can never be
    // parsed as newline-delimited JSON: the file is compressed a second time, or
    // the source/extractor's compression setting does not match the file. Only the
    // start of a part is decisive - mid-file chunks can coincidentally begin with
    // these bytes - so this is gated on offset 0. Fail fast with an actionable,
    // non-transient error (a UserError pauses the job) instead of letting the
    // parser emit a confusing "invalid utf-8" failure or crawl one byte at a time.
    if next_part.current_offset == 0 {
        if let Some(fmt) = detect_compression_magic(&next_chunk) {
            let internal = Error::msg(format!(
                "part {} begins with {fmt} magic bytes at offset 0 ({chunk_bytes} bytes read)",
                next_part.key
            ));
            return Err(internal.context(UserError::new(format!(
                "Part {} begins with {fmt}-compressed data where newline-delimited JSON was \
                 expected - the file may be compressed twice, or the import's compression \
                 setting doesn't match the file.",
                next_part.key
            ))));
        }
    }

    info!(job_id = %job_id, "Fetched part chunk {:?}", next_part);
    let m_tf = transform.clone();
    let key_for_error = key.clone();
    // This is computationally expensive, so we run it in a blocking task
    let parsed = tokio::task::spawn_blocking(move || (m_tf)(next_chunk))
        .await?
        .map_err(|e| {
            let inner_msg = get_user_message(&e);
            e.context(UserError::new(format!(
                "Parsing data in file '{key_for_error}' failed: {inner_msg}"
            )))
        })
        .context(format!("Processing part chunk {next_part:?}"))?;

    info!(
        job_id = %job_id,
        "Parsed part chunk {:?}, consumed {} bytes",
        next_part, parsed.consumed
    );

    // Consuming more bytes than were read is always a parser bug: advancing the
    // offset past real data would silently skip it, or overshoot the end of the
    // part and make it permanently un-finishable.
    if parsed.consumed > chunk_bytes {
        return Err(Error::msg(format!(
            "Parser consumed {} bytes but only {} were read from part {} at offset {}",
            parsed.consumed, chunk_bytes, next_part.key, next_part.current_offset
        )));
    }

    // If this is the last chunk and we didn't consume all of it, we have leftover unparseable data.
    if parsed.consumed < chunk_bytes && is_last_chunk {
        return Err(Error::msg(format!(
            "Failed to parse data from part {} at offset {} - {} bytes left unconsumed",
            next_part.key,
            next_part.current_offset,
            chunk_bytes - parsed.consumed
        )));
    }

    // A single record larger than the read window fills a whole non-final chunk with no
    // line terminator, so the parser can't advance past it. Without this the loop would
    // crawl one byte per iteration (and, on the temp-bucket backend, issue a ranged GET
    // of up to chunk_size per byte). Fail fast with an actionable, non-transient error
    // that pauses the job.
    if !is_last_chunk && chunk_bytes == chunk_size && parsed.consumed <= 1 && parsed.data.is_empty()
    {
        // Public message stays free of internal tuning values (chunk_size); the root
        // error carries them for status_message and logs.
        return Err(Error::msg(format!(
            "single record exceeds chunk_size={chunk_size} in part {} at offset {}",
            next_part.key, next_part.current_offset
        ))
        .context(UserError::new(format!(
            "A single record in part {} is too large to process. Split the oversized \
             record into smaller records and re-run this date range or file.",
            next_part.key
        ))));
    }

    // If we consumed no bytes and have no parsed data but there was data to consume, something went wrong.
    // Note: don't error if we have no parsed data but have consumed bytes - invalid events may have been all filtered out
    if parsed.consumed == 0 && parsed.data.is_empty() && chunk_bytes > 0 {
        return Err(Error::msg(format!(
            "Failed to parse any data from part {} at offset {}",
            next_part.key, next_part.current_offset
        )));
    }

    // Update the in-memory part state (the read will be committed to the DB once the write is done)
    next_part.current_offset += parsed.consumed as u64;

    Ok(Some((key, parsed, true)))
}

/// Load the job's state from the model, initializing the parts list from the
/// source on first pickup. A freshly initialized state is mirrored back into
/// the model so the next flush persists it. Shared by the import path
/// ([`Job::new`]) and the trial path.
pub(crate) async fn load_or_init_state(
    source: &dyn DataSource,
    model: &mut JobModel,
) -> Result<JobState, Error> {
    let mut state = model.state.clone().unwrap_or_default();

    if state.parts.is_empty() {
        info!(job_id = %model.id, "Found job with no parts, initializing parts list");
        // If we have no parts, we assume this is the first time picking up the job,
        // and populate the parts list
        let mut parts = Vec::new();
        let keys = source
            .keys()
            .await
            .with_context(|| "Failed to get source keys".to_string())?;
        // An empty listing would leave nothing to process, and the job would complete
        // at once with nothing imported. Pause instead, so the user can fix the source.
        if keys.is_empty() {
            return Err(
                Error::msg("source listed no keys to import").context(UserError::new(
                    "No files were found to import. Check the source settings (for example the \
                 bucket, the prefix and the file extension), then resume the import.",
                )),
            );
        }
        for key in keys {
            let size = source
                .size(&key)
                .await
                .with_context(|| format!("Failed to get size for part {key}"))?;
            debug!("Got size for part {key}: {size:?}");
            parts.push(PartState {
                key,
                current_offset: 0,
                total_size: size,
            });
        }
        state.parts = parts;
        model.state = Some(state.clone());
        info!(job_id = %model.id, "Initialized parts list: {:?}", state.parts);
    }

    Ok(state)
}

/// Free a committed part's staging once it is fully consumed, best-effort.
///
/// Called from `do_commit` after `complete_commit` persists the advanced offsets. Checks
/// the *persisted* model view (not the in-memory fetch state, which runs one iteration
/// ahead) so staging is only freed for parts whose completion is durable. Failures are
/// logged and counted, never propagated: a missed delete leaks only transient storage,
/// reclaimed by `cleanup_after_job` and (for the temp bucket) the bucket TTL.
pub(crate) async fn cleanup_committed_part_if_done(
    source: &dyn DataSource,
    model: &Mutex<JobModel>,
    key: &str,
) {
    let part_done = {
        let model = model.lock().await;
        model
            .state
            .as_ref()
            .and_then(|state| state.parts.iter().find(|p| p.key == key))
            .is_some_and(|part| part.is_done())
    };
    if !part_done {
        return;
    }
    match source.cleanup_key(key).await {
        Ok(()) => crate::metrics::part_cleanup("ok"),
        Err(e) => {
            warn!("Failed to clean up staging for committed part {key}: {e:#}");
            crate::metrics::part_cleanup("error");
        }
    }
}

pub struct Job {
    pub context: Arc<AppContext>,
    handle: Handle,

    // Once created the job id doesn't change, store it on the Job struct for easy access, to avoid contention/locking on the model
    pub job_id: Uuid,

    // How many bytes to fetch from the source per iteration. Varies by sink
    // type: the capture sink uses a smaller value to stay within the capture
    // service's HTTP body size limit.
    pub chunk_size: usize,

    // The job maintains a copy of the job state, outside of the model,
    // because process in a pipelined fashion, and to do that we need to
    // seperate "in-memory job state" from "database job state"
    pub state: Mutex<JobState>,
    // We also maintain a copy of the model. This and the above are wrapped in a mutex
    // because we simultaneously modify both, modifying our in-memory state when we
    // fetch and parse data, and then updating the model state when we commit data
    pub model: Mutex<JobModel>,

    pub source: Box<dyn DataSource>,
    pub transform: Arc<ParserFn>,

    // We keep a mutex here so we can mutably borrow this and the job state at the same time
    pub sink: Mutex<Box<dyn Emitter>>,

    // We want to fetch data and send it at the same time, and this acts as a temporary store
    // for the data we've fetched, but not yet sent
    checkpoint: Mutex<Option<Checkpoint>>,
}

struct Checkpoint {
    key: String,
    data: Parsed<Vec<InternallyCapturedEvent>>,
}

/// Whether the job keeps going after a commit.
#[derive(Debug, PartialEq, Eq)]
enum CommitOutcome {
    Continue,
    /// The row is no longer running because an operator or a customer changed its status.
    Stopped,
}

impl Job {
    pub async fn new(
        mut model: JobModel,
        context: Arc<AppContext>,
        handle: Handle,
    ) -> Result<Self, Error> {
        let is_restarting = model.state.is_some();

        let source = model
            .import_config
            .source
            .construct(
                &model.secrets,
                context.clone(),
                is_restarting,
                model.id,
                model.team_id,
            )
            .await
            .with_context(|| "Failed to construct data source for job".to_string())?;

        // Some sources need to prepare for the job before we can start processing it
        source.prepare_for_job().await?;

        let transform = Box::new(
            model
                .import_config
                .data_format
                .get_parser(&model, context.clone())
                .await?,
        );

        let sink = model
            .import_config
            .sink
            .construct(context.clone(), &model)
            .await
            .with_context(|| format!("Failed to construct sink for job {}", model.id))?;

        let state = load_or_init_state(source.as_ref(), &mut model).await?;

        let job_id = model.id;
        let chunk_size = match &model.import_config.sink {
            SinkConfig::Capture(_) => context.config.capture_chunk_size,
            _ => context.config.chunk_size,
        };

        Ok(Self {
            context,
            handle,
            job_id,
            chunk_size,
            model: Mutex::new(model),
            state: Mutex::new(state),
            source,
            transform: Arc::new(transform),
            sink: Mutex::new(sink),
            checkpoint: Mutex::new(None),
        })
    }

    pub async fn process(self) -> Result<Option<Self>, Error> {
        let next_chunk_fut = self.get_next_chunk();
        let next_commit_fut = self.do_commit();

        let (next_chunk, next_commit) = tokio::join!(next_chunk_fut, next_commit_fut);

        if let Ok(CommitOutcome::Stopped) = next_commit {
            // Keep staged data for the resume, like a shutdown does. The chunk fetched in
            // parallel is dropped: its offsets live only in memory.
            info!(job_id = %self.job_id, "Job is no longer running, stopping");
            if let Err(cleanup_err) = self.source.release_job_resources().await {
                warn!(
                    "Failed to release job resources after stop: {:?}",
                    cleanup_err
                );
            }
            return Ok(None);
        }

        if let Err(e) = next_commit {
            // If we fail to commit, we just log and bail out - the job will be paused if it needs to be,
            // but this pod should restart, in case it's sink is in some bad state.
            // The failure is sink-side and the offset was rolled back, so keep the
            // staged remote data: the resume re-reads the byte-identical chunk
            // instead of re-downloading from an origin that may have changed.
            error!("Failed to commit chunk: {:?}", e);
            if let Err(cleanup_err) = self.source.release_job_resources().await {
                warn!("Failed to cleanup after commit failure: {:?}", cleanup_err);
            }
            return Err(e);
        }

        let next = match next_chunk {
            Ok(Some(chunk)) => chunk,
            Ok(None) => {
                // We're done fetching and parsing, so we can complete the job
                if let Err(e) = self.source.cleanup_after_job().await {
                    warn!("Failed to cleanup after job: {:?}", e);
                }
                self.successfully_complete().await?;
                return Ok(None);
            }
            Err(e) => {
                handle_fetch_error(
                    &e,
                    &self.context,
                    &self.model,
                    &self.state,
                    self.source.as_ref(),
                )
                .await?;
                return Ok(None);
            }
        };

        let mut checkpoint = self.checkpoint.lock().await;
        *checkpoint = Some(Checkpoint {
            key: next.0,
            data: next.1,
        });

        drop(checkpoint);

        // This wasn't the last part/chunk, so we return the job to let it be processed again
        Ok(Some(self))
    }

    async fn get_next_chunk(
        &self,
    ) -> Result<Option<(String, Parsed<Vec<InternallyCapturedEvent>>)>, Error> {
        let Some((key, parsed, reset_backoff)) = select_and_fetch_next_chunk(
            &self.state,
            &self.model,
            self.source.as_ref(),
            &self.transform,
            self.chunk_size,
            self.job_id,
        )
        .await?
        else {
            return Ok(None); // We're done fetching
        };

        // The only DB write in this path, kept out of the core above so the
        // fetch/select/cleanup loop stays testable without a database. Skipped on
        // the empty-key short-circuit, matching the original control flow.
        if reset_backoff {
            let mut model = self.model.lock().await;
            reset_backoff_after_success(self.context.clone(), &mut model).await?;
        }

        Ok(Some((key, parsed)))
    }

    async fn do_commit(&self) -> Result<CommitOutcome, Error> {
        self.shutdown_guard()?;
        let mut checkpoint_lock = self.checkpoint.lock().await;

        let Some(checkpoint) = checkpoint_lock.take() else {
            info!(job_id = %self.job_id, "No checkpointed data to commit, returning");
            return Ok(CommitOutcome::Continue);
        };

        let (key, parsed) = (checkpoint.key, checkpoint.data);

        info!(job_id = %self.job_id, "Committing part {} consumed {} bytes", key, parsed.consumed);
        info!(job_id = %self.job_id, "Committing {} events", parsed.data.len());

        let mut sink = self.sink.lock().await;
        self.shutdown_guard()?;
        let txn = sink.begin_write().await?;
        info!(job_id = %self.job_id, "Writing {} events", parsed.data.len());
        self.shutdown_guard()?;
        txn.emit(&parsed.data).await?;
        // Save the advanced offset and a pending-commit marker before the sink commit. After a crash
        // between the two, the next claim finds the marker and sends the chunk again, or pauses the
        // job for review after `MAX_INTERRUPTED_SENDS` interruptions or an offset mismatch.
        self.shutdown_guard()?;
        info!(job_id = %self.job_id, "Beginning PG part commit");
        if self
            .begin_part_commit(&key, parsed.consumed, parsed.data.len())
            .await?
            == CommitOutcome::Stopped
        {
            if let Err(e) = txn.abort().await {
                warn!(job_id = %self.job_id, "Failed to abort the sink transaction: {e:#}");
            }
            metric_emit::operator_stop("before_send");
            return Ok(CommitOutcome::Stopped);
        }
        info!(job_id = %self.job_id, "Beginning emitter part commit");

        // A *returned* error (as opposed to a crash) means the sink definitively rejected the
        // chunk, so the speculative offset advance above is wrong: roll it back and record the
        // real error, so a resume re-processes the chunk instead of skipping it.
        let to_sleep = match txn.commit_write().await {
            Ok(to_sleep) => to_sleep,
            Err(e) => {
                self.rollback_part_commit(&key, parsed.consumed, parsed.data.len(), &e)
                    .await?;
                return Err(e);
            }
        };
        info!(job_id = %self.job_id, "Finishing PG part commit");
        let committed = self.complete_commit().await?;
        info!(job_id = %self.job_id, "Committed part {} consumed {} bytes", key, parsed.consumed);
        if let Some(send_rate) = committed.send_rate {
            sink.set_send_rate(send_rate);
        }

        // The committed part may now be fully consumed: free its staging eagerly and only
        // after its offsets are durable. Safe against concurrent reads: the in-memory state
        // marked this part done no later than the iteration before this commit (offsets
        // advance at fetch time, and the completion short-circuits - size-discovery and
        // empty-chunk - mark a part done in the same iteration that produces their
        // synthetic zero-consumed checkpoint), so the read loop can never re-select it.
        // Those synthetic checkpoints can make the hook fire a second time for a part
        // whose data commit already cleaned up; cleanup_key is idempotent, so the double
        // call is a tolerated no-op rather than a correctness issue. A rollback or a crash
        // between the two commit stages returns/aborts before reaching this point,
        // preserving the staged data for the byte-identical re-read on resume.
        cleanup_committed_part_if_done(self.source.as_ref(), &self.model, &key).await;

        if committed.status != model::JobStatus::Running {
            metric_emit::operator_stop("after_send");
            self.model
                .lock()
                .await
                .release_lease(&self.context.db)
                .await?;
            return Ok(CommitOutcome::Stopped);
        }

        info!(job_id = %self.job_id, "Sleeping for {:?}", to_sleep);
        tokio::select! {
            _ = tokio::time::sleep(to_sleep) => {},
            _ = self.handle.shutdown_recv() => {},
        }

        Ok(CommitOutcome::Continue)
    }

    async fn successfully_complete(self) -> Result<(), Error> {
        let mut model = self.model.lock().await;
        let stored = model.state.clone().unwrap_or_default();

        if let Some(blocker) = stored.completion_blocker() {
            error!(job_id = %self.job_id, "Refusing to complete job: {blocker}");
            metric_emit::job_completion("blocked");
            return model
                .pause(
                    self.context.clone(),
                    format!("Refused to complete job: {blocker}"),
                    Some(
                        "The import stopped before all files were processed. Resume the import \
                         to continue."
                            .to_string(),
                    ),
                )
                .await;
        }

        let status = model.complete(&self.context.db).await?;
        if status != model::JobStatus::Completed {
            info!(job_id = %self.job_id, %status, "Job reached its end, but an operator status was kept");
            return Ok(());
        }
        let parts = stored.parts.len();
        let events = stored.events_emitted;
        metric_emit::completed_job_size(parts, events);
        if events == 0 {
            metric_emit::job_completion("completed_empty");
            warn!(job_id = %self.job_id, parts, "Batch import job complete, but it emitted no events");
        } else {
            metric_emit::job_completion("completed");
            info!(job_id = %self.job_id, parts, events, "Batch import job complete");
        }
        Ok(())
    }

    // Saves the advanced offset and a pending-commit marker. Writes nothing and returns `Stopped`
    // when the row is no longer running.
    async fn begin_part_commit(
        &self,
        key: &str,
        consumed: usize,
        events: usize,
    ) -> Result<CommitOutcome, Error> {
        let mut model = self.model.lock().await;
        let Some(model_state) = &mut model.state else {
            return Err(Error::msg("No model state found"));
        };

        let Some(new_offset) = model_state.advance_part_offset(key, consumed as u64) else {
            return Err(Error::msg(format!("No part found with key {key}")));
        };
        model_state.events_emitted = model_state.events_emitted.saturating_add(events as u64);

        let from_offset = new_offset.saturating_sub(consumed as u64);
        let interrupted = model
            .recovered_commit
            .take()
            .filter(|recovered| recovered.key == key && recovered.from_offset == from_offset)
            .map_or(0, |recovered| recovered.interrupted);
        model.pending_commit = Some(model::PendingCommit {
            key: key.to_string(),
            from_offset,
            to_offset: new_offset,
            interrupted,
            events: events as u64,
        });

        let written = model.save_progress(&self.context.db, true, true).await?;
        if written.applied {
            return Ok(CommitOutcome::Continue);
        }

        // Keep the in-memory model equal to the row, which the refused write left unchanged.
        model.pending_commit = None;
        if let Some(model_state) = &mut model.state {
            model_state.revert_part_offset(key, consumed as u64);
            model_state.events_emitted = model_state.events_emitted.saturating_sub(events as u64);
        }
        model.release_lease(&self.context.db).await?;
        Ok(CommitOutcome::Stopped)
    }

    // Reverts the speculative offset advance from begin_part_commit after a definitive sink
    // rejection, leaving the job paused with the real error and the original offset, so a resume
    // re-processes the chunk rather than skipping it.
    async fn rollback_part_commit(
        &self,
        key: &str,
        consumed: usize,
        events: usize,
        err: &Error,
    ) -> Result<(), Error> {
        let mut model = self.model.lock().await;
        let Some(model_state) = &mut model.state else {
            return Err(Error::msg("No model state found"));
        };

        let Some(new_offset) = model_state.revert_part_offset(key, consumed as u64) else {
            return Err(Error::msg(format!("No part found with key {key}")));
        };
        model_state.events_emitted = model_state.events_emitted.saturating_sub(events as u64);
        model.pending_commit = None;

        let status_message =
            format!("Commit of part {key} failed, rolled back to offset {new_offset}: {err:#}");

        model
            .pause(
                self.context.clone(),
                status_message,
                Some("Job paused after a failed commit. Resolve the error and resume.".to_string()),
            )
            .await?;
        model.release_lease(&self.context.db).await
    }

    // Clears the pending-commit marker, and reports the row's current status and send rate
    async fn complete_commit(&self) -> Result<model::RowWrite, Error> {
        let mut model = self.model.lock().await;
        model.pending_commit = None;
        model.save_committed(&self.context.db).await
    }

    fn shutdown_guard(&self) -> Result<(), Error> {
        if self.handle.is_shutting_down() {
            warn!("Shutdown signaled during job processing, bailing");
            Err(Error::msg("Shutdown signaled during job processing"))
        } else {
            Ok(())
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use async_trait::async_trait;
    use httpmock::Method;
    use httpmock::MockServer;
    use reqwest::Client;
    use std::collections::HashMap;

    struct MockDataSource {
        keys: Vec<String>,
        date_ranges: HashMap<String, String>,
    }

    impl MockDataSource {
        fn new() -> Self {
            let mut date_ranges = HashMap::new();
            date_ranges.insert(
                "2023-01-01T00:00:00+00:00_2023-01-01T01:00:00+00:00".to_string(),
                "2023-01-01 00:00 UTC to 2023-01-01 01:00 UTC".to_string(),
            );
            date_ranges.insert(
                "2023-01-01T01:00:00+00:00_2023-01-01T02:00:00+00:00".to_string(),
                "2023-01-01 01:00 UTC to 2023-01-01 02:00 UTC".to_string(),
            );

            Self {
                keys: vec![
                    "2023-01-01T00:00:00+00:00_2023-01-01T01:00:00+00:00".to_string(),
                    "2023-01-01T01:00:00+00:00_2023-01-01T02:00:00+00:00".to_string(),
                ],
                date_ranges,
            }
        }
    }

    #[async_trait]
    impl DataSource for MockDataSource {
        async fn keys(&self) -> Result<Vec<String>, Error> {
            Ok(self.keys.clone())
        }

        async fn size(&self, _key: &str) -> Result<Option<u64>, Error> {
            Ok(Some(100))
        }

        async fn get_chunk(&self, _key: &str, _offset: u64, _size: u64) -> Result<Vec<u8>, Error> {
            Err(Error::msg("Mock error for testing"))
        }

        fn get_date_range_for_key(&self, key: &str) -> Option<String> {
            self.date_ranges.get(key).cloned()
        }
    }

    struct MockDataSourceWithoutDateRange {
        keys: Vec<String>,
    }

    impl MockDataSourceWithoutDateRange {
        fn new() -> Self {
            Self {
                keys: vec!["some-key".to_string()],
            }
        }
    }

    #[async_trait]
    impl DataSource for MockDataSourceWithoutDateRange {
        async fn keys(&self) -> Result<Vec<String>, Error> {
            Ok(self.keys.clone())
        }

        async fn size(&self, _key: &str) -> Result<Option<u64>, Error> {
            Ok(Some(100))
        }

        async fn get_chunk(&self, _key: &str, _offset: u64, _size: u64) -> Result<Vec<u8>, Error> {
            Err(Error::msg("Mock error for testing"))
        }
    }

    #[test]
    fn test_error_message_includes_date_range_when_available() {
        let mock_source = MockDataSource::new();
        let key = "2023-01-01T00:00:00+00:00_2023-01-01T01:00:00+00:00";

        let date_range = mock_source.get_date_range_for_key(key);
        assert_eq!(
            date_range,
            Some("2023-01-01 00:00 UTC to 2023-01-01 01:00 UTC".to_string())
        );

        let error_message = format!(
            "Failed to fetch and parse chunk for date range {}: Mock error",
            date_range.unwrap()
        );
        assert_eq!(
            error_message,
            "Failed to fetch and parse chunk for date range 2023-01-01 00:00 UTC to 2023-01-01 01:00 UTC: Mock error"
        );
    }

    #[tokio::test]
    async fn test_decide_on_error_backoff_for_429() {
        let server = MockServer::start();
        let _mock = server.mock(|when, then| {
            when.method(httpmock::Method::GET).path("/export");
            then.status(429);
        });

        let resp = Client::new()
            .get(server.url("/export"))
            .send()
            .await
            .unwrap();
        let http_err = resp.error_for_status().unwrap_err();
        let err = anyhow::Error::from(http_err);

        let decision = decide_on_error(
            &err,
            Some("2023-01-01 00:00 UTC to 2023-01-01 01:00 UTC"),
            crate::job::backoff::BackoffPolicy::new(
                std::time::Duration::from_secs(60),
                2.0,
                std::time::Duration::from_secs(3600),
            ),
            0,
            "Rate limit exceeded",
        );

        match decision {
            ErrorHandlingDecision::Backoff {
                delay,
                status_msg,
                display_msg,
            } => {
                assert_eq!(delay.as_secs(), 60);
                assert!(status_msg.contains("retry"));
                assert!(display_msg.contains("Date range"));
            }
            _ => panic!("expected backoff"),
        }
    }

    #[tokio::test]
    async fn test_decide_on_error_pause_for_non_429() {
        let server = MockServer::start();
        let _mock = server.mock(|when, then| {
            when.method(httpmock::Method::GET).path("/export");
            then.status(500);
        });

        let resp = Client::new()
            .get(server.url("/export"))
            .send()
            .await
            .unwrap();
        let http_err = resp.error_for_status().unwrap_err();
        let err = anyhow::Error::from(http_err);

        let decision = decide_on_error(
            &err,
            None,
            crate::job::backoff::BackoffPolicy::new(
                std::time::Duration::from_secs(60),
                2.0,
                std::time::Duration::from_secs(3600),
            ),
            2,
            "Remote server error",
        );

        match decision {
            ErrorHandlingDecision::Pause {
                error_msg,
                display_msg,
            } => {
                assert!(error_msg.contains("500 Internal Server Error"));
                assert_eq!(display_msg, "Remote server error");
            }
            _ => panic!("expected pause"),
        }
    }

    /// Build the real object_store error a temp-bucket read/stage produces for the
    /// given S3 response status (client-level retries disabled for speed).
    async fn object_store_error_with_status(status: u16) -> Error {
        use object_store::ObjectStoreExt;
        let server = MockServer::start();
        let _mock = server.mock(|when, then| {
            when.any_request();
            then.status(status);
        });
        let store = object_store::aws::AmazonS3Builder::new()
            .with_bucket_name("b")
            .with_endpoint(server.base_url())
            .with_region("us-east-1")
            .with_allow_http(true)
            .with_virtual_hosted_style_request(false)
            .with_access_key_id("k")
            .with_secret_access_key("s")
            .with_retry(object_store::RetryConfig {
                max_retries: 0,
                retry_timeout: std::time::Duration::from_secs(5),
                ..Default::default()
            })
            .build()
            .unwrap();
        let err = store
            .get(&object_store::path::Path::from("k.data"))
            .await
            .unwrap_err();
        Error::from(err).context("Failed to read staged object for key: k")
    }

    fn policy_for_test() -> crate::job::backoff::BackoffPolicy {
        crate::job::backoff::BackoffPolicy::new(
            std::time::Duration::from_secs(60),
            2.0,
            std::time::Duration::from_secs(3600),
        )
    }

    #[tokio::test]
    async fn test_decide_on_error_backoff_for_transient_storage_error() {
        // A temp-bucket 503 that outlived the S3 client's internal retries must
        // reach job backoff, not pause a customer job over infrastructure weather.
        let err = object_store_error_with_status(503).await;
        let decision = decide_on_error(&err, None, policy_for_test(), 0, "unused");
        match decision {
            ErrorHandlingDecision::Backoff { status_msg, .. } => {
                assert!(
                    status_msg.contains("Temporary storage error"),
                    "unexpected status: {status_msg}"
                );
            }
            other => panic!("expected backoff for storage 503, got {other:?}"),
        }
    }

    #[tokio::test]
    async fn test_decide_on_error_pause_for_permanent_storage_error() {
        // 403 (IAM misconfig) must pause visibly: with unlimited backoff attempts,
        // classifying it transient would retry it invisibly forever.
        let err = object_store_error_with_status(403).await;
        let decision = decide_on_error(&err, None, policy_for_test(), 0, "Storage error");
        assert!(
            matches!(decision, ErrorHandlingDecision::Pause { .. }),
            "expected pause for storage 403, got {decision:?}"
        );
    }

    #[tokio::test]
    async fn test_retry_after_overrides_backoff() {
        let server = MockServer::start();
        let _mock = server.mock(|when, then| {
            when.method(Method::GET).path("/rl");
            then.status(429).header("Retry-After", "2");
        });

        let resp = Client::new().get(server.url("/rl")).send().await.unwrap();
        // Clone headers from the actual response before turning it into an error
        let headers_clone = resp.headers().clone();
        let http_err = resp.error_for_status().unwrap_err();

        // Wrap into our RateLimitedError manually to include Retry-After from response
        let retry_after =
            crate::source::date_range_export::parse_retry_after_header(&headers_clone)
                .expect("retry-after parsed");
        let rl = crate::error::RateLimitedError {
            retry_after: Some(retry_after),
            source: http_err,
        };
        let err = anyhow::Error::from(rl);

        let decision = super::decide_on_error(
            &err,
            None,
            crate::job::backoff::BackoffPolicy::new(
                std::time::Duration::from_secs(60),
                2.0,
                std::time::Duration::from_secs(3600),
            ),
            0,
            "Rate limit exceeded",
        );

        match decision {
            super::ErrorHandlingDecision::Backoff { delay, .. } => {
                assert_eq!(delay.as_secs(), 2);
            }
            _ => panic!("expected backoff"),
        }
    }

    #[test]
    fn test_reset_backoff_after_success() {
        let mut model = JobModel {
            // Minimal dummy values; only fields we need in this function
            id: uuid::Uuid::now_v7(),
            team_id: 1,
            created_at: chrono::Utc::now(),
            updated_at: chrono::Utc::now(),
            lease_id: None,
            leased_until: None,
            status: super::model::JobStatus::Running,
            status_message: None,
            display_status_message: None,
            state: Some(JobState {
                parts: vec![],
                trial: None,
                events_emitted: 0,
            }),
            import_config: super::config::JobConfig {
                // Construct a trivially valid config that won't be used by this test
                source: super::config::SourceConfig::Folder(super::config::FolderSourceConfig {
                    path: "/tmp".to_string(),
                }),
                data_format: crate::parse::format::FormatConfig::JsonLines {
                    skip_blanks: true,
                    content: crate::parse::content::ContentType::Captured,
                },
                sink: super::config::SinkConfig::NoOp,
                import_events: true,
                generate_identify_events: false,
                generate_group_identify_events: false,
            },
            secrets: super::config::JobSecrets {
                secrets: std::collections::HashMap::new(),
            },
            was_leased: false,
            recovered_commit: None,
            backoff_attempt: 5,
            backoff_until: None,
            pending_commit: None,
        };

        // Only verifies local field change; DB write path covered elsewhere
        model.backoff_attempt = 0;
        assert_eq!(model.backoff_attempt, 0);
    }

    #[test]
    fn test_error_message_without_date_range() {
        let mock_source = MockDataSourceWithoutDateRange::new();
        let key = "some-key";

        let date_range = mock_source.get_date_range_for_key(key);
        assert!(date_range.is_none());

        let error_message = "Failed to fetch and parse chunk: Mock error";
        assert_eq!(error_message, "Failed to fetch and parse chunk: Mock error");
    }

    #[test]
    fn test_display_message_includes_date_range() {
        let user_message = "Connection failed";
        let date_range = "2023-01-01 00:00 UTC to 2023-01-01 01:00 UTC";

        let display_message = format!("{user_message} (Date range: {date_range})");
        assert_eq!(
            display_message,
            "Connection failed (Date range: 2023-01-01 00:00 UTC to 2023-01-01 01:00 UTC)"
        );
    }

    #[tokio::test]
    async fn test_decide_on_error_backoff_for_timeout() {
        // Create a TCP listener that accepts but never responds, triggering a client timeout
        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
        let addr = listener.local_addr().unwrap();

        let client = Client::builder()
            .timeout(std::time::Duration::from_millis(50))
            .build()
            .unwrap();

        let err = client
            .get(format!("http://{addr}/export"))
            .send()
            .await
            .unwrap_err();
        let err = anyhow::Error::from(err);

        let decision = decide_on_error(
            &err,
            Some("2026-01-01 10:00 UTC to 2026-01-01 11:00 UTC"),
            crate::job::backoff::BackoffPolicy::new(
                std::time::Duration::from_secs(60),
                2.0,
                std::time::Duration::from_secs(3600),
            ),
            0,
            "Request timed out",
        );

        match decision {
            ErrorHandlingDecision::Backoff {
                delay,
                status_msg,
                display_msg,
            } => {
                assert_eq!(delay.as_secs(), 60);
                assert!(
                    status_msg.contains("Request timed out"),
                    "status_msg should mention timeout: {status_msg}"
                );
                assert!(
                    display_msg.contains("Date range"),
                    "display_msg should include date range: {display_msg}"
                );
                assert!(
                    display_msg.contains("Request timed out"),
                    "display_msg should mention timeout: {display_msg}"
                );
            }
            _ => panic!("expected backoff for timeout error"),
        }
    }

    #[tokio::test]
    async fn test_decide_on_error_backoff_for_transient_network_error() {
        // Bind then drop to close the port — connecting will be refused.
        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
        let addr = listener.local_addr().unwrap();
        drop(listener);

        let client = Client::new();
        let err = client
            .get(format!("http://{addr}/export"))
            .send()
            .await
            .unwrap_err();
        let err = anyhow::Error::from(err);

        let decision = decide_on_error(
            &err,
            Some("2026-01-01 10:00 UTC to 2026-01-01 11:00 UTC"),
            crate::job::backoff::BackoffPolicy::new(
                std::time::Duration::from_secs(60),
                2.0,
                std::time::Duration::from_secs(3600),
            ),
            0,
            "Transient network error",
        );

        match decision {
            ErrorHandlingDecision::Backoff {
                delay,
                status_msg,
                display_msg,
            } => {
                assert_eq!(delay.as_secs(), 60);
                assert!(
                    status_msg.contains("Transient network error"),
                    "status_msg should mention transient network error: {status_msg}"
                );
                assert!(
                    display_msg.contains("Date range"),
                    "display_msg should include date range: {display_msg}"
                );
                assert!(
                    display_msg.contains("Transient network error"),
                    "display_msg should mention transient network error: {display_msg}"
                );
            }
            _ => panic!("expected backoff for transient network error"),
        }
    }

    #[tokio::test]
    async fn test_decide_on_error_backoff_for_502() {
        let server = httpmock::MockServer::start();
        let _mock = server.mock(|when, then| {
            when.method(httpmock::Method::GET).path("/e");
            then.status(502);
        });
        let client = Client::new();
        let resp = client.get(server.url("/e")).send().await.unwrap();
        let http_err = resp.error_for_status().unwrap_err();
        let err = anyhow::Error::from(http_err);

        let decision = decide_on_error(
            &err,
            Some("2026-01-01 10:00 UTC to 2026-01-01 11:00 UTC"),
            crate::job::backoff::BackoffPolicy::new(
                std::time::Duration::from_secs(60),
                2.0,
                std::time::Duration::from_secs(3600),
            ),
            0,
            "Upstream server error (5xx)",
        );

        match decision {
            ErrorHandlingDecision::Backoff {
                delay,
                status_msg,
                display_msg,
            } => {
                assert_eq!(delay.as_secs(), 60);
                assert!(
                    status_msg.contains("Upstream server error"),
                    "status_msg should mention upstream server error: {status_msg}"
                );
                assert!(
                    display_msg.contains("Date range"),
                    "display_msg should include date range: {display_msg}"
                );
            }
            _ => panic!("expected backoff for 502"),
        }
    }

    #[test]
    fn test_should_pause_due_to_max_attempts() {
        assert!(!should_pause_due_to_max_attempts(0, 0)); // unlimited
        assert!(!should_pause_due_to_max_attempts(2, 3));
        assert!(should_pause_due_to_max_attempts(3, 3));
        assert!(should_pause_due_to_max_attempts(4, 3));
    }

    #[test]
    fn test_decide_on_error_separates_developer_and_user_messages() {
        let root_error =
            std::io::Error::new(std::io::ErrorKind::InvalidData, "invalid byte sequence");
        let error = anyhow::Error::from(root_error)
            .context("Failed to parse JSON")
            .context("Processing chunk");

        let decision = decide_on_error(
            &error,
            Some("2023-01-01 to 2023-01-02"),
            crate::job::backoff::BackoffPolicy::new(
                std::time::Duration::from_secs(60),
                2.0,
                std::time::Duration::from_secs(3600),
            ),
            0,
            "User-friendly error message",
        );

        match decision {
            ErrorHandlingDecision::Pause {
                error_msg,
                display_msg,
            } => {
                assert!(
                    error_msg.contains("Processing chunk"),
                    "Developer message should contain outer context: {error_msg}"
                );
                assert!(
                    error_msg.contains("Failed to parse JSON"),
                    "Developer message should contain inner context: {error_msg}"
                );
                assert!(
                    error_msg.contains("invalid byte sequence"),
                    "Developer message should contain root cause: {error_msg}"
                );

                assert_eq!(
                    display_msg,
                    "User-friendly error message (Date range: 2023-01-01 to 2023-01-02)"
                );
            }
            _ => panic!("Expected Pause decision"),
        }
    }

    mod operator_control_tests {
        use std::future::Future;
        use std::pin::Pin;

        use std::time::Duration;

        use super::*;
        use crate::emit::Transaction;
        use crate::job::model::{JobStatus, PartState};
        use crate::job::test_db;
        use sqlx::PgPool;
        use tokio_util::sync::CancellationToken;

        const PART_BYTES: usize = 64;
        const SEND_RATE: u64 = 1000;

        type SendHook =
            Arc<dyn Fn(usize) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync>;

        /// Parts held in memory, one chunk each.
        struct MemorySource {
            parts: Vec<String>,
        }

        #[async_trait]
        impl DataSource for MemorySource {
            async fn keys(&self) -> Result<Vec<String>, Error> {
                Ok(self.parts.clone())
            }

            async fn size(&self, _key: &str) -> Result<Option<u64>, Error> {
                Ok(Some(PART_BYTES as u64))
            }

            async fn get_chunk(
                &self,
                _key: &str,
                offset: u64,
                size: u64,
            ) -> Result<Vec<u8>, Error> {
                let end = (offset + size).min(PART_BYTES as u64);
                Ok(vec![b'x'; (end - offset) as usize])
            }
        }

        /// Delivers in `commit_write` like the capture sink, recording each send rate and abort.
        /// `on_send` runs during a delivery, which is where an operator change races the worker.
        struct RecordingEmitter {
            send_rate: u64,
            sends: Arc<std::sync::Mutex<Vec<u64>>>,
            aborts: Arc<std::sync::atomic::AtomicUsize>,
            on_send: Option<SendHook>,
            fail_sends: bool,
        }

        struct RecordingTransaction<'a> {
            emitter: &'a RecordingEmitter,
        }

        #[async_trait]
        impl Emitter for RecordingEmitter {
            async fn begin_write<'a>(&'a mut self) -> Result<Box<dyn Transaction<'a> + 'a>, Error> {
                Ok(Box::new(RecordingTransaction { emitter: self }))
            }

            fn set_send_rate(&mut self, send_rate: u64) {
                self.send_rate = send_rate;
            }
        }

        #[async_trait]
        impl<'a> Transaction<'a> for RecordingTransaction<'a> {
            async fn emit(&self, _data: &[InternallyCapturedEvent]) -> Result<(), Error> {
                Ok(())
            }

            async fn commit_write(self: Box<Self>) -> Result<Duration, Error> {
                let index = {
                    let mut sends = self.emitter.sends.lock().unwrap();
                    sends.push(self.emitter.send_rate);
                    sends.len() - 1
                };
                if let Some(hook) = &self.emitter.on_send {
                    hook(index).await;
                }
                if self.emitter.fail_sends {
                    return Err(Error::msg("sink rejected the chunk"));
                }
                Ok(Duration::ZERO)
            }

            async fn abort(self: Box<Self>) -> Result<(), Error> {
                self.emitter
                    .aborts
                    .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
                Ok(())
            }
        }

        fn consume_all() -> Arc<ParserFn> {
            Arc::new(Box::new(|bytes: Vec<u8>| {
                Ok(Parsed {
                    consumed: bytes.len(),
                    data: Vec::new(),
                })
            }))
        }

        fn one_event_per_chunk() -> Arc<ParserFn> {
            Arc::new(Box::new(|bytes: Vec<u8>| {
                let inner = common_types::CapturedEvent {
                    uuid: Uuid::now_v7(),
                    distinct_id: "user".to_string(),
                    session_id: None,
                    ip: "127.0.0.1".to_string(),
                    data: "{}".to_string(),
                    now: chrono::Utc::now().to_rfc3339(),
                    sent_at: None,
                    token: "token".to_string(),
                    event: "event".to_string(),
                    timestamp: chrono::Utc::now(),
                    is_cookieless_mode: false,
                    historical_migration: true,
                };
                Ok(Parsed {
                    consumed: bytes.len(),
                    data: vec![InternallyCapturedEvent { team_id: 1, inner }],
                })
            }))
        }

        fn make_handle() -> lifecycle::Handle {
            let mut manager = lifecycle::Manager::builder("test")
                .with_trap_signals(false)
                .with_prestop_check(false)
                .with_shutdown_token(CancellationToken::new())
                .build();
            manager.register("test-component", lifecycle::ComponentOptions::new())
        }

        struct Run {
            id: Uuid,
            context: Arc<AppContext>,
            sends: Arc<std::sync::Mutex<Vec<u64>>>,
            aborts: Arc<std::sync::atomic::AtomicUsize>,
            result: Result<(), Error>,
        }

        /// Run a three-part job on a fresh row, calling `before_process` between commits and the
        /// `on_send` hook during deliveries.
        async fn run_job<H, B, BF>(on_send: H, before_process: B, fail_sends: bool) -> Run
        where
            H: FnOnce(Uuid, PgPool) -> Option<SendHook>,
            B: Fn(usize, Uuid, Arc<AppContext>) -> BF,
            BF: Future<Output = ()>,
        {
            run_job_after_claim(None, consume_all(), on_send, before_process, fail_sends).await
        }

        async fn run_job_after_claim<H, B, BF>(
            recovered: Option<serde_json::Value>,
            transform: Arc<ParserFn>,
            on_send: H,
            before_process: B,
            fail_sends: bool,
        ) -> Run
        where
            H: FnOnce(Uuid, PgPool) -> Option<SendHook>,
            B: Fn(usize, Uuid, Arc<AppContext>) -> BF,
            BF: Future<Output = ()>,
        {
            let context = test_db::context().await;
            let parts: Vec<String> = (1..=3).map(|n| format!("part-{n}")).collect();
            let state = JobState {
                parts: parts
                    .iter()
                    .map(|key| PartState {
                        key: key.clone(),
                        current_offset: 0,
                        total_size: Some(PART_BYTES as u64),
                    })
                    .collect(),
                trial: None,
                events_emitted: 0,
            };
            let mut row = test_db::JobRowFixture {
                status: JobStatus::Running,
                lease_id: "lease-from-claim",
                state: state.clone(),
                sink: serde_json::json!({"type": "capture", "send_rate": SEND_RATE}),
                pending_commit: None,
            };
            if recovered.is_some() {
                row.state.parts[0].current_offset = PART_BYTES as u64;
                row.state.events_emitted = 1;
                row.pending_commit = recovered;
            }
            let id = test_db::insert_job(&context.db, &row).await;
            let mut model = test_db::claimed_model(id, &row);
            assert!(!model
                .recover_interrupted_commit(context.clone())
                .await
                .unwrap());
            let sends = Arc::new(std::sync::Mutex::new(Vec::new()));
            let aborts = Arc::new(std::sync::atomic::AtomicUsize::new(0));
            let mut next = Some(Job {
                context: context.clone(),
                handle: make_handle(),
                job_id: id,
                chunk_size: PART_BYTES,
                state: Mutex::new(state),
                model: Mutex::new(model),
                source: Box::new(MemorySource { parts }),
                transform,
                sink: Mutex::new(Box::new(RecordingEmitter {
                    send_rate: SEND_RATE,
                    sends: sends.clone(),
                    aborts: aborts.clone(),
                    on_send: on_send(id, context.db.clone()),
                    fail_sends,
                })),
                checkpoint: Mutex::new(None),
            });

            let mut step = 0;
            let result = loop {
                let Some(job) = next else { break Ok(()) };
                before_process(step, id, context.clone()).await;
                step += 1;
                match job.process().await {
                    Ok(job) => next = job,
                    Err(e) => break Err(e),
                }
            };
            Run {
                id,
                context,
                sends,
                aborts,
                result,
            }
        }

        /// A delivery hook that runs `action` on the job row during the first send.
        fn during_first_send<F, Fut>(action: F) -> impl FnOnce(Uuid, PgPool) -> Option<SendHook>
        where
            F: Fn(PgPool, Uuid) -> Fut + Send + Sync + 'static,
            Fut: Future<Output = ()> + Send + 'static,
        {
            move |id, pool| {
                let action = Arc::new(action);
                let hook: SendHook = Arc::new(move |index| {
                    let (action, pool) = (action.clone(), pool.clone());
                    Box::pin(async move {
                        if index == 0 {
                            action(pool, id).await;
                        }
                    })
                });
                Some(hook)
            }
        }

        fn no_hook(_id: Uuid, _pool: PgPool) -> Option<SendHook> {
            None
        }

        fn offsets(stored: &test_db::StoredJob) -> Vec<u64> {
            stored
                .state
                .parts
                .iter()
                .map(|p| p.current_offset)
                .collect()
        }

        async fn no_op(_step: usize, _id: Uuid, _context: Arc<AppContext>) {}

        #[tokio::test]
        async fn test_resent_chunk_keeps_its_interruption_count() {
            let markers = Arc::new(std::sync::Mutex::new(Vec::new()));
            let recorded = markers.clone();
            let run = run_job_after_claim(
                Some(serde_json::json!(
                    {"key": "part-1", "from_offset": 0, "to_offset": PART_BYTES, "interrupted": 1, "events": 1}
                )),
                one_event_per_chunk(),
                move |id, pool| {
                    let hook: SendHook = Arc::new(move |_index| {
                        let (pool, recorded) = (pool.clone(), recorded.clone());
                        Box::pin(async move {
                            let marker = test_db::read_job(&pool, id).await.pending_commit.unwrap();
                            recorded
                                .lock()
                                .unwrap()
                                .push((marker["interrupted"].clone(), marker["events"].clone()));
                        })
                    });
                    Some(hook)
                },
                no_op,
                false,
            )
            .await;

            run.result.unwrap();
            assert_eq!(
                *markers.lock().unwrap(),
                vec![
                    (serde_json::json!(2), serde_json::json!(1)),
                    (serde_json::json!(0), serde_json::json!(1)),
                    (serde_json::json!(0), serde_json::json!(1))
                ],
                "only the re-sent chunk carries the count, so a crash loop on it reaches the cap"
            );
            let stored = test_db::read_job(&run.context.db, run.id).await;
            assert_eq!(stored.status, "completed");
            assert_eq!(
                stored.state.events_emitted, 3,
                "the re-sent chunk's events count once"
            );
            assert_eq!(stored.pending_commit, None);
            test_db::delete_job(&run.context.db, run.id).await;
        }

        #[tokio::test]
        async fn test_send_rate_change_applies_from_the_next_chunk() {
            let run = run_job(
                during_first_send(|pool, id| async move {
                    test_db::set_send_rate(&pool, id, 250).await;
                }),
                no_op,
                false,
            )
            .await;

            run.result.unwrap();
            assert_eq!(*run.sends.lock().unwrap(), vec![SEND_RATE, 250, 250]);
            let stored = test_db::read_job(&run.context.db, run.id).await;
            assert_eq!(stored.status, "completed");
            assert_eq!(offsets(&stored), vec![PART_BYTES as u64; 3]);
            assert_eq!(stored.pending_commit, None);
            test_db::delete_job(&run.context.db, run.id).await;
        }

        #[tokio::test]
        async fn test_operator_change_between_chunks_sends_nothing_more() {
            for operator_status in [JobStatus::Paused, JobStatus::Completed, JobStatus::Failed] {
                // Step 0 fetches part 1 and step 1 commits it, so this lands before any send.
                let run = run_job(
                    no_hook,
                    |step, id, context| async move {
                        if step == 1 {
                            test_db::operator_sets(&context.db, id, operator_status).await;
                        }
                    },
                    false,
                )
                .await;

                run.result.unwrap();
                assert!(run.sends.lock().unwrap().is_empty(), "{operator_status}");
                assert_eq!(
                    run.aborts.load(std::sync::atomic::Ordering::SeqCst),
                    1,
                    "the emitted chunk's transaction is aborted, not left open"
                );
                let stored = test_db::read_job(&run.context.db, run.id).await;
                assert_eq!(stored.status, operator_status.to_string());
                assert_eq!(offsets(&stored), vec![0, 0, 0], "{operator_status}");
                assert_eq!(stored.pending_commit, None);
                assert_eq!(stored.lease_id, None, "{operator_status}");
                test_db::delete_job(&run.context.db, run.id).await;
            }
        }

        #[tokio::test]
        async fn test_operator_change_during_send_records_the_chunk_and_stops() {
            for operator_status in [JobStatus::Paused, JobStatus::Completed, JobStatus::Failed] {
                let run = run_job(
                    during_first_send(move |pool, id| async move {
                        test_db::operator_sets(&pool, id, operator_status).await;
                    }),
                    no_op,
                    false,
                )
                .await;

                run.result.unwrap();
                assert_eq!(run.sends.lock().unwrap().len(), 1, "{operator_status}");
                let stored = test_db::read_job(&run.context.db, run.id).await;
                assert_eq!(stored.status, operator_status.to_string());
                assert_eq!(
                    offsets(&stored),
                    vec![PART_BYTES as u64, 0, 0],
                    "the sent chunk is recorded, so a resume does not send it again"
                );
                assert_eq!(stored.pending_commit, None);
                assert_eq!(stored.lease_id, None, "{operator_status}");
                test_db::delete_job(&run.context.db, run.id).await;
            }
        }

        #[tokio::test]
        async fn test_lost_lease_drops_the_job_without_sending() {
            let run = run_job(
                no_hook,
                |step, id, context| async move {
                    if step == 1 {
                        test_db::resume_elsewhere(&context.db, id).await;
                    }
                },
                false,
            )
            .await;

            assert!(run.result.is_err());
            assert!(run.sends.lock().unwrap().is_empty());
            let stored = test_db::read_job(&run.context.db, run.id).await;
            assert_eq!(stored.status, "running");
            assert_eq!(offsets(&stored), vec![0, 0, 0]);
            test_db::delete_job(&run.context.db, run.id).await;
        }

        #[tokio::test]
        async fn test_failed_send_on_an_operator_paused_row_rolls_back_and_releases_the_lease() {
            let run = run_job(
                during_first_send(|pool, id| async move {
                    test_db::operator_sets(&pool, id, JobStatus::Paused).await;
                }),
                no_op,
                true,
            )
            .await;

            assert!(run.result.is_err());
            let stored = test_db::read_job(&run.context.db, run.id).await;
            assert_eq!(stored.status, "paused");
            assert_eq!(
                offsets(&stored),
                vec![0, 0, 0],
                "a resume re-sends the chunk"
            );
            assert_eq!(stored.pending_commit, None);
            assert_eq!(stored.lease_id, None);
            test_db::delete_job(&run.context.db, run.id).await;
        }
    }

    mod shutdown_guard_tests {
        use tokio_util::sync::CancellationToken;

        fn make_handle(token: CancellationToken) -> lifecycle::Handle {
            let mut manager = lifecycle::Manager::builder("test")
                .with_trap_signals(false)
                .with_prestop_check(false)
                .with_shutdown_token(token)
                .build();
            manager.register("test-component", lifecycle::ComponentOptions::new())
        }

        /// shutdown_guard delegates to handle.is_shutting_down(); verify the
        /// underlying predicate returns false before cancellation.
        #[test]
        fn handle_not_shutting_down_before_cancel() {
            let token = CancellationToken::new();
            let handle = make_handle(token);
            assert!(!handle.is_shutting_down());
        }

        /// After the token is cancelled, is_shutting_down() returns true --
        /// which is what shutdown_guard uses to bail out of commit phases.
        #[test]
        fn handle_shutting_down_after_cancel() {
            let token = CancellationToken::new();
            let handle = make_handle(token.clone());
            token.cancel();
            assert!(handle.is_shutting_down());
        }
    }

    /// Drives the real DB-free read loop (`select_and_fetch_next_chunk`) against
    /// the real `DateRangeExportSource` + `PlainGzipExtractor` over `httpmock`, to
    /// lock the end-to-end staging-cleanup invariant: across a whole run the loop
    /// alone (without `cleanup_after_job`) leaves no `.raw` behind and never keeps
    /// more than one part's `.raw` on disk at a time. The source frees each `.raw`
    /// on its own EOF read (see [`crate::source::read_prepared_chunk`]); the
    /// decisive read-level proof of that lives in `date_range_export`'s tests.
    mod staging_cleanup_invariant_tests {
        use super::*;
        use crate::extractor::ExtractorType;
        use crate::source::date_range_export::{AuthConfig, DateRangeExportSource};
        use chrono::{TimeZone, Utc};
        use flate2::write::GzEncoder;
        use flate2::Compression;
        use std::io::Write;
        use std::path::{Path, PathBuf};
        use tempfile::TempDir;

        pub(super) fn gzip_bytes(data: &[u8]) -> Vec<u8> {
            let mut encoder = GzEncoder::new(Vec::new(), Compression::default());
            encoder.write_all(data).unwrap();
            encoder.finish().unwrap()
        }

        /// Build a source spanning `hours` one-hour intervals (one key per hour),
        /// all served the same body by the mock, decoded with `extractor`.
        /// `remote` selects temp-bucket staging; `None` is the local streaming path.
        pub(super) fn build_source_with(
            base_url: String,
            staging: &Path,
            hours: u32,
            extractor: ExtractorType,
            remote: Option<crate::source::RemoteStaging>,
        ) -> DateRangeExportSource {
            let start = Utc.with_ymd_and_hms(2023, 1, 1, 0, 0, 0).unwrap();
            let end = Utc.with_ymd_and_hms(2023, 1, 1, hours, 0, 0).unwrap();
            DateRangeExportSource::builder(
                base_url,
                start,
                end,
                3600,
                extractor.create_extractor(),
                staging.to_path_buf(),
            )
            .with_auth(AuthConfig::None)
            .with_date_format("%Y-%m-%dT%H:%M:%SZ".to_string())
            .with_headers(HashMap::new())
            .with_remote_staging(remote)
            .build()
            .unwrap()
        }

        pub(super) fn build_source_with_extractor(
            base_url: String,
            staging: &Path,
            hours: u32,
            extractor: ExtractorType,
        ) -> DateRangeExportSource {
            build_source_with(base_url, staging, hours, extractor, None)
        }

        fn build_source(base_url: String, staging: &Path, hours: u32) -> DateRangeExportSource {
            build_source_with(base_url, staging, hours, ExtractorType::PlainGzip, None)
        }

        /// A transform that consumes the whole chunk and emits no events. The
        /// cleanup invariant is about offsets reaching `total` and each `.raw`
        /// being freed on its EOF read — parsing semantics are irrelevant here, so
        /// this keeps the loop advancing deterministically by exactly the bytes read.
        fn consume_all_transform() -> Arc<ParserFn> {
            Arc::new(Box::new(|bytes: Vec<u8>| {
                Ok(Parsed {
                    consumed: bytes.len(),
                    data: Vec::new(),
                })
            }))
        }

        /// A transform that runs the real newline parser to compute `consumed`
        /// (validating each complete line as JSON) but emits no events. This
        /// reproduces the parser's real boundary behavior — a chunk with no line
        /// terminator yields `consumed == 1` — which is what the oversized-record
        /// guard keys on, without standing up the full captured-event pipeline.
        pub(super) fn newline_consumed_transform() -> Arc<ParserFn> {
            Arc::new(Box::new(|bytes: Vec<u8>| {
                let parser = crate::parse::format::newline_delim(true, |line: &str| {
                    serde_json::from_str::<serde_json::Value>(line)
                        .map(|_| ())
                        .map_err(anyhow::Error::from)
                });
                let parsed = parser(bytes)?;
                Ok(Parsed {
                    consumed: parsed.consumed,
                    data: Vec::new(),
                })
            }))
        }

        fn count_raw_files(dir: &Path) -> usize {
            let mut count = 0;
            let mut stack = vec![dir.to_path_buf()];
            while let Some(d) = stack.pop() {
                let Ok(entries) = std::fs::read_dir(&d) else {
                    continue;
                };
                for entry in entries.flatten() {
                    let path: PathBuf = entry.path();
                    if path.is_dir() {
                        stack.push(path);
                    } else if path.extension().and_then(|e| e.to_str()) == Some("raw") {
                        count += 1;
                    }
                }
            }
            count
        }

        /// Drive `select_and_fetch_next_chunk` until it reports all parts done,
        /// up to `max_iterations`. Returns whether it completed within budget.
        pub(super) async fn drive_loop_to_completion(
            state: &Mutex<JobState>,
            model: &Mutex<JobModel>,
            source: &dyn DataSource,
            transform: &Arc<ParserFn>,
            chunk_size: usize,
            max_iterations: usize,
        ) -> Result<bool, Error> {
            for _ in 0..max_iterations {
                let next = select_and_fetch_next_chunk(
                    state,
                    model,
                    source,
                    transform,
                    chunk_size,
                    Uuid::now_v7(),
                )
                .await?;
                if next.is_none() {
                    return Ok(true);
                }
            }
            Ok(false)
        }

        pub(super) fn dummy_model(state: JobState) -> JobModel {
            JobModel {
                id: Uuid::now_v7(),
                team_id: 1,
                created_at: Utc::now(),
                updated_at: Utc::now(),
                lease_id: None,
                leased_until: None,
                status: model::JobStatus::Running,
                status_message: None,
                display_status_message: None,
                state: Some(state),
                import_config: config::JobConfig {
                    source: config::SourceConfig::Folder(config::FolderSourceConfig {
                        path: "/tmp".to_string(),
                    }),
                    data_format: crate::parse::format::FormatConfig::JsonLines {
                        skip_blanks: true,
                        content: crate::parse::content::ContentType::Captured,
                    },
                    sink: config::SinkConfig::NoOp,
                    import_events: true,
                    generate_identify_events: false,
                    generate_group_identify_events: false,
                },
                secrets: config::JobSecrets {
                    secrets: HashMap::new(),
                },
                backoff_attempt: 0,
                backoff_until: None,
                pending_commit: None,
                was_leased: false,
                recovered_commit: None,
            }
        }

        pub(super) fn job_state(keys: &[String]) -> JobState {
            JobState {
                parts: keys
                    .iter()
                    .map(|key| PartState {
                        key: key.clone(),
                        current_offset: 0,
                        total_size: None,
                    })
                    .collect(),
                trial: None,
                events_emitted: 0,
            }
        }

        #[tokio::test]
        async fn test_single_part_raw_cleaned_up_by_read_loop_without_job_cleanup() {
            let server = MockServer::start();
            let mut body = String::new();
            for i in 0..2000 {
                body.push_str(&format!("{{\"event\":\"e{i}\"}}\n"));
            }
            let _mock = server.mock(|when, then| {
                when.method(Method::GET).path("/export");
                then.status(200).body(gzip_bytes(body.as_bytes()));
            });

            let staging = TempDir::new().unwrap();
            let source = build_source(server.url("/export"), staging.path(), 1);
            source.prepare_for_job().await.unwrap();
            let keys = source.keys().await.unwrap();
            assert_eq!(keys.len(), 1);

            let state = Mutex::new(job_state(&keys));
            let model = Mutex::new(dummy_model(job_state(&keys)));
            let transform = consume_all_transform();

            // Small chunk so the loop takes several passes before EOF.
            let mut peak_raw = 0;
            loop {
                let next = select_and_fetch_next_chunk(
                    &state,
                    &model,
                    &source,
                    &transform,
                    1024,
                    Uuid::now_v7(),
                )
                .await
                .unwrap();
                peak_raw = peak_raw.max(count_raw_files(staging.path()));
                if next.is_none() {
                    break;
                }
            }

            // The loop alone (no cleanup_after_job) must have deleted the .raw:
            // the source frees it on the EOF read.
            assert_eq!(
                count_raw_files(staging.path()),
                0,
                "the read loop must delete the part's .raw once it is fully read, without cleanup_after_job"
            );
            assert_eq!(
                peak_raw, 1,
                "exactly one .raw should exist while the part is in flight"
            );
        }

        #[tokio::test]
        async fn test_part_fully_consumed_at_eof_read_completes_without_offset_overshoot() {
            // Streaming sources only learn a part's total size on the read that
            // observes EOF. When that same read fully consumes the remaining bytes,
            // the offset lands exactly at the total while the job still has
            // total_size = None. The next loop iteration must then recognize the
            // part as done — not fetch an empty chunk, "consume" a phantom byte,
            // and push the offset past the total (which makes is_done() unreachable
            // and the job crawl one byte per iteration forever).
            let server = MockServer::start();
            let mut body = String::new();
            for i in 0..200 {
                body.push_str(&format!("{{\"event\":\"e{i}\"}}\n"));
            }
            let total_size = body.len() as u64;
            let _mock = server.mock(|when, then| {
                when.method(Method::GET).path("/export");
                then.status(200).body(gzip_bytes(body.as_bytes()));
            });

            let staging = TempDir::new().unwrap();
            let source = build_source(server.url("/export"), staging.path(), 1);
            source.prepare_for_job().await.unwrap();
            let keys = source.keys().await.unwrap();
            assert_eq!(keys.len(), 1);

            let state = Mutex::new(job_state(&keys));
            let model = Mutex::new(dummy_model(job_state(&keys)));
            // Real newline parser so `consumed` behaves exactly as in production.
            let transform = newline_consumed_transform();

            // Generous iteration budget: a healthy run needs ~(total/chunk + 2)
            // iterations. Far more than that means the loop is stuck.
            let chunk_size = 1024usize;
            let max_iterations = (total_size as usize / chunk_size) + 10;
            let mut completed = false;
            for _ in 0..max_iterations {
                let next = select_and_fetch_next_chunk(
                    &state,
                    &model,
                    &source,
                    &transform,
                    chunk_size,
                    Uuid::now_v7(),
                )
                .await
                .unwrap();

                let offset = state.lock().await.parts[0].current_offset;
                assert!(
                    offset <= total_size,
                    "part offset {offset} overshot total size {total_size}: the job would crawl forever"
                );

                if next.is_none() {
                    completed = true;
                    break;
                }
            }

            assert!(
                completed,
                "job did not finish the part within {max_iterations} iterations (offset {} / total {total_size})",
                state.lock().await.parts[0].current_offset
            );
        }

        #[tokio::test]
        async fn test_part_with_trailing_blank_lines_completes() {
            // Data whose final bytes are blank lines must still complete: the blanks
            // are only seen on the EOF read, after which the streaming source has
            // freed the part and can never serve those bytes again.
            let server = MockServer::start();
            let mut body = String::new();
            for i in 0..200 {
                body.push_str(&format!("{{\"event\":\"e{i}\"}}\n"));
            }
            body.push('\n');
            let total_size = body.len() as u64;
            let _mock = server.mock(|when, then| {
                when.method(Method::GET).path("/export");
                then.status(200).body(gzip_bytes(body.as_bytes()));
            });

            let staging = TempDir::new().unwrap();
            let source = build_source(server.url("/export"), staging.path(), 1);
            source.prepare_for_job().await.unwrap();
            let keys = source.keys().await.unwrap();

            let state = Mutex::new(job_state(&keys));
            let model = Mutex::new(dummy_model(job_state(&keys)));
            let transform = newline_consumed_transform();

            let chunk_size = 1024usize;
            let max_iterations = (total_size as usize / chunk_size) + 10;
            let mut completed = false;
            for _ in 0..max_iterations {
                let next = select_and_fetch_next_chunk(
                    &state,
                    &model,
                    &source,
                    &transform,
                    chunk_size,
                    Uuid::now_v7(),
                )
                .await
                .unwrap();
                if next.is_none() {
                    completed = true;
                    break;
                }
            }

            assert!(completed, "trailing blank lines must not stall the part");
            assert_eq!(state.lock().await.parts[0].current_offset, total_size);
        }

        #[tokio::test]
        async fn test_poisoned_part_with_overshot_offset_self_heals() {
            // A job written by a worker version with the overshoot bug has a part
            // whose stored offset exceeds its total size. On resume that part must
            // be treated as done (its real bytes were all consumed before the
            // phantom crawl began) so the job can move on to the remaining parts.
            let server = MockServer::start();
            let mut body = String::new();
            for i in 0..200 {
                body.push_str(&format!("{{\"event\":\"e{i}\"}}\n"));
            }
            let _mock = server.mock(|when, then| {
                when.method(Method::GET).path("/export");
                then.status(200).body(gzip_bytes(body.as_bytes()));
            });

            let staging = TempDir::new().unwrap();
            let source = build_source(server.url("/export"), staging.path(), 2);
            source.prepare_for_job().await.unwrap();
            let keys = source.keys().await.unwrap();
            assert_eq!(keys.len(), 2);

            // Part 1 poisoned as in production: offset way past its recorded total.
            let poisoned = JobState {
                parts: vec![
                    PartState {
                        key: keys[0].clone(),
                        current_offset: 25_328_072,
                        total_size: Some(24_441_157),
                    },
                    PartState {
                        key: keys[1].clone(),
                        current_offset: 0,
                        total_size: None,
                    },
                ],
                trial: None,
                events_emitted: 0,
            };
            let state = Mutex::new(poisoned.clone());
            let model = Mutex::new(dummy_model(poisoned));
            let transform = newline_consumed_transform();

            let mut completed = false;
            for _ in 0..20 {
                let next = select_and_fetch_next_chunk(
                    &state,
                    &model,
                    &source,
                    &transform,
                    1024,
                    Uuid::now_v7(),
                )
                .await
                .unwrap();
                if next.is_none() {
                    completed = true;
                    break;
                }
            }

            assert!(
                completed,
                "job with an overshot part must skip it and finish the rest"
            );
            let state = state.lock().await;
            assert_eq!(
                state.parts[0].current_offset, 25_328_072,
                "poisoned part must be skipped untouched, not re-read"
            );
            assert_eq!(
                state.parts[1].total_size,
                Some(state.parts[1].current_offset)
            );
        }

        #[tokio::test]
        async fn test_record_larger_than_chunk_size_fails_fast() {
            // A single JSON record with no interior newline, larger than the read window.
            // Without the guard the loop would crawl one byte per iteration forever.
            let server = MockServer::start();
            let huge_record = format!("{{\"event\":\"{}\"}}", "x".repeat(5000));
            assert!(!huge_record.contains('\n'));
            let _mock = server.mock(|when, then| {
                when.method(Method::GET).path("/export");
                then.status(200).body(gzip_bytes(huge_record.as_bytes()));
            });

            let staging = TempDir::new().unwrap();
            let source = build_source(server.url("/export"), staging.path(), 1);
            source.prepare_for_job().await.unwrap();
            let keys = source.keys().await.unwrap();

            let state = Mutex::new(job_state(&keys));
            let model = Mutex::new(dummy_model(job_state(&keys)));
            let transform = newline_consumed_transform();

            // Read window (1024) is far smaller than the 5000-byte record.
            let result = select_and_fetch_next_chunk(
                &state,
                &model,
                &source,
                &transform,
                1024,
                Uuid::now_v7(),
            )
            .await;

            let err = result.expect_err("oversized record must fail fast, not crawl");
            // Actionable, user-facing message so the job pauses (not a transient retry),
            // free of internal tuning values; the internal chain keeps chunk_size.
            let user_msg = crate::error::get_user_message(&err);
            assert!(
                user_msg.contains("too large to process"),
                "unexpected message: {user_msg}"
            );
            assert!(
                !user_msg.contains("chunk_size") && !user_msg.contains("1024"),
                "public message must not leak internal tuning values: {user_msg}"
            );
            assert!(
                format!("{err:#}").contains("chunk_size=1024"),
                "internal chain must carry the limit detail: {err:#}"
            );
            assert!(!crate::error::is_rate_limited_error(&err));
            assert!(!crate::error::is_timeout_error(&err));
            assert!(!crate::error::is_transient_network_error(&err));
            assert!(!crate::error::is_transient_server_error(&err));
            assert!(!crate::error::is_transient_object_store_error(&err));
        }

        #[tokio::test]
        async fn test_multi_part_peak_staging_bounded_to_one_part() {
            let server = MockServer::start();
            let mut body = String::new();
            for i in 0..2000 {
                body.push_str(&format!("{{\"event\":\"e{i}\"}}\n"));
            }
            let _mock = server.mock(|when, then| {
                when.method(Method::GET).path("/export");
                then.status(200).body(gzip_bytes(body.as_bytes()));
            });

            let staging = TempDir::new().unwrap();
            // Two one-hour intervals -> two keys processed back to back.
            let source = build_source(server.url("/export"), staging.path(), 2);
            source.prepare_for_job().await.unwrap();
            let keys = source.keys().await.unwrap();
            assert_eq!(keys.len(), 2);

            let state = Mutex::new(job_state(&keys));
            let model = Mutex::new(dummy_model(job_state(&keys)));
            let transform = consume_all_transform();

            // Two invariants across the whole run:
            //  - peak coexisting .raw files stays at 1 (each part's .raw is freed
            //    on its EOF read, before the next part is read), and
            //  - after the loop, zero .raw remain — cleaned by the loop alone,
            //    including the last part (which has no later prepare-time sweep).
            let mut peak_raw = 0;
            loop {
                let next = select_and_fetch_next_chunk(
                    &state,
                    &model,
                    &source,
                    &transform,
                    1024,
                    Uuid::now_v7(),
                )
                .await
                .unwrap();
                peak_raw = peak_raw.max(count_raw_files(staging.path()));
                if next.is_none() {
                    break;
                }
            }

            assert_eq!(
                peak_raw, 1,
                "peak staging must be bounded to a single part's .raw; coexisting raws means a completed part was not cleaned up"
            );
            assert_eq!(
                count_raw_files(staging.path()),
                0,
                "all parts' .raw files must be cleaned up by the read loop alone"
            );
        }

        /// The real read loop over a remote-staged (temp-bucket) source: parts stage on
        /// first touch, sizes are known immediately (no lazy-size pass), every part
        /// completes with the parser consuming real newline-delimited records, and no
        /// `.raw` survives staging (deleted right after a successful ingest).
        #[tokio::test]
        async fn test_remote_staged_source_completes_all_parts_through_read_loop() {
            use crate::source::RemoteStaging;
            use crate::staging::TempBucketBackend;
            use object_store::memory::InMemory;

            let server = MockServer::start();
            let mut body = String::new();
            for i in 0..500 {
                body.push_str(&format!("{{\"event\":\"e{i}\"}}\n"));
            }
            let _mock = server.mock(|when, then| {
                when.method(Method::GET).path("/export");
                then.status(200).body(gzip_bytes(body.as_bytes()));
            });

            let staging = TempDir::new().unwrap();
            let remote = RemoteStaging {
                backend: Arc::new(TempBucketBackend::new(
                    Arc::new(InMemory::new()),
                    "staging/",
                    "job-LOOP",
                )),
                extractor_type: ExtractorType::PlainGzip,
                max_plaintext_bytes: 0,
            };
            let source = build_source_with(
                server.url("/export"),
                staging.path(),
                2,
                ExtractorType::PlainGzip,
                Some(remote),
            );
            source.prepare_for_job().await.unwrap();
            let keys = source.keys().await.unwrap();
            assert_eq!(keys.len(), 2);

            let state = Mutex::new(job_state(&keys));
            let model = Mutex::new(dummy_model(job_state(&keys)));
            // Real newline parser: proves record reassembly across chunk boundaries
            // works over ranged backend reads, not just a consume-everything stub.
            let transform = newline_consumed_transform();

            loop {
                let next = select_and_fetch_next_chunk(
                    &state,
                    &model,
                    &source,
                    &transform,
                    1024,
                    Uuid::now_v7(),
                )
                .await
                .unwrap();
                // A successful stage never leaves a .raw behind.
                assert_eq!(count_raw_files(staging.path()), 0);
                if next.is_none() {
                    break;
                }
            }

            let final_state = state.lock().await;
            for part in &final_state.parts {
                assert!(part.is_done(), "part {} must complete", part.key);
                assert_eq!(part.total_size, Some(body.len() as u64));
            }
        }

        /// Wraps a source to observe (and optionally fail) `cleanup_key` calls, so the
        /// post-commit hook's exactly-once / best-effort behavior is observable.
        struct SpySource<S> {
            inner: S,
            cleanup_calls: std::sync::atomic::AtomicUsize,
            fail_cleanup: bool,
        }

        impl<S> SpySource<S> {
            fn new(inner: S, fail_cleanup: bool) -> Self {
                Self {
                    inner,
                    cleanup_calls: std::sync::atomic::AtomicUsize::new(0),
                    fail_cleanup,
                }
            }

            fn cleanups(&self) -> usize {
                self.cleanup_calls.load(std::sync::atomic::Ordering::SeqCst)
            }
        }

        #[async_trait::async_trait]
        impl<S: DataSource> DataSource for SpySource<S> {
            async fn keys(&self) -> Result<Vec<String>, Error> {
                self.inner.keys().await
            }
            async fn size(&self, key: &str) -> Result<Option<u64>, Error> {
                self.inner.size(key).await
            }
            async fn get_chunk(&self, key: &str, offset: u64, size: u64) -> Result<Vec<u8>, Error> {
                self.inner.get_chunk(key, offset, size).await
            }
            async fn prepare_key(&self, key: &str) -> Result<(), Error> {
                self.inner.prepare_key(key).await
            }
            async fn prepare_for_job(&self) -> Result<(), Error> {
                self.inner.prepare_for_job().await
            }
            async fn cleanup_after_job(&self) -> Result<(), Error> {
                self.inner.cleanup_after_job().await
            }
            async fn cleanup_key(&self, key: &str) -> Result<(), Error> {
                self.cleanup_calls
                    .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
                if self.fail_cleanup {
                    return Err(Error::msg("simulated cleanup failure"));
                }
                self.inner.cleanup_key(key).await
            }
        }

        fn remote_staging_for(
            store: Arc<object_store::memory::InMemory>,
        ) -> crate::source::RemoteStaging {
            crate::source::RemoteStaging {
                backend: Arc::new(crate::staging::TempBucketBackend::new(
                    store, "staging/", "job-HOOK",
                )),
                extractor_type: ExtractorType::PlainGzip,
                max_plaintext_bytes: 0,
            }
        }

        /// Drive the real fetch loop over a remote source, mirroring do_commit's
        /// post-complete sequence per checkpoint: persist the offset advance to the model,
        /// then invoke the hook. Returns the spy so callers can assert cleanup counts.
        async fn run_loop_with_hook(
            source: &SpySource<DateRangeExportSource>,
            keys: &[String],
        ) -> (Mutex<JobModel>, usize) {
            let state = Mutex::new(job_state(keys));
            let model = Mutex::new(dummy_model(job_state(keys)));
            let transform = newline_consumed_transform();
            let mut commits = 0usize;
            loop {
                let next = select_and_fetch_next_chunk(
                    &state,
                    &model,
                    source,
                    &transform,
                    1024,
                    Uuid::now_v7(),
                )
                .await
                .unwrap();
                let Some((key, parsed, _)) = next else { break };
                {
                    let mut model = model.lock().await;
                    model
                        .state
                        .as_mut()
                        .unwrap()
                        .advance_part_offset(&key, parsed.consumed as u64)
                        .unwrap();
                }
                cleanup_committed_part_if_done(source, &model, &key).await;
                commits += 1;
            }
            (model, commits)
        }

        #[tokio::test]
        async fn test_post_commit_hook_frees_each_remote_part_exactly_once() {
            let server = MockServer::start();
            let mut body = String::new();
            for i in 0..500 {
                body.push_str(&format!("{{\"event\":\"e{i}\"}}\n"));
            }
            let _mock = server.mock(|when, then| {
                when.method(Method::GET).path("/export");
                then.status(200).body(gzip_bytes(body.as_bytes()));
            });

            let staging = TempDir::new().unwrap();
            let store = Arc::new(object_store::memory::InMemory::new());
            let remote = remote_staging_for(Arc::clone(&store));
            let backend = Arc::clone(&remote.backend);
            let source = SpySource::new(
                build_source_with(
                    server.url("/export"),
                    staging.path(),
                    2,
                    ExtractorType::PlainGzip,
                    Some(remote),
                ),
                false,
            );
            source.prepare_for_job().await.unwrap();
            let keys = source.keys().await.unwrap();

            let (_model, commits) = run_loop_with_hook(&source, &keys).await;

            // Multiple commits per part (small chunk size), but exactly one cleanup per
            // part: the hook fires only on the commit that makes the part done.
            assert!(commits > keys.len(), "expected multiple commits per part");
            assert_eq!(source.cleanups(), keys.len());
            for key in &keys {
                assert_eq!(
                    backend.size(key).await.unwrap(),
                    None,
                    "staged object for a committed part must be deleted"
                );
            }
        }

        #[tokio::test]
        async fn test_post_commit_hook_preserves_staging_until_part_is_done() {
            // Encodes the rollback / crash-between-commit-stages guarantee: the hook only
            // fires for a durably-done part, so a part mid-flight (or with its offset
            // reverted) keeps its staged object for the byte-identical re-read.
            let server = MockServer::start();
            let body = b"{\"event\":\"a\"}\n{\"event\":\"b\"}\n";
            let _mock = server.mock(|when, then| {
                when.method(Method::GET).path("/export");
                then.status(200).body(gzip_bytes(body));
            });

            let staging = TempDir::new().unwrap();
            let store = Arc::new(object_store::memory::InMemory::new());
            let remote = remote_staging_for(Arc::clone(&store));
            let backend = Arc::clone(&remote.backend);
            let source = SpySource::new(
                build_source_with(
                    server.url("/export"),
                    staging.path(),
                    1,
                    ExtractorType::PlainGzip,
                    Some(remote),
                ),
                false,
            );
            source.prepare_for_job().await.unwrap();
            let key = source.keys().await.unwrap().remove(0);
            source.prepare_key(&key).await.unwrap();

            let model = Mutex::new(dummy_model(job_state(std::slice::from_ref(&key))));
            {
                let mut model = model.lock().await;
                let state = model.state.as_mut().unwrap();
                state.parts[0].total_size = Some(body.len() as u64);
                // Partially consumed (e.g. after a rollback reverted a later advance).
                state.parts[0].current_offset = 14;
            }

            cleanup_committed_part_if_done(&source, &model, &key).await;
            assert_eq!(
                source.cleanups(),
                0,
                "hook must not fire for a part mid-flight"
            );
            assert_eq!(
                backend.size(&key).await.unwrap(),
                Some(body.len() as u64),
                "staged object must survive for the re-read"
            );
        }

        #[tokio::test]
        async fn test_post_commit_hook_cleanup_failure_is_best_effort() {
            let server = MockServer::start();
            let body = b"{\"event\":\"a\"}\n";
            let _mock = server.mock(|when, then| {
                when.method(Method::GET).path("/export");
                then.status(200).body(gzip_bytes(body));
            });

            let staging = TempDir::new().unwrap();
            let store = Arc::new(object_store::memory::InMemory::new());
            let remote = remote_staging_for(Arc::clone(&store));
            let source = SpySource::new(
                build_source_with(
                    server.url("/export"),
                    staging.path(),
                    1,
                    ExtractorType::PlainGzip,
                    Some(remote),
                ),
                true, // cleanup_key always fails
            );
            source.prepare_for_job().await.unwrap();
            let keys = source.keys().await.unwrap();

            // The loop must run to completion despite every cleanup failing.
            let (_model, _commits) = run_loop_with_hook(&source, &keys).await;
            assert_eq!(source.cleanups(), 1, "hook attempted the cleanup");
        }

        #[tokio::test]
        async fn test_post_commit_hook_empty_part_cleanup_is_no_op() {
            let server = MockServer::start();
            let _mock = server.mock(|when, then| {
                when.method(Method::GET).path("/export");
                then.status(404);
            });

            let staging = TempDir::new().unwrap();
            let store = Arc::new(object_store::memory::InMemory::new());
            let remote = remote_staging_for(Arc::clone(&store));
            let backend = Arc::clone(&remote.backend);
            let source = SpySource::new(
                build_source_with(
                    server.url("/export"),
                    staging.path(),
                    1,
                    ExtractorType::PlainGzip,
                    Some(remote),
                ),
                false,
            );
            source.prepare_for_job().await.unwrap();
            let keys = source.keys().await.unwrap();

            let (model, _commits) = run_loop_with_hook(&source, &keys).await;

            // The empty part commits to done (0 == 0), the hook fires, and the delete of
            // the (empty) staged object succeeds; a second invocation is a no-op.
            assert_eq!(source.cleanups(), 1);
            assert_eq!(backend.size(&keys[0]).await.unwrap(), None);
            cleanup_committed_part_if_done(&source, &model, &keys[0]).await;
            assert_eq!(source.cleanups(), 2, "double cleanup is tolerated");
        }
    }

    /// A part whose first bytes are a known compression magic is still compressed
    /// (double-compression, or a source/extractor whose compression setting does
    /// not match the file). The loop must fail fast at part offset 0 with an
    /// actionable, non-transient error - never crawl or emit a confusing utf-8
    /// parse failure. The guard sits in the shared loop, so one streaming source
    /// (post-decompression double-gzip) and one plain verbatim source (a gzip file
    /// on a non-decompressing source) together prove it covers every source shape.
    mod compression_magic_guard_tests {
        use super::staging_cleanup_invariant_tests::{
            build_source_with_extractor, dummy_model, gzip_bytes, job_state,
            newline_consumed_transform,
        };
        use super::*;
        use crate::extractor::ExtractorType;
        use crate::source::folder::FolderSource;
        use tempfile::TempDir;

        fn assert_non_transient(err: &Error) {
            assert!(!crate::error::is_rate_limited_error(err));
            assert!(!crate::error::is_timeout_error(err));
            assert!(!crate::error::is_transient_network_error(err));
            assert!(!crate::error::is_transient_server_error(err));
        }

        #[tokio::test]
        async fn test_double_gzipped_part_detected_at_offset_zero() {
            // gzip(gzip(jsonl)): the extractor strips the outer gzip, so the
            // plaintext stream *starts* with the inner gzip magic. The guard must
            // catch it at offset 0 and pause with the double-compression message.
            let server = MockServer::start();
            let mut inner = String::new();
            for i in 0..50 {
                inner.push_str(&format!("{{\"event\":\"e{i}\"}}\n"));
            }
            let double = gzip_bytes(&gzip_bytes(inner.as_bytes()));
            let _mock = server.mock(|when, then| {
                when.method(Method::GET).path("/export");
                then.status(200).body(double);
            });

            let staging = TempDir::new().unwrap();
            let source = build_source_with_extractor(
                server.url("/export"),
                staging.path(),
                1,
                ExtractorType::PlainGzip,
            );
            source.prepare_for_job().await.unwrap();
            let keys = source.keys().await.unwrap();

            let state = Mutex::new(job_state(&keys));
            let model = Mutex::new(dummy_model(job_state(&keys)));
            let transform = newline_consumed_transform();

            let err = select_and_fetch_next_chunk(
                &state,
                &model,
                &source,
                &transform,
                1024,
                Uuid::now_v7(),
            )
            .await
            .expect_err("double-gzipped data must be rejected at offset 0");

            let user_msg = crate::error::get_user_message(&err);
            assert!(
                user_msg.contains("compressed twice") && user_msg.contains("gzip"),
                "unexpected user message: {user_msg}"
            );
            assert!(
                format!("{err:#}").contains("offset 0"),
                "internal chain should record the offset, got: {err:#}"
            );
            assert_non_transient(&err);
            // The guard fires before advancing, so nothing is consumed.
            assert_eq!(state.lock().await.parts[0].current_offset, 0);
        }

        #[tokio::test]
        async fn test_plain_source_serving_gzip_file_detected_at_offset_zero() {
            // The reverse misconfiguration: a gzipped file handed to a plain
            // (verbatim, non-decompressing) source. The raw gzip magic is the first
            // thing the loop reads, so the same offset-0 guard must catch it.
            let staging = TempDir::new().unwrap();
            let gz = gzip_bytes(b"{\"event\":\"x\"}\n");
            std::fs::write(staging.path().join("data.jsonl"), &gz).unwrap();

            let source = FolderSource::new(staging.path().to_str().unwrap().to_string())
                .await
                .unwrap();
            let keys = source.keys().await.unwrap();

            let state = Mutex::new(job_state(&keys));
            let model = Mutex::new(dummy_model(job_state(&keys)));
            let transform = newline_consumed_transform();

            let err = select_and_fetch_next_chunk(
                &state,
                &model,
                &source,
                &transform,
                1024,
                Uuid::now_v7(),
            )
            .await
            .expect_err("a gzip file on a plain source must be rejected at offset 0");

            let user_msg = crate::error::get_user_message(&err);
            assert!(
                user_msg.contains("compressed") && user_msg.contains("gzip"),
                "unexpected user message: {user_msg}"
            );
            assert_non_transient(&err);
        }

        #[tokio::test]
        async fn test_non_magic_binary_still_yields_parse_error() {
            // Bucket 3: non-magic invalid-utf8 content is corrupt data, not
            // compression. The guard must NOT fire; the pre-existing utf-8 parse
            // error must surface unchanged, proving the guard does not over-claim.
            let staging = TempDir::new().unwrap();
            // 0xff is not a compression-magic first byte; the trailing newline forms
            // a complete line that fails utf-8 validation - the pre-existing path.
            std::fs::write(staging.path().join("data.jsonl"), [0xff, 0x28, 0xff, b'\n']).unwrap();

            let source = FolderSource::new(staging.path().to_str().unwrap().to_string())
                .await
                .unwrap();
            let keys = source.keys().await.unwrap();

            let state = Mutex::new(job_state(&keys));
            let model = Mutex::new(dummy_model(job_state(&keys)));
            let transform = newline_consumed_transform();

            let err = select_and_fetch_next_chunk(
                &state,
                &model,
                &source,
                &transform,
                1024,
                Uuid::now_v7(),
            )
            .await
            .expect_err("invalid-utf8 binary must still fail");

            let full = format!("{err:#}");
            assert!(
                full.contains("utf8") || full.contains("Failed to parse"),
                "expected the pre-existing parse error, got: {full}"
            );
            assert!(
                !crate::error::get_user_message(&err).contains("compressed twice"),
                "non-magic data must not be mislabeled as double-compression: {full}"
            );
            assert_non_transient(&err);
        }
    }

    #[tokio::test]
    async fn test_empty_source_listing_fails_init_with_user_message() {
        let source = MockDataSourceWithoutDateRange { keys: vec![] };
        let mut model = staging_cleanup_invariant_tests::dummy_model(JobState::default());

        let err = load_or_init_state(&source, &mut model)
            .await
            .expect_err("an empty listing must not start a job that completes at once");

        assert!(get_user_message(&err).contains("No files were found to import"));
        assert!(model.state.unwrap().parts.is_empty());
    }

    /// Streaming sources discover a part's decompressed size only on the read
    /// that observes end-of-stream. When that stream turns out to be empty (or
    /// already fully consumed at the resume offset), the first read returns an
    /// empty chunk before the job has ever seen a size - which must complete the
    /// part, not pause the job as "source returned no data". These tests pin
    /// every empty-stream shape plus the error path for genuinely broken sources.
    mod empty_part_completion_tests {
        use super::staging_cleanup_invariant_tests::{
            build_source_with_extractor, drive_loop_to_completion, dummy_model, gzip_bytes,
            job_state, newline_consumed_transform,
        };
        use super::*;
        use crate::extractor::ExtractorType;
        use std::io::Write;
        use tempfile::TempDir;
        use zip::{write::SimpleFileOptions, ZipWriter};

        /// A zip archive with no `.json.gz` members - the Amplitude-shaped
        /// equivalent of an empty export (decompresses to zero bytes).
        pub(super) fn zip_without_json_members() -> Vec<u8> {
            let mut zip = ZipWriter::new(std::io::Cursor::new(Vec::new()));
            zip.start_file("readme.txt", SimpleFileOptions::default())
                .unwrap();
            zip.write_all(b"no data for this day").unwrap();
            zip.finish().unwrap().into_inner()
        }

        /// Empty exports come in several shapes, all decompressing to zero
        /// bytes: a raw empty 200 body (Mixpanel day with no events), a valid
        /// gzip of empty content, and a zip with no data members. Each must
        /// complete the part exactly like the pre-streaming code did.
        #[tokio::test]
        async fn test_empty_export_completes_part() {
            let cases: [(&str, Vec<u8>, ExtractorType); 3] = [
                ("empty 200 body", Vec::new(), ExtractorType::PlainGzip),
                (
                    "gzip of empty content",
                    gzip_bytes(b""),
                    ExtractorType::PlainGzip,
                ),
                (
                    "zip with no json members",
                    zip_without_json_members(),
                    ExtractorType::ZipGzipJson,
                ),
            ];

            for (name, body, extractor) in cases {
                let server = MockServer::start();
                let _mock = server.mock(|when, then| {
                    when.method(Method::GET).path("/export");
                    then.status(200).body(body);
                });

                let staging = TempDir::new().unwrap();
                let source = build_source_with_extractor(
                    server.url("/export"),
                    staging.path(),
                    1,
                    extractor,
                );
                source.prepare_for_job().await.unwrap();
                let keys = source.keys().await.unwrap();

                let state = Mutex::new(job_state(&keys));
                let model = Mutex::new(dummy_model(job_state(&keys)));
                let transform = newline_consumed_transform();

                let completed =
                    drive_loop_to_completion(&state, &model, &source, &transform, 1024, 10)
                        .await
                        .unwrap_or_else(|e| panic!("case '{name}' errored: {e:#}"));

                assert!(completed, "case '{name}' did not complete the empty part");
                let state = state.lock().await;
                assert_eq!(state.parts[0].current_offset, 0, "case '{name}'");
                assert_eq!(
                    state.parts[0].total_size,
                    Some(0),
                    "case '{name}' must record the discovered size so the DB state is terminal"
                );
                // The mirrored model state is what gets persisted on commit.
                let model = model.lock().await;
                assert_eq!(
                    model.state.as_ref().unwrap().parts[0].total_size,
                    Some(0),
                    "case '{name}' must mirror the size into the model"
                );
            }
        }

        /// A job can crash after committing the offset of a part's final chunk
        /// but before the next iteration discovered (and persisted) the size.
        /// On resume the re-downloaded stream EOFs exactly at the stored offset:
        /// the first read is empty with total_size still None, and the part must
        /// complete. If the re-downloaded data is instead *smaller* than the
        /// stored offset (non-byte-stable export), everything available was
        /// already consumed, so the part must also complete rather than strand
        /// the job on an unresolvable error.
        #[tokio::test]
        async fn test_resume_at_or_past_eof_with_unknown_total_completes() {
            let mut body = String::new();
            for i in 0..200 {
                body.push_str(&format!("{{\"event\":\"e{i}\"}}\n"));
            }
            let total_size = body.len() as u64;

            for (name, resume_offset) in [
                ("resume exactly at EOF", total_size),
                ("resume past a shrunk export", total_size + 1000),
            ] {
                let server = MockServer::start();
                let _mock = server.mock(|when, then| {
                    when.method(Method::GET).path("/export");
                    then.status(200).body(gzip_bytes(body.as_bytes()));
                });

                let staging = TempDir::new().unwrap();
                let source = build_source_with_extractor(
                    server.url("/export"),
                    staging.path(),
                    1,
                    ExtractorType::PlainGzip,
                );
                source.prepare_for_job().await.unwrap();
                let keys = source.keys().await.unwrap();

                let resumed = JobState {
                    parts: vec![PartState {
                        key: keys[0].clone(),
                        current_offset: resume_offset,
                        total_size: None,
                    }],
                    trial: None,
                    events_emitted: 0,
                };
                let state = Mutex::new(resumed.clone());
                let model = Mutex::new(dummy_model(resumed));
                let transform = newline_consumed_transform();

                let completed =
                    drive_loop_to_completion(&state, &model, &source, &transform, 1024, 10)
                        .await
                        .unwrap_or_else(|e| panic!("case '{name}' errored: {e:#}"));

                assert!(completed, "case '{name}' did not complete");
                let state = state.lock().await;
                assert_eq!(
                    state.parts[0].total_size,
                    Some(total_size),
                    "case '{name}' must record the size discovered on resume"
                );
                assert_eq!(
                    state.parts[0].current_offset, resume_offset,
                    "case '{name}' must not rewind or advance the stored offset"
                );
            }
        }

        /// A source that reports a fixed size but serves empty chunks - the
        /// "genuinely broken source" shape (e.g. a ranged HTTP server returning
        /// empty 200s mid-part, or a stream that shrank to a size that no longer
        /// matches the part's recorded total). These must still pause the job
        /// loudly, proving the empty-part completion path did not neuter the
        /// no-progress guard.
        struct EmptyChunkSource {
            reported_size: Option<u64>,
        }

        #[async_trait]
        impl DataSource for EmptyChunkSource {
            async fn keys(&self) -> Result<Vec<String>, Error> {
                Ok(vec!["part".to_string()])
            }

            async fn size(&self, _key: &str) -> Result<Option<u64>, Error> {
                Ok(self.reported_size)
            }

            async fn get_chunk(
                &self,
                _key: &str,
                _offset: u64,
                _size: u64,
            ) -> Result<Vec<u8>, Error> {
                Ok(Vec::new())
            }
        }

        #[tokio::test]
        async fn test_empty_chunk_before_end_of_part_still_errors() {
            // (case, part state total, part offset, source-reported size)
            let cases: [(&str, Option<u64>, u64, Option<u64>); 3] = [
                // Ranged source serving empty bodies mid-part.
                ("empty chunk below known size", Some(1000), 400, Some(1000)),
                // Stream ended early relative to what the source itself reports.
                ("source size past offset", None, 400, Some(1000)),
                // Stored total disagrees with what the source now reports.
                (
                    "shrunk source with recorded total",
                    Some(1000),
                    400,
                    Some(400),
                ),
            ];

            for (name, part_total, offset, reported_size) in cases {
                let source = EmptyChunkSource { reported_size };
                let part_state = JobState {
                    parts: vec![PartState {
                        key: "part".to_string(),
                        current_offset: offset,
                        total_size: part_total,
                    }],
                    trial: None,
                    events_emitted: 0,
                };
                let state = Mutex::new(part_state.clone());
                let model = Mutex::new(dummy_model(part_state));
                let transform = newline_consumed_transform();

                let result =
                    drive_loop_to_completion(&state, &model, &source, &transform, 1024, 5).await;

                let err = result.unwrap_err();
                assert!(
                    format!("{err:#}").contains("returned no data"),
                    "case '{name}' expected the no-data error, got: {err:#}"
                );
            }
        }
    }

    /// Remote (temp-bucket) staging must satisfy the same read-loop contracts the
    /// local streaming path is pinned to above and in `empty_part_completion_tests`:
    /// empty exports complete, trailing blank lines complete, and parts already done
    /// (including offsets overshot by a past worker bug) are skipped without ever
    /// touching the origin or the backend. All drive the real
    /// `select_and_fetch_next_chunk` loop over a temp-bucket-staged source.
    mod remote_read_loop_parity_tests {
        use super::empty_part_completion_tests::zip_without_json_members;
        use super::staging_cleanup_invariant_tests::{
            build_source_with, drive_loop_to_completion, dummy_model, gzip_bytes, job_state,
            newline_consumed_transform,
        };
        use super::*;
        use crate::extractor::ExtractorType;
        use crate::source::RemoteStaging;
        use crate::staging::{StagingBackend, TempBucketBackend};
        use object_store::memory::InMemory;
        use tempfile::TempDir;

        fn remote_staging(store: Arc<InMemory>) -> (RemoteStaging, Arc<TempBucketBackend>) {
            let backend = Arc::new(TempBucketBackend::new(store, "staging/", "job-PARITY"));
            let backend_dyn: Arc<dyn StagingBackend> = backend.clone();
            (
                RemoteStaging {
                    backend: backend_dyn,
                    extractor_type: ExtractorType::PlainGzip,
                    max_plaintext_bytes: 0,
                },
                backend,
            )
        }

        /// Remote counterpart of `test_empty_export_completes_part`: every empty
        /// export shape stages an empty object and completes the part.
        #[tokio::test]
        async fn test_remote_empty_export_completes_part() {
            let cases: [(&str, Vec<u8>, ExtractorType); 3] = [
                ("empty 200 body", Vec::new(), ExtractorType::PlainGzip),
                (
                    "gzip of empty content",
                    gzip_bytes(b""),
                    ExtractorType::PlainGzip,
                ),
                (
                    "zip with no json members",
                    zip_without_json_members(),
                    ExtractorType::ZipGzipJson,
                ),
            ];

            for (name, body, extractor) in cases {
                let server = MockServer::start();
                let _mock = server.mock(|when, then| {
                    when.method(Method::GET).path("/export");
                    then.status(200).body(body);
                });

                let staging = TempDir::new().unwrap();
                let (mut remote, backend) = remote_staging(Arc::new(InMemory::new()));
                remote.extractor_type = extractor.clone();
                let source = build_source_with(
                    server.url("/export"),
                    staging.path(),
                    1,
                    extractor,
                    Some(remote),
                );
                source.prepare_for_job().await.unwrap();
                let keys = source.keys().await.unwrap();

                let state = Mutex::new(job_state(&keys));
                let model = Mutex::new(dummy_model(job_state(&keys)));
                let transform = newline_consumed_transform();

                let completed =
                    drive_loop_to_completion(&state, &model, &source, &transform, 1024, 10)
                        .await
                        .unwrap_or_else(|e| panic!("case '{name}' errored: {e:#}"));

                assert!(completed, "case '{name}' did not complete the empty part");
                let state = state.lock().await;
                assert_eq!(state.parts[0].current_offset, 0, "case '{name}'");
                assert_eq!(state.parts[0].total_size, Some(0), "case '{name}'");
                // The empty object is durably staged: a resume attaches to it
                // instead of re-downloading from the origin.
                assert_eq!(
                    backend.size(&keys[0]).await.unwrap(),
                    Some(0),
                    "case '{name}' must stage an empty object"
                );
            }
        }

        /// Remote counterpart of `test_part_with_trailing_blank_lines_completes`:
        /// blank trailing content in the staged bytes is consumed over ranged
        /// backend reads exactly as over the local stream.
        #[tokio::test]
        async fn test_remote_part_with_trailing_blank_lines_completes() {
            let server = MockServer::start();
            let mut body = String::new();
            for i in 0..200 {
                body.push_str(&format!("{{\"event\":\"e{i}\"}}\n"));
            }
            body.push('\n');
            let total_size = body.len() as u64;
            let _mock = server.mock(|when, then| {
                when.method(Method::GET).path("/export");
                then.status(200).body(gzip_bytes(body.as_bytes()));
            });

            let staging = TempDir::new().unwrap();
            let (remote, _backend) = remote_staging(Arc::new(InMemory::new()));
            let source = build_source_with(
                server.url("/export"),
                staging.path(),
                1,
                ExtractorType::PlainGzip,
                Some(remote),
            );
            source.prepare_for_job().await.unwrap();
            let keys = source.keys().await.unwrap();

            let state = Mutex::new(job_state(&keys));
            let model = Mutex::new(dummy_model(job_state(&keys)));
            let transform = newline_consumed_transform();

            let completed = drive_loop_to_completion(
                &state,
                &model,
                &source,
                &transform,
                1024,
                (total_size as usize / 1024) + 10,
            )
            .await
            .unwrap();

            assert!(completed, "trailing blank lines must not stall the part");
            assert_eq!(state.lock().await.parts[0].current_offset, total_size);
        }

        /// Remote counterpart of `test_poisoned_part_with_overshot_offset_self_heals`:
        /// a part whose stored offset overshot its total (a past worker bug) is
        /// skipped as done without being staged or re-downloaded - `is_done` is
        /// checked before any prepare work, so the poisoned part costs nothing.
        #[tokio::test]
        async fn test_remote_poisoned_part_skipped_without_staging() {
            let server = MockServer::start();
            let mut body = String::new();
            for i in 0..200 {
                body.push_str(&format!("{{\"event\":\"e{i}\"}}\n"));
            }
            let mock = server.mock(|when, then| {
                when.method(Method::GET).path("/export");
                then.status(200).body(gzip_bytes(body.as_bytes()));
            });

            let staging = TempDir::new().unwrap();
            let (remote, backend) = remote_staging(Arc::new(InMemory::new()));
            let source = build_source_with(
                server.url("/export"),
                staging.path(),
                2,
                ExtractorType::PlainGzip,
                Some(remote),
            );
            source.prepare_for_job().await.unwrap();
            let keys = source.keys().await.unwrap();
            assert_eq!(keys.len(), 2);

            let poisoned = JobState {
                parts: vec![
                    PartState {
                        key: keys[0].clone(),
                        current_offset: 25_328_072,
                        total_size: Some(24_441_157),
                    },
                    PartState {
                        key: keys[1].clone(),
                        current_offset: 0,
                        total_size: None,
                    },
                ],
                trial: None,
                events_emitted: 0,
            };
            let state = Mutex::new(poisoned.clone());
            let model = Mutex::new(dummy_model(poisoned));
            let transform = newline_consumed_transform();

            let completed = drive_loop_to_completion(&state, &model, &source, &transform, 1024, 20)
                .await
                .unwrap();

            assert!(completed, "job with an overshot part must finish the rest");
            let state = state.lock().await;
            assert_eq!(
                state.parts[0].current_offset, 25_328_072,
                "poisoned part must be skipped untouched"
            );
            assert_eq!(state.parts[1].total_size, Some(body.len() as u64));
            assert_eq!(
                backend.size(&keys[0]).await.unwrap(),
                None,
                "poisoned part must never be staged"
            );
            assert_eq!(
                mock.hits(),
                1,
                "only the healthy part may hit the origin (one download)"
            );
        }
    }
}