Skip to content

A self-improving RLM agent for coding workflows and long-running autonomous tasks.

Rust
81.6 KB
Raw Blame
//! The per-session kernel provisioner: owns one kernel manager, guards its
//! startup, revives the saved namespace before the runtime bootstrap, and
//! disposes/kills on demand. Reader tasks hold only weak references, so no
//! kernel outlives its provisioner; `dispose()` flushes the snapshot first.

use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex};

use anyhow::anyhow;

use crate::kernel::bootstrap::{
    build_rlm_bootstrap_code, parse_unavailable_python_skills, KernelBootstrapProgressHandler,
    KernelPythonSkill, UnavailablePythonSkills,
};
use crate::kernel::cancellation::AbortSignal;
use crate::kernel::manager::{KernelStartOptions, ReplKernelManager};
use crate::kernel::shared::ExecuteStatus;
use crate::kernel::shared::{
    ExecuteOptions, HostRequestHandlers, KernelManagerOptions, KernelShutdownOptions,
    KernelSnapshotConfig, BASH_COMPLETED_REQUEST_TYPE, BOOTSTRAP_EXECUTION_TIMEOUT_MS,
};
use crate::kernel::state_snapshot::RestoreResult;
use crate::kernel::state_snapshot::{manifest_path_in, snapshot_path_in};

/// Above core count because boots are IO-bound, capped so a fan-out can't
/// thrash the FS past the ready-handshake window.
fn default_kernel_boot_concurrency() -> usize {
    let cores = std::thread::available_parallelism().map_or(4, std::num::NonZero::get);
    16.min((cores * 2).max(4))
}

fn resolve_kernel_boot_concurrency() -> usize {
    let Ok(raw) = std::env::var("PRIME_AGENT_MAX_CONCURRENT_KERNEL_BOOTS") else {
        return default_kernel_boot_concurrency();
    };
    if raw.is_empty() || !raw.chars().all(|c| c.is_ascii_digit()) {
        return default_kernel_boot_concurrency();
    }
    let parsed: usize = raw.parse().unwrap_or(0);
    if parsed < 1 {
        return default_kernel_boot_concurrency();
    }
    parsed.min(64)
}

/// Semaphore bounding concurrent kernel boots. Resolved lazily on first boot so
/// an env override set before the first kernel starts is honored.
static BOOT_PERMITS: Mutex<Option<Arc<tokio::sync::Semaphore>>> = Mutex::new(None);

async fn with_kernel_boot_permit<F, Fut>(boot: F) -> Fut::Output
where
    F: FnOnce() -> Fut,
    Fut: std::future::Future,
{
    let permits = {
        let mut guard = BOOT_PERMITS
            .lock()
            .unwrap_or_else(std::sync::PoisonError::into_inner);
        guard
            .get_or_insert_with(|| {
                Arc::new(tokio::sync::Semaphore::new(
                    resolve_kernel_boot_concurrency(),
                ))
            })
            .clone()
    };
    let _permit = permits.acquire().await;
    boot().await
}

/// Publishes the restore outcome once the kernel is usable.
pub type RestoreCallback = Arc<dyn Fn(&RestoreResult) + Send + Sync>;

/// Publishes the skills that failed to import into a freshly started
/// kernel (import name -> import error), once the kernel is usable.
pub type UnavailableSkillsCallback = Arc<dyn Fn(&UnavailablePythonSkills) + Send + Sync>;

/// Outcome of one full kernel bootstrap (spawn + handshake + namespace
/// restore + runtime bootstrap), reported once per actual boot.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum KernelBootstrapOutcome {
    /// The kernel process is running, revived, and runtime-bootstrapped.
    Ready,
    /// Any stage failed; the kernel was torn down before the error surfaced.
    Error,
}

/// One kernel boot's facts for the `kernel_bootstrap_*` session counters.
#[derive(Debug, Clone, Copy)]
pub struct KernelBootstrapStats {
    /// No prior namespace snapshot existed to restore (fresh session vs a
    /// revived one).
    pub cold: bool,
    pub outcome: KernelBootstrapOutcome,
    /// Wall time of the whole bootstrap, milliseconds.
    pub duration_ms: u64,
}

/// Reports kernel bootstrap results (the `kernel_bootstrap_*` session counters).
pub type KernelBootstrapResultHandler = Arc<dyn Fn(KernelBootstrapStats) + Send + Sync>;

#[derive(Default, Clone)]
pub struct IpythonKernelProvisionerOptions {
    /// Python override. Must have prime-agent-runtime installed.
    pub python: Option<PathBuf>,
    /// Per-cell watchdog budget for user `python_repl` cells, in milliseconds
    /// (`None`: the watchdog never arms). The daemon engine resolves it from
    /// settings and disarms it for daemonless sessions; the runtime bootstrap
    /// and internal state ops never arm it.
    pub cell_timeout_ms: Option<u64>,
    pub env: HashMap<String, String>,
    /// Command prefix prepended to every kernel `bash()` invocation.
    pub command_prefix: Option<String>,
    /// Trusted shell path injected for kernel `bash()`; `None` on platforms
    /// without one, where the runtime's teaching error fires instead.
    pub shell_path: Option<PathBuf>,
    pub session_id: Option<String>,
    pub host_handlers: HostRequestHandlers,
    pub python_skills: Vec<KernelPythonSkill>,
    /// Artifact directory of a persistent session; the revivable snapshot and
    /// the stderr log live there. `None` for ephemeral sessions.
    pub snapshot_dir: Option<PathBuf>,
    /// Await (e.g. a previous provisioner's dispose) before reading the snapshot.
    pub ready_gate: Option<Arc<dyn Fn() -> futures::future::BoxFuture<'static, ()> + Send + Sync>>,
    /// Publishes the restore outcome once the kernel is usable.
    pub on_restore: Option<RestoreCallback>,
    /// Fires when the kernel's last live background `bash()` handle
    /// settles, so owed continuations can resume.
    pub on_background_work_settled: Option<crate::kernel::shared::BackgroundWorkSettledCallback>,
    /// Fires once per kernel start when installed Python skills failed to import into the kernel,
    /// so the session can tell the model before it wastes turns calling them.
    pub on_unavailable_skills: Option<UnavailableSkillsCallback>,
    /// Publishes the per-boot result for the `kernel_bootstrap_*` counters.
    /// Telemetry only; kernel behavior never depends on it.
    pub on_bootstrap_result: Option<KernelBootstrapResultHandler>,
}

/// Why and how long one startup failed, published through the shared startup
/// watch so every joined `ensure()` caller sees the same cause and duration
/// instead of a bare "kernel startup failed".
#[derive(Clone)]
struct StartupFailure {
    /// Full error chain (`{:#}` formatting).
    message: String,
    duration_ms: u64,
}

type StartupResult = Result<ReplKernelManager, StartupFailure>;

/// How one boot's settle publishes: the normal park, one of the two
/// teardown rejects (a racing dispose; a memo generation invalidated by
/// `kill()`), or the boot's own failure.
enum Settle {
    /// The boot's manager parked into the provisioner.
    Published(ReplKernelManager),
    /// A dispose raced the boot: tear the kernel down with the dispose's
    /// snapshot policy and reject with its cause.
    TearDownForDispose {
        manager: ReplKernelManager,
        snapshot: bool,
        duration_ms: u64,
    },
    /// The memo this boot armed was cleared underneath it (`kill()`, or a
    /// defunct clear superseding the generation): never publish a kernel
    /// into a memo generation the boot no longer owns - kill it instead.
    TearDownForKill {
        manager: ReplKernelManager,
        duration_ms: u64,
    },
    /// The boot failed on its own; its failure already carries the cause
    /// and duration.
    Failed(StartupFailure),
}

/// One `stop_kernel` call's starting point.
enum StopArm {
    /// Neither a live kernel nor an in-flight boot: nothing to stop.
    Nothing,
    /// This stop's own gate, installed atomically with the manager take
    /// (direct) or the in-flight boot it joins.
    Armed {
        manager: Option<ReplKernelManager>,
        startup: Option<tokio::sync::watch::Receiver<Option<StartupResult>>>,
        stop_tx: tokio::sync::watch::Sender<bool>,
        /// The gate this arm superseded. The stop task waits it before
        /// opening its own, so a revival gated on this stop's gate can
        /// never cross before the superseded stop's final snapshot flush
        /// settles; and because every task waits only a gate OLDER than
        /// its own arm, the waits cannot form a cycle (the
        /// wait-the-installed-gate shape admitted both the missed
        /// supersede window and a three-task deadlock).
        previous_stop: Option<tokio::sync::watch::Receiver<bool>>,
    },
}

struct ProvisionerState {
    manager: Option<ReplKernelManager>,
    /// Shared result of the in-flight boot. Joined callers retain its manager
    /// even when a concurrent stop takes the provisioner's live owner.
    startup: Option<tokio::sync::watch::Receiver<Option<StartupResult>>>,
    startup_listeners: Vec<KernelBootstrapProgressHandler>,
    last_startup_message: Option<String>,
    last_restore: Option<RestoreResult>,
    disposed: bool,
    /// Snapshot policy of the dispose that aborted a startup, honored by
    /// the failed startup's own teardown.
    dispose_snapshot: bool,
    /// The in-flight `stop_kernel` shutdown (TS #2483's `pendingStop`): a
    /// revival boot waits for it to finish flushing its final snapshot
    /// before reading that snapshot back, so the two kernels never race
    /// over the same on-disk file. The receiver yields `true` once the
    /// recorded stop settles (a settled stop awaits instantly; a dead
    /// sender errs and unblocks the same way); each stop supersedes the
    /// previous.
    pending_stop: Option<tokio::sync::watch::Receiver<bool>>,
    /// The startup memo the installed pending-stop gate is armed against;
    /// `None` when the gate was armed with a directly-taken manager. The
    /// failed-boot teardown consults it to keep the gate of a stop
    /// ACTIVELY armed for its own boot (that stop's task waits the boot's
    /// memo, which settles only after the teardown): the teardown's flush
    /// is already covered, so it installs no replacement gate.
    pending_stop_for_startup: Option<tokio::sync::watch::Receiver<Option<StartupResult>>>,
    /// Startup memos `kill()` invalidated but whose boots have not settled
    /// yet. The memo's generation is dead (the boot tears down with `kill()`
    /// semantics) yet the boot's kernel still exists, so a later
    /// `dispose()` must wait its settle the same way it waits an armed
    /// memo — without the park, a kill followed by a dispose skips the
    /// doomed boot entirely and can orphan its kernel at worker exit.
    /// Each parked memo is spent by its own boot's settle.
    doomed_startups: Vec<tokio::sync::watch::Receiver<Option<StartupResult>>>,
}

/// Owns one kernel for one session: starts it, memoizes the startup so concurrent callers join the
/// same boot.
///
/// Cloning shares the same kernel and startup state.
#[derive(Clone)]
pub struct IpythonKernelProvisioner {
    inner: Arc<ProvisionerInner>,
}

struct ProvisionerInner {
    cwd: PathBuf,
    options: IpythonKernelProvisionerOptions,
    state: Mutex<ProvisionerState>,
    dispose_signal: AbortSignal,
}

impl IpythonKernelProvisioner {
    pub fn new(cwd: impl Into<PathBuf>, options: IpythonKernelProvisionerOptions) -> Self {
        Self {
            inner: Arc::new(ProvisionerInner {
                cwd: cwd.into(),
                options,
                state: Mutex::new(ProvisionerState {
                    manager: None,
                    startup: None,
                    startup_listeners: Vec::new(),
                    last_startup_message: None,
                    last_restore: None,
                    disposed: false,
                    dispose_snapshot: true,
                    pending_stop: None,
                    pending_stop_for_startup: None,
                    doomed_startups: Vec::new(),
                }),
                dispose_signal: AbortSignal::new(),
            }),
        }
    }

    fn lock_state(&self) -> std::sync::MutexGuard<'_, ProvisionerState> {
        self.inner
            .state
            .lock()
            .unwrap_or_else(std::sync::PoisonError::into_inner)
    }

    /// The kernel manager, once a startup has completed successfully.
    #[must_use]
    pub fn manager(&self) -> Option<ReplKernelManager> {
        self.lock_state().manager.clone()
    }

    /// The per-cell watchdog budget the kernel executor arms for user cells
    /// (`None`: the watchdog never arms).
    #[must_use]
    pub fn cell_timeout_ms(&self) -> Option<u64> {
        self.inner.options.cell_timeout_ms
    }

    /// The Python skills pre-imported at kernel bootstrap.
    #[cfg(test)]
    #[must_use]
    pub(crate) fn python_skills(&self) -> &[KernelPythonSkill] {
        &self.inner.options.python_skills
    }

    /// Result of reviving a prior session's namespace on the last kernel start.
    #[must_use]
    pub fn last_restore(&self) -> Option<RestoreResult> {
        self.lock_state().last_restore.clone()
    }

    /// Whether a kernel has finished starting and is currently running.
    #[must_use]
    pub fn has_running_kernel(&self) -> bool {
        self.manager().is_some_and(|m| m.is_running())
    }

    /// Start the kernel in the background. Failures are swallowed here and
    /// surface on the next `ensure()`.
    pub fn prewarm(&self) {
        let provisioner = self.clone();
        tokio::spawn(async move {
            let _ = provisioner.ensure(None, None).await;
        });
    }

    /// The kernel manager, starting it first when necessary. Concurrent callers join one startup.
    ///
    /// # Errors
    ///
    /// Returns an error when the abort signal is already cancelled or fires
    /// during the wait, when the provisioner was disposed, or when the kernel
    /// startup fails (all joined callers see the same failure).
    ///
    /// # Panics
    ///
    /// Panics if the shared startup watch settles without publishing a
    /// result; the publisher always sends the result before clearing the
    /// memo, so this is a construction invariant rather than a runtime
    /// path.
    pub async fn ensure(
        &self,
        on_progress: Option<KernelBootstrapProgressHandler>,
        signal: Option<AbortSignal>,
    ) -> anyhow::Result<ReplKernelManager> {
        if let Some(signal) = &signal {
            if signal.is_aborted() {
                return Err(anyhow!("Python execution aborted"));
            }
        }
        // The guard is strictly scoped to this decision block: a
        // conditionally-dropped non-Send MutexGuard would make ensure()
        // non-Send.
        let mut startup = {
            let mut state = self.lock_state();
            if state.disposed {
                return Err(anyhow!("Kernel provisioner disposed"));
            }
            if let Some(manager) = &state.manager {
                if manager.is_defunct() {
                    state.manager = None;
                    state.startup = None;
                }
            }
            if let Some(manager) = state.manager.clone() {
                return Ok(manager);
            }
            if let Some(progress) = &on_progress {
                if let Some(message) = state.last_startup_message.as_deref() {
                    progress(message);
                }
                state.startup_listeners.push(progress.clone());
            }
            if let Some(startup) = &state.startup {
                startup.clone()
            } else {
                let (done_tx, done_rx) = tokio::sync::watch::channel(None);
                let inner = Arc::clone(&self.inner);
                let pending_stop = state.pending_stop.clone();
                let startup_progress = on_progress.clone();
                let startup_memo = done_rx.clone();
                tokio::spawn(async move {
                    let boot = tokio::spawn(run_startup(
                        inner.clone(),
                        startup_progress,
                        pending_stop,
                        startup_memo.clone(),
                    ));
                    let result = match boot.await {
                        Ok(result) => result,
                        Err(error) => {
                            // A panicked boot never reached its own settle;
                            // the settle below still runs so the listener
                            // state clears and the next ensure() boots fresh.
                            Err(StartupFailure {
                                message: format!("kernel startup task failed: {error}"),
                                duration_ms: 0,
                            })
                        }
                    };
                    // Atomic settle/publish: the disposed check, the
                    // memo-generation check, the listener teardown, and the
                    // manager publication share ONE lock scope. Separate
                    // scopes let a dispose or a kill land between the check
                    // and the publish - the boot would park a live kernel
                    // into a provisioner that already reported itself torn
                    // down.
                    let settle = {
                        let mut state = inner
                            .state
                            .lock()
                            .unwrap_or_else(std::sync::PoisonError::into_inner);
                        let mine = state
                            .startup
                            .as_ref()
                            .is_some_and(|memo| memo.same_channel(&startup_memo));
                        // The listener state belongs to the ACTIVE memo
                        // generation: a doomed boot settling against a
                        // newer memo must not wipe the newer boot's
                        // progress listeners or its replayed stage. With
                        // no memo armed at all the entries are stale
                        // (their waiters already hold results) - clear.
                        if mine || state.startup.is_none() {
                            state.startup_listeners.clear();
                            state.last_startup_message = None;
                        }
                        match result {
                            Ok((manager, duration_ms)) => {
                                park_or_tear_down(&mut state, mine, manager, duration_ms)
                            }
                            Err(failure) => Settle::Failed(failure),
                        }
                    };
                    let settled = match settle {
                        Settle::Published(manager) => Ok(manager),
                        Settle::TearDownForDispose {
                            manager,
                            snapshot,
                            duration_ms,
                        } => {
                            // A dispose raced the boot: tear the kernel down
                            // with the dispose's snapshot policy and reject
                            // with its abort error instead of handing joined
                            // callers a manager that is already dead.
                            let _ = manager
                                .shutdown(KernelShutdownOptions {
                                    snapshot,
                                    drain_host_requests: true,
                                })
                                .await;
                            Err(StartupFailure {
                                message: "Kernel provisioner disposed during startup".to_string(),
                                duration_ms,
                            })
                        }
                        Settle::TearDownForKill {
                            manager,
                            duration_ms,
                        } => {
                            // TS kill() settles the pending startup and kills
                            // the manager it produced.
                            if !manager.is_defunct() {
                                manager.kill();
                            }
                            Err(StartupFailure {
                                message: "Kernel provisioner killed during startup".to_string(),
                                duration_ms,
                            })
                        }
                        Settle::Failed(failure) => Err(failure),
                    };
                    // Publish the completed result before clearing the memo:
                    // joined waiters keep the manager even if stop takes it.
                    let _ = done_tx.send(Some(settled));
                    // Clear only the memo this task installed (TS ensure()'s
                    // `managerPromise === startup` guard): a boot's manager
                    // can turn defunct inside the park->publish->clear window
                    // and another ensure() defunct-clears it and arms a NEWER
                    // memo already; an unconditional clear would wipe that
                    // memo mid-boot and arm a transient duplicate kernel.
                    let mut state = inner
                        .state
                        .lock()
                        .unwrap_or_else(std::sync::PoisonError::into_inner);
                    if state
                        .startup
                        .as_ref()
                        .is_some_and(|memo| memo.same_channel(&startup_memo))
                    {
                        state.startup = None;
                    }
                    release_spent_stop_arm(&mut state, &startup_memo);
                    // A memo kill() parked for this boot is spent with the
                    // settle the dispose() parker waits on.
                    state
                        .doomed_startups
                        .retain(|parked| !parked.same_channel(&startup_memo));
                });
                state.startup = Some(done_rx.clone());
                done_rx
            }
        };
        // Aborting one waiter leaves the shared startup alive for other
        // callers, like TS raceWithAbort(managerPromise, signal).
        let wait = async { startup.wait_for(Option::is_some).await };
        let completion = match &signal {
            Some(signal) => tokio::select! {
                result = wait => result,
                () = signal.cancelled() => return Err(anyhow!("Python execution aborted")),
            },
            None => wait.await,
        };
        if let Some(signal) = &signal {
            if signal.is_aborted() {
                return Err(anyhow!("Python execution aborted"));
            }
        }
        completion.map_err(|error| anyhow!("kernel startup task closed: {error}"))?;
        let settled = startup
            .borrow()
            .clone()
            .expect("settled startup has a result");
        match settled {
            Ok(manager) => Ok(manager),
            Err(failure) => Err(anyhow!(
                "kernel startup failed after {}ms: {}",
                failure.duration_ms,
                failure.message
            )),
        }
    }

    /// Remove live variables above the snapshot's per-variable size limit.
    pub async fn prune_oversized_variables(&self) -> Option<Vec<String>> {
        let manager = self.manager()?;
        manager
            .prune_oversized_variables()
            .await
            .and_then(|r| r.pruned)
    }

    /// Live user-defined names in the kernel namespace, or `None` if listing
    /// failed or no kernel is running.
    pub async fn list_namespace_names(&self, signal: Option<AbortSignal>) -> Option<Vec<String>> {
        let manager = self.manager()?;
        manager.list_namespace_names(signal).await
    }

    /// Stop the owned kernel without marking the provisioner disposed (TS
    /// #2483's `stopKernel`): the shutdown flushes the final snapshot, and
    /// the next `ensure()` boots a fresh kernel gated on this stop (the
    /// pending-stop revival gate), so a follow-up turn revives from the
    /// flushed snapshot instead of racing the flush over the same on-disk
    /// file. A kernel still starting up is joined first and shut down the
    /// same way (the TS `managerPromise` arm - the boot is not left
    /// resident); with neither a live kernel nor an in-flight boot there
    /// is nothing to stop.
    ///
    /// Concurrent stops of one in-flight boot chain instead of racing: each
    /// stop arms its own gate superseding the previous, and every stop task
    /// waits the strictly older gate it superseded before opening its own,
    /// so a revival gated on any of them cannot cross before the final
    /// snapshot flush settles.
    ///
    /// Best-effort by construction: a failed shutdown leaves no manager
    /// and the next `ensure()` boots fresh.
    pub async fn stop_kernel(&self, options: Option<KernelShutdownOptions>) {
        let snapshot = options.is_none_or(|o| o.snapshot);
        match self.arm_stop(snapshot) {
            StopArm::Nothing => {}
            StopArm::Armed {
                manager,
                startup,
                stop_tx,
                previous_stop,
                ..
            } => {
                let inner = Arc::clone(&self.inner);
                let stop = tokio::spawn(async move {
                    let manager = if let Some(manager) = manager {
                        Some(manager)
                    } else {
                        if let Some(mut startup) = startup {
                            let _ = startup.wait_for(Option::is_some).await;
                        }
                        inner
                            .state
                            .lock()
                            .unwrap_or_else(std::sync::PoisonError::into_inner)
                            .manager
                            .take()
                    };
                    if let Some(manager) = manager {
                        let _ = manager
                            .shutdown(KernelShutdownOptions {
                                snapshot,
                                drain_host_requests: true,
                            })
                            .await;
                    }
                    // Chained gate (the #3257 shape, kept under the join):
                    // open ours only after the gate THIS arm superseded
                    // settles - never the currently-installed one. A
                    // revival gated on ours then cannot cross before the
                    // superseded stop's final snapshot flush, whether this
                    // task shut its own kernel down or lost the manager
                    // take to the stop that gate belongs to; and because
                    // each task waits only a strictly older gate, the
                    // waits cannot form a cycle.
                    if let Some(mut previous) = previous_stop {
                        let _ = previous.wait_for(|done| *done).await;
                    }
                    let _ = stop_tx.send(true);
                });
                let _ = stop.await;
            }
        }
    }

    /// Claim one stop: take a live manager directly, or arm this stop's
    /// own gate for the in-flight boot's parked kernel. The gate is
    /// installed under the same lock `ensure()` uses to select its
    /// startup, so no revival can miss this stop; a second stop of the
    /// same boot simply supersedes the first's gate, and the stops chain
    /// through their superseded gates.
    fn arm_stop(&self, snapshot: bool) -> StopArm {
        let mut state = self.lock_state();
        state.dispose_snapshot = snapshot;
        let manager = state.manager.take();
        let startup = state.startup.clone();
        if manager.is_none() && startup.is_none() {
            return StopArm::Nothing;
        }
        let (stop_tx, stop_rx) = tokio::sync::watch::channel(false);
        let previous_stop = state.pending_stop.replace(stop_rx);
        // A gate armed with a directly-taken manager guards that manager's
        // shutdown, not a boot; the failed-boot teardown consults this
        // record to leave a stop ACTIVELY armed for the tearing-down boot
        // alone (its own task already covers the flush).
        state.pending_stop_for_startup = if manager.is_some() {
            None
        } else {
            startup.clone()
        };
        StopArm::Armed {
            manager,
            startup,
            stop_tx,
            previous_stop,
        }
    }

    pub async fn dispose(&self, options: Option<KernelShutdownOptions>) {
        let snapshot = options.is_none_or(|o| o.snapshot);
        let (mut startup, mut doomed) = {
            let mut state = self.lock_state();
            state.dispose_snapshot = snapshot;
            state.disposed = true;
            (state.startup.clone(), state.doomed_startups.clone())
        };
        self.inner.dispose_signal.abort();
        // A boot `kill()` invalidated settles like an armed one: the
        // dispose must not return while its kernel is still settling, or a
        // worker exit could orphan it.
        if let Some(startup) = &mut startup {
            let _ = startup.wait_for(Option::is_some).await;
        }
        for memo in &mut doomed {
            let _ = memo.wait_for(Option::is_some).await;
        }
        let manager = self.lock_state().manager.take();
        if let Some(manager) = manager {
            let _ = manager
                .shutdown(KernelShutdownOptions {
                    snapshot,
                    drain_host_requests: true,
                })
                .await;
        }
    }

    /// Kill the owned kernel without a final snapshot (busy-kernel restart).
    /// A kernel still starting up has not been published yet: like TS
    /// `kill()`, the startup memo leaves the generation (invalidation), so
    /// the doomed boot's own settle kills its kernel instead of parking
    /// it, and the next `ensure()` boots fresh - a kill racing a boot
    /// leaves no resident kernel behind. The invalidated memo is PARKED
    /// for `dispose()`: the doomed boot's kernel still exists until its
    /// settle, and a dispose racing the kill must wait it out.
    pub fn kill(&self) {
        let manager = {
            let mut state = self.lock_state();
            // The stop gate armed against the boot this kill invalidates
            // is spent with it.
            if let Some(armed_for) = state.startup.clone() {
                release_spent_stop_arm(&mut state, &armed_for);
            }
            if let Some(doomed) = state.startup.take() {
                state.doomed_startups.push(doomed);
            }
            // The shared progress state belongs to the memo generation
            // kill() just invalidated: its own emits already skip shared
            // writes, and the next ensure() must neither replay the
            // killed boot's stale stage to a fresh handler nor fan the
            // newer boot's stages out to the killed generation's
            // listeners.
            state.startup_listeners.clear();
            state.last_startup_message = None;
            state.manager.take()
        };
        if let Some(manager) = manager {
            manager.kill();
        }
    }
}

/// Release the stop gate armed for `memo`'s boot, if any. The arm's
/// memo receiver is a strong handle to the settled result's manager, and
/// a stale one keeps a parked-or-failed kernel alive past `stop_kernel()`,
/// so a later failed shutdown's process would leak with it as the sole
/// owner. Both spenders call this: the boot's settle and the `kill()` that
/// invalidates the boot.
fn release_spent_stop_arm(
    state: &mut ProvisionerState,
    memo: &tokio::sync::watch::Receiver<Option<StartupResult>>,
) {
    if state
        .pending_stop_for_startup
        .as_ref()
        .is_some_and(|armed| armed.same_channel(memo))
    {
        state.pending_stop_for_startup = None;
    }
}

/// The settled boot's disposition: apply the settle decision under the
/// settle's own lock scope.
fn park_or_tear_down(
    state: &mut ProvisionerState,
    mine: bool,
    manager: ReplKernelManager,
    duration_ms: u64,
) -> Settle {
    match settle_decision(state, mine) {
        SettleDecision::Kill => Settle::TearDownForKill {
            manager,
            duration_ms,
        },
        SettleDecision::Dispose => Settle::TearDownForDispose {
            manager,
            snapshot: state.dispose_snapshot,
            duration_ms,
        },
        SettleDecision::Publish => {
            state.manager = Some(manager.clone());
            Settle::Published(manager)
        }
    }
}

/// The settle's disposition order, isolated because the ORDER is the
/// contract: the generation check comes FIRST — a boot `kill()` doomed
/// tears down with `kill()` semantics (no flush) even when a dispose raced
/// in behind the kill, because the dispose's own snapshot policy cannot
/// resurrect a generation `kill()` invalidated (a killed+disposed boot
/// reaching Ok must never flush, and kill→ensure→dispose must never
/// overlap two flushes on one `snapshot_dir`). Only a live generation
/// honors the dispose.
fn settle_decision(state: &ProvisionerState, mine: bool) -> SettleDecision {
    if !mine {
        SettleDecision::Kill
    } else if state.disposed {
        SettleDecision::Dispose
    } else {
        SettleDecision::Publish
    }
}

#[derive(Clone, Copy, PartialEq, Eq, Debug)]
enum SettleDecision {
    Kill,
    Dispose,
    Publish,
}

/// The snapshot policy for a failed boot's teardown: the provisioner's
/// dispose policy while the boot's memo generation is still armed, but
/// NEVER a flush for a boot `kill()` invalidated — the replacement
/// generation is already restoring from the same on-disk payload, and a
/// failed doomed boot's teardown flushing over it would rewrite a healthy
/// snapshot (and the kernel stderr log) out from under the new kernel.
/// `dispose()` does not clear the memo, so a disposed boot keeps the
/// dispose's own policy.
///
/// When the policy still flushes, this ALSO installs a pending-stop gate
/// for the teardown's own flush — the decision and the gate share ONE
/// lock scope, so a `kill()` landing between the decision and the flush
/// still leaves the replacement boot gated on this teardown: it restores
/// only after the flush settles (the same revival gate `stop_kernel`
/// arms against its own flush). A stop actively armed for this boot
/// already covers the flush (its task waits this boot's memo, which
/// settles only after the teardown), so the gate stays untouched in
/// that one case; any older, merely installed gate - including one long
/// settled and left in place - covers nothing and is replaced. Hold the
/// returned sender across the shutdown — the gate opens when the
/// teardown scope drops it (a dead sender unblocks the waiters the same
/// way a settled one does).
#[must_use]
fn hold_snapshot_flush_gate(
    inner: &Arc<ProvisionerInner>,
    memo: &tokio::sync::watch::Receiver<Option<StartupResult>>,
) -> (bool, Option<tokio::sync::watch::Sender<bool>>) {
    let mut state = inner
        .state
        .lock()
        .unwrap_or_else(std::sync::PoisonError::into_inner);
    let flush = state
        .startup
        .as_ref()
        .is_some_and(|armed| armed.same_channel(memo))
        && state.dispose_snapshot;
    // A stop actively armed for THIS boot covers the teardown's flush
    // through its own memo wait (the stop's task waits this boot's
    // settle, which happens only after the teardown); a merely
    // installed older gate - including one long settled and left in
    // place - covers nothing, so this teardown's gate replaces it.
    let covered_by_active_stop = state
        .pending_stop_for_startup
        .as_ref()
        .is_some_and(|armed_for| armed_for.same_channel(memo));
    if !flush || covered_by_active_stop {
        return (flush, None);
    }
    let (gate_tx, gate_rx) = tokio::sync::watch::channel(false);
    state.pending_stop = Some(gate_rx);
    (flush, Some(gate_tx))
}

/// The boot's memo is still the armed startup generation AND the
/// provisioner is not disposed: a failed attempt may retry, and a
/// restore may surface. `kill()` (or a newer boot's defunct-clear)
/// invalidating the memo makes a further attempt stale work whose
/// kernel the settle would only have to kill.
fn boot_generation_is_live(
    inner: &Arc<ProvisionerInner>,
    memo: &tokio::sync::watch::Receiver<Option<StartupResult>>,
) -> bool {
    let state = inner
        .state
        .lock()
        .unwrap_or_else(std::sync::PoisonError::into_inner);
    !state.disposed
        && state
            .startup
            .as_ref()
            .is_some_and(|armed| armed.same_channel(memo))
}

fn emit_startup_progress(
    inner: &Arc<ProvisionerInner>,
    on_progress: Option<&KernelBootstrapProgressHandler>,
    memo: &tokio::sync::watch::Receiver<Option<StartupResult>>,
    message: &str,
) {
    let mut state = inner
        .state
        .lock()
        .unwrap_or_else(std::sync::PoisonError::into_inner);
    // The shared stage belongs to the ACTIVE memo generation: a boot
    // kill() invalidated must not overwrite the newer boot's replayed
    // stage nor fire the newer generation's listeners with its own
    // (the boot's own handler below is its caller's, not shared
    // state, so it keeps firing).
    if state
        .startup
        .as_ref()
        .is_some_and(|armed| armed.same_channel(memo))
    {
        state.last_startup_message = Some(message.to_string());
        for listener in &state.startup_listeners {
            listener(message);
        }
    }
    if let Some(on_progress) = on_progress {
        on_progress(message);
    }
}

/// Extra startup attempts beyond the first. The promise here is resilience against a wedged boot,
/// not masking a broken setup.
const DEFAULT_STARTUP_RETRIES: u32 = 1;
const DEFAULT_STARTUP_BUDGET_MS: u64 = 90_000;
const RETRY_BACKOFF_MS: [u64; 4] = [250, 1_000, 2_500, 5_000];

fn resolve_startup_retries() -> u32 {
    match std::env::var("PRIME_AGENT_KERNEL_STARTUP_RETRIES") {
        Ok(raw) => raw
            .trim()
            .parse::<u32>()
            .map_or(DEFAULT_STARTUP_RETRIES, |n| n.min(5)),
        Err(_) => DEFAULT_STARTUP_RETRIES,
    }
}

fn resolve_startup_budget_ms() -> u64 {
    match std::env::var("PRIME_AGENT_KERNEL_STARTUP_BUDGET_MS") {
        Ok(raw) => raw
            .trim()
            .parse::<u64>()
            .ok()
            .filter(|n| *n > 0)
            .unwrap_or(DEFAULT_STARTUP_BUDGET_MS),
        Err(_) => DEFAULT_STARTUP_BUDGET_MS,
    }
}

/// A failed boot the provisioner may retry on its own: transient spawn or
/// ready-handshake problems. Structural failures never auto-retry.
fn startup_failure_is_retryable(error: &anyhow::Error) -> bool {
    const FATAL_MARKERS: [&str; 9] = [
        "provisioner disposed",
        "provisioner killed",
        "aborted",
        "Failed to set up the Python kernel runtime",
        "PRIME_AGENT_KERNEL_PYTHON points to a Python",
        "Failed to initialize rlm runtime",
        "Update prime-agent-runtime in the kernel Python",
        "Kernel start superseded",
        "Kernel was disposed during startup",
    ];
    let chain = format!("{error:#}");
    !FATAL_MARKERS.iter().any(|marker| chain.contains(marker))
}

/// Boot the kernel, retrying transient failures with backoff until the
/// retry allowance or the hard startup budget runs out. Returns the booted
/// manager with the boot's duration, or the failure; the publisher task's
/// atomic settle decides whether the manager may park.
async fn run_startup(
    inner: Arc<ProvisionerInner>,
    on_progress: Option<KernelBootstrapProgressHandler>,
    pending_stop: Option<tokio::sync::watch::Receiver<bool>>,
    memo: tokio::sync::watch::Receiver<Option<StartupResult>>,
) -> Result<(ReplKernelManager, u64), StartupFailure> {
    let started = std::time::Instant::now();
    let budget = std::time::Duration::from_millis(resolve_startup_budget_ms());
    let mut remaining_retries = resolve_startup_retries();
    let mut attempt: u32 = 0;
    let outcome = loop {
        attempt += 1;
        match start_kernel(&inner, on_progress.as_ref(), pending_stop.clone(), &memo).await {
            Ok(manager) => break Ok(manager),
            Err(error) => {
                // A boot whose memo generation kill() (or a newer boot's
                // defunct-clear) invalidated must not retry: each attempt
                // spawns an interpreter and runs restore/bootstrap, and a
                // stale attempt's restore would surface notifications and
                // state a fresher generation never asked for.
                if !boot_generation_is_live(&inner, &memo)
                    || !startup_failure_is_retryable(&error)
                    || remaining_retries == 0
                    || started.elapsed() >= budget
                {
                    break Err(error);
                }
                remaining_retries -= 1;
                let backoff_ms =
                    RETRY_BACKOFF_MS[(attempt as usize - 1).min(RETRY_BACKOFF_MS.len() - 1)];
                emit_startup_progress(
                    &inner,
                    on_progress.as_ref(),
                    &memo,
                    &format!("Kernel start failed; retrying in {backoff_ms}ms…"),
                );
                tokio::select! {
                    () = tokio::time::sleep(std::time::Duration::from_millis(backoff_ms)) => {}
                    () = inner.dispose_signal.cancelled() => break Err(error),
                }
                // Re-check after the backoff: kill() can invalidate the
                // generation while this task sleeps it out.
                if !boot_generation_is_live(&inner, &memo) {
                    break Err(error);
                }
            }
        }
    };
    let duration_ms = started.elapsed().as_millis() as u64;
    // The settle (disposed check, memo-generation check, park-or-reject)
    // runs in the publisher task's atomic scope; this task only reports
    // the boot's own outcome and duration.
    match outcome {
        Ok(manager) => Ok((manager, duration_ms)),
        Err(error) => Err(StartupFailure {
            message: format!("{error:#}"),
            duration_ms,
        }),
    }
}

/// Boot one kernel, restore the prior namespace, then run the runtime bootstrap. Reports the result
/// through `on_bootstrap_result` once per actual boot.
async fn start_kernel(
    inner: &Arc<ProvisionerInner>,
    on_progress: Option<&KernelBootstrapProgressHandler>,
    pending_stop: Option<tokio::sync::watch::Receiver<bool>>,
    memo: &tokio::sync::watch::Receiver<Option<StartupResult>>,
) -> anyhow::Result<ReplKernelManager> {
    let started = std::time::Instant::now();
    let cold = !inner
        .options
        .snapshot_dir
        .as_ref()
        .is_some_and(|dir| snapshot_path_in(dir).exists());
    let result = start_kernel_impl(inner, on_progress, pending_stop, memo).await;
    if let Some(report) = &inner.options.on_bootstrap_result {
        report(KernelBootstrapStats {
            cold,
            duration_ms: started.elapsed().as_millis() as u64,
            outcome: match &result {
                Ok(_) => KernelBootstrapOutcome::Ready,
                Err(_) => KernelBootstrapOutcome::Error,
            },
        });
    }
    result
}

/// The bootstrap itself; see [`start_kernel`].
async fn start_kernel_impl(
    inner: &Arc<ProvisionerInner>,
    on_progress: Option<&KernelBootstrapProgressHandler>,
    mut pending_stop: Option<tokio::sync::watch::Receiver<bool>>,
    memo: &tokio::sync::watch::Receiver<Option<StartupResult>>,
) -> anyhow::Result<ReplKernelManager> {
    let options = &inner.options;
    let cwd = inner.cwd.clone();
    let dispose_signal = inner.dispose_signal.clone();
    // The boot-permit closure moves its own clone; the bootstrap below runs
    // on the same shared signal.
    let permit_dispose_signal = dispose_signal.clone();
    // Wait for this provisioner's own in-flight stop_kernel() — and its
    // final snapshot flush — before reading that snapshot back (TS #2483's
    // `pendingStop` gate; a completed stop awaits instantly and each stop
    // supersedes the previous). `ready_gate` stays the cross-provisioner
    // /reload arm.
    if let Some(stop_gate) = &mut pending_stop {
        if !*stop_gate.borrow() {
            emit_startup_progress(
                inner,
                on_progress,
                memo,
                "Waiting for the previous kernel to stop...",
            );
        }
        let _ = stop_gate.wait_for(|done| *done).await;
    }
    // Wait for a previous provisioner (e.g. on /reload) to finish disposing —
    // and flushing its final snapshot — before reading that snapshot back.
    if let Some(gate) = options.ready_gate.clone() {
        gate().await;
    }
    let snapshot_dir = options.snapshot_dir.clone();
    // The bounded-await default keys on the bash.completed host handler:
    // one source of truth (the registry the daemon registers the notice in).
    let bootstrap_code = build_rlm_bootstrap_code(
        &options.python_skills,
        options
            .host_handlers
            .get(BASH_COMPLETED_REQUEST_TYPE)
            .is_some(),
    );
    let mut env = options.env.clone();
    if let Some(shell_path) = &options.shell_path {
        env.insert(
            "PRIME_AGENT_BASH_SHELL".into(),
            shell_path.to_string_lossy().to_string(),
        );
    }
    if let Some(command_prefix) = &options.command_prefix {
        env.insert(
            "PRIME_AGENT_BASH_COMMAND_PREFIX".into(),
            command_prefix.clone(),
        );
    }
    let snapshot = snapshot_dir.as_ref().map(|dir| KernelSnapshotConfig {
        path: snapshot_path_in(dir),
        manifest_path: manifest_path_in(dir),
        max_bytes: None,
        max_variable_bytes: None,
        debounce_ms: None,
    });
    let stderr_log_path = snapshot_dir
        .as_ref()
        .map(|dir| dir.join("kernel-stderr.log"));
    let manager = ReplKernelManager::new(KernelManagerOptions {
        python: options.python.clone(),
        cwd: Some(cwd),
        env,
        session_id: options.session_id.clone(),
        host_handlers: options.host_handlers.clone(),
        python_skills: options.python_skills.clone(),
        on_background_work_settled: options.on_background_work_settled.clone(),
        snapshot,
        bootstrap_code: Some(bootstrap_code.clone()),
        stderr_log_path,
    });

    emit_startup_progress(inner, on_progress, memo, "Starting Python kernel...");
    // Only the process spawn + ready handshake contends for OS resources under
    // a fan-out, and it is bounded by start()'s own timeout — so the permit
    // covers only start(). Restore/bootstrap run per-kernel afterwards.
    let start = manager.start(KernelStartOptions {
        signal: None,
        on_bootstrap_progress: on_progress.cloned(),
    });
    let permit_inner = Arc::clone(inner);
    let permit_memo = memo.clone();
    let boot = async {
        with_kernel_boot_permit(move || async move {
            // Disposed, or its memo generation kill() invalidated, while
            // queued for the permit — don't spawn a kernel nobody wants.
            // The permit frees after the stop/ready gates below, so a boot
            // that waited those gates rechecks liveness here, the last
            // point before the interpreter spawn.
            if permit_dispose_signal.is_aborted() {
                return Err(anyhow!("Kernel provisioner disposed before start"));
            }
            if !boot_generation_is_live(&permit_inner, &permit_memo) {
                return Err(anyhow!("Kernel provisioner killed before start"));
            }
            start.await
        })
        .await
    };
    if let Err(error) = boot.await {
        // Never leak the kernel process if startup fails after spawn — and
        // never surface the failure before the teardown (final snapshot flush
        // included) finished.
        let (snapshot_policy, _teardown_gate) = hold_snapshot_flush_gate(inner, memo);
        let _ = manager
            .shutdown(KernelShutdownOptions {
                snapshot: snapshot_policy,
                drain_host_requests: true,
            })
            .await;
        // The drained stderr tail is the only extra evidence a failed boot
        // leaves; attach it to the cause.
        let stderr_tail = {
            let tail = manager.kernel_stderr();
            let chars: Vec<char> = tail.chars().collect();
            let start = chars.len().saturating_sub(2048);
            chars[start..].iter().collect::<String>()
        };
        let error = if stderr_tail.trim().is_empty() {
            error.context("kernel start")
        } else {
            error.context(format!(
                "kernel start; kernel stderr tail:
{stderr_tail}"
            ))
        };
        return Err(error);
    }

    // Revive a prior session's namespace before the bootstrap, so the
    // bootstrap then overwrites live handles (rlm, skills) on top of anything restored.
    let mut pending_restore: Option<RestoreResult> = None;
    let mut snapshot_existed = false;
    if let Some(dir) = &snapshot_dir {
        snapshot_existed = snapshot_path_in(dir).exists();
        emit_startup_progress(inner, on_progress, memo, "Restoring Python state...");
        let restore = manager.restore_state().await;
        if snapshot_existed {
            pending_restore = Some(restore.unwrap_or_default());
        }
    }
    emit_startup_progress(inner, on_progress, memo, "Preparing Python runtime...");
    // The bootstrap runs on the dispose signal (TS startKernel races every
    // boot stage against the dispose-linked abort): a dispose mid-bootstrap
    // settles the cell aborted, and the aborted-status arm below tears the
    // kernel down instead of leaking it into a disposed provisioner.
    let bootstrap = manager
        .execute_bounded(
            &bootstrap_code,
            ExecuteOptions {
                signal: Some(dispose_signal.clone()),
                ..Default::default()
            },
            Some(BOOTSTRAP_EXECUTION_TIMEOUT_MS),
        )
        .await;
    match bootstrap {
        Ok(bootstrap) if bootstrap.status == ExecuteStatus::Ok && dispose_signal.is_aborted() => {
            // The cell completed, but the provisioner was disposed under it:
            // the same teardown as the aborted-status arm, with the honest
            // disposed-startup cause (TS startKernel's abort error).
            let (snapshot_policy, _teardown_gate) = hold_snapshot_flush_gate(inner, memo);
            let _ = manager
                .shutdown(KernelShutdownOptions {
                    snapshot: snapshot_policy,
                    drain_host_requests: true,
                })
                .await;
            return Err(anyhow!("Kernel provisioner disposed during startup"));
        }
        Ok(bootstrap) if bootstrap.status == ExecuteStatus::Ok => {
            if snapshot_existed {
                // The just-restored namespace is fresh: the debounced
                // auto-snapshot would rewrite identical content.
                manager.mark_restored_namespace_fresh();
            }
            // Broken skill imports stay importable-looking placeholders;
            // report them so the model learns before its first call, not
            // from the placeholder's error (TS startKernel) - but only a
            // LIVE generation may report: the callback shares the restore
            // notice's mailbox, and a boot kill() invalidated must not
            // append stale skills-unavailable rows the next turn shows the
            // model. Same contract as the restore gate below: the
            // generation check and the callback share ONE lock scope, and
            // the callbacks must not re-enter the provisioner.
            let unavailable = parse_unavailable_python_skills(&bootstrap.stdout);
            if let Some(errors) = unavailable {
                let state = inner
                    .state
                    .lock()
                    .unwrap_or_else(std::sync::PoisonError::into_inner);
                if !state.disposed
                    && state
                        .startup
                        .as_ref()
                        .is_some_and(|armed| armed.same_channel(memo))
                {
                    if let Some(on_unavailable_skills) = &inner.options.on_unavailable_skills {
                        on_unavailable_skills(&errors);
                    }
                }
            }
        }
        Ok(bootstrap) => {
            // The kernel booted but its runtime did not initialize: the venv is the prime suspect,
            // so drop the memoized runtime-ready result.
            crate::kernel::bootstrap::invalidate_runtime_probe_cache();
            let details = [bootstrap.stderr.clone()]
                .into_iter()
                .chain(bootstrap.error.iter().map(|e| e.traceback.join("\n")))
                .filter(|s| !s.is_empty())
                .collect::<Vec<_>>()
                .join("\n");
            // TS startKernel's catch shuts the failed boot down with the
            // dispose snapshot policy (default true), not `false`.
            let (snapshot_policy, _teardown_gate) = hold_snapshot_flush_gate(inner, memo);
            let _ = manager
                .shutdown(KernelShutdownOptions {
                    snapshot: snapshot_policy,
                    drain_host_requests: true,
                })
                .await;
            // An aborted bootstrap with a live dispose signal is the bound
            // firing on a kernel that stopped answering: name the lost
            // bootstrap instead of a bare runtime failure.
            let error = if bootstrap.status == ExecuteStatus::Aborted
                && !dispose_signal.is_aborted()
            {
                anyhow!(
                    "Failed to initialize rlm runtime in the Python kernel: \
                     the runtime bootstrap did not finish within {BOOTSTRAP_EXECUTION_TIMEOUT_MS}ms:\n{details}"
                )
            } else {
                anyhow!("Failed to initialize rlm runtime in the Python kernel:\n{details}")
            };
            return Err(error);
        }
        Err(error) => {
            let (snapshot_policy, _teardown_gate) = hold_snapshot_flush_gate(inner, memo);
            let _ = manager
                .shutdown(KernelShutdownOptions {
                    snapshot: snapshot_policy,
                    drain_host_requests: true,
                })
                .await;
            return Err(error);
        }
    }

    // Only tell the model what was revived once the kernel is actually usable —
    // a notice claiming restored state must never outlive a failed bootstrap,
    // and only a LIVE generation may surface it: a boot kill() invalidated
    // restores its snapshot only for the settle to kill its kernel, so its
    // restore must not fire the notification nor pollute last_restore for
    // the fresher generation. The liveness check, the write, AND the
    // callback share ONE lock scope (this provisioner invokes its
    // callbacks under the state lock - the startup-progress listeners
    // already do - and they must not re-enter the provisioner), so a
    // kill() can no longer slip between the check and the notice.
    if let Some(restore) = pending_restore {
        let mut state = inner
            .state
            .lock()
            .unwrap_or_else(std::sync::PoisonError::into_inner);
        if !state.disposed
            && state
                .startup
                .as_ref()
                .is_some_and(|armed| armed.same_channel(memo))
        {
            state.last_restore = Some(restore.clone());
            if let Some(on_restore) = &inner.options.on_restore {
                on_restore(&restore);
            }
        }
    }
    Ok(manager)
}

/// Same as [`IpythonKernelProvisioner::new`] for a `Path`-shaped cwd.
#[must_use]
pub fn provisioner_for_path(
    cwd: &Path,
    options: IpythonKernelProvisionerOptions,
) -> IpythonKernelProvisioner {
    IpythonKernelProvisioner::new(cwd.to_path_buf(), options)
}

#[cfg(test)]
mod tests {
    use super::*;

    /// A shell file the resolver accepts: existing, and executable on unix.
    fn executable_shell(dir: &Path, name: &str) -> PathBuf {
        let path = dir.join(name);
        std::fs::write(&path, b"").unwrap();
        #[cfg(unix)]
        {
            use std::os::unix::fs::PermissionsExt;
            std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o755)).unwrap();
        }
        path
    }

    /// The scripted session both shell-setting tests build: a model and
    /// `stream_fn` the settings validation never reaches, no kernel prewarm.
    async fn shell_test_session(
        cwd: PathBuf,
        agent_dir: PathBuf,
    ) -> anyhow::Result<crate::session_engine::engine::SessionEngine> {
        use crate::session_engine::engine::{create_session, SessionEngineConfig};

        let model = pa_agent::types::Model {
            id: "shell-test".into(),
            name: "Shell test".into(),
            api: "test".into(),
            provider: "test".into(),
            base_url: "http://localhost".into(),
            reasoning: false,
            cost: pa_agent::types::UsageCost::default(),
            context_window: 1_000,
            max_tokens: 100,
            max_tokens_explicit: false,
        };
        let provider = Arc::new(pa_agent::scripted::ScriptedProvider::new(model.clone()));
        create_session(SessionEngineConfig {
            cwd,
            agent_dir,
            model: Some(model),
            stream_fn: Some(provider.stream_fn()),
            prewarm_ipython_kernel: Some(false),
            ..Default::default()
        })
        .await
    }

    #[tokio::test]
    async fn session_settings_reach_the_kernel_shell() {
        let dir = tempfile::tempdir().unwrap();
        let agent_dir = dir.path().join("agent");
        let cwd = dir.path().join("project");
        std::fs::create_dir_all(&agent_dir).unwrap();
        std::fs::create_dir_all(cwd.join(".prime/agent")).unwrap();
        // Project settings override the global setting before the session captures it.
        let global = executable_shell(dir.path(), "global-bash");
        let project = executable_shell(dir.path(), "project-bash");
        std::fs::write(
            agent_dir.join("settings.json"),
            serde_json::json!({ "shellPath": global.to_string_lossy() }).to_string(),
        )
        .unwrap();
        std::fs::write(
            cwd.join(".prime/agent/settings.json"),
            serde_json::json!({ "shellPath": format!("  {}  ", project.to_string_lossy()) })
                .to_string(),
        )
        .unwrap();
        let engine = shell_test_session(cwd, agent_dir).await.unwrap();
        assert_eq!(engine.provisioner.inner.options.shell_path, Some(project));
    }

    #[tokio::test]
    async fn session_settings_reach_the_kernel_command_prefix() {
        let dir = tempfile::tempdir().unwrap();
        let agent_dir = dir.path().join("agent");
        let cwd = dir.path().join("project");
        std::fs::create_dir_all(&agent_dir).unwrap();
        std::fs::create_dir_all(cwd.join(".prime/agent")).unwrap();
        std::fs::write(
            agent_dir.join("settings.json"),
            serde_json::json!({ "shellCommandPrefix": "shopt -s expand_aliases" }).to_string(),
        )
        .unwrap();
        let engine = shell_test_session(cwd, agent_dir).await.unwrap();
        assert_eq!(
            engine.provisioner.inner.options.command_prefix.as_deref(),
            Some("shopt -s expand_aliases")
        );
    }

    #[tokio::test]
    async fn a_stored_serper_key_reaches_the_kernel_env() {
        let dir = tempfile::tempdir().unwrap();
        let agent_dir = dir.path().join("agent");
        let cwd = dir.path().join("project");
        std::fs::create_dir_all(&agent_dir).unwrap();
        std::fs::create_dir_all(cwd.join(".prime/agent")).unwrap();
        std::fs::write(
            agent_dir.join("auth.json"),
            r#"{ "serper": { "type": "api_key", "key": "!printf key-from-cmd" } }"#,
        )
        .unwrap();
        let engine = shell_test_session(cwd, agent_dir).await.unwrap();
        let injected = engine
            .provisioner
            .inner
            .options
            .env
            .get("SERPER_API_KEY")
            .map(String::as_str);
        // A nonblank ambient SERPER_API_KEY wins (TS parity): the ambient key
        // reaches the kernel through the inherited env, not the wiring.
        if std::env::var("SERPER_API_KEY").is_ok_and(|value| !value.trim().is_empty()) {
            assert_eq!(injected, None);
        } else {
            assert_eq!(injected, Some("key-from-cmd"));
        }
    }

    #[tokio::test]
    async fn an_unsaved_session_blanks_the_kernel_session_id() {
        let dir = tempfile::tempdir().unwrap();
        let agent_dir = dir.path().join("agent");
        let cwd = dir.path().join("project");
        std::fs::create_dir_all(&agent_dir).unwrap();
        std::fs::create_dir_all(cwd.join(".prime/agent")).unwrap();
        let engine = shell_test_session(cwd, agent_dir).await.unwrap();
        assert_eq!(
            engine
                .provisioner
                .inner
                .options
                .env
                .get("RLM_SESSION_ID")
                .map(String::as_str),
            Some(""),
            "an inherited RLM_SESSION_ID must not reach an unsaved session's kernel"
        );
    }

    #[tokio::test]
    async fn without_a_websearch_skill_no_serper_key_env() {
        let dir = tempfile::tempdir().unwrap();
        let agent_dir = dir.path().join("agent");
        let cwd = dir.path().join("project");
        std::fs::create_dir_all(&agent_dir).unwrap();
        std::fs::create_dir_all(cwd.join(".prime/agent")).unwrap();
        std::fs::write(
            agent_dir.join("settings.json"),
            serde_json::json!({ "bundledSkills": { "websearch": false } }).to_string(),
        )
        .unwrap();
        std::fs::write(
            agent_dir.join("auth.json"),
            r#"{ "serper": { "type": "api_key", "key": "!printf key-from-cmd" } }"#,
        )
        .unwrap();
        let engine = shell_test_session(cwd, agent_dir).await.unwrap();
        assert!(
            !engine
                .provisioner
                .inner
                .options
                .env
                .contains_key("SERPER_API_KEY"),
            "the key rides the env only when a websearch skill is loaded"
        );
    }

    #[tokio::test]
    async fn session_settings_reject_a_missing_shell_path() {
        let dir = tempfile::tempdir().unwrap();
        let agent_dir = dir.path().join("agent");
        let cwd = dir.path().join("project");
        std::fs::create_dir_all(&agent_dir).unwrap();
        std::fs::create_dir_all(cwd.join(".prime/agent")).unwrap();
        let missing = cwd.join("no-such-shell");
        std::fs::write(
            cwd.join(".prime/agent/settings.json"),
            serde_json::json!({ "shellPath": missing.to_string_lossy() }).to_string(),
        )
        .unwrap();
        let Err(err) = shell_test_session(cwd, agent_dir).await else {
            panic!("create_session must reject a missing shellPath");
        };
        let message = err.to_string();
        assert!(
            message.contains("shellPath"),
            "error names the setting: {message}"
        );
        assert!(
            message.contains(missing.to_string_lossy().as_ref()),
            "error names the path: {message}"
        );
    }

    #[test]
    fn a_spent_stop_arm_releases_only_its_own_boot() {
        let provisioner = IpythonKernelProvisioner::new(
            PathBuf::from("/tmp"),
            IpythonKernelProvisionerOptions::default(),
        );
        let (armed_tx, armed_rx) = tokio::sync::watch::channel(None::<StartupResult>);
        let (other_tx, other_rx) = tokio::sync::watch::channel(None::<StartupResult>);
        drop((armed_tx, other_tx));
        {
            let mut state = provisioner.lock_state();
            state.pending_stop_for_startup = Some(armed_rx.clone());
            release_spent_stop_arm(&mut state, &other_rx);
            assert!(
                state.pending_stop_for_startup.is_some(),
                "an arm armed for a different boot is not spent by this boot's settle"
            );
            release_spent_stop_arm(&mut state, &armed_rx);
            assert!(
                state.pending_stop_for_startup.is_none(),
                "the arm armed for this boot is spent by its settle"
            );
        }
    }

    #[test]
    fn kill_releases_the_stop_arm_armed_for_the_killed_boot() {
        let provisioner = IpythonKernelProvisioner::new(
            PathBuf::from("/tmp"),
            IpythonKernelProvisionerOptions::default(),
        );
        let (memo_tx, memo_rx) = tokio::sync::watch::channel(None::<StartupResult>);
        drop(memo_tx);
        {
            let mut state = provisioner.lock_state();
            state.startup = Some(memo_rx.clone());
            state.pending_stop_for_startup = Some(memo_rx);
        }
        provisioner.kill();
        let state = provisioner.lock_state();
        assert!(state.startup.is_none(), "kill() invalidates the memo");
        assert!(
            state.pending_stop_for_startup.is_none(),
            "kill() releases the stop arm armed for the boot it invalidates"
        );
        assert_eq!(
            state.doomed_startups.len(),
            1,
            "kill() parks the invalidated memo for dispose() to wait"
        );
    }

    #[test]
    fn a_killed_boot_never_takes_the_dispose_teardown() {
        let provisioner = IpythonKernelProvisioner::new(
            PathBuf::from("/tmp"),
            IpythonKernelProvisionerOptions::default(),
        );
        {
            let mut state = provisioner.lock_state();
            state.disposed = true;
            state.dispose_snapshot = true;
        }
        // A killed generation tears down with kill() semantics even under a
        // racing dispose: the dispose's own snapshot policy cannot resurrect
        // it. The order is the contract — checking disposed first would
        // flush a snapshot kill() forbade.
        {
            let state = provisioner.lock_state();
            assert_eq!(settle_decision(&state, false), SettleDecision::Kill);
            // A live generation honors the dispose; a live undisposed one
            // parks.
            assert_eq!(settle_decision(&state, true), SettleDecision::Dispose);
        }
        {
            let mut state = provisioner.lock_state();
            state.disposed = false;
        }
        let state = provisioner.lock_state();
        assert_eq!(settle_decision(&state, true), SettleDecision::Publish);
    }

    #[test]
    fn a_doomed_boot_teardown_never_flushes_a_snapshot() {
        let provisioner = IpythonKernelProvisioner::new(
            PathBuf::from("/tmp"),
            IpythonKernelProvisionerOptions::default(),
        );
        let (memo_tx, memo_rx) = tokio::sync::watch::channel(None::<StartupResult>);
        drop(memo_tx);
        // A live boot's failed teardown keeps the dispose policy (true by
        // default) and installs the revival gate for its own flush: a
        // replacement boot that arms after this decision restores only
        // after the flush settles.
        {
            let mut state = provisioner.lock_state();
            state.startup = Some(memo_rx.clone());
        }
        let (flushes, gate) = hold_snapshot_flush_gate(&provisioner.inner, &memo_rx);
        assert!(flushes);
        assert!(
            provisioner.lock_state().pending_stop.is_some(),
            "a flushing teardown gates the revival on its own flush"
        );
        assert!(!provisioner
            .lock_state()
            .pending_stop
            .as_ref()
            .is_some_and(|g| *g.borrow()));
        drop(gate);
        // A boot kill() invalidated must never flush: the replacement
        // generation is already restoring the same on-disk payload.
        provisioner.kill();
        {
            let mut state = provisioner.lock_state();
            state.pending_stop = None;
        }
        let (flushes, gate) = hold_snapshot_flush_gate(&provisioner.inner, &memo_rx);
        assert!(!flushes);
        assert!(gate.is_none());
        assert!(
            provisioner.lock_state().pending_stop.is_none(),
            "a doomed teardown installs no gate"
        );
        // A disposed boot keeps the dispose's own policy (dispose does not
        // clear the memo) and gates the same way.
        let (disposed_tx, disposed_rx) = tokio::sync::watch::channel(None::<StartupResult>);
        drop(disposed_tx);
        {
            let mut state = provisioner.lock_state();
            state.startup = Some(disposed_rx.clone());
            state.disposed = true;
        }
        let (flushes, gate) = hold_snapshot_flush_gate(&provisioner.inner, &disposed_rx);
        assert!(flushes);
        assert!(gate.is_some());
        drop(gate);
        {
            let mut state = provisioner.lock_state();
            state.dispose_snapshot = false;
        }
        let (flushes, gate) = hold_snapshot_flush_gate(&provisioner.inner, &disposed_rx);
        assert!(!flushes);
        assert!(gate.is_none());
        // A stop actively armed for THIS boot covers the flush through its
        // own memo wait (it settles only after this teardown): the
        // teardown leaves that stop's gate untouched.
        let (armed_tx, armed_rx) = tokio::sync::watch::channel(false);
        {
            let mut state = provisioner.lock_state();
            state.dispose_snapshot = true;
            state.pending_stop = Some(armed_rx.clone());
            state.pending_stop_for_startup = Some(disposed_rx.clone());
        }
        let (flushes, gate) = hold_snapshot_flush_gate(&provisioner.inner, &disposed_rx);
        assert!(flushes);
        assert!(
            gate.is_none(),
            "an active stop armed for this boot covers the teardown flush"
        );
        assert!(
            provisioner
                .lock_state()
                .pending_stop
                .as_ref()
                .is_some_and(|g| g.same_channel(&armed_rx)),
            "the active stop's gate survives the teardown decision"
        );
        drop(armed_tx);
        // A merely installed, long-settled older gate covers nothing:
        // the teardown gate replaces it, or a kill() racing the teardown
        // would leave the replacement boot restoring against this
        // teardown's flush.
        {
            let mut state = provisioner.lock_state();
            state.pending_stop_for_startup = None;
        }
        let (flushes, gate) = hold_snapshot_flush_gate(&provisioner.inner, &disposed_rx);
        assert!(flushes);
        assert!(
            gate.is_some(),
            "a stale settled gate must not suppress the teardown gate"
        );
        assert!(
            !provisioner
                .lock_state()
                .pending_stop
                .as_ref()
                .is_some_and(|g| g.same_channel(&armed_rx)),
            "the stale gate is replaced"
        );
        drop(gate);
    }

    #[test]
    fn boot_concurrency_defaults_and_override() {
        let default = default_kernel_boot_concurrency();
        assert!(default >= 4);
        std::env::set_var("PRIME_AGENT_MAX_CONCURRENT_KERNEL_BOOTS", "2");
        assert_eq!(resolve_kernel_boot_concurrency(), 2);
        std::env::set_var("PRIME_AGENT_MAX_CONCURRENT_KERNEL_BOOTS", "0");
        assert_eq!(resolve_kernel_boot_concurrency(), default);
        std::env::set_var("PRIME_AGENT_MAX_CONCURRENT_KERNEL_BOOTS", "junk");
        assert_eq!(resolve_kernel_boot_concurrency(), default);
        std::env::set_var("PRIME_AGENT_MAX_CONCURRENT_KERNEL_BOOTS", "1000");
        assert_eq!(resolve_kernel_boot_concurrency(), 64);
        std::env::remove_var("PRIME_AGENT_MAX_CONCURRENT_KERNEL_BOOTS");
    }

    #[tokio::test]
    async fn ensure_rejects_aborted_signal() {
        let provisioner =
            IpythonKernelProvisioner::new("/tmp", IpythonKernelProvisionerOptions::default());
        let error = provisioner
            .ensure(None, Some(AbortSignal::aborted()))
            .await
            .expect_err("aborted startup must reject");
        assert!(error.to_string().contains("aborted"));
    }

    #[tokio::test]
    async fn dispose_then_ensure_fails() {
        let provisioner =
            IpythonKernelProvisioner::new("/tmp", IpythonKernelProvisionerOptions::default());
        provisioner.dispose(None).await;
        let error = provisioner
            .ensure(None, None)
            .await
            .expect_err("disposed provisioner");
        assert!(error.to_string().contains("disposed"));
    }

    #[test]
    fn retryable_failure_classification() {
        assert!(startup_failure_is_retryable(&anyhow!(
            "failed to spawn kernel python /x"
        )));
        assert!(startup_failure_is_retryable(&anyhow!(
            "Kernel did not become ready within 30000ms. stderr tail: ..."
        )));
        assert!(!startup_failure_is_retryable(&anyhow!(
            "Kernel provisioner disposed before start"
        )));
        assert!(!startup_failure_is_retryable(&anyhow!(
            "Kernel provisioner killed before start"
        )));
        assert!(!startup_failure_is_retryable(&anyhow!(
            "Python execution aborted"
        )));
        assert!(!startup_failure_is_retryable(&anyhow!(
            "PRIME_AGENT_KERNEL_PYTHON points to a Python missing a current prime-agent-runtime: /bad"
        )));
        assert!(!startup_failure_is_retryable(&anyhow!(
            "Kernel runtime speaks protocol 2, expected 3. \
             Update prime-agent-runtime in the kernel Python (PRIME_AGENT_KERNEL_PYTHON) to match this prime-agent."
        )));
    }

    #[tokio::test]
    async fn startup_failure_surfaces_cause_and_duration() {
        let options = IpythonKernelProvisionerOptions {
            python: Some(PathBuf::from("/nonexistent/kernel-python-for-test")),
            ..Default::default()
        };
        let provisioner = IpythonKernelProvisioner::new("/tmp", options);
        let error = provisioner
            .ensure(None, None)
            .await
            .expect_err("bogus python must fail");
        let message = format!("{error:#}");
        assert!(
            message.contains("kernel startup failed after "),
            "error must carry the duration: {message}"
        );
        assert!(
            message.contains("ms: "),
            "error must carry the duration unit: {message}"
        );
        assert!(
            message.contains("failed to spawn"),
            "error must carry the spawn cause: {message}"
        );
        // The next ensure() retries fresh rather than rethrowing the memo.
        assert!(provisioner.ensure(None, None).await.is_err());
    }

    #[tokio::test]
    async fn startup_retries_transient_failure() {
        // One retry (the default), so the bogus-python boot fails twice and
        // the backoff (>= 250ms) shows up in the elapsed time.
        let started = std::time::Instant::now();
        let options = IpythonKernelProvisionerOptions {
            python: Some(PathBuf::from("/nonexistent/kernel-python-for-test")),
            ..Default::default()
        };
        let provisioner = IpythonKernelProvisioner::new("/tmp", options);
        assert!(provisioner.ensure(None, None).await.is_err());
        assert!(
            started.elapsed() >= std::time::Duration::from_millis(250),
            "the retry backoff must elapse before the failure surfaces"
        );
    }

    #[tokio::test]
    async fn startup_retry_cancelled_by_dispose() {
        let options = IpythonKernelProvisionerOptions {
            python: Some(PathBuf::from("/nonexistent/kernel-python-for-test")),
            ..Default::default()
        };
        let provisioner = IpythonKernelProvisioner::new("/tmp", options);
        let p = provisioner.clone();
        tokio::spawn(async move {
            tokio::time::sleep(std::time::Duration::from_millis(10)).await;
            p.dispose(None).await;
        });
        // The boot fails; the dispose cancels any pending retry, so ensure()
        // settles without hanging on the backoff chain.
        let started = std::time::Instant::now();
        let _ = provisioner.ensure(None, None).await;
        assert!(
            started.elapsed() < std::time::Duration::from_secs(5),
            "dispose during retry must cancel the backoff promptly"
        );
    }

    #[tokio::test]
    async fn clone_shares_kernel_state() {
        let provisioner =
            IpythonKernelProvisioner::new("/tmp", IpythonKernelProvisionerOptions::default());
        let clone = provisioner.clone();
        provisioner.dispose(None).await;
        assert!(clone.ensure(None, None).await.is_err());
    }

    #[tokio::test]
    async fn prewarm_swallows_failure_and_keeps_lazy_fallback() {
        let options = IpythonKernelProvisionerOptions {
            python: Some(PathBuf::from("/nonexistent/kernel-python-for-test")),
            ..Default::default()
        };
        let provisioner = IpythonKernelProvisioner::new("/tmp", options);
        provisioner.prewarm();
        // Let the background startup settle into its failure.
        tokio::time::sleep(std::time::Duration::from_millis(300)).await;
        assert!(
            !provisioner.has_running_kernel(),
            "the failed prewarm must not leave a running kernel"
        );
        // The next ensure() surfaces the swallowed cause, not a memo hang.
        let error = provisioner
            .ensure(None, None)
            .await
            .expect_err("the bogus python must fail ensure too");
        assert!(
            format!("{error:#}").contains("failed to spawn"),
            "ensure must surface the spawn cause: {error:#}"
        );
    }

    /// A ready kernel that stops reading stdin must not wedge its bootstrap
    /// request or the failed boot's shutdown behind a full pipe.
    #[cfg(unix)]
    #[tokio::test]
    async fn nonreading_kernel_fails_bootstrap_without_parking_shutdown() {
        use std::os::unix::fs::PermissionsExt;

        let dir = tempfile::TempDir::new().expect("temp dir");
        let python = dir.path().join("ready-but-not-reading");
        std::fs::write(
            &python,
            "#!/usr/bin/env python3\nimport json, time\nprint(json.dumps({'event': 'ready', 'protocol': 3, 'python': '3.13.0'}), flush=True)\ntime.sleep(30)\n",
        )
        .expect("write fake kernel");
        std::fs::set_permissions(&python, std::fs::Permissions::from_mode(0o755))
            .expect("chmod fake kernel");
        let provisioner = IpythonKernelProvisioner::new(
            dir.path(),
            IpythonKernelProvisionerOptions {
                python: Some(python),
                python_skills: vec![KernelPythonSkill {
                    name: "oversized".into(),
                    import_name: "x".repeat(1024 * 1024),
                    package_path: dir.path().into(),
                    pyproject_path: dir.path().join("pyproject.toml"),
                }],
                ..Default::default()
            },
        );
        let error = tokio::time::timeout(
            std::time::Duration::from_secs(9),
            provisioner.ensure(None, None),
        )
        .await
        .expect("timed out bootstrap must not park during failed-boot cleanup")
        .expect_err("kernel never reads bootstrap");
        assert!(format!("{error:#}").contains("runtime bootstrap did not finish"));
        assert!(!provisioner.has_running_kernel());
    }

    /// A kernel that answers the ready handshake but never answers the
    /// bootstrap execute (the wedged-kernel shape: the frame is lost inside
    /// the kernel) fails `ensure` with the bound's message instead of parking
    /// forever, tears its kernel down, and does not auto-retry the fatal
    /// failure.
    #[cfg(unix)]
    #[tokio::test]
    async fn silent_bootstrap_fails_bounded_and_leaves_no_kernel() {
        use std::os::unix::fs::PermissionsExt;

        // Speaks protocol v3: answers the ready handshake, stays silent on
        // every execute (the runtime bootstrap included), and answers the
        // shutdown frame so a teardown does not wait out its kill deadline.
        const SILENT_BOOTSTRAP_RUNTIME: &str = r#"#!/usr/bin/env python3
import json
import os
import sys

base = os.path.dirname(os.path.abspath(__file__))
with open(os.path.join(base, "starts"), "a") as f:
    f.write("x")
print(json.dumps({"event": "ready", "protocol": 3, "python": "3.13.0"}), flush=True)
for line in sys.stdin:
    try:
        req = json.loads(line)
    except Exception:
        continue
    if req.get("type") == "shutdown":
        print(json.dumps({"event": "done", "id": req.get("id"), "status": "ok"}), flush=True)
        break
"#;

        let dir = tempfile::TempDir::new().expect("temp dir");
        let python = dir.path().join("fake-kernel");
        std::fs::write(&python, SILENT_BOOTSTRAP_RUNTIME).expect("write fake runtime");
        std::fs::set_permissions(&python, std::fs::Permissions::from_mode(0o755))
            .expect("chmod fake runtime");
        let provisioner = IpythonKernelProvisioner::new(
            dir.path(),
            IpythonKernelProvisionerOptions {
                python: Some(python),
                ..Default::default()
            },
        );

        let outcome = tokio::time::timeout(
            std::time::Duration::from_secs(10),
            provisioner.ensure(None, None),
        )
        .await
        .expect("the bounded bootstrap must settle the boot")
        .expect_err("a kernel that never answers the bootstrap must not report success");
        let chain = format!("{outcome:#}");
        assert!(
            chain.contains("Failed to initialize rlm runtime"),
            "{chain}"
        );
        assert!(
            chain.contains(&format!(
                "the runtime bootstrap did not finish within {BOOTSTRAP_EXECUTION_TIMEOUT_MS}ms"
            )),
            "{chain}"
        );
        assert!(
            !provisioner.has_running_kernel(),
            "the failed boot must tear its kernel down"
        );
        assert_eq!(
            std::fs::metadata(dir.path().join("starts")).map_or(0, |m| m.len()),
            1,
            "the fatal classification must not auto-retry the boot"
        );
    }
}