The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.
Rust agent-orchestrationai-agentsai-assistantdesktopllmlocal-firstmcppersonal-aiprivacyrustsecond-braintauri
feat(sandbox): a shell run can return at a yield point and keep running
sanil-23c72f8e25a09b
diff --git a/crates/openhuman-core/src/sandbox/mod.rs b/crates/openhuman-core/src/sandbox/mod.rsindex f6b817bcec..8e2139b9d1 100644--- a/crates/openhuman-core/src/sandbox/mod.rs+++ b/crates/openhuman-core/src/sandbox/mod.rs@@ -16,6 +16,8 @@ pub(crate) mod native_jail;pub mod ops;pub mod schemas;pub(crate) mod shell_exec;+pub(crate) mod shell_live;+pub(crate) mod shell_processes;pub(crate) mod shell_progress;pub mod types;diff --git a/crates/openhuman-core/src/sandbox/shell_exec.rs b/crates/openhuman-core/src/sandbox/shell_exec.rsindex b2c95abf49..5e1e2dd47c 100644--- a/crates/openhuman-core/src/sandbox/shell_exec.rs+++ b/crates/openhuman-core/src/sandbox/shell_exec.rs@@ -10,6 +10,7 @@use std::{collections::BTreeMap, path::PathBuf, time::Duration};+use super::shell_live::LiveShell;use super::types::SandboxExecResult;use crate::config::Config;@@ -44,6 +45,10 @@ pub(crate) struct NativeShellRun {pub(crate) isolate_network: bool,/// Receives the command's output as it streams, for progress reporting.pub(crate) meter: Option<super::shell_progress::ShellOutputMeter>,+ /// Return with the process still running once it has run this long,+ /// instead of waiting for it (the shell tool's yield point). `None`+ /// waits for the command to end or its runtime budget to stop it.+ pub(crate) yield_after: Option<Duration>,/// Native confinement; `None` is ordinary local execution.#[cfg(feature = "modules")]pub(crate) jail: Option<contract::NativeJailConfig>,@@ -69,6 +74,7 @@ impl NativeShellRun {hide_window: false,isolate_network: false,meter: None,+ yield_after: None,#[cfg(feature = "modules")]jail: None,}@@ -94,14 +100,38 @@ impl NativeShellRun {pub(crate) static USE_FIXTURE: std::sync::atomic::AtomicBool =std::sync::atomic::AtomicBool::new(true);-/// Run `run` through the module and collect its output.-///-/// A worker owns cancellation and resource cleanup if the caller drops its wait.+/// How a run came back: the command ended, or it was still running at its+/// yield point and is handed over to be followed.+pub(crate) enum ShellRunOutcome {+ Finished(SandboxExecResult),+ Running(LiveShell),+}++/// Run `run` through the module and collect its output, waiting for the+/// command to end (any yield point is ignored).pub(crate) async fn run(ctx: &crate::core::runtime::DispatchCtx,config: &Config,mut run: NativeShellRun,) -> anyhow::Result<SandboxExecResult> {+ run.yield_after = None;+ match run_yielding(ctx, config, run).await? {+ ShellRunOutcome::Finished(result) => Ok(result),+ ShellRunOutcome::Running(_) => {+ anyhow::bail!("a shell run without a yield point returned while running")+ }+ }+}++/// Run `run` through the module. With a yield point, a command still running+/// there comes back as [`ShellRunOutcome::Running`] and keeps running.+///+/// A worker owns cancellation and resource cleanup if the caller drops its wait.+pub(crate) async fn run_yielding(+ ctx: &crate::core::runtime::DispatchCtx,+ config: &Config,+ mut run: NativeShellRun,+) -> anyhow::Result<ShellRunOutcome> {#[cfg(test)]if !run.is_jailed() && USE_FIXTURE.load(std::sync::atomic::Ordering::SeqCst) {let _ = (ctx, config);@@ -115,6 +145,7 @@ pub(crate) async fn run(tracing::debug!(jailed = run.jail.is_some(),timeout_ms = run.timeout.as_millis() as u64,+ yield_after_ms = run.yield_after.map(|after| after.as_millis() as u64),"[sandbox:shell] starting native shell through TinyBox");// A cancelled turn drops this wait mid-command. The worker then cancels@@ -143,11 +174,17 @@ async fn run_owned(run: NativeShellRun,abandoned: tokio::sync::oneshot::Receiver<std::convert::Infallible>,meter: Option<super::shell_progress::ShellOutputMeter>,-) -> anyhow::Result<SandboxExecResult> {+) -> anyhow::Result<ShellRunOutcome> {let resource = tinybox::reserve(config, contract::ReserveRequest::Resource).await.map_err(|_| anyhow::anyhow!("TinyBox could not reserve a command resource"))?;let timeout = run.timeout;+ // A yield point only counts when it comes before the command's own+ // deadline; otherwise the run ends the usual way.+ let yield_at = run+ .yield_after+ .filter(|after| *after < timeout)+ .map(|after| tokio::time::Instant::now() + after);let result = async {tinybox::create(config,@@ -178,9 +215,30 @@ async fn run_owned(let process = tinybox::start_native_shell_exec(config, request).await.map_err(|_| anyhow::anyhow!("TinyBox could not start the command process"))?;- collect_process(config, process, timeout, abandoned, meter).await+ collect_process(config, process, timeout, yield_at, abandoned, meter).await}.await;+ // Still running at its yield point: the process and its resource go to+ // the caller, who releases them once it has stopped.+ let result = match result {+ Ok(Collected::Running {+ process,+ cursor,+ stdout,+ stderr,+ }) => {+ tracing::debug!("[sandbox:shell] command still running at its yield point");+ return Ok(ShellRunOutcome::Running(LiveShell {+ stdout: String::from_utf8_lossy(&stdout).into_owned(),+ stderr: String::from_utf8_lossy(&stderr).into_owned(),+ handle: super::shell_live::LiveHandle::TinyBox(+ super::shell_live::TinyBoxLive::new(config.clone(), resource, process, cursor),+ ),+ }));+ }+ Ok(Collected::Done(output)) => Ok(ShellRunOutcome::Finished(output)),+ Err(error) => Err(error),+ };let close = tinybox::close(config, resource).await;match (result, close) {(Ok(output), Ok(())) => Ok(output),@@ -222,14 +280,28 @@ pub(crate) fn shell_exec_request(})}+/// What [`collect_process`] read: the whole run, or the run so far when it was+/// still going at its yield point.+#[cfg(feature = "modules")]+enum Collected {+ Done(SandboxExecResult),+ Running {+ process: contract::ProcessRef,+ cursor: u64,+ stdout: Vec<u8>,+ stderr: Vec<u8>,+ },+}+#[cfg(feature = "modules")]async fn collect_process(config: &Config,process: contract::ProcessRef,timeout: Duration,+ yield_at: Option<tokio::time::Instant>,mut abandoned: tokio::sync::oneshot::Receiver<std::convert::Infallible>,meter: Option<super::shell_progress::ShellOutputMeter>,-) -> anyhow::Result<SandboxExecResult> {+) -> anyhow::Result<Collected> {use contract::{ExecutionState, OutputStream};use tokio::sync::oneshot::error::TryRecvError;@@ -270,18 +342,29 @@ async fn collect_process(}cursor = batch.next_sequence;match state {- ExecutionState::Running => tokio::time::sleep(Duration::from_millis(20)).await,+ ExecutionState::Running => {+ if !requested_cancel && yield_at.is_some_and(|at| tokio::time::Instant::now() >= at)+ {+ return Ok(Collected::Running {+ process,+ cursor,+ stdout,+ stderr,+ });+ }+ tokio::time::sleep(Duration::from_millis(20)).await+ }ExecutionState::Exited { .. } if has_chunks => continue,ExecutionState::Exited { exit_code } => {tinybox::release_output(config, process).await.map_err(|_| anyhow::anyhow!("TinyBox could not release command output"))?;- return Ok(SandboxExecResult {+ return Ok(Collected::Done(SandboxExecResult {exit_code,stdout: String::from_utf8_lossy(&stdout).into_owned(),stderr: String::from_utf8_lossy(&stderr).into_owned(),timed_out: false,- });+ }));}ExecutionState::Cancelled if has_chunks => continue,ExecutionState::Cancelled => break requested_cancel,@@ -299,12 +382,12 @@ async fn collect_process(// or test killed at its deadline has usually written its progress and// first errors by then. `timed_out` says why it stopped; the shell tool// writes the timeout line.- Ok(SandboxExecResult {+ Ok(Collected::Done(SandboxExecResult {exit_code: -1,stdout: String::from_utf8_lossy(&stdout).into_owned(),stderr: String::from_utf8_lossy(&stderr).into_owned(),timed_out,- })+ }))}/// What a terminal TinyBox failure means for the command. The module enforces@@ -313,7 +396,7 @@ async fn collect_process(/// the command's timeout, reported like one (`Ok(true)`). Anything else is a/// real sandbox failure, named by its code.#[cfg(feature = "modules")]-fn failure_timed_out(code: contract::ExecutionFailure) -> anyhow::Result<bool> {+pub(super) fn failure_timed_out(code: contract::ExecutionFailure) -> anyhow::Result<bool> {use contract::ExecutionFailure;match code {ExecutionFailure::TimedOut => Ok(true),diff --git a/crates/openhuman-core/src/sandbox/shell_exec_fixture.rs b/crates/openhuman-core/src/sandbox/shell_exec_fixture.rsindex 6c498619fc..c4623b13ce 100644--- a/crates/openhuman-core/src/sandbox/shell_exec_fixture.rs+++ b/crates/openhuman-core/src/sandbox/shell_exec_fixture.rs@@ -10,14 +10,15 @@ use std::time::Duration;use tokio::io::{AsyncRead, AsyncReadExt};-use super::{NativeShellRun, SandboxExecResult};+use super::{NativeShellRun, SandboxExecResult, ShellRunOutcome};+use crate::sandbox::shell_live::{LiveHandle, LiveRead, LiveShell, LiveState};/// Variables the module's safe environment snapshot carries.const SNAPSHOT: &[&str] = &["PATH", "HOME", "TERM", "LANG", "LC_ALL", "LC_CTYPE", "USER", "SHELL", "TMPDIR",];-pub(crate) async fn run(run: NativeShellRun) -> anyhow::Result<SandboxExecResult> {+pub(crate) async fn run(run: NativeShellRun) -> anyhow::Result<ShellRunOutcome> {let bash = ["/usr/bin/bash", "/bin/bash"].into_iter().find(|path| std::path::Path::new(path).exists());@@ -73,7 +74,11 @@ pub(crate) async fn run(run: NativeShellRun) -> anyhow::Result<SandboxExecResult// command stopped at its deadline keeps what it printed before.let (mut stdout_pipe, mut stderr_pipe) = (child.stdout.take(), child.stderr.take());let (mut stdout, mut stderr) = (Vec::new(), Vec::new());- let finished = tokio::time::timeout(run.timeout, async {+ let started = tokio::time::Instant::now();+ // Like the module: a command still running at its yield point is handed+ // over running, with what it printed so far.+ let yield_after = run.yield_after.filter(|after| *after < run.timeout);+ let finished = tokio::time::timeout(yield_after.unwrap_or(run.timeout), async {let (status, (), ()) = tokio::join!(child.wait(),drain(&mut stdout_pipe, &mut stdout),@@ -84,6 +89,22 @@ pub(crate) async fn run(run: NativeShellRun) -> anyhow::Result<SandboxExecResult.await;let (exit_code, timed_out) = match finished {Ok(status) => (status?.code().unwrap_or(-1), false),+ Err(_) if yield_after.is_some() => {+ return Ok(ShellRunOutcome::Running(LiveShell {+ stdout: String::from_utf8_lossy(&stdout).into_owned(),+ stderr: String::from_utf8_lossy(&stderr).into_owned(),+ handle: LiveHandle::Fixture(FixtureLive {+ child,+ pid,+ stdout_pipe,+ stderr_pipe,+ stdout: Vec::new(),+ stderr: Vec::new(),+ deadline: started + run.timeout,+ ended: false,+ }),+ }));+ }Err(_) => {// Stop the command and everything it started, then give the// readers a moment to drain what is left in the pipes.@@ -107,12 +128,98 @@ pub(crate) async fn run(run: NativeShellRun) -> anyhow::Result<SandboxExecResultmeter.record(&stdout);meter.record(&stderr);}- Ok(SandboxExecResult {+ Ok(ShellRunOutcome::Finished(SandboxExecResult {exit_code,stdout: String::from_utf8_lossy(&stdout).into_owned(),stderr: String::from_utf8_lossy(&stderr).into_owned(),timed_out,- })+ }))+}++/// A fixture command past its yield point, read and stopped like the module's.+pub(crate) struct FixtureLive {+ child: tokio::process::Child,+ pid: Option<u32>,+ stdout_pipe: Option<tokio::process::ChildStdout>,+ stderr_pipe: Option<tokio::process::ChildStderr>,+ /// Output read since the previous [`LiveRead`].+ stdout: Vec<u8>,+ stderr: Vec<u8>,+ deadline: tokio::time::Instant,+ ended: bool,+}++impl FixtureLive {+ pub(crate) async fn read(&mut self, wait: Duration) -> anyhow::Result<LiveRead> {+ if self.ended {+ return Ok(self.take(LiveState::Stopped));+ }+ let until = (tokio::time::Instant::now() + wait).min(self.deadline);+ let finished = tokio::time::timeout_at(until, async {+ let (status, (), ()) = tokio::join!(+ self.child.wait(),+ drain(&mut self.stdout_pipe, &mut self.stdout),+ drain(&mut self.stderr_pipe, &mut self.stderr),+ );+ status+ })+ .await;+ let state = match finished {+ Ok(status) => {+ self.ended = true;+ LiveState::Exited(status?.code().unwrap_or(-1))+ }+ Err(_) if tokio::time::Instant::now() >= self.deadline => {+ self.stop().await;+ LiveState::TimedOut+ }+ Err(_) => LiveState::Running,+ };+ Ok(self.take(state))+ }++ pub(crate) async fn kill(&mut self) -> anyhow::Result<LiveRead> {+ if !self.ended {+ self.stop().await;+ }+ Ok(self.take(LiveState::Stopped))+ }++ /// Stop the command and everything it started, keeping what is left in+ /// the pipes.+ async fn stop(&mut self) {+ if let Some(pid) = self.pid {+ tinytools_std::timeout::kill_process_group(pid);+ }+ let _ = self.child.start_kill();+ let _ = tokio::time::timeout(Duration::from_secs(2), async {+ tokio::join!(+ drain(&mut self.stdout_pipe, &mut self.stdout),+ drain(&mut self.stderr_pipe, &mut self.stderr),+ )+ })+ .await;+ self.ended = true;+ }++ fn take(&mut self, state: LiveState) -> LiveRead {+ LiveRead {+ stdout: String::from_utf8_lossy(&std::mem::take(&mut self.stdout)).into_owned(),+ stderr: String::from_utf8_lossy(&std::mem::take(&mut self.stderr)).into_owned(),+ state,+ }+ }+}++impl Drop for FixtureLive {+ fn drop(&mut self) {+ if !self.ended {+ if let Some(pid) = self.pid {+ tinytools_std::timeout::kill_process_group(pid);+ }+ let _ = self.child.start_kill();+ }+ }}/// Appends what `pipe` delivers to `into` until it closes. Reads arediff --git a/crates/openhuman-core/src/sandbox/shell_live.rs b/crates/openhuman-core/src/sandbox/shell_live.rsnew file mode 100644index 0000000000..672aac3088--- /dev/null+++ b/crates/openhuman-core/src/sandbox/shell_live.rs@@ -0,0 +1,227 @@+//! A shell command still running after its call returned.+//!+//! The `shell` tool returns at its yield point instead of waiting out the+//! command, and leaves the process running. [`LiveShell`] is what it hands over:+//! the output read so far and a [`LiveHandle`] that reads more, stops the+//! process, and releases it. TinyBox already keeps a process running and its+//! output journalled whether or not the host reads it; this only stops the host+//! from cancelling it when the call returns.+//!+//! A handle that is dropped while its process still runs (the turn ended)+//! stops the process and releases its resource in the background.++use std::time::Duration;++#[cfg(feature = "modules")]+use crate::modules::tinybox::{self, contract};++/// Pause between output reads while a process runs.+#[cfg(feature = "modules")]+const READ_INTERVAL: Duration = Duration::from_millis(50);++/// How long [`LiveHandle::kill`] waits for the stopped process to report.+const KILL_GRACE: Duration = Duration::from_secs(10);++/// The output a process wrote so far, at its yield point.+pub(crate) struct LiveShell {+ pub(crate) stdout: String,+ pub(crate) stderr: String,+ pub(crate) handle: LiveHandle,+}++/// Where a process is after a read.+#[derive(Clone, Copy, Debug, PartialEq, Eq)]+pub(crate) enum LiveState {+ /// Still running.+ Running,+ /// Exited on its own with this code.+ Exited(i32),+ /// Stopped at its runtime budget (`timeout_secs`, clamped to the turn).+ TimedOut,+ /// Stopped by the model or the host.+ Stopped,+}++/// Output written since the previous read, and where the process is now.+pub(crate) struct LiveRead {+ pub(crate) stdout: String,+ pub(crate) stderr: String,+ pub(crate) state: LiveState,+}++/// A running process the host can read, stop and release.+pub(crate) enum LiveHandle {+ #[cfg(feature = "modules")]+ TinyBox(TinyBoxLive),+ #[cfg(test)]+ Fixture(super::shell_exec::fixture::FixtureLive),+}++impl LiveHandle {+ /// Read until the process ends or `wait` passes, whichever is first.+ pub(crate) async fn read(&mut self, wait: Duration) -> anyhow::Result<LiveRead> {+ match self {+ #[cfg(feature = "modules")]+ Self::TinyBox(live) => live.read(wait).await,+ #[cfg(test)]+ Self::Fixture(live) => live.read(wait).await,+ }+ }++ /// Stop the process and return what it wrote since the previous read.+ pub(crate) async fn kill(&mut self) -> anyhow::Result<LiveRead> {+ match self {+ #[cfg(feature = "modules")]+ Self::TinyBox(live) => live.kill().await,+ #[cfg(test)]+ Self::Fixture(live) => live.kill().await,+ }+ }+}++/// A TinyBox native shell process past its yield point. Owns the command's+/// resource until the process has stopped and its output is released.+#[cfg(feature = "modules")]+pub(crate) struct TinyBoxLive {+ config: crate::config::Config,+ resource: contract::ResourceId,+ process: contract::ProcessRef,+ cursor: u64,+ released: bool,+}++#[cfg(feature = "modules")]+impl TinyBoxLive {+ pub(crate) fn new(+ config: crate::config::Config,+ resource: contract::ResourceId,+ process: contract::ProcessRef,+ cursor: u64,+ ) -> Self {+ Self {+ config,+ resource,+ process,+ cursor,+ released: false,+ }+ }++ async fn read(&mut self, wait: Duration) -> anyhow::Result<LiveRead> {+ let until = tokio::time::Instant::now() + wait;+ self.read_until(until).await+ }++ async fn kill(&mut self) -> anyhow::Result<LiveRead> {+ if !self.released {+ tracing::debug!("[sandbox:shell] stopping a running process on request");+ // A process that ended on its own refuses the cancel; the read+ // below reports how it ended either way.+ let _ = tinybox::cancel(&self.config, self.process.clone()).await;+ }+ let mut read = self+ .read_until(tokio::time::Instant::now() + KILL_GRACE)+ .await?;+ if read.state == LiveState::Running {+ read.state = LiveState::Stopped;+ }+ Ok(read)+ }++ async fn read_until(&mut self, until: tokio::time::Instant) -> anyhow::Result<LiveRead> {+ use contract::{ExecutionState, OutputStream};+ let (mut stdout, mut stderr) = (Vec::new(), Vec::new());+ let done = |stdout: &[u8], stderr: &[u8], state| LiveRead {+ stdout: String::from_utf8_lossy(stdout).into_owned(),+ stderr: String::from_utf8_lossy(stderr).into_owned(),+ state,+ };+ if self.released {+ return Ok(done(&stdout, &stderr, LiveState::Stopped));+ }+ loop {+ let batch = tinybox::read_output(&self.config, self.process.clone(), self.cursor)+ .await+ .map_err(|_| anyhow::anyhow!("TinyBox could not read command output"))?;+ let has_chunks = !batch.chunks.is_empty();+ for chunk in batch.chunks {+ match chunk.stream {+ OutputStream::Stdout => stdout.extend_from_slice(&chunk.bytes),+ OutputStream::Stderr => stderr.extend_from_slice(&chunk.bytes),+ }+ }+ self.cursor = batch.next_sequence;+ let state = match batch.state {+ ExecutionState::Running => {+ if tokio::time::Instant::now() >= until {+ return Ok(done(&stdout, &stderr, LiveState::Running));+ }+ tokio::time::sleep(READ_INTERVAL).await;+ continue;+ }+ _ if has_chunks => continue,+ ExecutionState::Exited { exit_code } => LiveState::Exited(exit_code),+ ExecutionState::Cancelled => LiveState::Stopped,+ ExecutionState::Failed { code } => {+ tracing::debug!(?code, "[sandbox:shell] a running process failed");+ match super::shell_exec::failure_timed_out(code) {+ Ok(true) => LiveState::TimedOut,+ Ok(false) => LiveState::Stopped,+ Err(error) => {+ self.release().await;+ return Err(error);+ }+ }+ }+ };+ self.release().await;+ return Ok(done(&stdout, &stderr, state));+ }+ }++ /// Release the stopped process's output and close its resource. Once.+ async fn release(&mut self) {+ if self.released {+ return;+ }+ self.released = true;+ let _ = tinybox::release_output(&self.config, self.process.clone()).await;+ if tinybox::close(&self.config, self.resource.clone())+ .await+ .is_err()+ {+ tracing::warn!("[sandbox:shell] TinyBox did not acknowledge a process cleanup");+ }+ }+}++#[cfg(feature = "modules")]+impl Drop for TinyBoxLive {+ fn drop(&mut self) {+ if self.released {+ return;+ }+ let Ok(runtime) = tokio::runtime::Handle::try_current() else {+ tracing::warn!("[sandbox:shell] no runtime to stop a running process at turn end");+ return;+ };+ let (config, resource, process) = (+ self.config.clone(),+ self.resource.clone(),+ self.process.clone(),+ );+ tracing::debug!("[sandbox:shell] turn ended: stopping a process it left running");+ runtime.spawn(async move {+ let _ = tinybox::cancel(&config, process.clone()).await;+ let deadline = tokio::time::Instant::now() + KILL_GRACE;+ while tokio::time::Instant::now() < deadline {+ match tinybox::is_running(&config, process.clone()).await {+ Ok(true) => tokio::time::sleep(Duration::from_millis(100)).await,+ _ => break,+ }+ }+ let _ = tinybox::release_output(&config, process).await;+ let _ = tinybox::close(&config, resource).await;+ });+ }+}diff --git a/crates/openhuman-core/src/sandbox/shell_processes.rs b/crates/openhuman-core/src/sandbox/shell_processes.rsnew file mode 100644index 0000000000..d9604f595d--- /dev/null+++ b/crates/openhuman-core/src/sandbox/shell_processes.rs@@ -0,0 +1,241 @@+//! The shell commands a turn left running past their yield point.+//!+//! The `shell` tool adopts a [`LiveShell`] here and gives the model its id+//! (`p1`, `p2`, …); the `shell_process` tool waits on it, stops it or reports+//! its status. One registry belongs to one turn's run context, so dropping it+//! at the end of the turn stops every process still running (each+//! [`LiveHandle`] stops its own on drop). Every process also stops at its own+//! runtime budget, which the turn budget already clamps.++use std::collections::BTreeMap;+use std::sync::atomic::{AtomicU32, Ordering};+use std::sync::{Arc, Mutex};+use std::time::{Duration, Instant};++use super::shell_live::{LiveHandle, LiveShell, LiveState};++/// Processes a turn may leave running at once. A further command runs to its+/// end the usual way.+pub(crate) const MAX_RUNNING: usize = 4;++/// Consecutive waits with no new output after which the report suggests+/// stopping the process.+pub(crate) const QUIET_WAITS_NOTE: u32 = 3;++/// How a process ended, kept for later questions about it.+#[derive(Clone, Copy, Debug, PartialEq, Eq)]+pub(crate) enum Ended {+ Exited(i32),+ TimedOut,+ Stopped,+}++/// What a wait, stop or status call found.+#[derive(Clone, Debug, PartialEq, Eq)]+pub(crate) enum ReportState {+ /// Still running.+ Running,+ /// Ended during this call.+ Ended(Ended),+ /// Had already ended, and its output was already returned.+ AlreadyEnded(Ended),+}++/// One answer about a process, for the tool to render.+#[derive(Clone, Debug)]+pub(crate) struct Report {+ pub(crate) id: String,+ pub(crate) command: String,+ /// Time since the command started.+ pub(crate) elapsed: Duration,+ /// The process's runtime budget.+ pub(crate) timeout: Duration,+ /// Output written since the previous answer.+ pub(crate) stdout: String,+ pub(crate) stderr: String,+ pub(crate) state: ReportState,+ /// Consecutive waits, this one included, that brought no new output.+ pub(crate) quiet_waits: u32,+}++struct Entry {+ handle: LiveHandle,+ command: String,+ started: Instant,+ timeout: Duration,+ ended: Option<Ended>,+ quiet_waits: u32,+}++/// The running processes of one turn.+#[derive(Default)]+pub(crate) struct ShellProcesses {+ next: AtomicU32,+ entries: Mutex<BTreeMap<String, Arc<tokio::sync::Mutex<Entry>>>>,+ running: Mutex<usize>,+}++/// A registry is equal only to itself: two turns never share one.+impl PartialEq for ShellProcesses {+ fn eq(&self, other: &Self) -> bool {+ std::ptr::eq(self, other)+ }+}++impl Eq for ShellProcesses {}++impl ShellProcesses {+ pub(crate) fn new() -> Arc<Self> {+ Arc::new(Self::default())+ }++ /// Whether another process may be left running.+ pub(crate) fn has_room(&self) -> bool {+ *self.running() < MAX_RUNNING+ }++ /// Take over a process still running at its yield point and name it.+ pub(crate) fn adopt(+ &self,+ live: LiveShell,+ command: &str,+ started: Instant,+ timeout: Duration,+ ) -> String {+ let id = format!("p{}", self.next.fetch_add(1, Ordering::SeqCst) + 1);+ *self.running() += 1;+ self.entries().insert(+ id.clone(),+ Arc::new(tokio::sync::Mutex::new(Entry {+ handle: live.handle,+ command: command.to_owned(),+ started,+ timeout,+ ended: None,+ quiet_waits: 0,+ })),+ );+ tracing::debug!(process = %id, "[shell] process left running past its yield point");+ id+ }++ /// Wait up to `wait` for new output or the end of the process.+ pub(crate) async fn wait(&self, id: &str, wait: Duration) -> Result<Report, String> {+ let entry = self.entry(id)?;+ let mut entry = entry.lock().await;+ if let Some(ended) = entry.ended {+ return Ok(report(+ id,+ &entry,+ String::new(),+ String::new(),+ ReportState::AlreadyEnded(ended),+ ));+ }+ let read = entry+ .handle+ .read(wait)+ .await+ .map_err(|error| format!("Could not read process {id}: {error}"))?;+ let quiet = read.stdout.is_empty() && read.stderr.is_empty();+ entry.quiet_waits = if quiet { entry.quiet_waits + 1 } else { 0 };+ let state = self.settle(&mut entry, read.state);+ Ok(report(id, &entry, read.stdout, read.stderr, state))+ }++ /// Stop the process and return what it wrote since the previous answer.+ pub(crate) async fn kill(&self, id: &str) -> Result<Report, String> {+ let entry = self.entry(id)?;+ let mut entry = entry.lock().await;+ if let Some(ended) = entry.ended {+ return Ok(report(+ id,+ &entry,+ String::new(),+ String::new(),+ ReportState::AlreadyEnded(ended),+ ));+ }+ tracing::debug!(process = %id, "[shell] stopping a running process on request");+ let read = entry+ .handle+ .kill()+ .await+ .map_err(|error| format!("Could not stop process {id}: {error}"))?;+ let state = self.settle(&mut entry, read.state);+ Ok(report(id, &entry, read.stdout, read.stderr, state))+ }++ /// Where the process is, without its output.+ pub(crate) async fn status(&self, id: &str) -> Result<Report, String> {+ let entry = self.entry(id)?;+ let entry = entry.lock().await;+ let state = match entry.ended {+ Some(ended) => ReportState::AlreadyEnded(ended),+ None => ReportState::Running,+ };+ Ok(report(id, &entry, String::new(), String::new(), state))+ }++ /// Record an ending, once.+ fn settle(&self, entry: &mut Entry, state: LiveState) -> ReportState {+ let ended = match state {+ LiveState::Running => return ReportState::Running,+ LiveState::Exited(code) => Ended::Exited(code),+ LiveState::TimedOut => Ended::TimedOut,+ LiveState::Stopped => Ended::Stopped,+ };+ if entry.ended.is_none() {+ entry.ended = Some(ended);+ let mut running = self.running();+ *running = running.saturating_sub(1);+ }+ ReportState::Ended(ended)+ }++ fn entry(&self, id: &str) -> Result<Arc<tokio::sync::Mutex<Entry>>, String> {+ let entries = self.entries();+ entries.get(id.trim()).cloned().ok_or_else(|| {+ let known: Vec<&str> = entries.keys().map(String::as_str).collect();+ if known.is_empty() {+ format!("No process {id} in this turn: no command has been left running.")+ } else {+ format!(+ "No process {id} in this turn. Processes: {}.",+ known.join(", ")+ )+ }+ })+ }++ fn entries(+ &self,+ ) -> std::sync::MutexGuard<'_, BTreeMap<String, Arc<tokio::sync::Mutex<Entry>>>> {+ self.entries+ .lock()+ .unwrap_or_else(std::sync::PoisonError::into_inner)+ }++ fn running(&self) -> std::sync::MutexGuard<'_, usize> {+ self.running+ .lock()+ .unwrap_or_else(std::sync::PoisonError::into_inner)+ }+}++fn report(id: &str, entry: &Entry, stdout: String, stderr: String, state: ReportState) -> Report {+ Report {+ id: id.to_owned(),+ command: entry.command.clone(),+ elapsed: entry.started.elapsed(),+ timeout: entry.timeout,+ stdout,+ stderr,+ state,+ quiet_waits: entry.quiet_waits,+ }+}++#[cfg(test)]+#[path = "shell_processes_tests.rs"]+mod tests;diff --git a/crates/openhuman-core/src/sandbox/shell_processes_tests.rs b/crates/openhuman-core/src/sandbox/shell_processes_tests.rsnew file mode 100644index 0000000000..32d46e5660--- /dev/null+++ b/crates/openhuman-core/src/sandbox/shell_processes_tests.rs@@ -0,0 +1,157 @@+//! The per-turn registry of running shell commands, driven through the test+//! stand-in for the native shell.++use super::*;+use crate::sandbox::shell_exec::{run_yielding, NativeShellRun, ShellRunOutcome};++async fn start(command: &str, yield_after_ms: u64, timeout_secs: u64) -> ShellRunOutcome {+ let dir = std::env::temp_dir();+ let mut run = NativeShellRun::new(command, &dir, &dir, Duration::from_secs(timeout_secs));+ run.yield_after = Some(Duration::from_millis(yield_after_ms));+ run_yielding(+ &crate::core::runtime::DispatchCtx::host_default(),+ &crate::config::Config::default(),+ run,+ )+ .await+ .expect("the stand-in runs the command")+}++async fn adopted(+ processes: &ShellProcesses,+ command: &str,+ yield_after_ms: u64,+ timeout_secs: u64,+) -> (String, String) {+ match start(command, yield_after_ms, timeout_secs).await {+ ShellRunOutcome::Running(live) => {+ let printed = live.stdout.clone();+ let id = processes.adopt(+ live,+ command,+ Instant::now(),+ Duration::from_secs(timeout_secs),+ );+ (id, printed)+ }+ ShellRunOutcome::Finished(result) => panic!("expected a running process, got {result:?}"),+ }+}++#[cfg(not(windows))]+#[tokio::test]+async fn a_command_that_ends_before_its_yield_point_finishes_as_usual() {+ match start("printf done", 5_000, 30).await {+ ShellRunOutcome::Finished(result) => assert_eq!(result.stdout, "done"),+ ShellRunOutcome::Running(_) => panic!("a quick command must not be left running"),+ }+}++#[cfg(not(windows))]+#[tokio::test]+async fn a_process_past_its_yield_point_is_followed_to_its_end() {+ let processes = ShellProcesses::new();+ let (id, printed) = adopted(&processes, "echo first; sleep 1; echo second", 300, 30).await;+ assert_eq!(id, "p1");+ assert_eq!(+ printed.trim(),+ "first",+ "the yield carries the output so far"+ );++ let report = processes.wait(&id, Duration::from_secs(10)).await.unwrap();+ assert_eq!(report.state, ReportState::Ended(Ended::Exited(0)));+ assert_eq!(+ report.stdout.trim(),+ "second",+ "only output written since the yield"+ );++ let again = processes.wait(&id, Duration::from_secs(1)).await.unwrap();+ assert_eq!(again.state, ReportState::AlreadyEnded(Ended::Exited(0)));+ assert!(again.stdout.is_empty());+}++#[cfg(not(windows))]+#[tokio::test]+async fn kill_stops_a_running_process() {+ let processes = ShellProcesses::new();+ let (id, _) = adopted(&processes, "echo a; sleep 30", 200, 60).await;+ let report = processes.kill(&id).await.unwrap();+ assert_eq!(report.state, ReportState::Ended(Ended::Stopped));+ let status = processes.status(&id).await.unwrap();+ assert_eq!(status.state, ReportState::AlreadyEnded(Ended::Stopped));+}++#[cfg(not(windows))]+#[tokio::test]+async fn waits_with_nothing_new_are_counted() {+ let processes = ShellProcesses::new();+ let (id, _) = adopted(&processes, "sleep 30", 100, 60).await;+ for expected in 1..=QUIET_WAITS_NOTE {+ let report = processes+ .wait(&id, Duration::from_millis(100))+ .await+ .unwrap();+ assert_eq!(report.state, ReportState::Running);+ assert_eq!(report.quiet_waits, expected);+ }+ processes.kill(&id).await.unwrap();+}++#[cfg(not(windows))]+#[tokio::test]+async fn the_runtime_budget_still_stops_a_process() {+ let processes = ShellProcesses::new();+ let (id, _) = adopted(&processes, "echo x; sleep 30", 200, 1).await;+ let report = processes.wait(&id, Duration::from_secs(10)).await.unwrap();+ assert_eq!(report.state, ReportState::Ended(Ended::TimedOut));+}++#[cfg(not(windows))]+#[tokio::test]+async fn at_most_four_processes_are_left_running() {+ let processes = ShellProcesses::new();+ let mut ids = Vec::new();+ for _ in 0..MAX_RUNNING {+ assert!(processes.has_room());+ ids.push(adopted(&processes, "sleep 30", 100, 60).await.0);+ }+ assert!(!processes.has_room());+ processes.kill(&ids[0]).await.unwrap();+ assert!(processes.has_room(), "a stopped process frees its place");+ for id in &ids[1..] {+ processes.kill(id).await.unwrap();+ }+}++#[cfg(not(windows))]+#[tokio::test]+async fn dropping_the_registry_stops_its_processes() {+ let processes = ShellProcesses::new();+ let (_, printed) = adopted(&processes, "echo $$; sleep 30", 300, 60).await;+ let pid: i32 = printed.trim().parse().expect("the shell printed its pid");+ drop(processes);+ let mut gone = false;+ for _ in 0..100 {+ if unsafe { libc::kill(pid, 0) } != 0 {+ gone = true;+ break;+ }+ tokio::time::sleep(Duration::from_millis(20)).await;+ }+ assert!(gone, "a process left running outlived its turn");+}++#[tokio::test]+async fn an_unknown_process_names_the_known_ones() {+ let processes = ShellProcesses::new();+ let empty = processes+ .wait("p9", Duration::from_millis(1))+ .await+ .unwrap_err();+ assert!(+ empty.contains("no command has been left running"),+ "{empty}"+ );+}