Skip to content

The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.

feat(sandbox): a shell run can return at a yield point and keep running

sanil-23c72f8e25a09b
diff --git a/crates/openhuman-core/src/sandbox/mod.rs b/crates/openhuman-core/src/sandbox/mod.rs
index f6b817bcec..8e2139b9d1 100644
--- a/crates/openhuman-core/src/sandbox/mod.rs
+++ b/crates/openhuman-core/src/sandbox/mod.rs
@@ -16,6 +16,8 @@ pub(crate) mod native_jail;
pub mod ops;
pub mod schemas;
pub(crate) mod shell_exec;
+pub(crate) mod shell_live;
+pub(crate) mod shell_processes;
pub(crate) mod shell_progress;
pub mod types;
diff --git a/crates/openhuman-core/src/sandbox/shell_exec.rs b/crates/openhuman-core/src/sandbox/shell_exec.rs
index b2c95abf49..5e1e2dd47c 100644
--- a/crates/openhuman-core/src/sandbox/shell_exec.rs
+++ b/crates/openhuman-core/src/sandbox/shell_exec.rs
@@ -10,6 +10,7 @@
use std::{collections::BTreeMap, path::PathBuf, time::Duration};
+use super::shell_live::LiveShell;
use super::types::SandboxExecResult;
use crate::config::Config;
@@ -44,6 +45,10 @@ pub(crate) struct NativeShellRun {
pub(crate) isolate_network: bool,
/// Receives the command's output as it streams, for progress reporting.
pub(crate) meter: Option<super::shell_progress::ShellOutputMeter>,
+ /// Return with the process still running once it has run this long,
+ /// instead of waiting for it (the shell tool's yield point). `None`
+ /// waits for the command to end or its runtime budget to stop it.
+ pub(crate) yield_after: Option<Duration>,
/// Native confinement; `None` is ordinary local execution.
#[cfg(feature = "modules")]
pub(crate) jail: Option<contract::NativeJailConfig>,
@@ -69,6 +74,7 @@ impl NativeShellRun {
hide_window: false,
isolate_network: false,
meter: None,
+ yield_after: None,
#[cfg(feature = "modules")]
jail: None,
}
@@ -94,14 +100,38 @@ impl NativeShellRun {
pub(crate) static USE_FIXTURE: std::sync::atomic::AtomicBool =
std::sync::atomic::AtomicBool::new(true);
-/// Run `run` through the module and collect its output.
-///
-/// A worker owns cancellation and resource cleanup if the caller drops its wait.
+/// How a run came back: the command ended, or it was still running at its
+/// yield point and is handed over to be followed.
+pub(crate) enum ShellRunOutcome {
+ Finished(SandboxExecResult),
+ Running(LiveShell),
+}
+
+/// Run `run` through the module and collect its output, waiting for the
+/// command to end (any yield point is ignored).
pub(crate) async fn run(
ctx: &crate::core::runtime::DispatchCtx,
config: &Config,
mut run: NativeShellRun,
) -> anyhow::Result<SandboxExecResult> {
+ run.yield_after = None;
+ match run_yielding(ctx, config, run).await? {
+ ShellRunOutcome::Finished(result) => Ok(result),
+ ShellRunOutcome::Running(_) => {
+ anyhow::bail!("a shell run without a yield point returned while running")
+ }
+ }
+}
+
+/// Run `run` through the module. With a yield point, a command still running
+/// there comes back as [`ShellRunOutcome::Running`] and keeps running.
+///
+/// A worker owns cancellation and resource cleanup if the caller drops its wait.
+pub(crate) async fn run_yielding(
+ ctx: &crate::core::runtime::DispatchCtx,
+ config: &Config,
+ mut run: NativeShellRun,
+) -> anyhow::Result<ShellRunOutcome> {
#[cfg(test)]
if !run.is_jailed() && USE_FIXTURE.load(std::sync::atomic::Ordering::SeqCst) {
let _ = (ctx, config);
@@ -115,6 +145,7 @@ pub(crate) async fn run(
tracing::debug!(
jailed = run.jail.is_some(),
timeout_ms = run.timeout.as_millis() as u64,
+ yield_after_ms = run.yield_after.map(|after| after.as_millis() as u64),
"[sandbox:shell] starting native shell through TinyBox"
);
// A cancelled turn drops this wait mid-command. The worker then cancels
@@ -143,11 +174,17 @@ async fn run_owned(
run: NativeShellRun,
abandoned: tokio::sync::oneshot::Receiver<std::convert::Infallible>,
meter: Option<super::shell_progress::ShellOutputMeter>,
-) -> anyhow::Result<SandboxExecResult> {
+) -> anyhow::Result<ShellRunOutcome> {
let resource = tinybox::reserve(config, contract::ReserveRequest::Resource)
.await
.map_err(|_| anyhow::anyhow!("TinyBox could not reserve a command resource"))?;
let timeout = run.timeout;
+ // A yield point only counts when it comes before the command's own
+ // deadline; otherwise the run ends the usual way.
+ let yield_at = run
+ .yield_after
+ .filter(|after| *after < timeout)
+ .map(|after| tokio::time::Instant::now() + after);
let result = async {
tinybox::create(
config,
@@ -178,9 +215,30 @@ async fn run_owned(
let process = tinybox::start_native_shell_exec(config, request)
.await
.map_err(|_| anyhow::anyhow!("TinyBox could not start the command process"))?;
- collect_process(config, process, timeout, abandoned, meter).await
+ collect_process(config, process, timeout, yield_at, abandoned, meter).await
}
.await;
+ // Still running at its yield point: the process and its resource go to
+ // the caller, who releases them once it has stopped.
+ let result = match result {
+ Ok(Collected::Running {
+ process,
+ cursor,
+ stdout,
+ stderr,
+ }) => {
+ tracing::debug!("[sandbox:shell] command still running at its yield point");
+ return Ok(ShellRunOutcome::Running(LiveShell {
+ stdout: String::from_utf8_lossy(&stdout).into_owned(),
+ stderr: String::from_utf8_lossy(&stderr).into_owned(),
+ handle: super::shell_live::LiveHandle::TinyBox(
+ super::shell_live::TinyBoxLive::new(config.clone(), resource, process, cursor),
+ ),
+ }));
+ }
+ Ok(Collected::Done(output)) => Ok(ShellRunOutcome::Finished(output)),
+ Err(error) => Err(error),
+ };
let close = tinybox::close(config, resource).await;
match (result, close) {
(Ok(output), Ok(())) => Ok(output),
@@ -222,14 +280,28 @@ pub(crate) fn shell_exec_request(
})
}
+/// What [`collect_process`] read: the whole run, or the run so far when it was
+/// still going at its yield point.
+#[cfg(feature = "modules")]
+enum Collected {
+ Done(SandboxExecResult),
+ Running {
+ process: contract::ProcessRef,
+ cursor: u64,
+ stdout: Vec<u8>,
+ stderr: Vec<u8>,
+ },
+}
+
#[cfg(feature = "modules")]
async fn collect_process(
config: &Config,
process: contract::ProcessRef,
timeout: Duration,
+ yield_at: Option<tokio::time::Instant>,
mut abandoned: tokio::sync::oneshot::Receiver<std::convert::Infallible>,
meter: Option<super::shell_progress::ShellOutputMeter>,
-) -> anyhow::Result<SandboxExecResult> {
+) -> anyhow::Result<Collected> {
use contract::{ExecutionState, OutputStream};
use tokio::sync::oneshot::error::TryRecvError;
@@ -270,18 +342,29 @@ async fn collect_process(
}
cursor = batch.next_sequence;
match state {
- ExecutionState::Running => tokio::time::sleep(Duration::from_millis(20)).await,
+ ExecutionState::Running => {
+ if !requested_cancel && yield_at.is_some_and(|at| tokio::time::Instant::now() >= at)
+ {
+ return Ok(Collected::Running {
+ process,
+ cursor,
+ stdout,
+ stderr,
+ });
+ }
+ tokio::time::sleep(Duration::from_millis(20)).await
+ }
ExecutionState::Exited { .. } if has_chunks => continue,
ExecutionState::Exited { exit_code } => {
tinybox::release_output(config, process)
.await
.map_err(|_| anyhow::anyhow!("TinyBox could not release command output"))?;
- return Ok(SandboxExecResult {
+ return Ok(Collected::Done(SandboxExecResult {
exit_code,
stdout: String::from_utf8_lossy(&stdout).into_owned(),
stderr: String::from_utf8_lossy(&stderr).into_owned(),
timed_out: false,
- });
+ }));
}
ExecutionState::Cancelled if has_chunks => continue,
ExecutionState::Cancelled => break requested_cancel,
@@ -299,12 +382,12 @@ async fn collect_process(
// or test killed at its deadline has usually written its progress and
// first errors by then. `timed_out` says why it stopped; the shell tool
// writes the timeout line.
- Ok(SandboxExecResult {
+ Ok(Collected::Done(SandboxExecResult {
exit_code: -1,
stdout: String::from_utf8_lossy(&stdout).into_owned(),
stderr: String::from_utf8_lossy(&stderr).into_owned(),
timed_out,
- })
+ }))
}
/// What a terminal TinyBox failure means for the command. The module enforces
@@ -313,7 +396,7 @@ async fn collect_process(
/// the command's timeout, reported like one (`Ok(true)`). Anything else is a
/// real sandbox failure, named by its code.
#[cfg(feature = "modules")]
-fn failure_timed_out(code: contract::ExecutionFailure) -> anyhow::Result<bool> {
+pub(super) fn failure_timed_out(code: contract::ExecutionFailure) -> anyhow::Result<bool> {
use contract::ExecutionFailure;
match code {
ExecutionFailure::TimedOut => Ok(true),
diff --git a/crates/openhuman-core/src/sandbox/shell_exec_fixture.rs b/crates/openhuman-core/src/sandbox/shell_exec_fixture.rs
index 6c498619fc..c4623b13ce 100644
--- a/crates/openhuman-core/src/sandbox/shell_exec_fixture.rs
+++ b/crates/openhuman-core/src/sandbox/shell_exec_fixture.rs
@@ -10,14 +10,15 @@ use std::time::Duration;
use tokio::io::{AsyncRead, AsyncReadExt};
-use super::{NativeShellRun, SandboxExecResult};
+use super::{NativeShellRun, SandboxExecResult, ShellRunOutcome};
+use crate::sandbox::shell_live::{LiveHandle, LiveRead, LiveShell, LiveState};
/// Variables the module's safe environment snapshot carries.
const SNAPSHOT: &[&str] = &[
"PATH", "HOME", "TERM", "LANG", "LC_ALL", "LC_CTYPE", "USER", "SHELL", "TMPDIR",
];
-pub(crate) async fn run(run: NativeShellRun) -> anyhow::Result<SandboxExecResult> {
+pub(crate) async fn run(run: NativeShellRun) -> anyhow::Result<ShellRunOutcome> {
let bash = ["/usr/bin/bash", "/bin/bash"]
.into_iter()
.find(|path| std::path::Path::new(path).exists());
@@ -73,7 +74,11 @@ pub(crate) async fn run(run: NativeShellRun) -> anyhow::Result<SandboxExecResult
// command stopped at its deadline keeps what it printed before.
let (mut stdout_pipe, mut stderr_pipe) = (child.stdout.take(), child.stderr.take());
let (mut stdout, mut stderr) = (Vec::new(), Vec::new());
- let finished = tokio::time::timeout(run.timeout, async {
+ let started = tokio::time::Instant::now();
+ // Like the module: a command still running at its yield point is handed
+ // over running, with what it printed so far.
+ let yield_after = run.yield_after.filter(|after| *after < run.timeout);
+ let finished = tokio::time::timeout(yield_after.unwrap_or(run.timeout), async {
let (status, (), ()) = tokio::join!(
child.wait(),
drain(&mut stdout_pipe, &mut stdout),
@@ -84,6 +89,22 @@ pub(crate) async fn run(run: NativeShellRun) -> anyhow::Result<SandboxExecResult
.await;
let (exit_code, timed_out) = match finished {
Ok(status) => (status?.code().unwrap_or(-1), false),
+ Err(_) if yield_after.is_some() => {
+ return Ok(ShellRunOutcome::Running(LiveShell {
+ stdout: String::from_utf8_lossy(&stdout).into_owned(),
+ stderr: String::from_utf8_lossy(&stderr).into_owned(),
+ handle: LiveHandle::Fixture(FixtureLive {
+ child,
+ pid,
+ stdout_pipe,
+ stderr_pipe,
+ stdout: Vec::new(),
+ stderr: Vec::new(),
+ deadline: started + run.timeout,
+ ended: false,
+ }),
+ }));
+ }
Err(_) => {
// Stop the command and everything it started, then give the
// readers a moment to drain what is left in the pipes.
@@ -107,12 +128,98 @@ pub(crate) async fn run(run: NativeShellRun) -> anyhow::Result<SandboxExecResult
meter.record(&stdout);
meter.record(&stderr);
}
- Ok(SandboxExecResult {
+ Ok(ShellRunOutcome::Finished(SandboxExecResult {
exit_code,
stdout: String::from_utf8_lossy(&stdout).into_owned(),
stderr: String::from_utf8_lossy(&stderr).into_owned(),
timed_out,
- })
+ }))
+}
+
+/// A fixture command past its yield point, read and stopped like the module's.
+pub(crate) struct FixtureLive {
+ child: tokio::process::Child,
+ pid: Option<u32>,
+ stdout_pipe: Option<tokio::process::ChildStdout>,
+ stderr_pipe: Option<tokio::process::ChildStderr>,
+ /// Output read since the previous [`LiveRead`].
+ stdout: Vec<u8>,
+ stderr: Vec<u8>,
+ deadline: tokio::time::Instant,
+ ended: bool,
+}
+
+impl FixtureLive {
+ pub(crate) async fn read(&mut self, wait: Duration) -> anyhow::Result<LiveRead> {
+ if self.ended {
+ return Ok(self.take(LiveState::Stopped));
+ }
+ let until = (tokio::time::Instant::now() + wait).min(self.deadline);
+ let finished = tokio::time::timeout_at(until, async {
+ let (status, (), ()) = tokio::join!(
+ self.child.wait(),
+ drain(&mut self.stdout_pipe, &mut self.stdout),
+ drain(&mut self.stderr_pipe, &mut self.stderr),
+ );
+ status
+ })
+ .await;
+ let state = match finished {
+ Ok(status) => {
+ self.ended = true;
+ LiveState::Exited(status?.code().unwrap_or(-1))
+ }
+ Err(_) if tokio::time::Instant::now() >= self.deadline => {
+ self.stop().await;
+ LiveState::TimedOut
+ }
+ Err(_) => LiveState::Running,
+ };
+ Ok(self.take(state))
+ }
+
+ pub(crate) async fn kill(&mut self) -> anyhow::Result<LiveRead> {
+ if !self.ended {
+ self.stop().await;
+ }
+ Ok(self.take(LiveState::Stopped))
+ }
+
+ /// Stop the command and everything it started, keeping what is left in
+ /// the pipes.
+ async fn stop(&mut self) {
+ if let Some(pid) = self.pid {
+ tinytools_std::timeout::kill_process_group(pid);
+ }
+ let _ = self.child.start_kill();
+ let _ = tokio::time::timeout(Duration::from_secs(2), async {
+ tokio::join!(
+ drain(&mut self.stdout_pipe, &mut self.stdout),
+ drain(&mut self.stderr_pipe, &mut self.stderr),
+ )
+ })
+ .await;
+ self.ended = true;
+ }
+
+ fn take(&mut self, state: LiveState) -> LiveRead {
+ LiveRead {
+ stdout: String::from_utf8_lossy(&std::mem::take(&mut self.stdout)).into_owned(),
+ stderr: String::from_utf8_lossy(&std::mem::take(&mut self.stderr)).into_owned(),
+ state,
+ }
+ }
+}
+
+impl Drop for FixtureLive {
+ fn drop(&mut self) {
+ if !self.ended {
+ if let Some(pid) = self.pid {
+ tinytools_std::timeout::kill_process_group(pid);
+ }
+ let _ = self.child.start_kill();
+ }
+ }
}
/// Appends what `pipe` delivers to `into` until it closes. Reads are
diff --git a/crates/openhuman-core/src/sandbox/shell_live.rs b/crates/openhuman-core/src/sandbox/shell_live.rs
new file mode 100644
index 0000000000..672aac3088
--- /dev/null
+++ b/crates/openhuman-core/src/sandbox/shell_live.rs
@@ -0,0 +1,227 @@
+//! A shell command still running after its call returned.
+//!
+//! The `shell` tool returns at its yield point instead of waiting out the
+//! command, and leaves the process running. [`LiveShell`] is what it hands over:
+//! the output read so far and a [`LiveHandle`] that reads more, stops the
+//! process, and releases it. TinyBox already keeps a process running and its
+//! output journalled whether or not the host reads it; this only stops the host
+//! from cancelling it when the call returns.
+//!
+//! A handle that is dropped while its process still runs (the turn ended)
+//! stops the process and releases its resource in the background.
+
+use std::time::Duration;
+
+#[cfg(feature = "modules")]
+use crate::modules::tinybox::{self, contract};
+
+/// Pause between output reads while a process runs.
+#[cfg(feature = "modules")]
+const READ_INTERVAL: Duration = Duration::from_millis(50);
+
+/// How long [`LiveHandle::kill`] waits for the stopped process to report.
+const KILL_GRACE: Duration = Duration::from_secs(10);
+
+/// The output a process wrote so far, at its yield point.
+pub(crate) struct LiveShell {
+ pub(crate) stdout: String,
+ pub(crate) stderr: String,
+ pub(crate) handle: LiveHandle,
+}
+
+/// Where a process is after a read.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub(crate) enum LiveState {
+ /// Still running.
+ Running,
+ /// Exited on its own with this code.
+ Exited(i32),
+ /// Stopped at its runtime budget (`timeout_secs`, clamped to the turn).
+ TimedOut,
+ /// Stopped by the model or the host.
+ Stopped,
+}
+
+/// Output written since the previous read, and where the process is now.
+pub(crate) struct LiveRead {
+ pub(crate) stdout: String,
+ pub(crate) stderr: String,
+ pub(crate) state: LiveState,
+}
+
+/// A running process the host can read, stop and release.
+pub(crate) enum LiveHandle {
+ #[cfg(feature = "modules")]
+ TinyBox(TinyBoxLive),
+ #[cfg(test)]
+ Fixture(super::shell_exec::fixture::FixtureLive),
+}
+
+impl LiveHandle {
+ /// Read until the process ends or `wait` passes, whichever is first.
+ pub(crate) async fn read(&mut self, wait: Duration) -> anyhow::Result<LiveRead> {
+ match self {
+ #[cfg(feature = "modules")]
+ Self::TinyBox(live) => live.read(wait).await,
+ #[cfg(test)]
+ Self::Fixture(live) => live.read(wait).await,
+ }
+ }
+
+ /// Stop the process and return what it wrote since the previous read.
+ pub(crate) async fn kill(&mut self) -> anyhow::Result<LiveRead> {
+ match self {
+ #[cfg(feature = "modules")]
+ Self::TinyBox(live) => live.kill().await,
+ #[cfg(test)]
+ Self::Fixture(live) => live.kill().await,
+ }
+ }
+}
+
+/// A TinyBox native shell process past its yield point. Owns the command's
+/// resource until the process has stopped and its output is released.
+#[cfg(feature = "modules")]
+pub(crate) struct TinyBoxLive {
+ config: crate::config::Config,
+ resource: contract::ResourceId,
+ process: contract::ProcessRef,
+ cursor: u64,
+ released: bool,
+}
+
+#[cfg(feature = "modules")]
+impl TinyBoxLive {
+ pub(crate) fn new(
+ config: crate::config::Config,
+ resource: contract::ResourceId,
+ process: contract::ProcessRef,
+ cursor: u64,
+ ) -> Self {
+ Self {
+ config,
+ resource,
+ process,
+ cursor,
+ released: false,
+ }
+ }
+
+ async fn read(&mut self, wait: Duration) -> anyhow::Result<LiveRead> {
+ let until = tokio::time::Instant::now() + wait;
+ self.read_until(until).await
+ }
+
+ async fn kill(&mut self) -> anyhow::Result<LiveRead> {
+ if !self.released {
+ tracing::debug!("[sandbox:shell] stopping a running process on request");
+ // A process that ended on its own refuses the cancel; the read
+ // below reports how it ended either way.
+ let _ = tinybox::cancel(&self.config, self.process.clone()).await;
+ }
+ let mut read = self
+ .read_until(tokio::time::Instant::now() + KILL_GRACE)
+ .await?;
+ if read.state == LiveState::Running {
+ read.state = LiveState::Stopped;
+ }
+ Ok(read)
+ }
+
+ async fn read_until(&mut self, until: tokio::time::Instant) -> anyhow::Result<LiveRead> {
+ use contract::{ExecutionState, OutputStream};
+ let (mut stdout, mut stderr) = (Vec::new(), Vec::new());
+ let done = |stdout: &[u8], stderr: &[u8], state| LiveRead {
+ stdout: String::from_utf8_lossy(stdout).into_owned(),
+ stderr: String::from_utf8_lossy(stderr).into_owned(),
+ state,
+ };
+ if self.released {
+ return Ok(done(&stdout, &stderr, LiveState::Stopped));
+ }
+ loop {
+ let batch = tinybox::read_output(&self.config, self.process.clone(), self.cursor)
+ .await
+ .map_err(|_| anyhow::anyhow!("TinyBox could not read command output"))?;
+ let has_chunks = !batch.chunks.is_empty();
+ for chunk in batch.chunks {
+ match chunk.stream {
+ OutputStream::Stdout => stdout.extend_from_slice(&chunk.bytes),
+ OutputStream::Stderr => stderr.extend_from_slice(&chunk.bytes),
+ }
+ }
+ self.cursor = batch.next_sequence;
+ let state = match batch.state {
+ ExecutionState::Running => {
+ if tokio::time::Instant::now() >= until {
+ return Ok(done(&stdout, &stderr, LiveState::Running));
+ }
+ tokio::time::sleep(READ_INTERVAL).await;
+ continue;
+ }
+ _ if has_chunks => continue,
+ ExecutionState::Exited { exit_code } => LiveState::Exited(exit_code),
+ ExecutionState::Cancelled => LiveState::Stopped,
+ ExecutionState::Failed { code } => {
+ tracing::debug!(?code, "[sandbox:shell] a running process failed");
+ match super::shell_exec::failure_timed_out(code) {
+ Ok(true) => LiveState::TimedOut,
+ Ok(false) => LiveState::Stopped,
+ Err(error) => {
+ self.release().await;
+ return Err(error);
+ }
+ }
+ }
+ };
+ self.release().await;
+ return Ok(done(&stdout, &stderr, state));
+ }
+ }
+
+ /// Release the stopped process's output and close its resource. Once.
+ async fn release(&mut self) {
+ if self.released {
+ return;
+ }
+ self.released = true;
+ let _ = tinybox::release_output(&self.config, self.process.clone()).await;
+ if tinybox::close(&self.config, self.resource.clone())
+ .await
+ .is_err()
+ {
+ tracing::warn!("[sandbox:shell] TinyBox did not acknowledge a process cleanup");
+ }
+ }
+}
+
+#[cfg(feature = "modules")]
+impl Drop for TinyBoxLive {
+ fn drop(&mut self) {
+ if self.released {
+ return;
+ }
+ let Ok(runtime) = tokio::runtime::Handle::try_current() else {
+ tracing::warn!("[sandbox:shell] no runtime to stop a running process at turn end");
+ return;
+ };
+ let (config, resource, process) = (
+ self.config.clone(),
+ self.resource.clone(),
+ self.process.clone(),
+ );
+ tracing::debug!("[sandbox:shell] turn ended: stopping a process it left running");
+ runtime.spawn(async move {
+ let _ = tinybox::cancel(&config, process.clone()).await;
+ let deadline = tokio::time::Instant::now() + KILL_GRACE;
+ while tokio::time::Instant::now() < deadline {
+ match tinybox::is_running(&config, process.clone()).await {
+ Ok(true) => tokio::time::sleep(Duration::from_millis(100)).await,
+ _ => break,
+ }
+ }
+ let _ = tinybox::release_output(&config, process).await;
+ let _ = tinybox::close(&config, resource).await;
+ });
+ }
+}
diff --git a/crates/openhuman-core/src/sandbox/shell_processes.rs b/crates/openhuman-core/src/sandbox/shell_processes.rs
new file mode 100644
index 0000000000..d9604f595d
--- /dev/null
+++ b/crates/openhuman-core/src/sandbox/shell_processes.rs
@@ -0,0 +1,241 @@
+//! The shell commands a turn left running past their yield point.
+//!
+//! The `shell` tool adopts a [`LiveShell`] here and gives the model its id
+//! (`p1`, `p2`, …); the `shell_process` tool waits on it, stops it or reports
+//! its status. One registry belongs to one turn's run context, so dropping it
+//! at the end of the turn stops every process still running (each
+//! [`LiveHandle`] stops its own on drop). Every process also stops at its own
+//! runtime budget, which the turn budget already clamps.
+
+use std::collections::BTreeMap;
+use std::sync::atomic::{AtomicU32, Ordering};
+use std::sync::{Arc, Mutex};
+use std::time::{Duration, Instant};
+
+use super::shell_live::{LiveHandle, LiveShell, LiveState};
+
+/// Processes a turn may leave running at once. A further command runs to its
+/// end the usual way.
+pub(crate) const MAX_RUNNING: usize = 4;
+
+/// Consecutive waits with no new output after which the report suggests
+/// stopping the process.
+pub(crate) const QUIET_WAITS_NOTE: u32 = 3;
+
+/// How a process ended, kept for later questions about it.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub(crate) enum Ended {
+ Exited(i32),
+ TimedOut,
+ Stopped,
+}
+
+/// What a wait, stop or status call found.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub(crate) enum ReportState {
+ /// Still running.
+ Running,
+ /// Ended during this call.
+ Ended(Ended),
+ /// Had already ended, and its output was already returned.
+ AlreadyEnded(Ended),
+}
+
+/// One answer about a process, for the tool to render.
+#[derive(Clone, Debug)]
+pub(crate) struct Report {
+ pub(crate) id: String,
+ pub(crate) command: String,
+ /// Time since the command started.
+ pub(crate) elapsed: Duration,
+ /// The process's runtime budget.
+ pub(crate) timeout: Duration,
+ /// Output written since the previous answer.
+ pub(crate) stdout: String,
+ pub(crate) stderr: String,
+ pub(crate) state: ReportState,
+ /// Consecutive waits, this one included, that brought no new output.
+ pub(crate) quiet_waits: u32,
+}
+
+struct Entry {
+ handle: LiveHandle,
+ command: String,
+ started: Instant,
+ timeout: Duration,
+ ended: Option<Ended>,
+ quiet_waits: u32,
+}
+
+/// The running processes of one turn.
+#[derive(Default)]
+pub(crate) struct ShellProcesses {
+ next: AtomicU32,
+ entries: Mutex<BTreeMap<String, Arc<tokio::sync::Mutex<Entry>>>>,
+ running: Mutex<usize>,
+}
+
+/// A registry is equal only to itself: two turns never share one.
+impl PartialEq for ShellProcesses {
+ fn eq(&self, other: &Self) -> bool {
+ std::ptr::eq(self, other)
+ }
+}
+
+impl Eq for ShellProcesses {}
+
+impl ShellProcesses {
+ pub(crate) fn new() -> Arc<Self> {
+ Arc::new(Self::default())
+ }
+
+ /// Whether another process may be left running.
+ pub(crate) fn has_room(&self) -> bool {
+ *self.running() < MAX_RUNNING
+ }
+
+ /// Take over a process still running at its yield point and name it.
+ pub(crate) fn adopt(
+ &self,
+ live: LiveShell,
+ command: &str,
+ started: Instant,
+ timeout: Duration,
+ ) -> String {
+ let id = format!("p{}", self.next.fetch_add(1, Ordering::SeqCst) + 1);
+ *self.running() += 1;
+ self.entries().insert(
+ id.clone(),
+ Arc::new(tokio::sync::Mutex::new(Entry {
+ handle: live.handle,
+ command: command.to_owned(),
+ started,
+ timeout,
+ ended: None,
+ quiet_waits: 0,
+ })),
+ );
+ tracing::debug!(process = %id, "[shell] process left running past its yield point");
+ id
+ }
+
+ /// Wait up to `wait` for new output or the end of the process.
+ pub(crate) async fn wait(&self, id: &str, wait: Duration) -> Result<Report, String> {
+ let entry = self.entry(id)?;
+ let mut entry = entry.lock().await;
+ if let Some(ended) = entry.ended {
+ return Ok(report(
+ id,
+ &entry,
+ String::new(),
+ String::new(),
+ ReportState::AlreadyEnded(ended),
+ ));
+ }
+ let read = entry
+ .handle
+ .read(wait)
+ .await
+ .map_err(|error| format!("Could not read process {id}: {error}"))?;
+ let quiet = read.stdout.is_empty() && read.stderr.is_empty();
+ entry.quiet_waits = if quiet { entry.quiet_waits + 1 } else { 0 };
+ let state = self.settle(&mut entry, read.state);
+ Ok(report(id, &entry, read.stdout, read.stderr, state))
+ }
+
+ /// Stop the process and return what it wrote since the previous answer.
+ pub(crate) async fn kill(&self, id: &str) -> Result<Report, String> {
+ let entry = self.entry(id)?;
+ let mut entry = entry.lock().await;
+ if let Some(ended) = entry.ended {
+ return Ok(report(
+ id,
+ &entry,
+ String::new(),
+ String::new(),
+ ReportState::AlreadyEnded(ended),
+ ));
+ }
+ tracing::debug!(process = %id, "[shell] stopping a running process on request");
+ let read = entry
+ .handle
+ .kill()
+ .await
+ .map_err(|error| format!("Could not stop process {id}: {error}"))?;
+ let state = self.settle(&mut entry, read.state);
+ Ok(report(id, &entry, read.stdout, read.stderr, state))
+ }
+
+ /// Where the process is, without its output.
+ pub(crate) async fn status(&self, id: &str) -> Result<Report, String> {
+ let entry = self.entry(id)?;
+ let entry = entry.lock().await;
+ let state = match entry.ended {
+ Some(ended) => ReportState::AlreadyEnded(ended),
+ None => ReportState::Running,
+ };
+ Ok(report(id, &entry, String::new(), String::new(), state))
+ }
+
+ /// Record an ending, once.
+ fn settle(&self, entry: &mut Entry, state: LiveState) -> ReportState {
+ let ended = match state {
+ LiveState::Running => return ReportState::Running,
+ LiveState::Exited(code) => Ended::Exited(code),
+ LiveState::TimedOut => Ended::TimedOut,
+ LiveState::Stopped => Ended::Stopped,
+ };
+ if entry.ended.is_none() {
+ entry.ended = Some(ended);
+ let mut running = self.running();
+ *running = running.saturating_sub(1);
+ }
+ ReportState::Ended(ended)
+ }
+
+ fn entry(&self, id: &str) -> Result<Arc<tokio::sync::Mutex<Entry>>, String> {
+ let entries = self.entries();
+ entries.get(id.trim()).cloned().ok_or_else(|| {
+ let known: Vec<&str> = entries.keys().map(String::as_str).collect();
+ if known.is_empty() {
+ format!("No process {id} in this turn: no command has been left running.")
+ } else {
+ format!(
+ "No process {id} in this turn. Processes: {}.",
+ known.join(", ")
+ )
+ }
+ })
+ }
+
+ fn entries(
+ &self,
+ ) -> std::sync::MutexGuard<'_, BTreeMap<String, Arc<tokio::sync::Mutex<Entry>>>> {
+ self.entries
+ .lock()
+ .unwrap_or_else(std::sync::PoisonError::into_inner)
+ }
+
+ fn running(&self) -> std::sync::MutexGuard<'_, usize> {
+ self.running
+ .lock()
+ .unwrap_or_else(std::sync::PoisonError::into_inner)
+ }
+}
+
+fn report(id: &str, entry: &Entry, stdout: String, stderr: String, state: ReportState) -> Report {
+ Report {
+ id: id.to_owned(),
+ command: entry.command.clone(),
+ elapsed: entry.started.elapsed(),
+ timeout: entry.timeout,
+ stdout,
+ stderr,
+ state,
+ quiet_waits: entry.quiet_waits,
+ }
+}
+
+#[cfg(test)]
+#[path = "shell_processes_tests.rs"]
+mod tests;
diff --git a/crates/openhuman-core/src/sandbox/shell_processes_tests.rs b/crates/openhuman-core/src/sandbox/shell_processes_tests.rs
new file mode 100644
index 0000000000..32d46e5660
--- /dev/null
+++ b/crates/openhuman-core/src/sandbox/shell_processes_tests.rs
@@ -0,0 +1,157 @@
+//! The per-turn registry of running shell commands, driven through the test
+//! stand-in for the native shell.
+
+use super::*;
+use crate::sandbox::shell_exec::{run_yielding, NativeShellRun, ShellRunOutcome};
+
+async fn start(command: &str, yield_after_ms: u64, timeout_secs: u64) -> ShellRunOutcome {
+ let dir = std::env::temp_dir();
+ let mut run = NativeShellRun::new(command, &dir, &dir, Duration::from_secs(timeout_secs));
+ run.yield_after = Some(Duration::from_millis(yield_after_ms));
+ run_yielding(
+ &crate::core::runtime::DispatchCtx::host_default(),
+ &crate::config::Config::default(),
+ run,
+ )
+ .await
+ .expect("the stand-in runs the command")
+}
+
+async fn adopted(
+ processes: &ShellProcesses,
+ command: &str,
+ yield_after_ms: u64,
+ timeout_secs: u64,
+) -> (String, String) {
+ match start(command, yield_after_ms, timeout_secs).await {
+ ShellRunOutcome::Running(live) => {
+ let printed = live.stdout.clone();
+ let id = processes.adopt(
+ live,
+ command,
+ Instant::now(),
+ Duration::from_secs(timeout_secs),
+ );
+ (id, printed)
+ }
+ ShellRunOutcome::Finished(result) => panic!("expected a running process, got {result:?}"),
+ }
+}
+
+#[cfg(not(windows))]
+#[tokio::test]
+async fn a_command_that_ends_before_its_yield_point_finishes_as_usual() {
+ match start("printf done", 5_000, 30).await {
+ ShellRunOutcome::Finished(result) => assert_eq!(result.stdout, "done"),
+ ShellRunOutcome::Running(_) => panic!("a quick command must not be left running"),
+ }
+}
+
+#[cfg(not(windows))]
+#[tokio::test]
+async fn a_process_past_its_yield_point_is_followed_to_its_end() {
+ let processes = ShellProcesses::new();
+ let (id, printed) = adopted(&processes, "echo first; sleep 1; echo second", 300, 30).await;
+ assert_eq!(id, "p1");
+ assert_eq!(
+ printed.trim(),
+ "first",
+ "the yield carries the output so far"
+ );
+
+ let report = processes.wait(&id, Duration::from_secs(10)).await.unwrap();
+ assert_eq!(report.state, ReportState::Ended(Ended::Exited(0)));
+ assert_eq!(
+ report.stdout.trim(),
+ "second",
+ "only output written since the yield"
+ );
+
+ let again = processes.wait(&id, Duration::from_secs(1)).await.unwrap();
+ assert_eq!(again.state, ReportState::AlreadyEnded(Ended::Exited(0)));
+ assert!(again.stdout.is_empty());
+}
+
+#[cfg(not(windows))]
+#[tokio::test]
+async fn kill_stops_a_running_process() {
+ let processes = ShellProcesses::new();
+ let (id, _) = adopted(&processes, "echo a; sleep 30", 200, 60).await;
+ let report = processes.kill(&id).await.unwrap();
+ assert_eq!(report.state, ReportState::Ended(Ended::Stopped));
+ let status = processes.status(&id).await.unwrap();
+ assert_eq!(status.state, ReportState::AlreadyEnded(Ended::Stopped));
+}
+
+#[cfg(not(windows))]
+#[tokio::test]
+async fn waits_with_nothing_new_are_counted() {
+ let processes = ShellProcesses::new();
+ let (id, _) = adopted(&processes, "sleep 30", 100, 60).await;
+ for expected in 1..=QUIET_WAITS_NOTE {
+ let report = processes
+ .wait(&id, Duration::from_millis(100))
+ .await
+ .unwrap();
+ assert_eq!(report.state, ReportState::Running);
+ assert_eq!(report.quiet_waits, expected);
+ }
+ processes.kill(&id).await.unwrap();
+}
+
+#[cfg(not(windows))]
+#[tokio::test]
+async fn the_runtime_budget_still_stops_a_process() {
+ let processes = ShellProcesses::new();
+ let (id, _) = adopted(&processes, "echo x; sleep 30", 200, 1).await;
+ let report = processes.wait(&id, Duration::from_secs(10)).await.unwrap();
+ assert_eq!(report.state, ReportState::Ended(Ended::TimedOut));
+}
+
+#[cfg(not(windows))]
+#[tokio::test]
+async fn at_most_four_processes_are_left_running() {
+ let processes = ShellProcesses::new();
+ let mut ids = Vec::new();
+ for _ in 0..MAX_RUNNING {
+ assert!(processes.has_room());
+ ids.push(adopted(&processes, "sleep 30", 100, 60).await.0);
+ }
+ assert!(!processes.has_room());
+ processes.kill(&ids[0]).await.unwrap();
+ assert!(processes.has_room(), "a stopped process frees its place");
+ for id in &ids[1..] {
+ processes.kill(id).await.unwrap();
+ }
+}
+
+#[cfg(not(windows))]
+#[tokio::test]
+async fn dropping_the_registry_stops_its_processes() {
+ let processes = ShellProcesses::new();
+ let (_, printed) = adopted(&processes, "echo $$; sleep 30", 300, 60).await;
+ let pid: i32 = printed.trim().parse().expect("the shell printed its pid");
+ drop(processes);
+ let mut gone = false;
+ for _ in 0..100 {
+ if unsafe { libc::kill(pid, 0) } != 0 {
+ gone = true;
+ break;
+ }
+ tokio::time::sleep(Duration::from_millis(20)).await;
+ }
+ assert!(gone, "a process left running outlived its turn");
+}
+
+#[tokio::test]
+async fn an_unknown_process_names_the_known_ones() {
+ let processes = ShellProcesses::new();
+ let empty = processes
+ .wait("p9", Duration::from_millis(1))
+ .await
+ .unwrap_err();
+ assert!(
+ empty.contains("no command has been left running"),
+ "{empty}"
+ );
+}