Skip to content

The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.

feat(shell): timeout_secs is when a call comes back; the turn's deadline kills

sanil-23996027896c45
diff --git a/crates/openhuman-core/src/agent/tinyagents/middleware.rs b/crates/openhuman-core/src/agent/tinyagents/middleware.rs
index 00ffd02bcd..058a491e43 100644
--- a/crates/openhuman-core/src/agent/tinyagents/middleware.rs
+++ b/crates/openhuman-core/src/agent/tinyagents/middleware.rs
@@ -31,6 +31,7 @@ pub(crate) mod fetched_site;
pub(crate) mod memory_pack;
pub(crate) mod packed_tool_route;
pub(crate) mod repeated_failure;
+pub(crate) mod shell_yield;
pub(crate) mod tool_outcome_capture;
pub(crate) mod tool_output;
pub(crate) mod tool_policy;
diff --git a/crates/openhuman-core/src/agent/tinyagents/middleware/shell_yield.rs b/crates/openhuman-core/src/agent/tinyagents/middleware/shell_yield.rs
new file mode 100644
index 0000000000..3f355fe757
--- /dev/null
+++ b/crates/openhuman-core/src/agent/tinyagents/middleware/shell_yield.rs
@@ -0,0 +1,58 @@
+//! A turn's `shell` call: the model's `timeout_secs` is when the call comes
+//! back, not when the command is killed.
+//!
+//! A command still running at that point keeps running as a process the model
+//! follows with `shell_process` (`tools/impl/system/shell.rs`). The kill is the
+//! turn's: TinyAgents' `ShellTurnBudget` clamp writes the turn's remaining
+//! budget into `timeout_secs`, and the shell enforces that as the command's
+//! deadline. So this runs just before the clamp and moves the model's value to
+//! the shell's yield argument, leaving `timeout_secs` to the clamp. Outside a
+//! turn nothing moves it, and `timeout_secs` stays the caller's deadline.
+
+use async_trait::async_trait;
+use tinyagents_harness::context::RunContext;
+use tinyagents_harness::error::Result as TaResult;
+use tinyagents_harness::middleware::Middleware;
+use tinyinference_llm::tool::ToolCall as TaToolCall;
+
+use crate::agent::tinyagents::host::OpenHumanRunContext;
+use crate::tools::implementations::system::YIELD_AFTER_ARG;
+
+/// Moves a `shell` call's `timeout_secs` to its yield argument.
+pub(crate) struct ShellYieldMiddleware;
+
+#[async_trait]
+impl Middleware<(), OpenHumanRunContext> for ShellYieldMiddleware {
+ fn name(&self) -> &str {
+ "shell_timeout_yields"
+ }
+
+ async fn before_tool(
+ &self,
+ _ctx: &mut RunContext<OpenHumanRunContext>,
+ _state: &(),
+ call: &mut TaToolCall,
+ ) -> TaResult<()> {
+ if call.name == "shell" {
+ if let Some(arguments) = call.arguments.as_object_mut() {
+ timeout_to_yield(arguments);
+ }
+ }
+ Ok(())
+ }
+}
+
+/// Move `timeout_secs`, when present, to [`YIELD_AFTER_ARG`].
+pub(crate) fn timeout_to_yield(arguments: &mut serde_json::Map<String, serde_json::Value>) {
+ if let Some(requested) = arguments.remove("timeout_secs") {
+ tracing::debug!(
+ requested = %requested,
+ "[tinyagents::mw] shell_yield: timeout_secs is the call's yield point"
+ );
+ arguments.insert(YIELD_AFTER_ARG.to_owned(), requested);
+ }
+}
+
+#[cfg(test)]
+#[path = "shell_yield_tests.rs"]
+mod tests;
diff --git a/crates/openhuman-core/src/agent/tinyagents/middleware/shell_yield_tests.rs b/crates/openhuman-core/src/agent/tinyagents/middleware/shell_yield_tests.rs
new file mode 100644
index 0000000000..25eb0cffd5
--- /dev/null
+++ b/crates/openhuman-core/src/agent/tinyagents/middleware/shell_yield_tests.rs
@@ -0,0 +1,43 @@
+use super::*;
+use serde_json::json;
+
+fn moved(arguments: serde_json::Value) -> serde_json::Value {
+ let mut arguments = arguments.as_object().cloned().unwrap();
+ timeout_to_yield(&mut arguments);
+ serde_json::Value::Object(arguments)
+}
+
+#[test]
+fn the_models_timeout_becomes_the_yield_point() {
+ assert_eq!(
+ moved(json!({"command": "make", "timeout_secs": 300})),
+ json!({"command": "make", YIELD_AFTER_ARG: 300})
+ );
+}
+
+#[test]
+fn a_call_without_a_timeout_is_left_to_the_default() {
+ assert_eq!(moved(json!({"command": "ls"})), json!({"command": "ls"}));
+}
+
+#[test]
+fn zero_still_means_wait_for_the_end() {
+ assert_eq!(
+ moved(json!({"command": "make", "timeout_secs": 0})),
+ json!({"command": "make", YIELD_AFTER_ARG: 0})
+ );
+}
+
+#[test]
+fn the_turn_clamp_then_sets_the_deadline() {
+ use std::time::Duration;
+ use tinyagents_harness::middleware::clamp_shell_timeout;
+
+ // With the model's value moved aside, the clamp sees a call that asked for
+ // no deadline and gives it the turn's remainder (less its reserve).
+ let arguments = moved(json!({"command": "make", "timeout_secs": 300}));
+ let requested = arguments.get("timeout_secs").and_then(|v| v.as_u64());
+ let clamp = clamp_shell_timeout(requested, Duration::from_secs(600), 3600).unwrap();
+ assert_eq!(clamp.requested, None);
+ assert_eq!(clamp.clamped_secs, 510);
+}
diff --git a/crates/openhuman-core/src/agent/tinyagents/turn_middleware.rs b/crates/openhuman-core/src/agent/tinyagents/turn_middleware.rs
index 64ce426c8a..5e17162db0 100644
--- a/crates/openhuman-core/src/agent/tinyagents/turn_middleware.rs
+++ b/crates/openhuman-core/src/agent/tinyagents/turn_middleware.rs
@@ -620,6 +620,9 @@ pub(super) fn install_turn_middleware(
// validation error. It never reaches approval/policy wrappers or the tool.
harness.push_middleware(Arc::new(ArgRecoveryMiddleware::new(tool_sets.clone())));
+ // The model's shell `timeout_secs` is when the call comes back, not a kill:
+ // move it aside so the clamp below writes the command's deadline there.
+ harness.push_middleware(Arc::new(middleware::shell_yield::ShellYieldMiddleware));
// Clamp shell `timeout_secs` to the turn remainder (#6953), on recovered args.
harness.push_middleware(Arc::new(shell_turn_budget.clamp()));
diff --git a/crates/openhuman-core/src/tools/impl/system/mod.rs b/crates/openhuman-core/src/tools/impl/system/mod.rs
index 55bfda8ecd..e17be2e422 100644
--- a/crates/openhuman-core/src/tools/impl/system/mod.rs
+++ b/crates/openhuman-core/src/tools/impl/system/mod.rs
@@ -28,6 +28,7 @@ pub use retrieve_tool_output::{retrieve_tool_output_tool, RetrieveToolOutputTool
pub use schedule::ScheduleTool;
pub use shell::ShellTool;
pub use shell_process::ShellProcessTool;
+pub(crate) use shell_support::YIELD_AFTER_ARG;
pub use update_apply::UpdateApplyTool;
pub use update_check::UpdateCheckTool;
diff --git a/crates/openhuman-core/src/tools/impl/system/shell.rs b/crates/openhuman-core/src/tools/impl/system/shell.rs
index b79cab80c6..5c32074add 100644
--- a/crates/openhuman-core/src/tools/impl/system/shell.rs
+++ b/crates/openhuman-core/src/tools/impl/system/shell.rs
@@ -193,13 +193,7 @@ impl Tool for ShellTool {
"type": "integer",
"minimum": 0,
"maximum": 3600,
- "description": "Optional wall-clock timeout (seconds, up to 3600) for this command before it is killed. Use a larger value for long-running work (builds, test suites, solvers). Omitted or 0 runs without a deadline."
- },
- "yield_after_secs": {
- "type": "integer",
- "minimum": 0,
- "maximum": 3600,
- "description": "Seconds after which a command that is still running comes back with its output so far and keeps running as a process you follow with `shell_process`. Default 120; 0 waits for the command to end."
+ "description": "Optional: how long (seconds) to wait before coming back. A command still running then returns its output so far and keeps running as a process you follow with `shell_process`. Default 120, at most 600; 0 waits for the command to end. It does not kill the command: put `timeout N` in the command for that."
}
},
"required": ["command"]
@@ -300,12 +294,14 @@ impl ShellTool {
.and_then(|v| v.as_str())
.ok_or_else(|| anyhow::anyhow!("Missing 'command' parameter"))?;
- // Optional per-call wall-clock budget. `None`/`0` ⇒ run unbounded;
- // a positive value is clamped downstream by
- // `tool_timeout::explicit_call_timeout_*`. Shell has no default deadline
- // (issue #4023) — long scripts must run to completion.
+ // The command's deadline: in a turn, what TinyAgents' turn-budget clamp
+ // wrote (the model's own `timeout_secs` is its yield point, moved to
+ // `YIELD_AFTER_ARG` by `middleware::shell_yield`); outside a turn, the
+ // caller's. `None`/`0` ⇒ unbounded (issue #4023).
let requested_timeout = args.get("timeout_secs").and_then(|v| v.as_u64());
- let requested_yield = args.get("yield_after_secs").and_then(|v| v.as_u64());
+ let requested_yield = args
+ .get(super::shell_support::YIELD_AFTER_ARG)
+ .and_then(|v| v.as_u64());
let start = Instant::now();
let (allowed, result) = crate::sandbox::shell_progress::with_shell_progress(
context,
diff --git a/crates/openhuman-core/src/tools/impl/system/shell_description.rs b/crates/openhuman-core/src/tools/impl/system/shell_description.rs
index b5299345cf..8b849bf1ef 100644
--- a/crates/openhuman-core/src/tools/impl/system/shell_description.rs
+++ b/crates/openhuman-core/src/tools/impl/system/shell_description.rs
@@ -21,10 +21,10 @@ pub(super) fn shell_description(flavor: ShellFlavor) -> &'static str {
commands chain with `&&` (`;` is not a separator). For PowerShell run \
`powershell -NoProfile -Command \"...\"`. Only stdout/stderr comes back, so a script \
that computes silently or only writes a file returns nothing — print what you need, \
- or read the file afterwards. A command still running after `yield_after_secs` (default 120) comes back with its output so far and keeps running as a process: follow it with `shell_process` (wait for more, or kill it) instead of starting it again. Don't pipe a long command through `tail`/`head` to shorten it: long output is kept and capped for you, and a pipe holds everything back until the command ends."
+ or read the file afterwards. A command still running after `timeout_secs` (default 120) comes back with its output so far and keeps running as a process: follow it with `shell_process` (wait for more, or kill it) instead of starting it again; it is stopped when the turn's time runs out. `timeout_secs` never kills: to stop a command after a time, put `timeout N` in it. Don't pipe a long command through `tail`/`head` to shorten it: long output is kept and capped for you, and a pipe holds everything back until the command ends."
}
ShellFlavor::Posix => {
- "Execute a shell command: run code, manipulate workspace files, or launch applications (`open -a Music`, `xdg-open music://`). Only stdout/stderr comes back, so a script that computes silently or only writes a file returns nothing — print what you need, or read the file afterwards. A command still running after `yield_after_secs` (default 120) comes back with its output so far and keeps running as a process: follow it with `shell_process` (wait for more, or kill it) instead of starting it again. Don't pipe a long command through `tail`/`head` to shorten it: long output is kept and capped for you, and a pipe holds everything back until the command ends."
+ "Execute a shell command: run code, manipulate workspace files, or launch applications (`open -a Music`, `xdg-open music://`). Only stdout/stderr comes back, so a script that computes silently or only writes a file returns nothing — print what you need, or read the file afterwards. A command still running after `timeout_secs` (default 120) comes back with its output so far and keeps running as a process: follow it with `shell_process` (wait for more, or kill it) instead of starting it again; it is stopped when the turn's time runs out. `timeout_secs` never kills: to stop a command after a time, put `timeout N` in it. Don't pipe a long command through `tail`/`head` to shorten it: long output is kept and capped for you, and a pipe holds everything back until the command ends."
}
}
}
diff --git a/crates/openhuman-core/src/tools/impl/system/shell_support.rs b/crates/openhuman-core/src/tools/impl/system/shell_support.rs
index 1f826c154d..f0db41868f 100644
--- a/crates/openhuman-core/src/tools/impl/system/shell_support.rs
+++ b/crates/openhuman-core/src/tools/impl/system/shell_support.rs
@@ -7,15 +7,16 @@ use tinytools::ToolResult;
pub(super) const COMMAND_TIMEOUT_EXIT_CODE: i32 = 124;
/// Appended to an exit-124 failure: the command's own `timeout` fired, not the
/// tool's (#6953). Without it, `exit code 124 [stdout] ok` reads like a bug.
-pub(super) const COMMAND_TIMEOUT_NOTE: &str = "[exit 124: the command's own `timeout` limit expired; the shell tool's timeout_secs did not fire]";
+pub(super) const COMMAND_TIMEOUT_NOTE: &str =
+ "[exit 124: the command's own `timeout` limit expired; the shell tool's deadline did not fire]";
-/// The result when the tool's own `timeout_secs` deadline killed the command.
-/// It stays the first line of the result: TinyAgents' shell turn-budget note
-/// recognises a fired deadline by this prefix.
+/// The result when the tool's deadline killed the command: in a turn, the
+/// turn's remaining budget (TinyAgents' `ShellTurnBudget` clamp); outside one,
+/// the caller's `timeout_secs`. It stays the first line of the result:
+/// TinyAgents' shell turn-budget note recognises a fired deadline by this
+/// prefix.
pub(super) fn tool_timeout_message(secs: u64) -> String {
- format!(
- "Command timed out after {secs}s and was killed: the shell tool's timeout_secs limit fired."
- )
+ format!("Command timed out after {secs}s and was killed: the shell tool's deadline fired.")
}
/// [`tinytools::command_failure`], plus [`COMMAND_TIMEOUT_NOTE`] on exit 124.
@@ -231,16 +232,39 @@ pub(super) fn extend_explicit_environment(
}
}
+/// The argument a turn's `shell` call carries its yield point in. The model
+/// writes it as `timeout_secs`; the turn moves it here before TinyAgents'
+/// turn-budget clamp writes the command's deadline into `timeout_secs`
+/// (`agent::tinyagents::middleware::shell_yield`).
+pub(crate) const YIELD_AFTER_ARG: &str = "yield_after_secs";
/// When a command still running is handed back as a process, unless the call
-/// says otherwise (`yield_after_secs`).
+/// says otherwise.
pub(super) const DEFAULT_YIELD_AFTER_SECS: u64 = 120;
+/// The latest a call may ask to come back.
+pub(super) const MAX_YIELD_AFTER_SECS: u64 = 600;
+/// The earliest the deadline-relative cap brings the yield point forward to.
+pub(super) const MIN_YIELD_AFTER_SECS: u64 = 30;
-/// The yield point a call asked for: the default when absent, none for `0`.
-pub(super) fn yield_after(requested: Option<u64>) -> Option<Duration> {
- match requested.unwrap_or(DEFAULT_YIELD_AFTER_SECS) {
- 0 => None,
- secs => Some(Duration::from_secs(secs.min(3600))),
+/// The yield point for `command` with `deadline` to run: the requested time
+/// (default 120 s, at most 600 s), but no later than an eighth of the deadline
+/// (and not earlier than 30 s for that cap), so a command started late in a
+/// turn still comes back before the turn's deadline kills it. None for `0`, and
+/// none for a deliberate wait (a command that starts with `sleep`), which would
+/// only come back with nothing to show.
+pub(super) fn yield_after(
+ requested: Option<u64>,
+ deadline: Duration,
+ command: &str,
+) -> Option<Duration> {
+ let secs = requested
+ .unwrap_or(DEFAULT_YIELD_AFTER_SECS)
+ .min(MAX_YIELD_AFTER_SECS);
+ let first = command.split_whitespace().next();
+ if secs == 0 || first == Some("sleep") {
+ return None;
}
+ let cap = (deadline.as_secs() / 8).max(MIN_YIELD_AFTER_SECS);
+ Some(Duration::from_secs(secs.min(cap)))
}
/// The result of a call that came back with its command still running.
@@ -281,7 +305,7 @@ pub(super) async fn run_native(
let processes = crate::agent::tinyagents::host::run_context::shell_processes_for_tool(context)
.filter(|processes| processes.has_room());
if processes.is_some() {
- run.yield_after = yield_after(requested_yield);
+ run.yield_after = yield_after(requested_yield, run.timeout, &run.command);
}
let (command, timeout) = (run.command.clone(), run.timeout);
let started = std::time::Instant::now();
@@ -334,3 +358,7 @@ pub(super) fn human_duration(duration: Duration) -> String {
_ => format!("{}h {}m", secs / 3600, (secs % 3600) / 60),
}
}
+
+#[cfg(test)]
+#[path = "shell_support_tests.rs"]
+mod tests;
diff --git a/crates/openhuman-core/src/tools/impl/system/shell_support_tests.rs b/crates/openhuman-core/src/tools/impl/system/shell_support_tests.rs
new file mode 100644
index 0000000000..e1cd5f65e7
--- /dev/null
+++ b/crates/openhuman-core/src/tools/impl/system/shell_support_tests.rs
@@ -0,0 +1,55 @@
+use super::*;
+
+const LONG: Duration = Duration::from_secs(3600);
+
+#[test]
+fn a_call_comes_back_after_two_minutes_by_default() {
+ assert_eq!(
+ yield_after(None, LONG, "make -j8"),
+ Some(Duration::from_secs(DEFAULT_YIELD_AFTER_SECS))
+ );
+}
+
+#[test]
+fn a_requested_time_is_kept_up_to_ten_minutes() {
+ assert_eq!(
+ yield_after(Some(300), Duration::from_secs(7200), "make"),
+ Some(Duration::from_secs(300))
+ );
+ assert_eq!(
+ yield_after(Some(3600), Duration::from_secs(86_400), "make"),
+ Some(Duration::from_secs(MAX_YIELD_AFTER_SECS))
+ );
+}
+
+#[test]
+fn zero_waits_for_the_end() {
+ assert_eq!(yield_after(Some(0), LONG, "make"), None);
+}
+
+#[test]
+fn a_deliberate_wait_is_not_cut_short() {
+ assert_eq!(yield_after(None, LONG, "sleep 100; tail run.log"), None);
+ assert_eq!(yield_after(Some(30), LONG, " sleep 5"), None);
+ assert!(yield_after(None, LONG, "sleeper --run").is_some());
+ assert!(yield_after(None, LONG, "make; sleep 5").is_some());
+}
+
+#[test]
+fn a_short_deadline_brings_the_yield_point_forward() {
+ // An eighth of the deadline: 800 s left to run comes back at 100 s.
+ assert_eq!(
+ yield_after(None, Duration::from_secs(800), "make"),
+ Some(Duration::from_secs(100))
+ );
+ // Never earlier than 30 s for that cap.
+ assert_eq!(
+ yield_after(None, Duration::from_secs(120), "make"),
+ Some(Duration::from_secs(MIN_YIELD_AFTER_SECS))
+ );
+ // A shorter request still wins.
+ assert_eq!(
+ yield_after(Some(5), Duration::from_secs(120), "make"),
+ Some(Duration::from_secs(5))
+ );
+}
diff --git a/crates/openhuman-core/src/tools/impl/system/shell_tests_keep_running_tests.rs b/crates/openhuman-core/src/tools/impl/system/shell_tests_keep_running_tests.rs
index 698fd4f027..a17e5c3f44 100644
--- a/crates/openhuman-core/src/tools/impl/system/shell_tests_keep_running_tests.rs
+++ b/crates/openhuman-core/src/tools/impl/system/shell_tests_keep_running_tests.rs
@@ -74,6 +74,22 @@ async fn yield_after_zero_waits_for_the_end() {
assert_eq!(result.output().trim(), "done");
}
+#[cfg(not(windows))]
+#[tokio::test]
+async fn a_deliberate_wait_runs_to_its_end() {
+ let run = test_turn_run();
+ let call = ToolExecutionContext::from_run_context(&run, CallId::new("c1"));
+ let result = shell()
+ .execute_with_context(
+ json!({"command": "sleep 2; echo done", "yield_after_secs": 1}),
+ ToolCallOptions::default(),
+ Some(&call),
+ )
+ .await
+ .unwrap();
+ assert_eq!(result.output().trim(), "done");
+}
+
#[cfg(not(windows))]
#[tokio::test]
async fn a_failing_end_reads_like_a_failed_shell_call() {
@@ -81,7 +97,7 @@ async fn a_failing_end_reads_like_a_failed_shell_call() {
let call = ToolExecutionContext::from_run_context(&run, CallId::new("c1"));
let yielded = shell()
.execute_with_context(
- json!({"command": "sleep 2; echo broke >&2; exit 3", "yield_after_secs": 1}),
+ json!({"command": "true; sleep 2; echo broke >&2; exit 3", "yield_after_secs": 1}),
ToolCallOptions::default(),
Some(&call),
)
diff --git a/crates/openhuman-core/src/tools/impl/system/shell_tests_timeout_tests.rs b/crates/openhuman-core/src/tools/impl/system/shell_tests_timeout_tests.rs
index 787c76a0b2..f49abc3e5f 100644
--- a/crates/openhuman-core/src/tools/impl/system/shell_tests_timeout_tests.rs
+++ b/crates/openhuman-core/src/tools/impl/system/shell_tests_timeout_tests.rs
@@ -44,7 +44,7 @@ async fn exit_124_is_attributed_to_the_commands_own_timeout() {
"exit 124 not attributed: {out}"
);
assert!(
- out.contains("the shell tool's timeout_secs did not fire"),
+ out.contains("the shell tool's deadline did not fire"),
"{out}"
);
}
@@ -77,7 +77,7 @@ async fn the_tools_own_deadline_says_it_was_timeout_secs() {
let out = result.output();
assert!(out.contains("timed out after 1s"), "{out}");
assert!(
- out.contains("the shell tool's timeout_secs limit fired"),
+ out.contains("the shell tool's deadline fired"),
"tool timeout not attributed: {out}"
);
}
@@ -172,7 +172,7 @@ fn a_timed_out_result_carries_both_streams_after_the_timeout_line() {
assert!(result.is_error);
assert_eq!(
result.output(),
- "Command timed out after 1331s and was killed: the shell tool's timeout_secs limit fired. \
+ "Command timed out after 1331s and was killed: the shell tool's deadline fired. \
Output before the kill:\n[stdout]\ntested 40 positions\nFAIL: e1d2\n[stderr]\nwarning: slow"
);
}