Skip to content

The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.

feat(shell): Python children write their output unbuffered

sanil-236501a1ea84c4
diff --git a/crates/openhuman-core/src/tools/impl/system/shell.rs b/crates/openhuman-core/src/tools/impl/system/shell.rs
index 39c527234a..1b55fd9c7f 100644
--- a/crates/openhuman-core/src/tools/impl/system/shell.rs
+++ b/crates/openhuman-core/src/tools/impl/system/shell.rs
@@ -1,7 +1,7 @@
use super::shell_description::{command_param_description, shell_description};
#[cfg(feature = "modules")]
use super::shell_support::container_environment;
-use super::shell_support::{command_failure, exec_result, python_utf8_env, tool_timeout_message};
+use super::shell_support::{command_failure, exec_result, tool_timeout_message};
use crate::agent::tinyagents::host::run_context::{
command_environment_for_tool, dispatch_ctx_for_tool,
};
@@ -640,7 +640,7 @@ impl ShellTool {
&self.security.workspace_dir,
effective,
);
- run.env = env;
+ run.env = super::shell_support::with_unbuffered_python(env, command_environment.is_some());
run.python_utf8 = true;
run.hide_window = self.runtime.hide_window();
run.meter = meter;
@@ -708,8 +708,8 @@ impl ShellTool {
);
// Apply the same Git config hardening to local and sandboxed shells.
extra_env.extend(tinytools_std::filesystem::shell_git_env());
- // And the same Python UTF-8 defaults the native path sets.
- for (var, val) in python_utf8_env(|name| {
+ // And the same Python defaults the native path sets.
+ for (var, val) in super::shell_support::python_env_defaults(|name| {
command_environment
.as_ref()
.and_then(|overrides| overrides.get(name).map(std::ffi::OsString::from))
diff --git a/crates/openhuman-core/src/tools/impl/system/shell_support.rs b/crates/openhuman-core/src/tools/impl/system/shell_support.rs
index 71903ba766..be9b189559 100644
--- a/crates/openhuman-core/src/tools/impl/system/shell_support.rs
+++ b/crates/openhuman-core/src/tools/impl/system/shell_support.rs
@@ -94,19 +94,57 @@ fn timed_out_output(secs: u64, result: &crate::sandbox::SandboxExecResult) -> St
out
}
-/// Python's text-encoding defaults for a child, unless the parent sets its own.
-/// Under `cmd.exe` the console code page is a legacy one (cp1252, cp437), so a
-/// script printing anything outside it dies with `UnicodeEncodeError`. Both are
-/// no-ops for a non-Python command. Native runs get the same from TinyBox's
-/// `python_utf8` request flag; this serves the Docker and sandbox-policy paths,
-/// which forward their environment as plain variables.
-pub(super) const PYTHON_UTF8_DEFAULTS: &[(&str, &str)] =
- &[("PYTHONUTF8", "1"), ("PYTHONIOENCODING", "utf-8")];
+/// Python defaults for a child, unless the parent sets its own.
+///
+/// - `PYTHONUTF8`, `PYTHONIOENCODING`: under `cmd.exe` the console code page
+/// is a legacy one (cp1252, cp437), so a script printing anything outside it
+/// dies with `UnicodeEncodeError`. Native runs get these from TinyBox's
+/// `python_utf8` request flag.
+/// - `PYTHONUNBUFFERED`: Python block-buffers stdout into a pipe (about 8 KB),
+/// and a process killed at its deadline never flushes that buffer, so a
+/// script that printed its first failures and then ran long would show
+/// nothing. Any non-empty value turns buffering off; a parent that wants it
+/// on sets it to the empty string.
+///
+/// All are no-ops for a non-Python command. This serves the Docker and
+/// sandbox-policy paths, which forward their environment as plain variables;
+/// the native path adds `PYTHONUNBUFFERED` itself.
+pub(super) const PYTHON_ENV_DEFAULTS: &[(&str, &str)] = &[
+ ("PYTHONUTF8", "1"),
+ ("PYTHONIOENCODING", "utf-8"),
+ (PYTHON_UNBUFFERED, "1"),
+];
+
+/// The variable that turns off Python's output buffering.
+pub(super) const PYTHON_UNBUFFERED: &str = "PYTHONUNBUFFERED";
+
+/// A native run's environment with `PYTHONUNBUFFERED` added, so a script killed
+/// at its deadline has written what it printed (TinyBox's `python_utf8` flag
+/// covers the encoding). A value already in `env` stays. Otherwise the
+/// process's own value is inherited, unless the turn owns its environment
+/// (`turn_environment`), as on the sandbox path; else it is `1`.
+pub(super) fn with_unbuffered_python(
+ mut env: std::collections::BTreeMap<String, String>,
+ turn_environment: bool,
+) -> std::collections::BTreeMap<String, String> {
+ if !env.contains_key(PYTHON_UNBUFFERED) {
+ let inherited = (!turn_environment)
+ .then(|| std::env::var(PYTHON_UNBUFFERED).ok())
+ .flatten();
+ env.insert(
+ PYTHON_UNBUFFERED.to_owned(),
+ inherited.unwrap_or_else(|| "1".to_owned()),
+ );
+ }
+ env
+}
-pub(super) fn python_utf8_env(
+/// The [`PYTHON_ENV_DEFAULTS`] a child gets, each one the parent's own value
+/// when `lookup` has one.
+pub(super) fn python_env_defaults(
lookup: impl Fn(&str) -> Option<std::ffi::OsString>,
) -> Vec<(&'static str, std::ffi::OsString)> {
- PYTHON_UTF8_DEFAULTS
+ PYTHON_ENV_DEFAULTS
.iter()
.map(|(name, default)| {
let value = lookup(name).unwrap_or_else(|| (*default).into());
@@ -140,7 +178,7 @@ pub(super) fn container_environment(
Vec::new()
}
.into_iter()
- .chain(python_utf8_env(|name| {
+ .chain(python_env_defaults(|name| {
explicit
.and_then(|overrides| overrides.get(name).map(std::ffi::OsString::from))
.or_else(|| {
diff --git a/crates/openhuman-core/src/tools/impl/system/shell_tests_platform_tests.rs b/crates/openhuman-core/src/tools/impl/system/shell_tests_platform_tests.rs
index d833a45282..48240f7412 100644
--- a/crates/openhuman-core/src/tools/impl/system/shell_tests_platform_tests.rs
+++ b/crates/openhuman-core/src/tools/impl/system/shell_tests_platform_tests.rs
@@ -7,7 +7,7 @@
use super::*;
use crate::modules::tinybox::host_runtime::{host_shell_flavor, ShellFlavor};
#[cfg(feature = "modules")]
-use crate::tools::system::shell_support::PYTHON_UTF8_DEFAULTS;
+use crate::tools::system::shell_support::{python_env_defaults, PYTHON_ENV_DEFAULTS};
use std::collections::HashMap;
use std::ffi::OsString;
@@ -78,7 +78,7 @@ fn as_map(env: Vec<(&'static str, OsString)>) -> HashMap<&'static str, String> {
#[test]
fn a_users_own_python_encoding_settings_win() {
- let env = as_map(python_utf8_env(parent(&[
+ let env = as_map(python_env_defaults(parent(&[
("PYTHONUTF8", "0"),
("PYTHONIOENCODING", "latin-1"),
])));
@@ -110,7 +110,7 @@ async fn the_container_environment_forwards_only_the_allowlist_plus_python_defau
for key in env.keys() {
assert!(
crate::sandbox::ops::SANDBOX_ENV_PASSTHROUGH.contains(&key.as_str())
- || PYTHON_UTF8_DEFAULTS.iter().any(|(k, _)| k == key),
+ || PYTHON_ENV_DEFAULTS.iter().any(|(k, _)| k == key),
"unexpected {key}"
);
}
@@ -141,12 +141,37 @@ fn the_container_environment_forwards_explicit_turn_variables() {
#[test]
fn sandboxed_calls_get_the_same_python_defaults() {
- let env = as_map(python_utf8_env(parent(&[])));
+ let env = as_map(python_env_defaults(parent(&[])));
assert_eq!(env.get("PYTHONUTF8").map(String::as_str), Some("1"));
assert_eq!(
env.get("PYTHONIOENCODING").map(String::as_str),
Some("utf-8")
);
+ assert_eq!(env.get("PYTHONUNBUFFERED").map(String::as_str), Some("1"));
+}
+
+#[test]
+fn a_parent_can_keep_python_buffering_with_an_empty_value() {
+ let env = as_map(python_env_defaults(parent(&[("PYTHONUNBUFFERED", "")])));
+ assert_eq!(env.get("PYTHONUNBUFFERED").map(String::as_str), Some(""));
+}
+
+#[tokio::test]
+async fn a_spawned_shell_gets_unbuffered_python_output() {
+ // A developer's own setting wins, so expect it when present.
+ let expected = std::env::var("PYTHONUNBUFFERED").unwrap_or_else(|_| "1".to_string());
+ let tool = ShellTool::new(test_security_with_env_cmd(), test_runtime(), test_audit());
+ let command = if cfg!(windows) {
+ "echo [%PYTHONUNBUFFERED%]"
+ } else {
+ "echo [$PYTHONUNBUFFERED]"
+ };
+ let result = tool
+ .execute(serde_json::json!({ "command": command }))
+ .await
+ .unwrap();
+ assert!(!result.is_error, "{}", result.output());
+ assert_eq!(result.output().trim(), format!("[{expected}]"));
}
#[tokio::test]