The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.
Rust agent-orchestrationai-agentsai-assistantdesktopllmlocal-firstmcppersonal-aiprivacyrustsecond-braintauri
feat(shell): Python children write their output unbuffered
sanil-236501a1ea84c4
diff --git a/crates/openhuman-core/src/tools/impl/system/shell.rs b/crates/openhuman-core/src/tools/impl/system/shell.rsindex 39c527234a..1b55fd9c7f 100644--- a/crates/openhuman-core/src/tools/impl/system/shell.rs+++ b/crates/openhuman-core/src/tools/impl/system/shell.rs@@ -1,7 +1,7 @@use super::shell_description::{command_param_description, shell_description};#[cfg(feature = "modules")]use super::shell_support::container_environment;-use super::shell_support::{command_failure, exec_result, python_utf8_env, tool_timeout_message};+use super::shell_support::{command_failure, exec_result, tool_timeout_message};use crate::agent::tinyagents::host::run_context::{command_environment_for_tool, dispatch_ctx_for_tool,};@@ -640,7 +640,7 @@ impl ShellTool {&self.security.workspace_dir,effective,);- run.env = env;+ run.env = super::shell_support::with_unbuffered_python(env, command_environment.is_some());run.python_utf8 = true;run.hide_window = self.runtime.hide_window();run.meter = meter;@@ -708,8 +708,8 @@ impl ShellTool {);// Apply the same Git config hardening to local and sandboxed shells.extra_env.extend(tinytools_std::filesystem::shell_git_env());- // And the same Python UTF-8 defaults the native path sets.- for (var, val) in python_utf8_env(|name| {+ // And the same Python defaults the native path sets.+ for (var, val) in super::shell_support::python_env_defaults(|name| {command_environment.as_ref().and_then(|overrides| overrides.get(name).map(std::ffi::OsString::from))diff --git a/crates/openhuman-core/src/tools/impl/system/shell_support.rs b/crates/openhuman-core/src/tools/impl/system/shell_support.rsindex 71903ba766..be9b189559 100644--- a/crates/openhuman-core/src/tools/impl/system/shell_support.rs+++ b/crates/openhuman-core/src/tools/impl/system/shell_support.rs@@ -94,19 +94,57 @@ fn timed_out_output(secs: u64, result: &crate::sandbox::SandboxExecResult) -> Stout}-/// Python's text-encoding defaults for a child, unless the parent sets its own.-/// Under `cmd.exe` the console code page is a legacy one (cp1252, cp437), so a-/// script printing anything outside it dies with `UnicodeEncodeError`. Both are-/// no-ops for a non-Python command. Native runs get the same from TinyBox's-/// `python_utf8` request flag; this serves the Docker and sandbox-policy paths,-/// which forward their environment as plain variables.-pub(super) const PYTHON_UTF8_DEFAULTS: &[(&str, &str)] =- &[("PYTHONUTF8", "1"), ("PYTHONIOENCODING", "utf-8")];+/// Python defaults for a child, unless the parent sets its own.+///+/// - `PYTHONUTF8`, `PYTHONIOENCODING`: under `cmd.exe` the console code page+/// is a legacy one (cp1252, cp437), so a script printing anything outside it+/// dies with `UnicodeEncodeError`. Native runs get these from TinyBox's+/// `python_utf8` request flag.+/// - `PYTHONUNBUFFERED`: Python block-buffers stdout into a pipe (about 8 KB),+/// and a process killed at its deadline never flushes that buffer, so a+/// script that printed its first failures and then ran long would show+/// nothing. Any non-empty value turns buffering off; a parent that wants it+/// on sets it to the empty string.+///+/// All are no-ops for a non-Python command. This serves the Docker and+/// sandbox-policy paths, which forward their environment as plain variables;+/// the native path adds `PYTHONUNBUFFERED` itself.+pub(super) const PYTHON_ENV_DEFAULTS: &[(&str, &str)] = &[+ ("PYTHONUTF8", "1"),+ ("PYTHONIOENCODING", "utf-8"),+ (PYTHON_UNBUFFERED, "1"),+];++/// The variable that turns off Python's output buffering.+pub(super) const PYTHON_UNBUFFERED: &str = "PYTHONUNBUFFERED";++/// A native run's environment with `PYTHONUNBUFFERED` added, so a script killed+/// at its deadline has written what it printed (TinyBox's `python_utf8` flag+/// covers the encoding). A value already in `env` stays. Otherwise the+/// process's own value is inherited, unless the turn owns its environment+/// (`turn_environment`), as on the sandbox path; else it is `1`.+pub(super) fn with_unbuffered_python(+ mut env: std::collections::BTreeMap<String, String>,+ turn_environment: bool,+) -> std::collections::BTreeMap<String, String> {+ if !env.contains_key(PYTHON_UNBUFFERED) {+ let inherited = (!turn_environment)+ .then(|| std::env::var(PYTHON_UNBUFFERED).ok())+ .flatten();+ env.insert(+ PYTHON_UNBUFFERED.to_owned(),+ inherited.unwrap_or_else(|| "1".to_owned()),+ );+ }+ env+}-pub(super) fn python_utf8_env(+/// The [`PYTHON_ENV_DEFAULTS`] a child gets, each one the parent's own value+/// when `lookup` has one.+pub(super) fn python_env_defaults(lookup: impl Fn(&str) -> Option<std::ffi::OsString>,) -> Vec<(&'static str, std::ffi::OsString)> {- PYTHON_UTF8_DEFAULTS+ PYTHON_ENV_DEFAULTS.iter().map(|(name, default)| {let value = lookup(name).unwrap_or_else(|| (*default).into());@@ -140,7 +178,7 @@ pub(super) fn container_environment(Vec::new()}.into_iter()- .chain(python_utf8_env(|name| {+ .chain(python_env_defaults(|name| {explicit.and_then(|overrides| overrides.get(name).map(std::ffi::OsString::from)).or_else(|| {diff --git a/crates/openhuman-core/src/tools/impl/system/shell_tests_platform_tests.rs b/crates/openhuman-core/src/tools/impl/system/shell_tests_platform_tests.rsindex d833a45282..48240f7412 100644--- a/crates/openhuman-core/src/tools/impl/system/shell_tests_platform_tests.rs+++ b/crates/openhuman-core/src/tools/impl/system/shell_tests_platform_tests.rs@@ -7,7 +7,7 @@use super::*;use crate::modules::tinybox::host_runtime::{host_shell_flavor, ShellFlavor};#[cfg(feature = "modules")]-use crate::tools::system::shell_support::PYTHON_UTF8_DEFAULTS;+use crate::tools::system::shell_support::{python_env_defaults, PYTHON_ENV_DEFAULTS};use std::collections::HashMap;use std::ffi::OsString;@@ -78,7 +78,7 @@ fn as_map(env: Vec<(&'static str, OsString)>) -> HashMap<&'static str, String> {#[test]fn a_users_own_python_encoding_settings_win() {- let env = as_map(python_utf8_env(parent(&[+ let env = as_map(python_env_defaults(parent(&[("PYTHONUTF8", "0"),("PYTHONIOENCODING", "latin-1"),])));@@ -110,7 +110,7 @@ async fn the_container_environment_forwards_only_the_allowlist_plus_python_defaufor key in env.keys() {assert!(crate::sandbox::ops::SANDBOX_ENV_PASSTHROUGH.contains(&key.as_str())- || PYTHON_UTF8_DEFAULTS.iter().any(|(k, _)| k == key),+ || PYTHON_ENV_DEFAULTS.iter().any(|(k, _)| k == key),"unexpected {key}");}@@ -141,12 +141,37 @@ fn the_container_environment_forwards_explicit_turn_variables() {#[test]fn sandboxed_calls_get_the_same_python_defaults() {- let env = as_map(python_utf8_env(parent(&[])));+ let env = as_map(python_env_defaults(parent(&[])));assert_eq!(env.get("PYTHONUTF8").map(String::as_str), Some("1"));assert_eq!(env.get("PYTHONIOENCODING").map(String::as_str),Some("utf-8"));+ assert_eq!(env.get("PYTHONUNBUFFERED").map(String::as_str), Some("1"));+}++#[test]+fn a_parent_can_keep_python_buffering_with_an_empty_value() {+ let env = as_map(python_env_defaults(parent(&[("PYTHONUNBUFFERED", "")])));+ assert_eq!(env.get("PYTHONUNBUFFERED").map(String::as_str), Some(""));+}++#[tokio::test]+async fn a_spawned_shell_gets_unbuffered_python_output() {+ // A developer's own setting wins, so expect it when present.+ let expected = std::env::var("PYTHONUNBUFFERED").unwrap_or_else(|_| "1".to_string());+ let tool = ShellTool::new(test_security_with_env_cmd(), test_runtime(), test_audit());+ let command = if cfg!(windows) {+ "echo [%PYTHONUNBUFFERED%]"+ } else {+ "echo [$PYTHONUNBUFFERED]"+ };+ let result = tool+ .execute(serde_json::json!({ "command": command }))+ .await+ .unwrap();+ assert!(!result.is_error, "{}", result.output());+ assert_eq!(result.output().trim(), format!("[{expected}]"));}#[tokio::test]