Skip to content

The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.

test(e2e): add headless chat agent e2e test

Steven Enamakel9581ae319657
diff --git a/tests/cli_agent_chat_headless_e2e.rs b/tests/cli_agent_chat_headless_e2e.rs
new file mode 100644
index 0000000000..cefa2ca9ce
--- /dev/null
+++ b/tests/cli_agent_chat_headless_e2e.rs
@@ -0,0 +1,352 @@
+//! A one-shot compiled CLI call waits for its registered async child and
+//! reports child failure as a failed call.
+
+use serde_json::{json, Value};
+use std::fs::File;
+use std::net::TcpListener;
+use std::path::{Path, PathBuf};
+use std::process::{Child, Command, Stdio};
+use std::thread;
+use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
+
+const CHILD_PROMPT: &str = "CLI_CHILD_PROMPT_220";
+const CHILD_RESULT: &str = "CLI_CHILD_RESULT_220";
+const ROOT_RESULT: &str = "CLI_ROOT_FINISHED_220";
+const FAILURE_PROMPT: &str = "CLI_FAILURE_CHILD_PROMPT_220";
+const FAILURE_RESULT: &str = "CLI_CHILD_FAILURE_220";
+
+struct ChildGuard(Child);
+
+impl Drop for ChildGuard {
+ fn drop(&mut self) {
+ let _ = self.0.kill();
+ let _ = self.0.wait();
+ }
+}
+
+fn free_port() -> u16 {
+ TcpListener::bind("127.0.0.1:0")
+ .expect("bind ephemeral port")
+ .local_addr()
+ .expect("listener address")
+ .port()
+}
+
+fn logs_dir() -> PathBuf {
+ let path = Path::new(env!("CARGO_MANIFEST_DIR"))
+ .join("../../target/debug-logs")
+ .join(format!(
+ "cli-agent-chat-220-{}-{}",
+ std::process::id(),
+ SystemTime::now()
+ .duration_since(UNIX_EPOCH)
+ .expect("clock after epoch")
+ .as_nanos()
+ ));
+ std::fs::create_dir_all(&path).expect("create isolated CLI test log directory");
+ path
+}
+
+fn wait_for_mock(origin: &str, child: &mut Child, log_path: &Path) {
+ let client = reqwest::blocking::Client::builder()
+ .timeout(Duration::from_millis(250))
+ .build()
+ .expect("http client");
+ let until = Instant::now() + Duration::from_secs(15);
+ while Instant::now() < until {
+ if !matches!(child.try_wait(), Ok(None)) {
+ panic!(
+ "mock API exited before becoming healthy; {}",
+ std::fs::read_to_string(log_path).unwrap_or_default()
+ );
+ }
+ if client
+ .get(format!("{origin}/__admin/health"))
+ .send()
+ .is_ok_and(|response| response.status().is_success())
+ {
+ return;
+ }
+ thread::sleep(Duration::from_millis(50));
+ }
+ panic!(
+ "mock API did not become healthy at {origin}; {}",
+ std::fs::read_to_string(log_path).unwrap_or_default()
+ );
+}
+
+fn set_mock_behavior(origin: &str, behavior: Value) {
+ let response = reqwest::blocking::Client::new()
+ .post(format!("{origin}/__admin/behavior"))
+ .json(&json!({ "behavior": behavior, "mode": "replace" }))
+ .send()
+ .expect("configure mock behavior");
+ assert!(response.status().is_success(), "mock behavior setup: {response}");
+}
+
+fn mock_requests(origin: &str) -> Value {
+ reqwest::blocking::Client::new()
+ .get(format!("{origin}/__admin/requests"))
+ .send()
+ .expect("read mock request history")
+ .error_for_status()
+ .expect("mock request-history status")
+ .json()
+ .expect("mock request-history JSON")
+}
+
+fn primary_llm_probes(origin: &str) -> Vec<(String, bool)> {
+ mock_requests(origin)["data"]
+ .as_array()
+ .expect("request history data array")
+ .iter()
+ .filter(|entry| {
+ entry["method"].as_str() == Some("POST")
+ && entry["url"].as_str().is_some_and(|url| url.contains("chat/completions"))
+ })
+ .filter_map(|entry| {
+ let body: Value = serde_json::from_str(entry["body"].as_str()?).ok()?;
+ let messages = body["messages"].as_array()?;
+ let probe = messages.iter().rev().find_map(|message| {
+ if !matches!(message["role"].as_str(), Some("user" | "tool")) {
+ return None;
+ }
+ message["content"].as_str().map(str::to_owned).or_else(|| {
+ message["content"].as_array().map(|parts| {
+ parts.iter().filter_map(|part| part["text"].as_str())
+ .collect::<Vec<_>>().join(" ")
+ })
+ })
+ })?;
+ Some((probe, body["stream"] == true))
+ })
+ .collect()
+}
+
+fn configure(home: &Path, workspace: &Path, backend: &str) {
+ // OPENHUMAN_WORKSPACE points to `<home>/.openhuman/workspace`; per
+ // config/schema/load/dirs.rs this resolves config.toml from its parent.
+ let config = home.join(".openhuman");
+ std::fs::create_dir_all(workspace).expect("create workspace");
+ std::fs::create_dir_all(&config).expect("create config directory");
+ std::fs::write(
+ config.join("config.toml"),
+ format!(
+ "api_url = \"{backend}\"\ndefault_model = \"openai/mock\"\ndefault_temperature = 0.0\n\n[secrets]\nencrypt = false\n"
+ ),
+ )
+ .expect("write isolated config");
+}
+
+fn run_cli(
+ home: &Path,
+ workspace: &Path,
+ action_dir: &Path,
+ backend: &str,
+ log_dir: &Path,
+) -> (bool, String, String) {
+ let stdout_path = log_dir.join("cli.stdout");
+ let stderr_path = log_dir.join("cli.stderr");
+ let stdout = File::create(&stdout_path).expect("create CLI stdout log");
+ let stderr = File::create(&stderr_path).expect("create CLI stderr log");
+ let params = json!({ "message": "Delegate this request to a child and report its exact result", "model_override": "e2e-mock-model" });
+ let mut child = ChildGuard(
+ Command::new(env!("CARGO_BIN_EXE_openhuman-core"))
+ .args(["call", "--method", "inference.agent_chat", "--params"])
+ .arg(params.to_string())
+ .env("HOME", home)
+ .env("USERPROFILE", home)
+ .env("OPENHUMAN_WORKSPACE", workspace)
+ .env("OPENHUMAN_ACTION_DIR", action_dir)
+ .env("BACKEND_URL", backend)
+ .env("OPENHUMAN_BACKEND_API_KEY", "synthetic-cli-test-key")
+ .env("OPENHUMAN_KEYRING_BACKEND", "file")
+ .env_remove("OPENHUMAN_BACKEND_SESSION_TOKEN")
+ .env_remove("OPENHUMAN_KEYRING_MASTER_KEY_FILE")
+ .env("OPENHUMAN_KEYRING_MASTER_KEY", "22".repeat(32))
+ .stdout(Stdio::from(stdout))
+ .stderr(Stdio::from(stderr))
+ .spawn()
+ .expect("start one-shot openhuman-core call"),
+ );
+
+ let until = Instant::now() + Duration::from_secs(90);
+ let success = loop {
+ if let Some(status) = child.0.try_wait().expect("poll CLI process") {
+ break status.success();
+ }
+ if Instant::now() >= until {
+ let _ = child.0.kill();
+ let _ = child.0.wait();
+ panic!(
+ "one-shot CLI exceeded 90 second watchdog; logs: {}",
+ log_dir.display()
+ );
+ }
+ thread::sleep(Duration::from_millis(25));
+ };
+ let out = std::fs::read_to_string(stdout_path).expect("read CLI stdout");
+ let err = std::fs::read_to_string(stderr_path).expect("read CLI stderr");
+ (success, out, err)
+}
+
+#[test]
+fn one_shot_cli_waits_for_and_returns_its_spawned_child_result() {
+ let home = tempfile::tempdir().expect("isolated home");
+ let workspace = home.path().join(".openhuman/workspace");
+ let action_dir = home.path().join("OpenHuman/projects");
+ std::fs::create_dir_all(&action_dir).expect("create action directory");
+ let port = free_port();
+ let origin = format!("http://127.0.0.1:{port}");
+ let logs = logs_dir();
+ let repo = Path::new(env!("CARGO_MANIFEST_DIR")).join("../..");
+ let mock_log = File::create(logs.join("mock.log")).expect("create unique mock log");
+ let mock_err = mock_log.try_clone().expect("clone mock log handle");
+ let mut mock = ChildGuard(
+ Command::new("node")
+ .arg(repo.join("scripts/mock-api-server.mjs"))
+ .arg("--port")
+ .arg(port.to_string())
+ .env("MOCK_LOG", logs.join("mock-requests.log"))
+ .stdout(Stdio::from(mock_log))
+ .stderr(Stdio::from(mock_err))
+ .spawn()
+ .expect("start local mock backend (Node.js must be installed)"),
+ );
+ let mock_log_path = logs.join("mock.log");
+ wait_for_mock(&origin, &mut mock.0, &mock_log_path);
+ configure(home.path(), &workspace, &origin);
+
+ // `pickProbeText` in scripts/mock-api/routes/llm/shared.mjs selects only
+ // the latest user/tool message. The CHILD_PROMPT rule therefore routes
+ // only a child inference request, while the root accepted-spawn follow-up
+ // gets a separate ROOT_RESULT. `task_key` is explicit and unique.
+ set_mock_behavior(
+ &origin,
+ json!({
+ "llmRequestRules": [
+ {
+ "keyword": CHILD_PROMPT,
+ "content": CHILD_RESULT,
+ "streamScript": [
+ { "text": CHILD_RESULT },
+ { "finish": "stop" }
+ ]
+ },
+ {
+ "keyword": "Accepted async sub-agent `task_manager_agent`",
+ "content": ROOT_RESULT,
+ "streamScript": [
+ { "text": ROOT_RESULT },
+ { "finish": "stop" }
+ ]
+ }
+ ],
+ "llmStreamScript": [
+ { "toolCall": {
+ "id": "call_cli_child_220",
+ "name": "spawn_subagent",
+ "arguments": json!({
+ "agent_id": "task_manager_agent",
+ "prompt": format!("{CHILD_PROMPT}: return exact result {CHILD_RESULT}"),
+ "task_key": "compiled-cli-child-220"
+ }).to_string()
+ }},
+ { "finish": "tool_calls" }
+ ]
+ }),
+ );
+
+ let success_logs = logs.join("success");
+ std::fs::create_dir_all(&success_logs).expect("create success logs");
+ let (success, stdout, stderr) =
+ run_cli(home.path(), &workspace, &action_dir, &origin, &success_logs);
+ assert!(
+ success,
+ "one-shot CLI failed; logs: {}\nstdout: {stdout}\nstderr: {stderr}",
+ logs.display()
+ );
+ let response: Value = serde_json::from_str(stdout.trim())
+ .unwrap_or_else(|error| panic!("CLI stdout is not JSON ({error}): {stdout}"));
+ assert!(
+ response.to_string().contains(CHILD_RESULT),
+ "successful CLI response must include the awaited child result, got {response}"
+ );
+ assert!(
+ response.to_string().contains(ROOT_RESULT),
+ "root response did not complete after child settlement: {response}"
+ );
+
+ // The one-shot request has no thread/session metadata, so the mock has no
+ // llmThreads key for this run. Prove the child request from primary LLM
+ // request history and the child's unique response from CLI output. Only
+ // the child rule emits CHILD_RESULT; root completion emits ROOT_RESULT.
+ let probes = primary_llm_probes(&origin);
+ assert!(
+ probes.iter().any(|(probe, streaming)| probe.contains(CHILD_PROMPT) && *streaming),
+ "no actual streaming primary LLM request carried the child prompt: {probes:#?}"
+ );
+
+ // Repeat with a separate profile and a child model error. A failed
+ // descendant must make the one-shot CLI fail instead of returning the
+ // scripted successful root response.
+ let failed_home = tempfile::tempdir().expect("isolated failure home");
+ let failed_workspace = failed_home.path().join(".openhuman/workspace");
+ let failed_action_dir = failed_home.path().join("OpenHuman/projects");
+ std::fs::create_dir_all(&failed_action_dir).expect("create failure action directory");
+ configure(failed_home.path(), &failed_workspace, &origin);
+ set_mock_behavior(
+ &origin,
+ json!({
+ "llmRequestRules": [
+ {
+ "keyword": FAILURE_PROMPT,
+ "error": FAILURE_RESULT
+ },
+ {
+ "keyword": "Accepted async sub-agent `task_manager_agent`",
+ "content": ROOT_RESULT,
+ "streamScript": [
+ { "text": ROOT_RESULT },
+ { "finish": "stop" }
+ ]
+ }
+ ],
+ "llmStreamScript": [
+ { "toolCall": {
+ "id": "call_cli_child_failure_220",
+ "name": "spawn_subagent",
+ "arguments": json!({
+ "agent_id": "task_manager_agent",
+ "prompt": format!("{FAILURE_PROMPT}: trigger {FAILURE_RESULT}"),
+ "task_key": "compiled-cli-child-failure-220"
+ }).to_string()
+ }},
+ { "finish": "tool_calls" }
+ ]
+ }),
+ );
+ let failure_logs = logs.join("failure");
+ std::fs::create_dir_all(&failure_logs).expect("create failure logs");
+ let (failed, failed_stdout, failed_stderr) = run_cli(
+ failed_home.path(),
+ &failed_workspace,
+ &failed_action_dir,
+ &origin,
+ &failure_logs,
+ );
+ let failure_probes = primary_llm_probes(&origin);
+ assert!(
+ failure_probes.iter().any(|(probe, streaming)| probe.contains(FAILURE_PROMPT) && *streaming),
+ "failure case did not issue its distinct actual child request: {failure_probes:#?}"
+ );
+ assert!(
+ !failed,
+ "one-shot CLI reported success despite a failed child; stdout: {failed_stdout}\nstderr: {failed_stderr}"
+ );
+ assert!(
+ format!("{failed_stdout}{failed_stderr}").contains("task_manager_agent")
+ || format!("{failed_stdout}{failed_stderr}").contains(FAILURE_RESULT),
+ "failure output should identify the failed child or model error; stdout: {failed_stdout}\nstderr: {failed_stderr}"
+ );
+}