The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.
Rust agent-orchestrationai-agentsai-assistantdesktopllmlocal-firstmcppersonal-aiprivacyrustsecond-braintauri
test(e2e): add headless chat agent e2e test
Steven Enamakel9581ae319657
diff --git a/tests/cli_agent_chat_headless_e2e.rs b/tests/cli_agent_chat_headless_e2e.rsnew file mode 100644index 0000000000..cefa2ca9ce--- /dev/null+++ b/tests/cli_agent_chat_headless_e2e.rs@@ -0,0 +1,352 @@+//! A one-shot compiled CLI call waits for its registered async child and+//! reports child failure as a failed call.++use serde_json::{json, Value};+use std::fs::File;+use std::net::TcpListener;+use std::path::{Path, PathBuf};+use std::process::{Child, Command, Stdio};+use std::thread;+use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};++const CHILD_PROMPT: &str = "CLI_CHILD_PROMPT_220";+const CHILD_RESULT: &str = "CLI_CHILD_RESULT_220";+const ROOT_RESULT: &str = "CLI_ROOT_FINISHED_220";+const FAILURE_PROMPT: &str = "CLI_FAILURE_CHILD_PROMPT_220";+const FAILURE_RESULT: &str = "CLI_CHILD_FAILURE_220";++struct ChildGuard(Child);++impl Drop for ChildGuard {+ fn drop(&mut self) {+ let _ = self.0.kill();+ let _ = self.0.wait();+ }+}++fn free_port() -> u16 {+ TcpListener::bind("127.0.0.1:0")+ .expect("bind ephemeral port")+ .local_addr()+ .expect("listener address")+ .port()+}++fn logs_dir() -> PathBuf {+ let path = Path::new(env!("CARGO_MANIFEST_DIR"))+ .join("../../target/debug-logs")+ .join(format!(+ "cli-agent-chat-220-{}-{}",+ std::process::id(),+ SystemTime::now()+ .duration_since(UNIX_EPOCH)+ .expect("clock after epoch")+ .as_nanos()+ ));+ std::fs::create_dir_all(&path).expect("create isolated CLI test log directory");+ path+}++fn wait_for_mock(origin: &str, child: &mut Child, log_path: &Path) {+ let client = reqwest::blocking::Client::builder()+ .timeout(Duration::from_millis(250))+ .build()+ .expect("http client");+ let until = Instant::now() + Duration::from_secs(15);+ while Instant::now() < until {+ if !matches!(child.try_wait(), Ok(None)) {+ panic!(+ "mock API exited before becoming healthy; {}",+ std::fs::read_to_string(log_path).unwrap_or_default()+ );+ }+ if client+ .get(format!("{origin}/__admin/health"))+ .send()+ .is_ok_and(|response| response.status().is_success())+ {+ return;+ }+ thread::sleep(Duration::from_millis(50));+ }+ panic!(+ "mock API did not become healthy at {origin}; {}",+ std::fs::read_to_string(log_path).unwrap_or_default()+ );+}++fn set_mock_behavior(origin: &str, behavior: Value) {+ let response = reqwest::blocking::Client::new()+ .post(format!("{origin}/__admin/behavior"))+ .json(&json!({ "behavior": behavior, "mode": "replace" }))+ .send()+ .expect("configure mock behavior");+ assert!(response.status().is_success(), "mock behavior setup: {response}");+}++fn mock_requests(origin: &str) -> Value {+ reqwest::blocking::Client::new()+ .get(format!("{origin}/__admin/requests"))+ .send()+ .expect("read mock request history")+ .error_for_status()+ .expect("mock request-history status")+ .json()+ .expect("mock request-history JSON")+}++fn primary_llm_probes(origin: &str) -> Vec<(String, bool)> {+ mock_requests(origin)["data"]+ .as_array()+ .expect("request history data array")+ .iter()+ .filter(|entry| {+ entry["method"].as_str() == Some("POST")+ && entry["url"].as_str().is_some_and(|url| url.contains("chat/completions"))+ })+ .filter_map(|entry| {+ let body: Value = serde_json::from_str(entry["body"].as_str()?).ok()?;+ let messages = body["messages"].as_array()?;+ let probe = messages.iter().rev().find_map(|message| {+ if !matches!(message["role"].as_str(), Some("user" | "tool")) {+ return None;+ }+ message["content"].as_str().map(str::to_owned).or_else(|| {+ message["content"].as_array().map(|parts| {+ parts.iter().filter_map(|part| part["text"].as_str())+ .collect::<Vec<_>>().join(" ")+ })+ })+ })?;+ Some((probe, body["stream"] == true))+ })+ .collect()+}++fn configure(home: &Path, workspace: &Path, backend: &str) {+ // OPENHUMAN_WORKSPACE points to `<home>/.openhuman/workspace`; per+ // config/schema/load/dirs.rs this resolves config.toml from its parent.+ let config = home.join(".openhuman");+ std::fs::create_dir_all(workspace).expect("create workspace");+ std::fs::create_dir_all(&config).expect("create config directory");+ std::fs::write(+ config.join("config.toml"),+ format!(+ "api_url = \"{backend}\"\ndefault_model = \"openai/mock\"\ndefault_temperature = 0.0\n\n[secrets]\nencrypt = false\n"+ ),+ )+ .expect("write isolated config");+}++fn run_cli(+ home: &Path,+ workspace: &Path,+ action_dir: &Path,+ backend: &str,+ log_dir: &Path,+) -> (bool, String, String) {+ let stdout_path = log_dir.join("cli.stdout");+ let stderr_path = log_dir.join("cli.stderr");+ let stdout = File::create(&stdout_path).expect("create CLI stdout log");+ let stderr = File::create(&stderr_path).expect("create CLI stderr log");+ let params = json!({ "message": "Delegate this request to a child and report its exact result", "model_override": "e2e-mock-model" });+ let mut child = ChildGuard(+ Command::new(env!("CARGO_BIN_EXE_openhuman-core"))+ .args(["call", "--method", "inference.agent_chat", "--params"])+ .arg(params.to_string())+ .env("HOME", home)+ .env("USERPROFILE", home)+ .env("OPENHUMAN_WORKSPACE", workspace)+ .env("OPENHUMAN_ACTION_DIR", action_dir)+ .env("BACKEND_URL", backend)+ .env("OPENHUMAN_BACKEND_API_KEY", "synthetic-cli-test-key")+ .env("OPENHUMAN_KEYRING_BACKEND", "file")+ .env_remove("OPENHUMAN_BACKEND_SESSION_TOKEN")+ .env_remove("OPENHUMAN_KEYRING_MASTER_KEY_FILE")+ .env("OPENHUMAN_KEYRING_MASTER_KEY", "22".repeat(32))+ .stdout(Stdio::from(stdout))+ .stderr(Stdio::from(stderr))+ .spawn()+ .expect("start one-shot openhuman-core call"),+ );++ let until = Instant::now() + Duration::from_secs(90);+ let success = loop {+ if let Some(status) = child.0.try_wait().expect("poll CLI process") {+ break status.success();+ }+ if Instant::now() >= until {+ let _ = child.0.kill();+ let _ = child.0.wait();+ panic!(+ "one-shot CLI exceeded 90 second watchdog; logs: {}",+ log_dir.display()+ );+ }+ thread::sleep(Duration::from_millis(25));+ };+ let out = std::fs::read_to_string(stdout_path).expect("read CLI stdout");+ let err = std::fs::read_to_string(stderr_path).expect("read CLI stderr");+ (success, out, err)+}++#[test]+fn one_shot_cli_waits_for_and_returns_its_spawned_child_result() {+ let home = tempfile::tempdir().expect("isolated home");+ let workspace = home.path().join(".openhuman/workspace");+ let action_dir = home.path().join("OpenHuman/projects");+ std::fs::create_dir_all(&action_dir).expect("create action directory");+ let port = free_port();+ let origin = format!("http://127.0.0.1:{port}");+ let logs = logs_dir();+ let repo = Path::new(env!("CARGO_MANIFEST_DIR")).join("../..");+ let mock_log = File::create(logs.join("mock.log")).expect("create unique mock log");+ let mock_err = mock_log.try_clone().expect("clone mock log handle");+ let mut mock = ChildGuard(+ Command::new("node")+ .arg(repo.join("scripts/mock-api-server.mjs"))+ .arg("--port")+ .arg(port.to_string())+ .env("MOCK_LOG", logs.join("mock-requests.log"))+ .stdout(Stdio::from(mock_log))+ .stderr(Stdio::from(mock_err))+ .spawn()+ .expect("start local mock backend (Node.js must be installed)"),+ );+ let mock_log_path = logs.join("mock.log");+ wait_for_mock(&origin, &mut mock.0, &mock_log_path);+ configure(home.path(), &workspace, &origin);++ // `pickProbeText` in scripts/mock-api/routes/llm/shared.mjs selects only+ // the latest user/tool message. The CHILD_PROMPT rule therefore routes+ // only a child inference request, while the root accepted-spawn follow-up+ // gets a separate ROOT_RESULT. `task_key` is explicit and unique.+ set_mock_behavior(+ &origin,+ json!({+ "llmRequestRules": [+ {+ "keyword": CHILD_PROMPT,+ "content": CHILD_RESULT,+ "streamScript": [+ { "text": CHILD_RESULT },+ { "finish": "stop" }+ ]+ },+ {+ "keyword": "Accepted async sub-agent `task_manager_agent`",+ "content": ROOT_RESULT,+ "streamScript": [+ { "text": ROOT_RESULT },+ { "finish": "stop" }+ ]+ }+ ],+ "llmStreamScript": [+ { "toolCall": {+ "id": "call_cli_child_220",+ "name": "spawn_subagent",+ "arguments": json!({+ "agent_id": "task_manager_agent",+ "prompt": format!("{CHILD_PROMPT}: return exact result {CHILD_RESULT}"),+ "task_key": "compiled-cli-child-220"+ }).to_string()+ }},+ { "finish": "tool_calls" }+ ]+ }),+ );++ let success_logs = logs.join("success");+ std::fs::create_dir_all(&success_logs).expect("create success logs");+ let (success, stdout, stderr) =+ run_cli(home.path(), &workspace, &action_dir, &origin, &success_logs);+ assert!(+ success,+ "one-shot CLI failed; logs: {}\nstdout: {stdout}\nstderr: {stderr}",+ logs.display()+ );+ let response: Value = serde_json::from_str(stdout.trim())+ .unwrap_or_else(|error| panic!("CLI stdout is not JSON ({error}): {stdout}"));+ assert!(+ response.to_string().contains(CHILD_RESULT),+ "successful CLI response must include the awaited child result, got {response}"+ );+ assert!(+ response.to_string().contains(ROOT_RESULT),+ "root response did not complete after child settlement: {response}"+ );++ // The one-shot request has no thread/session metadata, so the mock has no+ // llmThreads key for this run. Prove the child request from primary LLM+ // request history and the child's unique response from CLI output. Only+ // the child rule emits CHILD_RESULT; root completion emits ROOT_RESULT.+ let probes = primary_llm_probes(&origin);+ assert!(+ probes.iter().any(|(probe, streaming)| probe.contains(CHILD_PROMPT) && *streaming),+ "no actual streaming primary LLM request carried the child prompt: {probes:#?}"+ );++ // Repeat with a separate profile and a child model error. A failed+ // descendant must make the one-shot CLI fail instead of returning the+ // scripted successful root response.+ let failed_home = tempfile::tempdir().expect("isolated failure home");+ let failed_workspace = failed_home.path().join(".openhuman/workspace");+ let failed_action_dir = failed_home.path().join("OpenHuman/projects");+ std::fs::create_dir_all(&failed_action_dir).expect("create failure action directory");+ configure(failed_home.path(), &failed_workspace, &origin);+ set_mock_behavior(+ &origin,+ json!({+ "llmRequestRules": [+ {+ "keyword": FAILURE_PROMPT,+ "error": FAILURE_RESULT+ },+ {+ "keyword": "Accepted async sub-agent `task_manager_agent`",+ "content": ROOT_RESULT,+ "streamScript": [+ { "text": ROOT_RESULT },+ { "finish": "stop" }+ ]+ }+ ],+ "llmStreamScript": [+ { "toolCall": {+ "id": "call_cli_child_failure_220",+ "name": "spawn_subagent",+ "arguments": json!({+ "agent_id": "task_manager_agent",+ "prompt": format!("{FAILURE_PROMPT}: trigger {FAILURE_RESULT}"),+ "task_key": "compiled-cli-child-failure-220"+ }).to_string()+ }},+ { "finish": "tool_calls" }+ ]+ }),+ );+ let failure_logs = logs.join("failure");+ std::fs::create_dir_all(&failure_logs).expect("create failure logs");+ let (failed, failed_stdout, failed_stderr) = run_cli(+ failed_home.path(),+ &failed_workspace,+ &failed_action_dir,+ &origin,+ &failure_logs,+ );+ let failure_probes = primary_llm_probes(&origin);+ assert!(+ failure_probes.iter().any(|(probe, streaming)| probe.contains(FAILURE_PROMPT) && *streaming),+ "failure case did not issue its distinct actual child request: {failure_probes:#?}"+ );+ assert!(+ !failed,+ "one-shot CLI reported success despite a failed child; stdout: {failed_stdout}\nstderr: {failed_stderr}"+ );+ assert!(+ format!("{failed_stdout}{failed_stderr}").contains("task_manager_agent")+ || format!("{failed_stdout}{failed_stderr}").contains(FAILURE_RESULT),+ "failure output should identify the failed child or model error; stdout: {failed_stdout}\nstderr: {failed_stderr}"+ );+}