The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.
Rust agent-orchestrationai-agentsai-assistantdesktopllmlocal-firstmcppersonal-aiprivacyrustsecond-braintauri
use super::*;
use serde_json::{json, Value};
use tinyagents_harness_tools::todos::{TodoItem, TodoStatus};
/// Serialize tests that share the process-global scratch store. Same lock
/// process-global scratch store.
static SCRATCH_TEST_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());
fn scratch_lock() -> tokio::sync::MutexGuard<'static, ()> {
SCRATCH_TEST_LOCK.blocking_lock()
}
async fn scratch_lock_async() -> tokio::sync::MutexGuard<'static, ()> {
SCRATCH_TEST_LOCK.lock().await
}
fn todo_store(
workspace_dir: &std::path::Path,
) -> std::sync::Arc<dyn tinyagents_harness::store::Store> {
crate::storage::sessions::stores_or_ephemeral(workspace_dir).kv
}
/// A fresh on-disk workspace root for one test's `FileStore`-backed todo
/// list. Each test gets its own tempdir so tests never see each other's
/// persisted lists.
fn test_workspace() -> std::path::PathBuf {
let dir = std::env::temp_dir().join(format!("openhuman-todo-test-{}", uuid::Uuid::new_v4()));
std::fs::create_dir_all(&dir).expect("create test workspace");
dir
}
async fn reset_scratch(workspace_dir: &std::path::Path) {
tinyagents_harness_tools::todos::store::clear(&todo_store(workspace_dir), "_scratch_")
.await
.expect("clear scratch");
}
fn payload(result: &ToolResult) -> Value {
serde_json::from_str(&result.output()).expect("json payload")
}
#[tokio::test]
async fn a_write_replaces_the_whole_list_and_a_read_returns_it() {
let _guard = scratch_lock_async().await;
let workspace_dir = test_workspace();
reset_scratch(&workspace_dir).await;
let tool = TodoTool::new(workspace_dir.clone());
let written = tool
.execute(json!({ "todos": [
{ "content": "Write tests", "status": "in_progress" },
{ "content": "Ship it", "status": "pending" }
] }))
.await
.unwrap();
assert!(!written.is_error, "{}", written.output());
let p = payload(&written);
assert_eq!(p["todos"].as_array().unwrap().len(), 2);
assert_eq!(p["todos"][0]["status"], "in_progress");
assert_eq!(p["todos"][1]["status"], "pending");
let markdown = p["markdown"].as_str().unwrap();
assert!(markdown.contains("[~] Write tests"), "{markdown}");
assert!(markdown.contains("[ ] Ship it"), "{markdown}");
// Omitting `todos` reads the list back.
let read = tool.execute(json!({})).await.unwrap();
assert_eq!(payload(&read)["todos"].as_array().unwrap().len(), 2);
// The next write is the whole list again, not a patch.
let rewritten = tool
.execute(json!({ "todos": [
{ "content": "Write tests", "status": "completed" }
] }))
.await
.unwrap();
let p = payload(&rewritten);
assert_eq!(p["todos"].as_array().unwrap().len(), 1);
assert_eq!(p["todos"][0]["status"], "completed");
assert!(p["markdown"].as_str().unwrap().contains("[x] Write tests"));
// An empty list clears it.
let cleared = tool.execute(json!({ "todos": [] })).await.unwrap();
assert!(payload(&cleared)["todos"].as_array().unwrap().is_empty());
reset_scratch(&workspace_dir).await;
}
/// Bad input is a tool error the model can correct, never an `Err`: a
/// dispatch `Err` is fatal to the whole run in the harness, and a turn died
/// exactly that way when a model sent the retired `{"cards": …}` shape.
#[tokio::test]
async fn bad_input_is_a_tool_error_not_a_harness_error() {
let tool = TodoTool::new(test_workspace());
for (args, expect) in [
(
json!({ "todos": [{ "content": " ", "status": "pending" }] }),
"content",
),
(
// The exact phrasing belongs to TinyAgents; assert only that the
// rejection names the field the model got wrong.
json!({ "todos": [{ "content": "x", "status": "someday" }] }),
"status",
),
(json!({ "todos": "not a list" }), "invalid `todos`"),
(
json!({ "cards": [{ "content": "x", "status": "todo" }] }),
"pass `todos`",
),
] {
let result = tool
.execute(args.clone())
.await
.expect("never an Err: {args}");
assert!(result.is_error, "{args}");
assert!(
result.output().contains(expect),
"{args}: {}",
result.output()
);
}
}
/// The orchestrator's list is its thread's list — keyed by the chat thread
/// id, not `ParentExecutionContext::session_id` (which for the web channel is
/// the `{client_id,thread_id}` JSON blob and would otherwise scatter one
/// thread's todos across every reconnect). It used to be routed to one
/// app-wide `orchestrator-tasks` board that nothing rendered, so the items
/// the model wrote never showed up in the thread the user was in.
#[test]
fn every_agent_binds_to_its_own_thread() {
struct ThreadContext(&'static str);
impl ToolRunContext for ThreadContext {
fn thread_id(&self) -> Option<&str> {
Some(self.0)
}
}
let parent = ParentExecutionContext {
agent_definition_id: "orchestrator".into(),
allowed_subagent_ids: std::collections::HashSet::new(),
turn_model_source: crate::agent::tinyagents::TurnModelSource::from_model(Arc::new(
tinyagents_harness::testkit::ScriptedModel::replies(vec!["done"]),
)),
all_tools: Arc::new(Vec::new()),
all_tool_specs: Arc::new(Vec::new()),
visible_tool_specs: Arc::new(Vec::new()),
visible_tool_names: std::collections::HashSet::new(),
subagent_tool_ceiling_names: std::collections::HashSet::new(),
model_name: "test-model".into(),
temperature: 0.0,
workspace_dir: std::path::PathBuf::from("/tmp/openhuman-todo-parent"),
workspace_descriptor: None,
agent_config: crate::config::AgentConfig::default(),
workflows: Arc::new(Vec::new()),
memory_context: Arc::new(None),
session_id: "orchestrator_thread-live".into(),
channel: "test".into(),
connected_integrations: Vec::new(),
tool_call_format: tinytools_agent::dialect::ToolCallFormat::Native,
session_key: "parent-key".into(),
session_parent_prefix: None,
on_progress: None,
run_queue: None,
host_env: Default::default(),
};
assert_eq!(
current_scope(Some(&parent), Some(&ThreadContext("thread-live"))).session_id(),
Some("thread-live"),
"the thread id wins over the parent's legacy session_id"
);
assert_eq!(
current_scope(None, Some(&ThreadContext("thread-live"))).session_id(),
Some("thread-live"),
"a thread-only caller keys on the thread"
);
assert_eq!(
current_scope(Some(&parent), None).session_id(),
Some("orchestrator_thread-live"),
"no thread id at all falls back to the legacy parent session_id"
);
assert_eq!(current_scope(None, None), TodoScope::Scratch);
}
/// A list left under the pre-rekey `session_id` key is picked up once by the
/// new thread-id key, instead of silently disappearing when this file's
/// scope switched from `session_id`-first to `thread_id`-first.
#[tokio::test]
async fn a_legacy_session_keyed_list_is_migrated_forward_once() {
let workspace_dir = test_workspace();
struct ThreadContext(&'static str);
impl ToolRunContext for ThreadContext {
fn thread_id(&self) -> Option<&str> {
Some(self.0)
}
}
let parent = ParentExecutionContext {
agent_definition_id: "orchestrator".into(),
allowed_subagent_ids: std::collections::HashSet::new(),
turn_model_source: crate::agent::tinyagents::TurnModelSource::from_model(Arc::new(
tinyagents_harness::testkit::ScriptedModel::replies(vec!["done"]),
)),
all_tools: Arc::new(Vec::new()),
all_tool_specs: Arc::new(Vec::new()),
visible_tool_specs: Arc::new(Vec::new()),
visible_tool_names: std::collections::HashSet::new(),
subagent_tool_ceiling_names: std::collections::HashSet::new(),
model_name: "test-model".into(),
temperature: 0.0,
workspace_dir: workspace_dir.clone(),
workspace_descriptor: None,
agent_config: crate::config::AgentConfig::default(),
workflows: Arc::new(Vec::new()),
memory_context: Arc::new(None),
session_id: "client-1|thread-legacy".into(),
channel: "test".into(),
connected_integrations: Vec::new(),
tool_call_format: tinytools_agent::dialect::ToolCallFormat::Native,
session_key: "parent-key".into(),
session_parent_prefix: None,
on_progress: None,
run_queue: None,
host_env: Default::default(),
};
// Seed the legacy session-keyed list directly through the store, as if a
// pre-rekey process had written it.
let legacy_scope = TodoScope::Session {
id: parent.session_id.clone(),
};
let item = TodoItem::with_status("carried over", TodoStatus::InProgress);
tinyagents_harness_tools::todos::store::replace(
&todo_store(&workspace_dir),
legacy_scope.key(),
vec![item],
)
.await
.unwrap();
let tool = TodoTool::new(workspace_dir.clone());
let read = tool
.execute_with_parent_context(
json!({}),
Some(parent.clone()),
Some(&ThreadContext("thread-legacy")),
)
.await
.unwrap();
let p = payload(&read);
let todos = p["todos"].as_array().unwrap();
assert_eq!(todos.len(), 1, "{p}");
assert_eq!(todos[0]["content"], "carried over");
// The migrated list now lives under the thread-id key too.
let thread_scope = TodoScope::Session {
id: "thread-legacy".into(),
};
let migrated = tinyagents_harness_tools::todos::store::list(
&todo_store(&workspace_dir),
thread_scope.key(),
)
.await
.unwrap();
assert_eq!(migrated.items.len(), 1);
// Clearing the new thread-keyed list must remain authoritative. The empty
// list is persisted, so a later read must not treat it as an absent target
// and copy the still-present legacy row back into it.
let cleared = tool
.execute_with_parent_context(
json!({ "todos": [] }),
Some(parent.clone()),
Some(&ThreadContext("thread-legacy")),
)
.await
.unwrap();
assert!(payload(&cleared)["todos"].as_array().unwrap().is_empty());
let read_after_clear = tool
.execute_with_parent_context(
json!({}),
Some(parent),
Some(&ThreadContext("thread-legacy")),
)
.await
.unwrap();
assert!(
payload(&read_after_clear)["todos"]
.as_array()
.unwrap()
.is_empty(),
"a cleared migrated list must not be resurrected from the legacy key"
);
}
#[tokio::test]
async fn sessions_do_not_see_each_other_and_a_list_survives_across_turns() {
let workspace_dir = test_workspace();
let a = TodoScope::Session {
id: "sess-a".into(),
};
let b = TodoScope::Session {
id: "sess-b".into(),
};
tinyagents_harness_tools::todos::store::clear(&todo_store(&workspace_dir), a.key())
.await
.unwrap();
tinyagents_harness_tools::todos::store::clear(&todo_store(&workspace_dir), b.key())
.await
.unwrap();
let item = TodoItem::with_status("only in a", TodoStatus::InProgress);
tinyagents_harness_tools::todos::store::replace(
&todo_store(&workspace_dir),
a.key(),
vec![item],
)
.await
.unwrap();
let a_again =
tinyagents_harness_tools::todos::store::list(&todo_store(&workspace_dir), a.key())
.await
.unwrap();
assert_eq!(
a_again.items.len(),
1,
"a later turn of the same session reads it back"
);
assert_eq!(a_again.thread_id, "sess-a");
assert!(
tinyagents_harness_tools::todos::store::list(&todo_store(&workspace_dir), b.key())
.await
.unwrap()
.items
.is_empty()
);
}
/// Runs `arguments` through the tool and returns the list the frontend's todo
/// drawer would be sent (`ThreadTodosChanged`), if any.
async fn sent_to_the_drawer(tool: &TodoTool, arguments: Value) -> Option<Value> {
let writes = writes_plan(&arguments);
let result = tool.execute(arguments).await.unwrap();
changed_todos(writes, &result)
}
fn drawer_statuses(todos: &Value) -> Vec<&str> {
todos
.as_array()
.unwrap()
.iter()
.map(|item| item["status"].as_str().unwrap())
.collect()
}
/// Every call that changes the plan reaches the drawer with the new list, in
/// each shape the tool takes (whole list, a `complete` list or one sent as a
/// string, `cancel` and `complete` together, a tick with notes); a read, a
/// notes-only call and a refused call send nothing.
#[tokio::test]
async fn every_plan_change_reaches_the_todo_drawer_and_nothing_else_does() {
let _guard = scratch_lock_async().await;
let workspace_dir = test_workspace();
reset_scratch(&workspace_dir).await;
let tool = TodoTool::new(workspace_dir.clone());
let written = sent_to_the_drawer(
&tool,
json!({ "todos": [
{ "content": "Inspect", "status": "pending" },
{ "content": "Configure", "status": "pending" },
{ "content": "Create list", "status": "pending" },
{ "content": "Start", "status": "pending" },
{ "content": "Verify", "status": "pending" }
] }),
)
.await
.expect("a whole-list write is sent");
assert_eq!(drawer_statuses(&written)[0], "in_progress");
let ticked = sent_to_the_drawer(&tool, json!({ "complete": [1, 2] }))
.await
.expect("a `complete` list is sent");
assert_eq!(
drawer_statuses(&ticked),
[
"completed",
"completed",
"in_progress",
"pending",
"pending"
]
);
let combined = sent_to_the_drawer(
&tool,
json!({ "complete": "[3]", "cancel": 4, "reason": "not needed" }),
)
.await
.expect("`complete` (as a string) with `cancel` is sent");
assert_eq!(
drawer_statuses(&combined),
[
"completed",
"completed",
"completed",
"cancelled",
"in_progress"
]
);
assert_eq!(combined[3]["reason"], "not needed");
assert!(
sent_to_the_drawer(&tool, json!({ "facts": ["postfix runs"] }))
.await
.is_none(),
"notes alone leave the plan as it was"
);
assert!(
sent_to_the_drawer(&tool, json!({})).await.is_none(),
"a read changes nothing"
);
let refused = tool.execute(json!({ "complete": 9 })).await.unwrap();
assert!(refused.is_error, "{}", refused.output());
assert!(
changed_todos(true, &refused).is_none(),
"a refused call leaves the plan as it was"
);
let finished = sent_to_the_drawer(&tool, json!({ "complete": 5, "facts": ["eval.py passes"] }))
.await
.expect("a tick that carries notes is sent");
assert_eq!(
drawer_statuses(&finished),
[
"completed",
"completed",
"completed",
"cancelled",
"completed"
]
);
reset_scratch(&workspace_dir).await;
}