Skip to content

The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.

7.9 KB
Raw Blame
use crate::channels::runtime::TestChannel;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Mutex;
use std::time::Duration;
use tempfile::TempDir;
use tinyinference_llm::message::{AssistantMessage, Message};
use tinyinference_llm::model::{ChatModel, ModelProfile, ModelRequest, ModelResponse};
use tinyinference_llm::tool::ToolCall;
use tinytools::{Tool, ToolResult};

fn message_role(message: &Message) -> &'static str {
    match message {
        Message::System(_) => "system",
        Message::User(_) => "user",
        Message::Assistant(_) => "assistant",
        Message::Tool(_) => "tool",
        Message::Custom(_) => "custom",
    }
}

fn native_tool_profile() -> &'static ModelProfile {
    static PROFILE: std::sync::OnceLock<ModelProfile> = std::sync::OnceLock::new();
    PROFILE.get_or_init(|| {
        let mut profile = ModelProfile::default();
        profile.tool_calling = true;
        profile.parallel_tool_calls = true;
        profile
    })
}

fn tool_call_response(step: Option<usize>) -> ModelResponse {
    let mut arguments = serde_json::json!({"symbol": "BTC"});
    if let Some(step) = step {
        arguments["step"] = serde_json::json!(step);
    }
    ModelResponse {
        message: AssistantMessage {
            id: None,
            content: Vec::new(),
            tool_calls: vec![ToolCall::new("mock-price-call", "mock_price", arguments)],
            usage: None,
            origin: None,
        },
        usage: None,
        finish_reason: Some("tool_calls".to_string()),
        raw: None,
        resolved_model: None,
        continue_turn: None,
        served_from_cache: false,
        correlation: None,
        resolved_route: None,
    }
}

// Note: the shared bus handler lock and the "install the real agent
// handler for this test" helper both live in
// `crate::agent::bus` as `BUS_HANDLER_LOCK` (re-exported from
// `crate::core::bus`) and `use_real_agent_handler` so any
// test in the workspace can drive the real `agent.run_turn` path without
// depending on channels-specific scaffolding.
//
// For stub installations use `mock_agent_run_turn` (also in
// `crate::agent::bus`) or the generic `mock_bus_stub` in
// `crate::core::bus` for arbitrary bus methods.
pub(super) use crate::agent::bus::use_real_agent_handler;

pub(super) fn make_workspace() -> TempDir {
    let tmp = TempDir::new().unwrap();
    // Create minimal workspace files — only the bundled identity prompts.
    std::fs::write(tmp.path().join("SOUL.md"), "# Soul\nBe helpful.").unwrap();
    std::fs::write(
        tmp.path().join("IDENTITY.md"),
        "# Identity\nName: OpenHuman",
    )
    .unwrap();
    tmp
}

pub(super) struct DummyModel;

#[async_trait::async_trait]
impl ChatModel<()> for DummyModel {
    async fn invoke(
        &self,
        _state: &(),
        _request: ModelRequest,
    ) -> tinyinference_llm::Result<ModelResponse> {
        Ok(ModelResponse::assistant("ok"))
    }
}

pub(super) type RecordingChannel = TestChannel;

pub(super) struct SlowModel {
    pub(super) delay: Duration,
}

#[async_trait::async_trait]
impl ChatModel<()> for SlowModel {
    async fn invoke(
        &self,
        _state: &(),
        request: ModelRequest,
    ) -> tinyinference_llm::Result<ModelResponse> {
        tokio::time::sleep(self.delay).await;
        let message = request
            .messages
            .iter()
            .rev()
            .find(|message| matches!(message, Message::User(_)))
            .map(Message::text)
            .unwrap_or_default();
        Ok(ModelResponse::assistant(format!("echo: {message}")))
    }
}

pub(super) struct ToolCallingModel;

#[async_trait::async_trait]
impl ChatModel<()> for ToolCallingModel {
    fn profile(&self) -> Option<&ModelProfile> {
        Some(native_tool_profile())
    }

    async fn invoke(
        &self,
        _state: &(),
        request: ModelRequest,
    ) -> tinyinference_llm::Result<ModelResponse> {
        let has_tool_results = request.messages.iter().any(|message| {
            matches!(message, Message::Tool(_)) || message.text().contains("[Tool results]")
        });
        if has_tool_results {
            Ok(ModelResponse::assistant(
                "BTC is currently around $65,000 based on latest tool output.",
            ))
        } else {
            Ok(tool_call_response(None))
        }
    }
}

pub(super) struct IterativeToolModel {
    pub(super) required_tool_iterations: usize,
}

impl IterativeToolModel {
    pub(super) fn completed_tool_iterations(messages: &[Message]) -> usize {
        messages
            .iter()
            .filter(|message| {
                matches!(message, Message::Tool(_)) || message.text().contains("[Tool results]")
            })
            .count()
    }
}

#[async_trait::async_trait]
impl ChatModel<()> for IterativeToolModel {
    fn profile(&self) -> Option<&ModelProfile> {
        Some(native_tool_profile())
    }

    async fn invoke(
        &self,
        _state: &(),
        request: ModelRequest,
    ) -> tinyinference_llm::Result<ModelResponse> {
        let completed_iterations = Self::completed_tool_iterations(&request.messages);
        if completed_iterations >= self.required_tool_iterations {
            Ok(ModelResponse::assistant(format!(
                "Completed after {completed_iterations} tool iterations."
            )))
        } else {
            // Prefix a per-iteration progress note so each turn's assistant
            // output is distinct. A healthy multi-step agent varies its
            // narration as it advances; only byte-identical repeats (the
            // degeneration signature) should trip the harness repeat guard.
            Ok(tool_call_response(Some(completed_iterations)))
        }
    }
}

#[derive(Default)]
pub(super) struct HistoryCaptureModel {
    pub(super) calls: Mutex<Vec<Vec<(String, String)>>>,
}

#[async_trait::async_trait]
impl ChatModel<()> for HistoryCaptureModel {
    async fn invoke(
        &self,
        _state: &(),
        request: ModelRequest,
    ) -> tinyinference_llm::Result<ModelResponse> {
        let snapshot = request
            .messages
            .iter()
            .map(|message| (message_role(message).to_string(), message.text()))
            .collect::<Vec<_>>();
        let mut calls = self.calls.lock().unwrap_or_else(|e| e.into_inner());
        calls.push(snapshot);
        Ok(ModelResponse::assistant(format!(
            "response-{}",
            calls.len()
        )))
    }
}

pub(super) struct MockPriceTool;

#[derive(Default)]
pub(super) struct ModelCaptureModel {
    pub(super) call_count: AtomicUsize,
    pub(super) models: Mutex<Vec<String>>,
    label: String,
}

impl ModelCaptureModel {
    pub(super) fn new(label: impl Into<String>) -> Self {
        Self {
            label: label.into(),
            ..Self::default()
        }
    }
}

#[async_trait::async_trait]
impl ChatModel<()> for ModelCaptureModel {
    async fn invoke(
        &self,
        _state: &(),
        _request: ModelRequest,
    ) -> tinyinference_llm::Result<ModelResponse> {
        self.call_count.fetch_add(1, Ordering::SeqCst);
        self.models
            .lock()
            .unwrap_or_else(|e| e.into_inner())
            .push(self.label.clone());
        Ok(ModelResponse::assistant("ok"))
    }
}

#[async_trait::async_trait]
impl Tool for MockPriceTool {
    fn name(&self) -> &str {
        "mock_price"
    }

    fn description(&self) -> &str {
        "Return a mocked BTC price"
    }

    fn parameters_schema(&self) -> serde_json::Value {
        serde_json::json!({
            "type": "object",
            "properties": {
                "symbol": { "type": "string" }
            },
            "required": ["symbol"]
        })
    }

    async fn execute(&self, args: serde_json::Value) -> anyhow::Result<ToolResult> {
        let symbol = args.get("symbol").and_then(serde_json::Value::as_str);
        if symbol != Some("BTC") {
            return Ok(ToolResult::error("unexpected symbol"));
        }

        Ok(ToolResult::success("BTC is $65,000"))
    }
}