Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust
7.3 KB
Raw Blame
//! `ai-memory llm-test` — smoke test an LLM provider end-to-end.

use ai_memory_llm::{ChatRequest, LlmError, ProviderChoice, ProviderConfig, build_provider};
use anyhow::{Context, Result};
use tracing::info;

use crate::cli::{LlmProviderChoice, LlmTestArgs};
use crate::config::Config;
use crate::http_client::{ServerEndpoint, post_json};

/// Run the `llm-test` subcommand.
///
/// # Errors
/// Returns an error if the provider cannot be constructed, the env
/// lacks the required keys, or the HTTP call fails.
pub async fn run(config: &Config, args: LlmTestArgs) -> Result<()> {
    if args.server {
        return run_against_server(config).await;
    }
    let provider = ProviderChoice::from(
        args.provider
            .context("--provider is required unless --server is passed")?,
    );
    let api_key_override = args
        .api_key
        .filter(|s| !s.is_empty())
        .map(secrecy::SecretString::from);
    let provider_config = ProviderConfig {
        provider,
        model: args
            .model
            .context("--model is required unless --server is passed")?,
        auth: config.provider_auth(provider, api_key_override),
        base_url: args.base_url.or_else(|| config.llm_test_base_url(provider)),
        compat_strict: config.llm_compat_strict,
        compat_disable_thinking: config.llm_compat_disable_thinking,
        request_timeout_secs: config.llm_timeout_secs,
        reasoning_effort: config.llm_reasoning_effort,
        extra_headers: config
            .llm_extra_headers()
            .context("parsing AI_MEMORY_LLM_HEADERS / llm_headers")?,
    };
    let client = build_provider(provider_config).context("building LLM provider")?;
    info!(
        provider = client.name(),
        model = client.model(),
        "sending prompt",
    );
    let request = representative_request(
        args.prompt
            .context("--prompt is required unless --server is passed")?,
    );
    if args.structured {
        let value = client
            .complete_structured_raw(
                request,
                serde_json::json!({
                    "type": "object",
                    "properties": {"answer": {"type": "string"}},
                    "required": ["answer"]
                }),
            )
            .await
            .map_err(|e| with_oauth_hint(provider, e))
            .context("calling provider for structured output")?;
        println!("--- model: {} ---", client.model());
        println!("{}", serde_json::to_string_pretty(&value)?);
        return Ok(());
    }
    let resp = client
        .complete(request)
        .await
        .map_err(|e| with_oauth_hint(provider, e))
        .context("calling provider")?;

    println!("--- model: {} ---", resp.model);
    if let Some(u) = resp.usage {
        println!(
            "--- usage: in={} out={} ---",
            u.input_tokens, u.output_tokens
        );
    }
    println!("{}", resp.text);
    Ok(())
}

/// Anthropic answers a subscription token it will not serve with a bare 429
/// that reads like a real rate limit (#1170).
const ANTHROPIC_OAUTH_429_HINT: &str = "Anthropic often answers `claude setup-token` \
tokens with this 429 for any client that is not Claude Code; it is not a real \
rate limit. Use `--provider anthropic` with an API key or an OpenAI-compatible \
gateway (see docs/llm-providers.md).";

fn with_oauth_hint(provider: ProviderChoice, err: LlmError) -> anyhow::Error {
    let is_oauth_429 = provider == ProviderChoice::AnthropicOAuth
        && matches!(err, LlmError::Provider { status: 429, .. });
    let err = anyhow::Error::new(err);
    if is_oauth_429 {
        err.context(ANTHROPIC_OAUTH_429_HINT)
    } else {
        err
    }
}

/// Use the same sampling value as bootstrap and consolidation so this smoke
/// test exercises provider-specific request normalization.
fn representative_request(prompt: String) -> ChatRequest {
    let mut request = ChatRequest::user_prompt(prompt);
    request.temperature = Some(0.2);
    request
}

impl From<LlmProviderChoice> for ProviderChoice {
    fn from(value: LlmProviderChoice) -> Self {
        match value {
            LlmProviderChoice::Anthropic => Self::Anthropic,
            LlmProviderChoice::AnthropicOauth => Self::AnthropicOAuth,
            LlmProviderChoice::Openai => Self::OpenAi,
            LlmProviderChoice::Gemini => Self::Gemini,
            LlmProviderChoice::OpenaiCompat => Self::OpenAiCompat,
            LlmProviderChoice::OpenaiOauth => Self::OpenAiOAuth,
            LlmProviderChoice::Codex => Self::Codex,
            LlmProviderChoice::Copilot => Self::Copilot,
            LlmProviderChoice::Opencode => Self::OpenCode,
            LlmProviderChoice::Cursor => Self::Cursor,
        }
    }
}

/// Probe the server's own configured provider (`--server`).
///
/// The server runs the probe with a fixed short prompt against what its
/// configuration holds, so this works from a machine that has no keys at all —
/// which is the whole point (#1639).
async fn run_against_server(config: &Config) -> Result<()> {
    let endpoint = ServerEndpoint::from_config_resolving_auth(config).await;
    let report: serde_json::Value = post_json(&endpoint, "/admin/llm-test", &()).await?;
    let provider = report["provider"].as_str().unwrap_or("(unknown)");
    let model = report["model"].as_str().unwrap_or("(unknown)");
    let latency = report["latency_ms"].as_u64().unwrap_or(0);
    println!("--- server provider: {provider} / {model} ({latency} ms) ---");
    println!("{}", serde_json::to_string_pretty(&report)?);
    Ok(())
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn anthropic_oauth_choice_maps_to_runtime_provider() {
        assert_eq!(
            ProviderChoice::from(LlmProviderChoice::AnthropicOauth),
            ProviderChoice::AnthropicOAuth
        );
    }

    #[test]
    fn cursor_choice_maps_to_runtime_provider() {
        assert_eq!(
            ProviderChoice::from(LlmProviderChoice::Cursor),
            ProviderChoice::Cursor
        );
    }

    #[test]
    fn codex_choice_maps_to_runtime_provider() {
        assert_eq!(
            ProviderChoice::from(LlmProviderChoice::Codex),
            ProviderChoice::Codex
        );
    }

    #[test]
    fn anthropic_oauth_429_carries_the_hint() {
        let err = with_oauth_hint(
            ProviderChoice::AnthropicOAuth,
            LlmError::Provider {
                status: 429,
                body: "rate_limit_error".into(),
                retry_after: None,
            },
        );
        assert!(format!("{err:#}").contains("not a real rate limit"));
    }

    #[test]
    fn other_providers_and_statuses_carry_no_hint() {
        for (provider, status) in [
            (ProviderChoice::Anthropic, 429),
            (ProviderChoice::AnthropicOAuth, 500),
        ] {
            let err = with_oauth_hint(
                provider,
                LlmError::Provider {
                    status,
                    body: String::new(),
                    retry_after: None,
                },
            );
            assert!(!format!("{err:#}").contains("not a real rate limit"));
        }
    }

    #[test]
    fn llm_test_exercises_pipeline_sampling_compatibility() {
        let request = representative_request("diagnostic".into());

        assert_eq!(request.temperature, Some(0.2));
        assert_eq!(request.messages[0].content, "diagnostic");
    }
}