Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust
23.5 KB
Raw Blame
//! Opt-in, live-LLM QA-accuracy mode for the retrieval harness (R2b).
//!
//! R2a scores *retrieval* (did the evidence session surface in the top k).
//! This module measures the next hop: end-to-end **answer quality**. Per
//! scored question, with QA enabled, it
//!
//! 1. obtains a candidate answer — either the server's own `answer` field
//!    (future `answer=true` feature; used verbatim when present) or, by
//!    default, a harness synthesis call over the retrieved snippets (the
//!    exact context an agent would receive);
//! 2. grades that candidate against the dataset's GOLD answer with an
//!    LLM-as-judge, following the LongMemEval convention (a boolean verdict
//!    plus a one-line reason);
//! 3. records the verdict, the answer source, and the answer-path latency +
//!    token estimate, which [`super::score`] aggregates into QA accuracy
//!    alongside the R2a triple.
//!
//! This makes REAL provider API calls and is gated behind a key. It is OFF
//! by default; when it is requested but a provider/key cannot be resolved,
//! the caller SKIPS QA (retrieval metrics are unaffected) rather than
//! failing the run. Provider construction and structured output reuse the
//! same `ai-memory-llm` path the product uses; keys are read from env by the
//! provider auth boundary and never logged, printed, or written to reports.

use std::sync::Arc;
use std::time::Instant;

use anyhow::{Result, bail};
use schemars::JsonSchema;
use secrecy::SecretString;
use serde::{Deserialize, Serialize};

use ai_memory_llm::{
    AuthRequirement, ChatMessage, ChatRequest, LlmProvider, ProviderAuth, ProviderChoice,
    ProviderConfig, Role, complete_structured,
};

use super::query::{HitContext, QueryOutcome};
use super::report::QaReportMeta;

/// Shared opt-in QA provider arguments. Both retrieval QA and the abstention
/// benchmark use this exact construction path so auth/provider behavior cannot
/// drift between evaluators.
#[derive(clap::Args, Debug, Default)]
pub struct QaArgs {
    /// Enable live-LLM QA. OFF by default; missing credentials produce a skip.
    #[arg(long = "qa")]
    pub enabled: bool,
    /// QA answerer/judge provider.
    #[arg(long)]
    qa_provider: Option<String>,
    /// QA answerer model id.
    #[arg(long)]
    qa_model: Option<String>,
    /// QA answerer base URL (omit for native vendors).
    #[arg(long)]
    qa_base_url: Option<String>,
    /// QA answerer API key. Prefer `--qa-api-key-env`.
    #[arg(long)]
    qa_api_key: Option<String>,
    /// Env var containing the QA answerer API key.
    #[arg(long)]
    qa_api_key_env: Option<String>,
    /// Auth file for OAuth/Codex/Copilot answerers.
    #[arg(long)]
    qa_token_file: Option<std::path::PathBuf>,
    /// QA grader provider; defaults to the answerer.
    #[arg(long)]
    qa_grader_provider: Option<String>,
    /// QA grader model; defaults to the answerer.
    #[arg(long)]
    qa_grader_model: Option<String>,
    /// QA grader base URL; defaults to the answerer.
    #[arg(long)]
    qa_grader_base_url: Option<String>,
    /// QA grader API key; defaults to the answerer.
    #[arg(long)]
    qa_grader_api_key: Option<String>,
    /// Env var for the QA grader API key; defaults to the answerer.
    #[arg(long)]
    qa_grader_api_key_env: Option<String>,
    /// QA grader auth file; defaults to the answerer.
    #[arg(long)]
    qa_grader_token_file: Option<std::path::PathBuf>,
}

/// chars/4 heuristic divisor, matching `query.rs`'s context-token estimate.
const CHARS_PER_TOKEN: usize = 4;
/// Cap on synthesized-answer output tokens (LongMemEval answers are short).
const ANSWER_MAX_TOKENS: u32 = 512;
/// Cap on grader output tokens (a verdict plus one sentence).
const GRADE_MAX_TOKENS: u32 = 256;

/// Per-question QA outcome: a candidate answer graded against the gold
/// answer. Serialized into the per-question report; carries no key material.
#[derive(Debug, Clone, Serialize)]
pub struct QaOutcome {
    /// LLM-judge verdict: did the candidate correctly answer the question?
    pub correct: bool,
    /// Structured evidence-sufficiency verdict from synthesis (or from the
    /// server answer's validated citation presence).
    pub answerability: Answerability,
    /// Where the candidate came from: `synthesized` (harness LLM over the
    /// retrieved snippets) or `memory_query` (a server-provided `answer`).
    pub answer_source: &'static str,
    /// The candidate answer that was graded (kept for forensics).
    pub answer: String,
    /// One-line judge rationale.
    pub grade_reason: String,
    /// Wall-clock of the answer step, milliseconds. 0 when the answer came
    /// straight from the `memory_query` result (no synthesis call was made).
    pub answer_latency_ms: u128,
    /// chars/4 estimate over the synthesis prompt + answer — the answer-path
    /// analogue of the retrieval context-token estimate. 0 for a
    /// server-provided answer.
    pub answer_tokens: usize,
}

/// Answer synthesized from retrieved snippets (structured output).
#[derive(Debug, Clone, Deserialize, JsonSchema)]
pub(crate) struct SynthesizedAnswer {
    /// Typed verdict; evaluators never infer abstention from answer prose.
    answerability: Answerability,
    /// The answer to the question, grounded strictly in the snippets.
    answer: String,
}

/// Structured answerability classification used by both QA evaluators.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
#[serde(rename_all = "snake_case")]
pub enum Answerability {
    /// The snippets contain enough evidence for the requested answer.
    Answerable,
    /// The snippets do not contain enough evidence.
    Insufficient,
    /// The snippets contain incompatible live claims.
    Conflicting,
    /// The snippets support only part of the requested answer.
    Partial,
}

/// LLM-as-judge verdict (structured output), LongMemEval-style.
#[derive(Debug, Clone, Deserialize, JsonSchema)]
struct Grade {
    /// True when the candidate answer is factually correct for the question,
    /// judged against the gold answer.
    correct: bool,
    /// One-sentence justification for the verdict.
    reason: String,
}

/// A resolved QA engine: the answerer + grader providers and their names,
/// built once and shared (cloned `Arc`) across concurrent question tasks.
pub struct QaEngine {
    answerer: Arc<dyn LlmProvider>,
    grader: Arc<dyn LlmProvider>,
    answer_provider: String,
    answer_model: String,
    grader_provider: String,
    grader_model: String,
}

impl QaEngine {
    /// Resolve both providers from `--qa-*` args + env.
    ///
    /// Returns `Err` (which the caller turns into a clean SKIP) when QA is
    /// requested but a provider is missing, an API key/token cannot be
    /// resolved, or provider construction fails. The default env var for a
    /// provider's key is used when the operator does not name one, so
    /// `--qa --qa-provider gemini --qa-model <m>` works with `GEMINI_API_KEY`
    /// already in the environment.
    pub fn resolve(args: &QaArgs) -> Result<Self> {
        let Some(answer_provider_str) = args.qa_provider.as_deref() else {
            bail!("QA mode needs --qa-provider (e.g. anthropic|gemini|openai|openai-compat)");
        };
        let Some(answer_model) = args.qa_model.as_deref() else {
            bail!("QA mode needs --qa-model");
        };
        let answerer = build_llm_provider(
            "answerer",
            answer_provider_str,
            answer_model,
            args.qa_base_url.as_deref(),
            args.qa_api_key.as_deref(),
            args.qa_api_key_env.as_deref(),
            args.qa_token_file.as_deref(),
        )?;

        // The grader defaults to the answerer's provider/model/key/URL (the
        // common LongMemEval setup: one judge for everything), each field
        // overridable independently.
        let grader_provider_str = args
            .qa_grader_provider
            .as_deref()
            .unwrap_or(answer_provider_str);
        let grader_model = args.qa_grader_model.as_deref().unwrap_or(answer_model);
        let grader = build_llm_provider(
            "grader",
            grader_provider_str,
            grader_model,
            args.qa_grader_base_url.or_ref(&args.qa_base_url),
            args.qa_grader_api_key.or_ref(&args.qa_api_key),
            args.qa_grader_api_key_env.or_ref(&args.qa_api_key_env),
            args.qa_grader_token_file
                .as_deref()
                .or(args.qa_token_file.as_deref()),
        )?;

        Ok(Self {
            answer_provider: answerer.name().to_string(),
            answer_model: answerer.model().to_string(),
            grader_provider: grader.name().to_string(),
            grader_model: grader.model().to_string(),
            answerer,
            grader,
        })
    }

    /// Report provenance (names only — never keys).
    pub fn report_meta(&self) -> QaReportMeta {
        QaReportMeta {
            answer_provider: self.answer_provider.clone(),
            answer_model: self.answer_model.clone(),
            grader_provider: self.grader_provider.clone(),
            grader_model: self.grader_model.clone(),
        }
    }

    /// Short one-line description for the run header (names only).
    pub fn summary(&self) -> String {
        format!(
            "answerer={} {} | grader={} {}",
            self.answer_provider, self.answer_model, self.grader_provider, self.grader_model
        )
    }

    /// Produce and grade a candidate answer for one question.
    pub async fn evaluate(
        &self,
        question: &str,
        gold_answer: &str,
        outcome: &QueryOutcome,
    ) -> Result<QaOutcome> {
        let (answer, answerability, answer_source, answer_latency_ms, answer_tokens) =
            match &outcome.answer {
                // Future `answer=true` feature: the server already produced the
                // answer, so grade it directly (no synthesis call).
                Some(server_answer) => (
                    server_answer.text.clone(),
                    if server_answer.citations.is_empty() {
                        Answerability::Insufficient
                    } else {
                        Answerability::Answerable
                    },
                    "memory_query",
                    0u128,
                    0usize,
                ),
                None => {
                    let synth = self.synthesize(question, &outcome.contexts).await?;
                    (
                        synth.answer,
                        synth.answerability,
                        "synthesized",
                        synth.latency_ms,
                        synth.tokens,
                    )
                }
            };
        let grade = self.grade(question, gold_answer, &answer).await?;
        Ok(QaOutcome {
            correct: grade.correct,
            answerability,
            answer_source,
            answer,
            grade_reason: grade.reason,
            answer_latency_ms,
            answer_tokens,
        })
    }

    /// Synthesize an answer from the retrieved snippets (the context an agent
    /// would actually get), measuring latency and a chars/4 token estimate.
    pub(crate) async fn synthesize(
        &self,
        question: &str,
        contexts: &[HitContext],
    ) -> Result<SynthResult> {
        let user = synthesis_user_prompt(question, contexts);
        let prompt_chars = SYNTHESIS_SYSTEM.chars().count() + user.chars().count();
        let request = ChatRequest {
            system: Some(SYNTHESIS_SYSTEM.to_string()),
            messages: vec![ChatMessage {
                role: Role::User,
                content: user,
            }],
            max_tokens: ANSWER_MAX_TOKENS,
            temperature: Some(0.0),
        };
        let started = Instant::now();
        let out: SynthesizedAnswer = complete_structured(self.answerer.as_ref(), request).await?;
        let latency_ms = started.elapsed().as_millis();
        let tokens = (prompt_chars + out.answer.chars().count()).div_ceil(CHARS_PER_TOKEN);
        Ok(SynthResult {
            answer: out.answer,
            answerability: out.answerability,
            latency_ms,
            tokens,
        })
    }

    /// Grade a candidate answer against the gold answer (LLM-as-judge).
    async fn grade(&self, question: &str, gold: &str, candidate: &str) -> Result<Grade> {
        let request = ChatRequest {
            system: Some(GRADE_SYSTEM.to_string()),
            messages: vec![ChatMessage {
                role: Role::User,
                content: grade_user_prompt(question, gold, candidate),
            }],
            max_tokens: GRADE_MAX_TOKENS,
            temperature: Some(0.0),
        };
        let grade: Grade = complete_structured(self.grader.as_ref(), request).await?;
        Ok(grade)
    }
}

/// Internal synthesis result carrier.
pub(crate) struct SynthResult {
    pub answer: String,
    pub answerability: Answerability,
    pub latency_ms: u128,
    pub tokens: usize,
}

/// Convenience: borrow an inner `Option<String>` as `Option<&str>`, falling
/// back to a default, so grader overrides read cleanly.
trait OrRef {
    fn or_ref<'a>(&'a self, fallback: &'a Option<String>) -> Option<&'a str>;
}
impl OrRef for Option<String> {
    fn or_ref<'a>(&'a self, fallback: &'a Option<String>) -> Option<&'a str> {
        self.as_deref().or(fallback.as_deref())
    }
}

const SYNTHESIS_SYSTEM: &str = "You answer a user's question using ONLY the memory snippets \
    retrieved for them. Base your answer strictly on those snippets — do not use outside \
    knowledge or guess. Classify `answerability` as `answerable`, `insufficient`, `conflicting`, \
    or `partial`, then give a concise factual `answer`. For anything except `answerable`, explain \
    the evidence limitation without filling the gap from outside knowledge.";

const GRADE_SYSTEM: &str = "You are a strict grader for a long-term-memory QA benchmark. Given a \
    question, the gold (reference) answer, and a candidate answer, decide whether the candidate \
    is CORRECT. Judge factual correctness, not wording: a paraphrase or extra detail is fine \
    when it matches the gold answer's facts. Mark it incorrect when it contradicts the gold \
    answer, omits the key fact the question asks for, or declines to answer. Return a boolean \
    `correct` and a one-sentence `reason`.";

/// Build the synthesis user prompt: the question plus ranked snippets.
fn synthesis_user_prompt(question: &str, contexts: &[HitContext]) -> String {
    let mut prompt = format!("Question:\n{question}\n\n");
    if contexts.is_empty() {
        prompt.push_str("Retrieved memory snippets: (none)\n");
    } else {
        prompt.push_str("Retrieved memory snippets (ranked, best first):\n");
        for (i, c) in contexts.iter().enumerate() {
            prompt.push_str(&format!("[{}] {}\n{}\n\n", i + 1, c.title, c.snippet));
        }
    }
    prompt.push_str("\nAnswer the question using only the snippets above.");
    prompt
}

/// Build the grader user prompt.
fn grade_user_prompt(question: &str, gold: &str, candidate: &str) -> String {
    format!(
        "Question:\n{question}\n\nGold answer:\n{gold}\n\nCandidate answer:\n{candidate}\n\n\
         Is the candidate answer correct?"
    )
}

/// Resolve one provider from explicit fields (mirrors the `ab` harness's
/// resolution, reused here for the QA answerer + grader). Fails closed when a
/// required key/token is absent so the caller can SKIP cleanly.
fn build_llm_provider(
    role: &str,
    provider_str: &str,
    model: &str,
    base_url: Option<&str>,
    api_key_inline: Option<&str>,
    api_key_env: Option<&str>,
    token_file: Option<&std::path::Path>,
) -> Result<Arc<dyn LlmProvider>> {
    let provider = match provider_str {
        "anthropic" => ProviderChoice::Anthropic,
        "openai" => ProviderChoice::OpenAi,
        "openai-compat" | "openai_compat" => ProviderChoice::OpenAiCompat,
        "openai-oauth" | "openai_oauth" => ProviderChoice::OpenAiOAuth,
        "codex" => ProviderChoice::Codex,
        "copilot" | "github-copilot" | "github_copilot" => ProviderChoice::Copilot,
        "gemini" | "google" => ProviderChoice::Gemini,
        other => bail!(
            "{role}: provider `{other}` not one of \
             anthropic|openai|openai-compat|openai-oauth|codex|copilot|gemini"
        ),
    };

    // The env var to read the key from: operator override, else the
    // provider's own canonical env var.
    let requirement = provider.auth_requirement();
    let default_env = match &requirement {
        AuthRequirement::RequiredApiKey { env_var }
        | AuthRequirement::OptionalApiKey { env_var } => Some(*env_var),
        _ => None,
    };
    let eff_env = api_key_env.or(default_env);

    let api_key = match api_key_inline {
        Some(s) if !s.is_empty() => Some(SecretString::from(s.to_string())),
        _ => eff_env.and_then(|env| match std::env::var(env) {
            Ok(v) if !v.is_empty() => Some(SecretString::from(v)),
            _ => None,
        }),
    };

    let auth = match requirement {
        AuthRequirement::RequiredApiKey { env_var } => {
            if api_key.is_none() {
                bail!(
                    "{role}: provider `{provider_str}` requires an API key; \
                     set {} (or pass --qa-api-key)",
                    eff_env.unwrap_or(env_var)
                );
            }
            ProviderAuth::required_api_key_from_env(env_var, api_key)
        }
        AuthRequirement::OptionalApiKey { env_var } => {
            ProviderAuth::optional_api_key_from_env(env_var, api_key)
        }
        AuthRequirement::OpenAiOAuthToken => {
            let tf = token_file.ok_or_else(|| {
                anyhow::anyhow!("{role}: --qa-token-file required for openai-oauth")
            })?;
            ProviderAuth::openai_oauth_token_file(tf.to_path_buf())
        }
        AuthRequirement::CodexAuthFile => {
            let tf = token_file
                .ok_or_else(|| anyhow::anyhow!("{role}: --qa-token-file required for codex"))?;
            ProviderAuth::codex(tf.to_path_buf(), "codex")
        }
        AuthRequirement::CopilotToken => {
            let tf = token_file
                .ok_or_else(|| anyhow::anyhow!("{role}: --qa-token-file required for copilot"))?;
            ProviderAuth::copilot(tf.to_path_buf(), None, None, base_url.map(str::to_string))
        }
        AuthRequirement::AnthropicOAuthToken => ProviderAuth::anthropic_oauth_token(api_key),
        AuthRequirement::CursorCli => ProviderAuth::cursor(ai_memory_llm::CursorAuth::new(
            "agent",
            std::env::temp_dir(),
            &std::env::vars_os().collect::<Vec<_>>(),
        )),
    };

    let config = ProviderConfig {
        provider,
        model: model.to_string(),
        auth,
        base_url: base_url.map(str::to_string),
        compat_strict: true,
        // Comparisons must not differ by engine-side thinking behaviour;
        // keep the product default (off) for every candidate.
        compat_disable_thinking: false,
        request_timeout_secs: ai_memory_llm::DEFAULT_REQUEST_TIMEOUT_SECS,
        reasoning_effort: None,
        extra_headers: ai_memory_llm::ExtraHeaders::default(),
    };
    ai_memory_llm::build_provider(config)
        .map_err(anyhow::Error::from)
        .map_err(|e| e.context(format!("{role}: building provider `{provider_str}`")))
}

#[cfg(test)]
mod tests {
    use super::*;

    fn ctx(title: &str, snippet: &str) -> HitContext {
        HitContext {
            title: title.into(),
            snippet: snippet.into(),
        }
    }

    #[test]
    fn synthesis_prompt_numbers_snippets_and_carries_the_question() {
        let p = synthesis_user_prompt(
            "What car?",
            &[ctx("Car", "A blue Civic"), ctx("More", "2019")],
        );
        assert!(p.contains("Question:\nWhat car?"));
        assert!(p.contains("[1] Car\nA blue Civic"));
        assert!(p.contains("[2] More\n2019"));
        assert!(p.contains("only the snippets"));
    }

    #[test]
    fn synthesis_prompt_handles_no_snippets() {
        let p = synthesis_user_prompt("Anything?", &[]);
        assert!(p.contains("(none)"), "{p}");
    }

    #[test]
    fn grade_prompt_lays_out_all_three_fields() {
        let p = grade_user_prompt("Q?", "gold-x", "cand-y");
        assert!(p.contains("Question:\nQ?"));
        assert!(p.contains("Gold answer:\ngold-x"));
        assert!(p.contains("Candidate answer:\ncand-y"));
    }

    #[test]
    fn synthesis_requires_structured_answerability() {
        let parsed: SynthesizedAnswer = serde_json::from_value(serde_json::json!({
            "answerability": "insufficient",
            "answer": "The snippets do not contain the port."
        }))
        .unwrap();
        assert_eq!(parsed.answerability, Answerability::Insufficient);
        assert!(
            serde_json::from_value::<SynthesizedAnswer>(serde_json::json!({
                "answer": "I don't know."
            }))
            .is_err()
        );
    }

    #[test]
    fn unknown_provider_is_rejected() {
        let err = build_llm_provider("answerer", "nope", "m", None, None, None, None)
            .map(|_| ())
            .unwrap_err()
            .to_string();
        assert!(err.contains("not one of"), "{err}");
    }

    #[test]
    fn required_key_provider_without_a_key_fails_closed() {
        // Point at an env var that is (almost certainly) unset, and pass no
        // inline key: a required-key provider must refuse so the caller skips.
        let err = build_llm_provider(
            "answerer",
            "gemini",
            "gemini-2.5-flash",
            None,
            None,
            Some("AI_MEMORY_EVAL_QA_DEFINITELY_UNSET_KEY"),
            None,
        )
        .map(|_| ())
        .unwrap_err()
        .to_string();
        assert!(err.contains("requires an API key"), "{err}");
    }

    /// Live end-to-end QA smoke over a hand-made retrieval outcome (no eval
    /// server needed). `#[ignore]` so CI without keys stays green; even when
    /// run explicitly it skips cleanly when no key is present.
    #[tokio::test]
    #[ignore = "live LLM: requires GEMINI_API_KEY or ANTHROPIC_API_KEY"]
    async fn live_synthesize_and_grade_smoke() {
        let (provider, model, key_env) = if std::env::var("GEMINI_API_KEY").is_ok() {
            ("gemini", "gemini-2.5-flash", "GEMINI_API_KEY")
        } else if std::env::var("ANTHROPIC_API_KEY").is_ok() {
            ("anthropic", "claude-3-5-haiku-latest", "ANTHROPIC_API_KEY")
        } else {
            eprintln!("skipping: no GEMINI_API_KEY / ANTHROPIC_API_KEY in env");
            return;
        };
        let answerer =
            build_llm_provider("answerer", provider, model, None, None, Some(key_env), None)
                .expect("provider builds with a key present");
        let engine = QaEngine {
            answer_provider: answerer.name().to_string(),
            answer_model: answerer.model().to_string(),
            grader_provider: answerer.name().to_string(),
            grader_model: answerer.model().to_string(),
            grader: answerer.clone(),
            answerer,
        };
        let outcome = QueryOutcome {
            retrieved: vec![],
            contexts: vec![ctx("Pet", "The user has a dog named Rex.")],
            answer: None,
            latency: std::time::Duration::ZERO,
            context_tokens: 0,
            raw_hits: 0,
        };
        let qa = engine
            .evaluate("What is the user's dog's name?", "Rex", &outcome)
            .await
            .expect("QA evaluate succeeds with a live provider");
        assert_eq!(qa.answer_source, "synthesized");
        assert_eq!(qa.answerability, Answerability::Answerable);
        assert!(qa.correct, "expected the judge to accept: {qa:?}");
    }
}