Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust
24.2 KB
Raw Blame
//! Offline evidence-sufficiency and abstention evaluation.

use std::collections::{BTreeMap, HashSet};
use std::path::{Path, PathBuf};

use anyhow::{Context, Result, bail};
use jiff::Timestamp;
use serde::{Deserialize, Serialize};

use crate::retrieval::dataset::{self, DatasetVariant};
use crate::retrieval::grep;
use crate::retrieval::ingest::stored_session_uuid;
use crate::retrieval::qa::{Answerability, QaArgs, QaEngine};
use crate::retrieval::query::HitContext;
use crate::retrieval::report::QaReportMeta;

const FIXTURE_VERSION: u32 = 1;
const DEFAULT_THRESHOLDS: [f64; 11] = [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0];
const LONGMEMEVAL_CONTEXTS: usize = 10;
const LONGMEMEVAL_SNIPPET_CHARS: usize = 4_000;

#[derive(clap::Args, Debug)]
pub struct AbstentionArgs {
    /// Versioned clean-room answerability fixture.
    #[arg(long, default_value = "evals/fixtures/abstention/v1.json")]
    fixtures: PathBuf,
    /// Output root; a timestamped run directory is created below it.
    #[arg(long, default_value = "evals/runs")]
    out: PathBuf,
    /// Also evaluate the pinned LongMemEval-S `*_abs` questions. The pinned
    /// dataset must already exist; this command never downloads it.
    #[arg(long)]
    longmemeval_abs: bool,
    /// Existing QA provider flags, shared with `retrieval --qa`.
    #[command(flatten)]
    qa: QaArgs,
}

#[derive(Debug, Deserialize)]
#[serde(deny_unknown_fields)]
struct FixtureSet {
    version: u32,
    cases: Vec<Case>,
}

#[derive(Debug, Clone, Deserialize)]
#[serde(deny_unknown_fields)]
struct Case {
    id: String,
    question: String,
    expected_verdict: Verdict,
    slice: Slice,
    evidence: Vec<Evidence>,
    #[serde(default)]
    target_evidence: Vec<String>,
    #[serde(default)]
    gold_answer: Option<String>,
    #[serde(default)]
    channels: Channels,
}

#[derive(Debug, Clone, Deserialize)]
#[serde(deny_unknown_fields)]
struct Evidence {
    path: String,
    snippet: String,
    score: f64,
    #[serde(default)]
    belief_strength: Option<f64>,
}

#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
enum Verdict {
    Answerable,
    Unanswerable,
    Conflicting,
    Partial,
}

#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
enum Slice {
    Baseline,
    Unrelated,
    HardNegative,
    Conflicting,
    Partial,
    MissingChannel,
    LongmemevalAbstention,
}

impl Slice {
    fn name(self) -> &'static str {
        match self {
            Self::Baseline => "baseline",
            Self::Unrelated => "unrelated",
            Self::HardNegative => "hard_negative",
            Self::Conflicting => "conflicting",
            Self::Partial => "partial",
            Self::MissingChannel => "missing_channel",
            Self::LongmemevalAbstention => "longmemeval_abstention",
        }
    }
}

#[derive(Debug, Clone, Copy, Deserialize, Serialize)]
#[serde(default, deny_unknown_fields)]
struct Channels {
    fts: bool,
    vector: bool,
}

impl Default for Channels {
    fn default() -> Self {
        Self {
            fts: true,
            vector: true,
        }
    }
}

#[derive(Debug, Clone, Serialize)]
struct Signals {
    lexical_coverage: f64,
    top_score: f64,
    top_score_margin: Option<f64>,
    score_mean: f64,
    score_stddev: f64,
    snippet_matching_term_density: f64,
    snippet_chars: usize,
    belief_mean: Option<f64>,
    belief_observations: usize,
}

#[derive(Debug, Clone, Serialize)]
struct CaseResult {
    id: String,
    expected_verdict: Verdict,
    slice: Slice,
    channels: Channels,
    signals: Signals,
    #[serde(skip_serializing_if = "Option::is_none")]
    qa: Option<QaResult>,
}

#[derive(Debug, Clone, Serialize)]
struct QaResult {
    answerability: Answerability,
    matches_expected: bool,
    answer: String,
    latency_ms: u128,
    estimated_tokens: usize,
}

#[derive(Debug, Clone, Serialize)]
struct ThresholdMetrics {
    threshold: f64,
    supported: usize,
    unsupported: usize,
    ambiguous_excluded: usize,
    predicted_answerable: usize,
    false_supported_rate: Option<f64>,
    false_abstention_rate: Option<f64>,
    coverage: f64,
}

#[derive(Debug, Serialize)]
struct SliceReport {
    cases: usize,
    thresholds: Vec<ThresholdMetrics>,
}

#[derive(Debug, Serialize)]
struct Report {
    generated_at: String,
    commit: String,
    hardware: String,
    fixture_version: u32,
    fixture_path: String,
    decision_signal: &'static str,
    binary_mapping: &'static str,
    qa_status: String,
    #[serde(skip_serializing_if = "Option::is_none")]
    qa: Option<QaReportMeta>,
    qa_cases: usize,
    qa_verdict_matches: usize,
    qa_latency_p50_ms: Option<u128>,
    qa_latency_p95_ms: Option<u128>,
    seed: Option<u64>,
    cases: Vec<CaseResult>,
    thresholds: Vec<ThresholdMetrics>,
    slices: BTreeMap<String, SliceReport>,
}

pub async fn run(args: AbstentionArgs) -> Result<()> {
    let mut fixture = load_fixture(&args.fixtures)?;
    if args.longmemeval_abs {
        fixture.cases.extend(load_longmemeval_abstentions()?);
    }

    let (qa_engine, qa_status) = if args.qa.enabled {
        match QaEngine::resolve(&args.qa) {
            Ok(engine) => {
                let status = format!("enabled: {}", engine.summary());
                (Some(engine), status)
            }
            Err(error) => (None, format!("skipped: {error:#}")),
        }
    } else {
        (None, "disabled (offline zero-LLM run)".to_string())
    };

    let mut results = Vec::with_capacity(fixture.cases.len());
    for case in &fixture.cases {
        let qa = if let Some(engine) = &qa_engine {
            let contexts = case
                .evidence
                .iter()
                .map(|e| HitContext {
                    title: e.path.clone(),
                    snippet: e.snippet.clone(),
                })
                .collect::<Vec<_>>();
            let synthesized = engine.synthesize(&case.question, &contexts).await?;
            Some(QaResult {
                answerability: synthesized.answerability,
                matches_expected: answerability_matches(
                    case.expected_verdict,
                    synthesized.answerability,
                ),
                answer: synthesized.answer,
                latency_ms: synthesized.latency_ms,
                estimated_tokens: synthesized.tokens,
            })
        } else {
            None
        };
        results.push(CaseResult {
            id: case.id.clone(),
            expected_verdict: case.expected_verdict,
            slice: case.slice,
            channels: case.channels,
            signals: signals(case),
            qa,
        });
    }

    let thresholds = threshold_sweep(&results);
    let mut slices = BTreeMap::new();
    for slice in [
        Slice::Baseline,
        Slice::Unrelated,
        Slice::HardNegative,
        Slice::Conflicting,
        Slice::Partial,
        Slice::MissingChannel,
        Slice::LongmemevalAbstention,
    ] {
        let group = results
            .iter()
            .filter(|result| result.slice == slice)
            .cloned()
            .collect::<Vec<_>>();
        if !group.is_empty() {
            slices.insert(
                slice.name().to_string(),
                SliceReport {
                    cases: group.len(),
                    thresholds: threshold_sweep(&group),
                },
            );
        }
    }

    let qa_latencies = results
        .iter()
        .filter_map(|result| result.qa.as_ref().map(|qa| qa.latency_ms))
        .collect::<Vec<_>>();
    let qa_cases = qa_latencies.len();
    let qa_verdict_matches = results
        .iter()
        .filter_map(|result| result.qa.as_ref())
        .filter(|qa| qa.matches_expected)
        .count();
    let report = Report {
        generated_at: Timestamp::now().to_string(),
        commit: crate::retrieval::report::commit_sha(),
        hardware: crate::retrieval::report::hardware(),
        fixture_version: fixture.version,
        fixture_path: args.fixtures.display().to_string(),
        decision_signal: "normalized lexical query-term coverage",
        binary_mapping: "answerable=supported; unanswerable=unsupported; conflicting/partial excluded from FSR/FAR denominators",
        qa_status,
        qa: qa_engine.as_ref().map(QaEngine::report_meta),
        qa_cases,
        qa_verdict_matches,
        qa_latency_p50_ms: percentile(&qa_latencies, 0.50),
        qa_latency_p95_ms: percentile(&qa_latencies, 0.95),
        seed: None,
        cases: results,
        thresholds,
        slices,
    };

    let stamp: String = Timestamp::now()
        .to_string()
        .chars()
        .map(|c| if c == ':' || c == '.' { '-' } else { c })
        .collect();
    let run_dir = args.out.join(format!("{stamp}-abstention"));
    std::fs::create_dir_all(&run_dir)?;
    std::fs::write(
        run_dir.join("report.json"),
        serde_json::to_vec_pretty(&report)?,
    )?;
    let markdown = report_markdown(&report);
    std::fs::write(run_dir.join("report.md"), &markdown)?;
    println!("{markdown}");
    println!("run dir: {}", run_dir.display());
    Ok(())
}

fn load_fixture(path: &Path) -> Result<FixtureSet> {
    let bytes = std::fs::read(path).with_context(|| format!("reading {}", path.display()))?;
    let fixture: FixtureSet =
        serde_json::from_slice(&bytes).with_context(|| format!("parsing {}", path.display()))?;
    validate_fixture(&fixture)?;
    Ok(fixture)
}

fn validate_fixture(fixture: &FixtureSet) -> Result<()> {
    if fixture.version != FIXTURE_VERSION {
        bail!(
            "fixture version {} is unsupported; expected {FIXTURE_VERSION}",
            fixture.version
        );
    }
    if fixture.cases.is_empty() {
        bail!("fixture contains no cases");
    }
    let mut ids = HashSet::new();
    for case in &fixture.cases {
        if case.id.trim().is_empty() || !ids.insert(case.id.as_str()) {
            bail!("fixture ids must be nonempty and unique: `{}`", case.id);
        }
        if case.question.trim().is_empty() || case.evidence.is_empty() {
            bail!("{}: question and evidence must be nonempty", case.id);
        }
        let paths = case
            .evidence
            .iter()
            .map(|e| e.path.as_str())
            .collect::<HashSet<_>>();
        for evidence in &case.evidence {
            if evidence.path.trim().is_empty() || evidence.snippet.trim().is_empty() {
                bail!("{}: evidence path and snippet must be nonempty", case.id);
            }
            if !evidence.score.is_finite() || !(0.0..=1.0).contains(&evidence.score) {
                bail!("{}: evidence scores must be finite and in [0,1]", case.id);
            }
            if evidence
                .belief_strength
                .is_some_and(|v| !v.is_finite() || !(0.0..=1.0).contains(&v))
            {
                bail!("{}: belief strength must be finite and in [0,1]", case.id);
            }
        }
        if case
            .target_evidence
            .iter()
            .any(|target| !paths.contains(target.as_str()))
        {
            bail!(
                "{}: target_evidence must name a supplied evidence path",
                case.id
            );
        }
        if case.expected_verdict == Verdict::Answerable
            && (case.target_evidence.is_empty()
                || case.gold_answer.as_deref().is_none_or(str::is_empty))
        {
            bail!(
                "{}: answerable cases need target evidence and a gold answer",
                case.id
            );
        }
    }
    Ok(())
}

fn load_longmemeval_abstentions() -> Result<Vec<Case>> {
    let variant: &'static DatasetVariant = &dataset::VARIANT_S;
    let path = Path::new(variant.default_path);
    let questions = dataset::load(path, variant).with_context(|| {
        "loading pinned LongMemEval abstentions; run `ai-memory-eval retrieval --fetch` first"
            .to_string()
    })?;
    let cases = questions
        .into_iter()
        .filter(|question| question.is_abstention())
        .map(|question| {
            let mut seen = HashSet::new();
            let sessions = question
                .haystack_session_ids
                .iter()
                .zip(&question.haystack_dates)
                .zip(&question.haystack_sessions)
                .filter(|((session_id, _), _)| seen.insert(session_id.as_str()))
                .map(|((session_id, date), turns)| {
                    (
                        stored_session_uuid(&question, session_id),
                        format!("sessions/{session_id}.md"),
                        grep::session_text(date, turns),
                    )
                })
                .collect::<Vec<_>>();
            let normalized = sessions
                .iter()
                .map(|(id, _, text)| (*id, text.to_lowercase()))
                .collect::<Vec<_>>();
            let scores = grep::rank_with_scores(
                &question.question,
                normalized.iter().map(|(id, text)| (*id, text.as_str())),
            );
            let top_score = scores.first().map(|(_, score)| *score).unwrap_or(1) as f64;
            let evidence = scores
                .into_iter()
                .take(LONGMEMEVAL_CONTEXTS)
                .filter_map(|(id, score)| {
                    let (_, path, text) =
                        sessions.iter().find(|(candidate, _, _)| *candidate == id)?;
                    Some(Evidence {
                        path: path.clone(),
                        snippet: text.chars().take(LONGMEMEVAL_SNIPPET_CHARS).collect(),
                        score: score as f64 / top_score,
                        belief_strength: None,
                    })
                })
                .collect();
            Case {
                id: format!("longmemeval-{}", question.question_id),
                question: question.question,
                expected_verdict: Verdict::Unanswerable,
                slice: Slice::LongmemevalAbstention,
                evidence,
                target_evidence: Vec::new(),
                gold_answer: None,
                channels: Channels::default(),
            }
        })
        .collect();
    Ok(cases)
}

fn content_terms(text: &str) -> Vec<String> {
    const STOP: &[&str] = &[
        "a", "an", "and", "are", "do", "does", "for", "how", "in", "is", "of", "on", "the", "to",
        "was", "what", "which", "with",
    ];
    text.split(|c: char| !c.is_alphanumeric() && c != '_')
        .map(str::to_lowercase)
        .filter(|term| term.len() > 1 && !STOP.contains(&term.as_str()))
        .collect()
}

fn signals(case: &Case) -> Signals {
    let query_terms = content_terms(&case.question)
        .into_iter()
        .collect::<HashSet<_>>();
    let snippets = case
        .evidence
        .iter()
        .flat_map(|e| content_terms(&e.snippet))
        .collect::<Vec<_>>();
    let snippet_terms = snippets.iter().cloned().collect::<HashSet<_>>();
    let matched = query_terms
        .iter()
        .filter(|term| snippet_terms.contains(*term))
        .count();
    let lexical_coverage = ratio(matched, query_terms.len()).unwrap_or(0.0);
    let matching_occurrences = snippets
        .iter()
        .filter(|term| query_terms.contains(*term))
        .count();
    let snippet_matching_term_density = ratio(matching_occurrences, snippets.len()).unwrap_or(0.0);

    let mut scores = case.evidence.iter().map(|e| e.score).collect::<Vec<_>>();
    scores.sort_by(f64::total_cmp);
    scores.reverse();
    let top_score = scores.first().copied().unwrap_or(0.0);
    let top_score_margin = scores.get(1).map(|second| top_score - second);
    let score_mean = scores.iter().sum::<f64>() / scores.len() as f64;
    let score_stddev = (scores
        .iter()
        .map(|score| (score - score_mean).powi(2))
        .sum::<f64>()
        / scores.len() as f64)
        .sqrt();
    let beliefs = case
        .evidence
        .iter()
        .filter_map(|e| e.belief_strength)
        .collect::<Vec<_>>();
    let belief_mean =
        (!beliefs.is_empty()).then(|| beliefs.iter().sum::<f64>() / beliefs.len() as f64);
    Signals {
        lexical_coverage,
        top_score,
        top_score_margin,
        score_mean,
        score_stddev,
        snippet_matching_term_density,
        snippet_chars: case
            .evidence
            .iter()
            .map(|e| e.snippet.chars().count())
            .sum(),
        belief_mean,
        belief_observations: beliefs.len(),
    }
}

fn ratio(numerator: usize, denominator: usize) -> Option<f64> {
    (denominator != 0).then(|| numerator as f64 / denominator as f64)
}

fn answerability_matches(expected: Verdict, actual: Answerability) -> bool {
    matches!(
        (expected, actual),
        (Verdict::Answerable, Answerability::Answerable)
            | (Verdict::Unanswerable, Answerability::Insufficient)
            | (Verdict::Conflicting, Answerability::Conflicting)
            | (Verdict::Partial, Answerability::Partial)
    )
}

fn percentile(values: &[u128], quantile: f64) -> Option<u128> {
    if values.is_empty() {
        return None;
    }
    let mut sorted = values.to_vec();
    sorted.sort_unstable();
    let rank = (quantile * sorted.len() as f64).ceil().max(1.0) as usize;
    sorted.get(rank - 1).copied()
}

fn threshold_sweep(cases: &[CaseResult]) -> Vec<ThresholdMetrics> {
    DEFAULT_THRESHOLDS
        .into_iter()
        .map(|threshold| {
            let supported = cases
                .iter()
                .filter(|case| case.expected_verdict == Verdict::Answerable)
                .count();
            let unsupported = cases
                .iter()
                .filter(|case| case.expected_verdict == Verdict::Unanswerable)
                .count();
            let ambiguous_excluded = cases.len() - supported - unsupported;
            let predicted_answerable = cases
                .iter()
                .filter(|case| case.signals.lexical_coverage >= threshold)
                .count();
            let false_supported = cases
                .iter()
                .filter(|case| {
                    case.expected_verdict == Verdict::Unanswerable
                        && case.signals.lexical_coverage >= threshold
                })
                .count();
            let false_abstained = cases
                .iter()
                .filter(|case| {
                    case.expected_verdict == Verdict::Answerable
                        && case.signals.lexical_coverage < threshold
                })
                .count();
            ThresholdMetrics {
                threshold,
                supported,
                unsupported,
                ambiguous_excluded,
                predicted_answerable,
                false_supported_rate: ratio(false_supported, unsupported),
                false_abstention_rate: ratio(false_abstained, supported),
                coverage: ratio(predicted_answerable, cases.len()).unwrap_or(0.0),
            }
        })
        .collect()
}

fn metric(value: Option<f64>) -> String {
    value.map_or_else(|| "undefined".into(), |v| format!("{v:.3}"))
}

fn report_markdown(report: &Report) -> String {
    let mut markdown = format!(
        "# Abstention evaluation\n\n- commit: `{}`\n- fixture: `{}` (v{})\n- cases: {}\n- QA: {}\n- binary mapping: {}\n\n",
        report.commit,
        report.fixture_path,
        report.fixture_version,
        report.cases.len(),
        report.qa_status,
        report.binary_mapping,
    );
    if report.qa_cases > 0 {
        markdown.push_str(&format!(
            "- QA verdict matches: {}/{}; latency p50/p95: {}/{} ms\n\n",
            report.qa_verdict_matches,
            report.qa_cases,
            report.qa_latency_p50_ms.unwrap_or_default(),
            report.qa_latency_p95_ms.unwrap_or_default(),
        ));
    }
    markdown.push_str("| lexical threshold | coverage | FSR | FAR | ambiguous excluded |\n");
    markdown.push_str("|---:|---:|---:|---:|---:|\n");
    for row in &report.thresholds {
        markdown.push_str(&format!(
            "| {:.1} | {:.3} | {} | {} | {} |\n",
            row.threshold,
            row.coverage,
            metric(row.false_supported_rate),
            metric(row.false_abstention_rate),
            row.ambiguous_excluded,
        ));
    }
    markdown.push_str("\nNo threshold is designated as calibrated; the table is a trade-off sweep over a small synthetic corpus.\n");
    markdown
}

#[cfg(test)]
mod tests {
    use super::*;

    fn case(id: &str, verdict: Verdict, coverage_text: &str) -> Case {
        Case {
            id: id.into(),
            question: "Which deployment port is configured?".into(),
            expected_verdict: verdict,
            slice: Slice::Baseline,
            evidence: vec![Evidence {
                path: "rules/server.md".into(),
                snippet: coverage_text.into(),
                score: 0.8,
                belief_strength: Some(0.7),
            }],
            target_evidence: if verdict == Verdict::Answerable {
                vec!["rules/server.md".into()]
            } else {
                Vec::new()
            },
            gold_answer: (verdict == Verdict::Answerable).then(|| "49374".into()),
            channels: Channels::default(),
        }
    }

    #[test]
    fn fixture_validation_rejects_duplicate_ids_and_missing_support() {
        let duplicate = FixtureSet {
            version: 1,
            cases: vec![
                case("same", Verdict::Answerable, "deployment port 49374"),
                case("same", Verdict::Unanswerable, "deployment docs"),
            ],
        };
        assert!(
            validate_fixture(&duplicate)
                .unwrap_err()
                .to_string()
                .contains("unique")
        );

        let mut missing = case("missing", Verdict::Answerable, "deployment port 49374");
        missing.target_evidence.clear();
        assert!(
            validate_fixture(&FixtureSet {
                version: 1,
                cases: vec![missing]
            })
            .unwrap_err()
            .to_string()
            .contains("target evidence")
        );
    }

    #[test]
    fn raw_signals_are_deterministic_and_belief_is_optional() {
        let supported = case(
            "supported",
            Verdict::Answerable,
            "The deployment uses configured port 49374.",
        );
        let first = signals(&supported);
        let second = signals(&supported);
        assert_eq!(first.lexical_coverage, second.lexical_coverage);
        assert!(first.lexical_coverage > 0.5);
        assert_eq!(first.belief_mean, Some(0.7));

        let mut absent = supported;
        absent.evidence[0].belief_strength = None;
        assert_eq!(signals(&absent).belief_mean, None);
    }

    #[test]
    fn threshold_metrics_preserve_undefined_denominators_and_ambiguous_cases() {
        let cases = [
            case("yes", Verdict::Answerable, "configured deployment port"),
            case("no", Verdict::Unanswerable, "deployment guide only"),
            case("partial", Verdict::Partial, "deployment port"),
        ]
        .into_iter()
        .map(|case| CaseResult {
            id: case.id.clone(),
            expected_verdict: case.expected_verdict,
            slice: case.slice,
            channels: case.channels,
            signals: signals(&case),
            qa: None,
        })
        .collect::<Vec<_>>();
        let sweep = threshold_sweep(&cases);
        assert!(sweep.iter().all(|row| row.ambiguous_excluded == 1));
        assert!(sweep.iter().all(|row| row.false_supported_rate.is_some()));
        assert!(sweep.iter().all(|row| row.false_abstention_rate.is_some()));

        let only_partial = vec![cases[2].clone()];
        assert!(
            threshold_sweep(&only_partial).iter().all(
                |row| row.false_supported_rate.is_none() && row.false_abstention_rate.is_none()
            )
        );
    }

    #[test]
    fn verdict_mapping_and_latency_percentiles_are_explicit() {
        assert!(answerability_matches(
            Verdict::Unanswerable,
            Answerability::Insufficient
        ));
        assert!(!answerability_matches(
            Verdict::Conflicting,
            Answerability::Insufficient
        ));
        assert_eq!(percentile(&[], 0.95), None);
        assert_eq!(percentile(&[30, 10, 20], 0.50), Some(20));
        assert_eq!(percentile(&[30, 10, 20], 0.95), Some(30));
    }
}