Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

3.7 KB
Raw Blame
//! Vector-score threshold sweep. Embeds every labeled pair and scores A vs B by
//! the same best-across-fields cosine the pipeline's `collapse` uses, then sweeps
//! the similarity cutoff to produce a precision/recall curve plus the average
//! precision and best-F1 cutoff. This is what lets `min_vector_similarity`
//! (currently 0.75) be chosen from data instead of set blind.
//!
//! This measures the *vector floor in isolation* — score-only prediction, no
//! judge — so it shows how separable duplicates are by embedding similarity and
//! how aggressively the floor can filter before it starves the judge of real
//! duplicates. It needs only the embedder: no judge, no database.
//!
//! `#[ignore]` — hits OpenAI. Run with `just eval`.


use std::collections::HashMap;

use embedding::EmbeddingModel;
use embedding::LabeledEmbedding;
use embedding::embedding_provider::openai::TextEmbedding3Small;
use crate::util::corpus::{embeddable, load_corpus};
use crate::util::harness::{EVAL_CONCURRENCY, openai_key};
use crate::util::metrics::sweep_report;
use futures::StreamExt;

/// Cosine similarity between two equal-length vectors.
fn cosine(a: &[f32], b: &[f32]) -> f64 {
    let mut dot = 0.0f64;
    let mut norm_a = 0.0f64;
    let mut norm_b = 0.0f64;
    for (x, y) in a.iter().zip(b.iter()) {
        dot += f64::from(*x) * f64::from(*y);
        norm_a += f64::from(*x) * f64::from(*x);
        norm_b += f64::from(*y) * f64::from(*y);
    }
    if norm_a == 0.0 || norm_b == 0.0 {
        return 0.0;
    }
    dot / (norm_a.sqrt() * norm_b.sqrt())
}

/// Best cosine across the field cross-product, mirroring the pipeline's
/// `collapse` (title↔title, title↔body, body↔title, body↔body → max).
fn best_similarity<const DIMS: usize>(
    a: &[LabeledEmbedding<'static, DIMS>],
    b: &[LabeledEmbedding<'static, DIMS>],
) -> f64 {
    let mut best = f64::NEG_INFINITY;
    for field_a in a {
        for field_b in b {
            best = best.max(cosine(&field_a.embedding, &field_b.embedding));
        }
    }
    if best.is_finite() { best } else { 0.0 }
}

#[tokio::test]
#[ignore = "hits OpenAI; run locally with API keys"]
async fn threshold_sweep_baseline() {
    let corpus = load_corpus();
    let embedder = TextEmbedding3Small::new(openai_key());

    // Embed each task once, keyed by id.
    let embeddings: HashMap<String, _> = futures::stream::iter(&corpus.tasks)
        .map(|task| {
            let embedder = &embedder;
            async move {
                let fields = embedder.embed(&embeddable(task)).await.expect("embed task");
                (task.id.clone(), fields)
            }
        })
        .buffer_unordered(EVAL_CONCURRENCY)
        .collect()
        .await;

    // Score each labeled pair by best cross-field cosine, paired with its label.
    let mut points: Vec<(f64, bool)> = Vec::with_capacity(corpus.pairs.len());
    for pair in &corpus.pairs {
        let (Some(a), Some(b)) = (embeddings.get(&pair.a), embeddings.get(&pair.b)) else {
            continue;
        };
        if a.is_empty() || b.is_empty() {
            continue;
        }
        points.push((best_similarity(a, b), pair.expected_duplicate));
    }

    let thresholds: Vec<f64> = (30..=95).step_by(5).map(|t| f64::from(t) / 100.0).collect();
    println!(
        "{}",
        sweep_report(
            "vector-score threshold sweep (score-only, no judge)",
            &points,
            &thresholds,
        )
    );

    // Report-only: assert only that every pair was scored, so a seeding/wiring
    // regression still fails even though quality is not gated.
    assert_eq!(
        points.len(),
        corpus.pairs.len(),
        "every labeled pair should be scored"
    );
}