Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

4.0 KB
Raw Blame
//! Loading the labeled evaluation corpus, plus small per-task helpers.
//!
//! The corpus is [`EvalCorpus`] JSON merged from committed fixtures:
//! - two read-only snapshots of real employee tasks — a title-only set
//!   (`prod_title_only.json`) and a with-body set (`prod_with_body.json`), both
//!   screened for sensitive content and email-anonymized;
//! - labeled pairs over those tasks: hand-picked (`prod_pairs.json`),
//!   model-mined-then-verified (`prod_mined_pairs.json`), and an intentional
//!   look-alike hard-negative batch (`prod_hard_negatives.json` +
//!   `prod_hard_negative_pairs.json`);
//! - synthetic labeled pairs pinning the canonical cases: hand-authored
//!   (`synthetic.json`) and model-generated-then-verified
//!   (`synthetic_generated.json`);
//! - hand-authored non-rephrasing positives (`synthetic_hard_positives.json`):
//!   terse-title vs detailed writeup, and same-work/low-lexical-overlap pairs,
//!   so recall is measured on more than paraphrases.

use std::collections::HashMap;

use embedding::entity::Task;
use task_dedup::eval::{CorpusTask, EvalCorpus, TaskSource};

/// Loads and merges the committed fixtures into one corpus, panicking if any
/// labeled pair references a task that is missing.
pub fn load_corpus() -> EvalCorpus {
    let mut corpus = EvalCorpus::from_json(include_bytes!(
        "../../../fixtures/eval/prod_title_only.json"
    ))
    .expect("prod_title_only.json parses");
    for bytes in [
        include_bytes!("../../../fixtures/eval/prod_with_body.json").as_slice(),
        include_bytes!("../../../fixtures/eval/prod_pairs.json").as_slice(),
        include_bytes!("../../../fixtures/eval/prod_mined_pairs.json").as_slice(),
        include_bytes!("../../../fixtures/eval/prod_hard_negatives.json").as_slice(),
        include_bytes!("../../../fixtures/eval/prod_hard_negative_pairs.json").as_slice(),
        include_bytes!("../../../fixtures/eval/synthetic.json").as_slice(),
        include_bytes!("../../../fixtures/eval/synthetic_generated.json").as_slice(),
        include_bytes!("../../../fixtures/eval/synthetic_hard_positives.json").as_slice(),
    ] {
        corpus.merge(EvalCorpus::from_json(bytes).expect("fixture parses"));
    }

    let dangling = corpus.dangling_pair_ids();
    assert!(
        dangling.is_empty(),
        "labeled pairs reference missing tasks: {dangling:?}"
    );
    corpus
}

/// Maps each corpus task id to the document id it is seeded under.
///
/// Prod tasks keep their real UUID; synthetic tasks (readable slugs in the
/// fixture) are seeded under a generated UUID-shaped id. This matters because
/// the `task_duplicate_match_order` CHECK (`task_id < duplicate_task_id`) is
/// evaluated in the database's collation, while the service's `ordered_pair`
/// uses Rust byte ordering — the two disagree on strings with hyphens in
/// non-aligned positions (e.g. `syn-attach-drag-files` vs `syn-attach-dragdrop`).
/// Real document ids are UUIDs, whose hyphens always align, so production never
/// hits this; seeding UUID-shaped ids keeps the eval faithful to that.
pub fn seed_ids(corpus: &EvalCorpus) -> HashMap<String, String> {
    corpus
        .tasks
        .iter()
        .enumerate()
        .map(|(index, task)| {
            let doc_id = match task.source {
                TaskSource::Prod => task.id.clone(),
                TaskSource::Synthetic => format!("e5a10000-0000-4000-8000-{index:012x}"),
            };
            (task.id.clone(), doc_id)
        })
        .collect()
}

/// Joins a task's title and body the same way the service builds its judge /
/// rerank query, so isolated judge calls see exactly what the pipeline would.
pub fn full_text(title: &str, body: &str) -> String {
    format!("{}\n{}", title.trim(), body.trim())
}

/// Borrows a corpus task as an embeddable entity (title + body fields).
pub fn embeddable(task: &CorpusTask) -> Task<'_> {
    Task {
        title: std::borrow::Cow::Borrowed(&task.title),
        body: std::borrow::Cow::Borrowed(&task.body),
    }
}