Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

test(lint): add labeled contradiction evidence fixtures

Mihael Zamin Sousa81451f146b7d
diff --git a/crates/ai-memory-consolidate/tests/fixtures/contradiction_evidence.json b/crates/ai-memory-consolidate/tests/fixtures/contradiction_evidence.json
new file mode 100644
index 00000000..25ac5dcd
--- /dev/null
+++ b/crates/ai-memory-consolidate/tests/fixtures/contradiction_evidence.json
@@ -0,0 +1,112 @@
+[
+ {
+ "id": "conflicting_configuration_en",
+ "language": "en",
+ "label": "conflict",
+ "setup": "active",
+ "first": "For the production API in the current deployment, MAX_RETRIES must be exactly 3.",
+ "second": "For the production API in the current deployment, MAX_RETRIES must be exactly 5.",
+ "second_vector": [0.6, 0.8],
+ "expected_findings": 1
+ },
+ {
+ "id": "conflicting_configuration_pt_br",
+ "language": "pt-BR",
+ "label": "conflict",
+ "setup": "active",
+ "first": "Na implantação atual da API de produção, MAX_RETRIES deve ser exatamente 3.",
+ "second": "Na implantação atual da API de produção, MAX_RETRIES deve ser exatamente 5.",
+ "second_vector": [0.6, 0.8],
+ "expected_findings": 1
+ },
+ {
+ "id": "conflicting_configuration_high_similarity_zh",
+ "language": "zh",
+ "label": "conflict",
+ "setup": "active",
+ "first": "在生产 API 的当前部署中,MAX_RETRIES 必须恰好为 3。",
+ "second": "在生产 API 的当前部署中,MAX_RETRIES 必须恰好为 5。",
+ "second_vector": [0.98, 0.199],
+ "expected_findings": 0
+ },
+ {
+ "id": "compatible_same_topic_en",
+ "language": "en",
+ "label": "compatible",
+ "setup": "active",
+ "first": "The API retries failed requests up to three times.",
+ "second": "The API records a metric for each retry attempt.",
+ "second_vector": [0.6, 0.8],
+ "expected_findings": 1
+ },
+ {
+ "id": "compatible_procedures_pt_br",
+ "language": "pt-BR",
+ "label": "compatible",
+ "setup": "active",
+ "first": "A chave SSH da implantação deve ter permissão 0600.",
+ "second": "Remova os diretórios temporários abandonados após concluir a implantação.",
+ "second_vector": [0.6, 0.8],
+ "expected_findings": 1
+ },
+ {
+ "id": "compatible_hub_pages_zh",
+ "language": "zh",
+ "label": "compatible",
+ "setup": "active",
+ "first": "部署索引:服务清单、发布流程、监控指南。",
+ "second": "运维索引:服务清单、备份流程、值班指南。",
+ "second_vector": [0.6, 0.8],
+ "expected_findings": 1
+ },
+ {
+ "id": "compatible_near_duplicate_en",
+ "language": "en",
+ "label": "compatible",
+ "setup": "active",
+ "first": "The production API retry budget is three attempts.",
+ "second": "Production API requests have a retry budget of three attempts.",
+ "second_vector": [0.98, 0.199],
+ "expected_findings": 0
+ },
+ {
+ "id": "declared_contradiction_without_embeddings_en",
+ "language": "en",
+ "label": "conflict",
+ "setup": "declared",
+ "first": "For the current production API, the configured retry budget must be exactly 3.",
+ "second": "For the current production API, the configured retry budget must be exactly 5.",
+ "second_vector": [0.0, 1.0],
+ "expected_findings": 1
+ },
+ {
+ "id": "corrected_superseded_decision_pt_br",
+ "language": "pt-BR",
+ "label": "ineligible",
+ "setup": "superseded",
+ "first": "A decisão anterior da arquitetura usava Redis para a fila.",
+ "second": "Correção da decisão: a arquitetura atual usa PostgreSQL para a fila.",
+ "second_vector": [0.6, 0.8],
+ "expected_findings": 0
+ },
+ {
+ "id": "expired_configuration_zh",
+ "language": "zh",
+ "label": "ineligible",
+ "setup": "expired",
+ "first": "旧部署的 MAX_RETRIES 为 3,此记录已过期。",
+ "second": "当前部署的 MAX_RETRIES 为 5。",
+ "second_vector": [0.6, 0.8],
+ "expected_findings": 0
+ },
+ {
+ "id": "foreign_project_configuration_en",
+ "language": "en",
+ "label": "ineligible",
+ "setup": "foreign_project",
+ "first": "This project's production API requires MAX_RETRIES=3.",
+ "second": "A different project's production API requires MAX_RETRIES=5.",
+ "second_vector": [0.6, 0.8],
+ "expected_findings": 0
+ }
+]
diff --git a/crates/ai-memory-consolidate/tests/suite/contradiction_lint.rs b/crates/ai-memory-consolidate/tests/suite/contradiction_lint.rs
index 1509ed9e..e600cc00 100644
--- a/crates/ai-memory-consolidate/tests/suite/contradiction_lint.rs
+++ b/crates/ai-memory-consolidate/tests/suite/contradiction_lint.rs
@@ -450,3 +450,229 @@ async fn llm_lint_pass_does_not_audit_the_previous_lint_report() {
"{prompt}"
);
}
+
+/// Manually labeled, synthetic pairs. Vectors are controlled inputs, not model
+/// outputs: these cases measure the classification boundary, not embedding
+/// quality or production precision. `expected_findings` records today's
+/// behavior independently of the label; update it when semantics change.
+///
+/// A positive pair contains incompatible simultaneous values for the same
+/// setting and scope. Compatible active pairs are negatives. Expired,
+/// superseded and foreign-project pairs exercise eligibility and are excluded
+/// from the classification metrics. A prediction means any `kind = contradiction`
+/// finding, including an advisory similarity candidate; it does not mean the
+/// server independently verified the conflict. The typed-edge control also
+/// contains conflicting prose, so its label does not rely on the edge alone.
+#[derive(serde::Deserialize)]
+#[serde(deny_unknown_fields)]
+struct EvidenceCase {
+ id: String,
+ language: String,
+ label: EvidenceLabel,
+ setup: EvidenceSetup,
+ first: String,
+ second: String,
+ second_vector: [f32; 2],
+ expected_findings: usize,
+}
+
+#[derive(Clone, Copy, serde::Deserialize)]
+#[serde(rename_all = "snake_case")]
+enum EvidenceLabel {
+ Conflict,
+ Compatible,
+ Ineligible,
+}
+
+#[derive(Clone, Copy, serde::Deserialize)]
+#[serde(rename_all = "snake_case")]
+enum EvidenceSetup {
+ Active,
+ Declared,
+ Superseded,
+ Expired,
+ ForeignProject,
+}
+
+fn evidence_cases() -> Vec<EvidenceCase> {
+ serde_json::from_str(include_str!("../fixtures/contradiction_evidence.json")).unwrap()
+}
+
+async fn seed_evidence_case(case: &EvidenceCase) -> Fixture {
+ let fx = seed_fixture().await;
+ let first = write_semantic(&fx, "decisions/first.md", &case.first).await;
+ let project_id = if matches!(case.setup, EvidenceSetup::ForeignProject) {
+ fx.store
+ .writer
+ .get_or_create_project(fx.ws, "foreign", None)
+ .await
+ .unwrap()
+ } else {
+ fx.proj
+ };
+ let path = if matches!(case.setup, EvidenceSetup::Superseded) {
+ "decisions/first.md"
+ } else {
+ "decisions/second.md"
+ };
+ let frontmatter = if matches!(case.setup, EvidenceSetup::Declared) {
+ serde_json::json!({"relations": {"contradicts": ["decisions/first.md"]}})
+ } else {
+ serde_json::json!({})
+ };
+ let second = fx
+ .wiki
+ .write_page(WritePageRequest {
+ workspace_id: fx.ws,
+ project_id,
+ path: PagePath::new(path).unwrap(),
+ frontmatter,
+ body: case.second.clone(),
+ tier: Tier::Semantic,
+ pinned: false,
+ title: Some(path.into()),
+ admission_ctx: None,
+ author_id: None,
+ actor: ActorContext::anonymous(),
+ evidence: Vec::new(),
+ })
+ .await
+ .unwrap();
+ if !matches!(case.setup, EvidenceSetup::Declared) {
+ seed_embedding(&fx, first, &[1.0, 0.0]);
+ seed_embedding(&fx, second, &case.second_vector);
+ }
+ if matches!(case.setup, EvidenceSetup::Expired) {
+ let conn = rusqlite::Connection::open(fx.tmp.path().join("db/memory.sqlite")).unwrap();
+ conn.execute(
+ "UPDATE pages SET expires_at = 1 WHERE id = ?1",
+ params![first.as_bytes()],
+ )
+ .unwrap();
+ }
+ fx
+}
+
+async fn evidence_findings(fx: &Fixture) -> Vec<(String, Vec<String>)> {
+ let report = run_lint(
+ &fx.store.reader,
+ &fx.wiki,
+ None,
+ fx.ws,
+ fx.proj,
+ options(coord()),
+ )
+ .await
+ .unwrap();
+ report
+ .findings
+ .into_iter()
+ .filter(|finding| finding.kind == "contradiction")
+ .map(|finding| (finding.severity, finding.pages))
+ .collect()
+}
+
+#[derive(Default, Debug)]
+struct EvidenceCounts {
+ tp: usize,
+ fp: usize,
+ tn: usize,
+ fn_: usize,
+ excluded: usize,
+ findings: usize,
+}
+
+impl EvidenceCounts {
+ fn record(&mut self, case: &EvidenceCase, findings: usize) {
+ self.findings += findings;
+ match (case.label, findings > 0) {
+ (EvidenceLabel::Conflict, true) => self.tp += 1,
+ (EvidenceLabel::Conflict, false) => self.fn_ += 1,
+ (EvidenceLabel::Compatible, true) => self.fp += 1,
+ (EvidenceLabel::Compatible, false) => self.tn += 1,
+ (EvidenceLabel::Ineligible, _) => self.excluded += 1,
+ }
+ }
+
+ fn report(&self) {
+ // Undefined metrics stay None rather than being reported as zero.
+ let ratio = |n: usize, d: usize| (d != 0).then(|| n as f64 / d as f64);
+ println!(
+ "synthetic classification: {self:?}; precision={:?}; recall={:?}; f1={:?}; fpr={:?}; fnr={:?}",
+ ratio(self.tp, self.tp + self.fp),
+ ratio(self.tp, self.tp + self.fn_),
+ ratio(2 * self.tp, 2 * self.tp + self.fp + self.fn_),
+ ratio(self.fp, self.fp + self.tn),
+ ratio(self.fn_, self.tp + self.fn_),
+ );
+ }
+}
+
+#[tokio::test]
+async fn labeled_evidence_pairs_record_current_classification() {
+ let mut counts = EvidenceCounts::default();
+ let cases = evidence_cases();
+ let mut ids = std::collections::HashSet::new();
+ for case in &cases {
+ assert!(ids.insert(&case.id), "duplicate fixture id: {}", case.id);
+ let fx = seed_evidence_case(case).await;
+ let before = latest_page_count(&fx);
+ let first = evidence_findings(&fx).await;
+ assert_eq!(
+ first.len(),
+ case.expected_findings,
+ "{} ({})",
+ case.id,
+ case.language
+ );
+ if matches!(case.label, EvidenceLabel::Ineligible) {
+ assert!(first.is_empty(), "excluded pair: {}", case.id);
+ }
+ if matches!(case.setup, EvidenceSetup::Declared) {
+ assert_eq!(first[0].0, "warning", "typed edge control");
+ }
+ assert_eq!(
+ first,
+ evidence_findings(&fx).await,
+ "repeat parity: {}",
+ case.id
+ );
+ assert_eq!(
+ latest_page_count(&fx),
+ before,
+ "lint mutated pages: {}",
+ case.id
+ );
+ counts.record(case, first.len());
+ }
+ assert_eq!(cases.len(), 11);
+ counts.report();
+}
+
+/// Timing excludes fixture creation. Each trial is one isolated pair, not a
+/// scan over the whole corpus. Run explicitly; no wall-clock assertion enters
+/// CI, and the measurements do not imply a production throughput target.
+#[tokio::test]
+#[ignore = "manual offline lint timing receipt; run with --ignored --nocapture"]
+async fn labeled_evidence_pairs_timing_receipt() {
+ const TRIALS: usize = 20;
+ let mut timings = Vec::new();
+ for case in evidence_cases() {
+ let fx = seed_evidence_case(&case).await;
+ let expected = evidence_findings(&fx).await;
+ for _ in 0..TRIALS {
+ let started = std::time::Instant::now();
+ let actual = evidence_findings(&fx).await;
+ timings.push(started.elapsed().as_secs_f64() * 1_000.0);
+ assert_eq!(actual, expected, "repeat parity: {}", case.id);
+ }
+ }
+ timings.sort_by(f64::total_cmp);
+ let percentile = |pct: usize| timings[(pct * timings.len()).div_ceil(100) - 1];
+ println!(
+ "synthetic isolated-pair lint: trials={} p50_ms={:.6} p95_ms={:.6}; LLM calls=0",
+ timings.len(),
+ percentile(50),
+ percentile(95),
+ );
+}
diff --git a/docs/design-memory-aging.md b/docs/design-memory-aging.md
index 24342876..90b22c45 100644
--- a/docs/design-memory-aging.md
+++ b/docs/design-memory-aging.md
@@ -253,6 +253,36 @@ Everything in A works with no provider and does not require ranking changes.
- *Failure modes:* false-positive band → advisory-only output; the operator/lint
decides.
+**Classification fixtures ([RFC #1685](https://github.com/akitaonrails/ai-memory/issues/1685)).**
+The existing `contradiction_lint` suite also runs
+[`contradiction_evidence.json`](../crates/ai-memory-consolidate/tests/fixtures/contradiction_evidence.json),
+a small manually labeled English, Portuguese and Chinese corpus. Its vectors
+are controlled inputs, not outputs from an embedding model. The fixtures record
+current classification, including compatible pairs inside the band and a
+conflicting pair above it; they do not estimate production precision or propose
+a new response contract.
+
+A prediction is any `contradiction` finding, including an advisory similarity
+candidate. Positives contain incompatible simultaneous values for the same
+setting and scope; compatible active pairs are negatives. Superseded, expired
+and foreign-project pairs test eligibility separately and do not enter the
+classification metrics. The declared-edge control contains conflicting
+prose and has no embeddings. Expected counts document the current behavior;
+labels should remain independent when an approved classifier change updates
+those counts.
+
+```bash
+cargo test -p ai-memory-consolidate labeled_evidence_pairs_record_current_classification -- --nocapture
+cargo test -p ai-memory-consolidate labeled_evidence_pairs_timing_receipt -- --ignored --nocapture
+```
+
+The first command reports the confusion counts, precision, recall, F1 and
+false-positive/false-negative rates and checks repeated classification parity.
+The optional second command reports lint p50/p95 over 20 runs per isolated pair after a
+warm-up, excluding fixture creation. Both run offline without an LLM. Timing has
+no pass threshold and is not a benchmark of a full store. Neither changes lint's
+public classification; that decision remains in the RFC.
+
### Bucket B — LLM "dream" pass (opt-in, OFF by default, R2-gated)
All of B is off by default and touches either ranking or LLM; each gates on R2.