Skip to content

Plan, apply and roll back changes to a production ontology, with a blast radius report and a proof an auditor can re-check.

fix(eval): use alignment set sizes consistently

Rudy Celekli37093cb620b0
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3d864085..1490c9a8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -46,6 +46,11 @@ All notable changes to Open Ontologies are documented here.
### Fixed
+- Calculate alignment evaluation metrics and reported sizes from the same
+ deduplicated alignment sets as TP/FP/FN, so repeated entries do not lower the
+ score of an otherwise identical alignment.
+
+
- **The embedding model is loaded once per process rather than once per HTTP
session** (#262). The HTTP arm hands the same server constructor to
`StreamableHttpService::new` as a per-session factory, so everything that
diff --git a/src/eval_alignment.rs b/src/eval_alignment.rs
index ba31c4d9..29cbe0cd 100644
--- a/src/eval_alignment.rs
+++ b/src/eval_alignment.rs
@@ -45,15 +45,15 @@ pub fn evaluate(reference: &[AlignmentEntry], computed: &[AlignmentEntry]) -> Ev
let fp = comp_set.len() - tp;
let fn_ = ref_set.len() - tp;
- let precision = if computed.is_empty() {
+ let precision = if comp_set.is_empty() {
0.0
} else {
- tp as f64 / computed.len() as f64
+ tp as f64 / comp_set.len() as f64
};
- let recall = if reference.is_empty() {
+ let recall = if ref_set.is_empty() {
0.0
} else {
- tp as f64 / reference.len() as f64
+ tp as f64 / ref_set.len() as f64
};
let f1 = if precision + recall == 0.0 {
0.0
@@ -68,8 +68,8 @@ pub fn evaluate(reference: &[AlignmentEntry], computed: &[AlignmentEntry]) -> Ev
true_positive: tp,
false_positive: fp,
false_negative: fn_,
- reference_size: reference.len(),
- computed_size: computed.len(),
+ reference_size: ref_set.len(),
+ computed_size: comp_set.len(),
}
}
diff --git a/tests/alignment_evaluation_set_test.rs b/tests/alignment_evaluation_set_test.rs
new file mode 100644
index 00000000..7baf8597
--- /dev/null
+++ b/tests/alignment_evaluation_set_test.rs
@@ -0,0 +1,237 @@
+mod common;
+
+use serde_json::{Value, json};
+use std::process::Stdio;
+use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};
+
+struct Mcp {
+ child: tokio::process::Child,
+ stdin: tokio::process::ChildStdin,
+ stdout: tokio::io::Lines<BufReader<tokio::process::ChildStdout>>,
+ next_id: u64,
+}
+
+impl Mcp {
+ async fn start(directory: &std::path::Path) -> Self {
+ let (child, stdin, stdout) = {
+ let _gate = common::exec_gate();
+ let config = directory.join("config.toml");
+ std::fs::write(
+ &config,
+ format!(
+ "[cache]\ndir = {}\n[embeddings]\nmodel_path = {}\ntokenizer_path = {}\n",
+ json!(directory.join("cache").to_string_lossy()),
+ json!(directory.join("absent.onnx").to_string_lossy()),
+ json!(directory.join("absent-tokenizer.json").to_string_lossy()),
+ ),
+ )
+ .unwrap();
+ let mut child = tokio::process::Command::new(env!("CARGO_BIN_EXE_open-ontologies"))
+ .args(["--no-connect", "--data-dir"])
+ .arg(directory)
+ .args(["serve", "--config"])
+ .arg(config)
+ .stdin(Stdio::piped())
+ .stdout(Stdio::piped())
+ .stderr(Stdio::inherit())
+ .kill_on_drop(true)
+ .spawn()
+ .unwrap();
+ let stdin = child.stdin.take().unwrap();
+ let stdout = BufReader::new(child.stdout.take().unwrap()).lines();
+ (child, stdin, stdout)
+ };
+ let mut session = Self {
+ child,
+ stdin,
+ stdout,
+ next_id: 1,
+ };
+ session.request("initialize", json!({"protocolVersion":"2025-03-26", "capabilities":{}, "clientInfo":{"name":"alignment-evaluation-test", "version":"1"}})).await;
+ session
+ .write(json!({"jsonrpc":"2.0", "method":"notifications/initialized"}))
+ .await;
+ session
+ }
+
+ async fn write(&mut self, message: Value) {
+ self.stdin
+ .write_all(message.to_string().as_bytes())
+ .await
+ .unwrap();
+ self.stdin.write_all(b"\n").await.unwrap();
+ self.stdin.flush().await.unwrap();
+ }
+
+ async fn request(&mut self, method: &str, params: Value) -> Value {
+ let id = self.next_id;
+ self.next_id += 1;
+ self.write(json!({"jsonrpc":"2.0", "id":id, "method":method, "params":params}))
+ .await;
+ loop {
+ let line =
+ tokio::time::timeout(std::time::Duration::from_secs(20), self.stdout.next_line())
+ .await
+ .expect("MCP response timeout")
+ .unwrap()
+ .expect("MCP output closed");
+ let response: Value = serde_json::from_str(&line).unwrap();
+ if response["id"] == id {
+ assert!(response.get("error").is_none(), "{response}");
+ return response["result"].clone();
+ }
+ }
+ }
+
+ async fn call(&mut self, name: &str, arguments: Value) -> Value {
+ let result = self
+ .request("tools/call", json!({"name":name, "arguments":arguments}))
+ .await;
+ let value: Value =
+ serde_json::from_str(result["content"][0]["text"].as_str().unwrap()).unwrap();
+ assert!(value.get("error").is_none(), "{value}");
+ value
+ }
+}
+
+use open_ontologies::eval_alignment::{AlignmentEntry, evaluate};
+
+fn entry(source: &str, target: &str, relation: &str) -> AlignmentEntry {
+ AlignmentEntry {
+ source: source.into(),
+ target: target.into(),
+ relation: relation.into(),
+ }
+}
+
+fn cases() -> Vec<(
+ Vec<AlignmentEntry>,
+ Vec<AlignmentEntry>,
+ f64,
+ f64,
+ f64,
+ usize,
+ usize,
+ usize,
+)> {
+ let a = entry(
+ "https://example.org/Order",
+ "https://example.org/Purchase",
+ "equivalent",
+ );
+ let b = entry(
+ "https://example.org/Quantity",
+ "https://example.org/Amount",
+ "equivalent",
+ );
+ let c = entry(
+ "https://example.org/Customer",
+ "https://example.org/Buyer",
+ "subsumed_by",
+ );
+ let different_relation = entry(&a.source, &a.target, "subsumed_by");
+ vec![
+ (
+ vec![a.clone(), b.clone()],
+ vec![a.clone(), b.clone()],
+ 1.0,
+ 1.0,
+ 1.0,
+ 2,
+ 0,
+ 0,
+ ),
+ (
+ vec![a.clone(), a.clone(), b.clone()],
+ vec![a.clone(), b.clone()],
+ 1.0,
+ 1.0,
+ 1.0,
+ 2,
+ 0,
+ 0,
+ ),
+ (
+ vec![a.clone(), b.clone()],
+ vec![a.clone(), a.clone(), b.clone()],
+ 1.0,
+ 1.0,
+ 1.0,
+ 2,
+ 0,
+ 0,
+ ),
+ (
+ vec![a.clone(), a.clone(), b.clone()],
+ vec![a.clone(), b.clone(), b.clone()],
+ 1.0,
+ 1.0,
+ 1.0,
+ 2,
+ 0,
+ 0,
+ ),
+ (
+ vec![a.clone(), a.clone(), b],
+ vec![a.clone(), a.clone(), c],
+ 0.5,
+ 0.5,
+ 0.5,
+ 1,
+ 1,
+ 1,
+ ),
+ (
+ vec![a.clone(), a.clone()],
+ vec![different_relation.clone(), different_relation],
+ 0.0,
+ 0.0,
+ 0.0,
+ 0,
+ 1,
+ 1,
+ ),
+ (vec![a.clone(), a.clone()], vec![], 0.0, 0.0, 0.0, 0, 0, 1),
+ (vec![], vec![a.clone(), a], 0.0, 0.0, 0.0, 0, 1, 0),
+ (vec![], vec![], 0.0, 0.0, 0.0, 0, 0, 0),
+ ]
+}
+
+fn assert_report(result: &Value, expected: (f64, f64, f64, usize, usize, usize)) {
+ let (precision, recall, f1, tp, fp, fn_) = expected;
+ assert_eq!(result["precision"], precision);
+ assert_eq!(result["recall"], recall);
+ assert_eq!(result["f1"], f1);
+ assert_eq!(result["true_positive"], tp);
+ assert_eq!(result["false_positive"], fp);
+ assert_eq!(result["false_negative"], fn_);
+ assert_eq!(result["reference_size"], tp + fn_);
+ assert_eq!(result["computed_size"], tp + fp);
+}
+
+#[test]
+fn repeated_alignment_entries_do_not_change_set_scores_or_relation_identity() {
+ for (reference, computed, precision, recall, f1, tp, fp, fn_) in cases() {
+ let result = serde_json::to_value(evaluate(&reference, &computed)).unwrap();
+ assert_report(&result, (precision, recall, f1, tp, fp, fn_));
+ }
+}
+
+#[tokio::test]
+async fn public_mcp_alignment_scores_and_counts_agree_with_the_same_sets() {
+ let tmp = tempfile::tempdir().unwrap();
+ let mut mcp = Mcp::start(tmp.path()).await;
+ for (reference, computed, precision, recall, f1, tp, fp, fn_) in cases() {
+ let result = mcp
+ .call(
+ "onto_eval_alignment",
+ json!({
+ "reference_json": serde_json::to_string(&reference).unwrap(),
+ "computed_json": serde_json::to_string(&computed).unwrap(),
+ }),
+ )
+ .await;
+ assert_report(&result, (precision, recall, f1, tp, fp, fn_));
+ }
+ mcp.child.kill().await.unwrap();
+}