Plan, apply and roll back changes to a production ontology, with a blast radius report and a proof an auditor can re-check.
Rust ai-nativedescription-logicsformal-verificationinfrastructure-as-codeisabelle-languageknowledge-graphleanlean4mcpmcp-serverontologyowlowl2reasoningrustshaclsparqltheorem-proving
fix(eval): use alignment set sizes consistently
Rudy Celekli37093cb620b0
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 3d864085..1490c9a8 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -46,6 +46,11 @@ All notable changes to Open Ontologies are documented here.### Fixed+- Calculate alignment evaluation metrics and reported sizes from the same+ deduplicated alignment sets as TP/FP/FN, so repeated entries do not lower the+ score of an otherwise identical alignment.++- **The embedding model is loaded once per process rather than once per HTTPsession** (#262). The HTTP arm hands the same server constructor to`StreamableHttpService::new` as a per-session factory, so everything thatdiff --git a/src/eval_alignment.rs b/src/eval_alignment.rsindex ba31c4d9..29cbe0cd 100644--- a/src/eval_alignment.rs+++ b/src/eval_alignment.rs@@ -45,15 +45,15 @@ pub fn evaluate(reference: &[AlignmentEntry], computed: &[AlignmentEntry]) -> Evlet fp = comp_set.len() - tp;let fn_ = ref_set.len() - tp;- let precision = if computed.is_empty() {+ let precision = if comp_set.is_empty() {0.0} else {- tp as f64 / computed.len() as f64+ tp as f64 / comp_set.len() as f64};- let recall = if reference.is_empty() {+ let recall = if ref_set.is_empty() {0.0} else {- tp as f64 / reference.len() as f64+ tp as f64 / ref_set.len() as f64};let f1 = if precision + recall == 0.0 {0.0@@ -68,8 +68,8 @@ pub fn evaluate(reference: &[AlignmentEntry], computed: &[AlignmentEntry]) -> Evtrue_positive: tp,false_positive: fp,false_negative: fn_,- reference_size: reference.len(),- computed_size: computed.len(),+ reference_size: ref_set.len(),+ computed_size: comp_set.len(),}}diff --git a/tests/alignment_evaluation_set_test.rs b/tests/alignment_evaluation_set_test.rsnew file mode 100644index 00000000..7baf8597--- /dev/null+++ b/tests/alignment_evaluation_set_test.rs@@ -0,0 +1,237 @@+mod common;++use serde_json::{Value, json};+use std::process::Stdio;+use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};++struct Mcp {+ child: tokio::process::Child,+ stdin: tokio::process::ChildStdin,+ stdout: tokio::io::Lines<BufReader<tokio::process::ChildStdout>>,+ next_id: u64,+}++impl Mcp {+ async fn start(directory: &std::path::Path) -> Self {+ let (child, stdin, stdout) = {+ let _gate = common::exec_gate();+ let config = directory.join("config.toml");+ std::fs::write(+ &config,+ format!(+ "[cache]\ndir = {}\n[embeddings]\nmodel_path = {}\ntokenizer_path = {}\n",+ json!(directory.join("cache").to_string_lossy()),+ json!(directory.join("absent.onnx").to_string_lossy()),+ json!(directory.join("absent-tokenizer.json").to_string_lossy()),+ ),+ )+ .unwrap();+ let mut child = tokio::process::Command::new(env!("CARGO_BIN_EXE_open-ontologies"))+ .args(["--no-connect", "--data-dir"])+ .arg(directory)+ .args(["serve", "--config"])+ .arg(config)+ .stdin(Stdio::piped())+ .stdout(Stdio::piped())+ .stderr(Stdio::inherit())+ .kill_on_drop(true)+ .spawn()+ .unwrap();+ let stdin = child.stdin.take().unwrap();+ let stdout = BufReader::new(child.stdout.take().unwrap()).lines();+ (child, stdin, stdout)+ };+ let mut session = Self {+ child,+ stdin,+ stdout,+ next_id: 1,+ };+ session.request("initialize", json!({"protocolVersion":"2025-03-26", "capabilities":{}, "clientInfo":{"name":"alignment-evaluation-test", "version":"1"}})).await;+ session+ .write(json!({"jsonrpc":"2.0", "method":"notifications/initialized"}))+ .await;+ session+ }++ async fn write(&mut self, message: Value) {+ self.stdin+ .write_all(message.to_string().as_bytes())+ .await+ .unwrap();+ self.stdin.write_all(b"\n").await.unwrap();+ self.stdin.flush().await.unwrap();+ }++ async fn request(&mut self, method: &str, params: Value) -> Value {+ let id = self.next_id;+ self.next_id += 1;+ self.write(json!({"jsonrpc":"2.0", "id":id, "method":method, "params":params}))+ .await;+ loop {+ let line =+ tokio::time::timeout(std::time::Duration::from_secs(20), self.stdout.next_line())+ .await+ .expect("MCP response timeout")+ .unwrap()+ .expect("MCP output closed");+ let response: Value = serde_json::from_str(&line).unwrap();+ if response["id"] == id {+ assert!(response.get("error").is_none(), "{response}");+ return response["result"].clone();+ }+ }+ }++ async fn call(&mut self, name: &str, arguments: Value) -> Value {+ let result = self+ .request("tools/call", json!({"name":name, "arguments":arguments}))+ .await;+ let value: Value =+ serde_json::from_str(result["content"][0]["text"].as_str().unwrap()).unwrap();+ assert!(value.get("error").is_none(), "{value}");+ value+ }+}++use open_ontologies::eval_alignment::{AlignmentEntry, evaluate};++fn entry(source: &str, target: &str, relation: &str) -> AlignmentEntry {+ AlignmentEntry {+ source: source.into(),+ target: target.into(),+ relation: relation.into(),+ }+}++fn cases() -> Vec<(+ Vec<AlignmentEntry>,+ Vec<AlignmentEntry>,+ f64,+ f64,+ f64,+ usize,+ usize,+ usize,+)> {+ let a = entry(+ "https://example.org/Order",+ "https://example.org/Purchase",+ "equivalent",+ );+ let b = entry(+ "https://example.org/Quantity",+ "https://example.org/Amount",+ "equivalent",+ );+ let c = entry(+ "https://example.org/Customer",+ "https://example.org/Buyer",+ "subsumed_by",+ );+ let different_relation = entry(&a.source, &a.target, "subsumed_by");+ vec![+ (+ vec![a.clone(), b.clone()],+ vec![a.clone(), b.clone()],+ 1.0,+ 1.0,+ 1.0,+ 2,+ 0,+ 0,+ ),+ (+ vec![a.clone(), a.clone(), b.clone()],+ vec![a.clone(), b.clone()],+ 1.0,+ 1.0,+ 1.0,+ 2,+ 0,+ 0,+ ),+ (+ vec![a.clone(), b.clone()],+ vec![a.clone(), a.clone(), b.clone()],+ 1.0,+ 1.0,+ 1.0,+ 2,+ 0,+ 0,+ ),+ (+ vec![a.clone(), a.clone(), b.clone()],+ vec![a.clone(), b.clone(), b.clone()],+ 1.0,+ 1.0,+ 1.0,+ 2,+ 0,+ 0,+ ),+ (+ vec![a.clone(), a.clone(), b],+ vec![a.clone(), a.clone(), c],+ 0.5,+ 0.5,+ 0.5,+ 1,+ 1,+ 1,+ ),+ (+ vec![a.clone(), a.clone()],+ vec![different_relation.clone(), different_relation],+ 0.0,+ 0.0,+ 0.0,+ 0,+ 1,+ 1,+ ),+ (vec![a.clone(), a.clone()], vec![], 0.0, 0.0, 0.0, 0, 0, 1),+ (vec![], vec![a.clone(), a], 0.0, 0.0, 0.0, 0, 1, 0),+ (vec![], vec![], 0.0, 0.0, 0.0, 0, 0, 0),+ ]+}++fn assert_report(result: &Value, expected: (f64, f64, f64, usize, usize, usize)) {+ let (precision, recall, f1, tp, fp, fn_) = expected;+ assert_eq!(result["precision"], precision);+ assert_eq!(result["recall"], recall);+ assert_eq!(result["f1"], f1);+ assert_eq!(result["true_positive"], tp);+ assert_eq!(result["false_positive"], fp);+ assert_eq!(result["false_negative"], fn_);+ assert_eq!(result["reference_size"], tp + fn_);+ assert_eq!(result["computed_size"], tp + fp);+}++#[test]+fn repeated_alignment_entries_do_not_change_set_scores_or_relation_identity() {+ for (reference, computed, precision, recall, f1, tp, fp, fn_) in cases() {+ let result = serde_json::to_value(evaluate(&reference, &computed)).unwrap();+ assert_report(&result, (precision, recall, f1, tp, fp, fn_));+ }+}++#[tokio::test]+async fn public_mcp_alignment_scores_and_counts_agree_with_the_same_sets() {+ let tmp = tempfile::tempdir().unwrap();+ let mut mcp = Mcp::start(tmp.path()).await;+ for (reference, computed, precision, recall, f1, tp, fp, fn_) in cases() {+ let result = mcp+ .call(+ "onto_eval_alignment",+ json!({+ "reference_json": serde_json::to_string(&reference).unwrap(),+ "computed_json": serde_json::to_string(&computed).unwrap(),+ }),+ )+ .await;+ assert_report(&result, (precision, recall, f1, tp, fp, fn_));+ }+ mcp.child.kill().await.unwrap();+}