Plan, apply and roll back changes to a production ontology, with a blast radius report and a proof an auditor can re-check.
Rust ai-nativedescription-logicsformal-verificationinfrastructure-as-codeisabelle-languageknowledge-graphleanlean4mcpmcp-serverontologyowlowl2reasoningrustshaclsparqltheorem-proving
mod common;
use serde_json::{Value, json};
use std::process::Stdio;
use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};
struct Mcp {
child: tokio::process::Child,
stdin: tokio::process::ChildStdin,
stdout: tokio::io::Lines<BufReader<tokio::process::ChildStdout>>,
next_id: u64,
}
impl Mcp {
async fn start(directory: &std::path::Path) -> Self {
let (child, stdin, stdout) = {
let _gate = common::exec_gate();
let config = directory.join("config.toml");
std::fs::write(
&config,
format!(
"[cache]\ndir = {}\n[embeddings]\nmodel_path = {}\ntokenizer_path = {}\n",
json!(directory.join("cache").to_string_lossy()),
json!(directory.join("absent.onnx").to_string_lossy()),
json!(directory.join("absent-tokenizer.json").to_string_lossy()),
),
)
.unwrap();
let mut child = tokio::process::Command::new(env!("CARGO_BIN_EXE_open-ontologies"))
.args(["--no-connect", "--data-dir"])
.arg(directory)
.args(["serve", "--config"])
.arg(config)
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::inherit())
.kill_on_drop(true)
.spawn()
.unwrap();
let stdin = child.stdin.take().unwrap();
let stdout = BufReader::new(child.stdout.take().unwrap()).lines();
(child, stdin, stdout)
};
let mut session = Self {
child,
stdin,
stdout,
next_id: 1,
};
session.request("initialize", json!({"protocolVersion":"2025-03-26", "capabilities":{}, "clientInfo":{"name":"alignment-evaluation-test", "version":"1"}})).await;
session
.write(json!({"jsonrpc":"2.0", "method":"notifications/initialized"}))
.await;
session
}
async fn write(&mut self, message: Value) {
self.stdin
.write_all(message.to_string().as_bytes())
.await
.unwrap();
self.stdin.write_all(b"\n").await.unwrap();
self.stdin.flush().await.unwrap();
}
async fn request(&mut self, method: &str, params: Value) -> Value {
let id = self.next_id;
self.next_id += 1;
self.write(json!({"jsonrpc":"2.0", "id":id, "method":method, "params":params}))
.await;
loop {
let line =
tokio::time::timeout(std::time::Duration::from_secs(20), self.stdout.next_line())
.await
.expect("MCP response timeout")
.unwrap()
.expect("MCP output closed");
let response: Value = serde_json::from_str(&line).unwrap();
if response["id"] == id {
assert!(response.get("error").is_none(), "{response}");
return response["result"].clone();
}
}
}
async fn call(&mut self, name: &str, arguments: Value) -> Value {
let result = self
.request("tools/call", json!({"name":name, "arguments":arguments}))
.await;
let value: Value =
serde_json::from_str(result["content"][0]["text"].as_str().unwrap()).unwrap();
assert!(value.get("error").is_none(), "{value}");
value
}
}
use open_ontologies::eval_alignment::{AlignmentEntry, evaluate};
fn entry(source: &str, target: &str, relation: &str) -> AlignmentEntry {
AlignmentEntry {
source: source.into(),
target: target.into(),
relation: relation.into(),
}
}
type Scores = (f64, f64, f64, usize, usize, usize);
struct Case {
reference: Vec<AlignmentEntry>,
computed: Vec<AlignmentEntry>,
expected: Scores,
}
fn cases() -> Vec<Case> {
let a = entry(
"https://example.org/Order",
"https://example.org/Purchase",
"equivalent",
);
let b = entry(
"https://example.org/Quantity",
"https://example.org/Amount",
"equivalent",
);
let c = entry(
"https://example.org/Customer",
"https://example.org/Buyer",
"subsumed_by",
);
let different_relation = entry(&a.source, &a.target, "subsumed_by");
vec![
(
vec![a.clone(), b.clone()],
vec![a.clone(), b.clone()],
1.0,
1.0,
1.0,
2,
0,
0,
),
(
vec![a.clone(), a.clone(), b.clone()],
vec![a.clone(), b.clone()],
1.0,
1.0,
1.0,
2,
0,
0,
),
(
vec![a.clone(), b.clone()],
vec![a.clone(), a.clone(), b.clone()],
1.0,
1.0,
1.0,
2,
0,
0,
),
(
vec![a.clone(), a.clone(), b.clone()],
vec![a.clone(), b.clone(), b.clone()],
1.0,
1.0,
1.0,
2,
0,
0,
),
(
vec![a.clone(), a.clone(), b],
vec![a.clone(), a.clone(), c],
0.5,
0.5,
0.5,
1,
1,
1,
),
(
vec![a.clone(), a.clone()],
vec![different_relation.clone(), different_relation],
0.0,
0.0,
0.0,
0,
1,
1,
),
(vec![a.clone(), a.clone()], vec![], 0.0, 0.0, 0.0, 0, 0, 1),
(vec![], vec![a.clone(), a], 0.0, 0.0, 0.0, 0, 1, 0),
(vec![], vec![], 0.0, 0.0, 0.0, 0, 0, 0),
]
.into_iter()
.map(
|(reference, computed, precision, recall, f1, tp, fp, fn_)| Case {
reference,
computed,
expected: (precision, recall, f1, tp, fp, fn_),
},
)
.collect()
}
fn assert_report(result: &Value, expected: Scores) {
let (precision, recall, f1, tp, fp, fn_) = expected;
assert_eq!(result["precision"], precision);
assert_eq!(result["recall"], recall);
assert_eq!(result["f1"], f1);
assert_eq!(result["true_positive"], tp);
assert_eq!(result["false_positive"], fp);
assert_eq!(result["false_negative"], fn_);
assert_eq!(result["reference_size"], tp + fn_);
assert_eq!(result["computed_size"], tp + fp);
}
#[test]
fn repeated_alignment_entries_do_not_change_set_scores_or_relation_identity() {
for Case {
reference,
computed,
expected,
} in cases()
{
let result = serde_json::to_value(evaluate(&reference, &computed)).unwrap();
assert_report(&result, expected);
}
}
#[tokio::test]
async fn public_mcp_alignment_scores_and_counts_agree_with_the_same_sets() {
let tmp = tempfile::tempdir().unwrap();
let mut mcp = Mcp::start(tmp.path()).await;
for Case {
reference,
computed,
expected,
} in cases()
{
let result = mcp
.call(
"onto_eval_alignment",
json!({
"reference_json": serde_json::to_string(&reference).unwrap(),
"computed_json": serde_json::to_string(&computed).unwrap(),
}),
)
.await;
assert_report(&result, expected);
}
mcp.child.kill().await.unwrap();
}