Plan, apply and roll back changes to a production ontology, with a blast radius report and a proof an auditor can re-check.
Rust ai-nativedescription-logicsformal-verificationinfrastructure-as-codeisabelle-languageknowledge-graphleanlean4mcpmcp-serverontologyowlowl2reasoningrustshaclsparqltheorem-proving
feat: run benchmarks with real HermiT comparison
FABIOTESSa64d44058ded
diff --git a/benchmark/reasoner/.gitignore b/benchmark/reasoner/.gitignoreindex 1d045fb0..f8d385b9 100644--- a/benchmark/reasoner/.gitignore+++ b/benchmark/reasoner/.gitignore@@ -1,3 +1,4 @@lib/-results/*.class+*.owl+target/diff --git a/benchmark/reasoner/JavaReasoner.java b/benchmark/reasoner/JavaReasoner.javaindex 48e1baff..2089ebd2 100644--- a/benchmark/reasoner/JavaReasoner.java+++ b/benchmark/reasoner/JavaReasoner.java@@ -21,7 +21,21 @@ public class JavaReasoner {if (reasonerName.equals("hermit")) {factory = new org.semanticweb.HermiT.ReasonerFactory();} else if (reasonerName.equals("pellet")) {- factory = com.clarkparsia.pellet.owlapi.PelletReasonerFactory.getInstance();+ // Load Pellet via reflection (may not be on classpath)+ try {+ Class<?> cls = Class.forName("com.clarkparsia.pellet.owlapi.PelletReasonerFactory");+ factory = (OWLReasonerFactory) cls.getMethod("getInstance").invoke(null);+ } catch (ClassNotFoundException e) {+ // Try Openllet fork+ try {+ Class<?> cls = Class.forName("openllet.owlapi.OpenlletReasonerFactory");+ factory = (OWLReasonerFactory) cls.getMethod("getInstance").invoke(null);+ } catch (ClassNotFoundException e2) {+ System.err.println("ERROR: Neither Pellet nor Openllet found on classpath");+ System.exit(1);+ return;+ }+ }} else {throw new IllegalArgumentException("Unknown reasoner: " + reasonerName);}diff --git a/benchmark/reasoner/pom.xml b/benchmark/reasoner/pom.xmlnew file mode 100644index 00000000..eb757f44--- /dev/null+++ b/benchmark/reasoner/pom.xml@@ -0,0 +1,23 @@+<project>+ <modelVersion>4.0.0</modelVersion>+ <groupId>benchmark</groupId>+ <artifactId>reasoner</artifactId>+ <version>1.0</version>+ <dependencies>+ <dependency>+ <groupId>net.sourceforge.owlapi</groupId>+ <artifactId>owlapi-distribution</artifactId>+ <version>4.5.29</version>+ </dependency>+ <dependency>+ <groupId>net.sourceforge.owlapi</groupId>+ <artifactId>org.semanticweb.hermit</artifactId>+ <version>1.4.3.456</version>+ </dependency>+ <dependency>+ <groupId>org.slf4j</groupId>+ <artifactId>slf4j-simple</artifactId>+ <version>1.7.36</version>+ </dependency>+ </dependencies>+</project>diff --git a/benchmark/reasoner/results/lubm_results.json b/benchmark/reasoner/results/lubm_results.jsonnew file mode 100644index 00000000..05411d66--- /dev/null+++ b/benchmark/reasoner/results/lubm_results.json@@ -0,0 +1,22 @@+[+ {+ "size": 1000,+ "oo_ms": 15,+ "hermit_ms": 112+ },+ {+ "size": 5000,+ "oo_ms": 14,+ "hermit_ms": 410+ },+ {+ "size": 10000,+ "oo_ms": 14,+ "hermit_ms": 1200+ },+ {+ "size": 50000,+ "oo_ms": 15,+ "hermit_ms": 24490+ }+]\ No newline at end of filediff --git a/benchmark/reasoner/results/pizza_comparison.json b/benchmark/reasoner/results/pizza_comparison.jsonnew file mode 100644index 00000000..68412452--- /dev/null+++ b/benchmark/reasoner/results/pizza_comparison.json@@ -0,0 +1,20 @@+{+ "ontology": "Pizza",+ "source": "Manchester University reference (pizza-reference.owl)",+ "triples": 4179,+ "hermit": {+ "time_ms": 213,+ "classes": 100,+ "subsumptions": 312+ },+ "open_ontologies_owl_rl": {+ "time_ms": 43,+ "inferred_triples": 0+ },+ "open_ontologies_owl_dl": {+ "time_ms": 19,+ "inferred_subsumptions": 0,+ "named_classes": 1,+ "consistent": true+ }+}\ No newline at end of filediff --git a/benchmark/reasoner/run_benchmark.py b/benchmark/reasoner/run_benchmark.pynew file mode 100644index 00000000..07158399--- /dev/null+++ b/benchmark/reasoner/run_benchmark.py@@ -0,0 +1,220 @@+#!/usr/bin/env python3+"""+Run Pizza and LUBM benchmarks comparing Open Ontologies vs HermiT.++Open Ontologies uses an in-memory store per CLI invocation, so we measure+the full load+reason cycle for fair comparison. HermiT also loads the+ontology fresh each invocation.+"""+import json+import os+import subprocess+import sys+import time++SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))+BENCHMARK_DIR = os.path.dirname(SCRIPT_DIR)+RESULTS_DIR = os.path.join(SCRIPT_DIR, "results")+os.makedirs(RESULTS_DIR, exist_ok=True)++OO_BIN = os.environ.get("OO_BIN", "open-ontologies")+JAVA = os.environ.get("JAVA", "java")+PIZZA_OWL = os.path.join(BENCHMARK_DIR, "reference", "pizza-reference.owl")+++def run_oo_reason(owl_path, profile="owl-rl"):+ """Run Open Ontologies: load + reason in sequence (same data-dir)."""+ import tempfile+ data_dir = tempfile.mkdtemp()++ start = time.time()+ # Load+ load_result = subprocess.run(+ [OO_BIN, "load", owl_path, "--data-dir", data_dir],+ capture_output=True, text=True+ )+ load_data = json.loads(load_result.stdout) if load_result.stdout else {}++ # Reason+ reason_result = subprocess.run(+ [OO_BIN, "reason", "--profile", profile, "--data-dir", data_dir],+ capture_output=True, text=True+ )+ elapsed_ms = int((time.time() - start) * 1000)++ reason_data = json.loads(reason_result.stdout) if reason_result.stdout else {}+ reason_data["total_time_ms"] = elapsed_ms+ reason_data["triples_loaded"] = load_data.get("triples_loaded", 0)++ # Clean up+ import shutil+ shutil.rmtree(data_dir, ignore_errors=True)++ return reason_data+++def run_hermit(owl_path, output_path):+ """Run HermiT via JavaReasoner."""+ cp = f"{SCRIPT_DIR}:{SCRIPT_DIR}/lib/*"+ result = subprocess.run(+ [JAVA, "-cp", cp, "JavaReasoner", "hermit", owl_path, output_path],+ capture_output=True, text=True+ )+ if result.returncode != 0:+ print(f" HermiT error: {result.stderr[:200]}", file=sys.stderr)+ return None+ with open(output_path) as f:+ return json.load(f)+++def pizza_benchmark():+ """Compare Pizza ontology classification: HermiT vs Open Ontologies."""+ print("=" * 60)+ print("Pizza Ontology Correctness Benchmark")+ print("=" * 60)+ print()++ if not os.path.exists(PIZZA_OWL):+ print(f"ERROR: {PIZZA_OWL} not found")+ return None++ # HermiT+ print("Running HermiT...")+ hermit_path = os.path.join(RESULTS_DIR, "hermit_pizza.json")+ hermit = run_hermit(PIZZA_OWL, hermit_path)+ if hermit:+ print(f" HermiT: {hermit['time_ms']}ms, {hermit['classes']} classes, {len(hermit['subsumptions'])} subsumptions")+ else:+ print(" HermiT: FAILED")++ # Open Ontologies (owl-rl is the closest to classification)+ print("Running Open Ontologies (owl-rl)...")+ oo = run_oo_reason(PIZZA_OWL, "owl-rl")+ print(f" OO: {oo['total_time_ms']}ms, {oo.get('triples_loaded', 0)} triples loaded, "+ f"{oo.get('inferred_count', 0)} inferred")++ # Also run owl-dl+ print("Running Open Ontologies (owl-dl)...")+ oo_dl = run_oo_reason(PIZZA_OWL, "owl-dl")+ print(f" OO (owl-dl): {oo_dl['total_time_ms']}ms, "+ f"{oo_dl.get('inferred_subsumptions', 0)} subsumptions, "+ f"{oo_dl.get('named_classes', 0)} named classes")++ result = {+ "ontology": "Pizza",+ "source": "Manchester University reference (pizza-reference.owl)",+ "triples": oo.get("triples_loaded", 0),+ "hermit": {+ "time_ms": hermit["time_ms"] if hermit else None,+ "classes": hermit["classes"] if hermit else None,+ "subsumptions": len(hermit["subsumptions"]) if hermit else None,+ },+ "open_ontologies_owl_rl": {+ "time_ms": oo["total_time_ms"],+ "inferred_triples": oo.get("inferred_count", 0),+ },+ "open_ontologies_owl_dl": {+ "time_ms": oo_dl["total_time_ms"],+ "inferred_subsumptions": oo_dl.get("inferred_subsumptions", 0),+ "named_classes": oo_dl.get("named_classes", 0),+ "consistent": oo_dl.get("consistent", None),+ },+ }++ out_path = os.path.join(RESULTS_DIR, "pizza_comparison.json")+ with open(out_path, "w") as f:+ json.dump(result, f, indent=2)+ print(f"\nResults saved to {out_path}")++ return result+++def lubm_benchmark():+ """Performance benchmark at increasing ontology sizes."""+ print()+ print("=" * 60)+ print("LUBM Performance Benchmark")+ print("=" * 60)+ print()++ sizes = [1000, 5000, 10000, 50000]++ # Generate LUBM ontologies+ print("Generating LUBM ontologies...")+ gen_script = os.path.join(SCRIPT_DIR, "generate_lubm.py")+ subprocess.run([sys.executable, gen_script] + [str(s) for s in sizes], check=True)+ print()++ results = []+ for size in sizes:+ owl_path = os.path.join(RESULTS_DIR, f"lubm_{size}.owl")+ if not os.path.exists(owl_path):+ print(f"SKIP: {owl_path} not found")+ continue++ print(f"--- Size: {size} axioms ---")++ # Open Ontologies+ oo = run_oo_reason(owl_path, "owl-rl")+ print(f" OO: {oo['total_time_ms']}ms")++ # HermiT+ hermit_out = os.path.join(RESULTS_DIR, f"hermit_lubm_{size}.json")+ hermit = run_hermit(owl_path, hermit_out)+ hermit_ms = hermit["time_ms"] if hermit else None+ if hermit:+ print(f" HermiT: {hermit_ms}ms")+ else:+ print(f" HermiT: FAILED")++ results.append({+ "size": size,+ "oo_ms": oo["total_time_ms"],+ "hermit_ms": hermit_ms,+ })+ print()++ out_path = os.path.join(RESULTS_DIR, "lubm_results.json")+ with open(out_path, "w") as f:+ json.dump(results, f, indent=2)+ print(f"Results saved to {out_path}")++ return results+++def print_summary(pizza, lubm):+ """Print a summary table."""+ print()+ print("=" * 60)+ print("BENCHMARK SUMMARY")+ print("=" * 60)+ print()++ if pizza:+ h = pizza["hermit"]+ oo = pizza["open_ontologies_owl_rl"]+ oo_dl = pizza["open_ontologies_owl_dl"]+ print(f"Pizza Ontology ({pizza['triples']} triples)")+ print(f" {'Tool':<25} {'Time (ms)':<12} {'Result'}")+ print(f" {'-'*55}")+ if h["time_ms"]:+ print(f" {'HermiT':<25} {h['time_ms']:<12} {h['subsumptions']} subsumptions")+ print(f" {'Open Ontologies (OWL-RL)':<25} {oo['time_ms']:<12} {oo['inferred_triples']} inferred triples")+ print(f" {'Open Ontologies (OWL-DL)':<25} {oo_dl['time_ms']:<12} {oo_dl['inferred_subsumptions']} subsumptions, consistent={oo_dl['consistent']}")+ print()++ if lubm:+ print(f"LUBM Scaling")+ print(f" {'Size':<10} {'OO (ms)':<12} {'HermiT (ms)':<12} {'Ratio'}")+ print(f" {'-'*50}")+ for r in lubm:+ ratio = ""+ if r["hermit_ms"] and r["oo_ms"]:+ ratio = f"{r['hermit_ms']/r['oo_ms']:.1f}x"+ print(f" {r['size']:<10} {r['oo_ms']:<12} {str(r['hermit_ms'] or 'N/A'):<12} {ratio}")+++if __name__ == "__main__":+ pizza = pizza_benchmark()+ lubm = lubm_benchmark()+ print_summary(pizza, lubm)diff --git a/benchmark/reasoner/setup_jars.sh b/benchmark/reasoner/setup_jars.shindex 39fbb458..a7414a48 100755--- a/benchmark/reasoner/setup_jars.sh+++ b/benchmark/reasoner/setup_jars.sh@@ -1,5 +1,6 @@#!/bin/bash-# Download OWL API, HermiT, and Pellet JARs for benchmark comparisons.+# Download Java reasoner JARs for benchmark comparisons.+# Uses OWL API 4.5.29 + HermiT 1.4.3.456 + Pellet 2.4.0 (stable, compatible combo).set -eSCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"@@ -21,30 +22,50 @@ download() {echo "=== Setting up Java reasoner JARs ==="-# OWL API 5.1.20-download "$MAVEN/net/sourceforge/owlapi/owlapi-distribution/5.1.20/owlapi-distribution-5.1.20.jar" \- "$LIB_DIR/owlapi-distribution-5.1.20.jar"+# OWL API 4.5.29 (distribution = all-in-one)+download "$MAVEN/net/sourceforge/owlapi/owlapi-distribution/4.5.29/owlapi-distribution-4.5.29.jar" \+ "$LIB_DIR/owlapi-distribution-4.5.29.jar"-# HermiT 1.4.5.456 (OWL API-compatible build on Maven Central)-download "$MAVEN/net/sourceforge/owlapi/org.semanticweb.hermit/1.4.5.456/org.semanticweb.hermit-1.4.5.456.jar" \- "$LIB_DIR/HermiT-1.4.5.456.jar"+# HermiT 1.4.3.456 (OWL API 4.x compatible)+download "$MAVEN/net/sourceforge/owlapi/org.semanticweb.hermit/1.4.3.456/org.semanticweb.hermit-1.4.3.456.jar" \+ "$LIB_DIR/HermiT-1.4.3.456.jar"-# Openllet (actively maintained Pellet fork)-download "$MAVEN/com/github/galigator/openllet/openllet-owlapi/2.6.5/openllet-owlapi-2.6.5.jar" \- "$LIB_DIR/openllet-owlapi-2.6.5.jar"-download "$MAVEN/com/github/galigator/openllet/openllet-core/2.6.5/openllet-core-2.6.5.jar" \- "$LIB_DIR/openllet-core-2.6.5.jar"+# Pellet 2.4.0-ignazio (OWL API 4.x compatible)+for mod in owlapi core rules datatypes el common query; do+ download "$MAVEN/com/clarkparsia/pellet/pellet-${mod}-ignazio/2.4.0-ignazio1.6.0/pellet-${mod}-ignazio-2.4.0-ignazio1.6.0.jar" \+ "$LIB_DIR/pellet-${mod}-ignazio-2.4.0-ignazio1.6.0.jar"+done-# SLF4J (required by HermiT/Pellet)-download "$MAVEN/org/slf4j/slf4j-api/2.0.9/slf4j-api-2.0.9.jar" \- "$LIB_DIR/slf4j-api-2.0.9.jar"-download "$MAVEN/org/slf4j/slf4j-simple/2.0.9/slf4j-simple-2.0.9.jar" \- "$LIB_DIR/slf4j-simple-2.0.9.jar"+# Pellet's aterm dependency+download "$MAVEN/com/clarkparsia/pellet/aterm-java/1.8.2-p1/aterm-java-1.8.2-p1.jar" \+ "$LIB_DIR/aterm-java-1.8.2-p1.jar"-# Guava (Pellet dependency)-download "$MAVEN/com/google/guava/guava/33.0.0-jre/guava-33.0.0-jre.jar" \- "$LIB_DIR/guava-33.0.0-jre.jar"+# SLF4J 1.7 (compatible with OWL API 4.x)+download "$MAVEN/org/slf4j/slf4j-api/1.7.36/slf4j-api-1.7.36.jar" \+ "$LIB_DIR/slf4j-api-1.7.36.jar"+download "$MAVEN/org/slf4j/slf4j-simple/1.7.36/slf4j-simple-1.7.36.jar" \+ "$LIB_DIR/slf4j-simple-1.7.36.jar"++# Guava 18 (Pellet 2.4's expected version)+download "$MAVEN/com/google/guava/guava/18.0/guava-18.0.jar" \+ "$LIB_DIR/guava-18.0.jar"++# Caffeine (OWL API 4.5.29 dependency)+download "$MAVEN/com/github/ben-manes/caffeine/caffeine/2.9.3/caffeine-2.9.3.jar" \+ "$LIB_DIR/caffeine-2.9.3.jar"++# javax.inject (needed on Java 17+)+download "$MAVEN/javax/inject/javax.inject/1/javax.inject-1.jar" \+ "$LIB_DIR/javax.inject-1.jar"++# JGraphT (Pellet dependency)+download "$MAVEN/org/jgrapht/jgrapht-core/0.9.0/jgrapht-core-0.9.0.jar" \+ "$LIB_DIR/jgrapht-core-0.9.0.jar"++# ANTLR runtime (HermiT dependency)+download "$MAVEN/org/antlr/antlr-runtime/3.5.3/antlr-runtime-3.5.3.jar" \+ "$LIB_DIR/antlr-runtime-3.5.3.jar"echo ""-echo "All JARs in $LIB_DIR"-ls -la "$LIB_DIR"+echo "All JARs in $LIB_DIR:"+ls "$LIB_DIR"