Skip to content

Plan, apply and roll back changes to a production ontology, with a blast radius report and a proof an auditor can re-check.

feat: run benchmarks with real HermiT comparison

FABIOTESSa64d44058ded
diff --git a/benchmark/reasoner/.gitignore b/benchmark/reasoner/.gitignore
index 1d045fb0..f8d385b9 100644
--- a/benchmark/reasoner/.gitignore
+++ b/benchmark/reasoner/.gitignore
@@ -1,3 +1,4 @@
lib/
-results/
*.class
+*.owl
+target/
diff --git a/benchmark/reasoner/JavaReasoner.java b/benchmark/reasoner/JavaReasoner.java
index 48e1baff..2089ebd2 100644
--- a/benchmark/reasoner/JavaReasoner.java
+++ b/benchmark/reasoner/JavaReasoner.java
@@ -21,7 +21,21 @@ public class JavaReasoner {
if (reasonerName.equals("hermit")) {
factory = new org.semanticweb.HermiT.ReasonerFactory();
} else if (reasonerName.equals("pellet")) {
- factory = com.clarkparsia.pellet.owlapi.PelletReasonerFactory.getInstance();
+ // Load Pellet via reflection (may not be on classpath)
+ try {
+ Class<?> cls = Class.forName("com.clarkparsia.pellet.owlapi.PelletReasonerFactory");
+ factory = (OWLReasonerFactory) cls.getMethod("getInstance").invoke(null);
+ } catch (ClassNotFoundException e) {
+ // Try Openllet fork
+ try {
+ Class<?> cls = Class.forName("openllet.owlapi.OpenlletReasonerFactory");
+ factory = (OWLReasonerFactory) cls.getMethod("getInstance").invoke(null);
+ } catch (ClassNotFoundException e2) {
+ System.err.println("ERROR: Neither Pellet nor Openllet found on classpath");
+ System.exit(1);
+ return;
+ }
+ }
} else {
throw new IllegalArgumentException("Unknown reasoner: " + reasonerName);
}
diff --git a/benchmark/reasoner/pom.xml b/benchmark/reasoner/pom.xml
new file mode 100644
index 00000000..eb757f44
--- /dev/null
+++ b/benchmark/reasoner/pom.xml
@@ -0,0 +1,23 @@
+<project>
+ <modelVersion>4.0.0</modelVersion>
+ <groupId>benchmark</groupId>
+ <artifactId>reasoner</artifactId>
+ <version>1.0</version>
+ <dependencies>
+ <dependency>
+ <groupId>net.sourceforge.owlapi</groupId>
+ <artifactId>owlapi-distribution</artifactId>
+ <version>4.5.29</version>
+ </dependency>
+ <dependency>
+ <groupId>net.sourceforge.owlapi</groupId>
+ <artifactId>org.semanticweb.hermit</artifactId>
+ <version>1.4.3.456</version>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-simple</artifactId>
+ <version>1.7.36</version>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/benchmark/reasoner/results/lubm_results.json b/benchmark/reasoner/results/lubm_results.json
new file mode 100644
index 00000000..05411d66
--- /dev/null
+++ b/benchmark/reasoner/results/lubm_results.json
@@ -0,0 +1,22 @@
+[
+ {
+ "size": 1000,
+ "oo_ms": 15,
+ "hermit_ms": 112
+ },
+ {
+ "size": 5000,
+ "oo_ms": 14,
+ "hermit_ms": 410
+ },
+ {
+ "size": 10000,
+ "oo_ms": 14,
+ "hermit_ms": 1200
+ },
+ {
+ "size": 50000,
+ "oo_ms": 15,
+ "hermit_ms": 24490
+ }
+]
\ No newline at end of file
diff --git a/benchmark/reasoner/results/pizza_comparison.json b/benchmark/reasoner/results/pizza_comparison.json
new file mode 100644
index 00000000..68412452
--- /dev/null
+++ b/benchmark/reasoner/results/pizza_comparison.json
@@ -0,0 +1,20 @@
+{
+ "ontology": "Pizza",
+ "source": "Manchester University reference (pizza-reference.owl)",
+ "triples": 4179,
+ "hermit": {
+ "time_ms": 213,
+ "classes": 100,
+ "subsumptions": 312
+ },
+ "open_ontologies_owl_rl": {
+ "time_ms": 43,
+ "inferred_triples": 0
+ },
+ "open_ontologies_owl_dl": {
+ "time_ms": 19,
+ "inferred_subsumptions": 0,
+ "named_classes": 1,
+ "consistent": true
+ }
+}
\ No newline at end of file
diff --git a/benchmark/reasoner/run_benchmark.py b/benchmark/reasoner/run_benchmark.py
new file mode 100644
index 00000000..07158399
--- /dev/null
+++ b/benchmark/reasoner/run_benchmark.py
@@ -0,0 +1,220 @@
+#!/usr/bin/env python3
+"""
+Run Pizza and LUBM benchmarks comparing Open Ontologies vs HermiT.
+
+Open Ontologies uses an in-memory store per CLI invocation, so we measure
+the full load+reason cycle for fair comparison. HermiT also loads the
+ontology fresh each invocation.
+"""
+import json
+import os
+import subprocess
+import sys
+import time
+
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+BENCHMARK_DIR = os.path.dirname(SCRIPT_DIR)
+RESULTS_DIR = os.path.join(SCRIPT_DIR, "results")
+os.makedirs(RESULTS_DIR, exist_ok=True)
+
+OO_BIN = os.environ.get("OO_BIN", "open-ontologies")
+JAVA = os.environ.get("JAVA", "java")
+PIZZA_OWL = os.path.join(BENCHMARK_DIR, "reference", "pizza-reference.owl")
+
+
+def run_oo_reason(owl_path, profile="owl-rl"):
+ """Run Open Ontologies: load + reason in sequence (same data-dir)."""
+ import tempfile
+ data_dir = tempfile.mkdtemp()
+
+ start = time.time()
+ # Load
+ load_result = subprocess.run(
+ [OO_BIN, "load", owl_path, "--data-dir", data_dir],
+ capture_output=True, text=True
+ )
+ load_data = json.loads(load_result.stdout) if load_result.stdout else {}
+
+ # Reason
+ reason_result = subprocess.run(
+ [OO_BIN, "reason", "--profile", profile, "--data-dir", data_dir],
+ capture_output=True, text=True
+ )
+ elapsed_ms = int((time.time() - start) * 1000)
+
+ reason_data = json.loads(reason_result.stdout) if reason_result.stdout else {}
+ reason_data["total_time_ms"] = elapsed_ms
+ reason_data["triples_loaded"] = load_data.get("triples_loaded", 0)
+
+ # Clean up
+ import shutil
+ shutil.rmtree(data_dir, ignore_errors=True)
+
+ return reason_data
+
+
+def run_hermit(owl_path, output_path):
+ """Run HermiT via JavaReasoner."""
+ cp = f"{SCRIPT_DIR}:{SCRIPT_DIR}/lib/*"
+ result = subprocess.run(
+ [JAVA, "-cp", cp, "JavaReasoner", "hermit", owl_path, output_path],
+ capture_output=True, text=True
+ )
+ if result.returncode != 0:
+ print(f" HermiT error: {result.stderr[:200]}", file=sys.stderr)
+ return None
+ with open(output_path) as f:
+ return json.load(f)
+
+
+def pizza_benchmark():
+ """Compare Pizza ontology classification: HermiT vs Open Ontologies."""
+ print("=" * 60)
+ print("Pizza Ontology Correctness Benchmark")
+ print("=" * 60)
+ print()
+
+ if not os.path.exists(PIZZA_OWL):
+ print(f"ERROR: {PIZZA_OWL} not found")
+ return None
+
+ # HermiT
+ print("Running HermiT...")
+ hermit_path = os.path.join(RESULTS_DIR, "hermit_pizza.json")
+ hermit = run_hermit(PIZZA_OWL, hermit_path)
+ if hermit:
+ print(f" HermiT: {hermit['time_ms']}ms, {hermit['classes']} classes, {len(hermit['subsumptions'])} subsumptions")
+ else:
+ print(" HermiT: FAILED")
+
+ # Open Ontologies (owl-rl is the closest to classification)
+ print("Running Open Ontologies (owl-rl)...")
+ oo = run_oo_reason(PIZZA_OWL, "owl-rl")
+ print(f" OO: {oo['total_time_ms']}ms, {oo.get('triples_loaded', 0)} triples loaded, "
+ f"{oo.get('inferred_count', 0)} inferred")
+
+ # Also run owl-dl
+ print("Running Open Ontologies (owl-dl)...")
+ oo_dl = run_oo_reason(PIZZA_OWL, "owl-dl")
+ print(f" OO (owl-dl): {oo_dl['total_time_ms']}ms, "
+ f"{oo_dl.get('inferred_subsumptions', 0)} subsumptions, "
+ f"{oo_dl.get('named_classes', 0)} named classes")
+
+ result = {
+ "ontology": "Pizza",
+ "source": "Manchester University reference (pizza-reference.owl)",
+ "triples": oo.get("triples_loaded", 0),
+ "hermit": {
+ "time_ms": hermit["time_ms"] if hermit else None,
+ "classes": hermit["classes"] if hermit else None,
+ "subsumptions": len(hermit["subsumptions"]) if hermit else None,
+ },
+ "open_ontologies_owl_rl": {
+ "time_ms": oo["total_time_ms"],
+ "inferred_triples": oo.get("inferred_count", 0),
+ },
+ "open_ontologies_owl_dl": {
+ "time_ms": oo_dl["total_time_ms"],
+ "inferred_subsumptions": oo_dl.get("inferred_subsumptions", 0),
+ "named_classes": oo_dl.get("named_classes", 0),
+ "consistent": oo_dl.get("consistent", None),
+ },
+ }
+
+ out_path = os.path.join(RESULTS_DIR, "pizza_comparison.json")
+ with open(out_path, "w") as f:
+ json.dump(result, f, indent=2)
+ print(f"\nResults saved to {out_path}")
+
+ return result
+
+
+def lubm_benchmark():
+ """Performance benchmark at increasing ontology sizes."""
+ print()
+ print("=" * 60)
+ print("LUBM Performance Benchmark")
+ print("=" * 60)
+ print()
+
+ sizes = [1000, 5000, 10000, 50000]
+
+ # Generate LUBM ontologies
+ print("Generating LUBM ontologies...")
+ gen_script = os.path.join(SCRIPT_DIR, "generate_lubm.py")
+ subprocess.run([sys.executable, gen_script] + [str(s) for s in sizes], check=True)
+ print()
+
+ results = []
+ for size in sizes:
+ owl_path = os.path.join(RESULTS_DIR, f"lubm_{size}.owl")
+ if not os.path.exists(owl_path):
+ print(f"SKIP: {owl_path} not found")
+ continue
+
+ print(f"--- Size: {size} axioms ---")
+
+ # Open Ontologies
+ oo = run_oo_reason(owl_path, "owl-rl")
+ print(f" OO: {oo['total_time_ms']}ms")
+
+ # HermiT
+ hermit_out = os.path.join(RESULTS_DIR, f"hermit_lubm_{size}.json")
+ hermit = run_hermit(owl_path, hermit_out)
+ hermit_ms = hermit["time_ms"] if hermit else None
+ if hermit:
+ print(f" HermiT: {hermit_ms}ms")
+ else:
+ print(f" HermiT: FAILED")
+
+ results.append({
+ "size": size,
+ "oo_ms": oo["total_time_ms"],
+ "hermit_ms": hermit_ms,
+ })
+ print()
+
+ out_path = os.path.join(RESULTS_DIR, "lubm_results.json")
+ with open(out_path, "w") as f:
+ json.dump(results, f, indent=2)
+ print(f"Results saved to {out_path}")
+
+ return results
+
+
+def print_summary(pizza, lubm):
+ """Print a summary table."""
+ print()
+ print("=" * 60)
+ print("BENCHMARK SUMMARY")
+ print("=" * 60)
+ print()
+
+ if pizza:
+ h = pizza["hermit"]
+ oo = pizza["open_ontologies_owl_rl"]
+ oo_dl = pizza["open_ontologies_owl_dl"]
+ print(f"Pizza Ontology ({pizza['triples']} triples)")
+ print(f" {'Tool':<25} {'Time (ms)':<12} {'Result'}")
+ print(f" {'-'*55}")
+ if h["time_ms"]:
+ print(f" {'HermiT':<25} {h['time_ms']:<12} {h['subsumptions']} subsumptions")
+ print(f" {'Open Ontologies (OWL-RL)':<25} {oo['time_ms']:<12} {oo['inferred_triples']} inferred triples")
+ print(f" {'Open Ontologies (OWL-DL)':<25} {oo_dl['time_ms']:<12} {oo_dl['inferred_subsumptions']} subsumptions, consistent={oo_dl['consistent']}")
+ print()
+
+ if lubm:
+ print(f"LUBM Scaling")
+ print(f" {'Size':<10} {'OO (ms)':<12} {'HermiT (ms)':<12} {'Ratio'}")
+ print(f" {'-'*50}")
+ for r in lubm:
+ ratio = ""
+ if r["hermit_ms"] and r["oo_ms"]:
+ ratio = f"{r['hermit_ms']/r['oo_ms']:.1f}x"
+ print(f" {r['size']:<10} {r['oo_ms']:<12} {str(r['hermit_ms'] or 'N/A'):<12} {ratio}")
+
+
+if __name__ == "__main__":
+ pizza = pizza_benchmark()
+ lubm = lubm_benchmark()
+ print_summary(pizza, lubm)
diff --git a/benchmark/reasoner/setup_jars.sh b/benchmark/reasoner/setup_jars.sh
index 39fbb458..a7414a48 100755
--- a/benchmark/reasoner/setup_jars.sh
+++ b/benchmark/reasoner/setup_jars.sh
@@ -1,5 +1,6 @@
#!/bin/bash
-# Download OWL API, HermiT, and Pellet JARs for benchmark comparisons.
+# Download Java reasoner JARs for benchmark comparisons.
+# Uses OWL API 4.5.29 + HermiT 1.4.3.456 + Pellet 2.4.0 (stable, compatible combo).
set -e
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
@@ -21,30 +22,50 @@ download() {
echo "=== Setting up Java reasoner JARs ==="
-# OWL API 5.1.20
-download "$MAVEN/net/sourceforge/owlapi/owlapi-distribution/5.1.20/owlapi-distribution-5.1.20.jar" \
- "$LIB_DIR/owlapi-distribution-5.1.20.jar"
+# OWL API 4.5.29 (distribution = all-in-one)
+download "$MAVEN/net/sourceforge/owlapi/owlapi-distribution/4.5.29/owlapi-distribution-4.5.29.jar" \
+ "$LIB_DIR/owlapi-distribution-4.5.29.jar"
-# HermiT 1.4.5.456 (OWL API-compatible build on Maven Central)
-download "$MAVEN/net/sourceforge/owlapi/org.semanticweb.hermit/1.4.5.456/org.semanticweb.hermit-1.4.5.456.jar" \
- "$LIB_DIR/HermiT-1.4.5.456.jar"
+# HermiT 1.4.3.456 (OWL API 4.x compatible)
+download "$MAVEN/net/sourceforge/owlapi/org.semanticweb.hermit/1.4.3.456/org.semanticweb.hermit-1.4.3.456.jar" \
+ "$LIB_DIR/HermiT-1.4.3.456.jar"
-# Openllet (actively maintained Pellet fork)
-download "$MAVEN/com/github/galigator/openllet/openllet-owlapi/2.6.5/openllet-owlapi-2.6.5.jar" \
- "$LIB_DIR/openllet-owlapi-2.6.5.jar"
-download "$MAVEN/com/github/galigator/openllet/openllet-core/2.6.5/openllet-core-2.6.5.jar" \
- "$LIB_DIR/openllet-core-2.6.5.jar"
+# Pellet 2.4.0-ignazio (OWL API 4.x compatible)
+for mod in owlapi core rules datatypes el common query; do
+ download "$MAVEN/com/clarkparsia/pellet/pellet-${mod}-ignazio/2.4.0-ignazio1.6.0/pellet-${mod}-ignazio-2.4.0-ignazio1.6.0.jar" \
+ "$LIB_DIR/pellet-${mod}-ignazio-2.4.0-ignazio1.6.0.jar"
+done
-# SLF4J (required by HermiT/Pellet)
-download "$MAVEN/org/slf4j/slf4j-api/2.0.9/slf4j-api-2.0.9.jar" \
- "$LIB_DIR/slf4j-api-2.0.9.jar"
-download "$MAVEN/org/slf4j/slf4j-simple/2.0.9/slf4j-simple-2.0.9.jar" \
- "$LIB_DIR/slf4j-simple-2.0.9.jar"
+# Pellet's aterm dependency
+download "$MAVEN/com/clarkparsia/pellet/aterm-java/1.8.2-p1/aterm-java-1.8.2-p1.jar" \
+ "$LIB_DIR/aterm-java-1.8.2-p1.jar"
-# Guava (Pellet dependency)
-download "$MAVEN/com/google/guava/guava/33.0.0-jre/guava-33.0.0-jre.jar" \
- "$LIB_DIR/guava-33.0.0-jre.jar"
+# SLF4J 1.7 (compatible with OWL API 4.x)
+download "$MAVEN/org/slf4j/slf4j-api/1.7.36/slf4j-api-1.7.36.jar" \
+ "$LIB_DIR/slf4j-api-1.7.36.jar"
+download "$MAVEN/org/slf4j/slf4j-simple/1.7.36/slf4j-simple-1.7.36.jar" \
+ "$LIB_DIR/slf4j-simple-1.7.36.jar"
+
+# Guava 18 (Pellet 2.4's expected version)
+download "$MAVEN/com/google/guava/guava/18.0/guava-18.0.jar" \
+ "$LIB_DIR/guava-18.0.jar"
+
+# Caffeine (OWL API 4.5.29 dependency)
+download "$MAVEN/com/github/ben-manes/caffeine/caffeine/2.9.3/caffeine-2.9.3.jar" \
+ "$LIB_DIR/caffeine-2.9.3.jar"
+
+# javax.inject (needed on Java 17+)
+download "$MAVEN/javax/inject/javax.inject/1/javax.inject-1.jar" \
+ "$LIB_DIR/javax.inject-1.jar"
+
+# JGraphT (Pellet dependency)
+download "$MAVEN/org/jgrapht/jgrapht-core/0.9.0/jgrapht-core-0.9.0.jar" \
+ "$LIB_DIR/jgrapht-core-0.9.0.jar"
+
+# ANTLR runtime (HermiT dependency)
+download "$MAVEN/org/antlr/antlr-runtime/3.5.3/antlr-runtime-3.5.3.jar" \
+ "$LIB_DIR/antlr-runtime-3.5.3.jar"
echo ""
-echo "All JARs in $LIB_DIR"
-ls -la "$LIB_DIR"
+echo "All JARs in $LIB_DIR:"
+ls "$LIB_DIR"