Skip to content

Plan, apply and roll back changes to a production ontology, with a blast radius report and a proof an auditor can re-check.

fix: make benchmark numbers factual — extract from actual TTL files

FABIOTESS0d27404f3719
diff --git a/README.md b/README.md
index 91f0e303..235e770f 100644
--- a/README.md
+++ b/README.md
@@ -122,8 +122,8 @@ The reasoner is **conservative by design** — it will flag a safe mushroom as s
| Metric | Manual | Pure Claude | RDF Pipeline |
| ------ | ------ | ----------- | ------------ |
-| Object Recall | 100% | 89% | **100%** |
-| Category Recall | 100% | 79% | **91%** |
+| Object Recall | 100% | 89% | **95%** |
+| Category Recall | 100% | 79% | 32% |
| Total RDF Triples | 0 | 0 | **2,540** |
| skos:altLabel Synonyms | 0 | 0 | **612** |
| SPARQL Queryable | No | No | **Yes** |
@@ -131,9 +131,9 @@ The reasoner is **conservative by design** — it will flag a safe mushroom as s
| Effort per image | ~2 min | ~8 sec | ~8 sec |
| Scales to 1000+ images | No | Yes | Yes |
-**Why RDF Pipeline beats Pure Claude on recall:** The v2 pipeline uses `skos:altLabel` to generate synonym expansions for every detected object (e.g., "whiskers" + "vibrissae" + "whisker pads"), covering more matching surface. Pure Claude returns loose text that often uses a single label per object.
+**Object recall:** Both approaches detect most ground truth objects. The RDF pipeline uses `skos:altLabel` synonyms (612 total) to expand matching surface, achieving 95% recall. Category recall is lower (32%) because the TTL files use fine-grained categories ("animal body part", "vehicle part") while ground truth uses broad labels ("animal", "vehicle").
-**Why this matters:** Pure Claude gives you flat text — you can't query "find all images containing animals near water." The RDF pipeline produces a queryable knowledge graph with 2,540 triples and 612 synonyms across 10 images.
+**Why this matters:** Pure Claude gives you flat text — you can't query "find all images containing animals near water." The RDF pipeline produces a queryable knowledge graph with 2,540 triples, 204 spatial relationships, and 612 synonyms across 10 images.
**Full MCP pipeline:** The benchmark runs the complete `onto_clear` → `onto_validate` (×10) → `onto_load` (×10) → `onto_stats` → `onto_lint` (×10) → `onto_query` (×6) tool chain via the real MCP server (`open-ontologies serve`), using the official MCP Python SDK over JSON-RPC 2.0 stdio. All 10 images validated, loaded into Oxigraph, linted clean, and queried with SPARQL — the same protocol Claude uses.
diff --git a/benchmark/vision/compare_approaches.py b/benchmark/vision/compare_approaches.py
index 92791a4d..c3e7dc2c 100644
--- a/benchmark/vision/compare_approaches.py
+++ b/benchmark/vision/compare_approaches.py
@@ -3,23 +3,28 @@
Computes object recall, category recall, and structural metrics across
10 real photographs processed by parallel Claude agents.
+
+RDF pipeline data is extracted directly from the actual TTL files —
+no hardcoded values.
"""
import json
import os
+import re
import glob
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
DATASET_DIR = os.path.join(SCRIPT_DIR, "dataset")
-
-def fuzzy_match(detected: str, ground_truth_set: set) -> bool:
- """Check if a detected label fuzzy-matches any ground truth label."""
- d = detected.lower().strip()
- for gt in ground_truth_set:
- g = gt.lower().strip()
- if d == g or d in g or g in d:
- return True
- return False
+# Spatial relationship labels to exclude from object lists
+SPATIAL_LABELS = {
+ "above", "below", "behind", "in front of", "beside", "left of", "right of",
+ "near", "inside", "on top of", "beneath", "surrounding", "centered in",
+ "submerged in", "covered by", "surrounded by", "part of", "floating on",
+ "covering", "clinging to", "composed of", "above-left of", "resting on",
+ "between", "exhibited by", "provided by", "contains", "habitat for",
+ "standing on", "illuminated by", "same as", "source of", "illuminates",
+ "filters through", "on", "leading up to", "partially obscured by",
+}
def compute_recall(detected: list, ground_truth: list) -> float:
@@ -49,6 +54,36 @@ def compute_precision(detected: list, ground_truth: list) -> float:
return hits / len(detected)
+def extract_from_ttl(ttl_path):
+ """Extract objects, categories, and counts from a TTL file."""
+ content = open(ttl_path).read()
+
+ # rdfs:label values (exclude spatial relationship labels)
+ labels = []
+ for m in re.findall(r'rdfs:label\s+"([^"]+)"', content):
+ if m.lower() not in SPATIAL_LABELS:
+ labels.append(m)
+
+ # skos:altLabel values
+ alt_labels = re.findall(r'skos:altLabel\s+"([^"]+)"', content)
+
+ # Categories
+ categories = sorted(set(re.findall(r'schema:category\s+"([^"]+)"', content)))
+
+ # Relationship count (spatial predicates)
+ rel_patterns = r'ex:spatialRelation|ex:near|ex:above|ex:below|ex:beside|ex:inside|ex:behind|ex:leftOf|ex:rightOf|ex:surrounding|ex:partOf|ex:adjacentTo'
+ relationships = len(re.findall(rel_patterns, content))
+
+ # Combined object list (labels + altLabels, deduplicated)
+ all_objects = list(dict.fromkeys(labels + alt_labels))
+
+ return {
+ "objects": all_objects,
+ "categories": categories,
+ "relationships": relationships,
+ }
+
+
# Ground truth (manual annotation)
with open(os.path.join(DATASET_DIR, "ground_truth.json")) as f:
ground_truth = json.load(f)
@@ -97,88 +132,39 @@ pure_claude = {
},
}
-# RDF Pipeline v2 results — with skos:altLabel synonyms for matching
-# "all_labels" includes rdfs:label + all skos:altLabel values
-rdf_pipeline = {
- "img_1.jpg": {
- "objects": ["truck", "vehicle", "steering wheel", "dashboard", "windshield", "tree",
- "plant", "bush", "leaves", "rust", "metal", "seat", "gauges", "sky",
- "steering column", "glass", "moss", "dirt", "foliage", "branches"],
- "categories": ["vehicle", "nature", "decay", "material"],
- "triples": 370, "relationships": 20,
- },
- "img_2.jpg": {
- "objects": ["laptop", "notebook", "pen", "camera", "earphones", "earbuds", "headphones",
- "desk", "wooden desk", "screen", "keyboard", "sketches", "notes",
- "camera lens", "earphone cable"],
- "categories": ["technology", "workspace", "photography", "stationery"],
- "triples": 254, "relationships": 14,
- },
- "img_3.jpg": {
- "objects": ["frog", "amphibian", "duckweed", "water", "pond", "eyes", "insect",
- "frog head", "frog nostrils", "eyelids", "skin", "camouflage",
- "vegetation mat", "wildlife"],
- "categories": ["animal", "amphibian", "nature", "water", "aquatic plant", "insect"],
- "triples": 286, "relationships": 32,
- },
- "img_4.jpg": {
- "objects": ["sky", "trees", "conifers", "pines", "evergreens", "forest", "woods",
- "field", "meadow", "grassland", "hill", "hills", "grass", "tree trunks",
- "mist", "haze", "fog", "canopy", "slope", "clouds", "landscape"],
- "categories": ["landscape", "nature", "forest", "vegetation", "terrain"],
- "triples": 275, "relationships": 18,
- },
- "img_5.jpg": {
- "objects": ["boardwalk", "wood planks", "wooden planks", "deck boards", "planks",
- "sky", "buildings", "tower", "water tower", "horizon",
- "sunlight", "shadows", "nail holes", "rooflines"],
- "categories": ["architecture", "urban", "nature", "outdoor"],
- "triples": 355, "relationships": 16,
- },
- "img_6.jpg": {
- "objects": ["canoe", "boat", "people", "persons", "paddlers", "paddler", "person",
- "paddle", "oar", "water", "lake", "fog", "mist", "reflection",
- "hat", "jacket", "coat", "sky", "horizon"],
- "categories": ["recreation", "water", "nature", "people", "watercraft"],
- "triples": 250, "relationships": 22,
- },
- "img_7.jpg": {
- "objects": ["ocean", "sea", "beach", "sand", "coastline", "coast", "shoreline",
- "cliffs", "bluffs", "buildings", "houses", "homes", "roads", "streets",
- "trees", "vegetation", "greenery", "boat", "vessel",
- "golf course", "waves", "surf", "haze", "canyon", "headland"],
- "categories": ["landscape", "nature", "coastal", "urban", "water", "aerial"],
- "triples": 320, "relationships": 28,
- },
- "img_8.jpg": {
- "objects": ["fawn", "deer", "animal", "leaves", "fallen leaves", "trees", "forest",
- "sunlight", "light", "branch", "log", "dust motes", "foliage",
- "spots", "forest floor", "backlight"],
- "categories": ["animal", "wildlife", "nature", "forest", "vegetation"],
- "triples": 257, "relationships": 16,
- },
- "img_9.jpg": {
- "objects": ["cat", "feline", "tabby", "nose", "whiskers", "vibrissae",
- "fur", "coat", "hair", "mouth", "lips", "nostrils", "muzzle",
- "philtrum", "chin", "nose bridge", "whisker pads", "eyelids"],
- "categories": ["animal", "pet", "body part", "covering"],
- "triples": 300, "relationships": 20,
- },
- "img_10.jpg": {
- "objects": ["van", "volkswagen", "VW bus", "camper van", "vehicle", "minibus",
- "trees", "foliage", "vegetation", "road", "asphalt", "street",
- "wheels", "tires", "windows", "roof", "rust", "corrosion",
- "fallen leaves", "door", "bumper"],
- "categories": ["vehicle", "nature", "transportation", "vintage"],
- "triples": 300, "relationships": 22,
- },
-}
+# RDF Pipeline — extract directly from actual TTL files
+rdf_pipeline = {}
+for ttl in sorted(glob.glob(os.path.join(DATASET_DIR, "img_*.ttl"))):
+ img_key = os.path.basename(ttl).replace(".ttl", ".jpg")
+ rdf_pipeline[img_key] = extract_from_ttl(ttl)
+
+# Get validated triple counts from MCP pipeline results
+mcp_results_path = os.path.join(DATASET_DIR, "mcp_pipeline_results.json")
+if os.path.exists(mcp_results_path):
+ with open(mcp_results_path) as f:
+ mcp_results = json.load(f)
+ total_triples_from_mcp = mcp_results.get("total_triples_validated", 0)
+else:
+ total_triples_from_mcp = 0
+
+# Get per-file triple counts by running onto_validate via CLI
+OO_BIN = os.path.join(SCRIPT_DIR, "..", "..", "target", "release", "open-ontologies")
+import subprocess
+for img_key, data in rdf_pipeline.items():
+ ttl_path = os.path.join(DATASET_DIR, img_key.replace(".jpg", ".ttl"))
+ try:
+ result = subprocess.run(
+ [OO_BIN, "validate", ttl_path],
+ capture_output=True, text=True, timeout=10
+ )
+ parsed = json.loads(result.stdout.strip())
+ data["triples"] = parsed.get("triple_count", parsed.get("triples", 0))
+ except Exception:
+ data["triples"] = 0
def main():
results = {}
- total_manual_obj = 0
- total_manual_cat = 0
for img in sorted(ground_truth.keys()):
gt = ground_truth[img]
diff --git a/benchmark/vision/dataset/benchmark_results.json b/benchmark/vision/dataset/benchmark_results.json
index 3b20cb9a..94464ba6 100644
--- a/benchmark/vision/dataset/benchmark_results.json
+++ b/benchmark/vision/dataset/benchmark_results.json
@@ -20,12 +20,12 @@
},
"rdf_pipeline": {
"description": "Claude vision \u2192 structured Turtle with ontology, validated with Open Ontologies",
- "avg_object_recall": 1.0,
+ "avg_object_recall": 0.95,
"avg_object_precision": 0.35,
- "avg_category_recall": 0.91,
- "avg_objects_per_image": 18.3,
- "total_triples": 2967,
- "total_relationships": 208,
+ "avg_category_recall": 0.33,
+ "avg_objects_per_image": 29.7,
+ "total_triples": 2540,
+ "total_relationships": 204,
"ttl_files_generated": 10,
"queryable": true,
"relationships": true,
@@ -43,10 +43,10 @@
"category_recall": 1.0
},
"rdf_pipeline": {
- "objects_detected": 20,
+ "objects_detected": 46,
"object_recall": 1.0,
- "object_precision": 0.25,
- "category_recall": 1.0,
+ "object_precision": 0.3,
+ "category_recall": 0.67,
"triples": 370,
"relationships": 20
}
@@ -61,12 +61,12 @@
"category_recall": 0.33
},
"rdf_pipeline": {
- "objects_detected": 21,
- "object_recall": 1.0,
- "object_precision": 0.24,
- "category_recall": 0.67,
- "triples": 300,
- "relationships": 22
+ "objects_detected": 27,
+ "object_recall": 0.75,
+ "object_precision": 0.26,
+ "category_recall": 0.33,
+ "triples": 184,
+ "relationships": 23
}
},
"img_2.jpg": {
@@ -79,12 +79,12 @@
"category_recall": 1.0
},
"rdf_pipeline": {
- "objects_detected": 15,
+ "objects_detected": 24,
"object_recall": 1.0,
- "object_precision": 0.53,
- "category_recall": 1.0,
+ "object_precision": 0.54,
+ "category_recall": 0.0,
"triples": 254,
- "relationships": 14
+ "relationships": 12
}
},
"img_3.jpg": {
@@ -97,10 +97,10 @@
"category_recall": 0.75
},
"rdf_pipeline": {
- "objects_detected": 14,
+ "objects_detected": 27,
"object_recall": 1.0,
- "object_precision": 0.36,
- "category_recall": 1.0,
+ "object_precision": 0.37,
+ "category_recall": 0.75,
"triples": 286,
"relationships": 32
}
@@ -115,12 +115,12 @@
"category_recall": 1.0
},
"rdf_pipeline": {
- "objects_detected": 21,
+ "objects_detected": 24,
"object_recall": 1.0,
- "object_precision": 0.29,
- "category_recall": 1.0,
+ "object_precision": 0.33,
+ "category_recall": 0.67,
"triples": 275,
- "relationships": 18
+ "relationships": 23
}
},
"img_5.jpg": {
@@ -133,12 +133,12 @@
"category_recall": 0.67
},
"rdf_pipeline": {
- "objects_detected": 14,
+ "objects_detected": 34,
"object_recall": 1.0,
- "object_precision": 0.36,
- "category_recall": 1.0,
+ "object_precision": 0.15,
+ "category_recall": 0.0,
"triples": 355,
- "relationships": 16
+ "relationships": 13
}
},
"img_6.jpg": {
@@ -151,12 +151,12 @@
"category_recall": 1.0
},
"rdf_pipeline": {
- "objects_detected": 19,
+ "objects_detected": 25,
"object_recall": 1.0,
- "object_precision": 0.42,
- "category_recall": 1.0,
- "triples": 250,
- "relationships": 22
+ "object_precision": 0.52,
+ "category_recall": 0.25,
+ "triples": 155,
+ "relationships": 8
}
},
"img_7.jpg": {
@@ -169,12 +169,12 @@
"category_recall": 0.5
},
"rdf_pipeline": {
- "objects_detected": 25,
+ "objects_detected": 40,
"object_recall": 1.0,
"object_precision": 0.28,
- "category_recall": 0.75,
- "triples": 320,
- "relationships": 28
+ "category_recall": 0.0,
+ "triples": 259,
+ "relationships": 30
}
},
"img_8.jpg": {
@@ -187,12 +187,12 @@
"category_recall": 1.0
},
"rdf_pipeline": {
- "objects_detected": 16,
+ "objects_detected": 22,
"object_recall": 1.0,
- "object_precision": 0.5,
- "category_recall": 1.0,
+ "object_precision": 0.32,
+ "category_recall": 0.25,
"triples": 257,
- "relationships": 16
+ "relationships": 25
}
},
"img_9.jpg": {
@@ -205,12 +205,12 @@
"category_recall": 0.67
},
"rdf_pipeline": {
- "objects_detected": 18,
- "object_recall": 1.0,
- "object_precision": 0.28,
- "category_recall": 0.67,
- "triples": 300,
- "relationships": 20
+ "objects_detected": 28,
+ "object_recall": 0.75,
+ "object_precision": 0.43,
+ "category_recall": 0.33,
+ "triples": 145,
+ "relationships": 18
}
}
}