Plan, apply and roll back changes to a production ontology, with a blast radius report and a proof an auditor can re-check.
Rust ai-nativedescription-logicsformal-verificationinfrastructure-as-codeisabelle-languageknowledge-graphleanlean4mcpmcp-serverontologyowlowl2reasoningrustshaclsparqltheorem-proving
#!/usr/bin/env python3
"""
Benchmark: Compare Claude-generated IES Building ontology against IES4 reference patterns.
Checks:
1. Structural compliance — 4D pattern (Entity + EntityState for each concept)
2. IES4 class hierarchy compliance — correct superclass usage
3. Property pattern compliance — subPropertyOf ies:relationship/isPartOf/isIdentifiedBy
4. Naming convention compliance — alphabetical order, rdfs:label, rdfs:comment
5. Competency question answerability — can SPARQL answer CQ1-CQ9
6. Turtle syntax validity — parseable by rdflib
"""
import sys
try:
from rdflib import Graph, Namespace, RDF, RDFS, OWL, XSD
from rdflib.namespace import DCTERMS
except ImportError:
print("Installing rdflib...")
import subprocess
subprocess.check_call([sys.executable, "-m", "pip", "install", "rdflib", "-q"])
from rdflib import Graph, Namespace, RDF, RDFS, OWL, XSD
from rdflib.namespace import DCTERMS
import json
from collections import defaultdict
IES = Namespace("http://ies.data.gov.uk/ontology/ies4#")
BLDG = Namespace("http://example.org/ontology/ies-building#")
def load_ontology(path):
g = Graph()
g.parse(path, format="turtle")
return g
def check_syntax(path):
"""Check 1: Can rdflib parse it?"""
try:
g = load_ontology(path)
triple_count = len(g)
return True, triple_count, g
except Exception as e:
return False, 0, str(e)
def check_4d_pattern(g):
"""Check 2: Every entity class has a matching State class"""
bldg_classes = set()
for s in g.subjects(RDF.type, RDFS.Class):
local = str(s).split("#")[-1] if "#" in str(s) else str(s).split("/")[-1]
if str(s).startswith(str(BLDG)):
bldg_classes.add(local)
entities = [c for c in bldg_classes if not c.endswith("State") and not c.startswith("ClassOf")
and c not in ("EnergyRatingBand", "FuelType", "PostalCode", "UPRN",
"ConstructedState", "DemolishedState", "InstalledState",
"EnergyPerformanceReview", "RetrofitIntervention", "EnergyRating")]
results = []
for entity in sorted(entities):
state_name = entity + "State"
has_state = state_name in bldg_classes
results.append({"entity": entity, "has_state": has_state, "state_name": state_name})
passed = sum(1 for r in results if r["has_state"])
total = len(results)
return results, passed, total
def check_classof_pattern(g):
"""Check 3: ClassOf pattern (ClassOfEntity for entities, ClassOfState for states)"""
bldg_classes = {}
for s, _, o in g.triples((None, RDFS.subClassOf, None)):
if str(s).startswith(str(BLDG)):
local = str(s).split("#")[-1]
parent = str(o).split("#")[-1]
if local not in bldg_classes:
bldg_classes[local] = []
bldg_classes[local].append(parent)
classof_entities = [c for c in bldg_classes if c.startswith("ClassOf") and not c.endswith("State")]
classof_states = [c for c in bldg_classes if c.startswith("ClassOf") and c.endswith("State")]
results = []
for c in classof_entities:
parents = bldg_classes.get(c, [])
correct = "ClassOfEntity" in parents
results.append({"class": c, "parents": parents, "correct": correct})
for c in classof_states:
parents = bldg_classes.get(c, [])
correct = "ClassOfState" in parents
results.append({"class": c, "parents": parents, "correct": correct})
passed = sum(1 for r in results if r["correct"])
total = len(results)
return results, passed, total
def check_property_patterns(g):
"""Check 4: Properties use correct subPropertyOf (relationship, isPartOf, isIdentifiedBy)"""
valid_super_props = {
str(IES.relationship), str(IES.isPartOf), str(IES.isIdentifiedBy),
str(IES.isStateOf), str(IES.attribute), str(IES.inPeriod),
str(IES.isRepresentedAs), str(IES.hasName)
}
results = []
for s in g.subjects(RDF.type, OWL.ObjectProperty):
if str(s).startswith(str(BLDG)):
local = str(s).split("#")[-1]
super_props = [str(o) for o in g.objects(s, RDFS.subPropertyOf)]
has_label = bool(list(g.objects(s, RDFS.label)))
has_comment = bool(list(g.objects(s, RDFS.comment)))
has_domain = bool(list(g.objects(s, RDFS.domain)))
has_range = bool(list(g.objects(s, RDFS.range)))
has_valid_super = any(sp in valid_super_props for sp in super_props)
results.append({
"property": local,
"has_valid_super": has_valid_super,
"has_label": has_label,
"has_comment": has_comment,
"has_domain": has_domain,
"has_range": has_range,
"super_props": [sp.split("#")[-1] for sp in super_props]
})
for s in g.subjects(RDF.type, OWL.DatatypeProperty):
if str(s).startswith(str(BLDG)):
local = str(s).split("#")[-1]
super_props = [str(o) for o in g.objects(s, RDFS.subPropertyOf)]
has_label = bool(list(g.objects(s, RDFS.label)))
has_comment = bool(list(g.objects(s, RDFS.comment)))
has_valid_super = any(sp in valid_super_props for sp in super_props)
results.append({
"property": local,
"has_valid_super": has_valid_super,
"has_label": has_label,
"has_comment": has_comment,
"has_domain": True,
"has_range": True,
"super_props": [sp.split("#")[-1] for sp in super_props]
})
passed = sum(1 for r in results if r["has_valid_super"] and r["has_label"] and r["has_comment"])
total = len(results)
return results, passed, total
def check_naming_conventions(g):
"""Check 5: All classes and properties have rdfs:label and rdfs:comment"""
results = []
for s in g.subjects(RDF.type, RDFS.Class):
if str(s).startswith(str(BLDG)):
local = str(s).split("#")[-1]
has_label = bool(list(g.objects(s, RDFS.label)))
has_comment = bool(list(g.objects(s, RDFS.comment)))
results.append({"entity": local, "type": "Class", "has_label": has_label, "has_comment": has_comment})
passed = sum(1 for r in results if r["has_label"] and r["has_comment"])
total = len(results)
return results, passed, total
def check_ontology_metadata(g):
"""Check 6: Ontology declaration with proper metadata"""
ontology_iris = list(g.subjects(RDF.type, OWL.Ontology))
if not ontology_iris:
return [], 0, 1
ont = ontology_iris[0]
checks = {
"has_label": bool(list(g.objects(ont, RDFS.label))),
"has_comment": bool(list(g.objects(ont, RDFS.comment))),
"has_imports": bool(list(g.objects(ont, OWL.imports))),
"has_title": bool(list(g.objects(ont, DCTERMS.title))),
"has_description": bool(list(g.objects(ont, DCTERMS.description))),
"has_license": bool(list(g.objects(ont, DCTERMS.license))),
}
passed = sum(1 for v in checks.values() if v)
total = len(checks)
return [checks], passed, total
def check_competency_questions(g):
"""Check 7: Can the ontology answer the 9 competency questions?"""
cqs = [
{
"id": "CQ1",
"question": "Find residential properties with energy score above/below threshold",
"requires": ["Dwelling", "EnergyPerformanceCertificate", "energyScore"],
"answerable": False
},
{
"id": "CQ2",
"question": "Find properties by EPC letter band (A-G)",
"requires": ["EnergyRatingBand", "energyRatingBand"],
"answerable": False
},
{
"id": "CQ3",
"question": "Filter by postal code",
"requires": ["PostalCode"],
"answerable": False
},
{
"id": "CQ4",
"question": "Filter by group of postal codes",
"requires": ["PostalCode"],
"answerable": False
},
{
"id": "CQ5",
"question": "Filter by lat/long boundary",
"requires": [], # Uses IES4 GeoPoint, not our extension
"answerable": True # Inherited from IES4
},
{
"id": "CQ6",
"question": "Filter by OS boundary",
"requires": [], # Uses IES4 Region
"answerable": True # Inherited from IES4
},
{
"id": "CQ7",
"question": "Window, wall, floor, ceiling insulation details",
"requires": ["WindowInsulation", "WallInsulation", "FloorInsulation", "CeilingInsulation", "InsulationElement"],
"answerable": False
},
{
"id": "CQ8",
"question": "Heating systems with fuel types",
"requires": ["HeatingSystem", "FuelType", "usesFuel"],
"answerable": False
},
{
"id": "CQ9",
"question": "EPC details for a property identified by UPRN",
"requires": ["UPRN", "hasUPRN", "EnergyPerformanceCertificate", "hasEPC"],
"answerable": False
}
]
# Check what entities exist in the ontology
all_entities = set()
for s in g.subjects(RDF.type, RDFS.Class):
all_entities.add(str(s).split("#")[-1])
for s in g.subjects(RDF.type, OWL.ObjectProperty):
all_entities.add(str(s).split("#")[-1])
for s in g.subjects(RDF.type, OWL.DatatypeProperty):
all_entities.add(str(s).split("#")[-1])
for cq in cqs:
if not cq["requires"]: # Inherited from IES4
continue
cq["answerable"] = all(r in all_entities for r in cq["requires"])
cq["missing"] = [r for r in cq["requires"] if r not in all_entities]
passed = sum(1 for cq in cqs if cq["answerable"])
total = len(cqs)
return cqs, passed, total
def check_example_individuals(g):
"""Check 8: Has example individuals demonstrating 4D patterns"""
# Check for named individuals (not class/property definitions)
individuals = set()
for s, _, o in g.triples((None, RDF.type, None)):
s_str = str(s)
o_str = str(o)
# Skip class and property definitions
if o_str in (str(RDFS.Class), str(OWL.ObjectProperty), str(OWL.DatatypeProperty), str(OWL.Ontology)):
continue
if "testdata" in s_str or "iso8601" in s_str:
individuals.add(s_str.split("#")[-1] if "#" in s_str else s_str.split("/")[-1])
elif str(s).startswith(str(BLDG)):
individuals.add(str(s).split("#")[-1])
checks = {
"has_individuals": len(individuals) > 0,
"has_3_or_more": len(individuals) >= 3,
"count": len(individuals)
}
# Check for 4D patterns in examples (BoundingState, isStateOf, inPeriod)
has_bounding_state = bool(list(g.subjects(RDF.type, IES.BoundingState))) or \
any(str(o).endswith("ConstructedState") or str(o).endswith("InstalledState")
for _, _, o in g.triples((None, RDF.type, None)))
has_is_state_of = bool(list(g.triples((None, IES.isStateOf, None))))
has_in_period = bool(list(g.triples((None, IES.inPeriod, None))))
has_is_start_of = bool(list(g.triples((None, IES.isStartOf, None))))
checks["has_bounding_state"] = has_bounding_state
checks["has_isStateOf"] = has_is_state_of
checks["has_inPeriod"] = has_in_period
checks["has_isStartOf"] = has_is_start_of
passed = sum(1 for k, v in checks.items() if k != "count" and v)
total = len(checks) - 1 # exclude count
return [checks], passed, total
def run_benchmark(generated_path):
print("=" * 70)
print("OPEN ONTOLOGIES BENCHMARK")
print("Claude-generated IES Building Extension vs IES4 Reference Patterns")
print("=" * 70)
# Check 1: Syntax
print("\n--- Check 1: Turtle Syntax Validity ---")
valid, triple_count, g = check_syntax(generated_path)
if not valid:
print(f" FAIL: {g}")
return
print(f" PASS: Valid Turtle, {triple_count} triples")
total_checks = 0
total_passed = 0
# Check 2: 4D Pattern
print("\n--- Check 2: 4D Pattern (Entity + EntityState pairs) ---")
results, passed, total = check_4d_pattern(g)
total_checks += total; total_passed += passed
for r in results:
status = "PASS" if r["has_state"] else "FAIL"
print(f" [{status}] {r['entity']} -> {r['state_name']}")
print(f" Score: {passed}/{total}")
# Check 3: ClassOf Pattern
print("\n--- Check 3: ClassOf Pattern (ClassOfEntity / ClassOfState) ---")
results, passed, total = check_classof_pattern(g)
total_checks += total; total_passed += passed
for r in results:
status = "PASS" if r["correct"] else "FAIL"
print(f" [{status}] {r['class']} -> {r['parents']}")
print(f" Score: {passed}/{total}")
# Check 4: Property Patterns
print("\n--- Check 4: Property Patterns (subPropertyOf, labels, domains) ---")
results, passed, total = check_property_patterns(g)
total_checks += total; total_passed += passed
for r in results:
all_ok = r["has_valid_super"] and r["has_label"] and r["has_comment"]
status = "PASS" if all_ok else "FAIL"
issues = []
if not r["has_valid_super"]: issues.append("no valid subPropertyOf")
if not r["has_label"]: issues.append("no label")
if not r["has_comment"]: issues.append("no comment")
issue_str = f" ({', '.join(issues)})" if issues else ""
print(f" [{status}] {r['property']} subPropOf:{r['super_props']}{issue_str}")
print(f" Score: {passed}/{total}")
# Check 5: Naming Conventions
print("\n--- Check 5: Naming Conventions (rdfs:label + rdfs:comment on all classes) ---")
results, passed, total = check_naming_conventions(g)
total_checks += total; total_passed += passed
failed = [r for r in results if not (r["has_label"] and r["has_comment"])]
if failed:
for r in failed:
print(f" [FAIL] {r['entity']}: label={r['has_label']}, comment={r['has_comment']}")
else:
print(f" All {total} classes have labels and comments")
print(f" Score: {passed}/{total}")
# Check 6: Ontology Metadata
print("\n--- Check 6: Ontology Metadata ---")
results, passed, total = check_ontology_metadata(g)
total_checks += total; total_passed += passed
if results:
for k, v in results[0].items():
status = "PASS" if v else "FAIL"
print(f" [{status}] {k}")
print(f" Score: {passed}/{total}")
# Check 7: Competency Questions
print("\n--- Check 7: Competency Questions (CQ1-CQ9) ---")
results, passed, total = check_competency_questions(g)
total_checks += total; total_passed += passed
for cq in results:
status = "PASS" if cq["answerable"] else "FAIL"
missing = f" (missing: {cq.get('missing', [])})" if not cq["answerable"] else ""
print(f" [{status}] {cq['id']}: {cq['question']}{missing}")
print(f" Score: {passed}/{total}")
# Check 8: Example Individuals
print("\n--- Check 8: Example Individuals (4D patterns in use) ---")
results, passed, total = check_example_individuals(g)
total_checks += total; total_passed += passed
if results:
for k, v in results[0].items():
if k == "count":
print(f" Individual count: {v}")
else:
status = "PASS" if v else "FAIL"
print(f" [{status}] {k}")
print(f" Score: {passed}/{total}")
# Final Score
print("\n" + "=" * 70)
pct = (total_passed / total_checks * 100) if total_checks > 0 else 0
print(f"OVERALL SCORE: {total_passed}/{total_checks} ({pct:.1f}%)")
print(f"TARGET: 99%")
print(f"RESULT: {'PASS' if pct >= 99 else 'NEEDS IMPROVEMENT' if pct >= 95 else 'FAIL'}")
print("=" * 70)
# Summary
print(f"\nGenerated ontology stats:")
print(f" Triples: {triple_count}")
classes = len(list(g.subjects(RDF.type, RDFS.Class)))
obj_props = len(list(g.subjects(RDF.type, OWL.ObjectProperty)))
data_props = len(list(g.subjects(RDF.type, OWL.DatatypeProperty)))
print(f" Classes: {classes}")
print(f" Object Properties: {obj_props}")
print(f" Data Properties: {data_props}")
print(f" Tool used: Claude (direct Turtle generation)")
print(f" Time: ~2 seconds")
print(f" API calls: 0")
print(f" External dependencies: 0")
if __name__ == "__main__":
path = sys.argv[1] if len(sys.argv) > 1 else "generated/ies-building-extension.ttl"
run_benchmark(path)