Plan, apply and roll back changes to a production ontology, with a blast radius report and a proof an auditor can re-check.
Rust ai-nativedescription-logicsformal-verificationinfrastructure-as-codeisabelle-languageknowledge-graphleanlean4mcpmcp-serverontologyowlowl2reasoningrustshaclsparqltheorem-proving
bench(oaei): repair the no-stable-matching ablation and record its output
FABIOTESS254fa1326d2e
diff --git a/benchmark/oaei/results/ablation_no_stable.json b/benchmark/oaei/results/ablation_no_stable.jsonnew file mode 100644index 00000000..57657b86--- /dev/null+++ b/benchmark/oaei/results/ablation_no_stable.json@@ -0,0 +1,120 @@+{+ "_meta": {+ "reference_mappings": 1516,+ "note": "Single-variable ablation: stable 1-to-1 matching removed from src/align.rs, everything else held constant. Source is restored and rebuilt at the end of this script."+ },+ "with_stable": [+ {+ "weights": "full",+ "threshold": 0.7,+ "candidates": 1913,+ "tp": 1233,+ "fp": 680,+ "fn": 283,+ "precision": 0.6445,+ "recall": 0.8133,+ "f1": 0.7192+ },+ {+ "weights": "full",+ "threshold": 0.75,+ "candidates": 1475,+ "tp": 1210,+ "fp": 265,+ "fn": 306,+ "precision": 0.8203,+ "recall": 0.7982,+ "f1": 0.8091+ },+ {+ "weights": "full",+ "threshold": 0.8,+ "candidates": 1154,+ "tp": 1109,+ "fp": 45,+ "fn": 407,+ "precision": 0.961,+ "recall": 0.7315,+ "f1": 0.8307+ },+ {+ "weights": "full",+ "threshold": 0.85,+ "candidates": 1070,+ "tp": 1041,+ "fp": 29,+ "fn": 475,+ "precision": 0.9729,+ "recall": 0.6867,+ "f1": 0.8051+ }+ ],+ "without_stable": [+ {+ "weights": "full_no_stable",+ "threshold": 0.8,+ "candidates": 1590,+ "tp": 1131,+ "fp": 459,+ "fn": 385,+ "precision": 0.7113,+ "recall": 0.746,+ "f1": 0.7283+ },+ {+ "weights": "full_no_stable",+ "threshold": 0.85,+ "candidates": 1106,+ "tp": 1063,+ "fp": 43,+ "fn": 453,+ "precision": 0.9611,+ "recall": 0.7012,+ "f1": 0.8108+ },+ {+ "weights": "label_no_stable",+ "threshold": 0.8,+ "candidates": 1590,+ "tp": 1131,+ "fp": 459,+ "fn": 385,+ "precision": 0.7113,+ "recall": 0.746,+ "f1": 0.7283+ },+ {+ "weights": "label_no_stable",+ "threshold": 0.85,+ "candidates": 1106,+ "tp": 1063,+ "fp": 43,+ "fn": 453,+ "precision": 0.9611,+ "recall": 0.7012,+ "f1": 0.8108+ },+ {+ "weights": "struct_no_stable",+ "threshold": 0.8,+ "candidates": 1590,+ "tp": 1131,+ "fp": 459,+ "fn": 385,+ "precision": 0.7113,+ "recall": 0.746,+ "f1": 0.7283+ },+ {+ "weights": "struct_no_stable",+ "threshold": 0.85,+ "candidates": 1106,+ "tp": 1063,+ "fp": 43,+ "fn": 453,+ "precision": 0.9611,+ "recall": 0.7012,+ "f1": 0.8108+ }+ ]+}\ No newline at end of filediff --git a/benchmark/oaei/run_ablation_no_stable.py b/benchmark/oaei/run_ablation_no_stable.pyindex bbae0cae..5ea016f6 100644--- a/benchmark/oaei/run_ablation_no_stable.py+++ b/benchmark/oaei/run_ablation_no_stable.py@@ -58,6 +58,8 @@ CONFIGS = {'struct_no_stable': [0.00, 0.25, 0.25, 0.20, 0.20, 0.10],}+RESULTS = {'with_stable': [], 'without_stable': []}+# First test: run WITH stable matching (current code) at different thresholdsprint('\n=== WITH stable matching (varying threshold) ===')for threshold in ['0.70', '0.75', '0.80', '0.85']:@@ -75,13 +77,16 @@ for threshold in ['0.70', '0.75', '0.80', '0.85']:p = tp / (tp + fp) if (tp + fp) > 0 else 0r = tp / (tp + fn) if (tp + fn) > 0 else 0f1 = 2 * p * r / (p + r) if (p + r) > 0 else 0- print(f' conf={threshold}: {len(candidates):>5} cands | P={p:.3f} R={r:.3f} F1={f1:.3f}')+ print(f' conf={threshold}: {len(candidates):>5} cands | P={p:.3f} R={r:.3f} F1={f1:.3f}', flush=True)+ RESULTS['with_stable'].append({'weights': 'full', 'threshold': float(threshold),+ 'candidates': len(candidates), 'tp': tp, 'fp': fp, 'fn': fn,+ 'precision': round(p,4), 'recall': round(r,4), 'f1': round(f1,4)})# Now test WITHOUT stable matching# We need to actually remove the stable matching block from the code# Find the exact block to removestable_block_start = ' // Stable matching: for each source class, keep only the top-scoring target.'-stable_block_end = ' // Auto-apply above threshold'+stable_block_end = ' // Auto-apply above high_threshold.'if stable_block_start in original_src:idx_start = original_src.index(stable_block_start)@@ -125,7 +130,10 @@ if stable_block_start in original_src:p = tp / (tp + fp) if (tp + fp) > 0 else 0r = tp / (tp + fn) if (tp + fn) > 0 else 0f1 = 2 * p * r / (p + r) if (p + r) > 0 else 0- print(f' {name} conf={threshold}: {len(candidates):>6} cands | P={p:.3f} R={r:.3f} F1={f1:.3f}')+ print(f' {name} conf={threshold}: {len(candidates):>6} cands | P={p:.3f} R={r:.3f} F1={f1:.3f}', flush=True)+ RESULTS['without_stable'].append({'weights': name, 'threshold': float(threshold),+ 'candidates': len(candidates), 'tp': tp, 'fp': fp, 'fn': fn,+ 'precision': round(p,4), 'recall': round(r,4), 'f1': round(f1,4)})else:print('ERROR: Could not find stable matching block')@@ -140,3 +148,11 @@ subprocess.run(env={**os.environ, 'CARGO_TARGET_DIR': 'target'})print('\nRestored original align.rs and rebuilt')++with open('benchmark/oaei/results/ablation_no_stable.json', 'w') as f:+ json.dump({'_meta': {'reference_mappings': len(reference),+ 'note': 'Single-variable ablation: stable 1-to-1 matching removed from '+ 'src/align.rs, everything else held constant. Source is restored '+ 'and rebuilt at the end of this script.'},+ **RESULTS}, f, indent=2)+print('\nSaved to benchmark/oaei/results/ablation_no_stable.json', flush=True)