Skip to content

Plan, apply and roll back changes to a production ontology, with a blast radius report and a proof an auditor can re-check.

bench(oaei): repair the no-stable-matching ablation and record its output

FABIOTESS254fa1326d2e
diff --git a/benchmark/oaei/results/ablation_no_stable.json b/benchmark/oaei/results/ablation_no_stable.json
new file mode 100644
index 00000000..57657b86
--- /dev/null
+++ b/benchmark/oaei/results/ablation_no_stable.json
@@ -0,0 +1,120 @@
+{
+ "_meta": {
+ "reference_mappings": 1516,
+ "note": "Single-variable ablation: stable 1-to-1 matching removed from src/align.rs, everything else held constant. Source is restored and rebuilt at the end of this script."
+ },
+ "with_stable": [
+ {
+ "weights": "full",
+ "threshold": 0.7,
+ "candidates": 1913,
+ "tp": 1233,
+ "fp": 680,
+ "fn": 283,
+ "precision": 0.6445,
+ "recall": 0.8133,
+ "f1": 0.7192
+ },
+ {
+ "weights": "full",
+ "threshold": 0.75,
+ "candidates": 1475,
+ "tp": 1210,
+ "fp": 265,
+ "fn": 306,
+ "precision": 0.8203,
+ "recall": 0.7982,
+ "f1": 0.8091
+ },
+ {
+ "weights": "full",
+ "threshold": 0.8,
+ "candidates": 1154,
+ "tp": 1109,
+ "fp": 45,
+ "fn": 407,
+ "precision": 0.961,
+ "recall": 0.7315,
+ "f1": 0.8307
+ },
+ {
+ "weights": "full",
+ "threshold": 0.85,
+ "candidates": 1070,
+ "tp": 1041,
+ "fp": 29,
+ "fn": 475,
+ "precision": 0.9729,
+ "recall": 0.6867,
+ "f1": 0.8051
+ }
+ ],
+ "without_stable": [
+ {
+ "weights": "full_no_stable",
+ "threshold": 0.8,
+ "candidates": 1590,
+ "tp": 1131,
+ "fp": 459,
+ "fn": 385,
+ "precision": 0.7113,
+ "recall": 0.746,
+ "f1": 0.7283
+ },
+ {
+ "weights": "full_no_stable",
+ "threshold": 0.85,
+ "candidates": 1106,
+ "tp": 1063,
+ "fp": 43,
+ "fn": 453,
+ "precision": 0.9611,
+ "recall": 0.7012,
+ "f1": 0.8108
+ },
+ {
+ "weights": "label_no_stable",
+ "threshold": 0.8,
+ "candidates": 1590,
+ "tp": 1131,
+ "fp": 459,
+ "fn": 385,
+ "precision": 0.7113,
+ "recall": 0.746,
+ "f1": 0.7283
+ },
+ {
+ "weights": "label_no_stable",
+ "threshold": 0.85,
+ "candidates": 1106,
+ "tp": 1063,
+ "fp": 43,
+ "fn": 453,
+ "precision": 0.9611,
+ "recall": 0.7012,
+ "f1": 0.8108
+ },
+ {
+ "weights": "struct_no_stable",
+ "threshold": 0.8,
+ "candidates": 1590,
+ "tp": 1131,
+ "fp": 459,
+ "fn": 385,
+ "precision": 0.7113,
+ "recall": 0.746,
+ "f1": 0.7283
+ },
+ {
+ "weights": "struct_no_stable",
+ "threshold": 0.85,
+ "candidates": 1106,
+ "tp": 1063,
+ "fp": 43,
+ "fn": 453,
+ "precision": 0.9611,
+ "recall": 0.7012,
+ "f1": 0.8108
+ }
+ ]
+}
\ No newline at end of file
diff --git a/benchmark/oaei/run_ablation_no_stable.py b/benchmark/oaei/run_ablation_no_stable.py
index bbae0cae..5ea016f6 100644
--- a/benchmark/oaei/run_ablation_no_stable.py
+++ b/benchmark/oaei/run_ablation_no_stable.py
@@ -58,6 +58,8 @@ CONFIGS = {
'struct_no_stable': [0.00, 0.25, 0.25, 0.20, 0.20, 0.10],
}
+RESULTS = {'with_stable': [], 'without_stable': []}
+
# First test: run WITH stable matching (current code) at different thresholds
print('\n=== WITH stable matching (varying threshold) ===')
for threshold in ['0.70', '0.75', '0.80', '0.85']:
@@ -75,13 +77,16 @@ for threshold in ['0.70', '0.75', '0.80', '0.85']:
p = tp / (tp + fp) if (tp + fp) > 0 else 0
r = tp / (tp + fn) if (tp + fn) > 0 else 0
f1 = 2 * p * r / (p + r) if (p + r) > 0 else 0
- print(f' conf={threshold}: {len(candidates):>5} cands | P={p:.3f} R={r:.3f} F1={f1:.3f}')
+ print(f' conf={threshold}: {len(candidates):>5} cands | P={p:.3f} R={r:.3f} F1={f1:.3f}', flush=True)
+ RESULTS['with_stable'].append({'weights': 'full', 'threshold': float(threshold),
+ 'candidates': len(candidates), 'tp': tp, 'fp': fp, 'fn': fn,
+ 'precision': round(p,4), 'recall': round(r,4), 'f1': round(f1,4)})
# Now test WITHOUT stable matching
# We need to actually remove the stable matching block from the code
# Find the exact block to remove
stable_block_start = ' // Stable matching: for each source class, keep only the top-scoring target.'
-stable_block_end = ' // Auto-apply above threshold'
+stable_block_end = ' // Auto-apply above high_threshold.'
if stable_block_start in original_src:
idx_start = original_src.index(stable_block_start)
@@ -125,7 +130,10 @@ if stable_block_start in original_src:
p = tp / (tp + fp) if (tp + fp) > 0 else 0
r = tp / (tp + fn) if (tp + fn) > 0 else 0
f1 = 2 * p * r / (p + r) if (p + r) > 0 else 0
- print(f' {name} conf={threshold}: {len(candidates):>6} cands | P={p:.3f} R={r:.3f} F1={f1:.3f}')
+ print(f' {name} conf={threshold}: {len(candidates):>6} cands | P={p:.3f} R={r:.3f} F1={f1:.3f}', flush=True)
+ RESULTS['without_stable'].append({'weights': name, 'threshold': float(threshold),
+ 'candidates': len(candidates), 'tp': tp, 'fp': fp, 'fn': fn,
+ 'precision': round(p,4), 'recall': round(r,4), 'f1': round(f1,4)})
else:
print('ERROR: Could not find stable matching block')
@@ -140,3 +148,11 @@ subprocess.run(
env={**os.environ, 'CARGO_TARGET_DIR': 'target'}
)
print('\nRestored original align.rs and rebuilt')
+
+with open('benchmark/oaei/results/ablation_no_stable.json', 'w') as f:
+ json.dump({'_meta': {'reference_mappings': len(reference),
+ 'note': 'Single-variable ablation: stable 1-to-1 matching removed from '
+ 'src/align.rs, everything else held constant. Source is restored '
+ 'and rebuilt at the end of this script.'},
+ **RESULTS}, f, indent=2)
+print('\nSaved to benchmark/oaei/results/ablation_no_stable.json', flush=True)