Skip to content

28 eval-informed mental models and critical-thinking skills for Claude Code, GitHub Copilot, Codex, Cursor, and other Agent Skills-compatible tools

feat(m5): powered primary batch — 7 gated skills evaluated, all NO-LIFT

Travis Boudreauxb11f5538afcc
diff --git a/analysis/ELEVATE-OR-KILL-SCORECARD.json b/analysis/ELEVATE-OR-KILL-SCORECARD.json
index ca87b40..0b5fef2 100644
--- a/analysis/ELEVATE-OR-KILL-SCORECARD.json
+++ b/analysis/ELEVATE-OR-KILL-SCORECARD.json
@@ -1,6 +1,6 @@
{
"meta": {
- "generated": "2026-06-06T00:00:00Z",
+ "generated": "2026-06-07T05:00:00Z",
"source_documents": [
"analysis/EVAL-COVERAGE.md",
"analysis/ELEVATE-OR-KILL.md (final two sections only)",
@@ -26,11 +26,42 @@
"measured": 17,
"unmeasured": 22
},
- "notes": "Stale claims from run1/scorecard.md (n=3 wins) and mid-document ELEVATE-OR-KILL.md narrative are flagged superseded, not deleted. scientific-method is the sole robust objective ELEVATE."
+ "notes": "Stale claims from run1/scorecard.md (n=3 wins) and mid-document ELEVATE-OR-KILL.md narrative are flagged superseded, not deleted. scientific-method is the sole robust objective ELEVATE.",
+ "m5_powered_run": "m5-primary",
+ "m5_powered_skills": [
+ "red-team",
+ "occams-razor",
+ "five-whys-plus",
+ "systems",
+ "archetypes",
+ "map-territory",
+ "kepner-tregoe"
+ ],
+ "m5_powered_count": 7,
+ "m5_powered_solver": "claude-sonnet-4-6",
+ "m5_powered_conc": 4
},
"skills": {
"archetypes": {
- "objective_evidence": [],
+ "objective_evidence": [
+ {
+ "metric": "binary_decision_accuracy",
+ "dataset": "Systems/product/strategy pairwise decisive split (117 items: 42 target bottleneck/margin-of-safety, 29 near-miss, 46 distractor)",
+ "n": 117,
+ "placebo": 0.72,
+ "treatment": 0.73,
+ "delta_pp": 0.9,
+ "p": 1,
+ "significant": false,
+ "verdict": "NO-LIFT",
+ "provenance": "OBJ-powered-null",
+ "temporal": "post-edit",
+ "significance": "null",
+ "replicated": false,
+ "source": "evals/results/m5-primary/binary-archetypes.json",
+ "note": "M5 powered run on systems-product-strategy-pairwise decisive split (aggregate baseline 57.4% — in-band). In-band quarantine candidate given a fair calibrated chance. +0.9pp over placebo not significant. Target-only: 88% vs 88% (Δ0pp, p=0.479). Distractor FPR: 36% with skill vs 37% placebo (trivially better). No lift detected."
+ }
+ ],
"judge_evidence": [
{
"metric": "behavioral_win_rate",
@@ -47,9 +78,10 @@
"note": "Mid-document ELEVATE-OR-KILL.md pooled 63% ELEVATE claim superseded; judge-only n=9, no objective formulation"
}
],
- "verdict": "unmeasured",
- "eval_family": "Conceptual reframes",
- "value_surface": "paired reasoning quality"
+ "verdict": "no-lift",
+ "eval_family": "Systems/product/strategy pairwise",
+ "value_surface": "paired reasoning quality",
+ "powered_run": "OBJ-powered-null"
},
"bayesian": {
"objective_evidence": [
@@ -444,6 +476,23 @@
"replicated": false,
"source": "evals/results/run1/swe-five-whys-plus.json",
"note": "Earlier n=45 run; directional only"
+ },
+ {
+ "metric": "fault_localization_accuracy",
+ "dataset": "SWE-bench fault localization (decisive split, 150 items)",
+ "n": 150,
+ "placebo": 0.84,
+ "treatment": 0.84,
+ "delta_pp": 0,
+ "p": 0.752,
+ "significant": false,
+ "verdict": "NO-LIFT",
+ "provenance": "OBJ-powered-null",
+ "temporal": "post-edit",
+ "significance": "null",
+ "replicated": false,
+ "source": "evals/results/m5-primary/swe-five-whys-plus.json",
+ "note": "M5 powered run with M4-reworked skill. No lift over placebo on SWE-bench fault localization. Confirms previous replication failure."
}
],
"judge_evidence": [
@@ -464,7 +513,8 @@
],
"verdict": "no-lift",
"eval_family": "Debugging/fault-localization",
- "value_surface": "localization"
+ "value_surface": "localization",
+ "powered_run": "OBJ-powered-null"
},
"inversion": {
"objective_evidence": [],
@@ -511,7 +561,27 @@
"value_surface": "paired reasoning quality"
},
"kepner-tregoe": {
- "objective_evidence": [],
+ "objective_evidence": [
+ {
+ "metric": "fault_localization_accuracy",
+ "dataset": "SWE-bench fault localization (decisive split, 150 items)",
+ "n": 150,
+ "placebo": 0.81,
+ "treatment": 0.84,
+ "delta_pp": 2.7,
+ "p": 0.289,
+ "significant": false,
+ "verdict": "NO-LIFT",
+ "provenance": "OBJ-powered-null",
+ "temporal": "post-edit",
+ "significance": "null",
+ "replicated": false,
+ "source": "evals/results/m5-primary/swe-kepner-tregoe.json",
+ "note": "EXPLORATORY: surface-mismatch (powered on fault-localization, not native surface). M5 powered run with M4-reworked skill. kepner-tregoe's true value surface is paired-reasoning quality, not fault localization. +2.7pp over placebo not significant; this is NOT recorded as an honest kill — the eval surface does not match the skill's native value surface. Contract eval_family is Debugging/fault-localization which MISMATCHES the skill's paired reasoning quality value surface.",
+ "exploratory": true,
+ "exploratory_reason": "surface-mismatch: powered on fault-localization, not native surface (paired-reasoning quality)"
+ }
+ ],
"judge_evidence": [
{
"metric": "behavioral_win_rate",
@@ -528,9 +598,12 @@
"note": "Run1 scorecard n=3 win claim superseded; audit recommends split (PA for debugging, retire DA/PPA)"
}
],
- "verdict": "unmeasured",
+ "verdict": "no-lift",
"eval_family": "Debugging/fault-localization",
- "value_surface": "paired reasoning quality"
+ "value_surface": "paired reasoning quality",
+ "powered_run": "OBJ-powered-null",
+ "exploratory": true,
+ "exploratory_reason": "surface-mismatch: powered on fault-localization, not native surface (paired-reasoning quality)"
},
"leverage-points": {
"objective_evidence": [],
@@ -593,6 +666,25 @@
"significance": "null",
"replicated": false,
"source": "evals/results/rerun/map-territory-verify.json"
+ },
+ {
+ "metric": "fault_localization_accuracy",
+ "dataset": "SWE-bench fault localization (decisive split, 150 items)",
+ "n": 150,
+ "placebo": 0.84,
+ "treatment": 0.87,
+ "delta_pp": 2.7,
+ "p": 0.221,
+ "significant": false,
+ "verdict": "NO-LIFT",
+ "provenance": "OBJ-powered-null",
+ "temporal": "post-edit",
+ "significance": "null",
+ "replicated": false,
+ "source": "evals/results/m5-primary/swe-map-territory.json",
+ "note": "EXPLORATORY: surface-mismatch (powered on fault-localization, not native surface). M5 powered run with M4-reworked skill. map-territory's true value surface is routing/discoverability, not fault localization. +2.7pp over placebo not significant; this is NOT recorded as an honest kill — the eval surface does not match the skill's native value surface. Contract eval_family is Debugging/fault-localization which MISMATCHES the skill's routing/discoverability value surface.",
+ "exploratory": true,
+ "exploratory_reason": "surface-mismatch: powered on fault-localization, not native surface (routing/discoverability)"
}
],
"judge_evidence": [
@@ -611,9 +703,12 @@
"note": "Mid-document ELEVATE-OR-KILL.md pooled 63% ELEVATE claim superseded"
}
],
- "verdict": "ceiling",
- "eval_family": "Routing/abstention",
- "value_surface": "routing/discoverability"
+ "verdict": "no-lift",
+ "eval_family": "Debugging/fault-localization",
+ "value_surface": "routing/discoverability",
+ "powered_run": "OBJ-powered-null",
+ "exploratory": true,
+ "exploratory_reason": "surface-mismatch: powered on fault-localization, not native surface (routing/discoverability)"
},
"margin-of-safety": {
"objective_evidence": [
@@ -754,6 +849,23 @@
"replicated": false,
"source": "evals/results/run1/swe-occams-razor-improved.json",
"note": "Trigger-scoped rework did not move needle"
+ },
+ {
+ "metric": "fault_localization_accuracy",
+ "dataset": "SWE-bench fault localization (decisive split, 150 items)",
+ "n": 150,
+ "placebo": 0.8,
+ "treatment": 0.85,
+ "delta_pp": 4.7,
+ "p": 0.096,
+ "significant": false,
+ "verdict": "NO-LIFT",
+ "provenance": "OBJ-powered-directional",
+ "temporal": "post-edit",
+ "significance": "directional",
+ "replicated": false,
+ "source": "evals/results/m5-primary/swe-occams-razor.json",
+ "note": "M5 powered run with M4-reworked skill. +4.7pp directional improvement over placebo but p=0.096 (not significant) and below 5pp threshold. Prior run1 gave +2.0pp p=0.505."
}
],
"judge_evidence": [
@@ -772,9 +884,10 @@
"note": "Run1 scorecard n=3 win claim superseded; audit recommends TRIGGER-ONLY"
}
],
- "verdict": "directional, not confirmed",
+ "verdict": "directional-not-replicated",
"eval_family": "Debugging/fault-localization",
- "value_surface": "paired reasoning quality"
+ "value_surface": "paired reasoning quality",
+ "powered_run": "OBJ-powered-directional"
},
"ooda": {
"objective_evidence": [],
@@ -899,6 +1012,23 @@
"replicated": false,
"source": "evals/results/confirm/redteam-diversevul-balanced-n200.json",
"note": "Powered confirmation at n=200; +11.3pp at n=80 collapsed to +5.0pp p=0.10"
+ },
+ {
+ "metric": "vulnerability_detection_accuracy",
+ "dataset": "Security/adversarial decisive split (70 items: diverse CWE, near-miss safe, distractors)",
+ "n": 70,
+ "placebo": 0.43,
+ "treatment": 0.44,
+ "delta_pp": 1.4,
+ "p": 1,
+ "significant": false,
+ "verdict": "NO-LIFT",
+ "provenance": "OBJ-powered-null",
+ "temporal": "post-edit",
+ "significance": "null",
+ "replicated": false,
+ "source": "evals/results/m5-primary/redteam-security-decisive.json",
+ "note": "M5 powered run with M4-reworked skill on harder diverse-CWE decisive split (baseline 43% — in-band headroom). +1.4pp over placebo not significant. Prior +5.0pp at n=200 p=0.10 on easier diversevul-balanced pool did not replicate on harder items."
}
],
"judge_evidence": [
@@ -917,9 +1047,10 @@
"note": "Mid-document ELEVATE-OR-KILL.md pooled 63% ELEVATE claim superseded"
}
],
- "verdict": "directional-not-replicated",
+ "verdict": "no-lift",
"eval_family": "Security/adversarial",
- "value_surface": "false-positive reduction"
+ "value_surface": "false-positive reduction",
+ "powered_run": "OBJ-powered-null"
},
"regret-minimization": {
"objective_evidence": [],
@@ -1191,6 +1322,23 @@
"replicated": false,
"source": "evals/results/run1/swe-systems.json",
"note": "n=45 native domain run; directional only"
+ },
+ {
+ "metric": "fault_localization_accuracy",
+ "dataset": "SWE-bench fault localization (decisive split, 150 items)",
+ "n": 150,
+ "placebo": 0.83,
+ "treatment": 0.84,
+ "delta_pp": 1.3,
+ "p": 0.724,
+ "significant": false,
+ "verdict": "NO-LIFT",
+ "provenance": "OBJ-powered-null",
+ "temporal": "post-edit",
+ "significance": "null",
+ "replicated": false,
+ "source": "evals/results/m5-primary/swe-systems.json",
+ "note": "M5 powered run with M4-reworked skill. +1.3pp over placebo not significant. Confirms previous replication failure."
}
],
"judge_evidence": [
@@ -1211,7 +1359,8 @@
],
"verdict": "no-lift",
"eval_family": "Debugging/fault-localization",
- "value_surface": "localization"
+ "value_surface": "localization",
+ "powered_run": "OBJ-powered-null"
},
"theory-of-constraints": {
"objective_evidence": [
diff --git a/analysis/ELEVATE-OR-KILL-SCORECARD.md b/analysis/ELEVATE-OR-KILL-SCORECARD.md
index 154cd79..f9c081f 100644
--- a/analysis/ELEVATE-OR-KILL-SCORECARD.md
+++ b/analysis/ELEVATE-OR-KILL-SCORECARD.md
@@ -44,8 +44,21 @@
| circle-of-competence | SelfAware (abstention) | 70 | 70% → 70% | 0 | 0.77 | no calibration benefit | `routing/abstention-null` |
| bayesian | authored base-rate | 40 | 98% → 100% | +2 | 1.0 | ceiling | `OBJ-small-ceiling` |
| debiasing | authored bias | 40 | 100% → 98% | −2 | 1.0 | ceiling | `OBJ-small-ceiling` |
+| **M5: red-team (reworked)** | Security decisive (CWEs) | 70 | 43% → 44% | +1.4 | 1.0 | NO-LIFT (post-edit) | `OBJ-powered-null` |
+| **M5: five-whys-plus (reworked)** | SWE-bench | 150 | 84% → 84% | 0 | 0.752 | NO-LIFT (post-edit) | `OBJ-powered-null` |
+| **M5: systems (reworked)** | SWE-bench | 150 | 83% → 84% | +1.3 | 0.724 | NO-LIFT (post-edit) | `OBJ-powered-null` |
+| **M5: occams-razor (reworked)** | SWE-bench | 150 | 80% → 85% | +4.7 | 0.096 | directional, NO-LIFT | `OBJ-powered-directional` |
+| **M5: map-territory (reworked)** ⚠️ | SWE-bench | 150 | 84% → 87% | +2.7 | 0.221 | NO-LIFT (exploratory) | `OBJ-powered-null` |
+| **M5: kepner-tregoe (reworked)** ⚠️ | SWE-bench | 150 | 81% → 84% | +2.7 | 0.289 | NO-LIFT (exploratory) | `OBJ-powered-null` |
+| **M5: archetypes (quarantine)** | Systems decisive (binary) | 117 | 72% → 73% | +0.9 | 1.0 | NO-LIFT (post-edit) | `OBJ-powered-null` |
-**Total objectively measured: 17 / 39 skills**
+⚠️ **map-territory** and **kepner-tregoe**: Tagged `exploratory: surface-mismatch (powered on fault-localization, not native surface)`. Their eval_family is Debugging/fault-localization (SWE-bench), which MISMATCHES their true value surfaces (routing/discoverability and paired-reasoning quality respectively). A no-lift here is NOT recorded as an honest kill.
+
+⚠️ **archetypes**: In-band quarantine candidate (aggregate baseline 57.4%) given a fair calibrated chance on the systems-product-strategy-pairwise decisive split. No lift detected.
+
+**M5 powered run:** solver `claude-sonnet-4-6`, CONC=4, isolation ON, EVAL_RUN=m5-primary. All 7 results are post-edit (run after M4 skill reworks). No skill passed the primary gate (≥5pp + p<0.05), so no replications were triggered.
+
+**Total objectively measured: 17 (+7 M5 powered = 24 with post-edit evidence) / 39 skills**
**Unmeasured (judge-only / thin / leakage-blocked / meta): 22 skills**
---
diff --git a/evals/results/m5-primary/binary-archetypes.json b/evals/results/m5-primary/binary-archetypes.json
new file mode 100644
index 0000000..d8a70a1
--- /dev/null
+++ b/evals/results/m5-primary/binary-archetypes.json
@@ -0,0 +1,30 @@
+{
+ "mode": "binary-decision",
+ "skill": "archetypes",
+ "solver": "claude-sonnet-4-6",
+ "n": 117,
+ "n_target": 42,
+ "n_distractor": 75,
+ "acc_with_skill": 0.726,
+ "acc_with_skill_ci": [
+ 0.639,
+ 0.799
+ ],
+ "acc_placebo": 0.718,
+ "acc_placebo_ci": [
+ 0.63,
+ 0.792
+ ],
+ "delta_pp": 0.9,
+ "mcnemar_p": 1,
+ "significant": false,
+ "discordant": 15,
+ "target_acc_with": 0.881,
+ "target_acc_placebo": 0.881,
+ "target_delta_pp": 0,
+ "target_mcnemar_p": 0.479,
+ "distractor_acc_with": 0.64,
+ "distractor_acc_placebo": 0.627,
+ "distractor_fpr_with": 0.36,
+ "distractor_fpr_placebo": 0.373
+}
\ No newline at end of file
diff --git a/evals/results/m5-primary/redteam-security-decisive.json b/evals/results/m5-primary/redteam-security-decisive.json
new file mode 100644
index 0000000..bd3d8c0
--- /dev/null
+++ b/evals/results/m5-primary/redteam-security-decisive.json
@@ -0,0 +1,30 @@
+{
+ "mode": "correctness",
+ "solver": "claude-sonnet-4-6",
+ "n": 70,
+ "acc_with_skill": 0.443,
+ "acc_with_skill_ci": [
+ 0.332,
+ 0.559
+ ],
+ "acc_placebo": 0.429,
+ "acc_placebo_ci": [
+ 0.319,
+ 0.545
+ ],
+ "delta_pp": 1.4,
+ "mcnemar_p": 1,
+ "significant": false,
+ "discordant": 11,
+ "by_skill": [
+ {
+ "skill": "red-team",
+ "n": 70,
+ "acc_with": 0.443,
+ "acc_placebo": 0.429,
+ "delta_pp": 1.4,
+ "discordant": 11,
+ "mcnemar_p": 1
+ }
+ ]
+}
\ No newline at end of file
diff --git a/evals/results/m5-primary/swe-five-whys-plus.json b/evals/results/m5-primary/swe-five-whys-plus.json
new file mode 100644
index 0000000..6013654
--- /dev/null
+++ b/evals/results/m5-primary/swe-five-whys-plus.json
@@ -0,0 +1,20 @@
+{
+ "mode": "swe-localize",
+ "skill": "five-whys-plus",
+ "solver": "claude-sonnet-4-6",
+ "n": 150,
+ "acc_with_skill": 0.84,
+ "acc_with_skill_ci": [
+ 0.773,
+ 0.89
+ ],
+ "acc_placebo": 0.84,
+ "acc_placebo_ci": [
+ 0.773,
+ 0.89
+ ],
+ "delta_pp": 0,
+ "mcnemar_p": 0.752,
+ "significant": false,
+ "discordant": 10
+}
\ No newline at end of file
diff --git a/evals/results/m5-primary/swe-kepner-tregoe.json b/evals/results/m5-primary/swe-kepner-tregoe.json
new file mode 100644
index 0000000..f75dd60
--- /dev/null
+++ b/evals/results/m5-primary/swe-kepner-tregoe.json
@@ -0,0 +1,20 @@
+{
+ "mode": "swe-localize",
+ "skill": "kepner-tregoe",
+ "solver": "claude-sonnet-4-6",
+ "n": 150,
+ "acc_with_skill": 0.84,
+ "acc_with_skill_ci": [
+ 0.773,
+ 0.89
+ ],
+ "acc_placebo": 0.813,
+ "acc_placebo_ci": [
+ 0.743,
+ 0.868
+ ],
+ "delta_pp": 2.7,
+ "mcnemar_p": 0.289,
+ "significant": false,
+ "discordant": 8
+}
\ No newline at end of file
diff --git a/evals/results/m5-primary/swe-map-territory.json b/evals/results/m5-primary/swe-map-territory.json
new file mode 100644
index 0000000..844d600
--- /dev/null
+++ b/evals/results/m5-primary/swe-map-territory.json
@@ -0,0 +1,20 @@
+{
+ "mode": "swe-localize",
+ "skill": "map-territory",
+ "solver": "claude-sonnet-4-6",
+ "n": 150,
+ "acc_with_skill": 0.867,
+ "acc_with_skill_ci": [
+ 0.803,
+ 0.912
+ ],
+ "acc_placebo": 0.84,
+ "acc_placebo_ci": [
+ 0.773,
+ 0.89
+ ],
+ "delta_pp": 2.7,
+ "mcnemar_p": 0.221,
+ "significant": false,
+ "discordant": 6
+}
\ No newline at end of file
diff --git a/evals/results/m5-primary/swe-occams-razor.json b/evals/results/m5-primary/swe-occams-razor.json
new file mode 100644
index 0000000..609549a
--- /dev/null
+++ b/evals/results/m5-primary/swe-occams-razor.json
@@ -0,0 +1,20 @@
+{
+ "mode": "swe-localize",
+ "skill": "occams-razor",
+ "solver": "claude-sonnet-4-6",
+ "n": 150,
+ "acc_with_skill": 0.847,
+ "acc_with_skill_ci": [
+ 0.78,
+ 0.896
+ ],
+ "acc_placebo": 0.8,
+ "acc_placebo_ci": [
+ 0.729,
+ 0.856
+ ],
+ "delta_pp": 4.7,
+ "mcnemar_p": 0.096,
+ "significant": false,
+ "discordant": 13
+}
\ No newline at end of file
diff --git a/evals/results/m5-primary/swe-systems.json b/evals/results/m5-primary/swe-systems.json
new file mode 100644
index 0000000..71a5de7
--- /dev/null
+++ b/evals/results/m5-primary/swe-systems.json
@@ -0,0 +1,20 @@
+{
+ "mode": "swe-localize",
+ "skill": "systems",
+ "solver": "claude-sonnet-4-6",
+ "n": 150,
+ "acc_with_skill": 0.84,
+ "acc_with_skill_ci": [
+ 0.773,
+ 0.89
+ ],
+ "acc_placebo": 0.827,
+ "acc_placebo_ci": [
+ 0.758,
+ 0.879
+ ],
+ "delta_pp": 1.3,
+ "mcnemar_p": 0.724,
+ "significant": false,
+ "discordant": 8
+}
\ No newline at end of file
diff --git a/evals/run-binary-decision.js b/evals/run-binary-decision.js
new file mode 100644
index 0000000..8b4dc25
--- /dev/null
+++ b/evals/run-binary-decision.js
@@ -0,0 +1,135 @@
+#!/usr/bin/env node
+'use strict';
+
+/**
+ * Binary-decision powered eval for the systems-product-strategy-pairwise family.
+ * Reads decisive split items (mode=binary-decision), runs paired skill+placebo,
+ * scores against the ground-truth label (true=YES, false=NO).
+ *
+ * Usage: EVAL_RUN=m5-primary FORCE_SKILL=archetypes CONC=4 node evals/run-binary-decision.js
+ */
+
+const fs = require('fs');
+const path = require('path');
+const { droidExecAsync } = require('./lib/droid');
+const { runDir, writeJson, mapPool } = require('./lib/io');
+const { neutralFiller, wordCount } = require('./lib/conditions');
+const { mcnemar, wilson } = require('./lib/stats');
+
+const SOLVER = process.env.SOLVER_MODEL || 'claude-sonnet-4-6';
+const SOLVER_EFFORT = process.env.SOLVER_EFFORT || 'medium';
+const CONC = parseInt(process.env.CONC || '4', 10);
+const FORCE_SKILL = process.env.FORCE_SKILL;
+const FILE = path.join(__dirname, 'datasets', 'calibrated', 'systems-product-strategy-pairwise-decisive.jsonl');
+const SKILLS_DIR = path.join(__dirname, '..', 'skills');
+
+function loadItems() {
+ let L = fs.readFileSync(FILE, 'utf8').trim().split('\n').filter(Boolean).map(JSON.parse);
+ if (process.env.LIMIT) L = L.slice(0, parseInt(process.env.LIMIT, 10));
+ return L;
+}
+
+function skillContent(s) {
+ return fs.readFileSync(path.join(SKILLS_DIR, 'thinking-' + s.replace(/^thinking-/, ''), 'SKILL.md'), 'utf8');
+}
+
+function extractYesNo(text) {
+ const m = text.match(/ANSWER:\s*(yes|no)/i) ||
+ text.match(/\b(YES|NO)\b[.!]*\s*$/im) ||
+ text.match(/\b(yes|no)\b[.!]*\s*$/im);
+ return m ? m[1].toLowerCase() : null;
+}
+
+function isCorrect(item, answer) {
+ if (answer == null) return false;
+ const expected = item.label === true ? 'yes' : 'no';
+ return answer === expected;
+}
+
+async function runItem(item, skillMd) {
+ const answerInstr = `\n\nEnd your response with exactly: ANSWER: <Yes or No>`;
+ const skillP = `Use the following thinking-skill guide to reason about the problem. Apply it substantively.\n\n=== THINKING SKILL ===\n${skillMd}\n=== END SKILL ===\n\n${item.decision_instruction || ''}\n\n${item.prompt}${answerInstr}`;
+ const placeboP = `Some general notes:\n\n${neutralFiller(wordCount(skillMd))}\n\n${item.decision_instruction || ''}\n\n${item.prompt}${answerInstr}`;
+
+ const [s, p] = await Promise.all([
+ droidExecAsync({ model: SOLVER, effort: SOLVER_EFFORT, prompt: skillP }),
+ droidExecAsync({ model: SOLVER, effort: SOLVER_EFFORT, prompt: placeboP }),
+ ]);
+ if (!s.ok || !p.ok) return null;
+ return {
+ id: item.id,
+ target: item.target === true,
+ skill_correct: isCorrect(item, extractYesNo(s.text)),
+ placebo_correct: isCorrect(item, extractYesNo(p.text)),
+ };
+}
+
+async function main() {
+ if (!FORCE_SKILL) { console.error('set FORCE_SKILL'); process.exit(1); }
+ const skillMd = skillContent(FORCE_SKILL);
+ const items = loadItems();
+ console.log(`Binary-decision eval: ${items.length} items, skill=${FORCE_SKILL}, solver=${SOLVER}(${SOLVER_EFFORT}), isolation ${process.env.EVAL_NO_ISOLATE === '1' ? 'OFF' : 'ON'}`);
+
+ let done = 0;
+ const results = (await mapPool(items, CONC, async it => { const x = await runItem(it, skillMd); done++; if (done % 10 === 0) console.log(` ${done}/${items.length}`); return x; })).filter(Boolean);
+
+ const targets = results.filter(r => r.target);
+ const distractors = results.filter(r => !r.target);
+
+ // Overall stats
+ const SC = results.filter(r => r.skill_correct).length;
+ const PC = results.filter(r => r.placebo_correct).length;
+ const N = results.length;
+ const b = results.filter(r => r.skill_correct && !r.placebo_correct).length;
+ const cC = results.filter(r => !r.skill_correct && r.placebo_correct).length;
+
+ // Target-only stats
+ const tSC = targets.filter(r => r.skill_correct).length;
+ const tPC = targets.filter(r => r.placebo_correct).length;
+ const tN = targets.length;
+ const tb = targets.filter(r => r.skill_correct && !r.placebo_correct).length;
+ const tc = targets.filter(r => !r.skill_correct && r.placebo_correct).length;
+
+ // Distractor stats (FPR)
+ const dSC = distractors.filter(r => r.skill_correct).length;
+ const dPC = distractors.filter(r => r.placebo_correct).length;
+ const dN = distractors.length;
+
+ const out = {
+ mode: 'binary-decision',
+ skill: FORCE_SKILL,
+ solver: SOLVER,
+ n: N,
+ n_target: tN,
+ n_distractor: dN,
+ // Overall
+ acc_with_skill: +(SC / N).toFixed(3),
+ acc_with_skill_ci: wilson(SC, N).map(x => +x.toFixed(3)),
+ acc_placebo: +(PC / N).toFixed(3),
+ acc_placebo_ci: wilson(PC, N).map(x => +x.toFixed(3)),
+ delta_pp: +(((SC - PC) / N) * 100).toFixed(1),
+ mcnemar_p: +mcnemar(b, cC).toFixed(3),
+ significant: mcnemar(b, cC) < 0.05,
+ discordant: b + cC,
+ // Target only
+ target_acc_with: +(tSC / tN).toFixed(3),
+ target_acc_placebo: +(tPC / tN).toFixed(3),
+ target_delta_pp: +(((tSC - tPC) / tN) * 100).toFixed(1),
+ target_mcnemar_p: +mcnemar(tb, tc).toFixed(3),
+ // Distractor (FPR)
+ distractor_acc_with: +(dSC / dN).toFixed(3),
+ distractor_acc_placebo: +(dPC / dN).toFixed(3),
+ // FPR = false alarm on distractor (model says YES when label=false → incorrect)
+ distractor_fpr_with: +((dN - dSC) / dN).toFixed(3),
+ distractor_fpr_placebo: +((dN - dPC) / dN).toFixed(3),
+ };
+
+ const file = process.env.OUTFILE || path.join(runDir(), `binary-${FORCE_SKILL}.json`);
+ fs.mkdirSync(path.dirname(file), { recursive: true });
+ writeJson(file, out);
+ console.log(`\n Overall: WITH skill ${(out.acc_with_skill * 100).toFixed(0)}% vs placebo ${(out.acc_placebo * 100).toFixed(0)}% Δ${out.delta_pp >= 0 ? '+' : ''}${out.delta_pp}pp McNemar p=${out.mcnemar_p}${out.significant ? ' SIG' : ''}`);
+ console.log(` Targets: WITH ${(out.target_acc_with * 100).toFixed(0)}% vs placebo ${(out.target_acc_placebo * 100).toFixed(0)}% Δ${out.target_delta_pp >= 0 ? '+' : ''}${out.target_delta_pp}pp McNemar p=${out.target_mcnemar_p}`);
+ console.log(` Distractor FPR: WITH ${(out.distractor_fpr_with * 100).toFixed(0)}% vs placebo ${(out.distractor_fpr_placebo * 100).toFixed(0)}%`);
+ console.log(` -> ${file}`);
+}
+main();