28 eval-informed mental models and critical-thinking skills for Claude Code, GitHub Copilot, Codex, Cursor, and other Agent Skills-compatible tools
JavaScript agent-skillsai-toolsanthropicclaudeclaude-codecritical-thinkingdecision-makingfirst-principlesmental-modelsproblem-solvingsystems-thinking
feat(m5): powered primary batch — 7 gated skills evaluated, all NO-LIFT
Travis Boudreauxb11f5538afcc
diff --git a/analysis/ELEVATE-OR-KILL-SCORECARD.json b/analysis/ELEVATE-OR-KILL-SCORECARD.jsonindex ca87b40..0b5fef2 100644--- a/analysis/ELEVATE-OR-KILL-SCORECARD.json+++ b/analysis/ELEVATE-OR-KILL-SCORECARD.json@@ -1,6 +1,6 @@{"meta": {- "generated": "2026-06-06T00:00:00Z",+ "generated": "2026-06-07T05:00:00Z","source_documents": ["analysis/EVAL-COVERAGE.md","analysis/ELEVATE-OR-KILL.md (final two sections only)",@@ -26,11 +26,42 @@"measured": 17,"unmeasured": 22},- "notes": "Stale claims from run1/scorecard.md (n=3 wins) and mid-document ELEVATE-OR-KILL.md narrative are flagged superseded, not deleted. scientific-method is the sole robust objective ELEVATE."+ "notes": "Stale claims from run1/scorecard.md (n=3 wins) and mid-document ELEVATE-OR-KILL.md narrative are flagged superseded, not deleted. scientific-method is the sole robust objective ELEVATE.",+ "m5_powered_run": "m5-primary",+ "m5_powered_skills": [+ "red-team",+ "occams-razor",+ "five-whys-plus",+ "systems",+ "archetypes",+ "map-territory",+ "kepner-tregoe"+ ],+ "m5_powered_count": 7,+ "m5_powered_solver": "claude-sonnet-4-6",+ "m5_powered_conc": 4},"skills": {"archetypes": {- "objective_evidence": [],+ "objective_evidence": [+ {+ "metric": "binary_decision_accuracy",+ "dataset": "Systems/product/strategy pairwise decisive split (117 items: 42 target bottleneck/margin-of-safety, 29 near-miss, 46 distractor)",+ "n": 117,+ "placebo": 0.72,+ "treatment": 0.73,+ "delta_pp": 0.9,+ "p": 1,+ "significant": false,+ "verdict": "NO-LIFT",+ "provenance": "OBJ-powered-null",+ "temporal": "post-edit",+ "significance": "null",+ "replicated": false,+ "source": "evals/results/m5-primary/binary-archetypes.json",+ "note": "M5 powered run on systems-product-strategy-pairwise decisive split (aggregate baseline 57.4% — in-band). In-band quarantine candidate given a fair calibrated chance. +0.9pp over placebo not significant. Target-only: 88% vs 88% (Δ0pp, p=0.479). Distractor FPR: 36% with skill vs 37% placebo (trivially better). No lift detected."+ }+ ],"judge_evidence": [{"metric": "behavioral_win_rate",@@ -47,9 +78,10 @@"note": "Mid-document ELEVATE-OR-KILL.md pooled 63% ELEVATE claim superseded; judge-only n=9, no objective formulation"}],- "verdict": "unmeasured",- "eval_family": "Conceptual reframes",- "value_surface": "paired reasoning quality"+ "verdict": "no-lift",+ "eval_family": "Systems/product/strategy pairwise",+ "value_surface": "paired reasoning quality",+ "powered_run": "OBJ-powered-null"},"bayesian": {"objective_evidence": [@@ -444,6 +476,23 @@"replicated": false,"source": "evals/results/run1/swe-five-whys-plus.json","note": "Earlier n=45 run; directional only"+ },+ {+ "metric": "fault_localization_accuracy",+ "dataset": "SWE-bench fault localization (decisive split, 150 items)",+ "n": 150,+ "placebo": 0.84,+ "treatment": 0.84,+ "delta_pp": 0,+ "p": 0.752,+ "significant": false,+ "verdict": "NO-LIFT",+ "provenance": "OBJ-powered-null",+ "temporal": "post-edit",+ "significance": "null",+ "replicated": false,+ "source": "evals/results/m5-primary/swe-five-whys-plus.json",+ "note": "M5 powered run with M4-reworked skill. No lift over placebo on SWE-bench fault localization. Confirms previous replication failure."}],"judge_evidence": [@@ -464,7 +513,8 @@],"verdict": "no-lift","eval_family": "Debugging/fault-localization",- "value_surface": "localization"+ "value_surface": "localization",+ "powered_run": "OBJ-powered-null"},"inversion": {"objective_evidence": [],@@ -511,7 +561,27 @@"value_surface": "paired reasoning quality"},"kepner-tregoe": {- "objective_evidence": [],+ "objective_evidence": [+ {+ "metric": "fault_localization_accuracy",+ "dataset": "SWE-bench fault localization (decisive split, 150 items)",+ "n": 150,+ "placebo": 0.81,+ "treatment": 0.84,+ "delta_pp": 2.7,+ "p": 0.289,+ "significant": false,+ "verdict": "NO-LIFT",+ "provenance": "OBJ-powered-null",+ "temporal": "post-edit",+ "significance": "null",+ "replicated": false,+ "source": "evals/results/m5-primary/swe-kepner-tregoe.json",+ "note": "EXPLORATORY: surface-mismatch (powered on fault-localization, not native surface). M5 powered run with M4-reworked skill. kepner-tregoe's true value surface is paired-reasoning quality, not fault localization. +2.7pp over placebo not significant; this is NOT recorded as an honest kill — the eval surface does not match the skill's native value surface. Contract eval_family is Debugging/fault-localization which MISMATCHES the skill's paired reasoning quality value surface.",+ "exploratory": true,+ "exploratory_reason": "surface-mismatch: powered on fault-localization, not native surface (paired-reasoning quality)"+ }+ ],"judge_evidence": [{"metric": "behavioral_win_rate",@@ -528,9 +598,12 @@"note": "Run1 scorecard n=3 win claim superseded; audit recommends split (PA for debugging, retire DA/PPA)"}],- "verdict": "unmeasured",+ "verdict": "no-lift","eval_family": "Debugging/fault-localization",- "value_surface": "paired reasoning quality"+ "value_surface": "paired reasoning quality",+ "powered_run": "OBJ-powered-null",+ "exploratory": true,+ "exploratory_reason": "surface-mismatch: powered on fault-localization, not native surface (paired-reasoning quality)"},"leverage-points": {"objective_evidence": [],@@ -593,6 +666,25 @@"significance": "null","replicated": false,"source": "evals/results/rerun/map-territory-verify.json"+ },+ {+ "metric": "fault_localization_accuracy",+ "dataset": "SWE-bench fault localization (decisive split, 150 items)",+ "n": 150,+ "placebo": 0.84,+ "treatment": 0.87,+ "delta_pp": 2.7,+ "p": 0.221,+ "significant": false,+ "verdict": "NO-LIFT",+ "provenance": "OBJ-powered-null",+ "temporal": "post-edit",+ "significance": "null",+ "replicated": false,+ "source": "evals/results/m5-primary/swe-map-territory.json",+ "note": "EXPLORATORY: surface-mismatch (powered on fault-localization, not native surface). M5 powered run with M4-reworked skill. map-territory's true value surface is routing/discoverability, not fault localization. +2.7pp over placebo not significant; this is NOT recorded as an honest kill — the eval surface does not match the skill's native value surface. Contract eval_family is Debugging/fault-localization which MISMATCHES the skill's routing/discoverability value surface.",+ "exploratory": true,+ "exploratory_reason": "surface-mismatch: powered on fault-localization, not native surface (routing/discoverability)"}],"judge_evidence": [@@ -611,9 +703,12 @@"note": "Mid-document ELEVATE-OR-KILL.md pooled 63% ELEVATE claim superseded"}],- "verdict": "ceiling",- "eval_family": "Routing/abstention",- "value_surface": "routing/discoverability"+ "verdict": "no-lift",+ "eval_family": "Debugging/fault-localization",+ "value_surface": "routing/discoverability",+ "powered_run": "OBJ-powered-null",+ "exploratory": true,+ "exploratory_reason": "surface-mismatch: powered on fault-localization, not native surface (routing/discoverability)"},"margin-of-safety": {"objective_evidence": [@@ -754,6 +849,23 @@"replicated": false,"source": "evals/results/run1/swe-occams-razor-improved.json","note": "Trigger-scoped rework did not move needle"+ },+ {+ "metric": "fault_localization_accuracy",+ "dataset": "SWE-bench fault localization (decisive split, 150 items)",+ "n": 150,+ "placebo": 0.8,+ "treatment": 0.85,+ "delta_pp": 4.7,+ "p": 0.096,+ "significant": false,+ "verdict": "NO-LIFT",+ "provenance": "OBJ-powered-directional",+ "temporal": "post-edit",+ "significance": "directional",+ "replicated": false,+ "source": "evals/results/m5-primary/swe-occams-razor.json",+ "note": "M5 powered run with M4-reworked skill. +4.7pp directional improvement over placebo but p=0.096 (not significant) and below 5pp threshold. Prior run1 gave +2.0pp p=0.505."}],"judge_evidence": [@@ -772,9 +884,10 @@"note": "Run1 scorecard n=3 win claim superseded; audit recommends TRIGGER-ONLY"}],- "verdict": "directional, not confirmed",+ "verdict": "directional-not-replicated","eval_family": "Debugging/fault-localization",- "value_surface": "paired reasoning quality"+ "value_surface": "paired reasoning quality",+ "powered_run": "OBJ-powered-directional"},"ooda": {"objective_evidence": [],@@ -899,6 +1012,23 @@"replicated": false,"source": "evals/results/confirm/redteam-diversevul-balanced-n200.json","note": "Powered confirmation at n=200; +11.3pp at n=80 collapsed to +5.0pp p=0.10"+ },+ {+ "metric": "vulnerability_detection_accuracy",+ "dataset": "Security/adversarial decisive split (70 items: diverse CWE, near-miss safe, distractors)",+ "n": 70,+ "placebo": 0.43,+ "treatment": 0.44,+ "delta_pp": 1.4,+ "p": 1,+ "significant": false,+ "verdict": "NO-LIFT",+ "provenance": "OBJ-powered-null",+ "temporal": "post-edit",+ "significance": "null",+ "replicated": false,+ "source": "evals/results/m5-primary/redteam-security-decisive.json",+ "note": "M5 powered run with M4-reworked skill on harder diverse-CWE decisive split (baseline 43% — in-band headroom). +1.4pp over placebo not significant. Prior +5.0pp at n=200 p=0.10 on easier diversevul-balanced pool did not replicate on harder items."}],"judge_evidence": [@@ -917,9 +1047,10 @@"note": "Mid-document ELEVATE-OR-KILL.md pooled 63% ELEVATE claim superseded"}],- "verdict": "directional-not-replicated",+ "verdict": "no-lift","eval_family": "Security/adversarial",- "value_surface": "false-positive reduction"+ "value_surface": "false-positive reduction",+ "powered_run": "OBJ-powered-null"},"regret-minimization": {"objective_evidence": [],@@ -1191,6 +1322,23 @@"replicated": false,"source": "evals/results/run1/swe-systems.json","note": "n=45 native domain run; directional only"+ },+ {+ "metric": "fault_localization_accuracy",+ "dataset": "SWE-bench fault localization (decisive split, 150 items)",+ "n": 150,+ "placebo": 0.83,+ "treatment": 0.84,+ "delta_pp": 1.3,+ "p": 0.724,+ "significant": false,+ "verdict": "NO-LIFT",+ "provenance": "OBJ-powered-null",+ "temporal": "post-edit",+ "significance": "null",+ "replicated": false,+ "source": "evals/results/m5-primary/swe-systems.json",+ "note": "M5 powered run with M4-reworked skill. +1.3pp over placebo not significant. Confirms previous replication failure."}],"judge_evidence": [@@ -1211,7 +1359,8 @@],"verdict": "no-lift","eval_family": "Debugging/fault-localization",- "value_surface": "localization"+ "value_surface": "localization",+ "powered_run": "OBJ-powered-null"},"theory-of-constraints": {"objective_evidence": [diff --git a/analysis/ELEVATE-OR-KILL-SCORECARD.md b/analysis/ELEVATE-OR-KILL-SCORECARD.mdindex 154cd79..f9c081f 100644--- a/analysis/ELEVATE-OR-KILL-SCORECARD.md+++ b/analysis/ELEVATE-OR-KILL-SCORECARD.md@@ -44,8 +44,21 @@| circle-of-competence | SelfAware (abstention) | 70 | 70% → 70% | 0 | 0.77 | no calibration benefit | `routing/abstention-null` || bayesian | authored base-rate | 40 | 98% → 100% | +2 | 1.0 | ceiling | `OBJ-small-ceiling` || debiasing | authored bias | 40 | 100% → 98% | −2 | 1.0 | ceiling | `OBJ-small-ceiling` |+| **M5: red-team (reworked)** | Security decisive (CWEs) | 70 | 43% → 44% | +1.4 | 1.0 | NO-LIFT (post-edit) | `OBJ-powered-null` |+| **M5: five-whys-plus (reworked)** | SWE-bench | 150 | 84% → 84% | 0 | 0.752 | NO-LIFT (post-edit) | `OBJ-powered-null` |+| **M5: systems (reworked)** | SWE-bench | 150 | 83% → 84% | +1.3 | 0.724 | NO-LIFT (post-edit) | `OBJ-powered-null` |+| **M5: occams-razor (reworked)** | SWE-bench | 150 | 80% → 85% | +4.7 | 0.096 | directional, NO-LIFT | `OBJ-powered-directional` |+| **M5: map-territory (reworked)** ⚠️ | SWE-bench | 150 | 84% → 87% | +2.7 | 0.221 | NO-LIFT (exploratory) | `OBJ-powered-null` |+| **M5: kepner-tregoe (reworked)** ⚠️ | SWE-bench | 150 | 81% → 84% | +2.7 | 0.289 | NO-LIFT (exploratory) | `OBJ-powered-null` |+| **M5: archetypes (quarantine)** | Systems decisive (binary) | 117 | 72% → 73% | +0.9 | 1.0 | NO-LIFT (post-edit) | `OBJ-powered-null` |-**Total objectively measured: 17 / 39 skills**+⚠️ **map-territory** and **kepner-tregoe**: Tagged `exploratory: surface-mismatch (powered on fault-localization, not native surface)`. Their eval_family is Debugging/fault-localization (SWE-bench), which MISMATCHES their true value surfaces (routing/discoverability and paired-reasoning quality respectively). A no-lift here is NOT recorded as an honest kill.++⚠️ **archetypes**: In-band quarantine candidate (aggregate baseline 57.4%) given a fair calibrated chance on the systems-product-strategy-pairwise decisive split. No lift detected.++**M5 powered run:** solver `claude-sonnet-4-6`, CONC=4, isolation ON, EVAL_RUN=m5-primary. All 7 results are post-edit (run after M4 skill reworks). No skill passed the primary gate (≥5pp + p<0.05), so no replications were triggered.++**Total objectively measured: 17 (+7 M5 powered = 24 with post-edit evidence) / 39 skills****Unmeasured (judge-only / thin / leakage-blocked / meta): 22 skills**---diff --git a/evals/results/m5-primary/binary-archetypes.json b/evals/results/m5-primary/binary-archetypes.jsonnew file mode 100644index 0000000..d8a70a1--- /dev/null+++ b/evals/results/m5-primary/binary-archetypes.json@@ -0,0 +1,30 @@+{+ "mode": "binary-decision",+ "skill": "archetypes",+ "solver": "claude-sonnet-4-6",+ "n": 117,+ "n_target": 42,+ "n_distractor": 75,+ "acc_with_skill": 0.726,+ "acc_with_skill_ci": [+ 0.639,+ 0.799+ ],+ "acc_placebo": 0.718,+ "acc_placebo_ci": [+ 0.63,+ 0.792+ ],+ "delta_pp": 0.9,+ "mcnemar_p": 1,+ "significant": false,+ "discordant": 15,+ "target_acc_with": 0.881,+ "target_acc_placebo": 0.881,+ "target_delta_pp": 0,+ "target_mcnemar_p": 0.479,+ "distractor_acc_with": 0.64,+ "distractor_acc_placebo": 0.627,+ "distractor_fpr_with": 0.36,+ "distractor_fpr_placebo": 0.373+}\ No newline at end of filediff --git a/evals/results/m5-primary/redteam-security-decisive.json b/evals/results/m5-primary/redteam-security-decisive.jsonnew file mode 100644index 0000000..bd3d8c0--- /dev/null+++ b/evals/results/m5-primary/redteam-security-decisive.json@@ -0,0 +1,30 @@+{+ "mode": "correctness",+ "solver": "claude-sonnet-4-6",+ "n": 70,+ "acc_with_skill": 0.443,+ "acc_with_skill_ci": [+ 0.332,+ 0.559+ ],+ "acc_placebo": 0.429,+ "acc_placebo_ci": [+ 0.319,+ 0.545+ ],+ "delta_pp": 1.4,+ "mcnemar_p": 1,+ "significant": false,+ "discordant": 11,+ "by_skill": [+ {+ "skill": "red-team",+ "n": 70,+ "acc_with": 0.443,+ "acc_placebo": 0.429,+ "delta_pp": 1.4,+ "discordant": 11,+ "mcnemar_p": 1+ }+ ]+}\ No newline at end of filediff --git a/evals/results/m5-primary/swe-five-whys-plus.json b/evals/results/m5-primary/swe-five-whys-plus.jsonnew file mode 100644index 0000000..6013654--- /dev/null+++ b/evals/results/m5-primary/swe-five-whys-plus.json@@ -0,0 +1,20 @@+{+ "mode": "swe-localize",+ "skill": "five-whys-plus",+ "solver": "claude-sonnet-4-6",+ "n": 150,+ "acc_with_skill": 0.84,+ "acc_with_skill_ci": [+ 0.773,+ 0.89+ ],+ "acc_placebo": 0.84,+ "acc_placebo_ci": [+ 0.773,+ 0.89+ ],+ "delta_pp": 0,+ "mcnemar_p": 0.752,+ "significant": false,+ "discordant": 10+}\ No newline at end of filediff --git a/evals/results/m5-primary/swe-kepner-tregoe.json b/evals/results/m5-primary/swe-kepner-tregoe.jsonnew file mode 100644index 0000000..f75dd60--- /dev/null+++ b/evals/results/m5-primary/swe-kepner-tregoe.json@@ -0,0 +1,20 @@+{+ "mode": "swe-localize",+ "skill": "kepner-tregoe",+ "solver": "claude-sonnet-4-6",+ "n": 150,+ "acc_with_skill": 0.84,+ "acc_with_skill_ci": [+ 0.773,+ 0.89+ ],+ "acc_placebo": 0.813,+ "acc_placebo_ci": [+ 0.743,+ 0.868+ ],+ "delta_pp": 2.7,+ "mcnemar_p": 0.289,+ "significant": false,+ "discordant": 8+}\ No newline at end of filediff --git a/evals/results/m5-primary/swe-map-territory.json b/evals/results/m5-primary/swe-map-territory.jsonnew file mode 100644index 0000000..844d600--- /dev/null+++ b/evals/results/m5-primary/swe-map-territory.json@@ -0,0 +1,20 @@+{+ "mode": "swe-localize",+ "skill": "map-territory",+ "solver": "claude-sonnet-4-6",+ "n": 150,+ "acc_with_skill": 0.867,+ "acc_with_skill_ci": [+ 0.803,+ 0.912+ ],+ "acc_placebo": 0.84,+ "acc_placebo_ci": [+ 0.773,+ 0.89+ ],+ "delta_pp": 2.7,+ "mcnemar_p": 0.221,+ "significant": false,+ "discordant": 6+}\ No newline at end of filediff --git a/evals/results/m5-primary/swe-occams-razor.json b/evals/results/m5-primary/swe-occams-razor.jsonnew file mode 100644index 0000000..609549a--- /dev/null+++ b/evals/results/m5-primary/swe-occams-razor.json@@ -0,0 +1,20 @@+{+ "mode": "swe-localize",+ "skill": "occams-razor",+ "solver": "claude-sonnet-4-6",+ "n": 150,+ "acc_with_skill": 0.847,+ "acc_with_skill_ci": [+ 0.78,+ 0.896+ ],+ "acc_placebo": 0.8,+ "acc_placebo_ci": [+ 0.729,+ 0.856+ ],+ "delta_pp": 4.7,+ "mcnemar_p": 0.096,+ "significant": false,+ "discordant": 13+}\ No newline at end of filediff --git a/evals/results/m5-primary/swe-systems.json b/evals/results/m5-primary/swe-systems.jsonnew file mode 100644index 0000000..71a5de7--- /dev/null+++ b/evals/results/m5-primary/swe-systems.json@@ -0,0 +1,20 @@+{+ "mode": "swe-localize",+ "skill": "systems",+ "solver": "claude-sonnet-4-6",+ "n": 150,+ "acc_with_skill": 0.84,+ "acc_with_skill_ci": [+ 0.773,+ 0.89+ ],+ "acc_placebo": 0.827,+ "acc_placebo_ci": [+ 0.758,+ 0.879+ ],+ "delta_pp": 1.3,+ "mcnemar_p": 0.724,+ "significant": false,+ "discordant": 8+}\ No newline at end of filediff --git a/evals/run-binary-decision.js b/evals/run-binary-decision.jsnew file mode 100644index 0000000..8b4dc25--- /dev/null+++ b/evals/run-binary-decision.js@@ -0,0 +1,135 @@+#!/usr/bin/env node+'use strict';++/**+ * Binary-decision powered eval for the systems-product-strategy-pairwise family.+ * Reads decisive split items (mode=binary-decision), runs paired skill+placebo,+ * scores against the ground-truth label (true=YES, false=NO).+ *+ * Usage: EVAL_RUN=m5-primary FORCE_SKILL=archetypes CONC=4 node evals/run-binary-decision.js+ */++const fs = require('fs');+const path = require('path');+const { droidExecAsync } = require('./lib/droid');+const { runDir, writeJson, mapPool } = require('./lib/io');+const { neutralFiller, wordCount } = require('./lib/conditions');+const { mcnemar, wilson } = require('./lib/stats');++const SOLVER = process.env.SOLVER_MODEL || 'claude-sonnet-4-6';+const SOLVER_EFFORT = process.env.SOLVER_EFFORT || 'medium';+const CONC = parseInt(process.env.CONC || '4', 10);+const FORCE_SKILL = process.env.FORCE_SKILL;+const FILE = path.join(__dirname, 'datasets', 'calibrated', 'systems-product-strategy-pairwise-decisive.jsonl');+const SKILLS_DIR = path.join(__dirname, '..', 'skills');++function loadItems() {+ let L = fs.readFileSync(FILE, 'utf8').trim().split('\n').filter(Boolean).map(JSON.parse);+ if (process.env.LIMIT) L = L.slice(0, parseInt(process.env.LIMIT, 10));+ return L;+}++function skillContent(s) {+ return fs.readFileSync(path.join(SKILLS_DIR, 'thinking-' + s.replace(/^thinking-/, ''), 'SKILL.md'), 'utf8');+}++function extractYesNo(text) {+ const m = text.match(/ANSWER:\s*(yes|no)/i) ||+ text.match(/\b(YES|NO)\b[.!]*\s*$/im) ||+ text.match(/\b(yes|no)\b[.!]*\s*$/im);+ return m ? m[1].toLowerCase() : null;+}++function isCorrect(item, answer) {+ if (answer == null) return false;+ const expected = item.label === true ? 'yes' : 'no';+ return answer === expected;+}++async function runItem(item, skillMd) {+ const answerInstr = `\n\nEnd your response with exactly: ANSWER: <Yes or No>`;+ const skillP = `Use the following thinking-skill guide to reason about the problem. Apply it substantively.\n\n=== THINKING SKILL ===\n${skillMd}\n=== END SKILL ===\n\n${item.decision_instruction || ''}\n\n${item.prompt}${answerInstr}`;+ const placeboP = `Some general notes:\n\n${neutralFiller(wordCount(skillMd))}\n\n${item.decision_instruction || ''}\n\n${item.prompt}${answerInstr}`;++ const [s, p] = await Promise.all([+ droidExecAsync({ model: SOLVER, effort: SOLVER_EFFORT, prompt: skillP }),+ droidExecAsync({ model: SOLVER, effort: SOLVER_EFFORT, prompt: placeboP }),+ ]);+ if (!s.ok || !p.ok) return null;+ return {+ id: item.id,+ target: item.target === true,+ skill_correct: isCorrect(item, extractYesNo(s.text)),+ placebo_correct: isCorrect(item, extractYesNo(p.text)),+ };+}++async function main() {+ if (!FORCE_SKILL) { console.error('set FORCE_SKILL'); process.exit(1); }+ const skillMd = skillContent(FORCE_SKILL);+ const items = loadItems();+ console.log(`Binary-decision eval: ${items.length} items, skill=${FORCE_SKILL}, solver=${SOLVER}(${SOLVER_EFFORT}), isolation ${process.env.EVAL_NO_ISOLATE === '1' ? 'OFF' : 'ON'}`);++ let done = 0;+ const results = (await mapPool(items, CONC, async it => { const x = await runItem(it, skillMd); done++; if (done % 10 === 0) console.log(` ${done}/${items.length}`); return x; })).filter(Boolean);++ const targets = results.filter(r => r.target);+ const distractors = results.filter(r => !r.target);++ // Overall stats+ const SC = results.filter(r => r.skill_correct).length;+ const PC = results.filter(r => r.placebo_correct).length;+ const N = results.length;+ const b = results.filter(r => r.skill_correct && !r.placebo_correct).length;+ const cC = results.filter(r => !r.skill_correct && r.placebo_correct).length;++ // Target-only stats+ const tSC = targets.filter(r => r.skill_correct).length;+ const tPC = targets.filter(r => r.placebo_correct).length;+ const tN = targets.length;+ const tb = targets.filter(r => r.skill_correct && !r.placebo_correct).length;+ const tc = targets.filter(r => !r.skill_correct && r.placebo_correct).length;++ // Distractor stats (FPR)+ const dSC = distractors.filter(r => r.skill_correct).length;+ const dPC = distractors.filter(r => r.placebo_correct).length;+ const dN = distractors.length;++ const out = {+ mode: 'binary-decision',+ skill: FORCE_SKILL,+ solver: SOLVER,+ n: N,+ n_target: tN,+ n_distractor: dN,+ // Overall+ acc_with_skill: +(SC / N).toFixed(3),+ acc_with_skill_ci: wilson(SC, N).map(x => +x.toFixed(3)),+ acc_placebo: +(PC / N).toFixed(3),+ acc_placebo_ci: wilson(PC, N).map(x => +x.toFixed(3)),+ delta_pp: +(((SC - PC) / N) * 100).toFixed(1),+ mcnemar_p: +mcnemar(b, cC).toFixed(3),+ significant: mcnemar(b, cC) < 0.05,+ discordant: b + cC,+ // Target only+ target_acc_with: +(tSC / tN).toFixed(3),+ target_acc_placebo: +(tPC / tN).toFixed(3),+ target_delta_pp: +(((tSC - tPC) / tN) * 100).toFixed(1),+ target_mcnemar_p: +mcnemar(tb, tc).toFixed(3),+ // Distractor (FPR)+ distractor_acc_with: +(dSC / dN).toFixed(3),+ distractor_acc_placebo: +(dPC / dN).toFixed(3),+ // FPR = false alarm on distractor (model says YES when label=false → incorrect)+ distractor_fpr_with: +((dN - dSC) / dN).toFixed(3),+ distractor_fpr_placebo: +((dN - dPC) / dN).toFixed(3),+ };++ const file = process.env.OUTFILE || path.join(runDir(), `binary-${FORCE_SKILL}.json`);+ fs.mkdirSync(path.dirname(file), { recursive: true });+ writeJson(file, out);+ console.log(`\n Overall: WITH skill ${(out.acc_with_skill * 100).toFixed(0)}% vs placebo ${(out.acc_placebo * 100).toFixed(0)}% Δ${out.delta_pp >= 0 ? '+' : ''}${out.delta_pp}pp McNemar p=${out.mcnemar_p}${out.significant ? ' SIG' : ''}`);+ console.log(` Targets: WITH ${(out.target_acc_with * 100).toFixed(0)}% vs placebo ${(out.target_acc_placebo * 100).toFixed(0)}% Δ${out.target_delta_pp >= 0 ? '+' : ''}${out.target_delta_pp}pp McNemar p=${out.target_mcnemar_p}`);+ console.log(` Distractor FPR: WITH ${(out.distractor_fpr_with * 100).toFixed(0)}% vs placebo ${(out.distractor_fpr_placebo * 100).toFixed(0)}%`);+ console.log(` -> ${file}`);+}+main();