Skip to content

28 eval-informed mental models and critical-thinking skills for Claude Code, GitHub Copilot, Codex, Cursor, and other Agent Skills-compatible tools

3.9 KB
Raw Blame
#!/usr/bin/env node
'use strict';

/**
 * In-domain capability-substitution test (removes the medical out-of-domain
 * confound). Same-family ladder haiku→sonnet→opus, each skill paired with ITS OWN
 * in-domain behavioral problems (software/product/decision), skill vs length-
 * matched placebo, a FIXED single judge across all tiers (so judge bias is
 * constant → clean slope). Pairwise (these problems have no ground truth).
 *
 * Reports win-rate + CI per tier and the lift-vs-capability slope. Tracked in the
 * ledger. Usage: node experiments/run-capability-indomain.js
 */

const fs = require('fs');
const path = require('path');
const { spawnSync } = require('child_process');
const lib = require('./lib');

const ROOT = lib.REPO_ROOT;
const LADDER = (process.env.LADDER || 'claude-haiku-4-5-20251001,claude-sonnet-4-6,claude-opus-4-8').split(',');
const tag = m => m.replace('claude-', '').replace('-4-8', '').replace('-4-5-20251001', '').replace('-4-6', '');
const JUDGE = process.env.JUDGES || 'gemini-3.1-pro-preview'; // fixed across tiers
const SKILLS = (process.env.SKILLS || 'thinking-second-order,thinking-opportunity-cost,thinking-theory-of-constraints,thinking-systems,thinking-pre-mortem,thinking-via-negativa,thinking-scientific-method,thinking-first-principles,thinking-probabilistic').split(',');

function runTier(model, idx) {
  const run = `indomain-${tag(model)}`;
  const env = { ...process.env, EVAL_RUN: run, SOLVER_MODEL: model, SOLVER_EFFORT: 'medium', JUDGES: JUDGE, PAIRS: 'skill:placebo', CONC: '4' };
  const r = spawnSync('node', ['evals/run-behavioral.js', ...SKILLS], { cwd: ROOT, env, encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 });
  if (r.status !== 0) { console.log(`  ${tag(model)} FAILED: ${(r.stderr || '').slice(-200)}`); return null; }
  const res = JSON.parse(fs.readFileSync(path.join(ROOT, 'evals', 'results', run, 'tier3-behavioral.json'), 'utf8'));
  const a = res.aggregate_significance;
  console.log(`  ${tag(model).padEnd(8)} skill beats placebo ${(a.win_rate * 100).toFixed(0)}%  95% CI [${(a.ci95[0] * 100).toFixed(0)},${(a.ci95[1] * 100).toFixed(0)}]  p=${a.p_value}${a.significant ? ' SIG' : ''}  (n=${a.n})`);
  return { tier: tag(model), rank: idx, win_rate: a.win_rate, ci: a.ci95, p: a.p_value, n: a.n };
}
function slope(pts) {
  const n = pts.length, sx = pts.reduce((a, p) => a + p.rank, 0), sy = pts.reduce((a, p) => a + p.win_rate, 0);
  const sxy = pts.reduce((a, p) => a + p.rank * p.win_rate, 0), sxx = pts.reduce((a, p) => a + p.rank * p.rank, 0);
  return (n * sxy - sx * sy) / (n * sxx - sx * sx) * 100; // pp per step
}

async function main() {
  lib.init();
  const id = lib.genId('CAP-MODEL-INDOMAIN');
  console.log(`▶ ${id} — in-domain capability ladder (${SKILLS.length} skills × 3 problems, judge=${JUDGE} fixed), isolation ${process.env.EVAL_NO_ISOLATE === '1' ? 'OFF' : 'ON'}`);
  lib.register({ id, hypothesis: 'CAP-MODEL-INDOMAIN', title: `In-domain capability ladder (skill vs placebo, ${SKILLS.length} software/decision skills)`, intervention: 'capability-ladder-indomain', skills: SKILLS, status: 'running', dataset: 'behavioral' });
  const pts = [];
  for (let i = 0; i < LADDER.length; i++) { const p = runTier(LADDER[i], i); if (p) { pts.push(p); lib.recordSkill(id, p.tier, 0.5, p.win_rate, p.n); } }
  const m = pts.length >= 2 ? slope(pts) : 0;
  const dir = m < -1 ? 'SUBSTITUTES (helps weak models more)' : m > 1 ? 'COMPLEMENTS (helps strong models more)' : 'flat/no interaction';
  lib.update(id, { status: 'evaluated', lift_pp: +m.toFixed(2), n_problems: pts.reduce((a, p) => a + p.n, 0),
    notes: `IN-DOMAIN (software/decision), pairwise, fixed judge ${JUDGE}, isolation ON. slope=${m.toFixed(1)}pp/step → ${dir}. tiers: ${pts.map(p => p.tier + ':' + (p.win_rate * 100).toFixed(0) + '%(p=' + p.p + ')').join(', ')}.` });
  console.log(`\n  slope = ${m.toFixed(1)} pp per capability step → ${dir}`);
  console.log(`  ledger id: ${id}`);
}
main();