Skip to content

28 eval-informed mental models and critical-thinking skills for Claude Code, GitHub Copilot, Codex, Cursor, and other Agent Skills-compatible tools

13.2 KB
Raw Blame
#!/usr/bin/env node
'use strict';

/**
 * Generic blind pairwise engine CLI.
 *
 * Supports deterministic fixture mode and authenticated live studies.
 *
 * Usage:
 *   node evals/run-pairwise.js --fixture path/to/fixture.json
 *   node evals/run-pairwise.js --study path/to/study.json
 *
 * Fixture JSON shape:
 *   {
 *     "study_id", "study_version", "preregistration_sha256",
 *     "dataset": { source, version, split, sha256 },
 *     "solver": { model, effort },
 *     "judges": ["gpt-5.5-pro", "gemini-3.1-pro-preview", "deepseek-v4-pro"],
 *     "arms": [{ "id": "none", "prompt_sha256"?: "..." }, { "id": "lean", ... }],
 *     "pair": { "left": "lean", "right": "none" },
 *     "seed": 0,
 *     "trials": 1,
 *     "items": [{
 *       "id", "prompt",
 *       "fixture_responses": { "none": "text...", "lean": "text..." },
 *       "fixture_judge": { "winner": "A"|"B"|"tie", "resolved": true } // optional override
 *     }]
 *   }
 */

const fs = require('fs');
const path = require('path');
const { sha256, validateResultEnvelope } = require('./lib/result');
const { runPairwiseItems } = require('./lib/pairwise');
const { panelModels, panelJudge } = require('./lib/judge');
const { executeDroid, usageSummary } = require('./lib/droid');

function usage(code = 0) {
  const msg = `Usage:
  node evals/run-pairwise.js --fixture <fixture.json> [--out file.json]
  node evals/run-pairwise.js --study <study.json> [--out file.json]

Fixture mode requires fixture_responses per arm. Live mode uses the authenticated droid transport.
`;
  if (code) console.error(msg);
  else console.log(msg);
  process.exit(code);
}

function parseArgs(argv) {
  const out = { fixture: null, study: null, out: null, trials: null, seed: null };
  for (let i = 0; i < argv.length; i++) {
    const a = argv[i];
    if (a === '--help' || a === '-h') usage(0);
    else if (a === '--fixture' && argv[i + 1]) out.fixture = argv[++i];
    else if (a === '--study' && argv[i + 1]) out.study = argv[++i];
    else if (a === '--out' && argv[i + 1]) out.out = argv[++i];
    else if (a === '--trials' && argv[i + 1]) out.trials = parseInt(argv[++i], 10);
    else if (a === '--seed' && argv[i + 1]) out.seed = argv[++i];
    else if (a.startsWith('--')) {
      console.error(`unknown flag: ${a}`);
      usage(1);
    }
  }
  return out;
}

function loadJson(file) {
  return JSON.parse(fs.readFileSync(file, 'utf8'));
}

function fixtureSolve(study) {
  return async function solve({ item, armId }) {
    const table = item.fixture_responses || item.responses || {};
    const entry = table[armId];
    if (entry == null) {
      return {
        ok: false,
        text: null,
        failure: {
          type: 'transport',
          message: `fixture mode: missing fixture_responses for arm ${armId}`,
        },
        usage: { calls: 0 },
      };
    }
    if (typeof entry === 'string') {
      return { ok: true, text: entry, usage: { calls: 0, input_tokens: 0, output_tokens: 0 } };
    }
    return {
      ok: entry.ok !== false,
      text: entry.text != null ? entry.text : null,
      usage: entry.usage || { calls: 0 },
      failure: entry.failure || null,
      durationMs: entry.durationMs || 0,
      attempts: entry.attempts != null ? entry.attempts : 1,
    };
  };
}

function liveSolve(study) {
  return async function solve({ prompt }) {
    const model = (study.solver && study.solver.model) || process.env.SOLVER_MODEL;
    const result = await executeDroid({
      model,
      effort: (study.solver && study.solver.effort) || process.env.SOLVER_EFFORT,
      prompt,
      timeoutMs: parseInt(process.env.DROID_TIMEOUT_MS || '180000', 10),
      attempts: parseInt(process.env.DROID_ATTEMPTS || '3', 10),
    });
    const normalized = usageSummary(result.usage, model);
    return {
      ...result,
      usage: {
        ...normalized,
        calls: result.attempts,
        latency_ms: result.durationMs || 0,
        estimated_cost_usd: normalized.est_cost_usd,
      },
    };
  };
}

/**
 * Fixture judge: uses item.fixture_judge, study.fixture_judge, or a deterministic
 * default that prefers arm A when responses differ in length (stable, no models).
 */
function fixtureJudge(study) {
  return async function judge(prompt, judges) {
    // Allow per-call override via a marker embedded only in tests; primary path is study items.
    // The runner rebinds per item via closure below when items carry fixture_judge.
    void prompt;
    const models = Array.isArray(judges) && judges.length ? judges : panelModels();
    const defaultWinner = study.fixture_judge && study.fixture_judge.winner
      ? study.fixture_judge.winner
      : 'A';
    const resolved = study.fixture_judge && study.fixture_judge.resolved === false
      ? false
      : true;
    if (!resolved) {
      return {
        winner: null,
        resolved: false,
        unresolved: true,
        failure: { type: 'judge_unresolved', message: 'fixture unresolved panel' },
        tally: { A: 0, B: 0, tie: 0, missing: models.length },
        votes: models.map(model => ({ model, winner: null, valid: false, failure: { type: 'parse', message: 'fixture missing' } })),
        failures: [],
        vocab_only: false,
        whys: [],
        judge_usage: [],
        judge_durationMs: 0,
      };
    }
    const winner = String(defaultWinner).toUpperCase() === 'B' ? 'B'
      : String(defaultWinner).toUpperCase() === 'TIE' ? 'tie'
        : 'A';
    // Two matching valid votes for A/B; for tie inject resolved=true path used by pairwise normalize.
    if (winner === 'tie') {
      return {
        winner: 'tie',
        resolved: true,
        unresolved: false,
        failure: null,
        tally: { A: 0, B: 0, tie: models.length, missing: 0 },
        votes: models.map(model => ({ model, winner: 'tie', valid: true, failure: null })),
        failures: [],
        vocab_only: false,
        whys: ['fixture tie'],
        judge_usage: [],
        judge_durationMs: 0,
      };
    }
    return {
      winner,
      resolved: true,
      unresolved: false,
      failure: null,
      tally: {
        A: winner === 'A' ? Math.max(2, models.length) : 0,
        B: winner === 'B' ? Math.max(2, models.length) : 0,
        tie: 0,
        missing: 0,
      },
      votes: models.map(model => ({ model, winner, valid: true, failure: null })),
      failures: [],
      vocab_only: false,
      whys: [`fixture winner ${winner}`],
      judge_usage: [],
      judge_durationMs: 0,
    };
  };
}

/**
 * Wrap fixture judge so per-item fixture_judge overrides study default.
 * We recover the item id from the judge prompt PROBLEM section when present,
 * else fall back to sequential consumption of items list.
 */
function makeItemAwareJudge(study, baseJudge) {
  const byId = new Map();
  for (const item of study.items || []) {
    const id = item.id || item.item_id;
    if (id != null && item.fixture_judge) byId.set(String(id), item.fixture_judge);
  }
  if (byId.size === 0) return baseJudge;

  return async function judge(prompt, judges) {
    // Prefer explicit id tag if buildJudgePrompt embeds it; else scan prompt for item prompts.
    let override = null;
    const idMatch = String(prompt).match(/\[item_id=([^\]]+)\]/);
    if (idMatch && byId.has(idMatch[1])) override = byId.get(idMatch[1]);
    if (!override) {
      for (const item of study.items || []) {
        const id = String(item.id || item.item_id || '');
        const text = item.prompt || item.problem || item.text || '';
        if (text && String(prompt).includes(text) && byId.has(id)) {
          override = byId.get(id);
          break;
        }
      }
    }
    if (!override) return baseJudge(prompt, judges);

    const models = Array.isArray(judges) && judges.length ? judges : panelModels();
    if (override.resolved === false || override.unresolved === true) {
      return {
        winner: null,
        resolved: false,
        unresolved: true,
        failure: override.failure || { type: 'judge_unresolved', message: 'fixture unresolved panel' },
        tally: override.tally || { A: 0, B: 0, tie: 0, missing: models.length },
        votes: override.votes || models.map(model => ({ model, winner: null, valid: false })),
        failures: override.failures || [],
        vocab_only: !!override.vocab_only,
        whys: override.whys || [],
        judge_usage: override.judge_usage || [],
        judge_durationMs: override.judge_durationMs || 0,
      };
    }
    const winnerRaw = override.winner == null ? 'A' : String(override.winner);
    const upper = winnerRaw.toUpperCase();
    const winner = upper === 'B' ? 'B' : upper === 'TIE' ? 'tie' : 'A';
    return {
      winner,
      resolved: true,
      unresolved: false,
      failure: null,
      tally: override.tally || {
        A: winner === 'A' ? 2 : 0,
        B: winner === 'B' ? 2 : 0,
        tie: winner === 'tie' ? 2 : 0,
        missing: 0,
      },
      votes: override.votes || models.map(model => ({ model, winner, valid: true, failure: null })),
      failures: [],
      vocab_only: !!override.vocab_only,
      whys: override.whys || [`fixture winner ${winner}`],
      judge_usage: override.judge_usage || [],
      judge_durationMs: override.judge_durationMs || 0,
    };
  };
}

async function main() {
  const args = parseArgs(process.argv.slice(2));
  if (Boolean(args.fixture) === Boolean(args.study)) usage(1);

  const fixtureMode = Boolean(args.fixture);
  const studyPath = args.fixture || args.study;
  const study = loadJson(path.resolve(studyPath));
  const items = study.items || [];

  if (fixtureMode && items.some(it => !it.fixture_responses && !it.responses)) {
    console.error('run-pairwise: each fixture item needs fixture_responses for both arms');
    process.exit(1);
  }

  const arms = (study.arms || []).map(a => ({
    ...a,
    id: a.id,
    prompt_sha256: a.prompt_sha256 || sha256(a.id),
    skill_sha256: a.skill_sha256 != null ? a.skill_sha256 : null,
  }));
  if (arms.length < 2) {
    console.error('run-pairwise: study needs at least two arms');
    process.exit(1);
  }

  const pair = study.pair || { left: arms[0].id, right: arms[1].id };
  const judges = study.judges || panelModels();
  const armById = new Map(arms.map(arm => [arm.id, arm]));
  const runOptions = {
    studyId: study.study_id || study.studyId,
    studyVersion: study.study_version || study.studyVersion || '1',
    preregistrationSha256: study.preregistration_sha256 || study.preregistrationSha256 || sha256('adhoc'),
    dataset: study.dataset || {
      source: fixtureMode ? 'fixture' : studyPath,
      version: '0',
      split: fixtureMode ? 'fixture' : 'adhoc',
      sha256: sha256(items.map(it => it.id || it.item_id)),
    },
    arms,
    pair,
    solver: study.solver || { model: fixtureMode ? 'fixture-model' : process.env.SOLVER_MODEL, effort: null },
    judges,
    items,
    trials: args.trials || study.trials || 1,
    seed: args.seed != null ? args.seed : (study.seed != null ? study.seed : 0),
    solve: fixtureMode ? fixtureSolve(study) : liveSolve(study),
    judge: fixtureMode
      ? makeItemAwareJudge(study, fixtureJudge(study))
      : panelJudge,
    createdAt: study.created_at,
    retainResponseText: study.retain_response_text === true,
  };

  if (fixtureMode) {
    runOptions.buildJudgePrompt = ({ item, responseA, responseB }) => {
      const problem = item.prompt || item.problem || item.text || '';
      const id = item.id || item.item_id || '';
      return [
        `Pairwise fixture judge [item_id=${id}]`,
        '',
        '=== PROBLEM ===',
        problem,
        '',
        '=== RESPONSE A ===',
        responseA,
        '',
        '=== RESPONSE B ===',
        responseB,
        '',
        '=== END ===',
      ].join('\n');
    };
  } else {
    runOptions.buildPrompt = ({ item, armId }) => {
      const problem = item.prompt || item.problem || item.text || '';
      const arm = armById.get(armId);
      const guide = arm && (arm.skillContent || arm.skill_content || arm.instruction || arm.prompt);
      if (armId === 'none') {
        return `Answer the problem directly without using a named thinking-skill guide.\n\n${problem}`;
      }
      if (!guide) throw new Error(`live arm ${armId} is missing skillContent/instruction`);
      return `Use the following reasoning guide to answer the problem. Apply it substantively without naming the guide.\n\n=== GUIDE ===\n${guide}\n=== END GUIDE ===\n\n${problem}`;
    };
  }

  const envelope = await runPairwiseItems(runOptions);
  const validation = validateResultEnvelope(envelope);
  if (!validation.ok) {
    console.error('run-pairwise: envelope validation failed:\n' + validation.errors.join('\n'));
    process.exit(2);
  }

  const outPath = args.out || process.env.OUTFILE;
  if (outPath) {
    fs.mkdirSync(path.dirname(path.resolve(outPath)), { recursive: true });
    fs.writeFileSync(outPath, `${JSON.stringify(envelope, null, 2)}\n`, 'utf8');
    console.log(`wrote ${outPath}`);
  } else {
    process.stdout.write(`${JSON.stringify(envelope, null, 2)}\n`);
  }

  const h = envelope.health;
  const s = envelope.statistics || {};
  console.error(
    `pairwise: attempted=${h.attempted} completed=${h.completed} parsed=${h.parsed} scored=${h.scored} ` +
    `failures=${h.failures} eligible=${h.decision_eligible} ` +
    `W/L/T=${s.wins || 0}/${s.losses || 0}/${s.ties || 0}`
  );
}

main().catch((err) => {
  console.error(err && err.stack ? err.stack : err);
  process.exit(1);
});