Skip to content

28 eval-informed mental models and critical-thinking skills for Claude Code, GitHub Copilot, Codex, Cursor, and other Agent Skills-compatible tools

37.0 KB
Raw Blame
'use strict';

const {
  sha256,
  itemKey,
  checkpointKey,
  observationCheckpointKey,
  failureRecord,
  createResultEnvelope,
  defaultUsage,
} = require('./result');
const { buildConditionPrompt, normalizeCondition, defaultTokenCounter } = require('./conditions');

function okScore({ parsed, value, correct }) {
  return {
    parsed,
    scored: true,
    value: value !== undefined ? value : parsed,
    correct: Boolean(correct),
    failure: null,
  };
}

function failScore({ type = 'parse', message, parsed = null, value = null, details = null }) {
  return {
    parsed,
    scored: false,
    value,
    correct: false,
    failure: failureRecord({ type, message, details }),
  };
}

function lastAnswerMatch(text, pattern) {
  if (text == null) return null;
  const re = new RegExp(pattern.source, pattern.flags.includes('g') ? pattern.flags : `${pattern.flags}g`);
  let m;
  let last = null;
  while ((m = re.exec(String(text))) !== null) last = m;
  return last;
}

/**
 * Pure boolean answer extraction. No gold/scoring.
 * Accepts terminal ANSWER lines, trailing yes/no, and typed JSON { "answer": bool }.
 * Schema literals such as `{ "answer": true | false }` do not parse.
 */
function parseBooleanAnswer(text) {
  const s = String(text || '');
  if (!s.trim()) return null;

  // Prefer the last complete JSON object with a boolean answer field.
  const objectMatches = s.match(/\{[\s\S]*?\}/g);
  if (objectMatches) {
    for (let i = objectMatches.length - 1; i >= 0; i--) {
      try {
        const obj = JSON.parse(objectMatches[i]);
        if (typeof obj.answer === 'boolean') return obj.answer;
      } catch (_) {
        // malformed / schema literal — keep scanning earlier objects
      }
    }
  }

  const m = lastAnswerMatch(s, /ANSWER:\s*(yes|no|true|false)\b/i)
    || s.match(/"answer"\s*:\s*(true|false)\s*(?:[,}])/i)
    || s.match(/\b(yes|no|true|false)\b\s*\.?\s*$/i);
  if (!m) return null;
  const token = m[1].toLowerCase();
  return token === 'yes' || token === 'true';
}

/** Boolean / yes-no gold. Accepts boolean gold or yes/no/true/false strings. */
function scoreBoolean(text, gold, options = {}) {
  void options;
  const parsed = parseBooleanAnswer(text);
  if (parsed == null) return failScore({ type: 'parse', message: 'no boolean ANSWER found' });

  let expected;
  if (typeof gold === 'boolean') expected = gold;
  else if (gold == null) return failScore({ type: 'scoring', message: 'missing boolean gold', parsed, value: parsed });
  else {
    const g = String(gold).trim().toLowerCase();
    if (g === 'yes' || g === 'true' || g === '1') expected = true;
    else if (g === 'no' || g === 'false' || g === '0') expected = false;
    else return failScore({ type: 'scoring', message: `unrecognized boolean gold: ${gold}`, parsed, value: parsed });
  }
  return okScore({ parsed, value: parsed, correct: parsed === expected });
}

/** Multiple choice letter A-E (or custom letters). */
function scoreMultipleChoice(text, gold, options = {}) {
  const letters = options.letters || 'ABCDE';
  const classRe = new RegExp(`ANSWER:\\s*([${letters}])\\b`, 'i');
  const m = lastAnswerMatch(text, classRe)
    || String(text || '').match(new RegExp(`\\b([${letters}])\\b\\s*\\.?\\s*$`, 'i'));
  if (!m) return failScore({ type: 'parse', message: 'no multiple-choice ANSWER found' });
  const parsed = m[1].toUpperCase();
  if (gold == null || gold === '') {
    return failScore({ type: 'scoring', message: 'missing multiple-choice gold', parsed, value: parsed });
  }
  const expected = String(gold).trim().toUpperCase();
  if (!new RegExp(`^[${letters}]$`, 'i').test(expected)) {
    return failScore({ type: 'scoring', message: `gold letter out of range: ${gold}`, parsed, value: parsed });
  }
  return okScore({ parsed, value: parsed, correct: parsed === expected });
}

/**
 * Abstention scorer. Gold is { answerable: bool } or boolean answerable.
 * Correct when abstain on unanswerable and answer on answerable.
 */
function scoreAbstention(text, gold, options = {}) {
  if (text == null || String(text).trim() === '') {
    return failScore({ type: 'parse', message: 'empty response for abstention scoring' });
  }
  const answerable = typeof gold === 'boolean' ? gold
    : (gold && typeof gold === 'object' ? Boolean(gold.answerable) : null);
  if (answerable == null) {
    return failScore({ type: 'scoring', message: 'missing answerable gold' });
  }

  const m = String(text).match(/ANSWER:\s*([\s\S]+)$/i);
  const tail = (m ? m[1] : text).trim();
  let abstained = false;
  if (/^unanswerable\b/i.test(tail)) abstained = true;
  else if (/\bunanswerable\b/i.test(tail) && tail.length < 60) abstained = true;
  else if (/\b(cannot|can'?t|impossible to|no one can|not knowable|unknowable|no way to know)\b/i.test(tail) && tail.length < 80) {
    abstained = true;
  }

  const correct = answerable ? !abstained : abstained;
  return okScore({
    parsed: abstained ? 'UNANSWERABLE' : 'ANSWERED',
    value: { abstained, answerable },
    correct,
  });
}

function parseScientificNumber(raw) {
  if (raw == null) return null;
  let s = String(raw).replace(/,/g, '').replace(/\s/g, '').replace(/[×x]10\^?/, 'e');
  const n = parseFloat(s);
  return Number.isFinite(n) ? n : null;
}

/** Numeric order-of-magnitude: |log10(|est|/|gold|)| <= tolerance (default 1). */
function scoreNumericOrderOfMagnitude(text, gold, options = {}) {
  const tolerance = options.tolerance != null ? Number(options.tolerance) : 1;
  const m = lastAnswerMatch(text, /ANSWER:\s*\$?\s*(-?[\d,]*\.?\d+(?:\s*[eE]\s*[-+]?\d+)?(?:\s*[×x]\s*10\^?-?\d+)?)/);
  let est = m ? parseScientificNumber(m[1]) : null;
  if (est == null) {
    const all = String(text || '').match(/-?\d[\d,]*\.?\d*(?:[eE][-+]?\d+)?/g);
    est = all ? parseScientificNumber(all[all.length - 1]) : null;
  }
  if (est == null || !Number.isFinite(est) || est === 0) {
    return failScore({ type: 'parse', message: 'no finite non-zero numeric ANSWER found', parsed: est });
  }
  const expected = typeof gold === 'number' ? gold
    : (gold && typeof gold === 'object' && gold.answer_num != null ? Number(gold.answer_num)
      : Number(gold));
  if (!Number.isFinite(expected) || expected === 0) {
    return failScore({ type: 'scoring', message: 'invalid numeric gold', parsed: est, value: est });
  }
  const delta = Math.abs(Math.log10(Math.abs(est) / Math.abs(expected)));
  return okScore({ parsed: est, value: { estimate: est, gold: expected, log10_delta: delta }, correct: delta <= tolerance });
}

/**
 * Probability Brier score. Gold is 0/1 outcome (or {answer_bin}).
 * value holds probability + brier; correct uses threshold 0.5 vs outcome.
 */
function scoreProbabilityBrier(text, gold, options = {}) {
  const m = lastAnswerMatch(text, /ANSWER:\s*(-?\d+(?:\.\d+)?)\s*%?/i);
  if (!m) return failScore({ type: 'parse', message: 'no probability ANSWER found' });
  let p = parseFloat(m[1]);
  if (/%/.test(m[0]) || Math.abs(p) > 1) p = p / 100;
  // Clamp to [0,1]
  p = Math.max(0, Math.min(1, p));
  if (!Number.isFinite(p)) return failScore({ type: 'parse', message: 'non-finite probability', parsed: p });

  let outcome;
  if (typeof gold === 'number') outcome = gold;
  else if (typeof gold === 'boolean') outcome = gold ? 1 : 0;
  else if (gold && typeof gold === 'object' && gold.answer_bin != null) outcome = Number(gold.answer_bin);
  else outcome = Number(gold);
  if (!(outcome === 0 || outcome === 1)) {
    return failScore({ type: 'scoring', message: 'probability gold must be 0 or 1', parsed: p, value: p });
  }
  const brier = Math.pow(p - outcome, 2);
  const correct = (p > 0.5 ? 1 : 0) === outcome;
  return okScore({
    parsed: p,
    value: { probability: p, outcome, brier },
    correct,
  });
}

function normalizeRepoPath(p) {
  if (p == null) return null;
  let s = String(p).trim().replace(/\\/g, '/');
  s = s.replace(/^[`'"]+|[`'"]+$/g, '');
  return s;
}

/** Reject absolute, drive, UNC, and parent-escape paths before inventory matching. */
function forbiddenRepoPathReason(raw) {
  if (raw == null) return 'empty path';
  const original = String(raw).trim();
  if (!original) return 'empty path';
  // Drive / UNC checks on the original (before slash normalization alone would hide them).
  if (/^[a-zA-Z]:/.test(original)) return 'drive path not allowed';
  if (original.startsWith('\\\\') || original.startsWith('//')) return 'UNC path not allowed';
  const s = original.replace(/\\/g, '/');
  if (s.startsWith('/')) return 'absolute path not allowed';
  // Normalize only relative form for segment checks; never strip a leading slash to salvage.
  const rel = s.replace(/^\.\//, '').replace(/\/{2,}/g, '/');
  const segments = rel.split('/').filter((part) => part && part !== '.');
  if (segments.some((part) => part === '..')) return 'parent segment not allowed';
  return null;
}

function toRepoRelativePath(raw) {
  const reason = forbiddenRepoPathReason(raw);
  if (reason) return { ok: false, path: null, reason };
  let s = String(raw).trim().replace(/\\/g, '/');
  s = s.replace(/^[`'"]+|[`'"]+$/g, '');
  s = s.replace(/^\.\//, '').replace(/\/{2,}/g, '/');
  if (!s || s.endsWith('/')) return { ok: false, path: s || null, reason: 'empty or directory path' };
  return { ok: true, path: s, reason: null };
}

/**
 * Strict file localization parse:
 * - exactly one terminal ANSWER: <repo-relative-path> line
 * - only whitespace may follow that line
 * - normalize separators and ./ only for relative paths
 * - reject absolute/drive/UNC/.. paths before inventory matching
 * - reject directories, multi-token paths, paths outside inventory
 */
function parseFileLocalization(text, options = {}) {
  const inventory = options.repositoryFiles || options.fileInventory || options.inventory || null;
  const raw = String(text == null ? '' : text);

  // Collect every ANSWER line. Exactly one is required — duplicates fail even if identical.
  const answerLineRe = /^\s*ANSWER:\s*(.+?)\s*$/gim;
  const answers = [];
  let m;
  while ((m = answerLineRe.exec(raw)) !== null) {
    answers.push(m[1].trim());
  }
  if (answers.length === 0) {
    return {
      ok: false,
      parsed: null,
      failure: failureRecord({ type: 'parse', message: 'no terminal ANSWER: path line found' }),
    };
  }
  if (answers.length !== 1) {
    return {
      ok: false,
      parsed: null,
      failure: failureRecord({
        type: 'parse',
        message: 'multiple ANSWER lines (exactly one terminal ANSWER required)',
        details: { answers },
      }),
    };
  }

  // Terminal: after stripping trailing whitespace, the sole ANSWER line must end the text.
  // Any non-whitespace after the ANSWER line is a parse failure.
  const stripped = raw.replace(/\s+$/u, '');
  const terminalRe = /(?:^|\n)\s*ANSWER:\s*(\S+)\s*$/i;
  const terminal = stripped.match(terminalRe);
  if (!terminal) {
    // Distinguish trailing prose after a path vs multi-token path on the ANSWER line.
    const looseTerminal = stripped.match(/(?:^|\n)\s*ANSWER:\s*(.+)$/i);
    if (!looseTerminal) {
      return {
        ok: false,
        parsed: null,
        failure: failureRecord({
          type: 'parse',
          message: 'ANSWER line is not terminal (non-whitespace content follows)',
        }),
      };
    }
    const loose = looseTerminal[1].trim();
    if (/\s/.test(loose)) {
      return {
        ok: false,
        parsed: null,
        failure: failureRecord({
          type: 'parse',
          message: 'ANSWER path must be a single terminal path token without trailing text',
          details: { raw: loose },
        }),
      };
    }
    return {
      ok: false,
      parsed: null,
      failure: failureRecord({
        type: 'parse',
        message: 'ANSWER line is not terminal (non-whitespace content follows)',
      }),
    };
  }

  const rawPath = terminal[1].trim();
  const converted = toRepoRelativePath(rawPath);
  if (!converted.ok) {
    return {
      ok: false,
      parsed: null,
      failure: failureRecord({
        type: 'parse',
        message: converted.reason || 'invalid ANSWER path',
        details: { raw: rawPath },
      }),
    };
  }
  const parsed = converted.path;

  if (inventory && Array.isArray(inventory)) {
    const inv = new Set();
    for (const entry of inventory) {
      const c = toRepoRelativePath(entry);
      if (c.ok) inv.add(c.path);
    }
    if (!inv.has(parsed)) {
      return {
        ok: false,
        parsed,
        failure: failureRecord({
          type: 'parse',
          message: 'ANSWER path outside repository file inventory',
          details: { path: parsed },
        }),
      };
    }
  } else if (!parsed.includes('/') && options.requireNestedPath) {
    return {
      ok: false,
      parsed,
      failure: failureRecord({ type: 'parse', message: 'basename-only ANSWER rejected without inventory match' }),
    };
  }

  return { ok: true, parsed, failure: null };
}

function scoreFileLocalization(text, gold, options = {}) {
  const parsedResult = parseFileLocalization(text, options);
  if (!parsedResult.ok) {
    return {
      parsed: parsedResult.parsed,
      scored: false,
      value: parsedResult.parsed,
      correct: false,
      failure: parsedResult.failure,
    };
  }
  const parsed = parsedResult.parsed;
  let goldPaths = [];
  if (Array.isArray(gold)) goldPaths = gold;
  else if (gold && typeof gold === 'object') {
    goldPaths = gold.gold_files || gold.files || gold.paths || [];
    if (!goldPaths.length && gold.path) goldPaths = [gold.path];
  } else if (gold != null) goldPaths = [gold];

  goldPaths = goldPaths.map((g) => {
    const c = toRepoRelativePath(g);
    return c.ok ? c.path : null;
  }).filter(Boolean);
  if (!goldPaths.length) {
    return failScore({ type: 'scoring', message: 'missing gold file path(s)', parsed, value: parsed });
  }
  const correct = goldPaths.includes(parsed);
  return okScore({ parsed, value: parsed, correct });
}

const SCORERS = {
  boolean: scoreBoolean,
  multiple_choice: scoreMultipleChoice,
  abstention: scoreAbstention,
  numeric_order_of_magnitude: scoreNumericOrderOfMagnitude,
  probability_brier: scoreProbabilityBrier,
  file_localization: scoreFileLocalization,
};

function scoreWithAdapter(name, text, gold, options = {}) {
  const key = String(name || '').toLowerCase().replace(/-/g, '_');
  const fn = SCORERS[key];
  if (!fn) {
    return failScore({ type: 'scoring', message: `unknown scorer: ${name}` });
  }
  try {
    return fn(text, gold, options);
  } catch (err) {
    return failScore({
      type: 'scoring',
      message: err && err.message ? err.message : String(err),
      details: { scorer: key },
    });
  }
}

function goldForItem(item, scorerName) {
  if (!item) return null;
  const key = String(scorerName || item.scorer || item.mode || '').toLowerCase().replace(/-/g, '_');
  if (item.gold !== undefined) return item.gold;
  if (key === 'boolean' || key === 'binary_decision') {
    if (typeof item.label === 'boolean') return item.label;
    if (item.answer != null) return item.answer;
  }
  if (key === 'multiple_choice') return item.answer_idx != null ? item.answer_idx : item.answer;
  if (key === 'abstention') return { answerable: item.answerable };
  if (key === 'numeric_order_of_magnitude') return item.answer_num != null ? item.answer_num : item.answer;
  if (key === 'probability_brier') return item.answer_bin != null ? item.answer_bin : item.answer;
  if (key === 'file_localization') return item.gold_files || item.gold || item.files;
  return item.answer != null ? item.answer : item.label;
}

function sumUsage(a, b) {
  const x = defaultUsage(a);
  const y = defaultUsage(b);
  return {
    input_tokens: x.input_tokens + y.input_tokens,
    output_tokens: x.output_tokens + y.output_tokens,
    cached_tokens: x.cached_tokens + y.cached_tokens,
    cache_creation_tokens: x.cache_creation_tokens + y.cache_creation_tokens,
    total_tokens: x.total_tokens + y.total_tokens,
    calls: x.calls + y.calls,
    latency_ms: x.latency_ms + y.latency_ms,
    estimated_cost_usd: x.estimated_cost_usd + y.estimated_cost_usd,
  };
}

function normalizeSolveUsage(result, countDefaultCall = false) {
  const value = result || {};
  const raw = value.usage || {};
  return defaultUsage({
    input_tokens: raw.input_tokens || 0,
    output_tokens: raw.output_tokens || 0,
    cached_tokens: raw.cached_tokens || raw.cache_read_tokens || raw.cache_read_input_tokens || 0,
    cache_creation_tokens: raw.cache_creation_tokens || raw.cache_creation_input_tokens || 0,
    total_tokens: raw.total_tokens,
    calls: raw.calls != null
      ? raw.calls
      : (value.attempts != null ? value.attempts : (countDefaultCall ? 1 : 0)),
    latency_ms: raw.latency_ms != null
      ? raw.latency_ms
      : (countDefaultCall && value.durationMs != null ? value.durationMs : 0),
    estimated_cost_usd: raw.estimated_cost_usd != null
      ? raw.estimated_cost_usd
      : (raw.est_cost_usd || 0),
  });
}

/** Mulberry32 PRNG — deterministic, no deps. */
function mulberry32(seed) {
  let a = seed >>> 0;
  return function next() {
    a = (a + 0x6D2B79F5) >>> 0;
    let t = a;
    t = Math.imul(t ^ (t >>> 15), t | 1);
    t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
    return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
  };
}

function seedToUint32(seed) {
  if (Number.isInteger(seed) && seed >= 0) return seed >>> 0;
  const hex = sha256(String(seed)).slice(0, 8);
  return parseInt(hex, 16) >>> 0;
}

function unitSeed(parts) {
  return seedToUint32(sha256(parts.map((p) => String(p)).join(':')));
}

/**
 * Deterministic arm execution order from a frozen seed.
 * Varies by itemId × trial when possible; stable across processes.
 * Canonicalizes input order so declared arm array order cannot change the shuffle.
 *
 * @param {string[]} armIds
 * @param {{seed:string|number, itemId:string, trial:number}} ctx
 * @returns {string[]} arm ids in execution order
 */
function deterministicArmOrder(armIds, ctx = {}) {
  if (!Array.isArray(armIds) || armIds.length === 0) {
    throw new TypeError('armIds must be a non-empty array');
  }
  const ids = armIds.map(String);
  const seed = ctx.seed == null ? 0 : ctx.seed;
  const itemId = ctx.itemId == null ? '' : String(ctx.itemId);
  const trial = Number.isInteger(ctx.trial) ? ctx.trial : 1;
  const rng = mulberry32(unitSeed([seed, itemId, trial, ids.slice().sort().join(',')]));
  const out = ids.slice().sort();
  for (let i = out.length - 1; i > 0; i--) {
    const j = Math.floor(rng() * (i + 1));
    const tmp = out[i];
    out[i] = out[j];
    out[j] = tmp;
  }
  return out;
}


/**
 * Run every arm × item × trial. Never drops attempted rows.
 *
 * @param {object} spec
 * @param {string} spec.studyId
 * @param {string} spec.studyVersion
 * @param {string} spec.preregistrationSha256
 * @param {object} spec.dataset
 * @param {Array}  spec.arms - [{id, prompt_sha256?, skill_sha256?, skillContent?, condition?}]
 * @param {object|string} spec.solver
 * @param {Array}  spec.items
 * @param {number} [spec.trials=1]
 * @param {string|number} [spec.armOrderSeed] - when set, shuffle arm order per item×trial deterministically
 * @param {string|function} spec.scorer - adapter name or custom (text,item,arm)=>score
 * @param {object} [spec.scorerOptions]
 * @param {function} [spec.solve] - async ({item, arm, trial, prompt}) => {ok,text,usage,failure,...}
 * @param {function} [spec.buildPrompt] - optional override ({item, arm}) => string
 * @param {string} [spec.createdAt]
 */
async function runObjectiveItems(spec) {
  if (!spec || typeof spec !== 'object') throw new TypeError('runObjectiveItems: spec required');
  const itemsIn = Array.isArray(spec.items) ? spec.items : [];
  const armsIn = Array.isArray(spec.arms) ? spec.arms : [];
  if (!armsIn.length) throw new TypeError('runObjectiveItems: arms required');
  const trials = Number.isInteger(spec.trials) && spec.trials > 0 ? spec.trials : 1;
  const armOrderSeed = Object.prototype.hasOwnProperty.call(spec, 'armOrderSeed')
    && spec.armOrderSeed != null
    ? spec.armOrderSeed
    : null;
  const armOrderPolicy = armOrderSeed == null ? 'declared' : 'seeded';
  const scorerName = typeof spec.scorer === 'string' ? spec.scorer : (spec.scorerName || null);
  const scorerFn = typeof spec.scorer === 'function' ? spec.scorer : null;
  const scorerOptions = spec.scorerOptions || {};
  const solve = typeof spec.solve === 'function' ? spec.solve : null;
  const buildPrompt = typeof spec.buildPrompt === 'function' ? spec.buildPrompt : null;
  const solverModel = typeof spec.solver === 'string'
    ? spec.solver
    : (spec.solver && spec.solver.model) || 'unknown';
  const judgeModels = (spec.judges || []).map((j) => (typeof j === 'string' ? j : j.model)).filter(Boolean);

  const armMeta = armsIn.map((arm) => {
    const id = arm.id || arm.arm_id || normalizeCondition(arm.condition) || 'arm';
    const skillContent = arm.skillContent != null ? arm.skillContent : (arm.skill_content || null);
    return {
      id: String(id),
      condition: normalizeCondition(arm.condition || arm.id),
      skillContent,
      skillName: arm.skillName || arm.skill_name || null,
      // Collect every item prompt hash for this arm; aggregate after the loop.
      promptHashes: [],
      fixed_prompt_sha256: arm.prompt_sha256 || null,
      skill_sha256: arm.skill_sha256 != null ? arm.skill_sha256 : (skillContent != null ? sha256(skillContent) : null),
      tokenBudget: arm.tokenBudget,
      raw: arm,
    };
  });

  // Fail fast: duplicate resolved IDs collapse the seeded map and duplicate keys.
  const seenArmIds = new Set();
  for (const arm of armMeta) {
    if (seenArmIds.has(arm.id)) {
      throw new TypeError(`runObjectiveItems: duplicate arm id "${arm.id}"`);
    }
    seenArmIds.add(arm.id);
  }

  const outItems = [];
  const failures = [];
  let usage = defaultUsage(spec.usage);
  let attempted = 0;
  let completed = 0;
  let scored = 0;

  function pushRow(row) {
    outItems.push(row);
  }

  function observationKeyFor({ itemId, trial, armId, promptHash, skillHash }) {
    // Bind row-specific prompt/skill/solver/judges into the compatibility key, then
    // observationCheckpointKey binds item/trial/arm.
    const compatibilityKey = checkpointKey({
      studyId: spec.studyId,
      studyVersion: spec.studyVersion,
      datasetSha256: (spec.dataset && spec.dataset.sha256) || 'missing-dataset-sha256',
      promptSha256: promptHash || 'missing-prompt-sha256',
      skillSha256: skillHash || null,
      solver: solverModel,
      judges: judgeModels,
    });
    return observationCheckpointKey({
      compatibilityKey,
      itemId,
      trial,
      armId,
    });
  }

  for (const item of itemsIn) {
    const itemId = String(item.id || item.item_id || item.instance_id || '');
    if (!itemId) throw new TypeError('runObjectiveItems: every item needs id');
    for (let trial = 1; trial <= trials; trial++) {
      // Optional frozen-seed arm order: independent solve per arm, identity keys unchanged.
      let armsForTrial = armMeta;
      if (armOrderSeed != null) {
        const orderedIds = deterministicArmOrder(
          armMeta.map((a) => a.id),
          { seed: armOrderSeed, itemId, trial },
        );
        const byId = new Map(armMeta.map((a) => [a.id, a]));
        armsForTrial = orderedIds.map((id) => byId.get(id));
      }
      for (const arm of armsForTrial) {
        attempted += 1;
        const identityKey = itemKey({
          studyId: spec.studyId,
          itemId,
          trial,
          armId: arm.id,
        });

        let prompt;
        try {
          prompt = buildPrompt
            ? buildPrompt({ item, arm: arm.raw, armId: arm.id, trial })
            : buildConditionPrompt(
              arm.condition,
              item.prompt || item.problem || item.text || '',
              arm.skillContent,
              arm.skillName || arm.id,
              {
                tokenBudget: arm.tokenBudget,
                tokenCounter: scorerOptions.tokenCounter || defaultTokenCounter,
                tail: scorerName === 'file_localization'
                  ? '\n\nYour final line must be exactly `ANSWER: path/to/file.ext` with no Markdown, bullets, or trailing text. Do not use Markdown on the ANSWER line.'
                  : (item.answer_instruction
                    ? `\n\n${item.answer_instruction}`
                    : (item.decision_instruction
                      ? `\n\n${item.decision_instruction}\nEnd your response with exactly: ANSWER: <Yes or No>`
                      : undefined)),
              },
            );
        } catch (err) {
          const failure = failureRecord({
            type: 'scoring',
            message: err && err.message ? err.message : String(err),
            itemId,
            armId: arm.id,
            trial,
          });
          failures.push(failure);
          const obsKey = observationKeyFor({
            itemId,
            trial,
            armId: arm.id,
            promptHash: null,
            skillHash: arm.skill_sha256,
          });
          pushRow({
            item_id: itemId,
            trial,
            arm_id: arm.id,
            key: identityKey,
            observation_checkpoint_key: obsKey,
            completed: false,
            parsed_success: false,
            scored: false,
            correct: false,
            parsed: null,
            value: null,
            prompt_sha256: null,
            response_sha256: null,
            usage: defaultUsage(),
            failure,
          });
          continue;
        }

        const promptHash = sha256(prompt);
        arm.promptHashes.push(promptHash);
        const obsKey = observationKeyFor({
          itemId,
          trial,
          armId: arm.id,
          promptHash,
          skillHash: arm.skill_sha256,
        });

        let solveResult;
        if (solve) {
          try {
            solveResult = await solve({
              item,
              arm: arm.raw,
              armId: arm.id,
              trial,
              prompt,
              key: identityKey,
              observationCheckpointKey: obsKey,
            });
          } catch (err) {
            solveResult = {
              ok: false,
              text: null,
              usage: { calls: 0 },
              failure: failureRecord({
                type: 'transport',
                message: err && err.message ? err.message : String(err),
                itemId,
                armId: arm.id,
                trial,
              }),
            };
          }
        } else if (item.fixture_responses && item.fixture_responses[arm.id] != null) {
          const fr = item.fixture_responses[arm.id];
          solveResult = typeof fr === 'string'
            ? { ok: true, text: fr, usage: { calls: 0 } }
            : {
              ok: fr.ok !== false,
              text: fr.text != null ? fr.text : null,
              usage: fr.usage,
              failure: fr.failure || null,
            };
        } else if (spec.fixtureMode) {
          solveResult = {
            ok: false,
            text: null,
            failure: failureRecord({
              type: 'transport',
              message: 'fixture mode: missing fixture_responses for arm',
              itemId,
              armId: arm.id,
              trial,
            }),
          };
        } else {
          throw new TypeError('runObjectiveItems: solve callback or fixture_responses required');
        }

        const rowUsage = normalizeSolveUsage(solveResult, Boolean(solve));
        usage = sumUsage(usage, rowUsage);

        if (!solveResult || solveResult.ok === false) {
          const failure = solveResult && solveResult.failure
            ? {
              ...solveResult.failure,
              item_id: solveResult.failure.item_id != null ? solveResult.failure.item_id : itemId,
              arm_id: solveResult.failure.arm_id != null ? solveResult.failure.arm_id : arm.id,
              trial: solveResult.failure.trial != null ? solveResult.failure.trial : trial,
            }
            : failureRecord({
              type: (solveResult && solveResult.failureType) || 'transport',
              message: (solveResult && (solveResult.error || solveResult.message)) || 'solve failed',
              itemId,
              armId: arm.id,
              trial,
              attempt: solveResult && solveResult.attempts,
            });
          failures.push(failure);
          pushRow({
            item_id: itemId,
            trial,
            arm_id: arm.id,
            key: identityKey,
            observation_checkpoint_key: obsKey,
            completed: false,
            parsed_success: false,
            scored: false,
            correct: false,
            parsed: null,
            value: null,
            prompt_sha256: promptHash,
            response_sha256: solveResult && solveResult.text != null ? sha256(solveResult.text) : null,
            archive_uri: solveResult && solveResult.archive_uri || null,
            usage: rowUsage,
            failure,
            attempts: solveResult && solveResult.attempts != null ? solveResult.attempts : null,
            duration_ms: solveResult && solveResult.durationMs != null ? solveResult.durationMs : null,
          });
          continue;
        }

        completed += 1;
        const text = solveResult.text;
        const responseHash = text != null ? sha256(text) : null;

        let score;
        if (scorerFn) {
          try {
            score = scorerFn(text, item, arm.raw, scorerOptions);
          } catch (err) {
            score = failScore({
              type: 'scoring',
              message: err && err.message ? err.message : String(err),
            });
          }
        } else {
          const name = scorerName || item.scorer || item.mode || 'boolean';
          const gold = goldForItem(item, name);
          const opts = {
            ...scorerOptions,
            repositoryFiles: item.repository_files || item.file_inventory || scorerOptions.repositoryFiles,
          };
          score = scoreWithAdapter(name, text, gold, opts);
        }

        if (!score || typeof score !== 'object') {
          score = failScore({ type: 'scoring', message: 'scorer returned no result' });
        }

        const parsedSuccess = score.failure
          ? (score.failure.type !== 'parse' && score.parsed !== null && score.parsed !== undefined)
          : (score.parsed !== null && score.parsed !== undefined);

        if (score.failure) {
          const failure = {
            ...score.failure,
            item_id: itemId,
            arm_id: arm.id,
            trial,
          };
          failures.push(failure);
          pushRow({
            item_id: itemId,
            trial,
            arm_id: arm.id,
            key: identityKey,
            observation_checkpoint_key: obsKey,
            completed: true,
            parsed_success: parsedSuccess,
            scored: false,
            correct: false,
            parsed: score.parsed,
            value: score.value,
            prompt_sha256: promptHash,
            response_sha256: responseHash,
            archive_uri: solveResult.archive_uri || null,
            usage: rowUsage,
            failure,
            attempts: solveResult.attempts != null ? solveResult.attempts : null,
            duration_ms: solveResult.durationMs != null ? solveResult.durationMs : null,
          });
          continue;
        }

        scored += 1;
        pushRow({
          item_id: itemId,
          trial,
          arm_id: arm.id,
          key: identityKey,
          observation_checkpoint_key: obsKey,
          completed: true,
          parsed_success: true,
          scored: true,
          correct: Boolean(score.correct),
          parsed: score.parsed,
          value: score.value,
          prompt_sha256: promptHash,
          response_sha256: responseHash,
          archive_uri: solveResult.archive_uri || null,
          usage: rowUsage,
          failure: null,
          attempts: solveResult.attempts != null ? solveResult.attempts : null,
          duration_ms: solveResult.durationMs != null ? solveResult.durationMs : null,
        });
      }
    }
  }

  const envelopeArms = armMeta.map((arm) => {
    // Always derive from every generated item prompt. A declared arm.prompt_sha256
    // is bound as an extra input, never a substitute that can mask item changes.
    const parts = [...arm.promptHashes];
    if (arm.fixed_prompt_sha256) parts.push(arm.fixed_prompt_sha256);
    const prompt_sha256 = parts.length ? sha256(parts) : sha256(arm.id);
    return {
      id: arm.id,
      prompt_sha256,
      skill_sha256: arm.skill_sha256,
    };
  });

  const statistics = {
    ...(spec.statistics || {}),
    arm_order_policy: armOrderPolicy,
    arm_order_seed: armOrderSeed,
    per_arm: {},
  };
  for (const arm of armMeta) {
    const rows = outItems.filter((r) => r.arm_id === arm.id);
    const scoredRows = rows.filter((r) => r.scored);
    // ITT: failures count as incorrect; denominator is attempted, never shrinks.
    const correct = rows.filter((r) => r.correct === true).length;
    const conditionalCorrect = scoredRows.filter((r) => r.correct === true).length;
    statistics.per_arm[arm.id] = {
      attempted: rows.length,
      completed: rows.filter((r) => r.completed).length,
      parsed: rows.filter((r) => r.parsed_success).length,
      scored: scoredRows.length,
      correct,
      accuracy: rows.length ? correct / rows.length : null,
      conditional_accuracy: scoredRows.length ? conditionalCorrect / scoredRows.length : null,
    };
  }

  const parsedCount = outItems.filter((r) => r.parsed_success).length;

  return createResultEnvelope({
    study_id: spec.studyId,
    study_version: spec.studyVersion,
    preregistration_sha256: spec.preregistrationSha256,
    dataset: spec.dataset,
    arms: envelopeArms,
    solver: spec.solver,
    judges: spec.judges || [],
    items: outItems,
    failures,
    usage,
    statistics,
    health: {
      attempted,
      completed,
      parsed: parsedCount,
      scored,
      failures: failures.length,
    },
    created_at: spec.createdAt,
  });
}

/**
 * Summarize a two-arm objective envelope into skill-vs-control paired stats.
 * Uses ITT: unscored/failed arms count as incorrect. Denominator is paired items
 * that have both arms attempted (any completion state).
 */
function summarizePairedArms(envelope, leftArm = 'skill', rightArm = 'placebo') {
  const { mcnemar, wilson } = require('./stats');
  const rows = Array.isArray(envelope && envelope.items) ? envelope.items : [];
  const byItem = new Map();
  for (const row of rows) {
    const key = `${row.item_id}::${row.trial || 1}`;
    if (!byItem.has(key)) byItem.set(key, { id: row.item_id, trial: row.trial || 1 });
    const slot = byItem.get(key);
    if (row.arm_id === leftArm) slot.left = row;
    if (row.arm_id === rightArm) slot.right = row;
  }
  const pairs = [...byItem.values()].filter((p) => p.left && p.right);
  const n = pairs.length;
  const leftCorrect = pairs.filter((p) => p.left.correct === true).length;
  const rightCorrect = pairs.filter((p) => p.right.correct === true).length;
  const b = pairs.filter((p) => p.left.correct === true && p.right.correct !== true).length;
  const c = pairs.filter((p) => p.left.correct !== true && p.right.correct === true).length;
  const mcn = mcnemar(b, c);
  return {
    n,
    acc_with_skill: n ? +(leftCorrect / n).toFixed(3) : 0,
    acc_with_skill_ci: wilson(leftCorrect, n || 1).map((x) => +x.toFixed(3)),
    acc_placebo: n ? +(rightCorrect / n).toFixed(3) : 0,
    acc_placebo_ci: wilson(rightCorrect, n || 1).map((x) => +x.toFixed(3)),
    delta_pp: n ? +(((leftCorrect - rightCorrect) / n) * 100).toFixed(1) : 0,
    mcnemar_p: +mcn.toFixed(3),
    significant: mcn < 0.05,
    discordant: b + c,
    left_wins: b,
    right_wins: c,
    left_arm: leftArm,
    right_arm: rightArm,
  };
}

/**
 * Extract a boolean answer from free text or typed JSON.
 * Schema literals such as `{ "answer": true | false }` do not parse.
 */
function extractYesNo(text) {
  return parseBooleanAnswer(text);
}

/**
 * Paired arm contrast on case_success (or correct) fields.
 * Returns null when any item is missing either arm.
 */
function pairedContrast(items, left, right, opts = {}) {
  const metric = opts.metric || 'case_success';
  if (!items || !items.length) {
    return {
      delta_pp: 0,
      mcnemar_p: 1,
      discordant: 0,
      left_wins: 0,
      right_wins: 0,
    };
  }
  if (!items.every((i) => i.by_arm && i.by_arm[left] && i.by_arm[right])) return null;
  const leftOnly = items.filter((i) => i.by_arm[left][metric] && !i.by_arm[right][metric]).length;
  const rightOnly = items.filter((i) => !i.by_arm[left][metric] && i.by_arm[right][metric]).length;
  const leftAcc = items.filter((i) => i.by_arm[left][metric]).length / items.length;
  const rightAcc = items.filter((i) => i.by_arm[right][metric]).length / items.length;
  // Lazy require to avoid circular dependency with stats at load time if any.
  const { mcnemar } = require('./stats');
  return {
    delta_pp: +((leftAcc - rightAcc) * 100).toFixed(1),
    mcnemar_p: +mcnemar(leftOnly, rightOnly).toFixed(3),
    discordant: leftOnly + rightOnly,
    left_wins: leftOnly,
    right_wins: rightOnly,
  };
}

module.exports = {
  SCORERS,
  parseBooleanAnswer,
  scoreBoolean,
  scoreMultipleChoice,
  scoreAbstention,
  scoreNumericOrderOfMagnitude,
  scoreProbabilityBrier,
  parseFileLocalization,
  scoreFileLocalization,
  scoreWithAdapter,
  runObjectiveItems,
  deterministicArmOrder,
  mulberry32,
  seedToUint32,
  goldForItem,
  normalizeRepoPath,
  normalizeSolveUsage,
  extractYesNo,
  pairedContrast,
  summarizePairedArms,
};