Skip to content

28 eval-informed mental models and critical-thinking skills for Claude Code, GitHub Copilot, Codex, Cursor, and other Agent Skills-compatible tools

14.9 KB
Raw Blame
'use strict';

const fs = require('fs');
const path = require('path');
const {
  analyzeObjectiveEnvelope,
} = require('./scientific-method-experiment');
const {
  buildToolLocalizationPrompt,
  parseToolLocalizationResponse,
  prepareRepository,
  TOOL_RESTRICTED_TOOLS,
  TOOL_SYSTEM_PROMPT,
} = require('./tool-localization');
const {
  executeDroid,
  usageSummary,
} = require('./droid');
const {
  mapPool,
  readJsonIfExists,
  writeJsonAtomic,
} = require('./io');
const { sha256 } = require('./result');
const { binomExactTwoSided, holmAdjustment } = require('./stats');

const EXPLORATORY_ARMS = Object.freeze([
  'none',
  'current',
  'clue-first',
  'module-role',
]);

function normalizeRepoPath(value) {
  return String(value || '').replace(/\\/g, '/').replace(/^\.\//, '');
}

function scoreToolLocalizationResponse(item, text, maxObservations = 4) {
  const parsed = parseToolLocalizationResponse(text, maxObservations);
  const gold = normalizeRepoPath(item && item.gold_files && item.gold_files[0]);
  return {
    parsed_success: Boolean(parsed),
    parsed: parsed ? parsed.answer : null,
    reported_observations: parsed ? parsed.reported_observations : null,
    budget_compliant: parsed ? parsed.budget_compliant : false,
    correct: Boolean(
      parsed
      && parsed.budget_compliant
      && normalizeRepoPath(parsed.answer) === gold,
    ),
  };
}

function analyzeToolStage(rows, candidateArms, health = {}) {
  const analysis = analyzeObjectiveEnvelope(
    { items: rows || [], health },
    {
      controlArm: 'none',
      leanArm: 'current',
      candidateArms: candidateArms || [],
    },
  );
  for (const armId of candidateArms || []) {
    const candidate = analysis.candidates[armId];
    for (const contrast of [candidate.vs_none, candidate.vs_lean]) {
      contrast.p_value = binomExactTwoSided(
        contrast.left_wins,
        contrast.left_wins + contrast.right_wins,
      );
      contrast.test = 'two-sided exact McNemar/binomial';
    }
  }
  const adjusted = holmAdjustment((candidateArms || []).map((armId) => ({
    id: armId,
    p: analysis.candidates[armId].vs_none.p_value,
  })));
  for (const result of adjusted) {
    analysis.candidates[result.id].vs_none.p_adjusted = result.p_adjusted;
    analysis.candidates[result.id].vs_none.holm_rank = result.rank;
  }
  return analysis;
}

function evaluateToolCandidateGate(analysis, candidateArm, options = {}) {
  const stage = options.stage || 'pilot';
  const candidate = analysis && analysis.candidates && analysis.candidates[candidateArm];
  const candidateUsage = analysis && analysis.usage_by_arm && analysis.usage_by_arm[candidateArm];
  const noneUsage = analysis && analysis.usage_by_arm && analysis.usage_by_arm.none;
  const reasons = [];
  if (!candidate) reasons.push('candidate contrast missing');
  if (!candidateUsage || !noneUsage) reasons.push('usage summary missing');
  if (analysis && analysis.health && analysis.health.decision_eligible === false) {
    reasons.push('run is not decision eligible');
  }
  if (!candidate || !candidateUsage || !noneUsage) {
    return { pass: false, stage, candidate_arm: candidateArm, reasons };
  }
  const tokenRatio = noneUsage.median_total_tokens
    ? candidateUsage.median_total_tokens / noneUsage.median_total_tokens
    : Infinity;
  const maxTokenRatio = stage === 'pilot' ? 1.05 : 1;
  if (candidate.vs_none.delta_pp < 5) reasons.push('lift versus no-skill is below 5pp');
  if (candidate.vs_lean.delta_pp < 0) reasons.push('accuracy is lower than current skill');
  if (tokenRatio > maxTokenRatio) {
    reasons.push(`median token ratio versus no-skill ${tokenRatio.toFixed(3)} exceeds ${maxTokenRatio}`);
  }
  if (stage !== 'pilot') {
    const minN = Number(options.minN || 100);
    if (candidate.vs_none.n < minN) reasons.push(`paired n ${candidate.vs_none.n} is below ${minN}`);
    if (!(candidate.vs_none.p_value < 0.05)) reasons.push('paired p is not below .05');
  }
  return {
    pass: reasons.length === 0,
    stage,
    candidate_arm: candidateArm,
    token_ratio_vs_none: tokenRatio,
    reasons,
  };
}

function selectPilotCandidate(analysis, candidateArms = ['clue-first', 'module-role']) {
  const passing = candidateArms
    .map((armId) => ({
      armId,
      gate: evaluateToolCandidateGate(analysis, armId, { stage: 'pilot' }),
      candidate: analysis && analysis.candidates && analysis.candidates[armId],
      usage: analysis && analysis.usage_by_arm && analysis.usage_by_arm[armId],
    }))
    .filter((entry) => entry.gate.pass);
  passing.sort((left, right) => (
    right.candidate.vs_none.delta_pp - left.candidate.vs_none.delta_pp
    || left.usage.median_total_tokens - right.usage.median_total_tokens
    || (left.armId === 'clue-first' ? -1 : 1)
  ));
  return passing.length ? passing[0].armId : null;
}

function stageArmIds(stage, selectedCandidate) {
  if (stage === 'calibration' || stage === 'pilot') return EXPLORATORY_ARMS.slice();
  if (stage === 'confirmation' || stage === 'replication') {
    if (!selectedCandidate || !EXPLORATORY_ARMS.includes(selectedCandidate)
      || selectedCandidate === 'none' || selectedCandidate === 'current') {
      throw new Error(`${stage} requires a selected candidate`);
    }
    return ['none', 'current', selectedCandidate];
  }
  throw new Error(`unknown tool-localization stage: ${stage}`);
}

function checkpointFile(checkpointDir, studyId, manifestHash, stage, itemId, armId) {
  return path.join(
    checkpointDir,
    `${sha256(`${studyId}:${manifestHash}:${stage}:${itemId}:${armId}`)}.json`,
  );
}

function verifyPinnedFiles(manifest, rootDir) {
  for (const [relativePath, expectedHash] of Object.entries(
    manifest && manifest.file_pins || {},
  )) {
    const file = path.resolve(rootDir, relativePath);
    const root = path.resolve(rootDir);
    if (file !== root && !file.startsWith(`${root}${path.sep}`)) {
      throw new Error(`pinned file escapes root: ${relativePath}`);
    }
    if (!fs.existsSync(file)) throw new Error(`pinned file missing: ${relativePath}`);
    const actualHash = sha256(fs.readFileSync(file));
    if (actualHash !== expectedHash) {
      throw new Error(`pinned file hash mismatch: ${relativePath}`);
    }
  }
  return true;
}

function normalizedUsage(result, model) {
  const summary = usageSummary(result && result.usage, model);
  return {
    input_tokens: summary.input_tokens,
    output_tokens: summary.output_tokens,
    cache_read_tokens: summary.cache_read_tokens,
    cache_creation_tokens: summary.cache_creation_tokens,
    total_tokens: summary.total_tokens,
    estimated_cost_usd: summary.est_cost_usd,
    latency_ms: Number(result && result.durationMs || 0),
    cost_model_version: summary.cost_model_version,
  };
}

function serializeFailure(failure) {
  if (!failure) return null;
  return typeof failure.toJSON === 'function' ? failure.toJSON() : failure;
}

async function runToolStage(options = {}) {
  const {
    studyId,
    manifestHash,
    stage,
    items = [],
    armDefinitions = [],
    model,
    effort = 'high',
    outputDir,
    cacheRoot,
    callCap,
    costCapUsd,
    maxObservations = 4,
    concurrency = 4,
  } = options;
  if (!studyId || !manifestHash || !stage || !model || !outputDir || !cacheRoot) {
    throw new Error('studyId, manifestHash, stage, model, outputDir, and cacheRoot are required');
  }
  if (!armDefinitions.length) throw new Error('armDefinitions are required');
  const prepare = options.prepare || ((item) => prepareRepository(item, { cacheRoot }));
  const execute = options.execute || executeDroid;
  const stageDir = path.join(outputDir, stage);
  const checkpointDir = path.join(stageDir, 'checkpoints');
  const rawDir = path.join(stageDir, 'raw');
  fs.mkdirSync(checkpointDir, { recursive: true });
  fs.mkdirSync(rawDir, { recursive: true });

  const rows = [];
  const loadedKeys = new Set();
  let solverCalls = 0;
  let reservedCalls = 0;
  let estimatedCostUsd = 0;
  let invalidReason = null;
  const errors = [];
  for (const item of items) {
    for (const arm of armDefinitions) {
      const file = checkpointFile(
        checkpointDir,
        studyId,
        manifestHash,
        stage,
        item.id,
        arm.id,
      );
      const saved = readJsonIfExists(file);
      if (!saved) continue;
      if (saved.manifest_hash !== manifestHash || saved.study_id !== studyId
        || saved.stage !== stage || saved.item_id !== item.id || saved.arm_id !== arm.id) {
        throw new Error(`incompatible checkpoint: ${file}`);
      }
      rows.push(saved);
      loadedKeys.add(`${item.id}:${arm.id}`);
      solverCalls += Number(saved.solver_calls || 0);
      estimatedCostUsd += Number(saved.usage && saved.usage.estimated_cost_usd || 0);
    }
  }

  const groupsByRepo = new Map();
  for (const item of items) {
    if (!groupsByRepo.has(item.repo)) groupsByRepo.set(item.repo, []);
    groupsByRepo.get(item.repo).push(item);
  }
  const groups = [...groupsByRepo.values()];

  await mapPool(groups, Math.max(1, Number(concurrency || 1)), async (group) => {
    for (const item of group) {
      if (invalidReason) break;
      const missingArms = armDefinitions.filter(
        (arm) => !loadedKeys.has(`${item.id}:${arm.id}`),
      );
      if (!missingArms.length) continue;
      let checkout;
      try {
        checkout = await prepare(item);
      } catch (error) {
        invalidReason = `repository preparation failed for ${item.id}`;
        errors.push({ item_id: item.id, type: 'repository', message: String(error.message || error) });
        break;
      }
      for (const arm of missingArms) {
        if (invalidReason) break;
        if (estimatedCostUsd >= Number(costCapUsd)) {
          invalidReason = `estimated cost cap reached before ${item.id}:${arm.id}`;
          break;
        }
        const availableAttempts = Number(callCap) - solverCalls - reservedCalls;
        const attempts = Math.min(2, availableAttempts);
        if (attempts < 1) {
          invalidReason = `solver call cap reached before ${item.id}:${arm.id}`;
          break;
        }
        reservedCalls += attempts;
        const prompt = buildToolLocalizationPrompt(item, {
          skillContent: arm.skillContent || '',
          maxObservations,
        });
        let result;
        try {
          result = await execute({
            model,
            effort,
            prompt,
            cwd: checkout,
            isolate: true,
            allowToolUse: true,
            restrictTools: TOOL_RESTRICTED_TOOLS,
            appendSystemPrompt: TOOL_SYSTEM_PROMPT,
            attempts,
            timeoutMs: Number(options.timeoutMs || 300000),
          });
        } catch (error) {
          result = {
            ok: false,
            text: '',
            usage: null,
            durationMs: 0,
            attempts: 1,
            failure: { type: 'transport', message: String(error.message || error) },
            raw: '',
          };
        } finally {
          reservedCalls -= attempts;
        }
        const actualCalls = Math.max(1, Number(result.attempts || 1));
        solverCalls += actualCalls;
        const usage = normalizedUsage(result, model);
        estimatedCostUsd += usage.estimated_cost_usd;
        const score = result.ok
          ? scoreToolLocalizationResponse(item, result.text, maxObservations)
          : {
              parsed_success: false,
              parsed: null,
              reported_observations: null,
              budget_compliant: false,
              correct: false,
            };
        const key = sha256(`${studyId}:${manifestHash}:${stage}:${item.id}:${arm.id}`);
        const rawFile = path.join(rawDir, `${key}.txt`);
        fs.writeFileSync(rawFile, String(result.raw || result.text || ''), 'utf8');
        const row = {
          schema_version: 1,
          study_id: studyId,
          manifest_hash: manifestHash,
          stage,
          item_id: item.id,
          repo: item.repo,
          base_commit: item.base_commit,
          trial: 1,
          arm_id: arm.id,
          completed: true,
          solver_ok: Boolean(result.ok),
          solver_calls: actualCalls,
          ...score,
          failure: serializeFailure(result.failure),
          usage,
          prompt_sha256: sha256(prompt),
          response_sha256: sha256(result.text || ''),
          archive_path: rawFile,
        };
        const file = checkpointFile(
          checkpointDir,
          studyId,
          manifestHash,
          stage,
          item.id,
          arm.id,
        );
        writeJsonAtomic(file, row);
        rows.push(row);
        loadedKeys.add(`${item.id}:${arm.id}`);
        if (!result.ok && (!result.failure || result.failure.type !== 'parse')) {
          invalidReason = `exhausted solver failure at ${item.id}:${arm.id}`;
          errors.push({
            item_id: item.id,
            arm_id: arm.id,
            type: result.failure && result.failure.type || 'transport',
            message: result.failure && result.failure.message || result.error || 'solver failure',
          });
        }
        if (estimatedCostUsd > Number(costCapUsd)) {
          invalidReason = `estimated cost cap exceeded at ${item.id}:${arm.id}`;
        }
        if (typeof options.onProgress === 'function') {
          options.onProgress({
            stage,
            completed: rows.length,
            assigned: items.length * armDefinitions.length,
            solver_calls: solverCalls,
            estimated_cost_usd: estimatedCostUsd,
            invalid_reason: invalidReason,
          });
        }
      }
    }
  });

  const itemOrder = new Map(items.map((item, index) => [item.id, index]));
  const armOrder = new Map(armDefinitions.map((arm, index) => [arm.id, index]));
  rows.sort((left, right) => (
    itemOrder.get(left.item_id) - itemOrder.get(right.item_id)
    || armOrder.get(left.arm_id) - armOrder.get(right.arm_id)
  ));
  const assigned = items.length * armDefinitions.length;
  const nonParseFailures = rows.filter(
    (row) => !row.solver_ok && (!row.failure || row.failure.type !== 'parse'),
  ).length;
  const health = {
    assigned_observations: assigned,
    completed_observations: rows.length,
    solver_calls: solverCalls,
    non_parse_solver_failures: nonParseFailures,
    parse_or_protocol_failures: rows.filter(
      (row) => !row.parsed_success || !row.budget_compliant,
    ).length,
    estimated_cost_usd: Number(estimatedCostUsd.toFixed(6)),
    call_cap: Number(callCap),
    cost_cap_usd: Number(costCapUsd),
    decision_eligible: !invalidReason && rows.length === assigned && nonParseFailures === 0,
    invalid_reason: invalidReason,
    errors,
  };
  const envelope = {
    schema_version: 1,
    study_id: studyId,
    manifest_hash: manifestHash,
    stage,
    model,
    effort,
    arm_ids: armDefinitions.map((arm) => arm.id),
    item_ids: items.map((item) => item.id),
    health,
    items: rows,
  };
  writeJsonAtomic(path.join(stageDir, 'envelope.json'), envelope);
  return envelope;
}

module.exports = {
  EXPLORATORY_ARMS,
  analyzeToolStage,
  evaluateToolCandidateGate,
  runToolStage,
  scoreToolLocalizationResponse,
  selectPilotCandidate,
  stageArmIds,
  verifyPinnedFiles,
};