Skip to content

armgit / ECC

Public Imported from GitHub
0

The agent harness performance optimization system. Skills, instincts, memory, security, and research-first development for Claude Code, Codex, Opencode, Cursor and beyond.

3.4 KB
Raw Blame
'use strict';
// Development tool: assembles a complex corpus JSON from a reviewed fixture
// tree. Usage: node build-corpus.js [casesDir=cases] [outFile=complex-corpus.json] [corpusId=complex-tasks@1]
// Run after editing any fixture, query, or grader; commit the tree and the
// regenerated corpus together.
const fs = require('node:fs');
const path = require('node:path');

const root = __dirname;
const casesDir = path.join(root, process.argv[2] || 'cases');
const OUT = path.join(root, '..', process.argv[3] || 'complex-corpus.json');
const corpusId = process.argv[4] || 'complex-tasks@1';

function collect(directory, prefix = '') {
  const files = {};
  for (const entry of fs.readdirSync(directory, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) {
    const relative = prefix ? `${prefix}/${entry.name}` : entry.name;
    if (entry.isDirectory()) Object.assign(files, collect(path.join(directory, entry.name), relative));
    else if (entry.isFile()) files[relative] = fs.readFileSync(path.join(directory, entry.name), 'utf8');
  }
  return files;
}

const tasks = [];
const selection = [];
for (const id of fs.readdirSync(casesDir).sort()) {
  const directory = path.join(casesDir, id);
  const meta = JSON.parse(fs.readFileSync(path.join(directory, 'meta.json'), 'utf8'));
  if (meta.id !== id || !/^[a-z][a-z0-9-]{0,63}$/.test(id)) throw new Error(`Invalid task metadata in ${id}`);
  const files = collect(path.join(directory, 'files'));
  const stepsDir = path.join(directory, 'steps');
  let task;
  if (fs.existsSync(stepsDir)) {
    const steps = fs.readdirSync(stepsDir).sort().map((name, index) => ({
      query: fs.readFileSync(path.join(stepsDir, name, 'query.md'), 'utf8').trim(),
      check: fs.readFileSync(path.join(stepsDir, name, 'check.cjs'), 'utf8'),
      ...(meta.steps?.[index]?.manualIds ? { manualIds: meta.steps[index].manualIds } : {}),
      ...((meta.steps?.[index]?.checkTimeoutMs || meta.checkTimeoutMs)
        ? { checkTimeoutMs: meta.steps?.[index]?.checkTimeoutMs || meta.checkTimeoutMs } : {}),
    }));
    task = { id, category: meta.category, manualIds: meta.manualIds || [], files, steps };
  } else {
    const query = fs.readFileSync(path.join(directory, 'query.md'), 'utf8').trim();
    task = { id, category: meta.category, manualIds: meta.manualIds,
      ...(meta.checkTimeoutMs ? { checkTimeoutMs: meta.checkTimeoutMs } : {}),
      query, files, check: fs.readFileSync(path.join(directory, 'check.cjs'), 'utf8') };
  }
  tasks.push(task);
  selection.push({ id: meta.selection.id, category: meta.selection.category,
    query: meta.selection.query || task.query || task.steps.map(step => step.query).join(' '),
    expectedIds: meta.selection.expectedIds });
}

const corpus = {
  schemaVersion: 'ecc.context-eval-complex-corpus.v1',
  id: corpusId,
  sampling: 'Realistic multi-file engineering tasks, fixed before any provider call, with deterministic '
    + 'hidden graders scoring partial credit (ECC_EVAL_SCORE). Descriptive pilot: no '
    + 'population-representativeness claim. See complex-eval/DESIGN.md for the preregistered methodology.',
  minimumDistinctTasks: tasks.length,
  nonInferiorityMargin: 0.05,
  selection,
  tasks,
};
fs.writeFileSync(OUT, `${JSON.stringify(corpus, null, 1)}\n`);
console.log(`wrote ${path.basename(OUT)} (${corpusId}): ${tasks.length} tasks, ${selection.length} selection probes, `
  + `${tasks.reduce((sum, task) => sum + Object.keys(task.files).length, 0)} fixture files`);