The agent harness performance optimization system. Skills, instincts, memory, security, and research-first development for Claude Code, Codex, Opencode, Cursor and beyond.
refactor(eval): split the context-profile evaluator run loop
Affaan Mustafa35f57c80e6b7
diff --git a/docker/context-profiles/ai-eval-lib.js b/docker/context-profiles/ai-eval-lib.jsindex c90793cf2..9b5397470 100644--- a/docker/context-profiles/ai-eval-lib.js+++ b/docker/context-profiles/ai-eval-lib.js@@ -75,6 +75,37 @@ function validateCorpusIds(cases) {// Complex corpora hold a few realistic multi-file tasks with scored hidden graders. Sample gates// are descriptive at this size, so the distinct-task minimum relaxes to the corpus itself.+function invalidComplexStep(step) {+ return typeof step.query !== 'string' || !bounded(Buffer.byteLength(step.query), 1, 8192)+ || typeof step.check !== 'string' || !bounded(Buffer.byteLength(step.check), 1, 65536)+ || (step.checkTimeoutMs !== undefined && !bounded(step.checkTimeoutMs, 1, 120000))+ || (step.manualIds !== undefined && (!Array.isArray(step.manualIds) || step.manualIds.length > 3));+}++function validateComplexTaskFiles(task) {+ const files = Object.entries(task.files || {});+ if (!Array.isArray(task.manualIds) || task.manualIds.length > 3 || !bounded(files.length, 1, 24)+ || files.some(([file, content]) => !safeRelative(file) || typeof content !== 'string' || Buffer.byteLength(content) > 65536)) {+ throw new Error('Invalid corpus task');+ }+}++function validateComplexTask(task) {+ if (!/^[a-z][a-z0-9-]{0,63}$/.test(task.id)) throw new Error('Invalid corpus case');+ if (task.steps === undefined+ && (typeof task.query !== 'string' || !bounded(Buffer.byteLength(task.query), 1, 8192))) throw new Error('Invalid corpus case');+ validateComplexTaskFiles(task);+ if (task.steps !== undefined) {+ // Stepped (chained) task: sequential tickets graded in one accumulating workspace.+ if (!Array.isArray(task.steps) || !bounded(task.steps.length, 2, 8) || task.steps.some(invalidComplexStep)) {+ throw new Error('Invalid corpus task');+ }+ } else if (typeof task.check !== 'string' || !bounded(Buffer.byteLength(task.check), 1, 65536)+ || (task.checkTimeoutMs !== undefined && !bounded(task.checkTimeoutMs, 1, 120000))) {+ throw new Error('Invalid corpus task');+ }+}+function validateComplexCorpus(corpus) {if (!Array.isArray(corpus.selection) || !Array.isArray(corpus.tasks)|| !bounded(corpus.selection.length, 0, 50) || !bounded(corpus.tasks.length, 1, 10)@@ -83,29 +114,7 @@ function validateComplexCorpus(corpus) {}validateCorpusIds(corpus.selection);if (new Set(corpus.tasks.map(c => c.id)).size !== corpus.tasks.length) throw new Error('Duplicate corpus ID');- for (const task of corpus.tasks) {- if (!/^[a-z][a-z0-9-]{0,63}$/.test(task.id)) throw new Error('Invalid corpus case');- if (task.steps === undefined- && (typeof task.query !== 'string' || !bounded(Buffer.byteLength(task.query), 1, 8192))) throw new Error('Invalid corpus case');- const files = Object.entries(task.files || {});- if (!Array.isArray(task.manualIds) || task.manualIds.length > 3 || !bounded(files.length, 1, 24)- || files.some(([file, content]) => !safeRelative(file) || typeof content !== 'string' || Buffer.byteLength(content) > 65536)) {- throw new Error('Invalid corpus task');- }- if (task.steps !== undefined) {- // Stepped (chained) task: sequential tickets graded in one accumulating workspace.- if (!Array.isArray(task.steps) || !bounded(task.steps.length, 2, 8)- || task.steps.some(step => typeof step.query !== 'string' || !bounded(Buffer.byteLength(step.query), 1, 8192)- || typeof step.check !== 'string' || !bounded(Buffer.byteLength(step.check), 1, 65536)- || (step.checkTimeoutMs !== undefined && !bounded(step.checkTimeoutMs, 1, 120000))- || (step.manualIds !== undefined && (!Array.isArray(step.manualIds) || step.manualIds.length > 3)))) {- throw new Error('Invalid corpus task');- }- } else if (typeof task.check !== 'string' || !bounded(Buffer.byteLength(task.check), 1, 65536)- || (task.checkTimeoutMs !== undefined && !bounded(task.checkTimeoutMs, 1, 120000))) {- throw new Error('Invalid corpus task');- }- }+ for (const task of corpus.tasks) validateComplexTask(task);}function sourceSnapshot(repoRoot) {@@ -601,36 +610,50 @@ function launchEnvironment(launch) {TMPDIR: launch.home, LANG: 'C.UTF-8' };}+// One provider call. A throwing provider still records its elapsed time and+// restores the environment; only source drift propagates as itself.+function callProvider(state, environment, request, started) {+ try {+ return state.provider(request);+ } catch (error) {+ state.metrics.push({ phase: request.phase, elapsedMs: Date.now() - started, usage: null });+ if (error?.code === 'source-drift') throw error;+ return fail('provider-failed');+ } finally { environment.restore(); }+}++function parseProviderOutput(state, raw) {+ const parsed = state.family === 'claude' ? parseClaudeJson(raw?.stdout) : parseCodexJsonl(raw?.stdout);+ const succeeded = raw?.status === 0 && !raw?.error;+ return { parsed, succeeded };+}++function adapterCall(state, cwd, environment, args, options) {+ if (state.calls >= state.maxCalls) fail('call-budget');+ state.assertCurrent();+ environment.verify();+ const remaining = state.deadline - Date.now();+ if (remaining <= 0) fail('deadline');+ const phase = options.phase || (args.includes('read-only') ? 'selection' : 'task');+ state.calls++;+ const started = Date.now();+ // Coding tasks outgrow the launcher's interactive default, so the evaluator's own call bound governs them.+ const timeoutMs = Math.min(phase === 'task' ? state.callTimeoutMs : options.timeout, state.callTimeoutMs, remaining);+ const env = options.env || launchEnvironment(environment.launch);+ const raw = callProvider(state, environment,+ { phase, input: options.input, cwd, env, timeoutMs, maxBuffer: 1024 * 1024 }, started);+ const elapsedMs = Date.now() - started;+ const { parsed, succeeded } = parseProviderOutput(state, raw);+ state.metrics.push({ phase, elapsedMs, usage: parsed.valid && succeeded ? parsed.usage : null });+ if (Date.now() >= state.deadline || elapsedMs > timeoutMs) fail('deadline');+ state.assertCurrent();+ if (!succeeded) fail('provider-failed');+ if (!parsed.valid) fail(parsed.error ? 'provider-failed' : 'invalid-jsonl');+ return { status: 0, stdout: parsed.text };+}+function executeAdapter(state, cwd, environment) {- return (_command, args, options) => {- if (state.calls >= state.maxCalls) fail('call-budget');- state.assertCurrent();- environment.verify();- const remaining = state.deadline - Date.now();- if (remaining <= 0) fail('deadline');- const phase = options.phase || (args.includes('read-only') ? 'selection' : 'task');- state.calls++;- const started = Date.now();- let raw;- // Coding tasks outgrow the launcher's interactive default, so the evaluator's own call bound governs them.- const timeoutMs = Math.min(phase === 'task' ? state.callTimeoutMs : options.timeout, state.callTimeoutMs, remaining);- const env = options.env || launchEnvironment(environment.launch);- try {- raw = state.provider({ phase, input: options.input, cwd, env, timeoutMs, maxBuffer: 1024 * 1024 });- } catch (error) {- state.metrics.push({ phase, elapsedMs: Date.now() - started, usage: null });- if (error?.code === 'source-drift') throw error;- fail('provider-failed');- } finally { environment.restore(); }- const elapsedMs = Date.now() - started;- const parsed = state.family === 'claude' ? parseClaudeJson(raw?.stdout) : parseCodexJsonl(raw?.stdout);- state.metrics.push({ phase, elapsedMs, usage: parsed.valid && raw?.status === 0 && !raw?.error ? parsed.usage : null });- if (Date.now() >= state.deadline || elapsedMs > timeoutMs) fail('deadline');- state.assertCurrent();- if (raw?.status !== 0 || raw?.error) fail('provider-failed');- if (!parsed.valid) fail(parsed.error ? 'provider-failed' : 'invalid-jsonl');- return { status: 0, stdout: parsed.text };- };+ return (_command, args, options) => adapterCall(state, cwd, environment, args, options);}function selectionProbe(item, repoRoot, execute, environment, target) {@@ -661,48 +684,63 @@ function selectionProbe(item, repoRoot, execute, environment, target) {// Full relies on native discovery of the whole install; the Lean arms receive ECC-selected skill bodies;// ecc-legacy runs bare against the pinned pre-scoping skill library; Baseline runs the bare task query.// Stepped tasks run each ticket in the same accumulating workspace, grading after every step.-function outcomeTrial(item, arm, repeat, repoRoot, execute, cwd, environment, target, harvest, metrics = null) {- const launchStep = (query, manualIds) => {- const task = { sessionId: 'ecc-eval', taskId: item.id, revision: 1, phase: 'evaluate', query };- return launchTaskContext({ repoRoot, execute, nativeEnvironment: environment.launch, target,- bare: arm === 'baseline' || arm === 'ecc-legacy',- task: { ...task, ...(arm === 'manual-lean' && manualIds?.length ? { explicitIds: manualIds } : {}) },- profileId: arm === 'full' ? 'full@1' : 'lean@1', selectionMode: arm === 'auto-lean' ? 'auto' : 'manual' });- };- try {- if (!item.steps) {- const result = launchStep(item.query, item.manualIds);- if (harvest) harvest(arm, item.id, repeat, environment);- const verdict = runScoredCheck(cwd, item.check, item.checkTimeoutMs);- const passed = result.status === 'completed' && verdict.passed && verdict.score >= 0.999;- return { id: item.id, arm, repeat, passed, score: result.status === 'completed' ? verdict.score : 0,- selectedIds: result.selection.selectedIds, failure: passed ? null : 'hidden-check' };- }- const steps = [];- const selectedIds = [];- for (let index = 0; index < item.steps.length; index++) {- const step = item.steps[index];- const start = metrics ? metrics.length : 0;- const result = launchStep(step.query, step.manualIds || item.manualIds);- if (harvest) harvest(arm, `${item.id}--step${index + 1}`, repeat, environment);- if (result.status !== 'completed') {- // A failed ticket ends the chain; remaining tickets are unscored.- steps.push({ score: 0, ...(metrics ? metricsSince(metrics, start) : {}) });- for (let rest = index + 1; rest < item.steps.length; rest++) {- steps.push({ score: 0, ...(metrics ? metricsSince(metrics, metrics.length) : {}) });- }- break;+// `trial` carries repoRoot, execute, cwd, environment, target, harvest (or null)+// and metrics (or null) for one task trial.+function launchTrialStep(item, arm, trial, query, manualIds) {+ const task = { sessionId: 'ecc-eval', taskId: item.id, revision: 1, phase: 'evaluate', query };+ return launchTaskContext({ repoRoot: trial.repoRoot, execute: trial.execute,+ nativeEnvironment: trial.environment.launch, target: trial.target,+ bare: arm === 'baseline' || arm === 'ecc-legacy',+ task: { ...task, ...(arm === 'manual-lean' && manualIds?.length ? { explicitIds: manualIds } : {}) },+ profileId: arm === 'full' ? 'full@1' : 'lean@1', selectionMode: arm === 'auto-lean' ? 'auto' : 'manual' });+}++function stepMetrics(metrics, start) {+ return metrics ? metricsSince(metrics, start) : {};+}++function singleStepTrial(item, arm, repeat, trial) {+ const { cwd, environment, harvest } = trial;+ const result = launchTrialStep(item, arm, trial, item.query, item.manualIds);+ if (harvest) harvest(arm, item.id, repeat, environment);+ const verdict = runScoredCheck(cwd, item.check, item.checkTimeoutMs);+ const passed = result.status === 'completed' && verdict.passed && verdict.score >= 0.999;+ return { id: item.id, arm, repeat, passed, score: result.status === 'completed' ? verdict.score : 0,+ selectedIds: result.selection.selectedIds, failure: passed ? null : 'hidden-check' };+}++function steppedTrial(item, arm, repeat, trial) {+ const { cwd, environment, harvest, metrics } = trial;+ const steps = [];+ const selectedIds = [];+ for (let index = 0; index < item.steps.length; index++) {+ const step = item.steps[index];+ const start = metrics ? metrics.length : 0;+ const result = launchTrialStep(item, arm, trial, step.query, step.manualIds || item.manualIds);+ if (harvest) harvest(arm, `${item.id}--step${index + 1}`, repeat, environment);+ if (result.status !== 'completed') {+ // A failed ticket ends the chain; remaining tickets are unscored.+ steps.push({ score: 0, ...stepMetrics(metrics, start) });+ for (let rest = index + 1; rest < item.steps.length; rest++) {+ steps.push({ score: 0, ...stepMetrics(metrics, metrics ? metrics.length : 0) });}- selectedIds.push(...result.selection.selectedIds);- const verdict = runScoredCheck(cwd, step.check, step.checkTimeoutMs, index + 1);- steps.push({ score: verdict.passed ? verdict.score : 0, ...(metrics ? metricsSince(metrics, start) : {}) });+ break;}- const score = steps.reduce((sum, step) => sum + step.score, 0) / item.steps.length;- const passed = steps.length === item.steps.length && steps.every(step => step.score >= 0.999);- return { id: item.id, arm, repeat, passed, score, selectedIds: [...new Set(selectedIds)], steps,- failure: passed ? null : 'hidden-check' };+ selectedIds.push(...result.selection.selectedIds);+ const verdict = runScoredCheck(cwd, step.check, step.checkTimeoutMs, index + 1);+ steps.push({ score: verdict.passed ? verdict.score : 0, ...stepMetrics(metrics, start) });+ }+ const score = steps.reduce((sum, step) => sum + step.score, 0) / item.steps.length;+ const passed = steps.length === item.steps.length && steps.every(step => step.score >= 0.999);+ return { id: item.id, arm, repeat, passed, score, selectedIds: [...new Set(selectedIds)], steps,+ failure: passed ? null : 'hidden-check' };+}++function outcomeTrial(item, arm, repeat, trial) {+ try {+ return item.steps ? steppedTrial(item, arm, repeat, trial) : singleStepTrial(item, arm, repeat, trial);} catch (error) {- if (harvest) harvest(arm, item.id, repeat, environment);+ if (trial.harvest) trial.harvest(arm, item.id, repeat, trial.environment);return { id: item.id, arm, repeat, passed: false, score: 0, selectedIds: [], failure: failureCode(error) };}}@@ -753,10 +791,10 @@ function createHarvester(artifactDir, envs) {};}-function runEvaluation({ repoRoot = DEFAULT_REPO_ROOT, corpus = loadCorpus(), registration,- repeats = 1, provider, family, allowRealProvider = false, allowCredentialedTools = false,- executable, model, effort, authHome, environments,- arms = undefined, artifactDir = null, maxCalls = 300, deadlineMs = 3600000, callTimeoutMs = 300000 } = {}) {+// Validate the caller's bounds and opt-ins and settle on the provider family.+function resolveEvaluationFamily(options) {+ const { provider, allowRealProvider, maxCalls, deadlineMs, callTimeoutMs, registration,+ family, executable, effort, allowCredentialedTools } = options;if (!provider && !allowRealProvider) throw new Error('Evaluation requires an injected provider or explicit opt-in');if (!bounded(maxCalls, 1, 2000) || !bounded(deadlineMs, 1, 8 * 3600000)|| !bounded(callTimeoutMs, 1, 600000)) throw new Error('Invalid call or deadline bound');@@ -766,21 +804,97 @@ function runEvaluation({ repoRoot = DEFAULT_REPO_ROOT, corpus = loadCorpus(), reif (!provider && resolvedFamily === 'claude' && !allowCredentialedTools) {throw new Error('Claude task tools can read provider credentials; explicit credentialed-tool opt-in is required');}+ return resolvedFamily;+}++function pinEvaluation(options, resolvedFamily) {+ const { repoRoot, corpus, repeats, model, executable, effort, arms, provider, registration } = options;const pin = preregister({ repoRoot, corpus, repeats, model, executable, effort, arms });if (!provider && resolvedFamily === 'codex' && pin.arms.includes('ecc-legacy')) {throw new Error('Codex real evaluation requires --arms without ecc-legacy; the pinned legacy skills arm is Claude-only');}if (registration && !isDeepStrictEqual(registration, pin)) throw new Error('Registration pin mismatch');- const injected = Boolean(provider);- const liveProvider = provider || (resolvedFamily === 'claude'+ return pin;+}++function createLiveProvider(resolvedFamily, options) {+ const { allowRealProvider, allowCredentialedTools, executable, model, effort, authHome, artifactDir } = options;+ return resolvedFamily === 'claude'? createClaudeProvider({ allowRealProvider, allowCredentialedTools, executable, model,persistSessions: Boolean(artifactDir) })- : createCodexProvider({ allowRealProvider, executable, model, effort, authHome }));- const state = { calls: 0, metrics: [], maxCalls, callTimeoutMs, family: resolvedFamily,+ : createCodexProvider({ allowRealProvider, executable, model, effort, authHome });+}++function createEvaluationState(options, plan) {+ const { repoRoot, corpus, maxCalls, callTimeoutMs, deadlineMs } = options;+ const { pin, resolvedFamily, liveProvider } = plan;+ return { calls: 0, metrics: [], maxCalls, callTimeoutMs, family: resolvedFamily,deadline: Date.now() + deadlineMs, provider: liveProvider,assertCurrent() {if (digestObject(corpus) !== pin.corpusDigest || sourceSnapshot(repoRoot).sourceDigest !== pin.sourceDigest) fail('source-drift');} };+}++function planEvaluation(options) {+ const resolvedFamily = resolveEvaluationFamily(options);+ const pin = pinEvaluation(options, resolvedFamily);+ const injected = Boolean(options.provider);+ const liveProvider = options.provider || createLiveProvider(resolvedFamily, options);+ const plan = { resolvedFamily, pin, injected, liveProvider };+ plan.state = createEvaluationState(options, plan);+ return plan;+}++function prepareEvaluationEnvironments(options, plan, installRoot) {+ const { repoRoot, executable } = options;+ const { injected, resolvedFamily, pin } = plan;+ if (injected) return syntheticEnvironments(installRoot);+ if (resolvedFamily !== 'claude') return prepareEnvironments({ repoRoot, executable, root: installRoot });+ return prepareClaudeEnvironments({ repoRoot, executable, root: installRoot,+ ...(pin.arms.includes('ecc-legacy')+ ? { legacySource: exportLegacySource({ repoRoot, destination: path.join(installRoot, 'legacy-source') }) }+ : {}) });+}++function describeInstalls(envs) {+ return Object.fromEntries(Object.entries(envs).map(([name, env]) => [name,+ { profileId: env.profileId, skills: env.skills, ...(env.sourceSha ? { sourceSha: env.sourceSha } : {}) }]));+}++function runSelectionProbes(run, selection) {+ const { corpus, repoRoot, state, temp, envs, resolvedFamily } = run;+ for (const item of corpus.selection) {+ const cwd = path.join(temp, `${item.id}--selection`);+ fs.mkdirSync(cwd);+ const start = state.metrics.length;+ selection.push({ ...selectionProbe(item, repoRoot, executeAdapter(state, cwd, envs.lean), envs.lean, resolvedFamily),+ ...metricsSince(state.metrics, start) });+ }+}++function runOutcomeTrials(run, outcomes) {+ const { corpus, pin, repoRoot, state, temp, envs, resolvedFamily, harvest } = run;+ for (const scheduled of pin.order) {+ const item = corpus.tasks.find(c => c.id === scheduled.id);+ for (const arm of scheduled.arms) {+ const cwd = path.join(temp, `${item.id}--${arm}--${scheduled.repeat}`);+ const environment = envs[['full', 'baseline', 'ecc-legacy'].includes(arm) ? arm : 'lean'];+ fs.mkdirSync(cwd);+ writeWorkspace(cwd, item.files);+ const start = state.metrics.length;+ const trial = { repoRoot, execute: executeAdapter(state, cwd, environment), cwd, environment,+ target: resolvedFamily, harvest, metrics: state.metrics };+ outcomes.push({ ...outcomeTrial(item, arm, scheduled.repeat, trial), ...metricsSince(state.metrics, start) });+ fs.rmSync(cwd, { recursive: true, force: true });+ }+ }+}++// Prepare installs in a private temp root, run every selection probe and task+// trial, and always remove the root afterwards.+function executeEvaluation(options, plan) {+ const { repoRoot, corpus, environments, artifactDir } = options;+ const { resolvedFamily, pin, injected, state } = plan;const temp = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'ecc-ai-eval-')));const selection = [];const outcomes = [];@@ -789,52 +903,38 @@ function runEvaluation({ repoRoot = DEFAULT_REPO_ROOT, corpus = loadCorpus(), retry {const installRoot = path.join(temp, 'installs');fs.mkdirSync(installRoot, { mode: 0o700 });- const envs = environments || (injected ? syntheticEnvironments(installRoot)- : resolvedFamily === 'claude'- ? prepareClaudeEnvironments({ repoRoot, executable, root: installRoot,- ...(pin.arms.includes('ecc-legacy')- ? { legacySource: exportLegacySource({ repoRoot, destination: path.join(installRoot, 'legacy-source') }) }- : {}) })- : prepareEnvironments({ repoRoot, executable, root: installRoot }));- installs = Object.fromEntries(Object.entries(envs).map(([name, env]) => [name,- { profileId: env.profileId, skills: env.skills, ...(env.sourceSha ? { sourceSha: env.sourceSha } : {}) }]));+ const envs = environments || prepareEvaluationEnvironments(options, plan, installRoot);+ installs = describeInstalls(envs);harvester = artifactDir && resolvedFamily === 'claude' && !injected ? createHarvester(artifactDir, envs) : null;const harvest = harvester ? (arm, id, repeat, env) => harvester.record(arm, id, repeat, env) : null;- for (const item of corpus.selection) {- const cwd = path.join(temp, `${item.id}--selection`);- fs.mkdirSync(cwd);- const start = state.metrics.length;- selection.push({ ...selectionProbe(item, repoRoot, executeAdapter(state, cwd, envs.lean), envs.lean, resolvedFamily),- ...metricsSince(state.metrics, start) });- }- for (const scheduled of pin.order) {- const item = corpus.tasks.find(c => c.id === scheduled.id);- for (const arm of scheduled.arms) {- const cwd = path.join(temp, `${item.id}--${arm}--${scheduled.repeat}`);- const environment = envs[['full', 'baseline', 'ecc-legacy'].includes(arm) ? arm : 'lean'];- fs.mkdirSync(cwd);- writeWorkspace(cwd, item.files);- const start = state.metrics.length;- outcomes.push({ ...outcomeTrial(item, arm, scheduled.repeat, repoRoot,- executeAdapter(state, cwd, environment), cwd, environment, resolvedFamily, harvest, state.metrics),- ...metricsSince(state.metrics, start) });- fs.rmSync(cwd, { recursive: true, force: true });- }- }+ const run = { corpus, pin, repoRoot, state, temp, envs, resolvedFamily, harvest };+ runSelectionProbes(run, selection);+ runOutcomeTrials(run, outcomes);if (harvester) harvester.writeIndex();} finally { if (harvester) harvester.writeIndex(); fs.rmSync(temp, { recursive: true, force: true }); }+ return { selection, outcomes, installs, harvester };+}++function summarizeSelection(selection) {+ const selectionSuccesses = selection.filter(row => row.passed).length;+ return { n: selection.length, successes: selectionSuccesses,+ categories: [...new Set(selection.map(row => row.category))].map(category => ({ category,+ n: selection.filter(row => row.category === category).length,+ successes: selection.filter(row => row.category === category && row.passed).length })),+ interval: wilson(selectionSuccesses, selection.length), method: 'wilson-95-descriptive-purposive-sample' };+}++function buildEvaluationReport(options, plan, run) {+ const { maxCalls, deadlineMs, callTimeoutMs } = options;+ const { pin, injected, liveProvider, state, resolvedFamily } = plan;+ const { selection, outcomes, installs, harvester } = run;const summary = summarize(outcomes, pin.arms);const insufficient = summary.distinctTasks < pin.minimumDistinctTasks || selection.length < pin.minimumDistinctTasks;- const selectionSuccesses = selection.filter(row => row.passed).length;return { schemaVersion: 'ecc.context-eval.v2', registration: pin,evidence: injected ? 'injected-provider' : resolvedFamily === 'claude' ? 'claude-json' : 'codex-jsonl', installs,authentication: injected ? 'injected' : liveProvider.authentication, credentialsRetained: false,calls: state.calls, bounds: { maxCalls, deadlineMs, callTimeoutMs }, selection, outcomes, summary,- selectionSummary: { n: selection.length, successes: selectionSuccesses,- categories: [...new Set(selection.map(row => row.category))].map(category => ({ category,- n: selection.filter(row => row.category === category).length,- successes: selection.filter(row => row.category === category && row.passed).length })),- interval: wilson(selectionSuccesses, selection.length), method: 'wilson-95-descriptive-purposive-sample' },+ selectionSummary: summarizeSelection(selection),gate: { status: insufficient ? 'insufficient-sample' : injected ? 'synthetic-only' : 'review-required',nonInferioritySupported: !insufficient && !injected && summary.pairs.every(p => p.interval[0] >= -pin.nonInferiorityMargin),releaseApproved: false }, nativeInvocation: 'unobserved',@@ -842,6 +942,18 @@ function runEvaluation({ repoRoot = DEFAULT_REPO_ROOT, corpus = loadCorpus(), reartifactRetention: harvester ? 'session-jsonl-per-task-trial' : 'none', ...metricsSince(state.metrics, 0) };}+function runEvaluation({ repoRoot = DEFAULT_REPO_ROOT, corpus = loadCorpus(), registration,+ repeats = 1, provider, family, allowRealProvider = false, allowCredentialedTools = false,+ executable, model, effort, authHome, environments,+ arms = undefined, artifactDir = null, maxCalls = 300, deadlineMs = 3600000, callTimeoutMs = 300000 } = {}) {+ const options = { repoRoot, corpus, registration, repeats, provider, family, allowRealProvider,+ allowCredentialedTools, executable, model, effort, authHome, environments, arms, artifactDir,+ maxCalls, deadlineMs, callTimeoutMs };+ const plan = planEvaluation(options);+ const run = executeEvaluation(options, plan);+ return buildEvaluationReport(options, plan, run);+}+module.exports = { loadCorpus, preregister, runEvaluation, parseCodexJsonl, parseClaudeJson, summarize, wilson,runCheck, runScoredCheck, createAuthLease, createCodexProvider, createClaudeProvider, prepareEnvironments,prepareClaudeEnvironments, exportLegacySource, providerFamily, resolveFamily, readClaudeKeychainToken };