Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(scripts): block measurement when git status cannot be read
Alfredo Lopezb675d459fd3e
diff --git a/docs/testing/local-measurements.md b/docs/testing/local-measurements.mdindex 393465ca2..56b5a6236 100644--- a/docs/testing/local-measurements.md+++ b/docs/testing/local-measurements.md@@ -8,8 +8,12 @@ bash scripts/dev/repo-intake-packet.sh --measure -- <benchmark-command> [argumenThe gate records full HEAD and base commits, dirty-file count and ancestry.It refuses to start the command when `origin/dev` is missing, has commits-absent from HEAD, HEAD is detached, or merge conflicts exist. Candidate edits-are allowed and reported. The benchmark's exit status is preserved.+absent from HEAD, HEAD is detached, merge conflicts exist, or `git status`+cannot be read. Candidate edits are allowed and reported. A blocked gate exits+2 and prints `measurement_readiness: blocked` with its `blockers`; otherwise it+prints `measurement_readiness: base_check_passed_not_a_lane_verdict` and+replaces itself with the command, so the command's exit status is returned+unchanged.This uses the locally recorded `origin/dev`; it does not fetch, update abranch, inspect a cluster, or certify T0/T1/T2. Refresh the remote referencethrough the normal repository workflow before measuring against current dev.@@ -23,20 +27,21 @@ bash scripts/dev/repo-intake-packet.sh --measure --historical-base <commit> -- <That base must exist and be an ancestor of HEAD. Detached HEAD is allowed inthis mode; output is marked `historical_non_certifying`. This option does not-relax the normal T2 intake or certify runtime evidence. Use the gate as the-entry command of a future approved AutoResearch benchmark; calling a benchmark-directly bypasses it. No global runner or scheduled automation is changed.+relax the normal T2 intake or certify runtime evidence. Start benchmarks+through this gate; calling one directly skips the base check.`make test-service-matrix` compares the expanded `TEST_SERVICES` list with the-literal service matrix in CI. It rejects omissions, extra entries, duplicates-and unsupported matrix shapes before `make test-unit-all` executes packages.-CI remains unchanged. Each package keeps its existing `npm test` semantics;-real PostgreSQL tests remain opt-in and are not T1 evidence in this unit lane.+literal service matrix of the `test` job in `.github/workflows/ci-public.yml`+(`strategy.matrix.service`). It rejects omissions, extra entries, duplicates+and unsupported matrix shapes, and runs before `make test-unit-all` executes+packages. Each package keeps its `npm test` semantics; real PostgreSQL tests+remain opt-in and are not T1 evidence in this unit lane.-Local contract checks (no cluster required):+CI runs these contract checks in the `shell-syntax` job; locally (no cluster+required):```bash-node --test scripts/tests/test-measurement-base.cjs scripts/tests/test-test-services.cjs+node scripts/tests/run-node-test-files.mjs scripts/tests/test-test-services.cjs scripts/tests/test-measurement-base.cjsbash scripts/tests/test-repo-intake-packet.shmake test-service-matrix```diff --git a/scripts/dev/check-test-services.cjs b/scripts/dev/check-test-services.cjsindex fd4ce48bf..1d8bf7d04 100644--- a/scripts/dev/check-test-services.cjs+++ b/scripts/dev/check-test-services.cjs@@ -5,7 +5,9 @@ const fs = require('node:fs')const path = require('node:path')// The CI contract is a literal service list. Refuse expressions or a changed-// matrix shape instead of silently checking a partial list.+// matrix shape instead of silently checking a partial list. The parser relies+// on the current 6/8/10-space indentation and on `service:` being the matrix's+// only key; changing either makes the check fail rather than pass.function checkServices(workflow, localServices) {const matrices = [...(workflow + '\n').matchAll(/^ matrix:\n((?: {8}.*\n|\n)*)/gm)]const lists = matrices.map((match) => match[1].split('\n').filter((line) => line.trim() && !line.trim().startsWith('#')))diff --git a/scripts/dev/repo-intake-packet.sh b/scripts/dev/repo-intake-packet.shindex 1f8cbed42..a9c181afc 100755--- a/scripts/dev/repo-intake-packet.sh+++ b/scripts/dev/repo-intake-packet.sh@@ -105,12 +105,19 @@ collect_touched_areas() {}count_status() {- local line xy x y+ local line xy x y status_outSTATUS_TOTAL=0STATUS_STAGED=0STATUS_UNSTAGED=0STATUS_UNTRACKED=0STATUS_CONFLICTS=0+ # Zero counts from an unreadable status would look like a clean tree; the+ # measurement gate blocks on STATUS_READABLE=false.+ STATUS_READABLE=true+ if ! status_out="$(git status --porcelain=v1 -uall 2>/dev/null)"; then+ STATUS_READABLE=false+ return 0+ fiwhile IFS= read -r line; do[[ -n "${line}" ]] || continue@@ -128,7 +135,7 @@ count_status() {case "${xy}" inDD|AU|UD|UA|DU|AA|UU) STATUS_CONFLICTS=$((STATUS_CONFLICTS + 1)) ;;esac- done < <(git status --porcelain=v1 -uall 2>/dev/null || true)+ done <<< "${status_out}"}resolve_primary_checkout() {@@ -209,6 +216,7 @@ fiif [[ "${MEASURE}" == true ]]; then[[ "${DETACHED}" == no || "${HISTORICAL}" == true ]] || BLOCKERS+=("detached_measurement")count_status+ [[ "${STATUS_READABLE}" == true ]] || BLOCKERS+=("status_unreadable")(( STATUS_CONFLICTS == 0 )) || BLOCKERS+=("unresolved_conflicts")kv "measurement_head" "${HEAD_FULL}"kv "measurement_base_ref" "${BASE_REF}"diff --git a/scripts/tests/test-measurement-base.cjs b/scripts/tests/test-measurement-base.cjsindex a356f83ba..6eaab90f7 100644--- a/scripts/tests/test-measurement-base.cjs+++ b/scripts/tests/test-measurement-base.cjs@@ -8,12 +8,16 @@ const path = require('node:path')const { spawnSync, execFileSync } = require('node:child_process')const root = path.resolve(__dirname, '../..')const script = path.join(root, 'scripts/dev/repo-intake-packet.sh')+// An inherited GIT_DIR or GIT_INDEX_FILE (a git hook, for example) would point+// the fixture commands at the host repository.+const env = Object.fromEntries(Object.entries(process.env).filter(([key]) => !key.startsWith('GIT_')))test('measurement gate enforces ancestry before executing the benchmark', () => {- const git = (cwd, ...args) => execFileSync('git', args, { cwd, encoding: 'utf8' }).trim()+ const git = (cwd, ...args) => execFileSync('git', args, { cwd, encoding: 'utf8', env }).trim()const hostState = () => ['rev-parse HEAD', 'branch --show-current', 'status --porcelain=v1'].map((args) => git(root, ...args.split(' ')))const before = hostState()const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'evenfire-measurement-'))+ let failuretry {git(tmp, 'init', '-q')git(tmp, 'config', 'user.email', '[email protected]')@@ -22,43 +26,80 @@ test('measurement gate enforces ancestry before executing the benchmark', () =>git(tmp, 'branch', '-M', 'test/measurement')const base = git(tmp, 'rev-parse', 'HEAD')git(tmp, 'update-ref', 'refs/remotes/origin/dev', base)- const run = (options = [], code = 'console.log("BENCHMARK_EXECUTED")') => spawnSync('bash', [script, '--measure', ...options, '--', process.execPath, '-e', code], {- cwd: tmp, encoding: 'utf8', env: { ...process.env, REPO_INTAKE_BASE_REF: 'HEAD' },+ const spawn = (args) => spawnSync('bash', [script, ...args], {+ cwd: tmp, encoding: 'utf8', env: { ...env, REPO_INTAKE_BASE_REF: 'HEAD' },})+ const run = (options = [], code = 'console.log("BENCHMARK_EXECUTED")') =>+ spawn(['--measure', ...options, '--', process.execPath, '-e', code])const allowed = (options, label) => {const result = run(options)assert.equal(result.status, 0, result.stderr + result.stdout)assert.match(result.stdout, /BENCHMARK_EXECUTED/, label)return result}- const blocked = (label) => {+ // Each blocked case names its blocker, so a gate that blocks for the wrong+ // reason fails here.+ const blocked = (blocker, label) => {const result = run()assert.equal(result.status, 2, label + result.stderr + result.stdout)+ assert.match(result.stdout, /measurement_readiness:\s+blocked/, label)+ assert.match(result.stdout, new RegExp(`blockers:\\s+.*${blocker}`), label + result.stdout)assert.doesNotMatch(result.stdout, /BENCHMARK_EXECUTED/, label)}- allowed([], 'identical base')+ const normal = allowed([], 'identical base')+ assert.match(normal.stdout, /measurement_readiness:\s+base_check_passed_not_a_lane_verdict/)assert.equal(run([], 'process.exit(7)').status, 7, 'benchmark failure preserved')git(tmp, 'commit', '--allow-empty', '-qm', 'candidate')allowed([], 'ahead of base')const candidate = git(tmp, 'rev-parse', 'HEAD')git(tmp, 'update-ref', 'refs/remotes/origin/dev', candidate)git(tmp, 'checkout', '-q', '-B', 'test/historical', base)- blocked('behind base, environment override ignored')+ blocked('branch_missing_origin/dev_commits', 'behind base, environment override ignored')const historical = allowed(['--historical-base', base], 'explicit historical comparison')assert.match(historical.stdout, /historical_non_certifying/)assert.match(historical.stdout, new RegExp(base))git(tmp, 'commit', '--allow-empty', '-qm', 'divergent')- blocked('divergent base')+ blocked('branch_missing_origin/dev_commits', 'divergent base')git(tmp, 'update-ref', '-d', 'refs/remotes/origin/dev')- blocked('missing base')+ blocked('missing_origin/dev', 'missing base')git(tmp, 'update-ref', 'refs/remotes/origin/dev', base)git(tmp, 'checkout', '-q', '--detach', base)- blocked('detached default')+ blocked('detached_measurement', 'detached default')allowed(['--historical-base', base], 'detached historical')assert.equal(run(['--historical-base', 'missing-ref']).status, 2)assert.equal(run(['--historical-base']).status, 2)++ const unknown = spawn(['--unexpected'])+ assert.equal(unknown.status, 2, unknown.stderr + unknown.stdout)+ assert.match(unknown.stderr, /unknown repo intake arguments/)++ git(tmp, 'checkout', '-q', '-B', 'test/conflict', base)+ fs.writeFileSync(path.join(tmp, 'f.txt'), 'base\n')+ git(tmp, 'add', 'f.txt')+ git(tmp, 'commit', '-qm', 'file')+ git(tmp, 'update-ref', 'refs/remotes/origin/dev', git(tmp, 'rev-parse', 'HEAD'))+ git(tmp, 'checkout', '-q', '-b', 'test/other')+ fs.writeFileSync(path.join(tmp, 'f.txt'), 'other\n')+ git(tmp, 'commit', '-qam', 'other')+ git(tmp, 'checkout', '-q', 'test/conflict')+ fs.writeFileSync(path.join(tmp, 'f.txt'), 'mine\n')+ git(tmp, 'commit', '-qam', 'mine')+ const merge = spawnSync('git', ['merge', '-q', 'test/other'], { cwd: tmp, encoding: 'utf8', env })+ assert.notEqual(merge.status, 0, 'fixture merge must conflict')+ assert.match(git(tmp, 'status', '--porcelain=v1'), /^UU f\.txt$/m)+ blocked('unresolved_conflicts', 'conflicted index')+ git(tmp, 'merge', '--abort')++ fs.writeFileSync(path.join(tmp, '.git', 'index'), 'not an index')+ assert.notEqual(spawnSync('git', ['status'], { cwd: tmp, env }).status, 0, 'fixture index must be unreadable')+ blocked('status_unreadable', 'unreadable status')+ } catch (error) {+ failure = error} finally {fs.rmSync(tmp, { recursive: true, force: true })- assert.deepEqual(hostState(), before, 'host checkout remains unchanged')}+ // Checked after the body so a host-state difference never replaces the+ // body's own error.+ if (failure) throw failure+ assert.deepEqual(hostState(), before, 'host checkout remains unchanged')})diff --git a/scripts/tests/test-repo-intake-packet.sh b/scripts/tests/test-repo-intake-packet.shindex 6883268b7..eed3fa56c 100755--- a/scripts/tests/test-repo-intake-packet.sh+++ b/scripts/tests/test-repo-intake-packet.sh@@ -27,7 +27,11 @@ cleanup() {}trap cleanup EXIT-pass() { echo "PASS: $1"; }+PASSED=0+pass() {+ echo "PASS: $1"+ PASSED=$((PASSED + 1))+}fail() {echo "FAIL: $1"@@ -276,4 +280,8 @@ assert_contains "${DETACHED_OUTPUT}" "profile_helper_primary:" "detached outputassert_contains "${DETACHED_OUTPUT}" "yes" "detached output finds primary .local-notes helper"assert_not_contains "${DETACHED_OUTPUT}" "unbound variable" "detached output has no shell unbound-variable warnings"+# A run that skipped its assertions must not exit 0.+if (( PASSED < 44 )); then+ fail "expected at least 44 passing assertions, got ${PASSED}"+fiexit "${FAIL}"diff --git a/scripts/tests/test-test-services.cjs b/scripts/tests/test-test-services.cjsindex 5fc74725d..89d246b43 100644--- a/scripts/tests/test-test-services.cjs+++ b/scripts/tests/test-test-services.cjs@@ -21,3 +21,8 @@ test('dynamic, missing and ambiguous matrices fail closed', () => {assert.throws(() => checkServices(workflow + workflow, []), /Expected one/)assert.throws(() => checkServices(workflow.replace('\n steps:', ' exclude:\n - service: one\n steps:'), ['one', 'packages/two']), /Unsupported/)})+test('empty, inline and re-indented matrices fail closed', () => {+ assert.throws(() => checkServices(' matrix:\n service:\n\n steps:\n', []), /Expected one/)+ assert.throws(() => checkServices(' matrix:\n service: [one, packages/two]\n steps:\n', ['one', 'packages/two']), /Expected one/)+ assert.throws(() => checkServices(workflow.replace(/^ {6}/gm, ' '), ['one', 'packages/two']), /Expected one/)+})