Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(scripts): block measurement when git status cannot be read

Alfredo Lopezb675d459fd3e
diff --git a/docs/testing/local-measurements.md b/docs/testing/local-measurements.md
index 393465ca2..56b5a6236 100644
--- a/docs/testing/local-measurements.md
+++ b/docs/testing/local-measurements.md
@@ -8,8 +8,12 @@ bash scripts/dev/repo-intake-packet.sh --measure -- <benchmark-command> [argumen
The gate records full HEAD and base commits, dirty-file count and ancestry.
It refuses to start the command when `origin/dev` is missing, has commits
-absent from HEAD, HEAD is detached, or merge conflicts exist. Candidate edits
-are allowed and reported. The benchmark's exit status is preserved.
+absent from HEAD, HEAD is detached, merge conflicts exist, or `git status`
+cannot be read. Candidate edits are allowed and reported. A blocked gate exits
+2 and prints `measurement_readiness: blocked` with its `blockers`; otherwise it
+prints `measurement_readiness: base_check_passed_not_a_lane_verdict` and
+replaces itself with the command, so the command's exit status is returned
+unchanged.
This uses the locally recorded `origin/dev`; it does not fetch, update a
branch, inspect a cluster, or certify T0/T1/T2. Refresh the remote reference
through the normal repository workflow before measuring against current dev.
@@ -23,20 +27,21 @@ bash scripts/dev/repo-intake-packet.sh --measure --historical-base <commit> -- <
That base must exist and be an ancestor of HEAD. Detached HEAD is allowed in
this mode; output is marked `historical_non_certifying`. This option does not
-relax the normal T2 intake or certify runtime evidence. Use the gate as the
-entry command of a future approved AutoResearch benchmark; calling a benchmark
-directly bypasses it. No global runner or scheduled automation is changed.
+relax the normal T2 intake or certify runtime evidence. Start benchmarks
+through this gate; calling one directly skips the base check.
`make test-service-matrix` compares the expanded `TEST_SERVICES` list with the
-literal service matrix in CI. It rejects omissions, extra entries, duplicates
-and unsupported matrix shapes before `make test-unit-all` executes packages.
-CI remains unchanged. Each package keeps its existing `npm test` semantics;
-real PostgreSQL tests remain opt-in and are not T1 evidence in this unit lane.
+literal service matrix of the `test` job in `.github/workflows/ci-public.yml`
+(`strategy.matrix.service`). It rejects omissions, extra entries, duplicates
+and unsupported matrix shapes, and runs before `make test-unit-all` executes
+packages. Each package keeps its `npm test` semantics; real PostgreSQL tests
+remain opt-in and are not T1 evidence in this unit lane.
-Local contract checks (no cluster required):
+CI runs these contract checks in the `shell-syntax` job; locally (no cluster
+required):
```bash
-node --test scripts/tests/test-measurement-base.cjs scripts/tests/test-test-services.cjs
+node scripts/tests/run-node-test-files.mjs scripts/tests/test-test-services.cjs scripts/tests/test-measurement-base.cjs
bash scripts/tests/test-repo-intake-packet.sh
make test-service-matrix
```
diff --git a/scripts/dev/check-test-services.cjs b/scripts/dev/check-test-services.cjs
index fd4ce48bf..1d8bf7d04 100644
--- a/scripts/dev/check-test-services.cjs
+++ b/scripts/dev/check-test-services.cjs
@@ -5,7 +5,9 @@ const fs = require('node:fs')
const path = require('node:path')
// The CI contract is a literal service list. Refuse expressions or a changed
-// matrix shape instead of silently checking a partial list.
+// matrix shape instead of silently checking a partial list. The parser relies
+// on the current 6/8/10-space indentation and on `service:` being the matrix's
+// only key; changing either makes the check fail rather than pass.
function checkServices(workflow, localServices) {
const matrices = [...(workflow + '\n').matchAll(/^ matrix:\n((?: {8}.*\n|\n)*)/gm)]
const lists = matrices.map((match) => match[1].split('\n').filter((line) => line.trim() && !line.trim().startsWith('#')))
diff --git a/scripts/dev/repo-intake-packet.sh b/scripts/dev/repo-intake-packet.sh
index 1f8cbed42..a9c181afc 100755
--- a/scripts/dev/repo-intake-packet.sh
+++ b/scripts/dev/repo-intake-packet.sh
@@ -105,12 +105,19 @@ collect_touched_areas() {
}
count_status() {
- local line xy x y
+ local line xy x y status_out
STATUS_TOTAL=0
STATUS_STAGED=0
STATUS_UNSTAGED=0
STATUS_UNTRACKED=0
STATUS_CONFLICTS=0
+ # Zero counts from an unreadable status would look like a clean tree; the
+ # measurement gate blocks on STATUS_READABLE=false.
+ STATUS_READABLE=true
+ if ! status_out="$(git status --porcelain=v1 -uall 2>/dev/null)"; then
+ STATUS_READABLE=false
+ return 0
+ fi
while IFS= read -r line; do
[[ -n "${line}" ]] || continue
@@ -128,7 +135,7 @@ count_status() {
case "${xy}" in
DD|AU|UD|UA|DU|AA|UU) STATUS_CONFLICTS=$((STATUS_CONFLICTS + 1)) ;;
esac
- done < <(git status --porcelain=v1 -uall 2>/dev/null || true)
+ done <<< "${status_out}"
}
resolve_primary_checkout() {
@@ -209,6 +216,7 @@ fi
if [[ "${MEASURE}" == true ]]; then
[[ "${DETACHED}" == no || "${HISTORICAL}" == true ]] || BLOCKERS+=("detached_measurement")
count_status
+ [[ "${STATUS_READABLE}" == true ]] || BLOCKERS+=("status_unreadable")
(( STATUS_CONFLICTS == 0 )) || BLOCKERS+=("unresolved_conflicts")
kv "measurement_head" "${HEAD_FULL}"
kv "measurement_base_ref" "${BASE_REF}"
diff --git a/scripts/tests/test-measurement-base.cjs b/scripts/tests/test-measurement-base.cjs
index a356f83ba..6eaab90f7 100644
--- a/scripts/tests/test-measurement-base.cjs
+++ b/scripts/tests/test-measurement-base.cjs
@@ -8,12 +8,16 @@ const path = require('node:path')
const { spawnSync, execFileSync } = require('node:child_process')
const root = path.resolve(__dirname, '../..')
const script = path.join(root, 'scripts/dev/repo-intake-packet.sh')
+// An inherited GIT_DIR or GIT_INDEX_FILE (a git hook, for example) would point
+// the fixture commands at the host repository.
+const env = Object.fromEntries(Object.entries(process.env).filter(([key]) => !key.startsWith('GIT_')))
test('measurement gate enforces ancestry before executing the benchmark', () => {
- const git = (cwd, ...args) => execFileSync('git', args, { cwd, encoding: 'utf8' }).trim()
+ const git = (cwd, ...args) => execFileSync('git', args, { cwd, encoding: 'utf8', env }).trim()
const hostState = () => ['rev-parse HEAD', 'branch --show-current', 'status --porcelain=v1'].map((args) => git(root, ...args.split(' ')))
const before = hostState()
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'evenfire-measurement-'))
+ let failure
try {
git(tmp, 'init', '-q')
git(tmp, 'config', 'user.email', '[email protected]')
@@ -22,43 +26,80 @@ test('measurement gate enforces ancestry before executing the benchmark', () =>
git(tmp, 'branch', '-M', 'test/measurement')
const base = git(tmp, 'rev-parse', 'HEAD')
git(tmp, 'update-ref', 'refs/remotes/origin/dev', base)
- const run = (options = [], code = 'console.log("BENCHMARK_EXECUTED")') => spawnSync('bash', [script, '--measure', ...options, '--', process.execPath, '-e', code], {
- cwd: tmp, encoding: 'utf8', env: { ...process.env, REPO_INTAKE_BASE_REF: 'HEAD' },
+ const spawn = (args) => spawnSync('bash', [script, ...args], {
+ cwd: tmp, encoding: 'utf8', env: { ...env, REPO_INTAKE_BASE_REF: 'HEAD' },
})
+ const run = (options = [], code = 'console.log("BENCHMARK_EXECUTED")') =>
+ spawn(['--measure', ...options, '--', process.execPath, '-e', code])
const allowed = (options, label) => {
const result = run(options)
assert.equal(result.status, 0, result.stderr + result.stdout)
assert.match(result.stdout, /BENCHMARK_EXECUTED/, label)
return result
}
- const blocked = (label) => {
+ // Each blocked case names its blocker, so a gate that blocks for the wrong
+ // reason fails here.
+ const blocked = (blocker, label) => {
const result = run()
assert.equal(result.status, 2, label + result.stderr + result.stdout)
+ assert.match(result.stdout, /measurement_readiness:\s+blocked/, label)
+ assert.match(result.stdout, new RegExp(`blockers:\\s+.*${blocker}`), label + result.stdout)
assert.doesNotMatch(result.stdout, /BENCHMARK_EXECUTED/, label)
}
- allowed([], 'identical base')
+ const normal = allowed([], 'identical base')
+ assert.match(normal.stdout, /measurement_readiness:\s+base_check_passed_not_a_lane_verdict/)
assert.equal(run([], 'process.exit(7)').status, 7, 'benchmark failure preserved')
git(tmp, 'commit', '--allow-empty', '-qm', 'candidate')
allowed([], 'ahead of base')
const candidate = git(tmp, 'rev-parse', 'HEAD')
git(tmp, 'update-ref', 'refs/remotes/origin/dev', candidate)
git(tmp, 'checkout', '-q', '-B', 'test/historical', base)
- blocked('behind base, environment override ignored')
+ blocked('branch_missing_origin/dev_commits', 'behind base, environment override ignored')
const historical = allowed(['--historical-base', base], 'explicit historical comparison')
assert.match(historical.stdout, /historical_non_certifying/)
assert.match(historical.stdout, new RegExp(base))
git(tmp, 'commit', '--allow-empty', '-qm', 'divergent')
- blocked('divergent base')
+ blocked('branch_missing_origin/dev_commits', 'divergent base')
git(tmp, 'update-ref', '-d', 'refs/remotes/origin/dev')
- blocked('missing base')
+ blocked('missing_origin/dev', 'missing base')
git(tmp, 'update-ref', 'refs/remotes/origin/dev', base)
git(tmp, 'checkout', '-q', '--detach', base)
- blocked('detached default')
+ blocked('detached_measurement', 'detached default')
allowed(['--historical-base', base], 'detached historical')
assert.equal(run(['--historical-base', 'missing-ref']).status, 2)
assert.equal(run(['--historical-base']).status, 2)
+
+ const unknown = spawn(['--unexpected'])
+ assert.equal(unknown.status, 2, unknown.stderr + unknown.stdout)
+ assert.match(unknown.stderr, /unknown repo intake arguments/)
+
+ git(tmp, 'checkout', '-q', '-B', 'test/conflict', base)
+ fs.writeFileSync(path.join(tmp, 'f.txt'), 'base\n')
+ git(tmp, 'add', 'f.txt')
+ git(tmp, 'commit', '-qm', 'file')
+ git(tmp, 'update-ref', 'refs/remotes/origin/dev', git(tmp, 'rev-parse', 'HEAD'))
+ git(tmp, 'checkout', '-q', '-b', 'test/other')
+ fs.writeFileSync(path.join(tmp, 'f.txt'), 'other\n')
+ git(tmp, 'commit', '-qam', 'other')
+ git(tmp, 'checkout', '-q', 'test/conflict')
+ fs.writeFileSync(path.join(tmp, 'f.txt'), 'mine\n')
+ git(tmp, 'commit', '-qam', 'mine')
+ const merge = spawnSync('git', ['merge', '-q', 'test/other'], { cwd: tmp, encoding: 'utf8', env })
+ assert.notEqual(merge.status, 0, 'fixture merge must conflict')
+ assert.match(git(tmp, 'status', '--porcelain=v1'), /^UU f\.txt$/m)
+ blocked('unresolved_conflicts', 'conflicted index')
+ git(tmp, 'merge', '--abort')
+
+ fs.writeFileSync(path.join(tmp, '.git', 'index'), 'not an index')
+ assert.notEqual(spawnSync('git', ['status'], { cwd: tmp, env }).status, 0, 'fixture index must be unreadable')
+ blocked('status_unreadable', 'unreadable status')
+ } catch (error) {
+ failure = error
} finally {
fs.rmSync(tmp, { recursive: true, force: true })
- assert.deepEqual(hostState(), before, 'host checkout remains unchanged')
}
+ // Checked after the body so a host-state difference never replaces the
+ // body's own error.
+ if (failure) throw failure
+ assert.deepEqual(hostState(), before, 'host checkout remains unchanged')
})
diff --git a/scripts/tests/test-repo-intake-packet.sh b/scripts/tests/test-repo-intake-packet.sh
index 6883268b7..eed3fa56c 100755
--- a/scripts/tests/test-repo-intake-packet.sh
+++ b/scripts/tests/test-repo-intake-packet.sh
@@ -27,7 +27,11 @@ cleanup() {
}
trap cleanup EXIT
-pass() { echo "PASS: $1"; }
+PASSED=0
+pass() {
+ echo "PASS: $1"
+ PASSED=$((PASSED + 1))
+}
fail() {
echo "FAIL: $1"
@@ -276,4 +280,8 @@ assert_contains "${DETACHED_OUTPUT}" "profile_helper_primary:" "detached output
assert_contains "${DETACHED_OUTPUT}" "yes" "detached output finds primary .local-notes helper"
assert_not_contains "${DETACHED_OUTPUT}" "unbound variable" "detached output has no shell unbound-variable warnings"
+# A run that skipped its assertions must not exit 0.
+if (( PASSED < 44 )); then
+ fail "expected at least 44 passing assertions, got ${PASSED}"
+fi
exit "${FAIL}"
diff --git a/scripts/tests/test-test-services.cjs b/scripts/tests/test-test-services.cjs
index 5fc74725d..89d246b43 100644
--- a/scripts/tests/test-test-services.cjs
+++ b/scripts/tests/test-test-services.cjs
@@ -21,3 +21,8 @@ test('dynamic, missing and ambiguous matrices fail closed', () => {
assert.throws(() => checkServices(workflow + workflow, []), /Expected one/)
assert.throws(() => checkServices(workflow.replace('\n steps:', ' exclude:\n - service: one\n steps:'), ['one', 'packages/two']), /Unsupported/)
})
+test('empty, inline and re-indented matrices fail closed', () => {
+ assert.throws(() => checkServices(' matrix:\n service:\n\n steps:\n', []), /Expected one/)
+ assert.throws(() => checkServices(' matrix:\n service: [one, packages/two]\n steps:\n', ['one', 'packages/two']), /Expected one/)
+ assert.throws(() => checkServices(workflow.replace(/^ {6}/gm, ' '), ['one', 'packages/two']), /Expected one/)
+})