Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

chore: improve local measurement reliability and test coverage parity

Alfredo Lopezfd23ebbb48dc
diff --git a/Makefile b/Makefile
index cce68c7a4..0d2179b93 100644
--- a/Makefile
+++ b/Makefile
@@ -54,6 +54,9 @@ SERVICES := \
# Services that have unit tests
TEST_SERVICES := \
+ channel-reader \
+ gfs-controller \
+ workspace-files-controller \
workflow-approval-request-reader \
mcp-host \
host-context-controller \
@@ -74,6 +77,7 @@ TEST_SERVICES := \
packages/workflow-runtime-core \
packages/workflow-sdk \
packages/network-policy-core \
+ packages/codex-catalog-projection \
packages/llm-provider-attempt-contract
# ── Optional private infra (gcp-*, promotion) ──────────────────────────────
@@ -110,7 +114,7 @@ install-all: ## npm install in all services (parallel)
# ── Unit Tests ───────────────────────────────────────────────────────
.PHONY: test-unit-all
-test-unit-all: ## Run unit tests across all services
+test-unit-all: test-service-matrix ## Run unit tests across all services
@echo "Running unit tests..."
@failed=""; \
for svc in $(TEST_SERVICES); do \
@@ -122,6 +126,10 @@ test-unit-all: ## Run unit tests across all services
fi
@echo "All unit tests passed."
+.PHONY: test-service-matrix
+test-service-matrix: ## Check local test coverage against the CI service matrix
+ @node scripts/dev/check-test-services.cjs $(TEST_SERVICES)
+
.PHONY: test-codex-subscription-t0
test-codex-subscription-t0: ## Run the Codex subscription T0 aggregator (counts, no skips)
@bash scripts/tests/test-codex-subscription-t0.sh
diff --git a/docs/testing/local-measurements.md b/docs/testing/local-measurements.md
new file mode 100644
index 000000000..393465ca2
--- /dev/null
+++ b/docs/testing/local-measurements.md
@@ -0,0 +1,42 @@
+# Local measurement base and unit-test coverage
+
+Run a benchmark from the intended Evenfire checkout through the intake gate:
+
+```bash
+bash scripts/dev/repo-intake-packet.sh --measure -- <benchmark-command> [arguments...]
+```
+
+The gate records full HEAD and base commits, dirty-file count and ancestry.
+It refuses to start the command when `origin/dev` is missing, has commits
+absent from HEAD, HEAD is detached, or merge conflicts exist. Candidate edits
+are allowed and reported. The benchmark's exit status is preserved.
+This uses the locally recorded `origin/dev`; it does not fetch, update a
+branch, inspect a cluster, or certify T0/T1/T2. Refresh the remote reference
+through the normal repository workflow before measuring against current dev.
+`REPO_INTAKE_BASE_REF` cannot override the measurement default.
+
+For an intentional historical comparison, pin the base explicitly:
+
+```bash
+bash scripts/dev/repo-intake-packet.sh --measure --historical-base <commit> -- <benchmark-command> [arguments...]
+```
+
+That base must exist and be an ancestor of HEAD. Detached HEAD is allowed in
+this mode; output is marked `historical_non_certifying`. This option does not
+relax the normal T2 intake or certify runtime evidence. Use the gate as the
+entry command of a future approved AutoResearch benchmark; calling a benchmark
+directly bypasses it. No global runner or scheduled automation is changed.
+
+`make test-service-matrix` compares the expanded `TEST_SERVICES` list with the
+literal service matrix in CI. It rejects omissions, extra entries, duplicates
+and unsupported matrix shapes before `make test-unit-all` executes packages.
+CI remains unchanged. Each package keeps its existing `npm test` semantics;
+real PostgreSQL tests remain opt-in and are not T1 evidence in this unit lane.
+
+Local contract checks (no cluster required):
+
+```bash
+node --test scripts/tests/test-measurement-base.cjs scripts/tests/test-test-services.cjs
+bash scripts/tests/test-repo-intake-packet.sh
+make test-service-matrix
+```
diff --git a/scripts/dev/check-test-services.cjs b/scripts/dev/check-test-services.cjs
new file mode 100644
index 000000000..fd4ce48bf
--- /dev/null
+++ b/scripts/dev/check-test-services.cjs
@@ -0,0 +1,41 @@
+#!/usr/bin/env node
+'use strict'
+
+const fs = require('node:fs')
+const path = require('node:path')
+
+// The CI contract is a literal service list. Refuse expressions or a changed
+// matrix shape instead of silently checking a partial list.
+function checkServices(workflow, localServices) {
+ const matrices = [...(workflow + '\n').matchAll(/^ matrix:\n((?: {8}.*\n|\n)*)/gm)]
+ const lists = matrices.map((match) => match[1].split('\n').filter((line) => line.trim() && !line.trim().startsWith('#')))
+ .filter((lines) => lines.some((line) => /^ service:/.test(line)))
+ if (lists.length !== 1 || lists[0][0] !== ' service:' || lists[0].length < 2) throw new Error('Expected one literal CI service matrix')
+ const services = lists[0].slice(1).map((line) => {
+ const match = line.match(/^ - ([a-z0-9][a-z0-9/-]*)\s*$/)
+ if (!match || match[1].split('/').some((part) => !part)) throw new Error('Unsupported CI service entry')
+ return match[1]
+ })
+ for (const [label, entries] of [['CI', services], ['Makefile', localServices]]) {
+ if (new Set(entries).size !== entries.length) throw new Error(`${label} contains duplicate services`)
+ }
+ const missing = services.filter((service) => !localServices.includes(service))
+ const extra = localServices.filter((service) => !services.includes(service))
+ if (missing.length || extra.length) {
+ throw new Error(`Test service drift: missing from Makefile: ${missing.join(', ') || 'none'}; absent from CI: ${extra.join(', ') || 'none'}`)
+ }
+ return services.length
+}
+
+if (require.main === module) {
+ try {
+ const workflow = fs.readFileSync(path.join(__dirname, '../../.github/workflows/ci-public.yml'), 'utf8')
+ const count = checkServices(workflow, process.argv.slice(2))
+ console.log(`PASS: Makefile and CI cover the same ${count} test services`)
+ } catch (error) {
+ console.error(error.message)
+ process.exitCode = 1
+ }
+}
+
+module.exports = { checkServices }
diff --git a/scripts/dev/repo-intake-packet.sh b/scripts/dev/repo-intake-packet.sh
index 6a6c61914..1f8cbed42 100755
--- a/scripts/dev/repo-intake-packet.sh
+++ b/scripts/dev/repo-intake-packet.sh
@@ -2,6 +2,26 @@
set -euo pipefail
BASE_REF="${REPO_INTAKE_BASE_REF:-origin/dev}"
+MEASURE=false
+HISTORICAL=false
+# Measurement mode uses the development base unless a historical comparison
+# is explicitly requested. It never changes the normal T2 intake policy.
+if [[ "${1:-}" == "--measure" ]]; then
+ MEASURE=true
+ BASE_REF=origin/dev
+ shift
+ if [[ "${1:-}" == "--historical-base" ]]; then
+ [[ $# -ge 2 && -n "$2" && "$2" != -* ]] || { echo 'ERROR: historical base required' >&2; exit 2; }
+ HISTORICAL=true
+ BASE_REF="$2"
+ shift 2
+ fi
+ [[ "${1:-}" == "--" && $# -ge 2 ]] || { echo 'Usage: repo-intake-packet.sh --measure [--historical-base REF] -- COMMAND [ARG...]' >&2; exit 2; }
+ shift
+elif (( $# )); then
+ echo 'ERROR: unknown repo intake arguments' >&2
+ exit 2
+fi
SCRIPT_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
BLOCKERS=()
WARNINGS=()
@@ -176,13 +196,39 @@ if [[ -n "${BASE_COMMIT}" ]]; then
BLOCKERS+=("branch_missing_${BASE_REF}_commits")
elif [[ "${BASE_AHEAD}" != "n/a" && "${BASE_AHEAD}" -gt 0 ]]; then
BASE_STATUS="contains_base_with_${BASE_AHEAD}_local_commits"
- else
+ elif [[ "${BASE_AHEAD}" == "0" && "${BASE_BEHIND}" == "0" ]]; then
BASE_STATUS="at_base"
+ else
+ BASE_STATUS="unknown"
+ BLOCKERS+=("base_comparison_failed")
fi
else
BLOCKERS+=("missing_${BASE_REF}")
fi
+if [[ "${MEASURE}" == true ]]; then
+ [[ "${DETACHED}" == no || "${HISTORICAL}" == true ]] || BLOCKERS+=("detached_measurement")
+ count_status
+ (( STATUS_CONFLICTS == 0 )) || BLOCKERS+=("unresolved_conflicts")
+ kv "measurement_head" "${HEAD_FULL}"
+ kv "measurement_base_ref" "${BASE_REF}"
+ kv "measurement_base_commit" "${BASE_COMMIT:-missing}"
+ kv "measurement_base_status" "${BASE_STATUS}"
+ kv "measurement_dirty_count" "${STATUS_TOTAL}"
+ kv "measurement_ref_source" "local_git_refs_no_fetch"
+ if (( ${#BLOCKERS[@]} )); then
+ kv "measurement_readiness" "blocked"
+ kv "blockers" "$(join_blockers)"
+ exit 2
+ fi
+ if [[ "${HISTORICAL}" == true ]]; then
+ kv "measurement_readiness" "historical_non_certifying"
+ else
+ kv "measurement_readiness" "base_check_passed_not_a_lane_verdict"
+ fi
+ exec "$@"
+fi
+
UPSTREAM_AHEAD="n/a"
UPSTREAM_BEHIND="n/a"
if [[ "${UPSTREAM}" != "none" ]]; then
diff --git a/scripts/tests/test-measurement-base.cjs b/scripts/tests/test-measurement-base.cjs
new file mode 100644
index 000000000..a356f83ba
--- /dev/null
+++ b/scripts/tests/test-measurement-base.cjs
@@ -0,0 +1,64 @@
+'use strict'
+
+const { test } = require('node:test')
+const assert = require('node:assert/strict')
+const fs = require('node:fs')
+const os = require('node:os')
+const path = require('node:path')
+const { spawnSync, execFileSync } = require('node:child_process')
+const root = path.resolve(__dirname, '../..')
+const script = path.join(root, 'scripts/dev/repo-intake-packet.sh')
+
+test('measurement gate enforces ancestry before executing the benchmark', () => {
+ const git = (cwd, ...args) => execFileSync('git', args, { cwd, encoding: 'utf8' }).trim()
+ const hostState = () => ['rev-parse HEAD', 'branch --show-current', 'status --porcelain=v1'].map((args) => git(root, ...args.split(' ')))
+ const before = hostState()
+ const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'evenfire-measurement-'))
+ try {
+ git(tmp, 'init', '-q')
+ git(tmp, 'config', 'user.email', '[email protected]')
+ git(tmp, 'config', 'user.name', 'Fixture')
+ git(tmp, 'commit', '--allow-empty', '-qm', 'base')
+ git(tmp, 'branch', '-M', 'test/measurement')
+ const base = git(tmp, 'rev-parse', 'HEAD')
+ git(tmp, 'update-ref', 'refs/remotes/origin/dev', base)
+ const run = (options = [], code = 'console.log("BENCHMARK_EXECUTED")') => spawnSync('bash', [script, '--measure', ...options, '--', process.execPath, '-e', code], {
+ cwd: tmp, encoding: 'utf8', env: { ...process.env, REPO_INTAKE_BASE_REF: 'HEAD' },
+ })
+ const allowed = (options, label) => {
+ const result = run(options)
+ assert.equal(result.status, 0, result.stderr + result.stdout)
+ assert.match(result.stdout, /BENCHMARK_EXECUTED/, label)
+ return result
+ }
+ const blocked = (label) => {
+ const result = run()
+ assert.equal(result.status, 2, label + result.stderr + result.stdout)
+ assert.doesNotMatch(result.stdout, /BENCHMARK_EXECUTED/, label)
+ }
+ allowed([], 'identical base')
+ assert.equal(run([], 'process.exit(7)').status, 7, 'benchmark failure preserved')
+ git(tmp, 'commit', '--allow-empty', '-qm', 'candidate')
+ allowed([], 'ahead of base')
+ const candidate = git(tmp, 'rev-parse', 'HEAD')
+ git(tmp, 'update-ref', 'refs/remotes/origin/dev', candidate)
+ git(tmp, 'checkout', '-q', '-B', 'test/historical', base)
+ blocked('behind base, environment override ignored')
+ const historical = allowed(['--historical-base', base], 'explicit historical comparison')
+ assert.match(historical.stdout, /historical_non_certifying/)
+ assert.match(historical.stdout, new RegExp(base))
+ git(tmp, 'commit', '--allow-empty', '-qm', 'divergent')
+ blocked('divergent base')
+ git(tmp, 'update-ref', '-d', 'refs/remotes/origin/dev')
+ blocked('missing base')
+ git(tmp, 'update-ref', 'refs/remotes/origin/dev', base)
+ git(tmp, 'checkout', '-q', '--detach', base)
+ blocked('detached default')
+ allowed(['--historical-base', base], 'detached historical')
+ assert.equal(run(['--historical-base', 'missing-ref']).status, 2)
+ assert.equal(run(['--historical-base']).status, 2)
+ } finally {
+ fs.rmSync(tmp, { recursive: true, force: true })
+ assert.deepEqual(hostState(), before, 'host checkout remains unchanged')
+ }
+})
diff --git a/scripts/tests/test-repo-intake-packet.sh b/scripts/tests/test-repo-intake-packet.sh
index aa0976b62..6883268b7 100755
--- a/scripts/tests/test-repo-intake-packet.sh
+++ b/scripts/tests/test-repo-intake-packet.sh
@@ -3,6 +3,9 @@ set -u
FAIL=0
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd -P)"
+HOST_HEAD="$(git -C "${ROOT}" rev-parse HEAD)"
+HOST_BRANCH="$(git -C "${ROOT}" branch --show-current)"
+HOST_STATUS="$(git -C "${ROOT}" status --porcelain=v1)"
SCRIPT="${ROOT}/scripts/dev/repo-intake-packet.sh"
OWNER_SCRIPT="${ROOT}/scripts/minikube/profile-owner.sh"
TMP_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/clerum-intake-test.XXXXXX")"
@@ -11,8 +14,16 @@ PROFILE_ROOT="${TMP_ROOT}/profiles"
mkdir -p "${PROFILE_ROOT}"
cleanup() {
+ local status=$?
chmod -R u+w "${TMP_ROOT}" 2>/dev/null || true
rm -rf "${TMP_ROOT}"
+ if [[ "$(git -C "${ROOT}" rev-parse HEAD)" != "${HOST_HEAD}" ||
+ "$(git -C "${ROOT}" branch --show-current)" != "${HOST_BRANCH}" ||
+ "$(git -C "${ROOT}" status --porcelain=v1)" != "${HOST_STATUS}" ]]; then
+ echo 'FAIL: fixture changed host checkout'
+ status=1
+ fi
+ exit "${status}"
}
trap cleanup EXIT
@@ -76,11 +87,17 @@ payload_value() {
write_ports() {
local destination="$1" base="$2"
- cat >"${destination}" <<EOF_PORTS
-PORT_BASE=${base}
-CONTROL_API_PORT=$((base + 90))
-CONTROL_API_URL=http://127.0.0.1:$((base + 90))
-EOF_PORTS
+ # Schema-v2 requires the full branch-profile mapping, not just control-api.
+ local spec name port
+ {
+ printf 'PORT_BASE=%s\n' "${base}"
+ for spec in CONTROL_UI:0 PROFILE_UI:1 MCP_HOST:80 REGISTRY_API:85 CONTROL_API:90 EXTERNAL_REST_API:91 MEMBER_REGISTRATION_SERVICE:92 RPC_PROXY:94 WORKFLOW_APPROVAL_READER:98; do
+ name="${spec%:*}"
+ port=$((base + ${spec#*:}))
+ printf '%s_PORT=%s\n%s_URL=http://127.0.0.1:%s\n' "${name}" "${port}" "${name}" "${port}"
+ done
+ printf 'PROFILE_UI_BASE_URL=http://127.0.0.1:%s\n' "$((base + 1))"
+ } >"${destination}"
}
if bash -n "${SCRIPT}"; then
diff --git a/scripts/tests/test-test-services.cjs b/scripts/tests/test-test-services.cjs
new file mode 100644
index 000000000..5fc74725d
--- /dev/null
+++ b/scripts/tests/test-test-services.cjs
@@ -0,0 +1,23 @@
+'use strict'
+
+const { test } = require('node:test')
+const assert = require('node:assert/strict')
+const { checkServices } = require('../dev/check-test-services.cjs')
+const workflow = ' matrix:\n service:\n - one\n - packages/two\n\n steps:\n'
+
+test('matrix parity is independent of order', () => {
+ assert.equal(checkServices(workflow, ['packages/two', 'one']), 2)
+})
+test('missing and extra services fail with actionable names', () => {
+ assert.throws(() => checkServices(workflow, ['one', 'three']), /missing from Makefile: packages\/two; absent from CI: three/)
+})
+test('duplicate services fail on either side', () => {
+ assert.throws(() => checkServices(workflow, ['one', 'one']), /Makefile contains duplicate/)
+ assert.throws(() => checkServices(workflow.replace('packages/two', 'one'), ['one']), /CI contains duplicate/)
+})
+test('dynamic, missing and ambiguous matrices fail closed', () => {
+ assert.throws(() => checkServices(workflow.replace('packages/two', '${{ inputs.service }}'), ['one']), /Unsupported/)
+ assert.throws(() => checkServices('', []), /Expected one/)
+ assert.throws(() => checkServices(workflow + workflow, []), /Expected one/)
+ assert.throws(() => checkServices(workflow.replace('\n steps:', ' exclude:\n - service: one\n steps:'), ['one', 'packages/two']), /Unsupported/)
+})