Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
chore: improve local measurement reliability and test coverage parity
Alfredo Lopezfd23ebbb48dc
diff --git a/Makefile b/Makefileindex cce68c7a4..0d2179b93 100644--- a/Makefile+++ b/Makefile@@ -54,6 +54,9 @@ SERVICES := \# Services that have unit testsTEST_SERVICES := \+ channel-reader \+ gfs-controller \+ workspace-files-controller \workflow-approval-request-reader \mcp-host \host-context-controller \@@ -74,6 +77,7 @@ TEST_SERVICES := \packages/workflow-runtime-core \packages/workflow-sdk \packages/network-policy-core \+ packages/codex-catalog-projection \packages/llm-provider-attempt-contract# ── Optional private infra (gcp-*, promotion) ──────────────────────────────@@ -110,7 +114,7 @@ install-all: ## npm install in all services (parallel)# ── Unit Tests ───────────────────────────────────────────────────────.PHONY: test-unit-all-test-unit-all: ## Run unit tests across all services+test-unit-all: test-service-matrix ## Run unit tests across all services@echo "Running unit tests..."@failed=""; \for svc in $(TEST_SERVICES); do \@@ -122,6 +126,10 @@ test-unit-all: ## Run unit tests across all servicesfi@echo "All unit tests passed."+.PHONY: test-service-matrix+test-service-matrix: ## Check local test coverage against the CI service matrix+ @node scripts/dev/check-test-services.cjs $(TEST_SERVICES)+.PHONY: test-codex-subscription-t0test-codex-subscription-t0: ## Run the Codex subscription T0 aggregator (counts, no skips)@bash scripts/tests/test-codex-subscription-t0.shdiff --git a/docs/testing/local-measurements.md b/docs/testing/local-measurements.mdnew file mode 100644index 000000000..393465ca2--- /dev/null+++ b/docs/testing/local-measurements.md@@ -0,0 +1,42 @@+# Local measurement base and unit-test coverage++Run a benchmark from the intended Evenfire checkout through the intake gate:++```bash+bash scripts/dev/repo-intake-packet.sh --measure -- <benchmark-command> [arguments...]+```++The gate records full HEAD and base commits, dirty-file count and ancestry.+It refuses to start the command when `origin/dev` is missing, has commits+absent from HEAD, HEAD is detached, or merge conflicts exist. Candidate edits+are allowed and reported. The benchmark's exit status is preserved.+This uses the locally recorded `origin/dev`; it does not fetch, update a+branch, inspect a cluster, or certify T0/T1/T2. Refresh the remote reference+through the normal repository workflow before measuring against current dev.+`REPO_INTAKE_BASE_REF` cannot override the measurement default.++For an intentional historical comparison, pin the base explicitly:++```bash+bash scripts/dev/repo-intake-packet.sh --measure --historical-base <commit> -- <benchmark-command> [arguments...]+```++That base must exist and be an ancestor of HEAD. Detached HEAD is allowed in+this mode; output is marked `historical_non_certifying`. This option does not+relax the normal T2 intake or certify runtime evidence. Use the gate as the+entry command of a future approved AutoResearch benchmark; calling a benchmark+directly bypasses it. No global runner or scheduled automation is changed.++`make test-service-matrix` compares the expanded `TEST_SERVICES` list with the+literal service matrix in CI. It rejects omissions, extra entries, duplicates+and unsupported matrix shapes before `make test-unit-all` executes packages.+CI remains unchanged. Each package keeps its existing `npm test` semantics;+real PostgreSQL tests remain opt-in and are not T1 evidence in this unit lane.++Local contract checks (no cluster required):++```bash+node --test scripts/tests/test-measurement-base.cjs scripts/tests/test-test-services.cjs+bash scripts/tests/test-repo-intake-packet.sh+make test-service-matrix+```diff --git a/scripts/dev/check-test-services.cjs b/scripts/dev/check-test-services.cjsnew file mode 100644index 000000000..fd4ce48bf--- /dev/null+++ b/scripts/dev/check-test-services.cjs@@ -0,0 +1,41 @@+#!/usr/bin/env node+'use strict'++const fs = require('node:fs')+const path = require('node:path')++// The CI contract is a literal service list. Refuse expressions or a changed+// matrix shape instead of silently checking a partial list.+function checkServices(workflow, localServices) {+ const matrices = [...(workflow + '\n').matchAll(/^ matrix:\n((?: {8}.*\n|\n)*)/gm)]+ const lists = matrices.map((match) => match[1].split('\n').filter((line) => line.trim() && !line.trim().startsWith('#')))+ .filter((lines) => lines.some((line) => /^ service:/.test(line)))+ if (lists.length !== 1 || lists[0][0] !== ' service:' || lists[0].length < 2) throw new Error('Expected one literal CI service matrix')+ const services = lists[0].slice(1).map((line) => {+ const match = line.match(/^ - ([a-z0-9][a-z0-9/-]*)\s*$/)+ if (!match || match[1].split('/').some((part) => !part)) throw new Error('Unsupported CI service entry')+ return match[1]+ })+ for (const [label, entries] of [['CI', services], ['Makefile', localServices]]) {+ if (new Set(entries).size !== entries.length) throw new Error(`${label} contains duplicate services`)+ }+ const missing = services.filter((service) => !localServices.includes(service))+ const extra = localServices.filter((service) => !services.includes(service))+ if (missing.length || extra.length) {+ throw new Error(`Test service drift: missing from Makefile: ${missing.join(', ') || 'none'}; absent from CI: ${extra.join(', ') || 'none'}`)+ }+ return services.length+}++if (require.main === module) {+ try {+ const workflow = fs.readFileSync(path.join(__dirname, '../../.github/workflows/ci-public.yml'), 'utf8')+ const count = checkServices(workflow, process.argv.slice(2))+ console.log(`PASS: Makefile and CI cover the same ${count} test services`)+ } catch (error) {+ console.error(error.message)+ process.exitCode = 1+ }+}++module.exports = { checkServices }diff --git a/scripts/dev/repo-intake-packet.sh b/scripts/dev/repo-intake-packet.shindex 6a6c61914..1f8cbed42 100755--- a/scripts/dev/repo-intake-packet.sh+++ b/scripts/dev/repo-intake-packet.sh@@ -2,6 +2,26 @@set -euo pipefailBASE_REF="${REPO_INTAKE_BASE_REF:-origin/dev}"+MEASURE=false+HISTORICAL=false+# Measurement mode uses the development base unless a historical comparison+# is explicitly requested. It never changes the normal T2 intake policy.+if [[ "${1:-}" == "--measure" ]]; then+ MEASURE=true+ BASE_REF=origin/dev+ shift+ if [[ "${1:-}" == "--historical-base" ]]; then+ [[ $# -ge 2 && -n "$2" && "$2" != -* ]] || { echo 'ERROR: historical base required' >&2; exit 2; }+ HISTORICAL=true+ BASE_REF="$2"+ shift 2+ fi+ [[ "${1:-}" == "--" && $# -ge 2 ]] || { echo 'Usage: repo-intake-packet.sh --measure [--historical-base REF] -- COMMAND [ARG...]' >&2; exit 2; }+ shift+elif (( $# )); then+ echo 'ERROR: unknown repo intake arguments' >&2+ exit 2+fiSCRIPT_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"BLOCKERS=()WARNINGS=()@@ -176,13 +196,39 @@ if [[ -n "${BASE_COMMIT}" ]]; thenBLOCKERS+=("branch_missing_${BASE_REF}_commits")elif [[ "${BASE_AHEAD}" != "n/a" && "${BASE_AHEAD}" -gt 0 ]]; thenBASE_STATUS="contains_base_with_${BASE_AHEAD}_local_commits"- else+ elif [[ "${BASE_AHEAD}" == "0" && "${BASE_BEHIND}" == "0" ]]; thenBASE_STATUS="at_base"+ else+ BASE_STATUS="unknown"+ BLOCKERS+=("base_comparison_failed")fielseBLOCKERS+=("missing_${BASE_REF}")fi+if [[ "${MEASURE}" == true ]]; then+ [[ "${DETACHED}" == no || "${HISTORICAL}" == true ]] || BLOCKERS+=("detached_measurement")+ count_status+ (( STATUS_CONFLICTS == 0 )) || BLOCKERS+=("unresolved_conflicts")+ kv "measurement_head" "${HEAD_FULL}"+ kv "measurement_base_ref" "${BASE_REF}"+ kv "measurement_base_commit" "${BASE_COMMIT:-missing}"+ kv "measurement_base_status" "${BASE_STATUS}"+ kv "measurement_dirty_count" "${STATUS_TOTAL}"+ kv "measurement_ref_source" "local_git_refs_no_fetch"+ if (( ${#BLOCKERS[@]} )); then+ kv "measurement_readiness" "blocked"+ kv "blockers" "$(join_blockers)"+ exit 2+ fi+ if [[ "${HISTORICAL}" == true ]]; then+ kv "measurement_readiness" "historical_non_certifying"+ else+ kv "measurement_readiness" "base_check_passed_not_a_lane_verdict"+ fi+ exec "$@"+fi+UPSTREAM_AHEAD="n/a"UPSTREAM_BEHIND="n/a"if [[ "${UPSTREAM}" != "none" ]]; thendiff --git a/scripts/tests/test-measurement-base.cjs b/scripts/tests/test-measurement-base.cjsnew file mode 100644index 000000000..a356f83ba--- /dev/null+++ b/scripts/tests/test-measurement-base.cjs@@ -0,0 +1,64 @@+'use strict'++const { test } = require('node:test')+const assert = require('node:assert/strict')+const fs = require('node:fs')+const os = require('node:os')+const path = require('node:path')+const { spawnSync, execFileSync } = require('node:child_process')+const root = path.resolve(__dirname, '../..')+const script = path.join(root, 'scripts/dev/repo-intake-packet.sh')++test('measurement gate enforces ancestry before executing the benchmark', () => {+ const git = (cwd, ...args) => execFileSync('git', args, { cwd, encoding: 'utf8' }).trim()+ const hostState = () => ['rev-parse HEAD', 'branch --show-current', 'status --porcelain=v1'].map((args) => git(root, ...args.split(' ')))+ const before = hostState()+ const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'evenfire-measurement-'))+ try {+ git(tmp, 'init', '-q')+ git(tmp, 'config', 'user.email', '[email protected]')+ git(tmp, 'config', 'user.name', 'Fixture')+ git(tmp, 'commit', '--allow-empty', '-qm', 'base')+ git(tmp, 'branch', '-M', 'test/measurement')+ const base = git(tmp, 'rev-parse', 'HEAD')+ git(tmp, 'update-ref', 'refs/remotes/origin/dev', base)+ const run = (options = [], code = 'console.log("BENCHMARK_EXECUTED")') => spawnSync('bash', [script, '--measure', ...options, '--', process.execPath, '-e', code], {+ cwd: tmp, encoding: 'utf8', env: { ...process.env, REPO_INTAKE_BASE_REF: 'HEAD' },+ })+ const allowed = (options, label) => {+ const result = run(options)+ assert.equal(result.status, 0, result.stderr + result.stdout)+ assert.match(result.stdout, /BENCHMARK_EXECUTED/, label)+ return result+ }+ const blocked = (label) => {+ const result = run()+ assert.equal(result.status, 2, label + result.stderr + result.stdout)+ assert.doesNotMatch(result.stdout, /BENCHMARK_EXECUTED/, label)+ }+ allowed([], 'identical base')+ assert.equal(run([], 'process.exit(7)').status, 7, 'benchmark failure preserved')+ git(tmp, 'commit', '--allow-empty', '-qm', 'candidate')+ allowed([], 'ahead of base')+ const candidate = git(tmp, 'rev-parse', 'HEAD')+ git(tmp, 'update-ref', 'refs/remotes/origin/dev', candidate)+ git(tmp, 'checkout', '-q', '-B', 'test/historical', base)+ blocked('behind base, environment override ignored')+ const historical = allowed(['--historical-base', base], 'explicit historical comparison')+ assert.match(historical.stdout, /historical_non_certifying/)+ assert.match(historical.stdout, new RegExp(base))+ git(tmp, 'commit', '--allow-empty', '-qm', 'divergent')+ blocked('divergent base')+ git(tmp, 'update-ref', '-d', 'refs/remotes/origin/dev')+ blocked('missing base')+ git(tmp, 'update-ref', 'refs/remotes/origin/dev', base)+ git(tmp, 'checkout', '-q', '--detach', base)+ blocked('detached default')+ allowed(['--historical-base', base], 'detached historical')+ assert.equal(run(['--historical-base', 'missing-ref']).status, 2)+ assert.equal(run(['--historical-base']).status, 2)+ } finally {+ fs.rmSync(tmp, { recursive: true, force: true })+ assert.deepEqual(hostState(), before, 'host checkout remains unchanged')+ }+})diff --git a/scripts/tests/test-repo-intake-packet.sh b/scripts/tests/test-repo-intake-packet.shindex aa0976b62..6883268b7 100755--- a/scripts/tests/test-repo-intake-packet.sh+++ b/scripts/tests/test-repo-intake-packet.sh@@ -3,6 +3,9 @@ set -uFAIL=0ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd -P)"+HOST_HEAD="$(git -C "${ROOT}" rev-parse HEAD)"+HOST_BRANCH="$(git -C "${ROOT}" branch --show-current)"+HOST_STATUS="$(git -C "${ROOT}" status --porcelain=v1)"SCRIPT="${ROOT}/scripts/dev/repo-intake-packet.sh"OWNER_SCRIPT="${ROOT}/scripts/minikube/profile-owner.sh"TMP_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/clerum-intake-test.XXXXXX")"@@ -11,8 +14,16 @@ PROFILE_ROOT="${TMP_ROOT}/profiles"mkdir -p "${PROFILE_ROOT}"cleanup() {+ local status=$?chmod -R u+w "${TMP_ROOT}" 2>/dev/null || truerm -rf "${TMP_ROOT}"+ if [[ "$(git -C "${ROOT}" rev-parse HEAD)" != "${HOST_HEAD}" ||+ "$(git -C "${ROOT}" branch --show-current)" != "${HOST_BRANCH}" ||+ "$(git -C "${ROOT}" status --porcelain=v1)" != "${HOST_STATUS}" ]]; then+ echo 'FAIL: fixture changed host checkout'+ status=1+ fi+ exit "${status}"}trap cleanup EXIT@@ -76,11 +87,17 @@ payload_value() {write_ports() {local destination="$1" base="$2"- cat >"${destination}" <<EOF_PORTS-PORT_BASE=${base}-CONTROL_API_PORT=$((base + 90))-CONTROL_API_URL=http://127.0.0.1:$((base + 90))-EOF_PORTS+ # Schema-v2 requires the full branch-profile mapping, not just control-api.+ local spec name port+ {+ printf 'PORT_BASE=%s\n' "${base}"+ for spec in CONTROL_UI:0 PROFILE_UI:1 MCP_HOST:80 REGISTRY_API:85 CONTROL_API:90 EXTERNAL_REST_API:91 MEMBER_REGISTRATION_SERVICE:92 RPC_PROXY:94 WORKFLOW_APPROVAL_READER:98; do+ name="${spec%:*}"+ port=$((base + ${spec#*:}))+ printf '%s_PORT=%s\n%s_URL=http://127.0.0.1:%s\n' "${name}" "${port}" "${name}" "${port}"+ done+ printf 'PROFILE_UI_BASE_URL=http://127.0.0.1:%s\n' "$((base + 1))"+ } >"${destination}"}if bash -n "${SCRIPT}"; thendiff --git a/scripts/tests/test-test-services.cjs b/scripts/tests/test-test-services.cjsnew file mode 100644index 000000000..5fc74725d--- /dev/null+++ b/scripts/tests/test-test-services.cjs@@ -0,0 +1,23 @@+'use strict'++const { test } = require('node:test')+const assert = require('node:assert/strict')+const { checkServices } = require('../dev/check-test-services.cjs')+const workflow = ' matrix:\n service:\n - one\n - packages/two\n\n steps:\n'++test('matrix parity is independent of order', () => {+ assert.equal(checkServices(workflow, ['packages/two', 'one']), 2)+})+test('missing and extra services fail with actionable names', () => {+ assert.throws(() => checkServices(workflow, ['one', 'three']), /missing from Makefile: packages\/two; absent from CI: three/)+})+test('duplicate services fail on either side', () => {+ assert.throws(() => checkServices(workflow, ['one', 'one']), /Makefile contains duplicate/)+ assert.throws(() => checkServices(workflow.replace('packages/two', 'one'), ['one']), /CI contains duplicate/)+})+test('dynamic, missing and ambiguous matrices fail closed', () => {+ assert.throws(() => checkServices(workflow.replace('packages/two', '${{ inputs.service }}'), ['one']), /Unsupported/)+ assert.throws(() => checkServices('', []), /Expected one/)+ assert.throws(() => checkServices(workflow + workflow, []), /Expected one/)+ assert.throws(() => checkServices(workflow.replace('\n steps:', ' exclude:\n - service: one\n steps:'), ['one', 'packages/two']), /Unsupported/)+})