Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

chore(dev): add autoresearch hard gates

Alfredo Lopez32338337a1dd
diff --git a/Makefile b/Makefile
index 1f036f4c1..644c872bc 100644
--- a/Makefile
+++ b/Makefile
@@ -137,6 +137,10 @@ test-unit-all: test-service-matrix ## Run unit tests across all services
test-service-matrix: ## Check local test coverage against the CI service matrix
@node scripts/dev/check-test-services.cjs $(TEST_SERVICES)
+.PHONY: autoresearch-checks
+autoresearch-checks: ## Run the local checks used by AutoResearch hard gates
+ @bash scripts/dev/autoresearch-checks.sh
+
.PHONY: test-codex-subscription-t0
test-codex-subscription-t0: ## Run the Codex subscription T0 aggregator (counts, no skips)
@bash scripts/tests/test-codex-subscription-t0.sh
diff --git a/docs/testing/local-measurements.md b/docs/testing/local-measurements.md
index 56b5a6236..6292ab95b 100644
--- a/docs/testing/local-measurements.md
+++ b/docs/testing/local-measurements.md
@@ -37,11 +37,33 @@ and unsupported matrix shapes, and runs before `make test-unit-all` executes
packages. Each package keeps its `npm test` semantics; real PostgreSQL tests
remain opt-in and are not T1 evidence in this unit lane.
+## AutoResearch hard-gate entrypoints
+
+Use `scripts/dev/autoresearch-benchmark.sh -- <benchmark-command>` as the
+AutoResearch benchmark command. It runs the measurement intake first, so a
+missing or stale `origin/dev`, default detached HEAD, or merge conflict blocks
+the benchmark before it starts. A passing intake emits
+`AUTORESEARCH_GATE branch_freshness=pass`.
+
+Configure `make autoresearch-checks` as AutoResearch's `--checks-command`.
+It runs the CI/Makefile service-matrix parity check and emits
+`AUTORESEARCH_GATE service_matrix_parity=pass`. The Ralph adapter maps a
+non-zero checks command to the packet's `tests_pass` hard gate, so service
+drift cannot be kept. The captured gate lines make the two reasons explicit
+in packet evidence.
+
+These entrypoints use only the current local checkout and locally recorded Git
+refs. They do not fetch, push, create commits, change Minikube/Kubernetes
+state, or publish AutoResearch session files; the ignored `autoresearch.*`
+files remain local unless deliberately copied elsewhere.
+
CI runs these contract checks in the `shell-syntax` job; locally (no cluster
required):
```bash
node scripts/tests/run-node-test-files.mjs scripts/tests/test-test-services.cjs scripts/tests/test-measurement-base.cjs
bash scripts/tests/test-repo-intake-packet.sh
+bash scripts/tests/test-autoresearch-gates.sh
make test-service-matrix
+make autoresearch-checks
```
diff --git a/scripts/dev/autoresearch-benchmark.sh b/scripts/dev/autoresearch-benchmark.sh
new file mode 100755
index 000000000..a2701b14c
--- /dev/null
+++ b/scripts/dev/autoresearch-benchmark.sh
@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [[ "${1:-}" != "--" || $# -lt 2 ]]; then
+ printf 'Usage: autoresearch-benchmark.sh -- BENCHMARK [ARG...]\n' >&2
+ exit 2
+fi
+
+SCRIPT_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
+REPO_ROOT="$(git rev-parse --show-toplevel 2>/dev/null || true)"
+if [[ -z "${REPO_ROOT}" ]]; then
+ printf 'ERROR: autoresearch benchmark must run inside a Git checkout\n' >&2
+ exit 2
+fi
+cd "${REPO_ROOT}"
+
+# The measurement intake is deliberately run before the candidate benchmark.
+# It refuses stale/missing origin/dev ancestry, detached default measurements,
+# and merge conflicts without fetching or changing repository state.
+if bash "${SCRIPT_ROOT}/scripts/dev/repo-intake-packet.sh" --measure -- true; then
+ printf 'AUTORESEARCH_GATE branch_freshness=pass\n'
+else
+ status=$?
+ printf 'AUTORESEARCH_GATE branch_freshness=fail status=%s\n' "${status}" >&2
+ exit "${status}"
+fi
+
+shift
+exec "$@"
diff --git a/scripts/dev/autoresearch-checks.sh b/scripts/dev/autoresearch-checks.sh
new file mode 100755
index 000000000..28ff8f15d
--- /dev/null
+++ b/scripts/dev/autoresearch-checks.sh
@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+REPO_ROOT="$(git rev-parse --show-toplevel 2>/dev/null || true)"
+if [[ -z "${REPO_ROOT}" ]]; then
+ printf 'ERROR: autoresearch checks must run inside a Git checkout\n' >&2
+ exit 2
+fi
+cd "${REPO_ROOT}"
+
+# AutoResearch maps a non-zero checks command to its tests_pass hard gate.
+# Keep the parity result explicit in the captured packet output as well.
+if make test-service-matrix; then
+ printf 'AUTORESEARCH_GATE service_matrix_parity=pass\n'
+else
+ status=$?
+ printf 'AUTORESEARCH_GATE service_matrix_parity=fail status=%s\n' "${status}" >&2
+ exit "${status}"
+fi
diff --git a/scripts/tests/test-autoresearch-gates.sh b/scripts/tests/test-autoresearch-gates.sh
new file mode 100755
index 000000000..334ec2622
--- /dev/null
+++ b/scripts/tests/test-autoresearch-gates.sh
@@ -0,0 +1,128 @@
+#!/usr/bin/env bash
+set -u
+
+FAIL=0
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd -P)"
+BENCHMARK_WRAPPER="${ROOT}/scripts/dev/autoresearch-benchmark.sh"
+CHECKS_WRAPPER="${ROOT}/scripts/dev/autoresearch-checks.sh"
+HOST_HEAD="$(git -C "${ROOT}" rev-parse HEAD)"
+HOST_BRANCH="$(git -C "${ROOT}" branch --show-current)"
+HOST_STATUS="$(git -C "${ROOT}" status --porcelain=v1)"
+TMP_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/evenfire-autoresearch-gates.XXXXXX")"
+
+cleanup() {
+ local status=$?
+ rm -rf "${TMP_ROOT}"
+ if [[ "$(git -C "${ROOT}" rev-parse HEAD)" != "${HOST_HEAD}" ||
+ "$(git -C "${ROOT}" branch --show-current)" != "${HOST_BRANCH}" ||
+ "$(git -C "${ROOT}" status --porcelain=v1)" != "${HOST_STATUS}" ]]; then
+ echo 'FAIL: gate fixture changed host checkout'
+ status=1
+ fi
+ exit "${status}"
+}
+trap cleanup EXIT
+
+pass() { echo "PASS: $1"; }
+fail() { echo "FAIL: $1"; FAIL=1; }
+
+FIXTURE="${TMP_ROOT}/fixture"
+MARKER="${TMP_ROOT}/benchmark-ran"
+mkdir -p "${FIXTURE}"
+git init -q "${FIXTURE}"
+git -C "${FIXTURE}" checkout -q -b test/autoresearch-gates
+git -C "${FIXTURE}" config user.email "[email protected]"
+git -C "${FIXTURE}" config user.name "Evenfire Fixture"
+git -C "${FIXTURE}" commit --allow-empty -qm base
+git -C "${FIXTURE}" commit --allow-empty -qm candidate
+git -C "${FIXTURE}" checkout -q -b fixture-origin-dev
+git -C "${FIXTURE}" commit --allow-empty -qm "origin dev ahead"
+ORIGIN_DEV_HEAD="$(git -C "${FIXTURE}" rev-parse HEAD)"
+git -C "${FIXTURE}" checkout -q test/autoresearch-gates
+git -C "${FIXTURE}" update-ref refs/remotes/origin/dev "${ORIGIN_DEV_HEAD}"
+
+OUTPUT="$(cd "${FIXTURE}" && bash "${BENCHMARK_WRAPPER}" -- bash -c "touch '${MARKER}'" 2>&1)"
+STATUS=$?
+
+if [[ "${STATUS}" -eq 2 ]]; then
+ pass "stale origin/dev blocks the benchmark before execution"
+else
+ fail "stale origin/dev blocks the benchmark before execution"
+ echo "${OUTPUT}"
+fi
+
+if [[ ! -e "${MARKER}" ]]; then
+ pass "blocked benchmark command was not executed"
+else
+ fail "blocked benchmark command was not executed"
+fi
+
+if grep -Fq 'branch_missing_origin/dev_commits' <<<"${OUTPUT}"; then
+ pass "stale-base blocker is preserved in gate output"
+else
+ fail "stale-base blocker is preserved in gate output"
+ echo "${OUTPUT}"
+fi
+
+if ! grep -Fq 'AUTORESEARCH_GATE branch_freshness=pass' <<<"${OUTPUT}"; then
+ pass "blocked freshness gate does not report success"
+else
+ fail "blocked freshness gate does not report success"
+fi
+
+git -C "${FIXTURE}" update-ref refs/remotes/origin/dev "$(git -C "${FIXTURE}" rev-parse HEAD)"
+FRESH_OUTPUT="$(cd "${FIXTURE}" && bash "${BENCHMARK_WRAPPER}" -- bash -c "touch '${MARKER}'")"
+FRESH_STATUS=$?
+
+if [[ "${FRESH_STATUS}" -eq 0 ]]; then
+ pass "fresh origin/dev allows the benchmark"
+else
+ fail "fresh origin/dev allows the benchmark"
+ echo "${FRESH_OUTPUT}"
+fi
+
+if [[ -e "${MARKER}" ]]; then
+ pass "benchmark arguments pass through after the freshness gate"
+else
+ fail "benchmark arguments pass through after the freshness gate"
+fi
+
+if grep -Fq 'AUTORESEARCH_GATE branch_freshness=pass' <<<"${FRESH_OUTPUT}"; then
+ pass "freshness pass is explicit in gate output"
+else
+ fail "freshness pass is explicit in gate output"
+ echo "${FRESH_OUTPUT}"
+fi
+
+mkdir -p "${FIXTURE}/scripts/dev" "${FIXTURE}/.github/workflows"
+cp "${ROOT}/Makefile" "${FIXTURE}/Makefile"
+cp "${ROOT}/scripts/dev/check-test-services.cjs" "${FIXTURE}/scripts/dev/check-test-services.cjs"
+cp "${ROOT}/.github/workflows/ci-public.yml" "${FIXTURE}/.github/workflows/ci-public.yml"
+
+CHECKS_OUTPUT="$(cd "${FIXTURE}" && bash "${CHECKS_WRAPPER}" 2>&1)"
+CHECKS_STATUS=$?
+if [[ "${CHECKS_STATUS}" -eq 0 ]] && grep -Fq 'AUTORESEARCH_GATE service_matrix_parity=pass' <<<"${CHECKS_OUTPUT}"; then
+ pass "matching CI and Makefile service matrices pass"
+else
+ fail "matching CI and Makefile service matrices pass"
+ echo "${CHECKS_OUTPUT}"
+fi
+
+sed 's/^ - channel-reader$/ - fixture-only/' \
+ "${ROOT}/.github/workflows/ci-public.yml" >"${FIXTURE}/.github/workflows/ci-public.yml"
+DRIFT_OUTPUT="$(cd "${FIXTURE}" && bash "${CHECKS_WRAPPER}" 2>&1)"
+DRIFT_STATUS=$?
+if [[ "${DRIFT_STATUS}" -ne 0 ]] && grep -Fq 'Test service drift:' <<<"${DRIFT_OUTPUT}"; then
+ pass "service-matrix drift blocks the checks gate"
+else
+ fail "service-matrix drift blocks the checks gate"
+ echo "${DRIFT_OUTPUT}"
+fi
+
+if ! grep -Fq 'AUTORESEARCH_GATE service_matrix_parity=pass' <<<"${DRIFT_OUTPUT}"; then
+ pass "blocked service-matrix gate does not report success"
+else
+ fail "blocked service-matrix gate does not report success"
+fi
+
+exit "${FAIL}"