Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
chore(dev): add autoresearch hard gates
Alfredo Lopez32338337a1dd
diff --git a/Makefile b/Makefileindex 1f036f4c1..644c872bc 100644--- a/Makefile+++ b/Makefile@@ -137,6 +137,10 @@ test-unit-all: test-service-matrix ## Run unit tests across all servicestest-service-matrix: ## Check local test coverage against the CI service matrix@node scripts/dev/check-test-services.cjs $(TEST_SERVICES)+.PHONY: autoresearch-checks+autoresearch-checks: ## Run the local checks used by AutoResearch hard gates+ @bash scripts/dev/autoresearch-checks.sh+.PHONY: test-codex-subscription-t0test-codex-subscription-t0: ## Run the Codex subscription T0 aggregator (counts, no skips)@bash scripts/tests/test-codex-subscription-t0.shdiff --git a/docs/testing/local-measurements.md b/docs/testing/local-measurements.mdindex 56b5a6236..6292ab95b 100644--- a/docs/testing/local-measurements.md+++ b/docs/testing/local-measurements.md@@ -37,11 +37,33 @@ and unsupported matrix shapes, and runs before `make test-unit-all` executespackages. Each package keeps its `npm test` semantics; real PostgreSQL testsremain opt-in and are not T1 evidence in this unit lane.+## AutoResearch hard-gate entrypoints++Use `scripts/dev/autoresearch-benchmark.sh -- <benchmark-command>` as the+AutoResearch benchmark command. It runs the measurement intake first, so a+missing or stale `origin/dev`, default detached HEAD, or merge conflict blocks+the benchmark before it starts. A passing intake emits+`AUTORESEARCH_GATE branch_freshness=pass`.++Configure `make autoresearch-checks` as AutoResearch's `--checks-command`.+It runs the CI/Makefile service-matrix parity check and emits+`AUTORESEARCH_GATE service_matrix_parity=pass`. The Ralph adapter maps a+non-zero checks command to the packet's `tests_pass` hard gate, so service+drift cannot be kept. The captured gate lines make the two reasons explicit+in packet evidence.++These entrypoints use only the current local checkout and locally recorded Git+refs. They do not fetch, push, create commits, change Minikube/Kubernetes+state, or publish AutoResearch session files; the ignored `autoresearch.*`+files remain local unless deliberately copied elsewhere.+CI runs these contract checks in the `shell-syntax` job; locally (no clusterrequired):```bashnode scripts/tests/run-node-test-files.mjs scripts/tests/test-test-services.cjs scripts/tests/test-measurement-base.cjsbash scripts/tests/test-repo-intake-packet.sh+bash scripts/tests/test-autoresearch-gates.shmake test-service-matrix+make autoresearch-checks```diff --git a/scripts/dev/autoresearch-benchmark.sh b/scripts/dev/autoresearch-benchmark.shnew file mode 100755index 000000000..a2701b14c--- /dev/null+++ b/scripts/dev/autoresearch-benchmark.sh@@ -0,0 +1,29 @@+#!/usr/bin/env bash+set -euo pipefail++if [[ "${1:-}" != "--" || $# -lt 2 ]]; then+ printf 'Usage: autoresearch-benchmark.sh -- BENCHMARK [ARG...]\n' >&2+ exit 2+fi++SCRIPT_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"+REPO_ROOT="$(git rev-parse --show-toplevel 2>/dev/null || true)"+if [[ -z "${REPO_ROOT}" ]]; then+ printf 'ERROR: autoresearch benchmark must run inside a Git checkout\n' >&2+ exit 2+fi+cd "${REPO_ROOT}"++# The measurement intake is deliberately run before the candidate benchmark.+# It refuses stale/missing origin/dev ancestry, detached default measurements,+# and merge conflicts without fetching or changing repository state.+if bash "${SCRIPT_ROOT}/scripts/dev/repo-intake-packet.sh" --measure -- true; then+ printf 'AUTORESEARCH_GATE branch_freshness=pass\n'+else+ status=$?+ printf 'AUTORESEARCH_GATE branch_freshness=fail status=%s\n' "${status}" >&2+ exit "${status}"+fi++shift+exec "$@"diff --git a/scripts/dev/autoresearch-checks.sh b/scripts/dev/autoresearch-checks.shnew file mode 100755index 000000000..28ff8f15d--- /dev/null+++ b/scripts/dev/autoresearch-checks.sh@@ -0,0 +1,19 @@+#!/usr/bin/env bash+set -euo pipefail++REPO_ROOT="$(git rev-parse --show-toplevel 2>/dev/null || true)"+if [[ -z "${REPO_ROOT}" ]]; then+ printf 'ERROR: autoresearch checks must run inside a Git checkout\n' >&2+ exit 2+fi+cd "${REPO_ROOT}"++# AutoResearch maps a non-zero checks command to its tests_pass hard gate.+# Keep the parity result explicit in the captured packet output as well.+if make test-service-matrix; then+ printf 'AUTORESEARCH_GATE service_matrix_parity=pass\n'+else+ status=$?+ printf 'AUTORESEARCH_GATE service_matrix_parity=fail status=%s\n' "${status}" >&2+ exit "${status}"+fidiff --git a/scripts/tests/test-autoresearch-gates.sh b/scripts/tests/test-autoresearch-gates.shnew file mode 100755index 000000000..334ec2622--- /dev/null+++ b/scripts/tests/test-autoresearch-gates.sh@@ -0,0 +1,128 @@+#!/usr/bin/env bash+set -u++FAIL=0+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd -P)"+BENCHMARK_WRAPPER="${ROOT}/scripts/dev/autoresearch-benchmark.sh"+CHECKS_WRAPPER="${ROOT}/scripts/dev/autoresearch-checks.sh"+HOST_HEAD="$(git -C "${ROOT}" rev-parse HEAD)"+HOST_BRANCH="$(git -C "${ROOT}" branch --show-current)"+HOST_STATUS="$(git -C "${ROOT}" status --porcelain=v1)"+TMP_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/evenfire-autoresearch-gates.XXXXXX")"++cleanup() {+ local status=$?+ rm -rf "${TMP_ROOT}"+ if [[ "$(git -C "${ROOT}" rev-parse HEAD)" != "${HOST_HEAD}" ||+ "$(git -C "${ROOT}" branch --show-current)" != "${HOST_BRANCH}" ||+ "$(git -C "${ROOT}" status --porcelain=v1)" != "${HOST_STATUS}" ]]; then+ echo 'FAIL: gate fixture changed host checkout'+ status=1+ fi+ exit "${status}"+}+trap cleanup EXIT++pass() { echo "PASS: $1"; }+fail() { echo "FAIL: $1"; FAIL=1; }++FIXTURE="${TMP_ROOT}/fixture"+MARKER="${TMP_ROOT}/benchmark-ran"+mkdir -p "${FIXTURE}"+git init -q "${FIXTURE}"+git -C "${FIXTURE}" checkout -q -b test/autoresearch-gates+git -C "${FIXTURE}" config user.email "[email protected]"+git -C "${FIXTURE}" config user.name "Evenfire Fixture"+git -C "${FIXTURE}" commit --allow-empty -qm base+git -C "${FIXTURE}" commit --allow-empty -qm candidate+git -C "${FIXTURE}" checkout -q -b fixture-origin-dev+git -C "${FIXTURE}" commit --allow-empty -qm "origin dev ahead"+ORIGIN_DEV_HEAD="$(git -C "${FIXTURE}" rev-parse HEAD)"+git -C "${FIXTURE}" checkout -q test/autoresearch-gates+git -C "${FIXTURE}" update-ref refs/remotes/origin/dev "${ORIGIN_DEV_HEAD}"++OUTPUT="$(cd "${FIXTURE}" && bash "${BENCHMARK_WRAPPER}" -- bash -c "touch '${MARKER}'" 2>&1)"+STATUS=$?++if [[ "${STATUS}" -eq 2 ]]; then+ pass "stale origin/dev blocks the benchmark before execution"+else+ fail "stale origin/dev blocks the benchmark before execution"+ echo "${OUTPUT}"+fi++if [[ ! -e "${MARKER}" ]]; then+ pass "blocked benchmark command was not executed"+else+ fail "blocked benchmark command was not executed"+fi++if grep -Fq 'branch_missing_origin/dev_commits' <<<"${OUTPUT}"; then+ pass "stale-base blocker is preserved in gate output"+else+ fail "stale-base blocker is preserved in gate output"+ echo "${OUTPUT}"+fi++if ! grep -Fq 'AUTORESEARCH_GATE branch_freshness=pass' <<<"${OUTPUT}"; then+ pass "blocked freshness gate does not report success"+else+ fail "blocked freshness gate does not report success"+fi++git -C "${FIXTURE}" update-ref refs/remotes/origin/dev "$(git -C "${FIXTURE}" rev-parse HEAD)"+FRESH_OUTPUT="$(cd "${FIXTURE}" && bash "${BENCHMARK_WRAPPER}" -- bash -c "touch '${MARKER}'")"+FRESH_STATUS=$?++if [[ "${FRESH_STATUS}" -eq 0 ]]; then+ pass "fresh origin/dev allows the benchmark"+else+ fail "fresh origin/dev allows the benchmark"+ echo "${FRESH_OUTPUT}"+fi++if [[ -e "${MARKER}" ]]; then+ pass "benchmark arguments pass through after the freshness gate"+else+ fail "benchmark arguments pass through after the freshness gate"+fi++if grep -Fq 'AUTORESEARCH_GATE branch_freshness=pass' <<<"${FRESH_OUTPUT}"; then+ pass "freshness pass is explicit in gate output"+else+ fail "freshness pass is explicit in gate output"+ echo "${FRESH_OUTPUT}"+fi++mkdir -p "${FIXTURE}/scripts/dev" "${FIXTURE}/.github/workflows"+cp "${ROOT}/Makefile" "${FIXTURE}/Makefile"+cp "${ROOT}/scripts/dev/check-test-services.cjs" "${FIXTURE}/scripts/dev/check-test-services.cjs"+cp "${ROOT}/.github/workflows/ci-public.yml" "${FIXTURE}/.github/workflows/ci-public.yml"++CHECKS_OUTPUT="$(cd "${FIXTURE}" && bash "${CHECKS_WRAPPER}" 2>&1)"+CHECKS_STATUS=$?+if [[ "${CHECKS_STATUS}" -eq 0 ]] && grep -Fq 'AUTORESEARCH_GATE service_matrix_parity=pass' <<<"${CHECKS_OUTPUT}"; then+ pass "matching CI and Makefile service matrices pass"+else+ fail "matching CI and Makefile service matrices pass"+ echo "${CHECKS_OUTPUT}"+fi++sed 's/^ - channel-reader$/ - fixture-only/' \+ "${ROOT}/.github/workflows/ci-public.yml" >"${FIXTURE}/.github/workflows/ci-public.yml"+DRIFT_OUTPUT="$(cd "${FIXTURE}" && bash "${CHECKS_WRAPPER}" 2>&1)"+DRIFT_STATUS=$?+if [[ "${DRIFT_STATUS}" -ne 0 ]] && grep -Fq 'Test service drift:' <<<"${DRIFT_OUTPUT}"; then+ pass "service-matrix drift blocks the checks gate"+else+ fail "service-matrix drift blocks the checks gate"+ echo "${DRIFT_OUTPUT}"+fi++if ! grep -Fq 'AUTORESEARCH_GATE service_matrix_parity=pass' <<<"${DRIFT_OUTPUT}"; then+ pass "blocked service-matrix gate does not report success"+else+ fail "blocked service-matrix gate does not report success"+fi++exit "${FAIL}"