Skip to content

armgit / ax

Public Imported from GitHub
0

Agent Executor (AX) is Google's open agentic orchestration runtime

Go

test: add Kind-based E2E test suite and CI workflow (#474)

Siyuan Zhang43439b411f01
diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
new file mode 100644
index 0000000..c6823b2
--- /dev/null
+++ b/.github/workflows/e2e.yml
@@ -0,0 +1,80 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: E2E
+
+on:
+ push:
+ branches: [ main ]
+ pull_request:
+ branches: [ main ]
+ # Nightly run against Substrate main to catch incompatibilities with
+ # unreleased Substrate changes early. Never blocks PRs.
+ schedule:
+ - cron: '0 6 * * *' # every day at 06:00am UTC
+ workflow_dispatch:
+ inputs:
+ substrate_ref:
+ description: 'Substrate branch, tag, or commit to test against (default: version in go.mod)'
+ required: false
+ default: ''
+ type: string
+
+permissions:
+ contents: read
+
+jobs:
+ e2e-kind:
+ name: E2E (Kind / Substrate ${{ github.event_name == 'schedule' && 'main' || inputs.substrate_ref || 'go.mod' }})
+ runs-on: ubuntu-latest
+ timeout-minutes: 30
+ steps:
+ - name: Checkout AX
+ uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+
+ - name: Set up Go
+ uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0
+ with:
+ go-version-file: go.mod
+
+ - name: Install Kind
+ uses: helm/kind-action@1544676c07bb3570fcb70166beaa07c4528fd4aa # v1.15.1
+ with:
+ install_only: true
+
+ # The runner is discarded after the job, so the cluster is left running for
+ # the diagnostics step below rather than cleaned up.
+ # PRs and pushes test the Substrate version in go.mod, so they are not
+ # blocked by unreleased changes on Substrate main. The nightly run tests
+ # main; manual runs test the requested ref (go.mod if empty).
+ - name: Run Kind E2E suite
+ env:
+ EVENT: ${{ github.event_name }}
+ INPUT_REF: ${{ inputs.substrate_ref }}
+ run: |
+ if [[ "${EVENT}" == "schedule" ]]; then
+ make test-e2e-head
+ else
+ make test-e2e ${INPUT_REF:+SUBSTRATE_REF="${INPUT_REF}"}
+ fi
+
+ - name: Collect cluster diagnostics on failure
+ if: failure()
+ env:
+ KUBECONFIG: .tmp/kubeconfig-ax-e2e
+ run: |
+ [[ -f "${KUBECONFIG}" ]] || exit 0
+ kubectl get pods -A -o wide || true
+ kubectl -n ax-system logs deployment/ax-server --tail=200 || true
+ kubectl -n ate-system logs deployment/ate-api-server --all-containers --tail=200 || true
diff --git a/.gitignore b/.gitignore
index 5bb6ff2..7f5c597 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,7 @@
# Binaries
bin/
/tmp/
+/.tmp/
*.exe
*.test
/ax
@@ -8,3 +9,4 @@ bin/
/ax-task-runner
cmd/ax-task-runner/__pycache__/antigravity_bootstrap.cpython-313.pyc
+.vscode/
diff --git a/Dockerfile.task-runner b/Dockerfile.task-runner
index 29dc3e1..3dab7ac 100644
--- a/Dockerfile.task-runner
+++ b/Dockerfile.task-runner
@@ -25,7 +25,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
RUN pip install --no-cache-dir google-antigravity
-COPY bin/linux_amd64/ax-task-runner /usr/local/bin/ax-task-runner
+ARG TARGETARCH=amd64
+COPY bin/linux_${TARGETARCH}/ax-task-runner /usr/local/bin/ax-task-runner
COPY cmd/ax-task-runner/antigravity_bootstrap.py /usr/local/bin/antigravity_bootstrap.py
ENTRYPOINT ["/usr/local/bin/ax-task-runner"]
diff --git a/Makefile b/Makefile
index 0ffba30..2d918f5 100644
--- a/Makefile
+++ b/Makefile
@@ -19,7 +19,7 @@ AX_IMAGE_REPO ?= gcr.io/ax-substrate/ate-images
TASK_RUNNER_REPO ?= $(AX_IMAGE_REPO)/ax-task-runner
CONTAINER_CLI ?= $(shell which podman 2>/dev/null || which docker 2>/dev/null)
-.PHONY: all build build-binaries build-task-runner install push push-task-runner deploy deploy-server deploy-redis apply-example test clean
+.PHONY: all build build-binaries build-task-runner install push push-task-runner deploy deploy-server deploy-redis apply-example test test-e2e test-e2e-head e2e-setup e2e-run e2e-cleanup clean
all: build
@@ -87,6 +87,28 @@ test:
@echo "==> Running tests..."
go test -v ./...
+# Provision Kind + Substrate + AX, then run the E2E suite. Substrate defaults to
+# the version in go.mod; override with SUBSTRATE_REF=<branch|tag|commit>.
+# Pass extra `go test` flags via E2E_ARGS, e.g. E2E_ARGS='-test.run TestE2E_ModelCRUD'.
+test-e2e:
+ ./hack/run-e2e-kind.sh $(E2E_ARGS)
+
+# Run the E2E suite against Substrate main.
+test-e2e-head:
+ SUBSTRATE_REF=main ./hack/run-e2e-kind.sh $(E2E_ARGS)
+
+# Provision Kind + Substrate + AX without running tests (accepts SUBSTRATE_REF).
+e2e-setup:
+ ./hack/setup-e2e-kind.sh
+
+# Run the E2E suite against a cluster provisioned by e2e-setup.
+e2e-run:
+ ./hack/run-e2e.sh $(E2E_ARGS)
+
+# Delete the Kind E2E cluster.
+e2e-cleanup:
+ ./hack/cleanup-e2e-kind.sh
+
clean:
@echo "==> Cleaning build artifacts..."
- rm -rf bin/
+ rm -rf bin/ .tmp/
diff --git a/cmd/ax-server/main.go b/cmd/ax-server/main.go
index 0b5cc4b..967b00b 100644
--- a/cmd/ax-server/main.go
+++ b/cmd/ax-server/main.go
@@ -46,6 +46,7 @@ func main() {
substratePlaintext bool
defaultTemplate string
defaultTemplateAtespace string
+ defaultTaskImage string
)
flag.StringVar(&listenAddr, "addr", ":8080", "HTTP listen address")
@@ -59,6 +60,7 @@ func main() {
flag.BoolVar(&substratePlaintext, "substrate-plaintext", false, "Use insecure plaintext gRPC connection to Substrate")
flag.StringVar(&defaultTemplate, "template", "default-template", "Default Substrate ActorTemplate name")
flag.StringVar(&defaultTemplateAtespace, "template-atespace", v1alpha1.DefaultSystemNamespace, "Default Substrate ActorTemplate atespace")
+ flag.StringVar(&defaultTaskImage, "default-task-image", v1alpha1.DefaultTaskImage, "Container image for tasks that do not set spec.image (env: AX_DEFAULT_TASK_IMAGE)")
flag.Parse()
if envAddr := os.Getenv("ADDR"); envAddr != "" {
@@ -70,6 +72,9 @@ func main() {
if envPass := os.Getenv("REDIS_PASSWORD"); envPass != "" {
redisPassword = envPass
}
+ if envImage := os.Getenv("AX_DEFAULT_TASK_IMAGE"); envImage != "" {
+ defaultTaskImage = envImage
+ }
logger := slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelInfo}))
slog.SetDefault(logger)
@@ -79,6 +84,7 @@ func main() {
"redisAddr", redisAddr,
"substrateEndpoint", substrateEndpoint,
"template", defaultTemplate,
+ "defaultTaskImage", defaultTaskImage,
)
rClient := goredis.NewClient(&goredis.Options{
@@ -103,7 +109,7 @@ func main() {
slog.Warn("could not initialize substrate client; running without substrate reconciliation", "error", err)
} else {
defer subClient.Close()
- reconciler = controller.NewTaskReconciler(subClient, defaultTemplate, defaultTemplateAtespace)
+ reconciler = controller.NewTaskReconciler(subClient, defaultTemplate, defaultTemplateAtespace, defaultTaskImage)
}
srv := server.NewServer(rStore, server.Options{
diff --git a/hack/cleanup-e2e-kind.sh b/hack/cleanup-e2e-kind.sh
new file mode 100755
index 0000000..aefb0e0
--- /dev/null
+++ b/hack/cleanup-e2e-kind.sh
@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Deletes the Kind cluster created for AX E2E tests.
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
+TMP_DIR="${REPO_ROOT}/.tmp"
+
+if [[ -z "${KIND_CLUSTER_NAME:-}" && -f "${TMP_DIR}/e2e.env" ]]; then
+ # shellcheck disable=SC1091
+ source "${TMP_DIR}/e2e.env"
+fi
+
+KIND_CLUSTER_NAME="${KIND_CLUSTER_NAME:-ax-e2e}"
+echo "==> Deleting Kind cluster '${KIND_CLUSTER_NAME}'..."
+kind delete cluster --name "${KIND_CLUSTER_NAME}" || true
+rm -f "${TMP_DIR}/kubeconfig-${KIND_CLUSTER_NAME}" "${TMP_DIR}/e2e.env"
diff --git a/hack/run-e2e-kind.sh b/hack/run-e2e-kind.sh
new file mode 100755
index 0000000..dc8bb91
--- /dev/null
+++ b/hack/run-e2e-kind.sh
@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Provisions a Kind cluster with Substrate (at SUBSTRATE_REF, default: go.mod version)
+# and AX, then runs the AX E2E test suite.
+#
+# Set SKIP_SETUP=true to reuse an already set up cluster.
+# Set CLEANUP_KIND=true to delete the Kind cluster on exit.
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+if [[ "${CLEANUP_KIND:-false}" == "true" ]]; then
+ trap '"${SCRIPT_DIR}/cleanup-e2e-kind.sh"' EXIT
+fi
+
+if [[ "${SKIP_SETUP:-false}" != "true" ]]; then
+ export KIND_CLUSTER_NAME="${KIND_CLUSTER_NAME:-ax-e2e}"
+ "${SCRIPT_DIR}/setup-e2e-kind.sh"
+fi
+
+"${SCRIPT_DIR}/run-e2e.sh" "$@"
diff --git a/hack/run-e2e.sh b/hack/run-e2e.sh
new file mode 100755
index 0000000..0ed5756
--- /dev/null
+++ b/hack/run-e2e.sh
@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Runs the AX Go E2E test suite against an already-provisioned Kind cluster.
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
+TMP_DIR="${REPO_ROOT}/.tmp"
+
+REQUESTED_AX_BIN="${AX_BIN:-}"
+if [[ -f "${TMP_DIR}/e2e.env" ]]; then
+ # shellcheck disable=SC1091
+ source "${TMP_DIR}/e2e.env"
+fi
+
+# Rebuild the host ax CLI so tests always exercise the current source, unless
+# AX_BIN points at a custom binary.
+DEFAULT_AX_BIN="${TMP_DIR}/bin/ax"
+export AX_BIN="${REQUESTED_AX_BIN:-${AX_BIN:-${DEFAULT_AX_BIN}}}"
+if [[ "${AX_BIN}" == "${DEFAULT_AX_BIN}" ]]; then
+ mkdir -p "$(dirname "${AX_BIN}")"
+ (cd "${REPO_ROOT}" && CGO_ENABLED=0 go build -o "${AX_BIN}" ./cmd/ax)
+fi
+
+cd "${REPO_ROOT}"
+exec go test -v -count=1 -timeout "${E2E_TIMEOUT:-30m}" ./test/e2e/... -args --e2e "$@"
diff --git a/hack/setup-e2e-kind.sh b/hack/setup-e2e-kind.sh
new file mode 100755
index 0000000..e9f7fbc
--- /dev/null
+++ b/hack/setup-e2e-kind.sh
@@ -0,0 +1,284 @@
+#!/usr/bin/env bash
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Sets up a Kind cluster with Substrate (at a configurable branch/ref) and AX
+# deployed for end-to-end testing. Re-running the script reuses the existing
+# cluster and only reinstalls Substrate when the requested commit changes.
+#
+# Configurable environment variables:
+# SUBSTRATE_REF Branch, tag, commit, or Go module version of substrate (default: version in go.mod)
+# SUBSTRATE_REPO Git repository URL for substrate (default: https://github.com/agent-substrate/substrate.git)
+# SUBSTRATE_DIR Existing substrate checkout (default: ../substrate if present, else clone to .tmp/substrate)
+# KIND_CLUSTER_NAME Name of the Kind cluster (default: ax-e2e)
+# KUBECONFIG Path to kubeconfig file (default: .tmp/kubeconfig-${KIND_CLUSTER_NAME})
+# AX_SYSTEM_NAMESPACE Kubernetes namespace for AX control plane (default: ax-system)
+# AX_WORKER_REPLICAS Number of gVisor worker replicas in the AX WorkerPool (default: 4)
+# AX_SNAPSHOTS_BUCKET Snapshot storage bucket URI (default: gs://ate-snapshots/ax/)
+# REGISTRY_ADDR Local container registry address (default: localhost:5001)
+# RECREATE_KIND Set to true to delete and recreate the Kind cluster
+# FORCE_INSTALL_SUBSTRATE Set to true to reinstall Substrate even if the commit is unchanged
+
+set -euo pipefail
+export GOWORK=off
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
+TMP_DIR="${REPO_ROOT}/.tmp"
+E2E_DIR="${REPO_ROOT}/test/e2e"
+mkdir -p "${TMP_DIR}/bin"
+
+export KIND_CLUSTER_NAME="${KIND_CLUSTER_NAME:-ax-e2e}"
+export KUBECONFIG="${KUBECONFIG:-${TMP_DIR}/kubeconfig-${KIND_CLUSTER_NAME}}"
+export KUBE_CONTEXT="kind-${KIND_CLUSTER_NAME}"
+export AX_SYSTEM_NAMESPACE="${AX_SYSTEM_NAMESPACE:-ax-system}"
+export AX_WORKER_REPLICAS="${AX_WORKER_REPLICAS:-4}"
+export AX_SNAPSHOTS_BUCKET="${AX_SNAPSHOTS_BUCKET:-gs://ate-snapshots/ax/}"
+export REGISTRY_ADDR="${REGISTRY_ADDR:-localhost:5001}"
+SUBSTRATE_REPO="${SUBSTRATE_REPO:-https://github.com/agent-substrate/substrate.git}"
+GOARCH="$(go env GOARCH)"
+
+# Default to the Substrate version AX is built against. Go pseudo-versions
+SUBSTRATE_REF="${SUBSTRATE_REF:-$(cd "${REPO_ROOT}" && go list -m -f '{{.Version}}' github.com/agent-substrate/substrate)}"
+if [[ "${SUBSTRATE_REF}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+-.*-([0-9a-f]{12})$ ]]; then
+ SUBSTRATE_REF="${BASH_REMATCH[1]}"
+fi
+
+# Use an isolated DOCKER_CONFIG by default so expired gcloud credential helpers
+# in ~/.docker/config.json do not break pulling public base images or pushing
+# to the local registry.
+if [[ -z "${DOCKER_CONFIG:-}" ]]; then
+ export DOCKER_CONFIG="${TMP_DIR}/docker-config"
+ mkdir -p "${DOCKER_CONFIG}"
+ [[ -f "${DOCKER_CONFIG}/config.json" ]] || printf '{}\n' > "${DOCKER_CONFIG}/config.json"
+fi
+
+log() {
+ printf '\n==> %s\n' "$*" >&2
+}
+
+k() {
+ kubectl --kubeconfig "${KUBECONFIG}" --context "${KUBE_CONTEXT}" "$@"
+}
+
+# render_manifest substitutes only the AX_* variables below so any other `$`
+# in the manifests is left untouched.
+MANIFEST_VARS=(AX_SYSTEM_NAMESPACE AX_WORKER_REPLICAS AX_SERVER_IMAGE AX_DEFAULT_TASK_IMAGE AX_SNAPSHOTS_BUCKET)
+render_manifest() {
+ local manifest="${E2E_DIR}/manifests/$1"
+ if command -v envsubst > /dev/null 2>&1; then
+ envsubst "$(printf '${%s} ' "${MANIFEST_VARS[@]}")" < "${manifest}"
+ else
+ local sed_args=() var
+ for var in "${MANIFEST_VARS[@]}"; do
+ sed_args+=(-e "s|\${${var}}|${!var:-}|g")
+ done
+ sed "${sed_args[@]}" "${manifest}"
+ fi
+}
+
+# resolve_commit prints the commit SHA that ref names in the git repo at dir,
+# trying the ref as-is, as a vX.Y.Z tag, and as a release branch, both locally
+# and on origin. Returns non-zero if nothing matches.
+resolve_commit() {
+ local dir="$1" ref="$2" candidate
+ for candidate in "${ref}" "v${ref}" "release-${ref}" "origin/${ref}" "origin/v${ref}" "origin/release-${ref}"; do
+ if git -C "${dir}" rev-parse --verify --quiet "${candidate}^{commit}" 2> /dev/null; then
+ return 0
+ fi
+ done
+ return 1
+}
+
+# resolve_substrate_dir prints a directory containing substrate checked out at
+# SUBSTRATE_REF. It never modifies the HEAD of a user-provided checkout: if
+# HEAD differs from the requested ref, a detached worktree under .tmp is used.
+resolve_substrate_dir() {
+ local base_dir="${SUBSTRATE_DIR:-}"
+ if [[ -n "${base_dir}" ]]; then
+ base_dir="$(cd "${base_dir}" && pwd)"
+ elif [[ -d "${REPO_ROOT}/../substrate/.git" ]]; then
+ base_dir="$(cd "${REPO_ROOT}/../substrate" && pwd)"
+ fi
+
+ if [[ -z "${base_dir}" ]]; then
+ base_dir="${TMP_DIR}/substrate"
+ if [[ ! -d "${base_dir}/.git" ]]; then
+ log "Cloning substrate from ${SUBSTRATE_REPO}..."
+ git clone --quiet "${SUBSTRATE_REPO}" "${base_dir}" >&2
+ fi
+ fi
+ [[ -e "${base_dir}/.git" ]] || { echo "SUBSTRATE_DIR=${base_dir} is not a git checkout" >&2; return 1; }
+
+ local target_sha
+ if [[ "${base_dir}" == "${TMP_DIR}/substrate" ]] &&
+ git -C "${base_dir}" show-ref --verify --quiet "refs/remotes/origin/${SUBSTRATE_REF}"; then
+ # Refresh branches in the managed clone without moving its HEAD or a user checkout.
+ log "Fetching branch ${SUBSTRATE_REF} in ${base_dir}..."
+ git -C "${base_dir}" fetch --quiet origin "+refs/heads/${SUBSTRATE_REF}:refs/remotes/origin/${SUBSTRATE_REF}" >&2 || return 1
+ target_sha="$(git -C "${base_dir}" rev-parse "refs/remotes/origin/${SUBSTRATE_REF}^{commit}")"
+ elif ! target_sha="$(resolve_commit "${base_dir}" "${SUBSTRATE_REF}")"; then
+ log "Fetching ${SUBSTRATE_REF} in ${base_dir}..."
+ git -C "${base_dir}" fetch --quiet --tags origin "${SUBSTRATE_REF}" >&2 ||
+ git -C "${base_dir}" fetch --quiet --all --tags >&2
+ target_sha="$(resolve_commit "${base_dir}" "${SUBSTRATE_REF}")" ||
+ { echo "cannot resolve SUBSTRATE_REF=${SUBSTRATE_REF} in ${base_dir}" >&2; return 1; }
+ fi
+
+ if [[ "$(git -C "${base_dir}" rev-parse HEAD)" == "${target_sha}" ]]; then
+ printf '%s\n' "${base_dir}"
+ return 0
+ fi
+
+ local wt_dir="${TMP_DIR}/substrate-worktree-${target_sha:0:12}"
+ if [[ ! -d "${wt_dir}" ]]; then
+ log "Creating detached substrate worktree at ${SUBSTRATE_REF} (${target_sha:0:12})..."
+ git -C "${base_dir}" worktree prune >&2 || true
+ git -C "${base_dir}" worktree add --quiet --detach "${wt_dir}" "${target_sha}" >&2
+ fi
+ # Share substrate's tool cache with the worktree to avoid re-downloading tools.
+ if [[ -d "${base_dir}/.cache" && ! -e "${wt_dir}/.cache" && ! -L "${wt_dir}/.cache" ]]; then
+ ln -s "${base_dir}/.cache" "${wt_dir}/.cache"
+ fi
+ printf '%s\n' "${wt_dir}"
+}
+
+# uninstall_substrate removes Substrate and the cluster-scoped resources it
+# owns so that a different Substrate version can be installed cleanly.
+uninstall_substrate() {
+ local ax_ns="${AX_SYSTEM_NAMESPACE}"
+ k -n "${ax_ns}" delete workerpool ax-workers --ignore-not-found --timeout=30s > /dev/null 2>&1 ||
+ k -n "${ax_ns}" patch workerpool ax-workers --type=merge -p '{"metadata":{"finalizers":null}}' > /dev/null 2>&1 || true
+ k delete validatingadmissionpolicybinding,validatingadmissionpolicy -l app.kubernetes.io/part-of=ate --ignore-not-found > /dev/null 2>&1 || true
+ k delete validatingadmissionpolicybinding,validatingadmissionpolicy sandboxconfig-assets --ignore-not-found > /dev/null 2>&1 || true
+ if ! k delete sandboxconfig --all --ignore-not-found --timeout=30s > /dev/null 2>&1; then
+ local configs
+ configs=$(k get sandboxconfig -o name 2> /dev/null || true)
+ if [[ -n "${configs}" ]]; then
+ echo "${configs}" | xargs kubectl --kubeconfig "${KUBECONFIG}" --context "${KUBE_CONTEXT}" patch --type=merge -p '{"metadata":{"finalizers":null}}' > /dev/null 2>&1 || true
+ fi
+ fi
+ local crds
+ crds=$(k get crd -o name 2> /dev/null | grep '\.ate\.dev$' || true)
+ if [[ -n "${crds}" ]]; then
+ echo "${crds}" | xargs kubectl --kubeconfig "${KUBECONFIG}" --context "${KUBE_CONTEXT}" delete --ignore-not-found > /dev/null 2>&1 || true
+ fi
+ k delete namespace ate-system --ignore-not-found --wait=true > /dev/null 2>&1 || true
+ k label nodes --all ate.dev/substrate-version- > /dev/null 2>&1 || true
+}
+
+# build_and_push builds an image, pushes it to the local registry, and prints
+# its digest reference.
+build_and_push() {
+ local tag="$1"
+ shift
+ docker build -q -t "${tag}" "$@" "${REPO_ROOT}" > /dev/null
+ docker push -q "${tag}" > /dev/null
+ docker inspect --format='{{index .RepoDigests 0}}' "${tag}"
+}
+
+SUBSTRATE_WORK_DIR="$(resolve_substrate_dir)"
+SUBSTRATE_COMMIT="$(git -C "${SUBSTRATE_WORK_DIR}" rev-parse HEAD)"
+log "Using Substrate checkout at ${SUBSTRATE_WORK_DIR} (ref=${SUBSTRATE_REF}, commit=${SUBSTRATE_COMMIT:0:12})"
+
+# 1. Ensure the Kind cluster and local registry exist.
+if [[ "${RECREATE_KIND:-false}" == "true" ]] || ! kind get clusters 2> /dev/null | grep -qx "${KIND_CLUSTER_NAME}"; then
+ log "Creating Kind cluster '${KIND_CLUSTER_NAME}' and registry '${REGISTRY_ADDR}'..."
+ (cd "${SUBSTRATE_WORK_DIR}" && ./hack/create-kind-cluster.sh)
+else
+ log "Reusing Kind cluster '${KIND_CLUSTER_NAME}' (set RECREATE_KIND=true to recreate)."
+fi
+kind export kubeconfig --name "${KIND_CLUSTER_NAME}" --kubeconfig "${KUBECONFIG}"
+
+# 2. Install Substrate unless the requested commit is already installed.
+INSTALLED_SUBSTRATE_COMMIT="$(k get namespace ate-system -o jsonpath='{.metadata.annotations.substrate-commit}' 2> /dev/null || true)"
+if [[ "${FORCE_INSTALL_SUBSTRATE:-false}" == "true" ]] ||
+ [[ "${INSTALLED_SUBSTRATE_COMMIT}" != "${SUBSTRATE_COMMIT}" ]] ||
+ ! k -n ate-system get deployment ate-api-server > /dev/null 2>&1; then
+ log "Installing Substrate (ate-system) at ${SUBSTRATE_COMMIT:0:12}..."
+ if [[ -n "${INSTALLED_SUBSTRATE_COMMIT}" ]] || k get namespace ate-system > /dev/null 2>&1; then
+ uninstall_substrate
+ fi
+
+ install_args=(--deploy-ate-system --rollout-timeout="${SUBSTRATE_ROLLOUT_TIMEOUT:-3m}")
+ # Older Substrate versions do not have --credential-provider.
+ if grep -q -- "--credential-provider" "${SUBSTRATE_WORK_DIR}/hack/install-ate.sh" 2> /dev/null; then
+ install_args+=(--credential-provider='{"enabled":false}')
+ fi
+ (cd "${SUBSTRATE_WORK_DIR}" && ./hack/install-ate-kind.sh "${install_args[@]}")
+ k annotate namespace ate-system "substrate-commit=${SUBSTRATE_COMMIT}" --overwrite > /dev/null
+else
+ log "Substrate ${SUBSTRATE_COMMIT:0:12} already installed; waiting for ate-api-server..."
+ k -n ate-system wait --for=condition=Available deployment/ate-api-server --timeout=180s
+fi
+
+# 3. Deploy the AX namespace and gVisor WorkerPool.
+log "Deploying namespace '${AX_SYSTEM_NAMESPACE}' and WorkerPool (${AX_WORKER_REPLICAS} gVisor replicas)..."
+render_manifest namespace.yaml | k apply -f -
+(
+ cd "${SUBSTRATE_WORK_DIR}"
+ render_manifest workerpool.yaml |
+ KO_DOCKER_REPO="${REGISTRY_ADDR}" ./hack/run-tool.sh ko resolve --platform="linux/${GOARCH}" -f -
+) | k apply -f -
+
+log "Waiting for ax-workers to become Ready..."
+for _ in $(seq 1 60); do
+ k -n "${AX_SYSTEM_NAMESPACE}" get deployment ax-workers > /dev/null 2>&1 && break
+ sleep 2
+done
+k -n "${AX_SYSTEM_NAMESPACE}" rollout status deployment/ax-workers --timeout=180s
+
+# 4. Build AX binaries and images. ax-server and ax-task-runner run in the
+# cluster (linux); the ax CLI runs on the host.
+log "Building ax (host), ax-server and ax-task-runner (linux/${GOARCH})..."
+(
+ cd "${REPO_ROOT}"
+ mkdir -p "bin/linux_${GOARCH}"
+ CGO_ENABLED=0 go build -o "${TMP_DIR}/bin/ax" ./cmd/ax
+ CGO_ENABLED=0 GOOS=linux GOARCH="${GOARCH}" go build -o "bin/linux_${GOARCH}/ax-server" ./cmd/ax-server
+ CGO_ENABLED=0 GOOS=linux GOARCH="${GOARCH}" go build -o "bin/linux_${GOARCH}/ax-task-runner" ./cmd/ax-task-runner
+)
+
+log "Building and pushing images to ${REGISTRY_ADDR}..."
+AX_SERVER_IMAGE="$(build_and_push "${REGISTRY_ADDR}/ax-server:e2e" \
+ --build-arg TARGETARCH="${GOARCH}" -f "${E2E_DIR}/Dockerfile.server")"
+# The E2E task-runner image extends the production image with test fixtures.
+docker build -q -t "${REGISTRY_ADDR}/ax-task-runner:base" \
+ --build-arg TARGETARCH="${GOARCH}" -f "${REPO_ROOT}/Dockerfile.task-runner" "${REPO_ROOT}" > /dev/null
+AX_DEFAULT_TASK_IMAGE="$(build_and_push "${REGISTRY_ADDR}/ax-task-runner:e2e" \
+ --build-arg BASE_IMAGE="${REGISTRY_ADDR}/ax-task-runner:base" -f "${E2E_DIR}/Dockerfile.task-runner.e2e")"
+export AX_SERVER_IMAGE AX_DEFAULT_TASK_IMAGE
+printf ' AX_SERVER_IMAGE=%s\n AX_DEFAULT_TASK_IMAGE=%s\n' "${AX_SERVER_IMAGE}" "${AX_DEFAULT_TASK_IMAGE}"
+
+# 5. Deploy Redis and ax-server. Delete existing deployments first so every
+# run starts from empty Redis state.
+log "Deploying ax-redis and ax-server in namespace '${AX_SYSTEM_NAMESPACE}'..."
+k -n "${AX_SYSTEM_NAMESPACE}" delete deployment ax-redis ax-server --ignore-not-found
+render_manifest redis.yaml | k apply -f -
+render_manifest ax-server.yaml | k apply -f -
+k -n "${AX_SYSTEM_NAMESPACE}" rollout status deployment/ax-redis --timeout=120s
+k -n "${AX_SYSTEM_NAMESPACE}" rollout status deployment/ax-server --timeout=120s
+
+# 6. Persist the environment for hack/run-e2e.sh and the Go test harness.
+cat > "${TMP_DIR}/e2e.env" << EOF
+export KIND_CLUSTER_NAME="${KIND_CLUSTER_NAME}"
+export KUBECONFIG="${KUBECONFIG}"
+export KUBE_CONTEXT="${KUBE_CONTEXT}"
+export AX_SYSTEM_NAMESPACE="${AX_SYSTEM_NAMESPACE}"
+export AX_BIN="${TMP_DIR}/bin/ax"
+export SUBSTRATE_REF="${SUBSTRATE_REF}"
+export SUBSTRATE_COMMIT="${SUBSTRATE_COMMIT}"
+EOF
+
+log "Kind E2E environment is ready (configuration written to ${TMP_DIR}/e2e.env)."
diff --git a/internal/controller/reconciler.go b/internal/controller/reconciler.go
index 5e31e1d..074a481 100644
--- a/internal/controller/reconciler.go
+++ b/internal/controller/reconciler.go
@@ -63,6 +63,8 @@ type TaskReconciler struct {
httpClient *http.Client
defaultTemplate string
defaultTemplateAtespace string
+ // defaultTaskImage is applied to spec.image for tasks that leave it empty.
+ defaultTaskImage string
// SecretResolver resolves the Gemini API key for task containers. It defaults
// to the Kubernetes secret lookup; tests replace it to avoid touching a cluster.
@@ -73,19 +75,25 @@ type TaskReconciler struct {
WorkspaceReadyTimeout time.Duration
}
-// NewTaskReconciler creates a new TaskReconciler.
-func NewTaskReconciler(client *substrate.Client, defaultTemplate, defaultTemplateAtespace string) *TaskReconciler {
+// NewTaskReconciler creates a new TaskReconciler. Empty defaults fall back to
+// "default-template", v1alpha1.DefaultSystemNamespace and
+// v1alpha1.DefaultTaskImage respectively.
+func NewTaskReconciler(client *substrate.Client, defaultTemplate, defaultTemplateAtespace, defaultTaskImage string) *TaskReconciler {
if defaultTemplate == "" {
defaultTemplate = "default-template"
}
if defaultTemplateAtespace == "" {
defaultTemplateAtespace = v1alpha1.DefaultSystemNamespace
}
+ if defaultTaskImage == "" {
+ defaultTaskImage = v1alpha1.DefaultTaskImage
+ }
return &TaskReconciler{
client: client,
httpClient: &http.Client{Timeout: 2 * time.Second},
defaultTemplate: defaultTemplate,
defaultTemplateAtespace: defaultTemplateAtespace,
+ defaultTaskImage: defaultTaskImage,
SecretResolver: model.GetKubernetesSecret,
WorkspaceReadyTimeout: defaultWorkspaceReadyTimeout,
}
@@ -109,7 +117,7 @@ func (r *TaskReconciler) Reconcile(ctx context.Context, task *v1alpha1.Task, ws
task.Spec = &v1alpha1.TaskSpec{}
}
if task.Spec.Image == "" {
- task.Spec.Image = v1alpha1.DefaultTaskImage
+ task.Spec.Image = r.defaultTaskImage
}
slog.Info("reconciling task", "name", task.Metadata.Name, "atespace", atespace, "image", task.Spec.Image)
diff --git a/internal/controller/reconciler_test.go b/internal/controller/reconciler_test.go
index 37842be..1ca0000 100644
--- a/internal/controller/reconciler_test.go
+++ b/internal/controller/reconciler_test.go
@@ -45,6 +45,7 @@ type mockControlServer struct {
deletedActors []string
actorTemplates map[string]bool
createdTemplates []*ateapipb.ActorTemplate
+ templateImages []string
deletedTemplates []string
getActorFunc func(ctx context.Context, req *ateapipb.GetActorRequest) (*ateapipb.Actor, error)
createAtespaceErr error
@@ -80,6 +81,9 @@ func (m *mockControlServer) CreateActorTemplate(_ context.Context, req *ateapipb
tmpl := req.GetActorTemplate()
m.actorTemplates[tmpl.GetMetadata().GetName()] = true
m.createdTemplates = append(m.createdTemplates, tmpl)
+ for _, c := range tmpl.GetContainers() {
+ m.templateImages = append(m.templateImages, c.GetImage())
+ }
return tmpl, nil
}
@@ -238,7 +242,7 @@ func TestTaskReconciler(t *testing.T) {
defer client.Close()
// 3. Reconcile Task
- reconciler := controller.NewTaskReconciler(client, "test-template", v1alpha1.DefaultSystemNamespace)
+ reconciler := controller.NewTaskReconciler(client, "test-template", v1alpha1.DefaultSystemNamespace, "")
reconciler.SecretResolver = noSecrets
reconciler.WorkspaceReadyTimeout = 200 * time.Millisecond
@@ -286,6 +290,63 @@ func TestTaskReconciler(t *testing.T) {
}
}
+// TestTaskReconciler_DefaultTaskImage verifies that a task without spec.image is
+// defaulted to the image passed to NewTaskReconciler (or the API default when
+// none is given), and that the image
+// is what ends up in the per-task ActorTemplate.
+func TestTaskReconciler_DefaultTaskImage(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ override string
+ want string
+ }{
+ {name: "api default", want: v1alpha1.DefaultTaskImage},
+ {name: "server override", override: "localhost:5001/ax-task-runner:e2e", want: "localhost:5001/ax-task-runner:e2e"},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ lis, err := net.Listen("tcp", "127.0.0.1:0")
+ if err != nil {
+ t.Fatalf("failed to listen: %v", err)
+ }
+ defer lis.Close()
+
+ mockSrv := &mockControlServer{}
+ grpcServer := grpc.NewServer()
+ ateapipb.RegisterControlServer(grpcServer, mockSrv)
+ go grpcServer.Serve(lis)
+ defer grpcServer.Stop()
+
+ client, err := substrate.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials()))
+ if err != nil {
+ t.Fatalf("failed to create substrate client: %v", err)
+ }
+ defer client.Close()
+
+ reconciler := controller.NewTaskReconciler(client, "test-template", v1alpha1.DefaultSystemNamespace, tc.override)
+ reconciler.SecretResolver = noSecrets
+ reconciler.WorkspaceReadyTimeout = 200 * time.Millisecond
+
+ task := &v1alpha1.Task{
+ ApiVersion: v1alpha1.APIVersion,
+ Kind: v1alpha1.KindTask,
+ Metadata: &v1alpha1.ObjectMeta{Name: "no-image", Atespace: "default"},
+ Spec: &v1alpha1.TaskSpec{},
+ Status: &v1alpha1.TaskStatus{Phase: "Running"},
+ }
+ reconciled, err := reconciler.Reconcile(context.Background(), task, nil)
+ if err != nil {
+ t.Fatalf("Reconcile failed: %v", err)
+ }
+ if got := reconciled.GetSpec().GetImage(); got != tc.want {
+ t.Errorf("spec.image = %q, want %q", got, tc.want)
+ }
+ if len(mockSrv.templateImages) != 1 || mockSrv.templateImages[0] != tc.want {
+ t.Errorf("ActorTemplate images = %v, want [%q]", mockSrv.templateImages, tc.want)
+ }
+ })
+ }
+}
+
func TestTaskReconciler_Suspend(t *testing.T) {
ctx := context.Background()
@@ -307,7 +368,7 @@ func TestTaskReconciler_Suspend(t *testing.T) {
}
defer client.Close()
- reconciler := controller.NewTaskReconciler(client, "test-template", v1alpha1.DefaultSystemNamespace)
+ reconciler := controller.NewTaskReconciler(client, "test-template", v1alpha1.DefaultSystemNamespace, "")
reconciler.SecretResolver = noSecrets
reconciler.WorkspaceReadyTimeout = 200 * time.Millisecond
@@ -362,7 +423,7 @@ func TestTaskReconciler_SuspendError(t *testing.T) {
}
defer client.Close()
- reconciler := controller.NewTaskReconciler(client, "test-template", "ax-system")
+ reconciler := controller.NewTaskReconciler(client, "test-template", "ax-system", "")
reconciler.SecretResolver = noSecrets
workerIp := "10.1.1.1"
@@ -421,7 +482,7 @@ func TestTaskReconciler_TemplateCreationFailed(t *testing.T) {
}
defer client.Close()
- reconciler := controller.NewTaskReconciler(client, "test-template", "ax-system")
+ reconciler := controller.NewTaskReconciler(client, "test-template", "ax-system", "")
reconciler.SecretResolver = noSecrets
reconciler.WorkspaceReadyTimeout = 200 * time.Millisecond
@@ -499,7 +560,7 @@ func TestTaskReconciler_TemplateCreationTransient(t *testing.T) {
}
defer client.Close()
- reconciler := controller.NewTaskReconciler(client, "test-template", "ax-system")
+ reconciler := controller.NewTaskReconciler(client, "test-template", "ax-system", "")
reconciler.SecretResolver = noSecrets
reconciler.WorkspaceReadyTimeout = 200 * time.Millisecond
@@ -572,7 +633,7 @@ func TestTaskReconciler_WorkspaceReady(t *testing.T) {
}
defer client.Close()
- reconciler := controller.NewTaskReconciler(client, "test-template", v1alpha1.DefaultSystemNamespace)
+ reconciler := controller.NewTaskReconciler(client, "test-template", v1alpha1.DefaultSystemNamespace, "")
reconciler.SecretResolver = noSecrets
reconciler.WorkspaceReadyTimeout = 200 * time.Millisecond
@@ -666,7 +727,7 @@ func TestTaskReconciler_ResourceLimits(t *testing.T) {
}
defer client.Close()
- reconciler := controller.NewTaskReconciler(client, "test-template", "ax-system")
+ reconciler := controller.NewTaskReconciler(client, "test-template", "ax-system", "")
reconciler.SecretResolver = noSecrets
reconciler.WorkspaceReadyTimeout = 200 * time.Millisecond
@@ -763,7 +824,7 @@ func TestTaskReconciler_ResourceLimitsFailurePaths(t *testing.T) {
grpcServer.Stop()
lis.Close()
})
- reconciler := controller.NewTaskReconciler(client, "test-template", "ax-system")
+ reconciler := controller.NewTaskReconciler(client, "test-template", "ax-system", "")
reconciler.SecretResolver = noSecrets
reconciler.WorkspaceReadyTimeout = 200 * time.Millisecond
return reconciler
@@ -881,7 +942,7 @@ func TestReconcileDelete_RemovesActorAndTemplates(t *testing.T) {
}
defer client.Close()
- reconciler := controller.NewTaskReconciler(client, "test-template", v1alpha1.DefaultSystemNamespace)
+ reconciler := controller.NewTaskReconciler(client, "test-template", v1alpha1.DefaultSystemNamespace, "")
reconciler.SecretResolver = noSecrets
reconciler.WorkspaceReadyTimeout = 200 * time.Millisecond
@@ -944,7 +1005,7 @@ func TestReconcileDelete_BlocksUntilActorDeleted(t *testing.T) {
}
defer client.Close()
- reconciler := controller.NewTaskReconciler(client, "test-template", v1alpha1.DefaultSystemNamespace)
+ reconciler := controller.NewTaskReconciler(client, "test-template", v1alpha1.DefaultSystemNamespace, "")
reconciler.SecretResolver = noSecrets
if err := reconciler.ReconcileDelete(ctx, "default", "slow-delete-task"); err != nil {
@@ -986,7 +1047,7 @@ func TestReconcileDelete_ActorDeletionTimeout(t *testing.T) {
}
defer client.Close()
- reconciler := controller.NewTaskReconciler(client, "test-template", v1alpha1.DefaultSystemNamespace)
+ reconciler := controller.NewTaskReconciler(client, "test-template", v1alpha1.DefaultSystemNamespace, "")
reconciler.SecretResolver = noSecrets
err = reconciler.ReconcileDelete(ctx, "default", "stuck-task")
@@ -1050,7 +1111,7 @@ func TestTaskReconciler_TransientSubstrateUnavailable(t *testing.T) {
}
defer client.Close()
- reconciler := controller.NewTaskReconciler(client, "test-template", v1alpha1.DefaultSystemNamespace)
+ reconciler := controller.NewTaskReconciler(client, "test-template", v1alpha1.DefaultSystemNamespace, "")
reconciler.SecretResolver = noSecrets
reconciler.WorkspaceReadyTimeout = 50 * time.Millisecond
diff --git a/test/e2e/Dockerfile.server b/test/e2e/Dockerfile.server
new file mode 100644
index 0000000..91d61f4
--- /dev/null
+++ b/test/e2e/Dockerfile.server
@@ -0,0 +1,22 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM alpine:3.21
+
+RUN apk add --no-cache ca-certificates
+
+ARG TARGETARCH=amd64
+COPY bin/linux_${TARGETARCH}/ax-server /usr/local/bin/ax-server
+
+ENTRYPOINT ["/usr/local/bin/ax-server"]
diff --git a/test/e2e/Dockerfile.task-runner.e2e b/test/e2e/Dockerfile.task-runner.e2e
new file mode 100644
index 0000000..6f65239
--- /dev/null
+++ b/test/e2e/Dockerfile.task-runner.e2e
@@ -0,0 +1,36 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG BASE_IMAGE=ax-task-runner:base
+FROM ${BASE_IMAGE}
+
+# Initialize the sample Git repository used by E2E tests
+RUN mkdir -p /workspace /tmp/sample-src && \
+ cd /tmp/sample-src && \
+ git init -b main && \
+ git config user.name "AX E2E" && \
+ git config user.email "[email protected]" && \
+ echo "# Sample E2E Repo" > README.md && \
+ echo "print('hello from git repo v1')" > app.py && \
+ git add README.md app.py && \
+ git commit -m "Initial commit" && \
+ git tag v1.0.0 && \
+ echo "print('hello from git repo main')" > app.py && \
+ git commit -am "Second commit on main" && \
+ git clone --bare /tmp/sample-src /opt/ax-e2e-sample-repo.git && \
+ rm -rf /tmp/sample-src
+
+WORKDIR /workspace
+EXPOSE 80
+ENTRYPOINT ["/usr/local/bin/ax-task-runner"]
diff --git a/test/e2e/README.md b/test/e2e/README.md
new file mode 100644
index 0000000..5754129
--- /dev/null
+++ b/test/e2e/README.md
@@ -0,0 +1,69 @@
+# AX End-to-End (E2E) Tests
+
+This directory contains the end-to-end test suite for **AX** running on
+**Substrate** in a local [Kind](https://kind.sigs.k8s.io/) cluster, along with
+the manifests and Dockerfiles used to deploy AX for the tests.
+
+The suite exercises real gVisor sandboxes, Substrate's `ate-api-server`,
+in-cluster RustFS snapshot storage, Redis, `ax-server`, `ax-task-runner`, and
+the `ax` CLI, against a configurable Substrate branch, tag, or commit.
+
+| Path | Purpose |
+|---|---|
+| `harness_test.go` | `TestMain`, cluster connection setup, and shared helpers |
+| `e2e_test.go` | The test cases |
+| `manifests/` | Namespace, WorkerPool, Redis, and ax-server manifests (rendered with `envsubst`) |
+| `Dockerfile.server` | ax-server image |
+| `Dockerfile.task-runner.e2e` | Extends `Dockerfile.task-runner` with test fixtures (a sample Git repo) |
+
+## Quick Start
+
+Provision a cluster and run the suite against the Substrate version pinned in `go.mod`:
+
+```bash
+make test-e2e
+```
+
+Against Substrate `main`, or any branch, tag, or commit:
+
+```bash
+make test-e2e-head
+make test-e2e SUBSTRATE_REF=release-0.4
+```
+
+### Iterating
+
+Provisioning takes several minutes; reuse the cluster between runs:
+
+```bash
+make e2e-setup # provision Kind + Substrate + AX (re-run after changing server code)
+make e2e-run # run all tests
+make e2e-run E2E_ARGS='-test.run TestE2E_ModelCRUD'
+make e2e-cleanup # delete the Kind cluster
+```
+
+`make e2e-run` rebuilds the `ax` CLI, but not the in-cluster `ax-server` or
+`ax-task-runner` images; re-run `make e2e-setup` to redeploy those.
+
+`go test ./...` skips this package unless `-args --e2e` is passed or `AX_E2E=true` is set.
+
+## Configuration
+
+`hack/setup-e2e-kind.sh` reads these environment variables and writes the
+resolved values the tests need to `.tmp/e2e.env`.
+
+| Variable | Default | Description |
+|---|---|---|
+| `SUBSTRATE_REF` | version in `go.mod` | Substrate branch, tag, commit, or Go module version |
+| `SUBSTRATE_DIR` | `../substrate` if present | Local Substrate checkout. If its `HEAD` is not `SUBSTRATE_REF`, a detached worktree is created under `.tmp/` |
+| `SUBSTRATE_REPO` | `https://github.com/agent-substrate/substrate.git` | Cloned into `.tmp/substrate` when no local checkout exists |
+| `KIND_CLUSTER_NAME` | `ax-e2e` | Kind cluster name |
+| `KUBECONFIG` | `.tmp/kubeconfig-${KIND_CLUSTER_NAME}` | Kubeconfig for the Kind cluster |
+| `AX_SYSTEM_NAMESPACE` | `ax-system` | Namespace for `ax-redis`, `ax-server`, and the `ax-workers` WorkerPool |
+| `AX_WORKER_REPLICAS` | `4` | gVisor worker replicas |
+| `AX_SNAPSHOTS_BUCKET` | `gs://ate-snapshots/ax/` | Snapshot location in Substrate's in-cluster RustFS |
+| `RECREATE_KIND` | `false` | Delete and recreate the Kind cluster |
+| `FORCE_INSTALL_SUBSTRATE` | `false` | Reinstall Substrate even if the requested commit is already installed |
+| `CLEANUP_KIND` | `false` | `hack/run-e2e-kind.sh` only: delete the cluster on exit |
+| `SKIP_SETUP` | `false` | `hack/run-e2e-kind.sh` only: skip provisioning |
+| `E2E_TIMEOUT` | `30m` | `go test` timeout |
diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go
new file mode 100644
index 0000000..fc9fffd
--- /dev/null
+++ b/test/e2e/e2e_test.go
@@ -0,0 +1,564 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package e2e_test
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/agent-substrate/substrate/pkg/proto/ateapipb"
+ "github.com/google/ax/pkg/apis/v1alpha1"
+ "google.golang.org/grpc/codes"
+ "google.golang.org/grpc/status"
+)
+
+const (
+ defaultAtespace = "default"
+
+ phaseSuspended = "Suspended"
+ phaseRunning = "Running"
+
+ // sampleRepo is a bare Git repository baked into the E2E task-runner image
+ // (see test/e2e/Dockerfile.task-runner.e2e). Tag v1.0.0 contains
+ // app.py printing "hello from git repo v1".
+ sampleRepo = "file:///opt/ax-e2e-sample-repo.git"
+
+ // probeURL is the in-sandbox address of the ax-task-runner probe server.
+ probeURL = "http://127.0.0.1:80"
+
+ // metadataURL is the in-sandbox address of the ax-task-runner metadata server.
+ metadataURL = "http://127.0.0.1:8099"
+)
+
+// TestE2E_CLIVersionAndAutoTunnel verifies `ax version` and that, without
+// --server, the CLI automatically opens a port-forward tunnel to ax-server
+// that `ax ctx` reports as healthy and `ax tunnel stop` tears down.
+func TestE2E_CLIVersionAndAutoTunnel(t *testing.T) {
+ assertContains(t, "'ax version'", env.mustRunAX(t, "", "version"), "ax version")
+
+ // Without --server, ax should auto-create a port-forward tunnel to
+ // ax-server using the kube context. Use a dedicated AX_HOME so the tunnel
+ // state is isolated from the rest of the suite.
+ axHome := t.TempDir()
+ runTunneled := func(args ...string) string {
+ t.Helper()
+ cmd, cancel := env.axCommand(axHome, append([]string{"--context", env.KubeContext, "-n", env.AXSystemNamespace}, args...)...)
+ defer cancel()
+ out, err := cmd.CombinedOutput()
+ if err != nil {
+ t.Fatalf("ax %s failed: %v\nOutput:\n%s", strings.Join(args, " "), err, out)
+ }
+ return string(out)
+ }
+ t.Cleanup(func() {
+ cmd, cancel := env.axCommand(axHome, "tunnel", "stop", env.KubeContext)
+ defer cancel()
+ _ = cmd.Run()
+ })
+
+ runTunneled("get", "tasks")
+
+ listOut := runTunneled("ctx")
+ assertContains(t, "'ax ctx'", strings.ToLower(listOut), strings.ToLower(env.KubeContext), "healthy")
+
+ runTunneled("tunnel", "stop", env.KubeContext)
+ assertContains(t, "'ax ctx' after stop", runTunneled("ctx"), "No background tunnel running")
+}
+
+const workspaceCRUDManifest = `apiVersion: ax.io/v1alpha1
+kind: Workspace
+metadata:
+ name: {{.Name}}
+ atespace: {{.Atespace}}
+spec:
+ git:
+ - name: origin
+ repo: {{.Repo}}
+ branch: v1.0.0
+ files:
+ - path: config/settings.json
+ content: '{"env":"{{.Env}}"}'
+ mcp:
+ servers:
+ - name: local-mcp
+ endpoint: http://localhost:9000
+ skills:
+ path: /workspace/.agents/skills
+`
+
+// TestE2E_WorkspaceCRUD verifies the Workspace lifecycle through the CLI:
+// `ax apply` reports created / unchanged / configured, `ax get` and
+// `ax describe` show the updated spec, and `ax delete` removes it.
+func TestE2E_WorkspaceCRUD(t *testing.T) {
+ ns := defaultAtespace
+ name := uniqueName("ws-crud")
+ env.deleteOnCleanup(t, ns, "workspace", name)
+
+ manifest := func(envValue string) string {
+ return renderYAML(t, workspaceCRUDManifest, map[string]string{
+ "Name": name, "Atespace": ns, "Repo": sampleRepo, "Env": envValue,
+ })
+ }
+ ref := "workspace.ax.io/" + name
+
+ assertContains(t, "first apply", env.apply(t, ns, manifest("e2e-v1")), ref+" created")
+ assertContains(t, "identical re-apply", env.apply(t, ns, manifest("e2e-v1")), ref+" unchanged")
+ assertContains(t, "modified apply", env.apply(t, ns, manifest("e2e-v2")), ref+" configured")
+
+ assertContains(t, "'ax get workspaces'", env.mustRunAX(t, ns, "get", "workspaces"), name)
+ assertContains(t, "'ax get workspace'", env.mustRunAX(t, ns, "get", "workspace", name),
+ "e2e-v2", "local-mcp", "config/settings.json")
+ assertContains(t, "'ax describe workspace'", env.mustRunAX(t, ns, "describe", "workspace", name),
+ "local-mcp", ".agents/skills")
+
+ assertContains(t, "'ax delete workspace'", env.mustRunAX(t, ns, "delete", "workspace", name), "deleted")
+ _, err := env.AXClient.GetWorkspace(t.Context(), &v1alpha1.GetWorkspaceRequest{Name: name, Atespace: ns})
+ if status.Code(err) != codes.NotFound {
+ t.Fatalf("GetWorkspace after delete: got %v, want NotFound", err)
+ }
+}
+
+const modelCRUDManifest = `apiVersion: ax.io/v1alpha1
+kind: Model
+metadata:
+ name: {{.Name}}
+ atespace: {{.Atespace}}
+spec:
+ provider: google
+ model: {{.Model}}
+ secretKey:
+ name: gemini-secret
+ key: api-key
+`
+
+// TestE2E_ModelCRUD verifies the same apply / get / describe / delete
+// lifecycle as TestE2E_WorkspaceCRUD for a Model.
+func TestE2E_ModelCRUD(t *testing.T) {
+ ns := defaultAtespace
+ name := uniqueName("model-crud")
+ env.deleteOnCleanup(t, ns, "model", name)
+
+ manifest := func(model string) string {
+ return renderYAML(t, modelCRUDManifest, map[string]string{"Name": name, "Atespace": ns, "Model": model})
+ }
+ ref := "model.ax.io/" + name
+
+ assertContains(t, "first apply", env.apply(t, ns, manifest("gemini-2.5-pro")), ref+" created")
+ assertContains(t, "identical re-apply", env.apply(t, ns, manifest("gemini-2.5-pro")), ref+" unchanged")
+ assertContains(t, "modified apply", env.apply(t, ns, manifest("gemini-2.5-flash")), ref+" configured")
+
+ assertContains(t, "'ax get models'", env.mustRunAX(t, ns, "get", "models"), name, "gemini-2.5-flash")
+ assertContains(t, "'ax get model'", env.mustRunAX(t, ns, "get", "model", name), "gemini-2.5-flash")
+ assertContains(t, "'ax describe model'", env.mustRunAX(t, ns, "describe", "model", name), "gemini-secret")
+
+ env.mustRunAX(t, ns, "delete", "model", name)
+ _, err := env.AXClient.GetModel(t.Context(), &v1alpha1.GetModelRequest{Name: name, Atespace: ns})
+ if status.Code(err) != codes.NotFound {
+ t.Fatalf("GetModel after delete: got %v, want NotFound", err)
+ }
+}
+
+const debugTaskManifest = `apiVersion: ax.io/v1alpha1
+kind: Task
+metadata:
+ name: {{.Name}}
+ atespace: {{.Atespace}}
+spec:
+ debug: {{.Debug}}
+`
+
+// taskManifest renders a minimal Task with no workspaces.
+func taskManifest(t *testing.T, atespace, name string, debug bool) string {
+ return renderYAML(t, debugTaskManifest, map[string]any{"Name": name, "Atespace": atespace, "Debug": debug})
+}
+
+// TestE2E_TaskLifecycleAndSubstrateSync verifies that a Task and its Substrate
+// resources stay in sync across its lifecycle:
+// - A new Task starts Suspended and is immutable.
+// - Substrate has a suspended Actor backed by a per-task ActorTemplate
+// (<task>-tmpl-<hash>) with the /readyz:80 wakeup probe and a snapshot config.
+// - Resuming moves the Task to Running with WorkspaceReady and Ready
+// conditions, on the worker Substrate reports.
+// - Deleting the Task removes the Actor and ActorTemplate.
+func TestE2E_TaskLifecycleAndSubstrateSync(t *testing.T) {
+ ctx := t.Context()
+ ns := defaultAtespace
+ name := uniqueName("task-life")
+ env.deleteOnCleanup(t, ns, "task", name)
+
+ manifest := taskManifest(t, ns, name, true)
+
+ // 1. A new task is created Suspended, backed by an Actor of the same name.
+ assertContains(t, "'ax apply'", env.apply(t, ns, manifest), "task.ax.io/"+name+" created")
+ task := env.getTask(t, ns, name)
+ assertPhase(t, task, phaseSuspended)
+ if got := task.GetStatus().GetActor(); got != name {
+ t.Fatalf("task actor = %q, want %q", got, name)
+ }
+
+ // 2. Tasks are immutable: both CreateTask and `ax apply` reject a duplicate.
+ _, err := env.AXClient.CreateTask(ctx, &v1alpha1.CreateTaskRequest{Task: &v1alpha1.Task{
+ Metadata: &v1alpha1.ObjectMeta{Name: name, Atespace: ns},
+ Spec: &v1alpha1.TaskSpec{},
+ }})
+ if status.Code(err) != codes.FailedPrecondition {
+ t.Fatalf("duplicate CreateTask: got %v, want FailedPrecondition", err)
+ }
+ out, err := env.runAX(ns, "apply", "-f", writeManifest(t, manifest))
+ if err == nil {
+ t.Fatalf("expected re-applying an existing task to fail, got:\n%s", out)
+ }
+ assertContains(t, "duplicate 'ax apply'", out, "already exists and is immutable")
+
+ // 3. Substrate has a suspended Actor using a per-task ActorTemplate.
+ actor := env.getActor(t, ns, name, ateapipb.ActorState_ACTOR_STATE_SUSPENDED)
+ tmplName := actor.GetActorTemplate().GetName()
+ if !strings.HasPrefix(tmplName, name+"-tmpl-") {
+ t.Fatalf("actor template = %q, want prefix %q", tmplName, name+"-tmpl-")
+ }
+ tmpl, err := env.SubstrateClient.GetActorTemplate(ctx, ns, tmplName)
+ if err != nil {
+ t.Fatalf("Substrate GetActorTemplate(%s/%s): %v", ns, tmplName, err)
+ }
+ if containers := tmpl.GetContainers(); len(containers) != 1 {
+ t.Fatalf("actor template has %d containers, want 1", len(containers))
+ }
+ if probe := tmpl.GetContainers()[0].GetWakeupProbe().GetHttpGet(); probe.GetPath() != "/readyz" || probe.GetPort() != 80 {
+ t.Fatalf("unexpected WakeupProbe: %+v", tmpl.GetContainers()[0].GetWakeupProbe())
+ }
+ // Newer Substrate versions may return SnapshotConfig fields AX's proto does
+ // not know about, so accept either a known storage location or unknown fields.
+ if sc := tmpl.GetSnapshotConfig(); sc.GetStorageLocation() == "" && len(sc.ProtoReflect().GetUnknown()) == 0 {
+ t.Fatalf("unexpected SnapshotConfig: %+v", sc)
+ }
+
+ // 4. Resuming moves the task to Running on a worker reported by Substrate.
+ assertContains(t, "'ax resume task'", env.mustRunAX(t, ns, "resume", "task", name), "resumed")
+ task = env.getTask(t, ns, name)
+ assertPhase(t, task, phaseRunning)
+ assertCondition(t, task, "WorkspaceReady", "True", "SetupComplete")
+ assertCondition(t, task, "Ready", "True", "TaskRunning")
+ workerIP := task.GetStatus().GetWorkerIp()
+ if workerIP == "" {
+ t.Fatal("expected non-empty WorkerIp on running task")
+ }
+ actor = env.getActor(t, ns, name, ateapipb.ActorState_ACTOR_STATE_RUNNING)
+ if ips := actor.GetStatus().GetWorkerAssignment().GetWorkerPodIps(); len(ips) == 0 || ips[0] != workerIP {
+ t.Fatalf("Substrate WorkerPodIps = %v, want first IP %q", ips, workerIP)
+ }
+
+ // 5. Deleting the task removes the AX task, the Actor, and its ActorTemplate.
+ assertContains(t, "'ax delete task'", env.mustRunAX(t, ns, "delete", "task", name), "deleted")
+ if _, err := env.AXClient.GetTask(ctx, &v1alpha1.GetTaskRequest{Name: name, Atespace: ns}); status.Code(err) != codes.NotFound {
+ t.Fatalf("AX GetTask after delete: got %v, want NotFound", err)
+ }
+ if _, err := env.SubstrateClient.GetActor(ctx, ns, name); status.Code(err) != codes.NotFound {
+ t.Fatalf("Substrate GetActor after delete: got %v, want NotFound", err)
+ }
+ if _, err := env.SubstrateClient.GetActorTemplate(ctx, ns, tmplName); status.Code(err) != codes.NotFound {
+ t.Fatalf("Substrate GetActorTemplate(%s) after delete: got %v, want NotFound", tmplName, err)
+ }
+}
+
+const metadataManifest = `apiVersion: ax.io/v1alpha1
+kind: Workspace
+metadata:
+ name: {{.Workspace}}
+ atespace: {{.Atespace}}
+spec:
+ files:
+ - path: hello.txt
+ content: "hello from workspace"
+---
+apiVersion: ax.io/v1alpha1
+kind: Task
+metadata:
+ name: {{.Task}}
+ atespace: {{.Atespace}}
+spec:
+ debug: true
+ workspace:
+ name: {{.Workspace}}
+ path: /workspace
+ env:
+ - name: CUSTOM_E2E_VAR
+ value: "custom-val-123"
+ command:
+ - sh
+ - -c
+ - "echo cmd-ran-ok > /workspace/cmd-ran.txt"
+`
+
+// TestE2E_MetadataServerAndGuestExec verifies the guest environment of a
+// running task: `ax ssh` sees spec.env, AX_METADATA_URL, and workspace files;
+// spec.command runs; the in-sandbox metadata server serves health and
+// task/workspace metadata; guest exit codes propagate; and `ax ssh` is
+// rejected when spec.debug is false.
+func TestE2E_MetadataServerAndGuestExec(t *testing.T) {
+ ns := defaultAtespace
+ ws := uniqueName("ws-meta")
+ task := uniqueName("task-meta")
+ env.deleteOnCleanup(t, ns, "workspace", ws)
+ env.deleteOnCleanup(t, ns, "task", task)
+
+ env.apply(t, ns, renderYAML(t, metadataManifest, map[string]string{"Workspace": ws, "Task": task, "Atespace": ns}))
+ env.mustRunAX(t, ns, "resume", "task", task)
+
+ // 1. `ax ssh` sees spec.env, the metadata URL, and workspace files.
+ out := env.ssh(t, ns, task, "echo ENV=$CUSTOM_E2E_VAR META=$AX_METADATA_URL FILE=$(cat /workspace/hello.txt)")
+ assertContains(t, "'ax ssh' output", out,
+ "ENV=custom-val-123", "META="+metadataURL, "FILE=hello from workspace")
+
+ // spec.command runs in the background after setup; wait for its output.
+ err := poll(10*time.Second, 250*time.Millisecond, func() error {
+ out, err := env.runAX(ns, "ssh", task, "--", "cat", "/workspace/cmd-ran.txt")
+ if err != nil {
+ return fmt.Errorf("%w: %s", err, out)
+ }
+ if got := strings.TrimSpace(out); got != "cmd-ran-ok" {
+ return fmt.Errorf("cmd-ran.txt = %q", got)
+ }
+ return nil
+ })
+ if err != nil {
+ t.Fatalf("spec.command did not write /workspace/cmd-ran.txt: %v", err)
+ }
+
+ // 2. The in-sandbox probe server serves health/readiness on port 80, and
+ // the loopback metadata server serves task/workspace metadata at AX_METADATA_URL.
+ curl := func(url string) string {
+ t.Helper()
+ return env.mustRunAX(t, ns, "ssh", task, "--", "curl", "-sf", url)
+ }
+ for _, path := range []string{"/healthz", "/readyz"} {
+ if got := strings.TrimSpace(curl(probeURL + path)); got != "ok" {
+ t.Fatalf("GET %s = %q, want \"ok\"", path, got)
+ }
+ }
+ assertContains(t, "task metadata", curl(metadataURL+"/metadata/v1alpha1/ax/task"), task, "custom-val-123")
+ assertContains(t, "workspace metadata", curl(metadataURL+"/metadata/v1alpha1/ax/workspace"), ws, "hello from workspace")
+
+ // 3. `ax ssh` propagates a non-zero guest exit code.
+ if _, err := env.runAX(ns, "ssh", task, "--", "sh", "-c", "exit 17"); err == nil {
+ t.Fatal("expected 'ax ssh' to fail when the guest command exits 17")
+ }
+
+ // 4. `ax ssh` is rejected for tasks without spec.debug.
+ noDebug := uniqueName("task-nodebug")
+ env.deleteOnCleanup(t, ns, "task", noDebug)
+ env.apply(t, ns, taskManifest(t, ns, noDebug, false))
+ env.mustRunAX(t, ns, "resume", "task", noDebug)
+ out, err = env.runAX(ns, "ssh", noDebug, "--", "echo", "should-fail")
+ if err == nil {
+ t.Fatalf("expected 'ax ssh' to fail when debug=false, got:\n%s", out)
+ }
+ if !strings.Contains(out, "spec.debug: true") && !strings.Contains(out, "debugging is not enabled") {
+ t.Fatalf("expected debug-disabled error, got:\n%s", out)
+ }
+}
+
+const workspaceSetupManifest = `apiVersion: ax.io/v1alpha1
+kind: Workspace
+metadata:
+ name: {{.GitWorkspace}}
+ atespace: {{.Atespace}}
+spec:
+ git:
+ - name: sample
+ dir: "."
+ repo: {{.Repo}}
+ branch: v1.0.0
+ files:
+ - path: overlay/config.yaml
+ content: "mode: git-plus-overlay"
+ skills:
+ path: /workspace/primary/.agents/skills
+---
+apiVersion: ax.io/v1alpha1
+kind: Task
+metadata:
+ name: {{.Task}}
+ atespace: {{.Atespace}}
+spec:
+ debug: true
+ workspace:
+ name: {{.GitWorkspace}}
+ path: /workspace/primary
+`
+
+// TestE2E_WorkspaceSetup_FilesAndGit verifies workspace setup for a task:
+// custom mount path (/workspace/primary), working directory set to the mount
+// path, spec.git checkout at a tag, overlay spec.files, spec.skills.path, and
+// the /ax/initialized-* maiden-run marker.
+func TestE2E_WorkspaceSetup_FilesAndGit(t *testing.T) {
+ ns := defaultAtespace
+ wsGit := uniqueName("ws-git")
+ task := uniqueName("task-ws")
+ env.deleteOnCleanup(t, ns, "workspace", wsGit)
+ env.deleteOnCleanup(t, ns, "task", task)
+
+ env.apply(t, ns, renderYAML(t, workspaceSetupManifest, map[string]string{
+ "GitWorkspace": wsGit, "Task": task, "Atespace": ns, "Repo": sampleRepo,
+ }))
+ env.mustRunAX(t, ns, "resume", "task", task)
+
+ // - The working directory is the workspace's mount path.
+ // - The workspace has the repo checked out at v1.0.0, the overlay file, and
+ // the skills directory.
+ // - A maiden-run marker is written for the workspace mount path.
+ out := env.ssh(t, ns, task, `set -e
+pwd
+cat /workspace/primary/app.py
+cat /workspace/primary/overlay/config.yaml
+test -d /workspace/primary/.agents/skills && echo SKILLS_DIR_OK
+test -f /ax/initialized-workspace-primary && echo MARKERS_OK
+`)
+ assertContains(t, "workspace setup check", out,
+ "/workspace/primary",
+ "hello from git repo v1",
+ "mode: git-plus-overlay",
+ "SKILLS_DIR_OK",
+ "MARKERS_OK",
+ )
+}
+
+const persistenceManifest = `apiVersion: ax.io/v1alpha1
+kind: Workspace
+metadata:
+ name: {{.Workspace}}
+ atespace: {{.Atespace}}
+spec:
+ files:
+ - path: seed.txt
+ content: "initial-seed"
+---
+apiVersion: ax.io/v1alpha1
+kind: Task
+metadata:
+ name: {{.Task}}
+ atespace: {{.Atespace}}
+spec:
+ debug: true
+ workspace:
+ name: {{.Workspace}}
+ path: /workspace
+`
+
+// TestE2E_TaskSuspendAndResumePersistence verifies that suspend/resume with
+// VOLUMES-fidelity snapshots preserves the durable /workspace volume while
+// the rootfs (/tmp) is reset.
+func TestE2E_TaskSuspendAndResumePersistence(t *testing.T) {
+ ns := defaultAtespace
+ ws := uniqueName("ws-persist")
+ task := uniqueName("task-persist")
+ env.deleteOnCleanup(t, ns, "workspace", ws)
+ env.deleteOnCleanup(t, ns, "task", task)
+
+ env.apply(t, ns, renderYAML(t, persistenceManifest, map[string]string{"Workspace": ws, "Task": task, "Atespace": ns}))
+ env.mustRunAX(t, ns, "resume", "task", task)
+
+ // 1. Write to the durable /workspace volume and to the ephemeral rootfs.
+ env.ssh(t, ns, task, "echo durable-data-42 > /workspace/durable.txt && echo ephemeral-tmp > /tmp/ephemeral.txt")
+
+ // 2. Suspend: the task and its Actor are suspended and the worker released.
+ assertContains(t, "'ax suspend task'", env.mustRunAX(t, ns, "suspend", "task", task), "suspended")
+ got := env.getTask(t, ns, task)
+ assertPhase(t, got, phaseSuspended)
+ assertCondition(t, got, "Ready", "False", "TaskSuspended")
+ if ip := got.GetStatus().GetWorkerIp(); ip != "" {
+ t.Fatalf("expected empty WorkerIp when suspended, got %q", ip)
+ }
+ actor := env.getActor(t, ns, task, ateapipb.ActorState_ACTOR_STATE_SUSPENDED)
+ if len(actor.GetStatus().GetSnapshots()) == 0 && len(actor.GetStatus().ProtoReflect().GetUnknown()) == 0 {
+ t.Fatal("expected non-empty Snapshots on suspended actor")
+ }
+
+ // 3. Resume: VOLUMES-fidelity snapshots restore /workspace but cold-boot the rootfs.
+ env.mustRunAX(t, ns, "resume", "task", task)
+ out := env.ssh(t, ns, task,
+ "echo DURABLE=$(cat /workspace/durable.txt); if [ -f /tmp/ephemeral.txt ]; then echo TMP_EXISTS; else echo TMP_GONE; fi")
+ assertContains(t, "post-resume check", out, "DURABLE=durable-data-42", "TMP_GONE")
+}
+
+// TestE2E_WatchDescribeAndAtespaceIsolation verifies that a task in a custom
+// atespace is invisible from the default atespace, that WatchTask emits
+// INITIAL and then MODIFIED events as the task is resumed, and that
+// `ax describe task` reports the running task.
+func TestE2E_WatchDescribeAndAtespaceIsolation(t *testing.T) {
+ ctx, cancel := context.WithTimeout(t.Context(), 2*time.Minute)
+ defer cancel()
+
+ ns := uniqueName("team-e2e")
+ task := uniqueName("task-watch")
+ env.deleteOnCleanup(t, ns, "task", task)
+
+ env.apply(t, ns, taskManifest(t, ns, task, true))
+
+ // 1. The task is visible only in its own atespace.
+ if _, err := env.AXClient.GetTask(ctx, &v1alpha1.GetTaskRequest{Name: task, Atespace: defaultAtespace}); status.Code(err) != codes.NotFound {
+ t.Fatalf("GetTask in %q atespace: got %v, want NotFound", defaultAtespace, err)
+ }
+ if out := env.mustRunAX(t, defaultAtespace, "get", "tasks"); strings.Contains(out, task) {
+ t.Fatalf("task %s unexpectedly listed in %q atespace:\n%s", task, defaultAtespace, out)
+ }
+ assertContains(t, "'ax get tasks' in "+ns, env.mustRunAX(t, ns, "get", "tasks"), task)
+
+ // 2. WatchTask emits INITIAL (Suspended), then MODIFIED (Running) on resume.
+ stream, err := env.AXClient.WatchTask(ctx, &v1alpha1.WatchTaskRequest{Name: task, Atespace: ns})
+ if err != nil {
+ t.Fatalf("WatchTask: %v", err)
+ }
+ evt, err := stream.Recv()
+ if err != nil {
+ t.Fatalf("receiving INITIAL event: %v", err)
+ }
+ if evt.GetAction() != "INITIAL" || evt.GetTask().GetStatus().GetPhase() != phaseSuspended {
+ t.Fatalf("first event: action=%q phase=%q, want INITIAL/%s", evt.GetAction(), evt.GetTask().GetStatus().GetPhase(), phaseSuspended)
+ }
+
+ resumeErr := make(chan error, 1)
+ go func() {
+ out, err := env.runAX(ns, "resume", "task", task)
+ if err != nil {
+ err = fmt.Errorf("%w: %s", err, out)
+ }
+ resumeErr <- err
+ }()
+
+ for {
+ evt, err := stream.Recv()
+ if errors.Is(err, io.EOF) {
+ t.Fatal("WatchTask stream ended before a MODIFIED/Running event")
+ }
+ if err != nil {
+ t.Fatalf("receiving WatchTask event: %v", err)
+ }
+ if evt.GetAction() == "MODIFIED" && evt.GetTask().GetStatus().GetPhase() == phaseRunning {
+ break
+ }
+ }
+ if err := <-resumeErr; err != nil {
+ t.Fatalf("ax resume task: %v", err)
+ }
+
+ // 3. `ax describe task` reflects the running task and its conditions.
+ assertContains(t, "'ax describe task'", env.mustRunAX(t, ns, "describe", "task", task),
+ task, ns, phaseRunning, "WorkspaceReady", "TaskRunning")
+}
diff --git a/test/e2e/harness_test.go b/test/e2e/harness_test.go
new file mode 100644
index 0000000..e5870a7
--- /dev/null
+++ b/test/e2e/harness_test.go
@@ -0,0 +1,516 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package e2e_test
+
+import (
+ "bufio"
+ "bytes"
+ "context"
+ "crypto/rand"
+ "encoding/hex"
+ "errors"
+ "flag"
+ "fmt"
+ "net"
+ "net/http"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strings"
+ "testing"
+ "text/template"
+ "time"
+
+ "github.com/agent-substrate/substrate/pkg/proto/ateapipb"
+ "github.com/google/ax/internal/substrate"
+ "github.com/google/ax/pkg/apis/v1alpha1"
+ "google.golang.org/grpc"
+ "google.golang.org/grpc/credentials/insecure"
+)
+
+var runE2E = flag.Bool("e2e", false, "run end-to-end tests against a live Kind/Substrate/AX cluster")
+
+const (
+ // ateSystemNamespace is the namespace Substrate's control plane is installed into.
+ ateSystemNamespace = "ate-system"
+ // ateAPIAudience is the token audience and TLS authority of the Substrate Control API.
+ ateAPIAudience = "api.ate-system.svc"
+
+ // axCommandTimeout bounds a single `ax` CLI invocation.
+ axCommandTimeout = 2 * time.Minute
+ // readyTimeout bounds waiting for port-forwards and health endpoints.
+ readyTimeout = 15 * time.Second
+)
+
+// suiteEnv holds the connections shared by every test in the suite. It is
+// populated once by TestMain.
+type suiteEnv struct {
+ Kubeconfig string
+ KubeContext string
+ AXSystemNamespace string
+ AXBin string
+ AXServerURL string
+
+ AXClient v1alpha1.AXClient
+ SubstrateClient *substrate.Client
+
+ // axHome isolates ax CLI state (tunnels, config) from the user's ~/.ax.
+ axHome string
+ closers []func()
+}
+
+var env *suiteEnv
+
+func TestMain(m *testing.M) {
+ flag.Parse()
+ if !*runE2E && os.Getenv("AX_E2E") != "true" {
+ fmt.Println("Skipping AX E2E tests (pass -args --e2e or set AX_E2E=true to run)")
+ os.Exit(0)
+ }
+
+ var err error
+ env, err = newSuiteEnv()
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "E2E setup failed: %v\n", err)
+ os.Exit(1)
+ }
+ code := m.Run()
+ env.Close()
+ os.Exit(code)
+}
+
+// newSuiteEnv verifies the cluster provisioned by hack/setup-e2e-kind.sh is
+// healthy and connects to both ax-server and the Substrate Control API.
+func newSuiteEnv() (_ *suiteEnv, err error) {
+ wd, err := os.Getwd()
+ if err != nil {
+ return nil, err
+ }
+ repoRoot := filepath.Clean(filepath.Join(wd, "..", ".."))
+ repoTmp := filepath.Join(repoRoot, ".tmp")
+
+ // Pick up defaults written by hack/setup-e2e-kind.sh; explicit env vars win.
+ if err := loadEnvFileIfUnset(filepath.Join(repoTmp, "e2e.env")); err != nil {
+ return nil, err
+ }
+
+ clusterName := getenvDefault("KIND_CLUSTER_NAME", "ax-e2e")
+ s := &suiteEnv{
+ Kubeconfig: getenvDefault("KUBECONFIG", filepath.Join(repoTmp, "kubeconfig-"+clusterName)),
+ KubeContext: getenvDefault("KUBE_CONTEXT", "kind-"+clusterName),
+ AXSystemNamespace: getenvDefault("AX_SYSTEM_NAMESPACE", v1alpha1.DefaultSystemNamespace),
+ AXBin: getenvDefault("AX_BIN", filepath.Join(repoTmp, "bin", "ax")),
+ }
+ defer func() {
+ if err != nil {
+ s.Close()
+ }
+ }()
+
+ suiteTmp, err := os.MkdirTemp("", "ax-e2e-")
+ if err != nil {
+ return nil, err
+ }
+ s.closers = append(s.closers, func() { _ = os.RemoveAll(suiteTmp) })
+ s.axHome = filepath.Join(suiteTmp, "ax-home")
+
+ if err := s.verifyCluster(); err != nil {
+ return nil, err
+ }
+ if err := ensureAXBinary(repoRoot, s.AXBin); err != nil {
+ return nil, err
+ }
+ if err := s.connectAXServer(); err != nil {
+ return nil, err
+ }
+ if err := s.connectSubstrate(filepath.Join(suiteTmp, "ateapi.token")); err != nil {
+ return nil, err
+ }
+ return s, nil
+}
+
+// Close releases all suite resources in reverse order of acquisition.
+func (s *suiteEnv) Close() {
+ for i := len(s.closers) - 1; i >= 0; i-- {
+ s.closers[i]()
+ }
+ s.closers = nil
+}
+
+func (s *suiteEnv) verifyCluster() error {
+ const hint = "run ./hack/setup-e2e-kind.sh first"
+ for _, ns := range []string{ateSystemNamespace, s.AXSystemNamespace} {
+ if _, err := s.kubectl("get", "namespace", ns); err != nil {
+ return fmt.Errorf("namespace %q not found (%s): %w", ns, hint, err)
+ }
+ }
+ for ns, deploy := range map[string]string{
+ ateSystemNamespace: "ate-api-server",
+ s.AXSystemNamespace: "ax-server",
+ } {
+ if _, err := s.kubectl("-n", ns, "wait", "--for=condition=Available", "deployment/"+deploy, "--timeout=30s"); err != nil {
+ return fmt.Errorf("deployment %s/%s not available (%s): %w", ns, deploy, hint, err)
+ }
+ }
+ return nil
+}
+
+// ensureAXBinary builds the ax CLI for the host if it does not already exist.
+func ensureAXBinary(repoRoot, path string) error {
+ if _, err := os.Stat(path); err == nil {
+ return nil
+ }
+ cmd := exec.Command("go", "build", "-o", path, "./cmd/ax")
+ cmd.Dir = repoRoot
+ cmd.Env = append(os.Environ(), "CGO_ENABLED=0")
+ if out, err := cmd.CombinedOutput(); err != nil {
+ return fmt.Errorf("building ax CLI: %w\n%s", err, out)
+ }
+ return nil
+}
+
+func (s *suiteEnv) connectAXServer() error {
+ port, err := s.startPortForward(s.AXSystemNamespace, "svc/ax-server", 8080)
+ if err != nil {
+ return fmt.Errorf("port-forwarding ax-server: %w", err)
+ }
+ addr := fmt.Sprintf("127.0.0.1:%d", port)
+ s.AXServerURL = "http://" + addr
+
+ if err := waitForHTTPHealth(s.AXServerURL+"/healthz", readyTimeout); err != nil {
+ return fmt.Errorf("waiting for ax-server /healthz: %w", err)
+ }
+
+ conn, err := grpc.NewClient(addr, grpc.WithTransportCredentials(insecure.NewCredentials()))
+ if err != nil {
+ return fmt.Errorf("connecting to ax-server gRPC: %w", err)
+ }
+ s.closers = append(s.closers, func() { _ = conn.Close() })
+ s.AXClient = v1alpha1.NewAXClient(conn)
+ return nil
+}
+
+// connectSubstrate connects directly to the Substrate Control API so tests can
+// verify the Actors and ActorTemplates AX creates on their behalf.
+func (s *suiteEnv) connectSubstrate(tokenFile string) error {
+ port, err := s.startPortForward(ateSystemNamespace, "svc/api", 443)
+ if err != nil {
+ return fmt.Errorf("port-forwarding %s/svc/api: %w", ateSystemNamespace, err)
+ }
+
+ // Authenticate as ax-server's service account, just like ax-server does.
+ token, err := s.kubectl("create", "token", "ax-server", "-n", s.AXSystemNamespace,
+ "--audience="+ateAPIAudience, "--duration=1h")
+ if err != nil {
+ return fmt.Errorf("minting Substrate Control API token: %w", err)
+ }
+ if err := os.WriteFile(tokenFile, []byte(strings.TrimSpace(token)), 0o600); err != nil {
+ return fmt.Errorf("writing Substrate token file: %w", err)
+ }
+
+ client, err := substrate.NewClientWithOptions(substrate.ClientOptions{
+ Target: fmt.Sprintf("127.0.0.1:%d", port),
+ Authority: ateAPIAudience,
+ InsecureTLS: true,
+ TokenFile: tokenFile,
+ })
+ if err != nil {
+ return fmt.Errorf("connecting to Substrate Control API: %w", err)
+ }
+ s.closers = append(s.closers, func() { _ = client.Close() })
+ s.SubstrateClient = client
+
+ ctx, cancel := context.WithTimeout(context.Background(), readyTimeout)
+ defer cancel()
+ if err := client.EnsureAtespace(ctx, defaultAtespace); err != nil {
+ return fmt.Errorf("verifying Substrate Control API readiness: %w", err)
+ }
+ return nil
+}
+
+// kubectlArgs prefixes args with the suite's kubeconfig and context.
+func (s *suiteEnv) kubectlArgs(args ...string) []string {
+ return append([]string{"--kubeconfig", s.Kubeconfig, "--context", s.KubeContext}, args...)
+}
+
+func (s *suiteEnv) kubectl(args ...string) (string, error) {
+ out, err := exec.Command("kubectl", s.kubectlArgs(args...)...).CombinedOutput()
+ if err != nil {
+ return string(out), fmt.Errorf("kubectl %s: %w (%s)", strings.Join(args, " "), err, bytes.TrimSpace(out))
+ }
+ return string(out), nil
+}
+
+// startPortForward runs `kubectl port-forward` in the background and returns
+// the local port once it accepts connections. The process is stopped by Close.
+func (s *suiteEnv) startPortForward(namespace, resource string, remotePort int) (int, error) {
+ localPort, err := freeLocalPort()
+ if err != nil {
+ return 0, err
+ }
+
+ cmd := exec.Command("kubectl", s.kubectlArgs(
+ "-n", namespace, "port-forward", resource, fmt.Sprintf("%d:%d", localPort, remotePort))...)
+ var stderr bytes.Buffer
+ cmd.Stderr = &stderr
+ if err := cmd.Start(); err != nil {
+ return 0, err
+ }
+ stop := func() {
+ _ = cmd.Process.Kill()
+ _ = cmd.Wait()
+ }
+
+ addr := fmt.Sprintf("127.0.0.1:%d", localPort)
+ err = poll(readyTimeout, 150*time.Millisecond, func() error {
+ conn, err := net.DialTimeout("tcp", addr, 250*time.Millisecond)
+ if err != nil {
+ return err
+ }
+ return conn.Close()
+ })
+ if err != nil {
+ stop()
+ return 0, fmt.Errorf("port-forward to %s/%s did not become ready: %w (%s)", namespace, resource, err, stderr.String())
+ }
+ s.closers = append(s.closers, stop)
+ return localPort, nil
+}
+
+func freeLocalPort() (int, error) {
+ ln, err := net.Listen("tcp", "127.0.0.1:0")
+ if err != nil {
+ return 0, err
+ }
+ defer ln.Close()
+ return ln.Addr().(*net.TCPAddr).Port, nil
+}
+
+func waitForHTTPHealth(url string, timeout time.Duration) error {
+ client := &http.Client{Timeout: time.Second}
+ return poll(timeout, 200*time.Millisecond, func() error {
+ resp, err := client.Get(url)
+ if err != nil {
+ return err
+ }
+ _ = resp.Body.Close()
+ if resp.StatusCode != http.StatusOK {
+ return fmt.Errorf("GET %s: status %d", url, resp.StatusCode)
+ }
+ return nil
+ })
+}
+
+// poll calls fn every interval until it returns nil or timeout elapses, in
+// which case the last error is returned.
+func poll(timeout, interval time.Duration, fn func() error) error {
+ deadline := time.Now().Add(timeout)
+ for {
+ err := fn()
+ if err == nil {
+ return nil
+ }
+ if time.Now().After(deadline) {
+ return err
+ }
+ time.Sleep(interval)
+ }
+}
+
+// ---------------------------------------------------------------------------
+// ax CLI helpers
+// ---------------------------------------------------------------------------
+
+// axCommand returns an `ax` invocation whose environment points at the E2E
+// cluster, with AX_SERVER and AX_HOME scrubbed so the caller's shell does not
+// leak in. The command is killed after axCommandTimeout.
+func (s *suiteEnv) axCommand(axHome string, args ...string) (*exec.Cmd, context.CancelFunc) {
+ ctx, cancel := context.WithTimeout(context.Background(), axCommandTimeout)
+ cmd := exec.CommandContext(ctx, s.AXBin, args...)
+ for _, kv := range os.Environ() {
+ switch k, _, _ := strings.Cut(kv, "="); k {
+ case "AX_SERVER", "AX_HOME", "KUBECONFIG":
+ continue
+ }
+ cmd.Env = append(cmd.Env, kv)
+ }
+ cmd.Env = append(cmd.Env, "KUBECONFIG="+s.Kubeconfig, "AX_HOME="+axHome)
+ return cmd, cancel
+}
+
+// runAX runs `ax --server <url> --context <context> [-a atespace] args...` and returns its
+// combined output.
+func (s *suiteEnv) runAX(atespace string, args ...string) (string, error) {
+ fullArgs := []string{"--server", s.AXServerURL, "--context", s.KubeContext}
+ if atespace != "" {
+ fullArgs = append(fullArgs, "-a", atespace)
+ }
+ cmd, cancel := s.axCommand(s.axHome, append(fullArgs, args...)...)
+ defer cancel()
+ out, err := cmd.CombinedOutput()
+ return string(out), err
+}
+
+func (s *suiteEnv) mustRunAX(t *testing.T, atespace string, args ...string) string {
+ t.Helper()
+ out, err := s.runAX(atespace, args...)
+ if err != nil {
+ t.Fatalf("ax %s failed: %v\nOutput:\n%s", strings.Join(args, " "), err, out)
+ }
+ return out
+}
+
+// ssh runs a shell script inside the task's sandbox via `ax ssh`.
+func (s *suiteEnv) ssh(t *testing.T, atespace, task, script string) string {
+ t.Helper()
+ return s.mustRunAX(t, atespace, "ssh", task, "--", "sh", "-c", script)
+}
+
+// writeManifest writes manifest to a temp file and returns its path.
+func writeManifest(t *testing.T, manifest string) string {
+ t.Helper()
+ f := filepath.Join(t.TempDir(), "manifest.yaml")
+ if err := os.WriteFile(f, []byte(manifest), 0o644); err != nil {
+ t.Fatalf("writing manifest: %v", err)
+ }
+ return f
+}
+
+// apply runs `ax apply -f` on manifest and returns the CLI output.
+func (s *suiteEnv) apply(t *testing.T, atespace, manifest string) string {
+ t.Helper()
+ return s.mustRunAX(t, atespace, "apply", "-f", writeManifest(t, manifest))
+}
+
+// deleteOnCleanup registers a best-effort `ax delete <kind> <name>` to run
+// when the test finishes. Resources are deleted in reverse registration order.
+func (s *suiteEnv) deleteOnCleanup(t *testing.T, atespace, kind, name string) {
+ t.Cleanup(func() {
+ if out, err := s.runAX(atespace, "delete", kind, name); err != nil && !strings.Contains(out, "NotFound") {
+ t.Logf("cleanup: ax delete %s %s: %v\n%s", kind, name, err, out)
+ }
+ })
+}
+
+// ---------------------------------------------------------------------------
+// Assertions and fixtures
+// ---------------------------------------------------------------------------
+
+// renderYAML executes tmpl (a text/template) with data.
+func renderYAML(t *testing.T, tmpl string, data any) string {
+ t.Helper()
+ var b strings.Builder
+ if err := template.Must(template.New("manifest").Option("missingkey=error").Parse(tmpl)).Execute(&b, data); err != nil {
+ t.Fatalf("rendering manifest: %v", err)
+ }
+ return b.String()
+}
+
+func uniqueName(prefix string) string {
+ var b [4]byte
+ _, _ = rand.Read(b[:])
+ return prefix + "-" + hex.EncodeToString(b[:])
+}
+
+// assertContains fails the test if out does not contain every want.
+func assertContains(t *testing.T, what, out string, wants ...string) {
+ t.Helper()
+ for _, want := range wants {
+ if !strings.Contains(out, want) {
+ t.Fatalf("expected %s to contain %q, got:\n%s", what, want, out)
+ }
+ }
+}
+
+func (s *suiteEnv) getTask(t *testing.T, atespace, name string) *v1alpha1.Task {
+ t.Helper()
+ task, err := s.AXClient.GetTask(t.Context(), &v1alpha1.GetTaskRequest{Name: name, Atespace: atespace})
+ if err != nil {
+ t.Fatalf("GetTask(%s/%s): %v", atespace, name, err)
+ }
+ return task
+}
+
+func assertPhase(t *testing.T, task *v1alpha1.Task, want string) {
+ t.Helper()
+ if got := task.GetStatus().GetPhase(); got != want {
+ t.Fatalf("task %s: phase = %q, want %q", task.GetMetadata().GetName(), got, want)
+ }
+}
+
+func assertCondition(t *testing.T, task *v1alpha1.Task, condType, wantStatus, wantReason string) {
+ t.Helper()
+ for _, c := range task.GetStatus().GetConditions() {
+ if c.GetType() != condType {
+ continue
+ }
+ if c.GetStatus() != wantStatus || (wantReason != "" && c.GetReason() != wantReason) {
+ t.Fatalf("condition %s: got status=%q reason=%q (message=%q), want status=%q reason=%q",
+ condType, c.GetStatus(), c.GetReason(), c.GetMessage(), wantStatus, wantReason)
+ }
+ return
+ }
+ t.Fatalf("condition %q not found on task %s (conditions: %+v)", condType, task.GetMetadata().GetName(), task.GetStatus().GetConditions())
+}
+
+// getActor fetches the Substrate Actor backing a task and checks its state.
+func (s *suiteEnv) getActor(t *testing.T, atespace, name string, wantState ateapipb.ActorState) *ateapipb.Actor {
+ t.Helper()
+ actor, err := s.SubstrateClient.GetActor(t.Context(), atespace, name)
+ if err != nil {
+ t.Fatalf("Substrate GetActor(%s/%s): %v", atespace, name, err)
+ }
+ if got := actor.GetStatus().GetState(); got != wantState {
+ t.Fatalf("Substrate actor %s/%s: state = %v, want %v", atespace, name, got, wantState)
+ }
+ return actor
+}
+
+func getenvDefault(key, def string) string {
+ if v := os.Getenv(key); v != "" {
+ return v
+ }
+ return def
+}
+
+// loadEnvFileIfUnset sets variables from a file of `export KEY="value"` lines,
+// skipping keys already present in the environment. A missing file is not an error.
+func loadEnvFileIfUnset(path string) error {
+ f, err := os.Open(path)
+ if errors.Is(err, os.ErrNotExist) {
+ return nil
+ }
+ if err != nil {
+ return err
+ }
+ defer f.Close()
+
+ scanner := bufio.NewScanner(f)
+ for scanner.Scan() {
+ line := strings.TrimSpace(scanner.Text())
+ if line == "" || strings.HasPrefix(line, "#") {
+ continue
+ }
+ k, v, ok := strings.Cut(strings.TrimPrefix(line, "export "), "=")
+ if !ok {
+ continue
+ }
+ k = strings.TrimSpace(k)
+ if os.Getenv(k) == "" {
+ _ = os.Setenv(k, strings.Trim(strings.TrimSpace(v), `"'`))
+ }
+ }
+ return scanner.Err()
+}
diff --git a/test/e2e/manifests/ax-server.yaml b/test/e2e/manifests/ax-server.yaml
new file mode 100644
index 0000000..c0127f6
--- /dev/null
+++ b/test/e2e/manifests/ax-server.yaml
@@ -0,0 +1,131 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: ax-server
+ namespace: ${AX_SYSTEM_NAMESPACE}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+ name: ax-server
+rules:
+ - apiGroups: [""]
+ resources: ["secrets"]
+ verbs: ["get", "list", "watch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+ name: ax-server
+subjects:
+ - kind: ServiceAccount
+ name: ax-server
+ namespace: ${AX_SYSTEM_NAMESPACE}
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: ClusterRole
+ name: ax-server
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: ax-server
+ namespace: ${AX_SYSTEM_NAMESPACE}
+ labels:
+ app.kubernetes.io/name: ax-server
+ app.kubernetes.io/part-of: ax
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app.kubernetes.io/name: ax-server
+ template:
+ metadata:
+ labels:
+ app.kubernetes.io/name: ax-server
+ spec:
+ serviceAccountName: ax-server
+ containers:
+ - name: ax-server
+ image: ${AX_SERVER_IMAGE}
+ imagePullPolicy: IfNotPresent
+ args:
+ - "--addr=:8080"
+ - "--redis-addr=ax-redis.${AX_SYSTEM_NAMESPACE}.svc.cluster.local:6379"
+ - "--substrate-endpoint=api.ate-system.svc.cluster.local:443"
+ - "--substrate-authority=api.ate-system.svc"
+ - "--substrate-token-file=/var/run/secrets/ateapi/token"
+ - "--substrate-ca-file=/run/servicedns-ca/trust-bundle.pem"
+ - "--template=default-template"
+ - "--template-atespace=${AX_SYSTEM_NAMESPACE}"
+ env:
+ - name: ATENET_ROUTER_ADDR
+ value: "atenet-router.ate-system.svc.cluster.local:80"
+ - name: AX_SNAPSHOTS_BUCKET
+ value: "${AX_SNAPSHOTS_BUCKET}"
+ - name: AX_DEFAULT_TASK_IMAGE
+ value: "${AX_DEFAULT_TASK_IMAGE}"
+ ports:
+ - containerPort: 8080
+ name: http
+ resources:
+ requests:
+ cpu: 100m
+ memory: 128Mi
+ limits:
+ cpu: 1000m
+ memory: 1Gi
+ readinessProbe:
+ httpGet:
+ path: /healthz
+ port: 8080
+ initialDelaySeconds: 2
+ periodSeconds: 5
+ livenessProbe:
+ httpGet:
+ path: /healthz
+ port: 8080
+ initialDelaySeconds: 5
+ periodSeconds: 10
+ volumeMounts:
+ - mountPath: /var/run/secrets/ateapi
+ name: ate-token
+ readOnly: true
+ - mountPath: /run/servicedns-ca
+ name: servicedns-ca
+ readOnly: true
+ volumes:
+ - name: ate-token
+ projected:
+ defaultMode: 420
+ sources:
+ - serviceAccountToken:
+ audience: api.ate-system.svc
+ expirationSeconds: 7200
+ path: token
+ - name: servicedns-ca
+ projected:
+ defaultMode: 420
+ sources:
+ - clusterTrustBundle:
+ labelSelector:
+ matchLabels:
+ podcert.ate.dev/canarying: live
+ path: trust-bundle.pem
+ signerName: servicedns.podcert.ate.dev/identity
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: ax-server
+ namespace: ${AX_SYSTEM_NAMESPACE}
+ labels:
+ app.kubernetes.io/name: ax-server
+ app.kubernetes.io/part-of: ax
+spec:
+ ports:
+ - port: 8080
+ targetPort: 8080
+ name: http
+ selector:
+ app.kubernetes.io/name: ax-server
diff --git a/test/e2e/manifests/namespace.yaml b/test/e2e/manifests/namespace.yaml
new file mode 100644
index 0000000..0992a08
--- /dev/null
+++ b/test/e2e/manifests/namespace.yaml
@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+ name: ${AX_SYSTEM_NAMESPACE}
+ labels:
+ ate.dev/api-access: "true"
diff --git a/test/e2e/manifests/redis.yaml b/test/e2e/manifests/redis.yaml
new file mode 100644
index 0000000..ddb3192
--- /dev/null
+++ b/test/e2e/manifests/redis.yaml
@@ -0,0 +1,47 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: ax-redis
+ namespace: ${AX_SYSTEM_NAMESPACE}
+ labels:
+ app.kubernetes.io/name: ax-redis
+ app.kubernetes.io/part-of: ax
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app.kubernetes.io/name: ax-redis
+ template:
+ metadata:
+ labels:
+ app.kubernetes.io/name: ax-redis
+ spec:
+ containers:
+ - name: redis
+ image: redis:7-alpine
+ ports:
+ - containerPort: 6379
+ name: redis
+ resources:
+ requests:
+ cpu: 100m
+ memory: 128Mi
+ limits:
+ cpu: 1000m
+ memory: 1Gi
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: ax-redis
+ namespace: ${AX_SYSTEM_NAMESPACE}
+ labels:
+ app.kubernetes.io/name: ax-redis
+ app.kubernetes.io/part-of: ax
+spec:
+ ports:
+ - port: 6379
+ targetPort: 6379
+ name: redis
+ selector:
+ app.kubernetes.io/name: ax-redis
diff --git a/test/e2e/manifests/workerpool.yaml b/test/e2e/manifests/workerpool.yaml
new file mode 100644
index 0000000..2def94b
--- /dev/null
+++ b/test/e2e/manifests/workerpool.yaml
@@ -0,0 +1,12 @@
+apiVersion: ate.dev/v1alpha1
+kind: WorkerPool
+metadata:
+ name: ax-workers
+ namespace: ${AX_SYSTEM_NAMESPACE}
+spec:
+ replicas: ${AX_WORKER_REPLICAS}
+ workerImage: ko://github.com/agent-substrate/substrate/cmd/ateom-gvisor
+ sandboxClasses:
+ - name: gvisor
+ configRef:
+ name: gvisor-default