Agent Executor (AX) is Google's open agentic orchestration runtime
Go
test: add Kind-based E2E test suite and CI workflow (#474)
Siyuan Zhang43439b411f01
diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.ymlnew file mode 100644index 0000000..c6823b2--- /dev/null+++ b/.github/workflows/e2e.yml@@ -0,0 +1,80 @@+# Copyright 2026 Google LLC+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++name: E2E++on:+ push:+ branches: [ main ]+ pull_request:+ branches: [ main ]+ # Nightly run against Substrate main to catch incompatibilities with+ # unreleased Substrate changes early. Never blocks PRs.+ schedule:+ - cron: '0 6 * * *' # every day at 06:00am UTC+ workflow_dispatch:+ inputs:+ substrate_ref:+ description: 'Substrate branch, tag, or commit to test against (default: version in go.mod)'+ required: false+ default: ''+ type: string++permissions:+ contents: read++jobs:+ e2e-kind:+ name: E2E (Kind / Substrate ${{ github.event_name == 'schedule' && 'main' || inputs.substrate_ref || 'go.mod' }})+ runs-on: ubuntu-latest+ timeout-minutes: 30+ steps:+ - name: Checkout AX+ uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2++ - name: Set up Go+ uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0+ with:+ go-version-file: go.mod++ - name: Install Kind+ uses: helm/kind-action@1544676c07bb3570fcb70166beaa07c4528fd4aa # v1.15.1+ with:+ install_only: true++ # The runner is discarded after the job, so the cluster is left running for+ # the diagnostics step below rather than cleaned up.+ # PRs and pushes test the Substrate version in go.mod, so they are not+ # blocked by unreleased changes on Substrate main. The nightly run tests+ # main; manual runs test the requested ref (go.mod if empty).+ - name: Run Kind E2E suite+ env:+ EVENT: ${{ github.event_name }}+ INPUT_REF: ${{ inputs.substrate_ref }}+ run: |+ if [[ "${EVENT}" == "schedule" ]]; then+ make test-e2e-head+ else+ make test-e2e ${INPUT_REF:+SUBSTRATE_REF="${INPUT_REF}"}+ fi++ - name: Collect cluster diagnostics on failure+ if: failure()+ env:+ KUBECONFIG: .tmp/kubeconfig-ax-e2e+ run: |+ [[ -f "${KUBECONFIG}" ]] || exit 0+ kubectl get pods -A -o wide || true+ kubectl -n ax-system logs deployment/ax-server --tail=200 || true+ kubectl -n ate-system logs deployment/ate-api-server --all-containers --tail=200 || truediff --git a/.gitignore b/.gitignoreindex 5bb6ff2..7f5c597 100644--- a/.gitignore+++ b/.gitignore@@ -1,6 +1,7 @@# Binariesbin//tmp/+/.tmp/*.exe*.test/ax@@ -8,3 +9,4 @@ bin//ax-task-runnercmd/ax-task-runner/__pycache__/antigravity_bootstrap.cpython-313.pyc+.vscode/diff --git a/Dockerfile.task-runner b/Dockerfile.task-runnerindex 29dc3e1..3dab7ac 100644--- a/Dockerfile.task-runner+++ b/Dockerfile.task-runner@@ -25,7 +25,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \RUN pip install --no-cache-dir google-antigravity-COPY bin/linux_amd64/ax-task-runner /usr/local/bin/ax-task-runner+ARG TARGETARCH=amd64+COPY bin/linux_${TARGETARCH}/ax-task-runner /usr/local/bin/ax-task-runnerCOPY cmd/ax-task-runner/antigravity_bootstrap.py /usr/local/bin/antigravity_bootstrap.pyENTRYPOINT ["/usr/local/bin/ax-task-runner"]diff --git a/Makefile b/Makefileindex 0ffba30..2d918f5 100644--- a/Makefile+++ b/Makefile@@ -19,7 +19,7 @@ AX_IMAGE_REPO ?= gcr.io/ax-substrate/ate-imagesTASK_RUNNER_REPO ?= $(AX_IMAGE_REPO)/ax-task-runnerCONTAINER_CLI ?= $(shell which podman 2>/dev/null || which docker 2>/dev/null)-.PHONY: all build build-binaries build-task-runner install push push-task-runner deploy deploy-server deploy-redis apply-example test clean+.PHONY: all build build-binaries build-task-runner install push push-task-runner deploy deploy-server deploy-redis apply-example test test-e2e test-e2e-head e2e-setup e2e-run e2e-cleanup cleanall: build@@ -87,6 +87,28 @@ test:@echo "==> Running tests..."go test -v ./...+# Provision Kind + Substrate + AX, then run the E2E suite. Substrate defaults to+# the version in go.mod; override with SUBSTRATE_REF=<branch|tag|commit>.+# Pass extra `go test` flags via E2E_ARGS, e.g. E2E_ARGS='-test.run TestE2E_ModelCRUD'.+test-e2e:+ ./hack/run-e2e-kind.sh $(E2E_ARGS)++# Run the E2E suite against Substrate main.+test-e2e-head:+ SUBSTRATE_REF=main ./hack/run-e2e-kind.sh $(E2E_ARGS)++# Provision Kind + Substrate + AX without running tests (accepts SUBSTRATE_REF).+e2e-setup:+ ./hack/setup-e2e-kind.sh++# Run the E2E suite against a cluster provisioned by e2e-setup.+e2e-run:+ ./hack/run-e2e.sh $(E2E_ARGS)++# Delete the Kind E2E cluster.+e2e-cleanup:+ ./hack/cleanup-e2e-kind.sh+clean:@echo "==> Cleaning build artifacts..."- rm -rf bin/+ rm -rf bin/ .tmp/diff --git a/cmd/ax-server/main.go b/cmd/ax-server/main.goindex 0b5cc4b..967b00b 100644--- a/cmd/ax-server/main.go+++ b/cmd/ax-server/main.go@@ -46,6 +46,7 @@ func main() {substratePlaintext booldefaultTemplate stringdefaultTemplateAtespace string+ defaultTaskImage string)flag.StringVar(&listenAddr, "addr", ":8080", "HTTP listen address")@@ -59,6 +60,7 @@ func main() {flag.BoolVar(&substratePlaintext, "substrate-plaintext", false, "Use insecure plaintext gRPC connection to Substrate")flag.StringVar(&defaultTemplate, "template", "default-template", "Default Substrate ActorTemplate name")flag.StringVar(&defaultTemplateAtespace, "template-atespace", v1alpha1.DefaultSystemNamespace, "Default Substrate ActorTemplate atespace")+ flag.StringVar(&defaultTaskImage, "default-task-image", v1alpha1.DefaultTaskImage, "Container image for tasks that do not set spec.image (env: AX_DEFAULT_TASK_IMAGE)")flag.Parse()if envAddr := os.Getenv("ADDR"); envAddr != "" {@@ -70,6 +72,9 @@ func main() {if envPass := os.Getenv("REDIS_PASSWORD"); envPass != "" {redisPassword = envPass}+ if envImage := os.Getenv("AX_DEFAULT_TASK_IMAGE"); envImage != "" {+ defaultTaskImage = envImage+ }logger := slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelInfo}))slog.SetDefault(logger)@@ -79,6 +84,7 @@ func main() {"redisAddr", redisAddr,"substrateEndpoint", substrateEndpoint,"template", defaultTemplate,+ "defaultTaskImage", defaultTaskImage,)rClient := goredis.NewClient(&goredis.Options{@@ -103,7 +109,7 @@ func main() {slog.Warn("could not initialize substrate client; running without substrate reconciliation", "error", err)} else {defer subClient.Close()- reconciler = controller.NewTaskReconciler(subClient, defaultTemplate, defaultTemplateAtespace)+ reconciler = controller.NewTaskReconciler(subClient, defaultTemplate, defaultTemplateAtespace, defaultTaskImage)}srv := server.NewServer(rStore, server.Options{diff --git a/hack/cleanup-e2e-kind.sh b/hack/cleanup-e2e-kind.shnew file mode 100755index 0000000..aefb0e0--- /dev/null+++ b/hack/cleanup-e2e-kind.sh@@ -0,0 +1,32 @@+#!/usr/bin/env bash+# Copyright 2026 Google LLC+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++# Deletes the Kind cluster created for AX E2E tests.++set -euo pipefail++SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"+REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"+TMP_DIR="${REPO_ROOT}/.tmp"++if [[ -z "${KIND_CLUSTER_NAME:-}" && -f "${TMP_DIR}/e2e.env" ]]; then+ # shellcheck disable=SC1091+ source "${TMP_DIR}/e2e.env"+fi++KIND_CLUSTER_NAME="${KIND_CLUSTER_NAME:-ax-e2e}"+echo "==> Deleting Kind cluster '${KIND_CLUSTER_NAME}'..."+kind delete cluster --name "${KIND_CLUSTER_NAME}" || true+rm -f "${TMP_DIR}/kubeconfig-${KIND_CLUSTER_NAME}" "${TMP_DIR}/e2e.env"diff --git a/hack/run-e2e-kind.sh b/hack/run-e2e-kind.shnew file mode 100755index 0000000..dc8bb91--- /dev/null+++ b/hack/run-e2e-kind.sh@@ -0,0 +1,35 @@+#!/usr/bin/env bash+# Copyright 2026 Google LLC+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++# Provisions a Kind cluster with Substrate (at SUBSTRATE_REF, default: go.mod version)+# and AX, then runs the AX E2E test suite.+#+# Set SKIP_SETUP=true to reuse an already set up cluster.+# Set CLEANUP_KIND=true to delete the Kind cluster on exit.++set -euo pipefail++SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"++if [[ "${CLEANUP_KIND:-false}" == "true" ]]; then+ trap '"${SCRIPT_DIR}/cleanup-e2e-kind.sh"' EXIT+fi++if [[ "${SKIP_SETUP:-false}" != "true" ]]; then+ export KIND_CLUSTER_NAME="${KIND_CLUSTER_NAME:-ax-e2e}"+ "${SCRIPT_DIR}/setup-e2e-kind.sh"+fi++"${SCRIPT_DIR}/run-e2e.sh" "$@"diff --git a/hack/run-e2e.sh b/hack/run-e2e.shnew file mode 100755index 0000000..0ed5756--- /dev/null+++ b/hack/run-e2e.sh@@ -0,0 +1,40 @@+#!/usr/bin/env bash+# Copyright 2026 Google LLC+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++# Runs the AX Go E2E test suite against an already-provisioned Kind cluster.++set -euo pipefail++SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"+REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"+TMP_DIR="${REPO_ROOT}/.tmp"++REQUESTED_AX_BIN="${AX_BIN:-}"+if [[ -f "${TMP_DIR}/e2e.env" ]]; then+ # shellcheck disable=SC1091+ source "${TMP_DIR}/e2e.env"+fi++# Rebuild the host ax CLI so tests always exercise the current source, unless+# AX_BIN points at a custom binary.+DEFAULT_AX_BIN="${TMP_DIR}/bin/ax"+export AX_BIN="${REQUESTED_AX_BIN:-${AX_BIN:-${DEFAULT_AX_BIN}}}"+if [[ "${AX_BIN}" == "${DEFAULT_AX_BIN}" ]]; then+ mkdir -p "$(dirname "${AX_BIN}")"+ (cd "${REPO_ROOT}" && CGO_ENABLED=0 go build -o "${AX_BIN}" ./cmd/ax)+fi++cd "${REPO_ROOT}"+exec go test -v -count=1 -timeout "${E2E_TIMEOUT:-30m}" ./test/e2e/... -args --e2e "$@"diff --git a/hack/setup-e2e-kind.sh b/hack/setup-e2e-kind.shnew file mode 100755index 0000000..e9f7fbc--- /dev/null+++ b/hack/setup-e2e-kind.sh@@ -0,0 +1,284 @@+#!/usr/bin/env bash+# Copyright 2026 Google LLC+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++# Sets up a Kind cluster with Substrate (at a configurable branch/ref) and AX+# deployed for end-to-end testing. Re-running the script reuses the existing+# cluster and only reinstalls Substrate when the requested commit changes.+#+# Configurable environment variables:+# SUBSTRATE_REF Branch, tag, commit, or Go module version of substrate (default: version in go.mod)+# SUBSTRATE_REPO Git repository URL for substrate (default: https://github.com/agent-substrate/substrate.git)+# SUBSTRATE_DIR Existing substrate checkout (default: ../substrate if present, else clone to .tmp/substrate)+# KIND_CLUSTER_NAME Name of the Kind cluster (default: ax-e2e)+# KUBECONFIG Path to kubeconfig file (default: .tmp/kubeconfig-${KIND_CLUSTER_NAME})+# AX_SYSTEM_NAMESPACE Kubernetes namespace for AX control plane (default: ax-system)+# AX_WORKER_REPLICAS Number of gVisor worker replicas in the AX WorkerPool (default: 4)+# AX_SNAPSHOTS_BUCKET Snapshot storage bucket URI (default: gs://ate-snapshots/ax/)+# REGISTRY_ADDR Local container registry address (default: localhost:5001)+# RECREATE_KIND Set to true to delete and recreate the Kind cluster+# FORCE_INSTALL_SUBSTRATE Set to true to reinstall Substrate even if the commit is unchanged++set -euo pipefail+export GOWORK=off++SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"+REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"+TMP_DIR="${REPO_ROOT}/.tmp"+E2E_DIR="${REPO_ROOT}/test/e2e"+mkdir -p "${TMP_DIR}/bin"++export KIND_CLUSTER_NAME="${KIND_CLUSTER_NAME:-ax-e2e}"+export KUBECONFIG="${KUBECONFIG:-${TMP_DIR}/kubeconfig-${KIND_CLUSTER_NAME}}"+export KUBE_CONTEXT="kind-${KIND_CLUSTER_NAME}"+export AX_SYSTEM_NAMESPACE="${AX_SYSTEM_NAMESPACE:-ax-system}"+export AX_WORKER_REPLICAS="${AX_WORKER_REPLICAS:-4}"+export AX_SNAPSHOTS_BUCKET="${AX_SNAPSHOTS_BUCKET:-gs://ate-snapshots/ax/}"+export REGISTRY_ADDR="${REGISTRY_ADDR:-localhost:5001}"+SUBSTRATE_REPO="${SUBSTRATE_REPO:-https://github.com/agent-substrate/substrate.git}"+GOARCH="$(go env GOARCH)"++# Default to the Substrate version AX is built against. Go pseudo-versions+SUBSTRATE_REF="${SUBSTRATE_REF:-$(cd "${REPO_ROOT}" && go list -m -f '{{.Version}}' github.com/agent-substrate/substrate)}"+if [[ "${SUBSTRATE_REF}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+-.*-([0-9a-f]{12})$ ]]; then+ SUBSTRATE_REF="${BASH_REMATCH[1]}"+fi++# Use an isolated DOCKER_CONFIG by default so expired gcloud credential helpers+# in ~/.docker/config.json do not break pulling public base images or pushing+# to the local registry.+if [[ -z "${DOCKER_CONFIG:-}" ]]; then+ export DOCKER_CONFIG="${TMP_DIR}/docker-config"+ mkdir -p "${DOCKER_CONFIG}"+ [[ -f "${DOCKER_CONFIG}/config.json" ]] || printf '{}\n' > "${DOCKER_CONFIG}/config.json"+fi++log() {+ printf '\n==> %s\n' "$*" >&2+}++k() {+ kubectl --kubeconfig "${KUBECONFIG}" --context "${KUBE_CONTEXT}" "$@"+}++# render_manifest substitutes only the AX_* variables below so any other `$`+# in the manifests is left untouched.+MANIFEST_VARS=(AX_SYSTEM_NAMESPACE AX_WORKER_REPLICAS AX_SERVER_IMAGE AX_DEFAULT_TASK_IMAGE AX_SNAPSHOTS_BUCKET)+render_manifest() {+ local manifest="${E2E_DIR}/manifests/$1"+ if command -v envsubst > /dev/null 2>&1; then+ envsubst "$(printf '${%s} ' "${MANIFEST_VARS[@]}")" < "${manifest}"+ else+ local sed_args=() var+ for var in "${MANIFEST_VARS[@]}"; do+ sed_args+=(-e "s|\${${var}}|${!var:-}|g")+ done+ sed "${sed_args[@]}" "${manifest}"+ fi+}++# resolve_commit prints the commit SHA that ref names in the git repo at dir,+# trying the ref as-is, as a vX.Y.Z tag, and as a release branch, both locally+# and on origin. Returns non-zero if nothing matches.+resolve_commit() {+ local dir="$1" ref="$2" candidate+ for candidate in "${ref}" "v${ref}" "release-${ref}" "origin/${ref}" "origin/v${ref}" "origin/release-${ref}"; do+ if git -C "${dir}" rev-parse --verify --quiet "${candidate}^{commit}" 2> /dev/null; then+ return 0+ fi+ done+ return 1+}++# resolve_substrate_dir prints a directory containing substrate checked out at+# SUBSTRATE_REF. It never modifies the HEAD of a user-provided checkout: if+# HEAD differs from the requested ref, a detached worktree under .tmp is used.+resolve_substrate_dir() {+ local base_dir="${SUBSTRATE_DIR:-}"+ if [[ -n "${base_dir}" ]]; then+ base_dir="$(cd "${base_dir}" && pwd)"+ elif [[ -d "${REPO_ROOT}/../substrate/.git" ]]; then+ base_dir="$(cd "${REPO_ROOT}/../substrate" && pwd)"+ fi++ if [[ -z "${base_dir}" ]]; then+ base_dir="${TMP_DIR}/substrate"+ if [[ ! -d "${base_dir}/.git" ]]; then+ log "Cloning substrate from ${SUBSTRATE_REPO}..."+ git clone --quiet "${SUBSTRATE_REPO}" "${base_dir}" >&2+ fi+ fi+ [[ -e "${base_dir}/.git" ]] || { echo "SUBSTRATE_DIR=${base_dir} is not a git checkout" >&2; return 1; }++ local target_sha+ if [[ "${base_dir}" == "${TMP_DIR}/substrate" ]] &&+ git -C "${base_dir}" show-ref --verify --quiet "refs/remotes/origin/${SUBSTRATE_REF}"; then+ # Refresh branches in the managed clone without moving its HEAD or a user checkout.+ log "Fetching branch ${SUBSTRATE_REF} in ${base_dir}..."+ git -C "${base_dir}" fetch --quiet origin "+refs/heads/${SUBSTRATE_REF}:refs/remotes/origin/${SUBSTRATE_REF}" >&2 || return 1+ target_sha="$(git -C "${base_dir}" rev-parse "refs/remotes/origin/${SUBSTRATE_REF}^{commit}")"+ elif ! target_sha="$(resolve_commit "${base_dir}" "${SUBSTRATE_REF}")"; then+ log "Fetching ${SUBSTRATE_REF} in ${base_dir}..."+ git -C "${base_dir}" fetch --quiet --tags origin "${SUBSTRATE_REF}" >&2 ||+ git -C "${base_dir}" fetch --quiet --all --tags >&2+ target_sha="$(resolve_commit "${base_dir}" "${SUBSTRATE_REF}")" ||+ { echo "cannot resolve SUBSTRATE_REF=${SUBSTRATE_REF} in ${base_dir}" >&2; return 1; }+ fi++ if [[ "$(git -C "${base_dir}" rev-parse HEAD)" == "${target_sha}" ]]; then+ printf '%s\n' "${base_dir}"+ return 0+ fi++ local wt_dir="${TMP_DIR}/substrate-worktree-${target_sha:0:12}"+ if [[ ! -d "${wt_dir}" ]]; then+ log "Creating detached substrate worktree at ${SUBSTRATE_REF} (${target_sha:0:12})..."+ git -C "${base_dir}" worktree prune >&2 || true+ git -C "${base_dir}" worktree add --quiet --detach "${wt_dir}" "${target_sha}" >&2+ fi+ # Share substrate's tool cache with the worktree to avoid re-downloading tools.+ if [[ -d "${base_dir}/.cache" && ! -e "${wt_dir}/.cache" && ! -L "${wt_dir}/.cache" ]]; then+ ln -s "${base_dir}/.cache" "${wt_dir}/.cache"+ fi+ printf '%s\n' "${wt_dir}"+}++# uninstall_substrate removes Substrate and the cluster-scoped resources it+# owns so that a different Substrate version can be installed cleanly.+uninstall_substrate() {+ local ax_ns="${AX_SYSTEM_NAMESPACE}"+ k -n "${ax_ns}" delete workerpool ax-workers --ignore-not-found --timeout=30s > /dev/null 2>&1 ||+ k -n "${ax_ns}" patch workerpool ax-workers --type=merge -p '{"metadata":{"finalizers":null}}' > /dev/null 2>&1 || true+ k delete validatingadmissionpolicybinding,validatingadmissionpolicy -l app.kubernetes.io/part-of=ate --ignore-not-found > /dev/null 2>&1 || true+ k delete validatingadmissionpolicybinding,validatingadmissionpolicy sandboxconfig-assets --ignore-not-found > /dev/null 2>&1 || true+ if ! k delete sandboxconfig --all --ignore-not-found --timeout=30s > /dev/null 2>&1; then+ local configs+ configs=$(k get sandboxconfig -o name 2> /dev/null || true)+ if [[ -n "${configs}" ]]; then+ echo "${configs}" | xargs kubectl --kubeconfig "${KUBECONFIG}" --context "${KUBE_CONTEXT}" patch --type=merge -p '{"metadata":{"finalizers":null}}' > /dev/null 2>&1 || true+ fi+ fi+ local crds+ crds=$(k get crd -o name 2> /dev/null | grep '\.ate\.dev$' || true)+ if [[ -n "${crds}" ]]; then+ echo "${crds}" | xargs kubectl --kubeconfig "${KUBECONFIG}" --context "${KUBE_CONTEXT}" delete --ignore-not-found > /dev/null 2>&1 || true+ fi+ k delete namespace ate-system --ignore-not-found --wait=true > /dev/null 2>&1 || true+ k label nodes --all ate.dev/substrate-version- > /dev/null 2>&1 || true+}++# build_and_push builds an image, pushes it to the local registry, and prints+# its digest reference.+build_and_push() {+ local tag="$1"+ shift+ docker build -q -t "${tag}" "$@" "${REPO_ROOT}" > /dev/null+ docker push -q "${tag}" > /dev/null+ docker inspect --format='{{index .RepoDigests 0}}' "${tag}"+}++SUBSTRATE_WORK_DIR="$(resolve_substrate_dir)"+SUBSTRATE_COMMIT="$(git -C "${SUBSTRATE_WORK_DIR}" rev-parse HEAD)"+log "Using Substrate checkout at ${SUBSTRATE_WORK_DIR} (ref=${SUBSTRATE_REF}, commit=${SUBSTRATE_COMMIT:0:12})"++# 1. Ensure the Kind cluster and local registry exist.+if [[ "${RECREATE_KIND:-false}" == "true" ]] || ! kind get clusters 2> /dev/null | grep -qx "${KIND_CLUSTER_NAME}"; then+ log "Creating Kind cluster '${KIND_CLUSTER_NAME}' and registry '${REGISTRY_ADDR}'..."+ (cd "${SUBSTRATE_WORK_DIR}" && ./hack/create-kind-cluster.sh)+else+ log "Reusing Kind cluster '${KIND_CLUSTER_NAME}' (set RECREATE_KIND=true to recreate)."+fi+kind export kubeconfig --name "${KIND_CLUSTER_NAME}" --kubeconfig "${KUBECONFIG}"++# 2. Install Substrate unless the requested commit is already installed.+INSTALLED_SUBSTRATE_COMMIT="$(k get namespace ate-system -o jsonpath='{.metadata.annotations.substrate-commit}' 2> /dev/null || true)"+if [[ "${FORCE_INSTALL_SUBSTRATE:-false}" == "true" ]] ||+ [[ "${INSTALLED_SUBSTRATE_COMMIT}" != "${SUBSTRATE_COMMIT}" ]] ||+ ! k -n ate-system get deployment ate-api-server > /dev/null 2>&1; then+ log "Installing Substrate (ate-system) at ${SUBSTRATE_COMMIT:0:12}..."+ if [[ -n "${INSTALLED_SUBSTRATE_COMMIT}" ]] || k get namespace ate-system > /dev/null 2>&1; then+ uninstall_substrate+ fi++ install_args=(--deploy-ate-system --rollout-timeout="${SUBSTRATE_ROLLOUT_TIMEOUT:-3m}")+ # Older Substrate versions do not have --credential-provider.+ if grep -q -- "--credential-provider" "${SUBSTRATE_WORK_DIR}/hack/install-ate.sh" 2> /dev/null; then+ install_args+=(--credential-provider='{"enabled":false}')+ fi+ (cd "${SUBSTRATE_WORK_DIR}" && ./hack/install-ate-kind.sh "${install_args[@]}")+ k annotate namespace ate-system "substrate-commit=${SUBSTRATE_COMMIT}" --overwrite > /dev/null+else+ log "Substrate ${SUBSTRATE_COMMIT:0:12} already installed; waiting for ate-api-server..."+ k -n ate-system wait --for=condition=Available deployment/ate-api-server --timeout=180s+fi++# 3. Deploy the AX namespace and gVisor WorkerPool.+log "Deploying namespace '${AX_SYSTEM_NAMESPACE}' and WorkerPool (${AX_WORKER_REPLICAS} gVisor replicas)..."+render_manifest namespace.yaml | k apply -f -+(+ cd "${SUBSTRATE_WORK_DIR}"+ render_manifest workerpool.yaml |+ KO_DOCKER_REPO="${REGISTRY_ADDR}" ./hack/run-tool.sh ko resolve --platform="linux/${GOARCH}" -f -+) | k apply -f -++log "Waiting for ax-workers to become Ready..."+for _ in $(seq 1 60); do+ k -n "${AX_SYSTEM_NAMESPACE}" get deployment ax-workers > /dev/null 2>&1 && break+ sleep 2+done+k -n "${AX_SYSTEM_NAMESPACE}" rollout status deployment/ax-workers --timeout=180s++# 4. Build AX binaries and images. ax-server and ax-task-runner run in the+# cluster (linux); the ax CLI runs on the host.+log "Building ax (host), ax-server and ax-task-runner (linux/${GOARCH})..."+(+ cd "${REPO_ROOT}"+ mkdir -p "bin/linux_${GOARCH}"+ CGO_ENABLED=0 go build -o "${TMP_DIR}/bin/ax" ./cmd/ax+ CGO_ENABLED=0 GOOS=linux GOARCH="${GOARCH}" go build -o "bin/linux_${GOARCH}/ax-server" ./cmd/ax-server+ CGO_ENABLED=0 GOOS=linux GOARCH="${GOARCH}" go build -o "bin/linux_${GOARCH}/ax-task-runner" ./cmd/ax-task-runner+)++log "Building and pushing images to ${REGISTRY_ADDR}..."+AX_SERVER_IMAGE="$(build_and_push "${REGISTRY_ADDR}/ax-server:e2e" \+ --build-arg TARGETARCH="${GOARCH}" -f "${E2E_DIR}/Dockerfile.server")"+# The E2E task-runner image extends the production image with test fixtures.+docker build -q -t "${REGISTRY_ADDR}/ax-task-runner:base" \+ --build-arg TARGETARCH="${GOARCH}" -f "${REPO_ROOT}/Dockerfile.task-runner" "${REPO_ROOT}" > /dev/null+AX_DEFAULT_TASK_IMAGE="$(build_and_push "${REGISTRY_ADDR}/ax-task-runner:e2e" \+ --build-arg BASE_IMAGE="${REGISTRY_ADDR}/ax-task-runner:base" -f "${E2E_DIR}/Dockerfile.task-runner.e2e")"+export AX_SERVER_IMAGE AX_DEFAULT_TASK_IMAGE+printf ' AX_SERVER_IMAGE=%s\n AX_DEFAULT_TASK_IMAGE=%s\n' "${AX_SERVER_IMAGE}" "${AX_DEFAULT_TASK_IMAGE}"++# 5. Deploy Redis and ax-server. Delete existing deployments first so every+# run starts from empty Redis state.+log "Deploying ax-redis and ax-server in namespace '${AX_SYSTEM_NAMESPACE}'..."+k -n "${AX_SYSTEM_NAMESPACE}" delete deployment ax-redis ax-server --ignore-not-found+render_manifest redis.yaml | k apply -f -+render_manifest ax-server.yaml | k apply -f -+k -n "${AX_SYSTEM_NAMESPACE}" rollout status deployment/ax-redis --timeout=120s+k -n "${AX_SYSTEM_NAMESPACE}" rollout status deployment/ax-server --timeout=120s++# 6. Persist the environment for hack/run-e2e.sh and the Go test harness.+cat > "${TMP_DIR}/e2e.env" << EOF+export KIND_CLUSTER_NAME="${KIND_CLUSTER_NAME}"+export KUBECONFIG="${KUBECONFIG}"+export KUBE_CONTEXT="${KUBE_CONTEXT}"+export AX_SYSTEM_NAMESPACE="${AX_SYSTEM_NAMESPACE}"+export AX_BIN="${TMP_DIR}/bin/ax"+export SUBSTRATE_REF="${SUBSTRATE_REF}"+export SUBSTRATE_COMMIT="${SUBSTRATE_COMMIT}"+EOF++log "Kind E2E environment is ready (configuration written to ${TMP_DIR}/e2e.env)."diff --git a/internal/controller/reconciler.go b/internal/controller/reconciler.goindex 5e31e1d..074a481 100644--- a/internal/controller/reconciler.go+++ b/internal/controller/reconciler.go@@ -63,6 +63,8 @@ type TaskReconciler struct {httpClient *http.ClientdefaultTemplate stringdefaultTemplateAtespace string+ // defaultTaskImage is applied to spec.image for tasks that leave it empty.+ defaultTaskImage string// SecretResolver resolves the Gemini API key for task containers. It defaults// to the Kubernetes secret lookup; tests replace it to avoid touching a cluster.@@ -73,19 +75,25 @@ type TaskReconciler struct {WorkspaceReadyTimeout time.Duration}-// NewTaskReconciler creates a new TaskReconciler.-func NewTaskReconciler(client *substrate.Client, defaultTemplate, defaultTemplateAtespace string) *TaskReconciler {+// NewTaskReconciler creates a new TaskReconciler. Empty defaults fall back to+// "default-template", v1alpha1.DefaultSystemNamespace and+// v1alpha1.DefaultTaskImage respectively.+func NewTaskReconciler(client *substrate.Client, defaultTemplate, defaultTemplateAtespace, defaultTaskImage string) *TaskReconciler {if defaultTemplate == "" {defaultTemplate = "default-template"}if defaultTemplateAtespace == "" {defaultTemplateAtespace = v1alpha1.DefaultSystemNamespace}+ if defaultTaskImage == "" {+ defaultTaskImage = v1alpha1.DefaultTaskImage+ }return &TaskReconciler{client: client,httpClient: &http.Client{Timeout: 2 * time.Second},defaultTemplate: defaultTemplate,defaultTemplateAtespace: defaultTemplateAtespace,+ defaultTaskImage: defaultTaskImage,SecretResolver: model.GetKubernetesSecret,WorkspaceReadyTimeout: defaultWorkspaceReadyTimeout,}@@ -109,7 +117,7 @@ func (r *TaskReconciler) Reconcile(ctx context.Context, task *v1alpha1.Task, wstask.Spec = &v1alpha1.TaskSpec{}}if task.Spec.Image == "" {- task.Spec.Image = v1alpha1.DefaultTaskImage+ task.Spec.Image = r.defaultTaskImage}slog.Info("reconciling task", "name", task.Metadata.Name, "atespace", atespace, "image", task.Spec.Image)diff --git a/internal/controller/reconciler_test.go b/internal/controller/reconciler_test.goindex 37842be..1ca0000 100644--- a/internal/controller/reconciler_test.go+++ b/internal/controller/reconciler_test.go@@ -45,6 +45,7 @@ type mockControlServer struct {deletedActors []stringactorTemplates map[string]boolcreatedTemplates []*ateapipb.ActorTemplate+ templateImages []stringdeletedTemplates []stringgetActorFunc func(ctx context.Context, req *ateapipb.GetActorRequest) (*ateapipb.Actor, error)createAtespaceErr error@@ -80,6 +81,9 @@ func (m *mockControlServer) CreateActorTemplate(_ context.Context, req *ateapipbtmpl := req.GetActorTemplate()m.actorTemplates[tmpl.GetMetadata().GetName()] = truem.createdTemplates = append(m.createdTemplates, tmpl)+ for _, c := range tmpl.GetContainers() {+ m.templateImages = append(m.templateImages, c.GetImage())+ }return tmpl, nil}@@ -238,7 +242,7 @@ func TestTaskReconciler(t *testing.T) {defer client.Close()// 3. Reconcile Task- reconciler := controller.NewTaskReconciler(client, "test-template", v1alpha1.DefaultSystemNamespace)+ reconciler := controller.NewTaskReconciler(client, "test-template", v1alpha1.DefaultSystemNamespace, "")reconciler.SecretResolver = noSecretsreconciler.WorkspaceReadyTimeout = 200 * time.Millisecond@@ -286,6 +290,63 @@ func TestTaskReconciler(t *testing.T) {}}+// TestTaskReconciler_DefaultTaskImage verifies that a task without spec.image is+// defaulted to the image passed to NewTaskReconciler (or the API default when+// none is given), and that the image+// is what ends up in the per-task ActorTemplate.+func TestTaskReconciler_DefaultTaskImage(t *testing.T) {+ for _, tc := range []struct {+ name string+ override string+ want string+ }{+ {name: "api default", want: v1alpha1.DefaultTaskImage},+ {name: "server override", override: "localhost:5001/ax-task-runner:e2e", want: "localhost:5001/ax-task-runner:e2e"},+ } {+ t.Run(tc.name, func(t *testing.T) {+ lis, err := net.Listen("tcp", "127.0.0.1:0")+ if err != nil {+ t.Fatalf("failed to listen: %v", err)+ }+ defer lis.Close()++ mockSrv := &mockControlServer{}+ grpcServer := grpc.NewServer()+ ateapipb.RegisterControlServer(grpcServer, mockSrv)+ go grpcServer.Serve(lis)+ defer grpcServer.Stop()++ client, err := substrate.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials()))+ if err != nil {+ t.Fatalf("failed to create substrate client: %v", err)+ }+ defer client.Close()++ reconciler := controller.NewTaskReconciler(client, "test-template", v1alpha1.DefaultSystemNamespace, tc.override)+ reconciler.SecretResolver = noSecrets+ reconciler.WorkspaceReadyTimeout = 200 * time.Millisecond++ task := &v1alpha1.Task{+ ApiVersion: v1alpha1.APIVersion,+ Kind: v1alpha1.KindTask,+ Metadata: &v1alpha1.ObjectMeta{Name: "no-image", Atespace: "default"},+ Spec: &v1alpha1.TaskSpec{},+ Status: &v1alpha1.TaskStatus{Phase: "Running"},+ }+ reconciled, err := reconciler.Reconcile(context.Background(), task, nil)+ if err != nil {+ t.Fatalf("Reconcile failed: %v", err)+ }+ if got := reconciled.GetSpec().GetImage(); got != tc.want {+ t.Errorf("spec.image = %q, want %q", got, tc.want)+ }+ if len(mockSrv.templateImages) != 1 || mockSrv.templateImages[0] != tc.want {+ t.Errorf("ActorTemplate images = %v, want [%q]", mockSrv.templateImages, tc.want)+ }+ })+ }+}+func TestTaskReconciler_Suspend(t *testing.T) {ctx := context.Background()@@ -307,7 +368,7 @@ func TestTaskReconciler_Suspend(t *testing.T) {}defer client.Close()- reconciler := controller.NewTaskReconciler(client, "test-template", v1alpha1.DefaultSystemNamespace)+ reconciler := controller.NewTaskReconciler(client, "test-template", v1alpha1.DefaultSystemNamespace, "")reconciler.SecretResolver = noSecretsreconciler.WorkspaceReadyTimeout = 200 * time.Millisecond@@ -362,7 +423,7 @@ func TestTaskReconciler_SuspendError(t *testing.T) {}defer client.Close()- reconciler := controller.NewTaskReconciler(client, "test-template", "ax-system")+ reconciler := controller.NewTaskReconciler(client, "test-template", "ax-system", "")reconciler.SecretResolver = noSecretsworkerIp := "10.1.1.1"@@ -421,7 +482,7 @@ func TestTaskReconciler_TemplateCreationFailed(t *testing.T) {}defer client.Close()- reconciler := controller.NewTaskReconciler(client, "test-template", "ax-system")+ reconciler := controller.NewTaskReconciler(client, "test-template", "ax-system", "")reconciler.SecretResolver = noSecretsreconciler.WorkspaceReadyTimeout = 200 * time.Millisecond@@ -499,7 +560,7 @@ func TestTaskReconciler_TemplateCreationTransient(t *testing.T) {}defer client.Close()- reconciler := controller.NewTaskReconciler(client, "test-template", "ax-system")+ reconciler := controller.NewTaskReconciler(client, "test-template", "ax-system", "")reconciler.SecretResolver = noSecretsreconciler.WorkspaceReadyTimeout = 200 * time.Millisecond@@ -572,7 +633,7 @@ func TestTaskReconciler_WorkspaceReady(t *testing.T) {}defer client.Close()- reconciler := controller.NewTaskReconciler(client, "test-template", v1alpha1.DefaultSystemNamespace)+ reconciler := controller.NewTaskReconciler(client, "test-template", v1alpha1.DefaultSystemNamespace, "")reconciler.SecretResolver = noSecretsreconciler.WorkspaceReadyTimeout = 200 * time.Millisecond@@ -666,7 +727,7 @@ func TestTaskReconciler_ResourceLimits(t *testing.T) {}defer client.Close()- reconciler := controller.NewTaskReconciler(client, "test-template", "ax-system")+ reconciler := controller.NewTaskReconciler(client, "test-template", "ax-system", "")reconciler.SecretResolver = noSecretsreconciler.WorkspaceReadyTimeout = 200 * time.Millisecond@@ -763,7 +824,7 @@ func TestTaskReconciler_ResourceLimitsFailurePaths(t *testing.T) {grpcServer.Stop()lis.Close()})- reconciler := controller.NewTaskReconciler(client, "test-template", "ax-system")+ reconciler := controller.NewTaskReconciler(client, "test-template", "ax-system", "")reconciler.SecretResolver = noSecretsreconciler.WorkspaceReadyTimeout = 200 * time.Millisecondreturn reconciler@@ -881,7 +942,7 @@ func TestReconcileDelete_RemovesActorAndTemplates(t *testing.T) {}defer client.Close()- reconciler := controller.NewTaskReconciler(client, "test-template", v1alpha1.DefaultSystemNamespace)+ reconciler := controller.NewTaskReconciler(client, "test-template", v1alpha1.DefaultSystemNamespace, "")reconciler.SecretResolver = noSecretsreconciler.WorkspaceReadyTimeout = 200 * time.Millisecond@@ -944,7 +1005,7 @@ func TestReconcileDelete_BlocksUntilActorDeleted(t *testing.T) {}defer client.Close()- reconciler := controller.NewTaskReconciler(client, "test-template", v1alpha1.DefaultSystemNamespace)+ reconciler := controller.NewTaskReconciler(client, "test-template", v1alpha1.DefaultSystemNamespace, "")reconciler.SecretResolver = noSecretsif err := reconciler.ReconcileDelete(ctx, "default", "slow-delete-task"); err != nil {@@ -986,7 +1047,7 @@ func TestReconcileDelete_ActorDeletionTimeout(t *testing.T) {}defer client.Close()- reconciler := controller.NewTaskReconciler(client, "test-template", v1alpha1.DefaultSystemNamespace)+ reconciler := controller.NewTaskReconciler(client, "test-template", v1alpha1.DefaultSystemNamespace, "")reconciler.SecretResolver = noSecretserr = reconciler.ReconcileDelete(ctx, "default", "stuck-task")@@ -1050,7 +1111,7 @@ func TestTaskReconciler_TransientSubstrateUnavailable(t *testing.T) {}defer client.Close()- reconciler := controller.NewTaskReconciler(client, "test-template", v1alpha1.DefaultSystemNamespace)+ reconciler := controller.NewTaskReconciler(client, "test-template", v1alpha1.DefaultSystemNamespace, "")reconciler.SecretResolver = noSecretsreconciler.WorkspaceReadyTimeout = 50 * time.Milliseconddiff --git a/test/e2e/Dockerfile.server b/test/e2e/Dockerfile.servernew file mode 100644index 0000000..91d61f4--- /dev/null+++ b/test/e2e/Dockerfile.server@@ -0,0 +1,22 @@+# Copyright 2026 Google LLC+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++FROM alpine:3.21++RUN apk add --no-cache ca-certificates++ARG TARGETARCH=amd64+COPY bin/linux_${TARGETARCH}/ax-server /usr/local/bin/ax-server++ENTRYPOINT ["/usr/local/bin/ax-server"]diff --git a/test/e2e/Dockerfile.task-runner.e2e b/test/e2e/Dockerfile.task-runner.e2enew file mode 100644index 0000000..6f65239--- /dev/null+++ b/test/e2e/Dockerfile.task-runner.e2e@@ -0,0 +1,36 @@+# Copyright 2026 Google LLC+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++ARG BASE_IMAGE=ax-task-runner:base+FROM ${BASE_IMAGE}++# Initialize the sample Git repository used by E2E tests+RUN mkdir -p /workspace /tmp/sample-src && \+ cd /tmp/sample-src && \+ git init -b main && \+ git config user.name "AX E2E" && \+ git config user.email "[email protected]" && \+ echo "# Sample E2E Repo" > README.md && \+ echo "print('hello from git repo v1')" > app.py && \+ git add README.md app.py && \+ git commit -m "Initial commit" && \+ git tag v1.0.0 && \+ echo "print('hello from git repo main')" > app.py && \+ git commit -am "Second commit on main" && \+ git clone --bare /tmp/sample-src /opt/ax-e2e-sample-repo.git && \+ rm -rf /tmp/sample-src++WORKDIR /workspace+EXPOSE 80+ENTRYPOINT ["/usr/local/bin/ax-task-runner"]diff --git a/test/e2e/README.md b/test/e2e/README.mdnew file mode 100644index 0000000..5754129--- /dev/null+++ b/test/e2e/README.md@@ -0,0 +1,69 @@+# AX End-to-End (E2E) Tests++This directory contains the end-to-end test suite for **AX** running on+**Substrate** in a local [Kind](https://kind.sigs.k8s.io/) cluster, along with+the manifests and Dockerfiles used to deploy AX for the tests.++The suite exercises real gVisor sandboxes, Substrate's `ate-api-server`,+in-cluster RustFS snapshot storage, Redis, `ax-server`, `ax-task-runner`, and+the `ax` CLI, against a configurable Substrate branch, tag, or commit.++| Path | Purpose |+|---|---|+| `harness_test.go` | `TestMain`, cluster connection setup, and shared helpers |+| `e2e_test.go` | The test cases |+| `manifests/` | Namespace, WorkerPool, Redis, and ax-server manifests (rendered with `envsubst`) |+| `Dockerfile.server` | ax-server image |+| `Dockerfile.task-runner.e2e` | Extends `Dockerfile.task-runner` with test fixtures (a sample Git repo) |++## Quick Start++Provision a cluster and run the suite against the Substrate version pinned in `go.mod`:++```bash+make test-e2e+```++Against Substrate `main`, or any branch, tag, or commit:++```bash+make test-e2e-head+make test-e2e SUBSTRATE_REF=release-0.4+```++### Iterating++Provisioning takes several minutes; reuse the cluster between runs:++```bash+make e2e-setup # provision Kind + Substrate + AX (re-run after changing server code)+make e2e-run # run all tests+make e2e-run E2E_ARGS='-test.run TestE2E_ModelCRUD'+make e2e-cleanup # delete the Kind cluster+```++`make e2e-run` rebuilds the `ax` CLI, but not the in-cluster `ax-server` or+`ax-task-runner` images; re-run `make e2e-setup` to redeploy those.++`go test ./...` skips this package unless `-args --e2e` is passed or `AX_E2E=true` is set.++## Configuration++`hack/setup-e2e-kind.sh` reads these environment variables and writes the+resolved values the tests need to `.tmp/e2e.env`.++| Variable | Default | Description |+|---|---|---|+| `SUBSTRATE_REF` | version in `go.mod` | Substrate branch, tag, commit, or Go module version |+| `SUBSTRATE_DIR` | `../substrate` if present | Local Substrate checkout. If its `HEAD` is not `SUBSTRATE_REF`, a detached worktree is created under `.tmp/` |+| `SUBSTRATE_REPO` | `https://github.com/agent-substrate/substrate.git` | Cloned into `.tmp/substrate` when no local checkout exists |+| `KIND_CLUSTER_NAME` | `ax-e2e` | Kind cluster name |+| `KUBECONFIG` | `.tmp/kubeconfig-${KIND_CLUSTER_NAME}` | Kubeconfig for the Kind cluster |+| `AX_SYSTEM_NAMESPACE` | `ax-system` | Namespace for `ax-redis`, `ax-server`, and the `ax-workers` WorkerPool |+| `AX_WORKER_REPLICAS` | `4` | gVisor worker replicas |+| `AX_SNAPSHOTS_BUCKET` | `gs://ate-snapshots/ax/` | Snapshot location in Substrate's in-cluster RustFS |+| `RECREATE_KIND` | `false` | Delete and recreate the Kind cluster |+| `FORCE_INSTALL_SUBSTRATE` | `false` | Reinstall Substrate even if the requested commit is already installed |+| `CLEANUP_KIND` | `false` | `hack/run-e2e-kind.sh` only: delete the cluster on exit |+| `SKIP_SETUP` | `false` | `hack/run-e2e-kind.sh` only: skip provisioning |+| `E2E_TIMEOUT` | `30m` | `go test` timeout |diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.gonew file mode 100644index 0000000..fc9fffd--- /dev/null+++ b/test/e2e/e2e_test.go@@ -0,0 +1,564 @@+// Copyright 2026 Google LLC+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package e2e_test++import (+ "context"+ "errors"+ "fmt"+ "io"+ "strings"+ "testing"+ "time"++ "github.com/agent-substrate/substrate/pkg/proto/ateapipb"+ "github.com/google/ax/pkg/apis/v1alpha1"+ "google.golang.org/grpc/codes"+ "google.golang.org/grpc/status"+)++const (+ defaultAtespace = "default"++ phaseSuspended = "Suspended"+ phaseRunning = "Running"++ // sampleRepo is a bare Git repository baked into the E2E task-runner image+ // (see test/e2e/Dockerfile.task-runner.e2e). Tag v1.0.0 contains+ // app.py printing "hello from git repo v1".+ sampleRepo = "file:///opt/ax-e2e-sample-repo.git"++ // probeURL is the in-sandbox address of the ax-task-runner probe server.+ probeURL = "http://127.0.0.1:80"++ // metadataURL is the in-sandbox address of the ax-task-runner metadata server.+ metadataURL = "http://127.0.0.1:8099"+)++// TestE2E_CLIVersionAndAutoTunnel verifies `ax version` and that, without+// --server, the CLI automatically opens a port-forward tunnel to ax-server+// that `ax ctx` reports as healthy and `ax tunnel stop` tears down.+func TestE2E_CLIVersionAndAutoTunnel(t *testing.T) {+ assertContains(t, "'ax version'", env.mustRunAX(t, "", "version"), "ax version")++ // Without --server, ax should auto-create a port-forward tunnel to+ // ax-server using the kube context. Use a dedicated AX_HOME so the tunnel+ // state is isolated from the rest of the suite.+ axHome := t.TempDir()+ runTunneled := func(args ...string) string {+ t.Helper()+ cmd, cancel := env.axCommand(axHome, append([]string{"--context", env.KubeContext, "-n", env.AXSystemNamespace}, args...)...)+ defer cancel()+ out, err := cmd.CombinedOutput()+ if err != nil {+ t.Fatalf("ax %s failed: %v\nOutput:\n%s", strings.Join(args, " "), err, out)+ }+ return string(out)+ }+ t.Cleanup(func() {+ cmd, cancel := env.axCommand(axHome, "tunnel", "stop", env.KubeContext)+ defer cancel()+ _ = cmd.Run()+ })++ runTunneled("get", "tasks")++ listOut := runTunneled("ctx")+ assertContains(t, "'ax ctx'", strings.ToLower(listOut), strings.ToLower(env.KubeContext), "healthy")++ runTunneled("tunnel", "stop", env.KubeContext)+ assertContains(t, "'ax ctx' after stop", runTunneled("ctx"), "No background tunnel running")+}++const workspaceCRUDManifest = `apiVersion: ax.io/v1alpha1+kind: Workspace+metadata:+ name: {{.Name}}+ atespace: {{.Atespace}}+spec:+ git:+ - name: origin+ repo: {{.Repo}}+ branch: v1.0.0+ files:+ - path: config/settings.json+ content: '{"env":"{{.Env}}"}'+ mcp:+ servers:+ - name: local-mcp+ endpoint: http://localhost:9000+ skills:+ path: /workspace/.agents/skills+`++// TestE2E_WorkspaceCRUD verifies the Workspace lifecycle through the CLI:+// `ax apply` reports created / unchanged / configured, `ax get` and+// `ax describe` show the updated spec, and `ax delete` removes it.+func TestE2E_WorkspaceCRUD(t *testing.T) {+ ns := defaultAtespace+ name := uniqueName("ws-crud")+ env.deleteOnCleanup(t, ns, "workspace", name)++ manifest := func(envValue string) string {+ return renderYAML(t, workspaceCRUDManifest, map[string]string{+ "Name": name, "Atespace": ns, "Repo": sampleRepo, "Env": envValue,+ })+ }+ ref := "workspace.ax.io/" + name++ assertContains(t, "first apply", env.apply(t, ns, manifest("e2e-v1")), ref+" created")+ assertContains(t, "identical re-apply", env.apply(t, ns, manifest("e2e-v1")), ref+" unchanged")+ assertContains(t, "modified apply", env.apply(t, ns, manifest("e2e-v2")), ref+" configured")++ assertContains(t, "'ax get workspaces'", env.mustRunAX(t, ns, "get", "workspaces"), name)+ assertContains(t, "'ax get workspace'", env.mustRunAX(t, ns, "get", "workspace", name),+ "e2e-v2", "local-mcp", "config/settings.json")+ assertContains(t, "'ax describe workspace'", env.mustRunAX(t, ns, "describe", "workspace", name),+ "local-mcp", ".agents/skills")++ assertContains(t, "'ax delete workspace'", env.mustRunAX(t, ns, "delete", "workspace", name), "deleted")+ _, err := env.AXClient.GetWorkspace(t.Context(), &v1alpha1.GetWorkspaceRequest{Name: name, Atespace: ns})+ if status.Code(err) != codes.NotFound {+ t.Fatalf("GetWorkspace after delete: got %v, want NotFound", err)+ }+}++const modelCRUDManifest = `apiVersion: ax.io/v1alpha1+kind: Model+metadata:+ name: {{.Name}}+ atespace: {{.Atespace}}+spec:+ provider: google+ model: {{.Model}}+ secretKey:+ name: gemini-secret+ key: api-key+`++// TestE2E_ModelCRUD verifies the same apply / get / describe / delete+// lifecycle as TestE2E_WorkspaceCRUD for a Model.+func TestE2E_ModelCRUD(t *testing.T) {+ ns := defaultAtespace+ name := uniqueName("model-crud")+ env.deleteOnCleanup(t, ns, "model", name)++ manifest := func(model string) string {+ return renderYAML(t, modelCRUDManifest, map[string]string{"Name": name, "Atespace": ns, "Model": model})+ }+ ref := "model.ax.io/" + name++ assertContains(t, "first apply", env.apply(t, ns, manifest("gemini-2.5-pro")), ref+" created")+ assertContains(t, "identical re-apply", env.apply(t, ns, manifest("gemini-2.5-pro")), ref+" unchanged")+ assertContains(t, "modified apply", env.apply(t, ns, manifest("gemini-2.5-flash")), ref+" configured")++ assertContains(t, "'ax get models'", env.mustRunAX(t, ns, "get", "models"), name, "gemini-2.5-flash")+ assertContains(t, "'ax get model'", env.mustRunAX(t, ns, "get", "model", name), "gemini-2.5-flash")+ assertContains(t, "'ax describe model'", env.mustRunAX(t, ns, "describe", "model", name), "gemini-secret")++ env.mustRunAX(t, ns, "delete", "model", name)+ _, err := env.AXClient.GetModel(t.Context(), &v1alpha1.GetModelRequest{Name: name, Atespace: ns})+ if status.Code(err) != codes.NotFound {+ t.Fatalf("GetModel after delete: got %v, want NotFound", err)+ }+}++const debugTaskManifest = `apiVersion: ax.io/v1alpha1+kind: Task+metadata:+ name: {{.Name}}+ atespace: {{.Atespace}}+spec:+ debug: {{.Debug}}+`++// taskManifest renders a minimal Task with no workspaces.+func taskManifest(t *testing.T, atespace, name string, debug bool) string {+ return renderYAML(t, debugTaskManifest, map[string]any{"Name": name, "Atespace": atespace, "Debug": debug})+}++// TestE2E_TaskLifecycleAndSubstrateSync verifies that a Task and its Substrate+// resources stay in sync across its lifecycle:+// - A new Task starts Suspended and is immutable.+// - Substrate has a suspended Actor backed by a per-task ActorTemplate+// (<task>-tmpl-<hash>) with the /readyz:80 wakeup probe and a snapshot config.+// - Resuming moves the Task to Running with WorkspaceReady and Ready+// conditions, on the worker Substrate reports.+// - Deleting the Task removes the Actor and ActorTemplate.+func TestE2E_TaskLifecycleAndSubstrateSync(t *testing.T) {+ ctx := t.Context()+ ns := defaultAtespace+ name := uniqueName("task-life")+ env.deleteOnCleanup(t, ns, "task", name)++ manifest := taskManifest(t, ns, name, true)++ // 1. A new task is created Suspended, backed by an Actor of the same name.+ assertContains(t, "'ax apply'", env.apply(t, ns, manifest), "task.ax.io/"+name+" created")+ task := env.getTask(t, ns, name)+ assertPhase(t, task, phaseSuspended)+ if got := task.GetStatus().GetActor(); got != name {+ t.Fatalf("task actor = %q, want %q", got, name)+ }++ // 2. Tasks are immutable: both CreateTask and `ax apply` reject a duplicate.+ _, err := env.AXClient.CreateTask(ctx, &v1alpha1.CreateTaskRequest{Task: &v1alpha1.Task{+ Metadata: &v1alpha1.ObjectMeta{Name: name, Atespace: ns},+ Spec: &v1alpha1.TaskSpec{},+ }})+ if status.Code(err) != codes.FailedPrecondition {+ t.Fatalf("duplicate CreateTask: got %v, want FailedPrecondition", err)+ }+ out, err := env.runAX(ns, "apply", "-f", writeManifest(t, manifest))+ if err == nil {+ t.Fatalf("expected re-applying an existing task to fail, got:\n%s", out)+ }+ assertContains(t, "duplicate 'ax apply'", out, "already exists and is immutable")++ // 3. Substrate has a suspended Actor using a per-task ActorTemplate.+ actor := env.getActor(t, ns, name, ateapipb.ActorState_ACTOR_STATE_SUSPENDED)+ tmplName := actor.GetActorTemplate().GetName()+ if !strings.HasPrefix(tmplName, name+"-tmpl-") {+ t.Fatalf("actor template = %q, want prefix %q", tmplName, name+"-tmpl-")+ }+ tmpl, err := env.SubstrateClient.GetActorTemplate(ctx, ns, tmplName)+ if err != nil {+ t.Fatalf("Substrate GetActorTemplate(%s/%s): %v", ns, tmplName, err)+ }+ if containers := tmpl.GetContainers(); len(containers) != 1 {+ t.Fatalf("actor template has %d containers, want 1", len(containers))+ }+ if probe := tmpl.GetContainers()[0].GetWakeupProbe().GetHttpGet(); probe.GetPath() != "/readyz" || probe.GetPort() != 80 {+ t.Fatalf("unexpected WakeupProbe: %+v", tmpl.GetContainers()[0].GetWakeupProbe())+ }+ // Newer Substrate versions may return SnapshotConfig fields AX's proto does+ // not know about, so accept either a known storage location or unknown fields.+ if sc := tmpl.GetSnapshotConfig(); sc.GetStorageLocation() == "" && len(sc.ProtoReflect().GetUnknown()) == 0 {+ t.Fatalf("unexpected SnapshotConfig: %+v", sc)+ }++ // 4. Resuming moves the task to Running on a worker reported by Substrate.+ assertContains(t, "'ax resume task'", env.mustRunAX(t, ns, "resume", "task", name), "resumed")+ task = env.getTask(t, ns, name)+ assertPhase(t, task, phaseRunning)+ assertCondition(t, task, "WorkspaceReady", "True", "SetupComplete")+ assertCondition(t, task, "Ready", "True", "TaskRunning")+ workerIP := task.GetStatus().GetWorkerIp()+ if workerIP == "" {+ t.Fatal("expected non-empty WorkerIp on running task")+ }+ actor = env.getActor(t, ns, name, ateapipb.ActorState_ACTOR_STATE_RUNNING)+ if ips := actor.GetStatus().GetWorkerAssignment().GetWorkerPodIps(); len(ips) == 0 || ips[0] != workerIP {+ t.Fatalf("Substrate WorkerPodIps = %v, want first IP %q", ips, workerIP)+ }++ // 5. Deleting the task removes the AX task, the Actor, and its ActorTemplate.+ assertContains(t, "'ax delete task'", env.mustRunAX(t, ns, "delete", "task", name), "deleted")+ if _, err := env.AXClient.GetTask(ctx, &v1alpha1.GetTaskRequest{Name: name, Atespace: ns}); status.Code(err) != codes.NotFound {+ t.Fatalf("AX GetTask after delete: got %v, want NotFound", err)+ }+ if _, err := env.SubstrateClient.GetActor(ctx, ns, name); status.Code(err) != codes.NotFound {+ t.Fatalf("Substrate GetActor after delete: got %v, want NotFound", err)+ }+ if _, err := env.SubstrateClient.GetActorTemplate(ctx, ns, tmplName); status.Code(err) != codes.NotFound {+ t.Fatalf("Substrate GetActorTemplate(%s) after delete: got %v, want NotFound", tmplName, err)+ }+}++const metadataManifest = `apiVersion: ax.io/v1alpha1+kind: Workspace+metadata:+ name: {{.Workspace}}+ atespace: {{.Atespace}}+spec:+ files:+ - path: hello.txt+ content: "hello from workspace"+---+apiVersion: ax.io/v1alpha1+kind: Task+metadata:+ name: {{.Task}}+ atespace: {{.Atespace}}+spec:+ debug: true+ workspace:+ name: {{.Workspace}}+ path: /workspace+ env:+ - name: CUSTOM_E2E_VAR+ value: "custom-val-123"+ command:+ - sh+ - -c+ - "echo cmd-ran-ok > /workspace/cmd-ran.txt"+`++// TestE2E_MetadataServerAndGuestExec verifies the guest environment of a+// running task: `ax ssh` sees spec.env, AX_METADATA_URL, and workspace files;+// spec.command runs; the in-sandbox metadata server serves health and+// task/workspace metadata; guest exit codes propagate; and `ax ssh` is+// rejected when spec.debug is false.+func TestE2E_MetadataServerAndGuestExec(t *testing.T) {+ ns := defaultAtespace+ ws := uniqueName("ws-meta")+ task := uniqueName("task-meta")+ env.deleteOnCleanup(t, ns, "workspace", ws)+ env.deleteOnCleanup(t, ns, "task", task)++ env.apply(t, ns, renderYAML(t, metadataManifest, map[string]string{"Workspace": ws, "Task": task, "Atespace": ns}))+ env.mustRunAX(t, ns, "resume", "task", task)++ // 1. `ax ssh` sees spec.env, the metadata URL, and workspace files.+ out := env.ssh(t, ns, task, "echo ENV=$CUSTOM_E2E_VAR META=$AX_METADATA_URL FILE=$(cat /workspace/hello.txt)")+ assertContains(t, "'ax ssh' output", out,+ "ENV=custom-val-123", "META="+metadataURL, "FILE=hello from workspace")++ // spec.command runs in the background after setup; wait for its output.+ err := poll(10*time.Second, 250*time.Millisecond, func() error {+ out, err := env.runAX(ns, "ssh", task, "--", "cat", "/workspace/cmd-ran.txt")+ if err != nil {+ return fmt.Errorf("%w: %s", err, out)+ }+ if got := strings.TrimSpace(out); got != "cmd-ran-ok" {+ return fmt.Errorf("cmd-ran.txt = %q", got)+ }+ return nil+ })+ if err != nil {+ t.Fatalf("spec.command did not write /workspace/cmd-ran.txt: %v", err)+ }++ // 2. The in-sandbox probe server serves health/readiness on port 80, and+ // the loopback metadata server serves task/workspace metadata at AX_METADATA_URL.+ curl := func(url string) string {+ t.Helper()+ return env.mustRunAX(t, ns, "ssh", task, "--", "curl", "-sf", url)+ }+ for _, path := range []string{"/healthz", "/readyz"} {+ if got := strings.TrimSpace(curl(probeURL + path)); got != "ok" {+ t.Fatalf("GET %s = %q, want \"ok\"", path, got)+ }+ }+ assertContains(t, "task metadata", curl(metadataURL+"/metadata/v1alpha1/ax/task"), task, "custom-val-123")+ assertContains(t, "workspace metadata", curl(metadataURL+"/metadata/v1alpha1/ax/workspace"), ws, "hello from workspace")++ // 3. `ax ssh` propagates a non-zero guest exit code.+ if _, err := env.runAX(ns, "ssh", task, "--", "sh", "-c", "exit 17"); err == nil {+ t.Fatal("expected 'ax ssh' to fail when the guest command exits 17")+ }++ // 4. `ax ssh` is rejected for tasks without spec.debug.+ noDebug := uniqueName("task-nodebug")+ env.deleteOnCleanup(t, ns, "task", noDebug)+ env.apply(t, ns, taskManifest(t, ns, noDebug, false))+ env.mustRunAX(t, ns, "resume", "task", noDebug)+ out, err = env.runAX(ns, "ssh", noDebug, "--", "echo", "should-fail")+ if err == nil {+ t.Fatalf("expected 'ax ssh' to fail when debug=false, got:\n%s", out)+ }+ if !strings.Contains(out, "spec.debug: true") && !strings.Contains(out, "debugging is not enabled") {+ t.Fatalf("expected debug-disabled error, got:\n%s", out)+ }+}++const workspaceSetupManifest = `apiVersion: ax.io/v1alpha1+kind: Workspace+metadata:+ name: {{.GitWorkspace}}+ atespace: {{.Atespace}}+spec:+ git:+ - name: sample+ dir: "."+ repo: {{.Repo}}+ branch: v1.0.0+ files:+ - path: overlay/config.yaml+ content: "mode: git-plus-overlay"+ skills:+ path: /workspace/primary/.agents/skills+---+apiVersion: ax.io/v1alpha1+kind: Task+metadata:+ name: {{.Task}}+ atespace: {{.Atespace}}+spec:+ debug: true+ workspace:+ name: {{.GitWorkspace}}+ path: /workspace/primary+`++// TestE2E_WorkspaceSetup_FilesAndGit verifies workspace setup for a task:+// custom mount path (/workspace/primary), working directory set to the mount+// path, spec.git checkout at a tag, overlay spec.files, spec.skills.path, and+// the /ax/initialized-* maiden-run marker.+func TestE2E_WorkspaceSetup_FilesAndGit(t *testing.T) {+ ns := defaultAtespace+ wsGit := uniqueName("ws-git")+ task := uniqueName("task-ws")+ env.deleteOnCleanup(t, ns, "workspace", wsGit)+ env.deleteOnCleanup(t, ns, "task", task)++ env.apply(t, ns, renderYAML(t, workspaceSetupManifest, map[string]string{+ "GitWorkspace": wsGit, "Task": task, "Atespace": ns, "Repo": sampleRepo,+ }))+ env.mustRunAX(t, ns, "resume", "task", task)++ // - The working directory is the workspace's mount path.+ // - The workspace has the repo checked out at v1.0.0, the overlay file, and+ // the skills directory.+ // - A maiden-run marker is written for the workspace mount path.+ out := env.ssh(t, ns, task, `set -e+pwd+cat /workspace/primary/app.py+cat /workspace/primary/overlay/config.yaml+test -d /workspace/primary/.agents/skills && echo SKILLS_DIR_OK+test -f /ax/initialized-workspace-primary && echo MARKERS_OK+`)+ assertContains(t, "workspace setup check", out,+ "/workspace/primary",+ "hello from git repo v1",+ "mode: git-plus-overlay",+ "SKILLS_DIR_OK",+ "MARKERS_OK",+ )+}++const persistenceManifest = `apiVersion: ax.io/v1alpha1+kind: Workspace+metadata:+ name: {{.Workspace}}+ atespace: {{.Atespace}}+spec:+ files:+ - path: seed.txt+ content: "initial-seed"+---+apiVersion: ax.io/v1alpha1+kind: Task+metadata:+ name: {{.Task}}+ atespace: {{.Atespace}}+spec:+ debug: true+ workspace:+ name: {{.Workspace}}+ path: /workspace+`++// TestE2E_TaskSuspendAndResumePersistence verifies that suspend/resume with+// VOLUMES-fidelity snapshots preserves the durable /workspace volume while+// the rootfs (/tmp) is reset.+func TestE2E_TaskSuspendAndResumePersistence(t *testing.T) {+ ns := defaultAtespace+ ws := uniqueName("ws-persist")+ task := uniqueName("task-persist")+ env.deleteOnCleanup(t, ns, "workspace", ws)+ env.deleteOnCleanup(t, ns, "task", task)++ env.apply(t, ns, renderYAML(t, persistenceManifest, map[string]string{"Workspace": ws, "Task": task, "Atespace": ns}))+ env.mustRunAX(t, ns, "resume", "task", task)++ // 1. Write to the durable /workspace volume and to the ephemeral rootfs.+ env.ssh(t, ns, task, "echo durable-data-42 > /workspace/durable.txt && echo ephemeral-tmp > /tmp/ephemeral.txt")++ // 2. Suspend: the task and its Actor are suspended and the worker released.+ assertContains(t, "'ax suspend task'", env.mustRunAX(t, ns, "suspend", "task", task), "suspended")+ got := env.getTask(t, ns, task)+ assertPhase(t, got, phaseSuspended)+ assertCondition(t, got, "Ready", "False", "TaskSuspended")+ if ip := got.GetStatus().GetWorkerIp(); ip != "" {+ t.Fatalf("expected empty WorkerIp when suspended, got %q", ip)+ }+ actor := env.getActor(t, ns, task, ateapipb.ActorState_ACTOR_STATE_SUSPENDED)+ if len(actor.GetStatus().GetSnapshots()) == 0 && len(actor.GetStatus().ProtoReflect().GetUnknown()) == 0 {+ t.Fatal("expected non-empty Snapshots on suspended actor")+ }++ // 3. Resume: VOLUMES-fidelity snapshots restore /workspace but cold-boot the rootfs.+ env.mustRunAX(t, ns, "resume", "task", task)+ out := env.ssh(t, ns, task,+ "echo DURABLE=$(cat /workspace/durable.txt); if [ -f /tmp/ephemeral.txt ]; then echo TMP_EXISTS; else echo TMP_GONE; fi")+ assertContains(t, "post-resume check", out, "DURABLE=durable-data-42", "TMP_GONE")+}++// TestE2E_WatchDescribeAndAtespaceIsolation verifies that a task in a custom+// atespace is invisible from the default atespace, that WatchTask emits+// INITIAL and then MODIFIED events as the task is resumed, and that+// `ax describe task` reports the running task.+func TestE2E_WatchDescribeAndAtespaceIsolation(t *testing.T) {+ ctx, cancel := context.WithTimeout(t.Context(), 2*time.Minute)+ defer cancel()++ ns := uniqueName("team-e2e")+ task := uniqueName("task-watch")+ env.deleteOnCleanup(t, ns, "task", task)++ env.apply(t, ns, taskManifest(t, ns, task, true))++ // 1. The task is visible only in its own atespace.+ if _, err := env.AXClient.GetTask(ctx, &v1alpha1.GetTaskRequest{Name: task, Atespace: defaultAtespace}); status.Code(err) != codes.NotFound {+ t.Fatalf("GetTask in %q atespace: got %v, want NotFound", defaultAtespace, err)+ }+ if out := env.mustRunAX(t, defaultAtespace, "get", "tasks"); strings.Contains(out, task) {+ t.Fatalf("task %s unexpectedly listed in %q atespace:\n%s", task, defaultAtespace, out)+ }+ assertContains(t, "'ax get tasks' in "+ns, env.mustRunAX(t, ns, "get", "tasks"), task)++ // 2. WatchTask emits INITIAL (Suspended), then MODIFIED (Running) on resume.+ stream, err := env.AXClient.WatchTask(ctx, &v1alpha1.WatchTaskRequest{Name: task, Atespace: ns})+ if err != nil {+ t.Fatalf("WatchTask: %v", err)+ }+ evt, err := stream.Recv()+ if err != nil {+ t.Fatalf("receiving INITIAL event: %v", err)+ }+ if evt.GetAction() != "INITIAL" || evt.GetTask().GetStatus().GetPhase() != phaseSuspended {+ t.Fatalf("first event: action=%q phase=%q, want INITIAL/%s", evt.GetAction(), evt.GetTask().GetStatus().GetPhase(), phaseSuspended)+ }++ resumeErr := make(chan error, 1)+ go func() {+ out, err := env.runAX(ns, "resume", "task", task)+ if err != nil {+ err = fmt.Errorf("%w: %s", err, out)+ }+ resumeErr <- err+ }()++ for {+ evt, err := stream.Recv()+ if errors.Is(err, io.EOF) {+ t.Fatal("WatchTask stream ended before a MODIFIED/Running event")+ }+ if err != nil {+ t.Fatalf("receiving WatchTask event: %v", err)+ }+ if evt.GetAction() == "MODIFIED" && evt.GetTask().GetStatus().GetPhase() == phaseRunning {+ break+ }+ }+ if err := <-resumeErr; err != nil {+ t.Fatalf("ax resume task: %v", err)+ }++ // 3. `ax describe task` reflects the running task and its conditions.+ assertContains(t, "'ax describe task'", env.mustRunAX(t, ns, "describe", "task", task),+ task, ns, phaseRunning, "WorkspaceReady", "TaskRunning")+}diff --git a/test/e2e/harness_test.go b/test/e2e/harness_test.gonew file mode 100644index 0000000..e5870a7--- /dev/null+++ b/test/e2e/harness_test.go@@ -0,0 +1,516 @@+// Copyright 2026 Google LLC+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package e2e_test++import (+ "bufio"+ "bytes"+ "context"+ "crypto/rand"+ "encoding/hex"+ "errors"+ "flag"+ "fmt"+ "net"+ "net/http"+ "os"+ "os/exec"+ "path/filepath"+ "strings"+ "testing"+ "text/template"+ "time"++ "github.com/agent-substrate/substrate/pkg/proto/ateapipb"+ "github.com/google/ax/internal/substrate"+ "github.com/google/ax/pkg/apis/v1alpha1"+ "google.golang.org/grpc"+ "google.golang.org/grpc/credentials/insecure"+)++var runE2E = flag.Bool("e2e", false, "run end-to-end tests against a live Kind/Substrate/AX cluster")++const (+ // ateSystemNamespace is the namespace Substrate's control plane is installed into.+ ateSystemNamespace = "ate-system"+ // ateAPIAudience is the token audience and TLS authority of the Substrate Control API.+ ateAPIAudience = "api.ate-system.svc"++ // axCommandTimeout bounds a single `ax` CLI invocation.+ axCommandTimeout = 2 * time.Minute+ // readyTimeout bounds waiting for port-forwards and health endpoints.+ readyTimeout = 15 * time.Second+)++// suiteEnv holds the connections shared by every test in the suite. It is+// populated once by TestMain.+type suiteEnv struct {+ Kubeconfig string+ KubeContext string+ AXSystemNamespace string+ AXBin string+ AXServerURL string++ AXClient v1alpha1.AXClient+ SubstrateClient *substrate.Client++ // axHome isolates ax CLI state (tunnels, config) from the user's ~/.ax.+ axHome string+ closers []func()+}++var env *suiteEnv++func TestMain(m *testing.M) {+ flag.Parse()+ if !*runE2E && os.Getenv("AX_E2E") != "true" {+ fmt.Println("Skipping AX E2E tests (pass -args --e2e or set AX_E2E=true to run)")+ os.Exit(0)+ }++ var err error+ env, err = newSuiteEnv()+ if err != nil {+ fmt.Fprintf(os.Stderr, "E2E setup failed: %v\n", err)+ os.Exit(1)+ }+ code := m.Run()+ env.Close()+ os.Exit(code)+}++// newSuiteEnv verifies the cluster provisioned by hack/setup-e2e-kind.sh is+// healthy and connects to both ax-server and the Substrate Control API.+func newSuiteEnv() (_ *suiteEnv, err error) {+ wd, err := os.Getwd()+ if err != nil {+ return nil, err+ }+ repoRoot := filepath.Clean(filepath.Join(wd, "..", ".."))+ repoTmp := filepath.Join(repoRoot, ".tmp")++ // Pick up defaults written by hack/setup-e2e-kind.sh; explicit env vars win.+ if err := loadEnvFileIfUnset(filepath.Join(repoTmp, "e2e.env")); err != nil {+ return nil, err+ }++ clusterName := getenvDefault("KIND_CLUSTER_NAME", "ax-e2e")+ s := &suiteEnv{+ Kubeconfig: getenvDefault("KUBECONFIG", filepath.Join(repoTmp, "kubeconfig-"+clusterName)),+ KubeContext: getenvDefault("KUBE_CONTEXT", "kind-"+clusterName),+ AXSystemNamespace: getenvDefault("AX_SYSTEM_NAMESPACE", v1alpha1.DefaultSystemNamespace),+ AXBin: getenvDefault("AX_BIN", filepath.Join(repoTmp, "bin", "ax")),+ }+ defer func() {+ if err != nil {+ s.Close()+ }+ }()++ suiteTmp, err := os.MkdirTemp("", "ax-e2e-")+ if err != nil {+ return nil, err+ }+ s.closers = append(s.closers, func() { _ = os.RemoveAll(suiteTmp) })+ s.axHome = filepath.Join(suiteTmp, "ax-home")++ if err := s.verifyCluster(); err != nil {+ return nil, err+ }+ if err := ensureAXBinary(repoRoot, s.AXBin); err != nil {+ return nil, err+ }+ if err := s.connectAXServer(); err != nil {+ return nil, err+ }+ if err := s.connectSubstrate(filepath.Join(suiteTmp, "ateapi.token")); err != nil {+ return nil, err+ }+ return s, nil+}++// Close releases all suite resources in reverse order of acquisition.+func (s *suiteEnv) Close() {+ for i := len(s.closers) - 1; i >= 0; i-- {+ s.closers[i]()+ }+ s.closers = nil+}++func (s *suiteEnv) verifyCluster() error {+ const hint = "run ./hack/setup-e2e-kind.sh first"+ for _, ns := range []string{ateSystemNamespace, s.AXSystemNamespace} {+ if _, err := s.kubectl("get", "namespace", ns); err != nil {+ return fmt.Errorf("namespace %q not found (%s): %w", ns, hint, err)+ }+ }+ for ns, deploy := range map[string]string{+ ateSystemNamespace: "ate-api-server",+ s.AXSystemNamespace: "ax-server",+ } {+ if _, err := s.kubectl("-n", ns, "wait", "--for=condition=Available", "deployment/"+deploy, "--timeout=30s"); err != nil {+ return fmt.Errorf("deployment %s/%s not available (%s): %w", ns, deploy, hint, err)+ }+ }+ return nil+}++// ensureAXBinary builds the ax CLI for the host if it does not already exist.+func ensureAXBinary(repoRoot, path string) error {+ if _, err := os.Stat(path); err == nil {+ return nil+ }+ cmd := exec.Command("go", "build", "-o", path, "./cmd/ax")+ cmd.Dir = repoRoot+ cmd.Env = append(os.Environ(), "CGO_ENABLED=0")+ if out, err := cmd.CombinedOutput(); err != nil {+ return fmt.Errorf("building ax CLI: %w\n%s", err, out)+ }+ return nil+}++func (s *suiteEnv) connectAXServer() error {+ port, err := s.startPortForward(s.AXSystemNamespace, "svc/ax-server", 8080)+ if err != nil {+ return fmt.Errorf("port-forwarding ax-server: %w", err)+ }+ addr := fmt.Sprintf("127.0.0.1:%d", port)+ s.AXServerURL = "http://" + addr++ if err := waitForHTTPHealth(s.AXServerURL+"/healthz", readyTimeout); err != nil {+ return fmt.Errorf("waiting for ax-server /healthz: %w", err)+ }++ conn, err := grpc.NewClient(addr, grpc.WithTransportCredentials(insecure.NewCredentials()))+ if err != nil {+ return fmt.Errorf("connecting to ax-server gRPC: %w", err)+ }+ s.closers = append(s.closers, func() { _ = conn.Close() })+ s.AXClient = v1alpha1.NewAXClient(conn)+ return nil+}++// connectSubstrate connects directly to the Substrate Control API so tests can+// verify the Actors and ActorTemplates AX creates on their behalf.+func (s *suiteEnv) connectSubstrate(tokenFile string) error {+ port, err := s.startPortForward(ateSystemNamespace, "svc/api", 443)+ if err != nil {+ return fmt.Errorf("port-forwarding %s/svc/api: %w", ateSystemNamespace, err)+ }++ // Authenticate as ax-server's service account, just like ax-server does.+ token, err := s.kubectl("create", "token", "ax-server", "-n", s.AXSystemNamespace,+ "--audience="+ateAPIAudience, "--duration=1h")+ if err != nil {+ return fmt.Errorf("minting Substrate Control API token: %w", err)+ }+ if err := os.WriteFile(tokenFile, []byte(strings.TrimSpace(token)), 0o600); err != nil {+ return fmt.Errorf("writing Substrate token file: %w", err)+ }++ client, err := substrate.NewClientWithOptions(substrate.ClientOptions{+ Target: fmt.Sprintf("127.0.0.1:%d", port),+ Authority: ateAPIAudience,+ InsecureTLS: true,+ TokenFile: tokenFile,+ })+ if err != nil {+ return fmt.Errorf("connecting to Substrate Control API: %w", err)+ }+ s.closers = append(s.closers, func() { _ = client.Close() })+ s.SubstrateClient = client++ ctx, cancel := context.WithTimeout(context.Background(), readyTimeout)+ defer cancel()+ if err := client.EnsureAtespace(ctx, defaultAtespace); err != nil {+ return fmt.Errorf("verifying Substrate Control API readiness: %w", err)+ }+ return nil+}++// kubectlArgs prefixes args with the suite's kubeconfig and context.+func (s *suiteEnv) kubectlArgs(args ...string) []string {+ return append([]string{"--kubeconfig", s.Kubeconfig, "--context", s.KubeContext}, args...)+}++func (s *suiteEnv) kubectl(args ...string) (string, error) {+ out, err := exec.Command("kubectl", s.kubectlArgs(args...)...).CombinedOutput()+ if err != nil {+ return string(out), fmt.Errorf("kubectl %s: %w (%s)", strings.Join(args, " "), err, bytes.TrimSpace(out))+ }+ return string(out), nil+}++// startPortForward runs `kubectl port-forward` in the background and returns+// the local port once it accepts connections. The process is stopped by Close.+func (s *suiteEnv) startPortForward(namespace, resource string, remotePort int) (int, error) {+ localPort, err := freeLocalPort()+ if err != nil {+ return 0, err+ }++ cmd := exec.Command("kubectl", s.kubectlArgs(+ "-n", namespace, "port-forward", resource, fmt.Sprintf("%d:%d", localPort, remotePort))...)+ var stderr bytes.Buffer+ cmd.Stderr = &stderr+ if err := cmd.Start(); err != nil {+ return 0, err+ }+ stop := func() {+ _ = cmd.Process.Kill()+ _ = cmd.Wait()+ }++ addr := fmt.Sprintf("127.0.0.1:%d", localPort)+ err = poll(readyTimeout, 150*time.Millisecond, func() error {+ conn, err := net.DialTimeout("tcp", addr, 250*time.Millisecond)+ if err != nil {+ return err+ }+ return conn.Close()+ })+ if err != nil {+ stop()+ return 0, fmt.Errorf("port-forward to %s/%s did not become ready: %w (%s)", namespace, resource, err, stderr.String())+ }+ s.closers = append(s.closers, stop)+ return localPort, nil+}++func freeLocalPort() (int, error) {+ ln, err := net.Listen("tcp", "127.0.0.1:0")+ if err != nil {+ return 0, err+ }+ defer ln.Close()+ return ln.Addr().(*net.TCPAddr).Port, nil+}++func waitForHTTPHealth(url string, timeout time.Duration) error {+ client := &http.Client{Timeout: time.Second}+ return poll(timeout, 200*time.Millisecond, func() error {+ resp, err := client.Get(url)+ if err != nil {+ return err+ }+ _ = resp.Body.Close()+ if resp.StatusCode != http.StatusOK {+ return fmt.Errorf("GET %s: status %d", url, resp.StatusCode)+ }+ return nil+ })+}++// poll calls fn every interval until it returns nil or timeout elapses, in+// which case the last error is returned.+func poll(timeout, interval time.Duration, fn func() error) error {+ deadline := time.Now().Add(timeout)+ for {+ err := fn()+ if err == nil {+ return nil+ }+ if time.Now().After(deadline) {+ return err+ }+ time.Sleep(interval)+ }+}++// ---------------------------------------------------------------------------+// ax CLI helpers+// ---------------------------------------------------------------------------++// axCommand returns an `ax` invocation whose environment points at the E2E+// cluster, with AX_SERVER and AX_HOME scrubbed so the caller's shell does not+// leak in. The command is killed after axCommandTimeout.+func (s *suiteEnv) axCommand(axHome string, args ...string) (*exec.Cmd, context.CancelFunc) {+ ctx, cancel := context.WithTimeout(context.Background(), axCommandTimeout)+ cmd := exec.CommandContext(ctx, s.AXBin, args...)+ for _, kv := range os.Environ() {+ switch k, _, _ := strings.Cut(kv, "="); k {+ case "AX_SERVER", "AX_HOME", "KUBECONFIG":+ continue+ }+ cmd.Env = append(cmd.Env, kv)+ }+ cmd.Env = append(cmd.Env, "KUBECONFIG="+s.Kubeconfig, "AX_HOME="+axHome)+ return cmd, cancel+}++// runAX runs `ax --server <url> --context <context> [-a atespace] args...` and returns its+// combined output.+func (s *suiteEnv) runAX(atespace string, args ...string) (string, error) {+ fullArgs := []string{"--server", s.AXServerURL, "--context", s.KubeContext}+ if atespace != "" {+ fullArgs = append(fullArgs, "-a", atespace)+ }+ cmd, cancel := s.axCommand(s.axHome, append(fullArgs, args...)...)+ defer cancel()+ out, err := cmd.CombinedOutput()+ return string(out), err+}++func (s *suiteEnv) mustRunAX(t *testing.T, atespace string, args ...string) string {+ t.Helper()+ out, err := s.runAX(atespace, args...)+ if err != nil {+ t.Fatalf("ax %s failed: %v\nOutput:\n%s", strings.Join(args, " "), err, out)+ }+ return out+}++// ssh runs a shell script inside the task's sandbox via `ax ssh`.+func (s *suiteEnv) ssh(t *testing.T, atespace, task, script string) string {+ t.Helper()+ return s.mustRunAX(t, atespace, "ssh", task, "--", "sh", "-c", script)+}++// writeManifest writes manifest to a temp file and returns its path.+func writeManifest(t *testing.T, manifest string) string {+ t.Helper()+ f := filepath.Join(t.TempDir(), "manifest.yaml")+ if err := os.WriteFile(f, []byte(manifest), 0o644); err != nil {+ t.Fatalf("writing manifest: %v", err)+ }+ return f+}++// apply runs `ax apply -f` on manifest and returns the CLI output.+func (s *suiteEnv) apply(t *testing.T, atespace, manifest string) string {+ t.Helper()+ return s.mustRunAX(t, atespace, "apply", "-f", writeManifest(t, manifest))+}++// deleteOnCleanup registers a best-effort `ax delete <kind> <name>` to run+// when the test finishes. Resources are deleted in reverse registration order.+func (s *suiteEnv) deleteOnCleanup(t *testing.T, atespace, kind, name string) {+ t.Cleanup(func() {+ if out, err := s.runAX(atespace, "delete", kind, name); err != nil && !strings.Contains(out, "NotFound") {+ t.Logf("cleanup: ax delete %s %s: %v\n%s", kind, name, err, out)+ }+ })+}++// ---------------------------------------------------------------------------+// Assertions and fixtures+// ---------------------------------------------------------------------------++// renderYAML executes tmpl (a text/template) with data.+func renderYAML(t *testing.T, tmpl string, data any) string {+ t.Helper()+ var b strings.Builder+ if err := template.Must(template.New("manifest").Option("missingkey=error").Parse(tmpl)).Execute(&b, data); err != nil {+ t.Fatalf("rendering manifest: %v", err)+ }+ return b.String()+}++func uniqueName(prefix string) string {+ var b [4]byte+ _, _ = rand.Read(b[:])+ return prefix + "-" + hex.EncodeToString(b[:])+}++// assertContains fails the test if out does not contain every want.+func assertContains(t *testing.T, what, out string, wants ...string) {+ t.Helper()+ for _, want := range wants {+ if !strings.Contains(out, want) {+ t.Fatalf("expected %s to contain %q, got:\n%s", what, want, out)+ }+ }+}++func (s *suiteEnv) getTask(t *testing.T, atespace, name string) *v1alpha1.Task {+ t.Helper()+ task, err := s.AXClient.GetTask(t.Context(), &v1alpha1.GetTaskRequest{Name: name, Atespace: atespace})+ if err != nil {+ t.Fatalf("GetTask(%s/%s): %v", atespace, name, err)+ }+ return task+}++func assertPhase(t *testing.T, task *v1alpha1.Task, want string) {+ t.Helper()+ if got := task.GetStatus().GetPhase(); got != want {+ t.Fatalf("task %s: phase = %q, want %q", task.GetMetadata().GetName(), got, want)+ }+}++func assertCondition(t *testing.T, task *v1alpha1.Task, condType, wantStatus, wantReason string) {+ t.Helper()+ for _, c := range task.GetStatus().GetConditions() {+ if c.GetType() != condType {+ continue+ }+ if c.GetStatus() != wantStatus || (wantReason != "" && c.GetReason() != wantReason) {+ t.Fatalf("condition %s: got status=%q reason=%q (message=%q), want status=%q reason=%q",+ condType, c.GetStatus(), c.GetReason(), c.GetMessage(), wantStatus, wantReason)+ }+ return+ }+ t.Fatalf("condition %q not found on task %s (conditions: %+v)", condType, task.GetMetadata().GetName(), task.GetStatus().GetConditions())+}++// getActor fetches the Substrate Actor backing a task and checks its state.+func (s *suiteEnv) getActor(t *testing.T, atespace, name string, wantState ateapipb.ActorState) *ateapipb.Actor {+ t.Helper()+ actor, err := s.SubstrateClient.GetActor(t.Context(), atespace, name)+ if err != nil {+ t.Fatalf("Substrate GetActor(%s/%s): %v", atespace, name, err)+ }+ if got := actor.GetStatus().GetState(); got != wantState {+ t.Fatalf("Substrate actor %s/%s: state = %v, want %v", atespace, name, got, wantState)+ }+ return actor+}++func getenvDefault(key, def string) string {+ if v := os.Getenv(key); v != "" {+ return v+ }+ return def+}++// loadEnvFileIfUnset sets variables from a file of `export KEY="value"` lines,+// skipping keys already present in the environment. A missing file is not an error.+func loadEnvFileIfUnset(path string) error {+ f, err := os.Open(path)+ if errors.Is(err, os.ErrNotExist) {+ return nil+ }+ if err != nil {+ return err+ }+ defer f.Close()++ scanner := bufio.NewScanner(f)+ for scanner.Scan() {+ line := strings.TrimSpace(scanner.Text())+ if line == "" || strings.HasPrefix(line, "#") {+ continue+ }+ k, v, ok := strings.Cut(strings.TrimPrefix(line, "export "), "=")+ if !ok {+ continue+ }+ k = strings.TrimSpace(k)+ if os.Getenv(k) == "" {+ _ = os.Setenv(k, strings.Trim(strings.TrimSpace(v), `"'`))+ }+ }+ return scanner.Err()+}diff --git a/test/e2e/manifests/ax-server.yaml b/test/e2e/manifests/ax-server.yamlnew file mode 100644index 0000000..c0127f6--- /dev/null+++ b/test/e2e/manifests/ax-server.yaml@@ -0,0 +1,131 @@+apiVersion: v1+kind: ServiceAccount+metadata:+ name: ax-server+ namespace: ${AX_SYSTEM_NAMESPACE}+---+apiVersion: rbac.authorization.k8s.io/v1+kind: ClusterRole+metadata:+ name: ax-server+rules:+ - apiGroups: [""]+ resources: ["secrets"]+ verbs: ["get", "list", "watch"]+---+apiVersion: rbac.authorization.k8s.io/v1+kind: ClusterRoleBinding+metadata:+ name: ax-server+subjects:+ - kind: ServiceAccount+ name: ax-server+ namespace: ${AX_SYSTEM_NAMESPACE}+roleRef:+ apiGroup: rbac.authorization.k8s.io+ kind: ClusterRole+ name: ax-server+---+apiVersion: apps/v1+kind: Deployment+metadata:+ name: ax-server+ namespace: ${AX_SYSTEM_NAMESPACE}+ labels:+ app.kubernetes.io/name: ax-server+ app.kubernetes.io/part-of: ax+spec:+ replicas: 1+ selector:+ matchLabels:+ app.kubernetes.io/name: ax-server+ template:+ metadata:+ labels:+ app.kubernetes.io/name: ax-server+ spec:+ serviceAccountName: ax-server+ containers:+ - name: ax-server+ image: ${AX_SERVER_IMAGE}+ imagePullPolicy: IfNotPresent+ args:+ - "--addr=:8080"+ - "--redis-addr=ax-redis.${AX_SYSTEM_NAMESPACE}.svc.cluster.local:6379"+ - "--substrate-endpoint=api.ate-system.svc.cluster.local:443"+ - "--substrate-authority=api.ate-system.svc"+ - "--substrate-token-file=/var/run/secrets/ateapi/token"+ - "--substrate-ca-file=/run/servicedns-ca/trust-bundle.pem"+ - "--template=default-template"+ - "--template-atespace=${AX_SYSTEM_NAMESPACE}"+ env:+ - name: ATENET_ROUTER_ADDR+ value: "atenet-router.ate-system.svc.cluster.local:80"+ - name: AX_SNAPSHOTS_BUCKET+ value: "${AX_SNAPSHOTS_BUCKET}"+ - name: AX_DEFAULT_TASK_IMAGE+ value: "${AX_DEFAULT_TASK_IMAGE}"+ ports:+ - containerPort: 8080+ name: http+ resources:+ requests:+ cpu: 100m+ memory: 128Mi+ limits:+ cpu: 1000m+ memory: 1Gi+ readinessProbe:+ httpGet:+ path: /healthz+ port: 8080+ initialDelaySeconds: 2+ periodSeconds: 5+ livenessProbe:+ httpGet:+ path: /healthz+ port: 8080+ initialDelaySeconds: 5+ periodSeconds: 10+ volumeMounts:+ - mountPath: /var/run/secrets/ateapi+ name: ate-token+ readOnly: true+ - mountPath: /run/servicedns-ca+ name: servicedns-ca+ readOnly: true+ volumes:+ - name: ate-token+ projected:+ defaultMode: 420+ sources:+ - serviceAccountToken:+ audience: api.ate-system.svc+ expirationSeconds: 7200+ path: token+ - name: servicedns-ca+ projected:+ defaultMode: 420+ sources:+ - clusterTrustBundle:+ labelSelector:+ matchLabels:+ podcert.ate.dev/canarying: live+ path: trust-bundle.pem+ signerName: servicedns.podcert.ate.dev/identity+---+apiVersion: v1+kind: Service+metadata:+ name: ax-server+ namespace: ${AX_SYSTEM_NAMESPACE}+ labels:+ app.kubernetes.io/name: ax-server+ app.kubernetes.io/part-of: ax+spec:+ ports:+ - port: 8080+ targetPort: 8080+ name: http+ selector:+ app.kubernetes.io/name: ax-serverdiff --git a/test/e2e/manifests/namespace.yaml b/test/e2e/manifests/namespace.yamlnew file mode 100644index 0000000..0992a08--- /dev/null+++ b/test/e2e/manifests/namespace.yaml@@ -0,0 +1,6 @@+apiVersion: v1+kind: Namespace+metadata:+ name: ${AX_SYSTEM_NAMESPACE}+ labels:+ ate.dev/api-access: "true"diff --git a/test/e2e/manifests/redis.yaml b/test/e2e/manifests/redis.yamlnew file mode 100644index 0000000..ddb3192--- /dev/null+++ b/test/e2e/manifests/redis.yaml@@ -0,0 +1,47 @@+apiVersion: apps/v1+kind: Deployment+metadata:+ name: ax-redis+ namespace: ${AX_SYSTEM_NAMESPACE}+ labels:+ app.kubernetes.io/name: ax-redis+ app.kubernetes.io/part-of: ax+spec:+ replicas: 1+ selector:+ matchLabels:+ app.kubernetes.io/name: ax-redis+ template:+ metadata:+ labels:+ app.kubernetes.io/name: ax-redis+ spec:+ containers:+ - name: redis+ image: redis:7-alpine+ ports:+ - containerPort: 6379+ name: redis+ resources:+ requests:+ cpu: 100m+ memory: 128Mi+ limits:+ cpu: 1000m+ memory: 1Gi+---+apiVersion: v1+kind: Service+metadata:+ name: ax-redis+ namespace: ${AX_SYSTEM_NAMESPACE}+ labels:+ app.kubernetes.io/name: ax-redis+ app.kubernetes.io/part-of: ax+spec:+ ports:+ - port: 6379+ targetPort: 6379+ name: redis+ selector:+ app.kubernetes.io/name: ax-redisdiff --git a/test/e2e/manifests/workerpool.yaml b/test/e2e/manifests/workerpool.yamlnew file mode 100644index 0000000..2def94b--- /dev/null+++ b/test/e2e/manifests/workerpool.yaml@@ -0,0 +1,12 @@+apiVersion: ate.dev/v1alpha1+kind: WorkerPool+metadata:+ name: ax-workers+ namespace: ${AX_SYSTEM_NAMESPACE}+spec:+ replicas: ${AX_WORKER_REPLICAS}+ workerImage: ko://github.com/agent-substrate/substrate/cmd/ateom-gvisor+ sandboxClasses:+ - name: gvisor+ configRef:+ name: gvisor-default