Agent Executor (AX) is Google's open agentic orchestration runtime
Go
// Copyright 2026 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package controller_test
import (
"context"
"errors"
"net"
"net/http"
"strings"
"testing"
"time"
"github.com/agent-substrate/substrate/pkg/proto/ateapipb"
"github.com/google/ax/internal/controller"
"github.com/google/ax/internal/substrate"
"github.com/google/ax/pkg/apis/v1alpha1"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/credentials/insecure"
"google.golang.org/grpc/status"
"google.golang.org/protobuf/proto"
)
type mockControlServer struct {
ateapipb.UnimplementedControlServer
workerIP string
createdAtespaces []string
createdActors []string
resumedActors []string
suspendedActors []string
suspendErr error
deletedActors []string
actorTemplates map[string]bool
createdTemplates []*ateapipb.ActorTemplate
templateImages []string
deletedTemplates []string
getActorFunc func(ctx context.Context, req *ateapipb.GetActorRequest) (*ateapipb.Actor, error)
createAtespaceErr error
createActorErr error
resumeActorErr error
crashedActor string
revertedActors []string
// createTemplateErr, when set, is returned by CreateActorTemplate to stand in
// for Substrate rejecting a template (for example an invalid quantity).
createTemplateErr error
}
// noSecrets is a SecretResolver for tests: it never finds a key and never touches a cluster.
func noSecrets(context.Context, string, string, string) (string, error) {
return "", nil
}
func (m *mockControlServer) GetActorTemplate(_ context.Context, req *ateapipb.GetActorTemplateRequest) (*ateapipb.ActorTemplate, error) {
ref := req.GetActorTemplate()
if !m.actorTemplates[ref.GetName()] {
return nil, status.Error(codes.NotFound, "template not found")
}
return &ateapipb.ActorTemplate{Metadata: &ateapipb.ResourceMetadata{Name: ref.GetName(), Atespace: ref.GetAtespace()}}, nil
}
func (m *mockControlServer) CreateActorTemplate(_ context.Context, req *ateapipb.CreateActorTemplateRequest) (*ateapipb.ActorTemplate, error) {
if m.createTemplateErr != nil {
return nil, m.createTemplateErr
}
if m.actorTemplates == nil {
m.actorTemplates = make(map[string]bool)
}
tmpl := req.GetActorTemplate()
m.actorTemplates[tmpl.GetMetadata().GetName()] = true
m.createdTemplates = append(m.createdTemplates, tmpl)
for _, c := range tmpl.GetContainers() {
m.templateImages = append(m.templateImages, c.GetImage())
}
return tmpl, nil
}
func (m *mockControlServer) CreateAtespace(ctx context.Context, req *ateapipb.CreateAtespaceRequest) (*ateapipb.Atespace, error) {
if m.createAtespaceErr != nil {
return nil, m.createAtespaceErr
}
name := ""
if req.Atespace != nil && req.Atespace.Metadata != nil {
name = req.Atespace.Metadata.Name
}
m.createdAtespaces = append(m.createdAtespaces, name)
return &ateapipb.Atespace{Metadata: &ateapipb.ResourceMetadata{Name: name}}, nil
}
func (m *mockControlServer) CreateActor(ctx context.Context, req *ateapipb.CreateActorRequest) (*ateapipb.Actor, error) {
if m.createActorErr != nil {
return nil, m.createActorErr
}
name := ""
if req.Actor != nil && req.Actor.Metadata != nil {
name = req.Actor.Metadata.Name
}
if name == m.crashedActor {
return nil, status.Error(codes.AlreadyExists, "actor exists")
}
m.createdActors = append(m.createdActors, name)
return &ateapipb.Actor{
Metadata: &ateapipb.ResourceMetadata{Name: name},
Status: &ateapipb.ActorStatus{
State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED,
},
}, nil
}
func (m *mockControlServer) ResumeActor(ctx context.Context, req *ateapipb.ResumeActorRequest) (*ateapipb.ResumeActorResponse, error) {
if m.resumeActorErr != nil {
return nil, m.resumeActorErr
}
name := ""
if req.Actor != nil {
name = req.Actor.Name
}
m.resumedActors = append(m.resumedActors, name)
wIP := "10.244.1.42"
if m.workerIP != "" {
wIP = m.workerIP
}
return &ateapipb.ResumeActorResponse{
Actor: &ateapipb.Actor{
Metadata: &ateapipb.ResourceMetadata{Name: name},
Status: &ateapipb.ActorStatus{
State: ateapipb.ActorState_ACTOR_STATE_RUNNING,
WorkerAssignment: &ateapipb.WorkerAssignment{
WorkerPod: "worker-pod-1",
WorkerPodIps: []string{wIP},
},
},
},
Resumed: true,
}, nil
}
func (m *mockControlServer) SuspendActor(ctx context.Context, req *ateapipb.SuspendActorRequest) (*ateapipb.SuspendActorResponse, error) {
if m.suspendErr != nil {
return nil, m.suspendErr
}
name := ""
if req.Actor != nil {
name = req.Actor.Name
}
m.suspendedActors = append(m.suspendedActors, name)
return &ateapipb.SuspendActorResponse{}, nil
}
func (m *mockControlServer) RevertActor(_ context.Context, req *ateapipb.RevertActorRequest) (*ateapipb.RevertActorResponse, error) {
name := req.GetActor().GetName()
m.revertedActors = append(m.revertedActors, name)
return &ateapipb.RevertActorResponse{Actor: &ateapipb.Actor{
Metadata: &ateapipb.ResourceMetadata{Name: name},
Status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED},
}}, nil
}
func (m *mockControlServer) DeleteActor(ctx context.Context, req *ateapipb.DeleteActorRequest) (*ateapipb.Actor, error) {
name := req.GetActor().GetName()
m.deletedActors = append(m.deletedActors, name)
return &ateapipb.Actor{Metadata: &ateapipb.ResourceMetadata{Name: name}}, nil
}
func (m *mockControlServer) GetActor(ctx context.Context, req *ateapipb.GetActorRequest) (*ateapipb.Actor, error) {
if m.getActorFunc != nil {
return m.getActorFunc(ctx, req)
}
name := req.GetActor().GetName()
if name == m.crashedActor {
return &ateapipb.Actor{
Metadata: &ateapipb.ResourceMetadata{Name: name},
Status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_CRASHED},
}, nil
}
for _, del := range m.deletedActors {
if del == name {
return nil, status.Errorf(codes.NotFound, "actor %q not found", name)
}
}
for _, a := range m.createdActors {
if a == name {
return &ateapipb.Actor{
Metadata: &ateapipb.ResourceMetadata{Name: name},
Status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_RUNNING},
}, nil
}
}
return nil, status.Errorf(codes.NotFound, "actor %q not found", name)
}
func (m *mockControlServer) ListActorTemplates(ctx context.Context, req *ateapipb.ListActorTemplatesRequest) (*ateapipb.ListActorTemplatesResponse, error) {
resp := &ateapipb.ListActorTemplatesResponse{}
for name := range m.actorTemplates {
resp.ActorTemplates = append(resp.ActorTemplates, &ateapipb.ActorTemplate{
Metadata: &ateapipb.ResourceMetadata{Name: name, Atespace: req.GetAtespace()},
})
}
return resp, nil
}
func (m *mockControlServer) DeleteActorTemplate(ctx context.Context, req *ateapipb.DeleteActorTemplateRequest) (*ateapipb.ActorTemplate, error) {
name := req.GetActorTemplate().GetName()
delete(m.actorTemplates, name)
m.deletedTemplates = append(m.deletedTemplates, name)
return &ateapipb.ActorTemplate{Metadata: &ateapipb.ResourceMetadata{Name: name}}, nil
}
func TestTaskReconciler(t *testing.T) {
ctx := context.Background()
// 1. Start in-process mock gRPC Substrate server
lis, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatalf("failed to listen: %v", err)
}
defer lis.Close()
mockSrv := &mockControlServer{}
grpcServer := grpc.NewServer()
ateapipb.RegisterControlServer(grpcServer, mockSrv)
go grpcServer.Serve(lis)
defer grpcServer.Stop()
// 2. Initialize Substrate client
client, err := substrate.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials()))
if err != nil {
t.Fatalf("failed to create substrate client: %v", err)
}
defer client.Close()
// 3. Reconcile Task
reconciler := controller.NewTaskReconciler(client, "test-template", v1alpha1.DefaultSystemNamespace, "")
reconciler.SecretResolver = noSecrets
reconciler.WorkspaceReadyTimeout = 200 * time.Millisecond
task := &v1alpha1.Task{
ApiVersion: v1alpha1.APIVersion,
Kind: v1alpha1.KindTask,
Metadata: &v1alpha1.ObjectMeta{
Name: "test-task",
Atespace: "default",
},
Spec: &v1alpha1.TaskSpec{
Image: "ghrc.io/my-org/my-image",
Command: []string{"/bin/task-runner"},
},
// A client-supplied actor name must not survive: the actor is always
// named after the task.
Status: &v1alpha1.TaskStatus{Actor: "not-the-task", Phase: "Running"},
}
reconciled, err := reconciler.Reconcile(ctx, task, nil)
if err != nil {
t.Fatalf("Reconcile failed: %v", err)
}
// 4. Validate reconciliation results
if reconciled.Status.Phase != "Running" {
t.Errorf("expected phase 'Running', got %q", reconciled.Status.Phase)
}
if reconciled.Status.Actor != "test-task" {
t.Errorf("expected actor 'test-task', got %q", reconciled.Status.Actor)
}
if reconciled.Status.WorkerIp != "10.244.1.42" {
t.Errorf("expected worker IP '10.244.1.42', got %q", reconciled.Status.WorkerIp)
}
// Verify mock was called
if len(mockSrv.createdAtespaces) != 1 || mockSrv.createdAtespaces[0] != "default" {
t.Errorf("expected atespace 'default' created, got %v", mockSrv.createdAtespaces)
}
if len(mockSrv.createdActors) != 1 || mockSrv.createdActors[0] != "test-task" {
t.Errorf("expected actor 'test-task' created, got %v", mockSrv.createdActors)
}
if len(mockSrv.resumedActors) != 1 || mockSrv.resumedActors[0] != "test-task" {
t.Errorf("expected actor 'test-task' resumed, got %v", mockSrv.resumedActors)
}
}
// TestTaskReconciler_DefaultTaskImage verifies that a task without spec.image is
// defaulted to the image passed to NewTaskReconciler (or the API default when
// none is given), and that the image
// is what ends up in the per-task ActorTemplate.
func TestTaskReconciler_DefaultTaskImage(t *testing.T) {
for _, tc := range []struct {
name string
override string
want string
}{
{name: "api default", want: v1alpha1.DefaultTaskImage},
{name: "server override", override: "localhost:5001/ax-task-runner:e2e", want: "localhost:5001/ax-task-runner:e2e"},
} {
t.Run(tc.name, func(t *testing.T) {
lis, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatalf("failed to listen: %v", err)
}
defer lis.Close()
mockSrv := &mockControlServer{}
grpcServer := grpc.NewServer()
ateapipb.RegisterControlServer(grpcServer, mockSrv)
go grpcServer.Serve(lis)
defer grpcServer.Stop()
client, err := substrate.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials()))
if err != nil {
t.Fatalf("failed to create substrate client: %v", err)
}
defer client.Close()
reconciler := controller.NewTaskReconciler(client, "test-template", v1alpha1.DefaultSystemNamespace, tc.override)
reconciler.SecretResolver = noSecrets
reconciler.WorkspaceReadyTimeout = 200 * time.Millisecond
task := &v1alpha1.Task{
ApiVersion: v1alpha1.APIVersion,
Kind: v1alpha1.KindTask,
Metadata: &v1alpha1.ObjectMeta{Name: "no-image", Atespace: "default"},
Spec: &v1alpha1.TaskSpec{},
Status: &v1alpha1.TaskStatus{Phase: "Running"},
}
reconciled, err := reconciler.Reconcile(context.Background(), task, nil)
if err != nil {
t.Fatalf("Reconcile failed: %v", err)
}
if got := reconciled.GetSpec().GetImage(); got != tc.want {
t.Errorf("spec.image = %q, want %q", got, tc.want)
}
if len(mockSrv.templateImages) != 1 || mockSrv.templateImages[0] != tc.want {
t.Errorf("ActorTemplate images = %v, want [%q]", mockSrv.templateImages, tc.want)
}
})
}
}
func TestTaskReconciler_Suspend(t *testing.T) {
ctx := context.Background()
lis, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatalf("failed to listen: %v", err)
}
defer lis.Close()
mockSrv := &mockControlServer{}
grpcServer := grpc.NewServer()
ateapipb.RegisterControlServer(grpcServer, mockSrv)
go grpcServer.Serve(lis)
defer grpcServer.Stop()
client, err := substrate.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials()))
if err != nil {
t.Fatalf("failed to create substrate client: %v", err)
}
defer client.Close()
reconciler := controller.NewTaskReconciler(client, "test-template", v1alpha1.DefaultSystemNamespace, "")
reconciler.SecretResolver = noSecrets
reconciler.WorkspaceReadyTimeout = 200 * time.Millisecond
task := &v1alpha1.Task{
ApiVersion: v1alpha1.APIVersion,
Kind: v1alpha1.KindTask,
Metadata: &v1alpha1.ObjectMeta{
Name: "suspend-task",
Atespace: "default",
},
Spec: &v1alpha1.TaskSpec{
Image: "ghrc.io/my-org/my-image",
},
}
reconciled, err := reconciler.Reconcile(ctx, task, nil)
if err != nil {
t.Fatalf("Reconcile failed: %v", err)
}
if reconciled.Status.Phase != "Suspended" {
t.Errorf("expected phase 'Suspended', got %q", reconciled.Status.Phase)
}
if reconciled.Status.WorkerIp != "" {
t.Errorf("expected empty worker IP, got %q", reconciled.Status.WorkerIp)
}
if len(mockSrv.suspendedActors) != 1 || mockSrv.suspendedActors[0] != "suspend-task" {
t.Errorf("expected actor 'suspend-task' suspended, got %v", mockSrv.suspendedActors)
}
}
func TestTaskReconciler_SuspendError(t *testing.T) {
ctx := context.Background()
lis, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatalf("failed to listen: %v", err)
}
defer lis.Close()
mockSrv := &mockControlServer{
suspendErr: status.Error(codes.Unavailable, "substrate unavailable"),
}
grpcServer := grpc.NewServer()
ateapipb.RegisterControlServer(grpcServer, mockSrv)
go grpcServer.Serve(lis)
defer grpcServer.Stop()
client, err := substrate.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials()))
if err != nil {
t.Fatalf("failed to create substrate client: %v", err)
}
defer client.Close()
reconciler := controller.NewTaskReconciler(client, "test-template", "ax-system", "")
reconciler.SecretResolver = noSecrets
workerIp := "10.1.1.1"
task := &v1alpha1.Task{
ApiVersion: v1alpha1.APIVersion,
Kind: v1alpha1.KindTask,
Metadata: &v1alpha1.ObjectMeta{
Name: "suspend-fail-task",
Atespace: "default",
},
Spec: &v1alpha1.TaskSpec{
Image: "ghrc.io/my-org/my-image",
},
Status: &v1alpha1.TaskStatus{
Phase: "Suspended",
WorkerIp: workerIp,
Conditions: []*v1alpha1.Condition{{Type: "Ready", Status: "True", Reason: "TaskRunning"}},
},
}
reconciled, err := reconciler.Reconcile(ctx, task, nil)
if err == nil {
t.Fatal("expected Reconcile to return an error when SuspendActor fails")
}
if reconciled.GetStatus().GetWorkerIp() != workerIp {
t.Errorf("expected worker IP %q to be preserved, got %q", workerIp, reconciled.GetStatus().GetWorkerIp())
}
assertCondition(t, reconciled, "Ready", "True", "TaskRunning")
}
// TestTaskReconciler_TemplateCreationFailed covers google/ax#366: when Substrate
// rejects the per-task ActorTemplate (for example because the image is referenced
// by tag rather than digest), the task must report that error directly instead of
// silently falling back to the default template.
func TestTaskReconciler_TemplateCreationFailed(t *testing.T) {
ctx := t.Context()
lis, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatalf("failed to listen: %v", err)
}
defer lis.Close()
const substrateMsg = `actor_template.containers[0].image: Invalid value: "localhost:5001/my-agent:v1": must be pinned by digest (changing the image invalidates snapshots)`
mockSrv := &mockControlServer{
createTemplateErr: status.Error(codes.InvalidArgument, substrateMsg),
}
grpcServer := grpc.NewServer()
ateapipb.RegisterControlServer(grpcServer, mockSrv)
go grpcServer.Serve(lis)
defer grpcServer.Stop()
client, err := substrate.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials()))
if err != nil {
t.Fatalf("failed to create substrate client: %v", err)
}
defer client.Close()
reconciler := controller.NewTaskReconciler(client, "test-template", "ax-system", "")
reconciler.SecretResolver = noSecrets
reconciler.WorkspaceReadyTimeout = 200 * time.Millisecond
task := &v1alpha1.Task{
ApiVersion: v1alpha1.APIVersion,
Kind: v1alpha1.KindTask,
Metadata: &v1alpha1.ObjectMeta{
Name: "tagged-image",
Atespace: "default",
},
Spec: &v1alpha1.TaskSpec{
Image: "localhost:5001/my-agent:v1",
Command: []string{"python3", "/app/agent.py"},
},
Status: &v1alpha1.TaskStatus{Phase: "Running"},
}
reconciled, err := reconciler.Reconcile(ctx, task, nil)
if err == nil {
t.Fatal("expected Reconcile to fail when the ActorTemplate cannot be created")
}
if !strings.Contains(err.Error(), "must be pinned by digest") {
t.Errorf("expected error to carry the Substrate message, got %v", err)
}
if reconciled.Status.Phase != "Failed" {
t.Errorf("expected phase 'Failed', got %q", reconciled.Status.Phase)
}
assertCondition(t, reconciled, "Ready", "False", "TemplateCreationFailed")
for _, c := range reconciled.Status.Conditions {
if c.Type != "Ready" {
continue
}
// The user-facing message should be exactly what Substrate said, without the
// "rpc error: code = ... desc = " wrapper or our internal context prefix.
if c.Message != substrateMsg {
t.Errorf("expected Ready message to be the bare Substrate error\n got: %q\nwant: %q", c.Message, substrateMsg)
}
}
// The reconcile must stop before touching the actor: no fallback to the default template.
if len(mockSrv.createdActors) != 0 {
t.Errorf("expected no actor to be created, got %v", mockSrv.createdActors)
}
if len(mockSrv.resumedActors) != 0 {
t.Errorf("expected no actor to be resumed, got %v", mockSrv.resumedActors)
}
}
// TestTaskReconciler_TemplateCreationTransient checks that a transient Substrate
// error during ActorTemplate creation (for example Unavailable after retries are
// exhausted) leaves the task Pending for a later retry rather than permanently Failed.
func TestTaskReconciler_TemplateCreationTransient(t *testing.T) {
ctx := t.Context()
lis, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatalf("failed to listen: %v", err)
}
defer lis.Close()
// DeadlineExceeded is transient per substrate.IsTransientError but is not in the
// client's retry list, so the test does not wait out the Unavailable backoff.
mockSrv := &mockControlServer{
createTemplateErr: status.Error(codes.DeadlineExceeded, "substrate control plane timed out"),
}
grpcServer := grpc.NewServer()
ateapipb.RegisterControlServer(grpcServer, mockSrv)
go grpcServer.Serve(lis)
defer grpcServer.Stop()
client, err := substrate.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials()))
if err != nil {
t.Fatalf("failed to create substrate client: %v", err)
}
defer client.Close()
reconciler := controller.NewTaskReconciler(client, "test-template", "ax-system", "")
reconciler.SecretResolver = noSecrets
reconciler.WorkspaceReadyTimeout = 200 * time.Millisecond
task := &v1alpha1.Task{
ApiVersion: v1alpha1.APIVersion,
Kind: v1alpha1.KindTask,
Metadata: &v1alpha1.ObjectMeta{
Name: "template-transient",
Atespace: "default",
},
Spec: &v1alpha1.TaskSpec{Image: "ghcr.io/my-org/my-agent@sha256:0000000000000000000000000000000000000000000000000000000000000000"},
Status: &v1alpha1.TaskStatus{Phase: "Running"},
}
reconciled, err := reconciler.Reconcile(ctx, task, nil)
if err == nil {
t.Fatal("expected Reconcile to return an error on transient template failure")
}
if reconciled.Status.Phase != "Pending" {
t.Errorf("expected phase 'Pending', got %q", reconciled.Status.Phase)
}
assertCondition(t, reconciled, "Ready", "False", "SubstrateUnavailable")
if len(mockSrv.createdActors) != 0 {
t.Errorf("expected no actor to be created, got %v", mockSrv.createdActors)
}
}
func TestTaskReconciler_WorkspaceReady(t *testing.T) {
ctx := context.Background()
// 1. Mock Substrate Control Server
lis, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatalf("failed to listen: %v", err)
}
defer lis.Close()
// 2. Mock Worker readyz HTTP server
httpLis, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatalf("failed to listen http: %v", err)
}
defer httpLis.Close()
httpMux := http.NewServeMux()
httpMux.HandleFunc("/readyz", func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte("ok\n"))
})
httpServer := &http.Server{Handler: httpMux}
go httpServer.Serve(httpLis)
defer httpServer.Close()
workerHost, workerPortStr, _ := net.SplitHostPort(httpLis.Addr().String())
// In our mock, the worker IP returned by ResumeActor will have our mock ready server listening.
// But our reconciler connects to port 9999 by default: fmt.Sprintf("http://%s:9999/readyz", workerIP).
// If workerIP includes a port or is a host, let's verify how it handles it.
_ = workerHost
_ = workerPortStr
mockSrv := &mockControlServer{}
grpcServer := grpc.NewServer()
ateapipb.RegisterControlServer(grpcServer, mockSrv)
go grpcServer.Serve(lis)
defer grpcServer.Stop()
client, err := substrate.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials()))
if err != nil {
t.Fatalf("failed to create substrate client: %v", err)
}
defer client.Close()
reconciler := controller.NewTaskReconciler(client, "test-template", v1alpha1.DefaultSystemNamespace, "")
reconciler.SecretResolver = noSecrets
reconciler.WorkspaceReadyTimeout = 200 * time.Millisecond
task := &v1alpha1.Task{
ApiVersion: v1alpha1.APIVersion,
Kind: v1alpha1.KindTask,
Metadata: &v1alpha1.ObjectMeta{
Name: "ready-task",
Atespace: "default",
},
Spec: &v1alpha1.TaskSpec{},
Status: &v1alpha1.TaskStatus{
Phase: "Running",
},
}
// Case 1: Worker not responding on readyz -> WorkspaceReady=False and Ready=False.
reconciled, err := reconciler.Reconcile(ctx, task, nil)
if err != nil {
t.Fatalf("Reconcile failed: %v", err)
}
assertCondition(t, reconciled, "WorkspaceReady", "False", "Initializing")
assertCondition(t, reconciled, "Ready", "False", "WorkspaceInitializing")
// Case 2: Worker readyz endpoint succeeds -> WorkspaceReady=True and Ready=True.
mockSrv.workerIP = httpLis.Addr().String()
reconciledReady, err := reconciler.Reconcile(ctx, task, nil)
if err != nil {
t.Fatalf("Reconcile with ready worker failed: %v", err)
}
assertCondition(t, reconciledReady, "WorkspaceReady", "True", "SetupComplete")
assertCondition(t, reconciledReady, "Ready", "True", "TaskRunning")
// Case 3: Suspending the task -> Ready=False (TaskSuspended), but the workspace was
// already initialized so WorkspaceReady stays True.
task = reconciledReady
task.Status.Phase = "Suspended"
reconciledSuspended, err := reconciler.Reconcile(ctx, task, nil)
if err != nil {
t.Fatalf("Reconcile with suspend failed: %v", err)
}
if reconciledSuspended.Status.Phase != "Suspended" {
t.Errorf("expected phase Suspended, got %s", reconciledSuspended.Status.Phase)
}
assertCondition(t, reconciledSuspended, "Ready", "False", "TaskSuspended")
assertCondition(t, reconciledSuspended, "WorkspaceReady", "True", "SetupComplete")
// Case 4: Resuming with the worker unreachable -> the reconciler trusts the recorded
// WorkspaceReady instead of re-polling, so the task is Ready again immediately.
mockSrv.workerIP = "127.0.0.1:1"
task = reconciledSuspended
task.Status.Phase = "Running"
reconciledResumed, err := reconciler.Reconcile(ctx, task, nil)
if err != nil {
t.Fatalf("Reconcile with resume failed: %v", err)
}
assertCondition(t, reconciledResumed, "WorkspaceReady", "True", "SetupComplete")
assertCondition(t, reconciledResumed, "Ready", "True", "TaskRunning")
if got := len(mockSrv.actorTemplates); got != 1 {
t.Fatalf("readiness updates and suspend/resume created %d templates, want 1", got)
}
// A launch configuration change must still get a distinct template.
task.Spec.Command = []string{"python3", "agent.py"}
if _, err := reconciler.Reconcile(ctx, task, nil); err != nil {
t.Fatalf("Reconcile with changed command failed: %v", err)
}
if got := len(mockSrv.actorTemplates); got != 2 {
t.Errorf("command change left %d templates, want 2", got)
}
}
func TestTaskReconciler_ResourceLimits(t *testing.T) {
ctx := context.Background()
lis, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatalf("failed to listen: %v", err)
}
defer lis.Close()
mockSrv := &mockControlServer{}
grpcServer := grpc.NewServer()
ateapipb.RegisterControlServer(grpcServer, mockSrv)
go grpcServer.Serve(lis)
defer grpcServer.Stop()
client, err := substrate.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials()))
if err != nil {
t.Fatalf("failed to create substrate client: %v", err)
}
defer client.Close()
reconciler := controller.NewTaskReconciler(client, "test-template", "ax-system", "")
reconciler.SecretResolver = noSecrets
reconciler.WorkspaceReadyTimeout = 200 * time.Millisecond
task := &v1alpha1.Task{
ApiVersion: v1alpha1.APIVersion,
Kind: v1alpha1.KindTask,
Metadata: &v1alpha1.ObjectMeta{
Name: "sized-task",
Atespace: "default",
},
Spec: &v1alpha1.TaskSpec{
Image: "ghrc.io/my-org/my-image",
Resources: &v1alpha1.ResourcesSpec{
Limits: &v1alpha1.ResourceList{Cpu: "2", Memory: "4Gi"},
},
},
}
if _, err := reconciler.Reconcile(ctx, task, nil); err != nil {
t.Fatalf("Reconcile failed: %v", err)
}
if len(mockSrv.createdTemplates) != 1 {
t.Fatalf("created %d templates, want 1", len(mockSrv.createdTemplates))
}
want := &ateapipb.Resources{Limits: []*ateapipb.Limits{
{Name: "cpu", Quantity: "2"},
{Name: "memory", Quantity: "4Gi"},
}}
if got := mockSrv.createdTemplates[0].GetResources(); !proto.Equal(got, want) {
t.Errorf("template resources = %v, want %v", got, want)
}
// Raising a limit is a launch configuration change and must provision a new
// template, under a new name, carrying the new value.
task.Spec.Resources.Limits.Memory = "8Gi"
if _, err := reconciler.Reconcile(ctx, task, nil); err != nil {
t.Fatalf("Reconcile with changed limits failed: %v", err)
}
if len(mockSrv.createdTemplates) != 2 {
t.Fatalf("limits change left %d templates, want 2", len(mockSrv.createdTemplates))
}
first, second := mockSrv.createdTemplates[0].GetMetadata().GetName(), mockSrv.createdTemplates[1].GetMetadata().GetName()
if first == second {
t.Errorf("limits change reused template name %q, want a distinct name", first)
}
want.Limits[1].Quantity = "8Gi"
if got := mockSrv.createdTemplates[1].GetResources(); !proto.Equal(got, want) {
t.Errorf("template resources after change = %v, want %v", got, want)
}
// A task without limits inherits the worker defaults: no resources block.
plain := &v1alpha1.Task{
ApiVersion: v1alpha1.APIVersion,
Kind: v1alpha1.KindTask,
Metadata: &v1alpha1.ObjectMeta{Name: "plain-task", Atespace: "default"},
Spec: &v1alpha1.TaskSpec{Image: "ghrc.io/my-org/my-image"},
}
if _, err := reconciler.Reconcile(ctx, plain, nil); err != nil {
t.Fatalf("Reconcile of task without limits failed: %v", err)
}
if got := mockSrv.createdTemplates[len(mockSrv.createdTemplates)-1].GetResources(); got != nil {
t.Errorf("template for task without limits has resources %v, want none", got)
}
}
// A task that asks for limits must never run without them: invalid limits and a
// Substrate rejection of the template both fail the reconcile instead of falling
// back to the default template. Without limits the fallback contract is unchanged.
func TestTaskReconciler_ResourceLimitsFailurePaths(t *testing.T) {
ctx := context.Background()
newTask := func(name string, resources *v1alpha1.ResourcesSpec) *v1alpha1.Task {
return &v1alpha1.Task{
ApiVersion: v1alpha1.APIVersion,
Kind: v1alpha1.KindTask,
Metadata: &v1alpha1.ObjectMeta{Name: name, Atespace: "default"},
Spec: &v1alpha1.TaskSpec{Image: "ghrc.io/my-org/my-image", Resources: resources},
}
}
setup := func(t *testing.T, mockSrv *mockControlServer) *controller.TaskReconciler {
lis, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatalf("failed to listen: %v", err)
}
grpcServer := grpc.NewServer()
ateapipb.RegisterControlServer(grpcServer, mockSrv)
go grpcServer.Serve(lis)
client, err := substrate.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials()))
if err != nil {
t.Fatalf("failed to create substrate client: %v", err)
}
t.Cleanup(func() {
client.Close()
grpcServer.Stop()
lis.Close()
})
reconciler := controller.NewTaskReconciler(client, "test-template", "ax-system", "")
reconciler.SecretResolver = noSecrets
reconciler.WorkspaceReadyTimeout = 200 * time.Millisecond
return reconciler
}
t.Run("invalid limits fail before provisioning", func(t *testing.T) {
mockSrv := &mockControlServer{}
reconciler := setup(t, mockSrv)
for _, reqs := range []*v1alpha1.ResourcesSpec{
{Limits: &v1alpha1.ResourceList{Cpu: "two"}},
{Limits: &v1alpha1.ResourceList{Cpu: "0"}},
{Limits: &v1alpha1.ResourceList{Cpu: "1000"}},
{Limits: &v1alpha1.ResourceList{Memory: "-4Gi"}},
{Requests: &v1alpha1.ResourceList{Cpu: "500m"}, Limits: &v1alpha1.ResourceList{Cpu: "2"}},
} {
reconciled, err := reconciler.Reconcile(ctx, newTask("bad-limits", reqs), nil)
if err == nil {
t.Errorf("Reconcile(%v) succeeded, want error", reqs)
}
if reconciled.Status.Phase != "Failed" {
t.Errorf("Reconcile(%v) phase = %q, want Failed", reqs, reconciled.Status.Phase)
}
assertCondition(t, reconciled, "Ready", "False", "InvalidResources")
}
if len(mockSrv.createdTemplates) != 0 || len(mockSrv.createdActors) != 0 || len(mockSrv.resumedActors) != 0 {
t.Errorf("invalid limits reached Substrate: templates=%d actors=%v resumed=%v",
len(mockSrv.createdTemplates), mockSrv.createdActors, mockSrv.resumedActors)
}
})
t.Run("rejected template with limits does not fall back", func(t *testing.T) {
mockSrv := &mockControlServer{
createTemplateErr: status.Error(codes.InvalidArgument, "actor_template.resources.limits[0].quantity: Invalid value"),
}
reconciler := setup(t, mockSrv)
reconciled, err := reconciler.Reconcile(ctx, newTask("rejected-limits", &v1alpha1.ResourcesSpec{
Limits: &v1alpha1.ResourceList{Cpu: "2", Memory: "4Gi"},
}), nil)
if err == nil {
t.Fatal("Reconcile succeeded although the template with limits was rejected")
}
if reconciled.Status.Phase != "Failed" {
t.Errorf("phase = %q, want Failed", reconciled.Status.Phase)
}
assertCondition(t, reconciled, "Ready", "False", "TemplateCreationFailed")
if len(mockSrv.createdActors) != 0 || len(mockSrv.resumedActors) != 0 {
t.Errorf("task ran on a fallback template without its limits: actors=%v resumed=%v", mockSrv.createdActors, mockSrv.resumedActors)
}
})
t.Run("rejected template without limits does not fall back either", func(t *testing.T) {
mockSrv := &mockControlServer{
createTemplateErr: status.Error(codes.InvalidArgument, "image must be pinned by digest"),
}
reconciler := setup(t, mockSrv)
reconciled, err := reconciler.Reconcile(ctx, newTask("no-limits", nil), nil)
if err == nil {
t.Fatal("Reconcile succeeded although the template was rejected")
}
// Falling back would run a different image than the task asked for, so the
// template error is terminal regardless of whether limits are set (#366).
if reconciled.Status.Phase != "Failed" {
t.Errorf("phase = %q, want Failed", reconciled.Status.Phase)
}
assertCondition(t, reconciled, "Ready", "False", "TemplateCreationFailed")
if len(mockSrv.createdActors) != 0 {
t.Errorf("task ran on a fallback template: actors=%v", mockSrv.createdActors)
}
})
}
// assertCondition fails the test unless the task has a condition of the given type with
// the expected status and reason.
func assertCondition(t *testing.T, task *v1alpha1.Task, condType, wantStatus, wantReason string) {
t.Helper()
for _, c := range task.Status.Conditions {
if c.Type != condType {
continue
}
if c.Status != wantStatus || c.Reason != wantReason {
t.Errorf("expected %s=%s (%s), got Status=%s Reason=%s", condType, wantStatus, wantReason, c.Status, c.Reason)
}
return
}
t.Errorf("expected %s condition to be set", condType)
}
func TestReconcileDelete_RemovesActorAndTemplates(t *testing.T) {
ctx := context.Background()
lis, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatalf("failed to listen: %v", err)
}
defer lis.Close()
mockSrv := &mockControlServer{actorTemplates: map[string]bool{
"job-tmpl-0a1b2c3d": true, // current revision of task "job"
"job-tmpl-deadbeef": true, // stale revision of task "job"
"job-tmpl-deadbeef-tmpl-01234567": true, // belongs to a task literally named "job-tmpl-deadbeef"
"jobs-tmpl-0a1b2c3d": true, // belongs to task "jobs"
"default-template": true,
}}
grpcServer := grpc.NewServer()
ateapipb.RegisterControlServer(grpcServer, mockSrv)
go grpcServer.Serve(lis)
defer grpcServer.Stop()
client, err := substrate.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials()))
if err != nil {
t.Fatalf("failed to create substrate client: %v", err)
}
defer client.Close()
reconciler := controller.NewTaskReconciler(client, "test-template", v1alpha1.DefaultSystemNamespace, "")
reconciler.SecretResolver = noSecrets
reconciler.WorkspaceReadyTimeout = 200 * time.Millisecond
if err := reconciler.ReconcileDelete(ctx, "default", "job"); err != nil {
t.Fatalf("ReconcileDelete failed: %v", err)
}
if len(mockSrv.deletedActors) != 1 || mockSrv.deletedActors[0] != "job" {
t.Errorf("expected actor 'job' to be deleted, got %v", mockSrv.deletedActors)
}
wantDeleted := map[string]bool{"job-tmpl-0a1b2c3d": true, "job-tmpl-deadbeef": true}
if len(mockSrv.deletedTemplates) != len(wantDeleted) {
t.Errorf("expected %d templates deleted, got %v", len(wantDeleted), mockSrv.deletedTemplates)
}
for _, name := range mockSrv.deletedTemplates {
if !wantDeleted[name] {
t.Errorf("unexpected template deleted: %s", name)
}
}
for _, keep := range []string{"job-tmpl-deadbeef-tmpl-01234567", "jobs-tmpl-0a1b2c3d", "default-template"} {
if !mockSrv.actorTemplates[keep] {
t.Errorf("template %s should not have been deleted", keep)
}
}
}
func TestReconcileDelete_BlocksUntilActorDeleted(t *testing.T) {
ctx := context.Background()
lis, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatalf("failed to listen: %v", err)
}
defer lis.Close()
getCalls := 0
mockSrv := &mockControlServer{
getActorFunc: func(ctx context.Context, req *ateapipb.GetActorRequest) (*ateapipb.Actor, error) {
getCalls++
if getCalls < 3 {
// Simulate actor in deleting state during the first two poll checks
return &ateapipb.Actor{
Metadata: &ateapipb.ResourceMetadata{Name: req.GetActor().GetName()},
Status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_DELETING},
}, nil
}
// Once actor is completely torn down
return nil, status.Errorf(codes.NotFound, "actor %q not found", req.GetActor().GetName())
},
}
grpcServer := grpc.NewServer()
ateapipb.RegisterControlServer(grpcServer, mockSrv)
go grpcServer.Serve(lis)
defer grpcServer.Stop()
client, err := substrate.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials()))
if err != nil {
t.Fatalf("failed to create substrate client: %v", err)
}
defer client.Close()
reconciler := controller.NewTaskReconciler(client, "test-template", v1alpha1.DefaultSystemNamespace, "")
reconciler.SecretResolver = noSecrets
if err := reconciler.ReconcileDelete(ctx, "default", "slow-delete-task"); err != nil {
t.Fatalf("ReconcileDelete failed: %v", err)
}
if getCalls < 3 {
t.Errorf("expected at least 3 GetActor calls before deletion completes, got %d", getCalls)
}
}
func TestReconcileDelete_ActorDeletionTimeout(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 250*time.Millisecond)
defer cancel()
lis, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatalf("failed to listen: %v", err)
}
defer lis.Close()
mockSrv := &mockControlServer{
getActorFunc: func(ctx context.Context, req *ateapipb.GetActorRequest) (*ateapipb.Actor, error) {
// Actor remains in deleting state indefinitely
return &ateapipb.Actor{
Metadata: &ateapipb.ResourceMetadata{Name: req.GetActor().GetName()},
Status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_DELETING},
}, nil
},
}
grpcServer := grpc.NewServer()
ateapipb.RegisterControlServer(grpcServer, mockSrv)
go grpcServer.Serve(lis)
defer grpcServer.Stop()
client, err := substrate.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials()))
if err != nil {
t.Fatalf("failed to create substrate client: %v", err)
}
defer client.Close()
reconciler := controller.NewTaskReconciler(client, "test-template", v1alpha1.DefaultSystemNamespace, "")
reconciler.SecretResolver = noSecrets
err = reconciler.ReconcileDelete(ctx, "default", "stuck-task")
if err == nil {
t.Fatal("expected error due to timeout, got nil")
}
if !errors.Is(err, context.DeadlineExceeded) && !strings.Contains(err.Error(), "context deadline exceeded") {
t.Errorf("expected deadline exceeded error, got %v", err)
}
}
func TestEnsureActor_RevertsCrashedActor(t *testing.T) {
lis, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatalf("failed to listen: %v", err)
}
defer lis.Close()
mockSrv := &mockControlServer{crashedActor: "job"}
grpcServer := grpc.NewServer()
ateapipb.RegisterControlServer(grpcServer, mockSrv)
go grpcServer.Serve(lis)
defer grpcServer.Stop()
client, err := substrate.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials()))
if err != nil {
t.Fatalf("failed to create substrate client: %v", err)
}
defer client.Close()
actor, err := client.EnsureActor(context.Background(), "default", "job", "default", "job-tmpl-0a1b2c3d")
if err != nil {
t.Fatalf("EnsureActor failed: %v", err)
}
if got := actor.GetStatus().GetState(); got != ateapipb.ActorState_ACTOR_STATE_SUSPENDED {
t.Errorf("actor state = %v, want SUSPENDED", got)
}
if len(mockSrv.revertedActors) != 1 || len(mockSrv.deletedActors) != 0 {
t.Errorf("crashed actor: reverted %v, deleted %v; want reverted only", mockSrv.revertedActors, mockSrv.deletedActors)
}
}
func TestTaskReconciler_TransientSubstrateUnavailable(t *testing.T) {
ctx := context.Background()
lis, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatalf("failed to listen: %v", err)
}
defer lis.Close()
mockSrv := &mockControlServer{}
grpcServer := grpc.NewServer()
ateapipb.RegisterControlServer(grpcServer, mockSrv)
go grpcServer.Serve(lis)
defer grpcServer.Stop()
client, err := substrate.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials()))
if err != nil {
t.Fatalf("failed to create substrate client: %v", err)
}
defer client.Close()
reconciler := controller.NewTaskReconciler(client, "test-template", v1alpha1.DefaultSystemNamespace, "")
reconciler.SecretResolver = noSecrets
reconciler.WorkspaceReadyTimeout = 50 * time.Millisecond
// 1. Transient error during EnsureAtespace leaves Task in Phase="Pending" with Reason="SubstrateUnavailable".
mockSrv.createAtespaceErr = status.Error(codes.DeadlineExceeded, "substrate control plane timed out")
task := &v1alpha1.Task{
Metadata: &v1alpha1.ObjectMeta{Name: "transient-atespace"},
Spec: &v1alpha1.TaskSpec{Image: "ghrc.io/my-org/my-image"},
}
reconciled, err := reconciler.Reconcile(ctx, task, nil)
if err == nil {
t.Fatal("expected error from Reconcile when EnsureAtespace fails, got nil")
}
if reconciled.GetStatus().GetPhase() != "Pending" {
t.Errorf("expected Phase='Pending' on transient EnsureAtespace error, got %q", reconciled.GetStatus().GetPhase())
}
assertCondition(t, reconciled, "Ready", "False", "SubstrateUnavailable")
mockSrv.createAtespaceErr = nil
// 2. Transient error during EnsureActor leaves Task in Phase="Pending" with Reason="SubstrateUnavailable".
mockSrv.createActorErr = status.Error(codes.DeadlineExceeded, "substrate actor creation timed out")
task2 := &v1alpha1.Task{
Metadata: &v1alpha1.ObjectMeta{Name: "transient-actor"},
Spec: &v1alpha1.TaskSpec{Image: "ghrc.io/my-org/my-image"},
}
reconciled2, err := reconciler.Reconcile(ctx, task2, nil)
if err == nil {
t.Fatal("expected error from Reconcile when EnsureActor fails, got nil")
}
if reconciled2.GetStatus().GetPhase() != "Pending" {
t.Errorf("expected Phase='Pending' on transient EnsureActor error, got %q", reconciled2.GetStatus().GetPhase())
}
assertCondition(t, reconciled2, "Ready", "False", "SubstrateUnavailable")
mockSrv.createActorErr = nil
// 3. Transient error during ResumeActor leaves Task in Phase="Pending" with Reason="SubstrateUnavailable",
// and a subsequent Reconcile once Substrate recovers transitions the Task to "Running".
mockSrv.resumeActorErr = status.Error(codes.DeadlineExceeded, "substrate worker pool temporarily unreachable")
task3 := &v1alpha1.Task{
Metadata: &v1alpha1.ObjectMeta{Name: "transient-resume"},
Spec: &v1alpha1.TaskSpec{Image: "ghrc.io/my-org/my-image"},
Status: &v1alpha1.TaskStatus{Phase: "Running"},
}
reconciled3, err := reconciler.Reconcile(ctx, task3, nil)
if err == nil {
t.Fatal("expected error from Reconcile when ResumeActor fails, got nil")
}
if reconciled3.GetStatus().GetPhase() != "Pending" {
t.Errorf("expected Phase='Pending' on transient ResumeActor error, got %q", reconciled3.GetStatus().GetPhase())
}
assertCondition(t, reconciled3, "Ready", "False", "SubstrateUnavailable")
// Recovery on retry:
mockSrv.resumeActorErr = nil
reconciled3.Status.Phase = "Running"
recovered, err := reconciler.Reconcile(ctx, reconciled3, nil)
if err != nil {
t.Fatalf("expected Reconcile to succeed after Substrate recovers, got %v", err)
}
if recovered.GetStatus().GetPhase() != "Running" {
t.Errorf("expected Phase='Running' after recovery, got %q", recovered.GetStatus().GetPhase())
}
// 4. Permanent (non-transient) error still marks Task as Phase="Failed".
mockSrv.createActorErr = status.Error(codes.InvalidArgument, "invalid actor specification")
task4 := &v1alpha1.Task{
Metadata: &v1alpha1.ObjectMeta{Name: "permanent-actor-err"},
Spec: &v1alpha1.TaskSpec{Image: "ghrc.io/my-org/my-image"},
}
reconciled4, err := reconciler.Reconcile(ctx, task4, nil)
if err == nil {
t.Fatal("expected error on InvalidArgument, got nil")
}
if reconciled4.GetStatus().GetPhase() != "Failed" {
t.Errorf("expected Phase='Failed' on permanent error, got %q", reconciled4.GetStatus().GetPhase())
}
mockSrv.createActorErr = nil
}