Agent Executor (AX) is Google's open agentic orchestration runtime
Go
fix(substrate): adapt to Substrate v0.4 API and harden actor RPCs (#457)
Siyuan Zhang2e06707c0024
diff --git a/cmd/ax-server/main.go b/cmd/ax-server/main.goindex 4f224fd..0b5cc4b 100644--- a/cmd/ax-server/main.go+++ b/cmd/ax-server/main.go@@ -29,6 +29,7 @@ import ("github.com/google/ax/internal/server""github.com/google/ax/internal/store/redis""github.com/google/ax/internal/substrate"+ "github.com/google/ax/pkg/apis/v1alpha1"goredis "github.com/redis/go-redis/v9")@@ -57,7 +58,7 @@ func main() {flag.BoolVar(&substrateInsecureTLS, "substrate-insecure-tls", false, "Skip Substrate TLS verification")flag.BoolVar(&substratePlaintext, "substrate-plaintext", false, "Use insecure plaintext gRPC connection to Substrate")flag.StringVar(&defaultTemplate, "template", "default-template", "Default Substrate ActorTemplate name")- flag.StringVar(&defaultTemplateAtespace, "template-atespace", "ax-system", "Default Substrate ActorTemplate atespace")+ flag.StringVar(&defaultTemplateAtespace, "template-atespace", v1alpha1.DefaultSystemNamespace, "Default Substrate ActorTemplate atespace")flag.Parse()if envAddr := os.Getenv("ADDR"); envAddr != "" {diff --git a/cmd/ax/main.go b/cmd/ax/main.goindex a7bfb3a..db06a87 100644--- a/cmd/ax/main.go+++ b/cmd/ax/main.go@@ -51,7 +51,7 @@ func main() {atespace = "default"explicitServer = ""kubeContext = ""- axNamespace = "ax-system"+ axNamespace = v1alpha1.DefaultSystemNamespace)args := os.Args[1:]diff --git a/go.mod b/go.modindex 89b69cf..eb48ce6 100644--- a/go.mod+++ b/go.mod@@ -4,7 +4,8 @@ go 1.27.1require (github.com/agent-substrate/env v0.0.11-0.20260912052224-4468a200b170- github.com/agent-substrate/substrate v0.0.0-20260918201817-944abe3278b8+ github.com/agent-substrate/substrate v0.4.1-0.20261009152331-f6a1dc72a35d+ github.com/cenkalti/backoff/v5 v5.0.3github.com/redis/go-redis/v9 v9.22.0google.golang.org/grpc v1.83.2google.golang.org/protobuf v1.36.12@@ -19,6 +20,6 @@ require (golang.org/x/net v0.58.0 // indirectgolang.org/x/sys v0.47.0 // indirectgolang.org/x/text v0.41.0 // indirect- google.golang.org/genproto/googleapis/rpc v0.0.0-20260720211330-0afa2a65878a // indirect+ google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d // indirectgopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect)diff --git a/go.sum b/go.sumindex 27abe18..efbaf8d 100644--- a/go.sum+++ b/go.sum@@ -1,11 +1,13 @@github.com/agent-substrate/env v0.0.11-0.20260912052224-4468a200b170 h1:8AxhEx3BL7J8c+zqnYq5S3aH0J/4z85/JYWoo1dr6xY=github.com/agent-substrate/env v0.0.11-0.20260912052224-4468a200b170/go.mod h1:RZ4ACmDqWQx7fgX9b1EaFJskeYTZh1Y7uZinDtp7U2o=-github.com/agent-substrate/substrate v0.0.0-20260918201817-944abe3278b8 h1:C39j2y88zDvg0VVrIfJqYejCjeHI25hugGubMeee74w=-github.com/agent-substrate/substrate v0.0.0-20260918201817-944abe3278b8/go.mod h1:pWnhZpFkHq9VjF2OxsqEbryLnnUYmohhC62bFoindCM=+github.com/agent-substrate/substrate v0.4.1-0.20261009152331-f6a1dc72a35d h1:LGw4Z7X0wXXDRLYRpo3j0GYGEnrqVL39K4NEdGdb578=+github.com/agent-substrate/substrate v0.4.1-0.20261009152331-f6a1dc72a35d/go.mod h1:IuJNQ+2si6Sxkg9b3+kfhaXQiaxSwHtsiQrpNFWbjZY=github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=+github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM=+github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=@@ -40,16 +42,16 @@ github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs=github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s=go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=-go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU=-go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc=-go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc=-go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo=-go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58=-go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0=-go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI=-go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA=-go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk=-go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE=+go.opentelemetry.io/otel v1.45.0 h1:pdrWmLHofpubmArBv1LgFSv1Z0Ie/ppdZzu+kUN5EeU=+go.opentelemetry.io/otel v1.45.0/go.mod h1:XZxIqPapzEYnhNSScF5DIqXhm/rYi0FzCe2XddAwZfQ=+go.opentelemetry.io/otel/metric v1.45.0 h1:7Eg1uH7CJ5cXv9is6tnBe1FI6rj1nwUdbFypRm3br/M=+go.opentelemetry.io/otel/metric v1.45.0/go.mod h1:HAPbm1nd3p1PmFH7v2dR+6BjXxw+Lq4a2+pndMAm08s=+go.opentelemetry.io/otel/sdk v1.45.0 h1:4VVSMgQ83dUgW2aoX5f6JgLvHwIvzcuLnF9lUdCSpCw=+go.opentelemetry.io/otel/sdk v1.45.0/go.mod h1:Sr40LgXV7DsKMMJMKOhUWOgMWTfAaqvm2kF0g7ilwuA=+go.opentelemetry.io/otel/sdk/metric v1.45.0 h1:oVFszMfyj1Am6s24Vtc7wBb8BKLcwepJjNEYILuiE3o=+go.opentelemetry.io/otel/sdk/metric v1.45.0/go.mod h1:vUWUxDZvu1WVRj8JA8S0AdhsPrZoDpA2DdZauIh4mDA=+go.opentelemetry.io/otel/trace v1.45.0 h1:l/mP6Uv7oNO7/TblbhpbgMidxhq1uO/rPsikOyVhxag=+go.opentelemetry.io/otel/trace v1.45.0/go.mod h1:qoJJA2xNMnxRrdISU/kLtfUH2wNeQbiv+jhs/CxI8bc=go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=@@ -62,8 +64,8 @@ golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8=golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M=gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=-google.golang.org/genproto/googleapis/rpc v0.0.0-20260720211330-0afa2a65878a h1:qI/YMH1ep2qQtqcp00gMQyoU7mjvbhg88GJKCvfoLj0=-google.golang.org/genproto/googleapis/rpc v0.0.0-20260720211330-0afa2a65878a/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=+google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d h1:IL4hdHzcUv2l/gcg98/Rj3FbtE6axwqslOW8SW0C+S0=+google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU=google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8=google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc=diff --git a/internal/controller/reconciler.go b/internal/controller/reconciler.goindex 55814d7..341ac5a 100644--- a/internal/controller/reconciler.go+++ b/internal/controller/reconciler.go@@ -79,7 +79,7 @@ func NewTaskReconciler(client *substrate.Client, defaultTemplate, defaultTemplatdefaultTemplate = "default-template"}if defaultTemplateAtespace == "" {- defaultTemplateAtespace = "ax-system"+ defaultTemplateAtespace = v1alpha1.DefaultSystemNamespace}return &TaskReconciler{client: client,@@ -222,7 +222,7 @@ func (r *TaskReconciler) Reconcile(ctx context.Context, task *v1alpha1.Task, worhost = hport = p}- readyURL := fmt.Sprintf("http://%s:%s/readyz?check=workspace", host, port)+ readyURL := fmt.Sprintf("http://%s/readyz?check=workspace", net.JoinHostPort(host, port))// Workspace setup happens once per task. After it has completed, WorkspaceReady stays// True across suspend/resume cycles, so only poll while it is still initializing.workspaceReady := r.conditionTrue(task, condWorkspaceReady)diff --git a/internal/controller/reconciler_test.go b/internal/controller/reconciler_test.goindex e91954e..f3e1d6b 100644--- a/internal/controller/reconciler_test.go+++ b/internal/controller/reconciler_test.go@@ -112,8 +112,8 @@ func (m *mockControlServer) ResumeActor(ctx context.Context, req *ateapipb.ResumStatus: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_RUNNING,WorkerAssignment: &ateapipb.WorkerAssignment{- WorkerPod: "worker-pod-1",- WorkerPodIp: wIP,+ WorkerPod: "worker-pod-1",+ WorkerPodIps: []string{wIP},},},},@@ -214,7 +214,7 @@ func TestTaskReconciler(t *testing.T) {defer client.Close()// 3. Reconcile Task- reconciler := controller.NewTaskReconciler(client, "test-template", "ax-system")+ reconciler := controller.NewTaskReconciler(client, "test-template", v1alpha1.DefaultSystemNamespace)reconciler.SecretResolver = noSecretsreconciler.WorkspaceReadyTimeout = 200 * time.Millisecond@@ -283,7 +283,7 @@ func TestTaskReconciler_Suspend(t *testing.T) {}defer client.Close()- reconciler := controller.NewTaskReconciler(client, "test-template", "ax-system")+ reconciler := controller.NewTaskReconciler(client, "test-template", v1alpha1.DefaultSystemNamespace)reconciler.SecretResolver = noSecretsreconciler.WorkspaceReadyTimeout = 200 * time.Millisecond@@ -360,7 +360,7 @@ func TestTaskReconciler_WorkspaceReady(t *testing.T) {}defer client.Close()- reconciler := controller.NewTaskReconciler(client, "test-template", "ax-system")+ reconciler := controller.NewTaskReconciler(client, "test-template", v1alpha1.DefaultSystemNamespace)reconciler.SecretResolver = noSecretsreconciler.WorkspaceReadyTimeout = 200 * time.Millisecond@@ -476,7 +476,7 @@ func TestReconcileDelete_RemovesActorAndTemplates(t *testing.T) {}defer client.Close()- reconciler := controller.NewTaskReconciler(client, "test-template", "ax-system")+ reconciler := controller.NewTaskReconciler(client, "test-template", v1alpha1.DefaultSystemNamespace)reconciler.SecretResolver = noSecretsreconciler.WorkspaceReadyTimeout = 200 * time.Millisecond@@ -539,7 +539,7 @@ func TestReconcileDelete_BlocksUntilActorDeleted(t *testing.T) {}defer client.Close()- reconciler := controller.NewTaskReconciler(client, "test-template", "ax-system")+ reconciler := controller.NewTaskReconciler(client, "test-template", v1alpha1.DefaultSystemNamespace)reconciler.SecretResolver = noSecretsif err := reconciler.ReconcileDelete(ctx, "default", "slow-delete-task"); err != nil {@@ -581,7 +581,7 @@ func TestReconcileDelete_ActorDeletionTimeout(t *testing.T) {}defer client.Close()- reconciler := controller.NewTaskReconciler(client, "test-template", "ax-system")+ reconciler := controller.NewTaskReconciler(client, "test-template", v1alpha1.DefaultSystemNamespace)reconciler.SecretResolver = noSecretserr = reconciler.ReconcileDelete(ctx, "default", "stuck-task")diff --git a/internal/substrate/client.go b/internal/substrate/client.goindex 7e65273..234016d 100644--- a/internal/substrate/client.go+++ b/internal/substrate/client.go@@ -18,13 +18,16 @@ import ("context""crypto/tls""crypto/x509"+ "errors""fmt""log/slog""os"+ "slices""strings""time""github.com/agent-substrate/substrate/pkg/proto/ateapipb"+ "github.com/cenkalti/backoff/v5""github.com/google/ax/pkg/apis/v1alpha1""google.golang.org/grpc""google.golang.org/grpc/codes"@@ -206,8 +209,60 @@ func (c *Client) GetActorTemplate(ctx context.Context, atespace, templateName stconst (DefaultGuestCommand = "/usr/local/bin/ax-task-runner"DefaultSnapshotsBucket = "gs://dberkov-gke-dev3/ate-env/"++ // defaultRetryAttempts is the maximum number of calls retryOnCodes makes. With+ // exponential backoff from defaultRetryInterval capped at maxRetryInterval+ // (±50% jitter), the total wait across all attempts is roughly 6s-20s.+ defaultRetryAttempts = 10++ // retryBackoffFactor is the multiplier applied to the backoff after each failed attempt.+ retryBackoffFactor = 2++ // actorDeletingWaitAttempts bounds how many times EnsureActor retries creation+ // while a previous actor with the same name is still being deleted.+ actorDeletingWaitAttempts = 20+)++var (+ // defaultRetryInterval is the initial backoff for retryOnCodes and the poll+ // interval used while waiting for actor deletion.+ defaultRetryInterval = 200 * time.Millisecond++ // maxRetryInterval caps the exponential backoff in retryOnCodes.+ maxRetryInterval = 2 * time.Second++ // actorDeletingWaitInterval is the interval between creation attempts while a+ // previous actor is still deleting (actorDeletingWaitAttempts * interval ~= 10s).+ actorDeletingWaitInterval = 500 * time.Millisecond)+// retryOnCodes calls fn until it succeeds, returns an error whose gRPC code is not in+// retryable, defaultRetryAttempts is exhausted, or ctx is done. Between attempts it+// waits with jittered exponential backoff starting at defaultRetryInterval and+// capped at maxRetryInterval.+func retryOnCodes(ctx context.Context, fn func() error, retryable ...codes.Code) error {+ b := backoff.NewExponentialBackOff()+ b.InitialInterval = defaultRetryInterval+ b.MaxInterval = maxRetryInterval+ b.Multiplier = retryBackoffFactor++ _, err := backoff.Retry(ctx, func() (struct{}, error) {+ err := fn()+ if err != nil && !slices.Contains(retryable, status.Code(err)) {+ return struct{}{}, backoff.Permanent(err)+ }+ return struct{}{}, err+ }, backoff.WithBackOff(b), backoff.WithMaxTries(defaultRetryAttempts))++ // backoff.Retry returns the *PermanentError wrapper as-is when the attempt limit+ // is hit on the same try, so unwrap it to always surface the original error.+ var permanent *backoff.PermanentError+ if errors.As(err, &permanent) {+ return permanent.Unwrap()+ }+ return err+}+// BuildActorTemplate constructs a Substrate ActorTemplate based on the standard ate-env specification.func BuildActorTemplate(atespace, name, image string, envMap map[string]string, command []string, snapshotsBucket string) *ateapipb.ActorTemplate {if atespace == "" {@@ -245,7 +300,7 @@ func BuildActorTemplate(atespace, name, image string, envMap map[string]string,Image: image,Command: command,Env: envList,- Readyz: &ateapipb.ContainerReadyz{+ WakeupProbe: &ateapipb.ContainerWakeupProbe{HttpGet: &ateapipb.HTTPGetAction{Path: "/readyz",Port: 80,@@ -260,13 +315,9 @@ func BuildActorTemplate(atespace, name, image string, envMap map[string]string,Name: "workspace",DurableDir: &ateapipb.DurableDirVolumeSource{},}},- SnapshotsConfig: &ateapipb.SnapshotsConfig{- StorageLocation: snapshotsBucket,- OnPause: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA,- OnCommit: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA,- OnResume: &ateapipb.OnResumeConfig{- FromData: ateapipb.ResumeSource_RESUME_SOURCE_GOLDEN,- },+ SnapshotConfig: &ateapipb.SnapshotConfig{+ StorageLocation: snapshotsBucket,+ PreferredFidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES,},SandboxConfig: &ateapipb.SandboxConfig{SandboxClass: ateapipb.SandboxClass_SANDBOX_CLASS_GVISOR,@@ -347,11 +398,11 @@ func (c *Client) EnsureActor(ctx context.Context, atespace, actorName, templateA}if state == ateapipb.ActorState_ACTOR_STATE_DELETING {// Wait briefly for previous actor deletion to finalize before recreating- for i := 0; i < 20; i++ {+ for i := 0; i < actorDeletingWaitAttempts; i++ {select {case <-ctx.Done():return nil, ctx.Err()- case <-time.After(500 * time.Millisecond):+ case <-time.After(actorDeletingWaitInterval):}actor, err = c.control.CreateActor(ctx, req)if err == nil {@@ -379,18 +430,23 @@ func (c *Client) ResumeActor(ctx context.Context, atespace, actorName string) (*Name: actorName,},}- resp, err := c.control.ResumeActor(ctx, req)+ var resp *ateapipb.ResumeActorResponse+ err := retryOnCodes(ctx, func() error {+ var err error+ resp, err = c.control.ResumeActor(ctx, req)+ return err+ }, codes.Aborted)if err != nil {return nil, "", fmt.Errorf("resuming actor %s/%s: %w", atespace, actorName, err)}- if resp.Actor == nil {+ if resp.GetActor() == nil {return nil, "", fmt.Errorf("nil actor returned when resuming %s/%s", atespace, actorName)}var workerIP string- if resp.Actor.Status != nil && resp.Actor.Status.WorkerAssignment != nil {- workerIP = resp.Actor.Status.WorkerAssignment.WorkerPodIp+ if ips := resp.GetActor().GetStatus().GetWorkerAssignment().GetWorkerPodIps(); len(ips) > 0 {+ workerIP = ips[0]}- return resp.Actor, workerIP, nil+ return resp.GetActor(), workerIP, nil}// SuspendActor suspends the specified actor, triggering state checkpointing.@@ -401,7 +457,10 @@ func (c *Client) SuspendActor(ctx context.Context, atespace, actorName string) eName: actorName,},}- _, err := c.control.SuspendActor(ctx, req)+ err := retryOnCodes(ctx, func() error {+ _, err := c.control.SuspendActor(ctx, req)+ return err+ }, codes.Aborted)if err != nil {return fmt.Errorf("suspending actor %s/%s: %w", atespace, actorName, err)}@@ -419,9 +478,6 @@ func (c *Client) GetActor(ctx context.Context, atespace, actorName string) (*atereturn c.control.GetActor(ctx, req)}-// actorDeletionPollInterval is the interval between checks when waiting for an actor to be deleted.-var actorDeletionPollInterval = 200 * time.Millisecond-// DeleteActor deletes the specified actor from Substrate and blocks until the actor// is fully deleted.func (c *Client) DeleteActor(ctx context.Context, atespace, actorName string) error {@@ -432,7 +488,10 @@ func (c *Client) DeleteActor(ctx context.Context, atespace, actorName string) er},AnyState: true,}- _, err := c.control.DeleteActor(ctx, req)+ err := retryOnCodes(ctx, func() error {+ _, err := c.control.DeleteActor(ctx, req)+ return err+ }, codes.Aborted)if err != nil {if status.Code(err) == codes.NotFound {return nil@@ -458,7 +517,7 @@ func (c *Client) DeleteActor(ctx context.Context, atespace, actorName string) er}}- ticker := time.NewTicker(actorDeletionPollInterval)+ ticker := time.NewTicker(defaultRetryInterval)defer ticker.Stop()for {@@ -499,11 +558,12 @@ func (c *Client) DeleteActorTemplate(ctx context.Context, atespace, templateNameName: templateName,},}- _, err := c.control.DeleteActorTemplate(ctx, req)+ err := retryOnCodes(ctx, func() error {+ _, err := c.control.DeleteActorTemplate(ctx, req)+ return err+ }, codes.Aborted)if err != nil && status.Code(err) != codes.NotFound {return fmt.Errorf("deleting actor template %s/%s: %w", atespace, templateName, err)}return nil}--diff --git a/internal/substrate/client_test.go b/internal/substrate/client_test.goindex 2d88949..8e4faff 100644--- a/internal/substrate/client_test.go+++ b/internal/substrate/client_test.go@@ -22,6 +22,7 @@ import ("time""github.com/agent-substrate/substrate/pkg/proto/ateapipb"+ "github.com/google/ax/pkg/apis/v1alpha1""google.golang.org/grpc""google.golang.org/grpc/codes""google.golang.org/grpc/status"@@ -30,11 +31,69 @@ import (type mockControlClient struct {ateapipb.ControlClient- mu sync.Mutex- deleteCalls []*ateapipb.DeleteActorRequest- getCalls []*ateapipb.GetActorRequest- deleteErr error- actorRemaining int // Number of GetActor calls before returning NotFound+ mu sync.Mutex+ deleteCalls []*ateapipb.DeleteActorRequest+ getCalls []*ateapipb.GetActorRequest+ deleteErr error+ actorRemaining int // Number of GetActor calls before returning NotFound+ resumeErrs []error+ resumeActor *ateapipb.Actor+ deleteTemplateErrs []error+ deleteTemplateCalls int+ templates map[string]*ateapipb.ActorTemplate+ createTemplateErrs []error+ createdTemplates []*ateapipb.ActorTemplate+}++func (m *mockControlClient) GetActorTemplate(ctx context.Context, req *ateapipb.GetActorTemplateRequest, opts ...grpc.CallOption) (*ateapipb.ActorTemplate, error) {+ m.mu.Lock()+ defer m.mu.Unlock()+ key := req.GetActorTemplate().GetAtespace() + "/" + req.GetActorTemplate().GetName()+ if tmpl, ok := m.templates[key]; ok {+ return tmpl, nil+ }+ return nil, status.Error(codes.NotFound, "template not found")+}++func (m *mockControlClient) CreateActorTemplate(ctx context.Context, req *ateapipb.CreateActorTemplateRequest, opts ...grpc.CallOption) (*ateapipb.ActorTemplate, error) {+ m.mu.Lock()+ defer m.mu.Unlock()+ if len(m.createTemplateErrs) > 0 {+ err := m.createTemplateErrs[0]+ m.createTemplateErrs = m.createTemplateErrs[1:]+ if err != nil {+ return nil, err+ }+ }+ m.createdTemplates = append(m.createdTemplates, req.GetActorTemplate())+ return req.GetActorTemplate(), nil+}++func (m *mockControlClient) ResumeActor(ctx context.Context, req *ateapipb.ResumeActorRequest, opts ...grpc.CallOption) (*ateapipb.ResumeActorResponse, error) {+ m.mu.Lock()+ defer m.mu.Unlock()+ if len(m.resumeErrs) > 0 {+ err := m.resumeErrs[0]+ m.resumeErrs = m.resumeErrs[1:]+ if err != nil {+ return nil, err+ }+ }+ return &ateapipb.ResumeActorResponse{Actor: m.resumeActor, Resumed: true}, nil+}++func (m *mockControlClient) DeleteActorTemplate(ctx context.Context, req *ateapipb.DeleteActorTemplateRequest, opts ...grpc.CallOption) (*ateapipb.ActorTemplate, error) {+ m.mu.Lock()+ defer m.mu.Unlock()+ m.deleteTemplateCalls+++ if len(m.deleteTemplateErrs) > 0 {+ err := m.deleteTemplateErrs[0]+ m.deleteTemplateErrs = m.deleteTemplateErrs[1:]+ if err != nil {+ return nil, err+ }+ }+ return &ateapipb.ActorTemplate{}, nil}func (m *mockControlClient) DeleteActor(ctx context.Context, req *ateapipb.DeleteActorRequest, opts ...grpc.CallOption) (*ateapipb.Actor, error) {@@ -91,9 +150,9 @@ func TestClient_DeleteActor_AnyStateTrue(t *testing.T) {}func TestClient_DeleteActor_PollsUntilNotFound(t *testing.T) {- oldInterval := actorDeletionPollInterval- actorDeletionPollInterval = 10 * time.Millisecond- defer func() { actorDeletionPollInterval = oldInterval }()+ oldInterval := defaultRetryInterval+ defaultRetryInterval = 10 * time.Millisecond+ defer func() { defaultRetryInterval = oldInterval }()mock := &mockControlClient{actorRemaining: 3,@@ -140,3 +199,218 @@ func TestClient_DeleteActor_PropagatesRPCError(t *testing.T) {t.Fatal("expected error, got nil")}}++func TestBuildActorTemplate(t *testing.T) {+ t.Setenv("AX_SNAPSHOTS_BUCKET", "gs://ate-snapshots/ax/")++ tmpl := BuildActorTemplate("", "test-tmpl", "", map[string]string{"FOO": "BAR"}, nil, "")+ if tmpl.GetMetadata().GetAtespace() != "default" {+ t.Errorf("expected default atespace, got %q", tmpl.GetMetadata().GetAtespace())+ }+ if len(tmpl.GetContainers()) != 1 {+ t.Fatalf("expected 1 container, got %d", len(tmpl.GetContainers()))+ }+ container := tmpl.GetContainers()[0]+ if container.GetImage() != v1alpha1.DefaultTaskImage {+ t.Errorf("unexpected image: %q", container.GetImage())+ }+ probe := container.GetWakeupProbe().GetHttpGet()+ if probe == nil || probe.GetPath() != "/readyz" || probe.GetPort() != 80 {+ t.Errorf("unexpected wakeupProbe: %+v", container.GetWakeupProbe())+ }+ snapCfg := tmpl.GetSnapshotConfig()+ if snapCfg == nil {+ t.Fatal("expected SnapshotConfig to be set")+ }+ if snapCfg.GetStorageLocation() != "gs://ate-snapshots/ax/" {+ t.Errorf("expected StorageLocation gs://ate-snapshots/ax/, got %q", snapCfg.GetStorageLocation())+ }+ if snapCfg.GetPreferredFidelity() != ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES {+ t.Errorf("expected PreferredFidelity VOLUMES, got %v", snapCfg.GetPreferredFidelity())+ }+}++func TestClient_ResumeActor_WorkerPodIpsAndRetry(t *testing.T) {+ oldInterval := defaultRetryInterval+ defaultRetryInterval = 5 * time.Millisecond+ defer func() { defaultRetryInterval = oldInterval }()++ mock := &mockControlClient{+ resumeErrs: []error{+ status.Error(codes.Aborted, "operation aborted"),+ nil,+ },+ resumeActor: &ateapipb.Actor{+ Metadata: &ateapipb.ResourceMetadata{Atespace: "default", Name: "task-1"},+ Status: &ateapipb.ActorStatus{+ State: ateapipb.ActorState_ACTOR_STATE_RUNNING,+ WorkerAssignment: &ateapipb.WorkerAssignment{+ // The first (primary) IP is used, even when it is IPv6.+ WorkerPodIps: []string{"fd00::99", "10.244.0.99"},+ },+ },+ },+ }+ c := &Client{control: mock}+ actor, ip, err := c.ResumeActor(context.Background(), "default", "task-1")+ if err != nil {+ t.Fatalf("ResumeActor failed: %v", err)+ }+ if actor == nil {+ t.Fatal("expected non-nil actor")+ }+ if ip != "fd00::99" {+ t.Errorf("expected primary worker IP fd00::99, got %q", ip)+ }+}++func TestClient_ResumeActor_ResourceExhaustedNotRetried(t *testing.T) {+ mock := &mockControlClient{+ resumeErrs: []error{+ status.Error(codes.ResourceExhausted, "no worker available"),+ },+ }+ c := &Client{control: mock}+ _, _, err := c.ResumeActor(context.Background(), "default", "task-1")+ if err == nil {+ t.Fatal("expected error, got nil")+ }+ if status.Code(err) != codes.ResourceExhausted {+ t.Errorf("expected ResourceExhausted error, got %v", err)+ }+}++func TestClient_DeleteActorTemplate_RetriesOnAborted(t *testing.T) {+ oldInterval := defaultRetryInterval+ defaultRetryInterval = 5 * time.Millisecond+ defer func() { defaultRetryInterval = oldInterval }()++ mock := &mockControlClient{+ deleteTemplateErrs: []error{+ status.Error(codes.Aborted, "another operation is in progress for this ActorTemplate"),+ status.Error(codes.Aborted, "another operation is in progress for this ActorTemplate"),+ nil,+ },+ }+ c := &Client{control: mock}+ if err := c.DeleteActorTemplate(context.Background(), "default", "tmpl-1"); err != nil {+ t.Fatalf("DeleteActorTemplate failed: %v", err)+ }+ if mock.deleteTemplateCalls != 3 {+ t.Errorf("expected 3 DeleteActorTemplate calls, got %d", mock.deleteTemplateCalls)+ }+}++func TestClient_EnsureActorTemplateWithImage_Success(t *testing.T) {+ mock := &mockControlClient{}+ c := &Client{control: mock}++ created, err := c.EnsureActorTemplateWithImage(+ context.Background(),+ v1alpha1.DefaultSystemNamespace,+ "default-template",+ "default",+ "task-1-tmpl-abcd",+ "custom-image:v1",+ map[string]string{"FOO": "BAR"},+ )+ if err != nil {+ t.Fatalf("EnsureActorTemplateWithImage failed: %v", err)+ }+ if created.GetMetadata().GetAtespace() != "default" || created.GetMetadata().GetName() != "task-1-tmpl-abcd" {+ t.Errorf("unexpected metadata: %+v", created.GetMetadata())+ }+ if len(created.GetContainers()) != 1 || created.GetContainers()[0].GetImage() != "custom-image:v1" {+ t.Errorf("expected container image custom-image:v1, got %+v", created.GetContainers())+ }++ mock.mu.Lock()+ defer mock.mu.Unlock()+ if len(mock.createdTemplates) != 1 {+ t.Fatalf("expected 1 CreateActorTemplate request, got %d", len(mock.createdTemplates))+ }+}++func TestClient_EnsureActorTemplateWithImage_PropagatesError(t *testing.T) {+ mock := &mockControlClient{+ createTemplateErrs: []error{+ status.Error(codes.InvalidArgument, "actor_template.snapshot_config: invalid field"),+ },+ }+ c := &Client{control: mock}++ _, err := c.EnsureActorTemplateWithImage(+ context.Background(),+ v1alpha1.DefaultSystemNamespace,+ "default-template",+ "default",+ "task-1-tmpl-abcd",+ "custom-image:v1",+ )+ if err == nil {+ t.Fatal("expected error, got nil")+ }+ if status.Code(err) != codes.InvalidArgument {+ t.Errorf("expected InvalidArgument error, got %v", err)+ }+ mock.mu.Lock()+ defer mock.mu.Unlock()+ if len(mock.createdTemplates) != 0 {+ t.Errorf("expected no created templates on failure, got %d", len(mock.createdTemplates))+ }+}++func TestRetryOnCodes(t *testing.T) {+ oldInterval, oldMax := defaultRetryInterval, maxRetryInterval+ defaultRetryInterval, maxRetryInterval = time.Millisecond, 2*time.Millisecond+ defer func() { defaultRetryInterval, maxRetryInterval = oldInterval, oldMax }()++ t.Run("non-retryable returns immediately", func(t *testing.T) {+ calls := 0+ wantErr := status.Error(codes.Internal, "boom")+ err := retryOnCodes(context.Background(), func() error {+ calls+++ return wantErr+ }, codes.Aborted)+ if err != wantErr || calls != 1 {+ t.Errorf("got err=%v (%T) calls=%d, want original Internal error after 1 call", err, err, calls)+ }+ })++ t.Run("non-retryable on final attempt is unwrapped", func(t *testing.T) {+ calls := 0+ wantErr := status.Error(codes.Internal, "boom")+ err := retryOnCodes(context.Background(), func() error {+ calls+++ if calls == defaultRetryAttempts {+ return wantErr+ }+ return status.Error(codes.Aborted, "busy")+ }, codes.Aborted)+ if err != wantErr || calls != defaultRetryAttempts {+ t.Errorf("got err=%v (%T) calls=%d, want original Internal error after %d calls", err, err, calls, defaultRetryAttempts)+ }+ })++ t.Run("exhausts attempts", func(t *testing.T) {+ calls := 0+ err := retryOnCodes(context.Background(), func() error {+ calls+++ return status.Error(codes.Aborted, "busy")+ }, codes.Aborted)+ if status.Code(err) != codes.Aborted || calls != defaultRetryAttempts {+ t.Errorf("got err=%v calls=%d, want Aborted after %d calls", err, calls, defaultRetryAttempts)+ }+ })++ t.Run("context cancelled", func(t *testing.T) {+ ctx, cancel := context.WithCancel(context.Background())+ cancel()+ err := retryOnCodes(ctx, func() error {+ return status.Error(codes.Aborted, "busy")+ }, codes.Aborted)+ if !errors.Is(err, context.Canceled) {+ t.Errorf("expected context.Canceled, got %v", err)+ }+ })+}diff --git a/internal/tunnel/tunnel.go b/internal/tunnel/tunnel.goindex 6ab44eb..b0633e5 100644--- a/internal/tunnel/tunnel.go+++ b/internal/tunnel/tunnel.go@@ -27,6 +27,8 @@ import ("strings""syscall""time"++ "github.com/google/ax/pkg/apis/v1alpha1")var portForwardRegex = regexp.MustCompile(`Forwarding from .*:(\d+)\s+->`)@@ -196,7 +198,7 @@ func EnsureServerURL(opts Options) (string, error) {if envNs := os.Getenv("AX_SYSTEM_NAMESPACE"); envNs != "" {opts.Namespace = envNs} else {- opts.Namespace = "ax-system"+ opts.Namespace = v1alpha1.DefaultSystemNamespace}}if opts.Service == "" {@@ -208,7 +210,11 @@ func EnsureServerURL(opts Options) (string, error) {// Check for existing active tunnelif existing, err := GetTunnel(ctxName); err == nil && existing != nil {- if IsTunnelHealthy(existing.Port) {+ // Empty Namespace/Service means the tunnel was recorded before these fields+ // existed; treat it as matching for backward compatibility.+ sameTarget := (existing.Namespace == "" || existing.Namespace == opts.Namespace) &&+ (existing.Service == "" || existing.Service == opts.Service)+ if sameTarget && IsTunnelHealthy(existing.Port) {return fmt.Sprintf("http://127.0.0.1:%d", existing.Port), nil}// Existing tunnel is unhealthy or stale, clean it updiff --git a/pkg/apis/v1alpha1/types.go b/pkg/apis/v1alpha1/types.goindex 362722b..befcfaf 100644--- a/pkg/apis/v1alpha1/types.go+++ b/pkg/apis/v1alpha1/types.go@@ -35,7 +35,8 @@ const (KindWorkspace = "Workspace"KindModel = "Model"- DefaultTaskImage = "gcr.io/ax-substrate/ate-images/ax-task-runner"+ DefaultTaskImage = "gcr.io/ax-substrate/ate-images/ax-task-runner"+ DefaultSystemNamespace = "ax-system"// PhaseTerminating marks a task whose deletion has been requested and whose// actor is being torn down. The record disappears once cleanup completes.