Skip to content

armgit / ax

Public Imported from GitHub
0

Agent Executor (AX) is Google's open agentic orchestration runtime

Go

fix(substrate): adapt to Substrate v0.4 API and harden actor RPCs (#457)

Siyuan Zhang2e06707c0024
diff --git a/cmd/ax-server/main.go b/cmd/ax-server/main.go
index 4f224fd..0b5cc4b 100644
--- a/cmd/ax-server/main.go
+++ b/cmd/ax-server/main.go
@@ -29,6 +29,7 @@ import (
"github.com/google/ax/internal/server"
"github.com/google/ax/internal/store/redis"
"github.com/google/ax/internal/substrate"
+ "github.com/google/ax/pkg/apis/v1alpha1"
goredis "github.com/redis/go-redis/v9"
)
@@ -57,7 +58,7 @@ func main() {
flag.BoolVar(&substrateInsecureTLS, "substrate-insecure-tls", false, "Skip Substrate TLS verification")
flag.BoolVar(&substratePlaintext, "substrate-plaintext", false, "Use insecure plaintext gRPC connection to Substrate")
flag.StringVar(&defaultTemplate, "template", "default-template", "Default Substrate ActorTemplate name")
- flag.StringVar(&defaultTemplateAtespace, "template-atespace", "ax-system", "Default Substrate ActorTemplate atespace")
+ flag.StringVar(&defaultTemplateAtespace, "template-atespace", v1alpha1.DefaultSystemNamespace, "Default Substrate ActorTemplate atespace")
flag.Parse()
if envAddr := os.Getenv("ADDR"); envAddr != "" {
diff --git a/cmd/ax/main.go b/cmd/ax/main.go
index a7bfb3a..db06a87 100644
--- a/cmd/ax/main.go
+++ b/cmd/ax/main.go
@@ -51,7 +51,7 @@ func main() {
atespace = "default"
explicitServer = ""
kubeContext = ""
- axNamespace = "ax-system"
+ axNamespace = v1alpha1.DefaultSystemNamespace
)
args := os.Args[1:]
diff --git a/go.mod b/go.mod
index 89b69cf..eb48ce6 100644
--- a/go.mod
+++ b/go.mod
@@ -4,7 +4,8 @@ go 1.27.1
require (
github.com/agent-substrate/env v0.0.11-0.20260912052224-4468a200b170
- github.com/agent-substrate/substrate v0.0.0-20260918201817-944abe3278b8
+ github.com/agent-substrate/substrate v0.4.1-0.20261009152331-f6a1dc72a35d
+ github.com/cenkalti/backoff/v5 v5.0.3
github.com/redis/go-redis/v9 v9.22.0
google.golang.org/grpc v1.83.2
google.golang.org/protobuf v1.36.12
@@ -19,6 +20,6 @@ require (
golang.org/x/net v0.58.0 // indirect
golang.org/x/sys v0.47.0 // indirect
golang.org/x/text v0.41.0 // indirect
- google.golang.org/genproto/googleapis/rpc v0.0.0-20260720211330-0afa2a65878a // indirect
+ google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d // indirect
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
)
diff --git a/go.sum b/go.sum
index 27abe18..efbaf8d 100644
--- a/go.sum
+++ b/go.sum
@@ -1,11 +1,13 @@
github.com/agent-substrate/env v0.0.11-0.20260912052224-4468a200b170 h1:8AxhEx3BL7J8c+zqnYq5S3aH0J/4z85/JYWoo1dr6xY=
github.com/agent-substrate/env v0.0.11-0.20260912052224-4468a200b170/go.mod h1:RZ4ACmDqWQx7fgX9b1EaFJskeYTZh1Y7uZinDtp7U2o=
-github.com/agent-substrate/substrate v0.0.0-20260918201817-944abe3278b8 h1:C39j2y88zDvg0VVrIfJqYejCjeHI25hugGubMeee74w=
-github.com/agent-substrate/substrate v0.0.0-20260918201817-944abe3278b8/go.mod h1:pWnhZpFkHq9VjF2OxsqEbryLnnUYmohhC62bFoindCM=
+github.com/agent-substrate/substrate v0.4.1-0.20261009152331-f6a1dc72a35d h1:LGw4Z7X0wXXDRLYRpo3j0GYGEnrqVL39K4NEdGdb578=
+github.com/agent-substrate/substrate v0.4.1-0.20261009152331-f6a1dc72a35d/go.mod h1:IuJNQ+2si6Sxkg9b3+kfhaXQiaxSwHtsiQrpNFWbjZY=
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
+github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM=
+github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
@@ -40,16 +42,16 @@ github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs=
github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s=
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
-go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU=
-go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc=
-go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc=
-go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo=
-go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58=
-go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0=
-go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI=
-go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA=
-go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk=
-go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE=
+go.opentelemetry.io/otel v1.45.0 h1:pdrWmLHofpubmArBv1LgFSv1Z0Ie/ppdZzu+kUN5EeU=
+go.opentelemetry.io/otel v1.45.0/go.mod h1:XZxIqPapzEYnhNSScF5DIqXhm/rYi0FzCe2XddAwZfQ=
+go.opentelemetry.io/otel/metric v1.45.0 h1:7Eg1uH7CJ5cXv9is6tnBe1FI6rj1nwUdbFypRm3br/M=
+go.opentelemetry.io/otel/metric v1.45.0/go.mod h1:HAPbm1nd3p1PmFH7v2dR+6BjXxw+Lq4a2+pndMAm08s=
+go.opentelemetry.io/otel/sdk v1.45.0 h1:4VVSMgQ83dUgW2aoX5f6JgLvHwIvzcuLnF9lUdCSpCw=
+go.opentelemetry.io/otel/sdk v1.45.0/go.mod h1:Sr40LgXV7DsKMMJMKOhUWOgMWTfAaqvm2kF0g7ilwuA=
+go.opentelemetry.io/otel/sdk/metric v1.45.0 h1:oVFszMfyj1Am6s24Vtc7wBb8BKLcwepJjNEYILuiE3o=
+go.opentelemetry.io/otel/sdk/metric v1.45.0/go.mod h1:vUWUxDZvu1WVRj8JA8S0AdhsPrZoDpA2DdZauIh4mDA=
+go.opentelemetry.io/otel/trace v1.45.0 h1:l/mP6Uv7oNO7/TblbhpbgMidxhq1uO/rPsikOyVhxag=
+go.opentelemetry.io/otel/trace v1.45.0/go.mod h1:qoJJA2xNMnxRrdISU/kLtfUH2wNeQbiv+jhs/CxI8bc=
go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
@@ -62,8 +64,8 @@ golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8=
golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M=
gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20260720211330-0afa2a65878a h1:qI/YMH1ep2qQtqcp00gMQyoU7mjvbhg88GJKCvfoLj0=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20260720211330-0afa2a65878a/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d h1:IL4hdHzcUv2l/gcg98/Rj3FbtE6axwqslOW8SW0C+S0=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU=
google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8=
google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc=
diff --git a/internal/controller/reconciler.go b/internal/controller/reconciler.go
index 55814d7..341ac5a 100644
--- a/internal/controller/reconciler.go
+++ b/internal/controller/reconciler.go
@@ -79,7 +79,7 @@ func NewTaskReconciler(client *substrate.Client, defaultTemplate, defaultTemplat
defaultTemplate = "default-template"
}
if defaultTemplateAtespace == "" {
- defaultTemplateAtespace = "ax-system"
+ defaultTemplateAtespace = v1alpha1.DefaultSystemNamespace
}
return &TaskReconciler{
client: client,
@@ -222,7 +222,7 @@ func (r *TaskReconciler) Reconcile(ctx context.Context, task *v1alpha1.Task, wor
host = h
port = p
}
- readyURL := fmt.Sprintf("http://%s:%s/readyz?check=workspace", host, port)
+ readyURL := fmt.Sprintf("http://%s/readyz?check=workspace", net.JoinHostPort(host, port))
// Workspace setup happens once per task. After it has completed, WorkspaceReady stays
// True across suspend/resume cycles, so only poll while it is still initializing.
workspaceReady := r.conditionTrue(task, condWorkspaceReady)
diff --git a/internal/controller/reconciler_test.go b/internal/controller/reconciler_test.go
index e91954e..f3e1d6b 100644
--- a/internal/controller/reconciler_test.go
+++ b/internal/controller/reconciler_test.go
@@ -112,8 +112,8 @@ func (m *mockControlServer) ResumeActor(ctx context.Context, req *ateapipb.Resum
Status: &ateapipb.ActorStatus{
State: ateapipb.ActorState_ACTOR_STATE_RUNNING,
WorkerAssignment: &ateapipb.WorkerAssignment{
- WorkerPod: "worker-pod-1",
- WorkerPodIp: wIP,
+ WorkerPod: "worker-pod-1",
+ WorkerPodIps: []string{wIP},
},
},
},
@@ -214,7 +214,7 @@ func TestTaskReconciler(t *testing.T) {
defer client.Close()
// 3. Reconcile Task
- reconciler := controller.NewTaskReconciler(client, "test-template", "ax-system")
+ reconciler := controller.NewTaskReconciler(client, "test-template", v1alpha1.DefaultSystemNamespace)
reconciler.SecretResolver = noSecrets
reconciler.WorkspaceReadyTimeout = 200 * time.Millisecond
@@ -283,7 +283,7 @@ func TestTaskReconciler_Suspend(t *testing.T) {
}
defer client.Close()
- reconciler := controller.NewTaskReconciler(client, "test-template", "ax-system")
+ reconciler := controller.NewTaskReconciler(client, "test-template", v1alpha1.DefaultSystemNamespace)
reconciler.SecretResolver = noSecrets
reconciler.WorkspaceReadyTimeout = 200 * time.Millisecond
@@ -360,7 +360,7 @@ func TestTaskReconciler_WorkspaceReady(t *testing.T) {
}
defer client.Close()
- reconciler := controller.NewTaskReconciler(client, "test-template", "ax-system")
+ reconciler := controller.NewTaskReconciler(client, "test-template", v1alpha1.DefaultSystemNamespace)
reconciler.SecretResolver = noSecrets
reconciler.WorkspaceReadyTimeout = 200 * time.Millisecond
@@ -476,7 +476,7 @@ func TestReconcileDelete_RemovesActorAndTemplates(t *testing.T) {
}
defer client.Close()
- reconciler := controller.NewTaskReconciler(client, "test-template", "ax-system")
+ reconciler := controller.NewTaskReconciler(client, "test-template", v1alpha1.DefaultSystemNamespace)
reconciler.SecretResolver = noSecrets
reconciler.WorkspaceReadyTimeout = 200 * time.Millisecond
@@ -539,7 +539,7 @@ func TestReconcileDelete_BlocksUntilActorDeleted(t *testing.T) {
}
defer client.Close()
- reconciler := controller.NewTaskReconciler(client, "test-template", "ax-system")
+ reconciler := controller.NewTaskReconciler(client, "test-template", v1alpha1.DefaultSystemNamespace)
reconciler.SecretResolver = noSecrets
if err := reconciler.ReconcileDelete(ctx, "default", "slow-delete-task"); err != nil {
@@ -581,7 +581,7 @@ func TestReconcileDelete_ActorDeletionTimeout(t *testing.T) {
}
defer client.Close()
- reconciler := controller.NewTaskReconciler(client, "test-template", "ax-system")
+ reconciler := controller.NewTaskReconciler(client, "test-template", v1alpha1.DefaultSystemNamespace)
reconciler.SecretResolver = noSecrets
err = reconciler.ReconcileDelete(ctx, "default", "stuck-task")
diff --git a/internal/substrate/client.go b/internal/substrate/client.go
index 7e65273..234016d 100644
--- a/internal/substrate/client.go
+++ b/internal/substrate/client.go
@@ -18,13 +18,16 @@ import (
"context"
"crypto/tls"
"crypto/x509"
+ "errors"
"fmt"
"log/slog"
"os"
+ "slices"
"strings"
"time"
"github.com/agent-substrate/substrate/pkg/proto/ateapipb"
+ "github.com/cenkalti/backoff/v5"
"github.com/google/ax/pkg/apis/v1alpha1"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
@@ -206,8 +209,60 @@ func (c *Client) GetActorTemplate(ctx context.Context, atespace, templateName st
const (
DefaultGuestCommand = "/usr/local/bin/ax-task-runner"
DefaultSnapshotsBucket = "gs://dberkov-gke-dev3/ate-env/"
+
+ // defaultRetryAttempts is the maximum number of calls retryOnCodes makes. With
+ // exponential backoff from defaultRetryInterval capped at maxRetryInterval
+ // (±50% jitter), the total wait across all attempts is roughly 6s-20s.
+ defaultRetryAttempts = 10
+
+ // retryBackoffFactor is the multiplier applied to the backoff after each failed attempt.
+ retryBackoffFactor = 2
+
+ // actorDeletingWaitAttempts bounds how many times EnsureActor retries creation
+ // while a previous actor with the same name is still being deleted.
+ actorDeletingWaitAttempts = 20
+)
+
+var (
+ // defaultRetryInterval is the initial backoff for retryOnCodes and the poll
+ // interval used while waiting for actor deletion.
+ defaultRetryInterval = 200 * time.Millisecond
+
+ // maxRetryInterval caps the exponential backoff in retryOnCodes.
+ maxRetryInterval = 2 * time.Second
+
+ // actorDeletingWaitInterval is the interval between creation attempts while a
+ // previous actor is still deleting (actorDeletingWaitAttempts * interval ~= 10s).
+ actorDeletingWaitInterval = 500 * time.Millisecond
)
+// retryOnCodes calls fn until it succeeds, returns an error whose gRPC code is not in
+// retryable, defaultRetryAttempts is exhausted, or ctx is done. Between attempts it
+// waits with jittered exponential backoff starting at defaultRetryInterval and
+// capped at maxRetryInterval.
+func retryOnCodes(ctx context.Context, fn func() error, retryable ...codes.Code) error {
+ b := backoff.NewExponentialBackOff()
+ b.InitialInterval = defaultRetryInterval
+ b.MaxInterval = maxRetryInterval
+ b.Multiplier = retryBackoffFactor
+
+ _, err := backoff.Retry(ctx, func() (struct{}, error) {
+ err := fn()
+ if err != nil && !slices.Contains(retryable, status.Code(err)) {
+ return struct{}{}, backoff.Permanent(err)
+ }
+ return struct{}{}, err
+ }, backoff.WithBackOff(b), backoff.WithMaxTries(defaultRetryAttempts))
+
+ // backoff.Retry returns the *PermanentError wrapper as-is when the attempt limit
+ // is hit on the same try, so unwrap it to always surface the original error.
+ var permanent *backoff.PermanentError
+ if errors.As(err, &permanent) {
+ return permanent.Unwrap()
+ }
+ return err
+}
+
// BuildActorTemplate constructs a Substrate ActorTemplate based on the standard ate-env specification.
func BuildActorTemplate(atespace, name, image string, envMap map[string]string, command []string, snapshotsBucket string) *ateapipb.ActorTemplate {
if atespace == "" {
@@ -245,7 +300,7 @@ func BuildActorTemplate(atespace, name, image string, envMap map[string]string,
Image: image,
Command: command,
Env: envList,
- Readyz: &ateapipb.ContainerReadyz{
+ WakeupProbe: &ateapipb.ContainerWakeupProbe{
HttpGet: &ateapipb.HTTPGetAction{
Path: "/readyz",
Port: 80,
@@ -260,13 +315,9 @@ func BuildActorTemplate(atespace, name, image string, envMap map[string]string,
Name: "workspace",
DurableDir: &ateapipb.DurableDirVolumeSource{},
}},
- SnapshotsConfig: &ateapipb.SnapshotsConfig{
- StorageLocation: snapshotsBucket,
- OnPause: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA,
- OnCommit: ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_DATA,
- OnResume: &ateapipb.OnResumeConfig{
- FromData: ateapipb.ResumeSource_RESUME_SOURCE_GOLDEN,
- },
+ SnapshotConfig: &ateapipb.SnapshotConfig{
+ StorageLocation: snapshotsBucket,
+ PreferredFidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES,
},
SandboxConfig: &ateapipb.SandboxConfig{
SandboxClass: ateapipb.SandboxClass_SANDBOX_CLASS_GVISOR,
@@ -347,11 +398,11 @@ func (c *Client) EnsureActor(ctx context.Context, atespace, actorName, templateA
}
if state == ateapipb.ActorState_ACTOR_STATE_DELETING {
// Wait briefly for previous actor deletion to finalize before recreating
- for i := 0; i < 20; i++ {
+ for i := 0; i < actorDeletingWaitAttempts; i++ {
select {
case <-ctx.Done():
return nil, ctx.Err()
- case <-time.After(500 * time.Millisecond):
+ case <-time.After(actorDeletingWaitInterval):
}
actor, err = c.control.CreateActor(ctx, req)
if err == nil {
@@ -379,18 +430,23 @@ func (c *Client) ResumeActor(ctx context.Context, atespace, actorName string) (*
Name: actorName,
},
}
- resp, err := c.control.ResumeActor(ctx, req)
+ var resp *ateapipb.ResumeActorResponse
+ err := retryOnCodes(ctx, func() error {
+ var err error
+ resp, err = c.control.ResumeActor(ctx, req)
+ return err
+ }, codes.Aborted)
if err != nil {
return nil, "", fmt.Errorf("resuming actor %s/%s: %w", atespace, actorName, err)
}
- if resp.Actor == nil {
+ if resp.GetActor() == nil {
return nil, "", fmt.Errorf("nil actor returned when resuming %s/%s", atespace, actorName)
}
var workerIP string
- if resp.Actor.Status != nil && resp.Actor.Status.WorkerAssignment != nil {
- workerIP = resp.Actor.Status.WorkerAssignment.WorkerPodIp
+ if ips := resp.GetActor().GetStatus().GetWorkerAssignment().GetWorkerPodIps(); len(ips) > 0 {
+ workerIP = ips[0]
}
- return resp.Actor, workerIP, nil
+ return resp.GetActor(), workerIP, nil
}
// SuspendActor suspends the specified actor, triggering state checkpointing.
@@ -401,7 +457,10 @@ func (c *Client) SuspendActor(ctx context.Context, atespace, actorName string) e
Name: actorName,
},
}
- _, err := c.control.SuspendActor(ctx, req)
+ err := retryOnCodes(ctx, func() error {
+ _, err := c.control.SuspendActor(ctx, req)
+ return err
+ }, codes.Aborted)
if err != nil {
return fmt.Errorf("suspending actor %s/%s: %w", atespace, actorName, err)
}
@@ -419,9 +478,6 @@ func (c *Client) GetActor(ctx context.Context, atespace, actorName string) (*ate
return c.control.GetActor(ctx, req)
}
-// actorDeletionPollInterval is the interval between checks when waiting for an actor to be deleted.
-var actorDeletionPollInterval = 200 * time.Millisecond
-
// DeleteActor deletes the specified actor from Substrate and blocks until the actor
// is fully deleted.
func (c *Client) DeleteActor(ctx context.Context, atespace, actorName string) error {
@@ -432,7 +488,10 @@ func (c *Client) DeleteActor(ctx context.Context, atespace, actorName string) er
},
AnyState: true,
}
- _, err := c.control.DeleteActor(ctx, req)
+ err := retryOnCodes(ctx, func() error {
+ _, err := c.control.DeleteActor(ctx, req)
+ return err
+ }, codes.Aborted)
if err != nil {
if status.Code(err) == codes.NotFound {
return nil
@@ -458,7 +517,7 @@ func (c *Client) DeleteActor(ctx context.Context, atespace, actorName string) er
}
}
- ticker := time.NewTicker(actorDeletionPollInterval)
+ ticker := time.NewTicker(defaultRetryInterval)
defer ticker.Stop()
for {
@@ -499,11 +558,12 @@ func (c *Client) DeleteActorTemplate(ctx context.Context, atespace, templateName
Name: templateName,
},
}
- _, err := c.control.DeleteActorTemplate(ctx, req)
+ err := retryOnCodes(ctx, func() error {
+ _, err := c.control.DeleteActorTemplate(ctx, req)
+ return err
+ }, codes.Aborted)
if err != nil && status.Code(err) != codes.NotFound {
return fmt.Errorf("deleting actor template %s/%s: %w", atespace, templateName, err)
}
return nil
}
-
-
diff --git a/internal/substrate/client_test.go b/internal/substrate/client_test.go
index 2d88949..8e4faff 100644
--- a/internal/substrate/client_test.go
+++ b/internal/substrate/client_test.go
@@ -22,6 +22,7 @@ import (
"time"
"github.com/agent-substrate/substrate/pkg/proto/ateapipb"
+ "github.com/google/ax/pkg/apis/v1alpha1"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
@@ -30,11 +31,69 @@ import (
type mockControlClient struct {
ateapipb.ControlClient
- mu sync.Mutex
- deleteCalls []*ateapipb.DeleteActorRequest
- getCalls []*ateapipb.GetActorRequest
- deleteErr error
- actorRemaining int // Number of GetActor calls before returning NotFound
+ mu sync.Mutex
+ deleteCalls []*ateapipb.DeleteActorRequest
+ getCalls []*ateapipb.GetActorRequest
+ deleteErr error
+ actorRemaining int // Number of GetActor calls before returning NotFound
+ resumeErrs []error
+ resumeActor *ateapipb.Actor
+ deleteTemplateErrs []error
+ deleteTemplateCalls int
+ templates map[string]*ateapipb.ActorTemplate
+ createTemplateErrs []error
+ createdTemplates []*ateapipb.ActorTemplate
+}
+
+func (m *mockControlClient) GetActorTemplate(ctx context.Context, req *ateapipb.GetActorTemplateRequest, opts ...grpc.CallOption) (*ateapipb.ActorTemplate, error) {
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ key := req.GetActorTemplate().GetAtespace() + "/" + req.GetActorTemplate().GetName()
+ if tmpl, ok := m.templates[key]; ok {
+ return tmpl, nil
+ }
+ return nil, status.Error(codes.NotFound, "template not found")
+}
+
+func (m *mockControlClient) CreateActorTemplate(ctx context.Context, req *ateapipb.CreateActorTemplateRequest, opts ...grpc.CallOption) (*ateapipb.ActorTemplate, error) {
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ if len(m.createTemplateErrs) > 0 {
+ err := m.createTemplateErrs[0]
+ m.createTemplateErrs = m.createTemplateErrs[1:]
+ if err != nil {
+ return nil, err
+ }
+ }
+ m.createdTemplates = append(m.createdTemplates, req.GetActorTemplate())
+ return req.GetActorTemplate(), nil
+}
+
+func (m *mockControlClient) ResumeActor(ctx context.Context, req *ateapipb.ResumeActorRequest, opts ...grpc.CallOption) (*ateapipb.ResumeActorResponse, error) {
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ if len(m.resumeErrs) > 0 {
+ err := m.resumeErrs[0]
+ m.resumeErrs = m.resumeErrs[1:]
+ if err != nil {
+ return nil, err
+ }
+ }
+ return &ateapipb.ResumeActorResponse{Actor: m.resumeActor, Resumed: true}, nil
+}
+
+func (m *mockControlClient) DeleteActorTemplate(ctx context.Context, req *ateapipb.DeleteActorTemplateRequest, opts ...grpc.CallOption) (*ateapipb.ActorTemplate, error) {
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ m.deleteTemplateCalls++
+ if len(m.deleteTemplateErrs) > 0 {
+ err := m.deleteTemplateErrs[0]
+ m.deleteTemplateErrs = m.deleteTemplateErrs[1:]
+ if err != nil {
+ return nil, err
+ }
+ }
+ return &ateapipb.ActorTemplate{}, nil
}
func (m *mockControlClient) DeleteActor(ctx context.Context, req *ateapipb.DeleteActorRequest, opts ...grpc.CallOption) (*ateapipb.Actor, error) {
@@ -91,9 +150,9 @@ func TestClient_DeleteActor_AnyStateTrue(t *testing.T) {
}
func TestClient_DeleteActor_PollsUntilNotFound(t *testing.T) {
- oldInterval := actorDeletionPollInterval
- actorDeletionPollInterval = 10 * time.Millisecond
- defer func() { actorDeletionPollInterval = oldInterval }()
+ oldInterval := defaultRetryInterval
+ defaultRetryInterval = 10 * time.Millisecond
+ defer func() { defaultRetryInterval = oldInterval }()
mock := &mockControlClient{
actorRemaining: 3,
@@ -140,3 +199,218 @@ func TestClient_DeleteActor_PropagatesRPCError(t *testing.T) {
t.Fatal("expected error, got nil")
}
}
+
+func TestBuildActorTemplate(t *testing.T) {
+ t.Setenv("AX_SNAPSHOTS_BUCKET", "gs://ate-snapshots/ax/")
+
+ tmpl := BuildActorTemplate("", "test-tmpl", "", map[string]string{"FOO": "BAR"}, nil, "")
+ if tmpl.GetMetadata().GetAtespace() != "default" {
+ t.Errorf("expected default atespace, got %q", tmpl.GetMetadata().GetAtespace())
+ }
+ if len(tmpl.GetContainers()) != 1 {
+ t.Fatalf("expected 1 container, got %d", len(tmpl.GetContainers()))
+ }
+ container := tmpl.GetContainers()[0]
+ if container.GetImage() != v1alpha1.DefaultTaskImage {
+ t.Errorf("unexpected image: %q", container.GetImage())
+ }
+ probe := container.GetWakeupProbe().GetHttpGet()
+ if probe == nil || probe.GetPath() != "/readyz" || probe.GetPort() != 80 {
+ t.Errorf("unexpected wakeupProbe: %+v", container.GetWakeupProbe())
+ }
+ snapCfg := tmpl.GetSnapshotConfig()
+ if snapCfg == nil {
+ t.Fatal("expected SnapshotConfig to be set")
+ }
+ if snapCfg.GetStorageLocation() != "gs://ate-snapshots/ax/" {
+ t.Errorf("expected StorageLocation gs://ate-snapshots/ax/, got %q", snapCfg.GetStorageLocation())
+ }
+ if snapCfg.GetPreferredFidelity() != ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES {
+ t.Errorf("expected PreferredFidelity VOLUMES, got %v", snapCfg.GetPreferredFidelity())
+ }
+}
+
+func TestClient_ResumeActor_WorkerPodIpsAndRetry(t *testing.T) {
+ oldInterval := defaultRetryInterval
+ defaultRetryInterval = 5 * time.Millisecond
+ defer func() { defaultRetryInterval = oldInterval }()
+
+ mock := &mockControlClient{
+ resumeErrs: []error{
+ status.Error(codes.Aborted, "operation aborted"),
+ nil,
+ },
+ resumeActor: &ateapipb.Actor{
+ Metadata: &ateapipb.ResourceMetadata{Atespace: "default", Name: "task-1"},
+ Status: &ateapipb.ActorStatus{
+ State: ateapipb.ActorState_ACTOR_STATE_RUNNING,
+ WorkerAssignment: &ateapipb.WorkerAssignment{
+ // The first (primary) IP is used, even when it is IPv6.
+ WorkerPodIps: []string{"fd00::99", "10.244.0.99"},
+ },
+ },
+ },
+ }
+ c := &Client{control: mock}
+ actor, ip, err := c.ResumeActor(context.Background(), "default", "task-1")
+ if err != nil {
+ t.Fatalf("ResumeActor failed: %v", err)
+ }
+ if actor == nil {
+ t.Fatal("expected non-nil actor")
+ }
+ if ip != "fd00::99" {
+ t.Errorf("expected primary worker IP fd00::99, got %q", ip)
+ }
+}
+
+func TestClient_ResumeActor_ResourceExhaustedNotRetried(t *testing.T) {
+ mock := &mockControlClient{
+ resumeErrs: []error{
+ status.Error(codes.ResourceExhausted, "no worker available"),
+ },
+ }
+ c := &Client{control: mock}
+ _, _, err := c.ResumeActor(context.Background(), "default", "task-1")
+ if err == nil {
+ t.Fatal("expected error, got nil")
+ }
+ if status.Code(err) != codes.ResourceExhausted {
+ t.Errorf("expected ResourceExhausted error, got %v", err)
+ }
+}
+
+func TestClient_DeleteActorTemplate_RetriesOnAborted(t *testing.T) {
+ oldInterval := defaultRetryInterval
+ defaultRetryInterval = 5 * time.Millisecond
+ defer func() { defaultRetryInterval = oldInterval }()
+
+ mock := &mockControlClient{
+ deleteTemplateErrs: []error{
+ status.Error(codes.Aborted, "another operation is in progress for this ActorTemplate"),
+ status.Error(codes.Aborted, "another operation is in progress for this ActorTemplate"),
+ nil,
+ },
+ }
+ c := &Client{control: mock}
+ if err := c.DeleteActorTemplate(context.Background(), "default", "tmpl-1"); err != nil {
+ t.Fatalf("DeleteActorTemplate failed: %v", err)
+ }
+ if mock.deleteTemplateCalls != 3 {
+ t.Errorf("expected 3 DeleteActorTemplate calls, got %d", mock.deleteTemplateCalls)
+ }
+}
+
+func TestClient_EnsureActorTemplateWithImage_Success(t *testing.T) {
+ mock := &mockControlClient{}
+ c := &Client{control: mock}
+
+ created, err := c.EnsureActorTemplateWithImage(
+ context.Background(),
+ v1alpha1.DefaultSystemNamespace,
+ "default-template",
+ "default",
+ "task-1-tmpl-abcd",
+ "custom-image:v1",
+ map[string]string{"FOO": "BAR"},
+ )
+ if err != nil {
+ t.Fatalf("EnsureActorTemplateWithImage failed: %v", err)
+ }
+ if created.GetMetadata().GetAtespace() != "default" || created.GetMetadata().GetName() != "task-1-tmpl-abcd" {
+ t.Errorf("unexpected metadata: %+v", created.GetMetadata())
+ }
+ if len(created.GetContainers()) != 1 || created.GetContainers()[0].GetImage() != "custom-image:v1" {
+ t.Errorf("expected container image custom-image:v1, got %+v", created.GetContainers())
+ }
+
+ mock.mu.Lock()
+ defer mock.mu.Unlock()
+ if len(mock.createdTemplates) != 1 {
+ t.Fatalf("expected 1 CreateActorTemplate request, got %d", len(mock.createdTemplates))
+ }
+}
+
+func TestClient_EnsureActorTemplateWithImage_PropagatesError(t *testing.T) {
+ mock := &mockControlClient{
+ createTemplateErrs: []error{
+ status.Error(codes.InvalidArgument, "actor_template.snapshot_config: invalid field"),
+ },
+ }
+ c := &Client{control: mock}
+
+ _, err := c.EnsureActorTemplateWithImage(
+ context.Background(),
+ v1alpha1.DefaultSystemNamespace,
+ "default-template",
+ "default",
+ "task-1-tmpl-abcd",
+ "custom-image:v1",
+ )
+ if err == nil {
+ t.Fatal("expected error, got nil")
+ }
+ if status.Code(err) != codes.InvalidArgument {
+ t.Errorf("expected InvalidArgument error, got %v", err)
+ }
+ mock.mu.Lock()
+ defer mock.mu.Unlock()
+ if len(mock.createdTemplates) != 0 {
+ t.Errorf("expected no created templates on failure, got %d", len(mock.createdTemplates))
+ }
+}
+
+func TestRetryOnCodes(t *testing.T) {
+ oldInterval, oldMax := defaultRetryInterval, maxRetryInterval
+ defaultRetryInterval, maxRetryInterval = time.Millisecond, 2*time.Millisecond
+ defer func() { defaultRetryInterval, maxRetryInterval = oldInterval, oldMax }()
+
+ t.Run("non-retryable returns immediately", func(t *testing.T) {
+ calls := 0
+ wantErr := status.Error(codes.Internal, "boom")
+ err := retryOnCodes(context.Background(), func() error {
+ calls++
+ return wantErr
+ }, codes.Aborted)
+ if err != wantErr || calls != 1 {
+ t.Errorf("got err=%v (%T) calls=%d, want original Internal error after 1 call", err, err, calls)
+ }
+ })
+
+ t.Run("non-retryable on final attempt is unwrapped", func(t *testing.T) {
+ calls := 0
+ wantErr := status.Error(codes.Internal, "boom")
+ err := retryOnCodes(context.Background(), func() error {
+ calls++
+ if calls == defaultRetryAttempts {
+ return wantErr
+ }
+ return status.Error(codes.Aborted, "busy")
+ }, codes.Aborted)
+ if err != wantErr || calls != defaultRetryAttempts {
+ t.Errorf("got err=%v (%T) calls=%d, want original Internal error after %d calls", err, err, calls, defaultRetryAttempts)
+ }
+ })
+
+ t.Run("exhausts attempts", func(t *testing.T) {
+ calls := 0
+ err := retryOnCodes(context.Background(), func() error {
+ calls++
+ return status.Error(codes.Aborted, "busy")
+ }, codes.Aborted)
+ if status.Code(err) != codes.Aborted || calls != defaultRetryAttempts {
+ t.Errorf("got err=%v calls=%d, want Aborted after %d calls", err, calls, defaultRetryAttempts)
+ }
+ })
+
+ t.Run("context cancelled", func(t *testing.T) {
+ ctx, cancel := context.WithCancel(context.Background())
+ cancel()
+ err := retryOnCodes(ctx, func() error {
+ return status.Error(codes.Aborted, "busy")
+ }, codes.Aborted)
+ if !errors.Is(err, context.Canceled) {
+ t.Errorf("expected context.Canceled, got %v", err)
+ }
+ })
+}
diff --git a/internal/tunnel/tunnel.go b/internal/tunnel/tunnel.go
index 6ab44eb..b0633e5 100644
--- a/internal/tunnel/tunnel.go
+++ b/internal/tunnel/tunnel.go
@@ -27,6 +27,8 @@ import (
"strings"
"syscall"
"time"
+
+ "github.com/google/ax/pkg/apis/v1alpha1"
)
var portForwardRegex = regexp.MustCompile(`Forwarding from .*:(\d+)\s+->`)
@@ -196,7 +198,7 @@ func EnsureServerURL(opts Options) (string, error) {
if envNs := os.Getenv("AX_SYSTEM_NAMESPACE"); envNs != "" {
opts.Namespace = envNs
} else {
- opts.Namespace = "ax-system"
+ opts.Namespace = v1alpha1.DefaultSystemNamespace
}
}
if opts.Service == "" {
@@ -208,7 +210,11 @@ func EnsureServerURL(opts Options) (string, error) {
// Check for existing active tunnel
if existing, err := GetTunnel(ctxName); err == nil && existing != nil {
- if IsTunnelHealthy(existing.Port) {
+ // Empty Namespace/Service means the tunnel was recorded before these fields
+ // existed; treat it as matching for backward compatibility.
+ sameTarget := (existing.Namespace == "" || existing.Namespace == opts.Namespace) &&
+ (existing.Service == "" || existing.Service == opts.Service)
+ if sameTarget && IsTunnelHealthy(existing.Port) {
return fmt.Sprintf("http://127.0.0.1:%d", existing.Port), nil
}
// Existing tunnel is unhealthy or stale, clean it up
diff --git a/pkg/apis/v1alpha1/types.go b/pkg/apis/v1alpha1/types.go
index 362722b..befcfaf 100644
--- a/pkg/apis/v1alpha1/types.go
+++ b/pkg/apis/v1alpha1/types.go
@@ -35,7 +35,8 @@ const (
KindWorkspace = "Workspace"
KindModel = "Model"
- DefaultTaskImage = "gcr.io/ax-substrate/ate-images/ax-task-runner"
+ DefaultTaskImage = "gcr.io/ax-substrate/ate-images/ax-task-runner"
+ DefaultSystemNamespace = "ax-system"
// PhaseTerminating marks a task whose deletion has been requested and whose
// actor is being torn down. The record disappears once cleanup completes.